"""Does SoFIFA squad strength beat, or add to, the market?

TURNKEY: drop a CSV at data/sofifa_players.csv (or data/sofifa_teams.csv) and run
this. Nothing else to configure.

WHY YOU HAVE TO SUPPLY THE FILE. sofifa.com is behind a Cloudflare JS challenge;
the public scrapers get past it by driving a real browser, which is circumventing
an access control against their terms. Not doing that. Export it from your own
session, or use a published dataset (Kaggle EA FC player data), and put it here.

ACCEPTED SHAPES (auto-detected):
  players: needs a club name column and `overall_rating` (or `overall`).
           Team strength = mean of that club's top 11 by rating.
  teams:   needs a club name column and `overall` (optionally attack/midfield/
           defence). Used directly.

WHAT IT TESTS, in the order that matters:
  1. NAME MATCH RATE first. Squad ratings are worthless if the clubs do not join
     to our fixtures. Reported before any modelling, and the run aborts if it is
     too low -- a 30%-matched test silently measures the leagues that happen to
     spell names the same way.
  2. Rating differential ALONE vs the Pinnacle close (log loss).
  3. Market + rating vs market alone, time-split, paired t. Success criterion is
     fixed BEFORE the run: paired t > 1.96. Three previous feature families were
     declared dead by exactly this bar; SoFIFA gets the same one, no softer.
  4. THE NO-MODEL CONTROL. project_edge_mine_exhausted records a trap: a model
     showed +2.7% ROI while being no better than the market on log loss, because
     it was just selecting favourites in a table where proportional devig
     understates them. So any betting-style result here is reported next to the
     same slice bet with NO model.

KNOWN LIMITATION, stated not buried: a single ratings snapshot applied to past
fixtures is mildly look-ahead (ratings partly reflect the season being tested).
That biases IN FAVOUR of the ratings. If they lose even with that help, the
verdict is safe; if they win narrowly, it needs point-in-time ratings
(sofifa supports historical versions via its `r=YYMMDD` parameter) before
believing it.

Usage:  python research\\sofifa_test.py
"""
import csv
import math
import os
import sys
from collections import defaultdict

sys.path.insert(0, r'E:\soccer\research')
sys.path.insert(0, r'E:\soccer\main')
import elo_model_test as E
import nonprice_edge_test as T
import daily_signals as DS

PLAYERS = r'E:\soccer\data\sofifa_players.csv'
TEAMS = r'E:\soccer\data\sofifa_teams.csv'
MIN_MATCH_RATE = 0.35
TOP_N = 11


def norm(s):
    return ''.join(ch for ch in (s or '').lower() if ch.isalnum())


def _f(x):
    try:
        return float(x)
    except (TypeError, ValueError):
        return None


def pick(cols, *cands):
    low = {c.lower().strip(): c for c in cols}
    for c in cands:
        if c in low:
            return low[c]
    for c in cands:                       # substring fallback
        for k, v in low.items():
            if c in k:
                return v
    return None


def load_strength():
    """club_norm -> rating. From a teams file if present, else aggregate players."""
    if os.path.exists(TEAMS):
        with open(TEAMS, newline='', encoding='utf-8', errors='replace') as fh:
            rd = csv.DictReader(fh)
            cn = pick(rd.fieldnames, 'club_name', 'team', 'name')
            ov = pick(rd.fieldnames, 'overall', 'ovr', 'rating')
            if not cn or not ov:
                raise SystemExit(f'{TEAMS}: need a club-name and an overall column; '
                                 f'saw {rd.fieldnames}')
            out = {}
            for r in rd:
                v = _f(r.get(ov))
                if v:
                    out[norm(r.get(cn))] = v
            print(f'loaded {len(out)} team ratings from {TEAMS}')
            return out

    if not os.path.exists(PLAYERS):
        raise SystemExit(f'no data: put a CSV at {PLAYERS} or {TEAMS}')
    with open(PLAYERS, newline='', encoding='utf-8', errors='replace') as fh:
        rd = csv.DictReader(fh)
        cn = pick(rd.fieldnames, 'club_name', 'club', 'team')
        ov = pick(rd.fieldnames, 'overall_rating', 'overall', 'ovr', 'rating')
        if not cn or not ov:
            raise SystemExit(f'{PLAYERS}: need a club and an overall column; '
                             f'saw {rd.fieldnames}')
        squads = defaultdict(list)
        for r in rd:
            v = _f(r.get(ov))
            if v:
                squads[norm(r.get(cn))].append(v)
    out = {k: sum(sorted(v, reverse=True)[:TOP_N]) / min(len(v), TOP_N)
           for k, v in squads.items() if v}
    print(f'loaded {len(out)} clubs from {PLAYERS} '
          f'(top-{TOP_N} mean; median squad size {sorted(len(v) for v in squads.values())[len(squads)//2]})')
    return out


def logistic_fit(X, Y, C, iters=260, lr=0.28):
    """Multinomial logistic, L2 = 1/C. Plain GD -- small feature counts only."""
    n, d, K = len(X), len(X[0]), 3
    W = [[0.0] * d for _ in range(K)]
    b = [0.0] * K
    lam = 1.0 / max(C, 1e-9)
    for _ in range(iters):
        gW = [[0.0] * d for _ in range(K)]
        gb = [0.0] * K
        for xi, yi in zip(X, Y):
            z = [sum(W[k][j] * xi[j] for j in range(d)) + b[k] for k in range(K)]
            m = max(z)
            e = [math.exp(v - m) for v in z]
            s = sum(e)
            p = [v / s for v in e]
            for k in range(K):
                g = p[k] - (1.0 if yi == k else 0.0)
                gb[k] += g
                for j in range(d):
                    gW[k][j] += g * xi[j]
        for k in range(K):
            b[k] -= lr * gb[k] / n
            for j in range(d):
                W[k][j] -= lr * (gW[k][j] / n + lam * W[k][j])
    return W, b


def predict(W, b, xi):
    z = [sum(W[k][j] * xi[j] for j in range(len(xi))) + b[k] for k in range(3)]
    m = max(z)
    e = [math.exp(v - m) for v in z]
    s = sum(e)
    return [v / s for v in e]


def main():
    strength = load_strength()
    fx = E.load_fixtures()
    odds = T.load_odds()

    # 1. match rate, before anything else
    rows, seen, miss = [], set(), defaultdict(int)
    for f in fx:
        key = (f['date'], T.norm(DS.fdco_name(f['home'])), T.norm(DS.fdco_name(f['away'])))
        hit = odds.get(key)
        if not hit:
            continue
        p, res = hit
        i = T.IDX.get(res)
        if i is None:
            continue
        sh, sa = strength.get(norm(f['home'])), strength.get(norm(f['away']))
        seen.add(norm(f['home']))
        if sh is None:
            miss[f['home']] += 1
        if sa is None:
            miss[f['away']] += 1
        if sh is None or sa is None:
            continue
        rows.append({'date': f['date'], 'p': p, 'y': i, 'sh': sh, 'sa': sa,
                     'div': f['div']})

    total_odds = sum(1 for f in fx if odds.get(
        (f['date'], T.norm(DS.fdco_name(f['home'])), T.norm(DS.fdco_name(f['away'])))))
    rate = len(rows) / total_odds if total_odds else 0
    print(f'\n1. NAME MATCH: {len(rows)}/{total_odds} fixtures ({rate*100:.1f}%) '
          f'have BOTH clubs rated')
    if miss:
        print('   most-missed clubs:', ', '.join(
            f'{k} ({v})' for k, v in sorted(miss.items(), key=lambda kv: -kv[1])[:8]))
    if rate < MIN_MATCH_RATE:
        raise SystemExit(f'\nABORT: match rate below {MIN_MATCH_RATE:.0%}. Fix the club-name '
                         'join before testing -- a partial join silently tests only the '
                         'leagues that spell names our way.')

    rows.sort(key=lambda r: r['date'])
    cut = int(len(rows) * 0.7)
    tr, te = rows[:cut], rows[cut:]
    print(f'   time split: train {len(tr)} -> test {len(te)}')

    def ll(ps, ys):
        return sum(-math.log(max(p[y], 1e-12)) for p, y in zip(ps, ys)) / len(ys)

    mkt = ll([r['p'] for r in te], [r['y'] for r in te])
    print(f'\n2. MARKET BASELINE (Pinnacle close)   log loss {mkt:.5f}')

    def mklog(p):
        return [math.log(max(v, 1e-9)) for v in p]

    sd = (sum((r['sh'] - r['sa']) ** 2 for r in rows) / len(rows)) ** 0.5 or 1.0
    featsets = {
        'rating diff ALONE': lambda r: [(r['sh'] - r['sa']) / sd, 1.0],
        'market only (control)': lambda r: mklog(r['p']),
        'market + rating diff': lambda r: mklog(r['p']) + [(r['sh'] - r['sa']) / sd],
    }
    best = {}
    for nm, fn in featsets.items():
        Xtr = [fn(r) for r in tr]
        Ytr = [r['y'] for r in tr]
        Xte = [fn(r) for r in te]
        bl = None
        for C in (0.01, 0.1, 1.0):
            W, b = logistic_fit(Xtr, Ytr, C)
            v = ll([predict(W, b, x) for x in Xte], [r['y'] for r in te])
            if bl is None or v < bl[0]:
                bl = (v, C, W, b)
        best[nm] = bl
        print(f'   {nm:24s} log loss {bl[0]:.5f}  (best C={bl[1]})  '
              f'vs market {mkt - bl[0]:+.5f}')

    # 3. paired t on the pre-registered criterion
    v, C, W, b = best['market + rating diff']
    mv, mC, mW, mb = best['market only (control)']
    d = []
    for r in te:
        a = -math.log(max(predict(W, b, featsets['market + rating diff'](r))[r['y']], 1e-12))
        c = -math.log(max(predict(mW, mb, featsets['market only (control)'](r))[r['y']], 1e-12))
        d.append(c - a)                    # positive = rating helps
    mu = sum(d) / len(d)
    sdv = (sum((x - mu) ** 2 for x in d) / (len(d) - 1)) ** 0.5
    t = mu / (sdv / len(d) ** 0.5) if sdv else 0.0
    print(f'\n3. PAIRED TEST vs market-only control: {mu:+.5f} per fixture, '
          f't = {t:+.2f} on n={len(d)}')
    print('   PRE-REGISTERED BAR: t > 1.96.  '
          + ('*** PASSES ***' if t > 1.96 else 'does not pass -> same verdict as Elo/form.'))

    # 4. no-model control
    print('\n4. NO-MODEL CONTROL (the favourite-selection trap)')
    fav = [r for r in te if max(r['p']) > 0.40]
    if fav:
        hit = sum(1 for r in fav if r['p'].index(max(r['p'])) == r['y'])
        imp = sum(max(r['p']) for r in fav) / len(fav)
        print(f'   betting favourites with NO model: n={len(fav)} '
              f'implied {imp*100:.2f}% actual {hit/len(fav)*100:.2f}%')
        print('   Any "profit" from the rating model must beat THIS, not zero.')


if __name__ == '__main__':
    main()