"""
Does the South American home/draw anomaly survive on THOUSANDS of games?
(read-only, trades nothing, downloads public CSVs)

WHY THIS EXISTS: research/region_calibration_test.py found, on 73 fixtures of our
own data, that Pinnacle appears to under-price SA homes (48.2% -> 60.3% actual)
and over-price draws (27.4% -> 16.4%). Suggestive, sign-consistent across 5/5 SA
leagues -- but 73 games is nothing, and the observed draw rate contradicted the
well-known ~26-28% historical base rate for these leagues.

This settles it with football-data.co.uk's /new/ archive, which carries PINNACLE
CLOSING odds (PSCH/PSCD/PSCA) -- the same anchor our live pipeline uses -- for
thousands of Brazilian and Argentine fixtures.

CONTROLS:
  - Mexico + USA as non-SA western-hemisphere comparisons (same data source,
    same devig, same code path) so any finding is not an artifact of the pipeline.
  - Per-league and per-season breakdowns: a real structural effect is stable
    across seasons; a fluke lives in one or two.
  - Reports our own 73-fixture sample alongside, to show directly how far it sat
    from the long-run rate.

Usage:  python research\\sa_history_calibration.py
"""
import urllib.request
import io
import csv
import math
from collections import defaultdict

FEEDS = {
    'BRA': 'S.America', 'ARG': 'S.America',
    'MEX': 'control',   'USA': 'control',
}
URL = 'https://www.football-data.co.uk/new/{}.csv'
# Pinnacle CLOSING odds columns; fall back to market average if Pinnacle is blank.
ODDS_SETS = (('PSCH', 'PSCD', 'PSCA'), ('AvgCH', 'AvgCD', 'AvgCA'))


def _f(x):
    try:
        v = float(x)
        return v if v > 1.0 else None
    except (TypeError, ValueError):
        return None


def fair(o):
    imp = [1.0 / v for v in o]
    s = sum(imp)
    return [v / s for v in imp], s


def z_for(n, p, actual):
    sd = math.sqrt(n * p * (1 - p))
    return (actual - p * n) / sd if sd > 0 else float('nan')


def fetch(code):
    raw = urllib.request.urlopen(URL.format(code), timeout=60).read()
    txt = raw.decode('utf-8-sig', 'replace')
    out = []
    for r in csv.DictReader(io.StringIO(txt)):
        res = (r.get('Res') or '').strip().upper()
        if res not in ('H', 'D', 'A'):
            continue
        odds = None
        for cols in ODDS_SETS:
            vals = [_f(r.get(c)) for c in cols]
            if all(vals):
                odds = vals
                break
        if not odds:
            continue
        pf, over = fair(odds)
        out.append(dict(lg=(r.get('League') or code).strip(),
                        season=(r.get('Season') or '').strip(),
                        res=res, pfh=pf[0], pfd=pf[1], pfa=pf[2], over=over))
    return out


def line(s, label, indent='  '):
    n = len(s)
    if n < 20:
        print(f'{indent}{label:34s} n={n:5d}  -- too few')
        return
    parts = [f'{indent}{label:34s} n={n:5d}']
    for key, letter in (('pfh', 'H'), ('pfd', 'D'), ('pfa', 'A')):
        imp = sum(x[key] for x in s) / n
        cnt = sum(x['res'] == letter for x in s)
        parts.append(f'{letter} {imp*100:5.1f}->{cnt/n*100:5.1f} '
                     f'(z{z_for(n, imp, cnt):+6.2f})')
    print('   '.join(parts))


def main():
    data = {}
    for code in FEEDS:
        try:
            data[code] = fetch(code)
            print(f'  fetched {code}: {len(data[code])} usable fixtures')
        except Exception as e:
            print(f'  fetch {code} FAILED: {e}')
    print()

    sa = [x for c, v in data.items() if FEEDS[c] == 'S.America' for x in v]
    ctl = [x for c, v in data.items() if FEEDS[c] == 'control' for x in v]

    print('HEADLINE  (implied -> actual, Pinnacle closing, proportional devig)')
    line(sa, 'SOUTH AMERICA (Bra+Arg)')
    line(ctl, 'CONTROL (Mex+USA)')
    print()

    print('BY COUNTRY / LEAGUE')
    for code in ('BRA', 'ARG', 'MEX', 'USA'):
        if code not in data:
            continue
        by = defaultdict(list)
        for x in data[code]:
            by[x['lg']].append(x)
        for lg, s in sorted(by.items(), key=lambda kv: -len(kv[1]))[:3]:
            line(s, f'{code} {lg}')
    print()

    print('SOUTH AMERICA BY SEASON  (a real effect is stable; a fluke is not)')
    by = defaultdict(list)
    for x in sa:
        by[x['season']].append(x)
    for season, s in sorted(by.items())[-12:]:
        line(s, f'season {season}')
    print()

    n = len(sa)
    iH = sum(x['pfh'] for x in sa) / n
    cH = sum(x['res'] == 'H' for x in sa)
    iD = sum(x['pfd'] for x in sa) / n
    cD = sum(x['res'] == 'D' for x in sa)
    print('VERDICT vs our 73-fixture live sample')
    print(f'  our sample said:  homes 48.2 -> 60.3   draws 27.4 -> 16.4')
    print(f'  {n} games say:  homes {iH*100:.1f} -> {cH/n*100:.1f}   '
          f'draws {iD*100:.1f} -> {cD/n*100:.1f}')
    print(f'  long-run home edge: {(cH/n - iH)*100:+.2f}pp   '
          f'draw edge: {(cD/n - iD)*100:+.2f}pp')
    print()
    print('  TRADE ECONOMICS implied by the long run:')
    print(f'    buy every home at Pinnacle fair : ROI {(cH/n/iH - 1)*100:+.1f}%')
    nd_imp = 1 - iD
    nd_act = 1 - cD / n
    print(f'    buy home+away (fade the draw)   : ROI {(nd_act/nd_imp - 1)*100:+.1f}%'
          f'   (before any vig -- real books charge ~4-7%)')
    print(f'    avg Pinnacle overround in sample: '
          f'{(sum(x["over"] for x in sa)/n - 1)*100:.1f}%')


if __name__ == '__main__':
    main()