"""
Is the 20-25% price bucket -- where xlow actually bets -- mis-calibrated?
(read-only, trades nothing, downloads public CSVs)

WHY: sa_history_calibration.py refuted the South America anomaly on 11.7k games,
but turned up something more relevant on the way. Bucketing outcomes by implied
probability, the 0.20-0.25 bucket came in ~7% below implied (z-3.72) while its
neighbours were clean. xlow's average bet sits at 23.7% implied -- dead centre of
that bucket. If the effect is real it eats the bottom half of the [5%,12%) CLV
band and would explain a forward test sitting at zero.

That first pass only covered Brazil/Argentina/Mexico/USA. This one covers OUR
league universe -- the divisions xlow actually trades, dominated by the European
leagues that were 78% of the backtest.

TWO DEVIGS, because the first pass showed it matters: proportional (what our
pipeline uses) systematically overstates longshots, so part of any apparent
longshot bias is our own arithmetic rather than the market's. Power devig is the
control. If the dip survives power devig, it is not a devig artifact.

Usage:  python research\\price_bucket_calibration.py [n_seasons]
"""
import sys
import urllib.request
import io
import csv
import math
from collections import defaultdict

# Our traded universe, mapped to football-data.co.uk sources.
EURO_DIVS = ['E0', 'E1', 'E2', 'E3', 'EC', 'SC0', 'SC1', 'SC2', 'SC3',
             'D1', 'D2', 'I1', 'I2', 'SP1', 'SP2', 'F1', 'F2', 'N1', 'B1',
             'P1', 'T1', 'G1']
NEW_FEEDS = {'BRA': 'BSA/BR2', 'ARG': 'AR1', 'MEX': 'MX1', 'USA': 'MLS',
             'SWE': 'SE1', 'NOR': 'NO1', 'DNK': 'DK1', 'POL': 'PL1',
             'ROU': 'RO1', 'CHN': 'CN1', 'JPN': 'JP1'}
N_SEASONS = int(sys.argv[1]) if len(sys.argv) > 1 else 10
ODDS = ('PSCH', 'PSCD', 'PSCA')      # Pinnacle CLOSING
FALLBACK = ('AvgCH', 'AvgCD', 'AvgCA')
BUCKETS = [(0.05, .10), (.10, .15), (.15, .20), (.20, .25), (.25, .30),
           (.30, .40), (.40, .55), (.55, .95)]


def _f(x):
    try:
        v = float(x)
        return v if v > 1.0 else None
    except (TypeError, ValueError):
        return None


def seasons(n):
    out = []
    for start in range(26 - n, 26):
        a, b = start % 100, (start + 1) % 100
        out.append(f'{a:02d}{b:02d}')
    return out


def grab(url):
    try:
        return urllib.request.urlopen(url, timeout=45).read().decode('utf-8-sig', 'replace')
    except Exception:
        return None


def parse(txt, res_key, cols_sets, tag):
    out = []
    if not txt:
        return out
    for r in csv.DictReader(io.StringIO(txt)):
        res = (r.get(res_key) or '').strip().upper()
        if res not in ('H', 'D', 'A'):
            continue
        for cols in cols_sets:
            v = [_f(r.get(c)) for c in cols]
            if all(v):
                out.append((v, res, tag))
                break
    return out


def prop(o):
    imp = [1.0 / x for x in o]
    s = sum(imp)
    return [i / s for i in imp]


def power(o):
    """Solve sum(p_i^k)=1. Flattens longshots; standard fix for proportional's bias."""
    imp = [1.0 / x for x in o]
    lo, hi = 0.5, 1.5
    for _ in range(60):
        k = (lo + hi) / 2
        if sum(i ** k for i in imp) > 1:
            lo = k
        else:
            hi = k
    k = (lo + hi) / 2
    return [i ** k for i in imp]


def table(games, devig, label, only=None):
    obs = []
    for o, res, tag in games:
        if only and tag not in only:
            continue
        p = devig(o)
        for i, letter in enumerate('HDA'):
            obs.append((p[i], res == letter))
    print(f'\n{label}   ({len(obs)} outcome observations)')
    print(f'  {"bucket":12s} {"n":>7s} {"implied":>8s} {"actual":>8s} {"rel":>8s} {"z":>7s}')
    for lo, hi in BUCKETS:
        s = [x for x in obs if lo <= x[0] < hi]
        n = len(s)
        if n < 200:
            continue
        exp = sum(p for p, _ in s)
        act = sum(1 for _, w in s if w)
        var = sum(p * (1 - p) for p, _ in s)
        rel = (act / n) / (exp / n) - 1
        flag = '  <-- xlow bets here' if lo == 0.20 else ''
        print(f'  {lo:.2f}-{hi:.2f}   {n:7d} {exp/n*100:7.2f}% {act/n*100:7.2f}% '
              f'{rel*100:+7.1f}% {(act-exp)/math.sqrt(var):+7.2f}{flag}')


def main():
    games = []
    ss = seasons(N_SEASONS)
    print(f'fetching {len(EURO_DIVS)} European divisions x {len(ss)} seasons '
          f'+ {len(NEW_FEEDS)} country feeds ...')
    for div in EURO_DIVS:
        got = 0
        for s in ss:
            g = parse(grab(f'https://www.football-data.co.uk/mmz4281/{s}/{div}.csv'),
                      'FTR', (ODDS, FALLBACK), 'euro')
            games += g
            got += len(g)
        print(f'  {div:4s} {got:6d}')
    for code in NEW_FEEDS:
        g = parse(grab(f'https://www.football-data.co.uk/new/{code}.csv'),
                  'Res', (ODDS, FALLBACK), 'new')
        games += g
        print(f'  {code:4s} {len(g):6d}   ({NEW_FEEDS[code]})')

    print(f'\nTOTAL {len(games)} games with Pinnacle closing odds')
    table(games, prop, 'PROPORTIONAL DEVIG  (what our pipeline uses)')
    table(games, power, 'POWER DEVIG  (control for proportional overstating longshots)')
    table(games, prop, 'EUROPEAN ONLY, proportional  (78% of the backtest)', only={'euro'})
    print('\nIf the 0.20-0.25 dip survives POWER devig it is a real market effect;')
    print('if it vanishes, our proportional devig was inflating CLV on longshots')
    print('-- which would be OUR bug, and equally worth knowing.')


if __name__ == '__main__':
    main()