"""Which asking price sells best -- as a regression, conditional on the book.

touch_premium_curve.py answers this MARGINALLY: one premium for everything. But
the best price for a busy Championship book is not the best price for a thin
Peruvian one, and a curve averaged over both hides that.

THE TRICK THAT MAKES THIS CHEAP. The trade tape gives, per leg, the highest
price anyone paid before kickoff. Define

    headroom = (max print - touch at post) x 100   [cents]

A resting ask at premium d would have been lifted iff headroom >= d. So one
number per leg encodes the fill/no-fill answer at EVERY premium -- no need to
re-run anything per price, and no dependence on the arm the quoter happened to
pick. headroom IS the market's willingness to pay above the touch.

WHAT IS FITTED. A logistic per premium level, P(headroom >= d | features), with
standard errors CLUSTERED BY BOOK: the three legs of a match share its flow and
its overround, so treating them as independent would overstate significance
roughly threefold -- the same pseudo-replication that inflated the pinnapi
t-stats earlier in this project.

WHAT IT IS FOR. Per book, pick the d that maximises

    E[book] = P(all three fill at d) x (overround + 3d)

and backtest that against every fixed d on a TEMPORAL split. If per-book pricing
does not beat the best single number, use the single number.

CEILING CAVEAT: a print at or above our price means we could have been hit, not
that we would have. Queue priority is assumed throughout, as in mm_settle.

Usage:  python research\\premium_regression.py   (needs .tape_cache.json)
"""
import csv
import json
import shutil
import sys
import tempfile
from collections import defaultdict
from pathlib import Path

import numpy as np

sys.path.insert(0, r'E:\soccer\main')
import mm_settle as MS

QUOTES = Path(r'E:\soccer\research\mm_paper_quotes.csv')
CACHE = Path(r'E:\soccer\research\.tape_cache.json')
PREMIUMS = [0, 1, 2, 3, 4, 5]
FEATS = ['min_leg_vol_k', 'hrs_to_ko', 'spread_c', 'overround_c',
         'touch_px', 'is_draw']


def _f(x):
    try:
        return float(x)
    except (TypeError, ValueError):
        return None


def load():
    if not CACHE.exists():
        raise SystemExit('no .tape_cache.json -- run touch_premium_curve.py first')
    cache = json.loads(CACHE.read_text())
    tmp = Path(tempfile.gettempdir()) / 'mm_paper_quotes.snap.csv'
    shutil.copyfile(QUOTES, tmp)
    first = {}
    with tmp.open(newline='', encoding='utf-8', errors='replace') as fh:
        for r in csv.DictReader(fh):
            t = r.get('leg_ticker')
            if t and t not in first and not (r.get('reason') or ''):
                first[t] = r

    vols = defaultdict(list)
    for t, r in first.items():
        v = _f(r.get('volume'))
        if v is not None and t in cache:
            vols[r.get('event')].append(v)

    legs = []
    for t, r in first.items():
        if t not in cache:
            continue
        ts, ko = MS._pt(r.get('ts')), MS._pt(r.get('kickoff'))
        ka, kb = _f(r.get('k_ask')), _f(r.get('k_bid'))
        over = _f(r.get('over_ask'))
        vv = vols.get(r.get('event')) or []
        if None in (ts, ko, ka, kb, over) or not vv or not (0 < kb < ka < 1):
            continue
        hrs = (ko - ts).total_seconds() / 3600.0
        if not (0 < hrs <= 24):
            continue
        legs.append({
            'tkr': t, 'event': r.get('event'), 'ko': ko,
            'headroom': (float(cache[t]) - ka) * 100.0,
            'over_c': (over - 1.0) * 100.0,
            'x': [min(vv) / 1000.0, hrs, (ka - kb) * 100.0, (over - 1.0) * 100.0,
                  ka, 1.0 if str(r.get('side')).lower() == 'draw' else 0.0]})
    return legs


def fit_report(legs):
    import statsmodels.api as sm
    X = sm.add_constant(np.array([l['x'] for l in legs], float))
    groups = np.array([l['event'] for l in legs])
    print(f'{len(legs)} legs, {len(set(groups))} books\n')
    print('headroom (cents above touch that the market actually paid):')
    h = np.array([l['headroom'] for l in legs])
    for q in (10, 25, 50, 75, 90):
        print(f'   p{q:<3} {np.percentile(h, q):+6.1f}c')
    print(f'   share with headroom >= 0: {100.0*(h >= 0).mean():.0f}%\n')

    print(f'P(fill at premium d), logistic, SEs clustered by book')
    print(f'{"d":>3} {"fills":>7} ' + ' '.join(f'{n:>14}' for n in FEATS))
    for d in PREMIUMS:
        y = (h >= d).astype(float)
        if y.sum() < 20 or (1 - y).sum() < 20:
            print(f'{d:>2}c  {int(y.sum()):>6}   (too few either way to fit)')
            continue
        try:
            res = sm.Logit(y, X).fit(disp=0, cov_type='cluster',
                                     cov_kwds={'groups': groups})
        except Exception as e:
            print(f'{d:>2}c  fit failed: {type(e).__name__}')
            continue
        cells = []
        for i, _n in enumerate(FEATS, start=1):
            star = '*' if res.pvalues[i] < 0.05 else ' '
            cells.append(f'{res.params[i]:+8.3f}{star}     ')
        print(f'{d:>2}c  {int(y.sum()):>6}   ' + ' '.join(c[:14] for c in cells))
    print('  * = p<0.05 with book-clustered SEs')


def backtest(legs, train_frac=0.65):
    from sklearn.linear_model import LogisticRegression
    from sklearn.pipeline import make_pipeline
    from sklearn.preprocessing import StandardScaler

    books = defaultdict(list)
    for l in legs:
        books[l['event']].append(l)
    full = sorted([(min(x['ko'] for x in v), e, v)
                   for e, v in books.items() if len(v) == 3], key=lambda t: t[0])
    if len(full) < 60:
        print(f'\nonly {len(full)} three-leg books -- too few to backtest')
        return
    cut = int(len(full) * train_frac)
    train, test = full[:cut], full[cut:]
    tr_legs = [l for _, _, v in train for l in v]
    Xtr = np.array([l['x'] for l in tr_legs], float)
    htr = np.array([l['headroom'] for l in tr_legs])

    models = {}
    for d in PREMIUMS:
        y = (htr >= d).astype(int)
        if y.sum() < 15 or (1 - y).sum() < 15:
            continue
        m = make_pipeline(StandardScaler(), LogisticRegression(max_iter=3000))
        m.fit(Xtr, y)
        models[d] = m

    print(f'\n{"="*66}\nTEMPORAL BACKTEST  train {len(train)} books / '
          f'test {len(test)} books')
    print(f'  train {train[0][0]:%m-%d} .. {train[-1][0]:%m-%d}   '
          f'test {test[0][0]:%m-%d} .. {test[-1][0]:%m-%d}\n')
    print(f'  {"policy":26} {"books quoted":>13} {"complete":>9} {"E[book]":>10}')

    def score(pick):
        tot, comp, quoted = 0.0, 0, 0
        for _, _, v in test:
            d = pick(v)
            if d is None:
                continue
            quoted += 1
            over = v[0]['over_c']
            if all(l['headroom'] >= d for l in v):
                comp += 1
                tot += over + 3 * d
        return quoted, comp, tot

    best_fixed, best_val = None, -1e9
    for d in PREMIUMS:
        q, c, t = score(lambda v, d=d: d)
        ev = t / q if q else 0
        print(f'  {"fixed +" + str(d) + "c":26} {q:>13} {c:>8} {ev:>9.2f}c')
        if ev > best_val:
            best_fixed, best_val = d, ev

    def model_pick(v):
        Xb = np.array([l['x'] for l in v], float)
        best, bev = None, -1e9
        for d, m in models.items():
            p = float(np.prod(m.predict_proba(Xb)[:, 1]))
            ev = p * (v[0]['over_c'] + 3 * d)
            if ev > bev:
                best, bev = d, ev
        return best
    q, c, t = score(model_pick)
    print(f'  {"model: per-book premium":26} {q:>13} {c:>8} '
          f'{t/q if q else 0:>9.2f}c')
    print(f'\n  best fixed premium on the test window: +{best_fixed}c '
          f'({best_val:.2f}c/book)')
    print('  E[book] is per book QUOTED. Fills assume queue priority (CEILING).')


if __name__ == '__main__':
    L = load()
    fit_report(L)
    backtest(L)