"""Can a regression pick which books to quote, under a QUEUE-AWARE fill model?

WHY THIS SUPERSEDES fill_model.py. That one labelled a leg "filled" if any print
landed at or above our ask -- the optimistic test queue_aware_fill.py was
written to discredit. At the touch we join an ask with thousands of contracts
already resting on it, so a print at our price is usually somebody else's fill.
Selecting books on a label that is inflated exactly where we intend to quote is
worse than not selecting at all.

THE LABEL HERE. Posting at price P with Q contracts resting at P, we fill once
the cumulative size of trades at price >= P after we post exceeds Q. Trades
ABOVE our price count: the book walked up through our level. Q is the depth at
the touch at post time. This is queue_aware_fill.py's model, applied to every
leg rather than one day.

  Optimistic label -> upper bound.  This label -> lower bound (nobody ahead of
  us ever cancels).  The truth is between, and that gap is the honest error bar.

THE SIMULATION. Quote all three legs at the touch, no premium, no repairs:
    all three fill  -> collect the overround, outcome-independent
    partial         -> the filled legs settle naked
Then fit P(all three fill) on post-time features, split TEMPORALLY, and ask
whether selecting on it beats quoting everything. A gate can only remove books,
so it has to earn its keep by removing losers.

Sized trade tape is cached to .tape_sized_cache.json (one number per leg), so
the slow part happens once.

Usage:  python research\\book_selection.py
"""
import csv
import json
import shutil
import sys
import tempfile
from collections import defaultdict
from pathlib import Path

import numpy as np

sys.path.insert(0, r'E:\soccer\main')
sys.path.insert(0, r'E:\soccer\research')
import mm_settle as MS
import queue_aware_fill as QA

QUOTES = Path(r'E:\soccer\research\mm_paper_quotes.csv')
RESULTS = Path(r'E:\soccer\research\mm_paper_results.csv')
CACHE = Path(r'E:\soccer\research\.tape_sized_cache.json')

FEATS = ['touch', 'queue_q', 'spread_c', 'overround_c', 'min_leg_vol_k',
         'vol24_k', 'open_interest_k', 'ask_size_k', 'bid_size_k',
         'hrs_to_ko', 'is_draw', 'fair_gap_c']


def _f(x):
    try:
        return float(x)
    except (TypeError, ValueError):
        return None


def build():
    settled = {}
    with RESULTS.open(newline='', encoding='utf-8', errors='replace') as fh:
        for r in csv.DictReader(fh):
            if r.get('leg_ticker'):
                settled[r['leg_ticker']] = _f(r.get('settled'))

    tmp = Path(tempfile.gettempdir()) / 'mm_paper_quotes.snap.csv'
    shutil.copyfile(QUOTES, tmp)
    first = {}
    with tmp.open(newline='', encoding='utf-8', errors='replace') as fh:
        for r in csv.DictReader(fh):
            t = r.get('leg_ticker')
            if t and t not in first and not (r.get('reason') or ''):
                first[t] = r

    cache = json.loads(CACHE.read_text()) if CACHE.exists() else {}
    bookvol = defaultdict(list)
    for t, r in first.items():
        v = _f(r.get('volume'))
        if v is not None:
            bookvol[r.get('event')].append(v)

    legs, new = {}, 0
    for i, (t, r) in enumerate(sorted(first.items()), 1):
        ts, ko = MS._pt(r.get('ts')), MS._pt(r.get('kickoff'))
        touch, kb = _f(r.get('k_ask')), _f(r.get('k_bid'))
        over, q = _f(r.get('over_ask')), _f(r.get('ask_size'))
        s = settled.get(t)
        vols = bookvol.get(r.get('event')) or []
        if None in (ts, ko, touch, kb, over, q, s) or not vols:
            continue
        if not (0 < kb < touch < 1):
            continue
        hrs = (ko - ts).total_seconds() / 3600.0
        if not (0 < hrs <= 24):
            continue
        if t in cache:
            vol_at = cache[t]
        else:
            try:
                tr = QA.trades_sized(t, ts)
            except Exception:
                tr = []
            # size traded AT OR ABOVE our price, before kickoff
            vol_at = sum(c for (tt, p, c) in tr if p >= touch - 1e-9 and tt <= ko)
            cache[t] = vol_at
            new += 1
            if new % 25 == 0:
                CACHE.write_text(json.dumps(cache))
                print(f'  ... {new} tape reads ({i}/{len(first)})', flush=True)
        fair = _f(r.get('pin_fair'))
        legs[t] = {
            'event': r.get('event'), 'ko': ko, 'touch': touch, 'settled': s,
            'q': q, 'vol_at': vol_at, 'over_c': (over - 1.0) * 100.0,
            'fill': 1 if vol_at >= q else 0,
            'x': [touch, q / 1000.0, (touch - kb) * 100.0, (over - 1.0) * 100.0,
                  min(vols) / 1000.0, (_f(r.get('volume_24h')) or 0) / 1000.0,
                  (_f(r.get('open_interest')) or 0) / 1000.0, q / 1000.0,
                  (_f(r.get('bid_size')) or 0) / 1000.0, hrs,
                  1.0 if str(r.get('side')).lower() == 'draw' else 0.0,
                  ((fair - touch) * 100.0) if fair is not None else 0.0]}
    CACHE.write_text(json.dumps(cache))
    print(f'{len(legs)} legs scored ({new} new tape reads)\n')
    return legs


def books_from(legs):
    b = defaultdict(list)
    for t, v in legs.items():
        b[v['event']].append(v)
    out = []
    for ev, v3 in b.items():
        if len(v3) != 3:
            continue
        comp = all(x['fill'] for x in v3)
        # quote all three at the touch: complete -> the overround, outcome
        # independent; partial -> the filled legs settle naked
        pnl = (v3[0]['over_c'] if comp else
               sum((x['touch'] - x['settled']) * 100.0 for x in v3 if x['fill']))
        out.append({'ev': ev, 'ko': min(x['ko'] for x in v3), 'comp': int(comp),
                    'pnl_c': pnl, 'legs': v3})
    out.sort(key=lambda z: z['ko'])
    return out


def main():
    legs = build()
    if not legs:
        print('nothing scored')
        return
    fr = sum(v['fill'] for v in legs.values()) / len(legs)
    print(f'queue-aware LEG fill at the touch: {100*fr:.0f}%')
    bk = books_from(legs)
    if len(bk) < 60:
        print(f'only {len(bk)} three-leg books -- too few')
        return
    cr = sum(b['comp'] for b in bk) / len(bk)
    tot = sum(b['pnl_c'] for b in bk) / 100.0
    print(f'three-leg books: {len(bk)}   completion {100*cr:.0f}%')
    print(f'quote-everything P&L at the touch, size 1: {tot:+.2f} dollars '
          f'({tot/len(bk)*100:+.2f}c per book)\n')

    from sklearn.linear_model import LogisticRegression
    from sklearn.pipeline import make_pipeline
    from sklearn.preprocessing import StandardScaler
    cut = int(len(bk) * 0.65)
    train, test = bk[:cut], bk[cut:]
    Xtr = np.array([l['x'] for b in train for l in b['legs']], float)
    ytr = np.array([l['fill'] for b in train for l in b['legs']], int)
    if ytr.sum() < 15 or (1 - ytr).sum() < 15:
        print('train set too one-sided to fit')
        return
    pipe = make_pipeline(StandardScaler(), LogisticRegression(max_iter=3000))
    pipe.fit(Xtr, ytr)

    print(f'TEMPORAL SPLIT  train {len(train)} books '
          f'({train[0]["ko"]:%m-%d}..{train[-1]["ko"]:%m-%d})  '
          f'test {len(test)} ({test[0]["ko"]:%m-%d}..{test[-1]["ko"]:%m-%d})\n')
    scored = []
    for b in test:
        p = float(np.prod(pipe.predict_proba(
            np.array([l['x'] for l in b['legs']], float))[:, 1]))
        scored.append((p, b))
    print(f'  {"rule":24} {"books":>6} {"complete":>9} {"P&L $":>9} {"per book":>10}')

    def row(lbl, sel):
        if not sel:
            print(f'  {lbl:24} {0:>6}')
            return
        p = sum(b['pnl_c'] for _, b in sel) / 100.0
        c = 100.0 * sum(b['comp'] for _, b in sel) / len(sel)
        print(f'  {lbl:24} {len(sel):>6} {c:>8.0f}% {p:>+9.2f} {p/len(sel):>+10.3f}')

    row('quote everything', scored)
    for thr in (0.05, 0.10, 0.20, 0.35, 0.50):
        row(f'P(all 3) >= {thr:.2f}', [s for s in scored if s[0] >= thr])
    print('\n  Size 1. A gate can only REMOVE books, so it must beat')
    print('  "quote everything" on the same test window to be worth anything.')


if __name__ == '__main__':
    main()