"""Does the market move TOWARD our side after xlow fires? The testable question.

WHY THIS EXISTS. Realised P&L cannot validate xlow. Measured 2026-08-08 at
n=234: ROI +1.2%/bet against a per-bet stdev of 1.94u, so the 95% CI runs
[-23.7%, +26.1%] and proving the observed effect needs ~100,000 bets -- about 46
years at 6 bets/day. The forward test is a smoke alarm (it would catch a -25%
disaster), not a scale.

Line DRIFT is the same question with ~1/80th the variance. If the rule finds
genuine mispricing, the market should converge toward our fair value after we
identify it. A first pass over 61 reconstructed bets gave -0.77c with z=-2.46
(95% CI [-1.38c, -0.16c]) -- the market moving AWAY from our side. That is
suggestive but was found by slicing data many ways in one session, so this
script exists to run it as a STANDING, pre-registered metric on bets whose entry
mid was recorded at fire time rather than reconstructed afterwards.

TWO TRAPS THIS AVOIDS.

  1. MID-to-MID only. Comparing our entry ASK to a closing MID is biased by half
     the spread -- median 0.50c here, almost exactly the size of the effect. That
     comparison manufactures a negative result out of nothing.
  2. Exact entry timing. `entry_k_mid`/`entry_p_mid`/`entry_ts` are written by
     live_monitor._log_pending_bets at the instant the signal fires. Bets logged
     before 2026-08-08 lack them and are RECONSTRUCTED from the first pre-kickoff
     snapshot; those are reported separately and never pooled, because the
     reconstruction is a proxy for entry time and the whole point is to stop
     relying on it.

READING THE RESULT. Positive drift = the market moved toward our side = we got a
good price = evidence of edge. Negative = we are buying prices that then revert
against us.

Usage:  python research\\clv_realisation.py
"""
import csv
import glob
import json
import math
import statistics
from collections import defaultdict
from pathlib import Path

HIST = Path(r'E:\soccer\data\bet_history.json')
PEND = Path(r'E:\soccer\data\pending_bets.json')
LIVE = r'E:\soccer\live_data\*.csv'
TARGET_EFFECT = 0.005      # 0.5c -- the size worth detecting


def _f(x):
    try:
        return float(x)
    except (TypeError, ValueError):
        return None


def load_bets():
    out = []
    for p in (HIST, PEND):
        try:
            raw = json.loads(p.read_text(encoding='utf-8'))
        except Exception:
            continue
        rows = raw if isinstance(raw, list) else raw.get('bets', raw)
        for b in rows:
            if b.get('strategy') in ('xlow', 'xlow_polyus'):
                out.append(b)
    return out


def closing_mids():
    """(game_date, home, away) -> last PRE-kickoff snapshot row."""
    last = {}
    for f in sorted(glob.glob(LIVE)):
        with open(f, encoding='utf-8') as fh:
            for r in csv.DictReader(fh):
                if (r.get('status') or '') != 'PRE':
                    continue
                k = (r.get('game_date'), r.get('home'), r.get('away'))
                if k not in last or r['timestamp'] > last[k]['timestamp']:
                    last[k] = r
    return last


def report(name, drifts):
    n = len(drifts)
    print(f'\n{name}  (n={n})')
    if n < 10:
        print('   too few to report')
        return
    m = statistics.mean(drifts)
    sd = statistics.pstdev(drifts) or 1e-9
    se = sd / math.sqrt(n)
    z = m / se
    lo, hi = m - 1.96 * se, m + 1.96 * se
    print(f'   mean drift {100*m:+.2f}c   stdev {100*sd:.2f}c   SE {100*se:.2f}c   z {z:+.2f}')
    print(f'   95% CI [{100*lo:+.2f}c, {100*hi:+.2f}c]')
    need = (1.96 * sd / TARGET_EFFECT) ** 2
    if lo > 0:
        print('   SIGNIFICANT and POSITIVE -- market converges toward our side.')
    elif hi < 0:
        print('   SIGNIFICANT and NEGATIVE -- we are buying prices that revert '
              'against us. This is evidence AGAINST the rule.')
    else:
        print(f'   not significant; n={need:,.0f} needed for a {100*TARGET_EFFECT:.1f}c effect '
              f'({max(0, need-n):,.0f} more)')


def main():
    last = closing_mids()
    exact, recon = [], []
    by_lg = defaultdict(list)
    for b in load_bets():
        row = last.get((str(b.get('date'))[:10], b.get('home'), b.get('away')))
        if not row:
            continue
        side = 'home' if 'HOME' in str(b.get('bet', '')).upper() else 'away'
        on_poly = bool(b.get('poly_stake') and b.get('poly_ask'))
        pre = 'poly' if on_poly else 'kalshi'
        close = _f(row.get(f'{pre}_{side}_mid'))
        if not close:
            continue
        entry = _f(b.get('entry_p_mid') if on_poly else b.get('entry_k_mid'))
        if entry:
            exact.append(close - entry)
            by_lg[b.get('div')].append(close - entry)
        else:
            # reconstructed: first PRE snapshot on the same book/side
            first = None
            for f in sorted(glob.glob(LIVE)):
                with open(f, encoding='utf-8') as fh:
                    for r in csv.DictReader(fh):
                        if (r.get('status') or '') != 'PRE':
                            continue
                        if (r.get('game_date'), r.get('home'), r.get('away')) != \
                                (str(b.get('date'))[:10], b.get('home'), b.get('away')):
                            continue
                        v = _f(r.get(f'{pre}_{side}_mid'))
                        if v and (first is None or r['timestamp'] < first[0]):
                            first = (r['timestamp'], v)
            if first:
                recon.append(close - first[1])

    print('CLV REALISATION -- does the market move toward our side after xlow fires?')
    print('positive = toward us (edge)   negative = reverts against us')
    report('EXACT entry mid (logged at fire time, 2026-08-08 onward)', exact)
    report('RECONSTRUCTED entry (pre-2026-08-08, proxy -- do NOT pool)', recon)
    if len(exact) >= 10:
        print('\n   by league (exact only):')
        for lg, ds in sorted(by_lg.items(), key=lambda x: -len(x[1]))[:10]:
            if len(ds) >= 3:
                print(f'     {lg:6s} n={len(ds):4d}  {100*statistics.mean(ds):+.2f}c')


if __name__ == '__main__':
    main()