"""
Systematic edge search against the ONLY objective that matters: net CLV in cents.
(read-only, trades nothing)

WHY THIS OBJECTIVE. project_xlow_spread_verdict established that xlow's signal is
real (+0.31c of line movement) but smaller than the 0.67c half-spread it costs to
enter. So "find a pattern that predicts outcomes" is the wrong search -- it is
both maximally noisy (1 bit per fixture, per-bet SD 1.99u) and it ignores the
constraint that actually binds. We mine instead for:

    net = close_mid - entry_ask        (cents)

i.e. "the market's final valuation, minus what I actually paid." Positive mean
net = a configuration where the edge survives execution. This is continuous,
measurable on every fixture-outcome without waiting for results, and has roughly
50x the power of outcome-based testing.

MULTIPLE-TESTING DISCIPLINE. Mining hundreds of slices on ~1,200 fixtures WILL
produce winners by chance -- that is exactly how the day's dead ends (South
America, the 20-25%% bucket) were born. So:
  - the null is built by PERMUTATION at the FIXTURE level, preserving the
    within-fixture correlation of the three outcomes (they sum to 1, so they are
    not independent and naive t-tests overstate significance);
  - we report the max-|t| across ALL slices under that null, which is the correct
    family-wise threshold, not per-slice p-values;
  - the number of hypotheses tested is printed, always.

A slice only counts if its |t| beats the permutation max-|t| threshold.

Usage:  python research\\edge_mine.py [n_permutations]
"""
import sys
import csv
import glob
import math
import random
import statistics
from collections import defaultdict

LIVE = r'E:\soccer\live_data\*.csv'
BOOKS = ('kalshi', 'poly', 'polyus')
SIDES = ('home', 'draw', 'away')
NPERM = int(sys.argv[1]) if len(sys.argv) > 1 else 400
random.seed(11)


def _f(x):
    try:
        v = float(x)
        return v if v == v else None
    except (TypeError, ValueError):
        return None


def load():
    """One record per (fixture, book, side): features at entry, close mid, result."""
    fx = defaultdict(list)
    for p in glob.glob(LIVE):
        try:
            with open(p, newline='', encoding='utf-8') as fh:
                for r in csv.DictReader(fh):
                    if (r.get('status') or '').upper() != 'PRE':
                        continue
                    fx[(r.get('game_date'), r.get('league'),
                        r.get('home'), r.get('away'))].append(r)
        except Exception:
            continue

    recs = []
    for key, rows in fx.items():
        if len(rows) < 10:
            continue
        ftr = None
        for r in rows:
            v = (r.get('ftr') or '').strip().upper()
            if v in ('H', 'D', 'A'):
                ftr = v
                break
        last = rows[-1]
        # entry candidates: how many snapshots before the close
        for lag_name, idx in (('early', max(0, len(rows) - 20)),
                              ('late', max(0, len(rows) - 3))):
            entry = rows[idx]
            for book in BOOKS:
                for si, side in enumerate(SIDES):
                    ea = _f(entry.get(f'{book}_{side}_ask'))
                    eb = _f(entry.get(f'{book}_{side}_bid'))
                    em = _f(entry.get(f'{book}_{side}_mid'))
                    cm = _f(last.get(f'{book}_{side}_mid'))
                    pf = _f(entry.get(f'pin_fair_{side}'))
                    if None in (ea, eb, em, cm, pf):
                        continue
                    if not (0 < eb < ea < 1) or not (0 < cm < 1) or pf <= 0:
                        continue
                    clv = (pf - em) / pf
                    # the other two exchanges' mids, for disagreement features
                    others = [_f(entry.get(f'{b}_{side}_mid')) for b in BOOKS if b != book]
                    others = [o for o in others if o and 0 < o < 1]
                    recs.append(dict(
                        fixture=key, lg=key[1], book=book, side=side, lag=lag_name,
                        price=em, spread=(ea - eb) * 100, halfsp=(ea - em) * 100,
                        clv=clv * 100, pf=pf,
                        disagree=(max(others) - em) * 100 if others else None,
                        net=(cm - ea) * 100,          # <-- the objective, cents
                        mv=(cm - em) * 100,           # raw line move, cents
                        won=(1 if (ftr == 'H' and side == 'home') or
                             (ftr == 'A' and side == 'away') or
                             (ftr == 'D' and side == 'draw') else 0) if ftr else None))
    return recs


def tstat(v):
    n = len(v)
    if n < 2:
        return 0.0, 0.0
    m = statistics.mean(v)
    sd = statistics.pstdev(v)
    se = sd / math.sqrt(n) if sd > 0 else 1e-9
    return m, m / se


def slices(recs):
    """Every candidate configuration to test. Returns {label: [rec,...]}."""
    out = {}
    for lag in ('early', 'late'):
        base = [r for r in recs if r['lag'] == lag]
        out[f'ALL/{lag}'] = base
        for b in BOOKS:
            out[f'book={b}/{lag}'] = [r for r in base if r['book'] == b]
        for s in SIDES:
            out[f'side={s}/{lag}'] = [r for r in base if r['side'] == s]
        for lo, hi in ((-99, 0), (0, 5), (5, 12), (12, 25), (25, 99)):
            out[f'clv[{lo},{hi})/{lag}'] = [r for r in base if lo <= r['clv'] < hi]
        for lo, hi in ((0, .15), (.15, .25), (.25, .40), (.40, 1.0)):
            out[f'px[{lo:.2f},{hi:.2f})/{lag}'] = [r for r in base if lo <= r['price'] < hi]
        for lo, hi in ((0, .8), (.8, 1.6), (1.6, 99)):
            out[f'spread[{lo},{hi})/{lag}'] = [r for r in base if lo <= r['spread'] < hi]
        for lo, hi in ((-99, -1), (-1, 1), (1, 99)):
            out[f'disagree[{lo},{hi})/{lag}'] = [
                r for r in base if r['disagree'] is not None and lo <= r['disagree'] < hi]
        # two-way: book x clv band, and spread x clv band
        for b in BOOKS:
            for lo, hi in ((0, 5), (5, 12), (12, 99)):
                out[f'{b}&clv[{lo},{hi})/{lag}'] = [
                    r for r in base if r['book'] == b and lo <= r['clv'] < hi]
        for lo, hi in ((0, .8), (.8, 99)):
            for c0, c1 in ((5, 12), (12, 99)):
                out[f'spr<{hi}&clv[{c0},{c1})/{lag}'] = [
                    r for r in base if lo <= r['spread'] < hi and c0 <= r['clv'] < c1]
    return {k: v for k, v in out.items() if len(v) >= 40}


def main():
    recs = load()
    fixtures = {r['fixture'] for r in recs}
    print(f'{len(recs)} (fixture,book,side,lag) records from {len(fixtures)} fixtures')
    print(f'objective: net = close_mid - entry_ask, in cents. Positive = edge '
          f'survives the spread.\n')

    sl = slices(recs)
    print(f'testing {len(sl)} slices\n')

    res = []
    for label, v in sl.items():
        m, t = tstat([r['net'] for r in v])
        mv, _ = tstat([r['mv'] for r in v])
        hs = statistics.mean([r['halfsp'] for r in v])
        res.append((abs(t), t, m, mv, hs, len(v), label))
    res.sort(reverse=True)

    # ── permutation null: shuffle net WITHIN fixtures, recompute max|t| ──────
    byfix = defaultdict(list)
    for r in recs:
        byfix[r['fixture']].append(r)
    nulls = []
    for _ in range(NPERM):
        pool = []
        for f, rs in byfix.items():
            vals = [r['net'] for r in rs]
            random.shuffle(vals)
            for r, v in zip(rs, vals):
                pool.append(dict(r, net=v))
        sl2 = slices(pool)
        mx = 0.0
        for label, v in sl2.items():
            _, t = tstat([r['net'] for r in v])
            mx = max(mx, abs(t))
        nulls.append(mx)
    nulls.sort()
    thr = nulls[int(0.95 * len(nulls))]

    print(f'permutation family-wise |t| threshold (95%, {NPERM} perms, shuffled '
          f'within fixture): {thr:.2f}')
    print(f'  (a naive per-slice threshold would be 1.96 -- using it here would '
          f'manufacture findings)\n')
    print(f'{"slice":34s} {"n":>5s} {"net":>7s} {"move":>7s} {"halfsp":>7s} {"t":>7s}  verdict')
    for at, t, m, mv, hs, n, label in res[:25]:
        v = 'SURVIVES' if at > thr else ''
        print(f'{label:34s} {n:5d} {m:+7.3f} {mv:+7.3f} {hs:7.3f} {t:+7.2f}  {v}')

    surv = [r for r in res if r[0] > thr]
    print(f'\n{len(surv)} of {len(sl)} slices beat the family-wise threshold.')
    if surv:
        pos = [s for s in surv if s[2] > 0]
        print(f'  of those, {len(pos)} have POSITIVE net (an actual tradeable edge):')
        for at, t, m, mv, hs, n, label in pos:
            print(f'    {label:32s} n={n:5d}  net={m:+.3f}c  t={t:+.2f}')
        if not pos:
            print('  ALL of them are NEGATIVE -- the mine found ways to lose, '
                  'not ways to win.')
    else:
        print('  No slice survives. Nothing here beats the spread.')


if __name__ == '__main__':
    main()