"""Candidate-rule sweep with an explicit train/test split. Trades nothing.

WHY THESE CANDIDATES AND NOT A GRID SEARCH. With 1,717 fixtures and a flexible
rule space, something will always backtest well -- `unique` shows +20.6% at n=55
and is a hindsight artifact whose executable cousin t60 runs -18.3%. So the
candidates here are limited to three MECHANISMS, each motivated by a measured
finding rather than by scanning for a winner:

  1. PRICE FLOOR. The binding problem is variance, not edge: xlow's per-bet stdev
     is 1.94u at a mean price of 0.26, so proving its observed +0.8% ROI needs
     ~100,000 bets (~46 years). Variance of a unit bet scales as (1-p)/p, so
     restricting to higher-priced outcomes cuts the required n roughly 4x at
     p=0.5 versus p=0.26. This does not create edge; it makes edge MEASURABLE.

  2. PERSISTENCE. forward_test_unique_book.fire_xlow's own docstring flags this
     as "a candidate refinement for the n>=100 review -- NOT wired now": on a 1c
     grid with flip-flopping books, scanning for the first qualifying snapshot
     finds TRANSIENT configurations. Requiring the shape to hold N consecutive
     snapshots should drop the noise fires. Pre-registered, not dredged.

  3. KALSHI-ONLY. xlow needs both books and only 411 of 1,717 fixtures carry
     both; 1,364 carry Kalshi+Pinnacle. A single-book rule has 3x the sample,
     which matters more than rule elegance when the whole problem is power.

DISCIPLINE. Train = on/before FREEZE_DATE, test = after. Report BOTH always;
a candidate that only works in train is dead. Note that xlow's core was itself
fitted on train, so train numbers for its variants are optimistic by
construction -- only the TEST column is evidence, and only the INCREMENT over
baseline xlow is attributable to the new dimension.

MULTIPLE COMPARISONS. Every candidate tried is printed, including losers, with
the count. At ~20 candidates, one crossing p=0.05 by luck is the expectation,
not a finding.

Usage:  python research\\rule_sweep.py
"""
import math
import statistics
import sys
from collections import defaultdict

sys.path.insert(0, r'E:\soccer\research')
import forward_test_unique_book as FT

BAND = FT.BAND
SIDX = FT.SIDX


def _pin_fair(r):
    ph, pd_, pa = FT._f(r.get('pin_home')), FT._f(r.get('pin_draw')), FT._f(r.get('pin_away'))
    if None in (ph, pd_, pa) or not (ph > 1 and pd_ > 1 and pa > 1):
        return None
    return FT.fair_mult((ph, pd_, pa))


def _qualifies(r, pf, side, book, other):
    """(clv, mid, ask) if `book` is the exclusive low on `side`, else None."""
    m, a = FT.book_quote(r, book, side)
    om, _ = FT.book_quote(r, other, side)
    if m is None or om is None or a is None:
        return None
    if not (0 < m < 1 and 0 < om < 1):
        return None
    if m >= om:
        return None                       # must be the CHEAPER book
    fair = pf[SIDX[side]]
    if fair <= 0:
        return None
    clv = (fair - m) / fair
    oclv = (fair - om) / fair
    if not (BAND[0] <= clv < BAND[1]):
        return None
    if BAND[0] <= oclv < BAND[1]:
        return None                       # other book must NOT be in band
    return clv, m, a


def run_xlow(fixtures, min_price=0.0, persist=1):
    """One bet per fixture: first pre-KO snapshot where the shape holds for
    `persist` CONSECUTIVE snapshots. Returns list of (ret, ask, won)."""
    out = []
    for fx in fixtures:
        streak = defaultdict(int)
        picked = None
        for r in fx['pre']:
            pf = _pin_fair(r)
            if pf is None:
                continue
            # Which (side, book) configurations qualify on THIS snapshot.
            live = {}
            for side in ('home', 'away'):
                for book, other in (('kalshi', 'poly'), ('poly', 'kalshi')):
                    q = _qualifies(r, pf, side, book, other)
                    if q and q[1] >= min_price:
                        live[(side, book)] = q
            # A streak must be CONSECUTIVE: anything absent now resets to zero.
            for k in list(streak):
                if k not in live:
                    streak[k] = 0
            hit = None
            for k, q in live.items():
                streak[k] += 1
                if streak[k] >= persist and hit is None:
                    hit = (k[0], k[1], q)
            if hit:
                picked = hit
                break
        if not picked:
            continue
        side, book, (clv, mid, ask) = picked
        won = (fx['ftr'] == ('H' if side == 'home' else 'A'))
        out.append(((1.0 / ask - 1.0) if won else -1.0, ask, won))
    return out


def run_kalshi_only(fixtures, min_price=0.0, persist=1):
    """Kalshi mid-CLV in band vs Pinnacle. No second book required -> 3x sample."""
    out = []
    for fx in fixtures:
        streak = defaultdict(int)
        picked = None
        for r in fx['pre']:
            pf = _pin_fair(r)
            if pf is None:
                continue
            live = {}
            for side in ('home', 'away'):
                m, a = FT.book_quote(r, 'kalshi', side)
                if m is None or a is None or not (0 < m < 1):
                    continue
                fair = pf[SIDX[side]]
                if fair <= 0:
                    continue
                clv = (fair - m) / fair
                if BAND[0] <= clv < BAND[1] and m >= min_price:
                    live[side] = a
            for k in list(streak):
                if k not in live:
                    streak[k] = 0
            hit = None
            for k, a in live.items():
                streak[k] += 1
                if streak[k] >= persist and hit is None:
                    hit = (k, a)
            if hit:
                picked = hit
                break
        if not picked:
            continue
        side, ask = picked
        won = (fx['ftr'] == ('H' if side == 'home' else 'A'))
        out.append(((1.0 / ask - 1.0) if won else -1.0, ask, won))
    return out


def run_book_only(fixtures, book, min_price=0.0, persist=1):
    """Single-book mid-CLV in band vs Pinnacle, for any one book.

    Poly-US matters here for two reasons beyond sample size: it pays a MAKER
    REBATE (0.0125 x p x (1-p)) where Kalshi charges a taker fee, and it is the
    only venue covering GAP_LEAGUES. Note its history only starts 2026-07-24,
    so it has no pre-freeze period -- the split for it is internal by date.
    """
    out = []
    for fx in fixtures:
        streak = defaultdict(int)
        picked = None
        for r in fx['pre']:
            pf = _pin_fair(r)
            if pf is None:
                continue
            live = {}
            for side in ('home', 'away'):
                m, a = FT.book_quote(r, book, side)
                if m is None or a is None or not (0 < m < 1 and 0 < a < 1):
                    continue
                fair = pf[SIDX[side]]
                if fair <= 0:
                    continue
                clv = (fair - m) / fair
                if BAND[0] <= clv < BAND[1] and m >= min_price:
                    live[side] = a
            for k in list(streak):
                if k not in live:
                    streak[k] = 0
            hit = None
            for k, a in live.items():
                streak[k] += 1
                if streak[k] >= persist and hit is None:
                    hit = (k, a)
            if hit:
                picked = hit
                break
        if not picked:
            continue
        side, ask = picked
        won = (fx['ftr'] == ('H' if side == 'home' else 'A'))
        out.append(((1.0 / ask - 1.0) if won else -1.0, ask, won))
    return out


def summarise(rs):
    n = len(rs)
    if n == 0:
        return None
    m = statistics.mean(x[0] for x in rs)
    sd = statistics.pstdev([x[0] for x in rs]) or 1e-9
    se = sd / math.sqrt(n)
    return dict(n=n, roi=m, z=m / se, wr=sum(x[2] for x in rs) / n,
                imp=statistics.mean(x[1] for x in rs), sd=sd,
                need=(1.96 * sd / max(abs(m), 1e-6)) ** 2)


def main():
    fx = FT.load_fixtures()
    train = [f for f in fx if f['gd'] <= FT.FREEZE_DATE]
    test = [f for f in fx if f['gd'] > FT.FREEZE_DATE]
    print(f'{len(fx)} fixtures | train (<={FT.FREEZE_DATE}) {len(train)} | test {len(test)}')
    print('train numbers for xlow variants are OPTIMISTIC -- xlow was fitted here.')
    print('only the TEST column is evidence.\n')

    cands = []
    for mp in (0.0, 0.20, 0.25, 0.30, 0.40):
        for pz in (1, 2, 3):
            cands.append((f'xlow  minP>={mp:.2f} persist={pz}',
                          lambda f, mp=mp, pz=pz: run_xlow(f, mp, pz)))
    for mp in (0.0, 0.20, 0.30, 0.40):
        for pz in (1, 2):
            cands.append((f'kalshi-only minP>={mp:.2f} persist={pz}',
                          lambda f, mp=mp, pz=pz: run_kalshi_only(f, mp, pz)))

    print(f'{"candidate":36s} {"TRAIN n":>8s} {"roi":>7s} {"z":>6s}   '
          f'{"TEST n":>7s} {"roi":>7s} {"z":>6s} {"need n":>9s}')
    rows = []
    for name, fn in cands:
        a, b = summarise(fn(train)), summarise(fn(test))
        if not a or not b:
            continue
        rows.append((name, a, b))
        print(f'{name:36s} {a["n"]:8d} {100*a["roi"]:+6.1f}% {a["z"]:+6.2f}   '
              f'{b["n"]:7d} {100*b["roi"]:+6.1f}% {b["z"]:+6.2f} {b["need"]:9,.0f}')

    print(f'\n{len(rows)} candidates tried. At p=0.05, ~{0.05*len(rows):.1f} '
          f'are expected to look significant by luck alone.')
    good = [r for r in rows if r[2]['z'] > 1.96]
    print(f'candidates with TEST z > 1.96: {len(good)}')
    for name, a, b in good:
        print(f'   {name}: test n={b["n"]} roi={100*b["roi"]:+.1f}% z={b["z"]:+.2f}')


if __name__ == '__main__':
    main()