"""Forward (OOS) xlow restricted to the leagues the BACKTEST actually covered. THE QUESTION. The frozen n=286 backtest ran on one set of leagues. Coverage has grown a lot since -- new competitions, a season rollover -- so the live forward test is not scoring the same universe the rule was validated on. How does forward look on the backtest's own leagues? WHERE THE BACKTEST LIVES -- and why this script exists at all. The n=286 backtest was NOT run on live_data. It ran on poly_backfill/ (April-June 2026), because live_data carries NO Polymarket columns before 2026-07: poly fill rate by month is 0.0% / 0.0% / 0.0% / 74.1% / 63.5% for Apr / May / Jun / Jul / Aug. fire_xlow is shared-price-only (it needs BOTH books), so running it over the pre-FREEZE_DATE slice of live_data fires SEVEN times in three leagues -- an artifact of missing poly capture, not the backtest. Anything that reads "in-sample" off live_data is measuring the poly rollout. Do not do it. So: backtest leagues come from poly_backfill via the same strict rule (research/kalshi_only_execution.py), forward numbers come from live_data via forward_test_unique_book.fire_xlow. WHAT THIS IS NOT. Not a league filter, and it must not become one. The league set is chosen knowing both samples, so "forward, backtest leagues only" is a post-hoc slice and not an out-of-sample result -- quoting it as the rule's true performance is the selection error project_xlow_test_segment_decomp warns about. project_league_filter_study already tested filtering ON league and found it buys little. This answers one narrow question: is the forward number dragged by leagues the backtest never saw? Read-only. Run: python research/xlow_backtest_leagues.py """ import csv import glob import os import sys from collections import defaultdict sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import forward_test_unique_book as F import kalshi_only_execution as K BACKFILL = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'poly_backfill') def load_backfill(): """kalshi_only_execution.load(), but carrying the league through. That loader keys on (game_date, home, away) and drops `league`, which is the one field this script needs. Same grouping, same PRE filter, same result rule -- league is read off the first row of the group. """ fixtures = defaultdict(list) for path in glob.glob(os.path.join(BACKFILL, '*_polybackfill.csv')): with open(path, newline='', encoding='utf-8') as fh: for r in csv.DictReader(fh): fixtures[(r.get('game_date'), r.get('home'), r.get('away'))].append(r) out = [] for key, rows in fixtures.items(): gd = key[0] if not gd: continue pre = sorted([(K._pts(r['timestamp']), r) for r in rows if (r.get('status') or '').upper() == 'PRE' and K._pts(r['timestamp'])], key=lambda t: t[0]) ftr = K._result(rows) if pre and ftr: out.append(dict(gd=gd, ftr=ftr, pre=[r for _, r in pre], league=(rows[0].get('league') or '?'))) out.sort(key=lambda f: f['gd']) return out def _fmt(label, s, width=32): if not s: return f'{label:<{width}} {"(no bets)":>12}' return (f'{label:<{width}} n={s["n"]:>4} WR {s["wr"]:>5.1f}% ' f'impl {s["iwr"]:>5.1f}% ROI {s["roi"]:>+7.2f}% z {s["z"]:>+5.2f}') def main(): # ── backtest side (poly_backfill, the real n=286 universe) back = load_backfill() b_lg = defaultdict(list) for fx in back: b = K.xlow(fx) if b: b_lg[fx['league']].append(b) back_bets = [b for bs in b_lg.values() for b in bs] gds = [fx['gd'] for fx in back] print(f'BACKTEST source=poly_backfill fixtures={len(back)} ' f'dates {min(gds)} .. {max(gds)}') print(_fmt(' backtest xlow (strict)', F.stat(back_bets))) print(f' leagues firing: {len(b_lg)} {sorted(b_lg)}') print() # ── forward side (live_data, OOS) prepped = F.load_fixtures() fwd = [fx for fx in prepped if fx['gd'] > F.FREEZE_DATE] f_lg = defaultdict(list) for fx in fwd: b = F.fire_xlow(fx) if b: f_lg[fx['league']].append(b) back_leagues = set(b_lg) fwd_leagues = set(f_lg) shared = fwd_leagues & back_leagues new = fwd_leagues - back_leagues unseen = back_leagues - fwd_leagues def flat(d, keys): return [b for k in keys for b in d.get(k, [])] print(f'FORWARD source=live_data FREEZE_DATE={F.FREEZE_DATE} ' f'fixtures={len(fwd)}') print(f' leagues firing: {len(f_lg)}') print(f' also in backtest : {len(shared)} {sorted(shared)}') print(f' NEW since : {len(new)} {sorted(new)}') print(f' in backtest, none forward : {len(unseen)} {sorted(unseen)}') print() print('=' * 92) print(_fmt('FORWARD all leagues', F.stat(flat(f_lg, fwd_leagues)))) print(_fmt('FORWARD backtest leagues only', F.stat(flat(f_lg, shared)))) print(_fmt('FORWARD NEW leagues only', F.stat(flat(f_lg, new)))) print('=' * 92) print() # ── IS THE SPLIT REAL? Two halves always differ; the question is whether # this one differs by more than a 1.99u/bet per-bet SD allows at these n. # Welch t on per-bet returns, plus a permutation test that makes no # distributional assumption (returns are wildly skewed -- a 12c winner pays # +7.3u -- so the t is the weaker of the two). import random import math random.seed(11) def rets(bets): return [((1 - b['px']) / b['px']) if b['won'] else -1.0 for b in bets] ra = rets(flat(f_lg, shared)) rb = rets(flat(f_lg, new)) def mean(xs): return sum(xs) / len(xs) def var(xs): m = mean(xs) return sum((x - m) ** 2 for x in xs) / (len(xs) - 1) diff = (mean(ra) - mean(rb)) * 100 se = math.sqrt(var(ra) / len(ra) + var(rb) / len(rb)) * 100 t = diff / se if se else 0.0 pool = ra + rb na = len(ra) hits = 0 TRIALS = 20000 for _ in range(TRIALS): random.shuffle(pool) if abs(mean(pool[:na]) - mean(pool[na:])) * 100 >= abs(diff): hits += 1 print(f'DIFFERENCE backtest-leagues minus new-leagues: {diff:+.2f} pp/bet') print(f' Welch t = {t:+.2f} per-bet SD {math.sqrt(var(ra)):.2f}u / ' f'{math.sqrt(var(rb)):.2f}u') print(f' permutation p = {hits / TRIALS:.3f} ({TRIALS:,} shuffles)') print() # ── SENSITIVITY. "Leagues from the backtest" is ambiguous: AR1 fired ONCE # in the backtest and carries 37 forward bets. Requiring a real backtest # presence is the stricter and more honest reading, so show both. print('SENSITIVITY -- require at least MIN backtest bets to count as a ' '"backtest league":') print(f' {"min":>4} {"leagues":>8} {"fwd n":>6} {"ROI":>9} {"z":>6} ' f'{"excluded n":>10} {"ROI":>9}') for lo in (1, 3, 5, 10, 20): keep = {lg for lg in shared if len(b_lg[lg]) >= lo} rest = fwd_leagues - keep s_in, s_out = F.stat(flat(f_lg, keep)), F.stat(flat(f_lg, rest)) print(f' {lo:>4} {len(keep):>8} {s_in["n"]:>6} {s_in["roi"]:>+8.2f}% ' f'{s_in["z"]:>+5.2f} {s_out["n"]:>10} {s_out["roi"]:>+8.2f}%') print() # ── DOES LEAGUE PERFORMANCE PERSIST AT ALL? The whole premise of slicing by # "backtest leagues" is that a league's behaviour carries from one period to # the next. Test it directly: rank-correlate backtest ROI against forward # ROI over the leagues with enough bets on BOTH sides to be worth ranking. def spearman(xs, ys): def ranks(v): order = sorted(range(len(v)), key=lambda i: v[i]) r = [0.0] * len(v) for pos, i in enumerate(order): r[i] = pos + 1.0 return r rx, ry = ranks(xs), ranks(ys) n = len(xs) mx, my = sum(rx) / n, sum(ry) / n num = sum((a - mx) * (b - my) for a, b in zip(rx, ry)) den = math.sqrt(sum((a - mx) ** 2 for a in rx) * sum((b - my) ** 2 for b in ry)) return num / den if den else 0.0 for lo in (5, 10): pairs = [(F.stat(b_lg[lg])['roi'], F.stat(f_lg[lg])['roi']) for lg in shared if len(b_lg[lg]) >= lo and len(f_lg[lg]) >= lo] if len(pairs) >= 4: rho = spearman([p[0] for p in pairs], [p[1] for p in pairs]) print(f'PERSISTENCE leagues with >={lo} bets both sides: ' f'k={len(pairs)} Spearman(backtest ROI, forward ROI) ' f'= {rho:+.3f}') print() print('FORWARD, backtest leagues only, per league (desc by n):') print(f' {"lg":<7}{"n":>4} {"WR":>6} {"impl":>6} {"ROI":>9} {"z":>6} backtest') for lg in sorted(shared, key=lambda k: -len(f_lg[k])): s, bs = F.stat(f_lg[lg]), F.stat(b_lg[lg]) print(f' {lg:<7}{s["n"]:>4} {s["wr"]:>5.1f}% {s["iwr"]:>5.1f}% ' f'{s["roi"]:>+8.2f}% {s["z"]:>+5.2f} ' f'n={bs["n"]:<3} ROI {bs["roi"]:+.1f}%') if __name__ == '__main__': main()