"""What actually predicts a LEG filling, and a BOOK completing? METHOD, and why each guard is here -- every one of them corresponds to a way this project has already been fooled: GROUPED CV BY EVENT. The three legs of one book share a market, a kickoff and a liquidity regime. Random k-fold puts siblings on both sides of the split and the model reads the answer off its own training set. Every split here is GroupKFold on the event. LEAKY FEATURES DROPPED. `minutes_exposed` and `n_posts` look like the strongest predictors in the file and are worthless: an episode STOPS when it fills, so a filled leg is short by construction. Using them rediscovers the label. A NO-FEATURE CONTROL. project_nonprice_features_dead: a model showed +2.7% ROI and the no-model control showed +1.89%, i.e. almost all of it was the baseline. Here the control is the base rate, and AUC 0.5. CAPACITY LADDER. project_nonprice_features_dead again: "every increase in model CAPACITY made it significantly worse (GBM d6 t=-5.71), the signature of no signal." If depth-1 beats depth-4, believe the null, not the model. PERMUTATION IMPORTANCE ON HELD-OUT FOLDS, not impurity importance -- the latter inflates high-cardinality and continuous features regardless of signal. Usage: python research\\mm_fill_ml.py """ import sys import warnings import numpy as np import pandas as pd from sklearn.ensemble import GradientBoostingClassifier from sklearn.linear_model import LogisticRegression from sklearn.dummy import DummyClassifier from sklearn.model_selection import GroupKFold, cross_val_predict from sklearn.metrics import roc_auc_score, log_loss from sklearn.pipeline import Pipeline from sklearn.preprocessing import StandardScaler from sklearn.impute import SimpleImputer from sklearn.inspection import permutation_importance warnings.filterwarnings('ignore') SRC = r'E:\soccer\research\mm_fill_dataset.csv' RNG = 0 LEAKY = ['minutes_exposed', 'n_posts', 'seconds_to_fill', 'fill_qty'] def load(): df = pd.read_csv(SRC) for c in df.columns: if c not in ('ts_first', 'league', 'event', 'ticker', 'side'): df[c] = pd.to_numeric(df[c], errors='coerce') return df def engineer(df): d = pd.DataFrame(index=df.index) # raw at-post market state for c in ('price_cents', 'ask_at_post', 'bid_at_post', 'spread_c', 'pin_fair', 'ask_vs_pin_c', 'above_touch_c', 'hrs_to_ko', 'collected', 'target', 'scale', 'fresh'): if c in df: d[c] = df[c] # heavy-tailed counts -> log. queue_ahead spans 0..40k; untransformed it # makes every linear model a step function on one outlier. for c in ('queue_ahead', 'bid_size', 'volume', 'volume_24h', 'book_volume', 'open_interest'): if c in df: d['log_' + c] = np.log1p(df[c].clip(lower=0)) # `side` is 127 team abbreviations -- useless raw. What matters is whether # the leg is the draw, and where it sits in the price distribution. d['is_tie'] = (df['side'].astype(str).str.upper() == 'TIE').astype(int) p = df['price_cents'] / 100.0 d['is_favourite'] = (p >= 0.45).astype(int) d['is_longshot'] = (p <= 0.15).astype(int) # book already part-filled at post time? d['book_started'] = (df['collected'].fillna(0) > 0).astype(int) return d def evaluate(X, y, groups, label): print(f'\n{"=" * 66}\n{label}\n{"=" * 66}') print(f'n={len(y)} positives={int(y.sum())} ({100*y.mean():.1f}%) ' f'groups={groups.nunique()}') if y.sum() < 25: print('too few positives to model honestly -- stopping here') return None n_splits = min(5, groups.nunique(), int(y.sum())) cv = GroupKFold(n_splits=n_splits) models = { 'baseline (no features)': DummyClassifier(strategy='prior'), 'logistic': Pipeline([('i', SimpleImputer(strategy='median')), ('s', StandardScaler()), ('m', LogisticRegression(max_iter=2000, C=0.5))]), 'GBM depth1': Pipeline([('i', SimpleImputer(strategy='median')), ('m', GradientBoostingClassifier( max_depth=1, n_estimators=150, learning_rate=0.05, random_state=RNG))]), 'GBM depth3': Pipeline([('i', SimpleImputer(strategy='median')), ('m', GradientBoostingClassifier( max_depth=3, n_estimators=150, learning_rate=0.05, random_state=RNG))]), } results = {} for name, mdl in models.items(): try: pr = cross_val_predict(mdl, X, y, cv=cv, groups=groups, method='predict_proba')[:, 1] except Exception as e: print(f' {name:24} FAILED {type(e).__name__}') continue auc = roc_auc_score(y, pr) if len(np.unique(y)) > 1 else float('nan') ll = log_loss(y, np.clip(pr, 1e-6, 1 - 1e-6)) results[name] = (auc, ll, pr) print(f' {name:24} AUC {auc:.3f} logloss {ll:.4f}') # bootstrap CI on the best non-baseline AUC -- one number with no error bar # is how "significant" results get quoted here best = max((k for k in results if 'baseline' not in k), key=lambda k: results[k][0], default=None) if best: pr = results[best][2] rng = np.random.default_rng(RNG) boots = [] idx = np.arange(len(y)) for _ in range(400): s = rng.choice(idx, len(idx), replace=True) if len(np.unique(y.values[s])) < 2: continue boots.append(roc_auc_score(y.values[s], pr[s])) lo, hi = np.percentile(boots, [2.5, 97.5]) print(f'\n best = {best}: AUC {results[best][0]:.3f} ' f'95% CI [{lo:.3f}, {hi:.3f}]') if lo <= 0.5: print(' -> CI includes 0.5: NOT distinguishable from no signal') else: print(' -> CI excludes 0.5: there is signal') d1 = results.get('GBM depth1', (np.nan,))[0] d3 = results.get('GBM depth3', (np.nan,))[0] if d1 == d1 and d3 == d3: print(f' capacity check: depth1 {d1:.3f} vs depth3 {d3:.3f} ' f'-> {"more capacity HURTS (null-like)" if d3 < d1 else "more capacity helps"}') # permutation importance on held-out folds if best and 'baseline' not in best: mdl = models[best] imps = [] for tr, te in cv.split(X, y, groups): if y.iloc[tr].sum() < 5 or y.iloc[te].sum() < 2: continue mdl.fit(X.iloc[tr], y.iloc[tr]) r = permutation_importance(mdl, X.iloc[te], y.iloc[te], n_repeats=8, random_state=RNG, scoring='roc_auc') imps.append(r.importances_mean) if imps: imp = np.mean(imps, axis=0) order = np.argsort(imp)[::-1] print(f'\n permutation importance (drop in held-out AUC):') for i in order[:12]: bar = '#' * max(0, int(imp[i] * 200)) print(f' {X.columns[i]:20} {imp[i]:+.4f} {bar}') return results def main(): df = load() print(f'loaded {len(df)} leg episodes from {SRC}') print(f'dropped as leaky: {LEAKY}') # ---------- LEG LEVEL ---------- X = engineer(df) y = df['filled'].astype(int) groups = df['event'].astype(str) evaluate(X, y, groups, 'LEG FILL -- does this resting order get lifted?') # ---------- BOOK LEVEL ---------- # one row per event: book-wide state at post, outcome = all three legs sold g = df.groupby('event') bk = pd.DataFrame({ 'legs': g.size(), 'n_filled': g['filled'].sum(), 'min_volume': g['volume'].min(), 'book_volume': g['book_volume'].max(), 'min_queue': g['queue_ahead'].min(), 'max_queue': g['queue_ahead'].max(), 'mean_spread': g['spread_c'].mean(), 'sum_ask': g['ask_at_post'].sum(), 'hrs_to_ko': g['hrs_to_ko'].max(), 'mean_above_touch': g['above_touch_c'].mean(), 'max_above_touch': g['above_touch_c'].max(), 'min_oi': g['open_interest'].min(), }) bk = bk[bk['legs'] >= 3] bk['gross_c'] = (bk['sum_ask'] - 1.0) * 100 bk['completed'] = (bk['n_filled'] >= 3).astype(int) for c in ('min_volume', 'book_volume', 'min_queue', 'max_queue', 'min_oi'): bk['log_' + c] = np.log1p(bk[c].clip(lower=0)) bk.drop(columns=[c], inplace=True) Xb = bk.drop(columns=['completed', 'n_filled', 'legs', 'sum_ask']) yb = bk['completed'] evaluate(Xb, yb, pd.Series(bk.index, index=bk.index), 'BOOK COMPLETION -- do all three legs sell?') # the one relationship memory already claims; check it holds here print(f'\n{"=" * 66}\nvolume vs completion (the claim from ' f'project_mm_completion_arithmetic)\n{"=" * 66}') if 'log_min_volume' in bk: bk['vol'] = np.expm1(bk['log_min_volume']) for lo, hi in ((0, 250), (250, 500), (500, 1000), (1000, 2500), (2500, 1e12)): sel = bk[(bk['vol'] >= lo) & (bk['vol'] < hi)] if len(sel): print(f' min leg volume {lo:>6,.0f}-{hi if hi < 1e11 else "inf":>9}' f' n={len(sel):>4} completion {100*sel["completed"].mean():5.1f}%') if __name__ == '__main__': sys.exit(main())