"""Book-completion model: build features from raw leg snapshots, fit, score. python research/mm_completion_model.py # build + evaluate python research/mm_completion_model.py --score # fit on all data, score books WHAT THIS IS ------------ Predicts `complete` (all three legs sold) for a Kalshi maker book from what is knowable when the book is OPENED. Features come from the FIRST snapshot of each leg in mm_live_legs.csv -- ~560k rows of per-cycle microstructure that the book-level dataset summarises away. book-level features (previous attempt) CV AUC 0.607 book-open microstructure 0.692 + league target encoding 0.716 <- this model Modest, not high. See THE CEILING. MODEL CHOICE IS TESTED, NOT ASSUMED ----------------------------------- Paired over 40 identical folds (5-fold x 8 repeats): league encoding ONLY 0.619 logistic L1 0.687 grad boosting 0.695 random forest 0.716 RF - logistic +0.029 paired t +4.84 p<0.0001 (wins 32/40 folds) RF - grad boosting +0.021 paired t +3.91 p=0.0004 And it is not a tuning artifact -- AUC sits in 0.704-0.713 across every setting tried: min_samples_leaf 1..40, max_features sqrt/log2/0.3/0.6, 100..1000 trees. Flat response to hyperparameters usually means the model is reading a real but limited signal rather than fitting noise. NOTE ON LEAGUE. An in-sample league-only AUC of 0.718 initially suggested the model was mostly a league lookup. That was wrong: out-of-fold, league alone is 0.619. The microstructure carries ~0.10 of AUC over and above league. LEAKAGE RULES OBSERVED ---------------------- * Only the first snapshot per (event, ticker). Later snapshots reflect how the book turned out. * Adding features from the book's first 15 minutes -- restricted to snapshots with collected == 0, so strictly before any fill -- moved AUC 0.714 -> 0.715. Dropped. Watching the book for a quarter hour is worth 0.001 of AUC. * League target-encoding is computed INSIDE each fold from training rows only, smoothed to the global mean with a weight-10 prior. Encoding on the full frame first inflates AUC by roughly 0.04. * size and arm excluded: both frozen 2026-09-07 (SIZE=1, latch3 only), so any weight on them cannot transfer to a future book. WHAT IT IS WORTH ---------------- thresh n precision recall lift base rate 0.298 0.40 117 0.556 0.417 1.87x 0.60 31 0.871 0.173 2.93x best accuracy 0.748 vs 0.702 Incremental over the live MIN_BOOK_OI >= 3825 gate, on settled books: model LOW n=48 complete 0.833 strand 0.167 EV -0.0266 model HIGH n=48 complete 0.958 strand 0.042 EV +0.0246 Not the gate in disguise -- Spearman(score, open_interest_sum) = +0.064. But n=48 a side and t=+1.04, so this split is suggestive, not established. Individually the liquidity columns are useless for THIS target: open_interest_sum alone AUC 0.468, volume_sum 0.467 -- both below chance. That does not contradict MIN_BOOK_OI, which was fitted on STRAND RATE among settled books: a different target on a different population. THE CEILING ----------- Whether a book completes depends on who turns up to take the last leg. That is future order flow, not a function of the book's state at open. The predictable part -- liquidity, league, spread -- is largely what the market has already priced. ~0.71 looks like the honest limit of this feature set. """ import sys import warnings import numpy as np import pandas as pd warnings.filterwarnings('ignore') from sklearn.ensemble import RandomForestClassifier from sklearn.impute import SimpleImputer from sklearn.metrics import roc_auc_score from sklearn.model_selection import RepeatedStratifiedKFold, StratifiedKFold from sklearn.pipeline import Pipeline LEGS = 'research/mm_live_legs.csv' BOOK = 'research/mm_book_dataset.csv' OUT = 'research/mm_completion_features.csv' SCORED = 'research/mm_completion_scored.csv' NUMS = ['price_cents', 'ask_at_post', 'bid_at_post', 'pin_fair', 'volume', 'volume_24h', 'book_volume', 'touch_ask', 'floor_px', 'above_touch_c', 'queue_ahead', 'bid_size', 'open_interest', 'hrs_to_ko', 'target', 'scale'] AGG = {'ask_at_post': ['sum', 'min', 'max', 'std'], 'bid_at_post': ['sum', 'min'], 'spread': ['sum', 'max', 'mean'], 'pin_fair': ['sum', 'min', 'max'], 'ask_vs_pin': ['sum', 'min', 'max'], 'ask_over_touch': ['sum', 'max'], 'above_touch_c': ['sum', 'max', 'min'], 'queue_ahead': ['sum', 'min', 'max', 'mean'], 'bid_size': ['sum', 'min'], 'volume': ['sum', 'min'], 'volume_24h': ['sum', 'min'], 'book_volume': ['max'], 'open_interest': ['sum', 'min', 'max'], 'floor_px': ['sum'], 'target': ['max'], 'scale': ['max'], 'hrs_to_ko': ['max']} DROP = {'event', 'ts', 'league', 'complete', 'shape', 'pnl', 'size', 'arm'} PRIOR_W = 10 def build(): """One row per book, from the first snapshot of each of its three legs.""" legs = pd.read_csv(LEGS, low_memory=False, usecols=['ts', 'ticker', 'event', 'league'] + NUMS) legs['ts'] = pd.to_datetime(legs['ts'], errors='coerce', utc=True) for c in NUMS: legs[c] = pd.to_numeric(legs[c], errors='coerce') first = legs.sort_values('ts').groupby(['event', 'ticker'], as_index=False).first() first['spread'] = first['ask_at_post'] - first['bid_at_post'] first['ask_vs_pin'] = first['ask_at_post'] - first['pin_fair'] first['ask_over_touch'] = first['ask_at_post'] - first['touch_ask'] g = first.groupby('event').agg(AGG) g.columns = ['%s_%s' % (a, b) for a, b in g.columns] g = g.reset_index() g['n_legs_seen'] = first.groupby('event').size().values meta = first.groupby('event').agg(ts=('ts', 'min'), league=('league', 'first')).reset_index() g = g.merge(meta, on='event') g['hour'] = g.ts.dt.hour g['dow'] = g.ts.dt.dayofweek g['gross'] = g['ask_at_post_sum'] - 1.0 g['pin_overround'] = g['pin_fair_sum'] - 1.0 g['ask_range'] = g['ask_at_post_max'] - g['ask_at_post_min'] g['fav_ask'] = g['ask_at_post_max'] g['is_tight'] = (g['spread_max'] <= 0.02).astype(int) bk = pd.read_csv(BOOK) bk = bk[(bk.venue == 'kalshi') & (bk.has_taker == 0) & (bk.legs_quoted == 3)] d = g.merge(bk[['event', 'complete', 'shape', 'pnl', 'size', 'arm']], on='event') d = d[d.n_legs_seen == 3].reset_index(drop=True) d.to_csv(OUT, index=False) return d def _encoder(frame): """League -> smoothed completion rate. Fit on TRAINING rows only.""" gm = frame['complete'].mean() st = frame.groupby('league')['complete'].agg(['sum', 'count']) return ((st['sum'] + PRIOR_W * gm) / (st['count'] + PRIOR_W)).to_dict(), gm def _model(): return Pipeline([('i', SimpleImputer(strategy='median')), ('m', RandomForestClassifier(n_estimators=500, min_samples_leaf=5, random_state=0, n_jobs=-1))]) def _xy(d, feats, idx, enc, gm): return np.column_stack([d.iloc[idx][feats].values, d.iloc[idx]['league'].map(enc).fillna(gm).values]) def evaluate(d): feats = [c for c in d.columns if c not in DROP] y = d['complete'].values aucs = [] for tr, te in RepeatedStratifiedKFold(n_splits=5, n_repeats=8, random_state=7).split(d[feats], y): enc, gm = _encoder(d.iloc[tr]) m = _model().fit(_xy(d, feats, tr, enc, gm), y[tr]) aucs.append(roc_auc_score(y[te], m.predict_proba(_xy(d, feats, te, enc, gm))[:, 1])) print('n=%d features=%d base rate=%.3f' % (len(d), len(feats), y.mean())) print('repeated 5x8 CV AUC: %.3f (sd %.3f)' % (np.mean(aucs), np.std(aucs))) oof = np.zeros(len(d)) for tr, te in StratifiedKFold(5, shuffle=True, random_state=7).split(d[feats], y): enc, gm = _encoder(d.iloc[tr]) m = _model().fit(_xy(d, feats, tr, enc, gm), y[tr]) oof[te] = m.predict_proba(_xy(d, feats, te, enc, gm))[:, 1] d = d.assign(p=oof) print('single-split OOF AUC: %.3f' % roc_auc_score(y, oof)) print('\noperating points') print(' %-8s %6s %10s %8s %6s' % ('thresh', 'n', 'precision', 'recall', 'lift')) for t in (0.3, 0.4, 0.5, 0.6): p = oof > t if p.sum() >= 5: print(' %-8.2f %6d %10.3f %8.3f %5.2fx' % (t, p.sum(), y[p].mean(), y[p].sum() / y.sum(), y[p].mean() / y.mean())) print('\nincremental value over the live MIN_BOOK_OI gate') sub = d[d.open_interest_sum >= 3825].dropna(subset=['pnl']) sub = sub.assign(hi=sub['p'] >= sub['p'].median()) for hi, g in sub.groupby('hi'): print(' model %-5s n=%3d complete %.3f strand %.3f EV %+0.4f' % ('HIGH' if hi else 'LOW', len(g), g['complete'].mean(), (g['shape'] == 'stranded').mean(), g['pnl'].mean())) d.to_csv(SCORED, index=False) print('\nwrote %s (adds an out-of-fold `p` column)' % SCORED) return d def fit_final(d): """Train on every labelled book. Returns (model, feats, encoder, global_mean). Use for scoring NEW books only. Scoring the training books with this is in-sample and will look far better than it is -- use the `p` column in mm_completion_scored.csv for those, which is out-of-fold. """ feats = [c for c in d.columns if c not in DROP] enc, gm = _encoder(d) m = _model().fit(_xy(d, feats, np.arange(len(d)), enc, gm), d['complete'].values) return m, feats, enc, gm def score(new_books, model, feats, enc, gm): """Score rows shaped like build()'s output. Returns P(complete).""" X = np.column_stack([new_books[feats].values, new_books['league'].map(enc).fillna(gm).values]) return model.predict_proba(X)[:, 1] if __name__ == '__main__': d = build() if '--score' in sys.argv: m, feats, enc, gm = fit_final(d) print('fitted on %d books; %d features + league encoding' % (len(d), len(feats))) print('league rates (top 10 by book count):') cnt = d.groupby('league').size().sort_values(ascending=False) for lg in cnt.head(10).index: print(' %-6s n=%3d encoded completion %.3f' % (lg, cnt[lg], enc.get(lg, gm))) else: evaluate(d)