"""Does selecting volatile, liquid books raise BOOK COMPLETION? Everything so far about volatility and volume is per LEG (2.9% -> 20.1%). A book only pays when all three legs fill, and the number the strategy lives or dies on is book completion -- 3.7% under gate+ratchet, queue-aware, in mm_ratchet_backtest.py. Legs inside one book are not independent: a volatile liquid book should have all three legs moving and trading together, so completion ought to rise by more than the cube of the leg rate. Ought to. This measures it. Same machinery as the ratchet backtest -- real Kalshi trade prints, queue-aware fills, the entry gate and the ratchet -- with books additionally split by their own pre-quote volatility (from live_data) and their volume (from the Kalshi market at post time). The comparison that matters is completion AND total P&L against "no selection". A filter can only remove books, so it must earn its lost volume. """ import csv, collections, json, os, statistics, sys sys.path.insert(0, r'E:\soccer\main') sys.path.insert(0, r'E:\soccer\research') import mm_settle as S import mm_maker_live as ML from mm_volatility_clean import load_legs, load_movement, CACHE def _f(x): try: return float(x) except (TypeError, ValueError): return None def load_touch(): """{leg: market ask at first quote}. The live maker rests AT the touch; the paper maker's our_ask sits 0-5c above it on a price ladder. Since cents above the touch is the single strongest fill killer measured (AUC 0.690), scoring the ladder prices answers a different question and is not comparable to the 3.7% baseline from mm_ratchet_backtest.""" out = {} with open(r'E:\soccer\research\mm_paper_quotes.csv', encoding='utf-8', errors='replace') as fh: for r in csv.DictReader(fh): t = r.get('leg_ticker') if not t or t in out or not (r.get('ask_size') or '').strip(): continue a = _f(r.get('k_ask')) if a and 0 < a < 1: out[t] = a return out def load_volume(): out = {} with open(r'E:\soccer\research\mm_paper_quotes.csv', encoding='utf-8', errors='replace') as fh: for r in csv.DictReader(fh): t = r.get('leg_ticker') if not t or t in out or not (r.get('ask_size') or '').strip(): continue out[t] = _f(r.get('volume_24h')) or _f(r.get('volume')) return out def filled(trades, key, price, queue, ko): if key not in trades: return None cum = 0.0 for ts_s, p, c in trades[key]: ts = S._pt(ts_s) if not ts: break if (ts.tzinfo is None) != (ko.tzinfo is None): ts = ts.astimezone() if ts.tzinfo else ts.replace(tzinfo=ko.tzinfo) if ts >= ko: break if p >= price - 1e-9: cum += c if cum > queue: return True return False def score(books, label): if not books: return None done = [b for b in books if b['complete']] net = [b['net'] for b in done] return {'label': label, 'n': len(books), 'done': len(done), 'rate': 100.0 * len(done) / len(books), 'mean': (statistics.mean(net) * 100) if net else 0.0, 'total': sum(net) * 100} def main(): legs = load_legs() mv = load_movement(legs) vols = load_volume() touch = load_touch() trades = json.load(open(CACHE, encoding='utf-8')) # group legs into books by_book = collections.defaultdict(dict) for t, v in legs.items(): ev, leg = t.rsplit('-', 1) by_book[ev][leg] = v books = [] for ev, ls in by_book.items(): if len(ls) != 3: continue if not all(t in mv and (f'{ev}-{t}') in vols for t in ls): pass tick = {leg: f'{ev}-{leg}' for leg in ls} if not all(tick[l] in mv for l in ls): continue vv = [vols.get(tick[l]) for l in ls] if any(x is None for x in vv): continue if not all(tick[l] in touch for l in ls): continue asks = {l: touch[tick[l]] for l in ls} # AT the touch b = ML.book_budget(asks, {}) if b['decision'] == 'gate': continue # entry gate refuses it # RATCHET: floor at the entry ask, never below res = {} for l in ls: v = ls[l] key = f"{tick[l]}|{v['ts'].isoformat()}" res[l] = filled(trades, key, asks[l], v['queue'], v['ko']) if any(r is None for r in res.values()): continue complete = all(res.values()) coll = sum(asks.values()) fees = sum(ML.maker_fee(a) for a in asks.values()) books.append({ 'ev': ev, 'complete': complete, 'net': (coll - fees - 1.0) if complete else 0.0, 'sd': statistics.median(mv[tick[l]]['sd'] for l in ls), 'liq': statistics.median(vv), 'legs_filled': sum(1 for r in res.values() if r), }) if len(books) < 40: print(f'only {len(books)} books survive the join -- too few') return base = score(books, 'gate+ratchet, no selection') print(f'{len(books)} books past the entry gate, queue-aware fills\n') print(f'{"selection":>34} {"books":>6} {"complete":>9} {"rate":>7} ' f'{"net/book":>9} {"total":>9}') def show(s): if s: print(f'{s["label"]:>34} {s["n"]:>6} {s["done"]:>9} {s["rate"]:>6.1f}% ' f'{s["mean"]:>8.2f}c {s["total"]:>8.1f}c') show(base) sd_med = statistics.median(b['sd'] for b in books) lq_med = statistics.median(b['liq'] for b in books) show(score([b for b in books if b['sd'] > sd_med], 'volatile half')) show(score([b for b in books if b['liq'] > lq_med], 'liquid half')) show(score([b for b in books if b['sd'] > sd_med and b['liq'] > lq_med], 'volatile AND liquid')) show(score([b for b in books if b['sd'] <= sd_med and b['liq'] <= lq_med], 'calm AND thin (the dregs)')) # top quartile on both sd_q = sorted(b['sd'] for b in books)[int(len(books) * 0.75)] lq_q = sorted(b['liq'] for b in books)[int(len(books) * 0.75)] show(score([b for b in books if b['sd'] > sd_q or b['liq'] > lq_q], 'top quartile on either')) print('\n=== how far books get, by selection ===') for lab, sel in (('all', books), ('volatile AND liquid', [b for b in books if b['sd'] > sd_med and b['liq'] > lq_med]), ('calm AND thin', [b for b in books if b['sd'] <= sd_med and b['liq'] <= lq_med])): if not sel: continue d = collections.Counter(b['legs_filled'] for b in sel) tot = len(sel) print(f' {lab:>20}: ' + ' '.join( f'{k}/3 {100.0*d.get(k,0)/tot:>4.0f}%' for k in (0, 1, 2, 3))) print('\nA filter can only REMOVE books, so judge it on TOTAL, not rate.') if __name__ == '__main__': main()