"""Book-level dataset: what makes a BOOK complete, rather than a leg fill. WHY BOOK LEVEL. project_mm_completion_arithmetic reduces the maker to one number -- the completion rate -- because a complete book is near-deterministic (+1.78c on Kalshi, SD 1.69) while a partial one is a coin flip at ~25x the SD. Leg fill rate is an input to that; it is not the thing being decided. A model fitted on legs answers "will this order fill", which is the wrong question: three legs filling INDEPENDENTLY is not the same event as a book completing, because the legs of one book share a market, a moment and a budget. BUILT FROM mm_fill_dataset.csv, not from the raw log, so it inherits that file's two corrections: one row per resting order (not per re-post), and rows dropped before features existed. Run mm_fill_dataset.py first, or just run this -- it rebuilds automatically. THE UNIT IS A BOOK WE QUOTED. Features are taken AT OPEN (the first resting order on each leg), because the question is what could have been known when the decision to open the book was made. Anything measured later is contaminated by the outcome: a book that fills one leg gets re-priced by the budget rule, so its later above_touch and scale are consequences of filling, not causes. WATCH OUT FOR legs_quoted. A book where only two legs were ever offered CANNOT complete, and including those in a completion rate silently understates it. `complete` is defined against 3 legs; filter on legs_quoted==3 when the question is "given a full book was offered, did it complete". Usage: python research\\mm_book_dataset.py Writes: research\\mm_book_dataset.csv """ import csv import io import os import re import subprocess import sys from collections import defaultdict HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) LEGS_DS = os.path.join(HERE, 'mm_fill_dataset.csv') OUT = os.path.join(HERE, 'mm_book_dataset.csv') FIELD_DOCS = [ ('venue', "'kalshi' or 'polyus'. Both makers quote the same fixtures on " 'different venues with different economics -- Kalshi CHARGES the ' 'maker 0.0175*p(1-p), Poly-US CREDITS 0.0125*p(1-p) -- so ' 'break-even completion differs and the two must never be pooled ' 'without splitting on this.'), ('event', 'Event/market identifier -- one match, one row.'), ('league', 'Division code.'), ('ts_open', 'When the first leg of this book was first posted.'), ('hrs_to_ko_open', 'Hours to kickoff when the book was opened. LEG fill ' 'rate climbs into kickoff (26.1% inside 30min vs 6.8% ' 'beyond 24h) but BOOK completion moves the other way: ' 'among books that took a leg, with the entry gate held ' 'fixed, strand rate is ~17% under 6h against ~9% over ' 'it. A late book fills a leg and runs out of time to ' 'finish, which is the expensive half. MIN_OPEN_H=6 ' 'exists for this.'), ('legs_quoted', 'How many of the three legs were ever offered. A book with ' '<3 CANNOT complete -- filter on ==3 for a clean ' 'completion question.'), ('gross_c_open', 'Sum of the three asks at open, minus 1.00, in cents. The ' 'overround the book was opened on, and the variable the ' 'entry gate tests. Blank unless all three legs were ' 'quoted.'), ('pin_sum_open', 'Sum of Pinnacle fair probabilities across the legs. ' 'Should be ~1.00; deviation flags a stale or mismatched ' 'fair.'), ('ask_max_open', 'Highest leg ask at open -- the favourite. Favourites fill ' 'far more often (49.7% vs 17.7% for draws), which is how ' 'a book strands short the leg most likely to win.'), ('ask_min_open', 'Lowest leg ask at open -- the longshot.'), ('spread_c_mean', 'Mean ask-minus-bid across the legs at open, in cents. ' 'Mostly 1c: the venue prices on a 1-cent grid.'), ('queue_ahead_min', 'Smallest queue ahead of us across the legs. The easiest ' 'leg to fill.'), ('queue_ahead_max', 'Largest queue ahead of us. The hardest leg -- and a ' 'book completes only if its WORST leg sells.'), ('book_volume_open', 'Contracts traded across all three legs. The strongest ' 'single completion predictor found so far (+0.091 AUC); ' 'min-leg volume was null (rho +0.095, p=0.189).'), ('leg_volume_min', 'Volume on the thinnest leg. Kept because the paper-era ' 'gate used it -- it did NOT replicate live, so treat a ' 'result here with suspicion.'), ('open_interest_sum', 'Open interest summed across the legs.'), ('n_posts_total', 'Re-posts across every leg. A proxy for how long the book ' 'was worked.'), ('minutes_exposed_max', 'Longest any single leg sat on the book.'), ('is_xlow_any', '1 if the frozen xlow rule fired on this fixture (either ' 'side), else 0. Blank if the fixture could not be resolved.'), ('is_xhigh_any', '1 if the frozen xhigh rule fired on this fixture. xhigh ' 'overlaps xlow heavily -- not an independent feature.'), ('legs_filled', 'OUTCOME. How many legs filled, 0-3, per the EXCHANGE ' 'fill record -- the authority. Maker fills only ' '(post_only, so never taker), which also excludes xlow ' 'taker buys on the same tickers.'), ('legs_filled_log', 'The same count derived from the LOCAL fills log, kept ' 'for comparison. It undercounts: the log joins on ' '(ticker, price_cents) so a re-priced leg misses, and ' 'the log only records while the maker is up. On the ' '2026-08-28 audit the exchange said 115 books complete ' 'and the log-derived figure said 45. A large gap here ' 'means fills are being missed locally.'), ('fill_source', "'exchange' normally; 'log' if the exchange could not be " 'read, in which case legs_filled falls back and will ' 'undercount.'), ('complete', 'OUTCOME. 1 if all three legs filled. This is the number the ' 'strategy lives or dies on -- break-even was 61.4% on Kalshi.'), ('leg_sold_first', "Which OUTCOME sold first, in 1X2 terms: '1' home, " "'2' away, 'X' draw. Blank when no leg filled, or when " "the abbreviation cannot be resolved against the event " "code. Derived from first_fill_side, which carries the " "Kalshi leg abbreviation ('CAT') rather than a side -- " "readable only if you know which team that was. A " "Kalshi event ticker ends in the two team codes " "concatenated (26AUG17GEMCAT = GEM at home to CAT), so " "the abbreviation is matched against that pair: prefix " "means home, suffix means away, TIE means draw."), ('first_fill_side', 'Leg abbreviation of the first leg to fill, blank if ' 'none did.'), ('first_fill_secs', 'Seconds from that leg being posted to it filling.'), # --- added 2026-08-30, after a day of arguments that each had to # --- reconstruct these from raw fills and settlements, and got them wrong ('size', 'Contracts per leg the maker was working, from the largest leg. ' 'SIZE went 1 -> 3 on 2026-08-23 and mixing the two eras is a trap: ' 'a size-1 completion pays ~+$0.05 and a size-3 one ~+$0.20, so an ' 'average across both understates a good book threefold.'), ('shape', "OUTCOME, stricter than `complete`. 'unfilled' = quoted but " "never touched. 'clean' = three legs at the " "SAME size, the only shape that is actually hedged. 'ragged' = " "three legs at different sizes (a leg that took a 0.01-contract " "fill and never finished still reads complete). 'over' = a leg " "exceeded size, which means two orders rested at once. " "'stranded' = fewer than three legs."), ('has_taker', '1 if WE traded this event as a taker -- xlow, or a manual ' 'trade. The maker is post_only so it never takes. Books with ' "this set have P&L that is not the maker own: it is the " 'difference between a -$3.22 maker and a -$53.05 account. ' 'NOT the same as is_xlow_any, which only says the SIGNAL ' 'fired (169 books) rather than that a trade happened (13).'), ('arm', "Which pricing arm owned the book: 'budget' (a part-filled leg " "prices at exactly what the book still owes) or 'latch3' (the " "ratchet survives the fill but sits at most 3c above that budget). " "Blank for books opened before the split. From mm_arm_log.csv."), ('settled', '1 once every leg has settled and pnl is final.'), ('pnl', 'OUTCOME. Realised dollars, from the venue settlement records ' '(revenue - cost - fees) summed across the legs. Blank until ' 'settled. This is the column every P&L question should be answered ' 'from; reconstructing it by hand is where the errors came from.'), ('premium_over_budget_med', 'Median cents a part-filled leg was offered ' 'ABOVE what the book still needed, over the ' "book's life. 0 under the 'budget' arm by " "construction; the old unbounded latch ran a " 'median +1c with a tail to +54c, and that tail ' 'is what stranded books.'), ] _DATE_RE = re.compile(r'^\d{2}[A-Z]{3}\d{2}') def leg_to_1x2(event, leg): """'1' / '2' / 'X' for the leg abbreviation, or '' if it cannot be resolved. A Kalshi event ticker is SERIES-, e.g. KXARGPREMDIVGAME-26AUG17GEMCAT: GEM at home to CAT. So stripping the date leaves the two abbreviations concatenated, and the leg is home if it is the prefix and away if it is the suffix. Returns '' rather than guessing when both or neither match -- a code like 'AAA' vs leg 'A' is genuinely ambiguous, and a wrong side here would be invisible in the data. """ if not leg: return '' leg = str(leg).upper().strip() if leg in ('TIE', 'DRAW', 'X'): return 'X' tail = str(event or '').rsplit('-', 1)[-1].upper() pair = _DATE_RE.sub('', tail) if not pair or leg not in pair: return '' starts, ends = pair.startswith(leg), pair.endswith(leg) if starts and not ends: return '1' if ends and not starts: return '2' return '' OUT_COLS = [c for c, _ in FIELD_DOCS] def _f(v): try: return float(v) except (TypeError, ValueError): return None def exchange_fills(): """event -> {leg: (qty, price)} from Kalshi's OWN fill record. WHY NOT THE LOCAL LOG. `filled` in the leg dataset is a join of mm_live_realfills.csv on (ticker, price_cents) PLUS a time window. Both halves lose fills: * the price key misses a leg that was re-priced between the resting order going on and the fill being recorded; * the local log is itself incomplete -- 354 legs against 391 the exchange reports, because the fill detector only runs while the maker is up. Audited 2026-08-28: the exchange said 115 books completed, the dataset said 45. Completion is the one number this strategy is judged on, so it cannot be derived from a lossy local join. Maker fills only: our orders are post_only, so `is_taker` is always false and the side is never 'yes'. That is the same filter mm_maker_live uses to attribute its own fills, and it excludes xlow's taker buys on the same tickers. Returns None if the exchange cannot be read, so the caller can fall back to the log rather than silently reporting every book as incomplete. """ try: sys.path.insert(0, os.path.join(HERE, '..', 'main')) import kalshi_trader as kt rows = (kt._get_all('/portfolio/fills', 'fills', {'limit': 200}).get('fills') or []) except Exception as e: print(f' [warn] exchange fills unreadable ({type(e).__name__}: {e}); ' f'falling back to the local fills log') return None out = defaultdict(dict) for f in rows: if str(f.get('action', '')).lower() != 'sell' or f.get('is_taker'): continue if str(f.get('side', '')).lower() == 'yes': continue t = f.get('ticker') or '' if '-' not in t: continue ev, leg = t.rsplit('-', 1) q, px = _f(f.get('count_fp')) or 0.0, _f(f.get('yes_price_dollars')) prev_q, prev_px = out[ev].get(leg, (0.0, px)) out[ev][leg] = (prev_q + q, px if px is not None else prev_px) return dict(out) def taker_events(): """Events where WE traded as a taker -- xlow, or a manual trade. The maker is post_only so it never takes. An event with a taker fill has P&L that is not the maker's, and conflating the two is how a -$3.22 maker was reported as losing $53 for most of a day. """ try: sys.path.insert(0, os.path.join(HERE, '..', 'main')) import kalshi_trader as kt # BOTH SUBACCOUNTS. xlow moved to subaccount 1 on 2026-09-01 and this # endpoint defaults to 0, so reading the default alone would find no # taker fills at all -- every xlow event would score has_taker=0, get # counted as maker-only, and put xlow's P&L back on the maker. That is # the exact attribution error this function exists to prevent, just # arriving through a different door. rows = [] for _sub in (0, 1): try: rows += (kt._get_all('/portfolio/fills', 'fills', {'limit': 200, 'subaccount': _sub} ).get('fills') or []) except Exception: continue except Exception: return set() return {(f.get('ticker') or '').rsplit('-', 1)[0] for f in rows if f.get('is_taker') and '-' in (f.get('ticker') or '')} def settled_pnl(): """event -> (realised dollars, all_legs_settled) from the venue's own books. revenue - cost - fees, per leg, summed. This is the authority: every P&L figure should come from here rather than being rebuilt from fill prices, which is where the size-1/size-3 mixing and the taker contamination crept in. """ try: sys.path.insert(0, os.path.join(HERE, '..', 'main')) import kalshi_trader as kt # BOTH SUBACCOUNTS, same reason as taker_events: settlements are # subaccount-scoped and default to 0, so xlow's realised P&L would be # invisible here from 2026-09-01 onward. rows = [] for _sub in (0, 1): try: rows += (kt._get_all('/portfolio/settlements', 'settlements', {'limit': 300, 'subaccount': _sub} ).get('settlements') or []) except Exception: continue except Exception: return {} agg = defaultdict(lambda: [0.0, set()]) for r in rows: t = r.get('ticker') or '' if '-' not in t: continue ev, leg = t.rsplit('-', 1) rev = (_f(r.get('revenue')) or 0.0) / 100.0 cost = (_f(r.get('no_total_cost_dollars')) or 0.0) + (_f(r.get('yes_total_cost_dollars')) or 0.0) a = agg[ev] a[0] += rev - cost - (_f(r.get('fee_cost')) or 0.0) a[1].add(leg) # SETTLED MEANS "EVERY LEG WE HOLD HAS SETTLED", NOT ">= 3 LEGS". # # The >=3 test was survivorship in a single line: a stranded book holds only # one or two legs, so only one or two settle, so it scored settled=0 and fell # out of every `settled == 1` filter -- which is to say every P&L total this # file was used for silently dropped the losers. It reported +$9.40 on # maker-only books where the venue's own settlements say -$4.82. # # The SET of settled legs comes back rather than a count, so the caller can # test it against the legs actually held. Counting is fragile -- it cannot # tell three settled legs from three settlements on two legs. return {ev: (round(v[0], 4), v[1]) for ev, v in agg.items()} def arm_of(): """event -> pricing arm, from the append-only assignment log.""" path = os.path.join(HERE, 'mm_arm_log.csv') if not os.path.exists(path): return {} # 'vol'/'ko' are labels from the retired SELECTION experiment, not pricing # arms. Keeping them would let a stale label win 'first assignment' and # report a book as belonging to an arm that no longer exists. # # 'tieload' ADDED 2026-09-02. It was introduced as the third pricing arm but # never added here, so all 405 of its assignments were relabelled '' and # pooled with the 153 pre-arm books -- making the arm invisible in every # comparison AND quietly contaminating the baseline it was being compared # against. Keep this set in step with ARM_SPEC in mm_maker_live. valid = {'budget', 'latch3', 'tieload'} out = {} for r in _read(path): if r.get('event') and r.get('arm') in valid: out.setdefault(r['event'], r['arm']) # first real assignment wins return out def budget_premium(): """event -> median cents a PART-FILLED leg was offered above its budget. The whole subject of the arm test. Recomputed from the leg log because nothing else records what the book was actually asked to fetch. """ path = os.path.join(HERE, 'mm_live_legs.csv') if not os.path.exists(path): return {} per = defaultdict(list) with open(path, 'rb') as fh: hdr = fh.readline().replace(b'\x00', b'').decode('utf-8', 'replace').strip().split(',') for line in fh: line = line.replace(b'\x00', b'') try: r = dict(zip(hdr, next(csv.reader(io.StringIO( line.decode('utf-8', 'replace')))))) except Exception: continue if r.get('live') != '1': continue col, px = _f(r.get('collected')), _f(r.get('price_cents')) touch, sc = _f(r.get('touch_ask')), _f(r.get('scale')) if None in (col, px, touch, sc) or col <= 0: continue budget = touch * sc if sc > 1.0 else touch per[r.get('event')].append(round((px / 100.0 - budget) * 100)) out = {} for ev, v in per.items(): v.sort() out[ev] = v[len(v) // 2] return out def fill_times(): """event -> earliest maker fill timestamp. Used to place backfilled books.""" try: sys.path.insert(0, os.path.join(HERE, '..', 'main')) import kalshi_trader as kt rows = (kt._get_all('/portfolio/fills', 'fills', {'limit': 200}).get('fills') or []) except Exception: return {} out = {} for f in rows: if str(f.get('action', '')).lower() != 'sell' or f.get('is_taker'): continue t = f.get('ticker') or '' if '-' not in t: continue ev, ts = t.rsplit('-', 1)[0], f.get('created_time') or '' if ev not in out or ts < out[ev]: out[ev] = ts return out def _read(path): raw = open(path, 'rb').read().replace(b'\x00', b'') return list(csv.DictReader(io.StringIO(raw.decode('utf-8', 'replace')))) def polyus_books(): """Poly-US books as rows in the same shape, from its own logs + venue record. FEWER FEATURES ON PURPOSE. pus_live_legs.csv records ts/slug/side/price/ touch/above_touch/gross/collected/need -- it has no volume, queue depth, open interest or Pinnacle fair, because that maker was written to a narrower log. Those columns come out BLANK rather than zero: a model that reads 0 for "queue depth unknown" is being lied to. Completion comes from the VENUE's settlement record, same as the Kalshi side, for the same reason -- the local state stops writing during 503s. """ legs_csv = os.path.join(HERE, 'pus_live_legs.csv') if not os.path.exists(legs_csv): return [] try: sys.path.insert(0, HERE) import pus_maker_pnl as P settled = P.settled_books() or {} except Exception as e: print(f' [warn] poly-US settlement unavailable ({type(e).__name__}: {e})') settled = {} per = defaultdict(lambda: defaultdict(list)) for r in _read(legs_csv): slug = r.get('slug') or '' if '-' not in slug: continue ev, leg = slug.rsplit('-', 1) per[ev][leg].append(r) out = [] for ev, legs in per.items(): firsts = {lg: sorted(v, key=lambda x: x.get('ts', ''))[0] for lg, v in legs.items()} asks = [_f(r.get('touch_ask')) for r in firsts.values()] asks = [a for a in asks if a is not None] gross = ((sum(asks) - 1.0) * 100.0) if len(asks) == 3 else None filled = list((settled.get(ev) or {}).keys()) spreads = [(_f(r.get('touch_ask')) or 0) - (_f(r.get('touch_bid')) or 0) for r in firsts.values()] out.append({ 'venue': 'polyus', 'event': ev, 'league': (firsts[next(iter(firsts))].get('event') or '')[:24], 'ts_open': min(r.get('ts', '') for v in legs.values() for r in v), 'hrs_to_ko_open': None, 'legs_quoted': len(firsts), 'gross_c_open': f'{gross:.2f}' if gross is not None else '', 'pin_sum_open': '', 'ask_max_open': max(asks) if asks else None, 'ask_min_open': min(asks) if asks else None, 'spread_c_mean': (f'{100*sum(spreads)/len(spreads):.2f}' if spreads else ''), 'queue_ahead_min': None, 'queue_ahead_max': None, 'book_volume_open': None, 'leg_volume_min': None, 'open_interest_sum': None, 'n_posts_total': sum(len(v) for v in legs.values()), 'minutes_exposed_max': None, 'is_xlow_any': '', 'is_xhigh_any': '', 'legs_filled': len(filled), 'legs_filled_log': len(filled), 'fill_source': 'exchange' if settled else 'log', 'complete': 1 if len(filled) >= 3 else 0, 'leg_sold_first': '', 'first_fill_side': '', 'first_fill_secs': '', }) return out def main(rebuild=True): """rebuild=False when the caller has ALREADY refreshed the leg dataset. The dashboard rebuilds mm_fill_dataset itself before embedding, so letting this re-run it as a subprocess would parse ~350k log rows twice on every page build for the same answer. """ if rebuild: if not os.path.exists(LEGS_DS): print('leg dataset missing; building it first') subprocess.run([sys.executable, os.path.join(HERE, 'mm_fill_dataset.py')], capture_output=True, text=True, timeout=1800) ex = exchange_fills() tak = taker_events() stl = settled_pnl() arms = arm_of() prem = budget_premium() rows = _read(LEGS_DS) books = defaultdict(list) for r in rows: if r.get('event'): books[r['event']].append(r) out = [] for ev, legs in books.items(): # ONE ENTRY PER LEG, taken at its FIRST resting order. A leg can appear # several times (re-priced at a new level after the budget rule moved), # and the later ones are already downstream of what happened to the book. by_leg = {} for r in sorted(legs, key=lambda x: x.get('ts_first', '')): by_leg.setdefault(r.get('side', ''), []).append(r) firsts = {s: v[0] for s, v in by_leg.items()} def col(name, agg): vals = [_f(r.get(name)) for r in firsts.values()] vals = [v for v in vals if v is not None] return agg(vals) if vals else None n_legs = len(firsts) asks = [_f(r.get('ask_at_post')) for r in firsts.values()] asks = [a for a in asks if a is not None] gross = ((sum(asks) - 1.0) * 100.0) if n_legs == 3 and len(asks) == 3 else None # OUTCOME FROM THE EXCHANGE where we can read it, from the local log # only as a fallback. Both are kept so the gap stays visible instead of # being smoothed away -- they disagreed on 70 of 288 books when this was # first audited. log_filled = [s for s, v in by_leg.items() if any(x.get('filled') == '1' for x in v)] if ex is None: filled_legs, src = log_filled, 'log' else: filled_legs, src = list((ex.get(ev) or {}).keys()), 'exchange' # first fill across the whole book ff = None for r in sorted(legs, key=lambda x: x.get('ts_first', '')): if r.get('filled') == '1' and _f(r.get('seconds_to_fill')) is not None: ff = r break def anyflag(name): vals = [r.get(name) for r in firsts.values()] if all(v == '' for v in vals): return '' return 1 if any(v == '1' for v in vals) else 0 # size and shape from the EXCHANGE quantities: `complete` counts legs # touched, which calls a book with a 0.01-contract leg finished. _held = set((ex or {}).get(ev, {})) # leg abbrevs we actually hold _qty = sorted(q for q, _ in (ex or {}).get(ev, {}).values()) if ex else [] # INTENDED size is the MEDIAN leg, not the max. Comparing each leg # against the max makes an over-fill undetectable by construction: # CHIFOR ran 9/3/3 and read 'clean' until this was fixed. _size = _qty[len(_qty) // 2] if _qty else 0.0 if not _qty: _shape = 'unfilled' # quoted, never touched elif len(_qty) < 3: _shape = 'stranded' # 1-of-3 or 2-of-3, real exposure elif any(q > _size + 1e-6 for q in _qty): _shape = 'over' # two orders rested at once elif _qty[-1] - _qty[0] > 1e-6: _shape = 'ragged' # three legs, uneven, not hedged else: _shape = 'clean' ts_open = min(r.get('ts_first', '') for r in legs) out.append({ 'venue': 'kalshi', 'event': ev, 'league': next(iter(firsts.values())).get('league', ''), 'ts_open': ts_open, 'hrs_to_ko_open': col('hrs_to_ko', max), 'legs_quoted': n_legs, 'gross_c_open': f'{gross:.2f}' if gross is not None else '', 'pin_sum_open': (f"{sum(x for x in [_f(r.get('pin_fair')) for r in firsts.values()] if x is not None):.4f}" if n_legs == 3 else ''), 'ask_max_open': col('ask_at_post', max), 'ask_min_open': col('ask_at_post', min), 'spread_c_mean': (f"{col('spread_c', lambda v: sum(v)/len(v)):.2f}" if col('spread_c', lambda v: sum(v)/len(v)) is not None else ''), 'queue_ahead_min': col('queue_ahead', min), 'queue_ahead_max': col('queue_ahead', max), 'book_volume_open': col('book_volume', max), 'leg_volume_min': col('volume', min), 'open_interest_sum': col('open_interest', sum), 'n_posts_total': sum(int(_f(r.get('n_posts')) or 0) for r in legs), 'minutes_exposed_max': col('minutes_exposed', max), 'is_xlow_any': anyflag('is_xlow'), 'is_xhigh_any': anyflag('is_xhigh'), 'legs_filled': len(filled_legs), 'legs_filled_log': len(log_filled), 'fill_source': src, 'complete': 1 if len(filled_legs) >= 3 else 0, 'leg_sold_first': leg_to_1x2(ev, ff.get('side') if ff else ''), 'first_fill_side': ff.get('side', '') if ff else '', 'first_fill_secs': ff.get('seconds_to_fill', '') if ff else '', 'size': (f'{_size:g}' if _size else ''), 'shape': _shape, 'has_taker': 1 if ev in tak else 0, 'arm': arms.get(ev, ''), # settled when every leg we HOLD has settled. `legs` here is the # QUOTED legs, which is the wrong denominator -- a 2-leg strand is # fully settled at 2. _held is the exchange's record of what we # actually hold, and comparing SETS beats comparing counts. 'settled': (1 if (ev in stl and _held and _held <= stl[ev][1]) else (0 if ev in stl else '')), 'pnl': (f'{stl[ev][0]:.4f}' if ev in stl else ''), 'premium_over_budget_med': prem.get(ev, ''), }) # BACKFILL BOOKS THE LEG LOG NEVER SAW. # # Everything above is built from mm_live_legs.csv, which starts # 2026-08-16T13:44. Books the maker traded before that -- and any book whose # legs went unlogged during a restart -- have real fills and real # settlements and were simply absent, so every P&L total computed off this # file quietly excluded them. 29 fully settled books were missing when this # was written. # # They come in with the market-data columns BLANK, because none of it can be # reconstructed after the fact: no touch, no queue depth, no volume at open. # The outcome columns are exact -- they come from the exchange. So these # rows are usable for P&L and completion and must be excluded from any # feature analysis, which the blank ts_open makes obvious. _seen = {r['event'] for r in out} _ft = fill_times() for ev, legs in (ex or {}).items(): if ev in _seen or not legs: continue cs = sorted(q for q, _ in legs.values()) sz = cs[len(cs) // 2] if cs else 0.0 if not cs: shape = 'unfilled' elif len(cs) < 3: shape = 'stranded' elif any(q > sz + 1e-6 for q in cs): shape = 'over' elif cs[-1] - cs[0] > 1e-6: shape = 'ragged' else: shape = 'clean' out.append({ 'venue': 'kalshi', 'event': ev, 'league': '', 'ts_open': _ft.get(ev, ''), 'legs_quoted': len(legs), 'legs_filled': len(legs), 'legs_filled_log': '', 'fill_source': 'exchange (backfilled, no leg log)', 'complete': 1 if len(legs) >= 3 else 0, 'size': (f'{sz:g}' if sz else ''), 'shape': shape, 'has_taker': 1 if ev in tak else 0, 'arm': arms.get(ev, ''), # here `legs` IS the exchange fill record, so its keys are the # legs we hold. Every one of them must have settled. 'settled': (1 if (ev in stl and set(legs) <= stl[ev][1]) else (0 if ev in stl else '')), 'pnl': (f'{stl[ev][0]:.4f}' if ev in stl else ''), 'premium_over_budget_med': prem.get(ev, ''), }) out.extend(polyus_books()) out.sort(key=lambda r: (r['ts_open'] or '')) with open(OUT, 'w', newline='', encoding='utf-8') as fh: w = csv.DictWriter(fh, fieldnames=OUT_COLS) w.writeheader() for r in out: w.writerow({k: ('' if r.get(k) is None else r.get(k)) for k in OUT_COLS}) full = [r for r in out if r['legs_quoted'] == 3] comp = [r for r in full if r['complete']] print(f'wrote {OUT}') print(f' books : {len(out)}') print(f' full 3-leg books : {len(full)}') print(f' completed : {len(comp)}' + (f' ({100*len(comp)/len(full):.1f}%)' if full else '')) print(f' size : {os.path.getsize(OUT)/1e6:.2f} MB') return 0 if __name__ == '__main__': sys.exit(main())