"""Build a modelling-ready view of mm_book_dataset.csv.

WHY THIS EXISTS
---------------
mm_book_dataset.csv is an audit log: one row per book, every field the maker
saw or produced. That makes it wrong to hand straight to a regression, for
three reasons this script fixes:

  1. STRUCTURAL CONSTANTS. legs_quoted is 3 by construction (a soccer book has
     three legs) and pin_sum_open is ~1.0 by construction (it is a sum of
     probabilities). They are data-quality flags, not features. Left in a
     design matrix with an intercept they make it singular.

  2. POST-OUTCOME COLUMNS. leg_sold_first / first_fill_* / shape / fill_source
     are only knowable AFTER legs start filling. Predicting `complete` from
     them leaks the answer (it inflated pseudo-R2 from 0.03 to 0.71).

  3. THE NEGATIVE CLASS. Filtering to legs_filled > 0 removes every book that
     never filled -- 350 of 517 rows -- which is precisely the outcome a fill
     model is supposed to predict.

WHAT IT KEEPS
-------------
venue == 'kalshi'    polyus rows have no kalshi microstructure fields at all.
legs_quoted == 3     the documented filter for well-formed books (see the
                     WATCH OUT note at the top of mm_book_dataset.py).
all legs_filled      including 0. This is the point.

Targets are emitted alongside the features but never inside them:
  complete (0/1), legs_filled (0-3), pnl (settled books only).
"""
import pandas as pd

SRC = 'research/mm_book_dataset.csv'
OUT = 'research/mm_model_dataset.csv'

# Known at post time -- legitimate features.
FEATURES = [
    'hrs_to_ko_open', 'gross_c_open', 'ask_max_open', 'ask_min_open',
    'spread_c_mean', 'queue_ahead_min', 'queue_ahead_max', 'book_volume_open',
    'leg_volume_min', 'open_interest_sum', 'is_xlow_any', 'size', 'arm', 'league',
]
# Only knowable after the fact -- targets and diagnostics, never predictors.
TARGETS = ['complete', 'legs_filled', 'pnl', 'shape', 'settled', 'has_taker']
# Structural constants and retired fields.
DROPPED = {
    'legs_quoted': 'constant (3) by construction after the legs_quoted==3 filter',
    'pin_sum_open': 'sum of probabilities, ~1.0 by construction',
    'is_xhigh_any': 'xhigh strategy retired; stale',
    'n_posts_total': 'post-outcome: counts reposts across the book life',
    'minutes_exposed_max': 'post-outcome: total time the quote lived',
    'leg_sold_first': 'post-outcome: leaks which leg filled first',
    'first_fill_side': 'post-outcome', 'first_fill_secs': 'post-outcome',
    'legs_filled_log': 'post-outcome', 'fill_source': 'post-outcome',
    'premium_over_budget_med': 'post-outcome',
}

def build():
    df = pd.read_csv(SRC)
    n0 = len(df)
    df = df[df['venue'] == 'kalshi']
    n1 = len(df)
    df = df[df['legs_quoted'] == 3]
    n2 = len(df)
    df = df.dropna(subset=[c for c in FEATURES if c not in ('arm', 'league', 'size')])
    n3 = len(df)

    df['arm'] = df['arm'].fillna('none').replace({'nan': 'none', '': 'none'})
    df['size'] = pd.to_numeric(df['size'], errors='coerce').fillna(1)
    df['ts_open'] = df['ts_open']

    out = df[['event', 'ts_open'] + FEATURES + TARGETS].copy()
    out.to_csv(OUT, index=False)

    print('rows: %d start -> %d kalshi -> %d legs_quoted==3 -> %d with microstructure'
          % (n0, n1, n2, n3))
    print('wrote %s  (%d rows, %d cols)' % (OUT, len(out), out.shape[1]))
    print('\ndropped columns and why:')
    for k, v in DROPPED.items():
        print('   %-24s %s' % (k, v))
    print('\ntarget balance:')
    print('   complete      :', out['complete'].value_counts().sort_index().to_dict())
    print('   legs_filled   :', out['legs_filled'].value_counts().sort_index().to_dict())
    print('   pnl non-null  :', int(out['pnl'].notna().sum()))
    return out

if __name__ == '__main__':
    build()