"""Is the volatility effect just illiquidity wearing a disguise?

Volatile legs fill 20.5% against 7.4% for calm ones (mm_volatility_clean.py).
But the leagues that top the volatility table -- CN1, NWSL, CAN, PE1 -- are
plausibly the THIN ones, and a thin book moves further per trade simply because
less volume is needed to shift it. Volume independently predicts fills at AUC
~0.66, and a book nobody trades cannot complete however much it swings.

So the +13pp may be measuring books that are cheap to move and impossible to
finish. This splits the two:

    volatility  from live_data, measured strictly BEFORE the first quote
    volume      from the Kalshi market at post time (volume_24h / open interest)
    fill        real trade prints, queue-aware

and reports the 2x2. If volatility only works in the low-volume cell, it is
illiquidity and worthless. If it works within BOTH volume halves, it is a real
and separable signal.
"""
import csv, collections, json, os, statistics, sys

sys.path.insert(0, r'E:\soccer\main')
sys.path.insert(0, r'E:\soccer\research')
import mm_settle as S
from mm_volatility_clean import load_legs, load_movement, CACHE


def _f(x):
    try:
        return float(x)
    except (TypeError, ValueError):
        return None


def load_volume():
    """{leg_ticker: (volume_24h, open_interest, volume)} at its first quote."""
    out = {}
    with open(r'E:\soccer\research\mm_paper_quotes.csv',
              encoding='utf-8', errors='replace') as fh:
        for r in csv.DictReader(fh):
            t = r.get('leg_ticker')
            if not t or t in out:
                continue
            if not (r.get('ask_size') or '').strip():
                continue
            out[t] = {'v24': _f(r.get('volume_24h')),
                      'oi': _f(r.get('open_interest')),
                      'vol': _f(r.get('volume'))}
    return out


def cell(rows, label):
    if not rows:
        return f'{label:>18}      -        -'
    f = sum(r['filled'] for r in rows)
    return (f'{label:>18} {len(rows):>5} legs {f:>4} filled '
            f'{100.0*f/len(rows):>6.1f}%')


def main():
    legs = load_legs()
    mv = load_movement(legs)
    vols = load_volume()
    trades = json.load(open(CACHE, encoding='utf-8'))

    rows = []
    for t, v in legs.items():
        if t not in mv or t not in vols:
            continue
        key = f"{t}|{v['ts'].isoformat()}"
        if key not in trades:
            continue
        vv = vols[t]
        liq = vv['v24'] if vv['v24'] is not None else vv['vol']
        if liq is None:
            continue
        cum, filled = 0.0, False
        for ts_s, p, c in trades[key]:
            ts = S._pt(ts_s)
            if not ts:
                break
            ko = v['ko']
            if (ts.tzinfo is None) != (ko.tzinfo is None):
                ts = ts.astimezone() if ts.tzinfo else ts.replace(tzinfo=ko.tzinfo)
            if ts >= ko:
                break
            if p >= v['our_ask'] - 1e-9:
                cum += c
                if cum > v['queue']:
                    filled = True
                    break
        rows.append({'sd': mv[t]['sd'], 'liq': liq, 'filled': int(filled),
                     'oi': vv['oi']})

    if len(rows) < 80:
        print(f'only {len(rows)} legs join -- too few')
        return
    base = 100.0 * sum(r['filled'] for r in rows) / len(rows)
    print(f'{len(rows)} legs with movement, volume AND queue-aware fills; '
          f'base fill {base:.1f}%\n')

    # are the two even correlated? if strongly, they are the same thing
    n = len(rows)
    rs = sorted(range(n), key=lambda i: rows[i]['sd'])
    rk_sd = {i: k for k, i in enumerate(rs)}
    rs2 = sorted(range(n), key=lambda i: rows[i]['liq'])
    rk_lq = {i: k for k, i in enumerate(rs2)}
    d2 = sum((rk_sd[i] - rk_lq[i]) ** 2 for i in range(n))
    rho = 1 - 6 * d2 / (n * (n * n - 1))
    print(f'Spearman rho(volatility, volume) = {rho:+.3f}')
    print('  strongly NEGATIVE would mean volatile == thin, i.e. the same thing.')
    print(f'  {abs(rho):.2f} is '
          f'{"strong -- they are largely one variable" if abs(rho) > 0.5 else "moderate" if abs(rho) > 0.3 else "weak -- they are separable"}\n')

    sd_med = statistics.median(r['sd'] for r in rows)
    lq_med = statistics.median(r['liq'] for r in rows)
    print(f'splitting at median volatility {sd_med*100:.2f}c and '
          f'median 24h volume {lq_med:,.0f}\n')

    print('=== the 2x2 ===')
    quad = {}
    for vlab, vsel in (('calm', lambda r: r['sd'] <= sd_med),
                       ('VOLATILE', lambda r: r['sd'] > sd_med)):
        for llab, lsel in (('thin', lambda r: r['liq'] <= lq_med),
                           ('LIQUID', lambda r: r['liq'] > lq_med)):
            sel = [r for r in rows if vsel(r) and lsel(r)]
            quad[(vlab, llab)] = sel
            print(cell(sel, f'{vlab} + {llab}'))

    print('\n=== does volatility still help WITHIN each volume half? ===')
    for llab in ('thin', 'LIQUID'):
        c, v = quad[('calm', llab)], quad[('VOLATILE', llab)]
        if len(c) < 12 or len(v) < 12:
            continue
        rc = 100.0 * sum(r['filled'] for r in c) / len(c)
        rv = 100.0 * sum(r['filled'] for r in v) / len(v)
        print(f'  {llab:>7}: calm {rc:>5.1f}%  ->  volatile {rv:>5.1f}%   '
              f'{rv-rc:+.1f}pp')
    print('\n=== and does volume still help WITHIN each volatility half? ===')
    for vlab in ('calm', 'VOLATILE'):
        t_, l_ = quad[(vlab, 'thin')], quad[(vlab, 'LIQUID')]
        if len(t_) < 12 or len(l_) < 12:
            continue
        rt = 100.0 * sum(r['filled'] for r in t_) / len(t_)
        rl = 100.0 * sum(r['filled'] for r in l_) / len(l_)
        print(f'  {vlab:>8}: thin {rt:>5.1f}%  ->  liquid {rl:>5.1f}%   '
              f'{rl-rt:+.1f}pp')


if __name__ == '__main__':
    main()