"""
Do Kalshi markets converge to Pinnacle's (margin-removed) fair price as kickoff
nears, and how fast? Uses live_data/*.csv PRE snapshots. 'elapsed' (negative) =
minutes to kickoff. gap = pin_fair - kalshi_mid (signed prob); |gap| = mispricing.

Outputs:
 1) mean |gap| and mean signed gap bucketed by time-to-kickoff -> convergence curve
 2) attribution: from each fixture/side's EARLIEST PRE snapshot to its LATEST,
    does Kalshi move toward early-Pinnacle (catch-up) and how much does Pinnacle move?
"""
import csv, glob, io, smtplib, sys
from collections import defaultdict
from pathlib import Path
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

sys.stdout.reconfigure(encoding='utf-8', errors='replace')
files = sorted(glob.glob(r'E:\soccer\live_data\*.csv'))

def fval(r, k):
    try: return float(r.get(k, ''))
    except (TypeError, ValueError): return None

# per (date,home,away,side) -> list of (mins_to_ko, pinfair, kalmid)
series = defaultdict(list)
for fp in files:
    with open(fp, newline='', encoding='utf-8', errors='replace') as f:
        for r in csv.DictReader(f):
            if r.get('status') != 'PRE':
                continue
            el = fval(r, 'elapsed')
            if el is None or el >= 0:
                continue
            mins = -el
            for side in ('home', 'away'):
                pf = fval(r, f'pin_fair_{side}')
                km = fval(r, f'kalshi_{side}_mid')
                if pf and km and 0 < pf < 1 and 0 < km < 1:
                    series[(r['game_date'], r['home'], r['away'], side)].append((mins, pf, km))

for v in series.values():
    v.sort()

# ---- 1) convergence curve by time-to-kickoff ----
buckets = [(0,60),(60,120),(120,180),(180,360),(360,720),(720,1440),(1440,99999)]
blab = ['<1h','1-2h','2-3h','3-6h','6-12h','12-24h','>24h']
agg = {b: {'n':0,'absgap':0.0,'sgap':0.0} for b in buckets}
for v in series.values():
    for mins, pf, km in v:
        for b in buckets:
            if b[0] <= mins < b[1]:
                a = agg[b]; a['n']+=1
                a['absgap'] += abs(pf-km); a['sgap'] += (pf-km)
                break

print("=== Mean Kalshi-vs-Pinnacle gap by time to kickoff ===")
print(f"{'window':>8}{'N':>8}{'mean|gap|':>11}{'mean gap':>11}  (gap=pin_fair - kalshi_mid, prob pts)")
for b,lab in zip(buckets,blab):
    a=agg[b]
    if a['n']:
        print(f"{lab:>8}{a['n']:>8}{100*a['absgap']/a['n']:>10.2f}c{100*a['sgap']/a['n']:>+10.2f}c")

# ---- 2) attribution: earliest -> latest PRE snapshot per fixture/side ----
n_fix=0; closed=0; widened=0
abs0_sum=abs1_sum=pinmove_sum=kalmove_sum=0.0
kal_toward=0; pin_toward=0
for v in series.values():
    if len(v) < 2: continue
    # earliest = max mins (furthest from ko), latest = min mins (closest)
    (m0,pf0,km0) = v[-1]   # largest mins (sorted asc) -> furthest out
    (m1,pf1,km1) = v[0]    # smallest mins -> closest to ko
    if m0 <= m1: continue
    n_fix+=1
    gap0=abs(pf0-km0); gap1=abs(pf1-km1)
    abs0_sum+=gap0; abs1_sum+=gap1
    if gap1 < gap0: closed+=1
    elif gap1 > gap0: widened+=1
    pinmove_sum += abs(pf1-pf0)
    kalmove_sum += abs(km1-km0)
    # did kalshi move toward the early pinnacle anchor?
    if abs(km1-pf0) < abs(km0-pf0): kal_toward+=1
    # did pinnacle move toward the early kalshi?
    if abs(pf1-km0) < abs(pf0-km0): pin_toward+=1

print(f"\n=== Attribution (earliest vs latest PRE snapshot, {n_fix} fixture-sides) ===")
print(f"mean |gap| earliest : {100*abs0_sum/n_fix:.2f}c")
print(f"mean |gap| latest   : {100*abs1_sum/n_fix:.2f}c   "
      f"({100*(1-abs1_sum/abs0_sum):+.0f}% change)")
print(f"gap CLOSED in {100*closed/n_fix:.0f}% of fixtures, widened in {100*widened/n_fix:.0f}%")
print(f"mean Kalshi mid move : {100*kalmove_sum/n_fix:.2f}c")
print(f"mean Pinnacle fair move: {100*pinmove_sum/n_fix:.2f}c")
print(f"Kalshi moved toward early-Pinnacle: {100*kal_toward/n_fix:.0f}% of fixtures")
print(f"Pinnacle moved toward early-Kalshi: {100*pin_toward/n_fix:.0f}% of fixtures")

# ---- chart ----
xs=[]; ys=[]
for b,lab in zip(buckets,blab):
    a=agg[b]
    if a['n']:
        mid=(b[0]+b[1])/2/60 if b[1]<99999 else 30
        xs.append(mid); ys.append(100*a['absgap']/a['n'])
fig,ax=plt.subplots(figsize=(10,5.5))
ax.plot(xs,ys,'o-',color='#d62728',lw=2)
ax.invert_xaxis()
ax.set_xlabel('Hours to kickoff (later -> right to left)'); ax.set_ylabel('Mean |gap| (cents)')
ax.set_title('Kalshi vs Pinnacle mispricing as kickoff approaches')
ax.grid(True,alpha=0.3); fig.tight_layout()
buf=io.BytesIO(); fig.savefig(buf,format='png',dpi=120); plt.close(fig)
Path(r'E:\soccer\_kalshi_convergence.png').write_bytes(buf.getvalue())
print("\nchart saved: _kalshi_convergence.png")