"""
PILOT (read-only research): backfill Polymarket historical prices into one live_data
day, matched by timestamp. Proves feasibility + alignment quality before a full run.

Polymarket exposes per-market price history via the CLOB endpoint
    GET https://clob.polymarket.com/prices-history?market={yesTokenId}&startTs&endTs&fidelity
returning [{t: unix_utc, p: price}] (price = midpoint; NO separate bid/ask exists
historically -- that's the one backfill limitation). We:
  1. group the live_data day into fixtures (covered leagues only),
  2. find each fixture's Polymarket CLOSED event by name+date, read the 3 markets'
     clobTokenIds (home/away/draw YES tokens),
  3. pull each token's price history over the snapshot window,
  4. nearest-timestamp match every snapshot row -> poly_home/draw/away_price.
NOTHING is traded. Writes a *_polybackfill.csv copy; originals untouched.
"""
import csv, json, re, sys, time, bisect
from datetime import datetime, timezone
import requests
sys.path.insert(0, r"E:\soccer\main")
import polymarket_live as P

sys.stdout.reconfigure(encoding='utf-8', errors='replace')

LIVE_FILE = r"E:\soccer\live_data\2026-04-25.csv"
# write OUTSIDE live_data/ so forward_test's glob('live_data/*.csv') never ingests it
OUT_FILE  = r"E:\soccer\poly_backfill\2026-04-25_polybackfill.csv"
COVERED   = {c for c, t in P._LEAGUE_TAG_BY_CODE.items() if t}
MATCH_TOL_MIN = 90          # max minutes between a snapshot and the nearest poly point

_TS_FMTS = ['%Y-%m-%d %H:%M:%S', '%m/%d/%Y %H:%M', '%m/%d/%Y %H:%M:%S', '%Y-%m-%d %H:%M']
def parse_ts(s):
    for f in _TS_FMTS:
        try: return datetime.strptime(s, f)
        except (ValueError, TypeError): pass
    return None

# ---- robust name match: distinctive-token subset -------------------------------
# Polymarket spells clubs out in full ('Bologna FC 1909', 'Hull City AFC', 'AS Roma',
# 'Olympique Lyonnais'), while our live_data uses short forms ('Bologna','Hull','Roma',
# 'Lyon'). Whole-string substring fails on the suffixes and on <5-char names. Instead we
# tokenize, drop generic filler/suffix tokens + year numbers, and require the smaller
# distinctive-token set to be contained in the larger (token a~b if equal, or one is a
# >=4-char substring of the other -> 'lyon' matches 'lyonnais'). The joint home+away+
# date+moneyline constraint makes this safe from cross-team false hits.
_FILLER = {'fc','sc','afc','cf','bsc','sk','sd','rc','ac','as','aj','ss','us','ssd','sv',
           'club','city','county','calcio','sport','lisboa','sl','rcd','cp',
           'de','e','la','el','los','del','fk','if','bk','cd','ud','ca','sco','do'}
# user-side acronym/nickname -> the distinctive token Polymarket actually uses
_ALIAS = {'psg': 'germain', 'qpr': 'rangers', 'guimaraes': 'vitoria', 'bilbao': 'athletic',
          'wolves': 'wolverhampton', 'rennes': 'rennais', 'nuremberg': 'nurnberg'}
# token rewrites applied before filtering: 'utd'->'united' (Sheffield/Dundee Utd),
# 'ii'->'b' (Spanish reserve sides Polymarket spells '... de Futbol B'), 'saint'->'st'.
_TOKMAP = {'saint': 'st', 'utd': 'united', 'ii': 'b'}
# whole-name repairs for live_data values truncated to 13 chars by an upstream writer
# ('Los Angeles Galaxy'->'Los Angeles G', 'Los Angeles FC'->'Los Angeles F'). Keyed on the
# normalized full string so we restore the dropped suffix token before tokenizing.
_NAME_FIX = {'los angeles g': 'los angeles galaxy', 'los angeles f': 'los angeles fc'}
# Nordic/Icelandic/Polish letters that do NOT NFKD-decompose, so P._norm's ascii-ignore
# silently DROPS them ('Lillestrom' from 'Lillestrøm', 'Bod' from 'Bodø'), breaking the
# Norway/Sweden name match. Transliterate them to ASCII BEFORE P._norm sees the name.
_TRANSLIT = str.maketrans({'ø': 'o', 'Ø': 'o', 'æ': 'ae', 'Æ': 'ae', 'ð': 'd', 'Ð': 'd',
                           'þ': 'th', 'Þ': 'th', 'ł': 'l', 'Ł': 'l', 'œ': 'oe', 'Œ': 'oe',
                           'å': 'a', 'Å': 'a', 'ß': 'ss'})

def _sig_tokens(name):
    # coalesce runs of single-char tokens first: 'D.C.'->'d c'->'dc' so it matches 'DC'
    s = P._norm((name or '').translate(_TRANSLIT))
    raw = _NAME_FIX.get(s, s).split()
    toks, buf = [], ''
    for t in raw:
        if len(t) == 1:
            buf += t
            continue
        if buf:
            toks.append(buf); buf = ''
        toks.append(t)
    if buf:
        toks.append(buf)
    out = []
    for t in toks:
        t = _TOKMAP.get(t, t)
        t = _ALIAS.get(t, t)
        if t in _FILLER or t.isdigit():
            continue
        out.append(t)
    return out

def _tok_eq(a, b):
    return a == b or (min(len(a), len(b)) >= 4 and (a in b or b in a))

def _match_score(a, b):
    """0 if a,b are not the same club; else the contained-token count (= specificity).
    Lets event_tokens assign 'Paris Saint-Germain' to PSG (score 3) over 'Paris FC'
    (score 1) when both share the 'paris' token."""
    sa, sb = _sig_tokens(a), _sig_tokens(b)
    if not sa or not sb:
        return 0
    small, big = (sa, sb) if len(sa) <= len(sb) else (sb, sa)
    if not all(any(_tok_eq(s, u) for u in big) for s in small):
        return 0
    return len(small)

def name_match(a, b):
    return _match_score(a, b) > 0

def _excess(who, side):
    """# of sig-tokens in `side` NOT covered by any token of `who` (lower = more exact).
    Tie-breaks same-name ambiguity ('Dundee' fits BOTH 'Dundee United' and 'Dundee FC';
    'New York City FC'->{new,york} fits BOTH NYCFC and 'New York Red Bulls')."""
    sw, ss = _sig_tokens(who), _sig_tokens(side)
    return sum(0 if any(_tok_eq(s, w) for w in sw) else 1 for s in ss)

# ---- closed-event fetch: full per-tag pull, then client-filter by eventDate -----
# We must NOT filter via Gamma's end_date_min/max: many BASE moneyline events carry a
# stale/garbage endDate (e.g. 'Aalesund vs Brann', game 2026-05-20, has endDate
# 2026-05-10 -- 10 days early), while only their prop sub-events get the correct endDate.
# A resolution-date window therefore silently drops whole leagues (Norway/Sweden looked
# like 'props-only gaps' but the games were there). Instead we fetch a tag's ENTIRE closed
# set once (paginated to a high cap, DEDUPED by id -> stable: premier-league = 1389 every
# run) and client-filter by the reliable eventDate field (_event_date) within +-2 days.
from datetime import timedelta
_tag_closed = {}    # tag -> deduped list of all closed events (fetched once per run)
def _all_closed(tag):
    if tag in _tag_closed:
        return _tag_closed[tag]
    seen, off = {}, 0
    while True:
        try:
            r = requests.get(f"{P.GAMMA}/events",
                             params={"closed": "true", "tag_slug": tag,
                                     "limit": 100, "offset": off}, timeout=25)
        except requests.RequestException:
            break
        if not r.ok: break
        b = r.json()
        if not b: break
        for e in b:
            seen[e.get('id')] = e
        off += 100
        if len(b) < 100 or off >= 3000: break
    _tag_closed[tag] = list(seen.values())
    return _tag_closed[tag]

# The generic 'soccer' umbrella tag (used by leagues with NO dedicated tag, e.g. CL1
# Chile — its games carry only ['games','sports','soccer']) is far too large to paginate
# fully (overflows the off>=3000 cap, so mid-window games never load). For such tags we
# fetch a server-side date WINDOW instead. end_date can be stale/early on some base events
# (see the warning above), so we pad the window wide (±12d) and still client-filter by the
# reliable eventDate ±2d in events_for_day. The window keeps the result set small (<1000).
_BIG_TAGS = {'soccer'}
_win_cache = {}
def _windowed_closed(tag, game_date, pad=12):
    key = (tag, game_date)
    if key in _win_cache: return _win_cache[key]
    try:
        d = datetime.strptime(game_date, '%Y-%m-%d')
    except (ValueError, TypeError):
        _win_cache[key] = []; return []
    lo = (d - timedelta(days=pad)).strftime('%Y-%m-%d')
    hi = (d + timedelta(days=pad)).strftime('%Y-%m-%d')
    seen, off = {}, 0
    while True:
        try:
            r = requests.get(f"{P.GAMMA}/events",
                             params={"closed": "true", "tag_slug": tag,
                                     "end_date_min": lo, "end_date_max": hi,
                                     "limit": 100, "offset": off}, timeout=25)
        except requests.RequestException:
            break
        if not r.ok: break
        b = r.json()
        if not b: break
        for e in b:
            seen[e.get('id')] = e
        off += 100
        if len(b) < 100 or off >= 1000: break
    _win_cache[key] = list(seen.values())
    return _win_cache[key]

_byday = {}
def events_for_day(tag, game_date):
    key = (tag, game_date)
    if key in _byday: return _byday[key]
    try:
        d = datetime.strptime(game_date, '%Y-%m-%d')
    except (ValueError, TypeError):
        _byday[key] = []; return []
    out = []
    src = _windowed_closed(tag, game_date) if tag in _BIG_TAGS else _all_closed(tag)
    for e in src:
        ed = P._event_date(e)
        if not ed:
            continue
        try:
            if abs((datetime.strptime(ed, '%Y-%m-%d') - d).days) <= 2:
                out.append(e)
        except (ValueError, TypeError):
            pass
    _byday[key] = out
    return out

def event_tokens(ev, home, away):
    """{label: yesTokenId} for home/draw/away markets of a matched event.
    Only the base moneyline markets match the win/beat/draw question patterns, so
    prop sub-events ('- Halftime Result' etc.) yield {} and are naturally rejected."""
    toks = {}
    for m in (ev.get('markets') or []):
        ql = (m.get('question', '') or '').lower()
        tok = m.get('clobTokenIds')
        if not tok: continue
        yes = (json.loads(tok) if isinstance(tok, str) else tok)[0]
        if 'end in a draw' in ql:
            toks['draw'] = yes; continue
        mm = re.search(r'will (.+?) (?:beat|win on)', ql)
        if not mm: continue
        who = mm.group(1)
        # assign to the side 'who' matches MORE specifically. Primary: contained-token
        # count (so 'Paris Saint-Germain'(3) beats 'Paris FC'(1) on shared 'paris'). On a
        # TIE (who fits both sides equally -> same-name pairs like 'Dundee United v Dundee
        # FC', 'NY Red Bulls v NY City FC'), break by exactness: fewest leftover tokens on
        # the side wins (who={new,york} fits NYCFC exactly, Red Bulls with 2 extras).
        sh, sa = _match_score(who, home), _match_score(who, away)
        if sh and not sa:   toks['home'] = yes
        elif sa and not sh: toks['away'] = yes
        elif sh > sa:       toks['home'] = yes
        elif sa > sh:       toks['away'] = yes
        elif sh:            # sh == sa > 0 : ambiguous, pick the more exact side (ties->home)
            toks['home' if _excess(who, home) <= _excess(who, away) else 'away'] = yes
    return toks

def find_closed_event(tag, game_date, home, away):
    """Match by name (space-insensitive) AND date (exact preferred, then +-1 day for
    UTC/local off-by-one on late kickoffs). Only accepts events that actually expose a
    priceable home+away moneyline (rejects '- Halftime/Props' sub-events)."""
    try:
        gd = datetime.strptime(game_date, '%Y-%m-%d')
    except (ValueError, TypeError):
        gd = None
    cands = []
    for e in events_for_day(tag, game_date):
        title = e.get('title', '')
        tl = title.lower()
        if ' vs' not in tl or 'more market' in tl:
            continue
        parts = re.split(r'\s+vs\.?\s+', P._strip_league_prefix(title))
        if len(parts) != 2: continue
        h, a = parts
        ok = (name_match(h, home) and name_match(a, away)) or \
             (name_match(h, away) and name_match(a, home))
        if not ok: continue
        toks = event_tokens(e, home, away)
        if 'home' not in toks or 'away' not in toks:   # not a priceable moneyline event
            continue
        cands.append((e, P._event_date(e)))
    if not cands:
        return None
    for e, d in cands:                                  # exact date wins
        if d == game_date:
            return e
    if gd is not None:                                  # else nearest within 1 day
        for e, d in cands:
            try:
                if abs((datetime.strptime(d, '%Y-%m-%d') - gd).days) <= 1:
                    return e
            except (ValueError, TypeError):
                pass
    return None

def price_history(token, start_ts, end_ts, fidelity=10):
    try:
        r = requests.get("https://clob.polymarket.com/prices-history",
                         params={"market": token, "startTs": start_ts,
                                 "endTs": end_ts, "fidelity": fidelity}, timeout=25)
        if not r.ok: return []
        return r.json().get('history', [])
    except requests.RequestException:
        return []

def nearest(series_t, series_p, ts):
    """series_t sorted unix list; return (price, gap_seconds) nearest to ts or (None,None)."""
    if not series_t: return None, None
    i = bisect.bisect_left(series_t, ts)
    best, bestgap = None, None
    for j in (i-1, i):
        if 0 <= j < len(series_t):
            g = abs(series_t[j] - ts)
            if bestgap is None or g < bestgap:
                bestgap, best = g, series_p[j]
    return best, bestgap


# assume live timestamps are US/Eastern host-local. EDT (Mar-Nov) = UTC-4; the whole
# 2026-04..06 backfill window is inside EDT, so a flat +4h shift is correct throughout.
TZ_OFFSET_H = 4

def backfill_file(live_file, out_file):
    """Backfill one live_data day with Polymarket historical prices; write *_polybackfill
    copy. Returns a stats dict. Rows with empty game_date (in-play snapshots that can't be
    grouped to a fixture) are skipped at grouping -- they were the old phantom 'no event'."""
    rows = list(csv.DictReader(open(live_file, newline='', encoding='utf-8')))
    if not rows:
        return {'rows': 0, 'fixtures': 0, 'matched': 0, 'no_event': 0, 'no_hist': 0, 'filled': 0}
    fixtures = {}
    for r in rows:
        if r.get('league') in COVERED and r.get('game_date'):
            fixtures.setdefault((r['game_date'], r['league'], r['home'], r['away']), []).append(r)

    matched = filled = no_event = no_hist = 0
    fix_series = {}     # fixture key -> {label: (sorted_t, p)}

    for key, frows in fixtures.items():
        gd, league, home, away = key
        tag = P._LEAGUE_TAG_BY_CODE.get(league)
        ev = find_closed_event(tag, gd, home, away)
        if not ev:
            no_event += 1; continue
        toks = event_tokens(ev, home, away)
        # window = snapshot span +- 3h, shifted to UTC
        dts = [parse_ts(r['timestamp']) for r in frows]
        dts = [d for d in dts if d]
        if not dts: continue
        lo = int(min(dts).replace(tzinfo=timezone.utc).timestamp()) + TZ_OFFSET_H*3600 - 3*3600
        hi = int(max(dts).replace(tzinfo=timezone.utc).timestamp()) + TZ_OFFSET_H*3600 + 3*3600
        series = {}
        for label, tok in toks.items():
            h = price_history(tok, lo, hi, fidelity=10)
            if h:
                h.sort(key=lambda x: x['t'])
                series[label] = ([x['t'] for x in h], [x['p'] for x in h])
        if not series:
            no_hist += 1; continue
        matched += 1
        fix_series[key] = series

    # second pass: fill rows
    out_fields = list(rows[0].keys()) + ['poly_home_price', 'poly_draw_price',
                                         'poly_away_price', 'poly_gap_min']
    for r in rows:
        key = (r.get('game_date'), r.get('league'), r.get('home'), r.get('away'))
        for c in ('poly_home_price', 'poly_draw_price', 'poly_away_price', 'poly_gap_min'):
            r[c] = ''
        series = fix_series.get(key)
        if not series: continue
        d = parse_ts(r['timestamp'])
        if not d: continue
        ts = int(d.replace(tzinfo=timezone.utc).timestamp()) + TZ_OFFSET_H*3600
        gaps = []
        any_fill = False
        for label in ('home', 'draw', 'away'):
            if label in series:
                p, gap = nearest(series[label][0], series[label][1], ts)
                if p is not None and gap is not None and gap <= MATCH_TOL_MIN*60:
                    r[f'poly_{label}_price'] = round(p, 4)
                    gaps.append(gap); any_fill = True
        if gaps:
            r['poly_gap_min'] = round(min(gaps)/60, 1)
            filled += 1

    with open(out_file, 'w', newline='', encoding='utf-8') as fh:
        w = csv.DictWriter(fh, fieldnames=out_fields)
        w.writeheader(); w.writerows(rows)

    return {'rows': len(rows), 'fixtures': len(fixtures), 'matched': matched,
            'no_event': no_event, 'no_hist': no_hist, 'filled': filled,
            'fix_series': fix_series, '_rows': rows}


def main():
    s = backfill_file(LIVE_FILE, OUT_FILE)
    print(f"{LIVE_FILE}: {s['rows']} rows, {s['fixtures']} covered fixtures")
    print(f"\nfixtures matched on Polymarket w/ history: {s['matched']}/{s['fixtures']} "
          f"(no event found: {s['no_event']}, no history: {s['no_hist']})")
    print(f"snapshot rows filled: {s['filled']}")
    print(f"wrote {OUT_FILE}")

    # quality sample: compare poly vs kalshi mid vs pin_fair on the LAST filled row per fixture
    print("\n--- alignment spot-check (last filled row per fixture) ---")
    rows, fix_series = s['_rows'], s['fix_series']
    shown = 0
    for key, series in fix_series.items():
        frows = [r for r in rows if (r.get('game_date'), r.get('league'),
                 r.get('home'), r.get('away')) == key and r.get('poly_home_price') != '']
        if not frows: continue
        r = frows[-1]
        print(f"  {key[1]:4s} {key[2][:16]:16s} v {key[3][:16]:16s} "
              f"poly H/D/A={r['poly_home_price']}/{r['poly_draw_price']}/{r['poly_away_price']} "
              f"kalshiHmid={r.get('kalshi_home_mid','')} pinH={r.get('pin_fair_home','')} "
              f"gap={r['poly_gap_min']}m")
        shown += 1
        if shown >= 15: break


if __name__ == '__main__':
    main()