"""Pre-match Elo for every match in league_data (ML experiment, 2026-09-29).

Stands in for ClubElo (api.clubelo.com returned 502 for every club). Built
from our own results files, 22 European divisions from 2019/20 on, so the
~6 seasons before Kalshi's May 2025 start are the burn-in.

Run with the isolated env:
    E:\\soccer\\ml_env\\Scripts\\python.exe research\\ml_elo.py

Model (World Football Elo style):
  * expected home score  E = 1 / (1 + 10^(-(Rh + HFA - Ra) / 400))
  * update  K * G * (S - E), S = 1 / 0.5 / 0, G = goal-margin multiplier
  * ratings carry across divisions: football-data spells a club the same way
    in E0 and E1, so a promoted club arrives with its Championship rating
  * a club never seen before starts at its division's 20th percentile
    (newcomers are usually promoted from a division we don't have), or at a
    tier default while the division is still empty
K and HFA are picked by grid search on 2021/22-2024/25 log loss of E against
the result -- strictly before the Kalshi window, so the evaluation period
never tunes its own feature.

Writes research/ml_data/elo_matches.csv: one row per match with both clubs'
pre-match ratings and the home expected score.
"""
from itertools import product
from pathlib import Path

import numpy as np
import pandas as pd

LD = Path(r'E:\soccer\league_data')
OUT = Path(r'E:\soccer\research\ml_data\elo_matches.csv')
TIER = {'E0': 1, 'E1': 2, 'E2': 3, 'E3': 4, 'EC': 5, 'SC0': 1, 'SC1': 2, 'SC2': 3,
        'SC3': 4, 'SP1': 1, 'SP2': 2, 'D1': 1, 'D2': 2, 'I1': 1, 'I2': 2, 'F1': 1,
        'F2': 2, 'N1': 1, 'B1': 1, 'P1': 1, 'T1': 1, 'G1': 1}
TIER_INIT = {1: 1500, 2: 1380, 3: 1280, 4: 1200, 5: 1120}
TUNE = ('2021-07-01', '2025-05-01')


def load():
    frames = []
    for f in sorted(LD.glob('*_[0-9][0-9][0-9][0-9].csv')):
        for enc in ('utf-8-sig', 'cp1252'):   # some seasons are saved as cp1252
            try:
                d = pd.read_csv(f, encoding=enc, on_bad_lines='skip',
                                usecols=lambda c: c in ('Div', 'Date', 'Time', 'HomeTeam',
                                                        'AwayTeam', 'FTHG', 'FTAG'))
                break
            except UnicodeDecodeError:
                continue
        frames.append(d)
    d = pd.concat(frames, ignore_index=True).dropna(subset=['HomeTeam', 'FTHG', 'FTAG'])
    d['Date'] = pd.to_datetime(d['Date'], dayfirst=True, errors='coerce')
    d = d.dropna(subset=['Date'])
    d = d.drop_duplicates(['Div', 'Date', 'HomeTeam', 'AwayTeam'])
    d['Time'] = d['Time'].fillna('12:00')
    return d.sort_values(['Date', 'Time', 'Div']).reset_index(drop=True)


def margin(gd):
    gd = abs(gd)
    return 1.0 if gd <= 1 else 1.5 if gd == 2 else (11 + gd) / 8


def run_elo(d, K, HFA):
    R, members = {}, {}
    eh, ea = np.empty(len(d)), np.empty(len(d))
    for i, (div, h, a, hg, ag) in enumerate(zip(d.Div, d.HomeTeam, d.AwayTeam, d.FTHG, d.FTAG)):
        for t in (h, a):
            if t not in R:
                pool = [R[x] for x in members.get(div, ())]
                R[t] = np.percentile(pool, 20) if len(pool) >= 6 else TIER_INIT[TIER.get(div, 1)]
            members.setdefault(div, set()).add(t)
        rh, ra = R[h], R[a]
        eh[i], ea[i] = rh, ra
        e = 1 / (1 + 10 ** (-(rh + HFA - ra) / 400))
        s = 1.0 if hg > ag else 0.5 if hg == ag else 0.0
        dlt = K * margin(hg - ag) * (s - e)
        R[h], R[a] = rh + dlt, ra - dlt
    return eh, ea


def score(d, eh, ea, HFA):
    m = (d.Date >= TUNE[0]) & (d.Date < TUNE[1])
    e = 1 / (1 + 10 ** (-(eh[m] + HFA - ea[m]) / 400))
    s = np.select([d.FTHG[m] > d.FTAG[m], d.FTHG[m] == d.FTAG[m]], [1.0, 0.5], 0.0)
    e = np.clip(e, 1e-6, 1 - 1e-6)
    return -np.mean(s * np.log(e) + (1 - s) * np.log(1 - e))


if __name__ == '__main__':
    d = load()
    print(f'{len(d)} matches, {d.Date.min():%Y-%m-%d} -> {d.Date.max():%Y-%m-%d}, '
          f'{d.Div.nunique()} divisions, {pd.concat([d.HomeTeam, d.AwayTeam]).nunique()} clubs')
    best = None
    for K, HFA in product((6, 8, 10, 12, 15, 20, 25), (40, 50, 60, 70)):
        eh, ea = run_elo(d, K, HFA)
        ll = score(d, eh, ea, HFA)
        if best is None or ll < best[0]:
            best = (ll, K, HFA)
    ll, K, HFA = best
    print(f'best K={K} HFA={HFA}  tune log loss {ll:.4f}')
    eh, ea = run_elo(d, K, HFA)
    d['elo_home'], d['elo_away'] = eh.round(1), ea.round(1)
    d['elo_diff'] = (eh + HFA - ea).round(1)
    d['elo_exp_home'] = (1 / (1 + 10 ** (-d.elo_diff / 400))).round(4)
    d['elo_K'], d['elo_HFA'] = K, HFA
    d.to_csv(OUT, index=False)
    print(f'wrote {OUT} ({len(d)} rows)')