"""Is API-Football's "Pinnacle" actually Pinnacle -- and how far behind is it?

We have an INDEPENDENT Pinnacle reference sitting on disk: football-data.co.uk
publishes Pinnacle opening (PSH/PSD/PSA) and CLOSING (PSCH/PSCD/PSCA) odds for
every match. Our live pipeline reads a different source entirely: API-Football's
`/odds` endpoint, bookmaker id 4, labelled "Pinnacle". The two have never been
compared.

TWO QUESTIONS, and they are separable:

  1. IS IT REALLY PINNACLE? If API-Football's line is a genuine Pinnacle feed,
     its last pre-kickoff quote should sit close to Pinnacle's published CLOSE.
     A large systematic gap would mean the label is wrong -- that we have been
     computing "CLV vs Pinnacle" against something else entirely, which would
     undermine every result in this project that references Pinnacle.

  2. WHAT DOES THE STALENESS COST? We measured API-Football's line changing only
     every ~5.8h ([[project-pin-line-staleness]]). Even if the source is genuine,
     a slow relay means our last pre-kickoff read is an OLD Pinnacle price. This
     measures, in cents, how far it lands from the real close -- which is the
     error sitting inside every CLV number we compute.

Question 1 is answered by the shape of the distribution (centred and tight =
genuine), question 2 by its width. A biased centre would indicate a different
book; a wide but unbiased spread indicates lag.

Comparison is on DEVIGGED probabilities so bookmaker margin differences cannot
masquerade as price differences.

Usage:  python research\\af_vs_real_pinnacle.py
"""
import csv
import glob
import statistics
import sys
from collections import defaultdict
from datetime import datetime

sys.path.insert(0, r'E:\soccer\main')
sys.path.insert(0, r'E:\soccer\research')
import daily_signals as DS
import nonprice_edge_test as T

LIVE = r'E:\soccer\live_data\*.csv'
LEAGUE = r'E:\soccer\league_data\*.csv'


def _f(x):
    try:
        return float(x)
    except (TypeError, ValueError):
        return None


def devig(o):
    r = [1.0 / x for x in o]
    s = sum(r)
    return [x / s for x in r]


def load_fd():
    """(date, home_fd, away_fd) -> (open_probs, close_probs)."""
    out = {}
    for p in glob.glob(LEAGUE):
        try:
            with open(p, newline='', encoding='utf-8', errors='replace') as fh:
                for r in csv.DictReader(fh):
                    op = [_f(r.get(c)) for c in ('PSH', 'PSD', 'PSA')]
                    cl = [_f(r.get(c)) for c in ('PSCH', 'PSCD', 'PSCA')]
                    if not (all(cl) and all(v > 1 for v in cl)):
                        continue
                    d = (r.get('Date') or '').strip()
                    dt = None
                    for fmt in ('%d/%m/%Y', '%d/%m/%y'):
                        try:
                            dt = datetime.strptime(d, fmt).date()
                            break
                        except ValueError:
                            pass
                    if not dt:
                        continue
                    key = (dt, T.norm(r.get('HomeTeam')), T.norm(r.get('AwayTeam')))
                    out[key] = (devig(op) if all(op) and all(v > 1 for v in op) else None,
                                devig(cl))
        except Exception:
            continue
    return out


def load_af():
    """(date, home_fd, away_fd) -> (last pre-KO AF pinnacle probs, hours before KO)."""
    best = {}
    for p in glob.glob(LIVE):
        with open(p, newline='', encoding='utf-8', errors='replace') as fh:
            for r in csv.DictReader(fh):
                if (r.get('status') or '').upper() != 'PRE':
                    continue
                el = _f(r.get('elapsed'))
                if el is None or el >= 0:
                    continue
                o = [_f(r.get('pin_home')), _f(r.get('pin_draw')), _f(r.get('pin_away'))]
                if not (all(o) and all(v > 1 for v in o)):
                    continue
                gd = r.get('game_date')
                try:
                    d = datetime.strptime(gd[:10], '%Y-%m-%d').date()
                except (ValueError, TypeError):
                    continue
                key = (d, T.norm(DS.fdco_name(r.get('home'))),
                       T.norm(DS.fdco_name(r.get('away'))))
                hrs = -el / 60.0
                # keep the LAST pre-kickoff observation (smallest hrs)
                if key not in best or hrs < best[key][1]:
                    best[key] = (devig(o), hrs)
    return best


def main():
    fd = load_fd()
    af = load_af()
    keys = set(fd) & set(af)
    print(f'football-data fixtures with Pinnacle close: {len(fd)}')
    print(f'live_data fixtures with an API-Football pin quote: {len(af)}')
    print(f'matched on (date, home, away): {len(keys)}\n')
    if len(keys) < 20:
        print('too few matched fixtures to conclude anything')
        return

    signed, absd, lags = [], [], []
    per_side = defaultdict(list)
    for k in keys:
        _, close = fd[k]
        afp, hrs = af[k]
        lags.append(hrs)
        for i, side in enumerate(('home', 'draw', 'away')):
            d = (afp[i] - close[i]) * 100
            signed.append(d)
            absd.append(abs(d))
            per_side[side].append(d)

    absd.sort()
    print('Q1 -- IS IT PINNACLE?  (signed: AF minus real Pinnacle close, cents)')
    print(f'    mean bias {statistics.mean(signed):+.3f}c   '
          f'median {statistics.median(signed):+.3f}c')
    for s, v in per_side.items():
        print(f'      {s:5s} mean {statistics.mean(v):+.3f}c')
    print('    A near-zero centre means the same book. A persistent bias would '
          'mean a different one.')

    print('\nQ2 -- WHAT DOES THE LAG COST?  (|AF - close|, cents)')
    print(f'    median {statistics.median(absd):.2f}c   p75 {absd[int(len(absd)*.75)]:.2f}c   '
          f'p90 {absd[int(len(absd)*.90)]:.2f}c   max {max(absd):.2f}c')
    print(f'    within 1c: {sum(1 for x in absd if x < 1)/len(absd)*100:.0f}%   '
          f'beyond 3c: {sum(1 for x in absd if x > 3)/len(absd)*100:.0f}%')
    print(f'    (our last pre-KO read is a median of {statistics.median(lags):.1f}h '
          f'before kickoff)')

    print('\n    For scale: the xlow band is 5-12% of fair. On a 25c leg that is '
          '1.25-3.0c,\n    so an error of this size is a material fraction of the '
          'signal being traded.')


if __name__ == '__main__':
    main()