"""Download SoFIFA PLAYER ratings (and optionally team ratings). RUN BY THE USER.

What it collects, for FC 25 (Sept 2024) through FC 27 (2026/27):
  * PLAYERS (default): every player in 18 leagues, with EVERY column SoFIFA's
    player list offers (80: all attributes, face stats, height/weight, foot,
    weak foot, skill moves, work rates, traits, PlayStyles, contract, wage,
    value, release clause...). See PLAYER_COL_NAMES.
    About monthly snapshots per edition (12 evenly spaced ratings updates),
    so each match has ratings from within a few weeks. ~6,000 pages, ~5-6 hours.
  * TEAMS (--teams): each team's overall/attack/midfield/defence, transfer
    budget and club worth, ~monthly. ~2,500 pages, ~2 hours.

How it behaves, deliberately:
  * Identifies itself honestly (USER_AGENT below), not as a browser.
  * Checks sofifa.com/robots.txt before EVERY request and refuses anything it
    disallows (currently /api/ and /players/compare, which it never needs).
  * One request every ~3 seconds; every page cached on disk, so stopping and
    re-running resumes where it left off and never re-downloads.
  * Stops at the first HTTP 403/429 instead of retrying: that is the site
    saying no, and the right response is to stop.

The site's robots.txt also carries `Content-Signal: ai-train=no` and blocks AI
crawlers. Using the data for your own model is your call (2026-09-29).

Usage (from E:\\soccer):
  1. Check league ids first -- one page per league, ~1 minute:
       ml_env\\Scripts\\python.exe research\\sofifa_ratings.py --check
  2. Players (default), ~5-6 hours, safe to stop and restart:
       ml_env\\Scripts\\python.exe research\\sofifa_ratings.py
  3. Team ratings too (optional), ~2 hours:
       ml_env\\Scripts\\python.exe research\\sofifa_ratings.py --teams

Output: research/ml_data/sofifa_player_ratings.csv
        research/ml_data/sofifa_team_ratings.csv   (with --teams)
"""
import argparse
import csv
import hashlib
import html
import random
import re
import sys
import time
import urllib.error
import urllib.request

from pathlib import Path

BASE = 'https://sofifa.com'
USER_AGENT = 'soccer-research-script/1.0 (personal, non-commercial; ~1 req/3s)'
DELAY_S = 3.0
OUT_DIR = Path(r'E:\soccer\research\ml_data')
CACHE = OUT_DIR / 'sofifa_cache'
OUT_PLAYERS = OUT_DIR / 'sofifa_player_ratings.csv'
OUT_TEAMS = OUT_DIR / 'sofifa_team_ratings.csv'

# FC 25 (Sept 2024) .. FC 27 (2026/27). Narrowed from FIFA 15+ on 2026-09-29
# (user): the ratings are to be paired with Kalshi prices, and Kalshi soccer
# markets only start in May 2025, so earlier editions have nothing to pair with.
VERSIONS = range(25, 28)
TEAM_UPDATES_PER_VERSION = 12     # evenly spaced, ~monthly
# Player snapshots: positions within each version's update list, ~monthly
# (an edition's updates run roughly Sept -> Aug), so each match can use ratings
# from within a few weeks of kickoff. A version with few updates yet (the
# current one) just yields whatever exists.
#
# FROM MAY 2025 ONLY (user, 2026-09-29: "go back to May 2025"). FC 25 keeps
# just its last five monthly points, ~April-August 2025: April stays because
# May's matches need the rating in force just before them.
_MONTHLY = tuple(i / 11 for i in range(12))
PLAYER_SNAPSHOT_POS = {25: _MONTHLY[7:]}          # other versions: all 12
PLAYER_SNAPSHOT_DEFAULT = _MONTHLY
PAGE_SIZE = 60
MAX_PAGES_PER_LEAGUE = 25         # safety stop; a league is ~10 pages

# SoFIFA league id -> football-data code. `--check` prints the name SoFIFA
# shows for each id; fix any mismatch before a full run. National League and
# the lower Scottish tiers are not in the game, so they are absent on purpose.
LEAGUES = {
    13: 'E0',  14: 'E1',  60: 'E2',  61: 'E3',
    53: 'SP1', 54: 'SP2',
    19: 'D1',  20: 'D2',
    31: 'I1',  32: 'I2',
    16: 'F1',  17: 'F2',
    10: 'N1', 308: 'P1',   4: 'B1',  50: 'SC0',
    68: 'T1',  63: 'G1',
}
TEAM_COLS = ['oa', 'at', 'md', 'df', 'tb', 'cw', 'sa', 'ps']
# EVERY column SoFIFA's player list can show (its "Columns selected" picker,
# read 2026-09-29) -- the user wants all the info for each player. Codes and
# labels are in PLAYER_COL_NAMES. What the list cannot show, only the player's
# own page does: the rating at each individual position.
PLAYER_COL_NAMES = {
    'pi': 'ID', 'ae': 'Age', 'by': 'Birth year', 'hi': 'Height', 'wi': 'Weight',
    'pf': 'Preferred foot', 'oa': 'Overall rating', 'pt': 'Potential',
    'bo': 'Best overall', 'bp': 'Best position', 'gu': 'Growth', 'jt': 'Joined',
    'le': 'Loan date end', 'wg': 'Wage', 'vl': 'Value', 'rc': 'Release clause',
    'ta': 'Total attacking', 'cr': 'Crossing', 'fi': 'Finishing',
    'he': 'Heading accuracy', 'sh': 'Short passing', 'vo': 'Volleys',
    'ts': 'Total skill', 'dr': 'Dribbling', 'cu': 'Curve', 'fr': 'FK Accuracy',
    'lo': 'Long passing', 'bl': 'Ball control', 'to': 'Total movement',
    'ac': 'Acceleration', 'sp': 'Sprint speed', 'ag': 'Agility', 're': 'Reactions',
    'ba': 'Balance', 'tp': 'Total power', 'so': 'Shot power', 'ju': 'Jumping',
    'st': 'Stamina', 'sr': 'Strength', 'ln': 'Long shots', 'te': 'Total mentality',
    'ar': 'Aggression', 'in': 'Interceptions', 'po': 'Attack position',
    'vi': 'Vision', 'pe': 'Penalties', 'cm': 'Composure', 'td': 'Total defending',
    'ma': 'Defensive awareness', 'sa': 'Standing tackle', 'sl': 'Sliding tackle',
    'tg': 'Total goalkeeping', 'gd': 'GK Diving', 'gh': 'GK Handling',
    'gc': 'GK Kicking', 'gp': 'GK Positioning', 'gr': 'GK Reflexes',
    'tt': 'Total stats', 'bs': 'Base stats', 'wk': 'Weak foot', 'sk': 'Skill moves',
    'aw': 'Attacking work rate', 'dw': 'Defensive work rate',
    'ir': 'International reputation', 'bt': 'Body type', 'hc': 'Real face',
    'pac': 'Pace / Diving', 'sho': 'Shooting / Handling', 'pas': 'Passing / Kicking',
    'dri': 'Dribbling / Reflexes', 'def': 'Defending / Pace',
    'phy': 'Physical / Positioning', 't1': 'Traits', 't2': 'Traits',
    'ps1': 'PlayStyles', 'ps2': 'PlayStyles +', 'tc': 'Number of playstyles',
    'at': 'Acceleration type', 'cp': 'Club position', 'cj': 'Club kit number',
}
PLAYER_COLS = list(PLAYER_COL_NAMES)

_last = [0.0]
_rules = None        # [(allow: bool, path_prefix)] for our user-agent group


class Refused(Exception):
    pass


def load_robots():
    """Fetch robots.txt AS OURSELVES and keep the rules that apply to us.

    Not urllib.robotparser, for two reasons found on 2026-09-29:
      * it downloads robots.txt as 'Python-urllib/3.x', which sofifa answers
        with 403, and it then treats the whole site as disallowed;
      * it applies rules in file order, so sofifa's leading `Allow: /` would
        have permitted /api/, which the site disallows. Standard (RFC 9309)
        semantics are longest match wins -- implemented in allowed() below.
    A 401/403 on robots.txt for OUR user agent is still read as "stay out".
    """
    global _rules
    req = urllib.request.Request(BASE + '/robots.txt', headers={'User-Agent': USER_AGENT})
    try:
        with urllib.request.urlopen(req, timeout=30) as r:
            text = r.read().decode('utf-8', errors='replace')
    except urllib.error.HTTPError as e:
        if e.code in (401, 403):
            raise Refused(f'robots.txt returned HTTP {e.code} to our user agent -- '
                          f'treating the whole site as off-limits')
        text = ''                                    # 404 etc.: no rules
    groups, agents, in_rules = {}, [], False
    for raw in text.splitlines():
        line = raw.split('#', 1)[0].strip()
        if ':' not in line:
            continue
        k, v = (s.strip() for s in line.split(':', 1))
        k = k.lower()
        if k == 'user-agent':
            if in_rules:
                agents, in_rules = [], False
            agents.append(v.lower())
            for a in agents:
                groups.setdefault(a, [])
        elif k in ('allow', 'disallow'):
            in_rules = True
            for a in agents:
                groups[a].append((k == 'allow', v))
    token = USER_AGENT.split('/', 1)[0].lower()
    _rules = groups.get(token, groups.get('*', []))


def allowed(path):
    """Longest matching rule wins; on a tie Allow wins; no match = allowed."""
    hits = [(len(p), a) for a, p in _rules if p and path.startswith(p)]
    if not hits:
        return True
    best = max(n for n, _ in hits)
    return any(a for n, a in hits if n == best)


def fetch(path):
    """Cached, robots-checked, rate-limited GET. Returns the page text."""
    url = BASE + path
    key = CACHE / (hashlib.sha1(url.encode()).hexdigest() + '.html')
    if key.exists():
        return key.read_text(encoding='utf-8', errors='replace')
    if _rules is None:
        load_robots()
    if not allowed(path):
        raise Refused(f'robots.txt disallows {url}')
    wait = DELAY_S + random.uniform(0, 1.0) - (time.time() - _last[0])
    if wait > 0:
        time.sleep(wait)
    req = urllib.request.Request(url, headers={'User-Agent': USER_AGENT})
    try:
        with urllib.request.urlopen(req, timeout=30) as r:
            text = r.read().decode('utf-8', errors='replace')
    except urllib.error.HTTPError as e:
        if e.code in (403, 429):
            raise Refused(f'HTTP {e.code} on {url} -- the site is refusing; stopping')
        if e.code == 404 and 'offset=' in path:
            # A page past the end of a list is a 404, not an empty page: a
            # league with exactly 540 players (P1, update 260017) stopped the
            # 09-29 run here. Not cached, so a real outage is retried later.
            print(f'  404 past the end: {path}')
            return ''
        raise
    finally:
        _last[0] = time.time()
    CACHE.mkdir(parents=True, exist_ok=True)
    key.write_text(text, encoding='utf-8')
    return text


def page_date(text):
    m = re.search(r'<title>\s*(?:FIFA|FC)\s?\d+\s*-\s*([^-<]+?)\s*-', text)
    return m.group(1).strip() if m else ''


LIST_COLS = ('t1', 't2', 'ps1', 'ps2')          # traits / PlayStyles: one span each


def _cell(tr, col):
    m = re.search(rf'data-col="{col}"[^>]*>(.*?)</td>', tr, re.S)
    if not m:
        return ''
    if col in LIST_COLS:
        items = re.findall(r'<span[^>]*>(.*?)</span>', m.group(1), re.S)
        return '; '.join(html.unescape(re.sub(r'<[^>]+>', '', x)).strip() for x in items)
    return html.unescape(re.sub(r'<[^>]+>', '', m.group(1))).strip()


def _rows(text):
    return [tr for tr in re.findall(r'<tr[^>]*>(.*?)</tr>', text, re.S)
            if 'data-col="oa"' in tr]


def parse_teams(text):
    out = []
    for tr in _rows(text):
        t = re.search(r'href="/team/(\d+)/[^/]+/\d+/">([^<]+)</a>', tr)
        lg = re.search(r'href="/league/(\d+)"[^>]*>([^<]+)</a>', tr)
        row = {'team_id': t.group(1) if t else '',
               'team_name': html.unescape(t.group(2)).strip() if t else '',
               'sofifa_league_id': lg.group(1) if lg else '',
               'sofifa_league_name': html.unescape(lg.group(2)).strip() if lg else ''}
        row.update({c: _cell(tr, c) for c in TEAM_COLS})
        out.append(row)
    return out


def parse_players(text):
    out = []
    for tr in _rows(text):
        p = re.search(r'href="/player/(\d+)/[^"]*"[^>]*data-tippy-content="([^"]+)"', tr)
        t = re.search(r'href="/team/(\d+)/[^/]+/\d+/">([^<]+)</a>', tr)
        # Loan players carry nested markup here (the loan end date), so take
        # everything up to </div> and strip tags rather than stopping at '<'.
        contract = re.search(r'<div class="sub">(.*?)</div>', tr, re.S)
        out.append({
            'player_id': p.group(1) if p else '',
            'player_name': html.unescape(p.group(2)).strip() if p else '',
            # the name cell only: best-position and club-position cells use
            # the same markup further along the row
            'positions': ' '.join(re.findall(r'class="pos pos\d+">([A-Z]+)<',
                                             tr.split('data-col=', 1)[0])),
            'team_id': t.group(1) if t else '',
            'team_name': html.unescape(t.group(2)).strip() if t else '',
            'contract': (re.sub(r'\s+', ' ', html.unescape(
                re.sub(r'<[^>]+>', ' ', contract.group(1)))).strip()
                if contract else ''),
            **{c: _cell(tr, c) for c in PLAYER_COLS},
        })
        # Ratings that moved in this update render as "79+1": the rating AT
        # this update, then the change since the previous one (Anderson reads
        # 78, "79+1", 79 across three updates). Split them.
        # Any attribute can carry the marker; keep the change only for overall
        # and potential, the rest keep just the rating.
        for c in PLAYER_COLS:
            m = re.fullmatch(r'(\d+)\s*([+-]\d+)', out[-1][c])
            if m:
                out[-1][c] = m.group(1)
            if c in ('oa', 'pt'):
                out[-1][f'{c}_chg'] = m.group(2) if m else ''
    return out


def all_updates(version):
    """Every ratings-update id SoFIFA lists for one game version, oldest first."""
    text = fetch(f'/teams?lg=13&r={version:02d}0001&set=true')
    return sorted(set(re.findall(rf'r=({version:02d}\d{{4}})', text)))


def pick(ids, positions):
    if not ids:
        return []
    return sorted({ids[min(len(ids) - 1, round(p * (len(ids) - 1)))] for p in positions})


def check():
    latest = all_updates(max(VERSIONS))[-1]
    print(f'latest update {latest}')
    for lg, code in LEAGUES.items():
        rows = parse_teams(fetch(f'/teams?lg={lg}&r={latest}&set=true'))
        names = {r['sofifa_league_name'] for r in rows}
        print(f'  lg={lg:<4} -> {code:<4} {len(rows):>3} teams  shown as: '
              f'{", ".join(sorted(names)) or "NOTHING -- wrong id"}')


def players_path(lg, update, offset):
    cols = ''.join(f'&showCol%5B%5D={c}' for c in PLAYER_COLS)
    return f'/players?lg={lg}&r={update}&set=true{cols}&offset={offset}'


def run_players():
    plan = [(v, u) for v in VERSIONS
            for u in pick(all_updates(v),
                          PLAYER_SNAPSHOT_POS.get(v, PLAYER_SNAPSHOT_DEFAULT))]
    print(f'{len(plan)} snapshots x {len(LEAGUES)} leagues, ~10 pages each '
          f'(cached pages are free)')
    fields = ['version', 'update_id', 'update_date', 'fdco_league', 'sofifa_league_id',
              'player_id', 'player_name', 'positions', 'team_id', 'team_name',
              'contract'] + PLAYER_COLS + ['oa_chg', 'pt_chg']
    pages = 0
    # Written beside the finished file and swapped in only at the end, so an
    # interrupted run never leaves a half-written sofifa_player_ratings.csv.
    part = OUT_PLAYERS.with_suffix('.partial.csv')
    with open(part, 'w', newline='', encoding='utf-8') as fh:
        w = csv.DictWriter(fh, fieldnames=fields)
        w.writeheader()
        for v, u in plan:
            for lg, code in LEAGUES.items():
                for n in range(MAX_PAGES_PER_LEAGUE):
                    text = fetch(players_path(lg, u, n * PAGE_SIZE))
                    rows = parse_players(text)
                    for r in rows:
                        w.writerow({'version': v, 'update_id': u,
                                    'update_date': page_date(text),
                                    'fdco_league': code, 'sofifa_league_id': lg, **r})
                    pages += 1
                    if pages % 50 == 0:
                        print(f'  {pages} pages  (FC/FIFA {v}, update {u}, {code})')
                    if len(rows) < PAGE_SIZE:
                        break
                fh.flush()
    part.replace(OUT_PLAYERS)
    print(f'done: {OUT_PLAYERS}')


def run_teams():
    plan = [(v, u) for v in VERSIONS
            for u in pick(all_updates(v), [i / (TEAM_UPDATES_PER_VERSION - 1)
                                           for i in range(TEAM_UPDATES_PER_VERSION)])]
    total = len(plan) * len(LEAGUES)
    print(f'{len(plan)} updates x {len(LEAGUES)} leagues = {total} pages')
    fields = ['version', 'update_id', 'update_date', 'fdco_league',
              'team_id', 'team_name', 'sofifa_league_id', 'sofifa_league_name'] + TEAM_COLS
    n = 0
    with open(OUT_TEAMS, 'w', newline='', encoding='utf-8') as fh:
        w = csv.DictWriter(fh, fieldnames=fields)
        w.writeheader()
        for v, u in plan:
            for lg, code in LEAGUES.items():
                text = fetch(f'/teams?lg={lg}&r={u}&set=true')
                for r in parse_teams(text):
                    w.writerow({'version': v, 'update_id': u,
                                'update_date': page_date(text), 'fdco_league': code, **r})
                n += 1
                if n % 50 == 0:
                    print(f'  {n}/{total} pages')
    print(f'done: {OUT_TEAMS}')


if __name__ == '__main__':
    ap = argparse.ArgumentParser()
    ap.add_argument('--check', action='store_true',
                    help='one page per league to confirm the league ids')
    ap.add_argument('--teams', action='store_true',
                    help='team ratings instead of players')
    a = ap.parse_args()
    OUT_DIR.mkdir(parents=True, exist_ok=True)
    try:
        load_robots()
        if a.check:
            check()
        elif a.teams:
            run_teams()
        else:
            run_players()
    except Refused as e:
        print(f'STOPPED: {e}')
        sys.exit(2)
    except KeyboardInterrupt:
        print('\ninterrupted -- re-run the same command to resume from the cache')
        sys.exit(130)