""" Daily watchlist updater — run at midnight via Task Scheduler. Queries API-Football for ALL fixtures in tracked leagues, then checks Kalshi for which ones have coverage. All matches are added to watchlist.txt (active/uncommented) so the live monitor tracks B365 CLV on everything. Kalshi coverage is noted with a [K] tag. Usage: python daily_watchlist.py """ import sys, requests, time as _time from datetime import date, timedelta, datetime, timezone from pathlib import Path sys.path.insert(0, str(Path(__file__).parent)) import maps from kalshi_live import list_matches WATCHLIST_F = Path(r"E:\soccer\data\watchlist.txt") PREDICTIONS_F = Path(r"E:\soccer\data\predictions_today.csv") AF_BASE = "https://v3.football.api-sports.io" AF_HEADERS = {"x-apisports-key": "81b487be73030f5dec980d2a1ef2c406"} DEFAULT_KICKOFFS = { 'E0': ['12:30', '15:00', '17:30', '20:00'], 'E1': ['15:00', '17:30'], 'I1': ['14:00', '17:00', '19:45'], 'SP1': ['14:00', '16:15', '18:30', '21:00'], 'F1': ['19:00', '21:00'], 'D1': ['14:30', '17:30'], 'D2': ['13:30', '18:00'], 'N1': ['14:45', '19:00'], 'P1': ['16:00', '19:00', '21:15'], 'SC0': ['13:00', '15:00'], 'T1': ['14:00', '17:00', '19:00'], 'G1': ['16:00', '19:00'], 'MLS': ['19:30', '20:30', '22:00', '22:30'], 'I2': ['17:00', '19:30'], 'SP2': ['18:00', '20:30'], } TEAM_ALIASES = { 'psg': 'paris saint germain', 'stade brest': 'brestois', 'stade rennais': 'rennes', 'strasbourg alsace': 'strasbourg', 'monchengladbach': 'gladbach', "m'gladbach": 'gladbach', 'mogladbach': 'gladbach', 'borussia mogladbach': 'gladbach', 'munster': 'preussen munster', 'bayern munich': 'munchen', 'deportivo de la coruna': 'deportivo la coruna', 'atletico': 'atletico madrid', 'real betis': 'betis', 'athletic': 'athletic club', 'alaves': 'deportivo alaves', 'inter': 'internazionale', 'ac milan': 'milan', 'wolves': 'wolverhampton', 'west brom': 'west bromwich', 'spurs': 'tottenham', 'man city': 'manchester city', 'man united': 'manchester united', 'man utd': 'manchester united', 'sl benfica': 'benfica', 'sporting cp': 'sporting', 'los angeles f': 'los angeles fc', 'salt lake': 'real salt lake', 'new york': 'new york red bulls', 'new york city': 'new york city', 'san diego fc': 'san diego', } def _normalize(name): replacements = { 'ü': 'u', 'ö': 'o', 'ä': 'a', 'ß': 'ss', 'é': 'e', 'è': 'e', 'ê': 'e', 'ë': 'e', 'á': 'a', 'à': 'a', 'â': 'a', 'ã': 'a', 'í': 'i', 'ì': 'i', 'î': 'i', 'ó': 'o', 'ò': 'o', 'ô': 'o', 'õ': 'o', 'ú': 'u', 'ù': 'u', 'û': 'u', 'ç': 'c', 'ñ': 'n', } n = name.lower() for k, v in replacements.items(): n = n.replace(k, v) return n # Network errors seen during this run's fixture fetches. main() reads it before # writing: a fetch that FAILED is not the same as a day with no fixtures, and # the file must not be overwritten on the strength of one. See main(). _FETCH_ERRORS = 0 def fetch_all_fixtures(game_date): """Fetch all fixtures for tracked leagues from API-Football. Returns list of (home, away, lc, fid, kickoff_et_str).""" global _FETCH_ERRORS month = game_date.month et_offset = timedelta(hours=-4) if 3 <= month <= 11 else timedelta(hours=-5) et_zone = timezone(et_offset) seen_leagues = {maps.league_api_id(d): d for d in maps.all_divs() if maps.league_api_id(d)} skipped_status = {} print(f" Fetching fixtures from API-Football for {game_date}...") results = [] seen_fids = set() # Season years to sweep. `season` filters /fixtures GLOBALLY, so querying # only [year, year-1] silently drops any league whose API-Football season # label runs ahead of the calendar -- which is exactly how J1 League went # untracked from its 2026-08-07 season opener (labelled season 2027). # maps.season_years() derives the set from what the leagues actually use. seasons = maps.season_years(AF_HEADERS, extra=(game_date.year, game_date.year - 1)) print(f" Season years to sweep: {seasons}") for delta in [0, 1]: search_date = game_date + timedelta(days=delta) for season_year in sorted(set(seasons) | {search_date.year, search_date.year - 1}): try: r = requests.get(f"{AF_BASE}/fixtures", headers=AF_HEADERS, params={'date': search_date.isoformat(), 'season': season_year}, timeout=15) for fix in r.json().get('response', []): lid = fix['league']['id'] if lid not in seen_leagues: continue fid = fix['fixture']['id'] if fid in seen_fids: continue seen_fids.add(fid) # SKIP FIXTURES THAT ARE NOT ACTUALLY BEING PLAYED # (added 2026-07-29). API-Football keeps a postponed match on # its ORIGINAL date with status PST, so it looks like a normal # fixture. On 2026-07-29 four of eight "BSA games today" were # postponed, and we quoted them: Kalshi had relisted those # events for the 30th, the books were empty (bids 4/5/2c vs # asks 79/78/78c), and Pinnacle had no line -- three separate # anomalies that were all this one cause. Nothing filled, but # complete_book() would eventually have crossed into a dead # book at whatever junk price was resting there. # Statuses: PST postponed, CANC cancelled, ABD abandoned, # SUSP suspended, AWD/WO awarded/walkover (no market). st = ((fix.get('fixture') or {}).get('status') or {}).get('short') if st in ('PST', 'CANC', 'ABD', 'SUSP', 'AWD', 'WO'): skipped_status[st] = skipped_status.get(st, 0) + 1 continue home = fix['teams']['home']['name'] away = fix['teams']['away']['name'] lc = seen_leagues[lid] utc_dt = datetime.fromisoformat(fix['fixture']['date'].replace('Z', '+00:00')) et_dt = utc_dt.astimezone(et_zone) if et_dt.date() != game_date: continue ko_str = et_dt.strftime('%H:%M') results.append((home, away, lc, fid, ko_str)) except Exception as e: _FETCH_ERRORS += 1 print(f" [fixture lookup {search_date} season={season_year}] Error: {e}") _time.sleep(0.5) if skipped_status: print(f" Skipped not-being-played: {skipped_status}") print(f" Found {len(results)} fixtures across {len(seen_leagues)} leagues") return results def build_kalshi_set(game_date): """Return set of (home_norm, away_norm) tuples for matches on Kalshi.""" try: games = list_matches(game_date) kalshi = set() for _, home, away in games: h = TEAM_ALIASES.get(_normalize(home), _normalize(home)) a = TEAM_ALIASES.get(_normalize(away), _normalize(away)) kalshi.add((h, a)) return kalshi except Exception as e: print(f" [Kalshi] Error fetching matches: {e}") return set() def has_kalshi(home, away, kalshi_set): h = _normalize(home) a = _normalize(away) h = TEAM_ALIASES.get(h, h) a = TEAM_ALIASES.get(a, a) if (h, a) in kalshi_set: return True # Partial match for (kh, ka) in kalshi_set: if (h in kh or kh in h) and (a in ka or ka in a): return True return False def load_predictions(): if not PREDICTIONS_F.exists(): return {} try: import pandas as pd df = pd.read_csv(PREDICTIONS_F) result = {} for _, row in df.iterrows(): key = (str(row.get('home', '')).lower().strip(), str(row.get('away', '')).lower().strip()) result[key] = float(row.get('pA', 0)) return result except Exception as e: print(f" [predictions] Could not load {PREDICTIONS_F}: {e}") return {} def main(): # Two-day horizon (added 2026-07-04): include TOMORROW's fixtures so # live_monitor's T-24h pre-match window (elapsed >= -1450) is fully used # for every game — previously morning games only got tracked from midnight. # The watchlist parser handles multiple DATE= sections natively; each match # inherits the most recent DATE= line above it. today = date.today() # today+3 rather than today+1, from 2026-08-17. The live maker's window is # 72h, but the watchlist was feeding it 48, so 45 open MLS books sitting # ~55h out were invisible -- MLS plays Wednesday and Saturday, so a two-day # horizon misses the entire midweek card until the night before. # # Costs one /fixtures call per league per extra date. Quota read 959/7500 # when this was raised, and the negative-cache backoff added yesterday took # daily usage from ~37,000 to a projected ~4,100, so there is room. Revisit # if usage climbs back toward the cap. dates = [today + timedelta(days=d) for d in range(4)] print(f"\n=== Daily Watchlist Update — {today} (+ {dates[1]}) ===") predictions = load_predictions() lines = [ f"# Watchlist generated {today} — covers today + tomorrow (24h pre-KO tracking)", f"# Format: home | away | league | kickoff (HH:MM ET) | fixture_id", f"# [K] = Kalshi coverage available * = model flagged (pA >= 0.31)", ] grand_total = 0 grand_kalshi = 0 for game_date in dates: fixtures = fetch_all_fixtures(game_date) kalshi_set = build_kalshi_set(game_date) by_league = {} for home, away, lc, fid, ko in fixtures: by_league.setdefault(lc, []).append((home, away, fid, ko)) lines += ["#", f"DATE={game_date}", f"# {len(fixtures)} fixtures across {len(by_league)} leagues on {game_date}"] for lc in sorted(by_league.keys()): lines.append(f"#") lines.append(f"# [{lc}]") matches = sorted(by_league[lc], key=lambda x: x[3]) for home, away, fid, ko in matches: key = (home.lower(), away.lower()) pA = predictions.get(key, 0) k_tag = " [K]" if has_kalshi(home, away, kalshi_set) else "" m_tag = f" # model pA={pA:.2f}" if pA >= 0.31 else "" fid_str = f" | {fid}" if fid else "" lines.append(f"{home} | {away} | {lc} | {ko}{fid_str}{k_tag}{m_tag}") if k_tag: grand_kalshi += 1 grand_total += 1 lines.append("#") lines.append(f"# End of schedule") lines.append("") # NEVER REPLACE A GOOD WATCHLIST WITH A BROKEN FETCH (2026-09-28). # # Every fetch error above is caught and printed, and the loop carries on -- # so a run with no network used to "succeed" and write an EMPTY file. That # happened at 08:00 on 2026-09-28, during a UDP port exhaustion that made # every DNS lookup fail: it overwrote a 30-fixture list with 0, leaving # live_monitor watching nothing and the maker blind to every club book. # 2026-09-02 was the same failure from the 00:00 build. # # So when this run hit network errors AND came back empty, or with less than # half of what the current file holds, keep the current file. It was built # with a 4-day horizon, so it still covers today; stale beats empty, and the # next scheduled build replaces it once the network is back. A clean run # (no errors) always writes, even if it finds nothing -- that is a real # empty day, not a failure. if _FETCH_ERRORS: try: _have = sum(1 for ln in WATCHLIST_F.read_text(encoding='utf-8').splitlines() if '|' in ln and not ln.lstrip().startswith('#')) except Exception: _have = 0 if grand_total == 0 or (_have and grand_total < _have / 2): print(f"\nREFUSING TO WRITE: {_FETCH_ERRORS} fetch error(s) and only " f"{grand_total} fixture(s) found, against {_have} in the current " f"file. Keeping {WATCHLIST_F} as it is.") sys.exit(1) WATCHLIST_F.write_text('\n'.join(lines), encoding='utf-8') print(f"\nWrote {grand_total} fixtures ({dates[0]} + {dates[1]}) to {WATCHLIST_F} " f"({grand_kalshi} with Kalshi coverage)" + (f" [{_FETCH_ERRORS} fetch error(s) this run]" if _FETCH_ERRORS else "")) if __name__ == '__main__': main()