a
    ïw4jíF  ã                   @   s¨  d Z ddlZddlZddlZddlZddlZddlZddlmZmZ ddl	Z	ej
 dd¡ ddlZejjddd� dZd	Zd
d„ ej ¡ D ƒZdZg d¢Zdd„ Zh d£ZddddddddœZddddœZdddœZe d d d!d!d"d"d#d#d$d$d%d%d&d&d'd(œ¡Zd)d*„ Zd+d,„ Z d-d.„ Z!d/d0„ Z"d1d2„ Z#dd3lm$Z$ i Z%d4d5„ Z&d6hZ'i Z(dKd8d9„Z)i Z*d:d;„ Z+d<d=„ Z,d>d?„ Z-dLdAdB„Z.dCdD„ Z/dEZ0dFdG„ Z1dHdI„ Z2e3dJk�r¤e2ƒ  dS )Mas  
PILOT (read-only research): backfill Polymarket historical prices into one live_data
day, matched by timestamp. Proves feasibility + alignment quality before a full run.

Polymarket exposes per-market price history via the CLOB endpoint
    GET https://clob.polymarket.com/prices-history?market={yesTokenId}&startTs&endTs&fidelity
returning [{t: unix_utc, p: price}] (price = midpoint; NO separate bid/ask exists
historically -- that's the one backfill limitation). We:
  1. group the live_data day into fixtures (covered leagues only),
  2. find each fixture's Polymarket CLOSED event by name+date, read the 3 markets'
     clobTokenIds (home/away/draw YES tokens),
  3. pull each token's price history over the snapshot window,
  4. nearest-timestamp match every snapshot row -> poly_home/draw/away_price.
NOTHING is traded. Writes a *_polybackfill.csv copy; originals untouched.
é    N)ÚdatetimeÚtimezonezE:\soccer\mainúutf-8Úreplace)ÚencodingÚerrorsz"E:\soccer\live_data\2026-04-25.csvz3E:\soccer\poly_backfill\2026-04-25_polybackfill.csvc                 C   s   h | ]\}}|r|’qS © r   )Ú.0ÚcÚtr   r   ú%E:\soccer\main\poly_backfill_pilot.pyÚ	<setcomp>   ó    r   éZ   )z%Y-%m-%d %H:%M:%Sz%m/%d/%Y %H:%Mz%m/%d/%Y %H:%M:%Sz%Y-%m-%d %H:%Mc              
   C   s8   t D ].}zt | |¡W   S  ttfy0   Y q0 qd S ©N)Ú_TS_FMTSr   ÚstrptimeÚ
ValueErrorÚ	TypeError)ÚsÚfr   r   r   Úparse_ts   s    r   >&   ZskZlosZslZacZsdZcpZfcÚdelÚifÚusZcaZlaZclubZcityZcdZcfZdoZscZdeZssdZsportZfkZbkZcountyZrcdZlisboaZrcZajZudZbscZelÚssZscoZcalcioÚeÚasZsvZafcZgermainZrangersZvitoriaZathleticZwolverhamptonZrennaisZnurnberg)ZpsgZqprZ	guimaraesZbilbaoZwolvesZrennesZ	nurembergÚstZunitedÚb)ZsaintZutdZiizlos angeles galaxyzlos angeles fc)zlos angeles gzlos angeles fÚoZaeÚdZthÚlZoeÚar   )õ   Ã¸õ   Ã˜õ   Ã¦õ   Ã†õ   Ã°õ   Ã�õ   Ã¾õ   Ãžu   Å‚u   Å�u   Å“u   Å’õ   Ã¥õ   Ã…õ   ÃŸc                 C   s¾   t  | p
d t¡¡}t ||¡ ¡ }g d }}|D ]6}t|ƒdkrL||7 }q2|r^| |¡ d}| |¡ q2|rx| |¡ g }|D ]8}t	 ||¡}t
 ||¡}|tv s€| ¡ r®q€| |¡ q€|S )NÚ é   )ÚPZ_normÚ	translateÚ	_TRANSLITÚ	_NAME_FIXÚgetÚsplitÚlenÚappendÚ_TOKMAPÚ_ALIASÚ_FILLERÚisdigit)Únamer   ÚrawÚtoksZbufr   Úoutr   r   r   Ú_sig_tokensA   s(    

rA   c                 C   s.   | |kp,t t| ƒt|ƒƒdko,| |v p,|| v S )Né   )Úminr7   ©r#   r   r   r   r   Ú_tok_eqX   s    rE   c                    sd   t | ƒt |ƒ }}|r|sdS t|ƒt|ƒkr6||fn||f\}‰ t‡ fdd„|D ƒƒs\dS t|ƒS )zÔ0 if a,b are not the same club; else the contained-token count (= specificity).
    Lets event_tokens assign 'Paris Saint-Germain' to PSG (score 3) over 'Paris FC'
    (score 1) when both share the 'paris' token.r   c                 3   s$   | ]‰ t ‡ fd d„ˆD ƒƒV  qdS )c                 3   s   | ]}t ˆ |ƒV  qd S r   ©rE   )r	   Úu©r   r   r   Ú	<genexpr>c   r   z)_match_score.<locals>.<genexpr>.<genexpr>N©Úany©r	   ©ZbigrH   r   rI   c   r   z_match_score.<locals>.<genexpr>)rA   r7   Úall)r#   r   ÚsaZsbZsmallr   rM   r   Ú_match_score[   s    $rP   c                 C   s   t | |ƒdkS )Nr   )rP   rD   r   r   r   Ú
name_matchg   s    rQ   c                    s(   t | ƒt |ƒ ‰ }t‡ fdd„|D ƒƒS )z÷# of sig-tokens in `side` NOT covered by any token of `who` (lower = more exact).
    Tie-breaks same-name ambiguity ('Dundee' fits BOTH 'Dundee United' and 'Dundee FC';
    'New York City FC'->{new,york} fits BOTH NYCFC and 'New York Red Bulls').c                 3   s,   | ]$‰ t ‡ fd d„ˆD ƒƒr dndV  qdS )c                 3   s   | ]}t ˆ |ƒV  qd S r   rF   )r	   ÚwrH   r   r   rI   o   r   z$_excess.<locals>.<genexpr>.<genexpr>r   r0   NrJ   rL   ©ÚswrH   r   rI   o   r   z_excess.<locals>.<genexpr>)rA   Úsum)ÚwhoZsider   r   rS   r   Ú_excessj   s    rW   )Ú	timedeltac                 C   sÀ   | t v rt |  S i d }}z&tjtj› d�d| d|dœdd�}W n tjyX   Y q¨Y n0 |jsbq¨| ¡ }|spq¨|D ]}||| d¡< qt|d7 }t|ƒdk s¨|d	krq¨qt	| 
¡ ƒt | < t |  S )
Nr   ú/eventsÚtrueéd   )ÚclosedÚtag_slugÚlimitÚoffseté   ©ÚparamsZtimeoutÚidi¸  )Ú_tag_closedÚrequestsr5   r1   ÚGAMMAÚRequestExceptionÚokÚjsonr7   ÚlistÚvalues)ÚtagÚseenÚoffÚrr   r   r   r   r   Ú_all_closed{   s*    
ÿþ

rp   Zsocceré   c              	   C   s2  | |f}|t v rt | S zt |d¡}W n" ttfyJ   g t |< g  Y S 0 |t|d�  d¡}|t|d�  d¡}i d }}z*tjt	j
› d�d| ||d|dœdd	�}	W n tjyÂ   Y �qY n0 |	jsÎ�q|	 ¡ }
|
sÞ�q|
D ]}||| d
¡< qâ|d7 }t|
ƒdk �s|dkr~�qq~t| ¡ ƒt |< t | S )Nú%Y-%m-%d)Údaysr   rY   rZ   r[   )r\   r]   Zend_date_minZend_date_maxr^   r_   r`   ra   rc   iè  )Ú
_win_cacher   r   r   r   rX   Ústrftimere   r5   r1   rf   rg   rh   ri   r7   rj   rk   )rl   Ú	game_dateZpadÚkeyr!   ÚloÚhirm   rn   ro   r   r   r   r   r   Ú_windowed_closed˜   s8    
þý

rz   c              
   C   sÐ   | |f}|t v rt | S zt |d¡}W n" ttfyJ   g t |< g  Y S 0 g }| tv rbt| |ƒnt| ƒ}|D ]T}t 	|¡}|s‚qnz(t
t |d¡| jƒdkr¨| |¡ W qn ttfyÀ   Y qn0 qn|t |< |S )Nrr   é   )Ú_bydayr   r   r   r   Ú	_BIG_TAGSrz   rp   r1   Ú_event_dateÚabsrs   r8   )rl   rv   rw   r!   r@   Úsrcr   Zedr   r   r   Úevents_for_dayµ   s&    
r�   c                 C   s  i }|   d¡pg D ]ö}|  dd¡p$d ¡ }|  d¡}|s:qt|tƒrNt |¡n|d }d|v rh||d< qt d|¡}|szq| d	¡}	t	|	|ƒt	|	|ƒ }
}|
r¬|s¬||d
< q|r¾|
s¾||d< q|
|krÐ||d
< q||
krâ||d< q|
r||t
|	|ƒt
|	|ƒk�rd
nd< q|S )zè{label: yesTokenId} for home/draw/away markets of a matched event.
    Only the base moneyline markets match the win/beat/draw question patterns, so
    prop sub-events ('- Halftime Result' etc.) yield {} and are naturally rejected.ZmarketsZquestionr/   ZclobTokenIdsr   zend in a drawÚdrawzwill (.+?) (?:beat|win on)r0   ÚhomeÚaway)r5   ÚlowerÚ
isinstanceÚstrri   ÚloadsÚreÚsearchÚgrouprP   rW   )Úevrƒ   r„   r?   ÚmZqlÚtokZyesZmmrV   ZshrO   r   r   r   Úevent_tokensÊ   s&    


$r�   c              
   C   sv  zt  |d¡}W n ttfy*   d}Y n0 g }t| |ƒD ]²}| dd¡}| ¡ }d|vs:d|v rdq:t dt	 
|¡¡}	t|	ƒdkr„q:|	\}
}t|
|ƒr t||ƒp²t|
|ƒo²t||ƒ}|sºq:t|||ƒ}d	|vs:d
|vrØq:| |t	 |¡f¡ q:|södS |D ]\}}||krú|  S qú|du�rr|D ]N\}}z*tt  |d¡| jƒdk�rR|W   S W n ttf�yl   Y n0 �q"dS )zïMatch by name (space-insensitive) AND date (exact preferred, then +-1 day for
    UTC/local off-by-one on late kickoffs). Only accepts events that actually expose a
    priceable home+away moneyline (rejects '- Halftime/Props' sub-events).rr   NÚtitler/   z vszmore marketz\s+vs\.?\s+r{   rƒ   r„   r0   )r   r   r   r   r�   r5   r…   r‰   r6   r1   Z_strip_league_prefixr7   rQ   r�   r8   r~   r   rs   )rl   rv   rƒ   r„   ÚgdZcandsr   r�   ZtlÚpartsÚhr#   rh   r?   r!   r   r   r   Úfind_closed_eventç   sD    
ÿ


r”   é
   c                 C   sV   z8t jd| |||dœdd�}|js(g W S | ¡  dg ¡W S  t jyP   g  Y S 0 d S )Nz*https://clob.polymarket.com/prices-history)ZmarketZstartTsZendTsÚfidelityr`   ra   Úhistory)re   r5   rh   ri   rg   )ÚtokenZstart_tsZend_tsr–   ro   r   r   r   Úprice_history  s    ÿþr™   c                 C   s€   | sdS t  | |¡}d\}}|d |fD ]N}d|  krDt| ƒk r(n q(t| | | ƒ}|du sh||k r(|||  }}q(||fS )zTseries_t sorted unix list; return (price, gap_seconds) nearest to ts or (None,None).)NNr0   r   N)ÚbisectZbisect_leftr7   r   )Zseries_tZseries_pÚtsÚiZbestZbestgapÚjÚgr   r   r   Únearest  s    rŸ   rB   c           $   	   C   s’  t t t| ddd�¡ƒ}|s.dddddddœS i }|D ]F}| d¡tv r6| d¡r6| |d |d |d |d	 fg ¡ |¡ q6d } } }}i }	| ¡ D �]6\}
}|
\}}}}t	j
 |¡}t||||ƒ}|sØ|d
7 }qšt|||ƒ}dd„ |D ƒ}dd„ |D ƒ}|�sqštt|ƒjtjd� ¡ ƒtd  d }tt|ƒjtjd� ¡ ƒtd  d }i }| ¡ D ]P\}}t|||dd�}|�r`|jdd„ d� dd„ |D ƒdd„ |D ƒf||< �q`|�sÂ|d
7 }qš|d
7 }||	|
< qšt |d  ¡ ƒg d¢ }|D �].}| d¡| d¡| d¡| d	¡f}
dD ]}d||< �q|	 |
¡}|�sB�qðt|d ƒ}|�sX�qðt|jtjd� ¡ ƒtd  }g }d}dD ]v}||v �r‚t|| d || d
 |ƒ\} }!| du�r‚|!du�r‚|!td k�r‚t| dƒ|d|› d �< | |!¡ d!}�q‚|�rðtt|ƒd d
ƒ|d"< |d
7 }�qðt|d#ddd��0}"tj|"|d$�}#|# ¡  |# |¡ W d  ƒ n1 �sj0    Y  t|ƒt|ƒ|||||	|d%œS )&a  Backfill one live_data day with Polymarket historical prices; write *_polybackfill
    copy. Returns a stats dict. Rows with empty game_date (in-play snapshots that can't be
    grouped to a fixture) are skipped at grouping -- they were the old phantom 'no event'.r/   r   )Únewliner   r   )ÚrowsÚfixturesÚmatchedÚno_eventÚno_histÚfilledÚleaguerv   rƒ   r„   r0   c                 S   s   g | ]}t |d  ƒ‘qS )Ú	timestamp)r   ©r	   ro   r   r   r   Ú
<listcomp>?  r   z!backfill_file.<locals>.<listcomp>c                 S   s   g | ]}|r|‘qS r   r   )r	   r!   r   r   r   rª   @  r   )Ztzinfoi  i0*  r•   )r–   c                 S   s   | d S )Nr   r   )Úxr   r   r   Ú<lambda>H  r   zbackfill_file.<locals>.<lambda>©rw   c                 S   s   g | ]}|d  ‘qS )r   r   ©r	   r«   r   r   r   rª   I  r   c                 S   s   g | ]}|d  ‘qS )Úpr   r®   r   r   r   rª   I  r   )Úpoly_home_priceÚpoly_draw_priceÚpoly_away_priceÚpoly_gap_minr¨   F)rƒ   r‚   r„   Né<   rB   Zpoly_Z_priceTr³   rR   )Z
fieldnames)r¡   r¢   r£   r¤   r¥   r¦   Ú
fix_seriesÚ_rows) rj   ÚcsvZ
DictReaderÚopenr5   ÚCOVEREDÚ
setdefaultr8   Úitemsr1   Ú_LEAGUE_TAG_BY_CODEr”   r�   ÚintrC   r   r   Zutcr¨   ÚTZ_OFFSET_HÚmaxr™   ÚsortÚkeysr   rŸ   ÚMATCH_TOL_MINÚroundZ
DictWriterZwriteheaderZ	writerowsr7   )$Z	live_fileZout_filer¡   r¢   ro   r£   r¦   r¤   r¥   rµ   rw   Úfrowsr‘   r§   rƒ   r„   rl   rŒ   r?   Zdtsrx   ry   ÚseriesZlabelrŽ   r“   Z
out_fieldsr
   r!   r›   ZgapsZany_fillr¯   ZgapZfhrR   r   r   r   Úbackfill_file(  sv    ,
&&$


$



 "2þrÆ   c                     sj  t ttƒ} tt› d| d › d| d › d�ƒ td| d › d| d › d	| d
 › d| d › d�	ƒ td| d › �ƒ tdt› �ƒ tdƒ | d | d  }}d}| ¡ D ]À\‰ }‡ fdd„|D ƒ}|sÄq¤|d }tdˆ d d›dˆ d d d… d›dˆ d  d d… d›d!|d" › d|d# › d|d$ › d%| d&d'¡› d(| d)d'¡› d*|d+ › d,�ƒ |d7 }|d-kr¤ �qfq¤d S ).Nz: r¡   z rows, r¢   z covered fixturesz,
fixtures matched on Polymarket w/ history: r£   ú/z (no event found: r¤   z, no history: r¥   ú)zsnapshot rows filled: r¦   zwrote z;
--- alignment spot-check (last filled row per fixture) ---r¶   rµ   r   c                    sF   g | ]>}|  d ¡|  d¡|  d¡|  d¡fˆ kr|  d¡dkr|‘qS )rv   r§   rƒ   r„   r°   r/   )r5   r©   r­   r   r   rª   }  s   ÿÿÿzmain.<locals>.<listcomp>éÿÿÿÿz  r0   Z4sú r{   é   Z16sz v é   z poly H/D/A=r°   r±   r²   z kalshiHmid=Zkalshi_home_midr/   z pinH=Zpin_fair_homez gap=r³   r�   é   )rÆ   Ú	LIVE_FILEÚOUT_FILEÚprintr»   r5   )r   r¡   rµ   ZshownrÅ   rÄ   ro   r   r­   r   Úmainp  s>    
"ÿÿ
8ÿÿÿ
þ
þý
rÑ   Ú__main__)rq   )r•   )4Ú__doc__r·   ri   r‰   ÚsysÚtimerš   r   r   re   ÚpathÚinsertZpolymarket_liver1   ÚstdoutÚreconfigurerÎ   rÏ   r¼   r»   r¹   rÂ   r   r   r;   r:   r9   r4   r‡   Ú	maketransr3   rA   rE   rP   rQ   rW   rX   rd   rp   r}   rt   rz   r|   r�   r�   r”   r™   rŸ   r¾   rÆ   rÑ   Ú__name__r   r   r   r   Ú<module>   sX   0ÿ
þ
&

H
