a
    åRejA  ã                   @   s    d Z ddlZddlZddlZddlZddlmZ ddlmZ ej 	dd¡ ddl
ZdZdZddd	d
œZdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zedkrœeƒ  dS )aÛ  
Does NON-PRICE information (API-Football team stats) beat the market? (read-only)

THE QUESTION. Everything mined so far has been derived from prices, and
[[project_edge_mine_exhausted]] showed that whole space is structurally dead:
predictable movement sits exactly where the spread is widest. The one untouched
hypothesis is genuinely different information -- form, shots, possession,
discipline -- and we already hold 70k team-match rows of it in raw_api/.

THE ONLY TEST THAT MATTERS. Not "can a model predict football" -- of course it
can, badly. The question is whether it knows something the CLOSING PRICE does
not. So:

    baseline  = Pinnacle closing odds, devigged        (the market)
    model     = API features alone
    combined  = API features + the market's own probability

If `combined` does not beat `baseline` out-of-sample on log loss, the features
add nothing the price has not already absorbed, and there is no edge to trade
regardless of how good the model looks in isolation.

DISCIPLINE.
  - Rolling features are .shift(1) lagged in refresh_api_football.build_features,
    so no same-match leakage. Verified before writing this.
  - TIME-BASED split (train on earlier seasons, test on later). Random CV would
    leak: teams' form is autocorrelated and the market improves over time.
  - Any edge found must then clear the ~0.67c half-spread -- see
    project_xlow_spread_verdict. Log-loss improvement alone is not money.

Usage:  python research\nonprice_edge_test.py
é    N)Údefaultdict)ÚdatetimezE:\soccer\mainz$E:\soccer\raw_api\match_features.csvzE:\soccer\league_data\*.csvé   é   ©ÚHÚDÚAc              	   C   s8   zt | ƒ}||kr|nd W S  ttfy2   Y d S 0 d S ©N)ÚfloatÚ	TypeErrorÚ
ValueError)ÚxÚv© r   ú(E:\soccer\research\nonprice_edge_test.pyÚ_f/   s
    r   c                 C   s    | pd  ¡  ¡  dd¡ dd¡S )NÚ Ú.ú-ú )ÚstripÚlowerÚreplace©Úsr   r   r   Únorm7   s    r   c            	         sž  i } t   t¡D �]ˆ}�zht|dddd���@}t |¡D �]"‰ ˆ  d¡pHd ¡  ¡ }|tvr\q6‡ fdd„dD ƒ}t	|ƒrˆt
d	d
„ |D ƒƒrš‡ fdd„dD ƒ}t	|ƒr6t
dd
„ |D ƒƒr¶q6ˆ  d¡pÂd ¡ }dD ]:}zt ||¡ ¡ }W  �qW qÌ t�y   d}Y qÌ0 qÌ|�sq6dd„ |D ƒ}t|ƒ‰‡fdd„|D ƒ|f| |tˆ  d¡ƒtˆ  d¡ƒf< q6W d  ƒ n1 �sr0    Y  W q t�y–   Y qY q0 q| S )z<(date, home_fd, away_fd) -> devigged Pinnacle closing probs.r   úutf-8r   )ÚnewlineÚencodingÚerrorsZFTRc                    s   g | ]}t ˆ  |¡ƒ‘qS r   ©r   Úget©Ú.0Úc©Úrr   r   Ú
<listcomp>E   ó    zload_odds.<locals>.<listcomp>)ZPSCHZPSCDZPSCAc                 s   s   | ]}|d kV  qdS ©r   Nr   ©r$   r   r   r   r   Ú	<genexpr>F   r)   zload_odds.<locals>.<genexpr>c                    s   g | ]}t ˆ  |¡ƒ‘qS r   r!   r#   r&   r   r   r(   G   r)   )ZPSHZPSDZPSAc                 s   s   | ]}|d kV  qdS r*   r   r+   r   r   r   r,   H   r)   ZDate)z%d/%m/%Yz%d/%m/%yNc                 S   s   g | ]}d | ‘qS )r   r   r+   r   r   r   r(   S   r)   c                    s   g | ]}|ˆ  ‘qS r   r   r+   r   r   r   r(   V   r)   ZHomeTeamZAwayTeam)ÚglobÚLEAGUEÚopenÚcsvÚ
DictReaderr"   r   ÚupperÚIDXÚallÚanyr   ÚstrptimeÚdater   Úsumr   Ú	Exception)	ÚoutÚpÚfhÚresÚoÚdZfmtÚdtÚimpr   )r'   r   r   Ú	load_odds;   s:    ÿFrB   c                     s~  t tƒ} d}ttddd��`}t |¡D ]D}|du rDtdd„ |D ƒƒ}| d¡dv rVd	nd
}|| |d  |< q&W d  ƒ n1 s€0    Y  g }|  ¡ D ]Þ\}}d	|vs–d
|vr°q–|d	 |d
  ‰‰ z t	 
ˆd dd… d¡ ¡ }W n tyø   Y q–Y n0 ‡fdd„|D ƒ}	‡ fdd„|D ƒ}
tdd„ |	D ƒƒs–tdd„ |
D ƒƒ�rFq–| t||ˆ d¡ˆ d¡ˆ d¡|	|
d�¡ q–||fS )z?fixture -> (home_feats, away_feats, meta). One row per fixture.Nr   r   )r   r   c                 s   s   | ]}|  d ¡r|V  qdS )Zapi_N)Ú
startswithr#   r   r   r   r,   c   r)   z load_features.<locals>.<genexpr>Zis_home)Ú1z1.0ÚTrueÚhÚaZ
fixture_idr7   é
   z%Y-%m-%dc                    s   g | ]}t ˆ  |¡ƒ‘qS r   r!   r#   )rF   r   r   r(   o   r)   z!load_features.<locals>.<listcomp>c                    s   g | ]}t ˆ  |¡ƒ‘qS r   r!   r#   )rG   r   r   r(   p   r)   c                 s   s   | ]}|d u V  qd S r
   r   r+   r   r   r   r,   q   r)   ÚdivZ	home_teamZ	away_team)Úfidr7   rI   ÚhomeÚawayÚhvÚav)r   Údictr/   ÚFEATr0   r1   Úsortedr"   Úitemsr   r6   r7   r   r5   Úappend)ZbyfixÚcolsr<   r'   Zsider:   rJ   r?   r@   rM   rN   r   )rG   rF   r   Úload_features\   s4    0 
&þ
rU   c                     s  t ƒ } tƒ \}}tdt| ƒ› dt|ƒ› �ƒ g }|D ]ˆ}|d tt |d ¡ƒtt |d ¡ƒf}|  |¡}|srq4|\}}| t	|d |||d |d  dd	„ t
|d |d ƒD ƒ d
�¡ q4tdt|ƒ› �ƒ t|ƒdk rètdƒ d S |jdd„ d� tt|ƒd ƒ}	|d |	… ||	d …  }
}tdt|
ƒ› d|
d d › dt|ƒ› d|d d › d�	ƒ dd l}ddlm} dd„ }dd	„ |
D ƒ}dd	„ |D ƒ}| dd	„ |
D ƒ¡}| dd	„ |D ƒ¡}| d d	„ |
D ƒ¡}| d!d	„ |D ƒ¡}|d"d	„ |D ƒ|ƒ}td#d$›d%d&d'›d(�ƒ td)d$›d%|d*›d+�ƒ d,||fd-| ||g¡| ||g¡ffD ]–\}}}|d.d/d0dd1�}| ||¡ | |¡‰t|jƒ‰ ‡ ‡fd2d	„tt|ƒƒD ƒ}|||ƒ}t|d$›d%|d*›d3|| d4›||k �rÚd5nd6› �ƒ �qPtd7ƒ td8ƒ td9ƒ d S ):Nzodds rows: z   feature fixtures: r7   rK   rL   rM   rN   c                 S   s   g | ]\}}|| ‘qS r   r   )r$   r   Úyr   r   r   r(   †   r)   zmain.<locals>.<listcomp>)r7   r;   r=   r   z1MATCHED fixtures (features + Pinnacle + result): iô  zCtoo few matched -- name translation is the bottleneck, not the ideac                 S   s   | d S )Nr7   r   r&   r   r   r   Ú<lambda>Œ   r)   zmain.<locals>.<lambda>)Úkeygffffffæ?z  train z (to éÿÿÿÿz	)   test z (r   z on)
)ÚHistGradientBoostingClassifierc                 S   s"   t dd„ t| |ƒD ƒƒ t|ƒ S )Nc                 s   s*   | ]"\}}t  t|t|  d ƒ¡V  qdS )g•Ö&è.>N)ÚmathÚlogÚmaxr3   )r$   r;   rV   r   r   r   r,   –   r)   z#main.<locals>.ll.<locals>.<genexpr>)r8   ÚzipÚlen)ÚprobsZysr   r   r   Úll•   s    zmain.<locals>.llc                 S   s   g | ]}|d  ‘qS ©r=   r   ©r$   r'   r   r   r   r(   ˜   r)   c                 S   s   g | ]}|d  ‘qS rb   r   rc   r   r   r   r(   ™   r)   c                 S   s   g | ]}|d  ‘qS ©r   r   rc   r   r   r   r(   š   r)   c                 S   s   g | ]}|d  ‘qS rd   r   rc   r   r   r   r(   ›   r)   c                 S   s   g | ]}|d  ‘qS ©r;   r   rc   r   r   r   r(   œ   r)   c                 S   s   g | ]}|d  ‘qS re   r   rc   r   r   r   r(   �   r)   c                 S   s   g | ]}|d  ‘qS re   r   rc   r   r   r   r(   Ÿ   r)   ZmodelZ34sr   ztest log lossz>14sz   vs marketzMARKET (Pinnacle closing)z14.5fz   --zAPI features ALONEzAPI features + market probé   i,  g¸…ëQ¸®?)Z	max_depthZmax_iterZlearning_rateZrandom_statec                    s"   g | ]‰ ‡ ‡‡fd d„dD ƒ‘qS )c                    s   g | ]}ˆˆ  ˆ  |¡ ‘qS r   )Úindex)r$   Úk)ÚiÚorderÚprr   r   r(   «   r)   z#main.<locals>.<listcomp>.<listcomp>r   r   )r$   ©rj   rk   )ri   r   r(   «   s   ÿz   z+.5fz  BEATS MARKETr   zI
A negative delta means the model is WORSE than simply reading the price.zIOnly "features + market" beating "market" is evidence of new information;zCand even then it must clear the ~0.67c half-spread to be tradeable.)rB   rU   Úprintr_   r   ÚDSZ	fdco_namer"   rS   rO   r^   ÚsortÚintZnumpyZsklearn.ensemblerZ   ZarrayZhstackZfitZpredict_probaÚlistZclasses_Úrange)ZoddsZfeatsrT   ZrowsÚfrX   Zhitr;   r=   ZcutZtrZteZnprZ   ra   ZytrZyteZXtrZXteZMtrZMteÚbaseZlabelr	   ÚBÚmr`   r   r   rl   r   Úmainy   sp    
*
*ÿ
&
ÿ
ÿÿÿ


ÿ
ÿrw   Ú__main__)Ú__doc__Úsysr0   r-   r[   Úcollectionsr   r   ÚpathÚinsertZdaily_signalsrn   rP   r.   r3   r   r   rB   rU   rw   Ú__name__r   r   r   r   Ú<module>   s$   !=