a
    )ªhj(  ã                   @   sÀ   d Z ddlZddlZddlZddlZddlmZ ej dd¡ ej dd¡ ddl	Z
ddlZddlZdZdZdZd	Zd
d„ Zdd„ Zdd„ Zdd„ Zddd„Zdd„ Zdd„ Zedkr¼eƒ  dS )aŠ  Does SoFIFA squad strength beat, or add to, the market?

TURNKEY: drop a CSV at data/sofifa_players.csv (or data/sofifa_teams.csv) and run
this. Nothing else to configure.

WHY YOU HAVE TO SUPPLY THE FILE. sofifa.com is behind a Cloudflare JS challenge;
the public scrapers get past it by driving a real browser, which is circumventing
an access control against their terms. Not doing that. Export it from your own
session, or use a published dataset (Kaggle EA FC player data), and put it here.

ACCEPTED SHAPES (auto-detected):
  players: needs a club name column and `overall_rating` (or `overall`).
           Team strength = mean of that club's top 11 by rating.
  teams:   needs a club name column and `overall` (optionally attack/midfield/
           defence). Used directly.

WHAT IT TESTS, in the order that matters:
  1. NAME MATCH RATE first. Squad ratings are worthless if the clubs do not join
     to our fixtures. Reported before any modelling, and the run aborts if it is
     too low -- a 30%-matched test silently measures the leagues that happen to
     spell names the same way.
  2. Rating differential ALONE vs the Pinnacle close (log loss).
  3. Market + rating vs market alone, time-split, paired t. Success criterion is
     fixed BEFORE the run: paired t > 1.96. Three previous feature families were
     declared dead by exactly this bar; SoFIFA gets the same one, no softer.
  4. THE NO-MODEL CONTROL. project_edge_mine_exhausted records a trap: a model
     showed +2.7% ROI while being no better than the market on log loss, because
     it was just selecting favourites in a table where proportional devig
     understates them. So any betting-style result here is reported next to the
     same slice bet with NO model.

KNOWN LIMITATION, stated not buried: a single ratings snapshot applied to past
fixtures is mildly look-ahead (ratings partly reflect the season being tested).
That biases IN FAVOUR of the ratings. If they lose even with that help, the
verdict is safe; if they win narrowly, it needs point-in-time ratings
(sofifa supports historical versions via its `r=YYMMDD` parameter) before
believing it.

Usage:  python research\sofifa_test.py
é    N)ÚdefaultdictzE:\soccer\researchzE:\soccer\mainz!E:\soccer\data\sofifa_players.csvzE:\soccer\data\sofifa_teams.csvgffffffÖ?é   c                 C   s   d  dd„ | pd ¡ D ƒ¡S )NÚ c                 s   s   | ]}|  ¡ r|V  qd S ©N)Úisalnum)Ú.0Zch© r   ú!E:\soccer\research\sofifa_test.pyÚ	<genexpr><   ó    znorm.<locals>.<genexpr>)ÚjoinÚlower©Úsr   r   r	   Únorm;   s    r   c              	   C   s(   z
t | ƒW S  ttfy"   Y d S 0 d S r   )ÚfloatÚ	TypeErrorÚ
ValueError)Úxr   r   r	   Ú_f?   s    
r   c                 G   s`   dd„ | D ƒ}|D ]}||v r||   S q|D ]*}|  ¡ D ]\}}||v r<|    S q<q0d S )Nc                 S   s   i | ]}|  ¡  ¡ |“qS r   )r   Ústrip)r   Úcr   r   r	   Ú
<dictcomp>G   r   zpick.<locals>.<dictcomp>)Úitems)ZcolsZcandsZlowr   ÚkÚvr   r   r	   ÚpickF   s    r   c                  C   s  t j t¡rÐttdddd��¤} t | ¡}t|jdddƒ}t|jdd	d
ƒ}|rP|sdt	t› d|j› �ƒ‚i }|D ](}t
| |¡ƒ}|rl||t| |¡ƒ< qltdt|ƒ› dt› �ƒ |W  d  ƒ S 1 sÆ0    Y  t j t¡sðt	dt› dt› �ƒ‚ttdddd��œ} t | ¡}t|jdddƒ}t|jddd	d
ƒ}|�r:|�sNt	t› d|j› �ƒ‚ttƒ}|D ]2}t
| |¡ƒ}|�rZ|t| |¡ƒ  |¡ �qZW d  ƒ n1 �s¤0    Y  dd„ | ¡ D ƒ}tdt|ƒ› dt› dt› dtdd„ | ¡ D ƒƒt|ƒd  › d�	ƒ |S )zJclub_norm -> rating. From a teams file if present, else aggregate players.r   zutf-8Úreplace)ÚnewlineÚencodingÚerrorsZ	club_nameZteamÚnameZoverallZovrZratingz.: need a club-name and an overall column; saw zloaded z team ratings from Nzno data: put a CSV at z or ZclubZoverall_ratingz): need a club and an overall column; saw c                 S   s<   i | ]4\}}|r|t t|d d�dt… ƒtt|ƒtƒ “qS )T)ÚreverseN)ÚsumÚsortedÚTOP_NÚminÚlen©r   r   r   r   r   r	   r   r   s   
ÿz!load_strength.<locals>.<dictcomp>z clubs from z (top-z mean; median squad size c                 s   s   | ]}t |ƒV  qd S r   )r'   ©r   r   r   r   r	   r
   u   r   z load_strength.<locals>.<genexpr>é   ú))ÚosÚpathÚexistsÚTEAMSÚopenÚcsvZ
DictReaderr   Z
fieldnamesÚ
SystemExitr   Úgetr   Úprintr'   ÚPLAYERSr   ÚlistÚappendr   r%   r$   Úvalues)ZfhZrdZcnZovÚoutÚrr   Zsquadsr   r   r	   Úload_strengthR   sP    
ÿ"
ÿ<ÿÿ ÿ
r;   é  çìQ¸…ëÑ?c                    s¸  t | ƒt | d ƒd  }‰}‡fdd„t|ƒD ƒ‰ dg| ‰dt|dƒ }t|ƒD �]Z}‡fdd„t|ƒD ƒ}	dg| }
t| |ƒD ]º\‰}‡ ‡‡‡fd	d„t|ƒD ƒ}t|ƒ‰‡fd
d„|D ƒ}t|ƒ‰‡fdd„|D ƒ}t|ƒD ]X}|| ||krødnd }|
|  |7  < tˆƒD ]"}|	| |  |ˆ|  7  < �qqâq‚t|ƒD ]f}ˆ|  ||
|  | 8  < tˆƒD ]:}ˆ | |  ||	| | | |ˆ | |    8  < �qn�qFqRˆ ˆfS )zFMultinomial logistic, L2 = 1/C. Plain GD -- small feature counts only.r   é   c                    s   g | ]}d gˆ  ‘qS ©ç        r   ©r   Ú_©Údr   r	   Ú
<listcomp>|   r   z logistic_fit.<locals>.<listcomp>r@   ç      ð?ç•Ö&è.>c                    s   g | ]}d gˆ  ‘qS r?   r   rA   rC   r   r	   rE   €   r   c                    s2   g | ]*‰ t ‡‡ ‡fd d„tˆƒD ƒƒˆˆ   ‘qS )c                 3   s"   | ]}ˆ ˆ | ˆ|  V  qd S r   r   ©r   Új©ÚWr   Úxir   r	   r
   ƒ   r   z*logistic_fit.<locals>.<listcomp>.<genexpr>)r#   Úrange©r   )rK   ÚbrD   rL   ©r   r	   rE   ƒ   r   c                    s   g | ]}t  |ˆ  ¡‘qS r   ©ÚmathZexpr)   ©Úmr   r	   rE   …   r   c                    s   g | ]}|ˆ  ‘qS r   r   r)   r   r   r	   rE   ‡   r   )r'   rM   ÚmaxÚzipr#   )ÚXÚYÚCZitersZlrÚnÚKZlamrB   ZgWZgbZyiÚzÚeÚpr   ÚgrI   r   )rK   rO   rD   rT   r   rL   r	   Úlogistic_fity   s.    

$>r`   c                    sN   ‡ ‡‡fdd„t dƒD ƒ}t|ƒ‰‡fdd„|D ƒ}t|ƒ‰‡fdd„|D ƒS )Nc                    s6   g | ].‰ t ‡‡ ‡fd d„ttˆƒƒD ƒƒˆˆ   ‘qS )c                 3   s"   | ]}ˆ ˆ | ˆ|  V  qd S r   r   rH   rJ   r   r	   r
   •   r   z%predict.<locals>.<listcomp>.<genexpr>)r#   rM   r'   rN   )rK   rO   rL   rP   r	   rE   •   r   zpredict.<locals>.<listcomp>r>   c                    s   g | ]}t  |ˆ  ¡‘qS r   rQ   r)   rS   r   r	   rE   —   r   c                    s   g | ]}|ˆ  ‘qS r   r   r)   r   r   r	   rE   ™   r   )rM   rU   r#   )rK   rO   rL   r\   r]   r   )rK   rO   rT   r   rL   r	   Úpredict”   s
    ra   c            )         sX  t ƒ } t ¡ }t ¡ ‰g tƒ ttƒ  }}}|D �]}|d t t	 
|d ¡¡t t	 
|d ¡¡f}ˆ |¡}|stq0|\}}	tj |	¡}
|
d u r’q0|  t|d ƒ¡|  t|d ƒ¡ }}| t|d ƒ¡ |d u ræ||d   d7  < |d u �r||d   d7  < |d u s0|d u �rq0| |d ||
|||d dœ¡ q0t‡fdd„|D ƒƒ}|�rbt|ƒ| nd	}td
t|ƒ› d|› d|d d›d�ƒ |�rÄtdd dd„ t| ¡ dd„ d�d d… D ƒ¡ƒ |tk �ràtdtd›d�ƒ‚|jdd„ d� tt|ƒd ƒ}|d |… ||d …  }}tdt|ƒ› dt|ƒ› �ƒ dd„ }|d d!„ |D ƒd"d!„ |D ƒƒ}td#|d$›�ƒ d%d&„ ‰td'd„ |D ƒƒt|ƒ d( �p–d)‰‡fd*d„‡fd+d„‡‡fd,d„d-œ}i }| ¡ D ]â\}‰‡fd.d!„|D ƒ}d/d!„ |D ƒ}‡fd0d!„|D ƒ}d }d1D ]^}t|||ƒ\‰ ‰|‡ ‡fd2d!„|D ƒd3d!„ |D ƒƒ}|d u �s\||d	 k �r||ˆ ˆf}�q|||< td4|d5›d6|d	 d$›d7|d › d8||d	  d9›�ƒ �qÊ|d: \}}‰ ‰|d; \}}}} g }!|D ]h}"t ttˆ ˆ|d: |"ƒƒ|"d<  d=ƒ¡ }#t tt|| |d; |"ƒƒ|"d<  d=ƒ¡ }$|! |$|# ¡ �qÖt|!ƒt|!ƒ ‰t‡fd>d„|!D ƒƒt|!ƒd  d( }%|%�r�ˆ|%t|!ƒd(   nd?}&td@ˆd9›dA|&dB›dCt|!ƒ› �ƒ tdD|&dEk�rÈdFndG ƒ tdHƒ dId!„ |D ƒ}'|'�rTtdJd„ |'D ƒƒ}tdKd„ |'D ƒƒt|'ƒ }(tdLt|'ƒ› dM|(d dN›dO|t|'ƒ d dN›dP�ƒ tdQƒ d S )RNÚdateÚhomeÚawayé   Údiv)rb   r^   ÚyÚshÚsarf   c                 3   sF   | ]>}ˆ   |d  t t |d ¡¡t t |d ¡¡f¡rdV  qdS )rb   rc   rd   re   N)r3   ÚTr   ÚDSÚ	fdco_name)r   Úf)Úoddsr   r	   r
   ·   s   
,ÿzmain.<locals>.<genexpr>r   z
1. NAME MATCH: ú/z fixtures (éd   z.1fz%) have BOTH clubs ratedz   most-missed clubs:z, c                 s   s"   | ]\}}|› d |› d�V  qdS )z (r+   Nr   r(   r   r   r	   r
   ½   s   c                 S   s
   | d  S )Nre   r   )Zkvr   r   r	   Ú<lambda>¾   r   zmain.<locals>.<lambda>)Úkeyé   z
ABORT: match rate below z.0%zs. Fix the club-name join before testing -- a partial join silently tests only the leagues that spell names our way.c                 S   s   | d S )Nrb   r   ©r:   r   r   r	   rq   Ä   r   gffffffæ?z   time split: train z	 -> test c                 S   s    t dd„ t| |ƒD ƒƒt|ƒ S )Nc                 s   s(   | ] \}}t  t|| d ƒ¡ V  qdS )çê-�™—q=N©rR   ÚlogrU   )r   r^   rg   r   r   r	   r
   Ê   r   z#main.<locals>.ll.<locals>.<genexpr>)r#   rV   r'   )ZpsZysr   r   r	   ÚllÉ   s    zmain.<locals>.llc                 S   s   g | ]}|d  ‘qS ©r^   r   ©r   r:   r   r   r	   rE   Ì   r   zmain.<locals>.<listcomp>c                 S   s   g | ]}|d  ‘qS ©rg   r   rz   r   r   r	   rE   Ì   r   z0
2. MARKET BASELINE (Pinnacle close)   log loss z.5fc                 S   s   dd„ | D ƒS )Nc                 S   s   g | ]}t  t|d ƒ¡‘qS )rG   rv   r)   r   r   r	   rE   Ð   r   z'main.<locals>.mklog.<locals>.<listcomp>r   ry   r   r   r	   ÚmklogÏ   s    zmain.<locals>.mklogc                 s   s"   | ]}|d  |d  d V  qdS )rh   ri   r*   Nr   rz   r   r   r	   r
   Ò   r   g      à?rF   c                    s   | d | d  ˆ  dgS )Nrh   ri   rF   r   rt   )Úsdr   r	   rq   Ô   r   c                    s   ˆ | d ƒS )Nr^   r   rt   )r|   r   r	   rq   Õ   r   c                    s"   ˆ | d ƒ| d | d  ˆ g S )Nr^   rh   ri   r   rt   )r|   r}   r   r	   rq   Ö   r   )zrating diff ALONEúmarket only (control)úmarket + rating diffc                    s   g | ]}ˆ |ƒ‘qS r   r   rz   ©Úfnr   r	   rE   Ú   r   c                 S   s   g | ]}|d  ‘qS r{   r   rz   r   r   r	   rE   Û   r   c                    s   g | ]}ˆ |ƒ‘qS r   r   rz   r€   r   r	   rE   Ü   r   )g{®Gáz„?gš™™™™™¹?rF   c                    s   g | ]}t ˆ ˆ|ƒ‘qS r   )ra   ©r   r   )rK   rO   r   r	   rE   à   r   c                 S   s   g | ]}|d  ‘qS r{   r   rz   r   r   r	   rE   à   r   z   Z24sz
 log loss z
  (best C=z)  vs market z+.5fr   r~   rg   ru   c                 3   s   | ]}|ˆ  d  V  qdS )r*   Nr   r‚   )Úmur   r	   r
   ð   r   r@   z(
3. PAIRED TEST vs market-only control: z per fixture, t = z+.2fz on n=z"   PRE-REGISTERED BAR: t > 1.96.  g\�Âõ(\ÿ?z*** PASSES ***z*does not pass -> same verdict as Elo/form.z3
4. NO-MODEL CONTROL (the favourite-selection trap)c                 S   s    g | ]}t |d  ƒdkr|‘qS )r^   gš™™™™™Ù?©rU   rz   r   r   r	   rE   ù   r   c                 s   s0   | ](}|d    t|d  ƒ¡|d krdV  qdS )r^   rg   re   N)ÚindexrU   rz   r   r   r	   r
   û   r   c                 s   s   | ]}t |d  ƒV  qdS )r^   Nr„   rz   r   r   r	   r
   ü   r   z'   betting favourites with NO model: n=z	 implied z.2fz	% actual ú%z?   Any "profit" from the rating model must beat THIS, not zero.)r;   ÚEZload_fixturesrj   Z	load_oddsÚsetr   Úintr   rk   rl   r3   ZIDXÚaddr7   r#   r'   r4   r   r$   r   ÚMIN_MATCH_RATEr2   Úsortr`   rR   rw   rU   ra   ))ZstrengthZfxZrowsÚseenZmissrm   rr   Zhitr^   ÚresÚirh   ri   Z
total_oddsZrateZcutZtrZterx   ZmktZfeatsetsZbestZnmZXtrZYtrZXteZblrY   r   ZmvZmCZmWZmbrD   r:   Úar   ZsdvÚtZfavÚimpr   )rK   rO   r�   r|   rƒ   rn   r}   r	   Úmainœ   s´    
.
&
ÿ
&ÿ

$

ý$"ÿ**&ÿÿÿÿÿ
r“   Ú__main__)r<   r=   )Ú__doc__r1   rR   r,   ÚsysÚcollectionsr   r-   ÚinsertZelo_model_testr‡   Znonprice_edge_testrj   Zdaily_signalsrk   r5   r/   r‹   r%   r   r   r   r;   r`   ra   r“   Ú__name__r   r   r   r	   Ú<module>   s.   ('
f