a
    ãPejä"  ã                   @   s°   d Z ddlZddlZddlZddlZddlZddlZddlmZ dZ	dZ
dZeejƒdkrheejd ƒndZe d	¡ d
d„ Zdd„ Zdd„ Zdd„ Zdd„ Zedkr¬eƒ  dS )a  
Systematic edge search against the ONLY objective that matters: net CLV in cents.
(read-only, trades nothing)

WHY THIS OBJECTIVE. project_xlow_spread_verdict established that xlow's signal is
real (+0.31c of line movement) but smaller than the 0.67c half-spread it costs to
enter. So "find a pattern that predicts outcomes" is the wrong search -- it is
both maximally noisy (1 bit per fixture, per-bet SD 1.99u) and it ignores the
constraint that actually binds. We mine instead for:

    net = close_mid - entry_ask        (cents)

i.e. "the market's final valuation, minus what I actually paid." Positive mean
net = a configuration where the edge survives execution. This is continuous,
measurable on every fixture-outcome without waiting for results, and has roughly
50x the power of outcome-based testing.

MULTIPLE-TESTING DISCIPLINE. Mining hundreds of slices on ~1,200 fixtures WILL
produce winners by chance -- that is exactly how the day's dead ends (South
America, the 20-25%% bucket) were born. So:
  - the null is built by PERMUTATION at the FIXTURE level, preserving the
    within-fixture correlation of the three outcomes (they sum to 1, so they are
    not independent and naive t-tests overstate significance);
  - we report the max-|t| across ALL slices under that null, which is the correct
    family-wise threshold, not per-slice p-values;
  - the number of hypotheses tested is printed, always.

A slice only counts if its |t| beats the permutation max-|t| threshold.

Usage:  python research\edge_mine.py [n_permutations]
é    N)ÚdefaultdictzE:\soccer\live_data\*.csv)ZkalshiZpolyZpolyus)ÚhomeÚdrawÚawayé   i�  é   c              	   C   s8   zt | ƒ}||kr|nd W S  ttfy2   Y d S 0 d S )N)ÚfloatÚ	TypeErrorÚ
ValueError)ÚxÚv© r   úE:\soccer\research\edge_mine.pyÚ_f/   s
    r   c                     sN  t tƒ} t t¡D ]¤}zˆt|ddd��f}t |¡D ]J}| d¡pBd ¡ dkrNq2| | d¡| d¡| d¡| d	¡f  	|¡ q2W d
  ƒ n1 s’0    Y  W q t
y´   Y qY q0 qg }|  ¡ D �]‚\}}t|ƒdk rÜqÄd
}|D ],}| d¡pôd ¡  ¡ }|dv rä|} �qqä|d }	dtdt|ƒd ƒfdtdt|ƒd ƒffD �]þ\}
}|| ‰tD �]ä‰ ttƒD �]Ò\}‰tˆ ˆ › dˆ› d�¡ƒ}tˆ ˆ › dˆ› d�¡ƒ}tˆ ˆ › dˆ› d�¡ƒ}t|	 ˆ › dˆ› d�¡ƒ}tˆ dˆ› �¡ƒ}d
|||||fv �r�qjd|  k �r,|  k �r,dk �rjn n&d|  k �rHdk �rjn n
|dk�rZ�qj|| | }‡ ‡‡fdd„tD ƒ}dd„ |D ƒ}| 	t||d ˆ ˆ|
||| d || d |d ||�rÒt|ƒ| d nd
|| d || d |�r2|dk�rˆdk�s*|dk�rˆd	k�s*|d k�r.ˆd!k�r.dndnd
d"�¡ �qj�q\�qFqÄ|S )#zKOne record per (fixture, book, side): features at entry, close mid, result.Ú zutf-8)ÚnewlineÚencodingZstatusZPREZ	game_dateZleaguer   r   Né
   Úftr)ÚHÚDÚAéÿÿÿÿÚearlyr   é   Úlateé   Ú_Z_askZ_bidÚ_midZ	pin_fair_r   c              	      s.   g | ]&}|ˆ krt ˆ |› d ˆ› d�¡ƒ‘qS )r   r   )r   Úget)Ú.0Úb©ÚbookÚentryÚsider   r   Ú
<listcomp>a   ó    zload.<locals>.<listcomp>c                 S   s,   g | ]$}|rd |  k r dk rn q|‘qS )r   r   r   )r    Úor   r   r   r&   b   r'   éd   r   r   r   r   )ÚfixtureZlgr#   r%   ÚlagÚpriceÚspreadÚhalfspÚclvÚpfÚdisagreeÚnetÚmvZwon)r   ÚlistÚglobÚLIVEÚopenÚcsvZ
DictReaderr   ÚupperÚappendÚ	ExceptionÚitemsÚlenÚstripÚmaxÚBOOKSÚ	enumerateÚSIDESr   Údict)ZfxÚpZfhÚrÚrecsÚkeyZrowsr   r   ZlastZlag_nameÚidxZsiZeaZebZemÚcmr0   r/   Zothersr   r"   r   Úload7   s‚    ÿÿ(ÿ
N

þÿÿþþþ÷
rJ   c                 C   sN   t | ƒ}|dk rdS t | ¡}t | ¡}|dkr>|t |¡ nd}||| fS )Né   )ç        rL   r   g•Ö&è.>)r=   Ú
statisticsÚmeanZpstdevÚmathZsqrt)r   ÚnÚmZsdZser   r   r   Útstatp   s    

rR   c                    s   i }dD �]‰‡fdd„| D ƒ}||dˆ› �< t D ]&‰ ‡ fdd„|D ƒ|dˆ › dˆ› �< q2tD ]&‰‡fdd„|D ƒ|d	ˆ› dˆ› �< q^d
D ]2\‰‰‡‡fdd„|D ƒ|dˆ› dˆ› dˆ› �< qŠdD ]6\‰‰‡‡fdd„|D ƒ|dˆd›dˆd›dˆ› �< qÂdD ]2\‰‰‡‡fdd„|D ƒ|dˆ› dˆ› dˆ› �< qþdD ]4\‰‰‡‡fdd„|D ƒ|dˆ› dˆ› dˆ› �< �q6t D ]F‰ dD ]:\‰‰‡ ‡‡fdd„|D ƒ|ˆ › dˆ› dˆ› dˆ› �< �qx�qpdD ]N\‰‰dD ]>\‰‰‡‡‡‡fdd„|D ƒ|dˆ› dˆ› dˆ› dˆ› �< �qÈ�q¼qd d!„ | ¡ D ƒS )"zBEvery candidate configuration to test. Returns {label: [rec,...]}.)r   r   c                    s   g | ]}|d  ˆ kr|‘qS ©r+   r   ©r    rE   rS   r   r   r&   ~   r'   zslices.<locals>.<listcomp>zALL/c                    s   g | ]}|d  ˆ kr|‘qS )r#   r   rT   )r!   r   r   r&   �   r'   zbook=ú/c                    s   g | ]}|d  ˆ kr|‘qS )r%   r   rT   )Úsr   r   r&   ƒ   r'   zside=))é�ÿÿÿr   ©r   é   ©rY   é   )r[   é   )r\   éc   c                    s,   g | ]$}ˆ|d    kr ˆ k rn q|‘qS )r/   r   rT   ©ÚhiÚlor   r   r&   …   r'   zclv[ú,z)/))r   ç333333Ã?)rb   ç      Ð?)rc   çš™™™™™Ù?)rd   g      ð?c                    s,   g | ]$}ˆ|d    kr ˆ k rn q|‘qS )r,   r   rT   r^   r   r   r&   ‡   r'   zpx[ú.2f)©r   çš™™™™™é?)rg   çš™™™™™ù?)rh   r]   c                    s,   g | ]$}ˆ|d    kr ˆ k rn q|‘qS )r-   r   rT   r^   r   r   r&   ‰   r'   zspread[))rW   r   )r   r   )r   r]   c                    s8   g | ]0}|d  durˆ|d    kr,ˆ k rn q|‘qS )r1   Nr   rT   r^   r   r   r&   ‹   s   z	disagree[)rX   rZ   ©r[   r]   c                    s8   g | ]0}|d  ˆ krˆ|d   kr,ˆk rn q|‘qS )r#   r/   r   rT   )r!   r_   r`   r   r   r&   �   s   z&clv[)rf   )rg   r]   )rZ   ri   c                    sH   g | ]@}ˆ|d    kr ˆk rn qˆ |d   kr<ˆk rn q|‘qS )r-   r/   r   rT   )Úc0Úc1r_   r`   r   r   r&   ”   s   zspr<c                 S   s"   i | ]\}}t |ƒd kr||“qS )é(   )r=   )r    Úkr   r   r   r   Ú
<dictcomp>–   r'   zslices.<locals>.<dictcomp>)r@   rB   r<   )rF   ÚoutÚbaser   )r!   rj   rk   r_   r+   r`   rV   r   Úslicesz   s:    
$$,0,ÿ ÿ(ÿ,rq   c                     sL  t ƒ } dd„ | D ƒ}tt| ƒ› dt|ƒ› d�ƒ tdƒ t| ƒ}tdt|ƒ› d�ƒ g }| ¡ D ]h\}}tdd	„ |D ƒƒ\}}td
d	„ |D ƒƒ\}}	t dd	„ |D ƒ¡}
| t	|ƒ||||
t|ƒ|f¡ q`|j
dd� ttƒ}| D ]}||d   |¡ qâg }ttƒD ]¬}	g }| ¡ D ]J\}}dd	„ |D ƒ}t |¡ t||ƒD ]\}}| t||d�¡ �q@�qt|ƒ}d}| ¡ D ].\}}tdd	„ |D ƒƒ\}	}t|t	|ƒƒ}�qv| |¡ �q| 
¡  |tdt|ƒ ƒ ‰ tdt› dˆ d›�ƒ tdƒ tdd›ddd›ddd›ddd›dd d›dd!d›d"�ƒ |d d#… D ]b\}}}}}
}}|ˆ k�rRd$nd%}t|d›d|d&›d|d'›d|d'›d|
d(›d|d)›d*|› �ƒ �q2‡ fd+d	„|D ƒ}td,t|ƒ› d-t|ƒ› d.�ƒ |�r@d/d	„ |D ƒ}td0t|ƒ› d1�ƒ |D ]<\}}}}}
}}td2|d3›d4|d&›d5|d6›d7|d8›�ƒ �qò|�sHtd9ƒ ntd:ƒ d S );Nc                 S   s   h | ]}|d  ’qS )r*   r   rT   r   r   r   Ú	<setcomp>›   r'   zmain.<locals>.<setcomp>z& (fixture,book,side,lag) records from z	 fixtureszWobjective: net = close_mid - entry_ask, in cents. Positive = edge survives the spread.
ztesting z slices
c                 S   s   g | ]}|d  ‘qS ©r2   r   rT   r   r   r   r&   ¥   r'   zmain.<locals>.<listcomp>c                 S   s   g | ]}|d  ‘qS )r3   r   rT   r   r   r   r&   ¦   r'   c                 S   s   g | ]}|d  ‘qS )r.   r   rT   r   r   r   r&   §   r'   T)Úreverser*   c                 S   s   g | ]}|d  ‘qS rs   r   rT   r   r   r   r&   ³   r'   rs   rL   c                 S   s   g | ]}|d  ‘qS rs   r   rT   r   r   r   r&   º   r'   gffffffî?z,permutation family-wise |t| threshold (95%, z" perms, shuffled within fixture): re   zZ  (a naive per-slice threshold would be 1.96 -- using it here would manufacture findings)
ÚsliceZ34sú rP   z>5sr2   z>7sZmover.   Útz	  verdictr\   ZSURVIVESr   Z5dz+7.3fz7.3fz+7.2fz  c                    s   g | ]}|d  ˆ kr|‘qS )r   r   rT   ©Zthrr   r   r&   É   r'   Ú
z of z' slices beat the family-wise threshold.c                 S   s   g | ]}|d  dkr|‘qS )rK   r   r   )r    rV   r   r   r   r&   Ì   r'   z  of those, z. have POSITIVE net (an actual tradeable edge):z    Z32sz n=z  net=z+.3fzc  t=z+.2fzK  ALL of them are NEGATIVE -- the mine found ways to lose, not ways to win.z3  No slice survives. Nothing here beats the spread.)rJ   Úprintr=   rq   r<   rR   rM   rN   r:   ÚabsÚsortr   r4   ÚrangeÚNPERMÚrandomZshuffleÚziprC   r?   Úint)rF   ZfixturesZslÚresZlabelr   rQ   rw   r3   r   ZhsZbyfixrE   ZnullsZpoolÚfZrsZvalsZsl2ZmxZatrP   ZsurvÚposr   rx   r   Úmain™   sd    "

ÿ8@,
r…   Ú__main__)Ú__doc__Úsysr8   r5   rO   r   rM   Úcollectionsr   r6   r@   rB   r=   Úargvr�   r~   Zseedr   rJ   rR   rq   r…   Ú__name__r   r   r   r   Ú<module>   s&    
9
>