a
    s†€jš  ã                   @   sÊ   d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
Zej dd¡ ej dd¡ ddlZddlZe	dƒZe	dƒZe	d	ƒZg d
¢Zdd„ Zdd„ Zdd„ Zdd„ ZedkrÆeƒ  dS )a  Can a regression pick which books to quote, under a QUEUE-AWARE fill model?

WHY THIS SUPERSEDES fill_model.py. That one labelled a leg "filled" if any print
landed at or above our ask -- the optimistic test queue_aware_fill.py was
written to discredit. At the touch we join an ask with thousands of contracts
already resting on it, so a print at our price is usually somebody else's fill.
Selecting books on a label that is inflated exactly where we intend to quote is
worse than not selecting at all.

THE LABEL HERE. Posting at price P with Q contracts resting at P, we fill once
the cumulative size of trades at price >= P after we post exceeds Q. Trades
ABOVE our price count: the book walked up through our level. Q is the depth at
the touch at post time. This is queue_aware_fill.py's model, applied to every
leg rather than one day.

  Optimistic label -> upper bound.  This label -> lower bound (nobody ahead of
  us ever cancels).  The truth is between, and that gap is the honest error bar.

THE SIMULATION. Quote all three legs at the touch, no premium, no repairs:
    all three fill  -> collect the overround, outcome-independent
    partial         -> the filled legs settle naked
Then fit P(all three fill) on post-time features, split TEMPORALLY, and ask
whether selecting on it beats quoting everything. A gate can only remove books,
so it has to earn its keep by removing losers.

Sized trade tape is cached to .tape_sized_cache.json (one number per leg), so
the slow part happens once.

Usage:  python research\book_selection.py
é    N)Údefaultdict)ÚPathzE:\soccer\mainzE:\soccer\researchz&E:\soccer\research\mm_paper_quotes.csvz'E:\soccer\research\mm_paper_results.csvz)E:\soccer\research\.tape_sized_cache.json)ÚtouchZqueue_qZspread_cZoverround_cZmin_leg_vol_kZvol24_kZopen_interest_kZ
ask_size_kZ
bid_size_kZ	hrs_to_koZis_drawZ
fair_gap_cc              	   C   s(   z
t | ƒW S  ttfy"   Y d S 0 d S )N)ÚfloatÚ	TypeErrorÚ
ValueError©Úx© r
   ú$E:\soccer\research\book_selection.pyÚ_f7   s    
r   c                     s:  i } t jdddd��@}t |¡D ]$}| d¡r t| d¡ƒ| |d < q W d   ƒ n1 sZ0    Y  tt ¡ ƒd }t	 
t|¡ i }|jdddd��L}t |¡D ]0}| d¡}|r ||vr | d¡s ds |||< q W d   ƒ n1 sæ0    Y  t ¡ �rt t ¡ ¡ni }ttƒ}| ¡ D ]6\}}t| d	¡ƒ}|d u�r|| d
¡  |¡ �qi d }	}
tt| ¡ ƒdƒD �]š\}\}}t | d¡¡t | d¡¡ }‰ t| d¡ƒt| d¡ƒ ‰}t| d¡ƒt| d¡ƒ }}|  |¡}| | d
¡¡�púg }d |ˆ ˆ||||fv �sp|�s�qpd|  k �rBˆ  k �rBdk �sHn �qpˆ |  ¡ d }d|  k �rpdk�svn �qp||v �rŠ|| }n”zt ||¡}W n t�y²   g }Y n0 t‡ ‡fdd„|D ƒƒ}|||< |
d7 }
|
d dk�rt t |¡¡ t d|
› d|› dt!|ƒ› d�dd� t| d¡ƒ}| d
¡ˆ ˆ||||d d  ||k�rVdndˆ|d! ˆ| d  |d d  t"|ƒd! t| d"¡ƒ�p�dd! t| d#¡ƒ�p¦dd! |d! t| d$¡ƒ�pÂdd! |t#| d%¡ƒ $¡ d&k�rädnd'|d u�rü|ˆ d  nd'gd(œ	|	|< �qpt t |¡¡ t t!|	ƒ› d)|
› d*�ƒ |	S )+NÚ zutf-8Úreplace)ÚnewlineÚencodingÚerrorsZ
leg_tickerÚsettledzmm_paper_quotes.snap.csvÚreasonZvolumeÚeventr   é   ÚtsZkickoffZk_askZk_bidZover_askZask_sizeg      ¬@é   c                 3   s,   | ]$\}}}|ˆd  kr|ˆ kr|V  qdS )g•Ö&è.>Nr
   )Ú.0ZttÚpÚc©Úkor   r
   r   Ú	<genexpr>k   ó    zbuild.<locals>.<genexpr>é   z  ... z tape reads (ú/ú)T)ÚflushZpin_fairg      ð?ç      Y@g     @�@Z
volume_24hZopen_interestZbid_sizeZsideZdrawg        )	r   r   r   r   ÚqÚvol_atÚover_cÚfillr	   z legs scored (z new tape reads)
)%ÚRESULTSÚopenÚcsvZ
DictReaderÚgetr   r   ÚtempfileZ
gettempdirÚshutilZcopyfileÚQUOTESÚCACHEÚexistsÚjsonÚloadsÚ	read_textr   ÚlistÚitemsÚappendÚ	enumerateÚsortedÚMSZ_ptZtotal_secondsÚQAZtrades_sizedÚ	ExceptionÚsumZ
write_textÚdumpsÚprintÚlenÚminÚstrÚlower)r   ZfhÚrÚtmpÚfirstÚtÚcacheZbookvolÚvÚlegsÚnewÚir   ZkbZoverr$   ÚsZvolsZhrsr%   ZtrZfairr
   r   r   Úbuild>   sx    
6
(

 "
&


$ ûü
rM   c           	   	   C   sÀ   t tƒ}|  ¡ D ]\}}||d   |¡ qg }| ¡ D ]r\}}t|ƒdkrNq8tdd„ |D ƒƒ}|rp|d d ntdd„ |D ƒƒ}| |tdd„ |D ƒƒt|ƒ||d	œ¡ q8|j	d
d„ d� |S )Nr   é   c                 s   s   | ]}|d  V  qdS ©r'   Nr
   ©r   r	   r
   r
   r   r   ‰   r   zbooks_from.<locals>.<genexpr>r   r&   c                 s   s*   | ]"}|d  r|d |d  d V  qdS )r'   r   r   r#   Nr
   rP   r
   r
   r   r   �   r   c                 s   s   | ]}|d  V  qdS )r   Nr
   rP   r
   r
   r   r   Ž   r   )Úevr   ÚcompÚpnl_crI   c                 S   s   | d S )Nr   r
   )Úzr
   r
   r   Ú<lambda>�   r   zbooks_from.<locals>.<lambda>)Úkey)
r   r4   r5   r6   r?   Úallr<   r@   ÚintÚsort)	rI   ÚbrF   rH   ÚoutrQ   Zv3rR   Zpnlr
   r
   r   Ú
books_from�   s     ÿÿ
r\   c                     sä  t ƒ } | stdƒ d S tdd„ |  ¡ D ƒƒt| ƒ }tdd| d›d�ƒ t| ƒ}t|ƒdk rvtd	t|ƒ› d
�ƒ d S tdd„ |D ƒƒt|ƒ }tdd„ |D ƒƒd }tdt|ƒ› dd| d›d�ƒ td|d›d|t|ƒ d d›d�ƒ ddlm} ddlm	} ddl
m} tt|ƒd ƒ}|d |… ||d …  }	}
t dd„ |	D ƒt¡}t dd„ |	D ƒt¡}| ¡ dk �s†d|  ¡ dk �r’tdƒ d S ||ƒ |dd �ƒ}| ||¡ td!t|	ƒ› d"|	d d# d$›d%|	d& d# d$›d't|
ƒ› d(|
d d# d$›d%|
d& d# d$›d)�ƒ g }|
D ]J}tt | t d*d„ |d+ D ƒt¡¡d d …df ¡ƒ}| ||f¡ �qtd,d-d.›d/d0d1›d/d2d3›d/d4d3›d/d5d6›�
ƒ d7d8„ }|d9|ƒ d:D ]&‰ |d;ˆ d<›�‡ fd=d„|D ƒƒ �q¨td>ƒ td?ƒ d S )@Nznothing scoredc                 s   s   | ]}|d  V  qdS rO   r
   )r   rH   r
   r
   r   r   ™   r   zmain.<locals>.<genexpr>z#queue-aware LEG fill at the touch: éd   z.0fú%é<   zonly z three-leg books -- too fewc                 s   s   | ]}|d  V  qdS ©rR   Nr
   ©r   rZ   r
   r
   r   r   Ÿ   r   c                 s   s   | ]}|d  V  qdS ©rS   Nr
   ra   r
   r
   r   r       r   r#   zthree-leg books: z   completion z+quote-everything P&L at the touch, size 1: z+.2fz
 dollars (zc per book)
r   )ÚLogisticRegression)Úmake_pipeline)ÚStandardScalergÍÌÌÌÌÌä?c                 S   s"   g | ]}|d  D ]}|d ‘qqS )rI   r	   r
   ©r   rZ   Úlr
   r
   r   Ú
<listcomp>ª   r   zmain.<locals>.<listcomp>c                 S   s"   g | ]}|d  D ]}|d ‘qqS )rI   r'   r
   rf   r
   r
   r   rh   «   r   é   r   ztrain set too one-sided to fiti¸  )Zmax_iterzTEMPORAL SPLIT  train z books (r   z%m-%dz..éÿÿÿÿz)  test z (z)
c                 S   s   g | ]}|d  ‘qS r   r
   )r   rg   r
   r
   r   rh   ¸   r   rI   ú  ZruleÚ24ú Zbooksú>6Zcompletez>9zP&L $zper bookz>10c                 S   s”   |s t d| d›ddd›�ƒ d S tdd„ |D ƒƒd }dtd	d„ |D ƒƒ t|ƒ }t d| d›dt|ƒd›d|d
›d|d›d|t|ƒ d›�
ƒ d S )Nrk   rl   rm   r   rn   c                 s   s   | ]\}}|d  V  qdS rb   r
   ©r   Ú_rZ   r
   r
   r   r   À   r   z$main.<locals>.row.<locals>.<genexpr>r#   c                 s   s   | ]\}}|d  V  qdS r`   r
   ro   r
   r
   r   r   Á   r   z>8.0fz% z>+9.2fz>+10.3f)r>   r<   r?   )ZlblZselr   r   r
   r
   r   Úrow¼   s    zmain.<locals>.rowzquote everything)gš™™™™™©?gš™™™™™¹?gš™™™™™É?gffffffÖ?g      à?zP(all 3) >= z.2fc                    s   g | ]}|d  ˆ kr|‘qS )r   r
   )r   rL   ©Zthrr
   r   rh   Æ   r   z8
  Size 1. A gate can only REMOVE books, so it must beatzB  "quote everything" on the same test window to be worth anything.)rM   r>   r<   Úvaluesr?   r\   Zsklearn.linear_modelrc   Zsklearn.pipelinerd   Zsklearn.preprocessingre   rX   ÚnpZarrayr   ZfitZprodZpredict_probar6   )rI   ÚfrZbkZcrZtotrc   rd   re   ZcutZtrainÚtestZXtrZytrÚpipeZscoredrZ   r   rq   r
   rr   r   Úmain”   sl     ÿ
 ÿÿþþþ

ÿ
ÿ0
$rx   Ú__main__)Ú__doc__r*   r1   r-   Úsysr,   Úcollectionsr   Úpathlibr   Znumpyrt   ÚpathÚinsertZ	mm_settler9   Zqueue_aware_fillr:   r.   r(   r/   ZFEATSr   rM   r\   rx   Ú__name__r
   r
   r
   r   Ú<module>   s,   C7