a
    8ñ»j>o  ã                   @   s8  d Z ddlZddlZddlZddlZddlmZ ddlmZ ddl	m
Z
 ddlmZmZmZmZmZ ddlZddlmZmZ dd	lmZmZmZmZ ed
 ZdZddiZdddddœZddddœZdddddddœZ G dd
„ d
eƒZ!eee" dœd d!„Z#eee$ dœd"d#„Z%eee& dœd$d%„Z'eee( dœd&d'„Z)dS )(zScraper for understat.com.é    N)ÚIterable)Úunescape)ÚPath)ÚAnyÚCallableÚIOÚOptionalÚUnioné   )ÚBaseRequestsReaderÚmake_game_id)ÚDATA_DIRÚNOCACHEÚNOSTOREÚTEAMNAME_REPLACEMENTSÚ	Understatzhttps://understat.comzX-Requested-WithZXMLHttpRequestz	Open PlayzFrom Cornerz	Set PiecezDirect Freekick)ZOpenPlayZ
FromCornerZSetPieceZDirectFreekickz
Right Footz	Left FootÚOther)Z	RightFootZLeftFootZOtherBodyPartsÚGoalzOwn GoalzBlocked Shotz
Saved ShotzMissed ShotzShot On Post)r   ZOwnGoalZBlockedShotZ	SavedShotZMissedShotsZ
ShotOnPostc                
       sÐ  e Zd ZdZdddeeefeee	e
e	 f  eee	eeee	ef  f  eee	e
e	 eg e	f f  eeedœ‡ fdd„Zddœdd„Zejdœd	d
„Zejdœdd„Zd.eeejdœdd„Zd/eejdœdd„Zd0eejdœdd„Zd1eeee
e f  ejdœdd„Zd2eeee
e f  ejdœdd„Zd3ejeeee
e f  ejdœdd„Zd4eedœd d!„Zd5e	eeeed"œd#d$„Ze	eee d%œd&d'„Zd6e	ee ee e! d(œd)d*„Z"e#e	e	d+œd,d-„ƒZ$‡  Z%S )7r   a  Provides pd.DataFrames from data at https://understat.com.

    Data will be downloaded as necessary and cached locally in
    ``~/soccerdata/data/Understat``.

    Parameters
    ----------
    proxy : 'tor' or or dict or list(dict) or callable, optional
        Use a proxy to hide your IP address. Valid options are:
            - "tor": Uses the Tor network. Tor should be running in
              the background on port 9050.
            - str: The address of the proxy server to use.
            - list(str): A list of proxies to choose from. A different proxy will
              be selected from this list after failed requests, allowing rotating
              proxies.
            - callable: A function that returns a valid proxy. This function will
              be called after failed requests, allowing rotating proxies.
    no_cache : bool
        If True, will not use cached data.
    no_store : bool
        If True, will not store downloaded data.
    data_dir : Path
        Path to directory where data will be cached.
    N)ÚleaguesÚseasonsÚproxyÚno_cacheÚno_storeÚdata_dirc                    s&   t ƒ j|||||d� || _d| _dS )z"Initialize a new Understat reader.)r   r   r   r   r   FN)ÚsuperÚ__init__r   Ú_cookies_initialized)Úselfr   r   r   r   r   r   ©Ú	__class__© ú:E:\soccer\ml_env\Lib\site-packages\soccerdata/understat.pyr   F   s    
ûzUnderstat.__init__)Úreturnc                 C   s   | j s| j t¡ d| _ dS )z1Ensure the session has cookies from the homepage.TN)r   Ú_sessionÚgetÚUNDERSTAT_URL)r   r    r    r!   Ú_ensure_cookiesZ   s    zUnderstat._ensure_cookiesc           
         sÀ   |   ¡ }i }|d }|D ]B}|d }||vr|d }| dd¡}||td|› � dœ||< qd}t|ƒdkrxtj|d	�S tj t| ¡ ƒ¡ 	| j
¡ |¡ ¡  ¡ ‰ ‡ fd
d„| jD ƒ}	ˆ j|	 S )zqRetrieve the selected leagues from the datasource.

        Returns
        -------
        pd.DataFrame
        ÚstatDataÚ	league_idÚleagueú Ú_ú/league/)r(   r)   Úurlr   ©Úindexc                    s   g | ]}|ˆ j v r|‘qS r    r.   )Ú.0r)   ©Údfr    r!   Ú
<listcomp>ƒ   ó    z*Understat.read_leagues.<locals>.<listcomp>)Ú_read_leaguesÚreplacer%   ÚlenÚpdÚ	DataFrameÚfrom_recordsÚlistÚvaluesÚpipeÚ_translate_leagueÚ	set_indexÚ
sort_indexÚconvert_dtypesr   Úloc)
r   Údatar   Úleague_dataÚleague_statr(   r)   Úleague_slugr/   Zvalid_leaguesr    r1   r!   Úread_leagues`   s.    ýÿþ
ÿzUnderstat.read_leaguesc              
      s*  |   ¡ }i }|d }|D ]š}|d }t|d ƒ}t|d ƒ}|dkrH|n|d }||f}	|	|vr|d }
|
 dd	¡}|› d
|d › �}||
|| j |¡td|› d
|› � dœ||	< qddg}t|ƒdkrÔtj|d�S tj 	t
| ¡ ƒ¡ | j¡ |¡ ¡  ¡ ‰ t | j| j¡}‡ fdd„|D ƒ}ˆ j| S )zqRetrieve the selected seasons from the datasource.

        Returns
        -------
        pd.DataFrame
        r'   r(   ÚyearÚmonthé   r
   r)   r*   r+   ú/r,   )r(   r)   Ú	season_idÚseasonr-   rM   r   r.   c                    s   g | ]}|ˆ j v r|‘qS r    r.   )r0   rM   r1   r    r!   r3   ±   r4   z*Understat.read_seasons.<locals>.<listcomp>)r5   Úintr6   Z_season_codeÚparser%   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   Ú	itertoolsÚproductr   r   rB   )r   rC   r   rD   rE   r(   rH   rI   rL   Úkeyr)   rF   rM   r/   Zall_seasonsZvalid_seasonsr    r1   r!   Úread_seasons†   s>    
ûÿþ
ÿzUnderstat.read_seasonsTF)Úinclude_matches_without_dataÚforce_cacher"   c                 C   sØ  |   ¡ }g }| ¡ D �]J\\}}}|d }|d }	|d }
|  ||¡ }|oP| }|  |
||	|¡}|d }|D ]ð}t|d ƒ}|d d dv}|d d	 dv}|p¤|}| |||	|||d
 t|d d ƒt|d	 d ƒt|d d ƒt|d	 d ƒ|d	 d |d d t|d d ƒt|d d	 ƒt|d d ƒt|d d	 ƒt|d ƒ|t	d|› � dœ¡ qnqg d¢}t
|ƒdk�r„tj|d�S tj |¡jdd„ d� ttdœ¡jdd„ d� |¡ ¡  ¡ }|�sÔ||d  }|S )aì  Retrieve the matches for the selected leagues and seasons.

        Parameters
        ----------
        include_matches_without_data : bool
            By default matches with and without data are returned.
            If False, will only return matches with data.

        force_cache : bool
             By default no cached data is used for the current season.
             If True, will force the use of cached data anyway.

        Returns
        -------
        pd.DataFrame
        r(   rL   r-   Ú	datesDataÚidÚxGÚh)Ú0NÚaÚdatetimeÚtitleÚshort_titleÚgoalsZisResultz/match/)r(   r)   rL   rM   Úgame_idÚdateÚhome_team_idÚaway_team_idÚ	home_teamÚ	away_teamÚaway_team_codeÚhome_team_codeZ
home_goalsZ
away_goalsZhome_xgZaway_xgZ	is_resultÚhas_datar-   ©r)   rM   Úgamer   r.   c                 S   s   t j| d dd�S ©Nra   z%Y-%m-%d %H:%M:%S)Úformat©r8   Úto_datetime©Úgr    r    r!   Ú<lambda>ù   r4   z)Understat.read_schedule.<locals>.<lambda>©ra   ©rd   re   c                 S   s   | j tdd�S ©Nr
   )Zaxis©Úapplyr   ro   r    r    r!   rq      r4   ©rj   rh   )rS   ÚiterrowsÚ_is_completeÚ_read_league_seasonÚ_as_intÚappendÚ_as_strÚ	_as_floatÚ_as_boolr%   r7   r8   r9   r:   Úassignr6   r   r?   r@   rA   )r   rT   rU   Ú
df_seasonsÚmatchesr)   rM   Úleague_seasonr(   rL   r-   Úis_current_seasonr   rC   Úmatches_dataÚmatchÚmatch_idZhas_home_xgZhas_away_xgrh   r/   r2   r    r    r!   Úread_schedule´   sn    



íÿÿþýø	÷
ÿzUnderstat.read_schedule)rU   r"   c                 C   sÒ  |   ¡ }i }| ¡ D �]R\\}}}|d }|d }|d }	|  ||¡ }
|
oP| }|  |	|||¡}i }i }|d }|D ]®}t|d ƒ}|d }||||||d t|d d ƒt|d d ƒt|d d	 ƒt|d d	 ƒt|d d
 ƒt|d d
 ƒdœ||< dD ]"}t|| d ƒ}||||f< �q qv|d }| ¡ D �].}t|d ƒ}|d D �]}|d }|||f }|d }|dk�r€dnd}||v�rš|| ||< |d }|d dk�rÀ|d |d  ntj}||  	|› d�t|d ƒ|› d�t
|d ƒ|› d�t|d ƒ|› d�t
|d ƒ|› d�t
|d  ƒ|› d!�t
|d" ƒ|› d#�t
|ƒ|› d$�t|d% ƒi¡ �qP�q6qg d&¢}t|ƒdk�rŒtj|d'�S tj t| ¡ ƒ¡jd(d)„ d*� ttd+œ¡jd,d)„ d-� |¡ ¡  ¡ S ).aK  Retrieve the team match stats for the selected leagues and seasons.

        Parameters
        ----------
        force_cache : bool
             By default no cached data is used for the current season.
             If True, will force the use of cached data anyway.

        Returns
        -------
        pd.DataFrame
        r(   rL   r-   rV   rW   r\   rY   r[   r]   r^   )r(   r)   rL   rM   r`   ra   rb   rc   rd   re   rf   rg   ©rY   r[   Ú	teamsDataÚhistoryra   Úh_aÚhomeÚawayÚppdaÚdefr   ZattZ_pointsZptsZ_expected_pointsZxptsZ_goalsZscoredZ_xgrX   Z_np_xgÚnpxGZ_np_xg_differenceZnpxGDZ_ppdaZ_deep_completionsÚdeepri   r.   c                 S   s   t j| d dd�S rk   rm   ro   r    r    r!   rq   a  r4   z1Understat.read_team_match_stats.<locals>.<lambda>rr   rs   c                 S   s   | j tdd�S rt   ru   ro   r    r    r!   rq   h  r4   rw   )rS   rx   ry   rz   r{   r}   r<   r8   ÚNAÚupdater~   r7   r9   r:   r;   r€   r6   r   r?   r@   rA   )r   rU   r�   Ústatsr)   rM   rƒ   r(   rL   r-   r„   r   rC   Zscheduler‚   r…   r†   r‡   Z
match_dateÚsideÚteam_idÚ
teams_dataÚteamÚ	team_sideÚprefixr�   Z	team_ppdar/   r    r    r!   Úread_team_match_stats  sŠ    
ô

$øÿÿþýø	÷
ÿzUnderstat.read_team_match_statsc                 C   sæ  |   ¡ }g }| ¡ D �]ˆ\\}}}|d }|d }|d }	|  ||¡ }
|
oP| }|  |	|||¡}|d }i }| ¡ D ]$}t|d ƒ}t|d ƒ}|||< qv|d }|D ]ô}|d }d	|v rÊ| d	¡d
 }t|ƒ}|| }| ||||||t|d ƒt|d ƒt|d ƒt|d ƒt|d ƒt|d ƒt	|d ƒt|d ƒt	|d ƒt|d ƒt	|d ƒt|d ƒt|d ƒt|d ƒt|d ƒt	|d ƒt	|d ƒdœ¡ q¨qg d¢}t
|ƒd
k�rÂtj|d�S tj |¡ dti¡ |¡ ¡  ¡ S )aN  Retrieve the player season stats for the selected leagues and seasons.

        Parameters
        ----------
        force_cache : bool
             By default no cached data is used for the current season.
             If True, will force the use of cached data anyway.

        Returns
        -------
        pd.DataFrame
        r(   rL   r-   rŠ   r]   rW   ÚplayersDataZ
team_titleú,r   Úplayer_nameÚpositionZgamesÚtimer_   rX   Znpgr‘   ÚassistsÚxAÚshotsÚ
key_passesÚyellow_cardsÚ	red_cardsÚxGChainÚ	xGBuildup)r)   r(   rM   rL   r™   r—   ÚplayerÚ	player_idr    r‚   Úminutesr_   ÚxgZnp_goalsZnp_xgr¢   Úxar¤   r¥   r¦   r§   Úxg_chainÚ
xg_buildup)r)   rM   r™   rª   r.   r™   )rS   rx   ry   rz   r<   r}   r{   Úsplitr|   r~   r7   r8   r9   r:   r6   r   r?   r@   rA   )r   rU   r�   r•   r)   rM   rƒ   r(   rL   r-   r„   r   rC   r˜   Zteam_mappingr™   Z	team_namer—   Úplayers_datarª   Zplayer_team_nameZplayer_team_idr/   r    r    r!   Úread_player_season_statsn  st    


















éÿÿþú
ÿz"Understat.read_player_season_stats)r‡   r"   c                    sÀ  | j dd�}|  ||¡}g }| ¡ D �]R\\}}}}|d }	|d }
|d }|d }|  ||¡}|du rjq$|d ‰ ‡ fd	d
„dD ƒ}|d }| ¡ D ]â}| ¡ D ]Ô}|d }|| }| ||	||
|||t|ƒt|d ƒt|d ƒt|d ƒt|d ƒt|d ƒt|d ƒt|d ƒt|d ƒt|d ƒt|d ƒt|d ƒt|d ƒt|d ƒt|d ƒt|d ƒt|d ƒdœ¡ q q”q$g d¢}t	|ƒd k�rœt
j|d!�S t
j |¡ d"ti¡ |¡ ¡  ¡ S )#a�  Retrieve the player match stats for the selected leagues and seasons.

        Parameters
        ----------
        match_id : int or list of int, optional
            Retrieve the player match stats for a specific match.

        Raises
        ------
        ValueError
            If the given match_id could not be found in the selected seasons.

        Returns
        -------
        pd.DataFrame
        F©rT   r(   rL   r`   r-   NÚ
match_infoc                    s$   i | ]}ˆ | t ˆ d |› � ƒ“qS ©Zteam_)r}   ©r0   r–   ©rµ   r    r!   Ú
<dictcomp>ä  s   z5Understat.read_player_match_stats.<locals>.<dictcomp>r‰   ÚrostersDatar—   rª   r«   r    ZpositionOrderr¡   r_   Ú	own_goalsr¤   rX   r¨   r©   r¢   r£   r¥   Zyellow_cardZred_card)r)   r(   rM   rL   r`   rj   r™   r—   rª   r«   r    Zposition_idr¬   r_   r»   r¤   r­   r¯   r°   r¢   r®   r¥   r¦   r§   ©r)   rM   rj   r™   rª   r   r.   r™   )rˆ   Ú_select_matchesrx   Ú_read_matchr<   r|   r{   r}   r~   r7   r8   r9   r:   r6   r   r?   r@   rA   )r   r‡   Údf_scheduleÚ
df_resultsr•   r)   rM   rj   Úleague_season_gamer(   rL   r`   r-   rC   Zteam_id_to_namer²   Zteam_playersrª   r—   r™   r/   r    r¸   r!   Úread_player_match_statsÂ  sr    
ÿ















èÿ
ÿþú
ÿz!Understat.read_player_match_statsc                    s  | j dd�}|  ||¡}g }| ¡ D �]š\\}}}}|d }	|d }
|d }|d }|  ||¡}|du rjq$|d ‰ ‡ fd	d
„dD ƒ}|d }i }| ¡ D ]2}| ¡ D ]$}t|d ƒ}t|d ƒ}|||< q¤q˜|d }| ¡ D ]â}|D ]Ø}|d }t||› d� ƒ}|| }t|d ƒ}| |tj	¡}| 
|	||
||||d t|d ƒ||t|d ƒ|d ||t|d ƒt|d ƒt|d ƒt|d ƒt |d tj	¡t |d tj	¡t |d tj	¡dœ¡ qäqÜq$g d¢}t|ƒdk�rätj|d�S tj |¡jd d!„ d"� d#ti¡ |¡ ¡  ¡ S )$a§  Retrieve the shot events for the selected matches or the selected leagues and seasons.

        Parameters
        ----------
        match_id : int or list of int, optional
            Retrieve the shot events for a specific match.

        Raises
        ------
        ValueError
            If the given match_id could not be found in the selected seasons.

        Returns
        -------
        pd.DataFrame
        Fr´   r(   rL   r`   r-   Nrµ   c                    s(   i | ] }t ˆ d |› � ƒtˆ | ƒ“qS r¶   )r}   r{   r·   r¸   r    r!   r¹   :  s   ÿz.Understat.read_shot_events.<locals>.<dictcomp>r‰   rº   rª   rW   Ú	shotsDatarŒ   Z_teamZplayer_assistedra   r«   rX   ÚXÚYÚminuteZshotTypeÚ	situationÚresult)r(   r)   rL   rM   r`   rj   ra   Zshot_idr—   r™   r«   rª   Úassist_player_idÚassist_playerr­   Z
location_xZ
location_yrÆ   Z	body_partrÇ   rÈ   r¼   r   r.   c                 S   s   t j| d dd�S rk   rm   ro   r    r    r!   rq   o  r4   z,Understat.read_shot_events.<locals>.<lambda>rr   r™   )rˆ   r½   rx   r¾   r<   r}   r{   r$   r8   r“   r|   r~   ÚSHOT_BODY_PARTSÚSHOT_SITUATIONSÚSHOT_RESULTSr7   r9   r:   r€   r6   r   r?   r@   rA   )r   r‡   r¿   rÀ   r¤   r)   rM   rj   rÁ   r(   rL   r`   r-   rC   Zteam_name_to_idZrosters_dataZplayer_name_to_idZ	team_datarª   rŸ   r«   Z
shots_dataZ
team_shotsZshotrš   r™   r—   rÊ   rÉ   r/   r    r¸   r!   Úread_shot_events  s„    
þ





ëÿ
ÿÿýù
ÿzUnderstat.read_shot_events)r¿   r‡   r"   c                 C   sF   |d ur>t |tƒr|gn|}||d  |¡ }|jrBtdƒ‚n|}|S )Nr`   z<No matches found with the given IDs in the selected seasons.)Ú
isinstancerN   ÚisinÚemptyÚ
ValueError)r   r¿   r‡   Z	match_idsr2   r    r    r!   r½   z  s    
zUnderstat._select_matches)r   r"   c                 C   s@   |   ¡  td }| jd }| j|||d�}t |¡}d|d iS )Nz/getStatDatazleagues.json©r   r'   Ústat)r&   r%   r   Ú_request_apiÚjsonÚload)r   r   r-   ÚfilepathÚreaderrC   r    r    r!   r5   ‰  s    

zUnderstat._read_leagues)r-   r(   rL   r   r"   c                 C   s†   |   ¡  | d¡ d¡}|d }|d }td|› d|› � }| jd|› d|› d� }	| j||	|d�}
t |
¡}|d	 |d
 |d dœS )NrK   éþÿÿÿéÿÿÿÿz/getLeagueData/Zleague_Z_season_ú.jsonrÓ   ÚdatesZplayersZteams)rV   r�   rŠ   )r&   Úrstripr±   r%   r   rÕ   rÖ   r×   )r   r-   r(   rL   r   ÚpartsrF   rM   Úapi_urlrØ   rÙ   rC   r    r    r!   rz   ‘  s    
ýzUnderstat._read_league_season)r-   r‡   r"   c                 C   sÔ   |   ¡  z²td|› � }| jd|› d� }|  ||¡}t |¡}|  |d d ¡}|  |d d ¡}|d }	tt|	d  	¡ ƒƒd	 }
tt|	d
  	¡ ƒƒd	 }|
|||dœ}||	|d dœW S  t
yÎ   Y d S 0 d S )Nz/getMatchData/Zmatch_rÜ   Útmplr�   rŽ   ÚrostersrY   r—   r[   )rY   r[   Zteam_hZteam_ar¤   )rµ   rº   rÃ   )r&   r%   r   rÕ   rÖ   r×   Ú_extract_team_nameÚnextÚiterr<   ÚConnectionError)r   r-   r‡   rà   rØ   rÙ   rC   Zhome_team_nameZaway_team_namerâ   rb   rc   rµ   r    r    r!   r¾   ¤  s,    
üýzUnderstat._read_match)r-   rØ   r   r"   c                 C   s²   |duo|  ¡ o| o| j }|r6|dur6|jdd�S | jj|td�}| ¡  |j}| js¨|dur¨|j	j
ddd� |jdd��}| |¡ W d  ƒ n1 sž0    Y  t |¡S )z4Make an API request with proper headers and caching.NÚrb)Úmode)ÚheadersT)ÚparentsÚexist_okÚwb)Úexistsr   Úopenr#   r$   ÚUNDERSTAT_HEADERSÚraise_for_statusÚcontentr   ÚparentÚmkdirÚwriteÚioÚBytesIO)r   r-   rØ   r   Z	is_cachedÚresponseÚpayloadÚfhr    r    r!   rÕ   Â  s    ÿ(zUnderstat._request_api)Úhtmlr"   c                 C   s   t  d| ¡}|r| d¡S dS )z!Extract team name from tmpl HTML.z<h3><a[^>]*>([^<]+)</a></h3>r
   Ú )ÚreÚsearchÚgroup)rú   r†   r    r    r!   rã   ×  s    
zUnderstat._extract_team_name)TF)F)F)N)N)N)F)F)NF)&Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   ÚUNDERSTAT_DATADIRr   r	   Ústrr;   rN   r   r   Úboolr   r   r&   r8   r9   rG   rS   rˆ   rœ   r³   rÂ   rÎ   r½   Údictr5   rz   r¾   r   ÚbytesrÕ   Ústaticmethodrã   Ú__classcell__r    r    r   r!   r   ,   sf   ùù&/ ÿþWcU ÿþX$c ýü	 ÿþ ÿ
þ)Úvaluer"   c              	   C   s(   z
t | ƒW S  ttfy"   Y d S 0 d S ©N)r  Ú	TypeErrorrÒ   ©r
  r    r    r!   r   à  s    
r   c              	   C   s(   z
t | ƒW S  ttfy"   Y d S 0 d S r  )Úfloatr  rÒ   r  r    r    r!   r~   ç  s    
r~   c              	   C   s(   z
t | ƒW S  ttfy"   Y d S 0 d S r  )rN   r  rÒ   r  r    r    r!   r{   î  s    
r{   c              	   C   s(   z
t | ƒW S  ttfy"   Y d S 0 d S r  )r   r  rÒ   r  r    r    r!   r}   õ  s    
r}   )*r  rP   rõ   rÖ   rü   Úcollections.abcr   rú   r   Úpathlibr   Útypingr   r   r   r   r	   Zpandasr8   Ú_commonr   r   Ú_configr   r   r   r   r  r%   rï   rÌ   rË   rÍ   r   r  r   r  r~   rN   r{   r  r}   r    r    r    r!   Ú<module>   sP   üýú
     9