a
    8ñ»jÖD  ã                   @   s´   d Z ddlZddlZddlmZ ddlmZ ddlmZ ddl	m
Z
mZmZ ddlZddlmZ dd	lmZmZmZmZ dd
lmZmZmZmZmZ ed ZdZG dd„ deƒZdS )zScraper for http://sofifa.com.é    N)Ú	timedelta)Úproduct)ÚPath)ÚCallableÚOptionalÚUnion)Úhtmlé   )ÚBaseRequestsReaderÚadd_standardized_team_nameÚsafe_xpath_textÚstandardize_colnames)ÚDATA_DIRÚNOCACHEÚNOSTOREÚTEAMNAME_REPLACEMENTSÚloggerÚSoFIFAzhttps://sofifa.comc                
       s  e Zd ZdZdddeeefeee	e
e	 f  ee	ee
e f eee	e
e	 eg e	f f  eeedœ‡ fdd„Zejdœdd	„Zdeeef ejdœdd„Zejdœdd„Zdeee	e
e	 f  ejdœdd„Zejdœdd„Zdeee	e
e	 f  eeee
e f  ejdœdd„Z‡  ZS )r   a’  Provides pd.DataFrames from data at http://sofifa.com.

    Data will be downloaded as necessary and cached locally in
    ``~/soccerdata/data/SoFIFA``.

    Parameters
    ----------
    leagues : string or iterable, optional
        IDs of leagues to include.
    versions : string, int or list of int, optional
        FIFA releases to include. Should be specified by their ID used in the URL
        (e.g., 230034). Alternatively, the string "all" can be used to include all
        versions and "latest" to include the latest version only. Defaults to
        "latest".
    proxy : 'tor' or or dict or list(dict) or callable, optional
        Use a proxy to hide your IP address. Valid options are:
            - "tor": Uses the Tor network. Tor should be running in
              the background on port 9050.
            - str: The address of the proxy server to use.
            - list(str): A list of proxies to choose from. A different proxy will
              be selected from this list after failed requests, allowing rotating
              proxies.
            - callable: A function that returns a valid proxy. This function will
              be called after failed requests, allowing rotating proxies.
    no_cache : bool
        If True, will not use cached data.
    no_store : bool
        If True, will not store downloaded data.
    data_dir : Path
        Path to directory where data will be cached.
    NÚlatest)ÚleaguesÚversionsÚproxyÚno_cacheÚno_storeÚdata_dirc                    sª   t ƒ j|||||d� d| _|dkr8|  ¡ jdd�| _nn|dkrL|  ¡ | _nZt|tƒrj|  ¡ j|g | _n<t|t	ƒr˜t
dd„ |D ƒƒr˜|  ¡ j| | _ntd|› �ƒ‚d	S )
zInitialize SoFIFA reader.)r   r   r   r   r   r	   r   )ÚnÚallc                 s   s   | ]}t |tƒV  qd S )N)Ú
isinstanceÚint)Ú.0Úv© r!   ú7E:\soccer\ml_env\Lib\site-packages\soccerdata/sofifa.pyÚ	<genexpr>R   ó    z"SoFIFA.__init__.<locals>.<genexpr>zInvalid value for versions: N)ÚsuperÚ__init__Z
rate_limitÚread_versionsÚtailr   r   r   ÚlocÚlistr   Ú
ValueError)Úselfr   r   r   r   r   r   ©Ú	__class__r!   r"   r&   :   s"    
û
zSoFIFA.__init__)Úreturnc              
   C   s˜   | j d }td }|  ||¡}t |¡}g }|d D ]:}|d D ],}| |d d|d › d|d	 › �d
œ¡ q@q4t |¡ | j	¡ 
d¡ ¡ j| j ¡  S )zmRetrieve selected leagues from the datasource.

        Returns
        -------
        pd.DataFrame
        zleagues.jsonz/api/leagueÚdataZchildsÚidú[Z
nationNamez] Úvalue)Ú	league_idÚleaguer5   )r   ÚSO_FIFA_APIÚgetÚjsonÚloadÚappendÚpdÚ	DataFrameÚpipeZ_translate_leagueÚ	set_indexÚ
sort_indexr)   Z_selected_leaguesÚkeys)r,   ÚfilepathÚurlmaskÚreaderÚresponser   ÚnodeÚchildr!   r!   r"   Úread_leaguesW   s(    

þÿ
ÿþüÿzSoFIFA.read_leaguesr	   )Úmax_ager/   c                 C   sâ   | j d }|  t||¡}g }t |¡}t| d¡ƒD ]˜\}}|j}| j d|› d� }t| d¡ }	| j|	||dkrv|ndd�}t |¡}| d	¡D ]8}
|
 d¡}t 	d
|¡ 
d¡}| t|ƒ||
jdœ¡ q’q4t |¡ d¡ ¡ S )a³  Retrieve available FIFA releases and rating updates.

        Parameters
        ----------
        max_age : int for age in days, or timedelta object
            The max. age of the locally cached release history before a new
            version is downloaded.

        Raises
        ------
        TypeError
            If max_age is not an integer or timedelta object.

        Returns
        -------
        pd.DataFrame
        z
index.htmlz#//header/section/p/select[1]/optionZupdates_z.htmlr3   r   N)rH   z#//header/section/p/select[2]/optionzr=(\d+)r	   )Ú
version_idÚfifa_editionÚupdaterI   )r   r7   r6   r   ÚparseÚ	enumerateÚxpathÚtextÚreÚsearchÚgroupr:   r   r;   r<   r>   r?   )r,   rH   rA   rC   r   ÚtreeÚiZnode_fifa_editionrJ   ÚurlZnode_fifa_updateÚhrefrI   r!   r!   r"   r'   v   s(    



ýÿzSoFIFA.read_versionsc                 C   s   t d }d}|  ¡ }g }tt| ¡ | j ¡ ƒƒ}t|ƒD ]Ê\}\\}}\}	}
t d|d t	|ƒ||
d ¡ |d }| j
| ||	¡ }| ||	¡}|  ||¡}t |¡}t d¡}| d¡D ]F}| d	¡d
 }| tt || d¡¡ d¡ƒ|j|dœ|
 ¡ ¥¡ qºq8t |¡ dti¡ dg¡S )zkRetrieve all teams for the selected leagues.

        Returns
        -------
        pd.DataFrame
        ú/teams?lg={}&r={}&set=trueúteams_{}_{}.htmlú-[%s/%s] Retrieving teams for %s in %s editionr	   rK   r4   z\/team\/(\d+)\/[\w-]+\/ú//table/tbody/trú.//td[2]//ar   rV   )Úteam_idÚteamr5   r]   r\   )r6   rG   r*   r   Úiterrowsr   rM   r   ÚinfoÚlenr   Úformatr7   r   rL   rP   ÚcompilerN   r:   r   rQ   rR   rO   Úto_dictr;   r<   Úreplacer   r>   )r,   rB   Úfilemaskr   ÚteamsÚiteratorrT   Úlkeyr5   rI   Úversionr4   rA   rU   rC   rS   Zpat_teamrE   Z	team_linkr!   r!   r"   Ú
read_teams£   s@    û

ÿûúÿzSoFIFA.read_teams)r]   r/   c                 C   sf  t d }d}|  ¡ }|durTt|ƒ}|j|j |¡dd…f }t|ƒdkrXtdƒ‚n|}g }tt	| j
 ¡ | ¡ ƒƒ}t|ƒD ]Ö\}\\}	}
\}}t d|d t|ƒ|d |
d	 ¡ | j| ||	¡ }| ||	¡}|  ||¡}t |¡}t d
¡}| d¡}|d  d¡D ]H}| tt || d¡¡ d¡ƒ| d¡|d |d dœ|
 ¡ ¥¡ �qq|t |¡ dg¡S )aƒ  Retrieve all players for the selected leagues.

        Parameters
        ----------
        team: str or list of str, optional
            Team(s) to retrieve. If None, will retrieve all teams.

        Raises
        ------
        ValueError
            If no data is found for the given team(s) in the selected leagues.

        Returns
        -------
        pd.DataFrame
        z/team/{}/?r={}&set=truezplayers_{}_{}.htmlNr   z:No data found for the given teams in the selected seasons.z7[%s/%s] Retrieving list of players for %s in %s editionr	   r]   rK   z\/player\/(\d+)\/[\w-]+\/z//article/tablez&.//td[2]/a[contains(@href,'/player/')]rV   zdata-tippy-contentr5   )Ú	player_idÚplayerr]   r5   rk   )r6   rj   r   r)   r]   Úisinr`   r+   r*   r   r   r^   rM   r   r_   r   ra   r7   r   rL   rP   rb   rN   r:   r   rQ   rR   rc   r;   r<   r>   )r,   r]   rB   re   Zdf_teamsZteams_to_checkrg   ÚplayersrT   rI   ri   r\   Zdf_teamrA   rU   rC   rS   Z
pat_playerZtable_squadrE   r!   r!   r"   Úread_playersÖ   sL    
û	


ÿúùÿ
zSoFIFA.read_playersc                    s`  ddddddddd	d
ddddddddddddddœ}t d }|D ]}|d|› �7 }q@d}|  ¡ }g }tt| ¡ | j ¡ ƒƒ}t|ƒD ]¸\}\\}	}
\}}t d|d t	|ƒ|	|d ¡ |
d }| j
| ||¡ }| ||¡}|  ||¡}t |¡}| d ¡D ]>‰ | |	ˆ  d!¡d" jd#œ‡ fd$d%„| ¡ D ƒ¥| ¡ ¥¡ qüq„t |¡ d&ti¡ d'd&g¡ ¡ S )(zvRetrieve ratings for all teams in the selected leagues.

        Returns
        -------
        pd.DataFrame
        ZoverallZattackZmidfieldZdefenceZtransfer_budgetZ
club_worthZbuild_up_speedZbuild_up_dribblingZbuild_up_passingZbuild_up_positioningZchance_creation_crossingZchance_creation_passingZchance_creation_shootingZchance_creation_positioningZdefence_aggressionZdefence_pressureZdefence_team_widthZdefence_defender_lineZdefence_domestic_prestigeZinternational_prestigern   Zstarting_xi_average_ageZwhole_team_average_age)ZoaÚatÚmdZdfÚtbZcwÚbsZbdÚbpZbpsÚccÚcpÚcsZcpsÚdaÚdmZdwÚddZdpÚipZpsÚsaÚtarW   z&showCol[]=rX   rY   r	   rK   r4   rZ   r[   r   )r5   r]   c                    s6   i | ].\}}|t ˆ d |› d�d|› d|› d�d�“qS )z.//td[@data-col='z
']//text()zCould not parse z (z) stat.)Úwarn)r   )r   ÚkeyÚdesc©rE   r!   r"   Ú
<dictcomp>c  s   û
ýz,SoFIFA.read_team_ratings.<locals>.<dictcomp>r]   r5   )r6   rG   r*   r   r^   r   rM   r   r_   r`   r   ra   r7   r   rL   rN   r:   rO   Úitemsrc   r;   r<   rd   r   r>   r?   )r,   ÚratingsrB   Z	rating_idre   r   rf   rg   rT   rh   r5   rI   ri   r4   rA   rU   rC   rS   r!   r�   r"   Úread_team_ratings  s|    	éû
þ
úýõÿ
ÿþÿzSoFIFA.read_team_ratings)r]   rl   r/   c              	   C   sº  t d }d}|du r(| j|d�j ¡ }nt|tƒr:|g}n|}g }g d¢}tt| j 	¡ |ƒƒ}t
|ƒD �]4\}	\\}
}}t d|	d t|ƒ||d ¡ | j| ||
¡ }| ||
¡}|  ||¡}tj|tjd	d
�d�}| d¡d }| d¡ ¡ }| d¡ ¡ }d|�r|n|i| ¡ ¥}|D ]r}d}d|› d�d|› d�d|› d�g}|D ]*}| |¡}|�rL|d j ¡ } �qx�qL|du�r†|nd||< �q| |¡ qft |¡ t¡ dg¡ ¡ S )ae  Retrieve ratings for players.

        Parameters
        ----------
        team: str or list of str, optional
            Team(s) to retrieve. If None, will retrieve all teams.
        player: int or list of int, optional
            Player(s) to retrieve. If None, will retrieve all players.

        Returns
        -------
        pd.DataFrame
        z/player/{}/?r={}&set=truezplayer_{}_{}.htmlN)r]   )$zOverall ratingZ	PotentialZCrossingZ	FinishingzHeading accuracyzShort passingZVolleysZ	DribblingZCurvezFK AccuracyzLong passingzBall controlZAccelerationzSprint speedZAgilityZ	ReactionsZBalancez
Shot powerZJumpingZStaminaZStrengthz
Long shotsZ
AggressionZInterceptionsZPositioningZVisionZ	PenaltiesZ	ComposurezDefensive awarenesszStanding tacklezSliding tacklez	GK DivingzGK Handlingz
GK KickingzGK PositioningzGK Reflexesz>[%s/%s] Retrieving ratings for player with ID %s in %s editionr	   rK   Úutf8)Úencoding)Úparserz%//div[contains(@class, 'profile')]/h1r   zstring(./text()[1])z)string(./br/following-sibling::text()[1])rl   z//p[.//text()[contains(.,'z')]]/span/emz//div[contains(.,'z')]/emz.//li[not(self::script)][.//text()[contains(.,'z')]]/em)r6   ro   ÚindexÚuniquer   r   r*   r   r   r^   rM   r   r_   r`   r   ra   r7   r   rL   Ú
HTMLParserrN   Ústriprc   rO   r:   r;   r<   r=   r   r>   r?   )r,   r]   rl   rB   re   rn   r„   Zscore_labelsrg   rT   rI   ri   rA   rU   rC   rS   Znode_player_nameZ	before_brZafter_brZscoresÚsr3   ZxpathsrN   Únodesr!   r!   r"   Úread_player_ratingsw  sV    
'û	ÿþ


ý

zSoFIFA.read_player_ratings)r	   )N)NN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   ÚSO_FIFA_DATADIRr   r   Ústrr*   r   r   Úboolr   r&   r;   r<   rG   r   r'   rj   ro   r…   r�   Ú__classcell__r!   r!   r-   r"   r      s6   "ùù-3$IZ  ýü)r“   r8   rP   Údatetimer   Ú	itertoolsr   Úpathlibr   Útypingr   r   r   Zpandasr;   Zlxmlr   Ú_commonr
   r   r   r   Ú_configr   r   r   r   r   r”   r6   r   r!   r!   r!   r"   Ú<module>   s   