a
    ‘ô»jé)  ã                   @   sH  d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z
ddlZ
ddlZ
ddlmZ dZdZdZedƒZed Zed	 Zed
 ZeddƒZdZdZdZdZdddddddddddddddd d!d"d#œZg d$¢Zg d%¢Ze
j  ed& ¡Z!d'gZ"G d(d)„ d)e#ƒZ$d*d+„ Z%d,d-„ Z&d.d/„ Z'd0d1„ Z(d2d3„ Z)d4d5„ Z*d6d7„ Z+d8d9„ Z,d:d;„ Z-d<d=„ Z.d>d?„ Z/e0d@k�rDe 1¡ Z2e2j3dAdBdCdD� e2j3dEdBdFdD� e2 4¡ Z5ej6dGdGdH� z2e! 7¡  e5j-�rÈe-ƒ  ne5j8�rØe/ƒ  ne.ƒ  W nb e$�y Z9 z$e:dIe9› �ƒ e ;dJ¡ W Y dZ9[9n.dZ9[90  e<�yB   e:dKƒ e ;dL¡ Y n0 dS )Maç  Download SoFIFA PLAYER ratings (and optionally team ratings). RUN BY THE USER.

What it collects, for FIFA 15 (season 2014/15) through FC 27 (2026/27):
  * PLAYERS (default): every player in 18 leagues -- id, full name, positions,
    age, overall, potential, club, contract, wage, market value, total stats.
    Two snapshots per season: early autumn (post-summer-window squads) and
    mid-season (post-January). ~4,700 pages, ~4 hours.
  * TEAMS (--teams): each team's overall/attack/midfield/defence, transfer
    budget and club worth, ~monthly. ~2,500 pages, ~2 hours.

How it behaves, deliberately:
  * Identifies itself honestly (USER_AGENT below), not as a browser.
  * Checks sofifa.com/robots.txt before EVERY request and refuses anything it
    disallows (currently /api/ and /players/compare, which it never needs).
  * One request every ~3 seconds; every page cached on disk, so stopping and
    re-running resumes where it left off and never re-downloads.
  * Stops at the first HTTP 403/429 instead of retrying: that is the site
    saying no, and the right response is to stop.

The site's robots.txt also carries `Content-Signal: ai-train=no` and blocks AI
crawlers. Using the data for your own model is your call (2026-09-29).

Usage (from E:\soccer):
  1. Check league ids first -- one page per league, ~1 minute:
       ml_env\Scripts\python.exe research\sofifa_team_ratings.py --check
  2. Players (default), ~4 hours, safe to stop and restart:
       ml_env\Scripts\python.exe research\sofifa_team_ratings.py
  3. Team ratings too (optional), ~2 hours:
       ml_env\Scripts\python.exe research\sofifa_team_ratings.py --teams

Output: research/ml_data/sofifa_player_ratings.csv
        research/ml_data/sofifa_team_ratings.csv   (with --teams)
é    N)ÚPathzhttps://sofifa.comz@soccer-research-script/1.0 (personal, non-commercial; ~1 req/3s)g      @zE:\soccer\research\ml_dataZsofifa_cachezsofifa_player_ratings.csvzsofifa_team_ratings.csvé   é   é   )g333333Ã?gš™™™™™á?é<   é   ZE0ZE1ZE2ZE3ZSP1ZSP2ZD1ZD2ZI1ZI2ZF1ZF2ZN1ZP1ZB1ZSC0ZT1ZG1)é   é   r   é=   é5   é6   é   é   é   é    é   é   é
   i4  é   é2   éD   é?   )ÚoaZatZmdZdfÚtbZcwZsaZps)Zaer   ZptZwgZvlZttz/robots.txtg        c                   @   s   e Zd ZdS )ÚRefusedN)Ú__name__Ú
__module__Ú__qualname__© r   r   ú)E:\soccer\research\sofifa_team_ratings.pyr   U   s   r   c              
   C   sp  t |  }tt | ¡ ¡ ¡ d  }| ¡ r8|jddd�S t 	t
|¡sRtd|› �ƒ‚tt dd¡ t ¡ td   }|dkr„t |¡ tjj|dt
id	�}z¨zFtjj|d
d��"}| ¡ jddd�}W d  ƒ n1 sÔ0    Y  W nP tjj�y0 } z2|jdv �rtd|j› d|› d�ƒ‚‚ W Y d}~n
d}~0 0 W t ¡ td< nt ¡ td< 0 tjddd� |j|dd� |S )z@Cached, robots-checked, rate-limited GET. Returns the page text.z.htmlúutf-8Úreplace)ÚencodingÚerrorszrobots.txt disallows r   g      ð?z
User-Agent)Zheadersé   )Ztimeout)r#   N)i“  i­  zHTTP z on z" -- the site is refusing; stoppingT©ZparentsÚexist_ok)r"   )ÚBASEÚCACHEÚhashlibZsha1ÚencodeZ	hexdigestÚexistsZ	read_textÚ_robotsZ	can_fetchÚ
USER_AGENTr   ÚDELAY_SÚrandomZuniformÚtimeÚ_lastÚsleepÚurllibZrequestZRequestZurlopenÚreadÚdecodeÚerrorZ	HTTPErrorÚcodeÚmkdirZ
write_text)ÚpathZurlÚkeyÚwaitZreqÚrÚtextÚer   r   r   ÚfetchY   s*     
4r?   c                 C   s"   t  d| ¡}|r| d¡ ¡ S dS )Nz/<title>\s*(?:FIFA|FC)\s?\d+\s*-\s*([^-<]+?)\s*-é   Ú )ÚreÚsearchÚgroupÚstrip)r=   Úmr   r   r   Ú	page_dates   s    rG   c              	   C   s>   t  d|› d�| t j¡}|r:t t  dd| d¡¡¡ ¡ S dS )Nz
data-col="z"[^>]*>(.*?)</td>ú<[^>]+>rA   r@   )rB   rC   ÚSÚhtmlÚunescapeÚsubrD   rE   )ÚtrZcolrF   r   r   r   Ú_cellx   s    rN   c                 C   s   dd„ t  d| t j¡D ƒS )Nc                 S   s   g | ]}d |v r|‘qS )zdata-col="oa"r   )Ú.0rM   r   r   r   Ú
<listcomp>~   s   ÿz_rows.<locals>.<listcomp>z<tr[^>]*>(.*?)</tr>)rB   ÚfindallrI   )r=   r   r   r   Ú_rows}   s    rR   c              	      sª   g }t | ƒD ]˜‰ t dˆ ¡}t dˆ ¡}|r6| d¡nd|rPt | d¡¡ ¡ nd|r`| d¡nd|rzt | d¡¡ ¡ nddœ}| ‡ fdd„tD ƒ¡ | 	|¡ q|S )	Nú)href="/team/(\d+)/[^/]+/\d+/">([^<]+)</a>z%href="/league/(\d+)"[^>]*>([^<]+)</a>r@   rA   é   )Úteam_idÚ	team_nameÚsofifa_league_idÚsofifa_league_namec                    s   i | ]}|t ˆ |ƒ“qS r   ©rN   ©rO   Úc©rM   r   r   Ú
<dictcomp>‹   ó    zparse_teams.<locals>.<dictcomp>)
rR   rB   rC   rD   rJ   rK   rE   ÚupdateÚ	TEAM_COLSÚappend)r=   ÚoutÚtÚlgZrowr   r\   r   Úparse_teams‚   s    ýre   c                    sî   g }t | ƒD ]Ü‰ t dˆ ¡}t dˆ ¡}t dˆ tj¡}| |rJ| d¡nd|rdt | d¡¡ ¡ ndd 	t 
dˆ ¡¡|r„| d¡nd|ržt | d¡¡ ¡ nd|rÌt d	dt t d
d| d¡¡¡¡ ¡ nddœ‡ fdd„tD ƒ¥¡ q|S )Nz;href="/player/(\d+)/[^"]*"[^>]*data-tippy-content="([^"]+)"rS   z<div class="sub">(.*?)</div>r@   rA   rT   ú zclass="pos pos\d+">([A-Z]+)<z\s+rH   )Ú	player_idÚplayer_nameÚ	positionsrU   rV   Úcontractc                    s   i | ]}|t ˆ |ƒ“qS r   rY   rZ   r\   r   r   r]   ¡   r^   z!parse_players.<locals>.<dictcomp>)rR   rB   rC   rI   ra   rD   rJ   rK   rE   ÚjoinrQ   rL   ÚPLAYER_COLS)r=   rb   Úprc   rj   r   r\   r   Úparse_players�   s(    þÿ
ø	÷rn   c                 C   s0   t d| d›d�ƒ}ttt d| d›d�|¡ƒƒS )zHEvery ratings-update id SoFIFA lists for one game version, oldest first.z/teams?lg=13&r=Z02dz0001&set=truezr=(z\d{4}))r?   ÚsortedÚsetrB   rQ   )Úversionr=   r   r   r   Úall_updates¦   s    rr   c                    s   ˆ sg S t ‡ fdd„|D ƒƒS )Nc              	      s2   h | ]*}ˆ t tˆ ƒd  t|tˆ ƒd   ƒƒ ’qS ©r@   )ÚminÚlenÚround)rO   rm   ©Úidsr   r   Ú	<setcomp>¯   r^   zpick.<locals>.<setcomp>)ro   )rx   ri   r   rw   r   Úpick¬   s    rz   c                  C   s”   t ttƒƒd } td| › �ƒ t ¡ D ]h\}}ttd|› d| › d�ƒƒ}dd„ |D ƒ}td|d	›d
|d	›dt|ƒd›dd 	t
|ƒ¡p†d› �ƒ q&d S )Néÿÿÿÿzlatest update ú
/teams?lg=ú&r=ú	&set=truec                 S   s   h | ]}|d  ’qS )rX   r   )rO   r<   r   r   r   ry   ·   r^   zcheck.<locals>.<setcomp>z  lg=z<4z -> rf   z>3z teams  shown as: ú, zNOTHING -- wrong id)rr   ÚmaxÚVERSIONSÚprintÚLEAGUESÚitemsre   r?   ru   rk   ro   )Zlatestrd   r7   ÚrowsÚnamesr   r   r   Úcheck²   s     ÿr‡   c                  C   sb  dd„ t D ƒ} tt| ƒ› dttƒ› d�ƒ g d¢t }d}ttddd	d
��ô}tj||d�}| 	¡  | D ]È\}}t 
¡ D ]¶\}}ttƒD ]œ}	td|› d|› d|	t › �ƒ}
t|
ƒ}|D ]"}| ||t|
ƒ||dœ|¥¡ q´|d7 }|d dk�rtd|› d|› d|› d|› d�	ƒ t|ƒtk r† �q$q†| ¡  qvqfW d   ƒ n1 �sF0    Y  tdt› �ƒ d S )Nc                 S   s(   g | ] }t t|ƒtƒD ]}||f‘qqS r   )rz   rr   ÚPLAYER_SNAPSHOT_POS©rO   ÚvÚur   r   r   rP   ½   r^   zrun_players.<locals>.<listcomp>z snapshots x z0 leagues, ~10 pages each (cached pages are free))rq   Ú	update_idÚupdate_dateÚfdco_leaguerW   rg   rh   ri   rU   rV   rj   r   ÚwrA   r    ©Únewliner"   ©Z
fieldnamesz/players?lg=r}   z&set=true&offset=)rq   rŒ   r�   rŽ   rW   r@   r   ú  z pages  (FC/FIFA z	, update r   ú)údone: )r�   r‚   ru   rƒ   rl   ÚopenÚOUT_PLAYERSÚcsvÚ
DictWriterÚwriteheaderr„   ÚrangeÚMAX_PAGES_PER_LEAGUEr?   Ú	PAGE_SIZErn   ÚwriterowrG   Úflush)ÚplanÚfieldsZpagesÚfhr�   rŠ   r‹   rd   r7   Únr=   r…   r<   r   r   r   Úrun_players¼   s8    þþþ",r¤   c                  C   s4  dd„ t D ƒ} t| ƒttƒ }tt| ƒ› dttƒ› d|› d�ƒ g d¢t }d}ttdd	d
d��°}tj||d�}| 	¡  | D ]„\}}t 
¡ D ]r\}}	td|› d|› d�ƒ}
t|
ƒD ] }| ||t|
ƒ|	dœ|¥¡ q²|d7 }|d dkrŒtd|› d|› d�ƒ qŒq|W d   ƒ n1 �s0    Y  tdt› �ƒ d S )Nc                 S   s6   g | ].}t t|ƒd d„ ttƒD ƒƒD ]}||f‘q$qS )c                 S   s   g | ]}|t d   ‘qS rs   )ÚTEAM_UPDATES_PER_VERSION)rO   Úir   r   r   rP   Û   s   ÿz(run_teams.<locals>.<listcomp>.<listcomp>)rz   rr   r›   r¥   r‰   r   r   r   rP   Ú   s   ÿÿzrun_teams.<locals>.<listcomp>z updates x z leagues = z pages)rq   rŒ   r�   rŽ   rU   rV   rW   rX   r   r�   rA   r    r�   r’   r|   r}   r~   )rq   rŒ   r�   rŽ   r@   r   r“   ú/r•   )r�   ru   rƒ   r‚   r`   r–   Ú	OUT_TEAMSr˜   r™   rš   r„   r?   re   rž   rG   )r    Ztotalr¡   r£   r¢   r�   rŠ   r‹   rd   r7   r=   r<   r   r   r   Ú	run_teamsÙ   s.    "ÿÿÿ:r©   Ú__main__z--checkZ
store_truez-one page per league to confirm the league ids)ZactionÚhelpz--teamszteam ratings instead of playersTr%   z	STOPPED: rT   z@
interrupted -- re-run the same command to resume from the cacheé‚   )=Ú__doc__Zargparser˜   r)   rJ   r/   rB   Úsysr0   Zurllib.errorr3   Zurllib.requestZurllib.robotparserÚpathlibr   r'   r-   r.   ZOUT_DIRr(   r—   r¨   r›   r�   r¥   rˆ   r�   rœ   rƒ   r`   rl   ZrobotparserZRobotFileParserr,   r1   Ú	Exceptionr   r?   rG   rN   rR   re   rn   rr   rz   r‡   r¤   r©   r   ZArgumentParserZapZadd_argumentZ
parse_argsÚar8   r4   Zteamsr>   r‚   ÚexitÚKeyboardInterruptr   r   r   r   Ú<module>   sŽ   !
ù	

ÿÿ
