a
    }`¼jäE  ã                Q   @   s  d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z
ddlZ
ddlmZ dZdZdZedƒZed Zed	 Zed
 ZeddƒZdZedd„ edƒD ƒƒZdedd… iZeZdZdZddddddddddddddd d!d"d#d$œZg d%¢Zd&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d6d7d8d9d:d;d<d=d>d?d@dAdBdCdDdEdFdGdHdIdJdKdLdMdNdOdPdQdRdSdTdUdVdWdXdYdZd[d\d]d^d_d`dadbdcdddedfdgdhdidjdkdldmdndndodpdqdrdsdtduœPZ e!e ƒZ"dvgZ#da$G dwdx„ dxe%ƒZ&dydz„ Z'd{d|„ Z(d}d~„ Z)dd€„ Z*d�Z+d‚dƒ„ Z,d„d…„ Z-d†d‡„ Z.dˆd‰„ Z/dŠd‹„ Z0dŒd�„ Z1dŽd�„ Z2d�d‘„ Z3d’d“„ Z4d”d•„ Z5e6d–k�re 7¡ Z8e8j9d—d˜d™dš� e8j9d›d˜dœdš� e8 :¡ Z;ej<d�d�dž� z0e'ƒ  e;j2�rše2ƒ  ne;j=�rªe5ƒ  ne4ƒ  W nb e&�yð Z> z$e?dŸe>› �ƒ e @d ¡ W Y dZ>[>n.dZ>[>0  eA�y   e?d¡ƒ e @d¢¡ Y n0 dS )£am  Download SoFIFA PLAYER ratings (and optionally team ratings). RUN BY THE USER.

What it collects, for FC 25 (Sept 2024) through FC 27 (2026/27):
  * PLAYERS (default): every player in 18 leagues, with EVERY column SoFIFA's
    player list offers (80: all attributes, face stats, height/weight, foot,
    weak foot, skill moves, work rates, traits, PlayStyles, contract, wage,
    value, release clause...). See PLAYER_COL_NAMES.
    About monthly snapshots per edition (12 evenly spaced ratings updates),
    so each match has ratings from within a few weeks. ~6,000 pages, ~5-6 hours.
  * TEAMS (--teams): each team's overall/attack/midfield/defence, transfer
    budget and club worth, ~monthly. ~2,500 pages, ~2 hours.

How it behaves, deliberately:
  * Identifies itself honestly (USER_AGENT below), not as a browser.
  * Checks sofifa.com/robots.txt before EVERY request and refuses anything it
    disallows (currently /api/ and /players/compare, which it never needs).
  * One request every ~3 seconds; every page cached on disk, so stopping and
    re-running resumes where it left off and never re-downloads.
  * Stops at the first HTTP 403/429 instead of retrying: that is the site
    saying no, and the right response is to stop.

The site's robots.txt also carries `Content-Signal: ai-train=no` and blocks AI
crawlers. Using the data for your own model is your call (2026-09-29).

Usage (from E:\soccer):
  1. Check league ids first -- one page per league, ~1 minute:
       ml_env\Scripts\python.exe research\sofifa_ratings.py --check
  2. Players (default), ~5-6 hours, safe to stop and restart:
       ml_env\Scripts\python.exe research\sofifa_ratings.py
  3. Team ratings too (optional), ~2 hours:
       ml_env\Scripts\python.exe research\sofifa_ratings.py --teams

Output: research/ml_data/sofifa_player_ratings.csv
        research/ml_data/sofifa_team_ratings.csv   (with --teams)
é    N)ÚPathzhttps://sofifa.comz@soccer-research-script/1.0 (personal, non-commercial; ~1 req/3s)g      @zE:\soccer\research\ml_dataZsofifa_cachezsofifa_player_ratings.csvzsofifa_team_ratings.csvé   é   é   c                 c   s   | ]}|d  V  qdS )é   N© ©Ú.0Úir   r   ú$E:\soccer\research\sofifa_ratings.pyÚ	<genexpr>F   ó    r   é   é<   ZE0ZE1ZE2ZE3ZSP1ZSP2ZD1ZD2ZI1ZI2ZF1ZF2ZN1ZP1ZB1ZSC0ZT1ZG1)é   é   r   é=   é5   é6   é   é   é   é    é   é   é
   i4  é   é2   éD   é?   )ÚoaÚatZmdZdfÚtbZcwÚsaZpsZIDZAgez
Birth yearZHeightZWeightzPreferred footzOverall ratingZ	PotentialzBest overallzBest positionZGrowthZJoinedzLoan date endZWageZValuezRelease clausezTotal attackingZCrossingZ	FinishingzHeading accuracyzShort passingZVolleyszTotal skillZ	DribblingZCurvezFK AccuracyzLong passingzBall controlzTotal movementZAccelerationzSprint speedZAgilityZ	ReactionsZBalancezTotal powerz
Shot powerZJumpingZStaminaZStrengthz
Long shotszTotal mentalityZ
AggressionZInterceptionszAttack positionZVisionZ	PenaltiesZ	ComposurezTotal defendingzDefensive awarenesszStanding tacklezSliding tacklezTotal goalkeepingz	GK DivingzGK Handlingz
GK KickingzGK PositioningzGK ReflexeszTotal statsz
Base statsz	Weak footzSkill moveszAttacking work ratezDefensive work ratezInternational reputationz	Body typez	Real facezPace / DivingzShooting / HandlingzPassing / KickingzDribbling / ReflexeszDefending / PacezPhysical / PositioningZTraitsZ
PlayStyleszPlayStyles +zNumber of playstyleszAcceleration typezClub positionzClub kit number)PZpiZaeZbyZhiZwiZpfr    ÚptZboZbpZguZjtZleZwgZvlZrcZtaZcrZfiZheZshZvoZtsZdrZcuZfrZloZblZtoZacZspZagÚreZbaZtpZsoZjuÚstÚsrZlnZteZarZinZpoZviZpeZcmZtdZmar#   ZslZtgZgdZghZgcZgpZgrZttZbsZwkZskZawZdwZirZbtZhcZpacZshoZpasZdriZdefZphyÚt1Út2Úps1Úps2Ztcr!   ZcpZcjg        c                   @   s   e Zd ZdS )ÚRefusedN)Ú__name__Ú
__module__Ú__qualname__r   r   r   r   r,   }   s   r,   c               
   C   s¢  t jjtd dtid�} zFt jj| dd��"}| ¡ jddd�}W d	  ƒ n1 sR0    Y  W nH t jj	y¦ } z,|j
d
v rŽtd|j
› d�ƒ‚d}W Y d	}~n
d	}~0 0 i g d  }}}| ¡ D ]´}| dd¡d  ¡ }d|vrâqÀdd„ | dd¡D ƒ\}	}
|	 ¡ }	|	dk�rF|�rg d }}| |
 ¡ ¡ |D ]}| |g ¡ �q0qÀ|	dv rÀd}|D ]}||  |	dk|
f¡ �qVqÀt dd¡d  ¡ }| || dg ¡¡ad	S )aJ  Fetch robots.txt AS OURSELVES and keep the rules that apply to us.

    Not urllib.robotparser, for two reasons found on 2026-09-29:
      * it downloads robots.txt as 'Python-urllib/3.x', which sofifa answers
        with 403, and it then treats the whole site as disallowed;
      * it applies rules in file order, so sofifa's leading `Allow: /` would
        have permitted /api/, which the site disallows. Standard (RFC 9309)
        semantics are longest match wins -- implemented in allowed() below.
    A 401/403 on robots.txt for OUR user agent is still read as "stay out".
    z/robots.txtú
User-Agent©Zheadersé   ©Ztimeoutúutf-8Úreplace©ÚerrorsN)i‘  é“  zrobots.txt returned HTTP z; to our user agent -- treating the whole site as off-limitsÚ Fú#é   r   ú:c                 s   s   | ]}|  ¡ V  qd S ©N)Ústrip)r	   Úsr   r   r   r   ›   r   zload_robots.<locals>.<genexpr>z
user-agent)ÚallowZdisallowTr@   ú/Ú*)ÚurllibÚrequestÚRequestÚBASEÚ
USER_AGENTÚurlopenÚreadÚdecodeÚerrorÚ	HTTPErrorÚcoder,   Ú
splitlinesÚsplitr>   ÚlowerÚappendÚ
setdefaultÚgetÚ_rules)ÚreqÚrÚtextÚeZgroupsZagentsZin_rulesÚrawÚlineÚkÚvÚaZtokenr   r   r   Úload_robots�   s6    4


r^   c                    sB   ‡fdd„t D ƒ}|sdS tdd„ |D ƒƒ‰ t‡ fdd„|D ƒƒS )zDLongest matching rule wins; on a tie Allow wins; no match = allowed.c                    s*   g | ]"\}}|rˆ   |¡rt|ƒ|f‘qS r   )Ú
startswithÚlen)r	   r]   Úp)Úpathr   r   Ú
<listcomp>­   r   zallowed.<locals>.<listcomp>Tc                 s   s   | ]\}}|V  qd S r=   r   )r	   ÚnÚ_r   r   r   r   °   r   zallowed.<locals>.<genexpr>c                 3   s   | ]\}}|ˆ kr|V  qd S r=   r   )r	   rd   r]   )Úbestr   r   r   ±   r   )rT   ÚmaxÚany)rb   Zhitsr   )rf   rb   r   Úallowed«   s
    ri   c              
   C   sº  t |  }tt | ¡ ¡ ¡ d  }| ¡ r8|jddd�S tdu rFt	ƒ  t
| ƒs\td|› �ƒ‚tt dd¡ t ¡ td   }|dkrŽt |¡ tjj|d	tid
�}zèzFtjj|dd��"}| ¡ jddd�}W d  ƒ n1 sÞ0    Y  W n� tjj�yz } zr|jdv �r$td|j› d|› d�ƒ‚|jdk�rdd| v �rdtd| › �ƒ W Y d}~W t ¡ td< dS ‚ W Y d}~n
d}~0 0 W t ¡ td< nt ¡ td< 0 tjddd� |j|dd� |S )z@Cached, robots-checked, rate-limited GET. Returns the page text.z.htmlr4   r5   )Úencodingr7   Nzrobots.txt disallows r   g      ð?r0   r1   r2   r3   r6   )r8   i­  zHTTP z on z" -- the site is refusing; stoppingi”  zoffset=z  404 past the end: r9   T©ZparentsÚexist_ok)rj   )rF   ÚCACHEÚhashlibÚsha1ÚencodeÚ	hexdigestÚexistsÚ	read_textrT   r^   ri   r,   ÚDELAY_SÚrandomZuniformÚtimeÚ_lastÚsleeprC   rD   rE   rG   rH   rI   rJ   rK   rL   rM   ÚprintÚmkdirZ
write_text)rb   ZurlÚkeyÚwaitrU   rV   rW   rX   r   r   r   Úfetch´   s8     
4ýr}   c                 C   s"   t  d| ¡}|r| d¡ ¡ S dS )Nz/<title>\s*(?:FIFA|FC)\s?\d+\s*-\s*([^-<]+?)\s*-r;   r9   )r%   ÚsearchÚgroupr>   )rW   Úmr   r   r   Ú	page_dateÖ   s    r�   )r(   r)   r*   r+   c              	   C   sp   t  d|› d�| t j¡}|s dS |tv rRt  d| d¡t j¡}d dd„ |D ƒ¡S t t  	d	d| d¡¡¡ 
¡ S )
Nz
data-col="z"[^>]*>(.*?)</td>r9   z<span[^>]*>(.*?)</span>r;   z; c                 s   s&   | ]}t  t d d|¡¡ ¡ V  qdS )ú<[^>]+>r9   N)ÚhtmlÚunescaper%   Úsubr>   )r	   Úxr   r   r   r   ä   r   z_cell.<locals>.<genexpr>r‚   )r%   r~   ÚSÚ	LIST_COLSÚfindallr   Újoinrƒ   r„   r…   r>   )ÚtrZcolr€   Úitemsr   r   r   Ú_cellÞ   s    r�   c                 C   s   dd„ t  d| t j¡D ƒS )Nc                 S   s   g | ]}d |v r|‘qS )zdata-col="oa"r   )r	   r‹   r   r   r   rc   é   s   ÿz_rows.<locals>.<listcomp>z<tr[^>]*>(.*?)</tr>)r%   r‰   r‡   )rW   r   r   r   Ú_rowsè   s    rŽ   c              	      sª   g }t | ƒD ]˜‰ t dˆ ¡}t dˆ ¡}|r6| d¡nd|rPt | d¡¡ ¡ nd|r`| d¡nd|rzt | d¡¡ ¡ nddœ}| ‡ fdd„tD ƒ¡ | 	|¡ q|S )	Nú)href="/team/(\d+)/[^/]+/\d+/">([^<]+)</a>z%href="/league/(\d+)"[^>]*>([^<]+)</a>r;   r9   é   )Úteam_idÚ	team_nameÚsofifa_league_idÚsofifa_league_namec                    s   i | ]}|t ˆ |ƒ“qS r   ©r�   ©r	   Úc©r‹   r   r   Ú
<dictcomp>ö   r   zparse_teams.<locals>.<dictcomp>)
rŽ   r%   r~   r   rƒ   r„   r>   ÚupdateÚ	TEAM_COLSrQ   )rW   ÚoutÚtÚlgZrowr   r˜   r   Úparse_teamsí   s    ýrŸ   c                    s\  g }t | ƒD �]H‰ t dˆ ¡}t dˆ ¡}t dˆ tj¡}| |rL| d¡nd|rft | d¡¡ ¡ ndd 	t 
dˆ  d	d¡d
 ¡¡|r’| d¡nd|r¬t | d¡¡ ¡ nd|rÚt ddt t dd| d¡¡¡¡ ¡ nddœ‡ fdd„tD ƒ¥¡ tD ]Z}t d|d | ¡}|�r*| d¡|d |< |dv rú|�rB| d¡nd|d |› d�< qúq|S )Nz;href="/player/(\d+)/[^"]*"[^>]*data-tippy-content="([^"]+)"r�   z<div class="sub">(.*?)</div>r;   r9   r�   ú zclass="pos pos\d+">([A-Z]+)<z	data-col=r   z\s+r‚   )Ú	player_idÚplayer_nameÚ	positionsr‘   r’   Úcontractc                    s   i | ]}|t ˆ |ƒ“qS r   r•   r–   r˜   r   r   r™     r   z!parse_players.<locals>.<dictcomp>z(\d+)\s*([+-]\d+)éÿÿÿÿ)r    r$   Z_chg)rŽ   r%   r~   r‡   rQ   r   rƒ   r„   r>   rŠ   r‰   rO   r…   ÚPLAYER_COLSZ	fullmatch)rW   rœ   ra   r�   r¤   r—   r€   r   r˜   r   Úparse_playersû   s8    
ÿþÿ
õô&r§   c                 C   s0   t d| d›d�ƒ}ttt d| d›d�|¡ƒƒS )zHEvery ratings-update id SoFIFA lists for one game version, oldest first.z/teams?lg=13&r=Z02dz0001&set=truezr=(z\d{4}))r}   ÚsortedÚsetr%   r‰   )ÚversionrW   r   r   r   Úall_updates  s    r«   c                    s   ˆ sg S t ‡ fdd„|D ƒƒS )Nc              	      s2   h | ]*}ˆ t tˆ ƒd  t|tˆ ƒd   ƒƒ ’qS ©r;   )Úminr`   Úround)r	   ra   ©Úidsr   r   Ú	<setcomp>(  r   zpick.<locals>.<setcomp>)r¨   )r°   r£   r   r¯   r   Úpick%  s    r²   c                  C   s”   t ttƒƒd } td| › �ƒ t ¡ D ]h\}}ttd|› d| › d�ƒƒ}dd„ |D ƒ}td|d	›d
|d	›dt|ƒd›dd 	t
|ƒ¡p†d› �ƒ q&d S )Nr¥   zlatest update ú
/teams?lg=ú&r=ú	&set=truec                 S   s   h | ]}|d  ’qS )r”   r   )r	   rV   r   r   r   r±   0  r   zcheck.<locals>.<setcomp>z  lg=z<4z -> r    z>3z teams  shown as: ú, zNOTHING -- wrong id)r«   rg   ÚVERSIONSry   ÚLEAGUESrŒ   rŸ   r}   r`   rŠ   r¨   )Zlatestrž   rM   ÚrowsÚnamesr   r   r   Úcheck+  s     ÿr»   c                 C   s0   d  dd„ tD ƒ¡}d| › d|› d|› d|› �S )Nr9   c                 s   s   | ]}d |› �V  qdS )z&showCol%5B%5D=Nr   r–   r   r   r   r   6  r   zplayers_path.<locals>.<genexpr>z/players?lg=r´   rµ   z&offset=)rŠ   r¦   )rž   rš   ÚoffsetZcolsr   r   r   Úplayers_path5  s    r½   c                  C   st  dd„ t D ƒ} tt| ƒ› dttƒ› d�ƒ g d¢t ddg }d}t d	¡}t|d
ddd��ê}tj	||d�}| 
¡  | D ]¾\}}t ¡ D ]¬\}}	ttƒD ]’}
tt|||
t ƒƒ}t|ƒ}|D ]"}| ||t|ƒ|	|dœ|¥¡ q¼|d7 }|d dk�rtd|› d|› d|› d|	› d�	ƒ t|ƒtk r˜ �q,q˜| ¡  qˆqxW d   ƒ n1 �sN0    Y  | t¡ tdt› �ƒ d S )Nc                 S   s0   g | ](}t t|ƒt |t¡ƒD ]}||f‘qqS r   )r²   r«   ÚPLAYER_SNAPSHOT_POSrS   ÚPLAYER_SNAPSHOT_DEFAULT©r	   r\   Úur   r   r   rc   ;  s   
ÿÿzrun_players.<locals>.<listcomp>z snapshots x z0 leagues, ~10 pages each (cached pages are free))rª   Ú	update_idÚupdate_dateÚfdco_leaguer“   r¡   r¢   r£   r‘   r’   r¤   Zoa_chgZpt_chgr   z.partial.csvÚwr9   r4   ©Únewlinerj   ©Z
fieldnames)rª   rÂ   rÃ   rÄ   r“   r;   r   ú  z pages  (FC/FIFA z	, update r¶   ú)údone: )r·   ry   r`   r¸   r¦   ÚOUT_PLAYERSZwith_suffixÚopenÚcsvÚ
DictWriterÚwriteheaderrŒ   ÚrangeÚMAX_PAGES_PER_LEAGUEr}   r½   Ú	PAGE_SIZEr§   Úwriterowr�   Úflushr5   )ÚplanÚfieldsZpagesÚpartÚfhrÅ   r\   rÁ   rž   rM   rd   rW   r¹   rV   r   r   r   Úrun_players:  s@    þþ
þþ",
rÚ   c                  C   s4  dd„ t D ƒ} t| ƒttƒ }tt| ƒ› dttƒ› d|› d�ƒ g d¢t }d}ttdd	d
d��°}tj||d�}| 	¡  | D ]„\}}t 
¡ D ]r\}}	td|› d|› d�ƒ}
t|
ƒD ] }| ||t|
ƒ|	dœ|¥¡ q²|d7 }|d dkrŒtd|› d|› d�ƒ qŒq|W d   ƒ n1 �s0    Y  tdt› �ƒ d S )Nc                 S   s6   g | ].}t t|ƒd d„ ttƒD ƒƒD ]}||f‘q$qS )c                 S   s   g | ]}|t d   ‘qS r¬   )ÚTEAM_UPDATES_PER_VERSIONr   r   r   r   rc   _  s   ÿz(run_teams.<locals>.<listcomp>.<listcomp>)r²   r«   rÑ   rÛ   rÀ   r   r   r   rc   ^  s   ÿÿzrun_teams.<locals>.<listcomp>z updates x z leagues = z pages)rª   rÂ   rÃ   rÄ   r‘   r’   r“   r”   r   rÅ   r9   r4   rÆ   rÈ   r³   r´   rµ   )rª   rÂ   rÃ   rÄ   r;   r   rÉ   rA   rË   )r·   r`   r¸   ry   r›   rÍ   Ú	OUT_TEAMSrÎ   rÏ   rÐ   rŒ   r}   rŸ   rÔ   r�   )rÖ   Ztotalr×   rd   rÙ   rÅ   r\   rÁ   rž   rM   rW   rV   r   r   r   Ú	run_teams]  s.    "ÿÿÿ:rÝ   Ú__main__z--checkZ
store_truez-one page per league to confirm the league ids)ZactionÚhelpz--teamszteam ratings instead of playersTrk   z	STOPPED: r�   z@
interrupted -- re-run the same command to resume from the cacheé‚   )BÚ__doc__ZargparserÎ   rn   rƒ   ru   r%   Úsysrv   Zurllib.errorrC   Zurllib.requestÚpathlibr   rF   rG   rt   ZOUT_DIRrm   rÌ   rÜ   rÑ   r·   rÛ   ÚtupleZ_MONTHLYr¾   r¿   rÓ   rÒ   r¸   r›   ZPLAYER_COL_NAMESÚlistr¦   rw   rT   Ú	Exceptionr,   r^   ri   r}   r�   rˆ   r�   rŽ   rŸ   r§   r«   r²   r»   r½   rÚ   rÝ   r-   ZArgumentParserZapZadd_argumentZ
parse_argsr]   rz   ZteamsrX   ry   ÚexitÚKeyboardInterruptr   r   r   r   Ú<module>   sÊ   #
	ù	
è*	"
$
#
ÿÿ
