a &j8 @sUddlmZddlZddlZddlZddlmZddlmZddl m Z ddl m Z ddl mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%d d d d d Z&d dd ddZ'd d d ddZ(e)dde*DZ+de,d<dde+DZ-de,d<d dd ddZ.d dd ddZ/d dd d d!Z0d dd d"d#Z1d dd d$d%Z2d dd d&d'Z3d dd d(d)Z4d dd d*d+Z5d dd d,d-Z6d dd d.d/Z7d dd d0d1Z8d dd d2d3Z9d dd d4d5Z:d dd d6d7Z;d dd d8d9ZZ>dfd@d ddAdBdCZ?e ddDd ddEdFdGZ@d@dHdIdJdKZAd ddLdMdNZBdgd dd dPdQdRZCd d dSdTdUdVZDd d ddTdWdXZEdYejFdZfd d d d[d\d]d^ZGdhd@d d`d dddaddddbdc dddeZHdS)i) annotationsN) bisect_right)IncrementalDecoder) lru_cache) Generator)ENCODING_MARKSIANA_SUPPORTED_SIMILARRE_POSSIBLE_ENCODING_INDICATIONUNICODE_RANGES_COMBINED_SECONDARY_RANGE_NAMESCOMMON_CJK_CHARACTERS_LATIN_CJK_HANGUL _KATAKANA _HIRAGANA_HALFWIDTH_KATAKANA_THAI_ARABIC_ARABIC_ISOLATED_FORM _LIGATURE _SUPERSCRIPT_SENTENCE_OPEN_PUNCTUATION_ACCENT_KEYWORDS _ACCENTUATED_KNOWN_MB_DECODERS_KNOWN_MB_CLASSES _IANA_NAMES_MULTIBYTE_SEARCH_RADIUSstrint) characterreturncCszt|}Wnty"YdS0d}d|vr8|tO}d|vrH|tO}d|vrX|tO}d|vrx|tO}d|vrx|tO}d|vr|tO}d|vr|t O}d |vr|t O}d |vr|t O}d |vs| d r|t O}d |vr|tO}|dvr|tO}tD]}||vr|tO}qq|S)zRCompute all name-based classification flags with a single unicodedata.name() call.rZLATINZCJKZHANGULZKATAKANAZ HALFWIDTHZHIRAGANAZTHAIZARABICz ISOLATED FORMZLIGATUREz LETTER AEZ SUPERSCRIPT>zINVERTED QUESTION MARKzINVERTED EXCLAMATION MARK) unicodedataname ValueErrorrrrrrrrrrendswithrrrrr)r"descflagskwr+>E:\soccer\ml_env\Lib\site-packages\charset_normalizer/utils.py_character_flags'sD r-boolcCstt|t@SN)r.r-rr"r+r+r,is_accentuatedQsr1cCs.t|}|s|S|d}tt|ddS)N r)r$ decompositionsplitchrr!)r"Z decomposedcodesr+r+r, remove_accentUs   r8ccs |]\}}|j|j|fVqdSr/)startstop).0r%Z ord_ranger+r+r, asr<zlist[tuple[int, int, str]]_UNICODE_RANGES_SORTEDcCsg|] }|dqS)rr+)r;er+r+r, er?z list[int]_UNICODE_RANGE_STARTSz str | NonecCsTt|}|dkrdS|dkr dStt|d}|dkrPt|\}}}||krP|SdS)zK Retrieve the Unicode range official name from a single character. zControl characterz Basic LatinrrN)ordrrAr=)r"Z character_ordidxr9r:r%r+r+r, unicode_rangehsrFcCstt|t@Sr/)r.r-rr0r+r+r,is_latin}srGcCs2t|}d|vrdSt|}|dur*dSd|vS)NPTF Punctuationr$categoryrFr"character_categorycharacter_ranger+r+r,is_punctuations rOcCsBt|}d|vsd|vrdSt|}|dur2dSd|vo@|dkS)NSNTFZFormsZLorJrLr+r+r, is_symbols rRcCs$t|}|durdSd|vp"d|vS)NFZ EmoticonsZ Pictographs)rF)r"rNr+r+r, is_emoticonsrScCs.|s|dvrdSt|}d|vp,|dvS)N>>u｜+ZPcZPdZPo)isspacer$rK)r"rMr+r+r, is_separators rYcCs||kSr/)islowerisupperr0r+r+r,is_case_variablesr\cCstt|t@Sr/)r.r-rr0r+r+r,is_cjksr]cCstt|t@Sr/)r.r-rr0r+r+r, is_hiraganasr^cCstt|t@Sr/)r.r-rr0r+r+r, is_katakanasr_cCstt|t@Sr/)r.r-rr0r+r+r, is_hangulsr`cCstt|t@Sr/)r.r-rr0r+r+r,is_thaisracCstt|t@Sr/)r.r-rr0r+r+r, is_arabicsrbcCstt|t@Sr/)r.r-rr0r+r+r,is_arabic_isolated_formsrccCs|tvSr/)r r0r+r+r,is_cjk_uncommonsrd) range_namer#cCs|tvSr/)r )rer+r+r,is_unicode_range_secondarysrfcCs$| o"| o"|dko"|dkS)Nu)rX isprintabler0r+r+r,is_unprintables ri zbytes | bytearray)sequence search_zoner#c Cst|ttfstt|}|dt||}|}d|vrHd|vrHdS|jddd}t |D]4}| d dd }t |}|dur`|Sq`dS) zW Extract using ASCII-only decoder any specified encoding in the first n-bytes. Nscodingscharsetasciiignoreerrorsr-_) isinstancebytes bytearray TypeErrorlenminlowerdecoder finditergroupreplacerget) rkrlZseq_lenZ search_bytesZ lowered_bytesZ decoded_zonematchZspecified_encoding encoding_ianar+r+r,any_specified_encodings  r)maxsize)r%r#c CsP|tvr dStD]:}zt||WntttfyDYqYn0dSdS)zQ Verify is a specific encoding is a multi byte one based on it IANA name TF)rr importlib import_moduleZgetcodec ImportErrorAttributeError LookupError)r%providerr+r+r,is_multi_byte_encodings  rztuple[str | None, bytes])rkr#cCsJtD]@}t|}t|tr |g}|D]}||r$||fSq$qdS)z9 Identify and extract SIG/BOM in given sequence. )Nr@)rrsrt startswith)rk iana_encodingZmarksmarkr+r+r,identify_sig_or_boms  r)rr#cCs|dvS)N>utf_16utf_32r+)rr+r+r,should_strip_sig_or_bom(srT)cp_namestrictr#cCs>|dd}t|}|dur&|S|r:td|d|S)zIReturns the Python normalized encoding name (Not the IANA official name).rqrrNzUnable to retrieve IANA for '')ryr}rr~r&)rrrr+r+r, iana_name,s rfloat) iana_name_a iana_name_br#c Cst|st|rdStd|j}td|j}|dd}|dd}d}tdD]*}t|g}||||krX|d7}qX|dS)Ngz encodings.rnrorr)rrrrrangertrz) rrZ decoder_aZ decoder_bZid_aZid_bZcharacter_match_countiZ to_be_decodedr+r+r, cp_similarity:s     rcCs|tvo|t|vS)z Determine if two code page are at least 80% similar. IANA_SUPPORTED_SIMILAR dict was generated using the function cp_similarity. )r )rrr+r+r, is_cp_similarNs rZcharset_normalizerz)%(asctime)s | %(levelname)s | %(message)sNone)r%level format_stringr#cCs:t|}||t}|t|||dSr/)logging getLoggersetLevel StreamHandler setFormatter Formatter addHandler)r%rrloggerhandlerr+r+r,set_logging_handlerYs   rFrrtzGenerator[str, None, None]) sequencesroffsets chunk_sizebom_or_sig_availablestrip_sig_or_bom sig_payloadis_multi_byte_decoderdecoded_payloaddeferred_decodingr#c cs.|rV|drVt|} t|} |D].} | | | } || | |}|sJqR|Vq"n|r|s|D]"} || | |}|s~q|Vqbn| r|s|n|t|d}|D](} || | |}|sqt||VqnR|D]J} | |}|t|dkrq|| | |}|r |s ||}|j||r0dndd}|r"| dkr"t|d}|d|}d}|rt|} | | t|}td|t}t| |t}||||dk}|r"|s"||vr"t| | d d D]L}|||}|r|s||}|j|dd}|d||vrq"q|VqdS) NZiso2022_rnrrorr3F) rrwr rzrxmaxrfindr)rrrrrrrrrrZdecoded_lengthZsequence_lengthrZdecoded_offsetchunkZ base_bytesZ cut_sequenceZ chunk_endZchunk_partial_size_chkZ chunk_prefixZ found_nearbyZexpected_offsetZ search_startZ search_endjr+r+r,cut_sequence_chunksfs~             r)rj)T)NF)I __future__rrrr$bisectrcodecsr functoolsrtypingrZconstantrr r r r r rrrrrrrrrrrrrrrrrrr-r1r8sorteditemsr=__annotations__rArFrGrOrRrSrYr\r]r^r_r`rarbrcrdrfrirrrrrrrINFOrrr+r+r+r,s`    h*