U j*7 @sUddlmZddlZddlZddlZddlmZddlmZddl m Z ddl m Z ddl mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%d d d d d Z&d dd ddZ'd d d ddZ(e)dde*DZ+de,d<dde+DZ-de,d<d dd ddZ.d dd ddZ/d dd d d!Z0d dd d"d#Z1d dd d$d%Z2d dd d&d'Z3d dd d(d)Z4d dd d*d+Z5d dd d,d-Z6d dd d.d/Z7d dd d0d1Z8d dd d2d3Z9d dd d4d5Z:d dd d6d7Z;d dd d8d9ZZ>dfd@d ddAdBdCZ?e ddDd ddEdFdGZ@d@dHdIdJdKZAd ddLdMdNZBdgd dd dPdQdRZCd d dSdTdUdVZDd d ddTdWdXZEdYejFdZfd d d d[d\d]d^ZGdhd@d d`d dddaddddbdc dddeZHdS)i) annotationsN) bisect_right)IncrementalDecoder) lru_cache) Generator)ENCODING_MARKSIANA_SUPPORTED_SIMILARRE_POSSIBLE_ENCODING_INDICATIONUNICODE_RANGES_COMBINED_SECONDARY_RANGE_NAMESCOMMON_CJK_CHARACTERS_LATIN_CJK_HANGUL _KATAKANA _HIRAGANA_HALFWIDTH_KATAKANA_THAI_ARABIC_ARABIC_ISOLATED_FORM _LIGATURE _SUPERSCRIPT_SENTENCE_OPEN_PUNCTUATION_ACCENT_KEYWORDS _ACCENTUATED_KNOWN_MB_DECODERS_KNOWN_MB_CLASSES _IANA_NAMES_MULTIBYTE_SEARCH_RADIUSstrint) characterreturncCszt|}Wntk r$YdSXd}d|kr:|tO}d|krJ|tO}d|krZ|tO}d|krz|tO}d|krz|tO}d|kr|tO}d|kr|t O}d |kr|t O}d |kr|t O}d |ks| d r|t O}d |kr|tO}|dkr|tO}tD]}||kr|tO}qq|S)zRCompute all name-based classification flags with a single unicodedata.name() call.rZLATINCJKZHANGULZKATAKANAZ HALFWIDTHZHIRAGANAZTHAIZARABICz ISOLATED FORMZLIGATUREz LETTER AEZ SUPERSCRIPT>INVERTED EXCLAMATION MARKINVERTED QUESTION MARK) unicodedataname ValueErrorrrrrrrrrrendswithrrrrr)r"descflagskwr.asr?zlist[tuple[int, int, str]]_UNICODE_RANGES_SORTEDcCsg|] }|dqS)rr.)r>er.r.r/ esrBz list[int]_UNICODE_RANGE_STARTSz str | NonecCsTt|}|dkrdS|dkr dStt|d}|dkrPt|\}}}||krP|SdS)zK Retrieve the Unicode range official name from a single character. zControl characterz Basic LatinrrN)ordrrCr@)r"Z character_ordidxr<r=r(r.r.r/ unicode_rangehsrHcCstt|t@Sr2)r1r0rr3r.r.r/is_latin}srIcCs2t|}d|krdSt|}|dkr*dSd|kS)NPTFZ Punctuationr'categoryrHr"character_categorycharacter_ranger.r.r/is_punctuations rPcCsBt|}d|ksd|krdSt|}|dkr2dSd|ko@|dkS)NSNTFZFormsZLorKrMr.r.r/ is_symbols rScCs$t|}|dkrdSd|kp"d|kS)NFZ EmoticonsZ Pictographs)rH)r"rOr.r.r/ is_emoticonsrTcCs.|s|dkrdSt|}d|kp,|dkS)N>+<>｜TZ>PcPoPd)isspacer'rL)r"rNr.r.r/ is_separators r^cCs||kSr2)islowerisupperr3r.r.r/is_case_variablesracCstt|t@Sr2)r1r0rr3r.r.r/is_cjksrbcCstt|t@Sr2)r1r0rr3r.r.r/ is_hiraganasrccCstt|t@Sr2)r1r0rr3r.r.r/ is_katakanasrdcCstt|t@Sr2)r1r0rr3r.r.r/ is_hangulsrecCstt|t@Sr2)r1r0rr3r.r.r/is_thaisrfcCstt|t@Sr2)r1r0rr3r.r.r/ is_arabicsrgcCstt|t@Sr2)r1r0rr3r.r.r/is_arabic_isolated_formsrhcCs|tkSr2)r r3r.r.r/is_cjk_uncommonsri) range_namer#cCs|tkSr2)r )rjr.r.r/is_unicode_range_secondarysrkcCs$| o"| o"|dko"|dkS)Nu)r] isprintabler3r.r.r/is_unprintables rn zbytes | bytearray)sequence search_zoner#c Cst|ttfstt|}|dt||}|}d|krHd|krHdS|jddd}t |D]4}| d dd }t |}|dk r`|Sq`dS) zW Extract using ASCII-only decoder any specified encoding in the first n-bytes. Nscodingscharsetasciiignoreerrorsr-_) isinstancebytes bytearray TypeErrorlenminlowerdecoder finditergroupreplacerget) rprqZseq_lenZ search_bytesZ lowered_bytesZ decoded_zonematchZspecified_encoding encoding_ianar.r.r/any_specified_encodings  r)maxsize)r(r#c CsR|tkr dStD]<}zt||Wntttfk rFYqYnXdSdS)zQ Verify is a specific encoding is a multi byte one based on it IANA name TF)rr importlib import_moduleZgetcodec ImportErrorAttributeError LookupError)r(Zproviderr.r.r/is_multi_byte_encodings  rztuple[str | None, bytes])rpr#cCsJtD]@}t|}t|tr |g}|D]}||r$||fSq$qdS)z9 Identify and extract SIG/BOM in given sequence. )N)rrxry startswith)rp iana_encodingZmarksmarkr.r.r/identify_sig_or_boms  r)rr#cCs|dkS)N>utf_16utf_32r.)rr.r.r/should_strip_sig_or_bom(srT)cp_namestrictr#cCs>|dd}t|}|dk r&|S|r:td|d|S)zIReturns the Python normalized encoding name (Not the IANA official name).rvrwNzUnable to retrieve IANA for '')r~rrrr))rrrr.r.r/ iana_name,s rfloat) iana_name_a iana_name_br#c Cst|st|rdStd|j}td|j}|dd}|dd}d}tdD]*}t|g}||||krX|d7}qX|dS)Ngz encodings.rsrtrr)rrrrrangeryr) rrZ decoder_aZ decoder_bZid_aZid_bZcharacter_match_countiZ to_be_decodedr.r.r/ cp_similarity:s     rcCs|tko|t|kS)z Determine if two code page are at least 80% similar. IANA_SUPPORTED_SIMILAR dict was generated using the function cp_similarity. )r )rrr.r.r/ is_cp_similarNs rZcharset_normalizerz)%(asctime)s | %(levelname)s | %(message)sNone)r(level format_stringr#cCs:t|}||t}|t|||dSr2)logging getLoggersetLevel StreamHandler setFormatter Formatter addHandler)r(rrloggerhandlerr.r.r/set_logging_handlerYs   rFrryzGenerator[str, None, None]) sequencesroffsets chunk_sizebom_or_sig_availablestrip_sig_or_bom sig_payloadis_multi_byte_decoderdecoded_payloaddeferred_decodingr#c cs.|rV|drVt|} t|} |D].} | | | } || | |}|sJqR|Vq"n|r|s|D]"} || | |}|s~q|Vqbn| r|s|n|t|d}|D](} || | |}|sqt||VqnR|D]J} | |}|t|dkrq|| | |}|r |s ||}|j||r0dndd}|r"| dkr"t|d}|d|}d}|rt|} | | t|}td|t}t| |t}||||dk}|r"|s"||kr"t| | d d D]L}|||}|r|s||}|j|dd}|d||krq"q|VqdS) NZiso2022_rsrrtrr6F) rr|r rr}maxrfindr)rrrrrrrrrrZdecoded_lengthZsequence_lengthrZdecoded_offsetchunkZ base_bytesZ cut_sequenceZ chunk_endZchunk_partial_size_chkZ chunk_prefixZ found_nearbyZexpected_offsetZ search_startZ search_endjr.r.r/cut_sequence_chunksfs             r)ro)T)NF)I __future__rrrr'bisectrcodecsr functoolsrtypingrZconstantrr r r r r rrrrrrrrrrrrrrrrrrr0r4r;sorteditemsr@__annotations__rCrHrIrPrSrTr^rarbrcrdrerfrgrhrirkrnrrrrrrrINFOrrr.r.r.r/sb     h*