U
    j                     @  s  U d dl mZ d dlmZ d dlmZ ddlmZmZm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ ddlmZmZmZm Z m!Z!m"Z"m#Z# eeB eB eB eB Z$de%d< G d	d
 d
Z&edddd
dddZ'dd e(dD Z)de%d< G dd dZ*G dd de*Z+G dd de*Z,G dd de*Z-G dd de*Z.G d d! d!e*Z/G d"d# d#e*Z0G d$d% d%e*Z1G d&d' d'e*Z2G d(d) d)e*Z3G d*d+ d+e*Z4eddd,d,d-d.d/d0Z5d7dd3d-d3d4d5d6Z6dS )8    )annotations)	lru_cache)	getLogger   )COMMON_CJK_CHARACTERSCOMMON_SAFE_ASCII_CHARACTERSTRACECompatibleFamillyRange_ACCENTUATED_ARABIC_ARABIC_ISOLATED_FORM&_BASIC_LATIN_COMPATIBLE_RANGE_FAMILIES_CJK_COMPATIBLE_RANGE_FAMILIES#_COMPATIBLE_WITH_ANY_RANGE_FAMILIES_HANGUL_HALFWIDTH_KATAKANA	_HIRAGANA	_KATAKANA_LATIN	_LIGATURE_RANGE_FAMILIES_SENTENCE_OPEN_PUNCTUATION_SUPERSCRIPT_THAI)_character_flagsis_emoticonis_punctuationis_separator	is_symbolremove_accentunicode_rangeint_GLYPH_MASKc                   @  s  e Zd ZU dZdZded< ded< ded< ded< ded	< ded
< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded < ded!< ded"< dd#d$d%d&Zd'S )(CharInfoz>Pre-computed character properties shared across all detectors.)	character	printablealphaupperlowerspacedigitis_asciicase_variableflagsaccentuatedlatinis_cjkis_katakanais_halfwidth_katakana	is_arabicis_ligatureis_superscriptis_sentence_open_punctuationis_glyphpunctsymrangesepemoticonsafe
common_cjk
unaccentedstrr%   boolr&   r'   r(   r)   r*   r+   r,   r-   r"   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   
str | Noner;   r<   r=   r>   r?   r@   Noner%   returnc                 C  s  || _ t|}|dk rfd| _d| _|| _d| _d| _|tk| _d| _	d| _
d| _d| _d| _d| _d| _d| _d|  krdkrn nFd| _d| _d| _d| _d| _d| _d| _t| _d| _d| _d| _qd|  krdkr.n nFd| _d| _d| _d| _d| _d| _d| _t| _d| _d| _d| _qd|  krFd	krn nDd| _d| _d| _d| _d| _d| _d| _d
| _d| _d| _d| _n|dksd|  krdkrn nHd| _d| _d| _d| _d| _|dk| _d| _d
| _d| _d| _d| _nf| | _d| _d| _d| _d| _d| _d| _d
| _d| _| jrFt|nd| _| jr\t|nd| _nhd| _d| _| | _|  | _|! | _|" | _|# | _|$ | _| j| jk| _t%|}| jrd| _n
t&|| _|| _t'|t(@ | _t'|t@ | _t'|t)@ | _	t'|t*@ | _
t'|t+@ | _t'|t,@ | _t'|t-@ | _t'|t.@ | _t'|t/@ | _t'|t0@ | _| jr| jrt1|| _n|| _| j	o|t2k| _| jrt|| _t|| _nd| _d| _t3|| _4t5|| _6dS )zkCompute all properties for *character* (built once per codepoint,
        every branch assigns every slot).   TFA   Z   a   z   0   9   r       	      N)7r%   ordr,   r/   r@   r=   r?   r   r>   r1   r2   r3   r4   r5   r6   r7   r8   r'   r(   r)   r*   r+   r&   r-   r   r.   r0   r9   r:   isprintabler   r   isalphaisupperislowerisspaceisdigitr   r   rB   r
   r   r   r   r   r   r   r   r#   r    r   r!   r;   r   r<   )selfr%   or.    rZ   9/tmp/pip-unpacked-wheel-bwd693va/charset_normalizer/md.py__init__k   s    



&










zCharInfo.__init__N)__name__
__module____qualname____doc__	__slots____annotations__r\   rZ   rZ   rZ   r[   r$   ,   s>   
r$   N)maxsizerA   rE   c                 C  s   t | S )zBBuild (once per codepoint) and cache the CharInfo for *character*.)r$   )r%   rZ   rZ   r[   
_char_info   s    rd   c                 C  s   g | ]}t t|qS rZ   )r$   chr).0Z
_codepointrZ   rZ   r[   
<listcomp>   s    rg   rG   zlist[CharInfo]_ASCII_CHAR_INFOc                   @  sF   e Zd ZdZdZddddddZdd	d
dZedd	ddZdS )MessDetectorPluginzy
    Base abstract class used for mess detection plugins.
    All detectors MUST extend and implement given methods.
    rZ   rA   r$   rD   r%   inforF   c                 C  s   t dS )z
        The main routine to be executed upon character.
        Insert the logic in witch the text would be considered chaotic.
        NNotImplementedErrorrX   r%   rk   rZ   rZ   r[   	feed_info  s    zMessDetectorPlugin.feed_inforF   c                 C  s   t dS )zB
        Permit to reset the plugin to the initial state.
        Nrl   rX   rZ   rZ   r[   reset  s    zMessDetectorPlugin.resetfloatc                 C  s   t dS )z
        Compute the chaos ratio based on what your feed() has seen.
        Must NOT be lower than 0.; No restriction gt 0.
        Nrl   rq   rZ   rZ   r[   ratio  s    zMessDetectorPlugin.ratioN)	r]   r^   r_   r`   ra   ro   rr   propertyrt   rZ   rZ   rZ   r[   ri     s   ri   c                   @  sP   e Zd ZdZddddZddddd	d
ZddddZeddddZdS ) TooManySymbolOrPunctuationPlugin_punctuation_count_symbol_count_character_count_last_printable_charrD   rp   c                 C  s   d| _ d| _d| _d | _d S Nr   rw   rq   rZ   rZ   r[   r\   *  s    z)TooManySymbolOrPunctuationPlugin.__init__rA   r$   rj   c                 C  s^   |  j d7  _ || jkrT|jsT|jr4|  jd7  _n |jsT|jrT|jsT|  jd7  _|| _dS )1Optimized feed using pre-computed character info.r      N)	rz   r{   r>   r9   rx   r+   r:   r=   ry   rn   rZ   rZ   r[   ro   1  s    z*TooManySymbolOrPunctuationPlugin.feed_infoc                 C  s   d| _ d| _d| _d S r|   )rx   rz   ry   rq   rZ   rZ   r[   rr   =  s    z&TooManySymbolOrPunctuationPlugin.resetrs   c                 C  s0   | j dkrdS | j| j | j  }|dkr,|S dS )Nr           333333?)rz   rx   ry   )rX   Zratio_of_punctuationrZ   rZ   r[   rt   B  s    

z&TooManySymbolOrPunctuationPlugin.ratioN	r]   r^   r_   ra   r\   ro   rr   ru   rt   rZ   rZ   rZ   r[   rv   "  s   rv   c                   @  sP   e Zd ZdZddddZddddd	d
ZddddZeddddZdS )TooManyAccentuatedPluginrz   _accentuated_countrD   rp   c                 C  s   d| _ d| _d S r|   r   rq   rZ   rZ   r[   r\   Q  s    z!TooManyAccentuatedPlugin.__init__rA   r$   rj   c                 C  s&   |  j d7  _ |jr"|  jd7  _dS r}   r   N)rz   r/   r   rn   rZ   rZ   r[   ro   U  s    z"TooManyAccentuatedPlugin.feed_infoc                 C  s   d| _ d| _d S r|   r   rq   rZ   rZ   r[   rr   \  s    zTooManyAccentuatedPlugin.resetrs   c                 C  s*   | j dk rdS | j| j  }|dkr&|S dS )N   r   gffffff?r   )rX   Zratio_of_accentuationrZ   rZ   r[   rt   `  s    
zTooManyAccentuatedPlugin.ratioNr   rZ   rZ   rZ   r[   r   N  s   r   c                   @  sP   e Zd ZdZddddZddddd	d
ZddddZeddddZdS )UnprintablePlugin_unprintable_countrz   _has_escaperD   rp   c                 C  s   d| _ d| _d| _d S Nr   Fr   rq   rZ   rZ   r[   r\   l  s    zUnprintablePlugin.__init__rA   r$   rj   c                 C  sJ   |dkrd| _ |js8|js8|dkr8|dkr8|  jd7  _|  jd7  _dS )r}   Tu   ﻿r   N)r   r&   r*   r   rz   rn   rZ   rZ   r[   ro   q  s    zUnprintablePlugin.feed_infoc                 C  s   d| _ d| _d S r   )r   r   rq   rZ   rZ   r[   rr     s    zUnprintablePlugin.resetrs   c                 C  s(   | j dkrdS | jrdS | jd | j  S )Nr   r         ?r   )rz   r   r   rq   rZ   rZ   r[   rt     s
    
zUnprintablePlugin.ratioNr   rZ   rZ   rZ   r[   r   i  s   r   c                   @  sP   e Zd ZdZddddZddddd	d
ZddddZeddddZdS )SuspiciousDuplicateAccentPlugin_successive_countrz   _last_latin_character_last_was_accentuatedrD   rp   c                 C  s   d| _ d| _d | _d| _d S r   r   rq   rZ   rZ   r[   r\     s    z(SuspiciousDuplicateAccentPlugin.__init__rA   r$   rj   c                 C  sn   |  j d7  _ | jdk	r\|jr\| jr\|jr@| jjr@|  jd7  _|j| jjkr\|  jd7  _|| _|j| _dS r   )rz   r   r/   r   r(   r   r@   rn   rZ   rZ   r[   ro     s    z)SuspiciousDuplicateAccentPlugin.feed_infoc                 C  s   d| _ d| _d | _d| _d S r   r   rq   rZ   rZ   r[   rr     s    z%SuspiciousDuplicateAccentPlugin.resetrs   c                 C  s   | j dkrdS | jd | j  S )Nr   r   r~   )rz   r   rq   rZ   rZ   r[   rt     s    
z%SuspiciousDuplicateAccentPlugin.ratioNr   rZ   rZ   rZ   r[   r     s   r   c                   @  sP   e Zd ZdZddddZddddd	d
ZddddZeddddZdS )SuspiciousRange"_suspicious_successive_range_countrz   _last_printable_seen_last_printable_rangerD   rp   c                 C  s   d| _ d| _d | _d | _d S r|   r   rq   rZ   rZ   r[   r\     s    zSuspiciousRange.__init__rA   r$   rj   c                 C  s   |  j d7  _ |js |js |jr0d| _d| _dS | jdkrL|| _|j| _dS | j}|j}||ksh|dkrt||r|  jd7  _|| _|| _dS r   )	rz   r*   r9   r>   r   r   r;    is_suspiciously_successive_ranger   )rX   r%   rk   unicode_range_aunicode_range_brZ   rZ   r[   ro     s     

zSuspiciousRange.feed_infoc                 C  s   d| _ d| _d | _d | _d S r|   )rz   r   r   r   rq   rZ   rZ   r[   rr     s    zSuspiciousRange.resetrs   c                 C  s"   | j dkrdS | jd | j  }|S )NrP   r   r~   )rz   r   )rX   Zratio_of_suspicious_range_usagerZ   rZ   r[   rt     s    
zSuspiciousRange.ratioNr   rZ   rZ   rZ   r[   r     s   r   c                   @  sP   e Zd ZdZddddZddddd	d
ZddddZeddddZdS )SuperWeirdWordPlugin_word_count_bad_word_count_foreign_long_count_is_current_word_bad_foreign_long_watchrz   _bad_character_count_buffer_length_buffer_last_char_buffer_last_char_accentuated_buffer_accent_count_buffer_glyph_count_buffer_upper_count_buffer_first_lower_buffer_has_non_ascii_buffer_last_char_ligature_buffer_has_internal_ligature_is_current_word_invalid_invalid_word_countrD   rp   c                 C  sv   d| _ d| _d| _d| _d| _d| _d| _d| _d | _d| _	d| _
d| _d| _d| _d| _d| _d| _d| _d| _d S r   r   rq   rZ   rZ   r[   r\   
  s&    zSuperWeirdWordPlugin.__init__rA   r$   rj   c                 C  s  |j r| jrd| _|j| _| jdkr,|j| _|  jd7  _|| _|jrT|  j	d7  _	|j
s`d| _|j| _|jr||  jd7  _|jr|  jd7  _n| js|jr|jrd| _dS | jsdS |js|jr| jrd| _d| _|js|js|jr|  jd7  _| j}|  j|7  _|dkr| j| dkr.d| _n| jrd| j rd| j	|krd|  jd7  _d| _nV| jdkrd| _|  jd7  _n4| jr| jr| j	|d kr|  jd7  _d| _|dkr | jr | j	dko| j	| dk}|s |  jd7  _d| _| jr*|  jd7  _|  j|7  _d	| _| jrF|  j d7  _ d	| _d	| _d| _d| _d	| _d| _d| _d| _	d	| _d	| _d	| _d	| _n:|d
kr|j!s|j"rd| _|  jd7  _|| _d	| _dS )r}   Tr   r   N         ?   r   F>   |=~_-<>)#r'   r   r   r5   r   r)   r   r   r(   r   r,   r   r/   r   r   r8   r   r   r0   r7   r6   r   r   r*   r9   r<   r   rz   rT   r   r   r   r   r+   r:   )rX   r%   rk   Zbuffer_lengthZprobable_camel_casedrZ   rZ   r[   ro   "  s    

zSuperWeirdWordPlugin.feed_infoc                 C  sv   d| _ d | _d| _d| _d| _d| _d| _d| _d| _d| _	d| _
d| _d| _d| _d| _d| _d| _d| _d| _d S r   )r   r   r   r   r   r   r   rz   r   r   r   r   r   r   r   r   r   r   r   rq   rZ   rZ   r[   rr     s&    zSuperWeirdWordPlugin.resetrs   c                 C  s.   | j r
dS | jdkr"| jdkr"dS | j| j S )Nr   
   r   r   )r   r   r   r   rz   rq   rZ   rZ   r[   rt     s
    zSuperWeirdWordPlugin.ratioNr   rZ   rZ   rZ   r[   r     s   dr   c                   @  sT   e Zd ZdZdZddddZdddd	d
dZddddZeddddZ	dS )CjkUncommonPluginz<
    Detect messy CJK text that probably means nothing.
    rz   _uncommon_countrD   rp   c                 C  s   d| _ d| _d S r|   r   rq   rZ   rZ   r[   r\     s    zCjkUncommonPlugin.__init__rA   r$   rj   c                 C  s&   |  j d7  _ |js"|  jd7  _dS r   )rz   r?   r   rn   rZ   rZ   r[   ro     s    zCjkUncommonPlugin.feed_infoc                 C  s   d| _ d| _d S r|   r   rq   rZ   rZ   r[   rr     s    zCjkUncommonPlugin.resetrs   c                 C  s.   | j dk rdS | j| j  }|dkr*|d S dS )Nr   r   r      r   )rX   Zuncommon_form_usagerZ   rZ   r[   rt     s    
zCjkUncommonPlugin.ratioN
r]   r^   r_   r`   ra   r\   ro   rr   ru   rt   rZ   rZ   rZ   r[   r     s   r   c                   @  sT   e Zd ZdZdZddddZdddd	d
dZddddZeddddZ	dS )SuspiciousKatakanaPluginzDDetect implausible halfwidth Katakana and uncommon CJK combinations._katakana_count_halfwidth_katakana_count
_cjk_count_uncommon_cjk_countrD   rp   c                 C  s   d| _ d| _d| _d| _d S r|   r   rq   rZ   rZ   r[   r\     s    z!SuspiciousKatakanaPlugin.__init__rA   r$   rj   c                 C  sR   |j r,|  jd7  _|jr(|  jd7  _dS |  jd7  _|jsN|  jd7  _dS r   )r2   r   r3   r   r   r?   r   rn   rZ   rZ   r[   ro     s    z"SuspiciousKatakanaPlugin.feed_infoc                 C  s   d| _ d| _d| _d| _d S r|   r   rq   rZ   rZ   r[   rr     s    zSuspiciousKatakanaPlugin.resetrs   c                 C  s:   | j dkr6| j | jkr6d| j  kr.| jkr6n ndS dS )Nr      r   r   )r   r   r   r   rq   rZ   rZ   r[   rt     s    
 
zSuspiciousKatakanaPlugin.ratioNr   rZ   rZ   rZ   r[   r     s   r   c                   @  sP   e Zd ZdZddddZddddd	d
ZddddZeddddZdS )ArchaicUpperLowerPlugin_buf_character_count_since_last_sep_successive_upper_lower_count#_successive_upper_lower_count_finalrz   _last_alpha_seen_upper_last_alpha_seen_lower_current_ascii_onlyrD   rp   c                 C  s4   d| _ d| _d| _d| _d| _d| _d| _d| _d S )NFr   Tr   rq   rZ   rZ   r[   r\     s    z ArchaicUpperLowerPlugin.__init__rA   r$   rj   c                 C  s   |j o
|j}| }|rp| jdkrp| jdkrF|jsF| jsF|  j| j7  _d| _d| _d| _|  jd7  _d| _dS | jr|j	sd| _| jdkr|j
r| js|jr| jr| jr|  jd7  _d| _qd| _nd| _|  jd7  _|  jd7  _|j
| _|j| _dS )r}   r   @   Fr   TNr~   )r'   r-   r   r+   r   r   r   r   rz   r,   r(   r   r)   r   )rX   r%   rk   Zis_concernedZ	chunk_seprZ   rZ   r[   ro     sF    
z!ArchaicUpperLowerPlugin.feed_infoc                 C  s4   d| _ d| _d| _d| _d| _d| _d| _d| _d S )Nr   FT)rz   r   r   r   r   r   r   r   rq   rZ   rZ   r[   rr   ;  s    zArchaicUpperLowerPlugin.resetrs   c                 C  s   | j dkrdS | j| j  S )Nr   r   )rz   r   rq   rZ   rZ   r[   rt   E  s    
zArchaicUpperLowerPlugin.ratioNr   rZ   rZ   rZ   r[   r     s   +
r   c                   @  sP   e Zd ZdZddddZddddZdd	dd
ddZeddddZdS )ArabicIsolatedFormPluginrz   _isolated_form_countrD   rp   c                 C  s   d| _ d| _d S r|   r   rq   rZ   rZ   r[   r\   P  s    z!ArabicIsolatedFormPlugin.__init__c                 C  s   d| _ d| _d S r|   r   rq   rZ   rZ   r[   rr   T  s    zArabicIsolatedFormPlugin.resetrA   r$   rj   c                 C  s*   |  j d7  _ |jt@ r&|  jd7  _dS r   )rz   r.   r   r   rn   rZ   rZ   r[   ro   X  s    
z"ArabicIsolatedFormPlugin.feed_infors   c                 C  s   | j dk rdS | j| j  }|S )Nr   r   r   )rX   Zisolated_form_usagerZ   rZ   r[   rt   _  s    
zArabicIsolatedFormPlugin.ratioN)	r]   r^   r_   ra   r\   rr   ro   ru   rt   rZ   rZ   rZ   r[   r   M  s   r   rC   rB   )r   r   rF   c                 C  sz   | dks|dkrdS t |  }t | }||kr0dS |tks@|tkrDdS t||tkrVdS | dkrf|tkS |dkrv|tkS dS )z_
    Determine if two Unicode ranges seen next to each other can be considered suspicious.
    NTFzBasic Latin)r   r   r	   r   r   )r   r   Z	familly_aZ	familly_brZ   rZ   r[   r   i  s$    r   皙?Frs   )decoded_sequencemaximum_thresholddebugrF   c           $   
   C  s  t | }|dk rd}n|dk r$d}nd}|  }t}t}t }t }	t }
t }t }t	 }t
 }t }t }t }|j}|	j}|
j}|j}|j}|j}|j}|j}|j}|j}td||D ]:}| |||  D ]}t|}|dk r|| }n||}||| ||| |r$|jr||| q||| |jrJ||| ||| |jr||| |jrl||| |jr||| ||| n|jr||| |jr||| q|j|	j |
j |j |j |j |j |j |j |j } | |kr qbq|d }!|d|! |s|d|! |d|! |j|	j |
j |j |j |j |j |j |j |j } |rtd	}"|"td
| d|  d|  |dkr|"td| dd   |"td| dd   ||	|
|||||||f
D ] }#|"t|#j d|#j  qt| dS )zw
    Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier.
    i  rN   i   r   rG   r   r   
Zcharset_normalizerzIMess-detector extended-analysis start. intermediary_mean_mess_ratio_calc=z mean_mess_ratio=z maximum_threshold=   zStarting with: NzEnding with: iz: r   )lenisasciirh   rd   rv   r   r   r   r   r   r   r   r   r   ro   r;   rQ   r&   r'   r0   r1   r2   r4   rt   r   logr   	__class__round)$r   r   r   Zseq_lenstepZis_pure_asciiZ
ascii_infoZ	char_infoZd_spZd_taZd_upZd_sdaZd_srZd_swZd_cuZd_skZd_auZd_aiZ	d_sp_feedZ	d_ta_feedZ	d_up_feedZ
d_sda_feedZ	d_sr_feedZ	d_sw_feedZ	d_cu_feedZ	d_sk_feedZ	d_au_feedZ	d_ai_feedZblock_startr%   	codepointrk   Zmean_mess_ratioZnl_infologgerdtrZ   rZ   r[   
mess_ratio  s    	










	


	
r   )r   F)7
__future__r   	functoolsr   loggingr   Zconstantr   r   r   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   utilsr   r   r   r   r   r    r!   r#   rb   r$   rd   r;   rh   ri   rv   r   r   r   r   r   r   r   r   r   r   r   rZ   rZ   rZ   r[   <module>   s<    \$ M,%,9 4".W$   