U
e5d?R
@ s d Z dZddlZddlZddlmZmZ ddlT ddlm Z dd e
e D d
d d g Z [ ze W n e
k
rz eZ Y nX d
d Zdd Zdd Zdd ZdZdZeede ee ZdZdZdZdZed d ZeeeeeZd Z ed d ee Z d!e Z ee e Z ed"e d# Z ee e eZ!d$Z"d%Z#d&Z$d'Z%d(Z&ee&d) e&d* Z'ee&d+ e&d, Z(ed-d.d/d0d1d2d3d4d5 Z)d6Z*ed7d8d9Z+ee)e*e+Z,ee!e,e(eZ-ee- Z.ee&d: ed;d e&d< ed=d Z/edee'Z0eee0e!e,e/e Z1e2ej3e.e1e$e%f\Z4Z5Z6Z7ed>d?d@dAed>d?dBdCB dDdEdFdGdHdIhB Z8e3e"e3e#e6e7dJdKdL e8D dMdL e8D dNdL e8D Z9d)d*hdOdP e8D B dQdP e8D B Z:d;d=hdRdP e8D B dSdP e8D B Z;dTZG dWdX dXe=Z?dYdZ Z@e@fd[d
ZAd\d] ZBG d^d_ d_ZCe3d`ejDZEe3daejDZFdbdc ZGddde ZHdfd ZIdgd ZJeKdhkrddlLZLeMeLjNdkreAeOeLjNd jP n eAeLjQjP dS )izKa-Ping Yee z@GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro N)BOM_UTF8lookup)* )tokenc C s g | ]}|d dkr|qS )r _ ).0xr r ./usr/lib64/python3.8/lib2to3/pgen2/tokenize.py
% s r tokenizegenerate_tokens
untokenizec G s dd | d S )N(|))joinchoicesr r r group0 r c G s t | d S )Nr r r r r r any1 r r c G s t | d S )N?r r r r r maybe2 r r c s t fdd D S )Nc 3 s4 | ],} d D ] }| | kr|| V qqdS )) N)casefold)r r
ylr r 4 s
z _combinations..)setr r r r
_combinations3 s r# z[ \f\t]*z #[^\r\n]*z\\\r?\nz\w+z0[bB]_?[01]+(?:_[01]+)*z(0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?z 0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?z[1-9]\d*(?:_\d+)*[lL]?z0[lL]?z[eE][-+]?\d+(?:_\d+)*z \d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?z\.\d+(?:_\d+)*z \d+(?:_\d+)*z\d+(?:_\d+)*[jJ]z[jJ]z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'(?:[uUrRbBfF]|[rR][fFbB]|[fFbBuU][rR])?'''"""z '[^\n'\\]*(?:\\.[^\n'\\]*)*'z "[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z<>z!=z//=?z->z[+\-*/%&@|^=<>]=?~z[][(){}]z\r?\nz:=z[:;.,`@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"rRfFbBuUZurZuRZUrZUR)r' r( r$ r% c C s i | ]}| d t qS r$ ) single3progr prefixr r r
y s r5 c C s i | ]}| d t qS r% ) double3progr3 r r r r5 z s c C s i | ]
}|d qS Nr r3 r r r r5 { s c C s h | ]}| d qS r1 r r3 r r r s r9 c C s h | ]}| d qS r6 r r3 r r r r9 s c C s h | ]}| d qS )r' r r3 r r r r9 s c C s h | ]}| d qS )r( r r3 r r r r9 s c @ s e Zd ZdS )
TokenErrorN__name__
__module__ __qualname__r r r r r; s r; c @ s e Zd ZdS )StopTokenizingNr< r r r r r@ s r@ c C s4 |\}}|\}}t d||||t| t|f d S )Nz%d,%d-%d,%d: %s %s)printtok_namerepr) typer Zxxx_todo_changemeZxxx_todo_changeme1lineZsrowZscolZerowZecolr r r
printtoken s
rF c C s( zt | | W n tk
r" Y nX d S r8 )
tokenize_loopr@ )readline
tokeneaterr r r r
s
c C s t | D ] }|| qd S r8 )r )rH rI Z
token_infor r r rG s rG c @ s, e Zd Zdd Zdd Zdd Zdd Zd S )
Untokenizerc C s g | _ d| _d| _d S )Nr r )tokensprev_rowprev_col)selfr r r __init__ s zUntokenizer.__init__c C s* |\}}|| j }|r&| jd| d S )N )rM rK append)rN startrowcol
col_offsetr r r add_whitespace s
zUntokenizer.add_whitespacec C s |D ]p}t |dkr$| || qv|\}}}}}| | | j| |\| _| _|ttfkr| jd7 _d| _qd | jS )N r r r )
lencompatrV rK rQ rL rM NEWLINENLr )rN iterablettok_typer rR endrE r r r r s
zUntokenizer.untokenizec C s d}g }| j j}|\}}|ttfkr,|d7 }|ttfkr|t kr|
q@n*|ttfkrd}n|r|r||d d}|| q@d S )NFrP TrW ) rK rQ NAMENUMBERrZ r[ ASYNCAWAITINDENTDEDENTpop) rN r r\ startlineindents toks_appendtoknumtokvaltokr r r rY s0
zUntokenizer.compatN)r= r> r? rO rV r rY r r r r rJ s rJ z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$)c C sH | d d dd}|dks*|dr.dS |dks@|drDdS | S ) N r -utf-8zutf-8-)zlatin-1
iso-8859-1z iso-latin-1)zlatin-1-z iso-8859-1-z iso-latin-1-rq )lowerreplace
startswith)orig_encencr r r _get_normal_name s rw c s d d }d}fdd} fdd}| }| trHd |dd }d }|sT|g fS ||}|rj||gfS t|s~||gfS | }|s||gfS ||}|r|||gfS |||gfS )
NFrp c s( z W S t k
r" t Y S X d S r8 )
StopIterationbytesr )rH r r read_or_stop s z%detect_encoding..read_or_stopc s z| d}W n tk
r$ Y d S X t|}|s8d S t|d}z t|}W n tk
rr td| Y nX r|j dkrtd|d7 }|S )Nasciir zunknown encoding: rp zencoding problem: utf-8z-sig)
decodeUnicodeDecodeError cookie_rematchrw r r LookupError SyntaxErrorname)rE line_stringr encodingcodec) bom_foundr r find_cookie s"
z$detect_encoding..find_cookieT z utf-8-sig)rt r blank_rer )rH r defaultrz r firstsecondr )r rH r detect_encoding s0
r c C s t }|| S r8 )rJ r )r\ utr r r r : s c c st d } }}d\}}d }dg}d }d} d}
d} z
| } W n t k
rP d} Y nX |d }dt| }
}|r2| s|td||| }|r|d }
}t|| d | |||f|| fV d\}}d }nd|r | dd dkr | d d d
kr t|| ||t| f|fV d}d }q.n|| }|| }q.nB|dkr\|s\| sLq d}|
|k r| |
d krr|d }n8| |
d kr|t d t }n| |
d
krd}nq|
d }
qP|
|krĐq |r|V d }| |
dkr| |
dkrT| |
d d}|
t| }t |||
f||
t| f| fV t
| |d ||f|t| f| fV q.t
t f| |
dk | |
d ||
f|t| f| fV q.||d kr| | t | d |
|df||
f| fV ||d k r4||krt
dd||
| f|d d }| r|
|d krd} d} d}
td||
f||
f| fV q| rt| rt|
|d krtd} d} d}
n| sptd|dfd}|
|k r.t| |
}|r|d\}}||f||f| }}}
| || | | }}|tjks|dkr|dkrt|||| fV q|dkrJt} |dkr t
} n
| r&d} |r6|V d }| |||| fV q|dkrx|rd|V d }t |||| fV q|tkrt| }|| |
}|r|d}
| ||
}|r|V d }t||||
f| fV n||f}| |d }| }q.q|tks$|d d tks$|d d tkr|d dkrx||f}t| p\t|d p\t|d }| |d d }}| }q.n |r|V d }t|||| fV q| rr|dkr| r|dkrtnt|||| fV qtt|||| f} |dkr|s| }qt|d krZ|rZ|d tkrZ|d dkrZd} |d }
t|d |d |d |d fV d }|rj|V d }| V nz|d kr|r|V d }t
||||
f| fV d}nF|d kr|d }n|d kr|d }|r|V d }t |||| fV n(t| |
||
f||
d f| fV |
d }
qtq.|r,|V d }|dd D ] } td|df|dfdfV q8t d|df|dfdfV d S )!Nr )r r Fr r zEOF in multi-line stringz\
z\
rP z#
#z
r` z3unindent does not match any outer indentation levelz
zEOF in multi-line statement.TrW r
)asyncawaitr def \z([{z)]}) rx rX r; r r_ STRING
ERRORTOKENtabsizerstripCOMMENTr[ rQ re IndentationErrorrf
pseudoprogspanstringZdigitsrb rZ
triple_quotedendprogs
single_quoted isidentifierrc rd ra OP ENDMARKER) rH lnumparenlev continuedcontstrneedcontcontlineri ZstashedZ async_defZasync_def_indentZ async_def_nlrE posmaxstrstartendprogendmatchr_ column
comment_tokenZnl_pos pseudomatchrR sposeposr initialnewlinerm indentr r r r O s
*
__main__)R
__author__ __credits__r recodecsr r Zlib2to3.pgen2.tokenr r dir__all__ry NameErrorstrr r r r#
WhitespaceCommentIgnoreName Binnumber Hexnumber Octnumber Decnumber IntnumberExponent
PointfloatExpfloat Floatnumber
ImagnumberNumberSingleDoubleSingle3Double3Z
_litprefixTripleStringZOperatorZBracketSpecialFunny
PlainTokenTokenContStr PseudoExtras PseudoTokenmapcompileZ tokenprogr r2 r7 Z _strprefixesr r r r Exceptionr; r@ rF r
rG rJ ASCIIr~ r rw r r r r= sysrX argvopenrH stdinr r r r s
8 I b