o
    ]’iJš  ã                   @  s4  d Z ddlmZ ddlmZmZ ddlZddlZddlm	Z	m
Z
 G dd„ deƒZG dd	„ d	eƒZG d
d„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ dejƒZeG dd„ dƒƒZG dd„ dƒZeG dd„ dƒƒZd d d!œd.d)d*„Zd/d,d-„ZdS )0zTpytokens - A Fast, spec compliant Python 3.12+ tokenizer that runs on older Pythons.é    )Úannotations)Ú	dataclassÚfieldN)ÚIteratorÚNewTypec                   @  ó   e Zd ZdS )ÚTokenizeErrorN©Ú__name__Ú
__module__Ú__qualname__© r   r   úV/var/www/html/stock_analysis/be/venv/lib/python3.10/site-packages/pytokens/__init__.pyr      ó    r   c                   @  r   )ÚIndentationErrorNr	   r   r   r   r   r      r   r   c                   @  r   )ÚInconsistentUseOfTabsAndSpacesNr	   r   r   r   r   r      r   r   c                   @  r   )Ú DedentDoesNotMatchAnyOuterIndentNr	   r   r   r   r   r      r   r   c                   @  r   )ÚUnterminatedStringNr	   r   r   r   r   r      r   r   c                   @  r   )ÚUnexpectedEOFNr	   r   r   r   r   r      r   r   c                   @  r   )Ú!UnexpectedCharacterAfterBackslashNr	   r   r   r   r   r      r   r   c                   @  r   )ÚNotAnIndentNr	   r   r   r   r   r       r   r   c                   @  r   )Ú	UnderflowNr	   r   r   r   r   r   #   r   r   c                   @  sš   e Zd ZdZdZdZdZdZdZdZ	dZ
d	Zd
ZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZd'dd „Zd'd!d"„Z d(d$d%„Z!d&S ))Ú	TokenTypeé   é   é   é   é   é   é   é   é	   é
   é   é   é   é   é   é   é   é   é   é   é   é   é   é   é   é   é   é   ÚreturnÚstrc                 C  s   d| j › S )Nz
TokenType.)Úname©Úselfr   r   r   Ú__repr__I   ó   zTokenType.__repr__c                 C  s$   | j dkrdS |  ¡ rdS | j  ¡ S )NÚ
identifierÚNAMEÚOP)r7   Úis_operatorÚupperr8   r   r   r   Úto_python_tokenL   s
   

zTokenType.to_python_tokenÚboolc                 C  s   t j|   k ot jk S   S ©N)r   Ú	_op_startÚ_op_endr8   r   r   r   r?   U   s   zTokenType.is_operatorN©r5   r6   ©r5   rB   )"r
   r   r   Ú
whitespaceÚindentÚdedentÚnewlineÚnlÚcommentrD   Ú	semicolonÚlparenÚrparenÚlbracketÚrbracketÚlbraceÚrbraceÚcolonÚoprE   r<   ÚnumberÚstringÚfstring_startÚfstring_middleÚfstring_endÚtstring_startÚtstring_middleÚtstring_endÚ	endmarkerÚ
errortokenr:   rA   r?   r   r   r   r   r   &   s@    

	r   c                   @  sP   e Zd ZU ded< ded< ded< ded< ded< ded< ded	< ddd„ZdS )ÚTokenr   ÚtypeÚintÚstart_indexÚ	end_indexÚ
start_lineÚ	start_colÚend_lineÚend_colÚsourcer6   r5   c                 C  s   | j tjks| j tjkr| jt|ƒkr| jt|ƒd krdS | j tjkr8| jt|ƒd kr8| jt|ƒd kr8dS | j tjkr@dS || j| j… S )Nr   Ú )	rb   r   rK   rL   rd   Úlenre   rJ   r_   )r9   rj   r   r   r   Úto_byte_slicee   s&   
ÿ
ÿþý
ÿþýzToken.to_byte_sliceN)rj   r6   r5   r6   )r
   r   r   Ú__annotations__rm   r   r   r   r   ra   Y   s   
 ra   c                   @  s–   e Zd ZedeƒZedƒZedƒZedƒZedƒZ	edƒZ
edƒZdd
d„Zddd„Zddd„Zddd„Zddd„Zddd„Zddd„Zddd„ZdS )ÚFStringStateÚStater   r   r   r   r   r   r5   ÚNonec                 C  s   t j| _g | _d S rC   )ro   Únot_fstringÚstateÚstackr8   r   r   r   Ú__init__‡   s   
zFStringState.__init__c                 C  s   | j  | j¡ tj| _d S rC   )rt   Úappendrs   ro   Úat_fstring_middler8   r   r   r   Úenter_fstring‹   s   zFStringState.enter_fstringc                 C  s    | j tjksJ ‚| j ¡ | _ d S rC   )rs   ro   Úat_fstring_endrt   Úpopr8   r   r   r   Úleave_fstring   s   zFStringState.leave_fstringc                 C  s&   | j tjkr| j | j ¡ tj| _ d S rC   )rs   ro   Úin_fstring_expr_modifierrt   rv   Úat_fstring_lbracer8   r   r   r   Ú!consume_fstring_middle_for_lbrace“   s   z.FStringState.consume_fstring_middle_for_lbracec                 C  ó   t j| _d S rC   )ro   ry   rs   r8   r   r   r   Úconsume_fstring_middle_for_end™   r;   z+FStringState.consume_fstring_middle_for_endc                 C  r   rC   )ro   Úin_fstring_exprrs   r8   r   r   r   Úconsume_lbraceœ   r;   zFStringState.consume_lbracec                 C  sV   | j tjks| j tjksJ ‚t| jƒdkr%| jd tjkr%| j ¡ | _ d S tj| _ d S )Nr   éÿÿÿÿ)rs   ro   r   r|   rl   rt   rz   rw   r8   r   r   r   Úconsume_rbraceŸ   s   
ÿ
þÿþzFStringState.consume_rbracec                 C  s   | j tjksJ ‚tj| _ d S rC   )rs   ro   r   r|   r8   r   r   r   Úconsume_colon­   s   zFStringState.consume_colonN©r5   rq   )r
   r   r   r   rc   rp   rr   rw   r}   r   r|   ry   ru   rx   r{   r~   r€   r‚   r„   r…   r   r   r   r   ro   }   s     







ro   c                   @  s.  e Zd ZU ded< ded< dZded< dZded< d	Zded
< d	Zded< dZded< dZ	ded< dZ
ded< dZded< eedZded< dZded< eedZded< dZded< eedZded< eedZded< dZded< dZded < d!Zded"< d!Zded#< d!Zded$< dgd&d'„Zdhd(d)„Zdhd*d+„Zdid-d.„Zdjd0d1„Zdid2d3„Zdid4d5„Z d!d6œdkd9d:„Z!dld>d?„Z"dmdBdC„Z#didDdE„Z$dndFdG„Z%dndHdI„Z&dndJdK„Z'dndLdM„Z(dndNdO„Z)dndPdQ„Z*dodRdS„Z+dpdUdV„Z,dndWdX„Z-dndYdZ„Z.dnd[d\„Z/dgd]d^„Z0dgd_d`„Z1dndadb„Z2dqdcdd„Z3dndedf„Z4dS )rÚTokenIteratorr6   rj   rB   Úissue_128233_handlingr   rc   Úcurrent_indexÚ
prev_indexr   Úline_numberÚprev_line_numberÚbyte_offsetÚprev_byte_offsetTÚall_whitespace_on_this_lineÚbracket_level)Údefault_factoryz	list[int]Úbracket_level_stackNzToken | NoneÚ
prev_tokenz	list[str]Úindent_stackÚdedent_counterro   Úfstring_statezlist[tuple[str, str]]Úfstring_prefix_quote_stackz
str | NoneÚfstring_prefixÚfstring_quoteFÚweird_op_caseÚweird_op_case_nlÚweird_whitespace_caser5   c                 C  s   | j t| jƒk S rC   ©r‰   rl   rj   r8   r   r   r   Úis_in_boundsÓ   s   zTokenIterator.is_in_boundsc                 C  s   |   ¡ sJ ‚| j| j S rC   )rž   rj   r‰   r8   r   r   r   ÚpeekÖ   s   zTokenIterator.peekc                 C  s(   | j d t| jƒk sJ ‚| j| j d  S ©Nr   r   r8   r   r   r   Ú	peek_nextÚ   s   zTokenIterator.peek_nextrq   c                 C  s    |  j d7  _ |  jd7  _d S r    ©r‰   r   r8   r   r   r   ÚadvanceÞ   ó   zTokenIterator.advanceÚcountc                 C  s    |  j |7  _ |  j|7  _d S rC   r¢   )r9   r¥   r   r   r   Ú
advance_byâ   r¤   zTokenIterator.advance_byc                 C  s   |  j d7  _ d| _d| _d S )Nr   r   T)r‹   r   r   r8   r   r   r   Ú	next_lineæ   s   
zTokenIterator.next_linec                 C  s6   | j | j dkr|  jd7  _|  ¡  d S |  ¡  d S )NÚ
r   )rj   r‰   r§   r£   r8   r   r   r   Úadvance_check_newlineë   s   z#TokenIterator.advance_check_newline©Úignore_caseÚoptionsr«   c                G  sd   |D ]-}| j t|ƒ t| jƒkrq| j| j | j t|ƒ … }|r(| ¡ }| ¡ }||kr/ dS qdS )NTF)r‰   rl   rj   Úlower)r9   r«   r¬   ÚoptionÚsnippetr   r   r   Úmatchò   s   ÿzTokenIterator.matchÚtok_typer   ra   c              	   C  s   | j d ur$d| j v r$|tjkrtj}n|tjkrtj}n|tjkr$tj}| jr6| 	¡ s6|tj
tjfvr6tjn|}| jrN| j| j| j… }|dkrKd| _d| _t|| j| j| j| j| j| jd}|tjksi|tjkrn|  ¡  n|tjksx|tjkrynd| _|| _| j| _| j| _| j| _d| _|S )NÚtz F)rb   rd   re   rf   rg   rh   ri   )r˜   r   rY   r\   rZ   r]   r[   r^   rš   r?   rW   rX   rV   rj   rŠ   r‰   r   ra   rŒ   rŽ   r‹   r   rK   rL   r§   rH   rM   r“   )r9   r±   Ú
token_typeÚ	token_strÚtokenr   r   r   Ú
make_token   sP   


ÿþýû
ù	
zTokenIterator.make_tokenÚprefixÚquotec                 C  s<   | j d ur| jd usJ ‚| j | j | jf¡ || _ || _d S rC   )r˜   r™   r—   rv   )r9   r·   r¸   r   r   r   Úpush_fstring_prefix_quote7  s   

ÿ
z'TokenIterator.push_fstring_prefix_quotec                 C  sD   | j d u r| jd u sJ ‚t‚t| jƒdkrdn| j ¡ \| _ | _d S )Nr   )NN)r˜   r™   r   rl   r—   rz   r8   r   r   r   Úpop_fstring_quoteA  s   
ÿýzTokenIterator.pop_fstring_quotec                 C  sl   |   ¡ r| j| j dkr|  ¡  |  ¡  | js&| jdks&| jjtj	ks&| j
r)tjntj}|  |¡}d| _|S )Núr   F)rž   rj   r‰   r£   r›   r   r–   rs   ro   r   r   r   rL   rK   r¶   )r9   r³   rµ   r   r   r   rK   L  s    þýüûø

zTokenIterator.newlinec                 C  s>   | j dkrt‚t| jƒdkr| j ¡ }|  tj¡S |  tj¡S )Nr   )	r   r   rl   r”   rz   r¶   r   rJ   r_   )r9   Ú_r   r   r   r_   ^  s   

zTokenIterator.endmarkerc                 C  sH  d}| j | j  ¡ rd}|  ¡  |  ¡ r<| j | j  ¡ s$| j | j dkr<|  ¡  |  ¡ r<| j | j  ¡ s$| j | j dks$|  ¡ rL| j | j dkrL|  ¡  |  ¡ rŒ| j | j  ¡ sj| j | j dkrŒ| j | jd   ¡ rŒ|  ¡  |  ¡ rŒ| j | j  ¡ sj| j | j dkrŒ| j | jd   ¡ sj| jd t| j ƒk rì|s¢| j | jd   ¡ rì| j | j dks²| j | j dkrì| j | jd   ¡ sä| jd t| j ƒk rì| j | jd  d	ksÚ| j | jd  d
krì| j | jd   ¡ rì|  ¡  |  ¡  |  ¡ r:| j | j  ¡ s|s| j | jd   ¡ r:| j | j dkr:|  ¡  |  ¡ r:| j | j  ¡ s|s1| j | jd   ¡ r:| j | j dks|  ¡ rc|sM| j | jd   ¡ rc| j | j dks_| j | j dkrc|  ¡  | j| j dkrž| j | jd  dkrž| jd t| j ƒkr˜| j | j| jd … dkr˜|  ¡  |  ¡  |  tj	¡S |  tj
¡S )NFTr¼   Ú.r   ÚeÚEr   ú+ú-ÚjÚJz..)rj   r‰   Úisdigitr£   rž   rl   rŠ   r¶   r   rV   rW   )r9   Údigit_before_decimalr   r   r   Údecimalh  s¼   ÿþüÿþÿýüùÿýü	ÿÿýüù	÷õôò
ÿýýü
ùÿýýü

ÿÿýüÿþÿþzTokenIterator.decimalc                 C  s\  |   ¡  |   ¡  |  ¡ rD| j| j dks$| j| j dks$| j| j dkrD|   ¡  |  ¡ rD| j| j dks$| j| j dks$| j| j dks$|  ¡ rl| j| j dksX| j| j dkrl|   ¡  |  ¡ rl| j| j dkrl|   ¡  |  ¡ r¨| j| j dksˆ| j| j dksˆ| j| j dkr¨|   ¡  |  ¡ r¨| j| j dksˆ| j| j dksˆ| j| j dksˆ|  tj¡S )NÚ0Ú1r¼   r¾   r¿   rÁ   ©r£   rž   rj   r‰   r¶   r   rW   r8   r   r   r   Úbinary»  óR   ÿþýûÿþýÿþÿþýûÿþýzTokenIterator.binaryc                 C  s\  |   ¡  |   ¡  |  ¡ rD| j| j dkr| j| j dks$| j| j dkrD|   ¡  |  ¡ rD| j| j dkr<| j| j dks$| j| j dks$|  ¡ rl| j| j dksX| j| j dkrl|   ¡  |  ¡ rl| j| j dkrl|   ¡  |  ¡ r¨| j| j dkr€| j| j dksˆ| j| j dkr¨|   ¡  |  ¡ r¨| j| j dkr | j| j dksˆ| j| j dksˆ|  tj¡S )NrÇ   Ú7r¼   r¾   r¿   rÁ   rÉ   r8   r   r   r   ÚoctalÕ  rË   zTokenIterator.octalc                 C  s$  |   ¡  |   ¡  |  ¡ r6| j| j tjv s| j| j dkr6|   ¡  |  ¡ r6| j| j tjv s| j| j dks|  ¡ r^| j| j dksJ| j| j dkr^|   ¡  |  ¡ r^| j| j dkr^|   ¡  |  ¡ rŒ| j| j tjv ss| j| j dkrŒ|   ¡  |  ¡ rŒ| j| j tjv ss| j| j dkss|  tj¡S )Nr¼   r¾   r¿   rÁ   )	r£   rž   rj   r‰   rX   Ú	hexdigitsr¶   r   rW   r8   r   r   r   Úhexadecimalï  sB   ÿþüÿþÿþÿþüÿþzTokenIterator.hexadecimalc                 C  sD   t dƒD ]}| j| j|  }|dks|dkr| j|   S qtdƒ‚)Nr   ú"ú'zQuote not found somehow)Úrangerj   r‰   ÚAssertionError)r9   ÚoffsetÚcharr   r   r   Úfind_opening_quote  s   ÿz TokenIterator.find_opening_quoteútuple[str, str]c                 C  s„   |   ¡ }| j| j|… }| j| }|d t| jƒkr5| j|d  |kr5| j|d  |kr5| j||d … n| j||d … }||fS )Nr   r   r   )rÖ   rj   r‰   rl   )r9   Úquote_indexr·   Ú
quote_charr¸   r   r   r   Ústring_prefix_and_quotes  s   
þýüù	z&TokenIterator.string_prefix_and_quotesc                 C  s‚  | j jtjtjfv r;|  ¡ \}}|  ||¡ tt|ƒƒD ]}|  	¡  qtt|ƒƒD ]}|  	¡  q)| j  
¡  |  tj¡S | j jtjkr| jd usJJ ‚t| jƒdk}| j}|  ¡ rÿ| j| j }|dkrf|rft‚|dkr¯|  	¡  | jd usuJ ‚d| j ¡ vrš| jd t| jƒk rš|  ¡ dkrš|  ¡ dkrš|  	¡  |  	¡  |  ¡ r®|  ¡ dks®|  ¡ dks®|  ¡  qT|dkrÖ|  ¡ dkrÂ|  	¡  |  	¡  qT| j  ¡  | j|krÐ|  ¡ S |  tj¡S | jd usÝJ ‚|  | j¡r÷| j  ¡  | j|krñ|  ¡ S |  tj¡S |  ¡  |  ¡ sXt‚| j jtjkr"|  	¡  | j  !| j"¡ d| _"| j  #¡  |  tj$¡S | j jtj%krR| jd us2J ‚tt| jƒƒD ]}|  	¡  q9|  tj&¡}|  '¡  | j  (¡  |S | j jtj)kr½| j}|  ¡ r»| j| j }| jd uspJ ‚|dksz|dkr¢t| jƒdkr¢|dkr| j  ¡  ntj| j _| j|krœ|  ¡ S |  tj¡S |dkr²tj| j _|  tj¡S |  ¡  |  ¡ sbt‚t*d	ƒ‚)
Nr   r¨   ú\ÚrÚNÚ{Ú}r   zUnhandled f-string state)+r–   rs   ro   rr   r   rÚ   r¹   rÒ   rl   r£   rx   r¶   r   rY   rw   r™   r‰   rž   rj   r   r˜   r­   rŸ   r¡   r©   r~   ÚfstringrZ   r°   r€   r   r}   r’   rv   r   r‚   rS   ry   r[   rº   r{   r|   rÓ   )r9   r·   r¸   r¼   Úis_single_quoterd   rÕ   rµ   r   r   r   rà   !  s¶   þ


ÿþ
ý
ü
ÿ
ÿ



Ì6


$




ízTokenIterator.fstringc                 C  s  |   ¡ \}}|r| jr|  ¡  | jtjdS |D ]}|dv r$|  ¡   S qtt|ƒƒD ]}|  ¡  q+tt|ƒƒD ]}|  ¡  q8t|ƒdk}|  	¡ r„| j
| j }|dkrW|rWt‚|dkrd|  ¡  |  ¡  qE|  |¡r|tt|ƒƒD ]}|  ¡  qo|  tj¡S |  ¡  |  	¡ sIt‚)N)r±   )ÚfÚFr²   ÚTr   r¨   rÛ   )rÚ   rš   r£   r¶   r   rV   rà   rÒ   rl   rž   rj   r‰   r   r©   r°   rX   r   )r9   r·   r¸   rÕ   r¼   rá   r   r   r   rX   —  s8   
ÿ



îzTokenIterator.stringc           
      C  sÒ  | j }d}d}|  ¡ r+| j| j  }|  ¡ r&|  ¡  d}|dks#|dkr%d}nn|  ¡ s|  ¡ s<| j |kr6t‚|  tj¡S |rI|sId| _	|  tj¡S |  
¡ }|dksY|dksY|  ¡ r_|  tj¡S | j|| j … }t| jƒdkrpdn| jd	 }t|ƒt|ƒkr‘t|ƒdkr…t‚||kr‹t‚|  tj¡S t|ƒt|ƒkr±t|ƒdkr¥||vr¥t‚| j |¡ |  tj¡S t| jƒdkrã| jd	 }t|ƒt|ƒk rÇt‚t|ƒt|ƒkrÐn| j ¡ }	|  jd
7  _t| jƒdks¸|  tj¡S )NFTú ú	ú#rÛ   r   rk   rƒ   r   )r‰   rž   rj   Úis_whitespacer£   r   r¶   r   rH   rœ   rŸ   Ú
is_newlinerl   r”   r   rv   rI   r   rz   r•   )
r9   rd   Úsaw_whitespaceÚsaw_tab_or_spacerÕ   Ú	next_charÚ
new_indentÚcurrent_indentÚ
top_indentr¼   r   r   r   rI   ¾  s\   €ø



÷zTokenIterator.indentc                 C  s@   |   ¡ rdS | j| j }|dkp|dkp|dkp|dkp|dkS )NFrå   r»   ræ   úú)ré   rj   r‰   )r9   rÕ   r   r   r   rè   ý  s   ÿþýûzTokenIterator.is_whitespacec                 C  sT   | j | j dkr
dS | j | j dkr(| jd t| j ƒk r(| j | jd  dkr(dS dS )Nr¨   Tr»   r   F)rj   r‰   rl   r8   r   r   r   ré   
  s   ÿþýzTokenIterator.is_newlinec                 C  s€   | j r|  ¡  |  tj¡S | j| jd … }t|ƒD ]\}}t|ƒdk r0t	 
|¡s0|dkr0|} nqt|ƒ}|  |¡ |  tj¡S )Né€   r¼   )rš   r£   r¶   r   r<   rj   r‰   Ú	enumerateÚordr6   Úisalnumrl   r¦   )r9   Ú	remainingÚindexrÕ   Úlengthr   r   r   r7     s   €
zTokenIterator.namec                 C  s   | S rC   r   r8   r   r   r   Ú__iter__(  s   zTokenIterator.__iter__c                 C  st  | j d ur| j jtjkrt‚| jt| jƒkr3| j d u r|  ¡ S | j jtjtj	tj
hv r/|  ¡ S |  ¡ S | jt| jƒkr?|  ¡ S | jjtjkrQ| jjtjkrQ|  ¡ S | j| j }| js†|dkr†|  ¡  |  ¡ sj|  ¡ S | j| j }|dkr†d| _| j d ur†| j jtjkr†d| _|dkr½| jr—|  ¡  |  tj¡S |  ¡ r·|  ¡ s·| js«| j| j dkr«n|  ¡  |  ¡ r·|  ¡ rŸ|  tj¡S | jdkrÏ|  jd8  _|  tj
¡S |  ¡ r×|  ¡ S |dkr:|  ¡  |  ¡ sæt‚d}d}|  ¡ r/|  ¡ rú|  ¡  d}n1|s*|  ¡ r*| j| j }|dkr|  ¡  |  ¡  d}d}| jr"|  ¡  n|  ¡  d| _nn|  ¡ sï|s4t‚|  tj¡S | j dksD| j!ro| j"dkro| jjtjkrod| _!z|  #¡ }W n t$yg   d }Y nw |d uro|S |  ¡ r’|  ¡ rŒ|  ¡ rŒ|  ¡  |  ¡ rŒ|  ¡ s~|  tj¡S |d	v r¬|  ¡  |  %¡ d
kr¦|  ¡  |  tj&¡S |dkrâ|  ¡  |  %¡ dkrÆ|  ¡  |  tj&¡S |  %¡ dkrÑ|  ¡  |  %¡ d
krÜ|  ¡  |  tj&¡S |dkr|  ¡  |  %¡ dkrö|  ¡  |  %¡ d
kr|  ¡  |  tj&¡S |dkr,|  ¡  |  %¡ dkr|  ¡  |  %¡ d
kr&|  ¡  |  tj&¡S |dkrQ|  ¡  |  %¡ dkr@|  ¡  |  %¡ d
krK|  ¡  |  tj&¡S |dkr||  ¡  |  %¡ dkrk|  ¡  |  tj&¡S |  %¡ d
krv|  ¡  |  tj&¡S |dv r‹|  ¡  |  tj&¡S |dkrš|  ¡  |  tj&¡S |dkr°|  ¡  |  j"d7  _"|  tj'¡S |dkrÏ|  ¡  |  j"d8  _"| j"dk rÉd| _"|  tj(¡S |dkrå|  ¡  |  j"d7  _"|  tj)¡S |dkr|  ¡  |  j"d8  _"| j"dk rþd| _"|  tj*¡S |dkr|  ¡  |  j"d7  _"|  tj+¡S |dkrS|  ¡  | j"dkr=| jjtjkr=| j ,¡  | j- .¡ | _"n|  j"d8  _"| j"dk rMd| _"|  tj/¡S |dkr†|  ¡  | j"dkru| jjtjkrutj0| j_|  tj&¡S |  %¡ d
kr€|  ¡  |  tj&¡S |dv ræ| jd t| jƒkr¨| j| j| jd … dv r¨|  1¡ S | jd t| jƒkrÅ| j| j| jd … dv rÅ|  2¡ S | jd t| jƒkrâ| j| j| jd … dv râ|  3¡ S |  4¡ S | jd t| jƒkrø|  5dd¡s2| jd t| jƒkr| j5d d!d"d#d$d%d&d'd(d)dd*s2| jd+ t| jƒkr6| j5d,d-d.d/d0d1d2d3d4d5d6d7dd*r6|  6¡ S |  7¡ S )8Nr»   r¨   Trç   r   r   rÛ   F)	rÀ   ú&ú|ú^ú@ú%ú=ú!ú~rÿ   ú<ú>ú/Ú*rÁ   )ú,ú;ú`ú(ú)ú[ú]rÞ   rß   ú:z.0123456789r   )Ú0bÚ0B)Ú0oÚ0O)Ú0xÚ0XrÐ   rÑ   zb"zb'zr"zr'zf"zf'zu"zu'zt'zt"rª   r   zbr"zbr'zrb"zrb'zfr"zfr'zrf"zrf'ztr'ztr"zrt'zrt")8r“   rb   r   r_   ÚStopIterationr‰   rl   rj   rK   rL   rJ   r–   rs   ro   rr   r   rà   rˆ   r£   rž   rš   rM   r›   r¶   ré   r•   r   rè   r   r§   r   rH   r   rœ   r   rI   r   rŸ   rV   rO   rP   rQ   rR   rS   r„   r’   rz   rT   r|   rÊ   rÍ   rÏ   rÆ   r°   rX   r7   )r9   Úcurrent_charÚfound_whitespaceÚseen_newlinerÕ   Úindent_tokenr   r   r   Ú__next__+  sþ  
ýÿþÿþÿþú



éÿÿþýÿ

ÿ













ÿþ

ÿþ

ÿþÿþÿþÿ
ÿýõüíóì%zTokenIterator.__next__rG   rF   r†   )r¥   rc   r5   rq   )r¬   r6   r«   rB   r5   rB   )r±   r   r5   ra   )r·   r6   r¸   r6   r5   rq   )r5   ra   )r5   rc   )r5   r×   )r5   r‡   )5r
   r   r   rn   r‰   rŠ   r‹   rŒ   r   rŽ   r   r   r   Úlistr’   r“   r”   r•   ro   r–   r—   r˜   r™   rš   r›   rœ   rž   rŸ   r¡   r£   r¦   r§   r©   r°   r¶   r¹   rº   rK   r_   rÆ   rÊ   rÍ   rÏ   rÖ   rÚ   rà   rX   rI   rè   ré   r7   rù   r  r   r   r   r   r‡   ²   sb   
 








7






S



	

v
'
?


r‡   T)Úfstring_tokensrˆ   rj   r6   r  rB   rˆ   r5   úIterator[Token]c                C  s    t | |d}|rt|ƒS t|ƒS )N)rˆ   )r‡   ÚiterÚmerge_fstring_tokens)rj   r  rˆ   Útoken_iteratorr   r   r   Útokenizeb  s   r   r  c              
   c  s®    | D ]Q}|j tjtjfvr|V  q|}|}d}d}| D ]$}|j tjtjfv r+|d7 }|j tjtjfv r8|d7 }||kr@|} nqttj|j|j	|j
|j|j|jdV  qdS )zETurn post-Python-3.12 FSTRING-* tokens back to a single STRING token.r   r   )rb   rd   rf   rg   re   rh   ri   N)rb   r   rY   r\   r[   r^   ra   rX   rd   rf   rg   re   rh   ri   )r  rµ   Ústart_tokenÚ	end_tokenÚfstring_startsÚfstring_endsr   r   r   r  o  s8   €þ
ùìr  )rj   r6   r  rB   rˆ   rB   r5   r  )r  r‡   r5   r  )Ú__doc__Ú
__future__r   Údataclassesr   r   ÚenumrX   Útypingr   r   Ú	Exceptionr   r   r   r   r   r   r   rÓ   r   r   ÚIntEnumr   ra   ro   r‡   r   r  r   r   r   r   Ú<module>   s@    3#5       9ü