
    hr                       d Z ddlmZ ddlmZmZ ddlZddlZddlm	Z	m
Z
  G d de      Z G d d	e      Z G d
 de      Z G d de      Z G d de      Z G d de      Z G d de      Z G d de      Z G d de      Z G d dej,                        Ze G d d             Z G d d      Ze G d d             Zd d d!	 	 	 	 	 	 	 d$d"Zd%d#Zy)&zTpytokens - A Fast, spec compliant Python 3.12+ tokenizer that runs on older Pythons.    )annotations)	dataclassfieldN)IteratorNewTypec                      e Zd Zy)TokenizeErrorN__name__
__module____qualname__     D/var/www/html/immo/lib/python3.12/site-packages/pytokens/__init__.pyr	   r	          r   r	   c                      e Zd Zy)IndentationErrorNr
   r   r   r   r   r      r   r   r   c                      e Zd Zy)InconsistentUseOfTabsAndSpacesNr
   r   r   r   r   r      r   r   r   c                      e Zd Zy) DedentDoesNotMatchAnyOuterIndentNr
   r   r   r   r   r      r   r   r   c                      e Zd Zy)UnterminatedStringNr
   r   r   r   r   r      r   r   r   c                      e Zd Zy)UnexpectedEOFNr
   r   r   r   r   r      r   r   r   c                      e Zd Zy)!UnexpectedCharacterAfterBackslashNr
   r   r   r   r   r      r   r   r   c                      e Zd Zy)NotAnIndentNr
   r   r   r   r   r       r   r   r   c                      e Zd Zy)	UnderflowNr
   r   r   r   r!   r!   #   r   r   r!   c                      e Zd ZdZdZdZdZdZdZdZ	dZ
d	Zd
ZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZddZddZddZy) 	TokenType                        	   
                                                c                     d| j                    S )Nz
TokenType.)nameselfs    r   __repr__zTokenType.__repr__E   s    DII;''r   c                x    | j                   dk(  ry| j                         ry| j                   j                         S )N
identifierNAMEOP)r>   is_operatorupperr?   s    r   to_python_tokenzTokenType.to_python_tokenH   s1    99$yy  r   c                Z    t         j                  | cxk  xr t         j                  k  S c S N)r#   	_op_start_op_endr?   s    r   rF   zTokenType.is_operatorQ   s$    ""T=I,=,=====r   NreturnstrrN   bool)r   r   r   
whitespaceindentdedentnewlinenlcommentrK   	semicolonlparenrparenlbracketrbracketlbracerbracecolonoprL   rC   numberstringfstring_startfstring_middlefstring_end	endmarker
errortokenrA   rH   rF   r   r   r   r#   r#   &   s    JFFG	
BGIIFFHHFFE	BGJFFMNKIJ(!>r   r#   c                  \    e Zd ZU ded<   ded<   ded<   ded<   ded<   ded<   ded	<   dd
Zy)Tokenr#   typeintstart_index	end_index
start_line	start_colend_lineend_colc                   | j                   t        j                  k(  s| j                   t        j                  k(  r4| j                  t        |      k(  r| j                  t        |      dz   k(  ry| j                   t        j                  k(  r7| j                  t        |      dz   k(  r| j                  t        |      dz   k(  ry| j                   t        j                  k(  ry|| j                  | j                   S )Nr$    )	rj   r#   rU   rV   rl   lenrm   rT   rf   )r@   sources     r   to_byte_slicezToken.to_byte_slicea   s     YY)+++tyyILL/H  CK/#f+/1 II)))  CK!O3#f+/1 99	+++d&&88r   N)ru   rO   rN   rO   )r   r   r   __annotations__rv   r   r   r   ri   ri   U   s+    
ONONML9r   ri   c                      e Zd Z ede      Z ed      Z ed      Z ed      Z ed      Z	 ed      Z
 ed      ZddZdd	Zdd
ZddZddZddZddZddZy)FStringStateStater$   r%   r&   r'   r(   r)   c                <    t         j                  | _        g | _        y rJ   )ry   not_fstringstatestackr?   s    r   __init__zFStringState.__init__   s    !--
/1
r   c                x    | j                   j                  | j                         t        j                  | _        y rJ   )r~   appendr}   ry   at_fstring_middler?   s    r   enter_fstringzFStringState.enter_fstring   s%    

$**%!33
r   c                    | j                   t        j                  k(  sJ | j                  j	                         | _         y rJ   )r}   ry   at_fstring_endr~   popr?   s    r   leave_fstringzFStringState.leave_fstring   s,    zz\88888ZZ^^%
r   c                    | j                   t        j                  k(  r%| j                  j	                  | j                          t        j
                  | _         y rJ   )r}   ry   in_fstring_expr_modifierr~   r   at_fstring_lbracer?   s    r   !consume_fstring_middle_for_lbracez.FStringState.consume_fstring_middle_for_lbrace   s7    ::>>>JJdjj)!33
r   c                .    t         j                  | _        y rJ   )ry   r   r}   r?   s    r   consume_fstring_middle_for_endz+FStringState.consume_fstring_middle_for_end   s    !00
r   c                .    t         j                  | _        y rJ   )ry   in_fstring_exprr}   r?   s    r   consume_lbracezFStringState.consume_lbrace   s    !11
r   c                V   | j                   t        j                  k(  s| j                   t        j                  k(  sJ t	        | j
                        dkD  r@| j
                  d   t        j                  k(  r | j
                  j                         | _         y t        j                  | _         y )Nr   )r}   ry   r   r   rt   r~   r   r   r?   s    r   consume_rbracezFStringState.consume_rbrace   sx    JJ,666zz\BBB	
C
 

Oa

2,"G"GG)DJ%77DJr   c                l    | j                   t        j                  k(  sJ t        j                  | _         y rJ   )r}   ry   r   r   r?   s    r   consume_colonzFStringState.consume_colon   s'    zz\99999!::
r   NrN   None)r   r   r   r   rk   rz   r|   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   ry   ry   y   so    GS!E(KaaAhO$Qx1XN24&4128;r   ry   c                  B   e Zd ZU ded<   ded<   dZded<   dZded<   d	Zded
<   d	Zded<   dZded<   dZ	ded<   dZ
ded<   dZded<    ee      Zded<   dZded<    ee      Zded<   dZded<    ee      Zded<    ee      Zded<   dZded<   dZded <   d!Zded"<   d!Zded#<   d!Zded$<   dAd%ZdBd&ZdBd'ZdCd(ZdDd)ZdCd*ZdCd+Z d!d,dEd-Z!dFd.Z"dGd/Z#dCd0Z$dHd1Z%dHd2Z&dHd3Z'dHd4Z(dHd5Z)dHd6Z*dId7Z+dJd8Z,dHd9Z-dHd:Z.dHd;Z/dAd<Z0dAd=Z1dHd>Z2dKd?Z3dHd@Z4y)LTokenIteratorrO   ru   rQ   issue_128233_handlingr   rk   current_index
prev_indexr$   line_numberprev_line_numberbyte_offsetprev_byte_offsetTall_whitespace_on_this_linebracket_level)default_factoryz	list[int]bracket_level_stackNzToken | None
prev_tokenz	list[str]indent_stackdedent_counterry   fstring_statezlist[tuple[str, str]]fstring_prefix_quote_stackz
str | Nonefstring_prefixfstring_quoteFweird_op_caseweird_op_case_nlweird_whitespace_casec                F    | j                   t        | j                        k  S rJ   r   rt   ru   r?   s    r   is_in_boundszTokenIterator.is_in_bounds   s    !!C$444r   c                X    | j                         sJ | j                  | j                     S rJ   )r   ru   r   r?   s    r   peekzTokenIterator.peek   s)      """{{4--..r   c                    | j                   dz   t        | j                        k  sJ | j                  | j                   dz      S Nr$   r   r?   s    r   	peek_nextzTokenIterator.peek_next   s>    !!A%DKK(8888{{4--122r   c                X    | xj                   dz  c_         | xj                  dz  c_        y r   r   r   r?   s    r   advancezTokenIterator.advance   s$    aAr   c                X    | xj                   |z  c_         | xj                  |z  c_        y rJ   r   )r@   counts     r   
advance_byzTokenIterator.advance_by   s$    e#E!r   c                J    | xj                   dz  c_         d| _        d| _        y )Nr$   r   T)r   r   r   r?   s    r   	next_linezTokenIterator.next_line   s#    A+/(r   c                    | j                   | j                     dk(  r&| xj                  dz  c_        | j                          y | j                          y )N
r$   )ru   r   r   r   r?   s    r   advance_check_newlinez#TokenIterator.advance_check_newline   s=    ;;t))*d2!#NNLLNr   ignore_casec                   |D ]  }| j                   t        |      z   t        | j                        kD  r2| j                  | j                   | j                   t        |      z    }|r |j                         }|j                         }||k(  s y y)NTF)r   rt   ru   lower)r@   r   optionsoptionsnippets        r   matchzTokenIterator.match   s     		F!!CK/#dkk2BBkk$"4"4t7I7ICPVK7WXG!--/ 		 r   c           	        | j                   rB|j                         s2|t        j                  t        j                  fvrt        j
                  n|}| j                   r6| j                  | j                  | j                   }|dk(  rd| _	        d| _         t        || j                  | j                  | j                  | j                  | j                  | j                        }|t        j                  k(  s|t        j                   k(  r| j#                          n.|t        j$                  k(  s|t        j&                  k(  rnd| _	        || _        | j                  | _        | j                  | _        | j                  | _        d| _         |S )Nz F)rj   rl   rm   rn   ro   rp   rq   )r   rF   r#   ra   rb   r`   ru   r   r   r   ri   r   r   r   r   rU   rV   r   rR   rW   r   )r@   tok_type
token_type	token_strtokens        r   
make_tokenzTokenIterator.make_token   sM    !!((*!1!193C3C DD LL  	  DOOd6H6HIIE!380!&D((,,++%%$$
 y(((H	,DNN---Y=N=N1N/4D,,, $ 0 0 $ 0 0"r   c                    | j                   ?| j                  J | j                  j                  | j                   | j                  f       || _         || _        y rJ   )r   r   r   r   )r@   prefixquotes      r   push_fstring_prefix_quotez'TokenIterator.push_fstring_prefix_quote+  sZ    *%%111++22$$d&8&89 %"r   c                    | j                   | j                  J t        t        | j                        dk(  rdn| j                  j                         \  | _         | _        y )Nr   )NN)r   r   r!   rt   r   r   r?   s    r   pop_fstring_quotezTokenIterator.pop_fstring_quote5  s`    &%%---O 4223q8 00446 	0T/r   c                   | j                         r,| j                  | j                     dk(  r| j                          | j                          | j                  sB| j
                  dkD  s3| j                  j                  t        j                  k(  s| j                  rt        j                  nt        j                  }| j                  |      }d| _        |S )Nr   F)r   ru   r   r   r   r   r   r}   ry   r   r   r#   rV   rU   r   )r@   r   r   s      r   rU   zTokenIterator.newline@  s    4;;t/A/A#Bd#JLLN %%%%)%%++|/K/KK33 LL "" 	 
+ %r   c                   | j                   dk7  rt        t        | j                        dkD  r9| j                  j	                         }| j                  t        j                        S | j                  t        j                        S )Nr   )	r   r   rt   r   r   r   r#   rT   rf   )r@   _s     r   rf   zTokenIterator.endmarkerR  sg    "t  !A%!!%%'A??9#3#344y2233r   c                4   d}| j                   | j                     j                         rd}| j                          | j	                         r| j                   | j                     j                         s| j                   | j                     dk(  re| j                          | j	                         rE| j                   | j                     j                         rH| j                   | j                     dk(  re| j	                         r,| j                   | j                     dk(  r| j                          | j	                         r| j                   | j                     j                         sF| j                   | j                     dk(  r| j                   | j                  dz
     j                         r| j                          | j	                         ro| j                   | j                     j                         rH| j                   | j                     dk(  r+| j                   | j                  dz
     j                         r| j                  dz   t        | j                         k  r<|s+| j                   | j                  dz
     j                         r| j                   | j                     dk(  s| j                   | j                     dk(  r| j                   | j                  dz      j                         s| j                  dz   t        | j                         k  r| j                   | j                  dz      d	k(  s| j                   | j                  dz      d
k(  rJ| j                   | j                  dz      j                         r | j                          | j                          | j	                         r | j                   | j                     j                         sH|s*| j                   | j                  dz
     j                         r| j                   | j                     dk(  r| j                          | j	                         rq| j                   | j                     j                         rH|s*| j                   | j                  dz
     j                         r| j                   | j                     dk(  r| j	                         rt|s*| j                   | j                  dz
     j                         rH| j                   | j                     dk(  s| j                   | j                     dk(  r| j                          | j                  | j                  z
  dk(  r| j                   | j                  dz
     dk(  r| j                  dz   t        | j                         k  rI| j                   | j                  | j                  dz    dk(  r | j                          | j                          | j                  t        j                        S | j                  t        j                        S )NFTr   .r$   eEr%   +-jJz..)ru   r   isdigitr   r   rt   r   r   r#   r`   ra   )r@   digit_before_decimals     r   decimalzTokenIterator.decimal\  s   $;;t))*224#' LLN !KK**+335{{4--.#5LLN	 !KK**+335{{4--.#5 4;;t/A/A#Bc#ILLN!KK**+335D../36KK 2 2Q 67??A LLN !KK**+335D../36KK 2 2Q 67??A
 !C$44!T[[1C1Ca1G%H%P%P%RD../36;;t112c9 D..23;;=&&*S-==D$6$6$:;sB;;t'9'9A'=>#ED$6$6$:;CCE LLNLLN !KK**+335%T5G5G!5K)L)T)T)VKK 2 23s: LLN !KK**+335%T5G5G!5K)L)T)T)VKK 2 23s: !T[[1C1Ca1G%H%P%P%RD../36;;t112c9 LLN 0A5D..23s: ""Q&#dkk*::KK 2 2T5G5G!5KLPTT??9<<00y//00r   c                   | j                          | j                          | j                         r| j                  | j                     dk(  s8| j                  | j                     dk(  s| j                  | j                     dk(  rw| j                          | j                         rW| j                  | j                     dk(  r=| j                  | j                     dk(  rZ| j                  | j                     dk(  rw| j                         r| j                  | j                     dk(  s| j                  | j                     dk(  rL| j                          | j                         r,| j                  | j                     dk(  r| j                          | j                         r| j                  | j                     dk(  s8| j                  | j                     dk(  s| j                  | j                     dk(  rw| j                          | j                         rW| j                  | j                     dk(  r=| j                  | j                     dk(  rZ| j                  | j                     dk(  rw| j	                  t
        j                        S )N01r   r   r   r   r   r   ru   r   r   r#   ra   r?   s    r   binaryzTokenIterator.binary  s   !KK**+s2{{4--.#5{{4--.#5LLN !KK**+s2{{4--.#5{{4--.#5 KK**+s2{{4--.#5LLN  "t{{43E3E'F#'M!KK**+s2{{4--.#5{{4--.#5LLN !KK**+s2{{4--.#5{{4--.#5 y//00r   c                   | j                          | j                          | j                         r| j                  | j                     dk\  r| j                  | j                     dk  s| j                  | j                     dk(  rv| j                          | j                         rV| j                  | j                     dk\  r| j                  | j                     dk  rY| j                  | j                     dk(  rv| j                         r| j                  | j                     dk(  s| j                  | j                     dk(  rL| j                          | j                         r,| j                  | j                     dk(  r| j                          | j                         r| j                  | j                     dk\  r| j                  | j                     dk  s| j                  | j                     dk(  rv| j                          | j                         rV| j                  | j                     dk\  r| j                  | j                     dk  rY| j                  | j                     dk(  rv| j	                  t
        j                        S )Nr   7r   r   r   r   r   r?   s    r   octalzTokenIterator.octal  s   !KK**+s2D../36{{4--.#5LLN !KK**+s2D../36{{4--.#5 KK**+s2{{4--.#5LLN  "t{{43E3E'F#'M!KK**+s2D../36{{4--.#5LLN !KK**+s2D../36{{4--.#5 y//00r   c                   | j                          | j                          | j                         r| j                  | j                     t        j
                  v s| j                  | j                     dk(  rg| j                          | j                         rG| j                  | j                     t        j
                  v rJ| j                  | j                     dk(  rg| j                         r| j                  | j                     dk(  s| j                  | j                     dk(  rL| j                          | j                         r,| j                  | j                     dk(  r| j                          | j                         r| j                  | j                     t        j
                  v s| j                  | j                     dk(  rg| j                          | j                         rG| j                  | j                     t        j
                  v rJ| j                  | j                     dk(  rg| j                  t        j                        S )Nr   r   r   r   )	r   r   ru   r   rb   	hexdigitsr   r#   ra   r?   s    r   hexadecimalzTokenIterator.hexadecimal  s   !KK**+v/?/??{{4--.#5LLN	 !KK**+v/?/??{{4--.#5 KK**+s2{{4--.#5LLN  "t{{43E3E'F#'M!KK**+v/?/??{{4--.#5LLN	 !KK**+v/?/??{{4--.#5 y//00r   c                    t        d      D ]9  }| j                  | j                  |z      }|dk(  s|dk(  s*| j                  |z   c S  t        d      )Nr&   "'zQuote not found somehow)rangeru   r   AssertionError)r@   offsetchars      r   find_opening_quotez TokenIterator.find_opening_quote  sZ    Ah 	3F;;t11F:;Ds{dck))F22	3
 677r   c                L   | j                         }| j                  | j                  | }| j                  |   }|dz   t        | j                        k  r<| j                  |dz      |k(  r'| j                  |dz      |k(  r| j                  ||dz    n| j                  ||dz    }||fS )Nr&   r$   r%   )r   ru   r   rt   )r@   quote_indexr   
quote_charr   s        r   string_prefix_and_quotesz&TokenIterator.string_prefix_and_quotes  s    --/T//+>[[-
 a3t{{#33KKa0J>KKa0J>	 KKkAo6 [;?; 	 u}r   c                   | j                   j                  t        j                  t        j                  fv r| j                         \  }}| j                  ||       t        t        |            D ]  }| j                           t        t        |            D ]  }| j                           | j                   j                          | j                  t        j                        S | j                   j                  t        j                  k(  r| j                  J t        | j                        dk(  }| j                   }| j#                         rM| j$                  | j                      }|dk(  r|rt&        |dk(  r| j                          | j(                  J d| j(                  j+                         vrk| j                   dz   t        | j$                        k  rF| j-                         dk(  r3| j/                         dk(  r | j                          | j                          | j#                         r6| j-                         dk(  s#| j-                         dk(  s| j1                          )|dk(  r| j/                         dk(  r"| j                          | j                          c| j                   j3                          | j                   |k(  r| j5                         S | j                  t        j6                        S | j                  J | j9                  | j                        rX| j                   j;                          | j                   |k(  r| j5                         S | j                  t        j6                        S | j1                          | j#                         rMt<        | j                   j                  t        j>                  k(  ru| j                          | j@                  jC                  | jD                         d| _"        | j                   jG                          | j                  t        jH                        S | j                   j                  t        jJ                  k(  r| j                  J t        t        | j                              D ]  }| j                           | jM                          | j                   jO                          | j                  t        jP                        S | j                   j                  t        jR                  k(  rN| j                   }| j#                         r+| j$                  | j                      }| j                  J |dk(  s|dk(  rt        | j                        dk(  r}|dk(  r| j                   j3                          nt        j                  | j                   _        | j                   |k(  r| j5                         S | j                  t        j6                        S |dk(  r>t        j                  | j                   _        | j                  t        j6                        S | j1                          | j#                         r+t<        tU        d	      )
Nr$   r   \rN{}r   zUnhandled f-string state)+r   r}   ry   r|   r   r   r   r   rt   r   r   r   r#   rc   r   r   r   r   ru   r   r   r   r   r   r   r   fstringrd   r   r   r   r   r   r   r   r   r]   r   r   r   re   r   r   )r@   r   r   r   is_single_quoterl   r   s          r   r  zTokenIterator.fstring  s   ##$$(((
 
 !99;MFE**6593v;' 3u:& ,,.??9#:#:;;##|'E'EE%%111!$"4"45:O,,K##%{{4#5#564<O,, 4<LLN  ..:::4#6#6#<#<#>> ..2S5EE IIK3. NN,3((*		s*diikS.@224 3;~~'3. **LLN--<#'<<>1#y/G/GHH))555::d001&&EEG))[8#||~-??9+C+CDD**,i ##%l  ##|'E'EELLN$$++D,>,>?!"D--/??9#3#344##|'B'BB%%1113t1123 ""$,,.??9#8#899##|'L'LL,,K##%{{4#5#56))555DLDCKS9K9K5LPQ5Qs{**LLN 4@3O3O**0 ))[8#||~-??9+C+CDDS[/;/K/KD&&,??9+C+CDD**,' ##%*  788r   c                b   | j                         \  }}|r<| j                  r0| j                          | j                  t        j
                        S |D ]  }|dk(  s|dk(  s| j                         c S  t        t        |            D ]  }| j                           t        t        |            D ]  }| j                           t        |      dk(  }| j                         r| j                  | j                     }|dk(  r|rt        |dk(  r!| j                          | j                          \| j                  |      rHt        t        |            D ]  }| j                           | j                  t        j                        S | j                          | j                         rt         )N)r   fFr$   r   r   )r   r   r   r   r#   r`   r  r   rt   r   ru   r   r   r   r   rb   r   )r@   r   r   r   r   r  s         r   rb   zTokenIterator.string  sn   557d((LLN??ILL?99 	&Ds{dck||~%	& s6{# 	ALLN	s5z" 	ALLN	 e*/!;;t112Dt|(( t|**, zz% s5z* #ALLN#y'7'788&&(% !( r   c                   | j                   }d}d}| j                         rZ| j                  | j                      }| j                         r| j	                          d}|dk(  s|dk(  rd}nn| j                         rZ| j                         s4| j                   |k(  rt
        | j                  t        j                        S |r(|s&d| _	        | j                  t        j                        S | j                         }|dk(  s|dk(  s| j                         r| j                  t        j                        S | j                  || j                    }t        | j                        dk(  rdn| j                  d	   }t        |      t        |      k(  r>t        |      dk(  rt
        ||k7  rt        | j                  t        j                        S t        |      t        |      kD  rRt        |      dkD  r
||vrt        | j                  j                  |       | j                  t        j                         S t        | j                        dkD  r| j                  d	   }t        |      t        |      k  rt"        t        |      t        |      k(  rnH| j                  j%                         }	| xj&                  d
z  c_        t        | j                        dkD  r| j                  t        j                        S )NFT 	#r   r   rs   r   r$   )r   r   ru   is_whitespacer   r   r   r#   rR   r   r   
is_newlinert   r   r   r   rS   r   r   r   )
r@   rl   saw_whitespacesaw_tab_or_spacer   	next_char
new_indentcurrent_indent
top_indentr   s
             r   rS   zTokenIterator.indent  s~   (( !;;t112D!!#!%3;$$,'+$ !   "!![0!!??9#7#788 "2)-D&??9#7#788 IIK	yD0DOO4E??9#7#788[[t/A/AB
"4#4#45:@Q@QRT@Uz?c.11:!#!!^+44??9#7#788_s>22>"Q&>+K44$$Z0??9#3#344d''(1,!..r2
z?S_4::z?c*o5%%))+##q(# d''(1, ??9#7#788r   c                    | j                         ry| j                  | j                     }|dk(  xs |dk(  xs |dk(  xs |dk(  xs |dk(  S )NFr  r   r	  )r  ru   r   )r@   r   s     r   r  zTokenIterator.is_whitespace  sc    ??{{4--.CK t|t| v~ v~	
r   c                    | j                   | j                     dk(  ry| j                   | j                     dk(  rE| j                  dz   t        | j                         k  r | j                   | j                  dz      dk(  ryy)Nr   Tr   r$   F)ru   r   rt   r?   s    r   r  zTokenIterator.is_newline  sq    ;;t))*d2KK**+t3""Q&T[[)99D..23t;r   c                   | j                   r/| j                          | j                  t        j                        S | j
                  | j                  d  }t        |      D ]3  \  }}t        |      dk  st        j                  |      r+|dk7  s1|} n t        |      }| j                  |       | j                  t        j                        S )N   r   )r   r   r   r#   rC   ru   r   	enumerateordrO   isalnumrt   r   )r@   	remainingindexr   lengths        r   r>   zTokenIterator.name  s    LLN??9#7#788 KK 2 2 45	$Y/ 	$KE44y3s{{4'8TS[	$
 ^Fy3344r   c                    | S rJ   r   r?   s    r   __iter__zTokenIterator.__iter__  s    r   c                t   | j                   -| j                   j                  t        j                  k(  rt        | j
                  t        | j                        k(  r| j                   | j                         S | j                   j                  t        j                  t        j                  t        j                  hv r| j                         S | j                         S | j
                  t        | j                        kD  r| j                         S | j                  j                  t        j                  k7  r7| j                  j                  t        j                  k7  r| j!                         S | j                  | j
                     }| j"                  s|dk(  r| j%                          | j'                         s| j                         S | j                  | j
                     }|dk7  rAd| _        | j                   .| j                   j                  t        j*                  k(  rd| _        |dk(  r| j(                  r/| j%                          | j/                  t        j*                        S | j'                         rj| j1                         sZ| j"                  s| j                  | j
                     dk(  rn1| j%                          | j'                         r| j1                         sZ| j/                  t        j*                        S | j2                  dkD  r4| xj2                  dz  c_        | j/                  t        j                        S | j1                         r| j                         S |dk(  r| j%                          | j'                         st4        d}d}| j'                         r| j7                         r| j%                          d}n|s| j1                         rw| j                  | j
                     }|dk(  r| j%                          | j%                          d}d}| j8                  r| j;                          n*| j;                          d| _        nn| j'                         r|st<        | j/                  t        j>                        S | j@                  dk(  s| jB                  rR| jD                  dk(  rC| j                  j                  t        j                  k(  rd| _!        	 | jG                         }||S | j7                         rp| j'                         rA| j7                         r1| j%                          | j'                         r| j7                         r1| j/                  t        j>                        S |d	v rR| j%                          | jK                         d
k(  r| j%                          | j/                  t        jL                        S |dk(  r| j%                          | jK                         dk(  r/| j%                          | j/                  t        jL                        S | jK                         dk(  r| j%                          | jK                         d
k(  r| j%                          | j/                  t        jL                        S |dk(  ru| j%                          | jK                         dk(  r| j%                          | jK                         d
k(  r| j%                          | j/                  t        jL                        S |dk(  ru| j%                          | jK                         dk(  r| j%                          | jK                         d
k(  r| j%                          | j/                  t        jL                        S |dk(  ru| j%                          | jK                         dk(  r| j%                          | jK                         d
k(  r| j%                          | j/                  t        jL                        S |dk(  r| j%                          | jK                         dk(  r/| j%                          | j/                  t        jL                        S | jK                         d
k(  r| j%                          | j/                  t        jL                        S |dv r/| j%                          | j/                  t        jL                        S |dk(  r/| j%                          | j/                  t        jL                        S |dk(  rD| j%                          | xjD                  dz  c_"        | j/                  t        jN                        S |dk(  rZ| j%                          | xjD                  dz  c_"        | jD                  dk  rd| _"        | j/                  t        jP                        S |dk(  rD| j%                          | xjD                  dz  c_"        | j/                  t        jR                        S |dk(  rZ| j%                          | xjD                  dz  c_"        | jD                  dk  rd| _"        | j/                  t        jT                        S |dk(  rD| j%                          | xjD                  dz  c_"        | j/                  t        jV                        S |dk(  r| j%                          | jD                  dk(  ra| j                  j                  t        j                  k(  r:| j                  jY                          | jZ                  j]                         | _"        n+| xjD                  dz  c_"        | jD                  dk  rd| _"        | j/                  t        j^                        S |dk(  r| j%                          | jD                  dk(  re| j                  j                  t        j                  k(  r>t        j`                  | j                  _        | j/                  t        jL                        S | jK                         d
k(  r| j%                          | j/                  t        jL                        S |dv r'| j
                  dz   t        | j                        k  r8| j                  | j
                  | j
                  dz    dv r| jc                         S | j
                  dz   t        | j                        k  r8| j                  | j
                  | j
                  dz    dv r| je                         S | j
                  dz   t        | j                        k  r8| j                  | j
                  | j
                  dz    dv r| jg                         S | ji                         S | j
                  dz   t        | j                        k  r| jk                  dd      s~| j
                  dz   t        | j                        k  r| jk                  d d!d"d#d$d%d&d'd(	      s?| j
                  d)z   t        | j                        k  r*| jk                  d*d+d,d-d.d/d0d1d(	      r| jm                         S | jo                         S # tH        $ r d }Y 	Fw xY w)2Nr   r   Tr
  r   r$   r   F)	r   &|^@%=!~r'  <>/*r   ),;`()[]r   r  :z.0123456789r%   )0b0B)0o0O)0x0Xr   r   zb"zb'zr"zr'zf"zf'zu"zu'r   r&   zbr"zbr'zrb"zrb'zfr"zfr'zrf"zrf')8r   rj   r#   rf   StopIterationr   rt   ru   rU   rV   rT   r   r}   ry   r|   r   r  r   r   r   r   rW   r   r   r  r   r   r  r   r   r   rR   r   r   r   rS   r   r   r`   rY   rZ   r[   r\   r]   r   r   r   r^   r   r   r   r   r   r   rb   r>   )r@   current_charfound_whitespaceseen_newliner   indent_tokens         r   __next__zTokenIterator.__next__  s   ??&4??+?+?9CVCV+V T[[!11&~~''##!!  ( 
 ~~''||~%DKK 00>>## $$(@(@@""((L,H,HH<<>!{{4#5#56 ))ld.BLLN$$&||~%;;t'9'9:Lt#%)"OO/,,	0A0AA,0D) 3!!y'8'899##%doo.?22D$6$674? ##%doo.? ??9#4#455 "1$??9#3#344 ??<<>! 4LLN$$&##  % L##%%%'LLN'+$%4??+<;;t'9'9:Dt|LLN'+$#'L 77((;@8 / ##%2 $77??9#7#788 "d&@&@""a'""((L,D,DD).D&$#{{} '####%$*<*<*> ##%$*<*<*>??9#7#788HHLLNyy{c!??9<<003LLNyy{c!y||44yy{c!yy{c!??9<<003LLNyy{c!yy{c!??9<<003LLNyy{c!yy{c!??9<<003LLNyy{c!yy{c!??9<<003LLNyy{c!y||44 yy{c!??9<<00:%LLN??9<<00 3LLN??9<<003LLN!#??9#3#3443LLN!#!!A%%&"??9#3#3443LLN!#??9#5#5663LLN!#!!A%%&"??9#5#5663LLN!#??9#3#3443LLN""a'&&,,0L0LL""113%)%=%=%A%A%C"""a'"%%))*D&??9#3#3443LLN""a'&&,,0L0LL+7+P+P""(y||4499;#%LLNy||44=(!!A%T[[)99dkk""T%7%7!%;?? {{}$##a'3t{{+;;""T%7%7!%;AA zz|###a'3t{{+;;""T%7%7!%;AA ''))||~% !#s4;;'77DJJsC<P""Q&#dkk*::JJ $  
 ""Q&#dkk*::JJ $  
 ;;= yy{m  $#$s    x( (x76x7rP   rM   r   )r   rk   rN   r   )r   rO   r   rQ   rN   rQ   )r   r#   rN   ri   )r   rO   r   rO   rN   r   )rN   ri   )rN   rk   )rN   ztuple[str, str])rN   r   )5r   r   r   rw   r   r   r   r   r   r   r   r   r   listr   r   r   r   ry   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rU   rf   r   r   r   r   r   r   r  rb   rS   r  r  r>   r   rA  r   r   r   r   r      s   KM3JKcKc(,,M3%*4%@@#J##D9L)9NC #("EM<E8=d8S 5S!%NJ% $M:$  M4"d""'4'5/3"0
 8= -^#	
$4Q1f1414108"r9h%N=9~

5$nr   r   T)fstring_tokensr   c               L    t        | |      }|rt        |      S t        |      S )N)r   )r   itermerge_fstring_tokens)ru   rC  r   token_iterators       r   tokenizerH  N  s*     #6AVWNN##//r   c           
   #    K   | D ]  }|j                   t        j                  k7  r| %|}|}d}d}| D ]O  }|j                   t        j                  k(  r|dz  }|j                   t        j                  k(  r|dz  }||k(  sM|} n t	        t        j
                  |j                  |j                  |j                  |j                  |j                  |j                          yw)zETurn post-Python-3.12 FSTRING-* tokens back to a single STRING token.r$   r   )rj   rl   rn   ro   rm   rp   rq   N)rj   r#   rc   re   ri   rb   rl   rn   ro   rm   rp   rq   )rG  r   start_token	end_tokenfstring_startsfstring_endss         r   rF  rF  [  s      
::000K	# 	EzzY444!#zzY222!-!		 !!#//"--!++))''%%
 	
)
s   A>C'A&C')ru   rO   rC  rQ   r   rQ   rN   Iterator[Token])rG  r   rN   rN  )__doc__
__future__r   dataclassesr   r   enumrb   typingr   r   	Exceptionr	   r   r   r   r   r   r   r   r   r!   IntEnumr#   ri   ry   r   rH  rF  r   r   r   <module>rV     s    Z " (   $ $I # +} * <%5 ; >'7 = - , (M ' < ; '. & % $,> ,>^  9  9  9F2; 2;j \ \ \D  "&	
0
0 
0  	
0
 
0
r   