+
    f4jZ/                         R t ^ RIt^ RIt^ RIt^ RIt^ RItR.t]P                  ! RR4      t	 ! R R4      t
 ! R R4      t ! R R	4      tR
 tR tR tR tR tR# )zrobotparser.py

Copyright (C) 2000  Bastian Kleineidam

You can choose between two licenses when using this package:
1) GNU GPLv2
2) PSF license for Python 2.2

The robots.txt Exclusion Protocol is implemented as specified in
RFC 9309
NRobotFileParserRequestRatezrequests secondsc                   v   a  ] tR t^t o RtRR ltR tR tR tR t	R t
R tR	 tR
 tR tR tR tR tRtV tR# )r   zjThis class provides a set of methods to read, parse and answer
questions about a single robots.txt file.

c                    . V n         / V n        . V n        R V n        RV n        RV n        V P                  V4       ^ V n        R # )NF)entriesgroupssitemapsdefault_entrydisallow_all	allow_allset_urllast_checkedselfurls   &&$lib/python3.14/urllib/robotparser.py__init__RobotFileParser.__init__   sB    !!S    c                    V P                   # )zReturns the time the robots.txt file was last fetched.

This is useful for long-running web spiders that need to
check for new robots.txt files periodically.

)r   r   s   &r   mtimeRobotFileParser.mtime(   s        r   c                8    ^ RI pVP                  4       V n        R# )zISets the time the robots.txt file was last fetched to the
current time.

N)timer   )r   r   s   & r   modifiedRobotFileParser.modified1   s    
 	 IIKr   c                x    Wn         \        P                  P                  V4      R,          w  V n        V n        R# )z,Sets the URL referring to a robots.txt file.:      NN)r   urllibparseurlsplithostpathr   s   &&r   r   RobotFileParser.set_url9   s)    %||44S9#>	49r   c                    \         P                  P                  V P                  4      pVP	                  4       pV P                  VP                  RR4      P                  4       4       R#   \         P                  P                   d^   pTP                  R9   d	   RT n        M)TP                  R8  d   TP                  R8  d   RT n        TP                  4         Rp?R# Rp?ii ; i)z4Reads the robots.txt URL and feeds it to the parser.zutf-8surrogateescapeTi  i  N)i  i  )r    requesturlopenr   readr!   decode
splitlineserror	HTTPErrorcoder
   r   close)r   frawerrs   &   r   r*   RobotFileParser.read>   s    
	L&&txx0A &&(CJJszz'+<=HHJK ||%% 	xx:%$(!SSXX^!%IIKK	s   )A, ,C(AC##C(c                   V P                   P                  V4       VP                   F]  pVP                  4       pW P                  9  d   WP                  V&   K3  \        V P                  V,          V4      V P                  V&   K_  	  R # N)r   append
useragentslowerr   merge_entries)r   entryagents   && r   
_add_entryRobotFileParser._add_entryL   sc    E"%%EKKMEKK'%*E"%24;;u3Eu%ME" &r   c                   . p^ p\        4       pV P                  4        V EF  pVP                  R4      pV^ 8  d   VRV pVP                  4       pV'       g   K;  VP	                  R^4      p\        V4      ^8X  g   K_  V^ ,          P                  4       P                  4       V^ &   V^,          P                  4       V^&   V^ ,          R8X  dK   V^8X  d   V P                  V4       \        4       pV^,          pVP                  P                  V4       ^pK  V^ ,          R8X  d=   V^ 8w  d3   ^p VP                  P                  \        V^,          R4      4       EK@  EKC  V^ ,          R8X  d=   V^ 8w  d3   ^p VP                  P                  \        V^,          R4      4       EK  EK  V^ ,          R	8X  dR   V^ 8w  dH   V^,          P                  4       P                  4       '       d   \        V^,          4      Vn        ^pEK  EK  V^ ,          R
8X  d   V^ 8w  d   V^,          P	                  R4      p\        V4      ^8X  d   V^ ,          P                  4       P                  4       '       d]   V^,          P                  4       P                  4       '       d2   \!        \        V^ ,          4      \        V^,          4      4      Vn        ^pEK  EK  V^ ,          R8X  g   EK  V P$                  P                  V^,          4       EK  	  V^ 8w  d   V P                  V4       R# R#   \         d     EK  i ; i  \         d     EK.  i ; i)z|Parse the input lines from a robots.txt file.

We allow that a user-agent: line is not preceded by
one or more blank lines.
#N:z
user-agentdisallowFallowTzcrawl-delayzrequest-rate/sitemap)Entryr   findstripsplitlenr9   r=   r8   r7   	rulelinesRuleLine
ValueErrorisdigitintdelayr   req_rater   )	r   linesr   stater;   lineiproduct_tokennumberss	   &&       r   r!   RobotFileParser.parseU   s    
 D		#AAvBQx::<D::c1%D4yA~q'--///1Qq'--/Q7l*z. %$(GM$$++M:E!W
*z !!!OO228DGU3KL " !W'z !!!OO228DGT3JK " !W-z  7==?2244*-d1g,EK ! " !W.z"&q'--"4LA-'!*2B2B2D2L2L2N2N '
 0 0 2 : : < <-8WQZ#gVWj/-ZEN ! " !W	)
 MM((a1m n A:OOE" ?  * ! !  * ! !s$   2,L-<,M -L=<L= MMc                   V P                   P                  VP                  4       4      pVe   V# V P                   P                  4        F  pVP	                  V4      '       g   K  Vu # 	  V P                   P                  R4      # )N*)r   getr9   values
applies_tor   	useragentr;   s   && r   _find_entryRobotFileParser._find_entry   sh    	 12L[['')E	** * {{s##r   c                   V P                   '       d   R# V P                  '       d   R# V P                  '       g   R# \        P                  P                  VR4      p\        P                  P                  ! RR.VR,          O5!  p\        V4      pV'       g   RpVR8X  d   R# V P                  V4      pVf   R# VP                  V4      # )z=using the parsed robots.txt decide if useragent can fetch urlFT N:   NNrD   z/robots.txt)
r
   r   r   r    r!   	_urlsplit_urlunsplitnormalize_urir`   	allowance)r   r_   r   
parsed_urlr;   s   &&&  r   	can_fetchRobotFileParser.can_fetch   s    >>>
     \\++C4
ll&&tTCJrNCC C-  +=s##r   c                x    V P                  4       '       g   R # V P                  V4      pVf   R # VP                  # r6   )r   r`   rP   r^   s   && r   crawl_delayRobotFileParser.crawl_delay   s2    zz||  +={{r   c                x    V P                  4       '       g   R # V P                  V4      pVf   R # VP                  # r6   )r   r`   rQ   r^   s   && r   request_rateRobotFileParser.request_rate   s2    zz||  +=~~r   c                B    V P                   '       g   R # V P                   # r6   )r   r   s   &r   	site_mapsRobotFileParser.site_maps   s    }}}}}r   c           	         V P                   pV P                  e   WP                  .,           pRP                  \        R \	        \
        V4      4      4      # )Nz

)r   r	   joinfiltermapstr)r   r   s   & r   __str__RobotFileParser.__str__   sE    ,,)!3!3 44G{{6$C(9:;;r   )
r   r	   r
   r   r   r#   r   r$   r   r   N)rc   )__name__
__module____qualname____firstlineno____doc__r   r   r   r   r*   r=   r!   r`   rj   rm   rp   rs   rz   __static_attributes____classdictcell____classdict__s   @r   r   r      sT     
!(?
LNG#R$$6
< <r   c                   6   a  ] tR t^t o RtR tR tR tRtV t	R# )rL   zhA rule line is a single "Allow:" (allowance==True) or "Disallow:"
(allowance==False) followed by a path.c                
   VR 8X  d   V'       g   Rp\         P                  ! RRV4      p\         P                  ! RRV4      p\        V4      pVP                  R4      V n        VP                  R4      pRV9   d   \        R4      hRV n        RV9   dd   \         P                  ! \        V4      \         P                  4      pV P                  '       d   VP                  V n        MVP                  V n        Wn        W n        R# )rc   Tz[*]{2,}rZ   z[$][$*]+$z$ not at the end of pathN)resubnormalize_patternendswith	fullmatchrstriprM   matchercompiletranslate_patternDOTALLmatchr$   rh   )r   r$   rh   patterns   &&& r   r   RuleLine.__init__   s    2:iIvvj#t,vvk3- &s+{{3$;788$;jj!24!8"))DG~~~&00&}}	"r   c                   V P                   e4   V P                  V4      pV'       d   VP                  4       ^,           #  ^ # V P                  '       d0   WP                  8X  d   \	        V P                  4      ^,           #  ^ # VP                  V P                  4      '       d   \	        V P                  4      ^,           # ^ # r6   )r   endr   r$   rJ   
startswith)r   filenamems   && r   r]   RuleLine.applies_to   s     <<#X&Auuw{"   ~~~yy(tyy>A-- )
  &&tyy11tyy>A--r   c                    V P                   '       d   R MRR,           V P                  ,           V P                  '       d	   R,           # R,           # )AllowDisallowz: r   rc   )rh   r$   r   r   s   &r   rz   RuleLine.__str__  s?     NNN
dBTYYN...32 	3.02 	3r   )rh   r   r   r$   N)
r|   r}   r~   r   r   r   r]   rz   r   r   r   s   @r   rL   rL      s     1#* 3 3r   rL   c                   <   a  ] tR tRt o RtR tR tR tR tRt	V t
R# )	rF   i
  z?An entry has one or more user-agents and zero or more rulelinesc                >    . V n         . V n        R V n        R V n        R # r6   )r8   rK   rP   rQ   r   s   &r   r   Entry.__init__  s    
r   c                   V P                   '       g   R # . pV P                    F  pVP                  RV 24       K  	  V P                  e   VP                  RV P                   24       V P                  e8   V P                  pVP                  RVP                   RVP
                   24       V P                  '       d+   VP                  \        \        V P                  4      4       MVP                  R4       RP                  V4      # )rc   zUser-agent: zCrawl-delay: zRequest-rate: rD   zAllow:
)r8   r7   rP   rQ   requestssecondsrK   extendrx   ry   rv   )r   retr<   rates   &   r   rz   Entry.__str__  s    __EJJeW-. %::!JJtzzl34==$==DJJa~FG>>>JJs3/0JJx yy~r   c                    Vf   RV P                   9   # VP                  R4      ^ ,          P                  4       pV P                    F%  pVR8w  g   K  VP                  4       pW!9   g   K$   R# 	  R# )z2check if this entry applies to the specified agentrZ   rD   TF)r8   rI   r9   )r   r_   r<   s   && r   r]   Entry.applies_to#  sa    $//))OOC(+113	__E|%	 %
 r   c                    RpRpV P                    FR  pVP                  V4      pV'       g   K  WR8  d   TpVP                  pK4  WR8X  g   K<  V'       d   KF  VP                  pKT  	  V# )zKPreconditions:
- our agent applies to this entry
- filename is URL encoded
T)rK   r]   rh   )r   r   
best_matchrh   rT   r   s   &&    r   rh   Entry.allowance0  s\    
 
	NND)Aq>!"J $I_YY $I # r   )rP   rQ   rK   r8   N)r|   r}   r~   r   r   r   rz   r]   rh   r   r   r   s   @r   rF   rF   
  s!     I" r   rF   c                     \         P                  P                  V R R7      p\         P                  P                  VR R7      # )r'   )errors)r    r!   unquotequote)r$   unquoteds   & r   	normalizer   B  s7    ||##D1B#CH<<h/@AAr   c                     V P                  R 4      w  rp\        V 4      p V'       d*   \        P                  ! RR V4      pV R V,           ,          p V # )?z[^=&]+c                 &    \        V ^ ,          4      #     r   r   s   &r   <lambda>normalize_uri.<locals>.<lambda>J  s    IadOr   )	partitionr   r   r   r$   sepquerys   &  r   rg   rg   F  sG    ~~c*DuT?D
y";UCeKr   c                     V P                  R 4      w  rp\        P                  ! RR V 4      p V'       d*   \        P                  ! RR V4      pV R V,           ,          p V # )r   z[^*$]+c                 &    \        V ^ ,          4      # r   r   r   s   &r   r   #normalize_pattern.<locals>.<lambda>P  s    y1r   z[^=&*$]+c                 &    \        V ^ ,          4      # r   r   r   s   &r   r   r   R  s    i!or   )r   r   r   r   s   &  r   r   r   N  sQ    ~~c*Du66)6=D
{$=uEeKr   c                    \        \        \        P                  V P	                  R 4      4      4      p\        ^\        V4      ^,
          4       F  pRW,           R2W&   K  	  RVR,           2VR&   RP                  V4      # )rZ   z(?>.*?)z.*rc   r   )listrx   r   escaperI   rangerJ   rv   )r$   partsrU   s   &  r   r   r   V  sl    RYY

301E1c%jl#EH:Q' $U2YK E"I775>r   c                    \        4       p\        \        \        VP                  4      P
                  V P                  4      4      Vn        V P                  VP                  ,           Vn        VP                  f   V P                  MVP                  Vn        VP                  f   V P                  Vn        V# VP                  Vn        V# r6   )	rF   r   rw   setr8   __contains__rK   rP   rQ   )e1e2r;   s   && r   r:   r:   ]  s    GEF3r}}#5#B#BBMMRSEllR\\1EO hh."((BHHEK$&KK$7R[[ENL >@[[ENLr   )r   collectionsr   urllib.errorr    urllib.parseurllib.request__all__
namedtupler   r   rL   rF   r   rg   r   r   r:    r   r   <module>r      ss   
  	   
$$]4FGC< C<J*3 *3Z5 5pBr   