o cx@ @sdZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z ddlmZddlmZmZmZmZmZmZmZmZmZmZmZddlmZddlmZddl m!Z!m"Z"dd l#m$Z$dd l%m&Z&dd l'm(Z(dd l)m*Z*dd l+m,Z,ddl-m.Z.ddl/m0Z0ddl1m2Z2ddl3m4Z4m5Z5m6Z6erddlm7Z7ne8Z7e9e:Z;eeGddde?Z@deddfddZAGddde?ZBdeZNd5e&de*ded/fd?d@ZOGdAdBdBeZPGdCdDdDZQdS)GzO The main purpose of this module is to expose LinkCollector.collect_sources(). N) HTMLParser)Values) TYPE_CHECKINGCallableDictIterableListMutableMapping NamedTupleOptionalSequenceTupleUnion)requests)Response) RetryErrorSSLError)NetworkConnectionError)Link) SearchScope) PipSession)raise_for_status)is_archive_fileredact_auth_from_url)vcs)CandidatesFromPage LinkSource build_source)ProtocolurlreturncCs6tjD]}||r|t|dvr|SqdS)zgLook for VCS schemes in the URL. Returns the matched VCS scheme, or None if there's no match. z+:N)rschemeslower startswithlen)r!schemer(J/opt/certbot/lib/python3.10/site-packages/pip/_internal/index/collector.py_match_vcs_scheme7s r*cs*eZdZdededdffdd ZZS)_NotAPIContent content_type request_descr"Ncst||||_||_dSN)super__init__r,r-)selfr,r- __class__r(r)r0Cs z_NotAPIContent.__init__)__name__ __module__ __qualname__strr0 __classcell__r(r(r2r)r+Bs"r+responsecCs2|jdd}|}|drdSt||jj)z Check the Content-Type header to ensure the response contains a Simple API Response. Raises `_NotAPIContent` if the content type is not a valid content-type. Content-TypeUnknown)z text/htmlz#application/vnd.pypi.simple.v1+html#application/vnd.pypi.simple.v1+jsonN)headersgetr$r%r+requestmethod)r9r,content_type_lr(r(r)_ensure_api_headerIsrBc@s eZdZdS)_NotHTTPN)r4r5r6r(r(r(r)rC_srCsessioncCsFtj|\}}}}}|dvrt|j|dd}t|t|dS)z Send a HEAD request to the URL, and ensure the response contains a simple API Response. Raises `_NotHTTP` if the URL is not available for a HEAD request, or `_NotAPIContent` if the content type is not a valid content type. >httphttpsT)allow_redirectsN)urllibparseurlsplitrCheadrrB)r!rDr'netlocpathqueryfragmentrespr(r(r)_ensure_api_responsecs  rQcCsxtt|jr t||dtdt||j|dgdddd}t |t |tdt||j d d |S) aYAccess an Simple API response with GET, and return the response. This consists of three parts: 1. If the URL looks suspiciously like an archive, send a HEAD first to check the Content-Type is HTML or Simple API, to avoid downloading a large file. Raise `_NotHTTP` if the content type cannot be determined, or `_NotAPIContent` if it is not HTML or a Simple API. 2. Actually perform the request. Raise HTTP exceptions on network failures. 3. Check the Content-Type header to make sure we got a Simple API response, and raise `_NotAPIContent` otherwise. rDzGetting page %sz, )r<z*application/vnd.pypi.simple.v1+html; q=0.1ztext/html; q=0.01z max-age=0)Acceptz Cache-Control)r=zFetched page %s as %sr:r;) rrfilenamerQloggerdebugrr>joinrrBr=)r!rDrPr(r(r)_get_simple_responseus&  rXr=cCs<|rd|vrtj}|d|d<|d}|rt|SdS)z=Determine if we have any encoding information in our headers.r:z content-typecharsetN)emailmessageMessage get_paramr7)r=mrYr(r(r)_get_encoding_from_headerss    r_c@s6eZdZd ddZdedefdd Zdefd d ZdS) CacheablePageContentpage IndexContentr"NcCs|jsJ||_dSr.)cache_link_parsingrar1rar(r(r)r0s  zCacheablePageContent.__init__othercCst|t|o|jj|jjkSr.) isinstancetyperar!)r1rer(r(r)__eq__szCacheablePageContent.__eq__cCs t|jjSr.)hashrar!r1r(r(r)__hash__s zCacheablePageContent.__hash__)rarbr"N) r4r5r6r0objectboolrhintrkr(r(r(r)r`s r`c@s"eZdZdddeefddZdS) ParseLinksrarbr"cCsdSr.r(rdr(r(r)__call__szParseLinks.__call__N)r4r5r6rrrpr(r(r(r)rosrofncsPtjdddtdttffdd tdddttffd d }|S) z Given a function that parses an Iterable[Link] from an IndexContent, cache the function's result (keyed by CacheablePageContent), unless the IndexContent `page` has `page.cache_link_parsing == False`. N)maxsizecacheable_pager"cst|jSr.)listra)rs)rqr(r)wrappersz*with_cached_index_content..wrapperrarbcs|jr t|St|Sr.)rcr`rt)rarqrur(r)wrapper_wrappers  z2with_cached_index_content..wrapper_wrapper) functools lru_cacher`rrwraps)rqrwr(rvr)with_cached_index_contents r{rarbc cs|j}|dr+t|j}|dgD]}t||j }|dur%q|VqdSt |j }|j p4d}| |j ||j }|jpE|}|jD]} tj| ||d}|durXqI|VqIdS)z\ Parse a Simple API's Index Content, and yield its anchor elements as Link objects. r<filesNzutf-8)page_urlbase_url)r,r$r%jsonloadscontentr>r from_jsonr!HTMLLinkParserencodingfeeddecoder~anchors from_element) rarAdatafilelinkparserrr!r~anchorr(r(r) parse_linkss*       rc @sHeZdZdZ ddededeedededd f d d Zdefd d Z d S)rbz5Represents one response (or page), along with its URLTrr,rr!rcr"NcCs"||_||_||_||_||_dS)am :param encoding: the encoding to decode the given content. :param url: the URL from which the HTML was downloaded. :param cache_link_parsing: whether links parsed from this page's url should be cached. PyPI index urls should have this set to False, for example. N)rr,rr!rc)r1rr,rr!rcr(r(r)r0s  zIndexContent.__init__cCs t|jSr.)rr!rjr(r(r)__str__s zIndexContent.__str__T) r4r5r6__doc__bytesr7r rmr0rr(r(r(r)rbs" csveZdZdZdeddffdd Zdedeeeeefddfd d Z deeeeefdeefd d Z Z S) rzf HTMLParser that keeps the first base HREF and a list of all anchor elements' attributes. r!r"Ncs$tjdd||_d|_g|_dS)NT)convert_charrefs)r/r0r!r~r)r1r!r2r(r)r0#s zHTMLLinkParser.__init__tagattrscCsR|dkr|jdur||}|dur||_dSdS|dkr'|jt|dSdS)Nbasea)r~get_hrefrappenddict)r1rrhrefr(r(r)handle_starttag*s  zHTMLLinkParser.handle_starttagcCs"|D] \}}|dkr|SqdS)Nrr()r1rnamevaluer(r(r)r2s zHTMLLinkParser.get_href) r4r5r6rr7r0rr r rrr8r(r(r2r)rs &.rrreasonmeth).NcCs|durtj}|d||dS)Nz%Could not fetch URL %s: %s - skipping)rUrV)rrrr(r(r)_handle_get_simple_fail9srTrccCs&t|j}t|j|jd||j|dS)Nr:)rr!rc)r_r=rbrr!)r9rcrr(r(r)_make_index_contentCs rc Cs|jddd}t|}|rtd||dStj|\}}}}}}|dkrHtj tj |rH| ds;|d7}tj|d}td|zt||d }Wnty`td |YdSty|}ztd ||j|jWYd}~dSd}~wty}z t||WYd}~dSd}~wty}z t||WYd}~dSd}~wty}zd } | t|7} t|| tjd WYd}~dSd}~wtjy}zt|d|WYd}~dSd}~wtjyt|dYdSwt||jdS)N#rrzICannot look at %s URL %s because it does not support lookup as web pages.r/z index.htmlz# file: URL is directory, getting %srRz`Skipping page %s because it looks like an archive, and cannot be checked by a HTTP HEAD request.zSkipping page %s because the %s request got Content-Type: %s. The only supported Content-Types are application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html, and text/htmlz4There was a problem confirming the ssl certificate: )rzconnection error: z timed out)rc) r!splitr*rUwarningrHrIurlparseosrMisdirr? url2pathnameendswithurljoinrVrXrCr+r-r,rrrrr7inforConnectionErrorTimeoutrrc) rrDr! vcs_schemer'_rMrPexcrr(r(r)_get_index_contentPsn       rc@s.eZdZUeeeed<eeeed<dS)CollectedSources find_links index_urlsN)r4r5r6r r r__annotations__r(r(r(r)rs rc @seZdZdZdededdfddZe dded ed e ddfd d Z e de e fd dZdedeefddZde dedefddZdS) LinkCollectorz Responsible for collecting Link objects from all configured locations, making network requests as needed. The class's main method is its collect_sources() method. rD search_scoper"NcCs||_||_dSr.)rrD)r1rDrr(r(r)r0s zLinkCollector.__init__Foptionssuppress_no_indexcCsd|jg|j}|jr|stdddd|Dg}|jp g}tj|||jd}t ||d}|S)z :param session: The Session to use to make requests. :param suppress_no_index: Whether to ignore the --no-index option when constructing the SearchScope object. zIgnoring indexes: %s,css|]}t|VqdSr.r).0r!r(r(r) sz'LinkCollector.create..)rrno_index)rDr) index_urlextra_index_urlsrrUrVrWrrcreater)clsrDrrrrrlink_collectorr(r(r)rs$  zLinkCollector.createcCs|jjSr.)rrrjr(r(r)rszLinkCollector.find_linkslocationcCst||jdS)z> Fetch an HTML page containing package links. rR)rrD)r1rr(r(r)fetch_responseszLinkCollector.fetch_response project_namecandidates_from_pagecstfddj|D}tfddjD}ttj rIddt ||D}t |d|dg|}t d|tt|t|d S) Nc3&|]}t|jjdddVqdS)Frpage_validator expand_dirrcNrrDis_secure_originrlocrr1r(r)r z0LinkCollector.collect_sources..c3r)TrNrrrr(r)rrcSs*g|]}|dur|jdurd|jqS)Nz* )r)rsr(r(r) s z1LinkCollector.collect_sources..z' location(s) to search for versions of : )rr) collections OrderedDictrget_index_urls_locationsvaluesrrU isEnabledForloggingDEBUG itertoolschainr&rVrWrrt)r1rrindex_url_sourcesfind_links_sourceslinesr(rr)collect_sourcess2       zLinkCollector.collect_sources)F)r4r5r6rrrr0 classmethodrrmrpropertyrr7rrr rbrrrrr(r(r(r)rs<  !rr.r)Rrr email.messagerZrxrrrr urllib.parserHurllib.request html.parserroptparsertypingrrrrrr r r r r r pip._vendorrpip._vendor.requestsrpip._vendor.requests.exceptionsrrpip._internal.exceptionsrpip._internal.models.linkr!pip._internal.models.search_scoperpip._internal.network.sessionrpip._internal.network.utilsrpip._internal.utils.filetypesrpip._internal.utils.miscrpip._internal.vcsrsourcesrrrr rl getLoggerr4rUr7ResponseHeadersr* Exceptionr+rBrCrQrXr_r`ror{rrbrrrmrrrrr(r(r(r)s  4             ?        =