capybara/virtualenv/lib/python2.7/site-packages/setuptools/package_index.py
def process_url(self, url, retrieve=False):
        """Evaluate a URL as a possible download, and maybe retrieve it"""
        if url in self.scanned_urls and not retrieve:
            return
        self.scanned_urls[url] = True
        if not URL_SCHEME(url):
            self.process_filename(url)
            return
        else:
            dists = list(distros_for_url(url))
            if dists:
                if not self.url_ok(url):
                    return
                self.debug("Found link: %s", url)

        if dists or not retrieve or url in self.fetched_urls:
            list(map(self.add, dists))
            return  # don't need the actual page

        if not self.url_ok(url):
            self.fetched_urls[url] = True
            return

        self.info("Reading %s", url)
        self.fetched_urls[url] = True   # prevent multiple fetch attempts
        f = self.open_url(url, "Download error on %s: %%s -- Some packages may not be found!" % url)
        if f is None: return
        self.fetched_urls[f.url] = True
        if 'html' not in f.headers.get('content-type', '').lower():
            f.close()   # not html, we can't process it
            return

        base = f.url     # handle redirects
        page = f.read()
        if not isinstance(page, str): # We are in Python 3 and got bytes. We want str.
            if isinstance(f, HTTPError):
                # Errors have no charset, assume latin1:
                charset = 'latin-1'
            else:
                charset = f.headers.get_param('charset') or 'latin-1'
            page = page.decode(charset, "ignore")
        f.close()
        for match in HREF.finditer(page):
            link = urljoin(base, htmldecode(match.group(1)))
            self.process_url(link)
        if url.startswith(self.index_url) and getattr(f,'code',None)!=404:
            page = self.process_index(url, page)
def process_url(self, url, retrieve=False):
        """Evaluate a URL as a possible download, and maybe retrieve it"""
        if url in self.scanned_urls and not retrieve:
            return
        self.scanned_urls[url] = True
        if not URL_SCHEME(url):
            self.process_filename(url)
            return
        else:
            dists = list(distros_for_url(url))
            if dists:
                if not self.url_ok(url):
                    return
                self.debug("Found link: %s", url)

        if dists or not retrieve or url in self.fetched_urls:
            list(map(self.add, dists))
            return  # don't need the actual page

        if not self.url_ok(url):
            self.fetched_urls[url] = True
            return

        self.info("Reading %s", url)
        self.fetched_urls[url] = True   # prevent multiple fetch attempts
        f = self.open_url(url, "Download error on %s: %%s -- Some packages may not be found!" % url)
        if f is None: return
        self.fetched_urls[f.url] = True
        if 'html' not in f.headers.get('content-type', '').lower():
            f.close()   # not html, we can't process it
            return

        base = f.url     # handle redirects
        page = f.read()
        if not isinstance(page, str): # We are in Python 3 and got bytes. We want str.
            if isinstance(f, HTTPError):
                # Errors have no charset, assume latin1:
                charset = 'latin-1'
            else:
                charset = f.headers.get_param('charset') or 'latin-1'
            page = page.decode(charset, "ignore")
        f.close()
        for match in HREF.finditer(page):
            link = urljoin(base, htmldecode(match.group(1)))
            self.process_url(link)
        if url.startswith(self.index_url) and getattr(f,'code',None)!=404:
            page = self.process_index(url, page)
def process_url ( self , url , retrieve = False ) : if url in self . scanned_urls and not retrieve : return self . scanned_urls [ url ] = True if not URL_SCHEME ( url ) : self . process_filename ( url ) return else : dists = list ( distros_for_url ( url ) ) if dists : if not self . url_ok ( url ) : return self . debug ( "Found link: %s" , url ) if dists or not retrieve or url in self . fetched_urls : list ( map ( self . add , dists ) ) return # don't need the actual page if not self . url_ok ( url ) : self . fetched_urls [ url ] = True return self . info ( "Reading %s" , url ) self . fetched_urls [ url ] = True # prevent multiple fetch attempts f = self . open_url ( url , "Download error on %s: %%s -- Some packages may not be found!" % url ) if f is None : return self . fetched_urls [ f . url ] = True if 'html' not in f . headers . get ( 'content-type' , '' ) . lower ( ) : f . close ( ) # not html, we can't process it return base = f . url # handle redirects page = f . read ( ) if not isinstance ( page , str ) : # We are in Python 3 and got bytes. We want str. if isinstance ( f , HTTPError ) : # Errors have no charset, assume latin1: charset = 'latin-1' else : charset = f . headers . get_param ( 'charset' ) or 'latin-1' page = page . decode ( charset , "ignore" ) f . close ( ) for match in HREF . finditer ( page ) : link = urljoin ( base , htmldecode ( match . group ( 1 ) ) ) self . process_url ( link ) if url . startswith ( self . index_url ) and getattr ( f , 'code' , None ) != 404 : page = self . process_index ( url , page )
Evaluate a URL as a possible download, and maybe retrieve it
Evaluate a URL as a possible download and maybe retrieve it