def get_from_cache(url: str, cache_dir: Path = None) -> Path:
    """
    Given a URL, look for the corresponding dataset in the local cache.
    If it's not there, download it. Then return the path to the cached file.
    """
    cache_dir.mkdir(parents=True, exist_ok=True)

    filename = re.sub(r'.+/', '', url)
    # get cache path to put the file
    cache_path = cache_dir / filename
    if cache_path.exists():
        return cache_path

    # make HEAD request to check ETag
    response = requests.head(url)

    if response.status_code != 200:
        if "www.dropbox.com" in url:
            # dropbox return code 301, so we ignore this error
            pass
        else:
            raise IOError("HEAD request failed for url {}".format(url))

    # add ETag to filename if it exists
    # etag = response.headers.get("ETag")

    if not cache_path.exists():
        # Download to temporary file, then copy to cache dir once finished.
        # Otherwise you get corrupt cache entries if the download gets interrupted.
        fd, temp_filename = tempfile.mkstemp()
        logger.info("%s not found in cache, downloading to %s", url, temp_filename)

        # GET file object
        req = requests.get(url, stream=True)
        content_length = req.headers.get('Content-Length')
        total = int(content_length) if content_length is not None else None
        progress = Tqdm.tqdm(unit="B", total=total)
        with open(temp_filename, 'wb') as temp_file:
            for chunk in req.iter_content(chunk_size=1024):
                if chunk: # filter out keep-alive new chunks
                    progress.update(len(chunk))
                    temp_file.write(chunk)

        progress.close()

        logger.info("copying %s to cache at %s", temp_filename, cache_path)
        shutil.copyfile(temp_filename, str(cache_path))
        logger.info("removing temp file %s", temp_filename)
        os.close(fd)
        os.remove(temp_filename)

    return cache_path
def get_from_cache(url: str, cache_dir: Path = None) -> Path:
    """
    Given a URL, look for the corresponding dataset in the local cache.
    If it's not there, download it. Then return the path to the cached file.
    """
    cache_dir.mkdir(parents=True, exist_ok=True)

    filename = re.sub(r'.+/', '', url)
    # get cache path to put the file
    cache_path = cache_dir / filename
    if cache_path.exists():
        return cache_path

    # make HEAD request to check ETag
    response = requests.head(url)

    if response.status_code != 200:
        if "www.dropbox.com" in url:
            # dropbox return code 301, so we ignore this error
            pass
        else:
            raise IOError("HEAD request failed for url {}".format(url))

    # add ETag to filename if it exists
    # etag = response.headers.get("ETag")

    if not cache_path.exists():
        # Download to temporary file, then copy to cache dir once finished.
        # Otherwise you get corrupt cache entries if the download gets interrupted.
        fd, temp_filename = tempfile.mkstemp()
        logger.info("%s not found in cache, downloading to %s", url, temp_filename)

        # GET file object
        req = requests.get(url, stream=True)
        content_length = req.headers.get('Content-Length')
        total = int(content_length) if content_length is not None else None
        progress = Tqdm.tqdm(unit="B", total=total)
        with open(temp_filename, 'wb') as temp_file:
            for chunk in req.iter_content(chunk_size=1024):
                if chunk: # filter out keep-alive new chunks
                    progress.update(len(chunk))
                    temp_file.write(chunk)

        progress.close()

        logger.info("copying %s to cache at %s", temp_filename, cache_path)
        shutil.copyfile(temp_filename, str(cache_path))
        logger.info("removing temp file %s", temp_filename)
        os.close(fd)
        os.remove(temp_filename)

    return cache_path
def get_from_cache ( url : str , cache_dir : Path = None ) -> Path : cache_dir . mkdir ( parents = True , exist_ok = True ) filename = re . sub ( r'.+/' , '' , url ) # get cache path to put the file cache_path = cache_dir / filename if cache_path . exists ( ) : return cache_path # make HEAD request to check ETag response = requests . head ( url ) if response . status_code != 200 : if "www.dropbox.com" in url : # dropbox return code 301, so we ignore this error pass else : raise IOError ( "HEAD request failed for url {}" . format ( url ) ) # add ETag to filename if it exists # etag = response.headers.get("ETag") if not cache_path . exists ( ) : # Download to temporary file, then copy to cache dir once finished. # Otherwise you get corrupt cache entries if the download gets interrupted. fd , temp_filename = tempfile . mkstemp ( ) logger . info ( "%s not found in cache, downloading to %s" , url , temp_filename ) # GET file object req = requests . get ( url , stream = True ) content_length = req . headers . get ( 'Content-Length' ) total = int ( content_length ) if content_length is not None else None progress = Tqdm . tqdm ( unit = "B" , total = total ) with open ( temp_filename , 'wb' ) as temp_file : for chunk in req . iter_content ( chunk_size = 1024 ) : if chunk : # filter out keep-alive new chunks progress . update ( len ( chunk ) ) temp_file . write ( chunk ) progress . close ( ) logger . info ( "copying %s to cache at %s" , temp_filename , cache_path ) shutil . copyfile ( temp_filename , str ( cache_path ) ) logger . info ( "removing temp file %s" , temp_filename ) os . close ( fd ) os . remove ( temp_filename ) return cache_path
Given a URL, look for the corresponding dataset in the local cache.
    If it's not there, download it. Then return the path to the cached file.
Given a URL look for the corresponding dataset in the local cache . If it s not there download it . Then return the path to the cached file .