deeppavlov/core/data/utils.py
def download(dest_file_path: [List[Union[str, Path]]], source_url: str, force_download=True):
    """Download a file from URL to one or several target locations

    Args:
        dest_file_path: path or list of paths to the file destination files (including file name)
        source_url: the source URL
        force_download: download file if it already exists, or not

    """

    if isinstance(dest_file_path, list):
        dest_file_paths = [Path(path) for path in dest_file_path]
    else:
        dest_file_paths = [Path(dest_file_path).absolute()]

    if not force_download:
        to_check = list(dest_file_paths)
        dest_file_paths = []
        for p in to_check:
            if p.exists():
                log.info(f'File already exists in {p}')
            else:
                dest_file_paths.append(p)

    if dest_file_paths:
        cache_dir = os.getenv('DP_CACHE_DIR')
        cached_exists = False
        if cache_dir:
            first_dest_path = Path(cache_dir) / md5(source_url.encode('utf8')).hexdigest()[:15]
            cached_exists = first_dest_path.exists()
        else:
            first_dest_path = dest_file_paths.pop()

        if not cached_exists:
            first_dest_path.parent.mkdir(parents=True, exist_ok=True)

            simple_download(source_url, first_dest_path)
        else:
            log.info(f'Found cached {source_url} in {first_dest_path}')

        for dest_path in dest_file_paths:
            dest_path.parent.mkdir(parents=True, exist_ok=True)
            shutil.copy(str(first_dest_path), str(dest_path))
def download(dest_file_path: [List[Union[str, Path]]], source_url: str, force_download=True):
    """Download a file from URL to one or several target locations

    Args:
        dest_file_path: path or list of paths to the file destination files (including file name)
        source_url: the source URL
        force_download: download file if it already exists, or not

    """

    if isinstance(dest_file_path, list):
        dest_file_paths = [Path(path) for path in dest_file_path]
    else:
        dest_file_paths = [Path(dest_file_path).absolute()]

    if not force_download:
        to_check = list(dest_file_paths)
        dest_file_paths = []
        for p in to_check:
            if p.exists():
                log.info(f'File already exists in {p}')
            else:
                dest_file_paths.append(p)

    if dest_file_paths:
        cache_dir = os.getenv('DP_CACHE_DIR')
        cached_exists = False
        if cache_dir:
            first_dest_path = Path(cache_dir) / md5(source_url.encode('utf8')).hexdigest()[:15]
            cached_exists = first_dest_path.exists()
        else:
            first_dest_path = dest_file_paths.pop()

        if not cached_exists:
            first_dest_path.parent.mkdir(parents=True, exist_ok=True)

            simple_download(source_url, first_dest_path)
        else:
            log.info(f'Found cached {source_url} in {first_dest_path}')

        for dest_path in dest_file_paths:
            dest_path.parent.mkdir(parents=True, exist_ok=True)
            shutil.copy(str(first_dest_path), str(dest_path))
def download ( dest_file_path : [ List [ Union [ str , Path ] ] ] , source_url : str , force_download = True ) : if isinstance ( dest_file_path , list ) : dest_file_paths = [ Path ( path ) for path in dest_file_path ] else : dest_file_paths = [ Path ( dest_file_path ) . absolute ( ) ] if not force_download : to_check = list ( dest_file_paths ) dest_file_paths = [ ] for p in to_check : if p . exists ( ) : log . info ( f'File already exists in {p}' ) else : dest_file_paths . append ( p ) if dest_file_paths : cache_dir = os . getenv ( 'DP_CACHE_DIR' ) cached_exists = False if cache_dir : first_dest_path = Path ( cache_dir ) / md5 ( source_url . encode ( 'utf8' ) ) . hexdigest ( ) [ : 15 ] cached_exists = first_dest_path . exists ( ) else : first_dest_path = dest_file_paths . pop ( ) if not cached_exists : first_dest_path . parent . mkdir ( parents = True , exist_ok = True ) simple_download ( source_url , first_dest_path ) else : log . info ( f'Found cached {source_url} in {first_dest_path}' ) for dest_path in dest_file_paths : dest_path . parent . mkdir ( parents = True , exist_ok = True ) shutil . copy ( str ( first_dest_path ) , str ( dest_path ) )
Download a file from URL to one or several target locations

    Args:
        dest_file_path: path or list of paths to the file destination files (including file name)
        source_url: the source URL
        force_download: download file if it already exists, or not
Download a file from URL to one or several target locations