src/toil_lib/urls.py
def _s3am_with_retry(job, num_cores, file_path, s3_url, mode='upload', s3_key_path=None):
    """
    Run s3am with 3 retries

    :param toil.job.Job job: Toil job that is calling this function
    :param int num_cores: Number of cores to pass to upload/download slots
    :param str file_path: Full path to the file
    :param str s3_url: S3 URL
    :param str mode: Mode to run s3am in. Either "upload" or "download"
    :param str s3_key_path: Path to the SSE-C key if using encryption
    """
    container_key_file = None
    # try to find suitable credentials
    base_boto = '.boto'
    base_aws = '.aws/credentials'
    docker_home_dir = '/root'
    # map existing credential paths to their mount point within the container
    credentials_to_mount = {os.path.join(os.path.expanduser("~"), path): os.path.join(docker_home_dir, path)
                            for path in [base_aws, base_boto]
                            if os.path.exists(os.path.join(os.path.expanduser("~"), path))}
    require(os.path.isabs(file_path), "'file_path' parameter must be an absolute path")
    dir_path, file_name = file_path.rsplit('/', 1)
    # Mirror user specified paths to simplify debugging
    container_dir_path = '/data' + dir_path
    container_file = os.path.join(container_dir_path, file_name)
    mounts = {dir_path: container_dir_path}
    if s3_key_path:
        require(os.path.isabs(s3_key_path), "'s3_key_path' parameter must be an absolute path")
        key_dir_path, key_name = s3_key_path.rsplit('/', 1)
        container_key_dir_path = '/data' + key_dir_path
        container_key_file = os.path.join(container_key_dir_path, key_name)
        # if the key directory is identical to the file directory this assignment is idempotent
        mounts[key_dir_path] = container_key_dir_path
    for k, v in credentials_to_mount.iteritems():
        mounts[k] = v
    arguments = []
    url_arguments = []
    if mode == 'upload':
        arguments.extend(['upload', '--force', '--upload-slots=%s' % num_cores, '--exists=overwrite'])
        url_arguments.extend(['file://' + container_file, s3_url])
    elif mode == 'download':
        arguments.extend(['download', '--file-exists=overwrite', '--download-exists=discard'])
        url_arguments.extend([s3_url, 'file://' + container_file])
    else:
        raise ValueError('Improper mode specified. mode must be equal to "upload" or "download".')
    if s3_key_path:
        arguments.extend(['--sse-key-is-master', '--sse-key-file', container_key_file])
    arguments.extend(['--part-size=50M', '--download-slots=%s' % num_cores])
    # finally, add the url path arguments after all the tool parameters are set
    arguments.extend(url_arguments)
    # Pass credential-related environment variables into container
    env = {}
    if 'AWS_PROFILE' in os.environ:
        env['AWS_PROFILE'] = os.environ['AWS_PROFILE']
    # Create parameters to pass to Docker
    docker_parameters = ['--rm', '--log-driver', 'none']
    if mounts:
        for k, v in mounts.iteritems():
            docker_parameters.extend(['-v', k + ':' + v])
    if env:
        for e, v in env.iteritems():
            docker_parameters.extend(['-e', '{}={}'.format(e, v)])
    # Run s3am with retries
    retry_count = 3
    for i in xrange(retry_count):
        try:
            dockerCall(job=job, tool='quay.io/ucsc_cgl/s3am:2.0--fed932897e7fd40f4ec878362e5dd6afe15caaf0',
                       parameters=arguments, dockerParameters=docker_parameters)
        except subprocess.CalledProcessError:
            _log.debug('S3AM %s failed', mode, exc_info=True)
        else:
            _log.debug('S3AM %s succeeded', mode)
            return
    raise RuntimeError("S3AM failed to %s after %i retries with arguments %s. Enable 'debug' "
                       "level logging to see more information about the failed attempts." %
                       (mode, retry_count, arguments))
def _s3am_with_retry(job, num_cores, file_path, s3_url, mode='upload', s3_key_path=None):
    """
    Run s3am with 3 retries

    :param toil.job.Job job: Toil job that is calling this function
    :param int num_cores: Number of cores to pass to upload/download slots
    :param str file_path: Full path to the file
    :param str s3_url: S3 URL
    :param str mode: Mode to run s3am in. Either "upload" or "download"
    :param str s3_key_path: Path to the SSE-C key if using encryption
    """
    container_key_file = None
    # try to find suitable credentials
    base_boto = '.boto'
    base_aws = '.aws/credentials'
    docker_home_dir = '/root'
    # map existing credential paths to their mount point within the container
    credentials_to_mount = {os.path.join(os.path.expanduser("~"), path): os.path.join(docker_home_dir, path)
                            for path in [base_aws, base_boto]
                            if os.path.exists(os.path.join(os.path.expanduser("~"), path))}
    require(os.path.isabs(file_path), "'file_path' parameter must be an absolute path")
    dir_path, file_name = file_path.rsplit('/', 1)
    # Mirror user specified paths to simplify debugging
    container_dir_path = '/data' + dir_path
    container_file = os.path.join(container_dir_path, file_name)
    mounts = {dir_path: container_dir_path}
    if s3_key_path:
        require(os.path.isabs(s3_key_path), "'s3_key_path' parameter must be an absolute path")
        key_dir_path, key_name = s3_key_path.rsplit('/', 1)
        container_key_dir_path = '/data' + key_dir_path
        container_key_file = os.path.join(container_key_dir_path, key_name)
        # if the key directory is identical to the file directory this assignment is idempotent
        mounts[key_dir_path] = container_key_dir_path
    for k, v in credentials_to_mount.iteritems():
        mounts[k] = v
    arguments = []
    url_arguments = []
    if mode == 'upload':
        arguments.extend(['upload', '--force', '--upload-slots=%s' % num_cores, '--exists=overwrite'])
        url_arguments.extend(['file://' + container_file, s3_url])
    elif mode == 'download':
        arguments.extend(['download', '--file-exists=overwrite', '--download-exists=discard'])
        url_arguments.extend([s3_url, 'file://' + container_file])
    else:
        raise ValueError('Improper mode specified. mode must be equal to "upload" or "download".')
    if s3_key_path:
        arguments.extend(['--sse-key-is-master', '--sse-key-file', container_key_file])
    arguments.extend(['--part-size=50M', '--download-slots=%s' % num_cores])
    # finally, add the url path arguments after all the tool parameters are set
    arguments.extend(url_arguments)
    # Pass credential-related environment variables into container
    env = {}
    if 'AWS_PROFILE' in os.environ:
        env['AWS_PROFILE'] = os.environ['AWS_PROFILE']
    # Create parameters to pass to Docker
    docker_parameters = ['--rm', '--log-driver', 'none']
    if mounts:
        for k, v in mounts.iteritems():
            docker_parameters.extend(['-v', k + ':' + v])
    if env:
        for e, v in env.iteritems():
            docker_parameters.extend(['-e', '{}={}'.format(e, v)])
    # Run s3am with retries
    retry_count = 3
    for i in xrange(retry_count):
        try:
            dockerCall(job=job, tool='quay.io/ucsc_cgl/s3am:2.0--fed932897e7fd40f4ec878362e5dd6afe15caaf0',
                       parameters=arguments, dockerParameters=docker_parameters)
        except subprocess.CalledProcessError:
            _log.debug('S3AM %s failed', mode, exc_info=True)
        else:
            _log.debug('S3AM %s succeeded', mode)
            return
    raise RuntimeError("S3AM failed to %s after %i retries with arguments %s. Enable 'debug' "
                       "level logging to see more information about the failed attempts." %
                       (mode, retry_count, arguments))
def _s3am_with_retry ( job , num_cores , file_path , s3_url , mode = 'upload' , s3_key_path = None ) : container_key_file = None # try to find suitable credentials base_boto = '.boto' base_aws = '.aws/credentials' docker_home_dir = '/root' # map existing credential paths to their mount point within the container credentials_to_mount = { os . path . join ( os . path . expanduser ( "~" ) , path ) : os . path . join ( docker_home_dir , path ) for path in [ base_aws , base_boto ] if os . path . exists ( os . path . join ( os . path . expanduser ( "~" ) , path ) ) } require ( os . path . isabs ( file_path ) , "'file_path' parameter must be an absolute path" ) dir_path , file_name = file_path . rsplit ( '/' , 1 ) # Mirror user specified paths to simplify debugging container_dir_path = '/data' + dir_path container_file = os . path . join ( container_dir_path , file_name ) mounts = { dir_path : container_dir_path } if s3_key_path : require ( os . path . isabs ( s3_key_path ) , "'s3_key_path' parameter must be an absolute path" ) key_dir_path , key_name = s3_key_path . rsplit ( '/' , 1 ) container_key_dir_path = '/data' + key_dir_path container_key_file = os . path . join ( container_key_dir_path , key_name ) # if the key directory is identical to the file directory this assignment is idempotent mounts [ key_dir_path ] = container_key_dir_path for k , v in credentials_to_mount . iteritems ( ) : mounts [ k ] = v arguments = [ ] url_arguments = [ ] if mode == 'upload' : arguments . extend ( [ 'upload' , '--force' , '--upload-slots=%s' % num_cores , '--exists=overwrite' ] ) url_arguments . extend ( [ 'file://' + container_file , s3_url ] ) elif mode == 'download' : arguments . extend ( [ 'download' , '--file-exists=overwrite' , '--download-exists=discard' ] ) url_arguments . extend ( [ s3_url , 'file://' + container_file ] ) else : raise ValueError ( 'Improper mode specified. mode must be equal to "upload" or "download".' ) if s3_key_path : arguments . extend ( [ '--sse-key-is-master' , '--sse-key-file' , container_key_file ] ) arguments . extend ( [ '--part-size=50M' , '--download-slots=%s' % num_cores ] ) # finally, add the url path arguments after all the tool parameters are set arguments . extend ( url_arguments ) # Pass credential-related environment variables into container env = { } if 'AWS_PROFILE' in os . environ : env [ 'AWS_PROFILE' ] = os . environ [ 'AWS_PROFILE' ] # Create parameters to pass to Docker docker_parameters = [ '--rm' , '--log-driver' , 'none' ] if mounts : for k , v in mounts . iteritems ( ) : docker_parameters . extend ( [ '-v' , k + ':' + v ] ) if env : for e , v in env . iteritems ( ) : docker_parameters . extend ( [ '-e' , '{}={}' . format ( e , v ) ] ) # Run s3am with retries retry_count = 3 for i in xrange ( retry_count ) : try : dockerCall ( job = job , tool = 'quay.io/ucsc_cgl/s3am:2.0--fed932897e7fd40f4ec878362e5dd6afe15caaf0' , parameters = arguments , dockerParameters = docker_parameters ) except subprocess . CalledProcessError : _log . debug ( 'S3AM %s failed' , mode , exc_info = True ) else : _log . debug ( 'S3AM %s succeeded' , mode ) return raise RuntimeError ( "S3AM failed to %s after %i retries with arguments %s. Enable 'debug' " "level logging to see more information about the failed attempts." % ( mode , retry_count , arguments ) )
Run s3am with 3 retries

    :param toil.job.Job job: Toil job that is calling this function
    :param int num_cores: Number of cores to pass to upload/download slots
    :param str file_path: Full path to the file
    :param str s3_url: S3 URL
    :param str mode: Mode to run s3am in. Either "upload" or "download"
    :param str s3_key_path: Path to the SSE-C key if using encryption
Run s3am with 3 retries