paved/s3.py
def download_s3(bucket_name, file_key, file_path, force=False):
    """Download a remote file from S3.
    """
    file_path = path(file_path)
    bucket = open_s3(bucket_name)

    file_dir = file_path.dirname()
    file_dir.makedirs()

    s3_key = bucket.get_key(file_key)
    if file_path.exists():
        file_data = file_path.bytes()
        file_md5, file_md5_64 = s3_key.get_md5_from_hexdigest(hashlib.md5(file_data).hexdigest())

        # Check the hash.
        try:
            s3_md5 = s3_key.etag.replace('"', '')
        except KeyError:
            pass
        else:
            if s3_md5 == file_md5:
                info('Hash is the same. Skipping %s' % file_path)
                return

            elif not force:
                # Check if file on S3 is older than local file.
                s3_datetime = datetime.datetime(*time.strptime(
                    s3_key.last_modified, '%a, %d %b %Y %H:%M:%S %Z')[0:6])
                local_datetime = datetime.datetime.utcfromtimestamp(file_path.stat().st_mtime)
                if s3_datetime < local_datetime:
                    info("File at %s is less recent than the local version." % (file_key))
                    return

    # If it is newer, let's process and upload
    info("Downloading %s..." % (file_key))

    try:
        with open(file_path, 'w') as fo:
            s3_key.get_contents_to_file(fo)
    except Exception as e:
        error("Failed: %s" % e)
        raise
def download_s3(bucket_name, file_key, file_path, force=False):
    """Download a remote file from S3.
    """
    file_path = path(file_path)
    bucket = open_s3(bucket_name)

    file_dir = file_path.dirname()
    file_dir.makedirs()

    s3_key = bucket.get_key(file_key)
    if file_path.exists():
        file_data = file_path.bytes()
        file_md5, file_md5_64 = s3_key.get_md5_from_hexdigest(hashlib.md5(file_data).hexdigest())

        # Check the hash.
        try:
            s3_md5 = s3_key.etag.replace('"', '')
        except KeyError:
            pass
        else:
            if s3_md5 == file_md5:
                info('Hash is the same. Skipping %s' % file_path)
                return

            elif not force:
                # Check if file on S3 is older than local file.
                s3_datetime = datetime.datetime(*time.strptime(
                    s3_key.last_modified, '%a, %d %b %Y %H:%M:%S %Z')[0:6])
                local_datetime = datetime.datetime.utcfromtimestamp(file_path.stat().st_mtime)
                if s3_datetime < local_datetime:
                    info("File at %s is less recent than the local version." % (file_key))
                    return

    # If it is newer, let's process and upload
    info("Downloading %s..." % (file_key))

    try:
        with open(file_path, 'w') as fo:
            s3_key.get_contents_to_file(fo)
    except Exception as e:
        error("Failed: %s" % e)
        raise
def download_s3 ( bucket_name , file_key , file_path , force = False ) : file_path = path ( file_path ) bucket = open_s3 ( bucket_name ) file_dir = file_path . dirname ( ) file_dir . makedirs ( ) s3_key = bucket . get_key ( file_key ) if file_path . exists ( ) : file_data = file_path . bytes ( ) file_md5 , file_md5_64 = s3_key . get_md5_from_hexdigest ( hashlib . md5 ( file_data ) . hexdigest ( ) ) # Check the hash. try : s3_md5 = s3_key . etag . replace ( '"' , '' ) except KeyError : pass else : if s3_md5 == file_md5 : info ( 'Hash is the same. Skipping %s' % file_path ) return elif not force : # Check if file on S3 is older than local file. s3_datetime = datetime . datetime ( * time . strptime ( s3_key . last_modified , '%a, %d %b %Y %H:%M:%S %Z' ) [ 0 : 6 ] ) local_datetime = datetime . datetime . utcfromtimestamp ( file_path . stat ( ) . st_mtime ) if s3_datetime < local_datetime : info ( "File at %s is less recent than the local version." % ( file_key ) ) return # If it is newer, let's process and upload info ( "Downloading %s..." % ( file_key ) ) try : with open ( file_path , 'w' ) as fo : s3_key . get_contents_to_file ( fo ) except Exception as e : error ( "Failed: %s" % e ) raise
Download a remote file from S3.
Download a remote file from S3 .