s3pipeline/pipelines.py
def _upload_chunk(self, spider):
        """
        Do upload items to S3.
        """

        if not self.items:
            return  # Do nothing when items is empty.

        f = self._make_fileobj()

        # Build object key by replacing variables in object key template.
        object_key = self.object_key_template.format(**self._get_uri_params(spider))

        try:
            self.s3.upload_fileobj(f, self.bucket_name, object_key)
        except ClientError:
            self.stats.inc_value('pipeline/s3/fail')
            raise
        else:
            self.stats.inc_value('pipeline/s3/success')
        finally:
            # Prepare for the next chunk
            self.chunk_number += len(self.items)
            self.items = []
def _upload_chunk(self, spider):
        """
        Do upload items to S3.
        """

        if not self.items:
            return  # Do nothing when items is empty.

        f = self._make_fileobj()

        # Build object key by replacing variables in object key template.
        object_key = self.object_key_template.format(**self._get_uri_params(spider))

        try:
            self.s3.upload_fileobj(f, self.bucket_name, object_key)
        except ClientError:
            self.stats.inc_value('pipeline/s3/fail')
            raise
        else:
            self.stats.inc_value('pipeline/s3/success')
        finally:
            # Prepare for the next chunk
            self.chunk_number += len(self.items)
            self.items = []
def _upload_chunk ( self , spider ) : if not self . items : return # Do nothing when items is empty. f = self . _make_fileobj ( ) # Build object key by replacing variables in object key template. object_key = self . object_key_template . format ( * * self . _get_uri_params ( spider ) ) try : self . s3 . upload_fileobj ( f , self . bucket_name , object_key ) except ClientError : self . stats . inc_value ( 'pipeline/s3/fail' ) raise else : self . stats . inc_value ( 'pipeline/s3/success' ) finally : # Prepare for the next chunk self . chunk_number += len ( self . items ) self . items = [ ]
Do upload items to S3.
Do upload items to S3 .