perceval/backends/core/twitter.py
def fetch_items(self, category, **kwargs):
        """Fetch the tweets

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        since_id = kwargs['since_id']
        max_id = kwargs['max_id']
        geocode = kwargs['geocode']
        lang = kwargs['lang']
        entities = kwargs['include_entities']
        tweets_type = kwargs['result_type']

        logger.info("Fetching tweets %s from %s to %s",
                    self.query, str(since_id),
                    str(max_id) if max_id else '--')

        tweets_ids = []
        min_date = None
        max_date = None
        group_tweets = self.client.tweets(self.query, since_id=since_id, max_id=max_id, geocode=geocode,
                                          lang=lang, include_entities=entities, result_type=tweets_type)

        for tweets in group_tweets:
            for i in range(len(tweets)):
                tweet = tweets[i]
                tweets_ids.append(tweet['id'])

                if tweets[-1] == tweet:
                    min_date = str_to_datetime(tweets[-1]['created_at'])

                if tweets[0] == tweet and not max_date:
                    max_date = str_to_datetime(tweets[0]['created_at'])

                yield tweet

        logger.info("Fetch process completed: %s (unique %s) tweets fetched, from %s to %s",
                    len(tweets_ids), len(list(set(tweets_ids))), min_date, max_date)
def fetch_items(self, category, **kwargs):
        """Fetch the tweets

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        since_id = kwargs['since_id']
        max_id = kwargs['max_id']
        geocode = kwargs['geocode']
        lang = kwargs['lang']
        entities = kwargs['include_entities']
        tweets_type = kwargs['result_type']

        logger.info("Fetching tweets %s from %s to %s",
                    self.query, str(since_id),
                    str(max_id) if max_id else '--')

        tweets_ids = []
        min_date = None
        max_date = None
        group_tweets = self.client.tweets(self.query, since_id=since_id, max_id=max_id, geocode=geocode,
                                          lang=lang, include_entities=entities, result_type=tweets_type)

        for tweets in group_tweets:
            for i in range(len(tweets)):
                tweet = tweets[i]
                tweets_ids.append(tweet['id'])

                if tweets[-1] == tweet:
                    min_date = str_to_datetime(tweets[-1]['created_at'])

                if tweets[0] == tweet and not max_date:
                    max_date = str_to_datetime(tweets[0]['created_at'])

                yield tweet

        logger.info("Fetch process completed: %s (unique %s) tweets fetched, from %s to %s",
                    len(tweets_ids), len(list(set(tweets_ids))), min_date, max_date)
def fetch_items ( self , category , * * kwargs ) : since_id = kwargs [ 'since_id' ] max_id = kwargs [ 'max_id' ] geocode = kwargs [ 'geocode' ] lang = kwargs [ 'lang' ] entities = kwargs [ 'include_entities' ] tweets_type = kwargs [ 'result_type' ] logger . info ( "Fetching tweets %s from %s to %s" , self . query , str ( since_id ) , str ( max_id ) if max_id else '--' ) tweets_ids = [ ] min_date = None max_date = None group_tweets = self . client . tweets ( self . query , since_id = since_id , max_id = max_id , geocode = geocode , lang = lang , include_entities = entities , result_type = tweets_type ) for tweets in group_tweets : for i in range ( len ( tweets ) ) : tweet = tweets [ i ] tweets_ids . append ( tweet [ 'id' ] ) if tweets [ - 1 ] == tweet : min_date = str_to_datetime ( tweets [ - 1 ] [ 'created_at' ] ) if tweets [ 0 ] == tweet and not max_date : max_date = str_to_datetime ( tweets [ 0 ] [ 'created_at' ] ) yield tweet logger . info ( "Fetch process completed: %s (unique %s) tweets fetched, from %s to %s" , len ( tweets_ids ) , len ( list ( set ( tweets_ids ) ) ) , min_date , max_date )
Fetch the tweets

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
Fetch the tweets