sirmordred/sirmordred.py
def start(self):
        """
        This method defines the workflow of SirMordred. So it calls to:
        - initialize the databases
        - execute the different phases for the first iteration
          (collection, identities, enrichment)
        - start the collection and enrichment in parallel by data source
        - start also the Sorting Hat merge
        """

        # logger.debug("Starting SirMordred engine ...")
        logger.info("")
        logger.info("----------------------------")
        logger.info("Starting SirMordred engine ...")
        logger.info("- - - - - - - - - - - - - - ")

        # check we have access to the needed ES
        if not self.check_es_access():
            print('Can not access Elasticsearch service. Exiting sirmordred ...')
            sys.exit(1)

        # If arthur is configured check that it is working
        if self.conf['es_collection']['arthur']:
            if not self.check_redis_access():
                print('Can not access redis service. Exiting sirmordred ...')
                sys.exit(1)
            if not self.check_arthur_access():
                print('Can not access arthur service. Exiting sirmordred ...')
                sys.exit(1)

        # If bestiary is configured check that it is working
        if self.conf['projects']['projects_url']:
            if not self.check_bestiary_access():
                print('Can not access bestiary service. Exiting sirmordred ...')
                sys.exit(1)

        # Initial round: panels and projects loading
        self.__execute_initial_load()

        # Tasks to be executed during updating process
        all_tasks_cls = []
        all_tasks_cls.append(TaskProjects)  # projects update is always needed
        if self.conf['phases']['collection']:
            if not self.conf['es_collection']['arthur']:
                all_tasks_cls.append(TaskRawDataCollection)
            else:
                all_tasks_cls.append(TaskRawDataArthurCollection)
        if self.conf['phases']['identities']:
            # load identities and orgs periodically for updates
            all_tasks_cls.append(TaskIdentitiesLoad)
            all_tasks_cls.append(TaskIdentitiesMerge)
            all_tasks_cls.append(TaskIdentitiesExport)
            # This is done in enrichement before doing the enrich
            # if self.conf['phases']['collection']:
            #     all_tasks_cls.append(TaskIdentitiesCollection)
        if self.conf['phases']['enrichment']:
            all_tasks_cls.append(TaskEnrich)
        if self.conf['phases']['track_items']:
            all_tasks_cls.append(TaskTrackItems)
        if self.conf['phases']['report']:
            all_tasks_cls.append(TaskReport)

        # this is the main loop, where the execution should spend
        # most of its time

        while True:

            if not all_tasks_cls:
                logger.warning("No tasks to execute.")
                break

            try:
                if not self.conf['general']['update']:
                    self.execute_batch_tasks(all_tasks_cls,
                                             self.conf['sortinghat']['sleep_for'],
                                             self.conf['general']['min_update_delay'])
                    self.execute_batch_tasks(all_tasks_cls,
                                             self.conf['sortinghat']['sleep_for'],
                                             self.conf['general']['min_update_delay'])
                    break
                else:
                    self.execute_nonstop_tasks(all_tasks_cls)

                # FIXME this point is never reached so despite the exception is
                # handled and the error is shown, the traceback is not printed

            except DataCollectionError as e:
                logger.error(str(e))
                var = traceback.format_exc()
                logger.error(var)

            except DataEnrichmentError as e:
                logger.error(str(e))
                var = traceback.format_exc()
                logger.error(var)

        logger.info("Finished SirMordred engine ...")
def start(self):
        """
        This method defines the workflow of SirMordred. So it calls to:
        - initialize the databases
        - execute the different phases for the first iteration
          (collection, identities, enrichment)
        - start the collection and enrichment in parallel by data source
        - start also the Sorting Hat merge
        """

        # logger.debug("Starting SirMordred engine ...")
        logger.info("")
        logger.info("----------------------------")
        logger.info("Starting SirMordred engine ...")
        logger.info("- - - - - - - - - - - - - - ")

        # check we have access to the needed ES
        if not self.check_es_access():
            print('Can not access Elasticsearch service. Exiting sirmordred ...')
            sys.exit(1)

        # If arthur is configured check that it is working
        if self.conf['es_collection']['arthur']:
            if not self.check_redis_access():
                print('Can not access redis service. Exiting sirmordred ...')
                sys.exit(1)
            if not self.check_arthur_access():
                print('Can not access arthur service. Exiting sirmordred ...')
                sys.exit(1)

        # If bestiary is configured check that it is working
        if self.conf['projects']['projects_url']:
            if not self.check_bestiary_access():
                print('Can not access bestiary service. Exiting sirmordred ...')
                sys.exit(1)

        # Initial round: panels and projects loading
        self.__execute_initial_load()

        # Tasks to be executed during updating process
        all_tasks_cls = []
        all_tasks_cls.append(TaskProjects)  # projects update is always needed
        if self.conf['phases']['collection']:
            if not self.conf['es_collection']['arthur']:
                all_tasks_cls.append(TaskRawDataCollection)
            else:
                all_tasks_cls.append(TaskRawDataArthurCollection)
        if self.conf['phases']['identities']:
            # load identities and orgs periodically for updates
            all_tasks_cls.append(TaskIdentitiesLoad)
            all_tasks_cls.append(TaskIdentitiesMerge)
            all_tasks_cls.append(TaskIdentitiesExport)
            # This is done in enrichement before doing the enrich
            # if self.conf['phases']['collection']:
            #     all_tasks_cls.append(TaskIdentitiesCollection)
        if self.conf['phases']['enrichment']:
            all_tasks_cls.append(TaskEnrich)
        if self.conf['phases']['track_items']:
            all_tasks_cls.append(TaskTrackItems)
        if self.conf['phases']['report']:
            all_tasks_cls.append(TaskReport)

        # this is the main loop, where the execution should spend
        # most of its time

        while True:

            if not all_tasks_cls:
                logger.warning("No tasks to execute.")
                break

            try:
                if not self.conf['general']['update']:
                    self.execute_batch_tasks(all_tasks_cls,
                                             self.conf['sortinghat']['sleep_for'],
                                             self.conf['general']['min_update_delay'])
                    self.execute_batch_tasks(all_tasks_cls,
                                             self.conf['sortinghat']['sleep_for'],
                                             self.conf['general']['min_update_delay'])
                    break
                else:
                    self.execute_nonstop_tasks(all_tasks_cls)

                # FIXME this point is never reached so despite the exception is
                # handled and the error is shown, the traceback is not printed

            except DataCollectionError as e:
                logger.error(str(e))
                var = traceback.format_exc()
                logger.error(var)

            except DataEnrichmentError as e:
                logger.error(str(e))
                var = traceback.format_exc()
                logger.error(var)

        logger.info("Finished SirMordred engine ...")
def start ( self ) : # logger.debug("Starting SirMordred engine ...") logger . info ( "" ) logger . info ( "----------------------------" ) logger . info ( "Starting SirMordred engine ..." ) logger . info ( "- - - - - - - - - - - - - - " ) # check we have access to the needed ES if not self . check_es_access ( ) : print ( 'Can not access Elasticsearch service. Exiting sirmordred ...' ) sys . exit ( 1 ) # If arthur is configured check that it is working if self . conf [ 'es_collection' ] [ 'arthur' ] : if not self . check_redis_access ( ) : print ( 'Can not access redis service. Exiting sirmordred ...' ) sys . exit ( 1 ) if not self . check_arthur_access ( ) : print ( 'Can not access arthur service. Exiting sirmordred ...' ) sys . exit ( 1 ) # If bestiary is configured check that it is working if self . conf [ 'projects' ] [ 'projects_url' ] : if not self . check_bestiary_access ( ) : print ( 'Can not access bestiary service. Exiting sirmordred ...' ) sys . exit ( 1 ) # Initial round: panels and projects loading self . __execute_initial_load ( ) # Tasks to be executed during updating process all_tasks_cls = [ ] all_tasks_cls . append ( TaskProjects ) # projects update is always needed if self . conf [ 'phases' ] [ 'collection' ] : if not self . conf [ 'es_collection' ] [ 'arthur' ] : all_tasks_cls . append ( TaskRawDataCollection ) else : all_tasks_cls . append ( TaskRawDataArthurCollection ) if self . conf [ 'phases' ] [ 'identities' ] : # load identities and orgs periodically for updates all_tasks_cls . append ( TaskIdentitiesLoad ) all_tasks_cls . append ( TaskIdentitiesMerge ) all_tasks_cls . append ( TaskIdentitiesExport ) # This is done in enrichement before doing the enrich # if self.conf['phases']['collection']: #     all_tasks_cls.append(TaskIdentitiesCollection) if self . conf [ 'phases' ] [ 'enrichment' ] : all_tasks_cls . append ( TaskEnrich ) if self . conf [ 'phases' ] [ 'track_items' ] : all_tasks_cls . append ( TaskTrackItems ) if self . conf [ 'phases' ] [ 'report' ] : all_tasks_cls . append ( TaskReport ) # this is the main loop, where the execution should spend # most of its time while True : if not all_tasks_cls : logger . warning ( "No tasks to execute." ) break try : if not self . conf [ 'general' ] [ 'update' ] : self . execute_batch_tasks ( all_tasks_cls , self . conf [ 'sortinghat' ] [ 'sleep_for' ] , self . conf [ 'general' ] [ 'min_update_delay' ] ) self . execute_batch_tasks ( all_tasks_cls , self . conf [ 'sortinghat' ] [ 'sleep_for' ] , self . conf [ 'general' ] [ 'min_update_delay' ] ) break else : self . execute_nonstop_tasks ( all_tasks_cls ) # FIXME this point is never reached so despite the exception is # handled and the error is shown, the traceback is not printed except DataCollectionError as e : logger . error ( str ( e ) ) var = traceback . format_exc ( ) logger . error ( var ) except DataEnrichmentError as e : logger . error ( str ( e ) ) var = traceback . format_exc ( ) logger . error ( var ) logger . info ( "Finished SirMordred engine ..." )
This method defines the workflow of SirMordred. So it calls to:
        - initialize the databases
        - execute the different phases for the first iteration
          (collection, identities, enrichment)
        - start the collection and enrichment in parallel by data source
        - start also the Sorting Hat merge
This method defines the workflow of SirMordred . So it calls to : - initialize the databases - execute the different phases for the first iteration ( collection identities enrichment ) - start the collection and enrichment in parallel by data source - start also the Sorting Hat merge