sirmordred/config.py
def general_params(cls):
        """ Define all the possible config params """

        params = {}

        # GENERAL CONFIG
        params_general = {
            "general": {
                "min_update_delay": {
                    "optional": True,
                    "default": 60,
                    "type": int,
                    "description": "Short delay between tasks (collect, enrich ...)"
                },
                "update": {
                    "optional": False,
                    "default": False,
                    "type": bool,
                    "description": "Execute the tasks in loop"
                },
                "short_name": {
                    "optional": False,
                    "default": "Short name",
                    "type": str,
                    "description": "Short name of the project"
                },
                "debug": {
                    "optional": False,
                    "default": True,
                    "type": bool,
                    "description": "Debug mode (logging mainly)"
                },
                "logs_dir": {
                    "optional": False,
                    "default": "logs",
                    "type": str,
                    "description": "Directory with the logs of sirmordred"
                },
                "log_handler": {
                    "optional": True,
                    "default": "file",
                    "type": str,
                    "description": "use rotate for rotating the logs automatically"
                },
                "log_max_bytes": {
                    "optional": True,
                    "default": 104857600,  # 100MB
                    "type": int,
                    "description": "Max number of bytes per log file"
                },
                "log_backup_count": {
                    "optional": True,
                    "default": 5,
                    "type": int,
                    "description": "Number of rotate logs files to preserve"
                },
                "bulk_size": {
                    "optional": True,
                    "default": 1000,
                    "type": int,
                    "description": "Number of items to write in Elasticsearch using bulk operations"
                },
                "scroll_size": {
                    "optional": True,
                    "default": 100,
                    "type": int,
                    "description": "Number of items to read from Elasticsearch when scrolling"
                },
                "aliases_file": {
                    "optional": True,
                    "default": ALIASES_JSON,
                    "type": str,
                    "description": "JSON file to define aliases for raw and enriched indexes"
                },
                "menu_file": {
                    "optional": True,
                    "default": MENU_YAML,
                    "type": str,
                    "description": "YAML file to define the menus to be shown in Kibiter"
                },
                "retention_time": {
                    "optional": True,
                    "default": None,
                    "type": int,
                    "description": "The maximum number of minutes wrt the current date to retain the data"
                }
            }
        }
        params_projects = {
            "projects": {
                "projects_file": {
                    "optional": True,
                    "default": PROJECTS_JSON,
                    "type": str,
                    "description": "Projects file path with repositories to be collected group by projects"
                },
                "projects_url": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "Projects file URL"
                },
                "load_eclipse": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Load the projects from Eclipse"
                }
            }
        }

        params_phases = {
            "phases": {
                "collection": {
                    "optional": False,
                    "default": True,
                    "type": bool,
                    "description": "Activate collection of items"
                },
                "enrichment": {
                    "optional": False,
                    "default": True,
                    "type": bool,
                    "description": "Activate enrichment of items"
                },
                "identities": {
                    "optional": False,
                    "default": True,
                    "type": bool,
                    "description": "Do the identities tasks"
                },
                "panels": {
                    "optional": False,
                    "default": True,
                    "type": bool,
                    "description": "Load panels, create alias and other tasks related"
                },
                "track_items": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Track specific items from a gerrit repository"
                },
                "report": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Generate the PDF report for a project (alpha)"
                }
            }
        }

        general_config_params = [params_general, params_projects, params_phases]

        for section_params in general_config_params:
            params.update(section_params)

        # Config provided by tasks
        params_collection = {
            "es_collection": {
                "password": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "Password for connection to Elasticsearch"
                },
                "user": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "User for connection to Elasticsearch"
                },
                "url": {
                    "optional": False,
                    "default": "http://172.17.0.1:9200",
                    "type": str,
                    "description": "Elasticsearch URL"
                },
                "arthur": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Use arthur for collecting items from perceval"
                },
                "arthur_url": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "URL for the arthur service"
                },
                "redis_url": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "URL for the redis service"
                }
            }
        }

        params_enrichment = {
            "es_enrichment": {
                "url": {
                    "optional": False,
                    "default": "http://172.17.0.1:9200",
                    "type": str,
                    "description": "Elasticsearch URL"
                },
                "autorefresh": {
                    "optional": True,
                    "default": True,
                    "type": bool,
                    "description": "Execute the autorefresh of identities"
                },
                "autorefresh_interval": {
                    "optional": True,
                    "default": 2,
                    "type": int,
                    "description": "Set time interval (days) for autorefresh identities"
                },
                "user": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "User for connection to Elasticsearch"
                },
                "password": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "Password for connection to Elasticsearch"
                }
            }
        }

        params_panels = {
            "panels": {
                "strict": {
                    "optional": True,
                    "default": True,
                    "type": bool,
                    "description": "Enable strict panels loading"
                },
                "kibiter_time_from": {
                    "optional": True,
                    "default": "now-90d",
                    "type": str,
                    "description": "Default time interval for Kibiter"
                },
                "kibiter_default_index": {
                    "optional": True,
                    "default": "git",
                    "type": str,
                    "description": "Default index pattern for Kibiter"
                },
                "kibiter_url": {
                    "optional": False,
                    "default": None,
                    "type": str,
                    "description": "Kibiter URL"
                },
                "kibiter_version": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "Kibiter version"
                },
                "community": {
                    "optional": True,
                    "default": True,
                    "type": bool,
                    "description": "Enable community structure menu"
                },
                "kafka": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Enable kafka menu"
                },
                "github-repos": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Enable GitHub repo stats menu"
                },
                "gitlab-issues": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Enable GitLab issues menu"
                },
                "gitlab-merges": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Enable GitLab merge requests menu"
                },
                "mattermost": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Enable Mattermost menu"
                }
            }
        }

        params_report = {
            "report": {
                "start_date": {
                    "optional": False,
                    "default": "1970-01-01",
                    "type": str,
                    "description": "Start date for the report"
                },
                "end_date": {
                    "optional": False,
                    "default": "2100-01-01",
                    "type": str,
                    "description": "End date for the report"
                },
                "interval": {
                    "optional": False,
                    "default": "quarter",
                    "type": str,
                    "description": "Interval for the report"
                },
                "config_file": {
                    "optional": False,
                    "default": "report.cfg",
                    "type": str,
                    "description": "Config file for the report"
                },
                "data_dir": {
                    "optional": False,
                    "default": "report_data",
                    "type": str,
                    "description": "Directory in which to store the report data"
                },
                "filters": {
                    "optional": True,
                    "default": [],
                    "type": list,
                    "description": "General filters to be applied to all queries"
                },
                "offset": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "Date offset to be applied to start and end"
                }
            }
        }

        params_sortinghat = {
            "sortinghat": {
                "affiliate": {
                    "optional": False,
                    "default": "True",
                    "type": bool,
                    "description": "Affiliate identities to organizations"
                },
                "unaffiliated_group": {
                    "optional": False,
                    "default": "Unknown",
                    "type": str,
                    "description": "Name of the organization for unaffiliated identities"
                },
                "matching": {
                    "optional": False,
                    "default": ["email"],
                    "type": list,
                    "description": "Algorithm for matching identities in Sortinghat"
                },
                "sleep_for": {
                    "optional": False,
                    "default": 3600,
                    "type": int,
                    "description": "Delay between task identities executions"
                },
                "database": {
                    "optional": False,
                    "default": "sortinghat_db",
                    "type": str,
                    "description": "Name of the Sortinghat database"
                },
                "host": {
                    "optional": False,
                    "default": "mariadb",
                    "type": str,
                    "description": "Host with the Sortinghat database"
                },
                "user": {
                    "optional": False,
                    "default": "root",
                    "type": str,
                    "description": "User to access the Sortinghat database"
                },
                "password": {
                    "optional": False,
                    "default": "",
                    "type": str,
                    "description": "Password to access the Sortinghat database"
                },
                "autoprofile": {
                    "optional": False,
                    "default": ["customer", "git", "github"],
                    "type": list,
                    "description": "Order in which to get the identities information for filling the profile"
                },
                "load_orgs": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "deprecated": "Load organizations in Sortinghat database",
                    "description": ""
                },
                "identities_format": {
                    "optional": True,
                    "default": "sortinghat",
                    "type": str,
                    "description": "Format of the identities data to be loaded"
                },
                "strict_mapping": {
                    "optional": True,
                    "default": True,
                    "type": bool,
                    "description": "rigorous check of values in identities matching "
                                   "(i.e, well formed email addresses)"
                },
                "reset_on_load": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Unmerge and remove affiliations for all identities on load"
                },
                "orgs_file": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "File path with the organizations to be loaded in Sortinghat"
                },
                "identities_file": {
                    "optional": True,
                    "default": [],
                    "type": list,
                    "description": "File path with the identities to be loaded in Sortinghat"
                },
                "identities_export_url": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "URL in which to export the identities in Sortinghat"
                },
                "identities_api_token": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "API token for remote operation with GitHub and Gitlab"
                },
                "bots_names": {
                    "optional": True,
                    "default": [],
                    "type": list,
                    "description": "Name of the identities to be marked as bots"
                },
                "no_bots_names": {
                    "optional": True,
                    "default": [],
                    "type": list,
                    "description": "Name of the identities to be unmarked as bots"
                },
                "autogender": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Add gender to the profiles (executes autogender)"
                }
            }
        }

        params_track_items = {
            "track_items": {
                "project": {
                    "optional": False,
                    "default": "TrackProject",
                    "type": str,
                    "description": "Gerrit project to track"
                },
                "upstream_raw_es_url": {
                    "optional": False,
                    "default": "",
                    "type": str,
                    "description": "URL with the file with the gerrit reviews to track"
                },
                "raw_index_gerrit": {
                    "optional": False,
                    "default": "",
                    "type": str,
                    "description": "Name of the gerrit raw index"
                },
                "raw_index_git": {
                    "optional": False,
                    "default": "",
                    "type": str,
                    "description": "Name of the git raw index"
                }
            }
        }

        tasks_config_params = [params_collection, params_enrichment, params_panels,
                               params_report, params_sortinghat, params_track_items]

        for section_params in tasks_config_params:
            params.update(section_params)

        return params
def general_params(cls):
        """ Define all the possible config params """

        params = {}

        # GENERAL CONFIG
        params_general = {
            "general": {
                "min_update_delay": {
                    "optional": True,
                    "default": 60,
                    "type": int,
                    "description": "Short delay between tasks (collect, enrich ...)"
                },
                "update": {
                    "optional": False,
                    "default": False,
                    "type": bool,
                    "description": "Execute the tasks in loop"
                },
                "short_name": {
                    "optional": False,
                    "default": "Short name",
                    "type": str,
                    "description": "Short name of the project"
                },
                "debug": {
                    "optional": False,
                    "default": True,
                    "type": bool,
                    "description": "Debug mode (logging mainly)"
                },
                "logs_dir": {
                    "optional": False,
                    "default": "logs",
                    "type": str,
                    "description": "Directory with the logs of sirmordred"
                },
                "log_handler": {
                    "optional": True,
                    "default": "file",
                    "type": str,
                    "description": "use rotate for rotating the logs automatically"
                },
                "log_max_bytes": {
                    "optional": True,
                    "default": 104857600,  # 100MB
                    "type": int,
                    "description": "Max number of bytes per log file"
                },
                "log_backup_count": {
                    "optional": True,
                    "default": 5,
                    "type": int,
                    "description": "Number of rotate logs files to preserve"
                },
                "bulk_size": {
                    "optional": True,
                    "default": 1000,
                    "type": int,
                    "description": "Number of items to write in Elasticsearch using bulk operations"
                },
                "scroll_size": {
                    "optional": True,
                    "default": 100,
                    "type": int,
                    "description": "Number of items to read from Elasticsearch when scrolling"
                },
                "aliases_file": {
                    "optional": True,
                    "default": ALIASES_JSON,
                    "type": str,
                    "description": "JSON file to define aliases for raw and enriched indexes"
                },
                "menu_file": {
                    "optional": True,
                    "default": MENU_YAML,
                    "type": str,
                    "description": "YAML file to define the menus to be shown in Kibiter"
                },
                "retention_time": {
                    "optional": True,
                    "default": None,
                    "type": int,
                    "description": "The maximum number of minutes wrt the current date to retain the data"
                }
            }
        }
        params_projects = {
            "projects": {
                "projects_file": {
                    "optional": True,
                    "default": PROJECTS_JSON,
                    "type": str,
                    "description": "Projects file path with repositories to be collected group by projects"
                },
                "projects_url": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "Projects file URL"
                },
                "load_eclipse": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Load the projects from Eclipse"
                }
            }
        }

        params_phases = {
            "phases": {
                "collection": {
                    "optional": False,
                    "default": True,
                    "type": bool,
                    "description": "Activate collection of items"
                },
                "enrichment": {
                    "optional": False,
                    "default": True,
                    "type": bool,
                    "description": "Activate enrichment of items"
                },
                "identities": {
                    "optional": False,
                    "default": True,
                    "type": bool,
                    "description": "Do the identities tasks"
                },
                "panels": {
                    "optional": False,
                    "default": True,
                    "type": bool,
                    "description": "Load panels, create alias and other tasks related"
                },
                "track_items": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Track specific items from a gerrit repository"
                },
                "report": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Generate the PDF report for a project (alpha)"
                }
            }
        }

        general_config_params = [params_general, params_projects, params_phases]

        for section_params in general_config_params:
            params.update(section_params)

        # Config provided by tasks
        params_collection = {
            "es_collection": {
                "password": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "Password for connection to Elasticsearch"
                },
                "user": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "User for connection to Elasticsearch"
                },
                "url": {
                    "optional": False,
                    "default": "http://172.17.0.1:9200",
                    "type": str,
                    "description": "Elasticsearch URL"
                },
                "arthur": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Use arthur for collecting items from perceval"
                },
                "arthur_url": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "URL for the arthur service"
                },
                "redis_url": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "URL for the redis service"
                }
            }
        }

        params_enrichment = {
            "es_enrichment": {
                "url": {
                    "optional": False,
                    "default": "http://172.17.0.1:9200",
                    "type": str,
                    "description": "Elasticsearch URL"
                },
                "autorefresh": {
                    "optional": True,
                    "default": True,
                    "type": bool,
                    "description": "Execute the autorefresh of identities"
                },
                "autorefresh_interval": {
                    "optional": True,
                    "default": 2,
                    "type": int,
                    "description": "Set time interval (days) for autorefresh identities"
                },
                "user": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "User for connection to Elasticsearch"
                },
                "password": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "Password for connection to Elasticsearch"
                }
            }
        }

        params_panels = {
            "panels": {
                "strict": {
                    "optional": True,
                    "default": True,
                    "type": bool,
                    "description": "Enable strict panels loading"
                },
                "kibiter_time_from": {
                    "optional": True,
                    "default": "now-90d",
                    "type": str,
                    "description": "Default time interval for Kibiter"
                },
                "kibiter_default_index": {
                    "optional": True,
                    "default": "git",
                    "type": str,
                    "description": "Default index pattern for Kibiter"
                },
                "kibiter_url": {
                    "optional": False,
                    "default": None,
                    "type": str,
                    "description": "Kibiter URL"
                },
                "kibiter_version": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "Kibiter version"
                },
                "community": {
                    "optional": True,
                    "default": True,
                    "type": bool,
                    "description": "Enable community structure menu"
                },
                "kafka": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Enable kafka menu"
                },
                "github-repos": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Enable GitHub repo stats menu"
                },
                "gitlab-issues": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Enable GitLab issues menu"
                },
                "gitlab-merges": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Enable GitLab merge requests menu"
                },
                "mattermost": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Enable Mattermost menu"
                }
            }
        }

        params_report = {
            "report": {
                "start_date": {
                    "optional": False,
                    "default": "1970-01-01",
                    "type": str,
                    "description": "Start date for the report"
                },
                "end_date": {
                    "optional": False,
                    "default": "2100-01-01",
                    "type": str,
                    "description": "End date for the report"
                },
                "interval": {
                    "optional": False,
                    "default": "quarter",
                    "type": str,
                    "description": "Interval for the report"
                },
                "config_file": {
                    "optional": False,
                    "default": "report.cfg",
                    "type": str,
                    "description": "Config file for the report"
                },
                "data_dir": {
                    "optional": False,
                    "default": "report_data",
                    "type": str,
                    "description": "Directory in which to store the report data"
                },
                "filters": {
                    "optional": True,
                    "default": [],
                    "type": list,
                    "description": "General filters to be applied to all queries"
                },
                "offset": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "Date offset to be applied to start and end"
                }
            }
        }

        params_sortinghat = {
            "sortinghat": {
                "affiliate": {
                    "optional": False,
                    "default": "True",
                    "type": bool,
                    "description": "Affiliate identities to organizations"
                },
                "unaffiliated_group": {
                    "optional": False,
                    "default": "Unknown",
                    "type": str,
                    "description": "Name of the organization for unaffiliated identities"
                },
                "matching": {
                    "optional": False,
                    "default": ["email"],
                    "type": list,
                    "description": "Algorithm for matching identities in Sortinghat"
                },
                "sleep_for": {
                    "optional": False,
                    "default": 3600,
                    "type": int,
                    "description": "Delay between task identities executions"
                },
                "database": {
                    "optional": False,
                    "default": "sortinghat_db",
                    "type": str,
                    "description": "Name of the Sortinghat database"
                },
                "host": {
                    "optional": False,
                    "default": "mariadb",
                    "type": str,
                    "description": "Host with the Sortinghat database"
                },
                "user": {
                    "optional": False,
                    "default": "root",
                    "type": str,
                    "description": "User to access the Sortinghat database"
                },
                "password": {
                    "optional": False,
                    "default": "",
                    "type": str,
                    "description": "Password to access the Sortinghat database"
                },
                "autoprofile": {
                    "optional": False,
                    "default": ["customer", "git", "github"],
                    "type": list,
                    "description": "Order in which to get the identities information for filling the profile"
                },
                "load_orgs": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "deprecated": "Load organizations in Sortinghat database",
                    "description": ""
                },
                "identities_format": {
                    "optional": True,
                    "default": "sortinghat",
                    "type": str,
                    "description": "Format of the identities data to be loaded"
                },
                "strict_mapping": {
                    "optional": True,
                    "default": True,
                    "type": bool,
                    "description": "rigorous check of values in identities matching "
                                   "(i.e, well formed email addresses)"
                },
                "reset_on_load": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Unmerge and remove affiliations for all identities on load"
                },
                "orgs_file": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "File path with the organizations to be loaded in Sortinghat"
                },
                "identities_file": {
                    "optional": True,
                    "default": [],
                    "type": list,
                    "description": "File path with the identities to be loaded in Sortinghat"
                },
                "identities_export_url": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "URL in which to export the identities in Sortinghat"
                },
                "identities_api_token": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "API token for remote operation with GitHub and Gitlab"
                },
                "bots_names": {
                    "optional": True,
                    "default": [],
                    "type": list,
                    "description": "Name of the identities to be marked as bots"
                },
                "no_bots_names": {
                    "optional": True,
                    "default": [],
                    "type": list,
                    "description": "Name of the identities to be unmarked as bots"
                },
                "autogender": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Add gender to the profiles (executes autogender)"
                }
            }
        }

        params_track_items = {
            "track_items": {
                "project": {
                    "optional": False,
                    "default": "TrackProject",
                    "type": str,
                    "description": "Gerrit project to track"
                },
                "upstream_raw_es_url": {
                    "optional": False,
                    "default": "",
                    "type": str,
                    "description": "URL with the file with the gerrit reviews to track"
                },
                "raw_index_gerrit": {
                    "optional": False,
                    "default": "",
                    "type": str,
                    "description": "Name of the gerrit raw index"
                },
                "raw_index_git": {
                    "optional": False,
                    "default": "",
                    "type": str,
                    "description": "Name of the git raw index"
                }
            }
        }

        tasks_config_params = [params_collection, params_enrichment, params_panels,
                               params_report, params_sortinghat, params_track_items]

        for section_params in tasks_config_params:
            params.update(section_params)

        return params
def general_params ( cls ) : params = { } # GENERAL CONFIG params_general = { "general" : { "min_update_delay" : { "optional" : True , "default" : 60 , "type" : int , "description" : "Short delay between tasks (collect, enrich ...)" } , "update" : { "optional" : False , "default" : False , "type" : bool , "description" : "Execute the tasks in loop" } , "short_name" : { "optional" : False , "default" : "Short name" , "type" : str , "description" : "Short name of the project" } , "debug" : { "optional" : False , "default" : True , "type" : bool , "description" : "Debug mode (logging mainly)" } , "logs_dir" : { "optional" : False , "default" : "logs" , "type" : str , "description" : "Directory with the logs of sirmordred" } , "log_handler" : { "optional" : True , "default" : "file" , "type" : str , "description" : "use rotate for rotating the logs automatically" } , "log_max_bytes" : { "optional" : True , "default" : 104857600 , # 100MB "type" : int , "description" : "Max number of bytes per log file" } , "log_backup_count" : { "optional" : True , "default" : 5 , "type" : int , "description" : "Number of rotate logs files to preserve" } , "bulk_size" : { "optional" : True , "default" : 1000 , "type" : int , "description" : "Number of items to write in Elasticsearch using bulk operations" } , "scroll_size" : { "optional" : True , "default" : 100 , "type" : int , "description" : "Number of items to read from Elasticsearch when scrolling" } , "aliases_file" : { "optional" : True , "default" : ALIASES_JSON , "type" : str , "description" : "JSON file to define aliases for raw and enriched indexes" } , "menu_file" : { "optional" : True , "default" : MENU_YAML , "type" : str , "description" : "YAML file to define the menus to be shown in Kibiter" } , "retention_time" : { "optional" : True , "default" : None , "type" : int , "description" : "The maximum number of minutes wrt the current date to retain the data" } } } params_projects = { "projects" : { "projects_file" : { "optional" : True , "default" : PROJECTS_JSON , "type" : str , "description" : "Projects file path with repositories to be collected group by projects" } , "projects_url" : { "optional" : True , "default" : None , "type" : str , "description" : "Projects file URL" } , "load_eclipse" : { "optional" : True , "default" : False , "type" : bool , "description" : "Load the projects from Eclipse" } } } params_phases = { "phases" : { "collection" : { "optional" : False , "default" : True , "type" : bool , "description" : "Activate collection of items" } , "enrichment" : { "optional" : False , "default" : True , "type" : bool , "description" : "Activate enrichment of items" } , "identities" : { "optional" : False , "default" : True , "type" : bool , "description" : "Do the identities tasks" } , "panels" : { "optional" : False , "default" : True , "type" : bool , "description" : "Load panels, create alias and other tasks related" } , "track_items" : { "optional" : True , "default" : False , "type" : bool , "description" : "Track specific items from a gerrit repository" } , "report" : { "optional" : True , "default" : False , "type" : bool , "description" : "Generate the PDF report for a project (alpha)" } } } general_config_params = [ params_general , params_projects , params_phases ] for section_params in general_config_params : params . update ( section_params ) # Config provided by tasks params_collection = { "es_collection" : { "password" : { "optional" : True , "default" : None , "type" : str , "description" : "Password for connection to Elasticsearch" } , "user" : { "optional" : True , "default" : None , "type" : str , "description" : "User for connection to Elasticsearch" } , "url" : { "optional" : False , "default" : "http://172.17.0.1:9200" , "type" : str , "description" : "Elasticsearch URL" } , "arthur" : { "optional" : True , "default" : False , "type" : bool , "description" : "Use arthur for collecting items from perceval" } , "arthur_url" : { "optional" : True , "default" : None , "type" : str , "description" : "URL for the arthur service" } , "redis_url" : { "optional" : True , "default" : None , "type" : str , "description" : "URL for the redis service" } } } params_enrichment = { "es_enrichment" : { "url" : { "optional" : False , "default" : "http://172.17.0.1:9200" , "type" : str , "description" : "Elasticsearch URL" } , "autorefresh" : { "optional" : True , "default" : True , "type" : bool , "description" : "Execute the autorefresh of identities" } , "autorefresh_interval" : { "optional" : True , "default" : 2 , "type" : int , "description" : "Set time interval (days) for autorefresh identities" } , "user" : { "optional" : True , "default" : None , "type" : str , "description" : "User for connection to Elasticsearch" } , "password" : { "optional" : True , "default" : None , "type" : str , "description" : "Password for connection to Elasticsearch" } } } params_panels = { "panels" : { "strict" : { "optional" : True , "default" : True , "type" : bool , "description" : "Enable strict panels loading" } , "kibiter_time_from" : { "optional" : True , "default" : "now-90d" , "type" : str , "description" : "Default time interval for Kibiter" } , "kibiter_default_index" : { "optional" : True , "default" : "git" , "type" : str , "description" : "Default index pattern for Kibiter" } , "kibiter_url" : { "optional" : False , "default" : None , "type" : str , "description" : "Kibiter URL" } , "kibiter_version" : { "optional" : True , "default" : None , "type" : str , "description" : "Kibiter version" } , "community" : { "optional" : True , "default" : True , "type" : bool , "description" : "Enable community structure menu" } , "kafka" : { "optional" : True , "default" : False , "type" : bool , "description" : "Enable kafka menu" } , "github-repos" : { "optional" : True , "default" : False , "type" : bool , "description" : "Enable GitHub repo stats menu" } , "gitlab-issues" : { "optional" : True , "default" : False , "type" : bool , "description" : "Enable GitLab issues menu" } , "gitlab-merges" : { "optional" : True , "default" : False , "type" : bool , "description" : "Enable GitLab merge requests menu" } , "mattermost" : { "optional" : True , "default" : False , "type" : bool , "description" : "Enable Mattermost menu" } } } params_report = { "report" : { "start_date" : { "optional" : False , "default" : "1970-01-01" , "type" : str , "description" : "Start date for the report" } , "end_date" : { "optional" : False , "default" : "2100-01-01" , "type" : str , "description" : "End date for the report" } , "interval" : { "optional" : False , "default" : "quarter" , "type" : str , "description" : "Interval for the report" } , "config_file" : { "optional" : False , "default" : "report.cfg" , "type" : str , "description" : "Config file for the report" } , "data_dir" : { "optional" : False , "default" : "report_data" , "type" : str , "description" : "Directory in which to store the report data" } , "filters" : { "optional" : True , "default" : [ ] , "type" : list , "description" : "General filters to be applied to all queries" } , "offset" : { "optional" : True , "default" : None , "type" : str , "description" : "Date offset to be applied to start and end" } } } params_sortinghat = { "sortinghat" : { "affiliate" : { "optional" : False , "default" : "True" , "type" : bool , "description" : "Affiliate identities to organizations" } , "unaffiliated_group" : { "optional" : False , "default" : "Unknown" , "type" : str , "description" : "Name of the organization for unaffiliated identities" } , "matching" : { "optional" : False , "default" : [ "email" ] , "type" : list , "description" : "Algorithm for matching identities in Sortinghat" } , "sleep_for" : { "optional" : False , "default" : 3600 , "type" : int , "description" : "Delay between task identities executions" } , "database" : { "optional" : False , "default" : "sortinghat_db" , "type" : str , "description" : "Name of the Sortinghat database" } , "host" : { "optional" : False , "default" : "mariadb" , "type" : str , "description" : "Host with the Sortinghat database" } , "user" : { "optional" : False , "default" : "root" , "type" : str , "description" : "User to access the Sortinghat database" } , "password" : { "optional" : False , "default" : "" , "type" : str , "description" : "Password to access the Sortinghat database" } , "autoprofile" : { "optional" : False , "default" : [ "customer" , "git" , "github" ] , "type" : list , "description" : "Order in which to get the identities information for filling the profile" } , "load_orgs" : { "optional" : True , "default" : False , "type" : bool , "deprecated" : "Load organizations in Sortinghat database" , "description" : "" } , "identities_format" : { "optional" : True , "default" : "sortinghat" , "type" : str , "description" : "Format of the identities data to be loaded" } , "strict_mapping" : { "optional" : True , "default" : True , "type" : bool , "description" : "rigorous check of values in identities matching " "(i.e, well formed email addresses)" } , "reset_on_load" : { "optional" : True , "default" : False , "type" : bool , "description" : "Unmerge and remove affiliations for all identities on load" } , "orgs_file" : { "optional" : True , "default" : None , "type" : str , "description" : "File path with the organizations to be loaded in Sortinghat" } , "identities_file" : { "optional" : True , "default" : [ ] , "type" : list , "description" : "File path with the identities to be loaded in Sortinghat" } , "identities_export_url" : { "optional" : True , "default" : None , "type" : str , "description" : "URL in which to export the identities in Sortinghat" } , "identities_api_token" : { "optional" : True , "default" : None , "type" : str , "description" : "API token for remote operation with GitHub and Gitlab" } , "bots_names" : { "optional" : True , "default" : [ ] , "type" : list , "description" : "Name of the identities to be marked as bots" } , "no_bots_names" : { "optional" : True , "default" : [ ] , "type" : list , "description" : "Name of the identities to be unmarked as bots" } , "autogender" : { "optional" : True , "default" : False , "type" : bool , "description" : "Add gender to the profiles (executes autogender)" } } } params_track_items = { "track_items" : { "project" : { "optional" : False , "default" : "TrackProject" , "type" : str , "description" : "Gerrit project to track" } , "upstream_raw_es_url" : { "optional" : False , "default" : "" , "type" : str , "description" : "URL with the file with the gerrit reviews to track" } , "raw_index_gerrit" : { "optional" : False , "default" : "" , "type" : str , "description" : "Name of the gerrit raw index" } , "raw_index_git" : { "optional" : False , "default" : "" , "type" : str , "description" : "Name of the git raw index" } } } tasks_config_params = [ params_collection , params_enrichment , params_panels , params_report , params_sortinghat , params_track_items ] for section_params in tasks_config_params : params . update ( section_params ) return params
Define all the possible config params
Define all the possible config params