Update app.py

app.py

@@ -16,7 +16,7 @@ import pandas as pd
 import plotly.express as px
 import plotly.graph_objects as go
 from bs4 import BeautifulSoup
-import html2text  # CORRECTION 1: Import manquant ajouté
+import html2text
 
 # Importation du module de configuration
 from config import app_config as config
@@ -32,7 +32,7 @@ st.set_page_config(
 config.init_session_state()
 
 # Initialisation du parser HTML
-h = html2text.HTML2Text() # CORRECTION 1: Initialisation du parser
+h = html2text.HTML2Text()
 h.ignore_links = True
 
 # Configuration du logging
@@ -209,7 +209,6 @@ class IAEnricher:
         
         if response_text:
             try:
-                # CORRECTION 3: Remplacement du regex fragile par une recherche de délimiteurs JSON
                 start = response_text.find('{')
                 end = response_text.rfind('}')
                 if start != -1 and end != -1:
@@ -238,7 +237,6 @@ class IAEnricher:
         
         if response_text:
             try:
-                # CORRECTION 3: Remplacement du regex fragile
                 start = response_text.find('{')
                 end = response_text.rfind('}')
                 if start != -1 and end != -1:
@@ -266,7 +264,6 @@ class IAEnricher:
         
         if response_text:
             try:
-                # CORRECTION 3: Remplacement du regex fragile
                 start = response_text.find('{')
                 end = response_text.rfind('}')
                 if start != -1 and end != -1:
@@ -289,14 +286,13 @@ def check_api_keys():
     
     valid_keys = {k: v for k, v in keys.items() if v and v != f'your_{k.lower()}_here'}
     
-    config.USE_API_KEYS = len(valid_keys) == len(keys)
+    config.USE_API_KEYS = len(valid_keys) > 0
     if not config.USE_API_KEYS:
-        missing = set(keys.keys()) - set(valid_keys.keys())
-        logger.warning(f"Clés d'API manquantes ou non configurées: {', '.join(missing)}")
-        logger.warning("Le bot fonctionnera en mode dégradé avec des pauses plus longues.")
+        logger.warning("Aucune clé d'API valide trouvée. Le bot fonctionnera en mode dégradé avec des pauses plus longues.")
     else:
-        logger.info("Toutes les clés d'API sont configurées.")
-    return config.USE_API_KEYS
+        logger.info(f"Clés d'API valides trouvées pour: {', '.join(valid_keys.keys())}.")
+        
+    return valid_keys
 
 def make_request(url, headers=None, params=None, is_api_call=True):
     config.REQUEST_COUNT += 1
@@ -377,9 +373,14 @@ def save_qa_pair(question, answer, category, subcategory, source, attack_signatu
     except Exception as e:
         logger.error(f"Erreur lors de la sauvegarde du fichier {filename}: {str(e)}")
 
-def collect_kaggle_data(queries):
+def collect_kaggle_data(queries, num_pages, results_per_page):
     logger.info("Début de la collecte des données Kaggle...")
     
+    if not os.getenv('KAGGLE_USERNAME') or not os.getenv('KAGGLE_KEY'):
+        logger.warning("Clés Kaggle non configurées. La collecte Kaggle est ignorée.")
+        st.session_state.logs.append("ATTENTION: Clés Kaggle non configurées. Collecte ignorée.")
+        return
+        
     os.environ['KAGGLE_USERNAME'] = os.getenv('KAGGLE_USERNAME')
     os.environ['KAGGLE_KEY'] = os.getenv('KAGGLE_KEY')
     import kaggle
@@ -394,7 +395,12 @@ def collect_kaggle_data(queries):
     for query in list(set(search_queries)):
         logger.info(f"Recherche de datasets Kaggle pour: {query}")
         try:
-            datasets = kaggle.api.dataset_list(search=query, max_results=5)
+            # Kaggle API ne supporte pas la pagination et "results_per_page"
+            datasets = kaggle.api.dataset_list(search=query, max_results=results_per_page)
+            if not datasets:
+                logger.info(f"Aucun dataset trouvé pour la requête '{query}'.")
+                continue
+            
             for dataset in datasets:
                 dataset_ref = dataset.ref
                 if ia_enricher.available and st.session_state.enable_enrichment:
@@ -405,6 +411,7 @@ def collect_kaggle_data(queries):
                 
                 logger.info(f"Traitement du dataset: {dataset_ref}")
                 download_dir = Path("data") / "security" / "kaggle" / dataset_ref.replace('/', '_')
+                shutil.rmtree(download_dir, ignore_errors=True) # S'assurer que le dossier est vide
                 download_dir.mkdir(parents=True, exist_ok=True)
                 kaggle.api.dataset_download_files(dataset_ref, path=download_dir, unzip=True)
                 
@@ -427,142 +434,197 @@ def collect_kaggle_data(queries):
             logger.error(f"Erreur lors de la collecte des données Kaggle pour {query}: {str(e)}")
     logger.info("Collecte des données Kaggle terminée.")
 
-def collect_github_data(queries):
+def collect_github_data(queries, num_pages, results_per_page):
     logger.info("Début de la collecte des données GitHub...")
     base_url = "https://api.github.com"
     headers = {"Accept": "application/vnd.github.v3+json"}
-    if config.USE_API_KEYS:
-        token = os.getenv('GITHUB_API_TOKEN')
-        headers["Authorization"] = f"token {token}"
+    
+    github_token = os.getenv('GITHUB_API_TOKEN')
+    if github_token:
+        headers["Authorization"] = f"token {github_token}"
+    else:
+        logger.warning("Clé GitHub non configurée. La collecte GitHub est ignorée.")
+        st.session_state.logs.append("ATTENTION: Clé GitHub non configurée. Collecte ignorée.")
+        return
     
     search_queries = queries.split('\n') if queries else ["topic:devsecops", "topic:security"]
     
     for query in search_queries:
-        logger.info(f"Recherche de repositories pour: {query}")
-        search_url = f"{base_url}/search/repositories"
-        params = {"q": query, "sort": "stars", "per_page": 10}
-        response = make_request(search_url, headers=headers, params=params)
-        if not response:
-            continue
+        logger.info(f"Recherche de repositories pour: '{query}' sur {num_pages} page(s)")
         
-        data = response.json()
-        for repo in data.get("items", []):
-            repo_name = repo["full_name"].replace("/", "_")
-            logger.info(f"Traitement du repository: {repo['full_name']}")
+        for page_number in range(1, num_pages + 1):
+            logger.info(f"Consultation de la page {page_number}...")
+            search_url = f"{base_url}/search/repositories"
             
-            issues_url = f"{base_url}/repos/{repo['full_name']}/issues"
-            issues_params = {"state": "closed", "labels": "security,bug,vulnerability", "per_page": 10}
-            issues_response = make_request(issues_url, headers=headers, params=issues_params)
+            params = {
+                "q": query,
+                "sort": "stars",
+                "per_page": results_per_page,
+                "page": page_number
+            }
             
-            if issues_response:
-                issues_data = issues_response.json()
-                for issue in issues_data:
-                    if "pull_request" in issue: continue
-                    question = issue.get("title", "")
-                    body = clean_html(issue.get("body", ""))
-                    if not question or not body or len(body) < 50: continue
-                    
-                    comments_url = issue.get("comments_url")
-                    comments_response = make_request(comments_url, headers=headers)
-                    answer_parts = []
-                    if comments_response:
-                        comments_data = comments_response.json()
-                        for comment in comments_data:
-                            comment_body = clean_html(comment.get("body", ""))
-                            if comment_body: answer_parts.append(comment_body)
-                    
-                    if answer_parts:
-                        answer = "\n\n".join(answer_parts)
-                        save_qa_pair(
-                            question=f"{question}: {body}", answer=answer, category="devsecops",
-                            subcategory="github", source=f"github_{repo_name}"
-                        )
-            time.sleep(random.uniform(1, 3))
+            response = make_request(search_url, headers=headers, params=params)
+            if not response:
+                break
+            
+            data = response.json()
+            items = data.get("items", [])
+            
+            if not items:
+                logger.info(f"Fin des résultats pour cette requête (page {page_number}).")
+                break
+            
+            for repo in items:
+                repo_name = repo["full_name"].replace("/", "_")
+                logger.info(f"Traitement du repository: {repo['full_name']}")
+                
+                issues_url = f"{base_url}/repos/{repo['full_name']}/issues"
+                issues_params = {"state": "closed", "labels": "security,bug,vulnerability", "per_page": 10}
+                issues_response = make_request(issues_url, headers=headers, params=issues_params)
+                
+                if issues_response:
+                    issues_data = issues_response.json()
+                    for issue in issues_data:
+                        if "pull_request" in issue: continue
+                        question = issue.get("title", "")
+                        body = clean_html(issue.get("body", ""))
+                        if not question or not body or len(body) < 50: continue
+                        
+                        comments_url = issue.get("comments_url")
+                        comments_response = make_request(comments_url, headers=headers)
+                        answer_parts = []
+                        if comments_response:
+                            comments_data = comments_response.json()
+                            for comment in comments_data:
+                                comment_body = clean_html(comment.get("body", ""))
+                                if comment_body: answer_parts.append(comment_body)
+                        
+                        if answer_parts:
+                            answer = "\n\n".join(answer_parts)
+                            save_qa_pair(
+                                question=f"{question}: {body}", answer=answer, category="devsecops",
+                                subcategory="github", source=f"github_{repo_name}"
+                            )
+                time.sleep(random.uniform(1, 3))
     logger.info("Collecte des données GitHub terminée.")
 
-def collect_huggingface_data(queries):
+def collect_huggingface_data(queries, num_pages, results_per_page):
     logger.info("Début de la collecte des données Hugging Face...")
     base_url = "https://huggingface.co/api"
     headers = {"Accept": "application/json"}
-    if config.USE_API_KEYS:
-        token = os.getenv('HUGGINGFACE_API_TOKEN')
-        headers["Authorization"] = f"Bearer {token}"
+    
+    hf_token = os.getenv('HUGGINGFACE_API_TOKEN')
+    if hf_token:
+        headers["Authorization"] = f"Bearer {hf_token}"
+    else:
+        logger.warning("Clé Hugging Face non configurée. La collecte Hugging Face est ignorée.")
+        st.session_state.logs.append("ATTENTION: Clé Hugging Face non configurée. Collecte ignorée.")
+        return
     
     search_queries = queries.split('\n') if queries else ["security", "devsecops"]
     for query in search_queries:
         logger.info(f"Recherche de datasets pour: {query}")
-        search_url = f"{base_url}/datasets"
-        params = {"search": query, "limit": 10}
-        response = make_request(search_url, headers=headers, params=params)
-        if not response: continue
         
-        data = response.json()
-        for dataset in data:
-            dataset_id = dataset["id"].replace("/", "_")
-            logger.info(f"Traitement du dataset: {dataset['id']}")
-            dataset_url = f"{base_url}/datasets/{dataset['id']}"
-            dataset_response = make_request(dataset_url, headers=headers)
+        # Hugging Face API ne supporte pas la pagination par page_number, mais par 'limit' et 'offset'
+        # On va simuler la pagination en ajustant l'offset
+        for page_number in range(num_pages):
+            offset = page_number * results_per_page
+            search_url = f"{base_url}/datasets"
+            params = {"search": query, "limit": results_per_page, "offset": offset}
             
-            if dataset_response:
-                dataset_data = dataset_response.json()
-                description = clean_html(dataset_data.get("description", ""))
-                if not description or len(description) < 100: continue
-                tags = dataset_data.get("tags", [])
-                tags_text = ", ".join(tags) if tags else "No tags"
-                answer = f"Dataset: {dataset_data.get('id', '')}\nDownloads: {dataset_data.get('downloads', 0)}\nTags: {tags_text}\n\n{description}"
+            response = make_request(search_url, headers=headers, params=params)
+            if not response: continue
+            
+            data = response.json()
+            if not data:
+                logger.info(f"Fin des résultats pour la requête '{query}'.")
+                break
                 
-                save_qa_pair(
-                    question=f"What is the {dataset_data.get('id', '')} dataset about?", answer=answer,
-                    category="security", subcategory="dataset", source=f"huggingface_{dataset_id}", tags=tags
-                )
-            time.sleep(random.uniform(1, 3))
+            for dataset in data:
+                dataset_id = dataset["id"].replace("/", "_")
+                logger.info(f"Traitement du dataset: {dataset['id']}")
+                dataset_url = f"{base_url}/datasets/{dataset['id']}"
+                dataset_response = make_request(dataset_url, headers=headers)
+                
+                if dataset_response:
+                    dataset_data = dataset_response.json()
+                    description = clean_html(dataset_data.get("description", ""))
+                    if not description or len(description) < 100: continue
+                    tags = dataset_data.get("tags", [])
+                    tags_text = ", ".join(tags) if tags else "No tags"
+                    answer = f"Dataset: {dataset_data.get('id', '')}\nDownloads: {dataset_data.get('downloads', 0)}\nTags: {tags_text}\n\n{description}"
+                    
+                    save_qa_pair(
+                        question=f"What is the {dataset_data.get('id', '')} dataset about?", answer=answer,
+                        category="security", subcategory="dataset", source=f"huggingface_{dataset_id}", tags=tags
+                    )
+                time.sleep(random.uniform(1, 3))
     logger.info("Collecte des données Hugging Face terminée.")
 
-def collect_nvd_data():
+def collect_nvd_data(queries, num_pages, results_per_page):
     logger.info("Début de la collecte des données NVD...")
     base_url = "https://services.nvd.nist.gov/rest/json/cves/2.0"
     headers = {"Accept": "application/json"}
-    if config.USE_API_KEYS:
-        key = os.getenv('NVD_API_KEY')
-        headers["apiKey"] = key
-    
-    params = {"resultsPerPage": 50}
-    response = make_request(base_url, headers=headers, params=params)
-    if not response:
-        logger.warning("Impossible de récupérer les données du NVD.")
-        return
     
-    data = response.json()
-    vulnerabilities = data.get("vulnerabilities", [])
-    logger.info(f"Traitement de {len(vulnerabilities)} vulnérabilités...")
+    nvd_key = os.getenv('NVD_API_KEY')
+    if nvd_key:
+        headers["apiKey"] = nvd_key
+    else:
+        logger.warning("Clé NVD non configurée. La collecte NVD est ignorée.")
+        st.session_state.logs.append("ATTENTION: Clé NVD non configurée. Collecte ignorée.")
+        return
     
-    for vuln in vulnerabilities:
-        cve_data = vuln.get("cve", {})
-        cve_id = cve_data.get("id", "")
-        descriptions = cve_data.get("descriptions", [])
-        description = next((desc.get("value", "") for desc in descriptions if desc.get("lang") == "en"), "")
-        if not description or len(description) < 50: continue
-        
-        cvss_v3 = cve_data.get("metrics", {}).get("cvssMetricV31", [{}])[0].get("cvssData", {})
-        severity = cvss_v3.get("baseSeverity", "UNKNOWN")
-        score = cvss_v3.get("baseScore", 0)
-        references = [ref.get("url", "") for ref in cve_data.get("references", [])]
+    for page in range(num_pages):
+        start_index = page * results_per_page
+        logger.info(f"Consultation de la page NVD, index de départ: {start_index}")
+        params = {"resultsPerPage": results_per_page, "startIndex": start_index}
+        response = make_request(base_url, headers=headers, params=params)
         
-        answer = f"CVE ID: {cve_id}\nSeverity: {severity}\nCVSS Score: {score}\nReferences: {', '.join(references[:5])}\n\nDescription: {description}"
-        
-        save_qa_pair(
-            question=f"What is the vulnerability {cve_id}?", answer=answer,
-            category="security", subcategory="vulnerability", source=f"nvd_{cve_id}"
-        )
+        if not response:
+            logger.warning("Impossible de récupérer les données du NVD. Arrêt de la collecte NVD.")
+            break
+            
+        data = response.json()
+        vulnerabilities = data.get("vulnerabilities", [])
+        if not vulnerabilities:
+            logger.info("Fin des résultats pour la collecte NVD.")
+            break
+        
+        logger.info(f"Traitement de {len(vulnerabilities)} vulnérabilités...")
+        
+        for vuln in vulnerabilities:
+            cve_data = vuln.get("cve", {})
+            cve_id = cve_data.get("id", "")
+            descriptions = cve_data.get("descriptions", [])
+            description = next((desc.get("value", "") for desc in descriptions if desc.get("lang") == "en"), "")
+            if not description or len(description) < 50: continue
+            
+            cvss_v3 = cve_data.get("metrics", {}).get("cvssMetricV31", [{}])[0].get("cvssData", {})
+            severity = cvss_v3.get("baseSeverity", "UNKNOWN")
+            score = cvss_v3.get("baseScore", 0)
+            references = [ref.get("url", "") for ref in cve_data.get("references", [])]
+            
+            answer = f"CVE ID: {cve_id}\nSeverity: {severity}\nCVSS Score: {score}\nReferences: {', '.join(references[:5])}\n\nDescription: {description}"
+            
+            save_qa_pair(
+                question=f"What is the vulnerability {cve_id}?", answer=answer,
+                category="security", subcategory="vulnerability", source=f"nvd_{cve_id}"
+            )
+        time.sleep(random.uniform(1, 3))
     logger.info("Collecte des données NVD terminée.")
 
-def collect_stack_exchange_data(queries):
+def collect_stack_exchange_data(queries, num_pages, results_per_page):
     logger.info("Début de la collecte des données Stack Exchange...")
     base_url = "https://api.stackexchange.com/2.3"
-    params_base = {"pagesize": 10, "order": "desc", "sort": "votes", "filter": "withbody"}
-    if config.USE_API_KEYS:
-        key = os.getenv('STACK_EXCHANGE_API_KEY')
-        params_base["key"] = key
+    params_base = {"pagesize": results_per_page, "order": "desc", "sort": "votes", "filter": "withbody"}
+    
+    se_key = os.getenv('STACK_EXCHANGE_API_KEY')
+    if se_key:
+        params_base["key"] = se_key
+    else:
+        logger.warning("Clé Stack Exchange non configurée. La collecte est ignorée.")
+        st.session_state.logs.append("ATTENTION: Clé Stack Exchange non configurée. Collecte ignorée.")
+        return
     
     sites = [
         {"site": "security", "category": "security", "subcategory": "security"},
@@ -582,40 +644,48 @@ def collect_stack_exchange_data(queries):
         tags = tags_by_site.get(site, []) + (queries.split('\n') if queries else [])
         
         for tag in list(set(tags)):
-            logger.info(f"Recherche de questions avec le tag: {tag}")
+            logger.info(f"Recherche de questions avec le tag: '{tag}'")
             questions_url = f"{base_url}/questions"
-            params = {**params_base, "site": site, "tagged": tag}
-            
-            response = make_request(questions_url, params=params)
-            if not response: continue
             
-            questions_data = response.json()
-            for question in questions_data.get("items", []):
-                question_id = question.get("question_id")
-                title = question.get("title", "")
-                body = clean_html(question.get("body", ""))
-                if not body or len(body) < 50: continue
+            for page_number in range(1, num_pages + 1):
+                params = {**params_base, "site": site, "tagged": tag, "page": page_number}
                 
-                answers_url = f"{base_url}/questions/{question_id}/answers"
-                answers_params = {**params_base, "site": site}
-                answers_response = make_request(answers_url, params=answers_params)
-                answer_body = ""
-                if answers_response and answers_response.json().get("items"):
-                    answer_body = clean_html(answers_response.json()["items"][0].get("body", ""))
+                response = make_request(questions_url, params=params)
+                if not response: continue
                 
-                if answer_body:
-                    save_qa_pair(
-                        question=title, answer=answer_body, category=category,
-                        subcategory=subcategory, source=f"{site}_{question_id}", tags=question.get("tags", [])
-                    )
-                time.sleep(random.uniform(1, 3))
+                questions_data = response.json()
+                items = questions_data.get("items", [])
+                
+                if not items:
+                    logger.info(f"Fin des résultats pour le tag '{tag}' à la page {page_number}.")
+                    break
+                    
+                for question in items:
+                    question_id = question.get("question_id")
+                    title = question.get("title", "")
+                    body = clean_html(question.get("body", ""))
+                    if not body or len(body) < 50: continue
+                    
+                    answers_url = f"{base_url}/questions/{question_id}/answers"
+                    answers_params = {**params_base, "site": site}
+                    answers_response = make_request(answers_url, params=answers_params)
+                    answer_body = ""
+                    if answers_response and answers_response.json().get("items"):
+                        answer_body = clean_html(answers_response.json()["items"][0].get("body", ""))
+                    
+                    if answer_body:
+                        save_qa_pair(
+                            question=title, answer=answer_body, category=category,
+                            subcategory=subcategory, source=f"{site}_{question_id}", tags=question.get("tags", [])
+                        )
+                    time.sleep(random.uniform(1, 3))
     logger.info("Collecte des données Stack Exchange terminée.")
 
-def run_data_collection(sources, queries):
+def run_data_collection(sources, queries, num_pages, results_per_page):
     st.session_state.bot_status = "En cours d'exécution"
     st.session_state.logs = []
     
-    check_api_keys()
+    valid_keys = check_api_keys()
     
     progress_bar = st.progress(0)
     status_text = st.empty()
@@ -625,18 +695,34 @@ def run_data_collection(sources, queries):
     completed_sources = 0
     
     for source_name in enabled_sources:
+        if source_name == "Kaggle" and 'KAGGLE_USERNAME' not in os.environ:
+            logger.warning("Clés Kaggle non définies dans les variables d'environnement. Saut de la collecte Kaggle.")
+            continue
+        if source_name == "GitHub" and not valid_keys.get('GITHUB_API_TOKEN'):
+            logger.warning("Clé GitHub non définie. Saut de la collecte GitHub.")
+            continue
+        if source_name == "Hugging Face" and not valid_keys.get('HUGGINGFACE_API_TOKEN'):
+            logger.warning("Clé Hugging Face non définie. Saut de la collecte Hugging Face.")
+            continue
+        if source_name == "NVD" and not valid_keys.get('NVD_API_KEY'):
+            logger.warning("Clé NVD non définie. Saut de la collecte NVD.")
+            continue
+        if source_name == "Stack Exchange" and not valid_keys.get('STACK_EXCHANGE_API_KEY'):
+            logger.warning("Clé Stack Exchange non définie. Saut de la collecte Stack Exchange.")
+            continue
+            
         status_text.text(f"Collecte des données de {source_name}...")
         try:
             if source_name == "Kaggle":
-                collect_kaggle_data(queries.get("Kaggle", ""))
+                collect_kaggle_data(queries.get("Kaggle", ""), num_pages, results_per_page)
             elif source_name == "GitHub":
-                collect_github_data(queries.get("GitHub", ""))
+                collect_github_data(queries.get("GitHub", ""), num_pages, results_per_page)
             elif source_name == "Hugging Face":
-                collect_huggingface_data(queries.get("Hugging Face", ""))
+                collect_huggingface_data(queries.get("Hugging Face", ""), num_pages, results_per_page)
             elif source_name == "NVD":
-                collect_nvd_data()
+                collect_nvd_data(queries.get("NVD", ""), num_pages, results_per_page)
             elif source_name == "Stack Exchange":
-                collect_stack_exchange_data(queries.get("Stack Exchange", ""))
+                collect_stack_exchange_data(queries.get("Stack Exchange", ""), num_pages, results_per_page)
         except Exception as e:
             logger.error(f"Erreur fatale lors de la collecte de {source_name}: {str(e)}")
         
@@ -648,7 +734,6 @@ def run_data_collection(sources, queries):
     progress_bar.empty()
     status_text.empty()
     
-    # CORRECTION 2: Forcer le rafraîchissement de l'UI pour afficher les résultats
     st.rerun()
 
 def main():
@@ -672,6 +757,19 @@ def main():
         
         st.markdown("---")
         
+        st.header("Paramètres de la collecte")
+        col1, col2 = st.columns(2)
+        num_pages = col1.slider(
+            "Nombre de pages à consulter par source",
+            min_value=1, max_value=20, value=5,
+            help="Le bot consultera jusqu'à X pages de résultats pour chaque source."
+        )
+        results_per_page = col2.slider(
+            "Nombre de résultats par page",
+            min_value=10, max_value=100, value=20,
+            help="Le bot demandera jusqu'à Y résultats pour chaque page consultée."
+        )
+        
         st.header("Lancer la collecte")
         
         st.subheader("Sources de données")
@@ -689,6 +787,7 @@ def main():
         queries["GitHub"] = st.text_area("Requêtes GitHub (une par ligne)", "topic:devsecops\ntopic:security\nvulnerability")
         queries["Kaggle"] = st.text_area("Requêtes Kaggle (une par ligne)", "cybersecurity\nvulnerability dataset\npenetration testing")
         queries["Hugging Face"] = st.text_area("Requêtes Hugging Face (une par ligne)", "security dataset\nvulnerability\nlanguage model security")
+        queries["NVD"] = ""
         queries["Stack Exchange"] = st.text_area("Tags Stack Exchange (un par ligne)", "devsecops\nsecurity\nvulnerability")
         
         st.markdown("---")
@@ -698,7 +797,7 @@ def main():
                 st.session_state.logs = []
                 st.session_state.qa_data = []
                 st.session_state.total_qa_pairs = 0
-                run_data_collection(sources, queries)
+                run_data_collection(sources, queries, num_pages, results_per_page)
         else:
             st.warning("La collecte est en cours. Veuillez attendre qu'elle se termine.")
             if st.button("Forcer l'arrêt", use_container_width=True, type="secondary"):
@@ -779,4 +878,4 @@ def main():
         st.session_state.n_predict = st.slider("Nombre de tokens", 128, 1024, st.session_state.n_predict, help="Nombre maximum de tokens à générer par l'IA.")
 
 if __name__ == "__main__":
-    main()
+    main()