update model

app.py

@@ -395,8 +395,9 @@ def initialize_llm():
     if LLM_PROVIDER == "huggingface":
         # Will use requests for HF Inference API
         if not HUGGINGFACE_API_KEY:
-            raise ValueError("HUGGINGFACE_API_KEY not set in environment variables")
-        print(f"Initialized HuggingFace Inference API with model: {HUGGINGFACE_MODEL}")
+            print("WARNING: HUGGINGFACE_API_KEY not set - HuggingFace provider will fail at runtime")
+        else:
+            print(f"Initialized HuggingFace Inference API with model: {HUGGINGFACE_MODEL}")
     elif LLM_PROVIDER == "local":
         ensure_llama_cpp_installed()
         try:
@@ -432,8 +433,11 @@ def initialize_llm():
                 model_path=local_model_path,
                 n_ctx=LOCAL_MODEL_CONTEXT_LENGTH,
                 n_threads=LOCAL_MODEL_THREADS,
+                n_threads_batch=LOCAL_MODEL_THREADS,  # Use all threads for batch processing
                 n_batch=LOCAL_MODEL_BATCH_SIZE,
                 n_gpu_layers=0,
+                use_mmap=True,  # Memory-mapped file loading (faster, less RAM)
+                use_mlock=False,  # Don't lock memory (not needed for HF Spaces)
                 verbose=True,  # Enable to see prompt formatting
             )
         except Exception as load_err:
@@ -525,7 +529,7 @@ def generate_response_local(system_prompt: str, user_prompt: str) -> str:
                 temperature=0.3,
                 top_p=0.7,
                 repeat_penalty=1.3,
-                stop=["<end_of_turn>", "</s>", "Question:", "Context:"],
+                stop=["<|im_end|>", "<|endoftext|>", "<think>"],  # Qwen3 stop tokens + thinking
             )
     except Exception as err:
         raise HTTPException(status_code=500, detail=f"Local model error: {err}") from err

config.py

@@ -15,11 +15,11 @@ HUGGINGFACE_API_KEY = os.getenv("HUGGINGFACE_API_KEY", "")
 HUGGINGFACE_MODEL = "google/gemma-2-2b-it"
 
 # Local model configuration (for quantized models hosted within the Space)
-LOCAL_MODEL_REPO = os.getenv("LOCAL_MODEL_REPO", "bartowski/Qwen_Qwen3-1.7B-GGUF")
-LOCAL_MODEL_FILENAME = os.getenv("LOCAL_MODEL_FILENAME", "Qwen_Qwen3-1.7B-Q4_K_M.gguf")
+LOCAL_MODEL_REPO = os.getenv("LOCAL_MODEL_REPO", "bartowski/Qwen_Qwen3-4B-Instruct-2507-GGUF")
+LOCAL_MODEL_FILENAME = os.getenv("LOCAL_MODEL_FILENAME", "Qwen_Qwen3-4B-Instruct-2507-Q4_K_M.gguf")  # Q4_K_M (2.50GB, recommended)
 LOCAL_MODEL_CONTEXT_LENGTH = int(os.getenv("LOCAL_MODEL_CONTEXT_LENGTH", "2048"))
-LOCAL_MODEL_THREADS = int(os.getenv("LOCAL_MODEL_THREADS", str(os.cpu_count() or 4)))
-LOCAL_MODEL_BATCH_SIZE = int(os.getenv("LOCAL_MODEL_BATCH_SIZE", "256"))
+LOCAL_MODEL_THREADS = int(os.getenv("LOCAL_MODEL_THREADS", str(os.cpu_count() or 2)))  # HF Spaces has 2 vCPUs
+LOCAL_MODEL_BATCH_SIZE = int(os.getenv("LOCAL_MODEL_BATCH_SIZE", "512"))  # Increased for better throughput
 LOCAL_MODEL_MAX_OUTPUT_TOKENS = int(os.getenv("LOCAL_MODEL_MAX_OUTPUT_TOKENS", "200"))
 LOCAL_MODEL_HF_TOKEN = os.getenv("LOCAL_MODEL_HF_TOKEN", HUGGINGFACE_API_KEY or "")