Upload runner.py with huggingface_hub

runner.py

@@ -0,0 +1,493 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import time
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+import requests
+
+
+# ---------------------------
+# Config data classes
+# ---------------------------
+
+@dataclass
+class BackendConfig:
+    name: str
+    endpoint: str
+    concurrency: int = field(default=20)
+    timeout: int = field(default=60)
+    max_retries: int = field(default=5)
+    retry_delay_seconds: float = field(default=2.0)
+
+
+@dataclass
+class ModelConfig:
+    name: Optional[str]
+    backend: str
+    endpoint_model_name: str
+    system_prompt: Optional[str] = None
+
+
+# ---------------------------
+# Utilities
+# ---------------------------
+
+def fetch_default_model(endpoint: str) -> str:
+    headers = {
+        'Content-Type': 'application/json',
+    }
+    url = f"{endpoint}/v1/models"
+    resp = requests.get(url, headers=headers, timeout=30)
+    resp.raise_for_status()
+    data = resp.json()
+    # Prefer first model id
+    if isinstance(data, dict) and 'data' in data and isinstance(data['data'], list) and data['data']:
+        first = data['data'][0]
+        model_id = first.get('id') or first.get('name')
+        if not model_id:
+            raise RuntimeError('Could not determine model id from /v1/models response')
+        return str(model_id)
+    raise RuntimeError('No models returned from /v1/models')
+
+
+def resolve_backend_and_model(cli_endpoint: str) -> tuple[BackendConfig, ModelConfig]:
+    if not cli_endpoint:
+        raise ValueError('--endpoint is required')
+
+    model_id = fetch_default_model(cli_endpoint)
+
+    backend = BackendConfig(
+        name='vllm',
+        endpoint=cli_endpoint,
+        concurrency=20,
+        timeout=60,
+        max_retries=5,
+        retry_delay_seconds=2.0,
+    )
+
+    model = ModelConfig(
+        name=None,
+        backend='vllm',
+        endpoint_model_name=model_id,
+        system_prompt=None,
+    )
+
+    return backend, model
+
+
+def post_chat_completion(backend: BackendConfig, model: ModelConfig, payload: Dict[str, Any]) -> Dict[str, Any]:
+    headers = {
+        'Content-Type': 'application/json',
+    }
+
+    last_exc: Optional[Exception] = None
+    for attempt in range(backend.max_retries):
+        try:
+            response = requests.post(
+                f"{backend.endpoint}/v1/chat/completions",
+                json=payload,
+                headers=headers,
+                timeout=backend.timeout,
+            )
+            response.raise_for_status()
+            return response.json()
+        except requests.exceptions.RequestException as exc:
+            last_exc = exc
+            if attempt < backend.max_retries - 1:
+                time.sleep(backend.retry_delay_seconds)
+            else:
+                raise
+    if last_exc:
+        raise last_exc
+    raise RuntimeError('Request failed without exception')
+
+
+# ---------------------------
+# Resource loading
+# ---------------------------
+
+def read_json(path: Path) -> Dict[str, Any]:
+    with path.open('r', encoding='utf-8') as f:
+        return json.load(f)
+
+
+def get_tool_calling_resource_paths() -> Dict[str, Path]:
+    base = Path(__file__).parent / 'resources'
+    return {
+        'simple-tool-calling': base / 'simple-tool-calling.json',
+        'image-generation': base / 'image-generation.json',
+        'internet-search': base / 'internet-search.json',
+        'weather': base / 'weather.json',
+        'currency-conversion': base / 'currency-conversion.json',
+    }
+
+
+# ---------------------------
+# Evaluation logic
+# ---------------------------
+
+def validate_argument_types(arguments: Dict[str, Any], properties: Dict[str, Any]) -> tuple[bool, Optional[str]]:
+    """
+    Validate that argument types match the schema.
+    Returns (valid, error_message)
+    """
+    for field_name, value in arguments.items():
+        if field_name not in properties:
+            continue  # Extra fields are okay
+            
+        expected_type = properties[field_name].get('type')
+        if expected_type == 'string' and not isinstance(value, str):
+            return False, f"Field '{field_name}' should be string, got {type(value).__name__}"
+        elif expected_type == 'number' and not isinstance(value, (int, float)):
+            return False, f"Field '{field_name}' should be number, got {type(value).__name__}"
+        elif expected_type == 'boolean':
+            # Handle both boolean and string representations
+            if isinstance(value, bool):
+                continue  # Valid boolean
+            elif isinstance(value, str) and value.lower() in ('true', 'false'):
+                # Accept string booleans but flag as warning (still pass)
+                continue
+            else:
+                return False, f"Field '{field_name}' should be boolean, got {type(value).__name__} with value {repr(value)}"
+        elif expected_type == 'integer' and not isinstance(value, int):
+            return False, f"Field '{field_name}' should be integer, got {type(value).__name__}"
+    
+    return True, None
+
+
+def validate_specific_test_case(func_name: str, arguments: Dict[str, Any], messages: List[Dict[str, Any]]) -> tuple[bool, Optional[str]]:
+    """
+    Add sanity checks for specific test cases to ensure the model actually understood the request.
+    Returns (valid, warning_message)
+    """
+    # Extract user query from messages
+    user_content = ""
+    for msg in messages:
+        if msg.get('role') == 'user':
+            user_content += msg.get('content', '') + " "
+    user_content = user_content.lower()
+    
+    # Weather test - should extract city name
+    if func_name == 'GetWeather':
+        city = arguments.get('city', '').lower()
+        if not city:
+            return False, "Weather: 'city' field is empty"
+        # Check if it extracted something reasonable from "New York City"
+        if 'new york' in user_content and 'new york' not in city and 'nyc' not in city:
+            return False, f"Weather: Expected 'New York' in city, got '{arguments.get('city')}'"
+    
+    # Internet search - should extract query
+    elif func_name == 'InternetSearch':
+        query = arguments.get('query', '').lower()
+        if not query:
+            return False, "Search: 'query' field is empty"
+        # Should mention AI or artificial intelligence
+        if ('artificial intelligence' in user_content or 'ai' in user_content) and \
+           ('ai' not in query and 'artificial intelligence' not in query):
+            return False, f"Search: Expected AI-related query, got '{arguments.get('query')}'"
+    
+    # Currency conversion - should extract currencies and amount
+    elif func_name == 'CurrencyConverter':
+        from_curr = arguments.get('fromCurrency', '')
+        to_curr = arguments.get('toCurrency', '')
+        amount = arguments.get('amount')
+        
+        if not from_curr or not to_curr:
+            return False, f"Currency: Missing currencies (from={from_curr}, to={to_curr})"
+        if amount is None:
+            return False, "Currency: 'amount' field is missing"
+        if amount <= 0:
+            return False, f"Currency: Invalid amount {amount}"
+        
+        # Check if it matches the expected conversion
+        if 'usd' in user_content and 'eur' in user_content:
+            if from_curr.upper() != 'USD' or to_curr.upper() != 'EUR':
+                return False, f"Currency: Expected USD->EUR, got {from_curr}->{to_curr}"
+            if '100' in user_content and amount != 100:
+                return False, f"Currency: Expected amount 100, got {amount}"
+    
+    # Image prompt enhancer
+    elif func_name == 'ImagePromptEnhancer':
+        enhanced = arguments.get('enhancedPrompt', '')
+        is_nsfw = arguments.get('isNSFW')
+        
+        if not enhanced:
+            return False, "Image: 'enhancedPrompt' field is empty"
+        if is_nsfw is None:
+            return False, "Image: 'isNSFW' field is missing"
+        
+        # Handle both boolean and string representations for isNSFW
+        if isinstance(is_nsfw, str):
+            if is_nsfw.lower() not in ('true', 'false'):
+                return False, f"Image: 'isNSFW' should be boolean or 'true'/'false' string, got '{is_nsfw}'"
+        elif not isinstance(is_nsfw, bool):
+            return False, f"Image: 'isNSFW' should be boolean, got {type(is_nsfw).__name__}"
+        
+        # Check if enhanced prompt is actually enhanced (longer than original)
+        if len(enhanced) < 10:
+            return False, f"Image: Enhanced prompt too short ({len(enhanced)} chars)"
+    
+    return True, None
+
+
+def evaluate_tool_call_response(message: Dict[str, Any], expected_tool: Dict[str, Any], messages: List[Dict[str, Any]]) -> tuple[bool, Optional[str]]:
+    """
+    Evaluate if the response contains valid tool calls matching the expected tool schema.
+    Returns (passed, debug_info)
+    """
+    tool_calls = message.get('tool_calls', [])
+    
+    if not tool_calls:
+        return False, "No tool_calls in response"
+    
+    if not isinstance(tool_calls, list):
+        return False, f"tool_calls should be a list, got {type(tool_calls).__name__}"
+    
+    # Get the expected function name and required parameters
+    expected_func = expected_tool.get('function', {})
+    expected_name = expected_func.get('name')
+    expected_params = expected_func.get('parameters', {})
+    required_fields = expected_params.get('required', [])
+    properties = expected_params.get('properties', {})
+    
+    # Check if any tool call matches
+    for i, tool_call in enumerate(tool_calls):
+        if not isinstance(tool_call, dict):
+            return False, f"tool_call[{i}] should be a dict, got {type(tool_call).__name__}"
+        
+        # Validate tool_call structure
+        if 'function' not in tool_call:
+            return False, f"tool_call[{i}] missing 'function' field"
+        
+        func = tool_call.get('function', {})
+        func_name = func.get('name')
+        
+        if not func_name:
+            return False, f"tool_call[{i}] missing function name"
+        
+        # Check if function name matches
+        if func_name != expected_name:
+            continue
+            
+        # Parse and validate arguments
+        try:
+            arguments_str = func.get('arguments', '{}')
+            if isinstance(arguments_str, str):
+                if not arguments_str.strip():
+                    arguments_str = '{}'
+                arguments = json.loads(arguments_str)
+            else:
+                arguments = arguments_str
+            
+            if not isinstance(arguments, dict):
+                return False, f"Arguments should be a dict, got {type(arguments).__name__}"
+                
+            # Check all required fields are present
+            missing = [f for f in required_fields if f not in arguments]
+            if missing:
+                return False, f"Missing required fields: {missing}"
+            
+            # Validate argument types
+            types_valid, type_error = validate_argument_types(arguments, properties)
+            if not types_valid:
+                return False, type_error
+            
+            # Sanity check for specific test cases
+            sanity_valid, sanity_error = validate_specific_test_case(func_name, arguments, messages)
+            if not sanity_valid:
+                return False, sanity_error
+            
+            # All checks passed!
+            return True, None
+            
+        except json.JSONDecodeError as e:
+            return False, f"Invalid JSON in arguments: {e}"
+    
+    return False, f"No matching tool call found. Expected: {expected_name}, Got: {[tc.get('function', {}).get('name') for tc in tool_calls]}"
+
+
+def infer_expected_tool(resource: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+    """Extract the first tool from the resource tools list."""
+    tools = resource.get('tools', [])
+    if tools and len(tools) > 0:
+        return tools[0]
+    return None
+
+
+def build_payload(model: ModelConfig, resource: Dict[str, Any]) -> Dict[str, Any]:
+    messages = resource.get('messages', [])
+    if model.system_prompt:
+        sys_present = any(m.get('role') == 'system' for m in messages)
+        if not sys_present:
+            messages = [{'role': 'system', 'content': model.system_prompt}] + messages
+
+    payload: Dict[str, Any] = {
+        'model': model.endpoint_model_name,
+        'messages': messages,
+        'stream': False,
+        'max_tokens': 1024,
+        'temperature': 0,
+        'top_k': 1
+    }
+
+    # Use vllm native tool calling instead of response_format
+    if 'tools' in resource:
+        payload['tools'] = resource['tools']
+        # Force the model to call a tool
+        payload['tool_choice'] = 'auto'
+
+    return payload
+
+
+def run_single_task(task_name: str, resource_path: Path, backend: BackendConfig, model: ModelConfig) -> Dict[str, Any]:
+    resource = read_json(resource_path)
+    payload = build_payload(model, resource)
+    response_data: Optional[str] = None
+    debug_info: Optional[str] = None
+    
+    try:
+        result_json = post_chat_completion(backend, model, payload)
+        message = result_json.get('choices', [{}])[0].get('message', {})
+        
+        # Get the expected tool from resource
+        expected_tool = infer_expected_tool(resource)
+        messages = resource.get('messages', [])
+        
+        if expected_tool:
+            # Evaluate tool calling response with sanity checks
+            passed, debug_info = evaluate_tool_call_response(message, expected_tool, messages)
+            # Store the full message as JSON for debugging
+            response_data = json.dumps(message, indent=2)
+        else:
+            # No tools defined, can't evaluate
+            passed = False
+            debug_info = "No tools defined in resource"
+            response_data = json.dumps(message, indent=2)
+            
+    except requests.exceptions.RequestException as exc:
+        err_text = None
+        try:
+            if hasattr(exc, 'response') and exc.response is not None:
+                err_text = exc.response.text
+        except Exception:
+            err_text = None
+        response_data = err_text or str(exc)
+        debug_info = "Request failed"
+        passed = False
+
+    # Match eval-suite per-task results structure
+    results = [{
+        'response': response_data,
+        'passed': passed,
+        'debug_info': debug_info,
+    }]
+
+    return {
+        'results': results,
+        'passed': int(passed),
+        'failed': int(not passed),
+        'total': 1,
+    }
+
+
+def format_suite_result(model: ModelConfig, task_name: str, run_result: Dict[str, Any]) -> Dict[str, Any]:
+    return {
+        'model': {
+            'name': model.name or None,
+            'endpoint_model_name': model.endpoint_model_name,
+        },
+        'benchmark': {
+            'pathname': f"tool_calling/{task_name}",
+            'safename': task_name,
+            'version': 1,
+            'task_results': run_result['results'],
+        },
+        'passed': run_result['passed'],
+        'failed': run_result['failed'],
+        'total': run_result['total'],
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description='Run tool-calling tests against a vllm chat-completions endpoint using native tool calling.')
+    parser.add_argument('--endpoint', default='http://localhost:8002', help='Chat completions base URL (default: http://localhost:8002)')
+    parser.add_argument('--debug', action='store_true', help='Show debug details for failures')
+    parser.add_argument('--verbose', action='store_true', help='Show details for all tests (passed and failed)')
+    args = parser.parse_args()
+
+    backend, model = resolve_backend_and_model(args.endpoint)
+
+    resource_paths = get_tool_calling_resource_paths()
+    selected = list(resource_paths.items())
+
+    GREEN = '\x1b[32m'
+    RED = '\x1b[31m'
+    RESET = '\x1b[0m'
+
+    total_passed = 0
+    total_failed = 0
+
+    for name, path in selected:
+        run_result = run_single_task(name, path, backend, model)
+        suite_line = format_suite_result(model=model, task_name=name, run_result=run_result)
+
+        passed = suite_line['passed']
+        failed = suite_line['failed']
+        total_passed += passed
+        total_failed += failed
+
+        status = f"{GREEN}PASS{RESET}" if passed == 1 else f"{RED}FAIL{RESET}"
+        sys.stdout.write(f"[{status}] {name}\n")
+        
+        # Show details based on flags
+        show_details = (failed and args.debug) or args.verbose
+        if show_details:
+            try:
+                task_result = suite_line['benchmark']['task_results'][0]
+                debug_info = task_result.get('debug_info')
+                response_str = task_result.get('response')
+                
+                # Try to parse response and extract tool calls
+                if response_str:
+                    try:
+                        response = json.loads(response_str)
+                        tool_calls = response.get('tool_calls', [])
+                        
+                        if tool_calls:
+                            for tc in tool_calls:
+                                func = tc.get('function', {})
+                                func_name = func.get('name', 'unknown')
+                                args_str = func.get('arguments', '{}')
+                                if isinstance(args_str, str):
+                                    tool_args = json.loads(args_str)
+                                else:
+                                    tool_args = args_str
+                                sys.stdout.write(f"  Tool: {func_name}\n")
+                                sys.stdout.write(f"  Args: {json.dumps(tool_args, indent=4)}\n")
+                        elif passed == 0:
+                            sys.stdout.write(f"  No tool calls found\n")
+                    except json.JSONDecodeError:
+                        pass
+                
+                # Show debug info (especially for failures)
+                if debug_info:
+                    sys.stdout.write(f"  {'?' if failed else '?'} {debug_info}\n")
+                elif passed == 1:
+                    sys.stdout.write(f"  ? All sanity checks passed\n")
+                    
+            except Exception as e:
+                sys.stdout.write(f"  Error printing details: {e}\n")
+
+    summary_color = GREEN if total_failed == 0 else RED
+    sys.stdout.write(f"\nSummary: {summary_color}{total_passed} passed, {total_failed} failed{RESET}\n")
+    sys.stdout.flush()
+    return 0
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())