diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3d4c34e5..7b10946f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,7 +4,6 @@ on: push: branches: [ master ] pull_request: - branches: [ master ] jobs: test: @@ -20,6 +19,9 @@ jobs: with: python-version: '3.11' + - name: Install dependencies + run: pip install httpx + - name: Run Circuit Breaker Tests run: python3 test_circuit_breaker.py diff --git a/README.md b/README.md index 4ce30b3b..ade210f3 100644 --- a/README.md +++ b/README.md @@ -253,18 +253,21 @@ Exposes the entry endpoint (`http://localhost:5000/v1`) and evaluates prompt com | Model | Classifier | Premium backend | Fallback | Context Length | |:---|---:|:---|:---|:---| -| `llm-routing-auto-free` | ✅ | — | LiteLLM with classified tier | 256K | -| `llm-routing-auto-agy` | ✅ | agy (gated: reasoning → gemini-3.5-flash, advanced → gemini-3.5-flash → claude-opus-4.6) | LiteLLM with classified tier | 256K | -| `llm-routing-auto-ollama` | ✅ | Ollama (gated: reasoning & advanced → ollama-deepseek-v4-pro, complex → ollama-deepseek-v4-flash, below → bypass) | LiteLLM with classified tier | 256K | -| `llm-routing-auto-agy-ollama` | ✅ | agy → Ollama (gated: reasoning/advanced/complex) | LiteLLM with classified tier | 256K | -| `llm-routing-agy` | ❌ | agy (Gemini/Claude) — unconditional | LiteLLM agent-advanced-core | 256K | -| `llm-routing-ollama` | ✅ | Ollama (gated: reasoning & advanced → ollama-deepseek-v4-pro, complex & below → ollama-deepseek-v4-flash) | LiteLLM agent-advanced-core / agent-reasoning-core | 256K | -| `agent-advanced-core` | ❌ | — | LiteLLM openrouter-auto | 256K | -| `agent-reasoning-core` | ❌ | — | LiteLLM fallback chain | 256K | -| `agent-complex-core` | ❌ | — | LiteLLM fallback chain | 256K | -| `agent-medium-core` | ❌ | — | LiteLLM fallback chain | 256K | +| `llm-routing-auto-free` | ✅ | — | LiteLLM with classified tier | 262K | +| `llm-routing-auto-agy` | ✅ | agy (gated: reasoning → gemini-3.5-flash, advanced → gemini-3.5-flash → claude-opus-4.6) | LiteLLM with classified tier | 262K | +| `llm-routing-auto-ollama` | ✅ | Ollama (gated: reasoning & advanced → ollama-deepseek-v4-pro, complex → ollama-deepseek-v4-flash, below → bypass) | LiteLLM with classified tier | 512K | +| `llm-routing-auto-agy-ollama` | ✅ | agy → Ollama (gated: reasoning/advanced/complex) | LiteLLM with classified tier | 512K | +| `llm-routing-agy` | ❌ | agy (Gemini/Claude) — unconditional | LiteLLM agent-advanced-core | 1M | +| `llm-routing-ollama` | ✅ | Ollama (gated: reasoning & advanced → ollama-deepseek-v4-pro, complex & below → ollama-deepseek-v4-flash) | LiteLLM agent-advanced-core / agent-reasoning-core | 512K | +| `agent-advanced-core` | ❌ | — | LiteLLM openrouter-auto | 262K | +| `agent-reasoning-core` | ❌ | — | LiteLLM fallback chain | 262K | +| `agent-complex-core` | ❌ | — | LiteLLM fallback chain | 262K | +| `agent-medium-core` | ❌ | — | LiteLLM fallback chain | 262K | | `agent-simple-core` | ❌ | — | LiteLLM fallback chain | 256K | +> [!TIP] +> Model capabilities, token limits, and costs are visible in LiteLLM's Model Hub Table at `http://localhost:4000/ui/?page=model-hub-table` (or port 4000 on the gateway host). + ### B. LiteLLM Proxy Gateway (`litellm/config.yaml`) - **Version Pinning**: The LiteLLM gateway runs `ghcr.io/berriai/litellm:v1.88.0` (latest stable as of June 2026). The tag is explicitly pinned in `pod.yaml` — never use `:latest`. Check available tags with `skopeo list-tags docker://ghcr.io/berriai/litellm` before upgrading. ClickHouse runs `docker.io/clickhouse/clickhouse-server:26.5.1` (upgraded from 24.8, June 2026). Valkey Cache runs `docker.io/valkey/valkey:9.1.0-alpine` (upgraded from 8, June 2026). Orchestrates routing fallback chains, Redis caching, and telemetry callbacks: diff --git a/litellm/config.yaml b/litellm/config.yaml index b731a2b7..4a791e45 100644 --- a/litellm/config.yaml +++ b/litellm/config.yaml @@ -20,6 +20,16 @@ litellm_settings: - langfuse detailed_debug: false drop_params: true + public_model_groups: + - openrouter-auto + - llm-routing-ollama + - ollama-deepseek-v4-pro + - ollama-deepseek-v4-flash + - agent-advanced-core + - agent-reasoning-core + - agent-complex-core + - agent-medium-core + - agent-simple-core fallbacks: - agent-simple-core: - agent-medium-core @@ -57,12 +67,28 @@ model_list: model: openrouter/openrouter/auto request_timeout: 120 model_name: openrouter-auto + model_info: + supports_vision: true + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 2000000 + max_input_tokens: 2000000 + is_public_model_group: true - litellm_params: model: openai/llm-routing-ollama api_base: http://127.0.0.1:5000/v1 api_key: os.environ/LITELLM_MASTER_KEY request_timeout: 120 model_name: llm-routing-ollama + model_info: + supports_vision: true + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 524288 + max_input_tokens: 524288 + is_public_model_group: true # DISABLED 2026-06-08 — 20GB on disk, 23GB GTT (system RAM as GPU buffer). # Uncomment to re-enable once a lighter model replaces it. @@ -93,6 +119,16 @@ model_list: api_base: https://api.ollama.com api_key: os.environ/OLLAMA_API_KEY request_timeout: 120 + model_info: + supports_vision: true + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 524288 + max_input_tokens: 524288 + input_cost_per_token: 0.00000174 + output_cost_per_token: 0.00000348 + is_public_model_group: true # ================================================================================ # OLLAMA FLASH TIER — lighter/faster model for reasoning-tier requests. @@ -104,6 +140,16 @@ model_list: api_base: https://api.ollama.com api_key: os.environ/OLLAMA_API_KEY request_timeout: 120 + model_info: + supports_vision: true + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 524288 + max_input_tokens: 524288 + input_cost_per_token: 0.00000014 + output_cost_per_token: 0.00000028 + is_public_model_group: true # ================================================================================ # AGENT TIER DEPLOYMENTS — safety-net entries (one per tier) @@ -133,22 +179,62 @@ model_list: litellm_params: model: openrouter/google/gemma-4-31b-it:free request_timeout: 20 + model_info: + supports_vision: false + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 262144 + max_input_tokens: 262144 + is_public_model_group: true - model_name: agent-reasoning-core litellm_params: model: openrouter/google/gemma-4-26b-a4b-it:free request_timeout: 20 + model_info: + supports_vision: false + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 262144 + max_input_tokens: 262144 + is_public_model_group: true - model_name: agent-complex-core litellm_params: model: openrouter/nvidia/nemotron-3-ultra-550b-a55b:free request_timeout: 20 + model_info: + supports_vision: false + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 262144 + max_input_tokens: 262144 + is_public_model_group: true - model_name: agent-medium-core litellm_params: model: openrouter/google/gemma-4-26b-a4b-it:free request_timeout: 20 + model_info: + supports_vision: false + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 262144 + max_input_tokens: 262144 + is_public_model_group: true - model_name: agent-simple-core litellm_params: model: openrouter/nvidia/nemotron-3-nano-30b-a3b:free request_timeout: 20 + model_info: + supports_vision: false + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 256000 + max_input_tokens: 256000 + is_public_model_group: true redis_settings: redis_host: 127.0.0.1 diff --git a/pod.yaml b/pod.yaml index bad4946c..a0bda21c 100644 --- a/pod.yaml +++ b/pod.yaml @@ -90,6 +90,8 @@ spec: env: - name: CONFIG_PATH value: /config/router_dir/config.yaml + - name: LITELLM_CONFIG_PATH + value: /config/litellm_dir/config.yaml - name: DBUS_SESSION_BUS_ADDRESS value: unix:path=/run/user/1000/bus - name: LITELLM_MASTER_KEY @@ -127,6 +129,8 @@ spec: volumeMounts: - mountPath: /config/router_dir name: router-config + - mountPath: /config/litellm_dir + name: litellm-config - mountPath: /app/data name: dataset-data - mountPath: /config/gemini_auth diff --git a/router/free_models_roster.json b/router/free_models_roster.json index 2d4e8998..1e050d29 100644 --- a/router/free_models_roster.json +++ b/router/free_models_roster.json @@ -145,6 +145,6 @@ "context_length": 32768 } ], - "updated_at": "2026-06-19T22:36:57.045439Z", + "updated_at": "2026-06-20T02:28:14.044490Z", "count": 24 } \ No newline at end of file diff --git a/router/main.py b/router/main.py index 8e1d7a92..1f01925b 100644 --- a/router/main.py +++ b/router/main.py @@ -351,6 +351,18 @@ async def save_persisted_stats(force=False): CACHE_TTL_SECONDS = 86400 # Decisions cached for 24 hours classification_lock = asyncio.Lock() +async def _purge_stale_deployments(db_url: str, pattern: str): + """Purge stale deployments matching the pattern from LiteLLM's DB.""" + import asyncpg + conn = await asyncpg.connect(db_url) + try: + await conn.execute( + 'DELETE FROM "LiteLLM_ProxyModelTable" WHERE model_name LIKE $1', + pattern + ) + finally: + await conn.close() + async def sync_adaptive_router_roster(master_key: str): """Fetch free OpenRouter models and register them as deployments in LiteLLM.""" if not master_key: @@ -369,6 +381,8 @@ async def sync_adaptive_router_roster(master_key: str): logger.warning(f"Failed to fetch OpenRouter models: {e}") return free_models = [] + model_contexts = {} + model_supported_params = {} for m in all_models: mid = m.get("id", "") # Skip internal OpenRouter encoded IDs that LiteLLM can't map to a provider @@ -398,6 +412,8 @@ async def sync_adaptive_router_roster(master_key: str): except Exception: score = 25.0 # conservative fallback for unparseable models free_models.append((score, mid)) + model_contexts[mid] = m.get("context_length") or 262144 + model_supported_params[mid] = supported_params free_models.sort(reverse=True) if not free_models: logger.warning("No free models found — skipping roster sync") @@ -459,13 +475,7 @@ def norm(s: float) -> float: # starts clean — delete all, then register only the current roster. try: db_url = os.getenv("DATABASE_URL", "postgresql://postgres@127.0.0.1:5432/postgres") - import asyncpg - conn = await asyncpg.connect(db_url) - await conn.execute( - 'DELETE FROM "LiteLLM_ProxyModelTable" WHERE model_name LIKE $1', - 'agent-%' - ) - await conn.close() + await _purge_stale_deployments(db_url, 'agent-%') logger.info("🧹 Purged stale agent-* deployments before roster sync") except Exception as e: logger.warning(f"Failed to purge stale deployments (non-fatal): {e}") @@ -474,9 +484,20 @@ def norm(s: float) -> float: failed = 0 for tier_name, model_ids in tier_assignments.items(): for mid in model_ids: + ctx_len = model_contexts.get(mid, 262144) + sp = model_supported_params.get(mid, []) payload = { "model_name": tier_name, - "litellm_params": {"model": f"openrouter/{mid}", "request_timeout": 20} + "litellm_params": {"model": f"openrouter/{mid}", "request_timeout": 20}, + "model_info": { + "supports_vision": "vision" in sp, + "supports_reasoning": True, # OpenRouter API has no "reasoning" param; assume all modern LLMs support it + "supports_function_calling": "tools" in sp, + "mode": "chat", + "max_tokens": ctx_len, + "max_input_tokens": ctx_len, + "is_public_model_group": True + } } try: r = await client.post(f"{admin_url}/model/new", headers=headers, json=payload) @@ -490,6 +511,120 @@ def norm(s: float) -> float: logger.warning(f"Failed to register {mid} under {tier_name}: {e}") logger.info(f"📊 Roster sync: registered {registered} deployments ({failed} failed) across 5 tiers — {sum(len(v) for v in tier_assignments.values())} attempted") +async def _register_ollama_models_in_db(master_key: str): + """Register static ollama models via /model/new so they become DB models. + + LiteLLM's /model_group/info endpoint aggregates model info using its internal + model cost map for known providers. For ollama_chat models not in the map, + capabilities (vision, reasoning, function_calling) and token limits come back + as null/false. Registering them as DB models ensures our model_info wins. + """ + if not master_key: + logger.warning("No LiteLLM master key provided — skipping Ollama DB registration") + return + + admin_url = os.getenv("LITELLM_ADMIN_URL", "http://127.0.0.1:4000") + headers = {"Authorization": f"Bearer {master_key}", "Content-Type": "application/json"} + + ollama_models = [] + litellm_config_path = os.getenv("LITELLM_CONFIG_PATH", "/config/litellm_dir/config.yaml") + + config_paths_to_try = [ + litellm_config_path, + str(Path(__file__).resolve().parent.parent / "litellm" / "config.yaml"), + "./litellm/config.yaml" + ] + + loaded_from_config = False + for path in config_paths_to_try: + if path and os.path.exists(path): + try: + with open(path, "r") as f: + litellm_config = yaml.safe_load(f) + if litellm_config and "model_list" in litellm_config: + for item in litellm_config["model_list"]: + model_name = item.get("model_name", "") + if model_name.startswith("ollama-deepseek-"): + # Create a clean deep copy to avoid mutating configuration structures + ollama_models.append(copy.deepcopy(item)) + if ollama_models: + logger.info(f"Loaded {len(ollama_models)} Ollama model configurations dynamically from {path}") + loaded_from_config = True + break + except Exception as e: + logger.warning(f"Failed to load/parse LiteLLM config at {path}: {e}") + + if not loaded_from_config: + logger.warning("Could not load Ollama models from config.yaml, falling back to static definitions") + ollama_models = [ + { + "model_name": "ollama-deepseek-v4-pro", + "litellm_params": { + "model": "ollama_chat/deepseek-v4-pro", + "api_base": "https://api.ollama.com", + "api_key": "os.environ/OLLAMA_API_KEY", + "request_timeout": 120, + }, + "model_info": { + "supports_vision": True, + "supports_reasoning": True, + "supports_function_calling": True, + "mode": "chat", + "max_tokens": 524288, + "max_input_tokens": 524288, + "input_cost_per_token": 0.00000174, + "output_cost_per_token": 0.00000348, + "is_public_model_group": True, + }, + }, + { + "model_name": "ollama-deepseek-v4-flash", + "litellm_params": { + "model": "ollama_chat/deepseek-v4-flash", + "api_base": "https://api.ollama.com", + "api_key": "os.environ/OLLAMA_API_KEY", + "request_timeout": 120, + }, + "model_info": { + "supports_vision": True, + "supports_reasoning": True, + "supports_function_calling": True, + "mode": "chat", + "max_tokens": 524288, + "max_input_tokens": 524288, + "input_cost_per_token": 0.00000014, + "output_cost_per_token": 0.00000028, + "is_public_model_group": True, + }, + }, + ] + + # Purge stale ollama-deepseek DB entries before re-registering. + # Mirrors the agent-* purge pattern above — delete all, then register fresh. + try: + db_url = os.getenv("DATABASE_URL", "postgresql://postgres@127.0.0.1:5432/postgres") + await _purge_stale_deployments(db_url, 'ollama-deepseek-%') + logger.info("🧹 Purged stale ollama-deepseek-* DB entries before registration") + except Exception as e: + logger.warning(f"Failed to purge stale ollama DB entries (non-fatal): {e}") + + async with httpx.AsyncClient(timeout=10.0) as client: + registered = 0 + failed = 0 + for payload in ollama_models: + try: + r = await client.post(f"{admin_url}/model/new", headers=headers, json=payload) + if r.status_code in (200, 201): + registered += 1 + else: + failed += 1 + logger.warning(f"model/new {payload['model_name']}: HTTP {r.status_code} — {r.text[:200]}") + except Exception as e: + failed += 1 + logger.warning(f"Failed to register {payload['model_name']}: {e}") + logger.info(f"📊 Ollama DB registration: {registered} registered, {failed} failed") + + @asynccontextmanager async def lifespan(app: FastAPI): """Startup: wait for LiteLLM readiness, then sync free-model roster.""" @@ -521,6 +656,15 @@ async def lifespan(app: FastAPI): except Exception as e: logger.error(f"Roster sync failed: {e}") + # Register static ollama models via /model/new so they become DB models. + # The ollama_chat provider's internal model lookup overrides static config + # model_info at the group aggregation level (/model_group/info), causing + # features and token limits to show as null/false. DB models get priority. + try: + await _register_ollama_models_in_db(litellm_master_key) + except Exception as e: + logger.warning(f"Ollama DB registration failed (non-fatal): {e}") + # Start background task before yield so it runs during app lifetime task = asyncio.create_task(push_aggregate_scores()) @@ -1198,15 +1342,17 @@ async def proxy_models(): data = r.json() # Inject llm-routing-* models at the top of the list. # Auto models (classifier pipeline) first, then direct models. - # context_length: 262144 (256K) — the downstream models (gemini-3.5-flash, - # claude-opus-4.6, deepseek-v4-pro/flash) all support 256K+ context. + # Context lengths aligned with downstream model targets: + # - auto-free / auto-agy: 262144 (262K) + # - auto-ollama / auto-agy-ollama / llm-routing-ollama: 524288 (512K) + # - llm-routing-agy: 1048576 (1M) routing_models = [ {"id": "llm-routing-auto-free", "object": "model", "created": 0, "owned_by": "llm-routing", "context_length": 262144}, {"id": "llm-routing-auto-agy", "object": "model", "created": 0, "owned_by": "llm-routing", "context_length": 262144}, - {"id": "llm-routing-auto-ollama", "object": "model", "created": 0, "owned_by": "llm-routing", "context_length": 262144}, - {"id": "llm-routing-auto-agy-ollama", "object": "model", "created": 0, "owned_by": "llm-routing", "context_length": 262144}, - {"id": "llm-routing-agy", "object": "model", "created": 0, "owned_by": "llm-routing", "context_length": 262144}, - {"id": "llm-routing-ollama", "object": "model", "created": 0, "owned_by": "llm-routing", "context_length": 262144}, + {"id": "llm-routing-auto-ollama", "object": "model", "created": 0, "owned_by": "llm-routing", "context_length": 524288}, + {"id": "llm-routing-auto-agy-ollama", "object": "model", "created": 0, "owned_by": "llm-routing", "context_length": 524288}, + {"id": "llm-routing-agy", "object": "model", "created": 0, "owned_by": "llm-routing", "context_length": 1048576}, + {"id": "llm-routing-ollama", "object": "model", "created": 0, "owned_by": "llm-routing", "context_length": 524288}, ] for entry in reversed(routing_models): data["data"].insert(0, entry) diff --git a/scripts/verification/verify_direct_ollama_cooldown.py b/scripts/verification/verify_direct_ollama_cooldown.py index 0691a29f..e5fa9430 100644 --- a/scripts/verification/verify_direct_ollama_cooldown.py +++ b/scripts/verification/verify_direct_ollama_cooldown.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -import urllib.request import json import time import os import uuid import sys +import httpx # Resolve the absolute path to .env file in the workspace workspace_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -23,11 +23,11 @@ def get_triage_request_count(): try: - with urllib.request.urlopen(METRICS_URL, timeout=5) as response: - lines = response.read().decode("utf-8").splitlines() - for line in lines: - if line.startswith("triage_requests_total"): - return int(float(line.split()[1])) + response = httpx.get(METRICS_URL, timeout=5.0) + lines = response.text.splitlines() + for line in lines: + if line.startswith("triage_requests_total"): + return int(float(line.split()[1])) except Exception as e: print(f"Error fetching metrics: {e}") return 0 @@ -42,28 +42,26 @@ def send_litellm_request(model: str, prompt: str): "temperature": 0.0, "max_tokens": 10 } - data = json.dumps(payload).encode("utf-8") - req = urllib.request.Request( - LITELLM_URL, - data=data, - headers={"Content-Type": "application/json", "Authorization": f"Bearer {litellm_key}"} - ) start_time = time.time() try: - with urllib.request.urlopen(req, timeout=30) as response: - res_body = response.read().decode("utf-8") - result = json.loads(res_body) - model_returned = result.get("model", "unknown") - text = (result["choices"][0]["message"].get("content") or "").strip() - print(f"Success in {time.time() - start_time:.1f}s: model={model_returned}, text='{text[:40]}'") - return True, model_returned + response = httpx.post( + LITELLM_URL, + json=payload, + headers={"Authorization": f"Bearer {litellm_key}"}, + timeout=30.0 + ) + response.raise_for_status() + result = response.json() + model_returned = result.get("model", "unknown") + text = (result["choices"][0]["message"].get("content") or "").strip() + print(f"Success in {time.time() - start_time:.1f}s: model={model_returned}, text='{text[:40]}'") + return True, model_returned + except httpx.HTTPStatusError as e: + err_msg = f"{e} - {e.response.text}" + print(f"Failed in {time.time() - start_time:.1f}s: {err_msg}") + return False, err_msg except Exception as e: err_msg = str(e) - if hasattr(e, "read"): - try: - err_msg += " - " + e.read().decode("utf-8") - except Exception: - pass print(f"Failed in {time.time() - start_time:.1f}s: {err_msg}") return False, err_msg diff --git a/scripts/verification/verify_ollama_cooldown.py b/scripts/verification/verify_ollama_cooldown.py index ab31a881..6d0e33d9 100644 --- a/scripts/verification/verify_ollama_cooldown.py +++ b/scripts/verification/verify_ollama_cooldown.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -import urllib.request import json import time import os import uuid import sys +import httpx # Resolve the absolute path to .env file in the workspace workspace_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -25,11 +25,12 @@ def get_triage_request_count(): try: - with urllib.request.urlopen(METRICS_URL, timeout=5) as response: - lines = response.read().decode("utf-8").splitlines() - for line in lines: - if line.startswith("triage_requests_total"): - return int(float(line.split()[1])) + response = httpx.get(METRICS_URL, timeout=5.0) + response.raise_for_status() + lines = response.text.splitlines() + for line in lines: + if line.startswith("triage_requests_total"): + return int(float(line.split()[1])) except Exception as e: print(f"Error fetching metrics: {e}") return 0 @@ -44,29 +45,26 @@ def send_litellm_request(model: str, prompt: str): "temperature": 0.0, "max_tokens": 10 } - data = json.dumps(payload).encode("utf-8") - req = urllib.request.Request( - LITELLM_URL, - data=data, - headers={"Content-Type": "application/json", "Authorization": f"Bearer {litellm_key}"} - ) start_time = time.time() try: - with urllib.request.urlopen(req, timeout=30) as response: - res_body = response.read().decode("utf-8") - result = json.loads(res_body) - model_returned = result.get("model", "unknown") - text = (result["choices"][0]["message"].get("content") or "").strip() - print(f"Success in {time.time() - start_time:.1f}s: model={model_returned}, text='{text[:40]}'") - return True, model_returned + response = httpx.post( + LITELLM_URL, + json=payload, + headers={"Authorization": f"Bearer {litellm_key}"}, + timeout=30.0 + ) + response.raise_for_status() + result = response.json() + model_returned = result.get("model", "unknown") + text = (result["choices"][0]["message"].get("content") or "").strip() + print(f"Success in {time.time() - start_time:.1f}s: model={model_returned}, text='{text[:40]}'") + return True, model_returned + except httpx.HTTPStatusError as e: + err_msg = f"{e} - {e.response.text}" + print(f"Failed in {time.time() - start_time:.1f}s: {err_msg}") + return False, err_msg except Exception as e: - # Check if the error body is readable err_msg = str(e) - if hasattr(e, "read"): - try: - err_msg += " - " + e.read().decode("utf-8") - except Exception: - pass print(f"Failed in {time.time() - start_time:.1f}s: {err_msg}") return False, err_msg diff --git a/scripts/verification/verify_ollama_routing.py b/scripts/verification/verify_ollama_routing.py index 562d45b3..42e5e492 100644 --- a/scripts/verification/verify_ollama_routing.py +++ b/scripts/verification/verify_ollama_routing.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 -import urllib.request import json import sys import os +import httpx URL = "http://localhost:5000/v1/chat/completions" @@ -29,24 +29,23 @@ def send_request(model: str, prompt: str, expected_model: str): "temperature": 0.0, "max_tokens": 10 } - data = json.dumps(payload).encode("utf-8") - req = urllib.request.Request( - URL, - data=data, - headers={"Content-Type": "application/json", "Authorization": f"Bearer {litellm_key}"} - ) try: - with urllib.request.urlopen(req, timeout=30) as response: - res_body = response.read().decode("utf-8") - result = json.loads(res_body) - model_returned = result.get("model", "unknown") - text = (result["choices"][0]["message"].get("content") or "").strip() - print(f"Request: model={model}, prompt='{prompt[:40]}...'") - print(f"Response: model={model_returned}, text='{text[:60]}...'") - if model_returned != expected_model: - print(f"❌ FAILURE: Expected model '{expected_model}', but got '{model_returned}'", file=sys.stderr) - sys.exit(1) - print("✓ SUCCESS: Routed correctly!") + response = httpx.post( + URL, + json=payload, + headers={"Authorization": f"Bearer {litellm_key}"}, + timeout=30.0 + ) + response.raise_for_status() + result = response.json() + model_returned = result.get("model", "unknown") + text = (result["choices"][0]["message"].get("content") or "").strip() + print(f"Request: model={model}, prompt='{prompt[:40]}...'") + print(f"Response: model={model_returned}, text='{text[:60]}...'") + if model_returned != expected_model: + print(f"❌ FAILURE: Expected model '{expected_model}', but got '{model_returned}'", file=sys.stderr) + sys.exit(1) + print("✓ SUCCESS: Routed correctly!") except Exception as e: print(f"❌ ERROR: Request to model={model} failed or timed out: {e}", file=sys.stderr) sys.exit(1)