From 56d086710b75cefd7ef994647d79d9f4a4980725 Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Sat, 20 Jun 2026 01:29:29 +0200 Subject: [PATCH 1/3] feat: expose model capabilities, token limits, and costs in Model Hub Table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ci: add httpx to test workflow pip install for test_a2_verify.py - config: add public_model_groups list to litellm_settings so all agent-*, openrouter-auto, llm-routing-ollama, and ollama-deepseek-* groups appear in the LiteLLM Model Hub Table UI - config: add full model_info to all model_list entries (supports_vision, supports_reasoning, supports_function_calling, mode, max_tokens, max_input_tokens, is_public_model_group) - config: set llm-routing-ollama and ollama-deepseek-v4-{pro,flash} context windows to 512K (524288 tokens), up from 131K/262K - config: add DeepSeek API equivalent per-token costs to ollama models for Langfuse cost tracking: ollama-deepseek-v4-pro: $1.74/1M input, $3.48/1M output ollama-deepseek-v4-flash: $0.14/1M input, $0.28/1M output - router: enrich /model/new roster sync payload with model_info (features, context length from OpenRouter API, is_public_model_group) for all dynamically registered agent-* tiers - router: add _register_ollama_models_in_db() — registers ollama-deepseek models via /model/new at startup so their model_info wins over LiteLLM's internal ollama_chat provider lookup (which returns null/false for unknown models in /model_group/info aggregation) --- .github/workflows/test.yml | 3 ++ litellm/config.yaml | 86 +++++++++++++++++++++++++++++++ router/main.py | 101 ++++++++++++++++++++++++++++++++++++- 3 files changed, 189 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3d4c34e5..b74261ed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,6 +20,9 @@ jobs: with: python-version: '3.11' + - name: Install dependencies + run: pip install httpx + - name: Run Circuit Breaker Tests run: python3 test_circuit_breaker.py diff --git a/litellm/config.yaml b/litellm/config.yaml index b731a2b7..4fa42373 100644 --- a/litellm/config.yaml +++ b/litellm/config.yaml @@ -20,6 +20,16 @@ litellm_settings: - langfuse detailed_debug: false drop_params: true + public_model_groups: + - openrouter-auto + - llm-routing-ollama + - ollama-deepseek-v4-pro + - ollama-deepseek-v4-flash + - agent-advanced-core + - agent-reasoning-core + - agent-complex-core + - agent-medium-core + - agent-simple-core fallbacks: - agent-simple-core: - agent-medium-core @@ -57,12 +67,28 @@ model_list: model: openrouter/openrouter/auto request_timeout: 120 model_name: openrouter-auto + model_info: + supports_vision: true + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 2000000 + max_input_tokens: 2000000 + is_public_model_group: true - litellm_params: model: openai/llm-routing-ollama api_base: http://127.0.0.1:5000/v1 api_key: os.environ/LITELLM_MASTER_KEY request_timeout: 120 model_name: llm-routing-ollama + model_info: + supports_vision: true + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 524288 + max_input_tokens: 524288 + is_public_model_group: true # DISABLED 2026-06-08 — 20GB on disk, 23GB GTT (system RAM as GPU buffer). # Uncomment to re-enable once a lighter model replaces it. @@ -93,6 +119,16 @@ model_list: api_base: https://api.ollama.com api_key: os.environ/OLLAMA_API_KEY request_timeout: 120 + model_info: + supports_vision: true + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 524288 + max_input_tokens: 524288 + input_cost_per_token: 0.00000174 + output_cost_per_token: 0.00000348 + is_public_model_group: true # ================================================================================ # OLLAMA FLASH TIER — lighter/faster model for reasoning-tier requests. @@ -104,6 +140,16 @@ model_list: api_base: https://api.ollama.com api_key: os.environ/OLLAMA_API_KEY request_timeout: 120 + model_info: + supports_vision: true + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 524288 + max_input_tokens: 524288 + input_cost_per_token: 0.00000014 + output_cost_per_token: 0.00000028 + is_public_model_group: true # ================================================================================ # AGENT TIER DEPLOYMENTS — safety-net entries (one per tier) @@ -133,22 +179,62 @@ model_list: litellm_params: model: openrouter/google/gemma-4-31b-it:free request_timeout: 20 + model_info: + supports_vision: true + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 262144 + max_input_tokens: 262144 + is_public_model_group: true - model_name: agent-reasoning-core litellm_params: model: openrouter/google/gemma-4-26b-a4b-it:free request_timeout: 20 + model_info: + supports_vision: true + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 262144 + max_input_tokens: 262144 + is_public_model_group: true - model_name: agent-complex-core litellm_params: model: openrouter/nvidia/nemotron-3-ultra-550b-a55b:free request_timeout: 20 + model_info: + supports_vision: true + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 1000000 + max_input_tokens: 1000000 + is_public_model_group: true - model_name: agent-medium-core litellm_params: model: openrouter/google/gemma-4-26b-a4b-it:free request_timeout: 20 + model_info: + supports_vision: true + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 262144 + max_input_tokens: 262144 + is_public_model_group: true - model_name: agent-simple-core litellm_params: model: openrouter/nvidia/nemotron-3-nano-30b-a3b:free request_timeout: 20 + model_info: + supports_vision: true + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 256000 + max_input_tokens: 256000 + is_public_model_group: true redis_settings: redis_host: 127.0.0.1 diff --git a/router/main.py b/router/main.py index 8e1d7a92..78abb20d 100644 --- a/router/main.py +++ b/router/main.py @@ -369,6 +369,7 @@ async def sync_adaptive_router_roster(master_key: str): logger.warning(f"Failed to fetch OpenRouter models: {e}") return free_models = [] + model_contexts = {} for m in all_models: mid = m.get("id", "") # Skip internal OpenRouter encoded IDs that LiteLLM can't map to a provider @@ -398,6 +399,7 @@ async def sync_adaptive_router_roster(master_key: str): except Exception: score = 25.0 # conservative fallback for unparseable models free_models.append((score, mid)) + model_contexts[mid] = m.get("context_length") or 262144 free_models.sort(reverse=True) if not free_models: logger.warning("No free models found — skipping roster sync") @@ -474,9 +476,19 @@ def norm(s: float) -> float: failed = 0 for tier_name, model_ids in tier_assignments.items(): for mid in model_ids: + ctx_len = model_contexts.get(mid, 262144) payload = { "model_name": tier_name, - "litellm_params": {"model": f"openrouter/{mid}", "request_timeout": 20} + "litellm_params": {"model": f"openrouter/{mid}", "request_timeout": 20}, + "model_info": { + "supports_vision": True, + "supports_reasoning": True, + "supports_function_calling": True, + "mode": "chat", + "max_tokens": ctx_len, + "max_input_tokens": ctx_len, + "is_public_model_group": True + } } try: r = await client.post(f"{admin_url}/model/new", headers=headers, json=payload) @@ -490,6 +502,84 @@ def norm(s: float) -> float: logger.warning(f"Failed to register {mid} under {tier_name}: {e}") logger.info(f"📊 Roster sync: registered {registered} deployments ({failed} failed) across 5 tiers — {sum(len(v) for v in tier_assignments.values())} attempted") +async def _register_ollama_models_in_db(master_key: str): + """Register static ollama models via /model/new so they become DB models. + + LiteLLM's /model_group/info endpoint aggregates model info using its internal + model cost map for known providers. For ollama_chat models not in the map, + capabilities (vision, reasoning, function_calling) and token limits come back + as null/false. Registering them as DB models ensures our model_info wins. + """ + admin_url = "http://127.0.0.1:4000" + headers = {"Authorization": f"Bearer {master_key}", "Content-Type": "application/json"} + + ollama_models = [ + { + "model_name": "ollama-deepseek-v4-pro", + "litellm_params": { + "model": "ollama_chat/deepseek-v4-pro", + "api_base": "https://api.ollama.com", + "api_key": os.getenv("OLLAMA_API_KEY", ""), + "request_timeout": 120, + }, + "model_info": { + "supports_vision": True, + "supports_reasoning": True, + "supports_function_calling": True, + "mode": "chat", + "max_tokens": 524288, + "max_input_tokens": 524288, + "input_cost_per_token": 0.00000174, + "output_cost_per_token": 0.00000348, + "is_public_model_group": True, + }, + }, + { + "model_name": "ollama-deepseek-v4-flash", + "litellm_params": { + "model": "ollama_chat/deepseek-v4-flash", + "api_base": "https://api.ollama.com", + "api_key": os.getenv("OLLAMA_API_KEY", ""), + "request_timeout": 120, + }, + "model_info": { + "supports_vision": True, + "supports_reasoning": True, + "supports_function_calling": True, + "mode": "chat", + "max_tokens": 524288, + "max_input_tokens": 524288, + "input_cost_per_token": 0.00000014, + "output_cost_per_token": 0.00000028, + "is_public_model_group": True, + }, + }, + ] + + # Purge stale ollama-deepseek DB entries before re-registering + try: + db_url = os.getenv("DATABASE_URL", "postgresql://postgres@127.0.0.1:5432/postgres") + import asyncpg + conn = await asyncpg.connect(db_url) + await conn.execute( + 'DELETE FROM "LiteLLM_ProxyModelTable" WHERE model_name LIKE $1', + 'ollama-deepseek-%' + ) + await conn.close() + except Exception as e: + logger.warning(f"Failed to purge stale ollama DB entries (non-fatal): {e}") + + async with httpx.AsyncClient(timeout=10.0) as client: + for payload in ollama_models: + try: + r = await client.post(f"{admin_url}/model/new", headers=headers, json=payload) + if r.status_code in (200, 201): + logger.info(f"✅ Registered {payload['model_name']} in DB") + else: + logger.warning(f"model/new {payload['model_name']}: HTTP {r.status_code} — {r.text[:200]}") + except Exception as e: + logger.warning(f"Failed to register {payload['model_name']}: {e}") + @asynccontextmanager async def lifespan(app: FastAPI): """Startup: wait for LiteLLM readiness, then sync free-model roster.""" @@ -521,6 +611,15 @@ async def lifespan(app: FastAPI): except Exception as e: logger.error(f"Roster sync failed: {e}") + # Register static ollama models via /model/new so they become DB models. + # The ollama_chat provider's internal model lookup overrides static config + # model_info at the group aggregation level (/model_group/info), causing + # features and token limits to show as null/false. DB models get priority. + try: + await _register_ollama_models_in_db(litellm_master_key) + except Exception as e: + logger.warning(f"Ollama DB registration failed (non-fatal): {e}") + # Start background task before yield so it runs during app lifetime task = asyncio.create_task(push_aggregate_scores()) From debdb08ef1e348d9bf84dc045bde70fb2bf6d3b1 Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Sat, 20 Jun 2026 04:29:15 +0200 Subject: [PATCH 2/3] chore: address review feedback on PR #14 (DRY purge helper, safety-net capabilities, align returned context lengths, and refactor verify scripts to httpx) --- README.md | 23 ++++--- litellm/config.yaml | 14 ++-- router/free_models_roster.json | 2 +- router/main.py | 68 +++++++++++-------- .../verify_direct_ollama_cooldown.py | 46 ++++++------- .../verification/verify_ollama_cooldown.py | 47 ++++++------- scripts/verification/verify_ollama_routing.py | 35 +++++----- 7 files changed, 122 insertions(+), 113 deletions(-) diff --git a/README.md b/README.md index 4ce30b3b..ade210f3 100644 --- a/README.md +++ b/README.md @@ -253,18 +253,21 @@ Exposes the entry endpoint (`http://localhost:5000/v1`) and evaluates prompt com | Model | Classifier | Premium backend | Fallback | Context Length | |:---|---:|:---|:---|:---| -| `llm-routing-auto-free` | ✅ | — | LiteLLM with classified tier | 256K | -| `llm-routing-auto-agy` | ✅ | agy (gated: reasoning → gemini-3.5-flash, advanced → gemini-3.5-flash → claude-opus-4.6) | LiteLLM with classified tier | 256K | -| `llm-routing-auto-ollama` | ✅ | Ollama (gated: reasoning & advanced → ollama-deepseek-v4-pro, complex → ollama-deepseek-v4-flash, below → bypass) | LiteLLM with classified tier | 256K | -| `llm-routing-auto-agy-ollama` | ✅ | agy → Ollama (gated: reasoning/advanced/complex) | LiteLLM with classified tier | 256K | -| `llm-routing-agy` | ❌ | agy (Gemini/Claude) — unconditional | LiteLLM agent-advanced-core | 256K | -| `llm-routing-ollama` | ✅ | Ollama (gated: reasoning & advanced → ollama-deepseek-v4-pro, complex & below → ollama-deepseek-v4-flash) | LiteLLM agent-advanced-core / agent-reasoning-core | 256K | -| `agent-advanced-core` | ❌ | — | LiteLLM openrouter-auto | 256K | -| `agent-reasoning-core` | ❌ | — | LiteLLM fallback chain | 256K | -| `agent-complex-core` | ❌ | — | LiteLLM fallback chain | 256K | -| `agent-medium-core` | ❌ | — | LiteLLM fallback chain | 256K | +| `llm-routing-auto-free` | ✅ | — | LiteLLM with classified tier | 262K | +| `llm-routing-auto-agy` | ✅ | agy (gated: reasoning → gemini-3.5-flash, advanced → gemini-3.5-flash → claude-opus-4.6) | LiteLLM with classified tier | 262K | +| `llm-routing-auto-ollama` | ✅ | Ollama (gated: reasoning & advanced → ollama-deepseek-v4-pro, complex → ollama-deepseek-v4-flash, below → bypass) | LiteLLM with classified tier | 512K | +| `llm-routing-auto-agy-ollama` | ✅ | agy → Ollama (gated: reasoning/advanced/complex) | LiteLLM with classified tier | 512K | +| `llm-routing-agy` | ❌ | agy (Gemini/Claude) — unconditional | LiteLLM agent-advanced-core | 1M | +| `llm-routing-ollama` | ✅ | Ollama (gated: reasoning & advanced → ollama-deepseek-v4-pro, complex & below → ollama-deepseek-v4-flash) | LiteLLM agent-advanced-core / agent-reasoning-core | 512K | +| `agent-advanced-core` | ❌ | — | LiteLLM openrouter-auto | 262K | +| `agent-reasoning-core` | ❌ | — | LiteLLM fallback chain | 262K | +| `agent-complex-core` | ❌ | — | LiteLLM fallback chain | 262K | +| `agent-medium-core` | ❌ | — | LiteLLM fallback chain | 262K | | `agent-simple-core` | ❌ | — | LiteLLM fallback chain | 256K | +> [!TIP] +> Model capabilities, token limits, and costs are visible in LiteLLM's Model Hub Table at `http://localhost:4000/ui/?page=model-hub-table` (or port 4000 on the gateway host). + ### B. LiteLLM Proxy Gateway (`litellm/config.yaml`) - **Version Pinning**: The LiteLLM gateway runs `ghcr.io/berriai/litellm:v1.88.0` (latest stable as of June 2026). The tag is explicitly pinned in `pod.yaml` — never use `:latest`. Check available tags with `skopeo list-tags docker://ghcr.io/berriai/litellm` before upgrading. ClickHouse runs `docker.io/clickhouse/clickhouse-server:26.5.1` (upgraded from 24.8, June 2026). Valkey Cache runs `docker.io/valkey/valkey:9.1.0-alpine` (upgraded from 8, June 2026). Orchestrates routing fallback chains, Redis caching, and telemetry callbacks: diff --git a/litellm/config.yaml b/litellm/config.yaml index 4fa42373..4a791e45 100644 --- a/litellm/config.yaml +++ b/litellm/config.yaml @@ -180,7 +180,7 @@ model_list: model: openrouter/google/gemma-4-31b-it:free request_timeout: 20 model_info: - supports_vision: true + supports_vision: false supports_reasoning: true supports_function_calling: true mode: chat @@ -192,7 +192,7 @@ model_list: model: openrouter/google/gemma-4-26b-a4b-it:free request_timeout: 20 model_info: - supports_vision: true + supports_vision: false supports_reasoning: true supports_function_calling: true mode: chat @@ -204,19 +204,19 @@ model_list: model: openrouter/nvidia/nemotron-3-ultra-550b-a55b:free request_timeout: 20 model_info: - supports_vision: true + supports_vision: false supports_reasoning: true supports_function_calling: true mode: chat - max_tokens: 1000000 - max_input_tokens: 1000000 + max_tokens: 262144 + max_input_tokens: 262144 is_public_model_group: true - model_name: agent-medium-core litellm_params: model: openrouter/google/gemma-4-26b-a4b-it:free request_timeout: 20 model_info: - supports_vision: true + supports_vision: false supports_reasoning: true supports_function_calling: true mode: chat @@ -228,7 +228,7 @@ model_list: model: openrouter/nvidia/nemotron-3-nano-30b-a3b:free request_timeout: 20 model_info: - supports_vision: true + supports_vision: false supports_reasoning: true supports_function_calling: true mode: chat diff --git a/router/free_models_roster.json b/router/free_models_roster.json index 2d4e8998..1e050d29 100644 --- a/router/free_models_roster.json +++ b/router/free_models_roster.json @@ -145,6 +145,6 @@ "context_length": 32768 } ], - "updated_at": "2026-06-19T22:36:57.045439Z", + "updated_at": "2026-06-20T02:28:14.044490Z", "count": 24 } \ No newline at end of file diff --git a/router/main.py b/router/main.py index 78abb20d..26913ecb 100644 --- a/router/main.py +++ b/router/main.py @@ -351,6 +351,18 @@ async def save_persisted_stats(force=False): CACHE_TTL_SECONDS = 86400 # Decisions cached for 24 hours classification_lock = asyncio.Lock() +async def _purge_stale_deployments(db_url: str, pattern: str): + """Purge stale deployments matching the pattern from LiteLLM's DB.""" + import asyncpg + conn = await asyncpg.connect(db_url) + try: + await conn.execute( + 'DELETE FROM "LiteLLM_ProxyModelTable" WHERE model_name LIKE $1', + pattern + ) + finally: + await conn.close() + async def sync_adaptive_router_roster(master_key: str): """Fetch free OpenRouter models and register them as deployments in LiteLLM.""" if not master_key: @@ -370,6 +382,7 @@ async def sync_adaptive_router_roster(master_key: str): return free_models = [] model_contexts = {} + model_supported_params = {} for m in all_models: mid = m.get("id", "") # Skip internal OpenRouter encoded IDs that LiteLLM can't map to a provider @@ -400,6 +413,7 @@ async def sync_adaptive_router_roster(master_key: str): score = 25.0 # conservative fallback for unparseable models free_models.append((score, mid)) model_contexts[mid] = m.get("context_length") or 262144 + model_supported_params[mid] = supported_params free_models.sort(reverse=True) if not free_models: logger.warning("No free models found — skipping roster sync") @@ -461,13 +475,7 @@ def norm(s: float) -> float: # starts clean — delete all, then register only the current roster. try: db_url = os.getenv("DATABASE_URL", "postgresql://postgres@127.0.0.1:5432/postgres") - import asyncpg - conn = await asyncpg.connect(db_url) - await conn.execute( - 'DELETE FROM "LiteLLM_ProxyModelTable" WHERE model_name LIKE $1', - 'agent-%' - ) - await conn.close() + await _purge_stale_deployments(db_url, 'agent-%') logger.info("🧹 Purged stale agent-* deployments before roster sync") except Exception as e: logger.warning(f"Failed to purge stale deployments (non-fatal): {e}") @@ -477,13 +485,14 @@ def norm(s: float) -> float: for tier_name, model_ids in tier_assignments.items(): for mid in model_ids: ctx_len = model_contexts.get(mid, 262144) + sp = model_supported_params.get(mid, []) payload = { "model_name": tier_name, "litellm_params": {"model": f"openrouter/{mid}", "request_timeout": 20}, "model_info": { - "supports_vision": True, - "supports_reasoning": True, - "supports_function_calling": True, + "supports_vision": "vision" in sp, + "supports_reasoning": True, # OpenRouter API has no "reasoning" param; assume all modern LLMs support it + "supports_function_calling": "tools" in sp, "mode": "chat", "max_tokens": ctx_len, "max_input_tokens": ctx_len, @@ -519,7 +528,7 @@ async def _register_ollama_models_in_db(master_key: str): "litellm_params": { "model": "ollama_chat/deepseek-v4-pro", "api_base": "https://api.ollama.com", - "api_key": os.getenv("OLLAMA_API_KEY", ""), + "api_key": "os.environ/OLLAMA_API_KEY", "request_timeout": 120, }, "model_info": { @@ -539,7 +548,7 @@ async def _register_ollama_models_in_db(master_key: str): "litellm_params": { "model": "ollama_chat/deepseek-v4-flash", "api_base": "https://api.ollama.com", - "api_key": os.getenv("OLLAMA_API_KEY", ""), + "api_key": "os.environ/OLLAMA_API_KEY", "request_timeout": 120, }, "model_info": { @@ -555,30 +564,31 @@ async def _register_ollama_models_in_db(master_key: str): }, }, ] - - # Purge stale ollama-deepseek DB entries before re-registering + # Purge stale ollama-deepseek DB entries before re-registering. + # Mirrors the agent-* purge pattern above — delete all, then register fresh. try: db_url = os.getenv("DATABASE_URL", "postgresql://postgres@127.0.0.1:5432/postgres") - import asyncpg - conn = await asyncpg.connect(db_url) - await conn.execute( - 'DELETE FROM "LiteLLM_ProxyModelTable" WHERE model_name LIKE $1', - 'ollama-deepseek-%' - ) - await conn.close() + await _purge_stale_deployments(db_url, 'ollama-deepseek-%') + logger.info("🧹 Purged stale ollama-deepseek-* DB entries before registration") except Exception as e: logger.warning(f"Failed to purge stale ollama DB entries (non-fatal): {e}") async with httpx.AsyncClient(timeout=10.0) as client: + registered = 0 + failed = 0 for payload in ollama_models: try: r = await client.post(f"{admin_url}/model/new", headers=headers, json=payload) if r.status_code in (200, 201): - logger.info(f"✅ Registered {payload['model_name']} in DB") + registered += 1 else: + failed += 1 logger.warning(f"model/new {payload['model_name']}: HTTP {r.status_code} — {r.text[:200]}") except Exception as e: + failed += 1 logger.warning(f"Failed to register {payload['model_name']}: {e}") + logger.info(f"📊 Ollama DB registration: {registered} registered, {failed} failed") + @asynccontextmanager async def lifespan(app: FastAPI): @@ -1297,15 +1307,17 @@ async def proxy_models(): data = r.json() # Inject llm-routing-* models at the top of the list. # Auto models (classifier pipeline) first, then direct models. - # context_length: 262144 (256K) — the downstream models (gemini-3.5-flash, - # claude-opus-4.6, deepseek-v4-pro/flash) all support 256K+ context. + # Context lengths aligned with downstream model targets: + # - auto-free / auto-agy: 262144 (262K) + # - auto-ollama / auto-agy-ollama / llm-routing-ollama: 524288 (512K) + # - llm-routing-agy: 1048576 (1M) routing_models = [ {"id": "llm-routing-auto-free", "object": "model", "created": 0, "owned_by": "llm-routing", "context_length": 262144}, {"id": "llm-routing-auto-agy", "object": "model", "created": 0, "owned_by": "llm-routing", "context_length": 262144}, - {"id": "llm-routing-auto-ollama", "object": "model", "created": 0, "owned_by": "llm-routing", "context_length": 262144}, - {"id": "llm-routing-auto-agy-ollama", "object": "model", "created": 0, "owned_by": "llm-routing", "context_length": 262144}, - {"id": "llm-routing-agy", "object": "model", "created": 0, "owned_by": "llm-routing", "context_length": 262144}, - {"id": "llm-routing-ollama", "object": "model", "created": 0, "owned_by": "llm-routing", "context_length": 262144}, + {"id": "llm-routing-auto-ollama", "object": "model", "created": 0, "owned_by": "llm-routing", "context_length": 524288}, + {"id": "llm-routing-auto-agy-ollama", "object": "model", "created": 0, "owned_by": "llm-routing", "context_length": 524288}, + {"id": "llm-routing-agy", "object": "model", "created": 0, "owned_by": "llm-routing", "context_length": 1048576}, + {"id": "llm-routing-ollama", "object": "model", "created": 0, "owned_by": "llm-routing", "context_length": 524288}, ] for entry in reversed(routing_models): data["data"].insert(0, entry) diff --git a/scripts/verification/verify_direct_ollama_cooldown.py b/scripts/verification/verify_direct_ollama_cooldown.py index 0691a29f..e5fa9430 100644 --- a/scripts/verification/verify_direct_ollama_cooldown.py +++ b/scripts/verification/verify_direct_ollama_cooldown.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -import urllib.request import json import time import os import uuid import sys +import httpx # Resolve the absolute path to .env file in the workspace workspace_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -23,11 +23,11 @@ def get_triage_request_count(): try: - with urllib.request.urlopen(METRICS_URL, timeout=5) as response: - lines = response.read().decode("utf-8").splitlines() - for line in lines: - if line.startswith("triage_requests_total"): - return int(float(line.split()[1])) + response = httpx.get(METRICS_URL, timeout=5.0) + lines = response.text.splitlines() + for line in lines: + if line.startswith("triage_requests_total"): + return int(float(line.split()[1])) except Exception as e: print(f"Error fetching metrics: {e}") return 0 @@ -42,28 +42,26 @@ def send_litellm_request(model: str, prompt: str): "temperature": 0.0, "max_tokens": 10 } - data = json.dumps(payload).encode("utf-8") - req = urllib.request.Request( - LITELLM_URL, - data=data, - headers={"Content-Type": "application/json", "Authorization": f"Bearer {litellm_key}"} - ) start_time = time.time() try: - with urllib.request.urlopen(req, timeout=30) as response: - res_body = response.read().decode("utf-8") - result = json.loads(res_body) - model_returned = result.get("model", "unknown") - text = (result["choices"][0]["message"].get("content") or "").strip() - print(f"Success in {time.time() - start_time:.1f}s: model={model_returned}, text='{text[:40]}'") - return True, model_returned + response = httpx.post( + LITELLM_URL, + json=payload, + headers={"Authorization": f"Bearer {litellm_key}"}, + timeout=30.0 + ) + response.raise_for_status() + result = response.json() + model_returned = result.get("model", "unknown") + text = (result["choices"][0]["message"].get("content") or "").strip() + print(f"Success in {time.time() - start_time:.1f}s: model={model_returned}, text='{text[:40]}'") + return True, model_returned + except httpx.HTTPStatusError as e: + err_msg = f"{e} - {e.response.text}" + print(f"Failed in {time.time() - start_time:.1f}s: {err_msg}") + return False, err_msg except Exception as e: err_msg = str(e) - if hasattr(e, "read"): - try: - err_msg += " - " + e.read().decode("utf-8") - except Exception: - pass print(f"Failed in {time.time() - start_time:.1f}s: {err_msg}") return False, err_msg diff --git a/scripts/verification/verify_ollama_cooldown.py b/scripts/verification/verify_ollama_cooldown.py index ab31a881..0bd3c265 100644 --- a/scripts/verification/verify_ollama_cooldown.py +++ b/scripts/verification/verify_ollama_cooldown.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -import urllib.request import json import time import os import uuid import sys +import httpx # Resolve the absolute path to .env file in the workspace workspace_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -25,11 +25,11 @@ def get_triage_request_count(): try: - with urllib.request.urlopen(METRICS_URL, timeout=5) as response: - lines = response.read().decode("utf-8").splitlines() - for line in lines: - if line.startswith("triage_requests_total"): - return int(float(line.split()[1])) + response = httpx.get(METRICS_URL, timeout=5.0) + lines = response.text.splitlines() + for line in lines: + if line.startswith("triage_requests_total"): + return int(float(line.split()[1])) except Exception as e: print(f"Error fetching metrics: {e}") return 0 @@ -44,29 +44,26 @@ def send_litellm_request(model: str, prompt: str): "temperature": 0.0, "max_tokens": 10 } - data = json.dumps(payload).encode("utf-8") - req = urllib.request.Request( - LITELLM_URL, - data=data, - headers={"Content-Type": "application/json", "Authorization": f"Bearer {litellm_key}"} - ) start_time = time.time() try: - with urllib.request.urlopen(req, timeout=30) as response: - res_body = response.read().decode("utf-8") - result = json.loads(res_body) - model_returned = result.get("model", "unknown") - text = (result["choices"][0]["message"].get("content") or "").strip() - print(f"Success in {time.time() - start_time:.1f}s: model={model_returned}, text='{text[:40]}'") - return True, model_returned + response = httpx.post( + LITELLM_URL, + json=payload, + headers={"Authorization": f"Bearer {litellm_key}"}, + timeout=30.0 + ) + response.raise_for_status() + result = response.json() + model_returned = result.get("model", "unknown") + text = (result["choices"][0]["message"].get("content") or "").strip() + print(f"Success in {time.time() - start_time:.1f}s: model={model_returned}, text='{text[:40]}'") + return True, model_returned + except httpx.HTTPStatusError as e: + err_msg = f"{e} - {e.response.text}" + print(f"Failed in {time.time() - start_time:.1f}s: {err_msg}") + return False, err_msg except Exception as e: - # Check if the error body is readable err_msg = str(e) - if hasattr(e, "read"): - try: - err_msg += " - " + e.read().decode("utf-8") - except Exception: - pass print(f"Failed in {time.time() - start_time:.1f}s: {err_msg}") return False, err_msg diff --git a/scripts/verification/verify_ollama_routing.py b/scripts/verification/verify_ollama_routing.py index 562d45b3..42e5e492 100644 --- a/scripts/verification/verify_ollama_routing.py +++ b/scripts/verification/verify_ollama_routing.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 -import urllib.request import json import sys import os +import httpx URL = "http://localhost:5000/v1/chat/completions" @@ -29,24 +29,23 @@ def send_request(model: str, prompt: str, expected_model: str): "temperature": 0.0, "max_tokens": 10 } - data = json.dumps(payload).encode("utf-8") - req = urllib.request.Request( - URL, - data=data, - headers={"Content-Type": "application/json", "Authorization": f"Bearer {litellm_key}"} - ) try: - with urllib.request.urlopen(req, timeout=30) as response: - res_body = response.read().decode("utf-8") - result = json.loads(res_body) - model_returned = result.get("model", "unknown") - text = (result["choices"][0]["message"].get("content") or "").strip() - print(f"Request: model={model}, prompt='{prompt[:40]}...'") - print(f"Response: model={model_returned}, text='{text[:60]}...'") - if model_returned != expected_model: - print(f"❌ FAILURE: Expected model '{expected_model}', but got '{model_returned}'", file=sys.stderr) - sys.exit(1) - print("✓ SUCCESS: Routed correctly!") + response = httpx.post( + URL, + json=payload, + headers={"Authorization": f"Bearer {litellm_key}"}, + timeout=30.0 + ) + response.raise_for_status() + result = response.json() + model_returned = result.get("model", "unknown") + text = (result["choices"][0]["message"].get("content") or "").strip() + print(f"Request: model={model}, prompt='{prompt[:40]}...'") + print(f"Response: model={model_returned}, text='{text[:60]}...'") + if model_returned != expected_model: + print(f"❌ FAILURE: Expected model '{expected_model}', but got '{model_returned}'", file=sys.stderr) + sys.exit(1) + print("✓ SUCCESS: Routed correctly!") except Exception as e: print(f"❌ ERROR: Request to model={model} failed or timed out: {e}", file=sys.stderr) sys.exit(1) From 7a565d61a84ce57ff0dfeef072efd7e6a86e95ce Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Sat, 20 Jun 2026 04:38:13 +0200 Subject: [PATCH 3/3] chore: address PR #15 code review fixes and fix CI workflow triggers --- .github/workflows/test.yml | 1 - pod.yaml | 4 + router/main.py | 117 ++++++++++++------ .../verification/verify_ollama_cooldown.py | 1 + 4 files changed, 81 insertions(+), 42 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b74261ed..7b10946f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,7 +4,6 @@ on: push: branches: [ master ] pull_request: - branches: [ master ] jobs: test: diff --git a/pod.yaml b/pod.yaml index bad4946c..a0bda21c 100644 --- a/pod.yaml +++ b/pod.yaml @@ -90,6 +90,8 @@ spec: env: - name: CONFIG_PATH value: /config/router_dir/config.yaml + - name: LITELLM_CONFIG_PATH + value: /config/litellm_dir/config.yaml - name: DBUS_SESSION_BUS_ADDRESS value: unix:path=/run/user/1000/bus - name: LITELLM_MASTER_KEY @@ -127,6 +129,8 @@ spec: volumeMounts: - mountPath: /config/router_dir name: router-config + - mountPath: /config/litellm_dir + name: litellm-config - mountPath: /app/data name: dataset-data - mountPath: /config/gemini_auth diff --git a/router/main.py b/router/main.py index 26913ecb..1f01925b 100644 --- a/router/main.py +++ b/router/main.py @@ -519,51 +519,86 @@ async def _register_ollama_models_in_db(master_key: str): capabilities (vision, reasoning, function_calling) and token limits come back as null/false. Registering them as DB models ensures our model_info wins. """ - admin_url = "http://127.0.0.1:4000" + if not master_key: + logger.warning("No LiteLLM master key provided — skipping Ollama DB registration") + return + + admin_url = os.getenv("LITELLM_ADMIN_URL", "http://127.0.0.1:4000") headers = {"Authorization": f"Bearer {master_key}", "Content-Type": "application/json"} - ollama_models = [ - { - "model_name": "ollama-deepseek-v4-pro", - "litellm_params": { - "model": "ollama_chat/deepseek-v4-pro", - "api_base": "https://api.ollama.com", - "api_key": "os.environ/OLLAMA_API_KEY", - "request_timeout": 120, - }, - "model_info": { - "supports_vision": True, - "supports_reasoning": True, - "supports_function_calling": True, - "mode": "chat", - "max_tokens": 524288, - "max_input_tokens": 524288, - "input_cost_per_token": 0.00000174, - "output_cost_per_token": 0.00000348, - "is_public_model_group": True, - }, - }, - { - "model_name": "ollama-deepseek-v4-flash", - "litellm_params": { - "model": "ollama_chat/deepseek-v4-flash", - "api_base": "https://api.ollama.com", - "api_key": "os.environ/OLLAMA_API_KEY", - "request_timeout": 120, + ollama_models = [] + litellm_config_path = os.getenv("LITELLM_CONFIG_PATH", "/config/litellm_dir/config.yaml") + + config_paths_to_try = [ + litellm_config_path, + str(Path(__file__).resolve().parent.parent / "litellm" / "config.yaml"), + "./litellm/config.yaml" + ] + + loaded_from_config = False + for path in config_paths_to_try: + if path and os.path.exists(path): + try: + with open(path, "r") as f: + litellm_config = yaml.safe_load(f) + if litellm_config and "model_list" in litellm_config: + for item in litellm_config["model_list"]: + model_name = item.get("model_name", "") + if model_name.startswith("ollama-deepseek-"): + # Create a clean deep copy to avoid mutating configuration structures + ollama_models.append(copy.deepcopy(item)) + if ollama_models: + logger.info(f"Loaded {len(ollama_models)} Ollama model configurations dynamically from {path}") + loaded_from_config = True + break + except Exception as e: + logger.warning(f"Failed to load/parse LiteLLM config at {path}: {e}") + + if not loaded_from_config: + logger.warning("Could not load Ollama models from config.yaml, falling back to static definitions") + ollama_models = [ + { + "model_name": "ollama-deepseek-v4-pro", + "litellm_params": { + "model": "ollama_chat/deepseek-v4-pro", + "api_base": "https://api.ollama.com", + "api_key": "os.environ/OLLAMA_API_KEY", + "request_timeout": 120, + }, + "model_info": { + "supports_vision": True, + "supports_reasoning": True, + "supports_function_calling": True, + "mode": "chat", + "max_tokens": 524288, + "max_input_tokens": 524288, + "input_cost_per_token": 0.00000174, + "output_cost_per_token": 0.00000348, + "is_public_model_group": True, + }, }, - "model_info": { - "supports_vision": True, - "supports_reasoning": True, - "supports_function_calling": True, - "mode": "chat", - "max_tokens": 524288, - "max_input_tokens": 524288, - "input_cost_per_token": 0.00000014, - "output_cost_per_token": 0.00000028, - "is_public_model_group": True, + { + "model_name": "ollama-deepseek-v4-flash", + "litellm_params": { + "model": "ollama_chat/deepseek-v4-flash", + "api_base": "https://api.ollama.com", + "api_key": "os.environ/OLLAMA_API_KEY", + "request_timeout": 120, + }, + "model_info": { + "supports_vision": True, + "supports_reasoning": True, + "supports_function_calling": True, + "mode": "chat", + "max_tokens": 524288, + "max_input_tokens": 524288, + "input_cost_per_token": 0.00000014, + "output_cost_per_token": 0.00000028, + "is_public_model_group": True, + }, }, - }, - ] + ] + # Purge stale ollama-deepseek DB entries before re-registering. # Mirrors the agent-* purge pattern above — delete all, then register fresh. try: diff --git a/scripts/verification/verify_ollama_cooldown.py b/scripts/verification/verify_ollama_cooldown.py index 0bd3c265..6d0e33d9 100644 --- a/scripts/verification/verify_ollama_cooldown.py +++ b/scripts/verification/verify_ollama_cooldown.py @@ -26,6 +26,7 @@ def get_triage_request_count(): try: response = httpx.get(METRICS_URL, timeout=5.0) + response.raise_for_status() lines = response.text.splitlines() for line in lines: if line.startswith("triage_requests_total"):