diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3d4c34e5..b74261ed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,6 +20,9 @@ jobs: with: python-version: '3.11' + - name: Install dependencies + run: pip install httpx + - name: Run Circuit Breaker Tests run: python3 test_circuit_breaker.py diff --git a/litellm/config.yaml b/litellm/config.yaml index b731a2b7..4fa42373 100644 --- a/litellm/config.yaml +++ b/litellm/config.yaml @@ -20,6 +20,16 @@ litellm_settings: - langfuse detailed_debug: false drop_params: true + public_model_groups: + - openrouter-auto + - llm-routing-ollama + - ollama-deepseek-v4-pro + - ollama-deepseek-v4-flash + - agent-advanced-core + - agent-reasoning-core + - agent-complex-core + - agent-medium-core + - agent-simple-core fallbacks: - agent-simple-core: - agent-medium-core @@ -57,12 +67,28 @@ model_list: model: openrouter/openrouter/auto request_timeout: 120 model_name: openrouter-auto + model_info: + supports_vision: true + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 2000000 + max_input_tokens: 2000000 + is_public_model_group: true - litellm_params: model: openai/llm-routing-ollama api_base: http://127.0.0.1:5000/v1 api_key: os.environ/LITELLM_MASTER_KEY request_timeout: 120 model_name: llm-routing-ollama + model_info: + supports_vision: true + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 524288 + max_input_tokens: 524288 + is_public_model_group: true # DISABLED 2026-06-08 — 20GB on disk, 23GB GTT (system RAM as GPU buffer). # Uncomment to re-enable once a lighter model replaces it. @@ -93,6 +119,16 @@ model_list: api_base: https://api.ollama.com api_key: os.environ/OLLAMA_API_KEY request_timeout: 120 + model_info: + supports_vision: true + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 524288 + max_input_tokens: 524288 + input_cost_per_token: 0.00000174 + output_cost_per_token: 0.00000348 + is_public_model_group: true # ================================================================================ # OLLAMA FLASH TIER — lighter/faster model for reasoning-tier requests. @@ -104,6 +140,16 @@ model_list: api_base: https://api.ollama.com api_key: os.environ/OLLAMA_API_KEY request_timeout: 120 + model_info: + supports_vision: true + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 524288 + max_input_tokens: 524288 + input_cost_per_token: 0.00000014 + output_cost_per_token: 0.00000028 + is_public_model_group: true # ================================================================================ # AGENT TIER DEPLOYMENTS — safety-net entries (one per tier) @@ -133,22 +179,62 @@ model_list: litellm_params: model: openrouter/google/gemma-4-31b-it:free request_timeout: 20 + model_info: + supports_vision: true + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 262144 + max_input_tokens: 262144 + is_public_model_group: true - model_name: agent-reasoning-core litellm_params: model: openrouter/google/gemma-4-26b-a4b-it:free request_timeout: 20 + model_info: + supports_vision: true + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 262144 + max_input_tokens: 262144 + is_public_model_group: true - model_name: agent-complex-core litellm_params: model: openrouter/nvidia/nemotron-3-ultra-550b-a55b:free request_timeout: 20 + model_info: + supports_vision: true + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 1000000 + max_input_tokens: 1000000 + is_public_model_group: true - model_name: agent-medium-core litellm_params: model: openrouter/google/gemma-4-26b-a4b-it:free request_timeout: 20 + model_info: + supports_vision: true + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 262144 + max_input_tokens: 262144 + is_public_model_group: true - model_name: agent-simple-core litellm_params: model: openrouter/nvidia/nemotron-3-nano-30b-a3b:free request_timeout: 20 + model_info: + supports_vision: true + supports_reasoning: true + supports_function_calling: true + mode: chat + max_tokens: 256000 + max_input_tokens: 256000 + is_public_model_group: true redis_settings: redis_host: 127.0.0.1 diff --git a/router/main.py b/router/main.py index 8e1d7a92..78abb20d 100644 --- a/router/main.py +++ b/router/main.py @@ -369,6 +369,7 @@ async def sync_adaptive_router_roster(master_key: str): logger.warning(f"Failed to fetch OpenRouter models: {e}") return free_models = [] + model_contexts = {} for m in all_models: mid = m.get("id", "") # Skip internal OpenRouter encoded IDs that LiteLLM can't map to a provider @@ -398,6 +399,7 @@ async def sync_adaptive_router_roster(master_key: str): except Exception: score = 25.0 # conservative fallback for unparseable models free_models.append((score, mid)) + model_contexts[mid] = m.get("context_length") or 262144 free_models.sort(reverse=True) if not free_models: logger.warning("No free models found — skipping roster sync") @@ -474,9 +476,19 @@ def norm(s: float) -> float: failed = 0 for tier_name, model_ids in tier_assignments.items(): for mid in model_ids: + ctx_len = model_contexts.get(mid, 262144) payload = { "model_name": tier_name, - "litellm_params": {"model": f"openrouter/{mid}", "request_timeout": 20} + "litellm_params": {"model": f"openrouter/{mid}", "request_timeout": 20}, + "model_info": { + "supports_vision": True, + "supports_reasoning": True, + "supports_function_calling": True, + "mode": "chat", + "max_tokens": ctx_len, + "max_input_tokens": ctx_len, + "is_public_model_group": True + } } try: r = await client.post(f"{admin_url}/model/new", headers=headers, json=payload) @@ -490,6 +502,84 @@ def norm(s: float) -> float: logger.warning(f"Failed to register {mid} under {tier_name}: {e}") logger.info(f"📊 Roster sync: registered {registered} deployments ({failed} failed) across 5 tiers — {sum(len(v) for v in tier_assignments.values())} attempted") +async def _register_ollama_models_in_db(master_key: str): + """Register static ollama models via /model/new so they become DB models. + + LiteLLM's /model_group/info endpoint aggregates model info using its internal + model cost map for known providers. For ollama_chat models not in the map, + capabilities (vision, reasoning, function_calling) and token limits come back + as null/false. Registering them as DB models ensures our model_info wins. + """ + admin_url = "http://127.0.0.1:4000" + headers = {"Authorization": f"Bearer {master_key}", "Content-Type": "application/json"} + + ollama_models = [ + { + "model_name": "ollama-deepseek-v4-pro", + "litellm_params": { + "model": "ollama_chat/deepseek-v4-pro", + "api_base": "https://api.ollama.com", + "api_key": os.getenv("OLLAMA_API_KEY", ""), + "request_timeout": 120, + }, + "model_info": { + "supports_vision": True, + "supports_reasoning": True, + "supports_function_calling": True, + "mode": "chat", + "max_tokens": 524288, + "max_input_tokens": 524288, + "input_cost_per_token": 0.00000174, + "output_cost_per_token": 0.00000348, + "is_public_model_group": True, + }, + }, + { + "model_name": "ollama-deepseek-v4-flash", + "litellm_params": { + "model": "ollama_chat/deepseek-v4-flash", + "api_base": "https://api.ollama.com", + "api_key": os.getenv("OLLAMA_API_KEY", ""), + "request_timeout": 120, + }, + "model_info": { + "supports_vision": True, + "supports_reasoning": True, + "supports_function_calling": True, + "mode": "chat", + "max_tokens": 524288, + "max_input_tokens": 524288, + "input_cost_per_token": 0.00000014, + "output_cost_per_token": 0.00000028, + "is_public_model_group": True, + }, + }, + ] + + # Purge stale ollama-deepseek DB entries before re-registering + try: + db_url = os.getenv("DATABASE_URL", "postgresql://postgres@127.0.0.1:5432/postgres") + import asyncpg + conn = await asyncpg.connect(db_url) + await conn.execute( + 'DELETE FROM "LiteLLM_ProxyModelTable" WHERE model_name LIKE $1', + 'ollama-deepseek-%' + ) + await conn.close() + except Exception as e: + logger.warning(f"Failed to purge stale ollama DB entries (non-fatal): {e}") + + async with httpx.AsyncClient(timeout=10.0) as client: + for payload in ollama_models: + try: + r = await client.post(f"{admin_url}/model/new", headers=headers, json=payload) + if r.status_code in (200, 201): + logger.info(f"✅ Registered {payload['model_name']} in DB") + else: + logger.warning(f"model/new {payload['model_name']}: HTTP {r.status_code} — {r.text[:200]}") + except Exception as e: + logger.warning(f"Failed to register {payload['model_name']}: {e}") + @asynccontextmanager async def lifespan(app: FastAPI): """Startup: wait for LiteLLM readiness, then sync free-model roster.""" @@ -521,6 +611,15 @@ async def lifespan(app: FastAPI): except Exception as e: logger.error(f"Roster sync failed: {e}") + # Register static ollama models via /model/new so they become DB models. + # The ollama_chat provider's internal model lookup overrides static config + # model_info at the group aggregation level (/model_group/info), causing + # features and token limits to show as null/false. DB models get priority. + try: + await _register_ollama_models_in_db(litellm_master_key) + except Exception as e: + logger.warning(f"Ollama DB registration failed (non-fatal): {e}") + # Start background task before yield so it runs during app lifetime task = asyncio.create_task(push_aggregate_scores())