diff --git a/litellm/config.yaml b/litellm/config.yaml index a93aa2d9..1bcce938 100644 --- a/litellm/config.yaml +++ b/litellm/config.yaml @@ -3,10 +3,10 @@ general_settings: litellm_settings: # ------------------------------------------------------------------------- # FALLBACK CHAINS (cascading, in order of escalation) - # Simple → medium → complex → reasoning → advanced → openrouter-auto + # Simple → medium → complex → reasoning → advanced → local-qwen-3.6 → ollama → openrouter-auto # When one tier fails, LiteLLM tries the next model in the chain. - # All tiers fall back through increasingly capable models, then to openrouter-auto. - # local-qwen-3.6 (35B) disabled 2026-06-08 — frees 23GB RAM/GTT. + # Static safety-net entries in model_list (local-qwen-3.6 per tier) guarantee + # each agent tier has at least one deployment even during OpenRouter outages. # ------------------------------------------------------------------------- cache: true cache_params: @@ -36,31 +36,31 @@ litellm_settings: - agent-complex-core - agent-reasoning-core - agent-advanced-core + - local-qwen-3.6 - llm-routing-ollama - openrouter-auto - - local-qwen-3.6 - agent-medium-core: - agent-complex-core - agent-reasoning-core - agent-advanced-core + - local-qwen-3.6 - llm-routing-ollama - openrouter-auto - - local-qwen-3.6 - agent-complex-core: - agent-reasoning-core - agent-advanced-core + - local-qwen-3.6 - llm-routing-ollama - openrouter-auto - - local-qwen-3.6 - agent-reasoning-core: - agent-advanced-core + - local-qwen-3.6 - llm-routing-ollama - openrouter-auto - - local-qwen-3.6 - agent-advanced-core: + - local-qwen-3.6 - llm-routing-ollama - openrouter-auto - - local-qwen-3.6 model_list: - litellm_params: @@ -151,88 +151,83 @@ model_list: is_public_model_group: true # ================================================================================ -# AGENT TIER DEPLOYMENTS — safety-net entries (one per tier) -# The DB roster sync (router/main.py) is the DYNAMIC source of truth and -# registers the full AA-scored deployment pool at startup via LiteLLM's -# /model/new admin API. These static entries guarantee each tier name exists -# even before the first roster sync runs (prevents "model not found" on cold -# start). Each tier gets exactly ONE reliable free model that is always -# available. The roster sync adds the remaining AA-scored models on startup. -# -# TIER THRESHOLDS (normalized AA score): -# n >= 80 → agent-advanced-core -# n >= 75 → agent-reasoning-core -# n >= 68 → agent-complex-core -# n >= 60 → agent-medium-core -# else → agent-simple-core -# -# CURRENT AA SCORES (2026-06-08, max=55.0): -# minimax/minimax-m2.5:free → 55.0 (norm=100.0) — advanced safety net -# moonshotai/kimi-k2.6:free → 54.0 (norm=98.2) — reasoning safety net -# nvidia/nemotron-3-ultra-550b-a55b → 48.0 (norm=87.3) — complex safety net -# google/gemma-4-26b-a4b-it:free → 40.0 (norm=72.7) — medium safety net -# nvidia/nemotron-3-nano-30b-a3b:free → 28.0 (norm=50.9) — simple safety net +# AGENT TIER STATIC SAFETY-NET — one local-qwen-3.6 deployment per tier. +# The dynamic roster sync (router/main.py) is the PRIMARY source of agent-tier +# deployments and registers AA-scored OpenRouter free models at startup via +# LiteLLM's /model/new API. These static entries guarantee each tier has at +# least one model (the local classifier) even if OpenRouter is unreachable at +# startup — preventing cold-start "Model NotFoundError" failures. # ================================================================================ - model_name: agent-advanced-core litellm_params: - model: openrouter/google/gemma-4-31b-it:free - request_timeout: 20 + api_base: LLAMA_CLASSIFIER_URL_PLACEHOLDER + api_key: local-token + model: openai/local-qwen-3.6 + request_timeout: 300 model_info: - supports_vision: false - supports_reasoning: true - supports_function_calling: true mode: chat - max_tokens: 262144 - max_input_tokens: 262144 + max_tokens: 524288 + max_input_tokens: 524288 + supports_function_calling: true + supports_reasoning: true + supports_vision: true is_public_model_group: true - model_name: agent-reasoning-core litellm_params: - model: openrouter/google/gemma-4-26b-a4b-it:free - request_timeout: 20 + api_base: LLAMA_CLASSIFIER_URL_PLACEHOLDER + api_key: local-token + model: openai/local-qwen-3.6 + request_timeout: 300 model_info: - supports_vision: false - supports_reasoning: true - supports_function_calling: true mode: chat - max_tokens: 262144 - max_input_tokens: 262144 + max_tokens: 524288 + max_input_tokens: 524288 + supports_function_calling: true + supports_reasoning: true + supports_vision: true is_public_model_group: true - model_name: agent-complex-core litellm_params: - model: openrouter/nvidia/nemotron-3-ultra-550b-a55b:free - request_timeout: 20 + api_base: LLAMA_CLASSIFIER_URL_PLACEHOLDER + api_key: local-token + model: openai/local-qwen-3.6 + request_timeout: 300 model_info: - supports_vision: false - supports_reasoning: true - supports_function_calling: true mode: chat - max_tokens: 262144 - max_input_tokens: 262144 + max_tokens: 524288 + max_input_tokens: 524288 + supports_function_calling: true + supports_reasoning: true + supports_vision: true is_public_model_group: true - model_name: agent-medium-core litellm_params: - model: openrouter/google/gemma-4-26b-a4b-it:free - request_timeout: 20 + api_base: LLAMA_CLASSIFIER_URL_PLACEHOLDER + api_key: local-token + model: openai/local-qwen-3.6 + request_timeout: 300 model_info: - supports_vision: false - supports_reasoning: true - supports_function_calling: true mode: chat - max_tokens: 262144 - max_input_tokens: 262144 + max_tokens: 524288 + max_input_tokens: 524288 + supports_function_calling: true + supports_reasoning: true + supports_vision: true is_public_model_group: true - model_name: agent-simple-core litellm_params: - model: openrouter/nvidia/nemotron-3-nano-30b-a3b:free - request_timeout: 20 + api_base: LLAMA_CLASSIFIER_URL_PLACEHOLDER + api_key: local-token + model: openai/local-qwen-3.6 + request_timeout: 300 model_info: - supports_vision: false - supports_reasoning: true - supports_function_calling: true mode: chat - max_tokens: 256000 - max_input_tokens: 256000 + max_tokens: 524288 + max_input_tokens: 524288 + supports_function_calling: true + supports_reasoning: true + supports_vision: true is_public_model_group: true redis_settings: diff --git a/pod.yaml b/pod.yaml index 75981ee1..8eb4449f 100644 --- a/pod.yaml +++ b/pod.yaml @@ -59,6 +59,8 @@ spec: value: LITELLM_UI_PASSWORD_PLACEHOLDER - name: OLLAMA_API_KEY value: OLLAMA_API_KEY_PLACEHOLDER + - name: OPENROUTER_API_KEY + value: OPENROUTER_API_KEY_PLACEHOLDER - name: SERVER_ROOT_PATH value: /llm-routing/litellm - name: PROXY_BASE_URL diff --git a/start-stack.sh b/start-stack.sh index 4eac51a0..657095be 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -613,7 +613,16 @@ render_router_config() { } render_pod_yaml() { - export WORKDIR HOME LITELLM_MASTER_KEY UI_USERNAME UI_PASSWORD POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT + export WORKDIR HOME LITELLM_MASTER_KEY UI_USERNAME UI_PASSWORD + export POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY + export LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD + export OLLAMA_API_KEY OPENROUTER_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY + export CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD + export PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT + export ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT + export LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT + export CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT + export MINIO_S3_PORT MINIO_CONSOLE_PORT python3 - "$WORKDIR/pod.yaml" <<'PY' import os, sys, urllib.parse, json uid = os.getuid() @@ -637,6 +646,7 @@ placeholders = [ "SALT_PLACEHOLDER", "ENCRYPTION_KEY_PLACEHOLDER", "OLLAMA_API_KEY_PLACEHOLDER", + "OPENROUTER_API_KEY_PLACEHOLDER", "LANGFUSE_PUBLIC_KEY_PLACEHOLDER", "LANGFUSE_SECRET_KEY_PLACEHOLDER", "MINIO_USER_PLACEHOLDER", @@ -680,6 +690,7 @@ text = text.replace("NEXTAUTH_SECRET_PLACEHOLDER", yaml_scalar(os.environ["NEXTA text = text.replace("SALT_PLACEHOLDER", yaml_scalar(os.environ["SALT"])) text = text.replace("ENCRYPTION_KEY_PLACEHOLDER", yaml_scalar(os.environ["ENCRYPTION_KEY"])) text = text.replace("OLLAMA_API_KEY_PLACEHOLDER", yaml_scalar(os.environ["OLLAMA_API_KEY"])) +text = text.replace("OPENROUTER_API_KEY_PLACEHOLDER", yaml_scalar(os.environ["OPENROUTER_API_KEY"])) text = text.replace("LANGFUSE_PUBLIC_KEY_PLACEHOLDER", yaml_scalar(os.environ["LANGFUSE_PUBLIC_KEY"])) text = text.replace("LANGFUSE_SECRET_KEY_PLACEHOLDER", yaml_scalar(os.environ["LANGFUSE_SECRET_KEY"])) text = text.replace("MINIO_USER_PLACEHOLDER", yaml_scalar(os.environ["MINIO_ROOT_USER"]))