From d12f214219445dffec6b128c9e6ea0336e74ca3a Mon Sep 17 00:00:00 2001 From: boy Date: Mon, 13 Jul 2026 00:36:41 +0200 Subject: [PATCH 1/3] fix: add OPENROUTER_API_KEY to LiteLLM container env The LiteLLM gateway container was missing OPENROUTER_API_KEY in its environment, causing all OpenRouter free model requests to fail with 'Missing Authentication header' (401). This cascaded through the fallback chain and eventually produced 429 errors for the llm-routing-auto-free routing model. Added OPENROUTER_API_KEY_PLACEHOLDER to pod.yaml LiteLLM container env and the corresponding placeholder rendering in start-stack.sh. Verified: 24/24 canonical endpoints pass in dev environment. Tested: llm-routing-auto-free chat completion succeeds (not 429). --- pod.yaml | 2 ++ start-stack.sh | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pod.yaml b/pod.yaml index 75981ee1..8eb4449f 100644 --- a/pod.yaml +++ b/pod.yaml @@ -59,6 +59,8 @@ spec: value: LITELLM_UI_PASSWORD_PLACEHOLDER - name: OLLAMA_API_KEY value: OLLAMA_API_KEY_PLACEHOLDER + - name: OPENROUTER_API_KEY + value: OPENROUTER_API_KEY_PLACEHOLDER - name: SERVER_ROOT_PATH value: /llm-routing/litellm - name: PROXY_BASE_URL diff --git a/start-stack.sh b/start-stack.sh index 4eac51a0..8e6e567d 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -613,7 +613,7 @@ render_router_config() { } render_pod_yaml() { - export WORKDIR HOME LITELLM_MASTER_KEY UI_USERNAME UI_PASSWORD POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT + export WORKDIR HOME LITELLM_MASTER_KEY UI_USERNAME UI_PASSWORD POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY OPENROUTER_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT python3 - "$WORKDIR/pod.yaml" <<'PY' import os, sys, urllib.parse, json uid = os.getuid() @@ -637,6 +637,7 @@ placeholders = [ "SALT_PLACEHOLDER", "ENCRYPTION_KEY_PLACEHOLDER", "OLLAMA_API_KEY_PLACEHOLDER", + "OPENROUTER_API_KEY_PLACEHOLDER", "LANGFUSE_PUBLIC_KEY_PLACEHOLDER", "LANGFUSE_SECRET_KEY_PLACEHOLDER", "MINIO_USER_PLACEHOLDER", @@ -680,6 +681,7 @@ text = text.replace("NEXTAUTH_SECRET_PLACEHOLDER", yaml_scalar(os.environ["NEXTA text = text.replace("SALT_PLACEHOLDER", yaml_scalar(os.environ["SALT"])) text = text.replace("ENCRYPTION_KEY_PLACEHOLDER", yaml_scalar(os.environ["ENCRYPTION_KEY"])) text = text.replace("OLLAMA_API_KEY_PLACEHOLDER", yaml_scalar(os.environ["OLLAMA_API_KEY"])) +text = text.replace("OPENROUTER_API_KEY_PLACEHOLDER", yaml_scalar(os.environ["OPENROUTER_API_KEY"])) text = text.replace("LANGFUSE_PUBLIC_KEY_PLACEHOLDER", yaml_scalar(os.environ["LANGFUSE_PUBLIC_KEY"])) text = text.replace("LANGFUSE_SECRET_KEY_PLACEHOLDER", yaml_scalar(os.environ["LANGFUSE_SECRET_KEY"])) text = text.replace("MINIO_USER_PLACEHOLDER", yaml_scalar(os.environ["MINIO_ROOT_USER"])) From fbd7388161066a94c4ae291b4293c06301bcc678 Mon Sep 17 00:00:00 2001 From: boy Date: Mon, 13 Jul 2026 01:00:26 +0200 Subject: [PATCH 2/3] refactor: remove static agent-* safety-net entries from LiteLLM config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 5 static entries (agent-simple-core through agent-advanced-core) were safety nets that defined hardcoded OpenRouter :free models, including 3 Google models (gemma-4-31b, gemma-4-26b) that frequently hit quota limits. The dynamic roster sync (router/main.py) already registers 31+ AA-scored free models at startup via LiteLLM's /model/new API. This is the sole source of truth for agent tier deployments now. Removing the static entries eliminates Google models that were polluting the routing pool alongside the dynamic roster. Also applies the OPENROUTER_API_KEY fix from the same branch. Tested: 101/101 pytest, 23/24 canonical endpoints (1 timeout is pre-existing — free model rate limits) --- litellm/config.yaml | 85 --------------------------------------------- 1 file changed, 85 deletions(-) diff --git a/litellm/config.yaml b/litellm/config.yaml index a93aa2d9..16272289 100644 --- a/litellm/config.yaml +++ b/litellm/config.yaml @@ -150,91 +150,6 @@ model_list: output_cost_per_token: 0.00000028 is_public_model_group: true -# ================================================================================ -# AGENT TIER DEPLOYMENTS — safety-net entries (one per tier) -# The DB roster sync (router/main.py) is the DYNAMIC source of truth and -# registers the full AA-scored deployment pool at startup via LiteLLM's -# /model/new admin API. These static entries guarantee each tier name exists -# even before the first roster sync runs (prevents "model not found" on cold -# start). Each tier gets exactly ONE reliable free model that is always -# available. The roster sync adds the remaining AA-scored models on startup. -# -# TIER THRESHOLDS (normalized AA score): -# n >= 80 → agent-advanced-core -# n >= 75 → agent-reasoning-core -# n >= 68 → agent-complex-core -# n >= 60 → agent-medium-core -# else → agent-simple-core -# -# CURRENT AA SCORES (2026-06-08, max=55.0): -# minimax/minimax-m2.5:free → 55.0 (norm=100.0) — advanced safety net -# moonshotai/kimi-k2.6:free → 54.0 (norm=98.2) — reasoning safety net -# nvidia/nemotron-3-ultra-550b-a55b → 48.0 (norm=87.3) — complex safety net -# google/gemma-4-26b-a4b-it:free → 40.0 (norm=72.7) — medium safety net -# nvidia/nemotron-3-nano-30b-a3b:free → 28.0 (norm=50.9) — simple safety net -# ================================================================================ - -- model_name: agent-advanced-core - litellm_params: - model: openrouter/google/gemma-4-31b-it:free - request_timeout: 20 - model_info: - supports_vision: false - supports_reasoning: true - supports_function_calling: true - mode: chat - max_tokens: 262144 - max_input_tokens: 262144 - is_public_model_group: true -- model_name: agent-reasoning-core - litellm_params: - model: openrouter/google/gemma-4-26b-a4b-it:free - request_timeout: 20 - model_info: - supports_vision: false - supports_reasoning: true - supports_function_calling: true - mode: chat - max_tokens: 262144 - max_input_tokens: 262144 - is_public_model_group: true -- model_name: agent-complex-core - litellm_params: - model: openrouter/nvidia/nemotron-3-ultra-550b-a55b:free - request_timeout: 20 - model_info: - supports_vision: false - supports_reasoning: true - supports_function_calling: true - mode: chat - max_tokens: 262144 - max_input_tokens: 262144 - is_public_model_group: true -- model_name: agent-medium-core - litellm_params: - model: openrouter/google/gemma-4-26b-a4b-it:free - request_timeout: 20 - model_info: - supports_vision: false - supports_reasoning: true - supports_function_calling: true - mode: chat - max_tokens: 262144 - max_input_tokens: 262144 - is_public_model_group: true -- model_name: agent-simple-core - litellm_params: - model: openrouter/nvidia/nemotron-3-nano-30b-a3b:free - request_timeout: 20 - model_info: - supports_vision: false - supports_reasoning: true - supports_function_calling: true - mode: chat - max_tokens: 256000 - max_input_tokens: 256000 - is_public_model_group: true - redis_settings: redis_host: 127.0.0.1 redis_port: VALKEY_CACHE_PORT_PLACEHOLDER From 32fc19f7df21f871c340d3521528209ea1129441 Mon Sep 17 00:00:00 2001 From: boy Date: Mon, 13 Jul 2026 01:04:27 +0200 Subject: [PATCH 3/3] style: split long export line in render_pod_yaml for readability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per Gemini Code Assist review: the 500+ character export statement was hard to read and prone to merge conflicts. Split into one var per line with backslash continuations. CodeRabbit's suggestion to add OPENROUTER_API_KEY non-empty check is already handled at lines 127-152 — the script exits before reaching this function if the key is empty. --- start-stack.sh | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/start-stack.sh b/start-stack.sh index 8e6e567d..8a547f3b 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -613,7 +613,43 @@ render_router_config() { } render_pod_yaml() { - export WORKDIR HOME LITELLM_MASTER_KEY UI_USERNAME UI_PASSWORD POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY OPENROUTER_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT + export \ + WORKDIR \ + HOME \ + LITELLM_MASTER_KEY \ + UI_USERNAME \ + UI_PASSWORD \ + POSTGRES_PASSWORD \ + NEXTAUTH_SECRET \ + SALT \ + ENCRYPTION_KEY \ + LANGFUSE_INIT_USER_PASSWORD \ + MINIO_ROOT_USER \ + MINIO_ROOT_PASSWORD \ + OLLAMA_API_KEY \ + OPENROUTER_API_KEY \ + LANGFUSE_PUBLIC_KEY \ + LANGFUSE_SECRET_KEY \ + CLASSIFIER_INPUT_MAX_CHARS \ + REDIS_AUTH \ + CLICKHOUSE_PASSWORD \ + PUBLIC_BASE_URL \ + ROUTING_DOMAIN \ + POD_NAME \ + DATA_ROOT \ + ROUTER_IMAGE \ + ROUTER_PORT \ + LITELLM_PORT \ + LANGFUSE_WEB_PORT \ + LANGFUSE_WORKER_PORT \ + POSTGRES_PORT \ + VALKEY_CACHE_PORT \ + VALKEY_LF_PORT \ + CLICKHOUSE_HTTP_PORT \ + CLICKHOUSE_TCP_PORT \ + CLICKHOUSE_INTERSERVER_PORT \ + MINIO_S3_PORT \ + MINIO_CONSOLE_PORT python3 - "$WORKDIR/pod.yaml" <<'PY' import os, sys, urllib.parse, json uid = os.getuid()