Skip to content

fix: OPENROUTER_API_KEY + chain-routed static safety-nets #294

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sheepdestroyer merged 6 commits into from
Jul 13, 2026
Merged

fix: OPENROUTER_API_KEY + chain-routed static safety-nets #294

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d12f214
fix: add OPENROUTER_API_KEY to LiteLLM container env
Jul 12, 2026
fbd7388
refactor: remove static agent-* safety-net entries from LiteLLM config
Jul 12, 2026
32fc19f
style: split long export line in render_pod_yaml for readability
Jul 12, 2026
1654dfd
fix: add local-qwen-3.6 static safety-nets + clean export statements
Jul 12, 2026
062fbc1
fix: add capability flags + reorder fallback chains
Jul 12, 2026
f032937
fix: chain-routed static safety-nets + reduce timeout to 120s
Jul 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
115 changes: 50 additions & 65 deletions litellm/config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,11 @@ general_settings:
litellm_settings:
# -------------------------------------------------------------------------
# FALLBACK CHAINS (cascading, in order of escalation)
# Simple → medium → complex → reasoning → advanced → openrouter-auto
# Simple → medium → complex → reasoning → advanced → local-qwen-3.6 → ollama → openrouter-auto
# Static entries chain-route upward — only advanced-core hits local-qwen-3.6 directly.
# When one tier fails, LiteLLM tries the next model in the chain.
# All tiers fall back through increasingly capable models, then to openrouter-auto.
# local-qwen-3.6 (35B) disabled 2026-06-08 — frees 23GB RAM/GTT.
# Static safety-net entries in model_list (local-qwen-3.6 per tier) guarantee
# each agent tier has at least one deployment even during OpenRouter outages.
# -------------------------------------------------------------------------
cache: true
cache_params:
Expand Down Expand Up @@ -38,29 +39,25 @@ litellm_settings:
- agent-advanced-core
- llm-routing-ollama
- openrouter-auto
- local-qwen-3.6
- agent-medium-core:
- agent-complex-core
- agent-reasoning-core
- agent-advanced-core
- llm-routing-ollama
- openrouter-auto
- local-qwen-3.6
- agent-complex-core:
- agent-reasoning-core
- agent-advanced-core
- llm-routing-ollama
- openrouter-auto
- local-qwen-3.6
- agent-reasoning-core:
- agent-advanced-core
- llm-routing-ollama
- openrouter-auto
- local-qwen-3.6
- agent-advanced-core:
- local-qwen-3.6
- llm-routing-ollama
- openrouter-auto
- local-qwen-3.6

model_list:
- litellm_params:
Expand Down Expand Up @@ -95,7 +92,7 @@ model_list:
api_base: LLAMA_CLASSIFIER_URL_PLACEHOLDER
api_key: local-token
model: openai/local-qwen-3.6
request_timeout: 300
request_timeout: 120
model_name: local-qwen-3.6
- litellm_params:
api_base: LLAMA_CLASSIFIER_URL_PLACEHOLDER
Expand Down Expand Up @@ -151,88 +148,76 @@ model_list:
is_public_model_group: true

# ================================================================================
# AGENT TIER DEPLOYMENTS — safety-net entries (one per tier)
# The DB roster sync (router/main.py) is the DYNAMIC source of truth and
# registers the full AA-scored deployment pool at startup via LiteLLM's
# /model/new admin API. These static entries guarantee each tier name exists
# even before the first roster sync runs (prevents "model not found" on cold
# start). Each tier gets exactly ONE reliable free model that is always
# available. The roster sync adds the remaining AA-scored models on startup.
#
# TIER THRESHOLDS (normalized AA score):
# n >= 80 → agent-advanced-core
# n >= 75 → agent-reasoning-core
# n >= 68 → agent-complex-core
# n >= 60 → agent-medium-core
# else → agent-simple-core
#
# CURRENT AA SCORES (2026-06-08, max=55.0):
# minimax/minimax-m2.5:free → 55.0 (norm=100.0) — advanced safety net
# moonshotai/kimi-k2.6:free → 54.0 (norm=98.2) — reasoning safety net
# nvidia/nemotron-3-ultra-550b-a55b → 48.0 (norm=87.3) — complex safety net
# google/gemma-4-26b-a4b-it:free → 40.0 (norm=72.7) — medium safety net
# nvidia/nemotron-3-nano-30b-a3b:free → 28.0 (norm=50.9) — simple safety net
# AGENT TIER STATIC SAFETY-NET — chain-routed, one deployment per tier.
# Each tier has exactly ONE static deployment routing to the next tier up:
# simple → medium → complex → reasoning → advanced → local-qwen-3.6
# This guarantees no tier is ever empty (prevents ModelNotFoundError on
# cold start) while avoiding redundant fallback loops — only advanced-core
# points directly to the local classifier. The dynamic roster sync
# (router/main.py) adds OpenRouter deployments on top at startup.
# ================================================================================

- model_name: agent-advanced-core
litellm_params:
model: openrouter/google/gemma-4-31b-it:free
request_timeout: 20
api_base: LLAMA_CLASSIFIER_URL_PLACEHOLDER
api_key: local-token
model: openai/local-qwen-3.6
request_timeout: 120
model_info:
supports_vision: false
supports_reasoning: true
supports_function_calling: true
mode: chat
max_tokens: 262144
max_input_tokens: 262144
max_tokens: 524288
max_input_tokens: 524288
supports_function_calling: true
supports_reasoning: true
supports_vision: true
is_public_model_group: true
- model_name: agent-reasoning-core
litellm_params:
model: openrouter/google/gemma-4-26b-a4b-it:free
request_timeout: 20
model: agent-advanced-core
request_timeout: 120
model_info:
supports_vision: false
supports_reasoning: true
supports_function_calling: true
mode: chat
max_tokens: 262144
max_input_tokens: 262144
max_tokens: 524288
max_input_tokens: 524288
supports_function_calling: true
supports_reasoning: true
supports_vision: true
is_public_model_group: true
- model_name: agent-complex-core
litellm_params:
model: openrouter/nvidia/nemotron-3-ultra-550b-a55b:free
request_timeout: 20
model: agent-reasoning-core
request_timeout: 120
model_info:
supports_vision: false
supports_reasoning: true
supports_function_calling: true
mode: chat
max_tokens: 262144
max_input_tokens: 262144
max_tokens: 524288
max_input_tokens: 524288
supports_function_calling: true
supports_reasoning: true
supports_vision: true
is_public_model_group: true
- model_name: agent-medium-core
litellm_params:
model: openrouter/google/gemma-4-26b-a4b-it:free
request_timeout: 20
model: agent-complex-core
request_timeout: 120
model_info:
supports_vision: false
supports_reasoning: true
supports_function_calling: true
mode: chat
max_tokens: 262144
max_input_tokens: 262144
max_tokens: 524288
max_input_tokens: 524288
supports_function_calling: true
supports_reasoning: true
supports_vision: true
is_public_model_group: true
- model_name: agent-simple-core
litellm_params:
model: openrouter/nvidia/nemotron-3-nano-30b-a3b:free
request_timeout: 20
model: agent-medium-core
request_timeout: 120
model_info:
supports_vision: false
supports_reasoning: true
supports_function_calling: true
mode: chat
max_tokens: 256000
max_input_tokens: 256000
max_tokens: 524288
max_input_tokens: 524288
supports_function_calling: true
supports_reasoning: true
supports_vision: true
is_public_model_group: true
Comment on lines 174 to 221

@gemini-code-assist gemini-code-assist Bot Jul 13, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

critical

🚨 Critical Correctness Issue: Broken Static Safety-Net Chain Routing

The current configuration for the static safety-net entries (agent-reasoning-core, agent-complex-core, agent-medium-core, and agent-simple-core) will completely fail to route when OpenRouter is down.

Why this happens:

  1. Missing Provider Prefix & API Base: LiteLLM requires a provider prefix (e.g., openai/) or an explicit custom_llm_provider to know how to route a model. Without these, and without an api_base, LiteLLM defaults to the public OpenAI API (https://api.openai.com/v1/chat/completions) and will attempt to call it with the model name agent-advanced-core, resulting in a 401 Unauthorized or 404 Not Found error.
  2. No Self-Referential Routing: LiteLLM does not support routing to another model group defined in its own model_list unless you configure a loopback HTTP call to itself (which is highly inefficient, prone to connection pool exhaustion, and increases latency).

Recommended Solution:

Point all static safety-net entries directly to the local classifier (openai/local-qwen-3.6) with the correct api_base and api_key, just like agent-advanced-core does. This guarantees that if OpenRouter is down, any tier's safety net will immediately and successfully route to the local model without nested loopback overhead.

- model_name: agent-reasoning-core
  litellm_params:
    api_base: LLAMA_CLASSIFIER_URL_PLACEHOLDER
    api_key: local-token
    model: openai/local-qwen-3.6
    request_timeout: 120
  model_info:
    mode: chat
    max_tokens: 524288
    max_input_tokens: 524288
    supports_function_calling: true
    supports_reasoning: true
    supports_vision: true
    is_public_model_group: true
- model_name: agent-complex-core
  litellm_params:
    api_base: LLAMA_CLASSIFIER_URL_PLACEHOLDER
    api_key: local-token
    model: openai/local-qwen-3.6
    request_timeout: 120
  model_info:
    mode: chat
    max_tokens: 524288
    max_input_tokens: 524288
    supports_function_calling: true
    supports_reasoning: true
    supports_vision: true
    is_public_model_group: true
- model_name: agent-medium-core
  litellm_params:
    api_base: LLAMA_CLASSIFIER_URL_PLACEHOLDER
    api_key: local-token
    model: openai/local-qwen-3.6
    request_timeout: 120
  model_info:
    mode: chat
    max_tokens: 524288
    max_input_tokens: 524288
    supports_function_calling: true
    supports_reasoning: true
    supports_vision: true
    is_public_model_group: true
- model_name: agent-simple-core
  litellm_params:
    api_base: LLAMA_CLASSIFIER_URL_PLACEHOLDER
    api_key: local-token
    model: openai/local-qwen-3.6
    request_timeout: 120
  model_info:
    mode: chat
    max_tokens: 524288
    max_input_tokens: 524288
    supports_function_calling: true
    supports_reasoning: true
    supports_vision: true
    is_public_model_group: true

@sheepdestroyer sheepdestroyer Jul 13, 2026

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

False positive — LiteLLM does support internal model group routing without a provider prefix. Verified live: agent-simple-core (which chains through model: agent-medium-core → agent-complex-core → agent-reasoning-core → agent-advanced-core → openai/local-qwen-3.6) returns HTTP 200 in 0.55s on the dev deployment. LiteLLM correctly resolves bare model group names as internal references. If it were falling through to OpenAI's public API as claimed, we would see 401/404 errors, not successful responses.


redis_settings:
Expand Down
2 changes: 2 additions & 0 deletions pod.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@ spec:
value: LITELLM_UI_PASSWORD_PLACEHOLDER
- name: OLLAMA_API_KEY
value: OLLAMA_API_KEY_PLACEHOLDER
- name: OPENROUTER_API_KEY
value: OPENROUTER_API_KEY_PLACEHOLDER
- name: SERVER_ROOT_PATH
value: /llm-routing/litellm
- name: PROXY_BASE_URL
Expand Down
13 changes: 12 additions & 1 deletion start-stack.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -613,7 +613,16 @@ render_router_config() {
}

render_pod_yaml() {
export WORKDIR HOME LITELLM_MASTER_KEY UI_USERNAME UI_PASSWORD POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT
export WORKDIR HOME LITELLM_MASTER_KEY UI_USERNAME UI_PASSWORD
export POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY
export LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD
export OLLAMA_API_KEY OPENROUTER_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY
export CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD
export PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT
export ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT
export LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT
export CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT
export MINIO_S3_PORT MINIO_CONSOLE_PORT
python3 - "$WORKDIR/pod.yaml" <<'PY'
import os, sys, urllib.parse, json
uid = os.getuid()
Expand All @@ -637,6 +646,7 @@ placeholders = [
"SALT_PLACEHOLDER",
"ENCRYPTION_KEY_PLACEHOLDER",
"OLLAMA_API_KEY_PLACEHOLDER",
"OPENROUTER_API_KEY_PLACEHOLDER",
"LANGFUSE_PUBLIC_KEY_PLACEHOLDER",
"LANGFUSE_SECRET_KEY_PLACEHOLDER",
"MINIO_USER_PLACEHOLDER",
Expand Down Expand Up @@ -680,6 +690,7 @@ text = text.replace("NEXTAUTH_SECRET_PLACEHOLDER", yaml_scalar(os.environ["NEXTA
text = text.replace("SALT_PLACEHOLDER", yaml_scalar(os.environ["SALT"]))
text = text.replace("ENCRYPTION_KEY_PLACEHOLDER", yaml_scalar(os.environ["ENCRYPTION_KEY"]))
text = text.replace("OLLAMA_API_KEY_PLACEHOLDER", yaml_scalar(os.environ["OLLAMA_API_KEY"]))
text = text.replace("OPENROUTER_API_KEY_PLACEHOLDER", yaml_scalar(os.environ["OPENROUTER_API_KEY"]))
text = text.replace("LANGFUSE_PUBLIC_KEY_PLACEHOLDER", yaml_scalar(os.environ["LANGFUSE_PUBLIC_KEY"]))
text = text.replace("LANGFUSE_SECRET_KEY_PLACEHOLDER", yaml_scalar(os.environ["LANGFUSE_SECRET_KEY"]))
text = text.replace("MINIO_USER_PLACEHOLDER", yaml_scalar(os.environ["MINIO_ROOT_USER"]))
Expand Down