fix: postgres password, circuit breaker, and defensive review fixes v3

.github/dependabot.yml

@@ -19,7 +19,7 @@ updates:
     directory: "/"
     schedule:
       interval: "daily"
-      time: "03:23"
+      time: "02:55"
     open-pull-requests-limit: 5
     labels:
       - "dependencies"

.github/workflows/test.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches: [ master ]
   pull_request:
-    branches: [ master ]
 
 jobs:
   test:
@@ -20,6 +19,9 @@ jobs:
         with:
           python-version: '3.11'
 
+      - name: Install dependencies
+        run: pip install httpx==0.28.1
+
       - name: Run Circuit Breaker Tests
         run: python3 test_circuit_breaker.py
 

litellm/config.yaml

@@ -1,5 +1,5 @@
 general_settings:
-  master_key: sk-lit...33bf
+  master_key: os.environ/LITELLM_MASTER_KEY
 litellm_settings:
   # -------------------------------------------------------------------------
   # FALLBACK CHAINS (cascading, in order of escalation)
@@ -20,41 +20,76 @@ litellm_settings:
   - langfuse
   detailed_debug: false
   drop_params: true
+  public_model_groups:
+    - openrouter-auto
+    - llm-routing-ollama
+    - ollama-deepseek-v4-pro
+    - ollama-deepseek-v4-flash
+    - agent-advanced-core
+    - agent-reasoning-core
+    - agent-complex-core
+    - agent-medium-core
+    - agent-simple-core
   fallbacks:
   - agent-simple-core:
     - agent-medium-core
     - agent-complex-core
     - agent-reasoning-core
     - agent-advanced-core
+    - llm-routing-ollama
     - openrouter-auto
     # - local-qwen-3.6  # DISABLED: 35B model unloaded (23GB GTT saved)
   - agent-medium-core:
     - agent-complex-core
     - agent-reasoning-core
     - agent-advanced-core
+    - llm-routing-ollama
     - openrouter-auto
     # - local-qwen-3.6  # DISABLED
   - agent-complex-core:
     - agent-reasoning-core
     - agent-advanced-core
+    - llm-routing-ollama
     - openrouter-auto
     # - local-qwen-3.6  # DISABLED
   - agent-reasoning-core:
     - agent-advanced-core
+    - llm-routing-ollama
     - openrouter-auto
     # - local-qwen-3.6  # DISABLED
   - agent-advanced-core:
+    - llm-routing-ollama
     - openrouter-auto
     # - local-qwen-3.6  # DISABLED
-  - ollama-deepseek-v4-pro:
-    - agent-advanced-core
-  - ollama-deepseek-v4-flash:
-    - agent-reasoning-core
+
 model_list:
 - litellm_params:
     model: openrouter/openrouter/auto
     request_timeout: 120
   model_name: openrouter-auto
+  model_info:
+    supports_vision: true
+    supports_reasoning: true
+    supports_function_calling: true
+    mode: chat
+    max_tokens: 2000000
+    max_input_tokens: 2000000
+    is_public_model_group: true
+- litellm_params:
+    model: openai/llm-routing-ollama
+    api_base: http://127.0.0.1:5000/v1
+    api_key: os.environ/LITELLM_MASTER_KEY
+    request_timeout: 120
+  model_name: llm-routing-ollama
+  model_info:
+    supports_vision: true
+    supports_reasoning: true
+    supports_function_calling: true
+    mode: chat
+    max_tokens: 524288
+    max_input_tokens: 524288
+    is_public_model_group: true
+
 # DISABLED 2026-06-08 — 20GB on disk, 23GB GTT (system RAM as GPU buffer).
 # Uncomment to re-enable once a lighter model replaces it.
 #- litellm_params:
@@ -73,25 +108,48 @@ model_list:
 # ================================================================================
 # OLLAMA PAID TIER — ollama.com via LiteLLM's native ollama_chat provider.
 # LiteLLM calls https://api.ollama.com/api/chat with Bearer auth (OLLAMA_API_KEY).
-# Fallback: ollama-deepseek-v4-pro → agent-advanced-core → openrouter-auto.
+# No LiteLLM-level fallbacks: failures propagate back to the triage router
+# (router/main.py) which manages Ollama cooldowns internally. When Ollama fails,
+# the router activates a 5-minute cooldown and skips Ollama on subsequent
+# requests, returning 429 so LiteLLM falls through to openrouter-auto.
 # ================================================================================
 - model_name: ollama-deepseek-v4-pro
   litellm_params:
     model: ollama_chat/deepseek-v4-pro
     api_base: https://api.ollama.com
     api_key: os.environ/OLLAMA_API_KEY
     request_timeout: 120
+  model_info:
+    supports_vision: true
+    supports_reasoning: true
+    supports_function_calling: true
+    mode: chat
+    max_tokens: 524288
+    max_input_tokens: 524288
+    input_cost_per_token: 0.00000174
+    output_cost_per_token: 0.00000348
+    is_public_model_group: true
 
 # ================================================================================
 # OLLAMA FLASH TIER — lighter/faster model for reasoning-tier requests.
-# Fallback: ollama-deepseek-v4-flash → agent-reasoning-core → openrouter-auto.
+# Same cooldown architecture as the pro tier above.
 # ================================================================================
 - model_name: ollama-deepseek-v4-flash
   litellm_params:
     model: ollama_chat/deepseek-v4-flash
     api_base: https://api.ollama.com
     api_key: os.environ/OLLAMA_API_KEY
     request_timeout: 120
+  model_info:
+    supports_vision: true
+    supports_reasoning: true
+    supports_function_calling: true
+    mode: chat
+    max_tokens: 524288
+    max_input_tokens: 524288
+    input_cost_per_token: 0.00000014
+    output_cost_per_token: 0.00000028
+    is_public_model_group: true
 
 # ================================================================================
 # AGENT TIER DEPLOYMENTS — safety-net entries (one per tier)
@@ -119,45 +177,84 @@ model_list:
 
 - model_name: agent-advanced-core
   litellm_params:
-    model: openrouter/minimax/minimax-m2.5:free
-    request_timeout: 120
+    model: openrouter/google/gemma-4-31b-it:free
+    request_timeout: 20
+  model_info:
+    supports_vision: false
+    supports_reasoning: true
+    supports_function_calling: true
+    mode: chat
+    max_tokens: 262144
+    max_input_tokens: 262144
+    is_public_model_group: true
 - model_name: agent-reasoning-core
   litellm_params:
-    model: openrouter/moonshotai/kimi-k2.6:free
-    request_timeout: 120
+    model: openrouter/google/gemma-4-26b-a4b-it:free
+    request_timeout: 20
+  model_info:
+    supports_vision: false
+    supports_reasoning: true
+    supports_function_calling: true
+    mode: chat
+    max_tokens: 262144
+    max_input_tokens: 262144
+    is_public_model_group: true
 - model_name: agent-complex-core
   litellm_params:
     model: openrouter/nvidia/nemotron-3-ultra-550b-a55b:free
-    request_timeout: 120
+    request_timeout: 20
+  model_info:
+    supports_vision: false
+    supports_reasoning: true
+    supports_function_calling: true
+    mode: chat
+    max_tokens: 262144
+    max_input_tokens: 262144
+    is_public_model_group: true
 - model_name: agent-medium-core
   litellm_params:
     model: openrouter/google/gemma-4-26b-a4b-it:free
-    request_timeout: 120
+    request_timeout: 20
+  model_info:
+    supports_vision: false
+    supports_reasoning: true
+    supports_function_calling: true
+    mode: chat
+    max_tokens: 262144
+    max_input_tokens: 262144
+    is_public_model_group: true
 - model_name: agent-simple-core
   litellm_params:
     model: openrouter/nvidia/nemotron-3-nano-30b-a3b:free
-    request_timeout: 120
+    request_timeout: 20
+  model_info:
+    supports_vision: false
+    supports_reasoning: true
+    supports_function_calling: true
+    mode: chat
+    max_tokens: 256000
+    max_input_tokens: 256000
+    is_public_model_group: true
 
 redis_settings:
   redis_host: 127.0.0.1
   redis_port: 6379
 router_settings:
-  allowed_fails: 2
+  allowed_fails: 0
   cooldown_time: 300
   enable_pre_call_checks: false
   num_retries: 1
   routing_strategy: latency-based-routing
-  # Per-error-type cooldown thresholds (overrides allowed_fails for specific errors).
-  # Upstream rate limits ("temporarily rate-limited upstream") can last minutes —
-  # a 30s cooldown just wastes retries. 300s gives providers time to clear the limit.
-  # RateLimitError: cooldown on FIRST 429 (upstream rate limits persist for minutes —
-  # allowing even 1 retry wastes a request against a provider that's still throttling)
-  allowed_fails_policy:
-    RateLimitErrorAllowedFails: 0
-    TimeoutErrorAllowedFails: 3
-    BadRequestErrorAllowedFails: 1
+  enable_health_check_routing: false
+  # NOTE: allowed_fails_policy is an enterprise-only feature in LiteLLM.
+  # allowed_fails is set to 0 to ensure that any model deployment failure
+  # immediately escalates through the fallback chain (to fallback models)
+  # instead of retrying the failed model. This does not interfere with the
+  # triage router's internal cooldowns, which return 429 when a specific
+  # backend (like Ollama) is cooling down, allowing LiteLLM to immediately
+  # trigger the fallback path.
 vector_store_settings:
   collection_name: litellm_semantic_cache
-  connection_string: postgresql://postgres:***@127.0.0.1:5432/postgres
+  connection_string: os.environ/DATABASE_URL
   embedding_model: local-nomic-embed
   store_type: postgres

litellm/entrypoint.py

@@ -53,5 +53,89 @@ def check_tcp_port(ip: str, port: int) -> bool:
 else:
     print(f"⚠️ Warning: PostgreSQL not ready after {max_wait}s — proceeding anyway")
 
+# Patch spend_management_endpoints.py to support flexible date formats for UI logs page
+import glob
+import sys
+import litellm
+
+litellm_path = os.path.dirname(litellm.__file__)
+endpoints_paths = [
+    os.path.join(litellm_path, "proxy/spend_tracking/spend_management_endpoints.py")
+] + glob.glob("/app/.venv/lib/python*/site-packages/litellm/proxy/spend_tracking/spend_management_endpoints.py")
+
+for endpoints_path in endpoints_paths:
+    if os.path.exists(endpoints_path):
+        print(f"🩹 Patching {endpoints_path} for flexible date formats...")
+        sys.stdout.flush()
+        try:
+            with open(endpoints_path, "r") as f:
+                code = f.read()
+
+            target1 = 'is_v2 = "/spend/logs/v2" in get_request_route(request)\n        formats = ["%Y-%m-%d %H:%M:%S", "%Y-%m-%d"] if is_v2 else ["%Y-%m-%d %H:%M:%S"]'
+            replacement1 = '''is_v2 = "/spend/logs/v2" in get_request_route(request)
+        formats = [
+            "%Y-%m-%d %H:%M:%S", "%Y-%m-%d",
+            "%Y-%m-%dT%H:%M:%S.%fZ", "%Y-%m-%dT%H:%M:%SZ",
+            "%Y-%m-%dT%H:%M:%S.%f", "%Y-%m-%dT%H:%M:%S",
+            "%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S%z",
+            "%Y-%m-%dT%H:%M:%S%z"
+        ]'''
+
+            target2 = '''    start_date_obj: Optional[datetime] = None
+    end_date_obj: Optional[datetime] = None
+    if start_date is not None:
+        start_date_obj = datetime.strptime(start_date, "%Y-%m-%d %H:%M:%S").replace(
+            tzinfo=timezone.utc
+        )
+    if end_date is not None:
+        end_date_obj = datetime.strptime(end_date, "%Y-%m-%d %H:%M:%S").replace(
+            tzinfo=timezone.utc
+        )'''
+            replacement2 = '''    start_date_obj: Optional[datetime] = None
+    end_date_obj: Optional[datetime] = None
+    def _parse_detail_date(date_str: str) -> datetime:
+        for fmt in [
+            "%Y-%m-%d %H:%M:%S", "%Y-%m-%d",
+            "%Y-%m-%dT%H:%M:%S.%fZ", "%Y-%m-%dT%H:%M:%SZ",
+            "%Y-%m-%dT%H:%M:%S.%f", "%Y-%m-%dT%H:%M:%S",
+            "%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S%z",
+            "%Y-%m-%dT%H:%M:%S%z"
+        ]:
+            try:
+                return datetime.strptime(date_str, fmt).replace(tzinfo=timezone.utc)
+            except ValueError:
+                continue
+        raise ValueError(f"Invalid date format: {date_str}")
+
+    if start_date is not None:
+        start_date_obj = _parse_detail_date(start_date)
+    if end_date is not None:
+        end_date_obj = _parse_detail_date(end_date)'''
+
+            patched = False
+            if target1 in code:
+                code = code.replace(target1, replacement1)
+                print("   ✓ Patched list endpoint date parsing")
+                patched = True
+            else:
+                print("   ⚠ Target 1 not found (already patched?)")
+
+            if target2 in code:
+                code = code.replace(target2, replacement2)
+                print("   ✓ Patched detail endpoint date parsing")
+                patched = True
+            else:
+                print("   ⚠ Target 2 not found (already patched?)")
+
+            if patched:
+                with open(endpoints_path, "w") as f:
+                    f.write(code)
+            sys.stdout.flush()
+
+        except Exception as e:
+            print(f"❌ Failed to patch {endpoints_path}: {e}")
+            sys.stdout.flush()
+
 # Exec into litellm
 os.execvp("litellm", ["litellm", "--config", "/app/config.yaml", "--port", "4000"])
+

pod.yaml

@@ -42,7 +42,7 @@ spec:
     - name: STORE_MODEL_IN_DB
       value: 'True'
     - name: LITELLM_LOG
-      value: WARNING
+      value: INFO
     - name: LANGFUSE_HOST
       value: http://127.0.0.1:3001
     - name: LITELLM_MASTER_KEY
@@ -90,6 +90,10 @@ spec:
     env:
     - name: CONFIG_PATH
       value: /config/router_dir/config.yaml
+    - name: LITELLM_CONFIG_PATH
+      value: /config/litellm_dir/config.yaml
+    - name: DATABASE_URL
+      value: postgresql://postgres:***@127.0.0.1:5432/postgres
     - name: DBUS_SESSION_BUS_ADDRESS
       value: unix:path=/run/user/1000/bus
     - name: LITELLM_MASTER_KEY
@@ -127,6 +131,8 @@ spec:
     volumeMounts:
     - mountPath: /config/router_dir
       name: router-config
+    - mountPath: /config/litellm_dir
+      name: litellm-config
     - mountPath: /app/data
       name: dataset-data
     - mountPath: /config/gemini_auth

router/Dockerfile

@@ -3,7 +3,7 @@ FROM python:3.14-slim
 WORKDIR /app
 
 # Install deps in a single layer (no pip cache, no extra files)
-RUN pip install --no-cache-dir fastapi uvicorn httpx pyyaml python-multipart asyncpg langfuse
+RUN pip install --no-cache-dir fastapi uvicorn httpx pyyaml python-multipart asyncpg langfuse redis
 
 # Copy all source in one layer — removes dead config COPY (volume-mounted at runtime)
 COPY main.py agy_proxy.py circuit_breaker.py aa_scores.json free_models_roster.json /app/