Comprehensive Fixes: Worktree Leak Cleanups, LiteLLM Authentication, and Code Review Feedback

litellm/config.yaml

@@ -1,5 +1,5 @@
 general_settings:
-  master_key: sk-lit...33bf
+  master_key: os.environ/LITELLM_MASTER_KEY
 litellm_settings:
   # -------------------------------------------------------------------------
   # FALLBACK CHAINS (cascading, in order of escalation)
@@ -26,35 +26,44 @@ litellm_settings:
     - agent-complex-core
     - agent-reasoning-core
     - agent-advanced-core
+    - llm-routing-ollama
     - openrouter-auto
     # - local-qwen-3.6  # DISABLED: 35B model unloaded (23GB GTT saved)
   - agent-medium-core:
     - agent-complex-core
     - agent-reasoning-core
     - agent-advanced-core
+    - llm-routing-ollama
     - openrouter-auto
     # - local-qwen-3.6  # DISABLED
   - agent-complex-core:
     - agent-reasoning-core
     - agent-advanced-core
+    - llm-routing-ollama
     - openrouter-auto
     # - local-qwen-3.6  # DISABLED
   - agent-reasoning-core:
     - agent-advanced-core
+    - llm-routing-ollama
     - openrouter-auto
     # - local-qwen-3.6  # DISABLED
   - agent-advanced-core:
+    - llm-routing-ollama
     - openrouter-auto
     # - local-qwen-3.6  # DISABLED
-  - ollama-deepseek-v4-pro:
-    - agent-advanced-core
-  - ollama-deepseek-v4-flash:
-    - agent-reasoning-core
+
 model_list:
 - litellm_params:
     model: openrouter/openrouter/auto
     request_timeout: 120
   model_name: openrouter-auto
+- litellm_params:
+    model: openai/llm-routing-ollama
+    api_base: http://127.0.0.1:5000/v1
+    api_key: os.environ/LITELLM_MASTER_KEY
+    request_timeout: 120
+  model_name: llm-routing-ollama
+
 # DISABLED 2026-06-08 — 20GB on disk, 23GB GTT (system RAM as GPU buffer).
 # Uncomment to re-enable once a lighter model replaces it.
 #- litellm_params:
@@ -73,7 +82,10 @@ model_list:
 # ================================================================================
 # OLLAMA PAID TIER — ollama.com via LiteLLM's native ollama_chat provider.
 # LiteLLM calls https://api.ollama.com/api/chat with Bearer auth (OLLAMA_API_KEY).
-# Fallback: ollama-deepseek-v4-pro → agent-advanced-core → openrouter-auto.
+# No LiteLLM-level fallbacks: failures propagate back to the triage router
+# (router/main.py) which manages Ollama cooldowns internally. When Ollama fails,
+# the router activates a 5-minute cooldown and skips Ollama on subsequent
+# requests, returning 429 so LiteLLM falls through to openrouter-auto.
 # ================================================================================
 - model_name: ollama-deepseek-v4-pro
   litellm_params:
@@ -84,7 +96,7 @@ model_list:
 
 # ================================================================================
 # OLLAMA FLASH TIER — lighter/faster model for reasoning-tier requests.
-# Fallback: ollama-deepseek-v4-flash → agent-reasoning-core → openrouter-auto.
+# Same cooldown architecture as the pro tier above.
 # ================================================================================
 - model_name: ollama-deepseek-v4-flash
   litellm_params:
@@ -119,43 +131,38 @@ model_list:
 
 - model_name: agent-advanced-core
   litellm_params:
-    model: openrouter/minimax/minimax-m2.5:free
-    request_timeout: 120
+    model: openrouter/google/gemma-4-31b-it:free
+    request_timeout: 20
 - model_name: agent-reasoning-core
   litellm_params:
-    model: openrouter/moonshotai/kimi-k2.6:free
-    request_timeout: 120
+    model: openrouter/google/gemma-4-26b-a4b-it:free
+    request_timeout: 20
 - model_name: agent-complex-core
   litellm_params:
     model: openrouter/nvidia/nemotron-3-ultra-550b-a55b:free
-    request_timeout: 120
+    request_timeout: 20
 - model_name: agent-medium-core
   litellm_params:
     model: openrouter/google/gemma-4-26b-a4b-it:free
-    request_timeout: 120
+    request_timeout: 20
 - model_name: agent-simple-core
   litellm_params:
     model: openrouter/nvidia/nemotron-3-nano-30b-a3b:free
-    request_timeout: 120
+    request_timeout: 20
 
 redis_settings:
   redis_host: 127.0.0.1
   redis_port: 6379
 router_settings:
-  allowed_fails: 2
+  allowed_fails: 0
   cooldown_time: 300
   enable_pre_call_checks: false
   num_retries: 1
   routing_strategy: latency-based-routing
-  # Per-error-type cooldown thresholds (overrides allowed_fails for specific errors).
-  # Upstream rate limits ("temporarily rate-limited upstream") can last minutes —
-  # a 30s cooldown just wastes retries. 300s gives providers time to clear the limit.
-  # RateLimitError: cooldown on FIRST 429 (upstream rate limits persist for minutes —
-  # allowing even 1 retry wastes a request against a provider that's still throttling)
-  allowed_fails_policy:
-    RateLimitErrorAllowedFails: 0
-    TimeoutErrorAllowedFails: 3
-    BadRequestErrorAllowedFails: 1
+  enable_health_check_routing: false
+  # NOTE: allowed_fails_policy is an enterprise-only feature in LiteLLM.
+  # Ollama cooldowns are handled by the triage router itself (router/main.py),
+  # not by LiteLLM's deployment cooldown mechanism.
 vector_store_settings:
   collection_name: litellm_semantic_cache
   connection_string: postgresql://postgres:***@127.0.0.1:5432/postgres

router/Containerfile

@@ -3,7 +3,7 @@ FROM python:3.14-slim
 WORKDIR /app
 
 # Install deps in a single layer (no pip cache, no extra files)
-RUN pip install --no-cache-dir fastapi uvicorn httpx pyyaml python-multipart asyncpg langfuse
+RUN pip install --no-cache-dir fastapi uvicorn httpx pyyaml python-multipart asyncpg langfuse redis
 
 # Copy all source in one layer — removes dead config COPY (volume-mounted at runtime)
 COPY main.py agy_proxy.py circuit_breaker.py aa_scores.json free_models_roster.json /app/

router/agy_proxy.py

@@ -94,18 +94,19 @@ async def _run_agy_print(prompt: str, model_override: str = "",
     logger.info(f"agy proxy forwarding to host: [{model_tag}]{conv_tag} {prompt[:60]}...")
 
     try:
-        async with httpx.AsyncClient(timeout=timeout + 5.0) as client:
-            r = await client.post(url, json=payload)
-            if r.status_code == 200:
-                result = r.json()
-                return (
-                    result.get("returncode", 0),
-                    result.get("stdout", ""),
-                    result.get("stderr", ""),
-                    result.get("conversation_id", None)
-                )
-            else:
-                return -1, "", f"Daemon returned HTTP status {r.status_code}", None
+        from main import get_http_client
+        client = get_http_client()
+        r = await client.post(url, json=payload, timeout=timeout + 5.0)
+        if r.status_code == 200:
+            result = r.json()
+            return (
+                result.get("returncode", 0),
+                result.get("stdout", ""),
+                result.get("stderr", ""),
+                result.get("conversation_id", None)
+            )
+        else:
+            return -1, "", f"Daemon returned HTTP status {r.status_code}", None
     except Exception as e:
         logger.error(f"Failed to communicate with Host agy Daemon: {e}")
         return -1, "", f"Daemon connection error: {e}", None
@@ -201,6 +202,12 @@ async def try_agy_proxy(prompt: str, messages: list = None,
         ]
     else:
         agy_tiers = AGY_FALLBACK_TIERS  # full chain: gemini-3.5-flash → claude-opus-4.6
+    try:
+        from main import sync_cooldowns_from_valkey
+        await sync_cooldowns_from_valkey()
+    except Exception as e:
+        logger.warning(f"Failed to sync state from Valkey: {e}")
+
     # Per-model circuit breakers — Google and vendor (Claude/GPT) have independent
     # rate-limit windows (separate 5-hour quota refresh cycles).
     google_breaker = get_google_breaker()
@@ -220,7 +227,7 @@ async def try_agy_proxy(prompt: str, messages: list = None,
         context_parts = []
         for msg in messages[-10:]:
             role = msg.get("role", "user")
-            content = msg.get("content", "")
+            content = msg.get("content") or ""
             if role == "user":
                 context_parts.append(f"User: {content}")
             elif role == "assistant":
@@ -278,13 +285,16 @@ async def try_agy_proxy(prompt: str, messages: list = None,
             model_tag = tier["env_override"] if tier["env_override"] else "default (gemini-3.5-flash)"
             logger.info(f"agy proxy connecting stream to daemon: [{model_tag}]...")
 
-            client = httpx.AsyncClient(timeout=tier_timeout + 5.0)
-            req = client.build_request("POST", url, json=payload)
+            from main import get_http_client
+            client = get_http_client()
+            should_close_client = False
+            req = client.build_request("POST", url, json=payload, timeout=tier_timeout + 5.0)
             try:
                 r = await client.send(req, stream=True)
             except Exception as e:
                 logger.error(f"Failed to connect stream to daemon: {e}")
-                await client.aclose()
+                if should_close_client:
+                    await client.aclose()
                 continue
 
             # Read first line to see if it's successful or quota error
@@ -297,15 +307,17 @@ async def try_agy_proxy(prompt: str, messages: list = None,
 
             if not first_line:
                 await r.aclose()
-                await client.aclose()
+                if should_close_client:
+                    await client.aclose()
                 logger.warning(f"agy proxy: tier {tier['model_name']} returned empty stream. Trying next tier...")
                 continue
 
             try:
                 first_data = json.loads(first_line)
             except Exception:
                 await r.aclose()
-                await client.aclose()
+                if should_close_client:
+                    await client.aclose()
                 logger.error(f"agy proxy: invalid JSON from daemon: {first_line}")
                 continue
 
@@ -316,13 +328,24 @@ async def try_agy_proxy(prompt: str, messages: list = None,
                 if _is_quota_exhausted(rc, "", stderr_content) or rc != 0:
                     if _is_quota_exhausted(rc, "", stderr_content):
                         tier_breaker.record_failure()
+                        try:
+                            from main import save_cooldowns_to_valkey
+                            await save_cooldowns_to_valkey()
+                        except Exception as e:
+                            logger.warning(f"Failed to save cooldowns to Valkey: {e}")
                     await r.aclose()
-                    await client.aclose()
+                    if should_close_client:
+                        await client.aclose()
                     logger.warning(f"agy proxy: tier {tier['model_name']} failed immediately (rc={rc}). Trying next tier...")
                     continue
 
             # Success! Stream has started.
             tier_breaker.record_success()
+            try:
+                from main import save_cooldowns_to_valkey
+                await save_cooldowns_to_valkey()
+            except Exception as e:
+                logger.warning(f"Failed to save cooldowns to Valkey: {e}")
             async def token_generator(stream_resp, httpx_client, initial_line, current_conv_id):
                 """Asynchronously yields tokens from the agy daemon stream and manages session state updates."""
                 # Yield the initial token if it was a token
@@ -353,7 +376,8 @@ async def token_generator(stream_resp, httpx_client, initial_line, current_conv_
                                 }
                 finally:
                     await stream_resp.aclose()
-                    await httpx_client.aclose()
+                    if should_close_client:
+                        await httpx_client.aclose()
 
             return {
                 "stream": token_generator(r, client, first_line, last_conv_id),
@@ -376,6 +400,11 @@ async def token_generator(stream_resp, httpx_client, initial_line, current_conv_
             # Check for quota exhaustion
             if _is_quota_exhausted(returncode, stdout, stderr):
                 tier_breaker.record_failure()
+                try:
+                    from main import save_cooldowns_to_valkey
+                    await save_cooldowns_to_valkey()
+                except Exception as e:
+                    logger.warning(f"Failed to save cooldowns to Valkey: {e}")
                 logger.warning(
                     f"agy proxy: tier {tier['model_name']} quota exhausted. "
                     f"Falling to tier {actual_tier_idx + 2}..."
@@ -409,6 +438,11 @@ async def token_generator(stream_resp, httpx_client, initial_line, current_conv_
                     f"({len(stdout)} chars, {elapsed_total:.1f}s)"
                 )
                 tier_breaker.record_success()
+                try:
+                    from main import save_cooldowns_to_valkey
+                    await save_cooldowns_to_valkey()
+                except Exception as e:
+                    logger.warning(f"Failed to save cooldowns to Valkey: {e}")
                 return _wrap_response(stdout, tier["model_name"], proxy_prompt)
             else:
                 logger.warning(

router/circuit_breaker.py

@@ -121,6 +121,42 @@ def status(self) -> dict:
             "probe_granted": self.probe_granted,
         }
 
+    async def sync_from_valkey(self, redis_client) -> None:
+        """Synchronize circuit breaker state from Valkey."""
+        if not redis_client:
+            return
+        try:
+            state = await redis_client.hgetall(f"circuit_breaker:{self.name}")
+            if state:
+                self.tier = int(state.get("tier", "0"))
+                self.cooldown_until = float(state.get("cooldown_until", "0.0"))
+                self.probe_granted = state.get("probe_granted", "False") == "True"
+                self.total_trips = int(state.get("total_trips", "0"))
+                self.last_trip_time = float(state.get("last_trip_time", "0.0"))
+        except Exception as e:
+            logger.warning(f"Valkey circuit_breaker [{self.name}] sync failed: {e}")
+
+    async def save_to_valkey(self, redis_client) -> None:
+        """Persist circuit breaker state to Valkey."""
+        if not redis_client:
+            return
+        try:
+            key = f"circuit_breaker:{self.name}"
+            state = {
+                "tier": str(self.tier),
+                "cooldown_until": str(self.cooldown_until),
+                "probe_granted": "True" if self.probe_granted else "False",
+                "total_trips": str(self.total_trips),
+                "last_trip_time": str(self.last_trip_time),
+            }
+            await redis_client.hset(key, mapping=state)
+            now = time.time()
+            ttl = int(max(3600.0, self.cooldown_until - now + 3600.0))
+            await redis_client.expire(key, ttl)
+        except Exception as e:
+            logger.warning(f"Valkey circuit_breaker [{self.name}] save failed: {e}")
+
+
 
 class DualCircuitBreaker:
     """
@@ -162,6 +198,23 @@ def status(self) -> dict:
             "vendor": self.vendor.status(),
         }
 
+    async def sync_from_valkey(self, redis_client) -> None:
+        """Synchronize both sub-breakers from Valkey."""
+        import asyncio
+        await asyncio.gather(
+            self.google.sync_from_valkey(redis_client),
+            self.vendor.sync_from_valkey(redis_client)
+        )
+
+    async def save_to_valkey(self, redis_client) -> None:
+        """Persist both sub-breakers to Valkey."""
+        import asyncio
+        await asyncio.gather(
+            self.google.save_to_valkey(redis_client),
+            self.vendor.save_to_valkey(redis_client)
+        )
+
+
 
 # Module-level singleton
 _breaker = DualCircuitBreaker()

router/free_models_roster.json

@@ -42,6 +42,12 @@
       "score": 28.0,
       "context_length": 256000
     },
+    {
+      "id": "cohere/north-mini-code:free",
+      "name": "Cohere: North Mini Code (free)",
+      "score": 25.0,
+      "context_length": 256000
+    },
     {
       "id": "nex-agi/nex-n2-pro:free",
       "name": "Nex AGI: Nex-N2-Pro (free)",
@@ -139,6 +145,6 @@
       "context_length": 32768
     }
   ],
-  "updated_at": "2026-06-17T20:17:56.981195Z",
-  "count": 23
+  "updated_at": "2026-06-19T22:04:43.058592Z",
+  "count": 24
 }