fix: HAProxy routing, env vars, + canonical endpoint verification

README.md

@@ -822,6 +822,32 @@ Execute the verification script:
 ./scripts/verification/verify_reasoning_tiers.py
 ```
 
+### Canonical Endpoint Verification
+
+A comprehensive endpoint health check that validates all services across both prod and dev environments:
+
+```bash
+# Prod (default)
+python scripts/verification/verify_canonical_endpoints.py
+
+# Dev
+python scripts/verification/verify_canonical_endpoints.py --dev
+```
+
+Tests cover:
+
+| Section | Endpoints |
+|---------|-----------|
+| Router API | `/v1/models`, `/metrics`, `/dashboard`, `/api/dashboard-stats`, `/visualizer` |
+| LiteLLM | `/health/liveness`, `/health/readiness`, `/v1/models`, `/llm-routing/litellm/ui/` |
+| Langfuse | `/api/public/health`, `/` (web UI) |
+| Infrastructure | MinIO `/minio/health/live`, ClickHouse `/ping` |
+| E2E chat | 3 completions through triage router |
+| LiteLLM direct | 1 completion directly to LiteLLM |
+| Canonical URLs | 6 GET + 1 POST through public HTTPS (graceful DNS skip) |
+
+Requires `PUBLIC_BASE_URL` in `.env` for canonical URL tests. Dev `.env.dev` already has it; prod `.env` should include `PUBLIC_BASE_URL="https://x570.vendeuvre.lan/llm-routing"`.
+
 ## 10. Performance Benchmarks
 
 Through our local benchmarks, the following performance characteristics have been achieved:

pod.yaml

@@ -53,6 +53,10 @@ spec:
       value: http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER
     - name: LITELLM_MASTER_KEY
       value: LITELLM_MASTER_KEY_PLACEHOLDER
+    - name: LITELLM_UI_USERNAME
+      value: LITELLM_UI_USERNAME_PLACEHOLDER
+    - name: LITELLM_UI_PASSWORD
+      value: LITELLM_UI_PASSWORD_PLACEHOLDER
     - name: OLLAMA_API_KEY
       value: OLLAMA_API_KEY_PLACEHOLDER
     - name: SERVER_ROOT_PATH
@@ -318,7 +322,7 @@ spec:
     - name: NEXTAUTH_SECRET
       value: NEXTAUTH_SECRET_PLACEHOLDER
     - name: NEXTAUTH_URL
-      value: http://localhost:LANGFUSE_WEB_PORT_PLACEHOLDER
+      value: NEXTAUTH_URL_PLACEHOLDER
     - name: SALT
       value: SALT_PLACEHOLDER
     - name: ENCRYPTION_KEY

scripts/benchmark_classifier.py

@@ -42,10 +42,18 @@ def classify(prompt):
     )
     with urllib.request.urlopen(req, timeout=30) as resp:
         data = json.loads(resp.read())
-    choices = data.get("choices", [])
-    if not choices:
+    if not isinstance(data, dict):
         return "ERROR"
-    return choices[0].get("message", {}).get("content", "").strip()
+    choices = data.get("choices")
+    if not isinstance(choices, list) or not choices:
+        return "ERROR"
+    first = choices[0]
+    if not isinstance(first, dict):
+        return "ERROR"
+    message = first.get("message")
+    if not isinstance(message, dict):
+        return "ERROR"
+    return (message.get("content") or "").strip()
 
 total = len(dataset.get("prompts", []))
 print(f"Benchmark: gemma4-26a4b-routing vs {total} labeled prompts\n")

scripts/classify_direct.py

@@ -33,7 +33,18 @@ def classify(prompt):
     )
     with urllib.request.urlopen(req, timeout=30) as resp:
         data = json.loads(resp.read())
-    return data["choices"][0]["message"].get("content", "").strip()
+    if not isinstance(data, dict):
+        return "ERROR"
+    choices = data.get("choices")
+    if not isinstance(choices, list) or not choices:
+        return "ERROR"
+    first = choices[0]
+    if not isinstance(first, dict):
+        return "ERROR"
+    message = first.get("message")
+    if not isinstance(message, dict):
+        return "ERROR"
+    return (message.get("content") or "").strip()
 
 # Load prompts
 data_dir = Path(__file__).resolve().parent.parent / "data"

scripts/reclassify_all.py

@@ -32,10 +32,18 @@ def classify(prompt):
     req = urllib.request.Request(LLAMA_SERVER_URL, data=json.dumps(payload).encode(), headers={'Content-Type':'application/json','Authorization': f'Bearer {os.environ.get("ROUTER_API_KEY", "local-token")}'})
     with urllib.request.urlopen(req, timeout=30) as resp:
         data = json.loads(resp.read())
-    choices = data.get('choices', [])
-    if not choices:
+    if not isinstance(data, dict):
         return "ERROR: empty response"
-    return choices[0].get('message', {}).get('content', '').strip()
+    choices = data.get('choices')
+    if not isinstance(choices, list) or not choices:
+        return "ERROR: empty response"
+    first = choices[0]
+    if not isinstance(first, dict):
+        return "ERROR: invalid choice"
+    message = first.get('message')
+    if not isinstance(message, dict):
+        return "ERROR: invalid message"
+    return (message.get('content') or '').strip()
 
 # Load existing dataset (kanban/llm evals)
 data_dir = Path(__file__).resolve().parent.parent / 'data'

scripts/retry_errors.py

@@ -50,10 +50,16 @@ def classify(prompt):
     )
     with urllib.request.urlopen(req, timeout=30) as resp:
         data = json.loads(resp.read())
-    choices = data.get("choices", [])
+    if not isinstance(data, dict):
+        return "ERROR"
+    choices = data.get("choices")
     content = ""
-    if choices:
-        content = choices[0].get("message", {}).get("content", "").strip()
+    if isinstance(choices, list) and choices:
+        first = choices[0]
+        if isinstance(first, dict):
+            message = first.get("message")
+            if isinstance(message, dict):
+                content = (message.get("content") or "").strip()
     # Normalize: strip "tier:" prefix, extract just the tier name
     for tier in TIERS:
         if tier in content: