fix: HAProxy routing, env vars, + canonical endpoint verification

README.md

@@ -822,6 +822,32 @@ Execute the verification script:
 ./scripts/verification/verify_reasoning_tiers.py
 ```
 
+### Canonical Endpoint Verification
+
+A comprehensive endpoint health check that validates all services across both prod and dev environments:
+
+```bash
+# Prod (default)
+python scripts/verification/verify_canonical_endpoints.py
+
+# Dev
+python scripts/verification/verify_canonical_endpoints.py --dev
+```
+
+Tests cover:
+
+| Section | Endpoints |
+|---------|-----------|
+| Router API | `/v1/models`, `/metrics`, `/dashboard`, `/api/dashboard-stats`, `/visualizer` |
+| LiteLLM | `/health/liveness`, `/health/readiness`, `/v1/models`, `/llm-routing/litellm/ui/` |
+| Langfuse | `/api/public/health`, `/` (web UI) |
+| Infrastructure | MinIO `/minio/health/live`, ClickHouse `/ping` |
+| E2E chat | 3 completions through triage router |
+| LiteLLM direct | 1 completion directly to LiteLLM |
+| Canonical URLs | 6 GET + 1 POST through public HTTPS (graceful DNS skip) |
+
+Requires `PUBLIC_BASE_URL` in `.env` for canonical URL tests. Dev `.env.dev` already has it; prod `.env` should include `PUBLIC_BASE_URL="https://x570.vendeuvre.lan/llm-routing"`.
+
 ## 10. Performance Benchmarks
 
 Through our local benchmarks, the following performance characteristics have been achieved:

pod.yaml

@@ -53,6 +53,10 @@ spec:
       value: http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER
     - name: LITELLM_MASTER_KEY
       value: LITELLM_MASTER_KEY_PLACEHOLDER
+    - name: LITELLM_UI_USERNAME
+      value: LITELLM_UI_USERNAME_PLACEHOLDER
+    - name: LITELLM_UI_PASSWORD
+      value: LITELLM_UI_PASSWORD_PLACEHOLDER
     - name: OLLAMA_API_KEY
       value: OLLAMA_API_KEY_PLACEHOLDER
     - name: SERVER_ROOT_PATH
@@ -318,7 +322,7 @@ spec:
     - name: NEXTAUTH_SECRET
       value: NEXTAUTH_SECRET_PLACEHOLDER
     - name: NEXTAUTH_URL
-      value: http://localhost:LANGFUSE_WEB_PORT_PLACEHOLDER
+      value: NEXTAUTH_URL_PLACEHOLDER
     - name: SALT
       value: SALT_PLACEHOLDER
     - name: ENCRYPTION_KEY

scripts/benchmark_classifier.py

@@ -6,6 +6,10 @@
 from collections import defaultdict, Counter
 from pathlib import Path
 
+# Shared chat response parser (used by verification scripts too)
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from chat_helpers import parse_chat_response
+
 # Load dataset
 dataset_path = Path(__file__).resolve().parent.parent / "data" / "classified_dataset.json"
 with open(dataset_path) as f:
@@ -42,10 +46,8 @@ def classify(prompt):
     )
     with urllib.request.urlopen(req, timeout=30) as resp:
         data = json.loads(resp.read())
-    choices = data.get("choices", [])
-    if not choices:
-        return "ERROR"
-    return choices[0].get("message", {}).get("content", "").strip()
+    content, _ = parse_chat_response(data)
+    return content if content else "ERROR"
 
 total = len(dataset.get("prompts", []))
 print(f"Benchmark: gemma4-26a4b-routing vs {total} labeled prompts\n")

scripts/chat_helpers.py

@@ -0,0 +1,32 @@
+"""Shared helpers for defensive chat completion response parsing.
+
+Used by the canonical endpoint verification script and the classifier scripts
+to safely extract content and reasoning_content from OpenAI-compatible API responses.
+"""
+
+from typing import Any
+
+
+def parse_chat_response(data: Any) -> tuple[str, str]:
+    """Safely extract content and reasoning_content from a chat completion response.
+
+    Args:
+        data: Parsed JSON response from a chat completion endpoint (expected to be a dict).
+
+    Returns:
+        (content, reasoning_content) — both may be empty strings.
+    """
+    if not isinstance(data, dict):
+        return "", ""
+    choices = data.get("choices")
+    if not isinstance(choices, list) or not choices:
+        return "", ""
+    first_choice = choices[0]
+    if not isinstance(first_choice, dict):
+        return "", ""
+    message = first_choice.get("message")
+    if not isinstance(message, dict):
+        return "", ""
+    content = (message.get("content") or "").strip()
+    reasoning = (message.get("reasoning_content") or "").strip()
+    return content, reasoning

scripts/classify_direct.py

@@ -1,8 +1,12 @@
 """Direct classification of Hermes prompts using gemma4-26a4b-routing."""
 import os
-import json, urllib.request, time
+import json, urllib.request, time, sys
 from pathlib import Path
 
+# Shared chat response parser (used by verification scripts too)
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from chat_helpers import parse_chat_response
+
 PROMPT_TEMPLATE = """Classify the coding task complexity. Output ONLY the tier name.
 
 agent-simple-core: trivial one-liners, syntax fixes, single-line edits
@@ -33,7 +37,8 @@ def classify(prompt):
     )
     with urllib.request.urlopen(req, timeout=30) as resp:
         data = json.loads(resp.read())
-    return data["choices"][0]["message"].get("content", "").strip()
+    content, _ = parse_chat_response(data)
+    return content if content else "ERROR"
 
 # Load prompts
 data_dir = Path(__file__).resolve().parent.parent / "data"

scripts/reclassify_all.py

@@ -7,6 +7,10 @@
 from pathlib import Path
 from collections import Counter
 
+# Shared chat response parser (used by verification scripts too)
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from chat_helpers import parse_chat_response
+
 TIERS = ['agent-simple-core','agent-medium-core','agent-complex-core','agent-reasoning-core','agent-advanced-core']
 
 LLAMA_SERVER_URL = "http://127.0.0.1:8080/v1/chat/completions"
@@ -32,10 +36,8 @@ def classify(prompt):
     req = urllib.request.Request(LLAMA_SERVER_URL, data=json.dumps(payload).encode(), headers={'Content-Type':'application/json','Authorization': f'Bearer {os.environ.get("ROUTER_API_KEY", "local-token")}'})
     with urllib.request.urlopen(req, timeout=30) as resp:
         data = json.loads(resp.read())
-    choices = data.get('choices', [])
-    if not choices:
-        return "ERROR: empty response"
-    return choices[0].get('message', {}).get('content', '').strip()
+    content, _ = parse_chat_response(data)
+    return content if content else "ERROR: empty response"
 
 # Load existing dataset (kanban/llm evals)
 data_dir = Path(__file__).resolve().parent.parent / 'data'

scripts/retry_errors.py

@@ -1,8 +1,12 @@
 """Retry the 94 failed prompts with 800-char truncation (safe for 4096-ctx model)."""
-import json, urllib.request, time, tempfile, os
+import json, urllib.request, time, tempfile, os, sys
 from pathlib import Path
 from collections import Counter
 
+# Shared chat response parser (used by verification scripts too)
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from chat_helpers import parse_chat_response
+
 PROMPT_TEMPLATE = """Classify the coding task complexity. Output ONLY the tier name.
 
 agent-simple-core: trivial one-liners, syntax fixes, single-line edits
@@ -50,11 +54,10 @@ def classify(prompt):
     )
     with urllib.request.urlopen(req, timeout=30) as resp:
         data = json.loads(resp.read())
-    choices = data.get("choices", [])
-    content = ""
-    if choices:
-        content = choices[0].get("message", {}).get("content", "").strip()
-    # Normalize: strip "tier:" prefix, extract just the tier name
+    content, _ = parse_chat_response(data)
+    if not content:
+        return "ERROR"
+    # retry_errors specific: normalize tier name
     for tier in TIERS:
         if tier in content:
             return tier