From fb94aec07fa2dc46bd0b4125479dd79a4e740360 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 22:45:50 +0200 Subject: [PATCH 1/9] test: add canonical endpoint verification script Reads ports/URLs from .env (and .env.dev overlay), validates: - Router: /v1/models, /metrics, /dashboard, /api/dashboard-stats - LiteLLM: /health/liveness, /health/readiness, /v1/models - Langfuse: /api/public/health - E2E: 3 chat completions through triage router - Canonical HTTPS URLs (graceful skip on DNS failure) Usage: python scripts/verification/verify_canonical_endpoints.py # prod python scripts/verification/verify_canonical_endpoints.py --dev # dev --- .../verify_canonical_endpoints.py | 323 ++++++++++++++++++ 1 file changed, 323 insertions(+) create mode 100644 scripts/verification/verify_canonical_endpoints.py diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py new file mode 100644 index 00000000..d7721d75 --- /dev/null +++ b/scripts/verification/verify_canonical_endpoints.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +""" +Canonical endpoint verification — reads ports/URLs from .env files, +validates basic API endpoints, and runs E2E chat completion requests. + +Usage: + python scripts/verification/verify_canonical_endpoints.py # prod (default) + python scripts/verification/verify_canonical_endpoints.py --dev # dev overlay + python scripts/verification/verify_canonical_endpoints.py --prod # explicit prod +""" +import os +import sys +import json +import time +import argparse +import httpx +from pathlib import Path + +WORKDIR = Path(__file__).resolve().parent.parent.parent + + +def load_env(dev: bool = False) -> dict: + """Load .env (and optionally .env.dev overlay), return resolved config dict.""" + env = {} + + def _parse(path: Path): + if not path.exists(): + return + with open(path) as f: + for line in f: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, val = line.partition("=") + key = key.strip() + val = val.strip().strip('"').strip("'") + env[key] = val + + _parse(WORKDIR / ".env") + if dev: + _parse(WORKDIR / ".env.dev") + + # Resolve with defaults + return { + "router_port": env.get("ROUTER_PORT", "5000"), + "litellm_port": env.get("LITELLM_PORT", "4000"), + "langfuse_web_port": env.get("LANGFUSE_WEB_PORT", "3001"), + "litellm_master_key": env.get("LITELLM_MASTER_KEY", "gateway-pass"), + "router_api_key": env.get("ROUTER_API_KEY", "gateway-pass"), + "public_base_url": env.get("PUBLIC_BASE_URL", ""), + "base_url": env.get("BASE_URL", env.get("BASEURL", "x570.vendeuvre.lan")), + } + + +def check(label: str, ok: bool, detail: str = "") -> bool: + mark = "✓" if ok else "✗" + extra = f" — {detail}" if detail else "" + print(f" {mark} {label}{extra}") + return ok + + +def test_router_endpoints(cfg: dict) -> tuple[int, int]: + """Test router API endpoints. Returns (passed, total).""" + base = f"http://127.0.0.1:{cfg['router_port']}" + key = cfg["router_api_key"] + headers = {"Authorization": f"Bearer {key}"} + passed = total = 0 + + print(f"\n── Router endpoints ({base}) ──") + + # /v1/models + total += 1 + try: + r = httpx.get(f"{base}/v1/models", headers=headers, timeout=10) + models = r.json() + model_ids = [m["id"] for m in models.get("data", [])] + ok = r.status_code == 200 and len(model_ids) > 0 + passed += check("/v1/models", ok, f"{len(model_ids)} models") + except Exception as e: + passed += check("/v1/models", False, str(e)) + + # /metrics + total += 1 + try: + r = httpx.get(f"{base}/metrics", timeout=10) + ok = r.status_code == 200 and "triage_requests_total" in r.text + passed += check("/metrics", ok) + except Exception as e: + passed += check("/metrics", False, str(e)) + + # /dashboard + total += 1 + try: + r = httpx.get(f"{base}/dashboard", timeout=10) + ok = r.status_code == 200 and " tuple[int, int]: + """Test LiteLLM health endpoints. Returns (passed, total).""" + base = f"http://127.0.0.1:{cfg['litellm_port']}" + key = cfg["litellm_master_key"] + headers = {"Authorization": f"Bearer {key}"} + passed = total = 0 + + print(f"\n── LiteLLM endpoints ({base}) ──") + + # /health/liveness + total += 1 + try: + r = httpx.get(f"{base}/health/liveness", timeout=10) + passed += check("/health/liveness", r.status_code == 200) + except Exception as e: + passed += check("/health/liveness", False, str(e)) + + # /health/readiness + total += 1 + try: + r = httpx.get(f"{base}/health/readiness", timeout=10) + passed += check("/health/readiness", r.status_code == 200) + except Exception as e: + passed += check("/health/readiness", False, str(e)) + + # /v1/models (LiteLLM) + total += 1 + try: + r = httpx.get(f"{base}/v1/models", headers=headers, timeout=10) + models = r.json() + model_ids = [m["id"] for m in models.get("data", [])] + ok = r.status_code == 200 and len(model_ids) > 0 + passed += check("/v1/models", ok, f"{len(model_ids)} models") + except Exception as e: + passed += check("/v1/models", False, str(e)) + + return passed, total + + +def test_langfuse_endpoints(cfg: dict) -> tuple[int, int]: + """Test Langfuse health endpoint. Returns (passed, total).""" + base = f"http://127.0.0.1:{cfg['langfuse_web_port']}" + passed = total = 0 + + print(f"\n── Langfuse endpoints ({base}) ──") + + total += 1 + try: + r = httpx.get(f"{base}/api/public/health", timeout=10) + ok = r.status_code == 200 + passed += check("/api/public/health", ok, r.json() if ok else r.text[:80]) + except Exception as e: + passed += check("/api/public/health", False, str(e)) + + return passed, total + + +def test_e2e_chat(cfg: dict) -> tuple[int, int]: + """Run E2E chat completion requests through the triage router. Returns (passed, total).""" + base = f"http://127.0.0.1:{cfg['router_port']}" + key = cfg["router_api_key"] + headers = { + "Authorization": f"Bearer {key}", + "Content-Type": "application/json", + } + passed = total = 0 + + print(f"\n── E2E chat completions ({base}/v1/chat/completions) ──") + + tests = [ + { + "label": "llm-routing-auto-free (simple)", + "model": "llm-routing-auto-free", + "prompt": "Say 'hello' in exactly one word.", + "max_tokens": 5, + }, + { + "label": "agent-simple-core (direct)", + "model": "agent-simple-core", + "prompt": "What is 2+2? Answer with just the number.", + "max_tokens": 5, + }, + { + "label": "llm-routing-auto-free (medium)", + "model": "llm-routing-auto-free", + "prompt": "Explain what a Python decorator is in one sentence.", + "max_tokens": 50, + }, + ] + + for test in tests: + total += 1 + payload = { + "model": test["model"], + "messages": [{"role": "user", "content": test["prompt"]}], + "max_tokens": test["max_tokens"], + } + try: + start = time.time() + r = httpx.post( + f"{base}/v1/chat/completions", + json=payload, + headers=headers, + timeout=120, + ) + elapsed = time.time() - start + if r.status_code == 200: + data = r.json() + content = (data["choices"][0]["message"].get("content") or "").strip() + reasoning = (data["choices"][0]["message"].get("reasoning_content") or "").strip() + model_used = data.get("model", "?") + ok = len(content) > 0 or len(reasoning) > 0 + detail = f"model={model_used}, {elapsed:.1f}s" + if content: + detail += f", '{content[:60]}'" + elif reasoning: + detail += f", reasoning='{reasoning[:60]}'" + passed += check( + test["label"], + ok, + detail, + ) + else: + passed += check( + test["label"], + False, + f"HTTP {r.status_code}: {r.text[:120]}", + ) + except Exception as e: + passed += check(test["label"], False, str(e)) + + return passed, total + + +def test_canonical_urls(cfg: dict) -> tuple[int, int]: + """Verify canonical HTTPS URLs are reachable (if PUBLIC_BASE_URL is set).""" + passed = total = 0 + public = cfg["public_base_url"] + if not public: + return 0, 0 + + print(f"\n── Canonical URLs ({public}) ──") + + endpoints = [ + ("/v1/models", "router models"), + ("/dashboard", "dashboard"), + ("/metrics", "metrics"), + ] + + for path, label in endpoints: + total += 1 + url = f"{public}{path}" + try: + r = httpx.get(url, timeout=15, follow_redirects=True) + ok = r.status_code == 200 + passed += check(f"GET {url}", ok, f"HTTP {r.status_code}") + except httpx.ConnectError as e: + # DNS/unreachable — skip gracefully (host may not resolve from test machine) + passed += check(f"GET {url}", True, f"SKIP: DNS/unreachable ({e})") + except Exception as e: + passed += check(f"GET {url}", False, str(e)[:100]) + + return passed, total + + +def main(): + parser = argparse.ArgumentParser(description="Verify canonical endpoints") + parser.add_argument( + "--dev", action="store_true", help="Test dev environment (dev-router-pod)" + ) + parser.add_argument( + "--prod", action="store_true", help="Test prod environment (agent-router-pod, default)" + ) + args = parser.parse_args() + + dev = args.dev and not args.prod + env_label = "DEV" if dev else "PROD" + cfg = load_env(dev=dev) + + print(f"=== Canonical Endpoint Verification [{env_label}] ===") + print(f" Router port : {cfg['router_port']}") + print(f" LiteLLM port: {cfg['litellm_port']}") + print(f" Langfuse port: {cfg['langfuse_web_port']}") + if cfg["public_base_url"]: + print(f" Public URL : {cfg['public_base_url']}") + + total_passed = total_tests = 0 + + for name, fn in [ + ("Router API", test_router_endpoints), + ("LiteLLM health", test_litellm_endpoints), + ("Langfuse health", test_langfuse_endpoints), + ("E2E chat", test_e2e_chat), + ("Canonical URLs", test_canonical_urls), + ]: + p, t = fn(cfg) + total_passed += p + total_tests += t + + print(f"\n{'='*60}") + print(f"Results [{env_label}]: {total_passed}/{total_tests} passed") + if total_passed < total_tests: + print(f"FAILED: {total_tests - total_passed} test(s)") + sys.exit(1) + else: + print("ALL PASSED") + sys.exit(0) + + +if __name__ == "__main__": + main() From 3201825763a2b0ee10e1305841a290ae4068ef9d Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 23:10:24 +0200 Subject: [PATCH 2/9] fix: LiteLLM UI credentials and Langfuse NEXTAUTH_URL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Pass UI_USERNAME/UI_PASSWORD from .env to LiteLLM container as LITELLM_UI_USERNAME/LITELLM_UI_PASSWORD (LiteLLM's expected env vars) - Derive NEXTAUTH_URL from PUBLIC_BASE_URL instead of hardcoding localhost — fixes Langfuse post-login redirect to broken URL --- pod.yaml | 6 +++++- start-stack.sh | 10 +++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/pod.yaml b/pod.yaml index 4214663e..45ee10e2 100644 --- a/pod.yaml +++ b/pod.yaml @@ -53,6 +53,10 @@ spec: value: http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER - name: LITELLM_MASTER_KEY value: LITELLM_MASTER_KEY_PLACEHOLDER + - name: LITELLM_UI_USERNAME + value: LITELLM_UI_USERNAME_PLACEHOLDER + - name: LITELLM_UI_PASSWORD + value: LITELLM_UI_PASSWORD_PLACEHOLDER - name: OLLAMA_API_KEY value: OLLAMA_API_KEY_PLACEHOLDER - name: SERVER_ROOT_PATH @@ -318,7 +322,7 @@ spec: - name: NEXTAUTH_SECRET value: NEXTAUTH_SECRET_PLACEHOLDER - name: NEXTAUTH_URL - value: http://localhost:LANGFUSE_WEB_PORT_PLACEHOLDER + value: NEXTAUTH_URL_PLACEHOLDER - name: SALT value: SALT_PLACEHOLDER - name: ENCRYPTION_KEY diff --git a/start-stack.sh b/start-stack.sh index f3399914..c1f274b4 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -612,7 +612,7 @@ render_router_config() { } render_pod_yaml() { - export WORKDIR HOME LITELLM_MASTER_KEY POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT + export WORKDIR HOME LITELLM_MASTER_KEY UI_USERNAME UI_PASSWORD POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT python3 - "$WORKDIR/pod.yaml" <<'PY' import os, sys, urllib.parse, json uid = os.getuid() @@ -627,9 +627,12 @@ placeholders = [ "HOME_PLACEHOLDER", "RUN_USER_PLACEHOLDER", "LITELLM_MASTER_KEY_PLACEHOLDER", + "LITELLM_UI_USERNAME_PLACEHOLDER", + "LITELLM_UI_PASSWORD_PLACEHOLDER", "POSTGRES_PASSWORD_RAW_PLACEHOLDER", "POSTGRES_PASSWORD_ENCODED_PLACEHOLDER", "NEXTAUTH_SECRET_PLACEHOLDER", + "NEXTAUTH_URL_PLACEHOLDER", "SALT_PLACEHOLDER", "ENCRYPTION_KEY_PLACEHOLDER", "OLLAMA_API_KEY_PLACEHOLDER", @@ -666,6 +669,8 @@ text = text.replace("WORKDIR_PLACEHOLDER", os.environ["WORKDIR"]) text = text.replace("HOME_PLACEHOLDER", os.environ["HOME"]) text = text.replace("RUN_USER_PLACEHOLDER", f"/run/user/{uid}") text = text.replace("LITELLM_MASTER_KEY_PLACEHOLDER", yaml_scalar(os.environ["LITELLM_MASTER_KEY"])) +text = text.replace("LITELLM_UI_USERNAME_PLACEHOLDER", yaml_scalar(os.environ.get("UI_USERNAME", "admin"))) +text = text.replace("LITELLM_UI_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ.get("UI_PASSWORD", "admin"))) text = text.replace("POSTGRES_PASSWORD_RAW_PLACEHOLDER", yaml_scalar(os.environ["POSTGRES_PASSWORD"])) # URL-encode the postgres password for DSN insertion encoded_password = urllib.parse.quote(os.environ['POSTGRES_PASSWORD'], safe="") @@ -686,6 +691,9 @@ public_base_url = os.environ["PUBLIC_BASE_URL"].rstrip("/") proxy_base_url = f"{public_base_url}/litellm" text = text.replace("PROXY_BASE_URL_PLACEHOLDER", yaml_scalar(proxy_base_url)) text = text.replace("PUBLIC_BASE_URL_PLACEHOLDER", yaml_scalar(os.environ["PUBLIC_BASE_URL"])) +# Derive NEXTAUTH_URL from PUBLIC_BASE_URL (Langfuse needs the public URL for OAuth redirects) +nextauth_url = f"{public_base_url}/langfuse" +text = text.replace("NEXTAUTH_URL_PLACEHOLDER", yaml_scalar(nextauth_url)) text = text.replace("ROUTING_DOMAIN_PLACEHOLDER", yaml_scalar(os.environ["ROUTING_DOMAIN"])) # Raw replacements (no quoting — used for pod name, data root, and integer port values) raw_replacements = { From 73b99fe29d30b1d8e59e411ef3acaec8ae137474 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 23:14:27 +0200 Subject: [PATCH 3/9] test: add LiteLLM UI and Langfuse web UI to canonical endpoint tests - LiteLLM: /llm-routing/litellm/ui/ (SERVER_ROOT_PATH prefix required) - Langfuse: / (web UI root) - Canonical URLs: /litellm/ui/ and /langfuse --- .../verify_canonical_endpoints.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index d7721d75..128fdceb 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -146,6 +146,15 @@ def test_litellm_endpoints(cfg: dict) -> tuple[int, int]: except Exception as e: passed += check("/v1/models", False, str(e)) + # /llm-routing/litellm/ui/ (LiteLLM admin UI — requires SERVER_ROOT_PATH prefix) + total += 1 + try: + r = httpx.get(f"{base}/llm-routing/litellm/ui/", timeout=10, follow_redirects=True) + ok = r.status_code == 200 and " tuple[int, int]: except Exception as e: passed += check("/api/public/health", False, str(e)) + # / (Langfuse web UI) + total += 1 + try: + r = httpx.get(f"{base}/", timeout=10, follow_redirects=True) + ok = r.status_code == 200 and " tuple[int, int]: ("/v1/models", "router models"), ("/dashboard", "dashboard"), ("/metrics", "metrics"), + ("/litellm/ui/", "LiteLLM admin UI"), + ("/langfuse", "Langfuse web UI"), ] for path, label in endpoints: From 664a1cdf25c68a25e928f56a5c01cdfbec32e1a6 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 23:19:23 +0200 Subject: [PATCH 4/9] test: add visualizer, infra health, and LiteLLM direct chat to verification - Router: /visualizer endpoint - Infrastructure: MinIO /minio/health/live, ClickHouse /ping - LiteLLM: direct chat completion (bypasses triage router) - Canonical URLs: /visualizer --- .../verify_canonical_endpoints.py | 80 ++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index 128fdceb..eb60b051 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -107,6 +107,15 @@ def test_router_endpoints(cfg: dict) -> tuple[int, int]: except Exception as e: passed += check("/api/dashboard-stats", False, str(e)) + # /visualizer + total += 1 + try: + r = httpx.get(f"{base}/visualizer", timeout=10) + ok = r.status_code == 200 and " tuple[int, int]: return passed, total +def test_infra_health(cfg: dict) -> tuple[int, int]: + """Test infrastructure services: MinIO, ClickHouse. Returns (passed, total).""" + passed = total = 0 + + print(f"\n── Infrastructure health ──") + + # MinIO S3 health + total += 1 + try: + r = httpx.get("http://127.0.0.1:9002/minio/health/live", timeout=10) + passed += check("MinIO /minio/health/live", r.status_code == 200) + except Exception as e: + passed += check("MinIO /minio/health/live", False, str(e)) + + # ClickHouse HTTP ping + total += 1 + try: + r = httpx.get("http://127.0.0.1:8123/ping", timeout=10) + passed += check("ClickHouse /ping", r.status_code == 200 and r.text.strip() == "Ok.") + except Exception as e: + passed += check("ClickHouse /ping", False, str(e)) + + return passed, total + + +def test_litellm_direct_chat(cfg: dict) -> tuple[int, int]: + """Test a direct chat completion through LiteLLM (bypassing the triage router).""" + base = f"http://127.0.0.1:{cfg['litellm_port']}" + key = cfg["litellm_master_key"] + headers = { + "Authorization": f"Bearer {key}", + "Content-Type": "application/json", + } + passed = total = 0 + + print(f"\n── LiteLLM direct chat ({base}/v1/chat/completions) ──") + + total += 1 + payload = { + "model": "agent-simple-core", + "messages": [{"role": "user", "content": "Say 'ok' and nothing else."}], + "max_tokens": 5, + } + try: + start = time.time() + r = httpx.post(f"{base}/v1/chat/completions", json=payload, headers=headers, timeout=60) + elapsed = time.time() - start + if r.status_code == 200: + data = r.json() + content = (data["choices"][0]["message"].get("content") or "").strip() + reasoning = (data["choices"][0]["message"].get("reasoning_content") or "").strip() + ok = len(content) > 0 or len(reasoning) > 0 + detail = f"{elapsed:.1f}s" + if content: + detail += f", '{content[:40]}'" + elif reasoning: + detail += f", reasoning='{reasoning[:40]}'" + passed += check("agent-simple-core (direct)", ok, detail) + else: + passed += check("agent-simple-core (direct)", False, f"HTTP {r.status_code}: {r.text[:100]}") + except Exception as e: + passed += check("agent-simple-core (direct)", False, str(e)) + + return passed, total + + def test_canonical_urls(cfg: dict) -> tuple[int, int]: """Verify canonical HTTPS URLs are reachable (if PUBLIC_BASE_URL is set).""" passed = total = 0 @@ -275,6 +350,7 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int]: ("/v1/models", "router models"), ("/dashboard", "dashboard"), ("/metrics", "metrics"), + ("/visualizer", "visualizer"), ("/litellm/ui/", "LiteLLM admin UI"), ("/langfuse", "Langfuse web UI"), ] @@ -322,7 +398,9 @@ def main(): ("Router API", test_router_endpoints), ("LiteLLM health", test_litellm_endpoints), ("Langfuse health", test_langfuse_endpoints), - ("E2E chat", test_e2e_chat), + ("Infrastructure", test_infra_health), + ("E2E chat (router)", test_e2e_chat), + ("LiteLLM direct chat", test_litellm_direct_chat), ("Canonical URLs", test_canonical_urls), ]: p, t = fn(cfg) From 678a8ccb93b6f88a5ccba710982e0fc4a284885d Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 00:12:37 +0200 Subject: [PATCH 5/9] test: add canonical POST chat completion to verification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - POST agent-simple-core through public URL (dev.vendeuvre.lan) - Validates full HTTPS path: HAProxy → dev router → LiteLLM → model - Graceful DNS skip when host unreachable --- .../verify_canonical_endpoints.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index eb60b051..ff338124 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -368,6 +368,32 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int]: except Exception as e: passed += check(f"GET {url}", False, str(e)[:100]) + # Canonical chat completion (POST through public URL) + total += 1 + url = f"{public}/v1/chat/completions" + try: + payload = { + "model": "agent-simple-core", + "messages": [{"role": "user", "content": "What is 2+2? Answer with just the number."}], + "max_tokens": 5, + } + r = httpx.post(url, json=payload, headers={"Authorization": f"Bearer {cfg['router_api_key']}"}, timeout=60, follow_redirects=True) + if r.status_code == 200: + data = r.json() + content = (data["choices"][0]["message"].get("content") or "").strip() + reasoning = (data["choices"][0]["message"].get("reasoning_content") or "").strip() + ok = len(content) > 0 or len(reasoning) > 0 + detail = f"HTTP {r.status_code}" + if content: + detail += f", '{content[:30]}'" + passed += check(f"POST {url}", ok, detail) + else: + passed += check(f"POST {url}", False, f"HTTP {r.status_code}: {r.text[:80]}") + except httpx.ConnectError as e: + passed += check(f"POST {url}", True, f"SKIP: DNS/unreachable ({e})") + except Exception as e: + passed += check(f"POST {url}", False, str(e)[:100]) + return passed, total From 208d2a369836a4017e1b293dbb22c6acb1743fd9 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 00:21:45 +0200 Subject: [PATCH 6/9] fix: defensive choices access + differentiate skipped from passed - All three chat completion parsers now use .get() on choices/message instead of direct indexing (Gemini Code Assist review) - Canonical URL skipped checks are tracked separately and shown in summary as 'X skipped' (Sourcery review) --- .../verify_canonical_endpoints.py | 49 +++++++++++++------ 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index ff338124..576f398e 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -245,8 +245,10 @@ def test_e2e_chat(cfg: dict) -> tuple[int, int]: elapsed = time.time() - start if r.status_code == 200: data = r.json() - content = (data["choices"][0]["message"].get("content") or "").strip() - reasoning = (data["choices"][0]["message"].get("reasoning_content") or "").strip() + choices = data.get("choices") + message = choices[0].get("message", {}) if choices else {} + content = (message.get("content") or "").strip() + reasoning = (message.get("reasoning_content") or "").strip() model_used = data.get("model", "?") ok = len(content) > 0 or len(reasoning) > 0 detail = f"model={model_used}, {elapsed:.1f}s" @@ -320,8 +322,10 @@ def test_litellm_direct_chat(cfg: dict) -> tuple[int, int]: elapsed = time.time() - start if r.status_code == 200: data = r.json() - content = (data["choices"][0]["message"].get("content") or "").strip() - reasoning = (data["choices"][0]["message"].get("reasoning_content") or "").strip() + choices = data.get("choices") + message = choices[0].get("message", {}) if choices else {} + content = (message.get("content") or "").strip() + reasoning = (message.get("reasoning_content") or "").strip() ok = len(content) > 0 or len(reasoning) > 0 detail = f"{elapsed:.1f}s" if content: @@ -337,12 +341,13 @@ def test_litellm_direct_chat(cfg: dict) -> tuple[int, int]: return passed, total -def test_canonical_urls(cfg: dict) -> tuple[int, int]: - """Verify canonical HTTPS URLs are reachable (if PUBLIC_BASE_URL is set).""" - passed = total = 0 +def test_canonical_urls(cfg: dict) -> tuple[int, int, int]: + """Verify canonical HTTPS URLs are reachable (if PUBLIC_BASE_URL is set). + Returns (passed, total, skipped).""" + passed = total = skipped = 0 public = cfg["public_base_url"] if not public: - return 0, 0 + return 0, 0, 0 print(f"\n── Canonical URLs ({public}) ──") @@ -364,7 +369,8 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int]: passed += check(f"GET {url}", ok, f"HTTP {r.status_code}") except httpx.ConnectError as e: # DNS/unreachable — skip gracefully (host may not resolve from test machine) - passed += check(f"GET {url}", True, f"SKIP: DNS/unreachable ({e})") + skipped += 1 + check(f"GET {url}", True, f"SKIP: DNS/unreachable ({e})") except Exception as e: passed += check(f"GET {url}", False, str(e)[:100]) @@ -380,8 +386,10 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int]: r = httpx.post(url, json=payload, headers={"Authorization": f"Bearer {cfg['router_api_key']}"}, timeout=60, follow_redirects=True) if r.status_code == 200: data = r.json() - content = (data["choices"][0]["message"].get("content") or "").strip() - reasoning = (data["choices"][0]["message"].get("reasoning_content") or "").strip() + choices = data.get("choices") + message = choices[0].get("message", {}) if choices else {} + content = (message.get("content") or "").strip() + reasoning = (message.get("reasoning_content") or "").strip() ok = len(content) > 0 or len(reasoning) > 0 detail = f"HTTP {r.status_code}" if content: @@ -390,11 +398,12 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int]: else: passed += check(f"POST {url}", False, f"HTTP {r.status_code}: {r.text[:80]}") except httpx.ConnectError as e: - passed += check(f"POST {url}", True, f"SKIP: DNS/unreachable ({e})") + skipped += 1 + check(f"POST {url}", True, f"SKIP: DNS/unreachable ({e})") except Exception as e: passed += check(f"POST {url}", False, str(e)[:100]) - return passed, total + return passed, total, skipped def main(): @@ -418,7 +427,7 @@ def main(): if cfg["public_base_url"]: print(f" Public URL : {cfg['public_base_url']}") - total_passed = total_tests = 0 + total_passed = total_tests = total_skipped = 0 for name, fn in [ ("Router API", test_router_endpoints), @@ -429,12 +438,20 @@ def main(): ("LiteLLM direct chat", test_litellm_direct_chat), ("Canonical URLs", test_canonical_urls), ]: - p, t = fn(cfg) + result = fn(cfg) + if len(result) == 3: + p, t, s = result + total_skipped += s + else: + p, t = result total_passed += p total_tests += t print(f"\n{'='*60}") - print(f"Results [{env_label}]: {total_passed}/{total_tests} passed") + parts = [f"{total_passed}/{total_tests} passed"] + if total_skipped: + parts.append(f"{total_skipped} skipped") + print(f"Results [{env_label}]: {', '.join(parts)}") if total_passed < total_tests: print(f"FAILED: {total_tests - total_passed} test(s)") sys.exit(1) From c591995e22f3edaabbb508ad683b1b716c738e8f Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 00:22:30 +0200 Subject: [PATCH 7/9] docs: add canonical endpoint verification to README --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index 351e6568..74a44c79 100644 --- a/README.md +++ b/README.md @@ -822,6 +822,32 @@ Execute the verification script: ./scripts/verification/verify_reasoning_tiers.py ``` +### Canonical Endpoint Verification + +A comprehensive endpoint health check that validates all services across both prod and dev environments: + +```bash +# Prod (default) +python scripts/verification/verify_canonical_endpoints.py + +# Dev +python scripts/verification/verify_canonical_endpoints.py --dev +``` + +Tests cover: + +| Section | Endpoints | +|---------|-----------| +| Router API | `/v1/models`, `/metrics`, `/dashboard`, `/api/dashboard-stats`, `/visualizer` | +| LiteLLM | `/health/liveness`, `/health/readiness`, `/v1/models`, `/llm-routing/litellm/ui/` | +| Langfuse | `/api/public/health`, `/` (web UI) | +| Infrastructure | MinIO `/minio/health/live`, ClickHouse `/ping` | +| E2E chat | 3 completions through triage router | +| LiteLLM direct | 1 completion directly to LiteLLM | +| Canonical URLs | 6 GET + 1 POST through public HTTPS (graceful DNS skip) | + +Requires `PUBLIC_BASE_URL` in `.env` for canonical URL tests. Dev `.env.dev` already has it; prod `.env` should include `PUBLIC_BASE_URL="https://x570.vendeuvre.lan/llm-routing"`. + ## 10. Performance Benchmarks Through our local benchmarks, the following performance characteristics have been achieved: From 993b19549f8462fe78786295e33007b95cecaae2 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 00:32:29 +0200 Subject: [PATCH 8/9] =?UTF-8?q?fix:=20address=20review=20comments=20?= =?UTF-8?q?=E2=80=94=20refactor=20chat=20parsing,=20env-aware=20infra=20po?= =?UTF-8?q?rts,=20.env=20logging?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Extract parse_chat_response() helper to deduplicate chat completion parsing across test_e2e_chat, test_litellm_direct_chat, and test_canonical_urls - Add isinstance guards for defensive JSON response parsing (Gemini suggestion) - Load MinIO and ClickHouse ports from .env (MINIO_S3_PORT, CLICKHOUSE_HTTP_PORT) instead of hardcoding 9002/8123 (Sourcery suggestion) - Log which .env files were loaded and warn on missing files (Sourcery suggestion) - Handle export prefix and inline comments in .env parser (Gemini suggestion) --- .../verify_canonical_endpoints.py | 64 ++++++++++++++----- 1 file changed, 48 insertions(+), 16 deletions(-) diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index 576f398e..2a831949 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -22,14 +22,30 @@ def load_env(dev: bool = False) -> dict: """Load .env (and optionally .env.dev overlay), return resolved config dict.""" env = {} + loaded_files = [] def _parse(path: Path): if not path.exists(): + print(f" ⚠ env file not found: {path}") return + loaded_files.append(str(path.name)) with open(path) as f: for line in f: line = line.strip() - if not line or line.startswith("#") or "=" not in line: + # Skip empty lines and full-line comments + if not line or line.startswith("#"): + continue + # Strip inline comments (but not inside quoted values) + if "#" in line: + # Simple heuristic: split on first # that's preceded by whitespace + for i, ch in enumerate(line): + if ch == "#" and (i == 0 or line[i - 1] in " \t"): + line = line[:i].strip() + break + # Handle export prefix + if line.startswith("export "): + line = line[7:].strip() + if "=" not in line: continue key, _, val = line.partition("=") key = key.strip() @@ -40,6 +56,8 @@ def _parse(path: Path): if dev: _parse(WORKDIR / ".env.dev") + print(f" Loaded env files: {', '.join(loaded_files)}") + # Resolve with defaults return { "router_port": env.get("ROUTER_PORT", "5000"), @@ -49,6 +67,8 @@ def _parse(path: Path): "router_api_key": env.get("ROUTER_API_KEY", "gateway-pass"), "public_base_url": env.get("PUBLIC_BASE_URL", ""), "base_url": env.get("BASE_URL", env.get("BASEURL", "x570.vendeuvre.lan")), + "minio_s3_port": env.get("MINIO_S3_PORT", "9002"), + "clickhouse_http_port": env.get("CLICKHOUSE_HTTP_PORT", "8123"), } @@ -59,6 +79,25 @@ def check(label: str, ok: bool, detail: str = "") -> bool: return ok +def parse_chat_response(data: dict) -> tuple[str, str]: + """Safely extract content and reasoning_content from a chat completion response. + Returns (content, reasoning_content) — both may be empty strings.""" + if not isinstance(data, dict): + return "", "" + choices = data.get("choices") + if not isinstance(choices, list) or not choices: + return "", "" + first_choice = choices[0] + if not isinstance(first_choice, dict): + return "", "" + message = first_choice.get("message") + if not isinstance(message, dict): + return "", "" + content = (message.get("content") or "").strip() + reasoning = (message.get("reasoning_content") or "").strip() + return content, reasoning + + def test_router_endpoints(cfg: dict) -> tuple[int, int]: """Test router API endpoints. Returns (passed, total).""" base = f"http://127.0.0.1:{cfg['router_port']}" @@ -245,11 +284,8 @@ def test_e2e_chat(cfg: dict) -> tuple[int, int]: elapsed = time.time() - start if r.status_code == 200: data = r.json() - choices = data.get("choices") - message = choices[0].get("message", {}) if choices else {} - content = (message.get("content") or "").strip() - reasoning = (message.get("reasoning_content") or "").strip() - model_used = data.get("model", "?") + content, reasoning = parse_chat_response(data) + model_used = data.get("model", "?") if isinstance(data, dict) else "?" ok = len(content) > 0 or len(reasoning) > 0 detail = f"model={model_used}, {elapsed:.1f}s" if content: @@ -282,7 +318,8 @@ def test_infra_health(cfg: dict) -> tuple[int, int]: # MinIO S3 health total += 1 try: - r = httpx.get("http://127.0.0.1:9002/minio/health/live", timeout=10) + minio_port = cfg.get("minio_s3_port", "9002") + r = httpx.get(f"http://127.0.0.1:{minio_port}/minio/health/live", timeout=10) passed += check("MinIO /minio/health/live", r.status_code == 200) except Exception as e: passed += check("MinIO /minio/health/live", False, str(e)) @@ -290,7 +327,8 @@ def test_infra_health(cfg: dict) -> tuple[int, int]: # ClickHouse HTTP ping total += 1 try: - r = httpx.get("http://127.0.0.1:8123/ping", timeout=10) + ch_port = cfg.get("clickhouse_http_port", "8123") + r = httpx.get(f"http://127.0.0.1:{ch_port}/ping", timeout=10) passed += check("ClickHouse /ping", r.status_code == 200 and r.text.strip() == "Ok.") except Exception as e: passed += check("ClickHouse /ping", False, str(e)) @@ -322,10 +360,7 @@ def test_litellm_direct_chat(cfg: dict) -> tuple[int, int]: elapsed = time.time() - start if r.status_code == 200: data = r.json() - choices = data.get("choices") - message = choices[0].get("message", {}) if choices else {} - content = (message.get("content") or "").strip() - reasoning = (message.get("reasoning_content") or "").strip() + content, reasoning = parse_chat_response(data) ok = len(content) > 0 or len(reasoning) > 0 detail = f"{elapsed:.1f}s" if content: @@ -386,10 +421,7 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int, int]: r = httpx.post(url, json=payload, headers={"Authorization": f"Bearer {cfg['router_api_key']}"}, timeout=60, follow_redirects=True) if r.status_code == 200: data = r.json() - choices = data.get("choices") - message = choices[0].get("message", {}) if choices else {} - content = (message.get("content") or "").strip() - reasoning = (message.get("reasoning_content") or "").strip() + content, reasoning = parse_chat_response(data) ok = len(content) > 0 or len(reasoning) > 0 detail = f"HTTP {r.status_code}" if content: From f1d9cea1d0aea3a22d0cab02336187feb66aa073 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 00:33:37 +0200 Subject: [PATCH 9/9] fix: defensive chat response parsing in all classifier scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply isinstance guards on choices/message across benchmark_classifier.py, retry_errors.py, and reclassify_all.py — same pattern as the verification script's parse_chat_response() helper. Prevents unhandled exceptions when the API returns unexpected response structures. --- scripts/benchmark_classifier.py | 12 +++++++++--- scripts/reclassify_all.py | 12 +++++++++--- scripts/retry_errors.py | 10 +++++++--- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/scripts/benchmark_classifier.py b/scripts/benchmark_classifier.py index 301ad0e1..847edf98 100644 --- a/scripts/benchmark_classifier.py +++ b/scripts/benchmark_classifier.py @@ -42,10 +42,16 @@ def classify(prompt): ) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - choices = data.get("choices", []) - if not choices: + choices = data.get("choices") + if not isinstance(choices, list) or not choices: return "ERROR" - return choices[0].get("message", {}).get("content", "").strip() + first = choices[0] + if not isinstance(first, dict): + return "ERROR" + message = first.get("message") + if not isinstance(message, dict): + return "ERROR" + return (message.get("content") or "").strip() total = len(dataset.get("prompts", [])) print(f"Benchmark: gemma4-26a4b-routing vs {total} labeled prompts\n") diff --git a/scripts/reclassify_all.py b/scripts/reclassify_all.py index 1cdadbba..2d2e8f1e 100644 --- a/scripts/reclassify_all.py +++ b/scripts/reclassify_all.py @@ -32,10 +32,16 @@ def classify(prompt): req = urllib.request.Request(LLAMA_SERVER_URL, data=json.dumps(payload).encode(), headers={'Content-Type':'application/json','Authorization': f'Bearer {os.environ.get("ROUTER_API_KEY", "local-token")}'}) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - choices = data.get('choices', []) - if not choices: + choices = data.get('choices') + if not isinstance(choices, list) or not choices: return "ERROR: empty response" - return choices[0].get('message', {}).get('content', '').strip() + first = choices[0] + if not isinstance(first, dict): + return "ERROR: invalid choice" + message = first.get('message') + if not isinstance(message, dict): + return "ERROR: invalid message" + return (message.get('content') or '').strip() # Load existing dataset (kanban/llm evals) data_dir = Path(__file__).resolve().parent.parent / 'data' diff --git a/scripts/retry_errors.py b/scripts/retry_errors.py index a5bce029..ba6cb6a4 100644 --- a/scripts/retry_errors.py +++ b/scripts/retry_errors.py @@ -50,10 +50,14 @@ def classify(prompt): ) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - choices = data.get("choices", []) + choices = data.get("choices") content = "" - if choices: - content = choices[0].get("message", {}).get("content", "").strip() + if isinstance(choices, list) and choices: + first = choices[0] + if isinstance(first, dict): + message = first.get("message") + if isinstance(message, dict): + content = (message.get("content") or "").strip() # Normalize: strip "tier:" prefix, extract just the tier name for tier in TIERS: if tier in content: