diff --git a/README.md b/README.md index 351e6568..74a44c79 100644 --- a/README.md +++ b/README.md @@ -822,6 +822,32 @@ Execute the verification script: ./scripts/verification/verify_reasoning_tiers.py ``` +### Canonical Endpoint Verification + +A comprehensive endpoint health check that validates all services across both prod and dev environments: + +```bash +# Prod (default) +python scripts/verification/verify_canonical_endpoints.py + +# Dev +python scripts/verification/verify_canonical_endpoints.py --dev +``` + +Tests cover: + +| Section | Endpoints | +|---------|-----------| +| Router API | `/v1/models`, `/metrics`, `/dashboard`, `/api/dashboard-stats`, `/visualizer` | +| LiteLLM | `/health/liveness`, `/health/readiness`, `/v1/models`, `/llm-routing/litellm/ui/` | +| Langfuse | `/api/public/health`, `/` (web UI) | +| Infrastructure | MinIO `/minio/health/live`, ClickHouse `/ping` | +| E2E chat | 3 completions through triage router | +| LiteLLM direct | 1 completion directly to LiteLLM | +| Canonical URLs | 6 GET + 1 POST through public HTTPS (graceful DNS skip) | + +Requires `PUBLIC_BASE_URL` in `.env` for canonical URL tests. Dev `.env.dev` already has it; prod `.env` should include `PUBLIC_BASE_URL="https://x570.vendeuvre.lan/llm-routing"`. + ## 10. Performance Benchmarks Through our local benchmarks, the following performance characteristics have been achieved: diff --git a/pod.yaml b/pod.yaml index 4214663e..45ee10e2 100644 --- a/pod.yaml +++ b/pod.yaml @@ -53,6 +53,10 @@ spec: value: http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER - name: LITELLM_MASTER_KEY value: LITELLM_MASTER_KEY_PLACEHOLDER + - name: LITELLM_UI_USERNAME + value: LITELLM_UI_USERNAME_PLACEHOLDER + - name: LITELLM_UI_PASSWORD + value: LITELLM_UI_PASSWORD_PLACEHOLDER - name: OLLAMA_API_KEY value: OLLAMA_API_KEY_PLACEHOLDER - name: SERVER_ROOT_PATH @@ -318,7 +322,7 @@ spec: - name: NEXTAUTH_SECRET value: NEXTAUTH_SECRET_PLACEHOLDER - name: NEXTAUTH_URL - value: http://localhost:LANGFUSE_WEB_PORT_PLACEHOLDER + value: NEXTAUTH_URL_PLACEHOLDER - name: SALT value: SALT_PLACEHOLDER - name: ENCRYPTION_KEY diff --git a/scripts/benchmark_classifier.py b/scripts/benchmark_classifier.py index 301ad0e1..e12f9649 100644 --- a/scripts/benchmark_classifier.py +++ b/scripts/benchmark_classifier.py @@ -6,6 +6,10 @@ from collections import defaultdict, Counter from pathlib import Path +# Shared chat response parser (used by verification scripts too) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from chat_helpers import parse_chat_response + # Load dataset dataset_path = Path(__file__).resolve().parent.parent / "data" / "classified_dataset.json" with open(dataset_path) as f: @@ -42,10 +46,8 @@ def classify(prompt): ) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - choices = data.get("choices", []) - if not choices: - return "ERROR" - return choices[0].get("message", {}).get("content", "").strip() + content, _ = parse_chat_response(data) + return content if content else "ERROR" total = len(dataset.get("prompts", [])) print(f"Benchmark: gemma4-26a4b-routing vs {total} labeled prompts\n") diff --git a/scripts/chat_helpers.py b/scripts/chat_helpers.py new file mode 100644 index 00000000..a7cf85fe --- /dev/null +++ b/scripts/chat_helpers.py @@ -0,0 +1,32 @@ +"""Shared helpers for defensive chat completion response parsing. + +Used by the canonical endpoint verification script and the classifier scripts +to safely extract content and reasoning_content from OpenAI-compatible API responses. +""" + +from typing import Any + + +def parse_chat_response(data: Any) -> tuple[str, str]: + """Safely extract content and reasoning_content from a chat completion response. + + Args: + data: Parsed JSON response from a chat completion endpoint (expected to be a dict). + + Returns: + (content, reasoning_content) — both may be empty strings. + """ + if not isinstance(data, dict): + return "", "" + choices = data.get("choices") + if not isinstance(choices, list) or not choices: + return "", "" + first_choice = choices[0] + if not isinstance(first_choice, dict): + return "", "" + message = first_choice.get("message") + if not isinstance(message, dict): + return "", "" + content = (message.get("content") or "").strip() + reasoning = (message.get("reasoning_content") or "").strip() + return content, reasoning diff --git a/scripts/classify_direct.py b/scripts/classify_direct.py index 53556551..07748c40 100644 --- a/scripts/classify_direct.py +++ b/scripts/classify_direct.py @@ -1,8 +1,12 @@ """Direct classification of Hermes prompts using gemma4-26a4b-routing.""" import os -import json, urllib.request, time +import json, urllib.request, time, sys from pathlib import Path +# Shared chat response parser (used by verification scripts too) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from chat_helpers import parse_chat_response + PROMPT_TEMPLATE = """Classify the coding task complexity. Output ONLY the tier name. agent-simple-core: trivial one-liners, syntax fixes, single-line edits @@ -33,7 +37,8 @@ def classify(prompt): ) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - return data["choices"][0]["message"].get("content", "").strip() + content, _ = parse_chat_response(data) + return content if content else "ERROR" # Load prompts data_dir = Path(__file__).resolve().parent.parent / "data" diff --git a/scripts/reclassify_all.py b/scripts/reclassify_all.py index 1cdadbba..ed302438 100644 --- a/scripts/reclassify_all.py +++ b/scripts/reclassify_all.py @@ -7,6 +7,10 @@ from pathlib import Path from collections import Counter +# Shared chat response parser (used by verification scripts too) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from chat_helpers import parse_chat_response + TIERS = ['agent-simple-core','agent-medium-core','agent-complex-core','agent-reasoning-core','agent-advanced-core'] LLAMA_SERVER_URL = "http://127.0.0.1:8080/v1/chat/completions" @@ -32,10 +36,8 @@ def classify(prompt): req = urllib.request.Request(LLAMA_SERVER_URL, data=json.dumps(payload).encode(), headers={'Content-Type':'application/json','Authorization': f'Bearer {os.environ.get("ROUTER_API_KEY", "local-token")}'}) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - choices = data.get('choices', []) - if not choices: - return "ERROR: empty response" - return choices[0].get('message', {}).get('content', '').strip() + content, _ = parse_chat_response(data) + return content if content else "ERROR: empty response" # Load existing dataset (kanban/llm evals) data_dir = Path(__file__).resolve().parent.parent / 'data' diff --git a/scripts/retry_errors.py b/scripts/retry_errors.py index a5bce029..8ed6e34a 100644 --- a/scripts/retry_errors.py +++ b/scripts/retry_errors.py @@ -1,8 +1,12 @@ """Retry the 94 failed prompts with 800-char truncation (safe for 4096-ctx model).""" -import json, urllib.request, time, tempfile, os +import json, urllib.request, time, tempfile, os, sys from pathlib import Path from collections import Counter +# Shared chat response parser (used by verification scripts too) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from chat_helpers import parse_chat_response + PROMPT_TEMPLATE = """Classify the coding task complexity. Output ONLY the tier name. agent-simple-core: trivial one-liners, syntax fixes, single-line edits @@ -50,11 +54,10 @@ def classify(prompt): ) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - choices = data.get("choices", []) - content = "" - if choices: - content = choices[0].get("message", {}).get("content", "").strip() - # Normalize: strip "tier:" prefix, extract just the tier name + content, _ = parse_chat_response(data) + if not content: + return "ERROR" + # retry_errors specific: normalize tier name for tier in TIERS: if tier in content: return tier diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py new file mode 100644 index 00000000..ba38c652 --- /dev/null +++ b/scripts/verification/verify_canonical_endpoints.py @@ -0,0 +1,477 @@ +#!/usr/bin/env python3 +""" +Canonical endpoint verification — reads ports/URLs from .env files, +validates basic API endpoints, and runs E2E chat completion requests. + +Usage: + python scripts/verification/verify_canonical_endpoints.py # prod (default) + python scripts/verification/verify_canonical_endpoints.py --dev # dev overlay + python scripts/verification/verify_canonical_endpoints.py --prod # explicit prod +""" +import os +import sys +import json +import time +import argparse +import httpx +from pathlib import Path + +WORKDIR = Path(__file__).resolve().parent.parent.parent + +# Import shared chat response parser (also used by classifier scripts) +sys.path.insert(0, str(WORKDIR / "scripts")) +from chat_helpers import parse_chat_response + + +def load_env(dev: bool = False) -> dict: + """Load .env (and optionally .env.dev overlay), return resolved config dict.""" + env = {} + loaded_files = [] + + def _parse(path: Path): + if not path.exists(): + print(f" ⚠ env file not found: {path}") + return + loaded_files.append(str(path.name)) + with open(path, encoding="utf-8") as f: + for line in f: + line = line.strip() + # Skip empty lines and full-line comments (no inline comments — + # they would risk corrupting values containing literal # characters + # such as passwords or URL fragments) + if not line or line.startswith("#"): + continue + # Handle export prefix + if line.startswith("export "): + line = line[7:].strip() + if "=" not in line: + continue + key, _, val = line.partition("=") + key = key.strip() + val = val.strip().strip('"').strip("'") + env[key] = val + + _parse(WORKDIR / ".env") + if dev: + _parse(WORKDIR / ".env.dev") + + print(f" Loaded env files: {', '.join(loaded_files)}") + + # Resolve with defaults + return { + "router_port": env.get("ROUTER_PORT", "5000"), + "litellm_port": env.get("LITELLM_PORT", "4000"), + "langfuse_web_port": env.get("LANGFUSE_WEB_PORT", "3001"), + "litellm_master_key": env.get("LITELLM_MASTER_KEY", "gateway-pass"), + "router_api_key": env.get("ROUTER_API_KEY", "gateway-pass"), + "public_base_url": env.get("PUBLIC_BASE_URL", "").rstrip("/"), + "minio_s3_port": env.get("MINIO_S3_PORT", "9002"), + "clickhouse_http_port": env.get("CLICKHOUSE_HTTP_PORT", "8123"), + } + + +def check(label: str, ok: bool, detail: str = "") -> bool: + mark = "✓" if ok else "✗" + extra = f" — {detail}" if detail else "" + print(f" {mark} {label}{extra}") + return ok + + +def test_router_endpoints(cfg: dict) -> tuple[int, int]: + """Test router API endpoints. Returns (passed, total).""" + base = f"http://127.0.0.1:{cfg['router_port']}" + key = cfg["router_api_key"] + headers = {"Authorization": f"Bearer {key}"} + passed = total = 0 + + print(f"\n── Router endpoints ({base}) ──") + + # /v1/models + total += 1 + try: + r = httpx.get(f"{base}/v1/models", headers=headers, timeout=10) + models = r.json() + model_ids = [m["id"] for m in models.get("data", [])] + ok = r.status_code == 200 and len(model_ids) > 0 + passed += check("/v1/models", ok, f"{len(model_ids)} models") + except Exception as e: + passed += check("/v1/models", False, str(e)) + + # /metrics + total += 1 + try: + r = httpx.get(f"{base}/metrics", timeout=10) + ok = r.status_code == 200 and "triage_requests_total" in r.text + passed += check("/metrics", ok) + except Exception as e: + passed += check("/metrics", False, str(e)) + + # /dashboard + total += 1 + try: + r = httpx.get(f"{base}/dashboard", timeout=10) + ok = r.status_code == 200 and " tuple[int, int]: + """Test LiteLLM health endpoints. Returns (passed, total).""" + base = f"http://127.0.0.1:{cfg['litellm_port']}" + key = cfg["litellm_master_key"] + headers = {"Authorization": f"Bearer {key}"} + passed = total = 0 + + print(f"\n── LiteLLM endpoints ({base}) ──") + + # /health/liveness + total += 1 + try: + r = httpx.get(f"{base}/health/liveness", timeout=10) + passed += check("/health/liveness", r.status_code == 200) + except Exception as e: + passed += check("/health/liveness", False, str(e)) + + # /health/readiness + total += 1 + try: + r = httpx.get(f"{base}/health/readiness", timeout=10) + passed += check("/health/readiness", r.status_code == 200) + except Exception as e: + passed += check("/health/readiness", False, str(e)) + + # /v1/models (LiteLLM) + total += 1 + try: + r = httpx.get(f"{base}/v1/models", headers=headers, timeout=10) + models = r.json() + model_ids = [m["id"] for m in models.get("data", [])] + ok = r.status_code == 200 and len(model_ids) > 0 + passed += check("/v1/models", ok, f"{len(model_ids)} models") + except Exception as e: + passed += check("/v1/models", False, str(e)) + + # /llm-routing/litellm/ui/ (LiteLLM admin UI — requires SERVER_ROOT_PATH prefix) + total += 1 + try: + r = httpx.get(f"{base}/llm-routing/litellm/ui/", timeout=10, follow_redirects=True) + ok = r.status_code == 200 and " tuple[int, int]: + """Test Langfuse health endpoint. Returns (passed, total).""" + base = f"http://127.0.0.1:{cfg['langfuse_web_port']}" + passed = total = 0 + + print(f"\n── Langfuse endpoints ({base}) ──") + + total += 1 + try: + r = httpx.get(f"{base}/api/public/health", timeout=10) + ok = r.status_code == 200 + passed += check("/api/public/health", ok, r.json() if ok else r.text[:80]) + except Exception as e: + passed += check("/api/public/health", False, str(e)) + + # / (Langfuse web UI) + total += 1 + try: + r = httpx.get(f"{base}/", timeout=10, follow_redirects=True) + ok = r.status_code == 200 and " tuple[int, int]: + """Run E2E chat completion requests through the triage router. Returns (passed, total).""" + base = f"http://127.0.0.1:{cfg['router_port']}" + key = cfg["router_api_key"] + headers = { + "Authorization": f"Bearer {key}", + "Content-Type": "application/json", + } + passed = total = 0 + + print(f"\n── E2E chat completions ({base}/v1/chat/completions) ──") + + tests = [ + { + "label": "llm-routing-auto-free (simple)", + "model": "llm-routing-auto-free", + "prompt": "Say 'hello' in exactly one word.", + "max_tokens": 5, + }, + { + "label": "agent-simple-core (direct)", + "model": "agent-simple-core", + "prompt": "What is 2+2? Answer with just the number.", + "max_tokens": 5, + }, + { + "label": "llm-routing-auto-free (medium)", + "model": "llm-routing-auto-free", + "prompt": "Explain what a Python decorator is in one sentence.", + "max_tokens": 50, + }, + ] + + for test in tests: + total += 1 + payload = { + "model": test["model"], + "messages": [{"role": "user", "content": test["prompt"]}], + "max_tokens": test["max_tokens"], + } + try: + start = time.time() + r = httpx.post( + f"{base}/v1/chat/completions", + json=payload, + headers=headers, + timeout=120, + ) + elapsed = time.time() - start + if r.status_code == 200: + data = r.json() + content, reasoning = parse_chat_response(data) + model_used = data.get("model", "?") if isinstance(data, dict) else "?" + ok = len(content) > 0 or len(reasoning) > 0 + detail = f"model={model_used}, {elapsed:.1f}s" + if content: + detail += f", '{content[:60]}'" + elif reasoning: + detail += f", reasoning='{reasoning[:60]}'" + passed += check( + test["label"], + ok, + detail, + ) + else: + passed += check( + test["label"], + False, + f"HTTP {r.status_code}: {r.text[:120]}", + ) + except Exception as e: + passed += check(test["label"], False, str(e)) + + return passed, total + + +def test_infra_health(cfg: dict) -> tuple[int, int]: + """Test infrastructure services: MinIO, ClickHouse. Returns (passed, total).""" + passed = total = 0 + + print(f"\n── Infrastructure health ──") + + # MinIO S3 health + total += 1 + try: + minio_port = cfg.get("minio_s3_port", "9002") + r = httpx.get(f"http://127.0.0.1:{minio_port}/minio/health/live", timeout=10) + passed += check("MinIO /minio/health/live", r.status_code == 200) + except Exception as e: + passed += check("MinIO /minio/health/live", False, str(e)) + + # ClickHouse HTTP ping + total += 1 + try: + ch_port = cfg.get("clickhouse_http_port", "8123") + r = httpx.get(f"http://127.0.0.1:{ch_port}/ping", timeout=10) + passed += check("ClickHouse /ping", r.status_code == 200 and r.text.strip() == "Ok.") + except Exception as e: + passed += check("ClickHouse /ping", False, str(e)) + + return passed, total + + +def test_litellm_direct_chat(cfg: dict) -> tuple[int, int]: + """Test a direct chat completion through LiteLLM (bypassing the triage router).""" + base = f"http://127.0.0.1:{cfg['litellm_port']}" + key = cfg["litellm_master_key"] + headers = { + "Authorization": f"Bearer {key}", + "Content-Type": "application/json", + } + passed = total = 0 + + print(f"\n── LiteLLM direct chat ({base}/v1/chat/completions) ──") + + total += 1 + payload = { + "model": "agent-simple-core", + "messages": [{"role": "user", "content": "Say 'ok' and nothing else."}], + "max_tokens": 5, + } + try: + start = time.time() + r = httpx.post(f"{base}/v1/chat/completions", json=payload, headers=headers, timeout=60) + elapsed = time.time() - start + if r.status_code == 200: + data = r.json() + content, reasoning = parse_chat_response(data) + ok = len(content) > 0 or len(reasoning) > 0 + detail = f"{elapsed:.1f}s" + if content: + detail += f", '{content[:40]}'" + elif reasoning: + detail += f", reasoning='{reasoning[:40]}'" + passed += check("agent-simple-core (direct)", ok, detail) + else: + passed += check("agent-simple-core (direct)", False, f"HTTP {r.status_code}: {r.text[:100]}") + except Exception as e: + passed += check("agent-simple-core (direct)", False, str(e)) + + return passed, total + + +def test_canonical_urls(cfg: dict) -> tuple[int, int, int]: + """Verify canonical HTTPS URLs are reachable (if PUBLIC_BASE_URL is set). + Returns (passed, total, skipped).""" + passed = total = skipped = 0 + public = cfg["public_base_url"] + if not public: + return 0, 0, 0 + + print(f"\n── Canonical URLs ({public}) ──") + + endpoints = [ + ("/v1/models", "router models"), + ("/dashboard", "dashboard"), + ("/metrics", "metrics"), + ("/visualizer", "visualizer"), + ("/litellm/ui/", "LiteLLM admin UI"), + ("/langfuse", "Langfuse web UI"), + ] + + for path, label in endpoints: + total += 1 + url = f"{public}{path}" + try: + r = httpx.get(url, timeout=15, follow_redirects=True, + headers={"Authorization": f"Bearer {cfg['router_api_key']}"}) + ok = r.status_code == 200 + passed += check(f"GET {url}", ok, f"HTTP {r.status_code}") + except httpx.ConnectError as e: + # DNS/unreachable — skip gracefully (host may not resolve from test machine) + skipped += 1 + print(f" ⚠ GET {url} — SKIP: DNS/unreachable") + except Exception as e: + passed += check(f"GET {url}", False, str(e)[:100]) + + # Canonical chat completion (POST through public URL) + total += 1 + url = f"{public}/v1/chat/completions" + try: + payload = { + "model": "agent-simple-core", + "messages": [{"role": "user", "content": "What is 2+2? Answer with just the number."}], + "max_tokens": 5, + } + r = httpx.post(url, json=payload, headers={"Authorization": f"Bearer {cfg['router_api_key']}"}, timeout=60, follow_redirects=True) + if r.status_code == 200: + data = r.json() + content, reasoning = parse_chat_response(data) + ok = len(content) > 0 or len(reasoning) > 0 + detail = f"HTTP {r.status_code}" + if content: + detail += f", '{content[:30]}'" + passed += check(f"POST {url}", ok, detail) + else: + passed += check(f"POST {url}", False, f"HTTP {r.status_code}: {r.text[:80]}") + except httpx.ConnectError as e: + skipped += 1 + print(f" ⚠ POST {url} — SKIP: DNS/unreachable") + except Exception as e: + passed += check(f"POST {url}", False, str(e)[:100]) + + return passed, total, skipped + + +def main(): + parser = argparse.ArgumentParser(description="Verify canonical endpoints") + parser.add_argument( + "--dev", action="store_true", help="Test dev environment (dev-router-pod)" + ) + parser.add_argument( + "--prod", action="store_true", help="Test prod environment (agent-router-pod, default)" + ) + args = parser.parse_args() + + dev = args.dev and not args.prod + env_label = "DEV" if dev else "PROD" + cfg = load_env(dev=dev) + + print(f"=== Canonical Endpoint Verification [{env_label}] ===") + print(f" Router port : {cfg['router_port']}") + print(f" LiteLLM port: {cfg['litellm_port']}") + print(f" Langfuse port: {cfg['langfuse_web_port']}") + if cfg["public_base_url"]: + print(f" Public URL : {cfg['public_base_url']}") + + total_passed = total_tests = total_skipped = 0 + + for name, fn in [ + ("Router API", test_router_endpoints), + ("LiteLLM health", test_litellm_endpoints), + ("Langfuse health", test_langfuse_endpoints), + ("Infrastructure", test_infra_health), + ("E2E chat (router)", test_e2e_chat), + ("LiteLLM direct chat", test_litellm_direct_chat), + ("Canonical URLs", test_canonical_urls), + ]: + result = fn(cfg) + if len(result) == 3: + p, t, s = result + total_skipped += s + else: + p, t = result + total_passed += p + total_tests += t + + print(f"\\n{'='*60}") + actual_tests = total_tests - total_skipped + parts = [f"{total_passed}/{actual_tests} passed"] + if total_skipped: + parts.append(f"{total_skipped} skipped") + print(f"Results [{env_label}]: {', '.join(parts)}") + if total_passed < actual_tests: + print(f"FAILED: {actual_tests - total_passed} test(s)") + sys.exit(1) + else: + print("ALL PASSED") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/start-stack.sh b/start-stack.sh index f3399914..db94b065 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -612,7 +612,7 @@ render_router_config() { } render_pod_yaml() { - export WORKDIR HOME LITELLM_MASTER_KEY POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT + export WORKDIR HOME LITELLM_MASTER_KEY UI_USERNAME UI_PASSWORD POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT python3 - "$WORKDIR/pod.yaml" <<'PY' import os, sys, urllib.parse, json uid = os.getuid() @@ -627,9 +627,12 @@ placeholders = [ "HOME_PLACEHOLDER", "RUN_USER_PLACEHOLDER", "LITELLM_MASTER_KEY_PLACEHOLDER", + "LITELLM_UI_USERNAME_PLACEHOLDER", + "LITELLM_UI_PASSWORD_PLACEHOLDER", "POSTGRES_PASSWORD_RAW_PLACEHOLDER", "POSTGRES_PASSWORD_ENCODED_PLACEHOLDER", "NEXTAUTH_SECRET_PLACEHOLDER", + "NEXTAUTH_URL_PLACEHOLDER", "SALT_PLACEHOLDER", "ENCRYPTION_KEY_PLACEHOLDER", "OLLAMA_API_KEY_PLACEHOLDER", @@ -666,6 +669,8 @@ text = text.replace("WORKDIR_PLACEHOLDER", os.environ["WORKDIR"]) text = text.replace("HOME_PLACEHOLDER", os.environ["HOME"]) text = text.replace("RUN_USER_PLACEHOLDER", f"/run/user/{uid}") text = text.replace("LITELLM_MASTER_KEY_PLACEHOLDER", yaml_scalar(os.environ["LITELLM_MASTER_KEY"])) +text = text.replace("LITELLM_UI_USERNAME_PLACEHOLDER", yaml_scalar(os.environ.get("UI_USERNAME") or "admin")) +text = text.replace("LITELLM_UI_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ.get("UI_PASSWORD") or os.environ.get("LITELLM_MASTER_KEY") or "admin")) text = text.replace("POSTGRES_PASSWORD_RAW_PLACEHOLDER", yaml_scalar(os.environ["POSTGRES_PASSWORD"])) # URL-encode the postgres password for DSN insertion encoded_password = urllib.parse.quote(os.environ['POSTGRES_PASSWORD'], safe="") @@ -686,6 +691,9 @@ public_base_url = os.environ["PUBLIC_BASE_URL"].rstrip("/") proxy_base_url = f"{public_base_url}/litellm" text = text.replace("PROXY_BASE_URL_PLACEHOLDER", yaml_scalar(proxy_base_url)) text = text.replace("PUBLIC_BASE_URL_PLACEHOLDER", yaml_scalar(os.environ["PUBLIC_BASE_URL"])) +# Derive NEXTAUTH_URL from PUBLIC_BASE_URL (Langfuse needs the public URL for OAuth redirects) +nextauth_url = f"{public_base_url}/langfuse" +text = text.replace("NEXTAUTH_URL_PLACEHOLDER", yaml_scalar(nextauth_url)) text = text.replace("ROUTING_DOMAIN_PLACEHOLDER", yaml_scalar(os.environ["ROUTING_DOMAIN"])) # Raw replacements (no quoting — used for pod name, data root, and integer port values) raw_replacements = {