diff --git a/.env.dev b/.env.dev index 9dc4b1d0..4565148a 100644 --- a/.env.dev +++ b/.env.dev @@ -6,8 +6,9 @@ POD_NAME="dev-router-pod" # Public URLs BASEURL="dev.vendeuvre.lan" -BASE_URL="dev.vendeuvre.lan" -PUBLIC_BASE_URL="https://dev.vendeuvre.lan/llm-routing" +PUBLIC_BASE_URL="https://dev.vendeuvre.lan" +PROXY_BASE_URL="https://litellm.dev.vendeuvre.lan" +NEXTAUTH_URL="https://langfuse.dev.vendeuvre.lan" # Dev port assignments (+10 offset from production) ROUTER_PORT="5010" @@ -26,6 +27,7 @@ MINIO_CONSOLE_PORT="9011" # Dev uses locally-built router image (for testing code changes) ROUTER_IMAGE="localhost/llm-routing-dev:latest" -# Shared infrastructure (same llama classifier across dev & prod) -LLAMA_CLASSIFIER_URL="https://x570.vendeuvre.lan/llm-routing/llama/v1" -LLAMA_SERVER_URL="https://x570.vendeuvre.lan/llm-routing/llama" +# Shared llama.cpp infrastructure is host-networked, so the dev safety net +# reaches its local listener directly rather than any TLS-terminated hostname. +LLAMA_CLASSIFIER_URL="http://127.0.0.1:8083/v1" +LLAMA_SERVER_URL="http://127.0.0.1:8083" diff --git a/README.md b/README.md index 061ec42b..a1324e18 100644 --- a/README.md +++ b/README.md @@ -389,15 +389,20 @@ For secure production deployments, the gateway services are configured to run un ### 1. Launching the Stack Run the startup script from the root of the repository: ```bash -./start-stack.sh # Fast restart (preserves container IDs and logs) -./start-stack.sh --replace # Recreate pod from YAML (picks up new ports, - # health probes, env vars, containers — no rebuild) -./start-stack.sh --full-rebuild # Full reset: rebuild image + recreate pod +./start-stack.sh # Restart the systemd-managed Quadlet stack +./start-stack.sh --replace # Render Quadlets, daemon-reload, and recreate the stack + # (picks up ports, probes, env vars, and containers) +./start-stack.sh --full-rebuild # Same as --replace plus rebuild the router image + +# Inspect the generated systemd units and their logs +systemctl --user status llm-routing-pod.service --no-pager +systemctl --user list-units 'llm-routing-*' --no-pager +journalctl --user -u llm-routing-router.service -n 100 --no-pager ``` *Note: If running for the first time, the script will prompt you for your `OpenRouter API Key`, securely saving it inside `.env` with restrictive permissions (`chmod 600`). The script also automatically generates and persists secure random secrets (`LITELLM_MASTER_KEY`, `POSTGRES_PASSWORD`, `NEXTAUTH_SECRET`, `SALT`, `ENCRYPTION_KEY`, and `ROUTER_API_KEY`) to this file on startup if they are missing.* ### 2. Verify Container Status -Check that all **10 containers** inside `prod-router-pod` are up and running: +Check that all **9 application containers** in `prod-router-pod` are up and running (the tenth pod infra container is Podman-managed): ```bash podman pod ps podman ps --pod --filter pod=prod-router-pod @@ -840,14 +845,15 @@ Tests cover: | Section | Endpoints | |---------|-----------| | Router API | `/v1/models`, `/metrics`, `/dashboard`, `/api/dashboard-stats`, `/visualizer` | -| LiteLLM | `/health/liveness`, `/health/readiness`, `/v1/models`, `/llm-routing/litellm/ui/` | -| Langfuse | `/api/public/health`, `/` (web UI) | +| LiteLLM | Local `/health/liveness`, `/health/readiness`, `/v1/models`; canonical `https://litellm./ui/` | +| Langfuse | Local `/api/public/health`, `/`; canonical `https://langfuse./` | +| llama.cpp | Canonical `https://llama./health` | | Infrastructure | MinIO `/minio/health/live`, ClickHouse `/ping` | | E2E chat | 3 completions through triage router | | LiteLLM direct | 1 completion directly to LiteLLM | -| Canonical URLs | 6 GET + 1 POST through public HTTPS (graceful DNS skip) | +| Canonical URLs | 7 GET + 1 POST through public HTTPS (graceful DNS skip) | -Requires `PUBLIC_BASE_URL` in `.env` for canonical URL tests. Dev `.env.dev` already has it; prod `.env` should include `PUBLIC_BASE_URL="https://x570.vendeuvre.lan/llm-routing"`. +Requires `PUBLIC_BASE_URL` in `.env` for canonical URL tests. The router remains under its configured path (for example `https://x570.vendeuvre.lan/llm-routing`), while the verifier derives service URLs from its host: `https://litellm./ui/`, `https://langfuse./`, and `https://llama./health`. Dev `.env.dev` already has it; prod `.env` should include `PUBLIC_BASE_URL="https://x570.vendeuvre.lan/llm-routing"`. The dev local-model safety net uses the host-networked local listener: `LLAMA_CLASSIFIER_URL=http://127.0.0.1:8083/v1` and `LLAMA_SERVER_URL=http://127.0.0.1:8083`; it must not depend on TLS-terminated dev or production hostnames. ## 10. Performance Benchmarks diff --git a/quadlets/llm-routing-clickhouse.container b/quadlets/llm-routing-clickhouse.container new file mode 100644 index 00000000..8696921f --- /dev/null +++ b/quadlets/llm-routing-clickhouse.container @@ -0,0 +1,27 @@ +[Unit] +Description=ClickHouse — Langfuse v3 OLAP trace store (POD_NAME_PLACEHOLDER) + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-clickhouse-db +Image=clickhouse/clickhouse-server:26.6.2.81 +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +User=101 +Group=101 +Environment=CLICKHOUSE_DB=default +Environment=CLICKHOUSE_USER=clickhouse +Environment=CLICKHOUSE_PASSWORD=CLICKHOUSE_PASSWORD_PLACEHOLDER +Volume=DATA_ROOT_PLACEHOLDER/clickhouse-data:/var/lib/clickhouse +Volume=DATA_ROOT_PLACEHOLDER/clickhouse-config/port-override.xml:/etc/clickhouse-server/config.d/port-override.xml +HealthCmd=clickhouse-client --host 127.0.0.1 --port CLICKHOUSE_TCP_PORT_PLACEHOLDER --user clickhouse --password CLICKHOUSE_PASSWORD_PLACEHOLDER --query "SELECT 1" +HealthInterval=15s +HealthTimeout=5s +HealthOnFailure=kill +HealthRetries=5 +HealthStartPeriod=20s + +[Service] +Restart=always +TimeoutStartSec=120 diff --git a/quadlets/llm-routing-langfuse-web.container b/quadlets/llm-routing-langfuse-web.container new file mode 100644 index 00000000..d6f4fb86 --- /dev/null +++ b/quadlets/llm-routing-langfuse-web.container @@ -0,0 +1,51 @@ +[Unit] +Description=Langfuse Web — observability dashboard (POD_NAME_PLACEHOLDER) +After=llm-routing-postgres.service llm-routing-clickhouse.service llm-routing-valkey-lf.service llm-routing-minio.service +Wants=llm-routing-postgres.service llm-routing-clickhouse.service llm-routing-valkey-lf.service llm-routing-minio.service + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-langfuse-web +Image=langfuse/langfuse:3.222.0 +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +Environment=DATABASE_URL=postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:POSTGRES_PORT_PLACEHOLDER/langfuse +Environment=NEXTAUTH_SECRET=NEXTAUTH_SECRET_PLACEHOLDER +Environment=NEXTAUTH_URL=NEXTAUTH_URL_PLACEHOLDER +Environment=SALT=SALT_PLACEHOLDER +Environment=ENCRYPTION_KEY=ENCRYPTION_KEY_PLACEHOLDER +Environment=HOSTNAME=0.0.0.0 +Environment=PORT=LANGFUSE_WEB_PORT_PLACEHOLDER +Environment=TELEMETRY_ENABLED=false +Environment=LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse-events +Environment=LANGFUSE_S3_EVENT_UPLOAD_REGION=us-east-1 +Environment=LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=MINIO_USER_PLACEHOLDER +Environment=LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=MINIO_PASSWORD_PLACEHOLDER +Environment=LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT=http://127.0.0.1:MINIO_S3_PORT_PLACEHOLDER +Environment=S3_FORCE_PATH_STYLE=true +Environment=CLICKHOUSE_URL=http://127.0.0.1:CLICKHOUSE_HTTP_PORT_PLACEHOLDER +Environment=CLICKHOUSE_MIGRATION_URL=clickhouse://127.0.0.1:CLICKHOUSE_TCP_PORT_PLACEHOLDER +Environment=CLICKHOUSE_USER=clickhouse +Environment=CLICKHOUSE_PASSWORD=CLICKHOUSE_PASSWORD_PLACEHOLDER +Environment=CLICKHOUSE_CLUSTER_ENABLED=false +Environment=REDIS_HOST=127.0.0.1 +Environment=REDIS_PORT=VALKEY_LF_PORT_PLACEHOLDER +Environment=REDIS_AUTH=REDIS_AUTH_PLACEHOLDER +Environment=LANGFUSE_INIT_ORG_ID=org-local-dev-id +Environment=LANGFUSE_INIT_ORG_NAME=Local Development +Environment=LANGFUSE_INIT_PROJECT_ID=proj-triage-gateway-id +Environment=LANGFUSE_INIT_PROJECT_NAME=Triage Gateway +Environment=LANGFUSE_INIT_USER_EMAIL=admin@local.dev +Environment=LANGFUSE_INIT_USER_PASSWORD=LANGFUSE_INIT_USER_PASSWORD_PLACEHOLDER +Environment=LANGFUSE_LOG_LEVEL=warn +HealthCmd=wget -q -O /dev/null http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER/api/public/health +HealthInterval=15s +HealthTimeout=5s +HealthOnFailure=kill +HealthRetries=5 +HealthStartPeriod=240s + +[Service] +Restart=always +TimeoutStartSec=300 diff --git a/quadlets/llm-routing-langfuse-worker.container b/quadlets/llm-routing-langfuse-worker.container new file mode 100644 index 00000000..b4e63724 --- /dev/null +++ b/quadlets/llm-routing-langfuse-worker.container @@ -0,0 +1,40 @@ +[Unit] +Description=Langfuse Worker — background job processor (POD_NAME_PLACEHOLDER) +After=llm-routing-postgres.service llm-routing-clickhouse.service llm-routing-valkey-lf.service llm-routing-minio.service +Wants=llm-routing-postgres.service llm-routing-clickhouse.service llm-routing-valkey-lf.service llm-routing-minio.service + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-langfuse-worker +Image=langfuse/langfuse-worker:3.222.0 +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +Environment=DATABASE_URL=postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:POSTGRES_PORT_PLACEHOLDER/langfuse +Environment=CLICKHOUSE_URL=http://127.0.0.1:CLICKHOUSE_HTTP_PORT_PLACEHOLDER +Environment=CLICKHOUSE_USER=clickhouse +Environment=CLICKHOUSE_PASSWORD=CLICKHOUSE_PASSWORD_PLACEHOLDER +Environment=CLICKHOUSE_CLUSTER_ENABLED=false +Environment=REDIS_HOST=127.0.0.1 +Environment=REDIS_PORT=VALKEY_LF_PORT_PLACEHOLDER +Environment=REDIS_AUTH=REDIS_AUTH_PLACEHOLDER +Environment=HOSTNAME=0.0.0.0 +Environment=PORT=LANGFUSE_WORKER_PORT_PLACEHOLDER +Environment=TELEMETRY_ENABLED=false +Environment=LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse-events +Environment=LANGFUSE_S3_EVENT_UPLOAD_REGION=us-east-1 +Environment=LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=MINIO_USER_PLACEHOLDER +Environment=LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=MINIO_PASSWORD_PLACEHOLDER +Environment=LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT=http://127.0.0.1:MINIO_S3_PORT_PLACEHOLDER +Environment=S3_FORCE_PATH_STYLE=true +Environment=LANGFUSE_LOG_LEVEL=warn +HealthCmd=pgrep node +HealthInterval=15s +HealthTimeout=5s +HealthOnFailure=kill +HealthRetries=5 +HealthStartPeriod=240s + +[Service] +Restart=always +TimeoutStartSec=300 diff --git a/quadlets/llm-routing-litellm.container b/quadlets/llm-routing-litellm.container new file mode 100644 index 00000000..aff6e059 --- /dev/null +++ b/quadlets/llm-routing-litellm.container @@ -0,0 +1,44 @@ +[Unit] +Description=LiteLLM Gateway — upstream proxy with adaptive routing (POD_NAME_PLACEHOLDER) +After=llm-routing-postgres.service llm-routing-valkey-cache.service +Wants=llm-routing-postgres.service llm-routing-valkey-cache.service + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-litellm-gateway +Image=ghcr.io/berriai/litellm:v1.93.0 +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +# Image entrypoint is docker/prod_entrypoint.sh (litellm CLI); pod.yaml used +# command: python3 /app/entrypoint.py which replaced it. Quadlet Exec= only sets +# args, so override Entrypoint= with the venv python (matches play-kube behavior). +Entrypoint=/app/.venv/bin/python3 +Exec=/app/entrypoint.py +Environment=DATABASE_URL=postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:POSTGRES_PORT_PLACEHOLDER/postgres +Environment=STORE_MODEL_IN_DB=True +Environment=LITELLM_LOG=INFO +Environment=LANGFUSE_HOST=http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER +Environment=LITELLM_MASTER_KEY=LITELLM_MASTER_KEY_PLACEHOLDER +Environment=LITELLM_UI_USERNAME=LITELLM_UI_USERNAME_PLACEHOLDER +Environment=LITELLM_UI_PASSWORD=LITELLM_UI_PASSWORD_PLACEHOLDER +Environment=OLLAMA_API_KEY=OLLAMA_API_KEY_PLACEHOLDER +Environment=OPENROUTER_API_KEY=OPENROUTER_API_KEY_PLACEHOLDER +Environment=SERVER_ROOT_PATH=/llm-routing/litellm +Environment=PROXY_BASE_URL=PROXY_BASE_URL_PLACEHOLDER +Environment=LITELLM_PORT=LITELLM_PORT_PLACEHOLDER +Environment=POSTGRES_PORT=POSTGRES_PORT_PLACEHOLDER +Volume=DATA_ROOT_PLACEHOLDER/litellm-rendered/config.yaml:/app/config.yaml +Volume=DATA_ROOT_PLACEHOLDER/litellm-rendered/entrypoint.py:/app/entrypoint.py +Volume=WORKDIR_PLACEHOLDER/.env:/config/.env +Volume=HOME_PLACEHOLDER/.gemini:/config/gemini_auth +HealthCmd=python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:LITELLM_PORT_PLACEHOLDER/health/liveness')" +HealthInterval=15s +HealthTimeout=5s +HealthOnFailure=kill +HealthRetries=5 +HealthStartPeriod=240s + +[Service] +Restart=always +TimeoutStartSec=300 diff --git a/quadlets/llm-routing-minio.container b/quadlets/llm-routing-minio.container new file mode 100644 index 00000000..4df6c6cb --- /dev/null +++ b/quadlets/llm-routing-minio.container @@ -0,0 +1,27 @@ +[Unit] +Description=MinIO S3 — Langfuse v3 event storage (POD_NAME_PLACEHOLDER) + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-minio-s3 +Image=minio/minio:RELEASE.2025-09-07T16-13-09Z +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +Environment=MINIO_ROOT_USER=MINIO_USER_PLACEHOLDER +Environment=MINIO_ROOT_PASSWORD=MINIO_PASSWORD_PLACEHOLDER +# pod.yaml used args: server ... with no command:, so image entrypoint +# /usr/bin/docker-entrypoint.sh ran and consumed them. Quadlet Exec= sets args +# the same way — image entrypoint stays, Exec provides the server args. +Exec=server --quiet /data --address :MINIO_S3_PORT_PLACEHOLDER --console-address :MINIO_CONSOLE_PORT_PLACEHOLDER +Volume=DATA_ROOT_PLACEHOLDER/minio-data:/data +HealthCmd=curl -f http://127.0.0.1:MINIO_S3_PORT_PLACEHOLDER/minio/health/live +HealthInterval=15s +HealthTimeout=5s +HealthOnFailure=kill +HealthRetries=5 +HealthStartPeriod=10s + +[Service] +Restart=always +TimeoutStartSec=90 diff --git a/quadlets/llm-routing-postgres.container b/quadlets/llm-routing-postgres.container new file mode 100644 index 00000000..de7db982 --- /dev/null +++ b/quadlets/llm-routing-postgres.container @@ -0,0 +1,26 @@ +[Unit] +Description=PostgreSQL — Langfuse + LiteLLM metadata store (POD_NAME_PLACEHOLDER) + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-postgres-db +Image=pgvector/pgvector:0.8.5-pg18 +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +Environment=POSTGRES_USER=postgres +Environment=POSTGRES_PASSWORD=POSTGRES_PASSWORD_RAW_PLACEHOLDER +Environment=POSTGRES_DB=langfuse +Environment=PGDATA=/var/lib/postgresql/data/pg18 +Environment=PGPORT=POSTGRES_PORT_PLACEHOLDER +Volume=DATA_ROOT_PLACEHOLDER/postgres-data:/var/lib/postgresql +HealthCmd=pg_isready -U postgres -p POSTGRES_PORT_PLACEHOLDER +HealthInterval=10s +HealthTimeout=5s +HealthOnFailure=kill +HealthRetries=5 +HealthStartPeriod=15s + +[Service] +Restart=always +TimeoutStartSec=120 diff --git a/quadlets/llm-routing-router.container b/quadlets/llm-routing-router.container new file mode 100644 index 00000000..72023f9d --- /dev/null +++ b/quadlets/llm-routing-router.container @@ -0,0 +1,57 @@ +[Unit] +Description=LLM Triage Router — classification + routing gateway (POD_NAME_PLACEHOLDER) +After=llm-routing-litellm.service +Wants=llm-routing-litellm.service + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-llm-triage-router +Image=ROUTER_IMAGE_PLACEHOLDER +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +# Router image has no entrypoint (EP=null, CMD=uvicorn...); pod.yaml used +# command: /bin/sh + args: -c ... — Entrypoint=/bin/sh, Exec=-c "..." matches. +Entrypoint=/bin/sh +Exec=-c "set -a && . /config/.env && set +a && LLAMA_CLASSIFIER_URL=LLAMA_CLASSIFIER_URL_PLACEHOLDER LLAMA_SERVER_URL=LLAMA_SERVER_URL_PLACEHOLDER LITELLM_PORT=LITELLM_PORT_PLACEHOLDER VALKEY_CACHE_PORT=VALKEY_CACHE_PORT_PLACEHOLDER exec uvicorn main:app --host 0.0.0.0 --port ROUTER_PORT_PLACEHOLDER --log-level $${LOG_LEVEL:-warning}" +Environment=CONFIG_PATH=/config/router_dir/config.yaml +Environment=LITELLM_CONFIG_PATH=/config/litellm_dir/config.yaml +Environment=DATABASE_URL=postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:POSTGRES_PORT_PLACEHOLDER/postgres +Environment=DBUS_SESSION_BUS_ADDRESS=unix:path=RUN_USER_PLACEHOLDER/bus +Environment=LITELLM_MASTER_KEY=LITELLM_MASTER_KEY_PLACEHOLDER +Environment=LANGFUSE_PUBLIC_KEY=LANGFUSE_PUBLIC_KEY_PLACEHOLDER +Environment=LANGFUSE_SECRET_KEY=LANGFUSE_SECRET_KEY_PLACEHOLDER +Environment=LANGFUSE_HOST=http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER +Environment=OLLAMA_API_KEY=OLLAMA_API_KEY_PLACEHOLDER +Environment=PUBLIC_BASE_URL=PUBLIC_BASE_URL_PLACEHOLDER +Environment=ROUTING_DOMAIN=ROUTING_DOMAIN_PLACEHOLDER +# Reassert these after /config/.env is sourced by Exec so a dev overlay cannot +# be overwritten by the production bind-mounted file. +Environment=LLAMA_CLASSIFIER_URL=LLAMA_CLASSIFIER_URL_PLACEHOLDER +Environment=LLAMA_SERVER_URL=LLAMA_SERVER_URL_PLACEHOLDER +Environment=LITELLM_PORT=LITELLM_PORT_PLACEHOLDER +Environment=VALKEY_CACHE_PORT=VALKEY_CACHE_PORT_PLACEHOLDER +Environment=ROUTER_PORT=ROUTER_PORT_PLACEHOLDER +Environment=LANGFUSE_WEB_PORT=LANGFUSE_WEB_PORT_PLACEHOLDER +Environment=LOG_LEVEL=info +Volume=DATA_ROOT_PLACEHOLDER/router-rendered:/config/router_dir +Volume=DATA_ROOT_PLACEHOLDER/litellm-rendered:/config/litellm_dir +Volume=DATA_ROOT_PLACEHOLDER/datasets:/app/data +Volume=HOME_PLACEHOLDER/.gemini:/config/gemini_auth +Volume=HOME_PLACEHOLDER/.gemini:/root/.gemini +Volume=HOME_PLACEHOLDER/.local/share/goose:/config/goose_sessions +Volume=HOME_PLACEHOLDER/.local/bin/agy:/usr/local/bin/agy +Volume=RUN_USER_PLACEHOLDER/bus:RUN_USER_PLACEHOLDER/bus +Volume=HOME_PLACEHOLDER/.local/share/keyrings:/root/.local/share/keyrings +Volume=RUN_USER_PLACEHOLDER:RUN_USER_PLACEHOLDER +Volume=WORKDIR_PLACEHOLDER/.env:/config/.env +HealthCmd=python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:ROUTER_PORT_PLACEHOLDER/metrics')" +HealthInterval=15s +HealthTimeout=5s +HealthOnFailure=kill +HealthRetries=5 +HealthStartPeriod=240s + +[Service] +Restart=always +TimeoutStartSec=300 diff --git a/quadlets/llm-routing-valkey-cache.container b/quadlets/llm-routing-valkey-cache.container new file mode 100644 index 00000000..8172afb7 --- /dev/null +++ b/quadlets/llm-routing-valkey-cache.container @@ -0,0 +1,25 @@ +[Unit] +Description=Valkey Cache — LiteLLM response cache (POD_NAME_PLACEHOLDER) + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-valkey-cache +Image=valkey/valkey:9.1.1 +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +# pod.yaml used command: (replaces entrypoint). Image entrypoint docker-entrypoint.sh +# would also work, but explicit Entrypoint= matches play-kube semantics exactly. +Entrypoint=valkey-server +Exec=--bind 127.0.0.1 --port VALKEY_CACHE_PORT_PLACEHOLDER --loglevel warning +Volume=DATA_ROOT_PLACEHOLDER/valkey-data:/data +HealthCmd=valkey-cli -p VALKEY_CACHE_PORT_PLACEHOLDER ping +HealthInterval=10s +HealthTimeout=2s +HealthOnFailure=kill +HealthRetries=5 +HealthStartPeriod=5s + +[Service] +Restart=always +TimeoutStartSec=90 diff --git a/quadlets/llm-routing-valkey-lf.container b/quadlets/llm-routing-valkey-lf.container new file mode 100644 index 00000000..afbfe1ee --- /dev/null +++ b/quadlets/llm-routing-valkey-lf.container @@ -0,0 +1,23 @@ +[Unit] +Description=Valkey LF — Langfuse v3 BullMQ job queue (POD_NAME_PLACEHOLDER) + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-valkey-lf +Image=valkey/valkey:9.1.1 +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +Entrypoint=redis-server +Exec=--bind 127.0.0.1 --port VALKEY_LF_PORT_PLACEHOLDER --requirepass REDIS_AUTH_PLACEHOLDER --maxmemory-policy noeviction --loglevel warning +Volume=DATA_ROOT_PLACEHOLDER/redis-lf-data:/data +HealthCmd=valkey-cli -p VALKEY_LF_PORT_PLACEHOLDER -a REDIS_AUTH_PLACEHOLDER ping +HealthInterval=10s +HealthTimeout=2s +HealthOnFailure=kill +HealthRetries=5 +HealthStartPeriod=5s + +[Service] +Restart=always +TimeoutStartSec=90 diff --git a/quadlets/llm-routing.pod b/quadlets/llm-routing.pod new file mode 100644 index 00000000..3f5e0c4a --- /dev/null +++ b/quadlets/llm-routing.pod @@ -0,0 +1,13 @@ +# LLM Triage Gateway pod — Quadlet template (rendered by start-stack.sh) +# Pod name is rendered from POD_NAME (prod-router-pod / dev-router-pod). +[Unit] +Description=LLM Triage Gateway Pod (POD_NAME_PLACEHOLDER) +Wants=network-online.target +After=network-online.target + +[Pod] +PodName=POD_NAME_PLACEHOLDER +Network=host + +[Install] +WantedBy=default.target diff --git a/router/main.py b/router/main.py index 567f6f09..d0774e15 100644 --- a/router/main.py +++ b/router/main.py @@ -3514,7 +3514,7 @@ def resolve_external_urls(request: Request) -> tuple[str, str, str]: # Basic sanity-check on external_host, but don't over-restrict valid hostnames; # fall back to the request base URL rather than silently forcing localhost. - if not isinstance(external_host, str) or not re.match(r"^[a-zA-Z0-9.-:]+$", external_host): + if not isinstance(external_host, str) or not re.match(r"^[a-zA-Z0-9.:-]+$", external_host): logger.warning( "Unexpected external_host %r, falling back to request.base_url.hostname (%r)", external_host, @@ -3546,21 +3546,26 @@ def resolve_external_urls(request: Request) -> tuple[str, str, str]: is_valid_external = external_host == domain or external_host.endswith("." + domain) is_valid_base = request.base_url.hostname == domain or (request.base_url.hostname or "").endswith("." + domain) - if is_valid_external: - # Centralized base URL path under subdomain/reverse proxy - return ( - f"{external_scheme}://{external_netloc}/llm-routing/langfuse", - f"{external_scheme}://{external_netloc}/llm-routing/litellm/ui", - f"{external_scheme}://{external_netloc}/llm-routing/llama/" - ) - elif is_valid_base: - parsed_netloc = urlparse(f"{external_scheme}://{request.url.netloc}") - netloc = request.url.netloc if parsed_netloc.hostname else "localhost" - base = f"{external_scheme}://{netloc}" + if is_valid_external or is_valid_base: + # Use configured routing domain if a proxy supplies no request hostname. + # Preserve an explicit public port from the selected netloc so dashboard + # links remain valid behind non-standard TLS listeners. + host_val = external_host if is_valid_external else (request.base_url.hostname or domain) + netloc_val = external_netloc if is_valid_external else (request.base_url.netloc or host_val) + parsed_public = urlparse(f"{external_scheme}://{netloc_val}") + try: + port_suffix = f":{parsed_public.port}" if parsed_public.port else "" + except ValueError: + logger.warning("Invalid public port in netloc %r; omitting port", netloc_val) + port_suffix = "" + + host_base = re.sub(r"^dashboard\.", "", host_val) + host_base = re.sub(r"^(?:litellm|langfuse|llama)\.", "", host_base) + service_netloc = f"{host_base}{port_suffix}" return ( - f"{base}/llm-routing/langfuse", - f"{base}/llm-routing/litellm/ui", - f"{base}/llm-routing/llama/" + f"{external_scheme}://langfuse.{service_netloc}", + f"{external_scheme}://litellm.{service_netloc}/ui/", + f"{external_scheme}://llama.{service_netloc}/" ) else: # Local development fallback: derive schemes, ports, and paths dynamically from configuration constants diff --git a/router/tests/test_resolve_external_urls.py b/router/tests/test_resolve_external_urls.py index 8cc79f0d..8db6d5fb 100644 --- a/router/tests/test_resolve_external_urls.py +++ b/router/tests/test_resolve_external_urls.py @@ -45,9 +45,9 @@ def test_resolve_with_public_base_url_vaild_domain(): req = MockRequest() lf, ll, lm = resolve_external_urls(req) - assert lf == "https://app.vendeuvre.lan/llm-routing/langfuse" - assert ll == "https://app.vendeuvre.lan/llm-routing/litellm/ui" - assert lm == "https://app.vendeuvre.lan/llm-routing/llama/" + assert lf == "https://langfuse.app.vendeuvre.lan" + assert ll == "https://litellm.app.vendeuvre.lan/ui/" + assert lm == "https://llama.app.vendeuvre.lan/" def test_resolve_with_valid_base_request(): @@ -64,11 +64,29 @@ def test_resolve_with_valid_base_request(): lf, ll, lm = resolve_external_urls(req) - # Netloc should use [::1]:8000 because request base host is valid, - # and netloc was validated successfully via urlparse. - assert lf == "http://[::1]:8000/llm-routing/langfuse" - assert ll == "http://[::1]:8000/llm-routing/litellm/ui" - assert lm == "http://[::1]:8000/llm-routing/llama/" + # Valid request host is converted to service subdomains while preserving + # its explicit port, rather than leaking the configured IPv6 fallback. + assert lf == "http://langfuse.sub.vendeuvre.lan:8000" + assert ll == "http://litellm.sub.vendeuvre.lan:8000/ui/" + assert lm == "http://llama.sub.vendeuvre.lan:8000/" + + +def test_resolve_with_valid_base_request_without_port(): + os.environ["ROUTING_DOMAIN"] = "vendeuvre.lan" + os.environ["PUBLIC_BASE_URL"] = "http://[::1]:5000" + + req = MockRequest( + base_host="sub.vendeuvre.lan", + base_netloc="sub.vendeuvre.lan", + url_scheme="https", + url_netloc="[::1]:8000", + ) + + lf, ll, lm = resolve_external_urls(req) + + assert lf == "http://langfuse.sub.vendeuvre.lan" + assert ll == "http://litellm.sub.vendeuvre.lan/ui/" + assert lm == "http://llama.sub.vendeuvre.lan/" def test_local_fallback_ipv6(): diff --git a/scripts/README.md b/scripts/README.md index 089f036d..cbb735cf 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -7,11 +7,12 @@ This directory and the repository root contain various scripts used for stack or ## 1. Stack Orchestration & Backups ### `start-stack.sh` (Root Directory) -Unified startup and credential extraction script for the Podman Kubernetes container stack. +Unified startup and credential extraction script for the systemd Quadlet-managed Podman stack. - **Usage**: - - `./start-stack.sh` (Restart existing pod — fast, preserves logs) - - `./start-stack.sh --replace` (Stop + clean ports + redeploy pod from `pod.yaml`) + - `./start-stack.sh` (Restart the generated `llm-routing-pod.service`) + - `./start-stack.sh --replace` (Stop + clean ports + render/install Quadlets + daemon-reload + recreate stack) - `./start-stack.sh --full-rebuild` (Same as `--replace` + rebuild the triage router image; required for code changes in `router/`) +- Quadlet templates live in `quadlets/`; rendered owner-only units are installed under `~/.config/containers/systemd/llm-routing/`. Use `systemctl --user status llm-routing-pod.service --no-pager` and `journalctl --user -u llm-routing-router.service --no-pager` for lifecycle diagnostics. ### `scripts/backup.sh` Automated database backup script that runs before every stack deployment. Uses `pg_isready` to safely wait for database connections and manages timestamped backups under `backups/`. diff --git a/scripts/upgrade-prod.sh b/scripts/upgrade-prod.sh index fb48eb48..c1b42e78 100755 --- a/scripts/upgrade-prod.sh +++ b/scripts/upgrade-prod.sh @@ -4,7 +4,7 @@ # Flow: # 1. Fetch the latest release tag from GitHub # 2. Check out a clean copy of that tag to a temp dir -# 3. Rsync runtime files (pod.yaml, start-stack.sh, litellm/, router/, scripts/) +# 3. Rsync runtime files (pod.yaml, start-stack.sh, quadlets/, litellm/, router/, scripts/) # into ~/prod/LLM-Routing — data/, backups/, and .env are NEVER touched # 4. Redeploy with start-stack.sh --pull (pulls latest container images) # @@ -67,7 +67,7 @@ echo "📥 Cloning $REPO @ $TAG..." git clone -q --depth 1 --branch "$TAG" "https://github.com/$REPO.git" "$TEMP_DIR" # ── verify the tag has the files we need ── -for f in pod.yaml start-stack.sh litellm/ router/ scripts/; do +for f in pod.yaml start-stack.sh quadlets/ litellm/ router/ scripts/; do if [ ! -e "$TEMP_DIR/$f" ]; then echo "❌ Release $TAG is missing expected file/dir: $f" exit 1 @@ -80,7 +80,7 @@ if $DRY_RUN; then echo "── Dry run: files that would change ──" diff -rq "$TEMP_DIR/pod.yaml" "$PROD_DIR/pod.yaml" 2>/dev/null || echo " pod.yaml differs" diff -rq "$TEMP_DIR/start-stack.sh" "$PROD_DIR/start-stack.sh" 2>/dev/null || echo " start-stack.sh differs" - for dir in litellm router scripts; do + for dir in quadlets litellm router scripts; do diff -rq "$TEMP_DIR/$dir" "$PROD_DIR/$dir" 2>/dev/null || echo " $dir/ differs" done echo "── End dry run ──" @@ -90,7 +90,7 @@ fi # ── confirm ── echo "" echo "⚠️ This will OVERWRITE the following in $PROD_DIR:" -echo " pod.yaml start-stack.sh litellm/ router/ scripts/" +echo " pod.yaml start-stack.sh quadlets/ litellm/ router/ scripts/" echo " .env and data/ are NEVER touched." echo "" # Require interactive confirmation in TTY mode; auto-proceed in non-interactive @@ -141,6 +141,7 @@ fi # handles graceful shutdown (including pre-deploy backup) before redeploy. echo "📋 Syncing runtime files..." # Sync directories with --delete (clean stale files within each dir) +rsync -a --delete "$TEMP_DIR/quadlets/" "$PROD_DIR/quadlets/" rsync -a --delete "$TEMP_DIR/litellm/" "$PROD_DIR/litellm/" rsync -a --delete "$TEMP_DIR/router/" "$PROD_DIR/router/" rsync -a --delete "$TEMP_DIR/scripts/" "$PROD_DIR/scripts/" diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index ff2c2497..458f0ae3 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -14,6 +14,7 @@ import time import datetime import argparse +import urllib.parse import httpx from pathlib import Path @@ -597,23 +598,32 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int, int]: print(f"\n── Canonical URLs ({public}) ──") + parsed_public = urllib.parse.urlparse(public if "://" in public else f"https://{public}") + scheme = parsed_public.scheme if parsed_public.scheme in ("http", "https") else "https" + host = parsed_public.netloc or parsed_public.path.split("/", 1)[0] + if not host: + print(" ⚠ Canonical URLs — SKIPPED (PUBLIC_BASE_URL has no host)") + return 0, 0, 0 + # Preserve a configured router path (e.g. /llm-routing) while normalizing + # scheme-less PUBLIC_BASE_URL values into valid absolute URLs. + router_base = f"{scheme}://{host}{parsed_public.path.rstrip('/')}" endpoints = [ - ("/v1/models", "router models"), - ("/dashboard", "dashboard"), - ("/metrics", "metrics"), - ("/visualizer", "visualizer"), - ("/litellm/ui/", "LiteLLM admin UI"), - ("/langfuse", "Langfuse web UI"), + (f"{router_base}/v1/models", "router models"), + (f"{router_base}/dashboard", "dashboard"), + (f"{router_base}/metrics", "metrics"), + (f"{router_base}/visualizer", "visualizer"), + (f"{scheme}://litellm.{host}/ui/", "LiteLLM admin UI"), + (f"{scheme}://langfuse.{host}/", "Langfuse web UI"), + (f"{scheme}://llama.{host}/health", "llama.cpp health"), ] - for path, label in endpoints: + for url, label in endpoints: total += 1 - url = f"{public}{path}" try: r = httpx.get(url, timeout=15, follow_redirects=True, headers={"Authorization": f"Bearer {cfg['router_api_key']}"}) ok = r.status_code == 200 - passed += check(f"GET {url}", ok, f"HTTP {r.status_code}") + passed += check(f"GET {url} ({label})", ok, f"HTTP {r.status_code}") except httpx.RequestError as e: # DNS/unreachable/timeout — skip gracefully (host may not resolve from test machine) skipped += 1 @@ -623,7 +633,7 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int, int]: # Canonical chat completion (POST through public URL) total += 1 - url = f"{public}/v1/chat/completions" + url = f"{router_base}/v1/chat/completions" try: payload = { "model": "agent-simple-core", diff --git a/start-stack.sh b/start-stack.sh index 6087a442..e2b1936e 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -530,11 +530,53 @@ verify_stack_health() { echo " ⚠️ Triage router pipeline not responding after 120s — dashboard may still work" return 1 } -# ── Safe pod teardown ── +# ── Stack ownership and teardown ── +# Keep the generated Quadlet unit name in one place: it is used for ownership +# detection, lifecycle operations, and user-facing diagnostics. +LLM_ROUTING_POD_UNIT="llm-routing-pod.service" +# Quadlet-managed pods carry PODMAN_SYSTEMD_UNIT on their infra container. +# Consult that metadata rather than inferring ownership from active state: a +# stopped or failed generated unit is still Quadlet-owned and must be reconciled +# through systemd before a replacement pod is created. +stack_ownership() { + local infra_unit + if podman pod exists "${POD_NAME}" 2>/dev/null; then + infra_unit=$(podman pod inspect "${POD_NAME}" --format '{{.InfraContainerID}}' 2>/dev/null | xargs -r podman inspect --format '{{ index .Config.Labels "PODMAN_SYSTEMD_UNIT" }}' 2>/dev/null || true) + if [[ "$infra_unit" == "$LLM_ROUTING_POD_UNIT" ]]; then + printf 'quadlet\n' + else + printf 'legacy\n' + fi + elif systemctl --user show "$LLM_ROUTING_POD_UNIT" -p LoadState --value 2>/dev/null | grep -qxv 'not-found'; then + printf 'quadlet\n' + else + printf 'absent\n' + fi +} + +require_user_systemd() { + if ! systemctl --user show-environment >/dev/null 2>&1; then + echo "❌ Error: the Quadlet deployment requires a reachable systemd --user manager." >&2 + echo " Log in through a user session or restore the user D-Bus before deploying." >&2 + return 1 + fi +} + # Graceful stop (SIGTERM with 30s timeout) lets ClickHouse/Postgres flush, # then force-remove if needed. Avoids data corruption from SIGKILL. safe_pod_teardown() { - if podman pod exists ${POD_NAME} 2>/dev/null; then + local ownership + ownership=$(stack_ownership) + if [[ "$ownership" == "quadlet" ]]; then + echo "🛑 Reconciling Quadlet-owned stack (unit may be active, inactive, or failed)..." + systemctl --user stop "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true + systemctl --user reset-failed "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true + podman pod rm -f "${POD_NAME}" 2>/dev/null || true + cleanup_zombie_ports + echo "✓ Quadlet stack stopped, state reconciled, ports cleaned" + return + fi + if [[ "$ownership" == "legacy" ]]; then echo "🛑 Gracefully stopping pod (SIGTERM, 30s timeout)..." podman pod stop -t 30 ${POD_NAME} 2>/dev/null || true # podman pod exists returns 0 for stopped pods too — check running state @@ -550,6 +592,25 @@ safe_pod_teardown() { fi } +# Derive service URLs once so legacy pod rendering and Quadlet rendering cannot drift. +derive_external_service_urls() { + local values + values=$(python3 -c ' +import os +from urllib.parse import urlparse +public = (os.environ.get("PUBLIC_BASE_URL") or "").rstrip("/") +routing_domain = os.environ.get("ROUTING_DOMAIN") or "vendeuvre.lan" +parsed = urlparse(public if "://" in public else f"https://{public}") +scheme = parsed.scheme if parsed.scheme in {"http", "https"} else "https" +host = parsed.netloc or parsed.path.split("/", 1)[0] or routing_domain +print(os.environ.get("PROXY_BASE_URL") or f"{scheme}://litellm.{host}") +print(os.environ.get("NEXTAUTH_URL") or f"{scheme}://langfuse.{host}") +') || return 1 + PROXY_BASE_URL_DERIVED=${values%%$'\n'*} + NEXTAUTH_URL_DERIVED=${values#*$'\n'} + export PROXY_BASE_URL_DERIVED NEXTAUTH_URL_DERIVED +} + # Pre-deploy database backup (runs before any pod modification) # Skip if pod doesn't exist (e.g., after manual cleanup) if podman pod exists ${POD_NAME} 2>/dev/null; then @@ -612,105 +673,74 @@ render_router_config() { echo "✓ Router config rendered to ${rendered_dir}/config.yaml" } -render_pod_yaml() { +# Legacy pod.yaml rendering was removed: all deployment paths render Quadlets. + +# ── Quadlet rendering + installation ── +# Renders quadlets/*.pod + quadlets/*.container templates (same _PLACEHOLDER +# convention as pod.yaml) into ~/.config/containers/systemd/llm-routing/ and +# lets systemd's podman-user-generator turn them into real units. +# Quadlet values are bare scalars (not YAML) so plain string replacement is used. +QUADLET_DIR="${HOME}/.config/containers/systemd/llm-routing" + +render_quadlets() { export WORKDIR HOME LITELLM_MASTER_KEY UI_USERNAME UI_PASSWORD export POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY export LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD export OLLAMA_API_KEY OPENROUTER_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY export CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD export PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT + export LLAMA_CLASSIFIER_URL LLAMA_SERVER_URL export ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT export LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT export CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT export MINIO_S3_PORT MINIO_CONSOLE_PORT - python3 - "$WORKDIR/pod.yaml" <<'PY' -import os, sys, urllib.parse, json + local src_dir="${WORKDIR}/quadlets" + if ! derive_external_service_urls; then + echo "❌ Error: failed to derive external service URLs for Quadlet rendering" >&2 + return 1 + fi + mkdir -p "$QUADLET_DIR" + chmod 700 "$QUADLET_DIR" + python3 - "$src_dir" "$QUADLET_DIR" <<'PY' +import os, sys, urllib.parse, re, glob, shutil, tempfile uid = os.getuid() -with open(sys.argv[1], "r", encoding="utf-8") as f: - text = f.read() - -def yaml_scalar(val): - return json.dumps(val) - -placeholders = [ - "WORKDIR_PLACEHOLDER", - "HOME_PLACEHOLDER", - "RUN_USER_PLACEHOLDER", - "LITELLM_MASTER_KEY_PLACEHOLDER", - "LITELLM_UI_USERNAME_PLACEHOLDER", - "LITELLM_UI_PASSWORD_PLACEHOLDER", - "POSTGRES_PASSWORD_RAW_PLACEHOLDER", - "POSTGRES_PASSWORD_ENCODED_PLACEHOLDER", - "NEXTAUTH_SECRET_PLACEHOLDER", - "NEXTAUTH_URL_PLACEHOLDER", - "SALT_PLACEHOLDER", - "ENCRYPTION_KEY_PLACEHOLDER", - "OLLAMA_API_KEY_PLACEHOLDER", - "OPENROUTER_API_KEY_PLACEHOLDER", - "LANGFUSE_PUBLIC_KEY_PLACEHOLDER", - "LANGFUSE_SECRET_KEY_PLACEHOLDER", - "MINIO_USER_PLACEHOLDER", - "MINIO_PASSWORD_PLACEHOLDER", - "LANGFUSE_INIT_USER_PASSWORD_PLACEHOLDER", - "REDIS_AUTH_PLACEHOLDER", - "CLICKHOUSE_PASSWORD_PLACEHOLDER", - "PROXY_BASE_URL_PLACEHOLDER", - "PUBLIC_BASE_URL_PLACEHOLDER", - "ROUTING_DOMAIN_PLACEHOLDER", - "POD_NAME_PLACEHOLDER", - "DATA_ROOT_PLACEHOLDER", - "ROUTER_PORT_PLACEHOLDER", - "LITELLM_PORT_PLACEHOLDER", - "LANGFUSE_WEB_PORT_PLACEHOLDER", - "LANGFUSE_WORKER_PORT_PLACEHOLDER", - "POSTGRES_PORT_PLACEHOLDER", - "VALKEY_CACHE_PORT_PLACEHOLDER", - "VALKEY_LF_PORT_PLACEHOLDER", - "CLICKHOUSE_HTTP_PORT_PLACEHOLDER", - "CLICKHOUSE_TCP_PORT_PLACEHOLDER", - "MINIO_S3_PORT_PLACEHOLDER", - "MINIO_CONSOLE_PORT_PLACEHOLDER", - "ROUTER_IMAGE_PLACEHOLDER", -] -for ph in placeholders: - if ph not in text: - sys.stderr.write(f"Error: Required placeholder '{ph}' not found in pod.yaml. Ensure you are using the latest version of the template.\n") - sys.exit(1) -text = text.replace("WORKDIR_PLACEHOLDER", os.environ["WORKDIR"]) -text = text.replace("HOME_PLACEHOLDER", os.environ["HOME"]) -text = text.replace("RUN_USER_PLACEHOLDER", f"/run/user/{uid}") -text = text.replace("LITELLM_MASTER_KEY_PLACEHOLDER", yaml_scalar(os.environ["LITELLM_MASTER_KEY"])) -text = text.replace("LITELLM_UI_USERNAME_PLACEHOLDER", yaml_scalar(os.environ.get("UI_USERNAME") or "admin")) -text = text.replace("LITELLM_UI_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ.get("UI_PASSWORD") or os.environ.get("LITELLM_MASTER_KEY") or "admin")) -text = text.replace("POSTGRES_PASSWORD_RAW_PLACEHOLDER", yaml_scalar(os.environ["POSTGRES_PASSWORD"])) -# URL-encode the postgres password for DSN insertion -encoded_password = urllib.parse.quote(os.environ['POSTGRES_PASSWORD'], safe="") -text = text.replace("POSTGRES_PASSWORD_ENCODED_PLACEHOLDER", encoded_password) -text = text.replace("NEXTAUTH_SECRET_PLACEHOLDER", yaml_scalar(os.environ["NEXTAUTH_SECRET"])) -text = text.replace("SALT_PLACEHOLDER", yaml_scalar(os.environ["SALT"])) -text = text.replace("ENCRYPTION_KEY_PLACEHOLDER", yaml_scalar(os.environ["ENCRYPTION_KEY"])) -text = text.replace("OLLAMA_API_KEY_PLACEHOLDER", yaml_scalar(os.environ["OLLAMA_API_KEY"])) -text = text.replace("OPENROUTER_API_KEY_PLACEHOLDER", yaml_scalar(os.environ["OPENROUTER_API_KEY"])) -text = text.replace("LANGFUSE_PUBLIC_KEY_PLACEHOLDER", yaml_scalar(os.environ["LANGFUSE_PUBLIC_KEY"])) -text = text.replace("LANGFUSE_SECRET_KEY_PLACEHOLDER", yaml_scalar(os.environ["LANGFUSE_SECRET_KEY"])) -text = text.replace("MINIO_USER_PLACEHOLDER", yaml_scalar(os.environ["MINIO_ROOT_USER"])) -text = text.replace("MINIO_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["MINIO_ROOT_PASSWORD"])) -text = text.replace("LANGFUSE_INIT_USER_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["LANGFUSE_INIT_USER_PASSWORD"])) -text = text.replace("REDIS_AUTH_PLACEHOLDER", yaml_scalar(os.environ["REDIS_AUTH"])) -text = text.replace("CLICKHOUSE_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["CLICKHOUSE_PASSWORD"])) -# Derive PROXY_BASE_URL from PUBLIC_BASE_URL -public_base_url = os.environ["PUBLIC_BASE_URL"].rstrip("/") -proxy_base_url = f"{public_base_url}/litellm" -text = text.replace("PROXY_BASE_URL_PLACEHOLDER", yaml_scalar(proxy_base_url)) -text = text.replace("PUBLIC_BASE_URL_PLACEHOLDER", yaml_scalar(os.environ["PUBLIC_BASE_URL"])) -# Derive NEXTAUTH_URL from PUBLIC_BASE_URL (Langfuse needs the public URL for OAuth redirects) -nextauth_url = f"{public_base_url}/langfuse" -text = text.replace("NEXTAUTH_URL_PLACEHOLDER", yaml_scalar(nextauth_url)) -text = text.replace("ROUTING_DOMAIN_PLACEHOLDER", yaml_scalar(os.environ["ROUTING_DOMAIN"])) -# Raw replacements (no quoting — used for pod name, data root, and integer port values) -raw_replacements = { +src_dir, out_dir = sys.argv[1], sys.argv[2] + +encoded_pg = urllib.parse.quote(os.environ['POSTGRES_PASSWORD'], safe="") +# Derived by derive_external_service_urls(), shared with render_pod_yaml. +proxy_base_url = os.environ["PROXY_BASE_URL_DERIVED"] +nextauth_url = os.environ["NEXTAUTH_URL_DERIVED"] + +repl = { + "WORKDIR_PLACEHOLDER": os.environ["WORKDIR"], + "HOME_PLACEHOLDER": os.environ["HOME"], + "RUN_USER_PLACEHOLDER": f"/run/user/{uid}", + "LITELLM_MASTER_KEY_PLACEHOLDER": os.environ["LITELLM_MASTER_KEY"], + "LITELLM_UI_USERNAME_PLACEHOLDER": os.environ.get("UI_USERNAME") or "admin", + "LITELLM_UI_PASSWORD_PLACEHOLDER": os.environ.get("UI_PASSWORD") or os.environ.get("LITELLM_MASTER_KEY") or "admin", + "POSTGRES_PASSWORD_RAW_PLACEHOLDER": os.environ["POSTGRES_PASSWORD"], + "POSTGRES_PASSWORD_ENCODED_PLACEHOLDER": encoded_pg, + "NEXTAUTH_SECRET_PLACEHOLDER": os.environ["NEXTAUTH_SECRET"], + "NEXTAUTH_URL_PLACEHOLDER": nextauth_url, + "SALT_PLACEHOLDER": os.environ["SALT"], + "ENCRYPTION_KEY_PLACEHOLDER": os.environ["ENCRYPTION_KEY"], + "OLLAMA_API_KEY_PLACEHOLDER": os.environ["OLLAMA_API_KEY"], + "OPENROUTER_API_KEY_PLACEHOLDER": os.environ["OPENROUTER_API_KEY"], + "LANGFUSE_PUBLIC_KEY_PLACEHOLDER": os.environ["LANGFUSE_PUBLIC_KEY"], + "LANGFUSE_SECRET_KEY_PLACEHOLDER": os.environ["LANGFUSE_SECRET_KEY"], + "MINIO_USER_PLACEHOLDER": os.environ["MINIO_ROOT_USER"], + "MINIO_PASSWORD_PLACEHOLDER": os.environ["MINIO_ROOT_PASSWORD"], + "LANGFUSE_INIT_USER_PASSWORD_PLACEHOLDER": os.environ["LANGFUSE_INIT_USER_PASSWORD"], + "REDIS_AUTH_PLACEHOLDER": os.environ["REDIS_AUTH"], + "CLICKHOUSE_PASSWORD_PLACEHOLDER": os.environ["CLICKHOUSE_PASSWORD"], + "PROXY_BASE_URL_PLACEHOLDER": proxy_base_url, + "PUBLIC_BASE_URL_PLACEHOLDER": os.environ["PUBLIC_BASE_URL"].rstrip("/"), + "ROUTING_DOMAIN_PLACEHOLDER": os.environ["ROUTING_DOMAIN"], + "LLAMA_CLASSIFIER_URL_PLACEHOLDER": os.environ["LLAMA_CLASSIFIER_URL"], + "LLAMA_SERVER_URL_PLACEHOLDER": os.environ["LLAMA_SERVER_URL"], "POD_NAME_PLACEHOLDER": os.environ["POD_NAME"], "DATA_ROOT_PLACEHOLDER": os.environ["DATA_ROOT"], + "ROUTER_IMAGE_PLACEHOLDER": os.environ["ROUTER_IMAGE"], "ROUTER_PORT_PLACEHOLDER": os.environ["ROUTER_PORT"], "LITELLM_PORT_PLACEHOLDER": os.environ["LITELLM_PORT"], "LANGFUSE_WEB_PORT_PLACEHOLDER": os.environ["LANGFUSE_WEB_PORT"], @@ -722,33 +752,91 @@ raw_replacements = { "CLICKHOUSE_TCP_PORT_PLACEHOLDER": os.environ["CLICKHOUSE_TCP_PORT"], "MINIO_S3_PORT_PLACEHOLDER": os.environ["MINIO_S3_PORT"], "MINIO_CONSOLE_PORT_PLACEHOLDER": os.environ["MINIO_CONSOLE_PORT"], - "ROUTER_IMAGE_PLACEHOLDER": os.environ["ROUTER_IMAGE"], } -for ph, val in raw_replacements.items(): - text = text.replace(ph, val) -import re -unresolved = sorted(set(re.findall(r"\b[A-Z0-9_]+_PLACEHOLDER\b", text))) -if unresolved: - sys.stderr.write( - "Error: Unresolved placeholders remain in rendered pod.yaml: " - + ", ".join(unresolved) - + "\n" - ) + +templates = sorted(glob.glob(os.path.join(src_dir, "*.pod")) + glob.glob(os.path.join(src_dir, "*.container"))) +if not templates: + sys.stderr.write(f"Error: no quadlet templates found in {src_dir}\n") sys.exit(1) -sys.stdout.write(text) + +staging_dir = tempfile.mkdtemp(prefix=".llm-routing-render-", dir=os.path.dirname(out_dir)) +try: + for tpl in templates: + with open(tpl, "r", encoding="utf-8") as f: + text = f.read() + for ph, val in repl.items(): + text = text.replace(ph, str(val)) + unresolved = sorted(set(re.findall(r"\b[A-Z0-9_]+_PLACEHOLDER\b", text))) + if unresolved: + sys.stderr.write(f"Error: Unresolved placeholders in {os.path.basename(tpl)}: {', '.join(unresolved)}\n") + sys.exit(1) + # Quadlet Environment= values use systemd's command-line parser. Quote each + # complete KEY=value assignment after substitution so spaces, quotes, and + # backslashes in configurable values remain one environment value. + def quote_environment(match): + value = match.group(1).replace("\\", "\\\\").replace('"', '\\"') + return f'Environment="{value}"' + text = re.sub(r"(?m)^Environment=(.*)$", quote_environment, text) + staged_path = os.path.join(staging_dir, os.path.basename(tpl)) + with open(staged_path, "w", encoding="utf-8") as f: + f.write(text) + # Rendered units include credentials; systemd user generator can read owner-only files. + os.chmod(staged_path, 0o600) + + # All templates are now valid. Replace individual files atomically, then + # remove stale units; a failed render above leaves the prior unit set intact. + rendered_names = {os.path.basename(tpl) for tpl in templates} + for name in rendered_names: + os.replace(os.path.join(staging_dir, name), os.path.join(out_dir, name)) + for stale in glob.glob(os.path.join(out_dir, "*.pod")) + glob.glob(os.path.join(out_dir, "*.container")): + if os.path.basename(stale) not in rendered_names: + os.unlink(stale) + for name in sorted(rendered_names): + print(f" ✓ {name}") +finally: + shutil.rmtree(staging_dir, ignore_errors=True) PY + echo "✓ Quadlets rendered to ${QUADLET_DIR}" +} + +# Install quadlets and (re)generate systemd units; start or restart the stack. +deploy_quadlets() { + require_user_systemd || exit 1 + echo "📋 Rendering quadlet units..." + render_quadlets + if ! systemctl --user daemon-reload; then + echo "❌ Error: systemctl --user daemon-reload failed (no systemd user session?)" >&2 + exit 1 + fi + echo "✓ systemd units regenerated" + if systemctl --user is-active --quiet "$LLM_ROUTING_POD_UNIT"; then + echo "🔄 Restarting ${LLM_ROUTING_POD_UNIT}..." + if ! systemctl --user restart "$LLM_ROUTING_POD_UNIT"; then + echo "❌ Error: failed to restart ${LLM_ROUTING_POD_UNIT}" >&2 + echo " Hint: run 'systemctl --user status ${LLM_ROUTING_POD_UNIT} --no-pager' to inspect the failure" >&2 + exit 1 + fi + else + echo "🚀 Starting ${LLM_ROUTING_POD_UNIT}..." + if ! systemctl --user start "$LLM_ROUTING_POD_UNIT"; then + echo "❌ Error: failed to start ${LLM_ROUTING_POD_UNIT}" >&2 + echo " Hint: run 'systemctl --user status ${LLM_ROUTING_POD_UNIT} --no-pager' to inspect the failure" >&2 + exit 1 + fi + fi } deploy_fresh_pod() { generate_clickhouse_config render_litellm_config render_router_config - render_pod_yaml | podman play kube - + deploy_quadlets setup_minio_buckets verify_stack_health } -if podman pod exists ${POD_NAME} 2>/dev/null; then +STACK_OWNERSHIP=$(stack_ownership) +if [[ "$STACK_OWNERSHIP" != "absent" ]]; then if $FULL_REBUILD; then echo "🔨 Building custom local triage router image..." podman build -t "${ROUTER_IMAGE}" -f router/Dockerfile router @@ -766,11 +854,26 @@ if podman pod exists ${POD_NAME} 2>/dev/null; then echo "🚀 Deploying replacement pod from YAML..." deploy_fresh_pod else - echo "🔄 Restarting existing ${POD_NAME} (use --replace or --pull to recreate)..." - podman pod restart ${POD_NAME} + if [[ "$STACK_OWNERSHIP" == "quadlet" ]]; then + require_user_systemd || exit 1 + echo "🔄 Restarting Quadlet-owned stack via systemd..." + systemctl --user reset-failed "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true + if ! systemctl --user restart "$LLM_ROUTING_POD_UNIT"; then + echo "❌ Error: failed to restart ${LLM_ROUTING_POD_UNIT}" >&2 + echo " Hint: run 'systemctl --user status ${LLM_ROUTING_POD_UNIT} --no-pager' to inspect the failure" >&2 + exit 1 + fi + else + echo "🔄 Restarting legacy ${POD_NAME} (use --replace or --pull to migrate)..." + if ! podman pod restart "${POD_NAME}"; then + echo "❌ Error: failed to restart legacy pod ${POD_NAME}" >&2 + exit 1 + fi + fi setup_minio_buckets verify_stack_health + derive_external_service_urls echo "" echo "=========================================================================" echo "🎉 SUCCESS: LLM Triage Gateway restarted!" @@ -778,7 +881,7 @@ if podman pod exists ${POD_NAME} 2>/dev/null; then echo " (local) : ${LOCAL_BASE_URL}/v1" echo "⚙️ Dashboard URL : ${PUBLIC_BASE_URL}/dashboard" echo "🔑 Gateway API Key : gateway-pass" - echo "🔐 LiteLLM Admin UI: ${PUBLIC_BASE_URL}/litellm/ui" + echo "🔐 LiteLLM Admin UI: ${PROXY_BASE_URL_DERIVED}/ui/" echo " Username: admin | Password: $LITELLM_MASTER_KEY" echo "=========================================================================" exit 0 @@ -804,12 +907,13 @@ else fi +derive_external_service_urls echo "=========================================================================" echo "🎉 SUCCESS: LLM Triage Gateway successfully deployed!" echo "📍 Entry endpoint : ${PUBLIC_BASE_URL}/v1" echo " (local) : ${LOCAL_BASE_URL}/v1" echo "⚙️ Dashboard URL : ${PUBLIC_BASE_URL}/dashboard" echo "🔑 Gateway API Key : gateway-pass" -echo "🔐 LiteLLM Admin UI: ${PUBLIC_BASE_URL}/litellm/ui" +echo "🔐 LiteLLM Admin UI: ${PROXY_BASE_URL_DERIVED}/ui/" echo " Username: admin | Password: $LITELLM_MASTER_KEY" echo "=========================================================================" diff --git a/tests/test_quadlet_templates.py b/tests/test_quadlet_templates.py new file mode 100644 index 00000000..22f02b6c --- /dev/null +++ b/tests/test_quadlet_templates.py @@ -0,0 +1,89 @@ +"""Static deployment-contract tests for the systemd Quadlet templates.""" +from pathlib import Path +import re + + +ROOT = Path(__file__).resolve().parent.parent +QUADLETS = ROOT / "quadlets" + + +def test_quadlet_inventory_and_pod_membership(): + containers = sorted(QUADLETS.glob("*.container")) + assert len(containers) == 9 + assert (QUADLETS / "llm-routing.pod").is_file() + for container in containers: + text = container.read_text() + assert "Pod=llm-routing.pod" in text, container.name + assert "ContainerName=POD_NAME_PLACEHOLDER-" in text, container.name + + +def test_liveness_healthchecks_restart_failed_containers(): + for container in sorted(QUADLETS.glob("*.container")): + text = container.read_text() + assert "HealthCmd=" in text, container.name + # Podman kills an unhealthy container; systemd Restart=always then replaces it. + assert "HealthOnFailure=kill" in text, container.name + assert "Restart=always" in text, container.name + + +def test_quadlet_templates_remain_env_rendered_and_secret_free(): + for template in [*QUADLETS.glob("*.container"), QUADLETS / "llm-routing.pod"]: + text = template.read_text() + assert "_PLACEHOLDER" in text, template.name + assert not re.search(r"sk-(?:or|lf|lit)-[A-Za-z0-9_-]{12,}", text), template.name + + +def test_upgrade_syncs_quadlets_before_quadlet_start_stack(): + script = (ROOT / "scripts" / "upgrade-prod.sh").read_text() + assert "quadlets/" in script + assert 'rsync -a --delete "$TEMP_DIR/quadlets/" "$PROD_DIR/quadlets/"' in script + assert "for f in pod.yaml start-stack.sh quadlets/" in script + + +def test_rendered_quadlets_are_owner_only(): + script = (ROOT / "start-stack.sh").read_text() + assert 'chmod 700 "$QUADLET_DIR"' in script + assert "os.chmod(staged_path, 0o600)" in script + assert 'LLM_ROUTING_POD_UNIT="llm-routing-pod.service"' in script + assert "systemctl --user daemon-reload" in script + assert "stack_ownership()" in script + assert "PODMAN_SYSTEMD_UNIT" in script + assert "require_user_systemd()" in script + + +def test_quadlet_renderer_quotes_environment_values_for_systemd(): + script = (ROOT / "start-stack.sh").read_text() + assert 'text = re.sub(r"(?m)^Environment=(.*)$", quote_environment, text)' in script + assert "def quote_environment(match):" in script + assert "return f'Environment=\"{value}\"'" in script + + +def test_quadlet_renderer_stages_before_replacing_live_units(): + script = (ROOT / "start-stack.sh").read_text() + assert "tempfile.mkdtemp(prefix=\".llm-routing-render-\"" in script + assert "os.replace(os.path.join(staging_dir, name), os.path.join(out_dir, name))" in script + assert "shutil.rmtree(staging_dir, ignore_errors=True)" in script + + +def test_quadlet_systemd_failures_are_reported(): + script = (ROOT / "start-stack.sh").read_text() + assert 'if ! systemctl --user restart "$LLM_ROUTING_POD_UNIT"; then' in script + assert 'if ! systemctl --user start "$LLM_ROUTING_POD_UNIT"; then' in script + + +def test_router_quadlet_reasserts_overlayed_llama_urls_after_env_source(): + template = (QUADLETS / "llm-routing-router.container").read_text() + assert "LLAMA_CLASSIFIER_URL=LLAMA_CLASSIFIER_URL_PLACEHOLDER" in template + assert "LLAMA_SERVER_URL=LLAMA_SERVER_URL_PLACEHOLDER" in template + assert template.index("LLAMA_SERVER_URL=LLAMA_SERVER_URL_PLACEHOLDER") < template.index("exec uvicorn") + script = (ROOT / "start-stack.sh").read_text() + assert '"LLAMA_CLASSIFIER_URL_PLACEHOLDER": os.environ["LLAMA_CLASSIFIER_URL"]' in script + assert '"LLAMA_SERVER_URL_PLACEHOLDER": os.environ["LLAMA_SERVER_URL"]' in script + + +def test_quadlet_deployment_enforces_prerequisite_and_restart_failures(): + script = (ROOT / "start-stack.sh").read_text() + assert "render_pod_yaml()" not in script + assert "require_user_systemd || exit 1" in script + assert "failed to derive external service URLs for Quadlet rendering" in script + assert 'if ! podman pod restart "${POD_NAME}"; then' in script