From d1624595f7d10a391f7f6baf73fe68c84a6d74d6 Mon Sep 17 00:00:00 2001 From: boy Date: Wed, 22 Jul 2026 23:46:22 +0200 Subject: [PATCH 1/8] feat(deploy): migrate stack management from play kube to systemd Quadlets Replace 'podman play kube' with declarative Quadlet units rendered from quadlets/*.pod + quadlets/*.container templates into ~/.config/containers/systemd/llm-routing/. systemd's podman-user-generator turns them into llm-routing-*.service units, making systemd the single supervisor: boot auto-start (WantedBy=default.target + linger) and crash recovery (Restart=always) now come from systemd instead of podman's restartPolicy. Key points: - Templates use the same _PLACEHOLDER convention as pod.yaml; a new render_quadlets() in start-stack.sh reuses the existing env-derived values (ports, secrets, DATA_ROOT, POD_NAME) so dev/prod parity is preserved with plain string replacement (bare scalars, not YAML). - deploy_fresh_pod() now: render configs -> render quadlets -> daemon-reload -> start/restart llm-routing-pod.service. The bare 'restart existing' path and safe_pod_teardown() detect systemd-managed stacks and route through systemctl accordingly. - Entrypoint semantics: Quadlet Exec= only sets args (appended to the image entrypoint), so containers where pod.yaml used command: now set Entrypoint= explicitly (litellm venv python, valkey-server, redis-server, /bin/sh for the router). MinIO keeps its image entrypoint with Exec= providing the server args, matching pod.yaml args: behavior. - AddHost=:127.0.0.1 on all containers replicates the /etc/hosts entry play kube injected, required by langfuse 3.222+ which resolves the pod hostname at startup. - Healthchecks ported from livenessProbe exec commands (HealthCmd, HealthStartPeriod maps initialDelaySeconds). Verified on dev: all 9 containers healthy, 193/193 unit tests pass, crash recovery (kill -> systemd auto-restart <12s), canonical HTTPS endpoints green (model timeouts on free tier are pre-existing, identical on prod play-kube). --- quadlets/llm-routing-clickhouse.container | 26 ++++ quadlets/llm-routing-langfuse-web.container | 50 +++++++ .../llm-routing-langfuse-worker.container | 39 +++++ quadlets/llm-routing-litellm.container | 43 ++++++ quadlets/llm-routing-minio.container | 26 ++++ quadlets/llm-routing-postgres.container | 25 ++++ quadlets/llm-routing-router.container | 52 +++++++ quadlets/llm-routing-valkey-cache.container | 24 +++ quadlets/llm-routing-valkey-lf.container | 22 +++ quadlets/llm-routing.pod | 13 ++ start-stack.sh | 139 +++++++++++++++++- 11 files changed, 456 insertions(+), 3 deletions(-) create mode 100644 quadlets/llm-routing-clickhouse.container create mode 100644 quadlets/llm-routing-langfuse-web.container create mode 100644 quadlets/llm-routing-langfuse-worker.container create mode 100644 quadlets/llm-routing-litellm.container create mode 100644 quadlets/llm-routing-minio.container create mode 100644 quadlets/llm-routing-postgres.container create mode 100644 quadlets/llm-routing-router.container create mode 100644 quadlets/llm-routing-valkey-cache.container create mode 100644 quadlets/llm-routing-valkey-lf.container create mode 100644 quadlets/llm-routing.pod diff --git a/quadlets/llm-routing-clickhouse.container b/quadlets/llm-routing-clickhouse.container new file mode 100644 index 00000000..65f0c2f4 --- /dev/null +++ b/quadlets/llm-routing-clickhouse.container @@ -0,0 +1,26 @@ +[Unit] +Description=ClickHouse — Langfuse v3 OLAP trace store (POD_NAME_PLACEHOLDER) + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-clickhouse-db +Image=clickhouse/clickhouse-server:26.6.2.81 +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +User=101 +Group=101 +Environment=CLICKHOUSE_DB=default +Environment=CLICKHOUSE_USER=clickhouse +Environment=CLICKHOUSE_PASSWORD=CLICKHOUSE_PASSWORD_PLACEHOLDER +Volume=DATA_ROOT_PLACEHOLDER/clickhouse-data:/var/lib/clickhouse +Volume=DATA_ROOT_PLACEHOLDER/clickhouse-config/port-override.xml:/etc/clickhouse-server/config.d/port-override.xml +HealthCmd=clickhouse-client --host 127.0.0.1 --port CLICKHOUSE_TCP_PORT_PLACEHOLDER --user clickhouse --password CLICKHOUSE_PASSWORD_PLACEHOLDER --query "SELECT 1" +HealthInterval=15s +HealthTimeout=5s +HealthRetries=5 +HealthStartPeriod=20s + +[Service] +Restart=always +TimeoutStartSec=120 diff --git a/quadlets/llm-routing-langfuse-web.container b/quadlets/llm-routing-langfuse-web.container new file mode 100644 index 00000000..e9fa0db4 --- /dev/null +++ b/quadlets/llm-routing-langfuse-web.container @@ -0,0 +1,50 @@ +[Unit] +Description=Langfuse Web — observability dashboard (POD_NAME_PLACEHOLDER) +After=llm-routing-postgres.service llm-routing-clickhouse.service llm-routing-valkey-lf.service llm-routing-minio.service +Wants=llm-routing-postgres.service llm-routing-clickhouse.service llm-routing-valkey-lf.service llm-routing-minio.service + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-langfuse-web +Image=langfuse/langfuse:3.222.0 +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +Environment=DATABASE_URL=postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:POSTGRES_PORT_PLACEHOLDER/langfuse +Environment=NEXTAUTH_SECRET=NEXTAUTH_SECRET_PLACEHOLDER +Environment=NEXTAUTH_URL=NEXTAUTH_URL_PLACEHOLDER +Environment=SALT=SALT_PLACEHOLDER +Environment=ENCRYPTION_KEY=ENCRYPTION_KEY_PLACEHOLDER +Environment=HOSTNAME=0.0.0.0 +Environment=PORT=LANGFUSE_WEB_PORT_PLACEHOLDER +Environment=TELEMETRY_ENABLED=false +Environment=LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse-events +Environment=LANGFUSE_S3_EVENT_UPLOAD_REGION=us-east-1 +Environment=LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=MINIO_USER_PLACEHOLDER +Environment=LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=MINIO_PASSWORD_PLACEHOLDER +Environment=LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT=http://127.0.0.1:MINIO_S3_PORT_PLACEHOLDER +Environment=S3_FORCE_PATH_STYLE=true +Environment=CLICKHOUSE_URL=http://127.0.0.1:CLICKHOUSE_HTTP_PORT_PLACEHOLDER +Environment=CLICKHOUSE_MIGRATION_URL=clickhouse://127.0.0.1:CLICKHOUSE_TCP_PORT_PLACEHOLDER +Environment=CLICKHOUSE_USER=clickhouse +Environment=CLICKHOUSE_PASSWORD=CLICKHOUSE_PASSWORD_PLACEHOLDER +Environment=CLICKHOUSE_CLUSTER_ENABLED=false +Environment=REDIS_HOST=127.0.0.1 +Environment=REDIS_PORT=VALKEY_LF_PORT_PLACEHOLDER +Environment=REDIS_AUTH=REDIS_AUTH_PLACEHOLDER +Environment=LANGFUSE_INIT_ORG_ID=org-local-dev-id +Environment=LANGFUSE_INIT_ORG_NAME=Local Development +Environment=LANGFUSE_INIT_PROJECT_ID=proj-triage-gateway-id +Environment=LANGFUSE_INIT_PROJECT_NAME=Triage Gateway +Environment=LANGFUSE_INIT_USER_EMAIL=admin@local.dev +Environment=LANGFUSE_INIT_USER_PASSWORD=LANGFUSE_INIT_USER_PASSWORD_PLACEHOLDER +Environment=LANGFUSE_LOG_LEVEL=warn +HealthCmd=wget -q -O /dev/null http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER/api/public/health +HealthInterval=15s +HealthTimeout=5s +HealthRetries=5 +HealthStartPeriod=240s + +[Service] +Restart=always +TimeoutStartSec=300 diff --git a/quadlets/llm-routing-langfuse-worker.container b/quadlets/llm-routing-langfuse-worker.container new file mode 100644 index 00000000..1872373c --- /dev/null +++ b/quadlets/llm-routing-langfuse-worker.container @@ -0,0 +1,39 @@ +[Unit] +Description=Langfuse Worker — background job processor (POD_NAME_PLACEHOLDER) +After=llm-routing-postgres.service llm-routing-clickhouse.service llm-routing-valkey-lf.service llm-routing-minio.service +Wants=llm-routing-postgres.service llm-routing-clickhouse.service llm-routing-valkey-lf.service llm-routing-minio.service + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-langfuse-worker +Image=langfuse/langfuse-worker:3.222.0 +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +Environment=DATABASE_URL=postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:POSTGRES_PORT_PLACEHOLDER/langfuse +Environment=CLICKHOUSE_URL=http://127.0.0.1:CLICKHOUSE_HTTP_PORT_PLACEHOLDER +Environment=CLICKHOUSE_USER=clickhouse +Environment=CLICKHOUSE_PASSWORD=CLICKHOUSE_PASSWORD_PLACEHOLDER +Environment=CLICKHOUSE_CLUSTER_ENABLED=false +Environment=REDIS_HOST=127.0.0.1 +Environment=REDIS_PORT=VALKEY_LF_PORT_PLACEHOLDER +Environment=REDIS_AUTH=REDIS_AUTH_PLACEHOLDER +Environment=HOSTNAME=0.0.0.0 +Environment=PORT=LANGFUSE_WORKER_PORT_PLACEHOLDER +Environment=TELEMETRY_ENABLED=false +Environment=LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse-events +Environment=LANGFUSE_S3_EVENT_UPLOAD_REGION=us-east-1 +Environment=LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=MINIO_USER_PLACEHOLDER +Environment=LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=MINIO_PASSWORD_PLACEHOLDER +Environment=LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT=http://127.0.0.1:MINIO_S3_PORT_PLACEHOLDER +Environment=S3_FORCE_PATH_STYLE=true +Environment=LANGFUSE_LOG_LEVEL=warn +HealthCmd=pgrep node +HealthInterval=15s +HealthTimeout=5s +HealthRetries=5 +HealthStartPeriod=240s + +[Service] +Restart=always +TimeoutStartSec=300 diff --git a/quadlets/llm-routing-litellm.container b/quadlets/llm-routing-litellm.container new file mode 100644 index 00000000..6ab1b902 --- /dev/null +++ b/quadlets/llm-routing-litellm.container @@ -0,0 +1,43 @@ +[Unit] +Description=LiteLLM Gateway — upstream proxy with adaptive routing (POD_NAME_PLACEHOLDER) +After=llm-routing-postgres.service llm-routing-valkey-cache.service +Wants=llm-routing-postgres.service llm-routing-valkey-cache.service + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-litellm-gateway +Image=ghcr.io/berriai/litellm:v1.93.0 +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +# Image entrypoint is docker/prod_entrypoint.sh (litellm CLI); pod.yaml used +# command: python3 /app/entrypoint.py which replaced it. Quadlet Exec= only sets +# args, so override Entrypoint= with the venv python (matches play-kube behavior). +Entrypoint=/app/.venv/bin/python3 +Exec=/app/entrypoint.py +Environment=DATABASE_URL=postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:POSTGRES_PORT_PLACEHOLDER/postgres +Environment=STORE_MODEL_IN_DB=True +Environment=LITELLM_LOG=INFO +Environment=LANGFUSE_HOST=http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER +Environment=LITELLM_MASTER_KEY=LITELLM_MASTER_KEY_PLACEHOLDER +Environment=LITELLM_UI_USERNAME=LITELLM_UI_USERNAME_PLACEHOLDER +Environment=LITELLM_UI_PASSWORD=LITELLM_UI_PASSWORD_PLACEHOLDER +Environment=OLLAMA_API_KEY=OLLAMA_API_KEY_PLACEHOLDER +Environment=OPENROUTER_API_KEY=OPENROUTER_API_KEY_PLACEHOLDER +Environment=SERVER_ROOT_PATH=/llm-routing/litellm +Environment=PROXY_BASE_URL=PROXY_BASE_URL_PLACEHOLDER +Environment=LITELLM_PORT=LITELLM_PORT_PLACEHOLDER +Environment=POSTGRES_PORT=POSTGRES_PORT_PLACEHOLDER +Volume=DATA_ROOT_PLACEHOLDER/litellm-rendered/config.yaml:/app/config.yaml +Volume=DATA_ROOT_PLACEHOLDER/litellm-rendered/entrypoint.py:/app/entrypoint.py +Volume=WORKDIR_PLACEHOLDER/.env:/config/.env +Volume=HOME_PLACEHOLDER/.gemini:/config/gemini_auth +HealthCmd=python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:LITELLM_PORT_PLACEHOLDER/health/liveness')" +HealthInterval=15s +HealthTimeout=5s +HealthRetries=5 +HealthStartPeriod=240s + +[Service] +Restart=always +TimeoutStartSec=300 diff --git a/quadlets/llm-routing-minio.container b/quadlets/llm-routing-minio.container new file mode 100644 index 00000000..56f59a0e --- /dev/null +++ b/quadlets/llm-routing-minio.container @@ -0,0 +1,26 @@ +[Unit] +Description=MinIO S3 — Langfuse v3 event storage (POD_NAME_PLACEHOLDER) + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-minio-s3 +Image=minio/minio:RELEASE.2025-09-07T16-13-09Z +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +Environment=MINIO_ROOT_USER=MINIO_USER_PLACEHOLDER +Environment=MINIO_ROOT_PASSWORD=MINIO_PASSWORD_PLACEHOLDER +# pod.yaml used args: server ... with no command:, so image entrypoint +# /usr/bin/docker-entrypoint.sh ran and consumed them. Quadlet Exec= sets args +# the same way — image entrypoint stays, Exec provides the server args. +Exec=server --quiet /data --address :MINIO_S3_PORT_PLACEHOLDER --console-address :MINIO_CONSOLE_PORT_PLACEHOLDER +Volume=DATA_ROOT_PLACEHOLDER/minio-data:/data +HealthCmd=curl -f http://127.0.0.1:MINIO_S3_PORT_PLACEHOLDER/minio/health/live +HealthInterval=15s +HealthTimeout=5s +HealthRetries=5 +HealthStartPeriod=10s + +[Service] +Restart=always +TimeoutStartSec=90 diff --git a/quadlets/llm-routing-postgres.container b/quadlets/llm-routing-postgres.container new file mode 100644 index 00000000..980a56bc --- /dev/null +++ b/quadlets/llm-routing-postgres.container @@ -0,0 +1,25 @@ +[Unit] +Description=PostgreSQL — Langfuse + LiteLLM metadata store (POD_NAME_PLACEHOLDER) + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-postgres-db +Image=pgvector/pgvector:0.8.5-pg18 +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +Environment=POSTGRES_USER=postgres +Environment=POSTGRES_PASSWORD=POSTGRES_PASSWORD_RAW_PLACEHOLDER +Environment=POSTGRES_DB=langfuse +Environment=PGDATA=/var/lib/postgresql/data/pg18 +Environment=PGPORT=POSTGRES_PORT_PLACEHOLDER +Volume=DATA_ROOT_PLACEHOLDER/postgres-data:/var/lib/postgresql +HealthCmd=pg_isready -U postgres -p POSTGRES_PORT_PLACEHOLDER +HealthInterval=10s +HealthTimeout=5s +HealthRetries=5 +HealthStartPeriod=15s + +[Service] +Restart=always +TimeoutStartSec=120 diff --git a/quadlets/llm-routing-router.container b/quadlets/llm-routing-router.container new file mode 100644 index 00000000..3c39430b --- /dev/null +++ b/quadlets/llm-routing-router.container @@ -0,0 +1,52 @@ +[Unit] +Description=LLM Triage Router — classification + routing gateway (POD_NAME_PLACEHOLDER) +After=llm-routing-litellm.service +Wants=llm-routing-litellm.service + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-llm-triage-router +Image=ROUTER_IMAGE_PLACEHOLDER +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +# Router image has no entrypoint (EP=null, CMD=uvicorn...); pod.yaml used +# command: /bin/sh + args: -c ... — Entrypoint=/bin/sh, Exec=-c "..." matches. +Entrypoint=/bin/sh +Exec=-c "set -a && . /config/.env && set +a && LITELLM_PORT=LITELLM_PORT_PLACEHOLDER VALKEY_CACHE_PORT=VALKEY_CACHE_PORT_PLACEHOLDER exec uvicorn main:app --host 0.0.0.0 --port ROUTER_PORT_PLACEHOLDER --log-level $${LOG_LEVEL:-warning}" +Environment=CONFIG_PATH=/config/router_dir/config.yaml +Environment=LITELLM_CONFIG_PATH=/config/litellm_dir/config.yaml +Environment=DATABASE_URL=postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:POSTGRES_PORT_PLACEHOLDER/postgres +Environment=DBUS_SESSION_BUS_ADDRESS=unix:path=RUN_USER_PLACEHOLDER/bus +Environment=LITELLM_MASTER_KEY=LITELLM_MASTER_KEY_PLACEHOLDER +Environment=LANGFUSE_PUBLIC_KEY=LANGFUSE_PUBLIC_KEY_PLACEHOLDER +Environment=LANGFUSE_SECRET_KEY=LANGFUSE_SECRET_KEY_PLACEHOLDER +Environment=LANGFUSE_HOST=http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER +Environment=OLLAMA_API_KEY=OLLAMA_API_KEY_PLACEHOLDER +Environment=PUBLIC_BASE_URL=PUBLIC_BASE_URL_PLACEHOLDER +Environment=ROUTING_DOMAIN=ROUTING_DOMAIN_PLACEHOLDER +Environment=LITELLM_PORT=LITELLM_PORT_PLACEHOLDER +Environment=VALKEY_CACHE_PORT=VALKEY_CACHE_PORT_PLACEHOLDER +Environment=ROUTER_PORT=ROUTER_PORT_PLACEHOLDER +Environment=LANGFUSE_WEB_PORT=LANGFUSE_WEB_PORT_PLACEHOLDER +Environment=LOG_LEVEL=info +Volume=DATA_ROOT_PLACEHOLDER/router-rendered:/config/router_dir +Volume=DATA_ROOT_PLACEHOLDER/litellm-rendered:/config/litellm_dir +Volume=DATA_ROOT_PLACEHOLDER/datasets:/app/data +Volume=HOME_PLACEHOLDER/.gemini:/config/gemini_auth +Volume=HOME_PLACEHOLDER/.gemini:/root/.gemini +Volume=HOME_PLACEHOLDER/.local/share/goose:/config/goose_sessions +Volume=HOME_PLACEHOLDER/.local/bin/agy:/usr/local/bin/agy +Volume=RUN_USER_PLACEHOLDER/bus:RUN_USER_PLACEHOLDER/bus +Volume=HOME_PLACEHOLDER/.local/share/keyrings:/root/.local/share/keyrings +Volume=RUN_USER_PLACEHOLDER:RUN_USER_PLACEHOLDER +Volume=WORKDIR_PLACEHOLDER/.env:/config/.env +HealthCmd=python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:ROUTER_PORT_PLACEHOLDER/metrics')" +HealthInterval=15s +HealthTimeout=5s +HealthRetries=5 +HealthStartPeriod=240s + +[Service] +Restart=always +TimeoutStartSec=300 diff --git a/quadlets/llm-routing-valkey-cache.container b/quadlets/llm-routing-valkey-cache.container new file mode 100644 index 00000000..97b497aa --- /dev/null +++ b/quadlets/llm-routing-valkey-cache.container @@ -0,0 +1,24 @@ +[Unit] +Description=Valkey Cache — LiteLLM response cache (POD_NAME_PLACEHOLDER) + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-valkey-cache +Image=valkey/valkey:9.1.1 +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +# pod.yaml used command: (replaces entrypoint). Image entrypoint docker-entrypoint.sh +# would also work, but explicit Entrypoint= matches play-kube semantics exactly. +Entrypoint=valkey-server +Exec=--bind 127.0.0.1 --port VALKEY_CACHE_PORT_PLACEHOLDER --loglevel warning +Volume=DATA_ROOT_PLACEHOLDER/valkey-data:/data +HealthCmd=valkey-cli -p VALKEY_CACHE_PORT_PLACEHOLDER ping +HealthInterval=10s +HealthTimeout=2s +HealthRetries=5 +HealthStartPeriod=5s + +[Service] +Restart=always +TimeoutStartSec=90 diff --git a/quadlets/llm-routing-valkey-lf.container b/quadlets/llm-routing-valkey-lf.container new file mode 100644 index 00000000..21f3ccee --- /dev/null +++ b/quadlets/llm-routing-valkey-lf.container @@ -0,0 +1,22 @@ +[Unit] +Description=Valkey LF — Langfuse v3 BullMQ job queue (POD_NAME_PLACEHOLDER) + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-valkey-lf +Image=valkey/valkey:9.1.1 +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +Entrypoint=redis-server +Exec=--bind 127.0.0.1 --port VALKEY_LF_PORT_PLACEHOLDER --requirepass REDIS_AUTH_PLACEHOLDER --maxmemory-policy noeviction --loglevel warning +Volume=DATA_ROOT_PLACEHOLDER/redis-lf-data:/data +HealthCmd=valkey-cli -p VALKEY_LF_PORT_PLACEHOLDER -a REDIS_AUTH_PLACEHOLDER ping +HealthInterval=10s +HealthTimeout=2s +HealthRetries=5 +HealthStartPeriod=5s + +[Service] +Restart=always +TimeoutStartSec=90 diff --git a/quadlets/llm-routing.pod b/quadlets/llm-routing.pod new file mode 100644 index 00000000..3f5e0c4a --- /dev/null +++ b/quadlets/llm-routing.pod @@ -0,0 +1,13 @@ +# LLM Triage Gateway pod — Quadlet template (rendered by start-stack.sh) +# Pod name is rendered from POD_NAME (prod-router-pod / dev-router-pod). +[Unit] +Description=LLM Triage Gateway Pod (POD_NAME_PLACEHOLDER) +Wants=network-online.target +After=network-online.target + +[Pod] +PodName=POD_NAME_PLACEHOLDER +Network=host + +[Install] +WantedBy=default.target diff --git a/start-stack.sh b/start-stack.sh index 6087a442..21d3a0b2 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -534,6 +534,16 @@ verify_stack_health() { # Graceful stop (SIGTERM with 30s timeout) lets ClickHouse/Postgres flush, # then force-remove if needed. Avoids data corruption from SIGKILL. safe_pod_teardown() { + # If the stack is systemd-managed (quadlets), stop via systemd so the + # manager state stays consistent; then remove any leftover pod. + if systemctl --user is-active --quiet llm-routing-pod.service 2>/dev/null; then + echo "🛑 Stopping systemd-managed stack (llm-routing-pod.service)..." + systemctl --user stop llm-routing-pod.service || true + podman pod rm -f ${POD_NAME} 2>/dev/null || true + cleanup_zombie_ports + echo "✓ Stack stopped, ports cleaned" + return + fi if podman pod exists ${POD_NAME} 2>/dev/null; then echo "🛑 Gracefully stopping pod (SIGTERM, 30s timeout)..." podman pod stop -t 30 ${POD_NAME} 2>/dev/null || true @@ -739,11 +749,129 @@ sys.stdout.write(text) PY } +# ── Quadlet rendering + installation ── +# Renders quadlets/*.pod + quadlets/*.container templates (same _PLACEHOLDER +# convention as pod.yaml) into ~/.config/containers/systemd/llm-routing/ and +# lets systemd's podman-user-generator turn them into real units. +# Quadlet values are bare scalars (not YAML) so plain string replacement is used. +QUADLET_DIR="${HOME}/.config/containers/systemd/llm-routing" + +render_quadlets() { + export WORKDIR HOME LITELLM_MASTER_KEY UI_USERNAME UI_PASSWORD + export POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY + export LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD + export OLLAMA_API_KEY OPENROUTER_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY + export CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD + export PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT + export ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT + export LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT + export CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT + export MINIO_S3_PORT MINIO_CONSOLE_PORT + local src_dir="${WORKDIR}/quadlets" + mkdir -p "$QUADLET_DIR" + python3 - "$src_dir" "$QUADLET_DIR" <<'PY' +import os, sys, urllib.parse, re, glob +uid = os.getuid() +src_dir, out_dir = sys.argv[1], sys.argv[2] + +encoded_pg = urllib.parse.quote(os.environ['POSTGRES_PASSWORD'], safe="") +# Derive PROXY_BASE_URL / NEXTAUTH_URL exactly as render_pod_yaml does +# (path-based on master; subdomain switch lands with the routing branch) +public_base_url = os.environ["PUBLIC_BASE_URL"].rstrip("/") +proxy_base_url = f"{public_base_url}/litellm" +nextauth_url = f"{public_base_url}/langfuse" + +repl = { + "WORKDIR_PLACEHOLDER": os.environ["WORKDIR"], + "HOME_PLACEHOLDER": os.environ["HOME"], + "RUN_USER_PLACEHOLDER": f"/run/user/{uid}", + "LITELLM_MASTER_KEY_PLACEHOLDER": os.environ["LITELLM_MASTER_KEY"], + "LITELLM_UI_USERNAME_PLACEHOLDER": os.environ.get("UI_USERNAME") or "admin", + "LITELLM_UI_PASSWORD_PLACEHOLDER": os.environ.get("UI_PASSWORD") or os.environ.get("LITELLM_MASTER_KEY") or "admin", + "POSTGRES_PASSWORD_RAW_PLACEHOLDER": os.environ["POSTGRES_PASSWORD"], + "POSTGRES_PASSWORD_ENCODED_PLACEHOLDER": encoded_pg, + "NEXTAUTH_SECRET_PLACEHOLDER": os.environ["NEXTAUTH_SECRET"], + "NEXTAUTH_URL_PLACEHOLDER": nextauth_url, + "SALT_PLACEHOLDER": os.environ["SALT"], + "ENCRYPTION_KEY_PLACEHOLDER": os.environ["ENCRYPTION_KEY"], + "OLLAMA_API_KEY_PLACEHOLDER": os.environ["OLLAMA_API_KEY"], + "OPENROUTER_API_KEY_PLACEHOLDER": os.environ["OPENROUTER_API_KEY"], + "LANGFUSE_PUBLIC_KEY_PLACEHOLDER": os.environ["LANGFUSE_PUBLIC_KEY"], + "LANGFUSE_SECRET_KEY_PLACEHOLDER": os.environ["LANGFUSE_SECRET_KEY"], + "MINIO_USER_PLACEHOLDER": os.environ["MINIO_ROOT_USER"], + "MINIO_PASSWORD_PLACEHOLDER": os.environ["MINIO_ROOT_PASSWORD"], + "LANGFUSE_INIT_USER_PASSWORD_PLACEHOLDER": os.environ["LANGFUSE_INIT_USER_PASSWORD"], + "REDIS_AUTH_PLACEHOLDER": os.environ["REDIS_AUTH"], + "CLICKHOUSE_PASSWORD_PLACEHOLDER": os.environ["CLICKHOUSE_PASSWORD"], + "PROXY_BASE_URL_PLACEHOLDER": proxy_base_url, + "PUBLIC_BASE_URL_PLACEHOLDER": public_base_url, + "ROUTING_DOMAIN_PLACEHOLDER": os.environ["ROUTING_DOMAIN"], + "POD_NAME_PLACEHOLDER": os.environ["POD_NAME"], + "DATA_ROOT_PLACEHOLDER": os.environ["DATA_ROOT"], + "ROUTER_IMAGE_PLACEHOLDER": os.environ["ROUTER_IMAGE"], + "ROUTER_PORT_PLACEHOLDER": os.environ["ROUTER_PORT"], + "LITELLM_PORT_PLACEHOLDER": os.environ["LITELLM_PORT"], + "LANGFUSE_WEB_PORT_PLACEHOLDER": os.environ["LANGFUSE_WEB_PORT"], + "LANGFUSE_WORKER_PORT_PLACEHOLDER": os.environ["LANGFUSE_WORKER_PORT"], + "POSTGRES_PORT_PLACEHOLDER": os.environ["POSTGRES_PORT"], + "VALKEY_CACHE_PORT_PLACEHOLDER": os.environ["VALKEY_CACHE_PORT"], + "VALKEY_LF_PORT_PLACEHOLDER": os.environ["VALKEY_LF_PORT"], + "CLICKHOUSE_HTTP_PORT_PLACEHOLDER": os.environ["CLICKHOUSE_HTTP_PORT"], + "CLICKHOUSE_TCP_PORT_PLACEHOLDER": os.environ["CLICKHOUSE_TCP_PORT"], + "MINIO_S3_PORT_PLACEHOLDER": os.environ["MINIO_S3_PORT"], + "MINIO_CONSOLE_PORT_PLACEHOLDER": os.environ["MINIO_CONSOLE_PORT"], +} + +templates = sorted(glob.glob(os.path.join(src_dir, "*.pod")) + glob.glob(os.path.join(src_dir, "*.container"))) +if not templates: + sys.stderr.write(f"Error: no quadlet templates found in {src_dir}\n") + sys.exit(1) + +# Remove stale rendered files from previous deploys (e.g. renamed containers) +for stale in glob.glob(os.path.join(out_dir, "*.pod")) + glob.glob(os.path.join(out_dir, "*.container")): + os.unlink(stale) + +for tpl in templates: + with open(tpl, "r", encoding="utf-8") as f: + text = f.read() + for ph, val in repl.items(): + text = text.replace(ph, str(val)) + unresolved = sorted(set(re.findall(r"\b[A-Z0-9_]+_PLACEHOLDER\b", text))) + if unresolved: + sys.stderr.write(f"Error: Unresolved placeholders in {os.path.basename(tpl)}: {', '.join(unresolved)}\n") + sys.exit(1) + out_path = os.path.join(out_dir, os.path.basename(tpl)) + with open(out_path, "w", encoding="utf-8") as f: + f.write(text) + os.chmod(out_path, 0o644) + print(f" ✓ {os.path.basename(tpl)}") +PY + echo "✓ Quadlets rendered to ${QUADLET_DIR}" +} + +# Install quadlets and (re)generate systemd units; start or restart the stack. +deploy_quadlets() { + echo "📋 Rendering quadlet units..." + render_quadlets + if ! systemctl --user daemon-reload; then + echo "❌ Error: systemctl --user daemon-reload failed (no systemd user session?)" >&2 + exit 1 + fi + echo "✓ systemd units regenerated" + if systemctl --user is-active --quiet llm-routing-pod.service; then + echo "🔄 Restarting llm-routing-pod.service..." + systemctl --user restart llm-routing-pod.service + else + echo "🚀 Starting llm-routing-pod.service..." + systemctl --user start llm-routing-pod.service + fi +} + deploy_fresh_pod() { generate_clickhouse_config render_litellm_config render_router_config - render_pod_yaml | podman play kube - + deploy_quadlets setup_minio_buckets verify_stack_health } @@ -766,8 +894,13 @@ if podman pod exists ${POD_NAME} 2>/dev/null; then echo "🚀 Deploying replacement pod from YAML..." deploy_fresh_pod else - echo "🔄 Restarting existing ${POD_NAME} (use --replace or --pull to recreate)..." - podman pod restart ${POD_NAME} + if systemctl --user is-active --quiet llm-routing-pod.service 2>/dev/null; then + echo "🔄 Restarting stack via systemd (llm-routing-pod.service)..." + systemctl --user restart llm-routing-pod.service + else + echo "🔄 Restarting existing ${POD_NAME} (use --replace or --pull to recreate)..." + podman pod restart ${POD_NAME} + fi setup_minio_buckets verify_stack_health From f8fed954408e7e95ed26e507d4fb61b4b54d7063 Mon Sep 17 00:00:00 2001 From: boy Date: Thu, 23 Jul 2026 00:00:01 +0200 Subject: [PATCH 2/8] fix(deploy): harden Quadlet rollout and health recovery --- README.md | 15 ++++-- quadlets/llm-routing-clickhouse.container | 1 + quadlets/llm-routing-langfuse-web.container | 1 + .../llm-routing-langfuse-worker.container | 1 + quadlets/llm-routing-litellm.container | 1 + quadlets/llm-routing-minio.container | 1 + quadlets/llm-routing-postgres.container | 1 + quadlets/llm-routing-router.container | 1 + quadlets/llm-routing-valkey-cache.container | 1 + quadlets/llm-routing-valkey-lf.container | 1 + scripts/README.md | 7 +-- scripts/upgrade-prod.sh | 9 ++-- start-stack.sh | 4 +- tests/test_quadlet_templates.py | 47 +++++++++++++++++++ 14 files changed, 78 insertions(+), 13 deletions(-) create mode 100644 tests/test_quadlet_templates.py diff --git a/README.md b/README.md index 061ec42b..9e944930 100644 --- a/README.md +++ b/README.md @@ -389,15 +389,20 @@ For secure production deployments, the gateway services are configured to run un ### 1. Launching the Stack Run the startup script from the root of the repository: ```bash -./start-stack.sh # Fast restart (preserves container IDs and logs) -./start-stack.sh --replace # Recreate pod from YAML (picks up new ports, - # health probes, env vars, containers — no rebuild) -./start-stack.sh --full-rebuild # Full reset: rebuild image + recreate pod +./start-stack.sh # Restart the systemd-managed Quadlet stack +./start-stack.sh --replace # Render Quadlets, daemon-reload, and recreate the stack + # (picks up ports, probes, env vars, and containers) +./start-stack.sh --full-rebuild # Same as --replace plus rebuild the router image + +# Inspect the generated systemd units and their logs +systemctl --user status llm-routing-pod.service --no-pager +systemctl --user list-units 'llm-routing-*' --no-pager +journalctl --user -u llm-routing-router.service -n 100 --no-pager ``` *Note: If running for the first time, the script will prompt you for your `OpenRouter API Key`, securely saving it inside `.env` with restrictive permissions (`chmod 600`). The script also automatically generates and persists secure random secrets (`LITELLM_MASTER_KEY`, `POSTGRES_PASSWORD`, `NEXTAUTH_SECRET`, `SALT`, `ENCRYPTION_KEY`, and `ROUTER_API_KEY`) to this file on startup if they are missing.* ### 2. Verify Container Status -Check that all **10 containers** inside `prod-router-pod` are up and running: +Check that all **9 application containers** in `prod-router-pod` are up and running (the tenth pod infra container is Podman-managed): ```bash podman pod ps podman ps --pod --filter pod=prod-router-pod diff --git a/quadlets/llm-routing-clickhouse.container b/quadlets/llm-routing-clickhouse.container index 65f0c2f4..8696921f 100644 --- a/quadlets/llm-routing-clickhouse.container +++ b/quadlets/llm-routing-clickhouse.container @@ -18,6 +18,7 @@ Volume=DATA_ROOT_PLACEHOLDER/clickhouse-config/port-override.xml:/etc/clickhouse HealthCmd=clickhouse-client --host 127.0.0.1 --port CLICKHOUSE_TCP_PORT_PLACEHOLDER --user clickhouse --password CLICKHOUSE_PASSWORD_PLACEHOLDER --query "SELECT 1" HealthInterval=15s HealthTimeout=5s +HealthOnFailure=kill HealthRetries=5 HealthStartPeriod=20s diff --git a/quadlets/llm-routing-langfuse-web.container b/quadlets/llm-routing-langfuse-web.container index e9fa0db4..d6f4fb86 100644 --- a/quadlets/llm-routing-langfuse-web.container +++ b/quadlets/llm-routing-langfuse-web.container @@ -42,6 +42,7 @@ Environment=LANGFUSE_LOG_LEVEL=warn HealthCmd=wget -q -O /dev/null http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER/api/public/health HealthInterval=15s HealthTimeout=5s +HealthOnFailure=kill HealthRetries=5 HealthStartPeriod=240s diff --git a/quadlets/llm-routing-langfuse-worker.container b/quadlets/llm-routing-langfuse-worker.container index 1872373c..b4e63724 100644 --- a/quadlets/llm-routing-langfuse-worker.container +++ b/quadlets/llm-routing-langfuse-worker.container @@ -31,6 +31,7 @@ Environment=LANGFUSE_LOG_LEVEL=warn HealthCmd=pgrep node HealthInterval=15s HealthTimeout=5s +HealthOnFailure=kill HealthRetries=5 HealthStartPeriod=240s diff --git a/quadlets/llm-routing-litellm.container b/quadlets/llm-routing-litellm.container index 6ab1b902..aff6e059 100644 --- a/quadlets/llm-routing-litellm.container +++ b/quadlets/llm-routing-litellm.container @@ -35,6 +35,7 @@ Volume=HOME_PLACEHOLDER/.gemini:/config/gemini_auth HealthCmd=python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:LITELLM_PORT_PLACEHOLDER/health/liveness')" HealthInterval=15s HealthTimeout=5s +HealthOnFailure=kill HealthRetries=5 HealthStartPeriod=240s diff --git a/quadlets/llm-routing-minio.container b/quadlets/llm-routing-minio.container index 56f59a0e..4df6c6cb 100644 --- a/quadlets/llm-routing-minio.container +++ b/quadlets/llm-routing-minio.container @@ -18,6 +18,7 @@ Volume=DATA_ROOT_PLACEHOLDER/minio-data:/data HealthCmd=curl -f http://127.0.0.1:MINIO_S3_PORT_PLACEHOLDER/minio/health/live HealthInterval=15s HealthTimeout=5s +HealthOnFailure=kill HealthRetries=5 HealthStartPeriod=10s diff --git a/quadlets/llm-routing-postgres.container b/quadlets/llm-routing-postgres.container index 980a56bc..de7db982 100644 --- a/quadlets/llm-routing-postgres.container +++ b/quadlets/llm-routing-postgres.container @@ -17,6 +17,7 @@ Volume=DATA_ROOT_PLACEHOLDER/postgres-data:/var/lib/postgresql HealthCmd=pg_isready -U postgres -p POSTGRES_PORT_PLACEHOLDER HealthInterval=10s HealthTimeout=5s +HealthOnFailure=kill HealthRetries=5 HealthStartPeriod=15s diff --git a/quadlets/llm-routing-router.container b/quadlets/llm-routing-router.container index 3c39430b..16a0d9e6 100644 --- a/quadlets/llm-routing-router.container +++ b/quadlets/llm-routing-router.container @@ -44,6 +44,7 @@ Volume=WORKDIR_PLACEHOLDER/.env:/config/.env HealthCmd=python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:ROUTER_PORT_PLACEHOLDER/metrics')" HealthInterval=15s HealthTimeout=5s +HealthOnFailure=kill HealthRetries=5 HealthStartPeriod=240s diff --git a/quadlets/llm-routing-valkey-cache.container b/quadlets/llm-routing-valkey-cache.container index 97b497aa..8172afb7 100644 --- a/quadlets/llm-routing-valkey-cache.container +++ b/quadlets/llm-routing-valkey-cache.container @@ -16,6 +16,7 @@ Volume=DATA_ROOT_PLACEHOLDER/valkey-data:/data HealthCmd=valkey-cli -p VALKEY_CACHE_PORT_PLACEHOLDER ping HealthInterval=10s HealthTimeout=2s +HealthOnFailure=kill HealthRetries=5 HealthStartPeriod=5s diff --git a/quadlets/llm-routing-valkey-lf.container b/quadlets/llm-routing-valkey-lf.container index 21f3ccee..afbfe1ee 100644 --- a/quadlets/llm-routing-valkey-lf.container +++ b/quadlets/llm-routing-valkey-lf.container @@ -14,6 +14,7 @@ Volume=DATA_ROOT_PLACEHOLDER/redis-lf-data:/data HealthCmd=valkey-cli -p VALKEY_LF_PORT_PLACEHOLDER -a REDIS_AUTH_PLACEHOLDER ping HealthInterval=10s HealthTimeout=2s +HealthOnFailure=kill HealthRetries=5 HealthStartPeriod=5s diff --git a/scripts/README.md b/scripts/README.md index 089f036d..cbb735cf 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -7,11 +7,12 @@ This directory and the repository root contain various scripts used for stack or ## 1. Stack Orchestration & Backups ### `start-stack.sh` (Root Directory) -Unified startup and credential extraction script for the Podman Kubernetes container stack. +Unified startup and credential extraction script for the systemd Quadlet-managed Podman stack. - **Usage**: - - `./start-stack.sh` (Restart existing pod — fast, preserves logs) - - `./start-stack.sh --replace` (Stop + clean ports + redeploy pod from `pod.yaml`) + - `./start-stack.sh` (Restart the generated `llm-routing-pod.service`) + - `./start-stack.sh --replace` (Stop + clean ports + render/install Quadlets + daemon-reload + recreate stack) - `./start-stack.sh --full-rebuild` (Same as `--replace` + rebuild the triage router image; required for code changes in `router/`) +- Quadlet templates live in `quadlets/`; rendered owner-only units are installed under `~/.config/containers/systemd/llm-routing/`. Use `systemctl --user status llm-routing-pod.service --no-pager` and `journalctl --user -u llm-routing-router.service --no-pager` for lifecycle diagnostics. ### `scripts/backup.sh` Automated database backup script that runs before every stack deployment. Uses `pg_isready` to safely wait for database connections and manages timestamped backups under `backups/`. diff --git a/scripts/upgrade-prod.sh b/scripts/upgrade-prod.sh index fb48eb48..c1b42e78 100755 --- a/scripts/upgrade-prod.sh +++ b/scripts/upgrade-prod.sh @@ -4,7 +4,7 @@ # Flow: # 1. Fetch the latest release tag from GitHub # 2. Check out a clean copy of that tag to a temp dir -# 3. Rsync runtime files (pod.yaml, start-stack.sh, litellm/, router/, scripts/) +# 3. Rsync runtime files (pod.yaml, start-stack.sh, quadlets/, litellm/, router/, scripts/) # into ~/prod/LLM-Routing — data/, backups/, and .env are NEVER touched # 4. Redeploy with start-stack.sh --pull (pulls latest container images) # @@ -67,7 +67,7 @@ echo "📥 Cloning $REPO @ $TAG..." git clone -q --depth 1 --branch "$TAG" "https://github.com/$REPO.git" "$TEMP_DIR" # ── verify the tag has the files we need ── -for f in pod.yaml start-stack.sh litellm/ router/ scripts/; do +for f in pod.yaml start-stack.sh quadlets/ litellm/ router/ scripts/; do if [ ! -e "$TEMP_DIR/$f" ]; then echo "❌ Release $TAG is missing expected file/dir: $f" exit 1 @@ -80,7 +80,7 @@ if $DRY_RUN; then echo "── Dry run: files that would change ──" diff -rq "$TEMP_DIR/pod.yaml" "$PROD_DIR/pod.yaml" 2>/dev/null || echo " pod.yaml differs" diff -rq "$TEMP_DIR/start-stack.sh" "$PROD_DIR/start-stack.sh" 2>/dev/null || echo " start-stack.sh differs" - for dir in litellm router scripts; do + for dir in quadlets litellm router scripts; do diff -rq "$TEMP_DIR/$dir" "$PROD_DIR/$dir" 2>/dev/null || echo " $dir/ differs" done echo "── End dry run ──" @@ -90,7 +90,7 @@ fi # ── confirm ── echo "" echo "⚠️ This will OVERWRITE the following in $PROD_DIR:" -echo " pod.yaml start-stack.sh litellm/ router/ scripts/" +echo " pod.yaml start-stack.sh quadlets/ litellm/ router/ scripts/" echo " .env and data/ are NEVER touched." echo "" # Require interactive confirmation in TTY mode; auto-proceed in non-interactive @@ -141,6 +141,7 @@ fi # handles graceful shutdown (including pre-deploy backup) before redeploy. echo "📋 Syncing runtime files..." # Sync directories with --delete (clean stale files within each dir) +rsync -a --delete "$TEMP_DIR/quadlets/" "$PROD_DIR/quadlets/" rsync -a --delete "$TEMP_DIR/litellm/" "$PROD_DIR/litellm/" rsync -a --delete "$TEMP_DIR/router/" "$PROD_DIR/router/" rsync -a --delete "$TEMP_DIR/scripts/" "$PROD_DIR/scripts/" diff --git a/start-stack.sh b/start-stack.sh index 21d3a0b2..e121a43b 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -769,6 +769,7 @@ render_quadlets() { export MINIO_S3_PORT MINIO_CONSOLE_PORT local src_dir="${WORKDIR}/quadlets" mkdir -p "$QUADLET_DIR" + chmod 700 "$QUADLET_DIR" python3 - "$src_dir" "$QUADLET_DIR" <<'PY' import os, sys, urllib.parse, re, glob uid = os.getuid() @@ -843,7 +844,8 @@ for tpl in templates: out_path = os.path.join(out_dir, os.path.basename(tpl)) with open(out_path, "w", encoding="utf-8") as f: f.write(text) - os.chmod(out_path, 0o644) + # Rendered units include credentials; systemd user generator can read owner-only files. + os.chmod(out_path, 0o600) print(f" ✓ {os.path.basename(tpl)}") PY echo "✓ Quadlets rendered to ${QUADLET_DIR}" diff --git a/tests/test_quadlet_templates.py b/tests/test_quadlet_templates.py new file mode 100644 index 00000000..821f2cea --- /dev/null +++ b/tests/test_quadlet_templates.py @@ -0,0 +1,47 @@ +"""Static deployment-contract tests for the systemd Quadlet templates.""" +from pathlib import Path +import re + + +ROOT = Path(__file__).resolve().parent.parent +QUADLETS = ROOT / "quadlets" + + +def test_quadlet_inventory_and_pod_membership(): + containers = sorted(QUADLETS.glob("*.container")) + assert len(containers) == 9 + assert (QUADLETS / "llm-routing.pod").is_file() + for container in containers: + text = container.read_text() + assert "Pod=llm-routing.pod" in text, container.name + assert "ContainerName=POD_NAME_PLACEHOLDER-" in text, container.name + + +def test_liveness_healthchecks_restart_failed_containers(): + for container in sorted(QUADLETS.glob("*.container")): + text = container.read_text() + assert "HealthCmd=" in text, container.name + # Podman kills an unhealthy container; systemd Restart=always then replaces it. + assert "HealthOnFailure=kill" in text, container.name + assert "Restart=always" in text, container.name + + +def test_quadlet_templates_remain_env_rendered_and_secret_free(): + for template in [*QUADLETS.glob("*.container"), QUADLETS / "llm-routing.pod"]: + text = template.read_text() + assert "_PLACEHOLDER" in text, template.name + assert not re.search(r"sk-(?:or|lf|lit)-[A-Za-z0-9_-]{12,}", text), template.name + + +def test_upgrade_syncs_quadlets_before_quadlet_start_stack(): + script = (ROOT / "scripts" / "upgrade-prod.sh").read_text() + assert "quadlets/" in script + assert 'rsync -a --delete "$TEMP_DIR/quadlets/" "$PROD_DIR/quadlets/"' in script + assert "for f in pod.yaml start-stack.sh quadlets/" in script + + +def test_rendered_quadlets_are_owner_only(): + script = (ROOT / "start-stack.sh").read_text() + assert 'chmod 700 "$QUADLET_DIR"' in script + assert "os.chmod(out_path, 0o600)" in script + assert "systemctl --user daemon-reload" in script From e046088b409a263b65961614e07a99af3a34a2e5 Mon Sep 17 00:00:00 2001 From: boy Date: Wed, 22 Jul 2026 17:28:22 +0200 Subject: [PATCH 3/8] fix(routing): implement pure subdomain routing for LiteLLM, Langfuse, and Llama --- .env.dev | 5 +++-- router/main.py | 22 ++++++++-------------- start-stack.sh | 19 +++++++++++++++---- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/.env.dev b/.env.dev index 9dc4b1d0..c36b1736 100644 --- a/.env.dev +++ b/.env.dev @@ -6,8 +6,9 @@ POD_NAME="dev-router-pod" # Public URLs BASEURL="dev.vendeuvre.lan" -BASE_URL="dev.vendeuvre.lan" -PUBLIC_BASE_URL="https://dev.vendeuvre.lan/llm-routing" +PUBLIC_BASE_URL="https://dev.vendeuvre.lan" +PROXY_BASE_URL="https://litellm.dev.vendeuvre.lan" +NEXTAUTH_URL="https://langfuse.dev.vendeuvre.lan" # Dev port assignments (+10 offset from production) ROUTER_PORT="5010" diff --git a/router/main.py b/router/main.py index 567f6f09..d2c88638 100644 --- a/router/main.py +++ b/router/main.py @@ -3546,21 +3546,15 @@ def resolve_external_urls(request: Request) -> tuple[str, str, str]: is_valid_external = external_host == domain or external_host.endswith("." + domain) is_valid_base = request.base_url.hostname == domain or (request.base_url.hostname or "").endswith("." + domain) - if is_valid_external: - # Centralized base URL path under subdomain/reverse proxy + if is_valid_external or is_valid_base: + host_val = external_host if is_valid_external else (request.base_url.hostname or "vendeuvre.lan") + host_base = host_val.replace("dashboard.", "") + if host_base.startswith("litellm.") or host_base.startswith("langfuse.") or host_base.startswith("llama."): + host_base = re.sub(r"^(litellm|langfuse|llama)\.", "", host_base) return ( - f"{external_scheme}://{external_netloc}/llm-routing/langfuse", - f"{external_scheme}://{external_netloc}/llm-routing/litellm/ui", - f"{external_scheme}://{external_netloc}/llm-routing/llama/" - ) - elif is_valid_base: - parsed_netloc = urlparse(f"{external_scheme}://{request.url.netloc}") - netloc = request.url.netloc if parsed_netloc.hostname else "localhost" - base = f"{external_scheme}://{netloc}" - return ( - f"{base}/llm-routing/langfuse", - f"{base}/llm-routing/litellm/ui", - f"{base}/llm-routing/llama/" + f"{external_scheme}://langfuse.{host_base}", + f"{external_scheme}://litellm.{host_base}/ui/", + f"{external_scheme}://llama.{host_base}/" ) else: # Local development fallback: derive schemes, ports, and paths dynamically from configuration constants diff --git a/start-stack.sh b/start-stack.sh index e121a43b..a04d6c7d 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -708,13 +708,24 @@ text = text.replace("MINIO_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["MINIO_ text = text.replace("LANGFUSE_INIT_USER_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["LANGFUSE_INIT_USER_PASSWORD"])) text = text.replace("REDIS_AUTH_PLACEHOLDER", yaml_scalar(os.environ["REDIS_AUTH"])) text = text.replace("CLICKHOUSE_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["CLICKHOUSE_PASSWORD"])) -# Derive PROXY_BASE_URL from PUBLIC_BASE_URL +# Derive PROXY_BASE_URL and NEXTAUTH_URL (favoring explicit env or subdomain formatting) public_base_url = os.environ["PUBLIC_BASE_URL"].rstrip("/") -proxy_base_url = f"{public_base_url}/litellm" +parsed_pub = urllib.parse.urlparse(public_base_url) +scheme = parsed_pub.scheme or "https" +host = parsed_pub.netloc or parsed_pub.path.split("/")[0] or "vendeuvre.lan" + +if "PROXY_BASE_URL" in os.environ: + proxy_base_url = os.environ["PROXY_BASE_URL"] +else: + proxy_base_url = f"{scheme}://litellm.{host}" + +if "NEXTAUTH_URL" in os.environ: + nextauth_url = os.environ["NEXTAUTH_URL"] +else: + nextauth_url = f"{scheme}://langfuse.{host}" + text = text.replace("PROXY_BASE_URL_PLACEHOLDER", yaml_scalar(proxy_base_url)) text = text.replace("PUBLIC_BASE_URL_PLACEHOLDER", yaml_scalar(os.environ["PUBLIC_BASE_URL"])) -# Derive NEXTAUTH_URL from PUBLIC_BASE_URL (Langfuse needs the public URL for OAuth redirects) -nextauth_url = f"{public_base_url}/langfuse" text = text.replace("NEXTAUTH_URL_PLACEHOLDER", yaml_scalar(nextauth_url)) text = text.replace("ROUTING_DOMAIN_PLACEHOLDER", yaml_scalar(os.environ["ROUTING_DOMAIN"])) # Raw replacements (no quoting — used for pod name, data root, and integer port values) From f8a4a535af8764371af743360684c5332ce82b80 Mon Sep 17 00:00:00 2001 From: boy Date: Thu, 23 Jul 2026 00:03:17 +0200 Subject: [PATCH 4/8] fix(routing): address subdomain URL review findings --- router/main.py | 25 ++++++++++++++++------ router/tests/test_resolve_external_urls.py | 16 +++++++------- start-stack.sh | 10 +++++---- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/router/main.py b/router/main.py index d2c88638..a66136f9 100644 --- a/router/main.py +++ b/router/main.py @@ -3547,14 +3547,25 @@ def resolve_external_urls(request: Request) -> tuple[str, str, str]: is_valid_base = request.base_url.hostname == domain or (request.base_url.hostname or "").endswith("." + domain) if is_valid_external or is_valid_base: - host_val = external_host if is_valid_external else (request.base_url.hostname or "vendeuvre.lan") - host_base = host_val.replace("dashboard.", "") - if host_base.startswith("litellm.") or host_base.startswith("langfuse.") or host_base.startswith("llama."): - host_base = re.sub(r"^(litellm|langfuse|llama)\.", "", host_base) + # Use configured routing domain if a proxy supplies no request hostname. + # Preserve an explicit public port from the selected netloc so dashboard + # links remain valid behind non-standard TLS listeners. + host_val = external_host if is_valid_external else (request.base_url.hostname or domain) + netloc_val = external_netloc if is_valid_external else (request.base_url.netloc or host_val) + parsed_public = urlparse(f"{external_scheme}://{netloc_val}") + try: + port_suffix = f":{parsed_public.port}" if parsed_public.port else "" + except ValueError: + logger.warning("Invalid public port in netloc %r; omitting port", netloc_val) + port_suffix = "" + + host_base = re.sub(r"^dashboard\.", "", host_val) + host_base = re.sub(r"^(?:litellm|langfuse|llama)\.", "", host_base) + service_netloc = f"{host_base}{port_suffix}" return ( - f"{external_scheme}://langfuse.{host_base}", - f"{external_scheme}://litellm.{host_base}/ui/", - f"{external_scheme}://llama.{host_base}/" + f"{external_scheme}://langfuse.{service_netloc}", + f"{external_scheme}://litellm.{service_netloc}/ui/", + f"{external_scheme}://llama.{service_netloc}/" ) else: # Local development fallback: derive schemes, ports, and paths dynamically from configuration constants diff --git a/router/tests/test_resolve_external_urls.py b/router/tests/test_resolve_external_urls.py index 8cc79f0d..9e45d1a2 100644 --- a/router/tests/test_resolve_external_urls.py +++ b/router/tests/test_resolve_external_urls.py @@ -45,9 +45,9 @@ def test_resolve_with_public_base_url_vaild_domain(): req = MockRequest() lf, ll, lm = resolve_external_urls(req) - assert lf == "https://app.vendeuvre.lan/llm-routing/langfuse" - assert ll == "https://app.vendeuvre.lan/llm-routing/litellm/ui" - assert lm == "https://app.vendeuvre.lan/llm-routing/llama/" + assert lf == "https://langfuse.app.vendeuvre.lan" + assert ll == "https://litellm.app.vendeuvre.lan/ui/" + assert lm == "https://llama.app.vendeuvre.lan/" def test_resolve_with_valid_base_request(): @@ -64,11 +64,11 @@ def test_resolve_with_valid_base_request(): lf, ll, lm = resolve_external_urls(req) - # Netloc should use [::1]:8000 because request base host is valid, - # and netloc was validated successfully via urlparse. - assert lf == "http://[::1]:8000/llm-routing/langfuse" - assert ll == "http://[::1]:8000/llm-routing/litellm/ui" - assert lm == "http://[::1]:8000/llm-routing/llama/" + # Valid request host is converted to service subdomains while preserving + # its explicit port, rather than leaking the configured IPv6 fallback. + assert lf == "http://langfuse.sub.vendeuvre.lan:8000" + assert ll == "http://litellm.sub.vendeuvre.lan:8000/ui/" + assert lm == "http://llama.sub.vendeuvre.lan:8000/" def test_local_fallback_ipv6(): diff --git a/start-stack.sh b/start-stack.sh index a04d6c7d..f828a37c 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -787,11 +787,13 @@ uid = os.getuid() src_dir, out_dir = sys.argv[1], sys.argv[2] encoded_pg = urllib.parse.quote(os.environ['POSTGRES_PASSWORD'], safe="") -# Derive PROXY_BASE_URL / NEXTAUTH_URL exactly as render_pod_yaml does -# (path-based on master; subdomain switch lands with the routing branch) +# Derive PROXY_BASE_URL / NEXTAUTH_URL exactly as render_pod_yaml does. public_base_url = os.environ["PUBLIC_BASE_URL"].rstrip("/") -proxy_base_url = f"{public_base_url}/litellm" -nextauth_url = f"{public_base_url}/langfuse" +parsed_pub = urllib.parse.urlparse(public_base_url) +scheme = parsed_pub.scheme or "https" +host = parsed_pub.netloc or parsed_pub.path.split("/")[0] or os.environ["ROUTING_DOMAIN"] +proxy_base_url = os.environ.get("PROXY_BASE_URL") or f"{scheme}://litellm.{host}" +nextauth_url = os.environ.get("NEXTAUTH_URL") or f"{scheme}://langfuse.{host}" repl = { "WORKDIR_PLACEHOLDER": os.environ["WORKDIR"], From 83fd42244b883d2d2fc010ad8d57e16ddce39550 Mon Sep 17 00:00:00 2001 From: boy Date: Thu, 23 Jul 2026 00:41:32 +0200 Subject: [PATCH 5/8] fix(deploy): harden Quadlet review findings --- README.md | 9 +- router/tests/test_resolve_external_urls.py | 18 +++ .../verify_canonical_endpoints.py | 24 ++-- start-stack.sh | 123 ++++++++++++------ tests/test_quadlet_templates.py | 10 ++ 5 files changed, 133 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 9e944930..7b69e77f 100644 --- a/README.md +++ b/README.md @@ -845,14 +845,15 @@ Tests cover: | Section | Endpoints | |---------|-----------| | Router API | `/v1/models`, `/metrics`, `/dashboard`, `/api/dashboard-stats`, `/visualizer` | -| LiteLLM | `/health/liveness`, `/health/readiness`, `/v1/models`, `/llm-routing/litellm/ui/` | -| Langfuse | `/api/public/health`, `/` (web UI) | +| LiteLLM | Local `/health/liveness`, `/health/readiness`, `/v1/models`; canonical `https://litellm./ui/` | +| Langfuse | Local `/api/public/health`, `/`; canonical `https://langfuse./` | +| llama.cpp | Canonical `https://llama./health` | | Infrastructure | MinIO `/minio/health/live`, ClickHouse `/ping` | | E2E chat | 3 completions through triage router | | LiteLLM direct | 1 completion directly to LiteLLM | -| Canonical URLs | 6 GET + 1 POST through public HTTPS (graceful DNS skip) | +| Canonical URLs | 7 GET + 1 POST through public HTTPS (graceful DNS skip) | -Requires `PUBLIC_BASE_URL` in `.env` for canonical URL tests. Dev `.env.dev` already has it; prod `.env` should include `PUBLIC_BASE_URL="https://x570.vendeuvre.lan/llm-routing"`. +Requires `PUBLIC_BASE_URL` in `.env` for canonical URL tests. The router remains under its configured path (for example `https://x570.vendeuvre.lan/llm-routing`), while the verifier derives service URLs from its host: `https://litellm./ui/`, `https://langfuse./`, and `https://llama./health`. Dev `.env.dev` already has it; prod `.env` should include `PUBLIC_BASE_URL="https://x570.vendeuvre.lan/llm-routing"`. ## 10. Performance Benchmarks diff --git a/router/tests/test_resolve_external_urls.py b/router/tests/test_resolve_external_urls.py index 9e45d1a2..8db6d5fb 100644 --- a/router/tests/test_resolve_external_urls.py +++ b/router/tests/test_resolve_external_urls.py @@ -71,6 +71,24 @@ def test_resolve_with_valid_base_request(): assert lm == "http://llama.sub.vendeuvre.lan:8000/" +def test_resolve_with_valid_base_request_without_port(): + os.environ["ROUTING_DOMAIN"] = "vendeuvre.lan" + os.environ["PUBLIC_BASE_URL"] = "http://[::1]:5000" + + req = MockRequest( + base_host="sub.vendeuvre.lan", + base_netloc="sub.vendeuvre.lan", + url_scheme="https", + url_netloc="[::1]:8000", + ) + + lf, ll, lm = resolve_external_urls(req) + + assert lf == "http://langfuse.sub.vendeuvre.lan" + assert ll == "http://litellm.sub.vendeuvre.lan/ui/" + assert lm == "http://llama.sub.vendeuvre.lan/" + + def test_local_fallback_ipv6(): # Force fallback to local development logic (domain check fails) os.environ["ROUTING_DOMAIN"] = "vendeuvre.lan" diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index ff2c2497..601aef47 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -14,6 +14,7 @@ import time import datetime import argparse +import urllib.parse import httpx from pathlib import Path @@ -597,18 +598,25 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int, int]: print(f"\n── Canonical URLs ({public}) ──") + parsed_public = urllib.parse.urlparse(public if "://" in public else f"https://{public}") + scheme = parsed_public.scheme if parsed_public.scheme in ("http", "https") else "https" + host = parsed_public.netloc or parsed_public.path.split("/", 1)[0] + if not host: + print(" ⚠ Canonical URLs — SKIPPED (PUBLIC_BASE_URL has no host)") + return 0, 0, 0 + router_base = public.rstrip("/") endpoints = [ - ("/v1/models", "router models"), - ("/dashboard", "dashboard"), - ("/metrics", "metrics"), - ("/visualizer", "visualizer"), - ("/litellm/ui/", "LiteLLM admin UI"), - ("/langfuse", "Langfuse web UI"), + (f"{router_base}/v1/models", "router models"), + (f"{router_base}/dashboard", "dashboard"), + (f"{router_base}/metrics", "metrics"), + (f"{router_base}/visualizer", "visualizer"), + (f"{scheme}://litellm.{host}/ui/", "LiteLLM admin UI"), + (f"{scheme}://langfuse.{host}/", "Langfuse web UI"), + (f"{scheme}://llama.{host}/health", "llama.cpp health"), ] - for path, label in endpoints: + for url, label in endpoints: total += 1 - url = f"{public}{path}" try: r = httpx.get(url, timeout=15, follow_redirects=True, headers={"Authorization": f"Bearer {cfg['router_api_key']}"}) diff --git a/start-stack.sh b/start-stack.sh index f828a37c..2f9ea34e 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -530,21 +530,50 @@ verify_stack_health() { echo " ⚠️ Triage router pipeline not responding after 120s — dashboard may still work" return 1 } -# ── Safe pod teardown ── +# ── Stack ownership and teardown ── +# Quadlet-managed pods carry PODMAN_SYSTEMD_UNIT on their infra container. +# Consult that metadata rather than inferring ownership from active state: a +# stopped or failed generated unit is still Quadlet-owned and must be reconciled +# through systemd before a replacement pod is created. +stack_ownership() { + local infra_unit + if podman pod exists "${POD_NAME}" 2>/dev/null; then + infra_unit=$(podman pod inspect "${POD_NAME}" --format '{{.InfraContainerID}}' 2>/dev/null | xargs -r podman inspect --format '{{ index .Config.Labels "PODMAN_SYSTEMD_UNIT" }}' 2>/dev/null || true) + if [[ "$infra_unit" == "llm-routing-pod.service" ]]; then + printf 'quadlet\n' + else + printf 'legacy\n' + fi + elif systemctl --user show llm-routing-pod.service -p LoadState --value 2>/dev/null | grep -qxv 'not-found'; then + printf 'quadlet\n' + else + printf 'absent\n' + fi +} + +require_user_systemd() { + if ! systemctl --user show-environment >/dev/null 2>&1; then + echo "❌ Error: the Quadlet deployment requires a reachable systemd --user manager." >&2 + echo " Log in through a user session or restore the user D-Bus before deploying." >&2 + return 1 + fi +} + # Graceful stop (SIGTERM with 30s timeout) lets ClickHouse/Postgres flush, # then force-remove if needed. Avoids data corruption from SIGKILL. safe_pod_teardown() { - # If the stack is systemd-managed (quadlets), stop via systemd so the - # manager state stays consistent; then remove any leftover pod. - if systemctl --user is-active --quiet llm-routing-pod.service 2>/dev/null; then - echo "🛑 Stopping systemd-managed stack (llm-routing-pod.service)..." - systemctl --user stop llm-routing-pod.service || true - podman pod rm -f ${POD_NAME} 2>/dev/null || true + local ownership + ownership=$(stack_ownership) + if [[ "$ownership" == "quadlet" ]]; then + echo "🛑 Reconciling Quadlet-owned stack (unit may be active, inactive, or failed)..." + systemctl --user stop llm-routing-pod.service 2>/dev/null || true + systemctl --user reset-failed llm-routing-pod.service 2>/dev/null || true + podman pod rm -f "${POD_NAME}" 2>/dev/null || true cleanup_zombie_ports - echo "✓ Stack stopped, ports cleaned" + echo "✓ Quadlet stack stopped, state reconciled, ports cleaned" return fi - if podman pod exists ${POD_NAME} 2>/dev/null; then + if [[ "$ownership" == "legacy" ]]; then echo "🛑 Gracefully stopping pod (SIGTERM, 30s timeout)..." podman pod stop -t 30 ${POD_NAME} 2>/dev/null || true # podman pod exists returns 0 for stopped pods too — check running state @@ -560,6 +589,25 @@ safe_pod_teardown() { fi } +# Derive service URLs once so legacy pod rendering and Quadlet rendering cannot drift. +derive_external_service_urls() { + local values + values=$(python3 -c ' +import os +from urllib.parse import urlparse +public = (os.environ.get("PUBLIC_BASE_URL") or "").rstrip("/") +routing_domain = os.environ.get("ROUTING_DOMAIN") or "vendeuvre.lan" +parsed = urlparse(public if "://" in public else f"https://{public}") +scheme = parsed.scheme if parsed.scheme in {"http", "https"} else "https" +host = parsed.netloc or parsed.path.split("/", 1)[0] or routing_domain +print(os.environ.get("PROXY_BASE_URL") or f"{scheme}://litellm.{host}") +print(os.environ.get("NEXTAUTH_URL") or f"{scheme}://langfuse.{host}") +') || return 1 + PROXY_BASE_URL_DERIVED=${values%%$'\n'*} + NEXTAUTH_URL_DERIVED=${values#*$'\n'} + export PROXY_BASE_URL_DERIVED NEXTAUTH_URL_DERIVED +} + # Pre-deploy database backup (runs before any pod modification) # Skip if pod doesn't exist (e.g., after manual cleanup) if podman pod exists ${POD_NAME} 2>/dev/null; then @@ -633,6 +681,7 @@ render_pod_yaml() { export LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT export CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT export MINIO_S3_PORT MINIO_CONSOLE_PORT + derive_external_service_urls python3 - "$WORKDIR/pod.yaml" <<'PY' import os, sys, urllib.parse, json uid = os.getuid() @@ -708,21 +757,9 @@ text = text.replace("MINIO_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["MINIO_ text = text.replace("LANGFUSE_INIT_USER_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["LANGFUSE_INIT_USER_PASSWORD"])) text = text.replace("REDIS_AUTH_PLACEHOLDER", yaml_scalar(os.environ["REDIS_AUTH"])) text = text.replace("CLICKHOUSE_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["CLICKHOUSE_PASSWORD"])) -# Derive PROXY_BASE_URL and NEXTAUTH_URL (favoring explicit env or subdomain formatting) -public_base_url = os.environ["PUBLIC_BASE_URL"].rstrip("/") -parsed_pub = urllib.parse.urlparse(public_base_url) -scheme = parsed_pub.scheme or "https" -host = parsed_pub.netloc or parsed_pub.path.split("/")[0] or "vendeuvre.lan" - -if "PROXY_BASE_URL" in os.environ: - proxy_base_url = os.environ["PROXY_BASE_URL"] -else: - proxy_base_url = f"{scheme}://litellm.{host}" - -if "NEXTAUTH_URL" in os.environ: - nextauth_url = os.environ["NEXTAUTH_URL"] -else: - nextauth_url = f"{scheme}://langfuse.{host}" +# Service URLs are derived once by the shell wrapper for both rendering paths. +proxy_base_url = os.environ["PROXY_BASE_URL_DERIVED"] +nextauth_url = os.environ["NEXTAUTH_URL_DERIVED"] text = text.replace("PROXY_BASE_URL_PLACEHOLDER", yaml_scalar(proxy_base_url)) text = text.replace("PUBLIC_BASE_URL_PLACEHOLDER", yaml_scalar(os.environ["PUBLIC_BASE_URL"])) @@ -779,6 +816,7 @@ render_quadlets() { export CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT export MINIO_S3_PORT MINIO_CONSOLE_PORT local src_dir="${WORKDIR}/quadlets" + derive_external_service_urls mkdir -p "$QUADLET_DIR" chmod 700 "$QUADLET_DIR" python3 - "$src_dir" "$QUADLET_DIR" <<'PY' @@ -787,13 +825,9 @@ uid = os.getuid() src_dir, out_dir = sys.argv[1], sys.argv[2] encoded_pg = urllib.parse.quote(os.environ['POSTGRES_PASSWORD'], safe="") -# Derive PROXY_BASE_URL / NEXTAUTH_URL exactly as render_pod_yaml does. -public_base_url = os.environ["PUBLIC_BASE_URL"].rstrip("/") -parsed_pub = urllib.parse.urlparse(public_base_url) -scheme = parsed_pub.scheme or "https" -host = parsed_pub.netloc or parsed_pub.path.split("/")[0] or os.environ["ROUTING_DOMAIN"] -proxy_base_url = os.environ.get("PROXY_BASE_URL") or f"{scheme}://litellm.{host}" -nextauth_url = os.environ.get("NEXTAUTH_URL") or f"{scheme}://langfuse.{host}" +# Derived by derive_external_service_urls(), shared with render_pod_yaml. +proxy_base_url = os.environ["PROXY_BASE_URL_DERIVED"] +nextauth_url = os.environ["NEXTAUTH_URL_DERIVED"] repl = { "WORKDIR_PLACEHOLDER": os.environ["WORKDIR"], @@ -818,7 +852,7 @@ repl = { "REDIS_AUTH_PLACEHOLDER": os.environ["REDIS_AUTH"], "CLICKHOUSE_PASSWORD_PLACEHOLDER": os.environ["CLICKHOUSE_PASSWORD"], "PROXY_BASE_URL_PLACEHOLDER": proxy_base_url, - "PUBLIC_BASE_URL_PLACEHOLDER": public_base_url, + "PUBLIC_BASE_URL_PLACEHOLDER": os.environ["PUBLIC_BASE_URL"].rstrip("/"), "ROUTING_DOMAIN_PLACEHOLDER": os.environ["ROUTING_DOMAIN"], "POD_NAME_PLACEHOLDER": os.environ["POD_NAME"], "DATA_ROOT_PLACEHOLDER": os.environ["DATA_ROOT"], @@ -854,6 +888,13 @@ for tpl in templates: if unresolved: sys.stderr.write(f"Error: Unresolved placeholders in {os.path.basename(tpl)}: {', '.join(unresolved)}\n") sys.exit(1) + # Quadlet Environment= values use systemd's command-line parser. Quote each + # complete KEY=value assignment after substitution so spaces, quotes, and + # backslashes in configurable values remain one environment value. + def quote_environment(match): + value = match.group(1).replace("\\", "\\\\").replace('"', '\\"') + return f'Environment="{value}"' + text = re.sub(r"(?m)^Environment=(.*)$", quote_environment, text) out_path = os.path.join(out_dir, os.path.basename(tpl)) with open(out_path, "w", encoding="utf-8") as f: f.write(text) @@ -866,6 +907,7 @@ PY # Install quadlets and (re)generate systemd units; start or restart the stack. deploy_quadlets() { + require_user_systemd echo "📋 Rendering quadlet units..." render_quadlets if ! systemctl --user daemon-reload; then @@ -891,7 +933,8 @@ deploy_fresh_pod() { verify_stack_health } -if podman pod exists ${POD_NAME} 2>/dev/null; then +STACK_OWNERSHIP=$(stack_ownership) +if [[ "$STACK_OWNERSHIP" != "absent" ]]; then if $FULL_REBUILD; then echo "🔨 Building custom local triage router image..." podman build -t "${ROUTER_IMAGE}" -f router/Dockerfile router @@ -909,12 +952,14 @@ if podman pod exists ${POD_NAME} 2>/dev/null; then echo "🚀 Deploying replacement pod from YAML..." deploy_fresh_pod else - if systemctl --user is-active --quiet llm-routing-pod.service 2>/dev/null; then - echo "🔄 Restarting stack via systemd (llm-routing-pod.service)..." + if [[ "$STACK_OWNERSHIP" == "quadlet" ]]; then + require_user_systemd + echo "🔄 Restarting Quadlet-owned stack via systemd..." + systemctl --user reset-failed llm-routing-pod.service 2>/dev/null || true systemctl --user restart llm-routing-pod.service else - echo "🔄 Restarting existing ${POD_NAME} (use --replace or --pull to recreate)..." - podman pod restart ${POD_NAME} + echo "🔄 Restarting legacy ${POD_NAME} (use --replace or --pull to migrate)..." + podman pod restart "${POD_NAME}" fi setup_minio_buckets verify_stack_health @@ -926,7 +971,7 @@ if podman pod exists ${POD_NAME} 2>/dev/null; then echo " (local) : ${LOCAL_BASE_URL}/v1" echo "⚙️ Dashboard URL : ${PUBLIC_BASE_URL}/dashboard" echo "🔑 Gateway API Key : gateway-pass" - echo "🔐 LiteLLM Admin UI: ${PUBLIC_BASE_URL}/litellm/ui" + echo "🔐 LiteLLM Admin UI: ${PROXY_BASE_URL_DERIVED}/ui/" echo " Username: admin | Password: $LITELLM_MASTER_KEY" echo "=========================================================================" exit 0 @@ -958,6 +1003,6 @@ echo "📍 Entry endpoint : ${PUBLIC_BASE_URL}/v1" echo " (local) : ${LOCAL_BASE_URL}/v1" echo "⚙️ Dashboard URL : ${PUBLIC_BASE_URL}/dashboard" echo "🔑 Gateway API Key : gateway-pass" -echo "🔐 LiteLLM Admin UI: ${PUBLIC_BASE_URL}/litellm/ui" +echo "🔐 LiteLLM Admin UI: ${PROXY_BASE_URL_DERIVED}/ui/" echo " Username: admin | Password: $LITELLM_MASTER_KEY" echo "=========================================================================" diff --git a/tests/test_quadlet_templates.py b/tests/test_quadlet_templates.py index 821f2cea..991a9309 100644 --- a/tests/test_quadlet_templates.py +++ b/tests/test_quadlet_templates.py @@ -45,3 +45,13 @@ def test_rendered_quadlets_are_owner_only(): assert 'chmod 700 "$QUADLET_DIR"' in script assert "os.chmod(out_path, 0o600)" in script assert "systemctl --user daemon-reload" in script + assert "stack_ownership()" in script + assert "PODMAN_SYSTEMD_UNIT" in script + assert "require_user_systemd()" in script + + +def test_quadlet_renderer_quotes_environment_values_for_systemd(): + script = (ROOT / "start-stack.sh").read_text() + assert 'text = re.sub(r"(?m)^Environment=(.*)$", quote_environment, text)' in script + assert "def quote_environment(match):" in script + assert "return f'Environment=\"{value}\"'" in script From f65b89eb2ceb954fe567938de8bba60299fef6b0 Mon Sep 17 00:00:00 2001 From: boy Date: Thu, 23 Jul 2026 01:06:49 +0200 Subject: [PATCH 6/8] fix(deploy): harden Quadlet review findings and dev safety net --- .env.dev | 7 +- README.md | 2 +- .../verify_canonical_endpoints.py | 6 +- start-stack.sh | 103 +++++++++++------- tests/test_quadlet_templates.py | 16 ++- 5 files changed, 89 insertions(+), 45 deletions(-) diff --git a/.env.dev b/.env.dev index c36b1736..4565148a 100644 --- a/.env.dev +++ b/.env.dev @@ -27,6 +27,7 @@ MINIO_CONSOLE_PORT="9011" # Dev uses locally-built router image (for testing code changes) ROUTER_IMAGE="localhost/llm-routing-dev:latest" -# Shared infrastructure (same llama classifier across dev & prod) -LLAMA_CLASSIFIER_URL="https://x570.vendeuvre.lan/llm-routing/llama/v1" -LLAMA_SERVER_URL="https://x570.vendeuvre.lan/llm-routing/llama" +# Shared llama.cpp infrastructure is host-networked, so the dev safety net +# reaches its local listener directly rather than any TLS-terminated hostname. +LLAMA_CLASSIFIER_URL="http://127.0.0.1:8083/v1" +LLAMA_SERVER_URL="http://127.0.0.1:8083" diff --git a/README.md b/README.md index 7b69e77f..a1324e18 100644 --- a/README.md +++ b/README.md @@ -853,7 +853,7 @@ Tests cover: | LiteLLM direct | 1 completion directly to LiteLLM | | Canonical URLs | 7 GET + 1 POST through public HTTPS (graceful DNS skip) | -Requires `PUBLIC_BASE_URL` in `.env` for canonical URL tests. The router remains under its configured path (for example `https://x570.vendeuvre.lan/llm-routing`), while the verifier derives service URLs from its host: `https://litellm./ui/`, `https://langfuse./`, and `https://llama./health`. Dev `.env.dev` already has it; prod `.env` should include `PUBLIC_BASE_URL="https://x570.vendeuvre.lan/llm-routing"`. +Requires `PUBLIC_BASE_URL` in `.env` for canonical URL tests. The router remains under its configured path (for example `https://x570.vendeuvre.lan/llm-routing`), while the verifier derives service URLs from its host: `https://litellm./ui/`, `https://langfuse./`, and `https://llama./health`. Dev `.env.dev` already has it; prod `.env` should include `PUBLIC_BASE_URL="https://x570.vendeuvre.lan/llm-routing"`. The dev local-model safety net uses the host-networked local listener: `LLAMA_CLASSIFIER_URL=http://127.0.0.1:8083/v1` and `LLAMA_SERVER_URL=http://127.0.0.1:8083`; it must not depend on TLS-terminated dev or production hostnames. ## 10. Performance Benchmarks diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index 601aef47..44d71239 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -604,7 +604,9 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int, int]: if not host: print(" ⚠ Canonical URLs — SKIPPED (PUBLIC_BASE_URL has no host)") return 0, 0, 0 - router_base = public.rstrip("/") + # Preserve a configured router path (e.g. /llm-routing) while normalizing + # scheme-less PUBLIC_BASE_URL values into valid absolute URLs. + router_base = f"{scheme}://{host}{parsed_public.path.rstrip('/')}" endpoints = [ (f"{router_base}/v1/models", "router models"), (f"{router_base}/dashboard", "dashboard"), @@ -631,7 +633,7 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int, int]: # Canonical chat completion (POST through public URL) total += 1 - url = f"{public}/v1/chat/completions" + url = f"{router_base}/v1/chat/completions" try: payload = { "model": "agent-simple-core", diff --git a/start-stack.sh b/start-stack.sh index 2f9ea34e..b6febd66 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -531,6 +531,9 @@ verify_stack_health() { return 1 } # ── Stack ownership and teardown ── +# Keep the generated Quadlet unit name in one place: it is used for ownership +# detection, lifecycle operations, and user-facing diagnostics. +LLM_ROUTING_POD_UNIT="llm-routing-pod.service" # Quadlet-managed pods carry PODMAN_SYSTEMD_UNIT on their infra container. # Consult that metadata rather than inferring ownership from active state: a # stopped or failed generated unit is still Quadlet-owned and must be reconciled @@ -539,12 +542,12 @@ stack_ownership() { local infra_unit if podman pod exists "${POD_NAME}" 2>/dev/null; then infra_unit=$(podman pod inspect "${POD_NAME}" --format '{{.InfraContainerID}}' 2>/dev/null | xargs -r podman inspect --format '{{ index .Config.Labels "PODMAN_SYSTEMD_UNIT" }}' 2>/dev/null || true) - if [[ "$infra_unit" == "llm-routing-pod.service" ]]; then + if [[ "$infra_unit" == "$LLM_ROUTING_POD_UNIT" ]]; then printf 'quadlet\n' else printf 'legacy\n' fi - elif systemctl --user show llm-routing-pod.service -p LoadState --value 2>/dev/null | grep -qxv 'not-found'; then + elif systemctl --user show "$LLM_ROUTING_POD_UNIT" -p LoadState --value 2>/dev/null | grep -qxv 'not-found'; then printf 'quadlet\n' else printf 'absent\n' @@ -566,8 +569,8 @@ safe_pod_teardown() { ownership=$(stack_ownership) if [[ "$ownership" == "quadlet" ]]; then echo "🛑 Reconciling Quadlet-owned stack (unit may be active, inactive, or failed)..." - systemctl --user stop llm-routing-pod.service 2>/dev/null || true - systemctl --user reset-failed llm-routing-pod.service 2>/dev/null || true + systemctl --user stop "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true + systemctl --user reset-failed "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true podman pod rm -f "${POD_NAME}" 2>/dev/null || true cleanup_zombie_ports echo "✓ Quadlet stack stopped, state reconciled, ports cleaned" @@ -820,7 +823,7 @@ render_quadlets() { mkdir -p "$QUADLET_DIR" chmod 700 "$QUADLET_DIR" python3 - "$src_dir" "$QUADLET_DIR" <<'PY' -import os, sys, urllib.parse, re, glob +import os, sys, urllib.parse, re, glob, shutil, tempfile uid = os.getuid() src_dir, out_dir = sys.argv[1], sys.argv[2] @@ -875,32 +878,42 @@ if not templates: sys.stderr.write(f"Error: no quadlet templates found in {src_dir}\n") sys.exit(1) -# Remove stale rendered files from previous deploys (e.g. renamed containers) -for stale in glob.glob(os.path.join(out_dir, "*.pod")) + glob.glob(os.path.join(out_dir, "*.container")): - os.unlink(stale) - -for tpl in templates: - with open(tpl, "r", encoding="utf-8") as f: - text = f.read() - for ph, val in repl.items(): - text = text.replace(ph, str(val)) - unresolved = sorted(set(re.findall(r"\b[A-Z0-9_]+_PLACEHOLDER\b", text))) - if unresolved: - sys.stderr.write(f"Error: Unresolved placeholders in {os.path.basename(tpl)}: {', '.join(unresolved)}\n") - sys.exit(1) - # Quadlet Environment= values use systemd's command-line parser. Quote each - # complete KEY=value assignment after substitution so spaces, quotes, and - # backslashes in configurable values remain one environment value. - def quote_environment(match): - value = match.group(1).replace("\\", "\\\\").replace('"', '\\"') - return f'Environment="{value}"' - text = re.sub(r"(?m)^Environment=(.*)$", quote_environment, text) - out_path = os.path.join(out_dir, os.path.basename(tpl)) - with open(out_path, "w", encoding="utf-8") as f: - f.write(text) - # Rendered units include credentials; systemd user generator can read owner-only files. - os.chmod(out_path, 0o600) - print(f" ✓ {os.path.basename(tpl)}") +staging_dir = tempfile.mkdtemp(prefix=".llm-routing-render-", dir=os.path.dirname(out_dir)) +try: + for tpl in templates: + with open(tpl, "r", encoding="utf-8") as f: + text = f.read() + for ph, val in repl.items(): + text = text.replace(ph, str(val)) + unresolved = sorted(set(re.findall(r"\b[A-Z0-9_]+_PLACEHOLDER\b", text))) + if unresolved: + sys.stderr.write(f"Error: Unresolved placeholders in {os.path.basename(tpl)}: {', '.join(unresolved)}\n") + sys.exit(1) + # Quadlet Environment= values use systemd's command-line parser. Quote each + # complete KEY=value assignment after substitution so spaces, quotes, and + # backslashes in configurable values remain one environment value. + def quote_environment(match): + value = match.group(1).replace("\\", "\\\\").replace('"', '\\"') + return f'Environment="{value}"' + text = re.sub(r"(?m)^Environment=(.*)$", quote_environment, text) + staged_path = os.path.join(staging_dir, os.path.basename(tpl)) + with open(staged_path, "w", encoding="utf-8") as f: + f.write(text) + # Rendered units include credentials; systemd user generator can read owner-only files. + os.chmod(staged_path, 0o600) + + # All templates are now valid. Replace individual files atomically, then + # remove stale units; a failed render above leaves the prior unit set intact. + rendered_names = {os.path.basename(tpl) for tpl in templates} + for name in rendered_names: + os.replace(os.path.join(staging_dir, name), os.path.join(out_dir, name)) + for stale in glob.glob(os.path.join(out_dir, "*.pod")) + glob.glob(os.path.join(out_dir, "*.container")): + if os.path.basename(stale) not in rendered_names: + os.unlink(stale) + for name in sorted(rendered_names): + print(f" ✓ {name}") +finally: + shutil.rmtree(staging_dir, ignore_errors=True) PY echo "✓ Quadlets rendered to ${QUADLET_DIR}" } @@ -915,12 +928,20 @@ deploy_quadlets() { exit 1 fi echo "✓ systemd units regenerated" - if systemctl --user is-active --quiet llm-routing-pod.service; then - echo "🔄 Restarting llm-routing-pod.service..." - systemctl --user restart llm-routing-pod.service + if systemctl --user is-active --quiet "$LLM_ROUTING_POD_UNIT"; then + echo "🔄 Restarting ${LLM_ROUTING_POD_UNIT}..." + if ! systemctl --user restart "$LLM_ROUTING_POD_UNIT"; then + echo "❌ Error: failed to restart ${LLM_ROUTING_POD_UNIT}" >&2 + echo " Hint: run 'systemctl --user status ${LLM_ROUTING_POD_UNIT} --no-pager' to inspect the failure" >&2 + exit 1 + fi else - echo "🚀 Starting llm-routing-pod.service..." - systemctl --user start llm-routing-pod.service + echo "🚀 Starting ${LLM_ROUTING_POD_UNIT}..." + if ! systemctl --user start "$LLM_ROUTING_POD_UNIT"; then + echo "❌ Error: failed to start ${LLM_ROUTING_POD_UNIT}" >&2 + echo " Hint: run 'systemctl --user status ${LLM_ROUTING_POD_UNIT} --no-pager' to inspect the failure" >&2 + exit 1 + fi fi } @@ -955,8 +976,12 @@ if [[ "$STACK_OWNERSHIP" != "absent" ]]; then if [[ "$STACK_OWNERSHIP" == "quadlet" ]]; then require_user_systemd echo "🔄 Restarting Quadlet-owned stack via systemd..." - systemctl --user reset-failed llm-routing-pod.service 2>/dev/null || true - systemctl --user restart llm-routing-pod.service + systemctl --user reset-failed "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true + if ! systemctl --user restart "$LLM_ROUTING_POD_UNIT"; then + echo "❌ Error: failed to restart ${LLM_ROUTING_POD_UNIT}" >&2 + echo " Hint: run 'systemctl --user status ${LLM_ROUTING_POD_UNIT} --no-pager' to inspect the failure" >&2 + exit 1 + fi else echo "🔄 Restarting legacy ${POD_NAME} (use --replace or --pull to migrate)..." podman pod restart "${POD_NAME}" @@ -964,6 +989,7 @@ if [[ "$STACK_OWNERSHIP" != "absent" ]]; then setup_minio_buckets verify_stack_health + derive_external_service_urls echo "" echo "=========================================================================" echo "🎉 SUCCESS: LLM Triage Gateway restarted!" @@ -997,6 +1023,7 @@ else fi +derive_external_service_urls echo "=========================================================================" echo "🎉 SUCCESS: LLM Triage Gateway successfully deployed!" echo "📍 Entry endpoint : ${PUBLIC_BASE_URL}/v1" diff --git a/tests/test_quadlet_templates.py b/tests/test_quadlet_templates.py index 991a9309..920deafd 100644 --- a/tests/test_quadlet_templates.py +++ b/tests/test_quadlet_templates.py @@ -43,7 +43,8 @@ def test_upgrade_syncs_quadlets_before_quadlet_start_stack(): def test_rendered_quadlets_are_owner_only(): script = (ROOT / "start-stack.sh").read_text() assert 'chmod 700 "$QUADLET_DIR"' in script - assert "os.chmod(out_path, 0o600)" in script + assert "os.chmod(staged_path, 0o600)" in script + assert 'LLM_ROUTING_POD_UNIT="llm-routing-pod.service"' in script assert "systemctl --user daemon-reload" in script assert "stack_ownership()" in script assert "PODMAN_SYSTEMD_UNIT" in script @@ -55,3 +56,16 @@ def test_quadlet_renderer_quotes_environment_values_for_systemd(): assert 'text = re.sub(r"(?m)^Environment=(.*)$", quote_environment, text)' in script assert "def quote_environment(match):" in script assert "return f'Environment=\"{value}\"'" in script + + +def test_quadlet_renderer_stages_before_replacing_live_units(): + script = (ROOT / "start-stack.sh").read_text() + assert "tempfile.mkdtemp(prefix=\".llm-routing-render-\"" in script + assert "os.replace(os.path.join(staging_dir, name), os.path.join(out_dir, name))" in script + assert "shutil.rmtree(staging_dir, ignore_errors=True)" in script + + +def test_quadlet_systemd_failures_are_reported(): + script = (ROOT / "start-stack.sh").read_text() + assert 'if ! systemctl --user restart "$LLM_ROUTING_POD_UNIT"; then' in script + assert 'if ! systemctl --user start "$LLM_ROUTING_POD_UNIT"; then' in script From fc57e6b5a22a8459c4cd92fc40b30360bd37d811 Mon Sep 17 00:00:00 2001 From: boy Date: Thu, 23 Jul 2026 01:24:53 +0200 Subject: [PATCH 7/8] fix(dev): preserve overlayed llama URLs in router --- quadlets/llm-routing-router.container | 6 +++++- start-stack.sh | 4 ++++ tests/test_quadlet_templates.py | 10 ++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/quadlets/llm-routing-router.container b/quadlets/llm-routing-router.container index 16a0d9e6..72023f9d 100644 --- a/quadlets/llm-routing-router.container +++ b/quadlets/llm-routing-router.container @@ -13,7 +13,7 @@ AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 # Router image has no entrypoint (EP=null, CMD=uvicorn...); pod.yaml used # command: /bin/sh + args: -c ... — Entrypoint=/bin/sh, Exec=-c "..." matches. Entrypoint=/bin/sh -Exec=-c "set -a && . /config/.env && set +a && LITELLM_PORT=LITELLM_PORT_PLACEHOLDER VALKEY_CACHE_PORT=VALKEY_CACHE_PORT_PLACEHOLDER exec uvicorn main:app --host 0.0.0.0 --port ROUTER_PORT_PLACEHOLDER --log-level $${LOG_LEVEL:-warning}" +Exec=-c "set -a && . /config/.env && set +a && LLAMA_CLASSIFIER_URL=LLAMA_CLASSIFIER_URL_PLACEHOLDER LLAMA_SERVER_URL=LLAMA_SERVER_URL_PLACEHOLDER LITELLM_PORT=LITELLM_PORT_PLACEHOLDER VALKEY_CACHE_PORT=VALKEY_CACHE_PORT_PLACEHOLDER exec uvicorn main:app --host 0.0.0.0 --port ROUTER_PORT_PLACEHOLDER --log-level $${LOG_LEVEL:-warning}" Environment=CONFIG_PATH=/config/router_dir/config.yaml Environment=LITELLM_CONFIG_PATH=/config/litellm_dir/config.yaml Environment=DATABASE_URL=postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:POSTGRES_PORT_PLACEHOLDER/postgres @@ -25,6 +25,10 @@ Environment=LANGFUSE_HOST=http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER Environment=OLLAMA_API_KEY=OLLAMA_API_KEY_PLACEHOLDER Environment=PUBLIC_BASE_URL=PUBLIC_BASE_URL_PLACEHOLDER Environment=ROUTING_DOMAIN=ROUTING_DOMAIN_PLACEHOLDER +# Reassert these after /config/.env is sourced by Exec so a dev overlay cannot +# be overwritten by the production bind-mounted file. +Environment=LLAMA_CLASSIFIER_URL=LLAMA_CLASSIFIER_URL_PLACEHOLDER +Environment=LLAMA_SERVER_URL=LLAMA_SERVER_URL_PLACEHOLDER Environment=LITELLM_PORT=LITELLM_PORT_PLACEHOLDER Environment=VALKEY_CACHE_PORT=VALKEY_CACHE_PORT_PLACEHOLDER Environment=ROUTER_PORT=ROUTER_PORT_PLACEHOLDER diff --git a/start-stack.sh b/start-stack.sh index b6febd66..a1655f70 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -680,6 +680,7 @@ render_pod_yaml() { export OLLAMA_API_KEY OPENROUTER_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY export CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD export PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT + export LLAMA_CLASSIFIER_URL LLAMA_SERVER_URL export ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT export LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT export CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT @@ -814,6 +815,7 @@ render_quadlets() { export OLLAMA_API_KEY OPENROUTER_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY export CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD export PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT + export LLAMA_CLASSIFIER_URL LLAMA_SERVER_URL export ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT export LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT export CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT @@ -857,6 +859,8 @@ repl = { "PROXY_BASE_URL_PLACEHOLDER": proxy_base_url, "PUBLIC_BASE_URL_PLACEHOLDER": os.environ["PUBLIC_BASE_URL"].rstrip("/"), "ROUTING_DOMAIN_PLACEHOLDER": os.environ["ROUTING_DOMAIN"], + "LLAMA_CLASSIFIER_URL_PLACEHOLDER": os.environ["LLAMA_CLASSIFIER_URL"], + "LLAMA_SERVER_URL_PLACEHOLDER": os.environ["LLAMA_SERVER_URL"], "POD_NAME_PLACEHOLDER": os.environ["POD_NAME"], "DATA_ROOT_PLACEHOLDER": os.environ["DATA_ROOT"], "ROUTER_IMAGE_PLACEHOLDER": os.environ["ROUTER_IMAGE"], diff --git a/tests/test_quadlet_templates.py b/tests/test_quadlet_templates.py index 920deafd..0b012d52 100644 --- a/tests/test_quadlet_templates.py +++ b/tests/test_quadlet_templates.py @@ -69,3 +69,13 @@ def test_quadlet_systemd_failures_are_reported(): script = (ROOT / "start-stack.sh").read_text() assert 'if ! systemctl --user restart "$LLM_ROUTING_POD_UNIT"; then' in script assert 'if ! systemctl --user start "$LLM_ROUTING_POD_UNIT"; then' in script + + +def test_router_quadlet_reasserts_overlayed_llama_urls_after_env_source(): + template = (QUADLETS / "llm-routing-router.container").read_text() + assert "LLAMA_CLASSIFIER_URL=LLAMA_CLASSIFIER_URL_PLACEHOLDER" in template + assert "LLAMA_SERVER_URL=LLAMA_SERVER_URL_PLACEHOLDER" in template + assert template.index("LLAMA_SERVER_URL=LLAMA_SERVER_URL_PLACEHOLDER") < template.index("exec uvicorn") + script = (ROOT / "start-stack.sh").read_text() + assert '"LLAMA_CLASSIFIER_URL_PLACEHOLDER": os.environ["LLAMA_CLASSIFIER_URL"]' in script + assert '"LLAMA_SERVER_URL_PLACEHOLDER": os.environ["LLAMA_SERVER_URL"]' in script From dcbf6d768f0c221dce249ac5e3af1c87ad90ed1a Mon Sep 17 00:00:00 2001 From: boy Date: Thu, 23 Jul 2026 01:28:03 +0200 Subject: [PATCH 8/8] fix(deploy): enforce Quadlet deployment prerequisites --- router/main.py | 2 +- .../verify_canonical_endpoints.py | 2 +- start-stack.sh | 142 ++---------------- tests/test_quadlet_templates.py | 8 + 4 files changed, 21 insertions(+), 133 deletions(-) diff --git a/router/main.py b/router/main.py index a66136f9..d0774e15 100644 --- a/router/main.py +++ b/router/main.py @@ -3514,7 +3514,7 @@ def resolve_external_urls(request: Request) -> tuple[str, str, str]: # Basic sanity-check on external_host, but don't over-restrict valid hostnames; # fall back to the request base URL rather than silently forcing localhost. - if not isinstance(external_host, str) or not re.match(r"^[a-zA-Z0-9.-:]+$", external_host): + if not isinstance(external_host, str) or not re.match(r"^[a-zA-Z0-9.:-]+$", external_host): logger.warning( "Unexpected external_host %r, falling back to request.base_url.hostname (%r)", external_host, diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index 44d71239..458f0ae3 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -623,7 +623,7 @@ def test_canonical_urls(cfg: dict) -> tuple[int, int, int]: r = httpx.get(url, timeout=15, follow_redirects=True, headers={"Authorization": f"Bearer {cfg['router_api_key']}"}) ok = r.status_code == 200 - passed += check(f"GET {url}", ok, f"HTTP {r.status_code}") + passed += check(f"GET {url} ({label})", ok, f"HTTP {r.status_code}") except httpx.RequestError as e: # DNS/unreachable/timeout — skip gracefully (host may not resolve from test machine) skipped += 1 diff --git a/start-stack.sh b/start-stack.sh index a1655f70..e2b1936e 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -673,133 +673,7 @@ render_router_config() { echo "✓ Router config rendered to ${rendered_dir}/config.yaml" } -render_pod_yaml() { - export WORKDIR HOME LITELLM_MASTER_KEY UI_USERNAME UI_PASSWORD - export POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY - export LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD - export OLLAMA_API_KEY OPENROUTER_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY - export CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD - export PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT - export LLAMA_CLASSIFIER_URL LLAMA_SERVER_URL - export ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT - export LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT - export CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT - export MINIO_S3_PORT MINIO_CONSOLE_PORT - derive_external_service_urls - python3 - "$WORKDIR/pod.yaml" <<'PY' -import os, sys, urllib.parse, json -uid = os.getuid() -with open(sys.argv[1], "r", encoding="utf-8") as f: - text = f.read() - -def yaml_scalar(val): - return json.dumps(val) - -placeholders = [ - "WORKDIR_PLACEHOLDER", - "HOME_PLACEHOLDER", - "RUN_USER_PLACEHOLDER", - "LITELLM_MASTER_KEY_PLACEHOLDER", - "LITELLM_UI_USERNAME_PLACEHOLDER", - "LITELLM_UI_PASSWORD_PLACEHOLDER", - "POSTGRES_PASSWORD_RAW_PLACEHOLDER", - "POSTGRES_PASSWORD_ENCODED_PLACEHOLDER", - "NEXTAUTH_SECRET_PLACEHOLDER", - "NEXTAUTH_URL_PLACEHOLDER", - "SALT_PLACEHOLDER", - "ENCRYPTION_KEY_PLACEHOLDER", - "OLLAMA_API_KEY_PLACEHOLDER", - "OPENROUTER_API_KEY_PLACEHOLDER", - "LANGFUSE_PUBLIC_KEY_PLACEHOLDER", - "LANGFUSE_SECRET_KEY_PLACEHOLDER", - "MINIO_USER_PLACEHOLDER", - "MINIO_PASSWORD_PLACEHOLDER", - "LANGFUSE_INIT_USER_PASSWORD_PLACEHOLDER", - "REDIS_AUTH_PLACEHOLDER", - "CLICKHOUSE_PASSWORD_PLACEHOLDER", - "PROXY_BASE_URL_PLACEHOLDER", - "PUBLIC_BASE_URL_PLACEHOLDER", - "ROUTING_DOMAIN_PLACEHOLDER", - "POD_NAME_PLACEHOLDER", - "DATA_ROOT_PLACEHOLDER", - "ROUTER_PORT_PLACEHOLDER", - "LITELLM_PORT_PLACEHOLDER", - "LANGFUSE_WEB_PORT_PLACEHOLDER", - "LANGFUSE_WORKER_PORT_PLACEHOLDER", - "POSTGRES_PORT_PLACEHOLDER", - "VALKEY_CACHE_PORT_PLACEHOLDER", - "VALKEY_LF_PORT_PLACEHOLDER", - "CLICKHOUSE_HTTP_PORT_PLACEHOLDER", - "CLICKHOUSE_TCP_PORT_PLACEHOLDER", - "MINIO_S3_PORT_PLACEHOLDER", - "MINIO_CONSOLE_PORT_PLACEHOLDER", - "ROUTER_IMAGE_PLACEHOLDER", -] -for ph in placeholders: - if ph not in text: - sys.stderr.write(f"Error: Required placeholder '{ph}' not found in pod.yaml. Ensure you are using the latest version of the template.\n") - sys.exit(1) -text = text.replace("WORKDIR_PLACEHOLDER", os.environ["WORKDIR"]) -text = text.replace("HOME_PLACEHOLDER", os.environ["HOME"]) -text = text.replace("RUN_USER_PLACEHOLDER", f"/run/user/{uid}") -text = text.replace("LITELLM_MASTER_KEY_PLACEHOLDER", yaml_scalar(os.environ["LITELLM_MASTER_KEY"])) -text = text.replace("LITELLM_UI_USERNAME_PLACEHOLDER", yaml_scalar(os.environ.get("UI_USERNAME") or "admin")) -text = text.replace("LITELLM_UI_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ.get("UI_PASSWORD") or os.environ.get("LITELLM_MASTER_KEY") or "admin")) -text = text.replace("POSTGRES_PASSWORD_RAW_PLACEHOLDER", yaml_scalar(os.environ["POSTGRES_PASSWORD"])) -# URL-encode the postgres password for DSN insertion -encoded_password = urllib.parse.quote(os.environ['POSTGRES_PASSWORD'], safe="") -text = text.replace("POSTGRES_PASSWORD_ENCODED_PLACEHOLDER", encoded_password) -text = text.replace("NEXTAUTH_SECRET_PLACEHOLDER", yaml_scalar(os.environ["NEXTAUTH_SECRET"])) -text = text.replace("SALT_PLACEHOLDER", yaml_scalar(os.environ["SALT"])) -text = text.replace("ENCRYPTION_KEY_PLACEHOLDER", yaml_scalar(os.environ["ENCRYPTION_KEY"])) -text = text.replace("OLLAMA_API_KEY_PLACEHOLDER", yaml_scalar(os.environ["OLLAMA_API_KEY"])) -text = text.replace("OPENROUTER_API_KEY_PLACEHOLDER", yaml_scalar(os.environ["OPENROUTER_API_KEY"])) -text = text.replace("LANGFUSE_PUBLIC_KEY_PLACEHOLDER", yaml_scalar(os.environ["LANGFUSE_PUBLIC_KEY"])) -text = text.replace("LANGFUSE_SECRET_KEY_PLACEHOLDER", yaml_scalar(os.environ["LANGFUSE_SECRET_KEY"])) -text = text.replace("MINIO_USER_PLACEHOLDER", yaml_scalar(os.environ["MINIO_ROOT_USER"])) -text = text.replace("MINIO_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["MINIO_ROOT_PASSWORD"])) -text = text.replace("LANGFUSE_INIT_USER_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["LANGFUSE_INIT_USER_PASSWORD"])) -text = text.replace("REDIS_AUTH_PLACEHOLDER", yaml_scalar(os.environ["REDIS_AUTH"])) -text = text.replace("CLICKHOUSE_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["CLICKHOUSE_PASSWORD"])) -# Service URLs are derived once by the shell wrapper for both rendering paths. -proxy_base_url = os.environ["PROXY_BASE_URL_DERIVED"] -nextauth_url = os.environ["NEXTAUTH_URL_DERIVED"] - -text = text.replace("PROXY_BASE_URL_PLACEHOLDER", yaml_scalar(proxy_base_url)) -text = text.replace("PUBLIC_BASE_URL_PLACEHOLDER", yaml_scalar(os.environ["PUBLIC_BASE_URL"])) -text = text.replace("NEXTAUTH_URL_PLACEHOLDER", yaml_scalar(nextauth_url)) -text = text.replace("ROUTING_DOMAIN_PLACEHOLDER", yaml_scalar(os.environ["ROUTING_DOMAIN"])) -# Raw replacements (no quoting — used for pod name, data root, and integer port values) -raw_replacements = { - "POD_NAME_PLACEHOLDER": os.environ["POD_NAME"], - "DATA_ROOT_PLACEHOLDER": os.environ["DATA_ROOT"], - "ROUTER_PORT_PLACEHOLDER": os.environ["ROUTER_PORT"], - "LITELLM_PORT_PLACEHOLDER": os.environ["LITELLM_PORT"], - "LANGFUSE_WEB_PORT_PLACEHOLDER": os.environ["LANGFUSE_WEB_PORT"], - "LANGFUSE_WORKER_PORT_PLACEHOLDER": os.environ["LANGFUSE_WORKER_PORT"], - "POSTGRES_PORT_PLACEHOLDER": os.environ["POSTGRES_PORT"], - "VALKEY_CACHE_PORT_PLACEHOLDER": os.environ["VALKEY_CACHE_PORT"], - "VALKEY_LF_PORT_PLACEHOLDER": os.environ["VALKEY_LF_PORT"], - "CLICKHOUSE_HTTP_PORT_PLACEHOLDER": os.environ["CLICKHOUSE_HTTP_PORT"], - "CLICKHOUSE_TCP_PORT_PLACEHOLDER": os.environ["CLICKHOUSE_TCP_PORT"], - "MINIO_S3_PORT_PLACEHOLDER": os.environ["MINIO_S3_PORT"], - "MINIO_CONSOLE_PORT_PLACEHOLDER": os.environ["MINIO_CONSOLE_PORT"], - "ROUTER_IMAGE_PLACEHOLDER": os.environ["ROUTER_IMAGE"], -} -for ph, val in raw_replacements.items(): - text = text.replace(ph, val) -import re -unresolved = sorted(set(re.findall(r"\b[A-Z0-9_]+_PLACEHOLDER\b", text))) -if unresolved: - sys.stderr.write( - "Error: Unresolved placeholders remain in rendered pod.yaml: " - + ", ".join(unresolved) - + "\n" - ) - sys.exit(1) -sys.stdout.write(text) -PY -} +# Legacy pod.yaml rendering was removed: all deployment paths render Quadlets. # ── Quadlet rendering + installation ── # Renders quadlets/*.pod + quadlets/*.container templates (same _PLACEHOLDER @@ -821,7 +695,10 @@ render_quadlets() { export CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT export MINIO_S3_PORT MINIO_CONSOLE_PORT local src_dir="${WORKDIR}/quadlets" - derive_external_service_urls + if ! derive_external_service_urls; then + echo "❌ Error: failed to derive external service URLs for Quadlet rendering" >&2 + return 1 + fi mkdir -p "$QUADLET_DIR" chmod 700 "$QUADLET_DIR" python3 - "$src_dir" "$QUADLET_DIR" <<'PY' @@ -924,7 +801,7 @@ PY # Install quadlets and (re)generate systemd units; start or restart the stack. deploy_quadlets() { - require_user_systemd + require_user_systemd || exit 1 echo "📋 Rendering quadlet units..." render_quadlets if ! systemctl --user daemon-reload; then @@ -978,7 +855,7 @@ if [[ "$STACK_OWNERSHIP" != "absent" ]]; then deploy_fresh_pod else if [[ "$STACK_OWNERSHIP" == "quadlet" ]]; then - require_user_systemd + require_user_systemd || exit 1 echo "🔄 Restarting Quadlet-owned stack via systemd..." systemctl --user reset-failed "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true if ! systemctl --user restart "$LLM_ROUTING_POD_UNIT"; then @@ -988,7 +865,10 @@ if [[ "$STACK_OWNERSHIP" != "absent" ]]; then fi else echo "🔄 Restarting legacy ${POD_NAME} (use --replace or --pull to migrate)..." - podman pod restart "${POD_NAME}" + if ! podman pod restart "${POD_NAME}"; then + echo "❌ Error: failed to restart legacy pod ${POD_NAME}" >&2 + exit 1 + fi fi setup_minio_buckets verify_stack_health diff --git a/tests/test_quadlet_templates.py b/tests/test_quadlet_templates.py index 0b012d52..22f02b6c 100644 --- a/tests/test_quadlet_templates.py +++ b/tests/test_quadlet_templates.py @@ -79,3 +79,11 @@ def test_router_quadlet_reasserts_overlayed_llama_urls_after_env_source(): script = (ROOT / "start-stack.sh").read_text() assert '"LLAMA_CLASSIFIER_URL_PLACEHOLDER": os.environ["LLAMA_CLASSIFIER_URL"]' in script assert '"LLAMA_SERVER_URL_PLACEHOLDER": os.environ["LLAMA_SERVER_URL"]' in script + + +def test_quadlet_deployment_enforces_prerequisite_and_restart_failures(): + script = (ROOT / "start-stack.sh").read_text() + assert "render_pod_yaml()" not in script + assert "require_user_systemd || exit 1" in script + assert "failed to derive external service URLs for Quadlet rendering" in script + assert 'if ! podman pod restart "${POD_NAME}"; then' in script