From d1624595f7d10a391f7f6baf73fe68c84a6d74d6 Mon Sep 17 00:00:00 2001 From: boy Date: Wed, 22 Jul 2026 23:46:22 +0200 Subject: [PATCH 1/4] feat(deploy): migrate stack management from play kube to systemd Quadlets Replace 'podman play kube' with declarative Quadlet units rendered from quadlets/*.pod + quadlets/*.container templates into ~/.config/containers/systemd/llm-routing/. systemd's podman-user-generator turns them into llm-routing-*.service units, making systemd the single supervisor: boot auto-start (WantedBy=default.target + linger) and crash recovery (Restart=always) now come from systemd instead of podman's restartPolicy. Key points: - Templates use the same _PLACEHOLDER convention as pod.yaml; a new render_quadlets() in start-stack.sh reuses the existing env-derived values (ports, secrets, DATA_ROOT, POD_NAME) so dev/prod parity is preserved with plain string replacement (bare scalars, not YAML). - deploy_fresh_pod() now: render configs -> render quadlets -> daemon-reload -> start/restart llm-routing-pod.service. The bare 'restart existing' path and safe_pod_teardown() detect systemd-managed stacks and route through systemctl accordingly. - Entrypoint semantics: Quadlet Exec= only sets args (appended to the image entrypoint), so containers where pod.yaml used command: now set Entrypoint= explicitly (litellm venv python, valkey-server, redis-server, /bin/sh for the router). MinIO keeps its image entrypoint with Exec= providing the server args, matching pod.yaml args: behavior. - AddHost=:127.0.0.1 on all containers replicates the /etc/hosts entry play kube injected, required by langfuse 3.222+ which resolves the pod hostname at startup. - Healthchecks ported from livenessProbe exec commands (HealthCmd, HealthStartPeriod maps initialDelaySeconds). Verified on dev: all 9 containers healthy, 193/193 unit tests pass, crash recovery (kill -> systemd auto-restart <12s), canonical HTTPS endpoints green (model timeouts on free tier are pre-existing, identical on prod play-kube). --- quadlets/llm-routing-clickhouse.container | 26 ++++ quadlets/llm-routing-langfuse-web.container | 50 +++++++ .../llm-routing-langfuse-worker.container | 39 +++++ quadlets/llm-routing-litellm.container | 43 ++++++ quadlets/llm-routing-minio.container | 26 ++++ quadlets/llm-routing-postgres.container | 25 ++++ quadlets/llm-routing-router.container | 52 +++++++ quadlets/llm-routing-valkey-cache.container | 24 +++ quadlets/llm-routing-valkey-lf.container | 22 +++ quadlets/llm-routing.pod | 13 ++ start-stack.sh | 139 +++++++++++++++++- 11 files changed, 456 insertions(+), 3 deletions(-) create mode 100644 quadlets/llm-routing-clickhouse.container create mode 100644 quadlets/llm-routing-langfuse-web.container create mode 100644 quadlets/llm-routing-langfuse-worker.container create mode 100644 quadlets/llm-routing-litellm.container create mode 100644 quadlets/llm-routing-minio.container create mode 100644 quadlets/llm-routing-postgres.container create mode 100644 quadlets/llm-routing-router.container create mode 100644 quadlets/llm-routing-valkey-cache.container create mode 100644 quadlets/llm-routing-valkey-lf.container create mode 100644 quadlets/llm-routing.pod diff --git a/quadlets/llm-routing-clickhouse.container b/quadlets/llm-routing-clickhouse.container new file mode 100644 index 00000000..65f0c2f4 --- /dev/null +++ b/quadlets/llm-routing-clickhouse.container @@ -0,0 +1,26 @@ +[Unit] +Description=ClickHouse — Langfuse v3 OLAP trace store (POD_NAME_PLACEHOLDER) + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-clickhouse-db +Image=clickhouse/clickhouse-server:26.6.2.81 +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +User=101 +Group=101 +Environment=CLICKHOUSE_DB=default +Environment=CLICKHOUSE_USER=clickhouse +Environment=CLICKHOUSE_PASSWORD=CLICKHOUSE_PASSWORD_PLACEHOLDER +Volume=DATA_ROOT_PLACEHOLDER/clickhouse-data:/var/lib/clickhouse +Volume=DATA_ROOT_PLACEHOLDER/clickhouse-config/port-override.xml:/etc/clickhouse-server/config.d/port-override.xml +HealthCmd=clickhouse-client --host 127.0.0.1 --port CLICKHOUSE_TCP_PORT_PLACEHOLDER --user clickhouse --password CLICKHOUSE_PASSWORD_PLACEHOLDER --query "SELECT 1" +HealthInterval=15s +HealthTimeout=5s +HealthRetries=5 +HealthStartPeriod=20s + +[Service] +Restart=always +TimeoutStartSec=120 diff --git a/quadlets/llm-routing-langfuse-web.container b/quadlets/llm-routing-langfuse-web.container new file mode 100644 index 00000000..e9fa0db4 --- /dev/null +++ b/quadlets/llm-routing-langfuse-web.container @@ -0,0 +1,50 @@ +[Unit] +Description=Langfuse Web — observability dashboard (POD_NAME_PLACEHOLDER) +After=llm-routing-postgres.service llm-routing-clickhouse.service llm-routing-valkey-lf.service llm-routing-minio.service +Wants=llm-routing-postgres.service llm-routing-clickhouse.service llm-routing-valkey-lf.service llm-routing-minio.service + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-langfuse-web +Image=langfuse/langfuse:3.222.0 +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +Environment=DATABASE_URL=postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:POSTGRES_PORT_PLACEHOLDER/langfuse +Environment=NEXTAUTH_SECRET=NEXTAUTH_SECRET_PLACEHOLDER +Environment=NEXTAUTH_URL=NEXTAUTH_URL_PLACEHOLDER +Environment=SALT=SALT_PLACEHOLDER +Environment=ENCRYPTION_KEY=ENCRYPTION_KEY_PLACEHOLDER +Environment=HOSTNAME=0.0.0.0 +Environment=PORT=LANGFUSE_WEB_PORT_PLACEHOLDER +Environment=TELEMETRY_ENABLED=false +Environment=LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse-events +Environment=LANGFUSE_S3_EVENT_UPLOAD_REGION=us-east-1 +Environment=LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=MINIO_USER_PLACEHOLDER +Environment=LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=MINIO_PASSWORD_PLACEHOLDER +Environment=LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT=http://127.0.0.1:MINIO_S3_PORT_PLACEHOLDER +Environment=S3_FORCE_PATH_STYLE=true +Environment=CLICKHOUSE_URL=http://127.0.0.1:CLICKHOUSE_HTTP_PORT_PLACEHOLDER +Environment=CLICKHOUSE_MIGRATION_URL=clickhouse://127.0.0.1:CLICKHOUSE_TCP_PORT_PLACEHOLDER +Environment=CLICKHOUSE_USER=clickhouse +Environment=CLICKHOUSE_PASSWORD=CLICKHOUSE_PASSWORD_PLACEHOLDER +Environment=CLICKHOUSE_CLUSTER_ENABLED=false +Environment=REDIS_HOST=127.0.0.1 +Environment=REDIS_PORT=VALKEY_LF_PORT_PLACEHOLDER +Environment=REDIS_AUTH=REDIS_AUTH_PLACEHOLDER +Environment=LANGFUSE_INIT_ORG_ID=org-local-dev-id +Environment=LANGFUSE_INIT_ORG_NAME=Local Development +Environment=LANGFUSE_INIT_PROJECT_ID=proj-triage-gateway-id +Environment=LANGFUSE_INIT_PROJECT_NAME=Triage Gateway +Environment=LANGFUSE_INIT_USER_EMAIL=admin@local.dev +Environment=LANGFUSE_INIT_USER_PASSWORD=LANGFUSE_INIT_USER_PASSWORD_PLACEHOLDER +Environment=LANGFUSE_LOG_LEVEL=warn +HealthCmd=wget -q -O /dev/null http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER/api/public/health +HealthInterval=15s +HealthTimeout=5s +HealthRetries=5 +HealthStartPeriod=240s + +[Service] +Restart=always +TimeoutStartSec=300 diff --git a/quadlets/llm-routing-langfuse-worker.container b/quadlets/llm-routing-langfuse-worker.container new file mode 100644 index 00000000..1872373c --- /dev/null +++ b/quadlets/llm-routing-langfuse-worker.container @@ -0,0 +1,39 @@ +[Unit] +Description=Langfuse Worker — background job processor (POD_NAME_PLACEHOLDER) +After=llm-routing-postgres.service llm-routing-clickhouse.service llm-routing-valkey-lf.service llm-routing-minio.service +Wants=llm-routing-postgres.service llm-routing-clickhouse.service llm-routing-valkey-lf.service llm-routing-minio.service + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-langfuse-worker +Image=langfuse/langfuse-worker:3.222.0 +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +Environment=DATABASE_URL=postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:POSTGRES_PORT_PLACEHOLDER/langfuse +Environment=CLICKHOUSE_URL=http://127.0.0.1:CLICKHOUSE_HTTP_PORT_PLACEHOLDER +Environment=CLICKHOUSE_USER=clickhouse +Environment=CLICKHOUSE_PASSWORD=CLICKHOUSE_PASSWORD_PLACEHOLDER +Environment=CLICKHOUSE_CLUSTER_ENABLED=false +Environment=REDIS_HOST=127.0.0.1 +Environment=REDIS_PORT=VALKEY_LF_PORT_PLACEHOLDER +Environment=REDIS_AUTH=REDIS_AUTH_PLACEHOLDER +Environment=HOSTNAME=0.0.0.0 +Environment=PORT=LANGFUSE_WORKER_PORT_PLACEHOLDER +Environment=TELEMETRY_ENABLED=false +Environment=LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse-events +Environment=LANGFUSE_S3_EVENT_UPLOAD_REGION=us-east-1 +Environment=LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=MINIO_USER_PLACEHOLDER +Environment=LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=MINIO_PASSWORD_PLACEHOLDER +Environment=LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT=http://127.0.0.1:MINIO_S3_PORT_PLACEHOLDER +Environment=S3_FORCE_PATH_STYLE=true +Environment=LANGFUSE_LOG_LEVEL=warn +HealthCmd=pgrep node +HealthInterval=15s +HealthTimeout=5s +HealthRetries=5 +HealthStartPeriod=240s + +[Service] +Restart=always +TimeoutStartSec=300 diff --git a/quadlets/llm-routing-litellm.container b/quadlets/llm-routing-litellm.container new file mode 100644 index 00000000..6ab1b902 --- /dev/null +++ b/quadlets/llm-routing-litellm.container @@ -0,0 +1,43 @@ +[Unit] +Description=LiteLLM Gateway — upstream proxy with adaptive routing (POD_NAME_PLACEHOLDER) +After=llm-routing-postgres.service llm-routing-valkey-cache.service +Wants=llm-routing-postgres.service llm-routing-valkey-cache.service + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-litellm-gateway +Image=ghcr.io/berriai/litellm:v1.93.0 +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +# Image entrypoint is docker/prod_entrypoint.sh (litellm CLI); pod.yaml used +# command: python3 /app/entrypoint.py which replaced it. Quadlet Exec= only sets +# args, so override Entrypoint= with the venv python (matches play-kube behavior). +Entrypoint=/app/.venv/bin/python3 +Exec=/app/entrypoint.py +Environment=DATABASE_URL=postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:POSTGRES_PORT_PLACEHOLDER/postgres +Environment=STORE_MODEL_IN_DB=True +Environment=LITELLM_LOG=INFO +Environment=LANGFUSE_HOST=http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER +Environment=LITELLM_MASTER_KEY=LITELLM_MASTER_KEY_PLACEHOLDER +Environment=LITELLM_UI_USERNAME=LITELLM_UI_USERNAME_PLACEHOLDER +Environment=LITELLM_UI_PASSWORD=LITELLM_UI_PASSWORD_PLACEHOLDER +Environment=OLLAMA_API_KEY=OLLAMA_API_KEY_PLACEHOLDER +Environment=OPENROUTER_API_KEY=OPENROUTER_API_KEY_PLACEHOLDER +Environment=SERVER_ROOT_PATH=/llm-routing/litellm +Environment=PROXY_BASE_URL=PROXY_BASE_URL_PLACEHOLDER +Environment=LITELLM_PORT=LITELLM_PORT_PLACEHOLDER +Environment=POSTGRES_PORT=POSTGRES_PORT_PLACEHOLDER +Volume=DATA_ROOT_PLACEHOLDER/litellm-rendered/config.yaml:/app/config.yaml +Volume=DATA_ROOT_PLACEHOLDER/litellm-rendered/entrypoint.py:/app/entrypoint.py +Volume=WORKDIR_PLACEHOLDER/.env:/config/.env +Volume=HOME_PLACEHOLDER/.gemini:/config/gemini_auth +HealthCmd=python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:LITELLM_PORT_PLACEHOLDER/health/liveness')" +HealthInterval=15s +HealthTimeout=5s +HealthRetries=5 +HealthStartPeriod=240s + +[Service] +Restart=always +TimeoutStartSec=300 diff --git a/quadlets/llm-routing-minio.container b/quadlets/llm-routing-minio.container new file mode 100644 index 00000000..56f59a0e --- /dev/null +++ b/quadlets/llm-routing-minio.container @@ -0,0 +1,26 @@ +[Unit] +Description=MinIO S3 — Langfuse v3 event storage (POD_NAME_PLACEHOLDER) + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-minio-s3 +Image=minio/minio:RELEASE.2025-09-07T16-13-09Z +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +Environment=MINIO_ROOT_USER=MINIO_USER_PLACEHOLDER +Environment=MINIO_ROOT_PASSWORD=MINIO_PASSWORD_PLACEHOLDER +# pod.yaml used args: server ... with no command:, so image entrypoint +# /usr/bin/docker-entrypoint.sh ran and consumed them. Quadlet Exec= sets args +# the same way — image entrypoint stays, Exec provides the server args. +Exec=server --quiet /data --address :MINIO_S3_PORT_PLACEHOLDER --console-address :MINIO_CONSOLE_PORT_PLACEHOLDER +Volume=DATA_ROOT_PLACEHOLDER/minio-data:/data +HealthCmd=curl -f http://127.0.0.1:MINIO_S3_PORT_PLACEHOLDER/minio/health/live +HealthInterval=15s +HealthTimeout=5s +HealthRetries=5 +HealthStartPeriod=10s + +[Service] +Restart=always +TimeoutStartSec=90 diff --git a/quadlets/llm-routing-postgres.container b/quadlets/llm-routing-postgres.container new file mode 100644 index 00000000..980a56bc --- /dev/null +++ b/quadlets/llm-routing-postgres.container @@ -0,0 +1,25 @@ +[Unit] +Description=PostgreSQL — Langfuse + LiteLLM metadata store (POD_NAME_PLACEHOLDER) + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-postgres-db +Image=pgvector/pgvector:0.8.5-pg18 +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +Environment=POSTGRES_USER=postgres +Environment=POSTGRES_PASSWORD=POSTGRES_PASSWORD_RAW_PLACEHOLDER +Environment=POSTGRES_DB=langfuse +Environment=PGDATA=/var/lib/postgresql/data/pg18 +Environment=PGPORT=POSTGRES_PORT_PLACEHOLDER +Volume=DATA_ROOT_PLACEHOLDER/postgres-data:/var/lib/postgresql +HealthCmd=pg_isready -U postgres -p POSTGRES_PORT_PLACEHOLDER +HealthInterval=10s +HealthTimeout=5s +HealthRetries=5 +HealthStartPeriod=15s + +[Service] +Restart=always +TimeoutStartSec=120 diff --git a/quadlets/llm-routing-router.container b/quadlets/llm-routing-router.container new file mode 100644 index 00000000..3c39430b --- /dev/null +++ b/quadlets/llm-routing-router.container @@ -0,0 +1,52 @@ +[Unit] +Description=LLM Triage Router — classification + routing gateway (POD_NAME_PLACEHOLDER) +After=llm-routing-litellm.service +Wants=llm-routing-litellm.service + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-llm-triage-router +Image=ROUTER_IMAGE_PLACEHOLDER +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +# Router image has no entrypoint (EP=null, CMD=uvicorn...); pod.yaml used +# command: /bin/sh + args: -c ... — Entrypoint=/bin/sh, Exec=-c "..." matches. +Entrypoint=/bin/sh +Exec=-c "set -a && . /config/.env && set +a && LITELLM_PORT=LITELLM_PORT_PLACEHOLDER VALKEY_CACHE_PORT=VALKEY_CACHE_PORT_PLACEHOLDER exec uvicorn main:app --host 0.0.0.0 --port ROUTER_PORT_PLACEHOLDER --log-level $${LOG_LEVEL:-warning}" +Environment=CONFIG_PATH=/config/router_dir/config.yaml +Environment=LITELLM_CONFIG_PATH=/config/litellm_dir/config.yaml +Environment=DATABASE_URL=postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:POSTGRES_PORT_PLACEHOLDER/postgres +Environment=DBUS_SESSION_BUS_ADDRESS=unix:path=RUN_USER_PLACEHOLDER/bus +Environment=LITELLM_MASTER_KEY=LITELLM_MASTER_KEY_PLACEHOLDER +Environment=LANGFUSE_PUBLIC_KEY=LANGFUSE_PUBLIC_KEY_PLACEHOLDER +Environment=LANGFUSE_SECRET_KEY=LANGFUSE_SECRET_KEY_PLACEHOLDER +Environment=LANGFUSE_HOST=http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER +Environment=OLLAMA_API_KEY=OLLAMA_API_KEY_PLACEHOLDER +Environment=PUBLIC_BASE_URL=PUBLIC_BASE_URL_PLACEHOLDER +Environment=ROUTING_DOMAIN=ROUTING_DOMAIN_PLACEHOLDER +Environment=LITELLM_PORT=LITELLM_PORT_PLACEHOLDER +Environment=VALKEY_CACHE_PORT=VALKEY_CACHE_PORT_PLACEHOLDER +Environment=ROUTER_PORT=ROUTER_PORT_PLACEHOLDER +Environment=LANGFUSE_WEB_PORT=LANGFUSE_WEB_PORT_PLACEHOLDER +Environment=LOG_LEVEL=info +Volume=DATA_ROOT_PLACEHOLDER/router-rendered:/config/router_dir +Volume=DATA_ROOT_PLACEHOLDER/litellm-rendered:/config/litellm_dir +Volume=DATA_ROOT_PLACEHOLDER/datasets:/app/data +Volume=HOME_PLACEHOLDER/.gemini:/config/gemini_auth +Volume=HOME_PLACEHOLDER/.gemini:/root/.gemini +Volume=HOME_PLACEHOLDER/.local/share/goose:/config/goose_sessions +Volume=HOME_PLACEHOLDER/.local/bin/agy:/usr/local/bin/agy +Volume=RUN_USER_PLACEHOLDER/bus:RUN_USER_PLACEHOLDER/bus +Volume=HOME_PLACEHOLDER/.local/share/keyrings:/root/.local/share/keyrings +Volume=RUN_USER_PLACEHOLDER:RUN_USER_PLACEHOLDER +Volume=WORKDIR_PLACEHOLDER/.env:/config/.env +HealthCmd=python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:ROUTER_PORT_PLACEHOLDER/metrics')" +HealthInterval=15s +HealthTimeout=5s +HealthRetries=5 +HealthStartPeriod=240s + +[Service] +Restart=always +TimeoutStartSec=300 diff --git a/quadlets/llm-routing-valkey-cache.container b/quadlets/llm-routing-valkey-cache.container new file mode 100644 index 00000000..97b497aa --- /dev/null +++ b/quadlets/llm-routing-valkey-cache.container @@ -0,0 +1,24 @@ +[Unit] +Description=Valkey Cache — LiteLLM response cache (POD_NAME_PLACEHOLDER) + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-valkey-cache +Image=valkey/valkey:9.1.1 +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +# pod.yaml used command: (replaces entrypoint). Image entrypoint docker-entrypoint.sh +# would also work, but explicit Entrypoint= matches play-kube semantics exactly. +Entrypoint=valkey-server +Exec=--bind 127.0.0.1 --port VALKEY_CACHE_PORT_PLACEHOLDER --loglevel warning +Volume=DATA_ROOT_PLACEHOLDER/valkey-data:/data +HealthCmd=valkey-cli -p VALKEY_CACHE_PORT_PLACEHOLDER ping +HealthInterval=10s +HealthTimeout=2s +HealthRetries=5 +HealthStartPeriod=5s + +[Service] +Restart=always +TimeoutStartSec=90 diff --git a/quadlets/llm-routing-valkey-lf.container b/quadlets/llm-routing-valkey-lf.container new file mode 100644 index 00000000..21f3ccee --- /dev/null +++ b/quadlets/llm-routing-valkey-lf.container @@ -0,0 +1,22 @@ +[Unit] +Description=Valkey LF — Langfuse v3 BullMQ job queue (POD_NAME_PLACEHOLDER) + +[Container] +ContainerName=POD_NAME_PLACEHOLDER-valkey-lf +Image=valkey/valkey:9.1.1 +Pod=llm-routing.pod +# play-kube injected -> 127.0.0.1 into /etc/hosts; replicate for apps +# (langfuse 3.222+) that resolve the pod hostname for self/intra-pod calls. +AddHost=POD_NAME_PLACEHOLDER:127.0.0.1 +Entrypoint=redis-server +Exec=--bind 127.0.0.1 --port VALKEY_LF_PORT_PLACEHOLDER --requirepass REDIS_AUTH_PLACEHOLDER --maxmemory-policy noeviction --loglevel warning +Volume=DATA_ROOT_PLACEHOLDER/redis-lf-data:/data +HealthCmd=valkey-cli -p VALKEY_LF_PORT_PLACEHOLDER -a REDIS_AUTH_PLACEHOLDER ping +HealthInterval=10s +HealthTimeout=2s +HealthRetries=5 +HealthStartPeriod=5s + +[Service] +Restart=always +TimeoutStartSec=90 diff --git a/quadlets/llm-routing.pod b/quadlets/llm-routing.pod new file mode 100644 index 00000000..3f5e0c4a --- /dev/null +++ b/quadlets/llm-routing.pod @@ -0,0 +1,13 @@ +# LLM Triage Gateway pod — Quadlet template (rendered by start-stack.sh) +# Pod name is rendered from POD_NAME (prod-router-pod / dev-router-pod). +[Unit] +Description=LLM Triage Gateway Pod (POD_NAME_PLACEHOLDER) +Wants=network-online.target +After=network-online.target + +[Pod] +PodName=POD_NAME_PLACEHOLDER +Network=host + +[Install] +WantedBy=default.target diff --git a/start-stack.sh b/start-stack.sh index 6087a442..21d3a0b2 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -534,6 +534,16 @@ verify_stack_health() { # Graceful stop (SIGTERM with 30s timeout) lets ClickHouse/Postgres flush, # then force-remove if needed. Avoids data corruption from SIGKILL. safe_pod_teardown() { + # If the stack is systemd-managed (quadlets), stop via systemd so the + # manager state stays consistent; then remove any leftover pod. + if systemctl --user is-active --quiet llm-routing-pod.service 2>/dev/null; then + echo "🛑 Stopping systemd-managed stack (llm-routing-pod.service)..." + systemctl --user stop llm-routing-pod.service || true + podman pod rm -f ${POD_NAME} 2>/dev/null || true + cleanup_zombie_ports + echo "✓ Stack stopped, ports cleaned" + return + fi if podman pod exists ${POD_NAME} 2>/dev/null; then echo "🛑 Gracefully stopping pod (SIGTERM, 30s timeout)..." podman pod stop -t 30 ${POD_NAME} 2>/dev/null || true @@ -739,11 +749,129 @@ sys.stdout.write(text) PY } +# ── Quadlet rendering + installation ── +# Renders quadlets/*.pod + quadlets/*.container templates (same _PLACEHOLDER +# convention as pod.yaml) into ~/.config/containers/systemd/llm-routing/ and +# lets systemd's podman-user-generator turn them into real units. +# Quadlet values are bare scalars (not YAML) so plain string replacement is used. +QUADLET_DIR="${HOME}/.config/containers/systemd/llm-routing" + +render_quadlets() { + export WORKDIR HOME LITELLM_MASTER_KEY UI_USERNAME UI_PASSWORD + export POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY + export LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD + export OLLAMA_API_KEY OPENROUTER_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY + export CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD + export PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT + export ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT + export LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT + export CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT + export MINIO_S3_PORT MINIO_CONSOLE_PORT + local src_dir="${WORKDIR}/quadlets" + mkdir -p "$QUADLET_DIR" + python3 - "$src_dir" "$QUADLET_DIR" <<'PY' +import os, sys, urllib.parse, re, glob +uid = os.getuid() +src_dir, out_dir = sys.argv[1], sys.argv[2] + +encoded_pg = urllib.parse.quote(os.environ['POSTGRES_PASSWORD'], safe="") +# Derive PROXY_BASE_URL / NEXTAUTH_URL exactly as render_pod_yaml does +# (path-based on master; subdomain switch lands with the routing branch) +public_base_url = os.environ["PUBLIC_BASE_URL"].rstrip("/") +proxy_base_url = f"{public_base_url}/litellm" +nextauth_url = f"{public_base_url}/langfuse" + +repl = { + "WORKDIR_PLACEHOLDER": os.environ["WORKDIR"], + "HOME_PLACEHOLDER": os.environ["HOME"], + "RUN_USER_PLACEHOLDER": f"/run/user/{uid}", + "LITELLM_MASTER_KEY_PLACEHOLDER": os.environ["LITELLM_MASTER_KEY"], + "LITELLM_UI_USERNAME_PLACEHOLDER": os.environ.get("UI_USERNAME") or "admin", + "LITELLM_UI_PASSWORD_PLACEHOLDER": os.environ.get("UI_PASSWORD") or os.environ.get("LITELLM_MASTER_KEY") or "admin", + "POSTGRES_PASSWORD_RAW_PLACEHOLDER": os.environ["POSTGRES_PASSWORD"], + "POSTGRES_PASSWORD_ENCODED_PLACEHOLDER": encoded_pg, + "NEXTAUTH_SECRET_PLACEHOLDER": os.environ["NEXTAUTH_SECRET"], + "NEXTAUTH_URL_PLACEHOLDER": nextauth_url, + "SALT_PLACEHOLDER": os.environ["SALT"], + "ENCRYPTION_KEY_PLACEHOLDER": os.environ["ENCRYPTION_KEY"], + "OLLAMA_API_KEY_PLACEHOLDER": os.environ["OLLAMA_API_KEY"], + "OPENROUTER_API_KEY_PLACEHOLDER": os.environ["OPENROUTER_API_KEY"], + "LANGFUSE_PUBLIC_KEY_PLACEHOLDER": os.environ["LANGFUSE_PUBLIC_KEY"], + "LANGFUSE_SECRET_KEY_PLACEHOLDER": os.environ["LANGFUSE_SECRET_KEY"], + "MINIO_USER_PLACEHOLDER": os.environ["MINIO_ROOT_USER"], + "MINIO_PASSWORD_PLACEHOLDER": os.environ["MINIO_ROOT_PASSWORD"], + "LANGFUSE_INIT_USER_PASSWORD_PLACEHOLDER": os.environ["LANGFUSE_INIT_USER_PASSWORD"], + "REDIS_AUTH_PLACEHOLDER": os.environ["REDIS_AUTH"], + "CLICKHOUSE_PASSWORD_PLACEHOLDER": os.environ["CLICKHOUSE_PASSWORD"], + "PROXY_BASE_URL_PLACEHOLDER": proxy_base_url, + "PUBLIC_BASE_URL_PLACEHOLDER": public_base_url, + "ROUTING_DOMAIN_PLACEHOLDER": os.environ["ROUTING_DOMAIN"], + "POD_NAME_PLACEHOLDER": os.environ["POD_NAME"], + "DATA_ROOT_PLACEHOLDER": os.environ["DATA_ROOT"], + "ROUTER_IMAGE_PLACEHOLDER": os.environ["ROUTER_IMAGE"], + "ROUTER_PORT_PLACEHOLDER": os.environ["ROUTER_PORT"], + "LITELLM_PORT_PLACEHOLDER": os.environ["LITELLM_PORT"], + "LANGFUSE_WEB_PORT_PLACEHOLDER": os.environ["LANGFUSE_WEB_PORT"], + "LANGFUSE_WORKER_PORT_PLACEHOLDER": os.environ["LANGFUSE_WORKER_PORT"], + "POSTGRES_PORT_PLACEHOLDER": os.environ["POSTGRES_PORT"], + "VALKEY_CACHE_PORT_PLACEHOLDER": os.environ["VALKEY_CACHE_PORT"], + "VALKEY_LF_PORT_PLACEHOLDER": os.environ["VALKEY_LF_PORT"], + "CLICKHOUSE_HTTP_PORT_PLACEHOLDER": os.environ["CLICKHOUSE_HTTP_PORT"], + "CLICKHOUSE_TCP_PORT_PLACEHOLDER": os.environ["CLICKHOUSE_TCP_PORT"], + "MINIO_S3_PORT_PLACEHOLDER": os.environ["MINIO_S3_PORT"], + "MINIO_CONSOLE_PORT_PLACEHOLDER": os.environ["MINIO_CONSOLE_PORT"], +} + +templates = sorted(glob.glob(os.path.join(src_dir, "*.pod")) + glob.glob(os.path.join(src_dir, "*.container"))) +if not templates: + sys.stderr.write(f"Error: no quadlet templates found in {src_dir}\n") + sys.exit(1) + +# Remove stale rendered files from previous deploys (e.g. renamed containers) +for stale in glob.glob(os.path.join(out_dir, "*.pod")) + glob.glob(os.path.join(out_dir, "*.container")): + os.unlink(stale) + +for tpl in templates: + with open(tpl, "r", encoding="utf-8") as f: + text = f.read() + for ph, val in repl.items(): + text = text.replace(ph, str(val)) + unresolved = sorted(set(re.findall(r"\b[A-Z0-9_]+_PLACEHOLDER\b", text))) + if unresolved: + sys.stderr.write(f"Error: Unresolved placeholders in {os.path.basename(tpl)}: {', '.join(unresolved)}\n") + sys.exit(1) + out_path = os.path.join(out_dir, os.path.basename(tpl)) + with open(out_path, "w", encoding="utf-8") as f: + f.write(text) + os.chmod(out_path, 0o644) + print(f" ✓ {os.path.basename(tpl)}") +PY + echo "✓ Quadlets rendered to ${QUADLET_DIR}" +} + +# Install quadlets and (re)generate systemd units; start or restart the stack. +deploy_quadlets() { + echo "📋 Rendering quadlet units..." + render_quadlets + if ! systemctl --user daemon-reload; then + echo "❌ Error: systemctl --user daemon-reload failed (no systemd user session?)" >&2 + exit 1 + fi + echo "✓ systemd units regenerated" + if systemctl --user is-active --quiet llm-routing-pod.service; then + echo "🔄 Restarting llm-routing-pod.service..." + systemctl --user restart llm-routing-pod.service + else + echo "🚀 Starting llm-routing-pod.service..." + systemctl --user start llm-routing-pod.service + fi +} + deploy_fresh_pod() { generate_clickhouse_config render_litellm_config render_router_config - render_pod_yaml | podman play kube - + deploy_quadlets setup_minio_buckets verify_stack_health } @@ -766,8 +894,13 @@ if podman pod exists ${POD_NAME} 2>/dev/null; then echo "🚀 Deploying replacement pod from YAML..." deploy_fresh_pod else - echo "🔄 Restarting existing ${POD_NAME} (use --replace or --pull to recreate)..." - podman pod restart ${POD_NAME} + if systemctl --user is-active --quiet llm-routing-pod.service 2>/dev/null; then + echo "🔄 Restarting stack via systemd (llm-routing-pod.service)..." + systemctl --user restart llm-routing-pod.service + else + echo "🔄 Restarting existing ${POD_NAME} (use --replace or --pull to recreate)..." + podman pod restart ${POD_NAME} + fi setup_minio_buckets verify_stack_health From f8fed954408e7e95ed26e507d4fb61b4b54d7063 Mon Sep 17 00:00:00 2001 From: boy Date: Thu, 23 Jul 2026 00:00:01 +0200 Subject: [PATCH 2/4] fix(deploy): harden Quadlet rollout and health recovery --- README.md | 15 ++++-- quadlets/llm-routing-clickhouse.container | 1 + quadlets/llm-routing-langfuse-web.container | 1 + .../llm-routing-langfuse-worker.container | 1 + quadlets/llm-routing-litellm.container | 1 + quadlets/llm-routing-minio.container | 1 + quadlets/llm-routing-postgres.container | 1 + quadlets/llm-routing-router.container | 1 + quadlets/llm-routing-valkey-cache.container | 1 + quadlets/llm-routing-valkey-lf.container | 1 + scripts/README.md | 7 +-- scripts/upgrade-prod.sh | 9 ++-- start-stack.sh | 4 +- tests/test_quadlet_templates.py | 47 +++++++++++++++++++ 14 files changed, 78 insertions(+), 13 deletions(-) create mode 100644 tests/test_quadlet_templates.py diff --git a/README.md b/README.md index 061ec42b..9e944930 100644 --- a/README.md +++ b/README.md @@ -389,15 +389,20 @@ For secure production deployments, the gateway services are configured to run un ### 1. Launching the Stack Run the startup script from the root of the repository: ```bash -./start-stack.sh # Fast restart (preserves container IDs and logs) -./start-stack.sh --replace # Recreate pod from YAML (picks up new ports, - # health probes, env vars, containers — no rebuild) -./start-stack.sh --full-rebuild # Full reset: rebuild image + recreate pod +./start-stack.sh # Restart the systemd-managed Quadlet stack +./start-stack.sh --replace # Render Quadlets, daemon-reload, and recreate the stack + # (picks up ports, probes, env vars, and containers) +./start-stack.sh --full-rebuild # Same as --replace plus rebuild the router image + +# Inspect the generated systemd units and their logs +systemctl --user status llm-routing-pod.service --no-pager +systemctl --user list-units 'llm-routing-*' --no-pager +journalctl --user -u llm-routing-router.service -n 100 --no-pager ``` *Note: If running for the first time, the script will prompt you for your `OpenRouter API Key`, securely saving it inside `.env` with restrictive permissions (`chmod 600`). The script also automatically generates and persists secure random secrets (`LITELLM_MASTER_KEY`, `POSTGRES_PASSWORD`, `NEXTAUTH_SECRET`, `SALT`, `ENCRYPTION_KEY`, and `ROUTER_API_KEY`) to this file on startup if they are missing.* ### 2. Verify Container Status -Check that all **10 containers** inside `prod-router-pod` are up and running: +Check that all **9 application containers** in `prod-router-pod` are up and running (the tenth pod infra container is Podman-managed): ```bash podman pod ps podman ps --pod --filter pod=prod-router-pod diff --git a/quadlets/llm-routing-clickhouse.container b/quadlets/llm-routing-clickhouse.container index 65f0c2f4..8696921f 100644 --- a/quadlets/llm-routing-clickhouse.container +++ b/quadlets/llm-routing-clickhouse.container @@ -18,6 +18,7 @@ Volume=DATA_ROOT_PLACEHOLDER/clickhouse-config/port-override.xml:/etc/clickhouse HealthCmd=clickhouse-client --host 127.0.0.1 --port CLICKHOUSE_TCP_PORT_PLACEHOLDER --user clickhouse --password CLICKHOUSE_PASSWORD_PLACEHOLDER --query "SELECT 1" HealthInterval=15s HealthTimeout=5s +HealthOnFailure=kill HealthRetries=5 HealthStartPeriod=20s diff --git a/quadlets/llm-routing-langfuse-web.container b/quadlets/llm-routing-langfuse-web.container index e9fa0db4..d6f4fb86 100644 --- a/quadlets/llm-routing-langfuse-web.container +++ b/quadlets/llm-routing-langfuse-web.container @@ -42,6 +42,7 @@ Environment=LANGFUSE_LOG_LEVEL=warn HealthCmd=wget -q -O /dev/null http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER/api/public/health HealthInterval=15s HealthTimeout=5s +HealthOnFailure=kill HealthRetries=5 HealthStartPeriod=240s diff --git a/quadlets/llm-routing-langfuse-worker.container b/quadlets/llm-routing-langfuse-worker.container index 1872373c..b4e63724 100644 --- a/quadlets/llm-routing-langfuse-worker.container +++ b/quadlets/llm-routing-langfuse-worker.container @@ -31,6 +31,7 @@ Environment=LANGFUSE_LOG_LEVEL=warn HealthCmd=pgrep node HealthInterval=15s HealthTimeout=5s +HealthOnFailure=kill HealthRetries=5 HealthStartPeriod=240s diff --git a/quadlets/llm-routing-litellm.container b/quadlets/llm-routing-litellm.container index 6ab1b902..aff6e059 100644 --- a/quadlets/llm-routing-litellm.container +++ b/quadlets/llm-routing-litellm.container @@ -35,6 +35,7 @@ Volume=HOME_PLACEHOLDER/.gemini:/config/gemini_auth HealthCmd=python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:LITELLM_PORT_PLACEHOLDER/health/liveness')" HealthInterval=15s HealthTimeout=5s +HealthOnFailure=kill HealthRetries=5 HealthStartPeriod=240s diff --git a/quadlets/llm-routing-minio.container b/quadlets/llm-routing-minio.container index 56f59a0e..4df6c6cb 100644 --- a/quadlets/llm-routing-minio.container +++ b/quadlets/llm-routing-minio.container @@ -18,6 +18,7 @@ Volume=DATA_ROOT_PLACEHOLDER/minio-data:/data HealthCmd=curl -f http://127.0.0.1:MINIO_S3_PORT_PLACEHOLDER/minio/health/live HealthInterval=15s HealthTimeout=5s +HealthOnFailure=kill HealthRetries=5 HealthStartPeriod=10s diff --git a/quadlets/llm-routing-postgres.container b/quadlets/llm-routing-postgres.container index 980a56bc..de7db982 100644 --- a/quadlets/llm-routing-postgres.container +++ b/quadlets/llm-routing-postgres.container @@ -17,6 +17,7 @@ Volume=DATA_ROOT_PLACEHOLDER/postgres-data:/var/lib/postgresql HealthCmd=pg_isready -U postgres -p POSTGRES_PORT_PLACEHOLDER HealthInterval=10s HealthTimeout=5s +HealthOnFailure=kill HealthRetries=5 HealthStartPeriod=15s diff --git a/quadlets/llm-routing-router.container b/quadlets/llm-routing-router.container index 3c39430b..16a0d9e6 100644 --- a/quadlets/llm-routing-router.container +++ b/quadlets/llm-routing-router.container @@ -44,6 +44,7 @@ Volume=WORKDIR_PLACEHOLDER/.env:/config/.env HealthCmd=python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:ROUTER_PORT_PLACEHOLDER/metrics')" HealthInterval=15s HealthTimeout=5s +HealthOnFailure=kill HealthRetries=5 HealthStartPeriod=240s diff --git a/quadlets/llm-routing-valkey-cache.container b/quadlets/llm-routing-valkey-cache.container index 97b497aa..8172afb7 100644 --- a/quadlets/llm-routing-valkey-cache.container +++ b/quadlets/llm-routing-valkey-cache.container @@ -16,6 +16,7 @@ Volume=DATA_ROOT_PLACEHOLDER/valkey-data:/data HealthCmd=valkey-cli -p VALKEY_CACHE_PORT_PLACEHOLDER ping HealthInterval=10s HealthTimeout=2s +HealthOnFailure=kill HealthRetries=5 HealthStartPeriod=5s diff --git a/quadlets/llm-routing-valkey-lf.container b/quadlets/llm-routing-valkey-lf.container index 21f3ccee..afbfe1ee 100644 --- a/quadlets/llm-routing-valkey-lf.container +++ b/quadlets/llm-routing-valkey-lf.container @@ -14,6 +14,7 @@ Volume=DATA_ROOT_PLACEHOLDER/redis-lf-data:/data HealthCmd=valkey-cli -p VALKEY_LF_PORT_PLACEHOLDER -a REDIS_AUTH_PLACEHOLDER ping HealthInterval=10s HealthTimeout=2s +HealthOnFailure=kill HealthRetries=5 HealthStartPeriod=5s diff --git a/scripts/README.md b/scripts/README.md index 089f036d..cbb735cf 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -7,11 +7,12 @@ This directory and the repository root contain various scripts used for stack or ## 1. Stack Orchestration & Backups ### `start-stack.sh` (Root Directory) -Unified startup and credential extraction script for the Podman Kubernetes container stack. +Unified startup and credential extraction script for the systemd Quadlet-managed Podman stack. - **Usage**: - - `./start-stack.sh` (Restart existing pod — fast, preserves logs) - - `./start-stack.sh --replace` (Stop + clean ports + redeploy pod from `pod.yaml`) + - `./start-stack.sh` (Restart the generated `llm-routing-pod.service`) + - `./start-stack.sh --replace` (Stop + clean ports + render/install Quadlets + daemon-reload + recreate stack) - `./start-stack.sh --full-rebuild` (Same as `--replace` + rebuild the triage router image; required for code changes in `router/`) +- Quadlet templates live in `quadlets/`; rendered owner-only units are installed under `~/.config/containers/systemd/llm-routing/`. Use `systemctl --user status llm-routing-pod.service --no-pager` and `journalctl --user -u llm-routing-router.service --no-pager` for lifecycle diagnostics. ### `scripts/backup.sh` Automated database backup script that runs before every stack deployment. Uses `pg_isready` to safely wait for database connections and manages timestamped backups under `backups/`. diff --git a/scripts/upgrade-prod.sh b/scripts/upgrade-prod.sh index fb48eb48..c1b42e78 100755 --- a/scripts/upgrade-prod.sh +++ b/scripts/upgrade-prod.sh @@ -4,7 +4,7 @@ # Flow: # 1. Fetch the latest release tag from GitHub # 2. Check out a clean copy of that tag to a temp dir -# 3. Rsync runtime files (pod.yaml, start-stack.sh, litellm/, router/, scripts/) +# 3. Rsync runtime files (pod.yaml, start-stack.sh, quadlets/, litellm/, router/, scripts/) # into ~/prod/LLM-Routing — data/, backups/, and .env are NEVER touched # 4. Redeploy with start-stack.sh --pull (pulls latest container images) # @@ -67,7 +67,7 @@ echo "📥 Cloning $REPO @ $TAG..." git clone -q --depth 1 --branch "$TAG" "https://github.com/$REPO.git" "$TEMP_DIR" # ── verify the tag has the files we need ── -for f in pod.yaml start-stack.sh litellm/ router/ scripts/; do +for f in pod.yaml start-stack.sh quadlets/ litellm/ router/ scripts/; do if [ ! -e "$TEMP_DIR/$f" ]; then echo "❌ Release $TAG is missing expected file/dir: $f" exit 1 @@ -80,7 +80,7 @@ if $DRY_RUN; then echo "── Dry run: files that would change ──" diff -rq "$TEMP_DIR/pod.yaml" "$PROD_DIR/pod.yaml" 2>/dev/null || echo " pod.yaml differs" diff -rq "$TEMP_DIR/start-stack.sh" "$PROD_DIR/start-stack.sh" 2>/dev/null || echo " start-stack.sh differs" - for dir in litellm router scripts; do + for dir in quadlets litellm router scripts; do diff -rq "$TEMP_DIR/$dir" "$PROD_DIR/$dir" 2>/dev/null || echo " $dir/ differs" done echo "── End dry run ──" @@ -90,7 +90,7 @@ fi # ── confirm ── echo "" echo "⚠️ This will OVERWRITE the following in $PROD_DIR:" -echo " pod.yaml start-stack.sh litellm/ router/ scripts/" +echo " pod.yaml start-stack.sh quadlets/ litellm/ router/ scripts/" echo " .env and data/ are NEVER touched." echo "" # Require interactive confirmation in TTY mode; auto-proceed in non-interactive @@ -141,6 +141,7 @@ fi # handles graceful shutdown (including pre-deploy backup) before redeploy. echo "📋 Syncing runtime files..." # Sync directories with --delete (clean stale files within each dir) +rsync -a --delete "$TEMP_DIR/quadlets/" "$PROD_DIR/quadlets/" rsync -a --delete "$TEMP_DIR/litellm/" "$PROD_DIR/litellm/" rsync -a --delete "$TEMP_DIR/router/" "$PROD_DIR/router/" rsync -a --delete "$TEMP_DIR/scripts/" "$PROD_DIR/scripts/" diff --git a/start-stack.sh b/start-stack.sh index 21d3a0b2..e121a43b 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -769,6 +769,7 @@ render_quadlets() { export MINIO_S3_PORT MINIO_CONSOLE_PORT local src_dir="${WORKDIR}/quadlets" mkdir -p "$QUADLET_DIR" + chmod 700 "$QUADLET_DIR" python3 - "$src_dir" "$QUADLET_DIR" <<'PY' import os, sys, urllib.parse, re, glob uid = os.getuid() @@ -843,7 +844,8 @@ for tpl in templates: out_path = os.path.join(out_dir, os.path.basename(tpl)) with open(out_path, "w", encoding="utf-8") as f: f.write(text) - os.chmod(out_path, 0o644) + # Rendered units include credentials; systemd user generator can read owner-only files. + os.chmod(out_path, 0o600) print(f" ✓ {os.path.basename(tpl)}") PY echo "✓ Quadlets rendered to ${QUADLET_DIR}" diff --git a/tests/test_quadlet_templates.py b/tests/test_quadlet_templates.py new file mode 100644 index 00000000..821f2cea --- /dev/null +++ b/tests/test_quadlet_templates.py @@ -0,0 +1,47 @@ +"""Static deployment-contract tests for the systemd Quadlet templates.""" +from pathlib import Path +import re + + +ROOT = Path(__file__).resolve().parent.parent +QUADLETS = ROOT / "quadlets" + + +def test_quadlet_inventory_and_pod_membership(): + containers = sorted(QUADLETS.glob("*.container")) + assert len(containers) == 9 + assert (QUADLETS / "llm-routing.pod").is_file() + for container in containers: + text = container.read_text() + assert "Pod=llm-routing.pod" in text, container.name + assert "ContainerName=POD_NAME_PLACEHOLDER-" in text, container.name + + +def test_liveness_healthchecks_restart_failed_containers(): + for container in sorted(QUADLETS.glob("*.container")): + text = container.read_text() + assert "HealthCmd=" in text, container.name + # Podman kills an unhealthy container; systemd Restart=always then replaces it. + assert "HealthOnFailure=kill" in text, container.name + assert "Restart=always" in text, container.name + + +def test_quadlet_templates_remain_env_rendered_and_secret_free(): + for template in [*QUADLETS.glob("*.container"), QUADLETS / "llm-routing.pod"]: + text = template.read_text() + assert "_PLACEHOLDER" in text, template.name + assert not re.search(r"sk-(?:or|lf|lit)-[A-Za-z0-9_-]{12,}", text), template.name + + +def test_upgrade_syncs_quadlets_before_quadlet_start_stack(): + script = (ROOT / "scripts" / "upgrade-prod.sh").read_text() + assert "quadlets/" in script + assert 'rsync -a --delete "$TEMP_DIR/quadlets/" "$PROD_DIR/quadlets/"' in script + assert "for f in pod.yaml start-stack.sh quadlets/" in script + + +def test_rendered_quadlets_are_owner_only(): + script = (ROOT / "start-stack.sh").read_text() + assert 'chmod 700 "$QUADLET_DIR"' in script + assert "os.chmod(out_path, 0o600)" in script + assert "systemctl --user daemon-reload" in script From e046088b409a263b65961614e07a99af3a34a2e5 Mon Sep 17 00:00:00 2001 From: boy Date: Wed, 22 Jul 2026 17:28:22 +0200 Subject: [PATCH 3/4] fix(routing): implement pure subdomain routing for LiteLLM, Langfuse, and Llama --- .env.dev | 5 +++-- router/main.py | 22 ++++++++-------------- start-stack.sh | 19 +++++++++++++++---- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/.env.dev b/.env.dev index 9dc4b1d0..c36b1736 100644 --- a/.env.dev +++ b/.env.dev @@ -6,8 +6,9 @@ POD_NAME="dev-router-pod" # Public URLs BASEURL="dev.vendeuvre.lan" -BASE_URL="dev.vendeuvre.lan" -PUBLIC_BASE_URL="https://dev.vendeuvre.lan/llm-routing" +PUBLIC_BASE_URL="https://dev.vendeuvre.lan" +PROXY_BASE_URL="https://litellm.dev.vendeuvre.lan" +NEXTAUTH_URL="https://langfuse.dev.vendeuvre.lan" # Dev port assignments (+10 offset from production) ROUTER_PORT="5010" diff --git a/router/main.py b/router/main.py index 567f6f09..d2c88638 100644 --- a/router/main.py +++ b/router/main.py @@ -3546,21 +3546,15 @@ def resolve_external_urls(request: Request) -> tuple[str, str, str]: is_valid_external = external_host == domain or external_host.endswith("." + domain) is_valid_base = request.base_url.hostname == domain or (request.base_url.hostname or "").endswith("." + domain) - if is_valid_external: - # Centralized base URL path under subdomain/reverse proxy + if is_valid_external or is_valid_base: + host_val = external_host if is_valid_external else (request.base_url.hostname or "vendeuvre.lan") + host_base = host_val.replace("dashboard.", "") + if host_base.startswith("litellm.") or host_base.startswith("langfuse.") or host_base.startswith("llama."): + host_base = re.sub(r"^(litellm|langfuse|llama)\.", "", host_base) return ( - f"{external_scheme}://{external_netloc}/llm-routing/langfuse", - f"{external_scheme}://{external_netloc}/llm-routing/litellm/ui", - f"{external_scheme}://{external_netloc}/llm-routing/llama/" - ) - elif is_valid_base: - parsed_netloc = urlparse(f"{external_scheme}://{request.url.netloc}") - netloc = request.url.netloc if parsed_netloc.hostname else "localhost" - base = f"{external_scheme}://{netloc}" - return ( - f"{base}/llm-routing/langfuse", - f"{base}/llm-routing/litellm/ui", - f"{base}/llm-routing/llama/" + f"{external_scheme}://langfuse.{host_base}", + f"{external_scheme}://litellm.{host_base}/ui/", + f"{external_scheme}://llama.{host_base}/" ) else: # Local development fallback: derive schemes, ports, and paths dynamically from configuration constants diff --git a/start-stack.sh b/start-stack.sh index e121a43b..a04d6c7d 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -708,13 +708,24 @@ text = text.replace("MINIO_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["MINIO_ text = text.replace("LANGFUSE_INIT_USER_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["LANGFUSE_INIT_USER_PASSWORD"])) text = text.replace("REDIS_AUTH_PLACEHOLDER", yaml_scalar(os.environ["REDIS_AUTH"])) text = text.replace("CLICKHOUSE_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["CLICKHOUSE_PASSWORD"])) -# Derive PROXY_BASE_URL from PUBLIC_BASE_URL +# Derive PROXY_BASE_URL and NEXTAUTH_URL (favoring explicit env or subdomain formatting) public_base_url = os.environ["PUBLIC_BASE_URL"].rstrip("/") -proxy_base_url = f"{public_base_url}/litellm" +parsed_pub = urllib.parse.urlparse(public_base_url) +scheme = parsed_pub.scheme or "https" +host = parsed_pub.netloc or parsed_pub.path.split("/")[0] or "vendeuvre.lan" + +if "PROXY_BASE_URL" in os.environ: + proxy_base_url = os.environ["PROXY_BASE_URL"] +else: + proxy_base_url = f"{scheme}://litellm.{host}" + +if "NEXTAUTH_URL" in os.environ: + nextauth_url = os.environ["NEXTAUTH_URL"] +else: + nextauth_url = f"{scheme}://langfuse.{host}" + text = text.replace("PROXY_BASE_URL_PLACEHOLDER", yaml_scalar(proxy_base_url)) text = text.replace("PUBLIC_BASE_URL_PLACEHOLDER", yaml_scalar(os.environ["PUBLIC_BASE_URL"])) -# Derive NEXTAUTH_URL from PUBLIC_BASE_URL (Langfuse needs the public URL for OAuth redirects) -nextauth_url = f"{public_base_url}/langfuse" text = text.replace("NEXTAUTH_URL_PLACEHOLDER", yaml_scalar(nextauth_url)) text = text.replace("ROUTING_DOMAIN_PLACEHOLDER", yaml_scalar(os.environ["ROUTING_DOMAIN"])) # Raw replacements (no quoting — used for pod name, data root, and integer port values) From f8a4a535af8764371af743360684c5332ce82b80 Mon Sep 17 00:00:00 2001 From: boy Date: Thu, 23 Jul 2026 00:03:17 +0200 Subject: [PATCH 4/4] fix(routing): address subdomain URL review findings --- router/main.py | 25 ++++++++++++++++------ router/tests/test_resolve_external_urls.py | 16 +++++++------- start-stack.sh | 10 +++++---- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/router/main.py b/router/main.py index d2c88638..a66136f9 100644 --- a/router/main.py +++ b/router/main.py @@ -3547,14 +3547,25 @@ def resolve_external_urls(request: Request) -> tuple[str, str, str]: is_valid_base = request.base_url.hostname == domain or (request.base_url.hostname or "").endswith("." + domain) if is_valid_external or is_valid_base: - host_val = external_host if is_valid_external else (request.base_url.hostname or "vendeuvre.lan") - host_base = host_val.replace("dashboard.", "") - if host_base.startswith("litellm.") or host_base.startswith("langfuse.") or host_base.startswith("llama."): - host_base = re.sub(r"^(litellm|langfuse|llama)\.", "", host_base) + # Use configured routing domain if a proxy supplies no request hostname. + # Preserve an explicit public port from the selected netloc so dashboard + # links remain valid behind non-standard TLS listeners. + host_val = external_host if is_valid_external else (request.base_url.hostname or domain) + netloc_val = external_netloc if is_valid_external else (request.base_url.netloc or host_val) + parsed_public = urlparse(f"{external_scheme}://{netloc_val}") + try: + port_suffix = f":{parsed_public.port}" if parsed_public.port else "" + except ValueError: + logger.warning("Invalid public port in netloc %r; omitting port", netloc_val) + port_suffix = "" + + host_base = re.sub(r"^dashboard\.", "", host_val) + host_base = re.sub(r"^(?:litellm|langfuse|llama)\.", "", host_base) + service_netloc = f"{host_base}{port_suffix}" return ( - f"{external_scheme}://langfuse.{host_base}", - f"{external_scheme}://litellm.{host_base}/ui/", - f"{external_scheme}://llama.{host_base}/" + f"{external_scheme}://langfuse.{service_netloc}", + f"{external_scheme}://litellm.{service_netloc}/ui/", + f"{external_scheme}://llama.{service_netloc}/" ) else: # Local development fallback: derive schemes, ports, and paths dynamically from configuration constants diff --git a/router/tests/test_resolve_external_urls.py b/router/tests/test_resolve_external_urls.py index 8cc79f0d..9e45d1a2 100644 --- a/router/tests/test_resolve_external_urls.py +++ b/router/tests/test_resolve_external_urls.py @@ -45,9 +45,9 @@ def test_resolve_with_public_base_url_vaild_domain(): req = MockRequest() lf, ll, lm = resolve_external_urls(req) - assert lf == "https://app.vendeuvre.lan/llm-routing/langfuse" - assert ll == "https://app.vendeuvre.lan/llm-routing/litellm/ui" - assert lm == "https://app.vendeuvre.lan/llm-routing/llama/" + assert lf == "https://langfuse.app.vendeuvre.lan" + assert ll == "https://litellm.app.vendeuvre.lan/ui/" + assert lm == "https://llama.app.vendeuvre.lan/" def test_resolve_with_valid_base_request(): @@ -64,11 +64,11 @@ def test_resolve_with_valid_base_request(): lf, ll, lm = resolve_external_urls(req) - # Netloc should use [::1]:8000 because request base host is valid, - # and netloc was validated successfully via urlparse. - assert lf == "http://[::1]:8000/llm-routing/langfuse" - assert ll == "http://[::1]:8000/llm-routing/litellm/ui" - assert lm == "http://[::1]:8000/llm-routing/llama/" + # Valid request host is converted to service subdomains while preserving + # its explicit port, rather than leaking the configured IPv6 fallback. + assert lf == "http://langfuse.sub.vendeuvre.lan:8000" + assert ll == "http://litellm.sub.vendeuvre.lan:8000/ui/" + assert lm == "http://llama.sub.vendeuvre.lan:8000/" def test_local_fallback_ipv6(): diff --git a/start-stack.sh b/start-stack.sh index a04d6c7d..f828a37c 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -787,11 +787,13 @@ uid = os.getuid() src_dir, out_dir = sys.argv[1], sys.argv[2] encoded_pg = urllib.parse.quote(os.environ['POSTGRES_PASSWORD'], safe="") -# Derive PROXY_BASE_URL / NEXTAUTH_URL exactly as render_pod_yaml does -# (path-based on master; subdomain switch lands with the routing branch) +# Derive PROXY_BASE_URL / NEXTAUTH_URL exactly as render_pod_yaml does. public_base_url = os.environ["PUBLIC_BASE_URL"].rstrip("/") -proxy_base_url = f"{public_base_url}/litellm" -nextauth_url = f"{public_base_url}/langfuse" +parsed_pub = urllib.parse.urlparse(public_base_url) +scheme = parsed_pub.scheme or "https" +host = parsed_pub.netloc or parsed_pub.path.split("/")[0] or os.environ["ROUTING_DOMAIN"] +proxy_base_url = os.environ.get("PROXY_BASE_URL") or f"{scheme}://litellm.{host}" +nextauth_url = os.environ.get("NEXTAUTH_URL") or f"{scheme}://langfuse.{host}" repl = { "WORKDIR_PLACEHOLDER": os.environ["WORKDIR"],