From e4e5780211d5f425b640ad2a5285583b54085933 Mon Sep 17 00:00:00 2001 From: boy Date: Thu, 23 Jul 2026 02:28:43 +0200 Subject: [PATCH 1/3] fix(deploy): isolate dev and prod Quadlet namespaces --- .env.dev | 3 ++ README.md | 10 +++++++ scripts/README.md | 2 +- start-stack.sh | 50 +++++++++++++++++++++++++++------ tests/test_quadlet_templates.py | 14 ++++++++- 5 files changed, 68 insertions(+), 11 deletions(-) diff --git a/.env.dev b/.env.dev index 4565148a..3ee30ba0 100644 --- a/.env.dev +++ b/.env.dev @@ -31,3 +31,6 @@ ROUTER_IMAGE="localhost/llm-routing-dev:latest" # reaches its local listener directly rather than any TLS-terminated hostname. LLAMA_CLASSIFIER_URL="http://127.0.0.1:8083/v1" LLAMA_SERVER_URL="http://127.0.0.1:8083" + +# Distinct systemd Quadlet namespace; never share generated units with prod. +QUADLET_NAMESPACE="llm-routing-dev" diff --git a/README.md b/README.md index a1324e18..e88941c8 100644 --- a/README.md +++ b/README.md @@ -855,6 +855,16 @@ Tests cover: Requires `PUBLIC_BASE_URL` in `.env` for canonical URL tests. The router remains under its configured path (for example `https://x570.vendeuvre.lan/llm-routing`), while the verifier derives service URLs from its host: `https://litellm./ui/`, `https://langfuse./`, and `https://llama./health`. Dev `.env.dev` already has it; prod `.env` should include `PUBLIC_BASE_URL="https://x570.vendeuvre.lan/llm-routing"`. The dev local-model safety net uses the host-networked local listener: `LLAMA_CLASSIFIER_URL=http://127.0.0.1:8083/v1` and `LLAMA_SERVER_URL=http://127.0.0.1:8083`; it must not depend on TLS-terminated dev or production hostnames. +## Environment-isolated Quadlet deployment + +Dev and production use distinct Quadlet namespaces because generated systemd +unit names are global within the user manager. Dev renders units under +`~/.config/containers/systemd/llm-routing-dev/` and uses +`llm-routing-dev-pod.service`; production uses +`~/.config/containers/systemd/llm-routing-prod/` and +`llm-routing-prod-pod.service`. Their pod/container names, ports, data roots, +and rendered configuration remain separate. + ## 10. Performance Benchmarks Through our local benchmarks, the following performance characteristics have been achieved: diff --git a/scripts/README.md b/scripts/README.md index cbb735cf..ffeac893 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -12,7 +12,7 @@ Unified startup and credential extraction script for the systemd Quadlet-managed - `./start-stack.sh` (Restart the generated `llm-routing-pod.service`) - `./start-stack.sh --replace` (Stop + clean ports + render/install Quadlets + daemon-reload + recreate stack) - `./start-stack.sh --full-rebuild` (Same as `--replace` + rebuild the triage router image; required for code changes in `router/`) -- Quadlet templates live in `quadlets/`; rendered owner-only units are installed under `~/.config/containers/systemd/llm-routing/`. Use `systemctl --user status llm-routing-pod.service --no-pager` and `journalctl --user -u llm-routing-router.service --no-pager` for lifecycle diagnostics. +- Quadlet templates live in `quadlets/`; rendered owner-only units use environment-specific namespaces: dev under `~/.config/containers/systemd/llm-routing-dev/` and prod under `~/.config/containers/systemd/llm-routing-prod/`. Dev uses `llm-routing-dev-pod.service`; prod uses `llm-routing-prod-pod.service`. Use the matching `systemctl --user status -pod.service --no-pager` and `journalctl --user -u -router.service --no-pager` for lifecycle diagnostics. ### `scripts/backup.sh` Automated database backup script that runs before every stack deployment. Uses `pg_isready` to safely wait for database connections and manages timestamped backups under `backups/`. diff --git a/start-stack.sh b/start-stack.sh index e2b1936e..8f4c2b2d 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -77,6 +77,11 @@ if [ -n "${DEV_ENV_FILE:-}" ] && [ -f "$DEV_ENV_FILE" ]; then set +a fi +# Quadlet namespace is environment-specific. This prevents dev and prod from +# sharing rendered files or generated systemd unit names. +QUADLET_NAMESPACE="${QUADLET_NAMESPACE:-llm-routing-prod}" +export QUADLET_NAMESPACE + # Port assignments — read from env (set by .env or .env.dev) with prod defaults POD_NAME="${POD_NAME:-prod-router-pod}" ROUTER_PORT="${ROUTER_PORT:-5000}" @@ -533,7 +538,9 @@ verify_stack_health() { # ── Stack ownership and teardown ── # Keep the generated Quadlet unit name in one place: it is used for ownership # detection, lifecycle operations, and user-facing diagnostics. -LLM_ROUTING_POD_UNIT="llm-routing-pod.service" +LLM_ROUTING_POD_UNIT="${QUADLET_NAMESPACE}-pod.service" +LEGACY_LLM_ROUTING_POD_UNIT="llm-routing-pod.service" +QUADLET_DIR="${HOME}/.config/containers/systemd/${QUADLET_NAMESPACE}" # Quadlet-managed pods carry PODMAN_SYSTEMD_UNIT on their infra container. # Consult that metadata rather than inferring ownership from active state: a # stopped or failed generated unit is still Quadlet-owned and must be reconciled @@ -542,7 +549,7 @@ stack_ownership() { local infra_unit if podman pod exists "${POD_NAME}" 2>/dev/null; then infra_unit=$(podman pod inspect "${POD_NAME}" --format '{{.InfraContainerID}}' 2>/dev/null | xargs -r podman inspect --format '{{ index .Config.Labels "PODMAN_SYSTEMD_UNIT" }}' 2>/dev/null || true) - if [[ "$infra_unit" == "$LLM_ROUTING_POD_UNIT" ]]; then + if [[ "$infra_unit" == "$LLM_ROUTING_POD_UNIT" || "$infra_unit" == "$LEGACY_LLM_ROUTING_POD_UNIT" ]]; then printf 'quadlet\n' else printf 'legacy\n' @@ -554,6 +561,20 @@ stack_ownership() { fi } +stop_quadlet_units() { + systemctl --user stop "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true + if [[ "$LLM_ROUTING_POD_UNIT" != "$LEGACY_LLM_ROUTING_POD_UNIT" ]]; then + systemctl --user stop "$LEGACY_LLM_ROUTING_POD_UNIT" 2>/dev/null || true + fi +} + +reset_quadlet_units() { + systemctl --user reset-failed "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true + if [[ "$LLM_ROUTING_POD_UNIT" != "$LEGACY_LLM_ROUTING_POD_UNIT" ]]; then + systemctl --user reset-failed "$LEGACY_LLM_ROUTING_POD_UNIT" 2>/dev/null || true + fi +} + require_user_systemd() { if ! systemctl --user show-environment >/dev/null 2>&1; then echo "❌ Error: the Quadlet deployment requires a reachable systemd --user manager." >&2 @@ -569,8 +590,8 @@ safe_pod_teardown() { ownership=$(stack_ownership) if [[ "$ownership" == "quadlet" ]]; then echo "🛑 Reconciling Quadlet-owned stack (unit may be active, inactive, or failed)..." - systemctl --user stop "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true - systemctl --user reset-failed "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true + stop_quadlet_units + reset_quadlet_units podman pod rm -f "${POD_NAME}" 2>/dev/null || true cleanup_zombie_ports echo "✓ Quadlet stack stopped, state reconciled, ports cleaned" @@ -680,8 +701,6 @@ render_router_config() { # convention as pod.yaml) into ~/.config/containers/systemd/llm-routing/ and # lets systemd's podman-user-generator turn them into real units. # Quadlet values are bare scalars (not YAML) so plain string replacement is used. -QUADLET_DIR="${HOME}/.config/containers/systemd/llm-routing" - render_quadlets() { export WORKDIR HOME LITELLM_MASTER_KEY UI_USERNAME UI_PASSWORD export POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY @@ -705,6 +724,7 @@ render_quadlets() { import os, sys, urllib.parse, re, glob, shutil, tempfile uid = os.getuid() src_dir, out_dir = sys.argv[1], sys.argv[2] +namespace = os.environ["QUADLET_NAMESPACE"] encoded_pg = urllib.parse.quote(os.environ['POSTGRES_PASSWORD'], safe="") # Derived by derive_external_service_urls(), shared with render_pod_yaml. @@ -766,6 +786,17 @@ try: text = f.read() for ph, val in repl.items(): text = text.replace(ph, str(val)) + # Quadlet generated unit names are global in the user systemd manager. + # Namespace both filenames and internal dependencies to isolate dev/prod. + # Namespace Quadlet identifiers only. Do not rewrite image names, URL + # paths, or other configuration values containing "llm-routing". + text = re.sub( + r"llm-routing-(?=(?:pod|clickhouse|langfuse|litellm|minio|postgres|router|valkey))", + namespace + "-", + text, + ) + text = text.replace("llm-routing.pod", namespace + ".pod") + text = text.replace("llm-routing-pod.service", namespace + "-pod.service") unresolved = sorted(set(re.findall(r"\b[A-Z0-9_]+_PLACEHOLDER\b", text))) if unresolved: sys.stderr.write(f"Error: Unresolved placeholders in {os.path.basename(tpl)}: {', '.join(unresolved)}\n") @@ -777,7 +808,8 @@ try: value = match.group(1).replace("\\", "\\\\").replace('"', '\\"') return f'Environment="{value}"' text = re.sub(r"(?m)^Environment=(.*)$", quote_environment, text) - staged_path = os.path.join(staging_dir, os.path.basename(tpl)) + rendered_name = os.path.basename(tpl).replace("llm-routing", namespace) + staged_path = os.path.join(staging_dir, rendered_name) with open(staged_path, "w", encoding="utf-8") as f: f.write(text) # Rendered units include credentials; systemd user generator can read owner-only files. @@ -785,7 +817,7 @@ try: # All templates are now valid. Replace individual files atomically, then # remove stale units; a failed render above leaves the prior unit set intact. - rendered_names = {os.path.basename(tpl) for tpl in templates} + rendered_names = {os.path.basename(tpl).replace("llm-routing", namespace) for tpl in templates} for name in rendered_names: os.replace(os.path.join(staging_dir, name), os.path.join(out_dir, name)) for stale in glob.glob(os.path.join(out_dir, "*.pod")) + glob.glob(os.path.join(out_dir, "*.container")): @@ -857,7 +889,7 @@ if [[ "$STACK_OWNERSHIP" != "absent" ]]; then if [[ "$STACK_OWNERSHIP" == "quadlet" ]]; then require_user_systemd || exit 1 echo "🔄 Restarting Quadlet-owned stack via systemd..." - systemctl --user reset-failed "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true + reset_quadlet_units if ! systemctl --user restart "$LLM_ROUTING_POD_UNIT"; then echo "❌ Error: failed to restart ${LLM_ROUTING_POD_UNIT}" >&2 echo " Hint: run 'systemctl --user status ${LLM_ROUTING_POD_UNIT} --no-pager' to inspect the failure" >&2 diff --git a/tests/test_quadlet_templates.py b/tests/test_quadlet_templates.py index 22f02b6c..cd294afe 100644 --- a/tests/test_quadlet_templates.py +++ b/tests/test_quadlet_templates.py @@ -44,7 +44,8 @@ def test_rendered_quadlets_are_owner_only(): script = (ROOT / "start-stack.sh").read_text() assert 'chmod 700 "$QUADLET_DIR"' in script assert "os.chmod(staged_path, 0o600)" in script - assert 'LLM_ROUTING_POD_UNIT="llm-routing-pod.service"' in script + assert 'LLM_ROUTING_POD_UNIT="${QUADLET_NAMESPACE}-pod.service"' in script + assert 'QUADLET_DIR="${HOME}/.config/containers/systemd/${QUADLET_NAMESPACE}"' in script assert "systemctl --user daemon-reload" in script assert "stack_ownership()" in script assert "PODMAN_SYSTEMD_UNIT" in script @@ -87,3 +88,14 @@ def test_quadlet_deployment_enforces_prerequisite_and_restart_failures(): assert "require_user_systemd || exit 1" in script assert "failed to derive external service URLs for Quadlet rendering" in script assert 'if ! podman pod restart "${POD_NAME}"; then' in script + + +def test_quadlet_namespace_is_environment_specific(): + dev_env = (ROOT / ".env.dev").read_text() + prod_env = (ROOT / ".env").read_text() + assert 'QUADLET_NAMESPACE="llm-routing-dev"' in dev_env + assert 'QUADLET_NAMESPACE="llm-routing-prod"' in prod_env + script = (ROOT / "start-stack.sh").read_text() + assert 'r"llm-routing-(?=(?:pod|clickhouse|langfuse|litellm|minio|postgres|router|valkey))"' in script + assert 'text = text.replace("llm-routing.pod", namespace + ".pod")' in script + assert 'rendered_name = os.path.basename(tpl).replace("llm-routing", namespace)' in script From 919f22a3b33d163e47587a70475826d4d13aa570 Mon Sep 17 00:00:00 2001 From: boy Date: Thu, 23 Jul 2026 02:32:43 +0200 Subject: [PATCH 2/3] test(deploy): avoid requiring ignored production env --- tests/test_quadlet_templates.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_quadlet_templates.py b/tests/test_quadlet_templates.py index cd294afe..660afb26 100644 --- a/tests/test_quadlet_templates.py +++ b/tests/test_quadlet_templates.py @@ -92,9 +92,10 @@ def test_quadlet_deployment_enforces_prerequisite_and_restart_failures(): def test_quadlet_namespace_is_environment_specific(): dev_env = (ROOT / ".env.dev").read_text() - prod_env = (ROOT / ".env").read_text() + prod_namespace = "llm-routing-prod" assert 'QUADLET_NAMESPACE="llm-routing-dev"' in dev_env - assert 'QUADLET_NAMESPACE="llm-routing-prod"' in prod_env + assert 'QUADLET_NAMESPACE="${QUADLET_NAMESPACE:-llm-routing-prod}"' in (ROOT / "start-stack.sh").read_text() + assert prod_namespace in (ROOT / "start-stack.sh").read_text() script = (ROOT / "start-stack.sh").read_text() assert 'r"llm-routing-(?=(?:pod|clickhouse|langfuse|litellm|minio|postgres|router|valkey))"' in script assert 'text = text.replace("llm-routing.pod", namespace + ".pod")' in script From 9c05821c24558a1428164cd540e60db88b5b3116 Mon Sep 17 00:00:00 2001 From: boy Date: Thu, 23 Jul 2026 02:46:13 +0200 Subject: [PATCH 3/3] fix(deploy): preserve exact Quadlet ownership during migration --- start-stack.sh | 23 +++++++++++++++-------- tests/test_quadlet_templates.py | 7 +++++++ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/start-stack.sh b/start-stack.sh index 8f4c2b2d..93a8d24c 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -80,6 +80,10 @@ fi # Quadlet namespace is environment-specific. This prevents dev and prod from # sharing rendered files or generated systemd unit names. QUADLET_NAMESPACE="${QUADLET_NAMESPACE:-llm-routing-prod}" +if [[ ! "$QUADLET_NAMESPACE" =~ ^[a-z0-9][a-z0-9-]*$ ]]; then + echo "❌ Error: QUADLET_NAMESPACE must contain only lowercase letters, digits, and hyphens" >&2 + exit 1 +fi export QUADLET_NAMESPACE # Port assignments — read from env (set by .env or .env.dev) with prod defaults @@ -550,12 +554,14 @@ stack_ownership() { if podman pod exists "${POD_NAME}" 2>/dev/null; then infra_unit=$(podman pod inspect "${POD_NAME}" --format '{{.InfraContainerID}}' 2>/dev/null | xargs -r podman inspect --format '{{ index .Config.Labels "PODMAN_SYSTEMD_UNIT" }}' 2>/dev/null || true) if [[ "$infra_unit" == "$LLM_ROUTING_POD_UNIT" || "$infra_unit" == "$LEGACY_LLM_ROUTING_POD_UNIT" ]]; then - printf 'quadlet\n' + printf 'quadlet:%s\n' "$infra_unit" else printf 'legacy\n' fi elif systemctl --user show "$LLM_ROUTING_POD_UNIT" -p LoadState --value 2>/dev/null | grep -qxv 'not-found'; then - printf 'quadlet\n' + printf 'quadlet:%s\n' "$LLM_ROUTING_POD_UNIT" + elif systemctl --user show "$LEGACY_LLM_ROUTING_POD_UNIT" -p LoadState --value 2>/dev/null | grep -qxv 'not-found'; then + printf 'quadlet:%s\n' "$LEGACY_LLM_ROUTING_POD_UNIT" else printf 'absent\n' fi @@ -588,10 +594,11 @@ require_user_systemd() { safe_pod_teardown() { local ownership ownership=$(stack_ownership) - if [[ "$ownership" == "quadlet" ]]; then + if [[ "$ownership" == quadlet:* ]]; then + local owner_unit="${ownership#quadlet:}" echo "🛑 Reconciling Quadlet-owned stack (unit may be active, inactive, or failed)..." - stop_quadlet_units - reset_quadlet_units + systemctl --user stop "$owner_unit" 2>/dev/null || true + systemctl --user reset-failed "$owner_unit" 2>/dev/null || true podman pod rm -f "${POD_NAME}" 2>/dev/null || true cleanup_zombie_ports echo "✓ Quadlet stack stopped, state reconciled, ports cleaned" @@ -886,11 +893,11 @@ if [[ "$STACK_OWNERSHIP" != "absent" ]]; then echo "🚀 Deploying replacement pod from YAML..." deploy_fresh_pod else - if [[ "$STACK_OWNERSHIP" == "quadlet" ]]; then + if [[ "$STACK_OWNERSHIP" == quadlet:* ]]; then require_user_systemd || exit 1 + owner_unit="${STACK_OWNERSHIP#quadlet:}" echo "🔄 Restarting Quadlet-owned stack via systemd..." - reset_quadlet_units - if ! systemctl --user restart "$LLM_ROUTING_POD_UNIT"; then + if ! systemctl --user restart "$owner_unit"; then echo "❌ Error: failed to restart ${LLM_ROUTING_POD_UNIT}" >&2 echo " Hint: run 'systemctl --user status ${LLM_ROUTING_POD_UNIT} --no-pager' to inspect the failure" >&2 exit 1 diff --git a/tests/test_quadlet_templates.py b/tests/test_quadlet_templates.py index 660afb26..e47cabe5 100644 --- a/tests/test_quadlet_templates.py +++ b/tests/test_quadlet_templates.py @@ -100,3 +100,10 @@ def test_quadlet_namespace_is_environment_specific(): assert 'r"llm-routing-(?=(?:pod|clickhouse|langfuse|litellm|minio|postgres|router|valkey))"' in script assert 'text = text.replace("llm-routing.pod", namespace + ".pod")' in script assert 'rendered_name = os.path.basename(tpl).replace("llm-routing", namespace)' in script + + +def test_namespace_is_validated_and_ownership_preserves_exact_unit(): + script = (ROOT / "start-stack.sh").read_text() + assert 'QUADLET_NAMESPACE" =~ ^[a-z0-9][a-z0-9-]*$' in script + assert "printf 'quadlet:%s\\n' \"$infra_unit\"" in script + assert 'owner_unit="${STACK_OWNERSHIP#quadlet:}"' in script