fix(deploy): finish Quadlet isolation hardening

.env.dev

@@ -31,3 +31,6 @@ ROUTER_IMAGE="localhost/llm-routing-dev:latest"
 # reaches its local listener directly rather than any TLS-terminated hostname.
 LLAMA_CLASSIFIER_URL="http://127.0.0.1:8083/v1"
 LLAMA_SERVER_URL="http://127.0.0.1:8083"
+
+# Distinct systemd Quadlet namespace; never share generated units with prod.
+QUADLET_NAMESPACE="llm-routing-dev"

README.md

@@ -395,9 +395,9 @@ Run the startup script from the root of the repository:
 ./start-stack.sh --full-rebuild  # Same as --replace plus rebuild the router image
 
 # Inspect the generated systemd units and their logs
-systemctl --user status llm-routing-pod.service --no-pager
-systemctl --user list-units 'llm-routing-*' --no-pager
-journalctl --user -u llm-routing-router.service -n 100 --no-pager
+systemctl --user status llm-routing-prod-pod.service --no-pager  # or llm-routing-dev-pod.service
+systemctl --user list-units 'llm-routing-*' --no-pager  # filter for dev/prod namespaces
+journalctl --user -u llm-routing-prod-router.service -n 100 --no-pager  # or llm-routing-dev-router.service
 ```
 *Note: If running for the first time, the script will prompt you for your `OpenRouter API Key`, securely saving it inside `.env` with restrictive permissions (`chmod 600`). The script also automatically generates and persists secure random secrets (`LITELLM_MASTER_KEY`, `POSTGRES_PASSWORD`, `NEXTAUTH_SECRET`, `SALT`, `ENCRYPTION_KEY`, and `ROUTER_API_KEY`) to this file on startup if they are missing.*
 
@@ -855,6 +855,19 @@ Tests cover:
 
 Requires `PUBLIC_BASE_URL` in `.env` for canonical URL tests. The router remains under its configured path (for example `https://x570.vendeuvre.lan/llm-routing`), while the verifier derives service URLs from its host: `https://litellm.<host>/ui/`, `https://langfuse.<host>/`, and `https://llama.<host>/health`. Dev `.env.dev` already has it; prod `.env` should include `PUBLIC_BASE_URL="https://x570.vendeuvre.lan/llm-routing"`. The dev local-model safety net uses the host-networked local listener: `LLAMA_CLASSIFIER_URL=http://127.0.0.1:8083/v1` and `LLAMA_SERVER_URL=http://127.0.0.1:8083`; it must not depend on TLS-terminated dev or production hostnames.
 
+## Environment-isolated Quadlet deployment
+
+Dev and production use distinct Quadlet namespaces because generated systemd
+unit names are global within the user manager. Dev renders units under
+`~/.config/containers/systemd/llm-routing-dev/` and uses
+`llm-routing-dev-pod.service`; production uses
+`~/.config/containers/systemd/llm-routing-prod/` and
+`llm-routing-prod-pod.service`. Their pod/container names, ports, data roots,
+and rendered configuration remain separate. Unless overridden explicitly,
+`DATA_ROOT` is `${WORKDIR}/data`; because dev and production run from separate
+`~/dev/` and `~/prod/` worktrees, their persistent data and rendered configs are
+also physically separate.
+
 ## 10. Performance Benchmarks
 
 Through our local benchmarks, the following performance characteristics have been achieved:

scripts/README.md

@@ -9,10 +9,10 @@ This directory and the repository root contain various scripts used for stack or
 ### `start-stack.sh` (Root Directory)
 Unified startup and credential extraction script for the systemd Quadlet-managed Podman stack.
 - **Usage**:
-  - `./start-stack.sh` (Restart the generated `llm-routing-pod.service`)
+  - `./start-stack.sh` (Restart the generated environment-specific pod service)
   - `./start-stack.sh --replace` (Stop + clean ports + render/install Quadlets + daemon-reload + recreate stack)
   - `./start-stack.sh --full-rebuild` (Same as `--replace` + rebuild the triage router image; required for code changes in `router/`)
-- Quadlet templates live in `quadlets/`; rendered owner-only units are installed under `~/.config/containers/systemd/llm-routing/`. Use `systemctl --user status llm-routing-pod.service --no-pager` and `journalctl --user -u llm-routing-router.service --no-pager` for lifecycle diagnostics.
+- Quadlet templates live in `quadlets/`; rendered owner-only units use environment-specific namespaces: dev under `~/.config/containers/systemd/llm-routing-dev/` and prod under `~/.config/containers/systemd/llm-routing-prod/`. Dev uses `llm-routing-dev-pod.service`; prod uses `llm-routing-prod-pod.service`. Use the matching `systemctl --user status <namespace>-pod.service --no-pager` and `journalctl --user -u <namespace>-router.service --no-pager` for lifecycle diagnostics.
 
 ### `scripts/backup.sh`
 Automated database backup script that runs before every stack deployment. Uses `pg_isready` to safely wait for database connections and manages timestamped backups under `backups/`.

start-stack.sh

@@ -77,6 +77,15 @@ if [ -n "${DEV_ENV_FILE:-}" ] && [ -f "$DEV_ENV_FILE" ]; then
     set +a
 fi
 
+# Quadlet namespace is environment-specific. This prevents dev and prod from
+# sharing rendered files or generated systemd unit names.
+QUADLET_NAMESPACE="${QUADLET_NAMESPACE:-llm-routing-prod}"
+if [[ ! "$QUADLET_NAMESPACE" =~ ^[a-z0-9][a-z0-9-]*$ ]]; then
+    echo "❌ Error: QUADLET_NAMESPACE must contain only lowercase letters, digits, and hyphens" >&2
+    exit 1
+fi
+export QUADLET_NAMESPACE
+
 # Port assignments — read from env (set by .env or .env.dev) with prod defaults
 POD_NAME="${POD_NAME:-prod-router-pod}"
 ROUTER_PORT="${ROUTER_PORT:-5000}"
@@ -533,7 +542,9 @@ verify_stack_health() {
 # ── Stack ownership and teardown ──
 # Keep the generated Quadlet unit name in one place: it is used for ownership
 # detection, lifecycle operations, and user-facing diagnostics.
-LLM_ROUTING_POD_UNIT="llm-routing-pod.service"
+LLM_ROUTING_POD_UNIT="${QUADLET_NAMESPACE}-pod.service"
+LEGACY_LLM_ROUTING_POD_UNIT="llm-routing-pod.service"
+QUADLET_DIR="${HOME}/.config/containers/systemd/${QUADLET_NAMESPACE}"
 # Quadlet-managed pods carry PODMAN_SYSTEMD_UNIT on their infra container.
 # Consult that metadata rather than inferring ownership from active state: a
 # stopped or failed generated unit is still Quadlet-owned and must be reconciled
@@ -542,13 +553,15 @@ stack_ownership() {
     local infra_unit
     if podman pod exists "${POD_NAME}" 2>/dev/null; then
         infra_unit=$(podman pod inspect "${POD_NAME}" --format '{{.InfraContainerID}}' 2>/dev/null | xargs -r podman inspect --format '{{ index .Config.Labels "PODMAN_SYSTEMD_UNIT" }}' 2>/dev/null || true)
-        if [[ "$infra_unit" == "$LLM_ROUTING_POD_UNIT" ]]; then
-            printf 'quadlet\n'
+        if [[ "$infra_unit" == "$LLM_ROUTING_POD_UNIT" || "$infra_unit" == "$LEGACY_LLM_ROUTING_POD_UNIT" ]]; then
+            printf 'quadlet:%s\n' "$infra_unit"
         else
             printf 'legacy\n'
         fi
     elif systemctl --user show "$LLM_ROUTING_POD_UNIT" -p LoadState --value 2>/dev/null | grep -qxv 'not-found'; then
-        printf 'quadlet\n'
+        printf 'quadlet:%s\n' "$LLM_ROUTING_POD_UNIT"
+    elif systemctl --user show "$LEGACY_LLM_ROUTING_POD_UNIT" -p LoadState --value 2>/dev/null | grep -qxv 'not-found'; then
+        printf 'quadlet:%s\n' "$LEGACY_LLM_ROUTING_POD_UNIT"
     else
         printf 'absent\n'
     fi
@@ -567,10 +580,11 @@ require_user_systemd() {
 safe_pod_teardown() {
     local ownership
     ownership=$(stack_ownership)
-    if [[ "$ownership" == "quadlet" ]]; then
+    if [[ "$ownership" == quadlet:* ]]; then
+        local owner_unit="${ownership#quadlet:}"
         echo "🛑 Reconciling Quadlet-owned stack (unit may be active, inactive, or failed)..."
-        systemctl --user stop "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true
-        systemctl --user reset-failed "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true
+        systemctl --user stop "$owner_unit" 2>/dev/null || true
+        systemctl --user reset-failed "$owner_unit" 2>/dev/null || true
         podman pod rm -f "${POD_NAME}" 2>/dev/null || true
         cleanup_zombie_ports
         echo "✓ Quadlet stack stopped, state reconciled, ports cleaned"
@@ -677,11 +691,10 @@ render_router_config() {
 
 # ── Quadlet rendering + installation ──
 # Renders quadlets/*.pod + quadlets/*.container templates (same _PLACEHOLDER
-# convention as pod.yaml) into ~/.config/containers/systemd/llm-routing/ and
+# convention as pod.yaml) into the environment-specific
+# ~/.config/containers/systemd/${QUADLET_NAMESPACE}/ directory and
 # lets systemd's podman-user-generator turn them into real units.
 # Quadlet values are bare scalars (not YAML) so plain string replacement is used.
-QUADLET_DIR="${HOME}/.config/containers/systemd/llm-routing"
-
 render_quadlets() {
     export WORKDIR HOME LITELLM_MASTER_KEY UI_USERNAME UI_PASSWORD
     export POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY
@@ -705,6 +718,11 @@ render_quadlets() {
 import os, sys, urllib.parse, re, glob, shutil, tempfile
 uid = os.getuid()
 src_dir, out_dir = sys.argv[1], sys.argv[2]
+namespace = os.environ["QUADLET_NAMESPACE"]
+identifier_suffixes = (
+    "pod", "clickhouse", "langfuse", "litellm", "minio", "postgres", "router", "valkey"
+)
+identifier_prefix = re.compile(r"\bllm-routing-(?=(?:" + "|".join(identifier_suffixes) + r"))")
 
 encoded_pg = urllib.parse.quote(os.environ['POSTGRES_PASSWORD'], safe="")
 # Derived by derive_external_service_urls(), shared with render_pod_yaml.
@@ -766,6 +784,22 @@ try:
             text = f.read()
         for ph, val in repl.items():
             text = text.replace(ph, str(val))
+        # Quadlet generated unit names are global in the user systemd manager.
+        # Namespace both filenames and internal dependencies to isolate dev/prod.
+        # Namespace Quadlet identifiers only. Do not rewrite image names, URL
+        # paths, or other configuration values containing "llm-routing".
+        # Namespace only Quadlet identifier lines and values. This preserves
+        # arbitrary image names, URLs, and credentials containing llm-routing.
+        def namespace_identifier(match):
+            field, value = match.group(1), match.group(2)
+            if field in {"Pod", "After", "Wants", "BindsTo", "Requires", "PartOf"}:
+                value = identifier_prefix.sub(namespace + "-", value)
+                value = value.replace("llm-routing.pod", namespace + ".pod")
+                value = value.replace("llm-routing-pod.service", namespace + "-pod.service")
+            elif field == "Pod":
+                value = value.replace("llm-routing.pod", namespace + ".pod")
+            return f"{field}={value}"
+        text = re.sub(r"(?m)^(Pod|After|Wants|BindsTo|Requires|PartOf)=(.*)$", namespace_identifier, text)
         unresolved = sorted(set(re.findall(r"\b[A-Z0-9_]+_PLACEHOLDER\b", text)))
         if unresolved:
             sys.stderr.write(f"Error: Unresolved placeholders in {os.path.basename(tpl)}: {', '.join(unresolved)}\n")
@@ -777,15 +811,16 @@ try:
             value = match.group(1).replace("\\", "\\\\").replace('"', '\\"')
             return f'Environment="{value}"'
         text = re.sub(r"(?m)^Environment=(.*)$", quote_environment, text)
-        staged_path = os.path.join(staging_dir, os.path.basename(tpl))
+        rendered_name = os.path.basename(tpl).replace("llm-routing", namespace)
+        staged_path = os.path.join(staging_dir, rendered_name)
         with open(staged_path, "w", encoding="utf-8") as f:
             f.write(text)
         # Rendered units include credentials; systemd user generator can read owner-only files.
         os.chmod(staged_path, 0o600)
 
     # All templates are now valid. Replace individual files atomically, then
     # remove stale units; a failed render above leaves the prior unit set intact.
-    rendered_names = {os.path.basename(tpl) for tpl in templates}
+    rendered_names = {os.path.basename(tpl).replace("llm-routing", namespace) for tpl in templates}
     for name in rendered_names:
         os.replace(os.path.join(staging_dir, name), os.path.join(out_dir, name))
     for stale in glob.glob(os.path.join(out_dir, "*.pod")) + glob.glob(os.path.join(out_dir, "*.container")):
@@ -854,13 +889,13 @@ if [[ "$STACK_OWNERSHIP" != "absent" ]]; then
         echo "🚀 Deploying replacement pod from YAML..."
         deploy_fresh_pod
     else
-        if [[ "$STACK_OWNERSHIP" == "quadlet" ]]; then
+        if [[ "$STACK_OWNERSHIP" == quadlet:* ]]; then
             require_user_systemd || exit 1
+            owner_unit="${STACK_OWNERSHIP#quadlet:}"
             echo "🔄 Restarting Quadlet-owned stack via systemd..."
-            systemctl --user reset-failed "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true
-            if ! systemctl --user restart "$LLM_ROUTING_POD_UNIT"; then
-                echo "❌ Error: failed to restart ${LLM_ROUTING_POD_UNIT}" >&2
-                echo "   Hint: run 'systemctl --user status ${LLM_ROUTING_POD_UNIT} --no-pager' to inspect the failure" >&2
+            if ! systemctl --user restart "$owner_unit"; then
+                echo "❌ Error: failed to restart ${owner_unit}" >&2
+                echo "   Hint: run 'systemctl --user status ${owner_unit} --no-pager' to inspect the failure" >&2
                 exit 1
             fi
         else

tests/test_quadlet_templates.py

@@ -1,6 +1,9 @@
 """Static deployment-contract tests for the systemd Quadlet templates."""
 from pathlib import Path
+import os
 import re
+import subprocess
+import tempfile
 
 
 ROOT = Path(__file__).resolve().parent.parent
@@ -44,7 +47,8 @@ def test_rendered_quadlets_are_owner_only():
     script = (ROOT / "start-stack.sh").read_text()
     assert 'chmod 700 "$QUADLET_DIR"' in script
     assert "os.chmod(staged_path, 0o600)" in script
-    assert 'LLM_ROUTING_POD_UNIT="llm-routing-pod.service"' in script
+    assert 'LLM_ROUTING_POD_UNIT="${QUADLET_NAMESPACE}-pod.service"' in script
+    assert 'QUADLET_DIR="${HOME}/.config/containers/systemd/${QUADLET_NAMESPACE}"' in script
     assert "systemctl --user daemon-reload" in script
     assert "stack_ownership()" in script
     assert "PODMAN_SYSTEMD_UNIT" in script
@@ -87,3 +91,81 @@ def test_quadlet_deployment_enforces_prerequisite_and_restart_failures():
     assert "require_user_systemd || exit 1" in script
     assert "failed to derive external service URLs for Quadlet rendering" in script
     assert 'if ! podman pod restart "${POD_NAME}"; then' in script
+
+
+def test_quadlet_namespace_is_environment_specific():
+    dev_env = (ROOT / ".env.dev").read_text()
+    prod_namespace = "llm-routing-prod"
+    assert 'QUADLET_NAMESPACE="llm-routing-dev"' in dev_env
+    assert 'QUADLET_NAMESPACE="${QUADLET_NAMESPACE:-llm-routing-prod}"' in (ROOT / "start-stack.sh").read_text()
+    assert prod_namespace in (ROOT / "start-stack.sh").read_text()
+    script = (ROOT / "start-stack.sh").read_text()
+    assert 'def namespace_identifier(match):' in script
+    assert 'identifier_suffixes = (' in script
+    assert 'identifier_prefix.sub(namespace + "-", value)' in script
+    assert 'rendered_name = os.path.basename(tpl).replace("llm-routing", namespace)' in script
+
+
+def test_namespace_rendering_preserves_non_identifiers():
+    script = (ROOT / "start-stack.sh").read_text()
+    embedded = script.split("python3 - \"$src_dir\" \"$QUADLET_DIR\" <<'PY'\n", 1)[1].split("\nPY\n", 1)[0]
+    env = os.environ.copy()
+    values = {
+        "POSTGRES_PASSWORD": "pg", "WORKDIR": str(ROOT), "HOME": str(ROOT),
+        "LITELLM_MASTER_KEY": "master", "NEXTAUTH_SECRET": "next", "SALT": "salt",
+        "ENCRYPTION_KEY": "encrypt", "OLLAMA_API_KEY": "ollama", "OPENROUTER_API_KEY": "openrouter",
+        "LANGFUSE_PUBLIC_KEY": "public", "LANGFUSE_SECRET_KEY": "secret", "MINIO_ROOT_USER": "minio",
+        "MINIO_ROOT_PASSWORD": "minio-pass", "LANGFUSE_INIT_USER_PASSWORD": "lf-pass",
+        "REDIS_AUTH": "redis", "CLICKHOUSE_PASSWORD": "click", "PROXY_BASE_URL_DERIVED": "https://proxy",
+        "NEXTAUTH_URL_DERIVED": "https://next", "PUBLIC_BASE_URL": "https://host/llm-routing",
+        "ROUTING_DOMAIN": "vendeuvre.lan", "LLAMA_CLASSIFIER_URL": "http://127.0.0.1:8083/v1",
+        "LLAMA_SERVER_URL": "http://127.0.0.1:8083", "POD_NAME": "dev-router-pod",
+        "DATA_ROOT": str(ROOT / "data"), "ROUTER_IMAGE": "registry/llm-routing-router:latest",
+        "ROUTER_PORT": "5010", "LITELLM_PORT": "4010", "LANGFUSE_WEB_PORT": "3011",
+        "LANGFUSE_WORKER_PORT": "3030", "POSTGRES_PORT": "5442", "VALKEY_CACHE_PORT": "6389",
+        "VALKEY_LF_PORT": "6390", "CLICKHOUSE_HTTP_PORT": "8123", "CLICKHOUSE_TCP_PORT": "9003",
+        "MINIO_S3_PORT": "9002", "MINIO_CONSOLE_PORT": "9001", "QUADLET_NAMESPACE": "llm-routing-dev",
+    }
+    env.update(values)
+    with tempfile.TemporaryDirectory() as tmp:
+        src, out = Path(tmp) / "src", Path(tmp) / "out"
+        src.mkdir(); out.mkdir()
+        (src / "llm-routing.pod").write_text("[Pod]\nPodName=llm-routing.pod\n")
+        (src / "llm-routing-router.container").write_text(
+            "[Unit]\nAfter=llm-routing-litellm.service\n[Container]\n"
+            "Image=registry/llm-routing-router:latest\n"
+            "Environment=PUBLIC_BASE_URL=https://host/llm-routing-router\nPod=llm-routing.pod\n"
+        )
+        subprocess.run(["python3", "-c", embedded, str(src), str(out)], env=env, check=True, capture_output=True, text=True)
+        rendered = (out / "llm-routing-dev-router.container").read_text()
+        assert "After=llm-routing-dev-litellm.service" in rendered
+        assert "Image=registry/llm-routing-router:latest" in rendered
+        assert "PUBLIC_BASE_URL=https://host/llm-routing-router" in rendered
+        assert "Pod=llm-routing-dev.pod" in rendered
+
+
+def test_namespace_is_validated_and_ownership_preserves_exact_unit():
+    script = (ROOT / "start-stack.sh").read_text()
+    assert '[[ ! "$QUADLET_NAMESPACE" =~ ^[a-z0-9][a-z0-9-]*$ ]]' in script
+    assert "printf 'quadlet:%s\\n' \"$infra_unit\"" in script
+    assert 'owner_unit="${STACK_OWNERSHIP#quadlet:}"' in script
+    assert 'failed to restart ${owner_unit}' in script
+    assert 'status ${owner_unit} --no-pager' in script
+
+
+def test_documentation_uses_environment_specific_units():
+    readme = (ROOT / "README.md").read_text()
+    scripts_readme = (ROOT / "scripts" / "README.md").read_text()
+    assert "systemctl --user status llm-routing-prod-pod.service" in readme
+    assert "llm-routing-dev-pod.service" in readme
+    assert "llm-routing-pod.service" not in scripts_readme
+    assert "llm-routing-{dev,prod}" not in readme
+
+
+def test_data_root_is_worktree_scoped_by_default():
+    script = (ROOT / "start-stack.sh").read_text()
+    assert 'DATA_ROOT="${DATA_ROOT:-${WORKDIR}/data}"' in script
+    dev_root = ROOT.resolve()
+    prod_root = (ROOT.parent.parent / "prod" / "LLM-Routing").resolve()
+    assert dev_root != prod_root
+    assert dev_root / "data" != prod_root / "data"