fix(deploy): complete Quadlet isolation review fixes

.env.dev

@@ -31,3 +31,6 @@ ROUTER_IMAGE="localhost/llm-routing-dev:latest"
 # reaches its local listener directly rather than any TLS-terminated hostname.
 LLAMA_CLASSIFIER_URL="http://127.0.0.1:8083/v1"
 LLAMA_SERVER_URL="http://127.0.0.1:8083"
+
+# Distinct systemd Quadlet namespace; never share generated units with prod.
+QUADLET_NAMESPACE="llm-routing-dev"

README.md

@@ -303,42 +303,47 @@ Orchestrates routing fallback chains, Redis caching, and telemetry callbacks:
           SM --> SC[agent-complex-core]:::complex
           SC --> SR[agent-reasoning-core]:::reasoning
           SR --> SA[agent-advanced-core]:::advanced
-          SA --> SO1[llm-routing-ollama]:::premium
+          SA --> SL[local-qwen-3.6]:::local
+          SL --> SO1[llm-routing-ollama]:::premium
           SO1 --> SAU[openrouter-auto]:::auto
       end
 
       subgraph Medium["agent-medium-core Fallback Tree"]
           M[agent-medium-core]:::medium --> MC[agent-complex-core]:::complex
           MC --> MR[agent-reasoning-core]:::reasoning
           MR --> MA[agent-advanced-core]:::advanced
-          MA --> MO1[llm-routing-ollama]:::premium
+          MA --> ML[local-qwen-3.6]:::local
+          ML --> MO1[llm-routing-ollama]:::premium
           MO1 --> MAU[openrouter-auto]:::auto
       end
 
       subgraph Complex["agent-complex-core Fallback Tree"]
           C[agent-complex-core]:::complex --> CR[agent-reasoning-core]:::reasoning
           CR --> CA[agent-advanced-core]:::advanced
-          CA --> CO1[llm-routing-ollama]:::premium
+          CA --> CL[local-qwen-3.6]:::local
+          CL --> CO1[llm-routing-ollama]:::premium
           CO1 --> CAU[openrouter-auto]:::auto
       end
 
       subgraph Reasoning["agent-reasoning-core Fallback Tree"]
           R[agent-reasoning-core]:::reasoning --> RA[agent-advanced-core]:::advanced
-          RA --> RO1[llm-routing-ollama]:::premium
+          RA --> RL[local-qwen-3.6]:::local
+          RL --> RO1[llm-routing-ollama]:::premium
           RO1 --> RAU[openrouter-auto]:::auto
       end
 
       subgraph Advanced["agent-advanced-core Fallback Tree"]
-          A[agent-advanced-core]:::advanced --> AO1[llm-routing-ollama]:::premium
+          A[agent-advanced-core]:::advanced --> AL[local-qwen-3.6]:::local
+          AL --> AO1[llm-routing-ollama]:::premium
           AO1 --> AAU[openrouter-auto]:::auto
       end
   ```
 
-  - **`agent-simple-core`**: medium-core → complex-core → reasoning-core → advanced-core → `llm-routing-ollama` → `openrouter-auto`
-  - **`agent-medium-core`**: complex-core → reasoning-core → advanced-core → `llm-routing-ollama` → `openrouter-auto`
-  - **`agent-complex-core`**: reasoning-core → advanced-core → `llm-routing-ollama` → `openrouter-auto`
-  - **`agent-reasoning-core`**: advanced-core → `llm-routing-ollama` → `openrouter-auto`
-  - **`agent-advanced-core`**: `llm-routing-ollama` → `openrouter-auto`
+  - **`agent-simple-core`**: medium-core → complex-core → reasoning-core → advanced-core → `local-qwen-3.6` → `llm-routing-ollama` → `openrouter-auto`
+  - **`agent-medium-core`**: complex-core → reasoning-core → advanced-core → `local-qwen-3.6` → `llm-routing-ollama` → `openrouter-auto`
+  - **`agent-complex-core`**: reasoning-core → advanced-core → `local-qwen-3.6` → `llm-routing-ollama` → `openrouter-auto`
+  - **`agent-reasoning-core`**: advanced-core → `local-qwen-3.6` → `llm-routing-ollama` → `openrouter-auto`
+  - **`agent-advanced-core`**: `local-qwen-3.6` → `llm-routing-ollama` → `openrouter-auto`
   - **`llm-routing-ollama`** (classifier-gated proxy): `reasoning & advanced` → `ollama-deepseek-v4-pro`, `complex & below` → `ollama-deepseek-v4-flash`. Note: Ollama cooldowns are managed by the triage router internally (5-minute window on failure); during cooldown the router returns 429 immediately so LiteLLM skips to `openrouter-auto`.
   All tiers ultimately land on OpenRouter auto/free model pools or the local Speculative MoE when enabled.
 *Note: Premium routing is controlled by the model name, not by the tier. `llm-routing-agy` and `llm-routing-auto-agy` trigger the agy proxy (Google/Claude via Cloud Code Assist) — but auto models only trigger agy if the classifier returns `agent-advanced-core`. `llm-routing-ollama` and `llm-routing-auto-ollama` route through Ollama.com (deepseek-v4-pro via LiteLLM's ollama_chat provider) — same gating for auto models. `llm-routing-auto-agy-ollama` chains both: agy first, then Ollama if agy is exhausted, both gated on advanced classification. The `agent-advanced-core` tier itself is a plain LiteLLM tier with no premium trigger. See §2 for the full routing table.*
@@ -395,9 +400,9 @@ Run the startup script from the root of the repository:
 ./start-stack.sh --full-rebuild  # Same as --replace plus rebuild the router image
 
 # Inspect the generated systemd units and their logs
-systemctl --user status llm-routing-pod.service --no-pager
-systemctl --user list-units 'llm-routing-*' --no-pager
-journalctl --user -u llm-routing-router.service -n 100 --no-pager
+systemctl --user status llm-routing-prod-pod.service --no-pager  # or llm-routing-dev-pod.service
+systemctl --user list-units 'llm-routing-*' --no-pager  # filter for dev/prod namespaces
+journalctl --user -u llm-routing-prod-router.service -n 100 --no-pager  # or llm-routing-dev-router.service
 ```
 *Note: If running for the first time, the script will prompt you for your `OpenRouter API Key`, securely saving it inside `.env` with restrictive permissions (`chmod 600`). The script also automatically generates and persists secure random secrets (`LITELLM_MASTER_KEY`, `POSTGRES_PASSWORD`, `NEXTAUTH_SECRET`, `SALT`, `ENCRYPTION_KEY`, and `ROUTER_API_KEY`) to this file on startup if they are missing.*
 
@@ -855,6 +860,19 @@ Tests cover:
 
 Requires `PUBLIC_BASE_URL` in `.env` for canonical URL tests. The router remains under its configured path (for example `https://x570.vendeuvre.lan/llm-routing`), while the verifier derives service URLs from its host: `https://litellm.<host>/ui/`, `https://langfuse.<host>/`, and `https://llama.<host>/health`. Dev `.env.dev` already has it; prod `.env` should include `PUBLIC_BASE_URL="https://x570.vendeuvre.lan/llm-routing"`. The dev local-model safety net uses the host-networked local listener: `LLAMA_CLASSIFIER_URL=http://127.0.0.1:8083/v1` and `LLAMA_SERVER_URL=http://127.0.0.1:8083`; it must not depend on TLS-terminated dev or production hostnames.
 
+## Environment-isolated Quadlet deployment
+
+Dev and production use distinct Quadlet namespaces because generated systemd
+unit names are global within the user manager. Dev renders units under
+`~/.config/containers/systemd/llm-routing-dev/` and uses
+`llm-routing-dev-pod.service`; production uses
+`~/.config/containers/systemd/llm-routing-prod/` and
+`llm-routing-prod-pod.service`. Their pod/container names, ports, data roots,
+and rendered configuration remain separate. Unless overridden explicitly,
+`DATA_ROOT` is `${WORKDIR}/data`; because dev and production run from separate
+`~/dev/` and `~/prod/` worktrees, their persistent data and rendered configs are
+also physically separate.
+
 ## 10. Performance Benchmarks
 
 Through our local benchmarks, the following performance characteristics have been achieved:

scripts/README.md

@@ -9,10 +9,10 @@ This directory and the repository root contain various scripts used for stack or
 ### `start-stack.sh` (Root Directory)
 Unified startup and credential extraction script for the systemd Quadlet-managed Podman stack.
 - **Usage**:
-  - `./start-stack.sh` (Restart the generated `llm-routing-pod.service`)
+  - `./start-stack.sh` (Restart the generated environment-specific pod service)
   - `./start-stack.sh --replace` (Stop + clean ports + render/install Quadlets + daemon-reload + recreate stack)
   - `./start-stack.sh --full-rebuild` (Same as `--replace` + rebuild the triage router image; required for code changes in `router/`)
-- Quadlet templates live in `quadlets/`; rendered owner-only units are installed under `~/.config/containers/systemd/llm-routing/`. Use `systemctl --user status llm-routing-pod.service --no-pager` and `journalctl --user -u llm-routing-router.service --no-pager` for lifecycle diagnostics.
+- Quadlet templates live in `quadlets/`; rendered owner-only units use environment-specific namespaces: dev under `~/.config/containers/systemd/llm-routing-dev/` and prod under `~/.config/containers/systemd/llm-routing-prod/`. Dev uses `llm-routing-dev-pod.service`; prod uses `llm-routing-prod-pod.service`. Use the matching `systemctl --user status <namespace>-pod.service --no-pager` and `journalctl --user -u <namespace>-router.service --no-pager` for lifecycle diagnostics.
 
 ### `scripts/backup.sh`
 Automated database backup script that runs before every stack deployment. Uses `pg_isready` to safely wait for database connections and manages timestamped backups under `backups/`.

start-stack.sh

@@ -77,6 +77,15 @@ if [ -n "${DEV_ENV_FILE:-}" ] && [ -f "$DEV_ENV_FILE" ]; then
     set +a
 fi
 
+# Quadlet namespace is environment-specific. This prevents dev and prod from
+# sharing rendered files or generated systemd unit names.
+QUADLET_NAMESPACE="${QUADLET_NAMESPACE:-llm-routing-prod}"
+if [[ ! "$QUADLET_NAMESPACE" =~ ^[a-z0-9][a-z0-9-]*$ ]]; then
+    echo "❌ Error: QUADLET_NAMESPACE must contain only lowercase letters, digits, and hyphens" >&2
+    exit 1
+fi
+export QUADLET_NAMESPACE
+
 # Port assignments — read from env (set by .env or .env.dev) with prod defaults
 POD_NAME="${POD_NAME:-prod-router-pod}"
 ROUTER_PORT="${ROUTER_PORT:-5000}"
@@ -533,22 +542,41 @@ verify_stack_health() {
 # ── Stack ownership and teardown ──
 # Keep the generated Quadlet unit name in one place: it is used for ownership
 # detection, lifecycle operations, and user-facing diagnostics.
-LLM_ROUTING_POD_UNIT="llm-routing-pod.service"
+LLM_ROUTING_POD_UNIT="${QUADLET_NAMESPACE}-pod.service"
+LEGACY_LLM_ROUTING_POD_UNIT="llm-routing-pod.service"
+QUADLET_DIR="${HOME}/.config/containers/systemd/${QUADLET_NAMESPACE}"
 # Quadlet-managed pods carry PODMAN_SYSTEMD_UNIT on their infra container.
 # Consult that metadata rather than inferring ownership from active state: a
 # stopped or failed generated unit is still Quadlet-owned and must be reconciled
 # through systemd before a replacement pod is created.
 stack_ownership() {
     local infra_unit
+
+    # A generic legacy unit is shared by old deployments. Only treat it as
+    # owned when its generated unit explicitly names this stack's pod; merely
+    # being loaded is not sufficient and could tear down the other environment.
+    legacy_unit_owns_pod() {
+        local unit="$1"
+        systemctl --user cat "$unit" --no-pager 2>/dev/null \
+            | grep -Fqx "PodName=${POD_NAME}"
+    }
+
     if podman pod exists "${POD_NAME}" 2>/dev/null; then
         infra_unit=$(podman pod inspect "${POD_NAME}" --format '{{.InfraContainerID}}' 2>/dev/null | xargs -r podman inspect --format '{{ index .Config.Labels "PODMAN_SYSTEMD_UNIT" }}' 2>/dev/null || true)
         if [[ "$infra_unit" == "$LLM_ROUTING_POD_UNIT" ]]; then
-            printf 'quadlet\n'
+            printf 'quadlet:%s\n' "$infra_unit"
+        elif [[ "$infra_unit" == "$LEGACY_LLM_ROUTING_POD_UNIT" ]] && legacy_unit_owns_pod "$LEGACY_LLM_ROUTING_POD_UNIT"; then
+            printf 'quadlet:%s\n' "$infra_unit"
+        elif [[ "$infra_unit" == "$LEGACY_LLM_ROUTING_POD_UNIT" ]]; then
+            printf 'absent\n'
         else
             printf 'legacy\n'
         fi
     elif systemctl --user show "$LLM_ROUTING_POD_UNIT" -p LoadState --value 2>/dev/null | grep -qxv 'not-found'; then
-        printf 'quadlet\n'
+        printf 'quadlet:%s\n' "$LLM_ROUTING_POD_UNIT"
+    elif systemctl --user show "$LEGACY_LLM_ROUTING_POD_UNIT" -p LoadState --value 2>/dev/null | grep -qxv 'not-found' \
+        && legacy_unit_owns_pod "$LEGACY_LLM_ROUTING_POD_UNIT"; then
+        printf 'quadlet:%s\n' "$LEGACY_LLM_ROUTING_POD_UNIT"
     else
         printf 'absent\n'
     fi
@@ -567,10 +595,11 @@ require_user_systemd() {
 safe_pod_teardown() {
     local ownership
     ownership=$(stack_ownership)
-    if [[ "$ownership" == "quadlet" ]]; then
+    if [[ "$ownership" == quadlet:* ]]; then
+        local owner_unit="${ownership#quadlet:}"
         echo "🛑 Reconciling Quadlet-owned stack (unit may be active, inactive, or failed)..."
-        systemctl --user stop "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true
-        systemctl --user reset-failed "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true
+        systemctl --user stop "$owner_unit" 2>/dev/null || true
+        systemctl --user reset-failed "$owner_unit" 2>/dev/null || true
         podman pod rm -f "${POD_NAME}" 2>/dev/null || true
         cleanup_zombie_ports
         echo "✓ Quadlet stack stopped, state reconciled, ports cleaned"
@@ -677,11 +706,10 @@ render_router_config() {
 
 # ── Quadlet rendering + installation ──
 # Renders quadlets/*.pod + quadlets/*.container templates (same _PLACEHOLDER
-# convention as pod.yaml) into ~/.config/containers/systemd/llm-routing/ and
+# convention as pod.yaml) into the environment-specific
+# ~/.config/containers/systemd/${QUADLET_NAMESPACE}/ directory and
 # lets systemd's podman-user-generator turn them into real units.
 # Quadlet values are bare scalars (not YAML) so plain string replacement is used.
-QUADLET_DIR="${HOME}/.config/containers/systemd/llm-routing"
-
 render_quadlets() {
     export WORKDIR HOME LITELLM_MASTER_KEY UI_USERNAME UI_PASSWORD
     export POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY
@@ -705,6 +733,11 @@ render_quadlets() {
 import os, sys, urllib.parse, re, glob, shutil, tempfile
 uid = os.getuid()
 src_dir, out_dir = sys.argv[1], sys.argv[2]
+namespace = os.environ["QUADLET_NAMESPACE"]
+identifier_suffixes = (
+    "pod", "clickhouse", "langfuse", "litellm", "minio", "postgres", "router", "valkey"
+)
+identifier_prefix = re.compile(r"\bllm-routing-(?=(?:" + "|".join(identifier_suffixes) + r"))")
 
 encoded_pg = urllib.parse.quote(os.environ['POSTGRES_PASSWORD'], safe="")
 # Derived by derive_external_service_urls(), shared with render_pod_yaml.
@@ -766,6 +799,22 @@ try:
             text = f.read()
         for ph, val in repl.items():
             text = text.replace(ph, str(val))
+        # Quadlet generated unit names are global in the user systemd manager.
+        # Namespace both filenames and internal dependencies to isolate dev/prod.
+        # Namespace Quadlet identifiers only. Do not rewrite image names, URL
+        # paths, or other configuration values containing "llm-routing".
+        # Namespace only Quadlet identifier lines and values. This preserves
+        # arbitrary image names, URLs, and credentials containing llm-routing.
+        def namespace_identifier(match):
+            field, value = match.group(1), match.group(2)
+            if field in {"Pod", "After", "Wants", "BindsTo", "Requires", "PartOf"}:
+                value = identifier_prefix.sub(namespace + "-", value)
+                value = value.replace("llm-routing.pod", namespace + ".pod")
+                value = value.replace("llm-routing-pod.service", namespace + "-pod.service")
+            elif field == "Pod":
+                value = value.replace("llm-routing.pod", namespace + ".pod")
+            return f"{field}={value}"
+        text = re.sub(r"(?m)^(Pod|After|Wants|BindsTo|Requires|PartOf)=(.*)$", namespace_identifier, text)
         unresolved = sorted(set(re.findall(r"\b[A-Z0-9_]+_PLACEHOLDER\b", text)))
         if unresolved:
             sys.stderr.write(f"Error: Unresolved placeholders in {os.path.basename(tpl)}: {', '.join(unresolved)}\n")
@@ -777,15 +826,16 @@ try:
             value = match.group(1).replace("\\", "\\\\").replace('"', '\\"')
             return f'Environment="{value}"'
         text = re.sub(r"(?m)^Environment=(.*)$", quote_environment, text)
-        staged_path = os.path.join(staging_dir, os.path.basename(tpl))
+        rendered_name = os.path.basename(tpl).replace("llm-routing", namespace)
+        staged_path = os.path.join(staging_dir, rendered_name)
         with open(staged_path, "w", encoding="utf-8") as f:
             f.write(text)
         # Rendered units include credentials; systemd user generator can read owner-only files.
         os.chmod(staged_path, 0o600)
 
     # All templates are now valid. Replace individual files atomically, then
     # remove stale units; a failed render above leaves the prior unit set intact.
-    rendered_names = {os.path.basename(tpl) for tpl in templates}
+    rendered_names = {os.path.basename(tpl).replace("llm-routing", namespace) for tpl in templates}
     for name in rendered_names:
         os.replace(os.path.join(staging_dir, name), os.path.join(out_dir, name))
     for stale in glob.glob(os.path.join(out_dir, "*.pod")) + glob.glob(os.path.join(out_dir, "*.container")):
@@ -854,13 +904,13 @@ if [[ "$STACK_OWNERSHIP" != "absent" ]]; then
         echo "🚀 Deploying replacement pod from YAML..."
         deploy_fresh_pod
     else
-        if [[ "$STACK_OWNERSHIP" == "quadlet" ]]; then
+        if [[ "$STACK_OWNERSHIP" == quadlet:* ]]; then
             require_user_systemd || exit 1
+            owner_unit="${STACK_OWNERSHIP#quadlet:}"
             echo "🔄 Restarting Quadlet-owned stack via systemd..."
-            systemctl --user reset-failed "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true
-            if ! systemctl --user restart "$LLM_ROUTING_POD_UNIT"; then
-                echo "❌ Error: failed to restart ${LLM_ROUTING_POD_UNIT}" >&2
-                echo "   Hint: run 'systemctl --user status ${LLM_ROUTING_POD_UNIT} --no-pager' to inspect the failure" >&2
+            if ! systemctl --user restart "$owner_unit"; then
+                echo "❌ Error: failed to restart ${owner_unit}" >&2
+                echo "   Hint: run 'systemctl --user status ${owner_unit} --no-pager' to inspect the failure" >&2
                 exit 1
             fi
         else