From e4e5780211d5f425b640ad2a5285583b54085933 Mon Sep 17 00:00:00 2001 From: boy Date: Thu, 23 Jul 2026 02:28:43 +0200 Subject: [PATCH 01/13] fix(deploy): isolate dev and prod Quadlet namespaces --- .env.dev | 3 ++ README.md | 10 +++++++ scripts/README.md | 2 +- start-stack.sh | 50 +++++++++++++++++++++++++++------ tests/test_quadlet_templates.py | 14 ++++++++- 5 files changed, 68 insertions(+), 11 deletions(-) diff --git a/.env.dev b/.env.dev index 4565148a..3ee30ba0 100644 --- a/.env.dev +++ b/.env.dev @@ -31,3 +31,6 @@ ROUTER_IMAGE="localhost/llm-routing-dev:latest" # reaches its local listener directly rather than any TLS-terminated hostname. LLAMA_CLASSIFIER_URL="http://127.0.0.1:8083/v1" LLAMA_SERVER_URL="http://127.0.0.1:8083" + +# Distinct systemd Quadlet namespace; never share generated units with prod. +QUADLET_NAMESPACE="llm-routing-dev" diff --git a/README.md b/README.md index a1324e18..e88941c8 100644 --- a/README.md +++ b/README.md @@ -855,6 +855,16 @@ Tests cover: Requires `PUBLIC_BASE_URL` in `.env` for canonical URL tests. The router remains under its configured path (for example `https://x570.vendeuvre.lan/llm-routing`), while the verifier derives service URLs from its host: `https://litellm./ui/`, `https://langfuse./`, and `https://llama./health`. Dev `.env.dev` already has it; prod `.env` should include `PUBLIC_BASE_URL="https://x570.vendeuvre.lan/llm-routing"`. The dev local-model safety net uses the host-networked local listener: `LLAMA_CLASSIFIER_URL=http://127.0.0.1:8083/v1` and `LLAMA_SERVER_URL=http://127.0.0.1:8083`; it must not depend on TLS-terminated dev or production hostnames. +## Environment-isolated Quadlet deployment + +Dev and production use distinct Quadlet namespaces because generated systemd +unit names are global within the user manager. Dev renders units under +`~/.config/containers/systemd/llm-routing-dev/` and uses +`llm-routing-dev-pod.service`; production uses +`~/.config/containers/systemd/llm-routing-prod/` and +`llm-routing-prod-pod.service`. Their pod/container names, ports, data roots, +and rendered configuration remain separate. + ## 10. Performance Benchmarks Through our local benchmarks, the following performance characteristics have been achieved: diff --git a/scripts/README.md b/scripts/README.md index cbb735cf..ffeac893 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -12,7 +12,7 @@ Unified startup and credential extraction script for the systemd Quadlet-managed - `./start-stack.sh` (Restart the generated `llm-routing-pod.service`) - `./start-stack.sh --replace` (Stop + clean ports + render/install Quadlets + daemon-reload + recreate stack) - `./start-stack.sh --full-rebuild` (Same as `--replace` + rebuild the triage router image; required for code changes in `router/`) -- Quadlet templates live in `quadlets/`; rendered owner-only units are installed under `~/.config/containers/systemd/llm-routing/`. Use `systemctl --user status llm-routing-pod.service --no-pager` and `journalctl --user -u llm-routing-router.service --no-pager` for lifecycle diagnostics. +- Quadlet templates live in `quadlets/`; rendered owner-only units use environment-specific namespaces: dev under `~/.config/containers/systemd/llm-routing-dev/` and prod under `~/.config/containers/systemd/llm-routing-prod/`. Dev uses `llm-routing-dev-pod.service`; prod uses `llm-routing-prod-pod.service`. Use the matching `systemctl --user status -pod.service --no-pager` and `journalctl --user -u -router.service --no-pager` for lifecycle diagnostics. ### `scripts/backup.sh` Automated database backup script that runs before every stack deployment. Uses `pg_isready` to safely wait for database connections and manages timestamped backups under `backups/`. diff --git a/start-stack.sh b/start-stack.sh index e2b1936e..8f4c2b2d 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -77,6 +77,11 @@ if [ -n "${DEV_ENV_FILE:-}" ] && [ -f "$DEV_ENV_FILE" ]; then set +a fi +# Quadlet namespace is environment-specific. This prevents dev and prod from +# sharing rendered files or generated systemd unit names. +QUADLET_NAMESPACE="${QUADLET_NAMESPACE:-llm-routing-prod}" +export QUADLET_NAMESPACE + # Port assignments — read from env (set by .env or .env.dev) with prod defaults POD_NAME="${POD_NAME:-prod-router-pod}" ROUTER_PORT="${ROUTER_PORT:-5000}" @@ -533,7 +538,9 @@ verify_stack_health() { # ── Stack ownership and teardown ── # Keep the generated Quadlet unit name in one place: it is used for ownership # detection, lifecycle operations, and user-facing diagnostics. -LLM_ROUTING_POD_UNIT="llm-routing-pod.service" +LLM_ROUTING_POD_UNIT="${QUADLET_NAMESPACE}-pod.service" +LEGACY_LLM_ROUTING_POD_UNIT="llm-routing-pod.service" +QUADLET_DIR="${HOME}/.config/containers/systemd/${QUADLET_NAMESPACE}" # Quadlet-managed pods carry PODMAN_SYSTEMD_UNIT on their infra container. # Consult that metadata rather than inferring ownership from active state: a # stopped or failed generated unit is still Quadlet-owned and must be reconciled @@ -542,7 +549,7 @@ stack_ownership() { local infra_unit if podman pod exists "${POD_NAME}" 2>/dev/null; then infra_unit=$(podman pod inspect "${POD_NAME}" --format '{{.InfraContainerID}}' 2>/dev/null | xargs -r podman inspect --format '{{ index .Config.Labels "PODMAN_SYSTEMD_UNIT" }}' 2>/dev/null || true) - if [[ "$infra_unit" == "$LLM_ROUTING_POD_UNIT" ]]; then + if [[ "$infra_unit" == "$LLM_ROUTING_POD_UNIT" || "$infra_unit" == "$LEGACY_LLM_ROUTING_POD_UNIT" ]]; then printf 'quadlet\n' else printf 'legacy\n' @@ -554,6 +561,20 @@ stack_ownership() { fi } +stop_quadlet_units() { + systemctl --user stop "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true + if [[ "$LLM_ROUTING_POD_UNIT" != "$LEGACY_LLM_ROUTING_POD_UNIT" ]]; then + systemctl --user stop "$LEGACY_LLM_ROUTING_POD_UNIT" 2>/dev/null || true + fi +} + +reset_quadlet_units() { + systemctl --user reset-failed "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true + if [[ "$LLM_ROUTING_POD_UNIT" != "$LEGACY_LLM_ROUTING_POD_UNIT" ]]; then + systemctl --user reset-failed "$LEGACY_LLM_ROUTING_POD_UNIT" 2>/dev/null || true + fi +} + require_user_systemd() { if ! systemctl --user show-environment >/dev/null 2>&1; then echo "❌ Error: the Quadlet deployment requires a reachable systemd --user manager." >&2 @@ -569,8 +590,8 @@ safe_pod_teardown() { ownership=$(stack_ownership) if [[ "$ownership" == "quadlet" ]]; then echo "🛑 Reconciling Quadlet-owned stack (unit may be active, inactive, or failed)..." - systemctl --user stop "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true - systemctl --user reset-failed "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true + stop_quadlet_units + reset_quadlet_units podman pod rm -f "${POD_NAME}" 2>/dev/null || true cleanup_zombie_ports echo "✓ Quadlet stack stopped, state reconciled, ports cleaned" @@ -680,8 +701,6 @@ render_router_config() { # convention as pod.yaml) into ~/.config/containers/systemd/llm-routing/ and # lets systemd's podman-user-generator turn them into real units. # Quadlet values are bare scalars (not YAML) so plain string replacement is used. -QUADLET_DIR="${HOME}/.config/containers/systemd/llm-routing" - render_quadlets() { export WORKDIR HOME LITELLM_MASTER_KEY UI_USERNAME UI_PASSWORD export POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY @@ -705,6 +724,7 @@ render_quadlets() { import os, sys, urllib.parse, re, glob, shutil, tempfile uid = os.getuid() src_dir, out_dir = sys.argv[1], sys.argv[2] +namespace = os.environ["QUADLET_NAMESPACE"] encoded_pg = urllib.parse.quote(os.environ['POSTGRES_PASSWORD'], safe="") # Derived by derive_external_service_urls(), shared with render_pod_yaml. @@ -766,6 +786,17 @@ try: text = f.read() for ph, val in repl.items(): text = text.replace(ph, str(val)) + # Quadlet generated unit names are global in the user systemd manager. + # Namespace both filenames and internal dependencies to isolate dev/prod. + # Namespace Quadlet identifiers only. Do not rewrite image names, URL + # paths, or other configuration values containing "llm-routing". + text = re.sub( + r"llm-routing-(?=(?:pod|clickhouse|langfuse|litellm|minio|postgres|router|valkey))", + namespace + "-", + text, + ) + text = text.replace("llm-routing.pod", namespace + ".pod") + text = text.replace("llm-routing-pod.service", namespace + "-pod.service") unresolved = sorted(set(re.findall(r"\b[A-Z0-9_]+_PLACEHOLDER\b", text))) if unresolved: sys.stderr.write(f"Error: Unresolved placeholders in {os.path.basename(tpl)}: {', '.join(unresolved)}\n") @@ -777,7 +808,8 @@ try: value = match.group(1).replace("\\", "\\\\").replace('"', '\\"') return f'Environment="{value}"' text = re.sub(r"(?m)^Environment=(.*)$", quote_environment, text) - staged_path = os.path.join(staging_dir, os.path.basename(tpl)) + rendered_name = os.path.basename(tpl).replace("llm-routing", namespace) + staged_path = os.path.join(staging_dir, rendered_name) with open(staged_path, "w", encoding="utf-8") as f: f.write(text) # Rendered units include credentials; systemd user generator can read owner-only files. @@ -785,7 +817,7 @@ try: # All templates are now valid. Replace individual files atomically, then # remove stale units; a failed render above leaves the prior unit set intact. - rendered_names = {os.path.basename(tpl) for tpl in templates} + rendered_names = {os.path.basename(tpl).replace("llm-routing", namespace) for tpl in templates} for name in rendered_names: os.replace(os.path.join(staging_dir, name), os.path.join(out_dir, name)) for stale in glob.glob(os.path.join(out_dir, "*.pod")) + glob.glob(os.path.join(out_dir, "*.container")): @@ -857,7 +889,7 @@ if [[ "$STACK_OWNERSHIP" != "absent" ]]; then if [[ "$STACK_OWNERSHIP" == "quadlet" ]]; then require_user_systemd || exit 1 echo "🔄 Restarting Quadlet-owned stack via systemd..." - systemctl --user reset-failed "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true + reset_quadlet_units if ! systemctl --user restart "$LLM_ROUTING_POD_UNIT"; then echo "❌ Error: failed to restart ${LLM_ROUTING_POD_UNIT}" >&2 echo " Hint: run 'systemctl --user status ${LLM_ROUTING_POD_UNIT} --no-pager' to inspect the failure" >&2 diff --git a/tests/test_quadlet_templates.py b/tests/test_quadlet_templates.py index 22f02b6c..cd294afe 100644 --- a/tests/test_quadlet_templates.py +++ b/tests/test_quadlet_templates.py @@ -44,7 +44,8 @@ def test_rendered_quadlets_are_owner_only(): script = (ROOT / "start-stack.sh").read_text() assert 'chmod 700 "$QUADLET_DIR"' in script assert "os.chmod(staged_path, 0o600)" in script - assert 'LLM_ROUTING_POD_UNIT="llm-routing-pod.service"' in script + assert 'LLM_ROUTING_POD_UNIT="${QUADLET_NAMESPACE}-pod.service"' in script + assert 'QUADLET_DIR="${HOME}/.config/containers/systemd/${QUADLET_NAMESPACE}"' in script assert "systemctl --user daemon-reload" in script assert "stack_ownership()" in script assert "PODMAN_SYSTEMD_UNIT" in script @@ -87,3 +88,14 @@ def test_quadlet_deployment_enforces_prerequisite_and_restart_failures(): assert "require_user_systemd || exit 1" in script assert "failed to derive external service URLs for Quadlet rendering" in script assert 'if ! podman pod restart "${POD_NAME}"; then' in script + + +def test_quadlet_namespace_is_environment_specific(): + dev_env = (ROOT / ".env.dev").read_text() + prod_env = (ROOT / ".env").read_text() + assert 'QUADLET_NAMESPACE="llm-routing-dev"' in dev_env + assert 'QUADLET_NAMESPACE="llm-routing-prod"' in prod_env + script = (ROOT / "start-stack.sh").read_text() + assert 'r"llm-routing-(?=(?:pod|clickhouse|langfuse|litellm|minio|postgres|router|valkey))"' in script + assert 'text = text.replace("llm-routing.pod", namespace + ".pod")' in script + assert 'rendered_name = os.path.basename(tpl).replace("llm-routing", namespace)' in script From 919f22a3b33d163e47587a70475826d4d13aa570 Mon Sep 17 00:00:00 2001 From: boy Date: Thu, 23 Jul 2026 02:32:43 +0200 Subject: [PATCH 02/13] test(deploy): avoid requiring ignored production env --- tests/test_quadlet_templates.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_quadlet_templates.py b/tests/test_quadlet_templates.py index cd294afe..660afb26 100644 --- a/tests/test_quadlet_templates.py +++ b/tests/test_quadlet_templates.py @@ -92,9 +92,10 @@ def test_quadlet_deployment_enforces_prerequisite_and_restart_failures(): def test_quadlet_namespace_is_environment_specific(): dev_env = (ROOT / ".env.dev").read_text() - prod_env = (ROOT / ".env").read_text() + prod_namespace = "llm-routing-prod" assert 'QUADLET_NAMESPACE="llm-routing-dev"' in dev_env - assert 'QUADLET_NAMESPACE="llm-routing-prod"' in prod_env + assert 'QUADLET_NAMESPACE="${QUADLET_NAMESPACE:-llm-routing-prod}"' in (ROOT / "start-stack.sh").read_text() + assert prod_namespace in (ROOT / "start-stack.sh").read_text() script = (ROOT / "start-stack.sh").read_text() assert 'r"llm-routing-(?=(?:pod|clickhouse|langfuse|litellm|minio|postgres|router|valkey))"' in script assert 'text = text.replace("llm-routing.pod", namespace + ".pod")' in script From 9c05821c24558a1428164cd540e60db88b5b3116 Mon Sep 17 00:00:00 2001 From: boy Date: Thu, 23 Jul 2026 02:46:13 +0200 Subject: [PATCH 03/13] fix(deploy): preserve exact Quadlet ownership during migration --- start-stack.sh | 23 +++++++++++++++-------- tests/test_quadlet_templates.py | 7 +++++++ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/start-stack.sh b/start-stack.sh index 8f4c2b2d..93a8d24c 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -80,6 +80,10 @@ fi # Quadlet namespace is environment-specific. This prevents dev and prod from # sharing rendered files or generated systemd unit names. QUADLET_NAMESPACE="${QUADLET_NAMESPACE:-llm-routing-prod}" +if [[ ! "$QUADLET_NAMESPACE" =~ ^[a-z0-9][a-z0-9-]*$ ]]; then + echo "❌ Error: QUADLET_NAMESPACE must contain only lowercase letters, digits, and hyphens" >&2 + exit 1 +fi export QUADLET_NAMESPACE # Port assignments — read from env (set by .env or .env.dev) with prod defaults @@ -550,12 +554,14 @@ stack_ownership() { if podman pod exists "${POD_NAME}" 2>/dev/null; then infra_unit=$(podman pod inspect "${POD_NAME}" --format '{{.InfraContainerID}}' 2>/dev/null | xargs -r podman inspect --format '{{ index .Config.Labels "PODMAN_SYSTEMD_UNIT" }}' 2>/dev/null || true) if [[ "$infra_unit" == "$LLM_ROUTING_POD_UNIT" || "$infra_unit" == "$LEGACY_LLM_ROUTING_POD_UNIT" ]]; then - printf 'quadlet\n' + printf 'quadlet:%s\n' "$infra_unit" else printf 'legacy\n' fi elif systemctl --user show "$LLM_ROUTING_POD_UNIT" -p LoadState --value 2>/dev/null | grep -qxv 'not-found'; then - printf 'quadlet\n' + printf 'quadlet:%s\n' "$LLM_ROUTING_POD_UNIT" + elif systemctl --user show "$LEGACY_LLM_ROUTING_POD_UNIT" -p LoadState --value 2>/dev/null | grep -qxv 'not-found'; then + printf 'quadlet:%s\n' "$LEGACY_LLM_ROUTING_POD_UNIT" else printf 'absent\n' fi @@ -588,10 +594,11 @@ require_user_systemd() { safe_pod_teardown() { local ownership ownership=$(stack_ownership) - if [[ "$ownership" == "quadlet" ]]; then + if [[ "$ownership" == quadlet:* ]]; then + local owner_unit="${ownership#quadlet:}" echo "🛑 Reconciling Quadlet-owned stack (unit may be active, inactive, or failed)..." - stop_quadlet_units - reset_quadlet_units + systemctl --user stop "$owner_unit" 2>/dev/null || true + systemctl --user reset-failed "$owner_unit" 2>/dev/null || true podman pod rm -f "${POD_NAME}" 2>/dev/null || true cleanup_zombie_ports echo "✓ Quadlet stack stopped, state reconciled, ports cleaned" @@ -886,11 +893,11 @@ if [[ "$STACK_OWNERSHIP" != "absent" ]]; then echo "🚀 Deploying replacement pod from YAML..." deploy_fresh_pod else - if [[ "$STACK_OWNERSHIP" == "quadlet" ]]; then + if [[ "$STACK_OWNERSHIP" == quadlet:* ]]; then require_user_systemd || exit 1 + owner_unit="${STACK_OWNERSHIP#quadlet:}" echo "🔄 Restarting Quadlet-owned stack via systemd..." - reset_quadlet_units - if ! systemctl --user restart "$LLM_ROUTING_POD_UNIT"; then + if ! systemctl --user restart "$owner_unit"; then echo "❌ Error: failed to restart ${LLM_ROUTING_POD_UNIT}" >&2 echo " Hint: run 'systemctl --user status ${LLM_ROUTING_POD_UNIT} --no-pager' to inspect the failure" >&2 exit 1 diff --git a/tests/test_quadlet_templates.py b/tests/test_quadlet_templates.py index 660afb26..e47cabe5 100644 --- a/tests/test_quadlet_templates.py +++ b/tests/test_quadlet_templates.py @@ -100,3 +100,10 @@ def test_quadlet_namespace_is_environment_specific(): assert 'r"llm-routing-(?=(?:pod|clickhouse|langfuse|litellm|minio|postgres|router|valkey))"' in script assert 'text = text.replace("llm-routing.pod", namespace + ".pod")' in script assert 'rendered_name = os.path.basename(tpl).replace("llm-routing", namespace)' in script + + +def test_namespace_is_validated_and_ownership_preserves_exact_unit(): + script = (ROOT / "start-stack.sh").read_text() + assert 'QUADLET_NAMESPACE" =~ ^[a-z0-9][a-z0-9-]*$' in script + assert "printf 'quadlet:%s\\n' \"$infra_unit\"" in script + assert 'owner_unit="${STACK_OWNERSHIP#quadlet:}"' in script From d1fca46e2716b839044e7fdad577a9a9cc8700d2 Mon Sep 17 00:00:00 2001 From: boy Date: Thu, 23 Jul 2026 02:48:36 +0200 Subject: [PATCH 04/13] fix(deploy): namespace only Quadlet identifier fields --- start-stack.sh | 23 ++++++++++++++++------- tests/test_quadlet_templates.py | 4 ++-- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/start-stack.sh b/start-stack.sh index 93a8d24c..e8f5a6e6 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -797,13 +797,22 @@ try: # Namespace both filenames and internal dependencies to isolate dev/prod. # Namespace Quadlet identifiers only. Do not rewrite image names, URL # paths, or other configuration values containing "llm-routing". - text = re.sub( - r"llm-routing-(?=(?:pod|clickhouse|langfuse|litellm|minio|postgres|router|valkey))", - namespace + "-", - text, - ) - text = text.replace("llm-routing.pod", namespace + ".pod") - text = text.replace("llm-routing-pod.service", namespace + "-pod.service") + # Namespace only Quadlet identifier lines and values. This preserves + # arbitrary image names, URLs, and credentials containing llm-routing. + def namespace_identifier(match): + field, value = match.group(1), match.group(2) + if field in {"Pod", "After", "Wants", "BindsTo", "Requires", "PartOf"}: + value = re.sub( + r"\bllm-routing-(?=(?:pod|clickhouse|langfuse|litellm|minio|postgres|router|valkey))", + namespace + "-", + value, + ) + value = value.replace("llm-routing.pod", namespace + ".pod") + value = value.replace("llm-routing-pod.service", namespace + "-pod.service") + elif field == "Pod": + value = value.replace("llm-routing.pod", namespace + ".pod") + return f"{field}={value}" + text = re.sub(r"(?m)^(Pod|After|Wants|BindsTo|Requires|PartOf)=(.*)$", namespace_identifier, text) unresolved = sorted(set(re.findall(r"\b[A-Z0-9_]+_PLACEHOLDER\b", text))) if unresolved: sys.stderr.write(f"Error: Unresolved placeholders in {os.path.basename(tpl)}: {', '.join(unresolved)}\n") diff --git a/tests/test_quadlet_templates.py b/tests/test_quadlet_templates.py index e47cabe5..e73175ad 100644 --- a/tests/test_quadlet_templates.py +++ b/tests/test_quadlet_templates.py @@ -97,8 +97,8 @@ def test_quadlet_namespace_is_environment_specific(): assert 'QUADLET_NAMESPACE="${QUADLET_NAMESPACE:-llm-routing-prod}"' in (ROOT / "start-stack.sh").read_text() assert prod_namespace in (ROOT / "start-stack.sh").read_text() script = (ROOT / "start-stack.sh").read_text() - assert 'r"llm-routing-(?=(?:pod|clickhouse|langfuse|litellm|minio|postgres|router|valkey))"' in script - assert 'text = text.replace("llm-routing.pod", namespace + ".pod")' in script + assert 'def namespace_identifier(match):' in script + assert 'namespace + "-"' in script assert 'rendered_name = os.path.basename(tpl).replace("llm-routing", namespace)' in script From cfc95e9b1f344ee88b18fc2e4d1fb2040d5a23d5 Mon Sep 17 00:00:00 2001 From: boy Date: Thu, 23 Jul 2026 03:04:02 +0200 Subject: [PATCH 05/13] fix(deploy): complete Quadlet review hardening --- README.md | 6 ++-- scripts/README.md | 2 +- start-stack.sh | 28 +++++------------ tests/test_quadlet_templates.py | 56 +++++++++++++++++++++++++++++++-- 4 files changed, 65 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index e88941c8..1d02f92a 100644 --- a/README.md +++ b/README.md @@ -395,9 +395,9 @@ Run the startup script from the root of the repository: ./start-stack.sh --full-rebuild # Same as --replace plus rebuild the router image # Inspect the generated systemd units and their logs -systemctl --user status llm-routing-pod.service --no-pager -systemctl --user list-units 'llm-routing-*' --no-pager -journalctl --user -u llm-routing-router.service -n 100 --no-pager +systemctl --user status llm-routing-prod-pod.service --no-pager # or llm-routing-dev-pod.service +systemctl --user list-units 'llm-routing-{dev,prod}-*' --no-pager +journalctl --user -u llm-routing-prod-router.service -n 100 --no-pager # or llm-routing-dev-router.service ``` *Note: If running for the first time, the script will prompt you for your `OpenRouter API Key`, securely saving it inside `.env` with restrictive permissions (`chmod 600`). The script also automatically generates and persists secure random secrets (`LITELLM_MASTER_KEY`, `POSTGRES_PASSWORD`, `NEXTAUTH_SECRET`, `SALT`, `ENCRYPTION_KEY`, and `ROUTER_API_KEY`) to this file on startup if they are missing.* diff --git a/scripts/README.md b/scripts/README.md index ffeac893..f893f3b1 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -9,7 +9,7 @@ This directory and the repository root contain various scripts used for stack or ### `start-stack.sh` (Root Directory) Unified startup and credential extraction script for the systemd Quadlet-managed Podman stack. - **Usage**: - - `./start-stack.sh` (Restart the generated `llm-routing-pod.service`) + - `./start-stack.sh` (Restart the generated environment-specific pod service) - `./start-stack.sh --replace` (Stop + clean ports + render/install Quadlets + daemon-reload + recreate stack) - `./start-stack.sh --full-rebuild` (Same as `--replace` + rebuild the triage router image; required for code changes in `router/`) - Quadlet templates live in `quadlets/`; rendered owner-only units use environment-specific namespaces: dev under `~/.config/containers/systemd/llm-routing-dev/` and prod under `~/.config/containers/systemd/llm-routing-prod/`. Dev uses `llm-routing-dev-pod.service`; prod uses `llm-routing-prod-pod.service`. Use the matching `systemctl --user status -pod.service --no-pager` and `journalctl --user -u -router.service --no-pager` for lifecycle diagnostics. diff --git a/start-stack.sh b/start-stack.sh index e8f5a6e6..6eb2fb87 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -567,20 +567,6 @@ stack_ownership() { fi } -stop_quadlet_units() { - systemctl --user stop "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true - if [[ "$LLM_ROUTING_POD_UNIT" != "$LEGACY_LLM_ROUTING_POD_UNIT" ]]; then - systemctl --user stop "$LEGACY_LLM_ROUTING_POD_UNIT" 2>/dev/null || true - fi -} - -reset_quadlet_units() { - systemctl --user reset-failed "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true - if [[ "$LLM_ROUTING_POD_UNIT" != "$LEGACY_LLM_ROUTING_POD_UNIT" ]]; then - systemctl --user reset-failed "$LEGACY_LLM_ROUTING_POD_UNIT" 2>/dev/null || true - fi -} - require_user_systemd() { if ! systemctl --user show-environment >/dev/null 2>&1; then echo "❌ Error: the Quadlet deployment requires a reachable systemd --user manager." >&2 @@ -732,6 +718,10 @@ import os, sys, urllib.parse, re, glob, shutil, tempfile uid = os.getuid() src_dir, out_dir = sys.argv[1], sys.argv[2] namespace = os.environ["QUADLET_NAMESPACE"] +identifier_suffixes = ( + "pod", "clickhouse", "langfuse", "litellm", "minio", "postgres", "router", "valkey" +) +identifier_prefix = re.compile(r"\bllm-routing-(?=(?:" + "|".join(identifier_suffixes) + r"))") encoded_pg = urllib.parse.quote(os.environ['POSTGRES_PASSWORD'], safe="") # Derived by derive_external_service_urls(), shared with render_pod_yaml. @@ -802,11 +792,7 @@ try: def namespace_identifier(match): field, value = match.group(1), match.group(2) if field in {"Pod", "After", "Wants", "BindsTo", "Requires", "PartOf"}: - value = re.sub( - r"\bllm-routing-(?=(?:pod|clickhouse|langfuse|litellm|minio|postgres|router|valkey))", - namespace + "-", - value, - ) + value = identifier_prefix.sub(namespace + "-", value) value = value.replace("llm-routing.pod", namespace + ".pod") value = value.replace("llm-routing-pod.service", namespace + "-pod.service") elif field == "Pod": @@ -907,8 +893,8 @@ if [[ "$STACK_OWNERSHIP" != "absent" ]]; then owner_unit="${STACK_OWNERSHIP#quadlet:}" echo "🔄 Restarting Quadlet-owned stack via systemd..." if ! systemctl --user restart "$owner_unit"; then - echo "❌ Error: failed to restart ${LLM_ROUTING_POD_UNIT}" >&2 - echo " Hint: run 'systemctl --user status ${LLM_ROUTING_POD_UNIT} --no-pager' to inspect the failure" >&2 + echo "❌ Error: failed to restart ${owner_unit}" >&2 + echo " Hint: run 'systemctl --user status ${owner_unit} --no-pager' to inspect the failure" >&2 exit 1 fi else diff --git a/tests/test_quadlet_templates.py b/tests/test_quadlet_templates.py index e73175ad..6a03fd66 100644 --- a/tests/test_quadlet_templates.py +++ b/tests/test_quadlet_templates.py @@ -1,6 +1,9 @@ """Static deployment-contract tests for the systemd Quadlet templates.""" from pathlib import Path +import os import re +import subprocess +import tempfile ROOT = Path(__file__).resolve().parent.parent @@ -98,12 +101,61 @@ def test_quadlet_namespace_is_environment_specific(): assert prod_namespace in (ROOT / "start-stack.sh").read_text() script = (ROOT / "start-stack.sh").read_text() assert 'def namespace_identifier(match):' in script - assert 'namespace + "-"' in script + assert 'identifier_suffixes = (' in script + assert 'identifier_prefix.sub(namespace + "-", value)' in script assert 'rendered_name = os.path.basename(tpl).replace("llm-routing", namespace)' in script +def test_namespace_rendering_preserves_non_identifiers(): + script = (ROOT / "start-stack.sh").read_text() + embedded = script.split("python3 - \"$src_dir\" \"$QUADLET_DIR\" <<'PY'\n", 1)[1].split("\nPY\n", 1)[0] + env = os.environ.copy() + values = { + "POSTGRES_PASSWORD": "pg", "WORKDIR": str(ROOT), "HOME": str(ROOT), + "LITELLM_MASTER_KEY": "master", "NEXTAUTH_SECRET": "next", "SALT": "salt", + "ENCRYPTION_KEY": "encrypt", "OLLAMA_API_KEY": "ollama", "OPENROUTER_API_KEY": "openrouter", + "LANGFUSE_PUBLIC_KEY": "public", "LANGFUSE_SECRET_KEY": "secret", "MINIO_ROOT_USER": "minio", + "MINIO_ROOT_PASSWORD": "minio-pass", "LANGFUSE_INIT_USER_PASSWORD": "lf-pass", + "REDIS_AUTH": "redis", "CLICKHOUSE_PASSWORD": "click", "PROXY_BASE_URL_DERIVED": "https://proxy", + "NEXTAUTH_URL_DERIVED": "https://next", "PUBLIC_BASE_URL": "https://host/llm-routing", + "ROUTING_DOMAIN": "vendeuvre.lan", "LLAMA_CLASSIFIER_URL": "http://127.0.0.1:8083/v1", + "LLAMA_SERVER_URL": "http://127.0.0.1:8083", "POD_NAME": "dev-router-pod", + "DATA_ROOT": str(ROOT / "data"), "ROUTER_IMAGE": "registry/llm-routing-router:latest", + "ROUTER_PORT": "5010", "LITELLM_PORT": "4010", "LANGFUSE_WEB_PORT": "3011", + "LANGFUSE_WORKER_PORT": "3030", "POSTGRES_PORT": "5442", "VALKEY_CACHE_PORT": "6389", + "VALKEY_LF_PORT": "6390", "CLICKHOUSE_HTTP_PORT": "8123", "CLICKHOUSE_TCP_PORT": "9003", + "MINIO_S3_PORT": "9002", "MINIO_CONSOLE_PORT": "9001", "QUADLET_NAMESPACE": "llm-routing-dev", + } + env.update(values) + with tempfile.TemporaryDirectory() as tmp: + src, out = Path(tmp) / "src", Path(tmp) / "out" + src.mkdir(); out.mkdir() + (src / "llm-routing.pod").write_text("[Pod]\nPodName=llm-routing.pod\n") + (src / "llm-routing-router.container").write_text( + "[Unit]\nAfter=llm-routing-litellm.service\n[Container]\n" + "Image=registry/llm-routing-router:latest\n" + "Environment=PUBLIC_BASE_URL=https://host/llm-routing-router\nPod=llm-routing.pod\n" + ) + subprocess.run(["python3", "-c", embedded, str(src), str(out)], env=env, check=True, capture_output=True, text=True) + rendered = (out / "llm-routing-dev-router.container").read_text() + assert "After=llm-routing-dev-litellm.service" in rendered + assert "Image=registry/llm-routing-router:latest" in rendered + assert "PUBLIC_BASE_URL=https://host/llm-routing-router" in rendered + assert "Pod=llm-routing-dev.pod" in rendered + + def test_namespace_is_validated_and_ownership_preserves_exact_unit(): script = (ROOT / "start-stack.sh").read_text() - assert 'QUADLET_NAMESPACE" =~ ^[a-z0-9][a-z0-9-]*$' in script + assert '[[ ! "$QUADLET_NAMESPACE" =~ ^[a-z0-9][a-z0-9-]*$ ]]' in script assert "printf 'quadlet:%s\\n' \"$infra_unit\"" in script assert 'owner_unit="${STACK_OWNERSHIP#quadlet:}"' in script + assert 'failed to restart ${owner_unit}' in script + assert 'status ${owner_unit} --no-pager' in script + + +def test_documentation_uses_environment_specific_units(): + readme = (ROOT / "README.md").read_text() + scripts_readme = (ROOT / "scripts" / "README.md").read_text() + assert "systemctl --user status llm-routing-prod-pod.service" in readme + assert "llm-routing-dev-pod.service" in readme + assert "llm-routing-pod.service" not in scripts_readme From 267270102da0c23b5e8eab93627ce8365650bf0a Mon Sep 17 00:00:00 2001 From: boy Date: Thu, 23 Jul 2026 03:07:00 +0200 Subject: [PATCH 06/13] fix(deploy): finish review documentation and diagnostics --- README.md | 2 +- start-stack.sh | 3 ++- tests/test_quadlet_templates.py | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1d02f92a..c14e84af 100644 --- a/README.md +++ b/README.md @@ -396,7 +396,7 @@ Run the startup script from the root of the repository: # Inspect the generated systemd units and their logs systemctl --user status llm-routing-prod-pod.service --no-pager # or llm-routing-dev-pod.service -systemctl --user list-units 'llm-routing-{dev,prod}-*' --no-pager +systemctl --user list-units 'llm-routing-*' --no-pager # filter for dev/prod namespaces journalctl --user -u llm-routing-prod-router.service -n 100 --no-pager # or llm-routing-dev-router.service ``` *Note: If running for the first time, the script will prompt you for your `OpenRouter API Key`, securely saving it inside `.env` with restrictive permissions (`chmod 600`). The script also automatically generates and persists secure random secrets (`LITELLM_MASTER_KEY`, `POSTGRES_PASSWORD`, `NEXTAUTH_SECRET`, `SALT`, `ENCRYPTION_KEY`, and `ROUTER_API_KEY`) to this file on startup if they are missing.* diff --git a/start-stack.sh b/start-stack.sh index 6eb2fb87..105f2020 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -691,7 +691,8 @@ render_router_config() { # ── Quadlet rendering + installation ── # Renders quadlets/*.pod + quadlets/*.container templates (same _PLACEHOLDER -# convention as pod.yaml) into ~/.config/containers/systemd/llm-routing/ and +# convention as pod.yaml) into the environment-specific +# ~/.config/containers/systemd/${QUADLET_NAMESPACE}/ directory and # lets systemd's podman-user-generator turn them into real units. # Quadlet values are bare scalars (not YAML) so plain string replacement is used. render_quadlets() { diff --git a/tests/test_quadlet_templates.py b/tests/test_quadlet_templates.py index 6a03fd66..da51b973 100644 --- a/tests/test_quadlet_templates.py +++ b/tests/test_quadlet_templates.py @@ -159,3 +159,4 @@ def test_documentation_uses_environment_specific_units(): assert "systemctl --user status llm-routing-prod-pod.service" in readme assert "llm-routing-dev-pod.service" in readme assert "llm-routing-pod.service" not in scripts_readme + assert "llm-routing-{dev,prod}" not in readme From 49584f68ac9a8c23fc72d1b331d1390492cbf408 Mon Sep 17 00:00:00 2001 From: boy Date: Thu, 23 Jul 2026 03:07:38 +0200 Subject: [PATCH 07/13] test(deploy): document worktree-scoped persistent data --- README.md | 5 ++++- tests/test_quadlet_templates.py | 9 +++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c14e84af..166dc495 100644 --- a/README.md +++ b/README.md @@ -863,7 +863,10 @@ unit names are global within the user manager. Dev renders units under `llm-routing-dev-pod.service`; production uses `~/.config/containers/systemd/llm-routing-prod/` and `llm-routing-prod-pod.service`. Their pod/container names, ports, data roots, -and rendered configuration remain separate. +and rendered configuration remain separate. Unless overridden explicitly, +`DATA_ROOT` is `${WORKDIR}/data`; because dev and production run from separate +`~/dev/` and `~/prod/` worktrees, their persistent data and rendered configs are +also physically separate. ## 10. Performance Benchmarks diff --git a/tests/test_quadlet_templates.py b/tests/test_quadlet_templates.py index da51b973..a2da13d5 100644 --- a/tests/test_quadlet_templates.py +++ b/tests/test_quadlet_templates.py @@ -160,3 +160,12 @@ def test_documentation_uses_environment_specific_units(): assert "llm-routing-dev-pod.service" in readme assert "llm-routing-pod.service" not in scripts_readme assert "llm-routing-{dev,prod}" not in readme + + +def test_data_root_is_worktree_scoped_by_default(): + script = (ROOT / "start-stack.sh").read_text() + assert 'DATA_ROOT="${DATA_ROOT:-${WORKDIR}/data}"' in script + dev_root = ROOT.resolve() + prod_root = (ROOT.parent.parent / "prod" / "LLM-Routing").resolve() + assert dev_root != prod_root + assert dev_root / "data" != prod_root / "data" From d220dd4547563b4689179f34afb4e147bd060e7c Mon Sep 17 00:00:00 2001 From: boy Date: Thu, 23 Jul 2026 03:10:47 +0200 Subject: [PATCH 08/13] fix(deploy): verify legacy Quadlet ownership before teardown --- start-stack.sh | 13 ++++++++++++- tests/test_quadlet_templates.py | 3 +++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/start-stack.sh b/start-stack.sh index 105f2020..7ff7784f 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -551,6 +551,16 @@ QUADLET_DIR="${HOME}/.config/containers/systemd/${QUADLET_NAMESPACE}" # through systemd before a replacement pod is created. stack_ownership() { local infra_unit + + # A generic legacy unit is shared by old deployments. Only treat it as + # owned when its generated unit explicitly names this stack's pod; merely + # being loaded is not sufficient and could tear down the other environment. + legacy_unit_owns_pod() { + local unit="$1" + systemctl --user cat "$unit" --no-pager 2>/dev/null \ + | grep -Fqx "PodName=${POD_NAME}" + } + if podman pod exists "${POD_NAME}" 2>/dev/null; then infra_unit=$(podman pod inspect "${POD_NAME}" --format '{{.InfraContainerID}}' 2>/dev/null | xargs -r podman inspect --format '{{ index .Config.Labels "PODMAN_SYSTEMD_UNIT" }}' 2>/dev/null || true) if [[ "$infra_unit" == "$LLM_ROUTING_POD_UNIT" || "$infra_unit" == "$LEGACY_LLM_ROUTING_POD_UNIT" ]]; then @@ -560,7 +570,8 @@ stack_ownership() { fi elif systemctl --user show "$LLM_ROUTING_POD_UNIT" -p LoadState --value 2>/dev/null | grep -qxv 'not-found'; then printf 'quadlet:%s\n' "$LLM_ROUTING_POD_UNIT" - elif systemctl --user show "$LEGACY_LLM_ROUTING_POD_UNIT" -p LoadState --value 2>/dev/null | grep -qxv 'not-found'; then + elif systemctl --user show "$LEGACY_LLM_ROUTING_POD_UNIT" -p LoadState --value 2>/dev/null | grep -qxv 'not-found' \ + && legacy_unit_owns_pod "$LEGACY_LLM_ROUTING_POD_UNIT"; then printf 'quadlet:%s\n' "$LEGACY_LLM_ROUTING_POD_UNIT" else printf 'absent\n' diff --git a/tests/test_quadlet_templates.py b/tests/test_quadlet_templates.py index a2da13d5..98f8416a 100644 --- a/tests/test_quadlet_templates.py +++ b/tests/test_quadlet_templates.py @@ -151,6 +151,9 @@ def test_namespace_is_validated_and_ownership_preserves_exact_unit(): assert 'owner_unit="${STACK_OWNERSHIP#quadlet:}"' in script assert 'failed to restart ${owner_unit}' in script assert 'status ${owner_unit} --no-pager' in script + assert 'legacy_unit_owns_pod()' in script + assert 'grep -Fqx "PodName=${POD_NAME}"' in script + assert '&& legacy_unit_owns_pod "$LEGACY_LLM_ROUTING_POD_UNIT"' in script def test_documentation_uses_environment_specific_units(): From 2529318e4bda728a71c70c44b44b27ca79a396ca Mon Sep 17 00:00:00 2001 From: boy Date: Thu, 23 Jul 2026 03:12:22 +0200 Subject: [PATCH 09/13] fix(deploy): guard legacy ownership and document local fallback --- README.md | 25 +++++++++++++++---------- start-stack.sh | 6 +++++- tests/test_quadlet_templates.py | 2 ++ 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 166dc495..23cb3d2c 100644 --- a/README.md +++ b/README.md @@ -303,7 +303,8 @@ Orchestrates routing fallback chains, Redis caching, and telemetry callbacks: SM --> SC[agent-complex-core]:::complex SC --> SR[agent-reasoning-core]:::reasoning SR --> SA[agent-advanced-core]:::advanced - SA --> SO1[llm-routing-ollama]:::premium + SA --> SL[local-qwen-3.6]:::local + SL --> SO1[llm-routing-ollama]:::premium SO1 --> SAU[openrouter-auto]:::auto end @@ -311,34 +312,38 @@ Orchestrates routing fallback chains, Redis caching, and telemetry callbacks: M[agent-medium-core]:::medium --> MC[agent-complex-core]:::complex MC --> MR[agent-reasoning-core]:::reasoning MR --> MA[agent-advanced-core]:::advanced - MA --> MO1[llm-routing-ollama]:::premium + MA --> ML[local-qwen-3.6]:::local + ML --> MO1[llm-routing-ollama]:::premium MO1 --> MAU[openrouter-auto]:::auto end subgraph Complex["agent-complex-core Fallback Tree"] C[agent-complex-core]:::complex --> CR[agent-reasoning-core]:::reasoning CR --> CA[agent-advanced-core]:::advanced - CA --> CO1[llm-routing-ollama]:::premium + CA --> CL[local-qwen-3.6]:::local + CL --> CO1[llm-routing-ollama]:::premium CO1 --> CAU[openrouter-auto]:::auto end subgraph Reasoning["agent-reasoning-core Fallback Tree"] R[agent-reasoning-core]:::reasoning --> RA[agent-advanced-core]:::advanced - RA --> RO1[llm-routing-ollama]:::premium + RA --> RL[local-qwen-3.6]:::local + RL --> RO1[llm-routing-ollama]:::premium RO1 --> RAU[openrouter-auto]:::auto end subgraph Advanced["agent-advanced-core Fallback Tree"] - A[agent-advanced-core]:::advanced --> AO1[llm-routing-ollama]:::premium + A[agent-advanced-core]:::advanced --> AL[local-qwen-3.6]:::local + AL --> AO1[llm-routing-ollama]:::premium AO1 --> AAU[openrouter-auto]:::auto end ``` - - **`agent-simple-core`**: medium-core → complex-core → reasoning-core → advanced-core → `llm-routing-ollama` → `openrouter-auto` - - **`agent-medium-core`**: complex-core → reasoning-core → advanced-core → `llm-routing-ollama` → `openrouter-auto` - - **`agent-complex-core`**: reasoning-core → advanced-core → `llm-routing-ollama` → `openrouter-auto` - - **`agent-reasoning-core`**: advanced-core → `llm-routing-ollama` → `openrouter-auto` - - **`agent-advanced-core`**: `llm-routing-ollama` → `openrouter-auto` + - **`agent-simple-core`**: medium-core → complex-core → reasoning-core → advanced-core → `local-qwen-3.6` → `llm-routing-ollama` → `openrouter-auto` + - **`agent-medium-core`**: complex-core → reasoning-core → advanced-core → `local-qwen-3.6` → `llm-routing-ollama` → `openrouter-auto` + - **`agent-complex-core`**: reasoning-core → advanced-core → `local-qwen-3.6` → `llm-routing-ollama` → `openrouter-auto` + - **`agent-reasoning-core`**: advanced-core → `local-qwen-3.6` → `llm-routing-ollama` → `openrouter-auto` + - **`agent-advanced-core`**: `local-qwen-3.6` → `llm-routing-ollama` → `openrouter-auto` - **`llm-routing-ollama`** (classifier-gated proxy): `reasoning & advanced` → `ollama-deepseek-v4-pro`, `complex & below` → `ollama-deepseek-v4-flash`. Note: Ollama cooldowns are managed by the triage router internally (5-minute window on failure); during cooldown the router returns 429 immediately so LiteLLM skips to `openrouter-auto`. All tiers ultimately land on OpenRouter auto/free model pools or the local Speculative MoE when enabled. *Note: Premium routing is controlled by the model name, not by the tier. `llm-routing-agy` and `llm-routing-auto-agy` trigger the agy proxy (Google/Claude via Cloud Code Assist) — but auto models only trigger agy if the classifier returns `agent-advanced-core`. `llm-routing-ollama` and `llm-routing-auto-ollama` route through Ollama.com (deepseek-v4-pro via LiteLLM's ollama_chat provider) — same gating for auto models. `llm-routing-auto-agy-ollama` chains both: agy first, then Ollama if agy is exhausted, both gated on advanced classification. The `agent-advanced-core` tier itself is a plain LiteLLM tier with no premium trigger. See §2 for the full routing table.* diff --git a/start-stack.sh b/start-stack.sh index 7ff7784f..6159ff9c 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -563,8 +563,12 @@ stack_ownership() { if podman pod exists "${POD_NAME}" 2>/dev/null; then infra_unit=$(podman pod inspect "${POD_NAME}" --format '{{.InfraContainerID}}' 2>/dev/null | xargs -r podman inspect --format '{{ index .Config.Labels "PODMAN_SYSTEMD_UNIT" }}' 2>/dev/null || true) - if [[ "$infra_unit" == "$LLM_ROUTING_POD_UNIT" || "$infra_unit" == "$LEGACY_LLM_ROUTING_POD_UNIT" ]]; then + if [[ "$infra_unit" == "$LLM_ROUTING_POD_UNIT" ]]; then printf 'quadlet:%s\n' "$infra_unit" + elif [[ "$infra_unit" == "$LEGACY_LLM_ROUTING_POD_UNIT" ]] && legacy_unit_owns_pod "$LEGACY_LLM_ROUTING_POD_UNIT"; then + printf 'quadlet:%s\n' "$infra_unit" + elif [[ "$infra_unit" == "$LEGACY_LLM_ROUTING_POD_UNIT" ]]; then + printf 'absent\n' else printf 'legacy\n' fi diff --git a/tests/test_quadlet_templates.py b/tests/test_quadlet_templates.py index 98f8416a..68279ed3 100644 --- a/tests/test_quadlet_templates.py +++ b/tests/test_quadlet_templates.py @@ -154,6 +154,8 @@ def test_namespace_is_validated_and_ownership_preserves_exact_unit(): assert 'legacy_unit_owns_pod()' in script assert 'grep -Fqx "PodName=${POD_NAME}"' in script assert '&& legacy_unit_owns_pod "$LEGACY_LLM_ROUTING_POD_UNIT"' in script + assert 'elif [[ "$infra_unit" == "$LEGACY_LLM_ROUTING_POD_UNIT" ]]; then' in script + assert "printf 'absent\\n'" in script def test_documentation_uses_environment_specific_units(): From 8ad1e17614508c98a4505be6613c3d6af4145a30 Mon Sep 17 00:00:00 2001 From: boy Date: Thu, 23 Jul 2026 03:17:03 +0200 Subject: [PATCH 10/13] fix(deploy): abort on foreign legacy pod conflicts --- README.md | 4 ++-- start-stack.sh | 8 +++++++- tests/test_quadlet_templates.py | 3 ++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 23cb3d2c..01951806 100644 --- a/README.md +++ b/README.md @@ -265,7 +265,7 @@ Exposes the entry endpoint (`http://localhost:5000/v1`) and evaluates prompt com | `llm-routing-auto-agy-ollama` | ✅ | agy → Ollama (gated: reasoning/advanced/complex) | LiteLLM with classified tier | 512K | | `llm-routing-agy` | ❌ | agy (Gemini/Claude) — unconditional | LiteLLM agent-advanced-core | 1M | | `llm-routing-ollama` | ✅ | Ollama (gated: reasoning & advanced → ollama-deepseek-v4-pro, complex & below → ollama-deepseek-v4-flash) | LiteLLM openrouter-auto | 512K | -| `agent-advanced-core` | ❌ | — | LiteLLM openrouter-auto | 262K | +| `agent-advanced-core` | ❌ | — | LiteLLM `local-qwen-3.6` → `llm-routing-ollama` → `openrouter-auto` | 262K | | `agent-reasoning-core` | ❌ | — | LiteLLM fallback chain | 262K | | `agent-complex-core` | ❌ | — | LiteLLM fallback chain | 262K | | `agent-medium-core` | ❌ | — | LiteLLM fallback chain | 262K | @@ -285,7 +285,7 @@ Orchestrates routing fallback chains, Redis caching, and telemetry callbacks: - `embedding_model: "local-nomic-embed"` — uses the local nomic-embed model (no API costs) - `collection_name: "litellm_semantic_cache"` — stores embeddings for similarity-based cache lookups - **Cascading Fallback Chains** (configured in `litellm_settings.fallbacks`): - Each tier escalates through increasingly capable free models, then paid local/remote Ollama models, and finally falls back to `openrouter-auto` (LiteLLM's internal fallback to OpenRouter `/auto`). + Each tier escalates through increasingly capable free models, then the local llama.cpp safety net (`local-qwen-3.6`), then the paid/remote Ollama tier, and finally falls back to `openrouter-auto` (LiteLLM's internal fallback to OpenRouter `/auto`). ```mermaid graph TD diff --git a/start-stack.sh b/start-stack.sh index 6159ff9c..04113cfd 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -568,7 +568,7 @@ stack_ownership() { elif [[ "$infra_unit" == "$LEGACY_LLM_ROUTING_POD_UNIT" ]] && legacy_unit_owns_pod "$LEGACY_LLM_ROUTING_POD_UNIT"; then printf 'quadlet:%s\n' "$infra_unit" elif [[ "$infra_unit" == "$LEGACY_LLM_ROUTING_POD_UNIT" ]]; then - printf 'absent\n' + printf 'conflict:%s\n' "$infra_unit" else printf 'legacy\n' fi @@ -886,6 +886,12 @@ deploy_fresh_pod() { } STACK_OWNERSHIP=$(stack_ownership) +if [[ "$STACK_OWNERSHIP" == conflict:* ]]; then + conflict_unit="${STACK_OWNERSHIP#conflict:}" + echo "❌ Error: pod ${POD_NAME} is attached to an unrelated legacy unit ${conflict_unit}; refusing to stop, replace, or deploy it." >&2 + echo " Inspect with: systemctl --user cat ${conflict_unit} --no-pager" >&2 + exit 1 +fi if [[ "$STACK_OWNERSHIP" != "absent" ]]; then if $FULL_REBUILD; then echo "🔨 Building custom local triage router image..." diff --git a/tests/test_quadlet_templates.py b/tests/test_quadlet_templates.py index 68279ed3..abc62963 100644 --- a/tests/test_quadlet_templates.py +++ b/tests/test_quadlet_templates.py @@ -155,7 +155,8 @@ def test_namespace_is_validated_and_ownership_preserves_exact_unit(): assert 'grep -Fqx "PodName=${POD_NAME}"' in script assert '&& legacy_unit_owns_pod "$LEGACY_LLM_ROUTING_POD_UNIT"' in script assert 'elif [[ "$infra_unit" == "$LEGACY_LLM_ROUTING_POD_UNIT" ]]; then' in script - assert "printf 'absent\\n'" in script + assert 'printf \'conflict:%s\\n\' "$infra_unit"' in script + assert 'STACK_OWNERSHIP" == conflict:*' in script def test_documentation_uses_environment_specific_units(): From f9caaf69baeaba6f24854fbb522e5722801631fe Mon Sep 17 00:00:00 2001 From: boy Date: Thu, 23 Jul 2026 17:31:21 +0200 Subject: [PATCH 11/13] fix(deploy): remove unreachable namespace branch --- start-stack.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/start-stack.sh b/start-stack.sh index 04113cfd..cdd775b8 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -801,9 +801,7 @@ try: text = text.replace(ph, str(val)) # Quadlet generated unit names are global in the user systemd manager. # Namespace both filenames and internal dependencies to isolate dev/prod. - # Namespace Quadlet identifiers only. Do not rewrite image names, URL - # paths, or other configuration values containing "llm-routing". - # Namespace only Quadlet identifier lines and values. This preserves + # Namespace Quadlet identifier lines and values only. This preserves # arbitrary image names, URLs, and credentials containing llm-routing. def namespace_identifier(match): field, value = match.group(1), match.group(2) @@ -811,8 +809,6 @@ try: value = identifier_prefix.sub(namespace + "-", value) value = value.replace("llm-routing.pod", namespace + ".pod") value = value.replace("llm-routing-pod.service", namespace + "-pod.service") - elif field == "Pod": - value = value.replace("llm-routing.pod", namespace + ".pod") return f"{field}={value}" text = re.sub(r"(?m)^(Pod|After|Wants|BindsTo|Requires|PartOf)=(.*)$", namespace_identifier, text) unresolved = sorted(set(re.findall(r"\b[A-Z0-9_]+_PLACEHOLDER\b", text))) From 88a61e9528c4231f9130d0191e961e8a36857f2d Mon Sep 17 00:00:00 2001 From: boy Date: Thu, 23 Jul 2026 17:50:43 +0200 Subject: [PATCH 12/13] fix(config): preserve dev environment overlay in containers --- README.md | 13 +++++++--- litellm/entrypoint.py | 8 +++++- pod.yaml | 6 ++--- quadlets/llm-routing-litellm.container | 2 +- quadlets/llm-routing-router.container | 2 +- scripts/README.md | 1 + start-stack.sh | 35 +++++++++++++++++++++++++- tests/test_quadlet_templates.py | 14 ++++++++++- 8 files changed, 70 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 01951806..2dfcd497 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ graph TD ## 1b. Container Health Checks & Auto-Restart -All core containers are configured with **Kubernetes-style liveness and readiness probes** in [`pod.yaml`](pod.yaml) to enable automatic container restart on crash via Podman. This ensures the stack self-heals without manual intervention. +All core containers are configured with health checks in the Quadlet templates under [`quadlets/`](quadlets/). The legacy [`pod.yaml`](pod.yaml) is retained as a compatibility template. Quadlet `HealthOnFailure=kill` together with systemd `Restart=always` enables automatic recovery from unhealthy containers. | Container | Liveness Probe | Readiness Probe | |:---|---:|---:| @@ -216,7 +216,8 @@ All configurations, automation scripts, and databases are self-contained within ├── .env # Environment file for API keys, passwords, and generated secrets (ignored by git) ├── .gitignore # Git ignore policy protecting secrets & database files ├── README.md # In-depth system and operational guide -├── pod.yaml # Podman Kubernetes template defining the 10-container stack +├── quadlets/ # Quadlet templates for the systemd-managed stack +├── pod.yaml # Legacy Podman Kubernetes compatibility template ├── start-stack.sh # Unified startup and credential extraction script ├── pytest.ini # Pytest configuration (test discovery, asyncio mode) ├── litellm/ @@ -858,7 +859,7 @@ Tests cover: | LiteLLM direct | 1 completion directly to LiteLLM | | Canonical URLs | 7 GET + 1 POST through public HTTPS (graceful DNS skip) | -Requires `PUBLIC_BASE_URL` in `.env` for canonical URL tests. The router remains under its configured path (for example `https://x570.vendeuvre.lan/llm-routing`), while the verifier derives service URLs from its host: `https://litellm./ui/`, `https://langfuse./`, and `https://llama./health`. Dev `.env.dev` already has it; prod `.env` should include `PUBLIC_BASE_URL="https://x570.vendeuvre.lan/llm-routing"`. The dev local-model safety net uses the host-networked local listener: `LLAMA_CLASSIFIER_URL=http://127.0.0.1:8083/v1` and `LLAMA_SERVER_URL=http://127.0.0.1:8083`; it must not depend on TLS-terminated dev or production hostnames. +Requires `PUBLIC_BASE_URL` in `.env` for canonical URL tests. The router remains under its configured path (for example `https://x570.vendeuvre.lan/llm-routing`), while the verifier derives service URLs from its host: `https://litellm./ui/`, `https://langfuse./`, and `https://llama./health`. Dev `.env.dev` overlays the base `.env` during `--dev` verification; production `.env` should include `PUBLIC_BASE_URL="https://x570.vendeuvre.lan/llm-routing"`. The dev local-model safety net uses the host-networked local listener: `LLAMA_CLASSIFIER_URL=http://127.0.0.1:8083/v1` and `LLAMA_SERVER_URL=http://127.0.0.1:8083`; it must not depend on TLS-terminated dev or production hostnames. ## Environment-isolated Quadlet deployment @@ -873,6 +874,12 @@ and rendered configuration remain separate. Unless overridden explicitly, `~/dev/` and `~/prod/` worktrees, their persistent data and rendered configs are also physically separate. +The deployment script writes the fully merged environment (base `.env` followed +by the optional `.env.dev` overlay) to `${DATA_ROOT}/effective.env`. The router +and LiteLLM containers source this generated file, so dev-only URLs, ports, and +other overrides reach the containers without modifying the production `.env`. +The generated file is owner-only and is never committed. + ## 10. Performance Benchmarks Through our local benchmarks, the following performance characteristics have been achieved: diff --git a/litellm/entrypoint.py b/litellm/entrypoint.py index 6993dc89..c6e4b7b6 100644 --- a/litellm/entrypoint.py +++ b/litellm/entrypoint.py @@ -6,6 +6,7 @@ import time import socket import datetime +import shlex from datetime import datetime as original_datetime, timezone # Load .env into os.environ @@ -17,7 +18,12 @@ if line and not line.startswith("#") and "=" in line: key, _, val = line.partition("=") key = key.strip() - val = val.strip().strip('"').strip("'") + # effective.env is shell-quoted by start-stack.sh; parse it with + # the same rules as the router's `source /config/.env`. + try: + val = shlex.split(val, comments=False, posix=True)[0] if val else "" + except ValueError: + val = val.strip().strip('"').strip("'") os.environ.setdefault(key, val) # Load Gemini OAuth token from credentials JSON diff --git a/pod.yaml b/pod.yaml index 0a3d18f5..f1e2ea95 100644 --- a/pod.yaml +++ b/pod.yaml @@ -98,7 +98,7 @@ spec: subPath: entrypoint.py - mountPath: /config/.env name: env-file - subPath: .env + subPath: effective.env - mountPath: /config/gemini_auth name: gemini-secrets # LLM Triage Router — classification + routing gateway @@ -184,7 +184,7 @@ spec: name: xdg-runtime - mountPath: /config/.env name: env-file - subPath: .env + subPath: effective.env # PostgreSQL — Langfuse + LiteLLM metadata store - env: - name: POSTGRES_USER @@ -527,7 +527,7 @@ spec: path: RUN_USER_PLACEHOLDER name: xdg-runtime - hostPath: - path: WORKDIR_PLACEHOLDER + path: DATA_ROOT_PLACEHOLDER name: env-file - hostPath: path: DATA_ROOT_PLACEHOLDER/datasets diff --git a/quadlets/llm-routing-litellm.container b/quadlets/llm-routing-litellm.container index aff6e059..09dda590 100644 --- a/quadlets/llm-routing-litellm.container +++ b/quadlets/llm-routing-litellm.container @@ -30,7 +30,7 @@ Environment=LITELLM_PORT=LITELLM_PORT_PLACEHOLDER Environment=POSTGRES_PORT=POSTGRES_PORT_PLACEHOLDER Volume=DATA_ROOT_PLACEHOLDER/litellm-rendered/config.yaml:/app/config.yaml Volume=DATA_ROOT_PLACEHOLDER/litellm-rendered/entrypoint.py:/app/entrypoint.py -Volume=WORKDIR_PLACEHOLDER/.env:/config/.env +Volume=EFFECTIVE_ENV_FILE_PLACEHOLDER:/config/.env Volume=HOME_PLACEHOLDER/.gemini:/config/gemini_auth HealthCmd=python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:LITELLM_PORT_PLACEHOLDER/health/liveness')" HealthInterval=15s diff --git a/quadlets/llm-routing-router.container b/quadlets/llm-routing-router.container index 72023f9d..5c10f197 100644 --- a/quadlets/llm-routing-router.container +++ b/quadlets/llm-routing-router.container @@ -44,7 +44,7 @@ Volume=HOME_PLACEHOLDER/.local/bin/agy:/usr/local/bin/agy Volume=RUN_USER_PLACEHOLDER/bus:RUN_USER_PLACEHOLDER/bus Volume=HOME_PLACEHOLDER/.local/share/keyrings:/root/.local/share/keyrings Volume=RUN_USER_PLACEHOLDER:RUN_USER_PLACEHOLDER -Volume=WORKDIR_PLACEHOLDER/.env:/config/.env +Volume=EFFECTIVE_ENV_FILE_PLACEHOLDER:/config/.env HealthCmd=python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:ROUTER_PORT_PLACEHOLDER/metrics')" HealthInterval=15s HealthTimeout=5s diff --git a/scripts/README.md b/scripts/README.md index f893f3b1..2370c4d2 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -13,6 +13,7 @@ Unified startup and credential extraction script for the systemd Quadlet-managed - `./start-stack.sh --replace` (Stop + clean ports + render/install Quadlets + daemon-reload + recreate stack) - `./start-stack.sh --full-rebuild` (Same as `--replace` + rebuild the triage router image; required for code changes in `router/`) - Quadlet templates live in `quadlets/`; rendered owner-only units use environment-specific namespaces: dev under `~/.config/containers/systemd/llm-routing-dev/` and prod under `~/.config/containers/systemd/llm-routing-prod/`. Dev uses `llm-routing-dev-pod.service`; prod uses `llm-routing-prod-pod.service`. Use the matching `systemctl --user status -pod.service --no-pager` and `journalctl --user -u -router.service --no-pager` for lifecycle diagnostics. +- Before rendering, the script writes the merged `.env` plus optional `.env.dev` overlay to `${DATA_ROOT}/effective.env` (mode `600`). Router and LiteLLM mount this generated file as `/config/.env`; the source `.env` remains unchanged. ### `scripts/backup.sh` Automated database backup script that runs before every stack deployment. Uses `pg_isready` to safely wait for database connections and manages timestamped backups under `backups/`. diff --git a/start-stack.sh b/start-stack.sh index cdd775b8..080ceb35 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -124,6 +124,10 @@ LOCAL_BASE_URL="${LOCAL_BASE_URL:-http://localhost:${ROUTER_PORT}}" LOCAL_BASE_URL="${LOCAL_BASE_URL%/}" export PUBLIC_BASE_URL LOCAL_BASE_URL +# Containers source this generated file, not the production .env bind mount. +# This preserves the .env + optional .env.dev overlay inside each container. +EFFECTIVE_ENV_FILE="${DATA_ROOT}/effective.env" +export EFFECTIVE_ENV_FILE # Ensure openssl is installed if we need to generate passwords/keys if [ -z "$POSTGRES_PASSWORD" ] || [ -z "$NEXTAUTH_SECRET" ] || [ -z "$SALT" ] || [ -z "$ENCRYPTION_KEY" ] || [ -z "$LITELLM_MASTER_KEY" ] || [ -z "$ROUTER_API_KEY" ] || [ -z "$MINIO_ROOT_USER" ] || [ -z "$MINIO_ROOT_PASSWORD" ] || [ -z "$LANGFUSE_INIT_USER_PASSWORD" ] || [ -z "$REDIS_AUTH" ] || [ -z "$CLICKHOUSE_PASSWORD" ] || [ -z "$LANGFUSE_PUBLIC_KEY" ] || [ -z "$LANGFUSE_SECRET_KEY" ]; then @@ -359,8 +363,36 @@ if [ -z "$CLASSIFIER_INPUT_MAX_CHARS" ]; then echo "✓ Set default CLASSIFIER_INPUT_MAX_CHARS=300 and saved to $ENV_FILE" fi +# Persist only the application environment for bind-mounted container consumers. +# Quote values as shell syntax so URLs, credentials, and special characters +# survive the second `source` performed by the router/LiteLLM entrypoints. +python3 - "$EFFECTIVE_ENV_FILE" <<'PY' +import os +import shlex +import sys + +# Explicit allowlist: never copy unrelated host credentials into containers. +APPLICATION_ENV = { + "CLASSIFIER_INPUT_MAX_CHARS", "CLICKHOUSE_HTTP_PORT", "CLICKHOUSE_PASSWORD", + "CLICKHOUSE_TCP_PORT", "DATA_ROOT", "LANGFUSE_INIT_USER_PASSWORD", + "LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", "LANGFUSE_WEB_PORT", + "LANGFUSE_WORKER_PORT", "LLAMA_CLASSIFIER_URL", "LLAMA_SERVER_URL", + "LITELLM_MASTER_KEY", "LITELLM_PORT", "MINIO_CONSOLE_PORT", "MINIO_ROOT_PASSWORD", + "MINIO_ROOT_USER", "MINIO_S3_PORT", "NEXTAUTH_SECRET", "NEXTAUTH_URL", + "OLLAMA_API_KEY", "OPENROUTER_API_KEY", "POD_NAME", "POSTGRES_PASSWORD", + "POSTGRES_PORT", "PROXY_BASE_URL", "PUBLIC_BASE_URL", "QUADLET_NAMESPACE", + "REDIS_AUTH", "ROUTER_API_KEY", "ROUTER_IMAGE", "ROUTER_PORT", "ROUTING_DOMAIN", + "SALT", "ENCRYPTION_KEY", "UI_PASSWORD", "UI_USERNAME", "VALKEY_CACHE_PORT", + "VALKEY_LF_PORT", +} - +target = sys.argv[1] +with open(target, "w", encoding="utf-8") as handle: + for key in sorted(APPLICATION_ENV): + if key in os.environ: + handle.write(f"{key}={shlex.quote(os.environ[key])}\n") +os.chmod(target, 0o600) +PY # DYNAMIC_LITELLM_MASTER_KEY_PLACEHOLDER in router config is resolved at runtime from env @@ -773,6 +805,7 @@ repl = { "LLAMA_SERVER_URL_PLACEHOLDER": os.environ["LLAMA_SERVER_URL"], "POD_NAME_PLACEHOLDER": os.environ["POD_NAME"], "DATA_ROOT_PLACEHOLDER": os.environ["DATA_ROOT"], + "EFFECTIVE_ENV_FILE_PLACEHOLDER": os.environ["EFFECTIVE_ENV_FILE"], "ROUTER_IMAGE_PLACEHOLDER": os.environ["ROUTER_IMAGE"], "ROUTER_PORT_PLACEHOLDER": os.environ["ROUTER_PORT"], "LITELLM_PORT_PLACEHOLDER": os.environ["LITELLM_PORT"], diff --git a/tests/test_quadlet_templates.py b/tests/test_quadlet_templates.py index abc62963..4466d1eb 100644 --- a/tests/test_quadlet_templates.py +++ b/tests/test_quadlet_templates.py @@ -85,6 +85,17 @@ def test_router_quadlet_reasserts_overlayed_llama_urls_after_env_source(): assert '"LLAMA_SERVER_URL_PLACEHOLDER": os.environ["LLAMA_SERVER_URL"]' in script +def test_containers_source_the_merged_effective_environment(): + script = (ROOT / "start-stack.sh").read_text() + for template in (QUADLETS / "llm-routing-router.container", QUADLETS / "llm-routing-litellm.container"): + assert "EFFECTIVE_ENV_FILE_PLACEHOLDER:/config/.env" in template.read_text() + assert 'EFFECTIVE_ENV_FILE="${DATA_ROOT}/effective.env"' in script + assert '"EFFECTIVE_ENV_FILE_PLACEHOLDER": os.environ["EFFECTIVE_ENV_FILE"]' in script + assert "shlex.quote(os.environ[key])" in script + assert "APPLICATION_ENV = {" in script + assert "for key in sorted(APPLICATION_ENV):" in script + + def test_quadlet_deployment_enforces_prerequisite_and_restart_failures(): script = (ROOT / "start-stack.sh").read_text() assert "render_pod_yaml()" not in script @@ -120,7 +131,8 @@ def test_namespace_rendering_preserves_non_identifiers(): "NEXTAUTH_URL_DERIVED": "https://next", "PUBLIC_BASE_URL": "https://host/llm-routing", "ROUTING_DOMAIN": "vendeuvre.lan", "LLAMA_CLASSIFIER_URL": "http://127.0.0.1:8083/v1", "LLAMA_SERVER_URL": "http://127.0.0.1:8083", "POD_NAME": "dev-router-pod", - "DATA_ROOT": str(ROOT / "data"), "ROUTER_IMAGE": "registry/llm-routing-router:latest", + "DATA_ROOT": str(ROOT / "data"), "EFFECTIVE_ENV_FILE": str(ROOT / "data" / "effective.env"), + "ROUTER_IMAGE": "registry/llm-routing-router:latest", "ROUTER_PORT": "5010", "LITELLM_PORT": "4010", "LANGFUSE_WEB_PORT": "3011", "LANGFUSE_WORKER_PORT": "3030", "POSTGRES_PORT": "5442", "VALKEY_CACHE_PORT": "6389", "VALKEY_LF_PORT": "6390", "CLICKHOUSE_HTTP_PORT": "8123", "CLICKHOUSE_TCP_PORT": "9003", From a24842bdca55314daab068679fd9cedf77b927e5 Mon Sep 17 00:00:00 2001 From: boy Date: Thu, 23 Jul 2026 17:54:12 +0200 Subject: [PATCH 13/13] fix(deploy): verify legacy pod owner command --- start-stack.sh | 9 ++++++--- tests/test_quadlet_templates.py | 6 ++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/start-stack.sh b/start-stack.sh index 080ceb35..2d560a8c 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -585,12 +585,15 @@ stack_ownership() { local infra_unit # A generic legacy unit is shared by old deployments. Only treat it as - # owned when its generated unit explicitly names this stack's pod; merely - # being loaded is not sufficient and could tear down the other environment. + # owned when its generated ExecStartPre command creates this stack's pod; + # a PodName= source line alone does not prove which pod the unit owns. legacy_unit_owns_pod() { local unit="$1" + local pod_name_pattern + pod_name_pattern=$(printf '%s' "$POD_NAME" | sed 's/[][\\.^$*+?(){}|]/\\&/g') systemctl --user cat "$unit" --no-pager 2>/dev/null \ - | grep -Fqx "PodName=${POD_NAME}" + | grep -E 'podman[[:space:]]+pod[[:space:]]+create' \ + | grep -Eq -- "--name[=[:space:]]${pod_name_pattern}([[:space:]]|$)" } if podman pod exists "${POD_NAME}" 2>/dev/null; then diff --git a/tests/test_quadlet_templates.py b/tests/test_quadlet_templates.py index 4466d1eb..c94bccec 100644 --- a/tests/test_quadlet_templates.py +++ b/tests/test_quadlet_templates.py @@ -141,7 +141,8 @@ def test_namespace_rendering_preserves_non_identifiers(): env.update(values) with tempfile.TemporaryDirectory() as tmp: src, out = Path(tmp) / "src", Path(tmp) / "out" - src.mkdir(); out.mkdir() + src.mkdir() + out.mkdir() (src / "llm-routing.pod").write_text("[Pod]\nPodName=llm-routing.pod\n") (src / "llm-routing-router.container").write_text( "[Unit]\nAfter=llm-routing-litellm.service\n[Container]\n" @@ -164,7 +165,8 @@ def test_namespace_is_validated_and_ownership_preserves_exact_unit(): assert 'failed to restart ${owner_unit}' in script assert 'status ${owner_unit} --no-pager' in script assert 'legacy_unit_owns_pod()' in script - assert 'grep -Fqx "PodName=${POD_NAME}"' in script + assert "grep -E 'podman[[:space:]]+pod[[:space:]]+create'" in script + assert 'grep -Eq -- "--name[=[:space:]]${pod_name_pattern}' in script assert '&& legacy_unit_owns_pod "$LEGACY_LLM_ROUTING_POD_UNIT"' in script assert 'elif [[ "$infra_unit" == "$LEGACY_LLM_ROUTING_POD_UNIT" ]]; then' in script assert 'printf \'conflict:%s\\n\' "$infra_unit"' in script