diff --git a/.env.dev b/.env.dev index 4565148a..3ee30ba0 100644 --- a/.env.dev +++ b/.env.dev @@ -31,3 +31,6 @@ ROUTER_IMAGE="localhost/llm-routing-dev:latest" # reaches its local listener directly rather than any TLS-terminated hostname. LLAMA_CLASSIFIER_URL="http://127.0.0.1:8083/v1" LLAMA_SERVER_URL="http://127.0.0.1:8083" + +# Distinct systemd Quadlet namespace; never share generated units with prod. +QUADLET_NAMESPACE="llm-routing-dev" diff --git a/README.md b/README.md index a1324e18..2dfcd497 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ graph TD ## 1b. Container Health Checks & Auto-Restart -All core containers are configured with **Kubernetes-style liveness and readiness probes** in [`pod.yaml`](pod.yaml) to enable automatic container restart on crash via Podman. This ensures the stack self-heals without manual intervention. +All core containers are configured with health checks in the Quadlet templates under [`quadlets/`](quadlets/). The legacy [`pod.yaml`](pod.yaml) is retained as a compatibility template. Quadlet `HealthOnFailure=kill` together with systemd `Restart=always` enables automatic recovery from unhealthy containers. | Container | Liveness Probe | Readiness Probe | |:---|---:|---:| @@ -216,7 +216,8 @@ All configurations, automation scripts, and databases are self-contained within ├── .env # Environment file for API keys, passwords, and generated secrets (ignored by git) ├── .gitignore # Git ignore policy protecting secrets & database files ├── README.md # In-depth system and operational guide -├── pod.yaml # Podman Kubernetes template defining the 10-container stack +├── quadlets/ # Quadlet templates for the systemd-managed stack +├── pod.yaml # Legacy Podman Kubernetes compatibility template ├── start-stack.sh # Unified startup and credential extraction script ├── pytest.ini # Pytest configuration (test discovery, asyncio mode) ├── litellm/ @@ -265,7 +266,7 @@ Exposes the entry endpoint (`http://localhost:5000/v1`) and evaluates prompt com | `llm-routing-auto-agy-ollama` | ✅ | agy → Ollama (gated: reasoning/advanced/complex) | LiteLLM with classified tier | 512K | | `llm-routing-agy` | ❌ | agy (Gemini/Claude) — unconditional | LiteLLM agent-advanced-core | 1M | | `llm-routing-ollama` | ✅ | Ollama (gated: reasoning & advanced → ollama-deepseek-v4-pro, complex & below → ollama-deepseek-v4-flash) | LiteLLM openrouter-auto | 512K | -| `agent-advanced-core` | ❌ | — | LiteLLM openrouter-auto | 262K | +| `agent-advanced-core` | ❌ | — | LiteLLM `local-qwen-3.6` → `llm-routing-ollama` → `openrouter-auto` | 262K | | `agent-reasoning-core` | ❌ | — | LiteLLM fallback chain | 262K | | `agent-complex-core` | ❌ | — | LiteLLM fallback chain | 262K | | `agent-medium-core` | ❌ | — | LiteLLM fallback chain | 262K | @@ -285,7 +286,7 @@ Orchestrates routing fallback chains, Redis caching, and telemetry callbacks: - `embedding_model: "local-nomic-embed"` — uses the local nomic-embed model (no API costs) - `collection_name: "litellm_semantic_cache"` — stores embeddings for similarity-based cache lookups - **Cascading Fallback Chains** (configured in `litellm_settings.fallbacks`): - Each tier escalates through increasingly capable free models, then paid local/remote Ollama models, and finally falls back to `openrouter-auto` (LiteLLM's internal fallback to OpenRouter `/auto`). + Each tier escalates through increasingly capable free models, then the local llama.cpp safety net (`local-qwen-3.6`), then the paid/remote Ollama tier, and finally falls back to `openrouter-auto` (LiteLLM's internal fallback to OpenRouter `/auto`). ```mermaid graph TD @@ -303,7 +304,8 @@ Orchestrates routing fallback chains, Redis caching, and telemetry callbacks: SM --> SC[agent-complex-core]:::complex SC --> SR[agent-reasoning-core]:::reasoning SR --> SA[agent-advanced-core]:::advanced - SA --> SO1[llm-routing-ollama]:::premium + SA --> SL[local-qwen-3.6]:::local + SL --> SO1[llm-routing-ollama]:::premium SO1 --> SAU[openrouter-auto]:::auto end @@ -311,34 +313,38 @@ Orchestrates routing fallback chains, Redis caching, and telemetry callbacks: M[agent-medium-core]:::medium --> MC[agent-complex-core]:::complex MC --> MR[agent-reasoning-core]:::reasoning MR --> MA[agent-advanced-core]:::advanced - MA --> MO1[llm-routing-ollama]:::premium + MA --> ML[local-qwen-3.6]:::local + ML --> MO1[llm-routing-ollama]:::premium MO1 --> MAU[openrouter-auto]:::auto end subgraph Complex["agent-complex-core Fallback Tree"] C[agent-complex-core]:::complex --> CR[agent-reasoning-core]:::reasoning CR --> CA[agent-advanced-core]:::advanced - CA --> CO1[llm-routing-ollama]:::premium + CA --> CL[local-qwen-3.6]:::local + CL --> CO1[llm-routing-ollama]:::premium CO1 --> CAU[openrouter-auto]:::auto end subgraph Reasoning["agent-reasoning-core Fallback Tree"] R[agent-reasoning-core]:::reasoning --> RA[agent-advanced-core]:::advanced - RA --> RO1[llm-routing-ollama]:::premium + RA --> RL[local-qwen-3.6]:::local + RL --> RO1[llm-routing-ollama]:::premium RO1 --> RAU[openrouter-auto]:::auto end subgraph Advanced["agent-advanced-core Fallback Tree"] - A[agent-advanced-core]:::advanced --> AO1[llm-routing-ollama]:::premium + A[agent-advanced-core]:::advanced --> AL[local-qwen-3.6]:::local + AL --> AO1[llm-routing-ollama]:::premium AO1 --> AAU[openrouter-auto]:::auto end ``` - - **`agent-simple-core`**: medium-core → complex-core → reasoning-core → advanced-core → `llm-routing-ollama` → `openrouter-auto` - - **`agent-medium-core`**: complex-core → reasoning-core → advanced-core → `llm-routing-ollama` → `openrouter-auto` - - **`agent-complex-core`**: reasoning-core → advanced-core → `llm-routing-ollama` → `openrouter-auto` - - **`agent-reasoning-core`**: advanced-core → `llm-routing-ollama` → `openrouter-auto` - - **`agent-advanced-core`**: `llm-routing-ollama` → `openrouter-auto` + - **`agent-simple-core`**: medium-core → complex-core → reasoning-core → advanced-core → `local-qwen-3.6` → `llm-routing-ollama` → `openrouter-auto` + - **`agent-medium-core`**: complex-core → reasoning-core → advanced-core → `local-qwen-3.6` → `llm-routing-ollama` → `openrouter-auto` + - **`agent-complex-core`**: reasoning-core → advanced-core → `local-qwen-3.6` → `llm-routing-ollama` → `openrouter-auto` + - **`agent-reasoning-core`**: advanced-core → `local-qwen-3.6` → `llm-routing-ollama` → `openrouter-auto` + - **`agent-advanced-core`**: `local-qwen-3.6` → `llm-routing-ollama` → `openrouter-auto` - **`llm-routing-ollama`** (classifier-gated proxy): `reasoning & advanced` → `ollama-deepseek-v4-pro`, `complex & below` → `ollama-deepseek-v4-flash`. Note: Ollama cooldowns are managed by the triage router internally (5-minute window on failure); during cooldown the router returns 429 immediately so LiteLLM skips to `openrouter-auto`. All tiers ultimately land on OpenRouter auto/free model pools or the local Speculative MoE when enabled. *Note: Premium routing is controlled by the model name, not by the tier. `llm-routing-agy` and `llm-routing-auto-agy` trigger the agy proxy (Google/Claude via Cloud Code Assist) — but auto models only trigger agy if the classifier returns `agent-advanced-core`. `llm-routing-ollama` and `llm-routing-auto-ollama` route through Ollama.com (deepseek-v4-pro via LiteLLM's ollama_chat provider) — same gating for auto models. `llm-routing-auto-agy-ollama` chains both: agy first, then Ollama if agy is exhausted, both gated on advanced classification. The `agent-advanced-core` tier itself is a plain LiteLLM tier with no premium trigger. See §2 for the full routing table.* @@ -395,9 +401,9 @@ Run the startup script from the root of the repository: ./start-stack.sh --full-rebuild # Same as --replace plus rebuild the router image # Inspect the generated systemd units and their logs -systemctl --user status llm-routing-pod.service --no-pager -systemctl --user list-units 'llm-routing-*' --no-pager -journalctl --user -u llm-routing-router.service -n 100 --no-pager +systemctl --user status llm-routing-prod-pod.service --no-pager # or llm-routing-dev-pod.service +systemctl --user list-units 'llm-routing-*' --no-pager # filter for dev/prod namespaces +journalctl --user -u llm-routing-prod-router.service -n 100 --no-pager # or llm-routing-dev-router.service ``` *Note: If running for the first time, the script will prompt you for your `OpenRouter API Key`, securely saving it inside `.env` with restrictive permissions (`chmod 600`). The script also automatically generates and persists secure random secrets (`LITELLM_MASTER_KEY`, `POSTGRES_PASSWORD`, `NEXTAUTH_SECRET`, `SALT`, `ENCRYPTION_KEY`, and `ROUTER_API_KEY`) to this file on startup if they are missing.* @@ -853,7 +859,26 @@ Tests cover: | LiteLLM direct | 1 completion directly to LiteLLM | | Canonical URLs | 7 GET + 1 POST through public HTTPS (graceful DNS skip) | -Requires `PUBLIC_BASE_URL` in `.env` for canonical URL tests. The router remains under its configured path (for example `https://x570.vendeuvre.lan/llm-routing`), while the verifier derives service URLs from its host: `https://litellm./ui/`, `https://langfuse./`, and `https://llama./health`. Dev `.env.dev` already has it; prod `.env` should include `PUBLIC_BASE_URL="https://x570.vendeuvre.lan/llm-routing"`. The dev local-model safety net uses the host-networked local listener: `LLAMA_CLASSIFIER_URL=http://127.0.0.1:8083/v1` and `LLAMA_SERVER_URL=http://127.0.0.1:8083`; it must not depend on TLS-terminated dev or production hostnames. +Requires `PUBLIC_BASE_URL` in `.env` for canonical URL tests. The router remains under its configured path (for example `https://x570.vendeuvre.lan/llm-routing`), while the verifier derives service URLs from its host: `https://litellm./ui/`, `https://langfuse./`, and `https://llama./health`. Dev `.env.dev` overlays the base `.env` during `--dev` verification; production `.env` should include `PUBLIC_BASE_URL="https://x570.vendeuvre.lan/llm-routing"`. The dev local-model safety net uses the host-networked local listener: `LLAMA_CLASSIFIER_URL=http://127.0.0.1:8083/v1` and `LLAMA_SERVER_URL=http://127.0.0.1:8083`; it must not depend on TLS-terminated dev or production hostnames. + +## Environment-isolated Quadlet deployment + +Dev and production use distinct Quadlet namespaces because generated systemd +unit names are global within the user manager. Dev renders units under +`~/.config/containers/systemd/llm-routing-dev/` and uses +`llm-routing-dev-pod.service`; production uses +`~/.config/containers/systemd/llm-routing-prod/` and +`llm-routing-prod-pod.service`. Their pod/container names, ports, data roots, +and rendered configuration remain separate. Unless overridden explicitly, +`DATA_ROOT` is `${WORKDIR}/data`; because dev and production run from separate +`~/dev/` and `~/prod/` worktrees, their persistent data and rendered configs are +also physically separate. + +The deployment script writes the fully merged environment (base `.env` followed +by the optional `.env.dev` overlay) to `${DATA_ROOT}/effective.env`. The router +and LiteLLM containers source this generated file, so dev-only URLs, ports, and +other overrides reach the containers without modifying the production `.env`. +The generated file is owner-only and is never committed. ## 10. Performance Benchmarks diff --git a/litellm/entrypoint.py b/litellm/entrypoint.py index 6993dc89..c6e4b7b6 100644 --- a/litellm/entrypoint.py +++ b/litellm/entrypoint.py @@ -6,6 +6,7 @@ import time import socket import datetime +import shlex from datetime import datetime as original_datetime, timezone # Load .env into os.environ @@ -17,7 +18,12 @@ if line and not line.startswith("#") and "=" in line: key, _, val = line.partition("=") key = key.strip() - val = val.strip().strip('"').strip("'") + # effective.env is shell-quoted by start-stack.sh; parse it with + # the same rules as the router's `source /config/.env`. + try: + val = shlex.split(val, comments=False, posix=True)[0] if val else "" + except ValueError: + val = val.strip().strip('"').strip("'") os.environ.setdefault(key, val) # Load Gemini OAuth token from credentials JSON diff --git a/pod.yaml b/pod.yaml index 0a3d18f5..f1e2ea95 100644 --- a/pod.yaml +++ b/pod.yaml @@ -98,7 +98,7 @@ spec: subPath: entrypoint.py - mountPath: /config/.env name: env-file - subPath: .env + subPath: effective.env - mountPath: /config/gemini_auth name: gemini-secrets # LLM Triage Router — classification + routing gateway @@ -184,7 +184,7 @@ spec: name: xdg-runtime - mountPath: /config/.env name: env-file - subPath: .env + subPath: effective.env # PostgreSQL — Langfuse + LiteLLM metadata store - env: - name: POSTGRES_USER @@ -527,7 +527,7 @@ spec: path: RUN_USER_PLACEHOLDER name: xdg-runtime - hostPath: - path: WORKDIR_PLACEHOLDER + path: DATA_ROOT_PLACEHOLDER name: env-file - hostPath: path: DATA_ROOT_PLACEHOLDER/datasets diff --git a/quadlets/llm-routing-litellm.container b/quadlets/llm-routing-litellm.container index aff6e059..09dda590 100644 --- a/quadlets/llm-routing-litellm.container +++ b/quadlets/llm-routing-litellm.container @@ -30,7 +30,7 @@ Environment=LITELLM_PORT=LITELLM_PORT_PLACEHOLDER Environment=POSTGRES_PORT=POSTGRES_PORT_PLACEHOLDER Volume=DATA_ROOT_PLACEHOLDER/litellm-rendered/config.yaml:/app/config.yaml Volume=DATA_ROOT_PLACEHOLDER/litellm-rendered/entrypoint.py:/app/entrypoint.py -Volume=WORKDIR_PLACEHOLDER/.env:/config/.env +Volume=EFFECTIVE_ENV_FILE_PLACEHOLDER:/config/.env Volume=HOME_PLACEHOLDER/.gemini:/config/gemini_auth HealthCmd=python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:LITELLM_PORT_PLACEHOLDER/health/liveness')" HealthInterval=15s diff --git a/quadlets/llm-routing-router.container b/quadlets/llm-routing-router.container index 72023f9d..5c10f197 100644 --- a/quadlets/llm-routing-router.container +++ b/quadlets/llm-routing-router.container @@ -44,7 +44,7 @@ Volume=HOME_PLACEHOLDER/.local/bin/agy:/usr/local/bin/agy Volume=RUN_USER_PLACEHOLDER/bus:RUN_USER_PLACEHOLDER/bus Volume=HOME_PLACEHOLDER/.local/share/keyrings:/root/.local/share/keyrings Volume=RUN_USER_PLACEHOLDER:RUN_USER_PLACEHOLDER -Volume=WORKDIR_PLACEHOLDER/.env:/config/.env +Volume=EFFECTIVE_ENV_FILE_PLACEHOLDER:/config/.env HealthCmd=python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:ROUTER_PORT_PLACEHOLDER/metrics')" HealthInterval=15s HealthTimeout=5s diff --git a/scripts/README.md b/scripts/README.md index cbb735cf..2370c4d2 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -9,10 +9,11 @@ This directory and the repository root contain various scripts used for stack or ### `start-stack.sh` (Root Directory) Unified startup and credential extraction script for the systemd Quadlet-managed Podman stack. - **Usage**: - - `./start-stack.sh` (Restart the generated `llm-routing-pod.service`) + - `./start-stack.sh` (Restart the generated environment-specific pod service) - `./start-stack.sh --replace` (Stop + clean ports + render/install Quadlets + daemon-reload + recreate stack) - `./start-stack.sh --full-rebuild` (Same as `--replace` + rebuild the triage router image; required for code changes in `router/`) -- Quadlet templates live in `quadlets/`; rendered owner-only units are installed under `~/.config/containers/systemd/llm-routing/`. Use `systemctl --user status llm-routing-pod.service --no-pager` and `journalctl --user -u llm-routing-router.service --no-pager` for lifecycle diagnostics. +- Quadlet templates live in `quadlets/`; rendered owner-only units use environment-specific namespaces: dev under `~/.config/containers/systemd/llm-routing-dev/` and prod under `~/.config/containers/systemd/llm-routing-prod/`. Dev uses `llm-routing-dev-pod.service`; prod uses `llm-routing-prod-pod.service`. Use the matching `systemctl --user status -pod.service --no-pager` and `journalctl --user -u -router.service --no-pager` for lifecycle diagnostics. +- Before rendering, the script writes the merged `.env` plus optional `.env.dev` overlay to `${DATA_ROOT}/effective.env` (mode `600`). Router and LiteLLM mount this generated file as `/config/.env`; the source `.env` remains unchanged. ### `scripts/backup.sh` Automated database backup script that runs before every stack deployment. Uses `pg_isready` to safely wait for database connections and manages timestamped backups under `backups/`. diff --git a/start-stack.sh b/start-stack.sh index e2b1936e..2d560a8c 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -77,6 +77,15 @@ if [ -n "${DEV_ENV_FILE:-}" ] && [ -f "$DEV_ENV_FILE" ]; then set +a fi +# Quadlet namespace is environment-specific. This prevents dev and prod from +# sharing rendered files or generated systemd unit names. +QUADLET_NAMESPACE="${QUADLET_NAMESPACE:-llm-routing-prod}" +if [[ ! "$QUADLET_NAMESPACE" =~ ^[a-z0-9][a-z0-9-]*$ ]]; then + echo "❌ Error: QUADLET_NAMESPACE must contain only lowercase letters, digits, and hyphens" >&2 + exit 1 +fi +export QUADLET_NAMESPACE + # Port assignments — read from env (set by .env or .env.dev) with prod defaults POD_NAME="${POD_NAME:-prod-router-pod}" ROUTER_PORT="${ROUTER_PORT:-5000}" @@ -115,6 +124,10 @@ LOCAL_BASE_URL="${LOCAL_BASE_URL:-http://localhost:${ROUTER_PORT}}" LOCAL_BASE_URL="${LOCAL_BASE_URL%/}" export PUBLIC_BASE_URL LOCAL_BASE_URL +# Containers source this generated file, not the production .env bind mount. +# This preserves the .env + optional .env.dev overlay inside each container. +EFFECTIVE_ENV_FILE="${DATA_ROOT}/effective.env" +export EFFECTIVE_ENV_FILE # Ensure openssl is installed if we need to generate passwords/keys if [ -z "$POSTGRES_PASSWORD" ] || [ -z "$NEXTAUTH_SECRET" ] || [ -z "$SALT" ] || [ -z "$ENCRYPTION_KEY" ] || [ -z "$LITELLM_MASTER_KEY" ] || [ -z "$ROUTER_API_KEY" ] || [ -z "$MINIO_ROOT_USER" ] || [ -z "$MINIO_ROOT_PASSWORD" ] || [ -z "$LANGFUSE_INIT_USER_PASSWORD" ] || [ -z "$REDIS_AUTH" ] || [ -z "$CLICKHOUSE_PASSWORD" ] || [ -z "$LANGFUSE_PUBLIC_KEY" ] || [ -z "$LANGFUSE_SECRET_KEY" ]; then @@ -350,8 +363,36 @@ if [ -z "$CLASSIFIER_INPUT_MAX_CHARS" ]; then echo "✓ Set default CLASSIFIER_INPUT_MAX_CHARS=300 and saved to $ENV_FILE" fi +# Persist only the application environment for bind-mounted container consumers. +# Quote values as shell syntax so URLs, credentials, and special characters +# survive the second `source` performed by the router/LiteLLM entrypoints. +python3 - "$EFFECTIVE_ENV_FILE" <<'PY' +import os +import shlex +import sys + +# Explicit allowlist: never copy unrelated host credentials into containers. +APPLICATION_ENV = { + "CLASSIFIER_INPUT_MAX_CHARS", "CLICKHOUSE_HTTP_PORT", "CLICKHOUSE_PASSWORD", + "CLICKHOUSE_TCP_PORT", "DATA_ROOT", "LANGFUSE_INIT_USER_PASSWORD", + "LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", "LANGFUSE_WEB_PORT", + "LANGFUSE_WORKER_PORT", "LLAMA_CLASSIFIER_URL", "LLAMA_SERVER_URL", + "LITELLM_MASTER_KEY", "LITELLM_PORT", "MINIO_CONSOLE_PORT", "MINIO_ROOT_PASSWORD", + "MINIO_ROOT_USER", "MINIO_S3_PORT", "NEXTAUTH_SECRET", "NEXTAUTH_URL", + "OLLAMA_API_KEY", "OPENROUTER_API_KEY", "POD_NAME", "POSTGRES_PASSWORD", + "POSTGRES_PORT", "PROXY_BASE_URL", "PUBLIC_BASE_URL", "QUADLET_NAMESPACE", + "REDIS_AUTH", "ROUTER_API_KEY", "ROUTER_IMAGE", "ROUTER_PORT", "ROUTING_DOMAIN", + "SALT", "ENCRYPTION_KEY", "UI_PASSWORD", "UI_USERNAME", "VALKEY_CACHE_PORT", + "VALKEY_LF_PORT", +} - +target = sys.argv[1] +with open(target, "w", encoding="utf-8") as handle: + for key in sorted(APPLICATION_ENV): + if key in os.environ: + handle.write(f"{key}={shlex.quote(os.environ[key])}\n") +os.chmod(target, 0o600) +PY # DYNAMIC_LITELLM_MASTER_KEY_PLACEHOLDER in router config is resolved at runtime from env @@ -533,22 +574,44 @@ verify_stack_health() { # ── Stack ownership and teardown ── # Keep the generated Quadlet unit name in one place: it is used for ownership # detection, lifecycle operations, and user-facing diagnostics. -LLM_ROUTING_POD_UNIT="llm-routing-pod.service" +LLM_ROUTING_POD_UNIT="${QUADLET_NAMESPACE}-pod.service" +LEGACY_LLM_ROUTING_POD_UNIT="llm-routing-pod.service" +QUADLET_DIR="${HOME}/.config/containers/systemd/${QUADLET_NAMESPACE}" # Quadlet-managed pods carry PODMAN_SYSTEMD_UNIT on their infra container. # Consult that metadata rather than inferring ownership from active state: a # stopped or failed generated unit is still Quadlet-owned and must be reconciled # through systemd before a replacement pod is created. stack_ownership() { local infra_unit + + # A generic legacy unit is shared by old deployments. Only treat it as + # owned when its generated ExecStartPre command creates this stack's pod; + # a PodName= source line alone does not prove which pod the unit owns. + legacy_unit_owns_pod() { + local unit="$1" + local pod_name_pattern + pod_name_pattern=$(printf '%s' "$POD_NAME" | sed 's/[][\\.^$*+?(){}|]/\\&/g') + systemctl --user cat "$unit" --no-pager 2>/dev/null \ + | grep -E 'podman[[:space:]]+pod[[:space:]]+create' \ + | grep -Eq -- "--name[=[:space:]]${pod_name_pattern}([[:space:]]|$)" + } + if podman pod exists "${POD_NAME}" 2>/dev/null; then infra_unit=$(podman pod inspect "${POD_NAME}" --format '{{.InfraContainerID}}' 2>/dev/null | xargs -r podman inspect --format '{{ index .Config.Labels "PODMAN_SYSTEMD_UNIT" }}' 2>/dev/null || true) if [[ "$infra_unit" == "$LLM_ROUTING_POD_UNIT" ]]; then - printf 'quadlet\n' + printf 'quadlet:%s\n' "$infra_unit" + elif [[ "$infra_unit" == "$LEGACY_LLM_ROUTING_POD_UNIT" ]] && legacy_unit_owns_pod "$LEGACY_LLM_ROUTING_POD_UNIT"; then + printf 'quadlet:%s\n' "$infra_unit" + elif [[ "$infra_unit" == "$LEGACY_LLM_ROUTING_POD_UNIT" ]]; then + printf 'conflict:%s\n' "$infra_unit" else printf 'legacy\n' fi elif systemctl --user show "$LLM_ROUTING_POD_UNIT" -p LoadState --value 2>/dev/null | grep -qxv 'not-found'; then - printf 'quadlet\n' + printf 'quadlet:%s\n' "$LLM_ROUTING_POD_UNIT" + elif systemctl --user show "$LEGACY_LLM_ROUTING_POD_UNIT" -p LoadState --value 2>/dev/null | grep -qxv 'not-found' \ + && legacy_unit_owns_pod "$LEGACY_LLM_ROUTING_POD_UNIT"; then + printf 'quadlet:%s\n' "$LEGACY_LLM_ROUTING_POD_UNIT" else printf 'absent\n' fi @@ -567,10 +630,11 @@ require_user_systemd() { safe_pod_teardown() { local ownership ownership=$(stack_ownership) - if [[ "$ownership" == "quadlet" ]]; then + if [[ "$ownership" == quadlet:* ]]; then + local owner_unit="${ownership#quadlet:}" echo "🛑 Reconciling Quadlet-owned stack (unit may be active, inactive, or failed)..." - systemctl --user stop "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true - systemctl --user reset-failed "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true + systemctl --user stop "$owner_unit" 2>/dev/null || true + systemctl --user reset-failed "$owner_unit" 2>/dev/null || true podman pod rm -f "${POD_NAME}" 2>/dev/null || true cleanup_zombie_ports echo "✓ Quadlet stack stopped, state reconciled, ports cleaned" @@ -677,11 +741,10 @@ render_router_config() { # ── Quadlet rendering + installation ── # Renders quadlets/*.pod + quadlets/*.container templates (same _PLACEHOLDER -# convention as pod.yaml) into ~/.config/containers/systemd/llm-routing/ and +# convention as pod.yaml) into the environment-specific +# ~/.config/containers/systemd/${QUADLET_NAMESPACE}/ directory and # lets systemd's podman-user-generator turn them into real units. # Quadlet values are bare scalars (not YAML) so plain string replacement is used. -QUADLET_DIR="${HOME}/.config/containers/systemd/llm-routing" - render_quadlets() { export WORKDIR HOME LITELLM_MASTER_KEY UI_USERNAME UI_PASSWORD export POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY @@ -705,6 +768,11 @@ render_quadlets() { import os, sys, urllib.parse, re, glob, shutil, tempfile uid = os.getuid() src_dir, out_dir = sys.argv[1], sys.argv[2] +namespace = os.environ["QUADLET_NAMESPACE"] +identifier_suffixes = ( + "pod", "clickhouse", "langfuse", "litellm", "minio", "postgres", "router", "valkey" +) +identifier_prefix = re.compile(r"\bllm-routing-(?=(?:" + "|".join(identifier_suffixes) + r"))") encoded_pg = urllib.parse.quote(os.environ['POSTGRES_PASSWORD'], safe="") # Derived by derive_external_service_urls(), shared with render_pod_yaml. @@ -740,6 +808,7 @@ repl = { "LLAMA_SERVER_URL_PLACEHOLDER": os.environ["LLAMA_SERVER_URL"], "POD_NAME_PLACEHOLDER": os.environ["POD_NAME"], "DATA_ROOT_PLACEHOLDER": os.environ["DATA_ROOT"], + "EFFECTIVE_ENV_FILE_PLACEHOLDER": os.environ["EFFECTIVE_ENV_FILE"], "ROUTER_IMAGE_PLACEHOLDER": os.environ["ROUTER_IMAGE"], "ROUTER_PORT_PLACEHOLDER": os.environ["ROUTER_PORT"], "LITELLM_PORT_PLACEHOLDER": os.environ["LITELLM_PORT"], @@ -766,6 +835,18 @@ try: text = f.read() for ph, val in repl.items(): text = text.replace(ph, str(val)) + # Quadlet generated unit names are global in the user systemd manager. + # Namespace both filenames and internal dependencies to isolate dev/prod. + # Namespace Quadlet identifier lines and values only. This preserves + # arbitrary image names, URLs, and credentials containing llm-routing. + def namespace_identifier(match): + field, value = match.group(1), match.group(2) + if field in {"Pod", "After", "Wants", "BindsTo", "Requires", "PartOf"}: + value = identifier_prefix.sub(namespace + "-", value) + value = value.replace("llm-routing.pod", namespace + ".pod") + value = value.replace("llm-routing-pod.service", namespace + "-pod.service") + return f"{field}={value}" + text = re.sub(r"(?m)^(Pod|After|Wants|BindsTo|Requires|PartOf)=(.*)$", namespace_identifier, text) unresolved = sorted(set(re.findall(r"\b[A-Z0-9_]+_PLACEHOLDER\b", text))) if unresolved: sys.stderr.write(f"Error: Unresolved placeholders in {os.path.basename(tpl)}: {', '.join(unresolved)}\n") @@ -777,7 +858,8 @@ try: value = match.group(1).replace("\\", "\\\\").replace('"', '\\"') return f'Environment="{value}"' text = re.sub(r"(?m)^Environment=(.*)$", quote_environment, text) - staged_path = os.path.join(staging_dir, os.path.basename(tpl)) + rendered_name = os.path.basename(tpl).replace("llm-routing", namespace) + staged_path = os.path.join(staging_dir, rendered_name) with open(staged_path, "w", encoding="utf-8") as f: f.write(text) # Rendered units include credentials; systemd user generator can read owner-only files. @@ -785,7 +867,7 @@ try: # All templates are now valid. Replace individual files atomically, then # remove stale units; a failed render above leaves the prior unit set intact. - rendered_names = {os.path.basename(tpl) for tpl in templates} + rendered_names = {os.path.basename(tpl).replace("llm-routing", namespace) for tpl in templates} for name in rendered_names: os.replace(os.path.join(staging_dir, name), os.path.join(out_dir, name)) for stale in glob.glob(os.path.join(out_dir, "*.pod")) + glob.glob(os.path.join(out_dir, "*.container")): @@ -836,6 +918,12 @@ deploy_fresh_pod() { } STACK_OWNERSHIP=$(stack_ownership) +if [[ "$STACK_OWNERSHIP" == conflict:* ]]; then + conflict_unit="${STACK_OWNERSHIP#conflict:}" + echo "❌ Error: pod ${POD_NAME} is attached to an unrelated legacy unit ${conflict_unit}; refusing to stop, replace, or deploy it." >&2 + echo " Inspect with: systemctl --user cat ${conflict_unit} --no-pager" >&2 + exit 1 +fi if [[ "$STACK_OWNERSHIP" != "absent" ]]; then if $FULL_REBUILD; then echo "🔨 Building custom local triage router image..." @@ -854,13 +942,13 @@ if [[ "$STACK_OWNERSHIP" != "absent" ]]; then echo "🚀 Deploying replacement pod from YAML..." deploy_fresh_pod else - if [[ "$STACK_OWNERSHIP" == "quadlet" ]]; then + if [[ "$STACK_OWNERSHIP" == quadlet:* ]]; then require_user_systemd || exit 1 + owner_unit="${STACK_OWNERSHIP#quadlet:}" echo "🔄 Restarting Quadlet-owned stack via systemd..." - systemctl --user reset-failed "$LLM_ROUTING_POD_UNIT" 2>/dev/null || true - if ! systemctl --user restart "$LLM_ROUTING_POD_UNIT"; then - echo "❌ Error: failed to restart ${LLM_ROUTING_POD_UNIT}" >&2 - echo " Hint: run 'systemctl --user status ${LLM_ROUTING_POD_UNIT} --no-pager' to inspect the failure" >&2 + if ! systemctl --user restart "$owner_unit"; then + echo "❌ Error: failed to restart ${owner_unit}" >&2 + echo " Hint: run 'systemctl --user status ${owner_unit} --no-pager' to inspect the failure" >&2 exit 1 fi else diff --git a/tests/test_quadlet_templates.py b/tests/test_quadlet_templates.py index 22f02b6c..c94bccec 100644 --- a/tests/test_quadlet_templates.py +++ b/tests/test_quadlet_templates.py @@ -1,6 +1,9 @@ """Static deployment-contract tests for the systemd Quadlet templates.""" from pathlib import Path +import os import re +import subprocess +import tempfile ROOT = Path(__file__).resolve().parent.parent @@ -44,7 +47,8 @@ def test_rendered_quadlets_are_owner_only(): script = (ROOT / "start-stack.sh").read_text() assert 'chmod 700 "$QUADLET_DIR"' in script assert "os.chmod(staged_path, 0o600)" in script - assert 'LLM_ROUTING_POD_UNIT="llm-routing-pod.service"' in script + assert 'LLM_ROUTING_POD_UNIT="${QUADLET_NAMESPACE}-pod.service"' in script + assert 'QUADLET_DIR="${HOME}/.config/containers/systemd/${QUADLET_NAMESPACE}"' in script assert "systemctl --user daemon-reload" in script assert "stack_ownership()" in script assert "PODMAN_SYSTEMD_UNIT" in script @@ -81,9 +85,107 @@ def test_router_quadlet_reasserts_overlayed_llama_urls_after_env_source(): assert '"LLAMA_SERVER_URL_PLACEHOLDER": os.environ["LLAMA_SERVER_URL"]' in script +def test_containers_source_the_merged_effective_environment(): + script = (ROOT / "start-stack.sh").read_text() + for template in (QUADLETS / "llm-routing-router.container", QUADLETS / "llm-routing-litellm.container"): + assert "EFFECTIVE_ENV_FILE_PLACEHOLDER:/config/.env" in template.read_text() + assert 'EFFECTIVE_ENV_FILE="${DATA_ROOT}/effective.env"' in script + assert '"EFFECTIVE_ENV_FILE_PLACEHOLDER": os.environ["EFFECTIVE_ENV_FILE"]' in script + assert "shlex.quote(os.environ[key])" in script + assert "APPLICATION_ENV = {" in script + assert "for key in sorted(APPLICATION_ENV):" in script + + def test_quadlet_deployment_enforces_prerequisite_and_restart_failures(): script = (ROOT / "start-stack.sh").read_text() assert "render_pod_yaml()" not in script assert "require_user_systemd || exit 1" in script assert "failed to derive external service URLs for Quadlet rendering" in script assert 'if ! podman pod restart "${POD_NAME}"; then' in script + + +def test_quadlet_namespace_is_environment_specific(): + dev_env = (ROOT / ".env.dev").read_text() + prod_namespace = "llm-routing-prod" + assert 'QUADLET_NAMESPACE="llm-routing-dev"' in dev_env + assert 'QUADLET_NAMESPACE="${QUADLET_NAMESPACE:-llm-routing-prod}"' in (ROOT / "start-stack.sh").read_text() + assert prod_namespace in (ROOT / "start-stack.sh").read_text() + script = (ROOT / "start-stack.sh").read_text() + assert 'def namespace_identifier(match):' in script + assert 'identifier_suffixes = (' in script + assert 'identifier_prefix.sub(namespace + "-", value)' in script + assert 'rendered_name = os.path.basename(tpl).replace("llm-routing", namespace)' in script + + +def test_namespace_rendering_preserves_non_identifiers(): + script = (ROOT / "start-stack.sh").read_text() + embedded = script.split("python3 - \"$src_dir\" \"$QUADLET_DIR\" <<'PY'\n", 1)[1].split("\nPY\n", 1)[0] + env = os.environ.copy() + values = { + "POSTGRES_PASSWORD": "pg", "WORKDIR": str(ROOT), "HOME": str(ROOT), + "LITELLM_MASTER_KEY": "master", "NEXTAUTH_SECRET": "next", "SALT": "salt", + "ENCRYPTION_KEY": "encrypt", "OLLAMA_API_KEY": "ollama", "OPENROUTER_API_KEY": "openrouter", + "LANGFUSE_PUBLIC_KEY": "public", "LANGFUSE_SECRET_KEY": "secret", "MINIO_ROOT_USER": "minio", + "MINIO_ROOT_PASSWORD": "minio-pass", "LANGFUSE_INIT_USER_PASSWORD": "lf-pass", + "REDIS_AUTH": "redis", "CLICKHOUSE_PASSWORD": "click", "PROXY_BASE_URL_DERIVED": "https://proxy", + "NEXTAUTH_URL_DERIVED": "https://next", "PUBLIC_BASE_URL": "https://host/llm-routing", + "ROUTING_DOMAIN": "vendeuvre.lan", "LLAMA_CLASSIFIER_URL": "http://127.0.0.1:8083/v1", + "LLAMA_SERVER_URL": "http://127.0.0.1:8083", "POD_NAME": "dev-router-pod", + "DATA_ROOT": str(ROOT / "data"), "EFFECTIVE_ENV_FILE": str(ROOT / "data" / "effective.env"), + "ROUTER_IMAGE": "registry/llm-routing-router:latest", + "ROUTER_PORT": "5010", "LITELLM_PORT": "4010", "LANGFUSE_WEB_PORT": "3011", + "LANGFUSE_WORKER_PORT": "3030", "POSTGRES_PORT": "5442", "VALKEY_CACHE_PORT": "6389", + "VALKEY_LF_PORT": "6390", "CLICKHOUSE_HTTP_PORT": "8123", "CLICKHOUSE_TCP_PORT": "9003", + "MINIO_S3_PORT": "9002", "MINIO_CONSOLE_PORT": "9001", "QUADLET_NAMESPACE": "llm-routing-dev", + } + env.update(values) + with tempfile.TemporaryDirectory() as tmp: + src, out = Path(tmp) / "src", Path(tmp) / "out" + src.mkdir() + out.mkdir() + (src / "llm-routing.pod").write_text("[Pod]\nPodName=llm-routing.pod\n") + (src / "llm-routing-router.container").write_text( + "[Unit]\nAfter=llm-routing-litellm.service\n[Container]\n" + "Image=registry/llm-routing-router:latest\n" + "Environment=PUBLIC_BASE_URL=https://host/llm-routing-router\nPod=llm-routing.pod\n" + ) + subprocess.run(["python3", "-c", embedded, str(src), str(out)], env=env, check=True, capture_output=True, text=True) + rendered = (out / "llm-routing-dev-router.container").read_text() + assert "After=llm-routing-dev-litellm.service" in rendered + assert "Image=registry/llm-routing-router:latest" in rendered + assert "PUBLIC_BASE_URL=https://host/llm-routing-router" in rendered + assert "Pod=llm-routing-dev.pod" in rendered + + +def test_namespace_is_validated_and_ownership_preserves_exact_unit(): + script = (ROOT / "start-stack.sh").read_text() + assert '[[ ! "$QUADLET_NAMESPACE" =~ ^[a-z0-9][a-z0-9-]*$ ]]' in script + assert "printf 'quadlet:%s\\n' \"$infra_unit\"" in script + assert 'owner_unit="${STACK_OWNERSHIP#quadlet:}"' in script + assert 'failed to restart ${owner_unit}' in script + assert 'status ${owner_unit} --no-pager' in script + assert 'legacy_unit_owns_pod()' in script + assert "grep -E 'podman[[:space:]]+pod[[:space:]]+create'" in script + assert 'grep -Eq -- "--name[=[:space:]]${pod_name_pattern}' in script + assert '&& legacy_unit_owns_pod "$LEGACY_LLM_ROUTING_POD_UNIT"' in script + assert 'elif [[ "$infra_unit" == "$LEGACY_LLM_ROUTING_POD_UNIT" ]]; then' in script + assert 'printf \'conflict:%s\\n\' "$infra_unit"' in script + assert 'STACK_OWNERSHIP" == conflict:*' in script + + +def test_documentation_uses_environment_specific_units(): + readme = (ROOT / "README.md").read_text() + scripts_readme = (ROOT / "scripts" / "README.md").read_text() + assert "systemctl --user status llm-routing-prod-pod.service" in readme + assert "llm-routing-dev-pod.service" in readme + assert "llm-routing-pod.service" not in scripts_readme + assert "llm-routing-{dev,prod}" not in readme + + +def test_data_root_is_worktree_scoped_by_default(): + script = (ROOT / "start-stack.sh").read_text() + assert 'DATA_ROOT="${DATA_ROOT:-${WORKDIR}/data}"' in script + dev_root = ROOT.resolve() + prod_root = (ROOT.parent.parent / "prod" / "LLM-Routing").resolve() + assert dev_root != prod_root + assert dev_root / "data" != prod_root / "data"