From 3084c68010fbd561cbb85bed559c05c6b1ff20ef Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 12:17:46 +0200 Subject: [PATCH 01/11] feat: parameterize all ports and pod name for parallel prod/dev deployments Replaces all hardcoded port values and the pod name in pod.yaml, start-stack.sh, litellm/config.yaml and litellm/entrypoint.py with environment-variable-driven placeholders. A single pod.yaml now serves both the production agent-router-pod and the development dev-router-pod. Changes: - pod.yaml: rewrite with *_PLACEHOLDER vars for pod name, all 13 service ports, and DATA_ROOT; add clickhouse-port-config volume; litellm-config volume now points to DATA_ROOT/litellm-rendered (rendered copy) - start-stack.sh: load optional DEV_ENV_FILE overlay; resolve port/name defaults after env sourcing; add generate_clickhouse_config() and render_litellm_config() helpers; parameterize all agent-router-pod refs and hardcoded ports throughout cleanup, health-check, and minio helpers - litellm/config.yaml: replace port 6379 / 5000 with *_PLACEHOLDER vars (rendered at deploy time via render_litellm_config) - litellm/entrypoint.py: read POSTGRES_PORT / LITELLM_PORT from env instead of hardcoded 5432 / 4000; use setdefault to not clobber container-injected env vars - .env: append production port defaults (POD_NAME, ROUTER_PORT, ...) - .env.dev: new dev overlay (POD_NAME=dev-router-pod, +10 port offsets, dev.vendeuvre.lan URLs) - start-dev.sh: thin wrapper that sets DEV_ENV_FILE + DATA_ROOT then delegates to start-stack.sh - .gitignore: exclude dev-data/ (dev-stack runtime volumes) --- .env.dev | 24 ++++++ .gitignore | 2 + litellm/config.yaml | 6 +- litellm/entrypoint.py | 11 +-- pod.yaml | 124 ++++++++++++++++++----------- start-dev.sh | 14 ++++ start-stack.sh | 178 +++++++++++++++++++++++++++++++++--------- 7 files changed, 270 insertions(+), 89 deletions(-) create mode 100644 .env.dev create mode 100755 start-dev.sh diff --git a/.env.dev b/.env.dev new file mode 100644 index 00000000..7cc97b63 --- /dev/null +++ b/.env.dev @@ -0,0 +1,24 @@ +# Dev environment overlay โ€” sourced AFTER .env to override prod values +# Secrets are inherited from .env; only env-specific config is set here. + +# Pod identity +POD_NAME="dev-router-pod" + +# Public URLs +BASEURL="dev.vendeuvre.lan" +BASE_URL="dev.vendeuvre.lan" +PUBLIC_BASE_URL="https://dev.vendeuvre.lan/llm-routing" + +# Dev port assignments (+10 offset from production) +ROUTER_PORT="5010" +LITELLM_PORT="4010" +LANGFUSE_WEB_PORT="3011" +LANGFUSE_WORKER_PORT="3040" +POSTGRES_PORT="5442" +VALKEY_CACHE_PORT="6389" +VALKEY_LF_PORT="6390" +CLICKHOUSE_HTTP_PORT="8133" +CLICKHOUSE_TCP_PORT="9010" +CLICKHOUSE_INTERSERVER_PORT="9019" +MINIO_S3_PORT="9012" +MINIO_CONSOLE_PORT="9011" diff --git a/.gitignore b/.gitignore index 5ccf9013..34bee0fb 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,8 @@ valkey-data/ clickhouse-data/ minio-data/ redis-lf-data/ +dev-data/ + # Exclude auto-generated backups backups/ diff --git a/litellm/config.yaml b/litellm/config.yaml index 195712f5..b204f9cc 100644 --- a/litellm/config.yaml +++ b/litellm/config.yaml @@ -12,7 +12,7 @@ litellm_settings: cache_params: host: 127.0.0.1 mode: default - port: 6379 + port: VALKEY_CACHE_PORT_PLACEHOLDER ttl: 3600 type: redis caching_backend: redis @@ -77,7 +77,7 @@ model_list: is_public_model_group: true - litellm_params: model: openai/llm-routing-ollama - api_base: http://127.0.0.1:5000/v1 + api_base: http://127.0.0.1:ROUTER_PORT_PLACEHOLDER/v1 api_key: os.environ/LITELLM_MASTER_KEY request_timeout: 120 model_name: llm-routing-ollama @@ -237,7 +237,7 @@ model_list: redis_settings: redis_host: 127.0.0.1 - redis_port: 6379 + redis_port: VALKEY_CACHE_PORT_PLACEHOLDER router_settings: allowed_fails: 0 cooldown_time: 300 diff --git a/litellm/entrypoint.py b/litellm/entrypoint.py index 65520978..85dca12b 100644 --- a/litellm/entrypoint.py +++ b/litellm/entrypoint.py @@ -16,8 +16,7 @@ line = line.strip() if line and not line.startswith("#") and "=" in line: key, _, val = line.partition("=") - val = val.strip().strip('"').strip("'") - os.environ[key] = val + os.environ.setdefault(key, val) # Load Gemini OAuth token from credentials JSON creds_path = "/config/gemini_auth/oauth_creds.json" @@ -46,9 +45,10 @@ def check_tcp_port(ip: str, port: int) -> bool: return False max_wait = 60 -print(f"๐Ÿ”Œ Waiting for PostgreSQL on :5432 (max {max_wait}s)...") +postgres_port = int(os.environ.get("POSTGRES_PORT", "5432")) +print(f"๐Ÿ”Œ Waiting for PostgreSQL on :{postgres_port} (max {max_wait}s)...") for i in range(max_wait): - if check_tcp_port("127.0.0.1", 5432): + if check_tcp_port("127.0.0.1", postgres_port): print(f"โœ… PostgreSQL ready after {i+1}s") break time.sleep(1) @@ -127,5 +127,6 @@ def _serialize_dt(dt): # Start LiteLLM Proxy import litellm from litellm.proxy.proxy_cli import run_server -sys.argv = ["litellm", "--config", "/app/config.yaml", "--port", "4000"] +litellm_port = os.environ.get("LITELLM_PORT", os.environ.get("PORT", "4000")) +sys.argv = ["litellm", "--config", "/app/config.yaml", "--port", litellm_port] run_server() diff --git a/pod.yaml b/pod.yaml index db122105..c225a57e 100644 --- a/pod.yaml +++ b/pod.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Pod metadata: - name: agent-router-pod + name: POD_NAME_PLACEHOLDER spec: containers: # Valkey Cache โ€” LiteLLM response cache (exact-match & semantic) @@ -9,19 +9,21 @@ spec: - valkey-server - --bind - 127.0.0.1 + - --port + - 'VALKEY_CACHE_PORT_PLACEHOLDER' - --loglevel - warning image: docker.io/valkey/valkey:9.1.0-alpine livenessProbe: tcpSocket: - port: 6379 + port: VALKEY_CACHE_PORT_PLACEHOLDER initialDelaySeconds: 5 periodSeconds: 10 timeoutSeconds: 2 name: valkey-cache readinessProbe: tcpSocket: - port: 6379 + port: VALKEY_CACHE_PORT_PLACEHOLDER initialDelaySeconds: 2 periodSeconds: 5 timeoutSeconds: 2 @@ -34,28 +36,34 @@ spec: - /app/entrypoint.py env: - name: DATABASE_URL - value: postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:5432/postgres + value: postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:POSTGRES_PORT_PLACEHOLDER/postgres - name: STORE_MODEL_IN_DB value: 'True' - name: LITELLM_LOG value: INFO - name: LANGFUSE_HOST - value: http://127.0.0.1:3001 + value: http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER - name: LITELLM_MASTER_KEY value: LITELLM_MASTER_KEY_PLACEHOLDER - name: OLLAMA_API_KEY value: OLLAMA_API_KEY_PLACEHOLDER - name: SERVER_ROOT_PATH - value: SERVER_ROOT_PATH_PLACEHOLDER + value: /llm-routing/litellm - name: PROXY_BASE_URL value: PROXY_BASE_URL_PLACEHOLDER + - name: PORT + value: 'LITELLM_PORT_PLACEHOLDER' + - name: LITELLM_PORT + value: 'LITELLM_PORT_PLACEHOLDER' + - name: POSTGRES_PORT + value: 'POSTGRES_PORT_PLACEHOLDER' image: ghcr.io/berriai/litellm:v1.91.0 livenessProbe: exec: command: - python3 - -c - - import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveness') + - import urllib.request; urllib.request.urlopen('http://localhost:LITELLM_PORT_PLACEHOLDER/ping') initialDelaySeconds: 240 periodSeconds: 15 timeoutSeconds: 5 @@ -65,16 +73,16 @@ spec: command: - python3 - -c - - import urllib.request; urllib.request.urlopen('http://localhost:4000/health/readiness') + - import urllib.request; urllib.request.urlopen('http://localhost:LITELLM_PORT_PLACEHOLDER/health/readiness') initialDelaySeconds: 45 periodSeconds: 10 timeoutSeconds: 5 volumeMounts: - mountPath: /app/config.yaml - name: litellm-config + name: litellm-rendered subPath: config.yaml - mountPath: /app/entrypoint.py - name: litellm-config + name: litellm-rendered subPath: entrypoint.py - mountPath: /config/.env name: env-file @@ -83,7 +91,7 @@ spec: name: gemini-secrets # LLM Triage Router โ€” classification + routing gateway - args: - - set -a && . /config/.env && set +a && exec uvicorn main:app --host 0.0.0.0 --port 5000 --log-level ${LOG_LEVEL:-warning} + - set -a && . /config/.env && set +a && LITELLM_PORT=LITELLM_PORT_PLACEHOLDER VALKEY_CACHE_PORT=VALKEY_CACHE_PORT_PLACEHOLDER exec uvicorn main:app --host 0.0.0.0 --port ROUTER_PORT_PLACEHOLDER --log-level ${LOG_LEVEL:-warning} command: - /bin/sh - -c @@ -93,7 +101,7 @@ spec: - name: LITELLM_CONFIG_PATH value: /config/litellm_dir/config.yaml - name: DATABASE_URL - value: postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:5432/postgres + value: postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:POSTGRES_PORT_PLACEHOLDER/postgres - name: DBUS_SESSION_BUS_ADDRESS value: unix:path=RUN_USER_PLACEHOLDER/bus - name: LITELLM_MASTER_KEY @@ -103,22 +111,26 @@ spec: - name: LANGFUSE_SECRET_KEY value: LANGFUSE_SECRET_KEY_PLACEHOLDER - name: LANGFUSE_HOST - value: http://127.0.0.1:3001 + value: http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER - name: OLLAMA_API_KEY value: OLLAMA_API_KEY_PLACEHOLDER - name: PUBLIC_BASE_URL value: PUBLIC_BASE_URL_PLACEHOLDER - name: ROUTING_DOMAIN value: ROUTING_DOMAIN_PLACEHOLDER + - name: LITELLM_PORT + value: 'LITELLM_PORT_PLACEHOLDER' + - name: VALKEY_CACHE_PORT + value: 'VALKEY_CACHE_PORT_PLACEHOLDER' - name: LOG_LEVEL value: info - image: localhost/llm-triage-router:latest + image: ghcr.io/sheepdestroyer/llm-routing:latest livenessProbe: exec: command: - python3 - -c - - import urllib.request; urllib.request.urlopen('http://localhost:5000/metrics') + - import urllib.request; urllib.request.urlopen('http://localhost:ROUTER_PORT_PLACEHOLDER/metrics') initialDelaySeconds: 20 periodSeconds: 15 timeoutSeconds: 5 @@ -128,7 +140,7 @@ spec: command: - python3 - -c - - import urllib.request; urllib.request.urlopen('http://localhost:5000/metrics') + - import urllib.request; urllib.request.urlopen('http://localhost:ROUTER_PORT_PLACEHOLDER/metrics') initialDelaySeconds: 10 periodSeconds: 10 timeoutSeconds: 5 @@ -136,7 +148,7 @@ spec: - mountPath: /config/router_dir name: router-config - mountPath: /config/litellm_dir - name: litellm-config + name: litellm-rendered - mountPath: /app/data name: dataset-data - mountPath: /config/gemini_auth @@ -167,6 +179,8 @@ spec: value: langfuse - name: PGDATA value: /var/lib/postgresql/data/pg18 + - name: PGPORT + value: 'POSTGRES_PORT_PLACEHOLDER' image: docker.io/pgvector/pgvector:0.8.3-pg18 livenessProbe: exec: @@ -174,6 +188,8 @@ spec: - pg_isready - -U - postgres + - -p + - 'POSTGRES_PORT_PLACEHOLDER' initialDelaySeconds: 15 periodSeconds: 10 timeoutSeconds: 5 @@ -184,6 +200,8 @@ spec: - pg_isready - -U - postgres + - -p + - 'POSTGRES_PORT_PLACEHOLDER' initialDelaySeconds: 5 periodSeconds: 5 timeoutSeconds: 5 @@ -203,6 +221,10 @@ spec: exec: command: - clickhouse-client + - --host + - 127.0.0.1 + - --port + - 'CLICKHOUSE_TCP_PORT_PLACEHOLDER' - --user - clickhouse - --password @@ -217,6 +239,10 @@ spec: exec: command: - clickhouse-client + - --host + - 127.0.0.1 + - --port + - 'CLICKHOUSE_TCP_PORT_PLACEHOLDER' - --user - clickhouse - --password @@ -230,13 +256,16 @@ spec: volumeMounts: - mountPath: /var/lib/clickhouse name: clickhouse-storage + - mountPath: /etc/clickhouse-server/config.d/port-override.xml + name: clickhouse-port-config + subPath: port-override.xml # Valkey LF โ€” Langfuse v3 BullMQ job queue - command: - redis-server - --bind - 127.0.0.1 - --port - - '6380' + - 'VALKEY_LF_PORT_PLACEHOLDER' - --requirepass - REDIS_AUTH_PLACEHOLDER - --maxmemory-policy @@ -246,7 +275,7 @@ spec: image: docker.io/valkey/valkey:9.1.0-alpine livenessProbe: tcpSocket: - port: 6380 + port: VALKEY_LF_PORT_PLACEHOLDER initialDelaySeconds: 5 periodSeconds: 10 timeoutSeconds: 2 @@ -256,7 +285,7 @@ spec: command: - valkey-cli - -p - - "6380" + - "VALKEY_LF_PORT_PLACEHOLDER" - -a - REDIS_AUTH_PLACEHOLDER - ping @@ -269,11 +298,11 @@ spec: # Langfuse Web โ€” observability dashboard - env: - name: DATABASE_URL - value: postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:5432/langfuse + value: postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:POSTGRES_PORT_PLACEHOLDER/langfuse - name: NEXTAUTH_SECRET value: NEXTAUTH_SECRET_PLACEHOLDER - name: NEXTAUTH_URL - value: NEXTAUTH_URL_PLACEHOLDER + value: http://localhost:LANGFUSE_WEB_PORT_PLACEHOLDER - name: SALT value: SALT_PLACEHOLDER - name: ENCRYPTION_KEY @@ -281,7 +310,7 @@ spec: - name: HOSTNAME value: 0.0.0.0 - name: PORT - value: '3001' + value: 'LANGFUSE_WEB_PORT_PLACEHOLDER' - name: TELEMETRY_ENABLED value: 'false' - name: LANGFUSE_S3_EVENT_UPLOAD_BUCKET @@ -293,13 +322,13 @@ spec: - name: LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY value: MINIO_PASSWORD_PLACEHOLDER - name: LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT - value: http://127.0.0.1:9002 + value: http://127.0.0.1:MINIO_S3_PORT_PLACEHOLDER - name: S3_FORCE_PATH_STYLE value: 'true' - name: CLICKHOUSE_URL - value: http://127.0.0.1:8123 + value: http://127.0.0.1:CLICKHOUSE_HTTP_PORT_PLACEHOLDER - name: CLICKHOUSE_MIGRATION_URL - value: clickhouse://127.0.0.1:9000 + value: clickhouse://127.0.0.1:CLICKHOUSE_TCP_PORT_PLACEHOLDER - name: CLICKHOUSE_USER value: clickhouse - name: CLICKHOUSE_PASSWORD @@ -309,7 +338,7 @@ spec: - name: REDIS_HOST value: 127.0.0.1 - name: REDIS_PORT - value: '6380' + value: 'VALKEY_LF_PORT_PLACEHOLDER' - name: REDIS_AUTH value: REDIS_AUTH_PLACEHOLDER - name: LANGFUSE_INIT_ORG_ID @@ -334,7 +363,7 @@ spec: - -q - -O - /dev/null - - http://127.0.0.1:3001/api/health + - http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER/api/health initialDelaySeconds: 240 periodSeconds: 15 timeoutSeconds: 5 @@ -346,16 +375,16 @@ spec: - -q - -O - /dev/null - - http://127.0.0.1:3001/api/health + - http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER/api/health initialDelaySeconds: 15 periodSeconds: 10 timeoutSeconds: 5 # Langfuse Worker โ€” background job processor - env: - name: DATABASE_URL - value: postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:5432/langfuse + value: postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:POSTGRES_PORT_PLACEHOLDER/langfuse - name: CLICKHOUSE_URL - value: http://127.0.0.1:8123 + value: http://127.0.0.1:CLICKHOUSE_HTTP_PORT_PLACEHOLDER - name: CLICKHOUSE_USER value: clickhouse - name: CLICKHOUSE_PASSWORD @@ -365,13 +394,13 @@ spec: - name: REDIS_HOST value: 127.0.0.1 - name: REDIS_PORT - value: '6380' + value: 'VALKEY_LF_PORT_PLACEHOLDER' - name: REDIS_AUTH value: REDIS_AUTH_PLACEHOLDER - name: HOSTNAME value: 0.0.0.0 - name: PORT - value: '3030' + value: 'LANGFUSE_WORKER_PORT_PLACEHOLDER' - name: TELEMETRY_ENABLED value: 'false' - name: LANGFUSE_S3_EVENT_UPLOAD_BUCKET @@ -383,7 +412,7 @@ spec: - name: S3_FORCE_PATH_STYLE value: 'true' - name: LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT - value: http://127.0.0.1:9002 + value: http://127.0.0.1:MINIO_S3_PORT_PLACEHOLDER - name: LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY value: MINIO_PASSWORD_PLACEHOLDER - name: LANGFUSE_LOG_LEVEL @@ -404,9 +433,9 @@ spec: - --quiet - /data - --address - - ":9002" + - ":MINIO_S3_PORT_PLACEHOLDER" - --console-address - - ":9001" + - ":MINIO_CONSOLE_PORT_PLACEHOLDER" env: - name: MINIO_ROOT_USER value: MINIO_USER_PLACEHOLDER @@ -416,7 +445,7 @@ spec: livenessProbe: httpGet: path: /minio/health/live - port: 9002 + port: MINIO_S3_PORT_PLACEHOLDER initialDelaySeconds: 10 periodSeconds: 15 timeoutSeconds: 5 @@ -424,7 +453,7 @@ spec: readinessProbe: httpGet: path: /minio/health/ready - port: 9002 + port: MINIO_S3_PORT_PLACEHOLDER initialDelaySeconds: 5 periodSeconds: 10 timeoutSeconds: 5 @@ -435,25 +464,26 @@ spec: restartPolicy: Always volumes: - hostPath: - path: WORKDIR_PLACEHOLDER/valkey-data + path: DATA_ROOT_PLACEHOLDER/valkey-data name: valkey-storage - hostPath: - path: WORKDIR_PLACEHOLDER/litellm - name: litellm-config + path: DATA_ROOT_PLACEHOLDER/litellm-rendered + type: DirectoryOrCreate + name: litellm-rendered - hostPath: path: WORKDIR_PLACEHOLDER/router name: router-config - hostPath: - path: WORKDIR_PLACEHOLDER/postgres-data + path: DATA_ROOT_PLACEHOLDER/postgres-data name: postgres-storage - hostPath: - path: WORKDIR_PLACEHOLDER/clickhouse-data + path: DATA_ROOT_PLACEHOLDER/clickhouse-data name: clickhouse-storage - hostPath: - path: WORKDIR_PLACEHOLDER/redis-lf-data + path: DATA_ROOT_PLACEHOLDER/redis-lf-data name: redis-lf-storage - hostPath: - path: WORKDIR_PLACEHOLDER/minio-data + path: DATA_ROOT_PLACEHOLDER/minio-data name: minio-storage - hostPath: path: HOME_PLACEHOLDER/.gemini @@ -480,3 +510,7 @@ spec: path: WORKDIR_PLACEHOLDER/data type: DirectoryOrCreate name: dataset-data + - hostPath: + path: DATA_ROOT_PLACEHOLDER/clickhouse-config + type: DirectoryOrCreate + name: clickhouse-port-config diff --git a/start-dev.sh b/start-dev.sh new file mode 100755 index 00000000..82c254e3 --- /dev/null +++ b/start-dev.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Dev deployment wrapper โ€” stands up dev-router-pod on distinct ports alongside prod +set -e +WORKDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$WORKDIR" + +export DEV_ENV_FILE="${WORKDIR}/.env.dev" +export DATA_ROOT="${WORKDIR}/dev-data" + +mkdir -p dev-data/valkey-data dev-data/postgres-data dev-data/clickhouse-data \ + dev-data/redis-lf-data dev-data/minio-data dev-data/clickhouse-config \ + dev-data/litellm-rendered + +exec bash start-stack.sh "$@" diff --git a/start-stack.sh b/start-stack.sh index 4f0f9588..1e949ab9 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -16,7 +16,8 @@ show_help() { echo "Usage:" echo " ./start-stack.sh โ†’ Restart existing pod (fast, preserves logs)" echo " ./start-stack.sh --replace โ†’ Stop, clean up zombie ports, and recreate/redeploy pod" - echo " ./start-stack.sh --full-rebuild โ†’ Rebuild router image and recreate/redeploy pod" + echo " ./start-stack.sh --pull โ†’ Pull latest router image from GHCR and recreate/redeploy pod" + echo " ./start-stack.sh --full-rebuild โ†’ Rebuild custom router image locally and recreate/redeploy pod" echo " ./start-stack.sh --help | -h โ†’ Show this help message and exit" } @@ -39,9 +40,12 @@ fi # Ensure local volume directories exist on the host for Podman mounts mkdir -p valkey-data postgres-data langfuse-data clickhouse-data redis-lf-data minio-data +PULL_MODE=false FULL_REBUILD=false REPLACE_MODE=false -if [ "${1:-}" = "--full-rebuild" ]; then +if [ "${1:-}" = "--pull" ] || [ "${1:-}" = "--pull-latest" ]; then + PULL_MODE=true +elif [ "${1:-}" = "--full-rebuild" ]; then FULL_REBUILD=true elif [ "${1:-}" = "--replace" ]; then REPLACE_MODE=true @@ -69,6 +73,30 @@ if [ -f "$ENV_FILE" ]; then set +a fi +# Load optional dev-environment overlay (set DEV_ENV_FILE before calling this script) +if [ -n "${DEV_ENV_FILE:-}" ] && [ -f "$DEV_ENV_FILE" ]; then + set -a + source "$DEV_ENV_FILE" + set +a +fi + +# Port assignments โ€” read from env (set by .env or .env.dev) with prod defaults +POD_NAME="${POD_NAME:-agent-router-pod}" +ROUTER_PORT="${ROUTER_PORT:-5000}" +LITELLM_PORT="${LITELLM_PORT:-4000}" +LANGFUSE_WEB_PORT="${LANGFUSE_WEB_PORT:-3001}" +LANGFUSE_WORKER_PORT="${LANGFUSE_WORKER_PORT:-3030}" +POSTGRES_PORT="${POSTGRES_PORT:-5432}" +VALKEY_CACHE_PORT="${VALKEY_CACHE_PORT:-6379}" +VALKEY_LF_PORT="${VALKEY_LF_PORT:-6380}" +CLICKHOUSE_HTTP_PORT="${CLICKHOUSE_HTTP_PORT:-8123}" +CLICKHOUSE_TCP_PORT="${CLICKHOUSE_TCP_PORT:-9000}" +CLICKHOUSE_INTERSERVER_PORT="${CLICKHOUSE_INTERSERVER_PORT:-9009}" +MINIO_S3_PORT="${MINIO_S3_PORT:-9002}" +MINIO_CONSOLE_PORT="${MINIO_CONSOLE_PORT:-9001}" +DATA_ROOT="${DATA_ROOT:-${WORKDIR}}" +export POD_NAME ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT DATA_ROOT + # Define and export the routing domain ROUTING_DOMAIN="${ROUTING_DOMAIN:-vendeuvre.lan}" export ROUTING_DOMAIN @@ -82,12 +110,9 @@ if [[ ! "$PUBLIC_BASE_URL" =~ /llm-routing ]]; then PUBLIC_BASE_URL="${PUBLIC_BASE_URL%/}/llm-routing" fi PUBLIC_BASE_URL="${PUBLIC_BASE_URL%/}" -LOCAL_BASE_URL="${LOCAL_BASE_URL:-http://localhost:5000}" +LOCAL_BASE_URL="${LOCAL_BASE_URL:-http://localhost:${ROUTER_PORT}}" LOCAL_BASE_URL="${LOCAL_BASE_URL%/}" -LITELLM_PUBLIC_URL="${LITELLM_PUBLIC_URL:-https://litellm.x570.${ROUTING_DOMAIN}}" -LANGFUSE_PUBLIC_URL="${LANGFUSE_PUBLIC_URL:-https://langfuse.x570.${ROUTING_DOMAIN}}" -LITELLM_ROOT_PATH="${LITELLM_ROOT_PATH:-}" -export PUBLIC_BASE_URL LOCAL_BASE_URL LITELLM_PUBLIC_URL LANGFUSE_PUBLIC_URL LITELLM_ROOT_PATH +export PUBLIC_BASE_URL LOCAL_BASE_URL # Ensure openssl is installed if we need to generate passwords/keys @@ -337,7 +362,7 @@ fi # Hermes profiles (e.g., llm-routing-openrouter) whose container storage # can leave surviving processes holding ports indefinitely. cleanup_zombie_ports() { - local ALL_PORTS="3000 3030 4000 5000 5005 5432 6379 6380 8080 8123 9000 9001 9002 9004 9005 9009" + local ALL_PORTS="$ROUTER_PORT $LITELLM_PORT $LANGFUSE_WEB_PORT $LANGFUSE_WORKER_PORT $POSTGRES_PORT $VALKEY_CACHE_PORT $VALKEY_LF_PORT $CLICKHOUSE_HTTP_PORT $CLICKHOUSE_TCP_PORT $CLICKHOUSE_INTERSERVER_PORT $MINIO_S3_PORT $MINIO_CONSOLE_PORT 8080 9004 9005" echo "๐Ÿงน Cleaning up zombie port bindings..." @@ -411,9 +436,9 @@ setup_minio_buckets() { echo "" echo "๐Ÿ“ฆ Ensuring MinIO buckets exist..." - # Wait for MinIO S3 API to be ready (port 9002) + # Wait for MinIO S3 API to be ready while [ $waited -lt $MAX_WAIT ]; do - if curl -sf --max-time 3 http://127.0.0.1:9002/minio/health/live >/dev/null 2>&1; then + if curl -sf --max-time 3 http://127.0.0.1:${MINIO_S3_PORT}/minio/health/live >/dev/null 2>&1; then echo " โœ“ MinIO S3 API ready after ${waited}s" break fi @@ -425,22 +450,22 @@ setup_minio_buckets() { return 1 fi - # Ensure mc alias points to the correct MinIO S3 API port (9002, not 9000) + # Ensure mc alias points to the correct MinIO S3 API port # The default 'local' alias in the MinIO image points to :9000 which is ClickHouse, # not MinIO. We must override it. - if ! podman exec agent-router-pod-minio-s3 mc alias set local http://127.0.0.1:9002 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"; then - echo "โŒ Error: Failed to set MinIO alias 'local' on http://127.0.0.1:9002" >&2 + if ! podman exec ${POD_NAME}-minio-s3 mc alias set local http://127.0.0.1:${MINIO_S3_PORT} "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"; then + echo "โŒ Error: Failed to set MinIO alias 'local' on http://127.0.0.1:${MINIO_S3_PORT}" >&2 exit 1 fi # Create required buckets (idempotent) local BUCKETS=("langfuse-events" "proj-triage-gateway-id") for bucket in "${BUCKETS[@]}"; do - if podman exec agent-router-pod-minio-s3 mc ls "local/${bucket}" >/dev/null 2>&1; then + if podman exec ${POD_NAME}-minio-s3 mc ls "local/${bucket}" >/dev/null 2>&1; then echo " โœ“ Bucket '${bucket}' exists" else echo " + Creating bucket '${bucket}'..." - podman exec agent-router-pod-minio-s3 mc mb "local/${bucket}" 2>/dev/null || { + podman exec ${POD_NAME}-minio-s3 mc mb "local/${bucket}" 2>/dev/null || { echo " โš ๏ธ Failed to create bucket '${bucket}'" } fi @@ -459,7 +484,7 @@ verify_stack_health() { # Wait for postgres first โ€” everything depends on it while [ $waited -lt $MAX_WAIT ]; do - if podman exec agent-router-pod-postgres-db pg_isready -U postgres -q 2>/dev/null; then + if podman exec ${POD_NAME}-postgres-db pg_isready -U postgres -p ${POSTGRES_PORT} -q 2>/dev/null; then echo " โœ“ PostgreSQL ready after ${waited}s" break fi @@ -475,7 +500,7 @@ verify_stack_health() { local litellm_ready=false waited=0 while [ $waited -lt $MAX_WAIT ]; do - if curl -sf --max-time 3 http://127.0.0.1:4000/health/readiness >/dev/null 2>&1; then + if curl -sf --max-time 3 http://127.0.0.1:${LITELLM_PORT}/health/readiness >/dev/null 2>&1; then echo " โœ“ LiteLLM ready after ${waited}s" litellm_ready=true break @@ -490,7 +515,7 @@ verify_stack_health() { # Wait for triage router + verify full pipeline waited=0 while [ $waited -lt 120 ]; do - local resp=$(curl -s --max-time 10 http://127.0.0.1:5000/v1/chat/completions \ + local resp=$(curl -s --max-time 10 http://127.0.0.1:${ROUTER_PORT}/v1/chat/completions \ -H 'Content-Type: application/json' \ -d '{"model":"agent-simple-core","messages":[{"role":"user","content":"Hi"}],"max_tokens":5}' 2>/dev/null) if echo "$resp" | grep -q '"choices"'; then @@ -508,16 +533,16 @@ verify_stack_health() { # Graceful stop (SIGTERM with 30s timeout) lets ClickHouse/Postgres flush, # then force-remove if needed. Avoids data corruption from SIGKILL. safe_pod_teardown() { - if podman pod exists agent-router-pod 2>/dev/null; then + if podman pod exists ${POD_NAME} 2>/dev/null; then echo "๐Ÿ›‘ Gracefully stopping pod (SIGTERM, 30s timeout)..." - podman pod stop -t 30 agent-router-pod 2>/dev/null || true + podman pod stop -t 30 ${POD_NAME} 2>/dev/null || true # If still running after graceful attempt, force-remove - if podman pod exists agent-router-pod 2>/dev/null; then + if podman pod exists ${POD_NAME} 2>/dev/null; then echo "โš ๏ธ Graceful stop timed out โ€” force-removing..." - podman pod rm -f agent-router-pod 2>/dev/null || true + podman pod rm -f ${POD_NAME} 2>/dev/null || true else # Already stopped, just remove - podman pod rm agent-router-pod 2>/dev/null || true + podman pod rm ${POD_NAME} 2>/dev/null || true fi cleanup_zombie_ports echo "โœ“ Pod torn down, ports cleaned" @@ -526,13 +551,43 @@ safe_pod_teardown() { # Pre-deploy database backup (runs before any pod modification) # Skip if pod doesn't exist (e.g., after manual cleanup) -if podman pod exists agent-router-pod 2>/dev/null; then +if podman pod exists ${POD_NAME} 2>/dev/null; then echo "๐Ÿ’พ Taking pre-deploy database backup..." bash scripts/backup.sh && echo "โœ“ Pre-deploy backup saved" || echo "โš ๏ธ Pre-deploy backup skipped" fi +# โ”€โ”€ ClickHouse port override XML โ”€โ”€ +# Writes a minimal config.d XML override so ClickHouse listens on the +# configured ports instead of its compiled-in defaults. +generate_clickhouse_config() { + local config_dir="${DATA_ROOT}/clickhouse-config" + mkdir -p "$config_dir" + cat > "${config_dir}/port-override.xml" << EOF + + ${CLICKHOUSE_HTTP_PORT} + ${CLICKHOUSE_TCP_PORT} + ${CLICKHOUSE_INTERSERVER_PORT} + +EOF + echo "โœ“ ClickHouse port config written to ${config_dir}/port-override.xml" +} + +# โ”€โ”€ LiteLLM rendered config โ”€โ”€ +# Generates a rendered config.yaml (with port substitutions) into DATA_ROOT/litellm-rendered/ +# so prod and dev each get their own copy with the correct port values. +render_litellm_config() { + local rendered_dir="${DATA_ROOT}/litellm-rendered" + mkdir -p "$rendered_dir" + sed -e "s/VALKEY_CACHE_PORT_PLACEHOLDER/${VALKEY_CACHE_PORT}/g" \ + -e "s/ROUTER_PORT_PLACEHOLDER/${ROUTER_PORT}/g" \ + "${WORKDIR}/litellm/config.yaml" > "${rendered_dir}/config.yaml" + # Copy entrypoint.py unchanged + cp "${WORKDIR}/litellm/entrypoint.py" "${rendered_dir}/entrypoint.py" + echo "โœ“ LiteLLM config rendered to ${rendered_dir}/config.yaml" +} + render_pod_yaml() { - export WORKDIR HOME LITELLM_MASTER_KEY POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL ROUTING_DOMAIN LITELLM_PUBLIC_URL LANGFUSE_PUBLIC_URL LITELLM_ROOT_PATH + export WORKDIR HOME LITELLM_MASTER_KEY POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT python3 - "$WORKDIR/pod.yaml" <<'PY' import os, sys, urllib.parse, json uid = os.getuid() @@ -563,8 +618,19 @@ placeholders = [ "PROXY_BASE_URL_PLACEHOLDER", "PUBLIC_BASE_URL_PLACEHOLDER", "ROUTING_DOMAIN_PLACEHOLDER", - "NEXTAUTH_URL_PLACEHOLDER", - "SERVER_ROOT_PATH_PLACEHOLDER" + "POD_NAME_PLACEHOLDER", + "DATA_ROOT_PLACEHOLDER", + "ROUTER_PORT_PLACEHOLDER", + "LITELLM_PORT_PLACEHOLDER", + "LANGFUSE_WEB_PORT_PLACEHOLDER", + "LANGFUSE_WORKER_PORT_PLACEHOLDER", + "POSTGRES_PORT_PLACEHOLDER", + "VALKEY_CACHE_PORT_PLACEHOLDER", + "VALKEY_LF_PORT_PLACEHOLDER", + "CLICKHOUSE_HTTP_PORT_PLACEHOLDER", + "CLICKHOUSE_TCP_PORT_PLACEHOLDER", + "MINIO_S3_PORT_PLACEHOLDER", + "MINIO_CONSOLE_PORT_PLACEHOLDER", ] for ph in placeholders: if ph not in text: @@ -589,11 +655,30 @@ text = text.replace("MINIO_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["MINIO_ text = text.replace("LANGFUSE_INIT_USER_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["LANGFUSE_INIT_USER_PASSWORD"])) text = text.replace("REDIS_AUTH_PLACEHOLDER", yaml_scalar(os.environ["REDIS_AUTH"])) text = text.replace("CLICKHOUSE_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["CLICKHOUSE_PASSWORD"])) -text = text.replace("PROXY_BASE_URL_PLACEHOLDER", yaml_scalar(os.environ["LITELLM_PUBLIC_URL"])) -text = text.replace("NEXTAUTH_URL_PLACEHOLDER", yaml_scalar(os.environ["LANGFUSE_PUBLIC_URL"])) -text = text.replace("SERVER_ROOT_PATH_PLACEHOLDER", yaml_scalar(os.environ["LITELLM_ROOT_PATH"])) +# Derive PROXY_BASE_URL from PUBLIC_BASE_URL +public_base_url = os.environ["PUBLIC_BASE_URL"].rstrip("/") +proxy_base_url = f"{public_base_url}/litellm" +text = text.replace("PROXY_BASE_URL_PLACEHOLDER", yaml_scalar(proxy_base_url)) text = text.replace("PUBLIC_BASE_URL_PLACEHOLDER", yaml_scalar(os.environ["PUBLIC_BASE_URL"])) text = text.replace("ROUTING_DOMAIN_PLACEHOLDER", yaml_scalar(os.environ["ROUTING_DOMAIN"])) +# Raw replacements (no quoting โ€” used for pod name, data root, and integer port values) +raw_replacements = { + "POD_NAME_PLACEHOLDER": os.environ["POD_NAME"], + "DATA_ROOT_PLACEHOLDER": os.environ["DATA_ROOT"], + "ROUTER_PORT_PLACEHOLDER": os.environ["ROUTER_PORT"], + "LITELLM_PORT_PLACEHOLDER": os.environ["LITELLM_PORT"], + "LANGFUSE_WEB_PORT_PLACEHOLDER": os.environ["LANGFUSE_WEB_PORT"], + "LANGFUSE_WORKER_PORT_PLACEHOLDER": os.environ["LANGFUSE_WORKER_PORT"], + "POSTGRES_PORT_PLACEHOLDER": os.environ["POSTGRES_PORT"], + "VALKEY_CACHE_PORT_PLACEHOLDER": os.environ["VALKEY_CACHE_PORT"], + "VALKEY_LF_PORT_PLACEHOLDER": os.environ["VALKEY_LF_PORT"], + "CLICKHOUSE_HTTP_PORT_PLACEHOLDER": os.environ["CLICKHOUSE_HTTP_PORT"], + "CLICKHOUSE_TCP_PORT_PLACEHOLDER": os.environ["CLICKHOUSE_TCP_PORT"], + "MINIO_S3_PORT_PLACEHOLDER": os.environ["MINIO_S3_PORT"], + "MINIO_CONSOLE_PORT_PLACEHOLDER": os.environ["MINIO_CONSOLE_PORT"], +} +for ph, val in raw_replacements.items(): + text = text.replace(ph, val) import re unresolved = sorted(set(re.findall(r"\b[A-Z0-9_]+_PLACEHOLDER\b", text))) if unresolved: @@ -607,24 +692,38 @@ sys.stdout.write(text) PY } -if podman pod exists agent-router-pod 2>/dev/null; then +if podman pod exists ${POD_NAME} 2>/dev/null; then if $FULL_REBUILD; then echo "๐Ÿ”จ Building custom local triage router image..." - podman build -t localhost/llm-triage-router:latest -f router/Dockerfile router + podman build -t ghcr.io/sheepdestroyer/llm-routing:latest -f router/Dockerfile router safe_pod_teardown echo "๐Ÿš€ Deploying fresh triage pod..." + generate_clickhouse_config + render_litellm_config + render_pod_yaml | podman play kube - + setup_minio_buckets + verify_stack_health + elif $PULL_MODE; then + echo "๐Ÿšš Pulling latest triage router image from GHCR..." + podman pull ghcr.io/sheepdestroyer/llm-routing:latest + safe_pod_teardown + echo "๐Ÿš€ Deploying fresh triage pod with pulled image..." + generate_clickhouse_config + render_litellm_config render_pod_yaml | podman play kube - setup_minio_buckets verify_stack_health elif $REPLACE_MODE; then safe_pod_teardown echo "๐Ÿš€ Deploying replacement pod from YAML..." + generate_clickhouse_config + render_litellm_config render_pod_yaml | podman play kube - setup_minio_buckets verify_stack_health else - echo "๐Ÿ”„ Restarting existing agent-router-pod (use --replace or --full-rebuild to recreate)..." - podman pod restart agent-router-pod + echo "๐Ÿ”„ Restarting existing ${POD_NAME} (use --replace or --pull to recreate)..." + podman pod restart ${POD_NAME} setup_minio_buckets verify_stack_health @@ -643,10 +742,17 @@ if podman pod exists agent-router-pod 2>/dev/null; then else # First deploy โ€” no pod exists, clean ports just in case cleanup_zombie_ports - echo "๐Ÿ”จ Building custom local triage router image..." - podman build -t localhost/llm-triage-router:latest -f router/Dockerfile router + if $FULL_REBUILD; then + echo "๐Ÿ”จ Building custom local triage router image..." + podman build -t ghcr.io/sheepdestroyer/llm-routing:latest -f router/Dockerfile router + else + echo "๐Ÿšš Pulling latest triage router image from GHCR..." + podman pull ghcr.io/sheepdestroyer/llm-routing:latest + fi echo "๐Ÿš€ No existing pod found. Deploying fresh triage pod..." + generate_clickhouse_config + render_litellm_config render_pod_yaml | podman play kube - setup_minio_buckets verify_stack_health From 9e86358bcd7cf7f19350f7122f4d130afe1215d7 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 15:31:11 +0200 Subject: [PATCH 02/11] fix: default DATA_ROOT to WORKDIR/data instead of WORKDIR --- start-stack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/start-stack.sh b/start-stack.sh index 1e949ab9..6c25d109 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -94,7 +94,7 @@ CLICKHOUSE_TCP_PORT="${CLICKHOUSE_TCP_PORT:-9000}" CLICKHOUSE_INTERSERVER_PORT="${CLICKHOUSE_INTERSERVER_PORT:-9009}" MINIO_S3_PORT="${MINIO_S3_PORT:-9002}" MINIO_CONSOLE_PORT="${MINIO_CONSOLE_PORT:-9001}" -DATA_ROOT="${DATA_ROOT:-${WORKDIR}}" +DATA_ROOT="${DATA_ROOT:-${WORKDIR}/data}" export POD_NAME ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT DATA_ROOT # Define and export the routing domain From 38d03b42195bee189ff78cc1f4d110bc733a4e64 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 15:31:25 +0200 Subject: [PATCH 03/11] fix: move dataset mount under DATA_ROOT as datasets/ --- pod.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pod.yaml b/pod.yaml index c9cc70a7..829ad1b5 100644 --- a/pod.yaml +++ b/pod.yaml @@ -507,7 +507,7 @@ spec: path: WORKDIR_PLACEHOLDER name: env-file - hostPath: - path: WORKDIR_PLACEHOLDER/data + path: DATA_ROOT_PLACEHOLDER/datasets type: DirectoryOrCreate name: dataset-data - hostPath: From bce02a67a04df850b1a55ab1bcab206a4c990950 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 15:31:36 +0200 Subject: [PATCH 04/11] =?UTF-8?q?fix:=20remove=20dev-data=20workaround=20?= =?UTF-8?q?=E2=80=94=20DATA=5FROOT=20now=20defaults=20to=20data/?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- start-dev.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/start-dev.sh b/start-dev.sh index 82c254e3..602d9746 100755 --- a/start-dev.sh +++ b/start-dev.sh @@ -5,10 +5,5 @@ WORKDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd "$WORKDIR" export DEV_ENV_FILE="${WORKDIR}/.env.dev" -export DATA_ROOT="${WORKDIR}/dev-data" - -mkdir -p dev-data/valkey-data dev-data/postgres-data dev-data/clickhouse-data \ - dev-data/redis-lf-data dev-data/minio-data dev-data/clickhouse-config \ - dev-data/litellm-rendered exec bash start-stack.sh "$@" From 24a701212fc5eda425c78e5b9fcba1a5d8bcbf4e Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 15:31:57 +0200 Subject: [PATCH 05/11] =?UTF-8?q?fix:=20consolidate=20gitignore=20?= =?UTF-8?q?=E2=80=94=20all=20data=20dirs=20now=20under=20data/?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 34bee0fb..41daa843 100644 --- a/.gitignore +++ b/.gitignore @@ -4,14 +4,8 @@ # Exclude Podman runtime file generated dynamically at startup pod-run.yaml -# Exclude persistent database and cache storages -postgres-data/ -langfuse-data/ -valkey-data/ -clickhouse-data/ -minio-data/ -redis-lf-data/ -dev-data/ +# Exclude persistent database and cache storages (all under data/) +data/ # Exclude auto-generated backups @@ -38,7 +32,5 @@ router/free_models_roster.json workdirs/ pr_description.txt -# Dataset work in progress -data/ .cache/ test_output*.log From 2fe95456b0709078068b49afffdca54055b6ad11 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 15:38:33 +0200 Subject: [PATCH 06/11] fix: router reads ports from env vars (LITELLM_PORT, VALKEY_CACHE_PORT, LANGFUSE_WEB_PORT, ROUTER_PORT) --- router/main.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/router/main.py b/router/main.py index 7b8df8d9..97a47d84 100644 --- a/router/main.py +++ b/router/main.py @@ -23,9 +23,9 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator, RootModel from typing import Dict, Optional, Union -LITELLM_URL = (os.getenv("LITELLM_ADMIN_URL") or "http://127.0.0.1:4000").rstrip("/") +LITELLM_URL = (os.getenv("LITELLM_ADMIN_URL") or f"http://127.0.0.1:{os.getenv('LITELLM_PORT', '4000')}").rstrip("/") LLAMA_SERVER_URL = (os.getenv("LLAMA_SERVER_URL") or "http://127.0.0.1:8080").rstrip("/") -LANGFUSE_HOST = (os.getenv("LANGFUSE_HOST") or "http://127.0.0.1:3001").rstrip("/") +LANGFUSE_HOST = (os.getenv("LANGFUSE_HOST") or f"http://127.0.0.1:{os.getenv('LANGFUSE_WEB_PORT', '3001')}").rstrip("/") GEMINI_OAUTH_CREDS_PATH = "/config/gemini_auth/oauth_creds.json" @@ -57,7 +57,7 @@ def get_redis(): logger.info("Valkey client initialized from URL") else: host = os.getenv("VALKEY_HOST", "127.0.0.1") - port = int(os.getenv("VALKEY_PORT", "6379")) + port = int(os.getenv("VALKEY_CACHE_PORT") or os.getenv("VALKEY_PORT", "6379")) _redis_client = aioredis.Redis(host=host, port=port, decode_responses=True, socket_timeout=1.0) logger.info(f"Valkey client initialized at {host}:{port}") except Exception as e: @@ -1590,7 +1590,7 @@ def get_pie_chart_gradient() -> str: @app.api_route("/v1/memory{path:path}", methods=["GET", "POST", "DELETE", "PUT"]) async def proxy_memory(request: Request, path: str = ""): """Proxies memory API calls to the LiteLLM gateway on port 4000.""" - litellm_base = "http://127.0.0.1:4000/v1/memory" + litellm_base = f"http://127.0.0.1:{os.getenv('LITELLM_PORT', '4000')}/v1/memory" # Resolve the destination URL url = f"{litellm_base}{path}" @@ -2666,10 +2666,10 @@ async def get_dashboard_data(): llamacpp, ) = await asyncio.gather( asyncio.wait_for(sync_cooldowns_from_valkey(), timeout=2.0), - check_tcp_port("127.0.0.1", 6379), - check_http_endpoint("http://127.0.0.1:4000/"), + check_tcp_port("127.0.0.1", int(os.getenv("VALKEY_CACHE_PORT") or os.getenv("VALKEY_PORT", "6379"))), + check_http_endpoint(f"http://127.0.0.1:{os.getenv('LITELLM_PORT', '4000')}/"), check_http_endpoint("http://127.0.0.1:8080/health"), - check_http_endpoint("http://127.0.0.1:3001"), + check_http_endpoint(f"http://127.0.0.1:{os.getenv('LANGFUSE_WEB_PORT', '3001')}"), get_gemini_oauth_status(), asyncio.wait_for(get_best_free_model(), timeout=5.0), asyncio.to_thread(get_goose_sessions), @@ -3867,7 +3867,7 @@ async def get_dashboard(request: Request):
Triage Router - :5000 + :{os.getenv('ROUTER_PORT', '5000')}
Online
@@ -3875,7 +3875,7 @@ async def get_dashboard(request: Request):
LiteLLM Proxy - :4000 + :{os.getenv('LITELLM_PORT', '4000')}
{"Online" if litellm_status else "Offline"} @@ -3885,7 +3885,7 @@ async def get_dashboard(request: Request):
Valkey Cache - :6379 + :{os.getenv('VALKEY_CACHE_PORT') or os.getenv('VALKEY_PORT', '6379')}
{"Online" if valkey_status else "Offline"} From fff4592941815f9796fba6d607c95eb10942afe0 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 15:48:45 +0200 Subject: [PATCH 07/11] fix: router image configurable via ROUTER_IMAGE env var; router reads ports from env --- .env.dev | 3 +++ pod.yaml | 2 +- start-stack.sh | 7 +++++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.env.dev b/.env.dev index 7cc97b63..a5393f15 100644 --- a/.env.dev +++ b/.env.dev @@ -22,3 +22,6 @@ CLICKHOUSE_TCP_PORT="9010" CLICKHOUSE_INTERSERVER_PORT="9019" MINIO_S3_PORT="9012" MINIO_CONSOLE_PORT="9011" + +# Dev uses locally-built router image (for testing code changes) +ROUTER_IMAGE="localhost/llm-routing-dev:latest" diff --git a/pod.yaml b/pod.yaml index 829ad1b5..0f476b06 100644 --- a/pod.yaml +++ b/pod.yaml @@ -124,7 +124,7 @@ spec: value: 'VALKEY_CACHE_PORT_PLACEHOLDER' - name: LOG_LEVEL value: info - image: ghcr.io/sheepdestroyer/llm-routing:latest + image: ROUTER_IMAGE_PLACEHOLDER livenessProbe: exec: command: diff --git a/start-stack.sh b/start-stack.sh index 6c25d109..27057c22 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -94,8 +94,9 @@ CLICKHOUSE_TCP_PORT="${CLICKHOUSE_TCP_PORT:-9000}" CLICKHOUSE_INTERSERVER_PORT="${CLICKHOUSE_INTERSERVER_PORT:-9009}" MINIO_S3_PORT="${MINIO_S3_PORT:-9002}" MINIO_CONSOLE_PORT="${MINIO_CONSOLE_PORT:-9001}" +ROUTER_IMAGE="${ROUTER_IMAGE:-ghcr.io/sheepdestroyer/llm-routing:latest}" DATA_ROOT="${DATA_ROOT:-${WORKDIR}/data}" -export POD_NAME ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT DATA_ROOT +export POD_NAME ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT ROUTER_IMAGE DATA_ROOT # Define and export the routing domain ROUTING_DOMAIN="${ROUTING_DOMAIN:-vendeuvre.lan}" @@ -587,7 +588,7 @@ render_litellm_config() { } render_pod_yaml() { - export WORKDIR HOME LITELLM_MASTER_KEY POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT + export WORKDIR HOME LITELLM_MASTER_KEY POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT python3 - "$WORKDIR/pod.yaml" <<'PY' import os, sys, urllib.parse, json uid = os.getuid() @@ -631,6 +632,7 @@ placeholders = [ "CLICKHOUSE_TCP_PORT_PLACEHOLDER", "MINIO_S3_PORT_PLACEHOLDER", "MINIO_CONSOLE_PORT_PLACEHOLDER", + "ROUTER_IMAGE_PLACEHOLDER", ] for ph in placeholders: if ph not in text: @@ -676,6 +678,7 @@ raw_replacements = { "CLICKHOUSE_TCP_PORT_PLACEHOLDER": os.environ["CLICKHOUSE_TCP_PORT"], "MINIO_S3_PORT_PLACEHOLDER": os.environ["MINIO_S3_PORT"], "MINIO_CONSOLE_PORT_PLACEHOLDER": os.environ["MINIO_CONSOLE_PORT"], + "ROUTER_IMAGE_PLACEHOLDER": os.environ["ROUTER_IMAGE"], } for ph, val in raw_replacements.items(): text = text.replace(ph, val) From 02182ff9edbb4e6851514aff232ac65549597ebf Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 15:55:39 +0200 Subject: [PATCH 08/11] fix: render router config.yaml with LITELLM_PORT placeholder at deploy time --- pod.yaml | 3 ++- router/config.yaml | 16 ++++++++-------- start-stack.sh | 15 +++++++++++++++ 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/pod.yaml b/pod.yaml index 0f476b06..f381d5e0 100644 --- a/pod.yaml +++ b/pod.yaml @@ -471,7 +471,8 @@ spec: type: DirectoryOrCreate name: litellm-rendered - hostPath: - path: WORKDIR_PLACEHOLDER/router + path: DATA_ROOT_PLACEHOLDER/router-rendered + type: DirectoryOrCreate name: router-config - hostPath: path: DATA_ROOT_PLACEHOLDER/postgres-data diff --git a/router/config.yaml b/router/config.yaml index 80b42939..a30ed2ad 100644 --- a/router/config.yaml +++ b/router/config.yaml @@ -34,26 +34,26 @@ classification_rules: # The api_key placeholder is resolved at runtime by main.py:788. backends: - name: "agent-simple-core" - api_base: "http://127.0.0.1:4000/v1" + api_base: "http://127.0.0.1:LITELLM_PORT_PLACEHOLDER/v1" api_key: "DYNAMIC_LITELLM_MASTER_KEY_PLACEHOLDER" - name: "agent-medium-core" - api_base: "http://127.0.0.1:4000/v1" + api_base: "http://127.0.0.1:LITELLM_PORT_PLACEHOLDER/v1" api_key: "DYNAMIC_LITELLM_MASTER_KEY_PLACEHOLDER" - name: "agent-complex-core" - api_base: "http://127.0.0.1:4000/v1" + api_base: "http://127.0.0.1:LITELLM_PORT_PLACEHOLDER/v1" api_key: "DYNAMIC_LITELLM_MASTER_KEY_PLACEHOLDER" - name: "agent-reasoning-core" - api_base: "http://127.0.0.1:4000/v1" + api_base: "http://127.0.0.1:LITELLM_PORT_PLACEHOLDER/v1" api_key: "DYNAMIC_LITELLM_MASTER_KEY_PLACEHOLDER" - name: "agent-advanced-core" - api_base: "http://127.0.0.1:4000/v1" + api_base: "http://127.0.0.1:LITELLM_PORT_PLACEHOLDER/v1" api_key: "DYNAMIC_LITELLM_MASTER_KEY_PLACEHOLDER" - name: "ollama-deepseek-v4-pro" - api_base: "http://127.0.0.1:4000/v1" + api_base: "http://127.0.0.1:LITELLM_PORT_PLACEHOLDER/v1" api_key: "DYNAMIC_LITELLM_MASTER_KEY_PLACEHOLDER" - name: "ollama-deepseek-v4-flash" - api_base: "http://127.0.0.1:4000/v1" + api_base: "http://127.0.0.1:LITELLM_PORT_PLACEHOLDER/v1" api_key: "DYNAMIC_LITELLM_MASTER_KEY_PLACEHOLDER" - name: "openrouter-auto" - api_base: "http://127.0.0.1:4000/v1" + api_base: "http://127.0.0.1:LITELLM_PORT_PLACEHOLDER/v1" api_key: "DYNAMIC_LITELLM_MASTER_KEY_PLACEHOLDER" diff --git a/start-stack.sh b/start-stack.sh index 27057c22..c4c680b8 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -587,6 +587,17 @@ render_litellm_config() { echo "โœ“ LiteLLM config rendered to ${rendered_dir}/config.yaml" } +# โ”€โ”€ Router rendered config โ”€โ”€ +# Generates a rendered config.yaml (with port substitutions) into DATA_ROOT/router-rendered/ +# so prod and dev each get their own copy with the correct LiteLLM port. +render_router_config() { + local rendered_dir="${DATA_ROOT}/router-rendered" + mkdir -p "$rendered_dir" + sed -e "s/LITELLM_PORT_PLACEHOLDER/${LITELLM_PORT}/g" \ + "${WORKDIR}/router/config.yaml" > "${rendered_dir}/config.yaml" + echo "โœ“ Router config rendered to ${rendered_dir}/config.yaml" +} + render_pod_yaml() { export WORKDIR HOME LITELLM_MASTER_KEY POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT python3 - "$WORKDIR/pod.yaml" <<'PY' @@ -703,6 +714,7 @@ if podman pod exists ${POD_NAME} 2>/dev/null; then echo "๐Ÿš€ Deploying fresh triage pod..." generate_clickhouse_config render_litellm_config + render_router_config render_pod_yaml | podman play kube - setup_minio_buckets verify_stack_health @@ -713,6 +725,7 @@ if podman pod exists ${POD_NAME} 2>/dev/null; then echo "๐Ÿš€ Deploying fresh triage pod with pulled image..." generate_clickhouse_config render_litellm_config + render_router_config render_pod_yaml | podman play kube - setup_minio_buckets verify_stack_health @@ -721,6 +734,7 @@ if podman pod exists ${POD_NAME} 2>/dev/null; then echo "๐Ÿš€ Deploying replacement pod from YAML..." generate_clickhouse_config render_litellm_config + render_router_config render_pod_yaml | podman play kube - setup_minio_buckets verify_stack_health @@ -756,6 +770,7 @@ else echo "๐Ÿš€ No existing pod found. Deploying fresh triage pod..." generate_clickhouse_config render_litellm_config + render_router_config render_pod_yaml | podman play kube - setup_minio_buckets verify_stack_health From f4befbb33119e6cc91a713841e39db25cc54fb6e Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 17:02:26 +0200 Subject: [PATCH 09/11] fix: replace tcpSocket probes with valkey-cli exec probes, switch to non-alpine valkey image Root cause: podman play kube converts tcpSocket liveness probes into 'nc -z -v localhost || exit 1' healthchecks. Alpine's BusyBox nc does not support -z, so the healthcheck always fails. With HealthcheckOnFailureAction=restart, this causes an infinite SIGTERM restart loop (~every 30s). Changes: - valkey-cache: tcpSocket -> exec valkey-cli ping (liveness + readiness) - valkey-lf: tcpSocket -> exec valkey-cli -a ping (liveness) - Image: valkey:9.1.0-alpine -> valkey:9.1.0 (non-alpine has no nc, so no broken built-in healthcheck to conflict with) Verified in dev: 0 SIGTERMs, 0 ECONNREFUSED, both valkey instances healthy, Langfuse web/worker stable, chat completions 200 OK. --- pod.yaml | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/pod.yaml b/pod.yaml index f381d5e0..0420e5d2 100644 --- a/pod.yaml +++ b/pod.yaml @@ -13,17 +13,21 @@ spec: - 'VALKEY_CACHE_PORT_PLACEHOLDER' - --loglevel - warning - image: docker.io/valkey/valkey:9.1.0-alpine + image: docker.io/valkey/valkey:9.1.0 livenessProbe: - tcpSocket: - port: VALKEY_CACHE_PORT_PLACEHOLDER + exec: + command: + - valkey-cli + - ping initialDelaySeconds: 5 periodSeconds: 10 timeoutSeconds: 2 name: valkey-cache readinessProbe: - tcpSocket: - port: VALKEY_CACHE_PORT_PLACEHOLDER + exec: + command: + - valkey-cli + - ping initialDelaySeconds: 2 periodSeconds: 5 timeoutSeconds: 2 @@ -272,10 +276,16 @@ spec: - noeviction - --loglevel - warning - image: docker.io/valkey/valkey:9.1.0-alpine + image: docker.io/valkey/valkey:9.1.0 livenessProbe: - tcpSocket: - port: VALKEY_LF_PORT_PLACEHOLDER + exec: + command: + - valkey-cli + - -p + - "VALKEY_LF_PORT_PLACEHOLDER" + - -a + - REDIS_AUTH_PLACEHOLDER + - ping initialDelaySeconds: 5 periodSeconds: 10 timeoutSeconds: 2 From 5f4c87efb05705080d295b259fbd0b9deb098e80 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 17:39:47 +0200 Subject: [PATCH 10/11] fix: add -p VALKEY_CACHE_PORT_PLACEHOLDER to valkey-cache probes Sourcery review: valkey-cli ping without -p targets default 6379, causing false negatives when VALKEY_CACHE_PORT is customized (e.g. dev=6389). Now matches the port-aware pattern already used by valkey-lf probes. --- pod.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pod.yaml b/pod.yaml index 0420e5d2..c871aabf 100644 --- a/pod.yaml +++ b/pod.yaml @@ -18,6 +18,8 @@ spec: exec: command: - valkey-cli + - -p + - 'VALKEY_CACHE_PORT_PLACEHOLDER' - ping initialDelaySeconds: 5 periodSeconds: 10 @@ -27,6 +29,8 @@ spec: exec: command: - valkey-cli + - -p + - 'VALKEY_CACHE_PORT_PLACEHOLDER' - ping initialDelaySeconds: 2 periodSeconds: 5 From 1afa7c1af184a9e1d88a4d134ad24ba005ceb5c4 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 17:53:37 +0200 Subject: [PATCH 11/11] fix: address PR #248 and #249 review comments - litellm/entrypoint.py: restore quote stripping before setdefault (prevents literal '"5432"' from breaking int() parsing) - start-stack.sh: use ROUTER_IMAGE var instead of hardcoded ghcr.io/sheepdestroyer/llm-routing:latest in build/pull paths - start-stack.sh: create volume dirs under DATA_ROOT instead of WORKDIR (fixes pod creation when DATA_ROOT != WORKDIR) - start-stack.sh: add placeholder validation to render_litellm_config and chmod 644 on rendered configs for non-root container access - scripts/backup.sh: use POD_NAME from .env instead of hardcoded agent-router-pod-postgres-db (fixes dev backup failures) - router/main.py: extract _valkey_port() helper to DRY the VALKEY_CACHE_PORT/VALKEY_PORT fallback (3 call sites) - router/main.py: replace hardcoded :3001 with LANGFUSE_WEB_PORT env var in dashboard Infrastructure Nodes display --- litellm/entrypoint.py | 1 + router/main.py | 12 ++++++++---- scripts/backup.sh | 12 +++++++++--- start-stack.sh | 22 +++++++++++++++------- 4 files changed, 33 insertions(+), 14 deletions(-) diff --git a/litellm/entrypoint.py b/litellm/entrypoint.py index 85dca12b..b8755d44 100644 --- a/litellm/entrypoint.py +++ b/litellm/entrypoint.py @@ -16,6 +16,7 @@ line = line.strip() if line and not line.startswith("#") and "=" in line: key, _, val = line.partition("=") + val = val.strip().strip('"').strip("'") os.environ.setdefault(key, val) # Load Gemini OAuth token from credentials JSON diff --git a/router/main.py b/router/main.py index 97a47d84..6ca2e7c3 100644 --- a/router/main.py +++ b/router/main.py @@ -41,6 +41,10 @@ _redis_last_init_attempt = 0.0 _REDIS_RETRY_INTERVAL_SECONDS = 5.0 +def _valkey_port() -> str: + """Resolve the Valkey cache port from env, preferring VALKEY_CACHE_PORT.""" + return os.getenv("VALKEY_CACHE_PORT") or os.getenv("VALKEY_PORT", "6379") + def get_redis(): """Lazily initialize and return the async Redis/Valkey client. Returns None if connection fails or is disabled (non-fatal fallback).""" @@ -57,7 +61,7 @@ def get_redis(): logger.info("Valkey client initialized from URL") else: host = os.getenv("VALKEY_HOST", "127.0.0.1") - port = int(os.getenv("VALKEY_CACHE_PORT") or os.getenv("VALKEY_PORT", "6379")) + port = int(_valkey_port()) _redis_client = aioredis.Redis(host=host, port=port, decode_responses=True, socket_timeout=1.0) logger.info(f"Valkey client initialized at {host}:{port}") except Exception as e: @@ -2666,7 +2670,7 @@ async def get_dashboard_data(): llamacpp, ) = await asyncio.gather( asyncio.wait_for(sync_cooldowns_from_valkey(), timeout=2.0), - check_tcp_port("127.0.0.1", int(os.getenv("VALKEY_CACHE_PORT") or os.getenv("VALKEY_PORT", "6379"))), + check_tcp_port("127.0.0.1", int(_valkey_port())), check_http_endpoint(f"http://127.0.0.1:{os.getenv('LITELLM_PORT', '4000')}/"), check_http_endpoint("http://127.0.0.1:8080/health"), check_http_endpoint(f"http://127.0.0.1:{os.getenv('LANGFUSE_WEB_PORT', '3001')}"), @@ -3885,7 +3889,7 @@ async def get_dashboard(request: Request):
Valkey Cache - :{os.getenv('VALKEY_CACHE_PORT') or os.getenv('VALKEY_PORT', '6379')} + :{_valkey_port()}
{"Online" if valkey_status else "Offline"} @@ -3905,7 +3909,7 @@ async def get_dashboard(request: Request):
Langfuse Traces - :3001 + :{os.getenv('LANGFUSE_WEB_PORT', '3001')}
{"Online" if langfuse_status else "Offline"} diff --git a/scripts/backup.sh b/scripts/backup.sh index 2000ddb7..4a4159e1 100755 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -13,6 +13,12 @@ BACKUP_DIR="${WORKDIR}/backups" RETENTION_DAYS=14 LOG_FILE="/tmp/llm-backup-${TIMESTAMP}.log" +# Source .env for POD_NAME (with prod default) +if [ -f "${WORKDIR}/.env" ]; then + set -a; source "${WORKDIR}/.env"; set +a +fi +POD_NAME="${POD_NAME:-agent-router-pod}" + mkdir -p "$BACKUP_DIR" log() { echo "[$(date '+%H:%M:%S')] $*" | tee -a "$LOG_FILE"; } @@ -21,12 +27,12 @@ log() { echo "[$(date '+%H:%M:%S')] $*" | tee -a "$LOG_FILE"; } log "โณ Checking PostgreSQL readiness..." PG_READY=0 # Check if container exists AND is running before looping -PG_RUNNING=$(podman inspect --format '{{.State.Running}}' agent-router-pod-postgres-db 2>/dev/null || echo "false") +PG_RUNNING=$(podman inspect --format '{{.State.Running}}' ${POD_NAME}-postgres-db 2>/dev/null || echo "false") if [ "$PG_RUNNING" != "true" ]; then log "โš ๏ธ PostgreSQL container not running โ€” skipping DB backup" else for i in {1..15}; do - if podman exec agent-router-pod-postgres-db pg_isready -U postgres 2>/dev/null; then + if podman exec ${POD_NAME}-postgres-db pg_isready -U postgres 2>/dev/null; then PG_READY=1 log "โœ… PostgreSQL is ready" break @@ -40,7 +46,7 @@ fi if [ $PG_READY -eq 1 ]; then for db in postgres langfuse; do FILE="${BACKUP_DIR}/${db}_db_${TIMESTAMP}.dump" - if podman exec agent-router-pod-postgres-db \ + if podman exec ${POD_NAME}-postgres-db \ pg_dump -U postgres -d "$db" -F c > "$FILE"; then log "โœ… ${db} db: $(ls -lh "$FILE" | awk '{print $5}')" else diff --git a/start-stack.sh b/start-stack.sh index c4c680b8..be86e4a5 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -37,9 +37,6 @@ if [ $# -gt 1 ]; then exit 1 fi -# Ensure local volume directories exist on the host for Podman mounts -mkdir -p valkey-data postgres-data langfuse-data clickhouse-data redis-lf-data minio-data - PULL_MODE=false FULL_REBUILD=false REPLACE_MODE=false @@ -98,6 +95,9 @@ ROUTER_IMAGE="${ROUTER_IMAGE:-ghcr.io/sheepdestroyer/llm-routing:latest}" DATA_ROOT="${DATA_ROOT:-${WORKDIR}/data}" export POD_NAME ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT ROUTER_IMAGE DATA_ROOT +# Ensure local volume directories exist on the host for Podman mounts +mkdir -p "${DATA_ROOT}/valkey-data" "${DATA_ROOT}/postgres-data" "${DATA_ROOT}/langfuse-data" "${DATA_ROOT}/clickhouse-data" "${DATA_ROOT}/redis-lf-data" "${DATA_ROOT}/minio-data" + # Define and export the routing domain ROUTING_DOMAIN="${ROUTING_DOMAIN:-vendeuvre.lan}" export ROUTING_DOMAIN @@ -582,8 +582,15 @@ render_litellm_config() { sed -e "s/VALKEY_CACHE_PORT_PLACEHOLDER/${VALKEY_CACHE_PORT}/g" \ -e "s/ROUTER_PORT_PLACEHOLDER/${ROUTER_PORT}/g" \ "${WORKDIR}/litellm/config.yaml" > "${rendered_dir}/config.yaml" + # Validate no unresolved placeholders remain + if grep -q 'VALKEY_CACHE_PORT_PLACEHOLDER\|ROUTER_PORT_PLACEHOLDER' "${rendered_dir}/config.yaml"; then + echo "โŒ Error: Unresolved placeholders remain in ${rendered_dir}/config.yaml" >&2 + exit 1 + fi + chmod 644 "${rendered_dir}/config.yaml" # Copy entrypoint.py unchanged cp "${WORKDIR}/litellm/entrypoint.py" "${rendered_dir}/entrypoint.py" + chmod 644 "${rendered_dir}/entrypoint.py" echo "โœ“ LiteLLM config rendered to ${rendered_dir}/config.yaml" } @@ -595,6 +602,7 @@ render_router_config() { mkdir -p "$rendered_dir" sed -e "s/LITELLM_PORT_PLACEHOLDER/${LITELLM_PORT}/g" \ "${WORKDIR}/router/config.yaml" > "${rendered_dir}/config.yaml" + chmod 644 "${rendered_dir}/config.yaml" echo "โœ“ Router config rendered to ${rendered_dir}/config.yaml" } @@ -709,7 +717,7 @@ PY if podman pod exists ${POD_NAME} 2>/dev/null; then if $FULL_REBUILD; then echo "๐Ÿ”จ Building custom local triage router image..." - podman build -t ghcr.io/sheepdestroyer/llm-routing:latest -f router/Dockerfile router + podman build -t "${ROUTER_IMAGE}" -f router/Dockerfile router safe_pod_teardown echo "๐Ÿš€ Deploying fresh triage pod..." generate_clickhouse_config @@ -720,7 +728,7 @@ if podman pod exists ${POD_NAME} 2>/dev/null; then verify_stack_health elif $PULL_MODE; then echo "๐Ÿšš Pulling latest triage router image from GHCR..." - podman pull ghcr.io/sheepdestroyer/llm-routing:latest + podman pull "${ROUTER_IMAGE}" safe_pod_teardown echo "๐Ÿš€ Deploying fresh triage pod with pulled image..." generate_clickhouse_config @@ -761,10 +769,10 @@ else cleanup_zombie_ports if $FULL_REBUILD; then echo "๐Ÿ”จ Building custom local triage router image..." - podman build -t ghcr.io/sheepdestroyer/llm-routing:latest -f router/Dockerfile router + podman build -t "${ROUTER_IMAGE}" -f router/Dockerfile router else echo "๐Ÿšš Pulling latest triage router image from GHCR..." - podman pull ghcr.io/sheepdestroyer/llm-routing:latest + podman pull "${ROUTER_IMAGE}" fi echo "๐Ÿš€ No existing pod found. Deploying fresh triage pod..."