From 3084c68010fbd561cbb85bed559c05c6b1ff20ef Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 12:17:46 +0200 Subject: [PATCH 01/24] feat: parameterize all ports and pod name for parallel prod/dev deployments Replaces all hardcoded port values and the pod name in pod.yaml, start-stack.sh, litellm/config.yaml and litellm/entrypoint.py with environment-variable-driven placeholders. A single pod.yaml now serves both the production agent-router-pod and the development dev-router-pod. Changes: - pod.yaml: rewrite with *_PLACEHOLDER vars for pod name, all 13 service ports, and DATA_ROOT; add clickhouse-port-config volume; litellm-config volume now points to DATA_ROOT/litellm-rendered (rendered copy) - start-stack.sh: load optional DEV_ENV_FILE overlay; resolve port/name defaults after env sourcing; add generate_clickhouse_config() and render_litellm_config() helpers; parameterize all agent-router-pod refs and hardcoded ports throughout cleanup, health-check, and minio helpers - litellm/config.yaml: replace port 6379 / 5000 with *_PLACEHOLDER vars (rendered at deploy time via render_litellm_config) - litellm/entrypoint.py: read POSTGRES_PORT / LITELLM_PORT from env instead of hardcoded 5432 / 4000; use setdefault to not clobber container-injected env vars - .env: append production port defaults (POD_NAME, ROUTER_PORT, ...) - .env.dev: new dev overlay (POD_NAME=dev-router-pod, +10 port offsets, dev.vendeuvre.lan URLs) - start-dev.sh: thin wrapper that sets DEV_ENV_FILE + DATA_ROOT then delegates to start-stack.sh - .gitignore: exclude dev-data/ (dev-stack runtime volumes) --- .env.dev | 24 ++++++ .gitignore | 2 + litellm/config.yaml | 6 +- litellm/entrypoint.py | 11 +-- pod.yaml | 124 ++++++++++++++++++----------- start-dev.sh | 14 ++++ start-stack.sh | 178 +++++++++++++++++++++++++++++++++--------- 7 files changed, 270 insertions(+), 89 deletions(-) create mode 100644 .env.dev create mode 100755 start-dev.sh diff --git a/.env.dev b/.env.dev new file mode 100644 index 00000000..7cc97b63 --- /dev/null +++ b/.env.dev @@ -0,0 +1,24 @@ +# Dev environment overlay β€” sourced AFTER .env to override prod values +# Secrets are inherited from .env; only env-specific config is set here. + +# Pod identity +POD_NAME="dev-router-pod" + +# Public URLs +BASEURL="dev.vendeuvre.lan" +BASE_URL="dev.vendeuvre.lan" +PUBLIC_BASE_URL="https://dev.vendeuvre.lan/llm-routing" + +# Dev port assignments (+10 offset from production) +ROUTER_PORT="5010" +LITELLM_PORT="4010" +LANGFUSE_WEB_PORT="3011" +LANGFUSE_WORKER_PORT="3040" +POSTGRES_PORT="5442" +VALKEY_CACHE_PORT="6389" +VALKEY_LF_PORT="6390" +CLICKHOUSE_HTTP_PORT="8133" +CLICKHOUSE_TCP_PORT="9010" +CLICKHOUSE_INTERSERVER_PORT="9019" +MINIO_S3_PORT="9012" +MINIO_CONSOLE_PORT="9011" diff --git a/.gitignore b/.gitignore index 5ccf9013..34bee0fb 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,8 @@ valkey-data/ clickhouse-data/ minio-data/ redis-lf-data/ +dev-data/ + # Exclude auto-generated backups backups/ diff --git a/litellm/config.yaml b/litellm/config.yaml index 195712f5..b204f9cc 100644 --- a/litellm/config.yaml +++ b/litellm/config.yaml @@ -12,7 +12,7 @@ litellm_settings: cache_params: host: 127.0.0.1 mode: default - port: 6379 + port: VALKEY_CACHE_PORT_PLACEHOLDER ttl: 3600 type: redis caching_backend: redis @@ -77,7 +77,7 @@ model_list: is_public_model_group: true - litellm_params: model: openai/llm-routing-ollama - api_base: http://127.0.0.1:5000/v1 + api_base: http://127.0.0.1:ROUTER_PORT_PLACEHOLDER/v1 api_key: os.environ/LITELLM_MASTER_KEY request_timeout: 120 model_name: llm-routing-ollama @@ -237,7 +237,7 @@ model_list: redis_settings: redis_host: 127.0.0.1 - redis_port: 6379 + redis_port: VALKEY_CACHE_PORT_PLACEHOLDER router_settings: allowed_fails: 0 cooldown_time: 300 diff --git a/litellm/entrypoint.py b/litellm/entrypoint.py index 65520978..85dca12b 100644 --- a/litellm/entrypoint.py +++ b/litellm/entrypoint.py @@ -16,8 +16,7 @@ line = line.strip() if line and not line.startswith("#") and "=" in line: key, _, val = line.partition("=") - val = val.strip().strip('"').strip("'") - os.environ[key] = val + os.environ.setdefault(key, val) # Load Gemini OAuth token from credentials JSON creds_path = "/config/gemini_auth/oauth_creds.json" @@ -46,9 +45,10 @@ def check_tcp_port(ip: str, port: int) -> bool: return False max_wait = 60 -print(f"πŸ”Œ Waiting for PostgreSQL on :5432 (max {max_wait}s)...") +postgres_port = int(os.environ.get("POSTGRES_PORT", "5432")) +print(f"πŸ”Œ Waiting for PostgreSQL on :{postgres_port} (max {max_wait}s)...") for i in range(max_wait): - if check_tcp_port("127.0.0.1", 5432): + if check_tcp_port("127.0.0.1", postgres_port): print(f"βœ… PostgreSQL ready after {i+1}s") break time.sleep(1) @@ -127,5 +127,6 @@ def _serialize_dt(dt): # Start LiteLLM Proxy import litellm from litellm.proxy.proxy_cli import run_server -sys.argv = ["litellm", "--config", "/app/config.yaml", "--port", "4000"] +litellm_port = os.environ.get("LITELLM_PORT", os.environ.get("PORT", "4000")) +sys.argv = ["litellm", "--config", "/app/config.yaml", "--port", litellm_port] run_server() diff --git a/pod.yaml b/pod.yaml index db122105..c225a57e 100644 --- a/pod.yaml +++ b/pod.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Pod metadata: - name: agent-router-pod + name: POD_NAME_PLACEHOLDER spec: containers: # Valkey Cache β€” LiteLLM response cache (exact-match & semantic) @@ -9,19 +9,21 @@ spec: - valkey-server - --bind - 127.0.0.1 + - --port + - 'VALKEY_CACHE_PORT_PLACEHOLDER' - --loglevel - warning image: docker.io/valkey/valkey:9.1.0-alpine livenessProbe: tcpSocket: - port: 6379 + port: VALKEY_CACHE_PORT_PLACEHOLDER initialDelaySeconds: 5 periodSeconds: 10 timeoutSeconds: 2 name: valkey-cache readinessProbe: tcpSocket: - port: 6379 + port: VALKEY_CACHE_PORT_PLACEHOLDER initialDelaySeconds: 2 periodSeconds: 5 timeoutSeconds: 2 @@ -34,28 +36,34 @@ spec: - /app/entrypoint.py env: - name: DATABASE_URL - value: postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:5432/postgres + value: postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:POSTGRES_PORT_PLACEHOLDER/postgres - name: STORE_MODEL_IN_DB value: 'True' - name: LITELLM_LOG value: INFO - name: LANGFUSE_HOST - value: http://127.0.0.1:3001 + value: http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER - name: LITELLM_MASTER_KEY value: LITELLM_MASTER_KEY_PLACEHOLDER - name: OLLAMA_API_KEY value: OLLAMA_API_KEY_PLACEHOLDER - name: SERVER_ROOT_PATH - value: SERVER_ROOT_PATH_PLACEHOLDER + value: /llm-routing/litellm - name: PROXY_BASE_URL value: PROXY_BASE_URL_PLACEHOLDER + - name: PORT + value: 'LITELLM_PORT_PLACEHOLDER' + - name: LITELLM_PORT + value: 'LITELLM_PORT_PLACEHOLDER' + - name: POSTGRES_PORT + value: 'POSTGRES_PORT_PLACEHOLDER' image: ghcr.io/berriai/litellm:v1.91.0 livenessProbe: exec: command: - python3 - -c - - import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveness') + - import urllib.request; urllib.request.urlopen('http://localhost:LITELLM_PORT_PLACEHOLDER/ping') initialDelaySeconds: 240 periodSeconds: 15 timeoutSeconds: 5 @@ -65,16 +73,16 @@ spec: command: - python3 - -c - - import urllib.request; urllib.request.urlopen('http://localhost:4000/health/readiness') + - import urllib.request; urllib.request.urlopen('http://localhost:LITELLM_PORT_PLACEHOLDER/health/readiness') initialDelaySeconds: 45 periodSeconds: 10 timeoutSeconds: 5 volumeMounts: - mountPath: /app/config.yaml - name: litellm-config + name: litellm-rendered subPath: config.yaml - mountPath: /app/entrypoint.py - name: litellm-config + name: litellm-rendered subPath: entrypoint.py - mountPath: /config/.env name: env-file @@ -83,7 +91,7 @@ spec: name: gemini-secrets # LLM Triage Router β€” classification + routing gateway - args: - - set -a && . /config/.env && set +a && exec uvicorn main:app --host 0.0.0.0 --port 5000 --log-level ${LOG_LEVEL:-warning} + - set -a && . /config/.env && set +a && LITELLM_PORT=LITELLM_PORT_PLACEHOLDER VALKEY_CACHE_PORT=VALKEY_CACHE_PORT_PLACEHOLDER exec uvicorn main:app --host 0.0.0.0 --port ROUTER_PORT_PLACEHOLDER --log-level ${LOG_LEVEL:-warning} command: - /bin/sh - -c @@ -93,7 +101,7 @@ spec: - name: LITELLM_CONFIG_PATH value: /config/litellm_dir/config.yaml - name: DATABASE_URL - value: postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:5432/postgres + value: postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:POSTGRES_PORT_PLACEHOLDER/postgres - name: DBUS_SESSION_BUS_ADDRESS value: unix:path=RUN_USER_PLACEHOLDER/bus - name: LITELLM_MASTER_KEY @@ -103,22 +111,26 @@ spec: - name: LANGFUSE_SECRET_KEY value: LANGFUSE_SECRET_KEY_PLACEHOLDER - name: LANGFUSE_HOST - value: http://127.0.0.1:3001 + value: http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER - name: OLLAMA_API_KEY value: OLLAMA_API_KEY_PLACEHOLDER - name: PUBLIC_BASE_URL value: PUBLIC_BASE_URL_PLACEHOLDER - name: ROUTING_DOMAIN value: ROUTING_DOMAIN_PLACEHOLDER + - name: LITELLM_PORT + value: 'LITELLM_PORT_PLACEHOLDER' + - name: VALKEY_CACHE_PORT + value: 'VALKEY_CACHE_PORT_PLACEHOLDER' - name: LOG_LEVEL value: info - image: localhost/llm-triage-router:latest + image: ghcr.io/sheepdestroyer/llm-routing:latest livenessProbe: exec: command: - python3 - -c - - import urllib.request; urllib.request.urlopen('http://localhost:5000/metrics') + - import urllib.request; urllib.request.urlopen('http://localhost:ROUTER_PORT_PLACEHOLDER/metrics') initialDelaySeconds: 20 periodSeconds: 15 timeoutSeconds: 5 @@ -128,7 +140,7 @@ spec: command: - python3 - -c - - import urllib.request; urllib.request.urlopen('http://localhost:5000/metrics') + - import urllib.request; urllib.request.urlopen('http://localhost:ROUTER_PORT_PLACEHOLDER/metrics') initialDelaySeconds: 10 periodSeconds: 10 timeoutSeconds: 5 @@ -136,7 +148,7 @@ spec: - mountPath: /config/router_dir name: router-config - mountPath: /config/litellm_dir - name: litellm-config + name: litellm-rendered - mountPath: /app/data name: dataset-data - mountPath: /config/gemini_auth @@ -167,6 +179,8 @@ spec: value: langfuse - name: PGDATA value: /var/lib/postgresql/data/pg18 + - name: PGPORT + value: 'POSTGRES_PORT_PLACEHOLDER' image: docker.io/pgvector/pgvector:0.8.3-pg18 livenessProbe: exec: @@ -174,6 +188,8 @@ spec: - pg_isready - -U - postgres + - -p + - 'POSTGRES_PORT_PLACEHOLDER' initialDelaySeconds: 15 periodSeconds: 10 timeoutSeconds: 5 @@ -184,6 +200,8 @@ spec: - pg_isready - -U - postgres + - -p + - 'POSTGRES_PORT_PLACEHOLDER' initialDelaySeconds: 5 periodSeconds: 5 timeoutSeconds: 5 @@ -203,6 +221,10 @@ spec: exec: command: - clickhouse-client + - --host + - 127.0.0.1 + - --port + - 'CLICKHOUSE_TCP_PORT_PLACEHOLDER' - --user - clickhouse - --password @@ -217,6 +239,10 @@ spec: exec: command: - clickhouse-client + - --host + - 127.0.0.1 + - --port + - 'CLICKHOUSE_TCP_PORT_PLACEHOLDER' - --user - clickhouse - --password @@ -230,13 +256,16 @@ spec: volumeMounts: - mountPath: /var/lib/clickhouse name: clickhouse-storage + - mountPath: /etc/clickhouse-server/config.d/port-override.xml + name: clickhouse-port-config + subPath: port-override.xml # Valkey LF β€” Langfuse v3 BullMQ job queue - command: - redis-server - --bind - 127.0.0.1 - --port - - '6380' + - 'VALKEY_LF_PORT_PLACEHOLDER' - --requirepass - REDIS_AUTH_PLACEHOLDER - --maxmemory-policy @@ -246,7 +275,7 @@ spec: image: docker.io/valkey/valkey:9.1.0-alpine livenessProbe: tcpSocket: - port: 6380 + port: VALKEY_LF_PORT_PLACEHOLDER initialDelaySeconds: 5 periodSeconds: 10 timeoutSeconds: 2 @@ -256,7 +285,7 @@ spec: command: - valkey-cli - -p - - "6380" + - "VALKEY_LF_PORT_PLACEHOLDER" - -a - REDIS_AUTH_PLACEHOLDER - ping @@ -269,11 +298,11 @@ spec: # Langfuse Web β€” observability dashboard - env: - name: DATABASE_URL - value: postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:5432/langfuse + value: postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:POSTGRES_PORT_PLACEHOLDER/langfuse - name: NEXTAUTH_SECRET value: NEXTAUTH_SECRET_PLACEHOLDER - name: NEXTAUTH_URL - value: NEXTAUTH_URL_PLACEHOLDER + value: http://localhost:LANGFUSE_WEB_PORT_PLACEHOLDER - name: SALT value: SALT_PLACEHOLDER - name: ENCRYPTION_KEY @@ -281,7 +310,7 @@ spec: - name: HOSTNAME value: 0.0.0.0 - name: PORT - value: '3001' + value: 'LANGFUSE_WEB_PORT_PLACEHOLDER' - name: TELEMETRY_ENABLED value: 'false' - name: LANGFUSE_S3_EVENT_UPLOAD_BUCKET @@ -293,13 +322,13 @@ spec: - name: LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY value: MINIO_PASSWORD_PLACEHOLDER - name: LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT - value: http://127.0.0.1:9002 + value: http://127.0.0.1:MINIO_S3_PORT_PLACEHOLDER - name: S3_FORCE_PATH_STYLE value: 'true' - name: CLICKHOUSE_URL - value: http://127.0.0.1:8123 + value: http://127.0.0.1:CLICKHOUSE_HTTP_PORT_PLACEHOLDER - name: CLICKHOUSE_MIGRATION_URL - value: clickhouse://127.0.0.1:9000 + value: clickhouse://127.0.0.1:CLICKHOUSE_TCP_PORT_PLACEHOLDER - name: CLICKHOUSE_USER value: clickhouse - name: CLICKHOUSE_PASSWORD @@ -309,7 +338,7 @@ spec: - name: REDIS_HOST value: 127.0.0.1 - name: REDIS_PORT - value: '6380' + value: 'VALKEY_LF_PORT_PLACEHOLDER' - name: REDIS_AUTH value: REDIS_AUTH_PLACEHOLDER - name: LANGFUSE_INIT_ORG_ID @@ -334,7 +363,7 @@ spec: - -q - -O - /dev/null - - http://127.0.0.1:3001/api/health + - http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER/api/health initialDelaySeconds: 240 periodSeconds: 15 timeoutSeconds: 5 @@ -346,16 +375,16 @@ spec: - -q - -O - /dev/null - - http://127.0.0.1:3001/api/health + - http://127.0.0.1:LANGFUSE_WEB_PORT_PLACEHOLDER/api/health initialDelaySeconds: 15 periodSeconds: 10 timeoutSeconds: 5 # Langfuse Worker β€” background job processor - env: - name: DATABASE_URL - value: postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:5432/langfuse + value: postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:POSTGRES_PORT_PLACEHOLDER/langfuse - name: CLICKHOUSE_URL - value: http://127.0.0.1:8123 + value: http://127.0.0.1:CLICKHOUSE_HTTP_PORT_PLACEHOLDER - name: CLICKHOUSE_USER value: clickhouse - name: CLICKHOUSE_PASSWORD @@ -365,13 +394,13 @@ spec: - name: REDIS_HOST value: 127.0.0.1 - name: REDIS_PORT - value: '6380' + value: 'VALKEY_LF_PORT_PLACEHOLDER' - name: REDIS_AUTH value: REDIS_AUTH_PLACEHOLDER - name: HOSTNAME value: 0.0.0.0 - name: PORT - value: '3030' + value: 'LANGFUSE_WORKER_PORT_PLACEHOLDER' - name: TELEMETRY_ENABLED value: 'false' - name: LANGFUSE_S3_EVENT_UPLOAD_BUCKET @@ -383,7 +412,7 @@ spec: - name: S3_FORCE_PATH_STYLE value: 'true' - name: LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT - value: http://127.0.0.1:9002 + value: http://127.0.0.1:MINIO_S3_PORT_PLACEHOLDER - name: LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY value: MINIO_PASSWORD_PLACEHOLDER - name: LANGFUSE_LOG_LEVEL @@ -404,9 +433,9 @@ spec: - --quiet - /data - --address - - ":9002" + - ":MINIO_S3_PORT_PLACEHOLDER" - --console-address - - ":9001" + - ":MINIO_CONSOLE_PORT_PLACEHOLDER" env: - name: MINIO_ROOT_USER value: MINIO_USER_PLACEHOLDER @@ -416,7 +445,7 @@ spec: livenessProbe: httpGet: path: /minio/health/live - port: 9002 + port: MINIO_S3_PORT_PLACEHOLDER initialDelaySeconds: 10 periodSeconds: 15 timeoutSeconds: 5 @@ -424,7 +453,7 @@ spec: readinessProbe: httpGet: path: /minio/health/ready - port: 9002 + port: MINIO_S3_PORT_PLACEHOLDER initialDelaySeconds: 5 periodSeconds: 10 timeoutSeconds: 5 @@ -435,25 +464,26 @@ spec: restartPolicy: Always volumes: - hostPath: - path: WORKDIR_PLACEHOLDER/valkey-data + path: DATA_ROOT_PLACEHOLDER/valkey-data name: valkey-storage - hostPath: - path: WORKDIR_PLACEHOLDER/litellm - name: litellm-config + path: DATA_ROOT_PLACEHOLDER/litellm-rendered + type: DirectoryOrCreate + name: litellm-rendered - hostPath: path: WORKDIR_PLACEHOLDER/router name: router-config - hostPath: - path: WORKDIR_PLACEHOLDER/postgres-data + path: DATA_ROOT_PLACEHOLDER/postgres-data name: postgres-storage - hostPath: - path: WORKDIR_PLACEHOLDER/clickhouse-data + path: DATA_ROOT_PLACEHOLDER/clickhouse-data name: clickhouse-storage - hostPath: - path: WORKDIR_PLACEHOLDER/redis-lf-data + path: DATA_ROOT_PLACEHOLDER/redis-lf-data name: redis-lf-storage - hostPath: - path: WORKDIR_PLACEHOLDER/minio-data + path: DATA_ROOT_PLACEHOLDER/minio-data name: minio-storage - hostPath: path: HOME_PLACEHOLDER/.gemini @@ -480,3 +510,7 @@ spec: path: WORKDIR_PLACEHOLDER/data type: DirectoryOrCreate name: dataset-data + - hostPath: + path: DATA_ROOT_PLACEHOLDER/clickhouse-config + type: DirectoryOrCreate + name: clickhouse-port-config diff --git a/start-dev.sh b/start-dev.sh new file mode 100755 index 00000000..82c254e3 --- /dev/null +++ b/start-dev.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Dev deployment wrapper β€” stands up dev-router-pod on distinct ports alongside prod +set -e +WORKDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$WORKDIR" + +export DEV_ENV_FILE="${WORKDIR}/.env.dev" +export DATA_ROOT="${WORKDIR}/dev-data" + +mkdir -p dev-data/valkey-data dev-data/postgres-data dev-data/clickhouse-data \ + dev-data/redis-lf-data dev-data/minio-data dev-data/clickhouse-config \ + dev-data/litellm-rendered + +exec bash start-stack.sh "$@" diff --git a/start-stack.sh b/start-stack.sh index 4f0f9588..1e949ab9 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -16,7 +16,8 @@ show_help() { echo "Usage:" echo " ./start-stack.sh β†’ Restart existing pod (fast, preserves logs)" echo " ./start-stack.sh --replace β†’ Stop, clean up zombie ports, and recreate/redeploy pod" - echo " ./start-stack.sh --full-rebuild β†’ Rebuild router image and recreate/redeploy pod" + echo " ./start-stack.sh --pull β†’ Pull latest router image from GHCR and recreate/redeploy pod" + echo " ./start-stack.sh --full-rebuild β†’ Rebuild custom router image locally and recreate/redeploy pod" echo " ./start-stack.sh --help | -h β†’ Show this help message and exit" } @@ -39,9 +40,12 @@ fi # Ensure local volume directories exist on the host for Podman mounts mkdir -p valkey-data postgres-data langfuse-data clickhouse-data redis-lf-data minio-data +PULL_MODE=false FULL_REBUILD=false REPLACE_MODE=false -if [ "${1:-}" = "--full-rebuild" ]; then +if [ "${1:-}" = "--pull" ] || [ "${1:-}" = "--pull-latest" ]; then + PULL_MODE=true +elif [ "${1:-}" = "--full-rebuild" ]; then FULL_REBUILD=true elif [ "${1:-}" = "--replace" ]; then REPLACE_MODE=true @@ -69,6 +73,30 @@ if [ -f "$ENV_FILE" ]; then set +a fi +# Load optional dev-environment overlay (set DEV_ENV_FILE before calling this script) +if [ -n "${DEV_ENV_FILE:-}" ] && [ -f "$DEV_ENV_FILE" ]; then + set -a + source "$DEV_ENV_FILE" + set +a +fi + +# Port assignments β€” read from env (set by .env or .env.dev) with prod defaults +POD_NAME="${POD_NAME:-agent-router-pod}" +ROUTER_PORT="${ROUTER_PORT:-5000}" +LITELLM_PORT="${LITELLM_PORT:-4000}" +LANGFUSE_WEB_PORT="${LANGFUSE_WEB_PORT:-3001}" +LANGFUSE_WORKER_PORT="${LANGFUSE_WORKER_PORT:-3030}" +POSTGRES_PORT="${POSTGRES_PORT:-5432}" +VALKEY_CACHE_PORT="${VALKEY_CACHE_PORT:-6379}" +VALKEY_LF_PORT="${VALKEY_LF_PORT:-6380}" +CLICKHOUSE_HTTP_PORT="${CLICKHOUSE_HTTP_PORT:-8123}" +CLICKHOUSE_TCP_PORT="${CLICKHOUSE_TCP_PORT:-9000}" +CLICKHOUSE_INTERSERVER_PORT="${CLICKHOUSE_INTERSERVER_PORT:-9009}" +MINIO_S3_PORT="${MINIO_S3_PORT:-9002}" +MINIO_CONSOLE_PORT="${MINIO_CONSOLE_PORT:-9001}" +DATA_ROOT="${DATA_ROOT:-${WORKDIR}}" +export POD_NAME ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT DATA_ROOT + # Define and export the routing domain ROUTING_DOMAIN="${ROUTING_DOMAIN:-vendeuvre.lan}" export ROUTING_DOMAIN @@ -82,12 +110,9 @@ if [[ ! "$PUBLIC_BASE_URL" =~ /llm-routing ]]; then PUBLIC_BASE_URL="${PUBLIC_BASE_URL%/}/llm-routing" fi PUBLIC_BASE_URL="${PUBLIC_BASE_URL%/}" -LOCAL_BASE_URL="${LOCAL_BASE_URL:-http://localhost:5000}" +LOCAL_BASE_URL="${LOCAL_BASE_URL:-http://localhost:${ROUTER_PORT}}" LOCAL_BASE_URL="${LOCAL_BASE_URL%/}" -LITELLM_PUBLIC_URL="${LITELLM_PUBLIC_URL:-https://litellm.x570.${ROUTING_DOMAIN}}" -LANGFUSE_PUBLIC_URL="${LANGFUSE_PUBLIC_URL:-https://langfuse.x570.${ROUTING_DOMAIN}}" -LITELLM_ROOT_PATH="${LITELLM_ROOT_PATH:-}" -export PUBLIC_BASE_URL LOCAL_BASE_URL LITELLM_PUBLIC_URL LANGFUSE_PUBLIC_URL LITELLM_ROOT_PATH +export PUBLIC_BASE_URL LOCAL_BASE_URL # Ensure openssl is installed if we need to generate passwords/keys @@ -337,7 +362,7 @@ fi # Hermes profiles (e.g., llm-routing-openrouter) whose container storage # can leave surviving processes holding ports indefinitely. cleanup_zombie_ports() { - local ALL_PORTS="3000 3030 4000 5000 5005 5432 6379 6380 8080 8123 9000 9001 9002 9004 9005 9009" + local ALL_PORTS="$ROUTER_PORT $LITELLM_PORT $LANGFUSE_WEB_PORT $LANGFUSE_WORKER_PORT $POSTGRES_PORT $VALKEY_CACHE_PORT $VALKEY_LF_PORT $CLICKHOUSE_HTTP_PORT $CLICKHOUSE_TCP_PORT $CLICKHOUSE_INTERSERVER_PORT $MINIO_S3_PORT $MINIO_CONSOLE_PORT 8080 9004 9005" echo "🧹 Cleaning up zombie port bindings..." @@ -411,9 +436,9 @@ setup_minio_buckets() { echo "" echo "πŸ“¦ Ensuring MinIO buckets exist..." - # Wait for MinIO S3 API to be ready (port 9002) + # Wait for MinIO S3 API to be ready while [ $waited -lt $MAX_WAIT ]; do - if curl -sf --max-time 3 http://127.0.0.1:9002/minio/health/live >/dev/null 2>&1; then + if curl -sf --max-time 3 http://127.0.0.1:${MINIO_S3_PORT}/minio/health/live >/dev/null 2>&1; then echo " βœ“ MinIO S3 API ready after ${waited}s" break fi @@ -425,22 +450,22 @@ setup_minio_buckets() { return 1 fi - # Ensure mc alias points to the correct MinIO S3 API port (9002, not 9000) + # Ensure mc alias points to the correct MinIO S3 API port # The default 'local' alias in the MinIO image points to :9000 which is ClickHouse, # not MinIO. We must override it. - if ! podman exec agent-router-pod-minio-s3 mc alias set local http://127.0.0.1:9002 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"; then - echo "❌ Error: Failed to set MinIO alias 'local' on http://127.0.0.1:9002" >&2 + if ! podman exec ${POD_NAME}-minio-s3 mc alias set local http://127.0.0.1:${MINIO_S3_PORT} "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"; then + echo "❌ Error: Failed to set MinIO alias 'local' on http://127.0.0.1:${MINIO_S3_PORT}" >&2 exit 1 fi # Create required buckets (idempotent) local BUCKETS=("langfuse-events" "proj-triage-gateway-id") for bucket in "${BUCKETS[@]}"; do - if podman exec agent-router-pod-minio-s3 mc ls "local/${bucket}" >/dev/null 2>&1; then + if podman exec ${POD_NAME}-minio-s3 mc ls "local/${bucket}" >/dev/null 2>&1; then echo " βœ“ Bucket '${bucket}' exists" else echo " + Creating bucket '${bucket}'..." - podman exec agent-router-pod-minio-s3 mc mb "local/${bucket}" 2>/dev/null || { + podman exec ${POD_NAME}-minio-s3 mc mb "local/${bucket}" 2>/dev/null || { echo " ⚠️ Failed to create bucket '${bucket}'" } fi @@ -459,7 +484,7 @@ verify_stack_health() { # Wait for postgres first β€” everything depends on it while [ $waited -lt $MAX_WAIT ]; do - if podman exec agent-router-pod-postgres-db pg_isready -U postgres -q 2>/dev/null; then + if podman exec ${POD_NAME}-postgres-db pg_isready -U postgres -p ${POSTGRES_PORT} -q 2>/dev/null; then echo " βœ“ PostgreSQL ready after ${waited}s" break fi @@ -475,7 +500,7 @@ verify_stack_health() { local litellm_ready=false waited=0 while [ $waited -lt $MAX_WAIT ]; do - if curl -sf --max-time 3 http://127.0.0.1:4000/health/readiness >/dev/null 2>&1; then + if curl -sf --max-time 3 http://127.0.0.1:${LITELLM_PORT}/health/readiness >/dev/null 2>&1; then echo " βœ“ LiteLLM ready after ${waited}s" litellm_ready=true break @@ -490,7 +515,7 @@ verify_stack_health() { # Wait for triage router + verify full pipeline waited=0 while [ $waited -lt 120 ]; do - local resp=$(curl -s --max-time 10 http://127.0.0.1:5000/v1/chat/completions \ + local resp=$(curl -s --max-time 10 http://127.0.0.1:${ROUTER_PORT}/v1/chat/completions \ -H 'Content-Type: application/json' \ -d '{"model":"agent-simple-core","messages":[{"role":"user","content":"Hi"}],"max_tokens":5}' 2>/dev/null) if echo "$resp" | grep -q '"choices"'; then @@ -508,16 +533,16 @@ verify_stack_health() { # Graceful stop (SIGTERM with 30s timeout) lets ClickHouse/Postgres flush, # then force-remove if needed. Avoids data corruption from SIGKILL. safe_pod_teardown() { - if podman pod exists agent-router-pod 2>/dev/null; then + if podman pod exists ${POD_NAME} 2>/dev/null; then echo "πŸ›‘ Gracefully stopping pod (SIGTERM, 30s timeout)..." - podman pod stop -t 30 agent-router-pod 2>/dev/null || true + podman pod stop -t 30 ${POD_NAME} 2>/dev/null || true # If still running after graceful attempt, force-remove - if podman pod exists agent-router-pod 2>/dev/null; then + if podman pod exists ${POD_NAME} 2>/dev/null; then echo "⚠️ Graceful stop timed out β€” force-removing..." - podman pod rm -f agent-router-pod 2>/dev/null || true + podman pod rm -f ${POD_NAME} 2>/dev/null || true else # Already stopped, just remove - podman pod rm agent-router-pod 2>/dev/null || true + podman pod rm ${POD_NAME} 2>/dev/null || true fi cleanup_zombie_ports echo "βœ“ Pod torn down, ports cleaned" @@ -526,13 +551,43 @@ safe_pod_teardown() { # Pre-deploy database backup (runs before any pod modification) # Skip if pod doesn't exist (e.g., after manual cleanup) -if podman pod exists agent-router-pod 2>/dev/null; then +if podman pod exists ${POD_NAME} 2>/dev/null; then echo "πŸ’Ύ Taking pre-deploy database backup..." bash scripts/backup.sh && echo "βœ“ Pre-deploy backup saved" || echo "⚠️ Pre-deploy backup skipped" fi +# ── ClickHouse port override XML ── +# Writes a minimal config.d XML override so ClickHouse listens on the +# configured ports instead of its compiled-in defaults. +generate_clickhouse_config() { + local config_dir="${DATA_ROOT}/clickhouse-config" + mkdir -p "$config_dir" + cat > "${config_dir}/port-override.xml" << EOF + + ${CLICKHOUSE_HTTP_PORT} + ${CLICKHOUSE_TCP_PORT} + ${CLICKHOUSE_INTERSERVER_PORT} + +EOF + echo "βœ“ ClickHouse port config written to ${config_dir}/port-override.xml" +} + +# ── LiteLLM rendered config ── +# Generates a rendered config.yaml (with port substitutions) into DATA_ROOT/litellm-rendered/ +# so prod and dev each get their own copy with the correct port values. +render_litellm_config() { + local rendered_dir="${DATA_ROOT}/litellm-rendered" + mkdir -p "$rendered_dir" + sed -e "s/VALKEY_CACHE_PORT_PLACEHOLDER/${VALKEY_CACHE_PORT}/g" \ + -e "s/ROUTER_PORT_PLACEHOLDER/${ROUTER_PORT}/g" \ + "${WORKDIR}/litellm/config.yaml" > "${rendered_dir}/config.yaml" + # Copy entrypoint.py unchanged + cp "${WORKDIR}/litellm/entrypoint.py" "${rendered_dir}/entrypoint.py" + echo "βœ“ LiteLLM config rendered to ${rendered_dir}/config.yaml" +} + render_pod_yaml() { - export WORKDIR HOME LITELLM_MASTER_KEY POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL ROUTING_DOMAIN LITELLM_PUBLIC_URL LANGFUSE_PUBLIC_URL LITELLM_ROOT_PATH + export WORKDIR HOME LITELLM_MASTER_KEY POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT python3 - "$WORKDIR/pod.yaml" <<'PY' import os, sys, urllib.parse, json uid = os.getuid() @@ -563,8 +618,19 @@ placeholders = [ "PROXY_BASE_URL_PLACEHOLDER", "PUBLIC_BASE_URL_PLACEHOLDER", "ROUTING_DOMAIN_PLACEHOLDER", - "NEXTAUTH_URL_PLACEHOLDER", - "SERVER_ROOT_PATH_PLACEHOLDER" + "POD_NAME_PLACEHOLDER", + "DATA_ROOT_PLACEHOLDER", + "ROUTER_PORT_PLACEHOLDER", + "LITELLM_PORT_PLACEHOLDER", + "LANGFUSE_WEB_PORT_PLACEHOLDER", + "LANGFUSE_WORKER_PORT_PLACEHOLDER", + "POSTGRES_PORT_PLACEHOLDER", + "VALKEY_CACHE_PORT_PLACEHOLDER", + "VALKEY_LF_PORT_PLACEHOLDER", + "CLICKHOUSE_HTTP_PORT_PLACEHOLDER", + "CLICKHOUSE_TCP_PORT_PLACEHOLDER", + "MINIO_S3_PORT_PLACEHOLDER", + "MINIO_CONSOLE_PORT_PLACEHOLDER", ] for ph in placeholders: if ph not in text: @@ -589,11 +655,30 @@ text = text.replace("MINIO_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["MINIO_ text = text.replace("LANGFUSE_INIT_USER_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["LANGFUSE_INIT_USER_PASSWORD"])) text = text.replace("REDIS_AUTH_PLACEHOLDER", yaml_scalar(os.environ["REDIS_AUTH"])) text = text.replace("CLICKHOUSE_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["CLICKHOUSE_PASSWORD"])) -text = text.replace("PROXY_BASE_URL_PLACEHOLDER", yaml_scalar(os.environ["LITELLM_PUBLIC_URL"])) -text = text.replace("NEXTAUTH_URL_PLACEHOLDER", yaml_scalar(os.environ["LANGFUSE_PUBLIC_URL"])) -text = text.replace("SERVER_ROOT_PATH_PLACEHOLDER", yaml_scalar(os.environ["LITELLM_ROOT_PATH"])) +# Derive PROXY_BASE_URL from PUBLIC_BASE_URL +public_base_url = os.environ["PUBLIC_BASE_URL"].rstrip("/") +proxy_base_url = f"{public_base_url}/litellm" +text = text.replace("PROXY_BASE_URL_PLACEHOLDER", yaml_scalar(proxy_base_url)) text = text.replace("PUBLIC_BASE_URL_PLACEHOLDER", yaml_scalar(os.environ["PUBLIC_BASE_URL"])) text = text.replace("ROUTING_DOMAIN_PLACEHOLDER", yaml_scalar(os.environ["ROUTING_DOMAIN"])) +# Raw replacements (no quoting β€” used for pod name, data root, and integer port values) +raw_replacements = { + "POD_NAME_PLACEHOLDER": os.environ["POD_NAME"], + "DATA_ROOT_PLACEHOLDER": os.environ["DATA_ROOT"], + "ROUTER_PORT_PLACEHOLDER": os.environ["ROUTER_PORT"], + "LITELLM_PORT_PLACEHOLDER": os.environ["LITELLM_PORT"], + "LANGFUSE_WEB_PORT_PLACEHOLDER": os.environ["LANGFUSE_WEB_PORT"], + "LANGFUSE_WORKER_PORT_PLACEHOLDER": os.environ["LANGFUSE_WORKER_PORT"], + "POSTGRES_PORT_PLACEHOLDER": os.environ["POSTGRES_PORT"], + "VALKEY_CACHE_PORT_PLACEHOLDER": os.environ["VALKEY_CACHE_PORT"], + "VALKEY_LF_PORT_PLACEHOLDER": os.environ["VALKEY_LF_PORT"], + "CLICKHOUSE_HTTP_PORT_PLACEHOLDER": os.environ["CLICKHOUSE_HTTP_PORT"], + "CLICKHOUSE_TCP_PORT_PLACEHOLDER": os.environ["CLICKHOUSE_TCP_PORT"], + "MINIO_S3_PORT_PLACEHOLDER": os.environ["MINIO_S3_PORT"], + "MINIO_CONSOLE_PORT_PLACEHOLDER": os.environ["MINIO_CONSOLE_PORT"], +} +for ph, val in raw_replacements.items(): + text = text.replace(ph, val) import re unresolved = sorted(set(re.findall(r"\b[A-Z0-9_]+_PLACEHOLDER\b", text))) if unresolved: @@ -607,24 +692,38 @@ sys.stdout.write(text) PY } -if podman pod exists agent-router-pod 2>/dev/null; then +if podman pod exists ${POD_NAME} 2>/dev/null; then if $FULL_REBUILD; then echo "πŸ”¨ Building custom local triage router image..." - podman build -t localhost/llm-triage-router:latest -f router/Dockerfile router + podman build -t ghcr.io/sheepdestroyer/llm-routing:latest -f router/Dockerfile router safe_pod_teardown echo "πŸš€ Deploying fresh triage pod..." + generate_clickhouse_config + render_litellm_config + render_pod_yaml | podman play kube - + setup_minio_buckets + verify_stack_health + elif $PULL_MODE; then + echo "🚚 Pulling latest triage router image from GHCR..." + podman pull ghcr.io/sheepdestroyer/llm-routing:latest + safe_pod_teardown + echo "πŸš€ Deploying fresh triage pod with pulled image..." + generate_clickhouse_config + render_litellm_config render_pod_yaml | podman play kube - setup_minio_buckets verify_stack_health elif $REPLACE_MODE; then safe_pod_teardown echo "πŸš€ Deploying replacement pod from YAML..." + generate_clickhouse_config + render_litellm_config render_pod_yaml | podman play kube - setup_minio_buckets verify_stack_health else - echo "πŸ”„ Restarting existing agent-router-pod (use --replace or --full-rebuild to recreate)..." - podman pod restart agent-router-pod + echo "πŸ”„ Restarting existing ${POD_NAME} (use --replace or --pull to recreate)..." + podman pod restart ${POD_NAME} setup_minio_buckets verify_stack_health @@ -643,10 +742,17 @@ if podman pod exists agent-router-pod 2>/dev/null; then else # First deploy β€” no pod exists, clean ports just in case cleanup_zombie_ports - echo "πŸ”¨ Building custom local triage router image..." - podman build -t localhost/llm-triage-router:latest -f router/Dockerfile router + if $FULL_REBUILD; then + echo "πŸ”¨ Building custom local triage router image..." + podman build -t ghcr.io/sheepdestroyer/llm-routing:latest -f router/Dockerfile router + else + echo "🚚 Pulling latest triage router image from GHCR..." + podman pull ghcr.io/sheepdestroyer/llm-routing:latest + fi echo "πŸš€ No existing pod found. Deploying fresh triage pod..." + generate_clickhouse_config + render_litellm_config render_pod_yaml | podman play kube - setup_minio_buckets verify_stack_health From 9e86358bcd7cf7f19350f7122f4d130afe1215d7 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 15:31:11 +0200 Subject: [PATCH 02/24] fix: default DATA_ROOT to WORKDIR/data instead of WORKDIR --- start-stack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/start-stack.sh b/start-stack.sh index 1e949ab9..6c25d109 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -94,7 +94,7 @@ CLICKHOUSE_TCP_PORT="${CLICKHOUSE_TCP_PORT:-9000}" CLICKHOUSE_INTERSERVER_PORT="${CLICKHOUSE_INTERSERVER_PORT:-9009}" MINIO_S3_PORT="${MINIO_S3_PORT:-9002}" MINIO_CONSOLE_PORT="${MINIO_CONSOLE_PORT:-9001}" -DATA_ROOT="${DATA_ROOT:-${WORKDIR}}" +DATA_ROOT="${DATA_ROOT:-${WORKDIR}/data}" export POD_NAME ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT DATA_ROOT # Define and export the routing domain From 38d03b42195bee189ff78cc1f4d110bc733a4e64 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 15:31:25 +0200 Subject: [PATCH 03/24] fix: move dataset mount under DATA_ROOT as datasets/ --- pod.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pod.yaml b/pod.yaml index c9cc70a7..829ad1b5 100644 --- a/pod.yaml +++ b/pod.yaml @@ -507,7 +507,7 @@ spec: path: WORKDIR_PLACEHOLDER name: env-file - hostPath: - path: WORKDIR_PLACEHOLDER/data + path: DATA_ROOT_PLACEHOLDER/datasets type: DirectoryOrCreate name: dataset-data - hostPath: From bce02a67a04df850b1a55ab1bcab206a4c990950 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 15:31:36 +0200 Subject: [PATCH 04/24] =?UTF-8?q?fix:=20remove=20dev-data=20workaround=20?= =?UTF-8?q?=E2=80=94=20DATA=5FROOT=20now=20defaults=20to=20data/?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- start-dev.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/start-dev.sh b/start-dev.sh index 82c254e3..602d9746 100755 --- a/start-dev.sh +++ b/start-dev.sh @@ -5,10 +5,5 @@ WORKDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd "$WORKDIR" export DEV_ENV_FILE="${WORKDIR}/.env.dev" -export DATA_ROOT="${WORKDIR}/dev-data" - -mkdir -p dev-data/valkey-data dev-data/postgres-data dev-data/clickhouse-data \ - dev-data/redis-lf-data dev-data/minio-data dev-data/clickhouse-config \ - dev-data/litellm-rendered exec bash start-stack.sh "$@" From 24a701212fc5eda425c78e5b9fcba1a5d8bcbf4e Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 15:31:57 +0200 Subject: [PATCH 05/24] =?UTF-8?q?fix:=20consolidate=20gitignore=20?= =?UTF-8?q?=E2=80=94=20all=20data=20dirs=20now=20under=20data/?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 34bee0fb..41daa843 100644 --- a/.gitignore +++ b/.gitignore @@ -4,14 +4,8 @@ # Exclude Podman runtime file generated dynamically at startup pod-run.yaml -# Exclude persistent database and cache storages -postgres-data/ -langfuse-data/ -valkey-data/ -clickhouse-data/ -minio-data/ -redis-lf-data/ -dev-data/ +# Exclude persistent database and cache storages (all under data/) +data/ # Exclude auto-generated backups @@ -38,7 +32,5 @@ router/free_models_roster.json workdirs/ pr_description.txt -# Dataset work in progress -data/ .cache/ test_output*.log From 2fe95456b0709078068b49afffdca54055b6ad11 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 15:38:33 +0200 Subject: [PATCH 06/24] fix: router reads ports from env vars (LITELLM_PORT, VALKEY_CACHE_PORT, LANGFUSE_WEB_PORT, ROUTER_PORT) --- router/main.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/router/main.py b/router/main.py index 7b8df8d9..97a47d84 100644 --- a/router/main.py +++ b/router/main.py @@ -23,9 +23,9 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator, RootModel from typing import Dict, Optional, Union -LITELLM_URL = (os.getenv("LITELLM_ADMIN_URL") or "http://127.0.0.1:4000").rstrip("/") +LITELLM_URL = (os.getenv("LITELLM_ADMIN_URL") or f"http://127.0.0.1:{os.getenv('LITELLM_PORT', '4000')}").rstrip("/") LLAMA_SERVER_URL = (os.getenv("LLAMA_SERVER_URL") or "http://127.0.0.1:8080").rstrip("/") -LANGFUSE_HOST = (os.getenv("LANGFUSE_HOST") or "http://127.0.0.1:3001").rstrip("/") +LANGFUSE_HOST = (os.getenv("LANGFUSE_HOST") or f"http://127.0.0.1:{os.getenv('LANGFUSE_WEB_PORT', '3001')}").rstrip("/") GEMINI_OAUTH_CREDS_PATH = "/config/gemini_auth/oauth_creds.json" @@ -57,7 +57,7 @@ def get_redis(): logger.info("Valkey client initialized from URL") else: host = os.getenv("VALKEY_HOST", "127.0.0.1") - port = int(os.getenv("VALKEY_PORT", "6379")) + port = int(os.getenv("VALKEY_CACHE_PORT") or os.getenv("VALKEY_PORT", "6379")) _redis_client = aioredis.Redis(host=host, port=port, decode_responses=True, socket_timeout=1.0) logger.info(f"Valkey client initialized at {host}:{port}") except Exception as e: @@ -1590,7 +1590,7 @@ def get_pie_chart_gradient() -> str: @app.api_route("/v1/memory{path:path}", methods=["GET", "POST", "DELETE", "PUT"]) async def proxy_memory(request: Request, path: str = ""): """Proxies memory API calls to the LiteLLM gateway on port 4000.""" - litellm_base = "http://127.0.0.1:4000/v1/memory" + litellm_base = f"http://127.0.0.1:{os.getenv('LITELLM_PORT', '4000')}/v1/memory" # Resolve the destination URL url = f"{litellm_base}{path}" @@ -2666,10 +2666,10 @@ async def get_dashboard_data(): llamacpp, ) = await asyncio.gather( asyncio.wait_for(sync_cooldowns_from_valkey(), timeout=2.0), - check_tcp_port("127.0.0.1", 6379), - check_http_endpoint("http://127.0.0.1:4000/"), + check_tcp_port("127.0.0.1", int(os.getenv("VALKEY_CACHE_PORT") or os.getenv("VALKEY_PORT", "6379"))), + check_http_endpoint(f"http://127.0.0.1:{os.getenv('LITELLM_PORT', '4000')}/"), check_http_endpoint("http://127.0.0.1:8080/health"), - check_http_endpoint("http://127.0.0.1:3001"), + check_http_endpoint(f"http://127.0.0.1:{os.getenv('LANGFUSE_WEB_PORT', '3001')}"), get_gemini_oauth_status(), asyncio.wait_for(get_best_free_model(), timeout=5.0), asyncio.to_thread(get_goose_sessions), @@ -3867,7 +3867,7 @@ async def get_dashboard(request: Request):
Triage Router - :5000 + :{os.getenv('ROUTER_PORT', '5000')}
Online
@@ -3875,7 +3875,7 @@ async def get_dashboard(request: Request):
LiteLLM Proxy - :4000 + :{os.getenv('LITELLM_PORT', '4000')}
{"Online" if litellm_status else "Offline"} @@ -3885,7 +3885,7 @@ async def get_dashboard(request: Request):
Valkey Cache - :6379 + :{os.getenv('VALKEY_CACHE_PORT') or os.getenv('VALKEY_PORT', '6379')}
{"Online" if valkey_status else "Offline"} From fff4592941815f9796fba6d607c95eb10942afe0 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 15:48:45 +0200 Subject: [PATCH 07/24] fix: router image configurable via ROUTER_IMAGE env var; router reads ports from env --- .env.dev | 3 +++ pod.yaml | 2 +- start-stack.sh | 7 +++++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.env.dev b/.env.dev index 7cc97b63..a5393f15 100644 --- a/.env.dev +++ b/.env.dev @@ -22,3 +22,6 @@ CLICKHOUSE_TCP_PORT="9010" CLICKHOUSE_INTERSERVER_PORT="9019" MINIO_S3_PORT="9012" MINIO_CONSOLE_PORT="9011" + +# Dev uses locally-built router image (for testing code changes) +ROUTER_IMAGE="localhost/llm-routing-dev:latest" diff --git a/pod.yaml b/pod.yaml index 829ad1b5..0f476b06 100644 --- a/pod.yaml +++ b/pod.yaml @@ -124,7 +124,7 @@ spec: value: 'VALKEY_CACHE_PORT_PLACEHOLDER' - name: LOG_LEVEL value: info - image: ghcr.io/sheepdestroyer/llm-routing:latest + image: ROUTER_IMAGE_PLACEHOLDER livenessProbe: exec: command: diff --git a/start-stack.sh b/start-stack.sh index 6c25d109..27057c22 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -94,8 +94,9 @@ CLICKHOUSE_TCP_PORT="${CLICKHOUSE_TCP_PORT:-9000}" CLICKHOUSE_INTERSERVER_PORT="${CLICKHOUSE_INTERSERVER_PORT:-9009}" MINIO_S3_PORT="${MINIO_S3_PORT:-9002}" MINIO_CONSOLE_PORT="${MINIO_CONSOLE_PORT:-9001}" +ROUTER_IMAGE="${ROUTER_IMAGE:-ghcr.io/sheepdestroyer/llm-routing:latest}" DATA_ROOT="${DATA_ROOT:-${WORKDIR}/data}" -export POD_NAME ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT DATA_ROOT +export POD_NAME ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT ROUTER_IMAGE DATA_ROOT # Define and export the routing domain ROUTING_DOMAIN="${ROUTING_DOMAIN:-vendeuvre.lan}" @@ -587,7 +588,7 @@ render_litellm_config() { } render_pod_yaml() { - export WORKDIR HOME LITELLM_MASTER_KEY POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT + export WORKDIR HOME LITELLM_MASTER_KEY POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT python3 - "$WORKDIR/pod.yaml" <<'PY' import os, sys, urllib.parse, json uid = os.getuid() @@ -631,6 +632,7 @@ placeholders = [ "CLICKHOUSE_TCP_PORT_PLACEHOLDER", "MINIO_S3_PORT_PLACEHOLDER", "MINIO_CONSOLE_PORT_PLACEHOLDER", + "ROUTER_IMAGE_PLACEHOLDER", ] for ph in placeholders: if ph not in text: @@ -676,6 +678,7 @@ raw_replacements = { "CLICKHOUSE_TCP_PORT_PLACEHOLDER": os.environ["CLICKHOUSE_TCP_PORT"], "MINIO_S3_PORT_PLACEHOLDER": os.environ["MINIO_S3_PORT"], "MINIO_CONSOLE_PORT_PLACEHOLDER": os.environ["MINIO_CONSOLE_PORT"], + "ROUTER_IMAGE_PLACEHOLDER": os.environ["ROUTER_IMAGE"], } for ph, val in raw_replacements.items(): text = text.replace(ph, val) From 02182ff9edbb4e6851514aff232ac65549597ebf Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 15:55:39 +0200 Subject: [PATCH 08/24] fix: render router config.yaml with LITELLM_PORT placeholder at deploy time --- pod.yaml | 3 ++- router/config.yaml | 16 ++++++++-------- start-stack.sh | 15 +++++++++++++++ 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/pod.yaml b/pod.yaml index 0f476b06..f381d5e0 100644 --- a/pod.yaml +++ b/pod.yaml @@ -471,7 +471,8 @@ spec: type: DirectoryOrCreate name: litellm-rendered - hostPath: - path: WORKDIR_PLACEHOLDER/router + path: DATA_ROOT_PLACEHOLDER/router-rendered + type: DirectoryOrCreate name: router-config - hostPath: path: DATA_ROOT_PLACEHOLDER/postgres-data diff --git a/router/config.yaml b/router/config.yaml index 80b42939..a30ed2ad 100644 --- a/router/config.yaml +++ b/router/config.yaml @@ -34,26 +34,26 @@ classification_rules: # The api_key placeholder is resolved at runtime by main.py:788. backends: - name: "agent-simple-core" - api_base: "http://127.0.0.1:4000/v1" + api_base: "http://127.0.0.1:LITELLM_PORT_PLACEHOLDER/v1" api_key: "DYNAMIC_LITELLM_MASTER_KEY_PLACEHOLDER" - name: "agent-medium-core" - api_base: "http://127.0.0.1:4000/v1" + api_base: "http://127.0.0.1:LITELLM_PORT_PLACEHOLDER/v1" api_key: "DYNAMIC_LITELLM_MASTER_KEY_PLACEHOLDER" - name: "agent-complex-core" - api_base: "http://127.0.0.1:4000/v1" + api_base: "http://127.0.0.1:LITELLM_PORT_PLACEHOLDER/v1" api_key: "DYNAMIC_LITELLM_MASTER_KEY_PLACEHOLDER" - name: "agent-reasoning-core" - api_base: "http://127.0.0.1:4000/v1" + api_base: "http://127.0.0.1:LITELLM_PORT_PLACEHOLDER/v1" api_key: "DYNAMIC_LITELLM_MASTER_KEY_PLACEHOLDER" - name: "agent-advanced-core" - api_base: "http://127.0.0.1:4000/v1" + api_base: "http://127.0.0.1:LITELLM_PORT_PLACEHOLDER/v1" api_key: "DYNAMIC_LITELLM_MASTER_KEY_PLACEHOLDER" - name: "ollama-deepseek-v4-pro" - api_base: "http://127.0.0.1:4000/v1" + api_base: "http://127.0.0.1:LITELLM_PORT_PLACEHOLDER/v1" api_key: "DYNAMIC_LITELLM_MASTER_KEY_PLACEHOLDER" - name: "ollama-deepseek-v4-flash" - api_base: "http://127.0.0.1:4000/v1" + api_base: "http://127.0.0.1:LITELLM_PORT_PLACEHOLDER/v1" api_key: "DYNAMIC_LITELLM_MASTER_KEY_PLACEHOLDER" - name: "openrouter-auto" - api_base: "http://127.0.0.1:4000/v1" + api_base: "http://127.0.0.1:LITELLM_PORT_PLACEHOLDER/v1" api_key: "DYNAMIC_LITELLM_MASTER_KEY_PLACEHOLDER" diff --git a/start-stack.sh b/start-stack.sh index 27057c22..c4c680b8 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -587,6 +587,17 @@ render_litellm_config() { echo "βœ“ LiteLLM config rendered to ${rendered_dir}/config.yaml" } +# ── Router rendered config ── +# Generates a rendered config.yaml (with port substitutions) into DATA_ROOT/router-rendered/ +# so prod and dev each get their own copy with the correct LiteLLM port. +render_router_config() { + local rendered_dir="${DATA_ROOT}/router-rendered" + mkdir -p "$rendered_dir" + sed -e "s/LITELLM_PORT_PLACEHOLDER/${LITELLM_PORT}/g" \ + "${WORKDIR}/router/config.yaml" > "${rendered_dir}/config.yaml" + echo "βœ“ Router config rendered to ${rendered_dir}/config.yaml" +} + render_pod_yaml() { export WORKDIR HOME LITELLM_MASTER_KEY POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL ROUTING_DOMAIN POD_NAME DATA_ROOT ROUTER_IMAGE ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT python3 - "$WORKDIR/pod.yaml" <<'PY' @@ -703,6 +714,7 @@ if podman pod exists ${POD_NAME} 2>/dev/null; then echo "πŸš€ Deploying fresh triage pod..." generate_clickhouse_config render_litellm_config + render_router_config render_pod_yaml | podman play kube - setup_minio_buckets verify_stack_health @@ -713,6 +725,7 @@ if podman pod exists ${POD_NAME} 2>/dev/null; then echo "πŸš€ Deploying fresh triage pod with pulled image..." generate_clickhouse_config render_litellm_config + render_router_config render_pod_yaml | podman play kube - setup_minio_buckets verify_stack_health @@ -721,6 +734,7 @@ if podman pod exists ${POD_NAME} 2>/dev/null; then echo "πŸš€ Deploying replacement pod from YAML..." generate_clickhouse_config render_litellm_config + render_router_config render_pod_yaml | podman play kube - setup_minio_buckets verify_stack_health @@ -756,6 +770,7 @@ else echo "πŸš€ No existing pod found. Deploying fresh triage pod..." generate_clickhouse_config render_litellm_config + render_router_config render_pod_yaml | podman play kube - setup_minio_buckets verify_stack_health From f4befbb33119e6cc91a713841e39db25cc54fb6e Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 17:02:26 +0200 Subject: [PATCH 09/24] fix: replace tcpSocket probes with valkey-cli exec probes, switch to non-alpine valkey image Root cause: podman play kube converts tcpSocket liveness probes into 'nc -z -v localhost || exit 1' healthchecks. Alpine's BusyBox nc does not support -z, so the healthcheck always fails. With HealthcheckOnFailureAction=restart, this causes an infinite SIGTERM restart loop (~every 30s). Changes: - valkey-cache: tcpSocket -> exec valkey-cli ping (liveness + readiness) - valkey-lf: tcpSocket -> exec valkey-cli -a ping (liveness) - Image: valkey:9.1.0-alpine -> valkey:9.1.0 (non-alpine has no nc, so no broken built-in healthcheck to conflict with) Verified in dev: 0 SIGTERMs, 0 ECONNREFUSED, both valkey instances healthy, Langfuse web/worker stable, chat completions 200 OK. --- pod.yaml | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/pod.yaml b/pod.yaml index f381d5e0..0420e5d2 100644 --- a/pod.yaml +++ b/pod.yaml @@ -13,17 +13,21 @@ spec: - 'VALKEY_CACHE_PORT_PLACEHOLDER' - --loglevel - warning - image: docker.io/valkey/valkey:9.1.0-alpine + image: docker.io/valkey/valkey:9.1.0 livenessProbe: - tcpSocket: - port: VALKEY_CACHE_PORT_PLACEHOLDER + exec: + command: + - valkey-cli + - ping initialDelaySeconds: 5 periodSeconds: 10 timeoutSeconds: 2 name: valkey-cache readinessProbe: - tcpSocket: - port: VALKEY_CACHE_PORT_PLACEHOLDER + exec: + command: + - valkey-cli + - ping initialDelaySeconds: 2 periodSeconds: 5 timeoutSeconds: 2 @@ -272,10 +276,16 @@ spec: - noeviction - --loglevel - warning - image: docker.io/valkey/valkey:9.1.0-alpine + image: docker.io/valkey/valkey:9.1.0 livenessProbe: - tcpSocket: - port: VALKEY_LF_PORT_PLACEHOLDER + exec: + command: + - valkey-cli + - -p + - "VALKEY_LF_PORT_PLACEHOLDER" + - -a + - REDIS_AUTH_PLACEHOLDER + - ping initialDelaySeconds: 5 periodSeconds: 10 timeoutSeconds: 2 From 5f4c87efb05705080d295b259fbd0b9deb098e80 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 17:39:47 +0200 Subject: [PATCH 10/24] fix: add -p VALKEY_CACHE_PORT_PLACEHOLDER to valkey-cache probes Sourcery review: valkey-cli ping without -p targets default 6379, causing false negatives when VALKEY_CACHE_PORT is customized (e.g. dev=6389). Now matches the port-aware pattern already used by valkey-lf probes. --- pod.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pod.yaml b/pod.yaml index 0420e5d2..c871aabf 100644 --- a/pod.yaml +++ b/pod.yaml @@ -18,6 +18,8 @@ spec: exec: command: - valkey-cli + - -p + - 'VALKEY_CACHE_PORT_PLACEHOLDER' - ping initialDelaySeconds: 5 periodSeconds: 10 @@ -27,6 +29,8 @@ spec: exec: command: - valkey-cli + - -p + - 'VALKEY_CACHE_PORT_PLACEHOLDER' - ping initialDelaySeconds: 2 periodSeconds: 5 From 1afa7c1af184a9e1d88a4d134ad24ba005ceb5c4 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 17:53:37 +0200 Subject: [PATCH 11/24] fix: address PR #248 and #249 review comments - litellm/entrypoint.py: restore quote stripping before setdefault (prevents literal '"5432"' from breaking int() parsing) - start-stack.sh: use ROUTER_IMAGE var instead of hardcoded ghcr.io/sheepdestroyer/llm-routing:latest in build/pull paths - start-stack.sh: create volume dirs under DATA_ROOT instead of WORKDIR (fixes pod creation when DATA_ROOT != WORKDIR) - start-stack.sh: add placeholder validation to render_litellm_config and chmod 644 on rendered configs for non-root container access - scripts/backup.sh: use POD_NAME from .env instead of hardcoded agent-router-pod-postgres-db (fixes dev backup failures) - router/main.py: extract _valkey_port() helper to DRY the VALKEY_CACHE_PORT/VALKEY_PORT fallback (3 call sites) - router/main.py: replace hardcoded :3001 with LANGFUSE_WEB_PORT env var in dashboard Infrastructure Nodes display --- litellm/entrypoint.py | 1 + router/main.py | 12 ++++++++---- scripts/backup.sh | 12 +++++++++--- start-stack.sh | 22 +++++++++++++++------- 4 files changed, 33 insertions(+), 14 deletions(-) diff --git a/litellm/entrypoint.py b/litellm/entrypoint.py index 85dca12b..b8755d44 100644 --- a/litellm/entrypoint.py +++ b/litellm/entrypoint.py @@ -16,6 +16,7 @@ line = line.strip() if line and not line.startswith("#") and "=" in line: key, _, val = line.partition("=") + val = val.strip().strip('"').strip("'") os.environ.setdefault(key, val) # Load Gemini OAuth token from credentials JSON diff --git a/router/main.py b/router/main.py index 97a47d84..6ca2e7c3 100644 --- a/router/main.py +++ b/router/main.py @@ -41,6 +41,10 @@ _redis_last_init_attempt = 0.0 _REDIS_RETRY_INTERVAL_SECONDS = 5.0 +def _valkey_port() -> str: + """Resolve the Valkey cache port from env, preferring VALKEY_CACHE_PORT.""" + return os.getenv("VALKEY_CACHE_PORT") or os.getenv("VALKEY_PORT", "6379") + def get_redis(): """Lazily initialize and return the async Redis/Valkey client. Returns None if connection fails or is disabled (non-fatal fallback).""" @@ -57,7 +61,7 @@ def get_redis(): logger.info("Valkey client initialized from URL") else: host = os.getenv("VALKEY_HOST", "127.0.0.1") - port = int(os.getenv("VALKEY_CACHE_PORT") or os.getenv("VALKEY_PORT", "6379")) + port = int(_valkey_port()) _redis_client = aioredis.Redis(host=host, port=port, decode_responses=True, socket_timeout=1.0) logger.info(f"Valkey client initialized at {host}:{port}") except Exception as e: @@ -2666,7 +2670,7 @@ async def get_dashboard_data(): llamacpp, ) = await asyncio.gather( asyncio.wait_for(sync_cooldowns_from_valkey(), timeout=2.0), - check_tcp_port("127.0.0.1", int(os.getenv("VALKEY_CACHE_PORT") or os.getenv("VALKEY_PORT", "6379"))), + check_tcp_port("127.0.0.1", int(_valkey_port())), check_http_endpoint(f"http://127.0.0.1:{os.getenv('LITELLM_PORT', '4000')}/"), check_http_endpoint("http://127.0.0.1:8080/health"), check_http_endpoint(f"http://127.0.0.1:{os.getenv('LANGFUSE_WEB_PORT', '3001')}"), @@ -3885,7 +3889,7 @@ async def get_dashboard(request: Request):
Valkey Cache - :{os.getenv('VALKEY_CACHE_PORT') or os.getenv('VALKEY_PORT', '6379')} + :{_valkey_port()}
{"Online" if valkey_status else "Offline"} @@ -3905,7 +3909,7 @@ async def get_dashboard(request: Request):
Langfuse Traces - :3001 + :{os.getenv('LANGFUSE_WEB_PORT', '3001')}
{"Online" if langfuse_status else "Offline"} diff --git a/scripts/backup.sh b/scripts/backup.sh index 2000ddb7..4a4159e1 100755 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -13,6 +13,12 @@ BACKUP_DIR="${WORKDIR}/backups" RETENTION_DAYS=14 LOG_FILE="/tmp/llm-backup-${TIMESTAMP}.log" +# Source .env for POD_NAME (with prod default) +if [ -f "${WORKDIR}/.env" ]; then + set -a; source "${WORKDIR}/.env"; set +a +fi +POD_NAME="${POD_NAME:-agent-router-pod}" + mkdir -p "$BACKUP_DIR" log() { echo "[$(date '+%H:%M:%S')] $*" | tee -a "$LOG_FILE"; } @@ -21,12 +27,12 @@ log() { echo "[$(date '+%H:%M:%S')] $*" | tee -a "$LOG_FILE"; } log "⏳ Checking PostgreSQL readiness..." PG_READY=0 # Check if container exists AND is running before looping -PG_RUNNING=$(podman inspect --format '{{.State.Running}}' agent-router-pod-postgres-db 2>/dev/null || echo "false") +PG_RUNNING=$(podman inspect --format '{{.State.Running}}' ${POD_NAME}-postgres-db 2>/dev/null || echo "false") if [ "$PG_RUNNING" != "true" ]; then log "⚠️ PostgreSQL container not running β€” skipping DB backup" else for i in {1..15}; do - if podman exec agent-router-pod-postgres-db pg_isready -U postgres 2>/dev/null; then + if podman exec ${POD_NAME}-postgres-db pg_isready -U postgres 2>/dev/null; then PG_READY=1 log "βœ… PostgreSQL is ready" break @@ -40,7 +46,7 @@ fi if [ $PG_READY -eq 1 ]; then for db in postgres langfuse; do FILE="${BACKUP_DIR}/${db}_db_${TIMESTAMP}.dump" - if podman exec agent-router-pod-postgres-db \ + if podman exec ${POD_NAME}-postgres-db \ pg_dump -U postgres -d "$db" -F c > "$FILE"; then log "βœ… ${db} db: $(ls -lh "$FILE" | awk '{print $5}')" else diff --git a/start-stack.sh b/start-stack.sh index c4c680b8..be86e4a5 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -37,9 +37,6 @@ if [ $# -gt 1 ]; then exit 1 fi -# Ensure local volume directories exist on the host for Podman mounts -mkdir -p valkey-data postgres-data langfuse-data clickhouse-data redis-lf-data minio-data - PULL_MODE=false FULL_REBUILD=false REPLACE_MODE=false @@ -98,6 +95,9 @@ ROUTER_IMAGE="${ROUTER_IMAGE:-ghcr.io/sheepdestroyer/llm-routing:latest}" DATA_ROOT="${DATA_ROOT:-${WORKDIR}/data}" export POD_NAME ROUTER_PORT LITELLM_PORT LANGFUSE_WEB_PORT LANGFUSE_WORKER_PORT POSTGRES_PORT VALKEY_CACHE_PORT VALKEY_LF_PORT CLICKHOUSE_HTTP_PORT CLICKHOUSE_TCP_PORT CLICKHOUSE_INTERSERVER_PORT MINIO_S3_PORT MINIO_CONSOLE_PORT ROUTER_IMAGE DATA_ROOT +# Ensure local volume directories exist on the host for Podman mounts +mkdir -p "${DATA_ROOT}/valkey-data" "${DATA_ROOT}/postgres-data" "${DATA_ROOT}/langfuse-data" "${DATA_ROOT}/clickhouse-data" "${DATA_ROOT}/redis-lf-data" "${DATA_ROOT}/minio-data" + # Define and export the routing domain ROUTING_DOMAIN="${ROUTING_DOMAIN:-vendeuvre.lan}" export ROUTING_DOMAIN @@ -582,8 +582,15 @@ render_litellm_config() { sed -e "s/VALKEY_CACHE_PORT_PLACEHOLDER/${VALKEY_CACHE_PORT}/g" \ -e "s/ROUTER_PORT_PLACEHOLDER/${ROUTER_PORT}/g" \ "${WORKDIR}/litellm/config.yaml" > "${rendered_dir}/config.yaml" + # Validate no unresolved placeholders remain + if grep -q 'VALKEY_CACHE_PORT_PLACEHOLDER\|ROUTER_PORT_PLACEHOLDER' "${rendered_dir}/config.yaml"; then + echo "❌ Error: Unresolved placeholders remain in ${rendered_dir}/config.yaml" >&2 + exit 1 + fi + chmod 644 "${rendered_dir}/config.yaml" # Copy entrypoint.py unchanged cp "${WORKDIR}/litellm/entrypoint.py" "${rendered_dir}/entrypoint.py" + chmod 644 "${rendered_dir}/entrypoint.py" echo "βœ“ LiteLLM config rendered to ${rendered_dir}/config.yaml" } @@ -595,6 +602,7 @@ render_router_config() { mkdir -p "$rendered_dir" sed -e "s/LITELLM_PORT_PLACEHOLDER/${LITELLM_PORT}/g" \ "${WORKDIR}/router/config.yaml" > "${rendered_dir}/config.yaml" + chmod 644 "${rendered_dir}/config.yaml" echo "βœ“ Router config rendered to ${rendered_dir}/config.yaml" } @@ -709,7 +717,7 @@ PY if podman pod exists ${POD_NAME} 2>/dev/null; then if $FULL_REBUILD; then echo "πŸ”¨ Building custom local triage router image..." - podman build -t ghcr.io/sheepdestroyer/llm-routing:latest -f router/Dockerfile router + podman build -t "${ROUTER_IMAGE}" -f router/Dockerfile router safe_pod_teardown echo "πŸš€ Deploying fresh triage pod..." generate_clickhouse_config @@ -720,7 +728,7 @@ if podman pod exists ${POD_NAME} 2>/dev/null; then verify_stack_health elif $PULL_MODE; then echo "🚚 Pulling latest triage router image from GHCR..." - podman pull ghcr.io/sheepdestroyer/llm-routing:latest + podman pull "${ROUTER_IMAGE}" safe_pod_teardown echo "πŸš€ Deploying fresh triage pod with pulled image..." generate_clickhouse_config @@ -761,10 +769,10 @@ else cleanup_zombie_ports if $FULL_REBUILD; then echo "πŸ”¨ Building custom local triage router image..." - podman build -t ghcr.io/sheepdestroyer/llm-routing:latest -f router/Dockerfile router + podman build -t "${ROUTER_IMAGE}" -f router/Dockerfile router else echo "🚚 Pulling latest triage router image from GHCR..." - podman pull ghcr.io/sheepdestroyer/llm-routing:latest + podman pull "${ROUTER_IMAGE}" fi echo "πŸš€ No existing pod found. Deploying fresh triage pod..." From fdfd49fd9fd828bbb4a92ea968e26ce172499daf Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Sat, 11 Jul 2026 21:31:15 +0200 Subject: [PATCH 12/24] Update litellm/entrypoint.py Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- litellm/entrypoint.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/litellm/entrypoint.py b/litellm/entrypoint.py index b8755d44..77f3e471 100644 --- a/litellm/entrypoint.py +++ b/litellm/entrypoint.py @@ -46,7 +46,12 @@ def check_tcp_port(ip: str, port: int) -> bool: return False max_wait = 60 -postgres_port = int(os.environ.get("POSTGRES_PORT", "5432")) +postgres_port_str = os.environ.get("POSTGRES_PORT", "5432") +try: + postgres_port = int(postgres_port_str) +except ValueError: + print(f"⚠️ Invalid POSTGRES_PORT '{postgres_port_str}', defaulting to 5432") + postgres_port = 5432 print(f"πŸ”Œ Waiting for PostgreSQL on :{postgres_port} (max {max_wait}s)...") for i in range(max_wait): if check_tcp_port("127.0.0.1", postgres_port): From 45e1e50e9ecd95145868d20ee20a4cb6193016b3 Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Sat, 11 Jul 2026 21:31:26 +0200 Subject: [PATCH 13/24] Update router/main.py Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- router/main.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/router/main.py b/router/main.py index 6ca2e7c3..2959f7b5 100644 --- a/router/main.py +++ b/router/main.py @@ -41,9 +41,14 @@ _redis_last_init_attempt = 0.0 _REDIS_RETRY_INTERVAL_SECONDS = 5.0 -def _valkey_port() -> str: +def _valkey_port() -> int: """Resolve the Valkey cache port from env, preferring VALKEY_CACHE_PORT.""" - return os.getenv("VALKEY_CACHE_PORT") or os.getenv("VALKEY_PORT", "6379") + port_str = os.getenv("VALKEY_CACHE_PORT") or os.getenv("VALKEY_PORT", "6379") + try: + return int(port_str) + except ValueError: + logger.warning(f"Invalid Valkey port '{port_str}', defaulting to 6379") + return 6379 def get_redis(): """Lazily initialize and return the async Redis/Valkey client. From 4865655e76e76506b8cc691b73b2f38df313a41e Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Sat, 11 Jul 2026 21:31:36 +0200 Subject: [PATCH 14/24] Update router/main.py Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- router/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router/main.py b/router/main.py index 2959f7b5..419fda6e 100644 --- a/router/main.py +++ b/router/main.py @@ -2675,7 +2675,7 @@ async def get_dashboard_data(): llamacpp, ) = await asyncio.gather( asyncio.wait_for(sync_cooldowns_from_valkey(), timeout=2.0), - check_tcp_port("127.0.0.1", int(_valkey_port())), + check_tcp_port("127.0.0.1", _valkey_port()), check_http_endpoint(f"http://127.0.0.1:{os.getenv('LITELLM_PORT', '4000')}/"), check_http_endpoint("http://127.0.0.1:8080/health"), check_http_endpoint(f"http://127.0.0.1:{os.getenv('LANGFUSE_WEB_PORT', '3001')}"), From 8e13aff8fb64c9c274385f0fc4838d01c69d11f6 Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Sat, 11 Jul 2026 21:31:52 +0200 Subject: [PATCH 15/24] Update router/main.py Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- router/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router/main.py b/router/main.py index 419fda6e..045bff5a 100644 --- a/router/main.py +++ b/router/main.py @@ -66,7 +66,7 @@ def get_redis(): logger.info("Valkey client initialized from URL") else: host = os.getenv("VALKEY_HOST", "127.0.0.1") - port = int(_valkey_port()) + port = _valkey_port() _redis_client = aioredis.Redis(host=host, port=port, decode_responses=True, socket_timeout=1.0) logger.info(f"Valkey client initialized at {host}:{port}") except Exception as e: From 93a839d50865564431ca25642f14e636ad4eba50 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 21:38:57 +0200 Subject: [PATCH 16/24] fix: address PR #250 review comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - scripts/backup.sh: wire POSTGRES_PORT through pg_isready -p flag (fixes backup failures when DB port is customized, e.g. dev=5442) - pod.yaml: remove redundant PORT env var (LITELLM_PORT already set) - start-stack.sh: remove undocumented --pull-latest alias - start-stack.sh: add placeholder validation to render_router_config (matches existing render_litellm_config guard) - start-stack.sh: extract deploy_fresh_pod() helper to DRY the repeated generateβ†’renderβ†’deployβ†’setupβ†’verify sequence (4 call sites) - router/main.py: use LLAMA_SERVER_URL for health check instead of hardcoded http://127.0.0.1:8080/health --- pod.yaml | 2 -- router/main.py | 2 +- scripts/backup.sh | 5 +++-- start-stack.sh | 44 +++++++++++++++++++------------------------- 4 files changed, 23 insertions(+), 30 deletions(-) diff --git a/pod.yaml b/pod.yaml index c871aabf..9e165adf 100644 --- a/pod.yaml +++ b/pod.yaml @@ -59,8 +59,6 @@ spec: value: /llm-routing/litellm - name: PROXY_BASE_URL value: PROXY_BASE_URL_PLACEHOLDER - - name: PORT - value: 'LITELLM_PORT_PLACEHOLDER' - name: LITELLM_PORT value: 'LITELLM_PORT_PLACEHOLDER' - name: POSTGRES_PORT diff --git a/router/main.py b/router/main.py index 045bff5a..cbf85e50 100644 --- a/router/main.py +++ b/router/main.py @@ -2677,7 +2677,7 @@ async def get_dashboard_data(): asyncio.wait_for(sync_cooldowns_from_valkey(), timeout=2.0), check_tcp_port("127.0.0.1", _valkey_port()), check_http_endpoint(f"http://127.0.0.1:{os.getenv('LITELLM_PORT', '4000')}/"), - check_http_endpoint("http://127.0.0.1:8080/health"), + check_http_endpoint(f"{LLAMA_SERVER_URL}/health"), check_http_endpoint(f"http://127.0.0.1:{os.getenv('LANGFUSE_WEB_PORT', '3001')}"), get_gemini_oauth_status(), asyncio.wait_for(get_best_free_model(), timeout=5.0), diff --git a/scripts/backup.sh b/scripts/backup.sh index 4a4159e1..4b6ebe96 100755 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -13,11 +13,12 @@ BACKUP_DIR="${WORKDIR}/backups" RETENTION_DAYS=14 LOG_FILE="/tmp/llm-backup-${TIMESTAMP}.log" -# Source .env for POD_NAME (with prod default) +# Source .env for POD_NAME and POSTGRES_PORT (with prod defaults) if [ -f "${WORKDIR}/.env" ]; then set -a; source "${WORKDIR}/.env"; set +a fi POD_NAME="${POD_NAME:-agent-router-pod}" +POSTGRES_PORT="${POSTGRES_PORT:-5432}" mkdir -p "$BACKUP_DIR" @@ -32,7 +33,7 @@ if [ "$PG_RUNNING" != "true" ]; then log "⚠️ PostgreSQL container not running β€” skipping DB backup" else for i in {1..15}; do - if podman exec ${POD_NAME}-postgres-db pg_isready -U postgres 2>/dev/null; then + if podman exec ${POD_NAME}-postgres-db pg_isready -U postgres -p ${POSTGRES_PORT} 2>/dev/null; then PG_READY=1 log "βœ… PostgreSQL is ready" break diff --git a/start-stack.sh b/start-stack.sh index be86e4a5..32e0174b 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -40,7 +40,7 @@ fi PULL_MODE=false FULL_REBUILD=false REPLACE_MODE=false -if [ "${1:-}" = "--pull" ] || [ "${1:-}" = "--pull-latest" ]; then +if [ "${1:-}" = "--pull" ]; then PULL_MODE=true elif [ "${1:-}" = "--full-rebuild" ]; then FULL_REBUILD=true @@ -602,6 +602,11 @@ render_router_config() { mkdir -p "$rendered_dir" sed -e "s/LITELLM_PORT_PLACEHOLDER/${LITELLM_PORT}/g" \ "${WORKDIR}/router/config.yaml" > "${rendered_dir}/config.yaml" + # Validate no unresolved placeholders remain + if grep -q 'LITELLM_PORT_PLACEHOLDER' "${rendered_dir}/config.yaml"; then + echo "❌ Error: Unresolved placeholders remain in ${rendered_dir}/config.yaml" >&2 + exit 1 + fi chmod 644 "${rendered_dir}/config.yaml" echo "βœ“ Router config rendered to ${rendered_dir}/config.yaml" } @@ -714,38 +719,32 @@ sys.stdout.write(text) PY } +deploy_fresh_pod() { + generate_clickhouse_config + render_litellm_config + render_router_config + render_pod_yaml | podman play kube - + setup_minio_buckets + verify_stack_health +} + if podman pod exists ${POD_NAME} 2>/dev/null; then if $FULL_REBUILD; then echo "πŸ”¨ Building custom local triage router image..." podman build -t "${ROUTER_IMAGE}" -f router/Dockerfile router safe_pod_teardown echo "πŸš€ Deploying fresh triage pod..." - generate_clickhouse_config - render_litellm_config - render_router_config - render_pod_yaml | podman play kube - - setup_minio_buckets - verify_stack_health + deploy_fresh_pod elif $PULL_MODE; then echo "🚚 Pulling latest triage router image from GHCR..." podman pull "${ROUTER_IMAGE}" safe_pod_teardown echo "πŸš€ Deploying fresh triage pod with pulled image..." - generate_clickhouse_config - render_litellm_config - render_router_config - render_pod_yaml | podman play kube - - setup_minio_buckets - verify_stack_health + deploy_fresh_pod elif $REPLACE_MODE; then safe_pod_teardown echo "πŸš€ Deploying replacement pod from YAML..." - generate_clickhouse_config - render_litellm_config - render_router_config - render_pod_yaml | podman play kube - - setup_minio_buckets - verify_stack_health + deploy_fresh_pod else echo "πŸ”„ Restarting existing ${POD_NAME} (use --replace or --pull to recreate)..." podman pod restart ${POD_NAME} @@ -776,12 +775,7 @@ else fi echo "πŸš€ No existing pod found. Deploying fresh triage pod..." - generate_clickhouse_config - render_litellm_config - render_router_config - render_pod_yaml | podman play kube - - setup_minio_buckets - verify_stack_health + deploy_fresh_pod fi From 69a3b8451ad1f20de178ff4e8a02a45025c55c88 Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Sat, 11 Jul 2026 21:42:30 +0200 Subject: [PATCH 17/24] Update start-stack.sh Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- start-stack.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/start-stack.sh b/start-stack.sh index 32e0174b..34d15404 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -769,10 +769,16 @@ else if $FULL_REBUILD; then echo "πŸ”¨ Building custom local triage router image..." podman build -t "${ROUTER_IMAGE}" -f router/Dockerfile router + if [[ "$ROUTER_IMAGE" == localhost/* ]]; then + if ! podman image exists "${ROUTER_IMAGE}"; then + echo "πŸ”¨ Local image not found. Building custom local triage router image..." + podman build -t "${ROUTER_IMAGE}" -f router/Dockerfile router + fi else echo "🚚 Pulling latest triage router image from GHCR..." podman pull "${ROUTER_IMAGE}" fi + fi echo "πŸš€ No existing pod found. Deploying fresh triage pod..." deploy_fresh_pod From 2e82b4a3eed52b670616e718a0b7979267504137 Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Sat, 11 Jul 2026 21:43:08 +0200 Subject: [PATCH 18/24] Update scripts/backup.sh Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- scripts/backup.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/backup.sh b/scripts/backup.sh index 4b6ebe96..5f873cb8 100755 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -17,6 +17,9 @@ LOG_FILE="/tmp/llm-backup-${TIMESTAMP}.log" if [ -f "${WORKDIR}/.env" ]; then set -a; source "${WORKDIR}/.env"; set +a fi +if [ -n "${DEV_ENV_FILE:-}" ] && [ -f "$DEV_ENV_FILE" ]; then + set -a; source "$DEV_ENV_FILE"; set +a +fi POD_NAME="${POD_NAME:-agent-router-pod}" POSTGRES_PORT="${POSTGRES_PORT:-5432}" From a84638bb5fe271a9369faeb37589e4829d31c632 Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Sat, 11 Jul 2026 21:43:17 +0200 Subject: [PATCH 19/24] Update start-stack.sh Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- start-stack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/start-stack.sh b/start-stack.sh index 34d15404..755ba500 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -583,7 +583,7 @@ render_litellm_config() { -e "s/ROUTER_PORT_PLACEHOLDER/${ROUTER_PORT}/g" \ "${WORKDIR}/litellm/config.yaml" > "${rendered_dir}/config.yaml" # Validate no unresolved placeholders remain - if grep -q 'VALKEY_CACHE_PORT_PLACEHOLDER\|ROUTER_PORT_PLACEHOLDER' "${rendered_dir}/config.yaml"; then + if grep -E -q 'VALKEY_CACHE_PORT_PLACEHOLDER|ROUTER_PORT_PLACEHOLDER' "${rendered_dir}/config.yaml"; then echo "❌ Error: Unresolved placeholders remain in ${rendered_dir}/config.yaml" >&2 exit 1 fi From a1f3a2d2c093c36fb21704b84e3cb06d658e0b06 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 21:48:47 +0200 Subject: [PATCH 20/24] fix: repair CI test and start-stack.sh logic from direct-applied commits - router/tests/test_get_redis.py: _valkey_port() now safely catches ValueError and returns 6379, so the old test that expected int('invalid') to crash inside get_redis() no longer applies. Split into two tests: test_valkey_port_invalid_fallback (new) and a rewritten test_get_redis_initialization_failure that uses aioredis.Redis side_effect instead. - start-stack.sh: fix broken if/elif/else chain in first-deploy path (direct-applied commit had 'if' instead of 'elif', causing the localhost check and pull branch to run inside FULL_REBUILD mode, plus an extra dangling 'fi') --- router/tests/test_get_redis.py | 20 ++++++++++++++++---- start-stack.sh | 3 +-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/router/tests/test_get_redis.py b/router/tests/test_get_redis.py index 10070ed5..92105d44 100644 --- a/router/tests/test_get_redis.py +++ b/router/tests/test_get_redis.py @@ -63,22 +63,34 @@ def test_get_redis_initialization_success(mock_redis, mock_monotonic): @patch("router.main.time.monotonic") @patch("router.main.logger.warning") @patch.dict(os.environ, {"VALKEY_HOST": "my-host", "VALKEY_PORT": "invalid"}) -def test_get_redis_initialization_failure(mock_logger_warning, mock_monotonic): - """If initialization fails, it should catch the exception, log a warning, and return None.""" +def test_valkey_port_invalid_fallback(mock_logger_warning, mock_monotonic): + """_valkey_port() should log a warning and fall back to 6379 on invalid port.""" + port = main._valkey_port() + assert port == 6379 + mock_logger_warning.assert_called_once() + assert "Invalid Valkey port" in mock_logger_warning.call_args[0][0] + +@patch("router.main.time.monotonic") +@patch("router.main.aioredis.Redis") +@patch("router.main.logger.warning") +@patch.dict(os.environ, {"VALKEY_HOST": "my-host", "VALKEY_PORT": "1234"}) +def test_get_redis_initialization_failure(mock_logger_warning, mock_redis, mock_monotonic): + """If aioredis.Redis throws, it should catch the exception, log a warning, and return None.""" main._redis_client = None main._redis_last_init_attempt = 100.0 # Time elapsed is 10.0 seconds mock_monotonic.return_value = 110.0 - # The int() conversion of VALKEY_PORT will raise ValueError + mock_redis.side_effect = Exception("Connection refused") + client = main.get_redis() assert client is None assert main._redis_client is None assert main._redis_last_init_attempt == 110.0 mock_logger_warning.assert_called_once() - assert "Failed to initialize Valkey client" in mock_logger_warning.call_args[0][0] + assert "Connection refused" in mock_logger_warning.call_args[0][0] @patch("router.main.time.monotonic") @patch("router.main.aioredis.Redis") diff --git a/start-stack.sh b/start-stack.sh index 755ba500..c8033628 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -769,7 +769,7 @@ else if $FULL_REBUILD; then echo "πŸ”¨ Building custom local triage router image..." podman build -t "${ROUTER_IMAGE}" -f router/Dockerfile router - if [[ "$ROUTER_IMAGE" == localhost/* ]]; then + elif [[ "$ROUTER_IMAGE" == localhost/* ]]; then if ! podman image exists "${ROUTER_IMAGE}"; then echo "πŸ”¨ Local image not found. Building custom local triage router image..." podman build -t "${ROUTER_IMAGE}" -f router/Dockerfile router @@ -778,7 +778,6 @@ else echo "🚚 Pulling latest triage router image from GHCR..." podman pull "${ROUTER_IMAGE}" fi - fi echo "πŸš€ No existing pod found. Deploying fresh triage pod..." deploy_fresh_pod From bbc56ee30c926ff0c1ea0ee6f9348c4f91c9e8a2 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 21:52:32 +0200 Subject: [PATCH 21/24] chore: add .venv/ to .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 41daa843..1d88c0a7 100644 --- a/.gitignore +++ b/.gitignore @@ -32,5 +32,8 @@ router/free_models_roster.json workdirs/ pr_description.txt +# Python virtual environment +.venv/ + .cache/ test_output*.log From 008b15bdd8d462451aba2cf1c2f1aa790869fabd Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Sat, 11 Jul 2026 21:58:23 +0200 Subject: [PATCH 22/24] Update litellm/entrypoint.py Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- litellm/entrypoint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/entrypoint.py b/litellm/entrypoint.py index 77f3e471..21622749 100644 --- a/litellm/entrypoint.py +++ b/litellm/entrypoint.py @@ -133,6 +133,6 @@ def _serialize_dt(dt): # Start LiteLLM Proxy import litellm from litellm.proxy.proxy_cli import run_server -litellm_port = os.environ.get("LITELLM_PORT", os.environ.get("PORT", "4000")) +litellm_port = os.environ.get("LITELLM_PORT") or os.environ.get("PORT") or "4000" sys.argv = ["litellm", "--config", "/app/config.yaml", "--port", litellm_port] run_server() From dab85018cb29275ff0100cfc5c94d53a7c06c169 Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 22:06:10 +0200 Subject: [PATCH 23/24] =?UTF-8?q?fix:=20address=20PR#252=20review=20commen?= =?UTF-8?q?ts=20=E2=80=94=20safe=5Fpod=5Fteardown,=20.env=20key=20strip,?= =?UTF-8?q?=20empty-string=20port=20handling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - safe_pod_teardown: use podman pod inspect --format '{{.State}}' instead of podman pod exists to distinguish running vs stopped pods (Gemini review) - litellm/entrypoint.py: strip key after line.partition('=') to handle spaces around '=' in .env files (Gemini review) - router/main.py + litellm/entrypoint.py: replace os.getenv('PORT', 'default') with os.getenv('PORT') or 'default' to handle empty-string env vars correctly, preventing broken URLs like http://127.0.0.1: (Gemini review) - README.md: update health check table β€” valkey probes now use exec valkey-cli ping instead of tcpSocket, postgres uses pg_isready -p - Replied to false positives: LiteLLM /ping is a valid endpoint; the or chain in entrypoint.py correctly handles empty LITELLM_PORT --- README.md | 6 +++--- litellm/entrypoint.py | 3 ++- router/main.py | 16 ++++++++-------- start-stack.sh | 4 ++-- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 20262cd8..351e6568 100644 --- a/README.md +++ b/README.md @@ -76,12 +76,12 @@ All core containers are configured with **Kubernetes-style liveness and readines | Container | Liveness Probe | Readiness Probe | |:---|---:|---:| -| **valkey-cache** | `tcpSocket` on port 6379 every 10s | Same, every 5s | +| **valkey-cache** | `valkey-cli -p ping` every 10s | Same, every 5s | | **litellm-gateway** | Python `urllib` GET `/ping` (port 4000) every 15s | Python `urllib` GET `/health/readiness` (port 4000) every 10s | | **llm-triage-router** | Python `urllib` GET `/metrics` (port 5000) every 15s | Same, every 10s | -| **postgres-db** | `pg_isready -U postgres` every 10s | Same, every 5s | +| **postgres-db** | `pg_isready -U postgres -p ` every 10s | Same, every 5s | | **clickhouse-db** | `clickhouse-client --user clickhouse --password --query "SELECT 1"` every 15s | `clickhouse-client --user clickhouse --password --query "SELECT 1"` every 10s | -| **valkey-lf** | `tcpSocket` on port 6380 every 10s | Same, every 5s | +| **valkey-lf** | `valkey-cli -p -a ping` every 10s | Same, every 5s | | **langfuse-web** | `wget` GET `/api/health` (port 3001) every 15s | Same, every 10s | | **langfuse-worker** | `pgrep node` every 15s | β€” | | **minio-s3** | `httpGet` `/minio/health/live` (port 9002) every 15s | `httpGet` `/minio/health/ready` (port 9002) every 10s | diff --git a/litellm/entrypoint.py b/litellm/entrypoint.py index 21622749..898ec971 100644 --- a/litellm/entrypoint.py +++ b/litellm/entrypoint.py @@ -16,6 +16,7 @@ line = line.strip() if line and not line.startswith("#") and "=" in line: key, _, val = line.partition("=") + key = key.strip() val = val.strip().strip('"').strip("'") os.environ.setdefault(key, val) @@ -46,7 +47,7 @@ def check_tcp_port(ip: str, port: int) -> bool: return False max_wait = 60 -postgres_port_str = os.environ.get("POSTGRES_PORT", "5432") +postgres_port_str = os.environ.get("POSTGRES_PORT") or "5432" try: postgres_port = int(postgres_port_str) except ValueError: diff --git a/router/main.py b/router/main.py index cbf85e50..0c976f4f 100644 --- a/router/main.py +++ b/router/main.py @@ -23,9 +23,9 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator, RootModel from typing import Dict, Optional, Union -LITELLM_URL = (os.getenv("LITELLM_ADMIN_URL") or f"http://127.0.0.1:{os.getenv('LITELLM_PORT', '4000')}").rstrip("/") +LITELLM_URL = (os.getenv("LITELLM_ADMIN_URL") or f"http://127.0.0.1:{os.getenv('LITELLM_PORT') or '4000'}").rstrip("/") LLAMA_SERVER_URL = (os.getenv("LLAMA_SERVER_URL") or "http://127.0.0.1:8080").rstrip("/") -LANGFUSE_HOST = (os.getenv("LANGFUSE_HOST") or f"http://127.0.0.1:{os.getenv('LANGFUSE_WEB_PORT', '3001')}").rstrip("/") +LANGFUSE_HOST = (os.getenv("LANGFUSE_HOST") or f"http://127.0.0.1:{os.getenv('LANGFUSE_WEB_PORT') or '3001'}").rstrip("/") GEMINI_OAUTH_CREDS_PATH = "/config/gemini_auth/oauth_creds.json" @@ -1599,7 +1599,7 @@ def get_pie_chart_gradient() -> str: @app.api_route("/v1/memory{path:path}", methods=["GET", "POST", "DELETE", "PUT"]) async def proxy_memory(request: Request, path: str = ""): """Proxies memory API calls to the LiteLLM gateway on port 4000.""" - litellm_base = f"http://127.0.0.1:{os.getenv('LITELLM_PORT', '4000')}/v1/memory" + litellm_base = f"http://127.0.0.1:{os.getenv('LITELLM_PORT') or '4000'}/v1/memory" # Resolve the destination URL url = f"{litellm_base}{path}" @@ -2676,9 +2676,9 @@ async def get_dashboard_data(): ) = await asyncio.gather( asyncio.wait_for(sync_cooldowns_from_valkey(), timeout=2.0), check_tcp_port("127.0.0.1", _valkey_port()), - check_http_endpoint(f"http://127.0.0.1:{os.getenv('LITELLM_PORT', '4000')}/"), + check_http_endpoint(f"http://127.0.0.1:{os.getenv('LITELLM_PORT') or '4000'}/"), check_http_endpoint(f"{LLAMA_SERVER_URL}/health"), - check_http_endpoint(f"http://127.0.0.1:{os.getenv('LANGFUSE_WEB_PORT', '3001')}"), + check_http_endpoint(f"http://127.0.0.1:{os.getenv('LANGFUSE_WEB_PORT') or '3001'}"), get_gemini_oauth_status(), asyncio.wait_for(get_best_free_model(), timeout=5.0), asyncio.to_thread(get_goose_sessions), @@ -3876,7 +3876,7 @@ async def get_dashboard(request: Request):
Triage Router - :{os.getenv('ROUTER_PORT', '5000')} + :{os.getenv('ROUTER_PORT') or '5000'}
Online
@@ -3884,7 +3884,7 @@ async def get_dashboard(request: Request):
LiteLLM Proxy - :{os.getenv('LITELLM_PORT', '4000')} + :{os.getenv('LITELLM_PORT') or '4000'}
{"Online" if litellm_status else "Offline"} @@ -3914,7 +3914,7 @@ async def get_dashboard(request: Request):
Langfuse Traces - :{os.getenv('LANGFUSE_WEB_PORT', '3001')} + :{os.getenv('LANGFUSE_WEB_PORT') or '3001'}
{"Online" if langfuse_status else "Offline"} diff --git a/start-stack.sh b/start-stack.sh index c8033628..f3399914 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -537,8 +537,8 @@ safe_pod_teardown() { if podman pod exists ${POD_NAME} 2>/dev/null; then echo "πŸ›‘ Gracefully stopping pod (SIGTERM, 30s timeout)..." podman pod stop -t 30 ${POD_NAME} 2>/dev/null || true - # If still running after graceful attempt, force-remove - if podman pod exists ${POD_NAME} 2>/dev/null; then + # podman pod exists returns 0 for stopped pods too β€” check running state + if podman pod inspect ${POD_NAME} --format '{{.State}}' 2>/dev/null | grep -q 'Running'; then echo "⚠️ Graceful stop timed out β€” force-removing..." podman pod rm -f ${POD_NAME} 2>/dev/null || true else From cdd31143125640392b1d602fe6ec993c13e8705f Mon Sep 17 00:00:00 2001 From: boy Date: Sat, 11 Jul 2026 22:12:05 +0200 Subject: [PATCH 24/24] fix: add ROUTER_PORT and LANGFUSE_WEB_PORT env vars to router container The router uses these for dashboard display and health checks. Without them, the dashboard always shows default ports even when custom ports are configured via .env.dev. Reported by Gemini Code Assist review on PR#253. --- pod.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pod.yaml b/pod.yaml index 9e165adf..4214663e 100644 --- a/pod.yaml +++ b/pod.yaml @@ -128,6 +128,10 @@ spec: value: 'LITELLM_PORT_PLACEHOLDER' - name: VALKEY_CACHE_PORT value: 'VALKEY_CACHE_PORT_PLACEHOLDER' + - name: ROUTER_PORT + value: 'ROUTER_PORT_PLACEHOLDER' + - name: LANGFUSE_WEB_PORT + value: 'LANGFUSE_WEB_PORT_PLACEHOLDER' - name: LOG_LEVEL value: info image: ROUTER_IMAGE_PLACEHOLDER