diff --git a/.agents/AGENTS.md b/.agents/AGENTS.md index 16a25bb5..33379171 100644 --- a/.agents/AGENTS.md +++ b/.agents/AGENTS.md @@ -10,9 +10,9 @@ When working on this project, always refer to the dedicated **NotebookLM Compani - Local model benchmark metrics and `llama-server` configurations ### Notebook Details -- **Notebook Name:** `TriageGate-Architect-KB` +- **Notebook Name:** `LLM-Routing-KB` - **Notebook ID:** `llm-triage-gateway` -- **Notebook URL:** [TriageGate-Architect-KB](https://notebooklm.google.com/notebook/826cbd87-7969-4b0e-a38e-5517b5ab7d28) +- **Notebook URL:** [LLM-Routing-KB](https://notebooklm.google.com/notebook/826cbd87-7969-4b0e-a38e-5517b5ab7d28) ### How to Query Use the `notebooklm` MCP tools to search or ask questions about this codebase and stack: @@ -26,3 +26,59 @@ To prevent directory reorganization regressions, outdated file restorations, or 2. **Directory Rename Safety**: If Git reports conflicts related to moved directories or files, do not manually stage deletions of tracked files from moved directories (e.g., under the old `tests/` or `scripts/` paths) or re-create files at the root level. Resolve conflicts by directing all changes and file operations to the newly refactored paths. 3. **Verify Security Credentials**: Never accept resolutions that overwrite configuration files (`pod.yaml`, `start-stack.sh`) with hardcoded default passwords. Ensure placeholder-based configurations are preserved. 4. **Enforce Test Suite Count**: Run the full unit test suite (`pytest`) after conflict resolution. Verify that the total number of passing tests is equal to or greater than before the resolution. + +## Production Deployment Checklist (boy user) + +### One-Time Host Prerequisites (already configured on x570.vendeuvre.lan) +- `net.ipv4.ip_unprivileged_port_start=80` persisted in `/etc/sysctl.d/99-unprivileged-ports.conf` +- Host firewall ports `80/tcp` and `443/tcp` opened in `firewalld` (e.g. `sudo firewall-cmd --zone=public --add-port=80/tcp --permanent && sudo firewall-cmd --zone=public --add-port=443/tcp --permanent && sudo firewall-cmd --reload`) +- `boy` SSH host alias in `~/.ssh/config` — use `ssh boy` / `rsync ... boy:` throughout +- Required mount directories created under `boy`'s home: + - `/mnt/DATA/boy/.gemini/` + - `/mnt/DATA/boy/.local/bin/agy` (copy of the `agy` binary) + - `/mnt/DATA/boy/.local/share/goose/` + - `/mnt/DATA/boy/.local/share/keyrings/` +- HAProxy SSL cert: `/mnt/DATA/boy/haproxy/certs/vendeuvre.pem` +- HAProxy config: `/mnt/DATA/boy/haproxy/haproxy.cfg` + +### Fresh Deploy Steps (after a PR is merged to master) +```bash +# 1. Clean up old deploy on boy +ssh boy "rm -rf /mnt/DATA/boy/LLM-Routing" + +# 2. Clone fresh from master +ssh boy "git clone https://github.com/sheepdestroyer/LLM-Routing.git /mnt/DATA/boy/LLM-Routing" + +# 3. Start the full stack (builds and launches all containers) +ssh boy "cd /mnt/DATA/boy/LLM-Routing && ./start-stack.sh --full-rebuild" + +# 4. Start (or restart) production HAProxy +ssh boy "podman rm -f production-haproxy || true" +ssh boy "podman run -d --name production-haproxy --restart always --net host \ + -v /mnt/DATA/boy/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro \ + -v /mnt/DATA/boy/haproxy/certs:/usr/local/etc/haproxy/certs:ro \ + docker.io/library/haproxy:alpine" + +# 5. Start the host-side agy daemon +ssh boy "pkill -f host_agy_daemon.py || true" +ssh boy "nohup python3 /mnt/DATA/boy/LLM-Routing/scripts/host_agy_daemon.py >/tmp/agy-daemon.log 2>&1

Per-model usage, token consumption & cost are tracked with full trace detail in Langfuse.

- Open Langfuse Observability → + Open Langfuse Observability →
@@ -3846,15 +3885,15 @@ async def get_dashboard():
- + {src_badge("LANGFUSE", "#e879f9")} Observability UI - + {src_badge("LITELLM", "#34d399")} Admin UI - + {src_badge("LLAMA.CPP", "#fb923c")} Server Router UI diff --git a/start-stack.sh b/start-stack.sh index 7a1a7bac..42b151c0 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -511,7 +511,8 @@ if podman pod exists agent-router-pod 2>/dev/null; then fi render_pod_yaml() { - export WORKDIR HOME LITELLM_MASTER_KEY POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD + PUBLIC_BASE_URL="${PUBLIC_BASE_URL:-${BASE_URL:-${BASEURL:-https://x570.vendeuvre.lan/llm-routing}}}" + export WORKDIR HOME LITELLM_MASTER_KEY POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL python3 - "$WORKDIR/pod.yaml" <<'PY' import os, sys, urllib.parse, json uid = os.getuid() @@ -538,7 +539,8 @@ placeholders = [ "MINIO_PASSWORD_PLACEHOLDER", "LANGFUSE_INIT_USER_PASSWORD_PLACEHOLDER", "REDIS_AUTH_PLACEHOLDER", - "CLICKHOUSE_PASSWORD_PLACEHOLDER" + "CLICKHOUSE_PASSWORD_PLACEHOLDER", + "PROXY_BASE_URL_PLACEHOLDER" ] for ph in placeholders: if ph not in text: @@ -563,6 +565,10 @@ text = text.replace("MINIO_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["MINIO_ text = text.replace("LANGFUSE_INIT_USER_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["LANGFUSE_INIT_USER_PASSWORD"])) text = text.replace("REDIS_AUTH_PLACEHOLDER", yaml_scalar(os.environ["REDIS_AUTH"])) text = text.replace("CLICKHOUSE_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["CLICKHOUSE_PASSWORD"])) +# Derive PROXY_BASE_URL from PUBLIC_BASE_URL +public_base_url = os.environ["PUBLIC_BASE_URL"].rstrip("/") +proxy_base_url = f"{public_base_url}/litellm" +text = text.replace("PROXY_BASE_URL_PLACEHOLDER", yaml_scalar(proxy_base_url)) import re unresolved = sorted(set(re.findall(r"\b[A-Z0-9_]+_PLACEHOLDER\b", text))) if unresolved: @@ -596,13 +602,18 @@ if podman pod exists agent-router-pod 2>/dev/null; then podman pod restart agent-router-pod setup_minio_buckets verify_stack_health + # Derive base URLs from configuration/env with sensible defaults + PUBLIC_BASE_URL="${PUBLIC_BASE_URL:-https://x570.vendeuvre.lan/llm-routing}" + LOCAL_BASE_URL="${LOCAL_BASE_URL:-http://localhost:5000}" + echo "" echo "=========================================================================" echo "🎉 SUCCESS: LLM Triage Gateway restarted!" - echo "📍 Entry endpoint : http://localhost:5000/v1" - echo "⚙️ Dashboard URL : http://localhost:5000/dashboard" + echo "📍 Entry endpoint : ${PUBLIC_BASE_URL}/v1" + echo " (local) : ${LOCAL_BASE_URL}/v1" + echo "⚙️ Dashboard URL : ${PUBLIC_BASE_URL}/dashboard" echo "🔑 Gateway API Key : gateway-pass" - echo "🔐 LiteLLM Admin UI: http://localhost:4000/ui" + echo "🔐 LiteLLM Admin UI: ${PUBLIC_BASE_URL}/litellm/ui" echo " Username: admin | Password: $LITELLM_MASTER_KEY" echo "=========================================================================" exit 0 @@ -619,11 +630,16 @@ else verify_stack_health fi +# Derive base URLs from configuration/env with sensible defaults +PUBLIC_BASE_URL="${PUBLIC_BASE_URL:-https://x570.vendeuvre.lan/llm-routing}" +LOCAL_BASE_URL="${LOCAL_BASE_URL:-http://localhost:5000}" + echo "=========================================================================" echo "🎉 SUCCESS: LLM Triage Gateway successfully deployed!" -echo "📍 Entry endpoint : http://localhost:5000/v1" -echo "⚙️ Dashboard URL : http://localhost:5000/dashboard" +echo "📍 Entry endpoint : ${PUBLIC_BASE_URL}/v1" +echo " (local) : ${LOCAL_BASE_URL}/v1" +echo "⚙️ Dashboard URL : ${PUBLIC_BASE_URL}/dashboard" echo "🔑 Gateway API Key : gateway-pass" -echo "🔐 LiteLLM Admin UI: http://localhost:4000/ui" +echo "🔐 LiteLLM Admin UI: ${PUBLIC_BASE_URL}/litellm/ui" echo " Username: admin | Password: $LITELLM_MASTER_KEY" echo "========================================================================="