From 47ce9c524b830f1567304f97010fa01cdc51c3e7 Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Wed, 8 Jul 2026 21:04:15 +0200 Subject: [PATCH 01/20] docs: document firewall rules and sudo password trailing space precautions in AGENTS.md --- .agents/AGENTS.md | 50 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/.agents/AGENTS.md b/.agents/AGENTS.md index 16a25bb5..a3260413 100644 --- a/.agents/AGENTS.md +++ b/.agents/AGENTS.md @@ -26,3 +26,53 @@ To prevent directory reorganization regressions, outdated file restorations, or 2. **Directory Rename Safety**: If Git reports conflicts related to moved directories or files, do not manually stage deletions of tracked files from moved directories (e.g., under the old `tests/` or `scripts/` paths) or re-create files at the root level. Resolve conflicts by directing all changes and file operations to the newly refactored paths. 3. **Verify Security Credentials**: Never accept resolutions that overwrite configuration files (`pod.yaml`, `start-stack.sh`) with hardcoded default passwords. Ensure placeholder-based configurations are preserved. 4. **Enforce Test Suite Count**: Run the full unit test suite (`pytest`) after conflict resolution. Verify that the total number of passing tests is equal to or greater than before the resolution. + +## Production Deployment Checklist (boy user) + +### One-Time Host Prerequisites (already configured on x570.vendeuvre.lan) +- `net.ipv4.ip_unprivileged_port_start=80` persisted in `/etc/sysctl.d/99-unprivileged-ports.conf` +- Host firewall ports `80/tcp` and `443/tcp` opened in `firewalld` (e.g. `sudo firewall-cmd --zone=public --add-port=80/tcp --permanent && sudo firewall-cmd --zone=public --add-port=443/tcp --permanent && sudo firewall-cmd --reload`) +- `boy` SSH host alias in `~/.ssh/config` — use `ssh boy` / `rsync ... boy:` throughout +- Required mount directories created under `boy`'s home: + - `/mnt/DATA/boy/.gemini/` + - `/mnt/DATA/boy/.local/bin/agy` (copy of the `agy` binary) + - `/mnt/DATA/boy/.local/share/goose/` + - `/mnt/DATA/boy/.local/share/keyrings/` +- HAProxy SSL cert: `/mnt/DATA/boy/haproxy/certs/vendeuvre.pem` +- HAProxy config: `/mnt/DATA/boy/haproxy/haproxy.cfg` + +### Fresh Deploy Steps (after a PR is merged to master) +```bash +# 1. Clean up old deploy on boy +ssh boy "rm -rf /mnt/DATA/boy/LLM-Routing" + +# 2. Clone fresh from master +ssh boy "git clone https://github.com/sheepdestroyer/LLM-Routing.git /mnt/DATA/boy/LLM-Routing" + +# 3. Start the full stack (builds and launches all containers) +ssh boy "cd /mnt/DATA/boy/LLM-Routing && ./start-stack.sh --full-rebuild" + +# 4. Start (or restart) production HAProxy +ssh boy "podman rm -f production-haproxy || true" +ssh boy "podman run -d --name production-haproxy --restart always --net host \ + -v /mnt/DATA/boy/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro \ + -v /mnt/DATA/boy/haproxy/certs:/usr/local/etc/haproxy/certs:ro \ + docker.io/library/haproxy:alpine" + +# 5. Start the host-side agy daemon (runs as sheepdestroyer, not boy) +pkill -f host_agy_daemon.py || true +nohup python3 ~/LAB/IA/LLM-Routing/scripts/host_agy_daemon.py >/tmp/agy-daemon.log 2>&1 & + +# 6. Verify end-to-end +curl -k -s --resolve x570.vendeuvre.lan:443:127.0.0.1 \ + https://x570.vendeuvre.lan/llm-routing/dashboard | head -5 +``` + +### Notes +- The `agy-daemon.service` systemd unit cannot be reloaded via `systemctl --user` from + the agent terminal (DBus is not connected). Start the daemon manually with `nohup` as + shown above, or instruct the user to run it in their own session. +- **Sudo Password Precaution**: Always preserve exact bytes (including trailing spaces or newlines) when reading `/home/sheepdestroyer/.sudo_password` (e.g. `'sakamoto '`). Stripping whitespace will cause authentication to fail. +- `start-stack.sh` without `--full-rebuild` will do a fast pod restart (reuses images). + Use `--full-rebuild` after code changes or image updates. + From ee818d70daea4e01fd66675a3f337269d73bb429 Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Wed, 8 Jul 2026 21:10:58 +0200 Subject: [PATCH 02/20] feat: resolve console links using request host or BASEURL/BASE_URL environment variable --- router/main.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/router/main.py b/router/main.py index d8be6cfb..62fc6595 100644 --- a/router/main.py +++ b/router/main.py @@ -3070,8 +3070,16 @@ async def get_dashboard_stats(): @app.get("/dashboard", response_class=HTMLResponse) -async def get_dashboard(): +async def get_dashboard(request: Request): """Render the router main dashboard HTML showing system metrics, health checks, and recent token usage.""" + external_host = os.getenv("BASEURL") or os.getenv("BASE_URL") + if not external_host: + external_host = request.base_url.host or "localhost" + else: + if "://" in external_host: + from urllib.parse import urlparse + external_host = urlparse(external_host).hostname or "localhost" + data = await get_dashboard_data() # Unpack data for the f-string template @@ -3710,7 +3718,7 @@ async def get_dashboard():

Per-model usage, token consumption & cost are tracked with full trace detail in Langfuse.

- Open Langfuse Observability → + Open Langfuse Observability →
@@ -3846,15 +3854,15 @@ async def get_dashboard():
- + {src_badge("LANGFUSE", "#e879f9")} Observability UI → - + {src_badge("LITELLM", "#34d399")} Admin UI → - + {src_badge("LLAMA.CPP", "#fb923c")} Server Router UI → From f542bd0e8034c880def2dbeb2b8ee88858411e47 Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Wed, 8 Jul 2026 21:58:41 +0200 Subject: [PATCH 03/20] Update router/main.py Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- router/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router/main.py b/router/main.py index 62fc6595..d54ea6d8 100644 --- a/router/main.py +++ b/router/main.py @@ -3074,7 +3074,7 @@ async def get_dashboard(request: Request): """Render the router main dashboard HTML showing system metrics, health checks, and recent token usage.""" external_host = os.getenv("BASEURL") or os.getenv("BASE_URL") if not external_host: - external_host = request.base_url.host or "localhost" + external_host = request.base_url.hostname or "localhost" else: if "://" in external_host: from urllib.parse import urlparse From 3292da2f1d3025454385f008af71aeaef65ab719 Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Wed, 8 Jul 2026 22:00:30 +0200 Subject: [PATCH 04/20] docs: add GitHub CLI auth mapping notes to AGENTS.md --- .agents/AGENTS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.agents/AGENTS.md b/.agents/AGENTS.md index a3260413..a5ec8b7c 100644 --- a/.agents/AGENTS.md +++ b/.agents/AGENTS.md @@ -75,4 +75,5 @@ curl -k -s --resolve x570.vendeuvre.lan:443:127.0.0.1 \ - **Sudo Password Precaution**: Always preserve exact bytes (including trailing spaces or newlines) when reading `/home/sheepdestroyer/.sudo_password` (e.g. `'sakamoto '`). Stripping whitespace will cause authentication to fail. - `start-stack.sh` without `--full-rebuild` will do a fast pod restart (reuses images). Use `--full-rebuild` after code changes or image updates. +- **GitHub CLI Authentication**: If running `gh` commands fails with a 401 error, ensure that `GITHUB_TOKEN` is exported (e.g., mapped from `GITHUB_MCP_PAT` in `~/.bashrc` via `export GITHUB_TOKEN="$GITHUB_MCP_PAT"`). From 75f4851e80c5ffd74ae6c7f5a0e61700f7b7df2c Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Wed, 8 Jul 2026 22:10:20 +0200 Subject: [PATCH 05/20] feat: conditionally resolve console links to subdomains if on vendeuvre.lan network --- router/main.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/router/main.py b/router/main.py index d54ea6d8..1c9edbbf 100644 --- a/router/main.py +++ b/router/main.py @@ -3080,6 +3080,20 @@ async def get_dashboard(request: Request): from urllib.parse import urlparse external_host = urlparse(external_host).hostname or "localhost" + domain = "vendeuvre.lan" + if external_host and domain in external_host: + langfuse_url = f"https://langfuse.{domain}" + litellm_url = f"https://litellm.{domain}/ui" + llama_url = f"https://llama.{domain}" + elif domain in (request.base_url.hostname or ""): + langfuse_url = f"https://langfuse.{domain}" + litellm_url = f"https://litellm.{domain}/ui" + llama_url = f"https://llama.{domain}" + else: + langfuse_url = f"http://{external_host}:3001" + litellm_url = f"http://{external_host}:4000/ui" + llama_url = f"http://{external_host}:8080" + data = await get_dashboard_data() # Unpack data for the f-string template @@ -3718,7 +3732,7 @@ async def get_dashboard(request: Request):

Per-model usage, token consumption & cost are tracked with full trace detail in Langfuse.

- Open Langfuse Observability → + Open Langfuse Observability →
@@ -3854,15 +3868,15 @@ async def get_dashboard(request: Request):
- + {src_badge("LANGFUSE", "#e879f9")} Observability UI → - + {src_badge("LITELLM", "#34d399")} Admin UI → - + {src_badge("LLAMA.CPP", "#fb923c")} Server Router UI → From 675a59c1bfa634203f41bd5a4f7c6bad534867bc Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Wed, 8 Jul 2026 22:17:32 +0200 Subject: [PATCH 06/20] feat: subdirectory routing for litellm, langfuse, and llama.cpp under single x570.vendeuvre.lan domain --- pod.yaml | 6 ++++-- router/main.py | 13 +++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/pod.yaml b/pod.yaml index fa8938ad..80d3bb74 100644 --- a/pod.yaml +++ b/pod.yaml @@ -45,13 +45,15 @@ spec: value: LITELLM_MASTER_KEY_PLACEHOLDER - name: OLLAMA_API_KEY value: OLLAMA_API_KEY_PLACEHOLDER + - name: SERVER_ROOT_PATH + value: /llm-routing/litellm image: ghcr.io/berriai/litellm:v1.91.0 livenessProbe: exec: command: - python3 - -c - - import urllib.request; urllib.request.urlopen('http://localhost:4000/ping') + - import urllib.request; urllib.request.urlopen('http://localhost:4000/llm-routing/litellm/ping') initialDelaySeconds: 240 periodSeconds: 15 timeoutSeconds: 5 @@ -61,7 +63,7 @@ spec: command: - python3 - -c - - import urllib.request; urllib.request.urlopen('http://localhost:4000/health/readiness') + - import urllib.request; urllib.request.urlopen('http://localhost:4000/llm-routing/litellm/health/readiness') initialDelaySeconds: 10 periodSeconds: 10 timeoutSeconds: 5 diff --git a/router/main.py b/router/main.py index 1c9edbbf..fadb794c 100644 --- a/router/main.py +++ b/router/main.py @@ -3082,13 +3082,14 @@ async def get_dashboard(request: Request): domain = "vendeuvre.lan" if external_host and domain in external_host: - langfuse_url = f"https://langfuse.{domain}" - litellm_url = f"https://litellm.{domain}/ui" - llama_url = f"https://llama.{domain}" + langfuse_url = f"https://{external_host}/llm-routing/langfuse" + litellm_url = f"https://{external_host}/llm-routing/litellm/ui" + llama_url = f"https://{external_host}/llm-routing/llama/" elif domain in (request.base_url.hostname or ""): - langfuse_url = f"https://langfuse.{domain}" - litellm_url = f"https://litellm.{domain}/ui" - llama_url = f"https://llama.{domain}" + base = f"{request.url.scheme}://{request.url.netloc}" + langfuse_url = f"{base}/llm-routing/langfuse" + litellm_url = f"{base}/llm-routing/litellm/ui" + llama_url = f"{base}/llm-routing/llama/" else: langfuse_url = f"http://{external_host}:3001" litellm_url = f"http://{external_host}:4000/ui" From 36cbc72505750c8c8d183de2751aaca01b42411c Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Wed, 8 Jul 2026 22:29:11 +0200 Subject: [PATCH 07/20] fix: address PR reviews - sanitize external_host and fix credentials leak / TLS bypass --- .agents/AGENTS.md | 4 ++-- router/main.py | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.agents/AGENTS.md b/.agents/AGENTS.md index a5ec8b7c..8309bd73 100644 --- a/.agents/AGENTS.md +++ b/.agents/AGENTS.md @@ -64,7 +64,7 @@ pkill -f host_agy_daemon.py || true nohup python3 ~/LAB/IA/LLM-Routing/scripts/host_agy_daemon.py >/tmp/agy-daemon.log 2>&1 & # 6. Verify end-to-end -curl -k -s --resolve x570.vendeuvre.lan:443:127.0.0.1 \ +curl -s --resolve x570.vendeuvre.lan:443:127.0.0.1 \ https://x570.vendeuvre.lan/llm-routing/dashboard | head -5 ``` @@ -72,7 +72,7 @@ curl -k -s --resolve x570.vendeuvre.lan:443:127.0.0.1 \ - The `agy-daemon.service` systemd unit cannot be reloaded via `systemctl --user` from the agent terminal (DBus is not connected). Start the daemon manually with `nohup` as shown above, or instruct the user to run it in their own session. -- **Sudo Password Precaution**: Always preserve exact bytes (including trailing spaces or newlines) when reading `/home/sheepdestroyer/.sudo_password` (e.g. `'sakamoto '`). Stripping whitespace will cause authentication to fail. +- **Sudo Password Precaution**: Always preserve exact bytes (including trailing spaces or newlines) when reading `/home/sheepdestroyer/.sudo_password` (e.g. `'your_password_here '`). Stripping whitespace will cause authentication to fail. - `start-stack.sh` without `--full-rebuild` will do a fast pod restart (reuses images). Use `--full-rebuild` after code changes or image updates. - **GitHub CLI Authentication**: If running `gh` commands fails with a 401 error, ensure that `GITHUB_TOKEN` is exported (e.g., mapped from `GITHUB_MCP_PAT` in `~/.bashrc` via `export GITHUB_TOKEN="$GITHUB_MCP_PAT"`). diff --git a/router/main.py b/router/main.py index fadb794c..e426c470 100644 --- a/router/main.py +++ b/router/main.py @@ -3081,12 +3081,18 @@ async def get_dashboard(request: Request): external_host = urlparse(external_host).hostname or "localhost" domain = "vendeuvre.lan" + import re + if not isinstance(external_host, str) or not re.match(r"^[a-zA-Z0-9.-]+$", external_host): + external_host = "localhost" + if external_host and domain in external_host: langfuse_url = f"https://{external_host}/llm-routing/langfuse" litellm_url = f"https://{external_host}/llm-routing/litellm/ui" llama_url = f"https://{external_host}/llm-routing/llama/" elif domain in (request.base_url.hostname or ""): - base = f"{request.url.scheme}://{request.url.netloc}" + scheme = request.url.scheme if re.match(r"^(?:http|https)$", request.url.scheme) else "https" + netloc = request.url.netloc if re.match(r"^[a-zA-Z0-9.-]+(?::\d+)?$", request.url.netloc) else "localhost" + base = f"{scheme}://{netloc}" langfuse_url = f"{base}/llm-routing/langfuse" litellm_url = f"{base}/llm-routing/litellm/ui" llama_url = f"{base}/llm-routing/llama/" From c2a27e8b435977929f326f650675bec991ec4fcf Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Wed, 8 Jul 2026 22:33:44 +0200 Subject: [PATCH 08/20] docs: document -k intent in verification curl (self-signed cert) --- .agents/AGENTS.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.agents/AGENTS.md b/.agents/AGENTS.md index 8309bd73..11a5f7f9 100644 --- a/.agents/AGENTS.md +++ b/.agents/AGENTS.md @@ -64,7 +64,9 @@ pkill -f host_agy_daemon.py || true nohup python3 ~/LAB/IA/LLM-Routing/scripts/host_agy_daemon.py >/tmp/agy-daemon.log 2>&1 & # 6. Verify end-to-end -curl -s --resolve x570.vendeuvre.lan:443:127.0.0.1 \ +# NOTE: -k is intentional — the HAProxy cert is self-signed (local CA). +# Replace the cert with a trusted CA-signed cert to remove -k. +curl -k -s --resolve x570.vendeuvre.lan:443:127.0.0.1 \ https://x570.vendeuvre.lan/llm-routing/dashboard | head -5 ``` From aded670bfbdaebf0d758d58ec43c3aee1484f017 Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Wed, 8 Jul 2026 22:39:26 +0200 Subject: [PATCH 09/20] Update AGENTS.md --- .agents/AGENTS.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.agents/AGENTS.md b/.agents/AGENTS.md index 11a5f7f9..62e5f172 100644 --- a/.agents/AGENTS.md +++ b/.agents/AGENTS.md @@ -10,9 +10,9 @@ When working on this project, always refer to the dedicated **NotebookLM Compani - Local model benchmark metrics and `llama-server` configurations ### Notebook Details -- **Notebook Name:** `TriageGate-Architect-KB` +- **Notebook Name:** `LLM-Routing-KB` - **Notebook ID:** `llm-triage-gateway` -- **Notebook URL:** [TriageGate-Architect-KB](https://notebooklm.google.com/notebook/826cbd87-7969-4b0e-a38e-5517b5ab7d28) +- **Notebook URL:** [LLM-Routing-KB](https://notebooklm.google.com/notebook/826cbd87-7969-4b0e-a38e-5517b5ab7d28) ### How to Query Use the `notebooklm` MCP tools to search or ask questions about this codebase and stack: @@ -59,9 +59,9 @@ ssh boy "podman run -d --name production-haproxy --restart always --net host \ -v /mnt/DATA/boy/haproxy/certs:/usr/local/etc/haproxy/certs:ro \ docker.io/library/haproxy:alpine" -# 5. Start the host-side agy daemon (runs as sheepdestroyer, not boy) +# 5. Start the host-side agy daemon pkill -f host_agy_daemon.py || true -nohup python3 ~/LAB/IA/LLM-Routing/scripts/host_agy_daemon.py >/tmp/agy-daemon.log 2>&1 & +nohup python3 ~/LLM-Routing/scripts/host_agy_daemon.py >/tmp/agy-daemon.log 2>&1 & # 6. Verify end-to-end # NOTE: -k is intentional — the HAProxy cert is self-signed (local CA). @@ -74,7 +74,7 @@ curl -k -s --resolve x570.vendeuvre.lan:443:127.0.0.1 \ - The `agy-daemon.service` systemd unit cannot be reloaded via `systemctl --user` from the agent terminal (DBus is not connected). Start the daemon manually with `nohup` as shown above, or instruct the user to run it in their own session. -- **Sudo Password Precaution**: Always preserve exact bytes (including trailing spaces or newlines) when reading `/home/sheepdestroyer/.sudo_password` (e.g. `'your_password_here '`). Stripping whitespace will cause authentication to fail. +- **Sudo Password Precaution**: Always preserve exact bytes (including trailing spaces or newlines) when reading `~/.sudo_password` (e.g. `'your_password_here '`). Stripping whitespace will cause authentication to fail. - `start-stack.sh` without `--full-rebuild` will do a fast pod restart (reuses images). Use `--full-rebuild` after code changes or image updates. - **GitHub CLI Authentication**: If running `gh` commands fails with a 401 error, ensure that `GITHUB_TOKEN` is exported (e.g., mapped from `GITHUB_MCP_PAT` in `~/.bashrc` via `export GITHUB_TOKEN="$GITHUB_MCP_PAT"`). From 49b4f3acbe156a914308f72c4bb7230ac3dcc28f Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Wed, 8 Jul 2026 23:25:58 +0200 Subject: [PATCH 10/20] chore: update success banner to show external /llm-routing/v1 endpoint URL --- start-stack.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/start-stack.sh b/start-stack.sh index 7a1a7bac..7d722e51 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -599,10 +599,11 @@ if podman pod exists agent-router-pod 2>/dev/null; then echo "" echo "=========================================================================" echo "🎉 SUCCESS: LLM Triage Gateway restarted!" - echo "📍 Entry endpoint : http://localhost:5000/v1" - echo "⚙️ Dashboard URL : http://localhost:5000/dashboard" + echo "📍 Entry endpoint : https://x570.vendeuvre.lan/llm-routing/v1" + echo " (local) : http://localhost:5000/v1" + echo "⚙️ Dashboard URL : https://x570.vendeuvre.lan/llm-routing/dashboard" echo "🔑 Gateway API Key : gateway-pass" - echo "🔐 LiteLLM Admin UI: http://localhost:4000/ui" + echo "🔐 LiteLLM Admin UI: https://x570.vendeuvre.lan/llm-routing/litellm/ui" echo " Username: admin | Password: $LITELLM_MASTER_KEY" echo "=========================================================================" exit 0 @@ -621,9 +622,10 @@ fi echo "=========================================================================" echo "🎉 SUCCESS: LLM Triage Gateway successfully deployed!" -echo "📍 Entry endpoint : http://localhost:5000/v1" -echo "⚙️ Dashboard URL : http://localhost:5000/dashboard" +echo "📍 Entry endpoint : https://x570.vendeuvre.lan/llm-routing/v1" +echo " (local) : http://localhost:5000/v1" +echo "⚙️ Dashboard URL : https://x570.vendeuvre.lan/llm-routing/dashboard" echo "🔑 Gateway API Key : gateway-pass" -echo "🔐 LiteLLM Admin UI: http://localhost:4000/ui" +echo "🔐 LiteLLM Admin UI: https://x570.vendeuvre.lan/llm-routing/litellm/ui" echo " Username: admin | Password: $LITELLM_MASTER_KEY" echo "=========================================================================" From e096379a0fcabbfeea42995d04803768865c136d Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Wed, 8 Jul 2026 23:33:00 +0200 Subject: [PATCH 11/20] fix: bump LiteLLM readiness probe initial delay to 45s to avoid startup false failures --- pod.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pod.yaml b/pod.yaml index 80d3bb74..e2b04663 100644 --- a/pod.yaml +++ b/pod.yaml @@ -63,8 +63,8 @@ spec: command: - python3 - -c - - import urllib.request; urllib.request.urlopen('http://localhost:4000/llm-routing/litellm/health/readiness') - initialDelaySeconds: 10 + - import urllib.request; urllib.request.urlopen('http://localhost:4000/health/readiness') + initialDelaySeconds: 45 periodSeconds: 10 timeoutSeconds: 5 volumeMounts: From a702b5c0f3be9931a2f03d88c2e8c409d172a047 Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Thu, 9 Jul 2026 01:06:49 +0200 Subject: [PATCH 12/20] chore: sync changes from boy prod including local-qwen-3.6 config and PROXY_BASE_URL --- litellm/config.yaml | 15 +++++++-------- pod.yaml | 4 +++- router/free_models_roster.json | 23 +++++++++++++++-------- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/litellm/config.yaml b/litellm/config.yaml index d1043342..61c477a2 100644 --- a/litellm/config.yaml +++ b/litellm/config.yaml @@ -90,14 +90,13 @@ model_list: max_input_tokens: 524288 is_public_model_group: true -# DISABLED 2026-06-08 — 20GB on disk, 23GB GTT (system RAM as GPU buffer). -# Uncomment to re-enable once a lighter model replaces it. -#- litellm_params: -# api_base: http://127.0.0.1:8080/v1 -# api_key: local-token -# model: openai/qwen-35b-q4ks -# request_timeout: 300 -# model_name: local-qwen-3.6 +# Re-enabled 2026-07-08 — llama.cpp on port 8081, alias local-qwen-3.6 +- litellm_params: + api_base: http://127.0.0.1:8081/v1 + api_key: local-token + model: openai/local-qwen-3.6 + request_timeout: 300 + model_name: local-qwen-3.6 - litellm_params: api_base: http://127.0.0.1:8080/v1 api_key: local-token diff --git a/pod.yaml b/pod.yaml index e2b04663..c954b57a 100644 --- a/pod.yaml +++ b/pod.yaml @@ -47,13 +47,15 @@ spec: value: OLLAMA_API_KEY_PLACEHOLDER - name: SERVER_ROOT_PATH value: /llm-routing/litellm + - name: PROXY_BASE_URL + value: https://x570.vendeuvre.lan/llm-routing/litellm image: ghcr.io/berriai/litellm:v1.91.0 livenessProbe: exec: command: - python3 - -c - - import urllib.request; urllib.request.urlopen('http://localhost:4000/llm-routing/litellm/ping') + - import urllib.request; urllib.request.urlopen('http://localhost:4000/ping') initialDelaySeconds: 240 periodSeconds: 15 timeoutSeconds: 5 diff --git a/router/free_models_roster.json b/router/free_models_roster.json index 5f1dfe0c..8a71d4a5 100644 --- a/router/free_models_roster.json +++ b/router/free_models_roster.json @@ -42,6 +42,18 @@ "score": 28.0, "context_length": 256000 }, + { + "id": "tencent/hy3:free", + "name": "Tencent: Hy3 (free)", + "score": 25.0, + "context_length": 262144 + }, + { + "id": "poolside/laguna-xs-2.1:free", + "name": "Poolside: Laguna XS 2.1 (free)", + "score": 25.0, + "context_length": 262144 + }, { "id": "cohere/north-mini-code:free", "name": "Cohere: North Mini Code (free)", @@ -54,12 +66,7 @@ "score": 25.0, "context_length": 128000 }, - { - "id": "openrouter/owl-alpha", - "name": "Owl Alpha", - "score": 25.0, - "context_length": 1048756 - }, + { "id": "google/lyria-3-pro-preview", "name": "Google: Lyria 3 Pro Preview", @@ -139,6 +146,6 @@ "context_length": 32768 } ], - "updated_at": "2026-06-24T18:40:15.482014Z", - "count": 23 + "updated_at": "2026-07-08T22:55:47.271165Z", + "count": 24 } \ No newline at end of file From 86442b51f6b195c0a2e29af83cb4899f3185f464 Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Thu, 9 Jul 2026 01:08:08 +0200 Subject: [PATCH 13/20] fix: address PR reviews - configurable ROUTING_DOMAIN, improve external_host/port resolution, fix agy startup SSH commands --- .agents/AGENTS.md | 4 ++-- router/main.py | 30 +++++++++++++++++++----------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/.agents/AGENTS.md b/.agents/AGENTS.md index 62e5f172..36cc022f 100644 --- a/.agents/AGENTS.md +++ b/.agents/AGENTS.md @@ -60,8 +60,8 @@ ssh boy "podman run -d --name production-haproxy --restart always --net host \ docker.io/library/haproxy:alpine" # 5. Start the host-side agy daemon -pkill -f host_agy_daemon.py || true -nohup python3 ~/LLM-Routing/scripts/host_agy_daemon.py >/tmp/agy-daemon.log 2>&1 & +ssh boy "pkill -f host_agy_daemon.py || true" +ssh boy "nohup python3 /mnt/DATA/boy/LLM-Routing/scripts/host_agy_daemon.py >/tmp/agy-daemon.log 2>&1 &" # 6. Verify end-to-end # NOTE: -k is intentional — the HAProxy cert is self-signed (local CA). diff --git a/router/main.py b/router/main.py index e426c470..690f4175 100644 --- a/router/main.py +++ b/router/main.py @@ -3072,23 +3072,31 @@ async def get_dashboard_stats(): @app.get("/dashboard", response_class=HTMLResponse) async def get_dashboard(request: Request): """Render the router main dashboard HTML showing system metrics, health checks, and recent token usage.""" - external_host = os.getenv("BASEURL") or os.getenv("BASE_URL") - if not external_host: - external_host = request.base_url.hostname or "localhost" - else: - if "://" in external_host: + external_host_env = os.getenv("BASEURL") or os.getenv("BASE_URL") + if external_host_env: + if "://" not in external_host_env: + from urllib.parse import urlparse + parsed = urlparse(f"http://{external_host_env}") + else: from urllib.parse import urlparse - external_host = urlparse(external_host).hostname or "localhost" + parsed = urlparse(external_host_env) + external_host = parsed.hostname or "localhost" + external_netloc = parsed.netloc or "localhost" + else: + external_host = request.base_url.hostname or "localhost" + external_netloc = request.base_url.netloc or "localhost" - domain = "vendeuvre.lan" + domain = os.getenv("ROUTING_DOMAIN") or "vendeuvre.lan" import re if not isinstance(external_host, str) or not re.match(r"^[a-zA-Z0-9.-]+$", external_host): external_host = "localhost" + if not isinstance(external_netloc, str) or not re.match(r"^[a-zA-Z0-9.-]+(?::\d+)?$", external_netloc): + external_netloc = "localhost" - if external_host and domain in external_host: - langfuse_url = f"https://{external_host}/llm-routing/langfuse" - litellm_url = f"https://{external_host}/llm-routing/litellm/ui" - llama_url = f"https://{external_host}/llm-routing/llama/" + if external_netloc and domain in external_netloc: + langfuse_url = f"https://{external_netloc}/llm-routing/langfuse" + litellm_url = f"https://{external_netloc}/llm-routing/litellm/ui" + llama_url = f"https://{external_netloc}/llm-routing/llama/" elif domain in (request.base_url.hostname or ""): scheme = request.url.scheme if re.match(r"^(?:http|https)$", request.url.scheme) else "https" netloc = request.url.netloc if re.match(r"^[a-zA-Z0-9.-]+(?::\d+)?$", request.url.netloc) else "localhost" From 0404eff80d68864ac94c2d6d63b854564e9e2844 Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Thu, 9 Jul 2026 01:23:12 +0200 Subject: [PATCH 14/20] fix: address review comments on PR #241 (domain validation, dynamic success endpoints, nohup stdin redirect) --- .agents/AGENTS.md | 9 ++++++--- pod.yaml | 32 ++++++++++++++++---------------- router/main.py | 12 +++++++----- start-stack.sh | 34 ++++++++++++++++++++++++---------- 4 files changed, 53 insertions(+), 34 deletions(-) diff --git a/.agents/AGENTS.md b/.agents/AGENTS.md index 36cc022f..33379171 100644 --- a/.agents/AGENTS.md +++ b/.agents/AGENTS.md @@ -61,13 +61,12 @@ ssh boy "podman run -d --name production-haproxy --restart always --net host \ # 5. Start the host-side agy daemon ssh boy "pkill -f host_agy_daemon.py || true" -ssh boy "nohup python3 /mnt/DATA/boy/LLM-Routing/scripts/host_agy_daemon.py >/tmp/agy-daemon.log 2>&1 &" +ssh boy "nohup python3 /mnt/DATA/boy/LLM-Routing/scripts/host_agy_daemon.py >/tmp/agy-daemon.log 2>&1 /dev/null; then fi render_pod_yaml() { - export WORKDIR HOME LITELLM_MASTER_KEY POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD + PUBLIC_BASE_URL="${PUBLIC_BASE_URL:-${BASE_URL:-${BASEURL:-https://x570.vendeuvre.lan/llm-routing}}}" + export WORKDIR HOME LITELLM_MASTER_KEY POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL python3 - "$WORKDIR/pod.yaml" <<'PY' import os, sys, urllib.parse, json uid = os.getuid() @@ -538,7 +539,8 @@ placeholders = [ "MINIO_PASSWORD_PLACEHOLDER", "LANGFUSE_INIT_USER_PASSWORD_PLACEHOLDER", "REDIS_AUTH_PLACEHOLDER", - "CLICKHOUSE_PASSWORD_PLACEHOLDER" + "CLICKHOUSE_PASSWORD_PLACEHOLDER", + "PROXY_BASE_URL_PLACEHOLDER" ] for ph in placeholders: if ph not in text: @@ -563,6 +565,10 @@ text = text.replace("MINIO_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["MINIO_ text = text.replace("LANGFUSE_INIT_USER_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["LANGFUSE_INIT_USER_PASSWORD"])) text = text.replace("REDIS_AUTH_PLACEHOLDER", yaml_scalar(os.environ["REDIS_AUTH"])) text = text.replace("CLICKHOUSE_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["CLICKHOUSE_PASSWORD"])) +# Derive PROXY_BASE_URL from PUBLIC_BASE_URL +public_base_url = os.environ["PUBLIC_BASE_URL"].rstrip("/") +proxy_base_url = f"{public_base_url}/litellm" +text = text.replace("PROXY_BASE_URL_PLACEHOLDER", yaml_scalar(proxy_base_url)) import re unresolved = sorted(set(re.findall(r"\b[A-Z0-9_]+_PLACEHOLDER\b", text))) if unresolved: @@ -596,14 +602,18 @@ if podman pod exists agent-router-pod 2>/dev/null; then podman pod restart agent-router-pod setup_minio_buckets verify_stack_health + # Derive base URLs from configuration/env with sensible defaults + PUBLIC_BASE_URL="${PUBLIC_BASE_URL:-https://x570.vendeuvre.lan/llm-routing}" + LOCAL_BASE_URL="${LOCAL_BASE_URL:-http://localhost:5000}" + echo "" echo "=========================================================================" echo "🎉 SUCCESS: LLM Triage Gateway restarted!" - echo "📍 Entry endpoint : https://x570.vendeuvre.lan/llm-routing/v1" - echo " (local) : http://localhost:5000/v1" - echo "⚙️ Dashboard URL : https://x570.vendeuvre.lan/llm-routing/dashboard" + echo "📍 Entry endpoint : ${PUBLIC_BASE_URL}/v1" + echo " (local) : ${LOCAL_BASE_URL}/v1" + echo "⚙️ Dashboard URL : ${PUBLIC_BASE_URL}/dashboard" echo "🔑 Gateway API Key : gateway-pass" - echo "🔐 LiteLLM Admin UI: https://x570.vendeuvre.lan/llm-routing/litellm/ui" + echo "🔐 LiteLLM Admin UI: ${PUBLIC_BASE_URL}/litellm/ui" echo " Username: admin | Password: $LITELLM_MASTER_KEY" echo "=========================================================================" exit 0 @@ -620,12 +630,16 @@ else verify_stack_health fi +# Derive base URLs from configuration/env with sensible defaults +PUBLIC_BASE_URL="${PUBLIC_BASE_URL:-https://x570.vendeuvre.lan/llm-routing}" +LOCAL_BASE_URL="${LOCAL_BASE_URL:-http://localhost:5000}" + echo "=========================================================================" echo "🎉 SUCCESS: LLM Triage Gateway successfully deployed!" -echo "📍 Entry endpoint : https://x570.vendeuvre.lan/llm-routing/v1" -echo " (local) : http://localhost:5000/v1" -echo "⚙️ Dashboard URL : https://x570.vendeuvre.lan/llm-routing/dashboard" +echo "📍 Entry endpoint : ${PUBLIC_BASE_URL}/v1" +echo " (local) : ${LOCAL_BASE_URL}/v1" +echo "⚙️ Dashboard URL : ${PUBLIC_BASE_URL}/dashboard" echo "🔑 Gateway API Key : gateway-pass" -echo "🔐 LiteLLM Admin UI: https://x570.vendeuvre.lan/llm-routing/litellm/ui" +echo "🔐 LiteLLM Admin UI: ${PUBLIC_BASE_URL}/litellm/ui" echo " Username: admin | Password: $LITELLM_MASTER_KEY" echo "=========================================================================" From 8d67ccf10483ca2cd7e135e6f429922296731fe0 Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Thu, 9 Jul 2026 01:33:35 +0200 Subject: [PATCH 15/20] fix: address review comments on PR #242 (revert pod.yaml host paths, add fallbacks, avoid duplicate variables) --- litellm/config.yaml | 10 +++++----- pod.yaml | 30 +++++++++++++++--------------- start-stack.sh | 15 ++++++++------- 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/litellm/config.yaml b/litellm/config.yaml index 61c477a2..195712f5 100644 --- a/litellm/config.yaml +++ b/litellm/config.yaml @@ -38,29 +38,29 @@ litellm_settings: - agent-advanced-core - llm-routing-ollama - openrouter-auto - # - local-qwen-3.6 # DISABLED: 35B model unloaded (23GB GTT saved) + - local-qwen-3.6 - agent-medium-core: - agent-complex-core - agent-reasoning-core - agent-advanced-core - llm-routing-ollama - openrouter-auto - # - local-qwen-3.6 # DISABLED + - local-qwen-3.6 - agent-complex-core: - agent-reasoning-core - agent-advanced-core - llm-routing-ollama - openrouter-auto - # - local-qwen-3.6 # DISABLED + - local-qwen-3.6 - agent-reasoning-core: - agent-advanced-core - llm-routing-ollama - openrouter-auto - # - local-qwen-3.6 # DISABLED + - local-qwen-3.6 - agent-advanced-core: - llm-routing-ollama - openrouter-auto - # - local-qwen-3.6 # DISABLED + - local-qwen-3.6 model_list: - litellm_params: diff --git a/pod.yaml b/pod.yaml index bd3a02a8..e93ff1a9 100644 --- a/pod.yaml +++ b/pod.yaml @@ -431,48 +431,48 @@ spec: restartPolicy: Always volumes: - hostPath: - path: /mnt/DATA/boy/LLM-Routing/valkey-data + path: /home/gpav/Vrac/LAB/AI/LLM-Routing/valkey-data name: valkey-storage - hostPath: - path: /mnt/DATA/boy/LLM-Routing/litellm + path: /home/gpav/Vrac/LAB/AI/LLM-Routing/litellm name: litellm-config - hostPath: - path: /mnt/DATA/boy/LLM-Routing/router + path: /home/gpav/Vrac/LAB/AI/LLM-Routing/router name: router-config - hostPath: - path: /mnt/DATA/boy/LLM-Routing/postgres-data + path: /home/gpav/Vrac/LAB/AI/LLM-Routing/postgres-data name: postgres-storage - hostPath: - path: /mnt/DATA/boy/LLM-Routing/clickhouse-data + path: /home/gpav/Vrac/LAB/AI/LLM-Routing/clickhouse-data name: clickhouse-storage - hostPath: - path: /mnt/DATA/boy/LLM-Routing/redis-lf-data + path: /home/gpav/Vrac/LAB/AI/LLM-Routing/redis-lf-data name: redis-lf-storage - hostPath: - path: /mnt/DATA/boy/LLM-Routing/minio-data + path: /home/gpav/Vrac/LAB/AI/LLM-Routing/minio-data name: minio-storage - hostPath: - path: /mnt/DATA/boy/.gemini + path: /home/gpav/.gemini name: gemini-secrets - hostPath: - path: /mnt/DATA/boy/.local/share/goose + path: /home/gpav/.local/share/goose name: goose-sessions - hostPath: - path: /mnt/DATA/boy/.local/bin + path: /home/gpav/.local/bin name: agy-bin - hostPath: - path: /run/user/1002/bus + path: /run/user/1000/bus name: dbus-socket - hostPath: - path: /mnt/DATA/boy/.local/share/keyrings + path: /home/gpav/.local/share/keyrings name: keyring-store - hostPath: - path: /run/user/1002 + path: /run/user/1000 name: xdg-runtime - hostPath: - path: /mnt/DATA/boy/LLM-Routing + path: /home/gpav/Vrac/LAB/AI/LLM-Routing name: env-file - hostPath: - path: /mnt/DATA/boy/LLM-Routing/data + path: /home/gpav/Vrac/LAB/AI/LLM-Routing/data type: DirectoryOrCreate name: dataset-data diff --git a/start-stack.sh b/start-stack.sh index 42b151c0..9eb2968e 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -69,6 +69,14 @@ if [ -f "$ENV_FILE" ]; then set +a fi +# Derive public/local base URLs from env/config with sensible defaults, removing trailing slash +PUBLIC_BASE_URL="${PUBLIC_BASE_URL:-${BASE_URL:-${BASEURL:-https://x570.vendeuvre.lan/llm-routing}}}" +PUBLIC_BASE_URL="${PUBLIC_BASE_URL%/}" +LOCAL_BASE_URL="${LOCAL_BASE_URL:-http://localhost:5000}" +LOCAL_BASE_URL="${LOCAL_BASE_URL%/}" +export PUBLIC_BASE_URL LOCAL_BASE_URL + + # Ensure openssl is installed if we need to generate passwords/keys if [ -z "$POSTGRES_PASSWORD" ] || [ -z "$NEXTAUTH_SECRET" ] || [ -z "$SALT" ] || [ -z "$ENCRYPTION_KEY" ] || [ -z "$LITELLM_MASTER_KEY" ] || [ -z "$ROUTER_API_KEY" ] || [ -z "$MINIO_ROOT_USER" ] || [ -z "$MINIO_ROOT_PASSWORD" ] || [ -z "$LANGFUSE_INIT_USER_PASSWORD" ] || [ -z "$REDIS_AUTH" ] || [ -z "$CLICKHOUSE_PASSWORD" ] || [ -z "$LANGFUSE_PUBLIC_KEY" ] || [ -z "$LANGFUSE_SECRET_KEY" ]; then if ! command -v openssl &>/dev/null; then @@ -511,7 +519,6 @@ if podman pod exists agent-router-pod 2>/dev/null; then fi render_pod_yaml() { - PUBLIC_BASE_URL="${PUBLIC_BASE_URL:-${BASE_URL:-${BASEURL:-https://x570.vendeuvre.lan/llm-routing}}}" export WORKDIR HOME LITELLM_MASTER_KEY POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL python3 - "$WORKDIR/pod.yaml" <<'PY' import os, sys, urllib.parse, json @@ -602,9 +609,6 @@ if podman pod exists agent-router-pod 2>/dev/null; then podman pod restart agent-router-pod setup_minio_buckets verify_stack_health - # Derive base URLs from configuration/env with sensible defaults - PUBLIC_BASE_URL="${PUBLIC_BASE_URL:-https://x570.vendeuvre.lan/llm-routing}" - LOCAL_BASE_URL="${LOCAL_BASE_URL:-http://localhost:5000}" echo "" echo "=========================================================================" @@ -630,9 +634,6 @@ else verify_stack_health fi -# Derive base URLs from configuration/env with sensible defaults -PUBLIC_BASE_URL="${PUBLIC_BASE_URL:-https://x570.vendeuvre.lan/llm-routing}" -LOCAL_BASE_URL="${LOCAL_BASE_URL:-http://localhost:5000}" echo "=========================================================================" echo "🎉 SUCCESS: LLM Triage Gateway successfully deployed!" From 855deb376ecf4afce59dfbe9f02818e7f90a6af7 Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Thu, 9 Jul 2026 01:36:25 +0200 Subject: [PATCH 16/20] refactor: replace legacy hardcoded placeholders with generic WORKDIR_PLACEHOLDER, HOME_PLACEHOLDER, and RUN_USER_PLACEHOLDER --- pod.yaml | 36 ++++++++++++++++++------------------ router/agy_proxy.py | 2 +- start-stack.sh | 12 ++++++------ 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/pod.yaml b/pod.yaml index e93ff1a9..023bdd71 100644 --- a/pod.yaml +++ b/pod.yaml @@ -95,7 +95,7 @@ spec: - name: DATABASE_URL value: postgresql://postgres:POSTGRES_PASSWORD_ENCODED_PLACEHOLDER@127.0.0.1:5432/postgres - name: DBUS_SESSION_BUS_ADDRESS - value: unix:path=/run/user/1000/bus + value: unix:path=RUN_USER_PLACEHOLDER/bus - name: LITELLM_MASTER_KEY value: LITELLM_MASTER_KEY_PLACEHOLDER - name: LANGFUSE_PUBLIC_KEY @@ -144,11 +144,11 @@ spec: - mountPath: /usr/local/bin/agy name: agy-bin subPath: agy - - mountPath: /run/user/1000/bus + - mountPath: RUN_USER_PLACEHOLDER/bus name: dbus-socket - mountPath: /root/.local/share/keyrings name: keyring-store - - mountPath: /run/user/1000 + - mountPath: RUN_USER_PLACEHOLDER name: xdg-runtime - mountPath: /config/.env name: env-file @@ -431,48 +431,48 @@ spec: restartPolicy: Always volumes: - hostPath: - path: /home/gpav/Vrac/LAB/AI/LLM-Routing/valkey-data + path: WORKDIR_PLACEHOLDER/valkey-data name: valkey-storage - hostPath: - path: /home/gpav/Vrac/LAB/AI/LLM-Routing/litellm + path: WORKDIR_PLACEHOLDER/litellm name: litellm-config - hostPath: - path: /home/gpav/Vrac/LAB/AI/LLM-Routing/router + path: WORKDIR_PLACEHOLDER/router name: router-config - hostPath: - path: /home/gpav/Vrac/LAB/AI/LLM-Routing/postgres-data + path: WORKDIR_PLACEHOLDER/postgres-data name: postgres-storage - hostPath: - path: /home/gpav/Vrac/LAB/AI/LLM-Routing/clickhouse-data + path: WORKDIR_PLACEHOLDER/clickhouse-data name: clickhouse-storage - hostPath: - path: /home/gpav/Vrac/LAB/AI/LLM-Routing/redis-lf-data + path: WORKDIR_PLACEHOLDER/redis-lf-data name: redis-lf-storage - hostPath: - path: /home/gpav/Vrac/LAB/AI/LLM-Routing/minio-data + path: WORKDIR_PLACEHOLDER/minio-data name: minio-storage - hostPath: - path: /home/gpav/.gemini + path: HOME_PLACEHOLDER/.gemini name: gemini-secrets - hostPath: - path: /home/gpav/.local/share/goose + path: HOME_PLACEHOLDER/.local/share/goose name: goose-sessions - hostPath: - path: /home/gpav/.local/bin + path: HOME_PLACEHOLDER/.local/bin name: agy-bin - hostPath: - path: /run/user/1000/bus + path: RUN_USER_PLACEHOLDER/bus name: dbus-socket - hostPath: - path: /home/gpav/.local/share/keyrings + path: HOME_PLACEHOLDER/.local/share/keyrings name: keyring-store - hostPath: - path: /run/user/1000 + path: RUN_USER_PLACEHOLDER name: xdg-runtime - hostPath: - path: /home/gpav/Vrac/LAB/AI/LLM-Routing + path: WORKDIR_PLACEHOLDER name: env-file - hostPath: - path: /home/gpav/Vrac/LAB/AI/LLM-Routing/data + path: WORKDIR_PLACEHOLDER/data type: DirectoryOrCreate name: dataset-data diff --git a/router/agy_proxy.py b/router/agy_proxy.py index ffccd733..8c1f8631 100644 --- a/router/agy_proxy.py +++ b/router/agy_proxy.py @@ -42,7 +42,7 @@ async def save(self) -> None: logger = logging.getLogger("agy-proxy") -# In container: mounted from host /home/gpav/.local/bin/agy +# In container: mounted from host ~/.local/bin/agy AGY_BINARY = os.environ.get("AGY_BINARY_PATH", "/usr/local/bin/agy") if not os.path.exists(AGY_BINARY): AGY_BINARY = os.path.expanduser("~/.local/bin/agy") diff --git a/start-stack.sh b/start-stack.sh index 9eb2968e..0a646b77 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -530,9 +530,9 @@ def yaml_scalar(val): return json.dumps(val) placeholders = [ - "/home/gpav/Vrac/LAB/AI/LLM-Routing", - "/home/gpav/", - "/run/user/1000", + "WORKDIR_PLACEHOLDER", + "HOME_PLACEHOLDER", + "RUN_USER_PLACEHOLDER", "LITELLM_MASTER_KEY_PLACEHOLDER", "POSTGRES_PASSWORD_RAW_PLACEHOLDER", "POSTGRES_PASSWORD_ENCODED_PLACEHOLDER", @@ -553,9 +553,9 @@ for ph in placeholders: if ph not in text: sys.stderr.write(f"Error: Required placeholder '{ph}' not found in pod.yaml. Ensure you are using the latest version of the template.\n") sys.exit(1) -text = text.replace("/home/gpav/Vrac/LAB/AI/LLM-Routing", os.environ["WORKDIR"]) -text = text.replace("/home/gpav/", os.environ["HOME"] + "/") -text = text.replace("/run/user/1000", f"/run/user/{uid}") +text = text.replace("WORKDIR_PLACEHOLDER", os.environ["WORKDIR"]) +text = text.replace("HOME_PLACEHOLDER", os.environ["HOME"]) +text = text.replace("RUN_USER_PLACEHOLDER", f"/run/user/{uid}") text = text.replace("LITELLM_MASTER_KEY_PLACEHOLDER", yaml_scalar(os.environ["LITELLM_MASTER_KEY"])) text = text.replace("POSTGRES_PASSWORD_RAW_PLACEHOLDER", yaml_scalar(os.environ["POSTGRES_PASSWORD"])) # URL-encode the postgres password for DSN insertion From 045284a8670a22c902546e8025303c0b8424126b Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Thu, 9 Jul 2026 01:40:12 +0200 Subject: [PATCH 17/20] fix: normalize PUBLIC_BASE_URL to include https scheme and llm-routing subpath if missing --- start-stack.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/start-stack.sh b/start-stack.sh index 0a646b77..34f50603 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -71,6 +71,12 @@ fi # Derive public/local base URLs from env/config with sensible defaults, removing trailing slash PUBLIC_BASE_URL="${PUBLIC_BASE_URL:-${BASE_URL:-${BASEURL:-https://x570.vendeuvre.lan/llm-routing}}}" +if [[ ! "$PUBLIC_BASE_URL" =~ ^https?:// ]]; then + PUBLIC_BASE_URL="https://${PUBLIC_BASE_URL}" +fi +if [[ ! "$PUBLIC_BASE_URL" =~ /llm-routing ]]; then + PUBLIC_BASE_URL="${PUBLIC_BASE_URL%/}/llm-routing" +fi PUBLIC_BASE_URL="${PUBLIC_BASE_URL%/}" LOCAL_BASE_URL="${LOCAL_BASE_URL:-http://localhost:5000}" LOCAL_BASE_URL="${LOCAL_BASE_URL%/}" From 60ea4af209696a5cddb696cbc88593b7d29e140a Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Thu, 9 Jul 2026 01:43:49 +0200 Subject: [PATCH 18/20] docs: sanitize legacy gpav paths in README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5b45456a..20262cd8 100644 --- a/README.md +++ b/README.md @@ -212,7 +212,7 @@ The gateway supports multiple routing modes controlled by the `model` field: All configurations, automation scripts, and databases are self-contained within this repository directory: ``` -/home/gpav/Vrac/LAB/AI/LLM-Routing/ +/path/to/LLM-Routing/ ├── .env # Environment file for API keys, passwords, and generated secrets (ignored by git) ├── .gitignore # Git ignore policy protecting secrets & database files ├── README.md # In-depth system and operational guide @@ -350,7 +350,7 @@ The stack also supports **semantic** (vector-similarity) caching via `vector_sto - **Embedding Model**: Zero-cost local `nomic-embed-text-v1.5-Q4_K_M` (~137MB GGUF) running on llama-server, loaded as `local-nomic-embed` in LiteLLM. Produces 768-dimension vectors with CLS pooling. - **Vector Store**: PostgreSQL with pgvector extension stores embeddings in the `litellm_semantic_cache` collection. - **Cost**: Completely free — no OpenRouter API calls for embedding generation. -- **Configuration**: The nomic-embed model profile in `models.ini` (`/home/gpav/Vrac/LAB/AI/models.ini`) includes `embedding = true`, `pooling = cls`, and `embd-normalize = 2` for proper vector similarity search. llama-server runs with `--models-max 3` to keep the classifier (0.8B), MoE (35B), and embedding model loaded simultaneously. +- **Configuration**: The nomic-embed model profile in `models.ini` (e.g., `/path/to/models.ini`) includes `embedding = true`, `pooling = cls`, and `embd-normalize = 2` for proper vector similarity search. llama-server runs with `--models-max 3` to keep the classifier (0.8B), MoE (35B), and embedding model loaded simultaneously. --- @@ -456,7 +456,7 @@ Uvicorn's log level follows the same env var via `${LOG_LEVEL:-warning}` in the | `PrivateTmp` | `yes` | Isolates `/tmp` namespace for the daemon | | `PrivateDevices` | `yes` | Restricts access to `/dev` (no raw disk/device access) | | `ProtectSystem` | `strict` | Makes `/usr` and `/etc` read-only | -| `ProtectHome` | `read-only` | `/home/gpav` is read-only except specific paths | +| `ProtectHome` | `read-only` | The user home directory is read-only except specific paths | | `ProtectKernelTunables` | `yes` | Makes `/sys` and `/proc/sys` read-only | | `ProtectKernelModules` | `yes` | Blocks loading or listing kernel modules | | `ProtectControlGroups` | `yes` | Makes cgroup filesystem read-only | @@ -758,7 +758,7 @@ To maximize throughput under concurrent queries, `llama-server` is configured wi #### 3. Custom Memory Endpoint Proxy & MCP Server To allow Goose (and other agents) to store, list, and delete persistent preference/factual memories, we implemented a custom memory stack: * **Triage Router Memory Proxy**: Exposes a catch-all route `@app.api_route("/v1/memory{path:path}", methods=["GET", "POST", "DELETE", "PUT"])` in `router/main.py` that intercepts memory calls and proxies them to the LiteLLM gateway (port 4000) using the securely-loaded `LITELLM_MASTER_KEY` authorization. -* **Memory MCP Bridge Server**: Created a custom stdio MCP server in [memory_mcp.py](file:///home/gpav/Vrac/LAB/AI/LLM-Routing/router/memory_mcp.py) that exposes the `rememberMemory`, `retrieveMemories`, and `removeSpecificMemory` tools. The script proxies these commands directly to `http://localhost:5000/v1/memory`. +* **Memory MCP Bridge Server**: Created a custom stdio MCP server in [memory_mcp.py](router/memory_mcp.py) that exposes the `rememberMemory`, `retrieveMemories`, and `removeSpecificMemory` tools. The script proxies these commands directly to `http://localhost:5000/v1/memory`. * **Goose Integration**: The built-in memory extension is disabled in `~/.config/goose/config.yaml` and replaced with the `litellm-memory` custom command-line extension running our bridge server. ## 9c. Ollama Proxy Integration (via LiteLLM ollama_chat) From 13ebc2cc2870f9f67f34d1f4c59861bfb4bc9738 Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Thu, 9 Jul 2026 01:47:48 +0200 Subject: [PATCH 19/20] refactor: centralize URL resolution to resolve_external_urls helper utilizing PUBLIC_BASE_URL --- pod.yaml | 2 ++ router/main.py | 53 ++++++++++++++++++++++++++++++++------------------ start-stack.sh | 4 +++- 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/pod.yaml b/pod.yaml index 023bdd71..06c393a3 100644 --- a/pod.yaml +++ b/pod.yaml @@ -106,6 +106,8 @@ spec: value: http://127.0.0.1:3001 - name: OLLAMA_API_KEY value: OLLAMA_API_KEY_PLACEHOLDER + - name: PUBLIC_BASE_URL + value: PUBLIC_BASE_URL_PLACEHOLDER - name: LOG_LEVEL value: info image: localhost/llm-triage-router:latest diff --git a/router/main.py b/router/main.py index 952758ed..668b78cb 100644 --- a/router/main.py +++ b/router/main.py @@ -3070,20 +3070,22 @@ async def get_dashboard_stats(): return await get_dashboard_data() -@app.get("/dashboard", response_class=HTMLResponse) -async def get_dashboard(request: Request): - """Render the router main dashboard HTML showing system metrics, health checks, and recent token usage.""" - external_host_env = os.getenv("BASEURL") or os.getenv("BASE_URL") - if external_host_env: - if "://" not in external_host_env: - parsed = urlparse(f"http://{external_host_env}") +def resolve_external_urls(request: Request) -> tuple[str, str, str]: + """Resolve and validate the base URLs for Langfuse, LiteLLM, and Llama.cpp.""" + # 1. Try to load centralized base URL from config/env + base_url_env = os.getenv("PUBLIC_BASE_URL") or os.getenv("BASEURL") or os.getenv("BASE_URL") + if base_url_env: + if "://" not in base_url_env: + parsed = urlparse(f"https://{base_url_env}") else: - parsed = urlparse(external_host_env) + parsed = urlparse(base_url_env) external_host = parsed.hostname or "localhost" external_netloc = parsed.netloc or "localhost" + external_scheme = parsed.scheme if parsed.scheme in ("http", "https") else "https" else: external_host = request.base_url.hostname or "localhost" external_netloc = request.base_url.netloc or "localhost" + external_scheme = request.url.scheme if request.url.scheme in ("http", "https") else "https" domain = os.getenv("ROUTING_DOMAIN") or "vendeuvre.lan" if not isinstance(external_host, str) or not re.match(r"^[a-zA-Z0-9.-]+$", external_host): @@ -3096,20 +3098,33 @@ async def get_dashboard(request: Request): is_valid_base = request.base_url.hostname == domain or (request.base_url.hostname or "").endswith("." + domain) if is_valid_external: - langfuse_url = f"https://{external_netloc}/llm-routing/langfuse" - litellm_url = f"https://{external_netloc}/llm-routing/litellm/ui" - llama_url = f"https://{external_netloc}/llm-routing/llama/" + # Centralized base URL path under subdomain/reverse proxy + return ( + f"{external_scheme}://{external_netloc}/llm-routing/langfuse", + f"{external_scheme}://{external_netloc}/llm-routing/litellm/ui", + f"{external_scheme}://{external_netloc}/llm-routing/llama/" + ) elif is_valid_base: - scheme = request.url.scheme if re.match(r"^(?:http|https)$", request.url.scheme) else "https" netloc = request.url.netloc if re.match(r"^[a-zA-Z0-9.-]+(?::\d+)?$", request.url.netloc) else "localhost" - base = f"{scheme}://{netloc}" - langfuse_url = f"{base}/llm-routing/langfuse" - litellm_url = f"{base}/llm-routing/litellm/ui" - llama_url = f"{base}/llm-routing/llama/" + base = f"{external_scheme}://{netloc}" + return ( + f"{base}/llm-routing/langfuse", + f"{base}/llm-routing/litellm/ui", + f"{base}/llm-routing/llama/" + ) else: - langfuse_url = f"http://{external_host}:3001" - litellm_url = f"http://{external_host}:4000/ui" - llama_url = f"http://{external_host}:8080" + # Local development fallback + return ( + f"http://{external_host}:3001", + f"http://{external_host}:4000/ui", + f"http://{external_host}:8080" + ) + + +@app.get("/dashboard", response_class=HTMLResponse) +async def get_dashboard(request: Request): + """Render the router main dashboard HTML showing system metrics, health checks, and recent token usage.""" + langfuse_url, litellm_url, llama_url = resolve_external_urls(request) data = await get_dashboard_data() diff --git a/start-stack.sh b/start-stack.sh index 34f50603..a9aa0e68 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -553,7 +553,8 @@ placeholders = [ "LANGFUSE_INIT_USER_PASSWORD_PLACEHOLDER", "REDIS_AUTH_PLACEHOLDER", "CLICKHOUSE_PASSWORD_PLACEHOLDER", - "PROXY_BASE_URL_PLACEHOLDER" + "PROXY_BASE_URL_PLACEHOLDER", + "PUBLIC_BASE_URL_PLACEHOLDER" ] for ph in placeholders: if ph not in text: @@ -582,6 +583,7 @@ text = text.replace("CLICKHOUSE_PASSWORD_PLACEHOLDER", yaml_scalar(os.environ["C public_base_url = os.environ["PUBLIC_BASE_URL"].rstrip("/") proxy_base_url = f"{public_base_url}/litellm" text = text.replace("PROXY_BASE_URL_PLACEHOLDER", yaml_scalar(proxy_base_url)) +text = text.replace("PUBLIC_BASE_URL_PLACEHOLDER", yaml_scalar(os.environ["PUBLIC_BASE_URL"])) import re unresolved = sorted(set(re.findall(r"\b[A-Z0-9_]+_PLACEHOLDER\b", text))) if unresolved: From 8331d9b0b134fbe55995020bc7485fd75ef6ea31 Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Thu, 9 Jul 2026 09:54:07 +0200 Subject: [PATCH 20/20] refactor: consolidate routing domain default, dynamic ports for fallback URLs, and generalize AGENTS.md runbook --- .agents/AGENTS.md | 44 +++++++++++++++++---------------- pod.yaml | 2 ++ router/main.py | 62 ++++++++++++++++++++++++++++++++++++++--------- start-stack.sh | 12 ++++++--- 4 files changed, 85 insertions(+), 35 deletions(-) diff --git a/.agents/AGENTS.md b/.agents/AGENTS.md index 33379171..0a69c6f9 100644 --- a/.agents/AGENTS.md +++ b/.agents/AGENTS.md @@ -27,46 +27,48 @@ To prevent directory reorganization regressions, outdated file restorations, or 3. **Verify Security Credentials**: Never accept resolutions that overwrite configuration files (`pod.yaml`, `start-stack.sh`) with hardcoded default passwords. Ensure placeholder-based configurations are preserved. 4. **Enforce Test Suite Count**: Run the full unit test suite (`pytest`) after conflict resolution. Verify that the total number of passing tests is equal to or greater than before the resolution. -## Production Deployment Checklist (boy user) +## Production Deployment Checklist -### One-Time Host Prerequisites (already configured on x570.vendeuvre.lan) +Note: Throughout this checklist, the production host SSH alias is represented by `` (e.g., `boy`), the deployer home path is represented by `` (e.g., `/mnt/DATA/boy`), and the domain is represented by `` (e.g., `vendeuvre.lan`). + +### One-Time Host Prerequisites - `net.ipv4.ip_unprivileged_port_start=80` persisted in `/etc/sysctl.d/99-unprivileged-ports.conf` - Host firewall ports `80/tcp` and `443/tcp` opened in `firewalld` (e.g. `sudo firewall-cmd --zone=public --add-port=80/tcp --permanent && sudo firewall-cmd --zone=public --add-port=443/tcp --permanent && sudo firewall-cmd --reload`) -- `boy` SSH host alias in `~/.ssh/config` — use `ssh boy` / `rsync ... boy:` throughout -- Required mount directories created under `boy`'s home: - - `/mnt/DATA/boy/.gemini/` - - `/mnt/DATA/boy/.local/bin/agy` (copy of the `agy` binary) - - `/mnt/DATA/boy/.local/share/goose/` - - `/mnt/DATA/boy/.local/share/keyrings/` -- HAProxy SSL cert: `/mnt/DATA/boy/haproxy/certs/vendeuvre.pem` -- HAProxy config: `/mnt/DATA/boy/haproxy/haproxy.cfg` +- SSH host alias configured in `~/.ssh/config` — use `ssh ` / `rsync ... :` throughout +- Required mount directories created under ``: + - `/.gemini/` + - `/.local/bin/agy` (copy of the `agy` binary) + - `/.local/share/goose/` + - `/.local/share/keyrings/` +- HAProxy SSL cert: `/haproxy/certs/.pem` +- HAProxy config: `/haproxy/haproxy.cfg` ### Fresh Deploy Steps (after a PR is merged to master) ```bash -# 1. Clean up old deploy on boy -ssh boy "rm -rf /mnt/DATA/boy/LLM-Routing" +# 1. Clean up old deploy +ssh "rm -rf /LLM-Routing" # 2. Clone fresh from master -ssh boy "git clone https://github.com/sheepdestroyer/LLM-Routing.git /mnt/DATA/boy/LLM-Routing" +ssh "git clone https://github.com/sheepdestroyer/LLM-Routing.git /LLM-Routing" # 3. Start the full stack (builds and launches all containers) -ssh boy "cd /mnt/DATA/boy/LLM-Routing && ./start-stack.sh --full-rebuild" +ssh "cd /LLM-Routing && ./start-stack.sh --full-rebuild" # 4. Start (or restart) production HAProxy -ssh boy "podman rm -f production-haproxy || true" -ssh boy "podman run -d --name production-haproxy --restart always --net host \ - -v /mnt/DATA/boy/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro \ - -v /mnt/DATA/boy/haproxy/certs:/usr/local/etc/haproxy/certs:ro \ +ssh "podman rm -f production-haproxy || true" +ssh "podman run -d --name production-haproxy --restart always --net host \ + -v /haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro \ + -v /haproxy/certs:/usr/local/etc/haproxy/certs:ro \ docker.io/library/haproxy:alpine" # 5. Start the host-side agy daemon -ssh boy "pkill -f host_agy_daemon.py || true" -ssh boy "nohup python3 /mnt/DATA/boy/LLM-Routing/scripts/host_agy_daemon.py >/tmp/agy-daemon.log 2>&1 "pkill -f host_agy_daemon.py || true" +ssh "nohup python3 /LLM-Routing/scripts/host_agy_daemon.py >/tmp/agy-daemon.log 2>&1 "curl -k -s --resolve .:443:127.0.0.1 https://./llm-routing/dashboard" | head -5 ``` ### Notes diff --git a/pod.yaml b/pod.yaml index 06c393a3..4541a55e 100644 --- a/pod.yaml +++ b/pod.yaml @@ -108,6 +108,8 @@ spec: value: OLLAMA_API_KEY_PLACEHOLDER - name: PUBLIC_BASE_URL value: PUBLIC_BASE_URL_PLACEHOLDER + - name: ROUTING_DOMAIN + value: ROUTING_DOMAIN_PLACEHOLDER - name: LOG_LEVEL value: info image: localhost/llm-triage-router:latest diff --git a/router/main.py b/router/main.py index 668b78cb..6324485e 100644 --- a/router/main.py +++ b/router/main.py @@ -18,15 +18,15 @@ from fastapi.responses import HTMLResponse, StreamingResponse, JSONResponse from fastapi.staticfiles import StaticFiles from pathlib import Path +from urllib.parse import urlparse from circuit_breaker import get_breaker from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator, RootModel from typing import Dict, Optional, Union from urllib.parse import urlparse LITELLM_URL = (os.getenv("LITELLM_ADMIN_URL") or "http://127.0.0.1:4000").rstrip("/") -LLAMA_SERVER_URL = (os.getenv("LLAMA_SERVER_URL") or "http://127.0.0.1:8080").rstrip( - "/" -) +LLAMA_SERVER_URL = (os.getenv("LLAMA_SERVER_URL") or "http://127.0.0.1:8080").rstrip("/") +LANGFUSE_HOST = (os.getenv("LANGFUSE_HOST") or "http://127.0.0.1:3001").rstrip("/") GEMINI_OAUTH_CREDS_PATH = "/config/gemini_auth/oauth_creds.json" @@ -234,7 +234,7 @@ def get_langfuse(): _langfuse_client = langfuse.Langfuse( public_key=os.getenv("LANGFUSE_PUBLIC_KEY", ""), secret_key=os.getenv("LANGFUSE_SECRET_KEY", ""), - host=os.getenv("LANGFUSE_HOST", "http://127.0.0.1:3001"), + host=LANGFUSE_HOST, release="llm-triage-router-v1", ) logger.info("Langfuse client initialized") @@ -3088,10 +3088,36 @@ def resolve_external_urls(request: Request) -> tuple[str, str, str]: external_scheme = request.url.scheme if request.url.scheme in ("http", "https") else "https" domain = os.getenv("ROUTING_DOMAIN") or "vendeuvre.lan" + + # Basic sanity-check on external_host, but don't over-restrict valid hostnames; + # fall back to the request base URL rather than silently forcing localhost. if not isinstance(external_host, str) or not re.match(r"^[a-zA-Z0-9.-]+$", external_host): - external_host = "localhost" - if not isinstance(external_netloc, str) or not re.match(r"^[a-zA-Z0-9.-]+(?::\d+)?$", external_netloc): - external_netloc = "localhost" + logger.warning( + "Unexpected external_host %r, falling back to request.base_url.hostname (%r)", + external_host, + request.base_url.hostname, + ) + external_host = request.base_url.hostname or "localhost" + + # Relax external_netloc validation: use urlparse so IPv6 literals, IDN/punycode, + # and reverse-proxy-modified netlocs are supported. Log and fall back instead of + # silently forcing localhost when invalid. + if isinstance(external_netloc, str): + parsed_netloc = urlparse(f"{external_scheme}://{external_netloc}") + if not parsed_netloc.hostname: + logger.warning( + "Invalid external_netloc %r, falling back to request.base_url.netloc (%r)", + external_netloc, + request.base_url.netloc, + ) + external_netloc = request.base_url.netloc or "localhost" + else: + logger.warning( + "Non-string external_netloc %r, falling back to request.base_url.netloc (%r)", + external_netloc, + request.base_url.netloc, + ) + external_netloc = request.base_url.netloc or "localhost" # Enforce strict domain validation to prevent loose substring match bypasses (e.g., attacker-vendeuvre.lan) is_valid_external = external_host == domain or external_host.endswith("." + domain) @@ -3113,11 +3139,25 @@ def resolve_external_urls(request: Request) -> tuple[str, str, str]: f"{base}/llm-routing/llama/" ) else: - # Local development fallback + # Local development fallback: derive ports and paths dynamically from configuration constants + parsed_lf = urlparse(LANGFUSE_HOST) + parsed_ll = urlparse(LITELLM_URL) + parsed_lm = urlparse(LLAMA_SERVER_URL) + + lf_port = f":{parsed_lf.port}" if parsed_lf.port else "" + ll_port = f":{parsed_ll.port}" if parsed_ll.port else "" + lm_port = f":{parsed_lm.port}" if parsed_lm.port else "" + + lf_path = parsed_lf.path + ll_path = parsed_ll.path or "/ui" + if not ll_path.endswith("/ui") and not ll_path.endswith("/ui/"): + ll_path = ll_path.rstrip("/") + "/ui" + lm_path = parsed_lm.path + return ( - f"http://{external_host}:3001", - f"http://{external_host}:4000/ui", - f"http://{external_host}:8080" + f"http://{external_host}{lf_port}{lf_path}", + f"http://{external_host}{ll_port}{ll_path}", + f"http://{external_host}{lm_port}{lm_path}" ) diff --git a/start-stack.sh b/start-stack.sh index a9aa0e68..fd4f18b5 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -69,8 +69,12 @@ if [ -f "$ENV_FILE" ]; then set +a fi +# Define and export the routing domain +ROUTING_DOMAIN="${ROUTING_DOMAIN:-vendeuvre.lan}" +export ROUTING_DOMAIN + # Derive public/local base URLs from env/config with sensible defaults, removing trailing slash -PUBLIC_BASE_URL="${PUBLIC_BASE_URL:-${BASE_URL:-${BASEURL:-https://x570.vendeuvre.lan/llm-routing}}}" +PUBLIC_BASE_URL="${PUBLIC_BASE_URL:-${BASE_URL:-${BASEURL:-https://x570.${ROUTING_DOMAIN}/llm-routing}}}" if [[ ! "$PUBLIC_BASE_URL" =~ ^https?:// ]]; then PUBLIC_BASE_URL="https://${PUBLIC_BASE_URL}" fi @@ -525,7 +529,7 @@ if podman pod exists agent-router-pod 2>/dev/null; then fi render_pod_yaml() { - export WORKDIR HOME LITELLM_MASTER_KEY POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL + export WORKDIR HOME LITELLM_MASTER_KEY POSTGRES_PASSWORD NEXTAUTH_SECRET SALT ENCRYPTION_KEY LANGFUSE_INIT_USER_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD OLLAMA_API_KEY LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY CLASSIFIER_INPUT_MAX_CHARS REDIS_AUTH CLICKHOUSE_PASSWORD PUBLIC_BASE_URL ROUTING_DOMAIN python3 - "$WORKDIR/pod.yaml" <<'PY' import os, sys, urllib.parse, json uid = os.getuid() @@ -554,7 +558,8 @@ placeholders = [ "REDIS_AUTH_PLACEHOLDER", "CLICKHOUSE_PASSWORD_PLACEHOLDER", "PROXY_BASE_URL_PLACEHOLDER", - "PUBLIC_BASE_URL_PLACEHOLDER" + "PUBLIC_BASE_URL_PLACEHOLDER", + "ROUTING_DOMAIN_PLACEHOLDER" ] for ph in placeholders: if ph not in text: @@ -584,6 +589,7 @@ public_base_url = os.environ["PUBLIC_BASE_URL"].rstrip("/") proxy_base_url = f"{public_base_url}/litellm" text = text.replace("PROXY_BASE_URL_PLACEHOLDER", yaml_scalar(proxy_base_url)) text = text.replace("PUBLIC_BASE_URL_PLACEHOLDER", yaml_scalar(os.environ["PUBLIC_BASE_URL"])) +text = text.replace("ROUTING_DOMAIN_PLACEHOLDER", yaml_scalar(os.environ["ROUTING_DOMAIN"])) import re unresolved = sorted(set(re.findall(r"\b[A-Z0-9_]+_PLACEHOLDER\b", text))) if unresolved: