From f669209dfb01e70943200fc2ee33878245d51bde Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Wed, 24 Jun 2026 14:42:52 +0000 Subject: [PATCH 1/3] feat: extract hardcoded microservice endpoints into environment variables Extracted core target URLs for LiteLLM, Llama.cpp, Langfuse, Valkey, and Agy daemon into environment variables with safe fallback constants in `router/main.py` and `router/agy_proxy.py`. This improves deployment agility across varied network environments. - Defined `LITELLM_URL`, `LLAMA_SERVER_URL`, `LANGFUSE_URL`, `VALKEY_HOST`, and `VALKEY_PORT` in `router/main.py`. - Defined `AGY_DAEMON_URL` in `router/agy_proxy.py`. - Updated health checks, proxy logic, and dashboard UI to use these constants. - Fixed a minor indentation error in `detect_active_tool`. Co-authored-by: sheepdestroyer <1377479+sheepdestroyer@users.noreply.github.com> --- router/agy_proxy.py | 6 +++-- router/main.py | 59 +++++++++++++++++++++++++-------------------- 2 files changed, 37 insertions(+), 28 deletions(-) diff --git a/router/agy_proxy.py b/router/agy_proxy.py index 16adb8e9..ad083b0c 100644 --- a/router/agy_proxy.py +++ b/router/agy_proxy.py @@ -43,6 +43,8 @@ async def save(self) -> None: logger = logging.getLogger("agy-proxy") +AGY_DAEMON_URL = os.getenv("AGY_DAEMON_URL", "http://127.0.0.1:5005") + # In container: mounted from host /home/gpav/.local/bin/agy AGY_BINARY = os.environ.get("AGY_BINARY_PATH", "/usr/local/bin/agy") if not os.path.exists(AGY_BINARY): @@ -91,7 +93,7 @@ async def _run_agy_print(client: httpx.AsyncClient, prompt: str, model_override: """ Forward the agy execution request to the host-side agy daemon. """ - url = "http://127.0.0.1:5005/run" + url = f"{AGY_DAEMON_URL}/run" payload = { "prompt": prompt, "model_override": model_override, @@ -299,7 +301,7 @@ async def try_agy_proxy(prompt: str, messages: list = None, tier_timeout = min(AGY_TIMEOUT_SECS, remaining) if stream: - url = "http://127.0.0.1:5005/run" + url = f"{AGY_DAEMON_URL}/run" payload = { "prompt": proxy_prompt, "model_override": tier["env_override"], diff --git a/router/main.py b/router/main.py index 90cd9a4c..2f11e4a3 100644 --- a/router/main.py +++ b/router/main.py @@ -39,6 +39,13 @@ except ImportError: try_agy_proxy = None +# Global Configuration from Environment +LITELLM_URL = os.getenv("LITELLM_ADMIN_URL", "http://127.0.0.1:4000") +LLAMA_SERVER_URL = os.getenv("LLAMA_SERVER_URL", "http://127.0.0.1:8080") +LANGFUSE_URL = os.getenv("LANGFUSE_URL", "http://127.0.0.1:3001") +VALKEY_HOST = os.getenv("VALKEY_HOST", "127.0.0.1") +VALKEY_PORT = int(os.getenv("VALKEY_PORT", "6379")) + _redis_client = None _redis_last_init_attempt = 0.0 _REDIS_RETRY_INTERVAL_SECONDS = 5.0 @@ -53,8 +60,8 @@ def get_redis(): return None _redis_last_init_attempt = now try: - host = os.getenv("VALKEY_HOST", "127.0.0.1") - port = int(os.getenv("VALKEY_PORT", "6379")) + host = VALKEY_HOST + port = VALKEY_PORT _redis_client = aioredis.Redis(host=host, port=port, decode_responses=True, socket_timeout=1.0) logger.info(f"Valkey client initialized at {host}:{port}") except Exception as e: @@ -177,7 +184,7 @@ def get_langfuse(): _langfuse_client = langfuse.Langfuse( public_key=os.getenv("LANGFUSE_PUBLIC_KEY", ""), secret_key=os.getenv("LANGFUSE_SECRET_KEY", ""), - host=os.getenv("LANGFUSE_HOST", "http://127.0.0.1:3001"), + host=LANGFUSE_URL, release="llm-triage-router-v1", ) logger.info("Langfuse client initialized") @@ -242,7 +249,7 @@ async def push_aggregate_scores(): port = config.get("server", {}).get("port", 5000) router_model_conf = config.get("router", {}).get("router_model", {}) -router_api_base = router_model_conf.get("api_base", "http://127.0.0.1:8080/v1") +router_api_base = router_model_conf.get("api_base", f"{LLAMA_SERVER_URL}/v1") router_api_key = router_model_conf.get("api_key", "local-token") router_model_name = router_model_conf.get("model", "qwen-0.8b-routing") @@ -410,7 +417,7 @@ async def sync_adaptive_router_roster(master_key: str): logger.warning("No LITELLM_MASTER_KEY — skipping roster sync") return headers = {"Authorization": f"Bearer {master_key}", "Content-Type": "application/json"} - admin_url = "http://127.0.0.1:4000" + admin_url = LITELLM_URL try: async with httpx.AsyncClient(timeout=5.0) as client: r = await client.get("https://openrouter.ai/api/v1/models") @@ -567,7 +574,7 @@ async def _register_ollama_models_in_db(master_key: str): logger.warning("No LiteLLM master key provided — skipping Ollama DB registration") return - admin_url = os.getenv("LITELLM_ADMIN_URL", "http://127.0.0.1:4000") + admin_url = LITELLM_URL headers = {"Authorization": f"Bearer {master_key}", "Content-Type": "application/json"} ollama_models = [] @@ -680,10 +687,10 @@ async def lifespan(app: FastAPI): get_http_client() await sync_cooldowns_from_valkey() - litellm_ready_url = "http://127.0.0.1:4000/health/readiness" + litellm_ready_url = f"{LITELLM_URL}/health/readiness" litellm_master_key = os.getenv("LITELLM_MASTER_KEY", "") max_wait = 180 - logger.info(f"⏳ Waiting for LiteLLM on :4000 (max {max_wait}s)...") + logger.info(f"⏳ Waiting for LiteLLM on {LITELLM_URL} (max {max_wait}s)...") for i in range(max_wait): try: async with httpx.AsyncClient(timeout=2.0) as client: @@ -1131,7 +1138,7 @@ async def get_llamacpp_metrics() -> dict: try: async with httpx.AsyncClient(timeout=3.0) as client: # Fetch model list - r = await client.get("http://127.0.0.1:8080/v1/models") + r = await client.get(f"{LLAMA_SERVER_URL}/v1/models") if r.status_code == 200: data = r.json() for m in data.get("data", []): @@ -1146,7 +1153,7 @@ async def get_llamacpp_metrics() -> dict: "n_embd": meta.get("n_embd"), }) # Fetch props for build info - r2 = await client.get("http://127.0.0.1:8080/props") + r2 = await client.get(f"{LLAMA_SERVER_URL}/props") if r2.status_code == 200: props = r2.json() result["build"] = props.get("build_info", "unknown") @@ -1154,7 +1161,7 @@ async def get_llamacpp_metrics() -> dict: loaded = [m["id"] for m in result["models"] if m["status"] == "loaded"] slot_model = loaded[0] if loaded else (result["models"][0]["id"] if result["models"] else None) if slot_model: - r3 = await client.get(f"http://127.0.0.1:8080/slots?model={slot_model}") + r3 = await client.get(f"{LLAMA_SERVER_URL}/slots?model={slot_model}") if r3.status_code == 200: slots_data = r3.json() for s in slots_data: @@ -1330,8 +1337,8 @@ def get_pie_chart_gradient() -> str: @app.api_route("/v1/memory{path:path}", methods=["GET", "POST", "DELETE", "PUT"]) async def proxy_memory(request: Request, path: str = ""): - """Proxies memory API calls to the LiteLLM gateway on port 4000.""" - litellm_base = "http://127.0.0.1:4000/v1/memory" + """Proxies memory API calls to the LiteLLM gateway.""" + litellm_base = f"{LITELLM_URL}/v1/memory" # Resolve the destination URL url = f"{litellm_base}{path}" @@ -1384,7 +1391,7 @@ async def proxy_models(): async with httpx.AsyncClient(timeout=10.0) as client: auth_header = "Bearer " + (litellm_key or "") r = await client.get( - "http://127.0.0.1:4000/v1/models", + f"{LITELLM_URL}/v1/models", headers={"Authorization": auth_header} ) data = r.json() @@ -2131,10 +2138,10 @@ async def get_dashboard_data(): """Fetch all metrics and pre-compute HTML snippets for the dashboard.""" await sync_cooldowns_from_valkey() # 1. Run live health checks - valkey_status = await check_tcp_port("127.0.0.1", 6379) - litellm_status = await check_http_endpoint("http://127.0.0.1:4000/") - llama_server_status = await check_http_endpoint("http://127.0.0.1:8080/health") - langfuse_status = await check_http_endpoint("http://127.0.0.1:3001") + valkey_status = await check_tcp_port(VALKEY_HOST, VALKEY_PORT) + litellm_status = await check_http_endpoint(f"{LITELLM_URL}/") + llama_server_status = await check_http_endpoint(f"{LLAMA_SERVER_URL}/health") + langfuse_status = await check_http_endpoint(LANGFUSE_URL) # 1c. Check Gemini OAuth token status oauth_status = await asyncio.to_thread(get_gemini_oauth_status) @@ -3012,7 +3019,7 @@ async def get_dashboard():

Per-model usage, token consumption & cost are tracked with full trace detail in Langfuse.

- Open Langfuse Observability → + Open Langfuse Observability →
@@ -3077,7 +3084,7 @@ async def get_dashboard():
LiteLLM Proxy - :4000 + {':' + LITELLM_URL.split(':')[-1] if LITELLM_URL.count(':') > 1 else ''}
{'Online' if litellm_status else 'Offline'} @@ -3087,7 +3094,7 @@ async def get_dashboard():
Valkey Cache - :6379 + :{VALKEY_PORT}
{'Online' if valkey_status else 'Offline'} @@ -3097,7 +3104,7 @@ async def get_dashboard():
Llama-Server - :8080 + {':' + LLAMA_SERVER_URL.split(':')[-1] if LLAMA_SERVER_URL.count(':') > 1 else ''}
{'Online' if llama_server_status else 'Offline'} @@ -3107,7 +3114,7 @@ async def get_dashboard():
Langfuse Traces - :3001 + {':' + LANGFUSE_URL.split(':')[-1] if LANGFUSE_URL.count(':') > 1 else ''}
{'Online' if langfuse_status else 'Offline'} @@ -3144,15 +3151,15 @@ async def get_dashboard():
- + {src_badge('LANGFUSE', '#e879f9')} Observability UI → - + {src_badge('LITELLM', '#34d399')} Admin UI → - + {src_badge('LLAMA.CPP', '#fb923c')} Server Router UI → From 3d09707735613964791ed5d5b3bb1465460340f5 Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Wed, 24 Jun 2026 17:37:58 +0200 Subject: [PATCH 2/3] fix(router): address PR 47 review feedback on config & port robustness --- router/agy_proxy.py | 2 +- router/main.py | 37 ++++++++++++++++++++++++++++++------- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/router/agy_proxy.py b/router/agy_proxy.py index ad083b0c..991da897 100644 --- a/router/agy_proxy.py +++ b/router/agy_proxy.py @@ -43,7 +43,7 @@ async def save(self) -> None: logger = logging.getLogger("agy-proxy") -AGY_DAEMON_URL = os.getenv("AGY_DAEMON_URL", "http://127.0.0.1:5005") +AGY_DAEMON_URL = os.getenv("AGY_DAEMON_URL", "http://127.0.0.1:5005").rstrip("/") # In container: mounted from host /home/gpav/.local/bin/agy AGY_BINARY = os.environ.get("AGY_BINARY_PATH", "/usr/local/bin/agy") diff --git a/router/main.py b/router/main.py index 2f11e4a3..b87c8b61 100644 --- a/router/main.py +++ b/router/main.py @@ -39,12 +39,33 @@ except ImportError: try_agy_proxy = None +from urllib.parse import urlparse + # Global Configuration from Environment -LITELLM_URL = os.getenv("LITELLM_ADMIN_URL", "http://127.0.0.1:4000") -LLAMA_SERVER_URL = os.getenv("LLAMA_SERVER_URL", "http://127.0.0.1:8080") -LANGFUSE_URL = os.getenv("LANGFUSE_URL", "http://127.0.0.1:3001") +LITELLM_URL = os.getenv("LITELLM_ADMIN_URL", "http://127.0.0.1:4000").rstrip("/") +LLAMA_SERVER_URL = os.getenv("LLAMA_SERVER_URL", "http://127.0.0.1:8080").rstrip("/") +LANGFUSE_URL = os.getenv("LANGFUSE_URL", "http://127.0.0.1:3001").rstrip("/") VALKEY_HOST = os.getenv("VALKEY_HOST", "127.0.0.1") -VALKEY_PORT = int(os.getenv("VALKEY_PORT", "6379")) + +def _get_valkey_port() -> int: + try: + return int(os.getenv("VALKEY_PORT", "6379")) + except ValueError: + return 6379 + +VALKEY_PORT = _get_valkey_port() + +def _get_port_suffix(url: str) -> str: + try: + port = urlparse(url).port + return f":{port}" if port else "" + except Exception: + return "" + +LITELLM_PORT = _get_port_suffix(LITELLM_URL) +LLAMA_SERVER_PORT = _get_port_suffix(LLAMA_SERVER_URL) +LANGFUSE_PORT = _get_port_suffix(LANGFUSE_URL) + _redis_client = None _redis_last_init_attempt = 0.0 @@ -2463,6 +2484,7 @@ async def get_dashboard_data(): "t_tokens": t_tokens, "llamacpp_models_html": llamacpp_models_html, "llamacpp_slots_html": llamacpp_slots_html, + "llamacpp": llamacpp, "avg_triage_latency_ms": stats["avg_triage_latency_ms"], "avg_proxy_latency_ms": stats["avg_proxy_latency_ms"], "cache_hits": stats["cache_hits"], @@ -2500,6 +2522,7 @@ async def get_dashboard(): t_tokens = data["t_tokens"] llamacpp_models_html = data["llamacpp_models_html"] llamacpp_slots_html = data["llamacpp_slots_html"] + llamacpp = data["llamacpp"] avg_triage_latency_ms = data["avg_triage_latency_ms"] avg_proxy_latency_ms = data["avg_proxy_latency_ms"] cache_hits = data["cache_hits"] @@ -3084,7 +3107,7 @@ async def get_dashboard():
LiteLLM Proxy - {':' + LITELLM_URL.split(':')[-1] if LITELLM_URL.count(':') > 1 else ''} + {LITELLM_PORT}
{'Online' if litellm_status else 'Offline'} @@ -3104,7 +3127,7 @@ async def get_dashboard():
Llama-Server - {':' + LLAMA_SERVER_URL.split(':')[-1] if LLAMA_SERVER_URL.count(':') > 1 else ''} + {LLAMA_SERVER_PORT}
{'Online' if llama_server_status else 'Offline'} @@ -3114,7 +3137,7 @@ async def get_dashboard():
Langfuse Traces - {':' + LANGFUSE_URL.split(':')[-1] if LANGFUSE_URL.count(':') > 1 else ''} + {LANGFUSE_PORT}
{'Online' if langfuse_status else 'Offline'} From 1fd14b91f8b4518723c0b0016842870811c92c46 Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Wed, 24 Jun 2026 17:48:39 +0200 Subject: [PATCH 3/3] fix(router): fallback to default urls when environment variables are empty strings --- router/agy_proxy.py | 2 +- router/main.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/router/agy_proxy.py b/router/agy_proxy.py index 991da897..05368826 100644 --- a/router/agy_proxy.py +++ b/router/agy_proxy.py @@ -43,7 +43,7 @@ async def save(self) -> None: logger = logging.getLogger("agy-proxy") -AGY_DAEMON_URL = os.getenv("AGY_DAEMON_URL", "http://127.0.0.1:5005").rstrip("/") +AGY_DAEMON_URL = (os.getenv("AGY_DAEMON_URL") or "http://127.0.0.1:5005").rstrip("/") # In container: mounted from host /home/gpav/.local/bin/agy AGY_BINARY = os.environ.get("AGY_BINARY_PATH", "/usr/local/bin/agy") diff --git a/router/main.py b/router/main.py index b87c8b61..23a465e4 100644 --- a/router/main.py +++ b/router/main.py @@ -42,14 +42,17 @@ from urllib.parse import urlparse # Global Configuration from Environment -LITELLM_URL = os.getenv("LITELLM_ADMIN_URL", "http://127.0.0.1:4000").rstrip("/") -LLAMA_SERVER_URL = os.getenv("LLAMA_SERVER_URL", "http://127.0.0.1:8080").rstrip("/") -LANGFUSE_URL = os.getenv("LANGFUSE_URL", "http://127.0.0.1:3001").rstrip("/") -VALKEY_HOST = os.getenv("VALKEY_HOST", "127.0.0.1") +LITELLM_URL = (os.getenv("LITELLM_ADMIN_URL") or "http://127.0.0.1:4000").rstrip("/") +LLAMA_SERVER_URL = (os.getenv("LLAMA_SERVER_URL") or "http://127.0.0.1:8080").rstrip("/") +LANGFUSE_URL = (os.getenv("LANGFUSE_URL") or "http://127.0.0.1:3001").rstrip("/") +VALKEY_HOST = os.getenv("VALKEY_HOST") or "127.0.0.1" def _get_valkey_port() -> int: + val = os.getenv("VALKEY_PORT") + if not val: + return 6379 try: - return int(os.getenv("VALKEY_PORT", "6379")) + return int(val) except ValueError: return 6379