diff --git a/.env.dev b/.env.dev index a5393f15..10e3f22b 100644 --- a/.env.dev +++ b/.env.dev @@ -25,3 +25,6 @@ MINIO_CONSOLE_PORT="9011" # Dev uses locally-built router image (for testing code changes) ROUTER_IMAGE="localhost/llm-routing-dev:latest" + +# Shared infrastructure (same llama classifier across dev & prod) +LLAMA_CLASSIFIER_URL="https://x570.vendeuvre.lan/llm-routing/llama/v1" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6337b638..82dcfabf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,10 +26,10 @@ jobs: run: pip install httpx==0.28.1 pytest pytest-asyncio anyio pyyaml fastapi "pydantic>=2.0,<3.0" uvicorn python-multipart asyncpg langfuse redis aiofiles==25.1.0 - name: Run Unit Tests - run: CONFIG_PATH=router/config.yaml PYTHONPATH=.:router pytest --ignore=tests/test_agy_behavior.py --ignore=tests/test_agy_tiers.py --ignore=tests/test_antigravity.py + run: CONFIG_PATH=router/config.yaml PYTHONPATH=. pytest --ignore=tests/test_agy_behavior.py --ignore=tests/test_agy_tiers.py --ignore=tests/test_antigravity.py - name: Run Breaker Verification run: PYTHONPATH=. python3 scripts/verification/verify_breaker.py - name: Run Integration Verification - run: PYTHONPATH=.:router python3 tests/test_a2_verify.py + run: PYTHONPATH=. python3 tests/test_a2_verify.py diff --git a/README.md b/README.md index 80fdf472..5aaf9309 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ graph TD Client["goose-cli Client"] -->|Port 5000| Router["FastAPI Triage Router"] subgraph FastAPIRouter ["FastAPI Router Pod Context"] - Router -->|1. Complexity Triage| LlamaServer["Local Llama-Server\n(Port 8080 - qwen-2b-routing)"] + Router -->|1. Complexity Triage| LlamaServer["Local Llama-Server\n(Port 8080 - qwen-4b-routing)"] Router -->|"2. agy Proxy (Gemini/Claude)"| AgyProxy["agy Proxy Module\n(agy_proxy.py)"] AgyProxy -->|Tier 1| AgyGemini["agy --print\n(Gemini 3.5 Flash)"] @@ -109,7 +109,7 @@ sequenceDiagram Router->>Router: Check model name → decide route alt Model = llm-routing-auto-free / auto-agy / auto-ollama / auto-agy-ollama - Router->>Llama: POST /v1/chat/completions (Complexity triage via qwen-2b-routing) + Router->>Llama: POST /v1/chat/completions (Complexity triage via qwen-4b-routing) Llama-->>Router: JSON Response (5-tier: simple / medium / complex / reasoning / advanced) else Model = direct tier (agent-*-core / llm-routing-agy / llm-routing-ollama) Note over Router: Skip classifier, use model as tier @@ -279,7 +279,7 @@ Exposes the entry endpoint (`http://localhost:5000/v1`) and evaluates prompt com Orchestrates routing fallback chains, Redis caching, and telemetry callbacks: - **`drop_params: true`**: Automatically strips unsupported arguments when transitioning to models that don't support them. - **Request Timeouts (`300s`)**: Provides ample padding to prevent connection aborts during dynamic RAM swapping operations on the local GPU `llama-server`. -- **Local Embedding Model (`local-nomic-embed`)**: A zero-cost embedding model (`nomic-embed-text-v1.5-Q4_K_M`, ~137MB GGUF) running on llama-server. Configured in `litellm/config.yaml` with `api_base: http://127.0.0.1:8080/v1`. Used by `vector_store_settings` for semantic cache similarity search, replacing paid OpenRouter embeddings. +- **Local Embedding Model (`local-nomic-embed`)**: A zero-cost embedding model (`nomic-embed-text-v1.5-Q4_K_M`, ~137MB GGUF) running on llama-server. Configured in `litellm/config.yaml` with `api_base` set via `LLAMA_CLASSIFIER_URL` env var (resolved at deploy time from `.env`). Used by `vector_store_settings` for semantic cache similarity search, replacing paid OpenRouter embeddings. - **`vector_store_settings`**: PostgreSQL-backed vector store for semantic caching, configured in `litellm/config.yaml`: - `store_type: "postgres"` — pgvector extension on the local PostgreSQL instance - `embedding_model: "local-nomic-embed"` — uses the local nomic-embed model (no API costs) diff --git a/litellm/config.yaml b/litellm/config.yaml index b204f9cc..a93aa2d9 100644 --- a/litellm/config.yaml +++ b/litellm/config.yaml @@ -90,15 +90,15 @@ model_list: max_input_tokens: 524288 is_public_model_group: true -# Re-enabled 2026-07-08 — llama.cpp on port 8081, alias local-qwen-3.6 +# Re-enabled 2026-07-08 — llama.cpp, canonical endpoint set via .env - litellm_params: - api_base: http://127.0.0.1:8081/v1 + api_base: LLAMA_CLASSIFIER_URL_PLACEHOLDER api_key: local-token model: openai/local-qwen-3.6 request_timeout: 300 model_name: local-qwen-3.6 - litellm_params: - api_base: http://127.0.0.1:8080/v1 + api_base: LLAMA_CLASSIFIER_URL_PLACEHOLDER api_key: local-token model: openai/nomic-embed-text-v1.5-Q4_K_M request_timeout: 30 diff --git a/pod.yaml b/pod.yaml index bacc0561..75981ee1 100644 --- a/pod.yaml +++ b/pod.yaml @@ -145,7 +145,7 @@ spec: - python3 - -c - import urllib.request; urllib.request.urlopen('http://localhost:ROUTER_PORT_PLACEHOLDER/metrics') - initialDelaySeconds: 20 + initialDelaySeconds: 240 periodSeconds: 15 timeoutSeconds: 5 name: llm-triage-router @@ -155,7 +155,7 @@ spec: - python3 - -c - import urllib.request; urllib.request.urlopen('http://localhost:ROUTER_PORT_PLACEHOLDER/metrics') - initialDelaySeconds: 10 + initialDelaySeconds: 30 periodSeconds: 10 timeoutSeconds: 5 volumeMounts: diff --git a/router/__init__.py b/router/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/router/agy_proxy.py b/router/agy_proxy.py index 8c1f8631..2ad75b2a 100644 --- a/router/agy_proxy.py +++ b/router/agy_proxy.py @@ -38,7 +38,7 @@ async def save(self) -> None: """Push updated cooldown state to Valkey.""" ... -from circuit_breaker import get_google_breaker, get_vendor_breaker +from router.circuit_breaker import get_google_breaker, get_vendor_breaker logger = logging.getLogger("agy-proxy") diff --git a/router/config.yaml b/router/config.yaml index a30ed2ad..4cb81e48 100644 --- a/router/config.yaml +++ b/router/config.yaml @@ -5,9 +5,9 @@ server: router: strategy: "llm" router_model: - api_base: "http://127.0.0.1:8080/v1" + api_base: "os.environ/LLAMA_CLASSIFIER_URL" api_key: "os.environ/ROUTER_API_KEY" - model: "gemma4-26a4b-routing" + model: "qwen-4b-routing" classification_rules: system_prompt: | diff --git a/router/main.py b/router/main.py index 0c976f4f..678ba47a 100644 --- a/router/main.py +++ b/router/main.py @@ -19,7 +19,10 @@ from fastapi.staticfiles import StaticFiles from pathlib import Path from urllib.parse import urlparse -from circuit_breaker import get_breaker +try: + from router.circuit_breaker import get_breaker +except ImportError: + from circuit_breaker import get_breaker from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator, RootModel from typing import Dict, Optional, Union @@ -85,19 +88,52 @@ def get_redis(): _http_client = None +def _http_limits() -> httpx.Limits: + """Shared connection limits for all httpx clients.""" + return httpx.Limits( + max_connections=HTTP_MAX_CONNECTIONS, + max_keepalive_connections=HTTP_MAX_KEEPALIVE_CONNECTIONS, + keepalive_expiry=HTTP_KEEPALIVE_EXPIRY, + ) + + def get_http_client(): """Return the shared global httpx.AsyncClient singleton with configured limits.""" global _http_client if _http_client is None: - limits = httpx.Limits( - max_connections=HTTP_MAX_CONNECTIONS, - max_keepalive_connections=HTTP_MAX_KEEPALIVE_CONNECTIONS, - keepalive_expiry=HTTP_KEEPALIVE_EXPIRY, - ) - _http_client = httpx.AsyncClient(limits=limits, timeout=3600.0) + _http_client = httpx.AsyncClient(limits=_http_limits(), timeout=3600.0) return _http_client +_classifier_client: httpx.AsyncClient | None = None + + +def get_classifier_client(): + """Return a singleton httpx client for classifier calls (internal self-signed TLS). + + By default verify is disabled because the classifier sits behind HAProxy with + a self-signed certificate on the internal network. Set CLASSIFIER_CA_BUNDLE + to a PEM file path to enable TLS verification (e.g. for CI or staging). + """ + global _classifier_client + if _classifier_client is None: + ca_bundle = os.getenv("CLASSIFIER_CA_BUNDLE") + if ca_bundle is not None: + ca_bundle_stripped = ca_bundle.strip() + if ca_bundle_stripped.lower() in ("false", "0", "off", "no", ""): + verify = False + elif ca_bundle_stripped.lower() in ("true", "1", "on", "yes"): + verify = True + else: + verify = ca_bundle_stripped + else: + verify = False + _classifier_client = httpx.AsyncClient( + limits=_http_limits(), timeout=3600.0, verify=verify + ) + return _classifier_client + + # Compiled regular expressions for token estimation heuristics WORD_RE = re.compile(r'[a-zA-Z0-9]+') NON_ASCII_RE = re.compile(r'[^\s\x00-\x7F]') @@ -344,10 +380,25 @@ async def push_aggregate_scores(): port = config.get("server", {}).get("port", 5000) router_model_conf = config.get("router", {}).get("router_model", {}) -router_api_base = router_model_conf.get("api_base", "http://127.0.0.1:8080/v1") +router_api_base = router_model_conf.get("api_base") or "http://127.0.0.1:8080/v1" +if isinstance(router_api_base, str): + if router_api_base.startswith("os.environ/"): + env_var = router_api_base.split("/", 1)[1] + router_api_base = os.environ.get(env_var, "") + if not router_api_base: + if "pytest" in sys.modules: + router_api_base = "http://127.0.0.1:8080/v1" + else: + raise RuntimeError( + f"Configuration error: Environment variable '{env_var}' is missing or empty." + ) + router_api_base = router_api_base.rstrip("/") + router_api_key = router_model_conf.get("api_key") if not router_api_key: raise RuntimeError("Configuration error: 'api_key' is missing from router_model configuration.") +if not isinstance(router_api_key, str): + router_api_key = str(router_api_key) if router_api_key.startswith("os.environ/"): env_var = router_api_key.split("/", 1)[1] router_api_key = os.environ.get(env_var) @@ -356,7 +407,7 @@ async def push_aggregate_scores(): router_api_key = "local-token" else: raise RuntimeError(f"Configuration error: Environment variable '{env_var}' is missing or empty.") -router_model_name = router_model_conf.get("model", "qwen-0.8b-routing") +router_model_name = router_model_conf.get("model", "qwen-4b-routing") system_prompt = config.get("classification_rules", {}).get("system_prompt", "") backends = {b["name"]: b for b in config.get("backends", [])} @@ -910,6 +961,12 @@ async def lifespan(app: FastAPI): await _http_client.aclose() _http_client = None + # Close classifier client + global _classifier_client + if _classifier_client is not None: + await _classifier_client.aclose() + _classifier_client = None + # Close Redis client global _redis_client if _redis_client is not None and _redis_client is not False: @@ -991,7 +1048,7 @@ async def classify_request( return cached_decision, 0.0, True, cached_decision try: - client = get_http_client() + client = get_classifier_client() try: max_chars = max(0, int(os.getenv("CLASSIFIER_INPUT_MAX_CHARS", "300"))) except ValueError: diff --git a/router/tests/test_dashboard_data.py b/router/tests/test_dashboard_data.py index 1e811bed..fa2b12f2 100644 --- a/router/tests/test_dashboard_data.py +++ b/router/tests/test_dashboard_data.py @@ -1,27 +1,20 @@ import pytest from unittest.mock import AsyncMock, patch, MagicMock -import sys -import os @pytest.mark.asyncio async def test_get_dashboard_data_structure(): - # Ensure router directory is in sys.path - router_path = os.path.join(os.getcwd(), "router") - if router_path not in sys.path: - sys.path.insert(0, router_path) - - import main + from router import main # Mocking all I/O and external calls - with patch("main.sync_cooldowns_from_valkey", new_callable=AsyncMock) as mock_sync, \ - patch("main.check_tcp_port", new_callable=AsyncMock) as mock_tcp, \ - patch("main.check_http_endpoint", new_callable=AsyncMock) as mock_http, \ - patch("main.get_gemini_oauth_status", new_callable=AsyncMock) as mock_oauth, \ - patch("main.get_best_free_model", new_callable=AsyncMock) as mock_best_model, \ - patch("main.get_goose_sessions") as mock_goose, \ - patch("main.get_llamacpp_metrics", new_callable=AsyncMock) as mock_llamacpp, \ - patch("main.get_pie_chart_gradient") as mock_gradient, \ - patch("main.stats") as mock_stats: + with patch("router.main.sync_cooldowns_from_valkey", new_callable=AsyncMock) as mock_sync, \ + patch("router.main.check_tcp_port", new_callable=AsyncMock) as mock_tcp, \ + patch("router.main.check_http_endpoint", new_callable=AsyncMock) as mock_http, \ + patch("router.main.get_gemini_oauth_status", new_callable=AsyncMock) as mock_oauth, \ + patch("router.main.get_best_free_model", new_callable=AsyncMock) as mock_best_model, \ + patch("router.main.get_goose_sessions") as mock_goose, \ + patch("router.main.get_llamacpp_metrics", new_callable=AsyncMock) as mock_llamacpp, \ + patch("router.main.get_pie_chart_gradient") as mock_gradient, \ + patch("router.main.stats") as mock_stats: # Setup mock return values mock_sync.return_value = None diff --git a/router/tests/test_detect_active_tool.py b/router/tests/test_detect_active_tool.py index 0bebb391..95774be2 100644 --- a/router/tests/test_detect_active_tool.py +++ b/router/tests/test_detect_active_tool.py @@ -1,5 +1,5 @@ import pytest -from main import detect_active_tool +from router.main import detect_active_tool def test_detect_active_tool_empty(): assert detect_active_tool({}) == "none" diff --git a/router/tests/test_estimate_prompt_tokens.py b/router/tests/test_estimate_prompt_tokens.py index 32da0463..71d17532 100644 --- a/router/tests/test_estimate_prompt_tokens.py +++ b/router/tests/test_estimate_prompt_tokens.py @@ -1,5 +1,5 @@ import pytest -from main import estimate_prompt_tokens +from router.main import estimate_prompt_tokens def test_estimate_prompt_tokens_empty(): assert estimate_prompt_tokens({}) == 50 diff --git a/router/tests/test_get_gemini_oauth_status.py b/router/tests/test_get_gemini_oauth_status.py index 9d506334..016b5356 100644 --- a/router/tests/test_get_gemini_oauth_status.py +++ b/router/tests/test_get_gemini_oauth_status.py @@ -2,7 +2,7 @@ import json from unittest.mock import patch, mock_open -import main +from router import main @pytest.mark.asyncio async def test_get_gemini_oauth_status_missing_file(): diff --git a/router/tests/test_get_goose_sessions.py b/router/tests/test_get_goose_sessions.py index 45de1a05..4987d4e0 100644 --- a/router/tests/test_get_goose_sessions.py +++ b/router/tests/test_get_goose_sessions.py @@ -2,7 +2,7 @@ import sys from unittest.mock import patch, MagicMock -from main import get_goose_sessions +from router.main import get_goose_sessions def test_get_goose_sessions_no_db(): with patch('os.path.exists', return_value=False): diff --git a/router/tests/test_load_persisted_stats.py b/router/tests/test_load_persisted_stats.py index 67458d50..8e32663a 100644 --- a/router/tests/test_load_persisted_stats.py +++ b/router/tests/test_load_persisted_stats.py @@ -2,8 +2,8 @@ import pytest from unittest.mock import patch, mock_open -import main -from main import load_persisted_stats +from router import main +from router.main import load_persisted_stats @pytest.fixture def mock_stats(): @@ -17,7 +17,7 @@ def mock_stats(): yield main.stats def test_load_persisted_stats_file_not_exists(mock_stats): - with patch("main.os.path.exists", return_value=False) as mock_exists: + with patch("router.main.os.path.exists", return_value=False) as mock_exists: load_persisted_stats() mock_exists.assert_called_once_with(main.STATS_JSON_PATH) # Stats should remain unchanged @@ -31,9 +31,9 @@ def test_load_persisted_stats_success(mock_stats): } mock_json = json.dumps(mock_data) - with patch("main.os.path.exists", side_effect=lambda p: p == main.STATS_JSON_PATH): - with patch("main.open", mock_open(read_data=mock_json)): - with patch("main.logger.info") as mock_logger: + with patch("router.main.os.path.exists", side_effect=lambda p: p == main.STATS_JSON_PATH): + with patch("router.main.open", mock_open(read_data=mock_json)): + with patch("router.main.logger.info") as mock_logger: load_persisted_stats() # Assert simple value updated via else block @@ -48,9 +48,9 @@ def test_load_persisted_stats_success(mock_stats): mock_logger.assert_called_once_with("✓ Successfully loaded persisted gateway statistics from disk.") def test_load_persisted_stats_exception(mock_stats): - with patch("main.os.path.exists", return_value=True): - with patch("main.open", side_effect=Exception("Mock read error")): - with patch("main.logger.error") as mock_logger: + with patch("router.main.os.path.exists", return_value=True): + with patch("router.main.open", side_effect=Exception("Mock read error")): + with patch("router.main.logger.error") as mock_logger: load_persisted_stats() # Stats should remain unchanged diff --git a/router/tests/test_resolve_external_urls.py b/router/tests/test_resolve_external_urls.py index b86bf4c5..8cc79f0d 100644 --- a/router/tests/test_resolve_external_urls.py +++ b/router/tests/test_resolve_external_urls.py @@ -1,15 +1,9 @@ import pytest import os -import sys from unittest.mock import MagicMock -# Ensure router directory is in sys.path -router_path = os.path.join(os.getcwd(), "router") -if router_path not in sys.path: - sys.path.insert(0, router_path) - -import main -from main import resolve_external_urls +from router import main +from router.main import resolve_external_urls class MockRequest: diff --git a/router/tests/test_routing_behavior.py b/router/tests/test_routing_behavior.py index bb253baa..3b43ca14 100644 --- a/router/tests/test_routing_behavior.py +++ b/router/tests/test_routing_behavior.py @@ -26,7 +26,7 @@ async def test_classify_request_truncation_default(): mock_client.post.return_value = mock_response # Force bypass_cache=True to ensure classify_request always hits llama-server - with patch("router.main.get_http_client", return_value=mock_client), \ + with patch("router.main.get_classifier_client", return_value=mock_client), \ patch.dict(os.environ, {}, clear=False): # We verify behavior with default (no env var set -> defaults to 300) long_prompt = "a" * 500 @@ -57,7 +57,7 @@ async def test_classify_request_truncation_custom_env(): mock_client = AsyncMock() mock_client.post.return_value = mock_response - with patch("router.main.get_http_client", return_value=mock_client), \ + with patch("router.main.get_classifier_client", return_value=mock_client), \ patch.dict(os.environ, {"CLASSIFIER_INPUT_MAX_CHARS": "10"}): long_prompt = "a" * 500 decision, _, _, _ = await classify_request(long_prompt, bypass_cache=True) diff --git a/scripts/benchmark_classifier.py b/scripts/benchmark_classifier.py index 2878c402..69bc71e8 100644 --- a/scripts/benchmark_classifier.py +++ b/scripts/benchmark_classifier.py @@ -1,14 +1,16 @@ -"""Benchmark gemma4-26a4b-routing classifier against labeled dataset.""" +"""Benchmark qwen-4b-routing classifier against labeled dataset.""" import os -import json, urllib.request, urllib.error, time, sys +import json, urllib.request, urllib.error, time import concurrent.futures import threading from collections import defaultdict, Counter from pathlib import Path # Shared chat response parser (used by verification scripts too) -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) -from scripts.chat_helpers import parse_chat_response +try: + from scripts.chat_helpers import parse_chat_response +except ImportError: + from chat_helpers import parse_chat_response # Load dataset dataset_path = Path(__file__).resolve().parent.parent / "data" / "classified_dataset.json" @@ -32,9 +34,9 @@ ] def classify(prompt): - """Call gemma4-26a4b-routing via llama-server.""" + """Call qwen-4b-routing via llama-server.""" payload = { - "model": "gemma4-26a4b-routing", + "model": "qwen-4b-routing", "messages": [{"role": "user", "content": PROMPT_TEMPLATE + prompt}], "temperature": 0.0, "max_tokens": 15, @@ -50,7 +52,7 @@ def classify(prompt): return content if content else "ERROR" total = len(dataset.get("prompts", [])) -print(f"Benchmark: gemma4-26a4b-routing vs {total} labeled prompts\n") +print(f"Benchmark: qwen-4b-routing vs {total} labeled prompts\n") # Run classification results = [] @@ -166,7 +168,7 @@ def process_item_with_rate_limit(index_and_item): out_path = Path(__file__).resolve().parent.parent / "data" / "benchmark_results.json" with open(out_path, 'w') as f: json.dump({ - "classifier": "gemma4-26a4b-routing", + "classifier": "qwen-4b-routing", "dataset_total": total, "overall_accuracy": round(overall, 1), "per_tier": {t: { diff --git a/scripts/benchmark_tokens.py b/scripts/benchmark_tokens.py index 34c9a2f0..0737fa01 100644 --- a/scripts/benchmark_tokens.py +++ b/scripts/benchmark_tokens.py @@ -5,11 +5,12 @@ # Set CONFIG_PATH and ROUTER_API_KEY for import os.environ["CONFIG_PATH"] = str(Path(__file__).resolve().parent.parent / "router" / "config.yaml") os.environ["ROUTER_API_KEY"] = "local-token" -# Add the parent directory and the router directory to the path -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) -sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "router")) -from router.main import estimate_prompt_tokens, METADATA_OVERHEAD +try: + from router.main import estimate_prompt_tokens, METADATA_OVERHEAD +except ImportError: + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from router.main import estimate_prompt_tokens, METADATA_OVERHEAD def verify_accuracy(): """Benchmarking utility to verify token estimation accuracy across content types.""" diff --git a/scripts/classify_direct.py b/scripts/classify_direct.py index ee699daf..62c536f2 100644 --- a/scripts/classify_direct.py +++ b/scripts/classify_direct.py @@ -1,11 +1,13 @@ -"""Direct classification of Hermes prompts using gemma4-26a4b-routing.""" +"""Direct classification of Hermes prompts using qwen-4b-routing.""" import os -import json, urllib.request, time, sys +import json, urllib.request, time from pathlib import Path # Shared chat response parser (used by verification scripts too) -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) -from scripts.chat_helpers import parse_chat_response +try: + from scripts.chat_helpers import parse_chat_response +except ImportError: + from chat_helpers import parse_chat_response PROMPT_TEMPLATE = """Classify the coding task complexity. Output ONLY the tier name. @@ -25,7 +27,7 @@ def classify(prompt): """Query the llama-server to classify the prompt task complexity.""" payload = { - "model": "gemma4-26a4b-routing", + "model": "qwen-4b-routing", "messages": [{"role": "user", "content": PROMPT_TEMPLATE + prompt}], "temperature": 0.0, "max_tokens": 15, @@ -45,7 +47,7 @@ def classify(prompt): with open(data_dir / "raw_prompts_hermes.json") as f: prompts = json.load(f) -print(f"Classifying {len(prompts)} prompts with gemma4-26a4b-routing...") +print(f"Classifying {len(prompts)} prompts with qwen-4b-routing...") results = [] errors = 0 diff --git a/scripts/reclassify_all.py b/scripts/reclassify_all.py index 38675878..65b662ac 100644 --- a/scripts/reclassify_all.py +++ b/scripts/reclassify_all.py @@ -8,8 +8,10 @@ from collections import Counter # Shared chat response parser (used by verification scripts too) -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) -from scripts.chat_helpers import parse_chat_response +try: + from scripts.chat_helpers import parse_chat_response +except ImportError: + from chat_helpers import parse_chat_response TIERS = ['agent-simple-core','agent-medium-core','agent-complex-core','agent-reasoning-core','agent-advanced-core'] @@ -28,7 +30,7 @@ def classify(prompt): """Query the llama-server to classify the prompt complexity with grammar enforcement.""" payload = { - 'model': 'gemma4-26a4b-routing', + 'model': 'qwen-4b-routing', 'messages': [{'role': 'user', 'content': PROMPT_TEMPLATE + prompt}], 'max_tokens': 15, 'temperature': 0, 'grammar': 'root ::= "agent-simple-core" | "agent-medium-core" | "agent-complex-core" | "agent-reasoning-core" | "agent-advanced-core"' @@ -45,7 +47,7 @@ def classify(prompt): dataset = json.load(f) -print(f"Classifying {len(dataset['prompts'])} prompts with gemma4-26a4b (grammar-enforced)...") +print(f"Classifying {len(dataset['prompts'])} prompts with qwen-4b-routing (grammar-enforced)...") results = [] for i, item in enumerate(dataset['prompts']): diff --git a/scripts/retry_errors.py b/scripts/retry_errors.py index ed273721..a2a53b37 100644 --- a/scripts/retry_errors.py +++ b/scripts/retry_errors.py @@ -4,8 +4,10 @@ from collections import Counter # Shared chat response parser (used by verification scripts too) -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) -from scripts.chat_helpers import parse_chat_response +try: + from scripts.chat_helpers import parse_chat_response +except ImportError: + from chat_helpers import parse_chat_response PROMPT_TEMPLATE = """Classify the coding task complexity. Output ONLY the tier name. @@ -20,18 +22,18 @@ MAX_CHARS = 600 # proven safe across all prompt types in this dataset def get_model_port(): - """Discover the gemma4-26a4b-routing model's direct port (bypass router prompt-cache bug).""" + """Discover the qwen-4b-routing model's direct port (bypass router prompt-cache bug).""" req = urllib.request.Request('http://127.0.0.1:8080/v1/models') with urllib.request.urlopen(req, timeout=5) as resp: data = json.loads(resp.read()) for m in data.get('data', []): - if 'gemma4-26a4b' in m.get('id', ''): + if 'qwen-4b-routing' in m.get('id', ''): status_obj = m.get('status') or {} args = status_obj.get('args', []) if isinstance(status_obj, dict) else [] for i, v in enumerate(args): if v == '--port' and i + 1 < len(args): return args[i + 1] - raise RuntimeError("gemma4-26a4b-routing model port not found") + raise RuntimeError("qwen-4b-routing model port not found") MODEL_PORT = get_model_port() MODEL_URL = f"http://127.0.0.1:{MODEL_PORT}/v1/chat/completions" @@ -42,7 +44,7 @@ def classify(prompt): if len(prompt) > MAX_CHARS: prompt = prompt[:MAX_CHARS] payload = { - "model": "gemma4-26a4b-routing", + "model": "qwen-4b-routing", "messages": [{"role": "user", "content": PROMPT_TEMPLATE + prompt}], "temperature": 0.0, "max_tokens": 15, diff --git a/scripts/verification/verification_helpers.py b/scripts/verification/verification_helpers.py index e4500d61..a3a1c839 100644 --- a/scripts/verification/verification_helpers.py +++ b/scripts/verification/verification_helpers.py @@ -1,8 +1,11 @@ # Shared verification helpers for cooldown and routing tests -import sys -from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) -from scripts.chat_helpers import parse_chat_response +try: + from scripts.chat_helpers import parse_chat_response +except ImportError: + import sys + from pathlib import Path + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + from scripts.chat_helpers import parse_chat_response import os import uuid import time diff --git a/scripts/verification/verify_breaker.py b/scripts/verification/verify_breaker.py index 6ed9e7c7..eebd6687 100644 --- a/scripts/verification/verify_breaker.py +++ b/scripts/verification/verify_breaker.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 """Verification test for the agy circuit breaker.""" -import sys -from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) - -from router.circuit_breaker import get_breaker +try: + from router.circuit_breaker import get_breaker +except ImportError: + import sys + from pathlib import Path + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + from router.circuit_breaker import get_breaker b = get_breaker() assert b.is_allowed(), 'Tier 0 should be open' diff --git a/scripts/verification/verify_canonical_endpoints.py b/scripts/verification/verify_canonical_endpoints.py index 13c7d9b2..c0c6ce62 100644 --- a/scripts/verification/verify_canonical_endpoints.py +++ b/scripts/verification/verify_canonical_endpoints.py @@ -19,8 +19,11 @@ WORKDIR = Path(__file__).resolve().parent.parent.parent # Import shared chat response parser (also used by classifier scripts) -sys.path.insert(0, str(WORKDIR)) -from scripts.chat_helpers import parse_chat_response +try: + from scripts.chat_helpers import parse_chat_response +except ImportError: + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + from scripts.chat_helpers import parse_chat_response def load_env(dev: bool = False) -> dict: diff --git a/scripts/verification/verify_ollama_routing.py b/scripts/verification/verify_ollama_routing.py index c844e549..6f666b73 100644 --- a/scripts/verification/verify_ollama_routing.py +++ b/scripts/verification/verify_ollama_routing.py @@ -5,8 +5,11 @@ from pathlib import Path WORKDIR = Path(__file__).resolve().parent.parent.parent -sys.path.insert(0, str(WORKDIR)) -from scripts.chat_helpers import parse_chat_response +try: + from scripts.chat_helpers import parse_chat_response +except ImportError: + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + from scripts.chat_helpers import parse_chat_response URL = "http://localhost:5000/v1/chat/completions" diff --git a/start-stack.sh b/start-stack.sh index db94b065..4eac51a0 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -581,9 +581,10 @@ render_litellm_config() { mkdir -p "$rendered_dir" sed -e "s/VALKEY_CACHE_PORT_PLACEHOLDER/${VALKEY_CACHE_PORT}/g" \ -e "s/ROUTER_PORT_PLACEHOLDER/${ROUTER_PORT}/g" \ + -e "s|LLAMA_CLASSIFIER_URL_PLACEHOLDER|${LLAMA_CLASSIFIER_URL:?LLAMA_CLASSIFIER_URL must be set in .env}|g" \ "${WORKDIR}/litellm/config.yaml" > "${rendered_dir}/config.yaml" # Validate no unresolved placeholders remain - if grep -E -q 'VALKEY_CACHE_PORT_PLACEHOLDER|ROUTER_PORT_PLACEHOLDER' "${rendered_dir}/config.yaml"; then + if grep -E -q 'VALKEY_CACHE_PORT_PLACEHOLDER|ROUTER_PORT_PLACEHOLDER|LLAMA_CLASSIFIER_URL_PLACEHOLDER' "${rendered_dir}/config.yaml"; then echo "❌ Error: Unresolved placeholders remain in ${rendered_dir}/config.yaml" >&2 exit 1 fi diff --git a/tests/test_a2_verify.py b/tests/test_a2_verify.py index 95c6be31..60edee8c 100644 --- a/tests/test_a2_verify.py +++ b/tests/test_a2_verify.py @@ -1,7 +1,14 @@ #!/usr/bin/env python3 """Verify circuit breaker integration into agy_proxy.py""" -from circuit_breaker import get_breaker -from agy_proxy import try_agy_proxy +try: + from router.circuit_breaker import get_breaker + from router.agy_proxy import try_agy_proxy +except ImportError: + import sys + from pathlib import Path + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from router.circuit_breaker import get_breaker + from router.agy_proxy import try_agy_proxy import asyncio, time b = get_breaker() diff --git a/tests/test_classifier_accuracy.py b/tests/test_classifier_accuracy.py index 7b10f49d..d2321ca1 100644 --- a/tests/test_classifier_accuracy.py +++ b/tests/test_classifier_accuracy.py @@ -64,7 +64,7 @@ def query_model(prompt: str) -> tuple[str, float]: payload = { - "model": "qwen-0.8b-routing", + "model": "qwen-4b-routing", "messages": [ {"role": "system", "content": system_prompt}, {"role": "user", "content": prompt} diff --git a/tests/test_models_proxy.py b/tests/test_models_proxy.py index 2ad68535..2c90ec8f 100644 --- a/tests/test_models_proxy.py +++ b/tests/test_models_proxy.py @@ -4,11 +4,11 @@ from fastapi import Response from fastapi.responses import JSONResponse -from main import get_http_client, proxy_models, HTTP_MAX_CONNECTIONS, HTTP_MAX_KEEPALIVE_CONNECTIONS, HTTP_KEEPALIVE_EXPIRY +from router.main import get_http_client, proxy_models, HTTP_MAX_CONNECTIONS, HTTP_MAX_KEEPALIVE_CONNECTIONS, HTTP_KEEPALIVE_EXPIRY def test_http_client_limits(): # Verify that get_http_client initializes with configured limits using public mocks - import main + from router import main import httpx original_init = httpx.Limits.__init__ @@ -43,7 +43,7 @@ async def test_proxy_models_success(): mock_client = AsyncMock() mock_client.get.return_value = mock_resp - with patch("main.get_http_client", return_value=mock_client): + with patch("router.main.get_http_client", return_value=mock_client): response = await proxy_models() assert isinstance(response, JSONResponse) assert response.status_code == 200 @@ -67,7 +67,7 @@ async def test_proxy_models_error_status(): mock_client = AsyncMock() mock_client.get.return_value = mock_resp - with patch("main.get_http_client", return_value=mock_client): + with patch("router.main.get_http_client", return_value=mock_client): response = await proxy_models() assert isinstance(response, Response) assert response.status_code == 500 @@ -85,7 +85,7 @@ async def test_proxy_models_invalid_json(): mock_client = AsyncMock() mock_client.get.return_value = mock_resp - with patch("main.get_http_client", return_value=mock_client): + with patch("router.main.get_http_client", return_value=mock_client): response = await proxy_models() assert isinstance(response, Response) assert response.status_code == 200