From e34cd8169b6d97d3d73c00fcae0a0eef39d43f66 Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Wed, 1 Jul 2026 00:04:09 +0200 Subject: [PATCH 1/2] Refactor: Reorganize repository structure by moving scripts and tests to designated folders --- .github/workflows/test.yml | 6 +- README.md | 24 +- router/test_memory_mcp.py | 131 ------- router/tests/test_agy_proxy.py | 4 + router/tests/test_memory_mcp.py | 323 ++++++++++++++++++ scripts/README.md | 49 ++- get_pr_status.py => scripts/get_pr_status.py | 0 .../host_agy_daemon.py | 0 .../sync_gemini_token.py | 0 .../test_quota_reset.sh | 0 .../verification/verify_breaker.py | 2 +- watch_quota.sh => scripts/watch_quota.sh | 3 +- start-stack.sh | 2 +- test_memory_mcp.py | 167 --------- test_a2_verify.py => tests/test_a2_verify.py | 2 +- .../test_agy_behavior.py | 0 test_agy_tiers.py => tests/test_agy_tiers.py | 0 .../test_antigravity.py | 0 .../test_atomic_write.py | 0 .../test_check_http_endpoint.py | 0 .../test_circuit_breaker.py | 2 +- .../test_classifier_accuracy.py | 0 .../test_compute_free_model_score.py | 0 .../test_host_agy_daemon.py | 4 + .../test_map_tool_to_category.py | 0 .../test_models_proxy.py | 2 +- .../test_pie_chart_gradient.py | 0 .../test_record_tool_usage.py | 0 test_src_badge.py => tests/test_src_badge.py | 0 .../test_stream_latency.py | 0 .../test_sync_gemini_token.py | 3 + triage_upgrade_plan.md | 4 +- 32 files changed, 396 insertions(+), 332 deletions(-) delete mode 100644 router/test_memory_mcp.py create mode 100644 router/tests/test_memory_mcp.py rename get_pr_status.py => scripts/get_pr_status.py (100%) rename host_agy_daemon.py => scripts/host_agy_daemon.py (100%) rename sync_gemini_token.py => scripts/sync_gemini_token.py (100%) rename test_quota_reset.sh => scripts/test_quota_reset.sh (100%) rename verify_breaker.py => scripts/verification/verify_breaker.py (91%) rename watch_quota.sh => scripts/watch_quota.sh (92%) delete mode 100644 test_memory_mcp.py rename test_a2_verify.py => tests/test_a2_verify.py (88%) rename test_agy_behavior.py => tests/test_agy_behavior.py (100%) rename test_agy_tiers.py => tests/test_agy_tiers.py (100%) rename test_antigravity.py => tests/test_antigravity.py (100%) rename test_atomic_write.py => tests/test_atomic_write.py (100%) rename test_check_http_endpoint.py => tests/test_check_http_endpoint.py (100%) rename test_circuit_breaker.py => tests/test_circuit_breaker.py (99%) rename test_classifier_accuracy.py => tests/test_classifier_accuracy.py (100%) rename test_compute_free_model_score.py => tests/test_compute_free_model_score.py (100%) rename test_host_agy_daemon.py => tests/test_host_agy_daemon.py (99%) rename test_map_tool_to_category.py => tests/test_map_tool_to_category.py (100%) rename test_models_proxy.py => tests/test_models_proxy.py (97%) rename test_pie_chart_gradient.py => tests/test_pie_chart_gradient.py (100%) rename test_record_tool_usage.py => tests/test_record_tool_usage.py (100%) rename test_src_badge.py => tests/test_src_badge.py (100%) rename test_stream_latency.py => tests/test_stream_latency.py (100%) rename test_sync_gemini_token.py => tests/test_sync_gemini_token.py (98%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0b8e5878..4250afa9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,10 +26,10 @@ jobs: run: pip install httpx==0.28.1 pytest pytest-asyncio anyio pyyaml fastapi "pydantic>=2.0,<3.0" uvicorn python-multipart asyncpg langfuse redis - name: Run Unit Tests - run: CONFIG_PATH=router/config.yaml PYTHONPATH=.:router pytest --ignore=test_agy_behavior.py --ignore=test_agy_tiers.py + run: CONFIG_PATH=router/config.yaml PYTHONPATH=.:router pytest --ignore=tests/test_agy_behavior.py --ignore=tests/test_agy_tiers.py - name: Run Breaker Verification - run: python3 verify_breaker.py + run: python3 scripts/verification/verify_breaker.py - name: Run Integration Verification - run: python3 test_a2_verify.py + run: python3 tests/test_a2_verify.py diff --git a/README.md b/README.md index 70b66663..4ecbf930 100644 --- a/README.md +++ b/README.md @@ -228,16 +228,26 @@ All configurations, automation scripts, and databases are self-contained within │ ├── agy_proxy.py # 3-tier agy fallback with session continuation │ ├── circuit_breaker.py # Exponential cooldown breaker for agy proxy │ └── memory_mcp.py # MCP bridge server for Goose memory integration -├── scripts/ -│ └── backup.sh # Database backup with pg_isready retry logic +├── scripts/ # Automation, maintenance, and verification scripts +│ ├── backup.sh # Database backup with pg_isready retry logic +│ ├── host_agy_daemon.py # Real-time PTY-based streaming daemon for agy +│ ├── sync_gemini_token.py # Extraction/sync script for keyring credentials +│ ├── get_pr_status.py # PR state helper +│ ├── watch_quota.sh # Quota reset watcher +│ ├── test_quota_reset.sh # Quota reset test simulator +│ └── verification/ # Routing & cooldown verification tests +│ ├── verify_breaker.py # Circuit breaker verification +│ └── ... +├── tests/ # Integration & unit test suite +│ ├── test_agy_tiers.py # agy proxy model tier test suite +│ ├── test_classifier_accuracy.py # Classifier accuracy benchmark +│ └── ... ├── backups/ # Timestamped PostgreSQL dumps + config snapshots ├── valkey-data/ # [Git Ignored] Persistent memory volumes for Valkey Cache ├── postgres-data/ # [Git Ignored] Persistent tables for PostgreSQL ├── clickhouse-data/ # [Git Ignored] Persistent traces for Langfuse v3 ├── valkey-lf-data/ # [Git Ignored] Persistent job queues for Langfuse v3 -├── minio-data/ # [Git Ignored] S3-compatible event storage for Langfuse v3 -├── test_agy_tiers.py # agy proxy model tier test suite -└── test_classifier_accuracy.py # Classifier accuracy benchmark +└── minio-data/ # [Git Ignored] S3-compatible event storage for Langfuse v3 ``` --- @@ -719,7 +729,7 @@ agy --print "First message" # creates conversation agy --conversation --print "Follow-up" # continues same session # Run the full tier test suite -python3 test_agy_tiers.py +python3 tests/test_agy_tiers.py ``` ### 9b. Streaming & Concurrency Optimizations @@ -727,7 +737,7 @@ python3 test_agy_tiers.py To support production agentic environments (such as `goose-cli` or similar tools) that require low-latency streaming and high concurrent throughput, the following components were introduced: #### 1. Real-Time PTY-Based Streaming Bridge for `agy` Response -To support low-latency streaming for agent clients (such as `goose-cli`), the host-side `host_agy_daemon.py` runs `agy --print` inside a pseudo-terminal (PTY) using `pty.openpty()`. +To support low-latency streaming for agent clients (such as `goose-cli`), the host-side `scripts/host_agy_daemon.py` runs `agy --print` inside a pseudo-terminal (PTY) using `pty.openpty()`. * Running `agy` inside a PTY disables internal buffering, forcing it to write generated characters/lines progressively. * The host daemon streams these chunks in real-time as `application/x-ndjson` lines to the Triage Router. * The Triage Router immediately transforms these incoming chunks into standard OpenAI Server-Sent Event (SSE) packets and yields them to the client. This results in a true, low-latency stream with minimal Time-To-First-Token (TTFT) and eliminates synthetic buffering. diff --git a/router/test_memory_mcp.py b/router/test_memory_mcp.py deleted file mode 100644 index 24d23a34..00000000 --- a/router/test_memory_mcp.py +++ /dev/null @@ -1,131 +0,0 @@ -import time -import re -import sys -import json -from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent)) -from memory_mcp import _make_key, SCOPE_GLOBAL, SCOPE_LOCAL, PREFIX, _memory_value, _parse_memory_value - -def test_make_key_global(): - """Test generating a key for global scope.""" - category = "test_cat" - data = "test_data" - - before_ts = int(time.time() * 1000) - key = _make_key(category, True, data) - after_ts = int(time.time() * 1000) - - # Expected format: f"{PREFIX}:{scope}:{category}::{ts}:{h}" - parts = key.split(":") - - assert key.startswith(f"{PREFIX}:{SCOPE_GLOBAL}:{category}::") - - # Extract timestamp and hash part - # Format is memory:global:test_cat::1717612345:a1b2c3d4... - match = re.match(rf"^{PREFIX}:{SCOPE_GLOBAL}:{category}::(\d+):([a-f0-9]+)$", key) - assert match is not None, f"Key {key} does not match expected format" - - ts = int(match.group(1)) - h = match.group(2) - - assert before_ts <= ts <= after_ts - assert len(h) == 20 - -def test_make_key_local(): - """Test generating a key for local scope.""" - category = "another_cat" - data = "more_data" - - before_ts = int(time.time() * 1000) - key = _make_key(category, False, data) - after_ts = int(time.time() * 1000) - - assert key.startswith(f"{PREFIX}:{SCOPE_LOCAL}:{category}::") - - match = re.match(rf"^{PREFIX}:{SCOPE_LOCAL}:{category}::(\d+):([a-f0-9]+)$", key) - assert match is not None, f"Key {key} does not match expected format" - - ts = int(match.group(1)) - h = match.group(2) - - assert before_ts <= ts <= after_ts - assert len(h) == 20 - - -def test_make_key_formatting_details(monkeypatch): - """Test the exact output formatting of _make_key using deterministic BLAKE2b.""" - # Mock time.time to return a predictable float so ts = 1620000000123 - monkeypatch.setattr(time, "time", lambda: 1620000000.123) - - # data="data", ts=1620000000123 -> blake2b("data1620000000123", digest_size=10) -> 5e5dad075ca7764bc51f - key1 = _make_key("cat1", True, "data") - assert key1 == f"{PREFIX}:{SCOPE_GLOBAL}:cat1::1620000000123:5e5dad075ca7764bc51f" - - key2 = _make_key("cat2", False, "data") - assert key2 == f"{PREFIX}:{SCOPE_LOCAL}:cat2::1620000000123:5e5dad075ca7764bc51f" - - -def test_make_key_determinism_and_uniqueness(): - """Test determinism for same inputs within same timestamp, and uniqueness across timestamps/data.""" - category = "test_cat" - data1 = "data1" - data2 = "data2" - - key1 = _make_key(category, True, data1) - time.sleep(0.002) - key2 = _make_key(category, True, data1) - key3 = _make_key(category, True, data2) - - # Uniqueness across data - assert key1 != key3 - - # Check determinism: if the timestamp parts are the same, the keys should be identical - ts1 = key1.split("::")[1].split(":")[0] - ts2 = key2.split("::")[1].split(":")[0] - if ts1 == ts2: - assert key1 == key2 - else: - # If timestamp is different, keys should be different - assert key1 != key2 - -def test_memory_value_happy_path(): - """Test _memory_value with standard data and tags.""" - result = _memory_value("some data", ["tag1", "tag2"]) - parsed = json.loads(result) - assert parsed == {"data": "some data", "tags": ["tag1", "tag2"]} - -def test_memory_value_missing_tags(): - """Test _memory_value when tags is None.""" - result = _memory_value("some data", None) - parsed = json.loads(result) - assert parsed == {"data": "some data", "tags": []} - -def test_memory_value_unicode(): - """Test _memory_value properly handles unicode and ensure_ascii=False.""" - result = _memory_value("こんにちは", ["世界"]) - # If ensure_ascii=False, the unicode characters shouldn't be escaped (no \uXXXX) - assert "こんにちは" in result - assert "世界" in result - parsed = json.loads(result) - assert parsed == {"data": "こんにちは", "tags": ["世界"]} - -def test_parse_memory_value_success(): - """Test _parse_memory_value successfully decodes valid JSON.""" - raw = '{"data": "info", "tags": ["a"]}' - result = _parse_memory_value(raw) - assert result == {"data": "info", "tags": ["a"]} - -def test_parse_memory_value_invalid_json(): - """Test _parse_memory_value with invalid JSON.""" - result = _parse_memory_value("{invalid_json:") - assert result == {"data": "{invalid_json:", "tags": []} - -def test_parse_memory_value_type_error(): - """Test _parse_memory_value with TypeError (e.g. passing None).""" - result = _parse_memory_value(None) - assert result == {"data": None, "tags": []} - -def test_parse_memory_value_invalid_json_string(): - """Test _parse_memory_value with invalid JSON string.""" - result = _parse_memory_value("this is not a valid json string") - assert result == {"data": "this is not a valid json string", "tags": []} diff --git a/router/tests/test_agy_proxy.py b/router/tests/test_agy_proxy.py index 404059eb..174c9898 100644 --- a/router/tests/test_agy_proxy.py +++ b/router/tests/test_agy_proxy.py @@ -1,3 +1,7 @@ +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + from unittest.mock import patch, MagicMock from router.agy_proxy import _wrap_response, _is_quota_exhausted diff --git a/router/tests/test_memory_mcp.py b/router/tests/test_memory_mcp.py new file mode 100644 index 00000000..3bfd0ca7 --- /dev/null +++ b/router/tests/test_memory_mcp.py @@ -0,0 +1,323 @@ +import json +import re +import sys +import time +from pathlib import Path + +# Inject router path to allow importing memory_mcp directly +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import pytest +from memory_mcp import ( + PREFIX, + SCOPE_GLOBAL, + SCOPE_LOCAL, + _make_key, + _memory_entry, + _memory_value, + _parse_key, + _parse_memory_value, +) + + +# ===================================================================== +# Tests from router/test_memory_mcp.py +# ===================================================================== + +def test_make_key_global(): + """Test generating a key for global scope.""" + category = "test_cat" + data = "test_data" + + before_ts = int(time.time() * 1000) + key = _make_key(category, True, data) + after_ts = int(time.time() * 1000) + + # Expected format: f"{PREFIX}:{scope}:{category}::{ts}:{h}" + assert key.startswith(f"{PREFIX}:{SCOPE_GLOBAL}:{category}::") + + # Extract timestamp and hash part + match = re.match(rf"^{PREFIX}:{SCOPE_GLOBAL}:{category}::(\d+):([a-f0-9]+)$", key) + assert match is not None, f"Key {key} does not match expected format" + + ts = int(match.group(1)) + h = match.group(2) + + assert before_ts <= ts <= after_ts + assert len(h) == 20 + + +def test_make_key_local(): + """Test generating a key for local scope.""" + category = "another_cat" + data = "more_data" + + before_ts = int(time.time() * 1000) + key = _make_key(category, False, data) + after_ts = int(time.time() * 1000) + + assert key.startswith(f"{PREFIX}:{SCOPE_LOCAL}:{category}::") + + match = re.match(rf"^{PREFIX}:{SCOPE_LOCAL}:{category}::(\d+):([a-f0-9]+)$", key) + assert match is not None, f"Key {key} does not match expected format" + + ts = int(match.group(1)) + h = match.group(2) + + assert before_ts <= ts <= after_ts + assert len(h) == 20 + + +def test_make_key_formatting_details(monkeypatch): + """Test the exact output formatting of _make_key using deterministic BLAKE2b.""" + # Mock time.time to return a predictable float so ts = 1620000000123 + monkeypatch.setattr(time, "time", lambda: 1620000000.123) + + # data="data", ts=1620000000123 -> blake2b("data1620000000123", digest_size=10) -> 5e5dad075ca7764bc51f + key1 = _make_key("cat1", True, "data") + assert key1 == f"{PREFIX}:{SCOPE_GLOBAL}:cat1::1620000000123:5e5dad075ca7764bc51f" + + key2 = _make_key("cat2", False, "data") + assert key2 == f"{PREFIX}:{SCOPE_LOCAL}:cat2::1620000000123:5e5dad075ca7764bc51f" + + +def test_make_key_determinism_and_uniqueness(): + """Test determinism for same inputs within same timestamp, and uniqueness across timestamps/data.""" + category = "test_cat" + data1 = "data1" + data2 = "data2" + + key1 = _make_key(category, True, data1) + time.sleep(0.002) + key2 = _make_key(category, True, data1) + key3 = _make_key(category, True, data2) + + # Uniqueness across data + assert key1 != key3 + + # Check determinism: if the timestamp parts are the same, the keys should be identical + ts1 = key1.split("::")[1].split(":")[0] + ts2 = key2.split("::")[1].split(":")[0] + if ts1 == ts2: + assert key1 == key2 + else: + # If timestamp is different, keys should be different + assert key1 != key2 + + +def test_memory_value_happy_path(): + """Test _memory_value with standard data and tags.""" + result = _memory_value("some data", ["tag1", "tag2"]) + parsed = json.loads(result) + assert parsed == {"data": "some data", "tags": ["tag1", "tag2"]} + + +def test_memory_value_missing_tags(): + """Test _memory_value when tags is None.""" + result = _memory_value("some data", None) + parsed = json.loads(result) + assert parsed == {"data": "some data", "tags": []} + + +def test_memory_value_unicode(): + """Test _memory_value properly handles unicode and ensure_ascii=False.""" + result = _memory_value("こんにちは", ["世界"]) + # If ensure_ascii=False, the unicode characters shouldn't be escaped (no \uXXXX) + assert "こんにちは" in result + assert "世界" in result + parsed = json.loads(result) + assert parsed == {"data": "こんにちは", "tags": ["世界"]} + + +def test_parse_memory_value_success(): + """Test _parse_memory_value successfully decodes valid JSON.""" + raw = '{"data": "info", "tags": ["a"]}' + result = _parse_memory_value(raw) + assert result == {"data": "info", "tags": ["a"]} + + +def test_parse_memory_value_invalid_json(): + """Test _parse_memory_value with invalid JSON.""" + result = _parse_memory_value("{invalid_json:") + assert result == {"data": "{invalid_json:", "tags": []} + + +def test_parse_memory_value_type_error(): + """Test _parse_memory_value with TypeError (e.g. passing None).""" + result = _parse_memory_value(None) + assert result == {"data": None, "tags": []} + + +def test_parse_memory_value_invalid_json_string(): + """Test _parse_memory_value with invalid JSON string.""" + result = _parse_memory_value("this is not a valid json string") + assert result == {"data": "this is not a valid json string", "tags": []} + + +# ===================================================================== +# Tests from test_memory_mcp.py (root) +# ===================================================================== + +def test_memory_entry_happy_path(): + """Test correctly formatted and complete memory entry.""" + valid_key = "memory:global:project_standards::1689201948123:a1b2c3d4e5f6" + valid_value = json.dumps({"data": "Use pytest for all tests", "tags": ["testing", "python"]}) + lmem = { + "key": valid_key, + "value": valid_value, + "memory_id": "test_id_123" + } + + result = _memory_entry(lmem) + + assert result is not None + assert result["key"] == valid_key + assert result["category"] == "project_standards" + assert result["data"] == "Use pytest for all tests" + assert result["tags"] == ["testing", "python"] + assert result["scope"] == "global" + assert result["timestamp"] == "1689201948123" + assert result["memory_id"] == "test_id_123" + + +def test_memory_entry_invalid_key(): + """Test with a key that does not start with 'memory:'.""" + lmem = { + "key": "notamemory:global:cat::123:hash", + "value": json.dumps({"data": "test", "tags": []}) + } + + result = _memory_entry(lmem) + assert result is None + + +def test_memory_entry_malformed_json_value(): + """Test with malformed/string value where JSON parsing fails.""" + valid_key = "memory:local:notes::1689201948123:a1b2c3d4e5f6" + # value is just a raw string, not JSON + lmem = { + "key": valid_key, + "value": "This is just a raw string without tags" + } + + result = _memory_entry(lmem) + + assert result is not None + assert result["data"] == "This is just a raw string without tags" + assert result["tags"] == [] # Falls back to empty tags list + assert result["category"] == "notes" + assert result["scope"] == "local" + + +def test_memory_entry_missing_fields(): + """Test gracefully handling dictionaries with missing keys.""" + # Missing 'value' and 'memory_id' + lmem1 = { + "key": "memory:global:ideas::123:hash" + } + result1 = _memory_entry(lmem1) + assert result1 is not None + assert result1["data"] == "" + assert result1["tags"] == [] + assert result1["memory_id"] == "" + + # Missing 'key' + lmem2 = { + "value": json.dumps({"data": "test", "tags": []}) + } + result2 = _memory_entry(lmem2) + assert result2 is None + + # Empty dict + result3 = _memory_entry({}) + assert result3 is None + + +def test_parse_key_happy_path(): + """Test full standard key structure""" + key = "memory:local:code::20240101T120000Z:abc123hash" + result = _parse_key(key) + assert result == { + "scope": "local", + "category": "code", + "timestamp": "20240101T120000Z" + } + + +def test_parse_key_missing_timestamp_hash(): + """Test key without the :: delimiter section""" + key = "memory:global:general" + result = _parse_key(key) + assert result == { + "scope": "global", + "category": "general", + "timestamp": "" + } + + +def test_parse_key_missing_category(): + """Test key with missing category""" + key = "memory:local::20240101T120000Z:abc123hash" + result = _parse_key(key) + assert result == { + "scope": "local", + "category": "", + "timestamp": "20240101T120000Z" + } + + +def test_parse_key_missing_scope_and_category(): + """Test minimal key prefix""" + key = "memory" + result = _parse_key(key) + assert result == { + "scope": "", + "category": "", + "timestamp": "" + } + + +def test_parse_key_empty_string(): + """Test completely empty string""" + key = "" + result = _parse_key(key) + assert result == { + "scope": "", + "category": "", + "timestamp": "" + } + + +def test_parse_key_invalid_type(): + """Test handling of an invalid type that triggers the exception branch""" + key = None + result = _parse_key(key) + assert result == { + "scope": "", + "category": "", + "timestamp": "" + } + + +def test_parse_memory_value_valid_json(): + raw_data = json.dumps({"data": "some data", "tags": ["tag1", "tag2"]}) + result = _parse_memory_value(raw_data) + assert result == {"data": "some data", "tags": ["tag1", "tag2"]} + + +def test_parse_memory_value_invalid_json(): + raw_data = "this is not json" + result = _parse_memory_value(raw_data) + assert result == {"data": "this is not json", "tags": []} + + +def test_parse_memory_value_type_error(): + raw_data = 12345 + result = _parse_memory_value(raw_data) # type: ignore[arg-type] + assert result == {"data": 12345, "tags": []} + + +def test_parse_memory_value_non_dict_json(): + raw_data = '"just a string"' + result = _parse_memory_value(raw_data) + assert result == "just a string" diff --git a/scripts/README.md b/scripts/README.md index 0139e8d3..11f4fd2a 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -42,44 +42,61 @@ Simulates fallback cascades to verify that failed Ollama requests activate the r ### `scripts/verification/verify_direct_ollama_cooldown.py` Asserts that direct requests to `llm-routing-ollama` immediately trigger the cooldown response without hammering downstream endpoints. +### `scripts/verification/verify_breaker.py` +Sanity verification check for the dual circuit breaker logic. + ### `scripts/verification/mock_rate_limit_server.py` A simple HTTP server that returns `429 Rate Limit Exceeded` to simulate rate limits when testing cooldowns. - **Usage**: `python3 scripts/verification/mock_rate_limit_server.py` (Runs on `127.0.0.1:9999`) --- -## 3. Classifier & Dataset Maintenance (`scripts/`) +## 3. Classifier, Daemons & Maintenance (`scripts/`) -These tools are used to benchmark the prompt classifier and extract datasets from Langfuse traces: +These tools and helper daemons are used to benchmark the prompt classifier, extract datasets from Langfuse traces, and orchestrate client-host communication: - **`benchmark_classifier.py`**: Benchmarks latency and precision metrics of the Ryzen PRO APU-offloaded classifier. - **`classify_direct.py`**: Takes a string prompt argument and prints the classification decision directly. - **`extract_prompts.py` / `extract_complex.py` / `extract_gapfill.py`**: Mines prompt datasets from Langfuse PG/ClickHouse database traces for fine-tuning. - **`reclassify_all.py`**: Re-evaluates prompt classifications against updated models. - **`retry_errors.py`**: Retries failed queries. +- **`host_agy_daemon.py`**: Real-time PTY-based streaming daemon for low-latency streaming for agent clients. +- **`sync_gemini_token.py`**: Extraction and sync script for keyring OAuth credentials. +- **`get_pr_status.py`**: PR status query helper. +- **`watch_quota.sh`**: Watch/polling script for observing quota status. +- **`test_quota_reset.sh`**: Simulates/triggers quota reset conditions. --- -## 4. Integration Test Suite (Root Directory) +## 4. Integration Test Suite (`tests/`) -The integration test suite is located in the root directory. Tests are categorized below based on their primary function: +The integration test suite is located in the `tests/` directory. Tests are categorized below based on their primary function: ### Circuit Breaker Tests -- **`test_circuit_breaker.py`**: Unit/integration tests for the dual circuit breaker (`router/circuit_breaker.py`), covering independent Google/Vendor tiers and probe-granting logic. -- **`verify_breaker.py`**: Sanity verification check for the circuit breaker. -- **`test_a2_verify.py`**: Quick sanity integration check for the agy proxy circuit breaker. +- **`tests/test_circuit_breaker.py`**: Unit/integration tests for the dual circuit breaker (`router/circuit_breaker.py`), covering independent Google/Vendor tiers and probe-granting logic. +- **`tests/test_a2_verify.py`**: Quick sanity integration check for the agy proxy circuit breaker. ### Classifier Tests -- **`test_classifier_accuracy.py`**: Accuracy evaluation suite covering 25 system prompts. +- **`tests/test_classifier_accuracy.py`**: Accuracy evaluation suite covering 25 system prompts. +- **`tests/test_map_tool_to_category.py`**: Tests prompt-classifier category mapping. ### Routing & Proxy Tests -- **`test_agy_tiers.py`**: Validates `agy` proxy model tier routing. -- **`test_antigravity.py`**: Tests the connection to the host Antigravity CLI daemon (`agentapi`). +- **`tests/test_agy_tiers.py`**: Validates `agy` proxy model tier routing. +- **`tests/test_antigravity.py`**: Tests the connection to the host Antigravity CLI daemon (`agentapi`). +- **`tests/test_models_proxy.py`**: Tests direct reverse-proxy and router routing mechanics. ### Performance & Monitoring Tests -- **`test_stream_latency.py`**: Measures Time-To-First-Token (TTFT) and token generation speed. - -### Simulation Tests -- **`test_agy_behavior.py`**: Asserts the behavior of the `agy` CLI client under quota limits. -- **`test_quota_reset.sh`**: Simulates/triggers quota reset conditions. -- **`watch_quota.sh`**: Watch/polling script for observing quota status. +- **`tests/test_stream_latency.py`**: Measures Time-To-First-Token (TTFT) and token generation speed. + +### Simulation & Helper Daemon Tests +- **`tests/test_agy_behavior.py`**: Asserts the behavior of the `agy` CLI client under quota limits. +- **`tests/test_host_agy_daemon.py`**: Tests real-time streaming capabilities and connection handling of the host `agy` daemon. +- **`tests/test_sync_gemini_token.py`**: Tests OAuth credentials extraction and sync. + +### Utility Tests +- **`tests/test_atomic_write.py`**: Tests atomic file writing logic used for config updates. +- **`tests/test_check_http_endpoint.py`**: Tests health check monitoring logic for HTTP endpoints. +- **`tests/test_compute_free_model_score.py`**: Tests local free model load and accuracy scoring. +- **`tests/test_pie_chart_gradient.py`**: Tests dynamic CSS gradient calculation for stats visualization. +- **`tests/test_record_tool_usage.py`**: Tests Valkey/Redis recording of LLM tool usage. +- **`tests/test_src_badge.py`**: Tests dynamic visual badge generation for source status. diff --git a/get_pr_status.py b/scripts/get_pr_status.py similarity index 100% rename from get_pr_status.py rename to scripts/get_pr_status.py diff --git a/host_agy_daemon.py b/scripts/host_agy_daemon.py similarity index 100% rename from host_agy_daemon.py rename to scripts/host_agy_daemon.py diff --git a/sync_gemini_token.py b/scripts/sync_gemini_token.py similarity index 100% rename from sync_gemini_token.py rename to scripts/sync_gemini_token.py diff --git a/test_quota_reset.sh b/scripts/test_quota_reset.sh similarity index 100% rename from test_quota_reset.sh rename to scripts/test_quota_reset.sh diff --git a/verify_breaker.py b/scripts/verification/verify_breaker.py similarity index 91% rename from verify_breaker.py rename to scripts/verification/verify_breaker.py index daf41bef..6ed9e7c7 100644 --- a/verify_breaker.py +++ b/scripts/verification/verify_breaker.py @@ -3,7 +3,7 @@ import sys from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent)) +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) from router.circuit_breaker import get_breaker diff --git a/watch_quota.sh b/scripts/watch_quota.sh similarity index 92% rename from watch_quota.sh rename to scripts/watch_quota.sh index 8bf9d090..cb95debd 100755 --- a/watch_quota.sh +++ b/scripts/watch_quota.sh @@ -2,7 +2,8 @@ # Polling loop — checks quota every 30s and runs tests when reset # Log file to watch LOG_FILE="$HOME/.gemini/antigravity-cli/cli.log" -TEST_SCRIPT="test_quota_reset.sh" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TEST_SCRIPT="$SCRIPT_DIR/test_quota_reset.sh" POLL_INTERVAL=30 # seconds echo "=== Quota Reset Watcher ===" diff --git a/start-stack.sh b/start-stack.sh index c9b826dc..e77e76e0 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -74,7 +74,7 @@ if [ -f "$OAUTH_CREDS" ]; then fi fi if $NEED_SYNC; then - python3 sync_gemini_token.py || echo "⚠️ Warning: Failed to sync Gemini token from keyring" + python3 scripts/sync_gemini_token.py || echo "⚠️ Warning: Failed to sync Gemini token from keyring" fi ACTIVE_OAUTH="" diff --git a/test_memory_mcp.py b/test_memory_mcp.py deleted file mode 100644 index 0194dca9..00000000 --- a/test_memory_mcp.py +++ /dev/null @@ -1,167 +0,0 @@ -#!/usr/bin/env python3 -""" -Tests for memory_mcp.py -""" -import sys -import json -import pytest -from router.memory_mcp import _memory_entry, _parse_key, _parse_memory_value - -def test_memory_entry_happy_path(): - """Test correctly formatted and complete memory entry.""" - valid_key = "memory:global:project_standards::1689201948123:a1b2c3d4e5f6" - valid_value = json.dumps({"data": "Use pytest for all tests", "tags": ["testing", "python"]}) - lmem = { - "key": valid_key, - "value": valid_value, - "memory_id": "test_id_123" - } - - result = _memory_entry(lmem) - - assert result is not None - assert result["key"] == valid_key - assert result["category"] == "project_standards" - assert result["data"] == "Use pytest for all tests" - assert result["tags"] == ["testing", "python"] - assert result["scope"] == "global" - assert result["timestamp"] == "1689201948123" - assert result["memory_id"] == "test_id_123" - -def test_memory_entry_invalid_key(): - """Test with a key that does not start with 'memory:'.""" - lmem = { - "key": "notamemory:global:cat::123:hash", - "value": json.dumps({"data": "test", "tags": []}) - } - - result = _memory_entry(lmem) - assert result is None - -def test_memory_entry_malformed_json_value(): - """Test with malformed/string value where JSON parsing fails.""" - valid_key = "memory:local:notes::1689201948123:a1b2c3d4e5f6" - # value is just a raw string, not JSON - lmem = { - "key": valid_key, - "value": "This is just a raw string without tags" - } - - result = _memory_entry(lmem) - - assert result is not None - assert result["data"] == "This is just a raw string without tags" - assert result["tags"] == [] # Falls back to empty tags list - assert result["category"] == "notes" - assert result["scope"] == "local" - -def test_memory_entry_missing_fields(): - """Test gracefully handling dictionaries with missing keys.""" - # Missing 'value' and 'memory_id' - lmem1 = { - "key": "memory:global:ideas::123:hash" - } - result1 = _memory_entry(lmem1) - assert result1 is not None - assert result1["data"] == "" - assert result1["tags"] == [] - assert result1["memory_id"] == "" - - # Missing 'key' - lmem2 = { - "value": json.dumps({"data": "test", "tags": []}) - } - result2 = _memory_entry(lmem2) - assert result2 is None - - # Empty dict - result3 = _memory_entry({}) - assert result3 is None - -def test_parse_key_happy_path(): - """Test full standard key structure""" - key = "memory:local:code::20240101T120000Z:abc123hash" - result = _parse_key(key) - assert result == { - "scope": "local", - "category": "code", - "timestamp": "20240101T120000Z" - } - -def test_parse_key_missing_timestamp_hash(): - """Test key without the :: delimiter section""" - key = "memory:global:general" - result = _parse_key(key) - assert result == { - "scope": "global", - "category": "general", - "timestamp": "" - } - -def test_parse_key_missing_category(): - """Test key with missing category""" - key = "memory:local::20240101T120000Z:abc123hash" - result = _parse_key(key) - # The split(":") on "memory:local" results in ["memory", "local"] length 2 - # So category should be "" - assert result == { - "scope": "local", - "category": "", - "timestamp": "20240101T120000Z" - } - -def test_parse_key_missing_scope_and_category(): - """Test minimal key prefix""" - key = "memory" - result = _parse_key(key) - assert result == { - "scope": "", - "category": "", - "timestamp": "" - } - -def test_parse_key_empty_string(): - """Test completely empty string""" - key = "" - result = _parse_key(key) - assert result == { - "scope": "", - "category": "", - "timestamp": "" - } - -def test_parse_key_invalid_type(): - """Test handling of an invalid type that triggers the exception branch""" - key = None - result = _parse_key(key) - assert result == { - "scope": "", - "category": "", - "timestamp": "" - } - -def test_parse_memory_value_valid_json(): - raw_data = json.dumps({"data": "some data", "tags": ["tag1", "tag2"]}) - result = _parse_memory_value(raw_data) - assert result == {"data": "some data", "tags": ["tag1", "tag2"]} - -def test_parse_memory_value_invalid_json(): - raw_data = "this is not json" - result = _parse_memory_value(raw_data) - assert result == {"data": "this is not json", "tags": []} - -def test_parse_memory_value_type_error(): - # json.loads will raise TypeError if given something that isn't str, bytes, or bytearray - raw_data = 12345 - result = _parse_memory_value(raw_data) # type: ignore[arg-type] - assert result == {"data": 12345, "tags": []} - -def test_parse_memory_value_non_dict_json(): - # If the input is valid JSON but not a dictionary, it currently returns the parsed non-dict value, - # which violates the dict return type annotation and can cause downstream KeyErrors/TypeErrors. - raw_data = '"just a string"' - result = _parse_memory_value(raw_data) - assert result == "just a string" - -if __name__ == "__main__": - sys.exit(pytest.main(["-v", __file__])) diff --git a/test_a2_verify.py b/tests/test_a2_verify.py similarity index 88% rename from test_a2_verify.py rename to tests/test_a2_verify.py index 42cde1e1..d2f0be9a 100644 --- a/test_a2_verify.py +++ b/tests/test_a2_verify.py @@ -2,7 +2,7 @@ """Verify circuit breaker integration into agy_proxy.py""" import sys from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent / 'router')) +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / 'router')) from circuit_breaker import get_breaker from agy_proxy import try_agy_proxy diff --git a/test_agy_behavior.py b/tests/test_agy_behavior.py similarity index 100% rename from test_agy_behavior.py rename to tests/test_agy_behavior.py diff --git a/test_agy_tiers.py b/tests/test_agy_tiers.py similarity index 100% rename from test_agy_tiers.py rename to tests/test_agy_tiers.py diff --git a/test_antigravity.py b/tests/test_antigravity.py similarity index 100% rename from test_antigravity.py rename to tests/test_antigravity.py diff --git a/test_atomic_write.py b/tests/test_atomic_write.py similarity index 100% rename from test_atomic_write.py rename to tests/test_atomic_write.py diff --git a/test_check_http_endpoint.py b/tests/test_check_http_endpoint.py similarity index 100% rename from test_check_http_endpoint.py rename to tests/test_check_http_endpoint.py diff --git a/test_circuit_breaker.py b/tests/test_circuit_breaker.py similarity index 99% rename from test_circuit_breaker.py rename to tests/test_circuit_breaker.py index 8cdcad60..e4570c2a 100644 --- a/test_circuit_breaker.py +++ b/tests/test_circuit_breaker.py @@ -19,7 +19,7 @@ import pytest from unittest.mock import patch, AsyncMock from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent)) +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from router.circuit_breaker import get_breaker, TIER_COOLDOWNS, MAX_TIER diff --git a/test_classifier_accuracy.py b/tests/test_classifier_accuracy.py similarity index 100% rename from test_classifier_accuracy.py rename to tests/test_classifier_accuracy.py diff --git a/test_compute_free_model_score.py b/tests/test_compute_free_model_score.py similarity index 100% rename from test_compute_free_model_score.py rename to tests/test_compute_free_model_score.py diff --git a/test_host_agy_daemon.py b/tests/test_host_agy_daemon.py similarity index 99% rename from test_host_agy_daemon.py rename to tests/test_host_agy_daemon.py index 9e77f9f7..2966397a 100644 --- a/test_host_agy_daemon.py +++ b/tests/test_host_agy_daemon.py @@ -9,6 +9,10 @@ import pytest +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + import host_agy_daemon def find_free_port(): diff --git a/test_map_tool_to_category.py b/tests/test_map_tool_to_category.py similarity index 100% rename from test_map_tool_to_category.py rename to tests/test_map_tool_to_category.py diff --git a/test_models_proxy.py b/tests/test_models_proxy.py similarity index 97% rename from test_models_proxy.py rename to tests/test_models_proxy.py index bfaf3b81..c325a39b 100644 --- a/test_models_proxy.py +++ b/tests/test_models_proxy.py @@ -9,7 +9,7 @@ import sys from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent / "router")) +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "router")) from main import get_http_client, proxy_models, HTTP_MAX_CONNECTIONS, HTTP_MAX_KEEPALIVE_CONNECTIONS, HTTP_KEEPALIVE_EXPIRY diff --git a/test_pie_chart_gradient.py b/tests/test_pie_chart_gradient.py similarity index 100% rename from test_pie_chart_gradient.py rename to tests/test_pie_chart_gradient.py diff --git a/test_record_tool_usage.py b/tests/test_record_tool_usage.py similarity index 100% rename from test_record_tool_usage.py rename to tests/test_record_tool_usage.py diff --git a/test_src_badge.py b/tests/test_src_badge.py similarity index 100% rename from test_src_badge.py rename to tests/test_src_badge.py diff --git a/test_stream_latency.py b/tests/test_stream_latency.py similarity index 100% rename from test_stream_latency.py rename to tests/test_stream_latency.py diff --git a/test_sync_gemini_token.py b/tests/test_sync_gemini_token.py similarity index 98% rename from test_sync_gemini_token.py rename to tests/test_sync_gemini_token.py index d0cbe351..b7255cea 100644 --- a/test_sync_gemini_token.py +++ b/tests/test_sync_gemini_token.py @@ -6,6 +6,9 @@ import time # Import the module to test +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + import sync_gemini_token @pytest.fixture diff --git a/triage_upgrade_plan.md b/triage_upgrade_plan.md index dc8d283d..ada4d2ee 100644 --- a/triage_upgrade_plan.md +++ b/triage_upgrade_plan.md @@ -37,8 +37,8 @@ This document outlines the steps to upgrade the triage system in the LLM-Routing - `router/agy_proxy.py`: Contains the core triage logic. - `router/main.py`: Entry point that may call the triage functions. - `router/config.yaml`: Configuration for the classifier and triage parameters. -- `test_agy_tiers.py`: Tests for the triage tiers, may need updates. -- `test_classifier_accuracy.py`: Tests for classifier accuracy, may need updates. +- `tests/test_agy_tiers.py`: Tests for the triage tiers, may need updates. +- `tests/test_classifier_accuracy.py`: Tests for classifier accuracy, may need updates. ## Estimated Effort - Review and planning: 2 hours From cad53cb20153b02ee5e671f02bf67a32d5013cc7 Mon Sep 17 00:00:00 2001 From: sheepdestroyer Date: Wed, 1 Jul 2026 00:07:52 +0200 Subject: [PATCH 2/2] Refactor: Implement dynamic project root discovery and fail-fast check --- router/tests/test_agy_proxy.py | 8 +++++++- router/tests/test_memory_mcp.py | 8 ++++++-- scripts/verification/verify_breaker.py | 8 +++++++- start-stack.sh | 4 ++++ tests/test_a2_verify.py | 8 +++++++- tests/test_circuit_breaker.py | 7 ++++++- tests/test_host_agy_daemon.py | 8 +++++++- tests/test_models_proxy.py | 8 +++++++- tests/test_sync_gemini_token.py | 7 ++++++- 9 files changed, 57 insertions(+), 9 deletions(-) diff --git a/router/tests/test_agy_proxy.py b/router/tests/test_agy_proxy.py index 174c9898..3358439e 100644 --- a/router/tests/test_agy_proxy.py +++ b/router/tests/test_agy_proxy.py @@ -1,6 +1,12 @@ import sys from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +# Dynamic project root discovery +root = Path(__file__).resolve() +while root.parent != root and not (root / ".git").exists(): + root = root.parent +sys.path.insert(0, str(root)) +sys.path.insert(0, str(root / "router")) from unittest.mock import patch, MagicMock from router.agy_proxy import _wrap_response, _is_quota_exhausted diff --git a/router/tests/test_memory_mcp.py b/router/tests/test_memory_mcp.py index 3bfd0ca7..92c5bc96 100644 --- a/router/tests/test_memory_mcp.py +++ b/router/tests/test_memory_mcp.py @@ -4,8 +4,12 @@ import time from pathlib import Path -# Inject router path to allow importing memory_mcp directly -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +# Dynamic project root discovery +root = Path(__file__).resolve() +while root.parent != root and not (root / ".git").exists(): + root = root.parent +sys.path.insert(0, str(root)) +sys.path.insert(0, str(root / "router")) import pytest from memory_mcp import ( diff --git a/scripts/verification/verify_breaker.py b/scripts/verification/verify_breaker.py index 6ed9e7c7..db4db8af 100644 --- a/scripts/verification/verify_breaker.py +++ b/scripts/verification/verify_breaker.py @@ -3,7 +3,13 @@ import sys from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) + +# Dynamic project root discovery +root = Path(__file__).resolve() +while root.parent != root and not (root / ".git").exists(): + root = root.parent +sys.path.insert(0, str(root)) +sys.path.insert(0, str(root / "router")) from router.circuit_breaker import get_breaker diff --git a/start-stack.sh b/start-stack.sh index e77e76e0..ee181ee8 100755 --- a/start-stack.sh +++ b/start-stack.sh @@ -74,6 +74,10 @@ if [ -f "$OAUTH_CREDS" ]; then fi fi if $NEED_SYNC; then + if [ ! -f "scripts/sync_gemini_token.py" ]; then + echo "❌ Error: scripts/sync_gemini_token.py not found. Repository structure is invalid." >&2 + exit 1 + fi python3 scripts/sync_gemini_token.py || echo "⚠️ Warning: Failed to sync Gemini token from keyring" fi diff --git a/tests/test_a2_verify.py b/tests/test_a2_verify.py index d2f0be9a..ca9ca025 100644 --- a/tests/test_a2_verify.py +++ b/tests/test_a2_verify.py @@ -2,7 +2,13 @@ """Verify circuit breaker integration into agy_proxy.py""" import sys from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent.parent / 'router')) + +# Dynamic project root discovery +root = Path(__file__).resolve() +while root.parent != root and not (root / ".git").exists(): + root = root.parent +sys.path.insert(0, str(root)) +sys.path.insert(0, str(root / "router")) from circuit_breaker import get_breaker from agy_proxy import try_agy_proxy diff --git a/tests/test_circuit_breaker.py b/tests/test_circuit_breaker.py index e4570c2a..2dfd8064 100644 --- a/tests/test_circuit_breaker.py +++ b/tests/test_circuit_breaker.py @@ -19,7 +19,12 @@ import pytest from unittest.mock import patch, AsyncMock from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +# Dynamic project root discovery +root = Path(__file__).resolve() +while root.parent != root and not (root / ".git").exists(): + root = root.parent +sys.path.insert(0, str(root)) from router.circuit_breaker import get_breaker, TIER_COOLDOWNS, MAX_TIER diff --git a/tests/test_host_agy_daemon.py b/tests/test_host_agy_daemon.py index 2966397a..61f84e86 100644 --- a/tests/test_host_agy_daemon.py +++ b/tests/test_host_agy_daemon.py @@ -11,7 +11,13 @@ import sys from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +# Dynamic project root discovery +root = Path(__file__).resolve() +while root.parent != root and not (root / ".git").exists(): + root = root.parent +sys.path.insert(0, str(root)) +sys.path.insert(0, str(root / "scripts")) import host_agy_daemon diff --git a/tests/test_models_proxy.py b/tests/test_models_proxy.py index c325a39b..e0df6e41 100644 --- a/tests/test_models_proxy.py +++ b/tests/test_models_proxy.py @@ -9,7 +9,13 @@ import sys from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "router")) + +# Dynamic project root discovery +root = Path(__file__).resolve() +while root.parent != root and not (root / ".git").exists(): + root = root.parent +sys.path.insert(0, str(root)) +sys.path.insert(0, str(root / "router")) from main import get_http_client, proxy_models, HTTP_MAX_CONNECTIONS, HTTP_MAX_KEEPALIVE_CONNECTIONS, HTTP_KEEPALIVE_EXPIRY diff --git a/tests/test_sync_gemini_token.py b/tests/test_sync_gemini_token.py index b7255cea..9a3d8157 100644 --- a/tests/test_sync_gemini_token.py +++ b/tests/test_sync_gemini_token.py @@ -7,7 +7,12 @@ # Import the module to test from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) +# Dynamic project root discovery +root = Path(__file__).resolve() +while root.parent != root and not (root / ".git").exists(): + root = root.parent +sys.path.insert(0, str(root)) +sys.path.insert(0, str(root / "scripts")) import sync_gemini_token