From 08511715ae60d4c6a2816e67dff61101fb4b75d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=BB=D0=B0=D0=B4=D0=B8=D0=BC=D0=B8=D1=80=20=D0=A8?= =?UTF-8?q?=D0=BC=D0=B0=D0=BD?= Date: Sun, 16 Aug 2026 02:20:37 +0300 Subject: [PATCH 1/2] A ceiling that could see one per cent of the tokens, and a place to compact Five changes that are mostly one argument. `core/budget.py` charged a project's quota_tokens ceiling with `input + output`, which over the first fortnight of real use came to 149,063 tokens against 12,520,659 that had moved. The missing 98.8% is cache reads: a long conversation is re-read from cache on every tool round-trip. The counts were recorded correctly the whole time -- one line of arithmetic decided which of the four a ceiling could see. So: weight all four in `quota_log.billable`, the one place the four become one, with the ratios in `[quota]`. Output stays at 1.0 rather than its true multiple, because this is meant to reveal the 98.8% that was invisible and not to reprice the 1.2% that was not. The correction on the measured ledger is 12x. Which makes context length a cost rather than a curiosity, so the statusline carries a meter -- measured against whichever limit fires first, since 40% means different things at 300k and at the CLI's 967k -- and `core/compaction.py` compacts where you choose. There is no way to ask the SDK for that: its control protocol has ten subtypes and none of them is "compact". Ours is visible (a marker in the transcript, because an in-band compaction leaves the user reading evidence for a belief the agent no longer holds), metered under its own stage, and it keeps the handover note across a failed turn -- the turn right after a compaction is the one where losing it costs the whole session. Compacting is not obviously cheap and the threshold is not a "lower is better" dial: the seeded session starts on a cold cache, paying writes at 1.25x where it would have paid reads at 0.1x. The `compaction` stage is what makes that measurable, which is why the accounting landed first. `core/traces.py` tags each session -- tool:, gate:, ledger:, outcome:, cost: -- so a week of use leaves something sliceable behind, which is the unwritten prerequisite of HANDOFF 12 step 3. `traces harvest` turns real searches into eval rows, ungraded, because which papers were right is the one part a trace cannot recover. A verb only asked about does not count: four of the five `ledger:` tags on the busiest real session came from `--help` calls. And the licence. pyproject claimed MIT from the first commit with no LICENSE in the tree -- metadata granting a licence the repository did not. Validating the file under PEP 639 also surfaced `py-modules` sitting a table too low, under `package-data`, where setuptools reads keys as package names: the wheel declared data for a package called `py-modules` and no top-level modules at all, so an installed `grad` could not import `agent:main`. The editable install hid it completely. Both verified by building a wheel. Co-Authored-By: Claude Opus 5 --- .gitignore | 6 + LICENSE | 21 ++ README.md | 129 +++++++++++- agent.py | 112 ++++++++++- core/budget.py | 32 ++- core/compaction.py | 247 +++++++++++++++++++++++ core/config.py | 52 +++++ core/quota_log.py | 109 +++++++++- core/traces.py | 231 ++++++++++++++++++++++ pyproject.toml | 24 ++- tests/test_context_and_compaction.py | 279 ++++++++++++++++++++++++++ tests/test_traces.py | 211 ++++++++++++++++++++ tests/test_ui_compaction.py | 196 ++++++++++++++++++ tools/quota.py | 23 ++- tools/traces.py | 286 +++++++++++++++++++++++++++ ui/app.py | 170 +++++++++++++++- ui/models.py | 103 +++++++++- ui/tokens.py | 26 +++ ui/windows/chat.py | 117 +++++++++++ ui/windows/quota.py | 29 +++ 20 files changed, 2377 insertions(+), 26 deletions(-) create mode 100644 LICENSE create mode 100644 core/compaction.py create mode 100644 core/traces.py create mode 100644 tests/test_context_and_compaction.py create mode 100644 tests/test_traces.py create mode 100644 tests/test_ui_compaction.py create mode 100644 tools/traces.py diff --git a/.gitignore b/.gitignore index b21b7e3..0cad764 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,12 @@ __pycache__/ venv/ *.egg-info/ .pytest_cache/ +# Build outputs. Nothing here builds a wheel in the normal course of things -- +# the install is editable and `grad --update` moves the checkout rather than +# reinstalling from an artifact -- but the packaging metadata is only really +# checked by building one, and the leftovers should not land in a commit. +build/ +dist/ # Derived index - rebuildable from the JSONL at any time (HANDOFF §7). ledger/ledger.sqlite diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..38ba944 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Vladimir Shman + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 93a83d6..eb794a5 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ is a sentence in `prompts/system.md`. | Notebooks run clean top-to-bottom | `tools/nb.py verify` on a fresh kernel | | No general remote-execution capability | credentials in Windows Credential Manager, read only by `jobs.py` / `gpu.py` | | Concurrent ledger writes don't corrupt | one locked `core/jsonl.py:append`; no CLI writes a ledger file directly | -| Token and credit spend stays bounded, not merely measured | `core/budget.py`, checked at every gateable event | +| Token and credit spend stays bounded, not merely measured | `core/budget.py`, checked at every gateable event, over all four kinds of token | | An evolutionary campaign cannot outspend its allocation | the campaign gate in `tools/evolve.py`, before generation 0 and before each generation after it | | A job submitted to an org is collectable from that org | the namespace is persisted on the run handle, not just passed at submit | | Every number in a report traces to a run record | `tools/report.py check` refuses on an unresolved claim, on a `claims.tex` that has drifted from `claims.json`, and on a measured-looking number typed into the generated prose | @@ -56,6 +56,43 @@ limits are rolling windows (5-hour and weekly on Max) that the SDK does not expose as a remaining balance. **A token ceiling is a proxy you control, not a mirror of Anthropic's limit.** The meter says so on screen. +### The ceiling used to count about one per cent of the tokens + +Worth recording, because the row above claimed the opposite for two months and +nothing in the tests caught it. `core/budget.py` charged a project's +`quota_tokens` ceiling with `input_tokens + output_tokens`. Over the first +fortnight of real use that came to **149,063 tokens, against 12,520,659 that had +actually moved**. The missing 98.8% is cache reads: a long conversation is +re-read from the prompt cache on every tool round-trip, so cache traffic +dominates everything else by two orders of magnitude. One turn in that ledger +read 10.1M cached tokens to produce 104k of output. + +The counts were being recorded correctly the whole time — `quota_log.record` has +always stored all four — so this was never a measurement problem. It was one +line of arithmetic deciding which of the four a ceiling could see. + +Now all four are weighted into one number by `core/quota_log.py:billable`, which +is the only place the four become one, so a change to what a cache read is worth +lands on the gate, the meters and the summaries together. The weights are +`[quota]` in `config/grad.toml`, as ratios against one input token: + +| kind | weight | why | +|---|---|---| +| input | 1.0 | the unit | +| output | 1.0 | *not* its true multiple — see below | +| cache read | 0.1 | a tenth of an input token | +| cache write | 1.25 | a quarter more than one | + +Output stays at 1.0 deliberately. Weighting it by its real price would have been +more accurate and would also have silently reduced every existing ceiling; this +change is meant to reveal the 98.8% that was invisible, not to reprice the 1.2% +that was not. On the measured ledger the correction is **12×**. Set +`weight_cache_read = 0` to get the old arithmetic back. + +`python -m tools.quota summary --json` reports the four counts, the weighted +total and the weights it used, side by side, because a total that is mostly cache +traffic is unarguable with its components beside it and baffling without them. + ## Install ```bash @@ -176,6 +213,59 @@ else. Runs with no stamp at all pass silently: they predate the field, and refusing a report because its evidence is old would make the rule a reason to avoid updating. +### Where the conversation gets compacted, and who decides + +The CLI underneath compacts on its own, and a live session reports the threshold +as **967,000 of a 1,000,000 window**. That is a ceiling in the sense that a wall +at the end of a runway is one: by the time it is reached, every tool round-trip +has spent a long time re-reading most of a million cached tokens — which, with +the accounting above fixed, is now visible as the dominant cost it always was. + +There is no way to ask the SDK to compact, and no way to move its threshold from +here. The control protocol has ten subtypes — `initialize`, `mcp_status`, +`get_context_usage`, `interrupt`, `set_permission_mode`, `set_model`, +`rewind_files`, `mcp_reconnect`, `mcp_toggle`, `stop_task` — and none of them is +"compact"; the threshold comes from settings, and `agent.py` leaves +`setting_sources` unset on purpose so a stray `settings.json` cannot add +permission rules behind the code's back. + +So `core/compaction.py` does it, at `[agent] compact_at_tokens` (300k by +default, 0 to disable). Being ours buys three things the CLI's version cannot: + +* **It is visible.** A compaction performed in-band rewrites what the model + remembers while the transcript on screen still shows every turn — the user is + looking at evidence for a belief the agent no longer holds, and nothing says + so. Grad's writes a marker into the transcript where it happened, with the + handover note behind a disclosure. +* **It is metered.** The summary is charged to a `compaction` stage of its own, + so "what does compacting cost" is a question the ledger answers rather than a + cost folded into the conversation it was compacting. +* **It happens where you chose.** + +The mechanism has no clever part: ask the session, while it still remembers +everything, to write a note to whoever picks it up next; drop the client; start a +fresh conversation; hand it the note in front of the next prompt rather than as a +turn of its own, so it costs nothing extra. The note is asked for in the first +person and asks for paths, commands, and the ledger state the next turn is +expected to act on — an expectation registered and not yet judged, a run +submitted and not yet collected. A generic "summarise the conversation" prompt +drops those every time, and losing them does not read as a bad summary. It reads +as an agent that abandoned a run halfway. + +**Compacting is not obviously cheap, and the threshold is not a "lower is +better" dial.** The summary costs a turn, and the session it seeds starts with a +cold prompt cache — so the turn after a compaction pays cache *writes* at 1.25× +where it would have paid cache *reads* at 0.1×. There is a threshold below which +compacting costs more than not compacting. The `compaction` stage is what makes +that measurable, which is why the accounting split landed before this did. + +The chat window's statusline carries a context meter, measured against whichever +limit will actually be reached first — Grad's threshold when one is set, the +CLI's otherwise — because a meter reading 40% means quite different things at +300k and at 967k. It reads `—` rather than `0` before the first reading: an +unknown context and an empty one look identical at a glance and only one of them +is worth acting on. + ### Retrieval without an institutional email, and without waiting Tier 1 defaults to **Papers with Code** (`paperswithcode.co/api/v1`) — the @@ -253,6 +343,7 @@ that carry the literal next command. | `tools/evolve.py` | evolutionary search as a budgeted campaign, over ShinkaEvolve | | `tools/report.py` | `draft` / `write` / `cite` / `check` / `build` — the report and its gate | | `tools/lab.py` | the embedded JupyterLab server (human editing surface) | +| `tools/traces.py` | tag stored sessions, and harvest eval candidates from real use — **human-facing only** | | `tools/wiki.py` | RepoWiki over `core/` and `tools/` — **human-facing only**, not an agent tool | ### Exit codes @@ -313,6 +404,8 @@ prompts/system.md under 1000 tokens core/ the machinery the CLIs share, so no tool can forget a rule cli.py the §8 CLI contract, implemented once jsonl.py the single locked write path to the ledgers + compaction.py where a conversation is compacted, and what survives it + traces.py a session as tags a later query can slice on -- pure, tested submission.py the resolved submission and its hash gates.py the submit gates and the smoke carve-out budget.py the project dimension and its three ceilings @@ -458,6 +551,30 @@ reached for. The eval file here is a schema and a handful of seed rows, not a benchmark; authoring it cold would measure the imagination rather than the system. +That step had a prerequisite nobody wrote down: the week of use has to leave +something sliceable behind. A directory of transcripts is a record, but "every +session where a submitter refused" was a full-text search whose answer depended +on how the refusal happened to be phrased. `core/traces.py` tags each +trajectory — `tool:`, `gate:`, `ledger:`, `outcome:`, `turns:`, `cost:` — and +`python -m tools.traces list --json` reports what a week actually consisted of, +which is usually not what it felt like it consisted of. + +`gate:` is the namespace worth having, and the one ml-intern's equivalent has no +reason to want. Every row of the table at the top of this file is a claim that +some gate refuses under some condition; a corpus of real sessions tagged by +which gate refused is the difference between believing that and knowing it. A +verb that was only asked about does not count — `ledger expect --help` tags the +module and not the verb, because on the real corpus four of the five `ledger:` +tags on the busiest session came from `--help` calls, and a corpus that cannot +tell reading an interface from using it would answer the question wrongly. + +`python -m tools.traces harvest` turns the questions actually put to +`paper_search` into eval rows. They arrive **ungraded** — `relevant` is empty — +because which papers were the right answer is the one part of an eval row a +trace cannot recover, and a harvester that guessed would measure the guess. It +never rewrites an existing row and never appends a duplicate, so it is meant to +be re-run as the corpus grows. + ## Tests ```bash @@ -467,3 +584,13 @@ python -m pytest -q The gate tests run against a real ledger in a temp workspace rather than against mocks. A mock of a gate proves nothing about the gate, and these are the checks that stand between an agent under deadline pressure and a GPU bill. + +## Licence + +MIT — see [`LICENSE`](LICENSE). + +`pyproject.toml` claimed MIT from the first commit and the repository contained +no licence file, which is the one combination that is worse than saying nothing: +the package metadata grants a licence the repository does not. Both now say the +same thing, and `pyproject.toml` says it as an SPDX expression with +`license-files` rather than the deprecated free-text form. diff --git a/agent.py b/agent.py index cb7ed36..39c41da 100644 --- a/agent.py +++ b/agent.py @@ -194,11 +194,26 @@ async def run_session(prompt: str | None, *, once: bool) -> int: if env["removed_env"]: print(f"[grad] removed from the environment: {', '.join(env['removed_env'])}", file=sys.stderr) - async with sdk.ClaudeSDKClient(options=build_options(cfg)) as client: + # Held in a variable rather than in an `async with`, because compacting + # replaces it: the note is written by the outgoing session and the fresh one + # is built to hold it. A context manager binds the name for the whole block + # and there would be no way to swap what it holds -- which is how the CLI + # would have ended up as the surface that cannot compact, and the two + # surfaces disagreeing about a rule is the failure `drive_turn`'s docstring + # is about. + client = await _connect(sdk, cfg) + #: The handover note from a compaction, waiting for the next prompt to ride + #: in front of. Sending it as a turn of its own would spend a round-trip to + #: produce an answer nobody asked for. + seed: str | None = None + try: if prompt: - ran = await _turn(client, prompt) + ran, seed = await _turn(client, prompt, seed) if once: + # No compaction on a one-shot: the session ends here, so the only + # thing a compaction could buy is a summary nothing will read. return 0 if ran else EXIT_PROJECT_BUDGET + client, seed = await _maybe_compact(sdk, cfg, client, seed) while True: try: # In a worker thread: a bare input() blocks the event loop, and @@ -213,7 +228,71 @@ async def run_session(prompt: str | None, *, once: bool) -> int: continue if line in ("exit", "quit"): return 0 - await _turn(client, line) + _, seed = await _turn(client, line, seed) + client, seed = await _maybe_compact(sdk, cfg, client, seed) + finally: + await _disconnect(client) + + +async def _connect(sdk: Any, cfg: Any, *, resume: str | None = None) -> Any: + client = sdk.ClaudeSDKClient(options=build_options(cfg, resume=resume)) + await client.__aenter__() + return client + + +async def _disconnect(client: Any) -> None: + """Exit a client's context. Never raises on the way out.""" + if client is None: + return + try: + await client.__aexit__(None, None, None) + except Exception: # noqa: BLE001 - shutdown must not raise + pass + + +async def _maybe_compact(sdk: Any, cfg: Any, client: Any, seed: str | None) -> tuple[Any, str | None]: + """Compact between turns when the context has passed the threshold. + + The CLI's half of what `ui/app.py:Session.maybe_compact` does, and the same + order for the same reason: the note is written while the outgoing session + still remembers everything, and only then is the client replaced. + + A failure here returns the client unchanged. An oversized conversation is a + cost; a session taken down between turns by its own housekeeping is a loss. + """ + from core import compaction # noqa: PLC0415 + + if not compaction.threshold(cfg): + return client, seed + reader = getattr(client, "get_context_usage", None) + if reader is None: + return client, seed + try: + usage = await reader() + except Exception: # noqa: BLE001 - no reading is not a reason to compact + return client, seed + if not compaction.should_compact(usage, cfg): + return client, seed + + before = compaction.context_tokens(usage) + print(f"\n[grad] compacting at {before:,} tokens…", file=sys.stderr) + try: + handoff = await compaction.write_handoff(client, drive_turn) + except BudgetRefused as exc: + # No carve-out: a compaction is a model call and the allocation applies. + print(f"[grad] cannot compact: {exc.refusal['message']}", file=sys.stderr) + return client, seed + except Exception as exc: # noqa: BLE001 - the conversation survives a failed compaction + print(f"[grad] could not compact ({type(exc).__name__}); carrying on", file=sys.stderr) + return client, seed + + await _disconnect(client) + # `resume` is deliberately not passed. Resuming would restore the very + # conversation this just summarised, making the whole operation a cost with + # no effect. + fresh = await _connect(sdk, cfg) + print("[grad] compacted — the agent now knows this session by its handover note", file=sys.stderr) + return fresh, compaction.seed_message(handoff["note"], tokens_before=before) def check_turn_budget() -> dict[str, Any] | None: @@ -288,6 +367,8 @@ async def drive_turn( on_chunk: Any = None, on_session_id: Any = None, session: str | None = None, + stage: str = quota_log.STAGE_MAIN, + role: str = "research", ) -> dict[str, Any]: """One turn, for every surface that runs one. @@ -308,6 +389,12 @@ async def drive_turn( off the return value learned it only for turns that finished -- and an interrupted turn is precisely the one after which the client is rebuilt, so that was the case where losing the id cost the whole conversation. + + `stage` and `role` decide where the turn's tokens land in `ledger/quota.jsonl`. + They default to the conversation, and the one caller that overrides them is + `core/compaction.py`: a compaction is a model call this system makes on its + own initiative, and folding its cost into `main` would hide precisely the + number that says whether the threshold is set correctly. """ refusal = check_turn_budget() if refusal: @@ -347,26 +434,33 @@ async def drive_turn( # exactly the turns most worth accounting for. if last_usage is not None: recorded = quota_log.from_sdk_usage( - quota_log.STAGE_MAIN, last_usage, model=None, role="research", session=session + stage, last_usage, model=None, role=role, session=session ) return {"sdk_session_id": sdk_session_id, "quota": recorded} -async def _turn(client: Any, prompt: str) -> bool: - """Run one turn. Returns False if the budget refused it.""" +async def _turn(client: Any, prompt: str, seed: str | None = None) -> tuple[bool, str | None]: + """Run one turn. Returns whether it ran, and the seed still owed. + + `seed` is a handover note from a compaction, prepended to this prompt rather + than sent as a turn of its own. It is returned unconsumed when the turn does + not run, because a note dropped by a refused turn is the whole memory of + everything the compaction discarded. + """ stream = TurnStream() + sent = f"{seed}\n\n---\n\n{prompt}" if seed else prompt try: await drive_turn( - client, prompt, stream, on_chunk=lambda c: print(c, end="", flush=True) + client, sent, stream, on_chunk=lambda c: print(c, end="", flush=True) ) except BudgetRefused as exc: print( f"\n[grad] {exc.refusal['message']}\n[grad] fix: {exc.refusal['fix']}", file=sys.stderr, ) - return False + return False, seed print() - return True + return True, None def _text_of(message: Any) -> str: diff --git a/core/budget.py b/core/budget.py index cb8711a..3a8b07c 100644 --- a/core/budget.py +++ b/core/budget.py @@ -308,13 +308,29 @@ def spend(project_id: str) -> dict[str, Any]: if not (r.collected and r.get("cost_usd_actual") is not None): in_flight_usd += amount - quota_tokens = 0 + # The four kinds, weighted into one number by `quota_log.billable`. + # + # This line used to be `input + output`, which sounds like the whole of what + # a turn spends and is not close to it: a long context is re-read from cache + # on every tool round-trip, so cache reads dominate everything else by two + # orders of magnitude. Measured over the first fortnight of use, this sum saw + # 149k tokens while 12.5M had actually moved. The ceiling was real, the + # arithmetic under it was not. + # + # The raw counts are kept beside the weighted total rather than replaced by + # it. A ceiling needs one number; a person asking why they hit it needs four. + weight = quota_log.weights() + quota_billable = 0.0 + quota_counts = {field: 0 for field, _ in quota_log.KINDS} credits_usd = 0.0 for entry in quota_log.entries(): if project_of(entry) != project_id: continue - quota_tokens += int(entry.get("input_tokens", 0) or 0) + int(entry.get("output_tokens", 0) or 0) + for field, value in quota_log.counts(entry).items(): + quota_counts[field] += value + quota_billable += quota_log.billable(entry, weight) credits_usd += float(entry.get("credits_usd", 0.0) or 0.0) + quota_tokens = round(quota_billable) # Campaign candidates consume real resources and live outside runs.jsonl by # design (§23 item 4). Leaving them out here would make a campaign invisible @@ -343,6 +359,11 @@ def spend(project_id: str) -> dict[str, Any]: "gpu_in_flight_usd": round(in_flight_usd, 4), "gpu_candidate_usd": round(candidate_usd, 4), "quota_tokens": quota_tokens, + # What the weighted figure above is made of, and the weights that made + # it. Reported so `budget status` can answer "why is this number twelve + # times what I expected" without anyone having to read this file. + "quota_token_counts": dict(quota_counts), + "quota_weights": dict(weight), "credits_usd": round(credits_usd, 6), "runs": runs, "candidates": candidates, @@ -390,6 +411,13 @@ def status(project_id: str) -> dict[str, Any]: "gpu_in_flight_usd": used["gpu_in_flight_usd"], "run_count": len(used["runs"]), "over_budget": [r for r, d in resources.items() if d["over"]], + # What the `quota_tokens` figure is made of. Carried up from `spend` + # because this is the payload every surface reads -- `budget status`, + # the quota window, and the refusal message the agent is handed -- and a + # ceiling that is mostly cache traffic is unarguable with the four + # numbers beside it and baffling without them. + "quota_token_counts": used.get("quota_token_counts", {}), + "quota_weights": used.get("quota_weights", {}), # The meter must not imply a fuel gauge. Anthropic exposes no remaining # balance and the real limits are rolling windows, so a token ceiling is # a proxy the user controls. diff --git a/core/compaction.py b/core/compaction.py new file mode 100644 index 0000000..f8cb886 --- /dev/null +++ b/core/compaction.py @@ -0,0 +1,247 @@ +"""When a conversation is compacted, and what survives it. + +The SDK offers no way to ask for this. Its control protocol has ten subtypes -- +`initialize`, `mcp_status`, `get_context_usage`, `interrupt`, +`set_permission_mode`, `set_model`, `rewind_files`, `mcp_reconnect`, +`mcp_toggle`, `stop_task` -- and none of them is "compact". The CLI underneath +does compact on its own, and a live session reports the threshold as 967,000 of +a 1,000,000 window, but that number is not reachable from here either: it comes +from settings, and `agent.py` deliberately leaves `setting_sources` unset so +that a stray `settings.json` cannot add permission rules behind the code's back. + +So this is ours, and being ours is worth more than the convenience would have +been: + +* **It is visible.** A compaction that the CLI performs in-band rewrites what + the model remembers while the transcript on screen still shows every turn. + Nothing says so, and the divergence is invisible until the model fails to + remember something the user can see. A compaction performed here writes a + record, and the chat window draws it in the transcript where it happened. +* **It is metered.** The summary is a model call and it is charged to + `quota_log.STAGE_COMPACT`, so "what does compacting cost" is a question the + ledger answers. ml-intern's context manager carries a comment saying that not + doing this "used to hide a significant share of hosted inference spend"; the + same hole was available here for free. +* **It happens where we choose.** 967k is a wall at the end of a runway. By the + time it is reached, every tool round-trip has spent a long time re-reading + most of a million cached tokens. + +**Compacting is not obviously cheap.** The summary costs a turn, and the session +it seeds starts with a cold prompt cache -- so the turn after a compaction pays +cache *writes* at 1.25x where it would have paid cache *reads* at 0.1x. There is +a threshold below which compacting costs more than not compacting, this module +cannot know where it is, and `[quota]` weights plus the `compaction` stage are +what make it measurable. Do not lower `compact_at_tokens` on the theory that +less context is always cheaper. + +The mechanism is the one thing here with no clever part: ask the session, while +it still remembers everything, to write a note to whoever picks it up next; drop +the client; start a fresh conversation; hand that note to it as the first thing +it reads. The note is written in the first person and asks for specifics, +because the failure mode of a summary is that it reads well and contains +nothing actionable. +""" + +from __future__ import annotations + +from typing import Any + +from core import quota_log + +#: What the outgoing session is asked to leave behind. +#: +#: First person, and explicitly not a précis. A compaction summary is read by a +#: model that has to *continue* the work, not by a person deciding whether to, +#: and the two want opposite things: the reader of a précis wants the shape of +#: what happened, while the continuer wants the paths, the ids and the half +#: -finished intention. The instruction to be specific rather than brief is +#: doing the load-bearing work here. +#: +#: The Grad-specific paragraph is the reason this is not ml-intern's prompt +#: verbatim. An expectation that was registered and not yet judged, a run that +#: was submitted and not yet collected, a project that is selected: these are +#: pieces of state that live in the ledger, that the next turn is expected to +#: act on, and that a general "summarise the conversation" prompt drops on the +#: floor every time. Losing them does not read as a bad summary -- it reads as +#: an agent that abandoned a run halfway. +HANDOFF_PROMPT = """\ +You are about to be restored into a fresh session that has no memory of the \ +conversation above. Write a first-person note to your future self so you can \ +carry on exactly where you left off. This note is the only thing you will have. + +Cover, specifically and with real values rather than descriptions: + + * What was asked for, and what has actually been done about it so far. + * Every file you wrote or changed, by path. + * The commands you ran that mattered, and what each one returned. + * Decisions you made and *why* -- especially the ones you would otherwise \ +have to make again. + * What you were about to do next. + +Then, separately, the ledger state you were holding in your head: the project \ +selected, any expectation registered and not yet judged, any run submitted and \ +not yet collected, any deviation awaiting a verdict, and anything a gate has \ +already refused and why. If there is none of this, say so in one line. + +Do not be brief and do not be graceful. Be specific. Anything you leave out is \ +gone. +""" + + +def threshold(cfg: Any = None) -> int: + """Where Grad compacts, in tokens of context. 0 disables it. + + Read through the same tolerant path as the quota weights, and for the same + reason: this is consulted on the turn path, and a typo in `grad.toml` should + degrade to "do not compact" rather than take a session down. A negative or + non-finite value is treated as 0 -- disabled -- because the alternative is a + threshold that is always already exceeded, which would compact after every + single turn. + """ + value = _number(cfg, "compact_at_tokens", 0) + return int(value) if value > 0 else 0 + + +def keep_turns(cfg: Any = None) -> int: + """How many recent turns survive verbatim under the summary. + + Clamped to at least 0 and at most 10. The upper bound is not fussiness: the + turns kept are kept in full, and this agent's turns carry tool output, so a + generous number here is a compaction that does not compact. + """ + return max(0, min(10, int(_number(cfg, "compact_keep_turns", 2)))) + + +def _number(cfg: Any, key: str, default: float) -> float: + if cfg is None: + try: + from core import config as config_mod # noqa: PLC0415 + + cfg = config_mod.load() + except Exception: # noqa: BLE001 - see `threshold` + return default + try: + value = float(cfg.get("agent", key, default)) + except (TypeError, ValueError): + return default + if value != value or value in (float("inf"), float("-inf")): + return default + return value + + +def context_tokens(usage: Any) -> int: + """The context size out of a `get_context_usage` reading, or 0. + + 0 for an unreadable reading rather than a raise, and the caller treats 0 as + "do not compact" -- a missing measurement is not evidence of a large + context, and compacting on the strength of one would throw away a + conversation for no reason. + """ + if not isinstance(usage, dict): + return 0 + try: + return max(0, int(usage.get("totalTokens") or 0)) + except (TypeError, ValueError): + return 0 + + +def should_compact(usage: Any, cfg: Any = None) -> bool: + """Is this conversation over the threshold? Pure, so it is testable.""" + limit = threshold(cfg) + return bool(limit) and context_tokens(usage) > limit + + +def seed_message(note: str, *, tokens_before: int = 0) -> str: + """The first thing the fresh conversation reads. + + Framed as a handover rather than presented as though the model wrote it, + which is the honest description and also the useful one: a model told that + this is a reconstruction knows to distrust it where it is thin, and knows it + may have to re-read a file rather than assume it remembers the contents. + + The token figure is included because it is the one piece of context about + the compaction that the note itself cannot contain. + """ + note = (note or "").strip() + if not note: + # A summary that came back empty is not a summary. Saying so beats + # seeding a session with a blank handover, which would look to the model + # like a conversation that genuinely had nothing in it. + return ( + "[Grad compacted this conversation to keep it inside its context " + "budget, and the summary came back empty. Assume you have lost the " + "earlier turns entirely: re-read anything you need from disk and " + "check the ledger for open expectations and uncollected runs before " + "continuing.]" + ) + size = f" (it had reached {tokens_before:,} tokens)" if tokens_before else "" + return ( + f"[This session was compacted to keep it inside its context budget{size}. " + "The earlier turns are gone; what follows is the note the previous " + "session left for you. Treat it as a reconstruction rather than as a " + "record -- where it is thin, re-read the file or re-run the query " + "instead of assuming.]\n\n" + f"{note}" + ) + + +async def write_handoff(client: Any, drive: Any, *, session: str | None = None) -> dict[str, Any]: + """Ask the live session for its handoff note, and meter the asking. + + `drive` is `agent.drive_turn`, passed in rather than imported: `agent` is a + top-level module that imports `core`, and reaching back the other way would + make the dependency circular for the sake of one call. + + This runs while the outgoing client is still connected, which is what makes + it cheap: the whole conversation is already in that session's prompt cache, + so the summary is one more read of context that has been read many times + already, rather than a fresh upload of the transcript. + + Returns the note and what it cost. Raises nothing of its own -- a caller + that cannot get a summary should carry on with the conversation it has, and + the decision about whether that is acceptable is not this function's. + """ + from agent import TurnStream # noqa: PLC0415 - see the docstring + + stream = TurnStream() + result = await drive( + client, + HANDOFF_PROMPT, + stream, + session=session, + stage=quota_log.STAGE_COMPACT, + role="compaction", + ) + return { + "note": stream.text, + "quota": (result or {}).get("quota"), + "sdk_session_id": (result or {}).get("sdk_session_id"), + } + + +def record(*, tokens_before: int, tokens_after: int, note: str, cost: Any = None) -> dict[str, Any]: + """The transcript entry a compaction leaves behind. + + A record rather than a silent replacement, because the alternative -- what + the CLI does on its own -- is a transcript that still shows twenty turns + beside a model that remembers one. The user is looking at the evidence for a + belief the agent no longer holds, and nothing on screen distinguishes that + from the agent having forgotten something it should not have. + + `role` is `system`: `ui/app.py:restore` keeps records whose role it knows and + drops the rest, so this has to be one the chat window will draw. + """ + counts = quota_log.counts(cost or {}) + return { + "role": "system", + "kind": "compaction", + "text": ( + f"**Compacted.** The conversation reached {tokens_before:,} tokens and was " + "summarised into a handover note; the turns above are still here to read, " + "but the agent's memory of them is now that note." + ), + "note": note, + "tokens_before": tokens_before, + "tokens_after": tokens_after, + "cost_tokens": counts, + } diff --git a/core/config.py b/core/config.py index 03469e4..0a07b27 100644 --- a/core/config.py +++ b/core/config.py @@ -27,6 +27,33 @@ "stale_grace_factor": 3.0, "stale_grace_floor_s": 1800, }, + # What one token of each kind counts as against a `quota_tokens` ceiling. + # + # These exist because the ceiling used to count `input + output` and nothing + # else, and on the first fortnight of real use that was 149k tokens out of + # 12.5M actually moved -- 1.2% of the flow. The other 98.8% is cache reads, + # which are what a long context costs on every tool round-trip, and the row + # in the README promising that token spend is "bounded, not merely measured" + # could not see any of it. + # + # The weights are ratios against one input token, taken from published + # per-token pricing: a cache read is a tenth of an input token, a cache write + # is 1.25 of one. Output is left at 1.0 rather than at its true multiple so + # that an existing `quota_tokens` ceiling keeps roughly the meaning it had + # for the two components it could already see -- this change is meant to + # reveal the missing 98%, not to silently reprice the 1.2%. + # + # They are configuration and not constants for the reason §10 gives about + # the meter as a whole: subscription quota is not linear in tokens and + # Anthropic exposes no remaining balance, so this is a stated assumption you + # control rather than a mirror of anyone's billing. Set them all to 1.0 for + # a raw count, or `weight_cache_read = 0` to go back to what it did before. + "quota": { + "weight_input": 1.0, + "weight_output": 1.0, + "weight_cache_read": 0.1, + "weight_cache_write": 1.25, + }, "smoke": { # HANDOFF §6: the carve-out is hard-capped in code, not in prose. # "nothing useful can be trained inside them". @@ -184,6 +211,31 @@ # actually produces text; "omitted" is the SDK's own default and is here # so turning the feature off is a config edit rather than a code one. "reasoning": "summarized", + # Compact the conversation once it passes this many tokens of context. + # 0 disables it and leaves the matter to the CLI underneath. + # + # There is a threshold either way -- the CLI autocompacts on its own, and + # a live session reports it as 967,000 of a 1,000,000 window. That is a + # ceiling in the sense that a wall at the end of a runway is: by the time + # it is reached every tool round-trip has been re-reading the better part + # of a million cached tokens for a long time. 300k is roughly a third of + # the way in, which keeps a long session's per-turn cost bounded while + # leaving room for the kind of turn this agent actually runs -- the + # largest one in the ledger so far read 10.1M cached tokens. + # + # Compacting is not free and not obviously cheap: the summary costs a + # turn, and the session it seeds starts with a cold prompt cache, so the + # first turn after a compaction pays cache *writes* (1.25x) where it + # would have paid cache *reads* (0.1x). Compacting too eagerly costs more + # than not compacting. `python -m tools.quota summary --json` is where + # that trade becomes visible, which is why the accounting split landed + # before this did. + "compact_at_tokens": 300_000, + # How many of the most recent turns survive a compaction verbatim, below + # the summary. The summary is a model's account of the conversation and + # the last exchange is the one it is worst at compressing, because it has + # not yet had a consequence. + "compact_keep_turns": 2, }, "hosts": {}, } diff --git a/core/quota_log.py b/core/quota_log.py index bfbf326..d28fb58 100644 --- a/core/quota_log.py +++ b/core/quota_log.py @@ -18,6 +18,11 @@ # Funnel stages plus the main loop. Free-form strings are allowed; these are the # ones the UI knows how to group. STAGE_MAIN = "main" +#: The summary a compaction writes. Its own stage rather than folded into +#: `main`, because "what did compacting cost me" is the question that decides +#: whether the threshold is set right, and it is unanswerable if the compaction +#: turn is filed under the conversation it was compacting. +STAGE_COMPACT = "compaction" STAGE_EXPAND = "funnel.expand" # stage 0 STAGE_RETRIEVE = "funnel.retrieve" # stage 1 (free, logged for latency) STAGE_RERANK = "funnel.rerank" # stage 2 (credits, not quota) @@ -117,6 +122,83 @@ def entries() -> list[dict[str, Any]]: return jsonl.read(paths.quota_path()) +# --------------------------------------------------------------------------- +# what a token counts as +# --------------------------------------------------------------------------- +#: The four kinds, and the config key that weights each. Ordered as they are +#: displayed, which is also cheapest-to-dearest for everything except the first. +KINDS: tuple[tuple[str, str], ...] = ( + ("input_tokens", "weight_input"), + ("output_tokens", "weight_output"), + ("cache_read_tokens", "weight_cache_read"), + ("cache_write_tokens", "weight_cache_write"), +) + +#: Used when the config cannot be read at all. Same numbers as `config.DEFAULTS` +#: -- duplicated rather than imported, because `core.budget` calls into here on +#: the gate path and accounting must not be what takes a session down. +FALLBACK_WEIGHTS: dict[str, float] = { + "weight_input": 1.0, + "weight_output": 1.0, + "weight_cache_read": 0.1, + "weight_cache_write": 1.25, +} + + +def weights(cfg: Any = None) -> dict[str, float]: + """The `[quota]` weights, as floats, with every key present. + + A weight that is missing, non-numeric or negative falls back rather than + raising: this is read on the path that decides whether a turn may be issued, + and a typo in `grad.toml` should not be able to strand a session. A negative + weight is refused specifically because it would make spending *lower* the + measured total, which is the one error here that a ceiling cannot survive. + """ + if cfg is None: + try: + from core import config as config_mod # noqa: PLC0415 + + cfg = config_mod.load() + except Exception: # noqa: BLE001 - see the docstring + return dict(FALLBACK_WEIGHTS) + out: dict[str, float] = {} + for _, key in KINDS: + try: + value = float(cfg.get("quota", key, FALLBACK_WEIGHTS[key])) + except (TypeError, ValueError): + value = FALLBACK_WEIGHTS[key] + if value != value or value in (float("inf"), float("-inf")) or value < 0: + value = FALLBACK_WEIGHTS[key] + out[key] = value + return out + + +def counts(row: Any) -> dict[str, int]: + """The four raw token counts of one record, defaulting to zero.""" + get = row.get if isinstance(row, dict) else (lambda k, d=0: getattr(row, k, d)) + out: dict[str, int] = {} + for field, _ in KINDS: + try: + out[field] = int(get(field, 0) or 0) + except (TypeError, ValueError): + out[field] = 0 + return out + + +def billable(row: Any, weight: dict[str, float] | None = None) -> float: + """One record's tokens as a single weighted number. + + **This is the only place the four kinds become one.** `core/budget.py` + charges a ceiling with it, `summarise` totals it, and the UI meters it, so a + change to what a cache read is worth lands everywhere at once. The four raw + counts stay in the record and stay in every summary -- the weighting is how + they are *compared*, never a substitute for having them. + """ + weight = weights() if weight is None else weight + n = counts(row) + return sum(n[field] * weight.get(key, FALLBACK_WEIGHTS[key]) for field, key in KINDS) + + def summarise(days: int | None = None, *, project: str | None = None) -> dict[str, Any]: """Totals by stage, by role, and by project. @@ -143,21 +225,37 @@ def summarise(days: int | None = None, *, project: str | None = None) -> dict[st kept.append(r) rows = kept + weight = weights() + def _fold(key: str, fallback: str) -> dict[str, dict[str, Any]]: out: dict[str, dict[str, Any]] = {} for r in rows: node = out.setdefault( str(r.get(key) or fallback), {"calls": 0, "input_tokens": 0, "output_tokens": 0, - "cache_read_tokens": 0, "cache_write_tokens": 0, "credits_usd": 0.0}, + "cache_read_tokens": 0, "cache_write_tokens": 0, + "billable_tokens": 0.0, "credits_usd": 0.0}, ) node["calls"] += 1 - for k in ("input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens"): + for k, _ in KINDS: node[k] += int(r.get(k, 0) or 0) + node["billable_tokens"] += billable(r, weight) node["credits_usd"] += float(r.get("credits_usd", 0.0) or 0.0) - return {k: {**v, "credits_usd": round(v["credits_usd"], 4)} for k, v in sorted(out.items())} + return { + k: {**v, + "billable_tokens": round(v["billable_tokens"]), + "credits_usd": round(v["credits_usd"], 4)} + for k, v in sorted(out.items()) + } by_stage = _fold("stage", "unknown") + # `total_tokens` stays input + output, because that is what it has always + # meant and something reads every field in here. The number that a ceiling + # is charged against is `billable_tokens`, and the four raw counts are + # reported beside both so the difference between them is visible rather than + # buried in a weight -- on the first fortnight of use the two differ by a + # factor of twelve, and a reader who cannot see why would be right not to + # trust either. total_tokens = sum(n["input_tokens"] + n["output_tokens"] for n in by_stage.values()) return { "window_days": days, @@ -168,6 +266,11 @@ def _fold(key: str, fallback: str) -> dict[str, dict[str, Any]]: "by_role": _fold("role", "untagged"), "by_project": _fold("project", "unassigned"), "total_tokens": total_tokens, + "totals": { + field: sum(n[field] for n in by_stage.values()) for field, _ in KINDS + }, + "billable_tokens": round(sum(n["billable_tokens"] for n in by_stage.values())), + "weights": weight, "total_credits_usd": round(sum(n["credits_usd"] for n in by_stage.values()), 4), # Anthropic exposes no remaining-quota API and the Max 5x window is # opaque, so this is self-measured usage against an assumed budget -- diff --git a/core/traces.py b/core/traces.py new file mode 100644 index 0000000..31adbc0 --- /dev/null +++ b/core/traces.py @@ -0,0 +1,231 @@ +"""What happened in a session, as tags a later query can slice on. + +HANDOFF §12 step 3 says the eval set is harvested from a week of real use rather +than authored cold, because "a benchmark of imagined queries measures the +imagination". That step has a prerequisite nobody wrote down: the week of real +use has to leave something behind that can be sliced. A directory of transcripts +is a record, but "show me every session where a submitter refused" is a +full-text search over prose, and the answer depends on how the refusal happened +to be phrased. + +So each trajectory gets tags, in the shape ml-intern's `sft/tagger.py` uses -- +`namespace:value` strings, deduplicated, no filtering and no mutation -- and a +downstream pass selects on them. The namespaces here are not theirs, because the +interesting facts about a session are not the same: + + tool: a CLI the agent actually ran, by module name + gate: a gate that refused, by the exit code it refused with + ledger: expectation and verdict traffic: expect, verdict, collect… + outcome: how the session ended + turns: short (<5) / medium (5-20) / long (>20) + cost: by weighted tokens: low (<100k) / med (<1M) / high + search: whether the retrieval funnel was reached for, and how often + compaction: how many times the conversation was compacted + +`gate:` is the namespace this project has and ml-intern does not, and it is the +one worth having. Every claim in the README's table is that some gate refuses +under some condition; a corpus of real sessions tagged by which gate refused is +the difference between believing that and knowing it. It is also the raw +material for the gates-on/gates-off comparison, which is the only measurement +that is about this harness rather than about the model underneath it. + +**Nothing here reads a file.** The input is a trajectory -- the list of records +`ui/app.py:Session.restore` produces -- so tagging is a pure function over data, +testable without a workspace, and `tools/traces.py` owns the reading. Tags are +metadata and never a filter: this module's job is to describe a session, never +to decide that one does not count. +""" + +from __future__ import annotations + +import re +from typing import Any, Iterable + +from core import quota_log + +#: `python -m tools.X` / `python -m tools.X ...` in a Bash command. Anchored to +#: the module path rather than to a bare word so that a session *discussing* +#: `report check` is not tagged as having run it -- the tags exist to find +#: sessions where something happened, and prose is not an event. +TOOL_RE = re.compile(r"\bpython\s+-m\s+tools\.([a-z_]+)") + +#: A command that only asked what the interface is. Tagged for the module, since +#: reaching for a tool is a fact about the session, but not for the verb. +HELP_RE = re.compile(r"(?:^|\s)(?:--help|-h)(?:\s|$)") + +#: A ledger verb, as it appears after the module. `expect` and `verdict` are the +#: two halves of the pre-registration loop and are the point of the namespace; +#: the rest are here so that "an expectation was opened and never judged" is a +#: query rather than an inference. +LEDGER_VERBS = ("expect", "verdict", "falsify", "verify", "query", "collect", "submit") + +#: Exit codes that mean a gate refused, from `core/errors.py`. Named here rather +#: than imported as a set so the tag carries the *meaning* and not the number: a +#: corpus tagged `gate:4` ages badly the first time a code is renumbered. +GATE_EXITS = { + 4: "preflight", + 5: "expectation", + 6: "spend", + 7: "stale", + 12: "project_budget", +} + +#: Turn-count buckets, as (upper bound exclusive, label). The last is unbounded. +TURN_BUCKETS = ((5, "short"), (20, "medium")) +#: Weighted-token buckets, same shape. 100k and 1M because those are roughly +#: "one question" and "an afternoon" on the usage measured so far. +COST_BUCKETS = ((100_000, "low"), (1_000_000, "med")) + + +def tag_session(trajectory: Iterable[Any], *, usage: Iterable[Any] = ()) -> list[str]: + """Tags for one trajectory. Pure, order-stable, deduplicated. + + `usage` is this session's rows from `ledger/quota.jsonl`, which is where the + cost lives -- the transcript records what was said and never what it cost. + Passing none is fine and simply leaves the session untagged for cost, which + is honest: an unmeasured session is not a cheap one. + """ + records = [r for r in trajectory if isinstance(r, dict)] + tags: list[str] = [] + + turns = sum(1 for r in records if r.get("role") == "user") + tags.append(f"turns:{_bucket(turns, TURN_BUCKETS, 'long')}") + + compactions = sum(1 for r in records if r.get("kind") == "compaction") + if compactions: + tags.append(f"compaction:{compactions}") + + searches = 0 + for command in commands(records): + # A verb asked about is not a verb run. `ledger expect --help` would + # otherwise tag the session `ledger:expect`, and "sessions where an + # expectation was registered" is precisely the query this namespace + # exists to answer -- a corpus that cannot tell reading the interface + # from using it is no use for the gates-on/gates-off comparison. + # Measured on the real corpus, where four of the five `ledger:` verbs + # on the busiest session came from `--help` calls. + # + # The *module* is still tagged either way: reaching for a tool at all is + # a fact about the session, and one that reached for `ledger` and then + # did nothing with it is a more interesting row than one that never + # thought of it. + asking = bool(HELP_RE.search(command)) + for module in TOOL_RE.findall(command): + tags.append(f"tool:{module}") + if asking: + continue + if module == "paper_search": + searches += 1 + if module == "ledger": + tags.extend(f"ledger:{v}" for v in LEDGER_VERBS if _has_word(command, v)) + if module in ("jobs", "gpu"): + tags.extend(f"ledger:{v}" for v in ("submit", "collect") if _has_word(command, v)) + if searches: + tags.append(f"search:{searches}") + + for code in _exit_codes(records): + name = GATE_EXITS.get(code) + if name: + tags.append(f"gate:{name}") + + tags.append(f"outcome:{outcome(records)}") + + total = sum(quota_log.billable(row) for row in usage if isinstance(row, dict)) + if total: + tags.append(f"cost:{_bucket(total, COST_BUCKETS, 'high')}") + + return _dedupe(tags) + + +def outcome(records: list[dict[str, Any]]) -> str: + """How the session ended, from its last assistant turn. + + Five endings, and the ordering of the checks is the meaning. A session that + was refused by the budget and *then* went on to do more is not a refused + session, so only the last turn is consulted -- which is also why this is not + simply "did the word 'refused' ever appear". + """ + if not records: + return "empty" + last = records[-1] + if last.get("role") == "user": + # A prompt with no answer under it: the turn died before it settled, or + # the app was closed mid-turn. Either way nothing came back. + return "unanswered" + if last.get("kind") == "compaction": + return "compacted" + text = str(last.get("text") or "") + if "the session failed:" in text: + return "errored" + if "Refusing the next turn" in text or "token allocation" in text: + return "budget_refused" + return "completed" + + +def commands(records: list[dict[str, Any]]) -> list[str]: + """Every Bash command the agent actually ran. + + Off the `tool` blocks rather than out of the prose, which is the same + distinction the chat window draws: a command the agent ran and a command it + said it would run are different events, and only the first is evidence. + """ + out: list[str] = [] + for record in records: + for block in record.get("blocks") or []: + if not isinstance(block, dict) or block.get("kind") != "tool": + continue + if str(block.get("name") or "").lower() != "bash": + continue + out.append(str(block.get("text") or "")) + # `rows` is the rest of the tool input, and on a Bash call the + # command can land there rather than in `text` depending on which + # field `describe_tool` chose as the subject. + for row in block.get("rows") or []: + if isinstance(row, (list, tuple)) and len(row) == 2: + out.append(str(row[1])) + return out + + +def _exit_codes(records: list[dict[str, Any]]) -> list[int]: + """Exit codes reported by tool results, best effort. + + The CLIs answer with a JSON envelope carrying `exit`, and the result text is + what the block holds. Parsed leniently -- a truncated result is normal, since + `agent.clip` bounds what is kept -- because a missed code costs one tag and a + raise here would cost the whole tagging pass. + """ + codes: list[int] = [] + for record in records: + for block in record.get("blocks") or []: + if not isinstance(block, dict) or block.get("kind") != "tool": + continue + result = str(block.get("result") or "") + for match in re.finditer(r'"exit"\s*:\s*(\d+)', result): + codes.append(int(match.group(1))) + return codes + + +def _has_word(command: str, word: str) -> bool: + return re.search(rf"\b{re.escape(word)}\b", command) is not None + + +def _bucket(value: float, buckets: tuple[tuple[float, str], ...], last: str) -> str: + for limit, label in buckets: + if value < limit: + return label + return last + + +def _dedupe(tags: Iterable[str]) -> list[str]: + """Deduplicate, keeping first-seen order. + + Order-stable rather than sorted so a reader sees the session's shape -- + turns, then what it did, then how it ended -- in the order it happened. + """ + seen: set[str] = set() + out: list[str] = [] + for tag in tags: + if tag not in seen: + seen.add(tag) + out.append(tag) + return out diff --git a/pyproject.toml b/pyproject.toml index 0dd2129..29494bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,10 @@ [build-system] -requires = ["setuptools>=68"] +# 77 rather than 68 is the floor for PEP 639 (`license` as an SPDX expression +# plus `license-files`). The older `license = { text = "MIT" }` form is +# deprecated, and it was also the form that let this project claim a licence for +# two months with no LICENSE file in the tree -- metadata that says MIT and a +# repository that grants nothing is worse than saying nothing at all. +requires = ["setuptools>=77"] build-backend = "setuptools.build_meta" [project] @@ -8,7 +13,8 @@ version = "0.1.0" description = "Grad - a personal research agent for mathematics and machine learning" readme = "README.md" requires-python = ">=3.11" -license = { text = "MIT" } +license = "MIT" +license-files = ["LICENSE"] # The core (ledger, preflight, submitters) runs on the standard library plus a # file lock. Everything heavier is optional and imported lazily at the point of @@ -65,15 +71,23 @@ grad = "agent:main" [tool.setuptools] packages = ["core", "tools", "ui", "ui.windows"] +# Top-level modules, not packages. Without these the installed `grad` command +# cannot import `agent:main`, and `hooks` would be missing under it. +# +# This key belongs to `[tool.setuptools]` and spent its first months one table +# lower, under `package-data`, where setuptools reads keys as *package names* -- +# so it declared package data for a package called `py-modules` and declared no +# top-level modules at all. The editable install hid it completely, because an +# editable install puts the source tree on the path and `import agent` resolves +# whether or not the wheel would have contained it. Only a real wheel would have +# found this, and `grad --update` never builds one. +py-modules = ["agent", "hooks"] # The stylesheet is generated from `ui/tokens.py`, but `tiling.js` and any # vendored fonts are real files and have to travel with the wheel -- without # them an installed Grad has no pane dragging and no typefaces. [tool.setuptools.package-data] ui = ["static/*.js", "static/fonts/*", "assets/katex/**/*"] -# Top-level modules, not packages. Without these the installed `grad` command -# cannot import `agent:main`, and `hooks` would be missing under it. -py-modules = ["agent", "hooks"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/tests/test_context_and_compaction.py b/tests/test_context_and_compaction.py new file mode 100644 index 0000000..d730f6d --- /dev/null +++ b/tests/test_context_and_compaction.py @@ -0,0 +1,279 @@ +"""The context meter, the weighted quota ceiling, and compaction. + +Three changes that arrived together because they are one argument. The ceiling +counted `input + output` and nothing else, which on the first fortnight of real +use was 149k tokens out of 12.5M actually moved; the missing 98% is cache reads; +cache reads are what a long context costs on every tool round-trip; and the only +lever on that is where the conversation gets compacted. So: measure all four +kinds, show how full the window is, and compact somewhere you chose. + +The weighting tests use real records through `quota_log`, and the budget test +charges a real ceiling, for the reason `tests/test_budget.py` states -- a mock of +a gate proves nothing about the gate. The compaction tests are against the pure +half, which is deliberate: what is worth pinning down is *when* it fires and +*what survives*, and neither of those needs an SDK. +""" + +from __future__ import annotations + +import pytest + +from core import budget, compaction, quota_log +from ui import models + + +# --------------------------------------------------------------------------- +# what a token counts as +# --------------------------------------------------------------------------- +def test_cache_reads_reach_the_ceiling_at_a_tenth_of_an_input_token(): + """The bug this whole change exists for, at its smallest. + + A turn that reads a large cached context and says little is the shape of + every turn in a long session, and it used to register as almost nothing. + """ + row = { + "input_tokens": 100, + "output_tokens": 100, + "cache_read_tokens": 1_000_000, + "cache_write_tokens": 0, + } + weights = { + "weight_input": 1.0, "weight_output": 1.0, + "weight_cache_read": 0.1, "weight_cache_write": 1.25, + } + assert quota_log.billable(row, weights) == pytest.approx(100_200.0) + # What it did before: the same turn, counted as 200 tokens. + assert row["input_tokens"] + row["output_tokens"] == 200 + + +def test_the_weights_come_from_config_and_a_zero_restores_the_old_behaviour(workspace): + (workspace / "config").mkdir(exist_ok=True) + (workspace / "config" / "grad.toml").write_text( + "[quota]\nweight_cache_read = 0.0\nweight_cache_write = 0.0\n", encoding="utf-8" + ) + from core import config as config_mod + + config_mod._cache.clear() + row = {"input_tokens": 5, "output_tokens": 7, "cache_read_tokens": 9_000, "cache_write_tokens": 900} + assert quota_log.billable(row, quota_log.weights()) == pytest.approx(12.0) + + +@pytest.mark.parametrize("bad", ["-1.0", "'abc'", "nan"]) +def test_an_unusable_weight_falls_back_rather_than_stranding_a_session(workspace, bad): + """This is read on the path that decides whether a turn may be issued. + + A negative weight is refused specifically: it would make spending *lower* + the measured total, which is the one error a ceiling cannot survive. + """ + (workspace / "config").mkdir(exist_ok=True) + (workspace / "config" / "grad.toml").write_text( + f"[quota]\nweight_cache_read = {bad}\n", encoding="utf-8" + ) + from core import config as config_mod + + config_mod._cache.clear() + assert quota_log.weights()["weight_cache_read"] == quota_log.FALLBACK_WEIGHTS["weight_cache_read"] + + +def test_a_project_ceiling_is_charged_the_weighted_total(workspace): + """Through `budget.status`, against a real ledger, not through the helper. + + The point is that the *gate* sees the cache traffic. Testing `billable` alone + would have passed while `budget.py` went on adding two fields. + """ + budget.create("proj-w", title="weighting", budget={"quota_tokens": 50_000}) + budget.set_current("proj-w") + quota_log.record( + quota_log.STAGE_MAIN, + project="proj-w", + input_tokens=10, + output_tokens=10, + cache_read_tokens=1_000_000, + ) + state = budget.status("proj-w") + tokens = state["resources"]["quota_tokens"] + assert tokens["spent"] == pytest.approx(100_020, rel=1e-6) + assert tokens["over"] is True + # And the four raw counts survive beside the one number, because a person + # asking why they hit a ceiling needs them -- on `status`, which is the + # payload every surface actually reads. + assert state["quota_token_counts"]["cache_read_tokens"] == 1_000_000 + assert state["quota_weights"]["weight_cache_read"] == 0.1 + + +def test_the_summary_reports_the_four_kinds_and_the_weighted_total(workspace): + quota_log.record(quota_log.STAGE_MAIN, input_tokens=1, output_tokens=2, + cache_read_tokens=1_000, cache_write_tokens=100) + summary = quota_log.summarise() + assert summary["totals"] == { + "input_tokens": 1, "output_tokens": 2, + "cache_read_tokens": 1_000, "cache_write_tokens": 100, + } + assert summary["billable_tokens"] == round(1 + 2 + 100 + 125) + # `total_tokens` keeps its old meaning; nothing that read it starts lying. + assert summary["total_tokens"] == 3 + + +# --------------------------------------------------------------------------- +# the context meter +# --------------------------------------------------------------------------- +def test_an_unknown_context_reads_as_unknown_rather_than_as_empty(): + """Zero and "no reading yet" look identical at a glance and only one of them + is worth acting on.""" + model = models.context_model(None) + assert model["known"] is False + assert model["label"] == "ctx —" + assert model["tone"] == "" + + +def test_the_meter_measures_against_grads_threshold_when_there_is_one(): + """The whole point of the chip. Against the CLI's 967k the same session + reads as nearly empty; against the threshold that will actually fire it + reads as two thirds gone.""" + usage = {"totalTokens": 200_000, "maxTokens": 967_000, "categories": []} + cli = models.context_model(usage) + grad = models.context_model(usage, compact_at=300_000) + assert cli["limit_source"] == "cli" + assert cli["fraction"] == pytest.approx(200_000 / 967_000) + assert grad["limit_source"] == "grad" + assert grad["fraction"] == pytest.approx(2 / 3) + + +def test_a_threshold_above_the_models_window_does_not_win(): + """A threshold the conversation can never reach is not the binding one, and + drawing against it would show a meter that never fills while the CLI + compacts underneath.""" + usage = {"totalTokens": 500_000, "maxTokens": 967_000} + model = models.context_model(usage, compact_at=2_000_000) + assert model["limit_source"] == "cli" + assert model["limit"] == 967_000 + + +def test_the_chip_changes_tone_before_compaction_rather_than_after(): + near = models.context_model({"totalTokens": 295_000, "maxTokens": 1_000_000}, compact_at=300_000) + warn = models.context_model({"totalTokens": 240_000, "maxTokens": 1_000_000}, compact_at=300_000) + calm = models.context_model({"totalTokens": 10_000, "maxTokens": 1_000_000}, compact_at=300_000) + assert near["tone"] == "attention" + assert warn["tone"] == "warn" + assert calm["tone"] == "" + + +def test_free_space_is_not_listed_as_something_using_the_context(): + """It is the complement of everything else: always the largest entry in the + CLI's own breakdown, and never a consumer.""" + model = models.context_model( + { + "totalTokens": 1_469, + "maxTokens": 1_000_000, + "categories": [ + {"name": "Free space", "tokens": 998_531}, + {"name": "Skills", "tokens": 1_469}, + ], + } + ) + assert [c["name"] for c in model["categories"]] == ["Skills"] + + +# --------------------------------------------------------------------------- +# compaction +# --------------------------------------------------------------------------- +def test_compaction_is_off_unless_a_threshold_is_configured(workspace): + (workspace / "config").mkdir(exist_ok=True) + (workspace / "config" / "grad.toml").write_text( + "[agent]\ncompact_at_tokens = 0\n", encoding="utf-8" + ) + from core import config as config_mod + + config_mod._cache.clear() + cfg = config_mod.load() + assert compaction.threshold(cfg) == 0 + assert compaction.should_compact({"totalTokens": 10_000_000}, cfg) is False + + +@pytest.mark.parametrize("bad", ["-5", "nan", "'soon'"]) +def test_an_unusable_threshold_disables_compaction_rather_than_firing_every_turn(workspace, bad): + """A negative threshold is always already exceeded, which would compact + after every single turn -- the most expensive possible reading of a typo.""" + (workspace / "config").mkdir(exist_ok=True) + (workspace / "config" / "grad.toml").write_text( + f"[agent]\ncompact_at_tokens = {bad}\n", encoding="utf-8" + ) + from core import config as config_mod + + config_mod._cache.clear() + assert compaction.threshold(config_mod.load()) == 0 + + +def test_an_unreadable_context_never_triggers_a_compaction(workspace): + """A missing measurement is not evidence of a large context, and compacting + on the strength of one would discard a conversation for no reason.""" + cfg = _cfg(workspace, "[agent]\ncompact_at_tokens = 100\n") + assert compaction.should_compact(None, cfg) is False + assert compaction.should_compact({}, cfg) is False + assert compaction.should_compact({"totalTokens": "lots"}, cfg) is False + assert compaction.should_compact({"totalTokens": 101}, cfg) is True + + +def test_the_seed_says_it_is_a_reconstruction_and_carries_the_note(): + seed = compaction.seed_message("I was halfway through run-3.", tokens_before=412_000) + assert "412,000" in seed + assert "reconstruction" in seed + assert "I was halfway through run-3." in seed + + +def test_an_empty_summary_seeds_a_warning_rather_than_a_blank_handover(): + """A blank handover looks to the model like a conversation that genuinely + had nothing in it, which is the one reading that must not happen.""" + seed = compaction.seed_message(" ") + assert "came back empty" in seed + assert "re-read" in seed + + +def test_the_handoff_prompt_asks_for_the_ledger_state_a_generic_summary_drops(): + """The Grad-specific half. An expectation registered and not yet judged, or a + run submitted and not yet collected, is state the next turn is expected to + act on -- and losing it does not read as a bad summary, it reads as an agent + that abandoned a run halfway.""" + prompt = compaction.HANDOFF_PROMPT + for owed in ("expectation", "collected", "verdict", "gate"): + assert owed in prompt + + +def test_a_compaction_leaves_a_record_the_chat_window_can_draw(): + from ui.app import ROLES + + record = compaction.record( + tokens_before=310_000, + tokens_after=0, + note="what I was doing", + cost={"output_tokens": 900, "cache_read_tokens": 300_000}, + ) + # `restore` keeps records whose role it knows and drops the rest, so a + # marker with an unknown role would vanish on the next reload -- leaving a + # transcript that reads as one continuous conversation beside a model that + # remembers only the tail of it. + assert record["role"] in ROLES + assert record["kind"] == "compaction" + assert "310,000" in record["text"] + assert record["cost_tokens"]["cache_read_tokens"] == 300_000 + + +def test_the_compaction_turn_is_charged_to_its_own_stage(workspace): + """ml-intern's context manager carries a comment saying that not metering + this "used to hide a significant share of hosted inference spend". The same + hole was available here for free, and folding it into `main` would also hide + the number that says whether the threshold is set right.""" + quota_log.record(quota_log.STAGE_COMPACT, output_tokens=1_200, cache_read_tokens=250_000) + summary = quota_log.summarise() + assert quota_log.STAGE_COMPACT in summary["by_stage"] + assert quota_log.STAGE_COMPACT != quota_log.STAGE_MAIN + assert summary["by_stage"][quota_log.STAGE_COMPACT]["billable_tokens"] == round(1_200 + 25_000) + + +def _cfg(workspace, text: str): + (workspace / "config").mkdir(exist_ok=True) + (workspace / "config" / "grad.toml").write_text(text, encoding="utf-8") + from core import config as config_mod + + config_mod._cache.clear() + return config_mod.load() diff --git a/tests/test_traces.py b/tests/test_traces.py new file mode 100644 index 0000000..72be2a6 --- /dev/null +++ b/tests/test_traces.py @@ -0,0 +1,211 @@ +"""Tagging a session, and harvesting evals out of one (HANDOFF §12 step 3). + +The eval set is meant to be harvested from a week of real use rather than +authored cold. That step has a prerequisite: the week has to leave something +sliceable behind. These tests are about the two ways that can quietly fail -- +tagging a session for something it did not do, and harvesting a question twice. + +Both are precision failures rather than crashes, which is why they are worth +tests. A tagger that is merely noisy produces a corpus that looks usable and +answers the gates-on/gates-off question wrongly. +""" + +from __future__ import annotations + +import json + +from core import traces + + +def bash(command: str, *, result: str = "") -> dict: + """One tool block, shaped the way `agent.tool_block` shapes it.""" + return { + "kind": "tool", "id": "t1", "name": "Bash", + "title": command[:60], "text": command, + "rows": [], "status": "ok", "result": result, + } + + +def turn(*blocks: dict, text: str = "") -> dict: + return {"role": "assistant", "text": text, "blocks": list(blocks)} + + +def asked(text: str = "do the thing") -> dict: + return {"role": "user", "text": text} + + +# --------------------------------------------------------------------------- +# what a session did +# --------------------------------------------------------------------------- +def test_a_command_the_agent_ran_is_tagged_and_prose_about_it_is_not(): + """The same distinction the chat window draws: a command the agent ran and a + command it said it would run are different events, and only the first is + evidence.""" + ran = [asked(), turn(bash("python -m tools.ledger expect --task t --quantity q"))] + said = [asked(), turn(text="Next I will run `python -m tools.ledger expect` for this.")] + assert "ledger:expect" in traces.tag_session(ran) + assert "ledger:expect" not in traces.tag_session(said) + assert "tool:ledger" not in traces.tag_session(said) + + +def test_asking_for_an_interface_is_not_using_it(): + """Measured on the real corpus, where four of the five `ledger:` verbs on the + busiest session came from `--help` calls. "Sessions where an expectation was + registered" is the query this namespace exists to answer.""" + records = [asked(), turn(bash("python -m tools.ledger expect --help 2>&1 | head -40"))] + tags = traces.tag_session(records) + assert "ledger:expect" not in tags + # The module still counts: reaching for a tool at all is a fact about the + # session, and one that reached for the ledger and did nothing with it is a + # more interesting row than one that never thought of it. + assert "tool:ledger" in tags + + +def test_a_gate_refusal_is_tagged_by_meaning_rather_than_by_number(): + """A corpus tagged `gate:4` ages badly the first time a code is renumbered.""" + records = [ + asked(), + turn(bash("python -m tools.jobs submit --spec s.toml --json", + result='{"ok": false, "exit": 4, "error": {"message": "no preflight"}}')), + ] + tags = traces.tag_session(records) + assert "gate:preflight" in tags + assert "gate:4" not in tags + + +def test_the_gate_namespace_covers_every_refusing_exit_code(): + """These are the rows of the README's table. A corpus that could only see + some of them would answer "does the discipline pay for itself" from a + subset.""" + from core import errors + + refusing = { + errors.EXIT_PREFLIGHT, errors.EXIT_EXPECTATION, errors.EXIT_SPEND, + errors.EXIT_STALE_RUN, errors.EXIT_PROJECT_BUDGET, + } + assert set(traces.GATE_EXITS) == refusing + + +def test_a_compaction_shows_up_in_the_tags_and_in_the_outcome(): + records = [ + asked(), + turn(text="worked on it"), + {"role": "system", "kind": "compaction", "text": "**Compacted.**"}, + ] + tags = traces.tag_session(records) + assert "compaction:1" in tags + assert "outcome:compacted" in tags + + +def test_only_the_last_turn_decides_the_outcome(): + """A session refused by the budget and then continued is not a refused + session -- which is also why this is not "did the word ever appear".""" + recovered = [ + asked(), + turn(text="Refusing the next turn; the turn that crossed the ceiling finished."), + asked(), + turn(text="done"), + ] + stopped = [asked(), turn(text="Refusing the next turn — over the token allocation.")] + assert traces.outcome(recovered) == "completed" + assert traces.outcome(stopped) == "budget_refused" + + +def test_a_prompt_with_nothing_under_it_reads_as_unanswered(): + assert traces.outcome([asked()]) == "unanswered" + assert traces.outcome([]) == "empty" + + +def test_cost_is_tagged_from_the_ledger_and_absent_when_it_was_never_measured(): + """An unmeasured session is not a cheap one, so it gets no cost tag at all + rather than `cost:low`.""" + records = [asked(), turn(text="hi")] + assert not [t for t in traces.tag_session(records) if t.startswith("cost:")] + priced = traces.tag_session( + records, usage=[{"output_tokens": 1_000, "cache_read_tokens": 20_000_000}] + ) + assert "cost:high" in priced + + +def test_tags_are_deduplicated_and_keep_the_order_things_happened(): + records = [ + asked(), + turn(bash("python -m tools.paper_search search \"a\" --json")), + turn(bash("python -m tools.paper_search search \"b\" --json")), + ] + tags = traces.tag_session(records) + assert tags.count("tool:paper_search") == 1 + assert "search:2" in tags + assert tags[0].startswith("turns:") + + +# --------------------------------------------------------------------------- +# harvesting +# --------------------------------------------------------------------------- +def test_harvested_rows_are_never_graded_for_you(workspace, monkeypatch): + """Which papers were the right answer is the one part of an eval row a trace + cannot recover. The eval README's warning about authoring cold applies to an + automated harvester as much as to a person.""" + _store_session(monkeypatch, "s-1", [ + asked(), + turn(bash('python -m tools.paper_search search "how does loss scale with width" --json')), + ]) + from tools import traces as cli + + out = cli.cmd_harvest(_args(write=False)) + assert out["searches_found"] == 1 + row = out["candidates"][0] + assert row["question"] == "how does loss scale with width" + assert row["relevant"] == [] + assert row["seed"] is False + + +def test_harvesting_twice_does_not_add_the_question_twice(workspace, monkeypatch): + """This is meant to be re-run as the corpus grows, not once.""" + _store_session(monkeypatch, "s-1", [ + asked(), + turn(bash("python -m tools.paper_search search 'equivariance error with depth' --json")), + ]) + from tools import traces as cli + + first = cli.cmd_harvest(_args(write=True)) + assert first["written"] is True and len(first["candidates"]) == 1 + second = cli.cmd_harvest(_args(write=True)) + assert second["candidates"] == [] + assert second["already_present"] == 1 + lines = (workspace / "evals" / "retrieval.jsonl").read_text(encoding="utf-8").splitlines() + assert len([l for l in lines if l.strip()]) == 1 + + +def test_harvested_ids_continue_after_the_rows_already_there(workspace, monkeypatch): + evals = workspace / "evals" + evals.mkdir(parents=True, exist_ok=True) + (evals / "retrieval.jsonl").write_text( + json.dumps({"id": "q001", "question": "a seed row", "seed": True}) + "\n", + encoding="utf-8", + ) + _store_session(monkeypatch, "s-1", [ + asked(), + turn(bash('python -m tools.paper_search search "something new" --json')), + ]) + from tools import traces as cli + + out = cli.cmd_harvest(_args(write=False)) + assert [r["id"] for r in out["candidates"]] == ["q002"] + + +def _args(*, write: bool): + import argparse + + return argparse.Namespace(write=write) + + +def _store_session(monkeypatch, session_id: str, records: list[dict]) -> None: + """Write a session file the way `ui/sessions.py` would, and list it.""" + from ui import sessions + + path = sessions.path_for(session_id) + path.parent.mkdir(parents=True, exist_ok=True) + lines = [json.dumps({"meta": True, "title": "t", "created_at": "2026-08-15T00:00:00+00:00"})] + lines += [json.dumps(r) for r in records] + path.write_text("\n".join(lines) + "\n", encoding="utf-8") diff --git a/tests/test_ui_compaction.py b/tests/test_ui_compaction.py new file mode 100644 index 0000000..1857a8d --- /dev/null +++ b/tests/test_ui_compaction.py @@ -0,0 +1,196 @@ +"""What a compaction does to a session (`ui.app.Session`). + +Compaction discards the conversation and keeps a note. That makes the note the +only copy of everything it threw away, and the lifecycle of that one string is +where all the interesting failures are: + + * it must reach the model, but not as something the user appears to have said; + * it must survive a turn that fails, because the turn immediately after a + compaction is exactly the one where a failure costs the whole session; + * the resume id must go with it, or `start()` restores the very conversation + the compaction just paid to summarise. + +None of this needs the SDK. It is about what this class does with a client, so +the client is the same fake the interrupt suite uses. +""" + +from __future__ import annotations + +import asyncio + +import pytest + +app = pytest.importorskip("ui.app", reason="the ui extra is not installed") + +from tests.test_ui_turns import FakeClient, settled # noqa: E402 + +pytestmark = pytest.mark.asyncio + +NOTE = "I had just submitted run-3 and was waiting to collect it." + + +@pytest.fixture +def session(monkeypatch): + made_session = app.Session("compaction") + made_session.notify = lambda _msg: None + + async def start() -> None: + if made_session.client is None: + made_session.client = FakeClient() + + monkeypatch.setattr(made_session, "start", start) + return made_session + + +async def _run(session, prompt: str = "carry on"): + """One turn, driven to completion against the fake client.""" + task = asyncio.create_task(session.ask(prompt, settled)) + for _ in range(200): + await asyncio.sleep(0) + if session.client is not None and session.client.prompts: + session.client.finish.set() + break + await task + return task + + +async def test_the_handover_reaches_the_model_but_not_the_transcript(session): + """It goes in front of the prompt, and the transcript records what the user + actually typed. Putting it in `settled` would show the note as though they + had written it -- and, at its length, would be the thing you scroll past + forever afterwards.""" + session.pending_seed = NOTE + await _run(session, "what were we doing?") + + sent = session.client.prompts[0] + assert NOTE in sent + assert sent.endswith("what were we doing?") + assert [r["text"] for r in session.settled if r["role"] == "user"] == ["what were we doing?"] + + +async def test_the_handover_is_sent_once(session): + session.pending_seed = NOTE + await _run(session, "first") + await _run(session, "second") + assert NOTE in session.client.prompts[0] + assert NOTE not in session.client.prompts[1] + assert session.pending_seed is None + + +async def test_a_failed_turn_gives_the_handover_back(session, monkeypatch): + """The turn right after a compaction is the one where losing this costs the + session its entire memory. Re-sending it is at worst redundant context; the + model may not have read it at all.""" + import agent + + async def explode(*_a, **_k): + raise RuntimeError("the transport died") + + monkeypatch.setattr(agent, "drive_turn", explode) + session.pending_seed = NOTE + await _run(session, "carry on") + assert session.pending_seed == NOTE + + +async def test_a_budget_refusal_gives_the_handover_back(session, monkeypatch): + """A refused turn never reached the model at all, so the note is certainly + still owed.""" + import agent + + async def refuse(*_a, **_k): + raise agent.BudgetRefused( + {"message": "out of allocation", "fix": "python -m tools.budget raise"} + ) + + monkeypatch.setattr(agent, "drive_turn", refuse) + session.pending_seed = NOTE + await _run(session, "carry on") + assert session.pending_seed == NOTE + + +async def test_compacting_drops_the_resume_id(session, monkeypatch): + """Otherwise `start()` resumes the conversation the compaction just + summarised, quietly restoring the context it paid to discard -- a cost with + no effect, and one nothing on screen would explain.""" + from core import compaction + + async def handoff(*_a, **_k): + return {"note": NOTE, "quota": None, "sdk_session_id": "old"} + + monkeypatch.setattr(compaction, "write_handoff", handoff) + await session.start() + session.sdk_session_id = "sdk-abc" + session.context = {"totalTokens": 400_000} + + outcome = await session.compact() + + assert outcome["ok"] is True + assert session.sdk_session_id is None + assert session.client is None + assert NOTE in session.pending_seed + + +async def test_a_compaction_that_cannot_summarise_discards_nothing(session, monkeypatch): + """An oversized conversation is a cost. A conversation replaced by a failed + summary is a loss.""" + from core import compaction + + async def explode(*_a, **_k): + raise RuntimeError("the model refused") + + monkeypatch.setattr(compaction, "write_handoff", explode) + await session.start() + session.sdk_session_id = "sdk-abc" + client = session.client + + outcome = await session.compact() + + assert outcome["ok"] is False + assert session.sdk_session_id == "sdk-abc" + assert session.client is client + assert session.pending_seed is None + + +async def test_the_marker_lands_in_the_transcript_and_survives_a_reload(session, monkeypatch): + """`restore` keeps records whose role it knows and drops the rest, so a + marker it filtered out would leave a transcript reading as one continuous + conversation beside a model that remembers only the tail.""" + from core import compaction + + async def handoff(*_a, **_k): + return {"note": NOTE, "quota": None, "sdk_session_id": None} + + monkeypatch.setattr(compaction, "write_handoff", handoff) + await session.start() + session.context = {"totalTokens": 310_000} + await session.compact() + + reopened = app.Session("compaction-reader") + reopened.session_id = session.session_id + reopened.restore() + markers = [r for r in reopened.settled if r.get("kind") == "compaction"] + assert len(markers) == 1 + assert markers[0]["note"] == NOTE + + +async def test_the_threshold_is_what_decides_and_a_dropped_client_clears_the_reading(session, monkeypatch): + """A stale high reading kept across a client swap is exactly the reading that + would trigger a needless compaction on a conversation that is now empty.""" + from core import compaction + + calls: list[str] = [] + + async def handoff(*_a, **_k): + calls.append("compacted") + return {"note": NOTE, "quota": None, "sdk_session_id": None} + + monkeypatch.setattr(compaction, "write_handoff", handoff) + monkeypatch.setattr(compaction, "threshold", lambda _cfg=None: 300_000) + await session.start() + + session.context = {"totalTokens": 100} + assert await session.maybe_compact() is None + assert calls == [] + + await session.close() + assert session.context is None diff --git a/tools/quota.py b/tools/quota.py index 71825ee..63fdbbd 100644 --- a/tools/quota.py +++ b/tools/quota.py @@ -80,12 +80,18 @@ def cmd_funnel(args: argparse.Namespace) -> dict[str, Any]: quota_log.STAGE_RERANK, quota_log.STAGE_TRIAGE, ] - rows = {s: summary["by_stage"].get(s, {"calls": 0, "input_tokens": 0, "output_tokens": 0, "credits_usd": 0.0}) for s in stages} + empty = {"calls": 0, "billable_tokens": 0, "credits_usd": 0.0} + empty.update({field: 0 for field, _ in quota_log.KINDS}) + rows = {s: summary["by_stage"].get(s, dict(empty)) for s in stages} return { "window_days": args.days, "stages": rows, + # Weighted, like every other ceiling-facing total. The funnel's two + # subagent stages send a short prompt and read a long one, so counting + # only input + output flattered them by exactly the component that makes + # them worth questioning. "quota_tokens_stage0_and_3": sum( - rows[s]["input_tokens"] + rows[s]["output_tokens"] + rows[s]["billable_tokens"] for s in (quota_log.STAGE_EXPAND, quota_log.STAGE_TRIAGE) ), "credits_usd_stage2": rows[quota_log.STAGE_RERANK]["credits_usd"], @@ -118,6 +124,11 @@ def _record_args(p: argparse.ArgumentParser) -> None: p.add_argument("--model") p.add_argument("--input-tokens", type=int, default=0) p.add_argument("--output-tokens", type=int, default=0) + # The ledger has always stored these two; until now nothing but the SDK + # translation could write them, so a hand-written record could only ever + # describe the tenth of a turn that is not cache traffic. + p.add_argument("--cache-read-tokens", type=int, default=0) + p.add_argument("--cache-write-tokens", type=int, default=0) p.add_argument("--credits-usd", type=float, default=0.0) p.add_argument("--unit", choices=["quota", "credits"], default="quota") p.add_argument("--role", help="the §16 model role this call filled") @@ -130,7 +141,11 @@ def cmd_record(args: argparse.Namespace) -> dict[str, Any]: # This is the measurement instrument for every later cost decision, so a # negative count (which would reduce reported usage) or a NaN (which is not # valid JSON and would poison every later sum) is refused rather than stored. - if args.input_tokens < 0 or args.output_tokens < 0 or args.credits_usd < 0: + counts = ( + args.input_tokens, args.output_tokens, + args.cache_read_tokens, args.cache_write_tokens, + ) + if any(n < 0 for n in counts) or args.credits_usd < 0: raise UsageError("usage values must be non-negative", fix="check the arguments") if not math.isfinite(args.credits_usd): raise UsageError("--credits-usd must be a finite number", fix="pass a real dollar amount") @@ -140,6 +155,8 @@ def cmd_record(args: argparse.Namespace) -> dict[str, Any]: model=args.model, input_tokens=args.input_tokens, output_tokens=args.output_tokens, + cache_read_tokens=args.cache_read_tokens, + cache_write_tokens=args.cache_write_tokens, credits_usd=args.credits_usd, unit=args.unit, role=args.role, diff --git a/tools/traces.py b/tools/traces.py new file mode 100644 index 0000000..d4a37e7 --- /dev/null +++ b/tools/traces.py @@ -0,0 +1,286 @@ +"""grad-traces -- what the sessions so far actually contain (HANDOFF §12 step 3). + + "The order in §12 is deliberate -- build the agent, use it for a week, + *then* harvest `evals/retrieval.jsonl` from what retrieval was actually + reached for. Authoring it cold would measure the imagination rather than + the system." + +The harvesting step needs the week of use to have left something sliceable +behind, and until now it had not: the transcripts are a record, but "every +session where a submitter refused" was a full-text search whose answer depended +on phrasing. `core/traces.py` tags each trajectory; this reads the sessions and +applies it. + +`harvest` is the verb the handoff is actually asking for. It pulls the questions +that were really put to `paper_search` out of the transcripts and writes them as +candidate rows in the `evals/retrieval.jsonl` schema -- **unlabelled**, with +`relevant` empty and `seed` false, because which papers were the right answer is +the one part of an eval row that cannot be recovered from a trace. Grading is +yours. What this removes is the part that was never worth a human: remembering +what you asked. + +The tags are metadata and never a filter. Nothing here decides that a session +does not count. +""" + +from __future__ import annotations + +import argparse +import json +import re +from typing import Any + +from core import paths, quota_log, traces +from core.cli import Cli, main +from core.errors import NotFound, UsageError + +cli = Cli( + "grad-traces", + "Tag stored sessions, and harvest evaluation candidates from real use.", + epilog=( + "Tags are `namespace:value` and are metadata, not a filter.\n" + "`gate:` is the one worth having: every claim in the README is that some gate\n" + "refuses under some condition, and a corpus tagged by which one refused is the\n" + "difference between believing that and knowing it." + ), +) + + +def _sessions() -> Any: + """`ui.sessions`, imported at the point of use. + + It reads and writes files and imports nothing from NiceGUI, so this is safe + on a machine with no `ui` extra -- but the import still belongs here rather + than at module scope, so that `--help` works on a checkout where `ui/` has + been trimmed away. + """ + from ui import sessions # noqa: PLC0415 + + return sessions + + +def _trajectory(path: Any) -> list[dict[str, Any]]: + """The records of one session file, skipping the meta line and any junk. + + Deliberately the same tolerance `ui/app.py:restore` applies: these files + outlive the version that wrote them, and one malformed line should cost that + line rather than the command. + """ + if not path.exists(): + return [] + out: list[dict[str, Any]] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(record, dict) and record.get("role"): + out.append(record) + return out + + +def _usage_by_session() -> dict[str, list[dict[str, Any]]]: + """`ledger/quota.jsonl`, grouped by the session that spent it. + + Read once for the whole command rather than per session: the ledger is one + append-only file and re-reading it per session turns a listing into a + quadratic one. + """ + out: dict[str, list[dict[str, Any]]] = {} + for row in quota_log.entries(): + key = row.get("session") + if isinstance(key, str) and key: + out.setdefault(key, []).append(row) + return out + + +def _tagged(session: dict[str, Any], usage: dict[str, list[dict[str, Any]]]) -> dict[str, Any]: + sessions = _sessions() + session_id = session["id"] + records = _trajectory(sessions.path_for(session_id)) + rows = usage.get(session_id, []) + return { + "id": session_id, + "title": session.get("title") or "", + "created_at": session.get("created_at"), + "turns": sum(1 for r in records if r.get("role") == "user"), + "billable_tokens": round(sum(quota_log.billable(r) for r in rows)), + "tags": traces.tag_session(records, usage=rows), + } + + +def _list_args(p: argparse.ArgumentParser) -> None: + p.add_argument("--tag", action="append", help="only sessions carrying this tag (repeatable, AND)") + + +@cli.command("list", "every stored session, with its tags", setup=_list_args) +def cmd_list(args: argparse.Namespace) -> dict[str, Any]: + usage = _usage_by_session() + rows = [_tagged(s, usage) for s in _sessions().listing()] + wanted = set(args.tag or ()) + if wanted: + rows = [r for r in rows if wanted <= set(r["tags"])] + return { + "sessions": rows, + "count": len(rows), + "filtered_by": sorted(wanted), + # Every tag in the corpus with how many sessions carry it. This is the + # part worth reading first: it says what a week of use actually + # consisted of, which is usually not what it felt like it consisted of. + "tag_counts": _counts(rows), + } + + +def _counts(rows: list[dict[str, Any]]) -> dict[str, int]: + out: dict[str, int] = {} + for row in rows: + for tag in row["tags"]: + out[tag] = out.get(tag, 0) + 1 + return dict(sorted(out.items(), key=lambda kv: (-kv[1], kv[0]))) + + +@cli.command( + "show", + "one session's tags, and the commands behind them", + setup=lambda p: p.add_argument("session_id"), +) +def cmd_show(args: argparse.Namespace) -> dict[str, Any]: + sessions = _sessions() + if not sessions.is_id(args.session_id): + raise UsageError( + f"not a session id: {args.session_id!r}", + fix="python -m tools.traces list --json", + ) + path = sessions.path_for(args.session_id) + if not path.exists(): + raise NotFound( + f"no session {args.session_id!r}", + fix="python -m tools.traces list --json", + ) + records = _trajectory(path) + rows = _usage_by_session().get(args.session_id, []) + return { + "id": args.session_id, + "meta": sessions.read_meta(path), + "tags": traces.tag_session(records, usage=rows), + "outcome": traces.outcome(records), + "commands": traces.commands(records), + "questions": _questions(records), + "billable_tokens": round(sum(quota_log.billable(r) for r in rows)), + } + + +#: How a search reaches `paper_search`, as the system prompt tells the agent to +#: write it. Both quote styles, because the agent writes whichever the shell +#: wants and a harvester that only knew one would silently find half of them. +SEARCH_RE = re.compile( + r"python\s+-m\s+tools\.paper_search\s+(?:search|local)\s+(?P\"[^\"]+\"|'[^']+')" +) + + +def _questions(records: list[dict[str, Any]]) -> list[str]: + """The questions actually put to the funnel, in the order they were asked.""" + out: list[str] = [] + for command in traces.commands(records): + for match in SEARCH_RE.finditer(command): + question = match.group("q")[1:-1].strip() + if question and question not in out: + out.append(question) + return out + + +def _harvest_args(p: argparse.ArgumentParser) -> None: + p.add_argument( + "--write", + action="store_true", + help="append the candidates to evals/retrieval.jsonl instead of printing them", + ) + + +@cli.command("harvest", "candidate eval rows from the searches actually run", setup=_harvest_args) +def cmd_harvest(args: argparse.Namespace) -> dict[str, Any]: + """Turn real searches into unlabelled `evals/retrieval.jsonl` rows. + + Unlabelled on purpose. `relevant` is empty and has to be filled in by hand, + because which papers were the right answer is the one thing a transcript + cannot say -- the trace records what was asked and what came back, never + whether what came back was any good. The eval README's own warning applies + to an automated harvester as much as to a person: a row whose relevance + labels were guessed measures the guess. + + Existing rows are never rewritten and duplicates are never appended, so this + is safe to re-run as the corpus grows -- which is the intended use. It is + the mechanism §12 step 3 describes, not a one-off migration. + """ + sessions = _sessions() + seen: list[str] = [] + rows: list[dict[str, Any]] = [] + for session in sessions.listing(): + records = _trajectory(sessions.path_for(session["id"])) + for question in _questions(records): + if question in seen: + continue + seen.append(question) + rows.append( + { + "id": "", # assigned below, after the existing file is read + "question": question, + "asked_at": (session.get("created_at") or "")[:10], + "relevant": [], + "notes": f"harvested from session {session['id']}; relevance not yet graded", + "seed": False, + } + ) + + path = paths.root() / "evals" / "retrieval.jsonl" + existing, known = _existing_eval_rows(path) + fresh = [r for r in rows if r["question"] not in known] + for offset, row in enumerate(fresh, start=len(existing) + 1): + row["id"] = f"q{offset:03d}" + + written = False + if args.write and fresh: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as handle: + for row in fresh: + handle.write(json.dumps(row, ensure_ascii=False) + "\n") + written = True + + return { + "path": str(path), + "searches_found": len(seen), + "already_present": len(seen) - len(fresh), + "candidates": fresh, + "written": written, + "note": ( + "`relevant` is empty in every row: which papers were the right answer is the " + "one part of an eval row a trace cannot recover. Grade them by hand, then " + "drop the seed rows." + ), + "fix": None if written or not fresh else "re-run with --write to append them", + } + + +def _existing_eval_rows(path: Any) -> tuple[list[dict[str, Any]], set[str]]: + """What is already in the eval file, and the questions it already covers. + + Seed rows count as present. They are examples of the schema rather than real + queries, but a harvest that re-added a question a seed row already asks + would put the same question in the file twice, and the README's instruction + is to *replace* the seeds rather than pad around them. + """ + if not path.exists(): + return [], set() + rows: list[dict[str, Any]] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(row, dict): + rows.append(row) + return rows, {str(r.get("question") or "") for r in rows} + + +if __name__ == "__main__": + main(cli) diff --git a/ui/app.py b/ui/app.py index df45d3e..04cd6d4 100644 --- a/ui/app.py +++ b/ui/app.py @@ -45,7 +45,12 @@ from core import appdata, config as config_mod, instance, migrate, paths from ui import desktop, katex, kit, render, sessions, shell, state as state_mod -ROLES = ("user", "assistant") +#: Roles the chat window knows how to draw, and therefore the ones `restore` +#: keeps. `system` is the compaction marker: not something anyone said, but the +#: record of the moment the agent's memory of this transcript was replaced -- +#: which is the one event a reader needs in order to interpret everything above +#: it correctly. +ROLES = ("user", "assistant", "system") STATIC_URL = "/grad-static" #: The port `run()` bound, so the rest of the app can name its own origin. The @@ -121,6 +126,19 @@ def __init__(self, key: str = "default") -> None: #: for it: a fire-and-forget interrupt can outlive the turn it was aimed #: at and land on the one after it. self._stopping: asyncio.Task[None] | None = None + #: The last reading from `get_context_usage`, or None before the first + #: one. The statusline draws it; `ui/models.py:context_model` decides + #: what it means. None is a state the meter renders, not an error. + self.context: dict[str, Any] | None = None + #: One reading at a time. The call is a control request over the same + #: transport a turn is streaming on, and a slow one must not be able to + #: queue a second behind it every time the poll timer fires. + self._reading_context = False + #: The handover note a compaction left, waiting for somewhere to go. It + #: rides in front of the next prompt rather than being sent as a turn of + #: its own, which would spend a round-trip to produce an answer nobody + #: asked for. Cleared once it has been sent. + self.pending_seed: str | None = None async def start(self) -> None: if self.client is not None: @@ -245,6 +263,13 @@ async def close(self) -> None: the app, and leaks a whole set on any later reconnect. """ client, self.client = self.client, None + # The reading belongs to the client that answered it. Keeping it across a + # close would leave the meter reporting the context of a conversation + # that no longer exists -- and every path that drops a client (a session + # switch, a workspace rebind, an interrupt) is one where the next context + # is a different size, usually much smaller. A stale high reading there + # is exactly the reading that would trigger a needless compaction. + self.context = None if client is not None: try: await client.__aexit__(None, None, None) @@ -298,6 +323,12 @@ async def ask(self, prompt: str, on_settle: Any) -> None: self._idle.set() raise + # The transcript records what the *user* said; the seed is machinery and + # goes only to the model. Putting it in `settled` would show the handover + # note as though the user had typed it, which is both wrong and, given + # its length, the thing you would then have to scroll past forever. + seed, self.pending_seed = self.pending_seed, None + sent = f"{seed}\n\n---\n\n{prompt}" if seed else prompt self.settled.append({"role": "user", "text": prompt}) # The turn lands in the stream's blocks as it arrives; the chat window's # ~15 Hz timer is what turns that into something on screen. The same list @@ -312,7 +343,7 @@ async def ask(self, prompt: str, on_settle: Any) -> None: # whether the budget applies. result = await agent.drive_turn( self.client, - prompt, + sent, stream, # Recorded as it arrives rather than read off the return value, # which an interrupted turn never reaches. That id is what lets @@ -327,6 +358,7 @@ async def ask(self, prompt: str, on_settle: Any) -> None: # Not an error in the transcript's sense: the system did what it # says it does. It still has to be visible, because otherwise the # composer just goes quiet. + self.pending_seed = self.pending_seed or seed stream.note( f"\n\n**{exc.refusal['message']}**\n\n`{exc.refusal['fix']}`" ) @@ -337,6 +369,13 @@ async def ask(self, prompt: str, on_settle: Any) -> None: # message can carry a URL with a token in it, a header, or a path, # and both of those destinations are readable long after the fact. log.exception("session turn failed") + # Put the handover note back. It was consumed by a turn that did not + # complete, and it is the only remaining record of everything the + # compaction discarded -- dropping it here would mean one failed + # turn, immediately after a compaction, silently costs the session + # its entire memory. Re-sending it on the next turn is at worst + # redundant context; the model may not have read it at all. + self.pending_seed = self.pending_seed or seed # Whatever streamed before the failure is kept: a turn that died # half-way is more legible with its half than without it -- and a # tool card left mid-flight says which call it died on. @@ -474,6 +513,124 @@ async def _stopped(self) -> None: def _remember_sdk_session(self, sdk_session_id: str) -> None: self.sdk_session_id = sdk_session_id + async def read_context(self) -> dict[str, Any] | None: + """Refresh the context reading. Never raises, never blocks a turn. + + `get_context_usage` is a control request, not a model call: it costs no + tokens and it is answered by the CLI from state it already has. That is + what makes polling it reasonable at all, and it is why this is a poll + rather than something folded into `drive_turn` -- the number worth + watching is the one that climbs *during* a long turn, and a turn that + runs for forty minutes would otherwise report its context once, at the + end, when nothing can be done about it. + + Every failure is swallowed to None. An SDK without the method, a client + mid-restart, a control request that races a shutdown: none of them are + worth a line in the transcript, and all of them are indistinguishable to + a reader from "no session yet", which the meter already draws. + """ + client = self.client + if client is None or self._reading_context: + return self.context + reader = getattr(client, "get_context_usage", None) + if reader is None: + return self.context + self._reading_context = True + try: + usage = await reader() + except Exception as exc: # noqa: BLE001 - see the docstring + log.debug("context usage unavailable", exc_info=exc) + return self.context + finally: + self._reading_context = False + if isinstance(usage, dict): + self.context = usage + return self.context + + async def maybe_compact(self) -> dict[str, Any] | None: + """Compact if the context has passed the configured threshold. + + Called after a turn settles, which is the only safe moment: this drops + the client and builds another, and doing that underneath a running + `receive_response` is the failure `_stop_turn` exists to clean up after. + + The order matters and is the whole method. The note is written *first*, + while the outgoing session still remembers everything and its cache is + warm; only then is the client dropped. Reversed, there would be nothing + left to summarise. + + `pending_seed` rather than an immediate turn: sending the note now would + cost a whole round-trip to produce an answer nobody asked for. Held, it + rides along with whatever the user says next and costs nothing. + """ + from core import compaction # noqa: PLC0415 + + cfg = config_mod.load() + usage = await self.read_context() + if not compaction.should_compact(usage, cfg): + return None + before = compaction.context_tokens(usage) + return await self.compact(reason=f"context reached {before:,} tokens") + + async def compact(self, *, reason: str = "") -> dict[str, Any]: + """Summarise this conversation and start a fresh one holding the summary. + + Separated from `maybe_compact` so it can be asked for directly -- the + threshold is a default, not the only reason to want this, and a session + that has wandered is worth compacting at any size. + """ + import agent # noqa: PLC0415 + from core import compaction # noqa: PLC0415 + + if self.busy: + return {"ok": False, "message": "a turn is still running — interrupt it first"} + if self.client is None: + return {"ok": False, "message": "nothing to compact — no session is connected"} + + before = compaction.context_tokens(self.context) + self.busy = True + self._idle.clear() + try: + handoff = await compaction.write_handoff( + self.client, agent.drive_turn, session=self.session_id + ) + except agent.BudgetRefused as exc: + # No carve-out. A compaction is a model call and the allocation + # applies to it like any other -- exempting it would make "compact" + # the way to keep spending after the ceiling, and the ceiling is the + # feature. The conversation is left intact and oversized, which is + # recoverable; the alternative is not. + return {"ok": False, "message": exc.refusal["message"], "fix": exc.refusal["fix"]} + except Exception as exc: # noqa: BLE001 - a failed compaction keeps the session + log.exception("could not write the handoff note") + return { + "ok": False, + "message": f"could not summarise the session ({type(exc).__name__}) — nothing was discarded", + } + finally: + self.busy = False + self._idle.set() + + # The fresh conversation is a *new* SDK session, so the resume id has to + # go. Leaving it would have `start()` resume the very conversation this + # is discarding, quietly restoring the context that was just summarised + # and making the whole operation a cost with no effect. + await self.close() + self.sdk_session_id = None + self.pending_seed = compaction.seed_message(handoff["note"], tokens_before=before) + + entry = compaction.record( + tokens_before=before, + tokens_after=0, + note=handoff["note"], + cost=handoff.get("quota"), + ) + if reason: + entry["reason"] = reason + self.settled.append(entry) + self._persist() + return {"ok": True, "message": f"compacted at {before:,} tokens", "record": entry} + def say(self, message: str) -> None: """A line for the status bar, when there is one to put it in.""" log.info("%s", message) @@ -547,6 +704,15 @@ def restore(self) -> None: blocks = _drawable_blocks(record.get("blocks")) if blocks: kept["blocks"] = blocks + # Carried through the filter rather than dropped by it, so a + # compaction marker still reads as one after a reload instead of + # degrading into an anonymous message from nobody. Both are + # copied defensively: this file outlives the version that wrote + # it, and the renderer subscripts them. + if isinstance(record.get("kind"), str): + kept["kind"] = record["kind"] + if isinstance(record.get("note"), str): + kept["note"] = record["note"] self.settled.append(kept) diff --git a/ui/models.py b/ui/models.py index 4ea999b..c5177b8 100644 --- a/ui/models.py +++ b/ui/models.py @@ -272,6 +272,11 @@ def _session_window(*, hours: int = 5) -> dict[str, Any]: now = _dt.datetime.now(_dt.timezone.utc) cutoff = now - _dt.timedelta(hours=hours) + # Weighted, like the ceiling. This meter is meant to answer "how much of the + # rolling window have I used", and cache reads are most of what draws on it + # -- counting only input + output drew a bar that barely moved through a + # session that was in fact consuming the window steadily. + weight = quota_log.weights() chat = tool = 0.0 chat_tokens = tool_tokens = 0 tokens = 0 @@ -282,7 +287,7 @@ def _session_window(*, hours: int = 5) -> dict[str, Any]: continue oldest = at if oldest is None or at < oldest else oldest credits = float(entry.get("credits_usd", 0.0) or 0.0) - entry_tokens = int(entry.get("input_tokens", 0) or 0) + int(entry.get("output_tokens", 0) or 0) + entry_tokens = round(quota_log.billable(entry, weight)) tokens += entry_tokens if str(entry.get("stage") or "") == quota_log.STAGE_MAIN: chat += credits @@ -326,6 +331,96 @@ def _session_window(*, hours: int = 5) -> dict[str, Any]: } +# --------------------------------------------------------------------------- +# the context meter +# --------------------------------------------------------------------------- +#: Fractions of the way to compaction at which the chip changes tone. Below the +#: first it is ordinary; past the second, compaction is the next thing that will +#: happen and the strip should say so before it does rather than after. +CONTEXT_WARN = 0.75 +CONTEXT_NEAR = 0.92 + + +def context_model(usage: Any, *, compact_at: int = 0) -> dict[str, Any]: + """What the statusline's context chip shows, from one `get_context_usage`. + + Pure, and separate from the call that produces `usage`, because the + interesting part is not the request -- it is which limit the fraction is + measured against. There are two, and they mean different things: + + * `maxTokens` is where the *CLI* will compact, which a live session reports + as 967,000 of a 1,000,000 window. By the time that matters, every tool + round-trip has been re-reading most of a million cached tokens for a long + while, and the meter has been reading "ok" throughout. + * `compact_at` is where *Grad* will compact, from `[agent] + compact_at_tokens`. When it is set it is always the lower of the two, and + it is the one worth drawing, because it is the one that is going to fire. + + So the fraction is against whichever limit will actually be reached first, + and `limit_source` says which one that was -- a meter reading 40% means two + quite different things at a 300k threshold and at a 967k one. + + `usage` may be None (no client yet, or the call failed). That is a real and + ordinary state -- the meter is drawn as "—" rather than as zero, because a + context of zero and an unknown context look identical at a glance and only + one of them is worth acting on. + """ + if not isinstance(usage, dict): + return { + "known": False, "tokens": 0, "limit": 0, "fraction": 0.0, + "label": "ctx —", "tone": "", "limit_source": "unknown", + "detail": "no context reading yet — it arrives once a session is connected", + "categories": [], + } + + def _int(key: str) -> int: + try: + return max(0, int(usage.get(key) or 0)) + except (TypeError, ValueError): + return 0 + + tokens = _int("totalTokens") + ceiling = _int("maxTokens") or _int("rawMaxTokens") + source = "cli" + if compact_at and (not ceiling or compact_at < ceiling): + ceiling, source = compact_at, "grad" + fraction = min(1.0, tokens / ceiling) if ceiling else 0.0 + + tone = "" + if fraction >= CONTEXT_NEAR: + tone = "attention" + elif fraction >= CONTEXT_WARN: + tone = "warn" + + categories = [ + {"name": str(c.get("name") or "?"), "tokens": int(c.get("tokens") or 0)} + for c in (usage.get("categories") or []) + if isinstance(c, dict) and int(c.get("tokens") or 0) > 0 + # `Free space` is a category in the CLI's own breakdown and is the + # complement of everything else, so listing it in a tooltip about what + # is *using* the context is worse than noise -- it is always the largest + # entry and it is not a consumer. + and str(c.get("name") or "").strip().lower() != "free space" + ] + categories.sort(key=lambda c: -c["tokens"]) + + where = "Grad compacts" if source == "grad" else "the CLI compacts" + detail = f"{tokens:,} of {ceiling:,} tokens ({where} here)" if ceiling else f"{tokens:,} tokens" + if categories: + detail += " — " + ", ".join(f"{c['name']} {_tokens(c['tokens'])}" for c in categories[:4]) + return { + "known": True, + "tokens": tokens, + "limit": ceiling, + "fraction": fraction, + "label": f"ctx {_tokens(tokens)}" + (f" · {fraction * 100:.0f}%" if ceiling else ""), + "tone": tone, + "limit_source": source, + "detail": detail, + "categories": categories, + } + + def status_model() -> dict[str, Any]: """The workspace status bar: cwd, kernel, queue and gpu counts. @@ -1082,6 +1177,12 @@ def quota_model(*, days: int = 1) -> dict[str, Any]: "roles": roles, "stages": summary.get("by_stage") or {}, "total_tokens": summary.get("total_tokens", 0), + # The four kinds beside the one number, because they are the answer to + # the first question the one number provokes. `billable_tokens` is what + # a ceiling is charged; `totals` is what it is charged *for*. + "token_counts": summary.get("totals") or {}, + "billable_tokens": summary.get("billable_tokens", 0), + "token_weights": summary.get("weights") or {}, "total_credits_usd": summary.get("total_credits_usd", 0.0), "gpu": { "total_usd": rolling.get("total_usd", 0.0), diff --git a/ui/tokens.py b/ui/tokens.py index 7dacc03..30bf9e9 100644 --- a/ui/tokens.py +++ b/ui/tokens.py @@ -618,6 +618,17 @@ def _chat() -> str: .grad-statusline .activity { opacity: 0.62; min-width: 0; overflow: hidden; white-space: nowrap; text-overflow: ellipsis; } .grad-statusline .clock { opacity: 0.62; flex: 0 0 auto; } +/* How much of the context window is in use, and how close compaction is. + Dimmed like the clock until it matters, because for most of a session the + honest answer is "not yet" and a meter that shouts throughout is one nobody + reads by the time it should be shouting. `warn` is an outline and + `attention` is a fill: the first says the end is in sight, the second says + the next turn may be the one that triggers a compaction. */ +.grad-statusline .context { opacity: 0.62; flex: 0 0 auto; padding: 1px 5px; } +.grad-statusline .context.warn { + opacity: 1; border: 1.5px solid var(--grad-ink); } +.grad-statusline .context.attention { + opacity: 1; background: var(--grad-attention); color: var(--grad-ink); } /* The one part of the bar that is a control rather than a report, so it is the one part drawn as one. */ .grad-statusline .reasoning { flex: 0 0 auto; border: 1.5px solid var(--grad-ink); @@ -625,6 +636,21 @@ def _chat() -> str: .grad-chat.reasoning-on .grad-statusline .reasoning { background: var(--grad-ink); color: var(--grad-paper); } +/* The compaction marker: a rule across the transcript, not a message. Nothing + was said at this point -- what happened is that everything above it stopped + being something the agent remembers first-hand. Drawn as a rule so it reads + as a boundary rather than as a turn, and dashed for the same reason the rest + of the design uses a dashed border: what is above the line is no longer + solid. */ +.grad-compaction { margin: 14px 14px; border-top: 1.5px dashed var(--grad-rule-mid); + padding-top: 10px; font-size: 12px; } +.grad-compaction > .head { align-items: flex-start; } +.grad-compaction .mark { font-family: var(--grad-font-mono); opacity: 0.55; + flex: 0 0 auto; line-height: 1.5; } +.grad-compaction .head .q-markdown, .grad-compaction .head p { margin: 0; opacity: 0.75; } +.grad-compaction .note { margin-top: 6px; font-size: 11px; + background: var(--grad-paper-sunk); border: var(--grad-secondary); } + /* Reasoning is drawn either way and painted only when it is switched on: a toggle that rebuilt the transcript would take its scroll position with it, which is the same reason the poll never touches this window. */ diff --git a/ui/windows/chat.py b/ui/windows/chat.py index f3c2e07..269d378 100644 --- a/ui/windows/chat.py +++ b/ui/windows/chat.py @@ -49,6 +49,27 @@ #: "pending": an outcome that has not happened yet is not a green one. STATUS_TONE = {"running": "dashed", "ok": "ok", "error": "broken"} +#: Seconds between context readings. Four is slow enough that the call is +#: invisible next to a turn and fast enough that the chip is never far behind +#: what is actually in the window. +CONTEXT_POLL_S = 4.0 + + +def compact_threshold() -> int: + """Where Grad will compact, for the meter to measure against. + + Here rather than read from `core.compaction` at the call site so the import + stays lazy: `ui/windows/chat.py` is imported by the registry on a machine + that may have no config yet, and `config.load()` is cached, so the cost of + asking on every poll is a dict lookup. + """ + from core import compaction # noqa: PLC0415 + + try: + return compaction.threshold() + except Exception: # noqa: BLE001 - a meter must not be able to take the window down + return 0 + def subtitle(workspace: Any) -> str: session = getattr(workspace, "session", None) @@ -216,6 +237,24 @@ def flush() -> None: statusline.sync(blocks) ui.timer(1 / FLUSH_HZ, flush) + + async def poll_context() -> None: + """Ask the CLI how big the context is, and redraw the chip. + + On its own timer, several hundred times slower than the flush. The call + costs no tokens -- it is a control request answered from state the CLI + already holds -- but it is still a round-trip over the transport a turn + is streaming on, and there is nothing to learn from asking sixty times a + second about a number that moves once a turn. + + The interval is chosen for the case that matters: a turn that runs for + forty minutes, where the point of the meter is watching the context + climb while there is still time to do something about it. + """ + await session.read_context() + statusline.sync_context() + + ui.timer(CONTEXT_POLL_S, poll_context) # Once, at build: keep the transcript pinned to the bottom while a turn # streams. Doing it from here instead would be a `run_javascript` per flush. kit.run_js("window.gradStickBottom && window.gradStickBottom('grad-transcript')") @@ -267,10 +306,17 @@ def __init__(self, workspace: Any, root: Any) -> None: self.state = kit.text("IDLE", "state", tag="span") self.activity = kit.text("waiting for you", "activity", tag="span") kit.spacer() + # Before the clock rather than after it: the clock and the reasoning + # switch are about the turn in flight, and this is about the session + # as a whole -- it is the one thing on this strip that is still true + # when nothing is running. + self.context = kit.text("", "context", tag="span") self.clock = kit.text("", "clock", tag="span") self.reasoning = kit.text("", "reasoning", tag="span") self.bar = bar + self._context_mark: tuple[Any, ...] | None = None self._paint_reasoning() + self.sync_context() def toggle(self) -> None: showing = self.workspace.toggle_reasoning() @@ -296,6 +342,32 @@ def _paint_reasoning(self) -> None: showing = self.workspace.show_reasoning kit.set_text(self.reasoning, f"reasoning {'■ on' if showing else '□ off'}") + def sync_context(self) -> None: + """Redraw the context chip from the session's last reading. + + Called on its own timer rather than on the 15 Hz flush: the underlying + number changes once per control request, and repainting it a hundred + times between two readings is a hundred DOM writes that say the same + thing. Like `sync`, it touches nothing unless the line changed. + """ + session = self.workspace.session + model = models.context_model( + getattr(session, "context", None), + compact_at=compact_threshold(), + ) + mark = (model["label"], model["tone"], model["detail"]) + if mark == self._context_mark: + return + self._context_mark = mark + kit.set_text(self.context, model["label"]) + # Both removed before either is added: a chip that crossed from warn to + # attention would otherwise carry the old class as well as the new one, + # and the pair have different accents by design. + self.context.classes(remove="warn attention") + if model["tone"]: + self.context.classes(add=model["tone"]) + self.context.props(f'title="{kit.attr(model["detail"])}"') + def sync(self, blocks: list[dict[str, Any]]) -> None: """Called at the flush rate, so nothing here touches the DOM unless the line it draws actually changed.""" @@ -466,6 +538,31 @@ def _has_gate(record: dict[str, Any]) -> bool: return False +def _compaction(ui: Any, record: dict[str, Any]) -> None: + """The line across the transcript where the agent's memory was replaced. + + Drawn as a rule rather than as a message because that is what it is: nothing + was said here, and everything above it is now something the agent knows only + second-hand. Without a mark the transcript reads as one continuous + conversation, and the first time the model fails to remember a detail that + is plainly visible three turns up, the reasonable conclusion is that the + agent is broken. + + The note is behind a disclosure. It is long by design -- `HANDOFF_PROMPT` + asks for paths, commands and ledger state, not for brevity -- and it is + exactly what someone will want to read when the answer after a compaction is + worse than the answers before it. + """ + with kit.el("div", "grad-compaction"): + with kit.row("head", gap=8): + kit.text("⊟", "mark", tag="span") + ui.markdown(record.get("text") or "compacted") + note = record.get("note") + if isinstance(note, str) and note.strip(): + with ui.expansion("the handover note the previous session left").classes("note"): + ui.markdown(note) + + def _composer(ui: Any, workspace: Any, transcript: Any, tail: _Tail, statusline: Any) -> None: session = workspace.session @@ -506,6 +603,23 @@ async def send(prompt: str | None = None) -> None: # system does not have. Whether to plan first is something you say in # words, and the gates are what actually stop a spend. await session.ask(prompt, settle) + # After the turn has settled, never during it: compacting drops the + # client, and dropping it underneath a live `receive_response` is the + # failure `_stop_turn` exists to clean up after. Here rather than inside + # `ask` because the record has to be *drawn*, and the transcript is the + # window's to write to. + outcome = await session.maybe_compact() + if outcome is None: + return + if outcome.get("record"): + with transcript: + _message(outcome["record"], workspace) + else: + # A compaction that could not happen is worth saying out loud. The + # session carries on oversized, which is survivable, but silence + # here would leave a meter pinned at the threshold with nothing + # explaining why nothing is being done about it. + workspace.say(outcome.get("message") or "could not compact this session") with kit.el("div", "grad-composer"): # Right-aligned by the row rather than by a leading spacer: with the mode @@ -557,6 +671,9 @@ def _message(record: dict[str, Any], workspace: Any) -> None: from nicegui import ui text = record.get("text") or "" + if record.get("role") == "system": + _compaction(ui, record) + return if record.get("role") == "user": with kit.el("div", "grad-msg user"): kit.text("you", "role") diff --git a/ui/windows/quota.py b/ui/windows/quota.py index daf5822..bd5c2cb 100644 --- a/ui/windows/quota.py +++ b/ui/windows/quota.py @@ -61,6 +61,35 @@ def render(workspace: Any) -> None: kit.hr() + # The four kinds, spelled out. The ceiling is charged one weighted + # number, and the first question that number provokes is "why is it + # twelve times what I expected" -- which is unanswerable without this + # row and obvious with it. Cache reads are nearly always the largest + # entry, and that is the point rather than a defect. + kit.label(f"tokens · {model.get('days', 1)}d") + counts = model.get("token_counts") or {} + weights = model.get("token_weights") or {} + if not counts: + kit.text("nothing recorded in this window", "grad-caption") + else: + kit.kv([ + (label, f"{counts.get(field, 0):,} × {weights.get(key, 1.0):g}") + for field, key, label in ( + ("input_tokens", "weight_input", "input"), + ("output_tokens", "weight_output", "output"), + ("cache_read_tokens", "weight_cache_read", "cache read"), + ("cache_write_tokens", "weight_cache_write", "cache write"), + ) + ]) + with kit.row("", gap=14).style("margin-top: 6px"): + kit.text( + f"{model.get('billable_tokens', 0):,} charged to the ceiling", + "grad-mono", + tag="span", + ) + + kit.hr() + kit.label(f"spend today · {model.get('days', 1)}d") roles = model.get("roles") or [] if not roles: From 94a71b48f200817faa91a29789c0a48041c73955 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=BB=D0=B0=D0=B4=D0=B8=D0=BC=D0=B8=D1=80=20=D0=A8?= =?UTF-8?q?=D0=BC=D0=B0=D0=BD?= Date: Sun, 16 Aug 2026 02:40:42 +0300 Subject: [PATCH 2/2] Address review: a dead dial, an id derived from the wrong number, and three ways to round `compact_keep_turns` was a config key that did nothing and a docstring that described behaviour the code does not have -- no turns survive a compaction verbatim, only the note does. Removed rather than wired in: the conversation belongs to the SDK, so the desktop app has a transcript to keep turns from and the CLI has none, and implementing it now would mean the two surfaces compacting differently. The module docstring says so, and says what it would take. `harvest` numbered new rows from `len(existing) + 1`, which is right only while the ids are exactly q001..qN. Delete one graded row from a file of five and the next id is q005, which is already taken -- and a duplicate id in an eval set is two questions that cannot be told apart in a result table. Numbered from the highest qNNN now, preserving gaps, ignoring ids that were never qNNN at all. The append goes through `jsonl.append` rather than an `open(..., "a")`, which is the rule this project already has, and the free-id check runs in its precondition under the lock. `append` serialises the record *before* taking the lock, so a precondition cannot renumber a row in place -- it can only refuse, which makes this an ordinary compare-and-set retry. Tested by racing a second writer into the window between deriving an id and the write landing. Three rounding fixes, all the same shape: round once, at the end, not per group. `summarise` summed per-stage figures that had each already been rounded, and `_session_window` rounded every entry before adding it -- an error that grows with how busy the window was, in a meter about how busy the window was. And `counts()` now clamps at zero. `weights()` already refused a negative weight because it would make spending lower the measured total; the counts needed the same guard and did not have it, and `record`'s CLI-level check is not the only door -- `from_sdk_usage` records what the SDK reports and the ledger is a file that can be edited. `_fold` reads through `counts` too, so the raw figures and the weighted total cannot disagree about the same malformed row. `context_model` returned a confident zero for a reading it could not parse, which is the exact failure its own docstring is about: "ctx 0 - 0%" invites the conclusion that there is plenty of room. Missing or unparseable now reads unknown, and a genuine zero still reads as zero. Category parsing no longer raises on a non-numeric count -- it is drawn from a timer, so that was not one bad tooltip but an exception several times a second for the life of the session. Co-Authored-By: Claude Opus 5 --- core/compaction.py | 20 +++---- core/config.py | 5 -- core/quota_log.py | 30 ++++++++-- tests/test_context_and_compaction.py | 80 +++++++++++++++++++++++++ tests/test_traces.py | 88 ++++++++++++++++++++++++++++ tools/traces.py | 83 +++++++++++++++++++++++--- ui/models.py | 73 +++++++++++++++++------ 7 files changed, 333 insertions(+), 46 deletions(-) diff --git a/core/compaction.py b/core/compaction.py index f8cb886..22673a3 100644 --- a/core/compaction.py +++ b/core/compaction.py @@ -40,6 +40,16 @@ it reads. The note is written in the first person and asks for specifics, because the failure mode of a summary is that it reads well and contains nothing actionable. + +**The note is all that crosses.** No turns survive verbatim underneath it, which +is worth stating because the obvious improvement -- ml-intern's context manager +keeps the last five messages below its summary, on the sound theory that the most +recent exchange is the one a summary compresses worst -- has no seam to attach to +here. The conversation belongs to the SDK, not to this module: the desktop app +has a transcript in `settled` and the CLI has none at all, so keeping turns would +mean the two surfaces compacting differently, which is the thing `drive_turn`'s +docstring exists to prevent. It is worth doing when there is one transcript both +surfaces share, and not before. """ from __future__ import annotations @@ -102,16 +112,6 @@ def threshold(cfg: Any = None) -> int: return int(value) if value > 0 else 0 -def keep_turns(cfg: Any = None) -> int: - """How many recent turns survive verbatim under the summary. - - Clamped to at least 0 and at most 10. The upper bound is not fussiness: the - turns kept are kept in full, and this agent's turns carry tool output, so a - generous number here is a compaction that does not compact. - """ - return max(0, min(10, int(_number(cfg, "compact_keep_turns", 2)))) - - def _number(cfg: Any, key: str, default: float) -> float: if cfg is None: try: diff --git a/core/config.py b/core/config.py index 0a07b27..8df7454 100644 --- a/core/config.py +++ b/core/config.py @@ -231,11 +231,6 @@ # that trade becomes visible, which is why the accounting split landed # before this did. "compact_at_tokens": 300_000, - # How many of the most recent turns survive a compaction verbatim, below - # the summary. The summary is a model's account of the conversation and - # the last exchange is the one it is worst at compressing, because it has - # not yet had a consequence. - "compact_keep_turns": 2, }, "hosts": {}, } diff --git a/core/quota_log.py b/core/quota_log.py index d28fb58..14ccc52 100644 --- a/core/quota_log.py +++ b/core/quota_log.py @@ -174,12 +174,21 @@ def weights(cfg: Any = None) -> dict[str, float]: def counts(row: Any) -> dict[str, int]: - """The four raw token counts of one record, defaulting to zero.""" + """The four raw token counts of one record, defaulting to zero. + + **Clamped at zero**, for the same reason `weights` refuses a negative weight: + a negative count would *reduce* the measured total, which is the one error a + ceiling cannot survive -- one malformed row and a project has spending power + it was never allocated. `tools/quota.py record` already refuses a negative + at the CLI, but that is not the only door: `from_sdk_usage` records whatever + the SDK reports, and the ledger is a file on disk that can be edited. The + guard belongs at the read, where every path passes. + """ get = row.get if isinstance(row, dict) else (lambda k, d=0: getattr(row, k, d)) out: dict[str, int] = {} for field, _ in KINDS: try: - out[field] = int(get(field, 0) or 0) + out[field] = max(0, int(get(field, 0) or 0)) except (TypeError, ValueError): out[field] = 0 return out @@ -237,10 +246,18 @@ def _fold(key: str, fallback: str) -> dict[str, dict[str, Any]]: "billable_tokens": 0.0, "credits_usd": 0.0}, ) node["calls"] += 1 - for k, _ in KINDS: - node[k] += int(r.get(k, 0) or 0) + # Through `counts`, not straight off the record, so the raw figures + # and the weighted total are clamped by one rule. Read separately, + # a negative row would be excluded from `billable` and included in + # the counts printed beside it -- two numbers describing the same + # row and disagreeing, which is worse than either being wrong. + for k, value in counts(r).items(): + node[k] += value node["billable_tokens"] += billable(r, weight) node["credits_usd"] += float(r.get("credits_usd", 0.0) or 0.0) + # Rounded here and *not* re-rounded into the grand total below: rounding + # each group and then summing accumulates up to half a token of error per + # group, which is small but is also entirely avoidable. return { k: {**v, "billable_tokens": round(v["billable_tokens"]), @@ -269,7 +286,10 @@ def _fold(key: str, fallback: str) -> dict[str, dict[str, Any]]: "totals": { field: sum(n[field] for n in by_stage.values()) for field, _ in KINDS }, - "billable_tokens": round(sum(n["billable_tokens"] for n in by_stage.values())), + # From the rows, not from the rounded per-stage figures above. Summing + # values that have each already been rounded carries every group's + # rounding error into the one number a ceiling is compared against. + "billable_tokens": round(sum(billable(r, weight) for r in rows)), "weights": weight, "total_credits_usd": round(sum(n["credits_usd"] for n in by_stage.values()), 4), # Anthropic exposes no remaining-quota API and the Max 5x window is diff --git a/tests/test_context_and_compaction.py b/tests/test_context_and_compaction.py index d730f6d..85cd353 100644 --- a/tests/test_context_and_compaction.py +++ b/tests/test_context_and_compaction.py @@ -101,6 +101,42 @@ def test_a_project_ceiling_is_charged_the_weighted_total(workspace): assert state["quota_weights"]["weight_cache_read"] == 0.1 +def test_a_negative_count_cannot_buy_back_allocation(workspace): + """`weights` already refuses a negative weight for this reason; the counts + needed the same guard. `tools.quota record` refuses one at the CLI, but that + is not the only door -- `from_sdk_usage` records what the SDK reports, and + the ledger is a file on disk that can be edited.""" + budget.create("proj-neg", title="clamping", budget={"quota_tokens": 1_000}) + budget.set_current("proj-neg") + quota_log.record(quota_log.STAGE_MAIN, project="proj-neg", output_tokens=900) + quota_log.record(quota_log.STAGE_MAIN, project="proj-neg", output_tokens=-10_000) + + assert quota_log.counts({"output_tokens": -10_000})["output_tokens"] == 0 + assert budget.status("proj-neg")["resources"]["quota_tokens"]["spent"] == 900 + + +def test_the_raw_counts_and_the_weighted_total_clamp_by_one_rule(workspace): + """Read separately, a malformed row would be excluded from one and included + in the other -- two numbers describing the same row and disagreeing, which is + worse than either being wrong.""" + quota_log.record(quota_log.STAGE_MAIN, input_tokens=5, cache_read_tokens=-1_000) + summary = quota_log.summarise() + assert summary["totals"]["cache_read_tokens"] == 0 + assert summary["billable_tokens"] == 5 + + +def test_the_weighted_total_is_rounded_once_rather_than_per_group(workspace): + """Rounding each stage and then summing carries every group's error into the + one number a ceiling is compared against.""" + # Three stages, each landing on exactly half a token once weighted. + for stage in ("main", "funnel.expand", "funnel.triage"): + quota_log.record(stage, cache_read_tokens=5) + summary = quota_log.summarise() + # 3 x 0.5 = 1.5 -> 2. Summing three separately-rounded 0.5s gives 0. + assert summary["billable_tokens"] == 2 + assert sum(n["billable_tokens"] for n in summary["by_stage"].values()) == 0 + + def test_the_summary_reports_the_four_kinds_and_the_weighted_total(workspace): quota_log.record(quota_log.STAGE_MAIN, input_tokens=1, output_tokens=2, cache_read_tokens=1_000, cache_write_tokens=100) @@ -126,6 +162,50 @@ def test_an_unknown_context_reads_as_unknown_rather_than_as_empty(): assert model["tone"] == "" +@pytest.mark.parametrize( + "usage", + [ + {"maxTokens": 1_000_000}, # a reading with no total in it + {"totalTokens": None, "maxTokens": 1_000_000}, + {"totalTokens": "lots", "maxTokens": 1_000_000}, + ], +) +def test_an_unreadable_total_is_unknown_rather_than_a_confident_zero(usage): + """The failure this function's docstring is about, arriving through the + function itself. "ctx 0 · 0%" for a session that could not be measured is + worse than saying nothing, because it invites exactly the conclusion that + there is plenty of room.""" + model = models.context_model(usage, compact_at=300_000) + assert model["known"] is False + assert model["label"] == "ctx —" + + +def test_a_category_that_cannot_be_read_is_skipped_rather_than_raising(): + """This is drawn from a timer, so one odd category would not produce one bad + tooltip -- it would raise several times a second for the life of the + session.""" + model = models.context_model( + { + "totalTokens": 5_000, + "maxTokens": 1_000_000, + "categories": [ + {"name": "Tools", "tokens": "unknown"}, + {"name": "Skills", "tokens": 1_469}, + "not even a dict", + ], + } + ) + assert [c["name"] for c in model["categories"]] == ["Skills"] + + +def test_a_genuinely_empty_context_still_reads_as_known(): + """Zero is a real reading when the payload says so; only a missing or + unparseable one is unknown.""" + model = models.context_model({"totalTokens": 0, "maxTokens": 1_000_000}) + assert model["known"] is True + assert model["fraction"] == 0.0 + + def test_the_meter_measures_against_grads_threshold_when_there_is_one(): """The whole point of the chip. Against the CLI's 967k the same session reads as nearly empty; against the threshold that will actually fire it diff --git a/tests/test_traces.py b/tests/test_traces.py index 72be2a6..cd924fc 100644 --- a/tests/test_traces.py +++ b/tests/test_traces.py @@ -177,6 +177,94 @@ def test_harvesting_twice_does_not_add_the_question_twice(workspace, monkeypatch assert len([l for l in lines if l.strip()]) == 1 +def test_a_gap_in_the_ids_does_not_produce_a_duplicate(workspace, monkeypatch): + """The count-based numbering this started with was wrong the moment the ids + were not exactly q001..qN. Delete one graded row from a file of five and the + count says the next id is q005, which is already taken -- and a duplicate id + in an eval set is two different questions that cannot be told apart in a + result table.""" + evals = workspace / "evals" + evals.mkdir(parents=True, exist_ok=True) + (evals / "retrieval.jsonl").write_text( + "\n".join( + json.dumps({"id": i, "question": f"already asked {i}"}) + for i in ("q001", "q003", "q004", "q005") + ) + + "\n", + encoding="utf-8", + ) + _store_session(monkeypatch, "s-1", [ + asked(), + turn(bash('python -m tools.paper_search search "a brand new question" --json')), + ]) + from tools import traces as cli + + out = cli.cmd_harvest(_args(write=True)) + assert [r["id"] for r in out["candidates"]] == ["q006"] + ids = [ + json.loads(line)["id"] + for line in (evals / "retrieval.jsonl").read_text(encoding="utf-8").splitlines() + if line.strip() + ] + assert len(ids) == len(set(ids)) + + +def test_ids_that_are_not_qnnn_at_all_do_not_derail_the_numbering(workspace, monkeypatch): + evals = workspace / "evals" + evals.mkdir(parents=True, exist_ok=True) + (evals / "retrieval.jsonl").write_text( + json.dumps({"id": "scaling-laws-1", "question": "hand-named"}) + "\n" + + json.dumps({"id": "q007", "question": "numbered"}) + "\n", + encoding="utf-8", + ) + _store_session(monkeypatch, "s-1", [ + asked(), + turn(bash('python -m tools.paper_search search "something else" --json')), + ]) + from tools import traces as cli + + assert [r["id"] for r in cli.cmd_harvest(_args(write=True))["candidates"]] == ["q008"] + + +def test_an_id_taken_between_deriving_and_writing_is_derived_again(workspace, monkeypatch): + """`jsonl.append` serialises the record before it takes the lock, so a + precondition cannot renumber the row -- it can only refuse, which makes this + a retry. The precondition is where the race is actually decided.""" + from core import jsonl + + evals = workspace / "evals" + evals.mkdir(parents=True, exist_ok=True) + path = evals / "retrieval.jsonl" + path.write_text(json.dumps({"id": "q001", "question": "there first"}) + "\n", encoding="utf-8") + + real_append = jsonl.append + raced = {"done": False} + + def append_but_race_first(target, record, *, precondition=None): + # Another writer lands the id this attempt is about to ask for, in the + # window between deriving it and the lock closing over the write. + if not raced["done"]: + raced["done"] = True + with open(target, "a", encoding="utf-8", newline="\n") as fh: + fh.write(json.dumps({"id": "q002", "question": "raced in"}) + "\n") + return real_append(target, record, precondition=precondition) + + monkeypatch.setattr(jsonl, "append", append_but_race_first) + _store_session(monkeypatch, "s-1", [ + asked(), + turn(bash('python -m tools.paper_search search "mine" --json')), + ]) + from tools import traces as cli + + out = cli.cmd_harvest(_args(write=True)) + assert out["candidates"][0]["id"] == "q003" + ids = [ + json.loads(line)["id"] + for line in path.read_text(encoding="utf-8").splitlines() if line.strip() + ] + assert ids == ["q001", "q002", "q003"] + + def test_harvested_ids_continue_after_the_rows_already_there(workspace, monkeypatch): evals = workspace / "evals" evals.mkdir(parents=True, exist_ok=True) diff --git a/tools/traces.py b/tools/traces.py index d4a37e7..8b31884 100644 --- a/tools/traces.py +++ b/tools/traces.py @@ -30,7 +30,7 @@ import re from typing import Any -from core import paths, quota_log, traces +from core import jsonl, paths, quota_log, traces from core.cli import Cli, main from core.errors import NotFound, UsageError @@ -235,16 +235,18 @@ def cmd_harvest(args: argparse.Namespace) -> dict[str, Any]: path = paths.root() / "evals" / "retrieval.jsonl" existing, known = _existing_eval_rows(path) fresh = [r for r in rows if r["question"] not in known] - for offset, row in enumerate(fresh, start=len(existing) + 1): - row["id"] = f"q{offset:03d}" written = False if args.write and fresh: - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("a", encoding="utf-8") as handle: - for row in fresh: - handle.write(json.dumps(row, ensure_ascii=False) + "\n") + for row in fresh: + _append_numbered(path, row) written = True + else: + # Nothing is being written, so the ids are a proposal. Numbered from the + # same rule so a dry run says what a real one would do. + index = _next_index(existing) + for offset, row in enumerate(fresh, start=index): + row["id"] = f"q{offset:03d}" return { "path": str(path), @@ -261,6 +263,73 @@ def cmd_harvest(args: argparse.Namespace) -> dict[str, Any]: } +#: An eval row's id, as the schema in `evals/README.md` writes it. +ID_RE = re.compile(r"q(\d+)\Z") + +#: How many times `_append_numbered` will re-derive an id before giving up. Two +#: would do for the realistic case -- this is a human-run command on one desktop +#: -- and a handful costs nothing and covers a genuinely contended file. +ID_ATTEMPTS = 8 + + +class _Renumber(Exception): + """The id this row was given stopped being free before it landed.""" + + +def _next_index(existing: list[dict[str, Any]]) -> int: + """One past the highest `qNNN` in the file. + + From the highest id rather than from the row *count*, which is what this did + at first and which is wrong the moment the ids are not exactly `q001..qN`. + Delete one graded row from a file of five and the count says the next id is + `q005`, which is already taken -- and a duplicate id in an eval set is not a + cosmetic problem, it is two different questions that cannot be told apart in + a result table. Gaps are preserved rather than backfilled, because an id that + has been used once has probably been referred to somewhere. + """ + highest = 0 + for row in existing: + match = ID_RE.fullmatch(str(row.get("id") or "").strip()) + if match: + highest = max(highest, int(match.group(1))) + return highest + 1 + + +def _append_numbered(path: Any, row: dict[str, Any]) -> dict[str, Any]: + """Append one row under an id that is still free when it lands. + + Compare-and-set rather than assign-then-write. `core/jsonl.py:append` + serialises the record *before* it takes the lock, so a precondition cannot + renumber the row in place -- what it can do is refuse, which turns this into + an ordinary retry: derive an id, ask for it, and if the file gained that id + in the meantime, derive again against what is there now. + + Through `jsonl.append` rather than an `open(..., "a")` for the reason this + project states once and applies everywhere: no CLI opens an append-only file + for writing directly, because the locked path is the only one that cannot + interleave two writers mid-line. + """ + for _ in range(ID_ATTEMPTS): + row["id"] = f"q{_next_index(_existing_eval_rows(path)[0]):03d}" + wanted = row["id"] + + def _still_free(wanted: str = wanted) -> None: + # Bound as a default argument, so the check is against the id this + # attempt asked for rather than whatever the loop reaches next. + taken = {str(r.get("id") or "").strip() for r in _existing_eval_rows(path)[0]} + if wanted in taken: + raise _Renumber + + try: + return jsonl.append(path, row, precondition=_still_free) + except _Renumber: + continue + raise UsageError( + f"could not find a free id in {path} after {ID_ATTEMPTS} attempts", + fix="another harvest is writing to the eval file; re-run when it finishes", + ) + + def _existing_eval_rows(path: Any) -> tuple[list[dict[str, Any]], set[str]]: """What is already in the eval file, and the questions it already covers. diff --git a/ui/models.py b/ui/models.py index c5177b8..e3ac2d5 100644 --- a/ui/models.py +++ b/ui/models.py @@ -287,7 +287,11 @@ def _session_window(*, hours: int = 5) -> dict[str, Any]: continue oldest = at if oldest is None or at < oldest else oldest credits = float(entry.get("credits_usd", 0.0) or 0.0) - entry_tokens = round(quota_log.billable(entry, weight)) + # Kept fractional and rounded once, at the bottom. Rounding each entry + # first costs up to half a token per row, and this window folds every + # record in five hours -- an error that grows with how busy the window + # was is exactly the wrong shape for a meter about how busy it was. + entry_tokens = quota_log.billable(entry, weight) tokens += entry_tokens if str(entry.get("stage") or "") == quota_log.STAGE_MAIN: chat += credits @@ -321,12 +325,14 @@ def _session_window(*, hours: int = 5) -> dict[str, Any]: "credits_usd": total, "chat_usd": chat, "tool_usd": tool, - "chat_tokens": chat_tokens, - "tool_tokens": tool_tokens, + # Rounded once, on the way out. The fractions above are computed from the + # unrounded figures, so the split does not shift with the rounding. + "chat_tokens": round(chat_tokens), + "tool_tokens": round(tool_tokens), "chat_fraction": chat_fraction, "tool_fraction": tool_fraction, "split_basis": "credits" if total else ("tokens" if token_total else "empty"), - "tokens": tokens, + "tokens": round(tokens), "resets_in": resets_in, } @@ -341,6 +347,25 @@ def _session_window(*, hours: int = 5) -> dict[str, Any]: CONTEXT_NEAR = 0.92 +def _reading(source: Any, key: str) -> int | None: + """A non-negative integer out of a `get_context_usage` payload, or None. + + None and 0 are kept distinct all the way through this file, which is the + whole discipline of the context meter: "I could not read it" and "there is + nothing in it" are opposite facts and only one of them means there is room. + Every caller here decides for itself which way to fail. + """ + if not isinstance(source, dict): + return None + value = source.get(key) + if isinstance(value, bool) or value is None: + return None + try: + return max(0, int(value)) + except (TypeError, ValueError): + return None + + def context_model(usage: Any, *, compact_at: int = 0) -> dict[str, Any]: """What the statusline's context chip shows, from one `get_context_usage`. @@ -365,7 +390,14 @@ def context_model(usage: Any, *, compact_at: int = 0) -> dict[str, Any]: context of zero and an unknown context look identical at a glance and only one of them is worth acting on. """ - if not isinstance(usage, dict): + tokens = _reading(usage, "totalTokens") if isinstance(usage, dict) else None + if tokens is None: + # Not only "no dict yet". A reading whose `totalTokens` is missing or is + # not a number is *also* unknown, and it used to land here as a confident + # zero -- which is the one reading this function's docstring says must + # never happen, drawn by this function. A meter reporting "ctx 0 · 0%" + # for a session it cannot measure is worse than one reporting nothing, + # because it invites exactly the conclusion that there is plenty of room. return { "known": False, "tokens": 0, "limit": 0, "fraction": 0.0, "label": "ctx —", "tone": "", "limit_source": "unknown", @@ -373,14 +405,7 @@ def context_model(usage: Any, *, compact_at: int = 0) -> dict[str, Any]: "categories": [], } - def _int(key: str) -> int: - try: - return max(0, int(usage.get(key) or 0)) - except (TypeError, ValueError): - return 0 - - tokens = _int("totalTokens") - ceiling = _int("maxTokens") or _int("rawMaxTokens") + ceiling = _reading(usage, "maxTokens") or _reading(usage, "rawMaxTokens") or 0 source = "cli" if compact_at and (not ceiling or compact_at < ceiling): ceiling, source = compact_at, "grad" @@ -392,16 +417,26 @@ def _int(key: str) -> int: elif fraction >= CONTEXT_WARN: tone = "warn" - categories = [ - {"name": str(c.get("name") or "?"), "tokens": int(c.get("tokens") or 0)} - for c in (usage.get("categories") or []) - if isinstance(c, dict) and int(c.get("tokens") or 0) > 0 + # Built with a loop rather than a comprehension because a bare + # `int(c.get("tokens"))` raises on a non-numeric value, and this is drawn + # from a timer -- one odd category in one reading would not produce one bad + # tooltip, it would raise several times a second for as long as the session + # lasted. A category that cannot be read is skipped. + categories: list[dict[str, Any]] = [] + for entry in usage.get("categories") or []: + if not isinstance(entry, dict): + continue + size = _reading(entry, "tokens") + if not size: + continue + name = str(entry.get("name") or "?").strip() # `Free space` is a category in the CLI's own breakdown and is the # complement of everything else, so listing it in a tooltip about what # is *using* the context is worse than noise -- it is always the largest # entry and it is not a consumer. - and str(c.get("name") or "").strip().lower() != "free space" - ] + if name.lower() == "free space": + continue + categories.append({"name": name or "?", "tokens": size}) categories.sort(key=lambda c: -c["tokens"]) where = "Grad compacts" if source == "grad" else "the CLI compacts"