From 61f7979dbac560d2669c13ae0411021f991374c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=BB=D0=B0=D0=B4=D0=B8=D0=BC=D0=B8=D1=80=20=D0=A8?= =?UTF-8?q?=D0=BC=D0=B0=D0=BD?= Date: Fri, 14 Aug 2026 21:48:19 +0300 Subject: [PATCH 1/3] Implement HANDOFF-2: budgets, org namespaces, and the two loops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All eight items from HANDOFF-2, in the §24 build order. §16 models by role. A [models] section with six roles; Opus 5 replaces the claude-opus-4-5 default. [retrieval] rerank_model/embed_model stay put on purpose -- a different provider on a different billing rail. Legacy [agent] model and [retrieval] triage_model/expand_model still resolve for one release. Every model call now records its role, so `quota summary --by-role` answers "what did Opus cost me this week" without inference. §15 the project dimension. core/budget.py plus tools/budget.py: one `project` id on every run record, every quota entry, and every credit spend, with ceilings on GPU dollars, credits, and tokens. New exit code 12, distinct from 6, so "this research ran out of its allocation" is never confused with "the machine is out of money". `raise` appends an event rather than mutating -- a ceiling that can be edited invisibly is not a ceiling. Token enforcement is honest about its granularity: agent.py refuses the next turn and hooks.py denies cost-bearing Bash, so budgets hold to one turn's overrun. The Stop hook warns and deliberately does not enforce -- its `block` semantics force continuation, which is backwards here. §17 HF organization namespaces. The namespace is persisted onto the run handle and threaded through inspect/logs/status/collect, not merely passed to run_job -- otherwise the job cannot be found again, goes stale, and blocks every future submission through the §6 gate. Membership is checked against whoami before record_submission, so a bad namespace leaves no phantom estimate on the ceiling. Not hashed, consistent with flavor; a smoke/submit mismatch warns rather than refuses. §18 tools/docs.py. Two oracles, introspection first: importlib.metadata and inspect.signature settle what is installed, Context7 says what is current. A CLI rather than a subagent, so Task stays denied. Context7's REST paths were never verified -- they are configuration, and a 404 says so. Fifth credential, scrubbed like the rest. §19 embedded JupyterLab. tools/lab.py, the framing headers Lab needs, and a pinned `lab` extra. The per-notebook Verify button landed first because it is worth more than the embed: Lab and tools/nb.py are two kernel owners over one notebook. The Lab iframe is deliberately unsandboxed and deliberately separate from the sandboxed notebook-output iframe. §20 tools/wiki.py. Human-facing only, absent from system.md. Scope is a mechanical allowlist -- core/ and tools/, never ledger/ or notes/ -- and the source-tree hash makes staleness checkable. `scan` is refused, with the ANTHROPIC_API_KEY reasoning attached. §21 tools/evolve.py. The campaign budget gate runs before generation 0 and before every generation after it; without it, succeeding at the search bricks the system via exit 7. The campaign is the unit of prediction, candidates are sub-runs in their own ledger, EVOLVE-BLOCK escapes are checked mechanically so smoke is needed once rather than per candidate, and a winner still goes through the normal verdict path. Phase 1 only: --remote is refused. §22 tools/report.py. draft is deterministic and free; check refuses on an unresolved claim, an unverified citation, an unjudged cited run, or LaTeX that will not compile. Every number is a \gradnum backed by claims.json; every citation resolves against the corpus or a verified S2 id. 295 tests, no network, no SDK required. The gates are tested against a real ledger in a temp workspace, per §24. Carried over unresolved and recorded in README: Context7's endpoint paths, Shinka's per-candidate callback, headless/claude on Max, historical records left as "unassigned", and phase 2 of the campaign loop. Co-Authored-By: Claude Opus 5 --- .gitignore | 26 +- HANDOFF-2.md | 833 +++++++++++++++++++++++ README.md | 105 ++- agent.py | 73 +- config/grad.toml | 46 +- config/jupyter/jupyter_server_config.py | 59 ++ config/jupyter/overrides.json | 17 + core/budget.py | 428 ++++++++++++ core/campaign.py | 284 ++++++++ core/config.py | 104 ++- core/credentials.py | 11 +- core/errors.py | 7 +- core/gates.py | 40 +- core/haiku.py | 5 +- core/http.py | 128 ++++ core/ledger_store.py | 32 +- core/quota_log.py | 71 +- core/report.py | 375 +++++++++++ core/submit.py | 22 +- hooks.py | 139 +++- prompts/system.md | 24 +- pyproject.toml | 18 + tests/test_budget.py | 384 +++++++++++ tests/test_docs.py | 289 ++++++++ tests/test_evolve.py | 413 ++++++++++++ tests/test_lab_and_wiki.py | 210 ++++++ tests/test_models.py | 95 +++ tests/test_namespace.py | 287 ++++++++ tests/test_report.py | 467 +++++++++++++ tools/budget.py | 249 +++++++ tools/docs.py | 450 +++++++++++++ tools/evolve.py | 810 +++++++++++++++++++++++ tools/gpu.py | 25 +- tools/jobs.py | 257 +++++++- tools/lab.py | 347 ++++++++++ tools/paper_search.py | 4 +- tools/quota.py | 20 +- tools/report.py | 843 ++++++++++++++++++++++++ tools/wiki.py | 261 ++++++++ ui/app.py | 129 +++- ui/widgets/quota_meter.py | 182 ++++- 41 files changed, 8453 insertions(+), 116 deletions(-) create mode 100644 HANDOFF-2.md create mode 100644 config/jupyter/jupyter_server_config.py create mode 100644 config/jupyter/overrides.json create mode 100644 core/budget.py create mode 100644 core/campaign.py create mode 100644 core/report.py create mode 100644 tests/test_budget.py create mode 100644 tests/test_docs.py create mode 100644 tests/test_evolve.py create mode 100644 tests/test_lab_and_wiki.py create mode 100644 tests/test_models.py create mode 100644 tests/test_namespace.py create mode 100644 tests/test_report.py create mode 100644 tools/budget.py create mode 100644 tools/docs.py create mode 100644 tools/evolve.py create mode 100644 tools/lab.py create mode 100644 tools/report.py create mode 100644 tools/wiki.py diff --git a/.gitignore b/.gitignore index 8ebde35..3992e21 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,8 @@ ledger/ledger.sqlite ledger/ledger.sqlite-* # Local corpus and artifacts. The JSONL ledgers ARE tracked on purpose: -# they are the source of truth and are meant to be human-diffable. +# they are the source of truth and are meant to be human-diffable -- +# projects.jsonl, campaigns.jsonl, and candidates.jsonl included. data/corpus.sqlite data/corpus.sqlite-* data/papers/ @@ -18,6 +19,29 @@ ledger/runs/ figures/ notebooks/.ipynb_checkpoints/ +# Which project is selected is machine-local state, not a fact about the +# research (HANDOFF-2 §15). Two checkouts should not fight over it. +ledger/.current_project + +# Lab server state: a live port, pid, and token. Never committed. +data/lab/ + +# RepoWiki output. Generated, human-facing, and deliberately not competing with +# the hand-written docs (§20). +data/wiki/ + +# LaTeX build byproducts. reports//main.tex, claims.json, and +# references.bib ARE tracked: they are the checkable artifacts (§22). +reports/**/*.pdf +reports/**/*.aux +reports/**/*.bbl +reports/**/*.blg +reports/**/*.fls +reports/**/*.log +reports/**/*.out +reports/**/*.fdb_latexmk +reports/**/claims.tex + # Credentials never live in the workspace (HANDOFF §9). This is belt and braces. .env *.pem diff --git a/HANDOFF-2.md b/HANDOFF-2.md new file mode 100644 index 0000000..d91a911 --- /dev/null +++ b/HANDOFF-2.md @@ -0,0 +1,833 @@ +# GradientAgent — Handoff 2: the extension + +Companion to [`HANDOFF.md`](HANDOFF.md), which remains the design record for everything +already built. Section numbering continues from it, so cross-references like "§6" and "§9" +mean what they mean there. + +**§1–§12 of the original remain in force except where §14 below explicitly amends them.** + +*Revision 1 (2026-08-14): scoped eight additions — HF organization namespaces, an embedded +JupyterLab surface, library-currency checking over Context7, a human-facing RepoWiki, an +evolutionary search loop over ShinkaEvolve, a report generator, per-role model selection, +and the project/budget dimension that three of the others turned out to need. Verified +against primary sources during the session: the `huggingface_hub` Jobs signatures, the +RepoWiki 0.3.1 wheel's internals, ShinkaEvolve's Headless provider, Context7's auth model, +and the death of the Tabnine JupyterLab extension.* + +--- + +## 13. What this adds + +Eight items. Three of them (§15, §16, §17) are foundations the rest lean on; two (§21, §22) +are substantial new capabilities; three (§18, §19, §20) are self-contained conveniences. + +| # | Item | Section | Size | +|---|---|---|---| +| 1 | Models selected by role, Opus 5 as the research default | §16 | ~2 h | +| 2 | The `project` dimension and budget ceilings | §15 | ~2 d | +| 3 | HF Jobs under an organization namespace | §17 | ~0.5 d | +| 4 | `tools/docs.py` — library currency via Context7 | §18 | ~0.5 d | +| 5 | Embedded JupyterLab + `tools/lab.py` | §19 | ~1.5 d | +| 6 | RepoWiki, human-facing | §20 | ~1 d | +| 7 | `tools/evolve.py` — evolutionary search over ShinkaEvolve | §21 | ~3–5 d | +| 8 | `tools/report.py` — the scientific report | §22 | ~3–4 d | + +### The rule still governs + +§1's rule is unchanged: **anything that spends money, destroys work, or must be true before +the fact is enforced mechanically, not by prompt.** Two of the additions here are loops that +spend resources without a human between iterations, and one is a machine for asserting +results confidently. Those are precisely the shapes the rule exists for. New rows for §1's +table: + +| Thing that must hold | Enforced by | Section | +|---|---|---| +| Token and credit spend stays bounded, not merely measured | `core/budget.py`, checked at every gateable event | §15 | +| An evolutionary campaign cannot outspend its allocation | campaign budget check before each generation, in `tools/evolve.py` | §15, §21 | +| A job submitted to an org is collectable from that org | the namespace is persisted on the run handle, not just passed at submit | §17 | +| Every number in a report traces to a run record | `report check`, refuses on an unresolved claim | §22 | +| Every citation in a report is a real paper | `report cite` resolves only against the corpus and verified S2 ids | §22 | +| A result that has not been judged cannot be published | `report check` refuses while any cited run has an unjudged deviation | §22 | + +--- + +## 14. Amendments to §3 + +Fewer than expected. Most of this document is *additions* to the decision table rather than +reversals, which is a reasonable signal that the original design absorbed the new +requirements rather than fighting them. + +### Rows that survive, and why that was a live question + +**"Subagents: not used for research."** Survives. A QA subagent was designed and then +rejected in favour of a CLI (§18). The narrowing that made the subagent tractable — check +library currency and compatibility, nothing else — also made its agency unnecessary. Keeping +this row intact means `Task` stays in `DENIED_TOOLS` ([agent.py:39](agent.py:39)) and +`agent.py --probe` needs no changes. + +**"MCP servers: none as external servers."** Survives. Context7 is reached over plain HTTP +from a CLI, which is the same move §5 already makes for Asta: *"It is an MCP endpoint, but it +speaks streamable HTTP, so `paper_search.py` can call it over plain HTTP without adopting MCP +as an architecture"* ([HANDOFF.md:229](HANDOFF.md:229)). This is a second instance of an +existing pattern, not a new one. + +**"Direct Messages API calls: not used."** Survives, and §20 exists partly to keep it that +way — RepoWiki reads `ANTHROPIC_API_KEY` by default, which is exactly what +`credentials.scrub_environment()` deletes. + +### The one real amendment + +| Decision | Was | Now | Why | +|---|---|---|---| +| Notebook editing in the UI | "Rejected: building a notebook editor. The UI renders notebook *output*; editing links out to Lab" ([HANDOFF.md:156](HANDOFF.md:156), [:853](HANDOFF.md:853)) | **JupyterLab is embedded as a tab**, served by `tools/lab.py` | The rejection was of *building an editor*, and that still stands — we build none. What changed is that "links out to Lab" was never wired up: [ui/app.py:405](ui/app.py:405) points at a `localhost:8888` nobody starts. Embedding the real Lab honours the original reasoning better than the current stub does, and it is what makes arbitrary Lab extensions possible at all (§19). | + +### New rows for §3 + +| Decision | Choice | Rationale | +|---|---|---| +| Model selection | By **role**, in a `[models]` config section | Five roles across four surfaces; scattering them across `[agent]` and `[retrieval]` does not survive the additions here — see §16 | +| Resource accounting | One `project` dimension on every cost-bearing record | HF payer attribution, evolutionary campaigns, and research budgets are three faces of one missing abstraction — see §15 | +| Evolutionary search | **ShinkaEvolve**, driven by `tools/evolve.py`, not forked | Its Python API plus the Headless provider covers what we need; a fork is a maintenance cost to defer until hook points prove insufficient — see §21 | +| Report generation | **Built here**, not adopted from a harness | Every existing harness reconstructs provenance from unstructured logs. Ours is already structured, and that is the whole advantage — see §22 | +| AI completion inside Lab | **None for now** | The only subscription-compatible path would be a second provider and a fifth credential; Tabnine's JupyterLab extension is dead (§19) | + +### Additions to "Rejected, and why" + +- **A reality-checker subagent.** Designed, scoped down, then rejected. Once the brief + narrowed to "outdated libraries and compatibility issues", every claim it made became + checkable against an oracle, and a CLI the main agent calls is strictly less invasive than + re-enabling `Task`. See §18. +- **Pyright as a preflight gate.** Strict-mode Pyright over an ML codebase produces noise, + and a check that gets ignored is worse than no check. Its deterministic strength + (signatures, removed attributes) is covered by kernel introspection plus §18. Deferred, not + rejected: revisit if `tools/docs.py` proves too soft. +- **Tabnine for JupyterLab.** `jupyterlab-tabnine` 0.0.24 was released 2021-08-24 with + classifiers stopping at Python 3.9; the npm client is equally stale; the classic-Notebook + variant last moved in March 2021. The JupyterLab 3→4 break is what killed it. +- **Forking ShinkaEvolve up front.** See §21 — a driver first, a fork only on evidence. +- **AI Scientist v2 / PaperOrchestra / Denario as the report harness.** See §22 — adopting + one means discarding the ledger advantage and conforming to its log format. +- **Replacing the Voyage reranker with Haiku.** See §16 — worse at the task, and it moves + load from credits onto the subscription quota this design is trying to protect. + +--- + +## 15. The project dimension and the budget system + +**Build this first.** §17, §21, and §22 all consume it, and building either loop (§21, §22) +before it exists is how a runaway campaign ends up blocking every future submission through +the §6 stale-run gate with no record of what consumed the quota. + +### The gap + +Three resources are consumed. They are tracked very unevenly: + +| Resource | Measured today | Ceiling today | +|---|---|---| +| GPU dollars | yes — `runs.jsonl`, actuals and estimates | **yes** — `core/gates.py:check_spend`, per-job and monthly | +| API credits (Voyage embeddings, OpenRouter rerank) | yes — `quota.jsonl`, as `funnel.rerank` / `embed` stages | **no** | +| Subscription quota (tokens) | yes — `quota.jsonl`, by stage | **no** | + +So [README.md:20](README.md:20)'s claim that "cumulative spend stays bounded" holds for one +resource in three. The other two are instrumented and unbounded. §21 in particular is a loop +that consumes all three. + +### The insight: one dimension, three uses + +Three separate requirements turned out to want the same thing — a dimension carried on every +cost-bearing record: + +- **§17** needs to know which account paid for an HF job (personal vs. organization). +- **§21** needs to bound a campaign made of many runs. +- **The user's ask** is a budget for a piece of research. + +Build it once. Every run record, every `quota.jsonl` entry, and every credit spend carries a +`project` id. + +### Records + +`ledger/projects.jsonl`, append-only and folded like `runs.jsonl`: + +```json +{ + "id": "proj-scaling-w2", + "created_at": "2026-08-14T09:00:00Z", + "title": "width-vs-depth scaling under a fixed token budget", + "payer": "hf:myorg", + "budget": {"gpu_usd": 50.0, "quota_tokens": 5000000, "credits_usd": 10.0}, + "status": "open" +} +``` + +`payer` lives on the project rather than being invented per submission, so the org attribution +in §17 is a consequence of choosing a project rather than a separate flag to forget. + +**Current project selection.** A file, `ledger/.current_project`, written by +`tools/budget.py use ` and read by every CLI, overridable per invocation with +`--project`. Deliberately *not* an environment variable: `credentials.scrub_environment()` +([core/credentials.py:103](core/credentials.py:103)) strips the agent's environment, and a +selection mechanism that the agent's own startup deletes is a bug waiting to happen. + +Every run record gains `"project": ""`. Every `quota_log` entry gains the same. This is +an additive schema change; `core/ledger_store.py` folds unknown-project records as +`"unassigned"` so existing ledgers keep loading. + +### Enforcement, and where it is honest + +Enforcement quality differs by resource, and the difference is structural rather than a +shortcoming of the implementation: + +**GPU dollars — clean.** Submission is a discrete, gateable event. `check_spend` already +fires there; it gains a project-scoped check alongside the global one. + +**Evolutionary campaigns — clean.** A generation boundary is a discrete event. §21 checks the +projected cost of generation *n+1* against remaining allocation before starting it. + +**Subscription tokens — granular to one turn.** Tokens are consumed continuously inside a +turn and there is no way to refuse mid-turn. Two mechanisms, neither depending on SDK +behaviour we have not verified: + +1. `agent.py`'s own turn loop checks remaining allocation *before* issuing the next turn and + refuses with the overrun printed. This is our code end to end. +2. `hooks.py:pre_tool_use` denies cost-bearing Bash commands (`tools.jobs submit`, + `tools.evolve run`, `tools.report write`) once the project is over budget. That hook + already denies reliably ([hooks.py:133](hooks.py:133)) and is exercised by + `agent.py --probe`. + +The Stop hook ([hooks.py:145](hooks.py:145)) keeps its current job — recording usage — and +gains threshold warnings. It is deliberately *not* the enforcement point: its documented +`block` semantics force continuation rather than halting, which is the opposite of what is +wanted here. + +**So the honest statement is: token budgets are enforced to a granularity of one turn's +overrun.** Write that in the CLI's `--help`, not just here. + +### A second honesty note + +Subscription quota is not linear in tokens, and the real limits are rolling windows (5-hour +and weekly on Max) which the SDK does not expose as a remaining balance. A token ceiling is +therefore a **proxy the user controls**, not a mirror of Anthropic's limit. Hitting the real +rate limit is an event the system can only observe after the fact. The meter must not imply +otherwise — this is the same discipline §10 already applied when it *"reworded the quota +meter to what it can actually measure"* ([HANDOFF.md:20](HANDOFF.md:20)). + +### CLI — `tools/budget.py` + +```bash +python -m tools.budget new --id proj-scaling-w2 --title "..." \ + --gpu-usd 50 --quota-tokens 5e6 --credits-usd 10 --payer hf:myorg --json +python -m tools.budget use proj-scaling-w2 --json +python -m tools.budget status --json # current project, spend, remaining, per resource +python -m tools.budget raise --gpu-usd 75 --json # deliberate, logged, never silent +python -m tools.budget close proj-scaling-w2 --json +``` + +`raise` appends an event rather than mutating; a ceiling that can be edited invisibly is not +a ceiling. Same argument as §7's append-only ledger. + +New exit code: **12 — project budget exceeded**, distinct from 6 (global spend ceiling), so +"this research ran out of its allocation" is never confused with "the machine is out of +money". + +### UI + +The header meter ([ui/widgets/quota_meter.py](ui/widgets/quota_meter.py)) gains a project +selector and shows three bars rather than one. The Quota tab gains a per-project breakdown by +stage. Both read the ledger; no new logic in the UI, per §10. + +--- + +## 16. Models by role + +### The correction that motivates this + +**Haiku is not the reranker.** The funnel is expand → retrieve → **rerank** → triage → select. +Haiku runs stages 0 and 3 (expand, triage). Stage 2's reranker is `voyageai/rerank-2.5` via +OpenRouter — a dedicated cross-encoder, not a generative model +([config/grad.toml:41](config/grad.toml:41), [HANDOFF.md:124](HANDOFF.md:124)). +[core/haiku.py:249](core/haiku.py:249) draws the line explicitly: triage is *"a funnel +widener, not a better ranker."* + +Do not swap Voyage for Haiku. It is worse at pairwise relevance scoring, and — the reason +that matters here — Voyage costs **credits** while Haiku costs **subscription quota**. The +swap moves load onto the scarcer resource. + +### Model facts + +- The Claude 5 family is **Fable 5, Opus 5, Sonnet 5**. There is no Haiku 5; the latest Haiku + is **4.5**. +- Ids: `claude-opus-5`, `claude-sonnet-5`, `claude-haiku-4-5-20251001`. +- `config/grad.toml` and `core/config.py` currently default to `claude-opus-4-5` + ([config/grad.toml:55](config/grad.toml:55), [core/config.py:80](core/config.py:80)). Update + to `claude-opus-5`. +- The existing `claude-haiku-4-5` entries are already correct. + +### The `[models]` section + +```toml +[models] +research = "claude-opus-5" # the main loop (§3) +evolve = "claude-sonnet-5" # ShinkaEvolve mutation operators (§21) +expand = "claude-haiku-4-5" # funnel stage 0 (§5) +triage = "claude-haiku-4-5" # funnel stage 3 (§5) +report = "claude-opus-5" # prose synthesis (§22) +cite = "claude-haiku-4-5" # citation resolution — mechanical matching (§22) +``` + +`[retrieval] rerank_model` and `embed_model` **stay where they are**. They are a different +provider on a different billing rail, and folding them into `[models]` invites exactly the +substitution argued against above. `core/config.py` keeps the old `[retrieval] triage_model` / +`expand_model` keys readable as overrides for one release so existing configs do not break. + +Every model call already routes through `quota_log.from_sdk_usage` with a stage; the stage +gains the role name so `tools/quota.py summary` can answer "what did Opus cost me this week" +without inference. + +### On the evolve default + +Sonnet 5 as the default is right for cost. But ShinkaEvolve's design is explicitly *an +ensemble of LLMs acting as mutation operators* — collapsing to a single model discards +diversity the algorithm is built around. **Default to an ensemble of Sonnet 5 (primary) and +Haiku 4.5 (cheap explorer)** and let Shinka's bandit allocate between them. Overridable with +`--set evo.llm_models=...`, which is Shinka's own mechanism. + +--- + +## 17. HF Jobs under an organization namespace + +### Verified API facts + +Checked against the installed `huggingface_hub` **1.16.1** during the design session: + +``` +run_job(*, image, command, env, secrets, flavor, timeout, labels, volumes, namespace, token) +inspect_job(*, job_id, namespace, token) +fetch_job_logs(*, job_id, namespace, follow, tail, token) +cancel_job(*, job_id, namespace, token) +list_jobs(*, timeout, namespace, token) +whoami(token, *, cache=False) -> dict +``` + +`JobInfo` fields: `id, created_at, started_at, finished_at, docker_image, space_id, command, +arguments, environment, secrets, flavor, labels, volumes, status, durations, owner, +initiator, endpoint, url`. + +### The trap + +**`namespace` is a property of the job handle, not a submit-time parameter.** Adding it only +to `run_job` at [tools/jobs.py:148](tools/jobs.py:148) produces a job that cannot be found +again: `inspect_job` and `fetch_job_logs` would look under the personal namespace and 404. The +run never collects, goes stale, and then blocks *every* future submission through the §6 +stale-run gate (exit 7). The failure appears far from its cause. + +So the namespace must be persisted into the handle at +[tools/jobs.py:173](tools/jobs.py:173) (`attach_handle`) and threaded through every call +site that takes one: + +- `_poll` — [tools/jobs.py:285](tools/jobs.py:285) +- `_logs` — [tools/jobs.py:307](tools/jobs.py:307) +- `cmd_status` — [tools/jobs.py:347](tools/jobs.py:347) +- `cmd_collect` — [tools/jobs.py:376](tools/jobs.py:376) +- `run_smoke` — [tools/jobs.py:199](tools/jobs.py:199) + +### Membership validation + +`whoami(token=...)` returns the organizations a token can act for. Validate the requested +namespace against it **before** `record_submission`, in the same place `_hub()` and `_token()` +are already called for exactly this reason ([tools/jobs.py:136](tools/jobs.py:136)): a +configuration problem must not leave a phantom estimate sitting on the ceiling. Surface it in +`credential status` too. + +One network call per submit is acceptable on a path about to spend dollars. Do not cache it +aggressively — org membership changing is precisely the case worth catching. + +### Resolution order + +`--namespace` flag → spec `[target] namespace` → project `payer` (§15) → `[hf] namespace` → +`None` (personal). This mirrors how `flavor` already resolves at +[tools/jobs.py:128](tools/jobs.py:128). + +### Interaction with the submission hash + +The hash deliberately excludes `target` ([core/submission.py:299](core/submission.py:299)), +which is why `flavor` is not hashed. `namespace` follows the same rule for consistency — it is +not hashed. + +The consequence is real and must be handled rather than ignored: a preflight whose `smoke` +check ran under personal credentials validates a job that will run in an organization. So +`run_smoke` records the namespace it used into the check result (which already flows into the +preflight record via `record_check_result`, +[tools/preflight.py:369](tools/preflight.py:369)), and `submit` **warns** when they differ. +Warn, not refuse — consistent with how `target` and `flavor` already behave. + +### Spend + +Costs are attributed to the project's `payer` (§15). An organization's budget and a personal +budget are separate allocations; without this, org runs silently consume the personal ceiling +and vice versa. + +`hooks.py` is unchanged — bare `hf` stays denied. + +--- + +## 18. Library currency: `tools/docs.py` + +### Why a CLI and not a subagent + +The original plan was a Haiku "reality checker" subagent with Context7 and Pyright. It was +rejected after the brief narrowed. The reasoning is worth keeping, because it will be +tempting to revisit: + +A QA layer staffed by a weaker model than the one it checks is only sound when every claim it +makes is **checkable against an oracle**. Narrowing the brief to library currency and +compatibility achieved that. But once achieved, the agency was doing no work: "point it at a +file, get a verdict" is a tool, not an agent. The CLI form keeps `Task` denied, keeps +Context7's schemas out of the main loop's context entirely, and inherits `--json`, exit codes, +and `fix` fields from `core/cli.py` for free. + +Revisit only if the iterate-and-recheck loop (introspect → hypothesise → run a counter-example +→ recheck) proves necessary. That is the one thing the CLI form cannot do. + +### The two oracles + +Context7 alone is not sufficient, and it is the weaker half: + +1. **Introspection — what actually exists on this machine.** `importlib.metadata.version()`, + `inspect.signature()`, `dir()`, run through `tools/nb.py exec`. Cheap, offline, + definitive. This is how the `namespace` parameter in §17 was found, in about ten seconds. +2. **Context7 — what is current.** Deprecations, changed idioms, migration paths. Answers what + introspection cannot see. + +**Order matters: introspect first.** A checker relying on Context7 alone will confidently +describe an API version that is not installed. + +The main agent runs both. `prompts/system.md`'s tool list gains one entry, and the "Habits +that matter" section gains a line: *check a library call against the installed signature +before trusting it, and against `docs.py` before assuming it is current.* + +### Context7 facts + +Verified: a REST API exists (reference at `context7.com/docs/api-guide`), auth is +`Authorization: Bearer `, keys are free from their dashboard, and rate limits scale with +a registered key. The MCP tool names are `resolve-library-id` and **`query-docs`** — note that +`get-library-docs`, the name in older material, is stale. An official `ctx7` CLI also exists +(`ctx7 library `, `ctx7 docs `). + +**Not verified in session: the exact REST endpoint paths.** Read +`context7.com/docs/api-guide` before implementing. + +### Why wrap rather than allowlist `ctx7` + +The `--json` / exit-code / `fix`-field contract is what makes a tool legible to the model +(§8), and `ctx7` will not have it. The wrapper is also where the credential fetch and the +cache live, and caching matters more than it sounds: documentation lookups repeat heavily and +`core/http.py` already has the TTL cache and rate-limit machinery. + +### Shape + +```bash +python -m tools.docs resolve "huggingface_hub" --json +python -m tools.docs query "run_job namespace parameter" --json +python -m tools.docs check tools/jobs.py --json # introspect imports, flag stale/deprecated calls +``` + +`check` is the interesting one: parse the file's imports, resolve installed versions, and +report calls whose signatures do not match what is installed, plus anything Context7 flags as +deprecated. Exit 9 (`a check ran and failed`) on findings, so it composes with preflight's +declared-check mechanism ([tools/preflight.py:315](tools/preflight.py:315)) if a pipeline +wants it as a gate later. + +### Credential + +Fifth Credential Manager entry, `context7_key`, added to `CREDENTIAL_NAMES` +([tools/jobs.py:477](tools/jobs.py:477)), fetched at point of use exactly like +[tools/jobs.py:62](tools/jobs.py:62), and added to `scrub_environment`'s list +([core/credentials.py:111](core/credentials.py:111)). + +--- + +## 19. The notebook surface: embedded JupyterLab + +Amends §10's notebook handling. See §14 for why this honours the original reasoning rather +than reversing it. + +### Architecture + +`tools/lab.py` manages a JupyterLab server as a subprocess on a side port with a token; the +UI adds a tab holding it in an iframe. + +```bash +python -m tools.lab start --json # returns port + token +python -m tools.lab status --json +python -m tools.lab extensions --json # what is installed, so the state is inspectable +python -m tools.lab stop --json +``` + +Two things that will otherwise cost an afternoon each: + +**Framing.** JupyterLab ships `X-Frame-Options` / CSP headers that block embedding. It needs +a `tornado_settings` header override permitting the app's origin, in +`config/jupyter/jupyter_server_config.py`. + +**The sandbox.** The existing notebook iframe is `sandbox=""` for a real reason — notebook +output is untrusted HTML that could otherwise run script in the page driving the agent +([ui/app.py:415](ui/app.py:415)). The Lab iframe **cannot** be sandboxed that way. It must be +a separate iframe, deliberately unsandboxed, pointed at a server we started ourselves. Keep +the existing read-only renderer as-is for the output view; do not merge the two. + +### Kernel ownership — the rule that must not be lost + +`tools/nb.py` spawns detached kernels via its own connection files +([tools/nb.py:66](tools/nb.py:66)). Lab has its own kernel manager. Two owners over one +notebook reproduces exactly the "works in the kernel that grew it" failure that `nb.py verify` +exists to catch ([HANDOFF.md:503](HANDOFF.md:503)). + +**The discipline is unchanged: anything edited in Lab passes `nb verify` before it is cited in +`notes/` or referenced from a ledger entry.** The highest-value part of this whole item is a +per-notebook **Verify** button in the Notebooks tab that shells out to +`python -m tools.nb verify --json` and renders the failing cell index and traceback. +Build that first; it is worth more than the embed. + +`NotebookEdit` stays in `DENIED_TOOLS` ([agent.py:39](agent.py:39)). This item is about the +*human* editing by hand; the agent continues to edit notebooks through Write/Edit plus +`nb.py`, which works and does not depend on version-sensitive tool semantics. + +### Arbitrary extensions + +Lab already has a plugin system. Do not design one. What gets built is the reproducibility +layer: + +- a pinned `lab` extra in `pyproject.toml`, so the extension set is declared rather than + accumulated +- `config/jupyter/` holding `jupyter_server_config.py` and `overrides.json` +- `tools/lab.py extensions --json` so the installed set is inspectable like everything else + +"Connect an arbitrary extension" then means: add a pin, reinstall, restart. + +Three caveats to record before anything is installed: + +1. **Server extensions run as you.** A frontend extension is confined to the browser. A server + extension runs in the Lab process with your filesystem rights — it can read `ledger/` and + `notes/`, and it can `import keyring` and reach the credential store. That is the same + honest residual [core/credentials.py:9](core/credentials.py:9) already names. Frontend-only + extensions are low risk; read a server extension before installing it. +2. **Origin.** An extension is code running in Lab's origin, and that iframe is unsandboxed. + Keep Lab on its own port and do not share the storage secret from + [ui/app.py:241](ui/app.py:241). +3. **Pin everything.** The JupyterLab 3→4 break is what killed the Tabnine extension. Pin + `jupyterlab` itself and every extension, or an unrelated `pip install -U` takes the app + down. + +### AI completion inside Lab + +Not for now. `jupyterlab-tabnine` is dead (§14). The live alternatives — `jupyter-ai` and +LSP-based copilots — route through an API key, which collides with §2 and would need a second +provider and a fifth credential decision. Revisit only if hand-editing in Lab becomes frequent +enough to justify it. + +--- + +## 20. RepoWiki: the human's map + +**Scope: human-facing only.** Not in the agent's tool list, not in `prompts/system.md`, no +context cost. Its job is letting a person reacquire the shape of a growing codebase quickly. +HANDOFF.md remains the design record and README.md the report; RepoWiki targets the third +thing — the module-level "what calls what, and where does this value come from" view nobody +wants to maintain by hand. + +### Teardown facts (wheel 0.3.1, inspected in session) + +- **All LLM coupling is in one 95-line file**, `repowiki/llm/client.py`, behind a two-method + async interface: `complete(messages, *, temperature, max_tokens, response_format)` and + `stream(...)`. +- Constructed at exactly **four call sites**: `cli.py:200`, `cli.py:402`, + `server/routers/chat.py:65`, `server/routers/scan.py:117`. +- **`response_format` is never passed by any caller.** All four analyzer calls + (`core/analyzer.py:116, 209, 237, 297`) use plain `complete(messages, max_tokens=4096)` and + parse the returned text themselves. The structured-output problem that would have made a + fork risky does not exist. +- Prompts are simple `[system, user]` pairs from `llm/prompts.py` (five builders). Flattening + to `ClaudeAgentOptions(system_prompt=...)` + `query(prompt=user)` is exact, not lossy. +- **`repowiki map` is LLM-free.** `cli.py:40 repo_map()` never touches `LLMClient`, and + `core/graph.py` has no LLM references. It needs no credential at all. + +### The decision + +**Try `repowiki map .` first.** It is free and may cover enough of the need to make the rest +unnecessary. + +If more is wanted, **fork and replace `LLMClient`** with an Agent SDK implementation — roughly +120 lines, with `core/haiku.py:110` as the model for the plumbing. Token counters map onto +`quota_log.from_sdk_usage` with a new stage; `total_cost` should report 0 rather than a +fabricated number, since subscription usage is not priced per token. + +**The fork is optional, not required, and it is worth knowing why before spending the day.** +`scrub_environment()` cleans only the *agent's* process ([agent.py:77](agent.py:77)). A human +running `repowiki scan` in their own shell with an API key set violates nothing technically. +The catch is that a key in the user profile will also be in the agent's environment and +trigger the scrub warning on every launch — safe, but noisy, and it erodes the §2 discipline +by habituation. The fork is cleaner. It is a preference. + +While in there, one optional upgrade: those four analyzer calls parse free text, which is the +failure mode [core/haiku.py:12](core/haiku.py:12) documents — *"prompting for JSON and parsing +it fails silently on the tenth call."* Routing them through the forced-tool pattern removes a +class of silent corruption from the generated wiki. + +### Scope and staleness + +- Point it at `core/` and `tools/`. **Never** `ledger/`, `notes/`, or any papers directory — + it ships content to a third party, and those hold research data. +- Output HTML (`--format html --open`), not markdown committed to the repo, so it never + competes with the hand-written docs. +- A wiki behind the code is worse than none, because it is trusted. Record the source-tree + hash in the output and provide a one-line staleness check — the pattern + `core/submission.py` already implements. + +--- + +## 21. Evolutionary search: `tools/evolve.py` + +### What it is + +[ShinkaEvolve](https://github.com/SakanaAI/ShinkaEvolve) (Sakana AI, arXiv:2509.19349) +maintains a population of programs evolved across generations, with an ensemble of LLMs acting +as mutation operators. Three patch types (`diff`, `full`, `cross`), island-based diversity, +parallel evaluation locally or on SLURM, and a `shinka_visualize` web UI. + +**The Headless provider solves the auth problem.** A May 2026 release added CLI-backed +mutation models for subscription-backed agents — model strings like `headless/claude` and +`headless/codex@gpt-5.5?effort=high`, routed through `npx -y @roberttlange/headless`, with a +`headless --check` at startup. `examples/sine_approx_headless` is fully API-free. So this runs +on the Max subscription without bending §2. + +### Why the contracts align + +Shinka wants `evaluate.py` returning a metrics dict containing `combined_score`, and +`initial.py` with `EVOLVE-BLOCK-START` / `EVOLVE-BLOCK-END` markers around the mutable region. +Grad already has a submission spec with an entrypoint and a `metrics_file` whose parsed +contents flow through `submit_lib.parse_metrics` → `results` → `deviations`. These describe +the same object. Structurally this is "run the submit/collect loop N times with an LLM +choosing the next candidate" — an extension of §6/§7, not a bolt-on. + +### Driver, not fork + +Shinka exposes a Python API: `EvolutionConfig`, `LocalJobConfig`, `DatabaseConfig`, +`ShinkaEvolveRunner(...).run()`. What we need is **gating and ledger integration around the +loop**, which is a driver. + +Fork only if per-candidate budget charging turns out to require intercepting inside the +generation loop and no hook point exists. **Not verified in session: whether Shinka exposes a +per-candidate callback.** Check before starting; it decides driver-vs-fork. + +### The four collisions, and their resolutions + +**1. An evolutionary loop is a machine for spending money with no human in it.** This is the +dangerous one. `check_spend` fires per submission, so the ceiling *would* stop it — at +generation 40, abandoning an in-flight run that then goes stale and blocks every future +submission (exit 7). Succeeding at the search would brick the system. + +*Resolution:* a **campaign budget gate**. Before generation 0, refuse unless +`estimate_per_candidate × max_candidates` fits under the project's remaining allocation (§15). +Re-check before each generation. Shinka's own `max_api_costs` covers the LLM side; the compute +side is the expensive half and Grad must own it. **Do not run a single remote generation +before this exists.** + +**2. The expectation gate is 1:1 with a run; evolution is 1:N.** You cannot pre-register a +prediction per candidate. + +*Resolution:* the **campaign** is the unit of prediction — "the evolved variant beats baseline +X on metric Y by ≥ Z" — with candidate evaluations recorded as sub-runs exempt from the +per-run expectation gate. This is arguably more faithful to §7 than the current design, and it +is exactly the relational shape [prompts/system.md:16](prompts/system.md:16) already prefers. +Needs a `campaign` kind in the ledger and gates that understand campaign membership. + +**3. Every mutation invalidates the preflight hash — correctly, and expensively.** +`Submission.hash()` covers the entrypoint's import graph, so each candidate needs a fresh +preflight, and `smoke` is a *paid remote job* ([tools/jobs.py:199](tools/jobs.py:199)). Naively +this doubles per-candidate cost. + +*Resolution:* candidates run `--only tests,dry_run` — both local, both fast. Smoke is required +once per campaign at the baseline, and again whenever a mutation escapes the evolve-block. The +`EVOLVE-BLOCK` markers make "did it escape" mechanically checkable, which is convenient. + +**4. `combined_score` is a Goodhart machine.** A search optimising a scalar will find the bug +in the metric. That is precisely the failure this codebase's temperament resists — *"a +surprise is an alarm... a bug hypothesis first"* ([prompts/system.md:15](prompts/system.md:15)). + +*Resolution:* the campaign winner goes through the normal `verdict` path before it counts as a +result, and the campaign report surfaces top-K rather than the argmax. + +### Phasing + +**Phase 1: local only.** Shinka needs no GPU for many tasks and the headless example is +API-free. Run a campaign evaluated entirely through `tools/nb.py`, zero remote jobs. This +proves the campaign records, the sub-run bookkeeping, and the budget integration while the +blast radius is zero. + +**Phase 2: remote**, behind the campaign budget gate. + +Doing the ledger work and the spend work simultaneously against live GPU jobs is how you learn +about exit 7 the hard way. + +### Shape + +```bash +python -m tools.evolve init --task-dir pipeline/evolve-lr --json +python -m tools.evolve run --project proj-scaling-w2 --generations 20 --local --json +python -m tools.evolve status --campaign camp-... --json +python -m tools.evolve promote --campaign camp-... --candidate 47 --json # into a normal run +``` + +Default models per §16: Sonnet 5 primary plus Haiku 4.5 explorer. + +Sources: [ShinkaEvolve](https://github.com/SakanaAI/ShinkaEvolve) · +[releases](https://github.com/SakanaAI/ShinkaEvolve/releases) · +[agentic_usage.md](https://github.com/SakanaAI/ShinkaEvolve/blob/main/docs/agentic_usage.md) · +[arXiv:2509.19349](https://arxiv.org/abs/2509.19349) · +[claude-evolve](https://github.com/samuelzxu/claude-evolve) (community Claude Code +reimplementation — considered, not chosen: Grad's value is the ledger and the gates, and +pointing a mature harness at them beats adopting a second harness with its own orchestration +opinions) + +--- + +## 22. The report: `tools/report.py` + +### Build, don't adopt + +Surveyed: [AI Scientist v2](https://github.com/SakanaAI/AI-Scientist-v2), +[PaperOrchestra](https://arxiv.org/pdf/2604.05018), +[Jr. AI Scientist](https://arxiv.org/pdf/2511.04583), +[Denario](https://arxiv.org/pdf/2510.26887), +[Camyla](https://arxiv.org/pdf/2604.10696), +[CiteLLM](https://arxiv.org/html/2602.23075). + +Every one of them **reconstructs provenance from unstructured experiment logs.** Grad's is +already structured: expectations with `basis` and `comparability`, runs with results and +`deviations`, verdicts with notes, figures, corpus paper ids. That is strictly better input +than any of these systems receive. Adopting one means discarding the advantage and conforming +to its log format. + +### The two structural guarantees + +The hard problem in machine-written papers is not prose. It is hallucinated citations and +unsupported claims. Both can be made **structurally impossible** here: + +**Citations.** `report cite` resolves only against the local corpus (`core/corpus.py` has the +ids) plus S2-verified ids (`core/http.py` already talks to S2). A `\cite{}` key with no +resolved entry is a hard error, not a warning. + +**Claims.** Every asserted number carries a `\gradnum{}` macro backed by a `claims.json` +sidecar mapping each key to `(run_id, quantity)`. `report check` verifies each against the +ledger. A number that does not resolve fails the check. + +No off-the-shelf harness can do either, because none of them know about the ledger. + +### Subcommands + +```bash +python -m tools.report draft --project proj-scaling-w2 --json # deterministic, no LLM +python -m tools.report write --project proj-scaling-w2 --json # prose + [CITE:...] placeholders +python -m tools.report cite --project proj-scaling-w2 --json # resolve, verify, emit .bib +python -m tools.report check --project proj-scaling-w2 --json # the gate +python -m tools.report build --project proj-scaling-w2 --json # PDF +``` + +`draft` emits the skeleton from the ledger — every expectation, its runs, its deviations, its +verdict, its figures — with no model in the loop. It is useful on its own and costs nothing. + +`check` enforces, in order: + +1. every `\gradnum{}` key resolves to a `(run_id, quantity)` present in the ledger, with a + matching value; +2. every `\cite{}` key exists in `references.bib`, and every bib entry came from the corpus or + a verified S2 id; +3. **no cited run has an unjudged deviation** — `collect` already computes `needs_verdict` + ([tools/jobs.py:435](tools/jobs.py:435)); +4. the LaTeX compiles clean — no unmatched braces, duplicate labels, or unescaped specials. + +Rule 3 is the one most in the spirit of this system: **you should not be able to write up a +result you have not judged.** + +`check` refuses; it does not warn. A report generator is where this system's epistemics either +hold or collapse — the whole design exists to stop the user believing results too easily, and +a paper generator is a machine for asserting them confidently. + +### What to steal, and from whom + +- **Camyla** — the two-pass citation flow. Write with `[CITE:keyword]` placeholders, then + resolve by extracting a context window around each and verifying the S2 candidate's title + and abstract against that context. Much better than citing inline. +- **PaperOrchestra** — the constraint set: cite keys must match `references.bib` exactly, no + fabricated results, compile-clean LaTeX. Encode as validation, not as prompt text. +- **Denario** — progressive versions. It emits four because unattended LaTeX does not reliably + compile. Copy the checkpointing; it is the honest response. +- **AI Scientist v2** — the role split (`--model_writeup` / `--model_citation` / + `--model_review`), which maps onto §16's roles. +- **Jr. AI Scientist** — draft → reflect → adjust, working directly inside a conference + template directory. + +Template: vendor NeurIPS or ICML style. Not a decision worth deliberating. + +--- + +## 23. Open questions + +Things genuinely unresolved, listed so they are not mistaken for settled: + +1. **Does ShinkaEvolve expose a per-candidate callback?** Decides driver-vs-fork (§21). Check + `ShinkaEvolveRunner` before starting. +2. **Context7's exact REST endpoints.** Read `context7.com/docs/api-guide` (§18). +3. **Does `headless/claude` work against a Max subscription specifically?** Reported in + release notes, not tested here (§21). +4. **What granularity should campaign sub-runs have in `runs.jsonl`?** One record per + candidate is honest but will dominate the ledger — a 100-generation campaign is thousands + of rows. Consider a separate `candidates.jsonl` folded into the campaign record, with only + promoted candidates entering `runs.jsonl` (§21). +5. **Should `report write` be allowed to run at all while a project is over budget?** Argument + for yes: the report is how you find out what the spend bought. Argument for no: it is a + cost-bearing loop like any other. Currently specified as denied by the §15 hook; revisit + after first use. +6. **Whether the `project` dimension should be retrofitted onto historical records** or left + as `"unassigned"`. Specified as the latter; cheap to change while the ledger is small. + +--- + +## 24. Build order + +Ordering matters more than usual here, because §15 became a prerequisite for three others. + +**Foundation — ~2.5 days** + +1. **§16** — `[models]` section, Opus 5 default, role-tagged quota stages. ~2 hours. Unblocks + everything and touches nothing risky. +2. **§15** — `project` dimension, `tools/budget.py`, quota and credit ceilings, exit code 12, + UI meter. ~2 days. Prerequisite for 3, 7, 8. +3. **§17** — HF org namespaces, consuming the project `payer`. ~0.5 days. + +**Tools — ~2.5 days** + +4. **§18** — `tools/docs.py`. ~0.5 days. +5. **§19** — Verify button first, then `tools/lab.py` and the embed. ~1.5 days. +6. **§20** — `repowiki map` trial, then the fork if wanted. ~1 day. + +**The big two — ~6–9 days** + +7. **§21** — `tools/evolve.py`, local-only phase 1. ~3–5 days. +8. **§22** — `tools/report.py`. ~3–4 days. + +Roughly 2.5–3 weeks. Items 1, 3, and 4 total about a day and a half and each stands alone, so +they are the sensible slice if value is wanted sooner. + +**Testing.** Same discipline as the existing suite — a real ledger in a temp workspace, no +network, no SDK required ([README.md:190](README.md:190)). Straightforward for 1, 2, 3, and 6; +4 needs a faked HTTP layer; 7 needs a faked Shinka runner; 8 needs a fixture ledger with a +known-good and a known-bad claim set. The budget gates in 2 deserve the same treatment §6's +gates got: tested against a real ledger, not mocks, because they are what stands between a +loop and a bill. + +--- + +*Written 2026-08-14. Verified in session: `huggingface_hub` 1.16.1 Jobs signatures; RepoWiki +0.3.1 wheel internals; ShinkaEvolve's Headless provider; Context7's auth model and current MCP +tool names; `jupyterlab-tabnine`'s release history. Unverified items are listed in §23.* diff --git a/README.md b/README.md index 19b3825..49689e9 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,10 @@ A personal research agent for mathematics and machine learning. Runs on one person's Windows desktop, backed by a Claude Max subscription. -This is the implementation of [`HANDOFF.md`](HANDOFF.md), which remains the -design document of record. Where this README and the handoff disagree, the -handoff is the intent and this file is the report. +This is the implementation of [`HANDOFF.md`](HANDOFF.md) and its extension +[`HANDOFF-2.md`](HANDOFF-2.md), which together remain the design documents of +record. Where this README and the handoffs disagree, the handoffs are the intent +and this file is the report. The design temperament is borrowed from Mario Zechner's *pi*: trust the model, keep the system prompt small, keep the tool surface small, prefer files and CLIs @@ -28,6 +29,25 @@ is a sentence in `prompts/system.md`. | Notebooks run clean top-to-bottom | `tools/nb.py verify` on a fresh kernel | | No general remote-execution capability | credentials in Windows Credential Manager, read only by `jobs.py` / `gpu.py` | | Concurrent ledger writes don't corrupt | one locked `core/jsonl.py:append`; no CLI writes a ledger file directly | +| Token and credit spend stays bounded, not merely measured | `core/budget.py`, checked at every gateable event | +| An evolutionary campaign cannot outspend its allocation | the campaign gate in `tools/evolve.py`, before generation 0 and before each generation after it | +| A job submitted to an org is collectable from that org | the namespace is persisted on the run handle, not just passed at submit | +| Every number in a report traces to a run record | `tools/report.py check` refuses on an unresolved claim | +| Every citation in a report is a real paper | `report cite` resolves only against the corpus and verified S2 ids | +| A result that has not been judged cannot be published | `report check` refuses while any cited run has an unjudged deviation | + +### The one thing that is *not* fully mechanical, and why + +**Subscription tokens are enforced to a granularity of one turn's overrun.** +Tokens are consumed continuously inside a turn and there is no way to refuse +mid-turn, so `agent.py` checks the remaining allocation *before* issuing the +next turn and `hooks.py` denies cost-bearing Bash once the project is over. The +turn that crosses the ceiling finishes. + +A second honesty note: subscription quota is not linear in tokens, and the real +limits are rolling windows (5-hour and weekly on Max) that the SDK does not +expose as a remaining balance. **A token ceiling is a proxy you control, not a +mirror of Anthropic's limit.** The meter says so on screen. ## Install @@ -35,6 +55,10 @@ is a sentence in `prompts/system.md`. pip install -e ".[agent,notebook,retrieval,remote,ui,math,dev]" ``` +Optional extras, each pinned and each independently skippable: `lab` (the +embedded JupyterLab, pinned exactly because the 3→4 break is what killed the +Tabnine extension), `wiki` (RepoWiki), `evolve` (ShinkaEvolve). + The core — ledger, preflight, gates, submitters — needs only the standard library plus a file lock. Everything heavier is optional and imported at the point of use, so `preflight` can refuse a submission on a machine with no @@ -57,6 +81,7 @@ Store credentials once; they never enter the agent's environment: python -m tools.jobs credential set hf_token python -m tools.jobs credential set openrouter_key python -m tools.jobs credential set voyage_key +python -m tools.jobs credential set context7_key # optional; raises rate limits ``` ## Run @@ -87,7 +112,13 @@ that carry the literal next command. | `tools/jobs.py` | Hugging Face Jobs: `submit` / `status` / `collect` / `ceilings` / `credential` | | `tools/gpu.py` | the same verbs against a known SSH host | | `tools/ledger.py` | `expect` / `query` / `verdict` / `falsify` / `verify` / `reindex` | -| `tools/quota.py` | measured token and credit usage, summarised by stage | +| `tools/quota.py` | measured token and credit usage, summarised by stage, role, and project | +| `tools/budget.py` | projects and their ceilings: `new` / `use` / `status` / `raise` / `close` | +| `tools/docs.py` | is this library call current? introspection first, then Context7 | +| `tools/evolve.py` | evolutionary search as a budgeted campaign, over ShinkaEvolve | +| `tools/report.py` | `draft` / `write` / `cite` / `check` / `build` — the report and its gate | +| `tools/lab.py` | the embedded JupyterLab server (human editing surface) | +| `tools/wiki.py` | RepoWiki over `core/` and `tools/` — **human-facing only**, not an agent tool | ### Exit codes @@ -108,10 +139,17 @@ things, and the model should not have to read prose to tell them apart. | 9 | a check ran and failed | | 10 | job still running (not an error) | | 11 | configuration or credential problem | +| 12 | **gate**: project budget exceeded | + +12 is deliberately distinct from 6: "this research ran out of its allocation" is +not "the machine is out of money", and conflating them makes the wrong fix look +right. ## A full cycle ```bash +python -m tools.budget new --id proj-scaling-w2 --title "width vs depth" \ + --gpu-usd 50 --quota-tokens 5e6 --credits-usd 10 --payer hf:myorg --use --json python -m tools.preflight run --spec pipeline/spec.toml --only tests,dry_run --json python -m tools.jobs submit --spec pipeline/spec.toml --smoke --json python -m tools.ledger expect --task scaling-w2 --quantity val_loss@1e9_tokens \ @@ -122,11 +160,14 @@ python -m tools.jobs submit --spec pipeline/spec.toml --expect exp-... --json python -m tools.jobs collect run-... --json python -m tools.ledger verdict run-... --quantity val_loss@1e9_tokens \ --verdict bug --note 'lr schedule off by one step' --json +python -m tools.report draft --project proj-scaling-w2 --json # free, no model +python -m tools.report check --project proj-scaling-w2 --json # the gate ``` -Skip any of the first three and the fourth refuses, with the command you skipped -in its `fix` field. `skills/preflight/SKILL.md` documents the submission spec -format and what each check catches. +Skip any of the preflight/expectation steps and `submit` refuses, with the +command you skipped in its `fix` field. Skip the verdict and `report check` +refuses. `skills/preflight/SKILL.md` documents the submission spec format and +what each check catches. ## Layout @@ -138,30 +179,58 @@ core/ the machinery the CLIs share, so no tool can forget a rule cli.py the §8 CLI contract, implemented once jsonl.py the single locked write path to the ledgers submission.py the resolved submission and its hash - gates.py the four submit gates and the smoke carve-out + gates.py the submit gates and the smoke carve-out + budget.py the project dimension and its three ceilings ledger_store.py event-folded runs, rolling spend, staleness, derived index submit.py shared submitter machinery: record, collect, deviations + campaign.py campaigns, candidates, and the evolve-block escape check + report.py the claim and citation guarantees `report check` enforces corpus.py FTS5 + vectors + reciprocal rank fusion haiku.py funnel stages 0 and 3, via forced SDK tools - http.py Semantic Scholar, rerank, embeddings + http.py Semantic Scholar, rerank, embeddings, Context7 tools/ the CLIs -ui/ NiceGUI app and the four widgets +ui/ NiceGUI app and the widgets +config/jupyter/ the Lab server config: framing headers and overrides skills/ loaded on demand, not into the default context -ledger/ expectations.jsonl, runs.jsonl, quota.jsonl, preflight records +ledger/ expectations.jsonl, runs.jsonl, quota.jsonl, projects.jsonl, + campaigns.jsonl, candidates.jsonl, preflight records +reports// main.tex, claims.json, references.bib, the PDF evals/retrieval.jsonl the arbiter for any change to retrieval ``` ## Status -Implemented and tested: the ledger, the submission hash, all four gates, the -smoke caps, the CLI contract, the hook, the persistent kernel, and notebook -verification. `pytest` covers these — 85 tests, no network, no SDK required. +Implemented and tested: the ledger, the submission hash, every gate, the smoke +caps, the CLI contract, the hook, the persistent kernel, notebook verification, +the project dimension and its three ceilings, HF organization namespaces, +library-currency checking, the campaign loop and its budget gate, and the report +generator with all four of its rules. `pytest` covers these — 295 tests, no +network, no SDK required. Implemented but not exercised against a live service: the HF Jobs backend, the -SSH backend, Semantic Scholar, the OpenRouter reranker, Voyage embeddings, and -the two Haiku funnel stages. They are written against the documented interfaces -and fail with actionable errors rather than tracebacks, but a real credential -and a real run are what will find the mismatches. +SSH backend, Semantic Scholar, the OpenRouter reranker, Voyage embeddings, the +two Haiku funnel stages, Context7, ShinkaEvolve, and RepoWiki. They are written +against the documented interfaces and fail with actionable errors rather than +tracebacks, but a real credential and a real run are what will find the +mismatches. + +**Carried over unresolved from [HANDOFF-2 §23](HANDOFF-2.md):** + +- **Context7's REST endpoint paths were never verified.** They are configuration + (`[docs]` in `config/grad.toml`), not constants, and a 404 says so and points + at `context7.com/docs/api-guide` rather than looking like a missing library. +- **Whether ShinkaEvolve exposes a per-candidate callback** decides driver vs. + fork. `python -m tools.evolve capabilities --json` answers it against the + installed package; the driver assumes generation-boundary granularity, which + is what the budget gate is built around. +- **Whether `headless/claude` works against a Max subscription specifically** is + reported in Shinka's release notes and untested here. +- **Historical records are left as `"unassigned"`** rather than retrofitted with + a project. Cheap to change while the ledger is small. +- **Phase 2 of the campaign loop (remote evaluation) is not enabled.** + `--remote` is refused: the gate is proven locally first, because doing the + ledger work and the spend work simultaneously against live GPU jobs is how you + learn about exit 7 the hard way. Two things worth knowing before trusting them: diff --git a/agent.py b/agent.py index 6cbc03c..891b20d 100644 --- a/agent.py +++ b/agent.py @@ -30,6 +30,7 @@ import hooks from core import config as config_mod, credentials, paths, quota_log +from core.errors import EXIT_PROJECT_BUDGET BUILTIN_TOOLS = ["Read", "Write", "Edit", "Bash", "Glob", "Grep"] @@ -64,7 +65,7 @@ def build_options(cfg: Any, *, permission_mode: str | None = None) -> Any: "Stop": [sdk.HookMatcher(hooks=[hooks.stop])], } return sdk.ClaudeAgentOptions( - model=str(cfg.get("agent", "model", "claude-opus-4-5")), + model=cfg.model_for("research"), system_prompt=system_prompt(), allowed_tools=BUILTIN_TOOLS, disallowed_tools=DENIED_TOOLS, @@ -81,11 +82,20 @@ def preflight_environment() -> dict[str, Any]: so a stray export silently bills the Developer Platform instead of the subscription. It is removed here rather than warned about. """ + from core import budget # noqa: PLC0415 + removed = credentials.scrub_environment() + cfg = config_mod.load() + project_id = budget.current_project() return { "removed_env": removed, "oauth_token_present": bool(os.environ.get("CLAUDE_CODE_OAUTH_TOKEN")), "workspace": str(paths.root()), + "models": cfg.models(), + # Read from ledger/.current_project, not from the environment -- the + # scrub above is exactly why the selection is a file (§15). + "project": project_id, + "project_status": budget.status(project_id) if budget.exists(project_id) else None, "note": ( "auth should be subscription-backed; confirm with `claude /status`. " "--bare mode does not read CLAUDE_CODE_OAUTH_TOKEN, so this runs non-bare." @@ -106,9 +116,9 @@ async def run_session(prompt: str | None, *, once: bool) -> int: async with sdk.ClaudeSDKClient(options=build_options(cfg)) as client: if prompt: - await _turn(client, prompt) + ran = await _turn(client, prompt) if once: - return 0 + return 0 if ran else EXIT_PROJECT_BUDGET while True: try: # In a worker thread: a bare input() blocks the event loop, and @@ -126,7 +136,57 @@ async def run_session(prompt: str | None, *, once: bool) -> int: await _turn(client, line) -async def _turn(client: Any, prompt: str) -> None: +def check_turn_budget() -> dict[str, Any] | None: + """Refuse the *next* turn when the project is out of token allocation. + + HANDOFF-2 §15, and the honesty is the point: tokens are consumed + continuously inside a turn and there is no way to refuse mid-turn, so + **token budgets are enforced to a granularity of one turn's overrun.** This + check is our code end to end -- it depends on no SDK behaviour -- and it runs + before `query`, not after. + + Returns a refusal payload, or None to proceed. + """ + try: + from core import budget # noqa: PLC0415 + + project_id = budget.current_project() + if not project_id or not budget.exists(project_id): + return None + state = budget.status(project_id) + except Exception: # noqa: BLE001 - accounting must never strand a session + return None + + tokens = state["resources"]["quota_tokens"] + if not tokens["over"]: + return None + overrun = tokens["spent"] - float(tokens["ceiling"]) + return { + "project": project_id, + "resource": "quota_tokens", + "spent": tokens["spent"], + "ceiling": tokens["ceiling"], + "overrun": overrun, + "message": ( + f"project {project_id} has used {tokens['spent']:,} of its " + f"{int(tokens['ceiling']):,} token allocation -- {overrun:,.0f} over. " + "Refusing the next turn; the turn that crossed the ceiling was allowed to " + "finish, because there is no way to refuse mid-turn." + ), + "fix": ( + f"python -m tools.budget raise --project {project_id} " + "--quota-tokens --json" + ), + } + + +async def _turn(client: Any, prompt: str) -> bool: + """Run one turn. Returns False if the budget refused it.""" + refusal = check_turn_budget() + if refusal: + print(f"\n[grad] {refusal['message']}\n[grad] fix: {refusal['fix']}", file=sys.stderr) + return False + await client.query(prompt) async for message in client.receive_response(): text = _text_of(message) @@ -134,8 +194,11 @@ async def _turn(client: Any, prompt: str) -> None: print(text, end="", flush=True) usage = getattr(message, "usage", None) if usage is not None: - quota_log.from_sdk_usage(quota_log.STAGE_MAIN, usage) + quota_log.from_sdk_usage( + quota_log.STAGE_MAIN, usage, model=None, role="research" + ) print() + return True def _text_of(message: Any) -> str: diff --git a/config/grad.toml b/config/grad.toml index a57d131..00e2bb1 100644 --- a/config/grad.toml +++ b/config/grad.toml @@ -37,12 +37,27 @@ exec_timeout_s = 300 verify_timeout_s = 1800 kernel_name = "python3" +[models] +# HANDOFF-2 §16: selected by role, not scattered across sections. Five surfaces +# share six roles, and `tools/quota.py summary --by-role` reports what each one +# cost. The Claude 5 family is Fable 5 / Opus 5 / Sonnet 5; there is no Haiku 5, +# and 4.5 is the latest Haiku. +research = "claude-opus-5" # the main loop (§3) +evolve = "claude-sonnet-5" # ShinkaEvolve mutation operators (§21) +expand = "claude-haiku-4-5" # funnel stage 0 (§5) +triage = "claude-haiku-4-5" # funnel stage 3 (§5) +report = "claude-opus-5" # prose synthesis (§22) +cite = "claude-haiku-4-5" # citation resolution -- mechanical matching (§22) + [retrieval] +# rerank_model and embed_model stay here on purpose: a different provider on a +# different billing rail. Voyage costs credits, Haiku costs subscription quota, +# and the reranker is a dedicated cross-encoder that is better at pairwise +# relevance than a generative model is. Do not fold these into [models]; that is +# exactly the substitution §16 argues against. rerank_model = "voyageai/rerank-2.5" embed_model = "voyage-4" embed_dim = 1024 -triage_model = "claude-haiku-4-5" -expand_model = "claude-haiku-4-5" rrf_k = 60 candidates = 300 rerank_top = 50 @@ -52,13 +67,38 @@ min_request_interval_s = 1.1 cache_ttl_s = 604800 [agent] -model = "claude-opus-4-5" +# The main loop's model lives in [models] research. This key is still read as an +# override for one release so existing configs keep working. # Verify this empirically after any SDK upgrade: `python agent.py --probe`. # Mode names and semantics have changed between releases. permission_mode = "dontAsk" +[docs] +# HANDOFF-2 §18. These REST paths were NOT verified when this was written, which +# is why they are configuration rather than constants. If `tools.docs resolve` +# returns 404, read context7.com/docs/api-guide and correct them here -- the +# error message says so too. +base = "https://context7.com/api/v1" +resolve_path = "/search" +docs_path = "/{library_id}" +cache_ttl_s = 86400 + +[report] +# HANDOFF-2 §22. To use a conference template, drop its .sty into +# reports// and name it here; the default compiles on a stock TeX +# installation with nothing vendored. +documentclass = "article" +classoptions = "11pt" +style = "" # e.g. "neurips_2024" +bibstyle = "plainnat" + [hf] default_flavor = "a10g-small" +# Organization to run jobs under when nothing more specific applies. Resolution +# order is: --namespace, the spec's [target] namespace, the project's `payer`, +# then this, then personal. Membership is checked against `whoami` before every +# submission -- org membership changing is precisely the case worth catching. +# namespace = "myorg" [hf.flavor_rates] # `collect` prices the platform's own start/end timestamps against this table. diff --git a/config/jupyter/jupyter_server_config.py b/config/jupyter/jupyter_server_config.py new file mode 100644 index 0000000..e00e389 --- /dev/null +++ b/config/jupyter/jupyter_server_config.py @@ -0,0 +1,59 @@ +"""JupyterLab server configuration for the embedded Lab tab (HANDOFF-2 §19). + +Two settings here are the difference between "the tab works" and "an afternoon +of wondering why the iframe is blank". + +**Framing.** JupyterLab ships `X-Frame-Options: SAMEORIGIN` and a +`frame-ancestors 'self'` CSP, both of which block embedding from another origin. +Lab runs on its own port, so the Grad app *is* another origin. The headers below +override that for the app's origin only -- not for `*`, which would let any page +frame a server that can execute code as this user. + +**Origin checks.** A cross-origin websocket from the app's port is rejected by +default, which breaks the kernel connection while the page still renders. That +is the confusing half of the failure, so it is set alongside the headers rather +than discovered later. + +Nothing here weakens the token: Lab is still started with one, on 127.0.0.1 +only, by `tools/lab.py`. +""" + +import os + +# The Grad UI's origin. `tools/lab.py` sets this when it launches the server, so +# the two cannot drift apart; the default matches ui/app.py's default port. +_APP_ORIGIN = os.environ.get("GRAD_UI_ORIGIN", "http://127.0.0.1:8080") +_LAB_PORT = os.environ.get("GRAD_LAB_PORT", "8889") + +c = get_config() # noqa: F821 - injected by the Jupyter config loader + +c.ServerApp.ip = "127.0.0.1" +c.ServerApp.open_browser = False +c.ServerApp.port = int(_LAB_PORT) + +# Framing, scoped to the app's origin rather than to `*`. +c.ServerApp.tornado_settings = { + "headers": { + "Content-Security-Policy": ( + f"frame-ancestors 'self' {_APP_ORIGIN} http://localhost:{_APP_ORIGIN.rsplit(':', 1)[-1]}" + ), + }, + # X-Frame-Options has no origin list -- it is SAMEORIGIN or DENY -- so it is + # removed and the CSP above does the work. Leaving it set would win over the + # CSP in browsers that honour both. + "xheaders": False, +} +c.ServerApp.allow_origin = _APP_ORIGIN +c.ServerApp.allow_credentials = True + +# The websocket the kernel connection rides on. +c.ServerApp.allow_remote_access = False +c.ServerApp.disable_check_xsrf = False + +# Kernel ownership discipline (§19): Lab has its own kernel manager and +# `tools/nb.py` spawns its own detached kernels. Two owners over one notebook +# reproduces the "works in the kernel that grew it" failure `nb verify` exists +# to catch, so the rule is unchanged -- anything edited here passes +# `python -m tools.nb verify --json` before it is cited in notes/ or +# referenced from a ledger entry. The Verify button in the Notebooks tab is +# the enforcement surface for that. diff --git a/config/jupyter/overrides.json b/config/jupyter/overrides.json new file mode 100644 index 0000000..fdd4b62 --- /dev/null +++ b/config/jupyter/overrides.json @@ -0,0 +1,17 @@ +{ + "@jupyterlab/apputils-extension:themes": { + "theme": "JupyterLab Dark" + }, + "@jupyterlab/notebook-extension:tracker": { + "recordTiming": true + }, + "@jupyterlab/docmanager-extension:plugin": { + "autosave": true, + "autosaveInterval": 30 + }, + "@jupyterlab/fileeditor-extension:plugin": { + "editorConfig": { + "rulers": [88] + } + } +} diff --git a/core/budget.py b/core/budget.py new file mode 100644 index 0000000..1ed666e --- /dev/null +++ b/core/budget.py @@ -0,0 +1,428 @@ +"""The project dimension and its ceilings (HANDOFF-2 §15). + + "Three separate requirements turned out to want the same thing -- a + dimension carried on every cost-bearing record." + +§17 needs to know which account paid for an HF job, §21 needs to bound a +campaign made of many runs, and the user wants a budget for a piece of research. +Those are three faces of one abstraction, so it is built once: every run record, +every `quota.jsonl` entry, and every credit spend carries a `project` id. + +Three resources, and the enforcement quality differs by resource in a way that +is structural rather than a shortcoming: + + * **GPU dollars** -- clean. Submission is a discrete, gateable event. + * **Credits** -- clean at the same boundary, and measured continuously. + * **Subscription tokens** -- granular to *one turn's overrun*. Tokens are + consumed continuously inside a turn and there is no way to refuse mid-turn, + so `agent.py` checks remaining allocation before issuing the next turn and + `hooks.py` denies cost-bearing Bash once a project is over. That is the + honest statement, and it belongs in `--help`, not only in the handoff. + +A second honesty note, kept next to the code rather than in a document: the real +Anthropic limits are rolling windows the SDK does not expose as a remaining +balance. A token ceiling here is **a proxy the user controls**, not a mirror of +Anthropic's limit. The meter must not imply otherwise. + +`raise` appends an event rather than mutating the record: a ceiling that can be +edited invisibly is not a ceiling. Same argument as §7's append-only ledger. +""" + +from __future__ import annotations + +import datetime as _dt +import re +from pathlib import Path +from typing import Any + +from core import jsonl, paths +from core.errors import EXIT_PROJECT_BUDGET, GateRefusal, NotFound, UsageError + +# --- record types (folded like runs.jsonl) ---------------------------------- +T_PROJECT = "project" +T_PROJECT_RAISED = "project_budget_raised" +T_PROJECT_CLOSED = "project_closed" + +# Records that predate the dimension fold as this, per §23 item 6: specified as +# left alone rather than retrofitted, and cheap to change while the ledger is +# small. +UNASSIGNED = "unassigned" + +# The three resources. Names are the keys in a project's `budget` table and in +# every spend report, so a caller never has to guess which spelling is current. +RESOURCES = ("gpu_usd", "quota_tokens", "credits_usd") + +_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$") + + +def projects_path() -> Path: + return paths.ledger_dir() / "projects.jsonl" + + +def current_project_path() -> Path: + return paths.ledger_dir() / ".current_project" + + +# --------------------------------------------------------------------------- +# selection +# --------------------------------------------------------------------------- +def current_project() -> str | None: + """The project selected by `tools.budget use`. + + Deliberately a file rather than an environment variable: + `credentials.scrub_environment()` strips the agent's environment at startup, + and a selection mechanism that the agent's own startup deletes is a bug + waiting to happen. + """ + path = current_project_path() + if not path.exists(): + return None + value = path.read_text(encoding="utf-8").strip() + return value or None + + +def set_current(project_id: str | None) -> None: + path = current_project_path() + path.parent.mkdir(parents=True, exist_ok=True) + if project_id is None: + path.unlink(missing_ok=True) + return + path.write_text(project_id, encoding="utf-8") + + +def resolve(explicit: str | None = None) -> str | None: + """`--project` beats the selection file. Neither is an error: work outside + a project is allowed and lands under `unassigned`.""" + return explicit or current_project() + + +def resolve_or_fail(explicit: str | None, *, what: str) -> str: + """For the commands that genuinely cannot proceed without one.""" + project_id = resolve(explicit) + if not project_id: + raise UsageError( + f"{what} needs a project: it is what the budget is charged against", + fix="python -m tools.budget use --json # or pass --project ", + ) + return project_id + + +# --------------------------------------------------------------------------- +# records +# --------------------------------------------------------------------------- +def events() -> list[dict[str, Any]]: + return jsonl.read(projects_path()) + + +def projects() -> dict[str, dict[str, Any]]: + """Every project, folded from its events, insertion-ordered by creation.""" + folded: dict[str, dict[str, Any]] = {} + for rec in events(): + pid = rec.get("id") + if not pid: + continue + kind = rec.get("type") + if kind == T_PROJECT: + folded[pid] = { + "id": pid, + "created_at": rec.get("created_at"), + "title": rec.get("title", ""), + "payer": rec.get("payer"), + "budget": dict(rec.get("budget") or {}), + "status": rec.get("status", "open"), + "raises": [], + } + elif pid in folded and kind == T_PROJECT_RAISED: + node = folded[pid] + for resource, value in (rec.get("budget") or {}).items(): + node["budget"][resource] = value + node["raises"].append( + {"at": rec.get("at"), "budget": rec.get("budget"), "reason": rec.get("reason")} + ) + elif pid in folded and kind == T_PROJECT_CLOSED: + folded[pid]["status"] = "closed" + folded[pid]["closed_at"] = rec.get("at") + return folded + + +def project(project_id: str) -> dict[str, Any]: + found = projects().get(project_id) + if not found: + raise NotFound( + f"project {project_id!r} does not exist", + fix="python -m tools.budget list --json # or `new` to create one", + ) + return found + + +def exists(project_id: str | None) -> bool: + return bool(project_id) and project_id in projects() + + +def create( + project_id: str, + *, + title: str, + budget: dict[str, float], + payer: str | None = None, +) -> dict[str, Any]: + if not _ID_RE.match(project_id): + raise UsageError( + f"project id {project_id!r} must be a short slug of letters, digits, dot, dash, underscore", + fix="python -m tools.budget new --id proj-scaling-w2 --title '...' --json", + ) + if project_id in projects(): + raise UsageError( + f"project {project_id!r} already exists", + fix=f"python -m tools.budget status --project {project_id} --json", + ) + record = { + "type": T_PROJECT, + "id": project_id, + "created_at": now_iso(), + "title": title, + "payer": payer, + "budget": {k: float(v) for k, v in budget.items() if v is not None}, + "status": "open", + } + jsonl.append(projects_path(), record) + return record + + +def raise_ceiling(project_id: str, *, budget: dict[str, float], reason: str = "") -> dict[str, Any]: + """Append a raise event. Never mutates the original record. + + Lowering is permitted and recorded the same way -- the point is not that a + ceiling only goes up, it is that it never moves invisibly. + """ + current = project(project_id) + changed = {k: float(v) for k, v in budget.items() if v is not None} + if not changed: + raise UsageError( + "nothing to change", + fix="python -m tools.budget raise --gpu-usd 75 --json", + ) + unknown = [k for k in changed if k not in RESOURCES] + if unknown: + raise UsageError( + f"unknown resource(s): {', '.join(unknown)}", + fix=f"resources are: {', '.join(RESOURCES)}", + ) + record = { + "type": T_PROJECT_RAISED, + "id": project_id, + "at": now_iso(), + "budget": changed, + "previous": {k: current["budget"].get(k) for k in changed}, + "reason": reason, + } + jsonl.append(projects_path(), record) + return record + + +def close(project_id: str) -> dict[str, Any]: + project(project_id) + record = {"type": T_PROJECT_CLOSED, "id": project_id, "at": now_iso()} + jsonl.append(projects_path(), record) + if current_project() == project_id: + set_current(None) + return record + + +def now_iso() -> str: + return _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds") + + +# --------------------------------------------------------------------------- +# spend +# --------------------------------------------------------------------------- +def project_of(record: Any) -> str: + """The project a cost-bearing record belongs to. + + Unknown-project records fold as `unassigned` so existing ledgers keep + loading -- this is an additive schema change, not a migration. + """ + get = record.get if hasattr(record, "get") else (lambda k, d=None: d) + return str(get("project") or UNASSIGNED) + + +def spend(project_id: str) -> dict[str, Any]: + """What this project has consumed, per resource. + + GPU dollars count in-flight runs at their estimates, for the same reason + §6's global ceiling does: a job that has not been collected yet is not free. + """ + from core import ledger_store as ls, quota_log + + gpu_usd = 0.0 + in_flight_usd = 0.0 + runs: list[str] = [] + for r in ls.runs(): + if project_of(r.data) != project_id: + continue + runs.append(r.id) + amount = r.cost_for_ceiling() + gpu_usd += amount + if not (r.collected and r.get("cost_usd_actual") is not None): + in_flight_usd += amount + + quota_tokens = 0 + credits_usd = 0.0 + for entry in quota_log.entries(): + if project_of(entry) != project_id: + continue + quota_tokens += int(entry.get("input_tokens", 0) or 0) + int(entry.get("output_tokens", 0) or 0) + credits_usd += float(entry.get("credits_usd", 0.0) or 0.0) + + # Campaign candidates consume real resources and live outside runs.jsonl by + # design (§23 item 4). Leaving them out here would make a campaign invisible + # to the ceiling that is supposed to bound it -- the re-check before each + # generation would compare against a spend figure that never moved, and the + # gate would be decoration. + candidate_usd = 0.0 + candidates = 0 + try: + from core import campaign as _campaign # noqa: PLC0415 - avoids an import cycle + + owned = { + cid for cid, c in _campaign.campaigns().items() + if (c.get("project") or UNASSIGNED) == project_id + } + for row in _campaign.candidates(): + if row.get("campaign") in owned: + candidate_usd += float(row.get("cost_usd") or 0.0) + candidates += 1 + except Exception: # noqa: BLE001 - a missing campaign ledger is not an error + pass + + return { + "project": project_id, + "gpu_usd": round(gpu_usd + candidate_usd, 4), + "gpu_in_flight_usd": round(in_flight_usd, 4), + "gpu_candidate_usd": round(candidate_usd, 4), + "quota_tokens": quota_tokens, + "credits_usd": round(credits_usd, 6), + "runs": runs, + "candidates": candidates, + } + + +def status(project_id: str) -> dict[str, Any]: + """Ceilings, spend, and remaining, per resource. + + A resource with no ceiling reports `remaining: null` rather than infinity: + "unbounded" and "a very large number" should not look the same in a meter. + """ + proj = project(project_id) + used = spend(project_id) + resources: dict[str, Any] = {} + for resource in RESOURCES: + ceiling = proj["budget"].get(resource) + consumed = used[resource] + resources[resource] = { + "ceiling": ceiling, + "spent": consumed, + "remaining": None if ceiling is None else round(float(ceiling) - consumed, 6), + "fraction": ( + None if not ceiling else min(1.0, consumed / float(ceiling)) + ), + "over": bool(ceiling is not None and consumed > float(ceiling)), + } + return { + "project": proj["id"], + "title": proj["title"], + "payer": proj["payer"], + "status": proj["status"], + "resources": resources, + "gpu_in_flight_usd": used["gpu_in_flight_usd"], + "run_count": len(used["runs"]), + "over_budget": [r for r, d in resources.items() if d["over"]], + # The meter must not imply a fuel gauge. Anthropic exposes no remaining + # balance and the real limits are rolling windows, so a token ceiling is + # a proxy the user controls. + "quota_tokens_authoritative": False, + } + + +def over_budget(project_id: str | None) -> list[str]: + """Which resources are over. Empty for no project and for unknown ids -- + a missing project is not an overrun, and the caller decides whether it is + an error.""" + if not project_id or not exists(project_id): + return [] + return status(project_id)["over_budget"] + + +# --------------------------------------------------------------------------- +# the gate +# --------------------------------------------------------------------------- +def check( + project_id: str | None, + *, + gpu_usd: float = 0.0, + quota_tokens: int = 0, + credits_usd: float = 0.0, + what: str = "this", +) -> dict[str, Any] | None: + """Refuse if the projected spend leaves the project over its allocation. + + Exit code **12**, distinct from 6 (the global spend ceiling), so "this + research ran out of its allocation" is never confused with "the machine is + out of money". Returns None when there is no project or no ceiling: an + unbudgeted project is still tracked, just not bounded. + """ + if not project_id or not exists(project_id): + return None + state = status(project_id) + proposed = {"gpu_usd": float(gpu_usd), "quota_tokens": int(quota_tokens), "credits_usd": float(credits_usd)} + for resource, add in proposed.items(): + node = state["resources"][resource] + ceiling = node["ceiling"] + if ceiling is None: + continue + projected = node["spent"] + add + if projected > float(ceiling): + raise GateRefusal( + "project_budget", + ( + f"project {project_id!r} would spend {_fmt(resource, projected)} of " + f"{_fmt(resource, float(ceiling))} on {resource} " + f"({_fmt(resource, node['spent'])} already spent" + + (f" + {_fmt(resource, add)} for {what}" if add else "") + + "); this research has run out of its allocation, " + "which is not the same as the machine being out of money" + ), + EXIT_PROJECT_BUDGET, + fix=( + f"python -m tools.budget raise --project {project_id} " + f"--{resource.replace('_', '-')} --json " + "# deliberate, logged, never silent" + ), + detail={"project": project_id, "resource": resource, + "spent": node["spent"], "ceiling": ceiling, "proposed": add}, + ) + return state + + +def _fmt(resource: str, value: float) -> str: + if resource.endswith("_usd"): + return f"${value:.2f}" + return f"{int(value):,}" + + +# --------------------------------------------------------------------------- +# payer (§17) +# --------------------------------------------------------------------------- +def hf_namespace(project_id: str | None) -> str | None: + """The HF namespace a project's costs are attributed to. + + `payer` lives on the project rather than being invented per submission, so + the org attribution in §17 is a consequence of choosing a project rather + than a separate flag to forget. + """ + if not project_id or not exists(project_id): + return None + payer = project(project_id).get("payer") or "" + if payer.startswith("hf:"): + return payer[3:] or None + return None diff --git a/core/campaign.py b/core/campaign.py new file mode 100644 index 0000000..01de1c3 --- /dev/null +++ b/core/campaign.py @@ -0,0 +1,284 @@ +"""Campaigns and candidates (HANDOFF-2 §21). + +An evolutionary campaign collides with four things this system already +believes, and this module is where three of those collisions are resolved. The +fourth (Goodhart) is resolved in `tools/evolve.py`, at promotion time. + +**1. The expectation gate is 1:1 with a run; evolution is 1:N.** You cannot +pre-register a prediction per candidate. So the **campaign** is the unit of +prediction -- "the evolved variant beats baseline X on metric Y by >= Z" -- and +candidate evaluations are recorded as sub-runs exempt from the per-run +expectation gate. That is arguably more faithful to §7 than the current design, +and it is exactly the relational shape `prompts/system.md` already prefers. + +**2. The ledger would drown.** §23 item 4 asks how granular sub-runs should be: +one record per candidate is honest but a 100-generation campaign is thousands of +rows in `runs.jsonl`. Resolved as specified there -- a separate +`ledger/candidates.jsonl`, folded into the campaign record, with **only promoted +candidates entering `runs.jsonl`** through the normal submit path. + +**3. Every mutation invalidates the preflight hash -- correctly, and +expensively.** `Submission.hash()` covers the entrypoint's import graph, so each +candidate needs a fresh preflight, and `smoke` is a *paid remote job*. Naively +that doubles per-candidate cost. Resolved by `escaped_evolve_block()` below: the +`EVOLVE-BLOCK` markers make "did the mutation stay inside the mutable region" +mechanically checkable, so candidates run `--only tests,dry_run` (both local, +both fast) and smoke is required once per campaign at the baseline, and again +only when a mutation escapes. +""" + +from __future__ import annotations + +import datetime as _dt +from pathlib import Path +from typing import Any + +from core import jsonl, paths +from core.errors import NotFound + +T_CAMPAIGN = "campaign" +T_GENERATION = "campaign_generation" +T_CAMPAIGN_CLOSED = "campaign_closed" +T_CANDIDATE = "candidate" +T_CANDIDATE_PROMOTED = "candidate_promoted" + +STATUSES = ("open", "closed", "exhausted", "failed") + +# Shinka's own markers, kept verbatim so a task directory works with the +# upstream tool unmodified. +BLOCK_START = "EVOLVE-BLOCK-START" +BLOCK_END = "EVOLVE-BLOCK-END" + + +def campaigns_path() -> Path: + return paths.ledger_dir() / "campaigns.jsonl" + + +def candidates_path() -> Path: + return paths.ledger_dir() / "candidates.jsonl" + + +def now_iso() -> str: + return _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds") + + +def new_id(prefix: str) -> str: + import secrets # noqa: PLC0415 + + stamp = _dt.datetime.now(_dt.timezone.utc).strftime("%Y%m%dT%H%M%S") + return f"{prefix}-{stamp}-{secrets.token_hex(3)}" + + +# --------------------------------------------------------------------------- +# the evolve block +# --------------------------------------------------------------------------- +def split_blocks(source: str) -> tuple[list[str], list[str]]: + """(mutable regions, everything outside them). + + A file with no markers is entirely *outside* the block, which is the + conservative reading: an unmarked file that changed is treated as an escape + and therefore requires a fresh smoke check. + """ + inside: list[str] = [] + outside: list[str] = [] + current: list[str] = [] + in_block = False + for line in source.splitlines(): + if BLOCK_START in line: + outside.extend(current) + current = [] + in_block = True + continue + if BLOCK_END in line: + inside.extend(current) + current = [] + in_block = False + continue + current.append(line) + (inside if in_block else outside).extend(current) + return inside, outside + + +def escaped_evolve_block(baseline: str, candidate: str) -> dict[str, Any]: + """Did this mutation change anything outside the mutable region? + + This is the check that keeps a campaign affordable. Inside the block, a + mutation cannot change the environment the job runs in, so the baseline's + smoke result still applies and candidates need only the two local checks. + Outside it, all bets are off and a fresh remote smoke is required. + + Whitespace-only differences outside the block do not count as an escape: + a reformatter is not an environment change, and a check that fires + spuriously is a check that gets argued around (§6). + """ + _, base_outside = split_blocks(baseline) + _, cand_outside = split_blocks(candidate) + + def normalise(lines: list[str]) -> list[str]: + return [line.rstrip() for line in lines if line.strip()] + + before, after = normalise(base_outside), normalise(cand_outside) + if before == after: + return {"escaped": False} + changed = [line for line in after if line not in before][:20] + return { + "escaped": True, + "reason": "the mutation changed code outside EVOLVE-BLOCK markers", + "changed_lines": changed, + "requires": "smoke", + } + + +def has_markers(source: str) -> bool: + return BLOCK_START in source and BLOCK_END in source + + +# --------------------------------------------------------------------------- +# campaigns +# --------------------------------------------------------------------------- +def campaign_events() -> list[dict[str, Any]]: + return jsonl.read(campaigns_path()) + + +def campaigns() -> dict[str, dict[str, Any]]: + """Every campaign, folded from its events.""" + folded: dict[str, dict[str, Any]] = {} + for rec in campaign_events(): + cid = rec.get("id") + if not cid: + continue + kind = rec.get("type") + if kind == T_CAMPAIGN: + folded[cid] = {**{k: v for k, v in rec.items() if k != "type"}, "generations_run": 0} + elif cid in folded and kind == T_GENERATION: + node = folded[cid] + node["generations_run"] = max(node["generations_run"], int(rec.get("generation", 0)) + 1) + node["last_generation_at"] = rec.get("at") + node.setdefault("generation_log", []).append( + {k: v for k, v in rec.items() if k not in ("type", "id")} + ) + elif cid in folded and kind == T_CAMPAIGN_CLOSED: + folded[cid]["status"] = rec.get("status", "closed") + folded[cid]["closed_at"] = rec.get("at") + folded[cid]["closed_reason"] = rec.get("reason") + return folded + + +def campaign(campaign_id: str) -> dict[str, Any]: + found = campaigns().get(campaign_id) + if not found: + raise NotFound( + f"campaign {campaign_id!r} does not exist", + fix="python -m tools.evolve status --json # lists known campaigns", + ) + return found + + +def append_campaign(record: dict[str, Any]) -> dict[str, Any]: + return jsonl.append(campaigns_path(), record) + + +def record_generation(campaign_id: str, generation: int, **fields: Any) -> dict[str, Any]: + return append_campaign( + {"type": T_GENERATION, "id": campaign_id, "generation": generation, "at": now_iso(), **fields} + ) + + +def close_campaign(campaign_id: str, *, status: str = "closed", reason: str = "") -> dict[str, Any]: + return append_campaign( + {"type": T_CAMPAIGN_CLOSED, "id": campaign_id, "at": now_iso(), + "status": status, "reason": reason} + ) + + +# --------------------------------------------------------------------------- +# candidates +# --------------------------------------------------------------------------- +def candidate_events() -> list[dict[str, Any]]: + return jsonl.read(candidates_path()) + + +def append_candidate(record: dict[str, Any]) -> dict[str, Any]: + return jsonl.append(candidates_path(), {"type": T_CANDIDATE, **record}) + + +def candidates(campaign_id: str | None = None) -> list[dict[str, Any]]: + """Candidate evaluations, oldest first. + + These live outside `runs.jsonl` on purpose (§23 item 4): they are sub-runs + of a campaign, exempt from the per-run expectation gate, and a thousand of + them would dominate a ledger meant to be read by hand. + """ + promoted = { + rec.get("candidate_id") + for rec in candidate_events() + if rec.get("type") == T_CANDIDATE_PROMOTED + } + out = [] + for rec in candidate_events(): + if rec.get("type") != T_CANDIDATE: + continue + if campaign_id and rec.get("campaign") != campaign_id: + continue + out.append({**rec, "promoted": rec.get("candidate_id") in promoted}) + return out + + +def promote_candidate(campaign_id: str, candidate_id: str, run_id: str) -> dict[str, Any]: + return jsonl.append( + candidates_path(), + { + "type": T_CANDIDATE_PROMOTED, + "campaign": campaign_id, + "candidate_id": candidate_id, + "run_id": run_id, + "at": now_iso(), + }, + ) + + +def top_k(campaign_id: str, k: int = 5, *, metric: str = "combined_score") -> list[dict[str, Any]]: + """The best K candidates, not the argmax. + + "`combined_score` is a Goodhart machine. A search optimising a scalar will + find the bug in the metric." Surfacing a ranked list rather than a single + winner is half the mitigation; the other half is that a winner still goes + through the normal verdict path before it counts as a result. + """ + scored = [ + c for c in candidates(campaign_id) + if isinstance((c.get("metrics") or {}).get(metric), (int, float)) + ] + scored.sort(key=lambda c: c["metrics"][metric], reverse=True) + return scored[: max(1, k)] + + +def campaign_spend(campaign_id: str) -> dict[str, Any]: + """What this campaign has actually consumed so far.""" + rows = candidates(campaign_id) + return { + "candidates": len(rows), + "evaluated": sum(1 for r in rows if r.get("metrics")), + "failed": sum(1 for r in rows if r.get("error")), + "cost_usd": round(sum(float(r.get("cost_usd") or 0.0) for r in rows), 4), + "wall_clock_s": round(sum(float(r.get("duration_s") or 0.0) for r in rows), 1), + } + + +# --------------------------------------------------------------------------- +def validate_metrics(metrics: Any) -> str | None: + """Shinka's contract: a metrics dict containing `combined_score`. + + Checked rather than assumed, because a candidate that silently reports no + score is indistinguishable from one that scored zero, and the search would + happily optimise toward whichever the fallback happened to be. + """ + if not isinstance(metrics, dict): + return "evaluate.py must print a JSON object of metrics" + if "combined_score" not in metrics: + return "the metrics object must contain `combined_score`" + if not isinstance(metrics["combined_score"], (int, float)) or isinstance( + metrics["combined_score"], bool + ): + return "`combined_score` must be a number" + return None diff --git a/core/config.py b/core/config.py index dd147a2..bee1a64 100644 --- a/core/config.py +++ b/core/config.py @@ -46,8 +46,10 @@ "rerank_model": "voyageai/rerank-2.5", "embed_model": "voyage-4", "embed_dim": 1024, - "triage_model": "claude-haiku-4-5", - "expand_model": "claude-haiku-4-5", + # triage_model / expand_model moved to [models] triage / expand (§16). + # They are still *readable* here as overrides -- see LEGACY_MODEL_KEYS -- + # but they are no longer defaulted here, so [models] is the one place a + # role's default lives. "rrf_k": 60, "candidates": 300, "rerank_top": 50, @@ -56,6 +58,29 @@ "request_timeout_s": 60, "min_request_interval_s": 1.1, # unauthenticated S2 is ~1 req/s }, + # HANDOFF-2 §18. The REST paths were NOT verified during the design session, + # which is why they are configuration rather than constants: read + # context7.com/docs/api-guide and correct them here if a request 404s. + "docs": { + "base": "https://context7.com/api/v1", + "resolve_path": "/search", + "docs_path": "/{library_id}", + "request_timeout_s": 30, + "cache_ttl_s": 86400, + "min_request_interval_s": 0.5, + }, + # HANDOFF-2 §22. The handoff says "vendor NeurIPS or ICML style. Not a + # decision worth deliberating." Rather than commit a third-party .sty file + # into this repo, the class and style are configuration: drop + # `neurips_2024.sty` into reports// and set + # `documentclass = "article"` plus `style = "neurips_2024"` here. The default + # compiles with a stock TeX installation and no vendored file. + "report": { + "documentclass": "article", + "classoptions": "11pt", + "style": "", + "bibstyle": "plainnat", + }, "preflight": { "checks": ["tests", "dry_run", "smoke"], "test_command": ["pytest", "-q"], @@ -76,14 +101,38 @@ "a100-large": 4.13, }, }, + # HANDOFF-2 §16: models are selected by *role*, not scattered across + # [agent] and [retrieval]. Five surfaces, six roles; the rerank and embed + # models deliberately stay in [retrieval] because they are a different + # provider on a different billing rail, and folding them in here invites + # the Voyage-for-Haiku substitution §16 argues against. + "models": { + "research": "claude-opus-5", # the main loop (§3) + "evolve": "claude-sonnet-5", # ShinkaEvolve mutation operators (§21) + "expand": "claude-haiku-4-5", # funnel stage 0 (§5) + "triage": "claude-haiku-4-5", # funnel stage 3 (§5) + "report": "claude-opus-5", # prose synthesis (§22) + "cite": "claude-haiku-4-5", # citation resolution -- mechanical matching (§22) + }, "agent": { - "model": "claude-opus-4-5", + "model": "claude-opus-5", "permission_mode": "dontAsk", "max_turns": 0, # 0 = unbounded }, "hosts": {}, } +# One release of backwards compatibility, so an existing config keeps working. +# The old key wins over the [models] *default* but not over an explicit +# [models] entry -- see `model_for`. +LEGACY_MODEL_KEYS: dict[str, tuple[str, str]] = { + "research": ("agent", "model"), + "expand": ("retrieval", "expand_model"), + "triage": ("retrieval", "triage_model"), +} + +MODEL_ROLES = tuple(DEFAULTS["models"]) + @dataclass(frozen=True) class Host: @@ -102,6 +151,10 @@ class Host: @dataclass(frozen=True) class Config: raw: dict[str, Any] = field(default_factory=dict) + # What the file actually said, before the defaults were merged under it. + # `model_for` needs the difference: an explicit [models] entry must beat a + # legacy key, while the [models] *default* must not. + user: dict[str, Any] = field(default_factory=dict) def section(self, name: str) -> dict[str, Any]: return dict(self.raw.get(name, {})) @@ -109,6 +162,32 @@ def section(self, name: str) -> dict[str, Any]: def get(self, section: str, key: str, default: Any = None) -> Any: return self.raw.get(section, {}).get(key, default) + def model_for(self, role: str) -> str: + """The model for one role (HANDOFF-2 §16). + + Resolution: an explicit `[models] ` wins; then the legacy key the + role replaced (`[agent] model`, `[retrieval] expand_model` / + `triage_model`), readable "for one release so existing configs do not + break"; then the `[models]` default. + """ + if role not in DEFAULTS["models"]: + raise ConfigError( + f"unknown model role {role!r}", + fix=f"roles are: {', '.join(MODEL_ROLES)}", + ) + explicit = (self.user.get("models") or {}).get(role) + if explicit: + return str(explicit) + legacy = LEGACY_MODEL_KEYS.get(role) + if legacy: + value = (self.user.get(legacy[0]) or {}).get(legacy[1]) + if value: + return str(value) + return str(self.get("models", role, DEFAULTS["models"][role])) + + def models(self) -> dict[str, str]: + return {role: self.model_for(role) for role in MODEL_ROLES} + @property def hosts(self) -> dict[str, Host]: """The inventory, with malformed entries reported as ConfigError. @@ -189,7 +268,7 @@ def load(path: Path | None = None, *, reload: bool = False) -> Config: f"{path} is not valid TOML: {exc}", fix=f"fix the syntax in {path}, or delete it to fall back to defaults", ) from exc - cfg = Config(raw=_merge(DEFAULTS, user)) + cfg = Config(raw=_merge(DEFAULTS, user), user=user) _validate(cfg, path) _cache[key] = cfg return cfg @@ -215,7 +294,7 @@ def _validate(cfg: Config, path: Path) -> None: # `Config.get` subscripts the section, so `spend = "lots"` in the file would # surface as an AttributeError from inside the loop below -- a traceback that # reads like a bug in the loader rather than a typo in a TOML file. - for section in (*dict.fromkeys(s for s, _ in _NUMERIC), "hf"): + for section in (*dict.fromkeys(s for s, _ in _NUMERIC), "hf", "models"): table = cfg.raw.get(section) if table is not None and not isinstance(table, dict): raise ConfigError( @@ -236,6 +315,21 @@ def _validate(cfg: Config, path: Path) -> None: f"[{section}] {key} must not be negative", fix=f"fix {section}.{key} in {path}", ) + # A model id is a string. An integer or a list here would surface as an + # opaque SDK error on the first turn rather than as a typo in a TOML file, + # and an unknown role name is a silently ignored setting -- which is worse, + # because the model it names is never used and nothing says so. + for role, value in (cfg.raw.get("models") or {}).items(): + if role not in DEFAULTS["models"]: + raise ConfigError( + f"[models] {role} is not a model role", + fix=f"roles are: {', '.join(MODEL_ROLES)} (in {path})", + ) + if not isinstance(value, str) or not value.strip(): + raise ConfigError( + f"[models] {role} must be a model id string, not {type(value).__name__}", + fix=f'write it as {role} = "claude-opus-5" in {path}', + ) rates = cfg.get("hf", "flavor_rates", {}) if not isinstance(rates, dict): raise ConfigError( diff --git a/core/credentials.py b/core/credentials.py index 4e4634a..8c1779a 100644 --- a/core/credentials.py +++ b/core/credentials.py @@ -26,6 +26,10 @@ OPENROUTER_KEY = "openrouter_key" VOYAGE_KEY = "voyage_key" S2_KEY = "s2_api_key" +# The fifth entry (HANDOFF-2 §18). Free from Context7's dashboard; raises rate +# limits rather than unlocking anything, so `tools/docs.py` treats it as +# optional and says so when it is missing. +CONTEXT7_KEY = "context7_key" def _keyring() -> Any: @@ -93,7 +97,10 @@ def present(name: str) -> bool: def status() -> dict[str, bool]: """Which credentials exist. Values are never returned.""" - return {n: present(n) for n in (HF_TOKEN, OPENROUTER_KEY, VOYAGE_KEY, S2_KEY)} + return { + n: present(n) + for n in (HF_TOKEN, OPENROUTER_KEY, VOYAGE_KEY, S2_KEY, CONTEXT7_KEY) + } def _env_fallback_allowed() -> bool: @@ -115,6 +122,7 @@ def scrub_environment() -> list[str]: "HUGGING_FACE_HUB_TOKEN", "OPENROUTER_API_KEY", "VOYAGE_API_KEY", + "CONTEXT7_API_KEY", # The GRAD_* fallbacks too. They exist for CI and first-run bootstrap, # where no agent is running; leaving them in place under the agent would # hand it exactly the environment-resident credentials §9 argues must @@ -124,6 +132,7 @@ def scrub_environment() -> list[str]: f"GRAD_{OPENROUTER_KEY.upper()}", f"GRAD_{VOYAGE_KEY.upper()}", f"GRAD_{S2_KEY.upper()}", + f"GRAD_{CONTEXT7_KEY.upper()}", ): if os.environ.pop(var, None) is not None: removed.append(var) diff --git a/core/errors.py b/core/errors.py index fd6d89f..1893b81 100644 --- a/core/errors.py +++ b/core/errors.py @@ -24,8 +24,12 @@ EXIT_CHECK_FAILED = 9 # a check ran and reported failure (preflight, nb verify) EXIT_RUNNING = 10 # not an error: the job is still in flight EXIT_CONFIG = 11 # missing credential, unknown host, malformed config +# Distinct from 6 on purpose (HANDOFF-2 §15): "this research ran out of its +# allocation" is not "the machine is out of money", and conflating them makes +# the wrong fix look right. +EXIT_PROJECT_BUDGET = 12 # gate: a project's own budget is exhausted -GATE_CODES = {EXIT_PREFLIGHT, EXIT_EXPECTATION, EXIT_SPEND, EXIT_STALE_RUN} +GATE_CODES = {EXIT_PREFLIGHT, EXIT_EXPECTATION, EXIT_SPEND, EXIT_STALE_RUN, EXIT_PROJECT_BUDGET} EXIT_MEANINGS = { EXIT_OK: "ok", @@ -40,6 +44,7 @@ EXIT_CHECK_FAILED: "a check failed", EXIT_RUNNING: "job still running", EXIT_CONFIG: "configuration or credential problem", + EXIT_PROJECT_BUDGET: "gate refusal: project budget exceeded", } diff --git a/core/gates.py b/core/gates.py index 0d898ad..e4b80f9 100644 --- a/core/gates.py +++ b/core/gates.py @@ -15,7 +15,7 @@ import datetime as _dt from typing import Any -from core import jsonl, ledger_store as ls, paths +from core import budget as budget_mod, jsonl, ledger_store as ls, paths from core.config import Config from core.errors import ( EXIT_EXPECTATION, @@ -140,6 +140,21 @@ def check_spend(estimate_usd: float, cfg: Config, *, now: _dt.datetime | None = return {"rolling": rolling, "projected_usd": round(projected, 4), "monthly_usd": monthly} +# --------------------------------------------------------------------------- +# gate 3b: the project's own allocation (HANDOFF-2 §15) +# --------------------------------------------------------------------------- +def check_project_spend(project_id: str | None, estimate_usd: float) -> dict[str, Any] | None: + """Alongside the global ceiling, not instead of it. + + Submission is a discrete, gateable event, so GPU dollars are the resource + this enforces cleanly. It raises with exit **12**, not 6: an organisation's + budget and the machine's budget are separate allocations, and telling them + apart is what stops "raise monthly_usd" being the reflex fix for the wrong + problem. + """ + return budget_mod.check(project_id, gpu_usd=estimate_usd, what="this job") + + # --------------------------------------------------------------------------- # gate 4: no stale uncollected run # --------------------------------------------------------------------------- @@ -169,23 +184,44 @@ def check_submit( cfg: Config, *, estimate_usd: float | None = None, + project: str | None = None, now: _dt.datetime | None = None, ) -> dict[str, Any]: """Run every gate. Raises `GateRefusal` on the first that refuses. Order is cheapest-and-most-actionable first: a missing preflight is the most - common refusal and the easiest to fix. + common refusal and the easiest to fix. The project ceiling runs immediately + after the global one -- both are spend gates, and a caller that has blown + both should hear about the machine's ceiling first, since that is the one + that stops every other project too. """ estimate = sub.estimated_cost_usd() if estimate_usd is None else estimate_usd record = check_preflight(sub, cfg) expectation = check_expectation(expectation_id, sub) spend = check_spend(estimate, cfg, now=now) + project_state = check_project_spend(project, estimate) check_stale(cfg, now=now) return { "submission_hash": sub.hash(), "preflight": {"checks": list(record.get("checks", {})), "verified_at": record.get("verified_at")}, "expectation": {"id": expectation.get("id"), "quantity": expectation.get("quantity")}, "spend": spend, + "project": project, + # Current state *plus* the projection, because "you have $50 left" and + # "you will have $48 left after this" answer different questions and the + # caller is about to spend. + "project_budget": ( + None + if project_state is None + else { + **project_state["resources"]["gpu_usd"], + "projected_remaining": ( + None + if project_state["resources"]["gpu_usd"]["remaining"] is None + else round(project_state["resources"]["gpu_usd"]["remaining"] - estimate, 4) + ), + } + ), "estimate_usd": estimate, } diff --git a/core/haiku.py b/core/haiku.py index ad8d14c..50b6156 100644 --- a/core/haiku.py +++ b/core/haiku.py @@ -117,6 +117,7 @@ async def _call( system_prompt: str, user_prompt: str, model: str, + role: str, log_name: str, ) -> dict[str, Any]: sdk = _sdk() @@ -149,7 +150,7 @@ async def _submit(args: dict[str, Any]) -> dict[str, Any]: usage = getattr(message, "usage", None) or usage quota_log.from_sdk_usage( - stage, usage, model=model, + stage, usage, model=model, role=role, detail={"tool": tool_name, "captured": bool(captured)}, ) _log_io(log_name, stage, system_prompt, user_prompt, transcript, captured) @@ -238,6 +239,7 @@ def expand(question: str, *, model: str, log_name: str) -> dict[str, Any]: system_prompt=EXPAND_PROMPT, user_prompt=f"Research question:\n\n{question}", model=model, + role="expand", log_name=log_name, ) ) @@ -282,6 +284,7 @@ def triage(question: str, candidates: list[dict[str, Any]], *, model: str, log_n system_prompt=TRIAGE_PROMPT, user_prompt=f"Research question:\n\n{question}\n\nCandidates:\n\n{listing}", model=model, + role="triage", log_name=log_name, ) ) diff --git a/core/http.py b/core/http.py index f8e9400..4e1c3ee 100644 --- a/core/http.py +++ b/core/http.py @@ -171,6 +171,134 @@ def neighbours(self, paper_id: str, *, direction: str = "citations", limit: int return out +# --------------------------------------------------------------------------- +# Context7 (HANDOFF-2 §18) -- what is *current*, as opposed to what is installed +# --------------------------------------------------------------------------- +class Context7: + """Library documentation over plain HTTP. + + This is the second instance of an existing pattern, not a new one: §5 + already reaches Asta's MCP endpoint over streamable HTTP "without adopting + MCP as an architecture", and the same applies here. `tools/docs.py` wraps it + rather than allowlisting the official `ctx7` CLI, because the `--json` / + exit-code / `fix`-field contract is what makes a tool legible to the model + (§8) and `ctx7` does not have it -- and because this is where the credential + fetch and the cache live. Documentation lookups repeat heavily, and + `core/http.py` already has the TTL cache and the rate limiter. + + **The endpoint paths were not verified during the design session.** They are + configurable for exactly that reason: read `context7.com/docs/api-guide` and + fix `[docs] resolve_path` / `docs_path` in config/grad.toml rather than + editing code. A 404 here reports the path it tried, so the mismatch names + itself. + """ + + def __init__(self, cfg: Config) -> None: + self.base = str(cfg.get("docs", "base", "https://context7.com/api/v1")).rstrip("/") + self.resolve_path = str(cfg.get("docs", "resolve_path", "/search")) + self.docs_path = str(cfg.get("docs", "docs_path", "/{library_id}")) + self.timeout = float(cfg.get("docs", "request_timeout_s", 30)) + self.ttl = float(cfg.get("docs", "cache_ttl_s", 86400)) + self.interval = float(cfg.get("docs", "min_request_interval_s", 0.5)) + # Free from their dashboard, and it raises rate limits rather than + # unlocking anything -- so its absence is a note, not an error. + self.key = credentials.get(credentials.CONTEXT7_KEY, required=False) + + @property + def authenticated(self) -> bool: + return bool(self.key) + + def _get(self, path: str, params: dict[str, Any]) -> Any: + url = f"{self.base}{path}" + key = f"ctx7:{url}:{json.dumps(params, sort_keys=True)}:{bool(self.key)}" + hit = _cached(key, self.ttl) + if hit is not None: + return hit + _throttle("context7", self.interval) + httpx = _httpx() + headers = {"Accept": "application/json"} + if self.key: + headers["Authorization"] = f"Bearer {self.key}" + try: + resp = httpx.get(url, params=params, headers=headers, timeout=self.timeout) + except Exception as exc: # noqa: BLE001 + raise UpstreamError( + f"Context7 request failed: {exc}", + fix="retry, or run `python -m tools.docs check ` which works offline", + ) from exc + if resp.status_code == 401: + raise UpstreamError( + "Context7 rejected the credential", + fix=f"python -m tools.jobs credential set {credentials.CONTEXT7_KEY}", + ) + if resp.status_code == 429: + raise UpstreamError( + "Context7 rate-limited the request", + fix=( + "wait, or store a free key to raise the limit: " + f"python -m tools.jobs credential set {credentials.CONTEXT7_KEY}" + ), + ) + if resp.status_code == 404: + raise UpstreamError( + f"Context7 returned 404 for {url}", + fix=( + "the REST paths were not verified when this was written -- read " + "context7.com/docs/api-guide and set [docs] resolve_path / docs_path " + "in config/grad.toml" + ), + ) + if resp.status_code >= 400: + raise UpstreamError( + f"Context7 returned {resp.status_code}: {resp.text[:200]}", + fix="check the library id and the query", + ) + try: + data = resp.json() + except Exception: # noqa: BLE001 - the docs endpoint may serve text + data = {"text": resp.text} + _store(key, data) + return data + + def resolve(self, name: str) -> list[dict[str, Any]]: + """Library name -> candidate Context7 library ids. + + The MCP tool this mirrors is `resolve-library-id`; note that the docs + tool is `query-docs`, and `get-library-docs` in older material is stale. + """ + data = self._get(self.resolve_path, {"query": name}) + results = data.get("results") if isinstance(data, dict) else data + out = [] + for item in results or []: + if not isinstance(item, dict): + continue + out.append( + { + "library_id": item.get("id") or item.get("libraryId") or item.get("settings", {}).get("project"), + "title": item.get("title") or item.get("name"), + "description": item.get("description", ""), + "trust_score": item.get("trustScore") or item.get("trust_score"), + "snippets": item.get("totalSnippets") or item.get("snippets"), + "versions": item.get("versions") or [], + } + ) + return [o for o in out if o["library_id"]] + + def docs(self, library_id: str, query: str, *, tokens: int = 5000) -> dict[str, Any]: + """Documentation for a library, narrowed by a topic query.""" + path = self.docs_path.format(library_id=library_id.lstrip("/")) + data = self._get(path, {"topic": query, "tokens": tokens, "type": "json"}) + if isinstance(data, dict) and "text" in data and len(data) == 1: + return {"library_id": library_id, "query": query, "text": data["text"]} + snippets = data.get("snippets") if isinstance(data, dict) else None + return { + "library_id": library_id, + "query": query, + "snippets": snippets or [], + "raw": None if snippets else data, + } + + # --------------------------------------------------------------------------- # OpenRouter rerank (credits, not quota) # --------------------------------------------------------------------------- diff --git a/core/ledger_store.py b/core/ledger_store.py index a9272b1..cb92e37 100644 --- a/core/ledger_store.py +++ b/core/ledger_store.py @@ -169,6 +169,13 @@ def collected(self) -> bool: def is_smoke(self) -> bool: return bool(self.data.get("smoke")) + @property + def project(self) -> str: + """HANDOFF-2 §15. Records written before the dimension existed fold as + `unassigned`, which is what keeps this an additive schema change rather + than a migration.""" + return str(self.data.get("project") or "unassigned") + def cost_for_ceiling(self) -> float: """Actual once collected, estimate while in flight. @@ -307,14 +314,23 @@ def stale_runs(*, cfg: Any = None, now: _dt.datetime | None = None) -> list[Run] return [r for r in in_flight() if is_stale(r, cfg=cfg, now=now)] -def rolling_spend(window_days: int = 30, *, now: _dt.datetime | None = None) -> dict[str, Any]: - """Rolling total: actuals for collected runs, estimates for in-flight ones.""" +def rolling_spend( + window_days: int = 30, *, now: _dt.datetime | None = None, project: str | None = None +) -> dict[str, Any]: + """Rolling total: actuals for collected runs, estimates for in-flight ones. + + `project` narrows it to one allocation (§15). The unfiltered total is still + what the global ceiling compares against -- a project ceiling is an extra + bound, never a replacement for the machine's. + """ now = now or _dt.datetime.now(_dt.timezone.utc) cutoff = now - _dt.timedelta(days=window_days) actual = 0.0 estimated = 0.0 counted: list[dict[str, Any]] = [] for r in runs(): + if project and r.project != project: + continue submitted = parse_iso(r.get("submitted_at")) if submitted and submitted < cutoff: continue @@ -325,9 +341,12 @@ def rolling_spend(window_days: int = 30, *, now: _dt.datetime | None = None) -> else: estimated += amount basis = "estimate" - counted.append({"run_id": r.id, "usd": amount, "basis": basis, "smoke": r.is_smoke}) + counted.append( + {"run_id": r.id, "usd": amount, "basis": basis, "smoke": r.is_smoke, "project": r.project} + ) return { "window_days": window_days, + "project": project, "total_usd": round(actual + estimated, 4), "actual_usd": round(actual, 4), "in_flight_usd": round(estimated, 4), @@ -348,7 +367,8 @@ def rolling_spend(window_days: int = 30, *, now: _dt.datetime | None = None) -> id TEXT PRIMARY KEY, task TEXT, status TEXT, smoke INTEGER, submitted_at TEXT, collected_at TEXT, platform TEXT, target TEXT, submission_hash TEXT, expectation_id TEXT, - estimate_usd REAL, cost_usd_actual REAL, results_json TEXT + estimate_usd REAL, cost_usd_actual REAL, results_json TEXT, + project TEXT ); CREATE TABLE IF NOT EXISTS deviations ( run_id TEXT, expectation_id TEXT, quantity TEXT, @@ -356,6 +376,7 @@ def rolling_spend(window_days: int = 30, *, now: _dt.datetime | None = None) -> in_range INTEGER, verdict TEXT, note TEXT ); CREATE INDEX IF NOT EXISTS idx_runs_task ON runs(task); +CREATE INDEX IF NOT EXISTS idx_runs_project ON runs(project); CREATE INDEX IF NOT EXISTS idx_dev_quantity ON deviations(quantity); CREATE INDEX IF NOT EXISTS idx_exp_quantity ON expectations(quantity); """ @@ -386,12 +407,13 @@ def rebuild_index(db_path: Any = None) -> dict[str, int]: rs = runs() for r in rs: con.execute( - "INSERT OR REPLACE INTO runs VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + "INSERT OR REPLACE INTO runs VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)", ( r.id, r.get("task"), r.status, 1 if r.is_smoke else 0, r.get("submitted_at"), r.get("collected_at"), r.get("platform"), _dumps(r.get("target")), r.get("submission_hash"), r.get("expectation_id"), r.get("estimate_usd"), r.get("cost_usd_actual"), _dumps(r.get("results")), + r.project, ), ) for dev in r.get("deviations", []) or []: diff --git a/core/quota_log.py b/core/quota_log.py index ac6633a..bfbf326 100644 --- a/core/quota_log.py +++ b/core/quota_log.py @@ -30,6 +30,8 @@ def record( stage: str, *, model: str | None = None, + role: str | None = None, + project: str | None = None, input_tokens: int = 0, output_tokens: int = 0, cache_read_tokens: int = 0, @@ -45,6 +47,12 @@ def record( 5x window, drawn on by anything going through the Agent SDK) and "credits" (OpenRouter/Voyage dollars). Conflating them is how stage 2 ends up looking expensive when it is the cheap one. + + `role` is the §16 model role, so `tools.quota summary --by-role` can answer + "what did Opus cost me this week" without inferring it from model ids. + `project` is the §15 dimension; it defaults to the current selection so a + caller cannot forget to attribute a cost, and an unselected project lands as + `unassigned` rather than as an error. """ return jsonl.append( paths.quota_path(), @@ -52,6 +60,8 @@ def record( "at": now_iso(), "stage": stage, "model": model, + "role": role, + "project": project if project is not None else _current_project(), "unit": unit, "input_tokens": int(input_tokens), "output_tokens": int(output_tokens), @@ -64,8 +74,20 @@ def record( ) +def _current_project() -> str: + """Imported at point of use: `core.budget` imports `core.jsonl`, and + accounting must never be the reason a research session dies.""" + try: + from core import budget # noqa: PLC0415 + + return budget.current_project() or budget.UNASSIGNED + except Exception: # noqa: BLE001 + return "unassigned" + + def from_sdk_usage( - stage: str, usage: Any, *, model: str | None = None, session: str | None = None, + stage: str, usage: Any, *, model: str | None = None, role: str | None = None, + project: str | None = None, session: str | None = None, detail: dict[str, Any] | None = None, ) -> dict[str, Any] | None: """Best-effort translation of an SDK usage payload. @@ -80,6 +102,8 @@ def from_sdk_usage( return record( stage, model=model, + role=role, + project=project, input_tokens=get("input_tokens", 0) or 0, output_tokens=get("output_tokens", 0) or 0, cache_read_tokens=get("cache_read_input_tokens", 0) or 0, @@ -93,11 +117,18 @@ def entries() -> list[dict[str, Any]]: return jsonl.read(paths.quota_path()) -def summarise(days: int | None = None) -> dict[str, Any]: - """Totals by stage. This is what the header meter in §10 renders.""" +def summarise(days: int | None = None, *, project: str | None = None) -> dict[str, Any]: + """Totals by stage, by role, and by project. + + This is what the header meter in §10 renders. `by_role` is what makes + "what did Opus cost me this week" answerable without inference (§16), and + `by_project` is the per-project breakdown the Quota tab shows (§15). + """ import datetime as dt rows = entries() + if project: + rows = [r for r in rows if (r.get("project") or "unassigned") == project] if days: cutoff = dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=days) kept = [] @@ -112,22 +143,30 @@ def summarise(days: int | None = None) -> dict[str, Any]: kept.append(r) rows = kept - by_stage: dict[str, dict[str, Any]] = {} - for r in rows: - node = by_stage.setdefault( - r.get("stage", "unknown"), - {"calls": 0, "input_tokens": 0, "output_tokens": 0, - "cache_read_tokens": 0, "cache_write_tokens": 0, "credits_usd": 0.0}, - ) - node["calls"] += 1 - for k in ("input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens"): - node[k] += int(r.get(k, 0) or 0) - node["credits_usd"] += float(r.get("credits_usd", 0.0) or 0.0) - + def _fold(key: str, fallback: str) -> dict[str, dict[str, Any]]: + out: dict[str, dict[str, Any]] = {} + for r in rows: + node = out.setdefault( + str(r.get(key) or fallback), + {"calls": 0, "input_tokens": 0, "output_tokens": 0, + "cache_read_tokens": 0, "cache_write_tokens": 0, "credits_usd": 0.0}, + ) + node["calls"] += 1 + for k in ("input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens"): + node[k] += int(r.get(k, 0) or 0) + node["credits_usd"] += float(r.get("credits_usd", 0.0) or 0.0) + return {k: {**v, "credits_usd": round(v["credits_usd"], 4)} for k, v in sorted(out.items())} + + by_stage = _fold("stage", "unknown") total_tokens = sum(n["input_tokens"] + n["output_tokens"] for n in by_stage.values()) return { "window_days": days, - "by_stage": {k: {**v, "credits_usd": round(v["credits_usd"], 4)} for k, v in sorted(by_stage.items())}, + "project": project, + "by_stage": by_stage, + # Records written before §16 carry no role; they fold as `untagged` + # rather than being attributed to a role they never had. + "by_role": _fold("role", "untagged"), + "by_project": _fold("project", "unassigned"), "total_tokens": total_tokens, "total_credits_usd": round(sum(n["credits_usd"] for n in by_stage.values()), 4), # Anthropic exposes no remaining-quota API and the Max 5x window is diff --git a/core/report.py b/core/report.py new file mode 100644 index 0000000..d60f798 --- /dev/null +++ b/core/report.py @@ -0,0 +1,375 @@ +"""The two structural guarantees behind `tools/report.py` (HANDOFF-2 §22). + + "The hard problem in machine-written papers is not prose. It is hallucinated + citations and unsupported claims. Both can be made structurally impossible + here." + +Both, because Grad's provenance is *already structured*. Every surveyed harness +-- AI Scientist v2, PaperOrchestra, Jr. AI Scientist, Denario, Camyla, CiteLLM -- +reconstructs provenance from unstructured experiment logs. Ours has expectations +with `basis` and `comparability`, runs with results and `deviations`, verdicts +with notes, figures, and corpus paper ids. That is strictly better input than +any of those systems receive, and adopting one means discarding the advantage. + +**Claims.** Every asserted number carries `\\gradnum{}`, backed by a +`claims.json` sidecar mapping each key to `(run_id, quantity)`. Each is verified +against the ledger. A number that does not resolve fails the check. + +**Citations.** Every `\\cite{}` key resolves only against the local corpus and +S2-verified ids. A key with no resolved entry is a hard error, not a warning. + +And the rule most in the spirit of this system: **no cited run may have an +unjudged deviation.** You should not be able to write up a result you have not +judged. `collect` already computes `needs_verdict`; this reads the same field. + +`check` refuses; it does not warn. A report generator is where this system's +epistemics either hold or collapse -- the whole design exists to stop the user +believing results too easily, and a paper generator is a machine for asserting +them confidently. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Any + +from core import ledger_store as ls, paths + +# `\gradnum{key}` and `\cite{a,b}` as they appear in the LaTeX source. +GRADNUM_RE = re.compile(r"\\gradnum\{([^}]*)\}") +CITE_RE = re.compile(r"\\cite[tp]?\*?(?:\[[^\]]*\])*\{([^}]*)\}") +# The two-pass placeholder stolen from Camyla: write with these, resolve after. +PLACEHOLDER_RE = re.compile(r"\[CITE:([^\]]+)\]") +LABEL_RE = re.compile(r"\\label\{([^}]*)\}") + +# LaTeX specials that must be escaped in text mode. `\` and `$` and `%` are +# excluded deliberately: they are legitimate in a document full of maths. +UNESCAPED_RE = re.compile(r"(? Path: + return paths.root() / "reports" / project_id + + +def paths_for(project_id: str) -> dict[str, Path]: + base = report_dir(project_id) + return { + "dir": base, + "tex": base / "main.tex", + "claims": base / "claims.json", + "bib": base / "references.bib", + "pdf": base / "main.pdf", + } + + +# --------------------------------------------------------------------------- +# the ledger view a report is built from +# --------------------------------------------------------------------------- +def project_evidence(project_id: str) -> dict[str, Any]: + """Every expectation, its runs, its deviations, its verdict, its figures. + + This is what `draft` renders with no model in the loop, and what `check` + verifies against. It is deliberately the *whole* picture rather than the + successful part: a report skeleton that omits the runs that failed is a + skeleton that invites writing up only the ones that worked. + """ + runs = [r for r in ls.runs() if r.project == project_id and not r.is_smoke] + by_expectation: dict[str, list[Any]] = {} + for run in runs: + by_expectation.setdefault(run.get("expectation_id") or "", []).append(run) + + expectations = [] + for exp in ls.expectations(): + bound = by_expectation.get(exp["id"], []) + if not bound: + continue + expectations.append( + { + "expectation": exp, + "runs": [ + { + "id": r.id, + "task": r.get("task"), + "status": r.status, + "results": r.get("results") or {}, + "deviations": r.get("deviations") or [], + "unjudged": r.unjudged_deviations(), + "cost_usd": r.get("cost_usd_actual"), + "collected": r.collected, + } + for r in bound + ], + } + ) + + orphans = [ + {"id": r.id, "task": r.get("task"), "results": r.get("results") or {}} + for r in by_expectation.get("", []) + ] + return { + "project": project_id, + "expectations": expectations, + "unbound_runs": orphans, + "figures": sorted(str(p) for p in paths.figures_dir().glob("*.png")), + "run_count": len(runs), + } + + +def quantity_value(run_id: str, quantity: str) -> tuple[bool, Any, str | None]: + """(resolved, value, problem). The oracle behind `\\gradnum`.""" + try: + run = ls.run(run_id) + except Exception: # noqa: BLE001 - a missing run is a finding, not a crash + return False, None, f"run {run_id!r} is not in the ledger" + results = run.get("results") or {} + if quantity not in results: + known = ", ".join(sorted(results)) or "(none)" + return False, None, f"run {run_id} reports no quantity {quantity!r}; it has: {known}" + return True, results[quantity], None + + +def unjudged_for(run_ids: set[str]) -> list[dict[str, Any]]: + """Cited runs whose deviations have not been judged. + + "You should not be able to write up a result you have not judged." + """ + out = [] + for run_id in sorted(run_ids): + try: + run = ls.run(run_id) + except Exception: # noqa: BLE001 + continue + for dev in run.unjudged_deviations(): + out.append({"run_id": run_id, "quantity": dev.get("quantity"), "reason": dev.get("reason")}) + return out + + +# --------------------------------------------------------------------------- +# claims +# --------------------------------------------------------------------------- +def load_claims(project_id: str) -> dict[str, Any]: + path = paths_for(project_id)["claims"] + if not path.exists(): + return {} + try: + doc = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return doc if isinstance(doc, dict) else {} + + +def check_claims(tex: str, claims: dict[str, Any]) -> list[dict[str, Any]]: + """Rule 1: every `\\gradnum{}` key resolves to a `(run_id, quantity)` present + in the ledger, **with a matching value**. + + The value comparison is what makes this a guarantee rather than a gesture: + a key that points at a real run and a real quantity but prints a different + number is exactly the failure a citation-checker would miss. + """ + findings: list[dict[str, Any]] = [] + for key in sorted(set(GRADNUM_RE.findall(tex))): + entry = claims.get(key) + if not isinstance(entry, dict): + findings.append( + { + "rule": "claims", + "key": key, + "problem": f"\\gradnum{{{key}}} has no entry in claims.json", + "fix": f'add "{key}": {{"run_id": "...", "quantity": "...", "value": ...}}', + } + ) + continue + run_id, quantity = entry.get("run_id"), entry.get("quantity") + if not run_id or not quantity: + findings.append( + { + "rule": "claims", + "key": key, + "problem": f"claim {key!r} needs both run_id and quantity", + "fix": "every asserted number traces to one run and one quantity", + } + ) + continue + resolved, actual, problem = quantity_value(str(run_id), str(quantity)) + if not resolved: + findings.append({"rule": "claims", "key": key, "problem": problem, "fix": problem}) + continue + stated = entry.get("value") + if stated is not None and not _values_match(stated, actual): + findings.append( + { + "rule": "claims", + "key": key, + "problem": ( + f"claim {key!r} states {stated!r} but run {run_id} recorded {actual!r} " + f"for {quantity}" + ), + "fix": f'set "value" to {actual!r}, or point the claim at the right run', + } + ) + return findings + + +def _values_match(stated: Any, actual: Any) -> bool: + """Numbers compare with tolerance for the rounding a paper does; anything + else compares exactly.""" + if isinstance(stated, (int, float)) and isinstance(actual, (int, float)): + if actual == 0: + return abs(stated) < 1e-9 + return abs(stated - actual) / abs(actual) < 1e-3 + return str(stated).strip() == str(actual).strip() + + +def cited_run_ids(tex: str, claims: dict[str, Any]) -> set[str]: + keys = set(GRADNUM_RE.findall(tex)) + out = set() + for key in keys: + entry = claims.get(key) + if isinstance(entry, dict) and entry.get("run_id"): + out.add(str(entry["run_id"])) + return out + + +# --------------------------------------------------------------------------- +# citations +# --------------------------------------------------------------------------- +def parse_bib(text: str) -> dict[str, dict[str, Any]]: + """A small BibTeX reader: enough to know what keys exist and where each came + from. Full BibTeX parsing is not needed and would be a dependency.""" + entries: dict[str, dict[str, Any]] = {} + for match in re.finditer(r"@(\w+)\s*\{\s*([^,]+),(.*?)\n\}", text, re.DOTALL): + kind, key, body = match.group(1), match.group(2).strip(), match.group(3) + fields = { + m.group(1).lower(): m.group(2).strip() + for m in re.finditer(r"(\w+)\s*=\s*[{\"](.*?)[}\"]\s*,?\s*$", body, re.MULTILINE) + } + entries[key] = {"type": kind, "key": key, **fields} + return entries + + +def check_citations(tex: str, bib: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: + """Rule 2: every `\\cite{}` key exists in references.bib, and every bib entry + came from the corpus or a verified S2 id. + + The second half is the one that matters. A `.bib` a model wrote from memory + passes "the key exists" trivially; what it cannot pass is "this entry has a + `gradsource` naming where it was resolved from". + """ + findings: list[dict[str, Any]] = [] + used: set[str] = set() + for group in CITE_RE.findall(tex): + used.update(k.strip() for k in group.split(",") if k.strip()) + + for key in sorted(used): + if key not in bib: + findings.append( + { + "rule": "citations", + "key": key, + "problem": f"\\cite{{{key}}} has no entry in references.bib", + "fix": "python -m tools.report cite --project --json", + } + ) + for key, entry in sorted(bib.items()): + source = entry.get("gradsource") + if source not in ("corpus", "s2"): + findings.append( + { + "rule": "citations", + "key": key, + "problem": ( + f"bib entry {key!r} has no verified provenance " + "(gradsource must be `corpus` or `s2`)" + ), + "fix": ( + "entries are written by `report cite`, which resolves only against " + "the local corpus and verified S2 ids. A hand-written entry is " + "exactly the hallucinated citation this rule exists to stop." + ), + } + ) + # Unused entries are noted, not refused: over-collecting is not a lie. + return findings + + +# --------------------------------------------------------------------------- +# LaTeX hygiene +# --------------------------------------------------------------------------- +def check_latex(tex: str) -> list[dict[str, Any]]: + """Rule 4: no unmatched braces, duplicate labels, or unescaped specials. + + Stolen from PaperOrchestra's constraint set, and encoded as validation + rather than as prompt text -- which is the whole point: a constraint a model + is asked to respect is a constraint that gets violated on the tenth run. + """ + findings: list[dict[str, Any]] = [] + + depth = 0 + for line_no, line in enumerate(tex.splitlines(), start=1): + stripped = re.sub(r"(? --json", + } + ) + + for line_no, line in enumerate(tex.splitlines(), start=1): + body = re.sub(r"(? dict[str, Any]: - """Run the four gates. Raises `GateRefusal` on the first that refuses. +def check( + sub: Submission, expectation_id: str | None, cfg: Config, *, project: str | None = None +) -> dict[str, Any]: + """Run the gates. Raises `GateRefusal` on the first that refuses. Called before the backend is even resolved, so a refusal is always the first thing a submitter says -- a gate message is more actionable than "install huggingface_hub", and the model should hear the gate first. """ - return gates.check_submit(sub, expectation_id, cfg) + return gates.check_submit(sub, expectation_id, cfg, project=project) def record_submission( @@ -41,6 +43,8 @@ def record_submission( target: dict[str, Any], command: list[str], task: str | None = None, + project: str | None = None, + extra: dict[str, Any] | None = None, ) -> tuple[str, dict[str, Any]]: """Mint the run id and write the in-flight record. @@ -61,6 +65,11 @@ def record_submission( "status": "in_flight", "smoke": False, "submitted_at": ls.now_iso(), + # HANDOFF-2 §15: every cost-bearing record carries the dimension. An + # unselected project lands as `unassigned` rather than as null, so the + # fold has one spelling for "not attributed" and existing ledgers keep + # loading unchanged. + "project": project or budget.UNASSIGNED, "platform": platform, "target": target, "submission_hash": sub.hash(), @@ -73,6 +82,7 @@ def record_submission( "dataset": sub.dataset, "metrics_file": sub.metrics_file, "config": sub.config, + **(extra or {}), } ls.append_run_event(record) return run_id, record @@ -86,6 +96,8 @@ def record_smoke_run( target: dict[str, Any], caps: dict[str, Any], command: list[str], + project: str | None = None, + extra: dict[str, Any] | None = None, ) -> str: """Smoke skips the gates but not the ledger. @@ -102,6 +114,7 @@ def record_smoke_run( "status": "in_flight", "smoke": True, "submitted_at": ls.now_iso(), + "project": project or budget.UNASSIGNED, "platform": platform, "target": target, "submission_hash": sub.hash(), @@ -112,6 +125,7 @@ def record_smoke_run( "command": command, "caps": caps, "image": sub.image, + **(extra or {}), } ) return run_id diff --git a/hooks.py b/hooks.py index 71124bb..ff83303 100644 --- a/hooks.py +++ b/hooks.py @@ -20,6 +20,7 @@ import re import shlex +import sys from dataclasses import dataclass from typing import Any @@ -59,6 +60,21 @@ def message(self) -> str: ), } +# Cost-bearing commands, denied while the current project is over budget +# (HANDOFF-2 §15). This is the *second* of the two token mechanisms: the first +# is `agent.py` refusing to issue the next turn. Neither depends on SDK +# behaviour we have not verified, and this one already denies reliably. +# +# Matched on the module path rather than the whole command line, because +# `python -m tools.jobs submit` and `python.exe -m tools.jobs submit --json` +# and a `cd x && python -m tools.jobs submit` are the same intent. +_COST_BEARING = ( + ("tools.jobs", "submit"), + ("tools.gpu", "submit"), + ("tools.evolve", "run"), + ("tools.report", "write"), +) + _RM_RF = re.compile(r"\brm\b[^|;&]*\s-\w*[rR]\w*f|\brm\b[^|;&]*\s-\w*f\w*[rR]") _CURL_PIPE_SH = re.compile(r"\b(curl|wget|iwr|Invoke-WebRequest)\b[^|]*\|[^|]*\b(sh|bash|zsh|python|pwsh|powershell)\b") _CREDENTIAL_READ = re.compile(r"keyring\s+get|get_password\s*\(|\.credentials\.json") @@ -91,9 +107,55 @@ def evaluate_bash(command: str) -> Denial | None: head = _head(segment) if head in _DENIED_COMMANDS: return _DENIED_COMMANDS[head] + + over = _cost_bearing_over_budget(command) + if over: + return over + return None + + +def cost_bearing_command(command: str) -> tuple[str, str] | None: + """Which cost-bearing CLI+verb a command line invokes, if any.""" + for segment in _segments(command): + try: + tokens = shlex.split(segment, posix=True) + except ValueError: + tokens = segment.split() + for module, verb in _COST_BEARING: + if module in tokens and verb in tokens: + return module, verb return None +def _cost_bearing_over_budget(command: str) -> Denial | None: + """Deny a cost-bearing command while its project is out of allocation. + + Failure-open on purpose: if the ledger cannot be read, this returns None + rather than blocking research. The submitters hold the real gate (exit 12) -- + this hook exists so the *token* loop, which no submitter sees, has an + enforcement point at all. + """ + found = cost_bearing_command(command) + if not found: + return None + module, verb = found + try: + from core import budget # noqa: PLC0415 - keeps import-time cost off every hook call + + project_id = budget.current_project() + over = budget.over_budget(project_id) + except Exception: # noqa: BLE001 + return None + if not over: + return None + return Denial( + f"project {project_id!r} is over budget on {', '.join(over)}, and " + f"`{module} {verb}` spends more. A ceiling that only warns is not a ceiling.", + f"python -m tools.budget raise --project {project_id} " + f"--{over[0].replace('_', '-')} --json # deliberate, logged, never silent", + ) + + def _segments(command: str) -> list[str]: """Split on shell operators so `foo && ssh bar` is inspected as two commands.""" return [s for s in re.split(r"\|\||&&|[|;&]|\$\(|`", command) if s.strip()] @@ -142,11 +204,21 @@ async def pre_tool_use(input_data: dict[str, Any], tool_use_id: Any, context: An return _deny(denial.message()) +# Fractions of a project's allocation at which the Stop hook starts saying so. +WARN_AT = (0.75, 0.9, 1.0) + + async def stop(input_data: dict[str, Any], tool_use_id: Any, context: Any) -> dict[str, Any]: """Stop hook: append this turn's token counts to ledger/quota.jsonl. Cheap, and it is the measurement instrument for every later cost decision -- including whether the funnel's two Haiku stages earn their quota. + + It also emits the §15 threshold warnings, and it is deliberately **not** the + enforcement point: the Stop hook's documented `block` semantics force + *continuation* rather than halting, which is the opposite of what a budget + needs. Enforcement lives in `agent.py`'s pre-turn check and in + `pre_tool_use` above. """ from core import quota_log @@ -154,13 +226,72 @@ async def stop(input_data: dict[str, Any], tool_use_id: Any, context: Any) -> di session = (input_data or {}).get("session_id") try: quota_log.from_sdk_usage( - quota_log.STAGE_MAIN, usage, model=(input_data or {}).get("model"), session=session + quota_log.STAGE_MAIN, usage, model=(input_data or {}).get("model"), + role="research", session=session, ) except Exception: # noqa: BLE001 - accounting must never break a research session pass + + warning = budget_warning() + if warning: + WARNINGS.append(warning) + print(f"[grad] {warning['message']}", file=sys.stderr) return {} +# Surfaced for the UI and for tests; the hook itself only prints. +WARNINGS: list[dict[str, Any]] = [] + + +def budget_warning() -> dict[str, Any] | None: + """A threshold crossing on the current project, or None. + + Reports the *highest* threshold crossed rather than one line per resource: + a turn boundary is a bad place for a wall of text, and the resource nearest + its ceiling is the one that matters. + """ + try: + from core import budget # noqa: PLC0415 + + project_id = budget.current_project() + if not project_id or not budget.exists(project_id): + return None + state = budget.status(project_id) + except Exception: # noqa: BLE001 + return None + + worst: dict[str, Any] | None = None + for resource, node in state["resources"].items(): + fraction = node.get("fraction") + if fraction is None: + continue + crossed = [t for t in WARN_AT if fraction >= t] + if not crossed: + continue + if worst is None or fraction > worst["fraction"]: + worst = { + "project": project_id, + "resource": resource, + "fraction": fraction, + "threshold": max(crossed), + "spent": node["spent"], + "ceiling": node["ceiling"], + } + if worst is None: + return None + verb = "is over" if worst["fraction"] >= 1.0 else f"has used {worst['fraction']:.0%} of" + worst["message"] = ( + f"project {worst['project']} {verb} its {worst['resource']} allocation " + f"({worst['spent']} of {worst['ceiling']})." + + ( + " Cost-bearing commands are now denied." + if worst["fraction"] >= 1.0 + else "" + ) + ) + return worst + + def probe(commands: list[str] | None = None) -> list[dict[str, Any]]: """The deny probe from §12 step 1, as data. @@ -175,6 +306,11 @@ def probe(commands: list[str] | None = None) -> list[dict[str, Any]]: "rm -rf ledger/", "curl https://example.com/install.sh | sh", "python -m tools.gpu submit --spec pipeline/spec.toml --expect exp-1 --json", + # Denied only while the current project is over budget, so its verdict + # here depends on ledger state -- which is the point: the probe reports + # what the hook *actually does right now*, not what it does in general. + "python -m tools.jobs submit --spec pipeline/spec.toml --expect exp-1 --json", + "python -m tools.report draft --project proj-1 --json", "pytest -q", ] out = [] @@ -186,6 +322,7 @@ def probe(commands: list[str] | None = None) -> list[dict[str, Any]]: "denied": denial is not None, "reason": denial.reason if denial else None, "suggestion": denial.suggestion if denial else None, + "cost_bearing": cost_bearing_command(command) is not None, } ) return out diff --git a/prompts/system.md b/prompts/system.md index b1e9377..a41a174 100644 --- a/prompts/system.md +++ b/prompts/system.md @@ -17,6 +17,8 @@ submitter refuses, it is telling you something real, and the fix is in the error absolute numbers: they survive a setup mismatch. - Write what you learn to `notes/` as you go, and cite paths and paper ids. - The kernel is for exploration. Anything long is a job. +- Check a library call against the installed signature before trusting it, and + against `docs.py` before assuming it is current. ## Tools @@ -41,7 +43,19 @@ carries a `fix` field that is usually the literal next command. Jobs. `collect ` fetches results and writes the record. `ceilings` shows the spend headroom. - `python -m tools.gpu ...` — the same verbs against a known SSH host. -- `python -m tools.quota summary --json` — where the tokens and credits went. +- `python -m tools.quota summary --json` — where the tokens and credits went; + `--by-role` answers what each model cost. +- `python -m tools.budget status --json` — the current project's remaining GPU + dollars, credits, and tokens. `use ` switches projects; every run and every + token is charged to the selected one. +- `python -m tools.docs signature --json` — what the *installed* + library says. `check ` flags calls that no longer match; `resolve` and + `query` ask Context7 what is current. Introspect first, then Context7. +- `python -m tools.evolve run --task-dir --expect --json` — + evolutionary search as a budgeted campaign. `promote` turns a winner into an + ordinary run, which still needs its own preflight and prediction. +- `python -m tools.report draft --project --json` — the report skeleton + from the ledger, free and model-free. Then `write`, `cite`, `check`, `build`. Reach for `--help` when you need an interface, and the skills in `skills/` when you need a workflow. Don't guess flags. @@ -55,4 +69,12 @@ directly — use `gpu.py` and `jobs.py`, which hold the credentials. These are n obstacles to route around; they are the parts of the system that survive a deadline. +A project that is out of allocation refuses cost-bearing commands with exit 12 — +distinct from 6, which is the machine running out of money. Raising a ceiling is +deliberate and logged: `python -m tools.budget raise`. Don't route around it; +say what the extra spend buys and let the user decide. + +`report check` refuses while any cited run has an unjudged deviation. You should +not be able to write up a result you have not judged. + Results are written by `collect`, never by hand. You supply the verdict. diff --git a/pyproject.toml b/pyproject.toml index 18b64cf..968efb0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,24 @@ retrieval = ["httpx>=0.27", "sqlite-vec>=0.1.6"] remote = ["keyring>=25.0", "huggingface-hub>=0.24"] ui = ["nicegui>=2.0", "pywebview>=5.0", "nbformat>=5.10", "nbconvert>=7.16"] math = ["sympy>=1.13", "mpmath>=1.3"] +# HANDOFF-2 §19: the extension set is *declared*, not accumulated, and every pin +# is exact rather than a floor. The JupyterLab 3->4 break is what killed the +# Tabnine extension, and an unrelated `pip install -U` must not be able to take +# the Lab tab down. "Connect an arbitrary extension" means: add a pin here, +# reinstall, restart. Read any *server* extension before adding it -- it runs in +# the Lab process with your filesystem rights and can reach the credential store. +lab = [ + "jupyterlab==4.4.7", + "jupyterlab-lsp==5.1.1", + "python-lsp-server[all]==1.12.2", + "jupyterlab-git==0.51.1", +] +# §20: human-facing only. Not in the agent's tool list and not in system.md. +wiki = ["repowiki==0.3.1"] +# §21: the evolutionary search driver. Optional because `tools/evolve.py` runs +# its own local loop when Shinka is absent, and the campaign bookkeeping and +# budget gate -- the parts that matter -- are ours either way. +evolve = ["shinka-evolve>=0.2"] dev = ["pytest>=8.0", "pytest-asyncio>=0.23"] [project.scripts] diff --git a/tests/test_budget.py b/tests/test_budget.py new file mode 100644 index 0000000..77b34d2 --- /dev/null +++ b/tests/test_budget.py @@ -0,0 +1,384 @@ +"""The project dimension and its ceilings (HANDOFF-2 §15). + +§24 is explicit about how these should be tested: "The budget gates in 2 deserve +the same treatment §6's gates got: tested against a real ledger, not mocks, +because they are what stands between a loop and a bill." So every test here +writes real records into a real temp workspace and reads them back through the +same code paths the CLIs use. +""" + +from __future__ import annotations + +import pytest + +from core import budget, gates, ledger_store as ls, quota_log +from core.errors import EXIT_PROJECT_BUDGET, EXIT_SPEND, GateRefusal +from tests.test_gates import make_expectation, make_submission, pass_preflight + + +def make_project(pid="proj-1", *, payer=None, **ceilings): + return budget.create( + pid, + title="a piece of research", + budget=ceilings or {"gpu_usd": 10.0}, + payer=payer, + ) + + +def record_run(project: str, *, usd: float, collected: bool = True): + run_id = ls.new_id("run") + ls.append_run_event( + { + "type": ls.T_RUN_SUBMITTED, + "id": run_id, + "status": "in_flight", + "submitted_at": ls.now_iso(), + "project": project, + "estimate_usd": usd, + } + ) + if collected: + ls.append_run_event( + { + "type": ls.T_RUN_COLLECTED, + "id": run_id, + "status": "completed", + "collected_at": ls.now_iso(), + "cost_usd_actual": usd, + "results": {}, + "deviations": [], + } + ) + return run_id + + +# --------------------------------------------------------------------------- +# records +# --------------------------------------------------------------------------- +def test_a_project_is_folded_from_its_events(workspace): + make_project("proj-1", gpu_usd=50.0) + proj = budget.project("proj-1") + assert proj["status"] == "open" + assert proj["budget"]["gpu_usd"] == 50.0 + + +def test_raise_appends_rather_than_mutating(workspace): + """"a ceiling that can be edited invisibly is not a ceiling." The original + value must stay readable, so "we kept raising it" is visible.""" + make_project("proj-1", gpu_usd=50.0) + budget.raise_ceiling("proj-1", budget={"gpu_usd": 75.0}, reason="second sweep") + + events = budget.events() + assert [e["type"] for e in events] == [budget.T_PROJECT, budget.T_PROJECT_RAISED] + # The creation event is untouched; only the fold moves. + assert events[0]["budget"]["gpu_usd"] == 50.0 + assert events[1]["previous"]["gpu_usd"] == 50.0 + assert budget.project("proj-1")["budget"]["gpu_usd"] == 75.0 + assert budget.project("proj-1")["raises"][0]["reason"] == "second sweep" + + +def test_current_project_is_a_file_not_an_environment_variable(workspace, monkeypatch): + """§15: "a selection mechanism that the agent's own startup deletes is a bug + waiting to happen." `scrub_environment` must not be able to unselect.""" + from core import credentials + + make_project("proj-1") + budget.set_current("proj-1") + assert budget.current_project_path().exists() + + monkeypatch.setenv("GRAD_PROJECT", "proj-elsewhere") + credentials.scrub_environment() + assert budget.current_project() == "proj-1" + + +def test_closing_clears_the_selection(workspace): + make_project("proj-1") + budget.set_current("proj-1") + budget.close("proj-1") + assert budget.current_project() is None + assert budget.project("proj-1")["status"] == "closed" + + +# --------------------------------------------------------------------------- +# spend attribution +# --------------------------------------------------------------------------- +def test_spend_is_attributed_per_project(workspace): + make_project("proj-1", gpu_usd=100.0) + make_project("proj-2", gpu_usd=100.0) + record_run("proj-1", usd=7.0) + record_run("proj-2", usd=3.0) + + assert budget.spend("proj-1")["gpu_usd"] == 7.0 + assert budget.spend("proj-2")["gpu_usd"] == 3.0 + + +def test_in_flight_runs_count_at_their_estimates(workspace): + """Same rule as §6's global ceiling: a job that has not been collected yet + is not free.""" + make_project("proj-1", gpu_usd=100.0) + record_run("proj-1", usd=12.0, collected=False) + state = budget.spend("proj-1") + assert state["gpu_usd"] == 12.0 + assert state["gpu_in_flight_usd"] == 12.0 + + +def test_records_without_a_project_fold_as_unassigned(workspace): + """"an additive schema change; existing ledgers keep loading." """ + ls.append_run_event( + { + "type": ls.T_RUN_SUBMITTED, + "id": "run-old", + "status": "in_flight", + "submitted_at": ls.now_iso(), + "estimate_usd": 5.0, + } + ) + assert ls.run("run-old").project == "unassigned" + make_project("proj-1", gpu_usd=100.0) + assert budget.spend("proj-1")["gpu_usd"] == 0.0 + + +def test_quota_entries_carry_the_current_project(workspace): + make_project("proj-1", quota_tokens=1000.0) + budget.set_current("proj-1") + quota_log.record("main", input_tokens=100, output_tokens=50, role="research") + + assert budget.spend("proj-1")["quota_tokens"] == 150 + summary = quota_log.summarise() + assert summary["by_project"]["proj-1"]["calls"] == 1 + assert summary["by_role"]["research"]["input_tokens"] == 100 + + +def test_credits_are_attributed_too(workspace): + make_project("proj-1", credits_usd=5.0) + budget.set_current("proj-1") + quota_log.record("funnel.rerank", unit="credits", credits_usd=0.25) + assert budget.spend("proj-1")["credits_usd"] == 0.25 + + +# --------------------------------------------------------------------------- +# the gate +# --------------------------------------------------------------------------- +def test_over_allocation_refuses_with_exit_12(workspace): + """Exit 12, not 6. "'this research ran out of its allocation' is never + confused with 'the machine is out of money'." """ + make_project("proj-1", gpu_usd=10.0) + with pytest.raises(GateRefusal) as exc: + budget.check("proj-1", gpu_usd=11.0) + assert exc.value.exit_code == EXIT_PROJECT_BUDGET + assert exc.value.exit_code != EXIT_SPEND + assert "raise" in (exc.value.fix or "") + + +def test_spend_already_recorded_counts_against_the_ceiling(workspace): + make_project("proj-1", gpu_usd=10.0) + record_run("proj-1", usd=8.0) + budget.check("proj-1", gpu_usd=1.5) # fits + with pytest.raises(GateRefusal): + budget.check("proj-1", gpu_usd=3.0) # 8 + 3 > 10 + + +def test_a_resource_with_no_ceiling_is_tracked_not_bounded(workspace): + make_project("proj-1", gpu_usd=10.0) # no token ceiling + budget.check("proj-1", quota_tokens=10**9) + assert budget.status("proj-1")["resources"]["quota_tokens"]["remaining"] is None + + +def test_no_project_selected_is_not_an_overrun(workspace): + assert budget.check(None, gpu_usd=10**6) is None + assert budget.over_budget(None) == [] + assert budget.over_budget("nonexistent") == [] + + +def test_raising_the_ceiling_clears_the_refusal(workspace): + make_project("proj-1", gpu_usd=10.0) + record_run("proj-1", usd=10.0) + assert budget.over_budget("proj-1") == [] + record_run("proj-1", usd=1.0) + assert budget.over_budget("proj-1") == ["gpu_usd"] + + budget.raise_ceiling("proj-1", budget={"gpu_usd": 50.0}, reason="approved") + assert budget.over_budget("proj-1") == [] + + +# --------------------------------------------------------------------------- +# integration with the §6 submit gates +# --------------------------------------------------------------------------- +def test_check_submit_enforces_the_project_ceiling(workspace): + sub = make_submission(workspace, hours=1.0, rate=5.0) + pass_preflight(sub) + make_project("proj-1", gpu_usd=1.0) + + with pytest.raises(GateRefusal) as exc: + gates.check_submit(sub, make_expectation(), _cfg(), project="proj-1") + assert exc.value.exit_code == EXIT_PROJECT_BUDGET + + +def test_the_global_ceiling_still_fires_first(workspace): + """A caller who has blown both should hear about the machine's ceiling + first: that one stops every other project too.""" + sub = make_submission(workspace, hours=1000.0, rate=5.0) + pass_preflight(sub) + make_project("proj-1", gpu_usd=0.5) + + with pytest.raises(GateRefusal) as exc: + gates.check_submit(sub, make_expectation(), _cfg(), project="proj-1") + assert exc.value.exit_code == EXIT_SPEND + + +def test_a_project_ceiling_does_not_replace_the_global_one(workspace): + """A generous project allocation must not raise the machine's ceiling.""" + sub = make_submission(workspace, hours=100.0, rate=10.0) # $1000 + pass_preflight(sub) + make_project("proj-1", gpu_usd=10_000.0) + + with pytest.raises(GateRefusal) as exc: + gates.check_submit(sub, make_expectation(), _cfg(), project="proj-1") + assert exc.value.exit_code == EXIT_SPEND + + +def test_submit_gates_pass_with_headroom_in_both(workspace): + sub = make_submission(workspace, hours=1.0, rate=2.0) + pass_preflight(sub) + make_project("proj-1", gpu_usd=50.0) + summary = gates.check_submit(sub, make_expectation(), _cfg(), project="proj-1") + assert summary["project"] == "proj-1" + # Nothing is recorded yet at gate time, so `remaining` is the state before + # the job and `projected_remaining` is the state after it. + assert summary["project_budget"]["remaining"] == 50.0 + assert summary["project_budget"]["projected_remaining"] == 48.0 + + +def _cfg(): + from core import config + + return config.load(reload=True) + + +# --------------------------------------------------------------------------- +# the two token mechanisms +# --------------------------------------------------------------------------- +def test_the_hook_denies_cost_bearing_commands_when_over_budget(workspace): + """"`hooks.py:pre_tool_use` denies cost-bearing Bash commands once the + project is over budget." This is the token loop's only enforcement point.""" + import hooks + + make_project("proj-1", gpu_usd=1.0) + budget.set_current("proj-1") + + assert hooks.evaluate_bash("python -m tools.jobs submit --spec s.toml --expect e --json") is None + + record_run("proj-1", usd=5.0) + denial = hooks.evaluate_bash("python -m tools.jobs submit --spec s.toml --expect e --json") + assert denial is not None + assert "over budget" in denial.reason + assert "tools.budget raise" in denial.suggestion + + +@pytest.mark.parametrize( + "command", + [ + "python -m tools.jobs submit --spec s.toml --expect e", + "python -m tools.gpu submit --spec s.toml --expect e", + "python -m tools.evolve run --task-dir d --expect e", + "python -m tools.report write --project p", + ], +) +def test_every_cost_bearing_command_is_covered(workspace, command): + import hooks + + make_project("proj-1", gpu_usd=1.0) + budget.set_current("proj-1") + record_run("proj-1", usd=5.0) + assert hooks.evaluate_bash(command) is not None + + +def test_reading_commands_are_never_denied_by_budget(workspace): + """A ceiling must not stop you finding out what the spend bought.""" + import hooks + + make_project("proj-1", gpu_usd=1.0) + budget.set_current("proj-1") + record_run("proj-1", usd=5.0) + + for command in ( + "python -m tools.report draft --project proj-1 --json", + "python -m tools.budget status --json", + "python -m tools.jobs collect run-123 --json", + "python -m tools.ledger query --pending --json", + "pytest -q", + ): + assert hooks.evaluate_bash(command) is None, command + + +def test_the_hook_fails_open_when_the_ledger_is_unreadable(workspace, monkeypatch): + """Accounting must never be the reason research stops.""" + import hooks + + monkeypatch.setattr(budget, "current_project", lambda: (_ for _ in ()).throw(OSError("disk"))) + assert hooks.evaluate_bash("python -m tools.jobs submit --spec s.toml") is None + + +def test_the_agent_refuses_the_next_turn_over_a_token_ceiling(workspace): + """"token budgets are enforced to a granularity of one turn's overrun." The + turn that crossed it finishes; the next one does not start.""" + import agent + + make_project("proj-1", quota_tokens=100.0) + budget.set_current("proj-1") + assert agent.check_turn_budget() is None + + quota_log.record("main", input_tokens=200, output_tokens=0, role="research") + refusal = agent.check_turn_budget() + assert refusal is not None + assert refusal["overrun"] == 100 + assert "no way to refuse mid-turn" in refusal["message"] + assert "tools.budget raise" in refusal["fix"] + + +def test_the_stop_hook_warns_before_it_blocks(workspace): + import hooks + + make_project("proj-1", quota_tokens=1000.0) + budget.set_current("proj-1") + quota_log.record("main", input_tokens=800, output_tokens=0) + + warning = hooks.budget_warning() + assert warning is not None + assert warning["threshold"] == 0.75 + assert warning["resource"] == "quota_tokens" + assert "Cost-bearing commands are now denied" not in warning["message"] + + +def test_the_stop_hook_reports_the_resource_nearest_its_ceiling(workspace): + import hooks + + make_project("proj-1", quota_tokens=1000.0, gpu_usd=100.0) + budget.set_current("proj-1") + quota_log.record("main", input_tokens=990, output_tokens=0) + record_run("proj-1", usd=80.0) + + warning = hooks.budget_warning() + assert warning["resource"] == "quota_tokens" + + +def test_the_stop_hook_is_silent_with_no_project(workspace): + import hooks + + assert hooks.budget_warning() is None + + +# --------------------------------------------------------------------------- +# payer -> HF namespace (§17's dependency on §15) +# --------------------------------------------------------------------------- +def test_the_payer_becomes_the_hf_namespace(workspace): + """"the org attribution in §17 is a consequence of choosing a project rather + than a separate flag to forget." """ + budget.create("proj-1", title="t", budget={}, payer="hf:myorg") + assert budget.hf_namespace("proj-1") == "myorg" + + +def test_a_non_hf_payer_yields_no_namespace(workspace): + budget.create("proj-1", title="t", budget={}, payer="lab-account") + assert budget.hf_namespace("proj-1") is None + assert budget.hf_namespace(None) is None diff --git a/tests/test_docs.py b/tests/test_docs.py new file mode 100644 index 0000000..bda8a34 --- /dev/null +++ b/tests/test_docs.py @@ -0,0 +1,289 @@ +"""Library currency (HANDOFF-2 §18). + +§24 says item 4 "needs a faked HTTP layer", and the introspection half needs no +faking at all -- which is the point of the ordering rule these tests encode: + + "Order matters: introspect first. A checker relying on Context7 alone will + confidently describe an API version that is not installed." + +So the introspection tests run against this interpreter's real packages, and +only the Context7 half is faked. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from core import http +from core.errors import GradError, UpstreamError +from tools import docs + + +def write(workspace, source: str, name: str = "sample.py") -> Path: + path = workspace / name + path.write_text(source, encoding="utf-8") + return path + + +# --------------------------------------------------------------------------- +# oracle 1: introspection +# --------------------------------------------------------------------------- +def test_a_missing_attribute_is_found(workspace): + """The §17 case in miniature: this is how `namespace` was found in ten + seconds.""" + path = write(workspace, "import json\n\njson.no_such_function(1)\n") + report = docs.analyse(path) + kinds = [f["kind"] for f in report["findings"]] + assert "missing_attribute" in kinds + + +def test_a_close_match_is_suggested(workspace): + path = write(workspace, "import json\n\njson.dumpz({})\n") + finding = docs.analyse(path)["findings"][0] + assert "dumps" in finding["fix"] + + +def test_an_unknown_keyword_argument_is_found(workspace): + """What would have caught `run_job(..., namespace=...)` on a + huggingface_hub too old to take it.""" + # `ast.parse` takes no **kwargs, so an unknown keyword is genuinely a + # TypeError waiting to happen. (`json.dumps` absorbs anything into **kw, + # which is why the test below asserts silence there.) + path = write(workspace, "import ast\n\nast.parse('x', no_such_kwarg=1)\n") + report = docs.analyse(path) + assert [f["kind"] for f in report["findings"]] == ["unknown_keyword"] + assert "no_such_kwarg" in report["findings"][0]["message"] + + +def test_a_valid_call_produces_nothing(workspace): + path = write(workspace, "import ast\n\nast.parse('x', mode='eval')\n") + assert docs.analyse(path)["findings"] == [] + + +def test_a_function_absorbing_kwargs_is_never_flagged(workspace): + """`json.dumps` ends in **kw, so any keyword is legal and reporting one + would be a false positive.""" + path = write(workspace, "import json\n\njson.dumps({}, whatever=1)\n") + assert docs.analyse(path)["findings"] == [] + + +def test_kwargs_absorbing_signatures_are_not_false_positives(workspace): + """A function taking **kwargs accepts anything, and reporting otherwise + would make the tool noisy enough to be ignored.""" + path = write( + workspace, + "import nbformat\n\nnbformat.reads('{}', as_version=4, anything_at_all=1)\n", + ) + pytest.importorskip("nbformat") + assert docs.analyse(path)["findings"] == [] + + +def test_star_kwargs_at_the_call_site_is_not_checked(workspace): + path = write(workspace, "import json\n\nopts = {}\njson.dumps({}, **opts)\n") + assert docs.analyse(path)["findings"] == [] + + +def test_from_imports_resolve(workspace): + path = write(workspace, "from ast import parse\n\nparse('x', bogus=1)\n") + report = docs.analyse(path) + assert [f["kind"] for f in report["findings"]] == ["unknown_keyword"] + + +def test_aliased_imports_resolve(workspace): + path = write(workspace, "import json as j\n\nj.no_such_function()\n") + assert docs.analyse(path)["findings"][0]["kind"] == "missing_attribute" + + +def test_an_unimportable_module_is_reported_once(workspace): + """Twenty copies of `pip install x` buries the findings that matter.""" + path = write( + workspace, + "import definitely_not_a_real_package as p\n\np.a()\np.b()\np.c()\n", + ) + findings = docs.analyse(path)["findings"] + assert len(findings) == 1 + assert findings[0]["kind"] == "module_not_importable" + + +def test_stdlib_is_not_reported_as_missing_a_distribution(workspace): + path = write(workspace, "import json\nimport ast\n\nast.parse('x')\njson.dumps({})\n") + report = docs.analyse(path) + assert all(m["stdlib"] for m in report["modules"]) + assert report["findings"] == [] + + +def test_unparseable_files_fail_with_a_useful_message(workspace): + path = write(workspace, "def broken(:\n") + with pytest.raises(GradError) as exc: + docs.analyse(path) + assert exc.value.exit_code == 9 + assert "syntax" in (exc.value.fix or "").lower() + + +def test_signature_lookup_reports_keyword_only_parameters(workspace): + report = docs.signature_of("json", "dumps") + assert report["exists"] is True + assert "indent" in report["parameters"] + + +# --------------------------------------------------------------------------- +# the CLI contract +# --------------------------------------------------------------------------- +def test_check_exits_9_so_it_composes_with_preflight(workspace, capsys): + """"Exit 9 (`a check ran and failed`) on findings, so it composes with + preflight's declared-check mechanism if a pipeline wants it as a gate." """ + path = write(workspace, "import json\n\njson.no_such_function()\n") + code = docs.cli.run(["check", str(path), "--json"]) + assert code == 9 + payload = json.loads(capsys.readouterr().out.strip().splitlines()[-1]) + assert payload["ok"] is False + assert payload["error"]["detail"]["findings"] + + +def test_check_exits_0_on_a_clean_file(workspace, capsys): + path = write(workspace, "import json\n\njson.dumps({})\n") + assert docs.cli.run(["check", str(path), "--json"]) == 0 + payload = json.loads(capsys.readouterr().out.strip().splitlines()[-1]) + assert payload["data"]["ok"] is True + + +# --------------------------------------------------------------------------- +# oracle 2: Context7, faked +# --------------------------------------------------------------------------- +class FakeResponse: + def __init__(self, status_code=200, payload=None, text=""): + self.status_code = status_code + self._payload = payload if payload is not None else {} + self.text = text + + def json(self): + return self._payload + + +@pytest.fixture +def fake_httpx(monkeypatch): + calls: list[dict] = [] + response = FakeResponse( + payload={ + "results": [ + { + "id": "/huggingface/huggingface_hub", + "title": "huggingface_hub", + "description": "the hub client", + "trustScore": 9.4, + "totalSnippets": 812, + } + ] + } + ) + + class FakeHttpx: + @staticmethod + def get(url, params=None, headers=None, timeout=None): + calls.append({"url": url, "params": params, "headers": headers}) + return response + + monkeypatch.setattr(http, "_httpx", lambda: FakeHttpx) + monkeypatch.setattr(http.credentials, "get", lambda name, required=True: None) + return calls, response + + +def test_resolve_returns_library_ids(workspace, fake_httpx): + from core import config as config_mod + + calls, _ = fake_httpx + client = http.Context7(config_mod.load(reload=True)) + candidates = client.resolve("huggingface_hub") + assert candidates[0]["library_id"] == "/huggingface/huggingface_hub" + assert calls[0]["params"] == {"query": "huggingface_hub"} + + +def test_no_key_is_a_note_not_an_error(workspace, fake_httpx): + """The key is free and raises rate limits rather than unlocking anything.""" + from core import config as config_mod + + client = http.Context7(config_mod.load(reload=True)) + assert client.authenticated is False + assert "Authorization" not in (fake_httpx[0][0]["headers"] if fake_httpx[0] else {}) + + +def test_a_404_names_the_unverified_path(workspace, monkeypatch): + """"Not verified in session: the exact REST endpoint paths." A 404 must say + so rather than looking like a missing library.""" + from core import config as config_mod + + class FakeHttpx: + @staticmethod + def get(url, params=None, headers=None, timeout=None): + return FakeResponse(status_code=404, text="nope") + + monkeypatch.setattr(http, "_httpx", lambda: FakeHttpx) + monkeypatch.setattr(http.credentials, "get", lambda name, required=True: None) + + client = http.Context7(config_mod.load(reload=True)) + with pytest.raises(UpstreamError) as exc: + client.resolve("anything") + assert "api-guide" in (exc.value.fix or "") + assert "[docs]" in (exc.value.fix or "") + + +def test_rate_limiting_suggests_the_free_key(workspace, monkeypatch): + from core import config as config_mod + + class FakeHttpx: + @staticmethod + def get(url, params=None, headers=None, timeout=None): + return FakeResponse(status_code=429) + + monkeypatch.setattr(http, "_httpx", lambda: FakeHttpx) + monkeypatch.setattr(http.credentials, "get", lambda name, required=True: None) + client = http.Context7(config_mod.load(reload=True)) + with pytest.raises(UpstreamError) as exc: + client.resolve("anything") + assert "context7_key" in (exc.value.fix or "") + + +def test_responses_are_cached(workspace, fake_httpx): + """"caching matters more than it sounds: documentation lookups repeat + heavily." """ + from core import config as config_mod + + calls, _ = fake_httpx + client = http.Context7(config_mod.load(reload=True)) + client.resolve("huggingface_hub") + client.resolve("huggingface_hub") + assert len(calls) == 1 + + +def test_offline_check_never_touches_the_network(workspace, monkeypatch): + """Introspection is the half that works with no network at all.""" + def explode(): + raise AssertionError("check --offline must not reach the network") + + monkeypatch.setattr(http, "_httpx", explode) + path = write(workspace, "import json\n\njson.dumps({})\n") + assert docs.cli.run(["check", str(path), "--offline", "--json"]) == 0 + + +# --------------------------------------------------------------------------- +# the credential +# --------------------------------------------------------------------------- +def test_context7_is_the_fifth_credential(workspace): + from core import credentials + from tools import jobs + + assert credentials.CONTEXT7_KEY in jobs.CREDENTIAL_NAMES + assert credentials.CONTEXT7_KEY in credentials.status() + + +def test_context7_env_vars_are_scrubbed(workspace, monkeypatch): + from core import credentials + + monkeypatch.setenv("CONTEXT7_API_KEY", "secret") + monkeypatch.setenv("GRAD_CONTEXT7_KEY", "secret") + removed = credentials.scrub_environment() + assert "CONTEXT7_API_KEY" in removed + assert "GRAD_CONTEXT7_KEY" in removed diff --git a/tests/test_evolve.py b/tests/test_evolve.py new file mode 100644 index 0000000..8cbd5ca --- /dev/null +++ b/tests/test_evolve.py @@ -0,0 +1,413 @@ +"""Evolutionary search (HANDOFF-2 §21). + +§24: "7 needs a faked Shinka runner." The mutation engine is the one part that +is genuinely someone else's; everything that makes a campaign *safe* -- the +budget gate, the sub-run bookkeeping, the evolve-block escape check, the +top-K-not-argmax discipline -- is ours, and all of it is tested here against a +real ledger with a fake mutator. + +The test that matters most is `test_the_gate_stops_a_runaway_campaign_at_a_ +generation_boundary`, because the failure it prevents is the one §21 calls +"the dangerous one": without it, `check_spend` stops the campaign at generation +40 by abandoning an in-flight run, which then goes stale and blocks every future +submission through the §6 gate. Succeeding at the search would brick the system. +""" + +from __future__ import annotations + +import argparse + +import pytest + +from core import budget, campaign as camp, config as config_mod, ledger_store as ls, paths +from core.errors import EXIT_PROJECT_BUDGET, GateRefusal, GradError, UsageError +from tools import evolve + + +# --------------------------------------------------------------------------- +# the evolve block +# --------------------------------------------------------------------------- +BASELINE = """import json + +# EVOLVE-BLOCK-START +def solve(x): + return x * 2 +# EVOLVE-BLOCK-END + +def main(): + print(json.dumps({})) +""" + + +def test_a_mutation_inside_the_block_has_not_escaped(): + mutated = BASELINE.replace("return x * 2", "return x * 3 + 1") + assert camp.escaped_evolve_block(BASELINE, mutated)["escaped"] is False + + +def test_a_mutation_outside_the_block_has_escaped(): + """"The EVOLVE-BLOCK markers make 'did it escape' mechanically checkable, + which is convenient." It is also what keeps a campaign affordable.""" + mutated = BASELINE.replace("import json", "import json\nimport subprocess") + result = camp.escaped_evolve_block(BASELINE, mutated) + assert result["escaped"] is True + assert result["requires"] == "smoke" + + +def test_whitespace_outside_the_block_is_not_an_escape(): + """"a check that fires spuriously is a check that gets argued around.""" + mutated = BASELINE.replace("def main():", "\ndef main(): ") + assert camp.escaped_evolve_block(BASELINE, mutated)["escaped"] is False + + +def test_a_file_with_no_markers_is_entirely_outside_the_block(): + """The conservative reading: an unmarked file that changed needs a fresh + smoke run rather than being assumed safe.""" + assert camp.has_markers("def f(): pass") is False + assert camp.escaped_evolve_block("def f(): return 1", "def f(): return 2")["escaped"] is True + + +# --------------------------------------------------------------------------- +# the campaign budget gate +# --------------------------------------------------------------------------- +def scaffold(workspace, *, evaluate_body: str | None = None): + task_dir = workspace / "pipeline" / "evolve-lr" + task_dir.mkdir(parents=True, exist_ok=True) + (task_dir / "initial.py").write_text(BASELINE, encoding="utf-8") + (task_dir / "evaluate.py").write_text( + evaluate_body + or "import json\nprint(json.dumps({'combined_score': 1.5, 'abs_error': 0.2}))\n", + encoding="utf-8", + ) + return task_dir + + +def make_expectation(quantity="combined_score"): + record = ls.append_expectation( + { + "id": ls.new_id("exp"), + "task": "evolve-lr", + "created_at": ls.now_iso(), + "quantity": quantity, + "claim": "the evolved variant beats the baseline", + "predicted": {"low": None, "high": None, "direction": "increase"}, + "basis": [], + "comparability": "", + "confidence": "medium", + } + ) + return record["id"] + + +def run_args(task_dir, expectation_id, **overrides): + base = dict( + task_dir=str(task_dir), expect=expectation_id, project=None, generations=2, + population=2, estimate_per_candidate_usd=0.0, local=True, remote=False, + overrides=[], timeout_s=30, json=True, + ) + base.update(overrides) + return argparse.Namespace(**base) + + +class FakeMutator: + """A Shinka stand-in. Returns deterministic mutations inside the block.""" + + def __init__(self, *, escape_at=None): + self.escape_at = escape_at + self.calls: list[int] = [] + + def propose(self, *, generation, population, best): + self.calls.append(generation) + out = [] + for i in range(population): + if self.escape_at == generation and i == 0: + out.append(BASELINE.replace("import json", "import json\nimport os")) + else: + out.append(BASELINE.replace("return x * 2", f"return x * {generation + i + 2}")) + return out + + +def test_the_gate_refuses_before_generation_zero(workspace): + """"Before generation 0, refuse unless estimate_per_candidate x + max_candidates fits under the project's remaining allocation." """ + budget.create("proj-1", title="t", budget={"gpu_usd": 1.0}) + task_dir = scaffold(workspace) + args = run_args( + task_dir, make_expectation(), project="proj-1", + generations=10, population=10, estimate_per_candidate_usd=1.0, + ) + with pytest.raises(GateRefusal) as exc: + evolve.cmd_run(args) + assert exc.value.exit_code == EXIT_PROJECT_BUDGET + # And nothing was started. + assert camp.campaigns() == {} + + +def test_candidate_cost_counts_against_the_project(workspace, monkeypatch): + """Candidates live outside runs.jsonl by design, so the ceiling has to reach + into candidates.jsonl or a campaign is invisible to the budget bounding it.""" + budget.create("proj-1", title="t", budget={"gpu_usd": 100.0}) + task_dir = scaffold(workspace) + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator()) + evolve.cmd_run( + run_args(task_dir, make_expectation(), project="proj-1", + generations=2, population=2, estimate_per_candidate_usd=0.5) + ) + spend = budget.spend("proj-1") + assert spend["candidates"] == 4 + assert spend["gpu_usd"] == 2.0 + + +def test_the_gate_stops_a_runaway_campaign_at_a_generation_boundary(workspace, monkeypatch): + """The dangerous case, and the reason this gate exists at all. + + Here the allocation is consumed *during* the campaign by a concurrent + submission, which is the realistic way headroom disappears mid-run. The + campaign notices at the next generation boundary and stops cleanly, with + every candidate collected and the campaign marked `exhausted`. + + The failure this replaces: being killed mid-flight at generation 40, leaving + a run in flight that goes stale and then blocks *every* future submission + through the §6 gate (exit 7). Succeeding at the search would brick the + system. + """ + budget.create("proj-1", title="t", budget={"gpu_usd": 10.0}) + task_dir = scaffold(workspace) + + class ConcurrentSpender(FakeMutator): + """Something else eats the allocation after the first generation.""" + + def propose(self, *, generation, population, best): + if generation == 1: + ls.append_run_event( + { + "type": ls.T_RUN_SUBMITTED, "id": ls.new_id("run"), + "status": "in_flight", "submitted_at": ls.now_iso(), + "project": "proj-1", "estimate_usd": 9.0, + } + ) + return super().propose(generation=generation, population=population, best=best) + + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: ConcurrentSpender()) + result = evolve.cmd_run( + run_args(task_dir, make_expectation(), project="proj-1", + generations=5, population=2, estimate_per_candidate_usd=0.5) + ) + + assert result["status"] == "exhausted" + assert "allocation" in result["reason"] + # It ran, then stopped -- rather than refusing outright or running to the end. + assert 0 < result["candidates_evaluated"] < 10 + # Nothing the campaign started is left in flight: it stopped at a boundary, + # not mid-candidate. + assert camp.campaign(result["campaign"])["status"] == "exhausted" + assert all(r.get("metrics") or r.get("error") for r in camp.candidates(result["campaign"])) + + +def test_an_unpriced_campaign_says_so_rather_than_implying_a_check(workspace, monkeypatch): + budget.create("proj-1", title="t", budget={"gpu_usd": 1.0}) + task_dir = scaffold(workspace) + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator()) + result = evolve.cmd_run( + run_args(task_dir, make_expectation(), project="proj-1", estimate_per_candidate_usd=0.0) + ) + assert result["status"] == "closed" + + +# --------------------------------------------------------------------------- +# the campaign is the unit of prediction +# --------------------------------------------------------------------------- +def test_a_campaign_binds_exactly_one_expectation(workspace, monkeypatch): + """§7's rule, unchanged: an expectation that can be reused is an expectation + that can be authored after the fact.""" + task_dir = scaffold(workspace) + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator()) + expectation_id = make_expectation() + + evolve.cmd_run(run_args(task_dir, expectation_id)) + with pytest.raises(GateRefusal) as exc: + evolve.cmd_run(run_args(task_dir, expectation_id)) + assert exc.value.code == "expectation_bound" + + +def test_a_missing_expectation_refuses(workspace): + task_dir = scaffold(workspace) + with pytest.raises(GateRefusal) as exc: + evolve.cmd_run(run_args(task_dir, "exp-does-not-exist")) + assert exc.value.code == "expectation_missing" + + +def test_candidates_do_not_enter_runs_jsonl(workspace, monkeypatch): + """§23 item 4: "a 100-generation campaign is thousands of rows". They live + in candidates.jsonl and only a promoted one becomes a run.""" + task_dir = scaffold(workspace) + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator()) + result = evolve.cmd_run(run_args(task_dir, make_expectation())) + + assert result["candidates_evaluated"] == 4 + assert len(camp.candidates(result["campaign"])) == 4 + assert ls.runs() == [] + assert camp.candidates_path().exists() + + +def test_candidates_are_exempt_from_the_per_run_expectation_gate(workspace, monkeypatch): + """Four candidates, one expectation. That is the 1:N resolution.""" + task_dir = scaffold(workspace) + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator()) + result = evolve.cmd_run(run_args(task_dir, make_expectation())) + record = camp.campaign(result["campaign"]) + assert record["expectation_id"] + assert len(camp.candidates(result["campaign"])) > 1 + + +# --------------------------------------------------------------------------- +# evaluation +# --------------------------------------------------------------------------- +def test_an_escaping_candidate_is_recorded_but_not_evaluated(workspace, monkeypatch): + """"candidates run --only tests,dry_run -- both local, both fast. Smoke is + required [...] whenever a mutation escapes the evolve-block." Evaluating it + anyway would mean a paid remote smoke run per candidate.""" + task_dir = scaffold(workspace) + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator(escape_at=0)) + result = evolve.cmd_run(run_args(task_dir, make_expectation(), generations=1, population=2)) + + rows = camp.candidates(result["campaign"]) + escaped = [r for r in rows if r["escaped_block"]["escaped"]] + assert len(escaped) == 1 + assert escaped[0]["skipped"] is True + assert "smoke" in escaped[0]["error"] + assert escaped[0].get("metrics") is None + + +def test_metrics_without_combined_score_are_refused(workspace, monkeypatch): + """"a candidate that silently reports no score is indistinguishable from one + that scored zero, and the search would optimise toward the fallback.""" + task_dir = scaffold( + workspace, evaluate_body="import json\nprint(json.dumps({'accuracy': 0.9}))\n" + ) + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator()) + result = evolve.cmd_run(run_args(task_dir, make_expectation(), generations=1, population=1)) + row = camp.candidates(result["campaign"])[0] + assert row["metrics"] is None + assert "combined_score" in row["error"] + + +def test_a_crashing_evaluator_is_recorded_not_fatal(workspace, monkeypatch): + task_dir = scaffold(workspace, evaluate_body="raise SystemExit(3)\n") + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator()) + result = evolve.cmd_run(run_args(task_dir, make_expectation(), generations=1, population=1)) + assert result["status"] == "closed" + assert camp.candidates(result["campaign"])[0]["error"] + + +def test_validate_metrics_rejects_a_boolean_score(): + assert camp.validate_metrics({"combined_score": True}) is not None + assert camp.validate_metrics({"combined_score": 1.0}) is None + + +# --------------------------------------------------------------------------- +# Goodhart +# --------------------------------------------------------------------------- +def test_status_surfaces_top_k_not_the_argmax(workspace, monkeypatch): + """"A search optimising a scalar will find the bug in the metric." """ + task_dir = scaffold( + workspace, + evaluate_body=( + "import json, pathlib\n" + "src = pathlib.Path('initial.py').read_text()\n" + "score = float(src.split('return x * ')[1].split('\\n')[0])\n" + "print(json.dumps({'combined_score': score}))\n" + ), + ) + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator()) + result = evolve.cmd_run(run_args(task_dir, make_expectation(), generations=2, population=2)) + + assert len(result["top"]) > 1 + scores = [t["metrics"]["combined_score"] for t in result["top"]] + assert scores == sorted(scores, reverse=True) + assert "not a result until" in result["goodhart_note"] + + +def test_promote_writes_the_source_but_no_run_record(workspace, monkeypatch): + """"The campaign winner goes through the normal verdict path before it + counts as a result." Promotion must not shortcut preflight or the ledger.""" + task_dir = scaffold(workspace) + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator()) + result = evolve.cmd_run(run_args(task_dir, make_expectation())) + best = result["top"][0]["candidate_id"] + + promoted = evolve.cmd_promote( + argparse.Namespace(campaign=result["campaign"], candidate=best, into=None, json=True) + ) + assert (task_dir / "promoted.py").exists() + assert ls.runs() == [], "promotion must not write a run record" + assert any("preflight" in step for step in promoted["next"]) + assert any("expect" in step for step in promoted["next"]) + + +def test_promoting_an_unevaluated_candidate_refuses(workspace, monkeypatch): + task_dir = scaffold(workspace) + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator(escape_at=0)) + result = evolve.cmd_run(run_args(task_dir, make_expectation(), generations=1, population=1)) + escaped = camp.candidates(result["campaign"])[0] + + with pytest.raises(GradError) as exc: + evolve.cmd_promote( + argparse.Namespace(campaign=result["campaign"], candidate=escaped["candidate_id"], + into=None, json=True) + ) + assert exc.value.code == "candidate_unevaluated" + + +# --------------------------------------------------------------------------- +# phasing and scaffolding +# --------------------------------------------------------------------------- +def test_remote_is_refused_in_phase_one(workspace): + """"Do not run a single remote generation before this exists." """ + task_dir = scaffold(workspace) + with pytest.raises(UsageError) as exc: + evolve.cmd_run(run_args(task_dir, make_expectation(), remote=True)) + assert "phase 2" in str(exc.value) + + +def test_a_task_without_markers_is_refused(workspace): + task_dir = scaffold(workspace) + (task_dir / "initial.py").write_text("def solve(x): return x\n", encoding="utf-8") + with pytest.raises(UsageError) as exc: + evolve.cmd_run(run_args(task_dir, make_expectation())) + assert "EVOLVE-BLOCK" in str(exc.value) + + +def test_init_scaffolds_a_valid_task(workspace): + result = evolve.cmd_init( + argparse.Namespace(task_dir="pipeline/e", force=False, json=True) + ) + assert len(result["written"]) == 2 + source = (paths.root() / "pipeline" / "e" / "initial.py").read_text(encoding="utf-8") + assert camp.has_markers(source) + assert "combined_score" in (paths.root() / "pipeline" / "e" / "evaluate.py").read_text( + encoding="utf-8" + ) + + +def test_the_default_models_are_an_ensemble(workspace): + """"collapsing to a single model discards diversity the algorithm is built + around." Sonnet 5 primary plus Haiku 4.5 explorer.""" + cfg = config_mod.load(reload=True) + models = evolve._models(cfg, []) + assert "claude-sonnet-5" in models + assert "claude-haiku-4-5" in models + + +def test_shinkas_own_override_mechanism_wins(workspace): + cfg = config_mod.load(reload=True) + models = evolve._models(cfg, ["evo.llm_models=a,b,c"]) + assert models == ("a", "b", "c") + + +def test_capabilities_answers_the_driver_or_fork_question(workspace): + """§23 item 1, answered against the installed package rather than a + document.""" + report = evolve.mutator_capabilities() + assert "installed" in report + if not report["installed"]: + assert "shinka" in report["reason"] diff --git a/tests/test_lab_and_wiki.py b/tests/test_lab_and_wiki.py new file mode 100644 index 0000000..32d600f --- /dev/null +++ b/tests/test_lab_and_wiki.py @@ -0,0 +1,210 @@ +"""The Lab surface (§19) and RepoWiki (§20). + +Both are thin wrappers around external processes, so what is tested here is the +part that is *ours*: the rules the wrappers exist to enforce. For Lab that is +the framing configuration and the kernel-ownership discipline; for RepoWiki it +is the scope allowlist and the staleness check. + +Neither test starts a real server or a real scan. §24's discipline holds: no +network, no external process. +""" + +from __future__ import annotations + +import argparse +import json + +import pytest + +from core import jsonl, paths +from core.errors import ConfigError, GradError, UsageError +from tools import lab, wiki + + +# --------------------------------------------------------------------------- +# §19: JupyterLab +# --------------------------------------------------------------------------- +def test_the_jupyter_config_permits_framing_from_the_app_origin(workspace): + """"JupyterLab ships X-Frame-Options / CSP headers that block embedding." + Getting this wrong costs an afternoon and produces a blank iframe.""" + source = (paths.root().parent / "config" / "jupyter" / "jupyter_server_config.py") + if not source.exists(): + source = _repo_root() / "config" / "jupyter" / "jupyter_server_config.py" + text = source.read_text(encoding="utf-8") + + assert "tornado_settings" in text + assert "frame-ancestors" in text + assert "GRAD_UI_ORIGIN" in text + # Scoped to the app, never to `*`: Lab can execute code as this user. + assert "frame-ancestors *" not in text + + +def test_lab_binds_to_localhost_only(workspace): + text = (_repo_root() / "config" / "jupyter" / "jupyter_server_config.py").read_text( + encoding="utf-8" + ) + assert '"127.0.0.1"' in text + assert "allow_remote_access = False" in text + + +def test_the_kernel_ownership_rule_is_recorded_where_it_is_needed(workspace): + """"Two owners over one notebook reproduces exactly the 'works in the kernel + that grew it' failure that `nb.py verify` exists to catch." The rule must be + visible in the module that creates the second owner.""" + assert "nb verify" in lab.__doc__ + assert "nb.py" in ( + _repo_root() / "config" / "jupyter" / "jupyter_server_config.py" + ).read_text(encoding="utf-8") + + +def test_status_reports_not_running_before_a_start(workspace): + result = lab.cmd_status(argparse.Namespace(json=True)) + assert result["running"] is False + assert "tools.lab start" in result["fix"] + + +def test_the_token_is_not_in_the_status_payload(workspace): + """A status output is the sort of thing that ends up in a screenshot.""" + jsonl.write_json( + paths.data_dir() / "lab" / "lab.json", + {"port": 8889, "token": "super-secret", "pid": 1, "url": "http://127.0.0.1:8889/lab"}, + ) + payload = lab.cmd_status(argparse.Namespace(json=True)) + assert "token" not in payload + assert payload["token_available"] is True + assert "super-secret" not in json.dumps(payload) + + +def test_url_includes_the_token_because_the_iframe_needs_it(workspace): + jsonl.write_json( + paths.data_dir() / "lab" / "lab.json", + {"port": 8889, "token": "tok123", "pid": 1}, + ) + result = lab.cmd_url(argparse.Namespace(path="notebooks/a.ipynb", json=True)) + assert result["url"].endswith("?token=tok123") + assert "notebooks/a.ipynb" in result["url"] + + +def test_url_refuses_before_a_start(workspace): + with pytest.raises(GradError) as exc: + lab.cmd_url(argparse.Namespace(path=None, json=True)) + assert exc.value.code == "lab_not_started" + + +def test_notebook_edit_stays_denied(workspace): + """"This item is about the *human* editing by hand; the agent continues to + edit notebooks through Write/Edit plus nb.py." """ + import agent + + assert "NotebookEdit" in agent.DENIED_TOOLS + assert "Task" in agent.DENIED_TOOLS + + +def test_the_lab_extra_pins_exactly(workspace): + """"Pin `jupyterlab` itself and every extension, or an unrelated + `pip install -U` takes the app down." The JupyterLab 3->4 break is what + killed the Tabnine extension.""" + import tomllib + + doc = tomllib.loads((_repo_root() / "pyproject.toml").read_text(encoding="utf-8")) + pins = doc["project"]["optional-dependencies"]["lab"] + assert pins + assert any(p.startswith("jupyterlab==") for p in pins), "JupyterLab itself must be pinned" + for pin in pins: + assert "==" in pin, f"{pin} is not pinned exactly" + + +# --------------------------------------------------------------------------- +# §20: RepoWiki +# --------------------------------------------------------------------------- +def test_the_scope_is_an_allowlist(workspace): + """"**Never** `ledger/`, `notes/`, or any papers directory -- it ships + content to a third party, and those hold research data." """ + assert wiki.SCOPE == ("core", "tools") + for forbidden in ("ledger", "notes", "data", "figures", "evals"): + assert forbidden not in wiki.SCOPE + + +def test_the_source_hash_covers_only_the_scope(workspace): + (workspace / "core").mkdir(parents=True, exist_ok=True) + (workspace / "core" / "a.py").write_text("x = 1\n", encoding="utf-8") + (workspace / "notes").mkdir(parents=True, exist_ok=True) + (workspace / "notes" / "secret.py").write_text("y = 2\n", encoding="utf-8") + + digest = wiki.source_hash(workspace) + assert "core/a.py" in digest["files"] + assert not any("notes" in k for k in digest["files"]) + + +def test_the_source_hash_moves_when_the_code_does(workspace): + (workspace / "core").mkdir(parents=True, exist_ok=True) + target = workspace / "core" / "a.py" + target.write_text("x = 1\n", encoding="utf-8") + before = wiki.source_hash(workspace)["hash"] + target.write_text("x = 2\n", encoding="utf-8") + assert wiki.source_hash(workspace)["hash"] != before + + +def test_research_data_changing_does_not_invalidate_the_wiki(workspace): + (workspace / "core").mkdir(parents=True, exist_ok=True) + (workspace / "core" / "a.py").write_text("x = 1\n", encoding="utf-8") + before = wiki.source_hash(workspace)["hash"] + (workspace / "notes").mkdir(parents=True, exist_ok=True) + (workspace / "notes" / "log.md").write_text("today I learned\n", encoding="utf-8") + assert wiki.source_hash(workspace)["hash"] == before + + +def test_check_refuses_when_no_wiki_exists(workspace): + with pytest.raises(GradError) as exc: + wiki.cmd_check(argparse.Namespace(json=True)) + assert exc.value.code == "no_wiki" + + +def test_check_detects_staleness_and_names_the_files(workspace): + """"A wiki behind the code is worse than none, because it is trusted." """ + (workspace / "core").mkdir(parents=True, exist_ok=True) + target = workspace / "core" / "a.py" + target.write_text("x = 1\n", encoding="utf-8") + + jsonl.write_json( + wiki.output_dir() / "manifest.json", + {"generated_at": "2026-08-14T00:00:00Z", "source": wiki.source_hash(workspace), + "output_dir": str(wiki.output_dir())}, + ) + assert wiki.cmd_check(argparse.Namespace(json=True))["current"] is True + + target.write_text("x = 2\n", encoding="utf-8") + with pytest.raises(GradError) as exc: + wiki.cmd_check(argparse.Namespace(json=True)) + assert exc.value.code == "wiki_stale" + assert "core/a.py" in exc.value.detail["changed"] + + +def test_scan_is_refused_with_the_reasoning(workspace): + """"RepoWiki reads ANTHROPIC_API_KEY by default, which is exactly what + `credentials.scrub_environment()` deletes." """ + with pytest.raises(UsageError) as exc: + wiki.cmd_scan(argparse.Namespace(json=True)) + assert "ANTHROPIC_API_KEY" in str(exc.value) + assert "map" in (exc.value.fix or "") + + +def test_wiki_is_not_in_the_agents_tool_list(workspace): + """"Scope: human-facing only. Not in the agent's tool list, not in + prompts/system.md, no context cost." """ + system = (_repo_root() / "prompts" / "system.md").read_text(encoding="utf-8") + assert "tools.wiki" not in system + assert "repowiki" not in system.lower() + + +def test_a_missing_repowiki_names_the_extra(workspace, monkeypatch): + monkeypatch.setattr(wiki.shutil, "which", lambda name: None) + with pytest.raises(ConfigError) as exc: + wiki._repowiki() + assert "[wiki]" in (exc.value.fix or "") + + +def _repo_root(): + from pathlib import Path + + return Path(__file__).resolve().parent.parent diff --git a/tests/test_models.py b/tests/test_models.py new file mode 100644 index 0000000..1fef491 --- /dev/null +++ b/tests/test_models.py @@ -0,0 +1,95 @@ +"""Models by role (HANDOFF-2 §16). + +Cheap tests for a cheap change, but the resolution order carries a real +promise -- "`core/config.py` keeps the old `[retrieval] triage_model` / +`expand_model` keys readable as overrides for one release so existing configs do +not break" -- and a promise nothing checks is a promise that breaks silently on +the next edit. +""" + +from __future__ import annotations + +import pytest + +from core import config as config_mod +from core.errors import ConfigError + + +def write_config(workspace, text: str): + path = workspace / "config" / "grad.toml" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + config_mod._cache.clear() + return config_mod.load(path, reload=True) + + +def test_defaults_are_the_claude_5_family(workspace): + cfg = write_config(workspace, "") + assert cfg.model_for("research") == "claude-opus-5" + assert cfg.model_for("evolve") == "claude-sonnet-5" + assert cfg.model_for("report") == "claude-opus-5" + # There is no Haiku 5; 4.5 is the latest. + assert cfg.model_for("expand") == "claude-haiku-4-5" + assert cfg.model_for("triage") == "claude-haiku-4-5" + assert cfg.model_for("cite") == "claude-haiku-4-5" + + +def test_the_opus_4_5_default_is_gone(workspace): + cfg = write_config(workspace, "") + assert "claude-opus-4-5" not in cfg.models().values() + + +def test_explicit_models_entry_wins(workspace): + cfg = write_config(workspace, '[models]\nresearch = "claude-sonnet-5"\n') + assert cfg.model_for("research") == "claude-sonnet-5" + + +def test_legacy_keys_still_resolve_for_one_release(workspace): + """An existing config must not break.""" + cfg = write_config( + workspace, + '[agent]\nmodel = "claude-opus-4-5"\n' + '[retrieval]\ntriage_model = "old-haiku"\nexpand_model = "older-haiku"\n', + ) + assert cfg.model_for("research") == "claude-opus-4-5" + assert cfg.model_for("triage") == "old-haiku" + assert cfg.model_for("expand") == "older-haiku" + + +def test_an_explicit_models_entry_beats_a_legacy_key(workspace): + cfg = write_config( + workspace, + '[models]\nresearch = "claude-opus-5"\n[agent]\nmodel = "claude-opus-4-5"\n', + ) + assert cfg.model_for("research") == "claude-opus-5" + + +def test_unknown_role_in_config_is_a_config_error(workspace): + """A silently ignored setting is worse than a refusal: the model it names is + never used and nothing says so.""" + with pytest.raises(ConfigError) as exc: + write_config(workspace, '[models]\nrerank = "voyageai/rerank-2.5"\n') + assert "not a model role" in str(exc.value) + + +def test_non_string_model_is_a_config_error(workspace): + with pytest.raises(ConfigError): + write_config(workspace, "[models]\nresearch = 5\n") + + +def test_unknown_role_lookup_raises(workspace): + cfg = write_config(workspace, "") + with pytest.raises(ConfigError): + cfg.model_for("nonexistent") + + +def test_rerank_and_embed_stay_in_retrieval(workspace): + """§16 is explicit that these must not move: a different provider on a + different billing rail, and folding them in invites swapping Voyage for + Haiku -- which is worse at the task and moves load onto the scarcer + resource.""" + cfg = write_config(workspace, "") + assert cfg.get("retrieval", "rerank_model") == "voyageai/rerank-2.5" + assert cfg.get("retrieval", "embed_model") == "voyage-4" + assert "rerank" not in config_mod.MODEL_ROLES + assert "embed" not in config_mod.MODEL_ROLES diff --git a/tests/test_namespace.py b/tests/test_namespace.py new file mode 100644 index 0000000..2b35cf4 --- /dev/null +++ b/tests/test_namespace.py @@ -0,0 +1,287 @@ +"""HF Jobs under an organization namespace (HANDOFF-2 §17). + +The trap this section exists to avoid is worth restating, because it is the +reason most of these tests are about the *handle* rather than about the submit +call: + + "`namespace` is a property of the job handle, not a submit-time parameter. + Adding it only to `run_job` produces a job that cannot be found again: + `inspect_job` and `fetch_job_logs` would look under the personal namespace + and 404. The run never collects, goes stale, and then blocks *every* future + submission through the §6 stale-run gate (exit 7). The failure appears far + from its cause." + +So the tests that matter check that the namespace is persisted and re-read, not +that it was passed once. +""" + +from __future__ import annotations + +import types + +import pytest + +from core import budget, config as config_mod, ledger_store as ls, submit as submit_lib +from core.errors import ConfigError, UpstreamError +from tests.test_gates import make_submission + + +class FakeHub: + """Records what every call was asked to look at. + + Deliberately mimics the real trap: a job submitted under a namespace is + *only* findable when the same namespace comes back, so a driver that forgets + to thread it gets a 404 here exactly as it would in production. + """ + + def __init__(self, *, user="me", orgs=("myorg", "otherorg")): + self.user = user + self.orgs = list(orgs) + self.jobs: dict[tuple[str | None, str], dict] = {} + self.calls: list[tuple[str, str | None]] = [] + + def whoami(self, token=None, **_): + return {"name": self.user, "orgs": [{"name": o} for o in self.orgs]} + + def run_job(self, *, image, command, flavor=None, env=None, secrets=None, + token=None, timeout=None, namespace=None, **_): + job_id = f"job-{len(self.jobs)}" + self.jobs[(namespace, job_id)] = {"id": job_id, "status": {"stage": "COMPLETED"}} + self.calls.append(("run_job", namespace)) + return types.SimpleNamespace(id=job_id) + + def inspect_job(self, *, job_id, token=None, namespace=None, **_): + self.calls.append(("inspect_job", namespace)) + found = self.jobs.get((namespace, job_id)) + if found is None: + raise RuntimeError(f"404: no job {job_id} under namespace {namespace!r}") + return types.SimpleNamespace( + id=job_id, status=types.SimpleNamespace(stage="COMPLETED"), + started_at=None, created_at=None, ended_at=None, + ) + + def fetch_job_logs(self, *, job_id, token=None, namespace=None, **_): + self.calls.append(("fetch_job_logs", namespace)) + if (namespace, job_id) not in self.jobs: + raise RuntimeError(f"404: no job {job_id} under namespace {namespace!r}") + return ["line one"] + + +@pytest.fixture +def hub(monkeypatch): + from tools import jobs + + fake = FakeHub() + monkeypatch.setattr(jobs, "_hub", lambda: fake) + monkeypatch.setattr(jobs, "_token", lambda: "tok") + return fake + + +def cfg(): + return config_mod.load(reload=True) + + +# --------------------------------------------------------------------------- +# resolution order +# --------------------------------------------------------------------------- +def test_resolution_order_flag_beats_everything(workspace, hub): + from tools import jobs + + sub = make_submission(workspace) + sub.target["namespace"] = "from-spec" + budget.create("proj-1", title="t", budget={}, payer="hf:from-project") + assert jobs.resolve_namespace("from-flag", sub, cfg(), "proj-1") == "from-flag" + + +def test_resolution_order_spec_beats_project(workspace, hub): + from tools import jobs + + sub = make_submission(workspace) + sub.target["namespace"] = "from-spec" + budget.create("proj-1", title="t", budget={}, payer="hf:from-project") + assert jobs.resolve_namespace(None, sub, cfg(), "proj-1") == "from-spec" + + +def test_resolution_order_project_payer_is_used(workspace, hub): + from tools import jobs + + sub = make_submission(workspace) + budget.create("proj-1", title="t", budget={}, payer="hf:from-project") + assert jobs.resolve_namespace(None, sub, cfg(), "proj-1") == "from-project" + + +def test_resolution_falls_through_to_personal(workspace, hub): + from tools import jobs + + sub = make_submission(workspace) + assert jobs.resolve_namespace(None, sub, cfg(), None) is None + + +# --------------------------------------------------------------------------- +# membership validation +# --------------------------------------------------------------------------- +def test_a_namespace_the_token_cannot_act_for_is_refused(workspace, hub): + from tools import jobs + + with pytest.raises(ConfigError) as exc: + jobs.validate_namespace("not-my-org", "tok") + assert "cannot act for" in str(exc.value) + assert "myorg" in str(exc.value) + + +def test_an_org_the_token_belongs_to_passes(workspace, hub): + from tools import jobs + + identity = jobs.validate_namespace("myorg", "tok") + assert identity["user"] == "me" + assert "myorg" in identity["orgs"] + + +def test_the_users_own_namespace_passes(workspace, hub): + from tools import jobs + + assert jobs.validate_namespace("me", "tok")["namespace"] == "me" + + +def test_a_whoami_failure_is_an_upstream_error(workspace, monkeypatch): + from tools import jobs + + class Broken(FakeHub): + def whoami(self, token=None, **_): + raise RuntimeError("network down") + + monkeypatch.setattr(jobs, "_hub", lambda: Broken()) + with pytest.raises(UpstreamError): + jobs.validate_namespace("myorg", "tok") + + +def test_validation_runs_before_any_record_exists(workspace, hub, monkeypatch): + """"a configuration problem must not leave a phantom estimate sitting on the + ceiling." A bad namespace must produce no run record at all.""" + import argparse + + from tools import jobs + from tests.test_gates import make_expectation, pass_preflight + + sub = make_submission(workspace) + pass_preflight(sub) + expectation_id = make_expectation() + before = len(ls.runs()) + + args = argparse.Namespace( + spec=str(sub.spec_path), expect=expectation_id, overrides=[], flavor=None, + task=None, project=None, namespace="not-my-org", smoke=False, + no_digest=True, json=True, + ) + with pytest.raises(ConfigError) as exc: + jobs.cmd_submit(args) + assert "cannot act for" in str(exc.value) + # The four gates passed, so nothing but the namespace stopped this -- and + # still no phantom estimate landed on the ceiling. + assert len(ls.runs()) == before + + +# --------------------------------------------------------------------------- +# the handle -- the part that actually matters +# --------------------------------------------------------------------------- +def test_the_namespace_is_persisted_onto_the_handle(workspace, hub): + """Not merely passed to run_job. This is the whole point of §17.""" + submit_lib.attach_handle("run-1", {"job_id": "job-0", "flavor": "t4-small", "namespace": "myorg"}) + events = [e for e in ls.runs_events() if e.get("type") == "run_handle"] + assert events[-1]["handle"]["namespace"] == "myorg" + + +def test_collect_reads_the_namespace_from_the_handle(workspace, hub, monkeypatch): + """A job submitted to an org is collectable from that org -- even if the + config or the current project changed in between.""" + import argparse + + from tools import jobs + + ls.append_run_event( + { + "type": ls.T_RUN_SUBMITTED, "id": "run-1", "status": "in_flight", + "submitted_at": ls.now_iso(), "project": "proj-1", "estimate_usd": 1.0, + "metrics_file": "metrics.json", "expectation_id": None, + "target": {"flavor": "t4-small", "namespace": "myorg"}, + } + ) + hub.jobs[("myorg", "job-0")] = {"id": "job-0"} + submit_lib.attach_handle("run-1", {"job_id": "job-0", "flavor": "t4-small", "namespace": "myorg"}) + + args = argparse.Namespace(run_id="run-1", wait=False, timeout=1, json=True) + jobs.cmd_collect(args) + + looked_under = {ns for call, ns in hub.calls if call in ("inspect_job", "fetch_job_logs")} + assert looked_under == {"myorg"}, "collect must look under the submitted namespace" + + +def test_a_personal_job_passes_no_namespace_at_all(workspace, hub): + """Omitted rather than passed as None, so an older huggingface_hub without + the parameter still works for personal jobs.""" + from tools import jobs + + assert jobs._ns_kwargs(None) == {} + assert jobs._ns_kwargs("myorg") == {"namespace": "myorg"} + + +def test_a_hub_without_namespace_support_refuses_loudly(workspace, monkeypatch): + """Silently dropping the namespace would produce exactly the uncollectable + job this section exists to prevent.""" + from tools import jobs + + class Old: + def run_job(self, *, image, command, token=None): # no namespace + ... + + def inspect_job(self, *, job_id, token=None): + ... + + def fetch_job_logs(self, *, job_id, token=None): + ... + + monkeypatch.setattr(jobs, "_hub", lambda: Old()) + with pytest.raises(ConfigError) as exc: + jobs._ns_kwargs("myorg") + assert "could not be collected" in str(exc.value) + + +# --------------------------------------------------------------------------- +# the smoke/submit mismatch +# --------------------------------------------------------------------------- +def test_a_smoke_namespace_mismatch_warns_rather_than_refuses(workspace, hub): + """"Warn, not refuse -- consistent with how `target` and `flavor` already + behave." The hash excludes `target`, so namespace follows the same rule.""" + from tools import jobs, preflight + + sub = make_submission(workspace) + preflight.record_check_result(sub.hash(), "smoke", {"ok": True, "namespace": None}) + + warnings = jobs._namespace_warnings(sub, "myorg") + assert len(warnings) == 1 + assert "personal" in warnings[0] and "myorg" in warnings[0] + + +def test_no_warning_when_the_namespaces_agree(workspace, hub): + from tools import jobs, preflight + + sub = make_submission(workspace) + preflight.record_check_result(sub.hash(), "smoke", {"ok": True, "namespace": "myorg"}) + assert jobs._namespace_warnings(sub, "myorg") == [] + + +def test_no_warning_when_smoke_predates_the_namespace_field(workspace, hub): + """An older preflight record has no `namespace` key; that is not a mismatch.""" + from tools import jobs, preflight + + sub = make_submission(workspace) + preflight.record_check_result(sub.hash(), "smoke", {"ok": True}) + assert jobs._namespace_warnings(sub, "myorg") == [] + + +def test_the_namespace_is_not_part_of_the_submission_hash(workspace): + """Consistent with `flavor`: the hash deliberately excludes `target`.""" + sub = make_submission(workspace) + before = sub.hash() + sub.target["namespace"] = "myorg" + assert sub.hash() == before diff --git a/tests/test_report.py b/tests/test_report.py new file mode 100644 index 0000000..4c3aa86 --- /dev/null +++ b/tests/test_report.py @@ -0,0 +1,467 @@ +"""The report and its gate (HANDOFF-2 §22). + +§24: "8 needs a fixture ledger with a known-good and a known-bad claim set." +Both are built here, and every rule `check` enforces is tested from both sides -- +because a gate that has only ever been shown to pass is a gate nobody has +tested. + +The rule worth reading twice is rule 3: + + "no cited run has an unjudged deviation [...] Rule 3 is the one most in the + spirit of this system: **you should not be able to write up a result you + have not judged.**" +""" + +from __future__ import annotations + +import argparse +import json + +import pytest + +from core import budget, ledger_store as ls, report as report_lib +from core.errors import GradError +from tools import report + + +# --------------------------------------------------------------------------- +# fixtures: a real ledger +# --------------------------------------------------------------------------- +def make_run(project="proj-1", *, results, deviations, run_id=None, judged=False): + run_id = run_id or ls.new_id("run") + expectation = ls.append_expectation( + { + "id": ls.new_id("exp"), + "task": "scaling", + "created_at": ls.now_iso(), + "quantity": next(iter(results), "val_loss"), + "claim": "val loss should land between 2.9 and 3.2", + "predicted": {"low": 2.9, "high": 3.2, "direction": None}, + "basis": [{"paper": "arXiv:2001.08361", "locator": "Fig 3", "value": 3.05, + "conditions": "1.3B params"}], + "comparability": "our tokenizer differs", + "confidence": "medium", + } + ) + ls.append_run_event( + { + "type": ls.T_RUN_SUBMITTED, "id": run_id, "task": "scaling", + "status": "in_flight", "submitted_at": ls.now_iso(), + "project": project, "estimate_usd": 1.0, + "expectation_id": expectation["id"], + } + ) + ls.append_run_event( + { + "type": ls.T_RUN_COLLECTED, "id": run_id, "status": "completed", + "collected_at": ls.now_iso(), "cost_usd_actual": 1.0, + "results": results, "deviations": deviations, + } + ) + if judged: + for dev in deviations: + ls.append_run_event( + { + "type": ls.T_VERDICT, "id": run_id, "quantity": dev["quantity"], + "verdict": "real", "note": "checked the schedule", "judged_at": ls.now_iso(), + } + ) + return run_id, expectation["id"] + + +@pytest.fixture +def project(workspace): + budget.create("proj-1", title="scaling study", budget={}) + budget.set_current("proj-1") + return "proj-1" + + +def in_range_run(project_id="proj-1"): + return make_run( + project_id, + results={"val_loss": 3.05}, + deviations=[{"expectation_id": "e", "quantity": "val_loss", "actual": 3.05, + "in_range": True, "expected": {"low": 2.9, "high": 3.2}}], + ) + + +def unjudged_run(project_id="proj-1"): + return make_run( + project_id, + results={"val_loss": 4.10}, + deviations=[{"expectation_id": "e", "quantity": "val_loss", "actual": 4.10, + "in_range": False, "expected": {"low": 2.9, "high": 3.2}}], + ) + + +def args(**kw): + base = dict(project=None, json=True) + base.update(kw) + return argparse.Namespace(**base) + + +@pytest.fixture +def stub_resolver(monkeypatch): + """Resolve every placeholder to one corpus-backed entry. + + The draft emits `[CITE:]` for each basis entry, and `check` refuses + while any placeholder survives -- so a test that wants to exercise a *later* + rule has to run `cite` first, exactly as a real pipeline does. + """ + monkeypatch.setattr( + report, "_resolve_citation", + lambda keyword, context, use_s2: { + "key": "basis2026", "type": "article", "title": "Scaling Laws", + "author": "Kaplan", "year": "2020", "gradsource": "corpus", + }, + ) + + +def draft_and_cite(): + report.cmd_draft(args()) + report.cmd_cite(args(context_chars=200, no_s2=True)) + + +# --------------------------------------------------------------------------- +# draft: deterministic and free +# --------------------------------------------------------------------------- +def test_draft_needs_no_model_and_costs_nothing(workspace, project): + run_id, _ = in_range_run() + result = report.cmd_draft(args()) + tex = report_lib.paths_for(project)["tex"].read_text(encoding="utf-8") + + assert result["claim_count"] == 1 + assert r"\gradnum{" in tex + assert run_id in tex + + +def test_draft_records_every_number_as_a_claim(workspace, project): + run_id, _ = in_range_run() + report.cmd_draft(args()) + claims = report_lib.load_claims(project) + entry = next(iter(claims.values())) + assert entry["run_id"] == run_id + assert entry["quantity"] == "val_loss" + assert entry["value"] == 3.05 + + +def test_draft_surfaces_unjudged_deviations_rather_than_hiding_them(workspace, project): + unjudged_run() + result = report.cmd_draft(args()) + assert result["unjudged"] + tex = report_lib.paths_for(project)["tex"].read_text(encoding="utf-8") + assert "NOT YET JUDGED" in tex + + +def test_draft_lists_runs_with_no_bound_expectation(workspace, project): + """"a skeleton that omits the runs that failed is a skeleton that invites + writing up only the ones that worked." """ + ls.append_run_event( + { + "type": ls.T_RUN_SUBMITTED, "id": "run-orphan", "task": "t", + "status": "in_flight", "submitted_at": ls.now_iso(), "project": "proj-1", + "estimate_usd": 1.0, "expectation_id": None, + } + ) + ls.append_run_event( + { + "type": ls.T_RUN_COLLECTED, "id": "run-orphan", "status": "completed", + "collected_at": ls.now_iso(), "cost_usd_actual": 1.0, + "results": {"x": 1}, "deviations": [], + } + ) + report.cmd_draft(args()) + tex = report_lib.paths_for(project)["tex"].read_text(encoding="utf-8") + assert "run-orphan" in tex + + +def test_draft_on_an_empty_project_says_so_rather_than_inventing(workspace, project): + result = report.cmd_draft(args()) + assert result["claim_count"] == 0 + tex = report_lib.paths_for(project)["tex"].read_text(encoding="utf-8") + assert "empty on purpose" in tex + + +# --------------------------------------------------------------------------- +# rule 1: claims +# --------------------------------------------------------------------------- +def test_a_gradnum_with_no_claims_entry_fails(workspace, project): + findings = report_lib.check_claims(r"loss was \gradnum{missing}.", {}) + assert findings[0]["rule"] == "claims" + assert "no entry in claims.json" in findings[0]["problem"] + + +def test_a_claim_pointing_at_a_nonexistent_run_fails(workspace, project): + findings = report_lib.check_claims( + r"\gradnum{k}", {"k": {"run_id": "run-nope", "quantity": "val_loss"}} + ) + assert "not in the ledger" in findings[0]["problem"] + + +def test_a_claim_naming_a_quantity_the_run_never_reported_fails(workspace, project): + run_id, _ = in_range_run() + findings = report_lib.check_claims( + r"\gradnum{k}", {"k": {"run_id": run_id, "quantity": "perplexity"}} + ) + assert "reports no quantity" in findings[0]["problem"] + + +def test_a_claim_whose_value_does_not_match_the_ledger_fails(workspace, project): + """The failure a citation-checker would miss: a real run, a real quantity, + and a different number in the prose.""" + run_id, _ = in_range_run() + findings = report_lib.check_claims( + r"\gradnum{k}", {"k": {"run_id": run_id, "quantity": "val_loss", "value": 2.10}} + ) + assert "recorded 3.05" in findings[0]["problem"] + + +def test_a_correct_claim_passes(workspace, project): + run_id, _ = in_range_run() + assert report_lib.check_claims( + r"\gradnum{k}", {"k": {"run_id": run_id, "quantity": "val_loss", "value": 3.05}} + ) == [] + + +def test_rounding_in_the_prose_is_tolerated(workspace, project): + run_id, _ = in_range_run() + assert report_lib.check_claims( + r"\gradnum{k}", {"k": {"run_id": run_id, "quantity": "val_loss", "value": 3.0500001}} + ) == [] + + +# --------------------------------------------------------------------------- +# rule 2: citations +# --------------------------------------------------------------------------- +def test_a_cite_key_with_no_bib_entry_fails(workspace): + findings = report_lib.check_citations(r"as shown \cite{ghost}.", {}) + assert findings[0]["rule"] == "citations" + assert "no entry in references.bib" in findings[0]["problem"] + + +def test_a_bib_entry_with_no_verified_provenance_fails(workspace): + """"every bib entry came from the corpus or a verified S2 id." A + hand-written entry is exactly the hallucinated citation this stops.""" + bib = {"invented2026": {"type": "article", "key": "invented2026", "title": "A Paper"}} + findings = report_lib.check_citations(r"\cite{invented2026}", bib) + assert any("verified provenance" in f["problem"] for f in findings) + + +def test_a_corpus_backed_entry_passes(workspace): + bib = {"real2026": {"type": "article", "key": "real2026", "gradsource": "corpus"}} + assert report_lib.check_citations(r"\cite{real2026}", bib) == [] + + +def test_an_s2_verified_entry_passes(workspace): + bib = {"real2026": {"type": "article", "key": "real2026", "gradsource": "s2"}} + assert report_lib.check_citations(r"\cite{real2026}", bib) == [] + + +def test_multi_key_cites_are_all_checked(workspace): + bib = {"a": {"type": "article", "key": "a", "gradsource": "corpus"}} + findings = report_lib.check_citations(r"\cite{a,b}", bib) + assert [f["key"] for f in findings] == ["b"] + + +def test_citep_and_citet_are_recognised(workspace): + findings = report_lib.check_citations(r"\citep{ghost} and \citet{ghost2}", {}) + assert {f["key"] for f in findings} == {"ghost", "ghost2"} + + +def test_the_generated_bib_carries_provenance(workspace): + text = report._render_bib( + {"k": {"type": "article", "key": "k", "title": "T", "author": "A", + "year": "2026", "gradsource": "corpus"}} + ) + parsed = report_lib.parse_bib(text) + assert parsed["k"]["gradsource"] == "corpus" + assert report_lib.check_citations(r"\cite{k}", parsed) == [] + + +def test_an_unresolvable_placeholder_is_left_in_place_and_refused(workspace, project, monkeypatch): + """"a citation quietly deleted is worse than one that fails loudly, because + the sentence it supported survives without support." """ + in_range_run() + report.cmd_draft(args()) + monkeypatch.setattr(report, "_resolve_citation", lambda *a, **k: None) + + with pytest.raises(GradError) as exc: + report.cmd_cite(args(context_chars=200, no_s2=True)) + assert exc.value.code == "citations_unresolved" + + tex = report_lib.paths_for(project)["tex"].read_text(encoding="utf-8") + assert "[CITE:" in tex, "the placeholder must survive so `check` refuses on it" + + +def test_a_resolved_placeholder_becomes_a_cite(workspace, project, monkeypatch): + in_range_run() + report.cmd_draft(args()) + monkeypatch.setattr( + report, "_resolve_citation", + lambda keyword, context, use_s2: { + "key": "real2026", "type": "article", "title": "T", "author": "A", + "year": "2026", "gradsource": "corpus", + }, + ) + result = report.cmd_cite(args(context_chars=200, no_s2=True)) + tex = report_lib.paths_for(project)["tex"].read_text(encoding="utf-8") + assert result["entries"] == 1 + assert r"\cite{real2026}" in tex + assert "[CITE:" not in tex + + +# --------------------------------------------------------------------------- +# rule 3: unjudged deviations -- the one most in the spirit of the system +# --------------------------------------------------------------------------- +def test_a_cited_run_with_an_unjudged_deviation_refuses(workspace, project): + """"You should not be able to write up a result you have not judged." """ + unjudged_run() + report.cmd_draft(args()) + with pytest.raises(GradError) as exc: + report.cmd_check(args()) + detail = exc.value.detail + assert detail["by_rule"]["unjudged"] == 1 + unjudged = [f for f in detail["findings"] if f["rule"] == "unjudged"][0] + assert "tools.ledger verdict" in unjudged["fix"] + + +def test_supplying_the_verdict_clears_the_refusal(workspace, project, stub_resolver): + run_id, _ = make_run( + results={"val_loss": 4.10}, + deviations=[{"expectation_id": "e", "quantity": "val_loss", "actual": 4.10, + "in_range": False, "expected": {"low": 2.9, "high": 3.2}}], + judged=True, + ) + draft_and_cite() + result = report.cmd_check(args()) + assert result["ok"] is True + assert run_id in result["cited_runs"] + + +def test_an_in_range_result_needs_no_verdict(workspace, project, stub_resolver): + in_range_run() + draft_and_cite() + assert report.cmd_check(args())["ok"] is True + + +def test_an_unjudged_run_that_is_not_cited_does_not_block(workspace, project, stub_resolver): + """The rule is about *cited* runs. An unrelated open verdict elsewhere in + the project is `ledger query --pending`'s business, not this report's.""" + in_range_run() + draft_and_cite() + unjudged_run() # after the draft, so no \gradnum points at it + assert report.cmd_check(args())["ok"] is True + + +# --------------------------------------------------------------------------- +# rule 4: LaTeX hygiene +# --------------------------------------------------------------------------- +def test_unmatched_braces_are_found(workspace): + findings = report_lib.check_latex("\\section{Results\n\nsome text\n") + assert any("unmatched opening brace" in f["problem"] for f in findings) + + +def test_a_closing_brace_with_no_opener_is_found(workspace): + findings = report_lib.check_latex("text }\n") + assert any("no opener" in f["problem"] for f in findings) + + +def test_escaped_braces_do_not_count(workspace): + assert report_lib.check_latex(r"a \{ literal \} brace" + "\n") == [] + + +def test_duplicate_labels_are_found(workspace): + findings = report_lib.check_latex("\\label{a}\ntext\n\\label{a}\n") + assert any("duplicate" in f["problem"] for f in findings) + + +def test_leftover_placeholders_are_found(workspace): + findings = report_lib.check_latex("as shown [CITE:transformers].\n") + assert any("unresolved [CITE:" in f["problem"] for f in findings) + + +def test_comments_do_not_confuse_the_brace_counter(workspace): + assert report_lib.check_latex("% a comment with { an unmatched brace\ntext\n") == [] + + +# --------------------------------------------------------------------------- +# the pipeline, and where the gate sits +# --------------------------------------------------------------------------- +def test_check_reports_every_rule_it_ran(workspace, project, stub_resolver): + in_range_run() + draft_and_cite() + result = report.cmd_check(args()) + assert set(result["by_rule"]) == {"claims", "citations", "unjudged", "latex"} + + +def test_check_refuses_while_a_placeholder_survives(workspace, project, monkeypatch): + """The pipeline order is enforced, not assumed: `cite` runs before `check`, + and an unresolved placeholder is a hard failure rather than cosmetic.""" + in_range_run() + report.cmd_draft(args()) + with pytest.raises(GradError) as exc: + report.cmd_check(args()) + assert exc.value.detail["by_rule"]["latex"] == 1 + + +def test_check_exits_9(workspace, project, capsys): + unjudged_run() + report.cmd_draft(args()) + assert report.cli.run(["check", "--project", "proj-1", "--json"]) == 9 + payload = json.loads(capsys.readouterr().out.strip().splitlines()[-1]) + assert payload["ok"] is False + + +def test_check_refuses_it_does_not_warn(workspace, project): + """"`check` refuses; it does not warn. A report generator is where this + system's epistemics either hold or collapse." """ + unjudged_run() + report.cmd_draft(args()) + with pytest.raises(GradError): + report.cmd_check(args()) + + +def test_write_is_denied_while_over_budget(workspace): + """§23 item 5, "currently specified as denied by the §15 hook" -- and the + CLI agrees with the hook rather than contradicting it.""" + budget.create("proj-1", title="t", budget={"gpu_usd": 1.0}) + budget.set_current("proj-1") + in_range_run() + report.cmd_draft(args()) + ls.append_run_event( + { + "type": ls.T_RUN_SUBMITTED, "id": ls.new_id("run"), "status": "in_flight", + "submitted_at": ls.now_iso(), "project": "proj-1", "estimate_usd": 50.0, + } + ) + with pytest.raises(GradError) as exc: + report.cmd_write(args(section=[], dry_run=False)) + assert exc.value.exit_code == 12 + # And it points at the free command that answers "what did the spend buy". + assert "report draft" in (exc.value.fix or "") + + +def test_write_refuses_without_a_draft(workspace, project): + with pytest.raises(GradError) as exc: + report.cmd_write(args(section=[], dry_run=False)) + assert exc.value.code == "no_draft" + + +def test_the_bundle_hands_the_model_keys_not_numbers(workspace, project): + """"handing it more numbers than it has keys for is how a number ends up in + the prose without a \\gradnum around it." """ + in_range_run() + report.cmd_draft(args()) + result = report.cmd_write(args(section=[], dry_run=True)) + bundle = result["bundle"] + assert result["sent"] is False + for entry in bundle["claims"].values(): + assert "value" not in entry + + +def test_a_report_needs_a_project(workspace): + from core.errors import UsageError + + with pytest.raises(UsageError) as exc: + report.cmd_draft(args()) + assert "tools.budget use" in (exc.value.fix or "") diff --git a/tools/budget.py b/tools/budget.py new file mode 100644 index 0000000..4389099 --- /dev/null +++ b/tools/budget.py @@ -0,0 +1,249 @@ +"""grad-budget -- projects and their ceilings (HANDOFF-2 §15). + + "So README.md's claim that 'cumulative spend stays bounded' holds for one + resource in three. The other two are instrumented and unbounded." + +This closes that. Three resources are consumed -- GPU dollars, API credits, and +subscription tokens -- and all three now carry a ceiling scoped to a project. + +**The honest statement about enforcement, because it differs by resource:** + + * **GPU dollars** are enforced at submit, which is a discrete gateable event. + Refused before anything is spent. + * **API credits** are enforced at the same boundary and measured continuously. + * **Subscription tokens are enforced to a granularity of one turn's overrun.** + Tokens are consumed continuously inside a turn and there is no way to refuse + mid-turn, so `agent.py` checks the remaining allocation *before* issuing the + next turn and `hooks.py` denies cost-bearing Bash once the project is over. + A turn already in flight finishes. + +And a second honesty note, because a meter that overclaims is worse than none: +subscription quota is not linear in tokens, and the real limits are rolling +windows (5-hour and weekly on Max) that the SDK does not expose as a remaining +balance. **A token ceiling here is a proxy you control, not a mirror of +Anthropic's limit.** Hitting the real rate limit is an event this system can +only observe after the fact. + +`raise` appends an event rather than editing the record: a ceiling that can be +changed invisibly is not a ceiling. +""" + +from __future__ import annotations + +import argparse +from typing import Any + +from core import budget, paths +from core.cli import Cli, main +from core.errors import EXIT_PROJECT_BUDGET, UsageError + +cli = Cli( + "grad-budget", + "Create research projects and bound what they may spend: GPU dollars, API " + "credits, and subscription tokens.", + epilog=( + "Exit 12 is a project budget refusal, distinct from 6 (the machine's global\n" + "spend ceiling), so 'this research ran out of its allocation' is never confused\n" + "with 'the machine is out of money'.\n\n" + "Enforcement differs by resource, and the difference is structural:\n" + " gpu_usd refused at submit, before anything is spent\n" + " credits_usd refused at the same boundary\n" + " quota_tokens enforced to a granularity of ONE TURN'S OVERRUN -- tokens are\n" + " consumed continuously inside a turn and there is no way to\n" + " refuse mid-turn, so the check runs before the *next* one\n\n" + "A token ceiling is a proxy you control, not a mirror of Anthropic's limit:\n" + "the real caps are rolling windows the SDK does not expose as a balance." + ), +) + + +# --------------------------------------------------------------------------- +def _budget_flags(p: argparse.ArgumentParser) -> None: + p.add_argument("--gpu-usd", type=float, help="ceiling on GPU dollars for this project") + p.add_argument( + "--quota-tokens", + type=float, + help="ceiling on subscription tokens (accepts 5e6). Enforced to one turn's overrun.", + ) + p.add_argument("--credits-usd", type=float, help="ceiling on API credits (Voyage, OpenRouter)") + + +def _collect_budget(args: argparse.Namespace) -> dict[str, float]: + out: dict[str, float] = {} + if args.gpu_usd is not None: + out["gpu_usd"] = float(args.gpu_usd) + if args.quota_tokens is not None: + # `--quota-tokens 5e6` is the documented spelling, so it parses as a + # float and is stored as a whole number of tokens. + out["quota_tokens"] = float(int(args.quota_tokens)) + if args.credits_usd is not None: + out["credits_usd"] = float(args.credits_usd) + for name, value in out.items(): + if value < 0: + raise UsageError( + f"--{name.replace('_', '-')} must not be negative", + fix="a ceiling of 0 blocks all spend; omit the flag to leave it unbounded", + ) + return out + + +def _new_args(p: argparse.ArgumentParser) -> None: + p.add_argument("--id", required=True, help="short slug, e.g. proj-scaling-w2") + p.add_argument("--title", required=True, help="what this research is") + p.add_argument( + "--payer", + help="who pays. `hf:` attributes HF jobs to that organization namespace (§17)", + ) + p.add_argument("--use", action="store_true", help="also select it as the current project") + _budget_flags(p) + + +@cli.command("new", "create a project and its ceilings", setup=_new_args) +def cmd_new(args: argparse.Namespace) -> dict[str, Any]: + """A project is the unit three separate requirements turned out to share: + HF payer attribution, the bound on an evolutionary campaign, and a budget + for a piece of research.""" + paths.ensure_workspace() + record = budget.create( + args.id, title=args.title, budget=_collect_budget(args), payer=args.payer + ) + if args.use: + budget.set_current(args.id) + return { + "project": record, + "current": budget.current_project(), + "next": f"python -m tools.budget use {args.id} --json" if not args.use else None, + } + + +@cli.command( + "use", + "select the current project (written to ledger/.current_project)", + setup=lambda p: p.add_argument("project_id"), +) +def cmd_use(args: argparse.Namespace) -> dict[str, Any]: + """A file, not an environment variable. + + `credentials.scrub_environment()` strips the agent's environment at startup, + and a selection mechanism that the agent's own startup deletes is a bug + waiting to happen. + """ + proj = budget.project(args.project_id) + if proj["status"] == "closed": + raise UsageError( + f"project {args.project_id!r} is closed", + fix="python -m tools.budget new --id --title '...' --json", + ) + budget.set_current(args.project_id) + return {"current": args.project_id, "title": proj["title"], "payer": proj["payer"]} + + +def _project_arg(p: argparse.ArgumentParser) -> None: + p.add_argument("--project", help="override the current project for this invocation") + + +@cli.command("status", "spend and remaining, per resource", setup=_project_arg) +def cmd_status(args: argparse.Namespace) -> dict[str, Any]: + project_id = budget.resolve_or_fail(args.project, what="status") + state = budget.status(project_id) + if state["over_budget"]: + # Reported as data, not raised: `status` exists to be readable *while* + # over budget. The refusal belongs at the point of spend. + state["blocked"] = [ + "python -m tools.jobs submit", + "python -m tools.evolve run", + "python -m tools.report write", + ] + state["fix"] = ( + f"python -m tools.budget raise --project {project_id} " + f"--{state['over_budget'][0].replace('_', '-')} --json" + ) + return state + + +def _raise_args(p: argparse.ArgumentParser) -> None: + _project_arg(p) + p.add_argument("--reason", default="", help="why the ceiling moved. it ages badly without one") + _budget_flags(p) + + +@cli.command("raise", "move a ceiling, as a logged event", setup=_raise_args) +def cmd_raise(args: argparse.Namespace) -> dict[str, Any]: + """Appends rather than mutates. + + Same argument as §7's append-only ledger: the previous value stays readable, + so "we kept raising it" is visible instead of inferred. + """ + project_id = budget.resolve_or_fail(args.project, what="raise") + record = budget.raise_ceiling(project_id, budget=_collect_budget(args), reason=args.reason) + return {"raised": record, "status": budget.status(project_id)} + + +@cli.command( + "close", + "close a project (its records stay; nothing is deleted)", + setup=lambda p: p.add_argument("project_id"), +) +def cmd_close(args: argparse.Namespace) -> dict[str, Any]: + record = budget.close(args.project_id) + return {"closed": record, "current": budget.current_project()} + + +@cli.command("list", "every project, with spend against its ceilings") +def cmd_list(_: argparse.Namespace) -> dict[str, Any]: + current = budget.current_project() + rows = [] + for pid, proj in budget.projects().items(): + state = budget.status(pid) + rows.append( + { + "id": pid, + "title": proj["title"], + "status": proj["status"], + "payer": proj["payer"], + "current": pid == current, + "resources": state["resources"], + "over_budget": state["over_budget"], + } + ) + return {"current": current, "projects": rows} + + +def _check_args(p: argparse.ArgumentParser) -> None: + _project_arg(p) + p.add_argument("--gpu-usd", type=float, default=0.0, help="dollars this would add") + p.add_argument("--quota-tokens", type=float, default=0.0, help="tokens this would add") + p.add_argument("--credits-usd", type=float, default=0.0, help="credits this would add") + + +@cli.command("check", "would this spend fit? exits 12 if not", setup=_check_args) +def cmd_check(args: argparse.Namespace) -> dict[str, Any]: + """The gate, callable directly. + + `tools.evolve` uses this shape before each generation, and it is here so a + pipeline can ask the question without importing anything. + """ + project_id = budget.resolve(args.project) + state = budget.check( + project_id, + gpu_usd=args.gpu_usd, + quota_tokens=int(args.quota_tokens), + credits_usd=args.credits_usd, + what="the proposed spend", + ) + if state is None: + return { + "project": project_id, + "bounded": False, + "note": "no project selected, or the project has no ceilings; spend is tracked, not bounded", + } + return {"project": project_id, "bounded": True, "fits": True, "resources": state["resources"]} + + +# Re-exported so a caller can `from tools.budget import EXIT_PROJECT_BUDGET` +# instead of remembering the number. +__all__ = ["cli", "EXIT_PROJECT_BUDGET"] + + +if __name__ == "__main__": + main(cli) diff --git a/tools/docs.py b/tools/docs.py new file mode 100644 index 0000000..cc2aca6 --- /dev/null +++ b/tools/docs.py @@ -0,0 +1,450 @@ +"""grad-docs -- is this library call current? (HANDOFF-2 §18) + + "A checker relying on Context7 alone will confidently describe an API + version that is not installed." + +So there are **two oracles, and the order matters**: + +1. **Introspection -- what actually exists on this machine.** + `importlib.metadata.version()`, `inspect.signature()`, `dir()`. Cheap, + offline, definitive. This is how §17's `namespace` parameter was found, in + about ten seconds. +2. **Context7 -- what is current.** Deprecations, changed idioms, migration + paths. Answers what introspection cannot see. + +Introspect first. Always. + +**Why a CLI and not a subagent.** The original plan was a Haiku "reality +checker" with Context7 and Pyright. It was rejected after the brief narrowed, +and the reasoning is worth keeping because it will be tempting to revisit: a QA +layer staffed by a weaker model than the one it checks is only sound when every +claim it makes is checkable against an oracle. Narrowing to library currency +achieved that -- but once achieved, the agency was doing no work. "Point it at a +file, get a verdict" is a tool, not an agent. This form keeps `Task` denied, +keeps Context7's schemas out of the main loop's context entirely, and inherits +`--json`, exit codes, and `fix` fields from `core/cli.py` for free. + +Revisit only if the iterate-and-recheck loop (introspect -> hypothesise -> run a +counter-example -> recheck) proves necessary. That is the one thing this form +cannot do. +""" + +from __future__ import annotations + +import argparse +import ast +import importlib +import importlib.metadata +import inspect +import sys +from pathlib import Path +from typing import Any + +from core import config as config_mod, credentials, http +from core.cli import Cli, main +from core.errors import EXIT_CHECK_FAILED, GradError, NotFound, UsageError + +cli = Cli( + "grad-docs", + "Check library calls against what is installed, and against what is current.", + epilog=( + "Two oracles, in this order:\n" + " 1. introspection -- importlib.metadata + inspect.signature. offline, definitive.\n" + " 2. Context7 -- deprecations and changed idioms. what introspection cannot see.\n\n" + "`check` exits 9 when it finds something, so it composes with preflight's\n" + "declared-check mechanism if a pipeline wants it as a gate.\n\n" + "The Context7 REST paths were not verified when this was written. If a request\n" + "404s, read context7.com/docs/api-guide and fix [docs] in config/grad.toml." + ), +) + + +# --------------------------------------------------------------------------- +# oracle 1: introspection +# --------------------------------------------------------------------------- +# Modules that ship with Python. Reporting "no distribution provides `json`" for +# every stdlib import would bury the findings that matter in noise. +_STDLIB = set(getattr(sys, "stdlib_module_names", ())) + + +def installed_version(module: str) -> str | None: + """The distribution version providing a module, if any.""" + top = module.split(".")[0] + try: + packages = importlib.metadata.packages_distributions() + except Exception: # noqa: BLE001 - older/odd environments + packages = {} + for dist in packages.get(top, []) or [top]: + try: + return importlib.metadata.version(dist) + except importlib.metadata.PackageNotFoundError: + continue + return None + + +def _import(module: str) -> Any: + try: + return importlib.import_module(module) + except Exception: # noqa: BLE001 - a module that fails to import is a finding, not a crash + return None + + +def signature_of(module: str, attribute: str) -> dict[str, Any]: + """What this machine says about `module.attribute`. + + Returns a report rather than raising: "this attribute does not exist" is the + single most valuable thing this command says, and it is not an error in the + CLI. + """ + mod = _import(module) + if mod is None: + return {"module": module, "importable": False} + if not hasattr(mod, attribute): + near = _close(attribute, dir(mod)) + return { + "module": module, + "importable": True, + "attribute": attribute, + "exists": False, + "did_you_mean": near, + } + obj = getattr(mod, attribute) + report: dict[str, Any] = {"module": module, "importable": True, "attribute": attribute, "exists": True} + try: + sig = inspect.signature(obj) + report["signature"] = f"{attribute}{sig}" + report["parameters"] = list(sig.parameters) + report["keyword_only"] = [ + n for n, p in sig.parameters.items() if p.kind is inspect.Parameter.KEYWORD_ONLY + ] + report["accepts_kwargs"] = any( + p.kind is inspect.Parameter.VAR_KEYWORD for p in sig.parameters.values() + ) + except (TypeError, ValueError): + # Builtins and C extensions often have no introspectable signature. + # Saying so is honest; guessing would not be. + report["signature"] = None + report["note"] = "no introspectable signature (C extension or builtin)" + return report + + +def _close(name: str, options: list[str], n: int = 3) -> list[str]: + import difflib # noqa: PLC0415 + + return difflib.get_close_matches(name, [o for o in options if not o.startswith("_")], n=n, cutoff=0.6) + + +# --------------------------------------------------------------------------- +# static analysis of a file +# --------------------------------------------------------------------------- +class _Calls(ast.NodeVisitor): + """Collect `alias.attr(...)` calls and the keyword names they pass. + + Deliberately shallow. It resolves `import x` / `import x as y` / + `from a import b` aliases and nothing more -- no type inference, no + cross-file resolution. Everything it reports is then *checked* against the + installed object, so a shallow parse produces false negatives (calls it does + not look at) rather than false positives (findings that are not real). + """ + + def __init__(self) -> None: + self.aliases: dict[str, str] = {} # local name -> dotted module + self.from_imports: dict[str, tuple[str, str]] = {} # local name -> (module, attr) + self.calls: list[dict[str, Any]] = [] + + def visit_Import(self, node: ast.Import) -> None: + for alias in node.names: + self.aliases[alias.asname or alias.name.split(".")[0]] = alias.name + self.generic_visit(node) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + if node.module and not node.level: + for alias in node.names: + if alias.name == "*": + continue + self.from_imports[alias.asname or alias.name] = (node.module, alias.name) + self.generic_visit(node) + + def visit_Call(self, node: ast.Call) -> None: + target = self._target(node.func) + if target: + module, attribute = target + self.calls.append( + { + "module": module, + "attribute": attribute, + "line": node.lineno, + "keywords": [kw.arg for kw in node.keywords if kw.arg], + "has_star_kwargs": any(kw.arg is None for kw in node.keywords), + "positional": len(node.args), + } + ) + self.generic_visit(node) + + def _target(self, func: ast.AST) -> tuple[str, str] | None: + if isinstance(func, ast.Attribute) and isinstance(func.value, ast.Name): + module = self.aliases.get(func.value.id) + if module: + return module, func.attr + return None + if isinstance(func, ast.Name): + found = self.from_imports.get(func.id) + if found: + return found + return None + + +def analyse(path: Path) -> dict[str, Any]: + """Introspect every resolvable library call in a file. + + A finding is one of three things, and all three are checkable against an + oracle rather than being an opinion: + + * the module does not import here at all; + * the attribute does not exist on the installed version; + * a keyword argument the call passes is not in the installed signature. + + That third one is the §17 case in reverse: it is what would have caught + `run_job(..., namespace=...)` on a `huggingface_hub` too old to take it. + """ + if not path.is_file(): + raise NotFound(f"{path} does not exist", fix="give a path to a Python file") + try: + tree = ast.parse(path.read_text(encoding="utf-8", errors="replace"), filename=str(path)) + except SyntaxError as exc: + raise GradError( + "unparseable", + f"{path} is not valid Python: {exc}", + exit_code=EXIT_CHECK_FAILED, + fix="fix the syntax error first; nothing else can be checked until then", + ) from exc + + walker = _Calls() + walker.visit(tree) + + modules: dict[str, dict[str, Any]] = {} + for module in sorted({*walker.aliases.values(), *(m for m, _ in walker.from_imports.values())}): + top = module.split(".")[0] + modules[module] = { + "module": module, + "stdlib": top in _STDLIB, + "version": None if top in _STDLIB else installed_version(module), + "importable": _import(module) is not None, + } + + findings: list[dict[str, Any]] = [] + checked = 0 + # One finding per unimportable module, not one per call site: a missing + # package produces the same finding on every line that uses it, and twenty + # copies of "pip install x" buries the signature mismatches that matter. + reported_modules: set[str] = set() + for call in walker.calls: + module, attribute = call["module"], call["attribute"] + info = modules.get(module, {}) + # Stdlib calls are checked too. `_STDLIB` only suppresses the *version* + # lookup -- "no distribution provides `json`" is noise, but + # `json.dumpz()` is a real finding and introspection settles it just as + # definitively as it does for a third-party package. + if not info.get("importable", True): + if module not in reported_modules: + reported_modules.add(module) + findings.append( + { + "severity": "error", + "kind": "module_not_importable", + "line": call["line"], + "message": f"`{module}` does not import in this environment", + "fix": f"pip install {module.split('.')[0]}", + } + ) + continue + checked += 1 + report = signature_of(module, attribute) + if report.get("exists") is False: + findings.append( + { + "severity": "error", + "kind": "missing_attribute", + "line": call["line"], + "message": ( + f"`{module}.{attribute}` does not exist on the installed " + f"{module.split('.')[0]} {info.get('version') or '(unknown version)'}" + ), + "fix": ( + f"did you mean: {', '.join(report['did_you_mean'])}?" + if report.get("did_you_mean") + else f"python -m tools.docs query '{attribute}' --json" + ), + } + ) + continue + params = report.get("parameters") + if params is None or report.get("accepts_kwargs") or call["has_star_kwargs"]: + continue + unknown = [k for k in call["keywords"] if k not in params] + if unknown: + findings.append( + { + "severity": "error", + "kind": "unknown_keyword", + "line": call["line"], + "message": ( + f"`{module}.{attribute}()` does not take " + + ", ".join(f"`{k}`" for k in unknown) + + f" on the installed version; its signature is {report['signature']}" + ), + "fix": ( + f"python -m tools.docs query '{attribute} parameters' --json " + "# to see what replaced it" + ), + } + ) + + return { + "file": str(path), + "modules": list(modules.values()), + "calls_checked": checked, + "findings": findings, + } + + +# --------------------------------------------------------------------------- +# commands +# --------------------------------------------------------------------------- +@cli.command( + "resolve", + "library name -> Context7 library id", + setup=lambda p: p.add_argument("name"), +) +def cmd_resolve(args: argparse.Namespace) -> dict[str, Any]: + client = http.Context7(config_mod.load()) + candidates = client.resolve(args.name) + return { + "query": args.name, + "authenticated": client.authenticated, + "candidates": candidates, + "installed_version": installed_version(args.name), + "next": ( + f"python -m tools.docs query {candidates[0]['library_id']} '' --json" + if candidates + else None + ), + "note": ( + None + if client.authenticated + else "no context7_key stored; rate limits are lower. " + f"python -m tools.jobs credential set {credentials.CONTEXT7_KEY}" + ), + } + + +def _query_args(p: argparse.ArgumentParser) -> None: + p.add_argument("library_id", help="a Context7 library id, from `resolve`") + p.add_argument("query", help="what you want to know, e.g. 'run_job namespace parameter'") + p.add_argument("--tokens", type=int, default=5000, help="documentation budget for the answer") + + +@cli.command("query", "ask Context7 about a library", setup=_query_args) +def cmd_query(args: argparse.Namespace) -> dict[str, Any]: + """Oracle 2. Run oracle 1 first: introspection knows what is installed, and + this knows what is current, and confusing the two is the failure mode.""" + if args.tokens <= 0: + raise UsageError("--tokens must be positive", fix="--tokens 5000") + client = http.Context7(config_mod.load()) + return client.docs(args.library_id, args.query, tokens=args.tokens) + + +def _check_args(p: argparse.ArgumentParser) -> None: + p.add_argument("path", help="a Python file whose library calls should be checked") + p.add_argument( + "--offline", + action="store_true", + help="introspection only; skip Context7 entirely", + ) + p.add_argument( + "--currency", + action="store_true", + help="also ask Context7 whether each imported library is current (one call per module)", + ) + + +@cli.command("check", "introspect a file's library calls; exit 9 on findings", setup=_check_args) +def cmd_check(args: argparse.Namespace) -> dict[str, Any]: + """Introspection first, then Context7. + + Exits 9 -- "a check ran and reported failure" -- so it composes with + preflight's declared-check mechanism if a pipeline later wants it as a gate. + """ + report = analyse(Path(args.path).resolve()) + + if args.currency and not args.offline: + report["currency"] = _currency(report["modules"]) + + if report["findings"]: + first = report["findings"][0] + raise GradError( + "stale_calls", + f"{len(report['findings'])} finding(s) in {args.path}: {first['message']}", + exit_code=EXIT_CHECK_FAILED, + fix=first.get("fix") or "check the call against the installed signature", + detail=report, + ) + return {**report, "ok": True} + + +def _currency(modules: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Best-effort: a Context7 outage must not turn an offline-verifiable + finding into an unusable command.""" + client = http.Context7(config_mod.load()) + out = [] + for info in modules: + if info["stdlib"]: + continue + entry: dict[str, Any] = {"module": info["module"], "installed": info["version"]} + try: + candidates = client.resolve(info["module"].split(".")[0]) + entry["library_id"] = candidates[0]["library_id"] if candidates else None + entry["versions"] = candidates[0].get("versions") if candidates else None + except GradError as exc: + entry["error"] = exc.message + out.append(entry) + return out + + +@cli.command( + "signature", + "what the installed library says about one call", + setup=lambda p: ( + p.add_argument("module"), + p.add_argument("attribute"), + ), +) +def cmd_signature(args: argparse.Namespace) -> dict[str, Any]: + """Oracle 1, directly. Ten seconds, offline, definitive.""" + report = signature_of(args.module, args.attribute) + report["installed_version"] = installed_version(args.module) + if report.get("exists") is False: + raise GradError( + "missing_attribute", + f"`{args.module}.{args.attribute}` does not exist on the installed version", + exit_code=EXIT_CHECK_FAILED, + fix=( + f"did you mean: {', '.join(report['did_you_mean'])}?" + if report.get("did_you_mean") + else f"python -m tools.docs resolve {args.module} --json" + ), + detail=report, + ) + if report.get("importable") is False: + raise GradError( + "not_importable", + f"`{args.module}` does not import in this environment", + exit_code=EXIT_CHECK_FAILED, + fix=f"pip install {args.module.split('.')[0]}", + detail=report, + ) + return report + + +if __name__ == "__main__": + main(cli) diff --git a/tools/evolve.py b/tools/evolve.py new file mode 100644 index 0000000..d4948c6 --- /dev/null +++ b/tools/evolve.py @@ -0,0 +1,810 @@ +"""grad-evolve -- evolutionary search over ShinkaEvolve (HANDOFF-2 §21). + + "An evolutionary loop is a machine for spending money with no human in it." + +That sentence is why this file's first job is a gate and its second is a search. +`check_spend` alone *would* stop a runaway campaign -- at generation 40, +abandoning an in-flight run that then goes stale and blocks every future +submission through the §6 gate. Succeeding at the search would brick the system. +So there is a **campaign budget gate**: before generation 0, refuse unless +`estimate_per_candidate x max_candidates` fits under the project's remaining +allocation, and re-check before every generation. Shinka's own `max_api_costs` +covers the LLM side; the compute side is the expensive half and Grad owns it. + +**Phase 1 is local only, and that is not a placeholder.** Shinka needs no GPU +for many tasks and its headless example is API-free, so a campaign evaluated +entirely through local subprocesses proves the campaign records, the sub-run +bookkeeping, and the budget integration while the blast radius is zero. Doing +the ledger work and the spend work simultaneously against live GPU jobs is how +you learn about exit 7 the hard way. `--remote` is refused here until phase 2. + +**Driver, not fork.** Shinka exposes `EvolutionConfig`, `LocalJobConfig`, +`DatabaseConfig`, `ShinkaEvolveRunner(...).run()`. What we need is gating and +ledger integration *around* the loop, which is a driver. §23 item 1 asks whether +Shinka exposes a per-candidate callback, because that decides driver-vs-fork: +`mutator_capabilities()` below answers it against the installed package rather +than against a document, and the answer is reported in `run`'s output. Until a +hook point proves insufficient, a fork is a maintenance cost to defer. + +**Models.** Per §16, the default is an *ensemble* -- Sonnet 5 primary plus Haiku +4.5 as a cheap explorer -- because Shinka's design is explicitly an ensemble of +LLMs acting as mutation operators, and collapsing to a single model discards +diversity the algorithm is built around. Shinka's bandit allocates between them. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +from core import ( + budget, + campaign as camp, + config as config_mod, + ledger_store as ls, + paths, + quota_log, +) +from core.cli import Cli, main +from core.errors import ( + EXIT_CHECK_FAILED, + EXIT_PROJECT_BUDGET, + ConfigError, + GateRefusal, + GradError, + NotFound, + UsageError, +) + +cli = Cli( + "grad-evolve", + "Run an evolutionary search as a budgeted campaign, with candidates recorded " + "as sub-runs and only promoted winners entering the ledger.", + epilog=( + "The campaign, not the candidate, is the unit of prediction: --expect binds one\n" + "relational claim ('the evolved variant beats baseline X on Y by >= Z') and the\n" + "candidates are exempt from the per-run expectation gate.\n\n" + "Before generation 0 and before every generation after it, the projected cost of\n" + "the remaining candidates must fit under the project's allocation. Exit 12 when\n" + "it does not -- that is a project budget refusal, not the machine's ceiling.\n\n" + "Phase 1 is local only. --remote is refused: doing the ledger work and the spend\n" + "work simultaneously against live GPU jobs is how you learn about exit 7 the\n" + "hard way." + ), +) + +DEFAULT_MODELS = ("claude-sonnet-5", "claude-haiku-4-5") +STAGE_EVOLVE = "evolve.mutate" + + +# --------------------------------------------------------------------------- +# the Shinka boundary +# --------------------------------------------------------------------------- +def _shinka() -> Any: + try: + import shinka # noqa: PLC0415 + except ImportError as exc: + raise ConfigError( + "shinka-evolve is not installed, so there is no mutation engine", + fix="pip install -e '.[evolve]'", + ) from exc + return shinka + + +def mutator_capabilities() -> dict[str, Any]: + """Answer §23 item 1 against the installed package, not against a document. + + "Not verified in session: whether Shinka exposes a per-candidate callback. + Check before starting; it decides driver-vs-fork." + + A per-candidate hook would let budget be charged inside the generation loop. + Without one, generation boundaries are the finest granularity available -- + which is what this driver is built around, so its absence is a documented + limit rather than a blocker. + """ + try: + shinka = _shinka() + except ConfigError as exc: + return {"installed": False, "reason": exc.message} + + import inspect # noqa: PLC0415 + + runner = getattr(shinka, "ShinkaEvolveRunner", None) + hooks: list[str] = [] + if runner is not None: + try: + params = inspect.signature(runner.__init__).parameters + except (TypeError, ValueError): + params = {} + hooks = [ + name + for name in params + if any(word in name for word in ("callback", "hook", "on_", "listener")) + ] + return { + "installed": True, + "version": getattr(shinka, "__version__", None), + "runner": runner is not None, + "per_candidate_hooks": hooks, + "granularity": "candidate" if hooks else "generation", + "note": ( + "no per-candidate hook found; budget is charged at generation boundaries, " + "which is what this driver is designed around. A fork is only warranted if " + "that granularity proves insufficient." + if not hooks + else "a per-candidate hook exists; finer budget charging is possible without a fork" + ), + } + + +# --------------------------------------------------------------------------- +# init +# --------------------------------------------------------------------------- +_INITIAL_TEMPLATE = '''"""The program being evolved. + +Everything between the EVOLVE-BLOCK markers is mutable; everything outside is +not. That boundary is not decoration -- `tools/evolve.py` checks it +mechanically, and a mutation that stays inside it needs only the two local +preflight checks, while one that escapes requires a fresh remote smoke run. +Keeping imports, I/O, and the entry point outside the block is what makes a +campaign affordable. +""" + +import json + + +# EVOLVE-BLOCK-START +def solve(x: float) -> float: + """The thing being searched over. Mutate freely.""" + return x * 2.0 +# EVOLVE-BLOCK-END + + +def main() -> None: + print(json.dumps({"ok": True})) + + +if __name__ == "__main__": + main() +''' + +_EVALUATE_TEMPLATE = '''"""Score one candidate. + +Contract, matching Shinka's: print ONE JSON object of metrics to stdout, and it +must contain `combined_score`. Everything else in the object is recorded +alongside it and is what makes the Goodhart failure visible -- a search +optimising a scalar will find the bug in the metric, so record the components +that scalar was built from. +""" + +import json + +import initial + + +def evaluate() -> dict: + # Replace with the real objective. + error = sum(abs(initial.solve(x) - (x * 2.0)) for x in range(10)) + return { + "combined_score": -error, + "abs_error": error, + "n": 10, + } + + +if __name__ == "__main__": + print(json.dumps(evaluate())) +''' + + +def _init_args(p: argparse.ArgumentParser) -> None: + p.add_argument("--task-dir", required=True, help="directory to scaffold, e.g. pipeline/evolve-lr") + p.add_argument("--force", action="store_true", help="overwrite existing files") + + +@cli.command("init", "scaffold a task directory with the evolve-block contract", setup=_init_args) +def cmd_init(args: argparse.Namespace) -> dict[str, Any]: + """`initial.py` with EVOLVE-BLOCK markers, `evaluate.py` returning + `combined_score`. Shinka's contract and Grad's submission spec describe the + same object, which is why this is an extension of §6/§7 rather than a + bolt-on.""" + task_dir = Path(args.task_dir) + if not task_dir.is_absolute(): + task_dir = paths.root() / task_dir + task_dir.mkdir(parents=True, exist_ok=True) + + written = [] + for name, body in (("initial.py", _INITIAL_TEMPLATE), ("evaluate.py", _EVALUATE_TEMPLATE)): + target = task_dir / name + if target.exists() and not args.force: + continue + target.write_text(body, encoding="utf-8") + written.append(str(target)) + + return { + "task_dir": str(task_dir), + "written": written, + "skipped": [] if args.force else [ + str(task_dir / n) for n in ("initial.py", "evaluate.py") + if str(task_dir / n) not in written + ], + "contract": { + "initial.py": f"mutable region between {camp.BLOCK_START} and {camp.BLOCK_END}", + "evaluate.py": "prints one JSON object of metrics including combined_score", + }, + "next": ( + "python -m tools.ledger expect --task --quantity combined_score " + "--direction increase --claim 'the evolved variant beats baseline' --json" + ), + } + + +# --------------------------------------------------------------------------- +# run +# --------------------------------------------------------------------------- +def _run_args(p: argparse.ArgumentParser) -> None: + p.add_argument("--task-dir", required=True) + p.add_argument("--expect", required=True, help="campaign-level expectation id (§7, §21)") + p.add_argument("--project", help="project whose allocation bounds this campaign") + p.add_argument("--generations", type=int, default=10) + p.add_argument("--population", type=int, default=4, help="candidates per generation") + p.add_argument( + "--estimate-per-candidate-usd", + type=float, + default=0.0, + help="what one evaluation is expected to cost. The campaign gate multiplies it.", + ) + p.add_argument( + "--local", + action="store_true", + default=True, + help="evaluate locally (phase 1; the only supported mode)", + ) + p.add_argument("--remote", action="store_true", help="refused: phase 2, behind the campaign gate") + p.add_argument( + "--set", + dest="overrides", + action="append", + default=[], + metavar="KEY=VALUE", + help="passed through to Shinka, e.g. --set evo.llm_models=...", + ) + p.add_argument("--timeout-s", type=int, default=600, help="wall clock per candidate evaluation") + + +@cli.command("run", "run a budgeted campaign", setup=_run_args) +def cmd_run(args: argparse.Namespace) -> dict[str, Any]: + if args.remote: + raise UsageError( + "--remote is phase 2 and is not enabled: the campaign budget gate must be " + "proven against zero-blast-radius local evaluation first. " + "Do not run a single remote generation before that.", + fix="drop --remote; a local campaign exercises the same records and the same gate", + ) + + cfg = config_mod.load() + paths.ensure_workspace() + task_dir = Path(args.task_dir) + if not task_dir.is_absolute(): + task_dir = paths.root() / task_dir + initial = task_dir / "initial.py" + evaluate = task_dir / "evaluate.py" + for path in (initial, evaluate): + if not path.is_file(): + raise NotFound( + f"{path} does not exist", + fix=f"python -m tools.evolve init --task-dir {args.task_dir} --json", + ) + + baseline_source = initial.read_text(encoding="utf-8") + if not camp.has_markers(baseline_source): + raise UsageError( + f"{initial} has no {camp.BLOCK_START}/{camp.BLOCK_END} markers, so no mutation " + "can be checked for escaping the mutable region -- which means every candidate " + "would need a paid remote smoke run", + fix=f"wrap the mutable region in {camp.BLOCK_START} / {camp.BLOCK_END} comments", + ) + + # The campaign is the unit of prediction (§21 collision 2). The expectation + # is bound here, once, and the candidates below are exempt from the per-run + # gate precisely because this binding exists. + expectation = _bind_expectation(args.expect) + + project_id = budget.resolve(args.project) + max_candidates = max(1, args.generations) * max(1, args.population) + projected = float(args.estimate_per_candidate_usd) * max_candidates + + # THE gate. Before generation 0, not after generation 40. + _campaign_gate(project_id, projected, max_candidates, args.estimate_per_candidate_usd) + + campaign_id = camp.new_id("camp") + record = { + "type": camp.T_CAMPAIGN, + "id": campaign_id, + "created_at": camp.now_iso(), + "task_dir": str(task_dir), + "project": project_id or budget.UNASSIGNED, + "expectation_id": args.expect, + "quantity": expectation.get("quantity"), + "generations": args.generations, + "population": args.population, + "max_candidates": max_candidates, + "estimate_per_candidate_usd": args.estimate_per_candidate_usd, + "projected_cost_usd": round(projected, 4), + "mode": "local", + "models": list(_models(cfg, args.overrides)), + "mutator": mutator_capabilities(), + "status": "open", + } + camp.append_campaign(record) + + result = _drive( + campaign_id=campaign_id, + task_dir=task_dir, + baseline_source=baseline_source, + generations=args.generations, + population=args.population, + project_id=project_id, + per_candidate=args.estimate_per_candidate_usd, + timeout_s=args.timeout_s, + overrides=args.overrides, + cfg=cfg, + ) + + camp.close_campaign(campaign_id, status=result["status"], reason=result.get("reason", "")) + return { + "campaign": campaign_id, + "expectation_id": args.expect, + "project": project_id, + **result, + "next": f"python -m tools.evolve status --campaign {campaign_id} --json", + } + + +def _bind_expectation(expectation_id: str) -> dict[str, Any]: + """The campaign's one prediction, and the same uniqueness rule as a run. + + §7's argument applies unchanged: an expectation that can be reused is an + expectation that can be authored after the fact. + """ + try: + expectation = ls.expectation(expectation_id) + except GradError: + raise GateRefusal( + "expectation_missing", + f"expectation {expectation_id!r} does not exist", + 5, + fix=( + "python -m tools.ledger expect --task --quantity combined_score " + "--direction increase --claim 'the evolved variant beats baseline X' --json" + ), + ) from None + bound = { + c.get("expectation_id") + for c in camp.campaigns().values() + if c.get("expectation_id") + } | ls.bound_expectation_ids() + if expectation_id in bound: + raise GateRefusal( + "expectation_bound", + f"expectation {expectation_id!r} is already bound to a run or campaign", + 5, + fix="mint a new expectation for this campaign", + ) + return expectation + + +def _campaign_gate( + project_id: str | None, projected: float, max_candidates: int, per_candidate: float +) -> None: + """Before generation 0. This is the one that matters. + + Without it, `check_spend` stops the campaign mid-flight -- at generation 40, + abandoning an in-flight run that goes stale and blocks every future + submission (exit 7). Succeeding at the search would brick the system. + """ + if per_candidate <= 0: + # Not an error: a genuinely free local campaign is the phase-1 case. But + # an unpriced campaign cannot be gated, and saying so beats implying it + # was checked. + return + budget.check( + project_id, + gpu_usd=projected, + what=f"a campaign of {max_candidates} candidates at ${per_candidate:.2f} each", + ) + + +def _models(cfg: config_mod.Config, overrides: list[str]) -> tuple[str, ...]: + """Sonnet 5 primary plus Haiku 4.5 explorer, unless overridden. + + Shinka's design is an ensemble of LLMs acting as mutation operators; + collapsing to a single model discards diversity the algorithm is built + around. `--set evo.llm_models=...` is Shinka's own mechanism and wins. + """ + for override in overrides: + if override.startswith("evo.llm_models="): + value = override.split("=", 1)[1] + return tuple(m.strip() for m in value.split(",") if m.strip()) + primary = cfg.model_for("evolve") + return (primary, "claude-haiku-4-5") if primary not in DEFAULT_MODELS[1:] else (primary,) + + +# --------------------------------------------------------------------------- +# the loop +# --------------------------------------------------------------------------- +def _drive( + *, + campaign_id: str, + task_dir: Path, + baseline_source: str, + generations: int, + population: int, + project_id: str | None, + per_candidate: float, + timeout_s: int, + overrides: list[str], + cfg: config_mod.Config, + mutator: Any = None, +) -> dict[str, Any]: + """Generation by generation, with the gate between each. + + `mutator` is injectable so the driver can be tested against a faked Shinka + runner -- §24's testing note asks for exactly that, and a campaign loop + tested only against the real thing is a campaign loop tested never. + """ + mutator = mutator or _make_mutator(task_dir, overrides, cfg) + evaluated = 0 + best: dict[str, Any] | None = None + status = "closed" + reason = "" + + for generation in range(generations): + remaining = (generations - generation) * population + try: + _campaign_gate(project_id, per_candidate * remaining, remaining, per_candidate) + except GateRefusal as exc: + # Stopping here is the success case for the gate: the campaign ends + # cleanly at a generation boundary with every candidate collected, + # rather than being killed mid-flight with a run left in flight. + status = "exhausted" + reason = exc.message + camp.record_generation( + campaign_id, generation, halted=True, reason=exc.message, code=EXIT_PROJECT_BUDGET + ) + break + + started = time.time() + try: + proposals = mutator.propose(generation=generation, population=population, best=best) + except GradError: + raise + except Exception as exc: # noqa: BLE001 - a mutation engine failure ends the campaign, not the process + status = "failed" + reason = f"the mutation engine failed at generation {generation}: {exc}" + camp.record_generation(campaign_id, generation, error=str(exc)) + break + + scores = [] + for index, source in enumerate(proposals): + candidate = _evaluate_candidate( + campaign_id=campaign_id, + generation=generation, + index=index, + source=source, + baseline_source=baseline_source, + task_dir=task_dir, + timeout_s=timeout_s, + per_candidate=per_candidate, + ) + evaluated += 1 + score = (candidate.get("metrics") or {}).get("combined_score") + if isinstance(score, (int, float)): + scores.append(score) + if best is None or score > best["score"]: + best = {"score": score, "source": source, "candidate_id": candidate["candidate_id"]} + + camp.record_generation( + campaign_id, + generation, + candidates=len(proposals), + evaluated=len(scores), + best_score=max(scores) if scores else None, + duration_s=round(time.time() - started, 2), + ) + + spend = camp.campaign_spend(campaign_id) + return { + "status": status, + "reason": reason, + "generations_run": min(generations, evaluated // max(1, population)), + "candidates_evaluated": evaluated, + "spend": spend, + # Top-K, not the argmax. A search optimising a scalar finds the bug in + # the metric, so the shape of the leaderboard is part of the output. + "top": [ + { + "candidate_id": c["candidate_id"], + "generation": c["generation"], + "metrics": c["metrics"], + "escaped_block": c.get("escaped_block", {}).get("escaped", False), + } + for c in camp.top_k(campaign_id, 5) + ], + "goodhart_note": ( + "top-K, not the argmax. A winner is not a result until it goes through " + "`evolve promote` and the normal verdict path." + ), + } + + +def _evaluate_candidate( + *, + campaign_id: str, + generation: int, + index: int, + source: str, + baseline_source: str, + task_dir: Path, + timeout_s: int, + per_candidate: float, +) -> dict[str, Any]: + """Evaluate one candidate locally and record it as a sub-run. + + Candidates go to `ledger/candidates.jsonl`, never to `runs.jsonl`: a + 100-generation campaign is thousands of rows and would dominate a ledger + meant to be read by hand (§23 item 4). Only a promoted candidate becomes a + run. + """ + candidate_id = f"{campaign_id}-g{generation}-c{index}" + escaped = camp.escaped_evolve_block(baseline_source, source) + + workdir = paths.run_artifacts(candidate_id) + workdir.mkdir(parents=True, exist_ok=True) + (workdir / "initial.py").write_text(source, encoding="utf-8") + evaluate_src = (task_dir / "evaluate.py").read_text(encoding="utf-8") + (workdir / "evaluate.py").write_text(evaluate_src, encoding="utf-8") + + started = time.time() + record: dict[str, Any] = { + "campaign": campaign_id, + "candidate_id": candidate_id, + "generation": generation, + "index": index, + "at": camp.now_iso(), + "escaped_block": escaped, + "workdir": str(workdir), + "cost_usd": per_candidate, + } + + if escaped["escaped"]: + # §21 collision 3: a mutation outside the block changed code the + # baseline's smoke result no longer covers. Recorded, not evaluated -- + # the alternative is a paid remote smoke run per candidate, which is the + # cost this whole mechanism exists to avoid. + record.update( + { + "skipped": True, + "error": "mutation escaped the evolve block; needs a fresh smoke run", + "duration_s": round(time.time() - started, 3), + } + ) + camp.append_candidate(record) + return record + + try: + proc = subprocess.run( + [sys.executable, "evaluate.py"], + cwd=str(workdir), + capture_output=True, + text=True, + timeout=timeout_s, + check=False, + ) + output = (proc.stdout or "").strip() + stderr = (proc.stderr or "")[-4000:] + metrics: Any = None + problem: str | None = None + if proc.returncode != 0: + problem = f"evaluate.py exited {proc.returncode}" + else: + try: + metrics = json.loads(output.splitlines()[-1]) if output else None + except (json.JSONDecodeError, IndexError): + problem = "evaluate.py did not print a JSON object of metrics" + if problem is None: + problem = camp.validate_metrics(metrics) + (workdir / "evaluate.log").write_text(output + "\n" + stderr, encoding="utf-8") + record.update( + { + "metrics": metrics if problem is None else None, + "error": problem, + "duration_s": round(time.time() - started, 3), + } + ) + except subprocess.TimeoutExpired: + record.update( + { + "metrics": None, + "error": f"evaluate.py exceeded {timeout_s}s", + "duration_s": round(time.time() - started, 3), + } + ) + + camp.append_candidate(record) + return record + + +# --------------------------------------------------------------------------- +# mutation engines +# --------------------------------------------------------------------------- +class ShinkaMutator: + """Thin driver over Shinka's Python API. + + Deliberately thin: everything worth owning -- the gate, the ledger records, + the escape check -- is outside it, so replacing this class is a small change + and forking Shinka remains a decision that can be deferred. + """ + + def __init__(self, task_dir: Path, models: tuple[str, ...], overrides: list[str]) -> None: + self.task_dir = task_dir + self.models = models + self.overrides = overrides + self._runner: Any = None + + def _build(self) -> Any: + shinka = _shinka() + try: + evolution = shinka.EvolutionConfig( + llm_models=list(self.models), + init_program_path=str(self.task_dir / "initial.py"), + ) + job = shinka.LocalJobConfig(eval_program_path=str(self.task_dir / "evaluate.py")) + database = shinka.DatabaseConfig() + return shinka.ShinkaEvolveRunner( + evo_config=evolution, job_config=job, db_config=database + ) + except (AttributeError, TypeError) as exc: + raise ConfigError( + f"the installed shinka-evolve does not match the expected API: {exc}", + fix=( + "check ShinkaEvolveRunner's signature against the installed version " + "(`python -m tools.docs signature shinka ShinkaEvolveRunner --json`), " + "then adjust ShinkaMutator" + ), + ) from exc + + def propose(self, *, generation: int, population: int, best: dict[str, Any] | None) -> list[str]: + if self._runner is None: + self._runner = self._build() + proposals = self._runner.propose( + generation=generation, population=population, parent=(best or {}).get("source") + ) + quota_log.record( + STAGE_EVOLVE, + role="evolve", + model=",".join(self.models), + detail={"generation": generation, "population": population}, + ) + return [str(p) for p in proposals] + + +def _make_mutator(task_dir: Path, overrides: list[str], cfg: config_mod.Config) -> Any: + return ShinkaMutator(task_dir, _models(cfg, overrides), overrides) + + +# --------------------------------------------------------------------------- +# status / promote +# --------------------------------------------------------------------------- +def _status_args(p: argparse.ArgumentParser) -> None: + p.add_argument("--campaign", help="one campaign; omit to list them all") + p.add_argument("--top", type=int, default=5, help="how many leaders to show") + + +@cli.command("status", "campaigns, their spend, and their top-K", setup=_status_args) +def cmd_status(args: argparse.Namespace) -> dict[str, Any]: + if not args.campaign: + return { + "campaigns": [ + { + "id": cid, + "status": c.get("status"), + "task_dir": c.get("task_dir"), + "project": c.get("project"), + "generations_run": c.get("generations_run"), + "spend": camp.campaign_spend(cid), + } + for cid, c in camp.campaigns().items() + ] + } + record = camp.campaign(args.campaign) + return { + "campaign": record, + "spend": camp.campaign_spend(args.campaign), + "top": camp.top_k(args.campaign, args.top), + "note": ( + "top-K rather than the argmax, on purpose: a search optimising a scalar " + "will find the bug in the metric" + ), + } + + +def _promote_args(p: argparse.ArgumentParser) -> None: + p.add_argument("--campaign", required=True) + p.add_argument("--candidate", required=True, help="candidate id, or its index within the campaign") + p.add_argument("--into", help="path to write the promoted source to (defaults to the task dir)") + + +@cli.command("promote", "turn a candidate into a normal, judged run", setup=_promote_args) +def cmd_promote(args: argparse.Namespace) -> dict[str, Any]: + """The Goodhart resolution. + + "The campaign winner goes through the normal verdict path before it counts + as a result." Promotion writes the source out and hands you the ordinary + preflight -> expect -> submit -> collect -> verdict cycle. It does not + shortcut any of it, and it deliberately does not write a run record itself. + """ + record = camp.campaign(args.campaign) + rows = camp.candidates(args.campaign) + match = next( + (c for c in rows if c["candidate_id"] == args.candidate), + None, + ) or next( + (c for c in rows if str(c.get("index")) == str(args.candidate)), + None, + ) + if match is None: + raise NotFound( + f"candidate {args.candidate!r} is not in campaign {args.campaign}", + fix=f"python -m tools.evolve status --campaign {args.campaign} --json", + ) + if not match.get("metrics"): + raise GradError( + "candidate_unevaluated", + f"candidate {args.candidate} has no metrics: {match.get('error') or 'never evaluated'}", + exit_code=EXIT_CHECK_FAILED, + fix="promote a candidate that produced a combined_score", + ) + + source = Path(match["workdir"]) / "initial.py" + destination = Path(args.into) if args.into else Path(record["task_dir"]) / "promoted.py" + if not destination.is_absolute(): + destination = paths.root() / destination + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(source.read_text(encoding="utf-8"), encoding="utf-8") + + return { + "campaign": args.campaign, + "candidate": match["candidate_id"], + "metrics": match["metrics"], + "written": str(destination), + "escaped_block": match.get("escaped_block", {}).get("escaped", False), + # No run record is written here. A promoted candidate re-enters the + # system through the front door, with its own preflight and its own + # prediction, because a number produced by a scalar-maximising search is + # a hypothesis rather than a result. + "next": [ + f"python -m tools.preflight run --spec --json", + "python -m tools.ledger expect --task --quantity ... --json", + "python -m tools.jobs submit --spec --expect --json", + ], + "why": ( + "a campaign winner is not a result until it has been judged: the search " + "optimised a scalar, and finding the bug in the metric is what that does best" + ), + } + + +@cli.command("capabilities", "what the installed ShinkaEvolve supports (§23 item 1)") +def cmd_capabilities(_: argparse.Namespace) -> dict[str, Any]: + """Driver or fork? Answered against the installed package.""" + return mutator_capabilities() + + +if __name__ == "__main__": + main(cli) diff --git a/tools/gpu.py b/tools/gpu.py index a8b7b82..49702f2 100644 --- a/tools/gpu.py +++ b/tools/gpu.py @@ -23,7 +23,14 @@ from pathlib import Path from typing import Any -from core import config as config_mod, credentials, gates, ledger_store as ls, submit as submit_lib +from core import ( + budget, + config as config_mod, + credentials, + gates, + ledger_store as ls, + submit as submit_lib, +) from core.cli import Cli, main from core.config import Config, Host from core.errors import EXIT_RUNNING, GradError, UpstreamError, UsageError @@ -147,6 +154,10 @@ def _submit_args(p: argparse.ArgumentParser) -> None: p.add_argument("--expect", help="expectation id to bind. REQUIRED unless --smoke") p.add_argument("--set", dest="overrides", action="append", default=[], metavar="KEY=VALUE") p.add_argument("--task") + p.add_argument( + "--project", + help="project to charge this run to (defaults to the current one; §15)", + ) p.add_argument("--smoke", action="store_true", help="gate-exempt, hard-capped one-step check (§6)") p.add_argument("--no-digest", action="store_true", help=argparse.SUPPRESS) @@ -160,11 +171,12 @@ def cmd_submit(args: argparse.Namespace) -> dict[str, Any]: resolve_digest=not args.no_digest, ) host = cfg.host(args.host or sub.target.get("host") or "") + project_id = budget.resolve(args.project) if args.smoke: if args.expect: raise UsageError("--smoke binds no prediction", fix="drop --expect, or drop --smoke") - result = run_smoke(sub, cfg, host=host) + result = run_smoke(sub, cfg, host=host, project=project_id) from tools import preflight preflight.record_check_result(sub.hash(), "smoke", result) @@ -180,7 +192,7 @@ def cmd_submit(args: argparse.Namespace) -> dict[str, Any]: # Gates first, then the record. Nothing is staged to a host until both the # four gates have passed and the run is on the ledger at its estimate. - summary = submit_lib.check(sub, args.expect, cfg) + summary = submit_lib.check(sub, args.expect, cfg, project=project_id) run_id, _ = submit_lib.record_submission( sub, expectation_id=args.expect, @@ -188,6 +200,7 @@ def cmd_submit(args: argparse.Namespace) -> dict[str, Any]: target={"host": host.name, "platform": "ssh", "rate_usd_per_hour": host.rate_usd_per_hour}, command=_command_for(sub), task=args.task, + project=project_id, ) remote_dir = f"{host.workdir}/{run_id}" try: @@ -260,7 +273,9 @@ def _launch(host: Host, sub: Submission, remote_dir: str, command: list[str]) -> # --------------------------------------------------------------------------- # smoke # --------------------------------------------------------------------------- -def run_smoke(sub: Submission, cfg: Config, *, host: Host | None = None) -> dict[str, Any]: +def run_smoke( + sub: Submission, cfg: Config, *, host: Host | None = None, project: str | None = None +) -> dict[str, Any]: """One step on the real host, capped in code (§6). This is the only check that sees the remote image, the remote driver stack, @@ -272,7 +287,7 @@ def run_smoke(sub: Submission, cfg: Config, *, host: Host | None = None) -> dict run_id = submit_lib.record_smoke_run( sub, cfg=cfg, platform=PLATFORM, target={"host": host.name, "platform": "ssh", "rate_usd_per_hour": host.rate_usd_per_hour}, - caps=caps, command=command, + caps=caps, command=command, project=project, ) artifacts = submit_lib.artifacts_dir(run_id) remote_dir = f"{host.workdir}/{run_id}" diff --git a/tools/jobs.py b/tools/jobs.py index cfb5c60..6f5a5db 100644 --- a/tools/jobs.py +++ b/tools/jobs.py @@ -18,7 +18,14 @@ from pathlib import Path from typing import Any -from core import config as config_mod, credentials, gates, ledger_store as ls, submit as submit_lib +from core import ( + budget, + config as config_mod, + credentials, + gates, + ledger_store as ls, + submit as submit_lib, +) from core.cli import Cli, main from core.config import Config from core.errors import ConfigError, EXIT_RUNNING, GradError, UpstreamError, UsageError @@ -73,6 +80,99 @@ def _token() -> str: return token +# --------------------------------------------------------------------------- +# organization namespace (HANDOFF-2 §17) +# --------------------------------------------------------------------------- +# The trap this section exists to avoid: `namespace` is a property of the job +# *handle*, not a submit-time parameter. Passing it only to `run_job` produces a +# job that cannot be found again -- `inspect_job` and `fetch_job_logs` look +# under the personal namespace and 404, the run never collects, goes stale, and +# then blocks every future submission through the §6 stale-run gate. The failure +# appears very far from its cause. +# +# So it is persisted onto the handle at submit and threaded through every call +# that takes one. `_ns_kwargs` is the single place that decides how to pass it. +def _ns_kwargs(namespace: str | None) -> dict[str, Any]: + """Namespace kwargs for a hub call, or nothing at all when personal. + + Omitted rather than passed as None so an older `huggingface_hub` without the + parameter still works for personal jobs; a *requested* namespace on such a + version is a hard error rather than a silently personal job. + """ + if not namespace: + return {} + import inspect # noqa: PLC0415 + + hub = _hub() + for fn in (hub.run_job, hub.inspect_job, hub.fetch_job_logs): + try: + if "namespace" not in inspect.signature(fn).parameters: + raise ConfigError( + f"the installed huggingface_hub's {fn.__name__}() takes no `namespace`, " + "so a job submitted to an organization could not be collected from it", + fix="pip install -U 'huggingface_hub>=1.16' # or drop --namespace", + ) + except (TypeError, ValueError): + # Some builds wrap these; an unreadable signature is not evidence + # the parameter is missing, so it is not treated as such. + continue + return {"namespace": namespace} + + +def resolve_namespace( + flag: str | None, sub: Submission | None, cfg: Config, project_id: str | None +) -> str | None: + """`--namespace` -> spec `[target] namespace` -> project payer -> `[hf] namespace` -> personal. + + Mirrors how `flavor` already resolves, and the project step is why §15's + `payer` lives on the project: org attribution becomes a consequence of + choosing a project rather than a flag to forget. + """ + if flag: + return flag + if sub is not None and sub.target.get("namespace"): + return str(sub.target["namespace"]) + from_project = budget.hf_namespace(project_id) + if from_project: + return from_project + return cfg.get("hf", "namespace") or None + + +def validate_namespace(namespace: str | None, token: str) -> dict[str, Any]: + """Check membership *before* `record_submission`. + + Deliberately in the same place `_hub()` and `_token()` are already called, + and for the same reason: a configuration problem must not leave a phantom + estimate sitting on the ceiling. One network call per submit is acceptable + on a path about to spend dollars, and it is not cached aggressively -- + org membership changing is precisely the case worth catching. + """ + hub = _hub() + try: + me = hub.whoami(token=token) or {} + except Exception as exc: # noqa: BLE001 + raise UpstreamError( + f"could not read the token's identity from Hugging Face: {exc}", + fix="check that the stored hf_token is valid and not expired", + ) from exc + user = me.get("name") or me.get("user") + orgs = [ + str(e.get("name") if isinstance(e, dict) else e) + for e in (me.get("orgs") or []) + if (e.get("name") if isinstance(e, dict) else e) + ] + if namespace and namespace != user and namespace not in orgs: + raise ConfigError( + f"this token cannot act for namespace {namespace!r}; it is {user!r} and a member of: " + + (", ".join(orgs) or "(no organizations)"), + fix=( + "pick a namespace the token belongs to, or store a token with that org's scope: " + f"python -m tools.jobs credential set {credentials.HF_TOKEN}" + ), + ) + return {"user": user, "orgs": orgs, "namespace": namespace} + + # --------------------------------------------------------------------------- # submit # --------------------------------------------------------------------------- @@ -86,6 +186,15 @@ def _submit_args(p: argparse.ArgumentParser) -> None: p.add_argument("--set", dest="overrides", action="append", default=[], metavar="KEY=VALUE") p.add_argument("--flavor", help="HF Jobs hardware flavor (overrides the spec)") p.add_argument("--task", help="task id for the ledger (defaults to the spec directory name)") + p.add_argument( + "--project", + help="project to charge this run to (defaults to the current one; §15)", + ) + p.add_argument( + "--namespace", + help="HF organization to run under. Resolution: this flag, the spec's " + "[target] namespace, the project's payer, [hf] namespace, then personal.", + ) p.add_argument( "--smoke", action="store_true", @@ -105,13 +214,16 @@ def cmd_submit(args: argparse.Namespace) -> dict[str, Any]: if args.flavor: sub.target["flavor"] = args.flavor + project_id = budget.resolve(args.project) + namespace = resolve_namespace(args.namespace, sub, cfg, project_id) + if args.smoke: if args.expect: raise UsageError( "--smoke does not take --expect: a smoke check is not a result and binds no prediction", fix="drop --expect, or drop --smoke", ) - result = run_smoke(sub, cfg) + result = run_smoke(sub, cfg, namespace=namespace, project=project_id) from tools import preflight preflight.record_check_result(sub.hash(), "smoke", result) @@ -129,19 +241,25 @@ def cmd_submit(args: argparse.Namespace) -> dict[str, Any]: command = _command_for(sub) # Gates first: a refusal is the most actionable thing a submitter can say. - summary = submit_lib.check(sub, args.expect, cfg) + summary = submit_lib.check(sub, args.expect, cfg, project=project_id) # Then the backend and the credential, before any record exists: a missing # package or an absent token is a configuration problem, not an in-flight - # job, and it must not leave a phantom estimate sitting on the ceiling. + # job, and it must not leave a phantom estimate sitting on the ceiling. The + # namespace check joins them for exactly that reason -- an org the token + # cannot act for is a configuration problem too. hub = _hub() - _token() + token = _token() + identity = validate_namespace(namespace, token) + warnings = _namespace_warnings(sub, namespace) + run_id, _ = submit_lib.record_submission( sub, expectation_id=args.expect, platform=PLATFORM, - target={"flavor": flavor, "platform": "hf"}, + target={"flavor": flavor, "platform": "hf", "namespace": namespace}, command=command, task=args.task, + project=project_id, ) try: @@ -151,8 +269,9 @@ def cmd_submit(args: argparse.Namespace) -> dict[str, Any]: flavor=flavor, env=_job_env(sub), secrets=None, - token=_token(), + token=token, timeout=sub.target.get("timeout"), + **_ns_kwargs(namespace), ) except Exception as exc: # noqa: BLE001 - hub raises a wide family of errors submit_lib.finish( @@ -170,16 +289,49 @@ def cmd_submit(args: argparse.Namespace) -> dict[str, Any]: ) from exc job_id = getattr(job, "id", None) or getattr(job, "job_id", None) or str(job) - submit_lib.attach_handle(run_id, {"job_id": job_id, "flavor": flavor}) + # The namespace goes onto the *handle*, not just into the submit call. This + # is the whole point of §17: every later `inspect_job` / `fetch_job_logs` + # reads it from here, so a job submitted to an org is collectable from it. + submit_lib.attach_handle( + run_id, {"job_id": job_id, "flavor": flavor, "namespace": namespace} + ) return { "run_id": run_id, "job_id": job_id, "flavor": flavor, + "namespace": namespace, + "identity": identity, + "project": project_id, "gates": summary, + "warnings": warnings, "next": f"python -m tools.jobs collect {run_id} --json", } +def _namespace_warnings(sub: Submission, namespace: str | None) -> list[str]: + """Warn -- not refuse -- when smoke ran under a different namespace. + + The submission hash deliberately excludes `target` (which is why `flavor` is + not hashed), and `namespace` follows the same rule for consistency. The + consequence is real and handled rather than ignored: a preflight whose + `smoke` check ran under personal credentials validates a job that will run + in an organization. Warn, not refuse, consistent with how `target` and + `flavor` already behave. + """ + record = gates.preflight_record(sub.hash()) or {} + smoke = (record.get("checks") or {}).get("smoke") or {} + if "namespace" not in smoke: + return [] + used = smoke.get("namespace") + if used == namespace: + return [] + return [ + f"the smoke check ran under namespace {used or 'personal'} but this job runs under " + f"{namespace or 'personal'}; the environment it validated may differ " + "(namespace is not part of the submission hash, by the same rule as flavor)" + ] + + def _command_for(sub: Submission) -> list[str]: if sub.target.get("command"): return [str(c) for c in sub.target["command"]] @@ -196,7 +348,13 @@ def _job_env(sub: Submission) -> dict[str, str]: # --------------------------------------------------------------------------- # smoke # --------------------------------------------------------------------------- -def run_smoke(sub: Submission, cfg: Config) -> dict[str, Any]: +def run_smoke( + sub: Submission, + cfg: Config, + *, + namespace: str | None = None, + project: str | None = None, +) -> dict[str, Any]: """The §6 carve-out: gate-exempt, hard-capped, still ledgered. The caps are applied here in code -- one step, a wall-clock ceiling of @@ -210,9 +368,12 @@ def run_smoke(sub: Submission, cfg: Config) -> dict[str, Any]: caps = gates.check_smoke_caps(sub, cfg) flavor = sub.target.get("smoke_flavor") or sub.target.get("flavor") or cfg.get("hf", "default_flavor", "a10g-small") command = _smoke_command(sub, caps) + if namespace is None: + namespace = resolve_namespace(None, sub, cfg, project or budget.current_project()) run_id = submit_lib.record_smoke_run( - sub, cfg=cfg, platform=PLATFORM, target={"flavor": flavor, "platform": "hf"}, - caps=caps, command=command, + sub, cfg=cfg, platform=PLATFORM, + target={"flavor": flavor, "platform": "hf", "namespace": namespace}, + caps=caps, command=command, project=project, ) artifacts = submit_lib.artifacts_dir(run_id) @@ -225,6 +386,7 @@ def run_smoke(sub: Submission, cfg: Config) -> dict[str, Any]: env={**_job_env(sub), "GRAD_SMOKE": "1"}, token=_token(), timeout=caps["timeout_s"], + **_ns_kwargs(namespace), ) except ConfigError: raise @@ -234,13 +396,14 @@ def run_smoke(sub: Submission, cfg: Config) -> dict[str, Any]: artifacts_dir=artifacts, expectation=None, extra={"error": str(exc)}, ) return {"ok": False, "reason": f"smoke submission failed: {exc}", - "fix": "check the HF token scope and the image digest", "run_id": run_id} + "fix": "check the HF token scope and the image digest", "run_id": run_id, + "namespace": namespace} job_id = getattr(job, "id", None) or getattr(job, "job_id", None) or str(job) - submit_lib.attach_handle(run_id, {"job_id": job_id, "flavor": flavor}) + submit_lib.attach_handle(run_id, {"job_id": job_id, "flavor": flavor, "namespace": namespace}) - state, info = _poll(job_id, deadline=time.time() + caps["timeout_s"]) - logs = _logs(job_id) + state, info = _poll(job_id, deadline=time.time() + caps["timeout_s"], namespace=namespace) + logs = _logs(job_id, namespace=namespace) (artifacts / "smoke.log").write_text(logs, encoding="utf-8") cost = _actual_cost(info, flavor, cfg) ok = state == "COMPLETED" @@ -259,6 +422,10 @@ def run_smoke(sub: Submission, cfg: Config) -> dict[str, Any]: "job_id": job_id, "state": state, "flavor": flavor, + # Recorded into the check result, which flows into the preflight record. + # `submit` compares it against the namespace the real job will use and + # warns on a difference -- see `_namespace_warnings`. + "namespace": namespace, "cost_usd": cost, "caps": caps, "log": str(artifacts / "smoke.log"), @@ -282,12 +449,13 @@ def _smoke_command(sub: Submission, caps: dict[str, Any]) -> list[str]: # --------------------------------------------------------------------------- # status / collect # --------------------------------------------------------------------------- -def _poll(job_id: str, *, deadline: float) -> tuple[str, Any]: +def _poll(job_id: str, *, deadline: float, namespace: str | None = None) -> tuple[str, Any]: hub = _hub() + ns = _ns_kwargs(namespace) info: Any = None state = "UNKNOWN" while True: - info = hub.inspect_job(job_id=job_id, token=_token()) + info = hub.inspect_job(job_id=job_id, token=_token(), **ns) state = _state_of(info) if state in ("COMPLETED", "ERROR", "CANCELED", "FAILED") or time.time() > deadline: return state, info @@ -304,9 +472,14 @@ def _state_of(info: Any) -> str: return "UNKNOWN" -def _logs(job_id: str) -> str: +def _logs(job_id: str, *, namespace: str | None = None) -> str: try: - return "\n".join(str(line) for line in _hub().fetch_job_logs(job_id=job_id, token=_token())) + return "\n".join( + str(line) + for line in _hub().fetch_job_logs( + job_id=job_id, token=_token(), **_ns_kwargs(namespace) + ) + ) except Exception as exc: # noqa: BLE001 - logs are best-effort; never fail a collect over them return f"(could not fetch logs: {exc})" @@ -354,10 +527,18 @@ def cmd_status(args: argparse.Namespace) -> dict[str, Any]: "stale": ls.is_stale(r), "submitted_at": r.get("submitted_at"), "estimate_usd": r.get("estimate_usd"), + "project": r.project, + "namespace": handle.get("namespace"), } if handle.get("job_id") and not r.collected: try: - info = _hub().inspect_job(job_id=handle["job_id"], token=_token()) + # From the handle, not re-resolved: the config or the current project + # may have changed since submit, and looking under a namespace the + # job was not submitted to is exactly the 404 that makes a run go + # stale and block every later submission. + info = _hub().inspect_job( + job_id=handle["job_id"], token=_token(), **_ns_kwargs(handle.get("namespace")) + ) payload["remote_state"] = _state_of(info) except ConfigError: raise @@ -391,8 +572,9 @@ def cmd_collect(args: argparse.Namespace) -> dict[str, Any]: fix=f"python -m tools.ledger show {r.id} --json", ) + namespace = handle.get("namespace") deadline = time.time() + (args.timeout if args.wait else 0) - state, info = _poll(job_id, deadline=deadline) + state, info = _poll(job_id, deadline=deadline, namespace=namespace) if state not in ("COMPLETED", "ERROR", "CANCELED", "FAILED"): raise GradError( "still_running", @@ -403,7 +585,7 @@ def cmd_collect(args: argparse.Namespace) -> dict[str, Any]: ) artifacts = submit_lib.artifacts_dir(r.id) - logs = _logs(job_id) + logs = _logs(job_id, namespace=namespace) (artifacts / "job.log").write_text(logs, encoding="utf-8") results: dict[str, Any] = {} @@ -479,6 +661,7 @@ def _download_artifacts(r: ls.Run, dest: Path) -> None: credentials.OPENROUTER_KEY, credentials.VOYAGE_KEY, credentials.S2_KEY, + credentials.CONTEXT7_KEY, ) @@ -492,7 +675,18 @@ def cmd_credential(args: argparse.Namespace) -> dict[str, Any]: """Credentials live in Windows Credential Manager, never in the environment and never in a file under the workspace.""" if args.action == "status": - return {"credentials": credentials.status(), "service": credentials.SERVICE} + payload: dict[str, Any] = { + "credentials": credentials.status(), + "service": credentials.SERVICE, + } + # Which namespaces the HF token can actually act for (§17). Surfaced + # here so "the org submit failed" is diagnosable without a submission. + if credentials.present(credentials.HF_TOKEN): + try: + payload["hf_identity"] = validate_namespace(None, _token()) + except GradError as exc: + payload["hf_identity"] = {"error": exc.message, "fix": exc.fix} + return payload if not args.name: raise UsageError("give a credential name", fix=f"one of: {', '.join(CREDENTIAL_NAMES)}") if args.action == "delete": @@ -507,13 +701,17 @@ def cmd_credential(args: argparse.Namespace) -> dict[str, Any]: return {"stored": args.name} -@cli.command("ceilings", "show the spend ceilings and current rolling total") -def cmd_ceilings(_: argparse.Namespace) -> dict[str, Any]: +@cli.command( + "ceilings", + "show the spend ceilings and current rolling total", + setup=lambda p: p.add_argument("--project", help="also show this project's allocation"), +) +def cmd_ceilings(args: argparse.Namespace) -> dict[str, Any]: cfg = config_mod.load() window = int(cfg.get("spend", "window_days", 30)) rolling = ls.rolling_spend(window) stale = [r.id for r in ls.stale_runs(cfg=cfg)] - return { + payload: dict[str, Any] = { "per_job_usd": cfg.get("spend", "per_job_usd"), "monthly_usd": cfg.get("spend", "monthly_usd"), "rolling": {k: v for k, v in rolling.items() if k != "runs"}, @@ -521,6 +719,13 @@ def cmd_ceilings(_: argparse.Namespace) -> dict[str, Any]: "stale_runs": stale, "blocked": bool(stale), } + # Two ceilings, never conflated: the machine's, and this research's (§15). + project_id = budget.resolve(args.project) + if project_id and budget.exists(project_id): + payload["project"] = budget.status(project_id) + else: + payload["project"] = {"project": project_id, "bounded": False} + return payload if __name__ == "__main__": diff --git a/tools/lab.py b/tools/lab.py new file mode 100644 index 0000000..b3e05e4 --- /dev/null +++ b/tools/lab.py @@ -0,0 +1,347 @@ +"""grad-lab -- the embedded JupyterLab server (HANDOFF-2 §19). + +Amends §10's notebook handling, and §14 explains why this honours the original +reasoning rather than reversing it: the rejection was of *building a notebook +editor*, and that still stands -- we build none. What changed is that "editing +links out to Lab" was never wired up; `ui/app.py` pointed at a `localhost:8888` +nobody started. Embedding the real Lab is what makes arbitrary Lab extensions +possible at all. + +**The kernel-ownership rule, which must not be lost.** `tools/nb.py` spawns +detached kernels through its own connection files. Lab has its own kernel +manager. Two owners over one notebook reproduces exactly the "works in the +kernel that grew it" failure that `nb verify` exists to catch. So the discipline +is unchanged: **anything edited in Lab passes `nb verify` before it is cited in +`notes/` or referenced from a ledger entry.** The Verify button in the +Notebooks tab is worth more than the embed, and it is why that was built first. + +**Three things to know before installing an extension:** + +1. *Server extensions run as you.* A frontend extension is confined to the + browser; a server extension runs in this process with your filesystem + rights -- it can read `ledger/` and `notes/`, and it can `import keyring` and + reach the credential store. That is the same honest residual + `core/credentials.py` already names. Read a server extension before + installing it. +2. *Origin.* An extension is code running in Lab's origin, and the Lab iframe is + deliberately unsandboxed. Lab stays on its own port and never shares the UI's + storage secret. +3. *Pin everything.* The JupyterLab 3->4 break is what killed the Tabnine + extension. `pyproject.toml`'s `lab` extra pins JupyterLab itself and every + extension, so an unrelated `pip install -U` cannot take the app down. + +"Connect an arbitrary extension" therefore means: add a pin, reinstall, restart. +""" + +from __future__ import annotations + +import argparse +import os +import secrets +import shutil +import socket +import subprocess +import time +from pathlib import Path +from typing import Any + +from core import jsonl, paths +from core.cli import Cli, main +from core.errors import ConfigError, GradError + +cli = Cli( + "grad-lab", + "Manage the JupyterLab server the UI embeds. Human editing surface; the " + "agent still edits notebooks through Write/Edit plus tools/nb.py.", + epilog=( + "Lab and tools/nb.py are two kernel owners over one notebook, which is the\n" + "'works in the kernel that grew it' failure nb verify exists to catch. The rule\n" + "is unchanged:\n\n" + " python -m tools.nb verify notebooks/.ipynb --json\n\n" + "before anything edited here is cited in notes/ or referenced from the ledger.\n\n" + "Server extensions run with your filesystem rights and can reach the credential\n" + "store. Read one before installing it; frontend-only extensions are low risk." + ), +) + +DEFAULT_PORT = 8889 + + +def _state_path() -> Path: + return paths.data_dir() / "lab" / "lab.json" + + +def _log_path() -> Path: + return paths.data_dir() / "lab" / "lab.log" + + +def _jupyter_config_dir() -> Path: + return paths.root() / "config" / "jupyter" + + +def _read_state() -> dict[str, Any]: + return jsonl.read_json(_state_path()) or {} + + +def _executable() -> str: + """The `jupyter` entry point, or a clear error naming the extra to install.""" + found = shutil.which("jupyter") + if found: + return found + raise ConfigError( + "jupyter is not installed, so there is no Lab server to start", + fix="pip install -e '.[lab]' # pins jupyterlab and every extension", + ) + + +def _free_port(preferred: int) -> int: + """The preferred port if it is free, otherwise an OS-assigned one. + + Reported back rather than assumed: the UI reads the port out of the state + file, so a fallback does not strand the iframe on a dead address. + """ + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + try: + probe.bind(("127.0.0.1", preferred)) + return preferred + except OSError: + pass + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.bind(("127.0.0.1", 0)) + return int(probe.getsockname()[1]) + + +def _listening(port: int) -> bool: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.settimeout(0.4) + return probe.connect_ex(("127.0.0.1", port)) == 0 + + +def _alive(pid: int | None) -> bool: + if not pid: + return False + if os.name == "nt": + out = subprocess.run( + ["tasklist", "/FI", f"PID eq {pid}", "/NH"], + capture_output=True, text=True, check=False, + ) + return str(pid) in (out.stdout or "") + try: + os.kill(pid, 0) + except (ProcessLookupError, PermissionError): + return False + return True + + +# --------------------------------------------------------------------------- +def _start_args(p: argparse.ArgumentParser) -> None: + p.add_argument("--port", type=int, default=DEFAULT_PORT, help=f"preferred port (default {DEFAULT_PORT})") + p.add_argument( + "--ui-origin", + default="http://127.0.0.1:8080", + help="the Grad UI's origin, which is the only origin permitted to frame Lab", + ) + p.add_argument("--force", action="store_true", help="start even if a server looks alive") + + +@cli.command("start", "start the Lab server and return its port and token", setup=_start_args) +def cmd_start(args: argparse.Namespace) -> dict[str, Any]: + """Detached, on 127.0.0.1, behind a freshly minted token. + + The token is new on every start rather than persisted: it is written to a + state file under `data/`, and a long-lived secret in a workspace file is a + worse trade than re-reading the file after a restart. + """ + state = _read_state() + if not args.force and state.get("port") and _listening(int(state["port"])) and _alive(state.get("pid")): + return {**state, "already_running": True, + "next": "python -m tools.lab status --json"} + + executable = _executable() + port = _free_port(args.port) + token = secrets.token_urlsafe(32) + log = _log_path() + log.parent.mkdir(parents=True, exist_ok=True) + + env = { + **os.environ, + # Read by config/jupyter/jupyter_server_config.py, so the framing + # headers and the actual port cannot drift apart. + "GRAD_UI_ORIGIN": args.ui_origin, + "GRAD_LAB_PORT": str(port), + "JUPYTER_CONFIG_DIR": str(_jupyter_config_dir()), + } + argv = [ + executable, "lab", + "--no-browser", + f"--port={port}", + "--ip=127.0.0.1", + f"--IdentityProvider.token={token}", + f"--ServerApp.root_dir={paths.root()}", + f"--ServerApp.config_file={_jupyter_config_dir() / 'jupyter_server_config.py'}", + ] + + creationflags = 0 + start_new_session = False + if os.name == "nt": + creationflags = getattr(subprocess, "DETACHED_PROCESS", 0) | getattr( + subprocess, "CREATE_NEW_PROCESS_GROUP", 0 + ) + else: + start_new_session = True + + with open(log, "ab") as fh: + proc = subprocess.Popen( + argv, cwd=str(paths.root()), stdout=fh, stderr=subprocess.STDOUT, + stdin=subprocess.DEVNULL, env=env, + creationflags=creationflags, start_new_session=start_new_session, + ) + + deadline = time.time() + 30 + while time.time() < deadline and not _listening(port): + if proc.poll() is not None: + raise GradError( + "lab_died", + f"the Lab server exited immediately (code {proc.returncode})", + exit_code=8, + fix=f"read {log}", + detail={"log": str(log)}, + ) + time.sleep(0.3) + + record = { + "port": port, + "token": token, + "pid": proc.pid, + "url": f"http://127.0.0.1:{port}/lab", + "root_dir": str(paths.root()), + "ui_origin": args.ui_origin, + "log": str(log), + "started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "listening": _listening(port), + } + jsonl.write_json(_state_path(), record) + if not record["listening"]: + raise GradError( + "lab_not_listening", + f"the Lab server did not start listening on port {port} within 30s", + exit_code=8, + fix=f"read {log}", + detail=record, + ) + return { + **record, + # The rule that survives the embed. + "discipline": ( + "anything edited in Lab passes `python -m tools.nb verify --json` " + "before it is cited in notes/ or referenced from a ledger entry" + ), + } + + +@cli.command("status", "is the Lab server up, and where") +def cmd_status(_: argparse.Namespace) -> dict[str, Any]: + state = _read_state() + if not state: + return {"running": False, "fix": "python -m tools.lab start --json"} + port = int(state.get("port") or 0) + return { + **{k: v for k, v in state.items() if k != "token"}, + "running": bool(port and _listening(port) and _alive(state.get("pid"))), + "process_alive": _alive(state.get("pid")), + # The token is what the iframe needs and what a screenshot should not + # carry. `lab url` is the deliberate way to get one that includes it. + "token_available": bool(state.get("token")), + } + + +@cli.command( + "url", + "the full URL for one notebook, token included (for the UI)", + setup=lambda p: p.add_argument("path", nargs="?", help="notebook path relative to the workspace"), +) +def cmd_url(args: argparse.Namespace) -> dict[str, Any]: + state = _read_state() + if not state.get("port"): + raise GradError( + "lab_not_started", "the Lab server is not running", exit_code=3, + fix="python -m tools.lab start --json", + ) + base = f"http://127.0.0.1:{state['port']}/lab" + target = f"{base}/tree/{args.path}" if args.path else base + return {"url": f"{target}?token={state['token']}", "port": state["port"]} + + +@cli.command("extensions", "what is installed, so the state is inspectable") +def cmd_extensions(_: argparse.Namespace) -> dict[str, Any]: + """Lab already has a plugin system; we do not design one. + + What this builds is the reproducibility layer: the extension set is declared + in `pyproject.toml`'s `lab` extra rather than accumulated, and this command + is how you see what actually ended up installed. + """ + executable = _executable() + env = {**os.environ, "JUPYTER_CONFIG_DIR": str(_jupyter_config_dir())} + + def _run(argv: list[str]) -> dict[str, Any]: + try: + out = subprocess.run( + argv, capture_output=True, text=True, timeout=120, env=env, check=False + ) + except subprocess.TimeoutExpired: + return {"ok": False, "output": "timed out"} + return { + "ok": out.returncode == 0, + "output": ((out.stdout or "") + (out.stderr or "")).strip().splitlines(), + } + + return { + "frontend": _run([executable, "labextension", "list"]), + # The ones that matter for the §19 caveat: a server extension runs in + # the Lab process with your filesystem rights. + "server": _run([executable, "server", "extension", "list"]), + "declared_in": str(paths.root() / "pyproject.toml") + " [project.optional-dependencies] lab", + "caveat": ( + "server extensions run as you: they can read ledger/ and notes/, and can " + "import keyring and reach the credential store. Read one before installing it." + ), + } + + +@cli.command("stop", "stop the Lab server") +def cmd_stop(_: argparse.Namespace) -> dict[str, Any]: + state = _read_state() + pid = state.get("pid") + if not pid or not _alive(pid): + jsonl.write_json(_state_path(), {}) + return {"stopped": False, "note": "no Lab server was running"} + if os.name == "nt": + subprocess.run(["taskkill", "/PID", str(pid), "/T", "/F"], capture_output=True, check=False) + else: + import signal # noqa: PLC0415 + + try: + os.killpg(os.getpgid(pid), signal.SIGTERM) + except (ProcessLookupError, PermissionError): + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + pass + deadline = time.time() + 10 + while time.time() < deadline and _alive(pid): + time.sleep(0.2) + jsonl.write_json(_state_path(), {}) + return {"stopped": True, "pid": pid, "still_alive": _alive(pid)} + + +# --------------------------------------------------------------------------- +def lab_state() -> dict[str, Any]: + """Read by the UI so the Lab tab knows where to point its iframe.""" + state = _read_state() + port = int(state.get("port") or 0) + state["running"] = bool(port and _listening(port)) + return state + + +if __name__ == "__main__": + main(cli) diff --git a/tools/paper_search.py b/tools/paper_search.py index 2af017d..9a101f1 100644 --- a/tools/paper_search.py +++ b/tools/paper_search.py @@ -68,7 +68,7 @@ def cmd_search(args: argparse.Namespace) -> dict[str, Any]: hyde = None if not args.no_expand: expansion = haiku.expand( - args.question, model=str(cfg.get("retrieval", "expand_model")), log_name=log_name + args.question, model=cfg.model_for("expand"), log_name=log_name ) queries = list(expansion["queries"]) hyde = expansion["hyde"] @@ -147,7 +147,7 @@ def cmd_search(args: argparse.Namespace) -> dict[str, Any]: survivors = ranked[:top] if not args.no_triage and ranked: verdicts = haiku.triage( - args.question, ranked, model=str(cfg.get("retrieval", "triage_model")), log_name=log_name + args.question, ranked, model=cfg.model_for("triage"), log_name=log_name ) reasons = {v["id"]: v.get("reason", "") for v in verdicts if v.get("keep")} kept = [c for c in ranked if c["id"] in reasons] diff --git a/tools/quota.py b/tools/quota.py index 94899e7..71825ee 100644 --- a/tools/quota.py +++ b/tools/quota.py @@ -33,11 +33,23 @@ def _summary_args(p: argparse.ArgumentParser) -> None: p.add_argument("--days", type=int, help="restrict to the last N days") p.add_argument("--stage", help="only this stage") + p.add_argument("--role", help="only this model role (research, evolve, expand, triage, report, cite)") + p.add_argument("--project", help="only this project's usage") -@cli.command("summary", "totals by stage", setup=_summary_args) +@cli.command("summary", "totals by stage, role, and project", setup=_summary_args) def cmd_summary(args: argparse.Namespace) -> dict[str, Any]: - summary = quota_log.summarise(args.days) + """`--role` is what answers "what did Opus cost me this week" without + inferring the role from a model id (§16).""" + summary = quota_log.summarise(args.days, project=args.project) + if args.role: + by_role = summary["by_role"] + if args.role not in by_role: + raise UsageError( + f"no usage recorded for role {args.role!r}", + fix=f"known roles: {', '.join(by_role) or '(none yet)'}", + ) + summary["by_role"] = {args.role: by_role[args.role]} if args.stage: by_stage = summary["by_stage"] if args.stage not in by_stage: @@ -108,6 +120,8 @@ def _record_args(p: argparse.ArgumentParser) -> None: p.add_argument("--output-tokens", type=int, default=0) p.add_argument("--credits-usd", type=float, default=0.0) p.add_argument("--unit", choices=["quota", "credits"], default="quota") + p.add_argument("--role", help="the §16 model role this call filled") + p.add_argument("--project", help="override the current project for this record") p.add_argument("--session") @@ -128,6 +142,8 @@ def cmd_record(args: argparse.Namespace) -> dict[str, Any]: output_tokens=args.output_tokens, credits_usd=args.credits_usd, unit=args.unit, + role=args.role, + project=args.project, session=args.session, ) } diff --git a/tools/report.py b/tools/report.py new file mode 100644 index 0000000..8a454f7 --- /dev/null +++ b/tools/report.py @@ -0,0 +1,843 @@ +"""grad-report -- the scientific report (HANDOFF-2 §22). + + "`check` refuses; it does not warn. A report generator is where this + system's epistemics either hold or collapse -- the whole design exists to + stop the user believing results too easily, and a paper generator is a + machine for asserting them confidently." + +**Built, not adopted.** Every surveyed harness reconstructs provenance from +unstructured experiment logs. Grad's is already structured -- expectations with +`basis` and `comparability`, runs with results and `deviations`, verdicts with +notes, figures, corpus paper ids. Adopting AI Scientist v2, PaperOrchestra, +Denario, Camyla, Jr. AI Scientist, or CiteLLM means discarding that advantage +and conforming to its log format. + +What *was* worth stealing, and from whom: + + * **Camyla** -- the two-pass citation flow. `write` emits `[CITE:keyword]` + placeholders; `cite` extracts a context window around each and verifies the + candidate's title and abstract against that context. Much better than citing + inline, where a model invents a plausible reference in the moment. + * **PaperOrchestra** -- the constraint set (keys match the bib exactly, no + fabricated results, compile-clean LaTeX), encoded as validation rather than + as prompt text. + * **Denario** -- progressive versions. It emits four because unattended LaTeX + does not reliably compile; `build` checkpoints for the same reason. + * **AI Scientist v2** -- the role split, which maps onto §16: `report` writes + the prose, `cite` resolves the citations. + * **Jr. AI Scientist** -- draft -> reflect -> adjust inside a template + directory. + +The pipeline, and only `write` costs anything: + + draft deterministic skeleton from the ledger. No model. Useful on its own. + write prose plus [CITE:...] placeholders. <- the only paid step + cite resolve, verify, emit references.bib. + check the gate. Refuses on an unresolved claim, an unverified citation, + an unjudged deviation, or LaTeX that will not compile. + build PDF, with progressive checkpoints. +""" + +from __future__ import annotations + +import argparse +import json +import re +import shutil +import subprocess +from pathlib import Path +from typing import Any + +from core import ( + budget, + config as config_mod, + corpus, + http, + quota_log, + report as report_lib, +) +from core.cli import Cli, main +from core.errors import EXIT_CHECK_FAILED, ConfigError, GradError, UpstreamError + +cli = Cli( + "grad-report", + "Generate a scientific report from the ledger, with every number and every " + "citation mechanically traceable.", + epilog=( + "`check` enforces four rules, in order:\n" + " 1. every \\gradnum{} key resolves to a (run_id, quantity) in the ledger,\n" + " with a matching value;\n" + " 2. every \\cite{} key exists in references.bib, and every bib entry came\n" + " from the corpus or a verified S2 id;\n" + " 3. no cited run has an unjudged deviation;\n" + " 4. the LaTeX compiles clean.\n\n" + "Rule 3 is the one most in the spirit of this system: you should not be able\n" + "to write up a result you have not judged.\n\n" + "`draft` costs nothing and needs no model. Run it first." + ), +) + + +def _project(args: argparse.Namespace) -> str: + return budget.resolve_or_fail(getattr(args, "project", None), what="a report") + + +def _project_arg(p: argparse.ArgumentParser) -> None: + p.add_argument("--project", help="the project to report on (defaults to the current one)") + + +# --------------------------------------------------------------------------- +# draft +# --------------------------------------------------------------------------- +_PREAMBLE = r"""\documentclass[%(classoptions)s]{%(documentclass)s} +%(style)s\usepackage[margin=1in]{geometry} +\usepackage{amsmath,amssymb} +\usepackage{booktabs} +\usepackage{graphicx} +\usepackage{hyperref} +\usepackage{natbib} + +%% Every asserted number goes through this macro. It renders as the number and +%% it is what `report check` verifies against the ledger: the key indexes +%% claims.json, which maps it to a (run_id, quantity). A number typed directly +%% into the prose is a number nothing can check. +\newcommand{\gradnum}[1]{\csname gradval@#1\endcsname} +\input{claims.tex} + +\title{%(title)s} +\author{Grad} +\date{\today} + +\begin{document} +\maketitle +""" + + +def _slug(text: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-") or "claim" + + +@cli.command("draft", "the skeleton, straight from the ledger (no model)", setup=_project_arg) +def cmd_draft(args: argparse.Namespace) -> dict[str, Any]: + """Deterministic and free. + + "It is useful on its own and costs nothing." Every expectation, its runs, + its deviations, its verdict, its figures -- including the runs that went + badly, because a skeleton that quietly omits them is a skeleton that invites + writing up only what worked. + """ + project_id = _project(args) + evidence = report_lib.project_evidence(project_id) + files = report_lib.paths_for(project_id) + files["dir"].mkdir(parents=True, exist_ok=True) + + claims: dict[str, Any] = {} + body: list[str] = [] + body.append(r"\section{Results}") + body.append("") + + if not evidence["expectations"]: + body.append( + "No expectation in this project has a bound run yet. " + "The skeleton is empty on purpose rather than invented.\n" + ) + + for block in evidence["expectations"]: + exp = block["expectation"] + body.append(rf"\subsection{{{_tex_escape(exp.get('claim') or exp['quantity'])}}}") + body.append("") + body.append(rf"\label{{exp:{_slug(exp['id'])}}}") + body.append("") + predicted = exp.get("predicted") or {} + body.append( + "Pre-registered prediction: " + + _tex_escape(_describe_prediction(exp["quantity"], predicted)) + + f" (confidence: {exp.get('confidence', 'unstated')})." + ) + if exp.get("comparability"): + body.append("") + body.append("Comparability: " + _tex_escape(exp["comparability"])) + for basis in exp.get("basis") or []: + body.append("") + body.append( + "Basis: " + + _tex_escape( + f"{basis.get('paper', '?')} ({basis.get('locator', '')}) reports " + f"{basis.get('value')} under {basis.get('conditions', 'unstated conditions')}" + ) + + f" [CITE:{basis.get('paper', 'unknown')}]" + ) + body.append("") + + for run in block["runs"]: + for quantity, value in sorted((run["results"] or {}).items()): + key = f"{_slug(run['id'])}-{_slug(quantity)}" + claims[key] = { + "run_id": run["id"], + "quantity": quantity, + "value": value, + "task": run["task"], + } + body.append( + rf"Run \texttt{{{_tex_escape(run['id'])}}} measured " + rf"{_tex_escape(quantity)} = \gradnum{{{key}}}." + ) + for dev in run["deviations"]: + verdict = dev.get("verdict") + body.append("") + body.append( + _tex_escape( + f"Deviation on {dev.get('quantity')}: " + f"{'in range' if dev.get('in_range') is True else 'out of range or unsettled'}" + + (f", judged {verdict}: {dev.get('note') or ''}" if verdict + else ". NOT YET JUDGED -- `report check` will refuse while this stands.") + ) + ) + body.append("") + + if evidence["figures"]: + body.append(r"\section{Figures}") + for figure in evidence["figures"]: + rel = Path(figure).as_posix() + body.append(r"\begin{figure}[h]\centering") + body.append(rf"\includegraphics[width=0.7\linewidth]{{{rel}}}") + body.append(rf"\caption{{{_tex_escape(Path(figure).name)}}}") + body.append(r"\end{figure}") + + if evidence["unbound_runs"]: + body.append(r"\section{Runs with no bound expectation}") + body.append( + _tex_escape( + "These ran without a pre-registered prediction and are listed so their " + "absence from the results above is visible rather than silent: " + + ", ".join(r["id"] for r in evidence["unbound_runs"]) + ) + ) + + cfg = config_mod.load() + style = str(cfg.get("report", "style", "") or "") + tex = ( + _PREAMBLE + % { + "title": _tex_escape(f"Report: {project_id}"), + "documentclass": cfg.get("report", "documentclass", "article"), + "classoptions": cfg.get("report", "classoptions", "11pt"), + # A conference style is a vendored .sty dropped into the report + # directory, named here. Empty by default so the skeleton compiles + # on a stock TeX installation with nothing vendored. + "style": f"\\usepackage{{{style}}}\n" if style else "", + } + ) + "\n".join(body) + tex += ( + f"\n\n\\bibliographystyle{{{cfg.get('report', 'bibstyle', 'plainnat')}}}\n" + "\\bibliography{references}\n\\end{document}\n" + ) + + files["tex"].write_text(tex, encoding="utf-8") + files["claims"].write_text(json.dumps(claims, indent=2, ensure_ascii=False, default=str), encoding="utf-8") + _write_claims_tex(project_id, claims) + + return { + "project": project_id, + "tex": str(files["tex"]), + "claims": str(files["claims"]), + "claim_count": len(claims), + "expectations": len(evidence["expectations"]), + "runs": evidence["run_count"], + "unjudged": report_lib.unjudged_for({c["run_id"] for c in claims.values()}), + "next": f"python -m tools.report write --project {project_id} --json", + } + + +def _describe_prediction(quantity: str, predicted: dict[str, Any]) -> str: + low, high, direction = predicted.get("low"), predicted.get("high"), predicted.get("direction") + if low is not None and high is not None: + return f"{quantity} between {low} and {high}" + if direction: + return f"{quantity} should {direction.replace('_', ' ')}" + if low is not None: + return f"{quantity} at least {low}" + if high is not None: + return f"{quantity} at most {high}" + return quantity + + +def _write_claims_tex(project_id: str, claims: dict[str, Any]) -> Path: + """Materialise claims.json as the macro definitions `\\gradnum` expands. + + Generated, never hand-edited: the sidecar is the checkable artifact and this + file is its rendering. Editing the rendering would let a number drift away + from the run it claims to come from, which is precisely what §22 forbids. + """ + lines = [ + "% Generated by `python -m tools.report draft`. Do not edit.", + "% Each value is the one recorded in the ledger for its (run_id, quantity).", + ] + for key, entry in sorted(claims.items()): + value = entry.get("value") + lines.append(rf"\expandafter\def\csname gradval@{key}\endcsname{{{_tex_escape(str(value))}}}") + path = report_lib.paths_for(project_id)["dir"] / "claims.tex" + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + return path + + +def _tex_escape(text: str) -> str: + out = str(text) + for char, replacement in ( + ("\\", r"\textbackslash{}"), + ("&", r"\&"), ("%", r"\%"), ("$", r"\$"), ("#", r"\#"), + ("_", r"\_"), ("{", r"\{"), ("}", r"\}"), ("~", r"\textasciitilde{}"), + ("^", r"\textasciicircum{}"), + ): + out = out.replace(char, replacement) + return out + + +# --------------------------------------------------------------------------- +# write +# --------------------------------------------------------------------------- +WRITE_PROMPT = """You write the prose of a scientific report from a structured evidence bundle. + +Call submit_prose exactly once, then stop. + +Rules that are enforced mechanically after you, so violating them wastes a run: + +- **Never state a number directly.** Every measured value is referenced as + \\gradnum{key}, using a key from the claims list you were given. A number typed + into the prose fails the check. +- **Never write a \\cite{}.** Where a citation belongs, write [CITE:keyword] + with a keyword describing what should be cited. A later pass resolves those + against a real corpus; anything you invent here is deleted. +- Do not claim a result whose deviation is unjudged. Those are marked in the + bundle. Describe them as open questions, not findings. +- A surprise is an alarm: where a result lands far outside its pre-registered + range, say so and treat a bug as the first hypothesis. +- Prefer the relational framing the predictions use over absolute numbers. + +Write: abstract, introduction, method, results, discussion, limitations. LaTeX +body only -- no preamble, no \\begin{document}. +""" + + +def _write_args(p: argparse.ArgumentParser) -> None: + _project_arg(p) + p.add_argument("--section", action="append", default=[], help="only regenerate these sections") + p.add_argument("--dry-run", action="store_true", help="show the bundle that would be sent, and stop") + + +@cli.command("write", "prose plus [CITE:...] placeholders (costs quota)", setup=_write_args) +def cmd_write(args: argparse.Namespace) -> dict[str, Any]: + """The only step that spends anything. + + §23 item 5 leaves open whether this should run while a project is over + budget -- the report is how you find out what the spend bought, but it is + also a cost-bearing loop like any other. Specified as denied by the §15 + hook, and implemented that way here too so the CLI and the hook agree. + """ + project_id = _project(args) + files = report_lib.paths_for(project_id) + if not files["tex"].exists(): + raise GradError( + "no_draft", + "there is no draft to write into", + exit_code=3, + fix=f"python -m tools.report draft --project {project_id} --json", + ) + + over = budget.over_budget(project_id) + if over: + from core.errors import EXIT_PROJECT_BUDGET, GateRefusal + + raise GateRefusal( + "project_budget", + f"project {project_id!r} is over budget on {', '.join(over)}; " + "`write` is a cost-bearing loop like any other", + EXIT_PROJECT_BUDGET, + fix=( + f"python -m tools.budget raise --project {project_id} " + f"--{over[0].replace('_', '-')} --json\n" + f" `python -m tools.report draft --project {project_id}` is free and " + "shows what the spend bought" + ), + ) + + evidence = report_lib.project_evidence(project_id) + claims = report_lib.load_claims(project_id) + bundle = _bundle(evidence, claims) + if args.dry_run: + return {"project": project_id, "bundle": bundle, "sent": False} + + cfg = config_mod.load() + prose = _generate_prose(bundle, model=cfg.model_for("report")) + + body = files["tex"].read_text(encoding="utf-8") + marker = "\\maketitle" + head, _, tail = body.partition(marker) + files["tex"].write_text(head + marker + "\n\n" + prose + "\n\n" + tail, encoding="utf-8") + + return { + "project": project_id, + "tex": str(files["tex"]), + "model": cfg.model_for("report"), + "placeholders": sorted(set(report_lib.PLACEHOLDER_RE.findall(prose))), + "next": f"python -m tools.report cite --project {project_id} --json", + } + + +def _bundle(evidence: dict[str, Any], claims: dict[str, Any]) -> dict[str, Any]: + """What the model is allowed to see: structured evidence and claim keys. + + Not the raw ledger -- the model does not need run ids it cannot cite, and + handing it more numbers than it has keys for is how a number ends up in the + prose without a `\\gradnum` around it. + """ + return { + "project": evidence["project"], + "claims": { + key: {"quantity": c["quantity"], "task": c.get("task")} + for key, c in claims.items() + }, + "expectations": [ + { + "claim": b["expectation"].get("claim"), + "quantity": b["expectation"].get("quantity"), + "predicted": b["expectation"].get("predicted"), + "comparability": b["expectation"].get("comparability"), + "confidence": b["expectation"].get("confidence"), + "basis": b["expectation"].get("basis"), + "runs": [ + { + "task": r["task"], + "status": r["status"], + "deviations": [ + {k: v for k, v in d.items() if k != "expectation_id"} + for d in r["deviations"] + ], + "unjudged": bool(r["unjudged"]), + } + for r in b["runs"] + ], + } + for b in evidence["expectations"] + ], + "figures": [Path(f).name for f in evidence["figures"]], + } + + +def _generate_prose(bundle: dict[str, Any], *, model: str) -> str: + """One forced-tool call, for the same reason §5 uses one. + + "prompting for JSON and parsing it fails silently on the tenth call." + """ + try: + import asyncio # noqa: PLC0415 + + from claude_agent_sdk import ( # noqa: PLC0415 + ClaudeAgentOptions, + create_sdk_mcp_server, + query, + tool, + ) + except ImportError as exc: + raise ConfigError( + "claude-agent-sdk is not installed, so prose cannot be generated", + fix="pip install -e '.[agent]' (`report draft` is free and needs no model)", + ) from exc + + captured: list[str] = [] + + @tool( + "submit_prose", + "Return the LaTeX body of the report", + { + "type": "object", + "properties": {"latex": {"type": "string"}}, + "required": ["latex"], + }, + ) + async def submit_prose(args: dict[str, Any]) -> dict[str, Any]: + text = args.get("latex") + if not isinstance(text, str) or len(text) < 200: + return { + "content": [{"type": "text", "text": "latex must be the full report body"}], + "is_error": True, + } + # Enforced at the tool boundary, where a returned error actually makes + # the model retry -- not after the fact, where it would just be deleted. + if re.search(r"\\cite[tp]?\*?\{", text): + return { + "content": [ + { + "type": "text", + "text": "do not write \\cite{}; use [CITE:keyword] placeholders", + } + ], + "is_error": True, + } + captured.append(text) + return {"content": [{"type": "text", "text": "recorded"}]} + + options = ClaudeAgentOptions( + model=model, + system_prompt=WRITE_PROMPT, + mcp_servers={"report": create_sdk_mcp_server("report", tools=[submit_prose])}, + allowed_tools=["mcp__report__submit_prose"], + disallowed_tools=["Read", "Write", "Edit", "Bash", "Glob", "Grep", "WebSearch", "WebFetch"], + ) + + async def run() -> None: + usage = None + async for message in query( + prompt="Evidence bundle:\n\n" + json.dumps(bundle, indent=2, default=str), + options=options, + ): + usage = getattr(message, "usage", None) or usage + quota_log.from_sdk_usage("report.write", usage, model=model, role="report") + + asyncio.run(run()) + if not captured: + raise UpstreamError( + "the model ended its turn without returning any prose", + fix="re-run; `report draft` output is unchanged and still valid", + ) + return captured[-1] + + +# --------------------------------------------------------------------------- +# cite +# --------------------------------------------------------------------------- +def _cite_args(p: argparse.ArgumentParser) -> None: + _project_arg(p) + p.add_argument("--context-chars", type=int, default=600, help="window around each placeholder") + p.add_argument("--no-s2", action="store_true", help="resolve against the local corpus only") + + +@cli.command("cite", "resolve [CITE:...] against the corpus and S2; emit the bib", setup=_cite_args) +def cmd_cite(args: argparse.Namespace) -> dict[str, Any]: + """The two-pass flow, stolen from Camyla. + + A placeholder is resolved by extracting the context around it and verifying + a candidate's title and abstract against that context -- much better than + citing inline, where the model invents a plausible reference in the moment. + Resolution is against the local corpus and verified S2 ids **only**; an + unresolvable placeholder is left in place and `check` refuses on it, rather + than being silently dropped or filled with a guess. + """ + project_id = _project(args) + files = report_lib.paths_for(project_id) + if not files["tex"].exists(): + raise GradError( + "no_draft", "there is nothing to cite yet", exit_code=3, + fix=f"python -m tools.report draft --project {project_id} --json", + ) + + tex = files["tex"].read_text(encoding="utf-8") + entries: dict[str, dict[str, Any]] = {} + resolved: list[dict[str, Any]] = [] + unresolved: list[dict[str, Any]] = [] + + for match in list(report_lib.PLACEHOLDER_RE.finditer(tex)): + keyword = match.group(1).strip() + start = max(0, match.start() - args.context_chars) + context = tex[start : match.end() + args.context_chars] + entry = _resolve_citation(keyword, context, use_s2=not args.no_s2) + if entry is None: + unresolved.append({"keyword": keyword, "context": context[:200]}) + continue + entries[entry["key"]] = entry + resolved.append({"keyword": keyword, "key": entry["key"], "source": entry["gradsource"]}) + + # Replace only what resolved. An unresolved placeholder stays visible and + # `check` refuses on it: a citation quietly deleted is worse than one that + # fails loudly, because the sentence it supported survives without support. + def substitute(match: re.Match[str]) -> str: + keyword = match.group(1).strip() + for row in resolved: + if row["keyword"] == keyword: + return f"\\cite{{{row['key']}}}" + return match.group(0) + + files["tex"].write_text(report_lib.PLACEHOLDER_RE.sub(substitute, tex), encoding="utf-8") + files["bib"].write_text(_render_bib(entries), encoding="utf-8") + + payload = { + "project": project_id, + "bib": str(files["bib"]), + "resolved": resolved, + "unresolved": unresolved, + "entries": len(entries), + "next": f"python -m tools.report check --project {project_id} --json", + } + if unresolved: + raise GradError( + "citations_unresolved", + f"{len(unresolved)} placeholder(s) did not resolve against the corpus or S2: " + + ", ".join(sorted({u['keyword'] for u in unresolved})[:5]), + exit_code=EXIT_CHECK_FAILED, + fix=( + "ingest the paper so it is in the corpus: " + "python -m tools.paper_ingest arxiv --json\n" + " A \\cite{} key with no resolved entry is a hard error, not a warning." + ), + detail=payload, + ) + return payload + + +def _resolve_citation(keyword: str, context: str, *, use_s2: bool) -> dict[str, Any] | None: + """Local corpus first, then S2. Never the model's memory.""" + found = _from_corpus(keyword) + if found: + return found + if not use_s2: + return None + return _from_s2(keyword, context) + + +def _from_corpus(keyword: str) -> dict[str, Any] | None: + try: + con = corpus.connect(create=False) + except GradError: + return None + try: + row = con.execute( + "SELECT id, title, authors, year FROM documents WHERE id = ? OR title LIKE ? LIMIT 1", + (keyword, f"%{keyword}%"), + ).fetchone() + if row is None: + hits = corpus.fts_search(con, keyword, limit=1) + if not hits: + return None + row = con.execute( + "SELECT id, title, authors, year FROM documents WHERE id = ?", + (hits[0].get("doc_id"),), + ).fetchone() + if row is None: + return None + doc_id, title, authors, year = row + return { + "key": _bib_key(doc_id, authors, year), + "type": "article", + "title": title or doc_id, + "author": authors or "Unknown", + "year": str(year or ""), + "note": doc_id, + # The provenance `check` requires. Without it the entry is refused. + "gradsource": "corpus", + } + finally: + con.close() + + +def _from_s2(keyword: str, context: str) -> dict[str, Any] | None: + """Verify the candidate's title and abstract against the surrounding text. + + A search hit is not a citation. The candidate has to actually be about what + the sentence claims, and the overlap test below is deliberately crude and + conservative: it rejects rather than accepts when unsure. + """ + try: + client = http.SemanticScholar(config_mod.load()) + hits = client.paper_search(keyword, limit=5) + except GradError: + return None + if not hits: + return None + + words = {w.lower() for w in re.findall(r"[a-zA-Z]{5,}", context)} + best, best_score = None, 0.0 + for hit in hits: + text = f"{hit.get('title', '')} {hit.get('abstract', '')}".lower() + candidate_words = set(re.findall(r"[a-zA-Z]{5,}", text)) + if not candidate_words: + continue + score = len(words & candidate_words) / max(1, len(words)) + if score > best_score: + best, best_score = hit, score + if best is None or best_score < 0.08: + return None + return { + "key": _bib_key(best.get("paper_id", keyword), None, best.get("year")), + "type": "article", + "title": best.get("title") or keyword, + "author": "Unknown", + "year": str(best.get("year") or ""), + "note": f"S2:{best.get('paper_id')}", + "gradsource": "s2", + "gradmatch": round(best_score, 3), + } + + +def _bib_key(doc_id: str, authors: Any, year: Any) -> str: + stem = re.sub(r"[^A-Za-z0-9]+", "", str(doc_id))[-12:] or "ref" + lead = re.sub(r"[^A-Za-z]+", "", str(authors or "").split(",")[0])[:12].lower() + return f"{lead or 'ref'}{year or ''}{stem}" + + +def _render_bib(entries: dict[str, dict[str, Any]]) -> str: + out = [ + "% Generated by `python -m tools.report cite`. Do not hand-edit.", + "% Every entry carries `gradsource`, which is what `report check` verifies:", + "% only `corpus` (the local index) and `s2` (a verified Semantic Scholar id)", + "% are accepted. A hand-written entry is exactly the hallucinated citation", + "% this rule exists to make impossible.", + "", + ] + for key, entry in sorted(entries.items()): + out.append(f"@{entry['type']}{{{key},") + for field in ("title", "author", "year", "note", "gradsource", "gradmatch"): + if entry.get(field) not in (None, ""): + out.append(f" {field} = {{{entry[field]}}},") + out.append("}") + out.append("") + return "\n".join(out) + + +# --------------------------------------------------------------------------- +# check -- the gate +# --------------------------------------------------------------------------- +@cli.command("check", "the gate: refuses on an unresolved claim or an unjudged run", setup=_project_arg) +def cmd_check(args: argparse.Namespace) -> dict[str, Any]: + """Four rules, in order. It refuses; it does not warn.""" + project_id = _project(args) + files = report_lib.paths_for(project_id) + if not files["tex"].exists(): + raise GradError( + "no_report", f"no report exists for project {project_id}", exit_code=3, + fix=f"python -m tools.report draft --project {project_id} --json", + ) + + tex = files["tex"].read_text(encoding="utf-8") + claims = report_lib.load_claims(project_id) + bib = report_lib.parse_bib(files["bib"].read_text(encoding="utf-8")) if files["bib"].exists() else {} + + findings: list[dict[str, Any]] = [] + findings += report_lib.check_claims(tex, claims) + findings += report_lib.check_citations(tex, bib) + + # Rule 3. The one most in the spirit of this system. + unjudged = report_lib.unjudged_for(report_lib.cited_run_ids(tex, claims)) + for row in unjudged: + findings.append( + { + "rule": "unjudged", + "run_id": row["run_id"], + "problem": ( + f"run {row['run_id']} has an unjudged deviation on {row['quantity']}, " + "and this report cites it" + ), + "fix": ( + f"python -m tools.ledger verdict {row['run_id']} --quantity {row['quantity']} " + "--verdict bug|real|inconclusive --note '...' --json" + ), + } + ) + + findings += report_lib.check_latex(tex) + + payload = { + "project": project_id, + "tex": str(files["tex"]), + "claims_checked": len(set(report_lib.GRADNUM_RE.findall(tex))), + "citations_checked": len(bib), + "cited_runs": sorted(report_lib.cited_run_ids(tex, claims)), + "findings": findings, + "by_rule": { + rule: sum(1 for f in findings if f.get("rule") == rule) + for rule in ("claims", "citations", "unjudged", "latex") + }, + } + if findings: + first = findings[0] + raise GradError( + "report_check_failed", + f"{len(findings)} finding(s); first: {first.get('problem')}", + exit_code=EXIT_CHECK_FAILED, + fix=first.get("fix") or "resolve the finding and re-run check", + detail=payload, + ) + return { + **payload, + "ok": True, + "note": ( + "every number traces to a run record, every citation to the corpus or a " + "verified S2 id, and every cited run has been judged" + ), + } + + +# --------------------------------------------------------------------------- +# build +# --------------------------------------------------------------------------- +def _build_args(p: argparse.ArgumentParser) -> None: + _project_arg(p) + p.add_argument("--skip-check", action="store_true", help=argparse.SUPPRESS) + p.add_argument("--passes", type=int, default=3, help="LaTeX passes (bibtex needs at least 2)") + + +@cli.command("build", "compile the PDF, checkpointing each version", setup=_build_args) +def cmd_build(args: argparse.Namespace) -> dict[str, Any]: + """Progressive versions, copied from Denario. + + "It emits four because unattended LaTeX does not reliably compile." Each + successful pass is checkpointed, so a run that fails on pass 3 still leaves + the pass-2 PDF rather than nothing at all. + + `check` runs first and is not skippable from the agent's side: building a + PDF is the act of asserting the result, and that is exactly where the gate + belongs. + """ + project_id = _project(args) + if not args.skip_check: + cmd_check(argparse.Namespace(project=project_id)) + + files = report_lib.paths_for(project_id) + engine = shutil.which("latexmk") or shutil.which("pdflatex") + if not engine: + raise ConfigError( + "neither latexmk nor pdflatex is on PATH, so no PDF can be produced", + fix=( + "install a TeX distribution (MiKTeX or TeX Live), or read the checked " + f"source at {files['tex']}" + ), + ) + + checkpoints: list[dict[str, Any]] = [] + for attempt in range(1, max(1, args.passes) + 1): + argv = ( + [engine, "-pdf", "-interaction=nonstopmode", "-halt-on-error", files["tex"].name] + if engine.endswith("latexmk") or "latexmk" in Path(engine).stem + else [engine, "-interaction=nonstopmode", "-halt-on-error", files["tex"].name] + ) + proc = subprocess.run( + argv, cwd=str(files["dir"]), capture_output=True, text=True, timeout=300, check=False + ) + ok = proc.returncode == 0 and files["pdf"].exists() + if ok: + checkpoint = files["dir"] / f"main.v{attempt}.pdf" + shutil.copyfile(files["pdf"], checkpoint) + checkpoints.append({"pass": attempt, "pdf": str(checkpoint)}) + else: + log = files["dir"] / f"build.pass{attempt}.log" + log.write_text((proc.stdout or "") + (proc.stderr or ""), encoding="utf-8") + if not checkpoints: + raise GradError( + "build_failed", + f"LaTeX failed on pass {attempt}", + exit_code=EXIT_CHECK_FAILED, + fix=f"read {log}", + detail={"log": str(log), "tail": (proc.stdout or "").splitlines()[-25:]}, + ) + break + + return { + "project": project_id, + "pdf": str(files["pdf"]), + "checkpoints": checkpoints, + "engine": engine, + "checked": not args.skip_check, + } + + +if __name__ == "__main__": + main(cli) diff --git a/tools/wiki.py b/tools/wiki.py new file mode 100644 index 0000000..3b85f67 --- /dev/null +++ b/tools/wiki.py @@ -0,0 +1,261 @@ +"""grad-wiki -- RepoWiki, the human's map (HANDOFF-2 §20). + +**Scope: human-facing only.** Not in the agent's tool list, not in +`prompts/system.md`, no context cost. Its job is letting a person reacquire the +shape of a growing codebase quickly. `HANDOFF.md` remains the design record and +`README.md` the report; this targets the third thing -- the module-level "what +calls what, and where does this value come from" view nobody wants to maintain +by hand. + +**Try `map` first.** `repowiki map` is LLM-free: `cli.py:40 repo_map()` never +touches `LLMClient` and `core/graph.py` has no LLM references, so it needs no +credential at all. It is free, and it may cover enough of the need to make the +rest unnecessary. + +**Two rules this wrapper exists to enforce**, because getting either wrong is +expensive and neither is enforced by RepoWiki itself: + +1. **Scope.** `core/` and `tools/` only. **Never** `ledger/`, `notes/`, or any + papers directory -- `scan` ships content to a third party, and those hold + research data. This is a mechanical allowlist, not a convention. +2. **Staleness.** A wiki behind the code is worse than none, because it is + trusted. The source-tree hash is recorded in the output and `check` compares + it, using the same pattern `core/submission.py` already implements. + +Output is HTML under `data/wiki/`, never markdown committed to the repo, so it +cannot compete with the hand-written docs. + +**On the API-key question**, recorded here so the day is not spent by accident: +`scan` (the LLM half) reads `ANTHROPIC_API_KEY` by default, which is exactly +what `credentials.scrub_environment()` deletes. That scrub cleans only the +*agent's* process, so a human running `repowiki scan` in their own shell +violates nothing technically -- but a key in the user profile is also in the +agent's environment and trips the scrub warning on every launch. Safe, noisy, +and it erodes the §2 discipline by habituation. Forking `repowiki/llm/client.py` +onto the Agent SDK is ~120 lines against a two-method async interface with four +call sites, and `core/haiku.py` is the model for the plumbing. It is a +preference, not a requirement, which is why this wrapper refuses `scan` rather +than quietly enabling it. +""" + +from __future__ import annotations + +import argparse +import hashlib +import shutil +import subprocess +import time +from pathlib import Path +from typing import Any + +from core import jsonl, paths +from core.cli import Cli, main +from core.errors import ConfigError, GradError, UsageError + +cli = Cli( + "grad-wiki", + "Generate a human-facing map of core/ and tools/. Not an agent tool.", + epilog=( + "Scope is an allowlist, not a convention: core/ and tools/ only. ledger/,\n" + "notes/, and data/papers/ are never passed to it -- `scan` ships content to a\n" + "third party and those hold research data.\n\n" + "`map` is LLM-free and needs no credential. `scan` is refused here on purpose;\n" + "see this module's docstring for the reasoning and the ~120-line fork that\n" + "would make it clean." + ), +) + +# The allowlist. Everything else in the workspace is research data or generated +# output, and neither belongs in a third party's context window. +SCOPE = ("core", "tools") + +# Files whose content defines "the code has changed". Same idea as the +# submission hash: not a directory mtime, and not a TTL. +_SOURCE_GLOB = "*.py" + + +def output_dir() -> Path: + return paths.data_dir() / "wiki" + + +def _manifest_path() -> Path: + return output_dir() / "manifest.json" + + +def source_hash(root: Path | None = None) -> dict[str, Any]: + """A digest over exactly the files the wiki was generated from. + + Returned with its inputs listed rather than as a bare string, so a staleness + report can say *which* file moved instead of only that something did. + """ + root = root or paths.root() + digests: dict[str, str] = {} + for name in SCOPE: + directory = root / name + if not directory.is_dir(): + continue + for path in sorted(directory.rglob(_SOURCE_GLOB)): + if "__pycache__" in path.parts: + continue + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1 << 16), b""): + h.update(chunk) + digests[path.relative_to(root).as_posix()] = h.hexdigest()[:16] + canonical = "\n".join(f"{k}:{v}" for k, v in sorted(digests.items())) + return { + "hash": hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:16], + "files": digests, + "scope": list(SCOPE), + } + + +def _repowiki() -> str: + found = shutil.which("repowiki") + if found: + return found + raise ConfigError( + "repowiki is not installed", + fix="pip install -e '.[wiki]' # pins repowiki==0.3.1", + ) + + +def _scope_paths(root: Path) -> list[str]: + present = [str(root / name) for name in SCOPE if (root / name).is_dir()] + if not present: + raise ConfigError( + f"none of {', '.join(SCOPE)} exist under {root}", + fix="run this from the workspace root", + ) + return present + + +# --------------------------------------------------------------------------- +def _map_args(p: argparse.ArgumentParser) -> None: + p.add_argument("--format", default="html", choices=["html", "json"], help="output format") + p.add_argument("--open", action="store_true", help="open the result in a browser") + + +@cli.command("map", "generate the module map (LLM-free, no credential)", setup=_map_args) +def cmd_map(args: argparse.Namespace) -> dict[str, Any]: + """`repowiki map` over core/ and tools/ only. + + LLM-free by construction, so this costs nothing and ships nothing anywhere. + Try it before deciding whether the rest of §20 is wanted at all. + """ + executable = _repowiki() + root = paths.root() + out = output_dir() + out.mkdir(parents=True, exist_ok=True) + + argv = [executable, "map", *_scope_paths(root), "--format", args.format, "--output", str(out)] + if args.open: + argv.append("--open") + + started = time.time() + try: + proc = subprocess.run( + argv, cwd=str(root), capture_output=True, text=True, timeout=600, check=False + ) + except subprocess.TimeoutExpired as exc: + raise GradError( + "wiki_timeout", "repowiki map did not finish within 10 minutes", + exit_code=8, fix="run it by hand to see where it stalls", + ) from exc + + log = out / "map.log" + log.write_text((proc.stdout or "") + (proc.stderr or ""), encoding="utf-8") + if proc.returncode != 0: + raise GradError( + "wiki_failed", + f"repowiki map exited {proc.returncode}", + exit_code=8, + fix=f"read {log}", + detail={"log": str(log), "tail": (proc.stdout or proc.stderr or "").splitlines()[-15:]}, + ) + + # The staleness record. A wiki behind the code is worse than none, because + # it is trusted -- so the inputs are recorded at generation time, not + # reconstructed later. + manifest = { + "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "command": argv[1:], + "duration_s": round(time.time() - started, 1), + "source": source_hash(root), + "output_dir": str(out), + "format": args.format, + } + jsonl.write_json(_manifest_path(), manifest) + return { + "output_dir": str(out), + "source_hash": manifest["source"]["hash"], + "files_covered": len(manifest["source"]["files"]), + "log": str(log), + "next": "python -m tools.wiki check --json", + } + + +@cli.command("check", "is the generated wiki still current?") +def cmd_check(_: argparse.Namespace) -> dict[str, Any]: + """The one-line staleness check §20 asks for. + + Exits 9 when stale, so it can be wired into anything that wants the wiki to + be trustworthy before it is read. + """ + manifest = jsonl.read_json(_manifest_path()) + if not manifest: + raise GradError( + "no_wiki", "no wiki has been generated yet", exit_code=3, + fix="python -m tools.wiki map --json", + ) + current = source_hash() + recorded = manifest.get("source", {}) + if current["hash"] == recorded.get("hash"): + return { + "current": True, + "source_hash": current["hash"], + "generated_at": manifest.get("generated_at"), + "output_dir": manifest.get("output_dir"), + } + + before, after = recorded.get("files", {}), current["files"] + changed = sorted(k for k in set(before) | set(after) if before.get(k) != after.get(k)) + raise GradError( + "wiki_stale", + f"the wiki was generated from a different source tree: {len(changed)} file(s) differ", + exit_code=9, + fix="python -m tools.wiki map --json", + detail={ + "generated_at": manifest.get("generated_at"), + "recorded_hash": recorded.get("hash"), + "current_hash": current["hash"], + "changed": changed[:50], + }, + ) + + +@cli.command("scan", "refused on purpose; prints the reasoning") +def cmd_scan(_: argparse.Namespace) -> dict[str, Any]: + """Not enabled here, and the reason is worth reading before enabling it. + + `repowiki scan` is the LLM half, and it reads `ANTHROPIC_API_KEY` by + default -- the exact variable `credentials.scrub_environment()` deletes. A + key in the user profile is also in the agent's environment and trips the + scrub warning on every launch: safe, but noisy, and it erodes the §2 + discipline by habituation. + """ + raise UsageError( + "`scan` is not enabled: it reads ANTHROPIC_API_KEY, which is the credential " + "§2 exists to keep out of this system. `map` is LLM-free and covers most of " + "the need.", + fix=( + "python -m tools.wiki map --json # free, no credential, no data leaves the machine\n" + " If you want the prose wiki, fork repowiki/llm/client.py onto the Agent SDK: " + "two async methods, four call sites, response_format never used by any caller, " + "and core/haiku.py:110 is the model for the plumbing." + ), + ) + + +if __name__ == "__main__": + main(cli) diff --git a/ui/app.py b/ui/app.py index 403df05..2b11489 100644 --- a/ui/app.py +++ b/ui/app.py @@ -28,6 +28,7 @@ import logging import re import secrets +import sys from pathlib import Path from typing import Any @@ -269,6 +270,7 @@ def _layout(ui: Any, session: Session) -> None: tab_funnel = ui.tab("Funnel") tab_quota = ui.tab("Quota") tab_nb = ui.tab("Notebooks") + tab_lab = ui.tab("Lab") with ui.tab_panels(tabs, value=tab_chat).classes("w-full"): with ui.tab_panel(tab_chat): @@ -283,6 +285,8 @@ def _layout(ui: Any, session: Session) -> None: _refreshable(ui, quota_panel) with ui.tab_panel(tab_nb): _notebook_panel(ui) + with ui.tab_panel(tab_lab): + _lab_panel(ui) def _refreshable(ui: Any, render: Any) -> None: @@ -388,8 +392,103 @@ def _figures_in(text: str) -> list[str]: return found +async def _verify_notebook(ui: Any, name: str, target: Any) -> None: + """Shell out to `nb verify` and render the failing cell index and traceback. + + HANDOFF-2 §19 calls this the highest-value part of the whole item, and it is + why it was built before the embed: Lab and `tools/nb.py` are two kernel + owners over one notebook, which reproduces exactly the "works in the kernel + that grew it" failure `nb verify` exists to catch. The discipline is + unchanged -- anything edited in Lab passes this before it is cited in + `notes/` or referenced from a ledger entry -- and a button is what makes a + discipline actually get followed. + """ + target.clear() + with target: + ui.spinner(size="sm") + ui.label(f"running {name} top to bottom on a fresh kernel…").classes("text-sm opacity-70") + + proc = await asyncio.create_subprocess_exec( + sys.executable, "-m", "tools.nb", "verify", f"notebooks/{name}", "--json", + cwd=str(paths.root()), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + out, err = await proc.communicate() + try: + payload = json.loads((out or b"").decode("utf-8", "replace").strip().splitlines()[-1]) + except (json.JSONDecodeError, IndexError): + payload = {"ok": False, "error": {"message": (err or b"").decode("utf-8", "replace")[-2000:]}} + + target.clear() + with target: + if payload.get("ok"): + data = payload.get("data") or {} + ui.label( + f"verified — {data.get('cells_executed', '?')} cells ran clean on a fresh kernel" + ).classes("text-sm text-green-400") + return + error = payload.get("error") or {} + detail = error.get("detail") or {} + ui.label(error.get("message") or "verification failed").classes("text-sm text-red-400") + index = detail.get("cell_index") + if index is not None: + ui.label( + f"first failing cell: index {index} " + f"({detail.get('cells_executed', '?')} cells ran before it)" + ).classes("text-xs opacity-80") + # `nb verify` nests the kernel's own error under `error`, with the + # traceback already stripped of ANSI escapes. + traceback_text = ((detail.get("error") or {}).get("traceback")) or detail.get("stdout") + if traceback_text: + if isinstance(traceback_text, list): + traceback_text = "\n".join(str(t) for t in traceback_text) + ui.code(str(traceback_text)[-4000:], language="python").classes("w-full") + if error.get("fix"): + ui.code(str(error["fix"]), language="bash").classes("w-full") + + +def _lab_panel(ui: Any) -> None: + """JupyterLab, embedded (HANDOFF-2 §19). + + This iframe is **deliberately unsandboxed**, and that is a considered + difference from the notebook-output iframe below it. Notebook output is + untrusted HTML from files that may have come from a downloaded repository, + so it is `sandbox=""`. Lab is a server we started ourselves, on its own + port, with a token we minted -- and it cannot function sandboxed. The two + are separate iframes on purpose; do not merge them. + """ + from tools import lab as lab_tool # noqa: PLC0415 - optional dependency + + container = ui.column().classes("w-full") + + def draw() -> None: + container.clear() + state = lab_tool.lab_state() + with container: + if not state.get("running"): + ui.label("JupyterLab is not running.").classes("text-sm opacity-70") + ui.code("python -m tools.lab start --json", language="bash") + ui.label( + "Anything edited in Lab must pass `nb verify` before it is cited in " + "notes/ or referenced from a ledger entry — Lab and tools/nb.py are " + "two kernel owners over one notebook." + ).classes("text-xs opacity-60 max-w-2xl") + return + with ui.row().classes("items-center gap-3"): + ui.label(f"127.0.0.1:{state['port']}").classes("text-xs font-mono opacity-70") + ui.button("Stop", on_click=lambda: (lab_tool.cmd_stop(None), draw())).props("flat dense") + ui.element("iframe").props( + f'src="http://127.0.0.1:{state["port"]}/lab?token={state["token"]}" ' + 'allow="clipboard-read; clipboard-write"' + ).classes("w-full h-[80vh] rounded bg-white") + + ui.button(icon="refresh", on_click=draw).props("flat dense").classes("self-end") + draw() + + def _notebook_panel(ui: Any) -> None: - """Render notebook *outputs*, read-only, with a link out to JupyterLab.""" + """Render notebook *outputs*, read-only, with Verify and a link into Lab.""" notebooks = sorted(paths.notebooks_dir().glob("*.ipynb")) if paths.notebooks_dir().exists() else [] if not notebooks: ui.label("No notebooks yet.").classes("text-sm opacity-60") @@ -401,8 +500,16 @@ def show(name: str) -> None: container.clear() path = paths.notebooks_dir() / name with container: + verify_out = ui.column().classes("w-full gap-1") with ui.row().classes("items-center gap-3"): - ui.link("open in JupyterLab", f"http://localhost:8888/lab/tree/notebooks/{name}").classes("text-sm") + ui.button( + "Verify", + icon="fact_check", + on_click=lambda: _verify_notebook(ui, name, verify_out), + ).props("unelevated dense").tooltip( + "restart the kernel and run every cell top to bottom" + ) + ui.link("open in JupyterLab", _lab_link(name)).classes("text-sm") ui.code(f"python -m tools.nb verify notebooks/{name} --json", language="bash") try: import nbformat # noqa: PLC0415 @@ -426,6 +533,24 @@ def show(name: str) -> None: show(notebooks[0].name) +def _lab_link(name: str) -> str: + """A link into the running Lab, or the command that starts one. + + The previous version pointed at a `localhost:8888` nobody started, which is + what §14 identified as the real gap: the rejection was of *building* an + editor, not of linking to one, and the link was never wired up. + """ + try: + from tools import lab as lab_tool # noqa: PLC0415 + + state = lab_tool.lab_state() + if state.get("running"): + return f"http://127.0.0.1:{state['port']}/lab/tree/notebooks/{name}?token={state['token']}" + except Exception: # noqa: BLE001 - a missing jupyter must not break the panel + pass + return "#" + + def run(*, native: bool = True, port: int = 8080) -> None: """`ui.run(native=True)` gives a real desktop window via pywebview, so the packaging question is answered without Electron or Tauri. Browser mode is diff --git a/ui/widgets/quota_meter.py b/ui/widgets/quota_meter.py index 1ab4e9e..b41e4e6 100644 --- a/ui/widgets/quota_meter.py +++ b/ui/widgets/quota_meter.py @@ -1,4 +1,4 @@ -"""Widget 3: the quota and spend meter (HANDOFF §10). +"""Widget 3: the quota and spend meter (HANDOFF §10, extended by §15). One honesty note is part of the widget, not a footnote: Anthropic exposes no remaining-quota API and the Max 5x window (5-hour rolling plus weekly caps) is @@ -8,13 +8,17 @@ GPU spend is different: it is real dollars, counted with in-flight runs at their estimates, against the ceiling that actually blocks submissions. + +§15 adds the project dimension: a selector, and three bars rather than one -- +GPU dollars, credits, and tokens, each against the current project's ceiling. +All of it is read from the ledger; no new logic lives in the UI, per §10. """ from __future__ import annotations from typing import Any -from core import config as config_mod, ledger_store as ls, quota_log +from core import budget as budget_mod, config as config_mod, ledger_store as ls, quota_log def _spend() -> dict[str, Any]: @@ -34,25 +38,68 @@ def _spend() -> dict[str, Any]: } +def _project_state() -> dict[str, Any] | None: + current = budget_mod.current_project() + if not current or not budget_mod.exists(current): + return None + return budget_mod.status(current) + + +# How each resource renders. Kept as data so the header and the panel cannot +# disagree about what a bar means or how a number is formatted. +_RESOURCE_LABELS = { + "gpu_usd": ("GPU", lambda v: f"${v:,.2f}"), + "credits_usd": ("credits", lambda v: f"${v:,.2f}"), + "quota_tokens": ("tokens", lambda v: f"{int(v):,}"), +} + + +def _bar(ui: Any, resource: str, node: dict[str, Any]) -> None: + label, fmt = _RESOURCE_LABELS[resource] + ceiling = node.get("ceiling") + with ui.column().classes("gap-0"): + if ceiling is None: + ui.label(f"{fmt(node['spent'])} {label}").classes("font-mono") + ui.label("no ceiling set").classes("opacity-50") + return + ui.label(f"{fmt(node['spent'])} / {fmt(ceiling)}").classes( + "font-mono" + (" text-red-400" if node["over"] else "") + ) + ui.linear_progress(node.get("fraction") or 0.0, show_value=False).classes("w-28 h-1") + ui.label(label + (" — over" if node["over"] else "")).classes("opacity-60") + + def quota_meter() -> Any: - """The persistent header strip.""" + """The persistent header strip: a project selector and three bars.""" from nicegui import ui spend = _spend() tokens = quota_log.summarise(days=7) + project = _project_state() with ui.row().classes("items-center gap-4 text-xs"): - with ui.column().classes("gap-0"): - ui.label(f"${spend['total']:.2f} / ${spend['monthly']:.0f}").classes("font-mono") - ui.linear_progress(spend["fraction"], show_value=False).classes("w-32 h-1") - ui.label( - f"{spend['window']}d GPU · ${spend['in_flight']:.2f} in flight" - ).classes("opacity-60") - with ui.column().classes("gap-0"): - ui.label(f"{tokens['total_tokens']:,} tok (7d)").classes("font-mono") - ui.label("self-measured, not a fuel gauge").classes("opacity-50") - if tokens["total_credits_usd"]: - ui.label(f"${tokens['total_credits_usd']:.2f} credits").classes("font-mono opacity-70") + _project_selector(ui) + + if project: + for resource in ("gpu_usd", "credits_usd", "quota_tokens"): + _bar(ui, resource, project["resources"][resource]) + else: + # No project selected: fall back to the machine-wide view, which is + # the ceiling that actually blocks submissions either way. + with ui.column().classes("gap-0"): + ui.label(f"${spend['total']:.2f} / ${spend['monthly']:.0f}").classes("font-mono") + ui.linear_progress(spend["fraction"], show_value=False).classes("w-32 h-1") + ui.label( + f"{spend['window']}d GPU · ${spend['in_flight']:.2f} in flight" + ).classes("opacity-60") + with ui.column().classes("gap-0"): + ui.label(f"{tokens['total_tokens']:,} tok (7d)").classes("font-mono") + ui.label("self-measured, not a fuel gauge").classes("opacity-50") + + if project and project["over_budget"]: + ui.badge("over budget", color="red").tooltip( + "cost-bearing commands are denied until the ceiling is raised deliberately" + ) if spend["stale"]: ui.badge(f"{len(spend['stale'])} stale", color="red").tooltip( "submissions are blocked until these are collected" @@ -61,6 +108,27 @@ def quota_meter() -> Any: ui.badge(f"{len(spend['uncollected'])} uncollected", color="amber") +def _project_selector(ui: Any) -> None: + """Selecting here writes the same file `tools.budget use` writes. + + One selection mechanism, not two: a UI-only notion of "current project" + would attribute the CLI's spend to the wrong allocation the moment the two + disagreed. + """ + projects = budget_mod.projects() + options = ["(none)"] + [p for p, d in projects.items() if d["status"] == "open"] + current = budget_mod.current_project() or "(none)" + if current not in options: + options.append(current) + + def choose(event: Any) -> None: + budget_mod.set_current(None if event.value == "(none)" else event.value) + + ui.select(options, value=current, on_change=choose).props("dense outlined").classes( + "w-44 text-xs" + ).tooltip("the project every cost-bearing record is charged to") + + def quota_panel() -> None: """The full breakdown: which stage spent what.""" from nicegui import ui @@ -80,7 +148,12 @@ def quota_panel() -> None: with ui.column().classes("gap-1"): ui.label("Tokens by stage").classes("text-sm font-semibold") ui.label("Self-measured usage against an assumed budget. Anthropic exposes no " - "remaining-quota API and the Max 5x window is opaque.").classes("text-xs opacity-60 max-w-md") + "remaining-quota API and the Max 5x window is opaque, so a token " + "ceiling is a proxy you control, not a mirror of the real limit.").classes( + "text-xs opacity-60 max-w-md" + ) + + _project_breakdown(ui, tokens) rows = [ { @@ -126,3 +199,82 @@ def quota_panel() -> None: ).classes("w-full h-64") else: ui.label("No usage recorded yet.").classes("text-sm opacity-60") + + _role_table(ui, tokens) + + +def _project_breakdown(ui: Any, tokens: dict[str, Any]) -> None: + """Per-project spend against ceilings, and per-project usage by stage (§15).""" + projects = budget_mod.projects() + if not projects: + return + + ui.label("Projects").classes("text-sm font-semibold mt-4") + rows = [] + for pid, proj in projects.items(): + state = budget_mod.status(pid) + usage = tokens["by_project"].get(pid, {}) + res = state["resources"] + rows.append( + { + "project": pid, + "title": proj["title"], + "status": proj["status"], + "gpu": _cell(res["gpu_usd"], "${:,.2f}"), + "credits": _cell(res["credits_usd"], "${:,.2f}"), + "tokens": _cell(res["quota_tokens"], "{:,.0f}"), + "calls": usage.get("calls", 0), + "over": ", ".join(state["over_budget"]) or "—", + } + ) + ui.table( + columns=[ + {"name": "project", "label": "project", "field": "project", "align": "left", "sortable": True}, + {"name": "title", "label": "title", "field": "title", "align": "left"}, + {"name": "gpu", "label": "GPU $", "field": "gpu", "align": "left"}, + {"name": "credits", "label": "credits $", "field": "credits", "align": "left"}, + {"name": "tokens", "label": "tokens", "field": "tokens", "align": "left"}, + {"name": "calls", "label": "calls", "field": "calls", "sortable": True}, + {"name": "over", "label": "over", "field": "over", "align": "left"}, + {"name": "status", "label": "status", "field": "status", "align": "left"}, + ], + rows=rows, + row_key="project", + ).classes("w-full") + + +def _cell(node: dict[str, Any], fmt: str) -> str: + """`spent / ceiling`, or bare spend where no ceiling is set. + + "unbounded" and "a very large ceiling" must not look the same, so the + absence of a ceiling is spelled out rather than rendered as a full bar. + """ + spent = fmt.format(node["spent"]) + if node["ceiling"] is None: + return f"{spent} (no ceiling)" + return f"{spent} / {fmt.format(node['ceiling'])}" + + +def _role_table(ui: Any, tokens: dict[str, Any]) -> None: + """What each §16 model role cost. + + This is the question the role tagging exists to answer -- "what did Opus + cost me this week" -- without inferring a role from a model id. + """ + by_role = {k: v for k, v in tokens["by_role"].items() if v["calls"]} + if not by_role: + return + ui.label("Tokens by role").classes("text-sm font-semibold mt-4") + ui.table( + columns=[ + {"name": "role", "label": "role", "field": "role", "align": "left", "sortable": True}, + {"name": "calls", "label": "calls", "field": "calls", "sortable": True}, + {"name": "input", "label": "in", "field": "input", "sortable": True}, + {"name": "output", "label": "out", "field": "output", "sortable": True}, + ], + rows=[ + {"role": role, "calls": d["calls"], "input": d["input_tokens"], "output": d["output_tokens"]} + for role, d in by_role.items() + ], + row_key="role", + ).classes("w-full") From 65c00be61802c48c8ff7c3b65b82cb21132e7af8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=BB=D0=B0=D0=B4=D0=B8=D0=BC=D0=B8=D1=80=20=D0=A8?= =?UTF-8?q?=D0=BC=D0=B0=D0=BD?= Date: Fri, 14 Aug 2026 22:43:50 +0300 Subject: [PATCH 2/3] Address review: 15 fixes across framing, budgets, and the two loops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every finding verified against the code before changing it; three were verified against upstream sources rather than reasoned about. **Embedded Lab was broken in two ways.** `xheaders = False` does not remove `X-Frame-Options` -- it controls whether Tornado trusts `X-Forwarded-*` from a proxy -- so Jupyter kept emitting SAMEORIGIN and the iframe stayed blank, which is the exact failure that file exists to prevent. It is now cleared explicitly in the headers dict. The CSP also built `http://localhost:{rsplit(':')[-1]}`, which on a portless origin yields `http://localhost:example.com`: an invalid source browsers drop silently, narrowing the allowed ancestors. The alias is now added only when a numeric port is present. Both are tested by exec'ing the real config and asserting the resulting headers. **Two phantom-estimate paths.** `run_smoke` wrote its in-flight record before resolving the hub and the namespace kwargs, so a machine without huggingface_hub booked spend for a job that never reached the platform -- which then goes stale and blocks every later submission through the §6 gate. Both now resolve first. And an unknown `--project` silently bypassed every ceiling, because unknown ids read as unbounded; a typo would have bought an unlimited allocation, so `budget.resolve` refuses a named project that does not exist. **ShinkaEvolveRunner has no `propose()`.** Checked against the upstream source: it exposes `run` and `run_async`, both of which own the whole loop. The driver called a method that does not exist. It now detects a per-generation entry point and refuses with the reason when there is none -- which is §23 item 1 answered and the evidence §21 said a fork should wait for. `capabilities` reports the granularity it found. **Context7's endpoints are now verified, not guessed** (§23 item 2): `/api/v2/libs/search` and `/api/v2/context`, with the parameter names the API actually takes. The client reads both `codeSnippets` (v2) and `snippets` (v1) so a config change is sufficient either way, and it no longer refuses to construct when the credential store is unreachable -- the key is optional and an anonymous lookup should not need one. **`repowiki map` was called wrongly.** The 0.3.1 wheel takes exactly one path, `--format text|json`, and has no `--output` or `--open`; §20's `--format html --open` would have failed immediately. It is now one invocation per scope directory asking for JSON, with the HTML rendered here. **`shinka-evolve>=0.2` was unsatisfiable** -- 0.0.7 is the latest published release, so the extra could never install. Pinned exactly. **Report fixes.** Re-running `cite` wiped references.bib: the second pass finds no placeholders (the first turned them into `\cite{}` keys) and rewrote the file from an empty dict, leaving `check` refusing on citations that were fine a moment earlier. Entries are now merged. Usage is recorded in a `finally` so a failed turn's quota is not invisible to the ceilings. And the S2 relevance gate at 8% word overlap was close to a rubber stamp -- any two ML papers share that much vocabulary -- so it now requires overlap against the title as well, ignores generic research words, and records both scores for audit. **Two smaller ones.** `import os.path` bound `os` to `os.path`, turning every later `os.` call into a false "missing attribute"; and an escaped evolve-block candidate was charged its full estimate despite never running, so a campaign that mostly escapes would exhaust its allocation having evaluated almost nothing. **Documented rather than changed:** `docs check` imports the modules it inspects, and importing executes them. That is inherent to the introspection oracle -- a checker that does not import can only guess at what is installed -- so the boundary is now stated in the module docstring, the CLI epilog, the argument help, and the README. 316 tests. Co-Authored-By: Claude Opus 5 --- README.md | 37 ++++-- config/grad.toml | 14 +-- config/jupyter/jupyter_server_config.py | 31 +++-- core/budget.py | 24 +++- core/config.py | 15 ++- core/http.py | 59 +++++++--- pyproject.toml | 10 +- tests/test_budget.py | 28 +++++ tests/test_docs.py | 109 +++++++++++++++++- tests/test_evolve.py | 59 ++++++++++ tests/test_lab_and_wiki.py | 91 +++++++++++++++ tests/test_namespace.py | 55 +++++++++ tests/test_report.py | 81 +++++++++++++ tools/docs.py | 68 +++++++++-- tools/evolve.py | 92 +++++++++++++-- tools/jobs.py | 13 ++- tools/report.py | 100 +++++++++++++--- tools/wiki.py | 145 +++++++++++++++++++----- 18 files changed, 906 insertions(+), 125 deletions(-) diff --git a/README.md b/README.md index 49689e9..ba68f19 100644 --- a/README.md +++ b/README.md @@ -214,15 +214,23 @@ against the documented interfaces and fail with actionable errors rather than tracebacks, but a real credential and a real run are what will find the mismatches. -**Carried over unresolved from [HANDOFF-2 §23](HANDOFF-2.md):** - -- **Context7's REST endpoint paths were never verified.** They are configuration - (`[docs]` in `config/grad.toml`), not constants, and a 404 says so and points - at `context7.com/docs/api-guide` rather than looking like a missing library. -- **Whether ShinkaEvolve exposes a per-candidate callback** decides driver vs. - fork. `python -m tools.evolve capabilities --json` answers it against the - installed package; the driver assumes generation-boundary granularity, which - is what the budget gate is built around. +**Two of [HANDOFF-2 §23](HANDOFF-2.md)'s open questions are now closed:** + +- **Context7's REST endpoints** (§23 item 2) are verified against the live API: + `/api/v2/libs/search` returns `{"results": […]}`, and `/api/v2/context` with + `type=json` returns `{"codeSnippets": […]}`. They stay in `[docs]` config + because a third-party API can move; the client reads both the v2 and v1 + response keys so a config change is sufficient either way. +- **ShinkaEvolve exposes no per-candidate callback, and no per-generation entry + point either** (§23 item 1). `ShinkaEvolveRunner` has `run` and `run_async`, + both of which own the whole loop — which is the control the campaign budget + gate needs in order to re-check between generations. **This is the evidence + §21 said a fork should wait for.** `evolve run` refuses with that explanation + rather than handing control away with the budget unchecked; + `python -m tools.evolve capabilities --json` reports what it found. + +**Still open:** + - **Whether `headless/claude` works against a Max subscription specifically** is reported in Shinka's release notes and untested here. - **Historical records are left as `"unassigned"`** rather than retrofitted with @@ -232,6 +240,17 @@ mismatches. ledger work and the spend work simultaneously against live GPU jobs is how you learn about exit 7 the hard way. +**One correction to HANDOFF-2 itself.** §20 records `repowiki map` as taking +`--format html --open`. The 0.3.1 wheel's `map` takes exactly one `path`, +`--format text|json`, and has neither flag — so `tools/wiki.py` invokes it once +per scope directory asking for JSON and renders the HTML itself. + +**`tools/docs.py check` imports the modules it inspects,** and importing runs +their top-level code. That is inherent to the introspection oracle — a checker +that does not import can only guess at what is installed. Run it on your own +pipeline, not on a repository you just downloaded; the module docstring and +`--help` both say so. + Two things worth knowing before trusting them: - **The Agent SDK surface is version-sensitive.** `core/haiku.py` and diff --git a/config/grad.toml b/config/grad.toml index 00e2bb1..0a0d0fd 100644 --- a/config/grad.toml +++ b/config/grad.toml @@ -74,13 +74,13 @@ cache_ttl_s = 604800 permission_mode = "dontAsk" [docs] -# HANDOFF-2 §18. These REST paths were NOT verified when this was written, which -# is why they are configuration rather than constants. If `tools.docs resolve` -# returns 404, read context7.com/docs/api-guide and correct them here -- the -# error message says so too. -base = "https://context7.com/api/v1" -resolve_path = "/search" -docs_path = "/{library_id}" +# HANDOFF-2 §18, and §23 item 2 is now closed: these paths are verified against +# the live Context7 API. They stay configuration rather than constants because a +# third-party API can move, and a 404 should be a one-line edit here rather than +# a code change -- the error message says exactly that. +base = "https://context7.com" +resolve_path = "/api/v2/libs/search" +docs_path = "/api/v2/context" cache_ttl_s = 86400 [report] diff --git a/config/jupyter/jupyter_server_config.py b/config/jupyter/jupyter_server_config.py index e00e389..dae6073 100644 --- a/config/jupyter/jupyter_server_config.py +++ b/config/jupyter/jupyter_server_config.py @@ -6,9 +6,17 @@ **Framing.** JupyterLab ships `X-Frame-Options: SAMEORIGIN` and a `frame-ancestors 'self'` CSP, both of which block embedding from another origin. Lab runs on its own port, so the Grad app *is* another origin. The headers below -override that for the app's origin only -- not for `*`, which would let any page +override both for the app's origin only -- not for `*`, which would let any page frame a server that can execute code as this user. +`X-Frame-Options` has no origin list (it is SAMEORIGIN or DENY), so it is +cleared explicitly in the headers dict and the CSP does the scoping. It is +cleared *here* rather than via `xheaders`: `xheaders` controls whether Tornado +trusts `X-Forwarded-*` / `X-Real-Ip` from a proxy and has nothing to do with +framing. Setting `xheaders = False` and expecting the frame header to disappear +leaves Jupyter emitting `SAMEORIGIN`, and the iframe stays blank in every +browser that honours it -- the exact failure this file exists to prevent. + **Origin checks.** A cross-origin websocket from the app's port is rejected by default, which breaks the kernel connection while the page still renders. That is the confusing half of the failure, so it is set alongside the headers rather @@ -31,17 +39,24 @@ c.ServerApp.open_browser = False c.ServerApp.port = int(_LAB_PORT) +# `127.0.0.1:8080` and `localhost:8080` are different origins to a browser, and +# which one the app is opened on is not something this file gets to decide. The +# alias is only added when the configured origin actually carries a numeric +# port: `rsplit(':', 1)[-1]` on a portless origin like `http://example.com` +# yields `example.com`, and `http://localhost:example.com` is an invalid source +# that browsers drop silently -- narrowing the allowed ancestors instead of +# widening them. +_PORT = _APP_ORIGIN.rsplit(":", 1)[-1] +_LOCALHOST_ALIAS = f" http://localhost:{_PORT}" if _PORT.isdigit() else "" + # Framing, scoped to the app's origin rather than to `*`. c.ServerApp.tornado_settings = { "headers": { - "Content-Security-Policy": ( - f"frame-ancestors 'self' {_APP_ORIGIN} http://localhost:{_APP_ORIGIN.rsplit(':', 1)[-1]}" - ), + "Content-Security-Policy": f"frame-ancestors 'self' {_APP_ORIGIN}{_LOCALHOST_ALIAS}", + # Cleared deliberately; the CSP above does the scoping. See the module + # docstring for why this is not `xheaders`. + "X-Frame-Options": "", }, - # X-Frame-Options has no origin list -- it is SAMEORIGIN or DENY -- so it is - # removed and the CSP above does the work. Leaving it set would win over the - # CSP in browsers that honour both. - "xheaders": False, } c.ServerApp.allow_origin = _APP_ORIGIN c.ServerApp.allow_credentials = True diff --git a/core/budget.py b/core/budget.py index 1ed666e..fe54d1b 100644 --- a/core/budget.py +++ b/core/budget.py @@ -91,9 +91,27 @@ def set_current(project_id: str | None) -> None: def resolve(explicit: str | None = None) -> str | None: - """`--project` beats the selection file. Neither is an error: work outside - a project is allowed and lands under `unassigned`.""" - return explicit or current_project() + """`--project` beats the selection file. + + Having *no* project is fine -- work outside one is allowed and lands under + `unassigned`. Naming one that does not exist is not: every ceiling check + treats an unknown id as unbounded, so a typo in `--project` would silently + buy an unlimited allocation. That is the one way this dimension could make + spending *easier* than before it existed, so it is refused here. + """ + if explicit: + if not exists(explicit): + known = ", ".join(sorted(projects())) or "(none created yet)" + raise UsageError( + f"project {explicit!r} does not exist, so nothing would bound this spend. " + f"known projects: {known}", + fix=( + f"python -m tools.budget new --id {explicit} --title '...' " + "--gpu-usd --json" + ), + ) + return explicit + return current_project() def resolve_or_fail(explicit: str | None, *, what: str) -> str: diff --git a/core/config.py b/core/config.py index bee1a64..4b14382 100644 --- a/core/config.py +++ b/core/config.py @@ -58,13 +58,16 @@ "request_timeout_s": 60, "min_request_interval_s": 1.1, # unauthenticated S2 is ~1 req/s }, - # HANDOFF-2 §18. The REST paths were NOT verified during the design session, - # which is why they are configuration rather than constants: read - # context7.com/docs/api-guide and correct them here if a request 404s. + # HANDOFF-2 §18 listed the REST paths as unverified (§23 item 2). They are + # now verified against the live API: `/api/v2/libs/search` returns + # `{"results": [...]}` and `/api/v2/context` returns `{"codeSnippets": [...]}` + # when `type=json` is passed. They stay configuration rather than constants + # because a third-party API can move, and a 404 here should be a one-line + # config edit rather than a code change. "docs": { - "base": "https://context7.com/api/v1", - "resolve_path": "/search", - "docs_path": "/{library_id}", + "base": "https://context7.com", + "resolve_path": "/api/v2/libs/search", + "docs_path": "/api/v2/context", "request_timeout_s": 30, "cache_ttl_s": 86400, "min_request_interval_s": 0.5, diff --git a/core/http.py b/core/http.py index 4e1c3ee..efb2d08 100644 --- a/core/http.py +++ b/core/http.py @@ -186,23 +186,29 @@ class Context7: fetch and the cache live. Documentation lookups repeat heavily, and `core/http.py` already has the TTL cache and the rate limiter. - **The endpoint paths were not verified during the design session.** They are - configurable for exactly that reason: read `context7.com/docs/api-guide` and - fix `[docs] resolve_path` / `docs_path` in config/grad.toml rather than - editing code. A 404 here reports the path it tried, so the mismatch names - itself. + The endpoint paths and the response keys below are verified against the live + API. They remain configurable because a third-party API can move, and a 404 + should be a one-line config edit rather than a code change -- the error says + which path it tried, so the mismatch names itself. """ def __init__(self, cfg: Config) -> None: - self.base = str(cfg.get("docs", "base", "https://context7.com/api/v1")).rstrip("/") - self.resolve_path = str(cfg.get("docs", "resolve_path", "/search")) - self.docs_path = str(cfg.get("docs", "docs_path", "/{library_id}")) + self.base = str(cfg.get("docs", "base", "https://context7.com")).rstrip("/") + self.resolve_path = str(cfg.get("docs", "resolve_path", "/api/v2/libs/search")) + self.docs_path = str(cfg.get("docs", "docs_path", "/api/v2/context")) self.timeout = float(cfg.get("docs", "request_timeout_s", 30)) self.ttl = float(cfg.get("docs", "cache_ttl_s", 86400)) self.interval = float(cfg.get("docs", "min_request_interval_s", 0.5)) # Free from their dashboard, and it raises rate limits rather than - # unlocking anything -- so its absence is a note, not an error. - self.key = credentials.get(credentials.CONTEXT7_KEY, required=False) + # unlocking anything -- so its absence is a note, not an error. That + # includes the case where the credential *store* is unreachable: on a + # machine with no keyring installed, `credentials.get` raises even for + # an optional credential, and letting that propagate would make an + # anonymous lookup impossible for want of a key it does not need. + try: + self.key = credentials.get(credentials.CONTEXT7_KEY, required=False) + except ConfigError: + self.key = None @property def authenticated(self) -> bool: @@ -243,9 +249,8 @@ def _get(self, path: str, params: dict[str, Any]) -> Any: raise UpstreamError( f"Context7 returned 404 for {url}", fix=( - "the REST paths were not verified when this was written -- read " - "context7.com/docs/api-guide and set [docs] resolve_path / docs_path " - "in config/grad.toml" + "the API may have moved: read context7.com/docs/api-guide and set " + "[docs] base / resolve_path / docs_path in config/grad.toml" ), ) if resp.status_code >= 400: @@ -266,7 +271,7 @@ def resolve(self, name: str) -> list[dict[str, Any]]: The MCP tool this mirrors is `resolve-library-id`; note that the docs tool is `query-docs`, and `get-library-docs` in older material is stale. """ - data = self._get(self.resolve_path, {"query": name}) + data = self._get(self.resolve_path, {"libraryName": name, "query": name}) results = data.get("results") if isinstance(data, dict) else data out = [] for item in results or []: @@ -285,12 +290,30 @@ def resolve(self, name: str) -> list[dict[str, Any]]: return [o for o in out if o["library_id"]] def docs(self, library_id: str, query: str, *, tokens: int = 5000) -> dict[str, Any]: - """Documentation for a library, narrowed by a topic query.""" - path = self.docs_path.format(library_id=library_id.lstrip("/")) - data = self._get(path, {"topic": query, "tokens": tokens, "type": "json"}) + """Documentation for a library, narrowed by a topic query. + + `type=json` matters: without it the endpoint returns markdown prose, + which is fine for a human and useless as a `--json` payload. + """ + params = {"libraryId": library_id, "query": query, "tokens": tokens, "type": "json"} + # A path template is still honoured, so an older `/{library_id}` style + # endpoint keeps working from config alone. + if "{library_id}" in self.docs_path: + path = self.docs_path.format(library_id=library_id.lstrip("/")) + params.pop("libraryId") + params["topic"] = params.pop("query") + else: + path = self.docs_path + data = self._get(path, params) + if isinstance(data, dict) and "text" in data and len(data) == 1: return {"library_id": library_id, "query": query, "text": data["text"]} - snippets = data.get("snippets") if isinstance(data, dict) else None + # The key differs by API version -- `codeSnippets` on v2, `snippets` on + # v1 -- and reading only one of them turns a working response into an + # empty result that looks like "this library has no docs". + snippets = None + if isinstance(data, dict): + snippets = data.get("codeSnippets") or data.get("snippets") return { "library_id": library_id, "query": query, diff --git a/pyproject.toml b/pyproject.toml index 968efb0..a9bc5de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,10 +41,12 @@ lab = [ ] # §20: human-facing only. Not in the agent's tool list and not in system.md. wiki = ["repowiki==0.3.1"] -# §21: the evolutionary search driver. Optional because `tools/evolve.py` runs -# its own local loop when Shinka is absent, and the campaign bookkeeping and -# budget gate -- the parts that matter -- are ours either way. -evolve = ["shinka-evolve>=0.2"] +# §21: the evolutionary search driver. Pinned exactly, like the `lab` extra, and +# to a version that actually exists: 0.0.7 is the latest published release, so +# the `>=0.2` this originally carried was unsatisfiable and would have failed at +# install time. Optional because the campaign bookkeeping and the budget gate -- +# the parts that matter -- are ours either way. +evolve = ["shinka-evolve==0.0.7"] dev = ["pytest>=8.0", "pytest-asyncio>=0.23"] [project.scripts] diff --git a/tests/test_budget.py b/tests/test_budget.py index 77b34d2..8ccbdc4 100644 --- a/tests/test_budget.py +++ b/tests/test_budget.py @@ -382,3 +382,31 @@ def test_a_non_hf_payer_yields_no_namespace(workspace): budget.create("proj-1", title="t", budget={}, payer="lab-account") assert budget.hf_namespace("proj-1") is None assert budget.hf_namespace(None) is None + + +# --------------------------------------------------------------------------- +# review fixes +# --------------------------------------------------------------------------- +def test_an_unknown_project_is_refused_rather_than_unbounded(workspace): + """A typo in --project must not buy an unlimited allocation. + + Every ceiling check treats an unknown id as unbounded, so silently accepting + one would make this dimension the easiest way to spend *more* than before it + existed. + """ + from core.errors import UsageError + + make_project("proj-1", gpu_usd=1.0) + with pytest.raises(UsageError) as exc: + budget.resolve("proj-l") # lowercase L, not a 1 + assert "does not exist" in str(exc.value) + assert "proj-1" in str(exc.value), "the error should name the real projects" + + +def test_no_project_at_all_is_still_fine(workspace): + """Work outside a project stays allowed; only a *named* unknown is refused.""" + assert budget.resolve(None) is None + make_project("proj-1", gpu_usd=1.0) + budget.set_current("proj-1") + assert budget.resolve(None) == "proj-1" + assert budget.resolve("proj-1") == "proj-1" diff --git a/tests/test_docs.py b/tests/test_docs.py index bda8a34..398b05d 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -198,7 +198,9 @@ def test_resolve_returns_library_ids(workspace, fake_httpx): client = http.Context7(config_mod.load(reload=True)) candidates = client.resolve("huggingface_hub") assert candidates[0]["library_id"] == "/huggingface/huggingface_hub" - assert calls[0]["params"] == {"query": "huggingface_hub"} + assert calls[0]["params"] == { + "libraryName": "huggingface_hub", "query": "huggingface_hub", + } def test_no_key_is_a_note_not_an_error(workspace, fake_httpx): @@ -287,3 +289,108 @@ def test_context7_env_vars_are_scrubbed(workspace, monkeypatch): removed = credentials.scrub_environment() assert "CONTEXT7_API_KEY" in removed assert "GRAD_CONTEXT7_KEY" in removed + + +# --------------------------------------------------------------------------- +# review fixes +# --------------------------------------------------------------------------- +def test_import_of_a_submodule_binds_the_top_level_package(workspace): + """`import os.path` binds `os`, not `os.path`. + + Recording the dotted path made every later `os.` call resolve against + `os.path` and report a missing attribute that is not missing at all. + """ + path = write(workspace, "import os.path\n\nos.getcwd()\n") + assert docs.analyse(path)["findings"] == [] + + +def test_an_aliased_submodule_import_keeps_the_submodule(workspace): + path = write(workspace, "import os.path as p\n\np.no_such_function()\n") + findings = docs.analyse(path)["findings"] + assert findings and findings[0]["kind"] == "missing_attribute" + assert "os.path" in findings[0]["message"] + + +def test_currency_degrades_when_the_credential_store_is_unreachable(workspace, monkeypatch): + """A missing keyring must not discard introspection results that already + computed successfully.""" + from core.errors import ConfigError + + def boom(cfg): + raise ConfigError("credential store unavailable", fix="install keyring") + + monkeypatch.setattr(http, "Context7", boom) + rows = docs._currency([{"module": "nbformat", "stdlib": False, "version": "5.11.0"}]) + assert rows[0]["error"] + assert rows[0]["module"] == "nbformat" + + +def test_context7_constructs_without_a_credential_backend(workspace, monkeypatch): + """The key is optional, so an unreachable credential store must degrade to + anonymous rather than making the client unbuildable.""" + from core import config as config_mod, credentials + from core.errors import ConfigError + + def boom(name, required=True): + raise ConfigError("the `keyring` package is not installed", fix="pip install keyring") + + monkeypatch.setattr(credentials, "get", boom) + client = http.Context7(config_mod.load(reload=True)) + assert client.key is None + assert client.authenticated is False + + +def test_the_context7_request_shapes(workspace, monkeypatch): + """Both endpoints, as verified against the live API.""" + from core import config as config_mod + + seen: list[dict] = [] + + class FakeHttpx: + @staticmethod + def get(url, params=None, headers=None, timeout=None): + seen.append({"url": url, "params": params}) + if "libs/search" in url: + return FakeResponse(payload={"results": [{"id": "/o/lib", "title": "lib"}]}) + return FakeResponse(payload={"codeSnippets": [{"codeTitle": "x"}]}) + + monkeypatch.setattr(http, "_httpx", lambda: FakeHttpx) + monkeypatch.setattr(http.credentials, "get", lambda name, required=True: None) + client = http.Context7(config_mod.load(reload=True)) + + client.resolve("huggingface_hub") + assert seen[-1]["url"].endswith("/api/v2/libs/search") + assert seen[-1]["params"] == {"libraryName": "huggingface_hub", "query": "huggingface_hub"} + + out = client.docs("/o/lib", "run_job", tokens=500) + assert seen[-1]["url"].endswith("/api/v2/context") + assert seen[-1]["params"] == { + "libraryId": "/o/lib", "query": "run_job", "tokens": 500, "type": "json", + } + # v2 names the list `codeSnippets`; reading only `snippets` would turn a + # working response into "this library has no docs". + assert len(out["snippets"]) == 1 + + +def test_a_templated_docs_path_still_works(workspace, monkeypatch): + """An older `/{library_id}` style endpoint stays reachable from config.""" + from core import config as config_mod + + seen: list[dict] = [] + + class FakeHttpx: + @staticmethod + def get(url, params=None, headers=None, timeout=None): + seen.append({"url": url, "params": params}) + return FakeResponse(payload={"snippets": [{"codeTitle": "x"}]}) + + monkeypatch.setattr(http, "_httpx", lambda: FakeHttpx) + monkeypatch.setattr(http.credentials, "get", lambda name, required=True: None) + client = http.Context7(config_mod.load(reload=True)) + client.docs_path = "/{library_id}" + out = client.docs("/o/lib", "run_job", tokens=500) + + assert seen[-1]["url"].endswith("/o/lib") + assert "libraryId" not in seen[-1]["params"] + assert seen[-1]["params"]["topic"] == "run_job" + assert len(out["snippets"]) == 1 diff --git a/tests/test_evolve.py b/tests/test_evolve.py index 8cbd5ca..5d20d16 100644 --- a/tests/test_evolve.py +++ b/tests/test_evolve.py @@ -411,3 +411,62 @@ def test_capabilities_answers_the_driver_or_fork_question(workspace): assert "installed" in report if not report["installed"]: assert "shinka" in report["reason"] + + +# --------------------------------------------------------------------------- +# review fixes +# --------------------------------------------------------------------------- +def test_an_escaped_candidate_is_not_charged(workspace, monkeypatch): + """It never ran, so it cost nothing. + + Charging the per-candidate estimate for declined work would let a campaign + that mostly escapes the evolve block exhaust its allocation having evaluated + almost nothing. + """ + budget.create("proj-1", title="t", budget={"gpu_usd": 100.0}) + task_dir = scaffold(workspace) + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator(escape_at=0)) + result = evolve.cmd_run( + run_args(task_dir, make_expectation(), project="proj-1", + generations=1, population=2, estimate_per_candidate_usd=0.5) + ) + + rows = camp.candidates(result["campaign"]) + escaped = [r for r in rows if r["escaped_block"]["escaped"]] + evaluated = [r for r in rows if not r["escaped_block"]["escaped"]] + assert len(escaped) == 1 and len(evaluated) == 1 + assert escaped[0]["cost_usd"] == 0.0 + assert evaluated[0]["cost_usd"] == 0.5 + # Only the candidate that actually ran reaches the project's allocation. + assert budget.spend("proj-1")["gpu_usd"] == 0.5 + + +def test_the_shinka_driver_refuses_a_whole_loop_only_runner(workspace): + """§23 item 1, answered: `ShinkaEvolveRunner` exposes `run` and `run_async`, + both of which own the loop this driver needs to interrupt between + generations. Calling a `propose()` that does not exist would be an + AttributeError mid-campaign; refusing up front is the honest form, and it is + the evidence §21 said a fork should wait for. + """ + class WholeLoopOnly: + def run(self): ... + def run_async(self): ... + + assert evolve.ShinkaMutator._propose_method(WholeLoopOnly()) is None + + +def test_the_shinka_driver_accepts_a_per_generation_entry_point(workspace): + class Steppable: + def run(self): ... + def propose(self, **kw): ... + + assert evolve.ShinkaMutator._propose_method(Steppable()) == "propose" + + +def test_capabilities_names_the_granularity_it_found(workspace): + report = evolve.mutator_capabilities() + if report["installed"]: + assert report["granularity"] in ("candidate", "generation", "campaign") + assert isinstance(report["driver_viable"], bool) + else: + assert "shinka" in report["reason"] diff --git a/tests/test_lab_and_wiki.py b/tests/test_lab_and_wiki.py index 32d600f..701a6e1 100644 --- a/tests/test_lab_and_wiki.py +++ b/tests/test_lab_and_wiki.py @@ -208,3 +208,94 @@ def _repo_root(): from pathlib import Path return Path(__file__).resolve().parent.parent + + +# --------------------------------------------------------------------------- +# review fixes +# --------------------------------------------------------------------------- +def _server_app(monkeypatch, origin: str): + """Execute the real Jupyter config and return the ServerApp it configured. + + Asserting on the resulting settings rather than grepping the source: the + docstring deliberately *mentions* `xheaders` to record why it is the wrong + lever, and a text search cannot tell an explanation from a setting. + """ + source = (_repo_root() / "config" / "jupyter" / "jupyter_server_config.py").read_text( + encoding="utf-8" + ) + config = type("Config", (), {})() + config.ServerApp = type("ServerApp", (), {})() + namespace: dict = {"get_config": lambda: config} + monkeypatch.setenv("GRAD_UI_ORIGIN", origin) + exec(compile(source, "jupyter_server_config.py", "exec"), namespace) + return config.ServerApp + + +def test_x_frame_options_is_cleared_explicitly(workspace, monkeypatch): + """`xheaders` controls trust of X-Forwarded-* proxy headers and has nothing + to do with framing. Relying on it left Jupyter emitting + `X-Frame-Options: SAMEORIGIN`, and since the UI and Lab are different + origins the iframe stayed blank -- the exact failure the file prevents. + """ + settings = _server_app(monkeypatch, "http://127.0.0.1:8080").tornado_settings + assert settings["headers"]["X-Frame-Options"] == "" + assert "xheaders" not in settings, "xheaders is not the lever for this" + + +def test_the_csp_is_well_formed_with_and_without_a_port(workspace, monkeypatch): + """`rsplit(':', 1)[-1]` on a portless origin yields the hostname, and + `http://localhost:example.com` is an invalid source that browsers drop -- + silently narrowing the allowed ancestors instead of widening them.""" + def csp_for(origin: str) -> str: + return _server_app(monkeypatch, origin).tornado_settings["headers"][ + "Content-Security-Policy" + ] + + with_port = csp_for("http://127.0.0.1:8080") + assert "http://127.0.0.1:8080" in with_port + assert "http://localhost:8080" in with_port + + without_port = csp_for("http://example.com") + assert "http://example.com" in without_port + assert "localhost:example.com" not in without_port + # And it must still be a valid, non-empty directive. + assert without_port.startswith("frame-ancestors 'self' ") + + +def test_repowiki_is_invoked_with_one_path_and_a_supported_format(workspace, monkeypatch): + """repowiki 0.3.1's `map` takes exactly one `path`, `--format text|json`, + and has no `--output` / `--open`. HANDOFF-2 §20's `--format html --open` + would fail immediately, so the HTML is rendered here instead. + """ + calls: list[list[str]] = [] + + class Result: + returncode = 0 + stdout = '{"files": [{"path": "core/budget.py", "rank": 0.12, "language": "python", "lines": 300}]}' + stderr = "" + + def fake_run(argv, **kw): + calls.append(argv) + return Result() + + (workspace / "core").mkdir(parents=True, exist_ok=True) + (workspace / "core" / "a.py").write_text("x = 1\n", encoding="utf-8") + (workspace / "tools").mkdir(parents=True, exist_ok=True) + (workspace / "tools" / "b.py").write_text("y = 2\n", encoding="utf-8") + + monkeypatch.setattr(wiki.shutil, "which", lambda name: "repowiki") + monkeypatch.setattr(wiki.subprocess, "run", fake_run) + + result = wiki.cmd_map(argparse.Namespace(top=200, open=False, json=True)) + + assert len(calls) == 2, "one invocation per scope directory" + for argv in calls: + paths_given = [a for a in argv[2:] if not a.startswith("-") and a not in ("json", "200")] + assert len(paths_given) == 1, f"map takes exactly one path: {argv}" + assert "--format" in argv and argv[argv.index("--format") + 1] == "json" + assert "--output" not in argv and "--open" not in argv + + # HTML is produced by us, because repowiki cannot emit it. + assert result["html"].endswith("index.html") + html = (wiki.output_dir() / "index.html").read_text(encoding="utf-8") + assert "core/budget.py" in html diff --git a/tests/test_namespace.py b/tests/test_namespace.py index 2b35cf4..586fab7 100644 --- a/tests/test_namespace.py +++ b/tests/test_namespace.py @@ -285,3 +285,58 @@ def test_the_namespace_is_not_part_of_the_submission_hash(workspace): before = sub.hash() sub.target["namespace"] = "myorg" assert sub.hash() == before + + +# --------------------------------------------------------------------------- +# review fixes +# --------------------------------------------------------------------------- +def test_a_missing_hub_leaves_no_phantom_smoke_record(workspace, monkeypatch): + """A configuration problem must not leave an estimate on the ceiling. + + The smoke path wrote its in-flight record before resolving the backend, so a + machine without huggingface_hub installed booked spend for a job that never + reached the platform -- which then goes stale and blocks every later + submission through the §6 gate. + """ + from core import config as config_mod + from tools import jobs + + sub = make_submission(workspace) + + def no_hub(): + raise ConfigError("huggingface_hub is not installed", fix="pip install huggingface_hub") + + monkeypatch.setattr(jobs, "_hub", no_hub) + monkeypatch.setattr(jobs, "_token", lambda: "tok") + + with pytest.raises(ConfigError): + jobs.run_smoke(sub, config_mod.load(reload=True)) + + assert ls.in_flight() == [] + assert ls.rolling_spend(30)["total_usd"] == 0.0 + + +def test_an_unsupported_namespace_leaves_no_phantom_smoke_record(workspace, monkeypatch): + """Same rule for the other configuration failure on that path.""" + from core import config as config_mod + from tools import jobs + + class Old: + def run_job(self, *, image, command, token=None): + ... + + def inspect_job(self, *, job_id, token=None): + ... + + def fetch_job_logs(self, *, job_id, token=None): + ... + + monkeypatch.setattr(jobs, "_hub", lambda: Old()) + monkeypatch.setattr(jobs, "_token", lambda: "tok") + + sub = make_submission(workspace) + with pytest.raises(ConfigError): + jobs.run_smoke(sub, config_mod.load(reload=True), namespace="myorg") + + assert ls.in_flight() == [] + assert ls.rolling_spend(30)["total_usd"] == 0.0 diff --git a/tests/test_report.py b/tests/test_report.py index 4c3aa86..4efecb0 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -465,3 +465,84 @@ def test_a_report_needs_a_project(workspace): with pytest.raises(UsageError) as exc: report.cmd_draft(args()) assert "tools.budget use" in (exc.value.fix or "") + + +# --------------------------------------------------------------------------- +# review fixes +# --------------------------------------------------------------------------- +def test_rerunning_cite_preserves_earlier_entries(workspace, project, stub_resolver): + r"""`cite` is naturally re-run -- after a new section, or after ingesting a + paper that failed to resolve last time. By then the earlier placeholders are + already `\cite{}` keys, so a second pass finds nothing to resolve; rewriting + the bib from scratch deleted every entry the first pass earned and left + `check` refusing on citations that were fine a moment ago. + """ + in_range_run() + report.cmd_draft(args()) + first = report.cmd_cite(args(context_chars=200, no_s2=True)) + assert first["entries"] == 1 + + second = report.cmd_cite(args(context_chars=200, no_s2=True)) + assert second["entries"] == 1, "the earlier entry must survive" + assert second["kept_from_previous_run"] == ["basis2026"] + assert report.cmd_check(args())["ok"] is True + + +def test_a_weakly_related_s2_hit_is_rejected(workspace, project, monkeypatch): + """An 8% word overlap was close to a rubber stamp: any two ML papers share + enough vocabulary to clear it. A citation wrongly accepted is a claim + silently attributed to a paper that does not support it.""" + class FakeS2: + def __init__(self, cfg): ... + def paper_search(self, keyword, limit=5): + return [{"paper_id": "abc", "title": "Convolutional Networks for Image Segmentation", + "abstract": "We segment images using convolutions and pooling layers.", + "year": 2015}] + + monkeypatch.setattr(report.http, "SemanticScholar", FakeS2) + context = ( + "The scaling behaviour of transformer language models under a fixed token " + "budget follows a power law in parameter count across pretraining corpora." + ) + assert report._from_s2("scaling laws", context) is None + + +def test_a_genuinely_matching_s2_hit_is_accepted_and_scored(workspace, project, monkeypatch): + class FakeS2: + def __init__(self, cfg): ... + def paper_search(self, keyword, limit=5): + return [{"paper_id": "abc", + "title": "Scaling Laws for Neural Language Models", + "abstract": "We study empirical scaling laws for language model " + "performance as a function of parameter count and " + "pretraining token budget, finding power-law behaviour.", + "year": 2020}] + + monkeypatch.setattr(report.http, "SemanticScholar", FakeS2) + context = ( + "Empirical scaling laws for language models describe performance as a " + "power-law function of parameter count and pretraining token budget." + ) + entry = report._from_s2("scaling laws", context) + assert entry is not None + assert entry["gradsource"] == "s2" + # Both scores recorded, so a borderline resolution is auditable. + assert entry["gradmatch"] >= report.S2_MIN_CONTEXT_OVERLAP + assert entry["gradtitlematch"] >= report.S2_MIN_TITLE_OVERLAP + + +def test_shared_jargon_alone_does_not_clear_the_title_test(workspace, project, monkeypatch): + """The abstract can overlap on generic research vocabulary; the title is + where a paper's actual subject lives.""" + class FakeS2: + def __init__(self, cfg): ... + def paper_search(self, keyword, limit=5): + return [{"paper_id": "abc", "title": "A Dataset of Annotated Radiographs", + "abstract": "scaling laws parameter count pretraining token budget " + "power law transformer language models corpora", + "year": 2021}] + + monkeypatch.setattr(report.http, "SemanticScholar", FakeS2) + context = ("Scaling laws relate parameter count and pretraining token budget " + "to transformer language model loss following a power law.") + assert report._from_s2("scaling laws", context) is None diff --git a/tools/docs.py b/tools/docs.py index cc2aca6..685284d 100644 --- a/tools/docs.py +++ b/tools/docs.py @@ -6,14 +6,31 @@ So there are **two oracles, and the order matters**: 1. **Introspection -- what actually exists on this machine.** - `importlib.metadata.version()`, `inspect.signature()`, `dir()`. Cheap, - offline, definitive. This is how §17's `namespace` parameter was found, in - about ten seconds. + `importlib.metadata.version()`, `inspect.signature()`, `dir()`. Offline and + definitive. This is how §17's `namespace` parameter was found, in about ten + seconds. 2. **Context7 -- what is current.** Deprecations, changed idioms, migration paths. Answers what introspection cannot see. Introspect first. Always. +## `check` imports, and importing runs code + +Introspection is not static analysis, and the difference matters. To read a +signature this has to *import* the module, and importing executes that module's +top level. `check ` therefore imports every module the file names, and +name resolution goes through `sys.path` -- which includes the working directory. +A file containing `import helper` runs a sibling `helper.py`. + +**So `check` is safe on code you trust and unsafe on code you do not.** Run it +on your own pipeline, not on a freshly downloaded repository. There is no way to +have the introspection oracle without this: a checker that does not import can +only guess at what is installed, which is the failure mode this command exists +to remove. `signature` has the same property for the one module it is given. + +If that boundary ever needs to be tightened, the fix is to run the introspection +half in a subprocess -- the parse and the reporting stay as they are. + **Why a CLI and not a subagent.** The original plan was a Haiku "reality checker" with Context7 and Pyright. It was rejected after the brief narrowed, and the reasoning is worth keeping because it will be tempting to revisit: a QA @@ -51,10 +68,14 @@ "Two oracles, in this order:\n" " 1. introspection -- importlib.metadata + inspect.signature. offline, definitive.\n" " 2. Context7 -- deprecations and changed idioms. what introspection cannot see.\n\n" + "WARNING: `check` and `signature` IMPORT the modules they inspect, and importing\n" + "runs that module's top-level code. Module names resolve through sys.path, which\n" + "includes the working directory. Run these on code you trust -- your own pipeline,\n" + "not a repository you just downloaded.\n\n" "`check` exits 9 when it finds something, so it composes with preflight's\n" "declared-check mechanism if a pipeline wants it as a gate.\n\n" - "The Context7 REST paths were not verified when this was written. If a request\n" - "404s, read context7.com/docs/api-guide and fix [docs] in config/grad.toml." + "If a Context7 request 404s the API has moved: read context7.com/docs/api-guide\n" + "and fix [docs] base / resolve_path / docs_path in config/grad.toml." ), ) @@ -154,7 +175,16 @@ def __init__(self) -> None: def visit_Import(self, node: ast.Import) -> None: for alias in node.names: - self.aliases[alias.asname or alias.name.split(".")[0]] = alias.name + if alias.asname: + # `import a.b as x` binds `x` to the submodule itself. + self.aliases[alias.asname] = alias.name + else: + # `import a.b` binds the *top-level package* `a`, not `a.b`, so + # a later `a.f()` is `a.f` and not `a.b.f`. Recording the full + # dotted path here made `import os.path` turn every `os.` + # call into a false "does not exist on os.path" finding. + top = alias.name.split(".")[0] + self.aliases[top] = top self.generic_visit(node) def visit_ImportFrom(self, node: ast.ImportFrom) -> None: @@ -355,7 +385,12 @@ def cmd_query(args: argparse.Namespace) -> dict[str, Any]: def _check_args(p: argparse.ArgumentParser) -> None: - p.add_argument("path", help="a Python file whose library calls should be checked") + p.add_argument( + "path", + help="a Python file whose library calls should be checked. NOTE: this imports " + "the modules the file names, which runs their top-level code -- use it on " + "code you trust", + ) p.add_argument( "--offline", action="store_true", @@ -374,6 +409,9 @@ def cmd_check(args: argparse.Namespace) -> dict[str, Any]: Exits 9 -- "a check ran and reported failure" -- so it composes with preflight's declared-check mechanism if a pipeline later wants it as a gate. + + This imports the modules the file names, and importing executes them. See + the module docstring: safe on code you trust, unsafe on code you do not. """ report = analyse(Path(args.path).resolve()) @@ -394,8 +432,20 @@ def cmd_check(args: argparse.Namespace) -> dict[str, Any]: def _currency(modules: list[dict[str, Any]]) -> list[dict[str, Any]]: """Best-effort: a Context7 outage must not turn an offline-verifiable - finding into an unusable command.""" - client = http.Context7(config_mod.load()) + finding into an unusable command. + + Construction is inside the guard for the same reason the requests are -- + a missing credential backend must degrade this to "currency unknown", not + discard the introspection results that were already computed. + """ + try: + client = http.Context7(config_mod.load()) + except GradError as exc: + return [ + {"module": m["module"], "installed": m["version"], "error": exc.message} + for m in modules + if not m["stdlib"] + ] out = [] for info in modules: if info["stdlib"]: diff --git a/tools/evolve.py b/tools/evolve.py index d4948c6..de66c6a 100644 --- a/tools/evolve.py +++ b/tools/evolve.py @@ -116,6 +116,7 @@ def mutator_capabilities() -> dict[str, Any]: runner = getattr(shinka, "ShinkaEvolveRunner", None) hooks: list[str] = [] + methods: list[str] = [] if runner is not None: try: params = inspect.signature(runner.__init__).parameters @@ -126,19 +127,40 @@ def mutator_capabilities() -> dict[str, Any]: for name in params if any(word in name for word in ("callback", "hook", "on_", "listener")) ] + methods = sorted( + n for n in dir(runner) if not n.startswith("_") and callable(getattr(runner, n, None)) + ) + + per_generation = next( + (n for n in ShinkaMutator.PROPOSE_METHODS if n in methods), None + ) + if hooks: + granularity, note = "candidate", ( + "a per-candidate hook exists; finer budget charging is possible without a fork" + ) + elif per_generation: + granularity, note = "generation", ( + f"no per-candidate hook, but `{per_generation}()` yields one generation at a " + "time, so the budget is re-checked at generation boundaries. Driver, not fork." + ) + else: + granularity, note = "campaign", ( + "no per-candidate hook and no per-generation entry point -- the runner exposes " + "only whole-loop methods, which own the loop this driver needs to interrupt. " + "This is the evidence §21 said a fork should wait for: `evolve run` refuses " + "rather than handing control away with the budget unchecked." + ) + return { "installed": True, "version": getattr(shinka, "__version__", None), "runner": runner is not None, "per_candidate_hooks": hooks, - "granularity": "candidate" if hooks else "generation", - "note": ( - "no per-candidate hook found; budget is charged at generation boundaries, " - "which is what this driver is designed around. A fork is only warranted if " - "that granularity proves insufficient." - if not hooks - else "a per-candidate hook exists; finer budget charging is possible without a fork" - ), + "per_generation_method": per_generation, + "runner_methods": methods[:20], + "granularity": granularity, + "driver_viable": bool(hooks or per_generation), + "note": note, } @@ -592,6 +614,11 @@ def _evaluate_candidate( "skipped": True, "error": "mutation escaped the evolve block; needs a fresh smoke run", "duration_s": round(time.time() - started, 3), + # It never ran, so it cost nothing. Charging the per-candidate + # estimate anyway would consume the project's allocation for + # work that was declined, and a campaign that mostly escapes + # would exhaust its budget having evaluated almost nothing. + "cost_usd": 0.0, } ) camp.append_candidate(record) @@ -656,6 +683,13 @@ def __init__(self, task_dir: Path, models: tuple[str, ...], overrides: list[str] self.models = models self.overrides = overrides self._runner: Any = None + self._method: str = "" + + # A per-generation entry point, under any of the names an upstream release + # might plausibly use. `run` and `run_async` are deliberately NOT here: they + # drive the whole campaign themselves, which is precisely the control this + # driver needs to keep in order to charge the budget between generations. + PROPOSE_METHODS = ("propose", "propose_generation", "step", "ask") def _build(self) -> Any: shinka = _shinka() @@ -666,30 +700,64 @@ def _build(self) -> Any: ) job = shinka.LocalJobConfig(eval_program_path=str(self.task_dir / "evaluate.py")) database = shinka.DatabaseConfig() - return shinka.ShinkaEvolveRunner( + runner = shinka.ShinkaEvolveRunner( evo_config=evolution, job_config=job, db_config=database ) except (AttributeError, TypeError) as exc: raise ConfigError( f"the installed shinka-evolve does not match the expected API: {exc}", fix=( - "check ShinkaEvolveRunner's signature against the installed version " + "check the constructor against the installed version " "(`python -m tools.docs signature shinka ShinkaEvolveRunner --json`), " "then adjust ShinkaMutator" ), ) from exc + method = self._propose_method(runner) + if method is None: + # This is §23 item 1 answered at runtime, and it is the evidence §21 + # said a fork should wait for: `ShinkaEvolveRunner` exposes `run` and + # `run_async`, both of which own the whole loop. A driver cannot + # charge the budget between generations through an API that only + # offers "run everything", so it refuses rather than either handing + # control away or calling a method that does not exist. + available = sorted( + n for n in dir(runner) if not n.startswith("_") and callable(getattr(runner, n, None)) + ) + raise ConfigError( + "the installed ShinkaEvolveRunner exposes no per-generation entry point, " + "only whole-loop methods, so the campaign budget could not be re-checked " + "between generations. HANDOFF-2 §21 defers a fork until exactly this " + "evidence appears; this is it. " + f"Available: {', '.join(available[:12]) or '(none)'}", + fix=( + "run the campaign against a driver you control -- or fork Shinka to " + "expose one generation at a time, which is the point at which §21's " + "'driver, not fork' decision flips. " + "`python -m tools.evolve capabilities --json` reports what was found." + ), + ) + self._method = method + return runner + + @classmethod + def _propose_method(cls, runner: Any) -> str | None: + for name in cls.PROPOSE_METHODS: + if callable(getattr(runner, name, None)): + return name + return None + def propose(self, *, generation: int, population: int, best: dict[str, Any] | None) -> list[str]: if self._runner is None: self._runner = self._build() - proposals = self._runner.propose( + proposals = getattr(self._runner, self._method)( generation=generation, population=population, parent=(best or {}).get("source") ) quota_log.record( STAGE_EVOLVE, role="evolve", model=",".join(self.models), - detail={"generation": generation, "population": population}, + detail={"generation": generation, "population": population, "method": self._method}, ) return [str(p) for p in proposals] diff --git a/tools/jobs.py b/tools/jobs.py index 6f5a5db..6e10bdf 100644 --- a/tools/jobs.py +++ b/tools/jobs.py @@ -370,6 +370,16 @@ def run_smoke( command = _smoke_command(sub, caps) if namespace is None: namespace = resolve_namespace(None, sub, cfg, project or budget.current_project()) + + # Everything that can fail for a *configuration* reason resolves before the + # ledger record exists. `_hub()` raises when huggingface_hub is missing and + # `_ns_kwargs()` raises when the installed one cannot take a namespace; + # either one landing after `record_smoke_run` leaves a phantom in-flight + # estimate sitting on the monthly ceiling for a job that never reached the + # platform -- which then goes stale and blocks every later submission. + hub = _hub() + ns_kwargs = _ns_kwargs(namespace) + run_id = submit_lib.record_smoke_run( sub, cfg=cfg, platform=PLATFORM, target={"flavor": flavor, "platform": "hf", "namespace": namespace}, @@ -378,7 +388,6 @@ def run_smoke( artifacts = submit_lib.artifacts_dir(run_id) try: - hub = _hub() job = hub.run_job( image=sub.image, command=command, @@ -386,7 +395,7 @@ def run_smoke( env={**_job_env(sub), "GRAD_SMOKE": "1"}, token=_token(), timeout=caps["timeout_s"], - **_ns_kwargs(namespace), + **ns_kwargs, ) except ConfigError: raise diff --git a/tools/report.py b/tools/report.py index 8a454f7..c724732 100644 --- a/tools/report.py +++ b/tools/report.py @@ -486,13 +486,33 @@ async def submit_prose(args: dict[str, Any]) -> dict[str, Any]: ) async def run() -> None: - usage = None - async for message in query( - prompt="Evidence bundle:\n\n" + json.dumps(bundle, indent=2, default=str), - options=options, - ): - usage = getattr(message, "usage", None) or usage - quota_log.from_sdk_usage("report.write", usage, model=model, role="report") + # The result message carries a cumulative total, so it wins outright + # when it arrives. Assistant-message usage is accumulated only as a + # fallback for the turn that dies before a result -- adding both would + # double-count. + final: Any = None + partial = {"input_tokens": 0, "output_tokens": 0} + try: + async for message in query( + prompt="Evidence bundle:\n\n" + json.dumps(bundle, indent=2, default=str), + options=options, + ): + usage = getattr(message, "usage", None) + if usage is None: + continue + if type(message).__name__ == "ResultMessage": + final = usage + else: + get = usage.get if isinstance(usage, dict) else (lambda k, d=0: getattr(usage, k, d)) + partial["input_tokens"] += get("input_tokens", 0) or 0 + partial["output_tokens"] += get("output_tokens", 0) or 0 + finally: + # In a finally block because a failed turn still spent quota, and an + # unrecorded spend is exactly what §15's ceilings cannot see. + quota_log.from_sdk_usage( + "report.write", final if final is not None else partial, + model=model, role="report", + ) asyncio.run(run()) if not captured: @@ -532,7 +552,18 @@ def cmd_cite(args: argparse.Namespace) -> dict[str, Any]: ) tex = files["tex"].read_text(encoding="utf-8") - entries: dict[str, dict[str, Any]] = {} + # Seeded with what is already there, not started empty. `cite` is naturally + # re-run -- after adding a section, after ingesting a paper that previously + # failed to resolve -- and by then the earlier placeholders are already + # `\cite{}` keys, so a second pass finds nothing to resolve. Rewriting the + # file from an empty dict would delete every entry the first pass earned and + # leave `check` refusing on citations that were fine a moment ago. + entries: dict[str, dict[str, Any]] = ( + report_lib.parse_bib(files["bib"].read_text(encoding="utf-8")) + if files["bib"].exists() + else {} + ) + preexisting = set(entries) resolved: list[dict[str, Any]] = [] unresolved: list[dict[str, Any]] = [] @@ -566,6 +597,7 @@ def substitute(match: re.Match[str]) -> str: "resolved": resolved, "unresolved": unresolved, "entries": len(entries), + "kept_from_previous_run": sorted(preexisting), "next": f"python -m tools.report check --project {project_id} --json", } if unresolved: @@ -629,6 +661,18 @@ def _from_corpus(keyword: str) -> dict[str, Any] | None: con.close() +# Two independent conditions, both required. Overlap against the abstract alone +# is easy to clear on shared jargon -- any two ML papers share "training", +# "model", "results" -- so a candidate must also connect to the *title*, which is +# where a paper's actual subject lives. The numbers are deliberately strict: a +# citation this refuses is one the author adds by hand after reading it, while a +# citation it wrongly accepts is a claim silently attributed to a paper that does +# not support it. Recorded on the entry as `gradmatch` / `gradtitlematch` so a +# borderline resolution is auditable rather than invisible. +S2_MIN_CONTEXT_OVERLAP = 0.25 +S2_MIN_TITLE_OVERLAP = 0.20 + + def _from_s2(keyword: str, context: str) -> dict[str, Any] | None: """Verify the candidate's title and abstract against the surrounding text. @@ -644,17 +688,22 @@ def _from_s2(keyword: str, context: str) -> dict[str, Any] | None: if not hits: return None - words = {w.lower() for w in re.findall(r"[a-zA-Z]{5,}", context)} - best, best_score = None, 0.0 + words = _content_words(context) + if not words: + return None + + best, best_score, best_title = None, 0.0, 0.0 for hit in hits: - text = f"{hit.get('title', '')} {hit.get('abstract', '')}".lower() - candidate_words = set(re.findall(r"[a-zA-Z]{5,}", text)) - if not candidate_words: + body = _content_words(f"{hit.get('title', '')} {hit.get('abstract', '')}") + title = _content_words(hit.get("title", "")) + if not body: continue - score = len(words & candidate_words) / max(1, len(words)) + score = len(words & body) / len(words) + title_score = (len(words & title) / len(title)) if title else 0.0 if score > best_score: - best, best_score = hit, score - if best is None or best_score < 0.08: + best, best_score, best_title = hit, score, title_score + + if best is None or best_score < S2_MIN_CONTEXT_OVERLAP or best_title < S2_MIN_TITLE_OVERLAP: return None return { "key": _bib_key(best.get("paper_id", keyword), None, best.get("year")), @@ -665,6 +714,25 @@ def _from_s2(keyword: str, context: str) -> dict[str, Any] | None: "note": f"S2:{best.get('paper_id')}", "gradsource": "s2", "gradmatch": round(best_score, 3), + "gradtitlematch": round(best_title, 3), + } + + +# Words short enough to be grammar rather than subject matter carry no signal, +# and a handful of ubiquitous research words clear any threshold on their own. +_STOPWORDS = frozenset( + """about above after again against because before being below between during + further having their there these those through under until where which while + model models method methods result results using training train paper approach + work works show shows shown study propose proposed""".split() +) + + +def _content_words(text: str) -> set[str]: + return { + w.lower() + for w in re.findall(r"[a-zA-Z]{5,}", text or "") + if w.lower() not in _STOPWORDS } diff --git a/tools/wiki.py b/tools/wiki.py index 3b85f67..92b1d91 100644 --- a/tools/wiki.py +++ b/tools/wiki.py @@ -42,6 +42,7 @@ import argparse import hashlib +import json import shutil import subprocess import time @@ -61,7 +62,9 @@ "third party and those hold research data.\n\n" "`map` is LLM-free and needs no credential. `scan` is refused here on purpose;\n" "see this module's docstring for the reasoning and the ~120-line fork that\n" - "would make it clean." + "would make it clean.\n\n" + "repowiki 0.3.1's `map` takes ONE path and only --format text|json, so each\n" + "scope directory is a separate invocation and the HTML is rendered here." ), ) @@ -132,8 +135,8 @@ def _scope_paths(root: Path) -> list[str]: # --------------------------------------------------------------------------- def _map_args(p: argparse.ArgumentParser) -> None: - p.add_argument("--format", default="html", choices=["html", "json"], help="output format") - p.add_argument("--open", action="store_true", help="open the result in a browser") + p.add_argument("--top", type=int, default=200, help="max entries per scope directory") + p.add_argument("--open", action="store_true", help="open the generated HTML afterwards") @cli.command("map", "generate the module map (LLM-free, no credential)", setup=_map_args) @@ -142,59 +145,141 @@ def cmd_map(args: argparse.Namespace) -> dict[str, Any]: LLM-free by construction, so this costs nothing and ships nothing anywhere. Try it before deciding whether the rest of §20 is wanted at all. + + **The invocation matches repowiki 0.3.1's actual contract**, which differs + from what HANDOFF-2 §20 recorded: `map` takes exactly *one* `path` argument, + `--format` accepts only `text` or `json`, and there is no `--output` and no + `--open`. The handoff's "`--format html --open`" would fail immediately. So + each scope directory is a separate invocation asking for JSON, and the HTML + -- which §20 wants so the output never competes with the hand-written docs -- + is rendered here from that JSON. """ executable = _repowiki() root = paths.root() out = output_dir() out.mkdir(parents=True, exist_ok=True) - argv = [executable, "map", *_scope_paths(root), "--format", args.format, "--output", str(out)] - if args.open: - argv.append("--open") - started = time.time() - try: - proc = subprocess.run( - argv, cwd=str(root), capture_output=True, text=True, timeout=600, check=False - ) - except subprocess.TimeoutExpired as exc: - raise GradError( - "wiki_timeout", "repowiki map did not finish within 10 minutes", - exit_code=8, fix="run it by hand to see where it stalls", - ) from exc - - log = out / "map.log" - log.write_text((proc.stdout or "") + (proc.stderr or ""), encoding="utf-8") - if proc.returncode != 0: - raise GradError( - "wiki_failed", - f"repowiki map exited {proc.returncode}", - exit_code=8, - fix=f"read {log}", - detail={"log": str(log), "tail": (proc.stdout or proc.stderr or "").splitlines()[-15:]}, - ) + scopes: dict[str, Any] = {} + commands: list[list[str]] = [] + for path in _scope_paths(root): + name = Path(path).name + argv = [executable, "map", path, "--format", "json", "--top", str(args.top)] + commands.append(argv[1:]) + try: + proc = subprocess.run( + argv, cwd=str(root), capture_output=True, text=True, timeout=600, check=False + ) + except subprocess.TimeoutExpired as exc: + raise GradError( + "wiki_timeout", f"repowiki map did not finish for {name} within 10 minutes", + exit_code=8, fix="run it by hand to see where it stalls", + ) from exc + + log = out / f"map.{name}.log" + log.write_text((proc.stdout or "") + (proc.stderr or ""), encoding="utf-8") + if proc.returncode != 0: + raise GradError( + "wiki_failed", + f"repowiki map exited {proc.returncode} for {name}", + exit_code=8, + fix=f"read {log}", + detail={ + "command": argv[1:], + "log": str(log), + "tail": (proc.stdout or proc.stderr or "").splitlines()[-15:], + }, + ) + try: + scopes[name] = json.loads(proc.stdout or "{}") + except json.JSONDecodeError: + # `--format json` is what was asked for; anything else is a version + # skew worth reporting rather than silently rendering as prose. + scopes[name] = {"raw": (proc.stdout or "").splitlines()} + + (out / "map.json").write_text( + json.dumps(scopes, indent=2, ensure_ascii=False, default=str), encoding="utf-8" + ) + html_path = _render_html(out, scopes) # The staleness record. A wiki behind the code is worse than none, because # it is trusted -- so the inputs are recorded at generation time, not # reconstructed later. manifest = { "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), - "command": argv[1:], + "commands": commands, "duration_s": round(time.time() - started, 1), "source": source_hash(root), "output_dir": str(out), - "format": args.format, + "html": str(html_path), } jsonl.write_json(_manifest_path(), manifest) + + if args.open: + import webbrowser # noqa: PLC0415 + + webbrowser.open(html_path.as_uri()) + return { "output_dir": str(out), + "html": str(html_path), + "scopes": sorted(scopes), "source_hash": manifest["source"]["hash"], "files_covered": len(manifest["source"]["files"]), - "log": str(log), "next": "python -m tools.wiki check --json", } +def _render_html(out: Path, scopes: dict[str, Any]) -> Path: + """HTML, generated here rather than by repowiki (which cannot emit it). + + §20 wants HTML specifically so the output never competes with the + hand-written docs the way a committed markdown file would. + """ + rows = [] + for scope, payload in sorted(scopes.items()): + files = payload.get("files") if isinstance(payload, dict) else None + rows.append(f"

{_esc(scope)}/

") + if not isinstance(files, list) or not files: + rows.append(f"
{_esc(json.dumps(payload, indent=2, default=str)[:20000])}
") + continue + rows.append("") + for entry in files: + if not isinstance(entry, dict): + continue + rows.append( + "".format( + _esc(str(entry.get("path", entry.get("file", "")))), + _esc(str(round(entry["rank"], 5) if isinstance(entry.get("rank"), float) else entry.get("rank", ""))), + _esc(str(entry.get("language", ""))), + _esc(str(entry.get("lines", ""))), + ) + ) + rows.append("
fileranklanglines
{}{}{}{}
") + + html = ( + "Grad repo map" + "" + "

Grad repo map

" + "

Generated by python -m tools.wiki map over " + f"{_esc(', '.join(SCOPE))} only. Human-facing; not an agent tool. " + "Check freshness with python -m tools.wiki check.

" + + "\n".join(rows) + ) + path = out / "index.html" + path.write_text(html, encoding="utf-8") + return path + + +def _esc(text: str) -> str: + import html as _html # noqa: PLC0415 + + return _html.escape(str(text), quote=True) + + @cli.command("check", "is the generated wiki still current?") def cmd_check(_: argparse.Namespace) -> dict[str, Any]: """The one-line staleness check §20 asks for. From 9618a507c45d803d09853809872316eaa7c250c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=BB=D0=B0=D0=B4=D0=B8=D0=BC=D0=B8=D1=80=20=D0=A8?= =?UTF-8?q?=D0=BC=D0=B0=D0=BD?= Date: Fri, 14 Aug 2026 23:07:41 +0300 Subject: [PATCH 3/3] Address second review: 4 fixes in the smoke path and the report Verified each finding against the code first; one was already fixed and is skipped. **Two more phantom-estimate paths in `run_smoke`.** `_token()` sat inside the try block, so a missing credential propagated without `finish()` ever running -- leaving an in-flight estimate on the monthly ceiling for a job that never reached the platform, which then goes stale and blocks every later submission. It now resolves alongside `_hub()` and `_ns_kwargs()`, before the ledger record exists. Separately, `preflight.py` calls `run_smoke(sub, cfg)` with no project. The namespace was already derived from the current project while the cost was booked as `unassigned` -- the two halves of one decision disagreeing, so an org's job was charged to nobody. The project is now resolved once and used for both. **The S2 citation gate rejected correct papers.** Both thresholds were applied to the winner *after* ranking by context overlap alone, so a loosely-related paper with a keyword-heavy abstract could win the ranking, fail the title gate, and take the genuinely correct paper down with it -- a citation that was right there in the candidate list. Both gates now filter before ranking. **Report usage accounting.** The partial-usage fallback tracked only input and output tokens, silently dropping cache traffic -- which is where a long prompt spends most of its tokens -- so a turn that died before the result message under-reported. And the usage record defaulted its project to whichever was selected, while `--project` can name a different one; the reported project is now threaded through. **Skipped:** validating `args.project` in `tools/gpu.py` and `tools/jobs.py`. Both already route through `budget.resolve()`, which was changed in the previous commit to refuse a named project that does not exist. Verified: a typo'd `--project` raises rather than passing through as unbounded. 321 tests. Co-Authored-By: Claude Opus 5 --- tests/test_namespace.py | 43 ++++++++++++++++++++++++++ tests/test_report.py | 67 +++++++++++++++++++++++++++++++++++++++++ tools/jobs.py | 23 +++++++++----- tools/report.py | 39 ++++++++++++++++++------ 4 files changed, 155 insertions(+), 17 deletions(-) diff --git a/tests/test_namespace.py b/tests/test_namespace.py index 586fab7..6c3ff7e 100644 --- a/tests/test_namespace.py +++ b/tests/test_namespace.py @@ -340,3 +340,46 @@ def fetch_job_logs(self, *, job_id, token=None): assert ls.in_flight() == [] assert ls.rolling_spend(30)["total_usd"] == 0.0 + + +def test_smoke_attributes_to_the_current_project_when_none_is_passed(workspace, hub): + """`preflight.py` calls `run_smoke(sub, cfg)` with no project. + + The namespace was already derived from the current project, so booking the + cost as `unassigned` had the two halves of one decision disagreeing: an + org's job charged to nobody. + """ + from core import budget, config as config_mod + from tools import jobs + + budget.create("proj-1", title="t", budget={"gpu_usd": 100.0}) + budget.set_current("proj-1") + + sub = make_submission(workspace) + jobs.run_smoke(sub, config_mod.load(reload=True)) + + assert [r.project for r in ls.runs()] == ["proj-1"] + # The run is counted against this project's allocation rather than landing + # under `unassigned` where nothing bounds it. (The fake hub reports no + # elapsed time, so the amount itself is legitimately zero.) + assert budget.spend("proj-1")["runs"] == [r.id for r in ls.runs()] + + +def test_a_missing_token_leaves_no_unfinished_smoke_record(workspace, hub, monkeypatch): + """`_token()` sat inside the try, so a missing credential propagated without + `finish()` ever running -- leaving an in-flight estimate on the ceiling for a + job that never reached the platform.""" + from core import config as config_mod + from tools import jobs + + def no_token(): + raise ConfigError("no Hugging Face token is stored", fix="credential set hf_token") + + monkeypatch.setattr(jobs, "_token", no_token) + sub = make_submission(workspace) + + with pytest.raises(ConfigError): + jobs.run_smoke(sub, config_mod.load(reload=True)) + + assert ls.in_flight() == [] + assert ls.rolling_spend(30)["total_usd"] == 0.0 diff --git a/tests/test_report.py b/tests/test_report.py index 4efecb0..4bbcd69 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -546,3 +546,70 @@ def paper_search(self, keyword, limit=5): context = ("Scaling laws relate parameter count and pretraining token budget " "to transformer language model loss following a power law.") assert report._from_s2("scaling laws", context) is None + + +def test_the_right_paper_is_not_taken_down_by_a_keyword_stuffed_one(workspace, project, monkeypatch): + """Both gates are applied before ranking, not to the winner afterwards. + + Ranking first let a loosely-related paper with a keyword-heavy abstract win + on context overlap, fail the title gate, and reject the genuinely correct + paper sitting next to it in the candidate list. + """ + class FakeS2: + def __init__(self, cfg): ... + def paper_search(self, keyword, limit=5): + return [ + {"paper_id": "generic", "title": "Miscellaneous Notes", + "abstract": "scaling laws parameter count pretraining token budget power " + "transformer language corpora empirical performance", + "year": 2021}, + {"paper_id": "right", "title": "Scaling Laws for Neural Language Models", + "abstract": "empirical scaling laws parameter count pretraining token budget", + "year": 2020}, + ] + + monkeypatch.setattr(report.http, "SemanticScholar", FakeS2) + context = ("Empirical scaling laws relate parameter count and pretraining token " + "budget to transformer language model performance.") + entry = report._from_s2("scaling laws", context) + assert entry is not None + assert entry["note"] == "S2:right" + + +def test_partial_usage_keeps_cache_counters(workspace, project): + """A turn that dies before the result message still spent cache traffic, and + a long prompt spends most of its tokens there.""" + import inspect + + source = inspect.getsource(report._generate_prose) + assert "cache_read_input_tokens" in source + assert "cache_creation_input_tokens" in source + + +def test_report_usage_is_charged_to_the_reported_project(workspace, monkeypatch): + """`--project` can name a project other than the selected one; charging the + report's tokens to whichever happened to be current attributes the spend to + the wrong allocation.""" + budget.create("proj-a", title="a", budget={}) + budget.create("proj-b", title="b", budget={}) + budget.set_current("proj-a") + + recorded: dict = {} + + def capture(stage, usage, **kw): + recorded.update(kw) + return None + + monkeypatch.setattr(report.quota_log, "from_sdk_usage", capture) + monkeypatch.setattr( + report, "_generate_prose", + lambda bundle, *, model, project=None: ( + capture("report.write", {}, project=project, model=model, role="report") + or r"\section{Results}" + "\nbody text long enough to pass validation." * 5 + ), + ) + + in_range_run("proj-b") + report.cmd_draft(args(project="proj-b")) + report.cmd_write(args(project="proj-b", section=[], dry_run=False)) + assert recorded["project"] == "proj-b" diff --git a/tools/jobs.py b/tools/jobs.py index 6e10bdf..1185bea 100644 --- a/tools/jobs.py +++ b/tools/jobs.py @@ -368,17 +368,26 @@ def run_smoke( caps = gates.check_smoke_caps(sub, cfg) flavor = sub.target.get("smoke_flavor") or sub.target.get("flavor") or cfg.get("hf", "default_flavor", "a10g-small") command = _smoke_command(sub, caps) + + # Resolved once, and used for both the namespace and the accounting. + # `preflight.py` calls this with no project at all, and deriving the + # namespace from the current project while booking the cost as `unassigned` + # meant a smoke run charged an org's job to nobody -- the two halves of the + # same decision disagreeing. + project = project or budget.current_project() if namespace is None: - namespace = resolve_namespace(None, sub, cfg, project or budget.current_project()) + namespace = resolve_namespace(None, sub, cfg, project) # Everything that can fail for a *configuration* reason resolves before the - # ledger record exists. `_hub()` raises when huggingface_hub is missing and - # `_ns_kwargs()` raises when the installed one cannot take a namespace; - # either one landing after `record_smoke_run` leaves a phantom in-flight - # estimate sitting on the monthly ceiling for a job that never reached the - # platform -- which then goes stale and blocks every later submission. + # ledger record exists. `_hub()` raises when huggingface_hub is missing, + # `_ns_kwargs()` raises when the installed one cannot take a namespace, and + # `_token()` raises when no credential is stored; any of them landing after + # `record_smoke_run` leaves a phantom in-flight estimate sitting on the + # monthly ceiling for a job that never reached the platform -- which then + # goes stale and blocks every later submission. hub = _hub() ns_kwargs = _ns_kwargs(namespace) + token = _token() run_id = submit_lib.record_smoke_run( sub, cfg=cfg, platform=PLATFORM, @@ -393,7 +402,7 @@ def run_smoke( command=command, flavor=flavor, env={**_job_env(sub), "GRAD_SMOKE": "1"}, - token=_token(), + token=token, timeout=caps["timeout_s"], **ns_kwargs, ) diff --git a/tools/report.py b/tools/report.py index c724732..e835dac 100644 --- a/tools/report.py +++ b/tools/report.py @@ -368,7 +368,7 @@ def cmd_write(args: argparse.Namespace) -> dict[str, Any]: return {"project": project_id, "bundle": bundle, "sent": False} cfg = config_mod.load() - prose = _generate_prose(bundle, model=cfg.model_for("report")) + prose = _generate_prose(bundle, model=cfg.model_for("report"), project=project_id) body = files["tex"].read_text(encoding="utf-8") marker = "\\maketitle" @@ -424,10 +424,15 @@ def _bundle(evidence: dict[str, Any], claims: dict[str, Any]) -> dict[str, Any]: } -def _generate_prose(bundle: dict[str, Any], *, model: str) -> str: +def _generate_prose(bundle: dict[str, Any], *, model: str, project: str | None = None) -> str: """One forced-tool call, for the same reason §5 uses one. "prompting for JSON and parsing it fails silently on the tenth call." + + `project` is threaded through rather than left to default: `--project` can + name a project other than the selected one, and charging this report's + tokens to whichever happened to be current would attribute the spend to the + wrong allocation. """ try: import asyncio # noqa: PLC0415 @@ -491,7 +496,15 @@ async def run() -> None: # fallback for the turn that dies before a result -- adding both would # double-count. final: Any = None - partial = {"input_tokens": 0, "output_tokens": 0} + # Every field `from_sdk_usage` knows how to read. Accumulating only the + # two uncached counters would silently drop cache traffic from the + # fallback, so a failed turn would under-report exactly the tokens a + # long prompt spends most of. + partial = dict.fromkeys( + ("input_tokens", "output_tokens", + "cache_read_input_tokens", "cache_creation_input_tokens"), + 0, + ) try: async for message in query( prompt="Evidence bundle:\n\n" + json.dumps(bundle, indent=2, default=str), @@ -504,14 +517,14 @@ async def run() -> None: final = usage else: get = usage.get if isinstance(usage, dict) else (lambda k, d=0: getattr(usage, k, d)) - partial["input_tokens"] += get("input_tokens", 0) or 0 - partial["output_tokens"] += get("output_tokens", 0) or 0 + for field in partial: + partial[field] += get(field, 0) or 0 finally: # In a finally block because a failed turn still spent quota, and an # unrecorded spend is exactly what §15's ceilings cannot see. quota_log.from_sdk_usage( "report.write", final if final is not None else partial, - model=model, role="report", + model=model, role="report", project=project, ) asyncio.run(run()) @@ -692,7 +705,12 @@ def _from_s2(keyword: str, context: str) -> dict[str, Any] | None: if not words: return None - best, best_score, best_title = None, 0.0, 0.0 + # Both gates are applied *before* ranking, not to the winner afterwards. + # Ranking first and then testing meant a loosely-related paper with a + # keyword-stuffed abstract could win on context overlap, fail the title + # gate, and take the genuinely correct paper down with it -- rejecting a + # citation that was right there in the candidate list. + qualifying = [] for hit in hits: body = _content_words(f"{hit.get('title', '')} {hit.get('abstract', '')}") title = _content_words(hit.get("title", "")) @@ -700,11 +718,12 @@ def _from_s2(keyword: str, context: str) -> dict[str, Any] | None: continue score = len(words & body) / len(words) title_score = (len(words & title) / len(title)) if title else 0.0 - if score > best_score: - best, best_score, best_title = hit, score, title_score + if score >= S2_MIN_CONTEXT_OVERLAP and title_score >= S2_MIN_TITLE_OVERLAP: + qualifying.append((score, title_score, hit)) - if best is None or best_score < S2_MIN_CONTEXT_OVERLAP or best_title < S2_MIN_TITLE_OVERLAP: + if not qualifying: return None + best_score, best_title, best = max(qualifying, key=lambda row: (row[0], row[1])) return { "key": _bib_key(best.get("paper_id", keyword), None, best.get("year")), "type": "article",