diff --git a/.claude/launch.json b/.claude/launch.json index 9fb6c8b..f945b51 100644 --- a/.claude/launch.json +++ b/.claude/launch.json @@ -6,6 +6,15 @@ "runtimeExecutable": ".venv/Scripts/python.exe", "runtimeArgs": ["-c", "from ui.app import run; run(native=False, port=8123)"], "port": 8123 + }, + { + "name": "grad-ui-scratch", + "runtimeExecutable": ".venv/Scripts/python.exe", + "runtimeArgs": [ + "-c", + "import os, tempfile, pathlib; root = pathlib.Path(tempfile.gettempdir()) / 'grad-scratch-ws'; root.mkdir(parents=True, exist_ok=True); os.environ['GRAD_ROOT'] = str(root); os.environ['GRAD_APP_DIR'] = str(root / 'appdata'); from ui.app import run; run(native=False, port=8124)" + ], + "port": 8124 } ] } diff --git a/README.md b/README.md index a85caf9..26eda31 100644 --- a/README.md +++ b/README.md @@ -179,10 +179,16 @@ Then authenticate against the subscription, not the API: claude setup-token ``` -Export the result as `CLAUDE_CODE_OAUTH_TOKEN` and make sure `ANTHROPIC_API_KEY` -is **not** set — it outranks the OAuth token in the credential chain and will -silently bill the Developer Platform instead. `python agent.py --check` removes -it from the process environment and reports what it removed. +Export the result as `CLAUDE_CODE_OAUTH_TOKEN`, **or** store it as +`claude_oauth_token` below and skip the export — the two are equivalent for the +agent's own loop, and the stored copy is the one that works from the desktop +shortcut, which inherits whatever Explorer had and usually that is nothing. An +exported token wins over a stored one, so a terminal that set one deliberately +keeps it. `python agent.py --check` reports which of the two it is using. + +Make sure `ANTHROPIC_API_KEY` is **not** set — it outranks the OAuth token in +the credential chain and will silently bill the Developer Platform instead. +`--check` removes it from the process environment and reports what it removed. Store credentials once; they never enter the agent's environment: @@ -219,12 +225,19 @@ directory rather than in `config/grad.toml`: that file is hand-annotated and reformat it and drop every comment in it. `[kaggle] username` is still read as a fallback, and `account` says when a stored selection is shadowing one. -Or store them from the app: the workspace menu (`project ▾`) has a credentials -panel, which is the same command with `--stdin` instead of the `getpass` prompt. -That exists because the prompt needs a terminal, and needing one for this was -the only thing that forced a shell open beside the app on a fresh machine. The -value goes down a pipe rather than in an argument — an argv is visible to -anything that can list processes. +Or store them from the app, which is the shorter route: the **setup** window +asks for the subscription token first, then which model runs which role, then +which backends this machine can reach — and it is the same commands underneath, +with `--stdin` instead of the `getpass` prompt. That prompt needs a terminal, and +needing one for this was the only thing that forced a shell open beside the app +on a fresh machine. The value goes down a pipe rather than in an argument — an +argv is visible to anything that can list processes. + +A workspace that has never been arranged and has no subscription token opens on +that window, because the four windows it would otherwise open are four windows +that cannot do anything. Nothing else forces it: an unconfigured backend means +no remote training, which is a real limitation and not a reason to put a wizard +in front of someone who opened the app to read a ledger. ## Update @@ -687,7 +700,8 @@ that carry the literal next command. | `tools/kaggle.py` | the same verbs on Kaggle's free GPU/TPU, plus `account` / `quota` / `accelerators` | | `tools/ledger.py` | `expect` / `query` / `verdict` / `falsify` / `abandon` / `verify` / `reindex` | | `tools/quota.py` | measured token and credit usage, summarised by stage, role, and project | -| `tools/budget.py` | projects and their ceilings: `new` / `use` / `status` / `raise` / `close` | +| `tools/budget.py` | projects, their ceilings and their own models: `new` / `use` / `status` / `raise` / `configure` / `close` | +| `tools/setup.py` | the writable half of the configuration: `show` / `models` / `backend` / `host` / `check` | | `tools/docs.py` | is this library call current? introspection first, then Context7 | | `tools/evolve.py` | evolutionary search as a budgeted campaign, over our own operator | | `tools/task.py` | run a CLI in the background: `start` / `list` / `status` / `output` / `wait` / `stop` | @@ -790,7 +804,8 @@ core/ the machinery the CLIs share, so no tool can forget a rule traces.py a session as tags a later query can slice on -- pure, tested submission.py the resolved submission and its hash gates.py the submit gates and the smoke carve-out - budget.py the project dimension and its three ceilings + budget.py the project dimension, its ceilings and its own models + settings.py the writable overlay: what setup may change, and what it shadows kaggle_quota.py the weekly accelerator allowance, folded like rolling spend ledger_store.py event-folded runs, rolling spend, staleness, derived index submit.py shared submitter machinery: record, collect, deviations @@ -895,13 +910,37 @@ unrecognised one still raises rather than returning an empty list. on that rail spends subscription quota no ledger here can see — which is a reason not to use it whether or not it works. +- **Phase 2 of the campaign loop (remote evaluation) is enabled, on all three + backends.** The gate was proven locally first, which is what made it safe: + `--remote {ssh|hf_jobs|kaggle} --remote-spec ` evaluates every candidate + on real hardware and refuses unless that spec's preflight is complete and + passing *including the smoke run*. The required checks are named in + `tools/evolve.py` rather than read from `[preflight] checks`, so a machine + configured without `smoke` cannot let a loop with no human in it put forty + candidates on hardware nothing has ever run one step on. A candidate still + never becomes a run — the campaign remains the ledgered unit and its + expectation the bound prediction. + + The loop is local; the compute is not. A candidate changes an architecture or + an optimiser, so evaluating one is a training run — which is why each backend + gets a fresh remote per candidate and why every adapter bounds the work *where + it runs* rather than only where it is watched. The three differ in how the + mutated program gets there: `scp` to a host that stays up, a swapped file + inside Kaggle's embedded notebook payload, or a gzipped tar in an environment + variable for HF Jobs, whose pipeline lives in the image and has no upload step. + + Kaggle carries a second gate, because the dollar gate cannot see it: that + backend rations *hours*, so a campaign priced at zero would otherwise pass the + budget check and spend the week. `core/kaggle_quota.py` now folds candidate + rows beside runs, closing a hole exactly the size of a campaign. + **Still open:** - **Historical records are left as `"unassigned"`** rather than retrofitted with a project. Cheap to change while the ledger is small. -- **Phase 2 of the campaign loop (remote evaluation) is not enabled.** - `--remote` is refused: the gate is proven locally first, because doing the - ledger work and the spend work simultaneously against live GPU jobs is how you - learn about exit 7 the hard way. +- **Nothing has run a remote campaign against live hardware yet.** The gate, the + driver and the records are covered by `tests/test_evolve_remote.py` with the + ssh side stubbed; the first real campaign should be one generation of two + candidates on a host you can watch. **One correction to HANDOFF-2 itself.** §20 records `repowiki map` as taking `--format html --open`. The 0.3.1 wheel's `map` takes exactly one `path`, diff --git a/agent.py b/agent.py index 428a606..158f236 100644 --- a/agent.py +++ b/agent.py @@ -211,15 +211,30 @@ def preflight_environment() -> dict[str, Any]: ANTHROPIC_API_KEY outranks CLAUDE_CODE_OAUTH_TOKEN in the credential chain, so a stray export silently bills the Developer Platform instead of the subscription. It is removed here rather than warned about. + + Then the complement, and the order between the two is the point: the scrub + takes out what must not be there, and `hydrate_environment` supplies the one + thing that must -- the subscription token, from the credential store, for + the app that was launched from a shortcut and never saw an `export`. Both + entry points reach this function (`run_session` and `ui/app.py`'s client + start), which is why the bridge belongs here rather than in either one. """ from core import budget # noqa: PLC0415 removed = credentials.scrub_environment() + # Read before hydrating, so the report can distinguish a token someone + # exported from one this just fetched. They authenticate identically; they + # are very different answers to "why is it using that account?". + ambient = bool(os.environ.get("CLAUDE_CODE_OAUTH_TOKEN")) + hydrated = credentials.hydrate_environment() cfg = config_mod.load() project_id = budget.current_project() return { "removed_env": removed, "oauth_token_present": bool(os.environ.get("CLAUDE_CODE_OAUTH_TOKEN")), + "oauth_token_source": ( + "environment" if ambient else ("credential store" if hydrated else "absent") + ), "workspace": str(paths.root()), "models": cfg.models(), # Read from ledger/.current_project, not from the environment -- the @@ -1025,6 +1040,11 @@ def main() -> None: default=None, help="pin the --ui port; by default 8080, or the next free port above it", ) + parser.add_argument( + "--no-splash", + action="store_true", + help="do not show the loading mark while --ui starts", + ) parser.add_argument("--check", action="store_true", help="report environment and auth posture, then exit") parser.add_argument( "--update", @@ -1076,6 +1096,16 @@ def main() -> None: "`python` process holding it and start again." ) from None + # After the instance check and before the first expensive import, which + # is the only window where this is both correct and useful: a second + # launch that handed over above has nothing to load and must flash + # nothing, and everything below this line is the wait being covered. + # `ui/app.py` takes it down when the workspace's first client connects. + if not args.no_splash: + from ui import splash # noqa: PLC0415 - a docstring-only package + + splash.start() + from ui.app import run as run_ui # noqa: PLC0415 # A non-default port also moves the app's origin, and the embedded Lab diff --git a/core/budget.py b/core/budget.py index 3a8b07c..22aa8ac 100644 --- a/core/budget.py +++ b/core/budget.py @@ -42,6 +42,15 @@ T_PROJECT = "project" T_PROJECT_RAISED = "project_budget_raised" T_PROJECT_CLOSED = "project_closed" +#: A project choosing its own models, or the backend it reaches for. +#: +#: An event rather than a field, for the same reason a raise is one: the model a +#: campaign ran under is part of what produced its numbers, and "which model was +#: this project on in March" is a question the ledger should be able to answer +#: rather than one that only the current value can be guessed from. It also +#: means changing a model is recorded beside the ceiling changes, in one +#: append-only file, in order. +T_PROJECT_CONFIGURED = "project_configured" # Records that predate the dimension fold as this, per §23 item 6: specified as # left alone rather than retrofitted, and cheap to change while the ledger is @@ -160,6 +169,11 @@ def projects() -> dict[str, dict[str, Any]]: "budget": dict(rec.get("budget") or {}), "status": rec.get("status", "open"), "raises": [], + # What this project overrides about how it is run. Empty is the + # common case and means "whatever the workspace says". + "models": {}, + "backend": None, + "configured": [], } elif pid in folded and kind == T_PROJECT_RAISED: node = folded[pid] @@ -168,6 +182,27 @@ def projects() -> dict[str, dict[str, Any]]: node["raises"].append( {"at": rec.get("at"), "budget": rec.get("budget"), "reason": rec.get("reason")} ) + elif pid in folded and kind == T_PROJECT_CONFIGURED: + node = folded[pid] + # `None` clears rather than sets. Folding it as a value would leave a + # role overridden to nothing, which resolves as falsy everywhere and + # is therefore an override that is present, wrong and invisible -- + # `settings.clear_models` refuses the same shape for the same reason. + for role, value in (rec.get("models") or {}).items(): + if value is None: + node["models"].pop(role, None) + else: + node["models"][role] = str(value) + if "backend" in rec: + node["backend"] = rec["backend"] + node["configured"].append( + { + "at": rec.get("at"), + "models": rec.get("models"), + "backend": rec.get("backend"), + "reason": rec.get("reason"), + } + ) elif pid in folded and kind == T_PROJECT_CLOSED: folded[pid]["status"] = "closed" folded[pid]["closed_at"] = rec.get("at") @@ -262,6 +297,102 @@ def raise_ceiling(project_id: str, *, budget: dict[str, float], reason: str = "" return record +def configure( + project_id: str, + *, + models: dict[str, str | None] | None = None, + backend: str | None = None, + reason: str = "", +) -> dict[str, Any]: + """Record what this project overrides about how it is run. + + Appends, like `raise_ceiling`, and for a stronger reason than tidiness: the + model a candidate was mutated by is part of what produced the numbers in the + ledger beside it. A field that could be edited in place would let a project's + history claim it had always been on the model it is on today. + + `None` for a role clears it. The vocabulary is checked here rather than + trusted from the caller, because an unknown role stored in the ledger is a + setting nothing will ever read and nothing will ever mention again. + """ + from core import config as config_mod, settings as settings_mod # noqa: PLC0415 + + project(project_id) # refuses an unknown id before anything is appended + changed = dict(models or {}) + unknown = [r for r in changed if r not in config_mod.MODEL_ROLES] + if unknown: + raise UsageError( + f"unknown model role(s): {', '.join(sorted(unknown))}", + fix=f"roles are: {', '.join(config_mod.MODEL_ROLES)}", + ) + for role, value in changed.items(): + if value is not None and not str(value).strip(): + raise UsageError( + f"model for role {role!r} is empty", + fix=f"pass a model id, or clear the override for {role}", + ) + if backend is not None and backend not in settings_mod.BACKENDS: + raise UsageError( + f"unknown backend {backend!r}", + fix=f"backends are: {', '.join(settings_mod.BACKENDS)}", + ) + if not changed and backend is None: + raise UsageError( + "nothing to configure", + fix="pass a model for a role, or a backend", + ) + record: dict[str, Any] = { + "type": T_PROJECT_CONFIGURED, + "id": project_id, + "at": now_iso(), + "models": {r: (None if v is None else str(v).strip()) for r, v in changed.items()}, + "reason": reason, + } + if backend is not None: + record["backend"] = backend + jsonl.append(projects_path(), record) + return record + + +def project_overrides(project_id: str | None) -> dict[str, Any]: + """What one project overrides, or empty. Never raises. + + Read on the config path, so an unknown id, an unreadable ledger or no + selection at all has to mean "nothing overridden" rather than an exception + from inside `config.load()` -- which every surface in the app calls, most of + them while rendering. + """ + if not project_id: + return {"models": {}, "backend": None} + try: + found = projects().get(project_id) + except Exception: # noqa: BLE001 - see the docstring + return {"models": {}, "backend": None} + if not found: + return {"models": {}, "backend": None} + return { + "models": dict(found.get("models") or {}), + "backend": found.get("backend"), + } + + +def selection_stamp() -> tuple[int, int]: + """A cheap marker for "the project layer may have changed". + + `core/config.py` folds this into its cache key: which project is selected, + and whether the ledger that project's overrides live in has been written. + Two `stat` calls, and they are what make `budget configure` in a child + process visible to a parent that has already loaded a `Config`. + """ + out = [] + for path in (current_project_path(), projects_path()): + try: + out.append(path.stat().st_mtime_ns) + except OSError: + out.append(0) + return (out[0], out[1]) + + def close(project_id: str) -> dict[str, Any]: project(project_id) record = {"type": T_PROJECT_CLOSED, "id": project_id, "at": now_iso()} diff --git a/core/config.py b/core/config.py index eeccd9f..214140d 100644 --- a/core/config.py +++ b/core/config.py @@ -421,6 +421,18 @@ class Config: # `model_for` needs the difference: an explicit [models] entry must beat a # legacy key, while the [models] *default* must not. user: dict[str, Any] = field(default_factory=dict) + # `core/settings.py`: the writable half, chosen through the setup wizard and + # stored under the app directory rather than in the hand-annotated TOML. + # Outranks everything in the file, which is `kaggle account`'s rule and for + # the same reason -- a command that silently did nothing because a config + # file disagreed would be worse than one that overrides it, and + # `settings.shadowing` is what stops that being a surprise. + overlay: dict[str, Any] = field(default_factory=dict) + # The selected project's own overrides (`core/budget.py:configure`). Kept + # apart from `overlay` rather than merged into it, because "what would this + # role be without the project" is a question the projects window has to + # answer -- it draws every project, and only one of them is selected. + project_overlay: dict[str, Any] = field(default_factory=dict) def section(self, name: str) -> dict[str, Any]: return dict(self.raw.get(name, {})) @@ -428,19 +440,34 @@ def section(self, name: str) -> dict[str, Any]: def get(self, section: str, key: str, default: Any = None) -> Any: return self.raw.get(section, {}).get(key, default) - def model_for(self, role: str) -> str: + def model_for(self, role: str, *, project: bool = True) -> str: """The model for one role (HANDOFF-2 §16). - Resolution: an explicit `[models] ` wins; then the legacy key the - role replaced (`[agent] model`, `[retrieval] expand_model` / - `triage_model`), readable "for one release so existing configs do not - break"; then the `[models]` default. + Resolution, outermost first: the selected project's own override + (`core/budget.py:configure`); then a `core/settings.py` overlay entry + (what the setup window chose); then an explicit `[models] `; then + the legacy key the role replaced (`[agent] model`, + `[retrieval] expand_model` / `triage_model`), readable "for one release + so existing configs do not break"; then the `[models]` default. + + `project=False` answers the same question with the outermost layer + removed -- "what would this be if the project said nothing". The + projects window needs it because it draws every project and only one of + them is selected, so for the rest the project layer in effect is not + theirs. """ if role not in DEFAULTS["models"]: raise ConfigError( f"unknown model role {role!r}", fix=f"roles are: {', '.join(MODEL_ROLES)}", ) + if project: + chosen = (self.project_overlay.get("models") or {}).get(role) + if chosen: + return str(chosen) + chosen = (self.overlay.get("models") or {}).get(role) + if chosen: + return str(chosen) explicit = (self.user.get("models") or {}).get(role) if explicit: return str(explicit) @@ -469,6 +496,22 @@ def hosts(self) -> dict[str, Host]: f"[hosts] must be a table of host entries, not {type(raw).__name__}", fix=f"see the [hosts.*] example in {paths.config_path()}", ) + # The inventory has two sources and stays fixed: `[hosts.*]` in the TOML, + # and whatever `setup host add` wrote. Combined by name, so a machine + # that had hosts in its config keeps them, and validated below by the + # same code either way -- a host added through the wizard is not a host + # that skipped the rate check. + # + # **Whole entries, not `_merge`.** A recursive merge would have an + # overlay host inherit the fields it omitted from the config host of the + # same name -- including `key_credential`, which names the keyring entry + # that authenticates the connection. Replacing `gpu-box` through the + # wizard and getting the old box's credential, user and workdir attached + # to the new hostname is a connection nobody described, and it is the one + # field in this table where being wrong reaches a machine. + overlay_hosts = self.overlay.get("hosts") + if isinstance(overlay_hosts, dict) and overlay_hosts: + raw = {**raw, **{k: v for k, v in overlay_hosts.items() if isinstance(v, dict)}} out: dict[str, Host] = {} for name, spec in raw.items(): if not isinstance(spec, dict): @@ -525,7 +568,13 @@ def host(self, name: str) -> Host: known = ", ".join(sorted(hosts)) or "(none configured)" raise ConfigError( f"unknown host {name!r}; the inventory is fixed. known hosts: {known}", - fix=f"add a [hosts.{name}] block to {paths.config_path()}", + # Both halves named, because there are now two places a host can + # be defined and the wrong guess costs an edit to a file that was + # never going to be read. + fix=( + f"python -m tools.setup host add --name {name} --hostname … --user … --json" + f" # or add a [hosts.{name}] block to {paths.config_path()}" + ), ) return hosts[name] @@ -566,8 +615,18 @@ def _merge(base: dict[str, Any], over: dict[str, Any]) -> dict[str, Any]: def load(path: Path | None = None, *, reload: bool = False) -> Config: + from core import budget as budget_mod, settings as settings_mod # noqa: PLC0415 + path = Path(path) if path else paths.config_path() - key = str(path) + # The mtimes are part of the key, not just the path. `setup models` and + # `budget configure` run as child processes and write files this process has + # already read -- so without this the app goes on serving whatever it loaded + # at startup and the setup window appears to do nothing. Three `stat` calls + # and one tiny read per load, against a config that is read on the gate + # path; the alternative is every writer remembering to clear a cache in a + # module it does not import. + project_id = budget_mod.current_project() + key = f"{path}\x00{settings_mod.stamp()}\x00{project_id}\x00{budget_mod.selection_stamp()}" if not reload and key in _cache: return _cache[key] user: dict[str, Any] = {} @@ -579,7 +638,12 @@ def load(path: Path | None = None, *, reload: bool = False) -> Config: f"{path} is not valid TOML: {exc}", fix=f"fix the syntax in {path}, or delete it to fall back to defaults", ) from exc - cfg = Config(raw=_merge(DEFAULTS, user), user=user) + cfg = Config( + raw=_merge(DEFAULTS, user), + user=user, + overlay=settings_mod.load(), + project_overlay=budget_mod.project_overrides(project_id), + ) _validate(cfg, path) _cache[key] = cfg return cfg diff --git a/core/credentials.py b/core/credentials.py index ffb05ce..d37c2e0 100644 --- a/core/credentials.py +++ b/core/credentials.py @@ -185,6 +185,56 @@ def sdk_env() -> dict[str, str]: return {"CLAUDE_CODE_OAUTH_TOKEN": token} +def hydrate_environment() -> str | None: + """Put the stored subscription token into *this* process's environment. + + The complement of `scrub_environment`, and the fix for a gap between two + things that were each individually right. The main loop authenticates from + the ambient `CLAUDE_CODE_OAUTH_TOKEN` (`agent.preflight_environment`), while + `sdk_env` reads the credential store because the Bash hop strips that + variable from child processes. Nothing joined them up -- so a token stored + through the app's credentials panel authenticated the funnel and the + mutation operator, and left the agent's own loop with no credentials at all. + That is the worst shape for this to fail in: the panel says STORED, and the + first turn fails anyway. + + It bit the installed app rather than the terminal, which is why it survived. + `claude setup-token` is run in a shell, and a shell that exported the result + has it -- but the desktop shortcut launches `pythonw.exe` from Explorer, + which inherits whatever the user made persistent and usually that is + nothing. + + Three constraints, all of them load-bearing: + + **Ambient wins.** A token exported in a terminal is a deliberate choice -- + a second account, a token being tested -- and this must not quietly outrank + it. Same precedence `sdk_env` uses, for the same reason. + + **After the scrub, never before.** `ANTHROPIC_API_KEY` outranks this token + in the SDK's credential chain, so hydrating first would set a variable the + scrub had not yet cleared the way for, and the session would bill the + Developer Platform while reporting a subscription token present. + + **It never raises.** A missing `keyring`, a Credential Manager that will not + open for this user -- neither is a reason to refuse to start. The caller + reports what happened; the SDK still gives its own auth error if the token + was the only thing that could have helped. + """ + if os.environ.get("CLAUDE_CODE_OAUTH_TOKEN"): + return None + try: + token = get(CLAUDE_TOKEN, required=False) + except ConfigError: + # The store is unreachable. `status()` already reports that condition to + # the credentials panel, and there is nothing useful to do about it on + # the startup path. + return None + if not token: + return None + os.environ["CLAUDE_CODE_OAUTH_TOKEN"] = token + return "CLAUDE_CODE_OAUTH_TOKEN" + + def scrub_environment() -> list[str]: """Remove credential-shaped variables from the agent's own environment. diff --git a/core/kaggle_quota.py b/core/kaggle_quota.py index 102aad0..2a9a1f8 100644 --- a/core/kaggle_quota.py +++ b/core/kaggle_quota.py @@ -112,6 +112,69 @@ def hours_for_quota(run: ls.Run) -> float: return 0.0 +def _fold_candidates( + pools: dict[str, dict[str, Any]], *, cutoff: _dt.datetime, kind: str | None +) -> None: + """Evolve candidates evaluated on Kaggle, folded in beside the runs. + + **Without this the weekly allowance has a hole exactly the size of a + campaign.** Candidates deliberately never enter `runs.jsonl` -- §23 item 4, + so a hundred-generation search cannot dominate a ledger read by hand -- and + the quota fold reads runs. A remote campaign would therefore burn real GPU + hours that this function could not see, and the first thing to notice would + be an ordinary submission refused for an allowance something else had + already spent. + + Dollars do not have this problem: `core/budget.py` already reaches into + `candidates.jsonl` for exactly this reason. Hours needed the same treatment + and did not have it, because until now nothing could spend them from here. + + Always an estimate. A candidate records the wall clock its evaluation took, + which is a measurement -- but it is measured by us rather than reported by + Kaggle, and the run path's `actual` means "what the platform's own log said". + Counting ours as `in_flight` keeps that distinction honest and errs toward + the allowance being spent rather than free. + """ + from core import campaign as camp # noqa: PLC0415 - import cycle if hoisted + + try: + rows = camp.candidates() + except Exception: # noqa: BLE001 - a gate must not fail on an unreadable file + return + for row in rows: + if str(row.get("backend") or "") != PLATFORM: + continue + row_kind = str(row.get(F_KIND) or "") + if not row_kind or row_kind == UNMETERED: + continue + if kind and row_kind != kind: + continue + at = ls.parse_iso(row.get("at")) + if at and at < cutoff: + continue + try: + hours = max(0.0, float(row.get(F_ACTUAL) or row.get(F_ESTIMATE) or 0.0)) + except (TypeError, ValueError): + continue + if hours <= 0: + continue + pool = pools.setdefault( + row_kind, + {"kind": row_kind, "actual_hours": 0.0, "in_flight_hours": 0.0, "runs": []}, + ) + pool["in_flight_hours"] += hours + pool["runs"].append( + { + "run_id": row.get("candidate_id"), + "hours": round(hours, 4), + "basis": "estimate", + "accelerator": row.get(F_ACCELERATOR), + "smoke": False, + "campaign": row.get("campaign"), + } + ) + + def accelerator_hours( *, window_days: int = 7, now: _dt.datetime | None = None, kind: str | None = None ) -> dict[str, Any]: @@ -124,6 +187,7 @@ def accelerator_hours( now = now or _dt.datetime.now(_dt.timezone.utc) cutoff = now - _dt.timedelta(days=window_days) pools: dict[str, dict[str, Any]] = {} + _fold_candidates(pools, cutoff=cutoff, kind=kind) for r in ls.runs(): if r.get("platform") != PLATFORM: continue diff --git a/core/settings.py b/core/settings.py new file mode 100644 index 0000000..7810ff3 --- /dev/null +++ b/core/settings.py @@ -0,0 +1,328 @@ +"""The writable settings overlay: what a wizard is allowed to change. + +`config/grad.toml` is hand-annotated, and every value in it is annotated because +the reason for the value is worth more than the value. `tomllib` reads TOML and +does not write it, so a command that edited that file would reformat it and drop +every comment in it -- which is why nothing in this project ever has. The README +says so plainly, and `tools/kaggle.py` already solved the problem once: `account +--set` writes its choice to a file under the app directory, that choice *wins* +over `[kaggle] username`, and the command says when it is shadowing one. + +This is that mechanism, generalised, so an interactive setup can answer "which +model for which role", "which backend by default" and "which SSH hosts exist" +without touching a line anybody wrote. + +**Per workspace, not per machine.** `paths.config_path()` is +`_shipped("config", "grad.toml")` -- the workspace's copy when it has one, the +installation's otherwise -- so `grad.toml` is *already* overridable per +workspace. A machine-global overlay would silently flatten two workspaces that +had deliberately different model choices. An overlay has to mirror the scope of +the file it shadows, so this lives in `appdata.workspace_state_dir()`, keyed by +root, beside the window layouts. + +It stays out of the workspace *folder* for the reason `core/workspace.py` keeps +the root pointer beside the code: these are answers about how this install is +wired up, and a research folder handed to a colleague should not carry this +machine's SSH inventory with it. + +**What is not here.** Credentials: those are `core/credentials.py` and the +operating system's store, and a token in a JSON file under the app directory +would be exactly the environment-resident secret §9 argues against. Ceilings: +those are `core/budget.py`, append-only, because a ceiling that moved is an +event and not a setting. +""" + +from __future__ import annotations + +import datetime as _dt +import math +from pathlib import Path +from typing import Any + +from core import appdata, jsonl +from core.errors import UsageError + +#: The schema version, for a migration that has not been needed yet. Written so +#: that the first one does not have to guess what it is reading. +VERSION = 1 + +#: Where a candidate can be evaluated. The same three names `tools/evolve.py` +#: uses for `--remote` and the run records use for `platform` -- one vocabulary, +#: and `tests/test_settings.py` asserts the two lists still agree. +#: +#: In `core/` rather than in the tool, because a *setting* naming a backend is +#: read by the config layer, and `core` importing `tools` is backwards. +BACKENDS: tuple[str, ...] = ("ssh", "hf_jobs", "kaggle") + +#: The models the setup window offers as buttons. **Not a restriction.** +#: `set_models` takes any non-empty string, because a hardcoded list of model ids +#: in a UI ages the moment a new one ships -- and the one thing worse than not +#: offering the newest model is refusing it. These are the shortcut; the text +#: field beside them is the mechanism. +#: +#: The Claude 5 family is Fable 5 / Opus 5 / Sonnet 5. There is no Haiku 5, and +#: 4.5 is the latest Haiku -- the same note `config/grad.toml` carries, because +#: it is the thing people get wrong. +KNOWN_MODELS: tuple[str, ...] = ( + "claude-opus-5", + "claude-sonnet-5", + "claude-fable-5", + "claude-haiku-4-5", +) + +#: The keys an SSH host entry may carry, matching `config.Host`'s fields. A host +#: added here is merged into the `[hosts.*]` inventory, which is fixed by design +#: (`core/config.py:host`) -- this gives that inventory a second, writable +#: source; it does not make a connection ad-hoc. +HOST_FIELDS: tuple[str, ...] = ( + "hostname", + "user", + "rate_usd_per_hour", + "workdir", + "key_credential", + "gpus", + "notes", +) + + +def path(root: Path | None = None) -> Path: + return appdata.workspace_state_dir(root) / "settings.json" + + +def stamp(root: Path | None = None) -> int: + """A cheap marker that changes when this file does. + + `core/config.py` folds this into its cache key, so a wizard writing here is + picked up by a process that has already loaded a `Config` -- without which + the app would go on serving the models it read at startup and the setup + window would appear to do nothing. + + Never raises: an unreadable app directory is not a reason to refuse to load + a config, and the fallback -- treat it as absent -- is the same answer as a + machine with nothing configured. + """ + try: + return path(root).stat().st_mtime_ns + except OSError: + return 0 + + +def load(root: Path | None = None) -> dict[str, Any]: + """The overlay, or an empty one. Never raises and never partially applies. + + `jsonl.read_json` returns None for a file that will not parse, and that is + the right behaviour here rather than an error: a corrupt overlay should + leave the app running on `grad.toml`, which is a working configuration, not + stop it from starting. + """ + raw = jsonl.read_json(path(root)) + return raw if isinstance(raw, dict) else {} + + +def _write(document: dict[str, Any], root: Path | None = None) -> dict[str, Any]: + document["version"] = VERSION + document["set_at"] = _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds") + jsonl.write_json(path(root), document) + return document + + +# --------------------------------------------------------------------------- +# models +# --------------------------------------------------------------------------- +def models(root: Path | None = None) -> dict[str, str]: + """Only the roles that were actually chosen. A role absent here falls + through to `grad.toml` and then to the shipped default, which is what makes + this an overlay rather than a replacement.""" + chosen = load(root).get("models") + if not isinstance(chosen, dict): + return {} + return {str(k): str(v) for k, v in chosen.items() if v} + + +def _check_roles(roles: dict[str, str]) -> dict[str, str]: + from core import config as config_mod # noqa: PLC0415 - `config` reads this module + + unknown = [r for r in roles if r not in config_mod.MODEL_ROLES] + if unknown: + raise UsageError( + f"unknown model role(s): {', '.join(sorted(unknown))}", + fix=f"roles are: {', '.join(config_mod.MODEL_ROLES)}", + ) + cleaned = {} + for role, value in roles.items(): + text = str(value or "").strip() + if not text: + raise UsageError( + f"model for role {role!r} is empty", + fix=f"pass a model id, or `setup models --clear {role}` to fall back to the config", + ) + cleaned[role] = text + return cleaned + + +def set_models(roles: dict[str, str], root: Path | None = None) -> dict[str, Any]: + document = load(root) + current = dict(document.get("models") or {}) + current.update(_check_roles(roles)) + document["models"] = current + return _write(document, root) + + +def clear_models(roles: list[str], root: Path | None = None) -> dict[str, Any]: + """Drop an override, so the role falls back through the layers again. + + Retiring by making optional rather than by deleting: a role that was set + here and is now cleared resolves exactly as it did before anyone opened the + wizard. + """ + from core import config as config_mod # noqa: PLC0415 + + unknown = [r for r in roles if r not in config_mod.MODEL_ROLES] + if unknown: + raise UsageError( + f"unknown model role(s): {', '.join(sorted(unknown))}", + fix=f"roles are: {', '.join(config_mod.MODEL_ROLES)}", + ) + document = load(root) + current = dict(document.get("models") or {}) + for role in roles: + current.pop(role, None) + document["models"] = current + return _write(document, root) + + +# --------------------------------------------------------------------------- +# backend +# --------------------------------------------------------------------------- +def default_backend(root: Path | None = None) -> str | None: + value = str((load(root).get("backend") or {}).get("default") or "").strip() + return value or None + + +def set_backend(name: str, root: Path | None = None) -> dict[str, Any]: + chosen = str(name or "").strip() + if chosen not in BACKENDS: + raise UsageError( + f"unknown backend {name!r}", + fix=f"backends are: {', '.join(BACKENDS)}", + ) + document = load(root) + document["backend"] = {"default": chosen} + return _write(document, root) + + +# --------------------------------------------------------------------------- +# ssh hosts +# --------------------------------------------------------------------------- +def hosts(root: Path | None = None) -> dict[str, dict[str, Any]]: + stored = load(root).get("hosts") + if not isinstance(stored, dict): + return {} + return {str(k): dict(v) for k, v in stored.items() if isinstance(v, dict)} + + +def add_host(name: str, spec: dict[str, Any], root: Path | None = None) -> dict[str, Any]: + """Add or replace one host in the writable half of the inventory. + + The name is constrained the way a project id is, and for the same reason: + it is used to look a host up and it ends up in a command line. A host called + `-oProxyCommand=...` is not a naming problem. + """ + chosen = str(name or "").strip() + if not chosen or any(c.isspace() for c in chosen) or chosen.startswith("-"): + raise UsageError( + f"{name!r} is not a usable host name: no spaces, and it cannot start with a dash", + fix="python -m tools.setup host add --name gpu-box --hostname … --user … --json", + ) + unknown = [k for k in spec if k not in HOST_FIELDS] + if unknown: + raise UsageError( + f"unknown host field(s): {', '.join(sorted(unknown))}", + fix=f"fields are: {', '.join(HOST_FIELDS)}", + ) + if not str(spec.get("hostname") or "").strip(): + raise UsageError( + f"host {chosen!r} needs a hostname", + fix="--hostname is what ssh connects to; --name is what Grad calls it", + ) + rate = spec.get("rate_usd_per_hour", 0.0) + try: + rate = float(rate) + except (TypeError, ValueError): + raise UsageError( + f"host {chosen!r} has a malformed rate_usd_per_hour: {rate!r}", + fix="a number; use 0 for a host that is free to use", + ) from None + if not math.isfinite(rate): + # `nan` fails every comparison a gate makes against it and `inf` is a + # price no run can be under, so both are ceilings that stop bounding + # anything. `core/config.py` refuses them on the TOML side and this is + # the writable side of the same inventory -- a check that exists in only + # one of two entry points is a check with a way around it. + raise UsageError( + f"host {chosen!r} has a non-finite rate_usd_per_hour ({rate})", + fix="rate_usd_per_hour must be a finite number; use 0 for a free host", + ) + if rate < 0: + # `collect` prices wall clock against this, and a negative rate books + # negative actuals -- which *reduce* rolling spend. A typo that raises + # the ceiling is the one shape of error worth refusing here rather than + # at the point of accounting. + raise UsageError( + f"host {chosen!r} has a negative rate_usd_per_hour ({rate})", + fix="use 0 for a host that is free to use; negative spend is not a thing", + ) + document = load(root) + inventory = dict(document.get("hosts") or {}) + entry = {k: v for k, v in spec.items() if v is not None} + entry["rate_usd_per_hour"] = rate + inventory[chosen] = entry + document["hosts"] = inventory + return _write(document, root) + + +def remove_host(name: str, root: Path | None = None) -> dict[str, Any]: + document = load(root) + inventory = dict(document.get("hosts") or {}) + if name not in inventory: + raise UsageError( + f"no host {name!r} was added here", + fix="python -m tools.setup show --json # lists both halves of the inventory", + ) + inventory.pop(name) + document["hosts"] = inventory + return _write(document, root) + + +# --------------------------------------------------------------------------- +# what this is overriding +# --------------------------------------------------------------------------- +def shadowing(cfg: Any, root: Path | None = None) -> list[dict[str, Any]]: + """Every value here that is overriding one in `grad.toml`. + + This is what keeps the whole arrangement honest. Someone edits `[models] + evolve`, sees no change, and has no way to discover that a file they have + never heard of outranks the file they were told to edit. `kaggle account` + reports the same thing for the same reason, and the report is the price of + being allowed to win. + + Only genuine conflicts: an overlay value that matches the config, or one for + a key the config never set, is not shadowing anything. + """ + out: list[dict[str, Any]] = [] + configured_models = (getattr(cfg, "user", None) or {}).get("models") or {} + for role, value in models(root).items(): + configured = configured_models.get(role) + if configured and str(configured) != value: + out.append( + { + "what": f"[models] {role}", + "config": str(configured), + "overlay": value, + } + ) + configured_hosts = (getattr(cfg, "raw", None) or {}).get("hosts") or {} + for name in hosts(root): + if name in configured_hosts: + out.append({"what": f"[hosts.{name}]", "config": "defined", "overlay": "replaced"}) + return out diff --git a/core/spawn.py b/core/spawn.py index e63cd2c..00ae2ca 100644 --- a/core/spawn.py +++ b/core/spawn.py @@ -85,6 +85,41 @@ def run(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess: return subprocess.run(argv, **{**quiet(), **kwargs}) +def console_script(name: str) -> str | None: + """Find a console script installed *beside this interpreter*, then on PATH. + + `shutil.which` alone was wrong here, and wrong in the way that is hardest to + argue with: it searches `PATH`, and a virtualenv's `Scripts` directory is on + `PATH` only while the environment is *activated*. Grad is launched from a + desktop shortcut pointing at `.venv\\Scripts\\pythonw.exe`, and Explorer + starts it with the ambient environment -- so the interpreter is the venv's + and `PATH` is the machine's. + + That produced both halves of one bug report. `repowiki` was installed in the + venv, `which` did not find it, and `tools/wiki.py` reported "repowiki is not + installed" with a `pip install -e '.[wiki]'` that had already been run. And + `kaggle` was found -- in the *user-site* Python, not the venv -- so the wiki + said a package was missing while Kaggle silently shelled out to a different + installation's CLI, against a version this project pins. + + Beside the interpreter first, therefore, because `sys.executable` is the one + thing that is always right about which environment this is. PATH stays as + the fallback for a tool that genuinely lives elsewhere. + """ + import shutil # noqa: PLC0415 - only this function needs it + import sys # noqa: PLC0415 + + scripts = os.path.dirname(sys.executable) + if scripts: + # `shutil.which` with an explicit `path`, rather than joining a name and + # testing it: on Windows the extension is PATHEXT's business, and + # `repowiki` on disk is `repowiki.exe`. + found = shutil.which(name, path=scripts) + if found: + return found + return shutil.which(name) + + _sdk_masked = False diff --git a/core/wakeups.py b/core/wakeups.py new file mode 100644 index 0000000..ab37bf9 --- /dev/null +++ b/core/wakeups.py @@ -0,0 +1,610 @@ +"""Conditions the agent asks to be woken on, and the record of them. + + "`collect` is non-blocking by default: a two-hour poll inside the agent's + only shell is a tool timeout waiting to happen." + +That line has been in `tools/jobs.py` since remote jobs existed, and it names a +problem it does not solve. The agent has exactly one shell per turn, so waiting +for anything means either holding that shell -- which is a tool timeout, and a +turn that can do nothing else while it waits -- or coming back to look, over and +over, with a `sleep` between the looks. The second is what actually happened, and +it is worse than it sounds: every look is a turn, every turn is tokens, and the +sleeps have to grow or the polling costs more than the job. A four-hour training +run answered by `sleep 30`, `sleep 60`, `sleep 120` is a conversation whose +content is mostly the agent waiting. + +`core/tasks.py` fixed the half of this that is *starting* things without waiting. +This is the other half: **being told**. The agent arms a condition and ends its +turn. A detached watcher polls out of process -- no shell held, no tokens, no +model in the loop -- and when the condition is met it wakes the session with a +turn describing what happened. + +**A wake is a model call, so it is metered like one.** The turn a wake issues +goes through `agent.drive_turn` exactly as a typed prompt does, which is what +keeps it inside the token allocation and inside `ledger/quota.jsonl`. This is the +same rule that killed ShinkaEvolve's `headless/claude` rail and keeps `Task` +denied: the thing this project has learned three times is what an unmetered model +call costs. A wake is not an exception to the ceiling; it is a prompt with no +human typing it. + +**What the watcher may wait on is a closed list.** There is no `--command`, and +its absence is the security model rather than an omission. `hooks.py` gates every +`Bash` the agent runs, and `tools/task.py` re-uses `evaluate_bash` precisely so +that starting a background command cannot become the cheap way around it. A +wakeup that ran an arbitrary shell command on a timer would be that bypass, with +a delay on it -- so the conditions here are things this system already knows how +to read: a task's record, a run's status, a path, a clock. + +The registry is a JSONL under the workspace's app-state directory, for the +reasons `core/tasks.py` gives: three processes can hold an opinion about a wake +at once -- the agent that armed it, the watcher, and the desktop app -- and an +append-only event log is the only shape that survives that. +""" + +from __future__ import annotations + +import datetime as _dt +import json +import logging +import secrets +import subprocess +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any + +from core import appdata, jsonl + +log = logging.getLogger("grad.wakeups") + +T_ARMED = "wake_armed" +T_FIRED = "wake_fired" +T_CANCELLED = "wake_cancelled" +T_DELIVERED = "wake_delivered" +T_FORGOTTEN = "wake_forgotten" + +ARMED = "armed" +#: The condition happened. +FIRED = "fired" +#: The condition did not happen inside the agent's own timeout. Its own state, +#: not a kind of `fired`: "the job finished" and "the job has not finished in +#: four hours" are different facts and lead to different next actions. +EXPIRED = "expired" +CANCELLED = "cancelled" +#: Armed, never resolved, and its watcher is gone -- a reboot, or a kill. The +#: same honesty `core/tasks.py:LOST` is after: nothing was observed, so nothing +#: is claimed. +LOST = "lost" + +TERMINAL = (FIRED, EXPIRED, CANCELLED, LOST) + +#: The conditions a wake can carry. Adding one means teaching `check` to read +#: something this system already records -- not teaching it to run something. +KIND_AFTER = "after" +KIND_TASK = "task" +KIND_RUN = "run" +KIND_FILE = "file" +KINDS = (KIND_AFTER, KIND_TASK, KIND_RUN, KIND_FILE) + +#: The longest a wake may be armed for, whatever it asks. A watcher is cheap but +#: it is not free, and a condition nobody will ever meet should become an +#: `expired` record rather than a process that outlives the research. +MAX_TIMEOUT_S = 24 * 60 * 60 +#: What a wake waits if the agent names no timeout of its own. +DEFAULT_TIMEOUT_S = 4 * 60 * 60 + +#: How often the watcher looks, to begin with, and the ceiling it backs off to. +#: The point of the backoff is the same as the point of the whole module: the +#: first minute of a four-hour job is worth watching closely and the third hour +#: is not, and a fixed interval has to choose between being slow to notice and +#: being a process that wakes up ten thousand times. +POLL_START_S = 2.0 +POLL_MAX_S = 30.0 + +#: Remote states that mean a job has stopped, whichever backend reported it. +#: Deliberately a fixed vocabulary rather than "anything that is not running": +#: `tools/kaggle.py:_parse_status` returns `unknown` for output it cannot read, +#: and treating unknown as finished is how you collect a kernel mid-run. +TERMINAL_REMOTE = { + "COMPLETE", "COMPLETED", "DONE", "ERROR", "FAILED", "CANCELED", "CANCELLED", + "KILLED", "CANCELACKNOWLEDGED", +} + +#: Which CLI answers `status` for a run, per `platform`. The sibling of +#: `core/submit.py:COLLECTORS`, and kept beside it in spirit for the same reason: +#: an unknown platform degrades to "cannot tell", never to a guess. +STATUS_TOOLS = { + "hf_jobs": "tools.jobs", + "kaggle": "tools.kaggle", + "ssh": "tools.gpu", +} + + +# --------------------------------------------------------------------------- +# where things live +# --------------------------------------------------------------------------- +def registry_path() -> Path: + return appdata.workspace_state_dir() / "wakeups.jsonl" + + +def token_path() -> Path: + return appdata.state_dir() / "wake.token" + + +def now_iso() -> str: + return _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds") + + +def iso_at(epoch_s: float) -> str: + """A `time.time()` deadline as an ISO instant, for a record a human reads.""" + return _dt.datetime.fromtimestamp(float(epoch_s), _dt.timezone.utc).isoformat( + timespec="seconds" + ) + + +def new_id() -> str: + stamp = _dt.datetime.now(_dt.timezone.utc).strftime("%H%M%S") + return f"wake-{stamp}-{secrets.token_hex(2)}" + + +def token() -> str: + """The secret a wake must present to start a turn. Created on first use. + + **This is not decoration.** The app binds an unauthenticated loopback port + on purpose -- `/__grad/show` raises a window, which is harmless -- but the + endpoint a wake arrives on *starts a turn for an agent with Bash access*, and + that is the one thing on this port that must not be reachable by anything + that can open a socket to it. Any process on the machine can; only ours can + read a mode-600 file in the app directory. + + Persisted rather than generated per launch, because the watcher outlives the + app: a wake armed before a restart has to still be deliverable after it. + """ + path = token_path() + if not path.exists(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(secrets.token_urlsafe(32), encoding="utf-8") + try: + path.chmod(0o600) + except OSError: # not every filesystem honours it; the file is local either way + log.debug("could not restrict permissions on %s", path) + return path.read_text(encoding="utf-8").strip() + + +# --------------------------------------------------------------------------- +# reading +# --------------------------------------------------------------------------- +def events() -> list[dict[str, Any]]: + return jsonl.read(registry_path()) + + +def wakeups(*, check_liveness: bool = True) -> dict[str, dict[str, Any]]: + """Every wake, folded to its current state.""" + folded: dict[str, dict[str, Any]] = {} + for record in events(): + wake_id = record.get("id") + if not wake_id: + continue + kind = record.get("type") + if kind == T_ARMED: + folded[wake_id] = { + "id": wake_id, + "state": ARMED, + "armed_at": record.get("at"), + "condition": record.get("condition") or {}, + "note": record.get("note") or "", + "deadline": record.get("deadline"), + "pid": record.get("pid"), + "resume": bool(record.get("resume", True)), + "detail": None, + "finished_at": None, + "delivered": None, + } + elif wake_id in folded: + node = folded[wake_id] + if kind == T_FIRED: + node["state"] = EXPIRED if record.get("expired") else FIRED + node["detail"] = record.get("detail") + node["finished_at"] = record.get("at") + elif kind == T_CANCELLED: + node["state"] = CANCELLED + node["finished_at"] = record.get("at") + elif kind == T_DELIVERED: + node["delivered"] = record.get("delivered") + elif kind == T_FORGOTTEN: + folded.pop(wake_id, None) + + if check_liveness: + from core import tasks as tasklib # noqa: PLC0415 - shares the pid check + + pending = [n for n in folded.values() if n["state"] == ARMED and n.get("pid")] + if pending: + live = tasklib.alive_pids([n["pid"] for n in pending], max_age_s=2.0) + for node in pending: + if node["pid"] not in live: + node["state"] = LOST + return folded + + +def get(wake_id: str) -> dict[str, Any] | None: + return wakeups().get(wake_id) + + +def armed() -> list[dict[str, Any]]: + return [w for w in wakeups().values() if w["state"] == ARMED] + + +def pending_delivery() -> list[dict[str, Any]]: + """Wakes that fired and never reached a session. + + What the agent finds when it comes back to a workspace whose app was closed + while the watcher was still running. Without this a wake that fired into a + machine with no UI would simply be lost, which is the failure this whole + module exists to prevent, arrived at by a different road. + """ + return [ + w + for w in wakeups().values() + if w["state"] in (FIRED, EXPIRED) and not w.get("delivered") + ] + + +# --------------------------------------------------------------------------- +# writing +# --------------------------------------------------------------------------- +def append(record: dict[str, Any]) -> dict[str, Any]: + return jsonl.append(registry_path(), {"at": now_iso(), **record}) + + +def record_armed( + wake_id: str, + *, + condition: dict[str, Any], + deadline: float, + note: str, + pid: int, + resume: bool, +) -> dict[str, Any]: + return append( + { + "type": T_ARMED, + "id": wake_id, + "condition": condition, + "deadline": deadline, + "note": note, + "pid": pid, + "resume": resume, + } + ) + + +def record_fired(wake_id: str, *, detail: dict[str, Any], expired: bool = False) -> dict[str, Any]: + return append({"type": T_FIRED, "id": wake_id, "detail": detail, "expired": expired}) + + +def record_cancelled(wake_id: str, *, reason: str = "") -> dict[str, Any]: + return append({"type": T_CANCELLED, "id": wake_id, "reason": reason}) + + +def record_delivered(wake_id: str, *, delivered: str) -> dict[str, Any]: + return append({"type": T_DELIVERED, "id": wake_id, "delivered": delivered}) + + +def forget(wake_ids: list[str]) -> int: + for wake_id in wake_ids: + append({"type": T_FORGOTTEN, "id": wake_id}) + return len(wake_ids) + + +# --------------------------------------------------------------------------- +# the conditions +# --------------------------------------------------------------------------- +def describe(condition: dict[str, Any]) -> str: + """One line naming what is being waited for, for a prompt and a listing.""" + kind = condition.get("kind") + if kind == KIND_AFTER: + return f"{int(condition.get('seconds') or 0)}s elapse" + if kind == KIND_TASK: + return f"background task {condition.get('task')} finishes" + if kind == KIND_RUN: + return f"run {condition.get('run')} stops running on its backend" + if kind == KIND_FILE: + what = "changes" if condition.get("changed") else "appears" + return f"{condition.get('path')} {what}" + return str(kind or "an unknown condition") + + +def check(condition: dict[str, Any]) -> tuple[bool, dict[str, Any]]: + """Has it happened? Returns `(fired, detail)` and never raises. + + Never raises because this runs in a loop in a detached process with nowhere + to report to. A condition that cannot be read right now -- an unreadable + ledger, a `kaggle` CLI that timed out, a network that is down -- is not a + condition that has been met, and the honest answer is to look again in a few + seconds. The timeout is what stops that being forever. + """ + try: + return _check(condition) + except Exception as exc: # noqa: BLE001 - see the docstring + log.debug("could not evaluate %s", condition, exc_info=True) + return False, {"unreadable": f"{type(exc).__name__}: {exc}"} + + +def _check(condition: dict[str, Any]) -> tuple[bool, dict[str, Any]]: + kind = condition.get("kind") + + if kind == KIND_AFTER: + # Held by the watcher's own clock rather than measured here: the + # deadline is absolute, so a machine that slept through the interval + # wakes to a condition that is already true. + fire_at = float(condition.get("fire_at") or 0) + if time.time() >= fire_at: + return True, {"elapsed_s": int(condition.get("seconds") or 0)} + return False, {} + + if kind == KIND_TASK: + from core import tasks as tasklib # noqa: PLC0415 + + task = tasklib.get(str(condition.get("task") or "")) + if task is None: + return False, {"missing": "no such task in the registry"} + if task.get("state") in tasklib.TERMINAL: + return True, { + "task": task.get("id"), + "state": task.get("state"), + "exit_code": task.get("exit_code"), + "label": task.get("label"), + } + return False, {"state": task.get("state")} + + if kind == KIND_FILE: + path = Path(str(condition.get("path") or "")) + if not path.exists(): + return False, {} + if not condition.get("changed"): + return True, {"path": str(path), "appeared": True} + try: + mtime = path.stat().st_mtime_ns + except OSError: + return False, {} + baseline = condition.get("mtime_ns") + if baseline is None or mtime != baseline: + return True, {"path": str(path), "mtime_ns": mtime} + return False, {} + + if kind == KIND_RUN: + return _check_run(str(condition.get("run") or "")) + + return False, {"unknown_kind": str(kind)} + + +#: Run statuses that mean it is over. Every one of these is written by +#: `core/submit.py:record_collected`, which stamps `collected_at` too -- so in +#: practice the `collected` branch below catches them first, and this is the +#: backstop for a record where the two disagree. +#: +#: An explicit set, and that is the whole point. The test used to be `status and +#: status != "in_flight"`, which treats *everything* unrecognised as finished -- +#: including `"unknown"`, which is what `Run.status` returns for a fold with no +#: status in it. `ledger_store.runs()` builds a node from any event carrying an +#: id, and `jsonl.iter_records` skips a malformed line rather than raising, so a +#: torn `run_submitted` line followed by an intact `run_handle` produces exactly +#: that record. The wake then fired immediately, reported that the run had +#: stopped, and spent a metered turn on a claim nothing had checked -- when the +#: honest answer was "ask the backend", which is what falling through does. +TERMINAL_RUN_STATUSES = frozenset({"completed", "failed", "submit_failed", "abandoned"}) + + +def _check_run(run_id: str) -> tuple[bool, dict[str, Any]]: + """Has this run stopped running on whatever machine it went to? + + Two sources, cheapest first. The ledger is a local file and knows when a run + has already been collected; only if it is still in flight is the backend + asked, and that is a network call priced accordingly by the backoff. + + The backend is reached through its own `status` CLI rather than through its + client library, which is the same choice `tools/task.py` makes about running + commands: the CLI owns the credential handling, the timeouts and the §8 + envelope, and a second code path that talked to Hugging Face directly would + be a second place for the namespace bug to live. + """ + from core import ledger_store as ls # noqa: PLC0415 + + try: + record = ls.run(run_id) + except Exception as exc: # noqa: BLE001 - a missing run is not a fired wake + return False, {"unreadable": f"{type(exc).__name__}: {exc}"} + + if record.collected: + return True, {"run": run_id, "collected": True, "status": record.status} + if record.status in TERMINAL_RUN_STATUSES: + return True, {"run": run_id, "status": record.status} + + tool = STATUS_TOOLS.get(str(record.get("platform") or "")) + if tool is None: + # An unknown platform cannot be polled, so the wake rests on the ledger + # alone. Said in the detail rather than silently degrading, because + # "still in flight" and "nobody can tell you" are different answers. + return False, {"run": run_id, "unpollable": str(record.get("platform") or "unknown")} + + data = _status_envelope(tool, run_id) + if data is None: + return False, {"run": run_id, "unreadable": "the status command did not answer"} + if data.get("collected"): + return True, {"run": run_id, "collected": True} + if _remote_finished(data): + return True, {"run": run_id, "remote": _remote_state(data)} + return False, {"run": run_id, "remote": _remote_state(data)} + + +def _status_envelope(tool: str, run_id: str) -> dict[str, Any] | None: + """`python -m status --json`, decoded to its `data`.""" + from core import paths, spawn # noqa: PLC0415 + + try: + proc = spawn.run( + [sys.executable, "-m", tool, "status", run_id, "--json"], + cwd=str(paths.root()), + capture_output=True, + text=True, + timeout=180, + check=False, + ) + except (OSError, subprocess.SubprocessError): + return None + try: + envelope = json.loads((proc.stdout or "").strip().splitlines()[-1]) + except (json.JSONDecodeError, IndexError): + return None + if not isinstance(envelope, dict): + return None + data = envelope.get("data") + return data if isinstance(data, dict) else None + + +def _remote_state(data: dict[str, Any]) -> Any: + """Whatever this backend calls the remote's state, for the record.""" + if isinstance(data.get("remote"), dict): + return data["remote"] or None + if isinstance(data.get("kernel"), dict): + return data["kernel"].get("status") + return data.get("remote_state") + + +def _remote_finished(data: dict[str, Any]) -> bool: + """Does this status payload say the job has stopped? + + Three shapes, because there are three backends and they were never going to + agree: `gpu.py` reports the marker file it writes when the command exits, + `jobs.py` a stage string, `kaggle.py` a kernel status. An unrecognised value + is not finished -- see `TERMINAL_REMOTE`. + """ + marker = data.get("remote") + if isinstance(marker, dict) and marker: + return True + candidates = [data.get("remote_state")] + kernel = data.get("kernel") + if isinstance(kernel, dict): + candidates.append(kernel.get("status")) + return any( + isinstance(value, str) and value.strip().upper() in TERMINAL_REMOTE + for value in candidates + ) + + +# --------------------------------------------------------------------------- +# waking the agent +# --------------------------------------------------------------------------- +def prompt_for(wake: dict[str, Any], detail: dict[str, Any], *, expired: bool) -> str: + """The turn a fired wake issues. + + Written as a report to the agent rather than as an instruction, and the + difference matters: the agent armed this and knows why, and a prompt that + told it what to do next would be this module deciding research questions. It + states what was waited for, what happened, and the note the agent left + itself. + """ + what = describe(wake.get("condition") or {}) + head = ( + f"[wakeup {wake['id']}] the condition did not happen within the timeout you set: {what}." + if expired + else f"[wakeup {wake['id']}] {what} — this is the wake you armed." + ) + lines = [head] + if wake.get("note"): + lines.append(f"\nWhat you said you were waiting for: {wake['note']}") + readable = {k: v for k, v in (detail or {}).items() if v not in (None, {}, "")} + if readable: + lines.append(f"\nWhat the watcher saw: {json.dumps(readable, default=str)}") + return "\n".join(lines) + + +def deliver(wake_id: str, prompt: str) -> bool: + """Hand a wake to the running app, and say whether it took it. + + The same shape as `core/instance.py:show_running` -- the published port, a + short timeout, every failure folded to False -- with the one difference that + matters: this carries the token. See `token`. + + A False here is not an error. It is a workspace whose app is closed, which + is an ordinary thing for a four-hour job to finish into, and + `pending_delivery` is what makes it recoverable rather than lost. + """ + from core import instance # noqa: PLC0415 + + state = instance.read_state() + port = state.get("port") + if not port: + return False + body = json.dumps({"wake": wake_id, "token": token(), "prompt": prompt}).encode("utf-8") + request = urllib.request.Request( # noqa: S310 - fixed local scheme + f"http://127.0.0.1:{int(port)}/__grad/wake", + data=body, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10.0) as response: # noqa: S310 + return 200 <= response.status < 300 + except (urllib.error.URLError, OSError, ValueError): + return False + + +def spawn_watcher(wake_id: str) -> int: + """Start the detached process that does the waiting. Returns its pid. + + Detached for the reason `tools/task.py` spawns its supervisor detached: the + CLI that armed this is about to exit, and the whole point is that the wait + outlives it. `core/spawn.py` explains why that is `CREATE_NO_WINDOW` and not + `DETACHED_PROCESS`. + """ + from core import paths, spawn # noqa: PLC0415 + + child = subprocess.Popen( # noqa: S603 - our own module, our own argv + [sys.executable, "-u", "-m", "tools.wakeup", "_watch", "--wake-id", wake_id], + cwd=str(paths.root()), + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + **spawn.detached(), + ) + return int(child.pid) + + +def watch(wake_id: str) -> dict[str, Any]: + """Poll one wake until it fires, expires, or is cancelled. **Blocks.** + + The body of the detached watcher, and a plain function so it can be tested + without spawning anything. + """ + wake = get(wake_id) + if wake is None: + return {"wake": wake_id, "state": "unknown"} + condition = wake.get("condition") or {} + deadline = float(wake.get("deadline") or 0) + interval = POLL_START_S + + while True: + current = get(wake_id) + if current is None or current["state"] == CANCELLED: + # Cancelled out from under us. Nothing to fire and nothing to say. + return {"wake": wake_id, "state": CANCELLED} + + fired, detail = check(condition) + expired = not fired and time.time() >= deadline + if fired or expired: + record_fired(wake_id, detail=detail, expired=expired) + if wake.get("resume", True): + prompt = prompt_for(wake, detail, expired=expired) + if deliver(wake_id, prompt): + record_delivered(wake_id, delivered="session") + return { + "wake": wake_id, + "state": EXPIRED if expired else FIRED, + "detail": detail, + } + + time.sleep(min(interval, max(0.0, deadline - time.time()) + 0.1)) + interval = min(POLL_MAX_S, interval * 1.5) diff --git a/notes/plan-projects-and-setup-2026-08-17.md b/notes/plan-projects-and-setup-2026-08-17.md new file mode 100644 index 0000000..6a9903c --- /dev/null +++ b/notes/plan-projects-and-setup-2026-08-17.md @@ -0,0 +1,364 @@ +# Plan — the projects window and interactive setup, 2026-08-17 + +> **Implemented, 2026-08-17.** All five stages are in on `dev`, with 1550 tests +> green and the whole thing driven in a live browser preview against a scratch +> workspace. Three things came out differently from the plan and are recorded at +> the bottom under *What changed in the building*. + +Scope: a first-class projects surface, and a guided setup that asks for the +Claude token, the six model roles, the backends and the ceilings — with setup +opening automatically when a project is created. + +The feature as originally described is one wizard covering four subjects. The +central claim of this plan is that **it is two wizards**, because three of the +four subjects are machine- or workspace-scoped and only one is per-project. A +single wizard bolted to project creation re-asks for the Claude token every time +a project is made, and a user with six projects answers the same question six +times. + +--- + +## 1. Five decisions everything else follows from + +**1.1 Projects is a window, not a dialog.** `ui/registry.py`'s own docstring +says a thirteenth window is one `WindowSpec` and one module. The current project +surface is a 540px modal that closes on every action (`ui/shell.py:300`) and +cannot be tiled beside the ledger it explains. Projects keys every run, every +ceiling and every report; it is the only first-class concept in the app with no +window. + +**1.2 Setup splits by scope, not by page count.** + +| subject | scope | asked | +| --- | --- | --- | +| Claude OAuth token | machine | once, or when missing | +| six model roles | workspace | once, editable any time | +| backends + credentials | machine (credential) + workspace (inventory) | once per backend | +| ceilings, payer, default backend | **project** | every new project | + +First launch runs the machine half. Project creation runs the project half and +*skips* anything already satisfied, with a link to review the rest. + +**1.3 `config/grad.toml` is never machine-written.** README:216 states the +reason — it is hand-annotated, `tomllib` cannot write it, and an editing command +would reformat it and drop every comment in it. The precedent for a machine-set +value already exists: `tools/kaggle.py:179` writes +`appdata.state_dir()/kaggle.json`, that selection *wins* over `[kaggle] username`, +and `account` reports when it is shadowing a config value. Setup follows that +exactly. + +**1.4 Resolution is layered, and the layers are already the shape `model_for` +uses.** `core/config.py:431` resolves explicit → legacy → default, and +`paths.config_path()` is `_shipped("config", "grad.toml")` — the *workspace's* +copy when it has one, the installation's otherwise. So two config layers already +exist. Setup adds two more above them: + +``` +project override → workspace overlay → workspace grad.toml → installed grad.toml → legacy → DEFAULTS +``` + +Every layer must be reportable. "Why is it using Sonnet for evolve?" has to have +an answer, the way `[kaggle] username` shadowing does. + +**1.5 Budget stays an append-only event.** `core/budget.py:234` never mutates a +ceiling; it records `previous` and `reason`. Setup produces a create-time +`budget{}` through `budget.create`, and every later change is a raise with a +reason. A wizard that could be re-run and silently rewrite ceilings would +destroy the one property that module exists to hold. + +--- + +## 2. The layers, bottom up + +### 2.1 `core/settings.py` — the writable overlay (new) + +One JSON document at `appdata.workspace_state_dir()/settings.json` — **per +workspace, not per machine.** An earlier draft of this plan put it in +`state_dir()`, and that was wrong: `paths.config_path()` already resolves +`grad.toml` per workspace, so a machine-global overlay would silently flatten +two workspaces that had deliberately different model choices. An overlay must +mirror the scope of the file it shadows. + +It stays out of the workspace *folder* for the reason `core/workspace.py` keeps +the root pointer beside the code: these are answers about how this install is +wired up, and a research folder copied to a colleague's machine should not carry +this one's SSH inventory. `appdata.workspace_state_dir()` is exactly that seam — +keyed by workspace, stored with the installation — and it is already where the +window layouts live. + +Note an existing inconsistency this makes visible: `tools/kaggle.py:179` writes +`kaggle.json` to the machine-global `state_dir()` while shadowing `[kaggle] +username`, which is per-workspace. Not urgent, and not this plan's to fix — but +`setup show` will report it, so it should be a deliberate answer rather than a +surprise. + +```python +def load() -> dict[str, Any] +def models() -> dict[str, str] # role -> model, only what was set +def set_models(**roles: str) -> dict # validates against config.MODEL_ROLES +def default_backend() -> str | None +def set_backend(name: str) -> dict +def hosts() -> dict[str, dict] # SSH inventory added through setup +def add_host(name, hostname, user, *, rate_usd_per_hour, workdir, gpus) -> dict +def shadowing(cfg: Config) -> list[dict] # what here overrides what there +``` + +`shadowing()` is not optional decoration — it is the thing that keeps 1.3 +honest. A user who edits `[models] evolve` in the TOML and sees no change needs +the app to tell them why. + +**Validation belongs here, not in the UI.** Model role names against +`config.MODEL_ROLES`; host names against the same constraint `config.host()` +enforces; backend against `evolve.REMOTE_BACKENDS`. A bad value written by a +button is a bad value the next campaign reads. + +### 2.2 `core/config.py` — teach the resolver about the overlay + +`model_for` gains the overlay above `self.user`, and `hosts` merges +`settings.hosts()` into `[hosts.*]`. Both stay pure functions of their inputs; +the overlay is read once in `load()` and carried on `Config`, so nothing on the +gate path grows a file read. + +**The `[hosts.*]` inventory rule survives this.** `core/config.py:520` refuses an +unknown host name — "the inventory is fixed, never an ad-hoc connection". Setup +does not weaken that; it gives the inventory a second, writable source. The +refusal message should name both places a host can be defined. + +### 2.3 `core/budget.py` — per-project settings as an event + +```python +T_PROJECT_CONFIGURED = "project_configured" + +def configure(project_id, *, models=None, backend=None, reason="") -> dict +``` + +Folded in `projects()` beside `T_PROJECT_RAISED` (`core/budget.py:164`). An +event rather than a field for the same reason a raise is one: the record of what +this project was set to, and when, is worth more than the current value alone — +and a campaign whose model changed mid-flight is exactly the thing a ledger +should be able to answer. + +**Confirmed as real work, not a maybe** (decision, 2026-08-17): the model chosen +per role is the main lever on both cost and quality, and it is exactly the thing +that should differ between a cheap exploratory project and one being written up. + +Two consequences that are easy to miss: + +**The running session has to be rebuilt when `research` changes.** `ui/app.py:172` +builds `ClaudeSDKClient` options from `cfg` once, at client start. Switching to a +project that overrides `research` while a session is live leaves the old model +answering — silently, and while the projects window shows the new one. The +precedent is already there: `client_effort` (`ui/app.py:178`) exists so an effort +change can decide whether a rebuild is needed. Model selection needs the same +recorded-and-compared treatment, and `use_project` becomes a rebuild trigger. + +**Which roles may be overridden is a decision, not a default.** All six is the +obvious answer and probably right, but `research` is the one with the live-session +problem above, and `cite`/`triage` are described in `config/grad.toml:70-73` as +mechanical work where a cheaper model is the point. Start with all six overridable +and let the UI's ordering carry the advice. + +### 2.4 The CLIs — because every button runs one + +§10 and `tests/test_ui_argv.py:27`: the UI builds argv and a flag that does not +parse is a dead button. New commands: + +```bash +python -m tools.setup show --json # every layer, and what wins +python -m tools.setup models --evolve claude-opus-5 --json +python -m tools.setup backend --default kaggle --json +python -m tools.setup host add --name gpu-box --hostname … --user … --rate 1.20 --json +python -m tools.setup check --json # what is missing for each backend +``` + +`setup check` is the one that earns its place: it answers "can I actually submit +to Kaggle right now" by testing the credential *pair*, the way +`kaggle account --check` already does. A wizard that stores a token and never +tries it produces a green checkmark and a failure an hour later. + +Every new argv goes into `UI_COMMANDS`. + +### 2.5 `ui/models.py` — two shaped models + +- `projects_model()` — id, title, status, payer, ceilings *and* spend against + each, memory-file freshness (`core/projects.py` knows whether the generated + files are stale), last run. Every reader wrapped in `_safe`, per the rule + `workspaces_model` already follows: this panel has to render when the + workspace is wrong. +- `setup_model()` — per step: satisfied / missing / unknown, plus what each + answer would unlock. This is where `hf_token` stops being unconditionally + "required" (`ui/models.py:539`) and becomes required *for HF Jobs*, which is + the actual claim. + +### 2.6 The windows + +`ui/windows/projects.py` — list, select, create, close, per-project ceilings +inline, and a button that opens setup for that project. + +`ui/windows/setup.py` — the stepper. + +**Step state cannot live in a Python local.** `ui/static/tiling.js:184` — the +pane tree is rebuilt by the server on every retile, and non-persistent windows +are rebuilt on refresh ticks. The step index and the in-progress answers belong +in workspace state keyed by window id. Note also that `WindowSpec.persistent` is +declared (`ui/registry.py:36`) and asserted in `tests/test_ui_registry.py:106` +but has **no Python-side consumer I could find** — confirm what actually honours +it before relying on it for this window. + +`kit` needs one new primitive: a step header (n of m, back/next, a disabled next +until the step validates). Everything else — `kit.button`, `kit.chip`, +`kit.error_strip`, the password-shaped input from `_credentials` — already +exists. + +### 2.7 Dismantle the junk drawer + +Three scopes, three homes (decision, 2026-08-17): + +| control | goes to | +| --- | --- | +| switch project | the projects window, and `project ▾` for the quick switch | +| switch workspace folder, recent list | a new `workspace ▾` appbar control | +| credentials | the setup window | +| version and update | the setup window (the `↑ v0.2.0` appbar button stays) | + +`workspace ▾` shows the folder's **basename**, with the full path in the tooltip +— `model["root"]` is an absolute path and the appbar cannot carry one. It gets a +confirmation step, and it is the only control in the app that does: a project +switch changes what spend is charged to, while a folder switch replaces the +ledger, the project list, the notebooks and possibly the whole model config +under every open window. In the current menu those two sit six rows apart and +are styled identically (`ui/shell.py:342` and `:357`), which is the specific +thing this fixes. + +This is the change that makes the projects window worth having rather than a +fourth place to look. + +--- + +## 3. Sequence + +Each stage is shippable and independently useful. + +**Stage 1 — the projects window, and the `workspace ▾` split.** `projects_model`, +`ui/windows/projects.py`, the `WindowSpec`, the existing create/use/ceiling +actions moved into it, and the folder picker lifted into its own appbar control +(§2.7). No new storage — every action here already has a CLI behind it. Ends +with: projects is a tileable window, and switching folders no longer looks like +switching projects. + +**Stage 2 — `core/settings.py` + `tools/setup.py` + the resolver layers.** No +UI. Ends with: `python -m tools.setup models --evolve …` works, `setup show` +explains what wins, `config.model_for` honours it, and none of it touched +`grad.toml`. + +**Stage 3 — the setup window, machine half.** Token → models → backends, driven +by `setup_model`. Runs automatically on first launch when +`setup check` reports nothing configured. Ends with: a fresh machine is usable +without opening a terminal — which is the actual goal, and the README's install +section shrinks to `pip install` plus "open Grad". + +**Stage 4 — the project half, and the create hook.** Ceilings, payer, default +backend, on create. Skips satisfied machine steps. Ends with: the "created with +no ceilings" hole (`ui/shell.py:402`) is closed. + +**Stage 5 — per-project model overrides.** `T_PROJECT_CONFIGURED`, the top +resolution layer, and the session rebuild on project switch. Confirmed in scope. +Ends with: a project can be cheap or careful, and switching to it actually +changes which model answers. + +--- + +## 4. Tests each stage owes + +- Stage 1: `test_ui_registry` (id, defaults, the persistent set), `test_ui_argv` + for every button, `projects_model` against a real ledger — including a + workspace with an unparseable one, per `workspaces_model`'s rule. +- Stage 2: resolution order, with a case per layer; shadowing report; refusal on + an unknown role and an unknown backend; **`grad.toml` byte-identical after + every setup command** — that is the test 1.3 actually needs. +- Stage 3: `setup_model` step states; the credential-pair check surfacing a + wrong Kaggle key as a failed step rather than a stored one. +- Stage 4: create-with-ceilings goes through `budget.create`, not a raise; a + re-run wizard does not rewrite ceilings silently. +- Stage 5: the fold, and a project whose override survives a compaction. + +--- + +## 5. Decisions and open questions + +**Settled 2026-08-17:** + +1. **Per-project models are real.** They move cost and quality, which is the + whole reason to have them. Stage 5 is in scope; see §2.3 for the two + consequences. +2. **A hand-edited `grad.toml` is read, shown, and left alone.** Setup presents + its values as the current answers and writes to the overlay only when one is + changed — so a user who configured everything by hand meets a wizard that + agrees with them, and their comments survive (§1.3). + +3. **The workspace folder gets its own `workspace ▾` control**, not a strip + inside the projects window — because switching folders changes what *every* + open window is showing, and a control that destructive should not sit inside + a list of projects as one more row. See §2.7 for the full split. + +**Still open:** nothing blocking. Stage 1 can start. + +--- + +## 6. Fixed on the way here + +Two bugs found during the audit, fixed before this plan was written: + +- **`kaggle_key` had no purpose text.** It was in `credentials.ALL` and not in + `ui/models.py:CREDENTIAL_NOTES`, so the panel drew the free backend's + credential with an empty purpose column. Test asserts against `ALL` rather + than by name, because the hole was drift between two hand-written lists. +- **A stored Claude token never reached the agent's own loop.** The main loop + authenticates from ambient `CLAUDE_CODE_OAUTH_TOKEN` (`agent.py`), `sdk_env` + reads the credential store, and nothing joined them — so a token stored + through the credentials panel ran the funnel and the mutation operator and + left the main loop unauthenticated. It bit the installed app and not the + terminal, which is why it survived. `credentials.hydrate_environment()` now + bridges them, after the scrub and never over an exported token. + +The second one is why step 1 of the wizard is worth building: pasting a token +into the app is now sufficient, and before this it silently was not. + +--- + +## 7. What changed in the building + +**The overlay is per workspace, not per machine** (§2.1). `paths.config_path()` +is `_shipped(...)`, so `grad.toml` was already workspace-overridable and a +machine-global overlay would have flattened two workspaces that had deliberately +different model choices. Caught by reading the resolver rather than by a test, +which is the wrong order and worth saying. + +**The two overlays stayed separate rather than merging.** `Config` carries +`overlay` and `project_overlay` as distinct fields and `model_for` takes +`project=False`. Merging them would have been less code, and the projects window +could not then have answered "what would this role be if this project said +nothing" — which it has to, because it draws every project and only one of them +is selected. + +**`kit.attr` did not escape backslashes.** NiceGUI hands each props value to +`ast.literal_eval`, so `C:\Users\...` in a tooltip contains `\U`, begins a +unicode escape, and raises a SyntaxError out of `element.props()` — taking down +whatever was being built rather than the tooltip. It had never fired because no +control had ever put a *path* in a tooltip; `workspace ▾` is the first. + +**One bug the suite could not see.** `_project()` referenced a name that was not +in its scope, and the whole suite stayed green: the empty-workspace render test +returns before reaching that branch, and the populated one caught the NameError +in the failure card `shell._render` deliberately draws. That card is the right +behaviour and it is also a blind spot, so +`test_every_window_renders_with_real_data` now asserts on *content* per window +and that the string "failed to render" appears nowhere. Verified by +reintroducing the bug and watching the test fail. + +Two things the plan called for and did not need: a `persistent` flag on the +setup window (step state lives in `workspace.selection`, which survives both a +rebuild and a retile, so the flag's unclear Python-side consumer never became +load-bearing), and a `kit` stepper with back/next — the steps are tabs, because +none of them gates another and a wizard you cannot go back in is one people +abandon at question three. diff --git a/prompts/system.md b/prompts/system.md index 0582743..0f86228 100644 --- a/prompts/system.md +++ b/prompts/system.md @@ -30,6 +30,10 @@ submitter refuses, it is telling you something real, and the fix is in the error you get on with the next thing rather than blocking the turn on it. Independent commands can be started together; a command whose input is another's output cannot. You decide which is which. +- Anything that takes hours you do not wait for at all. Arm `tools.wakeup` and + end the turn; you will be woken when it finishes. Polling with a `sleep` that + keeps growing is the habit this replaces, and it spends a turn each time to + learn nothing. - Check a library call against the installed signature before trusting it, and against `docs.py` before assuming it is current. @@ -82,7 +86,13 @@ carries a `fix` field that is usually the literal next command. `--pressure` tune the search; `--jobs` proposes several at once. `TASK.md` in the task dir is put in front of the operator every time — write it. `promote` turns a winner into an ordinary run, which still needs its own preflight and - prediction. + prediction. `--remote {ssh|hf_jobs|kaggle} --remote-spec ` evaluates + every candidate on real hardware instead of on this machine — the loop stays + here, the training goes there. It refuses unless that spec's preflight is + complete and passing including the smoke run, so run the preflight first. A + Kaggle campaign is also projected against the weekly accelerator allowance, so + size `--generations`/`--population` against `tools.kaggle quota` before you + start. Candidates still never enter `runs.jsonl`. - `python -m tools.task start --label --json -- ` — run something in the background and come back to it. `list`, `status `, `output `, `wait `, `stop `. Use it for anything that takes minutes: `preflight @@ -90,6 +100,15 @@ carries a `fix` field that is usually the literal next command. and there is a ceiling on how many may run at once (exit 14). Give it `--halt` when the tool has its own stop verb, e.g. `--halt 'python -m tools.evolve halt --campaign camp-... --json'`. +- `python -m tools.wakeup arm --run --timeout --note "..." --json` — + wait for something without waiting. Arm it, **end your turn**, and you are + woken with a new turn when it happens. Also `--task `, `--file `, + `--after `. This is how you wait for a four-hour training run: not + with `sleep`, not by looking again every few minutes, and never by holding the + shell. Each of those costs a turn's tokens to learn nothing. `list` shows what + is armed and what fired while no window was open; `cancel ` stops one. + Set `--timeout` to what you actually expect plus a margin — a wake that + expires is a fact worth having, and it says so rather than pretending. - `python -m tools.report draft --project --json` — the report skeleton from the ledger, free and model-free. Then `write`, `cite`, `check`, `build`. - `python -m tools.project sync --json` — re-render this project's diff --git a/skills/remote-gpu/SKILL.md b/skills/remote-gpu/SKILL.md index b82013d..68f7c46 100644 --- a/skills/remote-gpu/SKILL.md +++ b/skills/remote-gpu/SKILL.md @@ -57,7 +57,41 @@ written to disk by us. `collect` is non-blocking by default and exits 10 while the job is running — a two-hour poll inside the agent's only shell is a tool timeout waiting to happen. -Use `--wait --timeout ` when you genuinely want to block. +Use `--wait --timeout ` when you genuinely want to block, which from the +agent is almost never: `python -m tools.wakeup arm --run --timeout ` +waits out of process and starts a new turn when the job stops, holding no shell. + +## Evolve campaigns on a host + +`python -m tools.evolve run --remote ssh --remote-spec ` evaluates every +candidate on the host that spec names. (`--remote hf_jobs` and `--remote kaggle` +do the same thing on those backends; this section is the SSH one.) The campaign +loop — mutation, selection, the ledger — stays local. What goes to the host is +the training. + +Each candidate is launched detached under `nohup` with a `grad_status.json` +marker and polled, exactly as `submit` does, and bounded by `timeout` on the host +itself. Both of those matter because a candidate is a training run: a held SSH +connection would be dropped by a NAT timeout or a sleeping laptop, and a poll +that gave up would leave the job running against the next candidate's GPU. + +Each candidate gets a fresh copy of the +pipeline directory under `/`, its own `initial.py` and +`evaluate.py` written over the top, one bounded run, and the directory removed +afterwards — so candidate N cannot see what candidate N-1 left behind. A search +that can accumulate state across evaluations is one whose scores stop being +comparable, and the failure looks like a real improvement. + +It refuses unless that spec has a complete, passing preflight *including the +smoke run*. That is stricter than an ordinary submission, deliberately: a +campaign is a loop with no human in it, so the environment is proven once, before +generation 0, rather than rediscovered forty times at the host's hourly rate. + +Candidates do not become runs. They stay in `ledger/candidates.jsonl`; the +campaign is the ledgered unit and its expectation is the bound prediction. Cost +per candidate is measured wall clock against the host rate, not the campaign's +flat estimate — so a host priced at 0 records a campaign that spent nothing, +which is another reason to set `rate_usd_per_hour` honestly. ## Conventions the pipeline must follow diff --git a/tests/test_candidate_hosts.py b/tests/test_candidate_hosts.py new file mode 100644 index 0000000..bcd48e0 --- /dev/null +++ b/tests/test_candidate_hosts.py @@ -0,0 +1,295 @@ +"""Running one evolve candidate on a real backend. + +`test_evolve_remote.py` covers the driver with the backend stubbed. This is the +other side of that seam: what each backend's adapter actually does to get a +mutated program onto a machine, bound it, and read a score back. + +The case being designed for is the one that matters -- a candidate is a changed +architecture or a changed optimiser, so evaluating it is a *training run* of +minutes to hours. Every property tested here follows from that: the job is +detached rather than held on a connection, it is bounded where it runs rather +than only where it is watched, and a candidate that overruns is killed rather +than left competing with its successor for the same GPU. + +No network. `_ssh` and `_scp` are stubbed with a fake host that records what it +was asked to do. +""" + +from __future__ import annotations + +import pytest + +from core.config import Host +from core.errors import GradError +from core.submission import Submission +from tools import gpu as gpu_tool + + +def a_host(rate: float = 2.0) -> Host: + return Host( + name="gpu-box", + hostname="10.0.0.7", + user="research", + workdir="~/grad", + rate_usd_per_hour=rate, + ) + + +def a_submission(workspace) -> Submission: + directory = workspace / "pipeline" + directory.mkdir(parents=True, exist_ok=True) + (directory / "train.py").write_text("print('x')\n", encoding="utf-8") + (directory / "spec.toml").write_text( + "entrypoint = 'train.py'\nimage = 'org/img@sha256:aaaa'\n" + "[target]\nhost = 'gpu-box'\n" + "[estimate]\nhours = 0.1\nrate_usd_per_hour = 1.0\n", + encoding="utf-8", + ) + return Submission.load(directory / "spec.toml", resolve_digest=False) + + +class FakeHost: + """Records every ssh command, and answers the ones with a known shape. + + `markers` is the sequence `grad_status.json` reads return, so a test can say + "not finished, not finished, finished" without any sleeping. + """ + + def __init__(self, *, markers=None, stdout="", stderr="", launch_pid="4242"): + self.commands: list[str] = [] + self.copies: list[tuple[str, str]] = [] + self.markers = list(markers or ['{"state":"finished","exit_code":0}']) + self.stdout = stdout + self.stderr = stderr + self.launch_pid = launch_pid + + def ssh(self, host, command, *, timeout=300.0): + self.commands.append(command) + if "grad_status.json" in command and command.startswith("cat "): + return self.markers.pop(0) if len(self.markers) > 1 else self.markers[0] + if "nohup" in command: + return self.launch_pid + if "stdout.log" in command and command.startswith("tail"): + return self.stdout + if "stderr.log" in command and command.startswith("tail"): + return self.stderr + return "" + + def scp(self, host, source, dest, **kwargs): + self.copies.append((source, dest)) + + def install(self, monkeypatch): + monkeypatch.setattr(gpu_tool, "_ssh", self.ssh) + monkeypatch.setattr(gpu_tool, "_scp", self.scp) + monkeypatch.setattr(gpu_tool.time, "sleep", lambda _: None) + return self + + def launched(self) -> str: + return next(c for c in self.commands if "nohup" in c) + + +def run_one(workspace, fake, monkeypatch, **kwargs): + from core import config as config_mod + + fake.install(monkeypatch) + options = { + "candidate_id": "camp-1-g0-c0", + "files": {"initial.py": "print('mutated')", "evaluate.py": "print('{}')"}, + "command": ["python", "evaluate.py"], + "timeout_s": 600, + } + options.update(kwargs) + return gpu_tool.evaluate_candidate( + a_submission(workspace), config_mod.load(), host=a_host(), **options + ) + + +# --------------------------------------------------------------------------- +# the shape of the run +# --------------------------------------------------------------------------- +def test_the_candidate_is_detached_not_held_on_the_connection(workspace, monkeypatch): + """A training run is minutes to hours. A single ssh channel held open across + that is one a NAT timeout, a sleeping laptop or a wifi handover will drop -- + and what that produces is not a failed candidate but a candidate that scored + nothing because the network moved, which the search then selects against.""" + fake = FakeHost() + run_one(workspace, fake, monkeypatch) + + launched = fake.launched() + assert "nohup" in launched + assert "grad_status.json" in launched + # Nothing ran the evaluator inline. + assert not any( + "python" in c and "nohup" not in c and "base64" not in c for c in fake.commands + ) + + +def test_the_candidate_is_bounded_where_it_runs(workspace, monkeypatch): + """Not only in the poll. The poll giving up ends the function; it does not + end a detached training run, and an abandoned candidate keeps holding the + GPU the next one is about to be measured on.""" + fake = FakeHost() + run_one(workspace, fake, monkeypatch, timeout_s=1800) + assert "timeout 1800" in fake.launched() + + +def test_the_mutated_files_are_written_after_the_pipeline_is_staged(workspace, monkeypatch): + """Order matters: the stage copies the preflighted pipeline, and the + candidate's own files go over the top of it.""" + fake = FakeHost() + run_one(workspace, fake, monkeypatch) + + writes = [i for i, c in enumerate(fake.commands) if "base64 -d" in c] + launch = fake.commands.index(fake.launched()) + assert writes, "the candidate's source never reached the host" + assert fake.copies, "the pipeline was not staged" + assert max(writes) < launch + + +def test_each_candidate_gets_its_own_directory(workspace, monkeypatch): + """Candidate N cannot see what candidate N-1 left behind. A loop that can + accumulate state across evaluations is one whose scores stop being + comparable, and the failure looks like a real improvement.""" + fake = FakeHost() + result = run_one(workspace, fake, monkeypatch, candidate_id="camp-1-g3-c2") + assert result["where"].endswith("camp-1-g3-c2") + assert any("camp-1-g3-c2" in c for c in fake.commands) + + +def test_the_directory_is_removed_afterwards(workspace, monkeypatch): + fake = FakeHost() + run_one(workspace, fake, monkeypatch) + assert any(c.startswith("rm -rf") for c in fake.commands) + + +# --------------------------------------------------------------------------- +# what comes back +# --------------------------------------------------------------------------- +def test_a_finished_candidate_reports_its_metrics_line_last(workspace, monkeypatch): + """`tools/evolve.py:_metrics_from` reads the last line, and the evaluator's + one JSON object is on stdout -- so stdout has to come after stderr, however + odd that reads in a log.""" + fake = FakeHost( + stdout='epoch 1\n{"combined_score": 2.5}', + stderr="a warning about a deprecated flag", + ) + result = run_one(workspace, fake, monkeypatch) + + assert result["ok"] is True + assert result["exit_code"] == 0 + assert result["output"].strip().endswith('{"combined_score": 2.5}') + assert "deprecated flag" in result["output"] + + +def test_the_exit_code_comes_from_the_marker(workspace, monkeypatch): + """Not from scraping a log. The marker is written by the runner itself and + is the one thing that knows how the process ended.""" + fake = FakeHost(markers=['{"state":"finished","exit_code":3}'], stdout="Traceback") + result = run_one(workspace, fake, monkeypatch) + + assert result["ok"] is False + assert result["exit_code"] == 3 + assert "exited 3" in result["error"] + + +def test_cost_is_wall_clock_against_the_host_rate(workspace, monkeypatch): + fake = FakeHost() + result = run_one(workspace, fake, monkeypatch) + assert result["cost_usd"] >= 0.0 + assert result["host"] == "gpu-box" + + +def test_a_free_host_still_records_a_cost_of_zero(workspace, monkeypatch): + """`rate_usd_per_hour = 0` means free to use and still ledgered.""" + from core import config as config_mod + + FakeHost().install(monkeypatch) + result = gpu_tool.evaluate_candidate( + a_submission(workspace), + config_mod.load(), + host=a_host(rate=0.0), + candidate_id="c1", + files={"initial.py": "x", "evaluate.py": "y"}, + command=["python", "evaluate.py"], + timeout_s=60, + ) + assert result["cost_usd"] == 0.0 + + +def test_the_poll_survives_an_unreadable_marker(workspace, monkeypatch): + """One failed read is a network hiccup, not a verdict -- and the job is + still running on the host either way, which is the point of detaching it.""" + fake = FakeHost(markers=["not json at all", '{"state":"finished","exit_code":0}']) + result = run_one(workspace, fake, monkeypatch) + assert result["ok"] is True + + +def test_a_candidate_that_never_finishes_is_killed(workspace, monkeypatch): + """The loop starts the next candidate the moment this returns. One left + running competes with its own successor for the same GPU, so the next score + would measure this one's overrun rather than the mutation.""" + # The grace is what the loop waits *past* the remote bound before deciding + # the host has stopped answering. Shrunk here because `install` stubs + # `time.sleep` to a no-op, so a minute of grace is a minute of real + # busy-looping in the suite rather than a minute of waiting. + monkeypatch.setattr(gpu_tool, "CANDIDATE_TIMEOUT_GRACE_S", 0.2) + fake = FakeHost(markers=['{"state":"running"}']) + result = run_one(workspace, fake, monkeypatch, timeout_s=0) + + assert result["ok"] is False + assert result["exit_code"] is None + assert "did not finish" in result["error"] + killed = [c for c in fake.commands if "kill" in c] + assert killed, "the overrunning candidate was left running" + # Children first: the pid `_launch` echoes is the wrapping shell's, and + # killing only that orphans the training process on the GPU. + assert "pkill -P 4242" in killed[0] + + +def test_a_host_that_refuses_the_stage_is_not_a_bad_mutation(workspace, monkeypatch): + def refuse(host, command, *, timeout=300.0): + raise GradError("ssh_failed", "ssh to gpu-box failed (exit 255)", exit_code=1) + + monkeypatch.setattr(gpu_tool, "_ssh", refuse) + monkeypatch.setattr(gpu_tool, "_scp", lambda *a, **k: None) + + from core import config as config_mod + + result = gpu_tool.evaluate_candidate( + a_submission(workspace), + config_mod.load(), + host=a_host(), + candidate_id="c1", + files={"initial.py": "x", "evaluate.py": "y"}, + command=["python", "evaluate.py"], + timeout_s=60, + ) + assert result["ok"] is False + assert result["exit_code"] is None, "a transport failure must not look like an exit code" + assert "ssh to gpu-box failed" in result["error"] + + +# --------------------------------------------------------------------------- +# what the host is allowed to be asked for +# --------------------------------------------------------------------------- +def test_a_candidate_file_may_not_be_a_path(workspace): + """Written on a machine we do not own, from a name a caller supplies. + Checked where it would be written rather than trusted to every caller.""" + for bad in ("../escape.py", "sub/dir.py", "..", ""): + with pytest.raises(GradError): + gpu_tool._write_remote(a_host(), "~/grad/c1", bad, "print(1)") + + +def test_source_travels_as_base64_not_as_a_heredoc(workspace, monkeypatch): + """The content is a language model's Python. It can contain anything a + heredoc terminator, a quote or a backtick means to a shell, and getting that + wrong does not raise -- it delivers a file subtly different from the one + that was scored.""" + fake = FakeHost().install(monkeypatch) + nasty = "s = '''\nEOF\n`whoami`\n$(rm -rf /)\n'''\n" + gpu_tool._write_remote(a_host(), "~/grad/c1", "initial.py", nasty) + + written = fake.commands[-1] + assert "base64 -d" in written + for fragment in ("EOF", "whoami", "rm -rf /"): + assert fragment not in written diff --git a/tests/test_candidate_jobs.py b/tests/test_candidate_jobs.py new file mode 100644 index 0000000..27f484b --- /dev/null +++ b/tests/test_candidate_jobs.py @@ -0,0 +1,309 @@ +"""Running one evolve candidate on the two job backends. + +`test_candidate_hosts.py` covers the SSH adapter, where the remote is a machine +that stays up and a candidate is a directory copied onto it. These two are +shaped differently and the difference is the whole reason each backend owns its +own adapter: + + * **Kaggle** has no `scp`. The pipeline already travels as a base64 tar inside + the generated notebook, so a candidate is that same payload with one file + swapped -- which means the secret scan and the size refusals apply to a + candidate exactly as they do to a submission. + * **HF Jobs** has no upload step at all. The pipeline is *in the image* and + `_command_for` just runs the entrypoint it contains, so a candidate -- a + program the image by definition does not have -- needs a way in. + +Neither test touches a network. The CLI runner, the Hub client and the pollers +are stubbed; what is under test is what each adapter asks them for. +""" + +from __future__ import annotations + +import base64 +import io +import secrets +import tarfile + +import pytest + +from core import campaign as camp, config as config_mod, kaggle_quota +from core.errors import UsageError +from tools import jobs as jobs_tool, kaggle as kaggle_tool + +from test_candidate_hosts import a_submission + + +def unpack(blob: str) -> dict[str, str]: + out: dict[str, str] = {} + with tarfile.open(fileobj=io.BytesIO(base64.b64decode(blob))) as tar: + for member in tar.getmembers(): + handle = tar.extractfile(member) + out[member.name] = handle.read().decode("utf-8") if handle else "" + return out + + +# --------------------------------------------------------------------------- +# Kaggle: the candidate rides inside the notebook's payload +# --------------------------------------------------------------------------- +def test_an_override_replaces_a_file_rather_than_shadowing_it(workspace): + """Two entries with one name in a tar is a file whose contents depend on + extraction order, which for a candidate means a score that depends on tar.""" + blob, packed = kaggle_tool._payload_b64( + a_submission(workspace), overrides={"train.py": "print('mutated')"} + ) + assert unpack(blob)["train.py"] == "print('mutated')" + assert packed.count("train.py") == 1 + + +def test_an_override_can_add_a_file_the_pipeline_does_not_have(workspace): + """A candidate is `initial.py` and `evaluate.py`, which a pipeline built for + ordinary submission has no reason to contain.""" + blob, _ = kaggle_tool._payload_b64( + a_submission(workspace), + overrides={"initial.py": "x = 1", "evaluate.py": "print('{}')"}, + ) + files = unpack(blob) + assert files["initial.py"] == "x = 1" + assert files["evaluate.py"] == "print('{}')" + assert "train.py" in files, "the proven pipeline stopped travelling" + + +def test_the_payload_is_a_function_of_its_inputs(workspace): + """Deterministic, so two pushes of one candidate are byte-identical and a + difference between two blobs means a difference in the code.""" + sub = a_submission(workspace) + first, _ = kaggle_tool._payload_b64(sub, overrides={"initial.py": "x = 1"}) + second, _ = kaggle_tool._payload_b64(sub, overrides={"initial.py": "x = 1"}) + assert first == second + + +def test_a_payload_entry_may_not_climb_out_of_the_pipeline(workspace): + for bad in ("../escape.py", "/etc/passwd", "a/../../b.py"): + with pytest.raises(UsageError): + kaggle_tool._payload_b64(a_submission(workspace), overrides={bad: "x"}) + + +def stub_kaggle(monkeypatch, *, status="complete", marker=None, hours=0.03, log=""): + pushed: dict = {} + monkeypatch.setattr(kaggle_tool, "_username", lambda cfg: "someone") + monkeypatch.setattr( + kaggle_tool, "_push", lambda cfg, workdir, **kw: pushed.update(kw) or "pushed" + ) + monkeypatch.setattr(kaggle_tool, "_wait", lambda cfg, ref, deadline: {"status": status}) + monkeypatch.setattr( + kaggle_tool, + "_fetch_output", + lambda cfg, ref, artifacts: (marker if marker is not None else {}, hours, log), + ) + return pushed + + +def run_kaggle(workspace, **kwargs): + options = { + "candidate_id": "camp-1-g0-c0", + "files": {"initial.py": "x = 1", "evaluate.py": "print('{}')"}, + "command": ["python", "evaluate.py"], + "timeout_s": 1800, + "artifacts": workspace / "artifacts", + } + options.update(kwargs) + return kaggle_tool.evaluate_candidate( + a_submission(workspace), config_mod.load(), **options + ) + + +def test_the_kernel_is_bounded_where_it_runs(workspace, monkeypatch): + """Without `--timeout` the only limit is how long we choose to poll, which + stops us waiting and does not stop the kernel -- and an abandoned kernel + keeps spending the weekly allowance.""" + pushed = stub_kaggle( + monkeypatch, + marker={"exit_code": 0, "elapsed_s": 120}, + log='epoch 1\n{"combined_score": 1}', + ) + result = run_kaggle(workspace, timeout_s=1800) + + assert pushed["kernel_timeout_s"] == 1800 + assert result["ok"] is True + assert result["hours"] == 0.03 + assert result["output"].endswith('{"combined_score": 1}') + + +def test_a_kaggle_candidate_is_priced_in_hours_not_dollars(workspace, monkeypatch): + """Kaggle rations hours. The zero is a fact about the backend rather than a + missing measurement, and `hours` is what bounds a campaign here.""" + stub_kaggle(monkeypatch, marker={"exit_code": 0}, hours=1.25) + result = run_kaggle(workspace) + assert result["cost_usd"] == 0.0 + assert result["hours"] == 1.25 + assert result["accelerator_kind"] in ("gpu", "tpu", "cpu") + + +def test_a_kernel_with_no_marker_is_not_an_exit_code(workspace, monkeypatch): + """A kernel that died before the marker cell has a real outcome and no exit + code. Reporting one would be a number nobody measured.""" + stub_kaggle(monkeypatch, status="error", marker={}, hours=None, log="boom") + result = run_kaggle(workspace) + + assert result["ok"] is False + assert result["exit_code"] is None + assert "without recording an outcome" in result["error"] + + +def test_a_kernel_that_never_leaves_the_queue_is_not_a_bad_mutation(workspace, monkeypatch): + stub_kaggle(monkeypatch, status="queued", marker={"exit_code": 0}) + result = run_kaggle(workspace, timeout_s=1) + + assert result["ok"] is False + assert result["exit_code"] is None + assert "still queued" in result["error"] + + +def test_kaggle_candidates_count_against_the_weekly_allowance(workspace): + """Candidates never reach `runs.jsonl`, so without the candidate fold a + campaign would burn real GPU hours the allowance cannot see -- surfacing as + an ordinary submission refused for hours nothing accounts for.""" + camp.append_candidate( + { + "campaign": "camp-1", + "candidate_id": "camp-1-g0-c0", + "generation": 0, + "index": 0, + "at": camp.now_iso(), + "backend": "kaggle", + "metrics": {"combined_score": 1.0}, + kaggle_quota.F_ACCELERATOR: "NvidiaTeslaP100", + kaggle_quota.F_KIND: "gpu", + kaggle_quota.F_ACTUAL: 3.5, + } + ) + + pools = kaggle_quota.accelerator_hours(kind="gpu")["pools"] + assert pools["gpu"]["total_hours"] == 3.5 + assert pools["gpu"]["runs"][0]["campaign"] == "camp-1" + + +def test_a_local_campaigns_candidates_do_not_touch_the_allowance(workspace): + """The fold keys on the backend, so a local campaign -- which spends no + Kaggle hours at all -- must not appear in the pool.""" + camp.append_candidate( + { + "campaign": "camp-2", + "candidate_id": "camp-2-g0-c0", + "generation": 0, + "index": 0, + "at": camp.now_iso(), + "metrics": {"combined_score": 1.0}, + "duration_s": 3600, + } + ) + assert kaggle_quota.accelerator_hours(kind="gpu")["pools"] == {} + + +# --------------------------------------------------------------------------- +# HF Jobs: the pipeline is in the image, so the candidate needs a way in +# --------------------------------------------------------------------------- +def test_the_candidate_travels_in_an_environment_variable(workspace): + blob = jobs_tool._candidate_blob({"initial.py": "x = 1", "evaluate.py": "print('{}')"}) + assert unpack(blob) == {"initial.py": "x = 1", "evaluate.py": "print('{}')"} + + +def test_the_hf_blob_is_a_function_of_its_inputs(workspace): + files = {"initial.py": "x = 1"} + assert jobs_tool._candidate_blob(files) == jobs_tool._candidate_blob(files) + + +def test_a_candidate_that_is_really_a_pipeline_is_refused(workspace): + """This is a container environment variable, not a file. The ceiling is the + platform's, undocumented, and discovering it by exceeding it means a job + that fails for a reason with nothing to do with the research.""" + # Incompressible, so it cannot slip under the limit by gzipping well. + bulk = secrets.token_hex(400_000) + with pytest.raises(UsageError) as exc: + jobs_tool._candidate_blob({"initial.py": bulk}) + assert "evolve block to code" in (exc.value.fix or "") + + +def test_an_hf_candidate_file_may_not_be_a_path(workspace): + for bad in ("../escape.py", "/etc/passwd"): + with pytest.raises(UsageError): + jobs_tool._candidate_blob({bad: "x"}) + + +def test_the_unpack_runs_before_the_command_and_gates_it(workspace): + """A failed unpack has to fail the job. Otherwise it runs the image's *own* + entrypoint and reports a score for the wrong program -- which would not look + like an error, it would look like every candidate scoring the same.""" + argv = jobs_tool._candidate_command(["python", "evaluate.py"]) + assert argv[0] == "sh" and argv[1] == "-c" + assert "&&" in argv[2] + assert argv[2].index("tarfile") < argv[2].index("evaluate.py") + + +def stub_hf(monkeypatch, *, state="COMPLETED", logs="", raises=None): + sent: dict = {} + + class FakeHub: + def run_job(self, **kwargs): + if raises is not None: + raise raises + sent.update(kwargs) + return type("J", (), {"id": "job-1"})() + + monkeypatch.setattr(jobs_tool, "_hub", lambda: FakeHub()) + monkeypatch.setattr(jobs_tool, "_token", lambda: "t") + monkeypatch.setattr(jobs_tool, "_ns_kwargs", lambda ns: {}) + monkeypatch.setattr(jobs_tool, "resolve_namespace", lambda *a, **k: "org") + monkeypatch.setattr(jobs_tool, "flavor_rate", lambda flavor, cfg: 1.0) + monkeypatch.setattr(jobs_tool, "_poll", lambda job_id, deadline, namespace=None: (state, {})) + monkeypatch.setattr(jobs_tool, "_logs", lambda job_id, namespace=None: logs) + monkeypatch.setattr(jobs_tool, "_actual_cost", lambda *a, **k: (0.4, None)) + return sent + + +def run_hf(workspace, **kwargs): + options = { + "candidate_id": "c1", + "files": {"initial.py": "x = 1"}, + "command": ["python", "evaluate.py"], + "timeout_s": 900, + "artifacts": workspace / "artifacts", + } + options.update(kwargs) + return jobs_tool.evaluate_candidate(a_submission(workspace), config_mod.load(), **options) + + +def test_the_blob_and_the_candidate_id_reach_the_job(workspace, monkeypatch): + sent = stub_hf(monkeypatch, logs='{"combined_score": 2}') + result = run_hf(workspace) + + assert jobs_tool.CANDIDATE_ENV in sent["env"] + assert sent["env"]["GRAD_CANDIDATE"] == "c1" + assert unpack(sent["env"][jobs_tool.CANDIDATE_ENV]) == {"initial.py": "x = 1"} + assert result["ok"] is True + assert result["output"].endswith('{"combined_score": 2}') + + +def test_an_hf_candidate_reports_a_state_not_an_invented_exit_code(workspace, monkeypatch): + stub_hf(monkeypatch, state="ERROR", logs="Traceback") + result = run_hf(workspace) + + assert result["ok"] is False + assert result["exit_code"] is None, "HF reports a state; an exit code would be invented" + assert result["job_state"] == "ERROR" + assert result["cost_usd"] == 0.4 + + +def test_the_job_is_bounded_where_it_runs(workspace, monkeypatch): + sent = stub_hf(monkeypatch) + run_hf(workspace, timeout_s=1200) + assert sent["timeout"] == 1200 + + +def test_a_refused_hf_submission_is_not_a_bad_mutation(workspace, monkeypatch): + stub_hf(monkeypatch, raises=RuntimeError("402 payment required")) + result = run_hf(workspace) + + assert result["ok"] is False + assert result["exit_code"] is None + assert "could not be submitted" in result["error"] diff --git a/tests/test_desktop_app.py b/tests/test_desktop_app.py index 9e53d1c..4e27f12 100644 --- a/tests/test_desktop_app.py +++ b/tests/test_desktop_app.py @@ -726,3 +726,401 @@ def _only(window_id: str, space): from ui import layout as layout_mod return layout_mod.Layout.default([window_id]) + + +# --------------------------------------------------------------------------- +# where the window was +# --------------------------------------------------------------------------- +@pytest.fixture +def fresh_geometry(monkeypatch): + """Empty the module's observation of the window between tests. + + `_geometry` is process-wide because there is one window per process, which + is true in production and would otherwise let one test decide another's + outcome -- the same reason `clean_process_state` exists in `conftest.py`. + """ + monkeypatch.setattr(desktop, "_geometry", {}) + monkeypatch.setattr(desktop, "_previous", {}) + monkeypatch.setattr(desktop, "_saved_at", 0.0) + yield + + +def _screens(monkeypatch, *rects): + monkeypatch.setattr(desktop, "_screens", lambda: list(rects)) + + +def test_a_window_nobody_has_moved_opens_at_the_default(workspace, fresh_geometry): + """No file, no position: the size is the documented default and pywebview is + left to place it, exactly as before any of this existed.""" + args = desktop.window_args() + assert (args["width"], args["height"]) == desktop.DEFAULT_SIZE + assert "x" not in args and "y" not in args + + +def test_the_window_reopens_where_it_was_left(workspace, fresh_geometry, monkeypatch): + """The whole feature in one test: move it, and the next launch is told to + put it back.""" + _screens(monkeypatch, (0, 0, 2560, 1440)) + desktop.remember(x=300, y=120) + desktop.remember(width=1200, height=800) + desktop.save_geometry(force=True) + + args = desktop.window_args() + assert (args["x"], args["y"]) == (300, 120) + assert (args["width"], args["height"]) == (1200, 800) + + +def test_a_position_on_a_screen_that_is_gone_is_dropped(workspace, fresh_geometry, monkeypatch): + """The failure this must not have. A saved position is a promise about a + monitor arrangement, and undocking breaks the promise -- restoring it puts + the window where there are no pixels, running and unreachable.""" + _screens(monkeypatch, (0, 0, 2560, 1440)) + desktop.remember(x=3000, y=200, width=1200, height=800) + desktop.save_geometry(force=True) + + args = desktop.window_args() + assert "x" not in args and "y" not in args, "restored onto a screen that is not there" + # The size is still honoured: it was never the part that stopped being true. + assert (args["width"], args["height"]) == (1200, 800) + + +def test_a_corner_on_a_screen_is_enough(workspace, fresh_geometry, monkeypatch): + """A window hanging mostly off the edge is where someone put it, and its + title bar is still grabbable. Only 'no pixels at all' is the broken case.""" + _screens(monkeypatch, (0, 0, 1920, 1080)) + assert desktop.on_screen(1800, 1000, 1200, 800) + assert not desktop.on_screen(1900, 1040, 1200, 800) + + +def test_a_second_screen_left_of_the_first_is_still_a_screen(workspace, fresh_geometry, monkeypatch): + """Negative coordinates are ordinary on Windows: a monitor placed to the + left of the primary one has them, and a naive `x >= 0` check would refuse to + restore onto it.""" + _screens(monkeypatch, (0, 0, 1920, 1080), (-1920, 0, 1920, 1080)) + assert desktop.on_screen(-1500, 100, 1200, 800) + + +def test_unknown_screens_do_not_veto_a_restore(workspace, fresh_geometry, monkeypatch): + """A backend that cannot enumerate displays must not turn 'remember the + position' off; it would fail closed on exactly the machines we cannot + check.""" + _screens(monkeypatch) + assert desktop.on_screen(-4000, -4000, 1200, 800) + + +def test_maximizing_does_not_eat_the_size_to_restore_to(workspace, fresh_geometry, monkeypatch): + """Maximizing moves and resizes the window, and pywebview reports both as + ordinary events. Recording them would restore a screen-sized window at 0,0 + on the next launch and lose the size the user actually chose.""" + _screens(monkeypatch, (0, 0, 2560, 1440)) + desktop.remember(x=300, y=120, width=1200, height=800) + # What the maximize looks like coming over the wire, worst case: the + # rectangle arrives first and the flag second. + desktop.remember(x=0, y=0, width=2560, height=1440) + desktop._note_maximized(True) + desktop.save_geometry(force=True) + + args = desktop.window_args() + assert args.get("maximized") is True + assert (args["width"], args["height"]) == (1200, 800), "the chosen size was overwritten" + assert (args["x"], args["y"]) == (300, 120) + + +def test_moving_a_maximized_window_is_not_a_choice_to_remember(workspace, fresh_geometry): + """Once maximized, the rectangle belongs to the window manager. The one to + restore to is the one from before.""" + desktop.remember(x=300, y=120, width=1200, height=800) + desktop._note_maximized(True) + desktop.remember(x=0, y=0, width=2560, height=1440) + + assert (desktop._geometry["width"], desktop._geometry["height"]) == (1200, 800) + + +def test_unmaximizing_clears_the_flag(workspace, fresh_geometry, monkeypatch): + _screens(monkeypatch, (0, 0, 2560, 1440)) + desktop.remember(x=300, y=120, width=1200, height=800) + desktop._note_maximized(True) + desktop._note_maximized(False) + desktop.save_geometry(force=True) + + assert "maximized" not in desktop.window_args() + + +def test_a_sliver_is_not_restored(workspace, fresh_geometry, monkeypatch): + """A window dragged down to nothing reopening at nothing looks like an app + that failed to start.""" + _screens(monkeypatch, (0, 0, 2560, 1440)) + desktop.remember(x=10, y=10, width=12, height=8) + desktop.save_geometry(force=True) + + args = desktop.window_args() + assert (args["width"], args["height"]) == desktop.MIN_SIZE + + +def test_a_corrupt_geometry_file_is_not_a_startup_failure(workspace, fresh_geometry): + """This file survives upgrades and can be edited by hand, and it is read at + the one moment where there is no UI to report a problem with.""" + path = desktop.geometry_path() + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("{not json at all", encoding="utf-8") + + assert desktop.read_geometry() == {} + assert (desktop.window_args()["width"], desktop.window_args()["height"]) == desktop.DEFAULT_SIZE + + +def test_junk_fields_are_rederived_rather_than_trusted(workspace, fresh_geometry): + """`create_window` is handed these directly, so a string where an int + belongs is a crash before there is a window to see it in.""" + from core import jsonl + + path = desktop.geometry_path() + path.parent.mkdir(parents=True, exist_ok=True) + jsonl.write_json(path, {"x": "left", "y": None, "width": 1200, "height": 800, "maximized": "yes"}) + + saved = desktop.read_geometry() + assert saved == {"width": 1200, "height": 800} + assert "maximized" not in desktop.window_args() + + +def test_the_save_is_throttled_but_the_last_word_is_not(workspace, fresh_geometry, monkeypatch): + """`moved` fires per pixel of a drag. The throttle drops the middle of a + burst; `force` is what makes sure it does not drop the end of one.""" + writes = [] + from core import jsonl + + monkeypatch.setattr(jsonl, "write_json", lambda path, obj: writes.append(dict(obj))) + + desktop.remember(x=1, y=1) + desktop.remember(x=2, y=2) + desktop.remember(x=3, y=3) + assert len(writes) == 1, "the throttle did not hold" + + desktop.save_geometry(force=True) + assert writes[-1]["x"] == 3 + + +def test_the_geometry_is_actually_handed_to_the_window(workspace, fresh_geometry, monkeypatch): + """The connecting wire for the geometry, tested for the reason the veto's is: + every test above passes with `window_args` never reaching `create_window`. + + It asserts the *merge order* as well as the values, because that is the part + that is easy to get backwards -- NiceGUI splices `native.window_args` in + after its own `width`/`height`, so these win over `ui.run(window_size=...)`. + A version that merged the other way would restore the position and silently + ignore the size. + """ + from nicegui import app as nicegui_app + + from ui import app as grad_app + + _screens(monkeypatch, (0, 0, 2560, 1440)) + desktop.remember(x=300, y=120, width=1200, height=800) + desktop.save_geometry(force=True) + + nicegui_app.native.window_args.clear() + grad_app._install_desktop(True) + + merged = {"width": 800, "height": 600, **nicegui_app.native.window_args} + assert (merged["x"], merged["y"]) == (300, 120) + assert (merged["width"], merged["height"]) == (1200, 800) + nicegui_app.native.window_args.clear() + + +def test_browser_mode_is_handed_no_window_at_all(workspace, fresh_geometry): + """`native=False` is the documented fallback and there is no window to place. + Setting these there would turn native mode back on -- the same trap the + `window_size` comment in `ui/app.py:run` already records.""" + from nicegui import app as nicegui_app + + from ui import app as grad_app + + nicegui_app.native.window_args.clear() + grad_app._install_desktop(False) + assert nicegui_app.native.window_args == {} + + +def test_the_window_events_are_subscribed_before_the_bridge_starts(workspace, fresh_geometry): + """`ui.run` is what starts the event manager that delivers these, so a + registration after it would arrive too late and nothing would ever be saved. + Asserted against NiceGUI's own registry rather than ours.""" + from nicegui import app as nicegui_app + from nicegui.native.event_manager import event_manager + + from ui import app as grad_app + + for name in ("moved", "resized", "maximized", "restored", "closed"): + event_manager._handlers.pop(name, None) + nicegui_app.native.window_args.clear() + grad_app._install_desktop(True) + + for name in ("moved", "resized", "maximized", "restored", "closed"): + assert event_manager._handlers.get(name), f"nothing is listening for {name}" + nicegui_app.native.window_args.clear() + + +# --------------------------------------------------------------------------- +# the loading mark +# --------------------------------------------------------------------------- +class _FakePipe: + def __init__(self) -> None: + self.closed = False + + def close(self) -> None: + self.closed = True + + +class _FakeProc: + """A splash process that stops when its pipe is closed, as the real one does.""" + + def __init__(self, argv, **kwargs) -> None: + self.argv = list(argv) + self.kwargs = kwargs + self.stdin = _FakePipe() + self.terminated = False + self.pid = 4242 + + def wait(self, timeout=None): + if not self.stdin.closed: + import subprocess as sp + + raise sp.TimeoutExpired(self.argv, timeout or 0) + return 0 + + def poll(self): + return 0 if self.stdin.closed else None + + def terminate(self) -> None: + self.terminated = True + + +@pytest.fixture +def fake_splash(monkeypatch): + """Never spawn a real window in the suite -- §24, and a Tk loop in CI.""" + import subprocess + + from ui import splash + + made: list[_FakeProc] = [] + + def popen(argv, **kwargs): + proc = _FakeProc(argv, **kwargs) + made.append(proc) + return proc + + monkeypatch.setattr(subprocess, "Popen", popen) + monkeypatch.setattr(splash, "_process", None) + yield made + monkeypatch.setattr(splash, "_process", None) + + +def test_the_loading_mark_is_a_separate_process(workspace, fake_splash): + """A thread would share the interpreter that is busy importing NiceGUI, and + a Tk window that does not paint is worse than no window -- Windows greys it + out and offers to close it.""" + from ui import splash + + splash.start() + assert len(fake_splash) == 1 + argv = fake_splash[0].argv + assert argv[1:3] == ["-m", "ui.splash"] + assert argv[0].endswith(("python.exe", "pythonw.exe", "python", "python3")) + + +def test_the_pipe_is_what_the_child_watches(workspace, fake_splash): + """The liveness channel. It has to be a pipe the parent holds open, because + EOF on it is the one signal that also arrives when the parent is killed.""" + import subprocess + + from ui import splash + + splash.start() + assert fake_splash[0].kwargs.get("stdin") is subprocess.PIPE + # And the child is told to watch it. Without the flag it ignores stdin + # entirely, which is what makes running the module by hand -- where stdin is + # a console or an already-closed handle -- not exit instantly. + assert "--watch-stdin" in fake_splash[0].argv + + +def test_stopping_closes_the_pipe_rather_than_killing(workspace, fake_splash): + """Closing is the same signal a crash sends, so there is one path to test + instead of two. `terminate` is only for a child that stopped reading.""" + from ui import splash + + splash.start() + proc = fake_splash[0] + splash.stop() + + assert proc.stdin.closed + assert not proc.terminated + assert not splash.running() + + +def test_a_child_that_will_not_go_is_terminated(workspace, fake_splash, monkeypatch): + from ui import splash + + splash.start() + proc = fake_splash[0] + monkeypatch.setattr(proc.stdin, "close", lambda: None) + splash.stop() + + assert proc.terminated + + +def test_starting_twice_shows_one_mark(workspace, fake_splash): + from ui import splash + + splash.start() + splash.start() + assert len(fake_splash) == 1 + + +def test_stopping_what_was_never_started_is_fine(workspace, fake_splash): + """`stop` runs from a connect handler, which fires in browser mode too -- + where nothing ever put a mark up.""" + from ui import splash + + splash.stop() + assert not splash.running() + + +def test_a_launch_that_cannot_spawn_still_launches(workspace, monkeypatch): + """Every reason this fails -- no display, no Tk, no permission to spawn -- + means the app starts exactly as it did before this existed.""" + import subprocess + + from ui import splash + + monkeypatch.setattr(splash, "_process", None) + monkeypatch.setattr( + subprocess, "Popen", lambda *a, **k: (_ for _ in ()).throw(OSError("nope")) + ) + splash.start() + assert not splash.running() + + +def test_the_mark_is_the_same_drawing_as_the_icon(workspace): + """`write_icon`'s rule: there is one drawing of this glyph and everything + reads it. A splash with its own hand-drawn nabla is the drift it prevents.""" + png = desktop.splash_png(96) + if png is None: + pytest.skip("Pillow is not installed") + assert Path(png).is_file() + assert Path(png).stat().st_size > 0 + # Cached, not redrawn: the launch path is the one place a couple of hundred + # milliseconds of Pillow is worth avoiding. + before = Path(png).stat().st_mtime_ns + assert desktop.splash_png(96) == png + assert Path(png).stat().st_mtime_ns == before + + +def test_the_mark_comes_down_when_a_client_connects(workspace): + """On connect, not on startup: the server listens several seconds before the + webview has rendered anything, and taking it down there puts the gap back.""" + from nicegui import app as nicegui_app + + from ui import app as grad_app + + before = len(nicegui_app._connect_handlers) + grad_app._install_desktop(True) + assert len(nicegui_app._connect_handlers) > before + nicegui_app.native.window_args.clear() diff --git a/tests/test_effort.py b/tests/test_effort.py index bfa1d0e..993536c 100644 --- a/tests/test_effort.py +++ b/tests/test_effort.py @@ -165,4 +165,85 @@ async def test_a_change_is_deferred_when_there_is_no_session_to_resume(workspace effort.set_current("low") session = _FakeSession(sdk_session_id=None, client_effort="auto") assert await Session.apply_effort(session) is False + + +# --------------------------------------------------------------------------- +# the same mechanism, for the model a project chose +# --------------------------------------------------------------------------- +class _FakeModelSession: + """`apply_model`'s two methods, and a record of the calls. The same shape as + `_FakeSession` above, because it is the same lazy-rebuild argument.""" + + def __init__(self, *, client=_A_CLIENT, sdk_session_id="sdk-1", client_model="claude-opus-5"): + self.client = client + self.sdk_session_id = sdk_session_id + self.client_model = client_model + self.calls: list[str] = [] + + async def close(self): + self.calls.append("close") + self.client = None + self.client_model = None + + async def start(self): + from core import config as config_mod + + self.calls.append("start") + self.client = _A_CLIENT + self.client_model = config_mod.load().model_for("research") + + +async def _apply_model(session): + from ui.app import Session + + return await Session.apply_model(session) + + +@pytest.mark.asyncio +async def test_a_project_that_overrides_research_rebuilds_the_client(workspace): + """The consequence that is easy to miss: `ClaudeSDKClient` options are built + once, at client start. Switching to a project that overrides `research` while + a session is live would otherwise leave the previous model answering -- + silently, and while every other surface says something different.""" + from core import budget, config as config_mod + + budget.create("proj-a", title="A", budget={}) + budget.set_current("proj-a") + budget.configure("proj-a", models={"research": "claude-haiku-4-5"}) + + session = _FakeModelSession(client_model="claude-opus-5") + assert await _apply_model(session) is True + assert session.calls == ["close", "start"] + assert session.client_model == config_mod.load().model_for("research") == "claude-haiku-4-5" + + +@pytest.mark.asyncio +async def test_a_project_on_the_model_already_running_rebuilds_nothing(workspace): + from core import config as config_mod + + session = _FakeModelSession(client_model=config_mod.load().model_for("research")) + assert await _apply_model(session) is False + assert session.calls == [] + + +@pytest.mark.asyncio +async def test_no_client_means_the_next_start_picks_the_model_up_for_free(workspace): + session = _FakeModelSession(client=None, client_model=None) + assert await _apply_model(session) is False assert session.calls == [] + + +@pytest.mark.asyncio +async def test_a_model_change_with_no_session_id_is_deferred_not_paid_for(workspace): + """A rebuild with nothing to resume would silently start a new conversation. + The session has said nothing yet in any case -- the id arrives with the first + turn.""" + from core import budget + + budget.create("proj-a", title="A", budget={}) + budget.set_current("proj-a") + budget.configure("proj-a", models={"research": "claude-haiku-4-5"}) + + session = _FakeModelSession(sdk_session_id=None, client_model="claude-opus-5") + assert await _apply_model(session) is False + assert session.calls == [], "a deferred change must not have cost a rebuild" diff --git a/tests/test_evolve.py b/tests/test_evolve.py index 6773c4d..c2c9d61 100644 --- a/tests/test_evolve.py +++ b/tests/test_evolve.py @@ -108,7 +108,8 @@ def make_expectation(quantity="combined_score"): def run_args(task_dir, expectation_id, **overrides): base = dict( task_dir=str(task_dir), expect=expectation_id, project=None, generations=2, - population=2, estimate_per_candidate_usd=0.0, local=True, remote=False, + population=2, estimate_per_candidate_usd=0.0, local=True, remote=None, + remote_spec=None, remote_timeout_s=0, overrides=[], timeout_s=30, json=True, # The search knobs. `islands=1` and no migration by default so the tests # that are about the *gate* are not also about the selection policy; @@ -402,12 +403,22 @@ def test_promoting_an_unevaluated_candidate_refuses(workspace, monkeypatch): # --------------------------------------------------------------------------- # phasing and scaffolding # --------------------------------------------------------------------------- -def test_remote_is_refused_in_phase_one(workspace): - """"Do not run a single remote generation before this exists." """ +def test_remote_needs_a_spec_to_be_remote_on(workspace): + """There is no such thing as 'the remote' in general, only a pipeline that + has been proven on one.""" task_dir = scaffold(workspace) with pytest.raises(UsageError) as exc: - evolve.cmd_run(run_args(task_dir, make_expectation(), remote=True)) - assert "phase 2" in str(exc.value) + evolve.cmd_run(run_args(task_dir, make_expectation(), remote="ssh")) + assert "--remote-spec" in str(exc.value.fix or "") + + +def test_a_spec_with_no_backend_is_refused(workspace): + """The mirror of the above: naming an environment without saying what to run + it on is a flag that would otherwise be silently ignored.""" + task_dir = scaffold(workspace) + with pytest.raises(UsageError) as exc: + evolve.cmd_run(run_args(task_dir, make_expectation(), remote_spec="pipeline/spec.toml")) + assert "no --remote backend" in str(exc.value) def test_a_task_without_markers_is_refused(workspace): diff --git a/tests/test_evolve_remote.py b/tests/test_evolve_remote.py new file mode 100644 index 0000000..e2b4fac --- /dev/null +++ b/tests/test_evolve_remote.py @@ -0,0 +1,614 @@ +"""Evolve phase 2: candidates evaluated on real hardware. + +Phase 1 was local-only on purpose -- prove the campaign records, the sub-run +bookkeeping and the budget gate while the blast radius is zero -- and these are +the tests for the thing that was being deferred until then. Two properties carry +most of the weight: + + * **the gate holds before generation 0.** A search is a loop with no human in + it, so the environment it lands in must already have a complete, passing + preflight *including the smoke run*, checked once, before anything is spent. + * **a candidate still never becomes a run.** §23 item 4 is the reason + candidates live in `candidates.jsonl`, and it would be undone the moment the + search left this machine if each remote evaluation wrote a ledger row. + +Nothing here opens an ssh connection. `tools/gpu.py:evaluate_candidate` is the +seam, and it is stubbed -- the point under test is what the driver does with +what a host says, not `ssh` itself. +""" + +from __future__ import annotations + +import pytest + +from core import campaign as camp, config as config_mod, jsonl, ledger_store as ls, paths +from core.errors import GateRefusal, GradError, UsageError +from core.submission import Submission +from tools import evolve + +from test_evolve import FakeMutator, make_expectation, run_args, scaffold + + +# --------------------------------------------------------------------------- +# fixtures of the world a remote campaign needs +# --------------------------------------------------------------------------- +def remote_spec( + workspace, + *, + host: str = "gpu-box", + hours: float = 0.1, + accelerator: str | None = None, + flavor: str | None = None, +) -> Submission: + """A pipeline spec, plus the config a campaign on it needs. + + One helper for all three backends: the spec names a host, and optionally an + accelerator or a flavor, so a test can point the same pipeline at whichever + backend it is about. + """ + directory = workspace / "pipeline" + directory.mkdir(parents=True, exist_ok=True) + (directory / "train.py").write_text("print('x')\n", encoding="utf-8") + target = [f"host = '{host}'"] + if accelerator: + target.append(f"accelerator = '{accelerator}'") + if flavor: + target.append(f"flavor = '{flavor}'") + (directory / "spec.toml").write_text( + "entrypoint = 'train.py'\nimage = 'org/img@sha256:aaaa'\n" + "[target]\n" + "\n".join(target) + "\n" + f"[estimate]\nhours = {hours}\nrate_usd_per_hour = 1.0\n", + encoding="utf-8", + ) + config_path = workspace / "config" / "grad.toml" + config_path.parent.mkdir(parents=True, exist_ok=True) + config_path.write_text( + f"[hosts.{host}]\nhostname = '10.0.0.7'\nuser = 'research'\n" + "workdir = '~/grad'\nrate_usd_per_hour = 2.0\n" + "\n[hf]\ndefault_flavor = 'a10g-small'\n" + "\n[hf.flavor_rates]\n'a10g-small' = 1.0\n" + "\n[kaggle]\ndefault_accelerator = 'NvidiaTeslaP100'\n" + "\n[kaggle.quota]\ngpu_hours_per_week = 30.0\nmax_session_hours = 12.0\n" + "\n[kaggle.accelerators]\nNvidiaTeslaP100 = 'gpu'\n", + encoding="utf-8", + ) + config_mod._cache.clear() + return Submission.load(directory / "spec.toml", resolve_digest=False) + + +def resolve_target(workspace, sub, **overrides): + """Just the gate and the resolution, without running a campaign.""" + args = campaign_args(workspace, sub, **overrides) + return evolve._remote_target(args, config_mod.load()) + + +def hostless_spec(workspace) -> Submission: + directory = workspace / "pipeline" + directory.mkdir(parents=True, exist_ok=True) + (directory / "train.py").write_text("print('x')\n", encoding="utf-8") + (directory / "spec.toml").write_text( + "entrypoint = 'train.py'\nimage = 'org/img@sha256:aaaa'\n" + "[estimate]\nhours = 0.1\nrate_usd_per_hour = 1.0\n", + encoding="utf-8", + ) + config_mod._cache.clear() + return Submission.load(directory / "spec.toml", resolve_digest=False) + + +def preflight(sub: Submission, **checks: bool) -> None: + """Write a preflight record for this exact submission hash.""" + results = {"tests": True, "dry_run": True, "smoke": True} + results.update(checks) + jsonl.write_json( + paths.preflight_record(sub.hash()), + { + "submission_hash": sub.hash(), + "verified_at": ls.now_iso(), + "checks": {name: {"ok": ok} for name, ok in results.items()}, + }, + ) + + +def missing_smoke(sub: Submission) -> None: + jsonl.write_json( + paths.preflight_record(sub.hash()), + { + "submission_hash": sub.hash(), + "verified_at": ls.now_iso(), + "checks": {"tests": {"ok": True}, "dry_run": {"ok": True}}, + }, + ) + + +def host_answers(monkeypatch, answer): + """Stand in for the whole ssh side. `answer(candidate_id) -> dict`. + + Returns the list of calls, so a test can assert on what actually reached the + host rather than only on what came back. + """ + from tools import gpu as gpu_tool + + seen: list[dict] = [] + + def evaluate_candidate(sub, cfg, *, candidate_id, files, command, timeout_s, host=None): + seen.append( + { + "candidate": candidate_id, + "files": dict(files), + "command": list(command), + "timeout_s": timeout_s, + } + ) + return answer(candidate_id) + + monkeypatch.setattr(gpu_tool, "evaluate_candidate", evaluate_candidate) + return seen + + +def scored(candidate_id: str, *, score: float = 1.5, cost: float = 0.05) -> dict: + return { + "ok": True, + "exit_code": 0, + "cost_usd": cost, + "host": "gpu-box", + "where": f"gpu-box:~/grad/{candidate_id}", + "error": None, + "output": f'training noise\n{{"combined_score": {score}, "abs_error": 2}}\nEXIT:0', + } + + +def campaign_args(workspace, sub, **overrides): + base = dict( + remote="ssh", + remote_spec=str(sub.spec_path), + generations=1, + population=2, + ) + base.update(overrides) + return run_args(scaffold(workspace), make_expectation(), **base) + + +def drive(workspace, sub, monkeypatch, **overrides): + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator()) + return evolve.cmd_run(campaign_args(workspace, sub, **overrides)) + + +# --------------------------------------------------------------------------- +# the gate +# --------------------------------------------------------------------------- +def test_a_remote_campaign_refuses_without_a_preflight(workspace): + """The environment is proven before generation 0, not rediscovered forty + times at a dollar apiece.""" + sub = remote_spec(workspace) + with pytest.raises(GateRefusal) as exc: + evolve.cmd_run(campaign_args(workspace, sub)) + assert exc.value.code == "preflight_missing" + assert "preflight run" in (exc.value.fix or "") + + +def test_a_remote_campaign_refuses_without_a_smoke_run(workspace): + """Specifically smoke, and specifically not read from `[preflight] checks`: + a machine configured without it would otherwise let a campaign put every + candidate it has on hardware nothing had ever run one step on.""" + sub = remote_spec(workspace) + missing_smoke(sub) + with pytest.raises(GateRefusal) as exc: + evolve.cmd_run(campaign_args(workspace, sub)) + assert "smoke missing" in exc.value.message + + +def test_a_failing_smoke_is_not_a_passing_preflight(workspace): + sub = remote_spec(workspace) + preflight(sub, smoke=False) + with pytest.raises(GateRefusal) as exc: + evolve.cmd_run(campaign_args(workspace, sub)) + assert "smoke failed" in exc.value.message + + +def test_a_preflight_for_a_different_submission_does_not_transfer(workspace): + """The record is keyed by submission hash, so editing the pipeline after + proving it puts the campaign back behind the gate.""" + sub = remote_spec(workspace) + preflight(sub) + (workspace / "pipeline" / "train.py").write_text("print('changed')\n", encoding="utf-8") + config_mod._cache.clear() + + with pytest.raises(GateRefusal) as exc: + evolve.cmd_run(campaign_args(workspace, sub)) + assert exc.value.code == "preflight_missing" + + +def test_the_gate_runs_before_the_expectation_is_bound(workspace): + """A configuration refusal must not cost an expectation. They are + single-use by §7, so being refused for a missing preflight would otherwise + burn one and force a re-mint before anything could be retried.""" + sub = remote_spec(workspace) + expectation = make_expectation() + args = run_args( + scaffold(workspace), expectation, remote="ssh", remote_spec=str(sub.spec_path) + ) + + with pytest.raises(GateRefusal): + evolve.cmd_run(args) + assert expectation not in ls.consumed_expectation_ids() + + +def test_a_spec_with_no_host_is_a_configuration_error(workspace): + sub = hostless_spec(workspace) + preflight(sub) + with pytest.raises(GradError) as exc: + evolve.cmd_run(campaign_args(workspace, sub)) + assert "no [target] host" in exc.value.message + + +def test_an_unknown_host_is_refused_before_generation_zero(workspace): + """Not forty evaluations in. An unknown host is a configuration error and + the campaign gate is where configuration errors belong.""" + sub = remote_spec(workspace, host="gpu-box") + preflight(sub) + (workspace / "config" / "grad.toml").write_text("", encoding="utf-8") + config_mod._cache.clear() + + with pytest.raises(GradError): + evolve.cmd_run(campaign_args(workspace, sub)) + + +def test_an_hf_campaign_refuses_a_flavor_nothing_prices(workspace): + """An unpriced flavor makes the campaign's projected cost a fiction, and the + campaign budget gate is the only thing between a search and an allocation. + Refused before generation 0 rather than booked as free.""" + sub = remote_spec(workspace, flavor="h200-quantum") + preflight(sub) + with pytest.raises(GradError) as exc: + evolve.cmd_run(campaign_args(workspace, sub, remote="hf_jobs")) + assert "flavor_rates" in exc.value.message + + +def test_a_kaggle_campaign_refuses_what_will_not_fit_the_week(workspace): + """Kaggle rations hours, not money, so a campaign priced at zero sails + through the dollar gate and would then spend the whole weekly allowance -- + surfacing as an ordinary submission refused for hours nothing accounts for.""" + sub = remote_spec(workspace, accelerator="NvidiaTeslaP100", hours=4.0) + preflight(sub) + with pytest.raises(GateRefusal) as exc: + # 10 generations x 4 candidates x 4h = 160h against a 30h week. + evolve.cmd_run( + campaign_args(workspace, sub, remote="kaggle", generations=10, population=4) + ) + assert exc.value.code == "quota_weekly" + assert "does not fit the week" in exc.value.message + # The campaign's own shape, because "this run estimates 160h" is a confusing + # way to describe forty four-hour candidates. + assert "40 candidates at 4.00h each" in exc.value.message + + +def test_the_session_cap_is_asked_about_one_candidate_not_the_campaign(workspace): + """The two Kaggle ceilings take different numbers. Handing the session cap + the campaign total would refuse an ordinary search of twenty one-hour + candidates for exceeding a twelve-hour session.""" + sub = remote_spec(workspace, accelerator="NvidiaTeslaP100", hours=1.0) + preflight(sub) + + # 20 candidates x 1h = 20h: inside the 30h week, and each one inside a 12h + # session. A gate that conflated the two would refuse this. + target = resolve_target(workspace, sub, remote="kaggle", generations=5, population=4) + assert target["backend"] == "kaggle" + assert target["accelerator_kind"] == "gpu" + + +def test_an_hf_campaign_resolves_and_prices_its_flavor(workspace): + sub = remote_spec(workspace, flavor="a10g-small") + preflight(sub) + target = resolve_target(workspace, sub, remote="hf_jobs") + assert target["flavor"] == "a10g-small" + assert target["rate_usd_per_hour"] == 1.0 + + +def test_a_single_candidate_past_the_session_cap_is_refused(workspace): + """Kaggle stops the kernel at the cap and hands back whatever it wrote, so a + candidate estimated past it has already been decided to fail.""" + sub = remote_spec(workspace, accelerator="NvidiaTeslaP100", hours=20.0) + preflight(sub) + with pytest.raises(GateRefusal) as exc: + evolve.cmd_run( + campaign_args(workspace, sub, remote="kaggle", generations=1, population=1) + ) + # Passed through from `kaggle_quota` untouched: its message already names + # the cap and says what happens to a kernel that hits it. + assert exc.value.code == "quota_session" + assert "single GPU session" in exc.value.message + + +def test_a_backend_without_a_spec_is_refused(workspace): + with pytest.raises(UsageError) as exc: + evolve.cmd_run(run_args(scaffold(workspace), make_expectation(), remote="ssh")) + assert "--remote-spec" in (exc.value.fix or "") + + +def test_a_spec_without_a_backend_is_refused(workspace): + """The mirror. A flag that names an environment and is then silently + ignored is worse than one that refuses.""" + with pytest.raises(UsageError) as exc: + evolve.cmd_run( + run_args(scaffold(workspace), make_expectation(), remote_spec="pipeline/spec.toml") + ) + assert "no --remote backend" in str(exc.value) + + +# --------------------------------------------------------------------------- +# what happens on the host +# --------------------------------------------------------------------------- +def test_the_mutated_source_is_what_reaches_the_host(workspace, monkeypatch): + """The whole point: the *candidate's* program runs inside the environment + the preflight proved.""" + sub = remote_spec(workspace) + preflight(sub) + seen = host_answers(monkeypatch, scored) + + drive(workspace, sub, monkeypatch) + + assert seen, "nothing reached the host" + for call in seen: + assert set(call["files"]) == {"initial.py", "evaluate.py"} + assert "EVOLVE-BLOCK-START" in call["files"]["initial.py"] + assert call["command"] == ["python", "evaluate.py"] + + +def test_a_remote_campaign_scores_what_the_host_printed(workspace, monkeypatch): + sub = remote_spec(workspace) + preflight(sub) + host_answers(monkeypatch, scored) + + result = drive(workspace, sub, monkeypatch) + candidates = [c for c in camp.candidates(result["campaign"]) if c.get("metrics")] + + assert candidates, "no candidate was scored from the host's output" + assert candidates[0]["metrics"]["combined_score"] == 1.5 + assert candidates[0]["ran_on"].startswith("gpu-box:") + assert candidates[0]["backend"] == "ssh" + + +def test_the_cost_recorded_is_measured_not_estimated(workspace, monkeypatch): + """The estimate is what the budget gate projects with; this is what was + actually spent, and on a remote campaign the two are not the same number.""" + sub = remote_spec(workspace) + preflight(sub) + host_answers(monkeypatch, lambda cid: scored(cid, cost=0.07)) + + result = drive(workspace, sub, monkeypatch, estimate_per_candidate_usd=0.5) + for candidate in camp.candidates(result["campaign"]): + assert candidate["cost_usd"] == 0.07 + + +def test_a_host_that_could_not_run_it_is_not_a_bad_mutation(workspace, monkeypatch): + """A search that reads 'the host refused the connection' as 'this idea + scored nothing' quietly selects against whatever was being proposed when + the network wobbled.""" + sub = remote_spec(workspace) + preflight(sub) + host_answers( + monkeypatch, + lambda cid: { + "ok": False, + "exit_code": None, + "cost_usd": 0.0, + "host": "gpu-box", + "where": f"gpu-box:~/grad/{cid}", + "output": "", + "error": "ssh to gpu-box failed (exit 255): connection refused", + }, + ) + + result = drive(workspace, sub, monkeypatch) + candidates = camp.candidates(result["campaign"]) + assert candidates + for candidate in candidates: + assert candidate["metrics"] is None + assert candidate["skipped"] is True + assert "could not run it" in candidate["error"] + + +def test_a_candidate_that_crashed_is_recorded_with_its_reason(workspace, monkeypatch): + sub = remote_spec(workspace) + preflight(sub) + host_answers( + monkeypatch, + lambda cid: { + "ok": False, + "exit_code": 1, + "cost_usd": 0.01, + "host": "gpu-box", + "where": f"gpu-box:~/grad/{cid}", + "error": "the candidate exited 1 on gpu-box", + "output": "Traceback (most recent call last):\nValueError: nope\nEXIT:1", + }, + ) + + result = drive(workspace, sub, monkeypatch) + candidates = camp.candidates(result["campaign"]) + assert all(c["metrics"] is None for c in candidates) + assert any("did not print a JSON object" in (c.get("error") or "") for c in candidates) + # Not skipped: it ran, it cost money, and it failed. That is a fact about + # the mutation and the next generation's prompt should see it. + assert not any(c.get("skipped") for c in candidates) + + +def test_a_remote_candidate_still_never_becomes_a_run(workspace, monkeypatch): + """§23 item 4 holds when the search leaves this machine.""" + sub = remote_spec(workspace) + preflight(sub) + host_answers(monkeypatch, scored) + + before = len(ls.runs()) + drive(workspace, sub, monkeypatch) + assert len(ls.runs()) == before + + +def test_the_campaign_record_says_where_it_ran(workspace, monkeypatch): + sub = remote_spec(workspace) + preflight(sub) + host_answers(monkeypatch, scored) + + result = drive(workspace, sub, monkeypatch) + record = camp.campaign(result["campaign"]) + assert record["mode"] == "remote" + assert record["backend"] == "ssh" + assert record["host"] == "gpu-box" + assert record["submission_hash"] == sub.hash() + assert "sub" not in record, "a live Submission object got into a JSON record" + + +def test_a_local_campaign_still_says_it_is_local(workspace, monkeypatch): + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator()) + result = evolve.cmd_run( + run_args(scaffold(workspace), make_expectation(), generations=1, population=2) + ) + record = camp.campaign(result["campaign"]) + assert record["mode"] == "local" + assert "host" not in record + + +def test_each_backend_is_handed_the_candidate_by_its_own_adapter(workspace, monkeypatch): + """One dispatcher, three modules. The adapters answer the same question and + return the same shape, but they get there differently enough that a shared + implementation would be a lie -- and this is the wire that would otherwise + send every campaign to whichever one happened to be first.""" + calls: list[str] = [] + + def spy(name): + def evaluate_candidate(sub, cfg, **kwargs): + calls.append(name) + return { + "ok": True, "exit_code": 0, "cost_usd": 0.0, "error": None, + "where": f"{name}:x", "output": '{"combined_score": 1}', + } + + return evaluate_candidate + + from tools import gpu as gpu_tool, jobs as jobs_tool, kaggle as kaggle_tool + + monkeypatch.setattr(gpu_tool, "evaluate_candidate", spy("ssh")) + monkeypatch.setattr(kaggle_tool, "evaluate_candidate", spy("kaggle")) + monkeypatch.setattr(jobs_tool, "evaluate_candidate", spy("hf_jobs")) + + for backend, sub in ( + ("ssh", remote_spec(workspace)), + ("kaggle", remote_spec(workspace, accelerator="NvidiaTeslaP100", hours=0.5)), + ("hf_jobs", remote_spec(workspace, flavor="a10g-small")), + ): + calls.clear() + preflight(sub) + evolve._run_on_backend( + resolve_target(workspace, sub, remote=backend), + config_mod.load(), + candidate_id="c1", + files={"initial.py": "x", "evaluate.py": "y"}, + timeout_s=60, + artifacts=workspace / "artifacts", + ) + assert calls == [backend] + + +def test_a_kaggle_campaign_records_the_hours_the_quota_fold_reads(workspace, monkeypatch): + """The candidate row is the only place those hours exist -- they never reach + `runs.jsonl` -- so the field names have to be the ones the fold looks for.""" + from core import kaggle_quota + from tools import kaggle as kaggle_tool + + sub = remote_spec(workspace, accelerator="NvidiaTeslaP100", hours=0.5) + preflight(sub) + monkeypatch.setattr( + kaggle_tool, + "evaluate_candidate", + lambda sub_, cfg, **kw: { + "ok": True, "exit_code": 0, "cost_usd": 0.0, "error": None, + "where": "kaggle:x", "output": '{"combined_score": 1}', + "hours": 0.4, "accelerator": "NvidiaTeslaP100", "accelerator_kind": "gpu", + }, + ) + + monkeypatch.setattr(evolve, "_make_mutator", lambda *a, **k: FakeMutator()) + result = evolve.cmd_run(campaign_args(workspace, sub, remote="kaggle")) + + rows = camp.candidates(result["campaign"]) + assert rows and all(r[kaggle_quota.F_ACTUAL] == 0.4 for r in rows) + pools = kaggle_quota.accelerator_hours(kind="gpu")["pools"] + assert pools["gpu"]["total_hours"] == pytest.approx(0.4 * len(rows)) + + +def test_the_remote_timeout_defaults_to_the_candidate_timeout(workspace, monkeypatch): + sub = remote_spec(workspace) + preflight(sub) + seen = host_answers(monkeypatch, scored) + + drive(workspace, sub, monkeypatch, timeout_s=45, remote_timeout_s=0) + assert seen and all(call["timeout_s"] == 45 for call in seen) + + +def test_the_remote_timeout_can_be_set_apart_from_the_local_one(workspace, monkeypatch): + """A remote evaluation is not bounded by the same number as a local one -- + the host is a different machine with different hardware.""" + sub = remote_spec(workspace) + preflight(sub) + seen = host_answers(monkeypatch, scored) + + drive(workspace, sub, monkeypatch, timeout_s=45, remote_timeout_s=1800) + assert seen and all(call["timeout_s"] == 1800 for call in seen) + + +# --------------------------------------------------------------------------- +# reading metrics out of a combined stream +# --------------------------------------------------------------------------- +def test_metrics_are_read_from_the_last_line_only(): + """The same rule the local path uses, deliberately. A search whose metric + can be found anywhere in the output is a search that can be fed a number by + a log line.""" + metrics, problem = evolve._metrics_from( + '{"combined_score": 99}\nreal work happened\n{"combined_score": 1}\nEXIT:0' + ) + assert problem is None + assert metrics["combined_score"] == 1 + + +def test_the_exit_marker_is_not_mistaken_for_output(): + metrics, problem = evolve._metrics_from('{"combined_score": 1}\nEXIT:0\n') + assert problem is None + assert metrics["combined_score"] == 1 + + +def test_silence_from_the_host_is_a_named_failure(): + metrics, problem = evolve._metrics_from("EXIT:0\n") + assert metrics is None + assert "printed nothing" in problem + + +def test_a_traceback_is_reported_with_its_last_line(): + metrics, problem = evolve._metrics_from("Traceback...\nValueError: nope\nEXIT:1") + assert metrics is None + assert "ValueError: nope" in problem + + +def test_metrics_without_a_combined_score_are_still_refused_remotely(): + """The metric contract does not relax because the number arrived over ssh.""" + metrics, problem = evolve._metrics_from('{"abs_error": 2}\nEXIT:0') + assert problem is not None + + +# --------------------------------------------------------------------------- +# the ssh adapter's own refusals +# --------------------------------------------------------------------------- +def test_a_candidate_file_may_not_be_a_path(workspace): + """It is written on a machine we do not own, from a name this module's + callers supply. Checked where it would be written rather than trusted to + every future caller.""" + from core.config import Host + from tools import gpu as gpu_tool + + host = Host( + name="gpu-box", hostname="10.0.0.7", user="research", workdir="~/grad", + rate_usd_per_hour=0.0, + ) + for bad in ("../escape.py", "sub/dir.py", ".."): + with pytest.raises(GradError) as exc: + gpu_tool._write_remote(host, "~/grad/c1", bad, "print(1)") + assert "plain name" in (exc.value.fix or "") or "refusing to write" in exc.value.message diff --git a/tests/test_lab_and_wiki.py b/tests/test_lab_and_wiki.py index 9c81e66..98f1ce0 100644 --- a/tests/test_lab_and_wiki.py +++ b/tests/test_lab_and_wiki.py @@ -11,6 +11,8 @@ from __future__ import annotations +import os + import argparse import json @@ -198,12 +200,65 @@ def test_wiki_is_not_in_the_agents_tool_list(workspace): def test_a_missing_repowiki_names_the_extra(workspace, monkeypatch): - monkeypatch.setattr(wiki.shutil, "which", lambda name: None) + from core import spawn + + monkeypatch.setattr(spawn, "console_script", lambda name: None) with pytest.raises(ConfigError) as exc: wiki._repowiki() assert "[wiki]" in (exc.value.fix or "") +def test_a_repowiki_in_the_venv_is_found_without_activating_it(workspace, monkeypatch, tmp_path): + r"""The bug this replaced `shutil.which` for. + + A virtualenv's `Scripts` directory is on PATH only while the environment is + *activated*, and the desktop shortcut points straight at + `.venv\Scripts\pythonw.exe` -- so the interpreter was the venv's and PATH + was the machine's. `repowiki` was installed, `which` did not find it, and + the error said "not installed" with a `pip install` that had already been + run. + """ + import shutil + import sys + + from core import spawn + + scripts = tmp_path / "Scripts" + scripts.mkdir() + installed = scripts / ("repowiki.exe" if os.name == "nt" else "repowiki") + installed.write_text("", encoding="utf-8") + installed.chmod(0o755) + + monkeypatch.setattr(sys, "executable", str(scripts / "python.exe")) + # Nothing on PATH at all: the venv copy is the only one, which is exactly the + # situation that used to report "not installed". + monkeypatch.setattr(shutil, "which", _venv_only_which()) + + found = spawn.console_script("repowiki") + # Case-folded: on Windows `which` returns the name with PATHEXT's casing, so + # a file written as `repowiki.exe` comes back as `repowiki.EXE`. + assert found is not None + assert found.casefold() == str(installed).casefold() + + +def _venv_only_which(): + """`shutil.which` that answers only when given an explicit `path`. + + Which is the whole shape of the bug: the tool exists on disk beside the + interpreter and is invisible to a PATH search. + """ + import shutil as _shutil + + real = _shutil.which + + def which(name, mode=os.F_OK | os.X_OK, path=None): + if path is None: + return None # PATH does not have it + return real(name, mode=mode, path=path) + + return which + + def _repo_root(): from pathlib import Path @@ -297,7 +352,9 @@ def fake_run(argv, **kw): (workspace / "tools").mkdir(parents=True, exist_ok=True) (workspace / "tools" / "b.py").write_text("y = 2\n", encoding="utf-8") - monkeypatch.setattr(wiki.shutil, "which", lambda name: "repowiki") + from core import spawn + + monkeypatch.setattr(spawn, "console_script", lambda name: "repowiki") monkeypatch.setattr(wiki.subprocess, "run", fake_run) result = wiki.cmd_map(argparse.Namespace(top=200, open=False, json=True)) diff --git a/tests/test_review_fixes_2.py b/tests/test_review_fixes_2.py index c829875..c44bf28 100644 --- a/tests/test_review_fixes_2.py +++ b/tests/test_review_fixes_2.py @@ -613,3 +613,121 @@ def test_the_stop_hook_no_longer_writes_a_usage_row(workspace): asyncio.run(hooks.stop({"session_id": "s-1"}, None, None)) assert [r for r in quota_log.entries() if r["stage"] == quota_log.STAGE_MAIN] == [] + + +# --------------------------------------------------------------------------- +# a stored token that authenticated everything except the agent +# --------------------------------------------------------------------------- +TOKEN = "sk-ant-oat-test" + + +def test_the_stored_token_reaches_the_main_loop(monkeypatch): + """The gap: the main loop authenticates from the ambient variable, `sdk_env` + reads the store, and nothing joined them. A token stored through the app's + credentials panel ran the funnel and the mutation operator and left the + agent's own loop with no credentials -- panel says STORED, first turn fails.""" + import os + + from core import credentials + + monkeypatch.delenv("CLAUDE_CODE_OAUTH_TOKEN", raising=False) + monkeypatch.setattr( + credentials, "get", lambda name, **k: TOKEN if name == credentials.CLAUDE_TOKEN else None + ) + + assert credentials.hydrate_environment() == "CLAUDE_CODE_OAUTH_TOKEN" + assert os.environ["CLAUDE_CODE_OAUTH_TOKEN"] == TOKEN + + +def test_a_token_exported_by_hand_outranks_the_stored_one(monkeypatch): + """A token exported in a terminal is a deliberate choice -- a second account, + a token being tested -- and the store must not quietly outrank it. Same + precedence `sdk_env` uses.""" + import os + + from core import credentials + + monkeypatch.setenv("CLAUDE_CODE_OAUTH_TOKEN", "ambient") + monkeypatch.setattr(credentials, "get", lambda *a, **k: TOKEN) + + assert credentials.hydrate_environment() is None + assert os.environ["CLAUDE_CODE_OAUTH_TOKEN"] == "ambient" + + +def test_an_unreachable_credential_store_does_not_stop_the_start(monkeypatch): + """A missing `keyring` is not a reason to refuse to start: the SDK still + gives its own auth error if the token was the only thing that could help.""" + import os + + from core import credentials + from core.errors import ConfigError + + monkeypatch.delenv("CLAUDE_CODE_OAUTH_TOKEN", raising=False) + + def boom(*_a, **_k): + raise ConfigError("credential store unavailable", fix="install keyring") + + monkeypatch.setattr(credentials, "get", boom) + + assert credentials.hydrate_environment() is None + assert "CLAUDE_CODE_OAUTH_TOKEN" not in os.environ + + +def test_preflight_hydrates_and_reports_where_the_token_came_from(monkeypatch): + """Both entry points reach `preflight_environment` -- `run_session` and the + UI's client start -- which is why the bridge lives there. The source is + reported because "why is it using that account?" is otherwise unanswerable.""" + import os + + import agent + from core import credentials + + monkeypatch.delenv("CLAUDE_CODE_OAUTH_TOKEN", raising=False) + monkeypatch.setattr( + credentials, "get", lambda name, **k: TOKEN if name == credentials.CLAUDE_TOKEN else None + ) + + report = agent.preflight_environment() + + assert report["oauth_token_present"] is True + assert report["oauth_token_source"] == "credential store" + assert os.environ["CLAUDE_CODE_OAUTH_TOKEN"] == TOKEN + + +def test_the_scrub_runs_before_the_hydrate(monkeypatch): + """ANTHROPIC_API_KEY outranks the OAuth token in the SDK's credential chain. + Hydrating first would set a variable the scrub had not yet cleared the way + for: a session billing the Developer Platform while reporting a subscription + token present.""" + import os + + import agent + from core import credentials + + monkeypatch.setenv("ANTHROPIC_API_KEY", "secret") + monkeypatch.delenv("CLAUDE_CODE_OAUTH_TOKEN", raising=False) + monkeypatch.setattr( + credentials, "get", lambda name, **k: TOKEN if name == credentials.CLAUDE_TOKEN else None + ) + + report = agent.preflight_environment() + + assert "ANTHROPIC_API_KEY" in report["removed_env"] + assert "ANTHROPIC_API_KEY" not in os.environ + assert os.environ["CLAUDE_CODE_OAUTH_TOKEN"] == TOKEN + + +# --------------------------------------------------------------------------- +# a credential row that said nothing about itself +# --------------------------------------------------------------------------- +def test_every_credential_the_store_knows_has_a_purpose_in_the_panel(): + """`kaggle_key` was in `credentials.ALL` and not in `CREDENTIAL_NOTES`, so + the panel drew it with an empty purpose column. Asserted against `ALL` + rather than by name, because the hole was drift between two hand-written + lists and a test naming the eighth entry would not catch the ninth.""" + from core import credentials + from ui import models + + for name in credentials.ALL: + purpose, _group = models.CREDENTIAL_NOTES.get(name, ("", "extras")) + assert purpose.strip(), f"{name} has no purpose text in the credentials panel" diff --git a/tests/test_settings.py b/tests/test_settings.py new file mode 100644 index 0000000..8bef02c --- /dev/null +++ b/tests/test_settings.py @@ -0,0 +1,393 @@ +"""The writable overlay, and the file it is not allowed to touch. + +Every test here is about one of two claims. The first is that a value chosen +through setup wins over the same value in `config/grad.toml` -- because a +command that silently did nothing when a config file disagreed would be worse +than one that overrides it. The second is that overriding it never *edits* it: +that file is hand-annotated, `tomllib` cannot write TOML, and the comments in it +are worth more than the values. +""" + +from __future__ import annotations + +import pytest + +from core import config as config_mod, paths, settings +from core.errors import UsageError + +#: A config with a comment on every line that matters, so "the file survived" +#: is a claim about the annotations and not only about the values. +ANNOTATED = """\ +# The main loop's model. Chosen deliberately -- see HANDOFF-2 §16. +[models] +research = "claude-opus-5" # the expensive one, on purpose +evolve = "claude-sonnet-5" + +[hosts.from-config] +hostname = "config-box.example" +user = "researcher" +rate_usd_per_hour = 2.5 +""" + + +@pytest.fixture +def annotated_config(): + """Write the config the workspace fixture points `GRAD_CONFIG` at.""" + path = paths.config_path() + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(ANNOTATED, encoding="utf-8") + config_mod._cache.clear() + return path + + +# --------------------------------------------------------------------------- +# the layers, one test per layer +# --------------------------------------------------------------------------- +def test_a_role_nobody_set_resolves_to_the_shipped_default(workspace): + assert config_mod.load(reload=True).model_for("cite") == config_mod.DEFAULTS["models"]["cite"] + + +def test_the_config_beats_the_default(workspace, annotated_config): + assert config_mod.load(reload=True).model_for("evolve") == "claude-sonnet-5" + + +def test_the_overlay_beats_the_config(workspace, annotated_config): + """`kaggle account`'s rule, generalised: a stored selection wins over the + config key it shadows, because a user who has just answered a wizard is + entitled to expect the answer to take effect.""" + settings.set_models({"evolve": "claude-opus-5"}) + assert config_mod.load(reload=True).model_for("evolve") == "claude-opus-5" + + +def test_clearing_an_override_falls_back_through_the_layers_again(workspace, annotated_config): + settings.set_models({"evolve": "claude-opus-5"}) + settings.clear_models(["evolve"]) + cfg = config_mod.load(reload=True) + assert cfg.model_for("evolve") == "claude-sonnet-5" # the config, again + assert cfg.model_for("cite") == config_mod.DEFAULTS["models"]["cite"] + + +def test_an_overlay_write_is_seen_by_a_process_that_already_loaded(workspace): + """The overlay's mtime is part of the cache key. Without it, `setup models` + -- which the UI runs as a child process -- writes a file the app has already + read, and the setup window appears to do nothing until a restart.""" + before = config_mod.load().model_for("evolve") + settings.set_models({"evolve": "claude-fable-5"}) + assert config_mod.load().model_for("evolve") == "claude-fable-5" + assert before != "claude-fable-5" + + +# --------------------------------------------------------------------------- +# the project layer, which sits above all of them +# --------------------------------------------------------------------------- +def test_a_project_override_beats_the_workspace_overlay(workspace, annotated_config): + """The model per role is the main lever on cost and quality, which is exactly + why a cheap exploratory project and one being written up should be able to + differ.""" + from core import budget + + settings.set_models({"evolve": "claude-sonnet-5"}) + budget.create("proj-a", title="A", budget={}) + budget.set_current("proj-a") + budget.configure("proj-a", models={"evolve": "claude-opus-5"}, reason="writing it up") + + assert config_mod.load(reload=True).model_for("evolve") == "claude-opus-5" + + +def test_a_project_overriding_one_role_leaves_the_others_alone(workspace, annotated_config): + """An override, not a replacement. "Opus for the write-up, everything else + as usual" has to be expressible.""" + from core import budget + + budget.create("proj-a", title="A", budget={}) + budget.set_current("proj-a") + budget.configure("proj-a", models={"report": "claude-opus-5"}) + + cfg = config_mod.load(reload=True) + assert cfg.model_for("report") == "claude-opus-5" + assert cfg.model_for("evolve") == "claude-sonnet-5" # still the config's + assert cfg.model_for("cite") == config_mod.DEFAULTS["models"]["cite"] + + +def test_the_project_layer_can_be_asked_to_stand_aside(workspace, annotated_config): + """The projects window draws every project and only one is selected, so for + all the others the project layer in effect belongs to somebody else.""" + from core import budget + + budget.create("proj-a", title="A", budget={}) + budget.set_current("proj-a") + budget.configure("proj-a", models={"evolve": "claude-opus-5"}) + + cfg = config_mod.load(reload=True) + assert cfg.model_for("evolve") == "claude-opus-5" + assert cfg.model_for("evolve", project=False) == "claude-sonnet-5" + + +def test_switching_project_changes_which_model_answers(workspace, annotated_config): + """Without the selection in the cache key, the app goes on serving whatever + it loaded at startup.""" + from core import budget + + budget.create("proj-cheap", title="cheap", budget={}) + budget.create("proj-careful", title="careful", budget={}) + budget.configure("proj-cheap", models={"research": "claude-haiku-4-5"}) + budget.configure("proj-careful", models={"research": "claude-opus-5"}) + + budget.set_current("proj-cheap") + assert config_mod.load().model_for("research") == "claude-haiku-4-5" + budget.set_current("proj-careful") + assert config_mod.load().model_for("research") == "claude-opus-5" + + +def test_clearing_a_project_override_falls_back_to_the_workspace(workspace, annotated_config): + from core import budget + + budget.create("proj-a", title="A", budget={}) + budget.set_current("proj-a") + budget.configure("proj-a", models={"evolve": "claude-opus-5"}) + budget.configure("proj-a", models={"evolve": None}) + + assert config_mod.load(reload=True).model_for("evolve") == "claude-sonnet-5" + assert budget.project_overrides("proj-a")["models"] == {} + + +def test_every_configure_is_kept_not_just_the_last(workspace): + """The model a candidate was mutated by is part of what produced the numbers + in the ledger beside it, so a field that could be edited in place would let a + project's history claim it had always been on today's model.""" + from core import budget + + budget.create("proj-a", title="A", budget={}) + budget.configure("proj-a", models={"evolve": "claude-sonnet-5"}, reason="first pass") + budget.configure("proj-a", models={"evolve": "claude-opus-5"}, reason="it was missing things") + + history = budget.projects()["proj-a"]["configured"] + assert [h["reason"] for h in history] == ["first pass", "it was missing things"] + assert budget.projects()["proj-a"]["models"]["evolve"] == "claude-opus-5" + + +def test_configure_refuses_a_role_and_a_backend_it_does_not_know(workspace): + from core import budget + + budget.create("proj-a", title="A", budget={}) + with pytest.raises(UsageError): + budget.configure("proj-a", models={"summarise": "claude-opus-5"}) + with pytest.raises(UsageError): + budget.configure("proj-a", backend="slurm") + with pytest.raises(UsageError): + budget.configure("proj-a") # nothing to change + assert budget.projects()["proj-a"]["configured"] == [] + + +def test_configure_refuses_a_project_that_does_not_exist(workspace): + from core import budget + from core.errors import NotFound + + with pytest.raises(NotFound): + budget.configure("proj-nope", models={"evolve": "claude-opus-5"}) + + +def test_overrides_for_an_unreadable_ledger_are_empty_not_an_exception(workspace, monkeypatch): + """`project_overrides` is read on the config path, which every surface calls + while rendering.""" + from core import budget + + monkeypatch.setattr( + budget, "projects", lambda: (_ for _ in ()).throw(RuntimeError("torn ledger")) + ) + assert budget.project_overrides("proj-a") == {"models": {}, "backend": None} + assert budget.project_overrides(None) == {"models": {}, "backend": None} + + +# --------------------------------------------------------------------------- +# the file that must not be written +# --------------------------------------------------------------------------- +def test_no_setup_command_edits_the_annotated_config(workspace, annotated_config): + """The whole reason this module exists. `tomllib` reads TOML and cannot + write it, so an editing command would reformat the file and drop every + comment in it -- and the comments are the reasoning.""" + from tools import setup as setup_tool + + before = annotated_config.read_bytes() + + setup_tool.cli.run(["models", "--evolve", "claude-opus-5", "--json"]) + setup_tool.cli.run(["backend", "--default", "kaggle", "--json"]) + setup_tool.cli.run( + ["host", "add", "--name", "gpu-box", "--hostname", "box.example", "--user", "me", "--json"] + ) + setup_tool.cli.run(["host", "remove", "--name", "gpu-box", "--json"]) + setup_tool.cli.run(["models", "--clear", "evolve", "--json"]) + setup_tool.cli.run(["show", "--json"]) + setup_tool.cli.run(["check", "--json"]) + + assert annotated_config.read_bytes() == before + + +def test_the_overlay_lives_beside_the_layouts_not_in_the_workspace(workspace): + """Per workspace, because `grad.toml` already is. Out of the workspace + folder, because a research folder handed to a colleague should not carry + this machine's SSH inventory.""" + from core import appdata + + assert settings.path().parent == appdata.workspace_state_dir() + assert paths.root() not in settings.path().parents + + +# --------------------------------------------------------------------------- +# what it refuses +# --------------------------------------------------------------------------- +def test_an_unknown_role_is_refused_with_the_roles_in_the_message(workspace): + with pytest.raises(UsageError) as caught: + settings.set_models({"summarise": "claude-opus-5"}) + assert "research" in caught.value.fix + + +def test_an_empty_model_is_refused_rather_than_stored(workspace): + """Stored, it would resolve to falsy and silently fall through -- a setting + that is present, wrong, and invisible.""" + with pytest.raises(UsageError): + settings.set_models({"evolve": " "}) + assert settings.models() == {} + + +def test_an_unknown_backend_is_refused(workspace): + with pytest.raises(UsageError) as caught: + settings.set_backend("slurm") + assert "kaggle" in caught.value.fix + + +def test_a_negative_host_rate_is_refused_on_the_writable_side_too(workspace): + """`collect` prices wall clock against this, so a negative rate books + negative actuals -- which *reduce* rolling spend. A typo that raises the + ceiling is worth refusing at the point of entry, on both sides of the + inventory.""" + with pytest.raises(UsageError) as caught: + settings.add_host("gpu-box", {"hostname": "box.example", "rate_usd_per_hour": -1.0}) + assert "negative spend" in caught.value.fix + + +def test_a_host_name_that_could_pass_for_an_ssh_flag_is_refused(workspace): + with pytest.raises(UsageError): + settings.add_host("-oProxyCommand=curl evil.example", {"hostname": "box.example"}) + + +def test_a_host_with_no_hostname_is_refused(workspace): + with pytest.raises(UsageError) as caught: + settings.add_host("gpu-box", {"user": "me"}) + assert "--hostname" in caught.value.fix + + +# --------------------------------------------------------------------------- +# the inventory has two sources and stays fixed +# --------------------------------------------------------------------------- +def test_an_added_host_joins_the_inventory_the_config_declared(workspace, annotated_config): + settings.add_host( + "gpu-box", {"hostname": "box.example", "user": "me", "rate_usd_per_hour": 1.25} + ) + hosts = config_mod.load(reload=True).hosts + assert set(hosts) == {"from-config", "gpu-box"} + assert hosts["gpu-box"].rate_usd_per_hour == 1.25 + assert hosts["from-config"].hostname == "config-box.example" + + +def test_an_overlay_host_replaces_the_config_one_rather_than_inheriting_it( + workspace, annotated_config +): + """A recursive merge would have the overlay host inherit the fields it + omitted -- including `key_credential`, the keyring entry that authenticates + the connection. Replacing `from-config` through the wizard and getting the + old box's credential and user attached to the new hostname is a connection + nobody described, and it is the one field here where being wrong reaches a + machine.""" + settings.add_host("from-config", {"hostname": "new-box.example", "rate_usd_per_hour": 0.0}) + + host = config_mod.load(reload=True).hosts["from-config"] + assert host.hostname == "new-box.example" + assert host.user == "", "the config host's user must not have carried over" + assert host.key_credential is None + assert host.rate_usd_per_hour == 0.0 + + +def test_a_non_finite_host_rate_is_refused_on_the_writable_side_too(workspace): + """`nan` fails every comparison a gate makes against it and `inf` is a price + no run can be under, so both are ceilings that stop bounding anything. + `core/config.py` refuses them on the TOML side; a check that exists in only + one of two entry points is a check with a way around it.""" + for rate in (float("nan"), float("inf"), float("-inf")): + with pytest.raises(UsageError) as caught: + settings.add_host("gpu-box", {"hostname": "box.example", "rate_usd_per_hour": rate}) + assert "finite" in caught.value.message or "negative" in caught.value.message + assert settings.hosts() == {} + + +def test_an_unknown_host_names_both_places_it_could_have_been_added(workspace): + """The inventory is fixed by design -- a host that can be named ad-hoc is a + general remote-execution capability. It now has two sources, so a refusal + that named one of them sent half the readers to the wrong file.""" + from core.errors import ConfigError + + with pytest.raises(ConfigError) as caught: + config_mod.load(reload=True).host("nowhere") + assert "tools.setup host add" in caught.value.fix + assert "grad.toml" in caught.value.fix + + +# --------------------------------------------------------------------------- +# the report that makes winning acceptable +# --------------------------------------------------------------------------- +def test_shadowing_names_the_config_value_that_stopped_taking_effect(workspace, annotated_config): + """Someone edits `[models] evolve`, sees no change, and has no way to + discover that a file they have never heard of outranks the file they were + told to edit.""" + settings.set_models({"evolve": "claude-opus-5"}) + report = settings.shadowing(config_mod.load(reload=True)) + assert report == [ + {"what": "[models] evolve", "config": "claude-sonnet-5", "overlay": "claude-opus-5"} + ] + + +def test_an_overlay_that_agrees_with_the_config_shadows_nothing(workspace, annotated_config): + settings.set_models({"evolve": "claude-sonnet-5"}) + assert settings.shadowing(config_mod.load(reload=True)) == [] + + +def test_a_role_the_config_never_set_shadows_nothing(workspace, annotated_config): + settings.set_models({"cite": "claude-opus-5"}) + assert settings.shadowing(config_mod.load(reload=True)) == [] + + +# --------------------------------------------------------------------------- +# one vocabulary for the three backends +# --------------------------------------------------------------------------- +def test_the_backend_names_are_the_ones_evolve_submits_to(): + """`core/settings.py` names them because a setting is read by the config + layer and `core` importing `tools` is backwards. Two lists is one list that + has not drifted yet.""" + from tools import evolve as evolve_tool + + assert set(settings.BACKENDS) == set(evolve_tool.REMOTE_BACKENDS) + + +# --------------------------------------------------------------------------- +# a broken overlay is not a broken app +# --------------------------------------------------------------------------- +def test_an_unparseable_overlay_leaves_the_app_on_the_config(workspace, annotated_config): + """`grad.toml` is a working configuration. An overlay that will not parse + should cost its own contents, not the ability to start.""" + settings.path().parent.mkdir(parents=True, exist_ok=True) + settings.path().write_text("{ not json at all", encoding="utf-8") + assert config_mod.load(reload=True).model_for("evolve") == "claude-sonnet-5" + + +def test_readiness_names_what_each_backend_is_missing(workspace, monkeypatch): + """`hf_token` is required *for HF Jobs*, which is a different claim from + "required" -- and the credentials panel used to make the stronger one at a + user who had chosen Kaggle.""" + from core import credentials + from tools import setup as setup_tool + + monkeypatch.setattr(credentials, "status", lambda: dict.fromkeys(credentials.ALL, False)) + rows = {r["backend"]: r for r in setup_tool.readiness(config_mod.load(reload=True))} + assert rows["hf_jobs"]["missing"] == ["hf_token"] + assert "an ssh host" in rows["ssh"]["missing"] + assert not any(r["ready"] for r in rows.values()) diff --git a/tests/test_ui_argv.py b/tests/test_ui_argv.py index f460bc1..0dd4841 100644 --- a/tests/test_ui_argv.py +++ b/tests/test_ui_argv.py @@ -28,10 +28,40 @@ ("tools.budget", ["raise", "--project", "proj-x", "--gpu-usd", "50"]), ("tools.budget", ["raise", "--project", "proj-x", "--quota-tokens", "5e6"]), ("tools.budget", ["raise", "--project", "proj-x", "--credits-usd", "10"]), + # The projects window offers the reason the CLI has always taken. A ceiling + # that moved without one is unarguable with six months later. + ("tools.budget", ["raise", "--project", "proj-x", "--gpu-usd", "50", "--reason", "why"]), ("tools.budget", ["new", "--id", "proj-x", "--title", "a title", "--use"]), + # The create form sends the ceilings and the payer on `new` itself, rather + # than as a raise afterwards -- a raise records a ceiling that *moved*. + ( + "tools.budget", + [ + "new", "--id", "proj-x", "--title", "a title", "--use", + "--gpu-usd", "120", "--quota-tokens", "5e6", "--credits-usd", "10", + "--payer", "hf:myorg", + ], + ), ("tools.budget", ["use", "proj-x"]), + ("tools.budget", ["close", "proj-x"]), + # Per-project overrides, from the projects window's model editor. + ("tools.budget", ["configure", "--project", "proj-x", "--research", "claude-opus-5"]), + ("tools.budget", ["configure", "--project", "proj-x", "--report", "claude-opus-5"]), + ("tools.budget", ["configure", "--project", "proj-x", "--clear", "evolve"]), + ("tools.budget", ["configure", "--project", "proj-x", "--backend", "kaggle"]), ("tools.budget", ["status", "--project", "proj-x"]), + # The setup window. Every one of these is a button. + ("tools.setup", ["models", "--research", "claude-opus-5"]), + ("tools.setup", ["models", "--evolve", "claude-sonnet-5"]), + ("tools.setup", ["models", "--clear", "evolve"]), + ("tools.setup", ["backend", "--default", "kaggle"]), + ("tools.setup", ["host", "add", "--name", "gpu-box", "--hostname", "h", "--user", "u", "--rate", "0"]), + ("tools.setup", ["host", "remove", "--name", "gpu-box"]), + ("tools.setup", ["show"]), + ("tools.setup", ["check"]), + ("tools.kaggle", ["account", "--set", "someone"]), ("tools.jobs", ["credential", "set", "hf_token", "--stdin"]), + ("tools.jobs", ["credential", "delete", "hf_token"]), ("tools.jobs", ["credential", "status"]), ("tools.jobs", ["collect", "run-1"]), ("tools.ledger", ["verdict", "run-1", "--quantity", "loss", "--verdict", "bug", "--note", "x"]), diff --git a/tests/test_ui_models.py b/tests/test_ui_models.py index 8b63cd9..52710ca 100644 --- a/tests/test_ui_models.py +++ b/tests/test_ui_models.py @@ -781,10 +781,225 @@ def test_a_damaged_ledger_line_degrades_to_an_error_not_a_crash(workspace): assert model["entries"] == [] +# --------------------------------------------------------------------------- +# the projects window +# --------------------------------------------------------------------------- +def test_a_project_with_no_ceilings_is_reported_as_unbounded(workspace): + """It passes every gate that reads a ceiling, silently. `tools/budget.py` + says so once at creation and nothing carried it further; the window puts it + on the row it is true of, for as long as it stays true.""" + from core import budget as budget_mod + + budget_mod.create("proj-open", title="no ceilings", budget={}) + budget_mod.create("proj-bound", title="bounded", budget={"gpu_usd": 10.0}) + + rows = {r["id"]: r for r in models.projects_model()["rows"]} + assert rows["proj-open"]["unbounded"] is True + assert rows["proj-bound"]["unbounded"] is False + assert models.projects_model()["unbounded"] == ["proj-open"] + + +def test_a_closed_project_is_not_counted_as_unbounded(workspace): + """Nothing will be charged to it, so an UNBOUNDED chip on a closed project is + a warning about a thing that cannot happen.""" + from core import budget as budget_mod + + budget_mod.create("proj-done", title="finished", budget={}) + budget_mod.close("proj-done") + + model = models.projects_model() + assert model["unbounded"] == [] + assert model["open_count"] == 0 + assert model["rows"][0]["status"] == "closed" + + +def test_every_project_carries_its_own_ceilings(workspace): + """The reason this window exists. The menu's raise controls addressed the + selected project only, so reading what bounded any other one meant switching + to it -- which reloads every window in the app to answer a question about a + number.""" + from core import budget as budget_mod + + budget_mod.create("proj-a", title="A", budget={"gpu_usd": 10.0}) + budget_mod.create("proj-b", title="B", budget={"quota_tokens": 5_000_000}) + budget_mod.set_current("proj-a") + + rows = {r["id"]: r for r in models.projects_model()["rows"]} + by_resource = {c["resource"]: c for c in rows["proj-b"]["ceilings"]} + assert by_resource["quota_tokens"]["set"] is True + assert by_resource["gpu_usd"]["set"] is False + # Not the selected one, and its ceiling is readable anyway. + assert rows["proj-b"]["current"] is False + + +def test_tokens_are_counted_and_dollars_are_priced(workspace): + """4.2M subscription tokens rendered as `$4,200,000.00` is the specific + thing the per-resource formatter separates.""" + from core import budget as budget_mod + + budget_mod.create("proj-a", title="A", budget={"quota_tokens": 4_200_000, "gpu_usd": 12.0}) + row = models.projects_model()["rows"][0] + by_resource = {c["resource"]: c for c in row["ceilings"]} + assert "4.2M" in by_resource["quota_tokens"]["label"] + assert "$" not in by_resource["quota_tokens"]["label"] + assert "$12.00" in by_resource["gpu_usd"]["label"] + + +def test_a_project_whose_spend_will_not_compute_does_not_take_the_list_down(workspace, monkeypatch): + """`status` folds the whole run ledger for one project. It is caught per row + so the broken one says so in its own row.""" + from core import budget as budget_mod + + budget_mod.create("proj-a", title="A", budget={}) + budget_mod.create("proj-b", title="B", budget={}) + + real_status = budget_mod.status + + def explode(project_id): + if project_id == "proj-a": + raise RuntimeError("this ledger is a lie") + return real_status(project_id) + + monkeypatch.setattr(budget_mod, "status", explode) + + rows = {r["id"]: r for r in models.projects_model()["rows"]} + assert "this ledger is a lie" in rows["proj-a"]["error"] + assert rows["proj-b"]["error"] is None + + +def test_scaffolded_but_empty_is_not_the_same_as_never_scaffolded(workspace): + """`budget new` guards the scaffold step so it cannot fail the creation. This + is where that consequence becomes visible instead of being discovered by + `project sync` weeks later.""" + from core import budget as budget_mod, projects as projects_mod + + budget_mod.create("proj-a", title="A", budget={}) + assert models.projects_model()["rows"][0]["memory"]["scaffolded"] is False + + projects_mod.scaffold("proj-a") + memory = models.projects_model()["rows"][0]["memory"] + assert memory["scaffolded"] is True + assert "MEMORY.md" in memory["present"] + + +# --------------------------------------------------------------------------- +# the setup window +# --------------------------------------------------------------------------- +def test_a_token_in_the_environment_is_ready_but_not_durable(workspace, monkeypatch): + """The distinction that only bites the installed app: a shell that exported + the token has it, and the desktop shortcut launches from Explorer with + whatever was made persistent -- usually nothing.""" + from core import credentials + + monkeypatch.setenv("CLAUDE_CODE_OAUTH_TOKEN", "sk-ant-oat-test") + monkeypatch.setattr(credentials, "status", lambda: dict.fromkeys(credentials.ALL, False)) + + token = models.setup_model()["token"] + assert token["state"] == "environment" + assert token["ready"] is True + assert token["durable"] is False + + +def test_a_stored_token_is_durable(workspace, monkeypatch): + from core import credentials + + monkeypatch.delenv("CLAUDE_CODE_OAUTH_TOKEN", raising=False) + monkeypatch.setattr( + credentials, + "status", + lambda: {**dict.fromkeys(credentials.ALL, False), credentials.CLAUDE_TOKEN: True}, + ) + token = models.setup_model()["token"] + assert token["state"] == "stored" + assert token["durable"] is True + + +def test_with_nothing_configured_the_first_step_is_the_one_that_blocks(workspace, monkeypatch): + from core import credentials + + monkeypatch.delenv("CLAUDE_CODE_OAUTH_TOKEN", raising=False) + monkeypatch.setattr(credentials, "status", lambda: dict.fromkeys(credentials.ALL, False)) + + model = models.setup_model() + steps = {s["id"]: s for s in model["steps"]} + assert steps["token"]["ready"] is False + assert steps["backends"]["ready"] is False + # Neither of these can be "unanswered": there are defaults for all six roles, + # and an optional key is optional. + assert steps["models"]["ready"] is True + assert steps["extras"]["ready"] is True + assert model["complete"] is False + + +def test_the_models_step_reports_where_each_role_resolved_from(workspace): + from core import settings + + settings.set_models({"evolve": "claude-opus-5"}) + roles = {r["role"]: r for r in models.setup_model()["roles"]} + assert roles["evolve"]["model"] == "claude-opus-5" + assert roles["evolve"]["source"] == "setup" + assert roles["evolve"]["overridden"] is True + assert roles["cite"]["source"] in ("config", "default") + assert roles["cite"]["overridden"] is False + + +def test_setup_is_needed_only_when_the_agent_cannot_authenticate(workspace, monkeypatch): + """Narrow on purpose. An unconfigured backend means no remote training, + which is a real limitation and not a reason to put a wizard in front of + someone who opened the app to read a ledger.""" + from core import credentials + + monkeypatch.delenv("CLAUDE_CODE_OAUTH_TOKEN", raising=False) + monkeypatch.setattr(credentials, "present", lambda _name: False) + assert models.setup_needed() is True + + monkeypatch.setattr(credentials, "present", lambda _name: True) + assert models.setup_needed() is False + + +def test_an_unreachable_credential_store_does_not_decide_the_app_is_broken(workspace, monkeypatch): + """`setup_needed` runs on the startup path. It must answer, not raise.""" + from core import credentials + from core.errors import ConfigError + + monkeypatch.delenv("CLAUDE_CODE_OAUTH_TOKEN", raising=False) + + def boom(_name): + raise ConfigError("no keyring here", fix="pip install keyring") + + monkeypatch.setattr(credentials, "present", boom) + assert models.setup_needed() is True # nothing configured is the same answer + + +def test_a_backend_credential_is_not_reported_as_unconditionally_required(workspace, monkeypatch): + """`hf_token` used to be marked required, so a user who had chosen Kaggle -- + the free backend -- saw a red MISSING for a token they will never need. What + is true is that HF Jobs needs it, which is a fact about a backend.""" + from core import credentials + + monkeypatch.setattr(credentials, "status", lambda: dict.fromkeys(credentials.ALL, False)) + rows = {r["name"]: r for r in models.credentials_model()["rows"]} + assert rows["hf_token"]["required"] is False + assert rows["hf_token"]["group"] == "backend" + assert rows["hf_token"]["tone"] != "broken" + # The one that genuinely is. + assert rows["claude_oauth_token"]["required"] is True + assert rows["claude_oauth_token"]["tone"] == "broken" + + +def test_the_header_carries_the_folder_basename_and_its_whole_path(workspace): + """The appbar cell cannot hold an absolute path, and the tooltip has to.""" + model = models.header_model() + assert model["root"] == str(paths.root()) + assert model["root_name"] == paths.root().name + + def test_every_model_survives_a_completely_empty_workspace(workspace): """Eleven windows over eight ledgers is eight chances per refresh for one bad file to take the workspace down. None of them may raise.""" for builder in ( + models.setup_model, + models.projects_model, models.ledger_model, models.quota_model, models.preflight_model, diff --git a/tests/test_ui_registry.py b/tests/test_ui_registry.py index 06c47b3..c8fed6e 100644 --- a/tests/test_ui_registry.py +++ b/tests/test_ui_registry.py @@ -25,12 +25,14 @@ def test_ids_are_unique(): def test_the_windows_the_handoff_lists_are_all_here(): - """The handoff's eleven, and `tasks` -- which is not one of them because the - handoff had nowhere to put a local command that runs for twenty minutes. - Everything long was awaited under a wall clock until it was.""" + """The handoff's eleven, and three it has nowhere to put: `tasks`, for a + local command that runs for twenty minutes; `projects`, the unit every run, + ceiling and report is keyed by; and `setup`, which is what the four scopes + crammed into one dropdown turned into once they were separated.""" assert set(registry.ids()) == { "chat", "notebook", "wiki", "papers", "evolve", "editor", "ledger", "preflight", "quota", "funnel", "queue", "tasks", + "projects", "setup", } @@ -77,6 +79,27 @@ def test_the_defaults_reproduce_the_mocks_opening_arrangement(): assert [c.windows for c in layout.columns] == [["chat"], ["notebook"], ["ledger", "quota"]] +def test_an_unauthenticated_machine_opens_on_setup(monkeypatch): + """Those four windows are four windows that cannot do anything without a + token, so the first thing on screen should be the one that fixes it.""" + monkeypatch.setattr(state_mod.models, "setup_needed", lambda: True) + assert state_mod.opening_windows()[0] == "setup" + assert set(registry.defaults()) <= set(state_mod.opening_windows()) + + +def test_a_configured_machine_opens_on_the_mocks_four(monkeypatch): + monkeypatch.setattr(state_mod.models, "setup_needed", lambda: False) + assert state_mod.opening_windows() == registry.defaults() + + +def test_a_credential_store_that_raises_does_not_stop_a_workspace_opening(monkeypatch): + def boom(): + raise RuntimeError("no keyring, no registry, nothing") + + monkeypatch.setattr(state_mod.models, "setup_needed", boom) + assert state_mod.opening_windows() == registry.defaults() + + def test_the_default_layout_fits_the_minimum_pane_width(): columns = len(layout_mod.Layout.default(registry.defaults()).columns) assert columns * layout_mod.MIN_PANE_PX <= 1600, "the default window would open below minimum" diff --git a/tests/test_ui_shell.py b/tests/test_ui_shell.py index 9585d0a..b317402 100644 --- a/tests/test_ui_shell.py +++ b/tests/test_ui_shell.py @@ -204,6 +204,24 @@ def test_a_quote_in_a_tooltip_cannot_truncate_the_props_string(rendered): assert "see below" in element.props["title"] +def test_a_windows_path_in_a_tooltip_does_not_raise_out_of_props(rendered): + """NiceGUI hands each props value to `ast.literal_eval`, so the text is read + as a Python string literal and a backslash is an escape in it. `C:\\Users\\…` + contains `\\U`, which begins a unicode escape and raises a SyntaxError from + inside `element.props()` -- taking down whatever was being built, not the + tooltip. It surfaced the first time a control put a path in a tooltip, which + is to say the first time the appbar said which folder this is. + """ + from ui import kit + + client, _ = rendered(["chat"]) + with client: + element = kit.button("OPEN", title=r"C:\Users\vovas\Grad — switch folder") + + assert "Users" in element.props["title"] + assert "switch folder" in element.props["title"] + + def test_a_handle_sits_between_every_pair_of_columns(rendered): client, space = rendered(["chat", "ledger", "quota"]) handles = [e for e in client.elements.values() if "grad-handle" in getattr(e, "classes", [])] @@ -667,7 +685,9 @@ def test_reopening_a_closed_window_builds_a_fresh_root(rendered): # --------------------------------------------------------------------------- # switching project and folder # --------------------------------------------------------------------------- -def test_the_project_menu_lists_the_folder_and_its_projects(rendered): +def test_the_project_menu_lists_the_projects_and_nothing_else(rendered): + """It is the quick switcher now. The folder, the credentials and the updater + are a different scope and live behind `workspace ▾`.""" from core import budget as budget_mod budget_mod.create("proj-a", title="Scaling laws", budget={}) @@ -681,13 +701,44 @@ def test_the_project_menu_lists_the_folder_and_its_projects(rendered): # Drawn on open, not at build time: creating a project makes the list it was # read from stale, so it is rebuilt each time. with client: - shell_mod._draw_project_menu( # noqa: SLF001 - no public hook - __import__("nicegui").ui, space, menu[0], _NullMenu() - ) + shell_mod._draw_project_menu(space, menu[0], _NullMenu()) # noqa: SLF001 - no public hook markup = html_of(client) assert "proj-a" in markup assert "Scaling laws" in markup + assert "CREDENTIALS" not in markup + assert "RECENT" not in markup + + +def test_the_workspace_menu_lists_the_folder_and_how_to_leave_it(rendered): + """The folder and the recent list. Credentials and the updater are facts + about the installation and live in the setup window; what is left here is a + button that opens it.""" + client, space = rendered(["chat"]) + + from ui import shell as shell_mod + + card = [e for e in client.elements.values() if "grad-card" in getattr(e, "classes", [])][0] + with client: + shell_mod._draw_workspace_menu( # noqa: SLF001 - no public hook + __import__("nicegui").ui, space, card, _NullMenu(), _NullConfirm() + ) + markup = html_of(client) assert "WORKSPACE" in markup + assert "SETUP" in markup + assert "hf_token" not in markup, "the credential rows moved to the setup window" + + +def test_the_appbar_carries_the_folder_basename_not_its_path(rendered): + """An absolute path does not fit an appbar cell, and the whole one is the + button's tooltip — "which folder is this?" has to be answerable without + opening anything.""" + from core import paths + + client, space = rendered(["chat"]) + markup = html_of(client) + root = paths.root() + assert f"{root.name} ▾" in markup + assert str(root) in markup, "the full path should still be reachable, as the tooltip" class _NullMenu: @@ -698,12 +749,140 @@ def redraw(self) -> None: pass -def test_the_workspace_menu_can_store_a_credential_without_a_terminal(rendered, monkeypatch): +class _NullConfirm: + """The folder switch asks before it moves. Nothing here answers it — these + tests draw the menu, they do not click through it.""" + + async def ask(self, *_a, **_k) -> bool: + return False + + +def test_creating_a_project_sets_its_ceilings_in_the_same_command(rendered, monkeypatch): + """Not as a raise afterwards. A raise appends an event carrying a `previous` + value, so setting the first ceiling through one would record that the GPU + allowance moved from nothing to $50 -- which is not what happened, and the + history is the reason that module is append-only at all.""" + import asyncio + + from ui import tasks as tasks_mod + + calls: list[tuple] = [] + + async def fake_run_tool(*argv, timeout=120.0, stdin=None): + calls.append(argv) + return {"ok": True, "data": {"message": "created"}} + + monkeypatch.setattr(tasks_mod, "run_tool", fake_run_tool) + monkeypatch.setattr("ui.state.run_tool", fake_run_tool) + + _, space = rendered(["projects"]) + asyncio.run( + space.create_project( + "proj-a", + "Scaling laws", + ceilings={"gpu-usd": "50", "quota-tokens": "5e6", "credits-usd": ""}, + payer="hf:myorg", + ) + ) + + assert len(calls) == 1, "the ceilings must not be a second command" + argv = calls[0] + assert argv[:2] == ("tools.budget", "new") + assert "--gpu-usd" in argv and "50" in argv + assert "--quota-tokens" in argv and "5e6" in argv + # Blank is left alone rather than sent as an empty ceiling. + assert "--credits-usd" not in argv + assert "--payer" in argv and "hf:myorg" in argv + assert "raise" not in argv + + +def test_a_create_with_no_ceilings_still_works(rendered, monkeypatch): + """They are strongly encouraged and not compulsory: `budget new` accepts a + project with none, and refusing here would be this window inventing a rule + the ledger does not have.""" + import asyncio + + from ui import tasks as tasks_mod + + calls: list[tuple] = [] + + async def fake_run_tool(*argv, timeout=120.0, stdin=None): + calls.append(argv) + return {"ok": True, "data": {"message": "created"}} + + monkeypatch.setattr(tasks_mod, "run_tool", fake_run_tool) + monkeypatch.setattr("ui.state.run_tool", fake_run_tool) + + _, space = rendered(["projects"]) + asyncio.run(space.create_project("proj-a", "A", ceilings={"gpu-usd": ""}, payer="")) + + argv = calls[0] + assert "--gpu-usd" not in argv + assert "--payer" not in argv + + +def test_creating_a_project_on_an_unconfigured_machine_opens_setup(rendered, monkeypatch): + """The user's "setup starts when a project is created", narrowed to the case + where there is something left to ask. A wizard that opened on every creation + would ask someone with six projects for their token six times.""" + import asyncio + + from ui import models as models_mod, tasks as tasks_mod + + async def fake_run_tool(*argv, timeout=120.0, stdin=None): + return {"ok": True, "data": {"message": "created"}} + + monkeypatch.setattr(tasks_mod, "run_tool", fake_run_tool) + monkeypatch.setattr("ui.state.run_tool", fake_run_tool) + monkeypatch.setattr(models_mod, "setup_needed", lambda: True) + + _, space = rendered(["projects"]) + asyncio.run(space.create_project("proj-a", "A")) + assert "setup" in space.layout.windows + + +def test_creating_a_project_on_a_configured_machine_opens_nothing(rendered, monkeypatch): + import asyncio + + from ui import models as models_mod, tasks as tasks_mod + + async def fake_run_tool(*argv, timeout=120.0, stdin=None): + return {"ok": True, "data": {"message": "created"}} + + monkeypatch.setattr(tasks_mod, "run_tool", fake_run_tool) + monkeypatch.setattr("ui.state.run_tool", fake_run_tool) + monkeypatch.setattr(models_mod, "setup_needed", lambda: False) + + _, space = rendered(["projects"]) + asyncio.run(space.create_project("proj-a", "A")) + assert "setup" not in space.layout.windows + + +def test_a_failed_create_does_not_open_setup(rendered, monkeypatch): + """`reload` and the wizard both hang off `ok`. A refused id that opened a + wizard would read as though the project had been made.""" + import asyncio + + from ui import models as models_mod, tasks as tasks_mod + + async def fake_run_tool(*argv, timeout=120.0, stdin=None): + return {"ok": False, "error": {"message": "that id is not a slug"}} + + monkeypatch.setattr(tasks_mod, "run_tool", fake_run_tool) + monkeypatch.setattr("ui.state.run_tool", fake_run_tool) + monkeypatch.setattr(models_mod, "setup_needed", lambda: True) + + _, space = rendered(["projects"]) + asyncio.run(space.create_project("not a slug!", "A")) + assert "setup" not in space.layout.windows + + +def test_the_setup_window_can_store_a_credential_without_a_terminal(rendered, monkeypatch): """The one thing the workspace could not do. `credential set` prompts with `getpass`, which needs a terminal -- so a fresh machine needed a shell open beside the app before the app was usable.""" from core import budget as budget_mod - from ui import shell as shell_mod, tasks as tasks_mod + from ui import tasks as tasks_mod calls: list[dict] = [] @@ -716,14 +895,11 @@ async def fake_run_tool(*argv, timeout=120.0, stdin=None): budget_mod.create("proj-a", title="A", budget={}) budget_mod.set_current("proj-a") - client, space = rendered(["chat"]) - card = [e for e in client.elements.values() if "grad-card" in getattr(e, "classes", [])][0] - with client: - shell_mod._draw_project_menu( # noqa: SLF001 - no public hook - __import__("nicegui").ui, space, card, _NullMenu() - ) + client, space = rendered(["setup"]) - assert "CREDENTIALS" in html_of(client) + markup = html_of(client) + assert "subscription token" in markup + assert "claude setup-token" in markup, "the window has to say how to mint one" import asyncio asyncio.run(space.set_credential("hf_token", "hf_the-actual-token")) @@ -986,9 +1162,24 @@ def test_every_window_renders_with_real_data(rendered): "gradnum", # editor source highlighting "different source tree", # wiki staleness "run-1", # queue + # The projects window, with a row in it. Asserted on *content* rather + # than on the element count, for the reason at the top of this file: a + # window whose render raises becomes a card saying so, so a broken + # window still adds elements. `_project` referenced a name that was not + # in its scope and the whole suite went green -- the empty-workspace + # test returned before reaching it, and this one caught the NameError in + # the failure card. + "Scaling", # projects, the seeded title + "ceiling raises", # projects, the per-project detail + "workspace defaults", # projects, models with no override + "subscription token", # setup, its first step ): assert expected in markup, expected + # And nothing anywhere is a failure card. Cheaper than asserting a string + # per window, and it is the actual claim: every window rendered. + assert "failed to render" not in markup + def test_a_populated_workspace_survives_the_full_gesture_sequence(rendered): seed_everything() diff --git a/tests/test_wakeup.py b/tests/test_wakeup.py new file mode 100644 index 0000000..40a87d3 --- /dev/null +++ b/tests/test_wakeup.py @@ -0,0 +1,441 @@ +"""Being woken instead of waiting (HANDOFF §6's unfinished half). + +The thing under test is a *replacement for a habit*: `sleep 30`, look, `sleep +60`, look. So these cover the two properties that make the replacement worth +having -- the agent's shell is not held, and nothing is lost when the condition +happens into a workspace whose window is closed -- and the one that makes it +safe, which is that the endpoint a wake arrives on cannot be driven by anything +that merely knows the port. + +Nothing here spawns a watcher. `wk.watch` is a plain function precisely so the +loop can be driven in-process; `arm` is exercised with the spawn stubbed. +""" + +from __future__ import annotations + +import time + +import pytest + +from core import tasks as tasklib, wakeups as wk +from core.errors import GradError +from tools import wakeup as wakeup_tool + + +@pytest.fixture +def no_spawn(monkeypatch): + """`arm` without the detached process. The watcher is tested separately. + + It reports *this* process as the watcher rather than an invented pid, and + that is not cosmetic: `wakeups()` folds an armed wake whose pid is gone to + `lost`, so a made-up number makes every wake in the suite arrive already + dead -- and `cancel` then correctly declines to cancel it. + """ + import os + + spawned: list[str] = [] + + def fake(wake_id: str) -> int: + spawned.append(wake_id) + return os.getpid() + + monkeypatch.setattr(wk, "spawn_watcher", fake) + return spawned + + +def _arm(**kwargs): + args = { + "after": None, "task": None, "run": None, "file": None, + "changed": False, "timeout": 3600.0, "note": "", "no_resume": False, + } + args.update(kwargs) + return wakeup_tool.cmd_arm(type("A", (), args)()) + + +# --------------------------------------------------------------------------- +# arming +# --------------------------------------------------------------------------- +def test_arming_returns_at_once_and_tells_the_agent_to_stop(workspace, no_spawn): + """The whole point. `arm` must not be a thing you wait on -- if it were, it + would be the `sleep` it replaces with extra steps.""" + started = time.monotonic() + out = _arm(after=3600) + assert time.monotonic() - started < 2.0 + assert out["wake"].startswith("wake-") + assert "end your turn" in out["next"] + + +def test_the_deadline_is_reported_as_a_deadline(workspace, no_spawn): + """It said `now` for its first hour of existence, which is the one value + that is never the answer to 'when does this expire'.""" + out = _arm(after=60, timeout=1800.0) + assert out["expires_at"] > out["wake"][5:], "not an instant at all" + armed_at = wk.get(out["wake"])["armed_at"] + assert out["expires_at"] > armed_at + + +def test_a_timeout_past_the_ceiling_is_refused(workspace, no_spawn): + with pytest.raises(GradError) as exc: + _arm(after=60, timeout=wk.MAX_TIMEOUT_S + 1) + assert "at most" in exc.value.message + assert exc.value.fix + + +def test_waiting_on_a_finished_task_is_refused(workspace, no_spawn): + """It would fire on its first look and spend a whole turn saying what a + `task status` in the same turn would have said for nothing.""" + task_id = tasklib.new_id() + tasklib.record_started( + task_id, label="done", argv=["python", "-c", "pass"], pid=1, halt=None, cwd="." + ) + tasklib.record_exited(task_id, 0) + + with pytest.raises(GradError) as exc: + _arm(task=task_id) + assert "already finished" in exc.value.message + + +def test_waiting_on_a_file_that_is_already_there_is_refused(workspace, no_spawn): + (workspace / "there.txt").write_text("hi", encoding="utf-8") + with pytest.raises(GradError) as exc: + _arm(file="there.txt") + assert "--changed" in (exc.value.fix or "") + + +def test_a_changed_file_baseline_is_taken_at_arm_time(workspace, no_spawn): + """'Changed since you asked', not 'changed since the watcher got round to + looking' -- otherwise a write in between is silently missed.""" + path = workspace / "metrics.json" + path.write_text("{}", encoding="utf-8") + out = _arm(file="metrics.json", changed=True) + + condition = wk.get(out["wake"])["condition"] + assert condition["mtime_ns"] == path.stat().st_mtime_ns + assert wk.check(condition)[0] is False + + time.sleep(0.01) + path.write_text('{"loss": 1}', encoding="utf-8") + fired, detail = wk.check(condition) + assert fired and detail["path"] == str(path) + + +def test_an_unknown_task_is_a_named_refusal_not_a_traceback(workspace, no_spawn): + with pytest.raises(GradError) as exc: + _arm(task="task-000000-dead") + assert "no background task" in exc.value.message + + +# --------------------------------------------------------------------------- +# the conditions +# --------------------------------------------------------------------------- +def test_a_clock_condition_survives_a_sleeping_machine(workspace): + """The deadline is absolute, so a laptop that slept through the interval + wakes to a condition that is already true rather than to a fresh countdown.""" + condition = {"kind": wk.KIND_AFTER, "seconds": 5, "fire_at": time.time() - 1} + assert wk.check(condition)[0] is True + + +def test_a_task_condition_fires_on_the_record_not_on_the_pid(workspace): + """`core/tasks.py` distinguishes 'exited' from 'stopped being alive', and a + wake that could not tell them apart would report a killed supervisor as a + finished job.""" + task_id = tasklib.new_id() + tasklib.record_started( + task_id, label="train", argv=["python", "train.py"], pid=999999, halt=None, cwd="." + ) + condition = {"kind": wk.KIND_TASK, "task": task_id} + + tasklib.record_exited(task_id, 3) + fired, detail = wk.check(condition) + assert fired + assert detail["exit_code"] == 3 + + +def test_an_unreadable_condition_is_not_a_fired_one(workspace, monkeypatch): + """This runs in a detached loop with nowhere to report to, so a condition + that cannot be read is 'look again', never 'it happened'.""" + def explode(_): + raise RuntimeError("the ledger is on fire") + + monkeypatch.setattr(wk, "_check", explode) + fired, detail = wk.check({"kind": wk.KIND_AFTER}) + assert fired is False + assert "unreadable" in detail + + +def test_an_unrecognised_remote_state_is_not_finished(workspace): + """`tools/kaggle.py:_parse_status` returns `unknown` for output it cannot + read, and treating unknown as finished is how you collect a kernel mid-run.""" + assert wk._remote_finished({"kernel": {"status": "unknown"}}) is False + assert wk._remote_finished({"remote_state": "RUNNING"}) is False + assert wk._remote_finished({"kernel": {"status": "complete"}}) is True + assert wk._remote_finished({"remote_state": "COMPLETED"}) is True + # The ssh backend writes a marker file only when the command exits, so the + # marker's presence *is* the terminal signal. + assert wk._remote_finished({"remote": {"exit_code": 0}}) is True + assert wk._remote_finished({"remote": {}}) is False + + +def test_a_run_with_no_status_is_not_a_finished_run(workspace, monkeypatch): + """The same mistake as the line above, one layer down. + + The ledger short-circuit was `status and status != "in_flight"`, which treats + everything unrecognised as finished -- including `"unknown"`, which is what + `Run.status` returns for a fold with no status in it. `runs()` builds a node + from any event carrying an id and `jsonl.iter_records` *skips* a malformed + line rather than raising, so a torn `run_submitted` followed by an intact + `run_handle` produces exactly that record. The wake then fired at once and + spent a metered turn reporting something nothing had checked. + """ + from core import ledger_store as ls + + # The record that damage leaves behind: an id, a platform, no status. + ls.append_run_event({"type": "run_handle", "id": "run-torn", "platform": "hf_jobs"}) + assert ls.run("run-torn").status == "unknown" + + polled: list[str] = [] + + def _never_answers(tool, run_id): + polled.append(run_id) + return None + + monkeypatch.setattr(wk, "_status_envelope", _never_answers) + + fired, detail = wk._check_run("run-torn") + assert fired is False, "unknown is not finished" + assert polled == ["run-torn"], "it has to fall through to the backend, which does know" + assert "unreadable" in detail + + +def test_a_terminal_status_still_fires_without_a_poll(workspace, monkeypatch): + """The short-circuit is still a short-circuit for the statuses that mean it.""" + from core import ledger_store as ls + + def _never(tool, run_id): + raise AssertionError("the backend must not be asked about a finished run") + + monkeypatch.setattr(wk, "_status_envelope", _never) + + for index, status in enumerate(sorted(wk.TERMINAL_RUN_STATUSES)): + run_id = f"run-{index}" + ls.append_run_event({"type": ls.T_RUN_SUBMITTED, "id": run_id, "status": "in_flight"}) + ls.append_run_event({"type": "run_finished", "id": run_id, "status": status}) + fired, detail = wk._check_run(run_id) + assert fired is True, status + assert detail["status"] == status + + +# --------------------------------------------------------------------------- +# the watcher +# --------------------------------------------------------------------------- +def test_the_watcher_fires_and_records_what_it_saw(workspace, no_spawn, monkeypatch): + out = _arm(after=0, timeout=30.0) + monkeypatch.setattr(wk, "deliver", lambda *a, **k: False) + + result = wk.watch(out["wake"]) + assert result["state"] == wk.FIRED + + wake = wk.get(out["wake"]) + assert wake["state"] == wk.FIRED + assert wake["detail"] == {"elapsed_s": 0} + + +def test_a_condition_that_never_happens_expires_as_its_own_state(workspace, no_spawn, monkeypatch): + """'The job finished' and 'the job has not finished in four hours' are + different facts and lead to different next actions, so `expired` is not a + kind of `fired`.""" + out = _arm(file="never.txt", timeout=0.4) + monkeypatch.setattr(wk, "deliver", lambda *a, **k: False) + + result = wk.watch(out["wake"]) + assert result["state"] == wk.EXPIRED + assert wk.get(out["wake"])["state"] == wk.EXPIRED + + +def test_an_expired_wake_says_so_in_the_turn_it_sends(workspace, no_spawn): + out = _arm(file="never.txt", timeout=0.3) + wake = wk.get(out["wake"]) + turn = wk.prompt_for(wake, {}, expired=True) + assert "did not happen within the timeout" in turn + + +def test_the_waking_turn_carries_the_note_back(workspace, no_spawn): + """The agent armed this and knows why; the note is how it tells itself.""" + out = _arm(after=0, note="the 4090 sweep — collect and judge against exp-7") + wake = wk.get(out["wake"]) + turn = wk.prompt_for(wake, {"elapsed_s": 0}, expired=False) + assert "exp-7" in turn + assert out["wake"] in turn + + +def test_cancelling_stops_the_watcher_without_a_signal(workspace, no_spawn): + out = _arm(file="never.txt", timeout=30.0) + wakeup_tool.cmd_cancel(type("A", (), {"wake_id": out["wake"], "reason": "changed my mind"})()) + + result = wk.watch(out["wake"]) + assert result["state"] == wk.CANCELLED + assert wk.get(out["wake"])["state"] == wk.CANCELLED + + +def test_a_wake_with_no_resume_never_tries_to_deliver(workspace, no_spawn, monkeypatch): + delivered: list[str] = [] + monkeypatch.setattr(wk, "deliver", lambda wid, prompt: delivered.append(wid) or True) + + out = _arm(after=0, no_resume=True) + wk.watch(out["wake"]) + assert delivered == [] + + +def test_a_fired_wake_nobody_took_is_kept_not_lost(workspace, no_spawn, monkeypatch): + """A four-hour job finishing into a closed app is ordinary. Losing the wake + there would be the same failure this module exists to prevent, reached by a + different road.""" + monkeypatch.setattr(wk, "deliver", lambda *a, **k: False) + out = _arm(after=0, timeout=30.0) + wk.watch(out["wake"]) + + pending = wk.pending_delivery() + assert [w["id"] for w in pending] == [out["wake"]] + + listing = wakeup_tool.cmd_list(type("A", (), {"all": False})()) + assert listing["undelivered"] == 1 + assert "fired while nothing was listening" in listing["note"] + + +def test_status_prints_the_turn_an_undelivered_wake_would_have_sent(workspace, no_spawn, monkeypatch): + monkeypatch.setattr(wk, "deliver", lambda *a, **k: False) + out = _arm(after=0, note="check the sweep") + wk.watch(out["wake"]) + + status = wakeup_tool.cmd_status(type("A", (), {"wake_id": out["wake"]})()) + assert "check the sweep" in status["turn"] + + +def test_a_delivered_wake_is_recorded_as_delivered(workspace, no_spawn, monkeypatch): + monkeypatch.setattr(wk, "deliver", lambda *a, **k: True) + out = _arm(after=0) + wk.watch(out["wake"]) + + assert wk.get(out["wake"])["delivered"] == "session" + assert wk.pending_delivery() == [] + + +# --------------------------------------------------------------------------- +# the token +# --------------------------------------------------------------------------- +def test_the_token_is_stable_across_calls(workspace): + """The watcher outlives the app: a wake armed before a restart has to still + be deliverable after it.""" + assert wk.token() == wk.token() + assert len(wk.token()) > 20 + + +def test_the_token_is_not_in_the_workspace(workspace): + """It is machine state and the workspace is a repository. A secret that + lands beside the ledger is a secret in someone's next commit.""" + assert workspace not in wk.token_path().parents + + +def test_delivery_without_a_running_instance_is_false_not_an_error(workspace, monkeypatch): + from core import instance + + monkeypatch.setattr(instance, "read_state", lambda: {}) + assert wk.deliver("wake-1", "hello") is False + + +# --------------------------------------------------------------------------- +# where a wake lands +# --------------------------------------------------------------------------- +class _Session: + busy = False + settled: list = [] + + +class _Workspace: + """A `Workspace` reduced to what delivery touches.""" + + from ui.state import Workspace as _real + + MAX_PENDING_WAKES = _real.MAX_PENDING_WAKES + accept_wake = _real.accept_wake + _deliver_wakes = _real._deliver_wakes + + def __init__(self) -> None: + self.session = _Session() + self.pending_wakes: list[str] = [] + self.sent: list[str] = [] + self.said: list[str] = [] + self.opened: list[str] = [] + self.state = "idle" + self.chat_send = lambda prompt: self.sent.append(prompt) + + def say(self, message: str) -> None: + self.said.append(message) + + def open(self, window_id: str) -> None: + self.opened.append(window_id) + + def set_agent_state(self, state: str) -> None: + self.state = state + + +@pytest.mark.asyncio +async def test_a_wake_becomes_a_turn_on_the_next_tick(workspace): + space = _Workspace() + assert space.accept_wake("[wakeup] the run finished") + assert await space._deliver_wakes() is True + assert space.sent == ["[wakeup] the run finished"] + assert space.state == "running" + + +@pytest.mark.asyncio +async def test_a_wake_never_lands_on_a_running_turn(workspace): + """`Session.ask` refuses a prompt during a turn, so delivering into one + would consume the wake and answer nothing -- the silent loss this whole + mechanism exists to prevent.""" + space = _Workspace() + space.session.busy = True + space.accept_wake("[wakeup] the run finished") + + assert await space._deliver_wakes() is False + assert space.sent == [] + assert space.pending_wakes == ["[wakeup] the run finished"] + + space.session.busy = False + assert await space._deliver_wakes() is True + assert space.sent == ["[wakeup] the run finished"] + + +@pytest.mark.asyncio +async def test_a_wake_with_no_chat_window_opens_one(workspace): + space = _Workspace() + space.chat_send = None + space.accept_wake("[wakeup] the run finished") + + assert await space._deliver_wakes() is False + assert space.opened == ["chat"] + assert space.pending_wakes, "the wake was dropped rather than held" + assert any("wakeup" in line for line in space.said) + + +@pytest.mark.asyncio +async def test_one_wake_per_tick(workspace): + space = _Workspace() + space.accept_wake("first") + space.accept_wake("second") + + await space._deliver_wakes() + assert space.sent == ["first"] + await space._deliver_wakes() + assert space.sent == ["first", "second"] + + +def test_a_runaway_watcher_cannot_flood_the_conversation(workspace): + space = _Workspace() + for i in range(_Workspace.MAX_PENDING_WAKES): + assert space.accept_wake(f"wake {i}") + assert space.accept_wake("one too many") is False + + +def test_an_empty_wake_is_refused(workspace): + assert _Workspace().accept_wake(" ") is False diff --git a/tools/budget.py b/tools/budget.py index 63fee36..005bf49 100644 --- a/tools/budget.py +++ b/tools/budget.py @@ -212,6 +212,73 @@ def cmd_raise(args: argparse.Namespace) -> dict[str, Any]: return {"raised": record, "status": budget.status(project_id)} +def _configure_args(p: argparse.ArgumentParser) -> None: + _project_arg(p) + from core import config as config_mod, settings as settings_mod + + for role in config_mod.MODEL_ROLES: + p.add_argument( + f"--{role}", + metavar="MODEL", + help=f"the model this project uses for the {role} role", + ) + p.add_argument( + "--clear", + action="append", + default=[], + metavar="ROLE", + help="drop this project's override, so the role resolves as the workspace's does", + ) + p.add_argument( + "--backend", + choices=settings_mod.BACKENDS, + help="the backend this project reaches for when nothing more specific applies", + ) + p.add_argument("--reason", default="", help="why. it ages badly without one") + + +@cli.command("configure", "what this project overrides about how it is run", setup=_configure_args) +def cmd_configure(args: argparse.Namespace) -> dict[str, Any]: + """A project's own models, as a logged event. + + The model chosen per role is the main lever on both cost and quality, and it + is exactly the thing that should differ between a cheap exploratory project + and one being written up. It is recorded rather than set, because the model a + candidate was mutated by is part of what produced the numbers in the ledger + beside it. + """ + from core import config as config_mod + + project_id = budget.resolve_or_fail(args.project, what="configure") + models: dict[str, str | None] = { + role: getattr(args, role) for role in config_mod.MODEL_ROLES if getattr(args, role, None) + } + for role in args.clear: + models[role] = None + record = budget.configure( + project_id, models=models, backend=args.backend, reason=args.reason + ) + overrides = budget.project_overrides(project_id) + # What *this* project resolves to, across every layer. `config.load()` folds + # in whichever project is currently selected, which is not necessarily the + # one being configured -- `--project other` would otherwise report the + # current project's models beside another project's overrides, and the two + # halves of one answer would be about different projects. + cfg = config_mod.load(reload=True) + models_now = { + role: overrides["models"].get(role) or cfg.model_for(role, project=False) + for role in config_mod.MODEL_ROLES + } + return { + "configured": record, + "overrides": overrides, + # The override alone does not answer "which model will this use", and + # that is the question anyone runs this command to settle. + "models": models_now, + "project": project_id, + } + + @cli.command( "close", "close a project (its records stay; nothing is deleted)", diff --git a/tools/evolve.py b/tools/evolve.py index ccf1a0d..e3b2abc 100644 --- a/tools/evolve.py +++ b/tools/evolve.py @@ -32,10 +32,46 @@ works the moment upstream grows a per-generation entry point: a path that already works should not stop working because a better one arrived. -**Phase 1 is local only, and that is not a placeholder.** A campaign evaluated -entirely through local subprocesses proves the campaign records, the sub-run -bookkeeping and the budget integration while the blast radius is zero. `--remote` -is refused here until phase 2. +**Remote evaluation is phase 2, and the gate is what made it safe to enable.** +Phase 1 was local-only on purpose: a campaign evaluated through local +subprocesses proves the campaign records, the sub-run bookkeeping and the budget +integration while the blast radius is zero. Those are proven, so `--remote` now +puts candidates on real hardware -- behind a refusal that is stricter than the +one for an ordinary job. `--remote-spec` names a pipeline whose preflight must be +complete and passing *including the smoke run*, which is the only check that sees +the real driver stack, the real data path and the real per-device batch size. The +config's `[preflight] checks` list is deliberately not consulted: a machine +configured without `smoke` would otherwise let a loop with no human in it put +forty candidates on hardware nothing had ever run one step on. + +**All three backends, because a candidate is a training run.** A mutation here +changes an architecture or an optimiser, so evaluating one takes minutes to +hours -- which is what makes a fresh container or kernel per candidate a +reasonable unit rather than an absurd one. Each backend owns its own adapter, +because they differ in the one thing that matters: how the mutated program +reaches the machine. `gpu.py` copies the pipeline directory to a host that stays +up; `kaggle.py` swaps one file inside the base64 payload already embedded in the +generated notebook; `jobs.py` has no upload step at all -- the pipeline is in the +image -- so the candidate rides in as a gzipped tar in one environment variable, +unpacked by a prelude in front of the command. + +Every adapter bounds the work *where it runs*, not only where it is watched. A +poll that gives up ends the function; it does not end a detached training run, +and an abandoned candidate keeps holding the GPU that the next one is about to be +measured on -- which would make the next score a measurement of this one's +overrun. + +**Kaggle gets a second gate, because the dollar gate cannot see it.** That +backend rations *hours*, so a campaign priced at zero passes `_campaign_gate` +unconditionally. `_kaggle_hours_gate` projects the campaign against the weekly +allowance before generation 0, and `core/kaggle_quota.py` folds candidate rows +beside runs so the hours are visible afterwards as well -- without that fold a +campaign would burn real GPU hours nothing could account for, and the first +symptom would be an ordinary submission refused. + +A candidate still never becomes a run: no adapter writes a ledger row, the +campaign remains the ledgered unit, and its expectation remains the bound +prediction. **Models.** Sonnet 5 by default, from `[models] evolve`. The ensemble Shinka built its bandit around is replaced by a bandit over *patch types* -- `diff`, @@ -67,6 +103,7 @@ quota_log, ) from core.cli import Cli, main +from core.submission import Submission from core.errors import ( EXIT_CHECK_FAILED, EXIT_PROJECT_BUDGET, @@ -92,9 +129,13 @@ "metered in ledger/quota.jsonl under the `evolve.mutate` stage and bounded by the\n" "project's token allocation. --mutator shinka switches to ShinkaEvolve, which\n" "refuses unless the installed release exposes a per-generation entry point.\n\n" - "Phase 1 is local only. --remote is refused: doing the ledger work and the spend\n" - "work simultaneously against live GPU jobs is how you learn about exit 7 the\n" - "hard way." + "--remote {ssh|hf_jobs|kaggle} --remote-spec evaluates every candidate on\n" + "real hardware. It refuses unless that spec's preflight is complete and passing --\n" + "tests, dry run, and a real smoke run on that hardware -- because a search is a\n" + "loop with no human in it and the environment it lands in has to be proven once,\n" + "before generation 0, rather than rediscovered forty times.\n\n" + "A Kaggle campaign is projected against the weekly accelerator allowance as well,\n" + "since that backend rations hours and the dollar gate cannot see them." ), ) @@ -102,6 +143,20 @@ MUTATOR_SHINKA = "shinka" STAGE_EVOLVE = quota_log.STAGE_EVOLVE +#: Where a candidate can be evaluated. The names are the `platform` strings the +#: run records already use, so a campaign's `backend` and a run's `platform` are +#: the same vocabulary rather than two spellings of one idea. +BACKEND_SSH = "ssh" +BACKEND_HF = "hf_jobs" +BACKEND_KAGGLE = "kaggle" +REMOTE_BACKENDS = (BACKEND_SSH, BACKEND_HF, BACKEND_KAGGLE) + +#: The checks a remote campaign's spec must have passed. Not `[preflight] checks` +#: from the config, and that is the point: a machine configured to skip `smoke` +#: would otherwise let a campaign put forty candidates on hardware nothing had +#: ever run one step on. The gate names what it needs. +REMOTE_REQUIRED_CHECKS = ("tests", "dry_run", "smoke") + # --------------------------------------------------------------------------- # the Shinka boundary -- kept, no longer the default @@ -382,9 +437,27 @@ def _run_args(p: argparse.ArgumentParser) -> None: "--local", action="store_true", default=True, - help="evaluate locally (phase 1; the only supported mode)", + help="evaluate locally, in a subprocess on this machine (the default)", + ) + p.add_argument( + "--remote", + choices=REMOTE_BACKENDS, + help=( + "evaluate every candidate on real hardware instead of locally. Requires " + "--remote-spec, and refuses unless that spec has a complete, passing " + "preflight -- tests, dry run, and a real smoke run on the hardware." + ), + ) + p.add_argument( + "--remote-spec", + help="the pipeline spec whose preflighted environment candidates run in", + ) + p.add_argument( + "--remote-timeout-s", + type=int, + default=0, + help="wall clock per remote candidate; defaults to --timeout-s", ) - p.add_argument("--remote", action="store_true", help="refused: phase 2, behind the campaign gate") p.add_argument( "--set", dest="overrides", @@ -398,14 +471,6 @@ def _run_args(p: argparse.ArgumentParser) -> None: @cli.command("run", "run a budgeted campaign", setup=_run_args) def cmd_run(args: argparse.Namespace) -> dict[str, Any]: - if args.remote: - raise UsageError( - "--remote is phase 2 and is not enabled: the campaign budget gate must be " - "proven against zero-blast-radius local evaluation first. " - "Do not run a single remote generation before that.", - fix="drop --remote; a local campaign exercises the same records and the same gate", - ) - cfg = config_mod.load() paths.ensure_workspace() task_dir = Path(args.task_dir) @@ -429,6 +494,11 @@ def cmd_run(args: argparse.Namespace) -> dict[str, Any]: fix=f"wrap the mutable region in {camp.BLOCK_START} / {camp.BLOCK_END} comments", ) + # Before the expectation is bound and before a campaign id exists, because + # every refusal in here is a configuration problem and none of them should + # cost an expectation that then has to be re-minted. + remote = _remote_target(args, cfg) + # The campaign is the unit of prediction (§21 collision 2). The expectation # is bound here, once, and the candidates below are exempt from the per-run # gate precisely because this binding exists. @@ -460,7 +530,7 @@ def cmd_run(args: argparse.Namespace) -> dict[str, Any]: "max_candidates": max_candidates, "estimate_per_candidate_usd": args.estimate_per_candidate_usd, "projected_cost_usd": round(projected, 4), - "mode": "local", + **_remote_note(remote), "mutator": args.mutator, "model": cfg.model_for("evolve"), "seed": seed, @@ -489,6 +559,7 @@ def cmd_run(args: argparse.Namespace) -> dict[str, Any]: project_id=project_id, seed=seed, cfg=cfg, + remote=remote, ) except BaseException as exc: # noqa: BLE001 - including KeyboardInterrupt camp.close_campaign( @@ -563,6 +634,188 @@ def _campaign_gate( ) +# --------------------------------------------------------------------------- +# where candidates run +# --------------------------------------------------------------------------- +def _remote_target(args: argparse.Namespace, cfg: config_mod.Config) -> dict[str, Any] | None: + """Resolve and gate `--remote`. `None` means the campaign evaluates locally. + + **The gate is the whole function.** Everything below the first two refusals + exists to answer one question: has the environment these candidates will land + in already been through the ordinary §6 path? A remote campaign is the one + place in this system where a loop with no human in it spends money in a + tight cycle, so the answer has to be yes *before* generation 0, for the same + reason `_campaign_gate` runs there rather than at generation 40. + """ + if not args.remote: + if args.remote_spec: + raise UsageError( + "--remote-spec names an environment but no --remote backend to run it on", + fix=f"--remote {BACKEND_SSH} --remote-spec {args.remote_spec}", + ) + return None + + if not args.remote_spec: + raise UsageError( + "--remote needs the spec whose preflighted environment candidates run in: " + "there is no such thing as 'the remote' in general, only a pipeline that has " + "been proven on one", + fix="--remote-spec pipeline/spec.toml", + ) + + sub = Submission.load(args.remote_spec) + _remote_gate(sub, cfg) + + target: dict[str, Any] = { + "backend": args.remote, + "spec": str(sub.spec_path), + "submission_hash": sub.hash(), + "sub": sub, + } + + if args.remote == BACKEND_SSH: + host_name = str(sub.target.get("host") or "") + if not host_name: + raise ConfigError( + f"{sub.spec_path} names no [target] host, so there is nowhere to send candidates", + fix="add `host = \"\"` under [target], matching a [hosts.*] entry", + ) + # Resolved here rather than per candidate: an unknown host is a + # configuration error and it should be one before generation 0, not + # forty evaluations in. Every backend below resolves the equivalent for + # the same reason. + host = cfg.host(host_name) + target.update({"host": host.name, "rate_usd_per_hour": host.rate_usd_per_hour}) + return target + + if args.remote == BACKEND_KAGGLE: + from tools import kaggle as kaggle_tool # noqa: PLC0415 - optional deps + + accelerator = kaggle_tool.resolve_accelerator(None, sub, cfg) + kind = cfg.accelerator_kind(accelerator) + target.update({"accelerator": accelerator, "accelerator_kind": kind}) + _kaggle_hours_gate(cfg, args, sub, accelerator=accelerator, kind=kind) + return target + + from tools import jobs as jobs_tool # noqa: PLC0415 - optional deps + + flavor = sub.target.get("flavor") or cfg.get("hf", "default_flavor", "a10g-small") + # Refused before generation 0 rather than priced at zero: an unpriced flavor + # makes the campaign's projected cost a fiction, and the campaign budget gate + # is the only thing standing between a search and an allocation. + if jobs_tool.flavor_rate(flavor, cfg) is None: + raise ConfigError( + f"flavor {flavor!r} has no rate in [hf.flavor_rates], so a campaign on it " + "cannot be priced", + fix=f'add `"{flavor}" = ` under [hf.flavor_rates] in config/grad.toml', + ) + target.update({"flavor": flavor, "rate_usd_per_hour": jobs_tool.flavor_rate(flavor, cfg)}) + return target + + +def _kaggle_hours_gate( + cfg: config_mod.Config, + args: argparse.Namespace, + sub: Submission, + *, + accelerator: str, + kind: str, +) -> None: + """Refuse a campaign that cannot fit in the week's accelerator hours. + + **The dollar gate cannot see this one.** Kaggle rations *hours*, not money, + so a campaign priced at zero passes `_campaign_gate` unconditionally and + would then spend the whole weekly GPU allowance -- with the first symptom + being an ordinary submission refused for hours nothing could account for. + `core/kaggle_quota.py` folds candidate rows beside runs so the hours are + visible after the fact; this is what stops them being spent in the first + place. + + Projected the same way the dollar gate projects: the per-candidate estimate + times the whole campaign, checked before generation 0 rather than at + generation 40. The per-candidate number is the spec's own estimate, which is + the same one a submission of this pipeline would be gated on. + """ + from core import kaggle_quota # noqa: PLC0415 + from tools import kaggle as kaggle_tool # noqa: PLC0415 + + per_candidate = kaggle_tool.estimated_hours(sub) + candidates = max(1, args.generations) * max(1, args.population) + projected = per_candidate * candidates + + # Two ceilings, and they take *different* numbers -- which is the whole + # reason this is not one call to `kaggle_quota.check`. The session cap is + # what Kaggle stops a single kernel at, so it is asked about one candidate; + # handing it the campaign total would refuse a perfectly ordinary search of + # twenty one-hour candidates for exceeding a twelve-hour session. The weekly + # allowance is the opposite: it is about the pool, so it gets the projection + # for the whole campaign. + # + # Both *raise* rather than return a refusal, and their messages and fixes are + # already the right ones -- `quota_weekly` names what is holding the hours + # and points at `kaggle quota --json`. The session cap passes through + # untouched for that reason. Only the weekly one is re-framed, because "this + # run estimates 160h" is a confusing way to describe forty four-hour + # candidates, and the number a reader needs is the shape of the campaign. + kaggle_quota.check_session(cfg, kind, per_candidate, accelerator=accelerator) + + try: + kaggle_quota.check_quota(cfg, kind, projected, accelerator=accelerator) + except GateRefusal as exc: + raise GateRefusal( + exc.code, + f"a campaign of {candidates} candidates at {per_candidate:.2f}h each projects " + f"{projected:.1f} {kind} hours, which does not fit the week's allowance. " + f"{exc.message}", + exc.exit_code, + fix=( + "lower --generations/--population, shorten the evaluation, or wait for the " + "rolling week to move: python -m tools.kaggle quota --json" + ), + detail=exc.detail, + ) from None + + +def _remote_gate(sub: Submission, cfg: config_mod.Config) -> None: + """Refuse unless this spec has a complete, passing preflight including smoke. + + `gates.check_preflight` is reused rather than reimplemented, and the required + list is named here rather than read from `[preflight] checks`. That is + deliberate: the config's list is a machine's policy for ordinary + submissions, and a machine configured without `smoke` would otherwise let a + campaign put every candidate it has on hardware that nothing has ever run a + single step on. The smoke run is the only check that sees the real driver + stack, the real data path and the real per-device batch size, which is + exactly the set of things a search will otherwise discover forty times. + """ + from core import gates # noqa: PLC0415 + + gates.check_preflight(sub, cfg, required=list(REMOTE_REQUIRED_CHECKS)) + + +def _remote_note(target: dict[str, Any] | None) -> dict[str, Any]: + """The target as it goes into the campaign record -- without the Submission. + + `Submission` is a live object with paths and a resolved config in it; the + campaign record is JSON that outlives this process and gets read by hand. + """ + if target is None: + return {"mode": "local"} + note = { + "mode": "remote", + "backend": target["backend"], + "remote_spec": target["spec"], + "submission_hash": target["submission_hash"], + } + # Only what this backend actually resolved. A record carrying `host: null` + # on a Kaggle campaign reads as a host that could not be found rather than + # as a dimension that does not apply. + for key in ("host", "rate_usd_per_hour", "accelerator", "accelerator_kind", "flavor"): + if target.get(key) is not None: + note[key] = target[key] + return note + + # --------------------------------------------------------------------------- # the loop # --------------------------------------------------------------------------- @@ -576,6 +829,7 @@ def _drive( seed: int, cfg: config_mod.Config, mutator: Any = None, + remote: dict[str, Any] | None = None, ) -> dict[str, Any]: """Generation by generation, with the gate between each. @@ -683,6 +937,9 @@ def _drive( timeout_s=args.timeout_s, per_candidate=per_candidate, jobs=max(1, args.eval_jobs), + remote=remote, + cfg=cfg, + remote_timeout_s=int(args.remote_timeout_s or args.timeout_s), ) evaluated += len(records) scores = [ @@ -742,6 +999,9 @@ def _evaluate_generation( timeout_s: int, per_candidate: float, jobs: int, + remote: dict[str, Any] | None = None, + cfg: Any = None, + remote_timeout_s: int = 0, ) -> list[dict[str, Any]]: """Evaluate one generation, at most `jobs` at once. @@ -751,6 +1011,11 @@ def _evaluate_generation( the thing holding the GPU, and four of those at once is four out-of-memory failures recorded as four bad mutations. + That default is *local* reasoning, and it is why `--eval-jobs` is worth + raising on a remote campaign: an ssh evaluation blocks the thread on the + network rather than on this machine's GPU, and the host's own capacity is + the thing to size it against instead. + Results come back in plan order rather than completion order, so a campaign with the same seed produces the same `candidate_id` for the same slot. The ledger appends in whatever order they finish, which is fine -- `candidates()` @@ -767,6 +1032,9 @@ def one(item: tuple[int, dict[str, Any]]) -> dict[str, Any]: task_dir=task_dir, timeout_s=timeout_s, per_candidate=per_candidate, + remote=remote, + cfg=cfg, + remote_timeout_s=remote_timeout_s or timeout_s, ) items = list(enumerate(proposals)) @@ -786,13 +1054,25 @@ def _evaluate_candidate( task_dir: Path, timeout_s: int, per_candidate: float, + remote: dict[str, Any] | None = None, + cfg: Any = None, + remote_timeout_s: int = 0, ) -> dict[str, Any]: - """Evaluate one candidate locally and record it as a sub-run. + """Evaluate one candidate and record it as a sub-run. Candidates go to `ledger/candidates.jsonl`, never to `runs.jsonl`: a 100-generation campaign is thousands of rows and would dominate a ledger meant to be read by hand (§23 item 4). Only a promoted candidate becomes a - run. + run. **That holds on a remote campaign too** -- see + `tools/gpu.py:evaluate_candidate` for why a per-candidate ledger row would + undo the rule the moment the search left this machine, and where the gate + sits instead. + + Everything up to the point of execution is identical either way: the same + escape check, the same two files written into the same local working + directory. A remote candidate then runs those files on the host instead of + in a subprocess, and its `cost_usd` becomes a measurement rather than the + campaign's per-candidate estimate. """ candidate_id = candidate_id_for(campaign_id, generation, index) source = proposal.get("source") or "" @@ -860,6 +1140,22 @@ def _evaluate_candidate( camp.append_candidate(record) return record + if remote is not None: + record.update( + _evaluate_remotely( + remote=remote, + cfg=cfg, + candidate_id=candidate_id, + source=source, + evaluator=evaluate_src, + timeout_s=remote_timeout_s or timeout_s, + workdir=workdir, + started=started, + ) + ) + camp.append_candidate(record) + return record + try: proc = subprocess.run( [sys.executable, "evaluate.py"], @@ -909,6 +1205,158 @@ def _evaluate_candidate( return record +def _evaluate_remotely( + *, + remote: dict[str, Any], + cfg: Any, + candidate_id: str, + source: str, + evaluator: str, + timeout_s: int, + workdir: Path, + started: float, +) -> dict[str, Any]: + """Run one candidate on the campaign's host and read its metrics back. + + The returned dict is the same set of fields the local path fills in, plus + where it ran. Two of them differ in meaning and both differences are the + point of going remote at all: + + * `cost_usd` is measured -- wall clock against the host's rate -- rather + than the campaign's flat per-candidate estimate. The estimate is what the + budget gate projects with; this is what was actually spent. + * `error` can now be a transport failure rather than a bad mutation. Those + are recorded distinctly, because a search that reads "the host refused the + connection" as "this idea scored nothing" will quietly select against + whatever was being proposed when the network wobbled. + """ + result = _run_on_backend( + remote, + cfg or config_mod.load(), + candidate_id=candidate_id, + files={"initial.py": source, "evaluate.py": evaluator}, + timeout_s=int(timeout_s), + artifacts=workdir, + ) + + output = str(result.get("output") or "") + (workdir / "evaluate.log").write_text(output, encoding="utf-8") + + fields: dict[str, Any] = { + "duration_s": round(time.time() - started, 3), + "cost_usd": float(result.get("cost_usd") or 0.0), + "ran_on": result.get("where"), + "backend": remote["backend"], + } + # Kaggle rations hours rather than dollars, so the number that bounds a + # campaign there is not `cost_usd`. Recorded under the field names + # `core/kaggle_quota.py` folds, which is what lets a campaign's candidates + # count against the weekly allowance at all -- they never reach `runs.jsonl`, + # so the fold has nowhere else to read them from. + if result.get("hours") is not None: + from core import kaggle_quota # noqa: PLC0415 + + fields[kaggle_quota.F_ACTUAL] = float(result["hours"]) + fields[kaggle_quota.F_ACCELERATOR] = result.get("accelerator") + fields[kaggle_quota.F_KIND] = result.get("accelerator_kind") + + if result.get("exit_code") is None: + # The candidate never ran. Recorded as `skipped` for the same reason an + # operator that produced nothing is: it has no score, and folding it into + # the population as a zero would teach the next generation that whatever + # was proposed here is bad. + fields.update( + { + "skipped": True, + "metrics": None, + "error": f"the host could not run it: {result.get('error')}", + } + ) + return fields + + metrics, problem = _metrics_from(output) + if not result.get("ok") and problem is None: + problem = str(result.get("error") or "the candidate exited non-zero") + fields.update({"metrics": metrics if problem is None else None, "error": problem}) + return fields + + +def _run_on_backend( + remote: dict[str, Any], + cfg: Any, + *, + candidate_id: str, + files: dict[str, str], + timeout_s: int, + artifacts: Path, +) -> dict[str, Any]: + """Hand one candidate to whichever backend the campaign is running on. + + The three adapters answer the same question and return the same shape -- ok, + exit code, output, error, cost, where -- but they get there differently + enough that a shared implementation would be a lie: `gpu.py` copies a + directory to a machine that stays up, `kaggle.py` packs the pipeline into a + notebook, and `jobs.py` has no upload step at all because the pipeline is + already in the image. Each one's own module owns that difference, which is + the same division `core/submit.py` already draws for real submissions. + + Imported at the point of use, because each backend brings optional + dependencies and a campaign on one of them must not need the others + installed. + """ + backend = remote["backend"] + common = { + "candidate_id": candidate_id, + "files": files, + "command": ["python", "evaluate.py"], + "timeout_s": int(timeout_s), + } + + if backend == BACKEND_SSH: + from tools import gpu as gpu_tool # noqa: PLC0415 + + return gpu_tool.evaluate_candidate(remote["sub"], cfg, **common) + + if backend == BACKEND_KAGGLE: + from tools import kaggle as kaggle_tool # noqa: PLC0415 + + return kaggle_tool.evaluate_candidate( + remote["sub"], cfg, artifacts=artifacts, + accelerator=remote.get("accelerator"), **common, + ) + + from tools import jobs as jobs_tool # noqa: PLC0415 + + return jobs_tool.evaluate_candidate( + remote["sub"], cfg, artifacts=artifacts, flavor=remote.get("flavor"), **common + ) + + +def _metrics_from(output: str) -> tuple[Any, str | None]: + """The evaluator's one JSON object, out of a combined stdout/stderr stream. + + The local path can read stdout on its own; over ssh the two are merged so a + traceback is not lost, and `_ssh` appends the `EXIT:` line the exit code is + read from. So the marker is dropped and the *last* line is taken -- the same + rule the local path uses, deliberately, rather than a more forgiving scan. + A search whose metric can be found anywhere in the output is a search that + can be fed a number by a log line. + """ + lines = [ + line for line in output.strip().splitlines() if not line.startswith("EXIT:") + ] + while lines and not lines[-1].strip(): + lines.pop() + if not lines: + return None, "the candidate printed nothing" + try: + metrics = json.loads(lines[-1]) + except json.JSONDecodeError: + tail = lines[-1].strip()[:300] + return None, f"evaluate.py did not print a JSON object of metrics: {tail}" + return metrics, camp.validate_metrics(metrics) + + # --------------------------------------------------------------------------- # mutation engines # --------------------------------------------------------------------------- diff --git a/tools/gpu.py b/tools/gpu.py index e3ef892..bfe4bf7 100644 --- a/tools/gpu.py +++ b/tools/gpu.py @@ -14,6 +14,8 @@ from __future__ import annotations import argparse +import base64 +import logging import os import shlex import stat @@ -33,9 +35,18 @@ ) from core.cli import Cli, main from core.config import Config, Host -from core.errors import EXIT_RUNNING, ConfigError, GradError, UpstreamError, UsageError +from core.errors import ( + EXIT_RUNNING, + EXIT_USAGE, + ConfigError, + GradError, + UpstreamError, + UsageError, +) from core.submission import Submission, parse_override +log = logging.getLogger("grad.gpu") + cli = Cli( "grad-gpu", "Submit and collect jobs on known SSH GPU hosts.", @@ -359,6 +370,235 @@ def run_smoke( } +# --------------------------------------------------------------------------- +# evolve candidates +# --------------------------------------------------------------------------- +#: How many bytes of a candidate's output are kept. An evaluator prints one JSON +#: object by contract, but a failing one prints a traceback, and a campaign of +#: forty candidates should not be able to write forty training logs into +#: `candidates.jsonl`. +CANDIDATE_OUTPUT_BYTES = 8000 +#: Seconds between marker reads while a candidate trains. Each one is an ssh +#: round trip, and the thing being waited for takes minutes to hours. +CANDIDATE_POLL_S = 10.0 +#: How long past its own `timeout` a candidate is given to record a marker. +#: Enough for the remote `timeout` to fire, the shell to write the marker and one +#: poll to read it; anything beyond this is a host that has stopped answering. +CANDIDATE_TIMEOUT_GRACE_S = 60.0 + + +def evaluate_candidate( + sub: Submission, + cfg: Config, + *, + candidate_id: str, + files: dict[str, str], + command: list[str], + timeout_s: int, + host: Host | None = None, +) -> dict[str, Any]: + """Run one evolve candidate on a known host, and return what it printed. + + **This deliberately writes no run record.** A candidate is not a run -- + `tools/evolve.py` records them in `ledger/candidates.jsonl` precisely so a + campaign of thousands cannot dominate a ledger meant to be read by hand + (§23 item 4) -- and a campaign that emitted a ledger row per candidate would + undo that the moment it went remote. The campaign is the ledgered unit, its + expectation is the bound prediction, and its cost is the sum of these. + + That is *not* a hole in the gates, and the difference is where the gate + sits. `tools/evolve.py:_remote_gate` refuses the whole campaign unless the + spec has a complete, passing preflight -- tests, dry run, **and a real smoke + run on this hardware** -- so the environment every candidate lands in is one + that has already been proven by the ordinary §6 path. What varies per + candidate is the contents of one marked region, which is the thing the + campaign is searching over. + + The pipeline directory is staged fresh per candidate rather than reused, + which is a few seconds of `scp` bought for the property that matters in a + search: candidate N cannot see a file candidate N-1 left behind. An + evolutionary loop that can accumulate state across evaluations is one whose + scores stop being comparable, and the failure looks like a real improvement. + + **The job is detached and its marker polled, not held open on the ssh + channel.** The first version of this ran the evaluator inside a single + synchronous `ssh` with a timeout, which is the right shape only if a + candidate is quick. A candidate here is a changed architecture or a changed + optimiser, so its evaluation is a training run -- minutes to hours -- and a + single TCP connection held open across that is a connection that a NAT + timeout, a sleeping laptop or a wifi handover will drop. What you get then + is not a failed candidate, it is a candidate that *scored nothing* because + the network moved, which the search then selects against. `cmd_submit` has + used `nohup` plus `grad_status.json` since it was written, for exactly this + reason; this uses the same two functions. + """ + host = host or cfg.host(sub.target.get("host") or "") + remote_dir = f"{host.workdir}/{candidate_id}" + started = time.time() + + def _failed(message: str) -> dict[str, Any]: + _discard(host, remote_dir) + return { + "ok": False, + "exit_code": None, + "output": "", + "error": message, + "cost_usd": round((time.time() - started) / 3600.0 * host.rate_usd_per_hour, 4), + "host": host.name, + "where": f"{host.name}:{remote_dir}", + } + + try: + _stage(host, sub, remote_dir) + for name, text in files.items(): + _write_remote(host, remote_dir, name, text) + # `timeout` on the *remote* side, not only in the poll below. The poll + # giving up would end this function; it would not end the training run, + # because the job is detached by design -- and an abandoned candidate + # keeps holding the GPU that the next one is about to be measured on. + # Bounding it where it runs means a candidate that overruns exits 124 + # and writes a marker like any other failure. + pid = _launch(host, sub, remote_dir, ["timeout", str(int(timeout_s)), *command]) + except GradError as exc: + return _failed(exc.message) + + # Longer than the remote bound, so the ordinary overrun is observed as a + # `timeout` exit rather than as this loop giving up on a job that is about + # to record one. + deadline = time.time() + int(timeout_s) + CANDIDATE_TIMEOUT_GRACE_S + marker: dict[str, Any] = {} + while time.time() < deadline: + try: + marker = _marker(host, remote_dir) + except GradError: + # A single unreadable marker is a network hiccup, not a verdict. The + # deadline is what stops this being forever, and the job is still + # running on the host either way -- which is the whole point of + # having detached it. + marker = {} + if marker.get("state") == "finished": + break + time.sleep(CANDIDATE_POLL_S) + + cost = round((time.time() - started) / 3600.0 * host.rate_usd_per_hour, 4) + if marker.get("state") != "finished": + # The remote `timeout` should have ended it and written a marker, so + # reaching here means the host stopped answering rather than that the + # candidate overran. Killed anyway, and killed before the directory is + # removed: the loop starts the next candidate the moment this returns, + # and one left running is a process competing with its own successor for + # the same GPU -- which would make the next score a measurement of this + # one's overrun rather than of the mutation. + _kill_remote(host, pid) + _discard(host, remote_dir) + return { + "ok": False, + "exit_code": None, + "output": "", + "error": f"the candidate did not finish within {int(timeout_s)}s on {host.name}", + "cost_usd": cost, + "host": host.name, + "where": f"{host.name}:{remote_dir}", + } + + exit_code = int(marker.get("exit_code") or 0) + output = _read_remote_logs(host, remote_dir) + _discard(host, remote_dir) + return { + "ok": exit_code == 0, + "exit_code": exit_code, + "output": output[-CANDIDATE_OUTPUT_BYTES:], + "error": None if exit_code == 0 else f"the candidate exited {exit_code} on {host.name}", + "cost_usd": cost, + "host": host.name, + "where": f"{host.name}:{remote_dir}", + } + + +def _read_remote_logs(host: Host, remote_dir: str) -> str: + """`stdout.log` then `stderr.log`, in that order and bounded. + + `_launch` sends the two streams to separate files, so unlike the synchronous + version there is no interleaving to rely on -- and the order matters to the + caller: `tools/evolve.py:_metrics_from` reads the *last* line, and the + evaluator's one JSON object is on stdout. Appending stderr after it would + put a traceback's last line where the metrics belong. + + So stdout is read last. What comes back is stderr first, then stdout, which + reads oddly in a log and is the only ordering that keeps the contract. + """ + parts: list[str] = [] + for name in ("stderr.log", "stdout.log"): + try: + text = _ssh( + host, + f"tail -c {CANDIDATE_OUTPUT_BYTES} {shlex.quote(remote_dir + '/' + name)} 2>/dev/null || true", + timeout=120, + ) + except GradError: + continue + if text.strip(): + parts.append(text.rstrip("\n")) + return "\n".join(parts) + + +def _write_remote(host: Host, remote_dir: str, name: str, text: str) -> None: + """Put one file on the host, through base64 rather than through a heredoc. + + The content is a mutated Python source produced by a language model. It can + contain anything a heredoc terminator, a quote or a backtick means to a + shell, and the failure mode of getting that wrong is not an error -- it is a + file that arrives subtly different from the one that was scored, which is a + campaign whose records describe code that never ran. + + The name is checked rather than quoted: it comes from this module's own + callers today, and a path that could climb out of the working directory is + worth refusing at the one place it would be written rather than trusting + every future caller. + """ + if "/" in name or "\\" in name or name in ("", ".", ".."): + raise GradError( + "bad_remote_name", + f"refusing to write {name!r} on {host.name}: a candidate file is a plain name", + exit_code=EXIT_USAGE, + fix="pass a file name, not a path", + ) + blob = base64.b64encode(text.encode("utf-8")).decode("ascii") + _ssh( + host, + f"cd {shlex.quote(remote_dir)} && printf %s {shlex.quote(blob)} | base64 -d > {shlex.quote(name)}", + timeout=120, + ) + + +def _kill_remote(host: Host, pid: str) -> None: + """End a detached candidate and whatever it started. Never raises. + + Children first, then the wrapper: `_launch` runs the command under + `sh -c`, so the pid it echoed is the shell's and killing only that leaves + the training process orphaned but very much alive on the GPU. + """ + if not pid or not pid.strip().isdigit(): + return + try: + _ssh(host, f"pkill -P {int(pid)} 2>/dev/null; kill {int(pid)} 2>/dev/null; true", timeout=120) + except Exception: # noqa: BLE001 - a cleanup must not become the failure + log.debug("could not kill %s on %s", pid, host.name, exc_info=True) + + +def _discard(host: Host, remote_dir: str) -> None: + """Remove a candidate's working directory. Never raises. + + Best-effort on purpose: a campaign must not fail because a cleanup did, and + the directory is named after the candidate, so what is left behind on a host + that refused is identifiable rather than anonymous. + """ + try: + _ssh(host, f"rm -rf {shlex.quote(remote_dir)}", timeout=120) + except Exception: # noqa: BLE001 - see the docstring + log.debug("could not remove %s on %s", remote_dir, host.name, exc_info=True) + + def _exit_code_from(output: str) -> int: for line in reversed(output.splitlines()): if line.startswith("EXIT:"): diff --git a/tools/jobs.py b/tools/jobs.py index 69a4e03..a68ed98 100644 --- a/tools/jobs.py +++ b/tools/jobs.py @@ -14,6 +14,7 @@ import argparse import datetime as _dt import json +import shlex import sys import time from pathlib import Path @@ -470,6 +471,205 @@ def run_smoke( } +# --------------------------------------------------------------------------- +# evolve candidates +# --------------------------------------------------------------------------- +#: How many bytes of a candidate's output are kept, matching `tools/gpu.py`. +CANDIDATE_OUTPUT_BYTES = 8000 +#: The environment variable the candidate's files ride in. Named rather than +#: positional so a person reading a job's configuration on the Hub can see what +#: it is. +CANDIDATE_ENV = "GRAD_CANDIDATE_B64" +#: How large that blob may get. This is a *container environment variable*, not +#: a file: the practical ceiling is the platform's, it is not documented, and +#: discovering it by exceeding it means a job that fails for a reason with +#: nothing to do with the research. A couple of Python modules gzip to a few +#: kilobytes, so anything approaching this is a pipeline being smuggled through +#: the wrong door. +MAX_CANDIDATE_B64 = 200_000 + + +def evaluate_candidate( + sub: Submission, + cfg: Config, + *, + candidate_id: str, + files: dict[str, str], + command: list[str], + timeout_s: int, + artifacts: Path, + flavor: str | None = None, + namespace: str | None = None, +) -> dict[str, Any]: + """Run one evolve candidate as a Hugging Face Job, and read its metrics back. + + **The delivery path is the interesting part, and it is different from the + other two backends.** `gpu.py` copies the pipeline directory to a host and + `kaggle.py` packs it into the notebook; on HF Jobs the pipeline is *in the + image*, and there is no upload step at all -- `_command_for` just runs the + entrypoint the image already contains. So a candidate, which is by definition + a program the image does not contain, needs a way in. + + It rides as a gzipped tar in one environment variable, unpacked by a prelude + in front of the command. That is the same shape as Kaggle's embedded payload + and it is deliberately the *small* version of it: only the candidate's own + files travel, because everything else is already in the image that the + preflight proved. If that starts to look like a way to ship a pipeline, the + size refusal above says so rather than letting it half-work. + + The files land in the container's working directory, which is the image's + `WORKDIR` -- the same place `_command_for` runs the entrypoint from. A spec + whose image puts the pipeline somewhere else needs `[target] command` to say + so, exactly as it already would for a submission. + + Like the other adapters: no ledger row, cost measured rather than estimated, + and a transport failure reported as one rather than as a candidate that + scored nothing. + """ + flavor = flavor or sub.target.get("flavor") or cfg.get("hf", "default_flavor", "a10g-small") + project = budget.current_project() + if namespace is None: + namespace = resolve_namespace(None, sub, cfg, project) + artifacts.mkdir(parents=True, exist_ok=True) + started = time.time() + + def _failed(message: str, **extra: Any) -> dict[str, Any]: + return { + "ok": False, + "exit_code": None, + "output": "", + "error": message, + "cost_usd": round(_elapsed_cost(started, flavor, cfg), 4), + "flavor": flavor, + "namespace": namespace, + "where": f"hf:{namespace or ''}", + **extra, + } + + try: + blob = _candidate_blob(files) + except GradError as exc: + return _failed(exc.message) + + # Everything that can fail for a configuration reason resolves before the + # job exists, for the reason `run_smoke` gives about phantom estimates. + try: + hub = _hub() + ns_kwargs = _ns_kwargs(namespace) + token = _token() + except GradError as exc: + return _failed(exc.message) + + try: + job = hub.run_job( + image=sub.image, + command=_candidate_command(command), + flavor=flavor, + env={**_job_env(sub), CANDIDATE_ENV: blob, "GRAD_CANDIDATE": candidate_id}, + token=token, + timeout=int(timeout_s), + **ns_kwargs, + ) + except Exception as exc: # noqa: BLE001 - a refused submission is not a bad mutation + return _failed(f"the candidate could not be submitted: {exc}") + + job_id = getattr(job, "id", None) or getattr(job, "job_id", None) or str(job) + state, info = _poll(job_id, deadline=time.time() + int(timeout_s), namespace=namespace) + logs = _logs(job_id, namespace=namespace) + (artifacts / "candidate.log").write_text(logs, encoding="utf-8") + cost, _warning = _actual_cost(info, flavor, cfg, estimate_usd=0.0) + + ok = state == "COMPLETED" + return { + "ok": ok, + # HF reports a *state*, not an exit code. `0` on COMPLETED and `1` + # otherwise would be inventing a number nobody measured, so the state is + # what is reported and `exit_code` stays None -- which the driver already + # distinguishes from a candidate that never ran. + "exit_code": 0 if ok else None, + "output": logs[-CANDIDATE_OUTPUT_BYTES:], + "error": None if ok else f"the candidate's job ended in state {state}", + "cost_usd": cost, + "flavor": flavor, + "namespace": namespace, + "job_state": state, + "where": f"hf:{namespace}/{job_id}" if namespace else f"hf:{job_id}", + } + + +def _elapsed_cost(started: float, flavor: str, cfg: Config) -> float: + rate = flavor_rate(flavor, cfg) or 0.0 + return (time.time() - started) / 3600.0 * float(rate) + + +def _candidate_blob(files: dict[str, str]) -> str: + """The candidate's files as one base64 gzipped tar. + + Deterministic in the same way `kaggle.py:_payload_b64` is -- sorted names, + `mtime=0` -- so the same candidate produces the same blob, which is what + makes two job configurations comparable when one of them misbehaves. + """ + import base64 # noqa: PLC0415 + import io # noqa: PLC0415 + import tarfile # noqa: PLC0415 + + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w:gz", compresslevel=9) as tar: + for name in sorted(files): + if name.startswith("/") or ".." in Path(name).parts: + raise UsageError( + f"refusing to send {name!r}: a candidate file is a path inside the workdir", + fix="pass a name relative to the image's working directory", + ) + data = str(files[name]).encode("utf-8") + info = tarfile.TarInfo(name) + info.size = len(data) + info.mtime = 0 + info.mode = 0o644 + info.uid = info.gid = 0 + info.uname = info.gname = "" + tar.addfile(info, io.BytesIO(data)) + blob = base64.b64encode(buffer.getvalue()).decode("ascii") + if len(blob) > MAX_CANDIDATE_B64: + raise UsageError( + f"the candidate packs to {len(blob):,} base64 bytes, past the " + f"{MAX_CANDIDATE_B64:,} this backend will put in an environment variable", + fix=( + "keep the evolve block to code -- the pipeline belongs in the image the " + "preflight proved, not in the candidate" + ), + ) + return blob + + +#: Unpacks `CANDIDATE_ENV` into the working directory. A `python -c` rather than +#: a shell one-liner because the payload is base64 and `base64 -d` is not on +#: every image; Python is, by construction, since the entrypoint is Python. +_UNPACK = ( + "import base64,io,os,tarfile;" + f"b=os.environ['{CANDIDATE_ENV}'];" + "t=tarfile.open(fileobj=io.BytesIO(base64.b64decode(b)));" + # `filter='data'` where the interpreter has it. The image's Python is not + # this machine's, so the version that matters cannot be checked from here -- + # the same reason `kaggle.py` names it conditionally. + "t.extractall('.', filter='data') if hasattr(tarfile,'data_filter') else t.extractall('.');" + "print('grad: unpacked', len(t.getnames()), 'candidate files')" +) + + +def _candidate_command(command: list[str]) -> list[str]: + """The command with the unpack in front of it. + + `sh -c` with the two joined by `&&`, so a failed unpack is a failed job + rather than a job that runs the image's *own* entrypoint against the + candidate's name and reports a score for the wrong program. That is the + failure worth engineering against here: it would not look like an error, it + would look like every candidate scoring the same. + """ + inner = " ".join(shlex.quote(c) for c in command) + return ["sh", "-c", f"python -c {shlex.quote(_UNPACK)} && {inner}"] + + def _smoke_command(sub: Submission, caps: dict[str, Any]) -> list[str]: """One step, real per-device batch size, truncated sequence count. diff --git a/tools/kaggle.py b/tools/kaggle.py index 99ae7bd..f256d61 100644 --- a/tools/kaggle.py +++ b/tools/kaggle.py @@ -36,7 +36,6 @@ import json import os import re -import shutil import subprocess import tarfile import tempfile @@ -155,11 +154,22 @@ def is_secret(rel: Path) -> bool: # the kaggle CLI # --------------------------------------------------------------------------- def _executable() -> str: - found = shutil.which("kaggle") + """The `kaggle` beside *this* interpreter, then the one on PATH. + + The order matters more here than for a missing tool. `shutil.which` alone + found the CLI in the user-site Python rather than in the venv, so this + project's pinned `kaggle` was installed and a *different* installation's was + what actually ran -- silently, against an API whose contract this module + encodes. See `core/spawn.py:console_script`. + """ + from core import spawn # noqa: PLC0415 + + found = spawn.console_script("kaggle") if found: return found raise ConfigError( - "the `kaggle` CLI is not on PATH, so Kaggle kernels cannot be reached", + "the `kaggle` CLI is not installed in this environment or on PATH, so Kaggle " + "kernels cannot be reached", fix="pip install -e '.[kaggle]'", ) @@ -290,13 +300,45 @@ def _run(argv: list[str], cfg: Config, *, timeout: float) -> str: # --------------------------------------------------------------------------- # staging: one uploadable file # --------------------------------------------------------------------------- -def _payload_b64(sub: Submission) -> tuple[str, list[str]]: +def _add_bytes(tar: Any, rel: str, data: bytes) -> None: + """One in-memory file into the tar, with the same determinism as the rest. + + The name is checked here rather than at the caller for the reason + `tools/gpu.py:_write_remote` checks its own: this is content going onto a + machine we do not own, and a relative path that climbs is worth refusing + where it is written rather than trusting every future caller. + """ + if rel.startswith("/") or ".." in Path(rel).parts: + raise UsageError( + f"refusing to pack {rel!r}: a payload entry is a path inside the pipeline", + fix="pass a name relative to the spec directory, with no '..' in it", + ) + info = tarfile.TarInfo(rel) + info.size = len(data) + info.mtime = 0 + info.mode = 0o644 + info.uid = info.gid = 0 + info.uname = info.gname = "" + tar.addfile(info, io.BytesIO(data)) + + +def _payload_b64( + sub: Submission, *, overrides: dict[str, str] | None = None +) -> tuple[str, list[str]]: """The spec directory, packed into one base64 blob. The whole directory rather than only the import graph, so this backend stages what `gpu.py`'s `scp -r` stages. A pipeline that reads a CSV sitting beside its entrypoint works on an SSH host and would fail here on a narrower rule, and "it ran on the other backend" is the least useful bug report there is. + + `overrides` replaces or adds files by relative path on the way into the tar, + without touching the directory on disk. That is what an evolve candidate is: + the preflighted pipeline with one program swapped for a mutated one. Doing it + here rather than by copying the tree to a temp directory keeps one packing + implementation -- the secret scan, the size refusals and the deterministic + mtimes all apply to a candidate exactly as they do to a submission, which + they would not if candidates got a packer of their own. """ base = sub.spec_path.parent eligible: list[Path] = [] @@ -359,9 +401,17 @@ def _payload_b64(sub: Submission) -> tuple[str, list[str]]: # `gzip` rather than `tar` alone, and deterministically: mtime=0 keeps the # blob byte-identical between two pushes of an unchanged directory, which is # what makes a re-push diffable. + replacements = {str(k): str(v) for k, v in (overrides or {}).items()} with tarfile.open(fileobj=buffer, mode="w:gz", compresslevel=9) as tar: for path in eligible: rel = path.relative_to(base).as_posix() + if rel in replacements: + # Replaced, not appended alongside. Two entries with one name in + # a tar is a file whose contents depend on extraction order, + # which for a candidate means a score that depends on tar. + _add_bytes(tar, rel, replacements.pop(rel).encode("utf-8")) + packed.append(rel) + continue info = tar.gettarinfo(str(path), arcname=rel) info.mtime = 0 info.uid = info.gid = 0 @@ -369,6 +419,11 @@ def _payload_b64(sub: Submission) -> tuple[str, list[str]]: with open(path, "rb") as fh: tar.addfile(info, fh) packed.append(rel) + # Whatever the overrides added rather than replaced. Sorted so the blob + # stays a function of its inputs, like the sorted walk above. + for rel in sorted(replacements): + _add_bytes(tar, rel, replacements[rel].encode("utf-8")) + packed.append(rel) blob = base64.b64encode(buffer.getvalue()).decode("ascii") if len(blob) > MAX_PAYLOAD_B64: raise UsageError( @@ -511,8 +566,9 @@ def _metadata(cfg: Config, sub: Submission, *, ref: str, slug: str, accelerator: def _stage(cfg: Config, sub: Submission, workdir: Path, *, ref: str, slug: str, - accelerator: str, command: list[str]) -> list[str]: - payload, packed = _payload_b64(sub) + accelerator: str, command: list[str], + overrides: dict[str, str] | None = None) -> list[str]: + payload, packed = _payload_b64(sub, overrides=overrides) notebook = _notebook_for(sub, command, payload=payload) (workdir / f"{slug}.ipynb").write_text( json.dumps(notebook, ensure_ascii=False), encoding="utf-8" @@ -974,6 +1030,133 @@ def run_smoke( } +# --------------------------------------------------------------------------- +# evolve candidates +# --------------------------------------------------------------------------- +#: How many bytes of a candidate's output are kept, matching `tools/gpu.py`. +CANDIDATE_OUTPUT_BYTES = 8000 + + +def evaluate_candidate( + sub: Submission, + cfg: Config, + *, + candidate_id: str, + files: dict[str, str], + command: list[str], + timeout_s: int, + artifacts: Path, + accelerator: str | None = None, +) -> dict[str, Any]: + """Run one evolve candidate as a Kaggle kernel, and read its metrics back. + + A candidate here is a changed architecture or a changed optimiser, so its + evaluation is a training run and one kernel per candidate is the right unit + -- the minute or two of queue and start-up is noise against it. This is the + same push/poll/fetch the real submitter does, with two differences that both + follow from a candidate not being a run. + + **No ledger row.** `tools/evolve.py` records candidates in + `candidates.jsonl` so a long campaign cannot dominate a ledger read by hand + (§23 item 4), and that holds when the search goes remote. The campaign is the + ledgered unit and its expectation is the bound prediction. + + **The hours are still counted.** `core/kaggle_quota.py` folds candidate rows + beside runs, because the weekly accelerator allowance is a hard external + limit and a campaign that spent it invisibly would surface as an ordinary + submission being refused for hours nothing could account for. The caller + checks the projection before generation 0; this records what was actually + used so the fold has something to read. + + The mutated program reaches the kernel the same way the pipeline does -- + inside the notebook's base64 payload, via `_payload_b64`'s `overrides`. There + is no second delivery path, so the secret scan and the size refusals apply to + a candidate exactly as they do to a submission. + """ + accelerator = resolve_accelerator(accelerator, sub, cfg) + kind = cfg.accelerator_kind(accelerator) + slug = _slug(cfg, candidate_id) + ref = f"{_username(cfg)}/{slug}" + artifacts.mkdir(parents=True, exist_ok=True) + started = time.time() + + def _failed(message: str, **extra: Any) -> dict[str, Any]: + return { + "ok": False, + "exit_code": None, + "output": "", + "error": message, + "hours": round((time.time() - started) / 3600.0, 4), + "cost_usd": 0.0, + "accelerator": accelerator, + "accelerator_kind": kind, + "where": _url(ref), + **extra, + } + + try: + with tempfile.TemporaryDirectory(prefix="grad-cand-") as tmp: + workdir = Path(tmp) + _stage( + cfg, sub, workdir, + ref=ref, slug=slug, accelerator=accelerator, command=command, + overrides=files, + ) + _push( + cfg, workdir, + accelerator=accelerator, + timeout_s=float(cfg.get("kaggle", "push_timeout_s", 600)), + # Bounded where it runs, not only where it is watched. Without + # this the only limit is how long we choose to poll, which stops + # us waiting and does not stop the kernel -- and an abandoned + # kernel keeps spending the weekly allowance. + kernel_timeout_s=int(timeout_s), + ) + except GradError as exc: + return _failed(exc.message) + + state = _wait(cfg, ref, deadline=time.time() + int(timeout_s) + _queue_grace(cfg)) + if state.get("status") not in _TERMINAL: + # The kernel's own timeout should have ended it. Reaching here means + # Kaggle is queueing or not answering, so it is left alone rather than + # guessed at -- `kernels status` is the only thing that knows. + return _failed( + f"the candidate was still {state.get('status') or 'unknown'} after " + f"{int(timeout_s)}s plus the queue grace", + kernel_status=state.get("status"), + ) + + marker, hours, log_text = _fetch_output(cfg, ref, artifacts) + exit_code = marker.get("exit_code") + if exit_code is None: + # No marker means the kernel died before cell 3, which is a real + # outcome and not an exit code. Reported as one would be a number + # nobody measured. + return _failed( + f"the kernel finished as {state.get('status')} without recording an outcome", + kernel_status=state.get("status"), + hours=round(hours or (time.time() - started) / 3600.0, 4), + output=log_text[-CANDIDATE_OUTPUT_BYTES:], + ) + + exit_code = int(exit_code) + return { + "ok": exit_code == 0, + "exit_code": exit_code, + "output": log_text[-CANDIDATE_OUTPUT_BYTES:], + "error": None if exit_code == 0 else f"the candidate exited {exit_code} on Kaggle", + "hours": round(hours or 0.0, 4), + # Kaggle rations hours, not dollars -- see `core/kaggle_quota.py`. The + # zero is a fact about the backend rather than a missing measurement, and + # `hours` is the number that bounds a campaign here. + "cost_usd": 0.0, + "accelerator": accelerator, + "accelerator_kind": kind, + "kernel_status": state.get("status"), + "where": _url(ref), + } + + def _queue_grace(cfg: Config) -> float: """How long a smoke run may sit in Kaggle's queue before the poll gives up. diff --git a/tools/setup.py b/tools/setup.py new file mode 100644 index 0000000..ff9dd59 --- /dev/null +++ b/tools/setup.py @@ -0,0 +1,337 @@ +"""grad-setup -- the answers a wizard is allowed to write. + +Three questions, and none of them belong in `config/grad.toml`: which model runs +which role, which backend to reach for by default, and which SSH hosts exist. +That file is hand-annotated and `tomllib` cannot write it, so a command that +edited it would reformat it and drop every comment in it. The answers go to +`core/settings.py` instead -- an overlay under the app directory, keyed by +workspace, which outranks the file and *says so*. + +`show` is the important command here. A layered resolution that cannot be +inspected is a layered resolution that will eventually be argued with, and the +argument is always the same one: someone edits `[models] evolve`, sees no +change, and has no way to find out that a file they have never heard of wins. + +Every command here is what the setup window's buttons run (§10), which is what +keeps the window from growing a second way to do any of it. +""" + +from __future__ import annotations + +import argparse +from typing import Any + +from core import config as config_mod, credentials, settings +from core.cli import Cli, main +from core.errors import UsageError + +cli = Cli( + "grad-setup", + "Models, backends and SSH hosts: the writable half of the configuration.", + epilog=( + "These are stored under the app directory, per workspace, and they win over\n" + "config/grad.toml -- which is hand-annotated and cannot be machine-written\n" + "without losing every comment in it. `show` reports what is overriding what.\n\n" + "Credentials are not here: they live in the OS credential store\n" + "(`python -m tools.jobs credential set `). Ceilings are not here either:\n" + "a ceiling that moved is an event, and it belongs in `python -m tools.budget raise`." + ), +) + + +# --------------------------------------------------------------------------- +# show +# --------------------------------------------------------------------------- +@cli.command("show", "every layer, and which one wins") +def cmd_show(_: argparse.Namespace) -> dict[str, Any]: + cfg = config_mod.load(reload=True) + overlay_models = settings.models() + project_models = (cfg.project_overlay.get("models") or {}) + roles = [] + for role in config_mod.MODEL_ROLES: + configured = (cfg.user.get("models") or {}).get(role) + legacy = config_mod.LEGACY_MODEL_KEYS.get(role) + from_legacy = (cfg.user.get(legacy[0]) or {}).get(legacy[1]) if legacy else None + roles.append( + { + "role": role, + "model": cfg.model_for(role), + # The layer that actually selected it, in `model_for`'s order. + # This listed three of the five and got the answer wrong for the + # other two -- a role set by the selected project reported as + # "config", and one coming from a legacy `[agent] model` key + # reported as "default". The whole point of `show` is that the + # resolution can be inspected, so a source that is nearly right + # is worse here than in most places. + "source": ( + "project" + if role in project_models + else "setup" + if role in overlay_models + else "config" + if configured + else "legacy" + if from_legacy + else "default" + ), + "project": project_models.get(role), + "overlay": overlay_models.get(role), + "config": configured, + "legacy": from_legacy, + "default": config_mod.DEFAULTS["models"][role], + } + ) + + inventory = [] + configured_hosts = cfg.raw.get("hosts") or {} + added = settings.hosts() + for name, host in sorted(cfg.hosts.items()): + inventory.append( + { + "name": name, + "hostname": host.hostname, + "user": host.user, + "rate_usd_per_hour": host.rate_usd_per_hour, + "gpus": host.gpus, + "source": "setup" if name in added else "config", + } + ) + + return { + "settings_path": str(settings.path()), + "config_path": str(config_mod.paths.config_path()), + "models": roles, + "backend": {"default": settings.default_backend(), "known": list(settings.BACKENDS)}, + "hosts": inventory, + # The report that makes winning acceptable. `kaggle account` says the + # same thing for the same reason. + "shadowing": settings.shadowing(cfg), + "config_hosts": sorted(configured_hosts), + } + + +# --------------------------------------------------------------------------- +# models +# --------------------------------------------------------------------------- +def _models_args(p: argparse.ArgumentParser) -> None: + # One flag per role, derived from the roles rather than written out, so a + # seventh role is reachable here the moment it exists. + for role in config_mod.MODEL_ROLES: + p.add_argument( + f"--{role}", + metavar="MODEL", + help=f"the model for the {role} role (default: {config_mod.DEFAULTS['models'][role]})", + ) + p.add_argument( + "--clear", + action="append", + default=[], + metavar="ROLE", + help="drop an override, so the role falls back to the config and then the default", + ) + + +@cli.command("models", "choose the model for one or more roles", setup=_models_args) +def cmd_models(args: argparse.Namespace) -> dict[str, Any]: + """Six roles, and the two the funnel does not name here. + + `[retrieval] rerank_model` and `embed_model` are deliberately not settable + through this command. They are a different provider on a different billing + rail -- Voyage costs credits, the roles below cost subscription quota -- and + `config/grad.toml` argues at length against folding the two together. A + wizard that offered all eight in one list would be making exactly the + substitution §16 exists to prevent. + """ + chosen = { + role: getattr(args, role) for role in config_mod.MODEL_ROLES if getattr(args, role, None) + } + if not chosen and not args.clear: + raise UsageError( + "nothing to set", + fix=f"--{config_mod.MODEL_ROLES[0]} claude-opus-5 # or --clear {config_mod.MODEL_ROLES[0]}", + ) + if chosen: + settings.set_models(chosen) + if args.clear: + settings.clear_models(list(args.clear)) + cfg = config_mod.load(reload=True) + return { + "set": chosen, + "cleared": list(args.clear), + "models": cfg.models(), + "shadowing": settings.shadowing(cfg), + } + + +# --------------------------------------------------------------------------- +# backend +# --------------------------------------------------------------------------- +def _backend_args(p: argparse.ArgumentParser) -> None: + p.add_argument( + "--default", + dest="default_backend", + choices=settings.BACKENDS, + help="which backend to reach for when nothing more specific applies", + ) + + +@cli.command("backend", "choose the default backend", setup=_backend_args) +def cmd_backend(args: argparse.Namespace) -> dict[str, Any]: + """A default, not a restriction. + + The three backends are not alternatives and the useful setup is a mixture: + Kaggle's free hours for a smoke run, HF Jobs for the one that matters. So + this records a preference and refuses nothing -- `--remote` still names a + backend per campaign, and a spec's `[target]` still wins over both. + """ + if args.default_backend: + settings.set_backend(args.default_backend) + return { + "default": settings.default_backend(), + "known": list(settings.BACKENDS), + "readiness": readiness(config_mod.load(reload=True)), + } + + +# --------------------------------------------------------------------------- +# hosts +# --------------------------------------------------------------------------- +def _host_args(p: argparse.ArgumentParser) -> None: + p.add_argument("action", choices=("add", "remove")) + p.add_argument("--name", required=True, help="what Grad calls this host") + p.add_argument("--hostname", help="what ssh connects to") + p.add_argument("--user", default="", help="the ssh user") + p.add_argument( + "--rate", + dest="rate_usd_per_hour", + default=0.0, + help="dollars per hour, for pricing wall clock at collect time. 0 for a free host", + ) + p.add_argument("--workdir", default="~/grad", help="where runs are staged on the host") + p.add_argument("--gpus", default=1, type=int) + p.add_argument( + "--key-credential", + dest="key_credential", + help="the keyring entry holding this host's key. never a path to a key file", + ) + p.add_argument("--notes", default="", help="anything worth remembering about this box") + + +@cli.command("host", "add or remove an SSH host in the inventory", setup=_host_args) +def cmd_host(args: argparse.Namespace) -> dict[str, Any]: + """The inventory stays fixed; this gives it a second, writable source. + + `core/config.py:host` refuses an unknown name because a host that can be + named ad-hoc is a general remote-execution capability the threat model does + not grant. Nothing about that changes here -- a host has to be added, on + purpose, before anything can reach it, and the refusal now names both places + it could have been added. + """ + if args.action == "remove": + settings.remove_host(args.name) + else: + settings.add_host( + args.name, + { + "hostname": args.hostname, + "user": args.user, + "rate_usd_per_hour": args.rate_usd_per_hour, + "workdir": args.workdir, + "gpus": args.gpus, + "key_credential": args.key_credential, + "notes": args.notes, + }, + ) + cfg = config_mod.load(reload=True) + return { + "action": args.action, + "name": args.name, + "hosts": sorted(cfg.hosts), + "added_here": sorted(settings.hosts()), + } + + +# --------------------------------------------------------------------------- +# check +# --------------------------------------------------------------------------- +#: What each backend needs before it can be submitted to, and what to run when it +#: is missing. `hf_token` is *required for HF Jobs* -- which is a different claim +#: from "required", and the credentials panel used to make the stronger one at a +#: user who had chosen Kaggle. +REQUIREMENTS: dict[str, dict[str, Any]] = { + "ssh": { + "credentials": (), + "needs_host": True, + "fix": "python -m tools.setup host add --name gpu-box --hostname … --user … --json", + }, + "hf_jobs": { + "credentials": (credentials.HF_TOKEN,), + "needs_host": False, + "fix": "python -m tools.jobs credential set hf_token", + }, + "kaggle": { + "credentials": (credentials.KAGGLE_KEY,), + "needs_host": False, + "needs_kaggle_account": True, + "fix": "python -m tools.kaggle account --set --json", + }, +} + + +def readiness(cfg: config_mod.Config) -> list[dict[str, Any]]: + """Which backends could actually take a submission right now. + + Reads state; runs nothing. `kaggle account --check` makes a real + authenticated call and this deliberately does not -- a readiness report that + takes several seconds and touches the network is one nothing will call on a + window's refresh. + """ + from tools import kaggle as kaggle_tool + + stored = credentials.status() + out = [] + for backend, needs in REQUIREMENTS.items(): + missing = [name for name in needs["credentials"] if not stored.get(name)] + if needs.get("needs_host") and not cfg.hosts: + missing.append("an ssh host") + if needs.get("needs_kaggle_account"): + username, _ = kaggle_tool.resolve_username(cfg) + if not username: + missing.append("a kaggle username") + out.append( + { + "backend": backend, + "ready": not missing, + "missing": missing, + "fix": None if not missing else needs["fix"], + } + ) + return out + + +@cli.command("check", "what is still missing, per backend") +def cmd_check(_: argparse.Namespace) -> dict[str, Any]: + cfg = config_mod.load(reload=True) + stored = credentials.status() + backend_readiness = readiness(cfg) + return { + # The subscription token is not a backend's business: it is what runs the + # agent at all, so it is reported on its own rather than folded into a + # row about GPUs. + "agent": { + "ready": bool(stored.get(credentials.CLAUDE_TOKEN)), + "fix": ( + None + if stored.get(credentials.CLAUDE_TOKEN) + else "claude setup-token, then: python -m tools.jobs credential set claude_oauth_token" + ), + }, + "backends": backend_readiness, + "any_backend_ready": any(r["ready"] for r in backend_readiness), + "default_backend": settings.default_backend(), + "models": cfg.models(), + } + + +if __name__ == "__main__": + main(cli) diff --git a/tools/wakeup.py b/tools/wakeup.py new file mode 100644 index 0000000..dbaf4db --- /dev/null +++ b/tools/wakeup.py @@ -0,0 +1,378 @@ +"""grad-wakeup -- arm a condition, end the turn, and be woken when it happens. + + "a two-hour poll inside the agent's only shell is a tool timeout waiting to + happen" + +`tools/task.py` gave the agent a way to *start* something without waiting for it. +This is the way to *stop* waiting for it. Arm a wake, end the turn, and a +detached watcher does the looking -- out of process, at a backing-off interval, +with no shell held and no tokens spent. When the condition happens it wakes the +session with a turn saying so. + +What it replaces is a pattern, not a command: `sleep 30`, look, `sleep 60`, look, +`sleep 120`, look. Every one of those looks is a turn and every turn is tokens, +so a four-hour training run produced a conversation that was mostly the agent +waiting -- and the sleeps had to keep growing or the watching cost more than the +job. None of that is necessary. The machine can tell us. + +**The conditions are a closed list, and that is the point.** There is no +`--command`. `hooks.py` gates every `Bash` the agent runs and `tools/task.py` +re-uses `evaluate_bash` so that backgrounding a command cannot become the way +around it; a wakeup that ran an arbitrary command on a timer would be the same +bypass with a delay on it. So a wake waits on things this system already knows +how to read: a background task's record, a run's status on its backend, a path, +a clock. `core/wakeups.py` has the rest of the argument. + +**The turn a wake issues is metered like any other.** It goes through +`agent.drive_turn`, inside the token allocation, into `ledger/quota.jsonl`. A +wake is a prompt with no human typing it, not an exception to the ceiling. +""" + +from __future__ import annotations + +import argparse +import time +from pathlib import Path +from typing import Any + +from core import paths, wakeups as wk +from core.cli import Cli, main +from core.errors import EXIT_RUNNING, GradError, NotFound, UsageError + +cli = Cli( + "grad-wakeup", + "Wait for something in the background and start a new turn when it happens.", + epilog=( + "The shape is always the same: arm, end your turn, get woken.\n\n" + " python -m tools.wakeup arm --run run-2026-08-17-a --timeout 14400 \\\n" + " --note 'the 4090 sweep; collect it and judge against exp-7' --json\n\n" + "Then stop. Do not poll it, do not sleep on it, and do not end the turn with a\n" + "`wait` unless a human is watching -- `wait` holds the shell, which is the thing\n" + "this tool exists to stop doing.\n\n" + "A wake fired into a workspace whose app is closed is kept, not lost: `list`\n" + "shows it as undelivered and `status` prints the turn it would have sent." + ), +) + +_WATCH = "_watch" + + +# --------------------------------------------------------------------------- +# arm +# --------------------------------------------------------------------------- +def _arm_args(p: argparse.ArgumentParser) -> None: + what = p.add_mutually_exclusive_group(required=True) + what.add_argument("--after", type=float, metavar="SECONDS", help="wake after a delay") + what.add_argument("--task", metavar="TASK_ID", help="wake when a background task finishes") + what.add_argument("--run", metavar="RUN_ID", help="wake when a run stops running on its backend") + what.add_argument("--file", metavar="PATH", help="wake when a path appears") + p.add_argument( + "--changed", + action="store_true", + help="with --file: wake when it changes, not merely when it exists", + ) + p.add_argument( + "--timeout", + type=float, + default=wk.DEFAULT_TIMEOUT_S, + help=( + f"seconds to wait before giving up and waking anyway " + f"(default {int(wk.DEFAULT_TIMEOUT_S)}, ceiling {int(wk.MAX_TIMEOUT_S)}). " + "Set it to what you actually expect plus a margin: an expired wake is a " + "fact worth learning, and the state it reports says so." + ), + ) + p.add_argument( + "--note", + default="", + help="what you are waiting for and why; it comes back to you in the waking turn", + ) + p.add_argument( + "--no-resume", + action="store_true", + help="record the wake but do not start a turn with it", + ) + + +@cli.command("arm", "wait for a condition in the background", setup=_arm_args) +def cmd_arm(args: argparse.Namespace) -> dict[str, Any]: + paths.ensure_workspace() + timeout = float(args.timeout) + if timeout <= 0: + raise UsageError( + "a timeout of zero would expire before the first look", + fix="--timeout 3600", + ) + if timeout > wk.MAX_TIMEOUT_S: + raise UsageError( + f"a wake may be armed for at most {int(wk.MAX_TIMEOUT_S)}s " + f"({wk.MAX_TIMEOUT_S // 3600} hours), and this asked for {int(timeout)}s", + fix=( + f"--timeout {int(wk.MAX_TIMEOUT_S)} # and re-arm if it is genuinely " + "still running then" + ), + ) + + condition = _condition(args) + deadline = time.time() + timeout + wake_id = wk.new_id() + + # The watcher is started first and the record written second, which is the + # opposite of `tools/task.py` and deliberate: the watcher's first act is to + # read its own record, so it is written before the process can look for it. + # See the retry below -- the alternative was a race between two processes + # over a file that exists to describe one of them. + pid = wk.spawn_watcher(wake_id) + wk.record_armed( + wake_id, + condition=condition, + deadline=deadline, + note=str(args.note or ""), + pid=pid, + resume=not args.no_resume, + ) + + return { + "wake": wake_id, + "waiting_for": wk.describe(condition), + "timeout_s": int(timeout), + "expires_at": wk.iso_at(deadline), + "pid": pid, + "resume": not args.no_resume, + "note": str(args.note or ""), + "next": ( + "end your turn. You will be woken with a new turn when this happens." + if not args.no_resume + else f"python -m tools.wakeup status {wake_id} --json" + ), + } + + +def _condition(args: argparse.Namespace) -> dict[str, Any]: + if args.after is not None: + seconds = float(args.after) + if seconds < 0: + raise UsageError("--after cannot be negative", fix="--after 600") + return {"kind": wk.KIND_AFTER, "seconds": seconds, "fire_at": time.time() + seconds} + + if args.task: + from core import tasks as tasklib # noqa: PLC0415 + + task = tasklib.get(str(args.task)) + if task is None: + raise NotFound( + f"no background task {args.task} in this workspace", + fix="python -m tools.task list --json", + ) + if task.get("state") in tasklib.TERMINAL: + # Refused rather than armed. A wake on something already finished + # would fire on its first look and spend a turn telling the agent + # what a `task status` in this same turn would have told it for free. + raise GradError( + "already_finished", + f"task {args.task} has already finished ({task.get('state')})", + exit_code=EXIT_RUNNING, + fix=f"python -m tools.task status {args.task} --json", + ) + return {"kind": wk.KIND_TASK, "task": str(args.task)} + + if args.run: + from core import ledger_store as ls # noqa: PLC0415 + + try: + record = ls.run(str(args.run)) + except GradError: + raise + except Exception as exc: # noqa: BLE001 + raise NotFound( + f"could not read run {args.run}: {type(exc).__name__}", + fix="python -m tools.ledger list --json", + ) from exc + if record.collected: + raise GradError( + "already_collected", + f"run {args.run} was collected at {record.get('collected_at')}", + exit_code=EXIT_RUNNING, + fix=f"python -m tools.ledger show {args.run} --json", + ) + return {"kind": wk.KIND_RUN, "run": str(args.run)} + + path = Path(str(args.file)) + if not path.is_absolute(): + path = paths.root() / path + condition: dict[str, Any] = { + "kind": wk.KIND_FILE, + "path": str(path), + "changed": bool(args.changed), + } + if args.changed: + # The baseline is taken here, at arm time, so "changed" means "changed + # since you asked" rather than "changed since the watcher got round to + # looking" -- which would silently miss a write in between. + try: + condition["mtime_ns"] = path.stat().st_mtime_ns + except OSError: + condition["mtime_ns"] = None + elif path.exists(): + raise GradError( + "already_there", + f"{path} already exists, so this wake would fire immediately", + exit_code=EXIT_RUNNING, + fix=f"python -m tools.wakeup arm --file {path} --changed --json", + ) + return condition + + +# --------------------------------------------------------------------------- +# reading +# --------------------------------------------------------------------------- +def _list_args(p: argparse.ArgumentParser) -> None: + p.add_argument("--all", action="store_true", help="include wakes that have already resolved") + + +@cli.command("list", "what is being waited for, and what has fired", setup=_list_args) +def cmd_list(args: argparse.Namespace) -> dict[str, Any]: + everything = wk.wakeups() + rows = [ + _summarise(w) + for w in everything.values() + if args.all or w["state"] == wk.ARMED or (w["state"] in (wk.FIRED, wk.EXPIRED) and not w["delivered"]) + ] + rows.sort(key=lambda r: r.get("armed_at") or "") + undelivered = [r for r in rows if r["state"] in (wk.FIRED, wk.EXPIRED) and not r["delivered"]] + return { + "wakeups": rows, + "armed": sum(1 for r in rows if r["state"] == wk.ARMED), + "undelivered": len(undelivered), + "note": ( + f"{len(undelivered)} wake(s) fired while nothing was listening; " + "`status ` prints what they would have said." + if undelivered + else "" + ), + } + + +@cli.command( + "status", + "one wake in full, including the turn it sent or would send", + setup=lambda p: p.add_argument("wake_id"), +) +def cmd_status(args: argparse.Namespace) -> dict[str, Any]: + wake = _require(args.wake_id) + out = _summarise(wake) + if wake["state"] in (wk.FIRED, wk.EXPIRED): + out["turn"] = wk.prompt_for(wake, wake.get("detail") or {}, expired=wake["state"] == wk.EXPIRED) + return out + + +@cli.command( + "cancel", + "stop waiting for one; it will not wake you", + setup=lambda p: (p.add_argument("wake_id"), p.add_argument("--reason", default="")), +) +def cmd_cancel(args: argparse.Namespace) -> dict[str, Any]: + wake = _require(args.wake_id) + if wake["state"] != wk.ARMED: + return {"wake": wake["id"], "state": wake["state"], "note": "it was not waiting for anything"} + wk.record_cancelled(wake["id"], reason=str(args.reason or "")) + # The watcher notices on its next look and exits. Not killed: it is sleeping + # on a bounded interval and reading its own record is the same check it makes + # every time round, so there is nothing to reach for a signal about. + return { + "wake": wake["id"], + "state": wk.CANCELLED, + "note": f"the watcher stops within {int(wk.POLL_MAX_S)}s", + } + + +def _wait_args(p: argparse.ArgumentParser) -> None: + p.add_argument("wake_id") + p.add_argument("--timeout", type=float, default=900.0, help="seconds to block for") + + +@cli.command( + "wait", + "block until one fires -- for a person at a terminal, not for the agent", + setup=_wait_args, +) +def cmd_wait(args: argparse.Namespace) -> dict[str, Any]: + """Deliberately the least useful command here. + + It exists because a person driving this from a terminal reasonably wants to + block on a wake, and because a test needs a synchronous way to observe one. + The agent should never reach for it: holding the shell is the thing the rest + of this module exists to stop, and `arm` says so in its own `next`. + """ + deadline = time.time() + max(0.0, float(args.timeout)) + while True: + wake = _require(args.wake_id) + if wake["state"] != wk.ARMED: + return _summarise(wake) + if time.time() >= deadline: + raise GradError( + "still_waiting", + f"{args.wake_id} is still armed after {int(args.timeout)}s", + exit_code=EXIT_RUNNING, + fix=f"python -m tools.wakeup status {args.wake_id} --json", + ) + time.sleep(1.0) + + +@cli.command("clear", "forget wakes that have already resolved") +def cmd_clear(_: argparse.Namespace) -> dict[str, Any]: + stale = [w["id"] for w in wk.wakeups().values() if w["state"] in wk.TERMINAL] + return {"forgotten": wk.forget(stale)} + + +def _require(wake_id: str) -> dict[str, Any]: + wake = wk.get(wake_id) + if wake is None: + raise NotFound( + f"no wake {wake_id} in this workspace", + fix="python -m tools.wakeup list --all --json", + ) + return wake + + +def _summarise(wake: dict[str, Any]) -> dict[str, Any]: + return { + "id": wake["id"], + "state": wake["state"], + "waiting_for": wk.describe(wake.get("condition") or {}), + "note": wake.get("note") or "", + "armed_at": wake.get("armed_at"), + "finished_at": wake.get("finished_at"), + "delivered": wake.get("delivered"), + "detail": wake.get("detail"), + } + + +# --------------------------------------------------------------------------- +# the watcher +# --------------------------------------------------------------------------- +def _watch_args(p: argparse.ArgumentParser) -> None: + p.add_argument("--wake-id", required=True) + + +@cli.command(_WATCH, None, setup=_watch_args) +def cmd_watch(args: argparse.Namespace) -> dict[str, Any]: + """The detached process. Hidden: spawned by name, never typed. + + It waits for its own record to appear before it starts looking. `arm` spawns + this and writes the record immediately afterwards, and on a cold filesystem + the process can be running before the append lands -- a watcher that gave up + there would leave a wake armed forever with nothing watching it. + """ + wake_id = str(args.wake_id) + for _ in range(100): + if wk.get(wake_id) is not None: + break + time.sleep(0.1) + else: + return {"wake": wake_id, "state": "unclaimed"} + return wk.watch(wake_id) + + +if __name__ == "__main__": + main(cli) diff --git a/tools/wiki.py b/tools/wiki.py index 92b1d91..c21989a 100644 --- a/tools/wiki.py +++ b/tools/wiki.py @@ -43,7 +43,6 @@ import argparse import hashlib import json -import shutil import subprocess import time from pathlib import Path @@ -114,12 +113,21 @@ def source_hash(root: Path | None = None) -> dict[str, Any]: def _repowiki() -> str: - found = shutil.which("repowiki") + """The `repowiki` beside *this* interpreter, then the one on PATH. + + `shutil.which` alone reported "not installed" for a repowiki sitting in the + venv this process is running from, because a virtualenv's `Scripts` + directory is on PATH only while the environment is activated and the desktop + shortcut does not activate anything. See `core/spawn.py:console_script`. + """ + from core import spawn # noqa: PLC0415 + + found = spawn.console_script("repowiki") if found: return found raise ConfigError( "repowiki is not installed", - fix="pip install -e '.[wiki]' # pins repowiki==0.3.1", + fix="pip install -e '.[wiki]' # pins repowiki==0.3.1; it is an optional extra", ) diff --git a/ui/app.py b/ui/app.py index fdb21a9..54845b4 100644 --- a/ui/app.py +++ b/ui/app.py @@ -145,6 +145,12 @@ def __init__(self, key: str = "default") -> None: #: change means a new client. Recorded here so `apply_effort` can tell a #: real change from a click that landed on the level already running. self.client_effort: str | None = None + #: The `research` model the live client was *built* with, for the same + #: reason and with the same consequence. A project may override it + #: (`core/budget.py:configure`), so switching to one that does while a + #: session is live would otherwise leave the previous model answering -- + #: silently, and while the projects window shows the new one. + self.client_model: str | None = None #: Held across `start` and `close`, so one can never run inside the #: other. The SDK's `connect` is not safe against a concurrent #: `disconnect`: disconnect nulls the client's transport while connect @@ -176,6 +182,7 @@ async def start(self) -> None: # does not leave this claiming a level nothing is running at -- which # would have `apply_effort` decide there was nothing to rebuild. self.client_effort = effort.current(cfg) + self.client_model = cfg.model_for("research") # -- named sessions ----------------------------------------------------- def adopt(self) -> None: @@ -301,6 +308,7 @@ async def _close_locked(self) -> None: # and reopened at a different level would compare the new selection # against the old client's and decide no rebuild was needed. self.client_effort = None + self.client_model = None # The reading belongs to the client that answered it. Keeping it across a # close would leave the meter reporting the context of a conversation # that no longer exists -- and every path that drops a client (a session @@ -360,6 +368,11 @@ async def ask(self, prompt: str, on_settle: Any) -> None: # client and doing that under a turn is what `_stop_turn` exists to # clean up after. await self.apply_effort() + # And the model, which a project switch can have changed underneath + # this session since the last turn. Both are lazy for the same + # reason: a rebuild is seconds, and doing it at the click would make + # idly switching between two projects cost more than using either. + await self.apply_model() await self.start() except Exception: self.busy = False @@ -628,6 +641,36 @@ async def apply_effort(self) -> bool: await self.start() return True + async def apply_model(self) -> bool: + """Rebuild the client if the `research` model is not the one it is + running. `apply_effort`'s sibling, and every line of its reasoning + applies unchanged. + + What makes it necessary is Stage 5: a project may override `research` + (`core/budget.py:configure`), and switching project is a thing that + happens *while a session is live*. Without this the previous model goes + on answering while the projects window and the ledger both say + otherwise -- which is the worst shape for it, because every surface + agrees on a claim that is false. + + Deliberately not folded into `apply_effort`. The two are checked + together at the same call site, but a rebuild that could have been + caused by either is a rebuild whose reason cannot be logged, and the log + line is what makes a mysterious reconnect explicable. + """ + if self.client is None: + return False + chosen = config_mod.load().model_for("research") + if chosen == self.client_model: + return False + if self.sdk_session_id is None: + log.info("model change deferred: no sdk session id to resume yet") + return False + log.info("rebuilding the client: %s -> %s", self.client_model, chosen) + await self.close() + await self.start() + return True + async def maybe_compact(self) -> dict[str, Any] | None: """Compact if the context has passed the configured threshold. @@ -842,6 +885,56 @@ def _show() -> dict[str, bool]: """ return {"shown": desktop.show_window()} + @nicegui_app.post("/__grad/wake") + async def _wake(payload: dict) -> Any: + """A watcher reporting that something the agent armed has happened. + + **This is the one endpoint on this port that is authenticated, and the + asymmetry is the point.** `/__grad/show` is unauthenticated because its + entire effect is that a window the user already owns becomes visible. + This one *starts a turn for an agent with Bash access*, so anything that + can open a loopback socket -- which is every process on the machine -- + would otherwise be able to drive it. The token is a mode-600 file in the + app directory; see `core/wakeups.py:token`. + + `compare_digest` rather than `==`, because a token compared with early + exit is a token that can be guessed a byte at a time by something already + able to time it. + + Queued rather than run here. A route has no client in scope and the turn + has to be drawn into one; `ui/state.py:accept_wake` explains the seam. + + **The body is taken as a plain `dict`, not as an injected `Request`.** + This module runs under `from __future__ import annotations`, so every + annotation reaches FastAPI as a *string* which it resolves against the + module's globals -- and `Request` imported inside this function is not + one. The result is not an error: FastAPI decides the unresolvable + parameter must be a query parameter, and every POST is rejected with a + 422 asking for a missing query field called `request`. `dict` is a + builtin, so it resolves, and FastAPI reads it from the body. + """ + from core import wakeups as wk # noqa: PLC0415 + + from fastapi.responses import JSONResponse # noqa: PLC0415 + + body = payload if isinstance(payload, dict) else {} + offered = str(body.get("token") or "") + if not offered or not secrets.compare_digest(offered, wk.token()): + log.warning("refused a wake with a bad token") + return JSONResponse({"error": "bad token"}, status_code=403) + + prompt = str(body.get("prompt") or "").strip() + if not prompt: + return JSONResponse({"error": "nothing to say"}, status_code=400) + + for workspace in list(_WAKE_TARGETS): + if workspace.accept_wake(prompt): + return {"queued": True, "wake": body.get("wake")} + # No window open, or every one of them is already holding a queue. The + # watcher records this as undelivered and `wakeup list` surfaces it, so + # the wake is deferred rather than lost. + return JSONResponse({"queued": False}, status_code=503) + @nicegui_app.get("/__grad/notebook/{name}") def _notebook(name: str) -> Any: """One notebook, rendered read-only, for the pane's iframe. @@ -887,6 +980,10 @@ def index() -> None: # interrupt the SDK refused, a client that had to be taken down -- which # have no turn to be written into. session.notify = workspace.say + # Where a wake can land. A list rather than one slot because there can + # be several windows open on one workspace, and the wake belongs to + # whichever of them can take it -- see `/__grad/wake`. + _WAKE_TARGETS.append(workspace) # Per client, not `app.on_shutdown`: that would accumulate one handler # per connection and hold every session's subprocess open until the app # itself exits. Graced, not immediate: NiceGUI fires this on any socket @@ -894,7 +991,9 @@ def index() -> None: # miss -- and the busiest moment this loop has is spawning the CLI for # the session's own first turn. Releasing right then closed the client # mid-connect; see `Session._lifecycle` for what that corrupted. - context.client.on_disconnect(lambda: _release_when_gone(context.client, session)) + context.client.on_disconnect( + lambda: _release_when_gone(context.client, session, workspace) + ) shell.build(workspace) @@ -909,8 +1008,14 @@ def index() -> None: #: is not the only one. Entries remove themselves when they finish. _RELEASES: set[Any] = set() +#: Workspaces a wakeup can be delivered to, oldest window first. Module-level +#: because the delivering end is an HTTP route with no client in scope -- see +#: `/__grad/wake` -- and per-process because that is what a wake reaches: one +#: app, holding one single-instance lock, on one published port. +_WAKE_TARGETS: list[Any] = [] + -def _release_when_gone(client: Any, session: Session) -> None: +def _release_when_gone(client: Any, session: Session, workspace: Any = None) -> None: """Hand the session back only if this client's socket stays gone. NiceGUI's disconnect handlers run on every socket drop, and a drop is not a @@ -924,6 +1029,12 @@ async def _check() -> None: await asyncio.sleep(RELEASE_GRACE_S) if getattr(client, "has_socket_connection", False): return + # Withdrawn on the same condition as the session, not on the disconnect: + # a window that reconnects inside the grace never stopped being somewhere + # a wake could land, and dropping it early would send a wake that had + # waited four hours to a 503. + if workspace is not None and workspace in _WAKE_TARGETS: + _WAKE_TARGETS.remove(workspace) await session.release() try: @@ -1065,6 +1176,31 @@ def _install_desktop(native: bool) -> None: icon = desktop.icon_path() if icon: nicegui_app.native.start_args["icon"] = icon + # Where the window was last time, and how big. Spliced into + # `create_window`'s arguments *after* NiceGUI's own `width`/`height`, so + # this is what decides the size and `window_size` below is only the + # fallback the first launch on a machine gets. `track_window` is what + # keeps the file current; it has to be registered before `ui.run`, + # because `ui.run` starts the bridge that delivers the events. + nicegui_app.native.window_args.update(desktop.window_args()) + desktop.track_window(nicegui_app) + + @nicegui_app.on_connect + def _drop_splash() -> None: + """The workspace is on screen, so the loading mark can go. + + On *connect*, not on startup: `on_startup` fires when the server is + listening, which is several seconds before the webview process has + rendered anything -- taking the splash down there would put the gap back + exactly where it was. A connected client is a page that has loaded and + opened its socket, which is the first moment there is something to look + at instead. + + Fires per client and `stop` is idempotent, so a reload costs a no-op. + """ + from ui import splash # noqa: PLC0415 + + splash.stop() @nicegui_app.on_startup def _wire() -> None: @@ -1159,7 +1295,12 @@ def run(*, native: bool = True, port: int | None = None) -> None: # nicety: NiceGUI turns `native` on whenever a window size is given, so # passing it unconditionally made `native=False` unreachable and the # documented browser fallback impossible to actually take. - extra = {"window_size": (1600, 1000)} if native else {} + # + # It is no longer what decides the size. `_install_desktop` puts the + # remembered geometry in `native.window_args`, which NiceGUI merges over + # these values -- so this is the size of a window nobody has moved yet, and + # it is spelled once, in `desktop.DEFAULT_SIZE`. + extra = {"window_size": desktop.DEFAULT_SIZE} if native else {} ui.run( native=native, title="Grad", diff --git a/ui/desktop.py b/ui/desktop.py index c8a9ff5..08003c5 100644 --- a/ui/desktop.py +++ b/ui/desktop.py @@ -423,6 +423,39 @@ def icon_path(*, refresh: bool = False) -> str | None: return None +def splash_png(size: int = 96) -> str | None: + """The mark as a PNG, for the loading window. Rendered once, then cached. + + A PNG and not the `.ico` beside it because the reader is Tk, which has read + PNG since 8.6 and has never read ICO -- and because the loading window is + the one consumer of this glyph that is not an operating-system icon slot. + + It is still `_icon_image`, which is the point: `write_icon`'s docstring says + there is one drawing of this mark and everything reads it, and a splash + screen with its own hand-drawn nabla is exactly the drift that rule exists to + prevent. The size is in the filename so changing it cannot silently serve a + stale render at the old one. + + Called from the *splash's own process*, never from the launch path. Importing + Pillow costs a couple of hundred milliseconds and the whole purpose of that + process is to be on screen before anything expensive happens here. + + Returns None rather than raising, like `icon_path`: a machine with no Pillow + still gets a loading window, just a wordmark instead of a glyph. + """ + try: + from core import appdata # noqa: PLC0415 + + target = appdata.app_dir() / f"grad-splash-{int(size)}.png" + if not target.is_file(): + target.parent.mkdir(parents=True, exist_ok=True) + _icon_image(int(size)).save(target, format="PNG") + return str(target) + except Exception: # noqa: BLE001 - see the docstring; cosmetic, never fatal + log.debug("could not render the splash mark", exc_info=True) + return None + + def _available_tag() -> str | None: """The release the last check found, or None. Cheap enough for a menu draw. @@ -616,6 +649,241 @@ def _closing() -> bool: window.events.closing += _closing +# --------------------------------------------------------------------------- +# where the window was +# --------------------------------------------------------------------------- +#: The size a machine that has never been told otherwise opens at. It was +#: written inline at the `ui.run` call for as long as this app has existed, +#: which is also exactly as long as the window has opened in the same place +#: every time no matter where it was left. +DEFAULT_SIZE = (1600, 1000) +#: Nothing smaller than this is restored. A window can be dragged down to a +#: sliver, and reopening at a sliver looks like an app that failed to start +#: rather than like the size someone chose. +MIN_SIZE = (640, 480) +#: How much of the window has to land on a real screen for its saved position +#: to be used. A corner is enough -- it is grabbable, which is the only thing +#: that matters -- and anything less is a window nobody can reach. +MIN_VISIBLE_PX = 80 +#: Seconds between writes while the window is being dragged. `moved` fires per +#: pixel of a drag; the file is 60 bytes and the disk should still not see all +#: of them. +SAVE_EVERY_S = 1.0 + +#: The geometry as last observed in a *normal* window state, and the one +#: previous to it. Two, not one: see `_note_maximized`. +_geometry: dict[str, Any] = {} +_previous: dict[str, Any] = {} +_saved_at = 0.0 + + +def geometry_path() -> Any: + """Where the window's own state lives. + + Beside `tray.flag` and `ui_storage_secret` in the app's state directory, not + under `workspaces/`: there is one window, and it does not become a different + window because the workspace root was pointed somewhere else. + """ + from core import appdata # noqa: PLC0415 - import cycle if hoisted + + return appdata.state_dir() / "window.json" + + +def read_geometry() -> dict[str, Any]: + """The saved geometry, or `{}`. Never raises and never returns nonsense. + + Every field is re-derived rather than trusted: this file survives upgrades, + can be edited by hand, and is read at the one moment where a bad value costs + the most -- deciding where to put a window before there is any UI to report + a problem with. + """ + from core import jsonl # noqa: PLC0415 + + try: + raw = jsonl.read_json(geometry_path()) + except Exception: # noqa: BLE001 - a missing or corrupt file is not an error + return {} + if not isinstance(raw, dict): + return {} + out: dict[str, Any] = {} + for key in ("x", "y", "width", "height"): + value = raw.get(key) + if isinstance(value, (int, float)) and not isinstance(value, bool): + out[key] = int(value) + if raw.get("maximized") is True: + out["maximized"] = True + return out + + +def _screens() -> list[tuple[int, int, int, int]]: + """Every attached screen as `(x, y, width, height)`, or `[]` if unknown. + + `webview.screens` answers without starting anything, which is what makes it + usable here -- this runs before `ui.run`, and the window process does not + exist yet. + """ + try: + import webview # noqa: PLC0415 + + return [ + (int(s.x), int(s.y), int(s.width), int(s.height)) + for s in webview.screens + if int(s.width) > 0 and int(s.height) > 0 + ] + except Exception: # noqa: BLE001 - no webview, no display, an odd backend + log.debug("could not enumerate screens", exc_info=True) + return [] + + +def on_screen(x: int, y: int, width: int, height: int) -> bool: + """Would a window at this rectangle be reachable? + + **This is the half of "remember the position" that is not optional.** A + saved position is a promise about a monitor arrangement, and the arrangement + is the part that changes: undock a laptop, unplug the second screen, and the + coordinates that were perfect yesterday put the window somewhere with no + pixels in it -- running, holding the port and the single-instance lock, and + invisible. That is a worse failure than opening in the wrong place, because + there is no way back from it that does not involve deleting a file. + + With no screen information at all this answers True. Refusing to restore + because we could not check would make the feature stop working on every + machine whose backend does not enumerate displays, in exchange for a + guarantee we have no evidence we need there. + """ + screens = _screens() + if not screens: + return True + for sx, sy, sw, sh in screens: + overlap_w = min(x + width, sx + sw) - max(x, sx) + overlap_h = min(y + height, sy + sh) - max(y, sy) + if overlap_w >= MIN_VISIBLE_PX and overlap_h >= MIN_VISIBLE_PX: + return True + return False + + +def window_args() -> dict[str, Any]: + """What `webview.create_window` should be given for the main window. + + Merged into `app.native.window_args`, which NiceGUI splices in *after* its + own `width`/`height` (see `native_mode._open_window`) -- so this overrides + `ui.run(window_size=...)` rather than fighting it, and the default lives + here rather than in two places. + + A size is always returned; a position only when there is a saved one that + lands on a screen that exists right now. + """ + saved = read_geometry() + width = max(MIN_SIZE[0], int(saved.get("width") or DEFAULT_SIZE[0])) + height = max(MIN_SIZE[1], int(saved.get("height") or DEFAULT_SIZE[1])) + args: dict[str, Any] = {"width": width, "height": height} + if saved.get("maximized"): + args["maximized"] = True + if "x" in saved and "y" in saved: + x, y = int(saved["x"]), int(saved["y"]) + if on_screen(x, y, width, height): + args["x"], args["y"] = x, y + else: + # Said out loud, once. A window that quietly ignores the position it + # was told to use looks identical to one that never saved it. + log.info( + "not restoring the window to %d,%d: no attached screen covers it", x, y + ) + return args + + +def _note_maximized(maximized: bool) -> None: + """Record the maximized flag without letting it eat the restored geometry. + + Maximizing a window also *moves and resizes* it, and pywebview reports those + as ordinary `moved`/`resized` events. Left alone, the last thing recorded + before `maximized` arrives is the maximized rectangle -- so un-maximizing on + the next launch would restore a window the size of the screen at its top + left corner, and the size the user actually chose would be gone. + + The order the two events arrive in is a backend detail, so this does not + depend on it: one observation of history is kept, and adopting it on the way + into the maximized state discards exactly the stray one. + """ + global _geometry + + if maximized: + if _previous: + _geometry = dict(_previous) + _geometry["maximized"] = True + else: + _geometry.pop("maximized", None) + + +def remember(**fields: int) -> None: + """Take one observation of the window's rectangle.""" + global _geometry, _previous + + if _geometry.get("maximized"): + # A move or resize *while* maximized is the window manager's business, + # not a choice to remember. The rectangle to restore to is the one from + # before it was maximized, which is already held. + return + _previous = dict(_geometry) + _geometry.update(fields) + save_geometry() + + +def save_geometry(*, force: bool = False) -> None: + """Write the geometry, at most every `SAVE_EVERY_S` unless forced. + + Throttled rather than deferred to close time, and that is the important + choice: closing the window *hides* it (see `hold_window_open`), so the + close-time hook that would be the obvious place to save this never runs on + the ordinary path. The app can then live in the notification area for days + and be quit from the tray, or killed, and either way the last thing written + should be roughly where the window was. + """ + global _saved_at + + import time # noqa: PLC0415 + + if not _geometry: + return + now = time.monotonic() + if not force and now - _saved_at < SAVE_EVERY_S: + return + _saved_at = now + from core import jsonl # noqa: PLC0415 + + try: + path = geometry_path() + path.parent.mkdir(parents=True, exist_ok=True) + jsonl.write_json(path, _geometry) + except OSError: # noqa: BLE001 - where the window was is never worth failing over + log.debug("could not save the window geometry", exc_info=True) + + +def track_window(nicegui_app: Any) -> None: + """Follow the window around and keep `window.json` current. + + All of this runs in *this* process. NiceGUI bridges pywebview's window + events from the child over a pipe and dispatches them onto the event loop + (`native/event_manager.py`), which is what makes a plain handler enough -- + the alternative would be another pickled function riding across in + `start_args`, like `hold_window_open`, for state this side is perfectly able + to keep. + + Registered before `ui.run`, because `ui.run` is what starts the event + manager that would deliver them. + """ + nicegui_app.native.on("moved", lambda e: remember(x=int(e.args["x"]), y=int(e.args["y"]))) + nicegui_app.native.on( + "resized", lambda e: remember(width=int(e.args["width"]), height=int(e.args["height"])) + ) + nicegui_app.native.on("maximized", lambda _: _note_maximized(True)) + nicegui_app.native.on("restored", lambda _: _note_maximized(False)) + # The throttle drops the last observation of a burst, and the last + # observation of a burst is the one worth keeping. + nicegui_app.native.on("closed", lambda _: save_geometry(force=True)) + nicegui_app.on_shutdown(lambda: save_geometry(force=True)) + + def bind_loop(loop: asyncio.AbstractEventLoop) -> None: global _loop diff --git a/ui/kit.py b/ui/kit.py index d07c649..2a2df3b 100644 --- a/ui/kit.py +++ b/ui/kit.py @@ -82,9 +82,20 @@ def attr(value: Any) -> str: has no reason to complain. That matters because these values are not all constants: a preflight remedy and a lineage bar's candidate id are ledger text, and ledger text can hold a quote. Newlines go for the same reason. + + **And the backslash, which is worse than the quote.** NiceGUI parses a props + string and hands each value to `ast.literal_eval`, so the text is read as a + Python string literal and a backslash is an escape character in it. On + Windows that is not a curiosity: `C:\\Users\\...` in a tooltip contains `\\U`, + which begins a unicode escape, and `literal_eval` raises a SyntaxError from + inside `element.props()` -- taking down not the tooltip but whatever was + being built. It surfaced the first time a control put a *path* in a tooltip, + which is to say the first time the appbar had to say which folder this is. """ collapsed = " ".join(str("" if value is None else value).split()) - return collapsed.replace('"', "'") + # Backslashes first: doing it after the quote swap would also escape the + # apostrophes this puts in. + return collapsed.replace("\\", "\\\\").replace('"', "'") def el(tag: str, classes: str = "", *, style: str = "") -> Any: @@ -362,6 +373,90 @@ def menu(draw: Callable[[Any, Any], None], *, width: int = 460) -> Menu: return Menu(dialog, lambda m: draw(body, m)) +def steps( + items: Sequence[dict[str, Any]], + active: str, + on_pick: Callable[[str], Any], +) -> Any: + """A row of numbered steps, each one a way back to itself. + + Not a wizard that marches forward: every step stays reachable, because a + setup that has to be restarted to change the answer to question two is a + setup people abandon at question three. The mark is the step's state -- a + tick when it is satisfied, its number when it is not -- so "what is left" + is answerable without opening anything. + + `items` are `setup_model`'s steps; each needs `id`, `caption`, `ready`. + """ + with el("div", "grad-steps") as element: + for index, item in enumerate(items, start=1): + current = item["id"] == active + classes = "grad-step" + (" open" if current else "") + step = el("button", classes) + step.props(f'title="{attr(item.get("hint", ""))}"') + step.on("click", lambda _=None, sid=item["id"]: on_pick(sid)) + with step: + text("✓" if item.get("ready") else str(index), "mark", tag="span") + text(item["caption"], "name", tag="span") + text(item.get("detail", ""), "hint", tag="span") + return element + + +class Confirm: + """A yes/no dialog, built once and reused for every question. + + Built during the page and only *opened* later, for `_install_quit_guard`'s + reason: a NiceGUI element belongs to the client whose slot context created + it, and the handler that wants to ask is running long after that context has + gone. So the shell constructs one of these while there is a client, and the + control that needs an answer awaits `ask`. + + It exists because exactly one control in the app is destructive enough to + need it. Switching *project* changes what spend is charged to; switching + *workspace folder* replaces the ledger, the project list, the notebooks and + the config under every open window at once. Those two sat six rows apart in + one dialog, styled identically, and the difference was discoverable only by + doing it. + """ + + def __init__(self, dialog: Any, card: Any) -> None: + self._dialog = dialog + self._card = card + + async def ask( + self, + title: str, + body: str, + *, + confirm: str = "CONTINUE", + cancel: str = "CANCEL", + tone: str = "danger", + note_text: str = "", + ) -> bool: + self._card.clear() + with self._card: + text(title, "grad-label") + text(body) + if note_text: + note(note_text) + with row("", gap=9): + button(cancel, tone="primary", on_click=lambda: self._dialog.submit(False)) + button(confirm, tone=tone, on_click=lambda: self._dialog.submit(True)) + self._dialog.open() + return bool(await self._dialog) + + +def confirm() -> Confirm: + """A `Confirm` over the app's own paper. Call this during the page build.""" + ui = _ui() + dialog = ui.dialog().props("persistent") + with dialog, el("div", "grad-app"): + card = column("grad-pad", gap=9).style( + "background: var(--grad-paper); border: var(--grad-border); min-width: 440px" + ) + return Confirm(dialog, card) + + def menu_row( mark: str, name: str, @@ -409,6 +504,21 @@ def run_js(code: str) -> None: zero-delay timer is NiceGUI's own answer -- it defers to the first tick after the page is live, and it makes the render function synchronous and testable rather than quietly scheduling background tasks. + + **It must be called with a live slot in scope, and that is a real trap.** The + timer is an element, so it is created in the enclosing slot -- and inside an + event handler the enclosing slot belongs to the element the handler was bound + to, which a handler that rebuilds the UI has usually just deleted. There is + no way to recover from here: `context.client` is itself reached *through* the + current slot, so a deleted one takes the client with it and the code is never + sent. All of it raises `RuntimeError: The parent element this slot belongs to + has been deleted` before the socket is touched. + + It is also quiet. `ui/shell.py:retile` runs from a titlebar button and + deletes every titlebar including the one that was clicked; the raise landed + in `ui/state.py:_guard`, which logs and carries on, so the JavaScript simply + never ran on any path but the first page build -- see `gradRearm`'s call + site, which enters a long-lived container precisely for this reason. """ ui = _ui() ui.timer(0.05, lambda: ui.run_javascript(code), once=True) diff --git a/ui/models.py b/ui/models.py index 3d40107..8c4192a 100644 --- a/ui/models.py +++ b/ui/models.py @@ -26,6 +26,7 @@ import datetime as _dt import json +import os import re import time as _time from pathlib import Path @@ -142,6 +143,170 @@ def workspaces_model() -> dict[str, Any]: } +#: The three ceilings a project carries: the resource name `core/budget.py` uses, +#: the `tools.budget raise` flag it maps to, and how to say it on screen. +#: +#: One list, because there were two. `ui/shell.py` kept its own copy for the +#: menu's raise controls, keyed by flag, while every meter in the app was keyed +#: by resource -- so the two spellings of one idea were maintained in different +#: files and neither knew the other existed. +CEILINGS: tuple[tuple[str, str, str, str], ...] = ( + ("gpu_usd", "gpu-usd", "GPU $", "dollars of remote compute"), + ("quota_tokens", "quota-tokens", "tokens", "subscription tokens, all roles"), + ("credits_usd", "credits-usd", "credits $", "the reranker and embeddings"), +) + +#: How each resource is written. Tokens are counted, not priced, and rendering +#: 4.2M of them as `$4,200,000.00` was the specific thing this separates. +_CEILING_FORMAT = {"gpu_usd": _usd, "quota_tokens": _tokens, "credits_usd": _usd} + + +def _project_memory(project_id: str) -> dict[str, Any]: + """Which of the six per-project documents exist. Six `exists()`, no reads. + + "Scaffolded but empty" and "never scaffolded" are different states and the + window says which: the first is a project nobody has written in yet, the + second is one whose `new` failed to scaffold -- `tools/budget.py` guards that + step precisely because it must not fail the creation, and this is where the + consequence becomes visible instead of being discovered by `project sync`. + """ + from core import projects as projects_mod + + directory, error = _safe(lambda: projects_mod.resolve_dir(project_id)) + if directory is None: + return {"dir": None, "present": [], "missing": list(projects_mod.DOCS), "error": error} + present, listing_error = _safe( + lambda: [name for name in projects_mod.DOCS if (directory / name).exists()], [] + ) + present = present or [] + return { + "dir": str(directory), + "present": present, + "missing": [name for name in projects_mod.DOCS if name not in present], + "scaffolded": bool(present), + "error": error or listing_error, + } + + +def projects_model() -> dict[str, Any]: + """Every project in this folder: what bounds it, what it has spent, and + whether anything has been written down about it. + + This replaced a section of the `project ▾` dialog and is deliberately not the + same data. A menu row had space for an id, a title and one summary line, and + the ceiling controls under the list addressed only the *selected* project -- + so reading what bounds a project you were not on meant switching to it first, + which charges nothing but reloads every window in the app. + + Wrapped reader by reader, for `workspaces_model`'s reason: this window has to + render when the workspace is wrong. `status` folds the whole run ledger for + one project, so it is caught per row -- one project whose spend will not + compute says so in its own row instead of taking the list down with it. + """ + from core import budget as budget_mod, config as config_mod, settings as settings_mod + + root, root_error = _safe(lambda: str(paths.root()), "") + current, _ = _safe(budget_mod.current_project) + records, projects_error = _safe(budget_mod.projects, {}) + cfg, _ = _safe(config_mod.load) + + # What a role resolves to with the project layer removed. Every project but + # the selected one has *someone else's* project layer in effect, so this is + # the only honest thing to show beside an override that is not set. + workspace_models: dict[str, str] = {} + for role in config_mod.MODEL_ROLES: + value, _ = _safe(lambda r=role: cfg.model_for(r, project=False) if cfg else "", "") + workspace_models[role] = value or config_mod.DEFAULTS["models"][role] + + rows: list[dict[str, Any]] = [] + for project_id, record in sorted((records or {}).items()): + state, state_error = _safe(lambda pid=project_id: budget_mod.status(pid), {}) + resources = (state or {}).get("resources") or {} + ceilings = [] + for resource, flag, caption_text, hint in CEILINGS: + node = resources.get(resource) or {} + ceiling = node.get("ceiling") + render = _CEILING_FORMAT[resource] + ceilings.append( + { + "resource": resource, + "flag": flag, + "caption": caption_text, + "hint": hint, + "ceiling": ceiling, + "spent": node.get("spent", 0.0), + "fraction": node.get("fraction"), + "over": bool(node.get("over")), + "set": ceiling is not None, + "label": ( + f"{render(node.get('spent', 0.0))} spent · no ceiling" + if ceiling is None + else f"{render(node.get('spent', 0.0))} / {render(ceiling)}" + ), + } + ) + rows.append( + { + "id": project_id, + "title": _short(record.get("title") or "", 90), + "status": record.get("status") or "open", + "current": project_id == current, + "payer": record.get("payer"), + # The date alone. A project list is read for "which of these am I + # still working on", and a timestamp to the second answers a + # question nobody asked while costing the row half its width. + "created": (record.get("created_at") or "")[:10], + "spend": _spend_line(state or {}), + "run_count": (state or {}).get("run_count", 0), + "over_budget": (state or {}).get("over_budget") or [], + "raise_count": len(record.get("raises") or []), + "ceilings": ceilings, + # What this project overrides about how it is run, and what each + # role would be without it. The model per role is the main lever + # on cost and quality, which is exactly why it should be able to + # differ between a cheap exploratory project and one being + # written up. + "models": [ + { + "role": role, + "override": (record.get("models") or {}).get(role), + "workspace": workspace_models[role], + "effective": (record.get("models") or {}).get(role) + or workspace_models[role], + } + for role in config_mod.MODEL_ROLES + ], + "override_count": len(record.get("models") or {}), + "backend": record.get("backend"), + "configured_count": len(record.get("configured") or []), + # A project with no ceilings bounds nothing and every gate that + # reads one passes silently. Surfaced as a flag so the window can + # say it where it is true, rather than in a caption under a form. + "unbounded": not any(c["set"] for c in ceilings), + "memory": _project_memory(project_id), + "error": state_error, + } + ) + + # Whether the machine half of setup still has something in it. Cheap -- one + # credential-store read -- and it is what lets the create form point at the + # wizard instead of containing it. + needs_setup, _ = _safe(setup_needed, False) + + return { + "root": root, + "rows": rows, + "current_project": current, + "needs_setup": bool(needs_setup), + "known_models": list(settings_mod.KNOWN_MODELS), + "known_backends": list(settings_mod.BACKENDS), + "count": len(rows), + "open_count": len([r for r in rows if r["status"] != "closed"]), + "unbounded": [r["id"] for r in rows if r["unbounded"] and r["status"] != "closed"], + "error": root_error or projects_error, + } + + def update_model() -> dict[str, Any]: """What the project menu says about updating, read from the cache only. @@ -206,7 +371,12 @@ def _spend_line(state: dict[str, Any]) -> str: """ resources = state.get("resources") or {} parts: list[str] = [] - for name, render in (("gpu_usd", _usd), ("quota_tokens", _tokens), ("credits_usd", _usd)): + # Derived from `CEILINGS` rather than listed again. This held its own copy of + # the three resources and their formatters, which is the third place that + # list has existed -- and a fourth ceiling added to `core/budget.py` would + # have appeared in every meter in the app except this line. + for name, _flag, _caption, _hint in CEILINGS: + render = _CEILING_FORMAT[name] entry = resources.get(name) or {} ceiling = entry.get("ceiling") if not ceiling: @@ -240,8 +410,17 @@ def header_model(*, agent_state: str = "idle", step: int | None = None) -> dict[ window, error = _safe(lambda: _session_window(hours=5), {}) window = window or {} used = float(window.get("credits_usd", 0.0)) + # The folder, for the appbar's `workspace ▾`. Read here rather than from + # `workspaces_model`, which folds every project and its spend to answer a + # question the title bar is not asking -- and which the title bar redraws on + # every tick. + root, root_error = _safe(lambda: paths.root(), None) return { "project": project or "unassigned", + "root": str(root) if root else "", + # The basename. An absolute path does not fit an appbar cell, and the + # full one is the button's tooltip. + "root_name": root.name if root else "—", "agent_state": agent_state if agent_state in AGENT_STATES else "idle", "accent": AGENT_ACCENT.get(agent_state, "neutral"), "step": step, @@ -254,7 +433,7 @@ def header_model(*, agent_state: str = "idle", step: int | None = None) -> dict[ "resets_in": window.get("resets_in", "—"), "tokens": window.get("tokens", 0), }, - "error": project_error or error, + "error": project_error or error or root_error, # Repeated wherever a token number appears, per §10. The provider # exposes no remaining-quota API; this is our own tally. "honesty": "self-measured tally, not the provider's — an estimate within ±5%", @@ -532,17 +711,51 @@ def sessions_model(current: str | None = None) -> dict[str, Any]: # --------------------------------------------------------------------------- # 0a. credentials # --------------------------------------------------------------------------- -#: What each credential unlocks, and whether the system works without it. The -#: text matters as much as the flag: "missing" is not the same fact for a token +#: What each credential unlocks, and which part of the system it belongs to. The +#: text matters as much as the group: "missing" is not the same fact for a token #: that gates GPU submission as for one that raises a rate limit. -CREDENTIAL_NOTES: dict[str, tuple[str, bool]] = { - "hf_token": ("Hugging Face Jobs — submitting and collecting runs", True), - "openrouter_key": ("optional second rail for the reranker; Voyage is used by default", False), - "voyage_key": ("the reranker and the local index's embeddings (costs credits)", False), - "asta_api_key": ("raises Asta's rate limits; discovery works without it", False), - "s2_api_key": ("Semantic Scholar direct — only issued to institutional addresses", False), - "context7_key": ("raises Context7's rate limits; lookups work without it", False), - "claude_oauth_token": ("the funnel's Haiku stages, when the agent runs them", True), +#: +#: The group replaced a bare `required` flag, and the flag was making a claim it +#: could not support. `hf_token` was marked required, so a user who had chosen +#: Kaggle -- the free backend, and the one a new user is most likely to start on +#: -- was shown a red MISSING for a token they will never need. What is actually +#: true is that HF Jobs needs it, which is a fact about a *backend*, and +#: `tools/setup.py:readiness` is where that belongs. +CREDENTIAL_GROUPS: dict[str, str] = { + # Nothing at all works without this one. + "agent": "the agent itself", + # Needed by one backend each, and only if you use that backend. + "backend": "where runs execute", + # Buys or widens retrieval. Everything here degrades rather than fails. + "retrieval": "papers and embeddings", + "extras": "nice to have", +} + +CREDENTIAL_NOTES: dict[str, tuple[str, str]] = { + "claude_oauth_token": ( + "the agent's own loop, the funnel's Haiku stages and the mutation operator", + "agent", + ), + "hf_token": ("Hugging Face Jobs — submitting and collecting runs", "backend"), + # The eighth credential. It was in `credentials.ALL` and not here, so the + # panel drew it with an empty purpose column -- the one credential whose row + # said nothing about what it was for, and the one belonging to the backend + # that costs nothing to try. + "kaggle_key": ( + "Kaggle kernels — the free GPU/TPU backend; useless without the username", + "backend", + ), + "voyage_key": ("the reranker and the local index's embeddings (costs credits)", "retrieval"), + "openrouter_key": ( + "optional second rail for the reranker; Voyage is used by default", + "retrieval", + ), + "asta_api_key": ("raises Asta's rate limits; discovery works without it", "retrieval"), + "s2_api_key": ( + "Semantic Scholar direct — only issued to institutional addresses", + "retrieval", + ), + "context7_key": ("raises Context7's rate limits; lookups work without it", "extras"), } @@ -559,12 +772,19 @@ def credentials_model() -> dict[str, Any]: present, error = _safe(credentials_mod.status, {}) rows = [] for name, stored in (present or {}).items(): - purpose, required = CREDENTIAL_NOTES.get(name, ("", False)) + purpose, group = CREDENTIAL_NOTES.get(name, ("", "extras")) + # Only the agent's own token is unconditionally required: without it + # nothing runs at all. A backend credential is required *for that + # backend*, which `tools/setup.py:readiness` reports against the backend + # rather than against the key. + required = group == "agent" rows.append( { "name": name, "stored": bool(stored), "purpose": purpose, + "group": group, + "group_label": CREDENTIAL_GROUPS.get(group, group), "required": required, "tone": "ok" if stored else ("broken" if required else "neutral"), "state": "STORED" if stored else ("MISSING" if required else "not set"), @@ -580,6 +800,174 @@ def credentials_model() -> dict[str, Any]: } +# --------------------------------------------------------------------------- +# 0a2. setup +# --------------------------------------------------------------------------- +#: The steps of the machine half, in order. `id`, the caption, and the one line +#: saying what answering it buys. +#: +#: Machine-scoped, all four of them, which is why they are not asked again when +#: a project is created. A project's own step -- ceilings, payer, the backend it +#: reaches for -- is Stage 4 and lives with the project, because it is the only +#: part of this that differs per project. +SETUP_STEPS: tuple[tuple[str, str, str], ...] = ( + ("token", "subscription", "the OAuth token every model call authenticates with"), + ("models", "models", "which model runs which of the six roles"), + ("backends", "backends", "where a training run actually executes"), + ("extras", "extras", "optional keys that widen retrieval or raise a rate limit"), +) + + +def setup_needed() -> bool: + """Whether this install has nothing to run with. + + Deliberately narrow: only the subscription token. An unconfigured backend + means no remote training, which is a real limitation and not a reason to put + a wizard in front of someone who wanted to read a ledger -- but with no token + the main loop cannot authenticate, so the four windows a fresh workspace + opens are four windows that can do nothing. + + Both places the token can be are checked, because both work: `credentials` + is the durable one and the environment is what a terminal export leaves. + Never raises -- `present` already swallows an unreachable store, and a + machine with no keyring is a machine with nothing configured, which is the + same answer. + """ + from core import credentials as credentials_mod + + if os.environ.get("CLAUDE_CODE_OAUTH_TOKEN"): + return False + stored, _ = _safe(lambda: credentials_mod.present(credentials_mod.CLAUDE_TOKEN), False) + return not stored + + +def setup_model() -> dict[str, Any]: + """What is configured, what is not, and what each answer would buy. + + Wrapped reader by reader like every other model here, and this one has the + strongest claim to it: the whole point of the window is to be usable on a + machine where nothing works yet, which is exactly the machine where a reader + is most likely to fail. + """ + from core import config as config_mod, credentials as credentials_mod, settings as settings_mod + from tools import setup as setup_tool + + cfg, cfg_error = _safe(config_mod.load) + stored, cred_error = _safe(credentials_mod.status, {}) + stored = stored or {} + + # -- the token --------------------------------------------------------- + ambient = bool(os.environ.get("CLAUDE_CODE_OAUTH_TOKEN")) + in_store = bool(stored.get("claude_oauth_token")) + token = { + "stored": in_store, + "ambient": ambient, + "state": "stored" if in_store else ("environment" if ambient else "missing"), + "ready": in_store or ambient, + # The distinction that matters on Windows: a token exported in a shell + # authenticates that shell, and the desktop shortcut launches from + # Explorer with whatever the user made persistent -- usually nothing. + "durable": in_store, + "name": credentials_mod.CLAUDE_TOKEN, + "mint": "claude setup-token", + } + + # -- the six roles ----------------------------------------------------- + overlay_models, _ = _safe(settings_mod.models, {}) + overlay_models = overlay_models or {} + roles = [] + for role in config_mod.MODEL_ROLES: + configured = ((getattr(cfg, "user", None) or {}).get("models") or {}).get(role) + chosen, _ = _safe(lambda r=role: cfg.model_for(r) if cfg else "", "") + roles.append( + { + "role": role, + "model": chosen, + "source": ( + "setup" if role in overlay_models else ("config" if configured else "default") + ), + "default": config_mod.DEFAULTS["models"][role], + "overridden": role in overlay_models, + } + ) + + # -- where a run executes ---------------------------------------------- + backends, backend_error = _safe(lambda: setup_tool.readiness(cfg) if cfg else [], []) + backends = backends or [] + hosts, _ = _safe(lambda: sorted(cfg.hosts) if cfg else [], []) + kaggle_account, _ = _safe(_kaggle_account, {}) + + credentials_panel = credentials_model() + shadowing, _ = _safe(lambda: settings_mod.shadowing(cfg) if cfg else [], []) + + steps = [] + for step_id, caption, hint in SETUP_STEPS: + if step_id == "token": + ready, detail = token["ready"], token["state"] + elif step_id == "models": + # Always resolvable -- there are defaults for all six. The step is + # here to be adjusted, not to be satisfied, so it never blocks. + ready, detail = True, f"{len(overlay_models)} of {len(roles)} chosen here" + elif step_id == "backends": + usable = [b["backend"] for b in backends if b["ready"]] + ready = bool(usable) + detail = ", ".join(usable) if usable else "none configured" + else: + optional = [r for r in credentials_panel["rows"] if r["group"] in ("retrieval", "extras")] + ready = True + detail = f"{len([r for r in optional if r['stored']])} of {len(optional)} stored" + steps.append( + { + "id": step_id, + "caption": caption, + "hint": hint, + "ready": ready, + "detail": detail, + "tone": "ok" if ready else "attention", + } + ) + + return { + "steps": steps, + "token": token, + "roles": roles, + "known_models": list(settings_mod.KNOWN_MODELS), + "backends": backends, + "default_backend": _safe(settings_mod.default_backend)[0], + "known_backends": list(settings_mod.BACKENDS), + "hosts": hosts, + "kaggle": kaggle_account, + "credentials": credentials_panel, + "settings_path": str(_safe(settings_mod.path, "")[0] or ""), + "config_path": str(_safe(paths.config_path, "")[0] or ""), + "shadowing": shadowing or [], + # Nothing here blocks the app; this is what the appbar and the first-run + # arrangement ask about. + "complete": token["ready"] and any(b["ready"] for b in backends), + "error": cfg_error or cred_error or backend_error, + } + + +def _kaggle_account() -> dict[str, Any]: + """The half of the Kaggle credential that is not a secret. + + Its own entry rather than a credential row, for the reason + `tools/kaggle.py` gives: only the key is secret, and an account name you + cannot read back is a worse answer to "whose kernels are these?" than a file + you can. + """ + from core import config as config_mod, credentials as credentials_mod + from tools import kaggle as kaggle_tool + + cfg = config_mod.load() + username, source = kaggle_tool.resolve_username(cfg) + return { + "username": username or "", + "source": source, + "key_stored": credentials_mod.present(credentials_mod.KAGGLE_KEY), + } + + # --------------------------------------------------------------------------- # 0b. background tasks # --------------------------------------------------------------------------- diff --git a/ui/registry.py b/ui/registry.py index 9ff1ef8..79ce5f0 100644 --- a/ui/registry.py +++ b/ui/registry.py @@ -1,9 +1,9 @@ """The window registry: the one list the whole shell is derived from. The `⋯` menu, the persisted layout's validation and the status bar's count all -read this tuple. Adding a thirteenth window is adding one `WindowSpec` and one -module -- if it is ever more than that, something has grown a second list and -the two will drift. +read this tuple. Adding a window is adding one `WindowSpec` and one module -- if +it is ever more than that, something has grown a second list and the two will +drift. `module` is resolved with `importlib` at first render rather than imported here, for the reason the rest of the app imports lazily: `ui.registry` has to stay @@ -37,6 +37,8 @@ class WindowSpec: WINDOWS: tuple[WindowSpec, ...] = ( + WindowSpec("setup", "setup", "ui.windows.setup", "token, models, backends — what this machine is wired to"), + WindowSpec("projects", "projects", "ui.windows.projects", "what the research is divided into, and what bounds it"), WindowSpec("chat", "chat", "ui.windows.chat", "the agent session", default=True, persistent=True), WindowSpec("notebook", "notebook", "ui.windows.notebook", "JupyterLab, and the verify banner", default=True, persistent=True), WindowSpec("ledger", "ledger", "ui.windows.ledger", "expectations against outcomes", default=True), diff --git a/ui/shell.py b/ui/shell.py index 8a5353a..bcd01d9 100644 --- a/ui/shell.py +++ b/ui/shell.py @@ -45,11 +45,12 @@ def build(workspace: Workspace) -> None: windows = _windows_menu(ui, workspace) projects = _project_menu(ui, workspace) + workspaces = _workspace_menu(ui, workspace) def draw_appbar() -> None: appbar.clear() with appbar: - _appbar(workspace, windows, projects) + _appbar(workspace, windows, projects, workspaces) def draw_status() -> None: statusbar.clear() @@ -97,6 +98,24 @@ def retile() -> None: with kit.el("div", "grad-slot", style=f"--grad-fraction: {slot.fraction:.6f}"): _frame(workspace, slot.window, roots, bars, attic, draw_window) + # Every root above has just been through `Element.move()`, and a + # moved root comes back as a *new DOM node* on the client -- which + # silently discards anything a window attached to its own node + # rather than to a NiceGUI element. The sticky transcript is the one + # that mattered: it was armed once at chat render and died on the + # first retile, so opening any window once stopped the chat + # scrolling for the rest of the session. `gradRearm` is idempotent + # and knows which ids to re-arm, so the shell does not have to know + # that `grad-transcript` exists. + # + # Inside `with tiles`, and that is not stylistic. This runs from a + # titlebar button, and by here the retile has deleted that button + # along with the slot it belonged to -- so at this point there is no + # current slot, no way to reach the client through one, and + # `kit.run_js` raises into `_guard` rather than sending anything. + # `tiles` is built once in `build` and outlives every retile. + kit.run_js("window.gradRearm && window.gradRearm()") + workspace.bind_chrome(draw_appbar) workspace.bind_chrome(draw_status) workspace.bind_retile(retile) @@ -154,7 +173,7 @@ async def confirm(report: dict[str, Any]) -> bool: # --------------------------------------------------------------------------- # chrome # --------------------------------------------------------------------------- -def _appbar(workspace: Workspace, windows: Any, projects: Any) -> None: +def _appbar(workspace: Workspace, windows: Any, projects: Any, workspaces: Any) -> None: header = workspace.header() session = header["session"] @@ -162,13 +181,30 @@ def _appbar(workspace: Workspace, windows: Any, projects: Any) -> None: kit.text("∇", "grad-mark") kit.text("GRAD", "grad-wordmark") + # Two scopes, two controls. One folder holds many projects, and switching + # folders replaces the ledger, the project list, the notebooks and the config + # under every open window -- while switching project changes what spend is + # charged to. They were one dialog, six rows apart, styled identically. + with kit.el("div", "grad-appbar-cell"): + kit.text("workspace", "dim") + kit.button( + f"{header['root_name']} ▾", + tone="ghost", + classes="grad-appbar-btn", + # The basename is on the button because an absolute path does not fit + # the cell; the whole one is here, because "which folder is this?" + # has to be answerable without opening anything. + title=f"{header['root']} — switch folder, credentials, and this installation", + on_click=workspaces.open, + ) + with kit.el("div", "grad-appbar-cell"): kit.text("project", "dim") kit.button( f"{header['project']} ▾", tone="ghost", classes="grad-appbar-btn", - title="switch project, or open another workspace folder", + title="switch the project runs and tokens are charged to", on_click=projects.open, ) # Only when there is something to do about it. A permanent "up to date" @@ -183,7 +219,7 @@ def _appbar(workspace: Workspace, windows: Any, projects: Any) -> None: tone="ghost", classes="grad-appbar-btn", title=f"{update['target']} is available — open the workspace menu to install it", - on_click=projects.open, + on_click=workspaces.open, ) with kit.el("div", "grad-appbar-cell"): @@ -269,14 +305,23 @@ def _statusbar(workspace: Workspace) -> None: def _project_menu(ui: Any, workspace: Workspace) -> kit.Menu: - """The workspace menu: which folder, which project, and how to change both.""" - return kit.menu( - lambda body, menu: _draw_project_menu(ui, workspace, body, menu), width=540 - ) + """The quick switcher: which project is charged, and nothing else. + It used to be the whole settings surface — folder, recent folders, projects, + creation, ceilings, credentials and the updater, in one 540px dialog behind a + button labelled `project`. Four scopes in one menu, and the two most + different actions in the app (switch project, switch folder) six rows apart + and styled the same. -def _draw_project_menu(ui: Any, workspace: Workspace, body: Any, menu: Any) -> None: - model = workspace.workspaces() + What is left here is the one thing worth a single click from the title bar. + Everything a project *is* — its ceilings, its memory, creating and closing — + is the projects window, which this opens. + """ + return kit.menu(lambda body, menu: _draw_project_menu(workspace, body, menu), width=460) + + +def _draw_project_menu(workspace: Workspace, body: Any, menu: Any) -> None: + model = workspace.projects() body.clear() def act(coro: Any, what: str) -> None: @@ -285,61 +330,24 @@ def act(coro: Any, what: str) -> None: menu.close() workspace.spawn(coro, what) + def open_window() -> None: + menu.close() + workspace.open("projects") + with body: - kit.text("WORKSPACE", "head ink") + kit.text("PROJECT", "head ink") with kit.el("div", "body"): kit.error_strip(model.get("error")) - kit.kv([("folder", model["root"]), ("chosen by", model["source"])]) - - with kit.row("", gap=6).style("margin-top: 10px"): - folder = ( - ui.input(placeholder="path to another workspace folder") - .props("borderless dense") - .classes("field") - .style("flex: 1 1 auto; padding: 0 8px") - ) - kit.button( - "BROWSE…", - tone="neutral", - title="pick a folder (needs the desktop window)", - on_click=lambda: workspace.spawn(_browse(workspace, folder), "folder picker"), - ) - kit.button( - "OPEN", - tone="primary", - title="switch this app to that folder", - on_click=lambda: act( - workspace.switch_root(folder.value or "", create=True), "workspace switch" - ), - ) - kit.text( - "a folder that does not exist yet is created; the agent's tools follow it", - "grad-caption", - ) - - if model["recent"]: - kit.text("RECENT", "grad-caption").style("margin-top: 12px") - for path in model["recent"]: - with kit.row("grad-row", gap=6): - kit.button( - "OPEN", - tone="neutral", - on_click=lambda _=None, p=path: act( - workspace.switch_root(p), "workspace switch" - ), - ) - kit.text(path, "grad-caption") - # -- projects --------------------------------------------------- - kit.text("PROJECTS IN THIS FOLDER", "grad-caption").style("margin-top: 16px") - if not model["projects"]: - kit.text("none yet — the first one is created below", "grad-empty") - for project in model["projects"]: + rows = [r for r in model.get("rows") or [] if r["status"] != "closed"] + if not rows: + kit.text("none in this folder yet", "grad-empty") + for project in rows: with kit.row("grad-row", gap=6): kit.button( "IN USE" if project["current"] else "USE", tone="active" if project["current"] else "neutral", - disabled=project["current"] or project["status"] == "closed", + disabled=project["current"], on_click=lambda _=None, pid=project["id"]: act( workspace.use_project(pid), "project switch" ), @@ -354,230 +362,149 @@ def act(coro: Any, what: str) -> None: " text-overflow: ellipsis; white-space: nowrap", ) kit.spacer() - if project["status"] == "closed": - kit.chip("CLOSED", "neutral") + if project["unbounded"]: + kit.chip("UNBOUNDED", "attention") kit.text(project["spend"], "grad-caption") - # -- a new one -------------------------------------------------- - kit.text("NEW PROJECT", "grad-caption").style("margin-top: 16px") - with kit.row("", gap=6): - project_id = ( - ui.input(placeholder="id, e.g. proj-scaling-w2") - .props("borderless dense") - .classes("field") - .style("flex: 0 0 220px; padding: 0 8px") - ) - title = ( - ui.input(placeholder="what this research is") - .props("borderless dense") - .classes("field") - .style("flex: 1 1 auto; padding: 0 8px") - ) + with kit.row("", gap=6).style("margin-top: 12px"): kit.button( - "CREATE", + "OPEN THE PROJECTS WINDOW", tone="primary", - on_click=lambda: act( - workspace.create_project(project_id.value or "", title.value or ""), - "project create", - ), + title="ceilings, memory, creating and closing — for every project, not just this one", + on_click=open_window, ) - kit.text( - "created with no ceilings — set them below once it is selected", - "grad-caption", - ) - _ceilings(ui, workspace, model, menu) - _credentials(ui, workspace, menu) - _updates(workspace, menu) +def _workspace_menu(ui: Any, workspace: Workspace) -> kit.Menu: + """Everything that is not a project: which folder, which credentials, which + Grad. -def _updates(workspace: Workspace, menu: _Menu) -> None: - """Which Grad this is, and the one button that changes it. - - Reads a cached answer -- `ui/models.py:update_model` explains why this must - never be the thing that talks to the network. The section is always drawn, - including when there is nothing to install: "you are on v0.2.0, checked an - hour ago" is the answer to a question people actually ask, and a section - that appeared only when an update existed would leave them with nowhere to - look for it. + The `Confirm` is built here, during the page, for the reason + `_install_quit_guard` builds its dialog here: a NiceGUI element belongs to + the client whose slot context made it, and by the time the answer is wanted + that context is gone. """ - model = workspace.update() - - def act(coro: Any, what: str) -> None: - menu.close() - workspace.spawn(coro, what) - - kit.text("THIS INSTALLATION", "grad-caption").style("margin-top: 16px") - kit.kv([("version", model["installed"]), ("last checked", model["checked"])]) - - if not model["is_checkout"]: - kit.note( - "This copy was not installed from a git checkout, so it cannot update itself. " - "Reinstall from the repository to get updates." - ) - return - - for warning in model["warnings"]: - kit.note(f"{warning['message']} — {warning['fix']}") - for blocker in model["blockers"]: - kit.error_strip(f"{blocker['message']} — {blocker['fix']}") - - with kit.row("", gap=6).style("margin-top: 8px"): - if model["available"]: - kit.chip(f"{model['target']} AVAILABLE", "attention") - kit.button( - "UPDATE", - tone="primary", - title=( - "quit first: this release changes dependencies" - if model["needs_reinstall"] - else "fast-forward this installation and migrate its state" - ), - on_click=lambda: act(workspace.apply_update(), "update"), - ) - kit.button( - "CHECK NOW", - tone="neutral", - title="ask the remote whether there is a newer release", - on_click=lambda: act(workspace.check_update(), "update check"), - ) - kit.spacer() - - if model["available"] and model["needs_reinstall"]: - kit.text( - "this release changes dependencies, so it needs Grad closed — quit, then run " - "`grad update` in a terminal", - "grad-caption", - ) - elif model["available"]: - kit.text( - f"{model['behind']} commit(s) behind · restart Grad afterwards to load it", - "grad-caption", - ) - if model["dirty"]: - kit.text( - "the installation has uncommitted edits; runs submitted from it are stamped " - "as modified and `report check` will say so", - "grad-caption", - ) + confirm = kit.confirm() + return kit.menu( + lambda body, menu: _draw_workspace_menu(ui, workspace, body, menu, confirm), width=540 + ) -#: The three ceilings a project carries, and the unit each is counted in. -#: `tools.budget raise` takes one flag per resource; this is that list, in the -#: order the quota window draws them. -CEILINGS = ( - ("gpu-usd", "GPU $", "dollars of remote compute"), - ("quota-tokens", "tokens", "subscription tokens, all roles"), - ("credits-usd", "credits $", "reranker and embeddings"), +#: What switching folders actually does, said before it happens. Every window in +#: the app re-reads from the new root, `config/grad.toml` may resolve to a +#: different file, and the agent's own tools follow — which is the whole point, +#: and is also nothing like the project switch two controls away. +SWITCH_NOTE = ( + "The ledger, the project list, the notebooks and the config all come from the folder, " + "so every open window re-reads from the new one and the agent's tools follow it. " + "Nothing is deleted, and the folder you are leaving is untouched." ) -def _ceilings(ui: Any, workspace: Workspace, model: dict[str, Any], menu: _Menu) -> None: - """Move a ceiling on the selected project. - - A logged event, not a setting: `budget raise` appends to the ledger, so the - history of what was raised and when survives. The UI runs the same command - for the same reason every other button does. - """ - current = next((p for p in model["projects"] if p["current"]), None) - if current is None: - return +def _draw_workspace_menu( + ui: Any, workspace: Workspace, body: Any, menu: Any, confirm: kit.Confirm +) -> None: + model = workspace.workspaces() + body.clear() - kit.text("CEILINGS", "grad-caption").style("margin-top: 16px") - fields: dict[str, Any] = {} - with kit.row("", gap=6): - for flag, caption, hint in CEILINGS: - field = ( - ui.input(placeholder=caption) - .props("borderless dense") - .classes("field") - .style("flex: 1 1 0; padding: 0 8px") - ) - field.props(f'title="{kit.attr(hint)}"') - fields[flag] = field - - def raise_them() -> None: - # `--project `, not a positional: `tools.budget raise` takes the - # project as a flag, and passing it positionally failed with a usage - # error on every click -- the one budget-mutating control in the app, - # dead since it was written, and the control the over-budget refusal - # tells you to use. `tests/test_ui_argv.py` now runs every button's - # argv through the real parser. - argv = ["tools.budget", "raise", "--project", current["id"]] - base = len(argv) - for flag, field in fields.items(): - if (field.value or "").strip(): - argv += [f"--{flag}", str(field.value).strip()] - if len(argv) == base: - workspace.say("no ceiling given — fill one of the three fields") - return - menu.close() - workspace.spawn(workspace.run_and_reload(*argv, "--json"), "ceiling raise") - - kit.button("RAISE", tone="primary", on_click=raise_them) - kit.text( - f"a logged event on {current['id']} — leave a field blank to leave that ceiling alone", - "grad-caption", - ) + def act(coro: Any, what: str) -> None: + menu.close() + workspace.spawn(coro, what) + def open_setup() -> None: + menu.close() + workspace.open("setup") -def _credentials(ui: Any, workspace: Workspace, menu: _Menu) -> None: - """Store the credentials the README's install section lists. + async def switch(path: str, *, create: bool = False) -> None: + if not (path or "").strip(): + workspace.say("no folder given") + return + if not await confirm.ask( + "Switch workspace folder?", + f"This app moves to {path}.", + confirm="SWITCH", + note_text=SWITCH_NOTE, + ): + return + await workspace.switch_root(path, create=create) - This is the one thing the workspace genuinely could not do: `credential set` - prompts with `getpass`, which needs a terminal, so a fresh machine needed a - shell open beside the app to become usable. The value goes down a pipe - rather than in an argument -- see `Workspace.set_credential`. + with body: + kit.text("WORKSPACE", "head ink") + with kit.el("div", "body"): + kit.error_strip(model.get("error")) + kit.kv([("folder", model["root"]), ("chosen by", model["source"])]) - Values are never shown, and there is nothing here that could show one: the - CLI does not print them and `credentials.status()` returns booleans. - """ - model = workspace.credentials() - kit.text("CREDENTIALS", "grad-caption").style("margin-top: 16px") - kit.error_strip(model.get("error")) - - for row in model["rows"]: - # Two lines, not one. On one line the fixed-width pieces -- chip, name, - # a 200px input, two buttons -- left the purpose text a few dozen - # pixels in a 540px dialog, and flex squeezed it to its min-content - # width: one word per line, a column taller than the rest of the row. - with kit.column("grad-row", gap=6): - # Full width explicitly: `.grad-row`'s `align-items: flex-start` - # would otherwise shrink each line to its content and the input - # with it. - with kit.row("", gap=6).style("width: 100%"): - kit.chip(row["state"], row["tone"]) - kit.text(row["name"], "grad-mono", tag="span") - kit.text( - row["purpose"], "grad-caption", tag="span", - style="flex: 1 1 auto; min-width: 0", - ) - with kit.row("", gap=6).style("width: 100%"): - value = ( - ui.input(placeholder="paste to set") - .props("borderless dense type=password") + with kit.row("", gap=6).style("margin-top: 10px"): + folder = ( + ui.input(placeholder="path to another workspace folder") + .props("borderless dense") .classes("field") .style("flex: 1 1 auto; padding: 0 8px") ) + kit.button( + "BROWSE…", + tone="neutral", + title="pick a folder (needs the desktop window)", + on_click=lambda: workspace.spawn(_browse(workspace, folder), "folder picker"), + ) + kit.button( + "OPEN", + tone="primary", + title="switch this app to that folder", + on_click=lambda: act(switch(folder.value or "", create=True), "workspace switch"), + ) + kit.text( + "a folder that does not exist yet is created; the agent's tools follow it", + "grad-caption", + ) - def store(_=None, name=row["name"], field=value) -> None: - pasted, field.value = field.value or "", "" - workspace.spawn(workspace.set_credential(name, pasted), "credential set") - menu.redraw() + if model["recent"]: + kit.text("RECENT", "grad-caption").style("margin-top: 12px") + for path in model["recent"]: + with kit.row("grad-row", gap=6): + kit.button( + "OPEN", + tone="neutral", + on_click=lambda _=None, p=path: act(switch(p), "workspace switch"), + ) + kit.text(path, "grad-caption") - def forget(_=None, name=row["name"]) -> None: - workspace.spawn(workspace.delete_credential(name), "credential delete") - menu.redraw() + # Credentials and the updater moved to the setup window. They are + # facts about this machine and this installation, they need more + # room than a 540px dialog, and neither was ever a thing you do + # *while* switching folders. + with kit.row("", gap=6).style("margin-top: 16px"): + kit.button( + "SETUP", + tone="primary", + title="token, models, backends, credentials and this installation", + on_click=open_setup, + ) + kit.text( + _setup_line(workspace), "grad-caption", tag="span", + style="flex: 1 1 auto; min-width: 0", + ) - kit.button("SET", tone="neutral", on_click=store) - kit.button("✕", tone="neutral", disabled=not row["stored"], title="forget it", - on_click=forget) - kit.text( - "stored in Windows Credential Manager, never in the workspace and never in the " - "agent's environment — they are fetched at the moment of use", - "grad-caption", - ) +def _setup_line(workspace: Workspace) -> str: + """One line saying whether this machine is wired up, beside the button that + wires it. Caught, because the menu has to open on a machine where nothing + reads -- that is the machine it exists for.""" + try: + model = workspace.model("setup") or {} + except Exception: # noqa: BLE001 - a caption must never break a menu + return "credentials, models, backends" + if not (model.get("token") or {}).get("ready"): + return "not authenticated — nothing can reach a model yet" + if not model.get("complete"): + return "no backend configured — runs cannot leave this machine" + steps = model.get("steps") or [] + return f"{len([s for s in steps if s['ready']])}/{len(steps)} answered" + + +# `_updates` and `_credentials` lived here and are gone: both panels moved to +# `ui/windows/setup.py`, which is where a fact about the installation belongs. +# `_Menu` stays -- `_bind_client_events` still reaches for it. def folder_dialog_type() -> int: diff --git a/ui/splash.py b/ui/splash.py new file mode 100644 index 0000000..80221af --- /dev/null +++ b/ui/splash.py @@ -0,0 +1,333 @@ +"""The mark on screen while the workspace is still loading. + +Starting Grad from the shortcut is slow in a way nothing on screen admits to. +`pythonw.exe` opens no console, `ui.run(native=True)` shows no window until +NiceGUI has imported, bound a port, built the page and handed a URL to a +pywebview process that then starts an Edge WebView2 host. On a cold start that +is several seconds of a double-click having produced *nothing at all*, which +reads exactly like a shortcut that does not work -- so people click it again, +and the second launch hands over to the first (`core/instance.py`) and still +shows nothing. + +Three decisions are load-bearing. + +**It is a separate process.** Not a thread: the reason there is nothing on +screen is that this interpreter is busy importing, and importing holds the GIL +for long stretches. A Tk loop sharing that interpreter would put up a window +that does not paint and does not answer, which is worse than no window -- +Windows greys out a hung window and offers to close it. A child process is +scheduled independently and is on screen while the parent is still importing. + +**It is Tk.** It is in the standard library, it starts in about a tenth of a +second, and it is already installed anywhere `pythonw.exe` came from. Every +other option here -- a second pywebview window, a NiceGUI page, anything with a +browser engine in it -- is the same slow thing whose slowness this exists to +cover. + +**It can be pushed out of the way.** A splash screen that insists on staying in +front is a splash screen that stops you doing anything else while an app you +were not waiting for starts. Clicking it drops it behind everything and it +carries on waiting; dragging moves it. Neither cancels the launch, because the +launch is not this process's to cancel. + +It closes when the workspace's first client connects -- the page is not merely +built by then, it is *shown* -- and, failing that, when the pipe from the parent +closes or the timeout runs out. See `stop`. +""" + +from __future__ import annotations + +import logging +import os +import subprocess +import sys +import threading +import time +from typing import Any + +log = logging.getLogger("grad.ui") + +#: How long the window lives if nobody ever tells it to go away. This is the +#: backstop behind the backstop -- the parent closes the pipe, and a parent that +#: died without closing anything is caught by the pipe reaching EOF regardless. +#: What is left is a machine where neither happened, and a mark that sits on +#: screen forever is a bug report about the splash rather than about the start. +MAX_SECONDS = 180.0 +#: How often the Tk loop asks whether it should still be here. +POLL_MS = 120 +#: Pixels of movement that turn a click into a drag. Below this, a press and +#: release in the same place means "get out of the way" rather than "move here". +DRAG_SLOP_PX = 4 + +_process: subprocess.Popen | None = None + + +# --------------------------------------------------------------------------- +# the parent's side +# --------------------------------------------------------------------------- +def start(*, timeout_s: float = MAX_SECONDS) -> None: + """Put the mark on screen. Returns immediately; never raises. + + Call this as early in the launch as it can be called and still be right -- + after the single-instance check, because a second launch that hands over to + the first has nothing to load and should flash nothing, and before the first + expensive import, because everything after that is the wait being covered. + + Failure here is silent by construction. There is no display, no Tk, no + permission to spawn: all of them mean the app starts exactly as it did + before this module existed, and none of them is worth refusing to start over. + """ + global _process + + if _process is not None: + return + argv = [ + sys.executable, + "-m", + "ui.splash", + "--parent-pid", + str(os.getpid()), + "--timeout", + str(float(timeout_s)), + # Only ever passed here, because only here is stdin known to be a pipe + # this process holds open. Run by hand it is a console or an + # already-closed handle, and watching that reads EOF at once -- a splash + # that vanishes the instant it is started, which is a confusing thing to + # meet while debugging one. + "--watch-stdin", + ] + try: + from core import paths, spawn # noqa: PLC0415 + + _process = subprocess.Popen( # noqa: S603 - our own module, our own argv + argv, + # The pipe is the liveness channel, not a channel for data: the + # child watches it for EOF. See `stop` and `_watch_parent`. + stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + cwd=str(paths.install_dir()), + **spawn.quiet(), + ) + except Exception: # noqa: BLE001 - see the docstring + log.debug("could not start the loading window", exc_info=True) + _process = None + + +def stop() -> None: + """Take the mark down. Idempotent, and safe to call from anywhere. + + **Closing the pipe is the signal**, and it is a better one than terminating + the process, because it is the same signal a crash sends. The child is + watching one file descriptor for EOF; it gets that whether this process + closed the handle deliberately, exited, or was killed -- so there is no path + that leaves a splash screen on a machine whose app is gone. `terminate` is + only what happens when a child has stopped reading it. + """ + global _process + + proc, _process = _process, None + if proc is None: + return + try: + if proc.stdin is not None: + proc.stdin.close() + except OSError: + pass + try: + proc.wait(timeout=3) + except subprocess.TimeoutExpired: + try: + proc.terminate() + except OSError: + pass + except Exception: # noqa: BLE001 - never worth failing a launch over + log.debug("could not close the loading window", exc_info=True) + + +def running() -> bool: + return _process is not None + + +# --------------------------------------------------------------------------- +# the child's side +# --------------------------------------------------------------------------- +def _watch_parent(gone: threading.Event) -> None: + """Set `gone` when the pipe from the parent closes. + + `os.read` on descriptor 0 rather than `sys.stdin`, because under + `pythonw.exe` there is no console and Python may leave `sys.stdin` as None + even though the descriptor this process was handed is perfectly real. + """ + while True: + try: + if not os.read(0, 1): + break + except (OSError, ValueError): + break + gone.set() + + +def _centre(window: Any, width: int, height: int) -> None: + """Put the window in the middle of the screen it is opening on.""" + screen_w = window.winfo_screenwidth() + screen_h = window.winfo_screenheight() + x = max(0, (screen_w - width) // 2) + # Slightly above centre. Optical centre sits higher than geometric centre, + # and a box placed at exactly half the height reads as low. + y = max(0, int((screen_h - height) * 0.42)) + window.geometry(f"{width}x{height}+{x}+{y}") + + +def show(*, timeout_s: float = MAX_SECONDS, watch_stdin: bool = False) -> int: + """The window itself. Runs the Tk loop until something says to stop.""" + import tkinter as tk # noqa: PLC0415 - the child's whole reason to exist + + from ui.tokens import COLOUR # noqa: PLC0415 + + # Read from the palette, not spelled here. `tests/test_ui_tokens.py` enforces + # that rule across the package and it applies to a window drawn in Tk exactly + # as it does to one drawn in CSS -- a splash screen with its own idea of the + # brand yellow is the first thing to drift when the palette changes. The + # module is pure Python with no dependencies of its own, so the cost of + # importing it in this process is a parse. + ink, paper, brand = COLOUR["ink"], COLOUR["paper"], COLOUR["attention"] + muted = COLOUR["muted"] + width, height = 340, 232 + + gone = threading.Event() + if watch_stdin: + threading.Thread(target=_watch_parent, args=(gone,), daemon=True).start() + + root = tk.Tk() + root.title("Grad") + # Borderless: this is a mark, not a window anyone should be asked to manage. + # It also keeps it out of the taskbar, where a second Grad entry that + # disappears on its own would be its own small confusion. + root.overrideredirect(True) + root.configure(bg=ink) + root.attributes("-topmost", True) + _centre(root, width, height) + + # The ink border is the frame's own background showing through a 2px inset, + # which is this design language's border everywhere else. + card = tk.Frame(root, bg=paper) + card.place(x=2, y=2, width=width - 4, height=height - 4) + + mark = tk.Label(card, bg=brand, bd=0) + image = None + try: + from ui import desktop # noqa: PLC0415 + + png = desktop.splash_png(96) + if png: + image = tk.PhotoImage(file=png) + mark.configure(image=image, width=96, height=96) + except Exception: # noqa: BLE001 - a wordmark is a fine degraded splash + image = None + if image is None: + # No Pillow, no PNG, or a Tk without the image reader. Deliberately not + # a second hand-drawn nabla -- see `desktop.splash_png`. + mark.configure(text="∇", fg=ink, font=("Segoe UI", 44, "bold"), width=3, height=1) + mark.pack(pady=(26, 14)) + + tk.Label( + card, text="GRAD", bg=paper, fg=ink, font=("Segoe UI", 15, "bold") + ).pack() + caption = tk.Label( + card, + text="starting the workspace…", + bg=paper, + fg=ink, + font=("Segoe UI", 9), + ) + caption.pack(pady=(6, 0)) + hint = tk.Label( + card, + text="click to send it behind · drag to move", + bg=paper, + fg=muted, + font=("Segoe UI", 8), + ) + hint.pack(pady=(2, 0)) + + state: dict[str, Any] = {"press": None, "moved": False, "backgrounded": False} + + def send_behind() -> None: + if state["backgrounded"]: + return + state["backgrounded"] = True + try: + root.attributes("-topmost", False) + root.lower() + except tk.TclError: + return + # Said, because a window that drops behind everything and still says + # "starting" is indistinguishable from one that gave up. + caption.configure(text="still starting — this closes itself") + hint.configure(text="") + + def on_press(event: Any) -> None: + state["press"] = (event.x_root, event.y_root, root.winfo_x(), root.winfo_y()) + state["moved"] = False + + def on_motion(event: Any) -> None: + press = state["press"] + if press is None: + return + dx, dy = event.x_root - press[0], event.y_root - press[1] + if abs(dx) > DRAG_SLOP_PX or abs(dy) > DRAG_SLOP_PX: + state["moved"] = True + if state["moved"]: + root.geometry(f"+{press[2] + dx}+{press[3] + dy}") + + def on_release(_: Any) -> None: + if state["press"] is not None and not state["moved"]: + send_behind() + state["press"] = None + + for widget in (root, card, mark, caption, hint): + widget.bind("", on_press) + widget.bind("", on_motion) + widget.bind("", on_release) + root.bind("", lambda _: send_behind()) + + deadline = time.monotonic() + max(1.0, float(timeout_s)) + + def tick() -> None: + if gone.is_set() or time.monotonic() >= deadline: + root.destroy() + return + root.after(POLL_MS, tick) + + root.after(POLL_MS, tick) + try: + root.mainloop() + except Exception: # noqa: BLE001 - a splash must never be the thing that fails + log.debug("the loading window stopped badly", exc_info=True) + return 0 + + +def main(argv: list[str] | None = None) -> int: + import argparse # noqa: PLC0415 + + parser = argparse.ArgumentParser( + prog="ui.splash", + description="The Grad mark, on screen while the workspace loads.", + ) + parser.add_argument("--timeout", type=float, default=MAX_SECONDS) + # Accepted and unused: the pipe is what liveness is actually read from, and + # a pid in the argument list is worth having when someone is looking at this + # process in a task manager wondering what started it. + parser.add_argument("--parent-pid", type=int, default=0) + parser.add_argument( + "--watch-stdin", + action="store_true", + help="close when the pipe from the launching process does", + ) + args = parser.parse_args(argv) + return show(timeout_s=args.timeout, watch_stdin=args.watch_stdin) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ui/state.py b/ui/state.py index bb9a842..b9c4420 100644 --- a/ui/state.py +++ b/ui/state.py @@ -69,7 +69,7 @@ def layout_path(project: str | None) -> Path: def load_layout(project: str | None) -> layout_mod.Layout: - """The saved arrangement, or the mock's opening one. + """The saved arrangement, or the one a workspace with no history opens with. Unknown window ids are dropped rather than raised on, so a layout written by a version with a window this one does not have still opens. @@ -78,7 +78,28 @@ def load_layout(project: str | None) -> layout_mod.Layout: restored = layout_mod.Layout.from_dict(data, known=registry.ids()) if restored.columns: return restored - return layout_mod.Layout.default(registry.defaults()) + return layout_mod.Layout.default(opening_windows()) + + +def opening_windows() -> tuple[str, ...]: + """What opens when this workspace has never been arranged. + + The mock's four -- unless the agent has no credentials, in which case those + four windows are four windows that cannot do anything, and the first thing + on screen should be the one that fixes it. + + Only reached when there is no saved layout, so this costs a credential-store + read once per fresh workspace and nothing thereafter. And only the *token* is + checked (`models.setup_needed`): an unconfigured backend means no remote + training, which is a real limitation and not a reason to put a wizard in + front of someone who opened the app to read a ledger. + """ + try: + if models.setup_needed(): + return ("setup", *registry.defaults()) + except Exception: # noqa: BLE001 - never the reason a workspace will not open + log.debug("could not decide the opening arrangement", exc_info=True) + return registry.defaults() def save_layout(project: str | None, value: layout_mod.Layout) -> None: @@ -94,6 +115,8 @@ def save_layout(project: str | None, value: layout_mod.Layout) -> None: # `chat` is absent on purpose: its state is the live SDK session, not a file, so # it redraws from its own stream rather than from the poll. MODEL_BUILDERS: dict[str, Callable[["Workspace"], Any]] = { + "setup": lambda w: models.setup_model(), + "projects": lambda w: models.projects_model(), "notebook": lambda w: models.notebook_model(), "ledger": lambda w: models.ledger_model(), "quota": lambda w: models.quota_model(), @@ -163,6 +186,14 @@ def __init__(self, session: Any, project: str | None) -> None: #: routing it through here keeps the gate card from having to reach into #: another window's closure. self.chat_send: Callable[[str], Any] | None = None + #: Wakes that have arrived and not yet been turned into a prompt. + #: + #: A queue rather than a direct call, because the two ends are in + #: different worlds: a wake arrives on a FastAPI route with no client in + #: scope, and `chat_send` draws into elements that belong to one. The + #: poll already runs in that client's context on a two-second tick, and + #: two seconds is nothing next to the hours a wake waits. + self.pending_wakes: list[str] = [] self._fingerprints: dict[str, str] = {} self._redraw: dict[str, Callable[[], None]] = {} #: Titlebar redraws, kept apart from the bodies: a titlebar reports live @@ -332,6 +363,7 @@ async def poll(self) -> None: try: if self._caught_up_after_move(): return + await self._deliver_wakes() # Snapshotted, because the rebuilds below await: a retile landing # mid-pass would otherwise have this iterating a list that changed # under it. A window that opened during the pass is picked up by the @@ -346,6 +378,60 @@ async def poll(self) -> None: finally: self._ticking = False + # -- wakeups ------------------------------------------------------------ + #: More than this many wakes waiting is a runaway watcher, not a research + #: session. They are dropped at the door with a line in the status bar + #: rather than queued into a conversation nobody asked for. + MAX_PENDING_WAKES = 8 + + def accept_wake(self, prompt: str) -> bool: + """Take a wake for delivery on the next tick. Returns whether it was kept. + + Called from the HTTP route, so it must do nothing that needs a client: + no elements, no drawing, no `say`. + """ + if not prompt.strip(): + return False + if len(self.pending_wakes) >= self.MAX_PENDING_WAKES: + log.warning("dropping a wake: %d already queued", len(self.pending_wakes)) + return False + self.pending_wakes.append(prompt) + return True + + async def _deliver_wakes(self) -> bool: + """Turn one queued wake into a turn, if now is a moment that can take it. + + One per tick and never while the session is busy. A wake is a prompt, and + `Session.ask` refuses a prompt during a turn -- so delivering into a + running turn would consume the wake and answer nothing, which is exactly + the silent loss this whole mechanism exists to prevent. Held instead, and + the next tick tries again. + """ + if not self.pending_wakes: + return False + if getattr(self.session, "busy", False): + return False + send = self.chat_send + if send is None: + # The chat window is closed, so there is nowhere for a turn to be + # drawn. Opening it is the honest response: something is trying to + # wake this session and the window that shows sessions is shut. + self.open("chat") + self.say("a wakeup arrived — opening the session window for it") + return False + + prompt = self.pending_wakes.pop(0) + self.set_agent_state("running") + try: + result = send(prompt) + if hasattr(result, "__await__"): + await result + except Exception: # noqa: BLE001 - a failed wake must not stop the poll + log.exception("could not deliver a wakeup") + self.set_agent_state("idle") + return False + return True + # -- layout moves ------------------------------------------------------- def toggle(self, window_id: str) -> None: self.layout.toggle(window_id) @@ -470,15 +556,51 @@ async def switch_root(self, folder: str, *, create: bool = False) -> None: self.reload() self.say(f"workspace: {chosen}") - async def create_project(self, project_id: str, title: str) -> None: + async def create_project( + self, + project_id: str, + title: str, + *, + ceilings: dict[str, str] | None = None, + payer: str = "", + ) -> None: """Create a project and select it, by running the same command the agent - would (§10) -- so it lands in the same ledger and reads back the same.""" - payload = await run_tool( - "tools.budget", "new", "--id", project_id, "--title", title, "--use", "--json" - ) + would (§10) -- so it lands in the same ledger and reads back the same. + + The ceilings go in *here*, on `budget new`, and not as a raise + afterwards. A raise appends a `project_budget_raised` event carrying a + `previous` value, so setting the first ceiling through one would record + that a project's GPU allowance moved from nothing to $50 -- which is not + what happened, and the history is the reason that module is append-only + at all. Creating with them is one record saying what the project was + allowed from the start. + + This is also the fix for the hole the old form left: it created with no + ceilings and put "set them below once it is selected" in a caption. A + project with no ceilings bounds nothing and every gate that reads one + passes silently. + """ + argv = ["tools.budget", "new", "--id", project_id, "--title", title, "--use"] + for flag, value in (ceilings or {}).items(): + if (value or "").strip(): + argv += [f"--{flag}", str(value).strip()] + if (payer or "").strip(): + argv += ["--payer", payer.strip()] + payload = await run_tool(*argv, "--json") self.say(envelope_message(payload)) - if payload.get("ok"): - self.reload() + if not payload.get("ok"): + return + self.reload() + # The machine half, and only when there is something in it left to + # answer. A wizard that opened on every project creation would ask a user + # with six projects for their Claude token six times; one that never + # opens leaves a fresh install with a project it cannot run. + try: + if models.setup_needed(): + self.open("setup") + self.say("no credentials yet — setup is open beside it") + except Exception: # noqa: BLE001 - never the reason a create is reported as failed + log.debug("could not decide whether to open setup", exc_info=True) async def use_project(self, project_id: str) -> None: payload = await run_tool("tools.budget", "use", project_id, "--json") @@ -486,6 +608,86 @@ async def use_project(self, project_id: str) -> None: if payload.get("ok"): self.reload() + async def configure_project( + self, + project_id: str, + *, + role: str = "", + model: str = "", + backend: str = "", + ) -> None: + """Set or clear one of this project's overrides. + + An empty `model` for a named role clears it, which is what the ✕ beside + each role sends -- `--clear` rather than an empty value, because an + override stored as empty resolves as falsy everywhere and is therefore + present, wrong and invisible. + + The rebuild this may need is not done here. `Session.apply_model` does it + lazily, immediately before the next turn, for the reason `apply_effort` + is lazy: dropping and respawning the SDK subprocess is seconds, and + paying it at the click would make idly comparing two projects cost more + than using either. + """ + argv = ["tools.budget", "configure", "--project", project_id] + # An explicit flag rather than counting argv. The length sentinel was + # correct only while the prefix stayed four words long, which is exactly + # the kind of thing a later flag breaks silently -- and the failure would + # be a command that runs with nothing to do and reports success. + changed = False + if role and model: + argv += [f"--{role}", model] + changed = True + elif role: + argv += ["--clear", role] + changed = True + if backend: + argv += ["--backend", backend] + changed = True + if not changed: + self.say("nothing to change — pick a model or a backend") + return + await self.run_and_reload(*argv, "--json") + + # -- setup --------------------------------------------------------------- + # Every one of these runs the same command a terminal would (§10), so the + # window cannot grow a second way to write a setting. They reload rather + # than invalidate: a model role or a host changes what `config.load()` + # answers, and that is read by every window and by the agent's own tools. + async def set_model(self, role: str, model: str) -> None: + if not (model or "").strip(): + self.say("no model given — paste an id or pick one of the buttons") + return + await self.run_and_reload("tools.setup", "models", f"--{role}", model.strip(), "--json") + + async def clear_model(self, role: str) -> None: + await self.run_and_reload("tools.setup", "models", "--clear", role, "--json") + + async def set_backend(self, name: str) -> None: + await self.run_and_reload("tools.setup", "backend", "--default", name, "--json") + + async def add_host(self, name: str, hostname: str, user: str, rate: str) -> None: + if not name or not hostname: + self.say("a host needs a name and a hostname") + return + await self.run_and_reload( + "tools.setup", "host", "add", + "--name", name, + "--hostname", hostname, + "--user", user, + "--rate", rate or "0", + "--json", + ) + + async def remove_host(self, name: str) -> None: + await self.run_and_reload("tools.setup", "host", "remove", "--name", name, "--json") + + async def set_kaggle_account(self, username: str) -> None: + if not (username or "").strip(): + self.say("no username given") + return + await self.run_and_reload("tools.kaggle", "account", "--set", username.strip(), "--json") + async def run_and_reload(self, *argv: str) -> None: """Run a CLI, report it, and re-read everything derived from it. @@ -501,6 +703,15 @@ async def run_and_reload(self, *argv: str) -> None: def workspaces(self) -> dict[str, Any]: return models.workspaces_model() + def projects(self) -> dict[str, Any]: + """A fresh read for the `project ▾` switcher. + + Not `model("projects")`: that is the poll's cached copy, and the window + it feeds may not even be open. A menu is redrawn on open precisely + because what it lists changes because of what it does. + """ + return models.projects_model() + def update(self) -> dict[str, Any]: return models.update_model() diff --git a/ui/static/tiling.js b/ui/static/tiling.js index 8608ce7..9220fff 100644 --- a/ui/static/tiling.js +++ b/ui/static/tiling.js @@ -437,8 +437,19 @@ * scrolling up to re-read a tool's output has to survive the next token, so * this pins only while the reader is already at the bottom. Server-side this * would be a `run_javascript` per flush, fifteen times a second. */ + + /* Every id this page has ever been asked to pin. It is a *set of ids* rather + * than a set of elements on purpose: the element behind an id does not + * survive a retile, and the id is the only handle that does. See `gradRearm`. */ + const stuck = new Set(); + window.gradStickBottom = (id) => { + stuck.add(id); const el = document.getElementById(id); + /* Two ways to do nothing, and they are different. No element: the window is + * closed, or the pane tree has not been patched in yet -- the next retile + * calls back through here and finds it. Already marked: this exact node is + * already observed, and a second observer on it would scroll it twice. */ if (!el || el.dataset.gradStuck) return; el.dataset.gradStuck = '1'; const SLACK_PX = 80; @@ -450,6 +461,27 @@ stick(); }; + /* Re-arm everything after the pane tree has been rebuilt. + * + * **This is what makes the transcript keep scrolling.** `gradStickBottom` was + * called once, from the chat window's render, and the marker plus the observer + * were the only record that it had been -- both of which live on the DOM node. + * A retile moves every window root through the attic with `Element.move()`, + * NiceGUI reparents server-side, and the client *re-creates* the node: new + * node, no marker, no observer, and the old observer left watching an orphan. + * Nothing re-ran the render, so nothing re-armed it. From the first time + * anyone opened, closed, focused or dragged a window, the transcript never + * scrolled itself again for the rest of the session. + * + * Cheap enough to call unconditionally: one `getElementById` and a dataset + * read per pinned id, on an event that happens when a human moves a window -- + * not on the flush, which is the thing the comment above refuses to pay for. + * + * An id whose element is genuinely gone (its window is closed) stays in the + * set and costs one lookup per retile. That is deliberate: the window can be + * reopened, and the set is the only thing that remembers it wanted pinning. */ + window.gradRearm = () => { stuck.forEach((id) => window.gradStickBottom(id)); }; + window.addEventListener('resize', reflowFrames); window.addEventListener('scroll', reflowFrames, true); diff --git a/ui/tokens.py b/ui/tokens.py index ff06148..0aaf985 100644 --- a/ui/tokens.py +++ b/ui/tokens.py @@ -286,6 +286,34 @@ def _shell() -> str: .grad-menu-row.disabled:hover { background: transparent; } .grad-menu-row.open.disabled { opacity: 1; } +/* The setup window's step row. A tab strip rather than a wizard's forward + march: every step stays clickable, because a setup that has to be restarted to + change the answer to question two is a setup people abandon at question + three. Sticky, so the steps do not scroll away underneath a long form. */ +.grad-steps { + display: flex; gap: 0; border-bottom: var(--grad-border); + position: sticky; top: 0; z-index: 2; background: var(--grad-paper); +} +.grad-step { + display: flex; flex-direction: column; align-items: flex-start; gap: 1px; + flex: 1 1 0; min-width: 0; padding: 8px 10px; cursor: pointer; + text-align: left; background: transparent; color: var(--grad-ink); + border: 0; border-right: var(--grad-border); + font: inherit; font-family: var(--grad-font-mono); font-size: 12px; +} +.grad-step:last-child { border-right: 0; } +.grad-step:hover { background: var(--grad-paper-sunk); } +.grad-step .mark { font-weight: 700; opacity: 0.55; } +.grad-step .name { + font-weight: 700; text-transform: uppercase; letter-spacing: 0.08em; +} +.grad-step .hint { + opacity: 0.55; font-size: 11px; width: 100%; + overflow: hidden; white-space: nowrap; text-overflow: ellipsis; +} +.grad-step.open { background: var(--grad-ink); color: var(--grad-paper); } +.grad-step.open .mark, .grad-step.open .hint { opacity: 0.7; } + .grad-statusbar { display: flex; align-items: center; gap: 14px; padding: 0 12px; background: var(--grad-ink); color: var(--grad-paper); diff --git a/ui/windows/__init__.py b/ui/windows/__init__.py index 43b8320..30d1139 100644 --- a/ui/windows/__init__.py +++ b/ui/windows/__init__.py @@ -1,4 +1,4 @@ -"""The twelve windows. +"""The fourteen windows. Each module here exposes `render(workspace)` and, optionally, `subtitle()` and `chips()` for its title bar. None of them import each other, and none of them @@ -21,8 +21,10 @@ "notebook", "papers", "preflight", + "projects", "quota", "queue", + "setup", "tasks", "wiki", ] diff --git a/ui/windows/chat.py b/ui/windows/chat.py index 155b223..3c2ec9c 100644 --- a/ui/windows/chat.py +++ b/ui/windows/chat.py @@ -272,8 +272,13 @@ async def poll_context() -> None: statusline.sync_context() ui.timer(CONTEXT_POLL_S, poll_context) - # Once, at build: keep the transcript pinned to the bottom while a turn - # streams. Doing it from here instead would be a `run_javascript` per flush. + # Keep the transcript pinned to the bottom while a turn streams. Doing it + # from the flush instead would be a `run_javascript` fifteen times a second. + # + # This is the *registration*, and it is no longer the only arming: the node + # behind this id is replaced whenever the pane tree is rebuilt, so + # `ui/shell.py:retile` re-arms through `gradRearm` afterwards. What this call + # does that the retile cannot is name the id in the window that owns it. kit.run_js("window.gradStickBottom && window.gradStickBottom('grad-transcript')") diff --git a/ui/windows/projects.py b/ui/windows/projects.py new file mode 100644 index 0000000..2ab5932 --- /dev/null +++ b/ui/windows/projects.py @@ -0,0 +1,409 @@ +"""Window 13 — projects: what the research is divided into, and what bounds it. + +A project is the unit three separate requirements turned out to share (§15): HF +payer attribution, the bound on an evolutionary campaign, and a budget for a +piece of research. Every run, every expectation and every report is keyed by +one. It was also, until this window, the only first-class concept in the app +with no window — it lived in a section of the `project ▾` dialog, between a +folder picker and a credentials panel. + +Two things the dialog could not do, and this exists for both: + +**Ceilings for a project you are not on.** The menu's raise controls addressed +the *selected* project only, so reading what bounded any other one meant +switching to it first — which charges nothing, and reloads every window in the +app to answer a question about a number. + +**Saying that a project bounds nothing.** A project with no ceilings passes +every gate that reads one, silently. `tools/budget.py` returns a warning saying +so at the moment of creation and nothing carried it any further; here it is a +chip on the row it is true of, for as long as it stays true. + +Closing is offered and deleting is not, because nothing here deletes: `close` +appends an event and the records stay (`core/budget.py`). +""" + +from __future__ import annotations + +from typing import Any + +from ui import kit +from ui.models import CEILINGS + + +def subtitle(workspace: Any) -> str: + model = workspace.model("projects") or {} + current = model.get("current_project") or "none selected" + return f"{model.get('open_count', 0)} open · on {current}" + + +def chips(workspace: Any) -> list[tuple[str, str]]: + model = workspace.model("projects") or {} + out: list[tuple[str, str]] = [] + unbounded = model.get("unbounded") or [] + if unbounded: + out.append((f"{len(unbounded)} UNBOUNDED", "attention")) + over = [r["id"] for r in model.get("rows") or [] if r.get("over_budget")] + if over: + out.append((f"{len(over)} OVER BUDGET", "broken")) + return out + + +def render(workspace: Any) -> None: + model = workspace.model("projects") or {} + kit.error_strip(model.get("error")) + + rows = model.get("rows") or [] + if not rows: + kit.empty( + "No projects in this folder yet.", + "python -m tools.budget new --id proj-scaling-w2 --title '...' --use --json", + ) + _new_project(workspace, model) + return + + for row in rows: + _project(workspace, model, row) + _new_project(workspace, model) + + +def _project(workspace: Any, model: dict[str, Any], row: dict[str, Any]) -> None: + """One project: who it is, what it has spent, and the two controls that + change either.""" + closed = row["status"] == "closed" + accent = "broken" if row.get("over_budget") else ("attention" if row["unbounded"] else "") + + with kit.el("div", "grad-card").style("margin: 9px"): + with kit.row(f"head {accent}".strip(), gap=9): + kit.text(row["id"], "", tag="span").style("font-weight: 700") + if row["current"]: + kit.chip("IN USE", "ok") + if closed: + kit.chip("CLOSED", "neutral") + if row["unbounded"] and not closed: + kit.chip("UNBOUNDED", "attention") + for resource in row.get("over_budget") or []: + kit.chip(f"OVER {resource.replace('_', ' ').upper()}", "broken") + kit.spacer() + kit.text(row["created"], "grad-caption", tag="span") + + with kit.el("div", "body"): + kit.error_strip(row.get("error")) + if row["title"]: + kit.text(row["title"], "").style("font-size: 13.5px; margin-bottom: 8px") + + kit.kv( + [ + ("payer", row["payer"] or "—"), + ("runs", row["run_count"]), + ("ceiling raises", row["raise_count"] or "none"), + ("memory", _memory_line(row["memory"])), + ] + ) + + _ceilings(workspace, row) + _models(workspace, model, row) + + with kit.row("", gap=6).style("margin-top: 10px"): + kit.button( + "IN USE" if row["current"] else "USE", + tone="active" if row["current"] else "primary", + disabled=row["current"] or closed, + title=( + "closed projects cannot be selected" + if closed + else "charge runs and tokens to this project" + ), + on_click=lambda _=None, pid=row["id"]: workspace.spawn( + workspace.use_project(pid), "project switch" + ), + ) + kit.spacer() + kit.button( + "CLOSE", + tone="neutral", + disabled=closed, + title="append a close event — nothing is deleted, and the records stay readable", + on_click=lambda _=None, pid=row["id"]: workspace.spawn( + workspace.run_and_reload("tools.budget", "close", pid, "--json"), + "project close", + ), + ) + + +def _memory_line(memory: dict[str, Any]) -> str: + """"Scaffolded but empty" and "never scaffolded" are different answers.""" + if memory.get("error"): + return "unreadable" + if not memory.get("scaffolded"): + return "not scaffolded — python -m tools.project init" + missing = memory.get("missing") or [] + present = len(memory.get("present") or []) + return f"{present} file(s)" + (f" · {len(missing)} missing" if missing else "") + + +def _ceilings(workspace: Any, row: dict[str, Any]) -> None: + """The three ceilings, and the one control that moves them. + + Drawn for every project rather than only the selected one, which is the + reason this window exists. A raise is a logged event and not a setting + (`core/budget.py`), so the reason field is offered here — the CLI defaults it + to empty and this does not force one, but a ceiling that moved without a + reason is unarguable with six months later. + """ + from nicegui import ui + + kit.label("ceilings").style("margin-top: 10px") + fields: dict[str, Any] = {} + + for ceiling in row["ceilings"]: + with kit.row("", gap=9).style("margin: 5px 0"): + kit.text(ceiling["caption"], "grad-mono", tag="span").style("min-width: 84px") + if not ceiling["set"]: + # "unbounded" and "nothing spent" must not look the same in a + # meter, so an unset ceiling gets no bar at all. + kit.text(ceiling["label"], "grad-caption", tag="span").style("flex: 1 1 auto") + else: + kit.bar( + [(ceiling["fraction"] or 0.0, "broken" if ceiling["over"] else "ink", "")] + ).style("flex: 1 1 auto") + kit.text(ceiling["label"], "grad-mono", tag="span") + + field = ( + ui.input(placeholder=ceiling["caption"]) + .props("borderless dense") + .classes("field") + .style("flex: 0 0 110px; padding: 0 8px") + ) + field.props(f'title="{kit.attr(ceiling["hint"])}"') + fields[ceiling["flag"]] = field + + with kit.row("", gap=6).style("margin-top: 6px"): + reason = ( + ui.input(placeholder="why it moved — optional, and it ages badly without one") + .props("borderless dense") + .classes("field") + .style("flex: 1 1 auto; padding: 0 8px") + ) + + def raise_them(_=None, pid: str = row["id"]) -> None: + # `--project ` as a flag, not a positional: `tools.budget raise` + # takes it as a flag, and passing it positionally is what made this + # control dead on every click for a release. `tests/test_ui_argv.py` + # runs this argv through the real parser. + argv = ["tools.budget", "raise", "--project", pid] + base = len(argv) + for flag, field in fields.items(): + if (field.value or "").strip(): + argv += [f"--{flag}", str(field.value).strip()] + if len(argv) == base: + workspace.say("no ceiling given — fill one of the three fields") + return + if (reason.value or "").strip(): + argv += ["--reason", str(reason.value).strip()] + workspace.spawn(workspace.run_and_reload(*argv, "--json"), "ceiling raise") + + kit.button( + "RAISE", + tone="primary", + title="a logged event — leave a field blank to leave that ceiling alone", + on_click=raise_them, + ) + + +def _models(workspace: Any, model: dict[str, Any], row: dict[str, Any]) -> None: + """What this project overrides about how it is run. + + Collapsed until asked for, because six roles times every project is a window + nobody can read, and the common case is a project that overrides nothing. + The expanded project id lives in `workspace.selection` rather than in a + closure — this window is rebuilt whenever a run lands. + + Beside each role is what it *would* be without the override, and that is the + workspace's answer rather than this project's: the window draws every + project and only one of them is selected, so for all the others the project + layer in effect belongs to somebody else. + """ + from nicegui import ui + + expanded = workspace.selection.get("projects.models") == row["id"] + overrides = [m for m in row["models"] if m["override"]] + + with kit.row("", gap=6).style("margin-top: 10px"): + kit.label("models").style("min-width: 84px") + if not overrides: + kit.text("workspace defaults", "grad-caption", tag="span") + for entry in overrides: + kit.chip(f"{entry['role']} → {entry['override']}", "ok") + if row["backend"]: + kit.chip(f"backend {row['backend']}", "outline") + kit.spacer() + kit.button( + "HIDE" if expanded else "CHANGE", + tone="neutral", + on_click=lambda _=None, pid=row["id"]: workspace.select( + "projects.models", None if expanded else pid, window="projects" + ), + ) + + if not expanded: + return + + for entry in row["models"]: + with kit.row("", gap=6).style("margin: 4px 0"): + kit.text(entry["role"], "grad-mono", tag="span").style("min-width: 84px") + kit.text( + entry["effective"], + "grad-mono" if entry["override"] else "grad-caption", + tag="span", + ).style("min-width: 150px") + if not entry["override"]: + kit.text( + "from the workspace", "grad-caption", tag="span", + style="flex: 1 1 auto; min-width: 0", + ) + else: + kit.text( + f"workspace says {entry['workspace']}", "grad-caption", tag="span", + style="flex: 1 1 auto; min-width: 0", + ) + field = ( + ui.input(placeholder="model id") + .props("borderless dense") + .classes("field") + .style("flex: 0 0 170px; padding: 0 8px") + ) + def set_model( + _=None, pid: str = row["id"], role: str = entry["role"], f: Any = field + ) -> None: + # An empty field means "I clicked the wrong button", not "clear + # it": `configure_project` reads an empty model for a named role + # as `--clear`, so SET on a blank field silently dropped the + # override that ✕ is there to drop deliberately. + chosen = (f.value or "").strip() + if not chosen: + workspace.say("no model given — type an id, or use ✕ to drop the override") + return + workspace.spawn( + workspace.configure_project(pid, role=role, model=chosen), "project model" + ) + + kit.button("SET", tone="primary", on_click=set_model) + kit.button( + "✕", + tone="neutral", + disabled=not entry["override"], + title="drop the override — the role resolves as the workspace's does", + on_click=lambda _=None, pid=row["id"], r=entry["role"]: workspace.spawn( + workspace.configure_project(pid, role=r, model=""), "project model" + ), + ) + + with kit.row("", gap=6).style("margin-top: 6px; flex-wrap: wrap"): + kit.text("backend", "grad-caption", tag="span").style("min-width: 84px") + for backend in model.get("known_backends") or []: + kit.button( + backend, + tone="active" if backend == row["backend"] else "neutral", + disabled=backend == row["backend"], + on_click=lambda _=None, pid=row["id"], b=backend: workspace.spawn( + workspace.configure_project(pid, backend=b), "project backend" + ), + ) + kit.text( + "a preference, not a restriction — --remote still names one per campaign, and a spec's " + "[target] wins over both", + "grad-caption", + ) + + +def _new_project(workspace: Any, model: dict[str, Any]) -> None: + """Create, with the ceilings in the same form. + + This is the project half of setup, and it is the whole of it: everything + else a wizard would ask -- the token, the six model roles, which backends + exist -- is a fact about this machine, answered once in the setup window and + not re-asked per project. + + The ceilings are here rather than in a second step because of what the old + form did without them. It created with none and said "set them below once it + is selected" in a caption under the button, so the common path produced a + project that bounds nothing and every gate that reads a ceiling passed + silently on it. + """ + from nicegui import ui + + with kit.el("div", "grad-card").style("margin: 9px"): + kit.text("NEW PROJECT", "head") + with kit.el("div", "body"): + with kit.row("", gap=6): + project_id = ( + ui.input(placeholder="id, e.g. proj-scaling-w2") + .props("borderless dense") + .classes("field") + .style("flex: 0 0 220px; padding: 0 8px") + ) + title = ( + ui.input(placeholder="what this research is") + .props("borderless dense") + .classes("field") + .style("flex: 1 1 auto; padding: 0 8px") + ) + + kit.label("ceilings").style("margin-top: 10px") + fields: dict[str, Any] = {} + with kit.row("", gap=6): + for _resource, flag, caption_text, hint in CEILINGS: + field = ( + ui.input(placeholder=caption_text) + .props("borderless dense") + .classes("field") + .style("flex: 1 1 0; padding: 0 8px") + ) + field.props(f'title="{kit.attr(hint)}"') + fields[flag] = field + payer = ( + ui.input(placeholder="payer, e.g. hf:myorg") + .props("borderless dense") + .classes("field") + .style("flex: 1 1 0; padding: 0 8px") + ) + payer.props( + 'title="who pays. hf:<org> attributes HF jobs to that organisation"' + ) + + with kit.row("", gap=6).style("margin-top: 10px"): + kit.button( + "CREATE", + tone="primary", + on_click=lambda: workspace.spawn( + workspace.create_project( + project_id.value or "", + title.value or "", + ceilings={f: (i.value or "") for f, i in fields.items()}, + payer=payer.value or "", + ), + "project create", + ), + ) + kit.text( + "set here, they are what the project was allowed from the start — a raise " + "afterwards records a ceiling that moved, which is a different claim", + "grad-caption", + style="flex: 1 1 auto; min-width: 0", + ) + + if model.get("needs_setup"): + # The machine half, pointed at rather than repeated. It opens on + # its own after a create, too -- but a form that is about to + # produce an unrunnable project should say so before the click, + # not after it. + kit.note( + "This machine has no subscription credentials yet, so a new project will have " + "nothing to run. Setup asks for those once, not per project." + ) + kit.button( + "OPEN SETUP", + tone="neutral", + on_click=lambda: workspace.open("setup"), + ) diff --git a/ui/windows/setup.py b/ui/windows/setup.py new file mode 100644 index 0000000..1a5136e --- /dev/null +++ b/ui/windows/setup.py @@ -0,0 +1,476 @@ +"""Window 14 — setup: the questions a fresh machine has to answer. + +Four steps, and all four are about *this machine and this workspace*: the +subscription token, which model runs which role, where a training run executes, +and the optional keys that widen retrieval. None of them is about a project, +which is the whole reason this is a separate surface from the projects window -- +a wizard that asked for a Claude token every time someone created a project +would ask a user with six projects the same question six times. + +Two decisions worth knowing before changing anything here. + +**The step is a tab, not a stage.** Every step stays reachable and none of them +gates another. A setup that has to be restarted to change the answer to question +two is a setup people abandon at question three, and there is nothing here whose +answer depends on an earlier one. + +**The step index lives in the workspace, not in a closure.** A non-persistent +window is rebuilt whenever its model changes and its whole subtree is rebuilt on +every retile (`ui/static/tiling.js`), so a step held in a Python local is a step +that resets when a background poll notices a new run. `workspace.selection` is +the same mechanism the ledger's filter chips and the funnel's trace picker use. + +Nothing here writes a credential to a file or reads one back: `credential set` +takes the value down a pipe (`ui/state.py:set_credential`), and +`credentials.status()` returns booleans. +""" + +from __future__ import annotations + +from typing import Any + +from ui import kit + + +def subtitle(workspace: Any) -> str: + model = workspace.model("setup") or {} + steps = model.get("steps") or [] + done = len([s for s in steps if s["ready"]]) + return f"{done}/{len(steps)} answered" if steps else "nothing configured yet" + + +def chips(workspace: Any) -> list[tuple[str, str]]: + model = workspace.model("setup") or {} + if not (model.get("token") or {}).get("ready"): + return [("NOT AUTHENTICATED", "broken")] + if not model.get("complete"): + return [("NO BACKEND", "attention")] + return [] + + +def render(workspace: Any) -> None: + model = workspace.model("setup") or {} + kit.error_strip(model.get("error")) + + steps = model.get("steps") or [] + if not steps: + kit.empty("Setup could not read this machine's configuration.") + return + + active = workspace.selection.get("setup.step") or steps[0]["id"] + if active not in {s["id"] for s in steps}: + active = steps[0]["id"] + + kit.steps(steps, active, lambda step_id: workspace.select("setup.step", step_id)) + + # `.get`, not `[]`. A step added to `SETUP_STEPS` without a body here would + # otherwise raise a KeyError out of `render` and take the whole window down + # -- the one window whose job is to be usable when nothing else works. + body = { + "token": _token, + "models": _models, + "backends": _backends, + "extras": _extras, + }.get(active) + if body is None: + kit.empty(f"the {active} step has no body yet") + else: + body(workspace, model) + + _installation(workspace) + + +# --------------------------------------------------------------------------- +# 1. the subscription +# --------------------------------------------------------------------------- +def _token(workspace: Any, model: dict[str, Any]) -> None: + from nicegui import ui + + token = model["token"] + with kit.pad(): + kit.label("subscription token") + if token["state"] == "stored": + kit.chip("STORED", "ok") + kit.text( + "The agent's own loop, the funnel's Haiku stages and the mutation operator all " + "authenticate with this.", + "grad-caption", + ) + elif token["state"] == "environment": + kit.chip("ENVIRONMENT ONLY", "attention") + # The distinction that actually bites, and only on the installed app: + # a shell that exported the token has it, and the desktop shortcut + # launches from Explorer with whatever was made persistent. + kit.note( + "A token is set in this process's environment, so the agent works right now — but " + "it is not stored. Launched from the desktop shortcut, which inherits whatever " + "Explorer had, there would be nothing to authenticate with. Paste it below to " + "keep it." + ) + else: + kit.chip("MISSING", "broken") + kit.note( + "Nothing can reach a model without this. Everything else in this window is " + "optional by comparison." + ) + + kit.text("mint one in a terminal, then paste the result:", "grad-caption").style( + "margin-top: 10px" + ) + kit.pre(token["mint"]) + + with kit.row("", gap=6).style("margin-top: 8px"): + value = ( + ui.input(placeholder="paste the token") + .props("borderless dense type=password") + .classes("field") + .style("flex: 1 1 auto; padding: 0 8px") + ) + + def store() -> None: + pasted, value.value = value.value or "", "" + workspace.spawn( + workspace.set_credential(token["name"], pasted), "credential set" + ) + + kit.button("STORE", tone="primary", on_click=store) + kit.button( + "✕", + tone="neutral", + disabled=not token["stored"], + title="forget it", + on_click=lambda: workspace.spawn( + workspace.delete_credential(token["name"]), "credential delete" + ), + ) + kit.text( + "stored in the OS credential store, never in the workspace and never in the agent's " + "environment — it is fetched at the moment of use", + "grad-caption", + ) + + +# --------------------------------------------------------------------------- +# 2. the six roles +# --------------------------------------------------------------------------- +def _models(workspace: Any, model: dict[str, Any]) -> None: + from nicegui import ui + + with kit.pad(): + kit.label("one model per role") + kit.text( + "Chosen here, these outrank config/grad.toml — which is hand-annotated and cannot be " + "machine-written without losing every comment in it.", + "grad-caption", + ) + + for role in model["roles"]: + with kit.el("div", "grad-card").style("margin: 9px 0"): + with kit.row("head", gap=9): + kit.text(role["role"], "", tag="span").style("font-weight: 700") + kit.spacer() + kit.chip(role["source"], "ok" if role["overridden"] else "neutral") + with kit.el("div", "body"): + kit.text(role["model"], "grad-mono") + with kit.row("", gap=6).style("margin-top: 8px; flex-wrap: wrap"): + for known in model["known_models"]: + kit.button( + known, + tone="active" if known == role["model"] else "neutral", + disabled=known == role["model"], + on_click=lambda _=None, r=role["role"], m=known: workspace.spawn( + workspace.set_model(r, m), "model set" + ), + ) + with kit.row("", gap=6).style("margin-top: 6px"): + # The list above ages the moment a new model ships. This + # is the mechanism; the buttons are the shortcut. + other = ( + ui.input(placeholder="or any other model id") + .props("borderless dense") + .classes("field") + .style("flex: 1 1 auto; padding: 0 8px") + ) + kit.button( + "SET", + tone="primary", + on_click=lambda _=None, r=role["role"], f=other: workspace.spawn( + workspace.set_model(r, (f.value or "").strip()), "model set" + ), + ) + kit.button( + "RESET", + tone="neutral", + disabled=not role["overridden"], + title=f"fall back to the config, then to {role['default']}", + on_click=lambda _=None, r=role["role"]: workspace.spawn( + workspace.clear_model(r), "model reset" + ), + ) + + _shadowing(model) + + +def _shadowing(model: dict[str, Any]) -> None: + """What these choices are overriding, said out loud. + + The price of being allowed to win. Someone edits `[models] evolve`, sees no + change, and has no way to discover that a file they have never heard of + outranks the one they were told to edit -- unless it is written here. + """ + rows = model.get("shadowing") or [] + if not rows: + return + kit.note( + "These override values set in " + + model.get("config_path", "config/grad.toml") + + ", which still says something different:" + ) + kit.kv([(row["what"], f"{row['config']} → {row['overlay']}") for row in rows]) + + +# --------------------------------------------------------------------------- +# 3. where a run executes +# --------------------------------------------------------------------------- +#: What each backend is, in the one line that decides whether to bother with it. +BACKEND_NOTES = { + "kaggle": "free GPU/TPU hours, rationed in hours rather than dollars", + "hf_jobs": "Hugging Face Jobs, priced per hour against the GPU ceiling", + "ssh": "your own machines, priced by the rate you record for each", +} + + +def _backends(workspace: Any, model: dict[str, Any]) -> None: + with kit.pad(): + kit.label("where a run executes") + kit.text( + "Not alternatives — the useful arrangement is a mixture, and the default below is a " + "preference rather than a restriction. --remote still names one per campaign.", + "grad-caption", + ) + + for backend in model["backends"]: + name = backend["backend"] + with kit.el("div", "grad-card").style("margin: 9px 0"): + with kit.row("head " + ("" if backend["ready"] else "attention"), gap=9): + kit.text(name, "", tag="span").style("font-weight: 700") + kit.chip("READY" if backend["ready"] else "NOT CONFIGURED", + "ok" if backend["ready"] else "attention") + kit.spacer() + if model["default_backend"] == name: + kit.chip("DEFAULT", "ok") + else: + kit.button( + "MAKE DEFAULT", + tone="neutral", + on_click=lambda _=None, b=name: workspace.spawn( + workspace.set_backend(b), "backend default" + ), + ) + with kit.el("div", "body"): + kit.text(BACKEND_NOTES.get(name, ""), "grad-caption") + if backend["missing"]: + kit.text( + "missing: " + ", ".join(backend["missing"]), "grad-caption" + ).style("margin-top: 6px") + # Named explicitly rather than falling through to `_hosts`. + # A fourth backend would otherwise be handed an SSH host + # editor, which is not merely useless -- it invites someone + # to add an inventory entry that backend will never read. + if name == "kaggle": + _kaggle(workspace, model) + elif name == "hf_jobs": + _credential_field(workspace, "hf_token") + elif name == "ssh": + _hosts(workspace, model) + + +def _kaggle(workspace: Any, model: dict[str, Any]) -> None: + """Two halves, and only one of them is a secret. + + The username is stored where it can be read back, because "whose kernels are + these?" deserves a file you can open. The key goes to the credential store. + """ + from nicegui import ui + + account = model.get("kaggle") or {} + kit.kv([("username", account.get("username") or "—"), ("from", account.get("source") or "—")]) + with kit.row("", gap=6).style("margin-top: 6px"): + username = ( + ui.input(placeholder="your kaggle username") + .props("borderless dense") + .classes("field") + .style("flex: 1 1 auto; padding: 0 8px") + ) + kit.button( + "SET", + tone="primary", + on_click=lambda _=None, f=username: workspace.spawn( + workspace.set_kaggle_account((f.value or "").strip()), "kaggle account" + ), + ) + _credential_field(workspace, "kaggle_key") + + +def _hosts(workspace: Any, model: dict[str, Any]) -> None: + """The inventory is fixed by design; this is its writable half. + + A host that can be named ad-hoc is a general remote-execution capability, so + one has to be added on purpose before anything can reach it. Adding it here + rather than by editing TOML does not change that. + """ + from nicegui import ui + + for name in model.get("hosts") or []: + with kit.row("grad-row", gap=6): + kit.chip(name, "outline") + kit.spacer() + kit.button( + "✕", + tone="neutral", + title="remove it from the inventory added here", + on_click=lambda _=None, h=name: workspace.spawn( + workspace.remove_host(h), "host remove" + ), + ) + + fields: dict[str, Any] = {} + with kit.row("", gap=6).style("margin-top: 6px; flex-wrap: wrap"): + for key, placeholder, width in ( + ("name", "name, e.g. gpu-box", 140), + ("hostname", "hostname ssh connects to", 200), + ("user", "ssh user", 110), + ("rate", "$/hour (0 if free)", 120), + ): + fields[key] = ( + ui.input(placeholder=placeholder) + .props("borderless dense") + .classes("field") + .style(f"flex: 0 0 {width}px; padding: 0 8px") + ) + kit.button( + "ADD HOST", + tone="primary", + on_click=lambda: workspace.spawn( + workspace.add_host( + (fields["name"].value or "").strip(), + (fields["hostname"].value or "").strip(), + (fields["user"].value or "").strip(), + (fields["rate"].value or "0").strip(), + ), + "host add", + ), + ) + kit.text( + "the rate is what `collect` prices wall clock against — a wrong one is a spend-accounting " + "problem, and 0 is correct for a machine you already pay for", + "grad-caption", + ) + + +# --------------------------------------------------------------------------- +# 4. the optional keys +# --------------------------------------------------------------------------- +def _extras(workspace: Any, model: dict[str, Any]) -> None: + rows = [ + r for r in model["credentials"]["rows"] if r["group"] in ("retrieval", "extras") + ] + with kit.pad(): + kit.label("optional keys") + kit.text( + "Everything here degrades rather than fails. Retrieval works without any of them; " + "each one widens or speeds up a stage.", + "grad-caption", + ) + kit.error_strip(model["credentials"].get("error")) + for row in rows: + with kit.column("grad-row", gap=6): + with kit.row("", gap=6).style("width: 100%"): + kit.chip(row["state"], row["tone"]) + kit.text(row["name"], "grad-mono", tag="span") + kit.text( + row["purpose"], "grad-caption", tag="span", + style="flex: 1 1 auto; min-width: 0", + ) + _credential_field(workspace, row["name"]) + + +def _credential_field(workspace: Any, name: str) -> None: + """One paste-to-store row. The value goes down a pipe, never in an argv.""" + from nicegui import ui + + with kit.row("", gap=6).style("width: 100%; margin-top: 6px"): + value = ( + ui.input(placeholder=f"paste {name}") + .props("borderless dense type=password") + .classes("field") + .style("flex: 1 1 auto; padding: 0 8px") + ) + + def store(_=None) -> None: + pasted, value.value = value.value or "", "" + workspace.spawn(workspace.set_credential(name, pasted), "credential set") + + kit.button("SET", tone="neutral", on_click=store) + kit.button( + "✕", + tone="neutral", + title="forget it", + on_click=lambda _=None: workspace.spawn( + workspace.delete_credential(name), "credential delete" + ), + ) + + +# --------------------------------------------------------------------------- +# the footer +# --------------------------------------------------------------------------- +def _installation(workspace: Any) -> None: + """Which Grad this is, and the one button that changes it. + + Not a step -- it is never *answered* -- but it belongs on this surface for + the same reason the credentials do: it is a fact about the installation, and + it used to live behind a control labelled `project`. + """ + model = workspace.update() + + kit.hr() + with kit.pad(): + kit.label("this installation") + kit.kv([("version", model["installed"]), ("last checked", model["checked"])]) + if not model["is_checkout"]: + kit.note( + "This copy was not installed from a git checkout, so it cannot update itself. " + "Reinstall from the repository to get updates." + ) + return + for warning in model["warnings"]: + kit.note(f"{warning['message']} — {warning['fix']}") + for blocker in model["blockers"]: + kit.error_strip(f"{blocker['message']} — {blocker['fix']}") + with kit.row("", gap=6).style("margin-top: 8px"): + if model["available"]: + kit.chip(f"{model['target']} AVAILABLE", "attention") + kit.button( + "UPDATE", + tone="primary", + title=( + "quit first: this release changes dependencies" + if model["needs_reinstall"] + else "fast-forward this installation and migrate its state" + ), + on_click=lambda: workspace.spawn(workspace.apply_update(), "update"), + ) + kit.button( + "CHECK NOW", + tone="neutral", + title="ask the remote whether there is a newer release", + on_click=lambda: workspace.spawn(workspace.check_update(), "update check"), + ) + kit.spacer() + if model["dirty"]: + kit.text( + "the installation has uncommitted edits; runs submitted from it are stamped " + "as modified and `report check` will say so", + "grad-caption", + )