diff --git a/.github/workflows/batch-check.yml b/.github/workflows/batch-check.yml index 04f16fdf..3ba4a984 100644 --- a/.github/workflows/batch-check.yml +++ b/.github/workflows/batch-check.yml @@ -768,6 +768,32 @@ jobs: run: | & tests\selfapps_cascade.ps1 + # CLAUDE.md Active Backlog Item 23: a genuine conda-create failure during a REQ-009 cascade + # re-entry previously fell through :die instead of gracefully keeping the previous working + # uv build. Must run AFTER the step directly above (selfapps_cascade.ps1) so Miniconda is + # already installed/cached from that step -- otherwise :cascade_acquire_conda would need a + # real Miniconda download of its own, same CI-ordering reasoning as + # selfapps_conda_bothfail.ps1's own placement note, just the opposite direction. + - name: "Self-test: cascade-reentry conda-create failure keeps previous build (uv lane, non-gating)" + if: ${{ matrix.mode == 'uv' }} + continue-on-error: true + shell: pwsh + run: | + & tests\selfapps_cascade_conda_create_fail.ps1 + + # PR #413 CodeRabbit review finding: the scenario above only exercises the fix's + # :conda_create_failed call site (create itself fails); this exercises the OTHER call site + # inside :conda_create_done (create genuinely succeeds but python.exe is missing + # afterward). Same placement requirement as the step above (Miniconda already cached). + - name: "Self-test: cascade-reentry conda-create missing python.exe keeps previous build (uv lane, non-gating)" + if: ${{ matrix.mode == 'uv' }} + continue-on-error: true + shell: pwsh + env: + CASCADE_CCF_SCENARIO: missing_python + run: | + & tests\selfapps_cascade_conda_create_fail.ps1 + # docs/agent-closed-backlog.md's Item 22: a real, non-simulated end-to-end test proving the # uv-to-conda cascade, warnfix repair (both success and failure in the same round), and # --hidden-import auto-recovery all fire for real in ONE run -- no HP_TEST_FORCE_*/ diff --git a/CLAUDE.md b/CLAUDE.md index c548dde4..d0b95513 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -523,57 +523,6 @@ Once an item is fully resolved it is removed from here entirely and archived (ke original number) in `docs/agent-closed-backlog.md`, which is why the numbering below does not start at 1 and has gaps. -- **Item 23: a genuine (non-test) conda-create failure during a REQ-009 cascade re-entry does - not restore the previous working build via `HP_CASCADE_SAVED_PY`, unlike every other - cascade-target failure.** Found via a CodeRabbit review finding on PR #412, verified by - reading the source directly (not taken on faith; re-verified again while fixing this entry's - own citations, which corrected the mechanism description below). `:try_conda_create`'s own - failure label (`:conda_create_failed`) does NOT hard-fail immediately -- it first calls - `:handle_conda_failure`, the same linear embed/venv/system fallback chain the ORIGINAL - (non-cascade) conda-create failure path already relies on; if any of those tiers succeeds, - `HP_ENV_READY` is set and control correctly `goto :after_env_mode_selection`. Only when - `:handle_conda_failure` ALSO exhausts every tier does `:conda_create_failed` fall through to - `call :die`, and only then does the actual bug surface: `:die` returns via `exit /b` - (subroutine return, not a process halt; see `docs/agent-lessons-learned.md`'s `:die` entry), - so execution falls straight through past it into `:conda_create_done`, which sets - `HP_PY=%CONDA_PREFIX%\python.exe` and checks `if not exist "%HP_PY%"` -- true in this case, so - it retries the identical `:handle_conda_failure` chain a second time (redundant, since nothing - changed) before a second `call :die`, after which execution again falls through, now carrying - a genuinely broken `HP_PY` into whatever code follows. Neither fall-through ever routes through - `:after_cascade_decision` (the label every OTHER cascade-target failure -- - `:cascade_conda_unavailable`, `:cascade_embed_unavailable`, `:cascade_venv_unavailable`, - `:cascade_system_unavailable` -- correctly uses, logging `[WARN] ... unavailable; keeping - current build.` and restoring `HP_CASCADE_SAVED_PY` into `HP_PY` so the bootstrap gracefully - continues on the PREVIOUS successful build, e.g. uv, if cascading uv-to-conda). `:try_conda_ - create`'s failure handling has no cascade-context-awareness at all -- it behaves identically - whether this is the very first creation attempt (where there is no earlier build to restore, - so eventually hard-failing is correct) or a `:cascade_from_uv` re-entry (where `HP_CASCADE_ - SAVED_PY` holds a known-working uv build that never gets restored). - **Practical impact, tempered but real:** `:die` already sets `HP_BOOTSTRAP_STATE=error` - unconditionally as of an earlier fix (see `docs/agent-lessons-learned.md`), so the FINAL - `~bootstrap.status.json` should still correctly read `state=error` rather than falsely - claiming success -- this is not the same severity as the empty-interpreter-command bug the - prior commit fixed. But a genuine, plausibly-transient conda-create failure during a cascade - (e.g. a real network blip while acquiring Miniconda on demand for the cascade, distinct from - "Miniconda not installed at all" which `:cascade_conda_unavailable` already handles gracefully) - now hard-fails the WHOLE bootstrap instead of gracefully keeping the already-working uv build, - and burns through the doubled `:handle_conda_failure` retry plus whatever broken-`HP_PY` log - noise follows before a terminal point is reached. - **Deliberately not fixed in the same commit as the Item 22 companion fix** -- properly fixing - this needs `:try_conda_create`'s failure branches to become cascade-context-aware (e.g. check - `if defined HP_CASCADE_APPROVED` or an equivalent re-entry signal and route to - `:after_cascade_decision`'s "keeping current build" pattern instead of `:die`, but ONLY when - there is a genuine earlier build to fall back to -- the first-attempt case must keep failing - hard), plus a new regression test that forces a genuine (not `HP_TEST_FORCE_CONDA_FAIL`-style - simulated) conda-create failure specifically during a cascade re-entry, which does not - currently exist. This is real design work, not a quick fix -- scoping it into the same commit - as the cStringIO warnfix-filter fix would have risked a rushed, undertested change to - already-sensitive cascade logic. `:hp_test_conda_fail` (the existing `HP_TEST_FORCE_CONDA_FAIL` - test hook) has the identical fallthrough shape and reaches the same `:after_env_mode_selection` - clear, but is scoped to the FIRST-attempt path only (its only call site is the top of - `:try_conda_create`, before any cascade re-entry could reach it) -- it is NOT itself evidence - this gap is already covered by existing tests. - - **Item 24: PyInstaller does not bundle `pygrib`'s native `eccodes.dll` dependency under the conda provider, so the frozen EXE fails at runtime even though the build itself succeeds.** Found via `self.layered_e2e.chain`'s real CI evidence (run `30875520181`, cache-lane job diff --git a/docs/agent-closed-backlog.md b/docs/agent-closed-backlog.md index 84023fe0..8d5bb188 100644 --- a/docs/agent-closed-backlog.md +++ b/docs/agent-closed-backlog.md @@ -1160,6 +1160,75 @@ this belongs to). and non-gating (`cache` lane, `continue-on-error`) in the interim, so no CI lane that gates PR merges is affected. +### Item 23 (closed 2026-08-04) + +- **A genuine (non-test) conda-create failure during a REQ-009 cascade re-entry did not restore + the previous working build via `HP_CASCADE_SAVED_PY`, unlike every other cascade-target + failure.** Found via a CodeRabbit review finding on PR #412, verified by reading the source + directly. `:try_conda_create`'s own failure label (`:conda_create_failed`) does NOT hard-fail + immediately -- it first calls `:handle_conda_failure`, the same linear embed/venv/system + fallback chain the ORIGINAL (non-cascade) conda-create failure path already relies on; if any + of those tiers succeeds, `HP_ENV_READY` is set and control correctly `goto + :after_env_mode_selection`. Only when `:handle_conda_failure` ALSO exhausts every tier does + `:conda_create_failed` fall through to `call :die`, and only then did the actual bug surface: + `:die` returns via `exit /b` (subroutine return, not a process halt; see + `docs/agent-lessons-learned.md`'s `:die` entry), so execution fell straight through past it + into `:conda_create_done`, which sets `HP_PY=%CONDA_PREFIX%\python.exe` and checks `if not + exist "%HP_PY%"` -- true in this case, so it retried the identical `:handle_conda_failure` + chain a second time (redundant, since nothing changed) before a second `call :die`, after which + execution again fell through, now carrying a genuinely broken `HP_PY` into whatever code + followed. Neither fall-through ever routed through `:after_cascade_decision` (the label every + OTHER cascade-target failure -- `:cascade_conda_unavailable`, `:cascade_embed_unavailable`, + `:cascade_venv_unavailable`, `:cascade_system_unavailable` -- correctly uses, logging `[WARN] + ... unavailable; keeping current build.` and restoring `HP_CASCADE_SAVED_PY` into `HP_PY` so + the bootstrap gracefully continues on the PREVIOUS successful build, e.g. uv, if cascading + uv-to-conda). `:try_conda_create`'s failure handling had no cascade-context-awareness at all -- + it behaved identically whether this was the very first creation attempt (where there is no + earlier build to restore, so eventually hard-failing is correct) or a `:cascade_from_uv` + re-entry (where `HP_CASCADE_SAVED_PY` holds a known-working uv build that never got restored). + **Fixed** by inserting `if defined HP_CASCADE_SAVED_PY goto :cascade_conda_create_failed` + immediately before both `call :die` fall-through sites (`:conda_create_failed`'s own line and + the companion `python.exe`-missing check inside `:conda_create_done`), and adding a new + `:cascade_conda_create_failed` label mirroring the existing sibling template exactly -- logs + `[WARN] REQ-009: cascade target conda create failed; keeping current build.` and `goto + :after_cascade_decision`, deliberately never calling `:die` (that label's own restore logic + only works correctly when `HP_BOOTSTRAP_STATE` is left as whatever it already was -- the prior + successful build's `ok` -- not overwritten to `error`). `HP_CASCADE_SAVED_PY`-definedness is a + safe signal to distinguish "first attempt" (never defined) from "cascade re-entry" (always + defined by `:provider_cascade` before dispatching to `:cascade_from_uv`, the only cascade + source that can reach `:try_conda_create`). + **New regression test, forcing a GENUINE (not `HP_TEST_FORCE_CONDA_FAIL`-style simulated) + failure through the real create/retry code path** -- `tests/selfapps_cascade_conda_create_fail. + ps1` (uv lane, non-gating, `self.cascade.conda_create_fail`; see `docs/agent-ndjson.md` for the + full assertion list). New hook `HP_TEST_FORCE_CONDA_CREATE_BOTH_FAIL=1` (distinct from the + existing `HP_TEST_FORCE_CONDA_CREATE_NETWORK_FAIL`, which only fails the first attempt then + clears itself so the retry can genuinely succeed) persists through both the initial attempt and + the retry, so `:conda_create_failed` is reached deterministically without depending on real + network conditions. Combined with `HP_TEST_FORCE_EMBED_FAIL=1`/`HP_TEST_FORCE_VENV_FAIL=1`/ + `HP_TEST_SYSCON_ANSWER=N` to exhaust `:handle_conda_failure`'s own fallback chain so the fix's + new check is actually reached. Placement is load-bearing: wired into `batch-check.yml` + immediately AFTER `selfapps_cascade.ps1`'s own step so Miniconda is already cached from that + step (mirrors `self.conda.bothfail`'s own placement note, opposite direction). `:hp_test_conda_ + fail` (the existing `HP_TEST_FORCE_CONDA_FAIL` test hook) has the identical fallthrough shape + and reaches the same `:after_env_mode_selection` clear, but is scoped to the FIRST-attempt path + only -- it was never evidence this gap was already covered by existing tests. + **Follow-up, same PR (#413), from a CodeRabbit review finding**: the original regression test + above only exercised `:conda_create_failed`'s own call site (create itself genuinely fails). The + fix's OTHER call site -- the `if not exist "%HP_PY%"` check inside `:conda_create_done`, reached + when conda create genuinely SUCCEEDS but the resulting environment is somehow missing + `python.exe` -- had zero coverage. Closed by adding a `missing_python` scenario + (`CASCADE_CCF_SCENARIO` env var) to the SAME test file, via a new hook + `HP_TEST_FORCE_CONDA_MISSING_PYTHON=1` (`run_setup.bat`, `:conda_create_done`) that lets the real + create succeed and then deletes the `python.exe` it just produced -- genuine success followed by + a genuinely missing interpreter, not a simulated create failure. Wired as a second CI step + (`CASCADE_CCF_SCENARIO: missing_python`) immediately after the original, same placement + constraint (Miniconda already cached from `selfapps_cascade.ps1`). See `docs/agent-ndjson.md`'s + updated `self.cascade.conda_create_fail` entry for the full two-scenario assertion list, + including the message-occurrence-COUNT technique used to distinguish "handle_conda_failure + logged the failure once, as expected" from "the old :die fall-through regression is back" + (both scenarios' own failure messages are logged once by `:handle_conda_failure` regardless of + outcome; a second occurrence would come only from `:die`'s own separate echo). + ### Item 13 (closed 2026-08-01) - **`self.warn.longpath`'s own real CI run showed an INCONCLUSIVE result (`ranBootstrap:false`), diff --git a/docs/agent-cold-storage.md b/docs/agent-cold-storage.md index 9bc30628..9a2128d2 100644 --- a/docs/agent-cold-storage.md +++ b/docs/agent-cold-storage.md @@ -114,3 +114,26 @@ its own named trigger genuinely fires -- do not speculatively build any of these was correctly ruled out elsewhere (its wheel bundles an ~40 MB SQLite package database, a non-starter for a single-file bootstrapper). +- **Conda native-DLL bundling repair loop (CLAUDE.md Active Backlog Item 24 -- pygrib/eccodes and + the general case).** Full PRD at `docs/prd-conda-native-dll-bundling.md`: a frozen EXE built + under the conda provider can fail at runtime with `ImportError: DLL load failed` when + PyInstaller doesn't discover/bundle a native DLL a conda-forge package depends on (confirmed via + real CI evidence for `pygrib`'s `eccodes.dll` dependency, `self.layered_e2e.chain`). The PRD + lays out a reactive, bounded repair loop mirroring `:hidden_import_recover`'s proven shape (see + `docs/agent-lessons-learned.md`'s "--hidden-import auto-recovery must stay STRICT" entry) -- + scan for the DLL-load-failure signature (or PyInstaller's own earlier build-time warning), + locate the missing DLL under `%CONDA_PREFIX%\Library\bin`, bundle via `--add-binary`, rebuild, + and iterate to catch transitive native deps one at a time. Not pursued now: the single highest- + leverage next step (verify whether `pyinstaller-hooks-contrib`'s existing `hook-gribapi.py` + already solves this for free via a plain `--hidden-import=gribapi` addition -- Finding 1 in the + PRD) is unverified and was not resolvable in the sandbox this PRD was drafted in (no access to + fetch `pyinstaller-hooks-contrib`'s source). Building any new mechanism before that verification + risks duplicating work an existing upstream hook may already cover. + **Trigger to thaw**: the owner explicitly brings this forward as the next active work item (per + their own stated intent when requesting this PRD: "I think I will bring it forward soon if level + check passes and the current todo is fully closed out") -- at that point, start with the PRD's + own Requirement 1 (the zero-new-code `--hidden-import=gribapi` verification), not with building + the repair loop directly. The PRD's own Open Questions section (narrow pygrib-only fix vs. a + general conda-native-DLL pattern-matcher) also needs a maintainer call before implementation + proceeds past that first verification step -- see `docs/open-questions.md`. + diff --git a/docs/agent-ndjson.md b/docs/agent-ndjson.md index 9add3f60..57d95b35 100644 --- a/docs/agent-ndjson.md +++ b/docs/agent-ndjson.md @@ -48,6 +48,7 @@ self.exe.hidden_import, self.exe.hidden_import.exhaust, self.preflight.syntax, self.cascade.detect, self.cascade.consent, self.cascade.timed, self.cascade.exec (uv lane only -- selfapps_cascade.ps1; non-gating), +self.cascade.conda_create_fail (uv lane only -- selfapps_cascade_conda_create_fail.ps1; non-gating), self.layered_e2e.chain (cache lane only -- selfapps_layered_e2e.ps1; non-gating), self.conda.bothfail (uv lane only -- selfapps_conda_bothfail.ps1; non-gating), self.exe.build.tiera (uv lane only -- selfapps_nuitka_tiera.ps1; non-gating), @@ -476,6 +477,78 @@ CI-ordering assumption above needs to soak before considering gating-lane promot self.conda.bothfail ``` +## selfapps-cascade-conda-create-fail NDJSON rows (selfapps_cascade_conda_create_fail.ps1, uv lane only, non-gating) + +Closes CLAUDE.md Active Backlog Item 23: a genuine (non-`HP_TEST_FORCE_CONDA_FAIL`-simulated) +conda-create failure reached during a REQ-009 cascade re-entry (`:cascade_from_uv -> +:try_conda_create`) previously fell through `:die` into `:conda_create_done`'s own success-path +continuation instead of routing through `:after_cascade_decision` like every other cascade-target +failure -- see `docs/agent-interconnect.md`'s "Provider cascade execution re-enters env-create" +and `CLAUDE.md`'s Item 23 entry for the full trace. Fixed by adding `if defined +HP_CASCADE_SAVED_PY goto :cascade_conda_create_failed` at both `:conda_create_failed` fall-through +sites, plus a new `:cascade_conda_create_failed` label mirroring the existing +`:cascade_conda_unavailable` / `:cascade_embed_unavailable` / `:cascade_venv_unavailable` / +`:cascade_system_unavailable` template (log a `[WARN] ... keeping current build.` line, restore +`HP_CASCADE_SAVED_PY` via `:after_cascade_decision`, deliberately never call `:die`). + +Two scenarios (`CASCADE_CCF_SCENARIO` env var; unset defaults to `create_fails`), covering the +fix's two distinct call sites -- both fall through the SAME `if defined HP_CASCADE_SAVED_PY goto +:cascade_conda_create_failed` pattern, but are reached via genuinely different real-code-path +triggers: + +- **`create_fails`** (default): new hook `HP_TEST_FORCE_CONDA_CREATE_BOTH_FAIL=1` + (`run_setup.bat`, `:try_conda_create`) forces a GENUINE failure through the real create/retry + code path -- unlike the existing `HP_TEST_FORCE_CONDA_CREATE_NETWORK_FAIL` (fails the first + attempt only, then clears itself so the retry can genuinely succeed), this flag persists through + the retry too, so both the initial attempt and the retry fail deterministically without + depending on real network conditions. Exercises `:conda_create_failed`'s own call site. +- **`missing_python`** (PR #413 CodeRabbit review finding -- the original scenario above never + exercised the fix's OTHER call site): new hook `HP_TEST_FORCE_CONDA_MISSING_PYTHON=1` + (`run_setup.bat`, `:conda_create_done`) lets the real `conda create` command run and succeed for + real, then deletes the `python.exe` it just produced -- a genuine successful create followed by + a genuinely missing interpreter, not a simulated create failure. `HP_TEST_FORCE_CONDA_CREATE_ + BOTH_FAIL` is deliberately unset in this scenario (create must succeed for real to reach + `:conda_create_done` at all). Exercises the `if not exist "%HP_PY%"` block's own call site + inside `:conda_create_done`. + +Both scenarios: `HP_TEST_FORCE_EMBED_FAIL=1`, `HP_TEST_FORCE_VENV_FAIL=1`, and +`HP_TEST_SYSCON_ANSWER=N` exhaust `:handle_conda_failure`'s own embed/venv/system fallback chain +deterministically, so `HP_ENV_READY` never gets set and the fix's own new check is actually +reached rather than short-circuited by an unrelated fallback tier succeeding. The app imports a +nonexistent module (`fake_pkg_cascade_xyz`, same trick as `selfapps_cascade.ps1`) so warnfix +genuinely fails to resolve it under uv, marking a cascade candidate and driving the uv-to-conda +cascade with `HP_TEST_CASCADE_ANSWER=Y`. + +**Placement is load-bearing for both scenarios.** Each must run AFTER `selfapps_cascade.ps1`'s own +step (`self.cascade.exec`) in the same `uv`-lane job -- that step already downloads and installs +Miniconda for real, so by the time either scenario's step runs `CONDA_BAT` is already cached and +`:cascade_acquire_conda`'s own real-install branch is skipped. Same CI-ordering reasoning as +`selfapps_conda_bothfail.ps1`'s own placement note above, just the opposite direction: that test +needs Miniconda NOT yet installed, these want it already installed. `missing_python` additionally +needs a real (not simulated-failure) conda create to complete, so its own step runs slower than +`create_fails`'s immediate simulated failure. + +Asserts, both scenarios: the cascade reached conda exactly once (`REQ-009: cascading provider uv +to conda`, single occurrence against `~setup.log` alone, matching `self.cascade.exec`'s own +single-source-for-counts convention), the new `[WARN] REQ-009: cascade target conda create failed; +keeping current build.` line fired, the scenario's own hard-failure message +(`[ERROR] conda env create failed.` for `create_fails`, `[ERROR] python.exe missing from conda +environment.` for `missing_python`) appears exactly ONCE in `~setup.log` -- not twice, since +`:handle_conda_failure` always logs its message once before attempting any fallback regardless of +outcome, so a SECOND occurrence (from `:die`'s own separate echo, if the old bug were still +present) is the actual fall-through-regression signal, not the message's mere presence -- the +bootstrap exited 0, and -- the key behavioral proof of the fix -- `~bootstrap.status.json` reads +`state=ok` (not `error`) since `HP_BOOTSTRAP_STATE`'s own default is `ok` at this point in the run +and `:after_cascade_decision` preserves whatever it already was. `create_fails` additionally +asserts both the simulated initial-attempt and retry-attempt failure signatures are PRESENT; +`missing_python` additionally asserts they are ABSENT (proving the real create genuinely +succeeded, not a simulated-failure path in disguise). Non-gating: depends on +`selfapps_cascade.ps1` having already run in the same job. + +``` +self.cascade.conda_create_fail +``` + ## selfapps-pyinstaller-fail NDJSON rows (selfapps_pyinstaller_fail.ps1, real/conda-full lanes) Three scenarios (`PYI_FAIL_SCENARIO` env var: `execfail` / `output_vanish` / diff --git a/docs/open-questions.md b/docs/open-questions.md index 37468e85..d932de8f 100644 --- a/docs/open-questions.md +++ b/docs/open-questions.md @@ -9,4 +9,21 @@ changelog-style sections are for. --- -_No open items right now._ +## 1. Conda native-DLL bundling repair loop: narrow (pygrib/eccodes-only) fix vs. a general pattern-matcher? + +Full context: `docs/prd-conda-native-dll-bundling.md` (CLAUDE.md Active Backlog Item 24, stored in +cold storage -- see `docs/agent-cold-storage.md`'s own entry for the thaw trigger). When this repair +loop is eventually built, should it hardcode the `eccodes.dll` name/glob (mirroring REQ-007's +existing libexpat pattern exactly -- cheap, directly closes the one known failure), or should it +detect ANY `Library not found: could not resolve 'X.dll'` PyInstaller warning generically and +bundle whatever `X` turns out to be (costs more to build/test, but silently covers any future +conda-forge package hitting the same gap without needing its own dedicated PRD each time)? + +The PRD leans toward "build it general is not much extra cost over building it narrow" -- the +loop's reactive/bounded/iterative shape (mirroring `:hidden_import_recover`) doesn't really care +whether the DLL name is hardcoded or parsed out of the warning text -- but this is exactly the +kind of proportionality judgment `docs/prd-av-safe-build-path.md`'s own "Notes from Claude" +section on pin-generalization warns against an agent deciding unilaterally. Needs the +maintainer's call, and only once the PRD's own Requirement 1 (verifying whether +`pyinstaller-hooks-contrib`'s existing `hook-gribapi.py` already solves this for free) has been +checked first -- that verification could make this whole question moot for the pygrib case. diff --git a/docs/prd-conda-native-dll-bundling.md b/docs/prd-conda-native-dll-bundling.md new file mode 100644 index 00000000..17ea2447 --- /dev/null +++ b/docs/prd-conda-native-dll-bundling.md @@ -0,0 +1,227 @@ +# PRD: Conda Native-DLL Bundling Repair Loop (pygrib/eccodes and the General Case) + +**Status:** Draft v1 -- planning only, no implementation started. Written in response to CLAUDE.md +Active Backlog Item 24 (found via `self.layered_e2e.chain`'s real CI evidence once the cStringIO +warnfix fix let that test reach this far for the first time -- see `docs/agent-closed-backlog.md`'s +Item 22 entry for the discovery trail). Deliberately stored in cold storage after this draft, not scheduled +-- see `docs/agent-cold-storage.md`'s own entry for the thaw trigger. +**Owner:** Supervisor (Python_vs_Windows) +**Related:** `docs/agent-closed-backlog.md` Item 22 (the layered E2E test that found this), +CLAUDE.md Active Backlog Item 24, `docs/prd-av-safe-build-path.md` (the repo's other PRD stored +in cold storage -- same "cheapest option first, reactive not proactive" design philosophy applies here) + +--- + +## Research Findings + +### Finding 1 -- PyInstaller already has SOME upstream machinery for this exact package family; untested whether it actually covers `pygrib` + +`pyinstaller-hooks-contrib` (already an installed dependency of this repo's build -- confirmed in +real CI logs as version `2026.6`) ships `hook-gribapi.py`, a standard hook for the `gribapi` +package -- eccodes's own official Python bindings. Per that project's own changelog, this hook was +specifically updated "to account for the possibility of bundled eccodes shared library," and a +companion `findlibs` runtime hook exists and was reworked for eccodes compatibility. PyInstaller's +own binary-collection machinery is also documented as aware of conda's `Library\bin` convention on +Windows. + +**This does not mean the problem is already solved.** `pygrib` is a *different* PyPI/conda package +name from `gribapi`, with its own compiled extension (`_pygrib.cp314-win_amd64.pyd`, per the real +build log this PRD is responding to). PyInstaller hooks trigger on the exact top-level Python +import PyInstaller's static analysis detects -- if `pygrib`'s own C extension links against +`eccodes.dll` directly at the OS/linker level, without ever doing `import gribapi` from Python +code, `hook-gribapi.py` would never fire for a pygrib-only project, regardless of how +conda-Library\bin-aware that hook actually is. This was NOT independently verified against the +hook's actual source in this research pass (the sandbox this PRD was written in could not fetch +`pyinstaller-hooks-contrib`'s repository directly) -- **the single highest-leverage next step, +cheaper than everything else in this document, is confirming this one way or the other** before +building any new bootstrapper-side mechanism. Two ways to check, in order of cost: +1. Add `--hidden-import=gribapi` to the PyInstaller build command for the failing case and see + whether that alone makes `hook-gribapi.py` fire and correctly bundle `eccodes.dll` (and its own + transitive deps) -- if yes, this problem may not need ANY new bootstrapper mechanism at all, + just a hidden-import addition scoped the same way REQ-016's existing loop already works. +2. Failing that, read `hook-gribapi.py`'s actual source (`pyinstaller-hooks-contrib`'s GitHub repo, + `src/_pyinstaller_hooks_contrib/stdhooks/hook-gribapi.py`) to see exactly what it collects and + whether it can be reused/imitated for `pygrib` specifically. + +### Finding 2 -- `--collect-binaries=PKG` is the wrong shape for this failure; it does not cross package boundaries + +Confirmed via PyInstaller's own documentation: `--collect-binaries=PKG` collects binaries found +*inside PKG's own installed package directory*. It is not a dependency-walker across unrelated +packages. Since `eccodes.dll` lives in conda's shared `Library\bin`, not inside `pygrib`'s own +site-packages folder, `--collect-binaries=pygrib` would very likely find nothing -- the DLL isn't +where that flag looks. `--collect-binaries=eccodes` is a closer guess (if eccodes's own Python +package tree contains anything binary) but still uncertain, since the DLL's actual location is the +shared `Library\bin` convention, not a package-local one. `--collect-all=PKG` has the identical +directory-scoping problem, plus it is much heavier (bundles all of a package's data+binaries+ +submodules, not just the one missing DLL) -- rejected for the same reason this repo's own +`--collect-submodules` mechanism deliberately excludes heavy stacks like torch/tensorflow (see +`docs/agent-lessons-learned.md`'s "Pre-build --collect-submodules must be DOUBLE-gated" entry). + +### Finding 3 -- this repo already has a working, shipped example of exactly this failure class: `libexpat` under REQ-007 -- but it is single-purpose, not generalized + +`run_setup.bat`'s `:compute_collect_flags`-adjacent REQ-007 block globs for +`%HP_PY_DIR%Library\bin\libexpat*.dll` and, if found, emits `--add-binary ";."` on the +PyInstaller build command line, logged as `[INFO] REQ-007: bundling conda libexpat DLL for +pyexpat: ...`. This is real, proven, shipped evidence that the "explicit `--add-binary` glob +against conda's `Library\bin`" approach genuinely works for at least one native DLL in this exact +codebase. It is entirely hardcoded to `libexpat` specifically -- the glob pattern, variable names +(`HP_PYI_EXPAT`), and log text are all expat-literal, with no generality and no "is this DLL +actually needed" gate beyond "does the file exist in `Library\bin`." Copy-pasting this pattern for +`eccodes` would work for the ONE named DLL, but inherits the same single-DLL blind spot Finding 4 +below addresses. + +### Finding 4 -- the existing `--hidden-import` auto-recovery loop is the right architectural template, not the collect/add-binary flags in isolation + +This repo already ships a REACTIVE, bounded, iterative repair loop for a structurally similar +problem: `:hidden_import_recover` (REQ-016 Slice 2) re-runs a failed frozen EXE, scans its captured +output for a *specific* error signature (`ModuleNotFoundError` for an installed module), and only +then attempts a targeted rebuild -- bounded to 3 attempts, with a tried-list so the same fix is +never attempted twice (see `docs/agent-lessons-learned.md`'s "--hidden-import auto-recovery must +stay STRICT" entry for the full design). This is deliberately **reactive, never proactive** -- +matching the same design principle `docs/prd-av-safe-build-path.md`'s Tier A already established +("let Nuitka do its own internal compiler discovery... do not build an independent detection +check"). + +A DLL-bundling repair loop should reuse this exact shape, not just the `--add-binary`/glob +mechanics from Finding 3: re-run the EXE, scan output for `ImportError: DLL load failed... The +specified module could not be found` (or the equivalent PyInstaller build-time `WARNING: Library +not found: could not resolve '.dll'` signal, which is available even earlier -- at build +time, before any smoke-test run is needed at all, see the Requirements section below for why this +matters), locate and bundle the NAMED missing DLL from `%CONDA_PREFIX%\Library\bin`, rebuild, and +re-run. If a DIFFERENT DLL is now missing (a transitive dependency of the first), the SAME loop +catches it on the next iteration -- this is what elegantly solves Finding 3's single-DLL blind +spot without needing to solve "walk the full transitive dependency graph up front," which would be +a much larger and more fragile undertaking (Windows has no single, universally-available tool for +this the way `ldd` serves on Linux -- `dumpbin` requires a Visual Studio install, which this +bootstrapper cannot assume is present). + +### Finding 5 -- the build-time warning is a stronger, earlier signal than the runtime crash, and may not even need a "post-smoke" trigger + +Unlike `--hidden-import` auto-recovery (which has no earlier signal than the runtime +`ModuleNotFoundError` -- PyInstaller's static analysis cannot tell whether a genuinely-installed +module will actually be imported by the running program), THIS failure class already announces +itself at BUILD time: `WARNING: Library not found: could not resolve 'eccodes.dll', dependency of +'...\pygrib\_pygrib.cp314-win_amd64.pyd'`. This is available in the PyInstaller build log +immediately after the build step, before the EXE is ever smoke-run. A design that reacts to this +build-time warning (parallel to how `parse_warn.py` already parses PyInstaller's warn file for a +DIFFERENT purpose -- missing Python modules, not missing binaries) could trigger the repair BEFORE +the first smoke-test run ever happens, instead of waiting for a guaranteed-to-fail runtime crash +first. This is a real design choice to make explicitly (see Requirements, P0 item 2) -- build-time +detection is strictly cheaper (skips one guaranteed-failing smoke run) but needs a new parser for a +different PyInstaller output stream than the existing `parse_warn.py` handles; runtime detection +reuses `:hidden_import_recover`'s existing scan-and-react shape almost exactly, at the cost of one +wasted smoke-test cycle per missing DLL. + +## Problem Statement + +`pygrib`'s conda-forge build genuinely installs and PyInstaller's build genuinely succeeds when the +uv-to-conda provider cascade lands a user on the conda provider for a package needing native +C-library dependencies conda ships separately (via its own `Library\bin` shared-library +convention). But the frozen EXE then fails at runtime (`ImportError: DLL load failed`) because +PyInstaller does not discover or bundle that native DLL dependency -- silently defeating the whole +point of having successfully cascaded to conda in the first place. This is currently the ONLY +remaining gap keeping `self.layered_e2e.chain` (the real end-to-end test proving "why you'd need +conda over uv") at `pass:false`; the two mechanisms that test actually exists to prove (REQ-009 +cascade, warnfix repair) both now genuinely pass. + +## Goals + +1. When a frozen EXE fails at runtime with a native-DLL-load-failure signature (or, per Finding 5, + when PyInstaller's own build log already warns about an unresolved `.dll` dependency), locate + the missing DLL in the conda environment's `Library\bin` and bundle it, then rebuild and + re-verify -- reactively, never proactively, matching this repo's established design philosophy. +2. Handle the "one DLL might not be the whole fix" case (transitive native dependencies) via + bounded iteration, reusing the shape of the existing `--hidden-import` auto-recovery loop, not + by trying to solve full dependency-graph resolution up front. +3. Scope the actual repair ACTION to the conda provider specifically (`%CONDA_PREFIX%\Library\bin` + has no meaning under uv/embed/venv/system) -- detection can stay provider-agnostic and cheap. +4. Before building any new mechanism, verify Finding 1's cheaper alternative + (`--hidden-import=gribapi` triggering an existing upstream hook) does not already solve this + for free. + +## Non-Goals + +- **Solving this for every possible native-DLL-dependent package speculatively.** Scope to what + the real, observed failure needs (`eccodes.dll`, and whatever its own transitive chain turns out + to be) first; generalize only if a second, unrelated conda-native-DLL gap is actually observed + (see the Open Question below on narrow-vs-general scope). +- **A full Windows DLL dependency-graph walker** (the `dumpbin`/`ldd`-equivalent problem). Too + heavy, needs tooling this bootstrapper cannot assume is present. The iterative reactive-loop + design (Finding 4) is the deliberate alternative to building this. +- **Fixing this under uv/embed/venv/system.** These providers install from PyPI wheels, which are + expected to vendor their own native dependencies inside the wheel (the whole point of the + manylinux/Windows-wheel packaging standard) -- this specific failure shape is conda-specific by + construction. A package that ships a broken/incomplete wheel is a different, unrelated problem + this PRD does not address. +- **Upstreaming a fix to `pyinstaller-hooks-contrib` itself.** Worth considering later if Finding 1 + confirms a real gap there, but out of scope for this bootstrapper's own PRD -- track separately if + it becomes relevant. + +## Requirements (sketch only -- not sequenced into P0/P1 until this is actually picked up) + +1. **Verify Finding 1 first, with zero new bootstrapper code.** Manually (or via a scratch CI + experiment) add `--hidden-import=gribapi` to a `pygrib` build and observe whether + `hook-gribapi.py` already resolves `eccodes.dll` correctly under conda on Windows. If yes, the + fix may be as small as extending the EXISTING `--hidden-import` mechanism's own scope (or a + small, targeted addition informed by exactly what the hook needs), not a new repair loop at all. +2. **Decide build-time vs. runtime detection (Finding 5).** Prototype whichever is cheaper to wire + given requirement 1's outcome; a build-time detector reuses `parse_warn.py`'s general shape + (new pattern, same file family) but targets a different PyInstaller output stream (`Library not + found: could not resolve` warnings, not the missing-module warn file); a runtime detector reuses + `:hidden_import_recover`'s existing scan-and-react loop shape almost directly. +3. **New repair loop, gated to `HP_ENV_MODE=conda` for the actual bundling action.** Mirror + `:hidden_import_recover`'s bounded-iteration, tried-list shape (see Finding 4) -- not a + single-shot fix. Detection stays provider-agnostic (cheap, always checked); the DLL-glob-and- + `--add-binary` action itself no-ops (with a clear log line) under any non-conda provider. +4. **Regression test forcing the real `pygrib`/`eccodes` failure**, modeled on how + `tests/selfapps_layered_e2e.ps1` already reaches this exact failure for real today (no new + simulated-failure hook needed -- the failure is already reliably reproducible via the existing + test's own uv-to-conda cascade). Assert the loop bundles the DLL, rebuilds, and the final EXE + genuinely runs and prints its token -- this is the acceptance criterion that would finally flip + `self.layered_e2e.chain`'s `chainPass` to `True`. +5. **Documentation**: CLAUDE.md's Item 24 gets its resolution written up (mirroring how Item 22 and + Item 23 were each closed with a full mechanism trace); `docs/agent-interconnect.md` gets a new + section analogous to "Tier A and hidden-import auto-recovery" describing how this new loop + relates to the existing one (touch one, must understand the other, if they end up sharing any + scan infrastructure). + +## Level Check (explicitly requested by the owner) + +**Still on-target, not scope creep.** This repo's own established narrative -- proven out in +`selfapps_cascade.ps1`'s header comment ("the main gain: conda is the strongest solver") and the +whole design of the layered E2E test -- is showcasing concrete, real reasons a user would need +conda's deeper bootstrapping over plain uv/pip. `pygrib` (zero Windows PyPI wheels, real +conda-forge binaries) is already exactly that story for the *install* step. Item 24 is that same +story's natural continuation into the *packaging* step: not just "conda can install what pip +can't," but "the bootstrapper can also freeze a conda-installed native-dependent package into a +working standalone EXE" -- arguably an even sharper, more specific value proposition, since +PyInstaller-plus-conda-native-packages is a genuinely gnarly combination many people struggle with +even by hand. Finishing this closes the loop on the exact demonstration this repo has been building +toward, rather than diverging from it. + +## Open Questions + +- **Narrow (pygrib/eccodes-only) fix vs. a general "conda native-DLL repair loop" that could catch + a FUTURE, different package's missing-DLL gap too -- genuinely undecided, needs the maintainer's + call.** A narrow fix (hardcode the eccodes DLL name/glob, mirroring REQ-007's libexpat pattern + exactly) is cheap and directly closes the one known failure. A general version (detect ANY + `Library not found: could not resolve 'X.dll'` pattern, not just eccodes by name, and bundle + whatever `X` turns out to be) costs more to build and test but would silently cover any future + conda-forge package hitting the same PyInstaller gap, without needing its own dedicated PRD each + time. Given Finding 4's iterative-loop design is barely more general than a single-DLL fix once + built (the loop shape does not care whether the DLL name is hardcoded or parsed from the + warning), this leans toward "build it general from the start is not much extra cost over + building it narrow" -- but this is exactly the kind of proportionality judgment + `docs/prd-av-safe-build-path.md`'s own "Notes from Claude" section on pin-generalization warns + against deciding unilaterally. Flagged here rather than resolved. + +## Confidence Assessment + +**Yellow -- real, well-evidenced problem with a plausible design direction, but the single most +important fact (does `--hidden-import=gribapi` already solve this via the existing upstream hook) +is unverified.** The reactive-loop architecture (Finding 4) is sound and directly reuses a proven, +shipped pattern from this same codebase -- low design risk there. The genuine uncertainty is +scope: this PRD could describe either "add one hidden-import flag" or "build a new repair +subsystem," and which one is true depends entirely on Finding 1's still-open verification. Do not +start implementation without resolving that first -- it changes the size of this work by an order +of magnitude in either direction. diff --git a/run_setup.bat b/run_setup.bat index 65676391..f36f7eb3 100644 --- a/run_setup.bat +++ b/run_setup.bat @@ -895,6 +895,14 @@ rem purpose"): the create call + %ERRORLEVEL% capture is never nested inside a p rem if/else block, so cmd's parse-time %VAR% expansion cannot freeze it to a stale value. if exist "~conda_create.tmp" del "~conda_create.tmp" >nul 2>&1 if "%HP_TEST_FORCE_CONDA_CREATE_NETWORK_FAIL%"=="1" goto :conda_create_test_network_fail +rem [TEST] HP_TEST_FORCE_CONDA_CREATE_BOTH_FAIL: like HP_TEST_FORCE_CONDA_CREATE_NETWORK_FAIL, +rem but genuinely fails BOTH the initial attempt and the retry (not cleared after the first +rem simulated failure -- see the retry call site below), so :conda_create_failed is reached +rem through the real create/retry code path instead of the :hp_test_conda_fail bypass. Exists +rem to test CLAUDE.md Active Backlog Item 23's cascade-restore fix, which specifically needs a +rem genuine (not HP_TEST_FORCE_CONDA_FAIL-style) failure reaching :conda_create_failed during a +rem cascade re-entry. +if "%HP_TEST_FORCE_CONDA_CREATE_BOTH_FAIL%"=="1" goto :conda_create_test_network_fail if "%PYSPEC%"=="" ( call "%CONDA_BAT%" create -y -n "%ENVNAME%" python pip --override-channels -c conda-forge > "~conda_create.tmp" 2>&1 ) else ( @@ -922,12 +930,14 @@ echo Conda environment creation failed -- possible network or repository issue. call :log "[INFO] conda create: transient failure detected; retrying after 15s." timeout /t 15 /nobreak >nul 2>&1 echo Retrying environment creation... +if "%HP_TEST_FORCE_CONDA_CREATE_BOTH_FAIL%"=="1" goto :conda_create_retry_forced_fail if "%PYSPEC%"=="" ( call "%CONDA_BAT%" create -y -n "%ENVNAME%" python pip --override-channels -c conda-forge >> "%LOG%" 2>&1 ) else ( call "%CONDA_BAT%" create -y -n "%ENVNAME%" %PYSPEC% pip --override-channels -c conda-forge >> "%LOG%" 2>&1 ) if not errorlevel 1 goto :conda_create_done +:conda_create_retry_forced_fail echo *** Conda environment creation could not complete. This may be a temporary network issue. echo *** See log file for details: ~setup.log call :log "[WARN] conda create: retry after transient failure also failed." @@ -935,15 +945,31 @@ call :log "[WARN] conda create: retry after transient failure also failed." set "HP_ENV_READY=" call :handle_conda_failure "[ERROR] conda env create failed." if defined HP_ENV_READY goto :after_env_mode_selection +rem derived requirement: a genuine conda-create failure reached via a REQ-009 cascade re-entry +rem (HP_CASCADE_SAVED_PY defined, see :provider_cascade) must gracefully keep the previous +rem working build instead of hard-failing the whole bootstrap, matching every other +rem cascade-target failure (:cascade_conda_unavailable etc.) -- see CLAUDE.md Active Backlog +rem Item 23 for the full trace of why this was previously missing. On a genuine first attempt +rem (no earlier build to fall back to), HP_CASCADE_SAVED_PY is never defined, so this check is a +rem no-op and the existing hard-failure behavior below is unchanged. +if defined HP_CASCADE_SAVED_PY goto :cascade_conda_create_failed call :die "[ERROR] conda env create failed." :conda_create_done set "CONDA_PREFIX=%ENV_PATH%" set "HP_PY=%CONDA_PREFIX%\python.exe" +rem [TEST] HP_TEST_FORCE_CONDA_MISSING_PYTHON: forces the "conda create genuinely succeeded but +rem python.exe is missing afterward" branch below through a REAL successful create (not a +rem simulated create-command failure) -- deletes the real python.exe a genuine create just +rem produced. Exists to test the OTHER call site of the CLAUDE.md Item 23 cascade-restore fix +rem (see :conda_create_failed's own comment above); selfapps_cascade_conda_create_fail.ps1's +rem missing_python scenario is the only caller. +if "%HP_TEST_FORCE_CONDA_MISSING_PYTHON%"=="1" if exist "%HP_PY%" del /f /q "%HP_PY%" >nul 2>&1 if not exist "%HP_PY%" ( set "HP_ENV_READY=" call :handle_conda_failure "[ERROR] python.exe missing from conda environment." if defined HP_ENV_READY goto :after_env_mode_selection + if defined HP_CASCADE_SAVED_PY goto :cascade_conda_create_failed call :die "[ERROR] python.exe missing from conda environment." ) @@ -1906,6 +1932,15 @@ goto :try_conda_create :cascade_conda_unavailable call :log "[WARN] REQ-009: cascade to conda unavailable (Miniconda not installed); keeping current build." goto :after_cascade_decision +:cascade_conda_create_failed +rem derived requirement: reached from :conda_create_failed / :conda_create_done (via goto, not +rem call) when a genuine conda-create failure occurs during THIS cascade re-entry specifically -- +rem see the comment at :conda_create_failed for the full rationale. Deliberately does NOT call +rem :die: :after_cascade_decision's own "keeping current build" restore only works correctly when +rem HP_BOOTSTRAP_STATE is left as whatever it already was (the prior successful build's "ok"), +rem not overwritten to "error" the way :die would. +call :log "[WARN] REQ-009: cascade target conda create failed; keeping current build." +goto :after_cascade_decision :cascade_from_conda if defined HP_CASCADE_TRIED_CONDA goto :after_cascade_decision diff --git a/tests/selfapps_cascade_conda_create_fail.ps1 b/tests/selfapps_cascade_conda_create_fail.ps1 new file mode 100644 index 00000000..b939103e --- /dev/null +++ b/tests/selfapps_cascade_conda_create_fail.ps1 @@ -0,0 +1,259 @@ +# ASCII only +# selfapps_cascade_conda_create_fail.ps1 - Regression coverage for CLAUDE.md Active Backlog +# Item 23: a genuine (non-test-bypassed) conda-create failure reached during a REQ-009 cascade +# re-entry previously fell through :die into :conda_create_done's own success path instead of +# routing through :after_cascade_decision like every other cascade-target failure, so the +# bootstrap could not gracefully keep the previous working uv build. Fixed by adding +# "if defined HP_CASCADE_SAVED_PY goto :cascade_conda_create_failed" at both :conda_create_failed +# fall-through sites, plus a new :cascade_conda_create_failed label mirroring the existing +# :cascade_conda_unavailable / :cascade_embed_unavailable / :cascade_venv_unavailable / +# :cascade_system_unavailable template. +# +# Two scenarios (CASCADE_CCF_SCENARIO env var; unset/anything-but-missing_python defaults to +# create_fails, the original scenario, for CI-wiring back-compat): +# +# create_fails (default): forces a GENUINE failure through the real conda-create/retry code path +# (not the :hp_test_conda_fail bypass HP_TEST_FORCE_CONDA_FAIL uses, which CLAUDE.md's Item 23 +# entry explicitly says is NOT sufficient evidence of this gap being covered) via +# HP_TEST_FORCE_CONDA_CREATE_BOTH_FAIL=1 (run_setup.bat, :try_conda_create) -- unlike the +# existing HP_TEST_FORCE_CONDA_CREATE_NETWORK_FAIL (first attempt only, then clears itself so +# the retry can genuinely succeed), this flag persists through the retry too, so both the +# initial attempt and the retry fail deterministically without depending on real network +# conditions. Exercises :conda_create_failed's own call site of the fix. +# +# missing_python: a CodeRabbit review finding on this PR (see PR #413) noted the ORIGINAL +# scenario never exercises the fix's OTHER call site -- inside :conda_create_done's own +# "if not exist %HP_PY%" block, reached when conda create itself genuinely SUCCEEDS but the +# resulting environment is somehow missing python.exe. Forces this via +# HP_TEST_FORCE_CONDA_MISSING_PYTHON=1 (run_setup.bat, :conda_create_done), which lets the real +# conda create command run and succeed for real, then deletes the python.exe it just produced -- +# a genuine successful create followed by a genuinely missing interpreter, not a simulated create +# failure. HP_TEST_FORCE_CONDA_CREATE_BOTH_FAIL is deliberately NOT set in this scenario (the +# create must succeed for real to reach :conda_create_done at all). +# +# Both scenarios: the app imports a nonexistent module (fake_pkg_cascade_xyz, same trick as +# selfapps_cascade.ps1) so warnfix genuinely fails to resolve it under uv, marking a cascade +# candidate. With HP_TEST_CASCADE_ANSWER=Y the run cascades uv -> conda, reaching +# :try_conda_create as a :cascade_from_uv re-entry (HP_CASCADE_SAVED_PY is set at this point, +# holding the working uv interpreter path). HP_TEST_FORCE_EMBED_FAIL=1, HP_TEST_FORCE_VENV_FAIL=1, +# and HP_TEST_SYSCON_ANSWER=N exhaust :handle_conda_failure's own embed/venv/system fallback chain +# deterministically in both scenarios, so HP_ENV_READY never gets set and the fix's own check is +# actually reached (not short-circuited by an unrelated fallback succeeding). +# +# Asserts (both scenarios): the cascade fired to conda exactly once, the new "cascade target +# conda create failed; keeping current build" WARN line fired, the old hard-failure +# "[ERROR] conda env create failed."/"[ERROR] python.exe missing from conda environment." lines +# never reached :die's own log output, and -- the key behavioral difference the fix makes -- +# ~bootstrap.status.json reads state=ok (not error), since HP_BOOTSTRAP_STATE is still its +# default "ok" at this point in the run and :after_cascade_decision preserves it, matching every +# other "keeping current build" cascade-decline exit. Before the fix this would have fallen +# through :die (which stamps state=error) and continued executing with a broken HP_PY instead. +# create_fails additionally asserts both the simulated initial-attempt and retry-attempt failures +# engaged; missing_python additionally asserts the real create succeeded and the missing-python.exe +# WARN specifically fired (distinguishing it from the create-itself-failed path). +# +# Placement: must run AFTER selfapps_cascade.ps1 in the same uv-lane job -- that step already +# downloads and installs Miniconda for real, so by the time this step runs CONDA_BAT is already +# cached and :cascade_acquire_conda's own real-install branch is skipped (mirrors +# selfapps_conda_bothfail.ps1's own documented CI-ordering constraint, just the opposite +# direction: that test needs Miniconda NOT yet installed, this one wants it already installed). +# missing_python additionally needs a real (not simulated-failure) conda create to complete, which +# is slower than create_fails's immediate simulated failure -- both scenarios still rely on the +# same already-cached Miniconda binary, just with a genuinely longer create step in this one. +# +# Emits: self.cascade.conda_create_fail +# +# Lane: uv only, non-gating (depends on selfapps_cascade.ps1 having already run in the same job). +param() +$ErrorActionPreference = 'Continue' +$scenario = $env:CASCADE_CCF_SCENARIO +if ($scenario -ne 'missing_python') { $scenario = 'create_fails' } +$here = $PSScriptRoot +$repo = Split-Path -Path $here -Parent +$nd = Join-Path $here '~test-results.ndjson' +$ciNd = Join-Path $repo 'ci_test_results.ndjson' +if (-not (Test-Path $nd)) { New-Item -ItemType File -Path $nd -Force | Out-Null } +if (-not (Test-Path $ciNd)) { New-Item -ItemType File -Path $ciNd -Force | Out-Null } + +function Write-NdjsonRow { + param([hashtable]$Row) + $lane = [Environment]::GetEnvironmentVariable('HP_CI_LANE') + if ($lane -and -not $Row.ContainsKey('lane')) { $Row['lane'] = $lane } + $json = $Row | ConvertTo-Json -Compress -Depth 8 + Add-Content -LiteralPath $nd -Value $json -Encoding Ascii + Add-Content -LiteralPath $ciNd -Value $json -Encoding Ascii +} + +# Non-Windows skip (batch bootstrapper only runs on Windows). +if (-not $IsWindows) { + $platform = [System.Environment]::OSVersion.Platform.ToString() + Write-NdjsonRow ([ordered]@{ + id = 'self.cascade.conda_create_fail' + req = 'REQ-009' + pass = $true + skip = $true + desc = 'cascade-reentry conda-create failure gracefully keeps previous build (skipped on non-Windows)' + details = [ordered]@{ platform = $platform } + }) + exit 0 +} + +$batchPath = Join-Path $repo 'run_setup.bat' +if (-not (Test-Path $batchPath)) { + Write-NdjsonRow ([ordered]@{ + id = 'self.cascade.conda_create_fail' + req = 'REQ-009' + pass = $false + desc = 'run_setup.bat not found' + details = [ordered]@{ error = 'run_setup.bat not found at ' + $batchPath } + }) + exit 1 +} + +$workDir = Join-Path $here '~selftest_cascade_conda_create_fail' +if (Test-Path $workDir) { Remove-Item -Recurse -Force $workDir } +New-Item -ItemType Directory -Force -Path $workDir | Out-Null +Copy-Item -Path $batchPath -Destination $workDir -Force + +# derived requirement: a static "import fake_pkg_cascade_xyz" makes PyInstaller's static +# analysis flag the module in warn-.txt. It does not exist on any index, so warnfix +# genuinely fails to install it under uv, marking a cascade candidate (same trick as +# selfapps_cascade.ps1). +$appCode = @' +import fake_pkg_cascade_xyz +print("should-not-run") +'@ +Set-Content -Path (Join-Path $workDir 'app.py') -Value $appCode -Encoding ASCII + +$prevSkip = if (Test-Path Env:HP_SKIP_PIPREQS) { $env:HP_SKIP_PIPREQS } else { $null } +$prevDisableH = if (Test-Path Env:HP_DISABLE_HEURISTICS) { $env:HP_DISABLE_HEURISTICS } else { $null } +$prevCascade = if (Test-Path Env:HP_TEST_CASCADE_ANSWER) { $env:HP_TEST_CASCADE_ANSWER } else { $null } +$prevCcBothFail = if (Test-Path Env:HP_TEST_FORCE_CONDA_CREATE_BOTH_FAIL){ $env:HP_TEST_FORCE_CONDA_CREATE_BOTH_FAIL }else { $null } +$prevCcMissPy = if (Test-Path Env:HP_TEST_FORCE_CONDA_MISSING_PYTHON) { $env:HP_TEST_FORCE_CONDA_MISSING_PYTHON } else { $null } +$prevEmbedFail = if (Test-Path Env:HP_TEST_FORCE_EMBED_FAIL) { $env:HP_TEST_FORCE_EMBED_FAIL } else { $null } +$prevVenvFail = if (Test-Path Env:HP_TEST_FORCE_VENV_FAIL) { $env:HP_TEST_FORCE_VENV_FAIL } else { $null } +$prevSyscon = if (Test-Path Env:HP_TEST_SYSCON_ANSWER) { $env:HP_TEST_SYSCON_ANSWER } else { $null } +$env:HP_SKIP_PIPREQS = '1' +$env:HP_DISABLE_HEURISTICS = '1' +$env:HP_TEST_CASCADE_ANSWER = 'Y' +if ($scenario -eq 'missing_python') { + $env:HP_TEST_FORCE_CONDA_MISSING_PYTHON = '1' +} else { + $env:HP_TEST_FORCE_CONDA_CREATE_BOTH_FAIL = '1' +} +$env:HP_TEST_FORCE_EMBED_FAIL = '1' +$env:HP_TEST_FORCE_VENV_FAIL = '1' +$env:HP_TEST_SYSCON_ANSWER = 'N' + +$bootstrapLog = '~cascade_conda_create_fail_bootstrap.log' +Push-Location $workDir +try { + cmd /c "call run_setup.bat > $bootstrapLog 2>&1" + $runExit = $LASTEXITCODE +} finally { + if ($null -eq $prevSkip) { Remove-Item Env:HP_SKIP_PIPREQS -ErrorAction SilentlyContinue } else { $env:HP_SKIP_PIPREQS = $prevSkip } + if ($null -eq $prevDisableH) { Remove-Item Env:HP_DISABLE_HEURISTICS -ErrorAction SilentlyContinue } else { $env:HP_DISABLE_HEURISTICS = $prevDisableH } + if ($null -eq $prevCascade) { Remove-Item Env:HP_TEST_CASCADE_ANSWER -ErrorAction SilentlyContinue } else { $env:HP_TEST_CASCADE_ANSWER = $prevCascade } + if ($null -eq $prevCcBothFail) { Remove-Item Env:HP_TEST_FORCE_CONDA_CREATE_BOTH_FAIL -ErrorAction SilentlyContinue }else { $env:HP_TEST_FORCE_CONDA_CREATE_BOTH_FAIL = $prevCcBothFail } + if ($null -eq $prevCcMissPy) { Remove-Item Env:HP_TEST_FORCE_CONDA_MISSING_PYTHON -ErrorAction SilentlyContinue } else { $env:HP_TEST_FORCE_CONDA_MISSING_PYTHON = $prevCcMissPy } + if ($null -eq $prevEmbedFail) { Remove-Item Env:HP_TEST_FORCE_EMBED_FAIL -ErrorAction SilentlyContinue } else { $env:HP_TEST_FORCE_EMBED_FAIL = $prevEmbedFail } + if ($null -eq $prevVenvFail) { Remove-Item Env:HP_TEST_FORCE_VENV_FAIL -ErrorAction SilentlyContinue } else { $env:HP_TEST_FORCE_VENV_FAIL = $prevVenvFail } + if ($null -eq $prevSyscon) { Remove-Item Env:HP_TEST_SYSCON_ANSWER -ErrorAction SilentlyContinue } else { $env:HP_TEST_SYSCON_ANSWER = $prevSyscon } + Pop-Location +} + +$logPath = Join-Path $workDir $bootstrapLog +$setupLog = Join-Path $workDir '~setup.log' +$logLines = if (Test-Path $logPath) { Get-Content -LiteralPath $logPath -Encoding ASCII } else { @() } +$setupText = if (Test-Path $setupLog) { Get-Content -LiteralPath $setupLog -Raw -Encoding ASCII } else { '' } +$combined = ($logLines -join "`n") + "`n" + $setupText + +# derived requirement: count against a SINGLE log source ($setupText), matching +# selfapps_cascade.ps1's own established convention -- :log writes every line to BOTH stdout +# (captured in $logLines) AND ~setup.log, so counting against $combined would double every +# occurrence. +$uvToConda = ([regex]::Matches($setupText, [regex]::Escape('REQ-009: cascading provider uv to conda'))).Count + +# Both the simulated initial attempt (the fake CondaHTTPError text written to ~conda_create.tmp, +# then typed into %LOG%) and the simulated retry failure (the existing REQ-022 retry-exhausted +# message, now reached via the forced-fail goto instead of a genuine failed real retry) must +# both appear in create_fails -- proving HP_TEST_FORCE_CONDA_CREATE_BOTH_FAIL engaged at both +# call sites, not just the first. In missing_python these must be ABSENT -- proving the real +# create genuinely succeeded rather than taking the simulated-failure path. +$initialAttemptFailed = $combined -match [regex]::Escape('CondaHTTPError: HTTP 000 CONNECTION FAILED (simulated)') +$retryAlsoFailed = $combined -match [regex]::Escape('conda create: retry after transient failure also failed') + +# The fix itself: the new cascade-target-failure label fired instead of falling through :die. +$cascadeRestoreFired = $combined -match [regex]::Escape('REQ-009: cascade target conda create failed; keeping current build') + +# derived requirement: :handle_conda_failure always logs its message argument once via :log +# BEFORE attempting any fallback (run_setup.bat, :handle_conda_failure's own first line), so +# "[ERROR] ... failed." legitimately appears once in EITHER scenario even on the graceful-restore +# path -- a single occurrence is expected, not a sign the old bug is back. :die (if it fires) +# echoes the SAME message text a second time (its own echo, not via :log) -- so a genuine +# fall-through-to-:die regression shows up as a SECOND occurrence of the scenario's own message, +# not merely a first one. Count, don't just match, for exactly this reason. +if ($scenario -eq 'missing_python') { + $scenarioMsg = '[ERROR] python.exe missing from conda environment.' +} else { + $scenarioMsg = '[ERROR] conda env create failed.' +} +$scenarioMsgCount = ([regex]::Matches($setupText, [regex]::Escape($scenarioMsg))).Count +$dieMessageFired = $scenarioMsgCount -gt 1 + +$statusPath = Join-Path $workDir '~bootstrap.status.json' +$statusExit = $null +$statusState = $null +if (Test-Path $statusPath) { + try { + $status = Get-Content -LiteralPath $statusPath -Raw -Encoding ASCII | ConvertFrom-Json + $statusExit = $status.exitCode + $statusState = $status.state + } catch { } +} + +# Primary criteria, common to both scenarios: the cascade reached conda exactly once, the new +# graceful-restore label fired, the scenario's own failure message reached the log exactly once +# (not the double occurrence a :die fall-through regression would produce), the bootstrap ended +# gracefully (exit 0), and -- the key behavioral proof of the fix -- ~bootstrap.status.json reads +# state=ok (HP_BOOTSTRAP_STATE's own default, preserved by :after_cascade_decision) rather than +# state=error. +$commonPass = ($uvToConda -eq 1) -and $cascadeRestoreFired -and (-not $dieMessageFired) -and ($runExit -eq 0) -and ($statusState -eq 'ok') -and ($statusExit -eq 0) +if ($scenario -eq 'missing_python') { + # The real create must have succeeded (neither simulated-failure signature present) -- + # otherwise this scenario would be indistinguishable from create_fails and prove nothing new. + $pass = $commonPass -and (-not $initialAttemptFailed) -and (-not $retryAlsoFailed) +} else { + $pass = $commonPass -and $initialAttemptFailed -and $retryAlsoFailed +} + +Write-Host "=== self.cascade.conda_create_fail evidence (scenario=$scenario) ===" +Write-Host ("uvToConda={0} initialAttemptFailed={1} retryAlsoFailed={2} cascadeRestoreFired={3} scenarioMsgCount={4} dieMessageFired={5} runExit={6} statusExit={7} statusState={8} pass={9}" -f ` + $uvToConda, $initialAttemptFailed, $retryAlsoFailed, $cascadeRestoreFired, $scenarioMsgCount, $dieMessageFired, $runExit, $statusExit, $statusState, $pass) +Write-Host "=== REQ-009 / conda create / cascade lines (setup log) ===" +($setupText -split "`n") | Where-Object { $_ -match 'REQ-009|conda create|conda env create|cascade|Creating Python environment|CondaHTTPError|python.exe missing' } | Select-Object -First 80 | ForEach-Object { Write-Host $_ } +Write-Host "=== bootstrap stdout log tail (50) ===" +$logLines | Select-Object -Last 50 | ForEach-Object { Write-Host $_ } +Write-Host "=== end self.cascade.conda_create_fail evidence ===" + +Write-NdjsonRow ([ordered]@{ + id = 'self.cascade.conda_create_fail' + req = 'REQ-009' + pass = [bool]$pass + desc = "a genuine conda-create failure (scenario=$scenario) during a uv-to-conda cascade re-entry gracefully keeps the previous uv build (CLAUDE.md Active Backlog Item 23)" + details = [ordered]@{ + scenario = $scenario + uvToConda = $uvToConda + initialAttemptFailed = [bool]$initialAttemptFailed + retryAlsoFailed = [bool]$retryAlsoFailed + scenarioMsgCount = $scenarioMsgCount + cascadeRestoreFired = [bool]$cascadeRestoreFired + dieMessageFired = [bool]$dieMessageFired + runExit = $runExit + statusExit = $statusExit + statusState = $statusState + } +}) + +if (-not $pass) { exit 1 } +exit 0