diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4338589..3d17308 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,6 +144,8 @@ jobs: run: git config --global url."https://${{ secrets.GITHUB_TOKEN }}@github.com/".insteadOf "https://github.com/" - name: Run Windows ACP background persistence tests run: go test ./internal/acpclient -run '^TestBackgroundWindows' -count=1 + - name: Run ACP client lifecycle smoke + run: go test ./internal/acpclient -run '^TestACPClientLifecycleBinarySmoke' -count=1 -timeout=5m - name: Run Windows managed policy security tests run: go test ./internal/hooks -run 'Managed|SecurePolicy' -count=1 - name: Run Windows daemon lock tests @@ -188,8 +190,10 @@ jobs: run: git config --global url."https://${{ secrets.GITHUB_TOKEN }}@github.com/".insteadOf "https://github.com/" - name: Run ACP client binary smoke run: go test ./cmd/ratchet -run '^TestACPClientExecBinarySmoke$' -count=1 -timeout=10m + - name: Run ACP client lifecycle smoke + run: go test ./internal/acpclient -run '^TestACPClientLifecycleBinarySmoke' -count=1 -timeout=5m - name: Test with race detector - run: go test -race -coverprofile=coverage.out -covermode=atomic -skip '^TestACPClientExecBinarySmoke$' ./... + run: go test -race -coverprofile=coverage.out -covermode=atomic -skip '^(TestACPClientExecBinarySmoke|TestACPClientLifecycleBinarySmoke)' ./... - name: Upload coverage uses: codecov/codecov-action@v4 with: diff --git a/docs/plans/2026-07-16-ratchet-cli-lifecycle-reliability-alignment.md b/docs/plans/2026-07-16-ratchet-cli-lifecycle-reliability-alignment.md new file mode 100644 index 0000000..33ed442 --- /dev/null +++ b/docs/plans/2026-07-16-ratchet-cli-lifecycle-reliability-alignment.md @@ -0,0 +1,46 @@ +### Alignment Report + +**Status:** PASS + +**Coverage:** + +| Design Requirement | Plan Task(s) | Status | +|---|---|---| +| cleanup dispatch returns candidate query/scan/iterate/close failures | Task 1 | Covered | +| primary and secondary row errors remain joined | Task 1 | Covered | +| repeated equivalent cleanup errors are rate-limited/reset | Task 1 | Covered | +| fairness waits for durable row deletion | Task 1 | Covered | +| service-level post-stop Get returns `Unavailable` before DB access | Task 2 | Covered | +| stop during startup reconcile cancels and joins | Task 2 | Covered | +| ACP cancel has one bounded send and no arbitrary post-send sleep | Task 3 | Covered | +| exact peer handling remains in-process; OS child proof covers reap | Task 3 | Covered | +| real-process tests share stable selector and 30-second bound | Task 3 | Covered | +| timed-out child is killed and reaped | Task 3 | Covered | +| process smoke is isolated from race coverage | Task 4 | Covered | +| same selector runs on Linux and existing native Windows runner | Task 4, Task 5 | Covered | +| releaseguard pins dedicated commands and paired skip | Task 4 | Covered | +| no secret/provider/prompt payload enters diagnostics | Task 1, Task 2 | Covered | +| no runner/resource/schema/API/dependency change | Scope Manifest, Task 4 | Covered | +| full tests/race/vet/lint/runtime/Windows/release proof | Task 5 | Covered | +| post-merge evidence, retro, closeout, second release | Task 6 | Covered | +| workspace state reconciliation and next phase continuation | Task 6 | Covered | + +**Scope Check:** + +| Plan Task | Design Requirement | Status | +|---|---|---| +| Task 1 | Provider Cleanup; Error Handling; Security | Justified | +| Task 2 | Provider Operation Lifecycle | Justified | +| Task 3 | ACP Cancellation And Process Smoke | Justified | +| Task 4 | CI isolation; native Windows; releaseguard | Justified | +| Task 5 | Multi-Component Validation; Release | Justified | +| Task 6 | two-PR post-merge closeout and continuation | Justified | + +**Manifest Check:** + +- `PR Count: 2` equals two PR grouping rows. +- `Tasks: 6` equals six `### Task N:` headings. +- Tasks 1-5 map only to PR 1; Task 6 maps only to PR 2. +- `plan-scope-check.sh` reports `PASS: scope-manifest checks succeeded`. + +**Drift Items:** None. diff --git a/docs/plans/2026-07-16-ratchet-cli-lifecycle-reliability-design-review.md b/docs/plans/2026-07-16-ratchet-cli-lifecycle-reliability-design-review.md new file mode 100644 index 0000000..ea9a135 --- /dev/null +++ b/docs/plans/2026-07-16-ratchet-cli-lifecycle-reliability-design-review.md @@ -0,0 +1,50 @@ +### Adversarial Review Report + +**Phase:** design +**Artifact:** `docs/plans/2026-07-16-ratchet-cli-lifecycle-reliability-design.md` +**Status:** PASS + +**Findings (Critical):** + +None. + +**Findings (Important):** + +- `D1` [Missing failure modes] [`Provider Cleanup`]: logging every failed dispatch on the 250 ms ticker can flood logs during a database outage. Recommendation: suppress equivalent repeats and emit at most once per minute until a successful dispatch. _Resolution: design revised in `## Provider Cleanup`._ +- `D2` [User-intent drift] [`ACP Cancellation And Process Smoke`]: moving the real-start test out of race coverage did not explicitly replace the named fixed five-second acknowledgment window. Recommendation: specify a larger isolated transition bound plus an outer package timeout. _Resolution: design now requires a 30-second transition bound and five-minute job timeout._ +- `D3` [Project-guidance conflict] [`ACP Cancellation And Process Smoke`; `docs/design-guidance.md` Quality/Security/Operations]: Linux-only process smoke conflicts with the native-operating-system proof rule for process behavior. Recommendation: run the same selector on the existing Windows runner without changing runner ownership. _Resolution: design now requires Linux and native Windows process smoke._ + +**Findings (Minor):** + +- `D4` [Declared integration proof] [`Multi-Component Validation`]: the original wording implied an OS-pipe write proved peer handling of an ACP notification. Recommendation: separate bounded OS-process cancellation/reap proof from exact in-process peer handling. _Resolution: validation wording revised._ + +**Bug-class scan transcript:** + +| Class | Result | Note | +|---|---|---| +| Project-guidance conflicts | Finding | D3 corrected native-platform process proof; no runner change is introduced. | +| Assumptions under attack | Clean | ACP notification, `exec.Cmd.Wait`, durable cleanup row, and smoke budget assumptions include challenges/fallbacks. | +| Repo-precedent conflicts | Clean | Dedicated smoke plus race skip follows `.github/workflows/ci.yml`'s existing ACP binary-smoke split. | +| Artifact-class precedent | Clean | CI assertions remain in `internal/releaseguard/workflow_test.go`; daemon/ACP regressions stay beside owning packages. | +| YAGNI violations | Clean | No observer API, schema, metrics backend, retry-policy, or new runner is added. | +| Missing failure modes | Finding | D1 corrected persistent query-error log flooding; joined close errors and restart-idempotent cleanup are covered. | +| Security / privacy at architecture level | Clean | Diagnostics exclude secret/provider/prompt/command payloads; cancellation reduces child lifetime. | +| Infrastructure impact | Clean | Existing jobs only; no cloud, IAM, migration, network, or production deployment impact. | +| Multi-component validation | Clean | SQLite/provider, service/manager, SDK/process, profile/process-lock, and CI guard boundaries have real proofs. | +| Declared integration proof | Finding | D4 corrected proof strength; runtime and config-only integrations are classified. | +| Contributed UI rendering proof | Clean | No UI contribution exists in scope. | +| Rollback story | Clean | Revert path has no data migration; ACP compatibility fallback retains reap guarantees. | +| Simpler alternative not considered | Clean | Tests-only and broad observer alternatives are explicitly evaluated and rejected. | +| User-intent drift | Finding | D2 corrected the fixed-start acknowledgment follow-up rather than only relocating it. | +| Existence / runtime-validity | Clean | Cited tests, CI jobs, structured workflow parser, service method, and cleanup paths exist on the base tree. | + +**Options the author may not have considered:** + +1. Keep all process tests in race coverage and only increase timeouts: smaller CI diff, but conflicts with durable guidance and continues whole-suite contention. +2. Add a public lifecycle-event API: richer diagnostics, but no current consumer justifies its compatibility and synchronization cost. + +**Verdict reasoning:** All Important findings were incorporated into the design. Remaining risk is bounded to internal lifecycle implementation and existing CI jobs; the design now matches native-platform guidance, directly addresses every named follow-up, and avoids new public state or infrastructure. + +### Process Amendment + +The release section now distinguishes the implementation PR from the required post-merge retro/closeout PR. This does not change feature scope; it prevents pre-merge CI/release evidence from being fabricated in a retro and makes both merges follow the operator's release rule. diff --git a/docs/plans/2026-07-16-ratchet-cli-lifecycle-reliability-design.md b/docs/plans/2026-07-16-ratchet-cli-lifecycle-reliability-design.md new file mode 100644 index 0000000..9a506b8 --- /dev/null +++ b/docs/plans/2026-07-16-ratchet-cli-lifecycle-reliability-design.md @@ -0,0 +1,151 @@ +# Ratchet CLI Lifecycle Reliability Design + +**Status:** Approved +**Date:** 2026-07-16 +**Scope:** Provider cleanup completion/diagnostics, provider-operation shutdown coverage, ACP cancellation teardown, and real-process smoke isolation. + +## Context + +The released `v0.30.35` tree passes the full serial suite, 30 repeated provider-cleanup runs, 50 repeated ACP cancellation/reap runs, and 20 repeated trusted-profile real-start runs. The remaining follow-ups are load-sensitive lifecycle gaps: + +- `TestProviderCleanupDispatcherFairness` observes the secret-provider delete hook before the worker removes its durable cleanup row. +- cleanup candidate query failures are silent; scan/iteration failures discard a secondary `rows.Close` error. +- ACP process cancellation sleeps for a fixed 100 ms after writing the cancellation notification even though ACP defines no peer acknowledgment. +- real-process cancellation and trusted-profile launch tests run inside the race/coverage job, where process startup and fixed five-second acknowledgments compete with the whole suite. +- post-stop `GetProviderOperation` is covered through the manager rather than the service boundary; concurrent stop during startup reconciliation is not covered. + +## Global Design Guidance + +Source: `docs/design-guidance.md` + +| Guidance | Design response | +|---|---| +| Prefer bounded failure and inspectable state. | Return cleanup dispatch errors, preserve close failures, and use the durable cleanup row as completion authority. | +| CLI, daemon, and provider contracts share one authority. | Add service-boundary shutdown coverage without adding a second operation state model. | +| Isolate heavyweight real-binary smoke from race/coverage. | Run ACP process lifecycle proofs in a dedicated CI step and retain in-process race coverage. | +| Preserve Windows, macOS, and Linux paths. | Keep process behavior portable; cross-build both Windows architectures and require existing native Windows jobs. | +| Do not duplicate SDK, plugin, or secret custody. | Continue using `acp-go-sdk`, Workflow `secrets.Provider`, and `secrets.Redactor`; add no integration or secret store. | + +## Approaches + +| Approach | Trade-off | Decision | +|---|---|---| +| Tests only | Smallest diff, but leaves silent cleanup errors and arbitrary runtime grace. | Reject. | +| Target lifecycle boundaries | Fixes observable runtime defects and makes tests causal without adding public API. | Choose. | +| General lifecycle event/observer API | Could support future diagnostics, but adds public surface and synchronization ownership without a consumer. | Reject as YAGNI. | + +## Design + +### Provider Cleanup + +- Change cleanup dispatch to return an error. The loop logs a stable operation label plus the error; secret names and credential values remain absent. +- Rate-limit repeated equivalent dispatch failures to one log per minute and clear the suppression state after a successful dispatch. A persistent database outage must not emit on every 250 ms tick. +- Extract candidate-row collection behind the minimal `Next`/`Scan`/`Err`/`Close` shape. Always close rows and return `errors.Join(primary, closeErr)` so a secondary close failure is preserved. +- Treat the SQL row removal as cleanup completion in the fairness regression. The test waits for both provider deletion and zero durable cleanup rows before asserting attempts/concurrency. +- Keep the worker count, retry schedule, secret naming, and durable schema unchanged. + +### Provider Operation Lifecycle + +- Exercise post-stop `GetProviderOperation` through `Service.GetProviderOperation`; expect `codes.Unavailable` before any database access. +- Add concurrent `Stop` coverage after `Start` has entered `reconcileStartup`, using a blocking secret read on an applied operation. `Stop` must cancel startup, `Start` must return a context-classified error, and shutdown must join. +- No RPC, protobuf, state, or error-code changes. + +### ACP Cancellation And Process Smoke + +- ACP cancellation remains one bounded notification attempt followed by transport closure and process kill/reap. Remove the fixed 100 ms sleep after `ClientSideConnection.Cancel` returns; notification write completion is the strongest available transport signal because ACP cancellation has no response. +- Keep exact-once peer handling assertions in the in-process pipe test, where write completion is synchronized with the reader. The real OS-process test proves prompt return and process reap, not peer handling before forced termination. +- Rename/collect real-process lifecycle tests under a stable binary-smoke selector. Run them once in a dedicated, bounded Linux CI step without race instrumentation; skip only that selector in the race/coverage command. +- Replace the trusted-profile helper's fixed five-second start wait with a shared 30-second process-smoke bound. The dedicated job retains an outer five-minute package timeout, so a deadlock still fails without competing with race/coverage load. +- Run the same real-process lifecycle selector on the existing `windows-2025` job. This adds a step but does not add or change runners. +- Keep in-process cancellation, blocked-send, queue, lock, and persistence tests in the race job. Keep native Windows persistence, daemon lock, release startup, and ConPTY jobs unchanged. +- Add a releaseguard assertion that the dedicated smoke command and matching race skip remain paired. + +## Error Handling + +| Failure | Result | +|---|---| +| Cleanup candidate query | Dispatch returns wrapped error; loop logs classification. | +| Scan/iteration plus close failure | Joined error retains both failures. | +| Cancellation write failure | Error joins with teardown result; process still killed/reaped. | +| Agent ignores cancellation | Transport/process teardown completes without waiting for a nonexistent ACP acknowledgment. | +| Startup reconcile canceled by stop | `Start` returns a context-classified failure; `Stop` joins startup. | +| Dedicated smoke timeout | Linux and Windows CI fail independently without consuming race/coverage timeout. | + +## Security Review + +- No auth/authz boundary changes. +- Cleanup diagnostics contain operation labels and driver errors only; they must not include secret names, values, provider request payloads, command environments, or raw ACP prompt text. +- Cancellation continues closing every owned transport and reaping the child. Removing grace reduces the interval in which an untrusted child remains alive after cancellation. +- No new dependency or executable trust boundary. + +## Infrastructure Impact + +- CI only: one additional Linux Go test step, one step on the existing Windows job, and a narrower race skip for named real-process tests. +- No cloud resources, storage schema, migration, IAM, network listener, secret, runner, or production deployment change. +- Existing Windows hosted jobs remain required; this design does not change runners. + +## Multi-Component Validation + +| Boundary | Proof | +|---|---| +| cleanup worker + SQLite + secret provider | Fairness test observes retry, bounded concurrency, secret deletion, and durable row removal. | +| service + provider operation manager | Post-stop service call returns `Unavailable`; reconcile-time stop cancels and joins startup. | +| ACP SDK + OS process | Linux and Windows dedicated smoke launch the real test child and prove bounded cancellation attempt, prompt return, and process reap; in-process tests prove exact peer handling. | +| trusted profile + process lock + real agent | Linux and Windows dedicated smoke prove the profile process lock remains held through real `Start` success/failure acknowledgment. | +| CI contract | releaseguard rejects a missing dedicated smoke or mismatched race skip. | + +Declared integrations: + +| Integration | Class | Validation | +|---|---|---| +| `github.com/coder/acp-go-sdk` | runtime-integrated | real child process plus in-process exact-once cancellation tests | +| Workflow `secrets.Provider` | runtime-integrated | provider cleanup service test with durable SQLite state | +| GitHub Actions | config-only | releaseguard workflow assertions and Linux/native-Windows PR check execution | + +## Assumptions + +| ID | Assumption | Challenge | Fallback | +|---|---|---|---| +| A1 | ACP cancel is a notification with no response/acknowledgment. | A future protocol version may add one. | Adopt the SDK acknowledgment when available; retain bounded teardown fallback. | +| A2 | `exec.Cmd.Wait` completion is the process-reap authority on supported OSes. | Platform-specific process groups may outlive the direct child. | Add process-group/job-object ownership in a separate design if a reproducer demonstrates descendants. | +| A3 | Cleanup-row deletion is the durable completion signal. | Delete can fail after secret removal. | Existing startup retry remains authoritative; dispatch diagnostics expose the failure. | +| A4 | In-process cancellation tests provide sufficient race coverage for notification ownership. | OS pipe behavior differs from `io.Pipe`. | Dedicated real-process smoke covers OS behavior without race load. | +| A5 | Existing provider/ACP public contracts are correct. | A consumer may need explicit lifecycle events later. | Add a consumer-driven API design; do not preemptively expose internals. | +| A6 | A 30-second isolated process-start bound is sufficient on hosted Linux and Windows. | Runner degradation may exceed it. | Preserve the outer five-minute job timeout and diagnose runner health before increasing the per-transition bound. | + +## Self-Challenge + +1. **Simplest solution:** Polling SQL and increasing timeouts would reduce flakes, but would leave silent cleanup errors and runtime delay; the targeted design is the minimum that fixes both behavior and evidence. +2. **Fragile assumption:** A1 means Ratchet cannot prove the child handled cancellation before kill. The contract is therefore explicitly notification-attempt plus bounded termination, not peer acknowledgment. +3. **Partial failure:** Cleanup can remove a secret and fail to remove its queue row. Startup/retry is idempotent through `secrets.ErrNotFound`, and the new joined error is observable. + +## Adversarial Review Resolutions + +- Repeated cleanup errors are transition/rate-limited rather than logged every tick. +- The fixed five-second real-start wait is replaced explicitly, not merely moved to another job. +- Real-process lifecycle proof runs on existing Linux and Windows runners; cross-compilation is not treated as native proof. + +## Backport 2026-07-16: ACP Handler Scheduling + +The in-process pipe synchronizes notification bytes with the peer reader, but +`acp-go-sdk` dispatches the decoded notification to `handleInbound` in a new +goroutine. Stress evidence observed the handler count before that goroutine ran. +Runtime behavior remains immediate bounded teardown because ACP has no cancel +acknowledgment; in-process exact-once tests now wait on an explicit peer-handler +barrier. The Scope Manifest is unchanged. + +## Non-Goals + +- New daemon lifecycle RPCs, metrics backend, log framework, retry policy, worker count, or cleanup schema. +- ACP protocol changes, process-tree management, arbitrary scheduler work, or runner changes. +- GitHub Actions major-version maintenance; that remains the next independent follow-up. + +## Rollback + +- Revert the implementation commit and restore the prior CI selector. No data rollback or migration is required. +- Cleanup rows and provider operations remain forward/backward compatible. +- If immediate ACP termination exposes an agent compatibility defect, restore the bounded grace in a patch release while collecting a protocol-level reproducer; do not weaken process reap guarantees. + +## Release + +Two PRs: one implementation PR, then one post-merge retro/plan-closeout PR containing evidence that can only exist after the first merge and release. For each PR, wait for green required checks and resolved review threads, admin squash-merge, publish the next patch release from the merge commit, verify every archive/checksum plus Homebrew, and run time-bounded installed `ratchet --version` and provider catalog probes. diff --git a/docs/plans/2026-07-16-ratchet-cli-lifecycle-reliability-plan-review.md b/docs/plans/2026-07-16-ratchet-cli-lifecycle-reliability-plan-review.md new file mode 100644 index 0000000..d114e2b --- /dev/null +++ b/docs/plans/2026-07-16-ratchet-cli-lifecycle-reliability-plan-review.md @@ -0,0 +1,59 @@ +### Adversarial Review Report + +**Phase:** plan +**Artifact:** `docs/plans/2026-07-16-ratchet-cli-lifecycle-reliability.md` +**Status:** PASS + +**Findings (Critical):** + +None. + +**Findings (Important):** + +- `P1` [Verification-class mismatch] [Task 1 Step 1]: stopping the manager before closing the DB lets canceled context mask the intended candidate-query failure. Recommendation: use an unstarted manager with live test context and closed DB. _Resolution: plan revised._ +- `P2` [Verification-class mismatch] [Task 1 Step 4]: restoring the old void signature would prove only a compile failure, not that tests catch dropped query/close/reporting behavior. Recommendation: retain signatures and behaviorally disable each fix during revert proof. _Resolution: plan revised._ +- `P3` [Missing failure mode] [Task 3 Step 2]: increasing child wait bounds without joining after timeout can leave a child/zombie on the failure path. Recommendation: kill and drain `Wait` before failing. _Resolution: plan revised._ + +**Findings (Minor):** + +- `P4` [Over-decomposition] [Task 6]: a docs-only closeout PR causes a second patch release. Recommendation: omit only if operator release policy changes. _Resolution: accepted because post-merge evidence cannot be written truthfully before PR #1 and the operator explicitly requires releases for every merge._ + +**Bug-class scan transcript:** + +| Class | Result | Note | +|---|---|---| +| Project-guidance conflicts | Clean | Tasks map bounded failure, shared contracts, smoke isolation, native Windows, and release proof. | +| Assumptions under attack | Clean | Plan implements the reviewed ACP notification, process reap, durable row, and smoke-budget assumptions. | +| Repo-precedent conflicts | Clean | Workflow tests use existing structured `loadWorkflow`/`requireRun` helpers and current jobs. | +| Artifact-class precedent | Clean | Daemon/ACP tests remain in owner packages; workflow contract remains in releaseguard. | +| YAGNI violations | Clean | No public API, schema, framework, runner, or dependency change. | +| Missing failure modes | Finding | P3 corrected process cleanup on timeout; cleanup restart and log-flood cases are planned. | +| Security / privacy at architecture level | Clean | Tests require sanitized startup errors and cleanup logs omit secret/prompt payloads. | +| Infrastructure impact | Clean | Existing Linux/Windows jobs only; no deploy, resource, IAM, network, or migration. | +| Multi-component validation | Clean | Real SQLite/provider, service/manager, SDK/process, profile/lock, CI, archive, and package boundaries are exercised. | +| Declared integration proof | Clean | ACP and secrets are runtime-proved; Actions is config-guarded and executed on PR. | +| Contributed UI rendering proof | Clean | No UI contribution. | +| Rollback story | Clean | Runtime/CI task has rollback; no data migration exists. | +| Simpler alternative not considered | Clean | Design rejects tests-only timeout changes with rationale. | +| User-intent drift | Clean | Every queued lifecycle follow-up, Windows proof, merge, release, and continuation is represented. | +| Existence / runtime-validity | Clean | Named files/tests/jobs/helpers and commands exist; new selectors are created before workflow use. | +| Over/under-decomposition | Finding | P4 is accepted process overhead; code tasks remain independently revertible. | +| Verification-class mismatch | Finding | P1/P2 corrected behavioral RED/revert evidence; runtime/CI/release classes have matching proof. | +| Auth/authz chain composition | Clean | No auth/authz change. | +| Hidden serial dependencies | Clean | Tasks 1-4 are explicitly serial on one branch; Task 6 depends on PR #1 evidence. | +| Missing rollback wiring | Clean | CI/runtime rollback is in Task 4/5; internal logic reverts cleanly. | +| Missing integration proof | Clean | Task 5 requires real host runtime and native Windows PR execution. | +| Missing declared integration matrix | Clean | Design matrix is implemented by Tasks 1, 3, 4, and 5. | +| Missing contributed UI route proof | Clean | No UI route. | +| Infrastructure verification mismatch | Clean | Structured YAML guard and hosted execution match the CI-only impact. | +| Plugin-loader runtime layout | Clean | No external Workflow plugin process is built or loaded. | +| Config-validation schema rules | Clean | No Workflow config is emitted. | +| Identifier/naming-convention match | Clean | Test names, job IDs, step names, branches, and Go identifiers match repository style. | +| Planned-code compile-validity | Clean | Rows interface matches `sql.Rows`; commands/selectors and Go snippets are compile-valid. | + +**Options the author may not have considered:** + +1. Keep only package-timeout enforcement and no per-transition process bound: fewer helpers, but slower deadlock diagnosis and less actionable failures. +2. Put all reliability work in separate daemon/ACP PRs: smaller diffs, but CI skip/selector changes would temporarily mismatch one side and require an extra release. + +**Verdict reasoning:** The three actionable Important findings are resolved. The accepted second-PR overhead is required for truthful post-merge evidence and explicit release policy. The plan is TDD-oriented, structurally bounded, native-Windows aware, and provides matching unit, race, process, workflow, package, and release verification. diff --git a/docs/plans/2026-07-16-ratchet-cli-lifecycle-reliability.md b/docs/plans/2026-07-16-ratchet-cli-lifecycle-reliability.md new file mode 100644 index 0000000..4c4faf2 --- /dev/null +++ b/docs/plans/2026-07-16-ratchet-cli-lifecycle-reliability.md @@ -0,0 +1,263 @@ +# Ratchet CLI Lifecycle Reliability Implementation Plan + +> **For the implementing agent:** REQUIRED SUB-SKILL: Use autodev:executing-plans to implement this plan task-by-task. + +**Goal:** Make provider cleanup and ACP child-process lifecycle evidence causal, observable, bounded, and reliable under Linux and native Windows CI. + +**Architecture:** Keep the daemon, SQLite, secret-provider, ACP SDK, and process ownership contracts. Return joined cleanup dispatch errors with rate-limited reporting, add shutdown/reconcile regressions, remove ACP's arbitrary post-send sleep, and isolate real-process tests from race coverage in dedicated Linux/Windows steps. + +**Tech Stack:** Go 1.26.4, `database/sql`, `testing/synctest`, `exec.Cmd`, `github.com/coder/acp-go-sdk`, GitHub Actions, releaseguard. + +**Base branch:** `master` + +--- + +## Scope Manifest + +**PR Count:** 2 +**Tasks:** 6 +**Estimated Lines of Change:** ~320 + +**Out of scope:** +- New RPCs, protobuf state, metrics/log frameworks, cleanup schema/retry changes, process-tree ownership, SDK replacement, or runner changes. +- GitHub Actions major-version and Docker dependency maintenance; those remain the next phase. +- Self-improving harness features; they follow reliability/dependency work. + +**PR Grouping:** + +| PR # | Title | Tasks | Branch | +|---|---|---|---| +| 1 | `fix: harden lifecycle cleanup and process smoke` | Task 1, Task 2, Task 3, Task 4, Task 5 | `fix/reliability-followups` | +| 2 | `docs: close lifecycle reliability plan` | Task 6 | `docs/lifecycle-reliability-closeout` | + +**Status:** Locked 2026-07-16T12:38:31Z + +## Guidance Mapping + +Source: `docs/design-guidance.md` + +| Guidance | Task | +|---|---| +| bounded/inspectable background failure | Task 1 | +| shared daemon/service contract | Task 2 | +| real-process smoke outside race coverage | Task 3, Task 4 | +| native Windows runtime proof | Task 4, Task 5 | +| release only merge commits with archive/Homebrew/runtime proof | Task 5, Task 6 | + +### Task 1: Provider Cleanup Completion And Diagnostics + +**Files:** +- Modify: `internal/daemon/provider_cleanup.go` +- Modify: `internal/daemon/provider_cleanup_test.go` + +**Step 1: Write RED tests** + +- `TestProviderCleanupCandidateRowsPreservePrimaryAndCloseErrors`: fake rows return a scan/iteration sentinel and independent close sentinel; require `errors.Is` for both. +- `TestProviderCleanupDispatchReturnsQueryFailure`: construct a manager without starting its loop, set its internal context to `t.Context()`, close its database, invoke dispatch, and require a classified candidate-query error. The live context prevents `context.Canceled` from masking the database failure. +- `TestProviderCleanupErrorReporterSuppressesEquivalentFailures`: inject clock/log function; same error logs once inside one minute, logs after one minute, and nil resets suppression. +- Make `TestProviderCleanupDispatcherFairness` wait for both four provider deletes and cleanup-row count zero. + +Run: `go test ./internal/daemon -run '^TestProviderCleanup(CandidateRows|DispatchReturnsQueryFailure|ErrorReporter|DispatcherFairness)' -count=1` + +Expected: FAIL because candidate collection/reporter APIs and dispatch error return do not exist. + +**Step 2: Implement minimal behavior** + +- Add a minimal rows interface (`Next`, `Scan`, `Err`, `Close`) and `collectProviderCleanupCandidates` with a named error return and deferred `errors.Join(err, rows.Close())`. +- Change `dispatchCleanup() error`; wrap query/scan/iterate/close errors and return nil after scheduling. +- Add loop-local reporter state. Equivalent error strings log at most once/minute; nil resets. Log only `provider cleanup: dispatch: ` without candidate names, provider payloads, prompts, or secrets. +- Keep worker count, retry schedule, schema, and cleanup semantics unchanged. + +**Step 3: Verify GREEN and stress** + +```bash +go test ./internal/daemon -run '^TestProviderCleanup' -count=1 +go test ./internal/daemon -run '^TestProviderCleanupDispatcherFairness$' -count=30 -timeout=2m +go test -race ./internal/daemon -run '^TestProviderCleanup' -count=10 -timeout=5m +``` + +Expected: PASS; four deletes, two poison attempts, max concurrency 1..2, zero cleanup rows. + +**Step 4: Regression proof and commit** + +Temporarily keep the new method signatures but discard the query error, drop the joined close error, and disable reporter suppression while retaining tests; focused selector must FAIL behaviorally rather than only failing to compile. Restore implementation; expect PASS. Record both outputs in PR #1. + +```bash +git add internal/daemon/provider_cleanup.go internal/daemon/provider_cleanup_test.go +git commit -m "fix(daemon): surface cleanup failures" +``` + +### Task 2: Provider Operation Shutdown Regressions + +**Files:** +- Modify: `internal/daemon/provider_operations_test.go` + +**Step 1: Add service and reconcile coverage** + +- In `TestProviderOperationRequestsAfterStopReturnUnavailable`, call `svc.GetProviderOperation(...)`, not the manager; expect `codes.Unavailable` after database close. +- Add `TestProviderOperationStopCancelsStartupReconciliation`: seed an applied operation/secret, block `provider.getHook` until context cancellation, start manager in a goroutine, wait for the hook, call `Stop`, then require `Start` matches `context.Canceled`, the error is sanitized/finalization-classified, and `Stop` joins. +- This is missing characterization coverage; no production change is expected. + +**Step 2: Verify and commit** + +```bash +go test ./internal/daemon -run '^TestProviderOperation(RequestsAfterStopReturnUnavailable|StopCancelsStartupReconciliation)$' -count=30 -timeout=3m +go test -race ./internal/daemon -run '^TestProviderOperation(RequestsAfterStopReturnUnavailable|StopCancelsStartupReconciliation)$' -count=10 -timeout=5m +git add internal/daemon/provider_operations_test.go +git commit -m "test(daemon): cover reconcile shutdown" +``` + +Expected: PASS; service calls are `Unavailable`; reconcile stop always cancels/joins without sensitive metadata. + +### Task 3: ACP Cancellation And Real-Process Contract + +**Files:** +- Modify: `internal/acpclient/client.go` +- Modify: `internal/acpclient/client_test.go` +- Modify: `internal/acpclient/fixture_test.go` +- Modify: `internal/acpclient/background_process_lock_test.go` +- Modify: `internal/acpclient/profiles_process_test.go` + +**Step 1: Write RED cancellation timing test** + +Add `TestClientCancellationNoPostSendDelay` with `testing/synctest`, closable in-process transport, and immediate cancellation. Fake elapsed time around the watcher must be zero. Current code advances 100 ms and must FAIL. + +Keep exact-once peer handling in the in-process test. Rename the three real-process tests to begin `TestACPClientLifecycleBinarySmoke`; the cancellation case proves prompt return/process reap/idempotent close, not peer handling before forced termination. + +Run: `go test ./internal/acpclient -run '^TestClientCancellationNoPostSendDelay$' -count=1` + +Expected: FAIL with elapsed `100ms`, want `0s`. + +**Step 2: Remove grace and stabilize process bounds** + +- After ACP `Cancel` send completion, immediately join the send error with `terminateCancellation()`; retain bounded send context, transport closure/join, process kill/reap, and cause precedence. +- Add shared test constant `acpClientProcessSmokeTimeout = 30 * time.Second` plus channel/poll helper. +- Use it for named profile launch/start-return/child-exit and cancellation prompt/return waits. Keep 200 ms lock-blocked negative assertions. +- On a child-exit timeout, kill the process and drain its `Wait` result before failing the test so the failure path does not leak a child/zombie. + +**Step 3: Verify, regression-proof, and commit** + +```bash +go test ./internal/acpclient -run '^TestClientCancellation' -count=10 -timeout=5m +go test ./internal/acpclient -run '^TestACPClientLifecycleBinarySmoke' -count=20 -timeout=10m +go test -race ./internal/acpclient -skip '^TestACPClientLifecycleBinarySmoke' -run 'Cancellation|BackgroundProfile' -count=5 -timeout=10m +``` + +Expected: PASS; zero fake-time grace, exact-once in-process cancel, reaped child, and profile lock held through real start success/failure acknowledgment. + +Restore only the 100 ms grace; timing test must FAIL. Remove it; expect PASS. Record both outputs. + +```bash +git add internal/acpclient/client.go internal/acpclient/client_test.go internal/acpclient/fixture_test.go internal/acpclient/background_process_lock_test.go internal/acpclient/profiles_process_test.go +git commit -m "fix(acp): bound process lifecycle smoke" +``` + +### Task 4: Linux And Windows CI Smoke Isolation + +**Files:** +- Modify: `.github/workflows/ci.yml` +- Modify: `internal/releaseguard/workflow_test.go` + +**Step 1: Write RED structured workflow guard** + +Add `TestCIIsolatesACPClientLifecycleBinarySmoke` using `loadWorkflow`, `requireJob`, and `requireRun`. Require: + +- Linux `test` step `Run ACP client lifecycle smoke`: `go test ./internal/acpclient -run '^TestACPClientLifecycleBinarySmoke' -count=1 -timeout=5m`. +- Existing native Windows job has the same step/command. +- Race step contains `-skip '^(TestACPClientExecBinarySmoke|TestACPClientLifecycleBinarySmoke)'`. + +Run: `go test ./internal/releaseguard -run '^TestCIIsolatesACPClientLifecycleBinarySmoke$' -count=1` + +Expected: FAIL because steps/paired skip are absent. + +**Step 2: Update CI without changing runners** + +- Add Linux lifecycle smoke after existing command-binary smoke. +- Add the same selector after Windows ACP persistence tests. +- Replace the race skip with the paired anchored regex. +- Do not change `runs-on`, action versions, job dependencies, release workflow, or runner count. + +**Step 3: Verify, regression-proof, and commit** + +```bash +go test ./internal/releaseguard -run 'CIIsolatesACPClientLifecycleBinarySmoke|CIRequiresNativeWindowsBackgroundPersistence|WorkflowsAvoidStaleWindowsRunner' -count=1 +go test ./internal/acpclient -run '^TestACPClientLifecycleBinarySmoke' -count=1 -timeout=5m +go test -race ./internal/acpclient -skip '^TestACPClientLifecycleBinarySmoke' -count=1 -timeout=15m +``` + +Expected: PASS; parser finds Linux/Windows steps and exact skip; host executes three process tests. + +Remove either dedicated step while retaining guard; guard must FAIL. Restore; expect PASS. Record both outputs. + +Rollback: revert commit, restore old race selector, rerun old race command plus one untagged process selector. + +```bash +git add .github/workflows/ci.yml internal/releaseguard/workflow_test.go +git commit -m "ci: isolate ACP process lifecycle smoke" +``` + +### Task 5: PR 1 Verification, Merge, And Release + +**Files:** +- Verify all PR #1 changes. + +**Step 1: Local gates** + +```bash +go test ./internal/daemon ./internal/acpclient ./internal/releaseguard -count=1 -timeout=15m +go test -race ./internal/daemon ./internal/acpclient ./internal/releaseguard -skip '^TestACPClientLifecycleBinarySmoke' -count=1 -timeout=20m +GOTMPDIR=$HOME/.rt GOCACHE=$HOME/.rc go test -p 1 ./... -count=1 -timeout=30m +go vet ./... +golangci-lint run --new-from-rev=origin/master +git diff --check +goreleaser check +``` + +Expected: exits 0; lint reports 0 issues. + +**Step 2: Runtime and Windows build proof** + +```bash +go build -o /tmp/ratchet-lifecycle ./cmd/ratchet +/tmp/ratchet-lifecycle --version +/tmp/ratchet-lifecycle provider setup list --json +GOOS=windows GOARCH=amd64 go build -o /tmp/ratchet-windows-amd64.exe ./cmd/ratchet +GOOS=windows GOARCH=arm64 go build -o /tmp/ratchet-windows-arm64.exe ./cmd/ratchet +``` + +Expected: host commands exit within 10 seconds; provider JSON has 22 entries; Windows outputs are non-empty PE files. Native Windows lifecycle smoke must pass on PR. + +Rollback: do not merge on failure; revert responsible task commit and rerun focused/runtime proof. + +**Step 3: Review, PR, monitor, merge** + +- Invoke `autodev:requesting-code-review`; process findings with `autodev:receiving-code-review`. +- Confirm `gh >= 2.88` before/after PR creation. Push branch, open PR with design/plan, RED/GREEN/revert proof, security boundary, verification, and Scope Manifest. +- Add Copilot immediately and invoke `autodev:pr-monitoring` immediately. +- Wait for all required checks, native Windows, CodeQL, and threads; fix and continue monitoring. +- Admin squash-merge only when green/resolved; delete remote branch. + +**Step 4: Release merge commit** + +Tag next patch from merge commit; wait for release workflow. Verify seven public assets, six archive checksums, Formula/Cask versions/hashes, Homebrew install/upgrade, bounded installed `ratchet --version`, and provider catalog count 22. + +Expected: public release; every package/runtime check passes; no draft remains. + +### Task 6: Retro, Closeout PR, And Release + +**Files:** +- Create: `docs/retros/2026-07-16-ratchet-cli-lifecycle-reliability-retro.md` +- Modify: `docs/plans/2026-07-16-ratchet-cli-lifecycle-reliability.md` +- Modify only for a durable new rule: `docs/design-guidance.md` + +**Step 1: Evidence-based retro** + +Invoke `autodev:post-merge-retrospective`. Correlate D1-D4, plan findings, RED/GREEN/revert proof, code review, PR/native-Windows checks, merge SHA, release run, archives/Homebrew/runtime, and misses. Do not claim unavailable activation evidence. + +**Step 2: Close plan and publish PR #2** + +Mark complete only with merge/release evidence. Verify manifest and `git diff --check`. Branch current `origin/master` as `docs/lifecycle-reliability-closeout`; review/Copilot/PR-monitoring; admin squash-merge only green/resolved. + +**Step 3: Release PR #2 and continue** + +Publish next patch from closeout merge; repeat seven-asset, checksum, Formula/Cask, Homebrew, bounded version, and 22-provider proofs. Reconcile workspace portfolio/projects/followups/phase JSONL through an isolated workspace PR without touching the dirty root branch. Continue automatically to Actions runtime maintenance and upstream Docker remediation, then self-improving harness work. diff --git a/docs/plans/2026-07-16-ratchet-cli-lifecycle-reliability.md.scope-lock b/docs/plans/2026-07-16-ratchet-cli-lifecycle-reliability.md.scope-lock new file mode 100644 index 0000000..f75e7c2 --- /dev/null +++ b/docs/plans/2026-07-16-ratchet-cli-lifecycle-reliability.md.scope-lock @@ -0,0 +1 @@ +0ffeb53f28790a0b6b87d941dcbe3d50a0a8c6daad67c810405c9b9219734a7f diff --git a/internal/acpclient/background_process_lock_test.go b/internal/acpclient/background_process_lock_test.go index c86909a..f241a0e 100644 --- a/internal/acpclient/background_process_lock_test.go +++ b/internal/acpclient/background_process_lock_test.go @@ -442,7 +442,7 @@ func TestBackgroundProfileUnsupportedLeaseFailsBeforeDurableWriteOrStart(t *test } } -func TestBackgroundProfileLaunchHoldsProcessLeaseThroughRealStart(t *testing.T) { +func TestACPClientLifecycleBinarySmokeBackgroundProfileLaunchHoldsProcessLeaseThroughRealStart(t *testing.T) { dir := t.TempDir() fixture := BuildFixtureAgent(t) manager, sessions, profiles, barriers := newBackgroundProfileProcessManager(t, dir, fixture, func( @@ -494,7 +494,7 @@ func TestBackgroundProfileLaunchHoldsProcessLeaseThroughRealStart(t *testing.T) }) } -func TestBackgroundProfileLaunchReleasesProcessLeaseAfterStartFailure(t *testing.T) { +func TestACPClientLifecycleBinarySmokeBackgroundProfileLaunchReleasesProcessLeaseAfterStartFailure(t *testing.T) { dir := t.TempDir() missingCommand := filepath.Join(dir, "missing-agent") manager, _, profiles, barriers := newBackgroundProfileProcessManager(t, dir, missingCommand, func( @@ -658,22 +658,12 @@ type backgroundLaunchBarriers struct { func waitBackgroundLaunchEntered(t *testing.T, entered <-chan struct{}) { t.Helper() - select { - case <-entered: - case <-time.After(5 * time.Second): - t.Fatal("profile launch did not reach StartRunner barrier") - } + waitACPClientProcessValue(t, entered, "profile launch StartRunner barrier") } func waitBackgroundStartReturned(t *testing.T, returned <-chan error) error { t.Helper() - select { - case err := <-returned: - return err - case <-time.After(5 * time.Second): - t.Fatal("profile launch StartRunner did not acknowledge real start") - return nil - } + return waitACPClientProcessValue(t, returned, "profile launch StartRunner acknowledgment") } type backgroundBarrierStartRunner func( diff --git a/internal/acpclient/client.go b/internal/acpclient/client.go index fa4524d..0f01001 100644 --- a/internal/acpclient/client.go +++ b/internal/acpclient/client.go @@ -402,9 +402,6 @@ func (c *Client) sendCancelAndTerminate(ctx context.Context, sessionID acpsdk.Se }() select { case sendErr := <-sendDone: - grace := time.NewTimer(min(c.timeout, 100*time.Millisecond)) - <-grace.C - grace.Stop() return errors.Join(sendErr, c.terminateCancellation()) case <-sendCtx.Done(): sendCancel() diff --git a/internal/acpclient/client_test.go b/internal/acpclient/client_test.go index 6a86593..8b83ae8 100644 --- a/internal/acpclient/client_test.go +++ b/internal/acpclient/client_test.go @@ -14,6 +14,7 @@ import ( "sync" "sync/atomic" "testing" + "testing/synctest" "time" acpsdk "github.com/coder/acp-go-sdk" @@ -324,7 +325,7 @@ func TestClientCancellationCommittedWithPromptResponseWins(t *testing.T) { }) requested := &atomic.Bool{} - agent := &cancelOnPromptResponseAgent{requested: requested} + agent := &cancelOnPromptResponseAgent{requested: requested, cancelCalled: make(chan struct{})} agentConn := acpsdk.NewAgentSideConnection(agent, agentToClientW, clientToAgentR) agent.conn = agentConn client := NewInProcessClient(clientToAgentW, agentToClientR, RunOptions{ @@ -339,14 +340,63 @@ func TestClientCancellationCommittedWithPromptResponseWins(t *testing.T) { if _, err := client.RunPrompt(t.Context(), "response race"); !errors.Is(err, ErrCancelRequested) { t.Fatalf("RunPrompt error = %v, want ErrCancelRequested", err) } + waitACPClientProcessValue(t, agent.cancelCalled, "in-process ACP peer cancellation") if got := agent.cancelCount.Load(); got != 1 { t.Fatalf("ACP cancel count = %d, want 1", got) } } -func TestClientCancellationIgnoringAgentReturnsAndReapsProcess(t *testing.T) { +func TestClientCancellationNoPostSendDelay(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + clientToAgentR, clientToAgentW := io.Pipe() + agentToClientR, agentToClientW := io.Pipe() + defer func() { + _ = clientToAgentR.Close() + _ = clientToAgentW.Close() + _ = agentToClientR.Close() + _ = agentToClientW.Close() + }() + + agent := &cancellationRaceAgent{cancelCalled: make(chan struct{})} + agentConn := acpsdk.NewAgentSideConnection(agent, agentToClientW, clientToAgentR) + client := NewInProcessClient(clientToAgentW, agentToClientR, RunOptions{ + Timeout: time.Second, + CancelRequested: func(string) (bool, error) { + return true, nil + }, + }) + + started := time.Now() + stopWatcher, err := client.startCancelWatcher(t.Context(), "immediate-cancel") + if !errors.Is(err, ErrCancelRequested) { + t.Fatalf("cancel watcher error = %v, want ErrCancelRequested", err) + } + if stopWatcher != nil { + t.Fatal("cancel watcher returned a stop function after immediate cancellation") + } + if elapsed := time.Since(started); elapsed != 0 { + t.Fatalf("cancel watcher elapsed = %v, want 0s", elapsed) + } + + synctest.Wait() + if got := agent.cancelCount.Load(); got != 1 { + t.Fatalf("ACP cancel count = %d, want 1", got) + } + select { + case <-agent.cancelCalled: + default: + t.Fatal("ACP peer did not handle cancellation") + } + select { + case <-agentConn.Done(): + default: + } + }) +} + +func TestACPClientLifecycleBinarySmokeCancellationIgnoringAgentReturnsAndReapsProcess(t *testing.T) { requested := &atomic.Bool{} - client, cancelMarker, promptMarker := startCancellationProcessClient(t, requested, nil) + client, _, promptMarker := startCancellationProcessClient(t, requested, nil) runner, err := client.StartSession(t.Context(), "") if err != nil { t.Fatalf("StartSession: %v", err) @@ -359,18 +409,10 @@ func TestClientCancellationIgnoringAgentReturnsAndReapsProcess(t *testing.T) { }() waitForCancellationMarker(t, promptMarker) requested.Store(true) - select { - case err := <-done: - if !errors.Is(err, ErrCancelRequested) { - t.Fatalf("Prompt error = %v, want ErrCancelRequested", err) - } - case <-time.After(3 * time.Second): - t.Fatal("Prompt did not return after cancellation") + if err := waitACPClientProcessValue(t, done, "canceled prompt return"); !errors.Is(err, ErrCancelRequested) { + t.Fatalf("Prompt error = %v, want ErrCancelRequested", err) } assertCancellationProcessReaped(t, client) - if got := markerLineCount(t, cancelMarker); got != 1 { - t.Fatalf("ACP cancel count = %d, want 1", got) - } if err := client.Close(); err != nil { t.Fatalf("idempotent Close: %v", err) } @@ -892,7 +934,7 @@ func cancellationProcessSpec() AgentSpec { func waitForCancellationMarker(t *testing.T, path string) { t.Helper() - deadline := time.Now().Add(3 * time.Second) + deadline := time.Now().Add(acpClientProcessSmokeTimeout) for time.Now().Before(deadline) { if _, err := os.Stat(path); err == nil { return @@ -1035,12 +1077,15 @@ type cancellationRaceAgent struct { type cancelOnPromptResponseAgent struct { echoAgent - requested *atomic.Bool - cancelCount atomic.Int64 + requested *atomic.Bool + cancelCalled chan struct{} + cancelOnce sync.Once + cancelCount atomic.Int64 } func (a *cancelOnPromptResponseAgent) Cancel(context.Context, acpsdk.CancelNotification) error { a.cancelCount.Add(1) + a.cancelOnce.Do(func() { close(a.cancelCalled) }) return nil } diff --git a/internal/acpclient/fixture_test.go b/internal/acpclient/fixture_test.go index a8b740b..b460b0b 100644 --- a/internal/acpclient/fixture_test.go +++ b/internal/acpclient/fixture_test.go @@ -5,8 +5,23 @@ import ( "path/filepath" "runtime" "testing" + "time" ) +const acpClientProcessSmokeTimeout = 30 * time.Second + +func waitACPClientProcessValue[T any](t *testing.T, values <-chan T, action string) T { + t.Helper() + select { + case value := <-values: + return value + case <-time.After(acpClientProcessSmokeTimeout): + t.Fatalf("timed out waiting for %s", action) + var zero T + return zero + } +} + func BuildFixtureAgent(t *testing.T) string { t.Helper() diff --git a/internal/acpclient/profiles_process_test.go b/internal/acpclient/profiles_process_test.go index 185860a..f112003 100644 --- a/internal/acpclient/profiles_process_test.go +++ b/internal/acpclient/profiles_process_test.go @@ -170,8 +170,13 @@ func waitProfileProcess(t *testing.T, child *profileProcess) { if err != nil { t.Fatalf("profile process failed: %v\n%s", err, child.output.String()) } - case <-time.After(5 * time.Second): - _ = child.cmd.Process.Kill() - t.Fatalf("profile process remained blocked\n%s", child.output.String()) + case <-time.After(acpClientProcessSmokeTimeout): + killErr := child.cmd.Process.Kill() + select { + case waitErr := <-child.done: + t.Fatalf("profile process remained blocked (kill: %v, wait: %v)\n%s", killErr, waitErr, child.output.String()) + case <-time.After(acpClientProcessSmokeTimeout): + t.Fatalf("profile process remained blocked and Wait did not return after kill (kill: %v)\n%s", killErr, child.output.String()) + } } } diff --git a/internal/daemon/provider_cleanup.go b/internal/daemon/provider_cleanup.go index 2cd3113..59971a0 100644 --- a/internal/daemon/provider_cleanup.go +++ b/internal/daemon/provider_cleanup.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "errors" + "fmt" "log" "time" @@ -12,6 +13,53 @@ import ( const providerCleanupWorkers = 2 +type providerCleanupRows interface { + Next() bool + Scan(...any) error + Err() error + Close() error +} + +type providerCleanupErrorReporter struct { + now func() time.Time + logf func(string, ...any) + lastError string + lastReportedAt time.Time +} + +func (r *providerCleanupErrorReporter) report(err error) { + if err == nil { + r.lastError = "" + r.lastReportedAt = time.Time{} + return + } + now := r.now() + errorText := err.Error() + if errorText == r.lastError && now.Sub(r.lastReportedAt) < time.Minute { + return + } + r.logf("provider cleanup: dispatch: %v", err) + r.lastError = errorText + r.lastReportedAt = now +} + +func collectProviderCleanupCandidates(rows providerCleanupRows) (names []string, err error) { + defer func() { + err = errors.Join(err, rows.Close()) + }() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, fmt.Errorf("scan provider cleanup candidate: %w", err) + } + names = append(names, name) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate provider cleanup candidates: %w", err) + } + return names, nil +} + func queueProviderSecretCleanupTx(ctx context.Context, tx *sql.Tx, secretName string) error { if secretName == "" { return nil @@ -36,55 +84,41 @@ func (m *providerOperationManager) WakeCleanup() { func (m *providerOperationManager) cleanupLoop() { ticker := time.NewTicker(250 * time.Millisecond) defer ticker.Stop() + reporter := providerCleanupErrorReporter{now: time.Now, logf: log.Printf} for { select { case <-m.ctx.Done(): return case <-m.cleanupWake: - m.dispatchCleanup() + reporter.report(m.dispatchCleanup()) case <-ticker.C: - m.dispatchCleanup() + reporter.report(m.dispatchCleanup()) } } } -func (m *providerOperationManager) dispatchCleanup() { +func (m *providerOperationManager) dispatchCleanup() error { m.cleanupMu.Lock() available := providerCleanupWorkers - len(m.cleaning) m.cleanupMu.Unlock() if available <= 0 { - return + return nil } rows, err := m.engine.DB.QueryContext(m.ctx, `SELECT secret_name FROM provider_secret_cleanup WHERE unixepoch(next_attempt_at) <= unixepoch() ORDER BY unixepoch(next_attempt_at), unixepoch(created_at) LIMIT ?`, available*4) if err != nil { - return + return fmt.Errorf("query provider cleanup candidates: %w", err) } - var names []string - for rows.Next() { - var name string - if err := rows.Scan(&name); err != nil { - log.Printf("provider cleanup: scan candidate: %v", err) - _ = rows.Close() - return - } - names = append(names, name) - } - if err := rows.Err(); err != nil { - log.Printf("provider cleanup: iterate candidates: %v", err) - _ = rows.Close() - return - } - if err := rows.Close(); err != nil { - log.Printf("provider cleanup: close candidates: %v", err) - return + names, err := collectProviderCleanupCandidates(rows) + if err != nil { + return fmt.Errorf("collect provider cleanup candidates: %w", err) } for _, name := range names { m.cleanupMu.Lock() if len(m.cleaning) >= providerCleanupWorkers { m.cleanupMu.Unlock() - return + return nil } if m.cleaning[name] { m.cleanupMu.Unlock() @@ -98,6 +132,7 @@ func (m *providerOperationManager) dispatchCleanup() { m.cleanupSecret(name) }() } + return nil } func (m *providerOperationManager) cleanupSecret(secretName string) { diff --git a/internal/daemon/provider_cleanup_test.go b/internal/daemon/provider_cleanup_test.go index 92c229c..6151bf5 100644 --- a/internal/daemon/provider_cleanup_test.go +++ b/internal/daemon/provider_cleanup_test.go @@ -3,6 +3,8 @@ package daemon import ( "context" "errors" + "fmt" + "strings" "sync" "testing" "testing/synctest" @@ -46,11 +48,15 @@ func TestProviderCleanupDispatcherFairness(t *testing.T) { svc.providerOps.WakeCleanup() deadline := time.Now().Add(3 * time.Second) + remaining := -1 for time.Now().Before(deadline) { mu.Lock() allDeleted := len(deleted) == 4 mu.Unlock() - if allDeleted { + if err := db.QueryRow(`SELECT count(*) FROM provider_secret_cleanup`).Scan(&remaining); err != nil { + t.Fatal(err) + } + if allDeleted && remaining == 0 { break } time.Sleep(10 * time.Millisecond) @@ -69,10 +75,6 @@ func TestProviderCleanupDispatcherFairness(t *testing.T) { if poisonAttempts != 2 { t.Fatalf("poison delete attempts = %d, want 2", poisonAttempts) } - var remaining int - if err := db.QueryRow(`SELECT count(*) FROM provider_secret_cleanup`).Scan(&remaining); err != nil { - t.Fatal(err) - } if remaining != 0 { t.Fatalf("cleanup rows remaining = %d, want 0", remaining) } @@ -81,6 +83,85 @@ func TestProviderCleanupDispatcherFairness(t *testing.T) { } } +func TestProviderCleanupCandidateRowsPreservePrimaryAndCloseErrors(t *testing.T) { + primaryErr := errors.New("candidate row primary failure") + closeErr := errors.New("candidate row close failure") + tests := map[string]*providerCleanupRowsStub{ + "scan": { + next: true, + scanErr: primaryErr, + closeErr: closeErr, + }, + "iteration": { + iterateErr: primaryErr, + closeErr: closeErr, + }, + } + for name, rows := range tests { + t.Run(name, func(t *testing.T) { + _, err := collectProviderCleanupCandidates(rows) + if !errors.Is(err, primaryErr) { + t.Fatalf("candidate row error = %v, want primary failure", err) + } + if !errors.Is(err, closeErr) { + t.Fatalf("candidate row error = %v, want close failure", err) + } + if !rows.closed { + t.Fatal("candidate rows were not closed") + } + }) + } +} + +func TestProviderCleanupDispatchReturnsQueryFailure(t *testing.T) { + db := openProviderOperationTestDB(t) + manager := newProviderOperationManager(&EngineContext{DB: db}) + manager.ctx = t.Context() + if err := db.Close(); err != nil { + t.Fatal(err) + } + + err := manager.dispatchCleanup() + if err == nil { + t.Fatal("dispatch cleanup succeeded with a closed database") + } + if !strings.Contains(err.Error(), "query provider cleanup candidates") { + t.Fatalf("dispatch cleanup error = %q, want candidate-query classification", err) + } +} + +func TestProviderCleanupErrorReporterSuppressesEquivalentFailures(t *testing.T) { + now := time.Date(2026, time.July, 16, 12, 0, 0, 0, time.UTC) + var logs []string + reporter := providerCleanupErrorReporter{ + now: func() time.Time { return now }, + logf: func(format string, args ...any) { + logs = append(logs, fmt.Sprintf(format, args...)) + }, + } + failure := errors.New("candidate query unavailable") + + reporter.report(failure) + now = now.Add(59 * time.Second) + reporter.report(errors.New(failure.Error())) + if len(logs) != 1 { + t.Fatalf("logs inside suppression window = %v, want one entry", logs) + } + + now = now.Add(2 * time.Second) + reporter.report(errors.New(failure.Error())) + reporter.report(nil) + reporter.report(errors.New(failure.Error())) + if len(logs) != 3 { + t.Fatalf("logs after interval and reset = %v, want three entries", logs) + } + for _, got := range logs { + if want := "provider cleanup: dispatch: candidate query unavailable"; got != want { + t.Fatalf("cleanup diagnostic = %q, want %q", got, want) + } + } +} + func TestProviderOperationStopWaitsForCleanupWorker(t *testing.T) { synctest.Test(t, func(t *testing.T) { const secretName = "provider-v2-stop-cleanup" @@ -109,3 +190,30 @@ func TestProviderOperationStopWaitsForCleanupWorker(t *testing.T) { waitProviderOperationValue(t, stopDone, "provider operation stop") }) } + +type providerCleanupRowsStub struct { + next bool + scanErr error + iterateErr error + closeErr error + closed bool +} + +func (r *providerCleanupRowsStub) Next() bool { + next := r.next + r.next = false + return next +} + +func (r *providerCleanupRowsStub) Scan(...any) error { + return r.scanErr +} + +func (r *providerCleanupRowsStub) Err() error { + return r.iterateErr +} + +func (r *providerCleanupRowsStub) Close() error { + r.closed = true + return r.closeErr +} diff --git a/internal/daemon/provider_operations_test.go b/internal/daemon/provider_operations_test.go index 684e479..d65d63f 100644 --- a/internal/daemon/provider_operations_test.go +++ b/internal/daemon/provider_operations_test.go @@ -551,7 +551,7 @@ func TestProviderOperationRequestsAfterStopReturnUnavailable(t *testing.T) { } }) } - if _, err := svc.providerOps.Get(t.Context(), "missing"); status.Code(err) != codes.Unavailable { + if _, err := svc.GetProviderOperation(t.Context(), &pb.GetProviderOperationReq{OperationId: "missing"}); status.Code(err) != codes.Unavailable { t.Fatalf("get after stop code = %v, want Unavailable (err=%v)", status.Code(err), err) } } @@ -622,6 +622,68 @@ func TestProviderOperationStopCancelsStart(t *testing.T) { waitProviderOperationValue(t, stopDone, "provider operation stop") } +func TestProviderOperationStopCancelsStartupReconciliation(t *testing.T) { + const ( + operationID = "d4c09fc0-296f-4b9f-b820-d01fe7bc5aa6" + alias = "startup-stop-reconcile" + secretName = "provider-v2-startup-stop-reconcile" + credential = "startup-stop-sensitive-credential" + ) + db := openProviderOperationTestDB(t) + if err := initDB(db); err != nil { + t.Fatal(err) + } + provider := newOperationSecrets() + if err := provider.Set(t.Context(), secretName, credential); err != nil { + t.Fatal(err) + } + insertProviderRow(t, db, alias, secretName, "startup-stop-model") + if _, err := db.Exec(`INSERT INTO provider_operations + (operation_id, alias, state, failure, secret_name, result_type, result_model, + result_is_default, created_at, updated_at, expires_at) + VALUES (?, ?, 'applied', '', ?, 'openai', 'startup-stop-model', 1, + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, datetime('now', '+1 day'))`, + operationID, alias, secretName); err != nil { + t.Fatal(err) + } + + started := make(chan struct{}) + provider.getHook = func(ctx context.Context, key string) error { + if key != secretName { + return nil + } + close(started) + <-ctx.Done() + return ctx.Err() + } + manager := newProviderOperationManager(&EngineContext{ + DB: db, + SecretsProvider: provider, + SecretsRedactor: secrets.NewRedactor(), + }) + t.Cleanup(manager.Stop) + startDone := make(chan error, 1) + go func() { startDone <- manager.Start(t.Context()) }() + waitProviderOperationValue(t, started, "provider operation startup reconciliation") + + stopDone := make(chan struct{}) + go func() { + manager.Stop() + close(stopDone) + }() + err := waitProviderOperationValue(t, startDone, "canceled provider operation startup reconciliation") + if !errors.Is(err, context.Canceled) { + t.Fatalf("startup reconciliation error = %v, want context canceled", err) + } + if got, want := err.Error(), "finalize provider operation: provider operation finalization secret read canceled"; got != want { + t.Fatalf("startup reconciliation error = %q, want %q", got, want) + } + if strings.Contains(err.Error(), secretName) || strings.Contains(err.Error(), credential) { + t.Fatalf("startup reconciliation error leaked sensitive metadata: %q", err) + } + waitProviderOperationValue(t, stopDone, "provider operation stop") +} + type providerOperationSignalContext struct { context.Context onDone sync.Once diff --git a/internal/releaseguard/workflow_test.go b/internal/releaseguard/workflow_test.go index e5e029d..e43055f 100644 --- a/internal/releaseguard/workflow_test.go +++ b/internal/releaseguard/workflow_test.go @@ -156,6 +156,19 @@ func TestCIRequiresNativeWindowsBackgroundPersistence(t *testing.T) { "go test ./internal/acpclient -run '^TestBackgroundWindows' -count=1") } +func TestCIIsolatesACPClientLifecycleBinarySmoke(t *testing.T) { + workflow := loadWorkflow(t, ".github/workflows/ci.yml") + const command = "go test ./internal/acpclient -run '^TestACPClientLifecycleBinarySmoke' -count=1 -timeout=5m" + + linux := requireJob(t, workflow, "test") + requireRun(t, linux, "Run ACP client lifecycle smoke", command) + requireRun(t, linux, "Test with race detector", + "-skip '^(TestACPClientExecBinarySmoke|TestACPClientLifecycleBinarySmoke)'") + + windows := requireJob(t, workflow, "windows-conpty-smoke") + requireRun(t, windows, "Run ACP client lifecycle smoke", command) +} + func TestReleaseWorkflowPrePublishGuards(t *testing.T) { workflow := loadWorkflow(t, ".github/workflows/release.yml") job := requireJob(t, workflow, "release")