From 6a3f151af8b6693dbceb73ce51f4bb10e9f50eb3 Mon Sep 17 00:00:00 2001 From: Jon Langevin Date: Mon, 13 Jul 2026 13:11:31 -0400 Subject: [PATCH 01/16] docs: design retained provider lifecycle --- ...0002-retained-runner-provider-lifecycle.md | 36 ++++ ...tained-runner-provider-lifecycle-design.md | 169 ++++++++++++++++++ 2 files changed, 205 insertions(+) create mode 100644 decisions/0002-retained-runner-provider-lifecycle.md create mode 100644 docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md diff --git a/decisions/0002-retained-runner-provider-lifecycle.md b/decisions/0002-retained-runner-provider-lifecycle.md new file mode 100644 index 0000000..551c149 --- /dev/null +++ b/decisions/0002-retained-runner-provider-lifecycle.md @@ -0,0 +1,36 @@ +# 0002. Keep retained runner-provider lifecycle plugin-owned + +**Status:** Accepted +**Date:** 2026-07-13 +**Related:** `docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md` + +## Context + +The GitHub runner provider needs a long-lived credential-bearing local service, +but retained-agent updates must not depend on a workflow-compute GitHub Actions +installer. Moving only the provider binary into this plugin would leave service +installation, update, and rollback behavior app-owned. + +## Decision + +Ship the retained Linux lifecycle adapter as subcommands of the existing +`github-runner-provider` release binary. One manual user-scope install creates +systemd/Podman wiring. Subsequent provider updates are driven by signed +workflow-compute package markers and cryptographically verified with the +provider-neutral compute-agent command before plugin-owned candidate preflight +and activation. + +Do not place GitHub API logic, provider credentials, provider service units, or +provider update orchestration in workflow-compute. Do not put STG API tokens on +the retained host for lifecycle observation. + +## Consequences + +- Provider lifecycle and rollback release with the provider implementation. +- Workflow-compute remains responsible only for generic package delivery, + maintenance fencing, provenance verification, dispatch, proof, and artifacts. +- Linux user-systemd/Podman is the first adapter. A generic managed-sidecar + contract remains deferred until another provider supplies evidence that the + lifecycle is reusable rather than merely similar. +- The stable launcher must preserve backward compatibility with versioned local + lifecycle config/state across provider updates. diff --git a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md new file mode 100644 index 0000000..da5d802 --- /dev/null +++ b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md @@ -0,0 +1,169 @@ +# Retained GitHub Runner Provider Lifecycle Design + +**Status:** Approved as a fix-forward refinement of the workspace GitHub Provider dogfood design +**Date:** 2026-07-13 +**Project:** Workflow-Compute slimming closure +**Related:** workspace `docs/plans/2026-06-26-github-provider-dogfood-agents-design.md`, Task 4 and Task 8 of its implementation plan + +## Problem + +The GitHub runner provider keeps its GitHub credential outside ephemeral job +containers, so a retained agent needs a local provider service. The discarded +workflow-compute implementation installed and refreshed that service from a +repository-owned GitHub workflow. That shape made provider updates depend on +GitHub Actions, left provider lifecycle in the private app repo, and required a +static STG read token on the host. + +The retained Linux proof needs one user-scope install, autonomous updates from +signed workflow-compute package campaigns, candidate validation, rollback, and +an independently invokable uninstall path. + +## Global Design Guidance + +Source: workspace `docs/design-guidance.md`. + +| guidance | design response | +|---|---| +| Workflow/plugin ecosystem is the substrate | GitHub-specific lifecycle ships in this plugin; workflow-compute supplies only generic signed package delivery and maintenance/provenance commands. | +| Reuse over rebuild | Reuse `github-runner-provider`, retained supervisor packages, user systemd, and Podman. | +| Primary language Go; strict boundaries | Lifecycle state/config/evidence use typed Go structs and strict decoders; no shell/JQ state machine or new `map[string]any` boundary. | +| Secrets never logged | Credentials enter through environment, are written only to mode-0600 files, and are excluded from status/evidence. | +| Multi-component validation | Release package, retained host, systemd/Podman, STG campaign, provider API, GitHub runner job, and STG proof/artifact APIs are all exercised. | + +## Approaches + +| approach | trade-off | decision | +|---|---|---| +| App-owned GitHub workflow installer | Existing code is available, but updates depend on GitHub and provider logic remains in workflow-compute. | Rejected. | +| Plugin-owned retained lifecycle with marker-triggered refresh | Smallest provider-owned path; one install, then server-mediated updates; Linux/systemd-specific adapter is explicit. | Selected. | +| Generic supervisor-managed sidecar framework | Strong long-term reuse, but expands core contracts/process supervision before one real provider proves the lifecycle requirements. | Deferred until a second provider needs the same mechanism. | + +## Architecture + +`github-runner-provider` gains commands that do not require provider service +environment parsing: + +- `version`: side-effect-free package probe. +- `probe`: authenticated TLS readiness and semantic GitHub preflight from + inside a candidate/stable provider container. +- `retained install`: one-time user-scope install/reinstall transaction. +- `retained refresh`: verify the current signed provider package, stage a + candidate container, preflight it, atomically activate it, and restart only + the provider service. +- `retained serve-active`: validate durable active state and exec Podman for + the selected immutable image ID. +- `retained status`: emit redacted machine-readable service/package state. +- `retained uninstall`: remove user-scope provider wiring under the same + maintenance fence; purge of provider state is explicit. + +The install transaction copies the reviewed provider binary to a stable +launcher path, writes provider and agent environment files, creates TLS +material, writes user-systemd provider/refresh/path units plus the retained +agent drop-in, activates the current signed package, and restarts the agent. +The installer uses `compute-agent supervisor-maintenance` and the local agent +status file; it never reads STG leases and receives no STG API token. + +The path unit watches the exact supervisor current-package marker. A marker +change runs `retained refresh`. Refresh uses `compute-agent supervisor-update +verify` to cryptographically bind worker, directive, artifact, path, and digest +before copying bytes. It builds a digest-unique scratch image, starts a +candidate with cloned provider state, runs authenticated readiness plus GitHub +preflight, and only then replaces durable active state and restarts the stable +provider. Candidate failure leaves the prior service and active state intact. + +The agent receives only provider URL, provider API token, and CA certificate. +The GitHub token remains in the provider container environment and is never +forwarded to the ephemeral runner-job container. + +## Integration Matrix + +| integration | classification | proof | +|---|---|---| +| `github-runner-provider` release binary | runtime-integrated | release archive runs `version`; digest equals promoted STG artifact. | +| `compute-agent` maintenance/update verification | runtime-integrated | real retained install verifies the current marker and drains/restarts the same worker identity. | +| user systemd + rootless Podman | runtime-integrated, Linux | service/path/refresh units active; candidate and stable authenticated probes pass. | +| GitHub org runner API | runtime-integrated | semantic preflight and one ephemeral job lifecycle succeed. | +| STG package campaign | runtime-integrated | a later plugin version promotes and refreshes without a GitHub install workflow. | +| STG task/proof/artifact APIs | runtime-integrated | accepted provider task and canonical artifact refs are retrieved from STG. | +| macOS/Windows retained provider lifecycle | deferred | runner-job payload is currently Linux-only; this slice proves retained Linux before adding launchd/Windows adapters. | + +## Security Review + +- Install paths must be absolute, under the invoking user's home, owned by that + user, and symlink-free. Generated files use atomic replacement and restrictive + modes. +- The executing installer binary must hash to the verified promoted artifact; + direct or stale release binaries cannot establish an unrelated package. +- Package verification is delegated to the compute-agent cryptographic reader; + this plugin does not duplicate signature-shape checks. +- Provider and agent tokens are rejected when empty or containing line breaks. + Commands and evidence never include credential values. +- Candidate provider state is a regular-file-only clone. Candidate failure or + interrupted activation preserves prior active state and service. +- Rootless Podman runs with a read-only root, dropped capabilities, no-new- + privileges, explicit state/TLS mounts, and no socket mount. Ephemeral workload + containers receive only the provider API credential. +- The provider API remains authenticated over a plugin-generated private CA; + readiness and semantic preflight require the provider token. +- Uninstall retains state and credentials unless explicit purge is requested. + +## Infrastructure Impact + +- Creates user-owned files below `~/.workflow-compute/github-runner-provider` + and user-systemd units/drop-ins below `~/.config/systemd/user`. +- Builds local rootless Podman images and runs one provider container. +- Adds no cloud resources, database migrations, public ports, or production + deployment. The first runtime proof is STG only. +- Initial install needs a self-hosted workflow on the retained Linux host. Once + installed, package refreshes are triggered by STG campaigns and local marker + observation, not GitHub workflows. + +## Multi-Component Validation + +1. Unit tests use fake command execution and filesystem roots to prove strict + config, transaction ordering, secret exclusion, candidate failure rollback, + marker-triggered unit content, status, and uninstall behavior. +2. Build Linux amd64/arm64 provider binaries and run `version` without provider + credentials. +3. Launch a real local provider process with TLS and an HTTP fake for the GitHub + boundary; run the real `probe` command. +4. Release the plugin and publish an executable, probe-capable provider package + through STG. +5. Run one retained Linux install workflow, then publish a second package + campaign and prove systemd refresh occurs without another install workflow. +6. Dispatch the real ephemeral GitHub runner workload from STG. Validate worker, + proof, logs, and artifact refs through STG; GitHub output alone is not proof. +7. Exercise the separate manual uninstall workflow only after update/reconnect + evidence is retained. + +## Assumptions + +| id | assumption | failure response | +|---|---|---| +| A1 | Retained Linux runs user systemd with lingering and rootless Podman. | Install preflight fails before mutation and emits a redacted diagnostic. | +| A2 | Podman default bridge provides name resolution between provider and workload containers. | Runtime proof fails before dogfood rollout; configure an explicit rootless network in a fix-forward release. | +| A3 | Current-package marker replacement is observable by a systemd path unit. | Runtime update proof must show the path unit invocation; otherwise replace it with a bounded user timer watching digest state. | +| A4 | Existing provider state consists only of regular files/directories. | Refresh rejects unsupported entries and preserves the active service. | +| A5 | Provider launcher schema remains backward-compatible across plugin updates. | Version the lifecycle config/state and fail closed before activation. | + +## Self-Challenge + +1. A generic managed-sidecar framework may eventually be cleaner. It is not + justified until another provider demonstrates identical lifecycle needs. +2. A path unit can miss or coalesce events. Refresh is idempotent and status + compares active/current digests; STG rollout proof must show actual version + transition, not merely an active unit. +3. Same-user host compromise can read provider files despite containerization. + The security boundary is untrusted workload versus trusted retained agent; + hardware-backed host isolation is explicitly outside this slimming slice. + +## Rollback + +- Candidate failure retains the previous active image/state and running service. +- A bad accepted release can be rolled back by a signed STG campaign to a prior + plugin version; the same refresh path preflights it before activation. +- `retained uninstall` removes provider units and the agent drop-in while + preserving worker identity and, by default, provider state. +- The dogfood workflow can return to the existing non-provider runner labels; + no production deployment is part of this design. + From 3387f91785e436a9416e684e29e5d6265c024233 Mon Sep 17 00:00:00 2001 From: Jon Langevin Date: Mon, 13 Jul 2026 13:20:00 -0400 Subject: [PATCH 02/16] docs: harden retained lifecycle design --- ...runner-provider-lifecycle-design-review.md | 63 +++++++++++++++++++ ...tained-runner-provider-lifecycle-design.md | 50 ++++++++++----- 2 files changed, 99 insertions(+), 14 deletions(-) create mode 100644 docs/plans/2026-07-13-retained-runner-provider-lifecycle-design-review.md diff --git a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design-review.md b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design-review.md new file mode 100644 index 0000000..306a82c --- /dev/null +++ b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design-review.md @@ -0,0 +1,63 @@ +### Adversarial Review Report + +**Phase:** design +**Artifact:** `docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md` +**Status:** PASS + +**Findings (Important):** + +- `D1` [Missing failure modes] `Architecture`: a systemd path event alone did + not guarantee catch-up after an inactive user session, coalesced event, or + reboot. Recommendation: add an idempotent boot/periodic reconciliation timer. + _Resolution: design now requires path plus bounded timer and a missed-event + runtime proof._ +- `D2` [Declared integration proof] `Architecture` / assumption A2: probing from + inside the provider container did not prove the ephemeral workload container + could resolve and authenticate the provider across Podman bridge networking. + Recommendation: probe from a separate workload-shaped container on the same + network. _Resolution: candidate and stable activation now require the separate + bridge-container probe._ +- `D3` [Rollback story] `Architecture` / `Rollback`: atomic active-state replace + did not define restart-mid-activation recovery or preservation of the prior + Podman image. Recommendation: use a crash-durable journal and retain referenced + current/prior immutable image IDs. _Resolution: design now requires + prepare/activate/commit journal recovery and image retention._ + +**Findings (Minor):** + +- `D4` [Infrastructure impact] Credential rotation was implicit in reinstall + but not named. Recommendation: state the rotation transaction and identity/ + state preservation. _Resolution: added to Infrastructure Impact._ + +**Bug-class scan transcript:** + +| Class | Result | Note | +|---|---|---| +| Project-guidance conflicts | Clean | Plugin ownership, Go implementation, secret handling, and real proof follow workspace guidance. | +| Assumptions under attack | Finding | A2/A3 were converted from rollout-time guesses into activation-time probes and timer reconciliation. | +| Repo-precedent conflicts | Clean | Existing provider binary/package and retained user-systemd/Podman patterns are reused without app-owned lifecycle. | +| Artifact-class precedent | Clean | Lifecycle remains a provider release command; scenario code will only orchestrate real STG proof. | +| YAGNI violations | Clean | Generic managed-sidecar framework remains explicitly deferred. | +| Missing failure modes | Finding | D1 and D3 address missed events and interruption during activation. | +| Security / privacy | Clean | GitHub credential remains provider-only; host receives no STG read token; package verification is delegated to compute-agent. | +| Infrastructure impact | Finding | D4 makes credential rotation and persistent user units/images explicit. | +| Multi-component validation | Clean | Release, STG campaign, host service, GitHub API/job, and STG artifacts are all runtime-proven. | +| Declared integration proof | Finding | D2 adds the missing workload-side network/DNS/TLS proof. | +| Contributed UI rendering proof | Clean | No UI contribution is declared. | +| Rollback story | Finding | D3 adds durable recovery and rollback image retention. | +| Simpler alternative | Clean | App-owned script and generic supervisor framework are compared and rejected/deferred with reasons. | +| User-intent drift | Clean | Initial workflow install is separated from autonomous STG-driven updates and manual uninstall. | +| Existence / runtime-validity | Clean | Existing release binary, provider endpoints, compute-agent commands, package markers, systemd, and Podman are named and must be runtime-probed before rollout. | + +**Options the author may not have considered:** + +1. A generic supervisor sidecar manager would remove systemd-specific provider + lifecycle, but it expands workflow-compute before a second provider proves a + reusable contract. Defer with evidence. +2. A timer-only reconciler is simpler than path plus timer, but increases update + latency. Path plus bounded timer keeps immediate updates and recovery. + +**Verdict reasoning:** The initial design had three tangible reliability/proof +gaps. They are resolved in the artifact without expanding provider ownership or +weakening the credential boundary. Remaining platform adapters are explicitly +deferred, so the revised Linux lifecycle design passes. diff --git a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md index da5d802..25dbd5a 100644 --- a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md +++ b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md @@ -64,12 +64,24 @@ The installer uses `compute-agent supervisor-maintenance` and the local agent status file; it never reads STG leases and receives no STG API token. The path unit watches the exact supervisor current-package marker. A marker -change runs `retained refresh`. Refresh uses `compute-agent supervisor-update -verify` to cryptographically bind worker, directive, artifact, path, and digest -before copying bytes. It builds a digest-unique scratch image, starts a -candidate with cloned provider state, runs authenticated readiness plus GitHub -preflight, and only then replaces durable active state and restarts the stable -provider. Candidate failure leaves the prior service and active state intact. +change runs `retained refresh`. A bounded user timer runs the same idempotent +refresh at boot and periodically, so a coalesced path event, disabled user +session, or server/agent restart still catches up. Refresh is serialized by a +user-owned OS lock and uses `compute-agent supervisor-update verify` to +cryptographically bind worker, directive, artifact, path, and digest before +copying bytes. + +Refresh writes a crash-durable transaction journal before mutation, builds a +digest-unique scratch image, starts a candidate with cloned provider state, and +runs authenticated readiness plus GitHub preflight from a separate probe +container on the same `--network bridge` used by provider workloads. This proves +container-name DNS and TLS from the workload side of the boundary, not merely +from inside the provider container. Only then does refresh fsync and atomically +replace durable active state, restart the stable provider, verify it again from +the separate probe container, mark the journal committed, and retain the prior +active image/state as rollback material. Candidate or stable activation failure +restores the prior active record and service. Startup recovery finishes or +rolls back an interrupted journal before any new refresh. The agent receives only provider URL, provider API token, and CA certificate. The GitHub token remains in the provider container environment and is never @@ -100,6 +112,9 @@ forwarded to the ephemeral runner-job container. Commands and evidence never include credential values. - Candidate provider state is a regular-file-only clone. Candidate failure or interrupted activation preserves prior active state and service. +- Refresh uses a single-owner lock and crash-durable prepare/activate/commit + journal. The current and immediately previous immutable image IDs are retained; + cleanup never removes rollback material referenced by active recovery state. - Rootless Podman runs with a read-only root, dropped capabilities, no-new- privileges, explicit state/TLS mounts, and no socket mount. Ephemeral workload containers receive only the provider API credential. @@ -117,12 +132,16 @@ forwarded to the ephemeral runner-job container. - Initial install needs a self-hosted workflow on the retained Linux host. Once installed, package refreshes are triggered by STG campaigns and local marker observation, not GitHub workflows. +- Credential rotation is an idempotent reinstall operation. It rewrites secret + files under maintenance, re-preflights provider/agent wiring, and preserves + worker identity and provider journal state. ## Multi-Component Validation 1. Unit tests use fake command execution and filesystem roots to prove strict config, transaction ordering, secret exclusion, candidate failure rollback, - marker-triggered unit content, status, and uninstall behavior. + crash-journal recovery, path+timer unit content, status, and uninstall + behavior. 2. Build Linux amd64/arm64 provider binaries and run `version` without provider credentials. 3. Launch a real local provider process with TLS and an HTTP fake for the GitHub @@ -130,7 +149,9 @@ forwarded to the ephemeral runner-job container. 4. Release the plugin and publish an executable, probe-capable provider package through STG. 5. Run one retained Linux install workflow, then publish a second package - campaign and prove systemd refresh occurs without another install workflow. + campaign and prove path/timer refresh plus reconnect occurs without another + install workflow. Restart the user service manager or retained agent between + promotion and observation to prove catch-up after a missed immediate event. 6. Dispatch the real ephemeral GitHub runner workload from STG. Validate worker, proof, logs, and artifact refs through STG; GitHub output alone is not proof. 7. Exercise the separate manual uninstall workflow only after update/reconnect @@ -141,8 +162,8 @@ forwarded to the ephemeral runner-job container. | id | assumption | failure response | |---|---|---| | A1 | Retained Linux runs user systemd with lingering and rootless Podman. | Install preflight fails before mutation and emits a redacted diagnostic. | -| A2 | Podman default bridge provides name resolution between provider and workload containers. | Runtime proof fails before dogfood rollout; configure an explicit rootless network in a fix-forward release. | -| A3 | Current-package marker replacement is observable by a systemd path unit. | Runtime update proof must show the path unit invocation; otherwise replace it with a bounded user timer watching digest state. | +| A2 | Podman `--network bridge` provides name resolution between provider and workload-shaped containers. | Every candidate/stable activation runs the real probe from a separate bridge container and fails closed before rollout if name resolution or TLS fails. | +| A3 | Current-package marker replacement is observable by a systemd path unit. | A bounded boot/periodic timer reconciles current versus active digest even when path observation is missed. | | A4 | Existing provider state consists only of regular files/directories. | Refresh rejects unsupported entries and preserves the active service. | | A5 | Provider launcher schema remains backward-compatible across plugin updates. | Version the lifecycle config/state and fail closed before activation. | @@ -150,9 +171,9 @@ forwarded to the ephemeral runner-job container. 1. A generic managed-sidecar framework may eventually be cleaner. It is not justified until another provider demonstrates identical lifecycle needs. -2. A path unit can miss or coalesce events. Refresh is idempotent and status - compares active/current digests; STG rollout proof must show actual version - transition, not merely an active unit. +2. A path unit can miss or coalesce events. The timer is required, refresh is + idempotent, and status compares active/current digests; STG rollout proof must + show actual version transition, not merely an active unit. 3. Same-user host compromise can read provider files despite containerization. The security boundary is untrusted workload versus trusted retained agent; hardware-backed host isolation is explicitly outside this slimming slice. @@ -160,10 +181,11 @@ forwarded to the ephemeral runner-job container. ## Rollback - Candidate failure retains the previous active image/state and running service. + Startup recovery consumes the fsynced transaction journal after interruption; + the current and prior immutable image IDs are never removed while referenced. - A bad accepted release can be rolled back by a signed STG campaign to a prior plugin version; the same refresh path preflights it before activation. - `retained uninstall` removes provider units and the agent drop-in while preserving worker identity and, by default, provider state. - The dogfood workflow can return to the existing non-provider runner labels; no production deployment is part of this design. - From add1618699a684c9d1fa9d3779863089776ba8d2 Mon Sep 17 00:00:00 2001 From: Jon Langevin Date: Mon, 13 Jul 2026 13:21:22 -0400 Subject: [PATCH 03/16] docs: plan retained provider lifecycle --- ...7-13-retained-runner-provider-lifecycle.md | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 docs/plans/2026-07-13-retained-runner-provider-lifecycle.md diff --git a/docs/plans/2026-07-13-retained-runner-provider-lifecycle.md b/docs/plans/2026-07-13-retained-runner-provider-lifecycle.md new file mode 100644 index 0000000..4488144 --- /dev/null +++ b/docs/plans/2026-07-13-retained-runner-provider-lifecycle.md @@ -0,0 +1,148 @@ +# Retained GitHub Runner Provider Lifecycle Implementation Plan + +> **For the implementing agent:** REQUIRED SUB-SKILL: Use autodev:executing-plans to implement this plan task-by-task. + +**Goal:** Ship a plugin-owned retained Linux provider install/refresh/uninstall lifecycle whose updates arrive through signed workflow-compute package campaigns and whose GitHub workloads return canonical proof through STG. + +**Architecture:** Extend the existing `github-runner-provider` binary with side-effect-free version/probe commands and retained lifecycle subcommands. A typed Go lifecycle package owns strict local config/state, crash recovery, Podman candidate activation, and user-systemd units; workflow-compute remains responsible only for generic maintenance fencing, cryptographic package verification, dispatch, proof, and artifacts. + +**Tech Stack:** Go stdlib, user systemd, rootless Podman, workflow-compute `compute-agent` CLI, GitHub Runner Provider HTTPS API, GoReleaser v2. + +**Base branch:** main + +--- + +## Scope Manifest + +**PR Count:** 1 +**Tasks:** 5 +**Estimated Lines of Change:** ~1500 + +**Out of scope:** +- Generic supervisor-managed sidecar framework. +- macOS launchd or Windows service/tray lifecycle for the Linux-only runner-job payload. +- App-owned workflow-compute provider installer or provider-specific server API. +- Production deployment or destructive production changes. +- Treating GitHub workflow artifacts as canonical workload proof. + +**PR Grouping:** + +| PR # | Title | Tasks | Branch | +|------|-------|-------|--------| +| 1 | Plugin-owned retained runner-provider lifecycle | Task 1, Task 2, Task 3, Task 4, Task 5 | codex/provider-retained-installer-20260713 | + +**Status:** Draft + +## Integration Matrix + +| integration | classification | task/proof | +|---|---|---| +| provider release binary | runtime-integrated | Task 1 and Task 5 launch `version` and `probe`; archive contains the command | +| compute-agent maintenance/update commands | runtime-integrated | Task 4 fake-command ordering; global plan Task 8 real retained host | +| user systemd and rootless Podman | runtime-integrated | Task 3/4 rendered units and local runtime launch; global plan Task 8 retained Linux apply | +| GitHub org runner API | runtime-integrated | Task 1 semantic preflight against HTTP fake; global plan Task 8 live org preflight/job | +| STG package campaign/task/proof/artifacts | runtime-integrated | global locked dogfood plan Task 8 after plugin release | +| macOS/Windows retained provider lifecycle | deferred | Linux-only runner-job package and explicit design non-goal | + +### Task 1: Provider Version And Workload-Side Probe + +**Files:** +- Modify: `cmd/github-runner-provider/main.go` +- Modify/Test: `cmd/github-runner-provider/main_test.go` +- Create/Test: `cmd/github-runner-provider/probe.go` +- Modify: `.goreleaser.yaml` +- Modify/Test: `release_packaging_test.go` + +**Steps:** +1. Add RED command tests proving `version` succeeds without provider credentials, unknown subcommands fail, legacy address invocation still serves, and `probe` rejects missing token/CA/HTTPS URL or unexpected JSON fields. +2. Add RED HTTP tests with a TLS server proving `probe` authenticates `GET /readyz`, then posts the strict org preflight request and emits a typed redacted result containing readiness, org/group/ref/workflow, runner-group id, resolved SHA, conflict count, and timestamp. +3. Run `GOWORK=off go test ./cmd/github-runner-provider -run 'Version|Probe|Dispatch' -count=1`; expected RED on missing dispatch/probe symbols or behavior. +4. Implement explicit command dispatch. Preserve no-subcommand and `host:port` service compatibility; `version` must print `internal.Version` and perform no environment/config reads. +5. Implement the bounded TLS probe using typed request/result structs, a private CA pool, provider-token bearer auth, strict single-value JSON decoding, response-size limits, and no credential-bearing errors/output. +6. Add the provider binary `-X .../internal.Version={{.Version}}` release ldflag and packaging assertions that the rendered archive command reports the release version. +7. Run focused tests, `GOWORK=off go test ./... -count=1`, `go vet ./...`, Linux/Windows/macOS cross-builds, and launch a built binary with `version`; expected exact non-empty version and exit 0. +8. Rollback: revert Task 1 commit; legacy provider service address invocation remains the prior behavior. +9. Commit: `feat(provider): add versioned readiness probe`. + +### Task 2: Strict Retained Lifecycle State And Recovery + +**Files:** +- Create: `internal/retainedprovider/config.go` +- Create: `internal/retainedprovider/state.go` +- Create: `internal/retainedprovider/files.go` +- Create/Test: `internal/retainedprovider/state_test.go` +- Create/Test: `internal/retainedprovider/files_test.go` + +**Steps:** +1. Add RED tests for versioned typed config, active state, transaction journal, verified-update projection, and redacted status. Reject unknown JSON fields, relative/out-of-home paths, unsafe unit/profile/component identifiers, line-breaking secrets, symlinked ancestors/files, wrong ownership/modes, malformed SHA256/image IDs, and mismatched worker/plugin/component identity. +2. Add RED filesystem tests for atomic mode-0600 writes with file/directory sync, regular-file-only bounded state cloning, install lock exclusivity, current/prior image reference retention, and prepare/activate/commit recovery after interruption at every journal phase. +3. Run `GOWORK=off go test ./internal/retainedprovider -run 'Config|State|Journal|Files|Recovery' -count=1`; expected RED on missing package/types. +4. Implement minimal typed structs and validation. Do not expose `map[string]any`; JSON decoders disallow unknown fields and multiple values. +5. Implement symlink-free user-home path validation, bounded secure copy, atomic durable writes, OS lock abstraction, and idempotent journal recovery that either restores prior active state or completes a verified commit. +6. Run focused tests, `GOWORK=off go test -race ./internal/retainedprovider -count=1`, full tests, `go vet ./...`, and `git diff --check`; expected PASS. +7. Rollback: revert Task 2 commit; no runtime wiring consumes the new package yet. +8. Commit: `feat(provider): add durable retained state`. + +### Task 3: Verified Podman Candidate Activation + +**Files:** +- Create: `internal/retainedprovider/command.go` +- Create: `internal/retainedprovider/refresh.go` +- Create/Test: `internal/retainedprovider/refresh_test.go` +- Modify: `cmd/github-runner-provider/main.go` +- Create/Test: `cmd/github-runner-provider/retained_test.go` + +**Steps:** +1. Add RED tests with a recording command runner for exact `compute-agent supervisor-update verify` arguments and typed output. Reject unverified identity/digest/path, installer self-digest mismatch during first install, and credential values in argv/errors/status. +2. Add RED refresh tests for serialized execution, digest-idempotent no-op, scratch image build with a static `FROM scratch` container file, immutable image-ID capture, candidate state clone, restrictive Podman flags, and separate bridge probe-container invocation. +3. Add RED failure tests for build/candidate/probe/stable failures and cancellation. Assert previous active state/service remains selected, journal recovery is possible, candidate containers are removed, and current/prior image IDs are not pruned. +4. Run `GOWORK=off go test ./internal/retainedprovider ./cmd/github-runner-provider -run 'Verify|Refresh|Candidate|Rollback|Retained' -count=1`; expected RED. +5. Implement bounded command execution without a shell. Parse verified-update JSON into strict typed projection, hash copied bytes, build a digest-unique image, and run candidate/stable probe containers on `--network bridge` with read-only root, dropped capabilities, no-new-privileges, and explicit mounts. +6. Implement prepare/activate/commit journal transitions with directory sync and recovery. Update active state only after candidate preflight; verify stable from a separate container after restart; roll back on failure. +7. Add `retained refresh` command wiring that loads strict config and emits only redacted typed status. +8. Run focused/race/full tests, `go vet ./...`, cross-build all release targets, and `git diff --check`; expected PASS. +9. Rollback: revert Task 3 commit; no service unit invokes refresh before Task 4. +10. Commit: `feat(provider): refresh retained provider safely`. + +### Task 4: User-Systemd Install, Status, And Uninstall + +**Files:** +- Create: `internal/retainedprovider/systemd.go` +- Create/Test: `internal/retainedprovider/systemd_test.go` +- Modify: `internal/retainedprovider/refresh.go` +- Modify: `cmd/github-runner-provider/main.go` +- Modify/Test: `cmd/github-runner-provider/retained_test.go` + +**Steps:** +1. Add RED golden/semantic tests for provider service, refresh service, marker path unit, boot/periodic timer, and retained-agent environment drop-in. Assert absolute escaped paths, no shell, stable launcher path, exact `--network bridge`, restart policy, no public port/socket mount, and no secret literals in units. +2. Add RED transaction tests proving install/reinstall ordering: preflight -> package verify/self-hash -> maintenance begin -> local status unavailable with empty task/lease -> stop agent -> durable files/units -> daemon-reload -> provider activation/probe -> start same agent -> status unavailable under same maintenance ID -> maintenance end -> idle/online observation. +3. Add RED tests that maintenance remains active on incomplete rollback, exact maintenance ID is required, transient local status is bounded, no STG token/API call is used, and credential rotation preserves worker/provider state. +4. Add RED uninstall tests proving a separate invocation fences the worker, disables/removes provider path/timer/services and agent drop-in, restarts the same retained agent, and preserves state/secrets unless `--purge` is explicit. +5. Run `GOWORK=off go test ./internal/retainedprovider ./cmd/github-runner-provider -run 'Systemd|Install|Reinstall|Status|Uninstall|Maintenance' -count=1`; expected RED. +6. Implement unit rendering and the install/reinstall/status/uninstall state machines with bounded systemctl/compute-agent calls, local status parsing, atomic files, rollback, and redacted evidence. +7. Wire `retained install|refresh|serve-active|status|uninstall`. Return an explicit unsupported-platform error outside Linux without breaking service/version/probe commands. +8. Run focused/race/full tests, `go vet ./...`, cross-build Linux/darwin/windows amd64/arm64, and `git diff --check`; expected PASS. +9. Runtime launch validation: in an isolated Linux user-systemd/Podman environment, render/install units against fake compute-agent/provider endpoints, fire path and timer events, observe refresh invocation, run status, then uninstall. Expected no leaked secrets, provider ready, and original agent unit active after uninstall. +10. Rollback: run `retained uninstall` without purge or revert Task 4 commit; restore the prior agent drop-in backup and restart the unchanged worker unit. +11. Commit: `feat(provider): install retained provider service`. + +### Task 5: Release Contract And Global Dogfood Handoff + +**Files:** +- Modify: `README.md` +- Modify: `.goreleaser.yaml` +- Modify/Test: `release_packaging_test.go` +- Modify: `docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md` +- Create: `docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md` + +**Steps:** +1. Add RED release tests requiring the versioned provider binary and retained lifecycle metadata/docs while keeping `github-actions-runner-job` Linux-only and existing archives compatible. +2. Document initial install/reinstall, autonomous STG campaign refresh, status, credential rotation, and separate uninstall. State that GitHub workflow output is orchestration evidence only. +3. Run `GOWORK=off go test ./... -count=1`, `GOWORK=off go test -race ./internal/retainedprovider ./cmd/github-runner-provider -count=1`, `go vet ./...`, `goreleaser release --snapshot --clean`, archive extraction plus `github-runner-provider version`, all release-target cross-builds, and `git diff --check`; expected PASS. +4. Run adversarial self code review over the complete diff. Scan secret flow, argv/env leakage, symlink/ownership checks, crash points, lock release, rollback, systemd escaping, Podman isolation/networking, unsupported OS behavior, and legacy CLI compatibility; fix every Critical/Important finding. +5. Open the plugin PR, request Copilot, monitor checks/threads until green, and admin-merge only with no unresolved findings. Verify merged commit equals intended release tag before tagging. +6. Publish the next plugin release and verify non-draft release assets, checksums, provider `version`, Linux runner-job image, and registry notification. +7. Hand off to Task 8 of the global locked plan: publish the executable/probe-capable provider package to STG; run one manual retained Linux install; dispatch a real GitHub ephemeral job from STG and validate proof/log/artifact refs through STG; publish a second package version or campaign, restart agent/user manager, and prove autonomous catch-up without another install workflow. +8. Keep uninstall as a separate manual workflow and do not run it until retained update/reconnect evidence is complete. +9. Rollback: pin/re-promote the previous plugin release, or invoke retained uninstall without purge; revert dogfood runner labels if STG proof fails. +10. Commit: `docs: document retained provider lifecycle`. From a40a5d65c4d7e1a4da28a297c4bab50cdd1f7b42 Mon Sep 17 00:00:00 2001 From: Jon Langevin Date: Mon, 13 Jul 2026 13:22:58 -0400 Subject: [PATCH 04/16] docs: review retained lifecycle plan --- ...d-runner-provider-lifecycle-plan-review.md | 73 +++++++++++++++++++ ...7-13-retained-runner-provider-lifecycle.md | 53 +++++++------- 2 files changed, 101 insertions(+), 25 deletions(-) create mode 100644 docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md diff --git a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md new file mode 100644 index 0000000..7a945a9 --- /dev/null +++ b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md @@ -0,0 +1,73 @@ +### Adversarial Review Report + +**Phase:** plan +**Artifact:** `docs/plans/2026-07-13-retained-runner-provider-lifecycle.md` +**Status:** PASS + +**Findings (Important):** + +- `P1` [Missing failure modes] Task 3 named `serve-active` only in final CLI + wiring and did not test immutable image validation or foreground process + tracking. Recommendation: add RED tests and implementation steps for image-ID + inspect/match plus shell-free foreground exec. _Resolution: added to Task 3._ +- `P2` [Security / missing integration proof] Task 3's separate probe container + did not constrain its environment, so an implementation could accidentally + inherit the GitHub credential. Recommendation: require a provider-token-only + probe env file and assert absence of GitHub credentials. _Resolution: added to + Task 3._ +- `P3` [Missing rollback wiring] Task 4 omitted executable TLS and environment + file tests even though those files define the provider/workload credential + boundary and rollback inputs. Recommendation: add CA/SAN, mode, syntax, + separation, and transaction-order tests. _Resolution: added to Task 4._ + +**Findings (Minor):** + +- `P4` [Verification-class mismatch] Go change verification named tests/vet but + omitted the workspace-required golangci gate. _Resolution: Task 5 now runs + `golangci-lint --new-from-rev` or records/uses the repository CI equivalent if + no pinned tool exists._ + +**Bug-class scan transcript:** + +| Class | Result | Note | +|---|---|---| +| Project-guidance conflicts | Clean | Plugin ownership, Go, no STG host token, redaction, and real dogfood proof are mapped to tasks. | +| Assumptions under attack | Clean | Path observation, bridge DNS, user systemd, and state shape all have fail-closed tests/proofs. | +| Repo-precedent conflicts | Clean | Existing provider binary, GoReleaser, user-systemd, Podman, and compute-agent commands are reused. | +| Artifact-class precedent | Clean | Provider lifecycle ships in provider release; scenario remains orchestration only. | +| YAGNI violations | Clean | Generic sidecar framework and non-Linux adapters remain out of scope. | +| Missing failure modes | Finding | P1 added active-service validation and foreground lifecycle coverage. | +| Security / privacy | Finding | P2/P3 make credential separation and TLS material executable gates. | +| Infrastructure impact | Clean | Units/images/files are rendered and safe-host applied; production is excluded. | +| Multi-component validation | Clean | Task 5 hands off to real STG campaign/job/proof/artifact validation. | +| Declared integration proof | Finding | P2 completes workload-side probe environment validation. | +| Contributed UI rendering proof | Clean | No UI contribution. | +| Rollback story | Finding | P3 wires durable secret/TLS assets into install rollback tests. | +| Simpler alternative | Clean | Design rejects app script and defers generic supervisor framework. | +| User-intent drift | Clean | One install, autonomous updates, and separate uninstall are explicit. | +| Existence / runtime-validity | Clean | Real commands/endpoints/artifacts are launched or cross-built before release. | +| Over/under-decomposition | Clean | Five cohesive TDD tasks fit one provider PR; global STG proof remains in locked Task 8. | +| Verification-class mismatch | Finding | P4 adds the missing lint/static-analysis gate. | +| Auth/authz chain composition | Clean | Provider bearer/TLS and compute-agent local verification are server/crypto enforced, not client-asserted payload claims. | +| Hidden serial dependencies | Clean | Tasks are deliberately sequential and touch shared command/lifecycle files. | +| Missing rollback wiring | Clean | Each runtime-affecting task includes an explicit rollback action. | +| Missing integration proof | Clean | Local launched runtime plus post-release retained STG proof are required. | +| Missing declared integration matrix | Clean | Every integration is runtime-integrated or explicitly deferred. | +| Missing contributed UI route proof | Clean | Not applicable. | +| Infrastructure verification mismatch | Clean | Unit rendering, safe-host apply, path/timer events, status, and uninstall are required. | +| Plugin-loader runtime layout | Clean | This release command is archive-contained, not a host-discovered plugin child. | +| Config-validation schema rules | Clean | Typed lifecycle JSON and unit semantics have strict tests. | +| Identifier/naming match | Clean | Existing `GITHUB_RUNNER_PROVIDER_*`, `COMPUTE_*`, and command naming are preserved. | +| Planned-code compile-validity | Clean | Plan embeds no pseudo-implementation that could fail compilation. | + +**Options the author may not have considered:** + +1. A single install shell script is shorter, but repeats the rejected untyped, + app-owned lifecycle and is unsuitable for crash recovery. +2. A supervisor-native sidecar manager could replace systemd, but belongs in a + future cross-provider design after a second concrete consumer exists. + +**Verdict reasoning:** The initial plan omitted three load-bearing executable +gates. The revised tasks now cover active-image process semantics, probe secret +isolation, TLS/env boundaries, and the Go static-analysis gate. No unresolved +Critical or Important findings remain. diff --git a/docs/plans/2026-07-13-retained-runner-provider-lifecycle.md b/docs/plans/2026-07-13-retained-runner-provider-lifecycle.md index 4488144..3afba73 100644 --- a/docs/plans/2026-07-13-retained-runner-provider-lifecycle.md +++ b/docs/plans/2026-07-13-retained-runner-provider-lifecycle.md @@ -94,15 +94,16 @@ **Steps:** 1. Add RED tests with a recording command runner for exact `compute-agent supervisor-update verify` arguments and typed output. Reject unverified identity/digest/path, installer self-digest mismatch during first install, and credential values in argv/errors/status. -2. Add RED refresh tests for serialized execution, digest-idempotent no-op, scratch image build with a static `FROM scratch` container file, immutable image-ID capture, candidate state clone, restrictive Podman flags, and separate bridge probe-container invocation. +2. Add RED refresh tests for serialized execution, digest-idempotent no-op, scratch image build with a static `FROM scratch` container file, immutable image-ID capture, candidate state clone, restrictive Podman flags, and separate bridge probe-container invocation. The probe environment file must contain only the provider API token, never the GitHub credential. 3. Add RED failure tests for build/candidate/probe/stable failures and cancellation. Assert previous active state/service remains selected, journal recovery is possible, candidate containers are removed, and current/prior image IDs are not pruned. -4. Run `GOWORK=off go test ./internal/retainedprovider ./cmd/github-runner-provider -run 'Verify|Refresh|Candidate|Rollback|Retained' -count=1`; expected RED. -5. Implement bounded command execution without a shell. Parse verified-update JSON into strict typed projection, hash copied bytes, build a digest-unique image, and run candidate/stable probe containers on `--network bridge` with read-only root, dropped capabilities, no-new-privileges, and explicit mounts. -6. Implement prepare/activate/commit journal transitions with directory sync and recovery. Update active state only after candidate preflight; verify stable from a separate container after restart; roll back on failure. -7. Add `retained refresh` command wiring that loads strict config and emits only redacted typed status. -8. Run focused/race/full tests, `go vet ./...`, cross-build all release targets, and `git diff --check`; expected PASS. -9. Rollback: revert Task 3 commit; no service unit invokes refresh before Task 4. -10. Commit: `feat(provider): refresh retained provider safely`. +4. Add RED `serve-active` tests proving it validates the durable image ID against `podman image inspect`, refuses mutable/mismatched state, passes provider secrets only through the provider env file, and replaces itself with the tracked foreground Podman process without a shell. +5. Run `GOWORK=off go test ./internal/retainedprovider ./cmd/github-runner-provider -run 'Verify|Refresh|Candidate|Rollback|ServeActive|Retained' -count=1`; expected RED. +6. Implement bounded command execution without a shell. Parse verified-update JSON into strict typed projection, hash copied bytes, build a digest-unique image, and run candidate/stable probe containers on `--network bridge` with read-only root, dropped capabilities, no-new-privileges, and explicit mounts. +7. Implement prepare/activate/commit journal transitions with directory sync and recovery. Update active state only after candidate preflight; verify stable from a separate container after restart; roll back on failure. +8. Implement `serve-active` immutable-image validation/foreground exec and `retained refresh` command wiring. Load strict config and emit only redacted typed status. +9. Run focused/race/full tests, `go vet ./...`, cross-build all release targets, and `git diff --check`; expected PASS. +10. Rollback: revert Task 3 commit; no service unit invokes refresh before Task 4. +11. Commit: `feat(provider): refresh retained provider safely`. ### Task 4: User-Systemd Install, Status, And Uninstall @@ -115,16 +116,17 @@ **Steps:** 1. Add RED golden/semantic tests for provider service, refresh service, marker path unit, boot/periodic timer, and retained-agent environment drop-in. Assert absolute escaped paths, no shell, stable launcher path, exact `--network bridge`, restart policy, no public port/socket mount, and no secret literals in units. -2. Add RED transaction tests proving install/reinstall ordering: preflight -> package verify/self-hash -> maintenance begin -> local status unavailable with empty task/lease -> stop agent -> durable files/units -> daemon-reload -> provider activation/probe -> start same agent -> status unavailable under same maintenance ID -> maintenance end -> idle/online observation. -3. Add RED tests that maintenance remains active on incomplete rollback, exact maintenance ID is required, transient local status is bounded, no STG token/API call is used, and credential rotation preserves worker/provider state. -4. Add RED uninstall tests proving a separate invocation fences the worker, disables/removes provider path/timer/services and agent drop-in, restarts the same retained agent, and preserves state/secrets unless `--purge` is explicit. -5. Run `GOWORK=off go test ./internal/retainedprovider ./cmd/github-runner-provider -run 'Systemd|Install|Reinstall|Status|Uninstall|Maintenance' -count=1`; expected RED. -6. Implement unit rendering and the install/reinstall/status/uninstall state machines with bounded systemctl/compute-agent calls, local status parsing, atomic files, rollback, and redacted evidence. -7. Wire `retained install|refresh|serve-active|status|uninstall`. Return an explicit unsupported-platform error outside Linux without breaking service/version/probe commands. -8. Run focused/race/full tests, `go vet ./...`, cross-build Linux/darwin/windows amd64/arm64, and `git diff --check`; expected PASS. -9. Runtime launch validation: in an isolated Linux user-systemd/Podman environment, render/install units against fake compute-agent/provider endpoints, fire path and timer events, observe refresh invocation, run status, then uninstall. Expected no leaked secrets, provider ready, and original agent unit active after uninstall. -10. Rollback: run `retained uninstall` without purge or revert Task 4 commit; restore the prior agent drop-in backup and restart the unchanged worker unit. -11. Commit: `feat(provider): install retained provider service`. +2. Add RED TLS and environment tests: private CA/server certificate SANs cover loopback plus stable/candidate names; keys/provider/agent/probe env files are mode 0600; Podman env syntax and systemd EnvironmentFile syntax are rendered separately; agent env contains provider URL/token/CA only; provider env contains GitHub token only in addition to provider configuration; probe env contains provider token only. +3. Add RED transaction tests proving install/reinstall ordering: preflight -> package verify/self-hash -> maintenance begin -> local status unavailable with empty task/lease -> stop agent -> durable TLS/env/config/units -> daemon-reload -> provider activation/probe -> start same agent -> status unavailable under same maintenance ID -> maintenance end -> idle/online observation. +4. Add RED tests that maintenance remains active on incomplete rollback, exact maintenance ID is required, transient local status is bounded, no STG token/API call is used, and credential rotation preserves worker/provider state. +5. Add RED uninstall tests proving a separate invocation fences the worker, disables/removes provider path/timer/services and agent drop-in, restarts the same retained agent, and preserves state/secrets unless `--purge` is explicit. +6. Run `GOWORK=off go test ./internal/retainedprovider ./cmd/github-runner-provider -run 'Systemd|TLS|Environment|Install|Reinstall|Status|Uninstall|Maintenance' -count=1`; expected RED. +7. Implement TLS/environment generation, unit rendering, and install/reinstall/status/uninstall state machines with bounded systemctl/compute-agent calls, local status parsing, atomic files, rollback, and redacted evidence. +8. Wire `retained install|refresh|serve-active|status|uninstall`. Return an explicit unsupported-platform error outside Linux without breaking service/version/probe commands. +9. Run focused/race/full tests, `go vet ./...`, cross-build Linux/darwin/windows amd64/arm64, and `git diff --check`; expected PASS. +10. Runtime launch validation: in an isolated Linux user-systemd/Podman environment, render/install units against fake compute-agent/provider endpoints, fire path and timer events, observe refresh invocation, run status, then uninstall. Expected no leaked secrets, provider ready, and original agent unit active after uninstall. +11. Rollback: run `retained uninstall` without purge or revert Task 4 commit; restore the prior agent drop-in backup and restart the unchanged worker unit. +12. Commit: `feat(provider): install retained provider service`. ### Task 5: Release Contract And Global Dogfood Handoff @@ -139,10 +141,11 @@ 1. Add RED release tests requiring the versioned provider binary and retained lifecycle metadata/docs while keeping `github-actions-runner-job` Linux-only and existing archives compatible. 2. Document initial install/reinstall, autonomous STG campaign refresh, status, credential rotation, and separate uninstall. State that GitHub workflow output is orchestration evidence only. 3. Run `GOWORK=off go test ./... -count=1`, `GOWORK=off go test -race ./internal/retainedprovider ./cmd/github-runner-provider -count=1`, `go vet ./...`, `goreleaser release --snapshot --clean`, archive extraction plus `github-runner-provider version`, all release-target cross-builds, and `git diff --check`; expected PASS. -4. Run adversarial self code review over the complete diff. Scan secret flow, argv/env leakage, symlink/ownership checks, crash points, lock release, rollback, systemd escaping, Podman isolation/networking, unsupported OS behavior, and legacy CLI compatibility; fix every Critical/Important finding. -5. Open the plugin PR, request Copilot, monitor checks/threads until green, and admin-merge only with no unresolved findings. Verify merged commit equals intended release tag before tagging. -6. Publish the next plugin release and verify non-draft release assets, checksums, provider `version`, Linux runner-job image, and registry notification. -7. Hand off to Task 8 of the global locked plan: publish the executable/probe-capable provider package to STG; run one manual retained Linux install; dispatch a real GitHub ephemeral job from STG and validate proof/log/artifact refs through STG; publish a second package version or campaign, restart agent/user manager, and prove autonomous catch-up without another install workflow. -8. Keep uninstall as a separate manual workflow and do not run it until retained update/reconnect evidence is complete. -9. Rollback: pin/re-promote the previous plugin release, or invoke retained uninstall without purge; revert dogfood runner labels if STG proof fails. -10. Commit: `docs: document retained provider lifecycle`. +4. Run `golangci-lint run --new-from-rev=origin/main`; expected exit 0. If the repository has no pinned lint configuration/tool, record that fact and use the CI lint/static-analysis command from `.github/workflows` rather than silently skipping the gate. +5. Run adversarial self code review over the complete diff. Scan secret flow, argv/env leakage, symlink/ownership checks, crash points, lock release, rollback, systemd escaping, Podman isolation/networking, unsupported OS behavior, and legacy CLI compatibility; fix every Critical/Important finding. +6. Open the plugin PR, request Copilot, monitor checks/threads until green, and admin-merge only with no unresolved findings. Verify merged commit equals intended release tag before tagging. +7. Publish the next plugin release and verify non-draft release assets, checksums, provider `version`, Linux runner-job image, and registry notification. +8. Hand off to Task 8 of the global locked plan: publish the executable/probe-capable provider package to STG; run one manual retained Linux install; dispatch a real GitHub ephemeral job from STG and validate proof/log/artifact refs through STG; publish a second package version or campaign, restart agent/user manager, and prove autonomous catch-up without another install workflow. +9. Keep uninstall as a separate manual workflow and do not run it until retained update/reconnect evidence is complete. +10. Rollback: pin/re-promote the previous plugin release, or invoke retained uninstall without purge; revert dogfood runner labels if STG proof fails. +11. Commit: `docs: document retained provider lifecycle`. From fc7fff99778a262d3914118076e60805996395d3 Mon Sep 17 00:00:00 2001 From: Jon Langevin Date: Mon, 13 Jul 2026 13:23:47 -0400 Subject: [PATCH 05/16] docs: align retained lifecycle plan --- ...ned-runner-provider-lifecycle-alignment.md | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 docs/plans/2026-07-13-retained-runner-provider-lifecycle-alignment.md diff --git a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-alignment.md b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-alignment.md new file mode 100644 index 0000000..abd363f --- /dev/null +++ b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-alignment.md @@ -0,0 +1,40 @@ +### Alignment Report + +**Status:** PASS + +**Coverage:** + +| Design requirement | Plan task(s) | Status | +|---|---|---| +| Version/probe/install/refresh/serve/status/uninstall command surface | Tasks 1, 3, 4 | Covered | +| Preserve legacy provider service invocation | Tasks 1, 5 | Covered | +| Typed strict state/config; no new untyped boundary | Task 2 | Covered | +| No STG token on host; local maintenance/status observation | Task 4 | Covered | +| Cryptographically verified package and installer identity | Tasks 2, 3 | Covered | +| Path plus boot/periodic reconciliation | Tasks 2, 4 | Covered | +| Crash-durable activation journal and prior-image rollback | Tasks 2, 3 | Covered | +| Separate workload-side bridge DNS/TLS/API probe | Tasks 1, 3 | Covered | +| GitHub credential isolated from agent/probe/workload | Tasks 1, 3, 4 | Covered | +| User-systemd/rootless Podman hardening | Tasks 3, 4 | Covered | +| Reinstall credential rotation and separate uninstall | Task 4 | Covered | +| Linux-only first adapter; other OS adapters deferred | Tasks 4, 5 | Covered | +| Real release, retained host, STG campaign/job/proof/artifact validation | Task 5 plus parent plan Task 8 | Covered | +| Rollback without worker identity loss | Tasks 2-5 | Covered | + +**Scope Check:** + +| Plan task | Design requirement | Status | +|---|---|---| +| Task 1 | Side-effect-free version and authenticated semantic probe | Justified | +| Task 2 | Strict durable local state and crash recovery | Justified | +| Task 3 | Verified candidate activation and active provider process | Justified | +| Task 4 | One-time install, autonomous reconciliation, status, rotation, uninstall | Justified | +| Task 5 | Release/runtime proof and parent-plan handoff | Justified | + +**Manifest trace:** `plan-scope-check.sh --plan` passed: one PR row, five +existing task headings, and complete task assignment. The workspace +`2026-06-26-github-provider-dogfood-agents.md` scope lock remains authoritative; +this subordinate fix-forward plan implements its existing plugin Task 4 and STG +Task 8 and therefore does not replace or create a competing active lock. + +**Drift Items:** None. From ad6d48abfa1a88180f7f83c5f17e9333be285211 Mon Sep 17 00:00:00 2001 From: Jon Langevin Date: Mon, 13 Jul 2026 13:31:21 -0400 Subject: [PATCH 06/16] feat(provider): add versioned readiness probe --- .goreleaser.yaml | 2 + cmd/github-runner-provider/main.go | 34 ++ cmd/github-runner-provider/main_test.go | 207 ++++++++++++ cmd/github-runner-provider/probe.go | 334 ++++++++++++++++++++ cmd/github-runner-provider/retained_stub.go | 12 + release_packaging_test.go | 11 + 6 files changed, 600 insertions(+) create mode 100644 cmd/github-runner-provider/probe.go create mode 100644 cmd/github-runner-provider/retained_stub.go diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 118fb79..0886f1f 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -42,6 +42,8 @@ builds: goarch: - amd64 - arm64 + ldflags: + - -s -w -X github.com/GoCodeAlone/workflow-plugin-github/internal.Version={{.Version}} - id: github-actions-runner-job main: ./cmd/github-actions-runner-job binary: github-actions-runner-job diff --git a/cmd/github-runner-provider/main.go b/cmd/github-runner-provider/main.go index f01ad26..d0a117a 100644 --- a/cmd/github-runner-provider/main.go +++ b/cmd/github-runner-provider/main.go @@ -7,6 +7,7 @@ import ( "crypto/tls" "errors" "fmt" + "io" "log/slog" "net" "net/http" @@ -45,6 +46,39 @@ func main() { } func run(ctx context.Context, logger *slog.Logger, args []string) error { + handled, err := dispatchProviderCommand(ctx, logger, args, os.Stdout) + if handled { + return err + } + return runProviderService(ctx, logger, args) +} + +func dispatchProviderCommand(ctx context.Context, logger *slog.Logger, args []string, stdout io.Writer) (bool, error) { + if len(args) == 0 { + return false, nil + } + switch args[0] { + case "version", "--version": + if len(args) != 1 { + return true, errors.New("version does not accept arguments") + } + _, err := fmt.Fprintln(stdout, internal.Version) + return true, err + case "probe": + return true, runProviderProbe(ctx, args[1:], stdout) + case "retained": + return true, runRetainedProviderCommand(ctx, logger, args[1:], stdout) + default: + if len(args) == 1 { + if _, _, err := net.SplitHostPort(args[0]); err == nil { + return false, nil + } + } + return true, fmt.Errorf("unknown command %q", args[0]) + } +} + +func runProviderService(ctx context.Context, logger *slog.Logger, args []string) error { addr := "127.0.0.1:8090" if len(args) > 0 { addr = args[0] diff --git a/cmd/github-runner-provider/main_test.go b/cmd/github-runner-provider/main_test.go index 0f43956..d407a0a 100644 --- a/cmd/github-runner-provider/main_test.go +++ b/cmd/github-runner-provider/main_test.go @@ -1,13 +1,17 @@ package main import ( + "bytes" "context" "crypto/tls" + "encoding/json" + "encoding/pem" "errors" "io" "log/slog" "net" "net/http" + "net/http/httptest" "os" "path/filepath" "reflect" @@ -16,6 +20,7 @@ import ( "time" githubplugin "github.com/GoCodeAlone/workflow-plugin-github" + "github.com/GoCodeAlone/workflow-plugin-github/internal" ) type deadlineShutdowner struct { @@ -85,6 +90,208 @@ func TestProviderHTTPServerHasBoundedConnectionTimeouts(t *testing.T) { } } +func TestProviderCommandVersionDoesNotRequireServiceCredentials(t *testing.T) { + t.Setenv("GITHUB_RUNNER_PROVIDER_GITHUB_TOKEN", "") + t.Setenv("GITHUB_TOKEN", "") + t.Setenv("GITHUB_RUNNER_PROVIDER_TOKEN", "") + var stdout bytes.Buffer + handled, err := dispatchProviderCommand(t.Context(), slog.New(slog.NewTextHandler(io.Discard, nil)), []string{"version"}, &stdout) + if err != nil { + t.Fatalf("version: %v", err) + } + if !handled { + t.Fatal("version was treated as a legacy listen address") + } + if got, want := strings.TrimSpace(stdout.String()), internal.Version; got != want || got == "" { + t.Fatalf("version output = %q want %q", got, want) + } +} + +func TestProviderCommandRejectsUnknownSubcommand(t *testing.T) { + var stdout bytes.Buffer + handled, err := dispatchProviderCommand(t.Context(), slog.New(slog.NewTextHandler(io.Discard, nil)), []string{"unknown-command"}, &stdout) + if !handled || err == nil || !strings.Contains(err.Error(), "unknown command") { + t.Fatalf("unknown command handled=%t err=%v", handled, err) + } + if stdout.Len() != 0 { + t.Fatalf("unknown command wrote stdout: %q", stdout.String()) + } +} + +func TestProviderCommandPreservesLegacyListenAddress(t *testing.T) { + var stdout bytes.Buffer + handled, err := dispatchProviderCommand(t.Context(), slog.New(slog.NewTextHandler(io.Discard, nil)), []string{"127.0.0.1:0"}, &stdout) + if handled || err != nil { + t.Fatalf("legacy address handled=%t err=%v", handled, err) + } +} + +func TestProviderProbeAuthenticatesAndEmitsRedactedSemanticEvidence(t *testing.T) { + const providerToken = "provider-secret-token" + const githubToken = "github-secret-token" + var readyAuth string + var preflightAuth string + var request providerProbePreflightRequest + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/readyz": + readyAuth = r.Header.Get("Authorization") + _ = json.NewEncoder(w).Encode(providerProbeReadyResponse{Status: "ok"}) + case "/v1/actions/orgs/GoCodeAlone/runners/preflight": + preflightAuth = r.Header.Get("Authorization") + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatalf("decode preflight request: %v", err) + } + _ = json.NewEncoder(w).Encode(internal.GitHubRunnerProviderPreflight{ + Organization: "GoCodeAlone", + RunnerGroup: "ephemeral", + RunnerGroupID: 41, + Ref: strings.Repeat("a", 40), + ResolvedWorkflowPath: ".github/workflows/dogfood-provider-target.yml", + ResolvedRefSHA: strings.Repeat("a", 40), + LabelsObserved: 5, + RunnerCountChecked: 3, + ActionsEnabled: true, + SelfHostedAllowed: true, + }) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + caFile := filepath.Join(t.TempDir(), "ca.pem") + caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}) + if err := os.WriteFile(caFile, caPEM, 0o600); err != nil { + t.Fatalf("write CA: %v", err) + } + t.Setenv("GITHUB_RUNNER_PROVIDER_TOKEN", providerToken) + t.Setenv("GITHUB_RUNNER_PROVIDER_GITHUB_TOKEN", githubToken) + var stdout bytes.Buffer + err := runProviderProbe(t.Context(), []string{ + "-url", server.URL, + "-ca-file", caFile, + "-organization", "GoCodeAlone", + "-repository", "GoCodeAlone/workflow-compute", + "-workflow", "dogfood-provider-target.yml", + "-ref", strings.Repeat("a", 40), + "-runner-name", "wfc-stg-ghp-linux-probe", + "-runner-group", "ephemeral", + "-label", "self-hosted", + "-label", "linux", + "-label", "wfc-ghp-stg", + }, &stdout) + if err != nil { + t.Fatalf("probe: %v", err) + } + if readyAuth != "Bearer "+providerToken || preflightAuth != "Bearer "+providerToken { + t.Fatalf("probe auth ready=%q preflight=%q", readyAuth, preflightAuth) + } + if request.Repository != "GoCodeAlone/workflow-compute" || request.RunnerName != "wfc-stg-ghp-linux-probe" || len(request.Labels) != 3 { + t.Fatalf("preflight request = %+v", request) + } + var result providerProbeResult + decoder := json.NewDecoder(bytes.NewReader(stdout.Bytes())) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&result); err != nil { + t.Fatalf("decode probe output: %v output=%s", err, stdout.String()) + } + if result.Status != "passed" || !result.Ready || result.Organization != "GoCodeAlone" || result.RunnerGroupID != 41 || result.ResolvedRefSHA != strings.Repeat("a", 40) || result.ObservedAt.IsZero() { + t.Fatalf("probe result = %+v", result) + } + if strings.Contains(stdout.String(), providerToken) || strings.Contains(stdout.String(), githubToken) { + t.Fatalf("probe output leaked credential: %s", stdout.String()) + } +} + +func TestProviderProbeFailsClosedOnInvalidConfiguration(t *testing.T) { + for _, tc := range []struct { + name string + args []string + token string + want string + }{ + {name: "missing token", args: []string{"-url", "https://provider.test", "-ca-file", "/ca.pem"}, want: "GITHUB_RUNNER_PROVIDER_TOKEN"}, + {name: "plaintext URL", args: []string{"-url", "http://provider.test", "-ca-file", "/ca.pem"}, token: "provider-token", want: "HTTPS"}, + {name: "missing CA", args: []string{"-url", "https://provider.test"}, token: "provider-token", want: "ca-file"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("GITHUB_RUNNER_PROVIDER_TOKEN", tc.token) + var stdout bytes.Buffer + err := runProviderProbe(t.Context(), tc.args, &stdout) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("probe err = %v want %q", err, tc.want) + } + if stdout.Len() != 0 { + t.Fatalf("failed probe wrote stdout: %q", stdout.String()) + } + }) + } +} + +func TestProviderProbeRejectsUnknownResponseFieldsAndDoesNotEchoErrorBody(t *testing.T) { + const providerToken = "provider-secret-token" + for _, tc := range []struct { + name string + handler http.HandlerFunc + want string + }{ + { + name: "unknown readiness field", + handler: func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{"status":"ok","unexpected":true}`) + }, + want: "invalid JSON", + }, + { + name: "secret upstream error", + handler: func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "upstream rejected "+providerToken, http.StatusBadGateway) + }, + want: "HTTP status 502", + }, + } { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewTLSServer(tc.handler) + defer server.Close() + caFile := writeProviderProbeTestCA(t, server) + t.Setenv("GITHUB_RUNNER_PROVIDER_TOKEN", providerToken) + var stdout bytes.Buffer + err := runProviderProbe(t.Context(), validProviderProbeTestArgs(server.URL, caFile), &stdout) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("probe err = %v want %q", err, tc.want) + } + if strings.Contains(err.Error(), providerToken) || stdout.Len() != 0 { + t.Fatalf("failed probe leaked output: err=%v stdout=%q", err, stdout.String()) + } + }) + } +} + +func writeProviderProbeTestCA(t *testing.T, server *httptest.Server) string { + t.Helper() + caFile := filepath.Join(t.TempDir(), "ca.pem") + caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}) + if err := os.WriteFile(caFile, caPEM, 0o600); err != nil { + t.Fatalf("write CA: %v", err) + } + return caFile +} + +func validProviderProbeTestArgs(providerURL, caFile string) []string { + return []string{ + "-url", providerURL, + "-ca-file", caFile, + "-organization", "GoCodeAlone", + "-repository", "GoCodeAlone/workflow-compute", + "-workflow", "dogfood-provider-target.yml", + "-ref", strings.Repeat("a", 40), + "-runner-name", "wfc-stg-ghp-linux-probe", + "-runner-group", "ephemeral", + "-label", "self-hosted", + } +} + func TestProviderTLSFilesRequireCertificateAndKeyTogether(t *testing.T) { for _, tc := range []struct { name string diff --git a/cmd/github-runner-provider/probe.go b/cmd/github-runner-provider/probe.go new file mode 100644 index 0000000..2e6b0cd --- /dev/null +++ b/cmd/github-runner-provider/probe.go @@ -0,0 +1,334 @@ +package main + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "strings" + "time" + + "github.com/GoCodeAlone/workflow-plugin-github/internal" +) + +const ( + providerProbeProtocolVersion = "github-runner-provider.probe.v1" + providerProbeMaxBodyBytes = 1 << 20 + providerProbeHTTPTimeout = 30 * time.Second +) + +type providerProbePreflightRequest struct { + Repository string `json:"repository"` + Workflow string `json:"workflow"` + Ref string `json:"ref"` + RunnerName string `json:"runner_name"` + RunnerGroup string `json:"runner_group"` + Labels []string `json:"labels"` +} + +type providerProbeReadyResponse struct { + Status string `json:"status"` +} + +type providerProbeResult struct { + ProtocolVersion string `json:"protocol_version"` + Status string `json:"status"` + Ready bool `json:"ready"` + Organization string `json:"organization"` + RunnerGroup string `json:"runner_group"` + RunnerGroupID int64 `json:"runner_group_id"` + Ref string `json:"ref"` + ResolvedWorkflowPath string `json:"resolved_workflow_path"` + ResolvedRefSHA string `json:"resolved_ref_sha"` + RunnerCountChecked int `json:"runner_count_checked"` + LabelsObserved int `json:"labels_observed"` + ConflictingLabelCount int `json:"conflicting_label_count"` + ExistingLabelsTruncated bool `json:"existing_labels_truncated"` + ActionsEnabled bool `json:"actions_enabled"` + SelfHostedAllowed bool `json:"self_hosted_allowed"` + ObservedAt time.Time `json:"observed_at"` +} + +type providerProbeFlags []string + +func (f *providerProbeFlags) String() string { + return strings.Join(*f, ",") +} + +func (f *providerProbeFlags) Set(value string) error { + value = strings.TrimSpace(value) + if value == "" { + return errors.New("label must not be empty") + } + *f = append(*f, value) + return nil +} + +func runProviderProbe(ctx context.Context, args []string, stdout io.Writer) error { + fs := flag.NewFlagSet("github-runner-provider probe", flag.ContinueOnError) + fs.SetOutput(io.Discard) + rawURL := fs.String("url", "", "provider HTTPS base URL") + caFile := fs.String("ca-file", "", "provider CA certificate file") + organization := fs.String("organization", "", "allowed GitHub organization") + repository := fs.String("repository", "", "target owner/repository") + workflow := fs.String("workflow", "", "target workflow file") + ref := fs.String("ref", "", "full target commit SHA") + runnerName := fs.String("runner-name", "", "unique preflight runner name") + runnerGroup := fs.String("runner-group", "", "allowed runner group") + var labels providerProbeFlags + fs.Var(&labels, "label", "required runner label; repeatable") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 0 { + return errors.New("probe does not accept positional arguments") + } + token := strings.TrimSpace(os.Getenv("GITHUB_RUNNER_PROVIDER_TOKEN")) + if token == "" { + return errors.New("GITHUB_RUNNER_PROVIDER_TOKEN is required") + } + if strings.ContainsAny(token, "\r\n\x00") { + return errors.New("GITHUB_RUNNER_PROVIDER_TOKEN contains unsupported characters") + } + baseURL, err := validateProviderProbeFlags(*rawURL, *caFile, *organization, *repository, *workflow, *ref, *runnerName, *runnerGroup, labels) + if err != nil { + return err + } + client, err := newProviderProbeHTTPClient(*caFile) + if err != nil { + return err + } + + var ready providerProbeReadyResponse + if err := providerProbeJSON(ctx, client, http.MethodGet, baseURL.ResolveReference(&url.URL{Path: "/readyz"}), token, nil, &ready); err != nil { + return fmt.Errorf("provider readiness probe: %w", err) + } + if ready.Status != "ok" { + return fmt.Errorf("provider readiness status is %q", ready.Status) + } + request := providerProbePreflightRequest{ + Repository: strings.TrimSpace(*repository), + Workflow: strings.TrimSpace(*workflow), + Ref: strings.TrimSpace(*ref), + RunnerName: strings.TrimSpace(*runnerName), + RunnerGroup: strings.TrimSpace(*runnerGroup), + Labels: append([]string(nil), labels...), + } + preflightURL := baseURL.ResolveReference(&url.URL{Path: "/v1/actions/orgs/" + url.PathEscape(strings.TrimSpace(*organization)) + "/runners/preflight"}) + var preflight internal.GitHubRunnerProviderPreflight + if err := providerProbeJSON(ctx, client, http.MethodPost, preflightURL, token, request, &preflight); err != nil { + return fmt.Errorf("provider semantic preflight: %w", err) + } + if err := validateProviderProbePreflight(preflight, strings.TrimSpace(*organization), request); err != nil { + return err + } + result := providerProbeResult{ + ProtocolVersion: providerProbeProtocolVersion, + Status: "passed", + Ready: true, + Organization: preflight.Organization, + RunnerGroup: preflight.RunnerGroup, + RunnerGroupID: preflight.RunnerGroupID, + Ref: preflight.Ref, + ResolvedWorkflowPath: preflight.ResolvedWorkflowPath, + ResolvedRefSHA: preflight.ResolvedRefSHA, + RunnerCountChecked: preflight.RunnerCountChecked, + LabelsObserved: preflight.LabelsObserved, + ConflictingLabelCount: len(preflight.ConflictingLabels), + ExistingLabelsTruncated: preflight.ExistingLabelsTruncated, + ActionsEnabled: preflight.ActionsEnabled, + SelfHostedAllowed: preflight.SelfHostedAllowed, + ObservedAt: time.Now().UTC(), + } + encoder := json.NewEncoder(stdout) + encoder.SetIndent("", " ") + return encoder.Encode(result) +} + +func validateProviderProbeFlags(rawURL, caFile, organization, repository, workflow, ref, runnerName, runnerGroup string, labels []string) (*url.URL, error) { + baseURL, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil { + return nil, fmt.Errorf("parse provider url: %w", err) + } + if baseURL.Scheme != "https" || baseURL.Host == "" || baseURL.User != nil || baseURL.RawQuery != "" || baseURL.Fragment != "" || (baseURL.Path != "" && baseURL.Path != "/") { + return nil, errors.New("provider url must be an HTTPS origin") + } + baseURL.Path = "" + if strings.TrimSpace(caFile) == "" || !filepath.IsAbs(strings.TrimSpace(caFile)) { + return nil, errors.New("-ca-file must be an absolute path") + } + for _, field := range []struct { + name string + value string + }{ + {name: "organization", value: organization}, + {name: "repository", value: repository}, + {name: "workflow", value: workflow}, + {name: "ref", value: ref}, + {name: "runner-name", value: runnerName}, + {name: "runner-group", value: runnerGroup}, + } { + name, value := field.name, field.value + if strings.TrimSpace(value) == "" || strings.TrimSpace(value) != value || strings.ContainsAny(value, "\r\n\x00") { + return nil, fmt.Errorf("-%s is required and must be canonical", name) + } + } + owner, repo, ok := strings.Cut(repository, "/") + if !ok || owner != organization || !safeProviderProbeIdentifier(owner) || !safeProviderProbeIdentifier(repo) || !safeProviderProbeIdentifier(organization) || !safeProviderProbeIdentifier(runnerGroup) { + return nil, errors.New("provider organization, repository, or runner group is invalid") + } + if !safeProviderProbeWorkflow(workflow) { + return nil, errors.New("provider workflow is invalid") + } + if !isProviderProbeFullSHA(ref) { + return nil, errors.New("provider ref must be a full lowercase commit SHA") + } + if len(runnerName) > 100 || !safeProviderProbeIdentifier(runnerName) { + return nil, errors.New("provider runner name is invalid") + } + if len(labels) == 0 { + return nil, errors.New("at least one -label is required") + } + seen := make(map[string]struct{}, len(labels)) + for _, label := range labels { + if len(label) > 100 || !safeProviderProbeIdentifier(label) { + return nil, fmt.Errorf("provider label %q is invalid", label) + } + if _, exists := seen[label]; exists { + return nil, fmt.Errorf("provider label %q is duplicated", label) + } + seen[label] = struct{}{} + } + return baseURL, nil +} + +func newProviderProbeHTTPClient(caFile string) (*http.Client, error) { + info, err := os.Lstat(caFile) + if err != nil { + return nil, fmt.Errorf("inspect provider CA file: %w", err) + } + if !info.Mode().IsRegular() || info.Size() <= 0 || info.Size() > providerProbeMaxBodyBytes { + return nil, errors.New("provider CA file must be a regular file of at most 1 MiB") + } + caPEM, err := os.ReadFile(caFile) + if err != nil { + return nil, fmt.Errorf("read provider CA file: %w", err) + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(caPEM) { + return nil, errors.New("provider CA file contains no certificates") + } + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: roots} + return &http.Client{Transport: transport, Timeout: providerProbeHTTPTimeout}, nil +} + +func providerProbeJSON(ctx context.Context, client *http.Client, method string, endpoint *url.URL, token string, input, output any) error { + var body io.Reader + if input != nil { + data, err := json.Marshal(input) + if err != nil { + return fmt.Errorf("encode request: %w", err) + } + body = bytes.NewReader(data) + } + req, err := http.NewRequestWithContext(ctx, method, endpoint.String(), body) + if err != nil { + return fmt.Errorf("create request: %w", err) + } + req.Header.Set("Authorization", "Bearer "+token) + if input != nil { + req.Header.Set("Content-Type", "application/json") + } + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("provider request failed: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, providerProbeMaxBodyBytes)) + return fmt.Errorf("provider returned HTTP status %d", resp.StatusCode) + } + data, err := io.ReadAll(io.LimitReader(resp.Body, providerProbeMaxBodyBytes+1)) + if err != nil { + return errors.New("read provider response") + } + if len(data) > providerProbeMaxBodyBytes { + return errors.New("provider response exceeds 1 MiB") + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(output); err != nil { + return errors.New("provider returned invalid JSON") + } + var extra json.RawMessage + if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) { + return errors.New("provider returned multiple JSON values") + } + return nil +} + +func validateProviderProbePreflight(preflight internal.GitHubRunnerProviderPreflight, organization string, request providerProbePreflightRequest) error { + expectedWorkflowPath := request.Workflow + if !strings.Contains(expectedWorkflowPath, "/") { + expectedWorkflowPath = path.Join(".github/workflows", expectedWorkflowPath) + } + if preflight.Organization != organization || preflight.RunnerGroup != request.RunnerGroup || preflight.RunnerGroupID <= 0 { + return errors.New("provider preflight identity or runner group mismatch") + } + if preflight.Ref != request.Ref || preflight.ResolvedRefSHA != request.Ref || preflight.ResolvedWorkflowPath != expectedWorkflowPath { + return errors.New("provider preflight workflow or ref mismatch") + } + if !isProviderProbeFullSHA(preflight.ResolvedRefSHA) || !preflight.ActionsEnabled || !preflight.SelfHostedAllowed { + return errors.New("provider preflight rejected self-hosted execution") + } + if preflight.ExistingLabelsTruncated || len(preflight.ConflictingLabels) != 0 { + return errors.New("provider preflight found incomplete or conflicting labels") + } + return nil +} + +func safeProviderProbeIdentifier(value string) bool { + if value == "" || strings.TrimSpace(value) != value { + return false + } + for _, r := range value { + if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' || r == '.' { + continue + } + return false + } + return true +} + +func safeProviderProbeWorkflow(value string) bool { + if value == "" || strings.TrimSpace(value) != value || strings.HasPrefix(value, "/") || strings.Contains(value, "..") || strings.ContainsAny(value, "\\\r\n\x00") { + return false + } + cleaned := path.Clean(value) + return cleaned == value && (strings.HasSuffix(value, ".yml") || strings.HasSuffix(value, ".yaml")) +} + +func isProviderProbeFullSHA(value string) bool { + if len(value) != 40 { + return false + } + for _, r := range value { + if r < '0' || r > '9' { + if r < 'a' || r > 'f' { + return false + } + } + } + return true +} diff --git a/cmd/github-runner-provider/retained_stub.go b/cmd/github-runner-provider/retained_stub.go new file mode 100644 index 0000000..54960e3 --- /dev/null +++ b/cmd/github-runner-provider/retained_stub.go @@ -0,0 +1,12 @@ +package main + +import ( + "context" + "errors" + "io" + "log/slog" +) + +func runRetainedProviderCommand(context.Context, *slog.Logger, []string, io.Writer) error { + return errors.New("retained provider lifecycle is not implemented") +} diff --git a/release_packaging_test.go b/release_packaging_test.go index 824dd8c..cce7b24 100644 --- a/release_packaging_test.go +++ b/release_packaging_test.go @@ -29,6 +29,17 @@ func TestReleaseArchiveIncludesGitHubRunnerProvider(t *testing.T) { } } +func TestGitHubRunnerProviderReleaseBuildInjectsVersion(t *testing.T) { + data, err := os.ReadFile(".goreleaser.yaml") + if err != nil { + t.Fatalf("read .goreleaser.yaml: %v", err) + } + build := listItemWithID(topLevelSection(string(data), "builds:"), "github-runner-provider") + if !strings.Contains(build, "-X github.com/GoCodeAlone/workflow-plugin-github/internal.Version={{.Version}}") { + t.Fatalf("github-runner-provider release build must inject internal.Version:\n%s", build) + } +} + func TestReleaseArchiveIncludesGitHubActionsRunnerJob(t *testing.T) { data, err := os.ReadFile(".goreleaser.yaml") if err != nil { From 39ee3661b00f4b28a4fc637005fe02609952274b Mon Sep 17 00:00:00 2001 From: Jon Langevin Date: Mon, 13 Jul 2026 13:45:49 -0400 Subject: [PATCH 07/16] feat(provider): add durable retained state --- internal/retainedprovider/config.go | 171 +++++++++ internal/retainedprovider/files.go | 355 +++++++++++++++++++ internal/retainedprovider/files_test.go | 200 +++++++++++ internal/retainedprovider/lock_other.go | 14 + internal/retainedprovider/lock_unix.go | 22 ++ internal/retainedprovider/lock_windows.go | 24 ++ internal/retainedprovider/mode_other.go | 7 + internal/retainedprovider/mode_unix.go | 15 + internal/retainedprovider/ownership_other.go | 7 + internal/retainedprovider/ownership_unix.go | 20 ++ internal/retainedprovider/state.go | 195 ++++++++++ internal/retainedprovider/state_test.go | 254 +++++++++++++ internal/retainedprovider/syncdir_other.go | 5 + internal/retainedprovider/syncdir_unix.go | 14 + 14 files changed, 1303 insertions(+) create mode 100644 internal/retainedprovider/config.go create mode 100644 internal/retainedprovider/files.go create mode 100644 internal/retainedprovider/files_test.go create mode 100644 internal/retainedprovider/lock_other.go create mode 100644 internal/retainedprovider/lock_unix.go create mode 100644 internal/retainedprovider/lock_windows.go create mode 100644 internal/retainedprovider/mode_other.go create mode 100644 internal/retainedprovider/mode_unix.go create mode 100644 internal/retainedprovider/ownership_other.go create mode 100644 internal/retainedprovider/ownership_unix.go create mode 100644 internal/retainedprovider/state.go create mode 100644 internal/retainedprovider/state_test.go create mode 100644 internal/retainedprovider/syncdir_other.go create mode 100644 internal/retainedprovider/syncdir_unix.go diff --git a/internal/retainedprovider/config.go b/internal/retainedprovider/config.go new file mode 100644 index 0000000..1905c77 --- /dev/null +++ b/internal/retainedprovider/config.go @@ -0,0 +1,171 @@ +package retainedprovider + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/url" + "path/filepath" + "regexp" + "strings" +) + +const ( + ConfigProtocolVersion = "retained-provider.config.v1" + GitHubPluginID = "workflow-plugin-github" + maxConfigBytes = 1 << 20 +) + +var ( + safeIdentifierPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`) + gitRefPattern = regexp.MustCompile(`^[0-9a-f]{40}$`) + workflowPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]{0,255}$`) +) + +// Config is the non-secret, versioned retained-provider installation contract. +type Config struct { + ProtocolVersion string `json:"protocol_version"` + WorkerID string `json:"worker_id"` + ProfileID string `json:"profile_id"` + PluginID string `json:"plugin_id"` + ComponentID string `json:"component_id"` + ComputeAgentPath string `json:"compute_agent_path"` + SupervisorConfigPath string `json:"supervisor_config_path"` + LocalStatusPath string `json:"local_status_path"` + InstallRoot string `json:"install_root"` + SystemdDir string `json:"systemd_dir"` + AgentUnit string `json:"agent_unit"` + PodmanPath string `json:"podman_path"` + ProviderURL string `json:"provider_url"` + StableContainer string `json:"stable_container"` + CandidateContainer string `json:"candidate_container"` + ContainerNetwork string `json:"container_network"` + Organization string `json:"organization"` + Repository string `json:"repository"` + Workflow string `json:"workflow"` + Ref string `json:"ref"` + RunnerName string `json:"runner_name"` + RunnerGroup string `json:"runner_group"` + Labels []string `json:"labels"` + RefreshIntervalSeconds int `json:"refresh_interval_seconds"` +} + +func DecodeConfig(reader io.Reader, home string) (Config, error) { + var config Config + data, err := io.ReadAll(io.LimitReader(reader, maxConfigBytes+1)) + if err != nil { + return Config{}, fmt.Errorf("read retained provider config: %w", err) + } + if len(data) > maxConfigBytes { + return Config{}, fmt.Errorf("retained provider config exceeds %d bytes", maxConfigBytes) + } + if err := decodeStrictJSON(bytes.NewReader(data), &config); err != nil { + return Config{}, fmt.Errorf("decode retained provider config: %w", err) + } + if err := config.Validate(home); err != nil { + return Config{}, err + } + return config, nil +} + +func (config Config) Validate(home string) error { + if config.ProtocolVersion != ConfigProtocolVersion { + return fmt.Errorf("protocol_version must be %q", ConfigProtocolVersion) + } + for field, value := range map[string]string{ + "worker_id": config.WorkerID, + "profile_id": config.ProfileID, + "component_id": config.ComponentID, + "organization": config.Organization, + "runner_name": config.RunnerName, + "runner_group": config.RunnerGroup, + "agent_unit": strings.TrimSuffix(config.AgentUnit, ".service"), + "stable_container": config.StableContainer, + "candidate_container": config.CandidateContainer, + } { + if !safeIdentifierPattern.MatchString(value) { + return fmt.Errorf("%s contains an unsafe identifier", field) + } + } + if !strings.HasSuffix(config.AgentUnit, ".service") { + return fmt.Errorf("agent_unit must end in .service") + } + if config.PluginID != GitHubPluginID { + return fmt.Errorf("plugin_id must be %q", GitHubPluginID) + } + if config.StableContainer == config.CandidateContainer { + return fmt.Errorf("candidate_container must differ from stable_container") + } + if config.ContainerNetwork != "bridge" { + return fmt.Errorf("container_network must be bridge") + } + if !filepath.IsAbs(config.PodmanPath) || containsControl(config.PodmanPath) { + return fmt.Errorf("podman_path must be an absolute safe path") + } + for field, path := range map[string]string{ + "compute_agent_path": config.ComputeAgentPath, + "supervisor_config_path": config.SupervisorConfigPath, + "local_status_path": config.LocalStatusPath, + "install_root": config.InstallRoot, + "systemd_dir": config.SystemdDir, + } { + if err := ValidateUserPath(home, path, false); err != nil { + return fmt.Errorf("%s: %w", field, err) + } + } + providerURL, err := url.Parse(config.ProviderURL) + if err != nil || providerURL.Scheme != "https" || providerURL.Host == "" || providerURL.User != nil || providerURL.RawQuery != "" || providerURL.Fragment != "" { + return fmt.Errorf("provider_url must be an HTTPS URL without credentials, query, or fragment") + } + if providerURL.Hostname() != config.StableContainer { + return fmt.Errorf("provider_url host must match stable_container") + } + parts := strings.Split(config.Repository, "/") + if len(parts) != 2 || parts[0] != config.Organization || !safeIdentifierPattern.MatchString(parts[1]) { + return fmt.Errorf("repository must be organization/name for the configured organization") + } + if !workflowPattern.MatchString(config.Workflow) || strings.Contains(config.Workflow, "..") || containsControl(config.Workflow) { + return fmt.Errorf("workflow contains an unsafe path") + } + if !gitRefPattern.MatchString(config.Ref) { + return fmt.Errorf("ref must be a full lowercase commit SHA") + } + if len(config.Labels) == 0 { + return fmt.Errorf("labels must not be empty") + } + seenLabels := make(map[string]struct{}, len(config.Labels)) + for _, label := range config.Labels { + if !safeIdentifierPattern.MatchString(label) { + return fmt.Errorf("labels contains an unsafe label") + } + if _, exists := seenLabels[label]; exists { + return fmt.Errorf("labels contains duplicate %q", label) + } + seenLabels[label] = struct{}{} + } + if config.RefreshIntervalSeconds < 60 || config.RefreshIntervalSeconds > 86_400 { + return fmt.Errorf("refresh_interval_seconds must be between 60 and 86400") + } + return nil +} + +func decodeStrictJSON(reader io.Reader, target any) error { + decoder := json.NewDecoder(reader) + decoder.DisallowUnknownFields() + if err := decoder.Decode(target); err != nil { + return err + } + var trailing json.RawMessage + if err := decoder.Decode(&trailing); err != io.EOF { + if err == nil { + return fmt.Errorf("multiple JSON values") + } + return fmt.Errorf("trailing JSON data: %w", err) + } + return nil +} + +func containsControl(value string) bool { + return strings.IndexFunc(value, func(r rune) bool { return r < 0x20 || r == 0x7f }) >= 0 +} diff --git a/internal/retainedprovider/files.go b/internal/retainedprovider/files.go new file mode 100644 index 0000000..fe9dac2 --- /dev/null +++ b/internal/retainedprovider/files.go @@ -0,0 +1,355 @@ +package retainedprovider + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "sync" +) + +const MaxStateFileBytes = 1 << 20 + +type CloneLimits struct { + MaxFiles int + MaxBytes int64 +} + +func AtomicWriteJSON(path string, value any) (returnErr error) { + encoded, err := json.Marshal(value) + if err != nil { + return fmt.Errorf("encode JSON: %w", err) + } + encoded = append(encoded, '\n') + if len(encoded) > MaxStateFileBytes { + return fmt.Errorf("encoded JSON exceeds %d bytes", MaxStateFileBytes) + } + directory := filepath.Dir(path) + if err := os.MkdirAll(directory, 0o700); err != nil { + return fmt.Errorf("create state directory: %w", err) + } + if err := rejectNonRegularDestination(path); err != nil { + return err + } + temporary, err := os.CreateTemp(directory, ".retained-provider-*.tmp") + if err != nil { + return fmt.Errorf("create temporary state: %w", err) + } + temporaryPath := temporary.Name() + defer func() { + _ = temporary.Close() + if removeErr := os.Remove(temporaryPath); returnErr == nil && removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { + returnErr = fmt.Errorf("remove temporary state: %w", removeErr) + } + }() + if err := temporary.Chmod(0o600); err != nil { + return fmt.Errorf("restrict temporary state: %w", err) + } + if _, err := temporary.Write(encoded); err != nil { + return fmt.Errorf("write temporary state: %w", err) + } + if err := temporary.Sync(); err != nil { + return fmt.Errorf("sync temporary state: %w", err) + } + if err := temporary.Close(); err != nil { + return fmt.Errorf("close temporary state: %w", err) + } + if err := rejectNonRegularDestination(path); err != nil { + return err + } + if err := os.Rename(temporaryPath, path); err != nil { + return fmt.Errorf("replace state: %w", err) + } + if err := syncDirectory(directory); err != nil { + return fmt.Errorf("sync state directory: %w", err) + } + return nil +} + +func ReadStrictJSONFile(path string, target any) error { + entry, err := os.Lstat(path) + if err != nil { + return fmt.Errorf("inspect state file: %w", err) + } + if !entry.Mode().IsRegular() { + return fmt.Errorf("state file must be regular") + } + if err := validateStateMode(entry); err != nil { + return err + } + if err := validateOwner(entry); err != nil { + return fmt.Errorf("state file ownership: %w", err) + } + if entry.Size() > MaxStateFileBytes { + return fmt.Errorf("state file exceeds %d bytes", MaxStateFileBytes) + } + file, err := os.Open(path) + if err != nil { + return fmt.Errorf("open state file: %w", err) + } + defer file.Close() + opened, err := file.Stat() + if err != nil { + return fmt.Errorf("stat opened state file: %w", err) + } + if !opened.Mode().IsRegular() || !os.SameFile(entry, opened) { + return fmt.Errorf("state file changed during open or is not regular") + } + if opened.Size() > MaxStateFileBytes { + return fmt.Errorf("state file exceeds %d bytes", MaxStateFileBytes) + } + data, err := io.ReadAll(io.LimitReader(file, MaxStateFileBytes+1)) + if err != nil { + return fmt.Errorf("read state file: %w", err) + } + if len(data) > MaxStateFileBytes { + return fmt.Errorf("state file exceeds %d bytes", MaxStateFileBytes) + } + if err := decodeStrictJSON(bytes.NewReader(data), target); err != nil { + return fmt.Errorf("decode state file: %w", err) + } + return nil +} + +// ValidateUserPath enforces a lexical user-home boundary and rejects symlinks +// or foreign-owned files in every existing component below that boundary. +func ValidateUserPath(home, path string, requireExisting bool) error { + if !filepath.IsAbs(home) || !filepath.IsAbs(path) { + return fmt.Errorf("path and home must be absolute") + } + home = filepath.Clean(home) + path = filepath.Clean(path) + relative, err := filepath.Rel(home, path) + if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) || filepath.IsAbs(relative) { + return fmt.Errorf("path must remain within home") + } + current := home + if relative != "." { + for _, component := range strings.Split(relative, string(filepath.Separator)) { + current = filepath.Join(current, component) + info, statErr := os.Lstat(current) + if errors.Is(statErr, os.ErrNotExist) { + if requireExisting { + return fmt.Errorf("path does not exist: %s", current) + } + return nil + } + if statErr != nil { + return fmt.Errorf("inspect path: %w", statErr) + } + if info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("path contains symlink: %s", current) + } + if err := validateOwner(info); err != nil { + return fmt.Errorf("path ownership: %w", err) + } + } + } + if requireExisting { + if _, err := os.Lstat(path); err != nil { + return fmt.Errorf("path does not exist: %w", err) + } + } + return nil +} + +func CloneRegularTree(source, destination string, limits CloneLimits) (returnErr error) { + if limits.MaxFiles <= 0 || limits.MaxBytes < 0 { + return fmt.Errorf("clone limits must be positive") + } + root, err := os.Lstat(source) + if err != nil { + return fmt.Errorf("inspect clone source: %w", err) + } + if !root.IsDir() || root.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("clone source must be a regular directory") + } + if _, err := os.Lstat(destination); !errors.Is(err, os.ErrNotExist) { + if err == nil { + return fmt.Errorf("clone destination already exists") + } + return fmt.Errorf("inspect clone destination: %w", err) + } + if err := os.MkdirAll(destination, 0o700); err != nil { + return fmt.Errorf("create clone destination: %w", err) + } + defer func() { + if returnErr != nil { + _ = os.RemoveAll(destination) + } + }() + files := 0 + var bytesCopied int64 + if err := filepath.WalkDir(source, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + relative, err := filepath.Rel(source, path) + if err != nil { + return err + } + if relative == "." { + return nil + } + target := filepath.Join(destination, relative) + info, err := entry.Info() + if err != nil { + return err + } + if entry.Type()&os.ModeSymlink != 0 { + return fmt.Errorf("clone source entries must be regular files or directories: %s", relative) + } + if info.IsDir() { + return os.Mkdir(target, 0o700) + } + if !info.Mode().IsRegular() { + return fmt.Errorf("clone source entries must be regular files or directories: %s", relative) + } + files++ + if files > limits.MaxFiles { + return fmt.Errorf("clone file limit exceeded") + } + if info.Size() > limits.MaxBytes-bytesCopied { + return fmt.Errorf("clone byte limit exceeded") + } + if err := cloneRegularFile(path, target, info); err != nil { + return err + } + bytesCopied += info.Size() + return nil + }); err != nil { + return fmt.Errorf("clone regular tree: %w", err) + } + return syncDirectory(destination) +} + +func cloneRegularFile(source, destination string, expected os.FileInfo) (returnErr error) { + input, err := os.Open(source) + if err != nil { + return err + } + defer input.Close() + opened, err := input.Stat() + if err != nil || !opened.Mode().IsRegular() || !os.SameFile(expected, opened) { + return fmt.Errorf("clone source file changed during open") + } + output, err := os.OpenFile(destination, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return err + } + defer func() { + if closeErr := output.Close(); returnErr == nil && closeErr != nil { + returnErr = closeErr + } + }() + if _, err := io.CopyN(output, input, expected.Size()); err != nil { + return err + } + var extra [1]byte + if count, err := input.Read(extra[:]); err != io.EOF || count != 0 { + return fmt.Errorf("clone source file grew during copy") + } + return output.Sync() +} + +func rejectNonRegularDestination(path string) error { + info, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("inspect state destination: %w", err) + } + if info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("state destination must not be a symlink") + } + if !info.Mode().IsRegular() { + return fmt.Errorf("state destination must be regular") + } + if err := validateOwner(info); err != nil { + return fmt.Errorf("state destination ownership: %w", err) + } + if err := validateStateMode(info); err != nil { + return err + } + return nil +} + +var ErrInstallLocked = errors.New("retained provider install is already locked") + +type InstallLock struct { + mu sync.Mutex + file *os.File + released bool +} + +func AcquireInstallLock(path string) (*InstallLock, error) { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return nil, fmt.Errorf("create lock directory: %w", err) + } + file, err := openRegularLockFile(path) + if err != nil { + return nil, fmt.Errorf("open install lock: %w", err) + } + if err := file.Chmod(0o600); err != nil { + _ = file.Close() + return nil, fmt.Errorf("restrict install lock: %w", err) + } + if err := lockFile(file); err != nil { + _ = file.Close() + if errors.Is(err, ErrInstallLocked) { + return nil, ErrInstallLocked + } + return nil, fmt.Errorf("acquire install lock: %w", err) + } + return &InstallLock{file: file}, nil +} + +func openRegularLockFile(path string) (*os.File, error) { + before, err := os.Lstat(path) + if err == nil { + if !before.Mode().IsRegular() { + return nil, fmt.Errorf("install lock must be a regular file") + } + if err := validateOwner(before); err != nil { + return nil, fmt.Errorf("install lock ownership: %w", err) + } + } else if !errors.Is(err, os.ErrNotExist) { + return nil, err + } + file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o600) + if err != nil { + return nil, err + } + opened, statErr := file.Stat() + after, lstatErr := os.Lstat(path) + if statErr != nil || lstatErr != nil || !opened.Mode().IsRegular() || !after.Mode().IsRegular() || !os.SameFile(opened, after) || (before != nil && !os.SameFile(before, opened)) { + _ = file.Close() + return nil, fmt.Errorf("install lock must remain the same regular file during open") + } + if err := validateOwner(after); err != nil { + _ = file.Close() + return nil, fmt.Errorf("install lock ownership: %w", err) + } + return file, nil +} + +func (lock *InstallLock) Release() error { + if lock == nil { + return nil + } + lock.mu.Lock() + defer lock.mu.Unlock() + if lock.released { + return nil + } + lock.released = true + unlockErr := unlockFile(lock.file) + closeErr := lock.file.Close() + return errors.Join(unlockErr, closeErr) +} diff --git a/internal/retainedprovider/files_test.go b/internal/retainedprovider/files_test.go new file mode 100644 index 0000000..bc847c1 --- /dev/null +++ b/internal/retainedprovider/files_test.go @@ -0,0 +1,200 @@ +package retainedprovider + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestAtomicWriteJSONUsesRestrictiveRegularFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "state", "active.json") + value := Status{ProtocolVersion: StatusProtocolVersion, Installed: true} + if err := AtomicWriteJSON(path, value); err != nil { + t.Fatalf("atomic write: %v", err) + } + info, err := os.Lstat(path) + if err != nil { + t.Fatalf("stat active state: %v", err) + } + if !info.Mode().IsRegular() || (runtime.GOOS != "windows" && info.Mode().Perm() != 0o600) { + t.Fatalf("active state mode = %v", info.Mode()) + } + var got Status + if err := ReadStrictJSONFile(path, &got); err != nil { + t.Fatalf("strict read: %v", err) + } + if !got.Installed { + t.Fatalf("active state = %+v", got) + } + value.ServiceActive = true + if err := AtomicWriteJSON(path, value); err != nil { + t.Fatalf("atomic replacement: %v", err) + } + if err := ReadStrictJSONFile(path, &got); err != nil || !got.ServiceActive { + t.Fatalf("replacement state = %+v err=%v", got, err) + } + + if err := os.Remove(path); err != nil { + t.Fatalf("remove state: %v", err) + } + if err := os.Symlink(filepath.Join(dir, "outside"), path); err != nil { + t.Skipf("symlink unsupported: %v", err) + } + if err := AtomicWriteJSON(path, value); err == nil || !strings.Contains(err.Error(), "symlink") { + t.Fatalf("symlink destination err = %v", err) + } +} + +func TestReadStrictJSONFileRejectsUnknownAndOversizedData(t *testing.T) { + dir := t.TempDir() + unknown := filepath.Join(dir, "unknown.json") + if err := os.WriteFile(unknown, []byte(`{"protocol_version":"retained-provider.status.v1","installed":true,"unknown":1}`), 0o600); err != nil { + t.Fatalf("write unknown: %v", err) + } + var status Status + if err := ReadStrictJSONFile(unknown, &status); err == nil || !strings.Contains(err.Error(), "unknown field") { + t.Fatalf("unknown field err = %v", err) + } + oversized := filepath.Join(dir, "oversized.json") + if err := os.WriteFile(oversized, []byte(strings.Repeat("x", MaxStateFileBytes+1)), 0o600); err != nil { + t.Fatalf("write oversized: %v", err) + } + if err := ReadStrictJSONFile(oversized, &status); err == nil || !strings.Contains(err.Error(), "exceeds") { + t.Fatalf("oversized err = %v", err) + } +} + +func TestReadStrictJSONFileRejectsSymlinksAndPermissiveModes(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows does not expose Unix permission bits") + } + dir := t.TempDir() + permissive := filepath.Join(dir, "permissive.json") + if err := os.WriteFile(permissive, []byte(`{"protocol_version":"retained-provider.status.v1"}`), 0o644); err != nil { + t.Fatalf("write permissive state: %v", err) + } + var status Status + if err := ReadStrictJSONFile(permissive, &status); err == nil || !strings.Contains(err.Error(), "mode") { + t.Fatalf("permissive mode err = %v", err) + } + + secure := filepath.Join(dir, "secure.json") + if err := os.WriteFile(secure, []byte(`{"protocol_version":"retained-provider.status.v1"}`), 0o600); err != nil { + t.Fatalf("write secure state: %v", err) + } + linked := filepath.Join(dir, "linked.json") + if err := os.Symlink(secure, linked); err != nil { + t.Skipf("symlink unsupported: %v", err) + } + if err := ReadStrictJSONFile(linked, &status); err == nil || !strings.Contains(err.Error(), "regular") { + t.Fatalf("symlink state err = %v", err) + } +} + +func TestValidateUserPathRejectsSymlinkedAncestorAndOutsideHome(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlink permission varies on Windows") + } + home := t.TempDir() + realDir := filepath.Join(home, "real") + if err := os.Mkdir(realDir, 0o700); err != nil { + t.Fatalf("mkdir real: %v", err) + } + alias := filepath.Join(home, "alias") + if err := os.Symlink(realDir, alias); err != nil { + t.Fatalf("symlink: %v", err) + } + if err := ValidateUserPath(home, filepath.Join(alias, "state.json"), false); err == nil || !strings.Contains(err.Error(), "symlink") { + t.Fatalf("symlink ancestor err = %v", err) + } + if err := ValidateUserPath(home, filepath.Join(filepath.Dir(home), "outside"), false); err == nil || !strings.Contains(err.Error(), "home") { + t.Fatalf("outside-home err = %v", err) + } +} + +func TestCloneRegularTreeCopiesOnlyBoundedRegularFiles(t *testing.T) { + source := filepath.Join(t.TempDir(), "source") + destination := filepath.Join(t.TempDir(), "destination") + if err := os.MkdirAll(filepath.Join(source, "nested"), 0o700); err != nil { + t.Fatalf("mkdir source: %v", err) + } + if err := os.WriteFile(filepath.Join(source, "nested", "state.json"), []byte(`{"ok":true}`), 0o600); err != nil { + t.Fatalf("write source: %v", err) + } + if err := CloneRegularTree(source, destination, CloneLimits{MaxFiles: 10, MaxBytes: 1024}); err != nil { + t.Fatalf("clone: %v", err) + } + data, err := os.ReadFile(filepath.Join(destination, "nested", "state.json")) + if err != nil || string(data) != `{"ok":true}` { + t.Fatalf("cloned data=%q err=%v", data, err) + } + + symlinkSource := filepath.Join(t.TempDir(), "symlink-source") + if err := os.Mkdir(symlinkSource, 0o700); err != nil { + t.Fatalf("mkdir symlink source: %v", err) + } + if err := os.Symlink(filepath.Join(source, "nested", "state.json"), filepath.Join(symlinkSource, "link")); err != nil { + t.Skipf("symlink unsupported: %v", err) + } + if err := CloneRegularTree(symlinkSource, filepath.Join(t.TempDir(), "rejected"), CloneLimits{MaxFiles: 10, MaxBytes: 1024}); err == nil || !strings.Contains(err.Error(), "regular") { + t.Fatalf("symlink clone err = %v", err) + } + if err := CloneRegularTree(source, filepath.Join(t.TempDir(), "too-small"), CloneLimits{MaxFiles: 10, MaxBytes: 1}); err == nil || !strings.Contains(err.Error(), "byte limit") { + t.Fatalf("byte limit err = %v", err) + } +} + +func TestInstallLockIsExclusive(t *testing.T) { + path := filepath.Join(t.TempDir(), "install.lock") + first, err := AcquireInstallLock(path) + if err != nil { + t.Fatalf("first lock: %v", err) + } + defer first.Release() + if _, err := AcquireInstallLock(path); err == nil || !errors.Is(err, ErrInstallLocked) { + t.Fatalf("second lock err = %v", err) + } + if err := first.Release(); err != nil { + t.Fatalf("release first: %v", err) + } + second, err := AcquireInstallLock(path) + if err != nil { + t.Fatalf("lock after release: %v", err) + } + if err := second.Release(); err != nil { + t.Fatalf("release second: %v", err) + } +} + +func TestInstallLockRejectsSymlink(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlink permission varies on Windows") + } + dir := t.TempDir() + target := filepath.Join(dir, "target.lock") + if err := os.WriteFile(target, nil, 0o600); err != nil { + t.Fatalf("write lock target: %v", err) + } + linked := filepath.Join(dir, "linked.lock") + if err := os.Symlink(target, linked); err != nil { + t.Fatalf("symlink lock: %v", err) + } + if _, err := AcquireInstallLock(linked); err == nil || !strings.Contains(err.Error(), "regular") { + t.Fatalf("symlink lock err = %v", err) + } +} + +func TestStatusJSONHasStableShape(t *testing.T) { + data, err := json.Marshal(Status{ProtocolVersion: StatusProtocolVersion}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if !strings.Contains(string(data), `"protocol_version":"retained-provider.status.v1"`) { + t.Fatalf("status JSON = %s", data) + } +} diff --git a/internal/retainedprovider/lock_other.go b/internal/retainedprovider/lock_other.go new file mode 100644 index 0000000..421815b --- /dev/null +++ b/internal/retainedprovider/lock_other.go @@ -0,0 +1,14 @@ +//go:build !darwin && !linux && !windows + +package retainedprovider + +import ( + "fmt" + "os" +) + +func lockFile(*os.File) error { + return fmt.Errorf("install locking is unsupported on this platform") +} + +func unlockFile(*os.File) error { return nil } diff --git a/internal/retainedprovider/lock_unix.go b/internal/retainedprovider/lock_unix.go new file mode 100644 index 0000000..4d10011 --- /dev/null +++ b/internal/retainedprovider/lock_unix.go @@ -0,0 +1,22 @@ +//go:build darwin || linux + +package retainedprovider + +import ( + "errors" + "os" + + "golang.org/x/sys/unix" +) + +func lockFile(file *os.File) error { + err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB) + if errors.Is(err, unix.EWOULDBLOCK) || errors.Is(err, unix.EAGAIN) { + return ErrInstallLocked + } + return err +} + +func unlockFile(file *os.File) error { + return unix.Flock(int(file.Fd()), unix.LOCK_UN) +} diff --git a/internal/retainedprovider/lock_windows.go b/internal/retainedprovider/lock_windows.go new file mode 100644 index 0000000..1810b7d --- /dev/null +++ b/internal/retainedprovider/lock_windows.go @@ -0,0 +1,24 @@ +//go:build windows + +package retainedprovider + +import ( + "errors" + "os" + + "golang.org/x/sys/windows" +) + +func lockFile(file *os.File) error { + var overlapped windows.Overlapped + err := windows.LockFileEx(windows.Handle(file.Fd()), windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, &overlapped) + if errors.Is(err, windows.ERROR_LOCK_VIOLATION) { + return ErrInstallLocked + } + return err +} + +func unlockFile(file *os.File) error { + var overlapped windows.Overlapped + return windows.UnlockFileEx(windows.Handle(file.Fd()), 0, 1, 0, &overlapped) +} diff --git a/internal/retainedprovider/mode_other.go b/internal/retainedprovider/mode_other.go new file mode 100644 index 0000000..873c54f --- /dev/null +++ b/internal/retainedprovider/mode_other.go @@ -0,0 +1,7 @@ +//go:build !darwin && !linux + +package retainedprovider + +import "os" + +func validateStateMode(os.FileInfo) error { return nil } diff --git a/internal/retainedprovider/mode_unix.go b/internal/retainedprovider/mode_unix.go new file mode 100644 index 0000000..0cdd000 --- /dev/null +++ b/internal/retainedprovider/mode_unix.go @@ -0,0 +1,15 @@ +//go:build darwin || linux + +package retainedprovider + +import ( + "fmt" + "os" +) + +func validateStateMode(info os.FileInfo) error { + if info.Mode().Perm() != 0o600 { + return fmt.Errorf("state file mode must be 0600") + } + return nil +} diff --git a/internal/retainedprovider/ownership_other.go b/internal/retainedprovider/ownership_other.go new file mode 100644 index 0000000..b5d6871 --- /dev/null +++ b/internal/retainedprovider/ownership_other.go @@ -0,0 +1,7 @@ +//go:build !darwin && !linux + +package retainedprovider + +import "os" + +func validateOwner(os.FileInfo) error { return nil } diff --git a/internal/retainedprovider/ownership_unix.go b/internal/retainedprovider/ownership_unix.go new file mode 100644 index 0000000..07ba708 --- /dev/null +++ b/internal/retainedprovider/ownership_unix.go @@ -0,0 +1,20 @@ +//go:build darwin || linux + +package retainedprovider + +import ( + "fmt" + "os" + "syscall" +) + +func validateOwner(info os.FileInfo) error { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return fmt.Errorf("ownership metadata is unavailable") + } + if stat.Uid != uint32(os.Geteuid()) { + return fmt.Errorf("owner uid %d does not match current uid", stat.Uid) + } + return nil +} diff --git a/internal/retainedprovider/state.go b/internal/retainedprovider/state.go new file mode 100644 index 0000000..3637c6d --- /dev/null +++ b/internal/retainedprovider/state.go @@ -0,0 +1,195 @@ +package retainedprovider + +import ( + "fmt" + "path/filepath" + "regexp" + "strings" + "time" +) + +const ( + ActiveStateProtocolVersion = "retained-provider.active.v1" + TransactionJournalProtocolVersion = "retained-provider.transaction.v1" + StatusProtocolVersion = "retained-provider.status.v1" +) + +var ( + digestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) + imageRefPattern = regexp.MustCompile(`^localhost/[a-z0-9]+(?:[._/-][a-z0-9]+)*:sha256-[0-9a-f]{12,64}$`) +) + +type VerifiedUpdate struct { + WorkerID string `json:"worker_id"` + DirectiveID string `json:"directive_id"` + CampaignID string `json:"campaign_id"` + Component string `json:"component"` + PluginID string `json:"plugin_id"` + ComponentID string `json:"component_id"` + Version string `json:"version"` + Format string `json:"format"` + Path string `json:"path"` + SHA256 string `json:"sha256"` +} + +func (update VerifiedUpdate) Validate() error { + for field, value := range map[string]string{ + "worker_id": update.WorkerID, "directive_id": update.DirectiveID, + "campaign_id": update.CampaignID, "component_id": update.ComponentID, + "version": update.Version, + } { + if !safeIdentifierPattern.MatchString(value) { + return fmt.Errorf("%s contains an unsafe identifier", field) + } + } + if update.Component != "provider" { + return fmt.Errorf("component must be provider") + } + if update.PluginID != GitHubPluginID { + return fmt.Errorf("plugin_id must be %q", GitHubPluginID) + } + if update.Format != "binary" { + return fmt.Errorf("format must be binary") + } + if !filepath.IsAbs(update.Path) || containsControl(update.Path) { + return fmt.Errorf("path must be absolute and safe") + } + if !digestPattern.MatchString(update.SHA256) { + return fmt.Errorf("sha256 must be a lowercase SHA-256 digest") + } + return nil +} + +type ImageSelection struct { + Update VerifiedUpdate `json:"update"` + ImageID string `json:"image_id"` + ImageRef string `json:"image_ref"` + ActivatedAt time.Time `json:"activated_at"` +} + +func (selection ImageSelection) Validate() error { + if err := selection.Update.Validate(); err != nil { + return err + } + if !digestPattern.MatchString(selection.ImageID) { + return fmt.Errorf("image_id must be an immutable SHA-256 digest") + } + digest := strings.TrimPrefix(selection.Update.SHA256, "sha256:") + if !imageRefPattern.MatchString(selection.ImageRef) || !strings.HasSuffix(selection.ImageRef, ":sha256-"+digest[:12]) { + return fmt.Errorf("image_ref must be a safe localhost reference derived from the update digest") + } + if selection.ActivatedAt.IsZero() { + return fmt.Errorf("activated_at must be set") + } + return nil +} + +type ActiveState struct { + ProtocolVersion string `json:"protocol_version"` + Current ImageSelection `json:"current"` + Previous *ImageSelection `json:"previous,omitempty"` + UpdatedAt time.Time `json:"updated_at"` +} + +func (state ActiveState) Validate() error { + if state.ProtocolVersion != ActiveStateProtocolVersion { + return fmt.Errorf("protocol_version must be %q", ActiveStateProtocolVersion) + } + if err := state.Current.Validate(); err != nil { + return fmt.Errorf("current: %w", err) + } + if state.Previous != nil { + if err := state.Previous.Validate(); err != nil { + return fmt.Errorf("previous: %w", err) + } + if state.Previous.ImageID == state.Current.ImageID || state.Previous.ImageRef == state.Current.ImageRef { + return fmt.Errorf("previous image must differ from current image") + } + } + if state.UpdatedAt.IsZero() { + return fmt.Errorf("updated_at must be set") + } + return nil +} + +type JournalPhase string + +const ( + JournalPrepared JournalPhase = "prepared" + JournalActivated JournalPhase = "activated" + JournalCommitted JournalPhase = "committed" +) + +type TransactionJournal struct { + ProtocolVersion string `json:"protocol_version"` + ID string `json:"id"` + Phase JournalPhase `json:"phase"` + Previous *ActiveState `json:"previous,omitempty"` + Candidate ImageSelection `json:"candidate"` + StartedAt time.Time `json:"started_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +func (journal TransactionJournal) Validate() error { + if journal.ProtocolVersion != TransactionJournalProtocolVersion { + return fmt.Errorf("protocol_version must be %q", TransactionJournalProtocolVersion) + } + if !safeIdentifierPattern.MatchString(journal.ID) { + return fmt.Errorf("id contains an unsafe identifier") + } + switch journal.Phase { + case JournalPrepared, JournalActivated, JournalCommitted: + default: + return fmt.Errorf("phase is invalid") + } + if journal.Previous != nil { + if err := journal.Previous.Validate(); err != nil { + return fmt.Errorf("previous: %w", err) + } + } + if err := journal.Candidate.Validate(); err != nil { + return fmt.Errorf("candidate: %w", err) + } + if journal.Previous != nil && (journal.Candidate.ImageID == journal.Previous.Current.ImageID || journal.Candidate.ImageRef == journal.Previous.Current.ImageRef) { + return fmt.Errorf("candidate image must differ from the active image") + } + if journal.StartedAt.IsZero() || journal.UpdatedAt.IsZero() || journal.UpdatedAt.Before(journal.StartedAt) { + return fmt.Errorf("journal timestamps are invalid") + } + return nil +} + +func RecoverActiveState(journal TransactionJournal) (ActiveState, error) { + if err := journal.Validate(); err != nil { + return ActiveState{}, err + } + if journal.Phase != JournalCommitted { + if journal.Previous == nil { + return ActiveState{}, fmt.Errorf("cannot recover %s transaction without previous state", journal.Phase) + } + return *journal.Previous, nil + } + recovered := ActiveState{ + ProtocolVersion: ActiveStateProtocolVersion, + Current: journal.Candidate, + UpdatedAt: journal.UpdatedAt, + } + if journal.Previous != nil { + previous := journal.Previous.Current + recovered.Previous = &previous + } + if err := recovered.Validate(); err != nil { + return ActiveState{}, fmt.Errorf("recover committed transaction: %w", err) + } + return recovered, nil +} + +// Status deliberately contains only redacted, local lifecycle observations. +type Status struct { + ProtocolVersion string `json:"protocol_version"` + Installed bool `json:"installed"` + ServiceActive bool `json:"service_active"` + CurrentVersion string `json:"current_version,omitempty"` + CurrentSHA256 string `json:"current_sha256,omitempty"` + ObservedAt time.Time `json:"observed_at,omitempty"` +} diff --git a/internal/retainedprovider/state_test.go b/internal/retainedprovider/state_test.go new file mode 100644 index 0000000..bc25b85 --- /dev/null +++ b/internal/retainedprovider/state_test.go @@ -0,0 +1,254 @@ +package retainedprovider + +import ( + "bytes" + "encoding/json" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestConfigDecodeAndValidation(t *testing.T) { + home := t.TempDir() + valid := validTestConfig(home) + data, err := json.Marshal(valid) + if err != nil { + t.Fatalf("marshal config: %v", err) + } + got, err := DecodeConfig(bytes.NewReader(data), home) + if err != nil { + t.Fatalf("decode config: %v", err) + } + if got.WorkerID != valid.WorkerID || got.ComponentID != valid.ComponentID || got.RefreshIntervalSeconds != 300 { + t.Fatalf("decoded config = %+v", got) + } + + unknown := append(append([]byte(nil), bytes.TrimSuffix(data, []byte("}"))...), []byte(`,"unexpected":true}`)...) + if _, err := DecodeConfig(bytes.NewReader(unknown), home); err == nil || !strings.Contains(err.Error(), "unknown field") { + t.Fatalf("unknown config field err = %v", err) + } + if _, err := DecodeConfig(strings.NewReader(string(data)+"\n{}"), home); err == nil || !strings.Contains(err.Error(), "multiple JSON") { + t.Fatalf("multiple config values err = %v", err) + } + oversized := append(bytes.Repeat([]byte(" "), maxConfigBytes+1), data...) + if _, err := DecodeConfig(bytes.NewReader(oversized), home); err == nil || !strings.Contains(err.Error(), "exceeds") { + t.Fatalf("oversized config err = %v", err) + } +} + +func TestConfigRejectsUnsafeIdentityAndPaths(t *testing.T) { + home := t.TempDir() + for _, tc := range []struct { + name string + mutate func(*Config) + want string + }{ + {name: "wrong protocol", mutate: func(c *Config) { c.ProtocolVersion = "retained.v0" }, want: "protocol_version"}, + {name: "unsafe worker", mutate: func(c *Config) { c.WorkerID = "../worker" }, want: "worker_id"}, + {name: "unsafe profile", mutate: func(c *Config) { c.ProfileID = "profile\nnext" }, want: "profile_id"}, + {name: "wrong plugin", mutate: func(c *Config) { c.PluginID = "other" }, want: "plugin_id"}, + {name: "unsafe component", mutate: func(c *Config) { c.ComponentID = "component;rm" }, want: "component_id"}, + {name: "unsafe unit", mutate: func(c *Config) { c.AgentUnit = "agent.service\nEnvironment=TOKEN" }, want: "agent_unit"}, + {name: "relative install root", mutate: func(c *Config) { c.InstallRoot = "relative" }, want: "install_root"}, + {name: "outside home", mutate: func(c *Config) { c.SystemdDir = filepath.Join(filepath.Dir(home), "outside") }, want: "systemd_dir"}, + {name: "plaintext provider URL", mutate: func(c *Config) { c.ProviderURL = "http://provider:18090" }, want: "provider_url"}, + {name: "wrong network", mutate: func(c *Config) { c.ContainerNetwork = "host" }, want: "container_network"}, + {name: "short ref", mutate: func(c *Config) { c.Ref = "main" }, want: "ref"}, + {name: "fast timer", mutate: func(c *Config) { c.RefreshIntervalSeconds = 10 }, want: "refresh_interval_seconds"}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := validTestConfig(home) + tc.mutate(&cfg) + if err := cfg.Validate(home); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("Validate err = %v want %q", err, tc.want) + } + }) + } +} + +func TestActiveStateAndVerifiedUpdateValidation(t *testing.T) { + now := time.Now().UTC() + selection := validTestSelection(now) + state := ActiveState{ + ProtocolVersion: ActiveStateProtocolVersion, + Current: selection, + UpdatedAt: now, + } + if err := state.Validate(); err != nil { + t.Fatalf("valid active state: %v", err) + } + + for _, tc := range []struct { + name string + mutate func(*ActiveState) + want string + }{ + {name: "worker", mutate: func(s *ActiveState) { s.Current.Update.WorkerID = "" }, want: "worker_id"}, + {name: "plugin", mutate: func(s *ActiveState) { s.Current.Update.PluginID = "other" }, want: "plugin_id"}, + {name: "component", mutate: func(s *ActiveState) { s.Current.Update.Component = "plugin" }, want: "component"}, + {name: "digest", mutate: func(s *ActiveState) { s.Current.Update.SHA256 = "sha256:bad" }, want: "sha256"}, + {name: "image id", mutate: func(s *ActiveState) { s.Current.ImageID = "latest" }, want: "image_id"}, + {name: "image ref digest", mutate: func(s *ActiveState) { s.Current.ImageRef = "localhost/provider:sha256-cccccccccccc" }, want: "image_ref"}, + {name: "duplicate previous", mutate: func(s *ActiveState) { previous := s.Current; s.Previous = &previous }, want: "previous"}, + } { + t.Run(tc.name, func(t *testing.T) { + candidate := state + tc.mutate(&candidate) + if err := candidate.Validate(); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("Validate err = %v want %q", err, tc.want) + } + }) + } +} + +func TestActiveStateRetainsDistinctCurrentAndPriorImages(t *testing.T) { + now := time.Now().UTC() + current := validTestSelection(now) + previous := validTestSelection(now.Add(-time.Hour)) + previous.ImageID = "sha256:" + strings.Repeat("c", 64) + previous.ImageRef = "localhost/workflow-plugin-github-runner-provider:sha256-dddddddddddd" + previous.Update.DirectiveID = "directive-prior" + previous.Update.SHA256 = "sha256:" + strings.Repeat("d", 64) + state := ActiveState{ + ProtocolVersion: ActiveStateProtocolVersion, + Current: current, + Previous: &previous, + UpdatedAt: now, + } + if err := state.Validate(); err != nil { + t.Fatalf("valid current/prior state: %v", err) + } + if state.Current.ImageID == state.Previous.ImageID || state.Current.ImageRef == state.Previous.ImageRef { + t.Fatalf("current and prior images were not retained distinctly: %+v", state) + } +} + +func TestRecoverySelectionForEveryJournalPhase(t *testing.T) { + now := time.Now().UTC() + previous := ActiveState{ProtocolVersion: ActiveStateProtocolVersion, Current: validTestSelection(now.Add(-time.Hour)), UpdatedAt: now.Add(-time.Hour)} + candidate := validTestSelection(now) + candidate.ImageID = "sha256:" + strings.Repeat("c", 64) + candidate.ImageRef = "localhost/workflow-plugin-github-runner-provider:sha256-dddddddddddd" + candidate.Update.SHA256 = "sha256:" + strings.Repeat("d", 64) + candidate.Update.DirectiveID = "directive-new" + + for _, tc := range []struct { + phase JournalPhase + want ImageSelection + }{ + {phase: JournalPrepared, want: previous.Current}, + {phase: JournalActivated, want: previous.Current}, + {phase: JournalCommitted, want: candidate}, + } { + t.Run(string(tc.phase), func(t *testing.T) { + journal := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, + ID: "txn-1", + Phase: tc.phase, + Previous: &previous, + Candidate: candidate, + StartedAt: now, + UpdatedAt: now, + } + if err := journal.Validate(); err != nil { + t.Fatalf("valid journal: %v", err) + } + got, err := RecoverActiveState(journal) + if err != nil { + t.Fatalf("recover: %v", err) + } + if got.Current.ImageID != tc.want.ImageID || got.Current.Update.DirectiveID != tc.want.Update.DirectiveID { + t.Fatalf("recovered state = %+v want selection %+v", got, tc.want) + } + }) + } +} + +func TestJournalRejectsCandidateMatchingActiveImage(t *testing.T) { + now := time.Now().UTC() + previous := ActiveState{ProtocolVersion: ActiveStateProtocolVersion, Current: validTestSelection(now.Add(-time.Hour)), UpdatedAt: now.Add(-time.Hour)} + journal := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, + ID: "txn-duplicate", + Phase: JournalPrepared, + Previous: &previous, + Candidate: previous.Current, + StartedAt: now, + UpdatedAt: now, + } + if err := journal.Validate(); err == nil || !strings.Contains(err.Error(), "candidate") { + t.Fatalf("duplicate candidate err = %v", err) + } +} + +func TestStatusContainsNoCredentialFields(t *testing.T) { + status := Status{ + ProtocolVersion: StatusProtocolVersion, + Installed: true, + ServiceActive: true, + CurrentVersion: "v1.0.32", + CurrentSHA256: "sha256:" + strings.Repeat("a", 64), + ObservedAt: time.Now().UTC(), + } + data, err := json.Marshal(status) + if err != nil { + t.Fatalf("marshal status: %v", err) + } + text := string(data) + for _, forbidden := range []string{"github_token", "provider_token", "credential", "secret", "ca_cert"} { + if strings.Contains(strings.ToLower(text), forbidden) { + t.Fatalf("status contains credential-shaped field %q: %s", forbidden, text) + } + } +} + +func validTestConfig(home string) Config { + root := filepath.Join(home, ".workflow-compute", "github-runner-provider") + return Config{ + ProtocolVersion: ConfigProtocolVersion, + WorkerID: "github-runner-linux-stg", + ProfileID: "github-runner-linux-stg", + PluginID: GitHubPluginID, + ComponentID: "github-runner-provider-sidecar", + ComputeAgentPath: filepath.Join(home, ".workflow-compute", "agent-core-bin", "github-runner-linux-stg", "compute-agent"), + SupervisorConfigPath: filepath.Join(home, ".workflow-compute", "github-runner-linux-stg", "supervisor.pb"), + LocalStatusPath: filepath.Join(home, ".workflow-compute", "github-runner-linux-stg", "agent-status.json"), + InstallRoot: root, + SystemdDir: filepath.Join(home, ".config", "systemd", "user"), + AgentUnit: "workflow-compute-github-runner-linux-stg.service", + PodmanPath: "/usr/bin/podman", + ProviderURL: "https://workflow-plugin-github-runner-provider:18090", + StableContainer: "workflow-plugin-github-runner-provider", + CandidateContainer: "workflow-plugin-github-runner-provider-candidate", + ContainerNetwork: "bridge", + Organization: "GoCodeAlone", + Repository: "GoCodeAlone/workflow-compute", + Workflow: "dogfood-provider-target.yml", + Ref: strings.Repeat("a", 40), + RunnerName: "wfc-stg-ghp-linux-probe", + RunnerGroup: "ephemeral", + Labels: []string{"self-hosted", "linux", "wfc-ghp-stg"}, + RefreshIntervalSeconds: 300, + } +} + +func validTestSelection(now time.Time) ImageSelection { + return ImageSelection{ + Update: VerifiedUpdate{ + WorkerID: "github-runner-linux-stg-supervisor", + DirectiveID: "directive-1", + CampaignID: "campaign-1", + Component: "provider", + PluginID: GitHubPluginID, + ComponentID: "github-runner-provider-sidecar", + Version: "v1.0.32", + Format: "binary", + Path: "/home/runner/.workflow-compute/updates/.candidate-provider", + SHA256: "sha256:" + strings.Repeat("a", 64), + }, + ImageID: "sha256:" + strings.Repeat("b", 64), + ImageRef: "localhost/workflow-plugin-github-runner-provider:sha256-aaaaaaaaaaaa", + ActivatedAt: now, + } +} diff --git a/internal/retainedprovider/syncdir_other.go b/internal/retainedprovider/syncdir_other.go new file mode 100644 index 0000000..1858c4c --- /dev/null +++ b/internal/retainedprovider/syncdir_other.go @@ -0,0 +1,5 @@ +//go:build !darwin && !linux + +package retainedprovider + +func syncDirectory(string) error { return nil } diff --git a/internal/retainedprovider/syncdir_unix.go b/internal/retainedprovider/syncdir_unix.go new file mode 100644 index 0000000..8804263 --- /dev/null +++ b/internal/retainedprovider/syncdir_unix.go @@ -0,0 +1,14 @@ +//go:build darwin || linux + +package retainedprovider + +import "os" + +func syncDirectory(path string) error { + directory, err := os.Open(path) + if err != nil { + return err + } + defer directory.Close() + return directory.Sync() +} From cfe59dd38d9313a0d26461da04af9a5be610dac6 Mon Sep 17 00:00:00 2001 From: Jon Langevin Date: Mon, 13 Jul 2026 14:12:59 -0400 Subject: [PATCH 08/16] feat(provider): refresh retained provider safely --- cmd/github-runner-provider/retained_stub.go | 70 +- cmd/github-runner-provider/retained_test.go | 137 ++++ internal/retainedprovider/command.go | 134 +++ internal/retainedprovider/config.go | 16 +- internal/retainedprovider/refresh.go | 800 ++++++++++++++++++ internal/retainedprovider/refresh_test.go | 866 ++++++++++++++++++++ internal/retainedprovider/replace_linux.go | 9 + internal/retainedprovider/replace_other.go | 9 + internal/retainedprovider/state.go | 12 +- internal/retainedprovider/state_test.go | 7 +- 10 files changed, 2052 insertions(+), 8 deletions(-) create mode 100644 cmd/github-runner-provider/retained_test.go create mode 100644 internal/retainedprovider/command.go create mode 100644 internal/retainedprovider/refresh.go create mode 100644 internal/retainedprovider/refresh_test.go create mode 100644 internal/retainedprovider/replace_linux.go create mode 100644 internal/retainedprovider/replace_other.go diff --git a/cmd/github-runner-provider/retained_stub.go b/cmd/github-runner-provider/retained_stub.go index 54960e3..572ba7d 100644 --- a/cmd/github-runner-provider/retained_stub.go +++ b/cmd/github-runner-provider/retained_stub.go @@ -3,10 +3,76 @@ package main import ( "context" "errors" + "flag" + "fmt" "io" "log/slog" + "os" + "runtime" + "strings" + "time" + + "github.com/GoCodeAlone/workflow-plugin-github/internal/retainedprovider" ) -func runRetainedProviderCommand(context.Context, *slog.Logger, []string, io.Writer) error { - return errors.New("retained provider lifecycle is not implemented") +type retainedProviderCommandDependencies struct { + GOOS string + HomeDir func() (string, error) + ReadConfig func(string, string) (retainedprovider.Config, error) + Refresh func(context.Context, retainedprovider.Config) (retainedprovider.Status, error) + ServeActive func(context.Context, retainedprovider.Config) error +} + +func runRetainedProviderCommand(ctx context.Context, logger *slog.Logger, args []string, stdout io.Writer) error { + runner := retainedprovider.OSCommandRunner{} + refresher := retainedprovider.Refresher{Runner: runner, ExecutablePath: os.Executable, Now: func() time.Time { return time.Now().UTC() }} + return runRetainedProviderCommandWithDependencies(ctx, logger, args, stdout, retainedProviderCommandDependencies{ + GOOS: runtime.GOOS, + HomeDir: os.UserHomeDir, + ReadConfig: retainedprovider.ReadConfigFile, + Refresh: refresher.Refresh, + ServeActive: refresher.ServeActive, + }) +} + +func runRetainedProviderCommandWithDependencies(ctx context.Context, _ *slog.Logger, args []string, stdout io.Writer, dependencies retainedProviderCommandDependencies) error { + if dependencies.GOOS != "linux" { + return fmt.Errorf("retained provider lifecycle is unsupported on %s", dependencies.GOOS) + } + if len(args) == 0 { + return errors.New("retained provider subcommand is required") + } + switch args[0] { + case "refresh", "serve-active": + default: + return fmt.Errorf("unknown retained provider subcommand %q", args[0]) + } + flags := flag.NewFlagSet("github-runner-provider retained "+args[0], flag.ContinueOnError) + flags.SetOutput(io.Discard) + configPath := flags.String("config", "", "absolute retained provider config path") + if err := flags.Parse(args[1:]); err != nil { + return err + } + if flags.NArg() != 0 { + return errors.New("retained provider command does not accept positional arguments") + } + if strings.TrimSpace(*configPath) == "" { + return errors.New("-config is required") + } + home, err := dependencies.HomeDir() + if err != nil { + return fmt.Errorf("resolve user home: %w", err) + } + config, err := dependencies.ReadConfig(*configPath, home) + if err != nil { + return err + } + if args[0] == "serve-active" { + return dependencies.ServeActive(ctx, config) + } + status, err := dependencies.Refresh(ctx, config) + if err != nil { + return err + } + return retainedprovider.WriteStatus(stdout, status) } diff --git a/cmd/github-runner-provider/retained_test.go b/cmd/github-runner-provider/retained_test.go new file mode 100644 index 0000000..4cb8e91 --- /dev/null +++ b/cmd/github-runner-provider/retained_test.go @@ -0,0 +1,137 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/GoCodeAlone/workflow-plugin-github/internal/retainedprovider" +) + +func TestRetainedRefreshLoadsStrictConfigAndEmitsTypedStatus(t *testing.T) { + home := t.TempDir() + config := retainedCommandTestConfig(home) + wantStatus := retainedprovider.Status{ + ProtocolVersion: retainedprovider.StatusProtocolVersion, + Installed: true, + ServiceActive: true, + CurrentVersion: "v1.0.32", + CurrentSHA256: "sha256:" + strings.Repeat("a", 64), + ObservedAt: time.Unix(1_700_000_000, 0).UTC(), + } + var received retainedprovider.Config + dependencies := retainedProviderCommandDependencies{ + GOOS: "linux", + HomeDir: func() (string, error) { return home, nil }, + ReadConfig: func(path, gotHome string) (retainedprovider.Config, error) { + if path != filepath.Join(home, "config.json") || gotHome != home { + t.Fatalf("ReadConfig path=%q home=%q", path, gotHome) + } + return config, nil + }, + Refresh: func(_ context.Context, got retainedprovider.Config) (retainedprovider.Status, error) { + received = got + return wantStatus, nil + }, + ServeActive: func(context.Context, retainedprovider.Config) error { return nil }, + } + var stdout bytes.Buffer + err := runRetainedProviderCommandWithDependencies(t.Context(), slog.New(slog.NewTextHandler(io.Discard, nil)), []string{ + "refresh", "-config", filepath.Join(home, "config.json"), + }, &stdout, dependencies) + if err != nil { + t.Fatalf("retained refresh: %v", err) + } + if received.WorkerID != config.WorkerID || received.ComponentID != config.ComponentID { + t.Fatalf("refresh config = %+v", received) + } + var status retainedprovider.Status + decoder := json.NewDecoder(bytes.NewReader(stdout.Bytes())) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&status); err != nil { + t.Fatalf("decode status: %v output=%s", err, stdout.String()) + } + if status != wantStatus { + t.Fatalf("status = %+v want %+v", status, wantStatus) + } +} + +func TestRetainedServeActiveDispatchesWithoutWritingOutput(t *testing.T) { + home := t.TempDir() + config := retainedCommandTestConfig(home) + sentinel := errors.New("foreground exec returned") + dependencies := retainedProviderCommandDependencies{ + GOOS: "linux", + HomeDir: func() (string, error) { return home, nil }, + ReadConfig: func(string, string) (retainedprovider.Config, error) { return config, nil }, + Refresh: func(context.Context, retainedprovider.Config) (retainedprovider.Status, error) { + return retainedprovider.Status{}, nil + }, + ServeActive: func(context.Context, retainedprovider.Config) error { return sentinel }, + } + var stdout bytes.Buffer + err := runRetainedProviderCommandWithDependencies(t.Context(), slog.New(slog.NewTextHandler(io.Discard, nil)), []string{ + "serve-active", "-config", filepath.Join(home, "config.json"), + }, &stdout, dependencies) + if !errors.Is(err, sentinel) { + t.Fatalf("serve-active err = %v", err) + } + if stdout.Len() != 0 { + t.Fatalf("serve-active wrote output: %q", stdout.String()) + } +} + +func TestRetainedCommandFailsClosedOnUnsupportedPlatformAndInvalidShape(t *testing.T) { + base := retainedProviderCommandDependencies{ + GOOS: "linux", + HomeDir: func() (string, error) { return t.TempDir(), nil }, + ReadConfig: func(string, string) (retainedprovider.Config, error) { return retainedprovider.Config{}, nil }, + Refresh: func(context.Context, retainedprovider.Config) (retainedprovider.Status, error) { + return retainedprovider.Status{}, nil + }, + ServeActive: func(context.Context, retainedprovider.Config) error { return nil }, + } + for _, tc := range []struct { + name string + deps retainedProviderCommandDependencies + args []string + want string + }{ + {name: "unsupported", deps: func() retainedProviderCommandDependencies { value := base; value.GOOS = "darwin"; return value }(), args: []string{"refresh"}, want: "unsupported"}, + {name: "missing subcommand", deps: base, want: "subcommand"}, + {name: "unknown", deps: base, args: []string{"install-now"}, want: "unknown"}, + {name: "missing config", deps: base, args: []string{"refresh"}, want: "-config"}, + {name: "positional", deps: base, args: []string{"refresh", "-config", "/tmp/config", "extra"}, want: "positional"}, + } { + t.Run(tc.name, func(t *testing.T) { + err := runRetainedProviderCommandWithDependencies(t.Context(), slog.New(slog.NewTextHandler(io.Discard, nil)), tc.args, io.Discard, tc.deps) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("err = %v want %q", err, tc.want) + } + }) + } +} + +func retainedCommandTestConfig(home string) retainedprovider.Config { + root := filepath.Join(home, ".workflow-compute", "github-runner-provider") + return retainedprovider.Config{ + ProtocolVersion: retainedprovider.ConfigProtocolVersion, + WorkerID: "github-runner-linux-stg", ProfileID: "github-runner-linux-stg", + PluginID: retainedprovider.GitHubPluginID, ComponentID: "github-runner-provider-sidecar", + ComputeAgentPath: filepath.Join(home, "compute-agent"), SupervisorConfigPath: filepath.Join(home, "supervisor.pb"), + LocalStatusPath: filepath.Join(home, "status.json"), InstallRoot: root, + SystemdDir: filepath.Join(home, ".config", "systemd", "user"), AgentUnit: "workflow-compute-agent.service", + PodmanPath: "/usr/bin/podman", ProviderURL: "https://workflow-plugin-github-runner-provider:18090", + StableContainer: "workflow-plugin-github-runner-provider", CandidateContainer: "workflow-plugin-github-runner-provider-candidate", ContainerNetwork: "bridge", + Organization: "GoCodeAlone", Repository: "GoCodeAlone/workflow-compute", Workflow: "dogfood-provider-target.yml", + Ref: strings.Repeat("a", 40), RunnerName: "wfc-stg-ghp-linux-probe", RunnerGroup: "ephemeral", + Labels: []string{"self-hosted", "linux", "wfc-ghp-stg"}, RefreshIntervalSeconds: 300, + } +} diff --git a/internal/retainedprovider/command.go b/internal/retainedprovider/command.go new file mode 100644 index 0000000..70da48b --- /dev/null +++ b/internal/retainedprovider/command.go @@ -0,0 +1,134 @@ +package retainedprovider + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" +) + +const defaultCommandOutputBytes = 1 << 20 + +type Command struct { + Path string + Args []string + Env []string + Dir string + Stdin []byte +} + +type CommandRunner interface { + Run(context.Context, Command) ([]byte, error) + Exec(Command) error +} + +type OSCommandRunner struct { + MaxOutputBytes int +} + +func (runner OSCommandRunner) Run(ctx context.Context, command Command) ([]byte, error) { + if err := validateCommand(command); err != nil { + return nil, err + } + limit := runner.MaxOutputBytes + if limit <= 0 { + limit = defaultCommandOutputBytes + } + stdout := &boundedCommandBuffer{remaining: limit} + process := exec.CommandContext(ctx, command.Path, command.Args...) + process.Stdout = stdout + process.Stderr = io.Discard + process.Dir = command.Dir + process.Env = commandEnvironment(command.Env) + if command.Stdin != nil { + process.Stdin = bytes.NewReader(command.Stdin) + } + if err := process.Run(); err != nil { + return nil, fmt.Errorf("command %q failed: %w", filepath.Base(command.Path), redactCommandError(err)) + } + if stdout.exceeded { + return nil, fmt.Errorf("command %q output exceeds %d bytes", filepath.Base(command.Path), limit) + } + return stdout.Bytes(), nil +} + +func (runner OSCommandRunner) Exec(command Command) error { + if err := validateCommand(command); err != nil { + return err + } + if command.Dir != "" || command.Stdin != nil { + return errors.New("foreground exec does not support a working directory or stdin") + } + environment := commandEnvironment(command.Env) + return replaceProcess(command.Path, append([]string{command.Path}, command.Args...), environment) +} + +func commandEnvironment(explicit []string) []string { + if explicit != nil { + return append([]string(nil), explicit...) + } + allowed := []string{ + "HOME", "USER", "LOGNAME", "PATH", "SHELL", "TMPDIR", + "LANG", "LC_ALL", "LC_CTYPE", + "XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_RUNTIME_DIR", + "DBUS_SESSION_BUS_ADDRESS", + "CONTAINERS_CONF", "CONTAINERS_STORAGE_CONF", "CONTAINERS_REGISTRIES_CONF", + } + environment := make([]string, 0, len(allowed)) + for _, key := range allowed { + if value, exists := os.LookupEnv(key); exists && !strings.ContainsAny(value, "\r\n\x00") { + environment = append(environment, key+"="+value) + } + } + return environment +} + +func validateCommand(command Command) error { + if !filepath.IsAbs(command.Path) || containsControl(command.Path) { + return errors.New("command path must be absolute and safe") + } + for _, value := range command.Args { + if strings.ContainsRune(value, 0) { + return errors.New("command argument contains NUL") + } + } + for _, value := range command.Env { + if strings.ContainsAny(value, "\r\n\x00") { + return errors.New("command environment contains unsupported characters") + } + } + return nil +} + +func redactCommandError(err error) error { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return fmt.Errorf("exit status %d", exitErr.ExitCode()) + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + return errors.New("process could not be started") +} + +type boundedCommandBuffer struct { + bytes.Buffer + remaining int + exceeded bool +} + +func (buffer *boundedCommandBuffer) Write(data []byte) (int, error) { + count := len(data) + if len(data) > buffer.remaining { + data = data[:buffer.remaining] + buffer.exceeded = true + } + buffer.remaining -= len(data) + _, _ = buffer.Buffer.Write(data) + return count, nil +} diff --git a/internal/retainedprovider/config.go b/internal/retainedprovider/config.go index 1905c77..8deecdb 100644 --- a/internal/retainedprovider/config.go +++ b/internal/retainedprovider/config.go @@ -69,6 +69,20 @@ func DecodeConfig(reader io.Reader, home string) (Config, error) { return config, nil } +func ReadConfigFile(path, home string) (Config, error) { + if err := ValidateUserPath(home, path, true); err != nil { + return Config{}, fmt.Errorf("config path: %w", err) + } + var config Config + if err := ReadStrictJSONFile(path, &config); err != nil { + return Config{}, fmt.Errorf("read retained provider config: %w", err) + } + if err := config.Validate(home); err != nil { + return Config{}, err + } + return config, nil +} + func (config Config) Validate(home string) error { if config.ProtocolVersion != ConfigProtocolVersion { return fmt.Errorf("protocol_version must be %q", ConfigProtocolVersion) @@ -115,7 +129,7 @@ func (config Config) Validate(home string) error { } } providerURL, err := url.Parse(config.ProviderURL) - if err != nil || providerURL.Scheme != "https" || providerURL.Host == "" || providerURL.User != nil || providerURL.RawQuery != "" || providerURL.Fragment != "" { + if err != nil || providerURL.Scheme != "https" || providerURL.Host == "" || providerURL.User != nil || providerURL.RawQuery != "" || providerURL.Fragment != "" || (providerURL.Path != "" && providerURL.Path != "/") || providerURL.Port() != "18090" { return fmt.Errorf("provider_url must be an HTTPS URL without credentials, query, or fragment") } if providerURL.Hostname() != config.StableContainer { diff --git a/internal/retainedprovider/refresh.go b/internal/retainedprovider/refresh.go new file mode 100644 index 0000000..2503884 --- /dev/null +++ b/internal/retainedprovider/refresh.go @@ -0,0 +1,800 @@ +package retainedprovider + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" +) + +const ( + providerServiceUnit = "workflow-plugin-github-runner-provider.service" + providerStateMount = "/var/lib/workflow-github-runner-provider" + providerTLSMount = "/tls" + providerCAPath = "/tls/ca.pem" + providerTLSCertPath = "/tls/server.crt" + providerTLSKeyPath = "/tls/server.key" + providerListenAddr = "0.0.0.0:18090" + maxProviderPackageBytes = 512 << 20 +) + +var providerContainerfile = []byte("FROM scratch\nCOPY --chmod=0555 github-runner-provider /github-runner-provider\nENTRYPOINT [\"/github-runner-provider\"]\n") + +type LifecyclePaths struct { + Root string + ActiveState string + Journal string + InstallLock string + ProviderState string + PackagesRoot string + CandidatesRoot string + ProviderEnv string + ProbeEnv string + TLSRoot string + CAFile string +} + +func LifecyclePathsFor(config Config) LifecyclePaths { + root := config.InstallRoot + return LifecyclePaths{ + Root: root, + ActiveState: filepath.Join(root, "lifecycle", "active.json"), + Journal: filepath.Join(root, "lifecycle", "transaction.json"), + InstallLock: filepath.Join(root, "lifecycle", "install.lock"), + ProviderState: filepath.Join(root, "provider-state"), + PackagesRoot: filepath.Join(root, "packages"), + CandidatesRoot: filepath.Join(root, "candidates"), + ProviderEnv: filepath.Join(root, "secrets", "provider.env"), + ProbeEnv: filepath.Join(root, "secrets", "probe.env"), + TLSRoot: filepath.Join(root, "tls"), + CAFile: filepath.Join(root, "tls", "ca.pem"), + } +} + +func (paths LifecyclePaths) CandidateState(digest string) string { + return filepath.Join(paths.CandidatesRoot, digestHex(digest), "state") +} + +func (paths LifecyclePaths) PackageDir(digest string) string { + return filepath.Join(paths.PackagesRoot, digestHex(digest)) +} + +func (paths LifecyclePaths) PackageBinary(digest string) string { + return filepath.Join(paths.PackageDir(digest), "github-runner-provider") +} + +type Refresher struct { + Runner CommandRunner + ExecutablePath func() (string, error) + Now func() time.Time + Sleep func(context.Context, time.Duration) error +} + +func (refresher Refresher) Refresh(ctx context.Context, config Config) (status Status, returnErr error) { + if refresher.Runner == nil { + return Status{}, errors.New("command runner is required") + } + paths := LifecyclePathsFor(config) + if err := validateInstallRoot(paths.Root); err != nil { + return Status{}, err + } + if err := ValidateUserPath(paths.Root, paths.InstallLock, false); err != nil { + return Status{}, fmt.Errorf("install lock path: %w", err) + } + lock, err := AcquireInstallLock(paths.InstallLock) + if err != nil { + return Status{}, err + } + defer func() { returnErr = errors.Join(returnErr, lock.Release()) }() + if err := refresher.recoverInterrupted(ctx, config, paths); err != nil { + return Status{}, err + } + update, err := VerifyCurrentUpdate(ctx, config, refresher.Runner) + if err != nil { + return Status{}, err + } + active, activeFound, err := readActiveState(paths.ActiveState) + if err != nil { + return Status{}, err + } + now := refresher.now() + if activeFound && active.Current.Update.SHA256 == update.SHA256 { + return statusForActive(active, true, now), nil + } + if !activeFound { + executablePath := refresher.ExecutablePath + if executablePath == nil { + executablePath = os.Executable + } + currentExecutable, err := executablePath() + if err != nil { + return Status{}, fmt.Errorf("resolve installer executable: %w", err) + } + digest, err := hashRegularFile(currentExecutable, true) + if err != nil { + return Status{}, fmt.Errorf("hash installer executable: %w", err) + } + if digest != update.SHA256 { + return Status{}, errors.New("installer digest does not match verified provider update") + } + } + for name, path := range map[string]string{ + "provider environment": paths.ProviderEnv, + "probe environment": paths.ProbeEnv, + "provider CA": paths.CAFile, + "provider state": paths.ProviderState, + } { + if err := ValidateUserPath(paths.Root, path, true); err != nil { + return Status{}, fmt.Errorf("%s path: %w", name, err) + } + } + if err := validateProviderEnvironment(config, paths.ProviderEnv); err != nil { + return Status{}, err + } + if err := validateProbeEnvironment(paths.ProbeEnv); err != nil { + return Status{}, err + } + if err := validateSecretFile(paths.CAFile); err != nil { + return Status{}, fmt.Errorf("provider CA file: %w", err) + } + if err := ValidateUserPath(paths.Root, paths.PackageDir(update.SHA256), false); err != nil { + return Status{}, fmt.Errorf("provider package path: %w", err) + } + if err := stageVerifiedProvider(update, paths); err != nil { + return Status{}, err + } + imageRef := providerImageRef(update.SHA256) + if _, err := refresher.Runner.Run(ctx, Command{ + Path: config.PodmanPath, + Args: []string{"build", "--file", "-", "--tag", imageRef, paths.PackageDir(update.SHA256)}, + Stdin: providerContainerfile, + }); err != nil { + return Status{}, fmt.Errorf("build provider candidate image: %w", err) + } + imageOutput, err := refresher.Runner.Run(ctx, Command{ + Path: config.PodmanPath, + Args: []string{"image", "inspect", "--format", "{{.Id}}", imageRef}, + }) + if err != nil { + return Status{}, fmt.Errorf("inspect provider candidate image: %w", err) + } + imageID := strings.TrimSpace(string(imageOutput)) + selection := ImageSelection{Update: update, ImageID: imageID, ImageRef: imageRef, ActivatedAt: now} + if err := selection.Validate(); err != nil { + return Status{}, fmt.Errorf("validate provider candidate image: %w", err) + } + candidateState := paths.CandidateState(update.SHA256) + if err := ValidateUserPath(paths.Root, candidateState, false); err != nil { + return Status{}, fmt.Errorf("provider candidate state path: %w", err) + } + if err := prepareCandidateState(paths.ProviderState, candidateState); err != nil { + return Status{}, err + } + journal := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, + ID: "refresh-" + digestHex(update.SHA256)[:16], + Phase: JournalPrepared, + Candidate: selection, + StartedAt: now, + UpdatedAt: now, + } + if activeFound { + previous := active + journal.Previous = &previous + } + if err := AtomicWriteJSON(paths.Journal, journal); err != nil { + return Status{}, fmt.Errorf("write prepared refresh journal: %w", err) + } + activeChanged := false + rollback := func(cause error) error { + return errors.Join(cause, refresher.rollback(ctx, config, paths, journal, activeChanged)) + } + if err := refresher.removeContainer(ctx, config, config.CandidateContainer); err != nil { + return Status{}, rollback(fmt.Errorf("remove stale provider candidate: %w", err)) + } + if _, err := refresher.Runner.Run(ctx, candidateProviderCommand(config, paths, candidateState, selection)); err != nil { + return Status{}, rollback(fmt.Errorf("start provider candidate: %w", err)) + } + if err := refresher.runProbe(ctx, providerProbeCommand(config, paths, config.CandidateContainer, selection)); err != nil { + return Status{}, rollback(fmt.Errorf("probe provider candidate: %w", err)) + } + journal.Phase = JournalActivated + journal.UpdatedAt = refresher.now() + if err := AtomicWriteJSON(paths.Journal, journal); err != nil { + return Status{}, rollback(fmt.Errorf("write activated refresh journal: %w", err)) + } + newActive := ActiveState{ProtocolVersion: ActiveStateProtocolVersion, Current: selection, UpdatedAt: journal.UpdatedAt} + if activeFound { + previous := active.Current + newActive.Previous = &previous + } + if err := AtomicWriteJSON(paths.ActiveState, newActive); err != nil { + return Status{}, rollback(fmt.Errorf("activate provider state: %w", err)) + } + activeChanged = true + if err := refresher.restartProvider(ctx); err != nil { + return Status{}, rollback(fmt.Errorf("restart active provider: %w", err)) + } + if err := refresher.runProbe(ctx, providerProbeCommand(config, paths, config.StableContainer, selection)); err != nil { + return Status{}, rollback(fmt.Errorf("probe active provider: %w", err)) + } + journal.Phase = JournalCommitted + journal.UpdatedAt = refresher.now() + if err := AtomicWriteJSON(paths.Journal, journal); err != nil { + return Status{}, rollback(fmt.Errorf("commit refresh journal: %w", err)) + } + if err := refresher.removeContainer(ctx, config, config.CandidateContainer); err != nil { + return Status{}, fmt.Errorf("remove provider candidate: %w", err) + } + if err := removeDurableFile(paths.Journal); err != nil { + return Status{}, fmt.Errorf("remove committed refresh journal: %w", err) + } + return statusForActive(newActive, true, refresher.now()), nil +} + +func (refresher Refresher) ServeActive(ctx context.Context, config Config) error { + if refresher.Runner == nil { + return errors.New("command runner is required") + } + paths := LifecyclePathsFor(config) + if err := validateInstallRoot(paths.Root); err != nil { + return err + } + active, found, err := readActiveState(paths.ActiveState) + if err != nil { + return err + } + if !found { + return errors.New("retained provider has no active image") + } + if err := validateProviderEnvironment(config, paths.ProviderEnv); err != nil { + return err + } + if err := validateSecretFile(paths.CAFile); err != nil { + return fmt.Errorf("provider CA file: %w", err) + } + if err := ValidateUserPath(config.InstallRoot, paths.ProviderState, true); err != nil { + return fmt.Errorf("provider state path: %w", err) + } + for name, path := range map[string]string{"provider environment": paths.ProviderEnv, "provider CA": paths.CAFile} { + if err := ValidateUserPath(paths.Root, path, true); err != nil { + return fmt.Errorf("%s path: %w", name, err) + } + } + output, err := refresher.Runner.Run(ctx, Command{ + Path: config.PodmanPath, + Args: []string{"image", "inspect", "--format", "{{.Id}}", active.Current.ImageRef}, + }) + if err != nil { + return fmt.Errorf("inspect active provider image: %w", err) + } + if imageID := strings.TrimSpace(string(output)); imageID != active.Current.ImageID { + return errors.New("active provider image id does not match durable state") + } + return refresher.Runner.Exec(Command{Path: config.PodmanPath, Args: []string{ + "run", "--rm", "--name", config.StableContainer, + "--network", config.ContainerNetwork, + "--read-only", "--cap-drop", "all", "--security-opt", "no-new-privileges", + "--env-file", paths.ProviderEnv, + "--volume", paths.ProviderState + ":" + providerStateMount + ":rw", + "--volume", paths.TLSRoot + ":" + providerTLSMount + ":ro", + active.Current.ImageID, providerListenAddr, + }}) +} + +func VerifyCurrentUpdate(ctx context.Context, config Config, runner CommandRunner) (VerifiedUpdate, error) { + output, err := runner.Run(ctx, Command{ + Path: config.ComputeAgentPath, + Args: []string{ + "supervisor-update", "verify", + "-config", config.SupervisorConfigPath, + "-format", "auto", + "-component", "provider", + "-plugin", GitHubPluginID, + "-component-id", config.ComponentID, + }, + }) + if err != nil { + return VerifiedUpdate{}, fmt.Errorf("verify supervisor provider update: %w", err) + } + if len(output) > MaxStateFileBytes { + return VerifiedUpdate{}, errors.New("verified update output exceeds 1 MiB") + } + var envelope verifiedUpdateCommandOutput + if err := decodeStrictJSON(bytes.NewReader(output), &envelope); err != nil { + return VerifiedUpdate{}, fmt.Errorf("decode verified update output: %w", err) + } + update := VerifiedUpdate{ + WorkerID: envelope.WorkerID, DirectiveID: envelope.DirectiveID, + CampaignID: envelope.CampaignID, Component: envelope.Component, + PluginID: envelope.PluginID, ComponentID: envelope.ComponentID, + Version: envelope.Version, Format: envelope.Format, + Path: envelope.Path, SHA256: envelope.SHA256, + } + if err := update.Validate(); err != nil { + return VerifiedUpdate{}, fmt.Errorf("validate verified update: %w", err) + } + if update.WorkerID != config.WorkerID { + return VerifiedUpdate{}, errors.New("verified update worker_id does not match retained worker") + } + if update.PluginID != config.PluginID || update.ComponentID != config.ComponentID { + return VerifiedUpdate{}, errors.New("verified update plugin_id or component_id does not match retained provider") + } + digest, err := hashRegularFile(update.Path, true) + if err != nil { + return VerifiedUpdate{}, fmt.Errorf("verify update path: %w", err) + } + if digest != update.SHA256 { + return VerifiedUpdate{}, errors.New("verified update path digest does not match command projection") + } + return update, nil +} + +type verifiedUpdateCommandOutput struct { + WorkerID string `json:"worker_id"` + DirectiveID string `json:"directive_id"` + CampaignID string `json:"campaign_id"` + DirectiveIssuedAt time.Time `json:"directive_issued_at"` + DirectiveExpiresAt time.Time `json:"directive_expires_at"` + DirectiveSignature json.RawMessage `json:"directive_signature"` + Component string `json:"component"` + PluginID string `json:"plugin_id"` + ComponentID string `json:"component_id"` + Version string `json:"version"` + Format string `json:"format"` + ArtifactURL string `json:"artifact_url"` + ArtifactSizeBytes int64 `json:"artifact_size_bytes"` + ArtifactSignature json.RawMessage `json:"artifact_signature"` + Directive json.RawMessage `json:"directive"` + Artifact json.RawMessage `json:"artifact"` + Path string `json:"path"` + SHA256 string `json:"sha256"` + AppliedAt time.Time `json:"applied_at"` +} + +func candidateProviderCommand(config Config, paths LifecyclePaths, candidateState string, selection ImageSelection) Command { + return Command{Path: config.PodmanPath, Args: []string{ + "run", "--detach", "--name", config.CandidateContainer, + "--network", config.ContainerNetwork, + "--read-only", "--cap-drop", "all", "--security-opt", "no-new-privileges", + "--env-file", paths.ProviderEnv, + "--volume", candidateState + ":" + providerStateMount + ":rw", + "--volume", paths.TLSRoot + ":" + providerTLSMount + ":ro", + selection.ImageID, providerListenAddr, + }} +} + +func providerProbeCommand(config Config, paths LifecyclePaths, target string, selection ImageSelection) Command { + arguments := []string{ + "run", "--rm", "--name", target + "-probe", + "--network", config.ContainerNetwork, + "--read-only", "--cap-drop", "all", "--security-opt", "no-new-privileges", + "--env-file", paths.ProbeEnv, + "--volume", paths.CAFile + ":" + providerCAPath + ":ro", + selection.ImageID, + "probe", "-url", "https://" + target + ":18090", "-ca-file", providerCAPath, + "-organization", config.Organization, "-repository", config.Repository, + "-workflow", config.Workflow, "-ref", config.Ref, + "-runner-name", config.RunnerName, "-runner-group", config.RunnerGroup, + } + for _, label := range config.Labels { + arguments = append(arguments, "-label", label) + } + return Command{Path: config.PodmanPath, Args: arguments} +} + +func (refresher Refresher) restartProvider(ctx context.Context) error { + _, err := refresher.Runner.Run(ctx, Command{Path: "/usr/bin/systemctl", Args: []string{"--user", "restart", providerServiceUnit}}) + return err +} + +func (refresher Refresher) removeContainer(ctx context.Context, config Config, name string) error { + _, err := refresher.Runner.Run(ctx, Command{Path: config.PodmanPath, Args: []string{"rm", "--force", "--ignore", name}}) + return err +} + +func (refresher Refresher) runProbe(ctx context.Context, command Command) error { + delays := []time.Duration{250 * time.Millisecond, 500 * time.Millisecond, time.Second, 2 * time.Second} + var lastErr error + for attempt := 0; attempt <= len(delays); attempt++ { + if _, err := refresher.Runner.Run(ctx, command); err == nil { + return nil + } else { + lastErr = err + } + if attempt == len(delays) { + break + } + if err := refresher.sleep(ctx, delays[attempt]); err != nil { + return err + } + } + return lastErr +} + +func (refresher Refresher) sleep(ctx context.Context, duration time.Duration) error { + if refresher.Sleep != nil { + return refresher.Sleep(ctx, duration) + } + timer := time.NewTimer(duration) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func (refresher Refresher) rollback(ctx context.Context, config Config, paths LifecyclePaths, journal TransactionJournal, activeChanged bool) error { + rollbackContext, cancelRollback := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancelRollback() + var rollbackErr error + if activeChanged { + if journal.Previous != nil { + if err := AtomicWriteJSON(paths.ActiveState, *journal.Previous); err != nil { + rollbackErr = errors.Join(rollbackErr, err) + } else if err := refresher.restartProvider(rollbackContext); err != nil { + rollbackErr = errors.Join(rollbackErr, err) + } else if err := refresher.runProbe(rollbackContext, providerProbeCommand(config, paths, config.StableContainer, journal.Previous.Current)); err != nil { + rollbackErr = errors.Join(rollbackErr, err) + } + } else { + rollbackErr = errors.Join(rollbackErr, removeDurableFile(paths.ActiveState)) + _, stopErr := refresher.Runner.Run(rollbackContext, Command{Path: "/usr/bin/systemctl", Args: []string{"--user", "stop", providerServiceUnit}}) + rollbackErr = errors.Join(rollbackErr, stopErr) + } + } + rollbackErr = errors.Join(rollbackErr, refresher.removeContainer(rollbackContext, config, config.CandidateContainer)) + if rollbackErr == nil { + rollbackErr = removeDurableFile(paths.Journal) + } + return rollbackErr +} + +func (refresher Refresher) recoverInterrupted(ctx context.Context, config Config, paths LifecyclePaths) error { + var journal TransactionJournal + if err := ReadStrictJSONFile(paths.Journal, &journal); err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return fmt.Errorf("read interrupted refresh journal: %w", err) + } + if err := journal.Validate(); err != nil { + return fmt.Errorf("validate interrupted refresh journal: %w", err) + } + if journal.Previous == nil && journal.Phase != JournalCommitted { + if err := removeDurableFile(paths.ActiveState); err != nil { + return fmt.Errorf("remove interrupted initial active state: %w", err) + } + if journal.Phase == JournalActivated { + if _, err := refresher.Runner.Run(ctx, Command{Path: "/usr/bin/systemctl", Args: []string{"--user", "stop", providerServiceUnit}}); err != nil { + return fmt.Errorf("stop interrupted initial provider: %w", err) + } + } + } else { + recovered, err := RecoverActiveState(journal) + if err != nil { + return err + } + if err := AtomicWriteJSON(paths.ActiveState, recovered); err != nil { + return fmt.Errorf("write recovered active state: %w", err) + } + if journal.Phase == JournalActivated { + if err := refresher.restartProvider(ctx); err != nil { + return fmt.Errorf("restart recovered provider: %w", err) + } + if err := refresher.runProbe(ctx, providerProbeCommand(config, paths, config.StableContainer, recovered.Current)); err != nil { + return fmt.Errorf("probe recovered provider: %w", err) + } + } + } + if err := refresher.removeContainer(ctx, config, config.CandidateContainer); err != nil { + return fmt.Errorf("remove interrupted provider candidate: %w", err) + } + if err := removeDurableFile(paths.Journal); err != nil { + return fmt.Errorf("remove recovered refresh journal: %w", err) + } + return nil +} + +func (refresher Refresher) now() time.Time { + if refresher.Now == nil { + return time.Now().UTC() + } + return refresher.Now().UTC() +} + +func stageVerifiedProvider(update VerifiedUpdate, paths LifecyclePaths) error { + destination := paths.PackageBinary(update.SHA256) + if existingDigest, err := hashRegularFile(destination, true); err == nil && existingDigest == update.SHA256 { + return nil + } + if err := os.MkdirAll(paths.PackageDir(update.SHA256), 0o700); err != nil { + return fmt.Errorf("create provider package directory: %w", err) + } + temporary, err := os.CreateTemp(paths.PackageDir(update.SHA256), ".provider-*.tmp") + if err != nil { + return fmt.Errorf("create provider package temporary file: %w", err) + } + temporaryPath := temporary.Name() + defer func() { + _ = temporary.Close() + _ = os.Remove(temporaryPath) + }() + source, err := os.Open(update.Path) + if err != nil { + return fmt.Errorf("open verified provider package: %w", err) + } + defer source.Close() + copied, err := io.Copy(temporary, io.LimitReader(source, maxProviderPackageBytes+1)) + if err != nil { + return fmt.Errorf("copy verified provider package: %w", err) + } + if copied > maxProviderPackageBytes { + return fmt.Errorf("verified provider package exceeds %d bytes", maxProviderPackageBytes) + } + if err := temporary.Chmod(0o700); err != nil { + return fmt.Errorf("mark provider package executable: %w", err) + } + if err := temporary.Sync(); err != nil { + return fmt.Errorf("sync provider package: %w", err) + } + if err := temporary.Close(); err != nil { + return fmt.Errorf("close provider package: %w", err) + } + if digest, err := hashRegularFile(temporaryPath, true); err != nil || digest != update.SHA256 { + return errors.New("staged provider package digest mismatch") + } + if err := os.Rename(temporaryPath, destination); err != nil { + return fmt.Errorf("activate provider package: %w", err) + } + return syncDirectory(filepath.Dir(destination)) +} + +func prepareCandidateState(source, destination string) error { + if err := os.MkdirAll(source, 0o700); err != nil { + return fmt.Errorf("create provider state: %w", err) + } + if err := os.RemoveAll(destination); err != nil { + return fmt.Errorf("remove stale candidate state: %w", err) + } + if err := CloneRegularTree(source, destination, CloneLimits{MaxFiles: 10_000, MaxBytes: 1 << 30}); err != nil { + return fmt.Errorf("clone provider candidate state: %w", err) + } + return nil +} + +func readActiveState(path string) (ActiveState, bool, error) { + var active ActiveState + if err := ReadStrictJSONFile(path, &active); err != nil { + if errors.Is(err, os.ErrNotExist) { + return ActiveState{}, false, nil + } + return ActiveState{}, false, err + } + if err := active.Validate(); err != nil { + return ActiveState{}, false, fmt.Errorf("validate active state: %w", err) + } + return active, true, nil +} + +func statusForActive(active ActiveState, serviceActive bool, now time.Time) Status { + return Status{ + ProtocolVersion: StatusProtocolVersion, + Installed: true, + ServiceActive: serviceActive, + CurrentVersion: active.Current.Update.Version, + CurrentSHA256: active.Current.Update.SHA256, + ObservedAt: now, + } +} + +func validateProviderEnvironment(config Config, path string) error { + values, err := readEnvironmentFile(path) + if err != nil { + return fmt.Errorf("provider environment: %w", err) + } + for _, required := range []string{"GITHUB_RUNNER_PROVIDER_TOKEN", "GITHUB_RUNNER_PROVIDER_GITHUB_TOKEN"} { + if values[required] == "" { + return fmt.Errorf("provider environment is missing %s", required) + } + } + for _, expected := range []struct { + key string + value string + }{ + {key: "GITHUB_RUNNER_PROVIDER_STATE_DIR", value: providerStateMount}, + {key: "GITHUB_RUNNER_PROVIDER_TLS_CERT_FILE", value: providerTLSCertPath}, + {key: "GITHUB_RUNNER_PROVIDER_TLS_KEY_FILE", value: providerTLSKeyPath}, + } { + if values[expected.key] != expected.value { + return errors.New("provider environment contains an invalid runtime path") + } + } + for _, expected := range []struct { + key string + value string + }{ + {key: "GITHUB_RUNNER_PROVIDER_REPOSITORIES", value: config.Repository}, + {key: "GITHUB_RUNNER_PROVIDER_ORGANIZATIONS", value: config.Organization}, + {key: "GITHUB_RUNNER_PROVIDER_RUNNER_GROUPS", value: config.RunnerGroup}, + } { + if !commaSeparatedEnvironmentContains(values[expected.key], expected.value) { + return errors.New("provider environment is missing a required GitHub allowlist value") + } + } + for key := range values { + if !allowedProviderEnvironmentKey(key) { + return errors.New("provider environment contains an unsupported key") + } + } + return nil +} + +func commaSeparatedEnvironmentContains(value, expected string) bool { + for item := range strings.SplitSeq(value, ",") { + if strings.TrimSpace(item) == expected { + return true + } + } + return false +} + +func allowedProviderEnvironmentKey(key string) bool { + switch key { + case "GITHUB_RUNNER_PROVIDER_TOKEN", + "GITHUB_RUNNER_PROVIDER_GITHUB_TOKEN", + "GITHUB_RUNNER_PROVIDER_STATE_DIR", + "GITHUB_RUNNER_PROVIDER_REPOSITORIES", + "GITHUB_RUNNER_PROVIDER_ORGANIZATIONS", + "GITHUB_RUNNER_PROVIDER_RUNNER_GROUPS", + "GITHUB_RUNNER_PROVIDER_TLS_CERT_FILE", + "GITHUB_RUNNER_PROVIDER_TLS_KEY_FILE", + "GITHUB_API_BASE_URL": + return true + default: + return false + } +} + +func validateProbeEnvironment(path string) error { + values, err := readEnvironmentFile(path) + if err != nil { + return fmt.Errorf("probe environment: %w", err) + } + if len(values) != 1 || values["GITHUB_RUNNER_PROVIDER_TOKEN"] == "" { + return errors.New("probe environment must contain only GITHUB_RUNNER_PROVIDER_TOKEN") + } + return nil +} + +func readEnvironmentFile(path string) (map[string]string, error) { + if err := validateSecretFile(path); err != nil { + return nil, err + } + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + values := make(map[string]string) + scanner := bufio.NewScanner(io.LimitReader(file, MaxStateFileBytes+1)) + for scanner.Scan() { + line := scanner.Text() + if line == "" { + continue + } + key, value, found := strings.Cut(line, "=") + if !found || !safeEnvironmentKey(key) || value == "" || strings.ContainsAny(value, "\r\n\x00") { + return nil, errors.New("environment file contains an invalid entry") + } + if _, exists := values[key]; exists { + return nil, errors.New("environment file contains a duplicate key") + } + values[key] = value + } + if err := scanner.Err(); err != nil { + return nil, errors.New("read environment file") + } + return values, nil +} + +func safeEnvironmentKey(value string) bool { + if value == "" { + return false + } + for index, r := range value { + if r >= 'A' && r <= 'Z' || r == '_' || index > 0 && r >= '0' && r <= '9' { + continue + } + return false + } + return true +} + +func validateSecretFile(path string) error { + info, err := os.Lstat(path) + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Size() <= 0 || info.Size() > MaxStateFileBytes { + return errors.New("secret file must be a non-empty regular file of at most 1 MiB") + } + if err := validateStateMode(info); err != nil { + return err + } + return validateOwner(info) +} + +func hashRegularFile(path string, requireExecutable bool) (string, error) { + entry, err := os.Lstat(path) + if err != nil { + return "", err + } + if !entry.Mode().IsRegular() { + return "", errors.New("path must be a regular file") + } + if requireExecutable && executableModeRequired() && entry.Mode().Perm()&0o111 == 0 { + return "", errors.New("path must be executable") + } + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + opened, err := file.Stat() + if err != nil || !opened.Mode().IsRegular() || !os.SameFile(entry, opened) { + return "", errors.New("path changed during open") + } + hasher := sha256.New() + if _, err := io.Copy(hasher, file); err != nil { + return "", err + } + return "sha256:" + hex.EncodeToString(hasher.Sum(nil)), nil +} + +func executableModeRequired() bool { + return os.PathSeparator == '/' +} + +func providerImageRef(digest string) string { + return "localhost/workflow-plugin-github-runner-provider:sha256-" + digestHex(digest) +} + +func digestHex(digest string) string { + return strings.TrimPrefix(digest, "sha256:") +} + +func removeDurableFile(path string) error { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return syncDirectory(filepath.Dir(path)) +} + +func validateInstallRoot(path string) error { + info, err := os.Lstat(path) + if err != nil { + return fmt.Errorf("inspect retained provider install root: %w", err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return errors.New("retained provider install root must be a real directory") + } + if err := validateOwner(info); err != nil { + return fmt.Errorf("retained provider install root ownership: %w", err) + } + if executableModeRequired() && info.Mode().Perm()&0o077 != 0 { + return errors.New("retained provider install root mode must not allow group or other access") + } + return nil +} diff --git a/internal/retainedprovider/refresh_test.go b/internal/retainedprovider/refresh_test.go new file mode 100644 index 0000000..255e19a --- /dev/null +++ b/internal/retainedprovider/refresh_test.go @@ -0,0 +1,866 @@ +package retainedprovider + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" +) + +type recordingCommandRunner struct { + commands []Command + run func(context.Context, Command) ([]byte, error) + exec func(Command) error +} + +func (runner *recordingCommandRunner) Run(ctx context.Context, command Command) ([]byte, error) { + runner.commands = append(runner.commands, command) + if runner.run == nil { + return nil, nil + } + return runner.run(ctx, command) +} + +func (runner *recordingCommandRunner) Exec(command Command) error { + runner.commands = append(runner.commands, command) + if runner.exec == nil { + return nil + } + return runner.exec(command) +} + +func TestVerifyCurrentUpdateUsesExactComputeAgentCommandAndStrictProjection(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + payload := writeTestProviderPayload(t, home, "verified-provider-v1") + digest := fileDigestForTest(t, payload) + runner := &recordingCommandRunner{} + runner.run = func(_ context.Context, _ Command) ([]byte, error) { + return testVerifiedUpdateJSON(config, payload, digest), nil + } + + update, err := VerifyCurrentUpdate(t.Context(), config, runner) + if err != nil { + t.Fatalf("verify current update: %v", err) + } + wantArgs := []string{ + "supervisor-update", "verify", + "-config", config.SupervisorConfigPath, + "-format", "auto", + "-component", "provider", + "-plugin", GitHubPluginID, + "-component-id", config.ComponentID, + } + if len(runner.commands) != 1 || runner.commands[0].Path != config.ComputeAgentPath || !reflect.DeepEqual(runner.commands[0].Args, wantArgs) { + t.Fatalf("verify commands = %+v want path=%q args=%q", runner.commands, config.ComputeAgentPath, wantArgs) + } + if update.WorkerID != config.WorkerID || update.ComponentID != config.ComponentID || update.Path != payload || update.SHA256 != digest { + t.Fatalf("verified update = %+v", update) + } + + runner.commands = nil + runner.run = func(_ context.Context, _ Command) ([]byte, error) { + data := testVerifiedUpdateJSON(config, payload, digest) + return append(data[:len(data)-2], []byte(`,"unexpected":true}`+"\n")...), nil + } + if _, err := VerifyCurrentUpdate(t.Context(), config, runner); err == nil || !strings.Contains(err.Error(), "unknown field") { + t.Fatalf("unknown verify field err = %v", err) + } +} + +func TestVerifyCurrentUpdateRejectsIdentityAndDigestMismatch(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + payload := writeTestProviderPayload(t, home, "verified-provider-v1") + digest := fileDigestForTest(t, payload) + + for _, tc := range []struct { + name string + mutate func(*Config, *string, *string) + want string + }{ + {name: "worker", mutate: func(c *Config, _, _ *string) { c.WorkerID = "other-worker" }, want: "worker_id"}, + {name: "component", mutate: func(c *Config, _, _ *string) { c.ComponentID = "other-component" }, want: "component_id"}, + {name: "digest", mutate: func(_ *Config, _ *string, d *string) { *d = "sha256:" + strings.Repeat("f", 64) }, want: "digest"}, + {name: "path", mutate: func(_ *Config, p, _ *string) { *p = filepath.Join(home, "missing-provider") }, want: "path"}, + } { + t.Run(tc.name, func(t *testing.T) { + expectedConfig := config + outputPath, outputDigest := payload, digest + tc.mutate(&expectedConfig, &outputPath, &outputDigest) + runner := &recordingCommandRunner{run: func(_ context.Context, _ Command) ([]byte, error) { + return testVerifiedUpdateJSON(expectedConfig, outputPath, outputDigest), nil + }} + if _, err := VerifyCurrentUpdate(t.Context(), config, runner); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("VerifyCurrentUpdate err = %v want %q", err, tc.want) + } + }) + } +} + +func TestInitialRefreshRequiresInstallerDigestMatch(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + if err := os.MkdirAll(config.InstallRoot, 0o700); err != nil { + t.Fatalf("mkdir install root: %v", err) + } + payload := writeTestProviderPayload(t, home, "verified-provider-v1") + digest := fileDigestForTest(t, payload) + other := writeTestProviderPayload(t, home, "different-installer") + runner := refreshTestRunner(config, payload, digest) + refresher := Refresher{ + Runner: runner, + ExecutablePath: func() (string, error) { return other, nil }, + Now: func() time.Time { return time.Unix(1_700_000_000, 0).UTC() }, + } + if _, err := refresher.Refresh(t.Context(), config); err == nil || !strings.Contains(err.Error(), "installer digest") { + t.Fatalf("initial refresh err = %v", err) + } + if len(runner.commands) != 1 || runner.commands[0].Path != config.ComputeAgentPath { + t.Fatalf("refresh mutated runtime before self-digest check: %+v", runner.commands) + } +} + +func TestRefreshBuildsAndPreflightsIsolatedCandidateThenStable(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + payload := writeTestProviderPayload(t, home, "verified-provider-v1") + digest := fileDigestForTest(t, payload) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + if err := os.WriteFile(filepath.Join(paths.ProviderState, "ownership.json"), []byte(`{"owner":"stg"}`), 0o600); err != nil { + t.Fatalf("write provider state: %v", err) + } + runner := refreshTestRunner(config, payload, digest) + now := time.Unix(1_700_000_000, 0).UTC() + refresher := Refresher{ + Runner: runner, + ExecutablePath: func() (string, error) { return payload, nil }, + Now: func() time.Time { return now }, + } + status, err := refresher.Refresh(t.Context(), config) + if err != nil { + t.Fatalf("refresh: %v\ncommands=%+v", err, runner.commands) + } + if !status.Installed || !status.ServiceActive || status.CurrentSHA256 != digest || status.ObservedAt != now { + t.Fatalf("refresh status = %+v", status) + } + var active ActiveState + if err := ReadStrictJSONFile(paths.ActiveState, &active); err != nil { + t.Fatalf("read active state: %v", err) + } + if active.Current.Update.SHA256 != digest || active.Current.ImageID != testProviderImageID { + t.Fatalf("active state = %+v", active) + } + if _, err := os.Stat(filepath.Join(paths.CandidateState(digest), "ownership.json")); err != nil { + t.Fatalf("candidate did not receive bounded state clone: %v", err) + } + assertRefreshCommandIsolation(t, runner.commands, config, paths) + + before := len(runner.commands) + status, err = refresher.Refresh(t.Context(), config) + if err != nil || status.CurrentSHA256 != digest { + t.Fatalf("idempotent refresh status=%+v err=%v", status, err) + } + for _, command := range runner.commands[before:] { + if command.Path == config.PodmanPath || filepath.Base(command.Path) == "systemctl" { + t.Fatalf("digest-idempotent refresh mutated runtime: %+v", command) + } + } +} + +func TestRefreshRejectsGitHubCredentialInProbeEnvironment(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + payload := writeTestProviderPayload(t, home, "verified-provider-v1") + digest := fileDigestForTest(t, payload) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + if err := os.WriteFile(paths.ProbeEnv, []byte("GITHUB_RUNNER_PROVIDER_TOKEN=provider-secret\nGITHUB_TOKEN=github-secret\n"), 0o600); err != nil { + t.Fatalf("write invalid probe env: %v", err) + } + runner := refreshTestRunner(config, payload, digest) + refresher := Refresher{Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }} + if _, err := refresher.Refresh(t.Context(), config); err == nil || !strings.Contains(err.Error(), "probe environment") { + t.Fatalf("probe credential isolation err = %v", err) + } + if strings.Contains(commandTranscript(runner.commands), "github-secret") { + t.Fatalf("command transcript leaked GitHub credential: %+v", runner.commands) + } +} + +func TestRefreshRejectsUnrelatedProviderEnvironmentVariable(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + payload := writeTestProviderPayload(t, home, "verified-provider-provider-env") + digest := fileDigestForTest(t, payload) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + if err := os.WriteFile(paths.ProviderEnv, []byte("GITHUB_RUNNER_PROVIDER_TOKEN=provider-secret\nGITHUB_RUNNER_PROVIDER_GITHUB_TOKEN=github-secret\nAWS_SECRET_ACCESS_KEY=unrelated-secret\n"), 0o600); err != nil { + t.Fatalf("write invalid provider env: %v", err) + } + runner := refreshTestRunner(config, payload, digest) + refresher := Refresher{Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }} + if _, err := refresher.Refresh(t.Context(), config); err == nil || !strings.Contains(err.Error(), "provider environment") { + t.Fatalf("provider credential isolation err = %v", err) + } + if strings.Contains(commandTranscript(runner.commands), "unrelated-secret") { + t.Fatalf("command transcript leaked unrelated credential: %+v", runner.commands) + } +} + +func TestRefreshRejectsIncompleteProviderEnvironment(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + payload := writeTestProviderPayload(t, home, "verified-provider-incomplete-env") + digest := fileDigestForTest(t, payload) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + if err := os.WriteFile(paths.ProviderEnv, []byte("GITHUB_RUNNER_PROVIDER_TOKEN=provider-secret\nGITHUB_RUNNER_PROVIDER_GITHUB_TOKEN=github-secret\n"), 0o600); err != nil { + t.Fatalf("write incomplete provider env: %v", err) + } + runner := refreshTestRunner(config, payload, digest) + refresher := Refresher{Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }} + if _, err := refresher.Refresh(t.Context(), config); err == nil || !strings.Contains(err.Error(), "provider environment") { + t.Fatalf("incomplete provider environment err = %v", err) + } +} + +func TestRefreshFailurePreservesPreviousActiveImageAndCleansCandidate(t *testing.T) { + for _, phase := range []string{"build", "stale-candidate", "candidate", "candidate-probe", "stable-restart", "stable-probe", "canceled"} { + t.Run(phase, func(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + previous := previousActiveStateForTest(t, home) + if err := AtomicWriteJSON(paths.ActiveState, previous); err != nil { + t.Fatalf("write previous active state: %v", err) + } + payload := writeTestProviderPayload(t, home, "verified-provider-v2") + digest := fileDigestForTest(t, payload) + runner := refreshTestRunner(config, payload, digest) + baseRun := runner.run + failedRestart := false + failedStaleCleanup := false + refreshContext := t.Context() + cancelRefresh := func() {} + if phase == "canceled" { + refreshContext, cancelRefresh = context.WithCancel(t.Context()) + } + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if phase == "canceled" && isCandidateStart(command, config) { + cancelRefresh() + return nil, ctx.Err() + } + if phase == "build" && command.Path == config.PodmanPath && firstArg(command.Args) == "build" { + return nil, errors.New("build failed") + } + if phase == "candidate" && isCandidateStart(command, config) { + return nil, errors.New("candidate failed") + } + if phase == "stale-candidate" && firstArg(command.Args) == "rm" && containsArg(command.Args, config.CandidateContainer) && !failedStaleCleanup { + failedStaleCleanup = true + return nil, errors.New("stale candidate cleanup failed") + } + if phase == "candidate-probe" && isProbeFor(command, config.CandidateContainer) { + return nil, errors.New("candidate probe failed") + } + if phase == "stable-restart" && filepath.Base(command.Path) == "systemctl" && !failedRestart { + failedRestart = true + return nil, errors.New("restart failed") + } + if phase == "stable-probe" && isProbeFor(command, config.StableContainer) && containsArg(command.Args, testProviderImageID) { + return nil, errors.New("stable probe failed") + } + return baseRun(ctx, command) + } + refresher := Refresher{ + Runner: runner, Now: func() time.Time { return time.Unix(1_700_000_100, 0).UTC() }, + Sleep: func(context.Context, time.Duration) error { return nil }, + } + if _, err := refresher.Refresh(refreshContext, config); err == nil { + t.Fatalf("%s refresh unexpectedly succeeded", phase) + } + var active ActiveState + if err := ReadStrictJSONFile(paths.ActiveState, &active); err != nil { + t.Fatalf("read active state after %s failure: %v", phase, err) + } + if active.Current.ImageID != previous.Current.ImageID || active.Current.Update.SHA256 != previous.Current.Update.SHA256 { + t.Fatalf("%s failure replaced active state: got=%+v want=%+v", phase, active, previous) + } + if _, err := os.Stat(paths.Journal); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("%s rollback journal remains: %v", phase, err) + } + transcript := commandTranscript(runner.commands) + if strings.Contains(transcript, "image rm") { + t.Fatalf("%s failure attempted to prune retained image:\n%s", phase, transcript) + } + if phase != "build" && !strings.Contains(transcript, "rm --force --ignore "+config.CandidateContainer) { + t.Fatalf("%s failure did not clean candidate:\n%s", phase, transcript) + } + }) + } +} + +func TestRefreshRecoversEveryInterruptedJournalPhaseIdempotently(t *testing.T) { + for _, phase := range []JournalPhase{JournalPrepared, JournalActivated, JournalCommitted} { + t.Run(string(phase), func(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + previous := previousActiveStateForTest(t, home) + candidatePayload := writeTestProviderPayload(t, home, "candidate-recovery") + candidateDigest := fileDigestForTest(t, candidatePayload) + candidate := selectionForDigest(candidatePayload, candidateDigest, "v1.0.32", "directive-candidate", "sha256:"+strings.Repeat("e", 64), time.Unix(1_700_000_100, 0).UTC()) + journal := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, + ID: "refresh-recovery", + Phase: phase, + Previous: &previous, + Candidate: candidate, + StartedAt: time.Unix(1_700_000_100, 0).UTC(), + UpdatedAt: time.Unix(1_700_000_101, 0).UTC(), + } + if err := AtomicWriteJSON(paths.Journal, journal); err != nil { + t.Fatalf("write journal: %v", err) + } + if err := AtomicWriteJSON(paths.ActiveState, ActiveState{ProtocolVersion: ActiveStateProtocolVersion, Current: candidate, Previous: &previous.Current, UpdatedAt: journal.UpdatedAt}); err != nil { + t.Fatalf("write interrupted active state: %v", err) + } + runner := &recordingCommandRunner{} + refresher := Refresher{Runner: runner, Sleep: func(context.Context, time.Duration) error { return nil }} + if err := refresher.recoverInterrupted(t.Context(), config, paths); err != nil { + t.Fatalf("recover %s: %v", phase, err) + } + if err := refresher.recoverInterrupted(t.Context(), config, paths); err != nil { + t.Fatalf("idempotent recover %s: %v", phase, err) + } + var active ActiveState + if err := ReadStrictJSONFile(paths.ActiveState, &active); err != nil { + t.Fatalf("read recovered active: %v", err) + } + wantImage := previous.Current.ImageID + if phase == JournalCommitted { + wantImage = candidate.ImageID + } + if active.Current.ImageID != wantImage { + t.Fatalf("%s recovered image = %s want %s", phase, active.Current.ImageID, wantImage) + } + if _, err := os.Stat(paths.Journal); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("%s journal remains after recovery: %v", phase, err) + } + }) + } +} + +func TestServeActiveValidatesImmutableImageThenExecsRestrictedPodman(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + active := previousActiveStateForTest(t, home) + if err := AtomicWriteJSON(paths.ActiveState, active); err != nil { + t.Fatalf("write active state: %v", err) + } + execSentinel := errors.New("exec invoked") + runner := &recordingCommandRunner{ + run: func(_ context.Context, command Command) ([]byte, error) { + if command.Path == config.PodmanPath && len(command.Args) > 1 && command.Args[0] == "image" { + return []byte(active.Current.ImageID + "\n"), nil + } + return nil, nil + }, + exec: func(Command) error { return execSentinel }, + } + refresher := Refresher{Runner: runner} + if err := refresher.ServeActive(t.Context(), config); !errors.Is(err, execSentinel) { + t.Fatalf("serve active err = %v", err) + } + if len(runner.commands) != 2 { + t.Fatalf("serve active commands = %+v", runner.commands) + } + execCommand := runner.commands[1] + if execCommand.Path != config.PodmanPath || firstArg(execCommand.Args) != "run" || !containsAdjacentArgs(execCommand.Args, "--name", config.StableContainer) || !containsAdjacentArgs(execCommand.Args, "--env-file", paths.ProviderEnv) { + t.Fatalf("serve active exec command = %+v", execCommand) + } + transcript := commandTranscript(runner.commands) + for _, required := range []string{"--network bridge", "--read-only", "--cap-drop all", "no-new-privileges", active.Current.ImageID} { + if !strings.Contains(transcript, required) { + t.Fatalf("serve active transcript missing %q:\n%s", required, transcript) + } + } + if strings.Contains(transcript, "provider-secret") || strings.Contains(transcript, "github-secret") || strings.Contains(transcript, "sock") { + t.Fatalf("serve active leaked secret or socket mount:\n%s", transcript) + } +} + +func TestServeActiveRefusesImageIdentityMismatch(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + active := previousActiveStateForTest(t, home) + if err := AtomicWriteJSON(paths.ActiveState, active); err != nil { + t.Fatalf("write active state: %v", err) + } + runner := &recordingCommandRunner{run: func(context.Context, Command) ([]byte, error) { + return []byte("sha256:" + strings.Repeat("f", 64) + "\n"), nil + }} + if err := (Refresher{Runner: runner}).ServeActive(t.Context(), config); err == nil || !strings.Contains(err.Error(), "image id") { + t.Fatalf("serve active mismatch err = %v", err) + } + if len(runner.commands) != 1 { + t.Fatalf("serve active executed mismatched image: %+v", runner.commands) + } +} + +func TestRefreshRetriesDetachedProviderProbe(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + payload := writeTestProviderPayload(t, home, "verified-provider-retry") + digest := fileDigestForTest(t, payload) + runner := refreshTestRunner(config, payload, digest) + baseRun := runner.run + attempts := 0 + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if isProbeFor(command, config.CandidateContainer) { + attempts++ + if attempts < 3 { + return nil, errors.New("provider not ready") + } + } + return baseRun(ctx, command) + } + var sleeps []time.Duration + refresher := Refresher{ + Runner: runner, + ExecutablePath: func() (string, error) { return payload, nil }, + Sleep: func(_ context.Context, duration time.Duration) error { + sleeps = append(sleeps, duration) + return nil + }, + } + if _, err := refresher.Refresh(t.Context(), config); err != nil { + t.Fatalf("refresh with readiness retry: %v", err) + } + if attempts != 3 || !reflect.DeepEqual(sleeps, []time.Duration{250 * time.Millisecond, 500 * time.Millisecond}) { + t.Fatalf("probe attempts=%d sleeps=%v", attempts, sleeps) + } +} + +func TestRefreshLockRejectsConcurrentMutationBeforeVerification(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + payload := writeTestProviderPayload(t, home, "verified-provider-lock") + digest := fileDigestForTest(t, payload) + runner := refreshTestRunner(config, payload, digest) + baseRun := runner.run + buildStarted := make(chan struct{}) + releaseBuild := make(chan struct{}) + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if command.Path == config.PodmanPath && firstArg(command.Args) == "build" { + close(buildStarted) + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-releaseBuild: + } + } + return baseRun(ctx, command) + } + refresher := Refresher{Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }} + done := make(chan error, 1) + go func() { + _, err := refresher.Refresh(t.Context(), config) + done <- err + }() + <-buildStarted + before := len(runner.commands) + if _, err := refresher.Refresh(t.Context(), config); !errors.Is(err, ErrInstallLocked) { + t.Fatalf("concurrent refresh err = %v", err) + } + if len(runner.commands) != before { + t.Fatalf("concurrent refresh reached command runner: before=%d after=%d", before, len(runner.commands)) + } + close(releaseBuild) + if err := <-done; err != nil { + t.Fatalf("first refresh: %v", err) + } +} + +func TestRefreshRejectsSymlinkedCandidateRootWithoutTouchingTarget(t *testing.T) { + if os.PathSeparator != '/' { + t.Skip("symlink behavior varies on Windows") + } + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + previous := previousActiveStateForTest(t, home) + if err := AtomicWriteJSON(paths.ActiveState, previous); err != nil { + t.Fatalf("write active state: %v", err) + } + payload := writeTestProviderPayload(t, home, "verified-provider-symlink") + digest := fileDigestForTest(t, payload) + outside := filepath.Join(t.TempDir(), "outside") + sentinel := filepath.Join(outside, digestHex(digest), "state", "sentinel") + if err := os.MkdirAll(filepath.Dir(sentinel), 0o700); err != nil { + t.Fatalf("mkdir outside target: %v", err) + } + if err := os.WriteFile(sentinel, []byte("keep"), 0o600); err != nil { + t.Fatalf("write outside sentinel: %v", err) + } + if err := os.Symlink(outside, paths.CandidatesRoot); err != nil { + t.Fatalf("symlink candidates root: %v", err) + } + runner := refreshTestRunner(config, payload, digest) + if _, err := (Refresher{Runner: runner}).Refresh(t.Context(), config); err == nil || !strings.Contains(err.Error(), "symlink") { + t.Fatalf("symlinked candidates err = %v", err) + } + if data, err := os.ReadFile(sentinel); err != nil || string(data) != "keep" { + t.Fatalf("outside sentinel changed: data=%q err=%v", data, err) + } +} + +func TestRollbackDoesNotRestartWhenDurableRestoreFails(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + previous := previousActiveStateForTest(t, home) + if err := AtomicWriteJSON(paths.ActiveState, previous); err != nil { + t.Fatalf("write active state: %v", err) + } + payload := writeTestProviderPayload(t, home, "verified-provider-restore-failure") + digest := fileDigestForTest(t, payload) + runner := refreshTestRunner(config, payload, digest) + baseRun := runner.run + restarts := 0 + poisoned := false + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if filepath.Base(command.Path) == "systemctl" && containsArg(command.Args, "restart") { + restarts++ + } + if isProbeFor(command, config.StableContainer) && containsArg(command.Args, testProviderImageID) { + if !poisoned { + poisoned = true + if err := os.Remove(paths.ActiveState); err != nil { + t.Fatalf("remove active state: %v", err) + } + if err := os.Symlink(filepath.Join(home, "outside-active"), paths.ActiveState); err != nil { + t.Fatalf("poison active state: %v", err) + } + } + return nil, errors.New("stable probe failed") + } + return baseRun(ctx, command) + } + refresher := Refresher{Runner: runner, Sleep: func(context.Context, time.Duration) error { return nil }} + if _, err := refresher.Refresh(t.Context(), config); err == nil || !strings.Contains(err.Error(), "symlink") { + t.Fatalf("restore failure err = %v", err) + } + if restarts != 1 { + t.Fatalf("provider restarted %d times after durable restore failure", restarts) + } +} + +func TestCommittedCleanupFailureLeavesRecoverableJournal(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + payload := writeTestProviderPayload(t, home, "verified-provider-cleanup") + digest := fileDigestForTest(t, payload) + runner := refreshTestRunner(config, payload, digest) + baseRun := runner.run + cleanupCalls := 0 + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if command.Path == config.PodmanPath && firstArg(command.Args) == "rm" && containsArg(command.Args, config.CandidateContainer) { + cleanupCalls++ + if cleanupCalls == 2 { + return nil, errors.New("candidate cleanup failed") + } + } + return baseRun(ctx, command) + } + refresher := Refresher{Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }} + if _, err := refresher.Refresh(t.Context(), config); err == nil || !strings.Contains(err.Error(), "candidate") { + t.Fatalf("cleanup failure err = %v", err) + } + var journal TransactionJournal + if err := ReadStrictJSONFile(paths.Journal, &journal); err != nil { + t.Fatalf("committed journal was not retained: %v", err) + } + if journal.Phase != JournalCommitted { + t.Fatalf("journal phase = %s", journal.Phase) + } +} + +func TestOSCommandRunnerDoesNotEchoArgumentsOrOutputOnFailure(t *testing.T) { + secret := "credential-that-must-not-leak" + runner := OSCommandRunner{MaxOutputBytes: 1024} + _, err := runner.Run(t.Context(), Command{ + Path: "/usr/bin/false", + Args: []string{secret}, + }) + if err == nil { + t.Fatal("failing command succeeded") + } + if strings.Contains(err.Error(), secret) { + t.Fatalf("command error leaked argument: %v", err) + } +} + +func TestOSCommandRunnerDoesNotInheritUnrelatedHostSecrets(t *testing.T) { + const secret = "aws-host-secret-that-must-not-leak" + t.Setenv("AWS_SECRET_ACCESS_KEY", secret) + runner := OSCommandRunner{MaxOutputBytes: 1 << 20} + output, err := runner.Run(t.Context(), Command{Path: "/usr/bin/env"}) + if err != nil { + t.Fatalf("run env: %v", err) + } + if strings.Contains(string(output), secret) || strings.Contains(string(output), "AWS_SECRET_ACCESS_KEY") { + t.Fatalf("subprocess inherited unrelated host secret: %s", output) + } +} + +const testProviderImageID = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + +func refreshTestRunner(config Config, payload, digest string) *recordingCommandRunner { + return &recordingCommandRunner{run: func(ctx context.Context, command Command) ([]byte, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + switch { + case command.Path == config.ComputeAgentPath: + return testVerifiedUpdateJSON(config, payload, digest), nil + case command.Path == config.PodmanPath && len(command.Args) >= 2 && command.Args[0] == "image" && command.Args[1] == "inspect": + return []byte(testProviderImageID + "\n"), nil + default: + return nil, nil + } + }} +} + +func assertRefreshCommandIsolation(t *testing.T, commands []Command, config Config, paths LifecyclePaths) { + t.Helper() + transcript := commandTranscript(commands) + for _, required := range []string{ + "build", "FROM scratch", config.CandidateContainer, config.StableContainer, + "--network bridge", "--read-only", "--cap-drop all", "no-new-privileges", + "--env-file " + paths.ProviderEnv, "--env-file " + paths.ProbeEnv, + "probe", "systemctl --user restart", + } { + if !strings.Contains(transcript, required) { + t.Fatalf("command transcript missing %q:\n%s", required, transcript) + } + } + if strings.Contains(transcript, "provider-secret") || strings.Contains(transcript, "github-secret") || strings.Contains(transcript, "/var/run/docker.sock") || strings.Contains(transcript, "/run/podman/podman.sock") { + t.Fatalf("command transcript leaked a secret or mounted a runtime socket:\n%s", transcript) + } + probeCommands := 0 + for _, command := range commands { + if command.Path != config.PodmanPath || !containsArg(command.Args, "probe") { + continue + } + probeCommands++ + if !containsAdjacentArgs(command.Args, "--env-file", paths.ProbeEnv) || containsAdjacentArgs(command.Args, "--env-file", paths.ProviderEnv) { + t.Fatalf("probe command has wrong environment: %+v", command) + } + } + if probeCommands != 2 { + t.Fatalf("probe command count = %d, commands=%+v", probeCommands, commands) + } +} + +func writeRefreshEnvironmentFiles(t *testing.T, paths LifecyclePaths) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(paths.ProviderEnv), 0o700); err != nil { + t.Fatalf("mkdir env dir: %v", err) + } + providerEnvironment := strings.Join([]string{ + "GITHUB_RUNNER_PROVIDER_TOKEN=provider-secret", + "GITHUB_RUNNER_PROVIDER_GITHUB_TOKEN=github-secret", + "GITHUB_RUNNER_PROVIDER_STATE_DIR=" + providerStateMount, + "GITHUB_RUNNER_PROVIDER_REPOSITORIES=GoCodeAlone/workflow-compute", + "GITHUB_RUNNER_PROVIDER_ORGANIZATIONS=GoCodeAlone", + "GITHUB_RUNNER_PROVIDER_RUNNER_GROUPS=ephemeral", + "GITHUB_RUNNER_PROVIDER_TLS_CERT_FILE=" + providerTLSCertPath, + "GITHUB_RUNNER_PROVIDER_TLS_KEY_FILE=" + providerTLSKeyPath, + "", + }, "\n") + if err := os.WriteFile(paths.ProviderEnv, []byte(providerEnvironment), 0o600); err != nil { + t.Fatalf("write provider env: %v", err) + } + if err := os.WriteFile(paths.ProbeEnv, []byte("GITHUB_RUNNER_PROVIDER_TOKEN=provider-secret\n"), 0o600); err != nil { + t.Fatalf("write probe env: %v", err) + } + if err := os.MkdirAll(paths.TLSRoot, 0o700); err != nil { + t.Fatalf("mkdir tls root: %v", err) + } + if err := os.WriteFile(paths.CAFile, []byte("test-ca"), 0o600); err != nil { + t.Fatalf("write ca: %v", err) + } +} + +func writeTestProviderPayload(t *testing.T, home, contents string) string { + t.Helper() + path := filepath.Join(home, contents) + if err := os.WriteFile(path, []byte(contents), 0o700); err != nil { + t.Fatalf("write provider payload: %v", err) + } + return path +} + +func fileDigestForTest(t *testing.T, path string) string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read payload: %v", err) + } + digest := sha256.Sum256(data) + return "sha256:" + hex.EncodeToString(digest[:]) +} + +func testVerifiedUpdateJSON(config Config, payload, digest string) []byte { + return []byte(`{ + "worker_id": "` + config.WorkerID + `", + "directive_id": "directive-1", + "campaign_id": "campaign-1", + "directive_issued_at": "2026-07-13T00:00:00Z", + "directive_expires_at": "2026-07-14T00:00:00Z", + "directive_signature": {}, + "component": "provider", + "plugin_id": "` + GitHubPluginID + `", + "component_id": "` + config.ComponentID + `", + "version": "v1.0.32", + "format": "binary", + "artifact_url": "/v1/artifacts/provider", + "artifact_size_bytes": 20, + "artifact_signature": {}, + "directive": {}, + "artifact": {}, + "path": "` + payload + `", + "sha256": "` + digest + `", + "applied_at": "2026-07-13T00:01:00Z" +} +`) +} + +func commandTranscript(commands []Command) string { + var builder strings.Builder + for _, command := range commands { + builder.WriteString(filepath.Base(command.Path)) + builder.WriteByte(' ') + builder.WriteString(strings.Join(command.Args, " ")) + builder.WriteByte('\n') + if command.Stdin != nil { + builder.Write(command.Stdin) + builder.WriteByte('\n') + } + } + return builder.String() +} + +func previousActiveStateForTest(t *testing.T, home string) ActiveState { + t.Helper() + payload := writeTestProviderPayload(t, home, "verified-provider-v1") + digest := fileDigestForTest(t, payload) + selection := selectionForDigest(payload, digest, "v1.0.31", "directive-previous", "sha256:"+strings.Repeat("c", 64), time.Unix(1_700_000_000, 0).UTC()) + return ActiveState{ProtocolVersion: ActiveStateProtocolVersion, Current: selection, UpdatedAt: selection.ActivatedAt} +} + +func selectionForDigest(path, digest, version, directiveID, imageID string, activatedAt time.Time) ImageSelection { + return ImageSelection{ + Update: VerifiedUpdate{ + WorkerID: "github-runner-linux-stg", DirectiveID: directiveID, CampaignID: "campaign-1", + Component: "provider", PluginID: GitHubPluginID, ComponentID: "github-runner-provider-sidecar", + Version: version, Format: "binary", Path: path, SHA256: digest, + }, + ImageID: imageID, ImageRef: providerImageRef(digest), ActivatedAt: activatedAt, + } +} + +func firstArg(args []string) string { + if len(args) == 0 { + return "" + } + return args[0] +} + +func isCandidateStart(command Command, config Config) bool { + return command.Path == config.PodmanPath && firstArg(command.Args) == "run" && containsAdjacentArgs(command.Args, "--name", config.CandidateContainer) && !containsArg(command.Args, "probe") +} + +func isProbeFor(command Command, target string) bool { + return firstArg(command.Args) == "run" && containsAdjacentArgs(command.Args, "--name", target+"-probe") && containsArg(command.Args, "probe") +} + +func containsArg(args []string, value string) bool { + for _, arg := range args { + if arg == value { + return true + } + } + return false +} + +func containsAdjacentArgs(args []string, first, second string) bool { + for index := 0; index+1 < len(args); index++ { + if args[index] == first && args[index+1] == second { + return true + } + } + return false +} diff --git a/internal/retainedprovider/replace_linux.go b/internal/retainedprovider/replace_linux.go new file mode 100644 index 0000000..85f0c17 --- /dev/null +++ b/internal/retainedprovider/replace_linux.go @@ -0,0 +1,9 @@ +//go:build linux + +package retainedprovider + +import "syscall" + +func replaceProcess(path string, args, environment []string) error { + return syscall.Exec(path, args, environment) +} diff --git a/internal/retainedprovider/replace_other.go b/internal/retainedprovider/replace_other.go new file mode 100644 index 0000000..600faba --- /dev/null +++ b/internal/retainedprovider/replace_other.go @@ -0,0 +1,9 @@ +//go:build !linux + +package retainedprovider + +import "errors" + +func replaceProcess(string, []string, []string) error { + return errors.New("retained foreground execution is supported only on Linux") +} diff --git a/internal/retainedprovider/state.go b/internal/retainedprovider/state.go index 3637c6d..894fbc6 100644 --- a/internal/retainedprovider/state.go +++ b/internal/retainedprovider/state.go @@ -1,7 +1,9 @@ package retainedprovider import ( + "encoding/json" "fmt" + "io" "path/filepath" "regexp" "strings" @@ -16,7 +18,7 @@ const ( var ( digestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) - imageRefPattern = regexp.MustCompile(`^localhost/[a-z0-9]+(?:[._/-][a-z0-9]+)*:sha256-[0-9a-f]{12,64}$`) + imageRefPattern = regexp.MustCompile(`^localhost/[a-z0-9]+(?:[._/-][a-z0-9]+)*:sha256-[0-9a-f]{64}$`) ) type VerifiedUpdate struct { @@ -75,7 +77,7 @@ func (selection ImageSelection) Validate() error { return fmt.Errorf("image_id must be an immutable SHA-256 digest") } digest := strings.TrimPrefix(selection.Update.SHA256, "sha256:") - if !imageRefPattern.MatchString(selection.ImageRef) || !strings.HasSuffix(selection.ImageRef, ":sha256-"+digest[:12]) { + if !imageRefPattern.MatchString(selection.ImageRef) || !strings.HasSuffix(selection.ImageRef, ":sha256-"+digest) { return fmt.Errorf("image_ref must be a safe localhost reference derived from the update digest") } if selection.ActivatedAt.IsZero() { @@ -193,3 +195,9 @@ type Status struct { CurrentSHA256 string `json:"current_sha256,omitempty"` ObservedAt time.Time `json:"observed_at,omitempty"` } + +func WriteStatus(writer io.Writer, status Status) error { + encoder := json.NewEncoder(writer) + encoder.SetIndent("", " ") + return encoder.Encode(status) +} diff --git a/internal/retainedprovider/state_test.go b/internal/retainedprovider/state_test.go index bc25b85..75e493e 100644 --- a/internal/retainedprovider/state_test.go +++ b/internal/retainedprovider/state_test.go @@ -53,6 +53,7 @@ func TestConfigRejectsUnsafeIdentityAndPaths(t *testing.T) { {name: "relative install root", mutate: func(c *Config) { c.InstallRoot = "relative" }, want: "install_root"}, {name: "outside home", mutate: func(c *Config) { c.SystemdDir = filepath.Join(filepath.Dir(home), "outside") }, want: "systemd_dir"}, {name: "plaintext provider URL", mutate: func(c *Config) { c.ProviderURL = "http://provider:18090" }, want: "provider_url"}, + {name: "wrong provider port", mutate: func(c *Config) { c.ProviderURL = "https://" + c.StableContainer + ":18091" }, want: "provider_url"}, {name: "wrong network", mutate: func(c *Config) { c.ContainerNetwork = "host" }, want: "container_network"}, {name: "short ref", mutate: func(c *Config) { c.Ref = "main" }, want: "ref"}, {name: "fast timer", mutate: func(c *Config) { c.RefreshIntervalSeconds = 10 }, want: "refresh_interval_seconds"}, @@ -107,7 +108,7 @@ func TestActiveStateRetainsDistinctCurrentAndPriorImages(t *testing.T) { current := validTestSelection(now) previous := validTestSelection(now.Add(-time.Hour)) previous.ImageID = "sha256:" + strings.Repeat("c", 64) - previous.ImageRef = "localhost/workflow-plugin-github-runner-provider:sha256-dddddddddddd" + previous.ImageRef = "localhost/workflow-plugin-github-runner-provider:sha256-" + strings.Repeat("d", 64) previous.Update.DirectiveID = "directive-prior" previous.Update.SHA256 = "sha256:" + strings.Repeat("d", 64) state := ActiveState{ @@ -129,7 +130,7 @@ func TestRecoverySelectionForEveryJournalPhase(t *testing.T) { previous := ActiveState{ProtocolVersion: ActiveStateProtocolVersion, Current: validTestSelection(now.Add(-time.Hour)), UpdatedAt: now.Add(-time.Hour)} candidate := validTestSelection(now) candidate.ImageID = "sha256:" + strings.Repeat("c", 64) - candidate.ImageRef = "localhost/workflow-plugin-github-runner-provider:sha256-dddddddddddd" + candidate.ImageRef = "localhost/workflow-plugin-github-runner-provider:sha256-" + strings.Repeat("d", 64) candidate.Update.SHA256 = "sha256:" + strings.Repeat("d", 64) candidate.Update.DirectiveID = "directive-new" @@ -248,7 +249,7 @@ func validTestSelection(now time.Time) ImageSelection { SHA256: "sha256:" + strings.Repeat("a", 64), }, ImageID: "sha256:" + strings.Repeat("b", 64), - ImageRef: "localhost/workflow-plugin-github-runner-provider:sha256-aaaaaaaaaaaa", + ImageRef: "localhost/workflow-plugin-github-runner-provider:sha256-" + strings.Repeat("a", 64), ActivatedAt: now, } } From f31ef1c3878fe38f63deabd45a283af4390fe3cf Mon Sep 17 00:00:00 2001 From: Jon Langevin Date: Tue, 14 Jul 2026 06:41:59 -0400 Subject: [PATCH 09/16] docs: redesign retained lifecycle recovery --- ...tained-runner-provider-lifecycle-design.md | 438 +++++++++++++++++- 1 file changed, 421 insertions(+), 17 deletions(-) diff --git a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md index 25dbd5a..a56c11b 100644 --- a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md +++ b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md @@ -71,17 +71,26 @@ user-owned OS lock and uses `compute-agent supervisor-update verify` to cryptographically bind worker, directive, artifact, path, and digest before copying bytes. -Refresh writes a crash-durable transaction journal before mutation, builds a -digest-unique scratch image, starts a candidate with cloned provider state, and -runs authenticated readiness plus GitHub preflight from a separate probe -container on the same `--network bridge` used by provider workloads. This proves +Refresh builds a digest-unique scratch image, writes a crash-durable transaction +journal, stops the stable provider to quiesce ownership writes, clones that +authoritative state, starts a candidate, and runs authenticated readiness plus +GitHub preflight from a separate probe container on the DNS-enabled +`wfcompute-github-provider` rootless network. The +installer makes that network the agent's default through a private +`containers.conf`, so generic workload `--network bridge` requests join the same +network without adding GitHub logic to workflow-compute. This proves container-name DNS and TLS from the workload side of the boundary, not merely -from inside the provider container. Only then does refresh fsync and atomically -replace durable active state, restart the stable provider, verify it again from -the separate probe container, mark the journal committed, and retain the prior -active image/state as rollback material. Candidate or stable activation failure -restores the prior active record and service. Startup recovery finishes or -rolls back an interrupted journal before any new refresh. +from inside the provider container. Only then does refresh stop the candidate +and stable service, journal the state-promoting boundary, rename the original +provider state to a rollback directory, rename the candidate state into the +stable mount path, and fsync both parents. It then replaces durable active image +state, restarts the stable provider, verifies it from the separate probe +container, and commits. The old state directory is removed only after the +stable probe commits; the previous immutable image remains in durable active +metadata. Candidate or stable activation failure restores the prior state +directory, active record, and service. Startup recovery deterministically +handles prepared, state-promoting, state-promoted, activated, and committed +journals before any new refresh. The agent receives only provider URL, provider API token, and CA certificate. The GitHub token remains in the provider container environment and is never @@ -102,8 +111,9 @@ forwarded to the ephemeral runner-job container. ## Security Review - Install paths must be absolute, under the invoking user's home, owned by that - user, and symlink-free. Generated files use atomic replacement and restrictive - modes. + user, and symlink-free. The purgeable root is fixed to the dedicated + `~/.workflow-compute/github-runner-provider` subtree. Generated files use + atomic replacement and restrictive modes. - The executing installer binary must hash to the verified promoted artifact; direct or stale release binaries cannot establish an unrelated package. - Package verification is delegated to the compute-agent cryptographic reader; @@ -112,9 +122,13 @@ forwarded to the ephemeral runner-job container. Commands and evidence never include credential values. - Candidate provider state is a regular-file-only clone. Candidate failure or interrupted activation preserves prior active state and service. -- Refresh uses a single-owner lock and crash-durable prepare/activate/commit - journal. The current and immediately previous immutable image IDs are retained; - cleanup never removes rollback material referenced by active recovery state. +- Install, refresh, and uninstall use one lifecycle-wide OS lock held from + before maintenance mutation through maintenance release. The lock inode is a + sibling of the purgeable install root so explicit purge cannot unlink it + while held. Refresh uses a + crash-durable prepare/state-promote/activate/commit journal. The current and + immediately previous immutable image IDs are retained; uncommitted provider + state and managed-file backups are never deleted after a failed restore. - Rootless Podman runs with a read-only root, dropped capabilities, no-new- privileges, explicit state/TLS mounts, and no socket mount. Ephemeral workload containers receive only the provider API credential. @@ -125,7 +139,8 @@ forwarded to the ephemeral runner-job container. ## Infrastructure Impact - Creates user-owned files below `~/.workflow-compute/github-runner-provider` - and user-systemd units/drop-ins below `~/.config/systemd/user`. + plus a sibling lifecycle lock, and user-systemd units/drop-ins below + `~/.config/systemd/user`. - Builds local rootless Podman images and runs one provider container. - Adds no cloud resources, database migrations, public ports, or production deployment. The first runtime proof is STG only. @@ -162,7 +177,7 @@ forwarded to the ephemeral runner-job container. | id | assumption | failure response | |---|---|---| | A1 | Retained Linux runs user systemd with lingering and rootless Podman. | Install preflight fails before mutation and emits a redacted diagnostic. | -| A2 | Podman `--network bridge` provides name resolution between provider and workload-shaped containers. | Every candidate/stable activation runs the real probe from a separate bridge container and fails closed before rollout if name resolution or TLS fails. | +| A2 | A dedicated rootless bridge with DNS can be the provider network and the agent's default generic bridge. | Install creates and validates `wfcompute-github-provider`; candidate/stable probes fail closed on DNS/TLS, and the agent receives only `CONTAINERS_CONF`. | | A3 | Current-package marker replacement is observable by a systemd path unit. | A bounded boot/periodic timer reconciles current versus active digest even when path observation is missed. | | A4 | Existing provider state consists only of regular files/directories. | Refresh rejects unsupported entries and preserves the active service. | | A5 | Provider launcher schema remains backward-compatible across plugin updates. | Version the lifecycle config/state and fail closed before activation. | @@ -189,3 +204,392 @@ forwarded to the ephemeral runner-job container. preserving worker identity and, by default, provider state. - The dogfood workflow can return to the existing non-provider runner labels; no production deployment is part of this design. + +### Backport 2026-07-13: Canonical Podman Image IDs + +Cause: Podman 5.8 `.Id` returned 64 lowercase hex characters, while durable +state validation required `sha256:`. +Change: accept only exact bare or `sha256:` SHA-256 forms at the Podman boundary; +store and compare the canonical prefixed digest in refresh and `serve-active`. +Scope: no manifest change. +Evidence: rootless Podman runtime install rejected +`d30ca04b79ef9de02c7dffd5f953561b5c437b6314c38810d6e05b9a3f581bf1` +before candidate activation; the stable service later rejected the equivalent +bare ID until the same canonicalizer guarded `serve-active`. + +### Backport 2026-07-13: DNS-Enabled Provider Network + +Cause: rootless Podman's default `bridge` had `dns_enabled=false`; a separate +workload-shaped probe could not resolve the candidate container. +Change: create and validate `wfcompute-github-provider` as a non-internal DNS +bridge; provider/probe containers join it explicitly; agent-local +`containers.conf` maps generic bridge workloads to it. No ports or sockets. +Scope: no manifest change. +Evidence: default bridge → `no such host`; named bridge → candidate DNS, TLS, +semantic GitHub preflight, stable activation, and marker refresh passed. + +### Backport 2026-07-13: Scratch Image Trust Roots + +Cause: the scratch provider image had no OS CA pool, so GitHub API preflight +failed with `x509: certificate signed by unknown authority`. +Change: the provider command imports Go's maintained fallback X.509 roots; +system roots remain preferred when available. +Scope: no manifest change. +Evidence: forced-empty-root subprocess failed before import and passed after; +real candidate preflight against `api.github.com` passed from scratch. + +### Backport 2026-07-13: Rootless Podman User Units + +Cause: `PrivateTmp=true` and, before Podman's pause process existed, +`NoNewPrivileges=true` blocked `newuidmap`; stable activation exited 125. +Change: omit those directives from user services that launch rootless Podman. +Provider and probe containers remain read-only, capability-dropped, and +`no-new-privileges`. +Scope: no manifest change. +Evidence: isolated user-systemd runtime failed namespace setup with each +directive and completed install after their removal. + +### Backport 2026-07-13: Path Unit Value Escaping + +Cause: `PathChanged="/absolute/path"` preserved the quote as path data; systemd +rejected it as non-absolute and refused the watch unit. +Change: render path-unit values without generic `ExecStart` quoting; encode +unsafe bytes with `\\xNN` and double `%` specifiers. +Scope: no manifest change. +Evidence: systemd journal reported `Path unit lacks path setting`; corrected +unit became active and marker creation invoked the real refresh oneshot. + +### Backport 2026-07-14: Lifecycle Transaction Hardening + +Cause: adversarial review found that install/uninstall acquired the mutation +lock after entering maintenance, candidate state was probed but never promoted, +failed managed-file restores deleted their backups, and same-digest refresh +reported success without checking the stable service. +Change: hold one lock across the complete maintenance transaction; promote and +recover provider state through explicit journal phases; preserve backups until +all restores succeed and propagate cleanup errors; inspect and probe the stable +provider on idempotent timer/path refreshes. A failed state restore leaves the +provider stopped and the journal intact. +Scope: no manifest change. +Evidence: contention performs no maintenance/agent mutation; simulated crashes +at every journal phase recover the expected state generation; stable-probe and +restore failures recover or fail closed; inactive same-digest service is +rejected. + +### Backport 2026-07-14: Observable Systemd Activation + +Cause: combined path/timer activation could partially mutate one unit before +returning an error, while the rollback model tracked only the combined call. +Change: activate path and timer independently; on failure inspect exact +`ActiveState` and `UnitFileState`, roll back observed mutations, and remain +conservative when state cannot be inspected. Bound refresh startup to 15 +minutes so candidate build/probe/promotion is not killed by systemd's default. +Scope: no manifest change. +Evidence: stateful partial-enable tests leave no enabled watch unit; confirmed +pre-mutation failures do not issue spurious disable operations; timeout unit +rendering is regression-tested. + +### Backport 2026-07-14: Cross-Transaction Rollback + +Cause: a provider refresh could commit migrated state before later installer +steps restarted and re-observed the retained agent; outer rollback then restored +old image metadata without restoring its matching provider state. Review also +found that cloning before provider quiescence could lose a final ownership +journal write. +Change: stop the stable provider before the authoritative clone; retain a +deferred committed refresh journal during install; let the outer failure path +restore image metadata plus provider state; finalize the state rollback target +only after provider, watch units, and fenced agent restart succeed. Journal +phase variables advance only after the corresponding durable write. +Scope: no manifest change. +Evidence: a quiesce-time write appears in candidate state; failed commit-journal +writes roll back the last durable phase; a post-refresh agent restart failure +restores the prior provider-state generation. + +### Backport 2026-07-14: Exact Host-State Restoration + +Cause: rollback inferred prior service activation from unit-file existence and +purge unlinked the held lifecycle lock inode. +Change: snapshot `UnitFileState` and `ActiveState` for each pre-existing managed +unit and restore enablement/activity independently; place the lock outside the +purged tree. Recovery stops candidate and stable containers before replacing +mounted state directories. +Scope: no manifest change. +Evidence: disabled units remain disabled after failed reinstall; active units +return active; a contender cannot acquire a replacement lock while purge holds +the original; permission-gated recovery proves process stop precedes filesystem +restore. + +### Backport 2026-07-14: Commit Recovery And Durable Cleanup + +Cause: adversarial review found that a process exit after a deferred provider +commit required manual journal repair, post-commit backup cleanup errors skipped +maintenance release, runtime-only systemd enablement became persistent, and +nested cloned state directories were not individually fsynced. +Change: a repeated install finalizes a previously probed deferred commit before +replaying the idempotent outer transaction; committed install/uninstall cleanup +always attempts maintenance release; runtime enablement is restored with +`enable --runtime`, while linked/transient activity states that cannot be +reconstructed are rejected before managed-unit mutation; cloned directories are +synced bottom-up through the first existing parent. Purge accepts only the +dedicated provider root. +Scope: no manifest change. +Evidence: process-restart fixtures consume the committed journal and rollback +directory; permission-gated backup cleanup still emits maintenance end; exact +systemd-state and bottom-up sync tests fail under fix reversion and pass after +restoration; shared/custom purge roots are rejected. + +### Backport 2026-07-14: Unified Identity-Bound Lifecycle Transaction + +Cause: the final code-review loop showed that separate installer and provider +journals still left autonomous-refresh fences unrecoverable, accepted only one +nested provider phase, and recovered using retry-time worker identity. +Change: replace the install-only outer record with one sibling lifecycle +operation journal shared by install, uninstall, autonomous refresh, and a +constrained legacy `refresh_recovery`. The journal separates immutable identity +(`worker_id`, `profile_id`, plugin/component, transaction id) from recovery +transport. Recovery transport records the strict non-secret config plus the +compute-agent executable digest, supervisor-config digest, and agent unit's +loaded fragment path/digest, ordered `DropInPaths` plus digests, effective +`ExecStart`, and relevant environment-file paths/digests. Before stop/start or +maintenance commands, recovery re-attests those regular files and verifies +sanitized local status still names the recorded worker. After this lifecycle +changes the managed drop-in, it captures the expected effective unit signature +after daemon-reload and before start; `ready` recovery accepts only that +journaled post-change signature. Before the first unit-file mutation, the +journal also stores deterministic intended bytes/digests for every managed unit +and drop-in (these contain paths only, no secret values). During `fenced`, each +managed path may independently match its recorded pre-state (including absence) +or intended state; this permits every partial sequential-write vector while +rejecting any third value. Loaded effective state may match only the complete +pre/intended signature because daemon-reload occurs after all writes. Rollback +restores every path to pre-state, runs daemon-reload, and requires the complete +normalized pre-signature. `ready`/`releasing` commit recovery requires every +path to match intended state and accepts loaded pre/intended signatures, +reloading only the exact pre-loaded case before requiring intended. This covers +crashes during sequential writes/restores, before/after daemon-reload, and +post-signature persistence. Retry-time config never becomes recovery authority. + +The state machine has seven durable phases: + +- `intent`: a canonical owner-only sibling transaction directory, bound to the + transaction id, is fsynced before any copies or maintenance. No mutation has + started; recovery removes that directory. +- `adopting`: only `refresh_recovery` uses this phase. It durably binds exactly + one legacy inner transaction's hash, candidate digest, and verified identity + before maintenance; recovery may establish/drain the exact fence but cannot + fabricate or restore a managed-file/systemd baseline. +- `fencing`: exact maintenance begin may be in flight. Recovery idempotently + establishes and drains that exact fence, performs no provider/managed-file + mutation, and durably writes `ready{outcome:rollback}` before maintenance end; + release and transaction-directory cleanup use the forward-only terminal path. + Once begin is durable and + sanitized status is lease/task-free, install/uninstall copy snapshots into + the sibling directory, append each completed `0600` snapshot atomically, + capture recovery attestations/unit state, and re-read them immediately before + advancing; a crash during copying remains `fencing` and deletes all copies. +- `fenced`: exact maintenance is durable and sanitized status is unavailable + with no task/lease. Only this phase permits stopping the recorded agent or + mutating provider/files/systemd. Recovery re-establishes the exact fence, + conservatively stops the re-attested agent, rolls back provider/files/units, + starts and observes that agent, confirms the inner journal is durably absent, + and writes `ready{outcome:rollback}` before any maintenance end. All release + then proceeds through forward-only `ready`/`releasing`. +- `ready`: a typed terminal outcome (`commit` or `rollback`) is durable after + provider/files/units restoration or activation and agent observation while + fenced. Commit-ready with `provider_effect:changed` requires a matching + deferred committed inner journal; commit-ready `unchanged` requires an absent + inner plus bound unchanged provenance; uninstall commit-ready requires + `not_applicable` and an absent inner. Rollback-ready requires the inner + transaction durably absent after rollback. + Recovery inspects maintenance: exact active fences are ended; inactive state + advances forward; conflicting identity fails closed. It never re-fences, + stops the agent, or mutates provider state. +- `releasing`: the same typed terminal outcome is retained while exact + maintenance end may have succeeded. Recovery applies the + same forward-only rule as `ready` using two sources: maintenance status + classifies exact-active, inactive, or conflicting identity; only for + exact-active does sanitized local status classify active task/lease versus + drained. Exact-active plus task/lease waits boundedly and retains `releasing` + on timeout; exact-active plus drained ends the exact fence; inactive advances + without stopping or re-fencing; conflicting identity fails closed. It then + advances to `committed`. A workload accepted after a successful end is never + interrupted by recovery. +- `committed`: maintenance is released. Recovery only finalizes deferred + provider state for the commit outcome, requested purge/preservation, + snapshots, and journal. Rollback outcome performs no provider finalization, + ignores any requested purge, preserves the restored pre-state, and removes + only snapshots/transaction evidence after audit drains. + +Install/uninstall snapshot and attest under the drained `fencing` phase and +immediately advance to `fenced`; refresh operations carry no managed-file +baseline. Uninstall has a typed +payload containing `purge`; committed cleanup either preserves provider state +or durably removes the dedicated root and fsyncs its parent before removing the +sibling journal. Transient/linked or otherwise unreconstructable systemd state +is rejected before `fencing`. + +The provider subtransaction remains a second file because provider rollback +already has a five-phase durable protocol, but it is no longer an independent +authority. New inner records contain outer transaction id, profile id, and +candidate digest. The outer record also has typed `provider_effect`: +`changed|unchanged|not_applicable`. `changed` is phase-relative: inner may be +absent in `intent`, `fencing`, and pre-provider `fenced`; once provider mutation +starts it requires the matching inner, and commit-ready requires that inner +deferred committed. `unchanged` requires install/refresh, a successful stable-provider +probe, an absent inner journal, and full agreement among outer identity, active +state, and verified candidate for worker id, plugin, component, component id, +and digest; it journals active plus candidate provenance so a new signed +directive reusing identical bytes remains valid. `not_applicable` is required +only for uninstall; uninstall rejects changed/unchanged, while install/refresh +reject not-applicable. This represents same-digest credential rotation without +fabricating a provider transaction. The accepted matrix is closed: + +| outer | install/refresh inner | uninstall inner | recovery | +|---|---|---|---| +| `intent`,`fencing` | absent only | absent only | abort before mutation | +| `adopting` | exact hash-bound legacy inner only for `refresh_recovery` | forbidden | establish/drain fence, then advance | +| `fenced` | absent or matching deferred `prepared`/`state_promoting`/`state_promoted`/`activated`/`committed` | absent | roll back | +| `ready`,`releasing` commit+changed | matching deferred `committed` | forbidden | finish forward without re-fence or mutation | +| `ready`,`releasing` commit+unchanged | absent; verified/active/probed digest bound by outer | forbidden | finish forward without re-fence or mutation | +| `ready`,`releasing` commit+not_applicable | forbidden | absent; uninstall only | finish forward without re-fence or mutation | +| `ready`,`releasing` rollback | absent after durable rollback | absent | release forward without re-fence or mutation | +| `committed` commit+changed | matching deferred `committed` or absent after finalized cleanup | forbidden | finalize provider and apply requested preserve/purge | +| `committed` commit+unchanged | absent; bound unchanged evidence | forbidden | apply requested preserve/purge without provider finalization | +| `committed` commit+not_applicable | forbidden | absent; uninstall only | apply requested preserve/purge without provider finalization | +| `committed` rollback | absent | absent | preserve restored root; clean transaction evidence only | + +Every other combination, transaction/profile/digest mismatch, or non-deferred +inner record fails closed without touching agent/provider state. Every entry +point recovers the outer journal before update-marker or same-digest decisions. +An orphan legacy inner journal may become only `refresh_recovery`: it fabricates +no file/unit baseline and mutates only the exact agent fence plus provider +transaction. Automatic adoption requires the previously installed strict +config and exact inner candidate worker/plugin/component agreement; new inner +journals also require profile agreement. Missing/invalid installed config or a +legacy identity mismatch requires `retained recover -config -confirm +`, which prints redacted identity, requires exact explicit +confirmation, and applies the same constrained recovery. It never accepts +credentials or releases an unrelated fence. + +Security: journals and audit records are strict, owner-only, contain no +credential values, accept only canonical managed paths/unit names, and bind all +nested records. Transaction snapshot directories necessarily contain sensitive +rollback bytes from provider/agent environment files and TLS private material; +they are canonical real `0700` owner-only directories containing only bounded +`0600` regular snapshots, are never named or copied into audit output, and are +retained only until terminal cleanup. Deletion is ordinary filesystem cleanup, +not a secure-erasure claim. Lifecycle intent, each phase transition, recovery +disposition, and terminal error class append redacted JSONL at +`${XDG_STATE_HOME:-$HOME/.local/state}/wfctl/plugins/workflow-plugin-github/retained-provider-audit.jsonl`; +records contain transaction/operation/phase and immutable identity only, never +config contents, paths, credentials, TLS material, or payload bytes. Audit is a +strict tagged union. Common fields are `event_id`, global `sequence`, timestamp, +transaction identity, operation, phase, and `kind`; allowed kind-specific +redacted fields are `outcome`, `provider_effect`, `purge`, `disposition`, +`error_class`, `count`, `first_seen`, and `last_seen`. + +Phase/outcome/purge safety events and recovery-disposition/error diagnostics use +two bounded journal lanes merged by global sequence. The safety lane reserves 16 +non-droppable immutable slots; a valid operation can emit at most one event for +each of seven forward phases plus terminal outcome/purge, so malformed capacity +is rejected before mutation and release always has reserved space. The diagnostic +lane has 32 keyed phase/kind/class summaries plus one `other` overflow summary; +matching events coalesce count/first/last timestamps. Once an event receives an +append offset its bytes/digest freeze; later matching diagnostics update/create +a tail summary, never the assigned head. Alternating/unrecognized classes count +into the overflow entry rather than consuming safety capacity. + +Each queued event has a stable id and exact canonical serialized digest. Under +the dedicated audit lock, only the lowest-sequence head may receive a pre-append +file offset and drain. The append is one canonical newline-terminated record and +the audit file is fsynced before that head is removed. Recovery clears the head +when the complete matching record exists at the offset, truncates a matching +partial final record back to the offset and re-appends, appends when the file +still ends exactly at the offset, and fails closed for a shorter file or +unrelated tail bytes. No later event can overwrite or bypass an unresolved one, +and every required event type uses this path. Audit failure never blocks release +of an established fence or rolls back a ready operation; it does block a new +operation and deletion of terminal journal/transaction evidence until both lanes +drain. + +Infrastructure impact: none beyond the existing user-systemd/Podman boundary; +the sibling journal, install lock, audit, and canonical transaction root +`$HOME/.workflow-compute/.workflow-plugin-github-runner-provider-transactions/` +survive provider-root purge. The directory and parent are owner-only and fsynced. +Committed cleanup may accept a missing transaction directory only after its +operation-specific preservation/purge outcome is already durable; root purge +is followed by parent fsync before snapshot-directory and journal removal. +Multi-component proof: operation-by-phase and legal outer/inner matrix tests +cover changed retry identity, partial snapshot cleanup, same-digest refresh +recovery, constrained legacy adoption/repair, purge intent, exact systemd state, +and retained-agent fence release. Maintenance is an explicit prerequisite from +merged workflow-compute commit `5472767de1e4629ab68337cd6dd1ac85f4b7577e` +(`cmd/compute-agent/main.go:454`, +`internal/agent/process_supervisor.go:1004-1147`, SPEC V971/V973). STG must serve +a signed agent bundle containing that commit before provider rollout. Runtime +proof exercises duplicate exact begin, begin/end/begin/end, status across crash, +wrong-ID rejection, end-to-job-assignment recovery, and post-release reconnect +against that real bundle before repeating the Podman matrix from an attributable +commit. + +Load-bearing assumptions: the cited workflow-compute contract guarantees +duplicate exact-id/profile/reason begin is idempotent and rejects conflicts; +maintenance status distinguishes exact active, inactive, and conflicting +identity; systemd start is idempotent; the sibling lock excludes plugin-owned +lifecycle mutation. Self-challenge: partial secret snapshots are owned by a +journaled sibling directory before copy; a crash around begin remains +`fencing`; a crash around end remains forward-only `releasing`; effective unit +attestation covers drop-ins rather than one fragment; persistent audit failure +preserves journal evidence without availability churn; malformed or +unattestable identity fails closed into the explicit recovery command. Scope: +no manifest change; the recovery command is required operational repair for the +locked lifecycle. + +## Task 4 Runtime Launch Transcript + +Environment: privileged Ubuntu 24.04 arm64 container booted with real user +systemd, lingering user manager, rootless Podman, real provider binary, and real +GitHub API access. The compute-agent maintenance/update/status dependency was a +disclosed deterministic CLI-seam substitute; it is not evidence for the later +real-agent/STG campaign gate. Docker Desktop nested storage required Podman +`vfs`; Ubuntu's CNI backend required the real `dnsname` plugin plus `dnsmasq`. + +```text +Build: +$ CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GOWORK=off go build \ + -o /tmp/github-runner-provider-runtime ./cmd/github-runner-provider +exit 0 + +Install: +$ github-runner-provider retained install -config bootstrap-config.json +installed=true service_active=true version=v1.0.32 + +Observe: +provider.service: active/enabled +refresh.path: active/enabled +refresh.timer: active/enabled +retained agent service: active/enabled +container: running, read-only, all capabilities dropped, + no-new-privileges, no published ports, named DNS bridge + +Marker refresh: +$ touch provider.json +refresh.service: Result=success ExecMainStatus=0 +probe container: created, started, exited successfully, removed + +Credential rotation: +$ github-runner-provider retained install -config bootstrap-config.json +provider environment digest changed; active metadata, provider-state sentinel, +worker identity, and retained agent service remained unchanged + +Uninstall and reinstall: +$ github-runner-provider retained uninstall -config bootstrap-config.json +installed=false; provider units/container absent; retained agent active; +provider-state sentinel unchanged +$ github-runner-provider retained install -config bootstrap-config.json +installed=true service_active=true; provider-state sentinel unchanged + +Failure-signature scrape: clean from the first successful install onward +Verdict: PASS for the Task 4 user-systemd/Podman lifecycle boundary +``` From 302028b962f9403d548568fb5c6195604e9c5916 Mon Sep 17 00:00:00 2001 From: Jon Langevin Date: Tue, 14 Jul 2026 09:35:25 -0400 Subject: [PATCH 10/16] feat(provider): install retained service --- cmd/github-runner-provider/main.go | 1 + cmd/github-runner-provider/main_test.go | 42 + cmd/github-runner-provider/retained_stub.go | 43 +- cmd/github-runner-provider/retained_test.go | 160 +- go.mod | 2 + go.sum | 2 + internal/retainedprovider/command.go | 30 +- internal/retainedprovider/config.go | 17 +- internal/retainedprovider/files.go | 52 +- internal/retainedprovider/files_test.go | 51 + internal/retainedprovider/lifecycle.go | 1584 +++++++++++++ internal/retainedprovider/lifecycle_test.go | 1424 ++++++++++++ internal/retainedprovider/refresh.go | 739 ++++++- internal/retainedprovider/refresh_test.go | 704 +++++- internal/retainedprovider/state.go | 31 +- internal/retainedprovider/state_test.go | 49 +- internal/retainedprovider/systemd.go | 2200 +++++++++++++++++++ internal/retainedprovider/systemd_test.go | 1890 ++++++++++++++++ 18 files changed, 8885 insertions(+), 136 deletions(-) create mode 100644 internal/retainedprovider/lifecycle.go create mode 100644 internal/retainedprovider/lifecycle_test.go create mode 100644 internal/retainedprovider/systemd.go create mode 100644 internal/retainedprovider/systemd_test.go diff --git a/cmd/github-runner-provider/main.go b/cmd/github-runner-provider/main.go index d0a117a..1f92fc8 100644 --- a/cmd/github-runner-provider/main.go +++ b/cmd/github-runner-provider/main.go @@ -17,6 +17,7 @@ import ( "time" "github.com/GoCodeAlone/workflow-plugin-github/internal" + _ "golang.org/x/crypto/x509roots/fallback" ) const providerShutdownTimeout = 10 * time.Second diff --git a/cmd/github-runner-provider/main_test.go b/cmd/github-runner-provider/main_test.go index d407a0a..f91e99c 100644 --- a/cmd/github-runner-provider/main_test.go +++ b/cmd/github-runner-provider/main_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "crypto/tls" + "crypto/x509" "encoding/json" "encoding/pem" "errors" @@ -13,6 +14,7 @@ import ( "net/http" "net/http/httptest" "os" + "os/exec" "path/filepath" "reflect" "strings" @@ -23,6 +25,46 @@ import ( "github.com/GoCodeAlone/workflow-plugin-github/internal" ) +func TestProviderBinaryHasFallbackCertificateRoots(t *testing.T) { + const helperEnvironment = "GITHUB_PROVIDER_TEST_FALLBACK_ROOTS" + if os.Getenv(helperEnvironment) == "1" { + pool, err := x509.SystemCertPool() + if err != nil { + panic(err) + } + if len(pool.Subjects()) == 0 { + panic("provider binary has no fallback certificate roots") + } + return + } + + command := exec.Command(os.Args[0], "-test.run=^TestProviderBinaryHasFallbackCertificateRoots$") + command.Env = append(environmentWithout(os.Environ(), helperEnvironment, "GODEBUG", "SSL_CERT_FILE", "SSL_CERT_DIR"), + helperEnvironment+"=1", + "GODEBUG=x509usefallbackroots=1", + "SSL_CERT_FILE=/nonexistent/provider-ca-bundle", + "SSL_CERT_DIR=/nonexistent/provider-ca-directory", + ) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("provider fallback roots subprocess: %v\n%s", err, output) + } +} + +func environmentWithout(environment []string, keys ...string) []string { + blocked := make(map[string]struct{}, len(keys)) + for _, key := range keys { + blocked[key] = struct{}{} + } + filtered := make([]string, 0, len(environment)) + for _, entry := range environment { + key, _, _ := strings.Cut(entry, "=") + if _, exists := blocked[key]; !exists { + filtered = append(filtered, entry) + } + } + return filtered +} + type deadlineShutdowner struct { closed bool } diff --git a/cmd/github-runner-provider/retained_stub.go b/cmd/github-runner-provider/retained_stub.go index 572ba7d..aee5e77 100644 --- a/cmd/github-runner-provider/retained_stub.go +++ b/cmd/github-runner-provider/retained_stub.go @@ -18,20 +18,31 @@ import ( type retainedProviderCommandDependencies struct { GOOS string HomeDir func() (string, error) + LookupEnv func(string) (string, bool) ReadConfig func(string, string) (retainedprovider.Config, error) + Install func(context.Context, string, retainedprovider.Config, retainedprovider.Credentials) (retainedprovider.Status, error) Refresh func(context.Context, retainedprovider.Config) (retainedprovider.Status, error) ServeActive func(context.Context, retainedprovider.Config) error + Status func(context.Context, string, retainedprovider.Config) (retainedprovider.Status, error) + Uninstall func(context.Context, string, retainedprovider.Config, bool) (retainedprovider.Status, error) + Recover func(context.Context, string, retainedprovider.Config, string) (retainedprovider.Status, error) } func runRetainedProviderCommand(ctx context.Context, logger *slog.Logger, args []string, stdout io.Writer) error { runner := retainedprovider.OSCommandRunner{} refresher := retainedprovider.Refresher{Runner: runner, ExecutablePath: os.Executable, Now: func() time.Time { return time.Now().UTC() }} + installer := retainedprovider.Installer{Runner: runner, ExecutablePath: os.Executable, Now: func() time.Time { return time.Now().UTC() }} return runRetainedProviderCommandWithDependencies(ctx, logger, args, stdout, retainedProviderCommandDependencies{ GOOS: runtime.GOOS, HomeDir: os.UserHomeDir, + LookupEnv: os.LookupEnv, ReadConfig: retainedprovider.ReadConfigFile, + Install: installer.Install, Refresh: refresher.Refresh, ServeActive: refresher.ServeActive, + Status: installer.Status, + Uninstall: installer.Uninstall, + Recover: installer.Recover, }) } @@ -43,13 +54,21 @@ func runRetainedProviderCommandWithDependencies(ctx context.Context, _ *slog.Log return errors.New("retained provider subcommand is required") } switch args[0] { - case "refresh", "serve-active": + case "install", "refresh", "serve-active", "status", "uninstall", "recover": default: return fmt.Errorf("unknown retained provider subcommand %q", args[0]) } flags := flag.NewFlagSet("github-runner-provider retained "+args[0], flag.ContinueOnError) flags.SetOutput(io.Discard) configPath := flags.String("config", "", "absolute retained provider config path") + var purge *bool + var confirmation *string + if args[0] == "uninstall" { + purge = flags.Bool("purge", false, "remove retained provider state and credentials") + } + if args[0] == "recover" { + confirmation = flags.String("confirm", "", "exact legacy provider transaction id") + } if err := flags.Parse(args[1:]); err != nil { return err } @@ -59,6 +78,9 @@ func runRetainedProviderCommandWithDependencies(ctx context.Context, _ *slog.Log if strings.TrimSpace(*configPath) == "" { return errors.New("-config is required") } + if args[0] == "recover" && strings.TrimSpace(*confirmation) == "" { + return errors.New("-confirm is required for retained provider recovery") + } home, err := dependencies.HomeDir() if err != nil { return fmt.Errorf("resolve user home: %w", err) @@ -70,7 +92,24 @@ func runRetainedProviderCommandWithDependencies(ctx context.Context, _ *slog.Log if args[0] == "serve-active" { return dependencies.ServeActive(ctx, config) } - status, err := dependencies.Refresh(ctx, config) + var status retainedprovider.Status + switch args[0] { + case "install": + githubToken, githubFound := dependencies.LookupEnv("GITHUB_RUNNER_PROVIDER_GITHUB_TOKEN") + providerToken, providerFound := dependencies.LookupEnv("GITHUB_RUNNER_PROVIDER_TOKEN") + if !githubFound || !providerFound || strings.TrimSpace(githubToken) == "" || strings.TrimSpace(providerToken) == "" { + return errors.New("provider and GitHub credentials are required in the retained install environment") + } + status, err = dependencies.Install(ctx, home, config, retainedprovider.Credentials{GitHubToken: githubToken, ProviderToken: providerToken}) + case "refresh": + status, err = dependencies.Refresh(ctx, config) + case "status": + status, err = dependencies.Status(ctx, home, config) + case "uninstall": + status, err = dependencies.Uninstall(ctx, home, config, *purge) + case "recover": + status, err = dependencies.Recover(ctx, home, config, *confirmation) + } if err != nil { return err } diff --git a/cmd/github-runner-provider/retained_test.go b/cmd/github-runner-provider/retained_test.go index 4cb8e91..56d8241 100644 --- a/cmd/github-runner-provider/retained_test.go +++ b/cmd/github-runner-provider/retained_test.go @@ -49,7 +49,7 @@ func TestRetainedRefreshLoadsStrictConfigAndEmitsTypedStatus(t *testing.T) { if err != nil { t.Fatalf("retained refresh: %v", err) } - if received.WorkerID != config.WorkerID || received.ComponentID != config.ComponentID { + if received.WorkerID != config.WorkerID || received.ProfileID != config.ProfileID || received.ComponentID != config.ComponentID { t.Fatalf("refresh config = %+v", received) } var status retainedprovider.Status @@ -88,6 +88,158 @@ func TestRetainedServeActiveDispatchesWithoutWritingOutput(t *testing.T) { } } +func TestRetainedInstallStatusAndUninstallDispatchTypedLifecycle(t *testing.T) { + home := t.TempDir() + config := retainedCommandTestConfig(home) + wantStatus := retainedprovider.Status{ProtocolVersion: retainedprovider.StatusProtocolVersion, Installed: true, ServiceActive: true} + var installedCredentials retainedprovider.Credentials + var events []string + dependencies := retainedProviderCommandDependencies{ + GOOS: "linux", + HomeDir: func() (string, error) { return home, nil }, + LookupEnv: func(key string) (string, bool) { + values := map[string]string{ + "GITHUB_RUNNER_PROVIDER_GITHUB_TOKEN": "github-secret", + "GITHUB_RUNNER_PROVIDER_TOKEN": "provider-secret", + } + value, found := values[key] + return value, found + }, + ReadConfig: func(path, gotHome string) (retainedprovider.Config, error) { + if path != filepath.Join(home, "bootstrap-config.json") || gotHome != home { + t.Fatalf("ReadConfig path=%q home=%q", path, gotHome) + } + return config, nil + }, + Install: func(_ context.Context, gotHome string, got retainedprovider.Config, credentials retainedprovider.Credentials) (retainedprovider.Status, error) { + if gotHome != home || got.WorkerID != config.WorkerID || got.ProfileID != config.ProfileID { + t.Fatalf("install home=%q config=%+v", gotHome, got) + } + installedCredentials = credentials + events = append(events, "install") + return wantStatus, nil + }, + Status: func(_ context.Context, gotHome string, got retainedprovider.Config) (retainedprovider.Status, error) { + if gotHome != home || got.WorkerID != config.WorkerID || got.ProfileID != config.ProfileID { + t.Fatalf("status home=%q config=%+v", gotHome, got) + } + events = append(events, "status") + return wantStatus, nil + }, + Uninstall: func(_ context.Context, gotHome string, got retainedprovider.Config, purge bool) (retainedprovider.Status, error) { + if gotHome != home || got.WorkerID != config.WorkerID || got.ProfileID != config.ProfileID || !purge { + t.Fatalf("uninstall home=%q config=%+v purge=%v", gotHome, got, purge) + } + events = append(events, "uninstall") + return retainedprovider.Status{ProtocolVersion: retainedprovider.StatusProtocolVersion}, nil + }, + Refresh: func(context.Context, retainedprovider.Config) (retainedprovider.Status, error) { + return retainedprovider.Status{}, nil + }, + ServeActive: func(context.Context, retainedprovider.Config) error { return nil }, + } + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + configPath := filepath.Join(home, "bootstrap-config.json") + for _, args := range [][]string{ + {"install", "-config", configPath}, + {"status", "-config", configPath}, + {"uninstall", "-config", configPath, "-purge"}, + } { + var stdout bytes.Buffer + if err := runRetainedProviderCommandWithDependencies(t.Context(), logger, args, &stdout, dependencies); err != nil { + t.Fatalf("retained %v: %v", args, err) + } + var status retainedprovider.Status + if err := json.Unmarshal(stdout.Bytes(), &status); err != nil || status.ProtocolVersion != retainedprovider.StatusProtocolVersion { + t.Fatalf("decode %v status=%+v err=%v output=%s", args, status, err, stdout.String()) + } + } + if installedCredentials != (retainedprovider.Credentials{GitHubToken: "github-secret", ProviderToken: "provider-secret"}) { + t.Fatalf("install credentials = %+v", installedCredentials) + } + if got, want := strings.Join(events, ","), "install,status,uninstall"; got != want { + t.Fatalf("events = %q want %q", got, want) + } +} + +func TestRetainedRecoverRequiresAndDispatchesExactConfirmation(t *testing.T) { + home := t.TempDir() + config := retainedCommandTestConfig(home) + wantStatus := retainedprovider.Status{ProtocolVersion: retainedprovider.StatusProtocolVersion, ObservedAt: time.Unix(1_700_000_000, 0).UTC()} + called := false + dependencies := retainedProviderCommandDependencies{ + GOOS: "linux", + HomeDir: func() (string, error) { return home, nil }, + ReadConfig: func(path, gotHome string) (retainedprovider.Config, error) { + if path != filepath.Join(home, "trusted-config.json") || gotHome != home { + t.Fatalf("ReadConfig path=%q home=%q", path, gotHome) + } + return config, nil + }, + Recover: func(_ context.Context, gotHome string, got retainedprovider.Config, confirmation string) (retainedprovider.Status, error) { + if gotHome != home || got.WorkerID != config.WorkerID || confirmation != "legacy-provider-transaction" { + t.Fatalf("recover home=%q config=%+v confirmation=%q", gotHome, got, confirmation) + } + called = true + return wantStatus, nil + }, + } + var stdout bytes.Buffer + if err := runRetainedProviderCommandWithDependencies(t.Context(), slog.New(slog.NewTextHandler(io.Discard, nil)), []string{ + "recover", "-config", filepath.Join(home, "trusted-config.json"), "-confirm", "legacy-provider-transaction", + }, &stdout, dependencies); err != nil { + t.Fatalf("retained recover: %v", err) + } + if !called { + t.Fatal("recover dependency was not called") + } + var status retainedprovider.Status + if err := json.Unmarshal(stdout.Bytes(), &status); err != nil || status != wantStatus { + t.Fatalf("recover status=%+v err=%v output=%s", status, err, stdout.String()) + } + + called = false + if err := runRetainedProviderCommandWithDependencies(t.Context(), slog.New(slog.NewTextHandler(io.Discard, nil)), []string{ + "recover", "-config", filepath.Join(home, "trusted-config.json"), + }, io.Discard, dependencies); err == nil || !strings.Contains(err.Error(), "-confirm") { + t.Fatalf("missing confirmation error = %v", err) + } + if called { + t.Fatal("recover called without exact confirmation") + } +} + +func TestRetainedInstallRejectsCredentialFlagsAndMissingEnvironment(t *testing.T) { + home := t.TempDir() + config := retainedCommandTestConfig(home) + base := retainedProviderCommandDependencies{ + GOOS: "linux", + HomeDir: func() (string, error) { return home, nil }, + LookupEnv: func(string) (string, bool) { return "", false }, + ReadConfig: func(string, string) (retainedprovider.Config, error) { return config, nil }, + Install: func(context.Context, string, retainedprovider.Config, retainedprovider.Credentials) (retainedprovider.Status, error) { + t.Fatal("install called without credentials") + return retainedprovider.Status{}, nil + }, + Refresh: func(context.Context, retainedprovider.Config) (retainedprovider.Status, error) { + return retainedprovider.Status{}, nil + }, + ServeActive: func(context.Context, retainedprovider.Config) error { return nil }, + } + for _, tc := range []struct { + args []string + want string + }{ + {args: []string{"install", "-config", filepath.Join(home, "config.json")}, want: "environment"}, + {args: []string{"install", "-config", filepath.Join(home, "config.json"), "-token", "secret"}, want: "flag"}, + } { + err := runRetainedProviderCommandWithDependencies(t.Context(), slog.New(slog.NewTextHandler(io.Discard, nil)), tc.args, io.Discard, base) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("args=%v err=%v want %q", tc.args, err, tc.want) + } + } +} + func TestRetainedCommandFailsClosedOnUnsupportedPlatformAndInvalidShape(t *testing.T) { base := retainedProviderCommandDependencies{ GOOS: "linux", @@ -123,13 +275,13 @@ func retainedCommandTestConfig(home string) retainedprovider.Config { root := filepath.Join(home, ".workflow-compute", "github-runner-provider") return retainedprovider.Config{ ProtocolVersion: retainedprovider.ConfigProtocolVersion, - WorkerID: "github-runner-linux-stg", ProfileID: "github-runner-linux-stg", + WorkerID: "github-runner-linux-stg", ProfileID: "github-runner-profile-stg", PluginID: retainedprovider.GitHubPluginID, ComponentID: "github-runner-provider-sidecar", ComputeAgentPath: filepath.Join(home, "compute-agent"), SupervisorConfigPath: filepath.Join(home, "supervisor.pb"), - LocalStatusPath: filepath.Join(home, "status.json"), InstallRoot: root, + LocalStatusPath: filepath.Join(home, "status.json"), ProviderMarkerPath: filepath.Join(home, "updates", "current-provider.json"), InstallRoot: root, SystemdDir: filepath.Join(home, ".config", "systemd", "user"), AgentUnit: "workflow-compute-agent.service", PodmanPath: "/usr/bin/podman", ProviderURL: "https://workflow-plugin-github-runner-provider:18090", - StableContainer: "workflow-plugin-github-runner-provider", CandidateContainer: "workflow-plugin-github-runner-provider-candidate", ContainerNetwork: "bridge", + StableContainer: "workflow-plugin-github-runner-provider", CandidateContainer: "workflow-plugin-github-runner-provider-candidate", ContainerNetwork: "wfcompute-github-provider", Organization: "GoCodeAlone", Repository: "GoCodeAlone/workflow-compute", Workflow: "dogfood-provider-target.yml", Ref: strings.Repeat("a", 40), RunnerName: "wfc-stg-ghp-linux-probe", RunnerGroup: "ephemeral", Labels: []string{"self-hosted", "linux", "wfc-ghp-stg"}, RefreshIntervalSeconds: 300, diff --git a/go.mod b/go.mod index 3f12402..82222c5 100644 --- a/go.mod +++ b/go.mod @@ -5,10 +5,12 @@ go 1.26.4 require ( github.com/GoCodeAlone/workflow v0.64.0 github.com/GoCodeAlone/workflow-plugin-compute-core v0.8.3 + github.com/coreos/go-systemd/v22 v22.7.0 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/go-github/v69 v69.2.0 github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 golang.org/x/crypto v0.51.0 + golang.org/x/crypto/x509roots/fallback v0.0.0-20260712151947-c1a3b97d708a golang.org/x/sys v0.44.0 google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af ) diff --git a/go.sum b/go.sum index fae30d7..8318678 100644 --- a/go.sum +++ b/go.sum @@ -665,6 +665,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/crypto/x509roots/fallback v0.0.0-20260712151947-c1a3b97d708a h1:Xc7UN/F6r6Hfr7Jfl1pe+JDFfHxHrE0U45fZVz28dpo= +golang.org/x/crypto/x509roots/fallback v0.0.0-20260712151947-c1a3b97d708a/go.mod h1:+UoQFNBq2p2wO+Q6ddVtYc25GZ6VNdOMyyrd4nrqrKs= golang.org/x/exp v0.0.0-20260508232706-74f9aab9d74a h1:+3jdDGGB8NGb1Zktc737jlt3/A5f6UlwSzmvqUuufxw= golang.org/x/exp v0.0.0-20260508232706-74f9aab9d74a/go.mod h1:d2fgXJLVs4dYDHUk5lwMIfzRzSrWCfGZb0ZqeLa/Vcw= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= diff --git a/internal/retainedprovider/command.go b/internal/retainedprovider/command.go index 70da48b..f0f0fe8 100644 --- a/internal/retainedprovider/command.go +++ b/internal/retainedprovider/command.go @@ -10,9 +10,16 @@ import ( "os/exec" "path/filepath" "strings" + "time" ) -const defaultCommandOutputBytes = 1 << 20 +const ( + defaultCommandOutputBytes = 1 << 20 + controlCommandTimeout = 30 * time.Second + containerStartTimeout = time.Minute + providerProbeTimeout = 2 * time.Minute + providerBuildTimeout = 10 * time.Minute +) type Command struct { Path string @@ -31,6 +38,27 @@ type OSCommandRunner struct { MaxOutputBytes int } +func runBoundedCommand(ctx context.Context, runner CommandRunner, command Command) ([]byte, error) { + timeout := controlCommandTimeout + if filepath.Base(command.Path) == "podman" && len(command.Args) > 0 { + switch command.Args[0] { + case "build": + timeout = providerBuildTimeout + case "run": + timeout = containerStartTimeout + for _, argument := range command.Args { + if argument == "probe" { + timeout = providerProbeTimeout + break + } + } + } + } + bounded, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + return runner.Run(bounded, command) +} + func (runner OSCommandRunner) Run(ctx context.Context, command Command) ([]byte, error) { if err := validateCommand(command); err != nil { return nil, err diff --git a/internal/retainedprovider/config.go b/internal/retainedprovider/config.go index 8deecdb..54d90f9 100644 --- a/internal/retainedprovider/config.go +++ b/internal/retainedprovider/config.go @@ -12,9 +12,10 @@ import ( ) const ( - ConfigProtocolVersion = "retained-provider.config.v1" - GitHubPluginID = "workflow-plugin-github" - maxConfigBytes = 1 << 20 + ConfigProtocolVersion = "retained-provider.config.v1" + GitHubPluginID = "workflow-plugin-github" + providerContainerNetwork = "wfcompute-github-provider" + maxConfigBytes = 1 << 20 ) var ( @@ -33,6 +34,7 @@ type Config struct { ComputeAgentPath string `json:"compute_agent_path"` SupervisorConfigPath string `json:"supervisor_config_path"` LocalStatusPath string `json:"local_status_path"` + ProviderMarkerPath string `json:"provider_marker_path"` InstallRoot string `json:"install_root"` SystemdDir string `json:"systemd_dir"` AgentUnit string `json:"agent_unit"` @@ -111,8 +113,8 @@ func (config Config) Validate(home string) error { if config.StableContainer == config.CandidateContainer { return fmt.Errorf("candidate_container must differ from stable_container") } - if config.ContainerNetwork != "bridge" { - return fmt.Errorf("container_network must be bridge") + if config.ContainerNetwork != providerContainerNetwork { + return fmt.Errorf("container_network must be %s", providerContainerNetwork) } if !filepath.IsAbs(config.PodmanPath) || containsControl(config.PodmanPath) { return fmt.Errorf("podman_path must be an absolute safe path") @@ -121,6 +123,7 @@ func (config Config) Validate(home string) error { "compute_agent_path": config.ComputeAgentPath, "supervisor_config_path": config.SupervisorConfigPath, "local_status_path": config.LocalStatusPath, + "provider_marker_path": config.ProviderMarkerPath, "install_root": config.InstallRoot, "systemd_dir": config.SystemdDir, } { @@ -128,6 +131,10 @@ func (config Config) Validate(home string) error { return fmt.Errorf("%s: %w", field, err) } } + expectedInstallRoot := filepath.Join(filepath.Clean(home), ".workflow-compute", "github-runner-provider") + if filepath.Clean(config.InstallRoot) != expectedInstallRoot { + return fmt.Errorf("install_root must be the dedicated provider root %s", expectedInstallRoot) + } providerURL, err := url.Parse(config.ProviderURL) if err != nil || providerURL.Scheme != "https" || providerURL.Host == "" || providerURL.User != nil || providerURL.RawQuery != "" || providerURL.Fragment != "" || (providerURL.Path != "" && providerURL.Path != "/") || providerURL.Port() != "18090" { return fmt.Errorf("provider_url must be an HTTPS URL without credentials, query, or fragment") diff --git a/internal/retainedprovider/files.go b/internal/retainedprovider/files.go index fe9dac2..e71557d 100644 --- a/internal/retainedprovider/files.go +++ b/internal/retainedprovider/files.go @@ -159,9 +159,16 @@ func ValidateUserPath(home, path string, requireExisting bool) error { } func CloneRegularTree(source, destination string, limits CloneLimits) (returnErr error) { + return cloneRegularTreeWithSync(source, destination, limits, syncDirectory) +} + +func cloneRegularTreeWithSync(source, destination string, limits CloneLimits, syncDir func(string) error) (returnErr error) { if limits.MaxFiles <= 0 || limits.MaxBytes < 0 { return fmt.Errorf("clone limits must be positive") } + if syncDir == nil { + return fmt.Errorf("clone directory sync is required") + } root, err := os.Lstat(source) if err != nil { return fmt.Errorf("inspect clone source: %w", err) @@ -175,8 +182,30 @@ func CloneRegularTree(source, destination string, limits CloneLimits) (returnErr } return fmt.Errorf("inspect clone destination: %w", err) } - if err := os.MkdirAll(destination, 0o700); err != nil { - return fmt.Errorf("create clone destination: %w", err) + missingDirectories := make([]string, 0, 2) + existingAncestor := destination + for { + info, err := os.Lstat(existingAncestor) + if err == nil { + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("clone destination ancestor must be a regular directory") + } + break + } + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("inspect clone destination ancestor: %w", err) + } + missingDirectories = append(missingDirectories, existingAncestor) + parent := filepath.Dir(existingAncestor) + if parent == existingAncestor { + return fmt.Errorf("clone destination has no existing ancestor") + } + existingAncestor = parent + } + for index := len(missingDirectories) - 1; index >= 0; index-- { + if err := os.Mkdir(missingDirectories[index], 0o700); err != nil { + return fmt.Errorf("create clone destination: %w", err) + } } defer func() { if returnErr != nil { @@ -185,6 +214,7 @@ func CloneRegularTree(source, destination string, limits CloneLimits) (returnErr }() files := 0 var bytesCopied int64 + createdTreeDirectories := make([]string, 0) if err := filepath.WalkDir(source, func(path string, entry fs.DirEntry, walkErr error) error { if walkErr != nil { return walkErr @@ -205,7 +235,11 @@ func CloneRegularTree(source, destination string, limits CloneLimits) (returnErr return fmt.Errorf("clone source entries must be regular files or directories: %s", relative) } if info.IsDir() { - return os.Mkdir(target, 0o700) + if err := os.Mkdir(target, 0o700); err != nil { + return err + } + createdTreeDirectories = append(createdTreeDirectories, target) + return nil } if !info.Mode().IsRegular() { return fmt.Errorf("clone source entries must be regular files or directories: %s", relative) @@ -225,7 +259,17 @@ func CloneRegularTree(source, destination string, limits CloneLimits) (returnErr }); err != nil { return fmt.Errorf("clone regular tree: %w", err) } - return syncDirectory(destination) + for index := len(createdTreeDirectories) - 1; index >= 0; index-- { + if err := syncDir(createdTreeDirectories[index]); err != nil { + return err + } + } + for _, directory := range missingDirectories { + if err := syncDir(directory); err != nil { + return err + } + } + return syncDir(existingAncestor) } func cloneRegularFile(source, destination string, expected os.FileInfo) (returnErr error) { diff --git a/internal/retainedprovider/files_test.go b/internal/retainedprovider/files_test.go index bc847c1..484c88d 100644 --- a/internal/retainedprovider/files_test.go +++ b/internal/retainedprovider/files_test.go @@ -149,6 +149,34 @@ func TestCloneRegularTreeCopiesOnlyBoundedRegularFiles(t *testing.T) { } } +func TestCloneRegularTreeSyncsCreatedDirectoriesBottomUp(t *testing.T) { + source := filepath.Join(t.TempDir(), "source") + if err := os.MkdirAll(filepath.Join(source, "nested", "deep"), 0o700); err != nil { + t.Fatalf("mkdir source: %v", err) + } + if err := os.WriteFile(filepath.Join(source, "nested", "deep", "state.json"), []byte(`{"ok":true}`), 0o600); err != nil { + t.Fatalf("write source: %v", err) + } + destinationParent := t.TempDir() + destination := filepath.Join(destinationParent, "destination") + var synced []string + if err := cloneRegularTreeWithSync(source, destination, CloneLimits{MaxFiles: 10, MaxBytes: 1024}, func(path string) error { + synced = append(synced, filepath.Clean(path)) + return nil + }); err != nil { + t.Fatalf("clone: %v", err) + } + want := []string{ + filepath.Join(destination, "nested", "deep"), + filepath.Join(destination, "nested"), + destination, + destinationParent, + } + if strings.Join(synced, "\n") != strings.Join(want, "\n") { + t.Fatalf("directory sync order = %v want %v", synced, want) + } +} + func TestInstallLockIsExclusive(t *testing.T) { path := filepath.Join(t.TempDir(), "install.lock") first, err := AcquireInstallLock(path) @@ -189,6 +217,29 @@ func TestInstallLockRejectsSymlink(t *testing.T) { } } +func TestLifecycleLockRemainsExclusiveWhileInstallRootIsPurged(t *testing.T) { + home := t.TempDir() + paths := LifecyclePathsFor(validTestConfig(home)) + if err := os.MkdirAll(paths.Root, 0o700); err != nil { + t.Fatalf("mkdir install root: %v", err) + } + lock, err := AcquireInstallLock(paths.InstallLock) + if err != nil { + t.Fatalf("acquire lifecycle lock: %v", err) + } + defer lock.Release() + if err := os.RemoveAll(paths.Root); err != nil { + t.Fatalf("purge install root: %v", err) + } + contender, err := AcquireInstallLock(paths.InstallLock) + if contender != nil { + _ = contender.Release() + } + if !errors.Is(err, ErrInstallLocked) { + t.Fatalf("contender acquired replacement lock inode: %v", err) + } +} + func TestStatusJSONHasStableShape(t *testing.T) { data, err := json.Marshal(Status{ProtocolVersion: StatusProtocolVersion}) if err != nil { diff --git a/internal/retainedprovider/lifecycle.go b/internal/retainedprovider/lifecycle.go new file mode 100644 index 0000000..f013ae8 --- /dev/null +++ b/internal/retainedprovider/lifecycle.go @@ -0,0 +1,1584 @@ +package retainedprovider + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +const ( + LifecycleJournalProtocolVersion = "retained-provider.lifecycle-transaction.v1" + maxLifecycleSafetyEvents = 16 + maxLifecycleDiagnosticEvents = 33 +) + +type LifecycleOperation string + +const ( + LifecycleInstall LifecycleOperation = "install" + LifecycleUninstall LifecycleOperation = "uninstall" + LifecycleRefresh LifecycleOperation = "refresh" + LifecycleRefreshRecovery LifecycleOperation = "refresh_recovery" +) + +type LifecyclePhase string + +const ( + LifecycleIntent LifecyclePhase = "intent" + LifecycleAdopting LifecyclePhase = "adopting" + LifecycleFencing LifecyclePhase = "fencing" + LifecycleFenced LifecyclePhase = "fenced" + LifecycleReady LifecyclePhase = "ready" + LifecycleReleasing LifecyclePhase = "releasing" + LifecycleCommitted LifecyclePhase = "committed" +) + +type LifecycleOutcome string + +const ( + LifecycleCommit LifecycleOutcome = "commit" + LifecycleRollback LifecycleOutcome = "rollback" +) + +type ProviderEffect string + +const ( + ProviderChanged ProviderEffect = "changed" + ProviderUnchanged ProviderEffect = "unchanged" + ProviderNotApplicable ProviderEffect = "not_applicable" +) + +type LifecycleIdentity struct { + WorkerID string `json:"worker_id"` + ProfileID string `json:"profile_id"` + PluginID string `json:"plugin_id"` + ComponentID string `json:"component_id"` +} + +func lifecycleIdentityFor(config Config) LifecycleIdentity { + return LifecycleIdentity{ + WorkerID: config.WorkerID, ProfileID: config.ProfileID, + PluginID: config.PluginID, ComponentID: config.ComponentID, + } +} + +func (identity LifecycleIdentity) Validate() error { + for name, value := range map[string]string{ + "worker_id": identity.WorkerID, "profile_id": identity.ProfileID, + "component_id": identity.ComponentID, + } { + if !safeIdentifierPattern.MatchString(value) { + return fmt.Errorf("lifecycle identity %s is invalid", name) + } + } + if identity.PluginID != GitHubPluginID { + return errors.New("lifecycle identity plugin_id is invalid") + } + return nil +} + +type LifecycleFileAttestation struct { + Path string `json:"path"` + SHA256 string `json:"sha256"` +} + +func (attestation LifecycleFileAttestation) Validate() error { + if !filepath.IsAbs(attestation.Path) || containsControl(attestation.Path) { + return errors.New("lifecycle attestation path is invalid") + } + if !digestPattern.MatchString(attestation.SHA256) { + return errors.New("lifecycle attestation digest is invalid") + } + return nil +} + +type LifecycleRecoveryAuthority struct { + Config Config `json:"config"` + ComputeAgent LifecycleFileAttestation `json:"compute_agent"` + SupervisorConfig LifecycleFileAttestation `json:"supervisor_config"` + AgentUnitBefore LifecycleSystemdSignature `json:"agent_unit_before"` +} + +type LifecycleSystemdSignature struct { + Fragment LifecycleFileAttestation `json:"fragment"` + DropIns []LifecycleFileAttestation `json:"drop_ins,omitempty"` + ExecStart string `json:"exec_start"` + EnvironmentFiles []LifecycleFileAttestation `json:"environment_files,omitempty"` +} + +func (signature LifecycleSystemdSignature) Validate(home string) error { + if err := signature.Fragment.Validate(); err != nil { + return fmt.Errorf("validate lifecycle systemd fragment: %w", err) + } + if strings.TrimSpace(signature.ExecStart) == "" || len(signature.ExecStart) > 16*1024 || containsControl(signature.ExecStart) { + return errors.New("lifecycle systemd ExecStart is invalid") + } + if len(signature.DropIns) > 64 || len(signature.EnvironmentFiles) > 64 { + return errors.New("lifecycle systemd signature has too many inputs") + } + seen := map[string]struct{}{signature.Fragment.Path: {}} + for label, attestations := range map[string][]LifecycleFileAttestation{ + "drop-in": signature.DropIns, "environment file": signature.EnvironmentFiles, + } { + for _, attestation := range attestations { + if err := attestation.Validate(); err != nil { + return fmt.Errorf("validate lifecycle systemd %s: %w", label, err) + } + if _, duplicate := seen[attestation.Path]; duplicate { + return fmt.Errorf("lifecycle systemd signature contains a duplicate %s path", label) + } + seen[attestation.Path] = struct{}{} + } + } + for path := range seen { + if err := ValidateUserPath(home, path, false); err != nil { + return fmt.Errorf("validate lifecycle systemd input path: %w", err) + } + } + return nil +} + +func (signature LifecycleSystemdSignature) Reattest() error { + if err := reattestLifecycleFile("fragment", signature.Fragment); err != nil { + return err + } + for _, attestation := range signature.DropIns { + if err := reattestLifecycleFile("drop-in", attestation); err != nil { + return err + } + } + for _, attestation := range signature.EnvironmentFiles { + if err := reattestLifecycleFile("environment file", attestation); err != nil { + return err + } + } + return nil +} + +func reattestLifecycleFile(label string, attestation LifecycleFileAttestation) error { + info, err := os.Lstat(attestation.Path) + if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("lifecycle systemd %s is not a regular file", label) + } + if err := validateOwner(info); err != nil { + return fmt.Errorf("lifecycle systemd %s ownership: %w", label, err) + } + digest, err := hashRegularFile(attestation.Path, false) + if err != nil { + return fmt.Errorf("re-attest lifecycle systemd %s: %w", label, err) + } + if digest != attestation.SHA256 { + return fmt.Errorf("lifecycle systemd %s attestation mismatch", label) + } + return nil +} + +func (authority LifecycleRecoveryAuthority) Validate(home string, identity LifecycleIdentity) error { + if err := authority.Config.Validate(home); err != nil { + return fmt.Errorf("validate lifecycle recovery config: %w", err) + } + if lifecycleIdentityFor(authority.Config) != identity { + return errors.New("lifecycle recovery config identity mismatch") + } + if err := authority.ComputeAgent.Validate(); err != nil { + return err + } + if authority.ComputeAgent.Path != authority.Config.ComputeAgentPath { + return errors.New("lifecycle compute-agent attestation path mismatch") + } + if err := authority.SupervisorConfig.Validate(); err != nil { + return err + } + if authority.SupervisorConfig.Path != authority.Config.SupervisorConfigPath { + return errors.New("lifecycle supervisor config attestation path mismatch") + } + if err := authority.AgentUnitBefore.Validate(home); err != nil { + return err + } + return nil +} + +type LifecycleProviderTransaction struct { + TransactionID string `json:"transaction_id"` + ProfileID string `json:"profile_id"` + Digest string `json:"digest"` + LegacyJournalSHA256 string `json:"legacy_journal_sha256,omitempty"` +} + +func (binding LifecycleProviderTransaction) Validate(identity LifecycleIdentity) error { + if !safeIdentifierPattern.MatchString(binding.TransactionID) || binding.ProfileID != identity.ProfileID || !digestPattern.MatchString(binding.Digest) { + return errors.New("lifecycle provider transaction binding is invalid") + } + if binding.LegacyJournalSHA256 != "" && !digestPattern.MatchString(binding.LegacyJournalSHA256) { + return errors.New("lifecycle legacy provider journal binding is invalid") + } + return nil +} + +type LifecycleUninstallPayload struct { + Purge bool `json:"purge"` +} + +type LifecycleUnchangedProvenance struct { + Active ImageSelection `json:"active"` + Candidate VerifiedUpdate `json:"candidate"` + StableProbeAt time.Time `json:"stable_probe_at,omitempty"` +} + +type LifecycleManagedFileIntent struct { + Path string `json:"path"` + Present bool `json:"present"` + Mode os.FileMode `json:"mode,omitempty"` + SHA256 string `json:"sha256,omitempty"` + Contents []byte `json:"contents,omitempty"` +} + +type lifecycleWiringExpectation uint8 + +const ( + lifecycleWiringMixed lifecycleWiringExpectation = iota + 1 + lifecycleWiringPre + lifecycleWiringIntended +) + +func (provenance LifecycleUnchangedProvenance) Validate(identity LifecycleIdentity, requireProbe bool) error { + if err := provenance.Active.Validate(); err != nil { + return fmt.Errorf("validate lifecycle unchanged active provenance: %w", err) + } + if err := provenance.Candidate.Validate(); err != nil { + return fmt.Errorf("validate lifecycle unchanged candidate provenance: %w", err) + } + active := provenance.Active.Update + if active.WorkerID != identity.WorkerID || active.PluginID != identity.PluginID || active.ComponentID != identity.ComponentID || + provenance.Candidate.WorkerID != identity.WorkerID || provenance.Candidate.PluginID != identity.PluginID || provenance.Candidate.ComponentID != identity.ComponentID { + return errors.New("lifecycle unchanged provenance identity mismatch") + } + if active.SHA256 != provenance.Candidate.SHA256 { + return errors.New("lifecycle unchanged provenance digest mismatch") + } + if requireProbe && provenance.StableProbeAt.IsZero() { + return errors.New("lifecycle unchanged provenance requires a successful stable probe") + } + return nil +} + +type LifecycleAuditKind string + +const ( + AuditPhase LifecycleAuditKind = "phase" + AuditRecovery LifecycleAuditKind = "recovery" + AuditError LifecycleAuditKind = "error" + AuditOverflow LifecycleAuditKind = "overflow" +) + +type LifecycleAuditEvent struct { + EventID string `json:"event_id"` + Sequence uint64 `json:"sequence"` + Timestamp time.Time `json:"timestamp"` + TransactionID string `json:"transaction_id"` + WorkerID string `json:"worker_id"` + Operation LifecycleOperation `json:"operation"` + Phase LifecyclePhase `json:"phase"` + Kind LifecycleAuditKind `json:"kind"` + Outcome LifecycleOutcome `json:"outcome,omitempty"` + ProviderEffect ProviderEffect `json:"provider_effect,omitempty"` + Purge *bool `json:"purge,omitempty"` + Disposition string `json:"disposition,omitempty"` + ErrorClass string `json:"error_class,omitempty"` + Count uint64 `json:"count,omitempty"` + FirstSeen time.Time `json:"first_seen,omitempty"` + LastSeen time.Time `json:"last_seen,omitempty"` + Digest string `json:"digest,omitempty"` + Offset *int64 `json:"offset,omitempty"` +} + +func (event LifecycleAuditEvent) Validate() error { + if !safeIdentifierPattern.MatchString(event.EventID) || event.Sequence == 0 || event.Timestamp.IsZero() || + !safeIdentifierPattern.MatchString(event.TransactionID) || !safeIdentifierPattern.MatchString(event.WorkerID) { + return errors.New("lifecycle audit event identity is invalid") + } + if !validLifecycleOperation(event.Operation) || !validLifecyclePhase(event.Phase) { + return errors.New("lifecycle audit event operation or phase is invalid") + } + switch event.Kind { + case AuditPhase: + if event.ErrorClass != "" || event.Disposition != "" || event.Count != 0 || !event.FirstSeen.IsZero() || !event.LastSeen.IsZero() { + return errors.New("phase audit event error_class or diagnostic fields are invalid") + } + case AuditRecovery: + if !safeIdentifierPattern.MatchString(event.Disposition) || event.ErrorClass != "" { + return errors.New("recovery audit event disposition is invalid") + } + case AuditError, AuditOverflow: + if !safeIdentifierPattern.MatchString(event.ErrorClass) || event.Count == 0 || event.FirstSeen.IsZero() || event.LastSeen.Before(event.FirstSeen) || event.Outcome != "" || event.ProviderEffect != "" { + return errors.New("error audit event error_class or summary is invalid") + } + default: + return errors.New("lifecycle audit event kind is invalid") + } + if event.Digest != "" && !digestPattern.MatchString(event.Digest) { + return errors.New("lifecycle audit event digest is invalid") + } + if event.Offset != nil && *event.Offset < 0 { + return errors.New("lifecycle audit event offset is invalid") + } + return nil +} + +type LifecycleAuditQueue struct { + NextSequence uint64 `json:"next_sequence"` + Safety []LifecycleAuditEvent `json:"safety,omitempty"` + Diagnostics []LifecycleAuditEvent `json:"diagnostics,omitempty"` +} + +func (queue *LifecycleAuditQueue) EnqueueDiagnostic(event LifecycleAuditEvent) error { + if queue == nil { + return errors.New("lifecycle audit queue is required") + } + if queue.NextSequence == 0 { + queue.NextSequence = 1 + } + for index := range queue.Diagnostics { + current := &queue.Diagnostics[index] + if current.Offset == nil && current.Kind == event.Kind && current.Phase == event.Phase && current.ErrorClass == event.ErrorClass { + current.Count++ + current.LastSeen = event.Timestamp + return current.Validate() + } + } + if len(queue.Diagnostics) >= maxLifecycleDiagnosticEvents-1 { + for index := range queue.Diagnostics { + current := &queue.Diagnostics[index] + if current.Offset == nil && current.Kind == AuditOverflow && current.ErrorClass == "other" { + current.Count++ + current.LastSeen = event.Timestamp + return current.Validate() + } + } + if len(queue.Diagnostics) >= maxLifecycleDiagnosticEvents { + return errors.New("lifecycle audit diagnostic queue is full") + } + event.Kind = AuditOverflow + event.ErrorClass = "other" + } + event.Sequence = queue.NextSequence + event.Count = 1 + event.FirstSeen = event.Timestamp + event.LastSeen = event.Timestamp + event.Outcome = "" + event.ProviderEffect = "" + if err := event.Validate(); err != nil { + return err + } + queue.NextSequence++ + queue.Diagnostics = append(queue.Diagnostics, event) + return nil +} + +func (queue *LifecycleAuditQueue) EnqueueSafety(event LifecycleAuditEvent) error { + if queue == nil { + return errors.New("lifecycle audit queue is required") + } + if queue.NextSequence == 0 { + queue.NextSequence = 1 + } + if len(queue.Safety) >= maxLifecycleSafetyEvents { + return errors.New("lifecycle audit safety queue is full") + } + event.Sequence = queue.NextSequence + if err := event.Validate(); err != nil { + return err + } + queue.NextSequence++ + queue.Safety = append(queue.Safety, event) + return nil +} + +func (queue LifecycleAuditQueue) Validate() error { + if queue.NextSequence == 0 || len(queue.Safety) > maxLifecycleSafetyEvents || len(queue.Diagnostics) > maxLifecycleDiagnosticEvents { + return errors.New("lifecycle audit queue bounds are invalid") + } + seen := map[uint64]struct{}{} + for _, events := range [][]LifecycleAuditEvent{queue.Safety, queue.Diagnostics} { + for _, event := range events { + if err := event.Validate(); err != nil { + return err + } + if _, duplicate := seen[event.Sequence]; duplicate || event.Sequence >= queue.NextSequence { + return errors.New("lifecycle audit queue sequence is invalid") + } + seen[event.Sequence] = struct{}{} + } + } + return nil +} + +func lifecycleAuditPayload(event LifecycleAuditEvent) ([]byte, error) { + event.Digest = "" + event.Offset = nil + payload, err := json.Marshal(event) + if err != nil { + return nil, fmt.Errorf("encode lifecycle audit event: %w", err) + } + return append(payload, '\n'), nil +} + +func digestBytes(data []byte) string { + digest := sha256.Sum256(data) + return "sha256:" + hex.EncodeToString(digest[:]) +} + +type LifecycleJournal struct { + ProtocolVersion string `json:"protocol_version"` + TransactionID string `json:"transaction_id"` + Operation LifecycleOperation `json:"operation"` + Phase LifecyclePhase `json:"phase"` + Outcome LifecycleOutcome `json:"outcome,omitempty"` + ProviderEffect ProviderEffect `json:"provider_effect"` + Identity LifecycleIdentity `json:"identity"` + Recovery LifecycleRecoveryAuthority `json:"recovery"` + ProviderTransaction *LifecycleProviderTransaction `json:"provider_transaction,omitempty"` + Unchanged *LifecycleUnchangedProvenance `json:"unchanged,omitempty"` + Uninstall *LifecycleUninstallPayload `json:"uninstall,omitempty"` + Snapshots []managedFileSnapshot `json:"snapshots,omitempty"` + WiringIntent []LifecycleManagedFileIntent `json:"wiring_intent,omitempty"` + PreviousUnits map[string]systemdUnitState `json:"previous_units,omitempty"` + Activation systemdActivation `json:"activation,omitempty"` + AgentUnitIntended *LifecycleSystemdSignature `json:"agent_unit_intended,omitempty"` + Audit LifecycleAuditQueue `json:"audit"` + StartedAt time.Time `json:"started_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +func (journal LifecycleJournal) Validate(home string, paths LifecyclePaths) error { + if journal.ProtocolVersion != LifecycleJournalProtocolVersion || !safeIdentifierPattern.MatchString(journal.TransactionID) { + return errors.New("lifecycle journal protocol or transaction id is invalid") + } + if !validLifecycleOperation(journal.Operation) || !validLifecyclePhase(journal.Phase) { + return errors.New("lifecycle journal operation or phase is invalid") + } + if err := journal.Identity.Validate(); err != nil { + return err + } + if err := journal.Recovery.Validate(home, journal.Identity); err != nil { + return err + } + if LifecyclePathsFor(journal.Recovery.Config).Root != paths.Root || paths.LifecycleTransactionRoot(journal.TransactionID) == paths.Root { + return errors.New("lifecycle journal path identity mismatch") + } + if err := journal.validateOperationEffect(); err != nil { + return err + } + if err := journal.validateSnapshots(home, paths); err != nil { + return err + } + if journal.ProviderTransaction != nil { + if err := journal.ProviderTransaction.Validate(journal.Identity); err != nil { + return err + } + } + terminal := journal.Phase == LifecycleReady || journal.Phase == LifecycleReleasing || journal.Phase == LifecycleCommitted + if terminal != (journal.Outcome != "") { + return errors.New("lifecycle journal outcome is invalid for phase") + } + if journal.Outcome != "" && journal.Outcome != LifecycleCommit && journal.Outcome != LifecycleRollback { + return errors.New("lifecycle journal outcome is invalid") + } + if terminal && journal.Outcome == LifecycleCommit && journal.ProviderEffect == ProviderChanged && journal.ProviderTransaction == nil { + return errors.New("changed commit lifecycle requires provider transaction binding") + } + if journal.Outcome == LifecycleRollback && journal.ProviderTransaction != nil { + return errors.New("rollback lifecycle provider transaction must be absent") + } + if journal.AgentUnitIntended != nil { + if journal.Operation != LifecycleInstall && journal.Operation != LifecycleUninstall { + return errors.New("refresh lifecycle must not contain an intended agent unit signature") + } + if err := journal.AgentUnitIntended.Validate(home); err != nil { + return err + } + } + if terminal && journal.Outcome == LifecycleCommit && (journal.Operation == LifecycleInstall || journal.Operation == LifecycleUninstall) && journal.AgentUnitIntended == nil { + return errors.New("committed wiring lifecycle requires an intended agent unit signature") + } + if err := journal.Audit.Validate(); err != nil { + return err + } + if len(journal.Audit.Safety)+journal.requiredSafetyReservation() > maxLifecycleSafetyEvents { + return errors.New("lifecycle audit safety queue has insufficient reserved terminal capacity") + } + if journal.StartedAt.IsZero() || journal.UpdatedAt.Before(journal.StartedAt) { + return errors.New("lifecycle journal timestamps are invalid") + } + return nil +} + +func (journal LifecycleJournal) validateSnapshots(home string, paths LifecyclePaths) error { + if journal.Operation != LifecycleInstall && journal.Operation != LifecycleUninstall { + if len(journal.Snapshots) != 0 || len(journal.WiringIntent) != 0 || len(journal.PreviousUnits) != 0 || journal.Activation != (systemdActivation{}) { + return errors.New("refresh lifecycle must not contain managed snapshots or systemd state") + } + return nil + } + allowedPaths := make(map[string]struct{}, len(managedInstallPaths(paths))) + for _, path := range managedInstallPaths(paths) { + allowedPaths[path] = struct{}{} + } + if len(journal.Snapshots) > len(allowedPaths) { + return errors.New("lifecycle snapshots exceed managed paths") + } + transactionRoot := paths.LifecycleTransactionRoot(journal.TransactionID) + snapshotRoot := filepath.Join(transactionRoot, "snapshots") + seenPaths := make(map[string]struct{}, len(journal.Snapshots)) + seenBackups := make(map[string]struct{}, len(journal.Snapshots)) + for _, snapshot := range journal.Snapshots { + if _, allowed := allowedPaths[snapshot.Path]; !allowed { + return errors.New("lifecycle snapshots contain an unmanaged path") + } + if _, duplicate := seenPaths[snapshot.Path]; duplicate { + return errors.New("lifecycle snapshots contain duplicate paths") + } + if _, duplicate := seenBackups[snapshot.Backup]; duplicate { + return errors.New("lifecycle snapshots contain duplicate backups") + } + seenPaths[snapshot.Path] = struct{}{} + seenBackups[snapshot.Backup] = struct{}{} + if filepath.Dir(snapshot.Backup) != snapshotRoot { + return errors.New("lifecycle snapshot must be inside the transaction root") + } + if snapshot.Existed { + if snapshot.Mode != 0o600 && snapshot.Mode != 0o700 { + return errors.New("lifecycle snapshot mode is invalid") + } + if err := ValidateUserPath(home, snapshot.Backup, true); err != nil { + return fmt.Errorf("validate lifecycle snapshot: %w", err) + } + info, err := os.Lstat(snapshot.Backup) + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != snapshot.Mode { + return errors.New("lifecycle snapshot is not an owner-only regular file") + } + if err := validateOwner(info); err != nil { + return fmt.Errorf("validate lifecycle snapshot owner: %w", err) + } + if !digestPattern.MatchString(snapshot.SHA256) { + return errors.New("lifecycle snapshot digest is invalid") + } + digest, err := hashRegularFile(snapshot.Backup, snapshot.Mode&0o100 != 0) + if err != nil || digest != snapshot.SHA256 { + return errors.New("lifecycle snapshot digest mismatch") + } + } else { + if snapshot.Mode != 0 || snapshot.SHA256 != "" { + return errors.New("absent lifecycle snapshot mode or digest is invalid") + } + if _, err := os.Lstat(snapshot.Backup); err == nil || !errors.Is(err, os.ErrNotExist) { + return errors.New("absent lifecycle snapshot backup unexpectedly exists") + } + } + } + if err := journal.validateWiringIntent(paths); err != nil { + return err + } + requiresComplete := journal.Phase == LifecycleFenced || journal.Outcome == LifecycleCommit + if requiresComplete && len(journal.Snapshots) != len(allowedPaths) { + return errors.New("fenced lifecycle requires complete snapshots") + } + if requiresComplete && len(journal.WiringIntent) != len(managedWiringPaths(paths)) { + return errors.New("fenced lifecycle requires complete wiring intent") + } + for unit, state := range journal.PreviousUnits { + if unit != providerServiceUnit && unit != refreshPathUnit && unit != refreshTimerUnit { + return errors.New("lifecycle snapshots contain an unmanaged systemd unit") + } + if err := validateRestorableUnitState(state); err != nil { + return fmt.Errorf("validate lifecycle previous unit %s: %w", unit, err) + } + } + return nil +} + +func (journal LifecycleJournal) validateWiringIntent(paths LifecyclePaths) error { + allowed := make(map[string]struct{}, len(managedWiringPaths(paths))) + for _, path := range managedWiringPaths(paths) { + allowed[path] = struct{}{} + } + seen := make(map[string]struct{}, len(journal.WiringIntent)) + for _, intent := range journal.WiringIntent { + if _, found := allowed[intent.Path]; !found { + return errors.New("lifecycle wiring intent contains an unmanaged path") + } + if _, duplicate := seen[intent.Path]; duplicate { + return errors.New("lifecycle wiring intent contains duplicate paths") + } + seen[intent.Path] = struct{}{} + switch journal.Operation { + case LifecycleInstall: + if !intent.Present || intent.Mode != 0o600 || len(intent.Contents) == 0 || !digestPattern.MatchString(intent.SHA256) || digestBytes(intent.Contents) != intent.SHA256 { + return errors.New("install lifecycle wiring intent digest or contents is invalid") + } + case LifecycleUninstall: + if intent.Present || intent.Mode != 0 || intent.SHA256 != "" || len(intent.Contents) != 0 { + return errors.New("uninstall lifecycle wiring intent must record absence") + } + } + } + return nil +} + +func validateLifecycleWiringVector(journal LifecycleJournal, paths LifecyclePaths, expectation lifecycleWiringExpectation) error { + if journal.Operation != LifecycleInstall && journal.Operation != LifecycleUninstall { + if len(journal.WiringIntent) != 0 { + return errors.New("refresh lifecycle unexpectedly contains wiring intent") + } + return nil + } + if expectation != lifecycleWiringMixed && expectation != lifecycleWiringPre && expectation != lifecycleWiringIntended { + return errors.New("lifecycle wiring expectation is invalid") + } + snapshots := make(map[string]managedFileSnapshot, len(journal.Snapshots)) + for _, snapshot := range journal.Snapshots { + snapshots[snapshot.Path] = snapshot + } + for _, intent := range journal.WiringIntent { + snapshot, found := snapshots[intent.Path] + if !found { + return errors.New("lifecycle wiring path has no pre-state snapshot") + } + matchesPre, err := managedFileMatches(intent.Path, snapshot.Existed, snapshot.Mode, snapshot.SHA256) + if err != nil { + return fmt.Errorf("inspect lifecycle wiring pre-state: %w", err) + } + matchesIntended, err := managedFileMatches(intent.Path, intent.Present, intent.Mode, intent.SHA256) + if err != nil { + return fmt.Errorf("inspect lifecycle intended wiring: %w", err) + } + switch expectation { + case lifecycleWiringMixed: + if !matchesPre && !matchesIntended { + return fmt.Errorf("lifecycle wiring %s matches neither pre-state nor intended state", intent.Path) + } + case lifecycleWiringPre: + if !matchesPre { + return fmt.Errorf("lifecycle wiring %s does not match pre-state", intent.Path) + } + case lifecycleWiringIntended: + if !matchesIntended { + return fmt.Errorf("lifecycle wiring %s does not match intended state", intent.Path) + } + } + } + return nil +} + +func managedFileMatches(path string, present bool, mode os.FileMode, digest string) (bool, error) { + info, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) { + return !present, nil + } + if err != nil { + return false, err + } + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return false, errors.New("managed lifecycle wiring must be a regular file") + } + if err := validateOwner(info); err != nil { + return false, err + } + if !present || info.Mode().Perm() != mode { + return false, nil + } + actual, err := hashRegularFile(path, mode&0o100 != 0) + if err != nil { + return false, err + } + return actual == digest, nil +} + +func writeLifecycleJournal(home string, paths LifecyclePaths, journal LifecycleJournal) error { + if err := validateLifecyclePathBoundary(home, paths); err != nil { + return err + } + if err := journal.Validate(home, paths); err != nil { + return err + } + if journal.Phase == LifecycleIntent { + transactionRoot := paths.LifecycleTransactionRoot(journal.TransactionID) + if err := os.MkdirAll(transactionRoot, 0o700); err != nil { + return fmt.Errorf("create lifecycle transaction root: %w", err) + } + if err := validateOwnedDirectory(transactionRoot); err != nil { + return fmt.Errorf("validate lifecycle transaction root: %w", err) + } + if err := syncDirectory(paths.LifecycleTransactions); err != nil { + return fmt.Errorf("sync lifecycle transactions: %w", err) + } + } + if err := AtomicWriteJSON(paths.LifecycleJournal, journal); err != nil { + return fmt.Errorf("write lifecycle journal: %w", err) + } + return nil +} + +func readLifecycleJournal(home string, paths LifecyclePaths) (LifecycleJournal, bool, error) { + if err := validateLifecyclePathBoundary(home, paths); err != nil { + return LifecycleJournal{}, false, err + } + var journal LifecycleJournal + if err := ReadStrictJSONFile(paths.LifecycleJournal, &journal); err != nil { + if errors.Is(err, os.ErrNotExist) { + return LifecycleJournal{}, false, nil + } + return LifecycleJournal{}, false, err + } + if err := journal.Validate(home, paths); err != nil { + return LifecycleJournal{}, false, err + } + return journal, true, nil +} + +func drainLifecycleAudit(home string, paths LifecyclePaths, journal *LifecycleJournal) (returnErr error) { + if journal == nil { + return errors.New("lifecycle journal is required") + } + if err := validateLifecyclePathBoundary(home, paths); err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(paths.LifecycleAudit), 0o700); err != nil { + return fmt.Errorf("create lifecycle audit directory: %w", err) + } + lock, err := AcquireInstallLock(paths.LifecycleAuditLock) + if err != nil { + return fmt.Errorf("acquire lifecycle audit lock: %w", err) + } + defer func() { returnErr = errors.Join(returnErr, lock.Release()) }() + + for len(journal.Audit.Safety) > 0 || len(journal.Audit.Diagnostics) > 0 { + lane := &journal.Audit.Safety + if len(*lane) == 0 || (len(journal.Audit.Diagnostics) > 0 && journal.Audit.Diagnostics[0].Sequence < (*lane)[0].Sequence) { + lane = &journal.Audit.Diagnostics + } + event := (*lane)[0] + payload, err := lifecycleAuditPayload(event) + if err != nil { + return err + } + file, err := openLifecycleAudit(paths.LifecycleAudit) + if err != nil { + return err + } + if event.Offset == nil { + info, statErr := file.Stat() + if statErr != nil { + _ = file.Close() + return fmt.Errorf("stat lifecycle audit: %w", statErr) + } + offset := info.Size() + event.Offset = &offset + event.Digest = digestBytes(payload) + (*lane)[0] = event + if err := writeLifecycleJournal(home, paths, *journal); err != nil { + _ = file.Close() + return err + } + } + if event.Digest != digestBytes(payload) { + _ = file.Close() + return errors.New("lifecycle audit pending digest mismatch") + } + if _, err := file.Seek(*event.Offset, io.SeekStart); err != nil { + _ = file.Close() + return fmt.Errorf("seek lifecycle audit: %w", err) + } + tail, err := io.ReadAll(io.LimitReader(file, int64(len(payload)+1))) + if err != nil { + _ = file.Close() + return fmt.Errorf("read lifecycle audit tail: %w", err) + } + switch { + case len(tail) == 0: + if err := appendLifecycleAuditAt(file, *event.Offset, payload); err != nil { + _ = file.Close() + return err + } + case len(tail) < len(payload) && bytes.Equal(tail, payload[:len(tail)]): + if err := appendLifecycleAuditAt(file, *event.Offset, payload); err != nil { + _ = file.Close() + return err + } + case len(tail) >= len(payload) && bytes.Equal(tail[:len(payload)], payload): + if err := file.Sync(); err != nil { + _ = file.Close() + return fmt.Errorf("sync completed lifecycle audit: %w", err) + } + default: + _ = file.Close() + return errors.New("lifecycle audit contains unrelated tail bytes") + } + if err := file.Close(); err != nil { + return fmt.Errorf("close lifecycle audit: %w", err) + } + *lane = append((*lane)[:0], (*lane)[1:]...) + if err := writeLifecycleJournal(home, paths, *journal); err != nil { + return err + } + } + return nil +} + +func openLifecycleAudit(path string) (*os.File, error) { + if err := rejectNonRegularDestination(path); err != nil { + return nil, err + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, fmt.Errorf("open lifecycle audit: %w", err) + } + info, err := file.Stat() + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != 0o600 { + _ = file.Close() + return nil, errors.New("lifecycle audit must be an owner-only regular file") + } + if err := validateOwner(info); err != nil { + _ = file.Close() + return nil, fmt.Errorf("lifecycle audit ownership: %w", err) + } + return file, nil +} + +func appendLifecycleAuditAt(file *os.File, offset int64, payload []byte) error { + if err := file.Truncate(offset); err != nil { + return fmt.Errorf("truncate lifecycle audit tail: %w", err) + } + if _, err := file.Seek(offset, io.SeekStart); err != nil { + return fmt.Errorf("seek lifecycle audit append: %w", err) + } + if _, err := file.Write(payload); err != nil { + return fmt.Errorf("append lifecycle audit: %w", err) + } + if err := file.Sync(); err != nil { + return fmt.Errorf("sync lifecycle audit: %w", err) + } + return nil +} + +func (journal LifecycleJournal) validateOperationEffect() error { + switch journal.Operation { + case LifecycleUninstall: + if journal.ProviderEffect != ProviderNotApplicable || journal.Uninstall == nil { + return errors.New("uninstall lifecycle requires not_applicable provider effect and payload") + } + case LifecycleInstall, LifecycleRefresh, LifecycleRefreshRecovery: + if journal.ProviderEffect != ProviderChanged && journal.ProviderEffect != ProviderUnchanged { + return errors.New("install or refresh lifecycle rejects not_applicable provider effect") + } + if journal.Uninstall != nil { + return errors.New("non-uninstall lifecycle contains uninstall payload") + } + default: + return errors.New("lifecycle operation is invalid") + } + if journal.ProviderEffect != ProviderChanged && journal.ProviderTransaction != nil { + return errors.New("unchanged or not_applicable lifecycle must not bind provider transaction") + } + if journal.ProviderEffect == ProviderUnchanged { + if journal.Unchanged == nil { + return errors.New("unchanged lifecycle requires unchanged provenance") + } + requireProbe := journal.Outcome == LifecycleCommit + if err := journal.Unchanged.Validate(journal.Identity, requireProbe); err != nil { + return err + } + } else if journal.Unchanged != nil { + return errors.New("non-unchanged lifecycle must not contain unchanged provenance") + } + return nil +} + +func (journal LifecycleJournal) requiredSafetyReservation() int { + switch journal.Phase { + case LifecycleIntent, LifecycleAdopting: + return 5 + case LifecycleFencing: + return 4 + case LifecycleFenced: + return 3 + case LifecycleReady: + return 2 + case LifecycleReleasing: + return 1 + case LifecycleCommitted: + return 0 + default: + return maxLifecycleSafetyEvents + 1 + } +} + +func validLifecycleOperation(operation LifecycleOperation) bool { + return operation == LifecycleInstall || operation == LifecycleUninstall || operation == LifecycleRefresh || operation == LifecycleRefreshRecovery +} + +func validLifecyclePhase(phase LifecyclePhase) bool { + switch phase { + case LifecycleIntent, LifecycleAdopting, LifecycleFencing, LifecycleFenced, LifecycleReady, LifecycleReleasing, LifecycleCommitted: + return true + default: + return false + } +} + +func lifecycleHome(paths LifecyclePaths) string { + return filepath.Dir(filepath.Dir(paths.Root)) +} + +func validateLifecyclePathBoundary(home string, paths LifecyclePaths) error { + for name, path := range map[string]string{ + "journal": paths.LifecycleJournal, + "transactions": paths.LifecycleTransactions, + "audit": paths.LifecycleAudit, + "audit lock": paths.LifecycleAuditLock, + } { + if err := ValidateUserPath(home, path, false); err != nil { + return fmt.Errorf("lifecycle %s path: %w", strings.TrimSpace(name), err) + } + } + return nil +} + +func (authority LifecycleRecoveryAuthority) Reattest() error { + computeAgentDigest, err := hashRegularFile(authority.ComputeAgent.Path, true) + if err != nil { + return fmt.Errorf("re-attest lifecycle compute-agent: %w", err) + } + if computeAgentDigest != authority.ComputeAgent.SHA256 { + return errors.New("lifecycle compute-agent attestation mismatch") + } + supervisorDigest, err := hashRegularFile(authority.SupervisorConfig.Path, false) + if err != nil { + return fmt.Errorf("re-attest lifecycle supervisor config: %w", err) + } + if supervisorDigest != authority.SupervisorConfig.SHA256 { + return errors.New("lifecycle supervisor config attestation mismatch") + } + return nil +} + +func lifecycleMaintenanceIdentity(operation LifecycleOperation) (id, reason string, err error) { + switch operation { + case LifecycleInstall: + return installMaintenanceID, installMaintenanceReason, nil + case LifecycleUninstall: + return uninstallMaintenanceID, uninstallMaintenanceReason, nil + case LifecycleRefresh, LifecycleRefreshRecovery: + return refreshMaintenanceID, refreshMaintenanceReason, nil + default: + return "", "", errors.New("lifecycle operation has no maintenance identity") + } +} + +func writeLifecycleTransition(home string, paths LifecyclePaths, journal *LifecycleJournal, phase LifecyclePhase, outcome LifecycleOutcome, now time.Time) error { + if journal == nil { + return errors.New("lifecycle journal is required") + } + next := *journal + next.Phase = phase + next.Outcome = outcome + next.UpdatedAt = now.UTC() + event := LifecycleAuditEvent{ + EventID: "event-" + fmt.Sprint(next.Audit.NextSequence), Timestamp: next.UpdatedAt, + TransactionID: next.TransactionID, WorkerID: next.Identity.WorkerID, + Operation: next.Operation, Phase: phase, Kind: AuditPhase, + Outcome: outcome, ProviderEffect: next.ProviderEffect, + } + if next.Uninstall != nil { + purge := next.Uninstall.Purge + event.Purge = &purge + } + if err := next.Audit.EnqueueSafety(event); err != nil { + return fmt.Errorf("enqueue lifecycle phase audit: %w", err) + } + if err := writeLifecycleJournal(home, paths, next); err != nil { + return err + } + *journal = next + return nil +} + +func newLifecycleJournal(config Config, operation LifecycleOperation, effect ProviderEffect, uninstall *LifecycleUninstallPayload, now time.Time) (LifecycleJournal, error) { + home := lifecycleHome(LifecyclePathsFor(config)) + if err := config.Validate(home); err != nil { + return LifecycleJournal{}, err + } + computeAgentDigest, err := hashRegularFile(config.ComputeAgentPath, true) + if err != nil { + return LifecycleJournal{}, fmt.Errorf("attest lifecycle compute-agent: %w", err) + } + supervisorDigest, err := hashRegularFile(config.SupervisorConfigPath, false) + if err != nil { + return LifecycleJournal{}, fmt.Errorf("attest lifecycle supervisor config: %w", err) + } + if now.IsZero() { + now = time.Now().UTC() + } + identity := lifecycleIdentityFor(config) + seed := strings.Join([]string{string(operation), identity.WorkerID, identity.ProfileID, now.UTC().Format(time.RFC3339Nano)}, "\x00") + transactionDigest := sha256.Sum256([]byte(seed)) + return LifecycleJournal{ + ProtocolVersion: LifecycleJournalProtocolVersion, + TransactionID: string(operation) + "-" + hex.EncodeToString(transactionDigest[:8]), + Operation: operation, + Phase: LifecycleIntent, + ProviderEffect: effect, + Identity: identity, + Recovery: LifecycleRecoveryAuthority{ + Config: config, + ComputeAgent: LifecycleFileAttestation{Path: config.ComputeAgentPath, SHA256: computeAgentDigest}, + SupervisorConfig: LifecycleFileAttestation{Path: config.SupervisorConfigPath, SHA256: supervisorDigest}, + }, + Uninstall: uninstall, + Audit: LifecycleAuditQueue{NextSequence: 1}, + StartedAt: now.UTC(), UpdatedAt: now.UTC(), + }, nil +} + +func startLifecycleTransaction(home string, paths LifecyclePaths, journal *LifecycleJournal) error { + if journal == nil { + return errors.New("lifecycle journal is required") + } + event := LifecycleAuditEvent{ + EventID: "event-" + fmt.Sprint(journal.Audit.NextSequence), Timestamp: journal.UpdatedAt, + TransactionID: journal.TransactionID, WorkerID: journal.Identity.WorkerID, + Operation: journal.Operation, Phase: journal.Phase, Kind: AuditPhase, + ProviderEffect: journal.ProviderEffect, + } + if journal.Uninstall != nil { + purge := journal.Uninstall.Purge + event.Purge = &purge + } + if err := journal.Audit.EnqueueSafety(event); err != nil { + return fmt.Errorf("enqueue lifecycle intent audit: %w", err) + } + if err := writeLifecycleJournal(home, paths, *journal); err != nil { + return err + } + if err := drainLifecycleAudit(home, paths, journal); err != nil { + return fmt.Errorf("drain lifecycle intent audit: %w", err) + } + return nil +} + +func (installer Installer) recoverLifecycleTransaction(ctx context.Context, home string, paths LifecyclePaths, refresher Refresher) error { + journal, found, err := readLifecycleJournal(home, paths) + if err != nil { + return fmt.Errorf("read retained provider lifecycle transaction: %w", err) + } + if !found { + if _, legacyOuterFound, legacyErr := readInstallTransactionJournal(paths); legacyErr != nil { + return fmt.Errorf("read legacy retained provider outer transaction: %w", legacyErr) + } else if legacyOuterFound { + return nil + } + return installer.adoptLegacyProviderTransaction(ctx, home, paths, refresher, nil, "") + } + if err := journal.Recovery.Reattest(); err != nil { + return err + } + if err := validateLifecycleProviderMatrix(paths, journal); err != nil { + return err + } + if err := installer.validateLifecycleAgentUnit(ctx, home, journal); err != nil { + return err + } + switch journal.Phase { + case LifecycleIntent: + return finishLifecycleTransaction(home, paths, &journal) + case LifecycleFencing: + return installer.recoverLifecycleFencing(ctx, home, paths, &journal) + case LifecycleFenced: + return installer.recoverLifecycleFenced(ctx, home, paths, &journal, refresher) + case LifecycleReady, LifecycleReleasing: + return installer.recoverLifecycleRelease(ctx, home, paths, &journal, refresher) + case LifecycleCommitted: + return finalizeLifecycleTransaction(home, paths, &journal, refresher) + case LifecycleAdopting: + return installer.recoverLifecycleAdopting(ctx, home, paths, &journal, refresher) + default: + return errors.New("lifecycle recovery phase is invalid") + } +} + +func (installer Installer) adoptLegacyProviderTransaction(ctx context.Context, home string, paths LifecyclePaths, refresher Refresher, trusted *Config, confirmation string) error { + inner, found, err := readTransactionJournal(paths.Journal) + if err != nil { + return fmt.Errorf("read legacy provider transaction: %w", err) + } + if !found { + return nil + } + if inner.OuterTransactionID != "" || inner.ProfileID != "" { + return errors.New("bound provider transaction has no matching outer lifecycle journal") + } + if confirmation != "" && confirmation != inner.ID { + return errors.New("legacy provider transaction confirmation does not match") + } + var config Config + if trusted == nil { + config, err = ReadConfigFile(paths.ConfigFile, home) + if err != nil { + return fmt.Errorf("automatic legacy recovery requires installed config; run retained recover with trusted config and exact transaction confirmation: %w", err) + } + } else { + config = *trusted + if err := config.Validate(home); err != nil { + return fmt.Errorf("validate trusted legacy recovery config: %w", err) + } + } + if LifecyclePathsFor(config).Root != paths.Root { + return errors.New("legacy recovery config does not identify this provider root") + } + update := inner.Candidate.Update + if update.WorkerID != config.WorkerID || update.PluginID != config.PluginID || update.ComponentID != config.ComponentID { + return errors.New("legacy provider transaction identity does not match recovery config") + } + journal, err := newLifecycleJournal(config, LifecycleRefreshRecovery, ProviderChanged, nil, installer.now()) + if err != nil { + return err + } + signature, err := installer.inspectAgentUnitSignature(ctx, home, config) + if err != nil { + return err + } + journal.Recovery.AgentUnitBefore = signature + legacyDigest, err := hashRegularFile(paths.Journal, false) + if err != nil { + return fmt.Errorf("hash legacy provider transaction: %w", err) + } + journal.ProviderTransaction = &LifecycleProviderTransaction{ + TransactionID: inner.ID, ProfileID: config.ProfileID, Digest: update.SHA256, LegacyJournalSHA256: legacyDigest, + } + if err := startLifecycleTransaction(home, paths, &journal); err != nil { + return err + } + if err := writeLifecycleTransition(home, paths, &journal, LifecycleAdopting, "", installer.now()); err != nil { + return err + } + return installer.recoverLifecycleAdopting(ctx, home, paths, &journal, refresher) +} + +func (installer Installer) recoverLifecycleAdopting(ctx context.Context, home string, paths LifecyclePaths, journal *LifecycleJournal, refresher Refresher) error { + if journal.Operation != LifecycleRefreshRecovery || journal.ProviderTransaction == nil || journal.ProviderTransaction.LegacyJournalSHA256 == "" { + return errors.New("adopting lifecycle is not bound to a legacy provider transaction") + } + if err := validateLifecycleProviderMatrix(paths, *journal); err != nil { + return err + } + id, reason, err := lifecycleMaintenanceIdentity(journal.Operation) + if err != nil { + return err + } + if err := installer.beginMaintenance(ctx, journal.Recovery.Config, id, reason); err != nil { + return fmt.Errorf("establish legacy recovery maintenance fence: %w", err) + } + if err := installer.waitLocalState(ctx, journal.Recovery.Config, "unavailable"); err != nil { + return fmt.Errorf("drain legacy recovery maintenance fence: %w", err) + } + if err := writeLifecycleTransition(home, paths, journal, LifecycleFenced, "", installer.now()); err != nil { + return err + } + return installer.recoverLifecycleFenced(ctx, home, paths, journal, refresher) +} + +func (installer Installer) validateLifecycleAgentUnit(ctx context.Context, home string, journal LifecycleJournal) error { + current, err := installer.inspectAgentUnitSignature(ctx, home, journal.Recovery.Config) + if err != nil { + return err + } + before := journal.Recovery.AgentUnitBefore + if journal.Operation == LifecycleRefresh || journal.Operation == LifecycleRefreshRecovery { + if !equalLifecycleSystemdSignature(current, before) { + return errors.New("effective retained agent unit does not match the recorded pre-signature") + } + return nil + } + switch journal.Phase { + case LifecycleFenced: + if equalLifecycleSystemdSignature(current, before) || journal.AgentUnitIntended != nil && equalLifecycleSystemdSignature(current, *journal.AgentUnitIntended) { + return nil + } + return errors.New("effective retained agent unit matches neither pre nor intended signature") + case LifecycleReady, LifecycleReleasing, LifecycleCommitted: + expected := before + if journal.Outcome == LifecycleCommit { + if journal.AgentUnitIntended == nil { + return errors.New("committed lifecycle has no intended agent unit signature") + } + expected = *journal.AgentUnitIntended + } + if !equalLifecycleSystemdSignature(current, expected) { + return errors.New("effective retained agent unit does not match the terminal signature") + } + return nil + default: + if !equalLifecycleSystemdSignature(current, before) { + return errors.New("effective retained agent unit does not match the recorded pre-signature") + } + return nil + } +} + +func equalLifecycleSystemdSignature(left, right LifecycleSystemdSignature) bool { + if left.Fragment != right.Fragment || left.ExecStart != right.ExecStart || len(left.DropIns) != len(right.DropIns) || len(left.EnvironmentFiles) != len(right.EnvironmentFiles) { + return false + } + for index := range left.DropIns { + if left.DropIns[index] != right.DropIns[index] { + return false + } + } + for index := range left.EnvironmentFiles { + if left.EnvironmentFiles[index] != right.EnvironmentFiles[index] { + return false + } + } + return true +} + +func deriveLifecycleAgentUnitSignature(before LifecycleSystemdSignature, paths LifecyclePaths, wiring []LifecycleManagedFileIntent, agentEnvironment *LifecycleFileAttestation) (LifecycleSystemdSignature, error) { + var dropInIntent *LifecycleManagedFileIntent + for index := range wiring { + if wiring[index].Path == paths.AgentDropIn { + dropInIntent = &wiring[index] + break + } + } + if dropInIntent == nil { + return LifecycleSystemdSignature{}, errors.New("lifecycle wiring has no retained agent drop-in intent") + } + if dropInIntent.Present != (agentEnvironment != nil) { + return LifecycleSystemdSignature{}, errors.New("lifecycle agent drop-in and environment intent disagree") + } + if agentEnvironment != nil { + if err := agentEnvironment.Validate(); err != nil || agentEnvironment.Path != paths.AgentEnv { + return LifecycleSystemdSignature{}, errors.New("lifecycle agent environment intent is invalid") + } + } + intended := before + intended.DropIns = replaceLifecycleAttestation(before.DropIns, paths.AgentDropIn, nil) + intended.EnvironmentFiles = replaceLifecycleAttestation(before.EnvironmentFiles, paths.AgentEnv, nil) + if dropInIntent.Present { + dropIn := LifecycleFileAttestation{Path: paths.AgentDropIn, SHA256: dropInIntent.SHA256} + intended.DropIns = append(intended.DropIns, dropIn) + intended.EnvironmentFiles = append(intended.EnvironmentFiles, *agentEnvironment) + } + sort.Slice(intended.DropIns, func(left, right int) bool { return intended.DropIns[left].Path < intended.DropIns[right].Path }) + sort.Slice(intended.EnvironmentFiles, func(left, right int) bool { + return intended.EnvironmentFiles[left].Path < intended.EnvironmentFiles[right].Path + }) + if err := intended.Validate(lifecycleHome(paths)); err != nil { + return LifecycleSystemdSignature{}, err + } + return intended, nil +} + +func replaceLifecycleAttestation(attestations []LifecycleFileAttestation, path string, replacement *LifecycleFileAttestation) []LifecycleFileAttestation { + result := make([]LifecycleFileAttestation, 0, len(attestations)+1) + for _, attestation := range attestations { + if attestation.Path != path { + result = append(result, attestation) + } + } + if replacement != nil { + result = append(result, *replacement) + } + return result +} + +func (installer Installer) recoverLifecycleFencing(ctx context.Context, home string, paths LifecyclePaths, journal *LifecycleJournal) error { + id, reason, err := lifecycleMaintenanceIdentity(journal.Operation) + if err != nil { + return err + } + config := journal.Recovery.Config + if err := installer.beginMaintenance(ctx, config, id, reason); err != nil { + return fmt.Errorf("establish lifecycle maintenance fence: %w", err) + } + if err := installer.waitLocalState(ctx, config, "unavailable"); err != nil { + return fmt.Errorf("drain lifecycle maintenance fence: %w", err) + } + journal.ProviderTransaction = nil + if err := writeLifecycleTransition(home, paths, journal, LifecycleReady, LifecycleRollback, installer.now()); err != nil { + return err + } + return installer.recoverLifecycleRelease(ctx, home, paths, journal, Refresher{Runner: installer.Runner, Now: installer.Now, Sleep: installer.Sleep}) +} + +func (installer Installer) recoverLifecycleFenced(ctx context.Context, home string, paths LifecyclePaths, journal *LifecycleJournal, refresher Refresher) error { + id, reason, err := lifecycleMaintenanceIdentity(journal.Operation) + if err != nil { + return err + } + config := journal.Recovery.Config + if err := installer.beginMaintenance(ctx, config, id, reason); err != nil { + return fmt.Errorf("re-establish fenced lifecycle maintenance: %w", err) + } + if err := installer.waitLocalState(ctx, config, "unavailable"); err != nil { + return fmt.Errorf("re-drain fenced lifecycle maintenance: %w", err) + } + if err := installer.reattestLifecycleAuthority(ctx, home, *journal); err != nil { + return err + } + if err := validateLifecycleProviderMatrix(paths, *journal); err != nil { + return err + } + if err := validateLifecycleWiringVector(*journal, paths, lifecycleWiringMixed); err != nil { + return err + } + if err := installer.systemctl(ctx, "stop", config.AgentUnit); err != nil { + return fmt.Errorf("stop fenced lifecycle agent: %w", err) + } + inner, found, err := readTransactionJournal(paths.Journal) + if err != nil { + return fmt.Errorf("read fenced provider transaction: %w", err) + } + if found { + activeChanged := false + if active, activeFound, activeErr := readActiveState(paths.ActiveState); activeErr != nil { + return fmt.Errorf("read fenced provider active state: %w", activeErr) + } else if activeFound { + activeChanged = active.Current.ImageID == inner.Candidate.ImageID && active.Current.ImageRef == inner.Candidate.ImageRef + } + rollbackJournal := inner + if rollbackJournal.Phase == JournalCommitted { + rollbackJournal.Phase = JournalActivated + activeChanged = true + } + if err := refresher.rollback(ctx, config, paths, rollbackJournal, activeChanged); err != nil { + return fmt.Errorf("rollback fenced provider transaction: %w", err) + } + } + if _, remains, err := readTransactionJournal(paths.Journal); err != nil { + return fmt.Errorf("verify fenced provider rollback: %w", err) + } else if remains { + return errors.New("fenced provider transaction remains after rollback") + } + if len(journal.Snapshots) > 0 || len(journal.PreviousUnits) > 0 || journal.Activation != (systemdActivation{}) { + if err := installer.rollbackInstallBeforeStart(ctx, config, journal.Snapshots, journal.PreviousUnits, true, false, id, journal.Activation, func(recoveryContext context.Context) error { + return installer.reattestLifecycleAuthority(recoveryContext, home, *journal) + }); err != nil { + return fmt.Errorf("rollback fenced lifecycle wiring: %w", err) + } + if err := validateLifecycleWiringVector(*journal, paths, lifecycleWiringPre); err != nil { + return fmt.Errorf("verify rolled back lifecycle wiring: %w", err) + } + journal.Snapshots = nil + journal.WiringIntent = nil + journal.PreviousUnits = nil + journal.Activation = systemdActivation{} + } else { + if err := installer.reattestLifecycleAuthority(ctx, home, *journal); err != nil { + return err + } + if err := installer.systemctl(ctx, "start", config.AgentUnit); err != nil { + return fmt.Errorf("restart fenced lifecycle agent: %w", err) + } + } + if err := installer.waitLocalState(ctx, config, "unavailable"); err != nil { + return fmt.Errorf("observe restarted fenced lifecycle agent: %w", err) + } + journal.ProviderTransaction = nil + if err := writeLifecycleTransition(home, paths, journal, LifecycleReady, LifecycleRollback, installer.now()); err != nil { + return err + } + return installer.recoverLifecycleRelease(ctx, home, paths, journal, refresher) +} + +func validateLifecycleProviderMatrix(paths LifecyclePaths, outer LifecycleJournal) error { + inner, found, err := readTransactionJournal(paths.Journal) + if err != nil { + return fmt.Errorf("read lifecycle provider transaction: %w", err) + } + legacyRecovery := outer.Operation == LifecycleRefreshRecovery && (outer.Phase == LifecycleAdopting || outer.Phase == LifecycleFenced) + if legacyRecovery { + if !found || outer.ProviderTransaction == nil || outer.ProviderTransaction.LegacyJournalSHA256 == "" { + return errors.New("legacy recovery lifecycle requires an exact provider transaction binding") + } + binding := outer.ProviderTransaction + if inner.ID != binding.TransactionID || inner.OuterTransactionID != "" || inner.ProfileID != "" || inner.Candidate.Update.SHA256 != binding.Digest { + return errors.New("legacy provider transaction binding mismatch") + } + update := inner.Candidate.Update + if update.WorkerID != outer.Identity.WorkerID || update.PluginID != outer.Identity.PluginID || update.ComponentID != outer.Identity.ComponentID { + return errors.New("legacy provider transaction identity mismatch") + } + digest, err := hashRegularFile(paths.Journal, false) + if err != nil || digest != binding.LegacyJournalSHA256 { + return errors.New("legacy provider transaction journal hash mismatch") + } + return nil + } + requiresAbsent := outer.ProviderEffect != ProviderChanged || outer.Outcome == LifecycleRollback || + outer.Phase == LifecycleIntent || outer.Phase == LifecycleFencing || outer.Operation == LifecycleUninstall + if requiresAbsent { + if found { + return errors.New("lifecycle phase requires an absent provider transaction") + } + return nil + } + if !found { + if outer.Phase == LifecycleFenced || outer.Phase == LifecycleCommitted { + return nil + } + return errors.New("lifecycle changed commit requires a provider transaction") + } + if outer.ProviderTransaction == nil { + return errors.New("lifecycle changed provider transaction binding is absent") + } + binding := outer.ProviderTransaction + if binding.LegacyJournalSHA256 != "" { + return errors.New("non-legacy lifecycle contains a legacy provider journal binding") + } + if inner.ID != binding.TransactionID || inner.OuterTransactionID != outer.TransactionID { + return errors.New("lifecycle provider outer transaction binding mismatch") + } + if inner.ProfileID != binding.ProfileID || inner.ProfileID != outer.Identity.ProfileID { + return errors.New("lifecycle provider profile binding mismatch") + } + if inner.Candidate.Update.SHA256 != binding.Digest { + return errors.New("lifecycle provider digest binding mismatch") + } + update := inner.Candidate.Update + if update.WorkerID != outer.Identity.WorkerID || update.PluginID != outer.Identity.PluginID || update.ComponentID != outer.Identity.ComponentID { + return errors.New("lifecycle provider candidate identity mismatch") + } + if !inner.DeferredCommit { + return errors.New("lifecycle provider transaction is not deferred") + } + if (outer.Phase == LifecycleReady || outer.Phase == LifecycleReleasing) && outer.Outcome == LifecycleCommit && inner.Phase != JournalCommitted { + return errors.New("lifecycle ready changed provider transaction is not committed") + } + return nil +} + +func (installer Installer) recoverLifecycleRelease(ctx context.Context, home string, paths LifecyclePaths, journal *LifecycleJournal, refresher Refresher) error { + id, reason, err := lifecycleMaintenanceIdentity(journal.Operation) + if err != nil { + return err + } + if err := installer.reattestLifecycleAuthority(ctx, home, *journal); err != nil { + return err + } + if len(journal.WiringIntent) > 0 { + expectation := lifecycleWiringPre + if journal.Outcome == LifecycleCommit { + expectation = lifecycleWiringIntended + } + if err := validateLifecycleWiringVector(*journal, paths, expectation); err != nil { + return err + } + } + if journal.Phase == LifecycleReady { + if err := writeLifecycleTransition(home, paths, journal, LifecycleReleasing, journal.Outcome, installer.now()); err != nil { + return err + } + _ = drainLifecycleAudit(home, paths, journal) + } + state, err := installer.maintenanceStatus(ctx, journal.Recovery.Config) + if err != nil { + return fmt.Errorf("read lifecycle maintenance status: %w", err) + } + switch classifyMaintenanceState(state, journal.Identity.ProfileID, id, reason) { + case maintenanceExactActive: + if err := installer.waitLocalDrained(ctx, journal.Recovery.Config); err != nil { + return fmt.Errorf("wait for lifecycle maintenance drain: %w", err) + } + if err := installer.releaseLifecycleMaintenance(ctx, home, *journal); err != nil { + return fmt.Errorf("release lifecycle maintenance: %w", err) + } + case maintenanceInactive: + case maintenanceConflicting: + return errors.New("lifecycle maintenance status has a conflicting active transaction") + default: + return errors.New("lifecycle maintenance status is invalid") + } + if err := writeLifecycleTransition(home, paths, journal, LifecycleCommitted, journal.Outcome, installer.now()); err != nil { + return err + } + return finalizeLifecycleTransaction(home, paths, journal, refresher) +} + +func (installer Installer) reattestLifecycleAuthority(ctx context.Context, home string, journal LifecycleJournal) error { + if err := journal.Recovery.Reattest(); err != nil { + return err + } + return installer.validateLifecycleAgentUnit(ctx, home, journal) +} + +func finalizeLifecycleTransaction(home string, paths LifecyclePaths, journal *LifecycleJournal, refresher Refresher) error { + if journal.Phase != LifecycleCommitted { + return errors.New("lifecycle transaction is not committed") + } + if journal.Outcome == LifecycleCommit && journal.ProviderEffect == ProviderChanged { + if _, found, err := readTransactionJournal(paths.Journal); err != nil { + return fmt.Errorf("read committed provider transaction: %w", err) + } else if found { + if err := refresher.finalizeDeferredRefresh(journal.Recovery.Config); err != nil { + return fmt.Errorf("finalize committed provider rollback target: %w", err) + } + } + } + if journal.Outcome == LifecycleCommit && journal.Operation == LifecycleUninstall && journal.Uninstall != nil && journal.Uninstall.Purge { + if info, err := os.Lstat(paths.Root); err == nil { + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return errors.New("purged lifecycle root is not a real directory") + } + if err := validateOwner(info); err != nil { + return fmt.Errorf("validate purged lifecycle root owner: %w", err) + } + if err := os.RemoveAll(paths.Root); err != nil { + return fmt.Errorf("purge retained provider state: %w", err) + } + if err := syncDirectory(filepath.Dir(paths.Root)); err != nil { + return fmt.Errorf("sync retained provider purge: %w", err) + } + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("inspect retained provider purge root: %w", err) + } + } + return finishLifecycleTransaction(home, paths, journal) +} + +func finishLifecycleTransaction(home string, paths LifecyclePaths, journal *LifecycleJournal) error { + if err := drainLifecycleAudit(home, paths, journal); err != nil { + return fmt.Errorf("drain lifecycle audit before cleanup: %w", err) + } + transactionRoot := paths.LifecycleTransactionRoot(journal.TransactionID) + if info, err := os.Lstat(transactionRoot); err == nil { + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return errors.New("lifecycle transaction root is not a real directory") + } + if err := validateOwner(info); err != nil { + return fmt.Errorf("validate lifecycle transaction root owner: %w", err) + } + if err := os.RemoveAll(transactionRoot); err != nil { + return fmt.Errorf("remove lifecycle transaction root: %w", err) + } + if err := syncDirectory(paths.LifecycleTransactions); err != nil { + return fmt.Errorf("sync lifecycle transactions after cleanup: %w", err) + } + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("inspect lifecycle transaction root: %w", err) + } + if err := removeDurableFile(paths.LifecycleJournal); err != nil { + return fmt.Errorf("remove lifecycle journal: %w", err) + } + return nil +} diff --git a/internal/retainedprovider/lifecycle_test.go b/internal/retainedprovider/lifecycle_test.go new file mode 100644 index 0000000..330454f --- /dev/null +++ b/internal/retainedprovider/lifecycle_test.go @@ -0,0 +1,1424 @@ +package retainedprovider + +import ( + "bytes" + "context" + "os" + "path/filepath" + "reflect" + "strconv" + "strings" + "testing" + "time" +) + +func TestRecoverReadyLifecycleReleasesForwardWithoutRefencing(t *testing.T) { + for _, tc := range []struct { + name string + status func(Config) []byte + wantErr string + wantEnd bool + wantCleanup bool + }{ + { + name: "exact active", + status: func(config Config) []byte { + return maintenanceStateJSON(true, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason) + }, + wantEnd: true, wantCleanup: true, + }, + { + name: "already inactive", + status: func(Config) []byte { return []byte(`{"active":false,"durable":true}`) }, + wantCleanup: true, + }, + { + name: "conflicting active", + status: func(config Config) []byte { + return maintenanceStateJSON(true, "other-transaction", config.ProfileID, refreshMaintenanceReason) + }, + wantErr: "conflicting", + }, + } { + t.Run(tc.name, func(t *testing.T) { + home := t.TempDir() + t.Setenv("XDG_STATE_HOME", filepath.Join(home, ".state")) + config := validTestConfig(home) + writeLifecycleRecoveryFiles(t, config) + paths := LifecyclePathsFor(config) + now := time.Unix(1_700_800_000, 0).UTC() + journal := lifecycleRecoveryJournalForTest(t, config, now) + journal.Operation = LifecycleRefresh + journal.ProviderEffect = ProviderUnchanged + setLifecycleUnchangedForTest(&journal, config, now) + if err := writeLifecycleJournal(home, paths, journal); err != nil { + t.Fatalf("write intent journal: %v", err) + } + journal.Phase = LifecycleReady + journal.Outcome = LifecycleRollback + if err := writeLifecycleJournal(home, paths, journal); err != nil { + t.Fatalf("write ready journal: %v", err) + } + + runner := &recordingCommandRunner{run: func(_ context.Context, command Command) ([]byte, error) { + switch installCommandEvent(command, config) { + case "agent-signature": + return agentUnitSystemdOutputForTest(t, config), nil + case "maintenance-status": + return tc.status(config), nil + case "local-status": + return localStatusJSON(config.WorkerID, "unavailable"), nil + case "maintenance-end": + return maintenanceStateJSON(false, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + default: + return nil, nil + } + }} + installer := Installer{Runner: runner, Now: func() time.Time { return now.Add(time.Minute) }, Sleep: func(context.Context, time.Duration) error { return nil }} + err := installer.recoverLifecycleTransaction(t.Context(), home, paths, Refresher{Runner: runner, Now: installer.Now, Sleep: installer.Sleep}) + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("recovery error = %v want %q", err, tc.wantErr) + } + if _, found, readErr := readLifecycleJournal(home, paths); readErr != nil || !found { + t.Fatalf("conflicting recovery journal found=%v err=%v", found, readErr) + } + } else if err != nil { + t.Fatalf("recover lifecycle: %v", err) + } + + transcript := commandTranscript(runner.commands) + if strings.Contains(transcript, "supervisor-maintenance begin") || strings.Contains(transcript, "systemctl --user stop "+config.AgentUnit) { + t.Fatalf("terminal recovery re-fenced or stopped the agent:\n%s", transcript) + } + if got := strings.Contains(transcript, "supervisor-maintenance end"); got != tc.wantEnd { + t.Fatalf("maintenance end present=%v want=%v:\n%s", got, tc.wantEnd, transcript) + } + if tc.wantCleanup { + if _, found, readErr := readLifecycleJournal(home, paths); readErr != nil || found { + t.Fatalf("terminal journal found=%v err=%v", found, readErr) + } + if _, statErr := os.Stat(paths.LifecycleTransactionRoot(journal.TransactionID)); !os.IsNotExist(statErr) { + t.Fatalf("transaction root remains: %v", statErr) + } + } + }) + } +} + +func TestRecoverLifecycleReattestsJournalAuthorityBeforeCommands(t *testing.T) { + home := t.TempDir() + t.Setenv("XDG_STATE_HOME", filepath.Join(home, ".state")) + config := validTestConfig(home) + writeLifecycleRecoveryFiles(t, config) + paths := LifecyclePathsFor(config) + now := time.Unix(1_700_800_000, 0).UTC() + journal := lifecycleRecoveryJournalForTest(t, config, now) + journal.Operation = LifecycleRefresh + journal.ProviderEffect = ProviderUnchanged + setLifecycleUnchangedForTest(&journal, config, now) + if err := writeLifecycleJournal(home, paths, journal); err != nil { + t.Fatalf("write lifecycle journal: %v", err) + } + if err := os.WriteFile(config.ComputeAgentPath, []byte("replaced compute-agent"), 0o700); err != nil { + t.Fatalf("replace compute-agent: %v", err) + } + runner := &recordingCommandRunner{} + err := (Installer{Runner: runner}).recoverLifecycleTransaction(t.Context(), home, paths, Refresher{Runner: runner}) + if err == nil || !strings.Contains(err.Error(), "attestation") { + t.Fatalf("recovery error = %v", err) + } + if len(runner.commands) != 0 { + t.Fatalf("recovery issued commands before re-attestation: %+v", runner.commands) + } +} + +func TestRecoverFencedLifecycleReattestsAfterDrainBeforeStop(t *testing.T) { + home := t.TempDir() + t.Setenv("XDG_STATE_HOME", filepath.Join(home, ".state")) + config := validTestConfig(home) + writeLifecycleRecoveryFiles(t, config) + paths := LifecyclePathsFor(config) + now := time.Unix(1_700_800_000, 0).UTC() + journal := lifecycleRecoveryJournalForTest(t, config, now) + journal.Operation = LifecycleRefresh + journal.ProviderEffect = ProviderUnchanged + setLifecycleUnchangedForTest(&journal, config, now) + if err := writeLifecycleJournal(home, paths, journal); err != nil { + t.Fatalf("write intent journal: %v", err) + } + journal.Phase = LifecycleFenced + if err := writeLifecycleJournal(home, paths, journal); err != nil { + t.Fatalf("write fenced journal: %v", err) + } + runner := &recordingCommandRunner{run: func(_ context.Context, command Command) ([]byte, error) { + switch installCommandEvent(command, config) { + case "agent-signature": + return agentUnitSystemdOutputForTest(t, config), nil + case "maintenance-begin": + return maintenanceStateJSON(true, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + case "local-status": + if err := os.WriteFile(config.ComputeAgentPath, []byte("replacement during drain"), 0o700); err != nil { + t.Fatalf("replace compute-agent during drain: %v", err) + } + return localStatusJSON(config.WorkerID, "unavailable"), nil + default: + return nil, nil + } + }} + err := (Installer{Runner: runner, Sleep: func(context.Context, time.Duration) error { return nil }}).recoverLifecycleTransaction(t.Context(), home, paths, Refresher{Runner: runner}) + if err == nil || !strings.Contains(err.Error(), "attestation") { + t.Fatalf("recovery error = %v", err) + } + transcript := commandTranscript(runner.commands) + if strings.Contains(transcript, "systemctl --user stop "+config.AgentUnit) || strings.Contains(transcript, "systemctl --user start "+config.AgentUnit) { + t.Fatalf("changed authority crossed stop boundary:\n%s", transcript) + } + if _, found, readErr := readLifecycleJournal(home, paths); readErr != nil || !found { + t.Fatalf("failed recovery lost journal found=%v err=%v", found, readErr) + } +} + +func TestRecoverLifecycleRejectsChangedEffectiveAgentUnitBeforeMutation(t *testing.T) { + home := t.TempDir() + t.Setenv("XDG_STATE_HOME", filepath.Join(home, ".state")) + config := validTestConfig(home) + writeLifecycleRecoveryFiles(t, config) + paths := LifecyclePathsFor(config) + now := time.Unix(1_700_800_000, 0).UTC() + journal := lifecycleRecoveryJournalForTest(t, config, now) + journal.Operation = LifecycleRefresh + journal.ProviderEffect = ProviderUnchanged + setLifecycleUnchangedForTest(&journal, config, now) + if err := writeLifecycleJournal(home, paths, journal); err != nil { + t.Fatalf("write intent journal: %v", err) + } + journal.Phase = LifecycleReady + journal.Outcome = LifecycleRollback + if err := writeLifecycleJournal(home, paths, journal); err != nil { + t.Fatalf("write ready journal: %v", err) + } + if err := os.WriteFile(agentUnitFragmentPathForTest(config), []byte("[Service]\nExecStart=/foreign/agent\n"), 0o600); err != nil { + t.Fatalf("replace agent fragment: %v", err) + } + runner := &recordingCommandRunner{run: func(_ context.Context, command Command) ([]byte, error) { + if installCommandEvent(command, config) == "agent-signature" { + return agentUnitSystemdOutputForTest(t, config), nil + } + return nil, nil + }} + err := (Installer{Runner: runner}).recoverLifecycleTransaction(t.Context(), home, paths, Refresher{Runner: runner}) + if err == nil || !strings.Contains(err.Error(), "pre-signature") { + t.Fatalf("recovery error = %v", err) + } + transcript := commandTranscript(runner.commands) + if !strings.Contains(transcript, "systemctl --user show "+config.AgentUnit) || strings.Contains(transcript, "supervisor-maintenance") || strings.Contains(transcript, "systemctl --user stop") || strings.Contains(transcript, "systemctl --user start") { + t.Fatalf("changed unit recovery crossed mutation boundary:\n%s", transcript) + } +} + +func TestRecoverLifecycleRejectsMismatchedProviderTransactionBeforeCommands(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(*LifecycleJournal, *TransactionJournal) + want string + }{ + {name: "outer id", mutate: func(_ *LifecycleJournal, inner *TransactionJournal) { inner.OuterTransactionID = "other-transaction" }, want: "outer transaction"}, + {name: "profile", mutate: func(_ *LifecycleJournal, inner *TransactionJournal) { inner.ProfileID = "other-profile" }, want: "profile"}, + {name: "digest", mutate: func(outer *LifecycleJournal, _ *TransactionJournal) { + outer.ProviderTransaction.Digest = "sha256:" + strings.Repeat("c", 64) + }, want: "digest"}, + {name: "non deferred", mutate: func(_ *LifecycleJournal, inner *TransactionJournal) { inner.DeferredCommit = false }, want: "outer transaction binding"}, + {name: "not committed", mutate: func(_ *LifecycleJournal, inner *TransactionJournal) { inner.Phase = JournalActivated }, want: "committed"}, + } { + t.Run(tc.name, func(t *testing.T) { + home := t.TempDir() + t.Setenv("XDG_STATE_HOME", filepath.Join(home, ".state")) + config := validTestConfig(home) + writeLifecycleRecoveryFiles(t, config) + paths := LifecyclePathsFor(config) + now := time.Unix(1_700_800_000, 0).UTC() + outer := lifecycleRecoveryJournalForTest(t, config, now) + outer.Operation = LifecycleRefresh + outer.ProviderEffect = ProviderChanged + selection := validTestSelection(now) + selection.Update.WorkerID = config.WorkerID + inner := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, + ID: "provider-transaction-123", Phase: JournalCommitted, DeferredCommit: true, + OuterTransactionID: outer.TransactionID, ProfileID: config.ProfileID, + Candidate: selection, StartedAt: now, UpdatedAt: now, + } + outer.ProviderTransaction = &LifecycleProviderTransaction{ + TransactionID: inner.ID, ProfileID: config.ProfileID, Digest: selection.Update.SHA256, + } + if err := writeLifecycleJournal(home, paths, outer); err != nil { + t.Fatalf("write intent lifecycle journal: %v", err) + } + outer.Phase = LifecycleReady + outer.Outcome = LifecycleCommit + tc.mutate(&outer, &inner) + if err := writeLifecycleJournal(home, paths, outer); err != nil { + t.Fatalf("write ready lifecycle journal: %v", err) + } + if err := AtomicWriteJSON(paths.Journal, inner); err != nil { + t.Fatalf("write provider transaction: %v", err) + } + runner := &recordingCommandRunner{} + err := (Installer{Runner: runner}).recoverLifecycleTransaction(t.Context(), home, paths, Refresher{Runner: runner}) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("recovery error = %v want %q", err, tc.want) + } + if len(runner.commands) != 0 { + t.Fatalf("recovery issued commands before provider binding validation: %+v", runner.commands) + } + }) + } +} + +func TestRecoverFencedLifecycleRollsBackEveryBoundProviderPhase(t *testing.T) { + for _, phase := range []JournalPhase{JournalPrepared, JournalStatePromoting, JournalStatePromoted, JournalActivated, JournalCommitted} { + t.Run(string(phase), func(t *testing.T) { + home := t.TempDir() + t.Setenv("XDG_STATE_HOME", filepath.Join(home, ".state")) + config := validTestConfig(home) + writeLifecycleRecoveryFiles(t, config) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("create provider state: %v", err) + } + if err := os.WriteFile(filepath.Join(paths.ProviderState, "generation"), []byte("previous"), 0o600); err != nil { + t.Fatalf("write previous provider state: %v", err) + } + now := time.Unix(1_700_800_000, 0).UTC() + previous := previousActiveStateForTest(t, home) + payload := writeTestProviderPayload(t, home, "outer-candidate-"+string(phase)) + digest := fileDigestForTest(t, payload) + candidate := selectionForDigest(payload, digest, "v1.0.32", "outer-directive-"+string(phase), "sha256:"+strings.Repeat("e", 64), now) + if err := prepareCandidateState(paths.ProviderState, paths.CandidateState(digest)); err != nil { + t.Fatalf("prepare candidate state: %v", err) + } + if err := os.WriteFile(filepath.Join(paths.CandidateState(digest), "generation"), []byte("candidate"), 0o600); err != nil { + t.Fatalf("write candidate provider state: %v", err) + } + switch phase { + case JournalStatePromoting: + if err := os.Rename(paths.ProviderState, paths.PreviousState(digest)); err != nil { + t.Fatalf("simulate state promoting: %v", err) + } + case JournalStatePromoted, JournalActivated, JournalCommitted: + if err := promoteCandidateProviderState(paths, digest); err != nil { + t.Fatalf("simulate promoted state: %v", err) + } + } + active := previous + if phase == JournalActivated || phase == JournalCommitted { + active = ActiveState{ProtocolVersion: ActiveStateProtocolVersion, Current: candidate, Previous: &previous.Current, UpdatedAt: now} + } + if err := AtomicWriteJSON(paths.ActiveState, active); err != nil { + t.Fatalf("write interrupted active state: %v", err) + } + outer := lifecycleRecoveryJournalForTest(t, config, now) + outer.Operation = LifecycleRefresh + outer.ProviderEffect = ProviderChanged + inner := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, + ID: "provider-transaction-" + string(phase), Phase: phase, DeferredCommit: true, + OuterTransactionID: outer.TransactionID, ProfileID: config.ProfileID, + Previous: &previous, Candidate: candidate, StartedAt: now, UpdatedAt: now, + } + outer.ProviderTransaction = &LifecycleProviderTransaction{TransactionID: inner.ID, ProfileID: config.ProfileID, Digest: digest} + if err := writeLifecycleJournal(home, paths, outer); err != nil { + t.Fatalf("write outer intent: %v", err) + } + outer.Phase = LifecycleFenced + if err := writeLifecycleJournal(home, paths, outer); err != nil { + t.Fatalf("write outer fenced: %v", err) + } + if err := AtomicWriteJSON(paths.Journal, inner); err != nil { + t.Fatalf("write bound inner: %v", err) + } + + maintenanceActive := false + runner := &recordingCommandRunner{run: func(_ context.Context, command Command) ([]byte, error) { + switch installCommandEvent(command, config) { + case "agent-signature": + return agentUnitSystemdOutputForTest(t, config), nil + case "maintenance-begin": + maintenanceActive = true + return maintenanceStateJSON(true, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + case "maintenance-status": + if maintenanceActive { + return maintenanceStateJSON(true, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + } + return []byte(`{"active":false,"durable":true}`), nil + case "maintenance-end": + maintenanceActive = false + return maintenanceStateJSON(false, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + case "local-status": + return localStatusJSON(config.WorkerID, "unavailable"), nil + default: + return nil, nil + } + }} + installer := Installer{Runner: runner, Now: func() time.Time { return now.Add(time.Minute) }, Sleep: func(context.Context, time.Duration) error { return nil }} + if err := installer.recoverLifecycleTransaction(t.Context(), home, paths, Refresher{Runner: runner, Now: installer.Now, Sleep: installer.Sleep}); err != nil { + t.Fatalf("recover outer %s: %v", phase, err) + } + if data, err := os.ReadFile(filepath.Join(paths.ProviderState, "generation")); err != nil || string(data) != "previous" { + t.Fatalf("restored provider state = %q err=%v", data, err) + } + recovered, found, err := readActiveState(paths.ActiveState) + if err != nil || !found || recovered.Current.ImageID != previous.Current.ImageID { + t.Fatalf("restored active = %+v found=%v err=%v", recovered, found, err) + } + if _, found, err := readLifecycleJournal(home, paths); err != nil || found { + t.Fatalf("outer remains found=%v err=%v", found, err) + } + if _, found, err := readTransactionJournal(paths.Journal); err != nil || found { + t.Fatalf("inner remains found=%v err=%v", found, err) + } + }) + } +} + +func TestRecoverFencingAndFencedLifecycleRollsBackBeforeRelease(t *testing.T) { + for _, tc := range []struct { + phase LifecyclePhase + wantStop bool + }{ + {phase: LifecycleFencing, wantStop: false}, + {phase: LifecycleFenced, wantStop: true}, + } { + t.Run(string(tc.phase), func(t *testing.T) { + home := t.TempDir() + t.Setenv("XDG_STATE_HOME", filepath.Join(home, ".state")) + config := validTestConfig(home) + writeLifecycleRecoveryFiles(t, config) + paths := LifecyclePathsFor(config) + now := time.Unix(1_700_800_000, 0).UTC() + journal := lifecycleRecoveryJournalForTest(t, config, now) + journal.Operation = LifecycleRefresh + journal.ProviderEffect = ProviderUnchanged + setLifecycleUnchangedForTest(&journal, config, now) + if err := writeLifecycleJournal(home, paths, journal); err != nil { + t.Fatalf("write intent journal: %v", err) + } + journal.Phase = tc.phase + if err := writeLifecycleJournal(home, paths, journal); err != nil { + t.Fatalf("write %s journal: %v", tc.phase, err) + } + + maintenanceActive := false + runner := &recordingCommandRunner{run: func(_ context.Context, command Command) ([]byte, error) { + switch installCommandEvent(command, config) { + case "agent-signature": + return agentUnitSystemdOutputForTest(t, config), nil + case "maintenance-begin": + maintenanceActive = true + return maintenanceStateJSON(true, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + case "maintenance-status": + if maintenanceActive { + return maintenanceStateJSON(true, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + } + return []byte(`{"active":false,"durable":true}`), nil + case "maintenance-end": + maintenanceActive = false + return maintenanceStateJSON(false, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + case "local-status": + return localStatusJSON(config.WorkerID, "unavailable"), nil + default: + return nil, nil + } + }} + installer := Installer{Runner: runner, Now: func() time.Time { return now.Add(time.Minute) }, Sleep: func(context.Context, time.Duration) error { return nil }} + if err := installer.recoverLifecycleTransaction(t.Context(), home, paths, Refresher{Runner: runner, Now: installer.Now, Sleep: installer.Sleep}); err != nil { + t.Fatalf("recover %s lifecycle: %v", tc.phase, err) + } + transcript := commandTranscript(runner.commands) + for _, required := range []string{"supervisor-maintenance begin", "supervisor-maintenance status", "supervisor-maintenance end"} { + if !strings.Contains(transcript, required) { + t.Fatalf("%s recovery missing %q:\n%s", tc.phase, required, transcript) + } + } + stopped := strings.Contains(transcript, "systemctl --user stop "+config.AgentUnit) + if stopped != tc.wantStop { + t.Fatalf("%s recovery stopped agent=%v want=%v:\n%s", tc.phase, stopped, tc.wantStop, transcript) + } + if tc.wantStop && !strings.Contains(transcript, "systemctl --user start "+config.AgentUnit) { + t.Fatalf("fenced recovery did not restart agent:\n%s", transcript) + } + if _, found, err := readLifecycleJournal(home, paths); err != nil || found { + t.Fatalf("recovered journal found=%v err=%v", found, err) + } + }) + } +} + +func TestRecoverLifecycleAdoptsLegacyInnerBeforeMaintenance(t *testing.T) { + home := t.TempDir() + t.Setenv("XDG_STATE_HOME", filepath.Join(home, ".state")) + config := validTestConfig(home) + writeLifecycleRecoveryFiles(t, config) + paths := LifecyclePathsFor(config) + if err := os.MkdirAll(paths.Root, 0o700); err != nil { + t.Fatalf("create provider root: %v", err) + } + if err := AtomicWriteJSON(paths.ConfigFile, config); err != nil { + t.Fatalf("write installed config: %v", err) + } + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("create provider state: %v", err) + } + if err := os.WriteFile(filepath.Join(paths.ProviderState, "generation"), []byte("previous"), 0o600); err != nil { + t.Fatalf("write previous provider state: %v", err) + } + now := time.Unix(1_700_800_000, 0).UTC() + previous := previousActiveStateForTest(t, home) + if err := AtomicWriteJSON(paths.ActiveState, previous); err != nil { + t.Fatalf("write previous active state: %v", err) + } + payload := writeTestProviderPayload(t, home, "legacy-candidate") + digest := fileDigestForTest(t, payload) + candidate := selectionForDigest(payload, digest, "v1.0.32", "legacy-directive", "sha256:"+strings.Repeat("e", 64), now) + if err := prepareCandidateState(paths.ProviderState, paths.CandidateState(digest)); err != nil { + t.Fatalf("prepare legacy candidate state: %v", err) + } + inner := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, + ID: "legacy-provider-transaction", Phase: JournalPrepared, + Previous: &previous, Candidate: candidate, StartedAt: now, UpdatedAt: now, + } + if err := AtomicWriteJSON(paths.Journal, inner); err != nil { + t.Fatalf("write legacy inner journal: %v", err) + } + + maintenanceActive := false + sawAdopting := false + runner := &recordingCommandRunner{run: func(_ context.Context, command Command) ([]byte, error) { + switch installCommandEvent(command, config) { + case "agent-signature": + return agentUnitSystemdOutputForTest(t, config), nil + case "maintenance-begin": + outer, found, err := readLifecycleJournal(home, paths) + wantPhase := LifecycleAdopting + if sawAdopting { + wantPhase = LifecycleFenced + } + if err != nil || !found || outer.Operation != LifecycleRefreshRecovery || outer.Phase != wantPhase || outer.ProviderTransaction == nil || outer.ProviderTransaction.LegacyJournalSHA256 == "" { + t.Fatalf("legacy adoption outer = %+v found=%v err=%v", outer, found, err) + } + if outer.Phase == LifecycleAdopting { + sawAdopting = true + } + maintenanceActive = true + return maintenanceStateJSON(true, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + case "maintenance-status": + if maintenanceActive { + return maintenanceStateJSON(true, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + } + return []byte(`{"active":false,"durable":true}`), nil + case "maintenance-end": + maintenanceActive = false + return maintenanceStateJSON(false, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + case "local-status": + return localStatusJSON(config.WorkerID, "unavailable"), nil + default: + return nil, nil + } + }} + installer := Installer{Runner: runner, Now: func() time.Time { return now.Add(time.Minute) }, Sleep: func(context.Context, time.Duration) error { return nil }} + if err := installer.recoverLifecycleTransaction(t.Context(), home, paths, Refresher{Runner: runner, Now: installer.Now, Sleep: installer.Sleep}); err != nil { + t.Fatalf("recover adopted legacy transaction: %v", err) + } + if !sawAdopting { + t.Fatal("legacy transaction was not durably adopting before maintenance") + } + if _, found, err := readLifecycleJournal(home, paths); err != nil || found { + t.Fatalf("outer lifecycle remains found=%v err=%v", found, err) + } + if _, found, err := readTransactionJournal(paths.Journal); err != nil || found { + t.Fatalf("legacy inner remains found=%v err=%v", found, err) + } + transcript := commandTranscript(runner.commands) + assertOrderedText(t, transcript, []string{"systemctl --user show " + config.AgentUnit, "supervisor-maintenance begin", "systemctl --user stop " + config.AgentUnit, "systemctl --user start " + config.AgentUnit, "supervisor-maintenance end"}) +} + +func TestExplicitLegacyRecoveryRequiresExactConfirmationBeforeCommands(t *testing.T) { + home := t.TempDir() + t.Setenv("XDG_STATE_HOME", filepath.Join(home, ".state")) + config := validTestConfig(home) + writeLifecycleRecoveryFiles(t, config) + paths := LifecyclePathsFor(config) + if err := os.MkdirAll(paths.Root, 0o700); err != nil { + t.Fatalf("create provider root: %v", err) + } + now := time.Unix(1_700_800_000, 0).UTC() + selection := validTestSelection(now) + selection.Update.WorkerID = config.WorkerID + selection.Update.PluginID = config.PluginID + selection.Update.ComponentID = config.ComponentID + inner := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, + ID: "legacy-provider-transaction", Phase: JournalPrepared, + Candidate: selection, StartedAt: now, UpdatedAt: now, + } + if err := AtomicWriteJSON(paths.Journal, inner); err != nil { + t.Fatalf("write legacy inner journal: %v", err) + } + runner := &recordingCommandRunner{} + installer := Installer{Runner: runner} + _, err := installer.Recover(t.Context(), home, config, "different-transaction") + if err == nil || !strings.Contains(err.Error(), "confirmation") { + t.Fatalf("explicit recovery error = %v", err) + } + if len(runner.commands) != 0 { + t.Fatalf("mismatched confirmation issued commands: %+v", runner.commands) + } +} + +func writeLifecycleRecoveryFiles(t *testing.T, config Config) { + t.Helper() + for _, file := range []struct { + path string + mode os.FileMode + data string + }{ + {path: config.ComputeAgentPath, mode: 0o700, data: "compute-agent fixture"}, + {path: config.SupervisorConfigPath, mode: 0o600, data: "supervisor config fixture"}, + {path: agentUnitFragmentPathForTest(config), mode: 0o600, data: "[Service]\nExecStart=" + config.ComputeAgentPath + " run\n"}, + } { + if err := os.MkdirAll(filepath.Dir(file.path), 0o700); err != nil { + t.Fatalf("create recovery file directory: %v", err) + } + if _, err := os.Lstat(file.path); err == nil { + continue + } else if !os.IsNotExist(err) { + t.Fatalf("inspect recovery file: %v", err) + } + if err := os.WriteFile(file.path, []byte(file.data), file.mode); err != nil { + t.Fatalf("write recovery file: %v", err) + } + } +} + +func lifecycleRecoveryJournalForTest(t *testing.T, config Config, now time.Time) LifecycleJournal { + t.Helper() + journal := validLifecycleJournalForTest(config, now) + computeAgentDigest, err := hashRegularFile(config.ComputeAgentPath, true) + if err != nil { + t.Fatalf("hash compute-agent: %v", err) + } + supervisorDigest, err := hashRegularFile(config.SupervisorConfigPath, false) + if err != nil { + t.Fatalf("hash supervisor config: %v", err) + } + journal.Recovery.ComputeAgent.SHA256 = computeAgentDigest + journal.Recovery.SupervisorConfig.SHA256 = supervisorDigest + journal.Recovery.AgentUnitBefore = agentUnitSignatureForTest(t, config) + return journal +} + +func setLifecycleUnchangedForTest(journal *LifecycleJournal, config Config, now time.Time) { + selection := validTestSelection(now) + selection.Update.WorkerID = config.WorkerID + selection.Update.PluginID = config.PluginID + selection.Update.ComponentID = config.ComponentID + journal.Unchanged = &LifecycleUnchangedProvenance{Active: selection, Candidate: selection.Update} +} + +func TestLifecyclePathsSurviveProviderRootPurge(t *testing.T) { + home := t.TempDir() + paths := LifecyclePathsFor(validTestConfig(home)) + transactionID := "install-transaction-123" + + for name, path := range map[string]string{ + "journal": paths.LifecycleJournal, + "transaction root": paths.LifecycleTransactionRoot(transactionID), + "audit": paths.LifecycleAudit, + } { + relative, err := filepath.Rel(paths.Root, path) + if err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + t.Fatalf("%s %q is inside purgeable root %q", name, path, paths.Root) + } + if err := ValidateUserPath(home, path, false); err != nil { + t.Fatalf("%s path: %v", name, err) + } + } +} + +func TestLifecycleAuditDrainRecoversCompleteAndTornAppend(t *testing.T) { + for _, tc := range []struct { + name string + prepare func(t *testing.T, paths LifecyclePaths, journal *LifecycleJournal, payload []byte) + wantErr string + }{ + {name: "new append"}, + {name: "complete append before pending clear", prepare: func(t *testing.T, paths LifecyclePaths, journal *LifecycleJournal, payload []byte) { + t.Helper() + writeAuditFixture(t, paths.LifecycleAudit, payload) + offset := int64(0) + journal.Audit.Safety[0].Offset = &offset + journal.Audit.Safety[0].Digest = digestBytes(payload) + }}, + {name: "torn append", prepare: func(t *testing.T, paths LifecyclePaths, journal *LifecycleJournal, payload []byte) { + t.Helper() + writeAuditFixture(t, paths.LifecycleAudit, payload[:len(payload)/2]) + offset := int64(0) + journal.Audit.Safety[0].Offset = &offset + journal.Audit.Safety[0].Digest = digestBytes(payload) + }}, + {name: "unrelated tail", prepare: func(t *testing.T, paths LifecyclePaths, journal *LifecycleJournal, payload []byte) { + t.Helper() + writeAuditFixture(t, paths.LifecycleAudit, []byte("unrelated\n")) + offset := int64(0) + journal.Audit.Safety[0].Offset = &offset + journal.Audit.Safety[0].Digest = digestBytes(payload) + }, wantErr: "unrelated"}, + } { + t.Run(tc.name, func(t *testing.T) { + home := t.TempDir() + t.Setenv("XDG_STATE_HOME", filepath.Join(home, ".state")) + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + now := time.Unix(1_700_800_000, 0).UTC() + journal := validLifecycleJournalForTest(config, now) + event := LifecycleAuditEvent{ + EventID: "event-1", Timestamp: now, TransactionID: journal.TransactionID, + WorkerID: config.WorkerID, Operation: LifecycleInstall, + Phase: LifecycleIntent, Kind: AuditPhase, ProviderEffect: ProviderChanged, + } + if err := journal.Audit.EnqueueSafety(event); err != nil { + t.Fatalf("enqueue safety event: %v", err) + } + payload, err := lifecycleAuditPayload(journal.Audit.Safety[0]) + if err != nil { + t.Fatalf("audit payload: %v", err) + } + if tc.prepare != nil { + tc.prepare(t, paths, &journal, payload) + } + if err := writeLifecycleJournal(home, paths, journal); err != nil { + t.Fatalf("write lifecycle journal: %v", err) + } + err = drainLifecycleAudit(home, paths, &journal) + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("drain error = %v want %q", err, tc.wantErr) + } + if len(journal.Audit.Safety) != 1 { + t.Fatalf("failed drain cleared queue: %+v", journal.Audit) + } + return + } + if err != nil { + t.Fatalf("drain audit: %v", err) + } + if len(journal.Audit.Safety) != 0 { + t.Fatalf("drained queue = %+v", journal.Audit) + } + data, err := os.ReadFile(paths.LifecycleAudit) + if err != nil || !bytes.Equal(data, payload) { + t.Fatalf("audit data = %q err=%v want=%q", data, err, payload) + } + }) + } +} + +func writeAuditFixture(t *testing.T, path string, data []byte) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatalf("mkdir audit dir: %v", err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("write audit fixture: %v", err) + } +} + +func TestLifecycleJournalValidatesOperationEffectAndIdentity(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + now := time.Unix(1_700_800_000, 0).UTC() + journal := validLifecycleJournalForTest(config, now) + + if err := journal.Validate(home, LifecyclePathsFor(config)); err != nil { + t.Fatalf("valid lifecycle journal: %v", err) + } + + for _, tc := range []struct { + name string + mutate func(*LifecycleJournal) + want string + }{ + {name: "uninstall changed effect", mutate: func(j *LifecycleJournal) { + j.Operation = LifecycleUninstall + j.ProviderEffect = ProviderChanged + j.Uninstall = &LifecycleUninstallPayload{Purge: true} + }, want: "not_applicable"}, + {name: "install not-applicable effect", mutate: func(j *LifecycleJournal) { + j.ProviderEffect = ProviderNotApplicable + }, want: "not_applicable"}, + {name: "ready without outcome", mutate: func(j *LifecycleJournal) { + j.Phase = LifecycleReady + j.Outcome = "" + }, want: "outcome"}, + {name: "changed ready without inner binding", mutate: func(j *LifecycleJournal) { + j.Operation = LifecycleRefresh + j.Phase = LifecycleReady + j.Outcome = LifecycleCommit + j.ProviderTransaction = nil + }, want: "provider transaction"}, + {name: "changed pre-provider without inner is valid", mutate: func(j *LifecycleJournal) { + j.Operation = LifecycleRefresh + j.Phase = LifecycleFenced + j.ProviderTransaction = nil + }, want: ""}, + {name: "retry identity mismatch", mutate: func(j *LifecycleJournal) { + j.Recovery.Config.WorkerID = "different-worker" + }, want: "identity"}, + } { + t.Run(tc.name, func(t *testing.T) { + candidate := journal + candidate.Uninstall = nil + candidate.ProviderTransaction = nil + tc.mutate(&candidate) + err := candidate.Validate(home, LifecyclePathsFor(config)) + if tc.want == "" { + if err != nil { + t.Fatalf("Validate = %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("Validate = %v want %q", err, tc.want) + } + }) + } +} + +func TestLifecycleJournalRequiresBoundUnchangedProvenance(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + now := time.Unix(1_700_800_000, 0).UTC() + active := validTestSelection(now) + active.Update.WorkerID = config.WorkerID + active.Update.PluginID = config.PluginID + active.Update.ComponentID = config.ComponentID + candidate := active.Update + candidate.DirectiveID = "directive-same-digest" + + valid := validLifecycleJournalForTest(config, now) + valid.Operation = LifecycleRefresh + valid.ProviderEffect = ProviderUnchanged + valid.Unchanged = &LifecycleUnchangedProvenance{ + Active: active, Candidate: candidate, StableProbeAt: now.Add(time.Minute), + } + valid.Phase = LifecycleReady + valid.Outcome = LifecycleCommit + if err := valid.Validate(home, LifecyclePathsFor(config)); err != nil { + t.Fatalf("valid unchanged lifecycle: %v", err) + } + + for _, tc := range []struct { + name string + mutate func(*LifecycleJournal) + want string + }{ + {name: "missing provenance", mutate: func(j *LifecycleJournal) { j.Unchanged = nil }, want: "unchanged provenance"}, + {name: "candidate digest mismatch", mutate: func(j *LifecycleJournal) { + j.Unchanged.Candidate.SHA256 = "sha256:" + strings.Repeat("c", 64) + }, want: "digest"}, + {name: "candidate worker mismatch", mutate: func(j *LifecycleJournal) { + j.Unchanged.Candidate.WorkerID = "other-worker" + }, want: "identity"}, + {name: "missing successful probe", mutate: func(j *LifecycleJournal) { + j.Unchanged.StableProbeAt = time.Time{} + }, want: "probe"}, + {name: "changed effect with provenance", mutate: func(j *LifecycleJournal) { + j.ProviderEffect = ProviderChanged + j.ProviderTransaction = &LifecycleProviderTransaction{ + TransactionID: "provider-transaction-123", ProfileID: config.ProfileID, Digest: candidate.SHA256, + } + }, want: "unchanged provenance"}, + } { + t.Run(tc.name, func(t *testing.T) { + journal := valid + provenance := *valid.Unchanged + journal.Unchanged = &provenance + tc.mutate(&journal) + err := journal.Validate(home, LifecyclePathsFor(config)) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("Validate = %v want %q", err, tc.want) + } + }) + } +} + +func TestLifecycleJournalReservesSafetyEventsForTerminalRelease(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + now := time.Unix(1_700_800_000, 0).UTC() + journal := validLifecycleJournalForTest(config, now) + journal.Operation = LifecycleRefresh + journal.ProviderEffect = ProviderChanged + journal.Phase = LifecycleReady + journal.Outcome = LifecycleRollback + for index := 0; index < maxLifecycleSafetyEvents-1; index++ { + journal.Audit.Safety = append(journal.Audit.Safety, LifecycleAuditEvent{ + EventID: "event-" + strconv.Itoa(index+1), Sequence: uint64(index + 1), Timestamp: now, + TransactionID: journal.TransactionID, WorkerID: config.WorkerID, + Operation: journal.Operation, Phase: journal.Phase, Kind: AuditPhase, + Outcome: journal.Outcome, ProviderEffect: journal.ProviderEffect, + }) + } + journal.Audit.NextSequence = maxLifecycleSafetyEvents + if err := journal.Validate(home, LifecyclePathsFor(config)); err == nil || !strings.Contains(err.Error(), "reserved") { + t.Fatalf("Validate overcommitted safety queue = %v", err) + } +} + +func TestLifecycleAuditEventStrictUnion(t *testing.T) { + now := time.Unix(1_700_800_000, 0).UTC() + event := LifecycleAuditEvent{ + EventID: "event-1", Sequence: 1, Timestamp: now, + TransactionID: "install-transaction-123", WorkerID: "worker-1", + Operation: LifecycleInstall, Phase: LifecycleReady, Kind: AuditPhase, + Outcome: LifecycleCommit, ProviderEffect: ProviderChanged, + } + if err := event.Validate(); err != nil { + t.Fatalf("valid phase event: %v", err) + } + + invalid := event + invalid.ErrorClass = "transport" + if err := invalid.Validate(); err == nil || !strings.Contains(err.Error(), "error_class") { + t.Fatalf("phase event with error class = %v", err) + } + + errorEvent := event + errorEvent.Kind = AuditError + errorEvent.Outcome = "" + errorEvent.ProviderEffect = "" + errorEvent.ErrorClass = "provider_probe" + errorEvent.Count = 1 + errorEvent.FirstSeen = now + errorEvent.LastSeen = now + if err := errorEvent.Validate(); err != nil { + t.Fatalf("valid error event: %v", err) + } +} + +func TestLifecycleAuditDiagnosticsCoalesceAndOverflow(t *testing.T) { + now := time.Unix(1_700_800_000, 0).UTC() + queue := LifecycleAuditQueue{NextSequence: 1} + base := LifecycleAuditEvent{ + EventID: "event-1", Timestamp: now, TransactionID: "install-transaction-123", + WorkerID: "worker-1", Operation: LifecycleInstall, Phase: LifecycleFenced, + Kind: AuditError, ErrorClass: "provider_probe", + } + if err := queue.EnqueueDiagnostic(base); err != nil { + t.Fatalf("enqueue diagnostic: %v", err) + } + repeated := base + repeated.EventID = "event-2" + repeated.Timestamp = now.Add(time.Minute) + if err := queue.EnqueueDiagnostic(repeated); err != nil { + t.Fatalf("coalesce diagnostic: %v", err) + } + if len(queue.Diagnostics) != 1 || queue.Diagnostics[0].Count != 2 || !queue.Diagnostics[0].LastSeen.Equal(repeated.Timestamp) { + t.Fatalf("coalesced diagnostics = %+v", queue.Diagnostics) + } + + assigned := queue.Diagnostics[0] + offset := int64(42) + assigned.Offset = &offset + assigned.Digest = "sha256:" + strings.Repeat("a", 64) + queue.Diagnostics[0] = assigned + third := base + third.EventID = "event-3" + third.Timestamp = now.Add(2 * time.Minute) + if err := queue.EnqueueDiagnostic(third); err != nil { + t.Fatalf("enqueue diagnostic after append assignment: %v", err) + } + if len(queue.Diagnostics) != 2 || queue.Diagnostics[0].Count != 2 || queue.Diagnostics[1].Count != 1 { + t.Fatalf("assigned diagnostic head was mutated: %+v", queue.Diagnostics) + } + + for index := len(queue.Diagnostics); index < maxLifecycleDiagnosticEvents-1; index++ { + event := base + event.EventID = "event-" + strconv.Itoa(index+2) + event.ErrorClass = "class-" + strconv.Itoa(index) + event.Timestamp = now.Add(time.Duration(index) * time.Minute) + if err := queue.EnqueueDiagnostic(event); err != nil { + t.Fatalf("fill diagnostic queue at %d: %v", index, err) + } + } + overflow := base + overflow.EventID = "overflow-source-1" + overflow.ErrorClass = "beyond-capacity" + overflow.Timestamp = now.Add(40 * time.Minute) + if err := queue.EnqueueDiagnostic(overflow); err != nil { + t.Fatalf("enqueue overflow diagnostic: %v", err) + } + if got := queue.Diagnostics[len(queue.Diagnostics)-1]; got.Kind != AuditOverflow || got.ErrorClass != "other" || got.Count != 1 { + t.Fatalf("overflow diagnostic = %+v", got) + } + overflow.EventID = "overflow-source-2" + overflow.Timestamp = now.Add(41 * time.Minute) + if err := queue.EnqueueDiagnostic(overflow); err != nil { + t.Fatalf("coalesce overflow diagnostic: %v", err) + } + if got := queue.Diagnostics[len(queue.Diagnostics)-1]; got.Count != 2 || !got.LastSeen.Equal(overflow.Timestamp) { + t.Fatalf("coalesced overflow diagnostic = %+v", got) + } +} + +func TestLifecycleJournalOwnsSensitiveSnapshotsOutsideProviderRoot(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + now := time.Unix(1_700_800_000, 0).UTC() + journal := validLifecycleJournalForTest(config, now) + journal.Phase = LifecycleFencing + transactionRoot := paths.LifecycleTransactionRoot(journal.TransactionID) + snapshotContents := []byte("secret rollback bytes") + journal.Snapshots = []managedFileSnapshot{{ + Path: paths.ProviderEnv, Backup: filepath.Join(transactionRoot, "snapshots", "0"), + Mode: 0o600, Existed: true, SHA256: digestBytes(snapshotContents), + }} + journal.PreviousUnits = map[string]systemdUnitState{} + if err := os.MkdirAll(filepath.Dir(journal.Snapshots[0].Backup), 0o700); err != nil { + t.Fatalf("create snapshot directory: %v", err) + } + if err := os.WriteFile(journal.Snapshots[0].Backup, snapshotContents, 0o600); err != nil { + t.Fatalf("write snapshot: %v", err) + } + if err := journal.Validate(home, paths); err != nil { + t.Fatalf("valid lifecycle snapshot: %v", err) + } + + insideProvider := journal + insideProvider.Snapshots = append([]managedFileSnapshot(nil), journal.Snapshots...) + insideProvider.Snapshots[0].Backup = filepath.Join(paths.Root, "snapshot") + if err := os.MkdirAll(paths.Root, 0o700); err != nil { + t.Fatalf("create provider root: %v", err) + } + if err := os.WriteFile(insideProvider.Snapshots[0].Backup, []byte("wrongly rooted rollback bytes"), 0o600); err != nil { + t.Fatalf("write provider-root snapshot: %v", err) + } + if err := insideProvider.Validate(home, paths); err == nil || !strings.Contains(err.Error(), "transaction root") { + t.Fatalf("provider-root snapshot validation = %v", err) + } + + refresh := journal + refresh.Operation = LifecycleRefresh + if err := refresh.Validate(home, paths); err == nil || !strings.Contains(err.Error(), "snapshots") { + t.Fatalf("refresh snapshot validation = %v", err) + } +} + +func TestSnapshotManagedFilesUsesLifecycleTransactionRoot(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + transactionRoot := paths.LifecycleTransactionRoot("install-transaction-123") + if err := os.MkdirAll(transactionRoot, 0o700); err != nil { + t.Fatalf("create lifecycle transaction root: %v", err) + } + if err := atomicWriteFile(paths.ProviderEnv, []byte("GITHUB_TOKEN=secret\n"), 0o600); err != nil { + t.Fatalf("write managed secret: %v", err) + } + snapshots, err := snapshotManagedFilesAt(paths, transactionRoot) + if err != nil { + t.Fatalf("snapshot managed files: %v", err) + } + snapshotRoot := filepath.Join(transactionRoot, "snapshots") + if len(snapshots) != len(managedInstallPaths(paths)) { + t.Fatalf("snapshot count = %d want %d", len(snapshots), len(managedInstallPaths(paths))) + } + for _, snapshot := range snapshots { + if filepath.Dir(snapshot.Backup) != snapshotRoot { + t.Fatalf("snapshot backup outside transaction root: %+v", snapshot) + } + } + if err := os.RemoveAll(paths.Root); err != nil { + t.Fatalf("purge provider root: %v", err) + } + var secretSnapshot managedFileSnapshot + for _, snapshot := range snapshots { + if snapshot.Path == paths.ProviderEnv { + secretSnapshot = snapshot + break + } + } + data, err := os.ReadFile(secretSnapshot.Backup) + if err != nil || string(data) != "GITHUB_TOKEN=secret\n" { + t.Fatalf("snapshot after purge = %q err=%v", data, err) + } +} + +func TestLifecycleSnapshotsAreJournaledIncrementallyAndDigestBound(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + now := time.Unix(1_700_800_000, 0).UTC() + journal := validLifecycleJournalForTest(config, now) + if err := writeLifecycleJournal(home, paths, journal); err != nil { + t.Fatalf("write lifecycle journal: %v", err) + } + journal.Phase = LifecycleFencing + if err := writeLifecycleJournal(home, paths, journal); err != nil { + t.Fatalf("write fencing lifecycle journal: %v", err) + } + if err := atomicWriteFile(paths.ConfigFile, []byte("first managed file"), 0o600); err != nil { + t.Fatalf("write first managed file: %v", err) + } + if err := os.MkdirAll(filepath.Dir(paths.Launcher), 0o700); err != nil { + t.Fatalf("create launcher directory: %v", err) + } + if err := os.Symlink(paths.ConfigFile, paths.Launcher); err != nil { + t.Fatalf("create invalid second managed file: %v", err) + } + + if err := snapshotManagedFilesForLifecycle(home, paths, &journal, now.Add(time.Second)); err == nil || !strings.Contains(err.Error(), "regular file") { + t.Fatalf("incremental snapshot error = %v", err) + } + persisted, found, err := readLifecycleJournal(home, paths) + if err != nil || !found { + t.Fatalf("read incremental lifecycle journal found=%v err=%v", found, err) + } + if len(persisted.Snapshots) != 1 || persisted.Snapshots[0].Path != paths.ConfigFile || persisted.Snapshots[0].SHA256 == "" { + t.Fatalf("incremental snapshots = %+v", persisted.Snapshots) + } + if err := os.WriteFile(persisted.Snapshots[0].Backup, []byte("tampered rollback bytes"), 0o600); err != nil { + t.Fatalf("tamper snapshot: %v", err) + } + if _, _, err := readLifecycleJournal(home, paths); err == nil || !strings.Contains(err.Error(), "digest") { + t.Fatalf("tampered snapshot read error = %v", err) + } +} + +func TestFencedLifecycleRequiresCompleteManagedWiringIntent(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + now := time.Unix(1_700_800_000, 0).UTC() + journal := validLifecycleJournalForTest(config, now) + if err := writeLifecycleJournal(home, paths, journal); err != nil { + t.Fatalf("write lifecycle journal: %v", err) + } + journal.Phase = LifecycleFencing + if err := writeLifecycleJournal(home, paths, journal); err != nil { + t.Fatalf("write fencing lifecycle journal: %v", err) + } + if err := snapshotManagedFilesForLifecycle(home, paths, &journal, now.Add(time.Second)); err != nil { + t.Fatalf("snapshot lifecycle files: %v", err) + } + units, err := RenderSystemdUnits(config, paths) + if err != nil { + t.Fatalf("render units: %v", err) + } + journal.WiringIntent = managedWiringIntent(paths, units, true) + journal.Phase = LifecycleFenced + journal.UpdatedAt = now.Add(2 * time.Second) + if err := journal.Validate(home, paths); err != nil { + t.Fatalf("valid fenced install journal: %v", err) + } + + missingSnapshot := journal + missingSnapshot.Snapshots = missingSnapshot.Snapshots[:len(missingSnapshot.Snapshots)-1] + if err := missingSnapshot.Validate(home, paths); err == nil || !strings.Contains(err.Error(), "complete snapshots") { + t.Fatalf("missing snapshot validation = %v", err) + } + missingIntent := journal + missingIntent.WiringIntent = missingIntent.WiringIntent[:len(missingIntent.WiringIntent)-1] + if err := missingIntent.Validate(home, paths); err == nil || !strings.Contains(err.Error(), "complete wiring intent") { + t.Fatalf("missing intent validation = %v", err) + } + badDigest := journal + badDigest.WiringIntent = append([]LifecycleManagedFileIntent(nil), journal.WiringIntent...) + badDigest.WiringIntent[0].SHA256 = "sha256:" + strings.Repeat("f", 64) + if err := badDigest.Validate(home, paths); err == nil || !strings.Contains(err.Error(), "wiring intent digest") { + t.Fatalf("incorrect intent digest validation = %v", err) + } +} + +func TestLifecycleManagedWiringAcceptsOnlyPreOrIntendedCrashVectors(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + now := time.Unix(1_700_800_000, 0).UTC() + journal := validLifecycleJournalForTest(config, now) + if err := writeLifecycleJournal(home, paths, journal); err != nil { + t.Fatalf("write intent journal: %v", err) + } + journal.Phase = LifecycleFencing + if err := writeLifecycleJournal(home, paths, journal); err != nil { + t.Fatalf("write fencing journal: %v", err) + } + if err := snapshotManagedFilesForLifecycle(home, paths, &journal, now.Add(time.Second)); err != nil { + t.Fatalf("snapshot lifecycle files: %v", err) + } + units, err := RenderSystemdUnits(config, paths) + if err != nil { + t.Fatalf("render units: %v", err) + } + journal.WiringIntent = managedWiringIntent(paths, units, true) + journal.Phase = LifecycleFenced + journal.UpdatedAt = now.Add(2 * time.Second) + if err := writeLifecycleJournal(home, paths, journal); err != nil { + t.Fatalf("write fenced journal: %v", err) + } + + for _, intent := range journal.WiringIntent[:2] { + if err := atomicWriteFile(intent.Path, intent.Contents, intent.Mode); err != nil { + t.Fatalf("write partial intended wiring: %v", err) + } + } + if err := validateLifecycleWiringVector(journal, paths, lifecycleWiringMixed); err != nil { + t.Fatalf("mixed pre/intended vector: %v", err) + } + if err := validateLifecycleWiringVector(journal, paths, lifecycleWiringIntended); err == nil || !strings.Contains(err.Error(), "intended") { + t.Fatalf("partial ready vector validation = %v", err) + } + + third := journal.WiringIntent[2] + if err := atomicWriteFile(third.Path, []byte("foreign wiring bytes"), 0o600); err != nil { + t.Fatalf("write foreign wiring: %v", err) + } + if err := validateLifecycleWiringVector(journal, paths, lifecycleWiringMixed); err == nil || !strings.Contains(err.Error(), "neither pre-state nor intended") { + t.Fatalf("foreign fenced vector validation = %v", err) + } + + for _, intent := range journal.WiringIntent { + if err := atomicWriteFile(intent.Path, intent.Contents, intent.Mode); err != nil { + t.Fatalf("write intended wiring: %v", err) + } + } + if err := validateLifecycleWiringVector(journal, paths, lifecycleWiringIntended); err != nil { + t.Fatalf("complete intended vector: %v", err) + } +} + +func TestLifecycleSystemdSignatureAttestsEveryEffectiveInput(t *testing.T) { + home := t.TempDir() + fragmentPath := filepath.Join(home, ".config", "systemd", "user", "workflow-compute-agent.service") + dropInPath := filepath.Join(home, ".config", "systemd", "user", "workflow-compute-agent.service.d", "20-provider.conf") + environmentPath := filepath.Join(home, ".workflow-compute", "agent.env") + for path, contents := range map[string]string{ + fragmentPath: "[Service]\nExecStart=/opt/workflow/compute-agent\n", + dropInPath: "[Service]\nEnvironmentFile=" + environmentPath + "\n", + environmentPath: "WORKER_ID=worker-1\n", + } { + if err := atomicWriteFile(path, []byte(contents), 0o600); err != nil { + t.Fatalf("write effective systemd input: %v", err) + } + } + signature := LifecycleSystemdSignature{ + Fragment: lifecycleAttestationForTest(t, fragmentPath), + DropIns: []LifecycleFileAttestation{lifecycleAttestationForTest(t, dropInPath)}, + ExecStart: "{ path=/opt/workflow/compute-agent ; argv[]=/opt/workflow/compute-agent run ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", + EnvironmentFiles: []LifecycleFileAttestation{lifecycleAttestationForTest(t, environmentPath)}, + } + if err := signature.Validate(home); err != nil { + t.Fatalf("valid effective signature: %v", err) + } + if err := signature.Reattest(); err != nil { + t.Fatalf("re-attest effective signature: %v", err) + } + + if err := os.WriteFile(dropInPath, []byte("[Service]\nEnvironment=FOREIGN=1\n"), 0o600); err != nil { + t.Fatalf("replace drop-in: %v", err) + } + if err := signature.Reattest(); err == nil || !strings.Contains(err.Error(), "drop-in") { + t.Fatalf("re-attest replaced drop-in = %v", err) + } + + duplicate := signature + duplicate.DropIns = append(duplicate.DropIns, duplicate.DropIns[0]) + if err := duplicate.Validate(home); err == nil || !strings.Contains(err.Error(), "duplicate") { + t.Fatalf("duplicate drop-in validation = %v", err) + } +} + +func TestDeriveIntendedAgentUnitSignatureClosesDaemonReloadCrashWindow(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + writeLifecycleRecoveryFiles(t, config) + paths := LifecyclePathsFor(config) + before := agentUnitSignatureForTest(t, config) + units, err := RenderSystemdUnits(config, paths) + if err != nil { + t.Fatalf("render systemd units: %v", err) + } + wiring := managedWiringIntent(paths, units, true) + agentEnvironment := []byte("COMPUTE_GITHUB_RUNNER_PROVIDER_URL=https://provider\n") + intended, err := deriveLifecycleAgentUnitSignature(before, paths, wiring, &LifecycleFileAttestation{ + Path: paths.AgentEnv, SHA256: digestBytes(agentEnvironment), + }) + if err != nil { + t.Fatalf("derive install signature: %v", err) + } + if len(intended.DropIns) != 1 || intended.DropIns[0].Path != paths.AgentDropIn || intended.DropIns[0].SHA256 != digestBytes([]byte(units.AgentDropIn)) { + t.Fatalf("derived drop-ins = %+v", intended.DropIns) + } + if len(intended.EnvironmentFiles) != 1 || intended.EnvironmentFiles[0].Path != paths.AgentEnv || intended.EnvironmentFiles[0].SHA256 != digestBytes(agentEnvironment) { + t.Fatalf("derived environment files = %+v", intended.EnvironmentFiles) + } + if err := atomicWriteFile(paths.AgentDropIn, []byte(units.AgentDropIn), 0o600); err != nil { + t.Fatalf("write intended drop-in: %v", err) + } + if err := atomicWriteFile(paths.AgentEnv, agentEnvironment, 0o600); err != nil { + t.Fatalf("write intended environment: %v", err) + } + if actual := agentUnitSignatureForTest(t, config); !equalLifecycleSystemdSignature(actual, intended) { + t.Fatalf("actual signature = %+v want %+v", actual, intended) + } + + uninstalled, err := deriveLifecycleAgentUnitSignature(intended, paths, managedWiringIntent(paths, SystemdUnits{}, false), nil) + if err != nil { + t.Fatalf("derive uninstall signature: %v", err) + } + if len(uninstalled.DropIns) != 0 || len(uninstalled.EnvironmentFiles) != 0 || uninstalled.Fragment != before.Fragment || uninstalled.ExecStart != before.ExecStart { + t.Fatalf("derived uninstall signature = %+v", uninstalled) + } +} + +func TestDeriveIntendedAgentUnitSignatureCanonicalizesEnvironmentAttestations(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + writeLifecycleRecoveryFiles(t, config) + paths := LifecyclePathsFor(config) + units, err := RenderSystemdUnits(config, paths) + if err != nil { + t.Fatalf("render systemd units: %v", err) + } + foreignEnvironment := filepath.Join(home, "zz-foreign.env") + if err := atomicWriteFile(foreignEnvironment, []byte("FOREIGN=1\n"), 0o600); err != nil { + t.Fatalf("write foreign environment: %v", err) + } + before := agentUnitSignatureForTest(t, config) + before.DropIns = append(before.DropIns, LifecycleFileAttestation{ + Path: filepath.Join(config.SystemdDir, config.AgentUnit+".d", "70-foreign.conf"), + SHA256: "sha256:" + strings.Repeat("d", 64), + }) + before.EnvironmentFiles = append(before.EnvironmentFiles, lifecycleAttestationForTest(t, foreignEnvironment)) + agentEnvironment := []byte("COMPUTE_GITHUB_RUNNER_PROVIDER_URL=https://provider\n") + intended, err := deriveLifecycleAgentUnitSignature(before, paths, managedWiringIntent(paths, units, true), &LifecycleFileAttestation{ + Path: paths.AgentEnv, SHA256: digestBytes(agentEnvironment), + }) + if err != nil { + t.Fatalf("derive install signature: %v", err) + } + want := []string{paths.AgentEnv, foreignEnvironment} + got := make([]string, 0, len(intended.EnvironmentFiles)) + for _, attestation := range intended.EnvironmentFiles { + got = append(got, attestation.Path) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("canonical environment attestations = %q want %q", got, want) + } +} + +func lifecycleAttestationForTest(t *testing.T, path string) LifecycleFileAttestation { + t.Helper() + digest, err := hashRegularFile(path, false) + if err != nil { + t.Fatalf("hash lifecycle attestation: %v", err) + } + return LifecycleFileAttestation{Path: path, SHA256: digest} +} + +func validLifecycleJournalForTest(config Config, now time.Time) LifecycleJournal { + return LifecycleJournal{ + ProtocolVersion: LifecycleJournalProtocolVersion, + TransactionID: "install-transaction-123", + Operation: LifecycleInstall, + Phase: LifecycleIntent, + ProviderEffect: ProviderChanged, + Identity: LifecycleIdentity{ + WorkerID: config.WorkerID, ProfileID: config.ProfileID, + PluginID: config.PluginID, ComponentID: config.ComponentID, + }, + Recovery: LifecycleRecoveryAuthority{ + Config: config, + ComputeAgent: LifecycleFileAttestation{Path: config.ComputeAgentPath, SHA256: "sha256:" + strings.Repeat("a", 64)}, + SupervisorConfig: LifecycleFileAttestation{Path: config.SupervisorConfigPath, SHA256: "sha256:" + strings.Repeat("b", 64)}, + AgentUnitBefore: LifecycleSystemdSignature{ + Fragment: LifecycleFileAttestation{Path: agentUnitFragmentPathForTest(config), SHA256: "sha256:" + strings.Repeat("c", 64)}, + ExecStart: staticExecStartForTest(config), + }, + }, + Audit: LifecycleAuditQueue{NextSequence: 1}, + StartedAt: now, + UpdatedAt: now, + } +} + +func agentUnitFragmentPathForTest(config Config) string { + return filepath.Join(lifecycleHome(LifecyclePathsFor(config)), ".config", "systemd", "user", config.AgentUnit) +} + +func agentUnitSignatureForTest(t *testing.T, config Config) LifecycleSystemdSignature { + t.Helper() + paths := LifecyclePathsFor(config) + signature := LifecycleSystemdSignature{ + Fragment: lifecycleAttestationForTest(t, agentUnitFragmentPathForTest(config)), + ExecStart: staticExecStartForTest(config), + } + if _, err := os.Lstat(paths.AgentDropIn); err == nil { + signature.DropIns = append(signature.DropIns, lifecycleAttestationForTest(t, paths.AgentDropIn)) + } else if !os.IsNotExist(err) { + t.Fatalf("inspect agent drop-in: %v", err) + } + if _, err := os.Lstat(paths.AgentEnv); err == nil { + signature.EnvironmentFiles = append(signature.EnvironmentFiles, lifecycleAttestationForTest(t, paths.AgentEnv)) + } else if !os.IsNotExist(err) { + t.Fatalf("inspect agent environment: %v", err) + } + return signature +} + +func agentUnitSystemdOutputForTest(t *testing.T, config Config) []byte { + t.Helper() + output, err := agentUnitSystemdOutput(config) + if err != nil { + t.Fatalf("build agent unit systemd output: %v", err) + } + return output +} + +func agentUnitSystemdOutput(config Config) ([]byte, error) { + fragmentDigest, err := hashRegularFile(agentUnitFragmentPathForTest(config), false) + if err != nil { + return nil, err + } + paths := LifecyclePathsFor(config) + signature := LifecycleSystemdSignature{ + Fragment: LifecycleFileAttestation{Path: agentUnitFragmentPathForTest(config), SHA256: fragmentDigest}, + } + if _, err := os.Lstat(paths.AgentDropIn); err == nil { + digest, hashErr := hashRegularFile(paths.AgentDropIn, false) + if hashErr != nil { + return nil, hashErr + } + signature.DropIns = append(signature.DropIns, LifecycleFileAttestation{Path: paths.AgentDropIn, SHA256: digest}) + } else if !os.IsNotExist(err) { + return nil, err + } + dropIns := make([]string, 0, len(signature.DropIns)) + for _, attestation := range signature.DropIns { + dropIns = append(dropIns, attestation.Path) + } + return []byte(strings.Join([]string{ + "LoadState=loaded", + "FragmentPath=" + signature.Fragment.Path, + "DropInPaths=" + strings.Join(dropIns, " "), + }, "\n") + "\n"), nil +} + +func staticExecStartForTest(config Config) string { + return "[" + strconv.Quote(config.ComputeAgentPath+" run") + "]" +} diff --git a/internal/retainedprovider/refresh.go b/internal/retainedprovider/refresh.go index 2503884..90840b9 100644 --- a/internal/retainedprovider/refresh.go +++ b/internal/retainedprovider/refresh.go @@ -30,40 +30,87 @@ const ( var providerContainerfile = []byte("FROM scratch\nCOPY --chmod=0555 github-runner-provider /github-runner-provider\nENTRYPOINT [\"/github-runner-provider\"]\n") type LifecyclePaths struct { - Root string - ActiveState string - Journal string - InstallLock string - ProviderState string - PackagesRoot string - CandidatesRoot string - ProviderEnv string - ProbeEnv string - TLSRoot string - CAFile string + Root string + ConfigFile string + Launcher string + ActiveState string + Journal string + InstallLock string + InstallJournal string + LifecycleJournal string + LifecycleTransactions string + LifecycleAudit string + LifecycleAuditLock string + ProviderState string + PackagesRoot string + CandidatesRoot string + ProviderEnv string + ProbeEnv string + AgentEnv string + TLSRoot string + CAFile string + ServerCert string + ServerKey string + ContainersConf string + ProviderUnit string + RefreshUnit string + PathUnit string + TimerUnit string + AgentDropIn string } func LifecyclePathsFor(config Config) LifecyclePaths { root := config.InstallRoot + workspaceRoot := filepath.Dir(root) + home := filepath.Dir(workspaceRoot) + stateHome := os.Getenv("XDG_STATE_HOME") + if stateHome == "" { + stateHome = filepath.Join(home, ".local", "state") + } + audit := filepath.Join(stateHome, "wfctl", "plugins", GitHubPluginID, "retained-provider-audit.jsonl") return LifecyclePaths{ - Root: root, - ActiveState: filepath.Join(root, "lifecycle", "active.json"), - Journal: filepath.Join(root, "lifecycle", "transaction.json"), - InstallLock: filepath.Join(root, "lifecycle", "install.lock"), - ProviderState: filepath.Join(root, "provider-state"), - PackagesRoot: filepath.Join(root, "packages"), - CandidatesRoot: filepath.Join(root, "candidates"), - ProviderEnv: filepath.Join(root, "secrets", "provider.env"), - ProbeEnv: filepath.Join(root, "secrets", "probe.env"), - TLSRoot: filepath.Join(root, "tls"), - CAFile: filepath.Join(root, "tls", "ca.pem"), + Root: root, + ConfigFile: filepath.Join(root, "config.json"), + Launcher: filepath.Join(root, "bin", "github-runner-provider"), + ActiveState: filepath.Join(root, "lifecycle", "active.json"), + Journal: filepath.Join(root, "lifecycle", "transaction.json"), + InstallLock: filepath.Join(filepath.Dir(root), ".workflow-plugin-github-runner-provider.install.lock"), + InstallJournal: filepath.Join(filepath.Dir(root), ".workflow-plugin-github-runner-provider.install-transaction.json"), + LifecycleJournal: filepath.Join(workspaceRoot, ".workflow-plugin-github-runner-provider.lifecycle-transaction.json"), + LifecycleTransactions: filepath.Join(workspaceRoot, ".workflow-plugin-github-runner-provider-transactions"), + LifecycleAudit: audit, + LifecycleAuditLock: audit + ".lock", + ProviderState: filepath.Join(root, "provider-state"), + PackagesRoot: filepath.Join(root, "packages"), + CandidatesRoot: filepath.Join(root, "candidates"), + ProviderEnv: filepath.Join(root, "secrets", "provider.env"), + ProbeEnv: filepath.Join(root, "secrets", "probe.env"), + AgentEnv: filepath.Join(root, "secrets", "agent.env"), + TLSRoot: filepath.Join(root, "tls"), + CAFile: filepath.Join(root, "tls", "ca.pem"), + ServerCert: filepath.Join(root, "tls", "server.crt"), + ServerKey: filepath.Join(root, "tls", "server.key"), + ContainersConf: filepath.Join(root, "runtime", "containers.conf"), + ProviderUnit: filepath.Join(config.SystemdDir, providerServiceUnit), + RefreshUnit: filepath.Join(config.SystemdDir, refreshServiceUnit), + PathUnit: filepath.Join(config.SystemdDir, refreshPathUnit), + TimerUnit: filepath.Join(config.SystemdDir, refreshTimerUnit), + AgentDropIn: filepath.Join(config.SystemdDir, config.AgentUnit+".d", "50-workflow-plugin-github-runner-provider.conf"), } } +func (paths LifecyclePaths) LifecycleTransactionRoot(transactionID string) string { + return filepath.Join(paths.LifecycleTransactions, transactionID) +} + func (paths LifecyclePaths) CandidateState(digest string) string { return filepath.Join(paths.CandidatesRoot, digestHex(digest), "state") } +func (paths LifecyclePaths) PreviousState(digest string) string { + return filepath.Join(paths.CandidatesRoot, digestHex(digest), "previous-state") +} + func (paths LifecyclePaths) PackageDir(digest string) string { return filepath.Join(paths.PackagesRoot, digestHex(digest)) } @@ -87,7 +134,7 @@ func (refresher Refresher) Refresh(ctx context.Context, config Config) (status S if err := validateInstallRoot(paths.Root); err != nil { return Status{}, err } - if err := ValidateUserPath(paths.Root, paths.InstallLock, false); err != nil { + if err := ValidateUserPath(filepath.Dir(paths.Root), paths.InstallLock, false); err != nil { return Status{}, fmt.Errorf("install lock path: %w", err) } lock, err := AcquireInstallLock(paths.InstallLock) @@ -95,6 +142,202 @@ func (refresher Refresher) Refresh(ctx context.Context, config Config) (status S return Status{}, err } defer func() { returnErr = errors.Join(returnErr, lock.Release()) }() + home := lifecycleHome(paths) + installer := Installer{Runner: refresher.Runner, Now: refresher.Now, Sleep: refresher.Sleep} + if err := installer.recoverLifecycleTransaction(ctx, home, paths, refresher); err != nil { + return Status{}, err + } + if err := installer.recoverInstallTransaction(ctx, config, paths, refresher); err != nil { + return Status{}, err + } + requiresMutation, err := refresher.requiresMutation(ctx, config, paths) + if err != nil { + return Status{}, err + } + if !requiresMutation { + return refresher.refreshUnchangedUnderLifecycleLock(ctx, home, config, paths) + } + return refresher.refreshFencedUnderLifecycleLock(ctx, home, config, paths) +} + +func (refresher Refresher) refreshUnchangedUnderLifecycleLock(ctx context.Context, home string, config Config, paths LifecyclePaths) (Status, error) { + update, err := VerifyCurrentUpdate(ctx, config, refresher.Runner) + if err != nil { + return Status{}, err + } + active, found, err := readActiveState(paths.ActiveState) + if err != nil { + return Status{}, err + } + if !found || active.Current.Update.SHA256 != update.SHA256 { + return refresher.refreshFencedUnderLifecycleLock(ctx, home, config, paths) + } + installer := Installer{Runner: refresher.Runner, Now: refresher.Now, Sleep: refresher.Sleep} + transaction, err := newLifecycleJournal(config, LifecycleRefresh, ProviderUnchanged, nil, refresher.now()) + if err != nil { + return Status{}, err + } + signature, err := installer.inspectAgentUnitSignature(ctx, home, config) + if err != nil { + return Status{}, err + } + transaction.Recovery.AgentUnitBefore = signature + transaction.Unchanged = &LifecycleUnchangedProvenance{Active: active.Current, Candidate: update} + if err := startLifecycleTransaction(home, paths, &transaction); err != nil { + return Status{}, err + } + status, err := refresher.refreshUnderLifecycleTransaction(ctx, config, true, false, "", "", update.SHA256) + if err != nil { + return Status{}, errors.Join(err, finishLifecycleTransaction(home, paths, &transaction)) + } + transaction.Unchanged.StableProbeAt = refresher.now() + if err := writeLifecycleTransition(home, paths, &transaction, LifecycleReady, LifecycleCommit, refresher.now()); err != nil { + return Status{}, err + } + if err := writeLifecycleTransition(home, paths, &transaction, LifecycleReleasing, LifecycleCommit, refresher.now()); err != nil { + return Status{}, err + } + _ = drainLifecycleAudit(home, paths, &transaction) + if err := writeLifecycleTransition(home, paths, &transaction, LifecycleCommitted, LifecycleCommit, refresher.now()); err != nil { + return Status{}, err + } + if err := finalizeLifecycleTransaction(home, paths, &transaction, refresher); err != nil { + return Status{}, err + } + return status, nil +} + +func (refresher Refresher) requiresMutation(ctx context.Context, config Config, paths LifecyclePaths) (bool, error) { + update, err := VerifyCurrentUpdate(ctx, config, refresher.Runner) + if err != nil { + return false, err + } + active, found, err := readActiveState(paths.ActiveState) + if err != nil { + return false, err + } + if !found { + if err := refresher.validateInitialInstaller(update); err != nil { + return false, err + } + } + return !found || active.Current.Update.SHA256 != update.SHA256, nil +} + +func (refresher Refresher) validateInitialInstaller(update VerifiedUpdate) error { + executablePath := refresher.ExecutablePath + if executablePath == nil { + executablePath = os.Executable + } + currentExecutable, err := executablePath() + if err != nil { + return fmt.Errorf("resolve installer executable: %w", err) + } + digest, err := hashRegularFile(currentExecutable, true) + if err != nil { + return fmt.Errorf("hash installer executable: %w", err) + } + if digest != update.SHA256 { + return errors.New("installer digest does not match verified provider update") + } + return nil +} + +func (refresher Refresher) refreshFencedUnderLifecycleLock(ctx context.Context, home string, config Config, paths LifecyclePaths) (Status, error) { + installer := Installer{Runner: refresher.Runner, Now: refresher.Now, Sleep: refresher.Sleep} + transaction, err := newLifecycleJournal(config, LifecycleRefresh, ProviderChanged, nil, refresher.now()) + if err != nil { + return Status{}, err + } + beforeSignature, err := installer.inspectAgentUnitSignature(ctx, home, config) + if err != nil { + return Status{}, err + } + transaction.Recovery.AgentUnitBefore = beforeSignature + if err := startLifecycleTransaction(home, paths, &transaction); err != nil { + return Status{}, err + } + if err := writeLifecycleTransition(home, paths, &transaction, LifecycleFencing, "", refresher.now()); err != nil { + return Status{}, err + } + fail := func(cause error) (Status, error) { + rollbackContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + return Status{}, errors.Join(cause, installer.recoverLifecycleTransaction(rollbackContext, home, paths, refresher)) + } + if err := installer.beginMaintenance(ctx, config, refreshMaintenanceID, refreshMaintenanceReason); err != nil { + return fail(err) + } + if err := installer.waitLocalState(ctx, config, "unavailable"); err != nil { + return Status{}, fmt.Errorf("wait for retained agent refresh fence: %w", err) + } + if err := installer.reattestLifecycleAuthority(ctx, home, transaction); err != nil { + return Status{}, err + } + if err := writeLifecycleTransition(home, paths, &transaction, LifecycleFenced, "", refresher.now()); err != nil { + return fail(err) + } + if err := installer.systemctl(ctx, "stop", config.AgentUnit); err != nil { + return fail(fmt.Errorf("stop retained agent for provider refresh: %w", err)) + } + status, err := refresher.refreshUnderLifecycleTransaction(ctx, config, true, true, transaction.TransactionID, config.ProfileID, "") + if err != nil { + return fail(err) + } + inner, found, err := readTransactionJournal(paths.Journal) + if err != nil || !found || inner.Phase != JournalCommitted { + return fail(errors.Join(errors.New("provider refresh did not leave a deferred committed transaction"), err)) + } + transaction.ProviderTransaction = &LifecycleProviderTransaction{ + TransactionID: inner.ID, ProfileID: config.ProfileID, Digest: inner.Candidate.Update.SHA256, + } + transaction.UpdatedAt = refresher.now() + if err := writeLifecycleJournal(home, paths, transaction); err != nil { + return fail(err) + } + if err := installer.reattestLifecycleAuthority(ctx, home, transaction); err != nil { + return fail(err) + } + if err := installer.systemctl(ctx, "start", config.AgentUnit); err != nil { + return fail(fmt.Errorf("restart retained agent after provider refresh: %w", err)) + } + if err := installer.waitLocalState(ctx, config, "unavailable"); err != nil { + return fail(fmt.Errorf("verify retained agent remains refresh-fenced: %w", err)) + } + if err := installer.reattestLifecycleAuthority(ctx, home, transaction); err != nil { + return fail(err) + } + if err := writeLifecycleTransition(home, paths, &transaction, LifecycleReady, LifecycleCommit, refresher.now()); err != nil { + return fail(err) + } + if err := writeLifecycleTransition(home, paths, &transaction, LifecycleReleasing, LifecycleCommit, refresher.now()); err != nil { + return fail(err) + } + _ = drainLifecycleAudit(home, paths, &transaction) + if err := installer.releaseLifecycleMaintenance(ctx, home, transaction); err != nil { + return Status{}, fmt.Errorf("release retained agent refresh fence: %w", err) + } + if err := writeLifecycleTransition(home, paths, &transaction, LifecycleCommitted, LifecycleCommit, refresher.now()); err != nil { + return Status{}, err + } + if err := finalizeLifecycleTransaction(home, paths, &transaction, refresher); err != nil { + return Status{}, err + } + if err := installer.waitLocalState(ctx, config, "idle"); err != nil { + return Status{}, fmt.Errorf("wait for retained agent after provider refresh: %w", err) + } + return status, nil +} + +func (refresher Refresher) refreshUnderLifecycleLock(ctx context.Context, config Config, verifyCurrent, deferCommit bool) (Status, error) { + return refresher.refreshUnderLifecycleTransaction(ctx, config, verifyCurrent, deferCommit, "", "", "") +} + +func (refresher Refresher) refreshUnderLifecycleTransaction(ctx context.Context, config Config, verifyCurrent, deferCommit bool, outerTransactionID, profileID, expectedDigest string) (Status, error) { + if refresher.Runner == nil { + return Status{}, errors.New("command runner is required") + } + paths := LifecyclePathsFor(config) if err := refresher.recoverInterrupted(ctx, config, paths); err != nil { return Status{}, err } @@ -102,29 +345,17 @@ func (refresher Refresher) Refresh(ctx context.Context, config Config) (status S if err != nil { return Status{}, err } + if expectedDigest != "" && update.SHA256 != expectedDigest { + return Status{}, errors.New("verified provider update changed during unchanged refresh") + } active, activeFound, err := readActiveState(paths.ActiveState) if err != nil { return Status{}, err } now := refresher.now() - if activeFound && active.Current.Update.SHA256 == update.SHA256 { - return statusForActive(active, true, now), nil - } if !activeFound { - executablePath := refresher.ExecutablePath - if executablePath == nil { - executablePath = os.Executable - } - currentExecutable, err := executablePath() - if err != nil { - return Status{}, fmt.Errorf("resolve installer executable: %w", err) - } - digest, err := hashRegularFile(currentExecutable, true) - if err != nil { - return Status{}, fmt.Errorf("hash installer executable: %w", err) - } - if digest != update.SHA256 { - return Status{}, errors.New("installer digest does not match verified provider update") + if err := refresher.validateInitialInstaller(update); err != nil { + return Status{}, err } } for name, path := range map[string]string{ @@ -146,6 +377,17 @@ func (refresher Refresher) Refresh(ctx context.Context, config Config) (status S if err := validateSecretFile(paths.CAFile); err != nil { return Status{}, fmt.Errorf("provider CA file: %w", err) } + if err := refresher.validateProviderNetwork(ctx, config); err != nil { + return Status{}, err + } + if activeFound && active.Current.Update.SHA256 == update.SHA256 { + if verifyCurrent { + if err := refresher.probeStableActive(ctx, config, paths, active.Current); err != nil { + return Status{}, err + } + } + return statusForActive(active, true, now), nil + } if err := ValidateUserPath(paths.Root, paths.PackageDir(update.SHA256), false); err != nil { return Status{}, fmt.Errorf("provider package path: %w", err) } @@ -153,21 +395,24 @@ func (refresher Refresher) Refresh(ctx context.Context, config Config) (status S return Status{}, err } imageRef := providerImageRef(update.SHA256) - if _, err := refresher.Runner.Run(ctx, Command{ + if _, err := refresher.run(ctx, Command{ Path: config.PodmanPath, Args: []string{"build", "--file", "-", "--tag", imageRef, paths.PackageDir(update.SHA256)}, Stdin: providerContainerfile, }); err != nil { return Status{}, fmt.Errorf("build provider candidate image: %w", err) } - imageOutput, err := refresher.Runner.Run(ctx, Command{ + imageOutput, err := refresher.run(ctx, Command{ Path: config.PodmanPath, Args: []string{"image", "inspect", "--format", "{{.Id}}", imageRef}, }) if err != nil { return Status{}, fmt.Errorf("inspect provider candidate image: %w", err) } - imageID := strings.TrimSpace(string(imageOutput)) + imageID, err := normalizePodmanImageID(string(imageOutput)) + if err != nil { + return Status{}, fmt.Errorf("validate provider candidate image id: %w", err) + } selection := ImageSelection{Update: update, ImageID: imageID, ImageRef: imageRef, ActivatedAt: now} if err := selection.Validate(); err != nil { return Status{}, fmt.Errorf("validate provider candidate image: %w", err) @@ -176,16 +421,16 @@ func (refresher Refresher) Refresh(ctx context.Context, config Config) (status S if err := ValidateUserPath(paths.Root, candidateState, false); err != nil { return Status{}, fmt.Errorf("provider candidate state path: %w", err) } - if err := prepareCandidateState(paths.ProviderState, candidateState); err != nil { - return Status{}, err - } journal := TransactionJournal{ - ProtocolVersion: TransactionJournalProtocolVersion, - ID: "refresh-" + digestHex(update.SHA256)[:16], - Phase: JournalPrepared, - Candidate: selection, - StartedAt: now, - UpdatedAt: now, + ProtocolVersion: TransactionJournalProtocolVersion, + ID: "refresh-" + digestHex(update.SHA256)[:16], + Phase: JournalPrepared, + DeferredCommit: deferCommit, + OuterTransactionID: outerTransactionID, + ProfileID: profileID, + Candidate: selection, + StartedAt: now, + UpdatedAt: now, } if activeFound { previous := active @@ -201,16 +446,29 @@ func (refresher Refresher) Refresh(ctx context.Context, config Config) (status S if err := refresher.removeContainer(ctx, config, config.CandidateContainer); err != nil { return Status{}, rollback(fmt.Errorf("remove stale provider candidate: %w", err)) } - if _, err := refresher.Runner.Run(ctx, candidateProviderCommand(config, paths, candidateState, selection)); err != nil { + if _, err := refresher.run(ctx, Command{Path: "/usr/bin/systemctl", Args: []string{"--user", "stop", providerServiceUnit}}); err != nil { + return Status{}, rollback(fmt.Errorf("quiesce active provider before state clone: %w", err)) + } + if err := prepareCandidateState(paths.ProviderState, candidateState); err != nil { + return Status{}, rollback(err) + } + if _, err := refresher.run(ctx, candidateProviderCommand(config, paths, candidateState, selection)); err != nil { return Status{}, rollback(fmt.Errorf("start provider candidate: %w", err)) } if err := refresher.runProbe(ctx, providerProbeCommand(config, paths, config.CandidateContainer, selection)); err != nil { return Status{}, rollback(fmt.Errorf("probe provider candidate: %w", err)) } - journal.Phase = JournalActivated - journal.UpdatedAt = refresher.now() - if err := AtomicWriteJSON(paths.Journal, journal); err != nil { - return Status{}, rollback(fmt.Errorf("write activated refresh journal: %w", err)) + if err := writeJournalPhase(paths.Journal, &journal, JournalStatePromoting, refresher.now()); err != nil { + return Status{}, rollback(fmt.Errorf("write state-promoting refresh journal: %w", err)) + } + if err := refresher.removeContainer(ctx, config, config.CandidateContainer); err != nil { + return Status{}, rollback(fmt.Errorf("stop probed provider candidate: %w", err)) + } + if err := promoteCandidateProviderState(paths, update.SHA256); err != nil { + return Status{}, rollback(fmt.Errorf("promote provider candidate state: %w", err)) + } + if err := writeJournalPhase(paths.Journal, &journal, JournalStatePromoted, refresher.now()); err != nil { + return Status{}, rollback(fmt.Errorf("write state-promoted refresh journal: %w", err)) } newActive := ActiveState{ProtocolVersion: ActiveStateProtocolVersion, Current: selection, UpdatedAt: journal.UpdatedAt} if activeFound { @@ -221,19 +479,23 @@ func (refresher Refresher) Refresh(ctx context.Context, config Config) (status S return Status{}, rollback(fmt.Errorf("activate provider state: %w", err)) } activeChanged = true + if err := writeJournalPhase(paths.Journal, &journal, JournalActivated, refresher.now()); err != nil { + return Status{}, rollback(fmt.Errorf("write activated refresh journal: %w", err)) + } if err := refresher.restartProvider(ctx); err != nil { return Status{}, rollback(fmt.Errorf("restart active provider: %w", err)) } if err := refresher.runProbe(ctx, providerProbeCommand(config, paths, config.StableContainer, selection)); err != nil { return Status{}, rollback(fmt.Errorf("probe active provider: %w", err)) } - journal.Phase = JournalCommitted - journal.UpdatedAt = refresher.now() - if err := AtomicWriteJSON(paths.Journal, journal); err != nil { + if err := writeJournalPhase(paths.Journal, &journal, JournalCommitted, refresher.now()); err != nil { return Status{}, rollback(fmt.Errorf("commit refresh journal: %w", err)) } - if err := refresher.removeContainer(ctx, config, config.CandidateContainer); err != nil { - return Status{}, fmt.Errorf("remove provider candidate: %w", err) + if deferCommit { + return statusForActive(newActive, true, refresher.now()), nil + } + if err := cleanupProviderStateTransaction(paths, update.SHA256); err != nil { + return Status{}, fmt.Errorf("remove committed provider state rollback target: %w", err) } if err := removeDurableFile(paths.Journal); err != nil { return Status{}, fmt.Errorf("remove committed refresh journal: %w", err) @@ -270,14 +532,18 @@ func (refresher Refresher) ServeActive(ctx context.Context, config Config) error return fmt.Errorf("%s path: %w", name, err) } } - output, err := refresher.Runner.Run(ctx, Command{ + output, err := refresher.run(ctx, Command{ Path: config.PodmanPath, Args: []string{"image", "inspect", "--format", "{{.Id}}", active.Current.ImageRef}, }) if err != nil { return fmt.Errorf("inspect active provider image: %w", err) } - if imageID := strings.TrimSpace(string(output)); imageID != active.Current.ImageID { + imageID, err := normalizePodmanImageID(string(output)) + if err != nil { + return fmt.Errorf("validate active provider image id: %w", err) + } + if imageID != active.Current.ImageID { return errors.New("active provider image id does not match durable state") } return refresher.Runner.Exec(Command{Path: config.PodmanPath, Args: []string{ @@ -292,7 +558,7 @@ func (refresher Refresher) ServeActive(ctx context.Context, config Config) error } func VerifyCurrentUpdate(ctx context.Context, config Config, runner CommandRunner) (VerifiedUpdate, error) { - output, err := runner.Run(ctx, Command{ + output, err := runBoundedCommand(ctx, runner, Command{ Path: config.ComputeAgentPath, Args: []string{ "supervisor-update", "verify", @@ -392,13 +658,63 @@ func providerProbeCommand(config Config, paths LifecyclePaths, target string, se return Command{Path: config.PodmanPath, Args: arguments} } +func (refresher Refresher) validateProviderNetwork(ctx context.Context, config Config) error { + output, err := refresher.run(ctx, Command{Path: config.PodmanPath, Args: []string{ + "network", "inspect", "--format", "{{.Driver}} {{.DNSEnabled}} {{.Internal}}", config.ContainerNetwork, + }}) + if err != nil { + return fmt.Errorf("inspect provider network: %w", err) + } + fields := strings.Fields(string(output)) + if len(fields) != 3 || fields[0] != "bridge" || fields[1] != "true" || fields[2] != "false" { + return errors.New("provider network must be a non-internal bridge with DNS enabled") + } + return nil +} + func (refresher Refresher) restartProvider(ctx context.Context) error { - _, err := refresher.Runner.Run(ctx, Command{Path: "/usr/bin/systemctl", Args: []string{"--user", "restart", providerServiceUnit}}) + _, err := refresher.run(ctx, Command{Path: "/usr/bin/systemctl", Args: []string{"--user", "restart", providerServiceUnit}}) return err } +// RestartAndProbeActive revalidates the stable service even when the package +// digest did not change, as happens during credential rotation. +func (refresher Refresher) RestartAndProbeActive(ctx context.Context, config Config) error { + if refresher.Runner == nil { + return errors.New("command runner is required") + } + paths := LifecyclePathsFor(config) + active, found, err := readActiveState(paths.ActiveState) + if err != nil { + return err + } + if !found { + return errors.New("retained provider has no active image") + } + if err := refresher.restartProvider(ctx); err != nil { + return fmt.Errorf("restart active provider: %w", err) + } + return refresher.probeStableActive(ctx, config, paths, active.Current) +} + +func (refresher Refresher) probeStableActive(ctx context.Context, config Config, paths LifecyclePaths, selection ImageSelection) error { + output, err := refresher.run(ctx, Command{Path: "/usr/bin/systemctl", Args: []string{ + "--user", "show", providerServiceUnit, "--property", "ActiveState", "--value", + }}) + if err != nil { + return fmt.Errorf("inspect active provider service: %w", err) + } + if strings.TrimSpace(string(output)) != "active" { + return errors.New("retained provider service is not active") + } + if err := refresher.runProbe(ctx, providerProbeCommand(config, paths, config.StableContainer, selection)); err != nil { + return fmt.Errorf("probe active provider: %w", err) + } + return nil +} + func (refresher Refresher) removeContainer(ctx context.Context, config Config, name string) error { - _, err := refresher.Runner.Run(ctx, Command{Path: config.PodmanPath, Args: []string{"rm", "--force", "--ignore", name}}) + _, err := refresher.run(ctx, Command{Path: config.PodmanPath, Args: []string{"rm", "--force", "--ignore", name}}) return err } @@ -406,7 +722,7 @@ func (refresher Refresher) runProbe(ctx context.Context, command Command) error delays := []time.Duration{250 * time.Millisecond, 500 * time.Millisecond, time.Second, 2 * time.Second} var lastErr error for attempt := 0; attempt <= len(delays); attempt++ { - if _, err := refresher.Runner.Run(ctx, command); err == nil { + if _, err := refresher.run(ctx, command); err == nil { return nil } else { lastErr = err @@ -439,7 +755,18 @@ func (refresher Refresher) rollback(ctx context.Context, config Config, paths Li rollbackContext, cancelRollback := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) defer cancelRollback() var rollbackErr error - if activeChanged { + rollbackErr = errors.Join(rollbackErr, refresher.removeContainer(rollbackContext, config, config.CandidateContainer)) + providerStateRestored := true + if journal.Phase != JournalPrepared && journal.Phase != JournalCommitted { + if _, err := refresher.run(rollbackContext, Command{Path: "/usr/bin/systemctl", Args: []string{"--user", "stop", providerServiceUnit}}); err != nil { + providerStateRestored = false + rollbackErr = errors.Join(rollbackErr, err) + } else if err := restorePreviousProviderState(paths, journal.Candidate.Update.SHA256, journal.Phase); err != nil { + providerStateRestored = false + rollbackErr = errors.Join(rollbackErr, err) + } + } + if activeChanged && providerStateRestored { if journal.Previous != nil { if err := AtomicWriteJSON(paths.ActiveState, *journal.Previous); err != nil { rollbackErr = errors.Join(rollbackErr, err) @@ -450,11 +777,35 @@ func (refresher Refresher) rollback(ctx context.Context, config Config, paths Li } } else { rollbackErr = errors.Join(rollbackErr, removeDurableFile(paths.ActiveState)) - _, stopErr := refresher.Runner.Run(rollbackContext, Command{Path: "/usr/bin/systemctl", Args: []string{"--user", "stop", providerServiceUnit}}) + _, stopErr := refresher.run(rollbackContext, Command{Path: "/usr/bin/systemctl", Args: []string{"--user", "stop", providerServiceUnit}}) + rollbackErr = errors.Join(rollbackErr, stopErr) + } + } + if !activeChanged && providerStateRestored && journal.Phase != JournalPrepared && journal.Phase != JournalCommitted { + if journal.Previous != nil { + if err := refresher.restartProvider(rollbackContext); err != nil { + rollbackErr = errors.Join(rollbackErr, err) + } else if err := refresher.runProbe(rollbackContext, providerProbeCommand(config, paths, config.StableContainer, journal.Previous.Current)); err != nil { + rollbackErr = errors.Join(rollbackErr, err) + } + } else { + _, stopErr := refresher.run(rollbackContext, Command{Path: "/usr/bin/systemctl", Args: []string{"--user", "stop", providerServiceUnit}}) + rollbackErr = errors.Join(rollbackErr, stopErr) + } + } + if journal.Phase == JournalPrepared { + rollbackErr = errors.Join(rollbackErr, cleanupProviderStateTransaction(paths, journal.Candidate.Update.SHA256)) + if journal.Previous != nil { + if err := refresher.restartProvider(rollbackContext); err != nil { + rollbackErr = errors.Join(rollbackErr, err) + } else if err := refresher.runProbe(rollbackContext, providerProbeCommand(config, paths, config.StableContainer, journal.Previous.Current)); err != nil { + rollbackErr = errors.Join(rollbackErr, err) + } + } else { + _, stopErr := refresher.run(rollbackContext, Command{Path: "/usr/bin/systemctl", Args: []string{"--user", "stop", providerServiceUnit}}) rollbackErr = errors.Join(rollbackErr, stopErr) } } - rollbackErr = errors.Join(rollbackErr, refresher.removeContainer(rollbackContext, config, config.CandidateContainer)) if rollbackErr == nil { rollbackErr = removeDurableFile(paths.Journal) } @@ -472,16 +823,47 @@ func (refresher Refresher) recoverInterrupted(ctx context.Context, config Config if err := journal.Validate(); err != nil { return fmt.Errorf("validate interrupted refresh journal: %w", err) } - if journal.Previous == nil && journal.Phase != JournalCommitted { + if err := refresher.removeContainer(ctx, config, config.CandidateContainer); err != nil { + return fmt.Errorf("remove interrupted provider candidate: %w", err) + } + if journal.Phase != JournalCommitted { + if _, err := refresher.run(ctx, Command{Path: "/usr/bin/systemctl", Args: []string{"--user", "stop", providerServiceUnit}}); err != nil { + return fmt.Errorf("stop interrupted provider before state recovery: %w", err) + } + } + if journal.Phase == JournalCommitted { + if journal.DeferredCommit { + return errors.New("deferred installer refresh requires installer finalization or rollback") + } + recovered, err := RecoverActiveState(journal) + if err != nil { + return err + } + if err := AtomicWriteJSON(paths.ActiveState, recovered); err != nil { + return fmt.Errorf("write recovered active state: %w", err) + } + if err := cleanupProviderStateTransaction(paths, journal.Candidate.Update.SHA256); err != nil { + return fmt.Errorf("clean committed provider state transaction: %w", err) + } + } else if journal.Previous == nil { + if journal.Phase != JournalPrepared { + if err := restorePreviousProviderState(paths, journal.Candidate.Update.SHA256, journal.Phase); err != nil { + return fmt.Errorf("restore interrupted initial provider state: %w", err) + } + } else if err := cleanupProviderStateTransaction(paths, journal.Candidate.Update.SHA256); err != nil { + return fmt.Errorf("clean interrupted initial candidate state: %w", err) + } if err := removeDurableFile(paths.ActiveState); err != nil { return fmt.Errorf("remove interrupted initial active state: %w", err) } - if journal.Phase == JournalActivated { - if _, err := refresher.Runner.Run(ctx, Command{Path: "/usr/bin/systemctl", Args: []string{"--user", "stop", providerServiceUnit}}); err != nil { - return fmt.Errorf("stop interrupted initial provider: %w", err) + } else { + if journal.Phase != JournalPrepared { + if err := restorePreviousProviderState(paths, journal.Candidate.Update.SHA256, journal.Phase); err != nil { + return fmt.Errorf("restore interrupted provider state: %w", err) } + } else if err := cleanupProviderStateTransaction(paths, journal.Candidate.Update.SHA256); err != nil { + return fmt.Errorf("clean interrupted candidate state: %w", err) } - } else { recovered, err := RecoverActiveState(journal) if err != nil { return err @@ -489,17 +871,12 @@ func (refresher Refresher) recoverInterrupted(ctx context.Context, config Config if err := AtomicWriteJSON(paths.ActiveState, recovered); err != nil { return fmt.Errorf("write recovered active state: %w", err) } - if journal.Phase == JournalActivated { - if err := refresher.restartProvider(ctx); err != nil { - return fmt.Errorf("restart recovered provider: %w", err) - } - if err := refresher.runProbe(ctx, providerProbeCommand(config, paths, config.StableContainer, recovered.Current)); err != nil { - return fmt.Errorf("probe recovered provider: %w", err) - } + if err := refresher.restartProvider(ctx); err != nil { + return fmt.Errorf("restart recovered provider: %w", err) + } + if err := refresher.runProbe(ctx, providerProbeCommand(config, paths, config.StableContainer, recovered.Current)); err != nil { + return fmt.Errorf("probe recovered provider: %w", err) } - } - if err := refresher.removeContainer(ctx, config, config.CandidateContainer); err != nil { - return fmt.Errorf("remove interrupted provider candidate: %w", err) } if err := removeDurableFile(paths.Journal); err != nil { return fmt.Errorf("remove recovered refresh journal: %w", err) @@ -507,6 +884,167 @@ func (refresher Refresher) recoverInterrupted(ctx context.Context, config Config return nil } +func writeJournalPhase(path string, journal *TransactionJournal, phase JournalPhase, updatedAt time.Time) error { + next := *journal + next.Phase = phase + next.UpdatedAt = updatedAt + if err := AtomicWriteJSON(path, next); err != nil { + return err + } + *journal = next + return nil +} + +func (refresher Refresher) finalizeDeferredRefresh(config Config) error { + paths := LifecyclePathsFor(config) + journal, found, err := readTransactionJournal(paths.Journal) + if err != nil || !found { + return err + } + if !journal.DeferredCommit || journal.Phase != JournalCommitted { + return errors.New("retained provider journal is not a deferred committed refresh") + } + if err := cleanupProviderStateTransaction(paths, journal.Candidate.Update.SHA256); err != nil { + return err + } + return removeDurableFile(paths.Journal) +} + +func (refresher Refresher) finalizeInterruptedDeferredCommit(config Config) error { + paths := LifecyclePathsFor(config) + journal, found, err := readTransactionJournal(paths.Journal) + if err != nil || !found || !journal.DeferredCommit || journal.Phase != JournalCommitted { + return err + } + return refresher.finalizeDeferredRefresh(config) +} + +func (refresher Refresher) rollbackDeferredRefresh(ctx context.Context, config Config) error { + paths := LifecyclePathsFor(config) + journal, found, err := readTransactionJournal(paths.Journal) + if err != nil || !found { + return err + } + if !journal.DeferredCommit || journal.Phase != JournalCommitted { + return errors.New("retained provider journal is not a deferred committed refresh") + } + journal.Phase = JournalActivated + return refresher.rollback(ctx, config, paths, journal, true) +} + +func readTransactionJournal(path string) (TransactionJournal, bool, error) { + var journal TransactionJournal + if err := ReadStrictJSONFile(path, &journal); err != nil { + if errors.Is(err, os.ErrNotExist) { + return TransactionJournal{}, false, nil + } + return TransactionJournal{}, false, err + } + if err := journal.Validate(); err != nil { + return TransactionJournal{}, false, err + } + return journal, true, nil +} + +func promoteCandidateProviderState(paths LifecyclePaths, digest string) error { + candidate := paths.CandidateState(digest) + previous := paths.PreviousState(digest) + if err := validateOwnedDirectory(paths.ProviderState); err != nil { + return fmt.Errorf("validate active provider state: %w", err) + } + if err := validateOwnedDirectory(candidate); err != nil { + return fmt.Errorf("validate candidate provider state: %w", err) + } + if _, err := os.Lstat(previous); !errors.Is(err, os.ErrNotExist) { + if err == nil { + return errors.New("provider state rollback target already exists") + } + return fmt.Errorf("inspect provider state rollback target: %w", err) + } + if err := os.Rename(paths.ProviderState, previous); err != nil { + return fmt.Errorf("retain previous provider state: %w", err) + } + if err := os.Rename(candidate, paths.ProviderState); err != nil { + restoreErr := os.Rename(previous, paths.ProviderState) + return errors.Join(fmt.Errorf("activate candidate provider state: %w", err), restoreErr) + } + return errors.Join(syncDirectory(paths.Root), syncDirectory(filepath.Dir(candidate))) +} + +func restorePreviousProviderState(paths LifecyclePaths, digest string, phase JournalPhase) error { + previous := paths.PreviousState(digest) + if _, err := os.Lstat(previous); errors.Is(err, os.ErrNotExist) { + if phase != JournalStatePromoting { + return fmt.Errorf("missing previous provider state during %s recovery", phase) + } + if err := validateOwnedDirectory(paths.ProviderState); err != nil { + return fmt.Errorf("validate unpromoted provider state: %w", err) + } + if err := validateOwnedDirectory(paths.CandidateState(digest)); err != nil { + return fmt.Errorf("validate unpromoted candidate state: %w", err) + } + return cleanupProviderStateTransaction(paths, digest) + } else if err != nil { + return fmt.Errorf("inspect previous provider state: %w", err) + } + if err := validateOwnedDirectory(previous); err != nil { + return fmt.Errorf("validate previous provider state: %w", err) + } + if _, err := os.Lstat(paths.ProviderState); err == nil { + if err := removeOwnedDirectory(paths.ProviderState); err != nil { + return fmt.Errorf("remove uncommitted provider state: %w", err) + } + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("inspect uncommitted provider state: %w", err) + } + if err := os.Rename(previous, paths.ProviderState); err != nil { + return fmt.Errorf("restore previous provider state: %w", err) + } + if err := syncDirectory(paths.Root); err != nil { + return err + } + return cleanupProviderStateTransaction(paths, digest) +} + +func cleanupProviderStateTransaction(paths LifecyclePaths, digest string) error { + transactionRoot := filepath.Join(paths.CandidatesRoot, digestHex(digest)) + info, err := os.Lstat(transactionRoot) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return errors.New("provider state transaction root must be a real directory") + } + if err := validateOwner(info); err != nil { + return err + } + if err := os.RemoveAll(transactionRoot); err != nil { + return err + } + return syncDirectory(paths.CandidatesRoot) +} + +func validateOwnedDirectory(path string) error { + info, err := os.Lstat(path) + if err != nil { + return err + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return errors.New("path must be a real directory") + } + return validateOwner(info) +} + +func removeOwnedDirectory(path string) error { + if err := validateOwnedDirectory(path); err != nil { + return err + } + return os.RemoveAll(path) +} + func (refresher Refresher) now() time.Time { if refresher.Now == nil { return time.Now().UTC() @@ -514,6 +1052,10 @@ func (refresher Refresher) now() time.Time { return refresher.Now().UTC() } +func (refresher Refresher) run(ctx context.Context, command Command) ([]byte, error) { + return runBoundedCommand(ctx, refresher.Runner, command) +} + func stageVerifiedProvider(update VerifiedUpdate, paths LifecyclePaths) error { destination := paths.PackageBinary(update.SHA256) if existingDigest, err := hashRegularFile(destination, true); err == nil && existingDigest == update.SHA256 { @@ -771,13 +1313,34 @@ func providerImageRef(digest string) string { return "localhost/workflow-plugin-github-runner-provider:sha256-" + digestHex(digest) } +func normalizePodmanImageID(value string) (string, error) { + value = strings.TrimSpace(value) + if digestPattern.MatchString(value) { + return value, nil + } + if len(value) == 64 { + candidate := "sha256:" + value + if digestPattern.MatchString(candidate) { + return candidate, nil + } + } + return "", errors.New("image id must be a lowercase SHA-256 digest") +} + func digestHex(digest string) string { return strings.TrimPrefix(digest, "sha256:") } func removeDurableFile(path string) error { - if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { - return err + if err := os.Remove(path); err != nil { + if !errors.Is(err, os.ErrNotExist) { + return err + } + if _, parentErr := os.Lstat(filepath.Dir(path)); errors.Is(parentErr, os.ErrNotExist) { + return nil + } else if parentErr != nil { + return parentErr + } } return syncDirectory(filepath.Dir(path)) } diff --git a/internal/retainedprovider/refresh_test.go b/internal/retainedprovider/refresh_test.go index 255e19a..f68c3b6 100644 --- a/internal/retainedprovider/refresh_test.go +++ b/internal/retainedprovider/refresh_test.go @@ -127,6 +127,200 @@ func TestInitialRefreshRequiresInstallerDigestMatch(t *testing.T) { } } +func TestRefreshBoundsEverySubprocessContext(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + payload := writeTestProviderPayload(t, home, "verified-provider-bounded-refresh-commands") + digest := fileDigestForTest(t, payload) + runner := refreshTestRunner(config, payload, digest) + baseRun := runner.run + var unbounded []Command + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if _, bounded := ctx.Deadline(); !bounded { + unbounded = append(unbounded, command) + } + return baseRun(ctx, command) + } + refresher := Refresher{Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, Sleep: func(context.Context, time.Duration) error { return nil }} + if _, err := refresher.Refresh(t.Context(), config); err != nil { + t.Fatalf("refresh: %v", err) + } + if len(unbounded) != 0 { + t.Fatalf("refresh issued unbounded subprocesses: %s", commandTranscript(unbounded)) + } +} + +func TestRefreshFencesAgentDuringProviderMutation(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + payload := writeTestProviderPayload(t, home, "verified-provider-fenced-refresh") + digest := fileDigestForTest(t, payload) + runner := refreshTestRunner(config, payload, digest) + baseRun := runner.run + statuses := []string{"unavailable", "unavailable", "idle"} + var events []string + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + switch installCommandEvent(command, config) { + case "maintenance-begin": + journal, found, err := readLifecycleJournal(home, paths) + if err != nil || !found || journal.Phase != LifecycleFencing { + t.Fatalf("maintenance begin lifecycle journal = %+v found=%v err=%v", journal, found, err) + } + events = append(events, "maintenance-begin") + return maintenanceStateJSON(true, "workflow-plugin-github-retained-provider-refresh", config.ProfileID, "workflow-plugin-github-retained-provider-refresh"), nil + case "maintenance-status": + events = append(events, "maintenance-status") + return maintenanceStateJSON(true, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + case "maintenance-end": + journal, found, err := readLifecycleJournal(home, paths) + if err != nil || !found || journal.Phase != LifecycleReleasing || journal.Outcome != LifecycleCommit || journal.ProviderTransaction == nil { + t.Fatalf("maintenance end lifecycle journal = %+v found=%v err=%v", journal, found, err) + } + inner, innerFound, err := readTransactionJournal(paths.Journal) + if err != nil || !innerFound || inner.Phase != JournalCommitted || inner.OuterTransactionID != journal.TransactionID || inner.ProfileID != config.ProfileID { + t.Fatalf("maintenance end provider journal = %+v found=%v err=%v", inner, innerFound, err) + } + events = append(events, "maintenance-end") + return maintenanceStateJSON(false, "workflow-plugin-github-retained-provider-refresh", config.ProfileID, "workflow-plugin-github-retained-provider-refresh"), nil + case "local-status": + if len(statuses) == 0 { + t.Fatal("unexpected extra local status read") + } + state := statuses[0] + statuses = statuses[1:] + events = append(events, "local-"+state) + return localStatusJSON(config.WorkerID, state), nil + case "agent-stop": + journal, found, err := readLifecycleJournal(home, paths) + if err != nil || !found || journal.Phase != LifecycleFenced { + t.Fatalf("agent stop lifecycle journal = %+v found=%v err=%v", journal, found, err) + } + events = append(events, "agent-stop") + case "agent-start": + events = append(events, "agent-start") + } + if isCandidateStart(command, config) { + events = append(events, "candidate-start") + } + return baseRun(ctx, command) + } + refresher := Refresher{Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, Sleep: func(context.Context, time.Duration) error { return nil }} + if _, err := refresher.Refresh(t.Context(), config); err != nil { + t.Fatalf("refresh: %v", err) + } + assertOrderedEvents(t, events, []string{ + "maintenance-begin", "local-unavailable", "agent-stop", "candidate-start", + "agent-start", "local-unavailable", "maintenance-end", "local-idle", + }) + if _, found, err := readLifecycleJournal(home, paths); err != nil || found { + t.Fatalf("completed refresh lifecycle journal found=%v err=%v", found, err) + } + if _, found, err := readTransactionJournal(paths.Journal); err != nil || found { + t.Fatalf("completed refresh provider journal found=%v err=%v", found, err) + } +} + +func TestSameDigestRefreshHealthCheckDoesNotFenceAgent(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + payload := writeTestProviderPayload(t, home, "verified-provider-same-digest") + digest := fileDigestForTest(t, payload) + now := time.Unix(1_700_000_000, 0).UTC() + selection := selectionForDigest(payload, digest, "v1.0.31", "directive-same", testProviderImageID, now) + active := ActiveState{ProtocolVersion: ActiveStateProtocolVersion, Current: selection, UpdatedAt: now} + if err := AtomicWriteJSON(paths.ActiveState, active); err != nil { + t.Fatalf("write active state: %v", err) + } + runner := refreshTestRunner(config, payload, digest) + originalRun := runner.run + sawJournaledProbe := false + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if isProbeFor(command, config.StableContainer) { + journal, found, err := readLifecycleJournal(home, paths) + if err != nil || !found || journal.Operation != LifecycleRefresh || journal.Phase != LifecycleIntent || journal.ProviderEffect != ProviderUnchanged || journal.Unchanged == nil || journal.Unchanged.Active.Update.SHA256 != digest || journal.Unchanged.Candidate.SHA256 != digest || !journal.Unchanged.StableProbeAt.IsZero() { + t.Fatalf("same-digest probe lifecycle = %+v found=%v err=%v", journal, found, err) + } + sawJournaledProbe = true + } + return originalRun(ctx, command) + } + refresher := Refresher{Runner: runner, Sleep: func(context.Context, time.Duration) error { return nil }} + if _, err := refresher.Refresh(t.Context(), config); err != nil { + t.Fatalf("same-digest refresh: %v", err) + } + transcript := commandTranscript(runner.commands) + for _, forbidden := range []string{"supervisor-maintenance", "stop " + config.AgentUnit, "start " + config.AgentUnit} { + if strings.Contains(transcript, forbidden) { + t.Fatalf("same-digest health check fenced agent with %q:\n%s", forbidden, transcript) + } + } + if !sawJournaledProbe { + t.Fatal("same-digest stable probe did not carry lifecycle provenance") + } + if _, found, err := readLifecycleJournal(home, paths); err != nil || found { + t.Fatalf("completed same-digest journal found=%v err=%v", found, err) + } +} + +func TestNormalizePodmanImageIDCanonicalizesOnlyImmutableSHA256(t *testing.T) { + hexDigest := strings.Repeat("a", 64) + for _, input := range []string{hexDigest, "sha256:" + hexDigest, "\n" + hexDigest + "\n"} { + got, err := normalizePodmanImageID(input) + if err != nil || got != "sha256:"+hexDigest { + t.Fatalf("normalize %q = %q err=%v", input, got, err) + } + } + for _, input := range []string{"", strings.Repeat("a", 63), strings.Repeat("A", 64), "sha512:" + hexDigest, hexDigest + " extra"} { + if got, err := normalizePodmanImageID(input); err == nil { + t.Fatalf("normalize invalid %q = %q", input, got) + } + } +} + +func TestRefreshRejectsProviderNetworkWithoutDNSBeforeBuild(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + payload := writeTestProviderPayload(t, home, "verified-provider-network") + digest := fileDigestForTest(t, payload) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + runner := refreshTestRunner(config, payload, digest) + baseRun := runner.run + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if command.Path == config.PodmanPath && containsAdjacentArgs(command.Args, "network", "inspect") { + return []byte("bridge false false\n"), nil + } + return baseRun(ctx, command) + } + refresher := Refresher{Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }} + if _, err := refresher.Refresh(t.Context(), config); err == nil || !strings.Contains(err.Error(), "DNS") { + t.Fatalf("refresh network err = %v", err) + } + for _, command := range runner.commands { + if command.Path == config.PodmanPath && firstArg(command.Args) == "build" { + t.Fatalf("refresh built image before network validation: %+v", runner.commands) + } + } +} + func TestRefreshBuildsAndPreflightsIsolatedCandidateThenStable(t *testing.T) { home := t.TempDir() config := validTestConfig(home) @@ -141,6 +335,23 @@ func TestRefreshBuildsAndPreflightsIsolatedCandidateThenStable(t *testing.T) { t.Fatalf("write provider state: %v", err) } runner := refreshTestRunner(config, payload, digest) + baseRun := runner.run + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if filepath.Base(command.Path) == "systemctl" && containsAdjacentArgs(command.Args, "--user", "stop") && containsArg(command.Args, providerServiceUnit) { + if err := os.WriteFile(filepath.Join(paths.ProviderState, "ownership.json"), []byte(`{"owner":"quiesced"}`), 0o600); err != nil { + t.Fatalf("write quiesced provider state: %v", err) + } + } + if isCandidateStart(command, config) { + if data, err := os.ReadFile(filepath.Join(paths.CandidateState(digest), "ownership.json")); err != nil || string(data) != `{"owner":"quiesced"}` { + t.Fatalf("candidate cloned live state before quiesce: data=%q err=%v", data, err) + } + if err := os.WriteFile(filepath.Join(paths.CandidateState(digest), "ownership.json"), []byte(`{"owner":"migrated"}`), 0o600); err != nil { + t.Fatalf("mutate candidate state: %v", err) + } + } + return baseRun(ctx, command) + } now := time.Unix(1_700_000_000, 0).UTC() refresher := Refresher{ Runner: runner, @@ -161,8 +372,11 @@ func TestRefreshBuildsAndPreflightsIsolatedCandidateThenStable(t *testing.T) { if active.Current.Update.SHA256 != digest || active.Current.ImageID != testProviderImageID { t.Fatalf("active state = %+v", active) } - if _, err := os.Stat(filepath.Join(paths.CandidateState(digest), "ownership.json")); err != nil { - t.Fatalf("candidate did not receive bounded state clone: %v", err) + if data, err := os.ReadFile(filepath.Join(paths.ProviderState, "ownership.json")); err != nil || string(data) != `{"owner":"migrated"}` { + t.Fatalf("candidate state was not promoted: data=%q err=%v", data, err) + } + if _, err := os.Stat(paths.CandidateState(digest)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("promoted candidate state remains at staging path: %v", err) } assertRefreshCommandIsolation(t, runner.commands, config, paths) @@ -171,10 +385,120 @@ func TestRefreshBuildsAndPreflightsIsolatedCandidateThenStable(t *testing.T) { if err != nil || status.CurrentSHA256 != digest { t.Fatalf("idempotent refresh status=%+v err=%v", status, err) } - for _, command := range runner.commands[before:] { - if command.Path == config.PodmanPath || filepath.Base(command.Path) == "systemctl" { - t.Fatalf("digest-idempotent refresh mutated runtime: %+v", command) + idempotentCommands := runner.commands[before:] + idempotentTranscript := commandTranscript(idempotentCommands) + for _, required := range []string{ + "systemctl --user show " + providerServiceUnit + " --property ActiveState --value", + "probe -url " + config.ProviderURL, + } { + if !strings.Contains(idempotentTranscript, required) { + t.Fatalf("digest-idempotent refresh skipped health check %q:\n%s", required, idempotentTranscript) + } + } + for _, forbidden := range []string{"podman build", "systemctl --user restart", config.CandidateContainer + " "} { + if strings.Contains(idempotentTranscript, forbidden) { + t.Fatalf("digest-idempotent refresh mutated runtime with %q:\n%s", forbidden, idempotentTranscript) + } + } +} + +func TestDigestIdempotentRefreshFailsWhenStableServiceIsInactive(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + payload := writeTestProviderPayload(t, home, "verified-provider-idempotent-health") + digest := fileDigestForTest(t, payload) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + active := ActiveState{ + ProtocolVersion: ActiveStateProtocolVersion, + Current: selectionForDigest(payload, digest, "v1.0.32", "directive-current", testProviderImageID, time.Unix(1_700_000_000, 0).UTC()), + UpdatedAt: time.Unix(1_700_000_000, 0).UTC(), + } + if err := AtomicWriteJSON(paths.ActiveState, active); err != nil { + t.Fatalf("write active state: %v", err) + } + runner := refreshTestRunner(config, payload, digest) + baseRun := runner.run + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if filepath.Base(command.Path) == "systemctl" && containsAdjacentArgs(command.Args, "--property", "ActiveState") { + return []byte("inactive\n"), nil } + return baseRun(ctx, command) + } + refresher := Refresher{Runner: runner} + if _, err := refresher.Refresh(t.Context(), config); err == nil || !strings.Contains(err.Error(), "not active") { + t.Fatalf("idempotent inactive refresh err = %v", err) + } + if strings.Contains(commandTranscript(runner.commands), "podman build") { + t.Fatalf("inactive idempotent refresh rebuilt unchanged image:\n%s", commandTranscript(runner.commands)) + } +} + +func TestDeferredRefreshRetainsRollbackStateUntilInstallerFinalizes(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + previous := previousActiveStateForTest(t, home) + if err := AtomicWriteJSON(paths.ActiveState, previous); err != nil { + t.Fatalf("write previous active state: %v", err) + } + payload := writeTestProviderPayload(t, home, "verified-provider-deferred-refresh") + digest := fileDigestForTest(t, payload) + runner := refreshTestRunner(config, payload, digest) + refresher := Refresher{Runner: runner, Sleep: func(context.Context, time.Duration) error { return nil }} + if _, err := refresher.refreshUnderLifecycleLock(t.Context(), config, false, true); err != nil { + t.Fatalf("deferred refresh: %v", err) + } + journal, found, err := readTransactionJournal(paths.Journal) + if err != nil || !found || journal.Phase != JournalCommitted || !journal.DeferredCommit { + t.Fatalf("deferred journal = %+v found=%v err=%v", journal, found, err) + } + if _, err := os.Stat(paths.PreviousState(digest)); err != nil { + t.Fatalf("deferred refresh removed rollback state: %v", err) + } + if err := refresher.finalizeDeferredRefresh(config); err != nil { + t.Fatalf("finalize deferred refresh: %v", err) + } + if _, err := os.Stat(paths.Journal); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("finalized journal remains: %v", err) + } + if _, err := os.Stat(filepath.Join(paths.CandidatesRoot, digestHex(digest))); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("finalized rollback state remains: %v", err) + } +} + +func TestDeferredRefreshBindsOuterLifecycleTransaction(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + previous := previousActiveStateForTest(t, home) + if err := AtomicWriteJSON(paths.ActiveState, previous); err != nil { + t.Fatalf("write previous active state: %v", err) + } + payload := writeTestProviderPayload(t, home, "verified-provider-bound-refresh") + digest := fileDigestForTest(t, payload) + runner := refreshTestRunner(config, payload, digest) + refresher := Refresher{Runner: runner, Sleep: func(context.Context, time.Duration) error { return nil }} + if _, err := refresher.refreshUnderLifecycleTransaction(t.Context(), config, false, true, "install-transaction-123", config.ProfileID, ""); err != nil { + t.Fatalf("bound deferred refresh: %v", err) + } + journal, found, err := readTransactionJournal(paths.Journal) + if err != nil || !found { + t.Fatalf("read bound provider journal found=%v err=%v", found, err) + } + if journal.OuterTransactionID != "install-transaction-123" || journal.ProfileID != config.ProfileID || !journal.DeferredCommit { + t.Fatalf("provider transaction binding = %+v", journal) } } @@ -245,7 +569,7 @@ func TestRefreshRejectsIncompleteProviderEnvironment(t *testing.T) { } func TestRefreshFailurePreservesPreviousActiveImageAndCleansCandidate(t *testing.T) { - for _, phase := range []string{"build", "stale-candidate", "candidate", "candidate-probe", "stable-restart", "stable-probe", "canceled"} { + for _, phase := range []string{"build", "stale-candidate", "stable-stop", "candidate", "candidate-probe", "stable-restart", "stable-probe", "canceled"} { t.Run(phase, func(t *testing.T) { home := t.TempDir() config := validTestConfig(home) @@ -287,7 +611,11 @@ func TestRefreshFailurePreservesPreviousActiveImageAndCleansCandidate(t *testing if phase == "candidate-probe" && isProbeFor(command, config.CandidateContainer) { return nil, errors.New("candidate probe failed") } - if phase == "stable-restart" && filepath.Base(command.Path) == "systemctl" && !failedRestart { + if phase == "stable-stop" && filepath.Base(command.Path) == "systemctl" && containsAdjacentArgs(command.Args, "stop", providerServiceUnit) && !failedRestart { + failedRestart = true + return nil, errors.New("stop failed") + } + if phase == "stable-restart" && filepath.Base(command.Path) == "systemctl" && containsAdjacentArgs(command.Args, "restart", providerServiceUnit) && !failedRestart { failedRestart = true return nil, errors.New("restart failed") } @@ -324,8 +652,105 @@ func TestRefreshFailurePreservesPreviousActiveImageAndCleansCandidate(t *testing } } +func TestStableProbeFailureRestoresPreviousProviderState(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + stateFile := filepath.Join(paths.ProviderState, "state.json") + if err := os.WriteFile(stateFile, []byte(`{"generation":"previous"}`), 0o600); err != nil { + t.Fatalf("write previous provider state: %v", err) + } + previous := previousActiveStateForTest(t, home) + if err := AtomicWriteJSON(paths.ActiveState, previous); err != nil { + t.Fatalf("write previous active state: %v", err) + } + payload := writeTestProviderPayload(t, home, "verified-provider-state-rollback") + digest := fileDigestForTest(t, payload) + runner := refreshTestRunner(config, payload, digest) + baseRun := runner.run + providerStops := 0 + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if filepath.Base(command.Path) == "systemctl" && containsAdjacentArgs(command.Args, "--user", "stop") && containsArg(command.Args, providerServiceUnit) { + providerStops++ + } + if isCandidateStart(command, config) { + if err := os.WriteFile(filepath.Join(paths.CandidateState(digest), "state.json"), []byte(`{"generation":"candidate"}`), 0o600); err != nil { + t.Fatalf("mutate candidate state: %v", err) + } + } + if isProbeFor(command, config.StableContainer) && containsArg(command.Args, testProviderImageID) { + return nil, errors.New("stable probe failed") + } + return baseRun(ctx, command) + } + refresher := Refresher{Runner: runner, Sleep: func(context.Context, time.Duration) error { return nil }} + if _, err := refresher.Refresh(t.Context(), config); err == nil || !strings.Contains(err.Error(), "stable probe failed") { + t.Fatalf("stable probe failure err = %v", err) + } + if data, err := os.ReadFile(stateFile); err != nil || string(data) != `{"generation":"previous"}` { + t.Fatalf("rollback state = %q err=%v", data, err) + } + if _, err := os.Stat(filepath.Join(paths.CandidatesRoot, digestHex(digest))); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("state transaction remains after rollback: %v", err) + } + if providerStops != 2 { + t.Fatalf("provider stop count = %d want promotion and rollback stops", providerStops) + } +} + +func TestCommitJournalWriteFailureRollsBackLastDurablePhase(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + stateFile := filepath.Join(paths.ProviderState, "state.json") + if err := os.WriteFile(stateFile, []byte(`{"generation":"previous"}`), 0o600); err != nil { + t.Fatalf("write previous provider state: %v", err) + } + previous := previousActiveStateForTest(t, home) + if err := AtomicWriteJSON(paths.ActiveState, previous); err != nil { + t.Fatalf("write previous active state: %v", err) + } + payload := writeTestProviderPayload(t, home, "verified-provider-commit-journal-failure") + digest := fileDigestForTest(t, payload) + runner := refreshTestRunner(config, payload, digest) + baseRun := runner.run + blockedCommit := false + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if isCandidateStart(command, config) { + if err := os.WriteFile(filepath.Join(paths.CandidateState(digest), "state.json"), []byte(`{"generation":"candidate"}`), 0o600); err != nil { + t.Fatalf("mutate candidate state: %v", err) + } + } + if isProbeFor(command, config.StableContainer) && !blockedCommit { + blockedCommit = true + if err := os.Remove(paths.Journal); err != nil { + t.Fatalf("remove journal before commit: %v", err) + } + if err := os.Mkdir(paths.Journal, 0o700); err != nil { + t.Fatalf("block journal commit: %v", err) + } + } + return baseRun(ctx, command) + } + refresher := Refresher{Runner: runner, Sleep: func(context.Context, time.Duration) error { return nil }} + if _, err := refresher.Refresh(t.Context(), config); err == nil { + t.Fatal("refresh with failed commit-journal write succeeded") + } + if data, err := os.ReadFile(stateFile); err != nil || string(data) != `{"generation":"previous"}` { + t.Fatalf("commit-write rollback state = %q err=%v", data, err) + } +} + func TestRefreshRecoversEveryInterruptedJournalPhaseIdempotently(t *testing.T) { - for _, phase := range []JournalPhase{JournalPrepared, JournalActivated, JournalCommitted} { + for _, phase := range []JournalPhase{JournalPrepared, JournalStatePromoting, JournalStatePromoted, JournalActivated, JournalCommitted} { t.Run(string(phase), func(t *testing.T) { home := t.TempDir() config := validTestConfig(home) @@ -334,10 +759,29 @@ func TestRefreshRecoversEveryInterruptedJournalPhaseIdempotently(t *testing.T) { if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { t.Fatalf("mkdir provider state: %v", err) } + if err := os.WriteFile(filepath.Join(paths.ProviderState, "generation"), []byte("previous"), 0o600); err != nil { + t.Fatalf("write previous provider state: %v", err) + } previous := previousActiveStateForTest(t, home) candidatePayload := writeTestProviderPayload(t, home, "candidate-recovery") candidateDigest := fileDigestForTest(t, candidatePayload) candidate := selectionForDigest(candidatePayload, candidateDigest, "v1.0.32", "directive-candidate", "sha256:"+strings.Repeat("e", 64), time.Unix(1_700_000_100, 0).UTC()) + if err := prepareCandidateState(paths.ProviderState, paths.CandidateState(candidateDigest)); err != nil { + t.Fatalf("prepare candidate state: %v", err) + } + if err := os.WriteFile(filepath.Join(paths.CandidateState(candidateDigest), "generation"), []byte("candidate"), 0o600); err != nil { + t.Fatalf("write candidate provider state: %v", err) + } + switch phase { + case JournalStatePromoting: + if err := os.Rename(paths.ProviderState, paths.PreviousState(candidateDigest)); err != nil { + t.Fatalf("simulate partial provider state promotion: %v", err) + } + case JournalStatePromoted, JournalActivated, JournalCommitted: + if err := promoteCandidateProviderState(paths, candidateDigest); err != nil { + t.Fatalf("simulate provider state promotion: %v", err) + } + } journal := TransactionJournal{ ProtocolVersion: TransactionJournalProtocolVersion, ID: "refresh-recovery", @@ -372,6 +816,13 @@ func TestRefreshRecoversEveryInterruptedJournalPhaseIdempotently(t *testing.T) { if active.Current.ImageID != wantImage { t.Fatalf("%s recovered image = %s want %s", phase, active.Current.ImageID, wantImage) } + wantGeneration := "previous" + if phase == JournalCommitted { + wantGeneration = "candidate" + } + if data, err := os.ReadFile(filepath.Join(paths.ProviderState, "generation")); err != nil || string(data) != wantGeneration { + t.Fatalf("%s recovered provider state = %q err=%v want %q", phase, data, err, wantGeneration) + } if _, err := os.Stat(paths.Journal); !errors.Is(err, os.ErrNotExist) { t.Fatalf("%s journal remains after recovery: %v", phase, err) } @@ -379,6 +830,94 @@ func TestRefreshRecoversEveryInterruptedJournalPhaseIdempotently(t *testing.T) { } } +func TestInterruptedRefreshStopsCandidateBeforeDeletingStagedState(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + previous := previousActiveStateForTest(t, home) + payload := writeTestProviderPayload(t, home, "candidate-recovery-order") + digest := fileDigestForTest(t, payload) + candidate := selectionForDigest(payload, digest, "v1.0.32", "directive-candidate-order", "sha256:"+strings.Repeat("e", 64), time.Unix(1_700_000_100, 0).UTC()) + if err := prepareCandidateState(paths.ProviderState, paths.CandidateState(digest)); err != nil { + t.Fatalf("prepare candidate state: %v", err) + } + journal := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, + ID: "refresh-recovery-order", + Phase: JournalPrepared, + Previous: &previous, + Candidate: candidate, + StartedAt: time.Unix(1_700_000_100, 0).UTC(), + UpdatedAt: time.Unix(1_700_000_101, 0).UTC(), + } + if err := AtomicWriteJSON(paths.Journal, journal); err != nil { + t.Fatalf("write journal: %v", err) + } + runner := &recordingCommandRunner{run: func(_ context.Context, command Command) ([]byte, error) { + if firstArg(command.Args) == "rm" && containsArg(command.Args, config.CandidateContainer) { + if _, err := os.Stat(paths.CandidateState(digest)); err != nil { + return nil, errors.New("candidate state deleted before container stop") + } + } + return nil, nil + }} + if err := (Refresher{Runner: runner}).recoverInterrupted(t.Context(), config, paths); err != nil { + t.Fatalf("recover interrupted candidate: %v", err) + } +} + +func TestInterruptedRecoveryStopsStableBeforeProviderStateRestore(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + if err := os.WriteFile(filepath.Join(paths.ProviderState, "generation"), []byte("previous"), 0o600); err != nil { + t.Fatalf("write previous provider state: %v", err) + } + previous := previousActiveStateForTest(t, home) + payload := writeTestProviderPayload(t, home, "candidate-recovery-stop-order") + digest := fileDigestForTest(t, payload) + candidate := selectionForDigest(payload, digest, "v1.0.32", "directive-candidate-stop-order", "sha256:"+strings.Repeat("e", 64), time.Unix(1_700_000_100, 0).UTC()) + if err := prepareCandidateState(paths.ProviderState, paths.CandidateState(digest)); err != nil { + t.Fatalf("prepare candidate state: %v", err) + } + if err := promoteCandidateProviderState(paths, digest); err != nil { + t.Fatalf("promote candidate state: %v", err) + } + journal := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, + ID: "refresh-recovery-stop-order", + Phase: JournalStatePromoted, + Previous: &previous, + Candidate: candidate, + StartedAt: time.Unix(1_700_000_100, 0).UTC(), + UpdatedAt: time.Unix(1_700_000_101, 0).UTC(), + } + if err := AtomicWriteJSON(paths.Journal, journal); err != nil { + t.Fatalf("write journal: %v", err) + } + if err := os.Chmod(paths.Root, 0o500); err != nil { + t.Fatalf("restrict provider root: %v", err) + } + t.Cleanup(func() { _ = os.Chmod(paths.Root, 0o700) }) + runner := &recordingCommandRunner{run: func(_ context.Context, command Command) ([]byte, error) { + if filepath.Base(command.Path) == "systemctl" && containsAdjacentArgs(command.Args, "--user", "stop") && containsArg(command.Args, providerServiceUnit) { + if err := os.Chmod(paths.Root, 0o700); err != nil { + t.Fatalf("unlock provider root after stop: %v", err) + } + } + return nil, nil + }} + if err := (Refresher{Runner: runner}).recoverInterrupted(t.Context(), config, paths); err != nil { + t.Fatalf("recover interrupted provider state: %v", err) + } +} + func TestServeActiveValidatesImmutableImageThenExecsRestrictedPodman(t *testing.T) { home := t.TempDir() config := validTestConfig(home) @@ -395,7 +934,7 @@ func TestServeActiveValidatesImmutableImageThenExecsRestrictedPodman(t *testing. runner := &recordingCommandRunner{ run: func(_ context.Context, command Command) ([]byte, error) { if command.Path == config.PodmanPath && len(command.Args) > 1 && command.Args[0] == "image" { - return []byte(active.Current.ImageID + "\n"), nil + return []byte(strings.TrimPrefix(active.Current.ImageID, "sha256:") + "\n"), nil } return nil, nil }, @@ -413,7 +952,7 @@ func TestServeActiveValidatesImmutableImageThenExecsRestrictedPodman(t *testing. t.Fatalf("serve active exec command = %+v", execCommand) } transcript := commandTranscript(runner.commands) - for _, required := range []string{"--network bridge", "--read-only", "--cap-drop all", "no-new-privileges", active.Current.ImageID} { + for _, required := range []string{"--network wfcompute-github-provider", "--read-only", "--cap-drop all", "no-new-privileges", active.Current.ImageID} { if !strings.Contains(transcript, required) { t.Fatalf("serve active transcript missing %q:\n%s", required, transcript) } @@ -612,6 +1151,94 @@ func TestRollbackDoesNotRestartWhenDurableRestoreFails(t *testing.T) { } } +func TestRollbackDoesNotRestartWhenProviderStateRestoreFails(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + previous := previousActiveStateForTest(t, home) + if err := AtomicWriteJSON(paths.ActiveState, previous); err != nil { + t.Fatalf("write active state: %v", err) + } + payload := writeTestProviderPayload(t, home, "verified-provider-state-restore-failure") + digest := fileDigestForTest(t, payload) + runner := refreshTestRunner(config, payload, digest) + baseRun := runner.run + restarts := 0 + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if filepath.Base(command.Path) == "systemctl" && containsArg(command.Args, "restart") { + restarts++ + } + if isProbeFor(command, config.StableContainer) && containsArg(command.Args, testProviderImageID) { + previousState := paths.PreviousState(digest) + if err := os.RemoveAll(previousState); err != nil { + t.Fatalf("remove previous provider state: %v", err) + } + if err := os.Symlink(filepath.Join(home, "outside-provider-state"), previousState); err != nil { + t.Fatalf("poison previous provider state: %v", err) + } + return nil, errors.New("stable probe failed") + } + return baseRun(ctx, command) + } + refresher := Refresher{Runner: runner, Sleep: func(context.Context, time.Duration) error { return nil }} + if _, err := refresher.Refresh(t.Context(), config); err == nil || !strings.Contains(err.Error(), "real directory") { + t.Fatalf("provider state restore failure err = %v", err) + } + if restarts != 1 { + t.Fatalf("provider restarted %d times after provider state restore failure", restarts) + } +} + +func TestInterruptedPromotedStateFailsClosedWhenRollbackStateIsMissing(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + stateFile := filepath.Join(paths.ProviderState, "generation") + if err := os.WriteFile(stateFile, []byte("candidate"), 0o600); err != nil { + t.Fatalf("write promoted provider state: %v", err) + } + previous := previousActiveStateForTest(t, home) + if err := AtomicWriteJSON(paths.ActiveState, previous); err != nil { + t.Fatalf("write previous active state: %v", err) + } + payload := writeTestProviderPayload(t, home, "verified-provider-missing-rollback") + digest := fileDigestForTest(t, payload) + now := time.Unix(1_700_400_000, 0).UTC() + candidate := selectionForDigest(payload, digest, "v1.0.32", "directive-missing-rollback", "sha256:"+strings.Repeat("d", 64), now) + journal := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, + ID: "refresh-missing-rollback", + Phase: JournalStatePromoted, + Previous: &previous, + Candidate: candidate, + StartedAt: now, + UpdatedAt: now, + } + if err := AtomicWriteJSON(paths.Journal, journal); err != nil { + t.Fatalf("write interrupted journal: %v", err) + } + runner := refreshTestRunner(config, payload, digest) + refresher := Refresher{Runner: runner, Sleep: func(context.Context, time.Duration) error { return nil }} + err := refresher.recoverInterrupted(t.Context(), config, paths) + if err == nil || !strings.Contains(err.Error(), "missing previous provider state") { + t.Fatalf("missing rollback recovery err = %v", err) + } + if data, readErr := os.ReadFile(stateFile); readErr != nil || string(data) != "candidate" { + t.Fatalf("ambiguous provider state changed: data=%q err=%v", data, readErr) + } + if _, statErr := os.Stat(paths.Journal); statErr != nil { + t.Fatalf("ambiguous recovery removed journal: %v", statErr) + } +} + func TestCommittedCleanupFailureLeavesRecoverableJournal(t *testing.T) { home := t.TempDir() config := validTestConfig(home) @@ -624,18 +1251,17 @@ func TestCommittedCleanupFailureLeavesRecoverableJournal(t *testing.T) { digest := fileDigestForTest(t, payload) runner := refreshTestRunner(config, payload, digest) baseRun := runner.run - cleanupCalls := 0 runner.run = func(ctx context.Context, command Command) ([]byte, error) { - if command.Path == config.PodmanPath && firstArg(command.Args) == "rm" && containsArg(command.Args, config.CandidateContainer) { - cleanupCalls++ - if cleanupCalls == 2 { - return nil, errors.New("candidate cleanup failed") + if isProbeFor(command, config.StableContainer) { + if err := os.Chmod(paths.CandidatesRoot, 0o500); err != nil { + t.Fatalf("restrict candidate cleanup root: %v", err) } } return baseRun(ctx, command) } + t.Cleanup(func() { _ = os.Chmod(paths.CandidatesRoot, 0o700) }) refresher := Refresher{Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }} - if _, err := refresher.Refresh(t.Context(), config); err == nil || !strings.Contains(err.Error(), "candidate") { + if _, err := refresher.Refresh(t.Context(), config); err == nil || !strings.Contains(err.Error(), "rollback target") { t.Fatalf("cleanup failure err = %v", err) } var journal TransactionJournal @@ -678,15 +1304,52 @@ func TestOSCommandRunnerDoesNotInheritUnrelatedHostSecrets(t *testing.T) { const testProviderImageID = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" func refreshTestRunner(config Config, payload, digest string) *recordingCommandRunner { + for _, file := range []struct { + path string + mode os.FileMode + data string + }{ + {path: config.ComputeAgentPath, mode: 0o700, data: "compute-agent fixture"}, + {path: config.SupervisorConfigPath, mode: 0o600, data: "supervisor config fixture"}, + {path: agentUnitFragmentPathForTest(config), mode: 0o600, data: "[Service]\nExecStart=" + config.ComputeAgentPath + " run\n"}, + } { + if err := os.MkdirAll(filepath.Dir(file.path), 0o700); err != nil { + panic(err) + } + if err := os.WriteFile(file.path, []byte(file.data), file.mode); err != nil { + panic(err) + } + } + maintenanceActive := false return &recordingCommandRunner{run: func(ctx context.Context, command Command) ([]byte, error) { if err := ctx.Err(); err != nil { return nil, err } + switch installCommandEvent(command, config) { + case "agent-signature": + return agentUnitSystemdOutput(config) + case "maintenance-begin": + maintenanceActive = true + return maintenanceStateJSON(true, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + case "maintenance-end": + maintenanceActive = false + return maintenanceStateJSON(false, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + case "local-status": + state := "idle" + if maintenanceActive { + state = "unavailable" + } + return localStatusJSON(config.WorkerID, state), nil + } switch { case command.Path == config.ComputeAgentPath: return testVerifiedUpdateJSON(config, payload, digest), nil case command.Path == config.PodmanPath && len(command.Args) >= 2 && command.Args[0] == "image" && command.Args[1] == "inspect": return []byte(testProviderImageID + "\n"), nil + case command.Path == config.PodmanPath && len(command.Args) >= 2 && command.Args[0] == "network" && command.Args[1] == "inspect": + return []byte("bridge true false\n"), nil + case filepath.Base(command.Path) == "systemctl" && containsAdjacentArgs(command.Args, "--property", "ActiveState"): + return []byte("active\n"), nil default: return nil, nil } @@ -698,9 +1361,11 @@ func assertRefreshCommandIsolation(t *testing.T, commands []Command, config Conf transcript := commandTranscript(commands) for _, required := range []string{ "build", "FROM scratch", config.CandidateContainer, config.StableContainer, - "--network bridge", "--read-only", "--cap-drop all", "no-new-privileges", + "network inspect --format {{.Driver}} {{.DNSEnabled}} {{.Internal}} wfcompute-github-provider", + "--network wfcompute-github-provider", "--read-only", "--cap-drop all", "no-new-privileges", "--env-file " + paths.ProviderEnv, "--env-file " + paths.ProbeEnv, - "probe", "systemctl --user restart", + "probe -url https://" + config.CandidateContainer + ":18090", "probe -url " + config.ProviderURL, + "systemctl --user restart", } { if !strings.Contains(transcript, required) { t.Fatalf("command transcript missing %q:\n%s", required, transcript) @@ -709,6 +1374,9 @@ func assertRefreshCommandIsolation(t *testing.T, commands []Command, config Conf if strings.Contains(transcript, "provider-secret") || strings.Contains(transcript, "github-secret") || strings.Contains(transcript, "/var/run/docker.sock") || strings.Contains(transcript, "/run/podman/podman.sock") { t.Fatalf("command transcript leaked a secret or mounted a runtime socket:\n%s", transcript) } + if strings.Contains(transcript, "--publish") { + t.Fatalf("command transcript exposed a host port:\n%s", transcript) + } probeCommands := 0 for _, command := range commands { if command.Path != config.PodmanPath || !containsArg(command.Args, "probe") { diff --git a/internal/retainedprovider/state.go b/internal/retainedprovider/state.go index 894fbc6..6debeef 100644 --- a/internal/retainedprovider/state.go +++ b/internal/retainedprovider/state.go @@ -117,19 +117,24 @@ func (state ActiveState) Validate() error { type JournalPhase string const ( - JournalPrepared JournalPhase = "prepared" - JournalActivated JournalPhase = "activated" - JournalCommitted JournalPhase = "committed" + JournalPrepared JournalPhase = "prepared" + JournalStatePromoting JournalPhase = "state_promoting" + JournalStatePromoted JournalPhase = "state_promoted" + JournalActivated JournalPhase = "activated" + JournalCommitted JournalPhase = "committed" ) type TransactionJournal struct { - ProtocolVersion string `json:"protocol_version"` - ID string `json:"id"` - Phase JournalPhase `json:"phase"` - Previous *ActiveState `json:"previous,omitempty"` - Candidate ImageSelection `json:"candidate"` - StartedAt time.Time `json:"started_at"` - UpdatedAt time.Time `json:"updated_at"` + ProtocolVersion string `json:"protocol_version"` + ID string `json:"id"` + Phase JournalPhase `json:"phase"` + DeferredCommit bool `json:"deferred_commit,omitempty"` + OuterTransactionID string `json:"outer_transaction_id,omitempty"` + ProfileID string `json:"profile_id,omitempty"` + Previous *ActiveState `json:"previous,omitempty"` + Candidate ImageSelection `json:"candidate"` + StartedAt time.Time `json:"started_at"` + UpdatedAt time.Time `json:"updated_at"` } func (journal TransactionJournal) Validate() error { @@ -139,8 +144,12 @@ func (journal TransactionJournal) Validate() error { if !safeIdentifierPattern.MatchString(journal.ID) { return fmt.Errorf("id contains an unsafe identifier") } + bound := journal.OuterTransactionID != "" || journal.ProfileID != "" + if bound && (!journal.DeferredCommit || !safeIdentifierPattern.MatchString(journal.OuterTransactionID) || !safeIdentifierPattern.MatchString(journal.ProfileID)) { + return fmt.Errorf("outer transaction binding is invalid") + } switch journal.Phase { - case JournalPrepared, JournalActivated, JournalCommitted: + case JournalPrepared, JournalStatePromoting, JournalStatePromoted, JournalActivated, JournalCommitted: default: return fmt.Errorf("phase is invalid") } diff --git a/internal/retainedprovider/state_test.go b/internal/retainedprovider/state_test.go index 75e493e..eb69ee7 100644 --- a/internal/retainedprovider/state_test.go +++ b/internal/retainedprovider/state_test.go @@ -9,6 +9,42 @@ import ( "time" ) +func TestProviderTransactionRequiresExactOuterBinding(t *testing.T) { + now := time.Unix(1_700_800_000, 0).UTC() + journal := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, + ID: "refresh-transaction-123", + Phase: JournalCommitted, + DeferredCommit: true, + OuterTransactionID: "install-transaction-123", + ProfileID: "github-runner-profile-stg", + Candidate: validTestSelection(now), + StartedAt: now, + UpdatedAt: now, + } + if err := journal.Validate(); err != nil { + t.Fatalf("valid bound provider transaction: %v", err) + } + + for _, tc := range []struct { + name string + mutate func(*TransactionJournal) + }{ + {name: "missing outer id", mutate: func(candidate *TransactionJournal) { candidate.OuterTransactionID = "" }}, + {name: "missing profile", mutate: func(candidate *TransactionJournal) { candidate.ProfileID = "" }}, + {name: "unsafe outer id", mutate: func(candidate *TransactionJournal) { candidate.OuterTransactionID = "../other" }}, + {name: "binding without deferred commit", mutate: func(candidate *TransactionJournal) { candidate.DeferredCommit = false }}, + } { + t.Run(tc.name, func(t *testing.T) { + candidate := journal + tc.mutate(&candidate) + if err := candidate.Validate(); err == nil || !strings.Contains(err.Error(), "outer transaction binding") { + t.Fatalf("Validate = %v", err) + } + }) + } +} + func TestConfigDecodeAndValidation(t *testing.T) { home := t.TempDir() valid := validTestConfig(home) @@ -51,10 +87,14 @@ func TestConfigRejectsUnsafeIdentityAndPaths(t *testing.T) { {name: "unsafe component", mutate: func(c *Config) { c.ComponentID = "component;rm" }, want: "component_id"}, {name: "unsafe unit", mutate: func(c *Config) { c.AgentUnit = "agent.service\nEnvironment=TOKEN" }, want: "agent_unit"}, {name: "relative install root", mutate: func(c *Config) { c.InstallRoot = "relative" }, want: "install_root"}, + {name: "shared workflow compute root", mutate: func(c *Config) { c.InstallRoot = filepath.Join(home, ".workflow-compute") }, want: "dedicated provider root"}, + {name: "systemd directory as install root", mutate: func(c *Config) { c.InstallRoot = c.SystemdDir }, want: "dedicated provider root"}, + {name: "arbitrary provider root", mutate: func(c *Config) { c.InstallRoot = filepath.Join(home, "provider") }, want: "dedicated provider root"}, {name: "outside home", mutate: func(c *Config) { c.SystemdDir = filepath.Join(filepath.Dir(home), "outside") }, want: "systemd_dir"}, {name: "plaintext provider URL", mutate: func(c *Config) { c.ProviderURL = "http://provider:18090" }, want: "provider_url"}, + {name: "wrong provider host", mutate: func(c *Config) { c.ProviderURL = "https://host.containers.internal:18090" }, want: "provider_url"}, {name: "wrong provider port", mutate: func(c *Config) { c.ProviderURL = "https://" + c.StableContainer + ":18091" }, want: "provider_url"}, - {name: "wrong network", mutate: func(c *Config) { c.ContainerNetwork = "host" }, want: "container_network"}, + {name: "default bridge network", mutate: func(c *Config) { c.ContainerNetwork = "bridge" }, want: "container_network"}, {name: "short ref", mutate: func(c *Config) { c.Ref = "main" }, want: "ref"}, {name: "fast timer", mutate: func(c *Config) { c.RefreshIntervalSeconds = 10 }, want: "refresh_interval_seconds"}, } { @@ -139,6 +179,8 @@ func TestRecoverySelectionForEveryJournalPhase(t *testing.T) { want ImageSelection }{ {phase: JournalPrepared, want: previous.Current}, + {phase: JournalStatePromoting, want: previous.Current}, + {phase: JournalStatePromoted, want: previous.Current}, {phase: JournalActivated, want: previous.Current}, {phase: JournalCommitted, want: candidate}, } { @@ -209,12 +251,13 @@ func validTestConfig(home string) Config { return Config{ ProtocolVersion: ConfigProtocolVersion, WorkerID: "github-runner-linux-stg", - ProfileID: "github-runner-linux-stg", + ProfileID: "github-runner-profile-stg", PluginID: GitHubPluginID, ComponentID: "github-runner-provider-sidecar", ComputeAgentPath: filepath.Join(home, ".workflow-compute", "agent-core-bin", "github-runner-linux-stg", "compute-agent"), SupervisorConfigPath: filepath.Join(home, ".workflow-compute", "github-runner-linux-stg", "supervisor.pb"), LocalStatusPath: filepath.Join(home, ".workflow-compute", "github-runner-linux-stg", "agent-status.json"), + ProviderMarkerPath: filepath.Join(home, ".workflow-compute", "updates", "updates", "current", "provider-workflow-plugin-github--component-Z2l0aHViLXJ1bm5lci1wcm92aWRlci1zaWRlY2Fy.json"), InstallRoot: root, SystemdDir: filepath.Join(home, ".config", "systemd", "user"), AgentUnit: "workflow-compute-github-runner-linux-stg.service", @@ -222,7 +265,7 @@ func validTestConfig(home string) Config { ProviderURL: "https://workflow-plugin-github-runner-provider:18090", StableContainer: "workflow-plugin-github-runner-provider", CandidateContainer: "workflow-plugin-github-runner-provider-candidate", - ContainerNetwork: "bridge", + ContainerNetwork: "wfcompute-github-provider", Organization: "GoCodeAlone", Repository: "GoCodeAlone/workflow-compute", Workflow: "dogfood-provider-target.yml", diff --git a/internal/retainedprovider/systemd.go b/internal/retainedprovider/systemd.go new file mode 100644 index 0000000..e180afa --- /dev/null +++ b/internal/retainedprovider/systemd.go @@ -0,0 +1,2200 @@ +package retainedprovider + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" + "math/big" + "net" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + systemdunit "github.com/coreos/go-systemd/v22/unit" +) + +const ( + refreshServiceUnit = "workflow-plugin-github-runner-provider-refresh.service" + refreshPathUnit = "workflow-plugin-github-runner-provider-refresh.path" + refreshTimerUnit = "workflow-plugin-github-runner-provider-refresh.timer" +) + +type SystemdUnits struct { + ProviderService string + RefreshService string + RefreshPath string + RefreshTimer string + AgentDropIn string +} + +func RenderSystemdUnits(config Config, paths LifecyclePaths) (SystemdUnits, error) { + for name, path := range map[string]string{ + "launcher": paths.Launcher, + "config": paths.ConfigFile, + "provider marker": config.ProviderMarkerPath, + "agent env": paths.AgentEnv, + } { + if !filepath.IsAbs(path) || containsControl(path) { + return SystemdUnits{}, fmt.Errorf("%s path must be absolute and safe", name) + } + } + interval := strconv.Itoa(config.RefreshIntervalSeconds) + "s" + return SystemdUnits{ + ProviderService: "[Unit]\n" + + "Description=Workflow Compute GitHub runner provider\n" + + "Wants=network-online.target\n" + + "After=network-online.target\n\n" + + "[Service]\n" + + "Type=simple\n" + + "ExecStart=" + systemdQuote(paths.Launcher) + " retained serve-active -config " + systemdQuote(paths.ConfigFile) + "\n" + + "Restart=on-failure\n" + + "RestartSec=5s\n\n" + + "[Install]\n" + + "WantedBy=default.target\n", + RefreshService: "[Unit]\n" + + "Description=Refresh Workflow Compute GitHub runner provider\n" + + "After=network-online.target\n\n" + + "[Service]\n" + + "Type=oneshot\n" + + "ExecStart=" + systemdQuote(paths.Launcher) + " retained refresh -config " + systemdQuote(paths.ConfigFile) + "\n" + + "TimeoutStartSec=15min\n", + RefreshPath: "[Unit]\n" + + "Description=Watch signed GitHub runner provider package marker\n\n" + + "[Path]\n" + + "PathChanged=" + systemdPathValue(config.ProviderMarkerPath) + "\n" + + "Unit=" + refreshServiceUnit + "\n\n" + + "[Install]\n" + + "WantedBy=default.target\n", + RefreshTimer: "[Unit]\n" + + "Description=Reconcile GitHub runner provider package\n\n" + + "[Timer]\n" + + "OnBootSec=30s\n" + + "OnUnitInactiveSec=" + interval + "\n" + + "Persistent=true\n" + + "Unit=" + refreshServiceUnit + "\n\n" + + "[Install]\n" + + "WantedBy=timers.target\n", + AgentDropIn: "[Service]\nEnvironmentFile=" + systemdPathValue(paths.AgentEnv) + "\n", + }, nil +} + +func systemdQuote(value string) string { + replacer := strings.NewReplacer(`\`, `\\`, `"`, `\"`, `%`, `%%`) + return `"` + replacer.Replace(value) + `"` +} + +func systemdPathValue(value string) string { + var escaped strings.Builder + escaped.Grow(len(value)) + for index := 0; index < len(value); index++ { + character := value[index] + switch { + case character == '%': + escaped.WriteString("%%") + case character == '/' || character == '.' || character == '_' || character == '-' || + character >= 'a' && character <= 'z' || character >= 'A' && character <= 'Z' || + character >= '0' && character <= '9': + escaped.WriteByte(character) + default: + _, _ = fmt.Fprintf(&escaped, `\x%02x`, character) + } + } + return escaped.String() +} + +type Credentials struct { + GitHubToken string + ProviderToken string +} + +type InstallMaterial struct { + ProviderEnv []byte + ProbeEnv []byte + AgentEnv []byte + ContainersConf []byte + CACert []byte + ServerCert []byte + ServerKey []byte +} + +func GenerateInstallMaterial(config Config, credentials Credentials, random io.Reader, now time.Time) (InstallMaterial, error) { + if err := validateCredential(credentials.GitHubToken); err != nil { + return InstallMaterial{}, errors.New("GitHub credential is required and must be canonical") + } + if err := validateCredential(credentials.ProviderToken); err != nil { + return InstallMaterial{}, errors.New("provider credential is required and must be canonical") + } + if random == nil { + random = rand.Reader + } + if now.IsZero() { + now = time.Now().UTC() + } + caCert, serverCert, serverKey, err := generateProviderTLS(config, random, now.UTC()) + if err != nil { + return InstallMaterial{}, err + } + providerEnvironment, err := renderPodmanEnvironment([]environmentValue{ + {Name: "GITHUB_RUNNER_PROVIDER_TOKEN", Value: credentials.ProviderToken}, + {Name: "GITHUB_RUNNER_PROVIDER_GITHUB_TOKEN", Value: credentials.GitHubToken}, + {Name: "GITHUB_RUNNER_PROVIDER_STATE_DIR", Value: providerStateMount}, + {Name: "GITHUB_RUNNER_PROVIDER_REPOSITORIES", Value: config.Repository}, + {Name: "GITHUB_RUNNER_PROVIDER_ORGANIZATIONS", Value: config.Organization}, + {Name: "GITHUB_RUNNER_PROVIDER_RUNNER_GROUPS", Value: config.RunnerGroup}, + {Name: "GITHUB_RUNNER_PROVIDER_TLS_CERT_FILE", Value: providerTLSCertPath}, + {Name: "GITHUB_RUNNER_PROVIDER_TLS_KEY_FILE", Value: providerTLSKeyPath}, + }) + if err != nil { + return InstallMaterial{}, err + } + probeEnvironment, err := renderPodmanEnvironment([]environmentValue{{Name: "GITHUB_RUNNER_PROVIDER_TOKEN", Value: credentials.ProviderToken}}) + if err != nil { + return InstallMaterial{}, err + } + agentEnvironment, err := renderSystemdEnvironment([]environmentValue{ + {Name: "WORKFLOW_COMPUTE_DYNAMIC_PROVIDER_GITHUB_ACTIONS_RUNNER_ENV_KEYS", Value: "COMPUTE_GITHUB_RUNNER_PROVIDER_URL,COMPUTE_GITHUB_RUNNER_PROVIDER_TOKEN,COMPUTE_GITHUB_RUNNER_PROVIDER_CA_CERT_B64"}, + {Name: "COMPUTE_GITHUB_RUNNER_PROVIDER_URL", Value: config.ProviderURL}, + {Name: "COMPUTE_GITHUB_RUNNER_PROVIDER_TOKEN", Value: credentials.ProviderToken}, + {Name: "COMPUTE_GITHUB_RUNNER_PROVIDER_CA_CERT_B64", Value: base64.StdEncoding.EncodeToString(caCert)}, + {Name: "CONTAINERS_CONF", Value: LifecyclePathsFor(config).ContainersConf}, + }) + if err != nil { + return InstallMaterial{}, err + } + return InstallMaterial{ + ProviderEnv: providerEnvironment, + ProbeEnv: probeEnvironment, + AgentEnv: agentEnvironment, + ContainersConf: []byte("[network]\ndefault_network = \"" + config.ContainerNetwork + "\"\n"), + CACert: caCert, + ServerCert: serverCert, + ServerKey: serverKey, + }, nil +} + +func WriteInstallMaterial(paths LifecyclePaths, material InstallMaterial) error { + for _, file := range []struct { + path string + data []byte + }{ + {path: paths.ProviderEnv, data: material.ProviderEnv}, + {path: paths.ProbeEnv, data: material.ProbeEnv}, + {path: paths.AgentEnv, data: material.AgentEnv}, + {path: paths.ContainersConf, data: material.ContainersConf}, + {path: paths.CAFile, data: material.CACert}, + {path: paths.ServerCert, data: material.ServerCert}, + {path: paths.ServerKey, data: material.ServerKey}, + } { + if err := atomicWriteFile(file.path, file.data, 0o600); err != nil { + return fmt.Errorf("write install material: %w", err) + } + } + return nil +} + +type environmentValue struct { + Name string + Value string +} + +func renderPodmanEnvironment(values []environmentValue) ([]byte, error) { + var builder strings.Builder + for _, value := range values { + if !safeEnvironmentKey(value.Name) || value.Value == "" || strings.ContainsAny(value.Value, "\r\n\x00") { + return nil, errors.New("Podman environment contains an invalid value") + } + builder.WriteString(value.Name) + builder.WriteByte('=') + builder.WriteString(value.Value) + builder.WriteByte('\n') + } + return []byte(builder.String()), nil +} + +func renderSystemdEnvironment(values []environmentValue) ([]byte, error) { + var builder strings.Builder + for _, value := range values { + if !safeEnvironmentKey(value.Name) || value.Value == "" || strings.ContainsAny(value.Value, "\r\n\x00") { + return nil, errors.New("systemd environment contains an invalid value") + } + builder.WriteString(value.Name) + builder.WriteString(`="`) + builder.WriteString(strings.NewReplacer(`\`, `\\`, `"`, `\"`).Replace(value.Value)) + builder.WriteString("\"\n") + } + return []byte(builder.String()), nil +} + +func validateCredential(value string) error { + if value == "" || strings.TrimSpace(value) != value || strings.ContainsAny(value, "\r\n\x00") { + return errors.New("invalid credential") + } + return nil +} + +func generateProviderTLS(config Config, random io.Reader, now time.Time) ([]byte, []byte, []byte, error) { + caKey, err := ecdsa.GenerateKey(elliptic.P256(), random) + if err != nil { + return nil, nil, nil, fmt.Errorf("generate provider CA key: %w", err) + } + serverKey, err := ecdsa.GenerateKey(elliptic.P256(), random) + if err != nil { + return nil, nil, nil, fmt.Errorf("generate provider server key: %w", err) + } + serial := big.NewInt(now.UnixNano()) + if serial.Sign() <= 0 { + serial = big.NewInt(1) + } + caTemplate := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: "Workflow Compute GitHub Runner Provider CA"}, + NotBefore: now.Add(-5 * time.Minute), + NotAfter: now.AddDate(10, 0, 0), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature, + BasicConstraintsValid: true, + IsCA: true, + } + caDER, err := x509.CreateCertificate(random, caTemplate, caTemplate, &caKey.PublicKey, caKey) + if err != nil { + return nil, nil, nil, fmt.Errorf("create provider CA certificate: %w", err) + } + serverTemplate := &x509.Certificate{ + SerialNumber: new(big.Int).Add(serial, big.NewInt(1)), + Subject: pkix.Name{CommonName: config.StableContainer}, + NotBefore: now.Add(-5 * time.Minute), + NotAfter: now.AddDate(1, 0, 0), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: []string{"localhost", config.StableContainer, config.CandidateContainer}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")}, + } + serverDER, err := x509.CreateCertificate(random, serverTemplate, caTemplate, &serverKey.PublicKey, caKey) + if err != nil { + return nil, nil, nil, fmt.Errorf("create provider server certificate: %w", err) + } + serverKeyDER, err := x509.MarshalECPrivateKey(serverKey) + if err != nil { + return nil, nil, nil, fmt.Errorf("marshal provider server key: %w", err) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}), + pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: serverDER}), + pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: serverKeyDER}), nil +} + +func atomicWriteFile(path string, data []byte, mode os.FileMode) (returnErr error) { + if len(data) == 0 || len(data) > MaxStateFileBytes { + return errors.New("generated file must be non-empty and at most 1 MiB") + } + if mode != 0o600 && mode != 0o700 { + return errors.New("generated file mode must be 0600 or 0700") + } + directory := filepath.Dir(path) + if err := os.MkdirAll(directory, 0o700); err != nil { + return err + } + if err := rejectWritableDestination(path); err != nil { + return err + } + temporary, err := os.CreateTemp(directory, ".retained-provider-install-*.tmp") + if err != nil { + return err + } + temporaryPath := temporary.Name() + defer func() { + _ = temporary.Close() + if err := os.Remove(temporaryPath); returnErr == nil && err != nil && !errors.Is(err, os.ErrNotExist) { + returnErr = err + } + }() + if err := temporary.Chmod(mode); err != nil { + return err + } + if _, err := temporary.Write(data); err != nil { + return err + } + if err := temporary.Sync(); err != nil { + return err + } + if err := temporary.Close(); err != nil { + return err + } + if err := rejectWritableDestination(path); err != nil { + return err + } + if err := os.Rename(temporaryPath, path); err != nil { + return err + } + return syncDirectory(directory) +} + +func rejectWritableDestination(path string) error { + info, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return errors.New("generated file destination must be regular and not a symlink") + } + return validateOwner(info) +} + +const ( + installMaintenanceID = "workflow-plugin-github-retained-provider-install" + installMaintenanceReason = "workflow-plugin-github-retained-provider-install" + refreshMaintenanceID = "workflow-plugin-github-retained-provider-refresh" + refreshMaintenanceReason = "workflow-plugin-github-retained-provider-refresh" + uninstallMaintenanceID = "workflow-plugin-github-retained-provider-uninstall" + uninstallMaintenanceReason = "workflow-plugin-github-retained-provider-uninstall" + maintenanceMarkerKind = "workflow-compute.supervisor-maintenance.v1" + localStatusProtocolVersion = "compute.local_status.v1" + localStatusAttempts = 30 + installTransactionProtocol = "retained-provider.install-transaction.v1" +) + +type Installer struct { + Runner CommandRunner + ExecutablePath func() (string, error) + Random io.Reader + Now func() time.Time + Sleep func(context.Context, time.Duration) error + Refresh func(context.Context, Config) (Status, error) + ProbeActive func(context.Context, Config) error +} + +type maintenanceRecord struct { + Kind string `json:"kind"` + ID string `json:"id"` + ProfileID string `json:"profile_id"` + Reason string `json:"reason"` + StartedAt time.Time `json:"started_at"` +} + +type maintenanceState struct { + Active bool `json:"active"` + Durable bool `json:"durable"` + Maintenance *maintenanceRecord `json:"maintenance,omitempty"` +} + +type maintenanceDisposition string + +const ( + maintenanceExactActive maintenanceDisposition = "exact_active" + maintenanceInactive maintenanceDisposition = "inactive" + maintenanceConflicting maintenanceDisposition = "conflicting" +) + +type localAgentStatus struct { + ProtocolVersion string `json:"protocol_version"` + WorkerID string `json:"worker_id"` + State string `json:"state"` + TaskID string `json:"task_id,omitempty"` + LeaseID string `json:"lease_id,omitempty"` + Message string `json:"message,omitempty"` + LastError string `json:"last_error,omitempty"` + Diagnostic json.RawMessage `json:"diagnostic,omitempty"` + UpdatedAt time.Time `json:"updated_at"` +} + +type managedFileSnapshot struct { + Path string `json:"path"` + Backup string `json:"backup"` + Mode os.FileMode `json:"mode"` + Existed bool `json:"existed"` + SHA256 string `json:"sha256,omitempty"` +} + +type systemdActivation struct { + ProviderService bool `json:"provider_service"` + RefreshPath bool `json:"refresh_path"` + RefreshTimer bool `json:"refresh_timer"` +} + +type systemdUnitState struct { + LoadState string `json:"load_state"` + FragmentPath string `json:"fragment_path"` + UnitFileState string `json:"unit_file_state"` + ActiveState string `json:"active_state"` +} + +type installTransactionPhase string + +const ( + installTransactionPrepared installTransactionPhase = "prepared" + installTransactionReady installTransactionPhase = "ready" + installTransactionCommitted installTransactionPhase = "committed" +) + +type installTransactionJournal struct { + ProtocolVersion string `json:"protocol_version"` + Operation string `json:"operation"` + Phase installTransactionPhase `json:"phase"` + MaintenanceID string `json:"maintenance_id"` + AgentStopped bool `json:"agent_stopped"` + Snapshots []managedFileSnapshot `json:"snapshots"` + PreviousUnits map[string]systemdUnitState `json:"previous_units"` + Activation systemdActivation `json:"activation"` + StartedAt time.Time `json:"started_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +func (journal installTransactionJournal) Validate(paths LifecyclePaths) error { + if journal.ProtocolVersion != installTransactionProtocol { + return errors.New("unsupported retained provider install transaction protocol") + } + expectedMaintenanceID := installMaintenanceID + if journal.Operation == "uninstall" { + expectedMaintenanceID = uninstallMaintenanceID + } else if journal.Operation != "install" { + return errors.New("retained provider install transaction operation is invalid") + } + if journal.MaintenanceID != expectedMaintenanceID || !journal.AgentStopped { + return errors.New("retained provider install transaction identity is invalid") + } + if journal.Phase != installTransactionPrepared && journal.Phase != installTransactionReady && journal.Phase != installTransactionCommitted { + return errors.New("retained provider install transaction phase is invalid") + } + if journal.StartedAt.IsZero() || journal.UpdatedAt.Before(journal.StartedAt) { + return errors.New("retained provider install transaction timestamps are invalid") + } + if len(journal.Snapshots) == 0 || len(journal.Snapshots) > len(managedInstallPaths(paths)) { + return errors.New("retained provider install transaction snapshots are invalid") + } + allowedPaths := make(map[string]struct{}, len(managedInstallPaths(paths))) + for _, path := range managedInstallPaths(paths) { + allowedPaths[path] = struct{}{} + } + seenPaths := make(map[string]struct{}, len(journal.Snapshots)) + seenBackups := make(map[string]struct{}, len(journal.Snapshots)) + backupRoot := "" + for _, snapshot := range journal.Snapshots { + if _, allowed := allowedPaths[snapshot.Path]; !allowed { + return errors.New("retained provider install transaction contains an unmanaged path") + } + if _, duplicate := seenPaths[snapshot.Path]; duplicate { + return errors.New("retained provider install transaction contains duplicate paths") + } + if _, duplicate := seenBackups[snapshot.Backup]; duplicate { + return errors.New("retained provider install transaction contains duplicate backups") + } + seenPaths[snapshot.Path] = struct{}{} + seenBackups[snapshot.Backup] = struct{}{} + root := filepath.Dir(snapshot.Backup) + if backupRoot == "" { + backupRoot = root + } else if root != backupRoot { + return errors.New("retained provider install transaction backup roots differ") + } + if !strings.HasPrefix(filepath.Base(root), ".install-backup-") { + return errors.New("retained provider install transaction backup root is invalid") + } + if err := ValidateUserPath(paths.Root, snapshot.Backup, journal.Phase == installTransactionPrepared && snapshot.Existed); err != nil { + return fmt.Errorf("validate retained provider install transaction backup: %w", err) + } + if snapshot.Existed { + if snapshot.Mode != 0o600 && snapshot.Mode != 0o700 { + return errors.New("retained provider install transaction snapshot mode is invalid") + } + if journal.Phase == installTransactionPrepared { + info, err := os.Lstat(snapshot.Backup) + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != snapshot.Mode { + return errors.New("retained provider install transaction backup is invalid") + } + if err := validateOwner(info); err != nil { + return fmt.Errorf("retained provider install transaction backup ownership: %w", err) + } + } + } else if snapshot.Mode != 0 { + return errors.New("retained provider absent snapshot mode is invalid") + } + } + for unit, state := range journal.PreviousUnits { + if unit != providerServiceUnit && unit != refreshPathUnit && unit != refreshTimerUnit { + return errors.New("retained provider install transaction contains an unmanaged unit") + } + if err := validateRestorableUnitState(state); err != nil { + return fmt.Errorf("validate retained provider install transaction unit %s: %w", unit, err) + } + } + return nil +} + +func newInstallTransactionJournal(operation string, snapshots []managedFileSnapshot, previousUnits map[string]systemdUnitState, now time.Time) installTransactionJournal { + maintenanceID := installMaintenanceID + if operation == "uninstall" { + maintenanceID = uninstallMaintenanceID + } + return installTransactionJournal{ + ProtocolVersion: installTransactionProtocol, + Operation: operation, + Phase: installTransactionPrepared, + MaintenanceID: maintenanceID, + AgentStopped: true, + Snapshots: snapshots, + PreviousUnits: previousUnits, + StartedAt: now, + UpdatedAt: now, + } +} + +func readInstallTransactionJournal(paths LifecyclePaths) (installTransactionJournal, bool, error) { + if err := ValidateUserPath(filepath.Dir(paths.Root), paths.InstallJournal, false); err != nil { + return installTransactionJournal{}, false, fmt.Errorf("install transaction path: %w", err) + } + var journal installTransactionJournal + if err := ReadStrictJSONFile(paths.InstallJournal, &journal); err != nil { + if errors.Is(err, os.ErrNotExist) { + return installTransactionJournal{}, false, nil + } + return installTransactionJournal{}, false, err + } + if err := journal.Validate(paths); err != nil { + return installTransactionJournal{}, false, err + } + return journal, true, nil +} + +func writeInstallTransactionJournal(paths LifecyclePaths, journal installTransactionJournal) error { + if err := journal.Validate(paths); err != nil { + return err + } + return AtomicWriteJSON(paths.InstallJournal, journal) +} + +func writeInstallTransactionPhase(paths LifecyclePaths, journal *installTransactionJournal, phase installTransactionPhase, now time.Time) error { + next := *journal + next.Phase = phase + next.UpdatedAt = now + if err := writeInstallTransactionJournal(paths, next); err != nil { + return err + } + *journal = next + return nil +} + +func writeInstallTransactionActivation(paths LifecyclePaths, journal *installTransactionJournal, activation systemdActivation, now time.Time) error { + next := *journal + next.Activation = activation + next.UpdatedAt = now + if err := writeInstallTransactionJournal(paths, next); err != nil { + return err + } + *journal = next + return nil +} + +func (installer Installer) recoverInstallTransaction(ctx context.Context, config Config, paths LifecyclePaths, refresher Refresher) error { + journal, found, err := readInstallTransactionJournal(paths) + if err != nil { + return fmt.Errorf("read retained provider install transaction: %w", err) + } + providerJournal, providerFound, err := readTransactionJournal(paths.Journal) + if err != nil { + return fmt.Errorf("read retained provider refresh transaction: %w", err) + } + if !found { + if providerFound && providerJournal.DeferredCommit && providerJournal.Phase == JournalCommitted { + return errors.New("deferred provider refresh has no durable outer install transaction") + } + return nil + } + if providerFound && (!providerJournal.DeferredCommit || providerJournal.Phase != JournalCommitted) { + return errors.New("outer install transaction references an incompatible provider refresh transaction") + } + switch journal.Phase { + case installTransactionPrepared: + var providerRollbackErr error + if providerFound { + providerRollbackErr = refresher.rollbackDeferredRefresh(ctx, config) + } + rollbackErr := installer.rollbackInstall(ctx, config, journal.Snapshots, journal.PreviousUnits, journal.AgentStopped, true, journal.MaintenanceID, journal.Activation) + if err := errors.Join(providerRollbackErr, rollbackErr); err != nil { + return fmt.Errorf("rollback interrupted retained provider %s: %w", journal.Operation, err) + } + case installTransactionReady: + reason := installMaintenanceReason + if journal.Operation == "uninstall" { + reason = uninstallMaintenanceReason + } + if err := installer.endMaintenance(ctx, config, journal.MaintenanceID, reason); err != nil { + return fmt.Errorf("release interrupted retained provider %s maintenance: %w", journal.Operation, err) + } + if err := writeInstallTransactionPhase(paths, &journal, installTransactionCommitted, installer.now()); err != nil { + return fmt.Errorf("commit interrupted retained provider %s: %w", journal.Operation, err) + } + fallthrough + case installTransactionCommitted: + if providerFound { + if err := refresher.finalizeDeferredRefresh(config); err != nil { + return fmt.Errorf("finalize interrupted retained provider %s: %w", journal.Operation, err) + } + } + if err := removeSnapshots(journal.Snapshots); err != nil { + return fmt.Errorf("remove interrupted retained provider %s snapshots: %w", journal.Operation, err) + } + default: + return errors.New("retained provider install transaction phase is invalid") + } + if err := removeDurableFile(paths.InstallJournal); err != nil { + return fmt.Errorf("remove recovered retained provider install transaction: %w", err) + } + return nil +} + +func (installer Installer) Install(ctx context.Context, home string, config Config, credentials Credentials) (status Status, returnErr error) { + if installer.Runner == nil { + return Status{}, errors.New("command runner is required") + } + if err := config.Validate(home); err != nil { + return Status{}, err + } + paths := LifecyclePathsFor(config) + lifecycleRefresher := Refresher{ + Runner: installer.Runner, + ExecutablePath: func() (string, error) { + return paths.Launcher, nil + }, + Now: installer.Now, + Sleep: installer.Sleep, + } + lock, err := AcquireInstallLock(paths.InstallLock) + if err != nil { + return Status{}, fmt.Errorf("acquire retained provider install lock: %w", err) + } + defer func() { returnErr = errors.Join(returnErr, lock.Release()) }() + if err := os.MkdirAll(paths.Root, 0o700); err != nil { + return Status{}, fmt.Errorf("create retained provider root: %w", err) + } + if err := validateInstallRoot(paths.Root); err != nil { + return Status{}, err + } + if err := installer.recoverLifecycleTransaction(ctx, home, paths, lifecycleRefresher); err != nil { + return Status{}, err + } + if err := installer.recoverInstallTransaction(ctx, config, paths, lifecycleRefresher); err != nil { + return Status{}, err + } + update, executable, material, units, err := installer.preflightInstall(ctx, config, paths, credentials) + if err != nil { + return Status{}, err + } + active, activeFound, err := readActiveState(paths.ActiveState) + if err != nil { + return Status{}, err + } + effect := ProviderChanged + if activeFound && active.Current.Update.SHA256 == update.SHA256 { + effect = ProviderUnchanged + } + transaction, err := newLifecycleJournal(config, LifecycleInstall, effect, nil, installer.now()) + if err != nil { + return Status{}, err + } + beforeSignature, err := installer.inspectAgentUnitSignature(ctx, home, config) + if err != nil { + return Status{}, err + } + transaction.Recovery.AgentUnitBefore = beforeSignature + if effect == ProviderUnchanged { + transaction.Unchanged = &LifecycleUnchangedProvenance{Active: active.Current, Candidate: update} + } + if err := startLifecycleTransaction(home, paths, &transaction); err != nil { + return Status{}, err + } + if err := writeLifecycleTransition(home, paths, &transaction, LifecycleFencing, "", installer.now()); err != nil { + return Status{}, err + } + fail := func(cause error) (Status, error) { + rollbackContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + return Status{}, errors.Join(cause, installer.recoverLifecycleTransaction(rollbackContext, home, paths, lifecycleRefresher)) + } + if err := installer.beginMaintenance(ctx, config, installMaintenanceID, installMaintenanceReason); err != nil { + return fail(err) + } + if err := installer.waitLocalState(ctx, config, "unavailable"); err != nil { + return Status{}, fmt.Errorf("wait for retained agent maintenance fence: %w", err) + } + if err := installer.reattestLifecycleAuthority(ctx, home, transaction); err != nil { + return Status{}, err + } + if err := snapshotManagedFilesForLifecycle(home, paths, &transaction, installer.now()); err != nil { + return fail(fmt.Errorf("snapshot retained provider wiring: %w", err)) + } + previousUnits, err := installer.captureManagedUnitStates(ctx) + if err != nil { + return fail(fmt.Errorf("snapshot retained provider unit state: %w", err)) + } + transaction.WiringIntent = managedWiringIntent(paths, units, true) + intendedSignature, err := deriveLifecycleAgentUnitSignature(beforeSignature, paths, transaction.WiringIntent, &LifecycleFileAttestation{ + Path: paths.AgentEnv, SHA256: digestBytes(material.AgentEnv), + }) + if err != nil { + return fail(err) + } + transaction.AgentUnitIntended = &intendedSignature + transaction.PreviousUnits = previousUnits + transaction.UpdatedAt = installer.now() + if err := writeLifecycleJournal(home, paths, transaction); err != nil { + return fail(fmt.Errorf("write retained provider install recovery state: %w", err)) + } + if err := writeLifecycleTransition(home, paths, &transaction, LifecycleFenced, "", installer.now()); err != nil { + return fail(err) + } + if err := installer.systemctl(ctx, "stop", config.AgentUnit); err != nil { + return fail(fmt.Errorf("stop retained agent: %w", err)) + } + watchUnits := previouslyLoadedWatchUnits(previousUnits) + if len(watchUnits) > 0 { + arguments := append([]string{"disable", "--now"}, watchUnits...) + if err := installer.systemctl(ctx, arguments...); err != nil { + return fail(fmt.Errorf("pause retained provider refresh: %w", err)) + } + } + if err := writeInstalledProvider(config, paths, executable, update.SHA256, material, units); err != nil { + return fail(err) + } + if err := installer.ensureProviderNetwork(ctx, config); err != nil { + return fail(err) + } + if err := installer.systemctl(ctx, "daemon-reload"); err != nil { + return fail(fmt.Errorf("reload user systemd: %w", err)) + } + loadedSignature, err := installer.inspectAgentUnitSignature(ctx, home, config) + if err != nil { + return fail(err) + } + if !equalLifecycleSystemdSignature(loadedSignature, intendedSignature) { + return fail(errors.New("reloaded retained agent unit does not match intended signature")) + } + activation := transaction.Activation + activation.ProviderService = true + transaction.Activation = activation + transaction.UpdatedAt = installer.now() + if err := writeLifecycleJournal(home, paths, transaction); err != nil { + return fail(fmt.Errorf("record provider service activation: %w", err)) + } + if err := installer.systemctl(ctx, "enable", providerServiceUnit); err != nil { + return fail(fmt.Errorf("enable provider service: %w", err)) + } + refresh := installer.Refresh + probeActive := installer.ProbeActive + if refresh == nil || probeActive == nil { + if refresh == nil { + refresh = func(ctx context.Context, config Config) (Status, error) { + return lifecycleRefresher.refreshUnderLifecycleTransaction(ctx, config, false, true, transaction.TransactionID, config.ProfileID, "") + } + } + if probeActive == nil { + probeActive = lifecycleRefresher.RestartAndProbeActive + } + } + status, err = refresh(ctx, config) + if err != nil { + return fail(fmt.Errorf("activate retained provider: %w", err)) + } + if err := probeActive(ctx, config); err != nil { + return fail(fmt.Errorf("revalidate retained provider: %w", err)) + } + if transaction.Unchanged != nil { + transaction.Unchanged.StableProbeAt = installer.now() + } + inner, innerFound, err := readTransactionJournal(paths.Journal) + if err != nil { + return fail(err) + } + if effect == ProviderChanged { + if !innerFound || inner.Phase != JournalCommitted || inner.OuterTransactionID != transaction.TransactionID || inner.ProfileID != config.ProfileID { + return fail(errors.New("changed install did not leave a matching deferred committed provider transaction")) + } + transaction.ProviderTransaction = &LifecycleProviderTransaction{TransactionID: inner.ID, ProfileID: config.ProfileID, Digest: inner.Candidate.Update.SHA256} + } else if innerFound { + return fail(errors.New("unchanged install unexpectedly created a provider transaction")) + } + transaction.UpdatedAt = installer.now() + if err := writeLifecycleJournal(home, paths, transaction); err != nil { + return fail(err) + } + _, err = installer.enableWatchUnitBefore(ctx, refreshPathUnit, func() error { + activation := transaction.Activation + activation.RefreshPath = true + transaction.Activation = activation + transaction.UpdatedAt = installer.now() + return writeLifecycleJournal(home, paths, transaction) + }) + if err != nil { + return fail(fmt.Errorf("enable retained provider refresh: %w", err)) + } + _, err = installer.enableWatchUnitBefore(ctx, refreshTimerUnit, func() error { + activation := transaction.Activation + activation.RefreshTimer = true + transaction.Activation = activation + transaction.UpdatedAt = installer.now() + return writeLifecycleJournal(home, paths, transaction) + }) + if err != nil { + return fail(fmt.Errorf("enable retained provider refresh: %w", err)) + } + if err := installer.reattestLifecycleAuthority(ctx, home, transaction); err != nil { + return fail(err) + } + if err := installer.systemctl(ctx, "start", config.AgentUnit); err != nil { + return fail(fmt.Errorf("restart retained agent: %w", err)) + } + if err := installer.waitLocalState(ctx, config, "unavailable"); err != nil { + return fail(fmt.Errorf("verify retained agent remains fenced: %w", err)) + } + if err := installer.reattestLifecycleAuthority(ctx, home, transaction); err != nil { + return fail(err) + } + if err := validateLifecycleWiringVector(transaction, paths, lifecycleWiringIntended); err != nil { + return fail(err) + } + if err := writeLifecycleTransition(home, paths, &transaction, LifecycleReady, LifecycleCommit, installer.now()); err != nil { + return fail(fmt.Errorf("prepare retained provider install commit: %w", err)) + } + if err := writeLifecycleTransition(home, paths, &transaction, LifecycleReleasing, LifecycleCommit, installer.now()); err != nil { + return fail(err) + } + _ = drainLifecycleAudit(home, paths, &transaction) + if err := installer.releaseLifecycleMaintenance(ctx, home, transaction); err != nil { + return Status{}, fmt.Errorf("release retained agent maintenance fence: %w", err) + } + if err := writeLifecycleTransition(home, paths, &transaction, LifecycleCommitted, LifecycleCommit, installer.now()); err != nil { + return Status{}, fmt.Errorf("commit retained provider install: %w", err) + } + if err := finalizeLifecycleTransaction(home, paths, &transaction, lifecycleRefresher); err != nil { + return Status{}, err + } + if err := installer.waitLocalState(ctx, config, "idle"); err != nil { + return Status{}, fmt.Errorf("wait for retained agent idle state: %w", err) + } + return status, nil +} + +func (installer Installer) Uninstall(ctx context.Context, home string, config Config, purge bool) (status Status, returnErr error) { + if installer.Runner == nil { + return Status{}, errors.New("command runner is required") + } + if err := config.Validate(home); err != nil { + return Status{}, err + } + paths := LifecyclePathsFor(config) + lock, err := AcquireInstallLock(paths.InstallLock) + if err != nil { + return Status{}, fmt.Errorf("acquire retained provider install lock: %w", err) + } + defer func() { returnErr = errors.Join(returnErr, lock.Release()) }() + lifecycleRefresher := Refresher{Runner: installer.Runner, Now: installer.Now, Sleep: installer.Sleep} + if err := os.MkdirAll(paths.Root, 0o700); err != nil { + return Status{}, fmt.Errorf("create retained provider root: %w", err) + } + if err := validateInstallRoot(paths.Root); err != nil { + return Status{}, err + } + if err := installer.recoverLifecycleTransaction(ctx, home, paths, lifecycleRefresher); err != nil { + return Status{}, err + } + if err := installer.recoverInstallTransaction(ctx, config, paths, lifecycleRefresher); err != nil { + return Status{}, err + } + if err := installer.runAgentSystemPreflight(ctx, config); err != nil { + return Status{}, err + } + transaction, err := newLifecycleJournal(config, LifecycleUninstall, ProviderNotApplicable, &LifecycleUninstallPayload{Purge: purge}, installer.now()) + if err != nil { + return Status{}, err + } + beforeSignature, err := installer.inspectAgentUnitSignature(ctx, home, config) + if err != nil { + return Status{}, err + } + transaction.Recovery.AgentUnitBefore = beforeSignature + if err := startLifecycleTransaction(home, paths, &transaction); err != nil { + return Status{}, err + } + if err := writeLifecycleTransition(home, paths, &transaction, LifecycleFencing, "", installer.now()); err != nil { + return Status{}, err + } + fail := func(cause error) (Status, error) { + rollbackContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + return Status{}, errors.Join(cause, installer.recoverLifecycleTransaction(rollbackContext, home, paths, lifecycleRefresher)) + } + if err := installer.beginMaintenance(ctx, config, uninstallMaintenanceID, uninstallMaintenanceReason); err != nil { + return fail(err) + } + if err := installer.waitLocalState(ctx, config, "unavailable"); err != nil { + return Status{}, fmt.Errorf("wait for retained agent maintenance fence: %w", err) + } + if err := installer.reattestLifecycleAuthority(ctx, home, transaction); err != nil { + return Status{}, err + } + if err := snapshotManagedFilesForLifecycle(home, paths, &transaction, installer.now()); err != nil { + return fail(fmt.Errorf("snapshot retained provider wiring: %w", err)) + } + previousUnits, err := installer.captureManagedUnitStates(ctx) + if err != nil { + return fail(fmt.Errorf("snapshot retained provider unit state: %w", err)) + } + transaction.WiringIntent = managedWiringIntent(paths, SystemdUnits{}, false) + intendedSignature, err := deriveLifecycleAgentUnitSignature(beforeSignature, paths, transaction.WiringIntent, nil) + if err != nil { + return fail(err) + } + transaction.AgentUnitIntended = &intendedSignature + transaction.PreviousUnits = previousUnits + transaction.UpdatedAt = installer.now() + if err := writeLifecycleJournal(home, paths, transaction); err != nil { + return fail(fmt.Errorf("write retained provider uninstall recovery state: %w", err)) + } + if err := writeLifecycleTransition(home, paths, &transaction, LifecycleFenced, "", installer.now()); err != nil { + return fail(err) + } + if err := installer.systemctl(ctx, "stop", config.AgentUnit); err != nil { + return fail(fmt.Errorf("stop retained agent: %w", err)) + } + if err := installer.systemctl(ctx, "disable", "--now", refreshPathUnit, refreshTimerUnit, providerServiceUnit); err != nil { + return fail(fmt.Errorf("disable retained provider wiring: %w", err)) + } + for _, path := range managedWiringPaths(paths) { + if err := removeDurableFile(path); err != nil { + return fail(fmt.Errorf("remove retained provider wiring: %w", err)) + } + } + if err := installer.systemctl(ctx, "daemon-reload"); err != nil { + return fail(fmt.Errorf("reload user systemd: %w", err)) + } + loadedSignature, err := installer.inspectAgentUnitSignature(ctx, home, config) + if err != nil { + return fail(err) + } + if !equalLifecycleSystemdSignature(loadedSignature, intendedSignature) { + return fail(errors.New("reloaded retained agent unit does not match intended signature")) + } + if err := installer.reattestLifecycleAuthority(ctx, home, transaction); err != nil { + return fail(err) + } + if err := installer.systemctl(ctx, "start", config.AgentUnit); err != nil { + return fail(fmt.Errorf("restart retained agent: %w", err)) + } + if err := installer.waitLocalState(ctx, config, "unavailable"); err != nil { + return fail(fmt.Errorf("verify retained agent remains fenced: %w", err)) + } + if err := installer.reattestLifecycleAuthority(ctx, home, transaction); err != nil { + return fail(err) + } + if err := validateLifecycleWiringVector(transaction, paths, lifecycleWiringIntended); err != nil { + return fail(err) + } + if err := writeLifecycleTransition(home, paths, &transaction, LifecycleReady, LifecycleCommit, installer.now()); err != nil { + return fail(fmt.Errorf("prepare retained provider uninstall commit: %w", err)) + } + if err := writeLifecycleTransition(home, paths, &transaction, LifecycleReleasing, LifecycleCommit, installer.now()); err != nil { + return fail(err) + } + _ = drainLifecycleAudit(home, paths, &transaction) + if err := installer.releaseLifecycleMaintenance(ctx, home, transaction); err != nil { + return Status{}, fmt.Errorf("release retained agent maintenance fence: %w", err) + } + if err := writeLifecycleTransition(home, paths, &transaction, LifecycleCommitted, LifecycleCommit, installer.now()); err != nil { + return Status{}, fmt.Errorf("commit retained provider uninstall: %w", err) + } + if err := installer.waitLocalState(ctx, config, "idle"); err != nil { + return Status{}, fmt.Errorf("wait for retained agent idle state: %w", err) + } + if err := finalizeLifecycleTransaction(home, paths, &transaction, lifecycleRefresher); err != nil { + return Status{}, err + } + return Status{ProtocolVersion: StatusProtocolVersion, ObservedAt: installer.now()}, nil +} + +func (installer Installer) Recover(ctx context.Context, home string, config Config, confirmation string) (status Status, returnErr error) { + if installer.Runner == nil { + return Status{}, errors.New("command runner is required") + } + if err := config.Validate(home); err != nil { + return Status{}, err + } + if !safeIdentifierPattern.MatchString(confirmation) { + return Status{}, errors.New("exact legacy provider transaction confirmation is required") + } + paths := LifecyclePathsFor(config) + lock, err := AcquireInstallLock(paths.InstallLock) + if err != nil { + return Status{}, fmt.Errorf("acquire retained provider install lock: %w", err) + } + defer func() { returnErr = errors.Join(returnErr, lock.Release()) }() + if err := validateInstallRoot(paths.Root); err != nil { + return Status{}, err + } + if _, found, err := readLifecycleJournal(home, paths); err != nil { + return Status{}, err + } else if found { + return Status{}, errors.New("explicit legacy recovery refuses an existing outer lifecycle transaction") + } + inner, found, err := readTransactionJournal(paths.Journal) + if err != nil { + return Status{}, err + } + if !found { + return Status{}, errors.New("no legacy provider transaction requires recovery") + } + if inner.ID != confirmation { + return Status{}, errors.New("legacy provider transaction confirmation does not match") + } + refresher := Refresher{Runner: installer.Runner, Now: installer.Now, Sleep: installer.Sleep} + if err := installer.adoptLegacyProviderTransaction(ctx, home, paths, refresher, &config, confirmation); err != nil { + return Status{}, err + } + return installer.Status(ctx, home, config) +} + +func (installer Installer) Status(ctx context.Context, home string, config Config) (Status, error) { + if installer.Runner == nil { + return Status{}, errors.New("command runner is required") + } + if err := config.Validate(home); err != nil { + return Status{}, err + } + paths := LifecyclePathsFor(config) + active, found, err := readActiveState(paths.ActiveState) + if err != nil { + return Status{}, err + } + if !found { + return Status{ProtocolVersion: StatusProtocolVersion, ObservedAt: installer.now()}, nil + } + unitInfo, err := os.Lstat(paths.ProviderUnit) + if errors.Is(err, os.ErrNotExist) { + return Status{ProtocolVersion: StatusProtocolVersion, ObservedAt: installer.now()}, nil + } + if err != nil || !unitInfo.Mode().IsRegular() { + return Status{}, errors.New("retained provider service unit must be a regular file") + } + if err := validateOwner(unitInfo); err != nil { + return Status{}, fmt.Errorf("retained provider service unit: %w", err) + } + output, err := installer.run(ctx, Command{Path: "/usr/bin/systemctl", Args: []string{ + "--user", "show", providerServiceUnit, "--property", "ActiveState", "--value", + }}) + if err != nil { + return Status{}, fmt.Errorf("inspect retained provider service: %w", err) + } + return statusForActive(active, strings.TrimSpace(string(output)) == "active", installer.now()), nil +} + +func (installer Installer) preflightInstall(ctx context.Context, config Config, paths LifecyclePaths, credentials Credentials) (VerifiedUpdate, string, InstallMaterial, SystemdUnits, error) { + if err := installer.runSystemPreflight(ctx, config); err != nil { + return VerifiedUpdate{}, "", InstallMaterial{}, SystemdUnits{}, err + } + update, err := VerifyCurrentUpdate(ctx, config, installer.Runner) + if err != nil { + return VerifiedUpdate{}, "", InstallMaterial{}, SystemdUnits{}, err + } + executablePath := installer.ExecutablePath + if executablePath == nil { + executablePath = os.Executable + } + executable, err := executablePath() + if err != nil { + return VerifiedUpdate{}, "", InstallMaterial{}, SystemdUnits{}, fmt.Errorf("resolve installer executable: %w", err) + } + digest, err := hashRegularFile(executable, true) + if err != nil { + return VerifiedUpdate{}, "", InstallMaterial{}, SystemdUnits{}, fmt.Errorf("hash installer executable: %w", err) + } + if digest != update.SHA256 { + return VerifiedUpdate{}, "", InstallMaterial{}, SystemdUnits{}, errors.New("installer digest does not match verified provider update") + } + material, err := GenerateInstallMaterial(config, credentials, installer.Random, installer.now()) + if err != nil { + return VerifiedUpdate{}, "", InstallMaterial{}, SystemdUnits{}, err + } + units, err := RenderSystemdUnits(config, paths) + if err != nil { + return VerifiedUpdate{}, "", InstallMaterial{}, SystemdUnits{}, err + } + return update, executable, material, units, nil +} + +func (installer Installer) runSystemPreflight(ctx context.Context, config Config) error { + if err := installer.systemctl(ctx, "show-environment"); err != nil { + return fmt.Errorf("user systemd preflight: %w", err) + } + if _, err := installer.run(ctx, Command{Path: config.PodmanPath, Args: []string{"version", "--format", "{{.Client.Version}}"}}); err != nil { + return fmt.Errorf("rootless Podman preflight: %w", err) + } + return installer.runSupervisorConfigPreflight(ctx, config) +} + +func (installer Installer) runAgentSystemPreflight(ctx context.Context, config Config) error { + if err := installer.systemctl(ctx, "show-environment"); err != nil { + return fmt.Errorf("user systemd preflight: %w", err) + } + return installer.runSupervisorConfigPreflight(ctx, config) +} + +func (installer Installer) runSupervisorConfigPreflight(ctx context.Context, config Config) error { + if _, err := installer.run(ctx, Command{Path: config.ComputeAgentPath, Args: []string{ + "supervisor-config", "validate", "-path", config.SupervisorConfigPath, "-format", "auto", + }}); err != nil { + return fmt.Errorf("supervisor config preflight: %w", err) + } + return nil +} + +func (installer Installer) ensureProviderNetwork(ctx context.Context, config Config) error { + if _, err := installer.run(ctx, Command{Path: config.PodmanPath, Args: []string{ + "network", "create", "--driver", "bridge", "--ignore", config.ContainerNetwork, + }}); err != nil { + return fmt.Errorf("create provider network: %w", err) + } + refresher := Refresher{Runner: installer.Runner} + if err := refresher.validateProviderNetwork(ctx, config); err != nil { + return err + } + return nil +} + +func (installer Installer) beginMaintenance(ctx context.Context, config Config, id, reason string) error { + state, err := installer.maintenanceCommand(ctx, config, "begin", id, reason) + if err != nil { + return err + } + return validateMaintenanceState(state, true, config.ProfileID, id, reason) +} + +func (installer Installer) endMaintenance(ctx context.Context, config Config, id, reason string) error { + state, err := installer.maintenanceCommand(ctx, config, "end", id, "") + if err != nil { + return err + } + return validateMaintenanceState(state, false, config.ProfileID, id, reason) +} + +func (installer Installer) releaseLifecycleMaintenance(ctx context.Context, home string, journal LifecycleJournal) error { + if err := installer.reattestLifecycleAuthority(ctx, home, journal); err != nil { + return err + } + id, reason, err := lifecycleMaintenanceIdentity(journal.Operation) + if err != nil { + return err + } + return installer.endMaintenance(ctx, journal.Recovery.Config, id, reason) +} + +func (installer Installer) maintenanceStatus(ctx context.Context, config Config) (maintenanceState, error) { + state, err := installer.maintenanceCommand(ctx, config, "status", "", "") + if err != nil { + return maintenanceState{}, err + } + if !state.Durable { + return maintenanceState{}, errors.New("supervisor maintenance status is not durable") + } + if !state.Active { + if state.Maintenance != nil { + return maintenanceState{}, errors.New("inactive supervisor maintenance status contains a marker") + } + return state, nil + } + if state.Maintenance == nil { + return maintenanceState{}, errors.New("active supervisor maintenance status is missing its marker") + } + maintenance := state.Maintenance + if maintenance.Kind != maintenanceMarkerKind || !safeIdentifierPattern.MatchString(maintenance.ID) || + !safeIdentifierPattern.MatchString(maintenance.ProfileID) || !safeIdentifierPattern.MatchString(maintenance.Reason) || maintenance.StartedAt.IsZero() { + return maintenanceState{}, errors.New("supervisor maintenance status contains an invalid marker") + } + return state, nil +} + +func classifyMaintenanceState(state maintenanceState, profileID, id, reason string) maintenanceDisposition { + if !state.Active { + return maintenanceInactive + } + maintenance := state.Maintenance + if maintenance != nil && maintenance.Kind == maintenanceMarkerKind && maintenance.ID == id && maintenance.ProfileID == profileID && maintenance.Reason == reason { + return maintenanceExactActive + } + return maintenanceConflicting +} + +func (installer Installer) maintenanceCommand(ctx context.Context, config Config, operation, id, reason string) (maintenanceState, error) { + arguments := []string{ + "supervisor-maintenance", operation, + "-config", config.SupervisorConfigPath, + "-format", "auto", + "-profile", config.ProfileID, + } + if id != "" { + arguments = append(arguments, "-id", id) + } + if reason != "" { + arguments = append(arguments, "-reason", reason) + } + output, err := installer.run(ctx, Command{Path: config.ComputeAgentPath, Args: arguments}) + if err != nil { + return maintenanceState{}, err + } + var state maintenanceState + if err := decodeStrictJSON(bytes.NewReader(output), &state); err != nil { + return maintenanceState{}, fmt.Errorf("decode supervisor maintenance state: %w", err) + } + return state, nil +} + +func validateMaintenanceState(state maintenanceState, active bool, profileID, id, reason string) error { + if state.Active != active || !state.Durable || state.Maintenance == nil { + return errors.New("supervisor maintenance command did not return the required durable state") + } + maintenance := state.Maintenance + if maintenance.Kind != maintenanceMarkerKind || maintenance.ID != id || maintenance.ProfileID != profileID || maintenance.Reason != reason || maintenance.StartedAt.IsZero() { + return errors.New("supervisor maintenance command returned a mismatched maintenance fence") + } + return nil +} + +func (installer Installer) waitLocalState(ctx context.Context, config Config, expected string) error { + for attempt := 0; attempt < localStatusAttempts; attempt++ { + output, err := installer.run(ctx, Command{Path: config.ComputeAgentPath, Args: []string{ + "local-status", "sanitize", "-path", config.LocalStatusPath, + }}) + if err != nil { + return err + } + var status localAgentStatus + if err := decodeStrictJSON(bytes.NewReader(output), &status); err != nil { + return fmt.Errorf("decode local agent status: %w", err) + } + if status.ProtocolVersion != localStatusProtocolVersion || status.WorkerID != config.WorkerID || status.UpdatedAt.IsZero() { + return errors.New("local agent status identity or protocol mismatch") + } + if status.State == expected && status.TaskID == "" && status.LeaseID == "" { + return nil + } + if attempt+1 < localStatusAttempts { + if err := installer.sleep(ctx, time.Second); err != nil { + return err + } + } + } + return fmt.Errorf("local agent did not reach %s without an active task or lease", expected) +} + +func (installer Installer) waitLocalDrained(ctx context.Context, config Config) error { + for attempt := 0; attempt < localStatusAttempts; attempt++ { + output, err := installer.run(ctx, Command{Path: config.ComputeAgentPath, Args: []string{ + "local-status", "sanitize", "-path", config.LocalStatusPath, + }}) + if err != nil { + return err + } + var status localAgentStatus + if err := decodeStrictJSON(bytes.NewReader(output), &status); err != nil { + return fmt.Errorf("decode local agent status: %w", err) + } + if status.ProtocolVersion != localStatusProtocolVersion || status.WorkerID != config.WorkerID || status.UpdatedAt.IsZero() { + return errors.New("local agent status identity or protocol mismatch") + } + if status.TaskID == "" && status.LeaseID == "" { + return nil + } + if attempt+1 < localStatusAttempts { + if err := installer.sleep(ctx, time.Second); err != nil { + return err + } + } + } + return errors.New("local agent remained assigned to a task or lease") +} + +func (installer Installer) systemctl(ctx context.Context, args ...string) error { + arguments := append([]string{"--user"}, args...) + _, err := installer.run(ctx, Command{Path: "/usr/bin/systemctl", Args: arguments}) + return err +} + +func (installer Installer) enableWatchUnit(ctx context.Context, unit string) (bool, error) { + return installer.enableWatchUnitBefore(ctx, unit, nil) +} + +func (installer Installer) enableWatchUnitBefore(ctx context.Context, unit string, beforeMutation func() error) (bool, error) { + if beforeMutation != nil { + if err := beforeMutation(); err != nil { + return false, err + } + } + if err := installer.systemctl(ctx, "enable", "--now", unit); err == nil { + return true, nil + } else { + activated, inspectErr := installer.inspectUnitActivation(ctx, unit) + if inspectErr != nil { + return true, errors.Join(err, fmt.Errorf("inspect failed unit activation: %w", inspectErr)) + } + return activated, err + } +} + +func (installer Installer) inspectUnitActivation(ctx context.Context, unit string) (bool, error) { + state, err := installer.inspectUnitState(ctx, unit) + if err != nil { + return false, err + } + return state.ActiveState != "inactive" || state.UnitFileState != "disabled", nil +} + +func (installer Installer) inspectAgentUnitSignature(ctx context.Context, home string, config Config) (LifecycleSystemdSignature, error) { + output, err := installer.run(ctx, Command{Path: "/usr/bin/systemctl", Args: []string{ + "--user", "show", config.AgentUnit, + "--property", "LoadState", "--property", "FragmentPath", "--property", "DropInPaths", + }}) + if err != nil { + return LifecycleSystemdSignature{}, fmt.Errorf("inspect retained agent effective unit: %w", err) + } + properties := map[string]string{} + for _, line := range strings.Split(strings.TrimSuffix(string(output), "\n"), "\n") { + key, value, found := strings.Cut(line, "=") + if !found || (key != "LoadState" && key != "FragmentPath" && key != "DropInPaths") { + return LifecycleSystemdSignature{}, errors.New("effective agent systemd state is malformed") + } + if _, duplicate := properties[key]; duplicate { + return LifecycleSystemdSignature{}, errors.New("effective agent systemd state contains a duplicate property") + } + properties[key] = value + } + if len(properties) != 3 || properties["LoadState"] != "loaded" { + return LifecycleSystemdSignature{}, errors.New("effective agent systemd state is incomplete or unloaded") + } + fragmentPath, err := decodeSystemdPath(properties["FragmentPath"]) + if err != nil { + return LifecycleSystemdSignature{}, fmt.Errorf("decode effective agent fragment: %w", err) + } + dropInPaths, err := decodeSystemdPathList(properties["DropInPaths"], false) + if err != nil { + return LifecycleSystemdSignature{}, fmt.Errorf("decode effective agent drop-ins: %w", err) + } + fragment, fragmentContents, err := readAndAttestLifecycleSystemdPath(home, fragmentPath) + if err != nil { + return LifecycleSystemdSignature{}, err + } + signature := LifecycleSystemdSignature{Fragment: fragment} + unitContents := [][]byte{fragmentContents} + for _, path := range dropInPaths { + attestation, contents, err := readAndAttestLifecycleSystemdPath(home, path) + if err != nil { + return LifecycleSystemdSignature{}, err + } + signature.DropIns = append(signature.DropIns, attestation) + unitContents = append(unitContents, contents) + } + environmentPaths, err := effectiveSystemdEnvironmentFiles(unitContents) + if err != nil { + return LifecycleSystemdSignature{}, fmt.Errorf("decode effective agent environment files: %w", err) + } + signature.ExecStart, err = effectiveSystemdExecStart(unitContents) + if err != nil { + return LifecycleSystemdSignature{}, fmt.Errorf("decode effective agent ExecStart: %w", err) + } + for _, path := range environmentPaths { + attestation, err := attestLifecycleSystemdPath(home, path) + if err != nil { + return LifecycleSystemdSignature{}, err + } + signature.EnvironmentFiles = append(signature.EnvironmentFiles, attestation) + } + sort.Slice(signature.EnvironmentFiles, func(left, right int) bool { + return signature.EnvironmentFiles[left].Path < signature.EnvironmentFiles[right].Path + }) + if err := signature.Validate(home); err != nil { + return LifecycleSystemdSignature{}, err + } + return signature, nil +} + +func effectiveSystemdExecStart(unitContents [][]byte) (string, error) { + var commands []string + for _, contents := range unitContents { + options, err := systemdunit.DeserializeOptions(bytes.NewReader(contents)) + if err != nil { + return "", fmt.Errorf("parse systemd unit: %w", err) + } + for _, option := range options { + if option.Section != "Service" || option.Name != "ExecStart" { + continue + } + if strings.TrimSpace(option.Value) == "" { + commands = nil + continue + } + if len(option.Value) > 16*1024 || containsControl(option.Value) { + return "", errors.New("systemd ExecStart contains an invalid value") + } + commands = append(commands, option.Value) + if len(commands) > 64 { + return "", errors.New("systemd unit contains too many ExecStart commands") + } + } + } + if len(commands) == 0 { + return "", errors.New("systemd unit has no ExecStart command") + } + encoded, err := json.Marshal(commands) + if err != nil { + return "", fmt.Errorf("encode static ExecStart: %w", err) + } + return string(encoded), nil +} + +func effectiveSystemdEnvironmentFiles(unitContents [][]byte) ([]string, error) { + var paths []string + for _, contents := range unitContents { + options, err := systemdunit.DeserializeOptions(bytes.NewReader(contents)) + if err != nil { + return nil, fmt.Errorf("parse systemd unit: %w", err) + } + for _, option := range options { + if option.Section != "Service" || option.Name != "EnvironmentFile" { + continue + } + if strings.TrimSpace(option.Value) == "" { + paths = nil + continue + } + values, err := splitSystemdPathWords(option.Value) + if err != nil { + return nil, err + } + paths = append(paths, values...) + if len(paths) > 64 { + return nil, errors.New("systemd unit references too many environment files") + } + } + } + return paths, nil +} + +func splitSystemdPathWords(value string) ([]string, error) { + var words []string + var word strings.Builder + var quote byte + started := false + flush := func() { + if started { + words = append(words, word.String()) + word.Reset() + started = false + } + } + for index := 0; index < len(value); index++ { + character := value[index] + if quote == 0 && (character == ' ' || character == '\t') { + flush() + continue + } + if character == '\'' || character == '"' { + if quote == 0 { + quote = character + started = true + continue + } + if quote == character { + quote = 0 + continue + } + } + if character == '\\' { + decoded, consumed, err := decodeSystemdEscape(value[index:]) + if err != nil { + return nil, err + } + word.WriteByte(decoded) + started = true + index += consumed - 1 + continue + } + if character < 0x20 || character == 0x7f { + return nil, errors.New("systemd environment file path contains control bytes") + } + word.WriteByte(character) + started = true + } + if quote != 0 { + return nil, errors.New("systemd environment file path has an unterminated quote") + } + flush() + if len(words) == 0 { + return nil, errors.New("systemd EnvironmentFile directive is empty") + } + for index, word := range words { + if strings.HasPrefix(word, "-") || strings.ContainsAny(word, "*?[") { + return nil, errors.New("optional or globbed systemd environment files are unsupported") + } + expanded, err := decodeLiteralSystemdPercents(word) + if err != nil { + return nil, err + } + if !filepath.IsAbs(expanded) || containsControl(expanded) { + return nil, errors.New("systemd environment file path must be absolute") + } + words[index] = expanded + } + return words, nil +} + +func decodeSystemdEscape(value string) (byte, int, error) { + if len(value) < 2 { + return 0, 0, errors.New("systemd environment file path has a trailing escape") + } + switch value[1] { + case '\\', '\'', '"', ' ': + return value[1], 2, nil + case 'x': + if len(value) < 4 { + return 0, 0, errors.New("systemd environment file path has a short hexadecimal escape") + } + high, okHigh := fromHex(value[2]) + low, okLow := fromHex(value[3]) + if !okHigh || !okLow || high<<4|low == 0 { + return 0, 0, errors.New("systemd environment file path has an invalid hexadecimal escape") + } + return high<<4 | low, 4, nil + default: + return 0, 0, errors.New("systemd environment file path has an unsupported escape") + } +} + +func decodeLiteralSystemdPercents(value string) (string, error) { + var decoded strings.Builder + for index := 0; index < len(value); index++ { + if value[index] != '%' { + decoded.WriteByte(value[index]) + continue + } + if index+1 >= len(value) || value[index+1] != '%' { + return "", errors.New("systemd environment file path contains an unsupported specifier") + } + decoded.WriteByte('%') + index++ + } + return decoded.String(), nil +} + +func decodeSystemdPathList(value string, environment bool) ([]string, error) { + if value == "" { + return nil, nil + } + fields := strings.Fields(value) + paths := make([]string, 0, len(fields)) + for _, field := range fields { + if environment && strings.HasPrefix(field, "(ignore_errors=") && strings.HasSuffix(field, ")") { + continue + } + path, err := decodeSystemdPath(field) + if err != nil { + return nil, err + } + paths = append(paths, path) + } + return paths, nil +} + +func decodeSystemdPath(value string) (string, error) { + if value == "" || value[0] != '/' || containsControl(value) { + return "", errors.New("systemd path is empty, relative, or contains control bytes") + } + var decoded strings.Builder + for index := 0; index < len(value); index++ { + if value[index] != '\\' { + decoded.WriteByte(value[index]) + continue + } + if index+1 < len(value) && value[index+1] == '\\' { + decoded.WriteByte('\\') + index++ + continue + } + if index+3 >= len(value) || value[index+1] != 'x' { + return "", errors.New("systemd path contains an unsupported escape") + } + high, okHigh := fromHex(value[index+2]) + low, okLow := fromHex(value[index+3]) + if !okHigh || !okLow { + return "", errors.New("systemd path contains an invalid hexadecimal escape") + } + decoded.WriteByte(high<<4 | low) + index += 3 + } + path := decoded.String() + if !filepath.IsAbs(path) || containsControl(path) { + return "", errors.New("decoded systemd path is invalid") + } + return path, nil +} + +func fromHex(value byte) (byte, bool) { + switch { + case value >= '0' && value <= '9': + return value - '0', true + case value >= 'a' && value <= 'f': + return value - 'a' + 10, true + case value >= 'A' && value <= 'F': + return value - 'A' + 10, true + default: + return 0, false + } +} + +func attestLifecycleSystemdPath(home, path string) (LifecycleFileAttestation, error) { + attestation, _, err := readAndAttestLifecycleSystemdPath(home, path) + return attestation, err +} + +func readAndAttestLifecycleSystemdPath(home, path string) (LifecycleFileAttestation, []byte, error) { + if err := ValidateUserPath(home, path, true); err != nil { + return LifecycleFileAttestation{}, nil, fmt.Errorf("validate effective agent systemd path: %w", err) + } + entry, err := os.Lstat(path) + if err != nil || !entry.Mode().IsRegular() || entry.Size() > MaxStateFileBytes { + return LifecycleFileAttestation{}, nil, errors.New("effective agent systemd input must be a regular file of at most 1 MiB") + } + if err := validateOwner(entry); err != nil { + return LifecycleFileAttestation{}, nil, fmt.Errorf("effective agent systemd input ownership: %w", err) + } + file, err := os.Open(path) + if err != nil { + return LifecycleFileAttestation{}, nil, fmt.Errorf("open effective agent systemd input: %w", err) + } + defer file.Close() + opened, err := file.Stat() + if err != nil || !opened.Mode().IsRegular() || !os.SameFile(entry, opened) || opened.Size() > MaxStateFileBytes { + return LifecycleFileAttestation{}, nil, errors.New("effective agent systemd input changed during open") + } + contents, err := io.ReadAll(io.LimitReader(file, MaxStateFileBytes+1)) + if err != nil { + return LifecycleFileAttestation{}, nil, fmt.Errorf("read effective agent systemd input: %w", err) + } + if len(contents) > MaxStateFileBytes { + return LifecycleFileAttestation{}, nil, errors.New("effective agent systemd input exceeds 1 MiB") + } + digest := digestBytes(contents) + attestation := LifecycleFileAttestation{Path: path, SHA256: digest} + if err := reattestLifecycleFile("input", attestation); err != nil { + return LifecycleFileAttestation{}, nil, err + } + return attestation, contents, nil +} + +func (installer Installer) inspectUnitState(ctx context.Context, unit string) (systemdUnitState, error) { + output, err := installer.run(ctx, Command{Path: "/usr/bin/systemctl", Args: []string{ + "--user", "show", unit, + "--property", "LoadState", "--property", "FragmentPath", + "--property", "ActiveState", "--property", "UnitFileState", + }}) + if err != nil { + return systemdUnitState{}, err + } + properties := map[string]string{} + for _, line := range strings.Split(strings.TrimSpace(string(output)), "\n") { + key, value, found := strings.Cut(line, "=") + if !found || (key != "LoadState" && key != "FragmentPath" && key != "ActiveState" && key != "UnitFileState") { + return systemdUnitState{}, errors.New("systemd unit activation state is malformed") + } + if _, duplicate := properties[key]; duplicate { + return systemdUnitState{}, errors.New("systemd unit activation state contains duplicate properties") + } + properties[key] = value + } + if len(properties) != 4 || properties["LoadState"] == "" || properties["ActiveState"] == "" { + return systemdUnitState{}, errors.New("systemd unit activation state is incomplete") + } + return systemdUnitState{ + LoadState: properties["LoadState"], FragmentPath: properties["FragmentPath"], + ActiveState: properties["ActiveState"], UnitFileState: properties["UnitFileState"], + }, nil +} + +func (installer Installer) captureManagedUnitStates(ctx context.Context) (map[string]systemdUnitState, error) { + states := map[string]systemdUnitState{} + for _, unit := range []string{providerServiceUnit, refreshPathUnit, refreshTimerUnit} { + state, err := installer.inspectUnitState(ctx, unit) + if err != nil { + return nil, fmt.Errorf("inspect %s: %w", unit, err) + } + if state.LoadState == "not-found" { + continue + } + if err := validateRestorableUnitState(state); err != nil { + return nil, fmt.Errorf("inspect %s: %w", unit, err) + } + states[unit] = state + } + return states, nil +} + +func (installer Installer) restoreUnitState(ctx context.Context, unit string, state systemdUnitState) error { + if err := validateRestorableUnitState(state); err != nil { + return err + } + var restoreErr error + switch state.UnitFileState { + case "enabled": + restoreErr = errors.Join(restoreErr, installer.systemctl(ctx, "enable", unit)) + case "enabled-runtime": + restoreErr = errors.Join(restoreErr, installer.systemctl(ctx, "enable", "--runtime", unit)) + case "disabled": + restoreErr = errors.Join(restoreErr, installer.systemctl(ctx, "disable", unit)) + case "static", "indirect", "generated", "transient": + } + switch state.ActiveState { + case "active": + restoreErr = errors.Join(restoreErr, installer.systemctl(ctx, "start", unit)) + case "inactive": + restoreErr = errors.Join(restoreErr, installer.systemctl(ctx, "stop", unit)) + } + return restoreErr +} + +func validateRestorableUnitState(state systemdUnitState) error { + if state.LoadState != "loaded" { + return fmt.Errorf("unsupported prior LoadState %q", state.LoadState) + } + if !filepath.IsAbs(state.FragmentPath) || containsControl(state.FragmentPath) { + return fmt.Errorf("unsupported prior FragmentPath %q", state.FragmentPath) + } + switch state.UnitFileState { + case "enabled", "enabled-runtime", "disabled", "static", "indirect", "generated", "transient": + default: + return fmt.Errorf("unsupported prior UnitFileState %q", state.UnitFileState) + } + switch state.ActiveState { + case "active", "inactive": + default: + return fmt.Errorf("unsupported prior ActiveState %q", state.ActiveState) + } + return nil +} + +func (installer Installer) rollbackInstall(ctx context.Context, config Config, snapshots []managedFileSnapshot, previousUnits map[string]systemdUnitState, agentStopped, maintenanceActive bool, maintenanceID string, activation systemdActivation) error { + return installer.rollbackInstallBeforeStart(ctx, config, snapshots, previousUnits, agentStopped, maintenanceActive, maintenanceID, activation, nil) +} + +func (installer Installer) rollbackInstallBeforeStart(ctx context.Context, config Config, snapshots []managedFileSnapshot, previousUnits map[string]systemdUnitState, agentStopped, maintenanceActive bool, maintenanceID string, activation systemdActivation, beforeStart func(context.Context) error) error { + rollbackContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + if !agentStopped { + cleanupErr := removeSnapshots(snapshots) + if maintenanceActive { + reason := installMaintenanceReason + if maintenanceID == uninstallMaintenanceID { + reason = uninstallMaintenanceReason + } + cleanupErr = errors.Join(cleanupErr, installer.endMaintenance(rollbackContext, config, maintenanceID, reason)) + } + return cleanupErr + } + var rollbackErr error + activatedUnits := make([]string, 0, 3) + if activation.RefreshPath { + activatedUnits = append(activatedUnits, refreshPathUnit) + } + if activation.RefreshTimer { + activatedUnits = append(activatedUnits, refreshTimerUnit) + } + if activation.ProviderService { + activatedUnits = append(activatedUnits, providerServiceUnit) + } + if len(activatedUnits) > 0 { + arguments := append([]string{"disable", "--now"}, activatedUnits...) + rollbackErr = errors.Join(rollbackErr, installer.systemctl(rollbackContext, arguments...)) + } + if err := restoreManagedFileContents(snapshots); err != nil { + rollbackErr = errors.Join(rollbackErr, err) + } else { + rollbackErr = errors.Join(rollbackErr, installer.systemctl(rollbackContext, "daemon-reload")) + for _, unit := range []string{providerServiceUnit, refreshPathUnit, refreshTimerUnit} { + if state, found := previousUnits[unit]; found { + rollbackErr = errors.Join(rollbackErr, installer.restoreUnitState(rollbackContext, unit, state)) + } + } + } + if beforeStart == nil { + rollbackErr = errors.Join(rollbackErr, installer.systemctl(rollbackContext, "start", config.AgentUnit)) + } else if rollbackErr == nil { + if err := beforeStart(rollbackContext); err != nil { + rollbackErr = err + } else { + rollbackErr = installer.systemctl(rollbackContext, "start", config.AgentUnit) + } + } + if rollbackErr == nil && maintenanceActive { + reason := installMaintenanceReason + if maintenanceID == uninstallMaintenanceID { + reason = uninstallMaintenanceReason + } + rollbackErr = installer.endMaintenance(rollbackContext, config, maintenanceID, reason) + } + if rollbackErr == nil { + rollbackErr = removeSnapshots(snapshots) + } + return rollbackErr +} + +func writeInstalledProvider(config Config, paths LifecyclePaths, executable, digest string, material InstallMaterial, units SystemdUnits) error { + if err := AtomicWriteJSON(paths.ConfigFile, config); err != nil { + return fmt.Errorf("write retained provider config: %w", err) + } + if err := replaceRegularFile(executable, paths.Launcher, 0o700, maxProviderPackageBytes); err != nil { + return fmt.Errorf("install retained provider launcher: %w", err) + } + if installedDigest, err := hashRegularFile(paths.Launcher, true); err != nil || installedDigest != digest { + return errors.New("installed retained provider launcher digest mismatch") + } + if err := WriteInstallMaterial(paths, material); err != nil { + return err + } + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + return fmt.Errorf("create provider state directory: %w", err) + } + for _, file := range []struct { + path string + data string + }{ + {paths.ProviderUnit, units.ProviderService}, + {paths.RefreshUnit, units.RefreshService}, + {paths.PathUnit, units.RefreshPath}, + {paths.TimerUnit, units.RefreshTimer}, + {paths.AgentDropIn, units.AgentDropIn}, + } { + if err := atomicWriteFile(file.path, []byte(file.data), 0o600); err != nil { + return fmt.Errorf("write user systemd unit: %w", err) + } + } + return nil +} + +func managedInstallPaths(paths LifecyclePaths) []string { + return append([]string{ + paths.ConfigFile, paths.Launcher, + paths.ActiveState, paths.Journal, + paths.ProviderEnv, paths.ProbeEnv, paths.AgentEnv, + paths.ContainersConf, + paths.CAFile, paths.ServerCert, paths.ServerKey, + }, managedWiringPaths(paths)...) +} + +func snapshotExisted(snapshots []managedFileSnapshot, path string) bool { + for _, snapshot := range snapshots { + if snapshot.Path == path { + return snapshot.Existed + } + } + return false +} + +func previouslyLoadedWatchUnits(states map[string]systemdUnitState) []string { + units := make([]string, 0, 2) + if _, found := states[refreshPathUnit]; found { + units = append(units, refreshPathUnit) + } + if _, found := states[refreshTimerUnit]; found { + units = append(units, refreshTimerUnit) + } + return units +} + +func managedWiringPaths(paths LifecyclePaths) []string { + return []string{paths.ProviderUnit, paths.RefreshUnit, paths.PathUnit, paths.TimerUnit, paths.AgentDropIn} +} + +func managedWiringIntent(paths LifecyclePaths, units SystemdUnits, present bool) []LifecycleManagedFileIntent { + contents := []string{units.ProviderService, units.RefreshService, units.RefreshPath, units.RefreshTimer, units.AgentDropIn} + managed := managedWiringPaths(paths) + intents := make([]LifecycleManagedFileIntent, 0, len(managed)) + for index, path := range managed { + intent := LifecycleManagedFileIntent{Path: path, Present: present} + if present { + intent.Mode = 0o600 + intent.Contents = []byte(contents[index]) + intent.SHA256 = digestBytes(intent.Contents) + } + intents = append(intents, intent) + } + return intents +} + +func snapshotManagedFiles(paths LifecyclePaths) ([]managedFileSnapshot, error) { + if err := os.MkdirAll(paths.Root, 0o700); err != nil { + return nil, err + } + backupRoot, err := os.MkdirTemp(paths.Root, ".install-backup-") + if err != nil { + return nil, err + } + if err := os.Chmod(backupRoot, 0o700); err != nil { + _ = os.RemoveAll(backupRoot) + return nil, err + } + snapshots := make([]managedFileSnapshot, 0, len(managedInstallPaths(paths))) + for index, path := range managedInstallPaths(paths) { + snapshot := managedFileSnapshot{Path: path, Backup: filepath.Join(backupRoot, strconv.Itoa(index))} + info, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) { + snapshots = append(snapshots, snapshot) + continue + } + if err != nil || !info.Mode().IsRegular() { + return nil, errors.Join(errors.New("managed install path must be a regular file"), os.RemoveAll(backupRoot)) + } + if err := validateOwner(info); err != nil { + return nil, errors.Join(err, os.RemoveAll(backupRoot)) + } + snapshot.Existed = true + snapshot.Mode = info.Mode().Perm() + if err := replaceRegularFile(path, snapshot.Backup, snapshot.Mode, maxProviderPackageBytes); err != nil { + return nil, errors.Join(err, os.RemoveAll(backupRoot)) + } + snapshot.SHA256, err = hashRegularFile(snapshot.Backup, snapshot.Mode&0o100 != 0) + if err != nil { + return nil, errors.Join(err, os.RemoveAll(backupRoot)) + } + snapshots = append(snapshots, snapshot) + } + return snapshots, nil +} + +func snapshotManagedFilesAt(paths LifecyclePaths, transactionRoot string) ([]managedFileSnapshot, error) { + if filepath.Dir(transactionRoot) != paths.LifecycleTransactions || !safeIdentifierPattern.MatchString(filepath.Base(transactionRoot)) { + return nil, errors.New("lifecycle snapshot transaction root is invalid") + } + info, err := os.Lstat(transactionRoot) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm() != 0o700 { + return nil, errors.New("lifecycle snapshot transaction root must be an owner-only real directory") + } + if err := validateOwner(info); err != nil { + return nil, fmt.Errorf("lifecycle snapshot transaction root ownership: %w", err) + } + snapshotRoot := filepath.Join(transactionRoot, "snapshots") + if err := os.Mkdir(snapshotRoot, 0o700); err != nil { + return nil, fmt.Errorf("create lifecycle snapshot root: %w", err) + } + cleanup := func(cause error) ([]managedFileSnapshot, error) { + return nil, errors.Join(cause, os.RemoveAll(snapshotRoot), syncDirectory(transactionRoot)) + } + if err := syncDirectory(transactionRoot); err != nil { + return cleanup(err) + } + snapshots := make([]managedFileSnapshot, 0, len(managedInstallPaths(paths))) + for index, path := range managedInstallPaths(paths) { + snapshot := managedFileSnapshot{Path: path, Backup: filepath.Join(snapshotRoot, strconv.Itoa(index))} + entry, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) { + snapshots = append(snapshots, snapshot) + continue + } + if err != nil || !entry.Mode().IsRegular() { + return cleanup(errors.New("managed lifecycle snapshot path must be a regular file")) + } + if err := validateOwner(entry); err != nil { + return cleanup(err) + } + snapshot.Existed = true + snapshot.Mode = entry.Mode().Perm() + if snapshot.Mode != 0o600 && snapshot.Mode != 0o700 { + return cleanup(errors.New("managed lifecycle snapshot path must be owner-only")) + } + if err := replaceRegularFile(path, snapshot.Backup, snapshot.Mode, maxProviderPackageBytes); err != nil { + return cleanup(err) + } + snapshot.SHA256, err = hashRegularFile(snapshot.Backup, snapshot.Mode&0o100 != 0) + if err != nil { + return cleanup(err) + } + snapshots = append(snapshots, snapshot) + } + if err := syncDirectory(snapshotRoot); err != nil { + return cleanup(err) + } + return snapshots, nil +} + +func snapshotManagedFilesForLifecycle(home string, paths LifecyclePaths, journal *LifecycleJournal, now time.Time) error { + if journal == nil || journal.Phase != LifecycleFencing || len(journal.Snapshots) != 0 { + return errors.New("lifecycle snapshot journal must be in an empty fencing phase") + } + transactionRoot := paths.LifecycleTransactionRoot(journal.TransactionID) + if filepath.Dir(transactionRoot) != paths.LifecycleTransactions || !safeIdentifierPattern.MatchString(filepath.Base(transactionRoot)) { + return errors.New("lifecycle snapshot transaction root is invalid") + } + info, err := os.Lstat(transactionRoot) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm() != 0o700 { + return errors.New("lifecycle snapshot transaction root must be an owner-only real directory") + } + if err := validateOwner(info); err != nil { + return fmt.Errorf("lifecycle snapshot transaction root ownership: %w", err) + } + snapshotRoot := filepath.Join(transactionRoot, "snapshots") + if err := os.Mkdir(snapshotRoot, 0o700); err != nil { + return fmt.Errorf("create lifecycle snapshot root: %w", err) + } + if err := syncDirectory(transactionRoot); err != nil { + return err + } + for index, path := range managedInstallPaths(paths) { + snapshot := managedFileSnapshot{Path: path, Backup: filepath.Join(snapshotRoot, strconv.Itoa(index))} + entry, err := os.Lstat(path) + switch { + case errors.Is(err, os.ErrNotExist): + case err != nil || !entry.Mode().IsRegular(): + return errors.New("managed lifecycle snapshot path must be a regular file") + default: + if err := validateOwner(entry); err != nil { + return err + } + snapshot.Existed = true + snapshot.Mode = entry.Mode().Perm() + if snapshot.Mode != 0o600 && snapshot.Mode != 0o700 { + return errors.New("managed lifecycle snapshot path must be owner-only") + } + if err := replaceRegularFile(path, snapshot.Backup, snapshot.Mode, maxProviderPackageBytes); err != nil { + return err + } + snapshot.SHA256, err = hashRegularFile(snapshot.Backup, snapshot.Mode&0o100 != 0) + if err != nil { + return err + } + } + journal.Snapshots = append(journal.Snapshots, snapshot) + journal.UpdatedAt = now.UTC() + if err := writeLifecycleJournal(home, paths, *journal); err != nil { + return fmt.Errorf("record lifecycle snapshot: %w", err) + } + } + return syncDirectory(snapshotRoot) +} + +func restoreManagedFiles(snapshots []managedFileSnapshot) error { + if err := restoreManagedFileContents(snapshots); err != nil { + return err + } + return removeSnapshots(snapshots) +} + +func restoreManagedFileContents(snapshots []managedFileSnapshot) error { + var restoreErr error + for _, snapshot := range snapshots { + if !snapshot.Existed { + restoreErr = errors.Join(restoreErr, removeDurableFile(snapshot.Path)) + continue + } + restoreErr = errors.Join(restoreErr, replaceRegularFile(snapshot.Backup, snapshot.Path, snapshot.Mode, maxProviderPackageBytes)) + } + if restoreErr != nil { + return restoreErr + } + return nil +} + +func removeSnapshots(snapshots []managedFileSnapshot) error { + if len(snapshots) == 0 { + return nil + } + return os.RemoveAll(filepath.Dir(snapshots[0].Backup)) +} + +func replaceRegularFile(source, destination string, mode os.FileMode, maxBytes int64) (returnErr error) { + if maxBytes <= 0 || mode&^os.FileMode(0o777) != 0 || mode&0o077 != 0 { + return errors.New("replacement file mode or size limit is invalid") + } + sourceInfo, err := os.Lstat(source) + if err != nil || !sourceInfo.Mode().IsRegular() || sourceInfo.Size() <= 0 || sourceInfo.Size() > maxBytes { + return errors.New("replacement source must be a bounded regular file") + } + input, err := os.Open(source) + if err != nil { + return err + } + defer input.Close() + opened, err := input.Stat() + if err != nil || !opened.Mode().IsRegular() || !os.SameFile(sourceInfo, opened) { + return errors.New("replacement source changed during open") + } + directory := filepath.Dir(destination) + if err := os.MkdirAll(directory, 0o700); err != nil { + return err + } + if err := rejectWritableDestination(destination); err != nil { + return err + } + temporary, err := os.CreateTemp(directory, ".retained-provider-copy-*.tmp") + if err != nil { + return err + } + temporaryPath := temporary.Name() + defer func() { + _ = temporary.Close() + if err := os.Remove(temporaryPath); returnErr == nil && err != nil && !errors.Is(err, os.ErrNotExist) { + returnErr = err + } + }() + if err := temporary.Chmod(mode); err != nil { + return err + } + if _, err := io.CopyN(temporary, input, sourceInfo.Size()); err != nil { + return err + } + var extra [1]byte + if count, err := input.Read(extra[:]); count != 0 || err != io.EOF { + return errors.New("replacement source size changed during copy") + } + if err := temporary.Sync(); err != nil { + return err + } + if err := temporary.Close(); err != nil { + return err + } + if err := os.Rename(temporaryPath, destination); err != nil { + return err + } + return syncDirectory(directory) +} + +func (installer Installer) now() time.Time { + if installer.Now != nil { + return installer.Now().UTC() + } + return time.Now().UTC() +} + +func (installer Installer) run(ctx context.Context, command Command) ([]byte, error) { + return runBoundedCommand(ctx, installer.Runner, command) +} + +func (installer Installer) sleep(ctx context.Context, duration time.Duration) error { + if installer.Sleep != nil { + return installer.Sleep(ctx, duration) + } + timer := time.NewTimer(duration) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} diff --git a/internal/retainedprovider/systemd_test.go b/internal/retainedprovider/systemd_test.go new file mode 100644 index 0000000..96278e3 --- /dev/null +++ b/internal/retainedprovider/systemd_test.go @@ -0,0 +1,1890 @@ +package retainedprovider + +import ( + "bytes" + "context" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "reflect" + "strconv" + "strings" + "testing" + "time" +) + +func TestRenderSystemdUnitsUsesStableAbsolutePathsAndNoShell(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + units, err := RenderSystemdUnits(config, paths) + if err != nil { + t.Fatalf("render units: %v", err) + } + + for name, unit := range map[string]string{ + "provider": units.ProviderService, + "refresh": units.RefreshService, + "path": units.RefreshPath, + "timer": units.RefreshTimer, + "drop-in": units.AgentDropIn, + } { + if strings.Contains(unit, "ExecStart=/bin/sh") || strings.Contains(unit, "ExecStart=\"/bin/sh\"") || strings.Contains(unit, " /bin/sh ") || strings.Contains(unit, "provider-secret") || strings.Contains(unit, "github-secret") { + t.Fatalf("%s unit contains shell or secret: %s", name, unit) + } + } + for _, required := range []string{ + "ExecStart=" + systemdQuote(paths.Launcher) + " retained serve-active -config " + systemdQuote(paths.ConfigFile), + "Restart=on-failure", + } { + if !strings.Contains(units.ProviderService, required) { + t.Fatalf("provider unit missing %q:\n%s", required, units.ProviderService) + } + } + if !strings.Contains(units.RefreshService, "ExecStart="+systemdQuote(paths.Launcher)+" retained refresh -config "+systemdQuote(paths.ConfigFile)) || !strings.Contains(units.RefreshService, "Type=oneshot") || !strings.Contains(units.RefreshService, "TimeoutStartSec=15min") { + t.Fatalf("refresh service = %s", units.RefreshService) + } + if !strings.Contains(units.RefreshPath, "PathChanged="+systemdPathValue(config.ProviderMarkerPath)) || !strings.Contains(units.RefreshPath, "Unit="+refreshServiceUnit) { + t.Fatalf("refresh path = %s", units.RefreshPath) + } + if !strings.Contains(units.RefreshTimer, "OnBootSec=30s") || !strings.Contains(units.RefreshTimer, "OnUnitInactiveSec=300s") || strings.Contains(units.RefreshTimer, "OnUnitActiveSec=") || !strings.Contains(units.RefreshTimer, "Persistent=true") { + t.Fatalf("refresh timer = %s", units.RefreshTimer) + } + if !strings.Contains(units.AgentDropIn, "EnvironmentFile="+systemdPathValue(paths.AgentEnv)) || strings.Contains(units.AgentDropIn, `EnvironmentFile="`) || strings.Contains(units.AgentDropIn, paths.ProviderEnv) { + t.Fatalf("agent drop-in = %s", units.AgentDropIn) + } + if strings.Contains(units.ProviderService, "--network") || strings.Contains(units.ProviderService, "podman") || strings.Contains(units.ProviderService, "EnvironmentFile=") { + t.Fatalf("provider unit bypasses serve-active/env boundary: %s", units.ProviderService) + } + for name, unit := range map[string]string{"provider": units.ProviderService, "refresh": units.RefreshService} { + if strings.Contains(unit, "NoNewPrivileges=true") || strings.Contains(unit, "PrivateTmp=true") { + t.Fatalf("%s unit blocks rootless Podman namespace setup: %s", name, unit) + } + } +} + +func TestSystemdQuoteEscapesSpecifierExpansion(t *testing.T) { + if got, want := systemdQuote(`/home/user%name/"provider"`), `"/home/user%%name/\"provider\""`; got != want { + t.Fatalf("systemdQuote = %q want %q", got, want) + } +} + +func TestRenderSystemdUnitsEscapesPathSettingWithoutGenericQuotes(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + config.ProviderMarkerPath = filepath.Join(home, `updates/current provider%marker`) + units, err := RenderSystemdUnits(config, LifecyclePathsFor(config)) + if err != nil { + t.Fatalf("render units: %v", err) + } + want := "PathChanged=" + strings.ReplaceAll(strings.ReplaceAll(config.ProviderMarkerPath, " ", `\x20`), "%", "%%") + if !strings.Contains(units.RefreshPath, want) { + t.Fatalf("refresh path missing %q:\n%s", want, units.RefreshPath) + } + if strings.Contains(units.RefreshPath, `PathChanged="`) { + t.Fatalf("refresh path used ExecStart-style quoting:\n%s", units.RefreshPath) + } +} + +func TestGenerateInstallMaterialSeparatesProviderProbeAndAgentSecrets(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + credentials := Credentials{GitHubToken: "github-secret", ProviderToken: "provider-secret"} + now := time.Unix(1_700_000_000, 0).UTC() + material, err := GenerateInstallMaterial(config, credentials, bytes.NewReader(bytes.Repeat([]byte{0x42}, 4096)), now) + if err != nil { + t.Fatalf("generate install material: %v", err) + } + if !strings.Contains(string(material.ProviderEnv), "GITHUB_RUNNER_PROVIDER_GITHUB_TOKEN=github-secret") || !strings.Contains(string(material.ProviderEnv), "GITHUB_RUNNER_PROVIDER_TOKEN=provider-secret") { + t.Fatalf("provider env missing credentials: %s", material.ProviderEnv) + } + if strings.Contains(string(material.ProbeEnv), "github-secret") || string(material.ProbeEnv) != "GITHUB_RUNNER_PROVIDER_TOKEN=provider-secret\n" { + t.Fatalf("probe env scope = %s", material.ProbeEnv) + } + agentText := string(material.AgentEnv) + for key, expected := range map[string]string{ + "WORKFLOW_COMPUTE_DYNAMIC_PROVIDER_GITHUB_ACTIONS_RUNNER_ENV_KEYS": "COMPUTE_GITHUB_RUNNER_PROVIDER_URL,COMPUTE_GITHUB_RUNNER_PROVIDER_TOKEN,COMPUTE_GITHUB_RUNNER_PROVIDER_CA_CERT_B64", + "COMPUTE_GITHUB_RUNNER_PROVIDER_URL": config.ProviderURL, + "COMPUTE_GITHUB_RUNNER_PROVIDER_TOKEN": "provider-secret", + "CONTAINERS_CONF": paths.ContainersConf, + } { + if got := systemdEnvironmentValue(agentText, key); got != expected { + t.Fatalf("agent env %s = %q want %q: %s", key, got, expected, agentText) + } + } + if strings.Contains(agentText, "github-secret") || strings.Contains(agentText, "GITHUB_RUNNER_PROVIDER_GITHUB_TOKEN") { + t.Fatalf("agent env contains GitHub credential: %s", agentText) + } + encodedCA := systemdEnvironmentValue(agentText, "COMPUTE_GITHUB_RUNNER_PROVIDER_CA_CERT_B64") + decodedCA, err := base64.StdEncoding.DecodeString(encodedCA) + if err != nil || !bytes.Equal(decodedCA, material.CACert) { + t.Fatalf("agent CA does not match generated CA: err=%v", err) + } + if bytes.Equal(material.ProviderEnv, material.AgentEnv) || bytes.Equal(material.ProviderEnv, material.ProbeEnv) { + t.Fatal("Podman and systemd environment files were not rendered separately") + } + + ca := parseCertificateForTest(t, material.CACert) + server := parseCertificateForTest(t, material.ServerCert) + if !ca.IsCA || server.NotBefore.After(now) || server.NotAfter.Before(now.Add(24*time.Hour)) { + t.Fatalf("certificate validity CA=%+v server=%+v", ca, server) + } + for _, dns := range []string{"localhost", config.StableContainer, config.CandidateContainer} { + if !containsString(server.DNSNames, dns) { + t.Fatalf("server certificate missing DNS SAN %q: %v", dns, server.DNSNames) + } + } + for _, ip := range []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")} { + if !containsIP(server.IPAddresses, ip) { + t.Fatalf("server certificate missing IP SAN %s: %v", ip, server.IPAddresses) + } + } + pool := x509.NewCertPool() + pool.AddCert(ca) + if _, err := server.Verify(x509.VerifyOptions{Roots: pool, DNSName: config.StableContainer, CurrentTime: now.Add(time.Hour)}); err != nil { + t.Fatalf("verify server certificate: %v", err) + } + + if err := WriteInstallMaterial(paths, material); err != nil { + t.Fatalf("write install material: %v", err) + } + for _, path := range []string{paths.ProviderEnv, paths.ProbeEnv, paths.AgentEnv, paths.ContainersConf, paths.CAFile, paths.ServerCert, paths.ServerKey} { + info, err := os.Stat(path) + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != 0o600 { + t.Fatalf("generated file %s mode=%v err=%v", path, info, err) + } + } + if data, err := os.ReadFile(paths.ContainersConf); err != nil || string(data) != "[network]\ndefault_network = \"wfcompute-github-provider\"\n" { + t.Fatalf("containers.conf data=%q err=%v", data, err) + } +} + +func TestRenderSystemdEnvironmentQuotesSpecialCharacters(t *testing.T) { + contents, err := renderSystemdEnvironment([]environmentValue{{Name: "PROVIDER_TOKEN", Value: `provider token"\tail#value`}}) + if err != nil { + t.Fatalf("render systemd environment: %v", err) + } + if got, want := string(contents), "PROVIDER_TOKEN=\"provider token\\\"\\\\tail#value\"\n"; got != want { + t.Fatalf("systemd environment = %q want %q", got, want) + } +} + +func TestGenerateInstallMaterialRejectsCredentialInjection(t *testing.T) { + config := validTestConfig(t.TempDir()) + for _, credentials := range []Credentials{ + {GitHubToken: "", ProviderToken: "provider-token"}, + {GitHubToken: "github-token", ProviderToken: ""}, + {GitHubToken: "github\nTOKEN=forged", ProviderToken: "provider-token"}, + {GitHubToken: "github-token", ProviderToken: "provider\rTOKEN=forged"}, + } { + if _, err := GenerateInstallMaterial(config, credentials, bytes.NewReader(bytes.Repeat([]byte{0x24}, 4096)), time.Now().UTC()); err == nil { + t.Fatalf("credential injection accepted: %+v", credentials) + } + } +} + +func TestInspectAgentUnitSignatureParsesAndAttestsSystemdShow(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + fragment := filepath.Join(home, ".config", "systemd", "user", config.AgentUnit) + dropIn := filepath.Join(home, ".config", "systemd", "user", config.AgentUnit+".d", "20-provider.conf") + environment := filepath.Join(home, ".workflow-compute", "agent.env") + for path, contents := range map[string]string{ + fragment: "[Service]\nExecStart=" + config.ComputeAgentPath + " run\n", + dropIn: "[Service]\nEnvironmentFile=" + environment + "\n", + environment: "WORKER_ID=" + config.WorkerID + "\n", + } { + if err := atomicWriteFile(path, []byte(contents), 0o600); err != nil { + t.Fatalf("write systemd signature fixture: %v", err) + } + } + output := strings.Join([]string{ + "LoadState=loaded", + "FragmentPath=" + fragment, + "DropInPaths=" + dropIn, + }, "\n") + "\n" + runner := &recordingCommandRunner{run: func(_ context.Context, command Command) ([]byte, error) { + if filepath.Base(command.Path) != "systemctl" || !containsArg(command.Args, config.AgentUnit) || !containsAdjacentArgs(command.Args, "--property", "DropInPaths") { + t.Fatalf("signature command = %+v", command) + } + if containsAdjacentArgs(command.Args, "--property", "EnvironmentFiles") { + t.Fatalf("signature command requested unavailable EnvironmentFiles property: %+v", command) + } + if containsAdjacentArgs(command.Args, "--property", "ExecStart") { + t.Fatalf("signature command requested runtime-varying ExecStart property: %+v", command) + } + return []byte(output), nil + }} + installer := Installer{Runner: runner} + signature, err := installer.inspectAgentUnitSignature(t.Context(), home, config) + if err != nil { + t.Fatalf("inspect agent unit signature: %v", err) + } + if signature.Fragment.Path != fragment || len(signature.DropIns) != 1 || signature.DropIns[0].Path != dropIn || len(signature.EnvironmentFiles) != 1 || signature.EnvironmentFiles[0].Path != environment { + t.Fatalf("signature = %+v", signature) + } + if want := `["` + config.ComputeAgentPath + ` run"]`; signature.ExecStart != want { + t.Fatalf("static ExecStart = %q want %q", signature.ExecStart, want) + } + if err := signature.Reattest(); err != nil { + t.Fatalf("re-attest inspected signature: %v", err) + } + + runner.run = func(_ context.Context, _ Command) ([]byte, error) { + return []byte(output + "FragmentPath=" + fragment + "\n"), nil + } + if _, err := installer.inspectAgentUnitSignature(t.Context(), home, config); err == nil || !strings.Contains(err.Error(), "duplicate") { + t.Fatalf("duplicate systemd property error = %v", err) + } +} + +func TestEffectiveSystemdEnvironmentFilesHonorsResetAndQuotedPaths(t *testing.T) { + paths, err := effectiveSystemdEnvironmentFiles([][]byte{ + []byte("[Service]\nEnvironmentFile=/home/runner/old.env\n"), + []byte("[Service]\nEnvironmentFile=\nEnvironmentFile=\"/home/runner/env files/agent%%active.env\"\n"), + }) + if err != nil { + t.Fatalf("derive effective environment files: %v", err) + } + want := []string{"/home/runner/env files/agent%active.env"} + if !reflect.DeepEqual(paths, want) { + t.Fatalf("environment files = %q want %q", paths, want) + } +} + +func TestEffectiveSystemdExecStartUsesStaticResetAwareCommands(t *testing.T) { + value, err := effectiveSystemdExecStart([][]byte{ + []byte("[Service]\nExecStart=/usr/bin/old-agent run\n"), + []byte("[Service]\nExecStart=\nExecStart=/usr/bin/current-agent run --profile retained\n"), + }) + if err != nil { + t.Fatalf("derive effective ExecStart: %v", err) + } + if want := `["/usr/bin/current-agent run --profile retained"]`; value != want { + t.Fatalf("static ExecStart = %q want %q", value, want) + } +} + +func TestEffectiveSystemdEnvironmentFilesRejectsUnattestablePaths(t *testing.T) { + for _, value := range []string{ + "-/home/runner/optional.env", + "/home/runner/*.env", + "/home/%h/agent.env", + `"/home/runner/trailing\`, + } { + t.Run(value, func(t *testing.T) { + _, err := effectiveSystemdEnvironmentFiles([][]byte{[]byte("[Service]\nEnvironmentFile=" + value + "\n")}) + if err == nil { + t.Fatalf("unsupported EnvironmentFile path %q was accepted", value) + } + }) + } +} + +func TestInstallTransactionOrdersMaintenanceAgentAndProviderActivation(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + writeLifecycleRecoveryFiles(t, config) + paths := LifecyclePathsFor(config) + payload := writeTestProviderPayload(t, home, "verified-provider-install") + digest := fileDigestForTest(t, payload) + events := make([]string, 0, 24) + statusQueue := []string{"unavailable", "unavailable", "idle"} + runner := &recordingCommandRunner{} + runner.run = func(_ context.Context, command Command) ([]byte, error) { + if command.Path == config.PodmanPath && len(command.Args) >= 2 && command.Args[0] == "image" && command.Args[1] == "inspect" { + return []byte(testProviderImageID + "\n"), nil + } + if filepath.Base(command.Path) == "systemctl" && containsArg(command.Args, "show") && (containsArg(command.Args, providerServiceUnit) || containsArg(command.Args, refreshPathUnit) || containsArg(command.Args, refreshTimerUnit)) { + if containsAdjacentArgs(command.Args, "--property", "ActiveState") && containsArg(command.Args, "--value") { + return []byte("active\n"), nil + } + return []byte("LoadState=not-found\nFragmentPath=\nActiveState=inactive\nUnitFileState=\n"), nil + } + if command.Path == config.PodmanPath && len(command.Args) >= 2 && command.Args[0] == "network" && command.Args[1] == "inspect" { + return []byte("bridge true false\n"), nil + } + event := installCommandEvent(command, config) + events = append(events, event) + switch event { + case "agent-signature": + return agentUnitSystemdOutputForTest(t, config), nil + case "verify-update": + return testVerifiedUpdateJSON(config, payload, digest), nil + case "maintenance-begin": + journal, found, err := readLifecycleJournal(home, paths) + if err != nil || !found || journal.Phase != LifecycleFencing { + t.Fatalf("install maintenance begin lifecycle = %+v found=%v err=%v", journal, found, err) + } + return maintenanceStateJSON(true, installMaintenanceID, config.ProfileID, installMaintenanceReason), nil + case "maintenance-end": + journal, found, err := readLifecycleJournal(home, paths) + if err != nil || !found || journal.Phase != LifecycleReleasing || journal.Outcome != LifecycleCommit || journal.ProviderTransaction == nil { + t.Fatalf("install maintenance end lifecycle = %+v found=%v err=%v", journal, found, err) + } + return maintenanceStateJSON(false, installMaintenanceID, config.ProfileID, installMaintenanceReason), nil + case "local-status": + if len(statusQueue) == 0 { + t.Fatal("unexpected extra local status read") + } + state := statusQueue[0] + statusQueue = statusQueue[1:] + return localStatusJSON(config.WorkerID, state), nil + default: + if event == "agent-stop" { + journal, found, err := readLifecycleJournal(home, paths) + if err != nil || !found || journal.Phase != LifecycleFenced { + t.Fatalf("install agent stop lifecycle = %+v found=%v err=%v", journal, found, err) + } + } + if isCandidateStart(command, config) { + events = append(events, "provider-refresh") + } + return nil, nil + } + } + installer := Installer{ + Runner: runner, + ExecutablePath: func() (string, error) { return payload, nil }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x31}, 4096)), + Now: func() time.Time { return time.Unix(1_700_000_000, 0).UTC() }, + Sleep: func(context.Context, time.Duration) error { return nil }, + } + status, err := installer.Install(t.Context(), home, config, Credentials{GitHubToken: "github-secret", ProviderToken: "provider-secret"}) + if err != nil { + t.Fatalf("install: %v\nevents=%v", err, events) + } + if status.ProtocolVersion != StatusProtocolVersion || !status.Installed || !status.ServiceActive || status.CurrentSHA256 != digest { + t.Fatalf("install status = %+v", status) + } + assertOrderedEvents(t, events, []string{ + "systemd-preflight", "podman-preflight", "supervisor-config-validate", "verify-update", + "maintenance-begin", "local-status", "agent-stop", "daemon-reload", "provider-enable", + "provider-refresh", "refresh-watch-enable", "agent-start", + "local-status", "maintenance-end", "local-status", + }) + transcript := commandTranscript(runner.commands) + if strings.Contains(transcript, "github-secret") || strings.Contains(transcript, "provider-secret") || strings.Contains(transcript, "COMPUTE_API_TOKEN") || strings.Contains(transcript, "https://stg") { + t.Fatalf("install command transcript leaked credential or STG access:\n%s", transcript) + } + for _, path := range []string{ + paths.ConfigFile, paths.Launcher, paths.ProviderEnv, paths.ProbeEnv, paths.AgentEnv, + paths.CAFile, paths.ServerCert, paths.ServerKey, + paths.ProviderUnit, paths.RefreshUnit, paths.PathUnit, paths.TimerUnit, paths.AgentDropIn, + } { + if info, err := os.Stat(path); err != nil || !info.Mode().IsRegular() { + t.Fatalf("installed file %s info=%v err=%v", path, info, err) + } + } + if data, err := os.ReadFile(paths.ProviderEnv); err != nil || !bytes.Contains(data, []byte("github-secret")) { + t.Fatalf("provider credential file data=%q err=%v", data, err) + } + if data, err := os.ReadFile(paths.AgentEnv); err != nil || bytes.Contains(data, []byte("github-secret")) { + t.Fatalf("agent credential file data=%q err=%v", data, err) + } +} + +func TestInstallReattestsAuthorityAfterDrainBeforeStop(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + payload := writeTestProviderPayload(t, home, "verified-provider-install-reattest") + digest := fileDigestForTest(t, payload) + runner := installSuccessRunner(t, config, payload, digest, new([]string)) + originalRun := runner.run + replaced := false + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + output, err := originalRun(ctx, command) + if err == nil && !replaced && installCommandEvent(command, config) == "local-status" { + replaced = true + if writeErr := os.WriteFile(config.ComputeAgentPath, []byte("replacement during install drain"), 0o700); writeErr != nil { + t.Fatalf("replace compute-agent during install drain: %v", writeErr) + } + } + return output, err + } + installer := Installer{ + Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x31}, 4096)), + Sleep: func(context.Context, time.Duration) error { return nil }, + } + _, err := installer.Install(t.Context(), home, config, Credentials{GitHubToken: "github-secret", ProviderToken: "provider-secret"}) + if err == nil || !strings.Contains(err.Error(), "attestation") { + t.Fatalf("install error = %v", err) + } + transcript := commandTranscript(runner.commands) + if strings.Contains(transcript, "systemctl --user stop "+config.AgentUnit) { + t.Fatalf("changed authority crossed install stop boundary:\n%s", transcript) + } +} + +func TestInstallReattestsAuthorityBeforeRestart(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + payload := writeTestProviderPayload(t, home, "verified-provider-install-restart-reattest") + digest := fileDigestForTest(t, payload) + runner := installSuccessRunner(t, config, payload, digest, new([]string)) + originalRun := runner.run + replaced := false + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + output, err := originalRun(ctx, command) + if err == nil && !replaced && installCommandEvent(command, config) == "refresh-watch-enable" { + replaced = true + if writeErr := os.WriteFile(config.ComputeAgentPath, []byte("replacement before install restart"), 0o700); writeErr != nil { + t.Fatalf("replace compute-agent before restart: %v", writeErr) + } + } + return output, err + } + installer := Installer{ + Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x31}, 4096)), + Sleep: func(context.Context, time.Duration) error { return nil }, + } + _, err := installer.Install(t.Context(), home, config, Credentials{GitHubToken: "github-secret", ProviderToken: "provider-secret"}) + if err == nil || !strings.Contains(err.Error(), "attestation") { + t.Fatalf("install error = %v", err) + } + transcript := commandTranscript(runner.commands) + if strings.Contains(transcript, "systemctl --user start "+config.AgentUnit) { + t.Fatalf("changed authority crossed install restart boundary:\n%s", transcript) + } +} + +func TestInstallReattestsAuthorityBeforeReadyAndMaintenanceEnd(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + payload := writeTestProviderPayload(t, home, "verified-provider-install-release-reattest") + digest := fileDigestForTest(t, payload) + runner := installSuccessRunner(t, config, payload, digest, new([]string)) + originalRun := runner.run + localStatusReads := 0 + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + output, err := originalRun(ctx, command) + if err == nil && installCommandEvent(command, config) == "local-status" { + localStatusReads++ + if localStatusReads == 2 { + if writeErr := os.WriteFile(config.ComputeAgentPath, []byte("replacement before install release"), 0o700); writeErr != nil { + t.Fatalf("replace compute-agent before release: %v", writeErr) + } + } + } + return output, err + } + installer := Installer{ + Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x31}, 4096)), + Sleep: func(context.Context, time.Duration) error { return nil }, + } + _, err := installer.Install(t.Context(), home, config, Credentials{GitHubToken: "github-secret", ProviderToken: "provider-secret"}) + if err == nil || !strings.Contains(err.Error(), "attestation") { + t.Fatalf("install error = %v", err) + } + transcript := commandTranscript(runner.commands) + if strings.Contains(transcript, "supervisor-maintenance end") { + t.Fatalf("changed authority crossed maintenance release boundary:\n%s", transcript) + } + journal, found, readErr := readLifecycleJournal(home, LifecyclePathsFor(config)) + if readErr != nil || !found || journal.Phase != LifecycleFenced { + t.Fatalf("failed release journal = %+v found=%v err=%v", journal, found, readErr) + } +} + +func TestReleaseLifecycleMaintenanceReattestsAuthorityBeforeCommand(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + writeLifecycleRecoveryFiles(t, config) + journal := lifecycleRecoveryJournalForTest(t, config, time.Now().UTC()) + journal.Operation = LifecycleInstall + if err := os.WriteFile(config.ComputeAgentPath, []byte("replacement before maintenance release"), 0o700); err != nil { + t.Fatalf("replace compute-agent before maintenance release: %v", err) + } + runner := &recordingCommandRunner{} + installer := Installer{Runner: runner} + + err := installer.releaseLifecycleMaintenance(t.Context(), home, journal) + if err == nil || !strings.Contains(err.Error(), "attestation") { + t.Fatalf("release error = %v", err) + } + if transcript := commandTranscript(runner.commands); strings.Contains(transcript, "supervisor-maintenance end") { + t.Fatalf("changed authority crossed maintenance release command:\n%s", transcript) + } +} + +func TestInstallBoundsEverySubprocessContext(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + payload := writeTestProviderPayload(t, home, "verified-provider-bounded-install-commands") + digest := fileDigestForTest(t, payload) + statuses := []string{"unavailable", "unavailable", "idle"} + runner := installSuccessRunner(t, config, payload, digest, &statuses) + baseRun := runner.run + var unbounded []Command + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if _, bounded := ctx.Deadline(); !bounded { + unbounded = append(unbounded, command) + } + return baseRun(ctx, command) + } + installer := Installer{ + Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x32}, 4096)), Sleep: func(context.Context, time.Duration) error { return nil }, + } + if _, err := installer.Install(t.Context(), home, config, Credentials{GitHubToken: "github-secret", ProviderToken: "provider-secret"}); err != nil { + t.Fatalf("install: %v", err) + } + if len(unbounded) != 0 { + t.Fatalf("install issued unbounded subprocesses: %s", commandTranscript(unbounded)) + } +} + +func TestInstallLockContentionDoesNotMutateMaintenanceOrAgent(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + payload := writeTestProviderPayload(t, home, "verified-provider-install-lock") + digest := fileDigestForTest(t, payload) + paths := LifecyclePathsFor(config) + lock, err := AcquireInstallLock(paths.InstallLock) + if err != nil { + t.Fatalf("hold install lock: %v", err) + } + defer lock.Release() + statuses := []string{"unavailable", "unavailable"} + runner := installSuccessRunner(t, config, payload, digest, &statuses) + installer := Installer{ + Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x32}, 4096)), Sleep: func(context.Context, time.Duration) error { return nil }, + Refresh: func(context.Context, Config) (Status, error) { return Status{}, nil }, + ProbeActive: func(context.Context, Config) error { return nil }, + } + if _, err := installer.Install(t.Context(), home, config, Credentials{GitHubToken: "github-secret", ProviderToken: "provider-secret"}); !errors.Is(err, ErrInstallLocked) { + t.Fatalf("contended install err = %v", err) + } + transcript := commandTranscript(runner.commands) + if strings.Contains(transcript, "supervisor-maintenance begin") || strings.Contains(transcript, "systemctl --user stop "+config.AgentUnit) { + t.Fatalf("contended install mutated maintenance or agent:\n%s", transcript) + } +} + +func TestInstallHoldsLifecycleLockUntilMaintenanceFenceReleased(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + payload := writeTestProviderPayload(t, home, "verified-provider-lifecycle-lock") + digest := fileDigestForTest(t, payload) + paths := LifecyclePathsFor(config) + statuses := []string{"unavailable", "unavailable", "idle"} + runner := installSuccessRunner(t, config, payload, digest, &statuses) + baseRun := runner.run + lockHeldAtFenceRelease := false + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if installCommandEvent(command, config) == "maintenance-end" { + contender, err := AcquireInstallLock(paths.InstallLock) + lockHeldAtFenceRelease = errors.Is(err, ErrInstallLocked) + if contender != nil { + _ = contender.Release() + } + } + return baseRun(ctx, command) + } + installer := Installer{ + Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x33}, 4096)), Sleep: func(context.Context, time.Duration) error { return nil }, + } + if _, err := installer.Install(t.Context(), home, config, Credentials{GitHubToken: "github-secret", ProviderToken: "provider-secret"}); err != nil { + t.Fatalf("install: %v", err) + } + if !lockHeldAtFenceRelease { + t.Fatal("install lock was released before maintenance fence") + } +} + +func TestInstallCredentialRotationPreservesProviderStateAndWorkerIdentity(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + payload := writeTestProviderPayload(t, home, "verified-provider-rotation") + digest := fileDigestForTest(t, payload) + paths := LifecyclePathsFor(config) + statuses := []string{"unavailable", "unavailable", "idle", "unavailable", "unavailable", "idle"} + runner := installSuccessRunner(t, config, payload, digest, &statuses) + newInstaller := func(randomByte byte) Installer { + return Installer{ + Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, + Random: bytes.NewReader(bytes.Repeat([]byte{randomByte}, 4096)), Now: func() time.Time { return time.Unix(1_700_000_000, 0).UTC() }, + Sleep: func(context.Context, time.Duration) error { return nil }, + } + } + if _, err := newInstaller(0x41).Install(t.Context(), home, config, Credentials{GitHubToken: "github-old", ProviderToken: "provider-old"}); err != nil { + t.Fatalf("initial install: %v", err) + } + if err := os.WriteFile(filepath.Join(paths.ProviderState, "retained.json"), []byte("retained-state"), 0o600); err != nil { + t.Fatalf("write retained state: %v", err) + } + if _, err := newInstaller(0x51).Install(t.Context(), home, config, Credentials{GitHubToken: "github-new", ProviderToken: "provider-new"}); err != nil { + t.Fatalf("credential rotation: %v", err) + } + if data, err := os.ReadFile(filepath.Join(paths.ProviderState, "retained.json")); err != nil || string(data) != "retained-state" { + t.Fatalf("provider state changed: data=%q err=%v", data, err) + } + providerEnv, _ := os.ReadFile(paths.ProviderEnv) + agentEnv, _ := os.ReadFile(paths.AgentEnv) + if !bytes.Contains(providerEnv, []byte("github-new")) || bytes.Contains(providerEnv, []byte("github-old")) || !bytes.Contains(agentEnv, []byte("provider-new")) || bytes.Contains(agentEnv, []byte("github-new")) { + t.Fatalf("rotated provider=%s agent=%s", providerEnv, agentEnv) + } + if config.ProfileID != "github-runner-profile-stg" || config.WorkerID != "github-runner-linux-stg" { + t.Fatalf("worker identity changed: %+v", config) + } +} + +func TestInstallLeavesMaintenanceActiveWhenRollbackCannotRestartAgent(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + payload := writeTestProviderPayload(t, home, "verified-provider-rollback") + digest := fileDigestForTest(t, payload) + statuses := []string{"unavailable"} + runner := installSuccessRunner(t, config, payload, digest, &statuses) + baseRun := runner.run + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + event := installCommandEvent(command, config) + if event == "daemon-reload" || event == "agent-start" { + return nil, errors.New("systemd unavailable") + } + return baseRun(ctx, command) + } + installer := Installer{ + Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x61}, 4096)), Sleep: func(context.Context, time.Duration) error { return nil }, + Refresh: func(context.Context, Config) (Status, error) { return Status{}, nil }, + ProbeActive: func(context.Context, Config) error { return nil }, + } + if _, err := installer.Install(t.Context(), home, config, Credentials{GitHubToken: "github-secret", ProviderToken: "provider-secret"}); err == nil { + t.Fatal("install with incomplete rollback succeeded") + } + transcript := commandTranscript(runner.commands) + if strings.Contains(transcript, "supervisor-maintenance end") { + t.Fatalf("incomplete rollback released maintenance:\n%s", transcript) + } +} + +func TestInstallRollbackDoesNotDisableInactiveRefreshWatchUnits(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + payload := writeTestProviderPayload(t, home, "verified-provider-inactive-watch-rollback") + digest := fileDigestForTest(t, payload) + statuses := []string{"unavailable"} + runner := installSuccessRunner(t, config, payload, digest, &statuses) + baseRun := runner.run + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if filepath.Base(command.Path) == "systemctl" && containsArg(command.Args, "disable") && (containsArg(command.Args, refreshPathUnit) || containsArg(command.Args, refreshTimerUnit)) { + return nil, errors.New("inactive refresh watch cannot be disabled") + } + return baseRun(ctx, command) + } + installer := Installer{ + Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x63}, 4096)), Sleep: func(context.Context, time.Duration) error { return nil }, + Refresh: func(context.Context, Config) (Status, error) { + return Status{}, errors.New("provider activation failed") + }, + ProbeActive: func(context.Context, Config) error { return nil }, + } + if _, err := installer.Install(t.Context(), home, config, Credentials{GitHubToken: "github-secret", ProviderToken: "provider-secret"}); err == nil || !strings.Contains(err.Error(), "provider activation failed") { + t.Fatalf("install activation failure = %v", err) + } + transcript := commandTranscript(runner.commands) + if strings.Contains(transcript, "disable --now "+refreshPathUnit) || strings.Contains(transcript, "disable --now "+refreshTimerUnit) { + t.Fatalf("rollback disabled inactive refresh watch units:\n%s", transcript) + } + if !strings.Contains(transcript, "systemctl --user disable --now "+providerServiceUnit) || !strings.Contains(transcript, "supervisor-maintenance end") { + t.Fatalf("rollback did not disable the activated provider and release maintenance:\n%s", transcript) + } +} + +func TestInstallRollbackDisablesPartiallyEnabledRefreshWatchUnits(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + payload := writeTestProviderPayload(t, home, "verified-provider-partial-watch-rollback") + digest := fileDigestForTest(t, payload) + statuses := []string{"unavailable"} + runner := installSuccessRunner(t, config, payload, digest, &statuses) + baseRun := runner.run + enabled := map[string]bool{} + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if filepath.Base(command.Path) == "systemctl" && containsAdjacentArgs(command.Args, "enable", "--now") { + hasPath := containsArg(command.Args, refreshPathUnit) + hasTimer := containsArg(command.Args, refreshTimerUnit) + if hasPath && hasTimer { + return nil, errors.New("refresh units must be activated independently") + } + if hasPath { + enabled[refreshPathUnit] = true + } + if hasTimer { + enabled[refreshTimerUnit] = true + return nil, errors.New("timer start failed after enable") + } + } + if filepath.Base(command.Path) == "systemctl" && containsArg(command.Args, "show") && containsArg(command.Args, refreshTimerUnit) { + return []byte("LoadState=loaded\nFragmentPath=/tmp/provider.service\nActiveState=active\nUnitFileState=enabled\n"), nil + } + if filepath.Base(command.Path) == "systemctl" && containsAdjacentArgs(command.Args, "disable", "--now") { + for unit := range enabled { + if containsArg(command.Args, unit) { + delete(enabled, unit) + } + } + } + return baseRun(ctx, command) + } + installer := Installer{ + Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x64}, 4096)), Sleep: func(context.Context, time.Duration) error { return nil }, + } + if _, err := installer.Install(t.Context(), home, config, Credentials{GitHubToken: "github-secret", ProviderToken: "provider-secret"}); err == nil || !strings.Contains(err.Error(), "timer start failed after enable") { + t.Fatalf("partial watch enable failure = %v", err) + } + transcript := commandTranscript(runner.commands) + wantDisable := "systemctl --user disable --now " + refreshPathUnit + " " + refreshTimerUnit + " " + providerServiceUnit + if !strings.Contains(transcript, wantDisable) || !strings.Contains(transcript, "supervisor-maintenance end") || len(enabled) != 0 { + t.Fatalf("rollback did not disable partially activated units and release maintenance:\n%s", transcript) + } +} + +func TestInstallRollbackConservativelyRestoresWatchUnitAfterAmbiguousEnableFailure(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + payload := writeTestProviderPayload(t, home, "verified-provider-unchanged-watch-rollback") + digest := fileDigestForTest(t, payload) + statuses := []string{"unavailable"} + runner := installSuccessRunner(t, config, payload, digest, &statuses) + baseRun := runner.run + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if filepath.Base(command.Path) == "systemctl" && containsAdjacentArgs(command.Args, "enable", "--now") && containsArg(command.Args, refreshPathUnit) { + return nil, errors.New("path enable failed before mutation") + } + if filepath.Base(command.Path) == "systemctl" && containsArg(command.Args, "show") && containsArg(command.Args, refreshPathUnit) { + return []byte("LoadState=loaded\nFragmentPath=/tmp/provider.service\nActiveState=inactive\nUnitFileState=disabled\n"), nil + } + return baseRun(ctx, command) + } + installer := Installer{ + Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x65}, 4096)), Sleep: func(context.Context, time.Duration) error { return nil }, + } + if _, err := installer.Install(t.Context(), home, config, Credentials{GitHubToken: "github-secret", ProviderToken: "provider-secret"}); err == nil || !strings.Contains(err.Error(), "path enable failed before mutation") { + t.Fatalf("unchanged watch enable failure = %v", err) + } + transcript := commandTranscript(runner.commands) + for _, command := range []string{ + "disable --now " + refreshPathUnit, + "disable " + refreshPathUnit, + "stop " + refreshPathUnit, + "disable --now " + refreshPathUnit + " " + providerServiceUnit, + "supervisor-maintenance end", + } { + if !strings.Contains(transcript, command) { + t.Fatalf("rollback did not conservatively restore ambiguous watch activation %q:\n%s", command, transcript) + } + } +} + +func TestInstallRollbackRestoresPriorActiveStateAndServiceWiring(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + payload := writeTestProviderPayload(t, home, "verified-provider-post-activation-rollback") + digest := fileDigestForTest(t, payload) + previous := previousActiveStateForTest(t, home) + if err := AtomicWriteJSON(paths.ActiveState, previous); err != nil { + t.Fatalf("write previous active state: %v", err) + } + if err := atomicWriteFile(paths.ProviderEnv, []byte("OLD_PROVIDER_ENV=retained\n"), 0o600); err != nil { + t.Fatalf("write previous provider env: %v", err) + } + for _, path := range managedWiringPaths(paths) { + if err := atomicWriteFile(path, []byte("prior-unit\n"), 0o600); err != nil { + t.Fatalf("write prior wiring %s: %v", path, err) + } + } + statuses := []string{"unavailable"} + runner := installSuccessRunner(t, config, payload, digest, &statuses) + installer := Installer{ + Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x62}, 4096)), Sleep: func(context.Context, time.Duration) error { return nil }, + Refresh: func(context.Context, Config) (Status, error) { + selection := selectionForDigest(payload, digest, "v1.0.32", "directive-new", "sha256:"+strings.Repeat("d", 64), time.Unix(1_700_100_000, 0).UTC()) + active := ActiveState{ProtocolVersion: ActiveStateProtocolVersion, Current: selection, Previous: &previous.Current, UpdatedAt: selection.ActivatedAt} + return statusForActive(active, true, active.UpdatedAt), AtomicWriteJSON(paths.ActiveState, active) + }, + ProbeActive: func(context.Context, Config) error { return errors.New("post-activation probe failed") }, + } + if _, err := installer.Install(t.Context(), home, config, Credentials{GitHubToken: "github-new", ProviderToken: "provider-new"}); err == nil { + t.Fatal("post-activation probe failure succeeded") + } + restored, found, err := readActiveState(paths.ActiveState) + if err != nil || !found || restored.Current.ImageID != previous.Current.ImageID { + t.Fatalf("restored active = %+v found=%v err=%v", restored, found, err) + } + providerEnv, err := os.ReadFile(paths.ProviderEnv) + if err != nil || string(providerEnv) != "OLD_PROVIDER_ENV=retained\n" { + t.Fatalf("restored provider env = %q err=%v", providerEnv, err) + } + transcript := commandTranscript(runner.commands) + for _, command := range []string{ + "systemctl --user enable " + providerServiceUnit, + "systemctl --user start " + providerServiceUnit, + "systemctl --user enable " + refreshPathUnit, + "systemctl --user start " + refreshPathUnit, + "systemctl --user enable " + refreshTimerUnit, + "systemctl --user start " + refreshTimerUnit, + } { + if !strings.Contains(transcript, command) { + t.Fatalf("rollback did not restore service activation %q:\n%s", command, transcript) + } + } +} + +func TestInstallRollbackPreservesPreviouslyDisabledProviderUnits(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + payload := writeTestProviderPayload(t, home, "verified-provider-disabled-unit-rollback") + digest := fileDigestForTest(t, payload) + for _, path := range managedWiringPaths(paths) { + if err := atomicWriteFile(path, []byte("prior-unit\n"), 0o600); err != nil { + t.Fatalf("write prior wiring %s: %v", path, err) + } + } + statuses := []string{"unavailable"} + runner := installSuccessRunner(t, config, payload, digest, &statuses) + baseRun := runner.run + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if filepath.Base(command.Path) == "systemctl" && containsArg(command.Args, "show") && (containsArg(command.Args, providerServiceUnit) || containsArg(command.Args, refreshPathUnit) || containsArg(command.Args, refreshTimerUnit)) { + return []byte("LoadState=loaded\nFragmentPath=/tmp/provider.service\nActiveState=inactive\nUnitFileState=disabled\n"), nil + } + return baseRun(ctx, command) + } + installer := Installer{ + Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x67}, 4096)), Sleep: func(context.Context, time.Duration) error { return nil }, + Refresh: func(context.Context, Config) (Status, error) { + return Status{}, errors.New("provider activation failed") + }, + ProbeActive: func(context.Context, Config) error { return nil }, + } + if _, err := installer.Install(t.Context(), home, config, Credentials{GitHubToken: "github-new", ProviderToken: "provider-new"}); err == nil || !strings.Contains(err.Error(), "provider activation failed") { + t.Fatalf("disabled-unit rollback failure = %v", err) + } + transcript := commandTranscript(runner.commands) + if strings.Contains(transcript, "restart "+providerServiceUnit) || strings.Contains(transcript, "enable --now "+refreshPathUnit) { + t.Fatalf("rollback activated previously disabled units:\n%s", transcript) + } +} + +func TestRestoreUnitStatePreservesRuntimeEnablement(t *testing.T) { + runner := &recordingCommandRunner{run: func(context.Context, Command) ([]byte, error) { return nil, nil }} + installer := Installer{Runner: runner} + if err := installer.restoreUnitState(t.Context(), providerServiceUnit, systemdUnitState{LoadState: "loaded", FragmentPath: "/tmp/provider.service", UnitFileState: "enabled-runtime", ActiveState: "active"}); err != nil { + t.Fatalf("restore runtime-enabled unit: %v", err) + } + transcript := commandTranscript(runner.commands) + if !strings.Contains(transcript, "systemctl --user enable --runtime "+providerServiceUnit) { + t.Fatalf("runtime enablement became persistent:\n%s", transcript) + } +} + +func TestCaptureManagedUnitStatesRejectsUnrestorableSemantics(t *testing.T) { + for _, tc := range []struct { + name string + state systemdUnitState + }{ + {name: "linked unit", state: systemdUnitState{LoadState: "loaded", FragmentPath: "/tmp/provider.service", UnitFileState: "linked", ActiveState: "inactive"}}, + {name: "failed unit", state: systemdUnitState{LoadState: "loaded", FragmentPath: "/tmp/provider.service", UnitFileState: "enabled", ActiveState: "failed"}}, + } { + t.Run(tc.name, func(t *testing.T) { + runner := &recordingCommandRunner{run: func(context.Context, Command) ([]byte, error) { + return []byte("LoadState=" + tc.state.LoadState + "\nFragmentPath=" + tc.state.FragmentPath + "\nActiveState=" + tc.state.ActiveState + "\nUnitFileState=" + tc.state.UnitFileState + "\n"), nil + }} + installer := Installer{Runner: runner} + if _, err := installer.captureManagedUnitStates(t.Context()); err == nil || !strings.Contains(err.Error(), "unsupported prior") { + t.Fatalf("capture state %+v err = %v", tc.state, err) + } + }) + } +} + +func TestCaptureManagedUnitStatesIncludesUnitsLoadedOutsideManagedPaths(t *testing.T) { + fragment := "/usr/lib/systemd/user/vendor-provider.service" + runner := &recordingCommandRunner{run: func(context.Context, Command) ([]byte, error) { + return []byte("LoadState=loaded\nFragmentPath=" + fragment + "\nActiveState=active\nUnitFileState=enabled\n"), nil + }} + states, err := (Installer{Runner: runner}).captureManagedUnitStates(t.Context()) + if err != nil { + t.Fatalf("capture loaded units: %v", err) + } + if len(states) != 3 { + t.Fatalf("captured states = %+v", states) + } + for _, unit := range []string{providerServiceUnit, refreshPathUnit, refreshTimerUnit} { + state, found := states[unit] + if !found || state.ActiveState != "active" || state.UnitFileState != "enabled" { + t.Fatalf("state[%s] = %+v found=%v", unit, state, found) + } + } +} + +func TestInstallRollbackRestoresProviderStateFromCommittedNestedRefresh(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + stateFile := filepath.Join(paths.ProviderState, "generation") + if err := os.WriteFile(stateFile, []byte("previous"), 0o600); err != nil { + t.Fatalf("write previous provider state: %v", err) + } + previous := previousActiveStateForTest(t, home) + if err := AtomicWriteJSON(paths.ActiveState, previous); err != nil { + t.Fatalf("write previous active state: %v", err) + } + payload := writeTestProviderPayload(t, home, "verified-provider-nested-refresh-rollback") + digest := fileDigestForTest(t, payload) + statuses := []string{"unavailable"} + runner := installSuccessRunner(t, config, payload, digest, &statuses) + baseRun := runner.run + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if installCommandEvent(command, config) == "agent-start" { + return nil, errors.New("agent restart failed after provider refresh") + } + return baseRun(ctx, command) + } + now := time.Unix(1_700_200_000, 0).UTC() + selection := selectionForDigest(payload, digest, "v1.0.32", "directive-nested", "sha256:"+strings.Repeat("d", 64), now) + installer := Installer{ + Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x66}, 4096)), Now: func() time.Time { return now }, + Sleep: func(context.Context, time.Duration) error { return nil }, + Refresh: func(context.Context, Config) (Status, error) { + outer, found, err := readLifecycleJournal(home, paths) + if err != nil || !found || outer.Phase != LifecycleFenced { + return Status{}, fmt.Errorf("read outer lifecycle binding found=%v: %w", found, err) + } + if err := prepareCandidateState(paths.ProviderState, paths.CandidateState(digest)); err != nil { + return Status{}, err + } + if err := os.WriteFile(filepath.Join(paths.CandidateState(digest), "generation"), []byte("candidate"), 0o600); err != nil { + return Status{}, err + } + if err := promoteCandidateProviderState(paths, digest); err != nil { + return Status{}, err + } + active := ActiveState{ProtocolVersion: ActiveStateProtocolVersion, Current: selection, Previous: &previous.Current, UpdatedAt: now} + if err := AtomicWriteJSON(paths.ActiveState, active); err != nil { + return Status{}, err + } + journal := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, ID: "refresh-nested-install", Phase: JournalCommitted, DeferredCommit: true, + OuterTransactionID: outer.TransactionID, ProfileID: config.ProfileID, + Previous: &previous, Candidate: selection, StartedAt: now, UpdatedAt: now, + } + if err := AtomicWriteJSON(paths.Journal, journal); err != nil { + return Status{}, err + } + return statusForActive(active, true, now), nil + }, + ProbeActive: func(context.Context, Config) error { return nil }, + } + if _, err := installer.Install(t.Context(), home, config, Credentials{GitHubToken: "github-new", ProviderToken: "provider-new"}); err == nil || !strings.Contains(err.Error(), "agent restart failed") { + t.Fatalf("install post-refresh failure = %v", err) + } + if data, err := os.ReadFile(stateFile); err != nil || string(data) != "previous" { + t.Fatalf("nested refresh rollback state = %q err=%v", data, err) + } +} + +func TestInstallRecoversDeferredCommittedRefreshAfterProcessRestart(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + payload := writeTestProviderPayload(t, home, "verified-provider-deferred-install-recovery") + digest := fileDigestForTest(t, payload) + now := time.Unix(1_700_300_000, 0).UTC() + selection := selectionForDigest(payload, digest, "v1.0.32", "directive-deferred-recovery", "sha256:"+strings.Repeat("d", 64), now) + active := ActiveState{ProtocolVersion: ActiveStateProtocolVersion, Current: selection, UpdatedAt: now} + if err := AtomicWriteJSON(paths.ActiveState, active); err != nil { + t.Fatalf("write committed active state: %v", err) + } + transactionRoot := filepath.Dir(paths.CandidateState(digest)) + if err := os.MkdirAll(transactionRoot, 0o700); err != nil { + t.Fatalf("create deferred transaction root: %v", err) + } + journal := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, + ID: "refresh-deferred-install-recovery", + Phase: JournalCommitted, + DeferredCommit: true, + Candidate: selection, + StartedAt: now, + UpdatedAt: now, + } + if err := AtomicWriteJSON(paths.Journal, journal); err != nil { + t.Fatalf("write deferred refresh journal: %v", err) + } + snapshots, err := snapshotManagedFiles(paths) + if err != nil { + t.Fatalf("snapshot committed outer install: %v", err) + } + outerJournal := newInstallTransactionJournal("install", snapshots, map[string]systemdUnitState{}, now) + outerJournal.Phase = installTransactionCommitted + if err := writeInstallTransactionJournal(paths, outerJournal); err != nil { + t.Fatalf("write committed outer install journal: %v", err) + } + statuses := []string{"unavailable", "unavailable", "idle"} + runner := installSuccessRunner(t, config, payload, digest, &statuses) + installer := Installer{ + Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x68}, 4096)), Now: func() time.Time { return now }, + Sleep: func(context.Context, time.Duration) error { return nil }, + Refresh: func(context.Context, Config) (Status, error) { + if _, err := os.Lstat(paths.Journal); !errors.Is(err, os.ErrNotExist) { + return Status{}, fmt.Errorf("deferred journal was not recovered before retry: %v", err) + } + if _, err := os.Lstat(transactionRoot); !errors.Is(err, os.ErrNotExist) { + return Status{}, fmt.Errorf("deferred rollback state was not finalized before retry: %v", err) + } + return statusForActive(active, true, now), nil + }, + ProbeActive: func(context.Context, Config) error { return nil }, + } + if _, err := installer.Install(t.Context(), home, config, Credentials{GitHubToken: "github-new", ProviderToken: "provider-new"}); err != nil { + t.Fatalf("recover deferred install: %v", err) + } + if transcript := commandTranscript(runner.commands); !strings.Contains(transcript, "supervisor-maintenance end") { + t.Fatalf("recovered install did not release maintenance:\n%s", transcript) + } +} + +func TestInstallCrashRecoveryRestoresDurableOuterBaselineBeforeRetry(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + payload := writeTestProviderPayload(t, home, "verified-provider-outer-crash-recovery") + digest := fileDigestForTest(t, payload) + now := time.Unix(1_700_500_000, 0).UTC() + previous := previousActiveStateForTest(t, home) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + stateFile := filepath.Join(paths.ProviderState, "generation") + if err := os.WriteFile(stateFile, []byte("previous"), 0o600); err != nil { + t.Fatalf("write previous provider state: %v", err) + } + if err := AtomicWriteJSON(paths.ActiveState, previous); err != nil { + t.Fatalf("write previous active state: %v", err) + } + if err := prepareCandidateState(paths.ProviderState, paths.CandidateState(digest)); err != nil { + t.Fatalf("prepare candidate state: %v", err) + } + if err := os.WriteFile(filepath.Join(paths.CandidateState(digest), "generation"), []byte("candidate"), 0o600); err != nil { + t.Fatalf("write candidate provider state: %v", err) + } + if err := promoteCandidateProviderState(paths, digest); err != nil { + t.Fatalf("promote candidate state: %v", err) + } + candidate := selectionForDigest(payload, digest, "v1.0.32", "directive-outer-crash", "sha256:"+strings.Repeat("d", 64), now) + active := ActiveState{ProtocolVersion: ActiveStateProtocolVersion, Current: candidate, Previous: &previous.Current, UpdatedAt: now} + if err := AtomicWriteJSON(paths.ActiveState, active); err != nil { + t.Fatalf("write candidate active state: %v", err) + } + providerJournal := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, ID: "refresh-outer-crash", Phase: JournalCommitted, DeferredCommit: true, + Previous: &previous, Candidate: candidate, StartedAt: now, UpdatedAt: now, + } + if err := AtomicWriteJSON(paths.Journal, providerJournal); err != nil { + t.Fatalf("write deferred provider journal: %v", err) + } + if err := atomicWriteFile(paths.AgentEnv, []byte("PARTIAL_AGENT_ENV=1\n"), 0o600); err != nil { + t.Fatalf("write partial agent wiring: %v", err) + } + backupRoot := filepath.Join(paths.Root, ".install-backup-crashed") + backup := filepath.Join(backupRoot, "0") + if err := atomicWriteFile(backup, []byte("ORIGINAL_AGENT_ENV=1\n"), 0o600); err != nil { + t.Fatalf("write durable outer backup: %v", err) + } + outerJournalPath := filepath.Join(filepath.Dir(paths.Root), ".workflow-plugin-github-runner-provider.install-transaction.json") + type snapshotFixture struct { + Path string `json:"path"` + Backup string `json:"backup"` + Mode os.FileMode `json:"mode"` + Existed bool `json:"existed"` + } + type activationFixture struct { + ProviderService bool `json:"provider_service"` + RefreshPath bool `json:"refresh_path"` + RefreshTimer bool `json:"refresh_timer"` + } + outerJournal := struct { + ProtocolVersion string `json:"protocol_version"` + Operation string `json:"operation"` + Phase string `json:"phase"` + MaintenanceID string `json:"maintenance_id"` + AgentStopped bool `json:"agent_stopped"` + Snapshots []snapshotFixture `json:"snapshots"` + PreviousUnits map[string]systemdUnitState `json:"previous_units"` + Activation activationFixture `json:"activation"` + StartedAt time.Time `json:"started_at"` + UpdatedAt time.Time `json:"updated_at"` + }{ + ProtocolVersion: "retained-provider.install-transaction.v1", Operation: "install", Phase: "prepared", + MaintenanceID: installMaintenanceID, AgentStopped: true, + Snapshots: []snapshotFixture{{Path: paths.AgentEnv, Backup: backup, Mode: 0o600, Existed: true}}, + PreviousUnits: map[string]systemdUnitState{}, Activation: activationFixture{}, StartedAt: now, UpdatedAt: now, + } + if err := AtomicWriteJSON(outerJournalPath, outerJournal); err != nil { + t.Fatalf("write outer install journal: %v", err) + } + statuses := []string{"unavailable"} + runner := installSuccessRunner(t, config, payload, digest, &statuses) + installer := Installer{ + Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x6a}, 4096)), Now: func() time.Time { return now }, + Sleep: func(context.Context, time.Duration) error { return nil }, + Refresh: func(context.Context, Config) (Status, error) { return Status{}, errors.New("retry activation failed") }, + ProbeActive: func(context.Context, Config) error { return nil }, + } + if _, err := installer.Install(t.Context(), home, config, Credentials{GitHubToken: "github-new", ProviderToken: "provider-new"}); err == nil || !strings.Contains(err.Error(), "retry activation failed") { + t.Fatalf("retry failure = %v", err) + } + if data, err := os.ReadFile(paths.AgentEnv); err != nil || string(data) != "ORIGINAL_AGENT_ENV=1\n" { + t.Fatalf("outer baseline wiring = %q err=%v", data, err) + } + if data, err := os.ReadFile(stateFile); err != nil || string(data) != "previous" { + t.Fatalf("outer baseline provider state = %q err=%v", data, err) + } + restored, found, err := readActiveState(paths.ActiveState) + if err != nil || !found || restored.Current.ImageID != previous.Current.ImageID { + t.Fatalf("outer baseline active = %+v found=%v err=%v", restored, found, err) + } + if _, err := os.Stat(outerJournalPath); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("recovered outer journal remains: %v", err) + } +} + +func TestInstallRecoversOuterTransactionBeforeNewUpdatePreflight(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + if err := atomicWriteFile(paths.AgentEnv, []byte("PARTIAL_AGENT_ENV=1\n"), 0o600); err != nil { + t.Fatalf("write partial agent env: %v", err) + } + backup := filepath.Join(paths.Root, ".install-backup-preflight-recovery", "0") + if err := atomicWriteFile(backup, []byte("ORIGINAL_AGENT_ENV=1\n"), 0o600); err != nil { + t.Fatalf("write agent env backup: %v", err) + } + now := time.Unix(1_700_550_000, 0).UTC() + journal := newInstallTransactionJournal("install", []managedFileSnapshot{{ + Path: paths.AgentEnv, Backup: backup, Mode: 0o600, Existed: true, + }}, map[string]systemdUnitState{}, now) + if err := writeInstallTransactionJournal(paths, journal); err != nil { + t.Fatalf("write prepared outer transaction: %v", err) + } + runner := installSuccessRunner(t, config, "", "", new([]string)) + baseRun := runner.run + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if installCommandEvent(command, config) == "verify-update" { + return nil, errors.New("new update preflight unavailable") + } + return baseRun(ctx, command) + } + installer := Installer{Runner: runner, Now: func() time.Time { return now }} + if _, err := installer.Install(t.Context(), home, config, Credentials{GitHubToken: "github-new", ProviderToken: "provider-new"}); err == nil || !strings.Contains(err.Error(), "new update preflight unavailable") { + t.Fatalf("install preflight err = %v", err) + } + if data, err := os.ReadFile(paths.AgentEnv); err != nil || string(data) != "ORIGINAL_AGENT_ENV=1\n" { + t.Fatalf("preflight failure blocked local recovery = %q err=%v", data, err) + } + if _, err := os.Stat(paths.InstallJournal); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("recovered outer transaction remains after preflight failure: %v", err) + } +} + +func TestInstallRejectsMalformedOuterTransactionBeforeMaintenanceMutation(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + payload := writeTestProviderPayload(t, home, "verified-provider-malformed-outer-transaction") + digest := fileDigestForTest(t, payload) + if err := os.MkdirAll(paths.Root, 0o700); err != nil { + t.Fatalf("mkdir install root: %v", err) + } + now := time.Unix(1_700_600_000, 0).UTC() + malformed := installTransactionJournal{ + ProtocolVersion: installTransactionProtocol, + Operation: "install", + Phase: installTransactionPrepared, + MaintenanceID: installMaintenanceID, + AgentStopped: true, + Snapshots: []managedFileSnapshot{{ + Path: filepath.Join(home, "unmanaged"), Backup: filepath.Join(paths.Root, ".install-backup-invalid", "0"), + }}, + PreviousUnits: map[string]systemdUnitState{}, + StartedAt: now, + UpdatedAt: now, + } + if err := AtomicWriteJSON(paths.InstallJournal, malformed); err != nil { + t.Fatalf("write malformed outer transaction: %v", err) + } + runner := installSuccessRunner(t, config, payload, digest, new([]string)) + installer := Installer{ + Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x6b}, 4096)), Sleep: func(context.Context, time.Duration) error { return nil }, + } + if _, err := installer.Install(t.Context(), home, config, Credentials{GitHubToken: "github-new", ProviderToken: "provider-new"}); err == nil || !strings.Contains(err.Error(), "unmanaged path") { + t.Fatalf("malformed outer transaction err = %v", err) + } + transcript := commandTranscript(runner.commands) + if strings.Contains(transcript, "supervisor-maintenance begin") || strings.Contains(transcript, "stop "+config.AgentUnit) { + t.Fatalf("malformed outer transaction mutated maintenance or agent:\n%s", transcript) + } +} + +func TestInstallRecoversReadyOuterTransactionForwardBeforeRetry(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + payload := writeTestProviderPayload(t, home, "verified-provider-ready-outer-transaction") + digest := fileDigestForTest(t, payload) + if err := atomicWriteFile(paths.AgentEnv, []byte("ORIGINAL_AGENT_ENV=1\n"), 0o600); err != nil { + t.Fatalf("write original agent env: %v", err) + } + snapshots, err := snapshotManagedFiles(paths) + if err != nil { + t.Fatalf("snapshot outer transaction: %v", err) + } + if err := atomicWriteFile(paths.AgentEnv, []byte("COMMITTED_AGENT_ENV=1\n"), 0o600); err != nil { + t.Fatalf("write committed agent env: %v", err) + } + now := time.Unix(1_700_700_000, 0).UTC() + journal := newInstallTransactionJournal("install", snapshots, map[string]systemdUnitState{}, now) + journal.Phase = installTransactionReady + if err := writeInstallTransactionJournal(paths, journal); err != nil { + t.Fatalf("write ready outer transaction: %v", err) + } + runner := installSuccessRunner(t, config, payload, digest, new([]string)) + baseRun := runner.run + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if installCommandEvent(command, config) == "maintenance-begin" { + return nil, errors.New("stop after recovered transaction") + } + return baseRun(ctx, command) + } + installer := Installer{ + Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x6c}, 4096)), Now: func() time.Time { return now }, + } + if _, err := installer.Install(t.Context(), home, config, Credentials{GitHubToken: "github-new", ProviderToken: "provider-new"}); err == nil || !strings.Contains(err.Error(), "stop after recovered transaction") { + t.Fatalf("retry after ready recovery err = %v", err) + } + if data, err := os.ReadFile(paths.AgentEnv); err != nil || string(data) != "COMMITTED_AGENT_ENV=1\n" { + t.Fatalf("ready recovery rolled back committed wiring = %q err=%v", data, err) + } + if _, err := os.Stat(paths.InstallJournal); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("ready outer transaction remains: %v", err) + } + transcript := commandTranscript(runner.commands) + if !strings.Contains(transcript, "supervisor-maintenance end") || strings.Contains(transcript, "disable --now") || strings.Contains(transcript, "start "+config.AgentUnit) { + t.Fatalf("ready outer transaction did not finish forward:\n%s", transcript) + } +} + +func TestInstallCleanupFailureStillReleasesMaintenance(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + payload := writeTestProviderPayload(t, home, "verified-provider-install-cleanup-failure") + digest := fileDigestForTest(t, payload) + statuses := []string{"unavailable", "unavailable"} + runner := installSuccessRunner(t, config, payload, digest, &statuses) + installer := Installer{ + Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x69}, 4096)), Sleep: func(context.Context, time.Duration) error { return nil }, + ProbeActive: func(context.Context, Config) error { + return os.Chmod(paths.LifecycleTransactions, 0o500) + }, + } + t.Cleanup(func() { _ = os.Chmod(paths.LifecycleTransactions, 0o700) }) + _, err := installer.Install(t.Context(), home, config, Credentials{GitHubToken: "github-new", ProviderToken: "provider-new"}) + if err == nil || !strings.Contains(err.Error(), "remove lifecycle transaction root") { + t.Fatalf("install cleanup failure = %v", err) + } + if transcript := commandTranscript(runner.commands); !strings.Contains(transcript, "supervisor-maintenance end") { + t.Fatalf("install cleanup failure stranded maintenance:\n%s", transcript) + } +} + +func TestUninstallCleanupFailureStillReleasesMaintenance(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + if err := atomicWriteFile(paths.ConfigFile, []byte("previous-config\n"), 0o600); err != nil { + t.Fatalf("write previous config: %v", err) + } + statuses := []string{"unavailable", "unavailable"} + runner := installSuccessRunner(t, config, "", "", &statuses) + baseRun := runner.run + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + output, err := baseRun(ctx, command) + if err == nil && installCommandEvent(command, config) == "maintenance-end" { + err = os.Chmod(paths.LifecycleTransactions, 0o500) + } + return output, err + } + installer := Installer{Runner: runner, Sleep: func(context.Context, time.Duration) error { return nil }} + t.Cleanup(func() { _ = os.Chmod(paths.LifecycleTransactions, 0o700) }) + _, err := installer.Uninstall(t.Context(), home, config, false) + if err == nil || !strings.Contains(err.Error(), "remove lifecycle transaction root") { + t.Fatalf("uninstall cleanup failure = %v", err) + } + if transcript := commandTranscript(runner.commands); !strings.Contains(transcript, "supervisor-maintenance end") { + t.Fatalf("uninstall cleanup failure stranded maintenance:\n%s", transcript) + } +} + +func TestInstallRejectsMismatchedMaintenanceIdentityBeforeMutation(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + payload := writeTestProviderPayload(t, home, "verified-provider-maintenance-mismatch") + digest := fileDigestForTest(t, payload) + runner := installSuccessRunner(t, config, payload, digest, new([]string)) + baseRun := runner.run + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if installCommandEvent(command, config) == "maintenance-begin" { + return maintenanceStateJSON(true, "different-transaction", config.ProfileID, installMaintenanceReason), nil + } + return baseRun(ctx, command) + } + installer := Installer{ + Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x71}, 4096)), Sleep: func(context.Context, time.Duration) error { return nil }, + } + if _, err := installer.Install(t.Context(), home, config, Credentials{GitHubToken: "github-secret", ProviderToken: "provider-secret"}); err == nil || !strings.Contains(err.Error(), "mismatched") { + t.Fatalf("mismatched maintenance err = %v", err) + } + if transcript := commandTranscript(runner.commands); strings.Contains(transcript, "systemctl --user stop "+config.AgentUnit) { + t.Fatalf("agent mutated after mismatched maintenance response:\n%s", transcript) + } +} + +func TestMaintenanceStatusClassifiesExactInactiveAndConflictingFence(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + for _, tc := range []struct { + name string + state []byte + want maintenanceDisposition + err string + }{ + { + name: "exact active", + state: maintenanceStateJSON(true, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), + want: maintenanceExactActive, + }, + { + name: "inactive", + state: []byte(`{"active":false,"durable":true}`), + want: maintenanceInactive, + }, + { + name: "conflicting active", + state: maintenanceStateJSON(true, "other-transaction", config.ProfileID, refreshMaintenanceReason), + want: maintenanceConflicting, + }, + { + name: "non-durable", + state: []byte(`{"active":false,"durable":false}`), + err: "durable", + }, + } { + t.Run(tc.name, func(t *testing.T) { + runner := &recordingCommandRunner{run: func(_ context.Context, command Command) ([]byte, error) { + if got := installCommandEvent(command, config); got != "maintenance-status" { + t.Fatalf("command event = %q command=%+v", got, command) + } + if containsArg(command.Args, "-id") || containsArg(command.Args, "-reason") { + t.Fatalf("status command carries transaction mutation arguments: %+v", command.Args) + } + return tc.state, nil + }} + installer := Installer{Runner: runner} + state, err := installer.maintenanceStatus(t.Context(), config) + if tc.err != "" { + if err == nil || !strings.Contains(err.Error(), tc.err) { + t.Fatalf("maintenanceStatus error = %v want %q", err, tc.err) + } + return + } + if err != nil { + t.Fatalf("maintenanceStatus: %v", err) + } + if got := classifyMaintenanceState(state, config.ProfileID, refreshMaintenanceID, refreshMaintenanceReason); got != tc.want { + t.Fatalf("classification = %q want %q", got, tc.want) + } + }) + } +} + +func TestInstallBoundsTransientLocalStatusPolling(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + payload := writeTestProviderPayload(t, home, "verified-provider-bounded-status") + digest := fileDigestForTest(t, payload) + statusReads := 0 + runner := installSuccessRunner(t, config, payload, digest, new([]string)) + baseRun := runner.run + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if installCommandEvent(command, config) == "local-status" { + statusReads++ + return localStatusJSON(config.WorkerID, "processing"), nil + } + return baseRun(ctx, command) + } + installer := Installer{ + Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x72}, 4096)), Sleep: func(context.Context, time.Duration) error { return nil }, + } + if _, err := installer.Install(t.Context(), home, config, Credentials{GitHubToken: "github-secret", ProviderToken: "provider-secret"}); err == nil || !strings.Contains(err.Error(), "did not reach unavailable") { + t.Fatalf("bounded status err = %v", err) + } + if statusReads != localStatusAttempts { + t.Fatalf("local status reads = %d want %d", statusReads, localStatusAttempts) + } + if transcript := commandTranscript(runner.commands); strings.Contains(transcript, "supervisor-maintenance end") || strings.Contains(transcript, "systemctl --user stop "+config.AgentUnit) { + t.Fatalf("pre-mutation timeout released the fence or stopped the agent:\n%s", transcript) + } + paths := LifecyclePathsFor(config) + if journal, found, err := readLifecycleJournal(home, paths); err != nil || !found || journal.Phase != LifecycleFencing { + t.Fatalf("bounded timeout lifecycle = %+v found=%v err=%v", journal, found, err) + } +} + +func TestInstallerStatusReportsOnlyLocalRedactedLifecycleState(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + active := previousActiveStateForTest(t, home) + if err := AtomicWriteJSON(paths.ActiveState, active); err != nil { + t.Fatalf("write active state: %v", err) + } + if err := atomicWriteFile(paths.ProviderUnit, []byte("[Service]\n"), 0o600); err != nil { + t.Fatalf("write provider service unit: %v", err) + } + runner := &recordingCommandRunner{run: func(_ context.Context, command Command) ([]byte, error) { + if filepath.Base(command.Path) == "systemctl" { + return []byte("active\n"), nil + } + return nil, nil + }} + now := time.Unix(1_700_100_000, 0).UTC() + status, err := (Installer{Runner: runner, Now: func() time.Time { return now }}).Status(t.Context(), home, config) + if err != nil { + t.Fatalf("status: %v", err) + } + want := statusForActive(active, true, now) + if status != want { + t.Fatalf("status = %+v want %+v", status, want) + } + transcript := commandTranscript(runner.commands) + if strings.Contains(transcript, "github-secret") || strings.Contains(transcript, "provider-token") || strings.Contains(transcript, "COMPUTE_API_TOKEN") || strings.Contains(transcript, "https://stg") { + t.Fatalf("status crossed a non-local boundary or leaked a secret:\n%s", transcript) + } +} + +func TestInstallerStatusReportsUninstalledWhenPreservedStateHasNoServiceUnit(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + if err := AtomicWriteJSON(paths.ActiveState, previousActiveStateForTest(t, home)); err != nil { + t.Fatalf("write preserved active state: %v", err) + } + runner := &recordingCommandRunner{run: func(context.Context, Command) ([]byte, error) { + return nil, errors.New("systemctl must not run for an uninstalled provider") + }} + now := time.Unix(1_700_200_000, 0).UTC() + status, err := (Installer{Runner: runner, Now: func() time.Time { return now }}).Status(t.Context(), home, config) + if err != nil { + t.Fatalf("status after preserved-state uninstall: %v", err) + } + want := Status{ProtocolVersion: StatusProtocolVersion, ObservedAt: now} + if status != want || len(runner.commands) != 0 { + t.Fatalf("status = %+v commands=%+v want=%+v", status, runner.commands, want) + } +} + +func TestRestoreManagedFilesPreservesBackupAfterRestoreFailure(t *testing.T) { + root := t.TempDir() + backupRoot := filepath.Join(root, "backup") + backup := filepath.Join(backupRoot, "0") + destination := filepath.Join(root, "managed") + if err := os.MkdirAll(backupRoot, 0o700); err != nil { + t.Fatalf("mkdir backup: %v", err) + } + if err := os.WriteFile(backup, []byte("prior managed data"), 0o600); err != nil { + t.Fatalf("write backup: %v", err) + } + if err := os.Mkdir(destination, 0o700); err != nil { + t.Fatalf("mkdir invalid destination: %v", err) + } + snapshots := []managedFileSnapshot{{Path: destination, Backup: backup, Mode: 0o600, Existed: true}} + if err := restoreManagedFiles(snapshots); err == nil { + t.Fatal("restore into directory succeeded") + } + if data, err := os.ReadFile(backup); err != nil || string(data) != "prior managed data" { + t.Fatalf("failed restore discarded backup: data=%q err=%v", data, err) + } +} + +func TestRollbackInstallPreservesSnapshotsUntilAgentAndMaintenanceRestore(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + backupRoot := filepath.Join(config.InstallRoot, ".install-backup-retry") + backup := filepath.Join(backupRoot, "0") + destination := LifecyclePathsFor(config).AgentEnv + if err := atomicWriteFile(backup, []byte("ORIGINAL_AGENT_ENV=1\n"), 0o600); err != nil { + t.Fatalf("write backup: %v", err) + } + if err := atomicWriteFile(destination, []byte("PARTIAL_AGENT_ENV=1\n"), 0o600); err != nil { + t.Fatalf("write partial destination: %v", err) + } + runner := &recordingCommandRunner{run: func(_ context.Context, command Command) ([]byte, error) { + if installCommandEvent(command, config) == "agent-start" { + return nil, errors.New("agent restart failed") + } + return nil, nil + }} + snapshots := []managedFileSnapshot{{Path: destination, Backup: backup, Mode: 0o600, Existed: true}} + err := (Installer{Runner: runner}).rollbackInstall(t.Context(), config, snapshots, map[string]systemdUnitState{}, true, true, installMaintenanceID, systemdActivation{}) + if err == nil || !strings.Contains(err.Error(), "agent restart failed") { + t.Fatalf("rollback err = %v", err) + } + if data, err := os.ReadFile(backup); err != nil || string(data) != "ORIGINAL_AGENT_ENV=1\n" { + t.Fatalf("rollback discarded retry backup = %q err=%v", data, err) + } +} + +func TestRestoreManagedFilesPropagatesSnapshotCleanupFailure(t *testing.T) { + root := t.TempDir() + cleanupParent := filepath.Join(root, "cleanup-parent") + backupRoot := filepath.Join(cleanupParent, "backup") + backup := filepath.Join(backupRoot, "0") + destination := filepath.Join(root, "managed") + if err := os.MkdirAll(backupRoot, 0o700); err != nil { + t.Fatalf("mkdir backup: %v", err) + } + if err := os.WriteFile(backup, []byte("prior managed data"), 0o600); err != nil { + t.Fatalf("write backup: %v", err) + } + if err := os.WriteFile(destination, []byte("current managed data"), 0o600); err != nil { + t.Fatalf("write destination: %v", err) + } + if err := os.Chmod(cleanupParent, 0o500); err != nil { + t.Fatalf("restrict cleanup parent: %v", err) + } + t.Cleanup(func() { _ = os.Chmod(cleanupParent, 0o700) }) + snapshots := []managedFileSnapshot{{Path: destination, Backup: backup, Mode: 0o600, Existed: true}} + if err := restoreManagedFiles(snapshots); err == nil { + t.Fatal("snapshot cleanup failure was discarded") + } +} + +func TestUninstallRemovesWiringAndPreservesStateUnlessPurged(t *testing.T) { + for _, purge := range []bool{false, true} { + t.Run("purge="+strings.ToLower(strconv.FormatBool(purge)), func(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + writeLifecycleRecoveryFiles(t, config) + paths := LifecyclePathsFor(config) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + if err := os.WriteFile(filepath.Join(paths.ProviderState, "retained.json"), []byte("state"), 0o600); err != nil { + t.Fatalf("write state: %v", err) + } + for _, path := range []string{paths.ProviderUnit, paths.RefreshUnit, paths.PathUnit, paths.TimerUnit, paths.AgentDropIn} { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatalf("mkdir unit dir: %v", err) + } + if err := os.WriteFile(path, []byte("unit"), 0o600); err != nil { + t.Fatalf("write unit: %v", err) + } + } + statuses := []string{"unavailable", "unavailable", "idle"} + runner := installSuccessRunner(t, config, "", "", &statuses) + baseRun := runner.run + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + switch installCommandEvent(command, config) { + case "maintenance-begin": + journal, found, err := readLifecycleJournal(home, paths) + if err != nil || !found || journal.Phase != LifecycleFencing || journal.Uninstall == nil || journal.Uninstall.Purge != purge { + t.Fatalf("uninstall maintenance begin lifecycle = %+v found=%v err=%v", journal, found, err) + } + case "agent-stop": + journal, found, err := readLifecycleJournal(home, paths) + if err != nil || !found || journal.Phase != LifecycleFenced { + t.Fatalf("uninstall agent stop lifecycle = %+v found=%v err=%v", journal, found, err) + } + case "maintenance-end": + journal, found, err := readLifecycleJournal(home, paths) + if err != nil || !found || journal.Phase != LifecycleReleasing || journal.Outcome != LifecycleCommit { + t.Fatalf("uninstall maintenance end lifecycle = %+v found=%v err=%v", journal, found, err) + } + } + return baseRun(ctx, command) + } + installer := Installer{Runner: runner, Sleep: func(context.Context, time.Duration) error { return nil }} + if _, err := installer.Uninstall(t.Context(), home, config, purge); err != nil { + t.Fatalf("uninstall: %v", err) + } + for _, path := range []string{paths.ProviderUnit, paths.RefreshUnit, paths.PathUnit, paths.TimerUnit, paths.AgentDropIn} { + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("wiring remains %s: %v", path, err) + } + } + _, rootErr := os.Stat(paths.Root) + if purge && !errors.Is(rootErr, os.ErrNotExist) { + t.Fatalf("purged root remains: %v", rootErr) + } + if !purge { + if data, err := os.ReadFile(filepath.Join(paths.ProviderState, "retained.json")); err != nil || string(data) != "state" { + t.Fatalf("non-purge state data=%q err=%v", data, err) + } + } + transcript := commandTranscript(runner.commands) + assertOrderedText(t, transcript, []string{"supervisor-maintenance begin", "local-status sanitize", "systemctl --user stop " + config.AgentUnit, "systemctl --user disable --now", "systemctl --user daemon-reload", "systemctl --user start " + config.AgentUnit, "supervisor-maintenance end", "local-status sanitize"}) + }) + } +} + +func TestUninstallLockContentionDoesNotMutateMaintenanceOrAgent(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + lock, err := AcquireInstallLock(paths.InstallLock) + if err != nil { + t.Fatalf("hold install lock: %v", err) + } + defer lock.Release() + runner := installSuccessRunner(t, config, "", "", new([]string)) + installer := Installer{Runner: runner, Sleep: func(context.Context, time.Duration) error { return nil }} + if _, err := installer.Uninstall(t.Context(), home, config, false); !errors.Is(err, ErrInstallLocked) { + t.Fatalf("contended uninstall err = %v", err) + } + transcript := commandTranscript(runner.commands) + if strings.Contains(transcript, "supervisor-maintenance begin") || strings.Contains(transcript, "systemctl --user stop "+config.AgentUnit) { + t.Fatalf("contended uninstall mutated maintenance or agent:\n%s", transcript) + } +} + +func installSuccessRunner(t *testing.T, config Config, payload, digest string, statuses *[]string) *recordingCommandRunner { + t.Helper() + writeLifecycleRecoveryFiles(t, config) + maintenanceActive := false + activeMaintenanceID := "" + activeMaintenanceReason := "" + runner := &recordingCommandRunner{} + runner.run = func(_ context.Context, command Command) ([]byte, error) { + if installCommandEvent(command, config) == "agent-signature" { + return agentUnitSystemdOutputForTest(t, config), nil + } + if command.Path == config.PodmanPath && len(command.Args) >= 2 && command.Args[0] == "image" && command.Args[1] == "inspect" { + return []byte(testProviderImageID + "\n"), nil + } + if command.Path == config.PodmanPath && len(command.Args) >= 2 && command.Args[0] == "network" && command.Args[1] == "inspect" { + return []byte("bridge true false\n"), nil + } + if filepath.Base(command.Path) == "systemctl" && containsArg(command.Args, "show") && (containsArg(command.Args, providerServiceUnit) || containsArg(command.Args, refreshPathUnit) || containsArg(command.Args, refreshTimerUnit)) { + if containsAdjacentArgs(command.Args, "--property", "ActiveState") && containsArg(command.Args, "--value") { + return []byte("active\n"), nil + } + unitPath := LifecyclePathsFor(config).ProviderUnit + switch { + case containsArg(command.Args, refreshPathUnit): + unitPath = LifecyclePathsFor(config).PathUnit + case containsArg(command.Args, refreshTimerUnit): + unitPath = LifecyclePathsFor(config).TimerUnit + } + if _, err := os.Lstat(unitPath); errors.Is(err, os.ErrNotExist) { + return []byte("LoadState=not-found\nFragmentPath=\nActiveState=inactive\nUnitFileState=\n"), nil + } else if err != nil { + return nil, err + } + return []byte("LoadState=loaded\nFragmentPath=" + unitPath + "\nActiveState=active\nUnitFileState=enabled\n"), nil + } + switch installCommandEvent(command, config) { + case "verify-update": + return testVerifiedUpdateJSON(config, payload, digest), nil + case "maintenance-begin": + maintenanceActive = true + reason := installMaintenanceReason + id := installMaintenanceID + if containsArg(command.Args, uninstallMaintenanceID) { + reason, id = uninstallMaintenanceReason, uninstallMaintenanceID + } + activeMaintenanceID, activeMaintenanceReason = id, reason + return maintenanceStateJSON(true, id, config.ProfileID, reason), nil + case "maintenance-status": + if !maintenanceActive { + return []byte(`{"active":false,"durable":true}`), nil + } + return maintenanceStateJSON(true, activeMaintenanceID, config.ProfileID, activeMaintenanceReason), nil + case "maintenance-end": + maintenanceActive = false + id := installMaintenanceID + reason := installMaintenanceReason + if containsArg(command.Args, uninstallMaintenanceID) { + id, reason = uninstallMaintenanceID, uninstallMaintenanceReason + } + return maintenanceStateJSON(false, id, config.ProfileID, reason), nil + case "local-status": + if len(*statuses) == 0 { + state := "idle" + if maintenanceActive { + state = "unavailable" + } + return localStatusJSON(config.WorkerID, state), nil + } + state := (*statuses)[0] + *statuses = (*statuses)[1:] + return localStatusJSON(config.WorkerID, state), nil + default: + return nil, nil + } + } + return runner +} + +func installCommandEvent(command Command, config Config) string { + if command.Path == config.ComputeAgentPath && len(command.Args) > 0 { + switch command.Args[0] { + case "supervisor-update": + return "verify-update" + case "supervisor-config": + return "supervisor-config-validate" + case "supervisor-maintenance": + if len(command.Args) > 1 && command.Args[1] == "begin" { + return "maintenance-begin" + } + if len(command.Args) > 1 && command.Args[1] == "status" { + return "maintenance-status" + } + return "maintenance-end" + case "local-status": + return "local-status" + } + } + if command.Path == config.PodmanPath { + return "podman-preflight" + } + if filepath.Base(command.Path) == "systemctl" { + joined := strings.Join(command.Args, " ") + switch { + case containsArg(command.Args, "show") && containsArg(command.Args, config.AgentUnit) && containsAdjacentArgs(command.Args, "--property", "DropInPaths"): + return "agent-signature" + case strings.Contains(joined, "show-environment"): + return "systemd-preflight" + case strings.Contains(joined, "daemon-reload"): + return "daemon-reload" + case strings.Contains(joined, "enable "+providerServiceUnit): + return "provider-enable" + case strings.Contains(joined, "enable --now "+refreshPathUnit): + return "refresh-watch-enable" + case strings.Contains(joined, "stop "+config.AgentUnit): + return "agent-stop" + case strings.Contains(joined, "start "+config.AgentUnit): + return "agent-start" + } + } + return filepath.Base(command.Path) + " " + strings.Join(command.Args, " ") +} + +func maintenanceStateJSON(active bool, id, profileID, reason string) []byte { + return []byte(`{"active":` + strconv.FormatBool(active) + `,"durable":true,"maintenance":{"kind":"workflow-compute.supervisor-maintenance.v1","id":"` + id + `","profile_id":"` + profileID + `","reason":"` + reason + `","started_at":"2026-07-13T00:00:00Z"}}`) +} + +func localStatusJSON(workerID, state string) []byte { + return []byte(`{"protocol_version":"compute.local_status.v1","worker_id":"` + workerID + `","state":"` + state + `","updated_at":"2026-07-13T00:00:00Z"}`) +} + +func assertOrderedEvents(t *testing.T, events, expected []string) { + t.Helper() + position := -1 + for _, want := range expected { + found := -1 + for index := position + 1; index < len(events); index++ { + if events[index] == want { + found = index + break + } + } + if found < 0 { + t.Fatalf("event %q missing after %d: %v", want, position, events) + } + position = found + } +} + +func assertOrderedText(t *testing.T, text string, expected []string) { + t.Helper() + position := -1 + for _, want := range expected { + found := strings.Index(text[position+1:], want) + if found < 0 { + t.Fatalf("text missing ordered %q after %d:\n%s", want, position, text) + } + position += found + 1 + } +} + +func parseCertificateForTest(t *testing.T, data []byte) *x509.Certificate { + t.Helper() + block, _ := pem.Decode(data) + if block == nil { + t.Fatalf("decode certificate PEM: %s", data) + } + certificate, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("parse certificate: %v", err) + } + return certificate +} + +func systemdEnvironmentValue(contents, key string) string { + for line := range strings.SplitSeq(contents, "\n") { + if value, found := strings.CutPrefix(line, key+"="); found { + return strings.Trim(value, "\"") + } + } + return "" +} + +func containsString(values []string, expected string) bool { + for _, value := range values { + if value == expected { + return true + } + } + return false +} + +func containsIP(values []net.IP, expected net.IP) bool { + for _, value := range values { + if value.Equal(expected) { + return true + } + } + return false +} From 744111d43564e49921bcdf49e599cd3620895d5b Mon Sep 17 00:00:00 2001 From: Jon Langevin Date: Tue, 14 Jul 2026 18:37:21 -0400 Subject: [PATCH 11/16] fix(provider): harden retained lifecycle Close crash-recovery, runtime ownership, timeout, and durable audit gaps found during adversarial review. Require patched TLS, HTTP, and transitive SDK dependencies before shipping the retained provider. --- .goreleaser.yaml | 3 + README.md | 97 ++ cmd/github-runner-provider/main_test.go | 52 +- cmd/github-runner-provider/probe.go | 15 +- cmd/github-runner-provider/retained_test.go | 3 +- ...tained-runner-provider-lifecycle-design.md | 541 ++++++- ...d-runner-provider-lifecycle-plan-review.md | 22 + examples/github-runner-retained-config.json | 29 + go.mod | 8 +- go.sum | 12 +- internal/retainedprovider/command.go | 27 +- internal/retainedprovider/config.go | 122 +- internal/retainedprovider/files.go | 70 +- internal/retainedprovider/files_test.go | 84 +- internal/retainedprovider/lifecycle.go | 249 ++- internal/retainedprovider/lifecycle_test.go | 348 +++- internal/retainedprovider/ownership_other.go | 16 +- internal/retainedprovider/ownership_unix.go | 24 + internal/retainedprovider/refresh.go | 930 ++++++++--- internal/retainedprovider/refresh_test.go | 1437 ++++++++++++++++- internal/retainedprovider/state.go | 119 +- internal/retainedprovider/state_test.go | 279 +++- internal/retainedprovider/syncdir_unix.go | 8 +- internal/retainedprovider/systemd.go | 502 ++++-- internal/retainedprovider/systemd_test.go | 574 ++++++- release_packaging_test.go | 149 ++ .../github-runner-retained-config.schema.json | 115 ++ 27 files changed, 5240 insertions(+), 595 deletions(-) create mode 100644 examples/github-runner-retained-config.json create mode 100644 schemas/github-runner-retained-config.schema.json diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 0886f1f..2a9bcd9 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -74,6 +74,9 @@ archives: - schemas/github-runner-provider.schema.json - schemas/github-runner-ephemeral-job-input.schema.json - schemas/github-runner-ephemeral-job-output.schema.json + - schemas/github-runner-retained-config.schema.json + - examples/github-runner-retained-config.json + - README.md - LICENSE checksum: diff --git a/README.md b/README.md index 81fca2b..4362aac 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,103 @@ provider-owned JIT runner ID recorded in the ownership journal. Workload outputs are returned through the declared `github-workload-outputs.tar.gz` provider artifact rather than arbitrary names. +#### Retained Linux provider + +The release archive includes `github-runner-provider` for a user-scoped Linux +installation alongside a retained workflow-compute agent. The host must have a +lingering user systemd manager, rootless Podman, and an agent bundle that +supports supervisor maintenance and signed package verification. The strict +non-secret config is generated for the registered worker and must use absolute +paths under that user's home. + +Enable the user manager once with administrative access, then verify the rest +as the retained agent user. Installation rejects UID 0, disabled linger, a +missing user manager, and a non-rootless Podman runtime. + +```sh +sudo loginctl enable-linger "$USER" +systemctl --user show-environment >/dev/null +test "$(loginctl show-user "$(id -u)" --property Linger --value)" = yes +test "$(podman info --format '{{.Host.Security.Rootless}}')" = true +``` + +The archive ships +`schemas/github-runner-retained-config.schema.json` and the runtime-validated +`examples/github-runner-retained-config.json`. Place the config in an +owner-only regular file under the retained user's home. Replace +`/home/wfcompute` with that exact home, and set the worker/profile IDs, agent +unit, supervisor paths, provider marker path, organization/repository, runner +group/labels, workflow, and full 40-character commit `ref` from the retained +agent registration and provider campaign. The following constraints are also +enforced by the runtime decoder: + +- `install_root` is exactly + `$HOME/.workflow-compute/github-runner-provider`. +- `provider_url` uses HTTPS on port `18090`; its host equals + `stable_container`, and `candidate_container` is different. +- `component_id` identifies the provider component in the agent supervisor + config, while `provider_marker_path` identifies that component's signed + current-update marker. +- `podman_path`, `systemctl_path`, and `loginctl_path` identify canonical + executable paths outside provider-managed state. Install rejects symlinks, + non-regular files, untrusted ownership, group/world-writable executables, and + lifecycle recovery re-attests their recorded digests before mutation. +- Every configured user path stays below the same home and has no symlinked + existing component. The config contains no credentials. + +Run the one-time install or an idempotent reinstall with credentials in the +process environment, never in the config or command arguments: + +```sh +GITHUB_RUNNER_PROVIDER_GITHUB_TOKEN="${GITHUB_TOKEN}" \ +GITHUB_RUNNER_PROVIDER_TOKEN="${PROVIDER_TOKEN}" \ + github-runner-provider retained install -config +``` + +After installation, autonomous refresh is driven by the retained agent's +signed package marker and a recurring user-systemd timer. A workflow is not +needed for routine provider updates. Check the redacted local state with: + +```sh +github-runner-provider retained status -config +``` + +For credential rotation, re-run `retained install` with the same config and the +replacement environment values. The transaction preserves the worker identity, +provider state, and retained-agent registration. A credential reinstall also +rotates the private CA and server key. Between reinstalls, refresh renews the +provider server certificate before its final 30 days while retaining the CA and +server key. + +Interrupted current-format transactions recover automatically on the next +lifecycle command. Only when an error explicitly reports an unbound legacy +provider transaction should an operator use the exact transaction ID printed +by that error: + +```sh +github-runner-provider retained recover -config \ + -confirm +``` + +Uninstall is deliberately separate from install/update orchestration. The +default retains provider state and credentials so a later reinstall can recover +ownership safely: + +```sh +github-runner-provider retained uninstall -config +``` + +Only remove retained state and credentials after update/reconnect evidence is +complete: + +```sh +github-runner-provider retained uninstall -config --purge +``` + +GitHub workflow output is orchestration evidence only. Acceptance requires a +job dispatched by workflow-compute STG to the registered agent and validation +through the STG task, proof, log, and artifact APIs. + ### Step: `step.gh_action_trigger` Triggers a GitHub Actions workflow via `workflow_dispatch`. diff --git a/cmd/github-runner-provider/main_test.go b/cmd/github-runner-provider/main_test.go index f91e99c..971a4c4 100644 --- a/cmd/github-runner-provider/main_test.go +++ b/cmd/github-runner-provider/main_test.go @@ -8,11 +8,13 @@ import ( "encoding/json" "encoding/pem" "errors" + "fmt" "io" "log/slog" "net" "net/http" "net/http/httptest" + "net/url" "os" "os/exec" "path/filepath" @@ -32,7 +34,7 @@ func TestProviderBinaryHasFallbackCertificateRoots(t *testing.T) { if err != nil { panic(err) } - if len(pool.Subjects()) == 0 { + if pool.Equal(x509.NewCertPool()) { panic("provider binary has no fallback certificate roots") } return @@ -271,6 +273,20 @@ func TestProviderProbeFailsClosedOnInvalidConfiguration(t *testing.T) { } } +func TestProviderProbeRejectsMoreThanRetainedConfigLabelLimit(t *testing.T) { + labels := make([]string, 65) + for index := range labels { + labels[index] = fmt.Sprintf("label-%d", index) + } + _, err := validateProviderProbeFlags( + "https://provider.test:18090", "/ca.pem", "GoCodeAlone", "GoCodeAlone/workflow-compute", + "dogfood-provider-target.yml", strings.Repeat("a", 40), "wfc-stg-ghp-linux-probe", "ephemeral", labels, + ) + if err == nil || !strings.Contains(err.Error(), "64") { + t.Fatalf("oversized label set err = %v", err) + } +} + func TestProviderProbeRejectsUnknownResponseFieldsAndDoesNotEchoErrorBody(t *testing.T) { const providerToken = "provider-secret-token" for _, tc := range []struct { @@ -310,6 +326,40 @@ func TestProviderProbeRejectsUnknownResponseFieldsAndDoesNotEchoErrorBody(t *tes } } +func TestProviderProbeRejectsRedirectWithoutForwardingBearer(t *testing.T) { + const providerToken = "provider-secret-token" + redirected := false + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/redirected" { + redirected = true + if r.Header.Get("Authorization") == "Bearer "+providerToken { + t.Error("redirected request received provider bearer token") + } + _, _ = io.WriteString(w, `{"status":"ok"}`) + return + } + http.Redirect(w, r, "/redirected", http.StatusTemporaryRedirect) + })) + defer server.Close() + caFile := writeProviderProbeTestCA(t, server) + client, err := newProviderProbeHTTPClient(caFile) + if err != nil { + t.Fatalf("build probe client: %v", err) + } + endpoint, err := url.Parse(server.URL + "/readyz") + if err != nil { + t.Fatalf("parse probe endpoint: %v", err) + } + var response providerProbeReadyResponse + err = providerProbeJSON(t.Context(), client, http.MethodGet, endpoint, providerToken, nil, &response) + if err == nil || !strings.Contains(err.Error(), "redirect") { + t.Fatalf("redirect probe error = %v", err) + } + if redirected { + t.Fatal("provider probe followed redirect") + } +} + func writeProviderProbeTestCA(t *testing.T, server *httptest.Server) string { t.Helper() caFile := filepath.Join(t.TempDir(), "ca.pem") diff --git a/cmd/github-runner-provider/probe.go b/cmd/github-runner-provider/probe.go index 2e6b0cd..25bf654 100644 --- a/cmd/github-runner-provider/probe.go +++ b/cmd/github-runner-provider/probe.go @@ -19,6 +19,7 @@ import ( "time" "github.com/GoCodeAlone/workflow-plugin-github/internal" + "github.com/GoCodeAlone/workflow-plugin-github/internal/retainedprovider" ) const ( @@ -196,8 +197,8 @@ func validateProviderProbeFlags(rawURL, caFile, organization, repository, workfl if len(runnerName) > 100 || !safeProviderProbeIdentifier(runnerName) { return nil, errors.New("provider runner name is invalid") } - if len(labels) == 0 { - return nil, errors.New("at least one -label is required") + if len(labels) == 0 || len(labels) > retainedprovider.MaxProviderProbeLabels { + return nil, fmt.Errorf("between 1 and %d -label values are required", retainedprovider.MaxProviderProbeLabels) } seen := make(map[string]struct{}, len(labels)) for _, label := range labels { @@ -230,7 +231,13 @@ func newProviderProbeHTTPClient(caFile string) (*http.Client, error) { } transport := http.DefaultTransport.(*http.Transport).Clone() transport.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: roots} - return &http.Client{Transport: transport, Timeout: providerProbeHTTPTimeout}, nil + return &http.Client{ + Transport: transport, + Timeout: providerProbeHTTPTimeout, + CheckRedirect: func(*http.Request, []*http.Request) error { + return errors.New("provider probe redirects are forbidden") + }, + }, nil } func providerProbeJSON(ctx context.Context, client *http.Client, method string, endpoint *url.URL, token string, input, output any) error { @@ -254,7 +261,7 @@ func providerProbeJSON(ctx context.Context, client *http.Client, method string, if err != nil { return fmt.Errorf("provider request failed: %w", err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, providerProbeMaxBodyBytes)) return fmt.Errorf("provider returned HTTP status %d", resp.StatusCode) diff --git a/cmd/github-runner-provider/retained_test.go b/cmd/github-runner-provider/retained_test.go index 56d8241..7bb48eb 100644 --- a/cmd/github-runner-provider/retained_test.go +++ b/cmd/github-runner-provider/retained_test.go @@ -280,7 +280,8 @@ func retainedCommandTestConfig(home string) retainedprovider.Config { ComputeAgentPath: filepath.Join(home, "compute-agent"), SupervisorConfigPath: filepath.Join(home, "supervisor.pb"), LocalStatusPath: filepath.Join(home, "status.json"), ProviderMarkerPath: filepath.Join(home, "updates", "current-provider.json"), InstallRoot: root, SystemdDir: filepath.Join(home, ".config", "systemd", "user"), AgentUnit: "workflow-compute-agent.service", - PodmanPath: "/usr/bin/podman", ProviderURL: "https://workflow-plugin-github-runner-provider:18090", + PodmanPath: "/usr/bin/podman", SystemctlPath: "/usr/bin/systemctl", LoginctlPath: "/usr/bin/loginctl", + ProviderURL: "https://workflow-plugin-github-runner-provider:18090", StableContainer: "workflow-plugin-github-runner-provider", CandidateContainer: "workflow-plugin-github-runner-provider-candidate", ContainerNetwork: "wfcompute-github-provider", Organization: "GoCodeAlone", Repository: "GoCodeAlone/workflow-compute", Workflow: "dogfood-provider-target.yml", Ref: strings.Repeat("a", 40), RunnerName: "wfc-stg-ghp-linux-probe", RunnerGroup: "ephemeral", diff --git a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md index a56c11b..b11d9a9 100644 --- a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md +++ b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md @@ -430,7 +430,7 @@ sibling journal. Transient/linked or otherwise unreconstructable systemd state is rejected before `fencing`. The provider subtransaction remains a second file because provider rollback -already has a five-phase durable protocol, but it is no longer an independent +already has a durable phase protocol, but it is no longer an independent authority. New inner records contain outer transaction id, profile id, and candidate digest. The outer record also has typed `provider_effect`: `changed|unchanged|not_applicable`. `changed` is phase-relative: inner may be @@ -449,7 +449,7 @@ fabricating a provider transaction. The accepted matrix is closed: |---|---|---|---| | `intent`,`fencing` | absent only | absent only | abort before mutation | | `adopting` | exact hash-bound legacy inner only for `refresh_recovery` | forbidden | establish/drain fence, then advance | -| `fenced` | absent or matching deferred `prepared`/`state_promoting`/`state_promoted`/`activated`/`committed` | absent | roll back | +| `fenced` | absent or matching deferred `staging`/`prepared`/`state_promoting`/`state_detached`/`state_promoted`/`activated`/`committed` or persisted rollback phase | absent | start or resume rollback | | `ready`,`releasing` commit+changed | matching deferred `committed` | forbidden | finish forward without re-fence or mutation | | `ready`,`releasing` commit+unchanged | absent; verified/active/probed digest bound by outer | forbidden | finish forward without re-fence or mutation | | `ready`,`releasing` commit+not_applicable | forbidden | absent; uninstall only | finish forward without re-fence or mutation | @@ -546,6 +546,57 @@ unattestable identity fails closed into the explicit recovery command. Scope: no manifest change; the recovery command is required operational repair for the locked lifecycle. +### Backport 2026-07-14: Effective User-Systemd Attestation + +Cause: systemd 255 omitted usable `EnvironmentFiles` data from `systemctl show`; +its `ExecStart` property also included runtime PID/start-time fields, so an +unchanged stopped unit produced a different signature. +Change: attest exact owner/identity/size-bounded fragment and drop-in bytes; +parse reset-aware static `EnvironmentFile` and `ExecStart` directives with +`go-systemd/unit`; hash the same opened bytes; reject optional, globbed, +specifier-bearing, relative, or otherwise unattestable environment paths; +canonicalize the redundant environment-file attestation list while preserving +ordered fragment/drop-in hashes that bind override semantics. +Scope: no manifest change. +Evidence: focused retained-provider tests cover omitted properties, reset +semantics, quoted paths, unsafe-path rejection, and stopped-unit stability; +the real Ubuntu 24.04 user manager completed install/uninstall/reinstall with +stable signatures. + +### Backport 2026-07-14: Recurring Refresh Timer + +Cause: `OnUnitActiveSec=300s` did not recur for an inactive `Type=oneshot` +refresh service; the timer became `active (elapsed)` without a next trigger. +Change: use `OnUnitInactiveSec=300s` so each completed refresh schedules the +next activation. +Scope: no manifest change. +Evidence: rendered-unit regression forbids `OnUnitActiveSec`; the runtime +timer fired naturally after five minutes, completed successfully, and exposed +a later next-elapse timestamp. + +### Backport 2026-07-14: Literal EnvironmentFile Path Encoding + +Cause: generic command quoting rendered `EnvironmentFile="/absolute/path"`; +systemd treated the quote as part of the path and ignored the file as +non-absolute. +Change: encode the absolute path with the systemd path-value encoder and reject +the quoted form in rendered-unit tests. +Scope: no manifest change. +Evidence: the corrected installed drop-in exposed the unquoted absolute path; +after daemon reload and agent restart, all four expected environment keys were +present and the user journal had no warnings. + +### Backport 2026-07-14: Branch-Wide Static Analysis + +Cause: Task 5's first branch-wide lint run found unchecked read-only closes, +one write-side directory close, deprecated certificate-pool inspection, and +four helpers made unreachable by the unified lifecycle redesign. +Change: make close handling explicit, join directory sync/close errors, compare +certificate pools semantically, and remove only proven-dead helpers. +Scope: no manifest change; no new product invariant because the existing +branch-wide lint gate directly detects recurrence. +Evidence: `golangci-lint run --new-from-rev=origin/main` → `0 issues`. + ## Task 4 Runtime Launch Transcript Environment: privileged Ubuntu 24.04 arm64 container booted with real user @@ -593,3 +644,489 @@ installed=true service_active=true; provider-state sentinel unchanged Failure-signature scrape: clean from the first successful install onward Verdict: PASS for the Task 4 user-systemd/Podman lifecycle boundary ``` + +### Backport 2026-07-14: Active-Only TLS Renewal Inspection + +Cause: renewal inspection was added before the refresh state machine separated +first activation from updates; initial activation then required an active state +that cannot exist yet and blocked the concurrency test behind its runner gate. +Change: first activation remains a fenced package mutation with installer +self-digest verification; TLS renewal inspection runs only when active state +exists. Refresh fixtures generate real CA/key/server material at the injected +clock instead of placeholder PEM. +Scope: no manifest change. +Evidence: removing the separation makes +`TestRefreshBuildsAndPreflightsIsolatedCandidateThenStable` fail with `fenced +refresh requires an active provider`; restored fix plus full retained package +tests pass in 27.7s. + +### Backport 2026-07-14: Current Server-Certificate Validity + +Cause: CA validity, signature, key binding, and SAN checks did not reject a +correctly signed server certificate whose `NotBefore` was still in the future. +Change: authority inspection fails closed for future-dated server material; +expired server certificates remain readable so the fenced renewal path can +replace them atomically. +Scope: no manifest change. +Evidence: fix removed → `TestProviderServerCertificateRejectsFutureValidity` +fails with `err = `; fix restored → test passes; branch lint → `0 issues`. + +### Backport 2026-07-14: Initial Installer Binding Across Marker Reads + +Cause: initial refresh verified the signed projection for installer self-digest, +then read the marker again before staging; a campaign update between reads could +replace the payload after the self-digest check. +Change: when no active provider exists, each verified projection used for +mutation is rebound to the running installer digest before any lifecycle journal, +maintenance, systemd, or Podman mutation. +Scope: no manifest change. +Evidence: fix removed → +`TestInitialRefreshRevalidatesInstallerAfterVerifiedUpdateChanges` advances into +runtime mutation; fix restored → exact test passes with two verify calls and no +other command. + +### Backport 2026-07-14: Installed Config Binds Every Mutation + +Cause: install accepted a different valid config at the same provider root and +could orphan the prior agent drop-in/unit/container identity; applying that +guard only to reinstall left refresh and uninstall asymmetric. +Change: after recovering any existing lifecycle journal, install, refresh, and +uninstall require the requested strict config to exactly equal owner-only +installed `config.json`; only first activation may lack that file. +Configuration migration requires a separate future transaction. +Scope: no manifest change. +Evidence: guard removed → +`TestReinstallRejectsChangedInstalledConfigBeforeCommands` and +`TestRefreshAndUninstallRejectChangedInstalledConfigBeforeCommands` reach host +preflight/verification; restored → changed worker/unit rejected with zero +commands while same-config credential rotation and uninstall cleanup recovery +pass. + +### Backport 2026-07-14: Fenced TLS Recovery Re-Proves Readiness + +Cause: atomic certificate renewal survived crashes but `Fenced + +ProviderUnchanged` recovery could restart/release the agent without restarting +or authenticating the provider. +Change: recovery retains maintenance, restarts provider, performs the real +stable semantic probe from durable unchanged provenance, records readiness, +then restarts the same agent and releases maintenance. +Scope: no manifest change. +Evidence: recovery branch removed → +`TestRecoverFencedTLSRefreshRestartsAndProbesProviderBeforeAgentRelease` observes +only agent start/end; restored → provider restart/probe precede both. + +### Backport 2026-07-14: SELinux And Bounded Runtime Retention + +Cause: unlabeled bind mounts fail on enforcing SELinux hosts; successful +campaigns retained all prior package directories and Podman image refs. +Change: mutable provider state mounts use private `Z`; TLS/CA shared by provider +and probes use read-only `z`. Post-commit, deferred-finalize, and committed +recovery GC remove only exact digest-owned image/package pairs image-first, +retain current+previous, and retry safely after interruption. +Scope: no manifest change. +Evidence: mount option removed → isolation test fails on exact volume; GC call +removed → deferred-finalize test leaves stale package. Full package → PASS. + +### Backport 2026-07-14: Config-Probe Validation Symmetry + +Cause: lifecycle config/schema accepted non-YAML workflow names and 101-128 byte +runner/label values rejected by the mandatory provider probe. +Change: runtime config and shipped schema require `.yml`/`.yaml` plus ≤100-byte +runner names/labels; the release example passes the runtime decoder. +Scope: no manifest change. +Evidence: suffix guard removed → config regression accepts non-YAML workflow; +restored config and release-contract tests → PASS. + +### Backport 2026-07-14: Terminal Cleanup Is Recoverable + +Cause: cleanup removed transaction snapshots before the durable committed +journal; a crash between removals made the remaining journal unreadable. +Change: only a fully committed journal may validate after any attested backup +has already been removed, including partial `RemoveAll` progress. Snapshot +metadata/path constraints remain mandatory; present backups are still +owner/mode/digest validated; every nonterminal journal requires every backup. +Scope: no manifest change. +Evidence: fix removed → +`TestRecoverCommittedLifecycleAfterTransactionRootCleanup` and +`TestRecoverCommittedLifecycleAfterPartialTransactionRootCleanup` fail on +missing backups; restored → both recover and remove the terminal journal. + +### Backport 2026-07-14: Audit Replay Rejects Truncation + +Cause: replay sought to a durable offset without proving the audit file still +covered it; append repair could create a sparse gap after external truncation. +Change: validate the opened file size before reading and immediately before an +offset `WriteAt`; never truncate during repair; sync and read back the exact +payload before removing it from the durable queue. Shorter files remain +unchanged and fail closed. +Scope: no manifest change. +Evidence: fix removed → +`TestLifecycleAuditDrainRejectsFileShorterThanDurableOffset` returns no error; +restored → exact test rejects the offset and preserves bytes. + +### Backport 2026-07-14: Audit Path Is Supervisor-Stable + +Cause: `LifecyclePathsFor` consulted ambient `XDG_STATE_HOME`; an interactive +install and its user-systemd refresh timer could persist audit queue offsets +against different files. +Change: derive audit and lock paths only from configured home at +`$HOME/.local/state/wfctl/plugins/workflow-plugin-github`; ambient process +environment cannot redirect lifecycle evidence. +Scope: no manifest change. +Evidence: fix removed → +`TestLifecycleAuditPathDoesNotDependOnAmbientStateHome` observes two files; +restored → interactive/systemd environments resolve the same path. + +### Backport 2026-07-14: Runtime State Is Config-Bound + +Cause: structural active-state and refresh-journal validation did not bind +worker/plugin/component/profile provenance to the installed config on every +runtime read. +Change: status, install, refresh, serve-active, deferred finalization, outer +install recovery, and lifecycle recovery validate current+previous selections +and nested interrupted journals against the strict installed identity before +commands or mutation. +Scope: no manifest change. +Evidence: binding removed → cross-worker serve reaches Podman validation and +cross-worker recovery completes; restored → +`TestServeActiveRejectsCrossWorkerActiveStateBeforePodman` and +`TestRecoverInterruptedRejectsCrossWorkerJournalBeforeCommands` reject with +zero commands; `TestRecoverInstallRejectsCrossWorkerDeferredJournalBeforeMutation` +also preserves the candidate transaction root. + +### Backport 2026-07-14: Executable Release Schema + +Cause: release tests checked schema JSON syntax and field substrings but did not +compile or consume the schema; its absolute-path regex was invalid for the +repository's schema engine. Unbounded labels also exceeded the probe/journal +contract. +Change: compile the shipped schema; validate the runtime-decodable example, +required-field/path/label boundaries; reject all ASCII controls/DEL in paths; +require an exact `podman` executable basename; and enforce one exported +64-label bound in runtime config, probe flags, and JSON Schema. +Scope: no manifest change. +Evidence: old regex → schema compilation fails on `\\u`; label guard removed → +65-label runtime test fails; restored release and config contract tests pass. + +### Backport 2026-07-14: Audit Variants Are Strict And Queue-Shaped + +Cause: audit validation accepted contradictory tagged-union fields, while the +first strict recovery rule overlooked that recovery events are coalesced +diagnostics with count/first/last summary fields. +Change: phase events alone carry outcome/provider effect/purge; recovery events +carry disposition plus a valid diagnostic summary; error/overflow events carry +error class plus summary. All variants reject foreign fields; overflow class is +`other`; pending digest and offset are all-or-none. +Scope: no manifest change. +Evidence: contradictory phase/recovery/error cases fail closed; recovery suite +and install rollback suite pass with queued recovery summaries. + +### Backport 2026-07-14: Pre-Mutation Artifact Ownership + +Cause: package copy and Podman build preceded the refresh journal, so failed or +crashed campaigns could retain up to 512 MiB packages plus images per digest. +Change: write a config-bound `staging` journal containing only signed update +provenance before package/image mutation. Rollback and restart recovery remove +only an image found at the deterministic ref with exact provider/worker/role/ +digest build labels, and remove its immutable image ID before the exact digest +package; current and rollback digest or image IDs are fail-closed exclusions. +Cleanup failure retains the journal and package for retry. The prepared phase +begins only after image ID inspection. +Scope: no manifest change; this replaces the rejected post-build transaction +approach. +Evidence: build/probe/activation failures remove only candidate artifacts; +staging crash recovery preserves the active provider without stopping it; +forced image-removal failure retains then successfully replays the journal. + +### Backport 2026-07-14: Durable Directory Entries + +Cause: `MkdirAll` plus leaf-directory sync did not persist each newly added +parent entry before a transaction could report durable completion. +Change: all retained-provider production directory creation is incremental; +each child creation is immediately followed by parent-directory sync. Atomic +JSON/files, locks, lifecycle/audit roots, packages, state, and systemd drop-ins +use the same primitive. +Scope: no manifest change. +Evidence: injected sync-order test proves `base -> base/one` after creating +`base/one/two`; production scan contains no `os.MkdirAll` outside tests. + +### Backport 2026-07-14: Cleanup Parent Roots Are Authoritative + +Cause: digest-child validation followed an intermediate `candidates` or +`packages` symlink before removal, allowing rollback cleanup outside the +managed install root. +Change: validate each managed parent as an owned real directory before child +lookup/removal; missing roots remain idempotent. Artifact cleanup also refuses +digests named by current or rollback active state. +Scope: no manifest change. +Evidence: candidate-state and package-root symlink regressions preserve outside +sentinels; focused and full retained suites pass. + +### Backport 2026-07-14: Fenced Reciprocal Rollback + +Cause: a crash inside nested refresh can leave a deferred inner journal before +the fenced outer journal copies its explicit binding. +Change: only a fenced outer transaction may roll back an unrecorded inner when +the inner reciprocally names the exact outer transaction/profile and matches +worker/plugin/component identity. Ready/commit still require explicit binding. +Scope: no manifest change. +Evidence: full recovery matrix passes for staging, prepared, promoting, +promoted, activated, and committed inner phases without forward adoption. + +### Backport 2026-07-14: Producer-Consumer Boundary Limits + +Cause: accepted credentials could exceed the environment scanner token limit; +the shipped schema rejected runtime-valid `/podman`. +Change: credentials are capped at 32 KiB before rendering; schema and runtime +share root/nested Podman path acceptance. Audit overflow is constructed as a +fresh variant so recovery disposition cannot leak into overflow fields. +Scope: no manifest change. +Evidence: exact credential boundary, `/podman`, and recovery-overflow tests +fail on the prior implementation and pass after the correction. + +### Backport 2026-07-14: Provider Environment Cannot Expand Authority + +Cause: runtime validation required configured repository, organization, and +runner-group values to appear in comma-separated lists, so a modified env file +could silently add scopes. It also accepted an unconfigured GitHub API base URL. +Change: retained provider allowlists are canonical singleton values that must +exactly equal strict config, and unbound API-base configuration is rejected. +The general provider executable retains its independently configured multi-scope +and GitHub Enterprise support; the retained installer does not acquire either +implicitly from mutable environment. +Scope: no manifest change. +Evidence: `TestProviderEnvironmentCannotBroadenConfiguredGitHubAuthority` fails +for repository, organization, runner-group, and API-base expansion when the old +contains/allow behavior is restored. + +### Backport 2026-07-14: Host Executors Are Explicit Recovery Authority + +Cause: Podman was path-configured but not durably attested, while systemctl and +loginctl were hard-coded. A mutable executable path could therefore change the +commands used by startup or crash recovery. +Change: strict config names canonical absolute Podman, systemctl, and loginctl +paths outside all managed and external authority paths. Runtime preflight +requires regular executable files with root/current-user ownership and no +group/world write permission. Lifecycle journals persist all three content +digests and re-attest them before recovery mutation; active startup and status +validate the executor immediately before first use. No ambient PATH lookup is +used. +Scope: no manifest change. +Evidence: `TestInstallHostPreflightRequiresNonRootLingeringAndRootlessPodman` +fails when systemctl/loginctl revert to hard-coded paths; +`TestHostPreflightRejectsUntrustedExecutableBeforeCommands` fails when host +authority validation is removed; and +`TestLifecycleRecoveryAttestsConfiguredHostExecutables` fails when recovery +stops checking the recorded Podman digest. + +### Backport 2026-07-14: Snapshot Authority Survives Rollback Transition + +Cause: fenced wiring recovery deleted snapshot backups before durably writing +`ready{outcome:rollback}`; a crash stranded a `fenced` journal whose recovery +authority no longer existed. +Change: lifecycle rollback restores bytes/units but retains snapshot metadata +and backups through ready/releasing/committed. Terminal transaction cleanup +removes them. Legacy one-shot rollback still removes backups after success. +Scope: no manifest change. +Evidence: `TestRollbackInstallBeforeStartRetainsSnapshotsForLifecycleCommit` +fails when the helper removes backups and passes when outer cleanup owns them. + +### Backport 2026-07-14: Durable Provider Rename And Rollback Phases + +Cause: two cross-directory state renames shared one journal phase; rollback +renamed previous state back and could crash before journal removal, making retry +misclassify the restored state as missing rollback authority. +Change: promotion persists `state_detached` between renames and syncs both +source/destination parents per rename. Rollback persists +`rollback_restoring -> rollback_restored -> rollback_cleaned`, records the exact +forward origin, and resumes each phase idempotently before journal removal. +Scope: no manifest change. +Evidence: `TestProviderStatePromotionPersistsEachCrossDirectoryRename`, +`TestRecoverInterruptedResumesAfterPreviousStateWasAlreadyRestored`, and +`TestRecoverInterruptedFinishesEveryPersistedRollbackPhase` pass. + +### Backport 2026-07-14: Config Paths Cannot Alias Managed State + +Cause: externally authoritative agent/supervisor/status/marker/systemd paths +could alias one another or installer-managed provider files, collapsing trust +boundaries and making rollback overwrite its own inputs. +Change: external authority paths are pairwise non-overlapping, outside the +dedicated install root, and cannot equal, contain, or be contained by any +generated managed state path. The systemd authority directory may contain only +its expected generated unit/drop-in paths and cannot overlap other authorities. +Scope: no manifest change. +Evidence: `TestConfigRejectsUnsafeIdentityAndPaths` and +`TestConfigRejectsAuthorityOverlapWithLifecycleState` reject managed aliases, +ancestor/descendant authority overlap, and agent/systemd paths inside +`install_root`. + +### Backport 2026-07-14: Fixed-Name Containers Require Podman Ownership + +Cause: cleanup force-removed the configured candidate name without proving +ownership, while stable and probe fixed names had no ownership or stale-crash +recovery contract. +Change: candidate, stable, and probe creation applies managed/worker/role +labels. Cleanup accepts only configured name/role pairs, queries the exact +regex-escaped name, validates one full immutable ID plus all labels, and removes +only that ID; absent is idempotent and collisions fail closed. Every probe +attempt cleans before and after execution, and post-run cleanup uses a detached +two-command budget so both ownership inspection and removal receive their own +bounded command window after caller cancellation. The aggregate probe budget +counts both pre-run and post-run inspect/remove paths for every attempt. Config validation requires the +stable, candidate, stable-probe, and candidate-probe derived names to be unique +so cleanup authority cannot collide across roles. +Scope: no manifest change. +Evidence: `TestProviderCommandsCarryRoleSpecificCleanupOwnershipLabels`, +`TestServeActiveValidatesImmutableImageThenExecsRestrictedPodman`, +`TestRefreshRemovesOwnedStaleProbeBeforeRetry`, and +`TestManagedProbeCleansOwnedOrphanAfterCallerCancellation` pass; +`TestConfigRejectsManagedContainerNameCollisions` fails when derived-name +validation is removed. + +### Backport 2026-07-14: Refresh Unit Covers Bounded Aggregate Runtime + +Cause: systemd allowed 15 minutes while a bounded build plus candidate/stable +probe retries, three status waits, and ownership cleanup can legitimately exceed +that duration. The first 45-minute correction still omitted complete status and +probe-cleanup budgets. +Change: shared computed refresh and rollback bounds cover three worst-case local +status loops, build, candidate/stable probe attempts and delays, four ownership +commands per probe attempt, container starts, control operations, and a +filesystem margin. The systemd start bound additionally composes initial +lifecycle recovery, deferred install/provider rollback, the full forward +refresh, and failure recovery. Its explicit stop bound leaves a complete +lifecycle-recovery window after cancellation; per-command limits remain +unchanged. +Scope: no manifest change. +Evidence: `TestRenderSystemdUnitsUsesStableAbsolutePathsAndNoShell` asserts the +rendered aggregate timeout; `TestRetainedTimeoutsCoverBoundedRefreshAndRollbackOperations` +proves both aggregate bounds dominate their component budgets. + +### Backport 2026-07-14: Failure Recovery Has An Aggregate Budget + +Cause: install, uninstall, and refresh failure handlers, plus legacy wiring +rollback, placed an entire multi-command recovery sequence under one 30-second +deadline even though each bounded control command may consume that duration. +Change: lifecycle and wiring rollback use computed aggregate deadlines derived +from the retained rollback, local-status, probe, and control-command budgets; +caller cancellation still cannot interrupt durable recovery. +Scope: no manifest change. +Evidence: the candidate-probe failure path observes the full stable-probe +deadline during recovery, and +`TestRollbackInstallBeforeStartRetainsSnapshotsForLifecycleCommit` observes a +multi-command rollback deadline above one control-command interval. Replacing +the computed wiring deadline with 30 seconds makes the latter test fail. + +### Backport 2026-07-14: Provider Image Cleanup Is Immutable And Owned + +Cause: cleanup selected an image by mutable deterministic tag even though the +prepared journal records its immutable ID; a same-name image could therefore be +deleted after tag rebinding. +Change: provider image builds carry exact managed/worker/role/digest labels. +Prepared cleanup and active startup inventory the exact durable image ID, +validate its managed/worker/role/digest labels, treat absence as idempotent for +cleanup, and remove or execute only that normalized ID. Pre-ID staging recovery +inventories by the complete ownership-label tuple and rejects ambiguity. +Mutable tags are never cleanup or startup authority. Garbage collection uses +the same owned-image path. +Scope: no manifest change. +Evidence: `TestRollbackImageCleanupRequiresOwnershipAndImmutableID` covers +absent, unowned, ID-mismatch, and owned cases; the build test proves all labels +are emitted and fails when they are removed. +`TestProviderImageCleanupUsesImmutableIDOrOwnershipLabelsWithoutTag` and +`TestServeActiveValidatesImmutableImageThenExecsRestrictedPodman` fail when the +inventory is changed back to a mutable reference filter. + +### Backport 2026-07-14: Same-Digest Runtime Artifacts Are Reconciled + +Cause: digest equality bypassed mutation and only probed the durable image ID, +so Podman storage loss or managed-label drift could never rebuild an otherwise +valid signed provider package. +Change: reconciliation inventories the durable active image before selecting the +unchanged path. Absence or exact-label drift enters a fenced `runtime_repair` +transaction that reuses the verified package, rebuilds and probes candidate and +stable containers, and preserves the older rollback selection instead of +duplicating the repaired digest. Failed repair removes only the rebuilt owned +image, retains the verified package and prior durable selection, skips a +knowingly absent-image probe, clears both journals, and leaves the next timer +free to retry. Malformed or ambiguous Podman inventory still fails closed. +Scope: no manifest change. +Evidence: `TestSameDigestRefreshRepairsMissingActiveImageUnderFence`, +`TestSameDigestOwnershipDriftRequiresRuntimeRepair`, +`TestFailedSameDigestRepairRemovesImageButRetainsVerifiedPackage`, and +`TestRuntimeRepairJournalAllowsSameDigestAndPreservesPriorSelection` cover +fencing, rebuild, ownership drift, rollback cleanup, package retention, and +committed recovery. Removing outer drift detection makes repair proceed without +the maintenance fence and fails the first test. + +### Backport 2026-07-14: Outer And Inner Provider Effects Must Agree + +Cause: install selected `unchanged` from digest equality before checking the +active Podman image, and image loss between outer classification and inner +refresh could trigger runtime mutation without the outer maintenance fence. +Change: install inventories the same-digest active image before selecting its +provider effect. The inner refresh receives the outer expected digest/effect, +rechecks immediately before mutation, and returns a mutation-required sentinel +on drift. An unchanged refresh then closes its clean outer transaction and +restarts through the changed, fenced lifecycle while retaining the install lock. +Scope: no manifest change. +Evidence: `TestReinstallMissingActiveImageUsesChangedProviderTransaction` and +`TestSameDigestImageLossRaceRestartsThroughFenceBeforeRepair` pass; reverting +either outer classification or inner effect enforcement reproduces the +transaction mismatch or unfenced build. + +### Backport 2026-07-14: Rendered Systemd Bounds Round Up + +Cause: converting computed durations with integer division truncated +fractional seconds, so a rendered systemd deadline could be shorter than the +bounded operation it protects. +Change: all rendered systemd timeout directives use ceiling conversion to +whole seconds. +Scope: no manifest change. +Evidence: `TestRenderSystemdUnitsUsesStableAbsolutePathsAndNoShell` parses each +directive and proves its duration is greater than or equal to the computed Go +budget; restoring truncation makes the test fail. + +### Backport 2026-07-14: Managed Paths Inherit Trusted Authority + +Cause: user-path validation skipped the home directory itself, and durable +directory creation trusted the first existing ancestor without checking its +owner or group/other writability. +Change: the home and every existing managed-path component must be a real +directory owned by the current user and not group/other writable on Unix. +Durable directory creation and tree cloning validate the nearest existing +ancestor before adding children. +Scope: no manifest change. +Evidence: `TestValidateUserPathRejectsSymlinkedHomeAndWritableAuthority` and +`TestDurableDirectoryCreationRejectsWritableExistingAncestor` pass; removing +the authority checks makes both regressions fail. + +### Backport 2026-07-14: Example Runtime Proof Uses Owned Host State + +Cause: the packaging test decoded the Linux operator example against its +literal `/home/wfcompute` path, assuming that account existed on every clean +test host after home-authority validation became mandatory. +Change: schema proof still consumes the exact shipped example; runtime proof +rebases only its documented home prefix onto an owned `t.TempDir()` and then +runs the strict production decoder. +Scope: no manifest change. +Evidence: `TestReleaseArchiveIncludesRetainedProviderConfigContract` fails with +`inspect home authority` on a host without `/home/wfcompute` before the change +and passes against real temporary authority afterward. + +### Backport 2026-07-14: Shipped Provider Uses Fixed Runtime Dependencies + +Cause: the release module still selected Go `1.26.4`, `x/net v0.54.0`, and +Kinesis `v1.43.4`; `govulncheck` reached the Go TLS and `x/net/idna` +advisories through the new provider server/probe and the Kinesis decoder panic +through transitive SDK initialization. +Change: require Go `1.26.5`, `x/net v0.55.0`, its compatible `x/sys v0.45.0`, +and Kinesis `v1.43.5`. Five inherited Docker advisories remain because no fixed +Docker module release exists; this provider path does not call the affected +archive/copy/AuthZ APIs. Removing the Workflow SDK's Docker dependency from the +provider binary is a later control-plane/dependency-light extraction, not a +manifest change here. +Scope: no manifest change. +Evidence: `govulncheck ./cmd/github-runner-provider` drops the two standard +library, `x/net`, and Kinesis findings and reports only the five no-fix Docker +advisories after the pins. diff --git a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md index 7a945a9..23d9bb8 100644 --- a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md +++ b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md @@ -71,3 +71,25 @@ gates. The revised tasks now cover active-image process semantics, probe secret isolation, TLS/env boundaries, and the Go static-analysis gate. No unresolved Critical or Important findings remain. + +## Implementation Adversarial Review + +| round | verdict | findings | resolution/evidence | +|---|---|---|---| +| 1 | REQUEST-CHANGES | rootless/linger docs; maintenance transaction IDs; local-status freshness; redirect handling; diagnostic audit durability; TLS renewal/current validity | hardened preflight, transaction-scoped maintenance, fresh status timestamps, redirect rejection, durable diagnostics, TLS renewal/validity tests; package tests PASS | +| 2 | REQUEST-CHANGES | reinstall config binding; TLS recovery restart/probe; SELinux labels; unbounded package/image retention; config/probe mismatch | exact installed-config guard on install/refresh/uninstall; fenced TLS probe; `Z`/`z`; current+previous GC; shared config/probe limits; focused + package tests PASS | +| 3 | REQUEST-CHANGES | committed cleanup ordered before journal removal; audit replay accepted truncation; active runtime provenance unbound; labels unbounded; schema not executed | committed cleanup recovery; durable offset checks; config-bound active/journal state; 64 labels; compiled schema/example/negative tests; focused + package tests PASS | +| 4 | REQUEST-CHANGES | nested deferred recovery not config-bound; audit path ambient; partial cleanup crash; audit tagged union loose; custom Podman timeout mismatch; host-specific test commands | config-bound reads before mutation; home-derived audit path; partial cleanup tolerance only after commit; strict queue-shaped variants; exact `podman` basename; current test-binary fixtures; focused + package tests PASS | +| 4 focused | REQUEST-CHANGES | proposed adding `ProfileID` to active artifact state | rejected: active artifact authorization identity is worker/plugin/component; profile is local outer transaction/supervisor routing and remains bound in config+journals. Adding it to provider artifact state would change protocol without closing an authorization gap. | +| 5 | REVERT-AND-REWRITE | failed updates leaked candidate packages/images; new directory entries lacked parent fsync; recovery-overflow conversion retained illegal disposition; credential scanner and root Podman schema mismatched | confirmed. Prior post-build transaction approach rejected. Rewritten with pre-mutation staging ownership, incremental parent-sync directory creation, fresh overflow construction, 32 KiB credential bound, and schema/runtime path parity. | +| post-rewrite 1 | REQUEST-CHANGES | fenced rollback removed snapshot authority before transition; provider-state rollback/promotion lacked replayable rename phases; config paths could alias managed state; candidate cleanup lacked ownership proof; refresh timeout was short | retained snapshots until terminal cleanup; added detached + rollback phases with parent fsync; enforced path separation; labeled/ID-bound cleanup; rendered 45-minute aggregate timeout; focused regressions PASS. | +| post-rewrite 2 | REQUEST-CHANGES | rollback timeout shorter than probes; incomplete authority overlap checks; aggregate timeout omitted worst-case loops; stable/probe names lacked ownership recovery; image cleanup used mutable tags | computed rollback/refresh budgets including probe cleanup; complete ancestor/descendant authority isolation; role-bound ownership and cancellation-safe stale cleanup for all fixed container names; labeled image inventory plus immutable-ID deletion; focused and package tests PASS. | +| post-rewrite 3 | REQUEST-CHANGES | lifecycle failure recovery had a 30-second aggregate deadline; Podman/systemctl/loginctl lacked complete path and recovery authority; prepared/active images still depended on mutable tags; retained provider env could broaden GitHub scopes | computed lifecycle/wiring recovery deadlines; explicit configured host tools with permission/content attestations; immutable-ID or complete-label image inventory; exact singleton GitHub authority and unbound API-base rejection; focused revert/restore proofs and retained package tests PASS. | +| post-rewrite 4 | REQUEST-CHANGES | systemd omitted initial/failure recovery from its service deadline; detached probe cleanup and aggregate arithmetic allowed only half the required commands; same-digest runtime loss had no rebuild path | composed start/stop service deadlines; two-command detached cleanup and four-command-per-attempt aggregate; explicit fenced same-digest repair with owned-image/package-safe rollback; focused revert/restore proofs and retained package tests PASS. | +| post-rewrite 5 | REVERT-AND-REWRITE | same-digest install could select unchanged without runtime integrity; image-loss race could mutate outside maintenance; rendered systemd timeouts truncated fractional seconds; managed path creation trusted home/ancestors without full authority validation | outer/inner digest+effect agreement with fenced restart; ceiling-rounded directives; home and nearest-existing-ancestor owner/writability validation; focused revert/restore proofs and retained package tests PASS. | +| post-rewrite 6 | REQUEST-CHANGES | packaged provider used vulnerable Go TLS and `x/net/idna` paths plus a fixed Kinesis decoder panic; path comment omitted writability; effect guard obscured precedence | Go 1.26.5, `x/net` 0.55.0, `x/sys` 0.45.0, Kinesis 1.43.5; comment and guard clarified; scoped vulnerability and focused tests rerun. | +| post-rewrite 7 | SHIP-IT | no Critical/Important; five inherited Docker advisories have no fixed release and affected archive/copy/AuthZ APIs are not called by this provider path | full scope/checklist pass; residual SDK linkage recorded for later dependency-light extraction; final verification gate required before PR. | + +Round 5 rejected the prior mechanism. The affected durability/recovery layer was +rewritten rather than advanced. A new post-rewrite review cycle must reach +`SHIP-IT` before PR creation. diff --git a/examples/github-runner-retained-config.json b/examples/github-runner-retained-config.json new file mode 100644 index 0000000..40d1a77 --- /dev/null +++ b/examples/github-runner-retained-config.json @@ -0,0 +1,29 @@ +{ + "protocol_version": "retained-provider.config.v1", + "worker_id": "github-runner-linux-stg", + "profile_id": "github-runner-profile-stg", + "plugin_id": "workflow-plugin-github", + "component_id": "github-runner-provider-sidecar", + "compute_agent_path": "/home/wfcompute/.workflow-compute/agent-core-bin/github-runner-linux-stg/compute-agent", + "supervisor_config_path": "/home/wfcompute/.workflow-compute/github-runner-linux-stg/supervisor.pb", + "local_status_path": "/home/wfcompute/.workflow-compute/github-runner-linux-stg/agent-status.json", + "provider_marker_path": "/home/wfcompute/.workflow-compute/updates/updates/current/provider-workflow-plugin-github--component-Z2l0aHViLXJ1bm5lci1wcm92aWRlci1zaWRlY2Fy.json", + "install_root": "/home/wfcompute/.workflow-compute/github-runner-provider", + "systemd_dir": "/home/wfcompute/.config/systemd/user", + "agent_unit": "workflow-compute-github-runner-linux-stg.service", + "podman_path": "/usr/bin/podman", + "systemctl_path": "/usr/bin/systemctl", + "loginctl_path": "/usr/bin/loginctl", + "provider_url": "https://workflow-plugin-github-runner-provider:18090", + "stable_container": "workflow-plugin-github-runner-provider", + "candidate_container": "workflow-plugin-github-runner-provider-candidate", + "container_network": "wfcompute-github-provider", + "organization": "GoCodeAlone", + "repository": "GoCodeAlone/workflow-compute", + "workflow": "dogfood-provider-target.yml", + "ref": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "runner_name": "wfc-stg-ghp-linux-probe", + "runner_group": "ephemeral", + "labels": ["self-hosted", "linux", "wfc-ghp-stg"], + "refresh_interval_seconds": 300 +} diff --git a/go.mod b/go.mod index 82222c5..54a8c36 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/GoCodeAlone/workflow-plugin-github -go 1.26.4 +go 1.26.5 require ( github.com/GoCodeAlone/workflow v0.64.0 @@ -11,7 +11,7 @@ require ( github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 golang.org/x/crypto v0.51.0 golang.org/x/crypto/x509roots/fallback v0.0.0-20260712151947-c1a3b97d708a - golang.org/x/sys v0.44.0 + golang.org/x/sys v0.45.0 google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af ) @@ -42,7 +42,7 @@ require ( github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.23 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.8 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.22 // indirect - github.com/aws/aws-sdk-go-v2/service/kinesis v1.43.4 // indirect + github.com/aws/aws-sdk-go-v2/service/kinesis v1.43.5 // indirect github.com/aws/aws-sdk-go-v2/service/signin v1.0.10 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.30.16 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.20 // indirect @@ -182,7 +182,7 @@ require ( go.yaml.in/yaml/v2 v2.4.4 // indirect golang.org/x/arch v0.27.0 // indirect golang.org/x/mod v0.36.0 // indirect - golang.org/x/net v0.54.0 // indirect + golang.org/x/net v0.55.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/text v0.37.0 // indirect diff --git a/go.sum b/go.sum index 8318678..2c2ed01 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.8 h1:HtOTYcb github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.8/go.mod h1:VsK9abqQeGlzPgUr+isNWzPlK2vKe9INMLWnY65f5Xs= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.22 h1:PUmZeJU6Y1Lbvt9WFuJ0ugUK2xn6hIWUBBbKuOWF30s= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.22/go.mod h1:nO6egFBoAaoXze24a2C0NjQCvdpk8OueRoYimvEB9jo= -github.com/aws/aws-sdk-go-v2/service/kinesis v1.43.4 h1:3m9iJtMtLq75jKRAfw0kapoHUlbzi0CRVigysBN/FHA= -github.com/aws/aws-sdk-go-v2/service/kinesis v1.43.4/go.mod h1:O2L6vGm4xacEuN2otHFMgn7yXXlgzFKzxrba0fy/yk8= +github.com/aws/aws-sdk-go-v2/service/kinesis v1.43.5 h1:LxgRVyuY+5DEPSX7kmin/V7toE8MWZ9U8n2dqRtX+RE= +github.com/aws/aws-sdk-go-v2/service/kinesis v1.43.5/go.mod h1:eUebEBEqVfOwEyDDDbGauH4PNqDCuepRvTaNbJeWr5w= github.com/aws/aws-sdk-go-v2/service/signin v1.0.10 h1:a1Fq/KXn75wSzoJaPQTgZO0wHGqE9mjFnylnqEPTchA= github.com/aws/aws-sdk-go-v2/service/signin v1.0.10/go.mod h1:p6+MXNxW7IA6dMgHfTAzljuwSKD0NCm/4lbS4t6+7vI= github.com/aws/aws-sdk-go-v2/service/sso v1.30.16 h1:x6bKbmDhsgSZwv6q19wY/u3rLk/3FGjJWyqKcIRufpE= @@ -688,8 +688,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= -golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= @@ -729,8 +729,8 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= -golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= diff --git a/internal/retainedprovider/command.go b/internal/retainedprovider/command.go index f0f0fe8..528a220 100644 --- a/internal/retainedprovider/command.go +++ b/internal/retainedprovider/command.go @@ -14,11 +14,28 @@ import ( ) const ( - defaultCommandOutputBytes = 1 << 20 - controlCommandTimeout = 30 * time.Second - containerStartTimeout = time.Minute - providerProbeTimeout = 2 * time.Minute - providerBuildTimeout = 10 * time.Minute + defaultCommandOutputBytes = 1 << 20 + controlCommandTimeout = 30 * time.Second + containerStartTimeout = time.Minute + providerProbeTimeout = 2 * time.Minute + providerBuildTimeout = 10 * time.Minute + providerProbeAttemptCount = 5 + providerProbeDelay1 = 250 * time.Millisecond + providerProbeDelay2 = 500 * time.Millisecond + providerProbeDelay3 = time.Second + providerProbeDelay4 = 2 * time.Second + providerProbeBudget = providerProbeAttemptCount*providerProbeTimeout + providerProbeDelay1 + providerProbeDelay2 + providerProbeDelay3 + providerProbeDelay4 + managedContainerCleanupTimeout = 2 * controlCommandTimeout + providerProbeCleanupBudget = 2 * providerProbeAttemptCount * managedContainerCleanupTimeout + managedProviderProbeBudget = providerProbeBudget + providerProbeCleanupBudget + localStatusWaitBudget = localStatusAttempts*controlCommandTimeout + (localStatusAttempts-1)*time.Second + retainedOperationMargin = 20*controlCommandTimeout + 5*time.Minute + retainedRefreshTimeout = 3*localStatusWaitBudget + providerBuildTimeout + 2*managedProviderProbeBudget + 2*containerStartTimeout + retainedOperationMargin + retainedRollbackTimeout = managedProviderProbeBudget + 6*controlCommandTimeout + 5*time.Minute + lifecycleRecoveryTimeout = retainedRollbackTimeout + 4*localStatusWaitBudget + 16*controlCommandTimeout + 5*time.Minute + installRollbackTimeout = 12*controlCommandTimeout + 5*time.Minute + retainedRefreshServiceStartTimeout = 2*lifecycleRecoveryTimeout + retainedRollbackTimeout + installRollbackTimeout + retainedRefreshTimeout + retainedRefreshServiceStopTimeout = lifecycleRecoveryTimeout ) type Command struct { diff --git a/internal/retainedprovider/config.go b/internal/retainedprovider/config.go index 54d90f9..a29bae7 100644 --- a/internal/retainedprovider/config.go +++ b/internal/retainedprovider/config.go @@ -16,6 +16,8 @@ const ( GitHubPluginID = "workflow-plugin-github" providerContainerNetwork = "wfcompute-github-provider" maxConfigBytes = 1 << 20 + providerProbeValueBytes = 100 + MaxProviderProbeLabels = 64 ) var ( @@ -39,6 +41,8 @@ type Config struct { SystemdDir string `json:"systemd_dir"` AgentUnit string `json:"agent_unit"` PodmanPath string `json:"podman_path"` + SystemctlPath string `json:"systemctl_path"` + LoginctlPath string `json:"loginctl_path"` ProviderURL string `json:"provider_url"` StableContainer string `json:"stable_container"` CandidateContainer string `json:"candidate_container"` @@ -107,17 +111,40 @@ func (config Config) Validate(home string) error { if !strings.HasSuffix(config.AgentUnit, ".service") { return fmt.Errorf("agent_unit must end in .service") } + if len(config.RunnerName) > providerProbeValueBytes { + return fmt.Errorf("runner_name must be at most %d bytes", providerProbeValueBytes) + } if config.PluginID != GitHubPluginID { return fmt.Errorf("plugin_id must be %q", GitHubPluginID) } - if config.StableContainer == config.CandidateContainer { - return fmt.Errorf("candidate_container must differ from stable_container") + managedContainerNames := []string{ + config.StableContainer, + config.CandidateContainer, + config.StableContainer + "-probe", + config.CandidateContainer + "-probe", + } + seenContainerNames := make(map[string]struct{}, len(managedContainerNames)) + for _, name := range managedContainerNames { + if _, exists := seenContainerNames[name]; exists { + return fmt.Errorf("managed container names must be distinct") + } + seenContainerNames[name] = struct{}{} } if config.ContainerNetwork != providerContainerNetwork { return fmt.Errorf("container_network must be %s", providerContainerNetwork) } - if !filepath.IsAbs(config.PodmanPath) || containsControl(config.PodmanPath) { - return fmt.Errorf("podman_path must be an absolute safe path") + for _, tool := range []struct { + field string + path string + base string + }{ + {field: "podman_path", path: config.PodmanPath, base: "podman"}, + {field: "systemctl_path", path: config.SystemctlPath, base: "systemctl"}, + {field: "loginctl_path", path: config.LoginctlPath, base: "loginctl"}, + } { + if !filepath.IsAbs(tool.path) || filepath.Clean(tool.path) != tool.path || containsControl(tool.path) || filepath.Base(tool.path) != tool.base { + return fmt.Errorf("%s must be an absolute canonical safe path to %s", tool.field, tool.base) + } } for field, path := range map[string]string{ "compute_agent_path": config.ComputeAgentPath, @@ -135,6 +162,56 @@ func (config Config) Validate(home string) error { if filepath.Clean(config.InstallRoot) != expectedInstallRoot { return fmt.Errorf("install_root must be the dedicated provider root %s", expectedInstallRoot) } + externalPaths := []struct { + field string + path string + }{ + {field: "compute_agent_path", path: filepath.Clean(config.ComputeAgentPath)}, + {field: "supervisor_config_path", path: filepath.Clean(config.SupervisorConfigPath)}, + {field: "local_status_path", path: filepath.Clean(config.LocalStatusPath)}, + {field: "provider_marker_path", path: filepath.Clean(config.ProviderMarkerPath)}, + {field: "systemd_dir", path: filepath.Clean(config.SystemdDir)}, + {field: "podman_path", path: filepath.Clean(config.PodmanPath)}, + {field: "systemctl_path", path: filepath.Clean(config.SystemctlPath)}, + {field: "loginctl_path", path: filepath.Clean(config.LoginctlPath)}, + } + for index, external := range externalPaths { + for prior := 0; prior < index; prior++ { + if external.path == externalPaths[prior].path { + return fmt.Errorf("%s and %s must be distinct", externalPaths[prior].field, external.field) + } + } + relative, err := filepath.Rel(expectedInstallRoot, external.path) + if err != nil { + return fmt.Errorf("%s relative to install_root: %w", external.field, err) + } + if relative == "." || relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return fmt.Errorf("%s must remain outside install_root", external.field) + } + } + paths := LifecyclePathsFor(config) + reserved := reservedProviderPaths(paths) + for _, external := range externalPaths { + for _, managed := range reserved { + if pathsOverlap(external.path, managed) { + return fmt.Errorf("%s must not overlap a managed provider path", external.field) + } + } + if external.field != "systemd_dir" { + for _, managed := range managedWiringPaths(paths) { + if pathsOverlap(external.path, managed) { + return fmt.Errorf("%s must not overlap a managed provider path", external.field) + } + } + } + } + for index, external := range externalPaths { + for prior := 0; prior < index; prior++ { + if pathsOverlap(external.path, externalPaths[prior].path) { + return fmt.Errorf("%s and %s authority paths overlap", externalPaths[prior].field, external.field) + } + } + } providerURL, err := url.Parse(config.ProviderURL) if err != nil || providerURL.Scheme != "https" || providerURL.Host == "" || providerURL.User != nil || providerURL.RawQuery != "" || providerURL.Fragment != "" || (providerURL.Path != "" && providerURL.Path != "/") || providerURL.Port() != "18090" { return fmt.Errorf("provider_url must be an HTTPS URL without credentials, query, or fragment") @@ -146,18 +223,18 @@ func (config Config) Validate(home string) error { if len(parts) != 2 || parts[0] != config.Organization || !safeIdentifierPattern.MatchString(parts[1]) { return fmt.Errorf("repository must be organization/name for the configured organization") } - if !workflowPattern.MatchString(config.Workflow) || strings.Contains(config.Workflow, "..") || containsControl(config.Workflow) { + if !workflowPattern.MatchString(config.Workflow) || strings.Contains(config.Workflow, "..") || containsControl(config.Workflow) || (!strings.HasSuffix(config.Workflow, ".yml") && !strings.HasSuffix(config.Workflow, ".yaml")) { return fmt.Errorf("workflow contains an unsafe path") } if !gitRefPattern.MatchString(config.Ref) { return fmt.Errorf("ref must be a full lowercase commit SHA") } - if len(config.Labels) == 0 { - return fmt.Errorf("labels must not be empty") + if len(config.Labels) == 0 || len(config.Labels) > MaxProviderProbeLabels { + return fmt.Errorf("labels must contain between 1 and %d entries", MaxProviderProbeLabels) } seenLabels := make(map[string]struct{}, len(config.Labels)) for _, label := range config.Labels { - if !safeIdentifierPattern.MatchString(label) { + if len(label) > providerProbeValueBytes || !safeIdentifierPattern.MatchString(label) { return fmt.Errorf("labels contains an unsafe label") } if _, exists := seenLabels[label]; exists { @@ -171,6 +248,35 @@ func (config Config) Validate(home string) error { return nil } +func reservedProviderPaths(paths LifecyclePaths) []string { + return []string{ + paths.Root, + paths.ConfigFile, paths.Launcher, paths.ActiveState, paths.Journal, + paths.InstallLock, paths.InstallJournal, + paths.LifecycleJournal, paths.LifecycleTransactions, + paths.LifecycleAudit, paths.LifecycleAuditLock, + paths.ProviderState, paths.PackagesRoot, paths.CandidatesRoot, + paths.ProviderEnv, paths.ProbeEnv, paths.AgentEnv, + paths.CAKey, paths.TLSRoot, paths.CAFile, paths.ServerCert, paths.ServerKey, + paths.ContainersConf, + } +} + +func pathsOverlap(left, right string) bool { + left = filepath.Clean(left) + right = filepath.Clean(right) + if left == right { + return true + } + for _, pair := range [][2]string{{left, right}, {right, left}} { + relative, err := filepath.Rel(pair[0], pair[1]) + if err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return true + } + } + return false +} + func decodeStrictJSON(reader io.Reader, target any) error { decoder := json.NewDecoder(reader) decoder.DisallowUnknownFields() diff --git a/internal/retainedprovider/files.go b/internal/retainedprovider/files.go index e71557d..ac7a875 100644 --- a/internal/retainedprovider/files.go +++ b/internal/retainedprovider/files.go @@ -30,7 +30,7 @@ func AtomicWriteJSON(path string, value any) (returnErr error) { return fmt.Errorf("encoded JSON exceeds %d bytes", MaxStateFileBytes) } directory := filepath.Dir(path) - if err := os.MkdirAll(directory, 0o700); err != nil { + if err := mkdirAllDurable(directory, 0o700); err != nil { return fmt.Errorf("create state directory: %w", err) } if err := rejectNonRegularDestination(path); err != nil { @@ -71,6 +71,49 @@ func AtomicWriteJSON(path string, value any) (returnErr error) { return nil } +func mkdirAllDurable(path string, mode fs.FileMode) error { + return mkdirAllDurableWithSync(path, mode, syncDirectory) +} + +func mkdirAllDurableWithSync(path string, mode fs.FileMode, syncDir func(string) error) error { + if path == "" || mode.Perm() != mode || mode&0o077 != 0 || syncDir == nil { + return errors.New("durable directory path, mode, and sync are required") + } + path = filepath.Clean(path) + missing := make([]string, 0, 4) + ancestor := path + for { + info, err := os.Lstat(ancestor) + if err == nil { + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("durable directory ancestor must be a real directory: %s", ancestor) + } + if err := validateManagedPathAuthority(info); err != nil { + return fmt.Errorf("durable directory ancestor authority: %w", err) + } + break + } + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("inspect durable directory ancestor: %w", err) + } + missing = append(missing, ancestor) + parent := filepath.Dir(ancestor) + if parent == ancestor { + return errors.New("durable directory has no existing ancestor") + } + ancestor = parent + } + for index := len(missing) - 1; index >= 0; index-- { + if err := os.Mkdir(missing[index], mode); err != nil { + return fmt.Errorf("create durable directory: %w", err) + } + if err := syncDir(filepath.Dir(missing[index])); err != nil { + return fmt.Errorf("sync durable directory parent: %w", err) + } + } + return nil +} + func ReadStrictJSONFile(path string, target any) error { entry, err := os.Lstat(path) if err != nil { @@ -92,7 +135,7 @@ func ReadStrictJSONFile(path string, target any) error { if err != nil { return fmt.Errorf("open state file: %w", err) } - defer file.Close() + defer func() { _ = file.Close() }() opened, err := file.Stat() if err != nil { return fmt.Errorf("stat opened state file: %w", err) @@ -117,13 +160,23 @@ func ReadStrictJSONFile(path string, target any) error { } // ValidateUserPath enforces a lexical user-home boundary and rejects symlinks -// or foreign-owned files in every existing component below that boundary. +// or untrusted ownership and writability in every existing component. func ValidateUserPath(home, path string, requireExisting bool) error { if !filepath.IsAbs(home) || !filepath.IsAbs(path) { return fmt.Errorf("path and home must be absolute") } home = filepath.Clean(home) path = filepath.Clean(path) + homeInfo, err := os.Lstat(home) + if err != nil { + return fmt.Errorf("inspect home authority: %w", err) + } + if !homeInfo.IsDir() || homeInfo.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("home authority must be a real directory without symlinks") + } + if err := validateManagedPathAuthority(homeInfo); err != nil { + return fmt.Errorf("home authority: %w", err) + } relative, err := filepath.Rel(home, path) if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) || filepath.IsAbs(relative) { return fmt.Errorf("path must remain within home") @@ -145,8 +198,8 @@ func ValidateUserPath(home, path string, requireExisting bool) error { if info.Mode()&os.ModeSymlink != 0 { return fmt.Errorf("path contains symlink: %s", current) } - if err := validateOwner(info); err != nil { - return fmt.Errorf("path ownership: %w", err) + if err := validateManagedPathAuthority(info); err != nil { + return fmt.Errorf("path authority: %w", err) } } } @@ -190,6 +243,9 @@ func cloneRegularTreeWithSync(source, destination string, limits CloneLimits, sy if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { return fmt.Errorf("clone destination ancestor must be a regular directory") } + if err := validateManagedPathAuthority(info); err != nil { + return fmt.Errorf("clone destination ancestor authority: %w", err) + } break } if !errors.Is(err, os.ErrNotExist) { @@ -277,7 +333,7 @@ func cloneRegularFile(source, destination string, expected os.FileInfo) (returnE if err != nil { return err } - defer input.Close() + defer func() { _ = input.Close() }() opened, err := input.Stat() if err != nil || !opened.Mode().IsRegular() || !os.SameFile(expected, opened) { return fmt.Errorf("clone source file changed during open") @@ -333,7 +389,7 @@ type InstallLock struct { } func AcquireInstallLock(path string) (*InstallLock, error) { - if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + if err := mkdirAllDurable(filepath.Dir(path), 0o700); err != nil { return nil, fmt.Errorf("create lock directory: %w", err) } file, err := openRegularLockFile(path) diff --git a/internal/retainedprovider/files_test.go b/internal/retainedprovider/files_test.go index 484c88d..0cd8669 100644 --- a/internal/retainedprovider/files_test.go +++ b/internal/retainedprovider/files_test.go @@ -50,6 +50,22 @@ func TestAtomicWriteJSONUsesRestrictiveRegularFile(t *testing.T) { } } +func TestDurableDirectoryCreationSyncsEachNewParentEntry(t *testing.T) { + base := t.TempDir() + target := filepath.Join(base, "one", "two") + var synced []string + if err := mkdirAllDurableWithSync(target, 0o700, func(path string) error { + synced = append(synced, filepath.Clean(path)) + return nil + }); err != nil { + t.Fatalf("create durable directory: %v", err) + } + want := []string{base, filepath.Join(base, "one")} + if strings.Join(synced, "\n") != strings.Join(want, "\n") { + t.Fatalf("directory sync order = %v want %v", synced, want) + } +} + func TestReadStrictJSONFileRejectsUnknownAndOversizedData(t *testing.T) { dir := t.TempDir() unknown := filepath.Join(dir, "unknown.json") @@ -117,6 +133,70 @@ func TestValidateUserPathRejectsSymlinkedAncestorAndOutsideHome(t *testing.T) { } } +func TestValidateUserPathRejectsSymlinkedHomeAndWritableAuthority(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows does not expose Unix directory authority semantics") + } + root := t.TempDir() + realHome := filepath.Join(root, "real-home") + if err := os.Mkdir(realHome, 0o700); err != nil { + t.Fatalf("mkdir real home: %v", err) + } + linkedHome := filepath.Join(root, "linked-home") + if err := os.Symlink(realHome, linkedHome); err != nil { + t.Fatalf("symlink home: %v", err) + } + if err := ValidateUserPath(linkedHome, filepath.Join(linkedHome, "state.json"), false); err == nil || !strings.Contains(err.Error(), "symlink") { + t.Fatalf("symlinked home err = %v", err) + } + + secureHome := filepath.Join(root, "secure-home") + if err := os.Mkdir(secureHome, 0o700); err != nil { + t.Fatalf("mkdir secure home: %v", err) + } + if err := os.Chmod(secureHome, 0o777); err != nil { + t.Fatalf("make home writable: %v", err) + } + if err := ValidateUserPath(secureHome, filepath.Join(secureHome, "state.json"), false); err == nil || !strings.Contains(err.Error(), "writable") { + t.Fatalf("writable home err = %v", err) + } + if err := os.Chmod(secureHome, 0o700); err != nil { + t.Fatalf("restore home mode: %v", err) + } + writable := filepath.Join(secureHome, "writable") + if err := os.Mkdir(writable, 0o700); err != nil { + t.Fatalf("mkdir writable ancestor: %v", err) + } + if err := os.Chmod(writable, 0o777); err != nil { + t.Fatalf("make ancestor writable: %v", err) + } + if err := ValidateUserPath(secureHome, filepath.Join(writable, "state.json"), false); err == nil || !strings.Contains(err.Error(), "writable") { + t.Fatalf("writable ancestor err = %v", err) + } +} + +func TestDurableDirectoryCreationRejectsWritableExistingAncestor(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows does not expose Unix directory authority semantics") + } + base := t.TempDir() + authority := filepath.Join(base, "authority") + if err := os.Mkdir(authority, 0o700); err != nil { + t.Fatalf("mkdir authority: %v", err) + } + if err := os.Chmod(authority, 0o777); err != nil { + t.Fatalf("make authority writable: %v", err) + } + t.Cleanup(func() { _ = os.Chmod(authority, 0o700) }) + target := filepath.Join(authority, "managed", "state") + if err := mkdirAllDurableWithSync(target, 0o700, func(string) error { return nil }); err == nil || !strings.Contains(err.Error(), "writable") { + t.Fatalf("writable durable ancestor err = %v", err) + } + if _, err := os.Lstat(filepath.Join(authority, "managed")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("durable creation mutated insecure authority: %v", err) + } +} + func TestCloneRegularTreeCopiesOnlyBoundedRegularFiles(t *testing.T) { source := filepath.Join(t.TempDir(), "source") destination := filepath.Join(t.TempDir(), "destination") @@ -183,7 +263,7 @@ func TestInstallLockIsExclusive(t *testing.T) { if err != nil { t.Fatalf("first lock: %v", err) } - defer first.Release() + defer func() { _ = first.Release() }() if _, err := AcquireInstallLock(path); err == nil || !errors.Is(err, ErrInstallLocked) { t.Fatalf("second lock err = %v", err) } @@ -227,7 +307,7 @@ func TestLifecycleLockRemainsExclusiveWhileInstallRootIsPurged(t *testing.T) { if err != nil { t.Fatalf("acquire lifecycle lock: %v", err) } - defer lock.Release() + defer func() { _ = lock.Release() }() if err := os.RemoveAll(paths.Root); err != nil { t.Fatalf("purge install root: %v", err) } diff --git a/internal/retainedprovider/lifecycle.go b/internal/retainedprovider/lifecycle.go index f013ae8..baf0454 100644 --- a/internal/retainedprovider/lifecycle.go +++ b/internal/retainedprovider/lifecycle.go @@ -106,6 +106,9 @@ type LifecycleRecoveryAuthority struct { Config Config `json:"config"` ComputeAgent LifecycleFileAttestation `json:"compute_agent"` SupervisorConfig LifecycleFileAttestation `json:"supervisor_config"` + Podman LifecycleFileAttestation `json:"podman"` + Systemctl LifecycleFileAttestation `json:"systemctl"` + Loginctl LifecycleFileAttestation `json:"loginctl"` AgentUnitBefore LifecycleSystemdSignature `json:"agent_unit_before"` } @@ -202,6 +205,22 @@ func (authority LifecycleRecoveryAuthority) Validate(home string, identity Lifec if authority.SupervisorConfig.Path != authority.Config.SupervisorConfigPath { return errors.New("lifecycle supervisor config attestation path mismatch") } + for _, executable := range []struct { + label string + attestation LifecycleFileAttestation + path string + }{ + {label: "podman", attestation: authority.Podman, path: authority.Config.PodmanPath}, + {label: "systemctl", attestation: authority.Systemctl, path: authority.Config.SystemctlPath}, + {label: "loginctl", attestation: authority.Loginctl, path: authority.Config.LoginctlPath}, + } { + if err := executable.attestation.Validate(); err != nil { + return fmt.Errorf("validate lifecycle %s attestation: %w", executable.label, err) + } + if executable.attestation.Path != executable.path { + return fmt.Errorf("lifecycle %s attestation path mismatch", executable.label) + } + } if err := authority.AgentUnitBefore.Validate(home); err != nil { return err } @@ -312,17 +331,31 @@ func (event LifecycleAuditEvent) Validate() error { } switch event.Kind { case AuditPhase: - if event.ErrorClass != "" || event.Disposition != "" || event.Count != 0 || !event.FirstSeen.IsZero() || !event.LastSeen.IsZero() { - return errors.New("phase audit event error_class or diagnostic fields are invalid") + if (event.Outcome != "" && event.Outcome != LifecycleCommit && event.Outcome != LifecycleRollback) || + (event.ProviderEffect != ProviderChanged && event.ProviderEffect != ProviderUnchanged && event.ProviderEffect != ProviderNotApplicable) || + event.ErrorClass != "" || event.Disposition != "" || event.Count != 0 || !event.FirstSeen.IsZero() || !event.LastSeen.IsZero() { + return errors.New("phase audit event outcome, provider effect, error_class, or diagnostic fields are invalid") + } + if event.Operation == LifecycleUninstall { + if event.ProviderEffect != ProviderNotApplicable || event.Purge == nil { + return errors.New("uninstall phase audit event requires purge intent") + } + } else if event.ProviderEffect == ProviderNotApplicable || event.Purge != nil { + return errors.New("non-uninstall phase audit event has invalid provider effect or purge intent") } case AuditRecovery: - if !safeIdentifierPattern.MatchString(event.Disposition) || event.ErrorClass != "" { + if !safeIdentifierPattern.MatchString(event.Disposition) || event.Outcome != "" || event.ProviderEffect != "" || event.Purge != nil || + event.ErrorClass != "" || event.Count == 0 || event.FirstSeen.IsZero() || event.LastSeen.Before(event.FirstSeen) { return errors.New("recovery audit event disposition is invalid") } case AuditError, AuditOverflow: - if !safeIdentifierPattern.MatchString(event.ErrorClass) || event.Count == 0 || event.FirstSeen.IsZero() || event.LastSeen.Before(event.FirstSeen) || event.Outcome != "" || event.ProviderEffect != "" { + if !safeIdentifierPattern.MatchString(event.ErrorClass) || event.Count == 0 || event.FirstSeen.IsZero() || event.LastSeen.Before(event.FirstSeen) || + event.Outcome != "" || event.ProviderEffect != "" || event.Purge != nil || event.Disposition != "" { return errors.New("error audit event error_class or summary is invalid") } + if event.Kind == AuditOverflow && event.ErrorClass != "other" { + return errors.New("overflow audit event error_class must be other") + } default: return errors.New("lifecycle audit event kind is invalid") } @@ -332,6 +365,9 @@ func (event LifecycleAuditEvent) Validate() error { if event.Offset != nil && *event.Offset < 0 { return errors.New("lifecycle audit event offset is invalid") } + if (event.Digest == "") != (event.Offset == nil) { + return errors.New("lifecycle audit event pending append metadata is incomplete") + } return nil } @@ -368,8 +404,12 @@ func (queue *LifecycleAuditQueue) EnqueueDiagnostic(event LifecycleAuditEvent) e if len(queue.Diagnostics) >= maxLifecycleDiagnosticEvents { return errors.New("lifecycle audit diagnostic queue is full") } - event.Kind = AuditOverflow - event.ErrorClass = "other" + event = LifecycleAuditEvent{ + EventID: event.EventID, Timestamp: event.Timestamp, + TransactionID: event.TransactionID, WorkerID: event.WorkerID, + Operation: event.Operation, Phase: event.Phase, + Kind: AuditOverflow, ErrorClass: "other", + } } event.Sequence = queue.NextSequence event.Count = 1 @@ -560,19 +600,23 @@ func (journal LifecycleJournal) validateSnapshots(home string, paths LifecyclePa if snapshot.Mode != 0o600 && snapshot.Mode != 0o700 { return errors.New("lifecycle snapshot mode is invalid") } - if err := ValidateUserPath(home, snapshot.Backup, true); err != nil { + if !digestPattern.MatchString(snapshot.SHA256) { + return errors.New("lifecycle snapshot digest is invalid") + } + requireBackup := journal.Phase != LifecycleCommitted + if err := ValidateUserPath(home, snapshot.Backup, requireBackup); err != nil { return fmt.Errorf("validate lifecycle snapshot: %w", err) } info, err := os.Lstat(snapshot.Backup) + if journal.Phase == LifecycleCommitted && errors.Is(err, os.ErrNotExist) { + continue + } if err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != snapshot.Mode { return errors.New("lifecycle snapshot is not an owner-only regular file") } if err := validateOwner(info); err != nil { return fmt.Errorf("validate lifecycle snapshot owner: %w", err) } - if !digestPattern.MatchString(snapshot.SHA256) { - return errors.New("lifecycle snapshot digest is invalid") - } digest, err := hashRegularFile(snapshot.Backup, snapshot.Mode&0o100 != 0) if err != nil || digest != snapshot.SHA256 { return errors.New("lifecycle snapshot digest mismatch") @@ -713,7 +757,7 @@ func writeLifecycleJournal(home string, paths LifecyclePaths, journal LifecycleJ } if journal.Phase == LifecycleIntent { transactionRoot := paths.LifecycleTransactionRoot(journal.TransactionID) - if err := os.MkdirAll(transactionRoot, 0o700); err != nil { + if err := mkdirAllDurable(transactionRoot, 0o700); err != nil { return fmt.Errorf("create lifecycle transaction root: %w", err) } if err := validateOwnedDirectory(transactionRoot); err != nil { @@ -753,7 +797,7 @@ func drainLifecycleAudit(home string, paths LifecyclePaths, journal *LifecycleJo if err := validateLifecyclePathBoundary(home, paths); err != nil { return err } - if err := os.MkdirAll(filepath.Dir(paths.LifecycleAudit), 0o700); err != nil { + if err := mkdirAllDurable(filepath.Dir(paths.LifecycleAudit), 0o700); err != nil { return fmt.Errorf("create lifecycle audit directory: %w", err) } lock, err := AcquireInstallLock(paths.LifecycleAuditLock) @@ -795,6 +839,10 @@ func drainLifecycleAudit(home string, paths LifecyclePaths, journal *LifecycleJo _ = file.Close() return errors.New("lifecycle audit pending digest mismatch") } + if err := validateLifecycleAuditOffset(file, *event.Offset); err != nil { + _ = file.Close() + return err + } if _, err := file.Seek(*event.Offset, io.SeekStart); err != nil { _ = file.Close() return fmt.Errorf("seek lifecycle audit: %w", err) @@ -856,18 +904,38 @@ func openLifecycleAudit(path string) (*os.File, error) { } func appendLifecycleAuditAt(file *os.File, offset int64, payload []byte) error { - if err := file.Truncate(offset); err != nil { - return fmt.Errorf("truncate lifecycle audit tail: %w", err) - } - if _, err := file.Seek(offset, io.SeekStart); err != nil { - return fmt.Errorf("seek lifecycle audit append: %w", err) + if err := validateLifecycleAuditOffset(file, offset); err != nil { + return err } - if _, err := file.Write(payload); err != nil { + written, err := file.WriteAt(payload, offset) + if err != nil { return fmt.Errorf("append lifecycle audit: %w", err) } + if written != len(payload) { + return io.ErrShortWrite + } if err := file.Sync(); err != nil { return fmt.Errorf("sync lifecycle audit: %w", err) } + stored := make([]byte, len(payload)) + read, err := file.ReadAt(stored, offset) + if err != nil && !errors.Is(err, io.EOF) { + return fmt.Errorf("verify lifecycle audit append: %w", err) + } + if read != len(payload) || !bytes.Equal(stored, payload) { + return errors.New("lifecycle audit append verification failed") + } + return nil +} + +func validateLifecycleAuditOffset(file *os.File, offset int64) error { + info, err := file.Stat() + if err != nil { + return fmt.Errorf("stat lifecycle audit offset: %w", err) + } + if info.Size() < offset { + return errors.New("lifecycle audit is shorter than pending offset") + } return nil } @@ -969,17 +1037,36 @@ func (authority LifecycleRecoveryAuthority) Reattest() error { if supervisorDigest != authority.SupervisorConfig.SHA256 { return errors.New("lifecycle supervisor config attestation mismatch") } + for _, executable := range []struct { + label string + attestation LifecycleFileAttestation + }{ + {label: "podman", attestation: authority.Podman}, + {label: "systemctl", attestation: authority.Systemctl}, + {label: "loginctl", attestation: authority.Loginctl}, + } { + digest, err := hashHostExecutable(executable.attestation.Path) + if err != nil { + return fmt.Errorf("re-attest lifecycle %s: %w", executable.label, err) + } + if digest != executable.attestation.SHA256 { + return fmt.Errorf("lifecycle %s attestation mismatch", executable.label) + } + } return nil } -func lifecycleMaintenanceIdentity(operation LifecycleOperation) (id, reason string, err error) { - switch operation { +func lifecycleMaintenanceIdentity(journal LifecycleJournal) (id, reason string, err error) { + if !safeIdentifierPattern.MatchString(journal.TransactionID) { + return "", "", errors.New("lifecycle transaction has no maintenance identity") + } + switch journal.Operation { case LifecycleInstall: - return installMaintenanceID, installMaintenanceReason, nil + return journal.TransactionID, installMaintenanceReason, nil case LifecycleUninstall: - return uninstallMaintenanceID, uninstallMaintenanceReason, nil + return journal.TransactionID, uninstallMaintenanceReason, nil case LifecycleRefresh, LifecycleRefreshRecovery: - return refreshMaintenanceID, refreshMaintenanceReason, nil + return journal.TransactionID, refreshMaintenanceReason, nil default: return "", "", errors.New("lifecycle operation has no maintenance identity") } @@ -1013,6 +1100,34 @@ func writeLifecycleTransition(home string, paths LifecyclePaths, journal *Lifecy return nil } +func writeLifecycleDiagnostic(home string, paths LifecyclePaths, journal *LifecycleJournal, kind LifecycleAuditKind, value string, now time.Time) error { + if journal == nil { + return errors.New("lifecycle journal is required") + } + event := LifecycleAuditEvent{ + EventID: "event-" + fmt.Sprint(journal.Audit.NextSequence), Timestamp: now.UTC(), + TransactionID: journal.TransactionID, WorkerID: journal.Identity.WorkerID, + Operation: journal.Operation, Phase: journal.Phase, Kind: kind, + } + switch kind { + case AuditError: + event.ErrorClass = value + case AuditRecovery: + event.Disposition = value + default: + return errors.New("unsupported lifecycle diagnostic kind") + } + if err := journal.Audit.EnqueueDiagnostic(event); err != nil { + return fmt.Errorf("enqueue lifecycle diagnostic audit: %w", err) + } + journal.UpdatedAt = now.UTC() + if err := writeLifecycleJournal(home, paths, *journal); err != nil { + return err + } + _ = drainLifecycleAudit(home, paths, journal) + return nil +} + func newLifecycleJournal(config Config, operation LifecycleOperation, effect ProviderEffect, uninstall *LifecycleUninstallPayload, now time.Time) (LifecycleJournal, error) { home := lifecycleHome(LifecyclePathsFor(config)) if err := config.Validate(home); err != nil { @@ -1026,6 +1141,18 @@ func newLifecycleJournal(config Config, operation LifecycleOperation, effect Pro if err != nil { return LifecycleJournal{}, fmt.Errorf("attest lifecycle supervisor config: %w", err) } + podmanDigest, err := hashHostExecutable(config.PodmanPath) + if err != nil { + return LifecycleJournal{}, fmt.Errorf("attest lifecycle podman: %w", err) + } + systemctlDigest, err := hashHostExecutable(config.SystemctlPath) + if err != nil { + return LifecycleJournal{}, fmt.Errorf("attest lifecycle systemctl: %w", err) + } + loginctlDigest, err := hashHostExecutable(config.LoginctlPath) + if err != nil { + return LifecycleJournal{}, fmt.Errorf("attest lifecycle loginctl: %w", err) + } if now.IsZero() { now = time.Now().UTC() } @@ -1043,6 +1170,9 @@ func newLifecycleJournal(config Config, operation LifecycleOperation, effect Pro Config: config, ComputeAgent: LifecycleFileAttestation{Path: config.ComputeAgentPath, SHA256: computeAgentDigest}, SupervisorConfig: LifecycleFileAttestation{Path: config.SupervisorConfigPath, SHA256: supervisorDigest}, + Podman: LifecycleFileAttestation{Path: config.PodmanPath, SHA256: podmanDigest}, + Systemctl: LifecycleFileAttestation{Path: config.SystemctlPath, SHA256: systemctlDigest}, + Loginctl: LifecycleFileAttestation{Path: config.LoginctlPath, SHA256: loginctlDigest}, }, Uninstall: uninstall, Audit: LifecycleAuditQueue{NextSequence: 1}, @@ -1089,6 +1219,9 @@ func (installer Installer) recoverLifecycleTransaction(ctx context.Context, home } return installer.adoptLegacyProviderTransaction(ctx, home, paths, refresher, nil, "") } + if err := writeLifecycleDiagnostic(home, paths, &journal, AuditRecovery, "resume_"+string(journal.Phase), installer.now()); err != nil { + return fmt.Errorf("record lifecycle recovery disposition: %w", err) + } if err := journal.Recovery.Reattest(); err != nil { return err } @@ -1181,14 +1314,15 @@ func (installer Installer) recoverLifecycleAdopting(ctx context.Context, home st if err := validateLifecycleProviderMatrix(paths, *journal); err != nil { return err } - id, reason, err := lifecycleMaintenanceIdentity(journal.Operation) + id, reason, err := lifecycleMaintenanceIdentity(*journal) if err != nil { return err } - if err := installer.beginMaintenance(ctx, journal.Recovery.Config, id, reason); err != nil { + maintenance, err := installer.beginMaintenance(ctx, journal.Recovery.Config, id, reason) + if err != nil { return fmt.Errorf("establish legacy recovery maintenance fence: %w", err) } - if err := installer.waitLocalState(ctx, journal.Recovery.Config, "unavailable"); err != nil { + if err := installer.waitLocalStateAfter(ctx, journal.Recovery.Config, "unavailable", maintenance.StartedAt); err != nil { return fmt.Errorf("drain legacy recovery maintenance fence: %w", err) } if err := writeLifecycleTransition(home, paths, journal, LifecycleFenced, "", installer.now()); err != nil { @@ -1303,15 +1437,16 @@ func replaceLifecycleAttestation(attestations []LifecycleFileAttestation, path s } func (installer Installer) recoverLifecycleFencing(ctx context.Context, home string, paths LifecyclePaths, journal *LifecycleJournal) error { - id, reason, err := lifecycleMaintenanceIdentity(journal.Operation) + id, reason, err := lifecycleMaintenanceIdentity(*journal) if err != nil { return err } config := journal.Recovery.Config - if err := installer.beginMaintenance(ctx, config, id, reason); err != nil { + maintenance, err := installer.beginMaintenance(ctx, config, id, reason) + if err != nil { return fmt.Errorf("establish lifecycle maintenance fence: %w", err) } - if err := installer.waitLocalState(ctx, config, "unavailable"); err != nil { + if err := installer.waitLocalStateAfter(ctx, config, "unavailable", maintenance.StartedAt); err != nil { return fmt.Errorf("drain lifecycle maintenance fence: %w", err) } journal.ProviderTransaction = nil @@ -1322,15 +1457,16 @@ func (installer Installer) recoverLifecycleFencing(ctx context.Context, home str } func (installer Installer) recoverLifecycleFenced(ctx context.Context, home string, paths LifecyclePaths, journal *LifecycleJournal, refresher Refresher) error { - id, reason, err := lifecycleMaintenanceIdentity(journal.Operation) + id, reason, err := lifecycleMaintenanceIdentity(*journal) if err != nil { return err } config := journal.Recovery.Config - if err := installer.beginMaintenance(ctx, config, id, reason); err != nil { + maintenance, err := installer.beginMaintenance(ctx, config, id, reason) + if err != nil { return fmt.Errorf("re-establish fenced lifecycle maintenance: %w", err) } - if err := installer.waitLocalState(ctx, config, "unavailable"); err != nil { + if err := installer.waitLocalStateAfter(ctx, config, "unavailable", maintenance.StartedAt); err != nil { return fmt.Errorf("re-drain fenced lifecycle maintenance: %w", err) } if err := installer.reattestLifecycleAuthority(ctx, home, *journal); err != nil { @@ -1342,16 +1478,16 @@ func (installer Installer) recoverLifecycleFenced(ctx context.Context, home stri if err := validateLifecycleWiringVector(*journal, paths, lifecycleWiringMixed); err != nil { return err } - if err := installer.systemctl(ctx, "stop", config.AgentUnit); err != nil { + if err := installer.systemctl(ctx, config.SystemctlPath, "stop", config.AgentUnit); err != nil { return fmt.Errorf("stop fenced lifecycle agent: %w", err) } - inner, found, err := readTransactionJournal(paths.Journal) + inner, found, err := readTransactionJournalForConfig(paths.Journal, config) if err != nil { return fmt.Errorf("read fenced provider transaction: %w", err) } if found { activeChanged := false - if active, activeFound, activeErr := readActiveState(paths.ActiveState); activeErr != nil { + if active, activeFound, activeErr := readActiveStateForConfig(paths.ActiveState, config); activeErr != nil { return fmt.Errorf("read fenced provider active state: %w", activeErr) } else if activeFound { activeChanged = active.Current.ImageID == inner.Candidate.ImageID && active.Current.ImageRef == inner.Candidate.ImageRef @@ -1370,8 +1506,25 @@ func (installer Installer) recoverLifecycleFenced(ctx context.Context, home stri } else if remains { return errors.New("fenced provider transaction remains after rollback") } + if journal.Operation == LifecycleRefresh && journal.ProviderEffect == ProviderUnchanged { + if journal.Unchanged == nil { + return errors.New("fenced unchanged refresh has no provider provenance") + } + if err := installer.systemctl(ctx, config.SystemctlPath, "restart", providerServiceUnit); err != nil { + return fmt.Errorf("restart provider during fenced TLS recovery: %w", err) + } + if err := refresher.probeStableActive(ctx, config, paths, journal.Unchanged.Active); err != nil { + return fmt.Errorf("probe provider during fenced TLS recovery: %w", err) + } + journal.Unchanged.StableProbeAt = installer.now() + journal.UpdatedAt = installer.now() + if err := writeLifecycleJournal(home, paths, *journal); err != nil { + return fmt.Errorf("record recovered provider TLS readiness: %w", err) + } + } + agentRestartedAfter := installer.now() if len(journal.Snapshots) > 0 || len(journal.PreviousUnits) > 0 || journal.Activation != (systemdActivation{}) { - if err := installer.rollbackInstallBeforeStart(ctx, config, journal.Snapshots, journal.PreviousUnits, true, false, id, journal.Activation, func(recoveryContext context.Context) error { + if err := installer.rollbackInstallBeforeStart(ctx, config, journal.Snapshots, journal.PreviousUnits, true, false, id, reason, journal.Activation, func(recoveryContext context.Context) error { return installer.reattestLifecycleAuthority(recoveryContext, home, *journal) }); err != nil { return fmt.Errorf("rollback fenced lifecycle wiring: %w", err) @@ -1379,19 +1532,15 @@ func (installer Installer) recoverLifecycleFenced(ctx context.Context, home stri if err := validateLifecycleWiringVector(*journal, paths, lifecycleWiringPre); err != nil { return fmt.Errorf("verify rolled back lifecycle wiring: %w", err) } - journal.Snapshots = nil - journal.WiringIntent = nil - journal.PreviousUnits = nil - journal.Activation = systemdActivation{} } else { if err := installer.reattestLifecycleAuthority(ctx, home, *journal); err != nil { return err } - if err := installer.systemctl(ctx, "start", config.AgentUnit); err != nil { + if err := installer.systemctl(ctx, config.SystemctlPath, "start", config.AgentUnit); err != nil { return fmt.Errorf("restart fenced lifecycle agent: %w", err) } } - if err := installer.waitLocalState(ctx, config, "unavailable"); err != nil { + if err := installer.waitLocalStateAfter(ctx, config, "unavailable", agentRestartedAfter); err != nil { return fmt.Errorf("observe restarted fenced lifecycle agent: %w", err) } journal.ProviderTransaction = nil @@ -1402,7 +1551,7 @@ func (installer Installer) recoverLifecycleFenced(ctx context.Context, home stri } func validateLifecycleProviderMatrix(paths LifecyclePaths, outer LifecycleJournal) error { - inner, found, err := readTransactionJournal(paths.Journal) + inner, found, err := readTransactionJournalForConfig(paths.Journal, outer.Recovery.Config) if err != nil { return fmt.Errorf("read lifecycle provider transaction: %w", err) } @@ -1440,6 +1589,14 @@ func validateLifecycleProviderMatrix(paths LifecyclePaths, outer LifecycleJourna return errors.New("lifecycle changed commit requires a provider transaction") } if outer.ProviderTransaction == nil { + if outer.Phase == LifecycleFenced && inner.DeferredCommit && + inner.OuterTransactionID == outer.TransactionID && inner.ProfileID == outer.Identity.ProfileID { + update := inner.Candidate.Update + if update.WorkerID != outer.Identity.WorkerID || update.PluginID != outer.Identity.PluginID || update.ComponentID != outer.Identity.ComponentID { + return errors.New("reciprocally bound provider transaction identity mismatch") + } + return nil + } return errors.New("lifecycle changed provider transaction binding is absent") } binding := outer.ProviderTransaction @@ -1469,7 +1626,7 @@ func validateLifecycleProviderMatrix(paths LifecyclePaths, outer LifecycleJourna } func (installer Installer) recoverLifecycleRelease(ctx context.Context, home string, paths LifecyclePaths, journal *LifecycleJournal, refresher Refresher) error { - id, reason, err := lifecycleMaintenanceIdentity(journal.Operation) + id, reason, err := lifecycleMaintenanceIdentity(*journal) if err != nil { return err } @@ -1497,7 +1654,7 @@ func (installer Installer) recoverLifecycleRelease(ctx context.Context, home str } switch classifyMaintenanceState(state, journal.Identity.ProfileID, id, reason) { case maintenanceExactActive: - if err := installer.waitLocalDrained(ctx, journal.Recovery.Config); err != nil { + if err := installer.waitLocalDrainedAfter(ctx, journal.Recovery.Config, state.Maintenance.StartedAt); err != nil { return fmt.Errorf("wait for lifecycle maintenance drain: %w", err) } if err := installer.releaseLifecycleMaintenance(ctx, home, *journal); err != nil { @@ -1527,7 +1684,7 @@ func finalizeLifecycleTransaction(home string, paths LifecyclePaths, journal *Li return errors.New("lifecycle transaction is not committed") } if journal.Outcome == LifecycleCommit && journal.ProviderEffect == ProviderChanged { - if _, found, err := readTransactionJournal(paths.Journal); err != nil { + if _, found, err := readTransactionJournalForConfig(paths.Journal, journal.Recovery.Config); err != nil { return fmt.Errorf("read committed provider transaction: %w", err) } else if found { if err := refresher.finalizeDeferredRefresh(journal.Recovery.Config); err != nil { diff --git a/internal/retainedprovider/lifecycle_test.go b/internal/retainedprovider/lifecycle_test.go index 330454f..b92657a 100644 --- a/internal/retainedprovider/lifecycle_test.go +++ b/internal/retainedprovider/lifecycle_test.go @@ -15,26 +15,26 @@ import ( func TestRecoverReadyLifecycleReleasesForwardWithoutRefencing(t *testing.T) { for _, tc := range []struct { name string - status func(Config) []byte + status func(Config, LifecycleJournal) []byte wantErr string wantEnd bool wantCleanup bool }{ { name: "exact active", - status: func(config Config) []byte { - return maintenanceStateJSON(true, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason) + status: func(config Config, journal LifecycleJournal) []byte { + return maintenanceStateJSON(true, journal.TransactionID, config.ProfileID, refreshMaintenanceReason) }, wantEnd: true, wantCleanup: true, }, { name: "already inactive", - status: func(Config) []byte { return []byte(`{"active":false,"durable":true}`) }, + status: func(Config, LifecycleJournal) []byte { return []byte(`{"active":false,"durable":true}`) }, wantCleanup: true, }, { name: "conflicting active", - status: func(config Config) []byte { + status: func(config Config, _ LifecycleJournal) []byte { return maintenanceStateJSON(true, "other-transaction", config.ProfileID, refreshMaintenanceReason) }, wantErr: "conflicting", @@ -65,11 +65,11 @@ func TestRecoverReadyLifecycleReleasesForwardWithoutRefencing(t *testing.T) { case "agent-signature": return agentUnitSystemdOutputForTest(t, config), nil case "maintenance-status": - return tc.status(config), nil + return tc.status(config, journal), nil case "local-status": return localStatusJSON(config.WorkerID, "unavailable"), nil case "maintenance-end": - return maintenanceStateJSON(false, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + return maintenanceStateJSON(false, journal.TransactionID, config.ProfileID, refreshMaintenanceReason), nil default: return nil, nil } @@ -106,6 +106,96 @@ func TestRecoverReadyLifecycleReleasesForwardWithoutRefencing(t *testing.T) { } } +func TestRecoverCommittedLifecycleAfterTransactionRootCleanup(t *testing.T) { + home, paths, journal := committedLifecycleCleanupJournalForTest(t) + transactionRoot := paths.LifecycleTransactionRoot(journal.TransactionID) + if err := os.RemoveAll(transactionRoot); err != nil { + t.Fatalf("simulate completed transaction-root cleanup: %v", err) + } + + persisted, found, err := readLifecycleJournal(home, paths) + if err != nil || !found { + t.Fatalf("read committed cleanup journal found=%v err=%v", found, err) + } + if err := finishLifecycleTransaction(home, paths, &persisted); err != nil { + t.Fatalf("finish committed cleanup recovery: %v", err) + } + if _, found, err := readLifecycleJournal(home, paths); err != nil || found { + t.Fatalf("terminal journal found=%v err=%v", found, err) + } +} + +func TestRecoverCommittedLifecycleAfterPartialTransactionRootCleanup(t *testing.T) { + home, paths, journal := committedLifecycleCleanupJournalForTest(t) + removed := "" + for _, snapshot := range journal.Snapshots { + if snapshot.Existed { + removed = snapshot.Backup + if err := os.Remove(snapshot.Backup); err != nil { + t.Fatalf("simulate partial transaction-root cleanup: %v", err) + } + break + } + } + if removed == "" { + t.Fatal("cleanup fixture has no existing snapshot backup") + } + if _, err := os.Stat(paths.LifecycleTransactionRoot(journal.TransactionID)); err != nil { + t.Fatalf("partial cleanup removed transaction root: %v", err) + } + + persisted, found, err := readLifecycleJournal(home, paths) + if err != nil || !found { + t.Fatalf("read partially cleaned committed journal found=%v err=%v", found, err) + } + if err := finishLifecycleTransaction(home, paths, &persisted); err != nil { + t.Fatalf("finish partial committed cleanup recovery: %v", err) + } +} + +func committedLifecycleCleanupJournalForTest(t *testing.T) (string, LifecyclePaths, LifecycleJournal) { + t.Helper() + home := t.TempDir() + t.Setenv("XDG_STATE_HOME", filepath.Join(home, ".state")) + config := validTestConfig(home) + writeLifecycleRecoveryFiles(t, config) + paths := LifecyclePathsFor(config) + if err := AtomicWriteJSON(paths.ConfigFile, config); err != nil { + t.Fatalf("write installed config: %v", err) + } + now := time.Unix(1_700_800_000, 0).UTC() + journal := lifecycleRecoveryJournalForTest(t, config, now) + if err := writeLifecycleJournal(home, paths, journal); err != nil { + t.Fatalf("write intent journal: %v", err) + } + journal.Phase = LifecycleFencing + if err := writeLifecycleJournal(home, paths, journal); err != nil { + t.Fatalf("write fencing journal: %v", err) + } + if err := snapshotManagedFilesForLifecycle(home, paths, &journal, now.Add(time.Second)); err != nil { + t.Fatalf("snapshot lifecycle files: %v", err) + } + units, err := RenderSystemdUnits(config, paths) + if err != nil { + t.Fatalf("render systemd units: %v", err) + } + journal.WiringIntent = managedWiringIntent(paths, units, true) + intended := journal.Recovery.AgentUnitBefore + journal.AgentUnitIntended = &intended + journal.ProviderTransaction = &LifecycleProviderTransaction{ + TransactionID: "provider-transaction-123", + ProfileID: config.ProfileID, + Digest: "sha256:" + strings.Repeat("d", 64), + } + journal.Phase = LifecycleCommitted + journal.Outcome = LifecycleCommit + journal.UpdatedAt = now.Add(2 * time.Second) + if err := writeLifecycleJournal(home, paths, journal); err != nil { + t.Fatalf("write committed journal: %v", err) + } + return home, paths, journal +} + func TestRecoverLifecycleReattestsJournalAuthorityBeforeCommands(t *testing.T) { home := t.TempDir() t.Setenv("XDG_STATE_HOME", filepath.Join(home, ".state")) @@ -156,7 +246,7 @@ func TestRecoverFencedLifecycleReattestsAfterDrainBeforeStop(t *testing.T) { case "agent-signature": return agentUnitSystemdOutputForTest(t, config), nil case "maintenance-begin": - return maintenanceStateJSON(true, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + return maintenanceStateJSON(true, adjacentArgValue(command.Args, "-id"), config.ProfileID, refreshMaintenanceReason), nil case "local-status": if err := os.WriteFile(config.ComputeAgentPath, []byte("replacement during drain"), 0o700); err != nil { t.Fatalf("replace compute-agent during drain: %v", err) @@ -179,6 +269,56 @@ func TestRecoverFencedLifecycleReattestsAfterDrainBeforeStop(t *testing.T) { } } +func TestRecoverFencedTLSRefreshRestartsAndProbesProviderBeforeAgentRelease(t *testing.T) { + home := t.TempDir() + t.Setenv("XDG_STATE_HOME", filepath.Join(home, ".state")) + config := validTestConfig(home) + writeLifecycleRecoveryFiles(t, config) + paths := LifecyclePathsFor(config) + now := time.Unix(1_700_800_000, 0).UTC() + journal := lifecycleRecoveryJournalForTest(t, config, now) + journal.Operation = LifecycleRefresh + journal.ProviderEffect = ProviderUnchanged + setLifecycleUnchangedForTest(&journal, config, now) + journal.Phase = LifecycleFenced + if err := writeLifecycleJournal(home, paths, journal); err != nil { + t.Fatalf("write fenced TLS refresh journal: %v", err) + } + + var events []string + runner := &recordingCommandRunner{run: func(_ context.Context, command Command) ([]byte, error) { + switch { + case filepath.Base(command.Path) == "systemctl" && containsAdjacentArgs(command.Args, "restart", providerServiceUnit): + events = append(events, "provider-restart") + return nil, nil + case filepath.Base(command.Path) == "systemctl" && containsArg(command.Args, providerServiceUnit) && containsAdjacentArgs(command.Args, "--property", "ActiveState"): + return []byte("active\n"), nil + case isProbeFor(command, config.StableContainer): + events = append(events, "provider-probe") + return nil, nil + } + switch installCommandEvent(command, config) { + case "agent-signature": + return agentUnitSystemdOutputForTest(t, config), nil + case "maintenance-begin", "maintenance-status": + return maintenanceStateJSON(true, journal.TransactionID, config.ProfileID, refreshMaintenanceReason), nil + case "local-status": + return localStatusJSON(config.WorkerID, "unavailable"), nil + case "agent-start": + events = append(events, "agent-start") + case "maintenance-end": + events = append(events, "maintenance-end") + return maintenanceStateJSON(false, journal.TransactionID, config.ProfileID, refreshMaintenanceReason), nil + } + return nil, nil + }} + installer := Installer{Runner: runner, Now: func() time.Time { return now.Add(time.Minute) }, Sleep: func(context.Context, time.Duration) error { return nil }} + if err := installer.recoverLifecycleTransaction(t.Context(), home, paths, Refresher{Runner: runner, Now: installer.Now, Sleep: installer.Sleep}); err != nil { + t.Fatalf("recover fenced TLS refresh: %v", err) + } + assertOrderedEvents(t, events, []string{"provider-restart", "provider-probe", "agent-start", "maintenance-end"}) +} + func TestRecoverLifecycleRejectsChangedEffectiveAgentUnitBeforeMutation(t *testing.T) { home := t.TempDir() t.Setenv("XDG_STATE_HOME", filepath.Join(home, ".state")) @@ -276,8 +416,8 @@ func TestRecoverLifecycleRejectsMismatchedProviderTransactionBeforeCommands(t *t } } -func TestRecoverFencedLifecycleRollsBackEveryBoundProviderPhase(t *testing.T) { - for _, phase := range []JournalPhase{JournalPrepared, JournalStatePromoting, JournalStatePromoted, JournalActivated, JournalCommitted} { +func TestRecoverFencedLifecycleRollsBackEveryReciprocallyBoundProviderPhase(t *testing.T) { + for _, phase := range []JournalPhase{JournalStaging, JournalPrepared, JournalStatePromoting, JournalStateDetached, JournalStatePromoted, JournalActivated, JournalCommitted} { t.Run(string(phase), func(t *testing.T) { home := t.TempDir() t.Setenv("XDG_STATE_HOME", filepath.Join(home, ".state")) @@ -296,17 +436,32 @@ func TestRecoverFencedLifecycleRollsBackEveryBoundProviderPhase(t *testing.T) { payload := writeTestProviderPayload(t, home, "outer-candidate-"+string(phase)) digest := fileDigestForTest(t, payload) candidate := selectionForDigest(payload, digest, "v1.0.32", "outer-directive-"+string(phase), "sha256:"+strings.Repeat("e", 64), now) - if err := prepareCandidateState(paths.ProviderState, paths.CandidateState(digest)); err != nil { - t.Fatalf("prepare candidate state: %v", err) - } - if err := os.WriteFile(filepath.Join(paths.CandidateState(digest), "generation"), []byte("candidate"), 0o600); err != nil { - t.Fatalf("write candidate provider state: %v", err) + journalCandidate := candidate + if phase == JournalStaging { + journalCandidate = ImageSelection{Update: candidate.Update} + if err := mkdirAllDurable(paths.PackageDir(digest), 0o700); err != nil { + t.Fatalf("create staged package: %v", err) + } + if err := os.WriteFile(paths.PackageBinary(digest), []byte("candidate"), 0o700); err != nil { + t.Fatalf("write staged package: %v", err) + } + } else { + if err := prepareCandidateState(paths.ProviderState, paths.CandidateState(digest)); err != nil { + t.Fatalf("prepare candidate state: %v", err) + } + if err := os.WriteFile(filepath.Join(paths.CandidateState(digest), "generation"), []byte("candidate"), 0o600); err != nil { + t.Fatalf("write candidate provider state: %v", err) + } } switch phase { case JournalStatePromoting: if err := os.Rename(paths.ProviderState, paths.PreviousState(digest)); err != nil { t.Fatalf("simulate state promoting: %v", err) } + case JournalStateDetached: + if err := detachProviderState(paths, digest); err != nil { + t.Fatalf("simulate detached provider state: %v", err) + } case JournalStatePromoted, JournalActivated, JournalCommitted: if err := promoteCandidateProviderState(paths, digest); err != nil { t.Fatalf("simulate promoted state: %v", err) @@ -326,9 +481,8 @@ func TestRecoverFencedLifecycleRollsBackEveryBoundProviderPhase(t *testing.T) { ProtocolVersion: TransactionJournalProtocolVersion, ID: "provider-transaction-" + string(phase), Phase: phase, DeferredCommit: true, OuterTransactionID: outer.TransactionID, ProfileID: config.ProfileID, - Previous: &previous, Candidate: candidate, StartedAt: now, UpdatedAt: now, + Previous: &previous, Candidate: journalCandidate, StartedAt: now, UpdatedAt: now, } - outer.ProviderTransaction = &LifecycleProviderTransaction{TransactionID: inner.ID, ProfileID: config.ProfileID, Digest: digest} if err := writeLifecycleJournal(home, paths, outer); err != nil { t.Fatalf("write outer intent: %v", err) } @@ -341,21 +495,23 @@ func TestRecoverFencedLifecycleRollsBackEveryBoundProviderPhase(t *testing.T) { } maintenanceActive := false + maintenanceID := "" runner := &recordingCommandRunner{run: func(_ context.Context, command Command) ([]byte, error) { switch installCommandEvent(command, config) { case "agent-signature": return agentUnitSystemdOutputForTest(t, config), nil case "maintenance-begin": maintenanceActive = true - return maintenanceStateJSON(true, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + maintenanceID = adjacentArgValue(command.Args, "-id") + return maintenanceStateJSON(true, maintenanceID, config.ProfileID, refreshMaintenanceReason), nil case "maintenance-status": if maintenanceActive { - return maintenanceStateJSON(true, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + return maintenanceStateJSON(true, maintenanceID, config.ProfileID, refreshMaintenanceReason), nil } return []byte(`{"active":false,"durable":true}`), nil case "maintenance-end": maintenanceActive = false - return maintenanceStateJSON(false, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + return maintenanceStateJSON(false, maintenanceID, config.ProfileID, refreshMaintenanceReason), nil case "local-status": return localStatusJSON(config.WorkerID, "unavailable"), nil default: @@ -411,21 +567,26 @@ func TestRecoverFencingAndFencedLifecycleRollsBackBeforeRelease(t *testing.T) { } maintenanceActive := false + maintenanceID := "" runner := &recordingCommandRunner{run: func(_ context.Context, command Command) ([]byte, error) { + if filepath.Base(command.Path) == "systemctl" && containsArg(command.Args, providerServiceUnit) && containsAdjacentArgs(command.Args, "--property", "ActiveState") { + return []byte("active\n"), nil + } switch installCommandEvent(command, config) { case "agent-signature": return agentUnitSystemdOutputForTest(t, config), nil case "maintenance-begin": maintenanceActive = true - return maintenanceStateJSON(true, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + maintenanceID = adjacentArgValue(command.Args, "-id") + return maintenanceStateJSON(true, maintenanceID, config.ProfileID, refreshMaintenanceReason), nil case "maintenance-status": if maintenanceActive { - return maintenanceStateJSON(true, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + return maintenanceStateJSON(true, maintenanceID, config.ProfileID, refreshMaintenanceReason), nil } return []byte(`{"active":false,"durable":true}`), nil case "maintenance-end": maintenanceActive = false - return maintenanceStateJSON(false, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + return maintenanceStateJSON(false, maintenanceID, config.ProfileID, refreshMaintenanceReason), nil case "local-status": return localStatusJSON(config.WorkerID, "unavailable"), nil default: @@ -449,6 +610,13 @@ func TestRecoverFencingAndFencedLifecycleRollsBackBeforeRelease(t *testing.T) { if tc.wantStop && !strings.Contains(transcript, "systemctl --user start "+config.AgentUnit) { t.Fatalf("fenced recovery did not restart agent:\n%s", transcript) } + if tc.wantStop { + for _, required := range []string{"systemctl --user restart " + providerServiceUnit, "probe -url " + config.ProviderURL} { + if !strings.Contains(transcript, required) { + t.Fatalf("fenced TLS recovery missing %q:\n%s", required, transcript) + } + } + } if _, found, err := readLifecycleJournal(home, paths); err != nil || found { t.Fatalf("recovered journal found=%v err=%v", found, err) } @@ -496,6 +664,7 @@ func TestRecoverLifecycleAdoptsLegacyInnerBeforeMaintenance(t *testing.T) { } maintenanceActive := false + maintenanceID := "" sawAdopting := false runner := &recordingCommandRunner{run: func(_ context.Context, command Command) ([]byte, error) { switch installCommandEvent(command, config) { @@ -514,15 +683,16 @@ func TestRecoverLifecycleAdoptsLegacyInnerBeforeMaintenance(t *testing.T) { sawAdopting = true } maintenanceActive = true - return maintenanceStateJSON(true, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + maintenanceID = adjacentArgValue(command.Args, "-id") + return maintenanceStateJSON(true, maintenanceID, config.ProfileID, refreshMaintenanceReason), nil case "maintenance-status": if maintenanceActive { - return maintenanceStateJSON(true, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + return maintenanceStateJSON(true, maintenanceID, config.ProfileID, refreshMaintenanceReason), nil } return []byte(`{"active":false,"durable":true}`), nil case "maintenance-end": maintenanceActive = false - return maintenanceStateJSON(false, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + return maintenanceStateJSON(false, maintenanceID, config.ProfileID, refreshMaintenanceReason), nil case "local-status": return localStatusJSON(config.WorkerID, "unavailable"), nil default: @@ -588,6 +758,9 @@ func writeLifecycleRecoveryFiles(t *testing.T, config Config) { }{ {path: config.ComputeAgentPath, mode: 0o700, data: "compute-agent fixture"}, {path: config.SupervisorConfigPath, mode: 0o600, data: "supervisor config fixture"}, + {path: config.PodmanPath, mode: 0o500, data: "podman fixture"}, + {path: config.SystemctlPath, mode: 0o500, data: "systemctl fixture"}, + {path: config.LoginctlPath, mode: 0o500, data: "loginctl fixture"}, {path: agentUnitFragmentPathForTest(config), mode: 0o600, data: "[Service]\nExecStart=" + config.ComputeAgentPath + " run\n"}, } { if err := os.MkdirAll(filepath.Dir(file.path), 0o700); err != nil { @@ -604,6 +777,43 @@ func writeLifecycleRecoveryFiles(t *testing.T, config Config) { } } +func TestLifecycleRecoveryAttestsConfiguredHostExecutables(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + writeLifecycleRecoveryFiles(t, config) + journal, err := newLifecycleJournal(config, LifecycleRefresh, ProviderChanged, nil, time.Now().UTC()) + if err != nil { + t.Fatalf("create lifecycle journal: %v", err) + } + for label, attestation := range map[string]LifecycleFileAttestation{ + "podman": journal.Recovery.Podman, + "systemctl": journal.Recovery.Systemctl, + "loginctl": journal.Recovery.Loginctl, + } { + var want string + switch label { + case "podman": + want = config.PodmanPath + case "systemctl": + want = config.SystemctlPath + case "loginctl": + want = config.LoginctlPath + } + if attestation.Path != want || attestation.SHA256 == "" { + t.Fatalf("%s attestation = %+v want path %q", label, attestation, want) + } + } + if err := os.Chmod(config.PodmanPath, 0o700); err != nil { + t.Fatalf("make podman fixture replaceable: %v", err) + } + if err := os.WriteFile(config.PodmanPath, []byte("replaced podman"), 0o500); err != nil { + t.Fatalf("replace podman fixture: %v", err) + } + if err := journal.Recovery.Reattest(); err == nil || !strings.Contains(err.Error(), "podman") { + t.Fatalf("re-attest replaced podman err = %v", err) + } +} + func lifecycleRecoveryJournalForTest(t *testing.T, config Config, now time.Time) LifecycleJournal { t.Helper() journal := validLifecycleJournalForTest(config, now) @@ -617,6 +827,17 @@ func lifecycleRecoveryJournalForTest(t *testing.T, config Config, now time.Time) } journal.Recovery.ComputeAgent.SHA256 = computeAgentDigest journal.Recovery.SupervisorConfig.SHA256 = supervisorDigest + for label, target := range map[string]*LifecycleFileAttestation{ + "podman": &journal.Recovery.Podman, + "systemctl": &journal.Recovery.Systemctl, + "loginctl": &journal.Recovery.Loginctl, + } { + digest, err := hashHostExecutable(target.Path) + if err != nil { + t.Fatalf("hash %s: %v", label, err) + } + target.SHA256 = digest + } journal.Recovery.AgentUnitBefore = agentUnitSignatureForTest(t, config) return journal } @@ -727,6 +948,44 @@ func TestLifecycleAuditDrainRecoversCompleteAndTornAppend(t *testing.T) { } } +func TestLifecycleAuditDrainRejectsFileShorterThanDurableOffset(t *testing.T) { + home := t.TempDir() + t.Setenv("XDG_STATE_HOME", filepath.Join(home, ".state")) + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + now := time.Unix(1_700_800_000, 0).UTC() + journal := validLifecycleJournalForTest(config, now) + event := LifecycleAuditEvent{ + EventID: "event-1", Timestamp: now, TransactionID: journal.TransactionID, + WorkerID: config.WorkerID, Operation: LifecycleInstall, + Phase: LifecycleIntent, Kind: AuditPhase, ProviderEffect: ProviderChanged, + } + if err := journal.Audit.EnqueueSafety(event); err != nil { + t.Fatalf("enqueue safety event: %v", err) + } + payload, err := lifecycleAuditPayload(journal.Audit.Safety[0]) + if err != nil { + t.Fatalf("audit payload: %v", err) + } + fixture := []byte("truncated\n") + writeAuditFixture(t, paths.LifecycleAudit, fixture) + offset := int64(len(fixture) + 32) + journal.Audit.Safety[0].Offset = &offset + journal.Audit.Safety[0].Digest = digestBytes(payload) + if err := writeLifecycleJournal(home, paths, journal); err != nil { + t.Fatalf("write lifecycle journal: %v", err) + } + + err = drainLifecycleAudit(home, paths, &journal) + if err == nil || !strings.Contains(err.Error(), "shorter than pending offset") { + t.Fatalf("truncated audit drain err = %v", err) + } + data, readErr := os.ReadFile(paths.LifecycleAudit) + if readErr != nil || !bytes.Equal(data, fixture) { + t.Fatalf("truncated audit mutated = %q err=%v", data, readErr) + } +} + func writeAuditFixture(t *testing.T, path string, data []byte) { t.Helper() if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { @@ -908,6 +1167,36 @@ func TestLifecycleAuditEventStrictUnion(t *testing.T) { if err := errorEvent.Validate(); err != nil { t.Fatalf("valid error event: %v", err) } + + recoveryEvent := event + recoveryEvent.Kind = AuditRecovery + recoveryEvent.Outcome = "" + recoveryEvent.ProviderEffect = "" + recoveryEvent.Disposition = "resume_fenced" + recoveryEvent.Count = 1 + recoveryEvent.FirstSeen = now + recoveryEvent.LastSeen = now + if err := recoveryEvent.Validate(); err != nil { + t.Fatalf("valid recovery event: %v", err) + } + for _, tc := range []struct { + name string + base LifecycleAuditEvent + mutate func(*LifecycleAuditEvent) + }{ + {name: "phase invalid outcome", base: event, mutate: func(candidate *LifecycleAuditEvent) { candidate.Outcome = "contradictory" }}, + {name: "phase invalid provider effect", base: event, mutate: func(candidate *LifecycleAuditEvent) { candidate.ProviderEffect = "contradictory" }}, + {name: "recovery provider effect", base: recoveryEvent, mutate: func(candidate *LifecycleAuditEvent) { candidate.ProviderEffect = ProviderChanged }}, + {name: "error disposition", base: errorEvent, mutate: func(candidate *LifecycleAuditEvent) { candidate.Disposition = "resume_fenced" }}, + } { + t.Run(tc.name, func(t *testing.T) { + candidate := tc.base + tc.mutate(&candidate) + if err := candidate.Validate(); err == nil { + t.Fatalf("contradictory audit event accepted: %+v", candidate) + } + }) + } } func TestLifecycleAuditDiagnosticsCoalesceAndOverflow(t *testing.T) { @@ -957,7 +1246,9 @@ func TestLifecycleAuditDiagnosticsCoalesceAndOverflow(t *testing.T) { } overflow := base overflow.EventID = "overflow-source-1" - overflow.ErrorClass = "beyond-capacity" + overflow.Kind = AuditRecovery + overflow.ErrorClass = "" + overflow.Disposition = "resume_fenced" overflow.Timestamp = now.Add(40 * time.Minute) if err := queue.EnqueueDiagnostic(overflow); err != nil { t.Fatalf("enqueue overflow diagnostic: %v", err) @@ -1346,6 +1637,9 @@ func validLifecycleJournalForTest(config Config, now time.Time) LifecycleJournal Config: config, ComputeAgent: LifecycleFileAttestation{Path: config.ComputeAgentPath, SHA256: "sha256:" + strings.Repeat("a", 64)}, SupervisorConfig: LifecycleFileAttestation{Path: config.SupervisorConfigPath, SHA256: "sha256:" + strings.Repeat("b", 64)}, + Podman: LifecycleFileAttestation{Path: config.PodmanPath, SHA256: "sha256:" + strings.Repeat("d", 64)}, + Systemctl: LifecycleFileAttestation{Path: config.SystemctlPath, SHA256: "sha256:" + strings.Repeat("e", 64)}, + Loginctl: LifecycleFileAttestation{Path: config.LoginctlPath, SHA256: "sha256:" + strings.Repeat("f", 64)}, AgentUnitBefore: LifecycleSystemdSignature{ Fragment: LifecycleFileAttestation{Path: agentUnitFragmentPathForTest(config), SHA256: "sha256:" + strings.Repeat("c", 64)}, ExecStart: staticExecStartForTest(config), diff --git a/internal/retainedprovider/ownership_other.go b/internal/retainedprovider/ownership_other.go index b5d6871..c6a768d 100644 --- a/internal/retainedprovider/ownership_other.go +++ b/internal/retainedprovider/ownership_other.go @@ -2,6 +2,20 @@ package retainedprovider -import "os" +import ( + "fmt" + "os" +) func validateOwner(os.FileInfo) error { return nil } + +func validateManagedPathAuthority(info os.FileInfo) error { + return validateOwner(info) +} + +func validateExecutableAuthority(info os.FileInfo) error { + if info.Mode().Perm()&0o022 != 0 { + return fmt.Errorf("executable must not be group- or world-writable") + } + return nil +} diff --git a/internal/retainedprovider/ownership_unix.go b/internal/retainedprovider/ownership_unix.go index 07ba708..dd2325c 100644 --- a/internal/retainedprovider/ownership_unix.go +++ b/internal/retainedprovider/ownership_unix.go @@ -18,3 +18,27 @@ func validateOwner(info os.FileInfo) error { } return nil } + +func validateManagedPathAuthority(info os.FileInfo) error { + if err := validateOwner(info); err != nil { + return err + } + if info.Mode().Perm()&0o022 != 0 { + return fmt.Errorf("managed path must not be group- or world-writable") + } + return nil +} + +func validateExecutableAuthority(info os.FileInfo) error { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return fmt.Errorf("ownership metadata is unavailable") + } + if stat.Uid != 0 && stat.Uid != uint32(os.Geteuid()) { + return fmt.Errorf("executable owner uid %d is not trusted", stat.Uid) + } + if info.Mode().Perm()&0o022 != 0 { + return fmt.Errorf("executable must not be group- or world-writable") + } + return nil +} diff --git a/internal/retainedprovider/refresh.go b/internal/retainedprovider/refresh.go index 90840b9..bb2d8b7 100644 --- a/internal/retainedprovider/refresh.go +++ b/internal/retainedprovider/refresh.go @@ -12,6 +12,7 @@ import ( "io" "os" "path/filepath" + "regexp" "strings" "time" ) @@ -25,10 +26,22 @@ const ( providerTLSKeyPath = "/tls/server.key" providerListenAddr = "0.0.0.0:18090" maxProviderPackageBytes = 512 << 20 + managedObjectLabel = "io.workflow.compute.managed" + managedWorkerLabel = "io.workflow.compute.worker" + managedRoleLabel = "io.workflow.compute.role" + managedDigestLabel = "io.workflow.compute.digest" + managedProviderValue = "github-runner-provider" + candidateContainerRole = "candidate" + stableContainerRole = "stable" + probeContainerRole = "probe" + providerImageRole = "provider-image" ) var providerContainerfile = []byte("FROM scratch\nCOPY --chmod=0555 github-runner-provider /github-runner-provider\nENTRYPOINT [\"/github-runner-provider\"]\n") +var errProviderImageOwnershipDrift = errors.New("provider image ownership drift") +var errProviderMutationRequired = errors.New("provider mutation requires a fenced lifecycle") + type LifecyclePaths struct { Root string ConfigFile string @@ -47,6 +60,7 @@ type LifecyclePaths struct { ProviderEnv string ProbeEnv string AgentEnv string + CAKey string TLSRoot string CAFile string ServerCert string @@ -63,10 +77,7 @@ func LifecyclePathsFor(config Config) LifecyclePaths { root := config.InstallRoot workspaceRoot := filepath.Dir(root) home := filepath.Dir(workspaceRoot) - stateHome := os.Getenv("XDG_STATE_HOME") - if stateHome == "" { - stateHome = filepath.Join(home, ".local", "state") - } + stateHome := filepath.Join(home, ".local", "state") audit := filepath.Join(stateHome, "wfctl", "plugins", GitHubPluginID, "retained-provider-audit.jsonl") return LifecyclePaths{ Root: root, @@ -86,6 +97,7 @@ func LifecyclePathsFor(config Config) LifecyclePaths { ProviderEnv: filepath.Join(root, "secrets", "provider.env"), ProbeEnv: filepath.Join(root, "secrets", "probe.env"), AgentEnv: filepath.Join(root, "secrets", "agent.env"), + CAKey: filepath.Join(root, "secrets", "ca.key"), TLSRoot: filepath.Join(root, "tls"), CAFile: filepath.Join(root, "tls", "ca.pem"), ServerCert: filepath.Join(root, "tls", "server.crt"), @@ -120,10 +132,12 @@ func (paths LifecyclePaths) PackageBinary(digest string) string { } type Refresher struct { - Runner CommandRunner - ExecutablePath func() (string, error) - Now func() time.Time - Sleep func(context.Context, time.Duration) error + Runner CommandRunner + ExecutablePath func() (string, error) + Random io.Reader + Now func() time.Time + Sleep func(context.Context, time.Duration) error + writeJournalPhaseFn func(string, *TransactionJournal, JournalPhase, time.Time) error } func (refresher Refresher) Refresh(ctx context.Context, config Config) (status Status, returnErr error) { @@ -147,6 +161,9 @@ func (refresher Refresher) Refresh(ctx context.Context, config Config) (status S if err := installer.recoverLifecycleTransaction(ctx, home, paths, refresher); err != nil { return Status{}, err } + if err := validateInstalledConfigBinding(home, config, paths); err != nil { + return Status{}, err + } if err := installer.recoverInstallTransaction(ctx, config, paths, refresher); err != nil { return Status{}, err } @@ -165,7 +182,7 @@ func (refresher Refresher) refreshUnchangedUnderLifecycleLock(ctx context.Contex if err != nil { return Status{}, err } - active, found, err := readActiveState(paths.ActiveState) + active, found, err := readActiveStateForConfig(paths.ActiveState, config) if err != nil { return Status{}, err } @@ -186,9 +203,13 @@ func (refresher Refresher) refreshUnchangedUnderLifecycleLock(ctx context.Contex if err := startLifecycleTransaction(home, paths, &transaction); err != nil { return Status{}, err } - status, err := refresher.refreshUnderLifecycleTransaction(ctx, config, true, false, "", "", update.SHA256) + status, err := refresher.refreshUnderLifecycleTransaction(ctx, config, true, false, "", "", update.SHA256, ProviderUnchanged) if err != nil { - return Status{}, errors.Join(err, finishLifecycleTransaction(home, paths, &transaction)) + finishErr := finishLifecycleTransaction(home, paths, &transaction) + if errors.Is(err, errProviderMutationRequired) && finishErr == nil { + return refresher.refreshFencedUnderLifecycleLock(ctx, home, config, paths) + } + return Status{}, errors.Join(err, finishErr) } transaction.Unchanged.StableProbeAt = refresher.now() if err := writeLifecycleTransition(home, paths, &transaction, LifecycleReady, LifecycleCommit, refresher.now()); err != nil { @@ -212,7 +233,7 @@ func (refresher Refresher) requiresMutation(ctx context.Context, config Config, if err != nil { return false, err } - active, found, err := readActiveState(paths.ActiveState) + active, found, err := readActiveStateForConfig(paths.ActiveState, config) if err != nil { return false, err } @@ -220,8 +241,17 @@ func (refresher Refresher) requiresMutation(ctx context.Context, config Config, if err := refresher.validateInitialInstaller(update); err != nil { return false, err } + return true, nil + } + runtimeRepair, err := refresher.activeProviderImageNeedsRepair(ctx, config, active.Current) + if err != nil { + return false, err + } + renewTLS, err := providerServerCertificateNeedsRenewal(config, paths, refresher.now()) + if err != nil { + return false, err } - return !found || active.Current.Update.SHA256 != update.SHA256, nil + return active.Current.Update.SHA256 != update.SHA256 || runtimeRepair || renewTLS, nil } func (refresher Refresher) validateInitialInstaller(update VerifiedUpdate) error { @@ -245,10 +275,45 @@ func (refresher Refresher) validateInitialInstaller(update VerifiedUpdate) error func (refresher Refresher) refreshFencedUnderLifecycleLock(ctx context.Context, home string, config Config, paths LifecyclePaths) (Status, error) { installer := Installer{Runner: refresher.Runner, Now: refresher.Now, Sleep: refresher.Sleep} - transaction, err := newLifecycleJournal(config, LifecycleRefresh, ProviderChanged, nil, refresher.now()) + update, err := VerifyCurrentUpdate(ctx, config, refresher.Runner) + if err != nil { + return Status{}, err + } + active, activeFound, err := readActiveStateForConfig(paths.ActiveState, config) + if err != nil { + return Status{}, err + } + if !activeFound { + if err := refresher.validateInitialInstaller(update); err != nil { + return Status{}, err + } + } + packageChanged := !activeFound || active.Current.Update.SHA256 != update.SHA256 + runtimeRepair := false + if activeFound && !packageChanged { + runtimeRepair, err = refresher.activeProviderImageNeedsRepair(ctx, config, active.Current) + if err != nil { + return Status{}, err + } + } + tlsRenewal := false + if activeFound { + tlsRenewal, err = providerServerCertificateNeedsRenewal(config, paths, refresher.now()) + if err != nil { + return Status{}, err + } + } + effect := ProviderUnchanged + if packageChanged || runtimeRepair { + effect = ProviderChanged + } + transaction, err := newLifecycleJournal(config, LifecycleRefresh, effect, nil, refresher.now()) if err != nil { return Status{}, err } + if !packageChanged && !runtimeRepair { + transaction.Unchanged = &LifecycleUnchangedProvenance{Active: active.Current, Candidate: update} + } beforeSignature, err := installer.inspectAgentUnitSignature(ctx, home, config) if err != nil { return Status{}, err @@ -261,14 +326,16 @@ func (refresher Refresher) refreshFencedUnderLifecycleLock(ctx context.Context, return Status{}, err } fail := func(cause error) (Status, error) { - rollbackContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + rollbackContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), lifecycleRecoveryTimeout) defer cancel() - return Status{}, errors.Join(cause, installer.recoverLifecycleTransaction(rollbackContext, home, paths, refresher)) + auditErr := writeLifecycleDiagnostic(home, paths, &transaction, AuditError, "operation_failed", refresher.now()) + return Status{}, errors.Join(cause, auditErr, installer.recoverLifecycleTransaction(rollbackContext, home, paths, refresher)) } - if err := installer.beginMaintenance(ctx, config, refreshMaintenanceID, refreshMaintenanceReason); err != nil { + maintenance, err := installer.beginMaintenance(ctx, config, transaction.TransactionID, refreshMaintenanceReason) + if err != nil { return fail(err) } - if err := installer.waitLocalState(ctx, config, "unavailable"); err != nil { + if err := installer.waitLocalStateAfter(ctx, config, "unavailable", maintenance.StartedAt); err != nil { return Status{}, fmt.Errorf("wait for retained agent refresh fence: %w", err) } if err := installer.reattestLifecycleAuthority(ctx, home, transaction); err != nil { @@ -277,19 +344,43 @@ func (refresher Refresher) refreshFencedUnderLifecycleLock(ctx context.Context, if err := writeLifecycleTransition(home, paths, &transaction, LifecycleFenced, "", refresher.now()); err != nil { return fail(err) } - if err := installer.systemctl(ctx, "stop", config.AgentUnit); err != nil { + if err := installer.systemctl(ctx, config.SystemctlPath, "stop", config.AgentUnit); err != nil { return fail(fmt.Errorf("stop retained agent for provider refresh: %w", err)) } - status, err := refresher.refreshUnderLifecycleTransaction(ctx, config, true, true, transaction.TransactionID, config.ProfileID, "") - if err != nil { - return fail(err) - } - inner, found, err := readTransactionJournal(paths.Journal) - if err != nil || !found || inner.Phase != JournalCommitted { - return fail(errors.Join(errors.New("provider refresh did not leave a deferred committed transaction"), err)) + if tlsRenewal { + renewed, err := renewProviderServerCertificate(config, paths, refresher.Random, refresher.now()) + if err != nil { + return fail(err) + } + if !renewed { + return fail(errors.New("provider TLS renewal was required but did not occur")) + } } - transaction.ProviderTransaction = &LifecycleProviderTransaction{ - TransactionID: inner.ID, ProfileID: config.ProfileID, Digest: inner.Candidate.Update.SHA256, + var status Status + if packageChanged || runtimeRepair { + status, err = refresher.refreshUnderLifecycleTransaction(ctx, config, true, true, transaction.TransactionID, config.ProfileID, "", ProviderChanged) + if err != nil { + return fail(err) + } + inner, found, err := readTransactionJournalForConfig(paths.Journal, config) + if err != nil || !found || inner.Phase != JournalCommitted { + return fail(errors.Join(errors.New("provider refresh did not leave a deferred committed transaction"), err)) + } + transaction.ProviderTransaction = &LifecycleProviderTransaction{ + TransactionID: inner.ID, ProfileID: config.ProfileID, Digest: inner.Candidate.Update.SHA256, + } + } else { + if !tlsRenewal { + return fail(errors.New("fenced refresh has no package or TLS mutation")) + } + if err := installer.systemctl(ctx, config.SystemctlPath, "restart", providerServiceUnit); err != nil { + return fail(fmt.Errorf("restart provider after TLS renewal: %w", err)) + } + if err := refresher.probeStableActive(ctx, config, paths, active.Current); err != nil { + return fail(fmt.Errorf("probe provider after TLS renewal: %w", err)) + } + transaction.Unchanged.StableProbeAt = refresher.now() + status = statusForActive(active, true, refresher.now()) } transaction.UpdatedAt = refresher.now() if err := writeLifecycleJournal(home, paths, transaction); err != nil { @@ -298,10 +389,11 @@ func (refresher Refresher) refreshFencedUnderLifecycleLock(ctx context.Context, if err := installer.reattestLifecycleAuthority(ctx, home, transaction); err != nil { return fail(err) } - if err := installer.systemctl(ctx, "start", config.AgentUnit); err != nil { + agentRestartedAfter := refresher.now() + if err := installer.systemctl(ctx, config.SystemctlPath, "start", config.AgentUnit); err != nil { return fail(fmt.Errorf("restart retained agent after provider refresh: %w", err)) } - if err := installer.waitLocalState(ctx, config, "unavailable"); err != nil { + if err := installer.waitLocalStateAfter(ctx, config, "unavailable", agentRestartedAfter); err != nil { return fail(fmt.Errorf("verify retained agent remains refresh-fenced: %w", err)) } if err := installer.reattestLifecycleAuthority(ctx, home, transaction); err != nil { @@ -314,6 +406,7 @@ func (refresher Refresher) refreshFencedUnderLifecycleLock(ctx context.Context, return fail(err) } _ = drainLifecycleAudit(home, paths, &transaction) + maintenanceReleasedAfter := refresher.now() if err := installer.releaseLifecycleMaintenance(ctx, home, transaction); err != nil { return Status{}, fmt.Errorf("release retained agent refresh fence: %w", err) } @@ -323,17 +416,17 @@ func (refresher Refresher) refreshFencedUnderLifecycleLock(ctx context.Context, if err := finalizeLifecycleTransaction(home, paths, &transaction, refresher); err != nil { return Status{}, err } - if err := installer.waitLocalState(ctx, config, "idle"); err != nil { + if err := installer.waitLocalStateAfter(ctx, config, "idle", maintenanceReleasedAfter); err != nil { return Status{}, fmt.Errorf("wait for retained agent after provider refresh: %w", err) } return status, nil } func (refresher Refresher) refreshUnderLifecycleLock(ctx context.Context, config Config, verifyCurrent, deferCommit bool) (Status, error) { - return refresher.refreshUnderLifecycleTransaction(ctx, config, verifyCurrent, deferCommit, "", "", "") + return refresher.refreshUnderLifecycleTransaction(ctx, config, verifyCurrent, deferCommit, "", "", "", "") } -func (refresher Refresher) refreshUnderLifecycleTransaction(ctx context.Context, config Config, verifyCurrent, deferCommit bool, outerTransactionID, profileID, expectedDigest string) (Status, error) { +func (refresher Refresher) refreshUnderLifecycleTransaction(ctx context.Context, config Config, verifyCurrent, deferCommit bool, outerTransactionID, profileID, expectedDigest string, expectedEffect ProviderEffect) (Status, error) { if refresher.Runner == nil { return Status{}, errors.New("command runner is required") } @@ -345,10 +438,7 @@ func (refresher Refresher) refreshUnderLifecycleTransaction(ctx context.Context, if err != nil { return Status{}, err } - if expectedDigest != "" && update.SHA256 != expectedDigest { - return Status{}, errors.New("verified provider update changed during unchanged refresh") - } - active, activeFound, err := readActiveState(paths.ActiveState) + active, activeFound, err := readActiveStateForConfig(paths.ActiveState, config) if err != nil { return Status{}, err } @@ -380,7 +470,24 @@ func (refresher Refresher) refreshUnderLifecycleTransaction(ctx context.Context, if err := refresher.validateProviderNetwork(ctx, config); err != nil { return Status{}, err } + runtimeRepair := false if activeFound && active.Current.Update.SHA256 == update.SHA256 { + runtimeRepair, err = refresher.activeProviderImageNeedsRepair(ctx, config, active.Current) + if err != nil { + return Status{}, err + } + } + actualEffect := ProviderChanged + if activeFound && active.Current.Update.SHA256 == update.SHA256 && !runtimeRepair { + actualEffect = ProviderUnchanged + } + if expectedDigest != "" && update.SHA256 != expectedDigest { + return Status{}, errProviderMutationRequired + } + if expectedEffect != "" && actualEffect != expectedEffect { + return Status{}, errProviderMutationRequired + } + if activeFound && active.Current.Update.SHA256 == update.SHA256 && !runtimeRepair { if verifyCurrent { if err := refresher.probeStableActive(ctx, config, paths, active.Current); err != nil { return Status{}, err @@ -391,62 +498,74 @@ func (refresher Refresher) refreshUnderLifecycleTransaction(ctx context.Context, if err := ValidateUserPath(paths.Root, paths.PackageDir(update.SHA256), false); err != nil { return Status{}, fmt.Errorf("provider package path: %w", err) } + journal := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, + ID: "refresh-" + digestHex(update.SHA256)[:16], + Phase: JournalStaging, + DeferredCommit: deferCommit, + RuntimeRepair: runtimeRepair, + OuterTransactionID: outerTransactionID, + ProfileID: profileID, + Candidate: ImageSelection{Update: update}, + StartedAt: now, + UpdatedAt: now, + } + if activeFound { + previous := active + journal.Previous = &previous + } + if err := AtomicWriteJSON(paths.Journal, journal); err != nil { + return Status{}, fmt.Errorf("write staging refresh journal: %w", err) + } + activeChanged := false + rollback := func(cause error) error { + return errors.Join(cause, refresher.rollback(ctx, config, paths, journal, activeChanged)) + } if err := stageVerifiedProvider(update, paths); err != nil { - return Status{}, err + return Status{}, rollback(err) } imageRef := providerImageRef(update.SHA256) if _, err := refresher.run(ctx, Command{ - Path: config.PodmanPath, - Args: []string{"build", "--file", "-", "--tag", imageRef, paths.PackageDir(update.SHA256)}, + Path: config.PodmanPath, + Args: []string{ + "build", "--file", "-", "--tag", imageRef, + "--label", managedObjectLabel + "=" + managedProviderValue, + "--label", managedWorkerLabel + "=" + config.WorkerID, + "--label", managedRoleLabel + "=" + providerImageRole, + "--label", managedDigestLabel + "=" + update.SHA256, + paths.PackageDir(update.SHA256), + }, Stdin: providerContainerfile, }); err != nil { - return Status{}, fmt.Errorf("build provider candidate image: %w", err) + return Status{}, rollback(fmt.Errorf("build provider candidate image: %w", err)) } imageOutput, err := refresher.run(ctx, Command{ Path: config.PodmanPath, Args: []string{"image", "inspect", "--format", "{{.Id}}", imageRef}, }) if err != nil { - return Status{}, fmt.Errorf("inspect provider candidate image: %w", err) + return Status{}, rollback(fmt.Errorf("inspect provider candidate image: %w", err)) } imageID, err := normalizePodmanImageID(string(imageOutput)) if err != nil { - return Status{}, fmt.Errorf("validate provider candidate image id: %w", err) + return Status{}, rollback(fmt.Errorf("validate provider candidate image id: %w", err)) } selection := ImageSelection{Update: update, ImageID: imageID, ImageRef: imageRef, ActivatedAt: now} if err := selection.Validate(); err != nil { - return Status{}, fmt.Errorf("validate provider candidate image: %w", err) + return Status{}, rollback(fmt.Errorf("validate provider candidate image: %w", err)) + } + journal.Candidate = selection + if err := refresher.writeJournal(paths.Journal, &journal, JournalPrepared, refresher.now()); err != nil { + return Status{}, rollback(fmt.Errorf("write prepared refresh journal: %w", err)) } candidateState := paths.CandidateState(update.SHA256) if err := ValidateUserPath(paths.Root, candidateState, false); err != nil { - return Status{}, fmt.Errorf("provider candidate state path: %w", err) + return Status{}, rollback(fmt.Errorf("provider candidate state path: %w", err)) } - journal := TransactionJournal{ - ProtocolVersion: TransactionJournalProtocolVersion, - ID: "refresh-" + digestHex(update.SHA256)[:16], - Phase: JournalPrepared, - DeferredCommit: deferCommit, - OuterTransactionID: outerTransactionID, - ProfileID: profileID, - Candidate: selection, - StartedAt: now, - UpdatedAt: now, - } - if activeFound { - previous := active - journal.Previous = &previous - } - if err := AtomicWriteJSON(paths.Journal, journal); err != nil { - return Status{}, fmt.Errorf("write prepared refresh journal: %w", err) - } - activeChanged := false - rollback := func(cause error) error { - return errors.Join(cause, refresher.rollback(ctx, config, paths, journal, activeChanged)) - } - if err := refresher.removeContainer(ctx, config, config.CandidateContainer); err != nil { + if err := refresher.removeManagedContainer(ctx, config, config.CandidateContainer, candidateContainerRole); err != nil { return Status{}, rollback(fmt.Errorf("remove stale provider candidate: %w", err)) } - if _, err := refresher.run(ctx, Command{Path: "/usr/bin/systemctl", Args: []string{"--user", "stop", providerServiceUnit}}); err != nil { + if _, err := refresher.run(ctx, Command{Path: config.SystemctlPath, Args: []string{"--user", "stop", providerServiceUnit}}); err != nil { return Status{}, rollback(fmt.Errorf("quiesce active provider before state clone: %w", err)) } if err := prepareCandidateState(paths.ProviderState, candidateState); err != nil { @@ -455,23 +574,32 @@ func (refresher Refresher) refreshUnderLifecycleTransaction(ctx context.Context, if _, err := refresher.run(ctx, candidateProviderCommand(config, paths, candidateState, selection)); err != nil { return Status{}, rollback(fmt.Errorf("start provider candidate: %w", err)) } - if err := refresher.runProbe(ctx, providerProbeCommand(config, paths, config.CandidateContainer, selection)); err != nil { + if err := refresher.runManagedProbe(ctx, config, config.CandidateContainer, providerProbeCommand(config, paths, config.CandidateContainer, selection)); err != nil { return Status{}, rollback(fmt.Errorf("probe provider candidate: %w", err)) } - if err := writeJournalPhase(paths.Journal, &journal, JournalStatePromoting, refresher.now()); err != nil { + if err := refresher.writeJournal(paths.Journal, &journal, JournalStatePromoting, refresher.now()); err != nil { return Status{}, rollback(fmt.Errorf("write state-promoting refresh journal: %w", err)) } - if err := refresher.removeContainer(ctx, config, config.CandidateContainer); err != nil { + if err := refresher.removeManagedContainer(ctx, config, config.CandidateContainer, candidateContainerRole); err != nil { return Status{}, rollback(fmt.Errorf("stop probed provider candidate: %w", err)) } - if err := promoteCandidateProviderState(paths, update.SHA256); err != nil { - return Status{}, rollback(fmt.Errorf("promote provider candidate state: %w", err)) + if err := detachProviderState(paths, update.SHA256); err != nil { + return Status{}, rollback(fmt.Errorf("detach active provider state: %w", err)) + } + if err := refresher.writeJournal(paths.Journal, &journal, JournalStateDetached, refresher.now()); err != nil { + return Status{}, rollback(fmt.Errorf("write state-detached refresh journal: %w", err)) } - if err := writeJournalPhase(paths.Journal, &journal, JournalStatePromoted, refresher.now()); err != nil { + if err := activateCandidateProviderState(paths, update.SHA256); err != nil { + return Status{}, rollback(fmt.Errorf("activate provider candidate state: %w", err)) + } + if err := refresher.writeJournal(paths.Journal, &journal, JournalStatePromoted, refresher.now()); err != nil { return Status{}, rollback(fmt.Errorf("write state-promoted refresh journal: %w", err)) } newActive := ActiveState{ProtocolVersion: ActiveStateProtocolVersion, Current: selection, UpdatedAt: journal.UpdatedAt} - if activeFound { + if runtimeRepair && active.Previous != nil { + previous := *active.Previous + newActive.Previous = &previous + } else if activeFound && !runtimeRepair { previous := active.Current newActive.Previous = &previous } @@ -479,16 +607,16 @@ func (refresher Refresher) refreshUnderLifecycleTransaction(ctx context.Context, return Status{}, rollback(fmt.Errorf("activate provider state: %w", err)) } activeChanged = true - if err := writeJournalPhase(paths.Journal, &journal, JournalActivated, refresher.now()); err != nil { + if err := refresher.writeJournal(paths.Journal, &journal, JournalActivated, refresher.now()); err != nil { return Status{}, rollback(fmt.Errorf("write activated refresh journal: %w", err)) } - if err := refresher.restartProvider(ctx); err != nil { + if err := refresher.restartProvider(ctx, config); err != nil { return Status{}, rollback(fmt.Errorf("restart active provider: %w", err)) } - if err := refresher.runProbe(ctx, providerProbeCommand(config, paths, config.StableContainer, selection)); err != nil { + if err := refresher.runManagedProbe(ctx, config, config.StableContainer, providerProbeCommand(config, paths, config.StableContainer, selection)); err != nil { return Status{}, rollback(fmt.Errorf("probe active provider: %w", err)) } - if err := writeJournalPhase(paths.Journal, &journal, JournalCommitted, refresher.now()); err != nil { + if err := refresher.writeJournal(paths.Journal, &journal, JournalCommitted, refresher.now()); err != nil { return Status{}, rollback(fmt.Errorf("commit refresh journal: %w", err)) } if deferCommit { @@ -497,6 +625,9 @@ func (refresher Refresher) refreshUnderLifecycleTransaction(ctx context.Context, if err := cleanupProviderStateTransaction(paths, update.SHA256); err != nil { return Status{}, fmt.Errorf("remove committed provider state rollback target: %w", err) } + if err := refresher.garbageCollectSupersededProviders(ctx, config, paths, newActive); err != nil { + return Status{}, fmt.Errorf("garbage collect committed provider update: %w", err) + } if err := removeDurableFile(paths.Journal); err != nil { return Status{}, fmt.Errorf("remove committed refresh journal: %w", err) } @@ -511,7 +642,7 @@ func (refresher Refresher) ServeActive(ctx context.Context, config Config) error if err := validateInstallRoot(paths.Root); err != nil { return err } - active, found, err := readActiveState(paths.ActiveState) + active, found, err := readActiveStateForConfig(paths.ActiveState, config) if err != nil { return err } @@ -532,27 +663,29 @@ func (refresher Refresher) ServeActive(ctx context.Context, config Config) error return fmt.Errorf("%s path: %w", name, err) } } - output, err := refresher.run(ctx, Command{ - Path: config.PodmanPath, - Args: []string{"image", "inspect", "--format", "{{.Id}}", active.Current.ImageRef}, - }) + if _, err := hashHostExecutable(config.PodmanPath); err != nil { + return fmt.Errorf("validate podman authority: %w", err) + } + _, imageFound, err := refresher.inspectOwnedProviderImage(ctx, config, active.Current.Update.SHA256, active.Current.ImageID) if err != nil { return fmt.Errorf("inspect active provider image: %w", err) } - imageID, err := normalizePodmanImageID(string(output)) - if err != nil { - return fmt.Errorf("validate active provider image id: %w", err) + if !imageFound { + return errors.New("active provider image is absent") } - if imageID != active.Current.ImageID { - return errors.New("active provider image id does not match durable state") + if err := refresher.removeManagedContainer(ctx, config, config.StableContainer, stableContainerRole); err != nil { + return fmt.Errorf("remove stale active provider: %w", err) } return refresher.Runner.Exec(Command{Path: config.PodmanPath, Args: []string{ "run", "--rm", "--name", config.StableContainer, + "--label", managedObjectLabel + "=" + managedProviderValue, + "--label", managedWorkerLabel + "=" + config.WorkerID, + "--label", managedRoleLabel + "=" + stableContainerRole, "--network", config.ContainerNetwork, "--read-only", "--cap-drop", "all", "--security-opt", "no-new-privileges", "--env-file", paths.ProviderEnv, - "--volume", paths.ProviderState + ":" + providerStateMount + ":rw", - "--volume", paths.TLSRoot + ":" + providerTLSMount + ":ro", + "--volume", paths.ProviderState + ":" + providerStateMount + ":rw,Z", + "--volume", paths.TLSRoot + ":" + providerTLSMount + ":ro,z", active.Current.ImageID, providerListenAddr, }}) } @@ -630,11 +763,14 @@ type verifiedUpdateCommandOutput struct { func candidateProviderCommand(config Config, paths LifecyclePaths, candidateState string, selection ImageSelection) Command { return Command{Path: config.PodmanPath, Args: []string{ "run", "--detach", "--name", config.CandidateContainer, + "--label", managedObjectLabel + "=" + managedProviderValue, + "--label", managedWorkerLabel + "=" + config.WorkerID, + "--label", managedRoleLabel + "=" + candidateContainerRole, "--network", config.ContainerNetwork, "--read-only", "--cap-drop", "all", "--security-opt", "no-new-privileges", "--env-file", paths.ProviderEnv, - "--volume", candidateState + ":" + providerStateMount + ":rw", - "--volume", paths.TLSRoot + ":" + providerTLSMount + ":ro", + "--volume", candidateState + ":" + providerStateMount + ":rw,Z", + "--volume", paths.TLSRoot + ":" + providerTLSMount + ":ro,z", selection.ImageID, providerListenAddr, }} } @@ -642,10 +778,13 @@ func candidateProviderCommand(config Config, paths LifecyclePaths, candidateStat func providerProbeCommand(config Config, paths LifecyclePaths, target string, selection ImageSelection) Command { arguments := []string{ "run", "--rm", "--name", target + "-probe", + "--label", managedObjectLabel + "=" + managedProviderValue, + "--label", managedWorkerLabel + "=" + config.WorkerID, + "--label", managedRoleLabel + "=" + probeContainerRole, "--network", config.ContainerNetwork, "--read-only", "--cap-drop", "all", "--security-opt", "no-new-privileges", "--env-file", paths.ProbeEnv, - "--volume", paths.CAFile + ":" + providerCAPath + ":ro", + "--volume", paths.CAFile + ":" + providerCAPath + ":ro,z", selection.ImageID, "probe", "-url", "https://" + target + ":18090", "-ca-file", providerCAPath, "-organization", config.Organization, "-repository", config.Repository, @@ -672,8 +811,8 @@ func (refresher Refresher) validateProviderNetwork(ctx context.Context, config C return nil } -func (refresher Refresher) restartProvider(ctx context.Context) error { - _, err := refresher.run(ctx, Command{Path: "/usr/bin/systemctl", Args: []string{"--user", "restart", providerServiceUnit}}) +func (refresher Refresher) restartProvider(ctx context.Context, config Config) error { + _, err := refresher.run(ctx, Command{Path: config.SystemctlPath, Args: []string{"--user", "restart", providerServiceUnit}}) return err } @@ -684,21 +823,21 @@ func (refresher Refresher) RestartAndProbeActive(ctx context.Context, config Con return errors.New("command runner is required") } paths := LifecyclePathsFor(config) - active, found, err := readActiveState(paths.ActiveState) + active, found, err := readActiveStateForConfig(paths.ActiveState, config) if err != nil { return err } if !found { return errors.New("retained provider has no active image") } - if err := refresher.restartProvider(ctx); err != nil { + if err := refresher.restartProvider(ctx, config); err != nil { return fmt.Errorf("restart active provider: %w", err) } return refresher.probeStableActive(ctx, config, paths, active.Current) } func (refresher Refresher) probeStableActive(ctx context.Context, config Config, paths LifecyclePaths, selection ImageSelection) error { - output, err := refresher.run(ctx, Command{Path: "/usr/bin/systemctl", Args: []string{ + output, err := refresher.run(ctx, Command{Path: config.SystemctlPath, Args: []string{ "--user", "show", providerServiceUnit, "--property", "ActiveState", "--value", }}) if err != nil { @@ -707,26 +846,84 @@ func (refresher Refresher) probeStableActive(ctx context.Context, config Config, if strings.TrimSpace(string(output)) != "active" { return errors.New("retained provider service is not active") } - if err := refresher.runProbe(ctx, providerProbeCommand(config, paths, config.StableContainer, selection)); err != nil { + if err := refresher.runManagedProbe(ctx, config, config.StableContainer, providerProbeCommand(config, paths, config.StableContainer, selection)); err != nil { return fmt.Errorf("probe active provider: %w", err) } return nil } -func (refresher Refresher) removeContainer(ctx context.Context, config Config, name string) error { - _, err := refresher.run(ctx, Command{Path: config.PodmanPath, Args: []string{"rm", "--force", "--ignore", name}}) - return err +func (refresher Refresher) removeManagedContainer(ctx context.Context, config Config, name, role string) error { + if !validManagedContainerTarget(config, name, role) { + return errors.New("container name and role are not managed by the retained provider") + } + format := "{{.ID}}\\t{{.Names}}\\t{{.Label \"" + managedObjectLabel + "\"}}\\t{{.Label \"" + managedWorkerLabel + "\"}}\\t{{.Label \"" + managedRoleLabel + "\"}}" + output, err := refresher.run(ctx, Command{Path: config.PodmanPath, Args: []string{ + "ps", "--all", "--no-trunc", "--filter", "name=^" + regexp.QuoteMeta(name) + "$", "--format", format, + }}) + if err != nil { + return fmt.Errorf("inspect %s container ownership: %w", role, err) + } + if len(output) > 4096 { + return fmt.Errorf("%s container inventory exceeds 4 KiB", role) + } + inventory := strings.TrimSpace(string(output)) + if inventory == "" { + return nil + } + lines := strings.Split(inventory, "\n") + if len(lines) != 1 { + return fmt.Errorf("%s container inventory is ambiguous", role) + } + fields := strings.Split(lines[0], "\t") + if len(fields) != 5 { + return fmt.Errorf("%s container inventory is malformed", role) + } + id := fields[0] + decodedID, decodeErr := hex.DecodeString(id) + if decodeErr != nil || len(decodedID) != 32 || strings.ToLower(id) != id { + return fmt.Errorf("%s container id is invalid", role) + } + if fields[1] != name || fields[2] != managedProviderValue || fields[3] != config.WorkerID || fields[4] != role { + return fmt.Errorf("%s container ownership does not match retained provider", role) + } + if _, err := refresher.run(ctx, Command{Path: config.PodmanPath, Args: []string{"rm", "--force", "--ignore", id}}); err != nil { + return fmt.Errorf("remove owned %s container: %w", role, err) + } + return nil } -func (refresher Refresher) runProbe(ctx context.Context, command Command) error { - delays := []time.Duration{250 * time.Millisecond, 500 * time.Millisecond, time.Second, 2 * time.Second} +func validManagedContainerTarget(config Config, name, role string) bool { + switch role { + case candidateContainerRole: + return name == config.CandidateContainer + case stableContainerRole: + return name == config.StableContainer + case probeContainerRole: + return name == config.CandidateContainer+"-probe" || name == config.StableContainer+"-probe" + default: + return false + } +} + +func (refresher Refresher) runManagedProbe(ctx context.Context, config Config, target string, command Command) error { + delays := []time.Duration{providerProbeDelay1, providerProbeDelay2, providerProbeDelay3, providerProbeDelay4} + probeName := target + "-probe" var lastErr error for attempt := 0; attempt <= len(delays); attempt++ { - if _, err := refresher.run(ctx, command); err == nil { + if err := refresher.removeManagedContainer(ctx, config, probeName, probeContainerRole); err != nil { + return fmt.Errorf("remove stale provider probe: %w", err) + } + _, runErr := refresher.run(ctx, command) + cleanupContext, cancelCleanup := context.WithTimeout(context.WithoutCancel(ctx), managedContainerCleanupTimeout) + cleanupErr := refresher.removeManagedContainer(cleanupContext, config, probeName, probeContainerRole) + cancelCleanup() + if runErr == nil && cleanupErr == nil { return nil - } else { - lastErr = err } + if cleanupErr != nil { + return errors.Join(runErr, fmt.Errorf("remove completed provider probe: %w", cleanupErr)) + } + lastErr = runErr if attempt == len(delays) { break } @@ -751,65 +948,102 @@ func (refresher Refresher) sleep(ctx context.Context, duration time.Duration) er } } -func (refresher Refresher) rollback(ctx context.Context, config Config, paths LifecyclePaths, journal TransactionJournal, activeChanged bool) error { - rollbackContext, cancelRollback := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) +func (refresher Refresher) rollback(ctx context.Context, config Config, paths LifecyclePaths, journal TransactionJournal, _ bool) error { + rollbackContext, cancelRollback := context.WithTimeout(context.WithoutCancel(ctx), retainedRollbackTimeout) defer cancelRollback() - var rollbackErr error - rollbackErr = errors.Join(rollbackErr, refresher.removeContainer(rollbackContext, config, config.CandidateContainer)) - providerStateRestored := true - if journal.Phase != JournalPrepared && journal.Phase != JournalCommitted { - if _, err := refresher.run(rollbackContext, Command{Path: "/usr/bin/systemctl", Args: []string{"--user", "stop", providerServiceUnit}}); err != nil { - providerStateRestored = false - rollbackErr = errors.Join(rollbackErr, err) - } else if err := restorePreviousProviderState(paths, journal.Candidate.Update.SHA256, journal.Phase); err != nil { - providerStateRestored = false - rollbackErr = errors.Join(rollbackErr, err) - } - } - if activeChanged && providerStateRestored { - if journal.Previous != nil { - if err := AtomicWriteJSON(paths.ActiveState, *journal.Previous); err != nil { - rollbackErr = errors.Join(rollbackErr, err) - } else if err := refresher.restartProvider(rollbackContext); err != nil { - rollbackErr = errors.Join(rollbackErr, err) - } else if err := refresher.runProbe(rollbackContext, providerProbeCommand(config, paths, config.StableContainer, journal.Previous.Current)); err != nil { - rollbackErr = errors.Join(rollbackErr, err) - } - } else { - rollbackErr = errors.Join(rollbackErr, removeDurableFile(paths.ActiveState)) - _, stopErr := refresher.run(rollbackContext, Command{Path: "/usr/bin/systemctl", Args: []string{"--user", "stop", providerServiceUnit}}) - rollbackErr = errors.Join(rollbackErr, stopErr) + if !isRollbackPhase(journal.Phase) { + if !isRollbackOrigin(journal.Phase) { + return fmt.Errorf("cannot roll back terminal provider phase %s", journal.Phase) + } + if err := validateForwardRollbackState(paths, journal); err != nil { + return err + } + journal.RollbackFrom = journal.Phase + if err := refresher.writeJournal(paths.Journal, &journal, JournalRollbackRestoring, rollbackTimestamp(journal, refresher.now())); err != nil { + return fmt.Errorf("record provider rollback intent: %w", err) } } - if !activeChanged && providerStateRestored && journal.Phase != JournalPrepared && journal.Phase != JournalCommitted { - if journal.Previous != nil { - if err := refresher.restartProvider(rollbackContext); err != nil { - rollbackErr = errors.Join(rollbackErr, err) - } else if err := refresher.runProbe(rollbackContext, providerProbeCommand(config, paths, config.StableContainer, journal.Previous.Current)); err != nil { - rollbackErr = errors.Join(rollbackErr, err) - } - } else { - _, stopErr := refresher.run(rollbackContext, Command{Path: "/usr/bin/systemctl", Args: []string{"--user", "stop", providerServiceUnit}}) - rollbackErr = errors.Join(rollbackErr, stopErr) + + if journal.Phase == JournalRollbackRestoring { + if err := refresher.restoreRollbackState(rollbackContext, config, paths, journal); err != nil { + return err + } + if err := refresher.writeJournal(paths.Journal, &journal, JournalRollbackRestored, rollbackTimestamp(journal, refresher.now())); err != nil { + return fmt.Errorf("record restored provider rollback: %w", err) + } + } + if journal.Phase == JournalRollbackRestored { + if err := refresher.cleanupCandidateArtifacts(rollbackContext, config, paths, journal.Candidate, journal.RuntimeRepair); err != nil { + return fmt.Errorf("clean provider rollback artifacts: %w", err) + } + if err := refresher.writeJournal(paths.Journal, &journal, JournalRollbackCleaned, rollbackTimestamp(journal, refresher.now())); err != nil { + return fmt.Errorf("record cleaned provider rollback: %w", err) + } + } + if journal.Phase != JournalRollbackCleaned { + return fmt.Errorf("provider rollback stopped in unsupported phase %s", journal.Phase) + } + return removeDurableFile(paths.Journal) +} + +func rollbackTimestamp(journal TransactionJournal, now time.Time) time.Time { + if now.Before(journal.UpdatedAt) { + return journal.UpdatedAt + } + return now +} + +func validateForwardRollbackState(paths LifecyclePaths, journal TransactionJournal) error { + switch journal.Phase { + case JournalStateDetached, JournalStatePromoted, JournalActivated: + if _, err := os.Lstat(paths.PreviousState(journal.Candidate.Update.SHA256)); errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("missing previous provider state during %s recovery", journal.Phase) + } else if err != nil { + return fmt.Errorf("inspect previous provider state during %s recovery: %w", journal.Phase, err) } } - if journal.Phase == JournalPrepared { - rollbackErr = errors.Join(rollbackErr, cleanupProviderStateTransaction(paths, journal.Candidate.Update.SHA256)) + return nil +} + +func (refresher Refresher) restoreRollbackState(ctx context.Context, config Config, paths LifecyclePaths, journal TransactionJournal) error { + if err := refresher.removeManagedContainer(ctx, config, config.CandidateContainer, candidateContainerRole); err != nil { + return fmt.Errorf("remove provider rollback candidate: %w", err) + } + origin := journal.RollbackFrom + if origin != JournalStaging { + if _, err := refresher.run(ctx, Command{Path: config.SystemctlPath, Args: []string{"--user", "stop", providerServiceUnit}}); err != nil { + return fmt.Errorf("stop provider before rollback restore: %w", err) + } + if origin == JournalPrepared { + if err := cleanupProviderStateTransaction(paths, journal.Candidate.Update.SHA256); err != nil { + return fmt.Errorf("clean prepared provider state: %w", err) + } + } else if err := restorePreviousProviderState(paths, journal.Candidate.Update.SHA256, origin); err != nil { + return err + } if journal.Previous != nil { - if err := refresher.restartProvider(rollbackContext); err != nil { - rollbackErr = errors.Join(rollbackErr, err) - } else if err := refresher.runProbe(rollbackContext, providerProbeCommand(config, paths, config.StableContainer, journal.Previous.Current)); err != nil { - rollbackErr = errors.Join(rollbackErr, err) + if err := AtomicWriteJSON(paths.ActiveState, *journal.Previous); err != nil { + return fmt.Errorf("restore previous active provider: %w", err) + } + if journal.RuntimeRepair { + return nil + } + if err := refresher.restartProvider(ctx, config); err != nil { + return fmt.Errorf("restart restored provider: %w", err) + } + if err := refresher.runManagedProbe(ctx, config, config.StableContainer, providerProbeCommand(config, paths, config.StableContainer, journal.Previous.Current)); err != nil { + return fmt.Errorf("probe restored provider: %w", err) } } else { - _, stopErr := refresher.run(rollbackContext, Command{Path: "/usr/bin/systemctl", Args: []string{"--user", "stop", providerServiceUnit}}) - rollbackErr = errors.Join(rollbackErr, stopErr) + if err := removeDurableFile(paths.ActiveState); err != nil { + return fmt.Errorf("remove initial provider active state: %w", err) + } + if _, err := refresher.run(ctx, Command{Path: config.SystemctlPath, Args: []string{"--user", "stop", providerServiceUnit}}); err != nil { + return fmt.Errorf("stop initial provider after rollback: %w", err) + } } } - if rollbackErr == nil { - rollbackErr = removeDurableFile(paths.Journal) - } - return rollbackErr + return nil } func (refresher Refresher) recoverInterrupted(ctx context.Context, config Config, paths LifecyclePaths) error { @@ -820,17 +1054,9 @@ func (refresher Refresher) recoverInterrupted(ctx context.Context, config Config } return fmt.Errorf("read interrupted refresh journal: %w", err) } - if err := journal.Validate(); err != nil { + if err := journal.ValidateForConfig(config); err != nil { return fmt.Errorf("validate interrupted refresh journal: %w", err) } - if err := refresher.removeContainer(ctx, config, config.CandidateContainer); err != nil { - return fmt.Errorf("remove interrupted provider candidate: %w", err) - } - if journal.Phase != JournalCommitted { - if _, err := refresher.run(ctx, Command{Path: "/usr/bin/systemctl", Args: []string{"--user", "stop", providerServiceUnit}}); err != nil { - return fmt.Errorf("stop interrupted provider before state recovery: %w", err) - } - } if journal.Phase == JournalCommitted { if journal.DeferredCommit { return errors.New("deferred installer refresh requires installer finalization or rollback") @@ -845,45 +1071,147 @@ func (refresher Refresher) recoverInterrupted(ctx context.Context, config Config if err := cleanupProviderStateTransaction(paths, journal.Candidate.Update.SHA256); err != nil { return fmt.Errorf("clean committed provider state transaction: %w", err) } - } else if journal.Previous == nil { - if journal.Phase != JournalPrepared { - if err := restorePreviousProviderState(paths, journal.Candidate.Update.SHA256, journal.Phase); err != nil { - return fmt.Errorf("restore interrupted initial provider state: %w", err) - } - } else if err := cleanupProviderStateTransaction(paths, journal.Candidate.Update.SHA256); err != nil { - return fmt.Errorf("clean interrupted initial candidate state: %w", err) - } - if err := removeDurableFile(paths.ActiveState); err != nil { - return fmt.Errorf("remove interrupted initial active state: %w", err) - } - } else { - if journal.Phase != JournalPrepared { - if err := restorePreviousProviderState(paths, journal.Candidate.Update.SHA256, journal.Phase); err != nil { - return fmt.Errorf("restore interrupted provider state: %w", err) - } - } else if err := cleanupProviderStateTransaction(paths, journal.Candidate.Update.SHA256); err != nil { - return fmt.Errorf("clean interrupted candidate state: %w", err) - } - recovered, err := RecoverActiveState(journal) - if err != nil { - return err - } - if err := AtomicWriteJSON(paths.ActiveState, recovered); err != nil { - return fmt.Errorf("write recovered active state: %w", err) + if err := refresher.garbageCollectSupersededProviders(ctx, config, paths, recovered); err != nil { + return fmt.Errorf("garbage collect recovered provider update: %w", err) } - if err := refresher.restartProvider(ctx); err != nil { - return fmt.Errorf("restart recovered provider: %w", err) + if err := removeDurableFile(paths.Journal); err != nil { + return fmt.Errorf("remove recovered refresh journal: %w", err) } - if err := refresher.runProbe(ctx, providerProbeCommand(config, paths, config.StableContainer, recovered.Current)); err != nil { - return fmt.Errorf("probe recovered provider: %w", err) + return nil + } + return refresher.rollback(ctx, config, paths, journal, false) +} + +func (refresher Refresher) cleanupCandidateArtifacts(ctx context.Context, config Config, paths LifecyclePaths, candidate ImageSelection, runtimeRepair bool) error { + digest := candidate.Update.SHA256 + if !digestPattern.MatchString(digest) { + return errors.New("candidate artifact digest is invalid") + } + active, found, err := readActiveStateForConfig(paths.ActiveState, config) + if err != nil { + return fmt.Errorf("read active state before candidate cleanup: %w", err) + } + if found && !runtimeRepair { + if selectionMatchesArtifact(active.Current, digest, candidate.ImageID) || active.Previous != nil && selectionMatchesArtifact(*active.Previous, digest, candidate.ImageID) { + return nil } } - if err := removeDurableFile(paths.Journal); err != nil { - return fmt.Errorf("remove recovered refresh journal: %w", err) + packagesExist := true + if _, err := os.Lstat(paths.PackagesRoot); errors.Is(err, os.ErrNotExist) { + packagesExist = false + } else if err != nil { + return fmt.Errorf("inspect provider packages root: %w", err) + } else if err := validateOwnedDirectory(paths.PackagesRoot); err != nil { + return fmt.Errorf("validate provider packages root: %w", err) + } + if err := refresher.removeOwnedProviderImage(ctx, config, digest, candidate.ImageID); err != nil { + return fmt.Errorf("remove candidate provider image: %w", err) + } + if runtimeRepair { + return nil + } + if !packagesExist { + return nil + } + packagePath := paths.PackageDir(digest) + if _, err := os.Lstat(packagePath); errors.Is(err, os.ErrNotExist) { + return nil + } else if err != nil { + return fmt.Errorf("inspect candidate provider package: %w", err) + } + if err := validateOwnedDirectory(packagePath); err != nil { + return fmt.Errorf("validate candidate provider package: %w", err) + } + if err := removeOwnedDirectory(packagePath); err != nil { + return fmt.Errorf("remove candidate provider package: %w", err) + } + return syncDirectory(paths.PackagesRoot) +} + +func selectionMatchesArtifact(selection ImageSelection, digest, imageID string) bool { + return selection.Update.SHA256 == digest || imageID != "" && selection.ImageID == imageID +} + +func (refresher Refresher) removeOwnedProviderImage(ctx context.Context, config Config, digest, expectedImageID string) error { + imageID, found, err := refresher.inspectOwnedProviderImage(ctx, config, digest, expectedImageID) + if err != nil { + return err + } + if !found { + return nil + } + if _, err := refresher.run(ctx, Command{Path: config.PodmanPath, Args: []string{"image", "rm", "--ignore", imageID}}); err != nil { + return fmt.Errorf("remove owned provider image: %w", err) } return nil } +func (refresher Refresher) inspectOwnedProviderImage(ctx context.Context, config Config, digest, expectedImageID string) (string, bool, error) { + if !digestPattern.MatchString(digest) { + return "", false, errors.New("provider image digest is invalid") + } + if expectedImageID != "" { + normalized, err := normalizePodmanImageID(expectedImageID) + if err != nil || normalized != expectedImageID { + return "", false, errors.New("expected provider image id is invalid") + } + } + arguments := []string{"images", "--no-trunc"} + if expectedImageID != "" { + arguments = append(arguments, "--filter", "id="+expectedImageID) + } else { + arguments = append(arguments, + "--filter", "label="+managedObjectLabel+"="+managedProviderValue, + "--filter", "label="+managedWorkerLabel+"="+config.WorkerID, + "--filter", "label="+managedRoleLabel+"="+providerImageRole, + "--filter", "label="+managedDigestLabel+"="+digest, + ) + } + format := "{{.ID}}\\t{{.Label \"" + managedObjectLabel + "\"}}\\t{{.Label \"" + managedWorkerLabel + "\"}}\\t{{.Label \"" + managedRoleLabel + "\"}}\\t{{.Label \"" + managedDigestLabel + "\"}}" + arguments = append(arguments, "--format", format) + output, err := refresher.run(ctx, Command{Path: config.PodmanPath, Args: arguments}) + if err != nil { + return "", false, fmt.Errorf("inspect provider image ownership: %w", err) + } + if len(output) > 4096 { + return "", false, errors.New("provider image inventory exceeds 4 KiB") + } + inventory := strings.TrimSpace(string(output)) + if inventory == "" { + return "", false, nil + } + lines := strings.Split(inventory, "\n") + if len(lines) != 1 { + return "", false, errors.New("provider image inventory is ambiguous") + } + fields := strings.Split(lines[0], "\t") + if len(fields) != 5 { + return "", false, errors.New("provider image inventory is malformed") + } + imageID, err := normalizePodmanImageID(fields[0]) + if err != nil { + return "", false, fmt.Errorf("provider image id is invalid: %w", err) + } + if fields[1] != managedProviderValue || fields[2] != config.WorkerID || fields[3] != providerImageRole || fields[4] != digest { + return "", false, fmt.Errorf("%w: provider image labels do not match retained provider", errProviderImageOwnershipDrift) + } + if expectedImageID != "" && imageID != expectedImageID { + return "", false, errors.New("provider image id does not match durable state") + } + return imageID, true, nil +} + +func (refresher Refresher) activeProviderImageNeedsRepair(ctx context.Context, config Config, selection ImageSelection) (bool, error) { + _, found, err := refresher.inspectOwnedProviderImage(ctx, config, selection.Update.SHA256, selection.ImageID) + if errors.Is(err, errProviderImageOwnershipDrift) { + return true, nil + } + if err != nil { + return false, fmt.Errorf("inspect active provider image: %w", err) + } + return !found, nil +} + func writeJournalPhase(path string, journal *TransactionJournal, phase JournalPhase, updatedAt time.Time) error { next := *journal next.Phase = phase @@ -895,9 +1223,16 @@ func writeJournalPhase(path string, journal *TransactionJournal, phase JournalPh return nil } +func (refresher Refresher) writeJournal(path string, journal *TransactionJournal, phase JournalPhase, updatedAt time.Time) error { + if refresher.writeJournalPhaseFn != nil { + return refresher.writeJournalPhaseFn(path, journal, phase, updatedAt) + } + return writeJournalPhase(path, journal, phase, updatedAt) +} + func (refresher Refresher) finalizeDeferredRefresh(config Config) error { paths := LifecyclePathsFor(config) - journal, found, err := readTransactionJournal(paths.Journal) + journal, found, err := readTransactionJournalForConfig(paths.Journal, config) if err != nil || !found { return err } @@ -907,21 +1242,64 @@ func (refresher Refresher) finalizeDeferredRefresh(config Config) error { if err := cleanupProviderStateTransaction(paths, journal.Candidate.Update.SHA256); err != nil { return err } + active, found, err := readActiveStateForConfig(paths.ActiveState, config) + if err != nil { + return err + } + if !found { + return errors.New("deferred committed refresh has no active provider") + } + if err := refresher.garbageCollectSupersededProviders(context.Background(), config, paths, active); err != nil { + return err + } return removeDurableFile(paths.Journal) } -func (refresher Refresher) finalizeInterruptedDeferredCommit(config Config) error { - paths := LifecyclePathsFor(config) - journal, found, err := readTransactionJournal(paths.Journal) - if err != nil || !found || !journal.DeferredCommit || journal.Phase != JournalCommitted { - return err +func (refresher Refresher) garbageCollectSupersededProviders(ctx context.Context, config Config, paths LifecyclePaths, active ActiveState) error { + retained := map[string]struct{}{digestHex(active.Current.Update.SHA256): {}} + if active.Previous != nil { + retained[digestHex(active.Previous.Update.SHA256)] = struct{}{} + } + entries, err := os.ReadDir(paths.PackagesRoot) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("read provider packages: %w", err) } - return refresher.finalizeDeferredRefresh(config) + if err := validateOwnedDirectory(paths.PackagesRoot); err != nil { + return fmt.Errorf("validate provider packages root: %w", err) + } + removed := false + for _, entry := range entries { + if _, keep := retained[entry.Name()]; keep { + continue + } + digest := "sha256:" + entry.Name() + if !digestPattern.MatchString(digest) { + return fmt.Errorf("provider package entry %q is not an owned digest", entry.Name()) + } + packagePath := filepath.Join(paths.PackagesRoot, entry.Name()) + if err := validateOwnedDirectory(packagePath); err != nil { + return fmt.Errorf("validate superseded provider package: %w", err) + } + if err := refresher.removeOwnedProviderImage(ctx, config, digest, ""); err != nil { + return fmt.Errorf("remove superseded provider image: %w", err) + } + if err := removeOwnedDirectory(packagePath); err != nil { + return fmt.Errorf("remove superseded provider package: %w", err) + } + removed = true + } + if removed { + return syncDirectory(paths.PackagesRoot) + } + return nil } func (refresher Refresher) rollbackDeferredRefresh(ctx context.Context, config Config) error { paths := LifecyclePathsFor(config) - journal, found, err := readTransactionJournal(paths.Journal) + journal, found, err := readTransactionJournalForConfig(paths.Journal, config) if err != nil || !found { return err } @@ -946,7 +1324,29 @@ func readTransactionJournal(path string) (TransactionJournal, bool, error) { return journal, true, nil } +func readTransactionJournalForConfig(path string, config Config) (TransactionJournal, bool, error) { + journal, found, err := readTransactionJournal(path) + if err != nil || !found { + return journal, found, err + } + if err := journal.ValidateForConfig(config); err != nil { + return TransactionJournal{}, false, fmt.Errorf("transaction journal identity: %w", err) + } + return journal, true, nil +} + func promoteCandidateProviderState(paths LifecyclePaths, digest string) error { + if err := detachProviderState(paths, digest); err != nil { + return err + } + return activateCandidateProviderState(paths, digest) +} + +func detachProviderState(paths LifecyclePaths, digest string) error { + return detachProviderStateWithSync(paths, digest, syncDirectory) +} + +func detachProviderStateWithSync(paths LifecyclePaths, digest string, syncDir func(string) error) error { candidate := paths.CandidateState(digest) previous := paths.PreviousState(digest) if err := validateOwnedDirectory(paths.ProviderState); err != nil { @@ -964,24 +1364,51 @@ func promoteCandidateProviderState(paths LifecyclePaths, digest string) error { if err := os.Rename(paths.ProviderState, previous); err != nil { return fmt.Errorf("retain previous provider state: %w", err) } + if err := syncDir(paths.Root); err != nil { + return fmt.Errorf("sync active provider state parent after detach: %w", err) + } + if err := syncDir(filepath.Dir(candidate)); err != nil { + return fmt.Errorf("sync provider transaction after detach: %w", err) + } + return nil +} + +func activateCandidateProviderState(paths LifecyclePaths, digest string) error { + return activateCandidateProviderStateWithSync(paths, digest, syncDirectory) +} + +func activateCandidateProviderStateWithSync(paths LifecyclePaths, digest string, syncDir func(string) error) error { + candidate := paths.CandidateState(digest) + previous := paths.PreviousState(digest) + if _, err := os.Lstat(paths.ProviderState); !errors.Is(err, os.ErrNotExist) { + if err == nil { + return errors.New("active provider state exists before candidate activation") + } + return fmt.Errorf("inspect active provider state before candidate activation: %w", err) + } + if err := validateOwnedDirectory(previous); err != nil { + return fmt.Errorf("validate detached provider state: %w", err) + } + if err := validateOwnedDirectory(candidate); err != nil { + return fmt.Errorf("validate candidate provider state: %w", err) + } if err := os.Rename(candidate, paths.ProviderState); err != nil { - restoreErr := os.Rename(previous, paths.ProviderState) - return errors.Join(fmt.Errorf("activate candidate provider state: %w", err), restoreErr) + return fmt.Errorf("activate candidate provider state: %w", err) + } + if err := syncDir(filepath.Dir(candidate)); err != nil { + return fmt.Errorf("sync provider transaction after activation: %w", err) + } + if err := syncDir(paths.Root); err != nil { + return fmt.Errorf("sync active provider state parent after activation: %w", err) } - return errors.Join(syncDirectory(paths.Root), syncDirectory(filepath.Dir(candidate))) + return nil } func restorePreviousProviderState(paths LifecyclePaths, digest string, phase JournalPhase) error { previous := paths.PreviousState(digest) if _, err := os.Lstat(previous); errors.Is(err, os.ErrNotExist) { - if phase != JournalStatePromoting { - return fmt.Errorf("missing previous provider state during %s recovery", phase) - } if err := validateOwnedDirectory(paths.ProviderState); err != nil { - return fmt.Errorf("validate unpromoted provider state: %w", err) - } - if err := validateOwnedDirectory(paths.CandidateState(digest)); err != nil { - return fmt.Errorf("validate unpromoted candidate state: %w", err) + return fmt.Errorf("validate restored provider state during %s recovery: %w", phase, err) } return cleanupProviderStateTransaction(paths, digest) } else if err != nil { @@ -1000,13 +1427,24 @@ func restorePreviousProviderState(paths LifecyclePaths, digest string, phase Jou if err := os.Rename(previous, paths.ProviderState); err != nil { return fmt.Errorf("restore previous provider state: %w", err) } + if err := syncDirectory(filepath.Dir(previous)); err != nil { + return fmt.Errorf("sync provider transaction after restore: %w", err) + } if err := syncDirectory(paths.Root); err != nil { - return err + return fmt.Errorf("sync active provider state parent after restore: %w", err) } return cleanupProviderStateTransaction(paths, digest) } func cleanupProviderStateTransaction(paths LifecyclePaths, digest string) error { + if _, err := os.Lstat(paths.CandidatesRoot); errors.Is(err, os.ErrNotExist) { + return nil + } else if err != nil { + return fmt.Errorf("inspect provider state transactions root: %w", err) + } + if err := validateOwnedDirectory(paths.CandidatesRoot); err != nil { + return fmt.Errorf("validate provider state transactions root: %w", err) + } transactionRoot := filepath.Join(paths.CandidatesRoot, digestHex(digest)) info, err := os.Lstat(transactionRoot) if errors.Is(err, os.ErrNotExist) { @@ -1061,7 +1499,7 @@ func stageVerifiedProvider(update VerifiedUpdate, paths LifecyclePaths) error { if existingDigest, err := hashRegularFile(destination, true); err == nil && existingDigest == update.SHA256 { return nil } - if err := os.MkdirAll(paths.PackageDir(update.SHA256), 0o700); err != nil { + if err := mkdirAllDurable(paths.PackageDir(update.SHA256), 0o700); err != nil { return fmt.Errorf("create provider package directory: %w", err) } temporary, err := os.CreateTemp(paths.PackageDir(update.SHA256), ".provider-*.tmp") @@ -1077,7 +1515,7 @@ func stageVerifiedProvider(update VerifiedUpdate, paths LifecyclePaths) error { if err != nil { return fmt.Errorf("open verified provider package: %w", err) } - defer source.Close() + defer func() { _ = source.Close() }() copied, err := io.Copy(temporary, io.LimitReader(source, maxProviderPackageBytes+1)) if err != nil { return fmt.Errorf("copy verified provider package: %w", err) @@ -1104,7 +1542,7 @@ func stageVerifiedProvider(update VerifiedUpdate, paths LifecyclePaths) error { } func prepareCandidateState(source, destination string) error { - if err := os.MkdirAll(source, 0o700); err != nil { + if err := mkdirAllDurable(source, 0o700); err != nil { return fmt.Errorf("create provider state: %w", err) } if err := os.RemoveAll(destination); err != nil { @@ -1130,6 +1568,17 @@ func readActiveState(path string) (ActiveState, bool, error) { return active, true, nil } +func readActiveStateForConfig(path string, config Config) (ActiveState, bool, error) { + active, found, err := readActiveState(path) + if err != nil || !found { + return active, found, err + } + if err := active.ValidateForConfig(config); err != nil { + return ActiveState{}, false, fmt.Errorf("validate active state identity: %w", err) + } + return active, true, nil +} + func statusForActive(active ActiveState, serviceActive bool, now time.Time) Status { return Status{ ProtocolVersion: StatusProtocolVersion, @@ -1171,8 +1620,8 @@ func validateProviderEnvironment(config Config, path string) error { {key: "GITHUB_RUNNER_PROVIDER_ORGANIZATIONS", value: config.Organization}, {key: "GITHUB_RUNNER_PROVIDER_RUNNER_GROUPS", value: config.RunnerGroup}, } { - if !commaSeparatedEnvironmentContains(values[expected.key], expected.value) { - return errors.New("provider environment is missing a required GitHub allowlist value") + if values[expected.key] != expected.value { + return errors.New("provider environment does not exactly match configured GitHub authority") } } for key := range values { @@ -1183,15 +1632,6 @@ func validateProviderEnvironment(config Config, path string) error { return nil } -func commaSeparatedEnvironmentContains(value, expected string) bool { - for item := range strings.SplitSeq(value, ",") { - if strings.TrimSpace(item) == expected { - return true - } - } - return false -} - func allowedProviderEnvironmentKey(key string) bool { switch key { case "GITHUB_RUNNER_PROVIDER_TOKEN", @@ -1201,8 +1641,7 @@ func allowedProviderEnvironmentKey(key string) bool { "GITHUB_RUNNER_PROVIDER_ORGANIZATIONS", "GITHUB_RUNNER_PROVIDER_RUNNER_GROUPS", "GITHUB_RUNNER_PROVIDER_TLS_CERT_FILE", - "GITHUB_RUNNER_PROVIDER_TLS_KEY_FILE", - "GITHUB_API_BASE_URL": + "GITHUB_RUNNER_PROVIDER_TLS_KEY_FILE": return true default: return false @@ -1228,7 +1667,7 @@ func readEnvironmentFile(path string) (map[string]string, error) { if err != nil { return nil, err } - defer file.Close() + defer func() { _ = file.Close() }() values := make(map[string]string) scanner := bufio.NewScanner(io.LimitReader(file, MaxStateFileBytes+1)) for scanner.Scan() { @@ -1279,6 +1718,14 @@ func validateSecretFile(path string) error { } func hashRegularFile(path string, requireExecutable bool) (string, error) { + return hashValidatedRegularFile(path, requireExecutable, nil) +} + +func hashHostExecutable(path string) (string, error) { + return hashValidatedRegularFile(path, true, validateExecutableAuthority) +} + +func hashValidatedRegularFile(path string, requireExecutable bool, validate func(os.FileInfo) error) (string, error) { entry, err := os.Lstat(path) if err != nil { return "", err @@ -1289,15 +1736,28 @@ func hashRegularFile(path string, requireExecutable bool) (string, error) { if requireExecutable && executableModeRequired() && entry.Mode().Perm()&0o111 == 0 { return "", errors.New("path must be executable") } + if validate != nil { + if err := validate(entry); err != nil { + return "", err + } + } file, err := os.Open(path) if err != nil { return "", err } - defer file.Close() + defer func() { _ = file.Close() }() opened, err := file.Stat() if err != nil || !opened.Mode().IsRegular() || !os.SameFile(entry, opened) { return "", errors.New("path changed during open") } + if requireExecutable && executableModeRequired() && opened.Mode().Perm()&0o111 == 0 { + return "", errors.New("path must be executable") + } + if validate != nil { + if err := validate(opened); err != nil { + return "", err + } + } hasher := sha256.New() if _, err := io.Copy(hasher, file); err != nil { return "", err diff --git a/internal/retainedprovider/refresh_test.go b/internal/retainedprovider/refresh_test.go index f68c3b6..899b641 100644 --- a/internal/retainedprovider/refresh_test.go +++ b/internal/retainedprovider/refresh_test.go @@ -1,13 +1,16 @@ package retainedprovider import ( + "bytes" "context" "crypto/sha256" "encoding/hex" "errors" + "fmt" "os" "path/filepath" "reflect" + "regexp" "strings" "testing" "time" @@ -127,6 +130,37 @@ func TestInitialRefreshRequiresInstallerDigestMatch(t *testing.T) { } } +func TestInitialRefreshRevalidatesInstallerAfterVerifiedUpdateChanges(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + if err := os.MkdirAll(config.InstallRoot, 0o700); err != nil { + t.Fatalf("mkdir install root: %v", err) + } + initial := writeTestProviderPayload(t, home, "verified-provider-initial") + initialDigest := fileDigestForTest(t, initial) + replacement := writeTestProviderPayload(t, home, "verified-provider-replacement") + replacementDigest := fileDigestForTest(t, replacement) + runner := refreshTestRunner(config, initial, initialDigest) + baseRun := runner.run + verifyCalls := 0 + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if command.Path == config.ComputeAgentPath && containsAdjacentArgs(command.Args, "supervisor-update", "verify") { + verifyCalls++ + if verifyCalls == 2 { + return testVerifiedUpdateJSON(config, replacement, replacementDigest), nil + } + } + return baseRun(ctx, command) + } + refresher := Refresher{Runner: runner, ExecutablePath: func() (string, error) { return initial, nil }} + if _, err := refresher.Refresh(t.Context(), config); err == nil || !strings.Contains(err.Error(), "installer digest") { + t.Fatalf("changed initial verified update err = %v", err) + } + if verifyCalls != 2 || len(runner.commands) != 2 { + t.Fatalf("changed initial verified update mutated runtime: verify_calls=%d commands=%+v", verifyCalls, runner.commands) + } +} + func TestRefreshBoundsEverySubprocessContext(t *testing.T) { home := t.TempDir() config := validTestConfig(home) @@ -169,6 +203,7 @@ func TestRefreshFencesAgentDuringProviderMutation(t *testing.T) { baseRun := runner.run statuses := []string{"unavailable", "unavailable", "idle"} var events []string + maintenanceID := "" runner.run = func(ctx context.Context, command Command) ([]byte, error) { switch installCommandEvent(command, config) { case "maintenance-begin": @@ -177,10 +212,11 @@ func TestRefreshFencesAgentDuringProviderMutation(t *testing.T) { t.Fatalf("maintenance begin lifecycle journal = %+v found=%v err=%v", journal, found, err) } events = append(events, "maintenance-begin") - return maintenanceStateJSON(true, "workflow-plugin-github-retained-provider-refresh", config.ProfileID, "workflow-plugin-github-retained-provider-refresh"), nil + maintenanceID = journal.TransactionID + return maintenanceStateJSON(true, maintenanceID, config.ProfileID, refreshMaintenanceReason), nil case "maintenance-status": events = append(events, "maintenance-status") - return maintenanceStateJSON(true, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + return maintenanceStateJSON(true, maintenanceID, config.ProfileID, refreshMaintenanceReason), nil case "maintenance-end": journal, found, err := readLifecycleJournal(home, paths) if err != nil || !found || journal.Phase != LifecycleReleasing || journal.Outcome != LifecycleCommit || journal.ProviderTransaction == nil { @@ -191,7 +227,7 @@ func TestRefreshFencesAgentDuringProviderMutation(t *testing.T) { t.Fatalf("maintenance end provider journal = %+v found=%v err=%v", inner, innerFound, err) } events = append(events, "maintenance-end") - return maintenanceStateJSON(false, "workflow-plugin-github-retained-provider-refresh", config.ProfileID, "workflow-plugin-github-retained-provider-refresh"), nil + return maintenanceStateJSON(false, maintenanceID, config.ProfileID, refreshMaintenanceReason), nil case "local-status": if len(statuses) == 0 { t.Fatal("unexpected extra local status read") @@ -259,7 +295,11 @@ func TestSameDigestRefreshHealthCheckDoesNotFenceAgent(t *testing.T) { } return originalRun(ctx, command) } - refresher := Refresher{Runner: runner, Sleep: func(context.Context, time.Duration) error { return nil }} + refresher := Refresher{ + Runner: runner, + ExecutablePath: func() (string, error) { return payload, nil }, + Sleep: func(context.Context, time.Duration) error { return nil }, + } if _, err := refresher.Refresh(t.Context(), config); err != nil { t.Fatalf("same-digest refresh: %v", err) } @@ -277,6 +317,202 @@ func TestSameDigestRefreshHealthCheckDoesNotFenceAgent(t *testing.T) { } } +func TestSameDigestRefreshRepairsMissingActiveImageUnderFence(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + payload := writeTestProviderPayload(t, home, "verified-provider-runtime-repair") + digest := fileDigestForTest(t, payload) + now := time.Unix(1_700_000_000, 0).UTC() + selection := selectionForDigest(payload, digest, "v1.0.31", "directive-repair", testProviderImageID, now) + if err := AtomicWriteJSON(paths.ActiveState, ActiveState{ProtocolVersion: ActiveStateProtocolVersion, Current: selection, UpdatedAt: now}); err != nil { + t.Fatalf("write active state: %v", err) + } + + runner := refreshTestRunner(config, payload, digest) + baseRun := runner.run + imageBuilt := false + sawRepairJournal := false + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if command.Path == config.PodmanPath && firstArg(command.Args) == "images" && !imageBuilt { + return nil, nil + } + if command.Path == config.PodmanPath && firstArg(command.Args) == "build" { + journal, found, err := readTransactionJournalForConfig(paths.Journal, config) + if err != nil || !found || !journal.RuntimeRepair || journal.Previous == nil || journal.Candidate.Update.SHA256 != journal.Previous.Current.Update.SHA256 { + t.Fatalf("runtime repair journal = %+v found=%v err=%v", journal, found, err) + } + sawRepairJournal = true + imageBuilt = true + } + return baseRun(ctx, command) + } + + status, err := (Refresher{ + Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, + Sleep: func(context.Context, time.Duration) error { return nil }, + }).Refresh(t.Context(), config) + if err != nil { + t.Fatalf("repair missing active image: %v\n%s", err, commandTranscript(runner.commands)) + } + if !sawRepairJournal || status.CurrentSHA256 != digest { + t.Fatalf("runtime repair status=%+v journaled=%v", status, sawRepairJournal) + } + transcript := commandTranscript(runner.commands) + for _, required := range []string{ + "supervisor-maintenance begin", "systemctl --user stop " + config.AgentUnit, + "podman build", config.CandidateContainer + "-probe", config.StableContainer + "-probe", + "supervisor-maintenance end", + } { + if !strings.Contains(transcript, required) { + t.Fatalf("runtime repair transcript missing %q:\n%s", required, transcript) + } + } + repaired, found, err := readActiveStateForConfig(paths.ActiveState, config) + if err != nil || !found || repaired.Current.Update.SHA256 != digest || repaired.Current.ImageID != testProviderImageID || repaired.Previous != nil { + t.Fatalf("repaired active state = %+v found=%v err=%v", repaired, found, err) + } +} + +func TestSameDigestImageLossRaceRestartsThroughFenceBeforeRepair(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + payload := writeTestProviderPayload(t, home, "verified-provider-raced-runtime-repair") + digest := fileDigestForTest(t, payload) + now := time.Unix(1_700_000_000, 0).UTC() + selection := selectionForDigest(payload, digest, "v1.0.31", "directive-raced-repair", testProviderImageID, now) + if err := AtomicWriteJSON(paths.ActiveState, ActiveState{ProtocolVersion: ActiveStateProtocolVersion, Current: selection, UpdatedAt: now}); err != nil { + t.Fatalf("write active state: %v", err) + } + + runner := refreshTestRunner(config, payload, digest) + baseRun := runner.run + imageChecks := 0 + imageBuilt := false + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if command.Path == config.PodmanPath && firstArg(command.Args) == "images" && !imageBuilt { + imageChecks++ + if imageChecks == 1 { + return ownedProviderImageInventory(config, digest, testProviderImageID), nil + } + return nil, nil + } + if command.Path == config.PodmanPath && firstArg(command.Args) == "build" { + imageBuilt = true + } + return baseRun(ctx, command) + } + if _, err := (Refresher{ + Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, + Sleep: func(context.Context, time.Duration) error { return nil }, + }).Refresh(t.Context(), config); err != nil { + t.Fatalf("repair raced image loss: %v\n%s", err, commandTranscript(runner.commands)) + } + transcript := commandTranscript(runner.commands) + maintenance := strings.Index(transcript, "supervisor-maintenance begin") + build := strings.Index(transcript, "podman build") + if maintenance < 0 || build < 0 || maintenance > build { + t.Fatalf("raced runtime repair was not fenced before build:\n%s", transcript) + } +} + +func TestSameDigestOwnershipDriftRequiresRuntimeRepair(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + payload := writeTestProviderPayload(t, home, "verified-provider-ownership-repair") + digest := fileDigestForTest(t, payload) + now := time.Unix(1_700_000_000, 0).UTC() + selection := selectionForDigest(payload, digest, "v1.0.31", "directive-ownership-repair", testProviderImageID, now) + if err := AtomicWriteJSON(paths.ActiveState, ActiveState{ProtocolVersion: ActiveStateProtocolVersion, Current: selection, UpdatedAt: now}); err != nil { + t.Fatalf("write active state: %v", err) + } + runner := refreshTestRunner(config, payload, digest) + baseRun := runner.run + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if command.Path == config.PodmanPath && firstArg(command.Args) == "images" { + return []byte(testProviderImageID + "\t" + managedProviderValue + "\tother-worker\t" + providerImageRole + "\t" + digest + "\n"), nil + } + return baseRun(ctx, command) + } + repair, err := (Refresher{Runner: runner}).requiresMutation(t.Context(), config, paths) + if err != nil || !repair { + t.Fatalf("ownership drift repair=%v err=%v", repair, err) + } +} + +func TestFailedSameDigestRepairRemovesImageButRetainsVerifiedPackage(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + payload := writeTestProviderPayload(t, home, "verified-provider-failed-repair") + digest := fileDigestForTest(t, payload) + now := time.Unix(1_700_000_000, 0).UTC() + selection := selectionForDigest(payload, digest, "v1.0.31", "directive-failed-repair", testProviderImageID, now) + original := ActiveState{ProtocolVersion: ActiveStateProtocolVersion, Current: selection, UpdatedAt: now} + if err := AtomicWriteJSON(paths.ActiveState, original); err != nil { + t.Fatalf("write active state: %v", err) + } + + runner := refreshTestRunner(config, payload, digest) + baseRun := runner.run + imageBuilt := false + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if command.Path == config.PodmanPath && firstArg(command.Args) == "images" && !imageBuilt { + return nil, nil + } + if command.Path == config.PodmanPath && firstArg(command.Args) == "build" { + imageBuilt = true + } + if isProbeFor(command, config.CandidateContainer) { + return nil, errors.New("candidate repair probe failed") + } + return baseRun(ctx, command) + } + _, err := (Refresher{ + Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, + Sleep: func(context.Context, time.Duration) error { return nil }, + }).Refresh(t.Context(), config) + if err == nil || !strings.Contains(err.Error(), "candidate repair probe failed") { + t.Fatalf("failed runtime repair err = %v", err) + } + repairErr := err + transcript := commandTranscript(runner.commands) + if !strings.Contains(transcript, "image rm --ignore "+testProviderImageID) { + t.Fatalf("failed runtime repair retained rebuilt image:\n%s", transcript) + } + if strings.Contains(transcript, "probe -url "+config.ProviderURL) { + t.Fatalf("failed runtime repair probed the known-missing prior image:\n%s", transcript) + } + if _, err := os.Stat(paths.PackageBinary(digest)); err != nil { + t.Fatalf("failed runtime repair removed verified package: %v", err) + } + restored, found, err := readActiveStateForConfig(paths.ActiveState, config) + if err != nil || !found || restored.Current != original.Current { + t.Fatalf("failed runtime repair active state = %+v found=%v err=%v", restored, found, err) + } + if _, found, err := readTransactionJournal(paths.Journal); err != nil || found { + t.Fatalf("failed runtime repair provider journal found=%v err=%v", found, err) + } + if _, found, err := readLifecycleJournal(home, paths); err != nil || found { + t.Fatalf("failed runtime repair lifecycle journal found=%v err=%v repair_err=%v", found, err, repairErr) + } +} + func TestNormalizePodmanImageIDCanonicalizesOnlyImmutableSHA256(t *testing.T) { hexDigest := strings.Repeat("a", 64) for _, input := range []string{hexDigest, "sha256:" + hexDigest, "\n" + hexDigest + "\n"} { @@ -292,6 +528,59 @@ func TestNormalizePodmanImageIDCanonicalizesOnlyImmutableSHA256(t *testing.T) { } } +func TestRefreshRenewsExpiringTLSUnderFenceWithoutPackageChange(t *testing.T) { + home := t.TempDir() + t.Setenv("XDG_STATE_HOME", filepath.Join(home, ".state")) + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + payload := writeTestProviderPayload(t, home, "verified-provider-tls-renewal") + digest := fileDigestForTest(t, payload) + issuedAt := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + material, err := GenerateInstallMaterial(config, Credentials{GitHubToken: "github-secret", ProviderToken: "provider-secret"}, bytes.NewReader(bytes.Repeat([]byte{0x47}, 4096)), issuedAt) + if err != nil { + t.Fatalf("generate install material: %v", err) + } + if err := WriteInstallMaterial(paths, material); err != nil { + t.Fatalf("write install material: %v", err) + } + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("create provider state: %v", err) + } + selection := selectionForDigest(payload, digest, "v1.0.32", "directive-1", testProviderImageID, issuedAt) + if err := AtomicWriteJSON(paths.ActiveState, ActiveState{ProtocolVersion: ActiveStateProtocolVersion, Current: selection, UpdatedAt: issuedAt}); err != nil { + t.Fatalf("write active state: %v", err) + } + runner := refreshTestRunner(config, payload, digest) + renewedAt := issuedAt.Add(350 * 24 * time.Hour) + status, err := (Refresher{ + Runner: runner, Random: bytes.NewReader(bytes.Repeat([]byte{0x48}, 4096)), + Now: func() time.Time { return renewedAt }, Sleep: func(context.Context, time.Duration) error { return nil }, + }).Refresh(t.Context(), config) + if err != nil { + t.Fatalf("refresh expiring TLS: %v\n%s", err, commandTranscript(runner.commands)) + } + if status.CurrentSHA256 != digest { + t.Fatalf("TLS-only refresh status = %+v", status) + } + transcript := commandTranscript(runner.commands) + for _, want := range []string{ + "supervisor-maintenance begin", "systemctl --user stop " + config.AgentUnit, + "systemctl --user restart " + providerServiceUnit, config.StableContainer + "-probe", + "systemctl --user start " + config.AgentUnit, "supervisor-maintenance end", + } { + if !strings.Contains(transcript, want) { + t.Fatalf("TLS renewal transcript missing %q:\n%s", want, transcript) + } + } + if strings.Contains(transcript, "podman build") { + t.Fatalf("TLS-only refresh rebuilt provider image:\n%s", transcript) + } + renewedPEM, err := os.ReadFile(paths.ServerCert) + if err != nil || bytes.Equal(renewedPEM, material.ServerCert) { + t.Fatalf("server certificate was not renewed: err=%v", err) + } +} + func TestRefreshRejectsProviderNetworkWithoutDNSBeforeBuild(t *testing.T) { home := t.TempDir() config := validTestConfig(home) @@ -327,7 +616,8 @@ func TestRefreshBuildsAndPreflightsIsolatedCandidateThenStable(t *testing.T) { payload := writeTestProviderPayload(t, home, "verified-provider-v1") digest := fileDigestForTest(t, payload) paths := LifecyclePathsFor(config) - writeRefreshEnvironmentFiles(t, paths) + now := time.Unix(1_700_000_000, 0).UTC() + writeRefreshEnvironmentFiles(t, paths, now) if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { t.Fatalf("mkdir provider state: %v", err) } @@ -352,7 +642,6 @@ func TestRefreshBuildsAndPreflightsIsolatedCandidateThenStable(t *testing.T) { } return baseRun(ctx, command) } - now := time.Unix(1_700_000_000, 0).UTC() refresher := Refresher{ Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, @@ -375,6 +664,26 @@ func TestRefreshBuildsAndPreflightsIsolatedCandidateThenStable(t *testing.T) { if data, err := os.ReadFile(filepath.Join(paths.ProviderState, "ownership.json")); err != nil || string(data) != `{"owner":"migrated"}` { t.Fatalf("candidate state was not promoted: data=%q err=%v", data, err) } + var buildCommand *Command + for index := range runner.commands { + if runner.commands[index].Path == config.PodmanPath && firstArg(runner.commands[index].Args) == "build" { + buildCommand = &runner.commands[index] + break + } + } + if buildCommand == nil { + t.Fatal("refresh did not build provider image") + } + for _, label := range []string{ + "io.workflow.compute.managed=github-runner-provider", + "io.workflow.compute.worker=" + config.WorkerID, + "io.workflow.compute.role=provider-image", + "io.workflow.compute.digest=" + digest, + } { + if !containsAdjacentArgs(buildCommand.Args, "--label", label) { + t.Fatalf("provider image build missing ownership label %q: %+v", label, buildCommand.Args) + } + } if _, err := os.Stat(paths.CandidateState(digest)); !errors.Is(err, os.ErrNotExist) { t.Fatalf("promoted candidate state remains at staging path: %v", err) } @@ -463,6 +772,13 @@ func TestDeferredRefreshRetainsRollbackStateUntilInstallerFinalizes(t *testing.T if _, err := os.Stat(paths.PreviousState(digest)); err != nil { t.Fatalf("deferred refresh removed rollback state: %v", err) } + staleDigest := "sha256:" + strings.Repeat("d", 64) + if err := os.MkdirAll(paths.PackageDir(staleDigest), 0o700); err != nil { + t.Fatalf("create stale provider package: %v", err) + } + if err := os.WriteFile(paths.PackageBinary(staleDigest), []byte("stale"), 0o700); err != nil { + t.Fatalf("write stale provider package: %v", err) + } if err := refresher.finalizeDeferredRefresh(config); err != nil { t.Fatalf("finalize deferred refresh: %v", err) } @@ -472,6 +788,12 @@ func TestDeferredRefreshRetainsRollbackStateUntilInstallerFinalizes(t *testing.T if _, err := os.Stat(filepath.Join(paths.CandidatesRoot, digestHex(digest))); !errors.Is(err, os.ErrNotExist) { t.Fatalf("finalized rollback state remains: %v", err) } + if _, err := os.Stat(paths.PackageDir(staleDigest)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("finalized superseded package remains: %v", err) + } + if !strings.Contains(commandTranscript(runner.commands), "image rm --ignore "+testProviderImageID) { + t.Fatalf("finalize did not remove superseded image: %s", commandTranscript(runner.commands)) + } } func TestDeferredRefreshBindsOuterLifecycleTransaction(t *testing.T) { @@ -490,7 +812,7 @@ func TestDeferredRefreshBindsOuterLifecycleTransaction(t *testing.T) { digest := fileDigestForTest(t, payload) runner := refreshTestRunner(config, payload, digest) refresher := Refresher{Runner: runner, Sleep: func(context.Context, time.Duration) error { return nil }} - if _, err := refresher.refreshUnderLifecycleTransaction(t.Context(), config, false, true, "install-transaction-123", config.ProfileID, ""); err != nil { + if _, err := refresher.refreshUnderLifecycleTransaction(t.Context(), config, false, true, "install-transaction-123", config.ProfileID, "", ""); err != nil { t.Fatalf("bound deferred refresh: %v", err) } journal, found, err := readTransactionJournal(paths.Journal) @@ -568,13 +890,51 @@ func TestRefreshRejectsIncompleteProviderEnvironment(t *testing.T) { } } +func TestProviderEnvironmentCannotBroadenConfiguredGitHubAuthority(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(string) string + }{ + {name: "repository", mutate: func(value string) string { + return strings.Replace(value, "GITHUB_RUNNER_PROVIDER_REPOSITORIES=GoCodeAlone/workflow-compute", "GITHUB_RUNNER_PROVIDER_REPOSITORIES=GoCodeAlone/workflow-compute,GoCodeAlone/other", 1) + }}, + {name: "organization", mutate: func(value string) string { + return strings.Replace(value, "GITHUB_RUNNER_PROVIDER_ORGANIZATIONS=GoCodeAlone", "GITHUB_RUNNER_PROVIDER_ORGANIZATIONS=GoCodeAlone,OtherOrg", 1) + }}, + {name: "runner group", mutate: func(value string) string { + return strings.Replace(value, "GITHUB_RUNNER_PROVIDER_RUNNER_GROUPS=ephemeral", "GITHUB_RUNNER_PROVIDER_RUNNER_GROUPS=ephemeral,Default", 1) + }}, + {name: "api base url", mutate: func(value string) string { + return value + "GITHUB_API_BASE_URL=https://github.example.invalid/api/v3\n" + }}, + } { + t.Run(tc.name, func(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + environment, err := os.ReadFile(paths.ProviderEnv) + if err != nil { + t.Fatalf("read provider environment: %v", err) + } + if err := os.WriteFile(paths.ProviderEnv, []byte(tc.mutate(string(environment))), 0o600); err != nil { + t.Fatalf("write broadened provider environment: %v", err) + } + if err := validateProviderEnvironment(config, paths.ProviderEnv); err == nil { + t.Fatal("broadened provider environment was accepted") + } + }) + } +} + func TestRefreshFailurePreservesPreviousActiveImageAndCleansCandidate(t *testing.T) { for _, phase := range []string{"build", "stale-candidate", "stable-stop", "candidate", "candidate-probe", "stable-restart", "stable-probe", "canceled"} { t.Run(phase, func(t *testing.T) { home := t.TempDir() config := validTestConfig(home) paths := LifecyclePathsFor(config) - writeRefreshEnvironmentFiles(t, paths) + now := time.Unix(1_700_000_100, 0).UTC() + writeRefreshEnvironmentFiles(t, paths, now) if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { t.Fatalf("mkdir provider state: %v", err) } @@ -588,6 +948,7 @@ func TestRefreshFailurePreservesPreviousActiveImageAndCleansCandidate(t *testing baseRun := runner.run failedRestart := false failedStaleCleanup := false + var rollbackProbeBudget time.Duration refreshContext := t.Context() cancelRefresh := func() {} if phase == "canceled" { @@ -604,13 +965,20 @@ func TestRefreshFailurePreservesPreviousActiveImageAndCleansCandidate(t *testing if phase == "candidate" && isCandidateStart(command, config) { return nil, errors.New("candidate failed") } - if phase == "stale-candidate" && firstArg(command.Args) == "rm" && containsArg(command.Args, config.CandidateContainer) && !failedStaleCleanup { + if phase == "stale-candidate" && firstArg(command.Args) == "ps" && !failedStaleCleanup { failedStaleCleanup = true return nil, errors.New("stale candidate cleanup failed") } if phase == "candidate-probe" && isProbeFor(command, config.CandidateContainer) { return nil, errors.New("candidate probe failed") } + if phase == "candidate-probe" && isProbeFor(command, config.StableContainer) { + deadline, ok := ctx.Deadline() + if !ok { + t.Fatal("rollback stable probe has no deadline") + } + rollbackProbeBudget = time.Until(deadline) + } if phase == "stable-stop" && filepath.Base(command.Path) == "systemctl" && containsAdjacentArgs(command.Args, "stop", providerServiceUnit) && !failedRestart { failedRestart = true return nil, errors.New("stop failed") @@ -625,7 +993,7 @@ func TestRefreshFailurePreservesPreviousActiveImageAndCleansCandidate(t *testing return baseRun(ctx, command) } refresher := Refresher{ - Runner: runner, Now: func() time.Time { return time.Unix(1_700_000_100, 0).UTC() }, + Runner: runner, Now: func() time.Time { return now }, Sleep: func(context.Context, time.Duration) error { return nil }, } if _, err := refresher.Refresh(refreshContext, config); err == nil { @@ -642,105 +1010,349 @@ func TestRefreshFailurePreservesPreviousActiveImageAndCleansCandidate(t *testing t.Fatalf("%s rollback journal remains: %v", phase, err) } transcript := commandTranscript(runner.commands) - if strings.Contains(transcript, "image rm") { - t.Fatalf("%s failure attempted to prune retained image:\n%s", phase, transcript) + if !strings.Contains(transcript, "image rm --ignore "+testProviderImageID) { + t.Fatalf("%s failure did not remove candidate image:\n%s", phase, transcript) + } + if strings.Contains(transcript, "image rm --ignore "+previous.Current.ImageRef) { + t.Fatalf("%s failure removed retained image:\n%s", phase, transcript) } - if phase != "build" && !strings.Contains(transcript, "rm --force --ignore "+config.CandidateContainer) { + if _, err := os.Stat(paths.PackageDir(digest)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("%s failure retained candidate package: %v", phase, err) + } + if phase != "build" && !strings.Contains(transcript, "rm --force --ignore "+testCandidateContainerID) { t.Fatalf("%s failure did not clean candidate:\n%s", phase, transcript) } + if phase == "candidate-probe" && rollbackProbeBudget < providerProbeTimeout-time.Second { + t.Fatalf("rollback stable probe deadline = %s want approximately %s", rollbackProbeBudget, providerProbeTimeout) + } }) } } -func TestStableProbeFailureRestoresPreviousProviderState(t *testing.T) { +func TestRecoverInterruptedStagingRemovesCandidateArtifactsWithoutStoppingActive(t *testing.T) { home := t.TempDir() config := validTestConfig(home) paths := LifecyclePathsFor(config) - writeRefreshEnvironmentFiles(t, paths) - if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { - t.Fatalf("mkdir provider state: %v", err) - } - stateFile := filepath.Join(paths.ProviderState, "state.json") - if err := os.WriteFile(stateFile, []byte(`{"generation":"previous"}`), 0o600); err != nil { - t.Fatalf("write previous provider state: %v", err) - } + now := time.Unix(1_700_000_100, 0).UTC() previous := previousActiveStateForTest(t, home) if err := AtomicWriteJSON(paths.ActiveState, previous); err != nil { t.Fatalf("write previous active state: %v", err) } - payload := writeTestProviderPayload(t, home, "verified-provider-state-rollback") + payload := writeTestProviderPayload(t, home, "verified-provider-staging-crash") digest := fileDigestForTest(t, payload) - runner := refreshTestRunner(config, payload, digest) - baseRun := runner.run - providerStops := 0 - runner.run = func(ctx context.Context, command Command) ([]byte, error) { - if filepath.Base(command.Path) == "systemctl" && containsAdjacentArgs(command.Args, "--user", "stop") && containsArg(command.Args, providerServiceUnit) { - providerStops++ - } - if isCandidateStart(command, config) { - if err := os.WriteFile(filepath.Join(paths.CandidateState(digest), "state.json"), []byte(`{"generation":"candidate"}`), 0o600); err != nil { - t.Fatalf("mutate candidate state: %v", err) - } - } - if isProbeFor(command, config.StableContainer) && containsArg(command.Args, testProviderImageID) { - return nil, errors.New("stable probe failed") + selection := selectionForDigest(payload, digest, "v1.0.32", "directive-staging-crash", "sha256:"+strings.Repeat("d", 64), now) + if err := mkdirAllDurable(paths.PackageDir(digest), 0o700); err != nil { + t.Fatalf("create staged package: %v", err) + } + if err := os.WriteFile(paths.PackageBinary(digest), []byte("candidate"), 0o700); err != nil { + t.Fatalf("write staged package: %v", err) + } + journal := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, + ID: "refresh-staging-crash", + Phase: JournalStaging, + Previous: &previous, + Candidate: ImageSelection{Update: selection.Update}, + StartedAt: now, + UpdatedAt: now, + } + if err := AtomicWriteJSON(paths.Journal, journal); err != nil { + t.Fatalf("write staging journal: %v", err) + } + runner := &recordingCommandRunner{run: func(_ context.Context, command Command) ([]byte, error) { + if command.Path == config.PodmanPath && firstArg(command.Args) == "images" { + return ownedProviderImageInventory(config, digest, testProviderImageID), nil } - return baseRun(ctx, command) + return nil, nil + }} + if err := (Refresher{Runner: runner}).recoverInterrupted(t.Context(), config, paths); err != nil { + t.Fatalf("recover staged update: %v", err) } - refresher := Refresher{Runner: runner, Sleep: func(context.Context, time.Duration) error { return nil }} - if _, err := refresher.Refresh(t.Context(), config); err == nil || !strings.Contains(err.Error(), "stable probe failed") { - t.Fatalf("stable probe failure err = %v", err) + transcript := commandTranscript(runner.commands) + if !strings.Contains(transcript, "image rm --ignore "+testProviderImageID) { + t.Fatalf("staging recovery did not remove image:\n%s", transcript) } - if data, err := os.ReadFile(stateFile); err != nil || string(data) != `{"generation":"previous"}` { - t.Fatalf("rollback state = %q err=%v", data, err) + if strings.Contains(transcript, "systemctl --user stop "+providerServiceUnit) { + t.Fatalf("staging recovery stopped active provider:\n%s", transcript) } - if _, err := os.Stat(filepath.Join(paths.CandidatesRoot, digestHex(digest))); !errors.Is(err, os.ErrNotExist) { - t.Fatalf("state transaction remains after rollback: %v", err) + if _, err := os.Stat(paths.PackageDir(digest)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("staging recovery retained package: %v", err) } - if providerStops != 2 { - t.Fatalf("provider stop count = %d want promotion and rollback stops", providerStops) + if _, err := os.Stat(paths.Journal); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("staging recovery retained journal: %v", err) + } + active, found, err := readActiveStateForConfig(paths.ActiveState, config) + if err != nil || !found || active.Current.ImageID != previous.Current.ImageID { + t.Fatalf("staging recovery changed active state: found=%v state=%+v err=%v", found, active, err) } } -func TestCommitJournalWriteFailureRollsBackLastDurablePhase(t *testing.T) { +func TestCleanupCandidateArtifactsRejectsSymlinkedPackagesRootWithoutTouchingTarget(t *testing.T) { + if os.PathSeparator != '/' { + t.Skip("symlink behavior varies on Windows") + } home := t.TempDir() config := validTestConfig(home) paths := LifecyclePathsFor(config) - writeRefreshEnvironmentFiles(t, paths) - if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { - t.Fatalf("mkdir provider state: %v", err) - } - stateFile := filepath.Join(paths.ProviderState, "state.json") - if err := os.WriteFile(stateFile, []byte(`{"generation":"previous"}`), 0o600); err != nil { - t.Fatalf("write previous provider state: %v", err) + digest := "sha256:" + strings.Repeat("e", 64) + outside := filepath.Join(t.TempDir(), "outside") + sentinel := filepath.Join(outside, digestHex(digest), "sentinel") + if err := os.MkdirAll(filepath.Dir(sentinel), 0o700); err != nil { + t.Fatalf("create outside package: %v", err) + } + if err := os.WriteFile(sentinel, []byte("keep"), 0o600); err != nil { + t.Fatalf("write outside sentinel: %v", err) + } + if err := mkdirAllDurable(filepath.Dir(paths.PackagesRoot), 0o700); err != nil { + t.Fatalf("create install root: %v", err) + } + if err := os.Symlink(outside, paths.PackagesRoot); err != nil { + t.Fatalf("symlink packages root: %v", err) + } + err := (Refresher{Runner: &recordingCommandRunner{}}).cleanupCandidateArtifacts(t.Context(), config, paths, ImageSelection{Update: VerifiedUpdate{SHA256: digest}}, false) + if err == nil || !strings.Contains(err.Error(), "real directory") { + t.Fatalf("symlinked package cleanup err = %v", err) + } + if data, err := os.ReadFile(sentinel); err != nil || string(data) != "keep" { + t.Fatalf("outside package changed: data=%q err=%v", data, err) + } +} + +func TestFailedRefreshRetainsStagingJournalUntilArtifactCleanupSucceeds(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + now := time.Unix(1_700_000_100, 0).UTC() + writeRefreshEnvironmentFiles(t, paths, now) + if err := mkdirAllDurable(paths.ProviderState, 0o700); err != nil { + t.Fatalf("create provider state: %v", err) } previous := previousActiveStateForTest(t, home) if err := AtomicWriteJSON(paths.ActiveState, previous); err != nil { t.Fatalf("write previous active state: %v", err) } - payload := writeTestProviderPayload(t, home, "verified-provider-commit-journal-failure") + payload := writeTestProviderPayload(t, home, "verified-provider-cleanup-retry") digest := fileDigestForTest(t, payload) runner := refreshTestRunner(config, payload, digest) baseRun := runner.run - blockedCommit := false runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if firstArg(command.Args) == "build" { + return nil, errors.New("build failed") + } + if containsAdjacentArgs(command.Args, "image", "rm") { + return nil, errors.New("image cleanup failed") + } + return baseRun(ctx, command) + } + refresher := Refresher{Runner: runner, Now: func() time.Time { return now }} + if _, err := refresher.Refresh(t.Context(), config); err == nil || !strings.Contains(err.Error(), "image cleanup failed") { + t.Fatalf("refresh cleanup failure err = %v", err) + } + journal, found, err := readTransactionJournalForConfig(paths.Journal, config) + if err != nil || !found || journal.Phase != JournalRollbackRestored || journal.RollbackFrom != JournalStaging { + t.Fatalf("retained rollback journal found=%v phase=%s from=%s err=%v", found, journal.Phase, journal.RollbackFrom, err) + } + if _, err := os.Stat(paths.PackageDir(digest)); err != nil { + t.Fatalf("retry package missing before recovery: %v", err) + } + recoveryRunner := &recordingCommandRunner{} + if err := (Refresher{Runner: recoveryRunner}).recoverInterrupted(t.Context(), config, paths); err != nil { + t.Fatalf("retry staged cleanup: %v", err) + } + if _, err := os.Stat(paths.PackageDir(digest)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("retry retained candidate package: %v", err) + } + if _, err := os.Stat(paths.Journal); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("retry retained staging journal: %v", err) + } +} + +func TestCleanupCandidateArtifactsPreservesRetainedRollbackDigest(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + now := time.Unix(1_700_000_100, 0).UTC() + active := previousActiveStateForTest(t, home) + rollback := active.Current + rollback.Update.DirectiveID = "directive-retained-rollback" + rollback.Update.SHA256 = "sha256:" + strings.Repeat("e", 64) + rollback.ImageID = "sha256:" + strings.Repeat("f", 64) + rollback.ImageRef = providerImageRef(rollback.Update.SHA256) + rollback.ActivatedAt = now.Add(-time.Hour) + active.Previous = &rollback + if err := AtomicWriteJSON(paths.ActiveState, active); err != nil { + t.Fatalf("write active rollback set: %v", err) + } + if err := mkdirAllDurable(paths.PackageDir(rollback.Update.SHA256), 0o700); err != nil { + t.Fatalf("create retained rollback package: %v", err) + } + runner := &recordingCommandRunner{} + if err := (Refresher{Runner: runner}).cleanupCandidateArtifacts(t.Context(), config, paths, rollback, false); err != nil { + t.Fatalf("preserve retained rollback artifact: %v", err) + } + if len(runner.commands) != 0 { + t.Fatalf("retained rollback cleanup issued commands: %+v", runner.commands) + } + if _, err := os.Stat(paths.PackageDir(rollback.Update.SHA256)); err != nil { + t.Fatalf("retained rollback package removed: %v", err) + } +} + +func TestGarbageCollectSupersededProviderPackagesAndImagesPreservesRollbackSet(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + digests := []string{ + "sha256:" + strings.Repeat("a", 64), + "sha256:" + strings.Repeat("b", 64), + "sha256:" + strings.Repeat("c", 64), + } + for _, digest := range digests { + if err := os.MkdirAll(paths.PackageDir(digest), 0o700); err != nil { + t.Fatalf("create provider package: %v", err) + } + if err := os.WriteFile(paths.PackageBinary(digest), []byte(digest), 0o700); err != nil { + t.Fatalf("write provider package: %v", err) + } + } + current := validTestSelection(time.Unix(1_700_000_000, 0).UTC()) + current.Update.SHA256 = digests[0] + current.ImageRef = providerImageRef(digests[0]) + previous := current + previous.Update.SHA256 = digests[1] + previous.ImageRef = providerImageRef(digests[1]) + active := ActiveState{ProtocolVersion: ActiveStateProtocolVersion, Current: current, Previous: &previous, UpdatedAt: current.ActivatedAt} + + failRemoval := true + runner := &recordingCommandRunner{run: func(_ context.Context, command Command) ([]byte, error) { + if command.Path == config.PodmanPath && firstArg(command.Args) == "images" { + return ownedProviderImageInventory(config, digests[2], testProviderImageID), nil + } + if command.Path == config.PodmanPath && containsAdjacentArgs(command.Args, "image", "rm") { + if _, err := os.Stat(paths.PackageDir(digests[2])); err != nil { + t.Fatalf("stale package removed before image: %v", err) + } + if failRemoval { + return nil, errors.New("image removal interrupted") + } + } + return nil, nil + }} + refresher := Refresher{Runner: runner} + if err := refresher.garbageCollectSupersededProviders(t.Context(), config, paths, active); err == nil || !strings.Contains(err.Error(), "image removal") { + t.Fatalf("interrupted provider GC err = %v", err) + } + if _, err := os.Stat(paths.PackageDir(digests[2])); err != nil { + t.Fatalf("interrupted provider GC removed package: %v", err) + } + failRemoval = false + if err := refresher.garbageCollectSupersededProviders(t.Context(), config, paths, active); err != nil { + t.Fatalf("retry provider GC: %v", err) + } + for _, digest := range digests[:2] { + if _, err := os.Stat(paths.PackageDir(digest)); err != nil { + t.Fatalf("retained rollback package %s missing: %v", digest, err) + } + } + if _, err := os.Stat(paths.PackageDir(digests[2])); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("stale provider package remains: %v", err) + } + transcript := commandTranscript(runner.commands) + if !strings.Contains(transcript, "image rm --ignore "+testProviderImageID) || strings.Contains(transcript, providerImageRef(digests[0])) || strings.Contains(transcript, providerImageRef(digests[1])) { + t.Fatalf("provider GC touched wrong images:\n%s", transcript) + } +} + +func TestStableProbeFailureRestoresPreviousProviderState(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + stateFile := filepath.Join(paths.ProviderState, "state.json") + if err := os.WriteFile(stateFile, []byte(`{"generation":"previous"}`), 0o600); err != nil { + t.Fatalf("write previous provider state: %v", err) + } + previous := previousActiveStateForTest(t, home) + if err := AtomicWriteJSON(paths.ActiveState, previous); err != nil { + t.Fatalf("write previous active state: %v", err) + } + payload := writeTestProviderPayload(t, home, "verified-provider-state-rollback") + digest := fileDigestForTest(t, payload) + runner := refreshTestRunner(config, payload, digest) + baseRun := runner.run + providerStops := 0 + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if filepath.Base(command.Path) == "systemctl" && containsAdjacentArgs(command.Args, "--user", "stop") && containsArg(command.Args, providerServiceUnit) { + providerStops++ + } if isCandidateStart(command, config) { if err := os.WriteFile(filepath.Join(paths.CandidateState(digest), "state.json"), []byte(`{"generation":"candidate"}`), 0o600); err != nil { t.Fatalf("mutate candidate state: %v", err) } } - if isProbeFor(command, config.StableContainer) && !blockedCommit { - blockedCommit = true - if err := os.Remove(paths.Journal); err != nil { - t.Fatalf("remove journal before commit: %v", err) - } - if err := os.Mkdir(paths.Journal, 0o700); err != nil { - t.Fatalf("block journal commit: %v", err) - } + if isProbeFor(command, config.StableContainer) && containsArg(command.Args, testProviderImageID) { + return nil, errors.New("stable probe failed") } return baseRun(ctx, command) } refresher := Refresher{Runner: runner, Sleep: func(context.Context, time.Duration) error { return nil }} + if _, err := refresher.Refresh(t.Context(), config); err == nil || !strings.Contains(err.Error(), "stable probe failed") { + t.Fatalf("stable probe failure err = %v", err) + } + if data, err := os.ReadFile(stateFile); err != nil || string(data) != `{"generation":"previous"}` { + t.Fatalf("rollback state = %q err=%v", data, err) + } + if _, err := os.Stat(filepath.Join(paths.CandidatesRoot, digestHex(digest))); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("state transaction remains after rollback: %v", err) + } + if providerStops != 2 { + t.Fatalf("provider stop count = %d want promotion and rollback stops", providerStops) + } +} + +func TestCommitJournalWriteFailureRollsBackLastDurablePhase(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + stateFile := filepath.Join(paths.ProviderState, "state.json") + if err := os.WriteFile(stateFile, []byte(`{"generation":"previous"}`), 0o600); err != nil { + t.Fatalf("write previous provider state: %v", err) + } + previous := previousActiveStateForTest(t, home) + if err := AtomicWriteJSON(paths.ActiveState, previous); err != nil { + t.Fatalf("write previous active state: %v", err) + } + payload := writeTestProviderPayload(t, home, "verified-provider-commit-journal-failure") + digest := fileDigestForTest(t, payload) + runner := refreshTestRunner(config, payload, digest) + baseRun := runner.run + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if isCandidateStart(command, config) { + if err := os.WriteFile(filepath.Join(paths.CandidateState(digest), "state.json"), []byte(`{"generation":"candidate"}`), 0o600); err != nil { + t.Fatalf("mutate candidate state: %v", err) + } + } + return baseRun(ctx, command) + } + blockedCommit := false + refresher := Refresher{ + Runner: runner, + Sleep: func(context.Context, time.Duration) error { return nil }, + writeJournalPhaseFn: func(path string, journal *TransactionJournal, phase JournalPhase, updatedAt time.Time) error { + if phase == JournalCommitted && !blockedCommit { + blockedCommit = true + return errors.New("commit journal write failed") + } + return writeJournalPhase(path, journal, phase, updatedAt) + }, + } if _, err := refresher.Refresh(t.Context(), config); err == nil { t.Fatal("refresh with failed commit-journal write succeeded") } @@ -750,7 +1362,7 @@ func TestCommitJournalWriteFailureRollsBackLastDurablePhase(t *testing.T) { } func TestRefreshRecoversEveryInterruptedJournalPhaseIdempotently(t *testing.T) { - for _, phase := range []JournalPhase{JournalPrepared, JournalStatePromoting, JournalStatePromoted, JournalActivated, JournalCommitted} { + for _, phase := range []JournalPhase{JournalPrepared, JournalStatePromoting, JournalStateDetached, JournalStatePromoted, JournalActivated, JournalCommitted} { t.Run(string(phase), func(t *testing.T) { home := t.TempDir() config := validTestConfig(home) @@ -777,6 +1389,10 @@ func TestRefreshRecoversEveryInterruptedJournalPhaseIdempotently(t *testing.T) { if err := os.Rename(paths.ProviderState, paths.PreviousState(candidateDigest)); err != nil { t.Fatalf("simulate partial provider state promotion: %v", err) } + case JournalStateDetached: + if err := detachProviderState(paths, candidateDigest); err != nil { + t.Fatalf("simulate detached provider state: %v", err) + } case JournalStatePromoted, JournalActivated, JournalCommitted: if err := promoteCandidateProviderState(paths, candidateDigest); err != nil { t.Fatalf("simulate provider state promotion: %v", err) @@ -830,6 +1446,57 @@ func TestRefreshRecoversEveryInterruptedJournalPhaseIdempotently(t *testing.T) { } } +func TestProviderStatePromotionPersistsEachCrossDirectoryRename(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + if err := mkdirAllDurable(paths.ProviderState, 0o700); err != nil { + t.Fatalf("create provider state: %v", err) + } + if err := os.WriteFile(filepath.Join(paths.ProviderState, "generation"), []byte("previous"), 0o600); err != nil { + t.Fatalf("write previous generation: %v", err) + } + payload := writeTestProviderPayload(t, home, "candidate-durable-promotion") + digest := fileDigestForTest(t, payload) + candidate := paths.CandidateState(digest) + if err := prepareCandidateState(paths.ProviderState, candidate); err != nil { + t.Fatalf("prepare candidate: %v", err) + } + if err := os.WriteFile(filepath.Join(candidate, "generation"), []byte("candidate"), 0o600); err != nil { + t.Fatalf("write candidate generation: %v", err) + } + + var synced []string + recordSync := func(path string) error { + synced = append(synced, path) + return nil + } + if err := detachProviderStateWithSync(paths, digest, recordSync); err != nil { + t.Fatalf("detach provider state: %v", err) + } + transactionRoot := filepath.Dir(candidate) + if want := []string{paths.Root, transactionRoot}; !reflect.DeepEqual(synced, want) { + t.Fatalf("detach sync order = %v want %v", synced, want) + } + if _, err := os.Stat(paths.ProviderState); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("provider state remains after detach: %v", err) + } + if data, err := os.ReadFile(filepath.Join(paths.PreviousState(digest), "generation")); err != nil || string(data) != "previous" { + t.Fatalf("detached generation = %q err=%v", data, err) + } + + synced = nil + if err := activateCandidateProviderStateWithSync(paths, digest, recordSync); err != nil { + t.Fatalf("activate candidate state: %v", err) + } + if want := []string{transactionRoot, paths.Root}; !reflect.DeepEqual(synced, want) { + t.Fatalf("activation sync order = %v want %v", synced, want) + } + if data, err := os.ReadFile(filepath.Join(paths.ProviderState, "generation")); err != nil || string(data) != "candidate" { + t.Fatalf("active generation = %q err=%v", data, err) + } +} + func TestInterruptedRefreshStopsCandidateBeforeDeletingStagedState(t *testing.T) { home := t.TempDir() config := validTestConfig(home) @@ -857,7 +1524,7 @@ func TestInterruptedRefreshStopsCandidateBeforeDeletingStagedState(t *testing.T) t.Fatalf("write journal: %v", err) } runner := &recordingCommandRunner{run: func(_ context.Context, command Command) ([]byte, error) { - if firstArg(command.Args) == "rm" && containsArg(command.Args, config.CandidateContainer) { + if firstArg(command.Args) == "rm" && containsArg(command.Args, testCandidateContainerID) { if _, err := os.Stat(paths.CandidateState(digest)); err != nil { return nil, errors.New("candidate state deleted before container stop") } @@ -918,11 +1585,165 @@ func TestInterruptedRecoveryStopsStableBeforeProviderStateRestore(t *testing.T) } } +func TestRecoverInterruptedResumesAfterPreviousStateWasAlreadyRestored(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := mkdirAllDurable(paths.ProviderState, 0o700); err != nil { + t.Fatalf("create restored provider state: %v", err) + } + if err := os.WriteFile(filepath.Join(paths.ProviderState, "generation"), []byte("previous"), 0o600); err != nil { + t.Fatalf("write restored provider state: %v", err) + } + previous := previousActiveStateForTest(t, home) + if err := AtomicWriteJSON(paths.ActiveState, previous); err != nil { + t.Fatalf("write restored active state: %v", err) + } + payload := writeTestProviderPayload(t, home, "candidate-already-restored") + digest := fileDigestForTest(t, payload) + now := time.Unix(1_700_000_100, 0).UTC() + candidate := selectionForDigest(payload, digest, "v1.0.32", "directive-already-restored", "sha256:"+strings.Repeat("e", 64), now) + journal := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, + ID: "refresh-already-restored", + Phase: JournalRollbackRestoring, + RollbackFrom: JournalStatePromoted, + Previous: &previous, + Candidate: candidate, + StartedAt: now, + UpdatedAt: now.Add(time.Second), + } + if err := AtomicWriteJSON(paths.Journal, journal); err != nil { + t.Fatalf("write restoring journal: %v", err) + } + refresher := Refresher{Runner: &recordingCommandRunner{}, Sleep: func(context.Context, time.Duration) error { return nil }} + if err := refresher.recoverInterrupted(t.Context(), config, paths); err != nil { + t.Fatalf("resume restored rollback: %v", err) + } + if _, err := os.Stat(paths.Journal); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("completed rollback journal remains: %v", err) + } + if data, err := os.ReadFile(filepath.Join(paths.ProviderState, "generation")); err != nil || string(data) != "previous" { + t.Fatalf("recovered provider state = %q err=%v", data, err) + } +} + +func TestRollbackPersistsRestoredPhaseBeforeArtifactCleanup(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := mkdirAllDurable(paths.ProviderState, 0o700); err != nil { + t.Fatalf("create provider state: %v", err) + } + if err := os.WriteFile(filepath.Join(paths.ProviderState, "generation"), []byte("previous"), 0o600); err != nil { + t.Fatalf("write previous provider state: %v", err) + } + previous := previousActiveStateForTest(t, home) + payload := writeTestProviderPayload(t, home, "candidate-cleanup-replay") + digest := fileDigestForTest(t, payload) + now := time.Unix(1_700_000_100, 0).UTC() + candidate := selectionForDigest(payload, digest, "v1.0.32", "directive-cleanup-replay", "sha256:"+strings.Repeat("e", 64), now) + if err := prepareCandidateState(paths.ProviderState, paths.CandidateState(digest)); err != nil { + t.Fatalf("prepare candidate: %v", err) + } + if err := promoteCandidateProviderState(paths, digest); err != nil { + t.Fatalf("promote candidate: %v", err) + } + if err := AtomicWriteJSON(paths.ActiveState, ActiveState{ProtocolVersion: ActiveStateProtocolVersion, Current: candidate, Previous: &previous.Current, UpdatedAt: now}); err != nil { + t.Fatalf("write candidate active state: %v", err) + } + journal := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, + ID: "refresh-cleanup-replay", + Phase: JournalStatePromoted, + Previous: &previous, + Candidate: candidate, + StartedAt: now, + UpdatedAt: now.Add(time.Second), + } + if err := AtomicWriteJSON(paths.Journal, journal); err != nil { + t.Fatalf("write promoted journal: %v", err) + } + runner := &recordingCommandRunner{run: func(_ context.Context, command Command) ([]byte, error) { + if command.Path == config.PodmanPath && firstArg(command.Args) == "images" { + return ownedProviderImageInventory(config, digest, candidate.ImageID), nil + } + if command.Path == config.PodmanPath && containsAdjacentArgs(command.Args, "image", "rm") { + return nil, errors.New("candidate image cleanup failed") + } + return nil, nil + }} + refresher := Refresher{Runner: runner, Sleep: func(context.Context, time.Duration) error { return nil }} + if err := refresher.recoverInterrupted(t.Context(), config, paths); err == nil || !strings.Contains(err.Error(), "candidate image cleanup failed") { + t.Fatalf("cleanup failure err = %v", err) + } + recovered, found, err := readTransactionJournal(paths.Journal) + if err != nil || !found { + t.Fatalf("read recoverable rollback journal: found=%v err=%v", found, err) + } + if recovered.Phase != JournalRollbackRestored || recovered.RollbackFrom != JournalStatePromoted { + t.Fatalf("rollback journal = phase %s from %s", recovered.Phase, recovered.RollbackFrom) + } +} + +func TestRecoverInterruptedFinishesEveryPersistedRollbackPhase(t *testing.T) { + for _, phase := range []JournalPhase{JournalRollbackRestored, JournalRollbackCleaned} { + t.Run(string(phase), func(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + previous := previousActiveStateForTest(t, home) + if err := AtomicWriteJSON(paths.ActiveState, previous); err != nil { + t.Fatalf("write restored active state: %v", err) + } + payload := writeTestProviderPayload(t, home, "candidate-"+string(phase)) + digest := fileDigestForTest(t, payload) + now := time.Unix(1_700_000_100, 0).UTC() + candidate := selectionForDigest(payload, digest, "v1.0.32", "directive-"+string(phase), "sha256:"+strings.Repeat("e", 64), now) + if phase == JournalRollbackRestored { + if err := stageVerifiedProvider(candidate.Update, paths); err != nil { + t.Fatalf("stage candidate package: %v", err) + } + } + journal := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, + ID: "refresh-" + string(phase), + Phase: phase, + RollbackFrom: JournalStatePromoted, + Previous: &previous, + Candidate: candidate, + StartedAt: now, + UpdatedAt: now.Add(time.Second), + } + if err := AtomicWriteJSON(paths.Journal, journal); err != nil { + t.Fatalf("write rollback journal: %v", err) + } + runner := &recordingCommandRunner{} + if err := (Refresher{Runner: runner}).recoverInterrupted(t.Context(), config, paths); err != nil { + t.Fatalf("recover %s: %v", phase, err) + } + if _, err := os.Stat(paths.Journal); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("completed rollback journal remains: %v", err) + } + _, packageErr := os.Stat(paths.PackageDir(digest)) + if !errors.Is(packageErr, os.ErrNotExist) { + t.Fatalf("rollback retained candidate package: %v", packageErr) + } + if phase == JournalRollbackCleaned && len(runner.commands) != 0 { + t.Fatalf("cleaned rollback issued commands: %+v", runner.commands) + } + }) + } +} + func TestServeActiveValidatesImmutableImageThenExecsRestrictedPodman(t *testing.T) { home := t.TempDir() config := validTestConfig(home) paths := LifecyclePathsFor(config) writeRefreshEnvironmentFiles(t, paths) + writeLifecycleRecoveryFiles(t, config) if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { t.Fatalf("mkdir provider state: %v", err) } @@ -933,8 +1754,16 @@ func TestServeActiveValidatesImmutableImageThenExecsRestrictedPodman(t *testing. execSentinel := errors.New("exec invoked") runner := &recordingCommandRunner{ run: func(_ context.Context, command Command) ([]byte, error) { - if command.Path == config.PodmanPath && len(command.Args) > 1 && command.Args[0] == "image" { - return []byte(strings.TrimPrefix(active.Current.ImageID, "sha256:") + "\n"), nil + if command.Path == config.PodmanPath && firstArg(command.Args) == "images" { + if !containsAdjacentArgs(command.Args, "--filter", "id="+active.Current.ImageID) || containsArg(command.Args, "reference="+active.Current.ImageRef) { + return nil, errors.New("active image was not looked up by immutable id") + } + return []byte(strings.Join([]string{ + active.Current.ImageID, managedProviderValue, config.WorkerID, providerImageRole, active.Current.Update.SHA256, + }, "\t") + "\n"), nil + } + if command.Path == config.PodmanPath && firstArg(command.Args) == "ps" { + return []byte(testStableContainerID + "\t" + config.StableContainer + "\tgithub-runner-provider\t" + config.WorkerID + "\tstable\n"), nil } return nil, nil }, @@ -944,19 +1773,42 @@ func TestServeActiveValidatesImmutableImageThenExecsRestrictedPodman(t *testing. if err := refresher.ServeActive(t.Context(), config); !errors.Is(err, execSentinel) { t.Fatalf("serve active err = %v", err) } - if len(runner.commands) != 2 { + if len(runner.commands) != 4 { t.Fatalf("serve active commands = %+v", runner.commands) } - execCommand := runner.commands[1] + if firstArg(runner.commands[1].Args) != "ps" || !containsAdjacentArgs(runner.commands[2].Args, "--ignore", testStableContainerID) { + t.Fatalf("serve active did not remove the owned stale stable container by immutable ID: %+v", runner.commands) + } + execCommand := runner.commands[3] if execCommand.Path != config.PodmanPath || firstArg(execCommand.Args) != "run" || !containsAdjacentArgs(execCommand.Args, "--name", config.StableContainer) || !containsAdjacentArgs(execCommand.Args, "--env-file", paths.ProviderEnv) { t.Fatalf("serve active exec command = %+v", execCommand) } transcript := commandTranscript(runner.commands) + if strings.Contains(transcript, active.Current.ImageRef) { + t.Fatalf("serve active depended on mutable image ref:\n%s", transcript) + } for _, required := range []string{"--network wfcompute-github-provider", "--read-only", "--cap-drop all", "no-new-privileges", active.Current.ImageID} { if !strings.Contains(transcript, required) { t.Fatalf("serve active transcript missing %q:\n%s", required, transcript) } } + for _, label := range []string{ + "io.workflow.compute.managed=github-runner-provider", + "io.workflow.compute.worker=" + config.WorkerID, + "io.workflow.compute.role=stable", + } { + if !containsAdjacentArgs(execCommand.Args, "--label", label) { + t.Fatalf("serve active command missing ownership label %q: %+v", label, execCommand) + } + } + for _, mount := range []string{ + paths.ProviderState + ":" + providerStateMount + ":rw,Z", + paths.TLSRoot + ":" + providerTLSMount + ":ro,z", + } { + if !containsAdjacentArgs(execCommand.Args, "--volume", mount) { + t.Fatalf("serve active command missing SELinux-safe mount %q: %+v", mount, execCommand) + } + } if strings.Contains(transcript, "provider-secret") || strings.Contains(transcript, "github-secret") || strings.Contains(transcript, "sock") { t.Fatalf("serve active leaked secret or socket mount:\n%s", transcript) } @@ -967,6 +1819,7 @@ func TestServeActiveRefusesImageIdentityMismatch(t *testing.T) { config := validTestConfig(home) paths := LifecyclePathsFor(config) writeRefreshEnvironmentFiles(t, paths) + writeLifecycleRecoveryFiles(t, config) if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { t.Fatalf("mkdir provider state: %v", err) } @@ -975,7 +1828,7 @@ func TestServeActiveRefusesImageIdentityMismatch(t *testing.T) { t.Fatalf("write active state: %v", err) } runner := &recordingCommandRunner{run: func(context.Context, Command) ([]byte, error) { - return []byte("sha256:" + strings.Repeat("f", 64) + "\n"), nil + return ownedProviderImageInventory(config, active.Current.Update.SHA256, "sha256:"+strings.Repeat("f", 64)), nil }} if err := (Refresher{Runner: runner}).ServeActive(t.Context(), config); err == nil || !strings.Contains(err.Error(), "image id") { t.Fatalf("serve active mismatch err = %v", err) @@ -985,6 +1838,68 @@ func TestServeActiveRefusesImageIdentityMismatch(t *testing.T) { } } +func TestServeActiveRejectsCrossWorkerActiveStateBeforePodman(t *testing.T) { + for _, target := range []string{"current", "previous"} { + t.Run(target, func(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + active := previousActiveStateForTest(t, home) + if target == "current" { + active.Current.Update.WorkerID = "other-retained-worker" + } else { + previous := active.Current + previous.Update.SHA256 = "sha256:" + strings.Repeat("e", 64) + previous.ImageID = "sha256:" + strings.Repeat("f", 64) + previous.ImageRef = providerImageRef(previous.Update.SHA256) + previous.Update.WorkerID = "other-retained-worker" + active.Previous = &previous + } + if err := AtomicWriteJSON(paths.ActiveState, active); err != nil { + t.Fatalf("write cross-worker active state: %v", err) + } + runner := &recordingCommandRunner{} + + err := (Refresher{Runner: runner}).ServeActive(t.Context(), config) + if err == nil || !strings.Contains(err.Error(), "identity") { + t.Fatalf("cross-worker active state err = %v", err) + } + if len(runner.commands) != 0 { + t.Fatalf("cross-worker active state reached Podman: %+v", runner.commands) + } + }) + } +} + +func TestRecoverInterruptedRejectsCrossWorkerJournalBeforeCommands(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + selection := validTestSelection(time.Unix(1_700_000_000, 0).UTC()) + selection.Update.WorkerID = "other-retained-worker" + journal := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, + ID: "refresh-cross-worker", Phase: JournalPrepared, + Candidate: selection, StartedAt: selection.ActivatedAt, UpdatedAt: selection.ActivatedAt, + } + if err := AtomicWriteJSON(paths.Journal, journal); err != nil { + t.Fatalf("write cross-worker refresh journal: %v", err) + } + runner := &recordingCommandRunner{} + + err := (Refresher{Runner: runner}).recoverInterrupted(t.Context(), config, paths) + if err == nil || !strings.Contains(err.Error(), "identity") { + t.Fatalf("cross-worker refresh journal err = %v", err) + } + if len(runner.commands) != 0 { + t.Fatalf("cross-worker refresh journal issued commands: %+v", runner.commands) + } +} + func TestRefreshRetriesDetachedProviderProbe(t *testing.T) { home := t.TempDir() config := validTestConfig(home) @@ -1024,6 +1939,283 @@ func TestRefreshRetriesDetachedProviderProbe(t *testing.T) { } } +func TestProviderCommandsCarryRoleSpecificCleanupOwnershipLabels(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + selection := validTestSelection(time.Unix(1_700_000_000, 0).UTC()) + commands := map[string]Command{ + "candidate": candidateProviderCommand(config, paths, paths.CandidateState(selection.Update.SHA256), selection), + "probe": providerProbeCommand(config, paths, config.CandidateContainer, selection), + } + for role, command := range commands { + for _, label := range []string{ + "io.workflow.compute.managed=github-runner-provider", + "io.workflow.compute.worker=" + config.WorkerID, + "io.workflow.compute.role=" + role, + } { + if !containsAdjacentArgs(command.Args, "--label", label) { + t.Fatalf("%s command missing ownership label %q: %v", role, label, command.Args) + } + } + } +} + +func TestRefreshRemovesOwnedStaleProbeBeforeRetry(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + writeRefreshEnvironmentFiles(t, paths) + if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + t.Fatalf("mkdir provider state: %v", err) + } + payload := writeTestProviderPayload(t, home, "verified-provider-probe-cleanup") + digest := fileDigestForTest(t, payload) + runner := refreshTestRunner(config, payload, digest) + baseRun := runner.run + probeAttempts := 0 + staleProbe := false + probeRemoved := false + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if firstArg(command.Args) == "ps" && containsAdjacentArgs(command.Args, "--filter", "name=^"+regexp.QuoteMeta(config.CandidateContainer+"-probe")+"$") { + if staleProbe { + return []byte(testProbeContainerID + "\t" + config.CandidateContainer + "-probe\tgithub-runner-provider\t" + config.WorkerID + "\tprobe\n"), nil + } + return nil, nil + } + if firstArg(command.Args) == "rm" && containsArg(command.Args, testProbeContainerID) { + staleProbe = false + probeRemoved = true + return nil, nil + } + if isProbeFor(command, config.CandidateContainer) { + probeAttempts++ + if probeAttempts == 1 { + staleProbe = true + return nil, errors.New("probe interrupted after container creation") + } + if staleProbe { + return nil, errors.New("probe name already in use") + } + } + return baseRun(ctx, command) + } + refresher := Refresher{ + Runner: runner, + ExecutablePath: func() (string, error) { return payload, nil }, + Sleep: func(context.Context, time.Duration) error { return nil }, + } + if _, err := refresher.Refresh(t.Context(), config); err != nil { + t.Fatalf("refresh with stale probe recovery: %v\n%s", err, commandTranscript(runner.commands)) + } + if probeAttempts != 2 || !probeRemoved { + t.Fatalf("probe attempts=%d removed=%v\n%s", probeAttempts, probeRemoved, commandTranscript(runner.commands)) + } +} + +func TestManagedProbeCleansOwnedOrphanAfterCallerCancellation(t *testing.T) { + config := validTestConfig(t.TempDir()) + paths := LifecyclePathsFor(config) + selection := validTestSelection(time.Unix(1_700_000_000, 0).UTC()) + ctx, cancel := context.WithCancel(t.Context()) + staleProbe := false + probeRemoved := false + var removeBudget time.Duration + runner := &recordingCommandRunner{run: func(commandContext context.Context, command Command) ([]byte, error) { + if err := commandContext.Err(); err != nil { + return nil, err + } + if firstArg(command.Args) == "ps" { + if staleProbe { + time.Sleep(time.Second) + return []byte(testProbeContainerID + "\t" + config.CandidateContainer + "-probe\tgithub-runner-provider\t" + config.WorkerID + "\tprobe\n"), nil + } + return nil, nil + } + if isProbeFor(command, config.CandidateContainer) { + staleProbe = true + cancel() + return nil, context.Canceled + } + if firstArg(command.Args) == "rm" && containsArg(command.Args, testProbeContainerID) { + deadline, ok := commandContext.Deadline() + if !ok { + t.Fatal("detached probe removal has no deadline") + } + removeBudget = time.Until(deadline) + staleProbe = false + probeRemoved = true + } + return nil, nil + }} + refresher := Refresher{Runner: runner, Sleep: func(ctx context.Context, _ time.Duration) error { return ctx.Err() }} + err := refresher.runManagedProbe(ctx, config, config.CandidateContainer, providerProbeCommand(config, paths, config.CandidateContainer, selection)) + if !errors.Is(err, context.Canceled) { + t.Fatalf("canceled probe error = %v", err) + } + if staleProbe || !probeRemoved { + t.Fatalf("canceled probe orphan remained: stale=%v removed=%v\n%s", staleProbe, probeRemoved, commandTranscript(runner.commands)) + } + if removeBudget < controlCommandTimeout-500*time.Millisecond { + t.Fatalf("probe removal budget = %s want a fresh command budget after ownership inspection", removeBudget) + } +} + +func TestRollbackImageCleanupRequiresOwnershipAndImmutableID(t *testing.T) { + const unownedImageID = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + for _, tc := range []struct { + name string + inventoryID string + managed string + wantErr string + wantRemove bool + }{ + {name: "absent"}, + {name: "unowned collision", inventoryID: testProviderImageID, managed: "other", wantErr: "ownership"}, + {name: "journal id mismatch", inventoryID: unownedImageID, managed: "github-runner-provider", wantErr: "image id"}, + {name: "owned", inventoryID: testProviderImageID, managed: "github-runner-provider", wantRemove: true}, + } { + t.Run(tc.name, func(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + payload := writeTestProviderPayload(t, home, "candidate-image-cleanup") + digest := fileDigestForTest(t, payload) + candidate := selectionForDigest(payload, digest, "v1.0.32", "directive-image-cleanup", testProviderImageID, time.Unix(1_700_000_100, 0).UTC()) + journal := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, + ID: "refresh-image-cleanup", + Phase: JournalRollbackRestored, + RollbackFrom: JournalPrepared, + Candidate: candidate, + StartedAt: candidate.ActivatedAt, + UpdatedAt: candidate.ActivatedAt, + } + if err := AtomicWriteJSON(paths.Journal, journal); err != nil { + t.Fatalf("write rollback journal: %v", err) + } + runner := &recordingCommandRunner{run: func(_ context.Context, command Command) ([]byte, error) { + if command.Path == config.PodmanPath && firstArg(command.Args) == "images" && tc.inventoryID != "" { + return []byte(strings.Join([]string{ + tc.inventoryID, tc.managed, config.WorkerID, providerImageRole, digest, + }, "\t") + "\n"), nil + } + return nil, nil + }} + err := (Refresher{Runner: runner}).rollback(t.Context(), config, paths, journal, false) + if tc.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tc.wantErr)) { + t.Fatalf("rollback cleanup error = %v want %q", err, tc.wantErr) + } + if tc.wantErr == "" && err != nil { + t.Fatalf("rollback cleanup: %v", err) + } + transcript := commandTranscript(runner.commands) + removed := strings.Contains(transcript, "image rm --ignore "+candidate.ImageID) + if removed != tc.wantRemove { + t.Fatalf("image cleanup transcript=%q wantRemove=%v", transcript, tc.wantRemove) + } + if strings.Contains(transcript, "image rm --ignore "+candidate.ImageRef) { + t.Fatalf("candidate image removed by mutable ref: %s", transcript) + } + }) + } +} + +func TestProviderImageCleanupUsesImmutableIDOrOwnershipLabelsWithoutTag(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + digest := "sha256:" + strings.Repeat("d", 64) + for _, tc := range []struct { + name string + expectedImageID string + wantFilters []string + }{ + {name: "durable image", expectedImageID: testProviderImageID, wantFilters: []string{"id=" + testProviderImageID}}, + {name: "staging image", wantFilters: []string{ + "label=" + managedObjectLabel + "=" + managedProviderValue, + "label=" + managedWorkerLabel + "=" + config.WorkerID, + "label=" + managedRoleLabel + "=" + providerImageRole, + "label=" + managedDigestLabel + "=" + digest, + }}, + } { + t.Run(tc.name, func(t *testing.T) { + runner := &recordingCommandRunner{run: func(_ context.Context, command Command) ([]byte, error) { + if firstArg(command.Args) != "images" { + return nil, nil + } + for _, filter := range tc.wantFilters { + if !containsAdjacentArgs(command.Args, "--filter", filter) { + return nil, fmt.Errorf("missing image filter %s", filter) + } + } + if strings.Contains(commandTranscript([]Command{command}), "reference=") { + return nil, errors.New("mutable reference image filter") + } + return []byte(strings.Join([]string{ + testProviderImageID, managedProviderValue, config.WorkerID, providerImageRole, digest, + }, "\t") + "\n"), nil + }} + if err := (Refresher{Runner: runner}).removeOwnedProviderImage(t.Context(), config, digest, tc.expectedImageID); err != nil { + t.Fatalf("remove owned provider image: %v", err) + } + if transcript := commandTranscript(runner.commands); !strings.Contains(transcript, "image rm --ignore "+testProviderImageID) { + t.Fatalf("immutable image was not removed:\n%s", transcript) + } + }) + } +} + +func TestRemoveCandidateContainerRequiresExactOwnershipAndUsesImmutableID(t *testing.T) { + containerID := strings.Repeat("a", 64) + for _, tc := range []struct { + name string + inventory string + wantErr string + wantRemove bool + }{ + {name: "absent"}, + {name: "unowned collision", inventory: containerID + "\tworkflow-plugin-github-runner-provider-candidate\tother\tgithub-runner-linux-stg\tcandidate\n", wantErr: "ownership"}, + {name: "owned", inventory: containerID + "\tworkflow-plugin-github-runner-provider-candidate\tgithub-runner-provider\tgithub-runner-linux-stg\tcandidate\n", wantRemove: true}, + } { + t.Run(tc.name, func(t *testing.T) { + config := validTestConfig(t.TempDir()) + runner := &recordingCommandRunner{run: func(_ context.Context, command Command) ([]byte, error) { + if firstArg(command.Args) == "ps" { + return []byte(tc.inventory), nil + } + return nil, nil + }} + err := (Refresher{Runner: runner}).removeManagedContainer(t.Context(), config, config.CandidateContainer, candidateContainerRole) + if tc.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tc.wantErr)) { + t.Fatalf("remove error = %v want %q", err, tc.wantErr) + } + if tc.wantErr == "" && err != nil { + t.Fatalf("remove candidate: %v", err) + } + transcript := commandTranscript(runner.commands) + removed := strings.Contains(transcript, "rm --force --ignore "+containerID) + if removed != tc.wantRemove { + t.Fatalf("remove transcript = %q wantRemove=%v", transcript, tc.wantRemove) + } + if strings.Contains(transcript, "rm --force --ignore "+config.CandidateContainer) { + t.Fatalf("candidate removed by mutable name: %s", transcript) + } + }) + } +} + +func TestRemoveCandidateContainerQuotesConfiguredNameFilter(t *testing.T) { + config := validTestConfig(t.TempDir()) + config.CandidateContainer = "candidate.v1" + runner := &recordingCommandRunner{} + if err := (Refresher{Runner: runner}).removeManagedContainer(t.Context(), config, config.CandidateContainer, candidateContainerRole); err != nil { + t.Fatalf("inspect absent candidate: %v", err) + } + if len(runner.commands) != 1 || !containsAdjacentArgs(runner.commands[0].Args, "--filter", `name=^candidate\.v1$`) { + t.Fatalf("candidate inventory filter is not exact: %+v", runner.commands) + } +} + func TestRefreshLockRejectsConcurrentMutationBeforeVerification(t *testing.T) { home := t.TempDir() config := validTestConfig(home) @@ -1277,8 +2469,8 @@ func TestOSCommandRunnerDoesNotEchoArgumentsOrOutputOnFailure(t *testing.T) { secret := "credential-that-must-not-leak" runner := OSCommandRunner{MaxOutputBytes: 1024} _, err := runner.Run(t.Context(), Command{ - Path: "/usr/bin/false", - Args: []string{secret}, + Path: os.Args[0], + Args: []string{"-test.not-a-real-flag=" + secret}, }) if err == nil { t.Fatal("failing command succeeded") @@ -1291,8 +2483,12 @@ func TestOSCommandRunnerDoesNotEchoArgumentsOrOutputOnFailure(t *testing.T) { func TestOSCommandRunnerDoesNotInheritUnrelatedHostSecrets(t *testing.T) { const secret = "aws-host-secret-that-must-not-leak" t.Setenv("AWS_SECRET_ACCESS_KEY", secret) + t.Setenv("LC_ALL", "workflow-provider-environment-helper") runner := OSCommandRunner{MaxOutputBytes: 1 << 20} - output, err := runner.Run(t.Context(), Command{Path: "/usr/bin/env"}) + output, err := runner.Run(t.Context(), Command{ + Path: os.Args[0], + Args: []string{"-test.run=^TestOSCommandRunnerEnvironmentHelper$"}, + }) if err != nil { t.Fatalf("run env: %v", err) } @@ -1301,7 +2497,23 @@ func TestOSCommandRunnerDoesNotInheritUnrelatedHostSecrets(t *testing.T) { } } -const testProviderImageID = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +func TestOSCommandRunnerEnvironmentHelper(t *testing.T) { + if os.Getenv("LC_ALL") != "workflow-provider-environment-helper" { + return + } + for _, entry := range os.Environ() { + if _, err := fmt.Fprintln(os.Stdout, entry); err != nil { + t.Fatalf("write environment fixture: %v", err) + } + } +} + +const ( + testProviderImageID = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + testCandidateContainerID = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + testStableContainerID = "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" + testProbeContainerID = "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" +) func refreshTestRunner(config Config, payload, digest string) *recordingCommandRunner { for _, file := range []struct { @@ -1311,6 +2523,9 @@ func refreshTestRunner(config Config, payload, digest string) *recordingCommandR }{ {path: config.ComputeAgentPath, mode: 0o700, data: "compute-agent fixture"}, {path: config.SupervisorConfigPath, mode: 0o600, data: "supervisor config fixture"}, + {path: config.PodmanPath, mode: 0o500, data: "podman fixture"}, + {path: config.SystemctlPath, mode: 0o500, data: "systemctl fixture"}, + {path: config.LoginctlPath, mode: 0o500, data: "loginctl fixture"}, {path: agentUnitFragmentPathForTest(config), mode: 0o600, data: "[Service]\nExecStart=" + config.ComputeAgentPath + " run\n"}, } { if err := os.MkdirAll(filepath.Dir(file.path), 0o700); err != nil { @@ -1321,6 +2536,8 @@ func refreshTestRunner(config Config, payload, digest string) *recordingCommandR } } maintenanceActive := false + maintenanceID := "" + maintenanceReason := "" return &recordingCommandRunner{run: func(ctx context.Context, command Command) ([]byte, error) { if err := ctx.Err(); err != nil { return nil, err @@ -1330,10 +2547,14 @@ func refreshTestRunner(config Config, payload, digest string) *recordingCommandR return agentUnitSystemdOutput(config) case "maintenance-begin": maintenanceActive = true - return maintenanceStateJSON(true, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + maintenanceID = adjacentArgValue(command.Args, "-id") + maintenanceReason = adjacentArgValue(command.Args, "-reason") + return maintenanceStateJSON(true, maintenanceID, config.ProfileID, maintenanceReason), nil case "maintenance-end": maintenanceActive = false - return maintenanceStateJSON(false, refreshMaintenanceID, config.ProfileID, refreshMaintenanceReason), nil + return maintenanceStateJSON(false, maintenanceID, config.ProfileID, maintenanceReason), nil + case "maintenance-status": + return maintenanceStateJSON(maintenanceActive, maintenanceID, config.ProfileID, maintenanceReason), nil case "local-status": state := "idle" if maintenanceActive { @@ -1344,6 +2565,32 @@ func refreshTestRunner(config Config, payload, digest string) *recordingCommandR switch { case command.Path == config.ComputeAgentPath: return testVerifiedUpdateJSON(config, payload, digest), nil + case command.Path == config.PodmanPath && firstArg(command.Args) == "ps": + var id, name, role string + switch adjacentArgValue(command.Args, "--filter") { + case "name=^" + regexp.QuoteMeta(config.CandidateContainer) + "$": + id, name, role = testCandidateContainerID, config.CandidateContainer, candidateContainerRole + case "name=^" + regexp.QuoteMeta(config.StableContainer) + "$": + id, name, role = testStableContainerID, config.StableContainer, stableContainerRole + case "name=^" + regexp.QuoteMeta(config.CandidateContainer+"-probe") + "$": + id, name, role = testProbeContainerID, config.CandidateContainer+"-probe", probeContainerRole + case "name=^" + regexp.QuoteMeta(config.StableContainer+"-probe") + "$": + id, name, role = testProbeContainerID, config.StableContainer+"-probe", probeContainerRole + default: + return nil, nil + } + return []byte(id + "\t" + name + "\t" + managedProviderValue + "\t" + config.WorkerID + "\t" + role + "\n"), nil + case command.Path == config.PodmanPath && firstArg(command.Args) == "images": + imageDigest := digest + for index := 0; index+1 < len(command.Args); index++ { + if command.Args[index] == "--filter" { + const prefix = "label=" + managedDigestLabel + "=" + if strings.HasPrefix(command.Args[index+1], prefix) { + imageDigest = strings.TrimPrefix(command.Args[index+1], prefix) + } + } + } + return ownedProviderImageInventory(config, imageDigest, testProviderImageID), nil case command.Path == config.PodmanPath && len(command.Args) >= 2 && command.Args[0] == "image" && command.Args[1] == "inspect": return []byte(testProviderImageID + "\n"), nil case command.Path == config.PodmanPath && len(command.Args) >= 2 && command.Args[0] == "network" && command.Args[1] == "inspect": @@ -1356,6 +2603,12 @@ func refreshTestRunner(config Config, payload, digest string) *recordingCommandR }} } +func ownedProviderImageInventory(config Config, digest, imageID string) []byte { + return []byte(strings.Join([]string{ + imageID, managedProviderValue, config.WorkerID, providerImageRole, digest, + }, "\t") + "\n") +} + func assertRefreshCommandIsolation(t *testing.T, commands []Command, config Config, paths LifecyclePaths) { t.Helper() transcript := commandTranscript(commands) @@ -1366,6 +2619,9 @@ func assertRefreshCommandIsolation(t *testing.T, commands []Command, config Conf "--env-file " + paths.ProviderEnv, "--env-file " + paths.ProbeEnv, "probe -url https://" + config.CandidateContainer + ":18090", "probe -url " + config.ProviderURL, "systemctl --user restart", + ":" + providerStateMount + ":rw,Z", + paths.TLSRoot + ":" + providerTLSMount + ":ro,z", + paths.CAFile + ":" + providerCAPath + ":ro,z", } { if !strings.Contains(transcript, required) { t.Fatalf("command transcript missing %q:\n%s", required, transcript) @@ -1392,8 +2648,15 @@ func assertRefreshCommandIsolation(t *testing.T, commands []Command, config Conf } } -func writeRefreshEnvironmentFiles(t *testing.T, paths LifecyclePaths) { +func writeRefreshEnvironmentFiles(t *testing.T, paths LifecyclePaths, observedAt ...time.Time) { t.Helper() + certificateTime := time.Now().UTC() + if len(observedAt) > 1 { + t.Fatalf("write refresh environment files accepts at most one observation time") + } + if len(observedAt) == 1 { + certificateTime = observedAt[0].UTC() + } if err := os.MkdirAll(filepath.Dir(paths.ProviderEnv), 0o700); err != nil { t.Fatalf("mkdir env dir: %v", err) } @@ -1417,8 +2680,20 @@ func writeRefreshEnvironmentFiles(t *testing.T, paths LifecyclePaths) { if err := os.MkdirAll(paths.TLSRoot, 0o700); err != nil { t.Fatalf("mkdir tls root: %v", err) } - if err := os.WriteFile(paths.CAFile, []byte("test-ca"), 0o600); err != nil { - t.Fatalf("write ca: %v", err) + config := validTestConfig(lifecycleHome(paths)) + material, err := GenerateInstallMaterial(config, Credentials{ + GitHubToken: "github-secret", ProviderToken: "provider-secret", + }, nil, certificateTime.Add(-time.Hour)) + if err != nil { + t.Fatalf("generate refresh TLS material: %v", err) + } + for path, data := range map[string][]byte{ + paths.CAFile: material.CACert, paths.CAKey: material.CAKey, + paths.ServerCert: material.ServerCert, paths.ServerKey: material.ServerKey, + } { + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("write refresh TLS material: %v", err) + } } } diff --git a/internal/retainedprovider/state.go b/internal/retainedprovider/state.go index 6debeef..438ef47 100644 --- a/internal/retainedprovider/state.go +++ b/internal/retainedprovider/state.go @@ -62,6 +62,13 @@ func (update VerifiedUpdate) Validate() error { return nil } +func (update VerifiedUpdate) validateConfigIdentity(config Config) error { + if update.WorkerID != config.WorkerID || update.PluginID != config.PluginID || update.ComponentID != config.ComponentID { + return fmt.Errorf("provider update identity does not match retained config") + } + return nil +} + type ImageSelection struct { Update VerifiedUpdate `json:"update"` ImageID string `json:"image_id"` @@ -114,21 +121,43 @@ func (state ActiveState) Validate() error { return nil } +func (state ActiveState) ValidateForConfig(config Config) error { + if err := state.Validate(); err != nil { + return err + } + if err := state.Current.Update.validateConfigIdentity(config); err != nil { + return fmt.Errorf("current: %w", err) + } + if state.Previous != nil { + if err := state.Previous.Update.validateConfigIdentity(config); err != nil { + return fmt.Errorf("previous: %w", err) + } + } + return nil +} + type JournalPhase string const ( - JournalPrepared JournalPhase = "prepared" - JournalStatePromoting JournalPhase = "state_promoting" - JournalStatePromoted JournalPhase = "state_promoted" - JournalActivated JournalPhase = "activated" - JournalCommitted JournalPhase = "committed" + JournalStaging JournalPhase = "staging" + JournalPrepared JournalPhase = "prepared" + JournalStatePromoting JournalPhase = "state_promoting" + JournalStateDetached JournalPhase = "state_detached" + JournalStatePromoted JournalPhase = "state_promoted" + JournalActivated JournalPhase = "activated" + JournalCommitted JournalPhase = "committed" + JournalRollbackRestoring JournalPhase = "rollback_restoring" + JournalRollbackRestored JournalPhase = "rollback_restored" + JournalRollbackCleaned JournalPhase = "rollback_cleaned" ) type TransactionJournal struct { ProtocolVersion string `json:"protocol_version"` ID string `json:"id"` Phase JournalPhase `json:"phase"` + RollbackFrom JournalPhase `json:"rollback_from,omitempty"` DeferredCommit bool `json:"deferred_commit,omitempty"` + RuntimeRepair bool `json:"runtime_repair,omitempty"` OuterTransactionID string `json:"outer_transaction_id,omitempty"` ProfileID string `json:"profile_id,omitempty"` Previous *ActiveState `json:"previous,omitempty"` @@ -148,21 +177,52 @@ func (journal TransactionJournal) Validate() error { if bound && (!journal.DeferredCommit || !safeIdentifierPattern.MatchString(journal.OuterTransactionID) || !safeIdentifierPattern.MatchString(journal.ProfileID)) { return fmt.Errorf("outer transaction binding is invalid") } + rollbackPhase := isRollbackPhase(journal.Phase) switch journal.Phase { - case JournalPrepared, JournalStatePromoting, JournalStatePromoted, JournalActivated, JournalCommitted: + case JournalStaging, JournalPrepared, JournalStatePromoting, JournalStateDetached, JournalStatePromoted, JournalActivated, JournalCommitted, + JournalRollbackRestoring, JournalRollbackRestored, JournalRollbackCleaned: default: return fmt.Errorf("phase is invalid") } + if rollbackPhase { + if !isRollbackOrigin(journal.RollbackFrom) { + return fmt.Errorf("rollback_from must identify a non-terminal forward phase") + } + } else if journal.RollbackFrom != "" { + return fmt.Errorf("forward phase must not contain rollback_from") + } + effectivePhase := journal.Phase + if rollbackPhase { + effectivePhase = journal.RollbackFrom + } if journal.Previous != nil { if err := journal.Previous.Validate(); err != nil { return fmt.Errorf("previous: %w", err) } } - if err := journal.Candidate.Validate(); err != nil { + if effectivePhase == JournalStaging { + if err := journal.Candidate.Update.Validate(); err != nil { + return fmt.Errorf("candidate update: %w", err) + } + if journal.Candidate.ImageID != "" || journal.Candidate.ImageRef != "" || !journal.Candidate.ActivatedAt.IsZero() { + return fmt.Errorf("staging candidate must not contain image activation state") + } + } else if err := journal.Candidate.Validate(); err != nil { return fmt.Errorf("candidate: %w", err) } - if journal.Previous != nil && (journal.Candidate.ImageID == journal.Previous.Current.ImageID || journal.Candidate.ImageRef == journal.Previous.Current.ImageRef) { - return fmt.Errorf("candidate image must differ from the active image") + if journal.Previous != nil { + if journal.RuntimeRepair { + if journal.Candidate.Update.SHA256 != journal.Previous.Current.Update.SHA256 { + return fmt.Errorf("runtime repair candidate must match the active update") + } + } else if journal.Candidate.Update.SHA256 == journal.Previous.Current.Update.SHA256 { + return fmt.Errorf("candidate update must differ from the active update") + } + if !journal.RuntimeRepair && effectivePhase != JournalStaging && (journal.Candidate.ImageID == journal.Previous.Current.ImageID || journal.Candidate.ImageRef == journal.Previous.Current.ImageRef) { + return fmt.Errorf("candidate image must differ from the active image") + } + } else if journal.RuntimeRepair { + return fmt.Errorf("runtime repair requires previous active state") } if journal.StartedAt.IsZero() || journal.UpdatedAt.IsZero() || journal.UpdatedAt.Before(journal.StartedAt) { return fmt.Errorf("journal timestamps are invalid") @@ -170,6 +230,42 @@ func (journal TransactionJournal) Validate() error { return nil } +func isRollbackPhase(phase JournalPhase) bool { + switch phase { + case JournalRollbackRestoring, JournalRollbackRestored, JournalRollbackCleaned: + return true + default: + return false + } +} + +func isRollbackOrigin(phase JournalPhase) bool { + switch phase { + case JournalStaging, JournalPrepared, JournalStatePromoting, JournalStateDetached, JournalStatePromoted, JournalActivated: + return true + default: + return false + } +} + +func (journal TransactionJournal) ValidateForConfig(config Config) error { + if err := journal.Validate(); err != nil { + return err + } + if err := journal.Candidate.Update.validateConfigIdentity(config); err != nil { + return fmt.Errorf("candidate: %w", err) + } + if journal.Previous != nil { + if err := journal.Previous.ValidateForConfig(config); err != nil { + return fmt.Errorf("previous: %w", err) + } + } + if journal.ProfileID != "" && journal.ProfileID != config.ProfileID { + return fmt.Errorf("profile identity does not match retained config") + } + return nil +} + func RecoverActiveState(journal TransactionJournal) (ActiveState, error) { if err := journal.Validate(); err != nil { return ActiveState{}, err @@ -185,7 +281,10 @@ func RecoverActiveState(journal TransactionJournal) (ActiveState, error) { Current: journal.Candidate, UpdatedAt: journal.UpdatedAt, } - if journal.Previous != nil { + if journal.RuntimeRepair && journal.Previous != nil && journal.Previous.Previous != nil { + previous := *journal.Previous.Previous + recovered.Previous = &previous + } else if !journal.RuntimeRepair && journal.Previous != nil { previous := journal.Previous.Current recovered.Previous = &previous } diff --git a/internal/retainedprovider/state_test.go b/internal/retainedprovider/state_test.go index eb69ee7..8baa532 100644 --- a/internal/retainedprovider/state_test.go +++ b/internal/retainedprovider/state_test.go @@ -3,6 +3,8 @@ package retainedprovider import ( "bytes" "encoding/json" + "fmt" + "os" "path/filepath" "strings" "testing" @@ -73,6 +75,45 @@ func TestConfigDecodeAndValidation(t *testing.T) { } } +func TestConfigDecodesExplicitHostToolPaths(t *testing.T) { + home := t.TempDir() + fields := map[string]json.RawMessage{} + encoded, err := json.Marshal(validTestConfig(home)) + if err != nil { + t.Fatalf("marshal config: %v", err) + } + if err := json.Unmarshal(encoded, &fields); err != nil { + t.Fatalf("decode config fields: %v", err) + } + fields["systemctl_path"], _ = json.Marshal("/usr/bin/systemctl") + fields["loginctl_path"], _ = json.Marshal("/usr/bin/loginctl") + encoded, err = json.Marshal(fields) + if err != nil { + t.Fatalf("marshal explicit tool config: %v", err) + } + config, err := DecodeConfig(bytes.NewReader(encoded), home) + if err != nil { + t.Fatalf("decode explicit host tools: %v", err) + } + if config.SystemctlPath != "/usr/bin/systemctl" || config.LoginctlPath != "/usr/bin/loginctl" { + t.Fatalf("decoded host tools = systemctl:%q loginctl:%q", config.SystemctlPath, config.LoginctlPath) + } +} + +func TestLifecycleAuditPathDoesNotDependOnAmbientStateHome(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + t.Setenv("XDG_STATE_HOME", filepath.Join(home, "interactive-state")) + interactive := LifecyclePathsFor(config) + t.Setenv("XDG_STATE_HOME", filepath.Join(home, "systemd-state")) + systemd := LifecyclePathsFor(config) + + want := filepath.Join(home, ".local", "state", "wfctl", "plugins", GitHubPluginID, "retained-provider-audit.jsonl") + if interactive.LifecycleAudit != want || systemd.LifecycleAudit != want { + t.Fatalf("audit path changed across environments: interactive=%q systemd=%q want=%q", interactive.LifecycleAudit, systemd.LifecycleAudit, want) + } +} + func TestConfigRejectsUnsafeIdentityAndPaths(t *testing.T) { home := t.TempDir() for _, tc := range []struct { @@ -91,10 +132,26 @@ func TestConfigRejectsUnsafeIdentityAndPaths(t *testing.T) { {name: "systemd directory as install root", mutate: func(c *Config) { c.InstallRoot = c.SystemdDir }, want: "dedicated provider root"}, {name: "arbitrary provider root", mutate: func(c *Config) { c.InstallRoot = filepath.Join(home, "provider") }, want: "dedicated provider root"}, {name: "outside home", mutate: func(c *Config) { c.SystemdDir = filepath.Join(filepath.Dir(home), "outside") }, want: "systemd_dir"}, + {name: "authority aliases managed file", mutate: func(c *Config) { c.SupervisorConfigPath = LifecyclePathsFor(*c).AgentDropIn }, want: "managed provider path"}, + {name: "authority paths alias", mutate: func(c *Config) { c.LocalStatusPath = c.SupervisorConfigPath }, want: "distinct"}, + {name: "agent inside install root", mutate: func(c *Config) { c.ComputeAgentPath = filepath.Join(c.InstallRoot, "compute-agent") }, want: "outside install_root"}, + {name: "systemd inside install root", mutate: func(c *Config) { c.SystemdDir = filepath.Join(c.InstallRoot, "systemd") }, want: "outside install_root"}, + {name: "podman wrapper", mutate: func(c *Config) { c.PodmanPath = filepath.Join(home, "podman-wrapper") }, want: "podman_path"}, + {name: "systemctl wrapper", mutate: func(c *Config) { c.SystemctlPath = filepath.Join(home, "systemctl-wrapper") }, want: "systemctl_path"}, + {name: "loginctl wrapper", mutate: func(c *Config) { c.LoginctlPath = filepath.Join(home, "loginctl-wrapper") }, want: "loginctl_path"}, {name: "plaintext provider URL", mutate: func(c *Config) { c.ProviderURL = "http://provider:18090" }, want: "provider_url"}, {name: "wrong provider host", mutate: func(c *Config) { c.ProviderURL = "https://host.containers.internal:18090" }, want: "provider_url"}, {name: "wrong provider port", mutate: func(c *Config) { c.ProviderURL = "https://" + c.StableContainer + ":18091" }, want: "provider_url"}, {name: "default bridge network", mutate: func(c *Config) { c.ContainerNetwork = "bridge" }, want: "container_network"}, + {name: "non-yaml workflow", mutate: func(c *Config) { c.Workflow = "dogfood-provider-target" }, want: "workflow"}, + {name: "long runner name", mutate: func(c *Config) { c.RunnerName = strings.Repeat("r", 101) }, want: "runner_name"}, + {name: "long label", mutate: func(c *Config) { c.Labels = []string{strings.Repeat("l", 101)} }, want: "labels"}, + {name: "too many labels", mutate: func(c *Config) { + c.Labels = make([]string, 65) + for index := range c.Labels { + c.Labels[index] = fmt.Sprintf("label-%d", index) + } + }, want: "labels"}, {name: "short ref", mutate: func(c *Config) { c.Ref = "main" }, want: "ref"}, {name: "fast timer", mutate: func(c *Config) { c.RefreshIntervalSeconds = 10 }, want: "refresh_interval_seconds"}, } { @@ -108,6 +165,69 @@ func TestConfigRejectsUnsafeIdentityAndPaths(t *testing.T) { } } +func TestConfigRejectsManagedContainerNameCollisions(t *testing.T) { + home := t.TempDir() + for _, tc := range []struct { + name string + mutate func(*Config) + }{ + { + name: "candidate aliases stable probe", + mutate: func(config *Config) { + config.CandidateContainer = config.StableContainer + "-probe" + }, + }, + { + name: "stable aliases candidate probe", + mutate: func(config *Config) { + config.StableContainer = config.CandidateContainer + "-probe" + config.ProviderURL = "https://" + config.StableContainer + ":18090" + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + config := validTestConfig(home) + tc.mutate(&config) + if err := config.Validate(home); err == nil || !strings.Contains(err.Error(), "container names") { + t.Fatalf("Validate err = %v want managed container name collision", err) + } + }) + } +} + +func TestConfigRejectsAuthorityOverlapWithLifecycleState(t *testing.T) { + home := t.TempDir() + base := validTestConfig(home) + paths := LifecyclePathsFor(base) + for _, tc := range []struct { + name string + mutate func(*Config) + want string + }{ + {name: "exact lifecycle journal", mutate: func(c *Config) { c.ProviderMarkerPath = paths.LifecycleJournal }}, + {name: "lifecycle journal parent", mutate: func(c *Config) { c.SupervisorConfigPath = filepath.Dir(paths.LifecycleJournal) }}, + {name: "inside lifecycle transactions", mutate: func(c *Config) { c.LocalStatusPath = filepath.Join(paths.LifecycleTransactions, "foreign-status.json") }}, + {name: "exact audit", mutate: func(c *Config) { c.ProviderMarkerPath = paths.LifecycleAudit }}, + {name: "authority paths nested", mutate: func(c *Config) { c.LocalStatusPath = filepath.Join(c.SupervisorConfigPath, "status.json") }}, + {name: "systemd contains lifecycle root", mutate: func(c *Config) { c.SystemdDir = filepath.Dir(paths.LifecycleJournal) }}, + {name: "agent contains install root", mutate: func(c *Config) { c.ComputeAgentPath = filepath.Dir(c.InstallRoot) }}, + {name: "podman inside install root", mutate: func(c *Config) { c.PodmanPath = filepath.Join(c.InstallRoot, "bin", "podman") }, want: "outside install_root"}, + {name: "systemctl contains compute agent", mutate: func(c *Config) { c.ComputeAgentPath = filepath.Join(c.SystemctlPath, "compute-agent") }}, + } { + t.Run(tc.name, func(t *testing.T) { + config := base + tc.mutate(&config) + want := tc.want + if want == "" { + want = "overlap" + } + if err := config.Validate(home); err == nil || !strings.Contains(err.Error(), want) { + t.Fatalf("Validate err = %v want %q", err, want) + } + }) + } +} + func TestActiveStateAndVerifiedUpdateValidation(t *testing.T) { now := time.Now().UTC() selection := validTestSelection(now) @@ -178,19 +298,25 @@ func TestRecoverySelectionForEveryJournalPhase(t *testing.T) { phase JournalPhase want ImageSelection }{ + {phase: JournalStaging, want: previous.Current}, {phase: JournalPrepared, want: previous.Current}, {phase: JournalStatePromoting, want: previous.Current}, + {phase: JournalPhase("state_detached"), want: previous.Current}, {phase: JournalStatePromoted, want: previous.Current}, {phase: JournalActivated, want: previous.Current}, {phase: JournalCommitted, want: candidate}, } { t.Run(string(tc.phase), func(t *testing.T) { + journalCandidate := candidate + if tc.phase == JournalStaging { + journalCandidate = ImageSelection{Update: candidate.Update} + } journal := TransactionJournal{ ProtocolVersion: TransactionJournalProtocolVersion, ID: "txn-1", Phase: tc.phase, Previous: &previous, - Candidate: candidate, + Candidate: journalCandidate, StartedAt: now, UpdatedAt: now, } @@ -208,6 +334,103 @@ func TestRecoverySelectionForEveryJournalPhase(t *testing.T) { } } +func TestRollbackJournalRequiresValidForwardOrigin(t *testing.T) { + now := time.Now().UTC() + base := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, + ID: "txn-rollback", + Phase: JournalActivated, + Candidate: validTestSelection(now), + StartedAt: now, + UpdatedAt: now, + } + encoded, err := json.Marshal(base) + if err != nil { + t.Fatalf("marshal base journal: %v", err) + } + var object map[string]any + if err := json.Unmarshal(encoded, &object); err != nil { + t.Fatalf("decode base journal: %v", err) + } + + for _, tc := range []struct { + name string + phase string + rollbackFrom any + wantValid bool + }{ + {name: "restoring", phase: "rollback_restoring", rollbackFrom: "activated", wantValid: true}, + {name: "restored", phase: "rollback_restored", rollbackFrom: "state_promoted", wantValid: true}, + {name: "cleaned", phase: "rollback_cleaned", rollbackFrom: "prepared", wantValid: true}, + {name: "missing origin", phase: "rollback_restoring", rollbackFrom: nil}, + {name: "terminal origin", phase: "rollback_restoring", rollbackFrom: "committed"}, + {name: "rollback origin", phase: "rollback_restoring", rollbackFrom: "rollback_restored"}, + {name: "origin on forward phase", phase: "prepared", rollbackFrom: "staging"}, + } { + t.Run(tc.name, func(t *testing.T) { + candidate := make(map[string]any, len(object)+1) + for key, value := range object { + candidate[key] = value + } + candidate["phase"] = tc.phase + if tc.rollbackFrom == nil { + delete(candidate, "rollback_from") + } else { + candidate["rollback_from"] = tc.rollbackFrom + } + data, err := json.Marshal(candidate) + if err != nil { + t.Fatalf("marshal candidate journal: %v", err) + } + path := filepath.Join(t.TempDir(), "journal.json") + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("write candidate journal: %v", err) + } + var journal TransactionJournal + err = ReadStrictJSONFile(path, &journal) + if err == nil { + err = journal.Validate() + } + if tc.wantValid && err != nil { + t.Fatalf("valid rollback journal: %v", err) + } + if !tc.wantValid && err == nil { + t.Fatal("invalid rollback journal was accepted") + } + }) + } +} + +func TestStagingJournalContainsOnlyDistinctVerifiedUpdateProvenance(t *testing.T) { + now := time.Now().UTC() + previous := ActiveState{ProtocolVersion: ActiveStateProtocolVersion, Current: validTestSelection(now.Add(-time.Hour)), UpdatedAt: now.Add(-time.Hour)} + update := validTestSelection(now).Update + update.SHA256 = "sha256:" + strings.Repeat("d", 64) + update.DirectiveID = "directive-staging" + journal := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, + ID: "txn-staging", + Phase: JournalStaging, + Previous: &previous, + Candidate: ImageSelection{Update: update}, + StartedAt: now, + UpdatedAt: now, + } + if err := journal.Validate(); err != nil { + t.Fatalf("valid staging journal: %v", err) + } + withImage := journal + withImage.Candidate.ImageID = "sha256:" + strings.Repeat("c", 64) + if err := withImage.Validate(); err == nil || !strings.Contains(err.Error(), "staging") { + t.Fatalf("staging journal accepted image activation: %v", err) + } + matching := journal + matching.Candidate.Update.SHA256 = previous.Current.Update.SHA256 + if err := matching.Validate(); err == nil || !strings.Contains(err.Error(), "differ") { + t.Fatalf("staging journal accepted active digest: %v", err) + } +} + func TestJournalRejectsCandidateMatchingActiveImage(t *testing.T) { now := time.Now().UTC() previous := ActiveState{ProtocolVersion: ActiveStateProtocolVersion, Current: validTestSelection(now.Add(-time.Hour)), UpdatedAt: now.Add(-time.Hour)} @@ -225,6 +448,56 @@ func TestJournalRejectsCandidateMatchingActiveImage(t *testing.T) { } } +func TestRuntimeRepairJournalAllowsSameDigestAndPreservesPriorSelection(t *testing.T) { + now := time.Now().UTC() + current := validTestSelection(now.Add(-time.Hour)) + prior := validTestSelection(now.Add(-2 * time.Hour)) + prior.Update.SHA256 = "sha256:" + strings.Repeat("c", 64) + prior.ImageID = "sha256:" + strings.Repeat("d", 64) + prior.ImageRef = providerImageRef(prior.Update.SHA256) + previous := ActiveState{ + ProtocolVersion: ActiveStateProtocolVersion, + Current: current, + Previous: &prior, + UpdatedAt: current.ActivatedAt, + } + candidate := current + candidate.Update.DirectiveID = "directive-runtime-repair" + candidate.ActivatedAt = now + journal := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, + ID: "txn-runtime-repair", + Phase: JournalCommitted, + RuntimeRepair: true, + Previous: &previous, + Candidate: candidate, + StartedAt: now, + UpdatedAt: now, + } + if err := journal.Validate(); err != nil { + t.Fatalf("valid runtime repair journal: %v", err) + } + recovered, err := RecoverActiveState(journal) + if err != nil { + t.Fatalf("recover runtime repair: %v", err) + } + if recovered.Current.Update.DirectiveID != candidate.Update.DirectiveID || recovered.Previous == nil || recovered.Previous.Update.SHA256 != prior.Update.SHA256 { + t.Fatalf("recovered runtime repair state = %+v", recovered) + } + + regular := journal + regular.RuntimeRepair = false + if err := regular.Validate(); err == nil || !strings.Contains(err.Error(), "differ") { + t.Fatalf("regular same-digest journal err = %v", err) + } + mismatched := journal + mismatched.Candidate.Update.SHA256 = "sha256:" + strings.Repeat("e", 64) + mismatched.Candidate.ImageRef = providerImageRef(mismatched.Candidate.Update.SHA256) + if err := mismatched.Validate(); err == nil || !strings.Contains(err.Error(), "repair") { + t.Fatalf("mismatched runtime repair journal err = %v", err) + } +} + func TestStatusContainsNoCredentialFields(t *testing.T) { status := Status{ ProtocolVersion: StatusProtocolVersion, @@ -261,7 +534,9 @@ func validTestConfig(home string) Config { InstallRoot: root, SystemdDir: filepath.Join(home, ".config", "systemd", "user"), AgentUnit: "workflow-compute-github-runner-linux-stg.service", - PodmanPath: "/usr/bin/podman", + PodmanPath: filepath.Join(home, ".local", "libexec", "podman"), + SystemctlPath: filepath.Join(home, ".local", "libexec", "systemctl"), + LoginctlPath: filepath.Join(home, ".local", "libexec", "loginctl"), ProviderURL: "https://workflow-plugin-github-runner-provider:18090", StableContainer: "workflow-plugin-github-runner-provider", CandidateContainer: "workflow-plugin-github-runner-provider-candidate", diff --git a/internal/retainedprovider/syncdir_unix.go b/internal/retainedprovider/syncdir_unix.go index 8804263..5e11c94 100644 --- a/internal/retainedprovider/syncdir_unix.go +++ b/internal/retainedprovider/syncdir_unix.go @@ -2,13 +2,15 @@ package retainedprovider -import "os" +import ( + "errors" + "os" +) func syncDirectory(path string) error { directory, err := os.Open(path) if err != nil { return err } - defer directory.Close() - return directory.Sync() + return errors.Join(directory.Sync(), directory.Close()) } diff --git a/internal/retainedprovider/systemd.go b/internal/retainedprovider/systemd.go index e180afa..f34d39e 100644 --- a/internal/retainedprovider/systemd.go +++ b/internal/retainedprovider/systemd.go @@ -17,7 +17,9 @@ import ( "math/big" "net" "os" + "os/user" "path/filepath" + "reflect" "sort" "strconv" "strings" @@ -70,7 +72,8 @@ func RenderSystemdUnits(config Config, paths LifecyclePaths) (SystemdUnits, erro "[Service]\n" + "Type=oneshot\n" + "ExecStart=" + systemdQuote(paths.Launcher) + " retained refresh -config " + systemdQuote(paths.ConfigFile) + "\n" + - "TimeoutStartSec=15min\n", + "TimeoutStartSec=" + systemdTimeout(retainedRefreshServiceStartTimeout) + "\n" + + "TimeoutStopSec=" + systemdTimeout(retainedRefreshServiceStopTimeout) + "\n", RefreshPath: "[Unit]\n" + "Description=Watch signed GitHub runner provider package marker\n\n" + "[Path]\n" + @@ -91,6 +94,11 @@ func RenderSystemdUnits(config Config, paths LifecyclePaths) (SystemdUnits, erro }, nil } +func systemdTimeout(duration time.Duration) string { + seconds := (duration + time.Second - 1) / time.Second + return strconv.FormatInt(int64(seconds), 10) + "s" +} + func systemdQuote(value string) string { replacer := strings.NewReplacer(`\`, `\\`, `"`, `\"`, `%`, `%%`) return `"` + replacer.Replace(value) + `"` @@ -120,12 +128,15 @@ type Credentials struct { ProviderToken string } +const maxCredentialBytes = 32 << 10 + type InstallMaterial struct { ProviderEnv []byte ProbeEnv []byte AgentEnv []byte ContainersConf []byte CACert []byte + CAKey []byte ServerCert []byte ServerKey []byte } @@ -143,7 +154,7 @@ func GenerateInstallMaterial(config Config, credentials Credentials, random io.R if now.IsZero() { now = time.Now().UTC() } - caCert, serverCert, serverKey, err := generateProviderTLS(config, random, now.UTC()) + caCert, caKey, serverCert, serverKey, err := generateProviderTLS(config, random, now.UTC()) if err != nil { return InstallMaterial{}, err } @@ -180,6 +191,7 @@ func GenerateInstallMaterial(config Config, credentials Credentials, random io.R AgentEnv: agentEnvironment, ContainersConf: []byte("[network]\ndefault_network = \"" + config.ContainerNetwork + "\"\n"), CACert: caCert, + CAKey: caKey, ServerCert: serverCert, ServerKey: serverKey, }, nil @@ -195,6 +207,7 @@ func WriteInstallMaterial(paths LifecyclePaths, material InstallMaterial) error {path: paths.AgentEnv, data: material.AgentEnv}, {path: paths.ContainersConf, data: material.ContainersConf}, {path: paths.CAFile, data: material.CACert}, + {path: paths.CAKey, data: material.CAKey}, {path: paths.ServerCert, data: material.ServerCert}, {path: paths.ServerKey, data: material.ServerKey}, } { @@ -214,7 +227,7 @@ func renderPodmanEnvironment(values []environmentValue) ([]byte, error) { var builder strings.Builder for _, value := range values { if !safeEnvironmentKey(value.Name) || value.Value == "" || strings.ContainsAny(value.Value, "\r\n\x00") { - return nil, errors.New("Podman environment contains an invalid value") + return nil, errors.New("podman environment contains an invalid value") } builder.WriteString(value.Name) builder.WriteByte('=') @@ -239,20 +252,20 @@ func renderSystemdEnvironment(values []environmentValue) ([]byte, error) { } func validateCredential(value string) error { - if value == "" || strings.TrimSpace(value) != value || strings.ContainsAny(value, "\r\n\x00") { + if value == "" || len(value) > maxCredentialBytes || strings.TrimSpace(value) != value || strings.ContainsAny(value, "\r\n\x00") { return errors.New("invalid credential") } return nil } -func generateProviderTLS(config Config, random io.Reader, now time.Time) ([]byte, []byte, []byte, error) { +func generateProviderTLS(config Config, random io.Reader, now time.Time) ([]byte, []byte, []byte, []byte, error) { caKey, err := ecdsa.GenerateKey(elliptic.P256(), random) if err != nil { - return nil, nil, nil, fmt.Errorf("generate provider CA key: %w", err) + return nil, nil, nil, nil, fmt.Errorf("generate provider CA key: %w", err) } serverKey, err := ecdsa.GenerateKey(elliptic.P256(), random) if err != nil { - return nil, nil, nil, fmt.Errorf("generate provider server key: %w", err) + return nil, nil, nil, nil, fmt.Errorf("generate provider server key: %w", err) } serial := big.NewInt(now.UnixNano()) if serial.Sign() <= 0 { @@ -269,7 +282,7 @@ func generateProviderTLS(config Config, random io.Reader, now time.Time) ([]byte } caDER, err := x509.CreateCertificate(random, caTemplate, caTemplate, &caKey.PublicKey, caKey) if err != nil { - return nil, nil, nil, fmt.Errorf("create provider CA certificate: %w", err) + return nil, nil, nil, nil, fmt.Errorf("create provider CA certificate: %w", err) } serverTemplate := &x509.Certificate{ SerialNumber: new(big.Int).Add(serial, big.NewInt(1)), @@ -283,17 +296,169 @@ func generateProviderTLS(config Config, random io.Reader, now time.Time) ([]byte } serverDER, err := x509.CreateCertificate(random, serverTemplate, caTemplate, &serverKey.PublicKey, caKey) if err != nil { - return nil, nil, nil, fmt.Errorf("create provider server certificate: %w", err) + return nil, nil, nil, nil, fmt.Errorf("create provider server certificate: %w", err) + } + caKeyDER, err := x509.MarshalECPrivateKey(caKey) + if err != nil { + return nil, nil, nil, nil, fmt.Errorf("marshal provider CA key: %w", err) } serverKeyDER, err := x509.MarshalECPrivateKey(serverKey) if err != nil { - return nil, nil, nil, fmt.Errorf("marshal provider server key: %w", err) + return nil, nil, nil, nil, fmt.Errorf("marshal provider server key: %w", err) } return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}), + pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: caKeyDER}), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: serverDER}), pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: serverKeyDER}), nil } +const providerServerCertificateRenewalWindow = 30 * 24 * time.Hour + +func renewProviderServerCertificate(config Config, paths LifecyclePaths, random io.Reader, now time.Time) (bool, error) { + caCertificate, caKey, serverCertificate, serverKey, err := readProviderTLSAuthority(config, paths, now) + if err != nil { + return false, err + } + if serverCertificate.NotAfter.After(now.Add(providerServerCertificateRenewalWindow)) { + return false, nil + } + if random == nil { + random = rand.Reader + } + notAfter := now.AddDate(1, 0, 0) + if caLimit := caCertificate.NotAfter.Add(-time.Hour); notAfter.After(caLimit) { + notAfter = caLimit + } + if !notAfter.After(now.Add(providerServerCertificateRenewalWindow)) { + return false, errors.New("provider CA is too close to expiry; credential reinstall is required") + } + serial := big.NewInt(now.UnixNano()) + if serial.Sign() <= 0 { + serial = big.NewInt(1) + } + template := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: config.StableContainer}, + NotBefore: now.Add(-5 * time.Minute), + NotAfter: notAfter, + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: []string{"localhost", config.StableContainer, config.CandidateContainer}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")}, + } + der, err := x509.CreateCertificate(random, template, caCertificate, &serverKey.PublicKey, caKey) + if err != nil { + return false, fmt.Errorf("renew provider server certificate: %w", err) + } + if err := atomicWriteFile(paths.ServerCert, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600); err != nil { + return false, fmt.Errorf("write renewed provider server certificate: %w", err) + } + return true, nil +} + +func providerServerCertificateNeedsRenewal(config Config, paths LifecyclePaths, now time.Time) (bool, error) { + _, _, certificate, _, err := readProviderTLSAuthority(config, paths, now) + if err != nil { + return false, err + } + return !certificate.NotAfter.After(now.Add(providerServerCertificateRenewalWindow)), nil +} + +func readProviderTLSAuthority(config Config, paths LifecyclePaths, now time.Time) (*x509.Certificate, *ecdsa.PrivateKey, *x509.Certificate, *ecdsa.PrivateKey, error) { + caCertificate, err := readProviderCertificate(paths.CAFile) + if err != nil { + return nil, nil, nil, nil, fmt.Errorf("read provider CA certificate: %w", err) + } + caKey, err := readProviderECKey(paths.CAKey) + if err != nil { + return nil, nil, nil, nil, fmt.Errorf("read provider CA key: %w", err) + } + serverCertificate, err := readProviderCertificate(paths.ServerCert) + if err != nil { + return nil, nil, nil, nil, fmt.Errorf("read provider server certificate: %w", err) + } + serverKey, err := readProviderECKey(paths.ServerKey) + if err != nil { + return nil, nil, nil, nil, fmt.Errorf("read provider server key: %w", err) + } + if !caCertificate.IsCA || caCertificate.NotBefore.After(now) || !caCertificate.NotAfter.After(now) || !sameECDSAPublicKey(caCertificate.PublicKey, &caKey.PublicKey) { + return nil, nil, nil, nil, errors.New("provider CA authority is invalid or expired") + } + if err := serverCertificate.CheckSignatureFrom(caCertificate); err != nil || !sameECDSAPublicKey(serverCertificate.PublicKey, &serverKey.PublicKey) { + return nil, nil, nil, nil, errors.New("provider server certificate authority or key binding is invalid") + } + if serverCertificate.NotBefore.After(now) { + return nil, nil, nil, nil, errors.New("provider server certificate is not currently valid") + } + for _, hostname := range []string{config.StableContainer, config.CandidateContainer} { + if err := serverCertificate.VerifyHostname(hostname); err != nil { + return nil, nil, nil, nil, errors.New("provider server certificate hostname binding is invalid") + } + } + return caCertificate, caKey, serverCertificate, serverKey, nil +} + +func readProviderCertificate(path string) (*x509.Certificate, error) { + data, err := readProviderSecretPEM(path) + if err != nil { + return nil, err + } + block, rest := pem.Decode(data) + if block == nil || block.Type != "CERTIFICATE" || len(bytes.TrimSpace(rest)) != 0 { + return nil, errors.New("provider certificate PEM is invalid") + } + certificate, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, errors.New("provider certificate is invalid") + } + return certificate, nil +} + +func readProviderECKey(path string) (*ecdsa.PrivateKey, error) { + data, err := readProviderSecretPEM(path) + if err != nil { + return nil, err + } + block, rest := pem.Decode(data) + if block == nil || block.Type != "EC PRIVATE KEY" || len(bytes.TrimSpace(rest)) != 0 { + return nil, errors.New("provider private key PEM is invalid") + } + key, err := x509.ParseECPrivateKey(block.Bytes) + if err != nil || key.Curve != elliptic.P256() { + return nil, errors.New("provider private key is invalid") + } + return key, nil +} + +func readProviderSecretPEM(path string) ([]byte, error) { + entry, err := os.Lstat(path) + if err != nil || !entry.Mode().IsRegular() || entry.Mode().Perm() != 0o600 || entry.Size() <= 0 || entry.Size() > MaxStateFileBytes { + return nil, errors.New("provider TLS input must be a bounded owner-only regular file") + } + if err := validateOwner(entry); err != nil { + return nil, err + } + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer func() { _ = file.Close() }() + opened, err := file.Stat() + if err != nil || !opened.Mode().IsRegular() || !os.SameFile(entry, opened) || opened.Size() > MaxStateFileBytes { + return nil, errors.New("provider TLS input changed during open") + } + data, err := io.ReadAll(io.LimitReader(file, MaxStateFileBytes+1)) + if err != nil || len(data) > MaxStateFileBytes { + return nil, errors.New("read provider TLS input") + } + return data, nil +} + +func sameECDSAPublicKey(value any, expected *ecdsa.PublicKey) bool { + actual, ok := value.(*ecdsa.PublicKey) + return ok && actual.Equal(expected) +} + func atomicWriteFile(path string, data []byte, mode os.FileMode) (returnErr error) { if len(data) == 0 || len(data) > MaxStateFileBytes { return errors.New("generated file must be non-empty and at most 1 MiB") @@ -302,7 +467,7 @@ func atomicWriteFile(path string, data []byte, mode os.FileMode) (returnErr erro return errors.New("generated file mode must be 0600 or 0700") } directory := filepath.Dir(path) - if err := os.MkdirAll(directory, 0o700); err != nil { + if err := mkdirAllDurable(directory, 0o700); err != nil { return err } if err := rejectWritableDestination(path); err != nil { @@ -370,6 +535,7 @@ const ( type Installer struct { Runner CommandRunner ExecutablePath func() (string, error) + UserID func() (string, error) Random io.Reader Now func() time.Time Sleep func(context.Context, time.Duration) error @@ -587,23 +753,12 @@ func writeInstallTransactionPhase(paths LifecyclePaths, journal *installTransact return nil } -func writeInstallTransactionActivation(paths LifecyclePaths, journal *installTransactionJournal, activation systemdActivation, now time.Time) error { - next := *journal - next.Activation = activation - next.UpdatedAt = now - if err := writeInstallTransactionJournal(paths, next); err != nil { - return err - } - *journal = next - return nil -} - func (installer Installer) recoverInstallTransaction(ctx context.Context, config Config, paths LifecyclePaths, refresher Refresher) error { journal, found, err := readInstallTransactionJournal(paths) if err != nil { return fmt.Errorf("read retained provider install transaction: %w", err) } - providerJournal, providerFound, err := readTransactionJournal(paths.Journal) + providerJournal, providerFound, err := readTransactionJournalForConfig(paths.Journal, config) if err != nil { return fmt.Errorf("read retained provider refresh transaction: %w", err) } @@ -622,7 +777,11 @@ func (installer Installer) recoverInstallTransaction(ctx context.Context, config if providerFound { providerRollbackErr = refresher.rollbackDeferredRefresh(ctx, config) } - rollbackErr := installer.rollbackInstall(ctx, config, journal.Snapshots, journal.PreviousUnits, journal.AgentStopped, true, journal.MaintenanceID, journal.Activation) + reason := installMaintenanceReason + if journal.Operation == "uninstall" { + reason = uninstallMaintenanceReason + } + rollbackErr := installer.rollbackInstall(ctx, config, journal.Snapshots, journal.PreviousUnits, journal.AgentStopped, true, journal.MaintenanceID, reason, journal.Activation) if err := errors.Join(providerRollbackErr, rollbackErr); err != nil { return fmt.Errorf("rollback interrupted retained provider %s: %w", journal.Operation, err) } @@ -677,7 +836,7 @@ func (installer Installer) Install(ctx context.Context, home string, config Conf return Status{}, fmt.Errorf("acquire retained provider install lock: %w", err) } defer func() { returnErr = errors.Join(returnErr, lock.Release()) }() - if err := os.MkdirAll(paths.Root, 0o700); err != nil { + if err := mkdirAllDurable(paths.Root, 0o700); err != nil { return Status{}, fmt.Errorf("create retained provider root: %w", err) } if err := validateInstallRoot(paths.Root); err != nil { @@ -686,6 +845,9 @@ func (installer Installer) Install(ctx context.Context, home string, config Conf if err := installer.recoverLifecycleTransaction(ctx, home, paths, lifecycleRefresher); err != nil { return Status{}, err } + if err := validateInstalledConfigBinding(home, config, paths); err != nil { + return Status{}, err + } if err := installer.recoverInstallTransaction(ctx, config, paths, lifecycleRefresher); err != nil { return Status{}, err } @@ -693,13 +855,19 @@ func (installer Installer) Install(ctx context.Context, home string, config Conf if err != nil { return Status{}, err } - active, activeFound, err := readActiveState(paths.ActiveState) + active, activeFound, err := readActiveStateForConfig(paths.ActiveState, config) if err != nil { return Status{}, err } effect := ProviderChanged if activeFound && active.Current.Update.SHA256 == update.SHA256 { - effect = ProviderUnchanged + runtimeRepair, err := lifecycleRefresher.activeProviderImageNeedsRepair(ctx, config, active.Current) + if err != nil { + return Status{}, err + } + if !runtimeRepair { + effect = ProviderUnchanged + } } transaction, err := newLifecycleJournal(config, LifecycleInstall, effect, nil, installer.now()) if err != nil { @@ -720,14 +888,16 @@ func (installer Installer) Install(ctx context.Context, home string, config Conf return Status{}, err } fail := func(cause error) (Status, error) { - rollbackContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + rollbackContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), lifecycleRecoveryTimeout) defer cancel() - return Status{}, errors.Join(cause, installer.recoverLifecycleTransaction(rollbackContext, home, paths, lifecycleRefresher)) + auditErr := writeLifecycleDiagnostic(home, paths, &transaction, AuditError, "operation_failed", installer.now()) + return Status{}, errors.Join(cause, auditErr, installer.recoverLifecycleTransaction(rollbackContext, home, paths, lifecycleRefresher)) } - if err := installer.beginMaintenance(ctx, config, installMaintenanceID, installMaintenanceReason); err != nil { + maintenance, err := installer.beginMaintenance(ctx, config, transaction.TransactionID, installMaintenanceReason) + if err != nil { return fail(err) } - if err := installer.waitLocalState(ctx, config, "unavailable"); err != nil { + if err := installer.waitLocalStateAfter(ctx, config, "unavailable", maintenance.StartedAt); err != nil { return Status{}, fmt.Errorf("wait for retained agent maintenance fence: %w", err) } if err := installer.reattestLifecycleAuthority(ctx, home, transaction); err != nil { @@ -736,7 +906,7 @@ func (installer Installer) Install(ctx context.Context, home string, config Conf if err := snapshotManagedFilesForLifecycle(home, paths, &transaction, installer.now()); err != nil { return fail(fmt.Errorf("snapshot retained provider wiring: %w", err)) } - previousUnits, err := installer.captureManagedUnitStates(ctx) + previousUnits, err := installer.captureManagedUnitStates(ctx, config.SystemctlPath) if err != nil { return fail(fmt.Errorf("snapshot retained provider unit state: %w", err)) } @@ -756,13 +926,13 @@ func (installer Installer) Install(ctx context.Context, home string, config Conf if err := writeLifecycleTransition(home, paths, &transaction, LifecycleFenced, "", installer.now()); err != nil { return fail(err) } - if err := installer.systemctl(ctx, "stop", config.AgentUnit); err != nil { + if err := installer.systemctl(ctx, config.SystemctlPath, "stop", config.AgentUnit); err != nil { return fail(fmt.Errorf("stop retained agent: %w", err)) } watchUnits := previouslyLoadedWatchUnits(previousUnits) if len(watchUnits) > 0 { arguments := append([]string{"disable", "--now"}, watchUnits...) - if err := installer.systemctl(ctx, arguments...); err != nil { + if err := installer.systemctl(ctx, config.SystemctlPath, arguments...); err != nil { return fail(fmt.Errorf("pause retained provider refresh: %w", err)) } } @@ -772,7 +942,7 @@ func (installer Installer) Install(ctx context.Context, home string, config Conf if err := installer.ensureProviderNetwork(ctx, config); err != nil { return fail(err) } - if err := installer.systemctl(ctx, "daemon-reload"); err != nil { + if err := installer.systemctl(ctx, config.SystemctlPath, "daemon-reload"); err != nil { return fail(fmt.Errorf("reload user systemd: %w", err)) } loadedSignature, err := installer.inspectAgentUnitSignature(ctx, home, config) @@ -789,7 +959,7 @@ func (installer Installer) Install(ctx context.Context, home string, config Conf if err := writeLifecycleJournal(home, paths, transaction); err != nil { return fail(fmt.Errorf("record provider service activation: %w", err)) } - if err := installer.systemctl(ctx, "enable", providerServiceUnit); err != nil { + if err := installer.systemctl(ctx, config.SystemctlPath, "enable", providerServiceUnit); err != nil { return fail(fmt.Errorf("enable provider service: %w", err)) } refresh := installer.Refresh @@ -797,7 +967,7 @@ func (installer Installer) Install(ctx context.Context, home string, config Conf if refresh == nil || probeActive == nil { if refresh == nil { refresh = func(ctx context.Context, config Config) (Status, error) { - return lifecycleRefresher.refreshUnderLifecycleTransaction(ctx, config, false, true, transaction.TransactionID, config.ProfileID, "") + return lifecycleRefresher.refreshUnderLifecycleTransaction(ctx, config, false, true, transaction.TransactionID, config.ProfileID, "", effect) } } if probeActive == nil { @@ -814,7 +984,7 @@ func (installer Installer) Install(ctx context.Context, home string, config Conf if transaction.Unchanged != nil { transaction.Unchanged.StableProbeAt = installer.now() } - inner, innerFound, err := readTransactionJournal(paths.Journal) + inner, innerFound, err := readTransactionJournalForConfig(paths.Journal, config) if err != nil { return fail(err) } @@ -830,7 +1000,7 @@ func (installer Installer) Install(ctx context.Context, home string, config Conf if err := writeLifecycleJournal(home, paths, transaction); err != nil { return fail(err) } - _, err = installer.enableWatchUnitBefore(ctx, refreshPathUnit, func() error { + _, err = installer.enableWatchUnitBefore(ctx, config.SystemctlPath, refreshPathUnit, func() error { activation := transaction.Activation activation.RefreshPath = true transaction.Activation = activation @@ -840,7 +1010,7 @@ func (installer Installer) Install(ctx context.Context, home string, config Conf if err != nil { return fail(fmt.Errorf("enable retained provider refresh: %w", err)) } - _, err = installer.enableWatchUnitBefore(ctx, refreshTimerUnit, func() error { + _, err = installer.enableWatchUnitBefore(ctx, config.SystemctlPath, refreshTimerUnit, func() error { activation := transaction.Activation activation.RefreshTimer = true transaction.Activation = activation @@ -853,10 +1023,11 @@ func (installer Installer) Install(ctx context.Context, home string, config Conf if err := installer.reattestLifecycleAuthority(ctx, home, transaction); err != nil { return fail(err) } - if err := installer.systemctl(ctx, "start", config.AgentUnit); err != nil { + agentRestartedAfter := installer.now() + if err := installer.systemctl(ctx, config.SystemctlPath, "start", config.AgentUnit); err != nil { return fail(fmt.Errorf("restart retained agent: %w", err)) } - if err := installer.waitLocalState(ctx, config, "unavailable"); err != nil { + if err := installer.waitLocalStateAfter(ctx, config, "unavailable", agentRestartedAfter); err != nil { return fail(fmt.Errorf("verify retained agent remains fenced: %w", err)) } if err := installer.reattestLifecycleAuthority(ctx, home, transaction); err != nil { @@ -872,6 +1043,7 @@ func (installer Installer) Install(ctx context.Context, home string, config Conf return fail(err) } _ = drainLifecycleAudit(home, paths, &transaction) + maintenanceReleasedAfter := installer.now() if err := installer.releaseLifecycleMaintenance(ctx, home, transaction); err != nil { return Status{}, fmt.Errorf("release retained agent maintenance fence: %w", err) } @@ -881,12 +1053,28 @@ func (installer Installer) Install(ctx context.Context, home string, config Conf if err := finalizeLifecycleTransaction(home, paths, &transaction, lifecycleRefresher); err != nil { return Status{}, err } - if err := installer.waitLocalState(ctx, config, "idle"); err != nil { + if err := installer.waitLocalStateAfter(ctx, config, "idle", maintenanceReleasedAfter); err != nil { return Status{}, fmt.Errorf("wait for retained agent idle state: %w", err) } return status, nil } +func validateInstalledConfigBinding(home string, requested Config, paths LifecyclePaths) error { + if _, err := os.Lstat(paths.ConfigFile); errors.Is(err, os.ErrNotExist) { + return nil + } else if err != nil { + return fmt.Errorf("inspect installed retained provider config: %w", err) + } + installed, err := ReadConfigFile(paths.ConfigFile, home) + if err != nil { + return fmt.Errorf("read installed retained provider config: %w", err) + } + if !reflect.DeepEqual(installed, requested) { + return errors.New("lifecycle config must exactly match the installed retained provider config") + } + return nil +} + func (installer Installer) Uninstall(ctx context.Context, home string, config Config, purge bool) (status Status, returnErr error) { if installer.Runner == nil { return Status{}, errors.New("command runner is required") @@ -901,7 +1089,7 @@ func (installer Installer) Uninstall(ctx context.Context, home string, config Co } defer func() { returnErr = errors.Join(returnErr, lock.Release()) }() lifecycleRefresher := Refresher{Runner: installer.Runner, Now: installer.Now, Sleep: installer.Sleep} - if err := os.MkdirAll(paths.Root, 0o700); err != nil { + if err := mkdirAllDurable(paths.Root, 0o700); err != nil { return Status{}, fmt.Errorf("create retained provider root: %w", err) } if err := validateInstallRoot(paths.Root); err != nil { @@ -910,6 +1098,9 @@ func (installer Installer) Uninstall(ctx context.Context, home string, config Co if err := installer.recoverLifecycleTransaction(ctx, home, paths, lifecycleRefresher); err != nil { return Status{}, err } + if err := validateInstalledConfigBinding(home, config, paths); err != nil { + return Status{}, err + } if err := installer.recoverInstallTransaction(ctx, config, paths, lifecycleRefresher); err != nil { return Status{}, err } @@ -932,14 +1123,16 @@ func (installer Installer) Uninstall(ctx context.Context, home string, config Co return Status{}, err } fail := func(cause error) (Status, error) { - rollbackContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + rollbackContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), lifecycleRecoveryTimeout) defer cancel() - return Status{}, errors.Join(cause, installer.recoverLifecycleTransaction(rollbackContext, home, paths, lifecycleRefresher)) + auditErr := writeLifecycleDiagnostic(home, paths, &transaction, AuditError, "operation_failed", installer.now()) + return Status{}, errors.Join(cause, auditErr, installer.recoverLifecycleTransaction(rollbackContext, home, paths, lifecycleRefresher)) } - if err := installer.beginMaintenance(ctx, config, uninstallMaintenanceID, uninstallMaintenanceReason); err != nil { + maintenance, err := installer.beginMaintenance(ctx, config, transaction.TransactionID, uninstallMaintenanceReason) + if err != nil { return fail(err) } - if err := installer.waitLocalState(ctx, config, "unavailable"); err != nil { + if err := installer.waitLocalStateAfter(ctx, config, "unavailable", maintenance.StartedAt); err != nil { return Status{}, fmt.Errorf("wait for retained agent maintenance fence: %w", err) } if err := installer.reattestLifecycleAuthority(ctx, home, transaction); err != nil { @@ -948,7 +1141,7 @@ func (installer Installer) Uninstall(ctx context.Context, home string, config Co if err := snapshotManagedFilesForLifecycle(home, paths, &transaction, installer.now()); err != nil { return fail(fmt.Errorf("snapshot retained provider wiring: %w", err)) } - previousUnits, err := installer.captureManagedUnitStates(ctx) + previousUnits, err := installer.captureManagedUnitStates(ctx, config.SystemctlPath) if err != nil { return fail(fmt.Errorf("snapshot retained provider unit state: %w", err)) } @@ -966,10 +1159,10 @@ func (installer Installer) Uninstall(ctx context.Context, home string, config Co if err := writeLifecycleTransition(home, paths, &transaction, LifecycleFenced, "", installer.now()); err != nil { return fail(err) } - if err := installer.systemctl(ctx, "stop", config.AgentUnit); err != nil { + if err := installer.systemctl(ctx, config.SystemctlPath, "stop", config.AgentUnit); err != nil { return fail(fmt.Errorf("stop retained agent: %w", err)) } - if err := installer.systemctl(ctx, "disable", "--now", refreshPathUnit, refreshTimerUnit, providerServiceUnit); err != nil { + if err := installer.systemctl(ctx, config.SystemctlPath, "disable", "--now", refreshPathUnit, refreshTimerUnit, providerServiceUnit); err != nil { return fail(fmt.Errorf("disable retained provider wiring: %w", err)) } for _, path := range managedWiringPaths(paths) { @@ -977,7 +1170,7 @@ func (installer Installer) Uninstall(ctx context.Context, home string, config Co return fail(fmt.Errorf("remove retained provider wiring: %w", err)) } } - if err := installer.systemctl(ctx, "daemon-reload"); err != nil { + if err := installer.systemctl(ctx, config.SystemctlPath, "daemon-reload"); err != nil { return fail(fmt.Errorf("reload user systemd: %w", err)) } loadedSignature, err := installer.inspectAgentUnitSignature(ctx, home, config) @@ -990,10 +1183,11 @@ func (installer Installer) Uninstall(ctx context.Context, home string, config Co if err := installer.reattestLifecycleAuthority(ctx, home, transaction); err != nil { return fail(err) } - if err := installer.systemctl(ctx, "start", config.AgentUnit); err != nil { + agentRestartedAfter := installer.now() + if err := installer.systemctl(ctx, config.SystemctlPath, "start", config.AgentUnit); err != nil { return fail(fmt.Errorf("restart retained agent: %w", err)) } - if err := installer.waitLocalState(ctx, config, "unavailable"); err != nil { + if err := installer.waitLocalStateAfter(ctx, config, "unavailable", agentRestartedAfter); err != nil { return fail(fmt.Errorf("verify retained agent remains fenced: %w", err)) } if err := installer.reattestLifecycleAuthority(ctx, home, transaction); err != nil { @@ -1009,13 +1203,14 @@ func (installer Installer) Uninstall(ctx context.Context, home string, config Co return fail(err) } _ = drainLifecycleAudit(home, paths, &transaction) + maintenanceReleasedAfter := installer.now() if err := installer.releaseLifecycleMaintenance(ctx, home, transaction); err != nil { return Status{}, fmt.Errorf("release retained agent maintenance fence: %w", err) } if err := writeLifecycleTransition(home, paths, &transaction, LifecycleCommitted, LifecycleCommit, installer.now()); err != nil { return Status{}, fmt.Errorf("commit retained provider uninstall: %w", err) } - if err := installer.waitLocalState(ctx, config, "idle"); err != nil { + if err := installer.waitLocalStateAfter(ctx, config, "idle", maintenanceReleasedAfter); err != nil { return Status{}, fmt.Errorf("wait for retained agent idle state: %w", err) } if err := finalizeLifecycleTransaction(home, paths, &transaction, lifecycleRefresher); err != nil { @@ -1073,7 +1268,7 @@ func (installer Installer) Status(ctx context.Context, home string, config Confi return Status{}, err } paths := LifecyclePathsFor(config) - active, found, err := readActiveState(paths.ActiveState) + active, found, err := readActiveStateForConfig(paths.ActiveState, config) if err != nil { return Status{}, err } @@ -1090,7 +1285,10 @@ func (installer Installer) Status(ctx context.Context, home string, config Confi if err := validateOwner(unitInfo); err != nil { return Status{}, fmt.Errorf("retained provider service unit: %w", err) } - output, err := installer.run(ctx, Command{Path: "/usr/bin/systemctl", Args: []string{ + if _, err := hashHostExecutable(config.SystemctlPath); err != nil { + return Status{}, fmt.Errorf("validate systemctl authority: %w", err) + } + output, err := installer.run(ctx, Command{Path: config.SystemctlPath, Args: []string{ "--user", "show", providerServiceUnit, "--property", "ActiveState", "--value", }}) if err != nil { @@ -1134,22 +1332,83 @@ func (installer Installer) preflightInstall(ctx context.Context, config Config, } func (installer Installer) runSystemPreflight(ctx context.Context, config Config) error { - if err := installer.systemctl(ctx, "show-environment"); err != nil { + if err := validateConfiguredHostExecutables(config); err != nil { + return err + } + uid, err := installer.currentUserID() + if err != nil { + return fmt.Errorf("resolve retained provider user identity: %w", err) + } + if uid == "0" { + return errors.New("retained provider install requires a non-root user") + } + if err := installer.systemctl(ctx, config.SystemctlPath, "show-environment"); err != nil { return fmt.Errorf("user systemd preflight: %w", err) } + linger, err := installer.run(ctx, Command{Path: config.LoginctlPath, Args: []string{ + "show-user", uid, "--property", "Linger", "--value", + }}) + if err != nil { + return fmt.Errorf("user systemd lingering preflight: %w", err) + } + if strings.TrimSpace(string(linger)) != "yes" { + return errors.New("user systemd lingering must be enabled") + } if _, err := installer.run(ctx, Command{Path: config.PodmanPath, Args: []string{"version", "--format", "{{.Client.Version}}"}}); err != nil { return fmt.Errorf("rootless Podman preflight: %w", err) } + rootless, err := installer.run(ctx, Command{Path: config.PodmanPath, Args: []string{ + "info", "--format", "{{.Host.Security.Rootless}}", + }}) + if err != nil { + return fmt.Errorf("inspect rootless Podman preflight: %w", err) + } + if strings.TrimSpace(string(rootless)) != "true" { + return errors.New("podman must report a rootless runtime") + } return installer.runSupervisorConfigPreflight(ctx, config) } +func (installer Installer) currentUserID() (string, error) { + if installer.UserID != nil { + return installer.UserID() + } + current, err := user.Current() + if err != nil { + return "", err + } + if _, err := strconv.ParseUint(current.Uid, 10, 32); err != nil { + return "", errors.New("current user has no numeric UID") + } + return current.Uid, nil +} + func (installer Installer) runAgentSystemPreflight(ctx context.Context, config Config) error { - if err := installer.systemctl(ctx, "show-environment"); err != nil { + if err := validateConfiguredHostExecutables(config); err != nil { + return err + } + if err := installer.systemctl(ctx, config.SystemctlPath, "show-environment"); err != nil { return fmt.Errorf("user systemd preflight: %w", err) } return installer.runSupervisorConfigPreflight(ctx, config) } +func validateConfiguredHostExecutables(config Config) error { + for _, executable := range []struct { + label string + path string + }{ + {label: "podman", path: config.PodmanPath}, + {label: "systemctl", path: config.SystemctlPath}, + {label: "loginctl", path: config.LoginctlPath}, + } { + if _, err := hashHostExecutable(executable.path); err != nil { + return fmt.Errorf("validate %s authority: %w", executable.label, err) + } + } + return nil +} + func (installer Installer) runSupervisorConfigPreflight(ctx context.Context, config Config) error { if _, err := installer.run(ctx, Command{Path: config.ComputeAgentPath, Args: []string{ "supervisor-config", "validate", "-path", config.SupervisorConfigPath, "-format", "auto", @@ -1172,12 +1431,15 @@ func (installer Installer) ensureProviderNetwork(ctx context.Context, config Con return nil } -func (installer Installer) beginMaintenance(ctx context.Context, config Config, id, reason string) error { +func (installer Installer) beginMaintenance(ctx context.Context, config Config, id, reason string) (maintenanceRecord, error) { state, err := installer.maintenanceCommand(ctx, config, "begin", id, reason) if err != nil { - return err + return maintenanceRecord{}, err + } + if err := validateMaintenanceState(state, true, config.ProfileID, id, reason); err != nil { + return maintenanceRecord{}, err } - return validateMaintenanceState(state, true, config.ProfileID, id, reason) + return *state.Maintenance, nil } func (installer Installer) endMaintenance(ctx context.Context, config Config, id, reason string) error { @@ -1192,7 +1454,7 @@ func (installer Installer) releaseLifecycleMaintenance(ctx context.Context, home if err := installer.reattestLifecycleAuthority(ctx, home, journal); err != nil { return err } - id, reason, err := lifecycleMaintenanceIdentity(journal.Operation) + id, reason, err := lifecycleMaintenanceIdentity(journal) if err != nil { return err } @@ -1270,7 +1532,10 @@ func validateMaintenanceState(state maintenanceState, active bool, profileID, id return nil } -func (installer Installer) waitLocalState(ctx context.Context, config Config, expected string) error { +func (installer Installer) waitLocalStateAfter(ctx context.Context, config Config, expected string, observedAfter time.Time) error { + if observedAfter.IsZero() { + return errors.New("local agent observation boundary is required") + } for attempt := 0; attempt < localStatusAttempts; attempt++ { output, err := installer.run(ctx, Command{Path: config.ComputeAgentPath, Args: []string{ "local-status", "sanitize", "-path", config.LocalStatusPath, @@ -1285,7 +1550,7 @@ func (installer Installer) waitLocalState(ctx context.Context, config Config, ex if status.ProtocolVersion != localStatusProtocolVersion || status.WorkerID != config.WorkerID || status.UpdatedAt.IsZero() { return errors.New("local agent status identity or protocol mismatch") } - if status.State == expected && status.TaskID == "" && status.LeaseID == "" { + if status.UpdatedAt.After(observedAfter) && status.State == expected && status.TaskID == "" && status.LeaseID == "" { return nil } if attempt+1 < localStatusAttempts { @@ -1297,7 +1562,10 @@ func (installer Installer) waitLocalState(ctx context.Context, config Config, ex return fmt.Errorf("local agent did not reach %s without an active task or lease", expected) } -func (installer Installer) waitLocalDrained(ctx context.Context, config Config) error { +func (installer Installer) waitLocalDrainedAfter(ctx context.Context, config Config, observedAfter time.Time) error { + if observedAfter.IsZero() { + return errors.New("local agent drain boundary is required") + } for attempt := 0; attempt < localStatusAttempts; attempt++ { output, err := installer.run(ctx, Command{Path: config.ComputeAgentPath, Args: []string{ "local-status", "sanitize", "-path", config.LocalStatusPath, @@ -1312,7 +1580,7 @@ func (installer Installer) waitLocalDrained(ctx context.Context, config Config) if status.ProtocolVersion != localStatusProtocolVersion || status.WorkerID != config.WorkerID || status.UpdatedAt.IsZero() { return errors.New("local agent status identity or protocol mismatch") } - if status.TaskID == "" && status.LeaseID == "" { + if status.UpdatedAt.After(observedAfter) && status.TaskID == "" && status.LeaseID == "" { return nil } if attempt+1 < localStatusAttempts { @@ -1324,26 +1592,22 @@ func (installer Installer) waitLocalDrained(ctx context.Context, config Config) return errors.New("local agent remained assigned to a task or lease") } -func (installer Installer) systemctl(ctx context.Context, args ...string) error { +func (installer Installer) systemctl(ctx context.Context, path string, args ...string) error { arguments := append([]string{"--user"}, args...) - _, err := installer.run(ctx, Command{Path: "/usr/bin/systemctl", Args: arguments}) + _, err := installer.run(ctx, Command{Path: path, Args: arguments}) return err } -func (installer Installer) enableWatchUnit(ctx context.Context, unit string) (bool, error) { - return installer.enableWatchUnitBefore(ctx, unit, nil) -} - -func (installer Installer) enableWatchUnitBefore(ctx context.Context, unit string, beforeMutation func() error) (bool, error) { +func (installer Installer) enableWatchUnitBefore(ctx context.Context, systemctlPath, unit string, beforeMutation func() error) (bool, error) { if beforeMutation != nil { if err := beforeMutation(); err != nil { return false, err } } - if err := installer.systemctl(ctx, "enable", "--now", unit); err == nil { + if err := installer.systemctl(ctx, systemctlPath, "enable", "--now", unit); err == nil { return true, nil } else { - activated, inspectErr := installer.inspectUnitActivation(ctx, unit) + activated, inspectErr := installer.inspectUnitActivation(ctx, systemctlPath, unit) if inspectErr != nil { return true, errors.Join(err, fmt.Errorf("inspect failed unit activation: %w", inspectErr)) } @@ -1351,8 +1615,8 @@ func (installer Installer) enableWatchUnitBefore(ctx context.Context, unit strin } } -func (installer Installer) inspectUnitActivation(ctx context.Context, unit string) (bool, error) { - state, err := installer.inspectUnitState(ctx, unit) +func (installer Installer) inspectUnitActivation(ctx context.Context, systemctlPath, unit string) (bool, error) { + state, err := installer.inspectUnitState(ctx, systemctlPath, unit) if err != nil { return false, err } @@ -1360,7 +1624,7 @@ func (installer Installer) inspectUnitActivation(ctx context.Context, unit strin } func (installer Installer) inspectAgentUnitSignature(ctx context.Context, home string, config Config) (LifecycleSystemdSignature, error) { - output, err := installer.run(ctx, Command{Path: "/usr/bin/systemctl", Args: []string{ + output, err := installer.run(ctx, Command{Path: config.SystemctlPath, Args: []string{ "--user", "show", config.AgentUnit, "--property", "LoadState", "--property", "FragmentPath", "--property", "DropInPaths", }}) @@ -1680,7 +1944,7 @@ func readAndAttestLifecycleSystemdPath(home, path string) (LifecycleFileAttestat if err != nil { return LifecycleFileAttestation{}, nil, fmt.Errorf("open effective agent systemd input: %w", err) } - defer file.Close() + defer func() { _ = file.Close() }() opened, err := file.Stat() if err != nil || !opened.Mode().IsRegular() || !os.SameFile(entry, opened) || opened.Size() > MaxStateFileBytes { return LifecycleFileAttestation{}, nil, errors.New("effective agent systemd input changed during open") @@ -1700,8 +1964,8 @@ func readAndAttestLifecycleSystemdPath(home, path string) (LifecycleFileAttestat return attestation, contents, nil } -func (installer Installer) inspectUnitState(ctx context.Context, unit string) (systemdUnitState, error) { - output, err := installer.run(ctx, Command{Path: "/usr/bin/systemctl", Args: []string{ +func (installer Installer) inspectUnitState(ctx context.Context, systemctlPath, unit string) (systemdUnitState, error) { + output, err := installer.run(ctx, Command{Path: systemctlPath, Args: []string{ "--user", "show", unit, "--property", "LoadState", "--property", "FragmentPath", "--property", "ActiveState", "--property", "UnitFileState", @@ -1729,10 +1993,10 @@ func (installer Installer) inspectUnitState(ctx context.Context, unit string) (s }, nil } -func (installer Installer) captureManagedUnitStates(ctx context.Context) (map[string]systemdUnitState, error) { +func (installer Installer) captureManagedUnitStates(ctx context.Context, systemctlPath string) (map[string]systemdUnitState, error) { states := map[string]systemdUnitState{} for _, unit := range []string{providerServiceUnit, refreshPathUnit, refreshTimerUnit} { - state, err := installer.inspectUnitState(ctx, unit) + state, err := installer.inspectUnitState(ctx, systemctlPath, unit) if err != nil { return nil, fmt.Errorf("inspect %s: %w", unit, err) } @@ -1747,25 +2011,25 @@ func (installer Installer) captureManagedUnitStates(ctx context.Context) (map[st return states, nil } -func (installer Installer) restoreUnitState(ctx context.Context, unit string, state systemdUnitState) error { +func (installer Installer) restoreUnitState(ctx context.Context, systemctlPath, unit string, state systemdUnitState) error { if err := validateRestorableUnitState(state); err != nil { return err } var restoreErr error switch state.UnitFileState { case "enabled": - restoreErr = errors.Join(restoreErr, installer.systemctl(ctx, "enable", unit)) + restoreErr = errors.Join(restoreErr, installer.systemctl(ctx, systemctlPath, "enable", unit)) case "enabled-runtime": - restoreErr = errors.Join(restoreErr, installer.systemctl(ctx, "enable", "--runtime", unit)) + restoreErr = errors.Join(restoreErr, installer.systemctl(ctx, systemctlPath, "enable", "--runtime", unit)) case "disabled": - restoreErr = errors.Join(restoreErr, installer.systemctl(ctx, "disable", unit)) + restoreErr = errors.Join(restoreErr, installer.systemctl(ctx, systemctlPath, "disable", unit)) case "static", "indirect", "generated", "transient": } switch state.ActiveState { case "active": - restoreErr = errors.Join(restoreErr, installer.systemctl(ctx, "start", unit)) + restoreErr = errors.Join(restoreErr, installer.systemctl(ctx, systemctlPath, "start", unit)) case "inactive": - restoreErr = errors.Join(restoreErr, installer.systemctl(ctx, "stop", unit)) + restoreErr = errors.Join(restoreErr, installer.systemctl(ctx, systemctlPath, "stop", unit)) } return restoreErr } @@ -1790,23 +2054,21 @@ func validateRestorableUnitState(state systemdUnitState) error { return nil } -func (installer Installer) rollbackInstall(ctx context.Context, config Config, snapshots []managedFileSnapshot, previousUnits map[string]systemdUnitState, agentStopped, maintenanceActive bool, maintenanceID string, activation systemdActivation) error { - return installer.rollbackInstallBeforeStart(ctx, config, snapshots, previousUnits, agentStopped, maintenanceActive, maintenanceID, activation, nil) +func (installer Installer) rollbackInstall(ctx context.Context, config Config, snapshots []managedFileSnapshot, previousUnits map[string]systemdUnitState, agentStopped, maintenanceActive bool, maintenanceID, maintenanceReason string, activation systemdActivation) error { + if err := installer.rollbackInstallBeforeStart(ctx, config, snapshots, previousUnits, agentStopped, maintenanceActive, maintenanceID, maintenanceReason, activation, nil); err != nil { + return err + } + return removeSnapshots(snapshots) } -func (installer Installer) rollbackInstallBeforeStart(ctx context.Context, config Config, snapshots []managedFileSnapshot, previousUnits map[string]systemdUnitState, agentStopped, maintenanceActive bool, maintenanceID string, activation systemdActivation, beforeStart func(context.Context) error) error { - rollbackContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) +func (installer Installer) rollbackInstallBeforeStart(ctx context.Context, config Config, snapshots []managedFileSnapshot, previousUnits map[string]systemdUnitState, agentStopped, maintenanceActive bool, maintenanceID, maintenanceReason string, activation systemdActivation, beforeStart func(context.Context) error) error { + rollbackContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), installRollbackTimeout) defer cancel() if !agentStopped { - cleanupErr := removeSnapshots(snapshots) if maintenanceActive { - reason := installMaintenanceReason - if maintenanceID == uninstallMaintenanceID { - reason = uninstallMaintenanceReason - } - cleanupErr = errors.Join(cleanupErr, installer.endMaintenance(rollbackContext, config, maintenanceID, reason)) + return installer.endMaintenance(rollbackContext, config, maintenanceID, maintenanceReason) } - return cleanupErr + return nil } var rollbackErr error activatedUnits := make([]string, 0, 3) @@ -1821,36 +2083,29 @@ func (installer Installer) rollbackInstallBeforeStart(ctx context.Context, confi } if len(activatedUnits) > 0 { arguments := append([]string{"disable", "--now"}, activatedUnits...) - rollbackErr = errors.Join(rollbackErr, installer.systemctl(rollbackContext, arguments...)) + rollbackErr = errors.Join(rollbackErr, installer.systemctl(rollbackContext, config.SystemctlPath, arguments...)) } if err := restoreManagedFileContents(snapshots); err != nil { rollbackErr = errors.Join(rollbackErr, err) } else { - rollbackErr = errors.Join(rollbackErr, installer.systemctl(rollbackContext, "daemon-reload")) + rollbackErr = errors.Join(rollbackErr, installer.systemctl(rollbackContext, config.SystemctlPath, "daemon-reload")) for _, unit := range []string{providerServiceUnit, refreshPathUnit, refreshTimerUnit} { if state, found := previousUnits[unit]; found { - rollbackErr = errors.Join(rollbackErr, installer.restoreUnitState(rollbackContext, unit, state)) + rollbackErr = errors.Join(rollbackErr, installer.restoreUnitState(rollbackContext, config.SystemctlPath, unit, state)) } } } if beforeStart == nil { - rollbackErr = errors.Join(rollbackErr, installer.systemctl(rollbackContext, "start", config.AgentUnit)) + rollbackErr = errors.Join(rollbackErr, installer.systemctl(rollbackContext, config.SystemctlPath, "start", config.AgentUnit)) } else if rollbackErr == nil { if err := beforeStart(rollbackContext); err != nil { rollbackErr = err } else { - rollbackErr = installer.systemctl(rollbackContext, "start", config.AgentUnit) + rollbackErr = installer.systemctl(rollbackContext, config.SystemctlPath, "start", config.AgentUnit) } } if rollbackErr == nil && maintenanceActive { - reason := installMaintenanceReason - if maintenanceID == uninstallMaintenanceID { - reason = uninstallMaintenanceReason - } - rollbackErr = installer.endMaintenance(rollbackContext, config, maintenanceID, reason) - } - if rollbackErr == nil { - rollbackErr = removeSnapshots(snapshots) + rollbackErr = installer.endMaintenance(rollbackContext, config, maintenanceID, maintenanceReason) } return rollbackErr } @@ -1868,7 +2123,7 @@ func writeInstalledProvider(config Config, paths LifecyclePaths, executable, dig if err := WriteInstallMaterial(paths, material); err != nil { return err } - if err := os.MkdirAll(paths.ProviderState, 0o700); err != nil { + if err := mkdirAllDurable(paths.ProviderState, 0o700); err != nil { return fmt.Errorf("create provider state directory: %w", err) } for _, file := range []struct { @@ -1892,21 +2147,12 @@ func managedInstallPaths(paths LifecyclePaths) []string { return append([]string{ paths.ConfigFile, paths.Launcher, paths.ActiveState, paths.Journal, - paths.ProviderEnv, paths.ProbeEnv, paths.AgentEnv, + paths.ProviderEnv, paths.ProbeEnv, paths.AgentEnv, paths.CAKey, paths.ContainersConf, paths.CAFile, paths.ServerCert, paths.ServerKey, }, managedWiringPaths(paths)...) } -func snapshotExisted(snapshots []managedFileSnapshot, path string) bool { - for _, snapshot := range snapshots { - if snapshot.Path == path { - return snapshot.Existed - } - } - return false -} - func previouslyLoadedWatchUnits(states map[string]systemdUnitState) []string { units := make([]string, 0, 2) if _, found := states[refreshPathUnit]; found { @@ -1939,7 +2185,7 @@ func managedWiringIntent(paths LifecyclePaths, units SystemdUnits, present bool) } func snapshotManagedFiles(paths LifecyclePaths) ([]managedFileSnapshot, error) { - if err := os.MkdirAll(paths.Root, 0o700); err != nil { + if err := mkdirAllDurable(paths.Root, 0o700); err != nil { return nil, err } backupRoot, err := os.MkdirTemp(paths.Root, ".install-backup-") @@ -2129,13 +2375,13 @@ func replaceRegularFile(source, destination string, mode os.FileMode, maxBytes i if err != nil { return err } - defer input.Close() + defer func() { _ = input.Close() }() opened, err := input.Stat() if err != nil || !opened.Mode().IsRegular() || !os.SameFile(sourceInfo, opened) { return errors.New("replacement source changed during open") } directory := filepath.Dir(destination) - if err := os.MkdirAll(directory, 0o700); err != nil { + if err := mkdirAllDurable(directory, 0o700); err != nil { return err } if err := rejectWritableDestination(destination); err != nil { diff --git a/internal/retainedprovider/systemd_test.go b/internal/retainedprovider/systemd_test.go index 96278e3..513fbfd 100644 --- a/internal/retainedprovider/systemd_test.go +++ b/internal/retainedprovider/systemd_test.go @@ -46,9 +46,31 @@ func TestRenderSystemdUnitsUsesStableAbsolutePathsAndNoShell(t *testing.T) { t.Fatalf("provider unit missing %q:\n%s", required, units.ProviderService) } } - if !strings.Contains(units.RefreshService, "ExecStart="+systemdQuote(paths.Launcher)+" retained refresh -config "+systemdQuote(paths.ConfigFile)) || !strings.Contains(units.RefreshService, "Type=oneshot") || !strings.Contains(units.RefreshService, "TimeoutStartSec=15min") { + wantRefreshStartTimeout := "TimeoutStartSec=" + strconv.FormatInt(ceilDurationSeconds(retainedRefreshServiceStartTimeout), 10) + "s" + wantRefreshStopTimeout := "TimeoutStopSec=" + strconv.FormatInt(ceilDurationSeconds(retainedRefreshServiceStopTimeout), 10) + "s" + if !strings.Contains(units.RefreshService, "ExecStart="+systemdQuote(paths.Launcher)+" retained refresh -config "+systemdQuote(paths.ConfigFile)) || !strings.Contains(units.RefreshService, "Type=oneshot") || !strings.Contains(units.RefreshService, wantRefreshStartTimeout) || !strings.Contains(units.RefreshService, wantRefreshStopTimeout) { t.Fatalf("refresh service = %s", units.RefreshService) } + for directive, minimum := range map[string]time.Duration{ + "TimeoutStartSec": retainedRefreshServiceStartTimeout, + "TimeoutStopSec": retainedRefreshServiceStopTimeout, + } { + var rendered time.Duration + for _, line := range strings.Split(units.RefreshService, "\n") { + value, found := strings.CutPrefix(line, directive+"=") + if !found { + continue + } + seconds, err := strconv.ParseInt(strings.TrimSuffix(value, "s"), 10, 64) + if err != nil || !strings.HasSuffix(value, "s") { + t.Fatalf("parse %s=%q: %v", directive, value, err) + } + rendered = time.Duration(seconds) * time.Second + } + if rendered < minimum { + t.Fatalf("%s = %s want at least %s", directive, rendered, minimum) + } + } if !strings.Contains(units.RefreshPath, "PathChanged="+systemdPathValue(config.ProviderMarkerPath)) || !strings.Contains(units.RefreshPath, "Unit="+refreshServiceUnit) { t.Fatalf("refresh path = %s", units.RefreshPath) } @@ -68,6 +90,52 @@ func TestRenderSystemdUnitsUsesStableAbsolutePathsAndNoShell(t *testing.T) { } } +func ceilDurationSeconds(duration time.Duration) int64 { + return int64((duration + time.Second - 1) / time.Second) +} + +func TestRetainedTimeoutsCoverBoundedRefreshAndRollbackOperations(t *testing.T) { + localStatusBudget := time.Duration(localStatusAttempts)*controlCommandTimeout + time.Duration(localStatusAttempts-1)*time.Second + probeBudget := 5*providerProbeTimeout + 250*time.Millisecond + 500*time.Millisecond + time.Second + 2*time.Second + probeOwnershipBudget := 4 * providerProbeAttemptCount * controlCommandTimeout + controlAndFilesystemMargin := 20*controlCommandTimeout + 5*time.Minute + minimumRefresh := 3*localStatusBudget + providerBuildTimeout + 2*(probeBudget+probeOwnershipBudget) + 2*containerStartTimeout + controlAndFilesystemMargin + if retainedRefreshTimeout < minimumRefresh { + t.Fatalf("refresh timeout = %s want at least %s", retainedRefreshTimeout, minimumRefresh) + } + minimumRollback := probeBudget + probeOwnershipBudget + 6*controlCommandTimeout + 5*time.Minute + if retainedRollbackTimeout < minimumRollback { + t.Fatalf("rollback timeout = %s want at least %s", retainedRollbackTimeout, minimumRollback) + } + minimumServiceStart := 2*lifecycleRecoveryTimeout + retainedRollbackTimeout + installRollbackTimeout + retainedRefreshTimeout + if retainedRefreshServiceStartTimeout < minimumServiceStart { + t.Fatalf("refresh service start timeout = %s want at least %s", retainedRefreshServiceStartTimeout, minimumServiceStart) + } + if retainedRefreshServiceStopTimeout < lifecycleRecoveryTimeout { + t.Fatalf("refresh service stop timeout = %s want at least %s", retainedRefreshServiceStopTimeout, lifecycleRecoveryTimeout) + } + + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + now := time.Unix(1_700_000_000, 0).UTC() + journal := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, + ID: "rollback-timeout", + Phase: JournalStaging, + Candidate: ImageSelection{Update: validTestSelection(now).Update}, + StartedAt: now, + UpdatedAt: now, + } + runner := &recordingCommandRunner{} + if err := AtomicWriteJSON(paths.Journal, journal); err != nil { + t.Fatalf("write rollback journal: %v", err) + } + if err := (Refresher{Runner: runner}).rollback(t.Context(), config, paths, journal, false); err != nil { + t.Fatalf("rollback timeout budget: %v", err) + } +} + func TestSystemdQuoteEscapesSpecifierExpansion(t *testing.T) { if got, want := systemdQuote(`/home/user%name/"provider"`), `"/home/user%%name/\"provider\""`; got != want { t.Fatalf("systemdQuote = %q want %q", got, want) @@ -154,7 +222,7 @@ func TestGenerateInstallMaterialSeparatesProviderProbeAndAgentSecrets(t *testing if err := WriteInstallMaterial(paths, material); err != nil { t.Fatalf("write install material: %v", err) } - for _, path := range []string{paths.ProviderEnv, paths.ProbeEnv, paths.AgentEnv, paths.ContainersConf, paths.CAFile, paths.ServerCert, paths.ServerKey} { + for _, path := range []string{paths.ProviderEnv, paths.ProbeEnv, paths.AgentEnv, paths.ContainersConf, paths.CAFile, paths.CAKey, paths.ServerCert, paths.ServerKey} { info, err := os.Stat(path) if err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != 0o600 { t.Fatalf("generated file %s mode=%v err=%v", path, info, err) @@ -163,6 +231,109 @@ func TestGenerateInstallMaterialSeparatesProviderProbeAndAgentSecrets(t *testing if data, err := os.ReadFile(paths.ContainersConf); err != nil || string(data) != "[network]\ndefault_network = \"wfcompute-github-provider\"\n" { t.Fatalf("containers.conf data=%q err=%v", data, err) } + if strings.HasPrefix(paths.CAKey, paths.TLSRoot+string(os.PathSeparator)) { + t.Fatalf("CA signing key is exposed through the provider TLS mount: %s", paths.CAKey) + } +} + +func TestCredentialsRespectEnvironmentReaderBoundary(t *testing.T) { + const credentialLimit = 32 << 10 + if err := validateCredential(strings.Repeat("x", credentialLimit)); err != nil { + t.Fatalf("credential at limit: %v", err) + } + if err := validateCredential(strings.Repeat("x", credentialLimit+1)); err == nil { + t.Fatal("credential above environment reader limit was accepted") + } +} + +func TestRenewProviderServerCertificateKeepsCAAndServerKeyStable(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + issuedAt := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + material, err := GenerateInstallMaterial(config, Credentials{GitHubToken: "github-secret", ProviderToken: "provider-secret"}, bytes.NewReader(bytes.Repeat([]byte{0x44}, 4096)), issuedAt) + if err != nil { + t.Fatalf("generate install material: %v", err) + } + if err := WriteInstallMaterial(paths, material); err != nil { + t.Fatalf("write install material: %v", err) + } + renewedAt := issuedAt.Add(350 * 24 * time.Hour) + renewed, err := renewProviderServerCertificate(config, paths, bytes.NewReader(bytes.Repeat([]byte{0x45}, 4096)), renewedAt) + if err != nil { + t.Fatalf("renew server certificate: %v", err) + } + if !renewed { + t.Fatal("expiring server certificate was not renewed") + } + for path, want := range map[string][]byte{paths.CAFile: material.CACert, paths.CAKey: material.CAKey, paths.ServerKey: material.ServerKey} { + got, err := os.ReadFile(path) + if err != nil || !bytes.Equal(got, want) { + t.Fatalf("stable TLS authority %s changed: err=%v", path, err) + } + } + renewedPEM, err := os.ReadFile(paths.ServerCert) + if err != nil { + t.Fatalf("read renewed server certificate: %v", err) + } + if bytes.Equal(renewedPEM, material.ServerCert) { + t.Fatal("server certificate bytes did not change") + } + renewedCertificate := parseCertificateForTest(t, renewedPEM) + if !renewedCertificate.NotAfter.After(parseCertificateForTest(t, material.ServerCert).NotAfter) { + t.Fatalf("renewed expiry = %s", renewedCertificate.NotAfter) + } + pool := x509.NewCertPool() + pool.AddCert(parseCertificateForTest(t, material.CACert)) + if _, err := renewedCertificate.Verify(x509.VerifyOptions{Roots: pool, DNSName: config.StableContainer, CurrentTime: renewedAt.Add(time.Hour)}); err != nil { + t.Fatalf("verify renewed server certificate: %v", err) + } + renewed, err = renewProviderServerCertificate(config, paths, bytes.NewReader(bytes.Repeat([]byte{0x46}, 4096)), renewedAt) + if err != nil || renewed { + t.Fatalf("fresh certificate renewed=%v err=%v", renewed, err) + } +} + +func TestProviderServerCertificateRejectsFutureValidity(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + now := time.Date(2026, 7, 14, 0, 0, 0, 0, time.UTC) + material, err := GenerateInstallMaterial(config, Credentials{GitHubToken: "github-secret", ProviderToken: "provider-secret"}, nil, now.Add(-time.Hour)) + if err != nil { + t.Fatalf("generate install material: %v", err) + } + if err := WriteInstallMaterial(paths, material); err != nil { + t.Fatalf("write install material: %v", err) + } + + caKeyBlock, _ := pem.Decode(material.CAKey) + serverKeyBlock, _ := pem.Decode(material.ServerKey) + if caKeyBlock == nil || serverKeyBlock == nil { + t.Fatal("decode generated private keys") + } + caKey, err := x509.ParseECPrivateKey(caKeyBlock.Bytes) + if err != nil { + t.Fatalf("parse CA key: %v", err) + } + serverKey, err := x509.ParseECPrivateKey(serverKeyBlock.Bytes) + if err != nil { + t.Fatalf("parse server key: %v", err) + } + template := *parseCertificateForTest(t, material.ServerCert) + template.NotBefore = now.Add(time.Hour) + template.NotAfter = now.AddDate(1, 0, 0) + der, err := x509.CreateCertificate(bytes.NewReader(bytes.Repeat([]byte{0x47}, 4096)), &template, parseCertificateForTest(t, material.CACert), &serverKey.PublicKey, caKey) + if err != nil { + t.Fatalf("create future-dated server certificate: %v", err) + } + if err := atomicWriteFile(paths.ServerCert, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600); err != nil { + t.Fatalf("write future-dated server certificate: %v", err) + } + + if _, err := providerServerCertificateNeedsRenewal(config, paths, now); err == nil || !strings.Contains(err.Error(), "not currently valid") { + t.Fatalf("future-dated server certificate err = %v", err) + } } func TestRenderSystemdEnvironmentQuotesSpecialCharacters(t *testing.T) { @@ -298,6 +469,12 @@ func TestInstallTransactionOrdersMaintenanceAgentAndProviderActivation(t *testin statusQueue := []string{"unavailable", "unavailable", "idle"} runner := &recordingCommandRunner{} runner.run = func(_ context.Context, command Command) ([]byte, error) { + if command.Path == config.LoginctlPath { + return []byte("yes\n"), nil + } + if command.Path == config.PodmanPath && containsArg(command.Args, "info") { + return []byte("true\n"), nil + } if command.Path == config.PodmanPath && len(command.Args) >= 2 && command.Args[0] == "image" && command.Args[1] == "inspect" { return []byte(testProviderImageID + "\n"), nil } @@ -322,13 +499,19 @@ func TestInstallTransactionOrdersMaintenanceAgentAndProviderActivation(t *testin if err != nil || !found || journal.Phase != LifecycleFencing { t.Fatalf("install maintenance begin lifecycle = %+v found=%v err=%v", journal, found, err) } - return maintenanceStateJSON(true, installMaintenanceID, config.ProfileID, installMaintenanceReason), nil + if !containsAdjacentArgs(command.Args, "-id", journal.TransactionID) { + t.Fatalf("install maintenance id is not the lifecycle transaction: %+v", command) + } + return maintenanceStateJSON(true, journal.TransactionID, config.ProfileID, installMaintenanceReason), nil case "maintenance-end": journal, found, err := readLifecycleJournal(home, paths) if err != nil || !found || journal.Phase != LifecycleReleasing || journal.Outcome != LifecycleCommit || journal.ProviderTransaction == nil { t.Fatalf("install maintenance end lifecycle = %+v found=%v err=%v", journal, found, err) } - return maintenanceStateJSON(false, installMaintenanceID, config.ProfileID, installMaintenanceReason), nil + if !containsAdjacentArgs(command.Args, "-id", journal.TransactionID) { + t.Fatalf("install maintenance release id is not the lifecycle transaction: %+v", command) + } + return maintenanceStateJSON(false, journal.TransactionID, config.ProfileID, installMaintenanceReason), nil case "local-status": if len(statusQueue) == 0 { t.Fatal("unexpected extra local status read") @@ -390,6 +573,99 @@ func TestInstallTransactionOrdersMaintenanceAgentAndProviderActivation(t *testin } } +func TestInstallHostPreflightRequiresNonRootLingeringAndRootlessPodman(t *testing.T) { + config := validTestConfig(t.TempDir()) + writeLifecycleRecoveryFiles(t, config) + for _, tc := range []struct { + name string + uid string + linger string + rootless string + want string + wantLoginctl bool + }{ + {name: "root user", uid: "0", want: "non-root"}, + {name: "linger disabled", uid: "1001", linger: "no\n", rootless: "true\n", want: "lingering", wantLoginctl: true}, + {name: "rootful podman", uid: "1001", linger: "yes\n", rootless: "false\n", want: "rootless", wantLoginctl: true}, + } { + t.Run(tc.name, func(t *testing.T) { + runner := &recordingCommandRunner{run: func(_ context.Context, command Command) ([]byte, error) { + if filepath.Base(command.Path) == "systemctl" && command.Path != config.SystemctlPath { + t.Fatalf("systemctl command path = %q want %q", command.Path, config.SystemctlPath) + } + if filepath.Base(command.Path) == "loginctl" && command.Path != config.LoginctlPath { + t.Fatalf("loginctl command path = %q want %q", command.Path, config.LoginctlPath) + } + switch command.Path { + case config.LoginctlPath: + return []byte(tc.linger), nil + case config.PodmanPath: + if containsArg(command.Args, "info") { + return []byte(tc.rootless), nil + } + return []byte("5.5.0\n"), nil + default: + return nil, nil + } + }} + installer := Installer{Runner: runner, UserID: func() (string, error) { return tc.uid, nil }} + err := installer.runSystemPreflight(t.Context(), config) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("preflight error = %v want %q", err, tc.want) + } + transcript := commandTranscript(runner.commands) + if strings.Contains(transcript, "supervisor-update verify") { + t.Fatalf("host preflight crossed package verification boundary:\n%s", transcript) + } + if tc.wantLoginctl != strings.Contains(transcript, "loginctl show-user "+tc.uid+" --property Linger --value") { + t.Fatalf("loginctl invocation mismatch:\n%s", transcript) + } + }) + } +} + +func TestHostPreflightRejectsUntrustedExecutableBeforeCommands(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + writeLifecycleRecoveryFiles(t, config) + if err := os.Chmod(config.PodmanPath, 0o722); err != nil { + t.Fatalf("make podman fixture group-writable: %v", err) + } + runner := &recordingCommandRunner{} + installer := Installer{Runner: runner, UserID: func() (string, error) { return "1001", nil }} + if err := installer.runSystemPreflight(t.Context(), config); err == nil || !strings.Contains(err.Error(), "writable") { + t.Fatalf("untrusted executable preflight err = %v", err) + } + if len(runner.commands) != 0 { + t.Fatalf("untrusted executable preflight issued commands: %+v", runner.commands) + } +} + +func TestWaitLocalStateRequiresObservationAfterFence(t *testing.T) { + config := validTestConfig(t.TempDir()) + fenceStartedAt := time.Date(2026, 7, 14, 14, 0, 0, 0, time.UTC) + statuses := [][]byte{ + localStatusAtJSON(config.WorkerID, "unavailable", fenceStartedAt.Add(-time.Second)), + localStatusAtJSON(config.WorkerID, "unavailable", fenceStartedAt.Add(time.Second)), + } + reads := 0 + runner := &recordingCommandRunner{run: func(_ context.Context, command Command) ([]byte, error) { + if !containsArg(command.Args, "local-status") { + t.Fatalf("unexpected command: %+v", command) + } + result := statuses[reads] + reads++ + return result, nil + }} + installer := Installer{Runner: runner, Sleep: func(context.Context, time.Duration) error { return nil }} + if err := installer.waitLocalStateAfter(t.Context(), config, "unavailable", fenceStartedAt); err != nil { + t.Fatalf("wait for fresh local state: %v", err) + } + if reads != 2 { + t.Fatalf("local status reads = %d want 2", reads) + } +} + func TestInstallReattestsAuthorityAfterDrainBeforeStop(t *testing.T) { home := t.TempDir() config := validTestConfig(home) @@ -423,6 +699,44 @@ func TestInstallReattestsAuthorityAfterDrainBeforeStop(t *testing.T) { } } +func TestInstallFailureEmitsRedactedErrorAndRecoveryAudit(t *testing.T) { + home := t.TempDir() + t.Setenv("XDG_STATE_HOME", filepath.Join(home, ".state")) + config := validTestConfig(home) + payload := writeTestProviderPayload(t, home, "verified-provider-audit-failure") + digest := fileDigestForTest(t, payload) + runner := installSuccessRunner(t, config, payload, digest, new([]string)) + baseRun := runner.run + failed := false + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if !failed && installCommandEvent(command, config) == "provider-enable" { + failed = true + return nil, errors.New("provider-enable-secret-detail") + } + return baseRun(ctx, command) + } + installer := Installer{ + Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x35}, 4096)), + Sleep: func(context.Context, time.Duration) error { return nil }, + } + if _, err := installer.Install(t.Context(), home, config, Credentials{GitHubToken: "github-secret", ProviderToken: "provider-secret"}); err == nil { + t.Fatal("install failure was not returned") + } + audit, err := os.ReadFile(LifecyclePathsFor(config).LifecycleAudit) + if err != nil { + t.Fatalf("read lifecycle audit: %v", err) + } + for _, want := range []string{`"kind":"error"`, `"error_class":"operation_failed"`, `"kind":"recovery"`, `"disposition":"resume_fenced"`} { + if !bytes.Contains(audit, []byte(want)) { + t.Fatalf("lifecycle audit missing %s:\n%s", want, audit) + } + } + if bytes.Contains(audit, []byte("provider-enable-secret-detail")) || bytes.Contains(audit, []byte("github-secret")) || bytes.Contains(audit, []byte("provider-secret")) { + t.Fatalf("lifecycle audit leaked failure or credentials:\n%s", audit) + } +} + func TestInstallReattestsAuthorityBeforeRestart(t *testing.T) { home := t.TempDir() config := validTestConfig(home) @@ -553,7 +867,7 @@ func TestInstallLockContentionDoesNotMutateMaintenanceOrAgent(t *testing.T) { if err != nil { t.Fatalf("hold install lock: %v", err) } - defer lock.Release() + defer func() { _ = lock.Release() }() statuses := []string{"unavailable", "unavailable"} runner := installSuccessRunner(t, config, payload, digest, &statuses) installer := Installer{ @@ -640,6 +954,107 @@ func TestInstallCredentialRotationPreservesProviderStateAndWorkerIdentity(t *tes } } +func TestReinstallMissingActiveImageUsesChangedProviderTransaction(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + payload := writeTestProviderPayload(t, home, "verified-provider-reinstall-repair") + digest := fileDigestForTest(t, payload) + statuses := []string{"unavailable", "unavailable", "idle", "unavailable", "unavailable", "idle"} + runner := installSuccessRunner(t, config, payload, digest, &statuses) + baseRun := runner.run + imagePresent := false + runner.run = func(ctx context.Context, command Command) ([]byte, error) { + if command.Path == config.PodmanPath && firstArg(command.Args) == "images" && containsAdjacentArgs(command.Args, "--filter", "id="+testProviderImageID) { + if !imagePresent { + return nil, nil + } + return ownedProviderImageInventory(config, digest, testProviderImageID), nil + } + if command.Path == config.PodmanPath && firstArg(command.Args) == "build" { + imagePresent = true + } + return baseRun(ctx, command) + } + newInstaller := func() Installer { + return Installer{ + Runner: runner, ExecutablePath: func() (string, error) { return payload, nil }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x61}, 4096)), + Now: func() time.Time { return time.Unix(1_700_000_000, 0).UTC() }, + Sleep: func(context.Context, time.Duration) error { return nil }, + } + } + if _, err := newInstaller().Install(t.Context(), home, config, Credentials{GitHubToken: "github-secret", ProviderToken: "provider-secret"}); err != nil { + t.Fatalf("initial install: %v", err) + } + imagePresent = false + commandCount := len(runner.commands) + if _, err := newInstaller().Install(t.Context(), home, config, Credentials{GitHubToken: "github-secret", ProviderToken: "provider-secret"}); err != nil { + t.Fatalf("repairing reinstall: %v\n%s", err, commandTranscript(runner.commands[commandCount:])) + } + if transcript := commandTranscript(runner.commands[commandCount:]); !strings.Contains(transcript, "podman build") { + t.Fatalf("repairing reinstall did not rebuild missing active image:\n%s", transcript) + } +} + +func TestReinstallRejectsChangedInstalledConfigBeforeCommands(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + if err := os.MkdirAll(paths.Root, 0o700); err != nil { + t.Fatalf("create install root: %v", err) + } + if err := AtomicWriteJSON(paths.ConfigFile, config); err != nil { + t.Fatalf("write installed config: %v", err) + } + changed := config + changed.WorkerID = "different-retained-worker" + changed.AgentUnit = "workflow-compute-different-retained-worker.service" + changed.Labels = append([]string(nil), config.Labels...) + + runner := &recordingCommandRunner{} + installer := Installer{Runner: runner} + if _, err := installer.Install(t.Context(), home, changed, Credentials{GitHubToken: "github-new", ProviderToken: "provider-new"}); err == nil || !strings.Contains(err.Error(), "must exactly match") { + t.Fatalf("identity-changing reinstall err = %v", err) + } + if len(runner.commands) != 0 { + t.Fatalf("identity-changing reinstall issued commands: %+v", runner.commands) + } +} + +func TestRefreshAndUninstallRejectChangedInstalledConfigBeforeCommands(t *testing.T) { + for _, operation := range []string{"refresh", "uninstall"} { + t.Run(operation, func(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + if err := os.MkdirAll(paths.Root, 0o700); err != nil { + t.Fatalf("create install root: %v", err) + } + if err := AtomicWriteJSON(paths.ConfigFile, config); err != nil { + t.Fatalf("write installed config: %v", err) + } + changed := config + changed.WorkerID = "different-retained-worker" + changed.AgentUnit = "workflow-compute-different-retained-worker.service" + changed.Labels = append([]string(nil), config.Labels...) + runner := &recordingCommandRunner{} + var err error + switch operation { + case "refresh": + _, err = (Refresher{Runner: runner}).Refresh(t.Context(), changed) + case "uninstall": + _, err = (Installer{Runner: runner}).Uninstall(t.Context(), home, changed, false) + } + if err == nil || !strings.Contains(err.Error(), "must exactly match") { + t.Fatalf("identity-changing %s err = %v", operation, err) + } + if len(runner.commands) != 0 { + t.Fatalf("identity-changing %s issued commands: %+v", operation, runner.commands) + } + }) + } +} + func TestInstallLeavesMaintenanceActiveWhenRollbackCannotRestartAgent(t *testing.T) { home := t.TempDir() config := validTestConfig(home) @@ -886,9 +1301,10 @@ func TestInstallRollbackPreservesPreviouslyDisabledProviderUnits(t *testing.T) { } func TestRestoreUnitStatePreservesRuntimeEnablement(t *testing.T) { + systemctlPath := validTestConfig(t.TempDir()).SystemctlPath runner := &recordingCommandRunner{run: func(context.Context, Command) ([]byte, error) { return nil, nil }} installer := Installer{Runner: runner} - if err := installer.restoreUnitState(t.Context(), providerServiceUnit, systemdUnitState{LoadState: "loaded", FragmentPath: "/tmp/provider.service", UnitFileState: "enabled-runtime", ActiveState: "active"}); err != nil { + if err := installer.restoreUnitState(t.Context(), systemctlPath, providerServiceUnit, systemdUnitState{LoadState: "loaded", FragmentPath: "/tmp/provider.service", UnitFileState: "enabled-runtime", ActiveState: "active"}); err != nil { t.Fatalf("restore runtime-enabled unit: %v", err) } transcript := commandTranscript(runner.commands) @@ -906,11 +1322,12 @@ func TestCaptureManagedUnitStatesRejectsUnrestorableSemantics(t *testing.T) { {name: "failed unit", state: systemdUnitState{LoadState: "loaded", FragmentPath: "/tmp/provider.service", UnitFileState: "enabled", ActiveState: "failed"}}, } { t.Run(tc.name, func(t *testing.T) { + systemctlPath := validTestConfig(t.TempDir()).SystemctlPath runner := &recordingCommandRunner{run: func(context.Context, Command) ([]byte, error) { return []byte("LoadState=" + tc.state.LoadState + "\nFragmentPath=" + tc.state.FragmentPath + "\nActiveState=" + tc.state.ActiveState + "\nUnitFileState=" + tc.state.UnitFileState + "\n"), nil }} installer := Installer{Runner: runner} - if _, err := installer.captureManagedUnitStates(t.Context()); err == nil || !strings.Contains(err.Error(), "unsupported prior") { + if _, err := installer.captureManagedUnitStates(t.Context(), systemctlPath); err == nil || !strings.Contains(err.Error(), "unsupported prior") { t.Fatalf("capture state %+v err = %v", tc.state, err) } }) @@ -918,11 +1335,12 @@ func TestCaptureManagedUnitStatesRejectsUnrestorableSemantics(t *testing.T) { } func TestCaptureManagedUnitStatesIncludesUnitsLoadedOutsideManagedPaths(t *testing.T) { + systemctlPath := validTestConfig(t.TempDir()).SystemctlPath fragment := "/usr/lib/systemd/user/vendor-provider.service" runner := &recordingCommandRunner{run: func(context.Context, Command) ([]byte, error) { return []byte("LoadState=loaded\nFragmentPath=" + fragment + "\nActiveState=active\nUnitFileState=enabled\n"), nil }} - states, err := (Installer{Runner: runner}).captureManagedUnitStates(t.Context()) + states, err := (Installer{Runner: runner}).captureManagedUnitStates(t.Context(), systemctlPath) if err != nil { t.Fatalf("capture loaded units: %v", err) } @@ -1014,7 +1432,7 @@ func TestInstallRecoversDeferredCommittedRefreshAfterProcessRestart(t *testing.T payload := writeTestProviderPayload(t, home, "verified-provider-deferred-install-recovery") digest := fileDigestForTest(t, payload) now := time.Unix(1_700_300_000, 0).UTC() - selection := selectionForDigest(payload, digest, "v1.0.32", "directive-deferred-recovery", "sha256:"+strings.Repeat("d", 64), now) + selection := selectionForDigest(payload, digest, "v1.0.32", "directive-deferred-recovery", testProviderImageID, now) active := ActiveState{ProtocolVersion: ActiveStateProtocolVersion, Current: selection, UpdatedAt: now} if err := AtomicWriteJSON(paths.ActiveState, active); err != nil { t.Fatalf("write committed active state: %v", err) @@ -1069,6 +1487,58 @@ func TestInstallRecoversDeferredCommittedRefreshAfterProcessRestart(t *testing.T } } +func TestRecoverInstallRejectsCrossWorkerDeferredJournalBeforeMutation(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + paths := LifecyclePathsFor(config) + now := time.Unix(1_700_300_000, 0).UTC() + payload := writeTestProviderPayload(t, home, "cross-worker-deferred-recovery") + digest := fileDigestForTest(t, payload) + selection := selectionForDigest(payload, digest, "v1.0.32", "directive-cross-worker", "sha256:"+strings.Repeat("d", 64), now) + selection.Update.WorkerID = "other-retained-worker" + providerJournal := TransactionJournal{ + ProtocolVersion: TransactionJournalProtocolVersion, + ID: "refresh-cross-worker-deferred", + Phase: JournalCommitted, + DeferredCommit: true, + Candidate: selection, + StartedAt: now, + UpdatedAt: now, + } + if err := AtomicWriteJSON(paths.Journal, providerJournal); err != nil { + t.Fatalf("write cross-worker provider journal: %v", err) + } + transactionRoot := filepath.Dir(paths.CandidateState(digest)) + if err := os.MkdirAll(transactionRoot, 0o700); err != nil { + t.Fatalf("create provider transaction root: %v", err) + } + sentinel := filepath.Join(transactionRoot, "must-remain") + if err := os.WriteFile(sentinel, []byte("retained"), 0o600); err != nil { + t.Fatalf("write provider transaction sentinel: %v", err) + } + snapshots, err := snapshotManagedFiles(paths) + if err != nil { + t.Fatalf("snapshot outer install: %v", err) + } + outer := newInstallTransactionJournal("install", snapshots, map[string]systemdUnitState{}, now) + outer.Phase = installTransactionCommitted + if err := writeInstallTransactionJournal(paths, outer); err != nil { + t.Fatalf("write outer install journal: %v", err) + } + runner := &recordingCommandRunner{} + + err = (Installer{Runner: runner}).recoverInstallTransaction(t.Context(), config, paths, Refresher{Runner: runner}) + if err == nil || !strings.Contains(err.Error(), "identity") { + t.Fatalf("cross-worker deferred recovery err = %v", err) + } + if len(runner.commands) != 0 { + t.Fatalf("cross-worker deferred recovery issued commands: %+v", runner.commands) + } + if data, readErr := os.ReadFile(sentinel); readErr != nil || string(data) != "retained" { + t.Fatalf("cross-worker deferred recovery mutated transaction state: %q err=%v", data, readErr) + } +} + func TestInstallCrashRecoveryRestoresDurableOuterBaselineBeforeRetry(t *testing.T) { home := t.TempDir() config := validTestConfig(home) @@ -1331,7 +1801,7 @@ func TestUninstallCleanupFailureStillReleasesMaintenance(t *testing.T) { home := t.TempDir() config := validTestConfig(home) paths := LifecyclePathsFor(config) - if err := atomicWriteFile(paths.ConfigFile, []byte("previous-config\n"), 0o600); err != nil { + if err := AtomicWriteJSON(paths.ConfigFile, config); err != nil { t.Fatalf("write previous config: %v", err) } statuses := []string{"unavailable", "unavailable"} @@ -1475,6 +1945,7 @@ func TestInstallBoundsTransientLocalStatusPolling(t *testing.T) { func TestInstallerStatusReportsOnlyLocalRedactedLifecycleState(t *testing.T) { home := t.TempDir() config := validTestConfig(home) + writeLifecycleRecoveryFiles(t, config) paths := LifecyclePathsFor(config) active := previousActiveStateForTest(t, home) if err := AtomicWriteJSON(paths.ActiveState, active); err != nil { @@ -1567,7 +2038,7 @@ func TestRollbackInstallPreservesSnapshotsUntilAgentAndMaintenanceRestore(t *tes return nil, nil }} snapshots := []managedFileSnapshot{{Path: destination, Backup: backup, Mode: 0o600, Existed: true}} - err := (Installer{Runner: runner}).rollbackInstall(t.Context(), config, snapshots, map[string]systemdUnitState{}, true, true, installMaintenanceID, systemdActivation{}) + err := (Installer{Runner: runner}).rollbackInstall(t.Context(), config, snapshots, map[string]systemdUnitState{}, true, true, installMaintenanceID, installMaintenanceReason, systemdActivation{}) if err == nil || !strings.Contains(err.Error(), "agent restart failed") { t.Fatalf("rollback err = %v", err) } @@ -1576,6 +2047,41 @@ func TestRollbackInstallPreservesSnapshotsUntilAgentAndMaintenanceRestore(t *tes } } +func TestRollbackInstallBeforeStartRetainsSnapshotsForLifecycleCommit(t *testing.T) { + home := t.TempDir() + config := validTestConfig(home) + backupRoot := filepath.Join(config.InstallRoot, ".install-backup-lifecycle") + backup := filepath.Join(backupRoot, "0") + destination := LifecyclePathsFor(config).AgentEnv + if err := atomicWriteFile(backup, []byte("ORIGINAL_AGENT_ENV=1\n"), 0o600); err != nil { + t.Fatalf("write backup: %v", err) + } + if err := atomicWriteFile(destination, []byte("PARTIAL_AGENT_ENV=1\n"), 0o600); err != nil { + t.Fatalf("write partial destination: %v", err) + } + snapshots := []managedFileSnapshot{{Path: destination, Backup: backup, Mode: 0o600, Existed: true}} + if err := (Installer{Runner: &recordingCommandRunner{}}).rollbackInstallBeforeStart( + t.Context(), config, snapshots, map[string]systemdUnitState{}, true, false, "", "", systemdActivation{}, func(ctx context.Context) error { + deadline, ok := ctx.Deadline() + if !ok { + t.Fatal("rollback callback has no deadline") + } + if remaining := time.Until(deadline); remaining < 2*controlCommandTimeout { + t.Fatalf("rollback callback deadline = %s want at least %s", remaining, 2*controlCommandTimeout) + } + return nil + }, + ); err != nil { + t.Fatalf("rollback before lifecycle commit: %v", err) + } + if data, err := os.ReadFile(backup); err != nil || string(data) != "ORIGINAL_AGENT_ENV=1\n" { + t.Fatalf("lifecycle rollback discarded durable backup = %q err=%v", data, err) + } + if data, err := os.ReadFile(destination); err != nil || string(data) != "ORIGINAL_AGENT_ENV=1\n" { + t.Fatalf("lifecycle rollback did not restore destination = %q err=%v", data, err) + } +} + func TestRestoreManagedFilesPropagatesSnapshotCleanupFailure(t *testing.T) { root := t.TempDir() cleanupParent := filepath.Join(root, "cleanup-parent") @@ -1677,7 +2183,7 @@ func TestUninstallLockContentionDoesNotMutateMaintenanceOrAgent(t *testing.T) { if err != nil { t.Fatalf("hold install lock: %v", err) } - defer lock.Release() + defer func() { _ = lock.Release() }() runner := installSuccessRunner(t, config, "", "", new([]string)) installer := Installer{Runner: runner, Sleep: func(context.Context, time.Duration) error { return nil }} if _, err := installer.Uninstall(t.Context(), home, config, false); !errors.Is(err, ErrInstallLocked) { @@ -1697,12 +2203,21 @@ func installSuccessRunner(t *testing.T, config Config, payload, digest string, s activeMaintenanceReason := "" runner := &recordingCommandRunner{} runner.run = func(_ context.Context, command Command) ([]byte, error) { + if command.Path == config.LoginctlPath { + return []byte("yes\n"), nil + } if installCommandEvent(command, config) == "agent-signature" { return agentUnitSystemdOutputForTest(t, config), nil } + if command.Path == config.PodmanPath && containsArg(command.Args, "info") { + return []byte("true\n"), nil + } if command.Path == config.PodmanPath && len(command.Args) >= 2 && command.Args[0] == "image" && command.Args[1] == "inspect" { return []byte(testProviderImageID + "\n"), nil } + if command.Path == config.PodmanPath && firstArg(command.Args) == "images" && containsAdjacentArgs(command.Args, "--filter", "id="+testProviderImageID) { + return ownedProviderImageInventory(config, digest, testProviderImageID), nil + } if command.Path == config.PodmanPath && len(command.Args) >= 2 && command.Args[0] == "network" && command.Args[1] == "inspect" { return []byte("bridge true false\n"), nil } @@ -1729,11 +2244,8 @@ func installSuccessRunner(t *testing.T, config Config, payload, digest string, s return testVerifiedUpdateJSON(config, payload, digest), nil case "maintenance-begin": maintenanceActive = true - reason := installMaintenanceReason - id := installMaintenanceID - if containsArg(command.Args, uninstallMaintenanceID) { - reason, id = uninstallMaintenanceReason, uninstallMaintenanceID - } + id := adjacentArgValue(command.Args, "-id") + reason := adjacentArgValue(command.Args, "-reason") activeMaintenanceID, activeMaintenanceReason = id, reason return maintenanceStateJSON(true, id, config.ProfileID, reason), nil case "maintenance-status": @@ -1743,10 +2255,15 @@ func installSuccessRunner(t *testing.T, config Config, payload, digest string, s return maintenanceStateJSON(true, activeMaintenanceID, config.ProfileID, activeMaintenanceReason), nil case "maintenance-end": maintenanceActive = false - id := installMaintenanceID - reason := installMaintenanceReason - if containsArg(command.Args, uninstallMaintenanceID) { - id, reason = uninstallMaintenanceID, uninstallMaintenanceReason + id, reason := activeMaintenanceID, activeMaintenanceReason + if id == "" { + id = adjacentArgValue(command.Args, "-id") + switch id { + case installMaintenanceID: + reason = installMaintenanceReason + case uninstallMaintenanceID: + reason = uninstallMaintenanceReason + } } return maintenanceStateJSON(false, id, config.ProfileID, reason), nil case "local-status": @@ -1816,7 +2333,20 @@ func maintenanceStateJSON(active bool, id, profileID, reason string) []byte { } func localStatusJSON(workerID, state string) []byte { - return []byte(`{"protocol_version":"compute.local_status.v1","worker_id":"` + workerID + `","state":"` + state + `","updated_at":"2026-07-13T00:00:00Z"}`) + return []byte(`{"protocol_version":"compute.local_status.v1","worker_id":"` + workerID + `","state":"` + state + `","updated_at":"2099-01-01T00:00:00Z"}`) +} + +func localStatusAtJSON(workerID, state string, updatedAt time.Time) []byte { + return []byte(`{"protocol_version":"compute.local_status.v1","worker_id":"` + workerID + `","state":"` + state + `","updated_at":` + strconv.Quote(updatedAt.UTC().Format(time.RFC3339Nano)) + `}`) +} + +func adjacentArgValue(args []string, key string) string { + for index := 0; index+1 < len(args); index++ { + if args[index] == key { + return args[index+1] + } + } + return "" } func assertOrderedEvents(t *testing.T, events, expected []string) { diff --git a/release_packaging_test.go b/release_packaging_test.go index cce7b24..85552c2 100644 --- a/release_packaging_test.go +++ b/release_packaging_test.go @@ -1,9 +1,16 @@ package githubplugin_test import ( + "bytes" + "encoding/json" + "fmt" "os" + "path/filepath" "strings" "testing" + + "github.com/GoCodeAlone/workflow-plugin-github/internal/retainedprovider" + "github.com/santhosh-tekuri/jsonschema/v6" ) func TestReleaseArchiveIncludesGitHubRunnerProvider(t *testing.T) { @@ -29,6 +36,148 @@ func TestReleaseArchiveIncludesGitHubRunnerProvider(t *testing.T) { } } +func TestReleaseArchiveIncludesRetainedProviderOperations(t *testing.T) { + data, err := os.ReadFile(".goreleaser.yaml") + if err != nil { + t.Fatalf("read .goreleaser.yaml: %v", err) + } + archive := listItemWithID(topLevelSection(string(data), "archives:"), "workflow-plugin-github") + if !strings.Contains(archive, "- README.md") { + t.Fatal("release archive must include retained-provider operator documentation") + } + + readmeData, err := os.ReadFile("README.md") + if err != nil { + t.Fatalf("read README.md: %v", err) + } + readme := string(readmeData) + for _, want := range []string{ + "loginctl enable-linger", + "podman info --format '{{.Host.Security.Rootless}}'", + "examples/github-runner-retained-config.json", + "schemas/github-runner-retained-config.schema.json", + "github-runner-provider retained install -config", + "GITHUB_RUNNER_PROVIDER_GITHUB_TOKEN", + "GITHUB_RUNNER_PROVIDER_TOKEN", + "github-runner-provider retained status -config", + "github-runner-provider retained uninstall -config", + "github-runner-provider retained uninstall -config --purge", + "credential rotation", + "github-runner-provider retained recover -config", + "autonomous refresh", + "GitHub workflow output is orchestration evidence only", + "STG task, proof, log, and artifact APIs", + } { + if !strings.Contains(readme, want) { + t.Fatalf("retained-provider operations documentation is missing %q", want) + } + } +} + +func TestReleaseArchiveIncludesRetainedProviderConfigContract(t *testing.T) { + data, err := os.ReadFile(".goreleaser.yaml") + if err != nil { + t.Fatalf("read .goreleaser.yaml: %v", err) + } + archive := listItemWithID(topLevelSection(string(data), "archives:"), "workflow-plugin-github") + for _, path := range []string{ + "schemas/github-runner-retained-config.schema.json", + "examples/github-runner-retained-config.json", + } { + if !strings.Contains(archive, "- "+path) { + t.Fatalf("release archive must include retained-provider config contract %q", path) + } + } + + example, err := os.ReadFile("examples/github-runner-retained-config.json") + if err != nil { + t.Fatalf("read retained-provider config example: %v", err) + } + home := t.TempDir() + runtimeExample := bytes.ReplaceAll(example, []byte("/home/wfcompute"), []byte(filepath.ToSlash(home))) + config, err := retainedprovider.DecodeConfig(bytes.NewReader(runtimeExample), home) + if err != nil { + t.Fatalf("retained-provider config example must pass the shipped runtime decoder: %v", err) + } + if config.ProtocolVersion != retainedprovider.ConfigProtocolVersion || config.PluginID != retainedprovider.GitHubPluginID { + t.Fatalf("retained-provider config example has wrong identity: %+v", config) + } + schema, err := os.ReadFile("schemas/github-runner-retained-config.schema.json") + if err != nil { + t.Fatalf("read retained-provider config schema: %v", err) + } + if !json.Valid(schema) { + t.Fatal("retained-provider config schema must be valid JSON") + } + compiled, err := jsonschema.NewCompiler().Compile("schemas/github-runner-retained-config.schema.json") + if err != nil { + t.Fatalf("compile retained-provider config schema: %v", err) + } + var exampleDocument any + if err := json.Unmarshal(example, &exampleDocument); err != nil { + t.Fatalf("decode retained-provider config example for schema validation: %v", err) + } + if err := compiled.Validate(exampleDocument); err != nil { + t.Fatalf("retained-provider config example must pass the shipped schema: %v", err) + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(example, &fields); err != nil { + t.Fatalf("decode retained-provider config fields: %v", err) + } + validateFields := func(fields map[string]json.RawMessage) error { + data, err := json.Marshal(fields) + if err != nil { + t.Fatalf("marshal retained-provider schema fixture: %v", err) + } + var document any + if err := json.Unmarshal(data, &document); err != nil { + t.Fatalf("decode retained-provider schema fixture: %v", err) + } + return compiled.Validate(document) + } + delete(fields, "worker_id") + if err := validateFields(fields); err == nil { + t.Fatal("retained-provider config schema accepted an example without worker_id") + } + if err := json.Unmarshal(example, &fields); err != nil { + t.Fatalf("reset retained-provider config fields: %v", err) + } + delete(fields, "systemctl_path") + if err := validateFields(fields); err == nil { + t.Fatal("retained-provider config schema accepted an example without systemctl_path") + } + if err := json.Unmarshal(example, &fields); err != nil { + t.Fatalf("reset retained-provider config fields: %v", err) + } + labels := make([]string, 64) + for index := range labels { + labels[index] = fmt.Sprintf("label-%d", index) + } + fields["labels"], _ = json.Marshal(labels) + if err := validateFields(fields); err != nil { + t.Fatalf("retained-provider config schema rejected 64 labels: %v", err) + } + labels = append(labels, "label-64") + fields["labels"], _ = json.Marshal(labels) + if err := validateFields(fields); err == nil { + t.Fatal("retained-provider config schema accepted 65 labels") + } + fields["labels"] = json.RawMessage(`["self-hosted"]`) + fields["podman_path"], _ = json.Marshal("/podman") + if err := validateFields(fields); err != nil { + t.Fatalf("retained-provider config schema rejected runtime-valid root Podman path: %v", err) + } + fields["podman_path"], _ = json.Marshal("/usr/bin/podman\t") + if err := validateFields(fields); err == nil { + t.Fatal("retained-provider config schema accepted a control character in an absolute path") + } + for _, field := range []string{"protocol_version", "worker_id", "provider_marker_path", "podman_path", "systemctl_path", "loginctl_path", "ref", "refresh_interval_seconds"} { + if !bytes.Contains(schema, []byte(`"`+field+`"`)) { + t.Fatalf("retained-provider config schema is missing %q", field) + } + } +} + func TestGitHubRunnerProviderReleaseBuildInjectsVersion(t *testing.T) { data, err := os.ReadFile(".goreleaser.yaml") if err != nil { diff --git a/schemas/github-runner-retained-config.schema.json b/schemas/github-runner-retained-config.schema.json new file mode 100644 index 0000000..b9e6d12 --- /dev/null +++ b/schemas/github-runner-retained-config.schema.json @@ -0,0 +1,115 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/GoCodeAlone/workflow-plugin-github/schemas/github-runner-retained-config.schema.json", + "title": "Retained GitHub Runner Provider Configuration", + "description": "Non-secret Linux lifecycle configuration consumed by github-runner-provider retained commands.", + "type": "object", + "additionalProperties": false, + "required": [ + "protocol_version", + "worker_id", + "profile_id", + "plugin_id", + "component_id", + "compute_agent_path", + "supervisor_config_path", + "local_status_path", + "provider_marker_path", + "install_root", + "systemd_dir", + "agent_unit", + "podman_path", + "systemctl_path", + "loginctl_path", + "provider_url", + "stable_container", + "candidate_container", + "container_network", + "organization", + "repository", + "workflow", + "ref", + "runner_name", + "runner_group", + "labels", + "refresh_interval_seconds" + ], + "properties": { + "protocol_version": { "const": "retained-provider.config.v1" }, + "worker_id": { "$ref": "#/$defs/identifier" }, + "profile_id": { "$ref": "#/$defs/identifier" }, + "plugin_id": { "const": "workflow-plugin-github" }, + "component_id": { "$ref": "#/$defs/identifier" }, + "compute_agent_path": { "$ref": "#/$defs/absolutePath" }, + "supervisor_config_path": { "$ref": "#/$defs/absolutePath" }, + "local_status_path": { "$ref": "#/$defs/absolutePath" }, + "provider_marker_path": { "$ref": "#/$defs/absolutePath" }, + "install_root": { "$ref": "#/$defs/absolutePath" }, + "systemd_dir": { "$ref": "#/$defs/absolutePath" }, + "agent_unit": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}\\.service$" + }, + "podman_path": { + "type": "string", + "pattern": "^/(?:podman|[^\\x00-\\x1F\\x7F]*/podman)$" + }, + "systemctl_path": { + "type": "string", + "pattern": "^/(?:systemctl|[^\\x00-\\x1F\\x7F]*/systemctl)$" + }, + "loginctl_path": { + "type": "string", + "pattern": "^/(?:loginctl|[^\\x00-\\x1F\\x7F]*/loginctl)$" + }, + "provider_url": { + "type": "string", + "format": "uri", + "pattern": "^https://[A-Za-z0-9][A-Za-z0-9._-]{0,127}:18090/?$" + }, + "stable_container": { "$ref": "#/$defs/identifier" }, + "candidate_container": { "$ref": "#/$defs/identifier" }, + "container_network": { "const": "wfcompute-github-provider" }, + "organization": { "$ref": "#/$defs/identifier" }, + "repository": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}/[A-Za-z0-9][A-Za-z0-9._-]{0,127}$" + }, + "workflow": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._/-]*\\.ya?ml$", + "not": { "pattern": "\\.\\." } + }, + "ref": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "runner_name": { "$ref": "#/$defs/probeIdentifier" }, + "runner_group": { "$ref": "#/$defs/identifier" }, + "labels": { + "type": "array", + "minItems": 1, + "maxItems": 64, + "uniqueItems": true, + "items": { "$ref": "#/$defs/probeIdentifier" } + }, + "refresh_interval_seconds": { + "type": "integer", + "minimum": 60, + "maximum": 86400 + } + }, + "$defs": { + "identifier": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$" + }, + "probeIdentifier": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$" + }, + "absolutePath": { + "type": "string", + "pattern": "^/[^\\x00-\\x1F\\x7F]*$" + } + } +} From 2dbd9ea48f4921658e536d0ecb7437fb6772b25e Mon Sep 17 00:00:00 2001 From: Jon Langevin Date: Tue, 14 Jul 2026 18:57:04 -0400 Subject: [PATCH 12/16] fix(provider): require DNS-safe names Managed container names become TLS SANs and Podman aliases. Restrict base and derived names to lowercase DNS labels and keep the shipped schema aligned. --- ...tained-runner-provider-lifecycle-design.md | 14 ++++++++ ...d-runner-provider-lifecycle-plan-review.md | 1 + internal/retainedprovider/config.go | 4 +++ internal/retainedprovider/state_test.go | 33 +++++++++++++++++++ release_packaging_test.go | 24 ++++++++++++++ .../github-runner-retained-config.schema.json | 11 +++++-- 6 files changed, 84 insertions(+), 3 deletions(-) diff --git a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md index b11d9a9..fa236a2 100644 --- a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md +++ b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md @@ -1130,3 +1130,17 @@ Scope: no manifest change. Evidence: `govulncheck ./cmd/github-runner-provider` drops the two standard library, `x/net`, and Kinesis findings and reports only the five no-fix Docker advisories after the pins. + +### Backport 2026-07-14: Managed Container Names Are TLS DNS Labels + +Cause: generic safe identifiers allowed underscores, uppercase, trailing dots, +and 63-byte base names even though stable/candidate names become HTTPS hosts, +certificate DNS SANs, Podman network aliases, and derived `-probe` names. +Change: runtime validation requires every base and derived managed container +name to be a lowercase single DNS label. The shipped schema applies the same +rule, caps base names at 57 bytes so `-probe` remains within 63, and constrains +the `provider_url` host identically. +Scope: no manifest change. +Evidence: `TestConfigRejectsContainerNamesThatCannotBeTLSDNSNames` and the +release schema contract reject underscore, uppercase, trailing-dot, and derived +overflow cases; reverting the fix makes all eight assertions fail. diff --git a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md index 23d9bb8..41f015e 100644 --- a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md +++ b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md @@ -89,6 +89,7 @@ Critical or Important findings remain. | post-rewrite 5 | REVERT-AND-REWRITE | same-digest install could select unchanged without runtime integrity; image-loss race could mutate outside maintenance; rendered systemd timeouts truncated fractional seconds; managed path creation trusted home/ancestors without full authority validation | outer/inner digest+effect agreement with fenced restart; ceiling-rounded directives; home and nearest-existing-ancestor owner/writability validation; focused revert/restore proofs and retained package tests PASS. | | post-rewrite 6 | REQUEST-CHANGES | packaged provider used vulnerable Go TLS and `x/net/idna` paths plus a fixed Kinesis decoder panic; path comment omitted writability; effect guard obscured precedence | Go 1.26.5, `x/net` 0.55.0, `x/sys` 0.45.0, Kinesis 1.43.5; comment and guard clarified; scoped vulnerability and focused tests rerun. | | post-rewrite 7 | SHIP-IT | no Critical/Important; five inherited Docker advisories have no fixed release and affected archive/copy/AuthZ APIs are not called by this provider path | full scope/checklist pass; residual SDK linkage recorded for later dependency-light extraction; final verification gate required before PR. | +| Copilot 1 | REQUEST-CHANGES | runtime and schema allowed managed container names that cannot serve as TLS DNS SANs/hosts; derived probe labels could exceed 63 bytes | shared lowercase DNS-label invariant, 57-byte base cap, provider URL parity, and RED/GREEN/revert/restore runtime+schema proofs. | Round 5 rejected the prior mechanism. The affected durability/recovery layer was rewritten rather than advanced. A new post-rewrite review cycle must reach diff --git a/internal/retainedprovider/config.go b/internal/retainedprovider/config.go index a29bae7..bda4479 100644 --- a/internal/retainedprovider/config.go +++ b/internal/retainedprovider/config.go @@ -22,6 +22,7 @@ const ( var ( safeIdentifierPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`) + dnsLabelPattern = regexp.MustCompile(`^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$`) gitRefPattern = regexp.MustCompile(`^[0-9a-f]{40}$`) workflowPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]{0,255}$`) ) @@ -125,6 +126,9 @@ func (config Config) Validate(home string) error { } seenContainerNames := make(map[string]struct{}, len(managedContainerNames)) for _, name := range managedContainerNames { + if !dnsLabelPattern.MatchString(name) { + return fmt.Errorf("managed container name %q must be a DNS label", name) + } if _, exists := seenContainerNames[name]; exists { return fmt.Errorf("managed container names must be distinct") } diff --git a/internal/retainedprovider/state_test.go b/internal/retainedprovider/state_test.go index 8baa532..8d4c217 100644 --- a/internal/retainedprovider/state_test.go +++ b/internal/retainedprovider/state_test.go @@ -195,6 +195,39 @@ func TestConfigRejectsManagedContainerNameCollisions(t *testing.T) { } } +func TestConfigRejectsContainerNamesThatCannotBeTLSDNSNames(t *testing.T) { + home := t.TempDir() + for _, tc := range []struct { + name string + mutate func(*Config) + }{ + { + name: "stable underscore", + mutate: func(config *Config) { + config.StableContainer = "provider_name" + config.ProviderURL = "https://provider_name:18090" + }, + }, + {name: "candidate uppercase", mutate: func(config *Config) { config.CandidateContainer = "Provider-Candidate" }}, + { + name: "stable trailing dot", + mutate: func(config *Config) { + config.StableContainer = "provider." + config.ProviderURL = "https://provider.:18090" + }, + }, + {name: "probe suffix exceeds label", mutate: func(config *Config) { config.CandidateContainer = strings.Repeat("c", 63) }}, + } { + t.Run(tc.name, func(t *testing.T) { + config := validTestConfig(home) + tc.mutate(&config) + if err := config.Validate(home); err == nil || !strings.Contains(err.Error(), "DNS label") { + t.Fatalf("Validate = %v", err) + } + }) + } +} + func TestConfigRejectsAuthorityOverlapWithLifecycleState(t *testing.T) { home := t.TempDir() base := validTestConfig(home) diff --git a/release_packaging_test.go b/release_packaging_test.go index 85552c2..0732be9 100644 --- a/release_packaging_test.go +++ b/release_packaging_test.go @@ -171,6 +171,30 @@ func TestReleaseArchiveIncludesRetainedProviderConfigContract(t *testing.T) { if err := validateFields(fields); err == nil { t.Fatal("retained-provider config schema accepted a control character in an absolute path") } + for _, tc := range []struct { + name string + field string + value string + stableURL string + }{ + {name: "stable underscore", field: "stable_container", value: "provider_name", stableURL: "https://provider_name:18090"}, + {name: "candidate uppercase", field: "candidate_container", value: "Provider-Candidate"}, + {name: "stable trailing dot", field: "stable_container", value: "provider.", stableURL: "https://provider.:18090"}, + {name: "probe suffix exceeds label", field: "candidate_container", value: strings.Repeat("c", 63)}, + } { + t.Run(tc.name, func(t *testing.T) { + if err := json.Unmarshal(example, &fields); err != nil { + t.Fatalf("reset retained-provider config fields: %v", err) + } + fields[tc.field], _ = json.Marshal(tc.value) + if tc.stableURL != "" { + fields["provider_url"], _ = json.Marshal(tc.stableURL) + } + if err := validateFields(fields); err == nil { + t.Fatalf("retained-provider config schema accepted non-DNS %s %q", tc.field, tc.value) + } + }) + } for _, field := range []string{"protocol_version", "worker_id", "provider_marker_path", "podman_path", "systemctl_path", "loginctl_path", "ref", "refresh_interval_seconds"} { if !bytes.Contains(schema, []byte(`"`+field+`"`)) { t.Fatalf("retained-provider config schema is missing %q", field) diff --git a/schemas/github-runner-retained-config.schema.json b/schemas/github-runner-retained-config.schema.json index b9e6d12..5ccfe49 100644 --- a/schemas/github-runner-retained-config.schema.json +++ b/schemas/github-runner-retained-config.schema.json @@ -65,10 +65,10 @@ "provider_url": { "type": "string", "format": "uri", - "pattern": "^https://[A-Za-z0-9][A-Za-z0-9._-]{0,127}:18090/?$" + "pattern": "^https://[a-z0-9](?:[a-z0-9-]{0,55}[a-z0-9])?:18090/?$" }, - "stable_container": { "$ref": "#/$defs/identifier" }, - "candidate_container": { "$ref": "#/$defs/identifier" }, + "stable_container": { "$ref": "#/$defs/containerName" }, + "candidate_container": { "$ref": "#/$defs/containerName" }, "container_network": { "const": "wfcompute-github-provider" }, "organization": { "$ref": "#/$defs/identifier" }, "repository": { @@ -107,6 +107,11 @@ "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$" }, + "containerName": { + "type": "string", + "maxLength": 57, + "pattern": "^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$" + }, "absolutePath": { "type": "string", "pattern": "^/[^\\x00-\\x1F\\x7F]*$" From 2f6d6399f4cfccc8a71e3ff748e75ba018a7a8c4 Mon Sep 17 00:00:00 2001 From: Jon Langevin Date: Tue, 14 Jul 2026 19:26:39 -0400 Subject: [PATCH 13/16] fix(provider): reject unsafe paths Retained config and probe CA paths cross systemd, process, and filesystem boundaries unchanged. Require one canonical control-free absolute-path invariant before use. --- cmd/github-runner-provider/main_test.go | 4 ++++ cmd/github-runner-provider/probe.go | 5 ++--- ...13-retained-runner-provider-lifecycle-design.md | 14 ++++++++++++++ ...tained-runner-provider-lifecycle-plan-review.md | 2 ++ internal/retainedprovider/config.go | 11 ++++++++++- internal/retainedprovider/state_test.go | 7 +++++++ 6 files changed, 39 insertions(+), 4 deletions(-) diff --git a/cmd/github-runner-provider/main_test.go b/cmd/github-runner-provider/main_test.go index 971a4c4..6cb6ff4 100644 --- a/cmd/github-runner-provider/main_test.go +++ b/cmd/github-runner-provider/main_test.go @@ -258,6 +258,10 @@ func TestProviderProbeFailsClosedOnInvalidConfiguration(t *testing.T) { {name: "missing token", args: []string{"-url", "https://provider.test", "-ca-file", "/ca.pem"}, want: "GITHUB_RUNNER_PROVIDER_TOKEN"}, {name: "plaintext URL", args: []string{"-url", "http://provider.test", "-ca-file", "/ca.pem"}, token: "provider-token", want: "HTTPS"}, {name: "missing CA", args: []string{"-url", "https://provider.test"}, token: "provider-token", want: "ca-file"}, + {name: "CA control", args: []string{"-url", "https://provider.test", "-ca-file", "/ca.pem\nnext"}, token: "provider-token", want: "ca-file"}, + {name: "CA DEL", args: []string{"-url", "https://provider.test", "-ca-file", "/ca.pem\x7f"}, token: "provider-token", want: "ca-file"}, + {name: "CA noncanonical", args: []string{"-url", "https://provider.test", "-ca-file", "/tmp/../ca.pem"}, token: "provider-token", want: "ca-file"}, + {name: "CA padded", args: []string{"-url", "https://provider.test", "-ca-file", " /ca.pem"}, token: "provider-token", want: "ca-file"}, } { t.Run(tc.name, func(t *testing.T) { t.Setenv("GITHUB_RUNNER_PROVIDER_TOKEN", tc.token) diff --git a/cmd/github-runner-provider/probe.go b/cmd/github-runner-provider/probe.go index 25bf654..04aef55 100644 --- a/cmd/github-runner-provider/probe.go +++ b/cmd/github-runner-provider/probe.go @@ -14,7 +14,6 @@ import ( "net/url" "os" "path" - "path/filepath" "strings" "time" @@ -165,8 +164,8 @@ func validateProviderProbeFlags(rawURL, caFile, organization, repository, workfl return nil, errors.New("provider url must be an HTTPS origin") } baseURL.Path = "" - if strings.TrimSpace(caFile) == "" || !filepath.IsAbs(strings.TrimSpace(caFile)) { - return nil, errors.New("-ca-file must be an absolute path") + if !retainedprovider.IsCanonicalSafeAbsolutePath(caFile) { + return nil, errors.New("-ca-file must be an absolute canonical safe path") } for _, field := range []struct { name string diff --git a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md index fa236a2..43906e1 100644 --- a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md +++ b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md @@ -1144,3 +1144,17 @@ Scope: no manifest change. Evidence: `TestConfigRejectsContainerNamesThatCannotBeTLSDNSNames` and the release schema contract reject underscore, uppercase, trailing-dot, and derived overflow cases; reverting the fix makes all eight assertions fail. + +### Backport 2026-07-14: Absolute Paths Cross Boundaries Unchanged + +Cause: home-boundary validation normalized retained filesystem paths before +checking them, while the probe accepted a trimmed CA path but later opened the +original value. C0/DEL and non-canonical paths could therefore cross systemd, +process, diagnostic, or filesystem boundaries despite the shipped schema's +control-character restriction. +Change: one shared predicate requires absolute, canonical, C0/DEL-free paths; +all retained config paths and the probe CA argument fail closed before use. +Scope: no manifest change. +Evidence: focused config/probe tests cover every retained path plus control, +DEL, padded, and non-canonical CA values; fix-revert reproduces all eleven +failures and restore returns both packages green. diff --git a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md index 41f015e..fc97343 100644 --- a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md +++ b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md @@ -90,6 +90,8 @@ Critical or Important findings remain. | post-rewrite 6 | REQUEST-CHANGES | packaged provider used vulnerable Go TLS and `x/net/idna` paths plus a fixed Kinesis decoder panic; path comment omitted writability; effect guard obscured precedence | Go 1.26.5, `x/net` 0.55.0, `x/sys` 0.45.0, Kinesis 1.43.5; comment and guard clarified; scoped vulnerability and focused tests rerun. | | post-rewrite 7 | SHIP-IT | no Critical/Important; five inherited Docker advisories have no fixed release and affected archive/copy/AuthZ APIs are not called by this provider path | full scope/checklist pass; residual SDK linkage recorded for later dependency-light extraction; final verification gate required before PR. | | Copilot 1 | REQUEST-CHANGES | runtime and schema allowed managed container names that cannot serve as TLS DNS SANs/hosts; derived probe labels could exceed 63 bytes | shared lowercase DNS-label invariant, 57-byte base cap, provider URL parity, and RED/GREEN/revert/restore runtime+schema proofs. | +| Copilot 2 | REQUEST-CHANGES | retained filesystem paths and probe CA path accepted control, DEL, padded, or non-canonical values before systemd/process/filesystem use | shared canonical safe absolute-path predicate plus RED/GREEN/revert/restore proof across every retained path and probe CA variants. | +| post-Copilot 2 | SHIP-IT | no Critical/Important after scope-compliance and full bug-class scan; predicate is symmetric across retained config and probe CA boundaries, errors fail closed, tests are hermetic/non-vacuous, and target-specific path semantics compile per OS | full race/static/six-target/snapshot/runtime gates rerun on final code; CI and Copilot re-review still required after push. | Round 5 rejected the prior mechanism. The affected durability/recovery layer was rewritten rather than advanced. A new post-rewrite review cycle must reach diff --git a/internal/retainedprovider/config.go b/internal/retainedprovider/config.go index bda4479..7e60b24 100644 --- a/internal/retainedprovider/config.go +++ b/internal/retainedprovider/config.go @@ -146,7 +146,7 @@ func (config Config) Validate(home string) error { {field: "systemctl_path", path: config.SystemctlPath, base: "systemctl"}, {field: "loginctl_path", path: config.LoginctlPath, base: "loginctl"}, } { - if !filepath.IsAbs(tool.path) || filepath.Clean(tool.path) != tool.path || containsControl(tool.path) || filepath.Base(tool.path) != tool.base { + if !IsCanonicalSafeAbsolutePath(tool.path) || filepath.Base(tool.path) != tool.base { return fmt.Errorf("%s must be an absolute canonical safe path to %s", tool.field, tool.base) } } @@ -158,6 +158,9 @@ func (config Config) Validate(home string) error { "install_root": config.InstallRoot, "systemd_dir": config.SystemdDir, } { + if !IsCanonicalSafeAbsolutePath(path) { + return fmt.Errorf("%s must be an absolute canonical safe path", field) + } if err := ValidateUserPath(home, path, false); err != nil { return fmt.Errorf("%s: %w", field, err) } @@ -300,3 +303,9 @@ func decodeStrictJSON(reader io.Reader, target any) error { func containsControl(value string) bool { return strings.IndexFunc(value, func(r rune) bool { return r < 0x20 || r == 0x7f }) >= 0 } + +// IsCanonicalSafeAbsolutePath reports whether value is safe to pass unchanged +// to filesystem and process boundaries. +func IsCanonicalSafeAbsolutePath(value string) bool { + return filepath.IsAbs(value) && filepath.Clean(value) == value && !containsControl(value) +} diff --git a/internal/retainedprovider/state_test.go b/internal/retainedprovider/state_test.go index 8d4c217..75f84b2 100644 --- a/internal/retainedprovider/state_test.go +++ b/internal/retainedprovider/state_test.go @@ -127,6 +127,13 @@ func TestConfigRejectsUnsafeIdentityAndPaths(t *testing.T) { {name: "wrong plugin", mutate: func(c *Config) { c.PluginID = "other" }, want: "plugin_id"}, {name: "unsafe component", mutate: func(c *Config) { c.ComponentID = "component;rm" }, want: "component_id"}, {name: "unsafe unit", mutate: func(c *Config) { c.AgentUnit = "agent.service\nEnvironment=TOKEN" }, want: "agent_unit"}, + {name: "compute agent path control", mutate: func(c *Config) { c.ComputeAgentPath += "\nnext" }, want: "canonical safe path"}, + {name: "compute agent path noncanonical", mutate: func(c *Config) { c.ComputeAgentPath = home + "/bin/../compute-agent" }, want: "canonical safe path"}, + {name: "supervisor config path control", mutate: func(c *Config) { c.SupervisorConfigPath += "\x7f" }, want: "canonical safe path"}, + {name: "local status path control", mutate: func(c *Config) { c.LocalStatusPath += "\rnext" }, want: "canonical safe path"}, + {name: "provider marker path control", mutate: func(c *Config) { c.ProviderMarkerPath += "\tnext" }, want: "canonical safe path"}, + {name: "install root path control", mutate: func(c *Config) { c.InstallRoot += "\x00next" }, want: "canonical safe path"}, + {name: "systemd path control", mutate: func(c *Config) { c.SystemdDir += "\x1fnext" }, want: "canonical safe path"}, {name: "relative install root", mutate: func(c *Config) { c.InstallRoot = "relative" }, want: "install_root"}, {name: "shared workflow compute root", mutate: func(c *Config) { c.InstallRoot = filepath.Join(home, ".workflow-compute") }, want: "dedicated provider root"}, {name: "systemd directory as install root", mutate: func(c *Config) { c.InstallRoot = c.SystemdDir }, want: "dedicated provider root"}, From 647456dade06169616ba264e9ab3167f44f9eab0 Mon Sep 17 00:00:00 2001 From: Jon Langevin Date: Tue, 14 Jul 2026 19:39:19 -0400 Subject: [PATCH 14/16] fix(provider): harden CLI errors Keep positional input out of unknown-command diagnostics while preserving causal response-read errors without exposing response content. --- cmd/github-runner-provider/main.go | 2 +- cmd/github-runner-provider/main_test.go | 33 ++++++++++++++++++- cmd/github-runner-provider/probe.go | 2 +- ...tained-runner-provider-lifecycle-design.md | 11 +++++++ ...d-runner-provider-lifecycle-plan-review.md | 2 ++ 5 files changed, 47 insertions(+), 3 deletions(-) diff --git a/cmd/github-runner-provider/main.go b/cmd/github-runner-provider/main.go index 1f92fc8..d69681c 100644 --- a/cmd/github-runner-provider/main.go +++ b/cmd/github-runner-provider/main.go @@ -75,7 +75,7 @@ func dispatchProviderCommand(ctx context.Context, logger *slog.Logger, args []st return false, nil } } - return true, fmt.Errorf("unknown command %q", args[0]) + return true, errors.New("unknown command") } } diff --git a/cmd/github-runner-provider/main_test.go b/cmd/github-runner-provider/main_test.go index 6cb6ff4..85a34a2 100644 --- a/cmd/github-runner-provider/main_test.go +++ b/cmd/github-runner-provider/main_test.go @@ -21,6 +21,7 @@ import ( "reflect" "strings" "testing" + "testing/iotest" "time" githubplugin "github.com/GoCodeAlone/workflow-plugin-github" @@ -99,6 +100,12 @@ type closeTrackingListener struct { closed bool } +type providerProbeRoundTripFunc func(*http.Request) (*http.Response, error) + +func (f providerProbeRoundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { + return f(request) +} + func (l *closeTrackingListener) Close() error { l.closed = true return l.Listener.Close() @@ -152,11 +159,15 @@ func TestProviderCommandVersionDoesNotRequireServiceCredentials(t *testing.T) { } func TestProviderCommandRejectsUnknownSubcommand(t *testing.T) { + const secretArgument = "github_pat_must-not-reach-provider-logs" var stdout bytes.Buffer - handled, err := dispatchProviderCommand(t.Context(), slog.New(slog.NewTextHandler(io.Discard, nil)), []string{"unknown-command"}, &stdout) + handled, err := dispatchProviderCommand(t.Context(), slog.New(slog.NewTextHandler(io.Discard, nil)), []string{secretArgument}, &stdout) if !handled || err == nil || !strings.Contains(err.Error(), "unknown command") { t.Fatalf("unknown command handled=%t err=%v", handled, err) } + if strings.Contains(err.Error(), secretArgument) { + t.Fatalf("unknown command leaked argument: %v", err) + } if stdout.Len() != 0 { t.Fatalf("unknown command wrote stdout: %q", stdout.String()) } @@ -364,6 +375,26 @@ func TestProviderProbeRejectsRedirectWithoutForwardingBearer(t *testing.T) { } } +func TestProviderProbePreservesResponseReadFailure(t *testing.T) { + readErr := errors.New("response read sentinel") + client := &http.Client{Transport: providerProbeRoundTripFunc(func(*http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(iotest.ErrReader(readErr)), + Header: make(http.Header), + }, nil + })} + endpoint, err := url.Parse("https://provider.test/readyz") + if err != nil { + t.Fatalf("parse probe endpoint: %v", err) + } + var response providerProbeReadyResponse + err = providerProbeJSON(t.Context(), client, http.MethodGet, endpoint, "provider-token", nil, &response) + if !errors.Is(err, readErr) || !strings.Contains(err.Error(), "read provider response") { + t.Fatalf("response read error = %v", err) + } +} + func writeProviderProbeTestCA(t *testing.T, server *httptest.Server) string { t.Helper() caFile := filepath.Join(t.TempDir(), "ca.pem") diff --git a/cmd/github-runner-provider/probe.go b/cmd/github-runner-provider/probe.go index 04aef55..433a590 100644 --- a/cmd/github-runner-provider/probe.go +++ b/cmd/github-runner-provider/probe.go @@ -267,7 +267,7 @@ func providerProbeJSON(ctx context.Context, client *http.Client, method string, } data, err := io.ReadAll(io.LimitReader(resp.Body, providerProbeMaxBodyBytes+1)) if err != nil { - return errors.New("read provider response") + return fmt.Errorf("read provider response: %w", err) } if len(data) > providerProbeMaxBodyBytes { return errors.New("provider response exceeds 1 MiB") diff --git a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md index 43906e1..28aabff 100644 --- a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md +++ b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md @@ -1158,3 +1158,14 @@ Scope: no manifest change. Evidence: focused config/probe tests cover every retained path plus control, DEL, padded, and non-canonical CA values; fix-revert reproduces all eleven failures and restore returns both packages green. + +### Backport 2026-07-14: CLI Errors Preserve Causes, Not Inputs + +Cause: unknown-subcommand errors echoed the raw positional value into provider +logs, while probe response-read failures discarded the underlying I/O cause. +Change: unknown-command diagnostics are constant and response-read failures +wrap the original error without including response bodies. +Scope: no manifest change. +Evidence: a token-shaped argument is absent from the command error and a +hermetic failing response body remains reachable through `errors.Is`; reverting +the two production lines makes both focused tests fail and restore passes. diff --git a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md index fc97343..8cee98d 100644 --- a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md +++ b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md @@ -92,6 +92,8 @@ Critical or Important findings remain. | Copilot 1 | REQUEST-CHANGES | runtime and schema allowed managed container names that cannot serve as TLS DNS SANs/hosts; derived probe labels could exceed 63 bytes | shared lowercase DNS-label invariant, 57-byte base cap, provider URL parity, and RED/GREEN/revert/restore runtime+schema proofs. | | Copilot 2 | REQUEST-CHANGES | retained filesystem paths and probe CA path accepted control, DEL, padded, or non-canonical values before systemd/process/filesystem use | shared canonical safe absolute-path predicate plus RED/GREEN/revert/restore proof across every retained path and probe CA variants. | | post-Copilot 2 | SHIP-IT | no Critical/Important after scope-compliance and full bug-class scan; predicate is symmetric across retained config and probe CA boundaries, errors fail closed, tests are hermetic/non-vacuous, and target-specific path semantics compile per OS | full race/static/six-target/snapshot/runtime gates rerun on final code; CI and Copilot re-review still required after push. | +| Copilot 3 | REQUEST-CHANGES | unknown subcommands echoed raw positional input into logs; probe response-read failures swallowed their causal I/O error | constant unknown-command diagnostic and `%w` response-read wrapping with token-shaped and hermetic failing-body RED/GREEN/revert/restore tests. | +| post-Copilot 3 | SHIP-IT | no Critical/Important after scope-compliance and full bug-class scan; unknown input cannot enter diagnostics, read causes propagate without response content, and both tests exercise real hermetic failure paths | full race/static/six-target/snapshot/runtime gates rerun; CI and Copilot re-review required after push. | Round 5 rejected the prior mechanism. The affected durability/recovery layer was rewritten rather than advanced. A new post-rewrite review cycle must reach From 925fc24c7b8128b7b4162ac193308d6cd3692761 Mon Sep 17 00:00:00 2001 From: Jon Langevin Date: Tue, 14 Jul 2026 19:55:17 -0400 Subject: [PATCH 15/16] fix(provider): reject raced symlinks Use platform no-follow opens for install locks and lifecycle audits before post-open authority checks. Keep retained unknown subcommand values out of diagnostics. --- cmd/github-runner-provider/retained_stub.go | 2 +- cmd/github-runner-provider/retained_test.go | 14 ++++-- ...tained-runner-provider-lifecycle-design.md | 16 ++++++ ...d-runner-provider-lifecycle-plan-review.md | 2 + internal/retainedprovider/files.go | 9 +++- .../files_nofollow_unix_test.go | 50 +++++++++++++++++++ internal/retainedprovider/lifecycle.go | 10 +++- internal/retainedprovider/lock_other.go | 5 ++ internal/retainedprovider/lock_unix.go | 14 ++++++ internal/retainedprovider/lock_windows.go | 26 ++++++++++ 10 files changed, 140 insertions(+), 8 deletions(-) create mode 100644 internal/retainedprovider/files_nofollow_unix_test.go diff --git a/cmd/github-runner-provider/retained_stub.go b/cmd/github-runner-provider/retained_stub.go index aee5e77..983d8d5 100644 --- a/cmd/github-runner-provider/retained_stub.go +++ b/cmd/github-runner-provider/retained_stub.go @@ -56,7 +56,7 @@ func runRetainedProviderCommandWithDependencies(ctx context.Context, _ *slog.Log switch args[0] { case "install", "refresh", "serve-active", "status", "uninstall", "recover": default: - return fmt.Errorf("unknown retained provider subcommand %q", args[0]) + return errors.New("unknown retained provider subcommand") } flags := flag.NewFlagSet("github-runner-provider retained "+args[0], flag.ContinueOnError) flags.SetOutput(io.Discard) diff --git a/cmd/github-runner-provider/retained_test.go b/cmd/github-runner-provider/retained_test.go index 7bb48eb..3e001ac 100644 --- a/cmd/github-runner-provider/retained_test.go +++ b/cmd/github-runner-provider/retained_test.go @@ -251,14 +251,15 @@ func TestRetainedCommandFailsClosedOnUnsupportedPlatformAndInvalidShape(t *testi ServeActive: func(context.Context, retainedprovider.Config) error { return nil }, } for _, tc := range []struct { - name string - deps retainedProviderCommandDependencies - args []string - want string + name string + deps retainedProviderCommandDependencies + args []string + want string + redactArg bool }{ {name: "unsupported", deps: func() retainedProviderCommandDependencies { value := base; value.GOOS = "darwin"; return value }(), args: []string{"refresh"}, want: "unsupported"}, {name: "missing subcommand", deps: base, want: "subcommand"}, - {name: "unknown", deps: base, args: []string{"install-now"}, want: "unknown"}, + {name: "unknown", deps: base, args: []string{"github_pat_must-not-reach-retained-logs"}, want: "unknown", redactArg: true}, {name: "missing config", deps: base, args: []string{"refresh"}, want: "-config"}, {name: "positional", deps: base, args: []string{"refresh", "-config", "/tmp/config", "extra"}, want: "positional"}, } { @@ -267,6 +268,9 @@ func TestRetainedCommandFailsClosedOnUnsupportedPlatformAndInvalidShape(t *testi if err == nil || !strings.Contains(err.Error(), tc.want) { t.Fatalf("err = %v want %q", err, tc.want) } + if tc.redactArg && strings.Contains(err.Error(), tc.args[0]) { + t.Fatalf("retained command error leaked argument: %v", err) + } }) } } diff --git a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md index 28aabff..259100a 100644 --- a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md +++ b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md @@ -1169,3 +1169,19 @@ Scope: no manifest change. Evidence: a token-shaped argument is absent from the command error and a hermetic failing response body remains reachable through `errors.Is`; reverting the two production lines makes both focused tests fail and restore passes. + +### Backport 2026-07-14: Security-Sensitive Creation Never Follows Symlinks + +Cause: install-lock and lifecycle-audit creation validated a missing final path +with `Lstat` and then used a following `OpenFile(O_CREATE)`, allowing a symlink +to be raced into that gap; retained unknown-subcommand diagnostics also echoed +their raw input like the already-corrected top-level dispatcher. +Change: both creation paths use one injected no-follow opener: `O_NOFOLLOW` and +`O_CLOEXEC` on Unix, `FILE_FLAG_OPEN_REPARSE_POINT` on Windows, and fail-closed +behavior elsewhere. Existing owner, mode, regular-file, and same-inode checks +remain after open. Retained unknown-subcommand diagnostics are constant. +Scope: no manifest change. +Evidence: deterministic Unix tests place a symlink after pre-open validation +for both files and prove the target stays untouched; the retained token-shaped +argument is absent from errors. Reverting production removes the no-follow +boundary and reproduces the leak; restore passes and Windows cross-compiles. diff --git a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md index 8cee98d..f86771b 100644 --- a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md +++ b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md @@ -94,6 +94,8 @@ Critical or Important findings remain. | post-Copilot 2 | SHIP-IT | no Critical/Important after scope-compliance and full bug-class scan; predicate is symmetric across retained config and probe CA boundaries, errors fail closed, tests are hermetic/non-vacuous, and target-specific path semantics compile per OS | full race/static/six-target/snapshot/runtime gates rerun on final code; CI and Copilot re-review still required after push. | | Copilot 3 | REQUEST-CHANGES | unknown subcommands echoed raw positional input into logs; probe response-read failures swallowed their causal I/O error | constant unknown-command diagnostic and `%w` response-read wrapping with token-shaped and hermetic failing-body RED/GREEN/revert/restore tests. | | post-Copilot 3 | SHIP-IT | no Critical/Important after scope-compliance and full bug-class scan; unknown input cannot enter diagnostics, read causes propagate without response content, and both tests exercise real hermetic failure paths | full race/static/six-target/snapshot/runtime gates rerun; CI and Copilot re-review required after push. | +| Copilot 4 | REQUEST-CHANGES | retained unknown subcommands still echoed raw input; lock creation could follow a final-component symlink raced between `Lstat` and `OpenFile(O_CREATE)` | constant retained diagnostic; shared Unix/Windows no-follow opener wired to lock and sibling lifecycle-audit creation; deterministic raced-symlink and RED/GREEN/revert/restore proofs plus Windows cross-compile. | +| post-Copilot 4 | SHIP-IT | no Critical/Important after scope-compliance and full bug-class scan; Unix rejects final symlinks, Windows opens/rejects reparse points through existing post-open checks, unsupported platforms fail closed, and no content mutates before validation | full race/static/seven-target/snapshot/runtime gates rerun; Windows CI journal and Copilot re-review required after push. | Round 5 rejected the prior mechanism. The affected durability/recovery layer was rewritten rather than advanced. A new post-rewrite review cycle must reach diff --git a/internal/retainedprovider/files.go b/internal/retainedprovider/files.go index ac7a875..d50ec4e 100644 --- a/internal/retainedprovider/files.go +++ b/internal/retainedprovider/files.go @@ -411,6 +411,13 @@ func AcquireInstallLock(path string) (*InstallLock, error) { } func openRegularLockFile(path string) (*os.File, error) { + return openRegularLockFileWith(path, openNoFollowFile) +} + +func openRegularLockFileWith(path string, opener func(string, fs.FileMode) (*os.File, error)) (*os.File, error) { + if opener == nil { + return nil, errors.New("install lock opener is required") + } before, err := os.Lstat(path) if err == nil { if !before.Mode().IsRegular() { @@ -422,7 +429,7 @@ func openRegularLockFile(path string) (*os.File, error) { } else if !errors.Is(err, os.ErrNotExist) { return nil, err } - file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o600) + file, err := opener(path, 0o600) if err != nil { return nil, err } diff --git a/internal/retainedprovider/files_nofollow_unix_test.go b/internal/retainedprovider/files_nofollow_unix_test.go new file mode 100644 index 0000000..c869d5c --- /dev/null +++ b/internal/retainedprovider/files_nofollow_unix_test.go @@ -0,0 +1,50 @@ +//go:build darwin || linux + +package retainedprovider + +import ( + "io/fs" + "os" + "path/filepath" + "testing" +) + +func TestSecuritySensitiveOpenRejectsRacedSymlink(t *testing.T) { + for _, tc := range []struct { + name string + open func(string, func(string, fs.FileMode) (*os.File, error)) (*os.File, error) + }{ + {name: "install lock", open: openRegularLockFileWith}, + {name: "lifecycle audit", open: openLifecycleAuditWith}, + } { + t.Run(tc.name, func(t *testing.T) { + directory := t.TempDir() + target := filepath.Join(directory, "target") + const targetContents = "must remain untouched" + if err := os.WriteFile(target, []byte(targetContents), 0o600); err != nil { + t.Fatalf("write target: %v", err) + } + path := filepath.Join(directory, "managed") + opener := func(path string, mode fs.FileMode) (*os.File, error) { + if err := os.Symlink(target, path); err != nil { + t.Fatalf("race symlink into place: %v", err) + } + return openNoFollowFile(path, mode) + } + file, err := tc.open(path, opener) + if file != nil { + _ = file.Close() + } + if err == nil { + t.Fatal("security-sensitive open followed raced symlink") + } + contents, readErr := os.ReadFile(target) + if readErr != nil { + t.Fatalf("read target: %v", readErr) + } + if string(contents) != targetContents { + t.Fatalf("target contents = %q", contents) + } + }) + } +} diff --git a/internal/retainedprovider/lifecycle.go b/internal/retainedprovider/lifecycle.go index baf0454..d4d234b 100644 --- a/internal/retainedprovider/lifecycle.go +++ b/internal/retainedprovider/lifecycle.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "io" + "io/fs" "os" "path/filepath" "sort" @@ -884,10 +885,17 @@ func drainLifecycleAudit(home string, paths LifecyclePaths, journal *LifecycleJo } func openLifecycleAudit(path string) (*os.File, error) { + return openLifecycleAuditWith(path, openNoFollowFile) +} + +func openLifecycleAuditWith(path string, opener func(string, fs.FileMode) (*os.File, error)) (*os.File, error) { + if opener == nil { + return nil, errors.New("lifecycle audit opener is required") + } if err := rejectNonRegularDestination(path); err != nil { return nil, err } - file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) + file, err := opener(path, 0o600) if err != nil { return nil, fmt.Errorf("open lifecycle audit: %w", err) } diff --git a/internal/retainedprovider/lock_other.go b/internal/retainedprovider/lock_other.go index 421815b..b90b2cf 100644 --- a/internal/retainedprovider/lock_other.go +++ b/internal/retainedprovider/lock_other.go @@ -4,9 +4,14 @@ package retainedprovider import ( "fmt" + "io/fs" "os" ) +func openNoFollowFile(string, fs.FileMode) (*os.File, error) { + return nil, fmt.Errorf("secure no-follow file open is unsupported on this platform") +} + func lockFile(*os.File) error { return fmt.Errorf("install locking is unsupported on this platform") } diff --git a/internal/retainedprovider/lock_unix.go b/internal/retainedprovider/lock_unix.go index 4d10011..efb9c7c 100644 --- a/internal/retainedprovider/lock_unix.go +++ b/internal/retainedprovider/lock_unix.go @@ -4,11 +4,25 @@ package retainedprovider import ( "errors" + "io/fs" "os" "golang.org/x/sys/unix" ) +func openNoFollowFile(path string, mode fs.FileMode) (*os.File, error) { + descriptor, err := unix.Open(path, unix.O_RDWR|unix.O_CREAT|unix.O_CLOEXEC|unix.O_NOFOLLOW, uint32(mode.Perm())) + if err != nil { + return nil, &os.PathError{Op: "open", Path: path, Err: err} + } + file := os.NewFile(uintptr(descriptor), path) + if file == nil { + _ = unix.Close(descriptor) + return nil, errors.New("convert no-follow file descriptor") + } + return file, nil +} + func lockFile(file *os.File) error { err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB) if errors.Is(err, unix.EWOULDBLOCK) || errors.Is(err, unix.EAGAIN) { diff --git a/internal/retainedprovider/lock_windows.go b/internal/retainedprovider/lock_windows.go index 1810b7d..13a4387 100644 --- a/internal/retainedprovider/lock_windows.go +++ b/internal/retainedprovider/lock_windows.go @@ -4,11 +4,37 @@ package retainedprovider import ( "errors" + "io/fs" "os" "golang.org/x/sys/windows" ) +func openNoFollowFile(path string, _ fs.FileMode) (*os.File, error) { + windowsPath, err := windows.UTF16PtrFromString(path) + if err != nil { + return nil, &os.PathError{Op: "open", Path: path, Err: err} + } + handle, err := windows.CreateFile( + windowsPath, + windows.GENERIC_READ|windows.GENERIC_WRITE, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE, + nil, + windows.OPEN_ALWAYS, + windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_OPEN_REPARSE_POINT, + 0, + ) + if err != nil { + return nil, &os.PathError{Op: "open", Path: path, Err: err} + } + file := os.NewFile(uintptr(handle), path) + if file == nil { + _ = windows.CloseHandle(handle) + return nil, errors.New("convert no-follow file handle") + } + return file, nil +} + func lockFile(file *os.File) error { var overlapped windows.Overlapped err := windows.LockFileEx(windows.Handle(file.Fd()), windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, &overlapped) From 47bf8795608f6abf4f071e63406c243157938d52 Mon Sep 17 00:00:00 2001 From: Jon Langevin Date: Tue, 14 Jul 2026 20:26:01 -0400 Subject: [PATCH 16/16] fix(provider): align path schema Reject noncanonical path segments in the shipped schema so operator validation matches the runtime security boundary. --- ...13-retained-runner-provider-lifecycle-design.md | 12 ++++++++++++ ...tained-runner-provider-lifecycle-plan-review.md | 2 ++ release_packaging_test.go | 14 ++++++++++++++ schemas/github-runner-retained-config.schema.json | 2 +- 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md index 259100a..4285dbc 100644 --- a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md +++ b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-design.md @@ -1185,3 +1185,15 @@ Evidence: deterministic Unix tests place a symlink after pre-open validation for both files and prove the target stays untouched; the retained token-shaped argument is absent from errors. Reverting production removes the no-follow boundary and reproduces the leak; restore passes and Windows cross-compiles. + +### Backport 2026-07-14: Schema Paths Use Canonical Segments + +Cause: the runtime canonical-path predicate rejected duplicate separators, +dot/parent segments, and trailing separators, while the shipped schema's broad +absolute-path pattern accepted them. +Change: replace the broad pattern with an explicit segment grammar. Root and +canonical dot-prefixed names remain valid; empty, `.`, and `..` segments do not. +Scope: no manifest change. +Evidence: the real release-schema test rejects `//`, `/./`, `/../`, and a +trailing separator while continuing to validate the shipped example; reverting +the grammar accepts the first invalid path and restoring it passes. diff --git a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md index f86771b..9f8f1e3 100644 --- a/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md +++ b/docs/plans/2026-07-13-retained-runner-provider-lifecycle-plan-review.md @@ -96,6 +96,8 @@ Critical or Important findings remain. | post-Copilot 3 | SHIP-IT | no Critical/Important after scope-compliance and full bug-class scan; unknown input cannot enter diagnostics, read causes propagate without response content, and both tests exercise real hermetic failure paths | full race/static/six-target/snapshot/runtime gates rerun; CI and Copilot re-review required after push. | | Copilot 4 | REQUEST-CHANGES | retained unknown subcommands still echoed raw input; lock creation could follow a final-component symlink raced between `Lstat` and `OpenFile(O_CREATE)` | constant retained diagnostic; shared Unix/Windows no-follow opener wired to lock and sibling lifecycle-audit creation; deterministic raced-symlink and RED/GREEN/revert/restore proofs plus Windows cross-compile. | | post-Copilot 4 | SHIP-IT | no Critical/Important after scope-compliance and full bug-class scan; Unix rejects final symlinks, Windows opens/rejects reparse points through existing post-open checks, unsupported platforms fail closed, and no content mutates before validation | full race/static/seven-target/snapshot/runtime gates rerun; Windows CI journal and Copilot re-review required after push. | +| Copilot 5 | REVERT-AND-REWRITE | shipped `absolutePath` schema accepted non-canonical paths rejected by the runtime predicate | replaced the broad regex approach with an explicit canonical path-segment grammar and real release-schema RED/GREEN/revert/restore proof; no sixth Copilot loop. | +| post-rewrite final | SHIP-IT | no Critical/Important; scope/dispatch, symmetry, error handling, comments, test names, edge cases, concurrency, type coercion, dead code, root-cause fidelity, boundary wiring, integration consumption, hermeticity, portability, and vacuous-assertion scans are clean for the schema-only rewrite | explicit segments match `filepath.Clean` semantics for root, dot-prefixed names, duplicate separators, dot/parent segments, and trailing separators; race/static/seven-target/snapshot/checksum/archive/runtime gates pass on the exact candidate. | Round 5 rejected the prior mechanism. The affected durability/recovery layer was rewritten rather than advanced. A new post-rewrite review cycle must reach diff --git a/release_packaging_test.go b/release_packaging_test.go index 0732be9..c3a0558 100644 --- a/release_packaging_test.go +++ b/release_packaging_test.go @@ -171,6 +171,20 @@ func TestReleaseArchiveIncludesRetainedProviderConfigContract(t *testing.T) { if err := validateFields(fields); err == nil { t.Fatal("retained-provider config schema accepted a control character in an absolute path") } + for _, value := range []string{ + "/home/wfcompute//compute-agent", + "/home/wfcompute/./compute-agent", + "/home/wfcompute/bin/../compute-agent", + "/home/wfcompute/compute-agent/", + } { + if err := json.Unmarshal(example, &fields); err != nil { + t.Fatalf("reset retained-provider config fields: %v", err) + } + fields["compute_agent_path"], _ = json.Marshal(value) + if err := validateFields(fields); err == nil { + t.Fatalf("retained-provider config schema accepted non-canonical absolute path %q", value) + } + } for _, tc := range []struct { name string field string diff --git a/schemas/github-runner-retained-config.schema.json b/schemas/github-runner-retained-config.schema.json index 5ccfe49..dbfacf7 100644 --- a/schemas/github-runner-retained-config.schema.json +++ b/schemas/github-runner-retained-config.schema.json @@ -114,7 +114,7 @@ }, "absolutePath": { "type": "string", - "pattern": "^/[^\\x00-\\x1F\\x7F]*$" + "pattern": "^/(?:(?:[^./\\x00-\\x1F\\x7F][^/\\x00-\\x1F\\x7F]*|\\.[^./\\x00-\\x1F\\x7F][^/\\x00-\\x1F\\x7F]*|\\.\\.[^/\\x00-\\x1F\\x7F][^/\\x00-\\x1F\\x7F]*)(?:/(?:[^./\\x00-\\x1F\\x7F][^/\\x00-\\x1F\\x7F]*|\\.[^./\\x00-\\x1F\\x7F][^/\\x00-\\x1F\\x7F]*|\\.\\.[^/\\x00-\\x1F\\x7F][^/\\x00-\\x1F\\x7F]*))*)?$" } } }