From 51d7c477b222cc2ada025d838265df3a0d0fb12b Mon Sep 17 00:00:00 2001 From: cmdr-chara <249489759+cmdr-chara@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:17:57 +0200 Subject: [PATCH 1/2] feat(suite): add declarative backend suite sync --- CHANGELOG.md | 1 + DOCUMENTATION.md | 64 +- README.md | 2 + src/commands/suite.test.ts | 455 ++++++ src/commands/suite.ts | 1237 +++++++++++++++++ src/index.ts | 4 +- test/__snapshots__/help.snapshot.test.ts.snap | 89 +- test/help.snapshot.test.ts | 5 + 8 files changed, 1852 insertions(+), 5 deletions(-) create mode 100644 src/commands/suite.test.ts create mode 100644 src/commands/suite.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 45803e1..4892090 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ All notable changes to `@testsprite/testsprite-cli` are documented here. The for ### Added +- **Declarative backend Suitefiles.** New `suite validate`, `suite graph`, `suite plan`, and `suite apply` commands validate versioned manifests and Python sources, compile dependency declarations into deterministic execution waves, detect remote metadata/code drift, and reconcile creates/updates with explicit confirmation. Adjacent atomic lockfiles preserve stable remote identities and interrupted-create idempotency; name collisions and changed pending creates fail as conflicts, while unmanaged remote tests are reported and never deleted. - **`test cancel `** — user-initiated cancel of in-flight runs (the real stop button; Ctrl-C only detaches). A single id renders the run card with status `cancelled` (plus an advisory when it was already cancelled); multiple ids print a `{cancelled, alreadyCancelled, conflicts, notFound}` summary. Exit codes: 4 when any id is not found, else 6 on conflicts, else 0. `--dry-run` supported. - **Graceful Ctrl-C during `--wait`** — SIGINT/SIGTERM now detaches cleanly instead of killing the process mid-poll: the in-flight request aborts immediately, stdout gets the same partial `{runId, status: "running"}` envelope as the request-timeout path, and stderr states the truth — the server-side run keeps executing (and billing) — with a re-attach hint and a `test cancel` pointer. Exit 130/143/129 per the documented signal contract; a second signal forces a hard exit. Interrupting never cancels the server-side run — that's what `test cancel` is for. - **`project delete --confirm`** — permanently delete a project and everything under it (its frontend/backend sub-projects, all their tests, and backend fixtures), mirroring the Portal's cascade delete. Requires `--confirm` (the CLI never prompts); `--dry-run` previews the response shape without a network call. Standard exit codes: 0 success, 3 auth, 4 not-found (or already-deleted), 5 validation (e.g. missing `--confirm`). diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index bf6b1b5..c2c6778 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -11,6 +11,7 @@ The full reference for the TestSprite CLI: install verification, manual setup, e - [Manual setup](#manual-setup) - [The complete agent loop](#the-complete-agent-loop) - [Agent onboarding (`agent install`)](#agent-onboarding-agent-install) +- [Declarative backend suites (`suite`)](#declarative-backend-suites-suite) - [Command reference](#command-reference) - [Read commands](#read-commands) - [Write commands](#write-commands) @@ -45,7 +46,7 @@ testsprite --version testsprite project list --dry-run --output json ``` -`--dry-run` is a global flag that skips the network, credentials, and the local filesystem and emits a canned sample matching the API contract. It's the right way to confirm an install or learn the surface before configuring auth — the response _shapes_ match the wire contract, but the data is fake. +`--dry-run` is a global flag that skips the network, credentials, and filesystem writes and emits a canned sample matching the API contract. File-backed plan inputs and Suitefile manifests/code are still read and validated locally. It's the right way to confirm an install or learn the surface before configuring auth — the response _shapes_ match the wire contract, but the remote data is fake. ## Manual setup @@ -131,6 +132,67 @@ The `codex` target uses **managed-section mode** — it writes only a sentinel-d Re-running with `--force` on **own-file targets** (claude, cursor, cline, antigravity, kiro, windsurf, copilot) backs up the existing file to `.bak` first. +## Declarative backend suites (`suite`) + +A Suitefile keeps a backend test suite in source control: test identity, metadata, Python source paths, and data dependencies. The CLI validates the complete local definition, compiles `produces`/`consumes` into deterministic execution waves, compares it with the remote project, then performs only the reviewed creates and updates. + +```json +{ + "schemaVersion": 1, + "projectId": "proj_xxxxxxxx", + "tests": [ + { + "key": "auth/session", + "name": "Create API session", + "codeFile": "tests/auth_session.py", + "priority": "p0", + "produces": ["session_token"], + "category": "setup" + }, + { + "key": "orders/create", + "name": "Create order", + "codeFile": "tests/create_order.py", + "consumes": ["session_token"] + }, + { + "key": "orders/cleanup", + "name": "Clean up order data", + "codeFile": "tests/cleanup.py", + "category": "teardown" + } + ] +} +``` + +Paths are relative to the Suitefile and must resolve to contained `.py` files. Unknown fields, duplicate keys/test ids, missing or ambiguous producers, self-dependencies, cycles, lexical or symlink path escapes, and oversized code files fail locally before credentials or network are touched. `teardown` and `cleanup` tests are forced behind non-teardown tests. + +```bash +# Pure-local checks +testsprite suite validate ./testsprite.suite.json +testsprite suite graph ./testsprite.suite.json --output json + +# Live comparison; makes no changes +testsprite suite plan ./testsprite.suite.json + +# Fully validate locally without credentials/network +testsprite suite plan ./testsprite.suite.json --dry-run --output json + +# Apply exactly the create/update plan +testsprite suite apply ./testsprite.suite.json --confirm +``` + +The adjacent `testsprite.suite.lock.json` records stable suite-key → remote-test identity, code version, and desired-state hash. Check it into source control with the manifest. You can override its location with `--lock-file `. + +Safety rules are deliberately strict: + +- Existing tests are adopted only through an explicit manifest `testId` or the lockfile. A same-name remote test is a conflict, not an implicit match. +- Unmanaged remote tests are reported by `plan` and are **never deleted** by `apply`. +- Backend tests only are supported in schema version 1; an adopted frontend test is a conflict. +- Creates and updates use deterministic idempotency keys. The lockfile is written atomically, and interrupted creates retain their request identity for safe replay; changing the definition while a create is pending produces a conflict. +- `name`, Python code, `produces`, and `consumes` are authoritative. Omitted `produces`/`consumes` mean empty lists. `priority` and `category` are changed only when declared. +- `apply` requires `--confirm` whenever the plan contains a mutation. It refuses any plan containing conflicts. + ## Command reference Every command supports the [global flags](#global-flags), and every example below pairs a real call with a `--dry-run` companion that works on a fresh install with no auth. diff --git a/README.md b/README.md index 3bb465e..2551b5b 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,8 @@ Prefer to configure each step by hand (or learn the surface offline with `--dry- | | `test failure summary` | One-screen triage card (no media download) | | | `test diff` | Compare two runs — verdict, failure kind, per-step status flips, code-version drift | | **Write** | `test scaffold` / `test lint` | Author plans locally: emit a schema-correct starter, validate plan files offline — no network, no credentials | +| | `suite validate` / `suite graph` / `suite plan` | Compile a declarative backend Suitefile into dependency waves and preview drift against the remote project | +| | `suite apply` | Reconcile backend test metadata and Python code with guarded, idempotent writes; unmanaged remote tests are never deleted | | | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | | | `test update` / `test delete` / `test delete-batch` | Edit metadata and BE dependency declarations (`--produces` / `--needs` / `--category`) / permanently delete (no restore window; `--confirm` required) | | | `test code put` | Replace generated code (etag-guarded) | diff --git a/src/commands/suite.test.ts b/src/commands/suite.test.ts new file mode 100644 index 0000000..62755c1 --- /dev/null +++ b/src/commands/suite.test.ts @@ -0,0 +1,455 @@ +import { mkdirSync, mkdtempSync, readFileSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { ApiError } from '../lib/errors.js'; +import type { FetchImpl } from '../lib/http.js'; +import { + buildSuiteGraph, + loadSuiteManifest, + runSuiteApply, + runSuitePlan, + type SuiteManifest, +} from './suite.js'; + +function writeSuite( + tests: Array>, + projectId = 'proj_suite_1', +): { dir: string; manifestPath: string; lockPath: string } { + const dir = mkdtempSync(join(tmpdir(), 'testsprite-suite-')); + for (const test of tests) { + const codeFile = String(test.codeFile); + writeFileSync( + join(dir, codeFile), + `def test_${String(test.key).replace(/\W/g, '_')}():\n assert True\n`, + ); + } + const manifestPath = join(dir, 'testsprite.suite.json'); + writeFileSync(manifestPath, JSON.stringify({ schemaVersion: 1, projectId, tests }, null, 2)); + return { dir, manifestPath, lockPath: join(dir, 'testsprite.suite.lock.json') }; +} + +function common(fetchImpl?: FetchImpl) { + return { + profile: 'default', + output: 'json' as const, + dryRun: false, + debug: false, + verbose: false, + endpointUrl: 'https://api.example.test', + fetchImpl, + env: { TESTSPRITE_API_KEY: 'sk-suite-test' }, + stdout: () => undefined, + stderr: () => undefined, + now: () => new Date('2026-07-21T12:00:00.000Z'), + }; +} + +function json(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json', 'x-request-id': 'req_suite_test' }, + }); +} + +describe('Suitefile validation and graph compilation', () => { + it('compiles producer, consumer, and teardown tests into stable waves', () => { + const { manifestPath } = writeSuite([ + { key: 'auth', name: 'Auth setup', codeFile: 'auth.py', produces: ['token'] }, + { + key: 'checkout', + name: 'Checkout', + codeFile: 'checkout.py', + consumes: ['token'], + produces: ['order'], + }, + { + key: 'cleanup', + name: 'Cleanup', + codeFile: 'cleanup.py', + consumes: ['order'], + category: 'teardown', + }, + ]); + + const loaded = loadSuiteManifest(manifestPath); + + expect(loaded.graph.waves).toEqual([ + { wave: 1, tests: ['auth'] }, + { wave: 2, tests: ['checkout'] }, + { wave: 3, tests: ['cleanup'] }, + ]); + expect(loaded.graph.edges).toEqual([ + { from: 'auth', to: 'checkout', variable: 'token' }, + { from: 'auth', to: 'cleanup', variable: '$teardown' }, + { from: 'checkout', to: 'cleanup', variable: '$teardown' }, + { from: 'checkout', to: 'cleanup', variable: 'order' }, + ]); + expect(loaded.graph.producers).toEqual({ order: 'checkout', token: 'auth' }); + }); + + it('rejects missing and ambiguous producers before any network access', () => { + const manifest: SuiteManifest = { + schemaVersion: 1, + projectId: 'proj_graph', + tests: [ + { + key: 'a', + name: 'A', + codeFile: 'a.py', + produces: ['token'], + consumes: [], + }, + { + key: 'b', + name: 'B', + codeFile: 'b.py', + produces: ['token'], + consumes: [], + }, + { + key: 'c', + name: 'C', + codeFile: 'c.py', + produces: [], + consumes: ['missing'], + }, + ], + }; + + try { + buildSuiteGraph(manifest); + throw new Error('expected graph validation to fail'); + } catch (error) { + expect(error).toBeInstanceOf(ApiError); + expect((error as ApiError).nextAction).toMatch( + /ambiguous producers.*no suite test produces/s, + ); + } + }); + + it('rejects code paths that escape the Suitefile directory', () => { + const dir = mkdtempSync(join(tmpdir(), 'testsprite-suite-escape-')); + const manifestPath = join(dir, 'suite.json'); + writeFileSync( + manifestPath, + JSON.stringify({ + schemaVersion: 1, + projectId: 'proj_escape', + tests: [{ key: 'escape', name: 'Escape', codeFile: '../secret.py' }], + }), + ); + + try { + loadSuiteManifest(manifestPath); + throw new Error('expected path validation to fail'); + } catch (error) { + expect(error).toBeInstanceOf(ApiError); + expect((error as ApiError).nextAction).toContain('escapes the Suitefile directory'); + } + }); + + it.runIf(process.platform !== 'win32')('rejects code-file symlinks that escape the suite', () => { + const parent = mkdtempSync(join(tmpdir(), 'testsprite-suite-symlink-')); + const suiteDir = join(parent, 'suite'); + mkdirSync(suiteDir); + writeFileSync(join(parent, 'outside.py'), 'SECRET = True\n'); + symlinkSync(join(parent, 'outside.py'), join(suiteDir, 'linked.py')); + const manifestPath = join(suiteDir, 'suite.json'); + writeFileSync( + manifestPath, + JSON.stringify({ + schemaVersion: 1, + projectId: 'proj_escape', + tests: [{ key: 'escape', name: 'Escape', codeFile: 'linked.py' }], + }), + ); + + expect(() => loadSuiteManifest(manifestPath)).toThrowError( + expect.objectContaining({ nextAction: expect.stringContaining('symlink escape') }), + ); + }); + + it('rejects unknown fields instead of silently ignoring manifest typos', () => { + const { manifestPath } = writeSuite([ + { + key: 'health', + name: 'Health', + codeFile: 'health.py', + consume: ['misspelled-consumes'], + }, + ]); + + expect(() => loadSuiteManifest(manifestPath)).toThrowError( + expect.objectContaining({ + code: 'VALIDATION_ERROR', + nextAction: expect.stringContaining('tests[0].consume is not supported'), + }), + ); + }); +}); + +describe('suite plan', () => { + it('provides an offline dry-run plan without credentials or network', async () => { + const { manifestPath } = writeSuite([{ key: 'health', name: 'Health', codeFile: 'health.py' }]); + const plan = await runSuitePlan( + { + profile: 'default', + output: 'json', + dryRun: true, + debug: false, + verbose: false, + manifestPath, + }, + { stdout: () => undefined, stderr: () => undefined }, + ); + + expect(plan.dryRun).toBe(true); + expect(plan.summary).toEqual({ create: 1, update: 0, noop: 0, conflict: 0 }); + }); + + it('plans an update from live metadata and code drift', async () => { + const { manifestPath } = writeSuite([ + { + key: 'auth', + testId: 'test_auth', + name: 'Auth current', + codeFile: 'auth.py', + priority: 'p0', + produces: ['token'], + }, + ]); + const fetchImpl: FetchImpl = async input => { + const url = new URL(String(input)); + if (url.pathname.endsWith('/tests')) { + return json({ + items: [ + { + id: 'test_auth', + projectId: 'proj_suite_1', + name: 'Auth old', + type: 'backend', + createdFrom: 'cli', + status: 'ready', + priority: 'p1', + produces: [], + consumes: [], + category: null, + createdAt: '2026-01-01T00:00:00.000Z', + updatedAt: '2026-01-01T00:00:00.000Z', + }, + ], + nextToken: null, + }); + } + if (url.pathname.endsWith('/tests/test_auth/code')) { + return json({ + testId: 'test_auth', + language: 'python', + framework: 'pytest', + code: 'def test_old():\n assert False\n', + codeVersion: 'v7', + }); + } + throw new Error(`unexpected request: ${url}`); + }; + + const plan = await runSuitePlan({ ...common(fetchImpl), manifestPath }, common(fetchImpl)); + + expect(plan.summary).toEqual({ create: 0, update: 1, noop: 0, conflict: 0 }); + expect(plan.items[0]).toMatchObject({ + key: 'auth', + testId: 'test_auth', + action: 'update', + changes: ['name', 'priority', 'produces', 'code'], + }); + }); + + it('refuses an implicit adoption when an unmanaged remote test has the same name', async () => { + const { manifestPath } = writeSuite([{ key: 'health', name: 'Health', codeFile: 'health.py' }]); + const fetchImpl: FetchImpl = async () => + json({ + items: [ + { + id: 'test_existing', + projectId: 'proj_suite_1', + name: 'Health', + type: 'backend', + createdFrom: 'portal', + status: 'ready', + createdAt: '2026-01-01T00:00:00.000Z', + updatedAt: '2026-01-01T00:00:00.000Z', + }, + ], + nextToken: null, + }); + + const plan = await runSuitePlan({ ...common(fetchImpl), manifestPath }, common(fetchImpl)); + + expect(plan.items[0]).toMatchObject({ action: 'conflict' }); + expect(plan.items[0]?.reason).toContain('add testId'); + }); +}); + +describe('suite apply', () => { + it('updates an existing test, creates a new test, and writes an atomic lockfile', async () => { + const { manifestPath, lockPath } = writeSuite([ + { + key: 'auth', + testId: 'test_auth', + name: 'Auth current', + codeFile: 'auth.py', + produces: ['token'], + }, + { + key: 'checkout', + name: 'Checkout', + codeFile: 'checkout.py', + consumes: ['token'], + }, + ]); + const requests: Array<{ + method: string; + path: string; + body: unknown; + idempotencyKey: string | null; + }> = []; + const fetchImpl: FetchImpl = async (input, init) => { + const url = new URL(String(input)); + const method = init?.method ?? 'GET'; + const headers = new Headers(init?.headers); + const body = typeof init?.body === 'string' ? JSON.parse(init.body) : undefined; + requests.push({ + method, + path: url.pathname, + body, + idempotencyKey: headers.get('idempotency-key'), + }); + if (method === 'GET' && url.pathname.endsWith('/tests')) { + return json({ + items: [ + { + id: 'test_auth', + projectId: 'proj_suite_1', + name: 'Auth old', + type: 'backend', + createdFrom: 'cli', + status: 'ready', + produces: [], + consumes: [], + createdAt: '2026-01-01T00:00:00.000Z', + updatedAt: '2026-01-01T00:00:00.000Z', + }, + ], + nextToken: null, + }); + } + if (method === 'GET' && url.pathname.endsWith('/tests/test_auth/code')) { + return json({ + testId: 'test_auth', + language: 'python', + framework: 'pytest', + code: 'old code', + codeVersion: 'v2', + }); + } + if (method === 'PUT' && url.pathname.endsWith('/tests/test_auth')) { + return json({ testId: 'test_auth', updatedFields: ['name', 'produces'], updatedAt: 'now' }); + } + if (method === 'PUT' && url.pathname.endsWith('/tests/test_auth/code')) { + return json({ testId: 'test_auth', codeVersion: 'v3', updatedAt: 'now' }); + } + if (method === 'POST' && url.pathname.endsWith('/tests')) { + return json({ + testId: 'test_checkout', + type: 'backend', + codeVersion: 'v1', + createdAt: 'now', + }); + } + throw new Error(`unexpected request: ${method} ${url.pathname}`); + }; + + const result = await runSuiteApply( + { ...common(fetchImpl), manifestPath, confirm: true }, + common(fetchImpl), + ); + + expect('summary' in result && result.summary).toEqual({ created: 1, updated: 1, unchanged: 0 }); + const lock = JSON.parse(readFileSync(lockPath, 'utf8')) as { + entries: Record; + }; + expect(lock.entries.auth).toMatchObject({ testId: 'test_auth', codeVersion: 'v3' }); + expect(lock.entries.checkout).toMatchObject({ testId: 'test_checkout', codeVersion: 'v1' }); + const mutationKeys = requests + .filter(request => request.method !== 'GET') + .map(request => request.idempotencyKey); + expect(mutationKeys).toHaveLength(3); + expect(mutationKeys.every(key => key?.startsWith('cli-suite-v1-'))).toBe(true); + }); + + it('requires explicit confirmation before remote mutation', async () => { + const { manifestPath } = writeSuite([{ key: 'health', name: 'Health', codeFile: 'health.py' }]); + const fetchImpl: FetchImpl = async () => json({ items: [], nextToken: null }); + + await expect( + runSuiteApply({ ...common(fetchImpl), manifestPath, confirm: false }, common(fetchImpl)), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + nextAction: expect.stringContaining('required to apply 1 suite mutation'), + }); + }); + + it('resumes an unchanged pending create and conflicts if its definition drifted', async () => { + const { manifestPath, lockPath, dir } = writeSuite([ + { key: 'health', name: 'Health', codeFile: 'health.py' }, + ]); + const createFetch: FetchImpl = async (input, init) => { + const url = new URL(String(input)); + if ((init?.method ?? 'GET') === 'GET' && url.pathname.endsWith('/tests')) { + return json({ items: [], nextToken: null }); + } + if ((init?.method ?? 'GET') === 'POST' && url.pathname.endsWith('/tests')) { + return json({ + testId: 'test_health', + type: 'backend', + codeVersion: 'v1', + createdAt: 'now', + }); + } + throw new Error(`unexpected request: ${init?.method ?? 'GET'} ${url.pathname}`); + }; + await runSuiteApply( + { ...common(createFetch), manifestPath, confirm: true }, + common(createFetch), + ); + + const completed = JSON.parse(readFileSync(lockPath, 'utf8')) as { + entries: Record; + }; + completed.entries.health = { + desiredHash: completed.entries.health!.desiredHash, + createKey: 'cli-suite-v1-create-pending', + updatedAt: completed.entries.health!.updatedAt, + } as { desiredHash: string; updatedAt: string }; + writeFileSync(lockPath, JSON.stringify(completed, null, 2)); + + const emptyRemote: FetchImpl = async () => json({ items: [], nextToken: null }); + const resumed = await runSuitePlan( + { ...common(emptyRemote), manifestPath }, + common(emptyRemote), + ); + expect(resumed.items[0]).toMatchObject({ + action: 'create', + changes: ['resume pending idempotent create'], + }); + + writeFileSync(join(dir, 'health.py'), 'def test_health():\n assert False\n'); + const drifted = await runSuitePlan( + { ...common(emptyRemote), manifestPath }, + common(emptyRemote), + ); + expect(drifted.items[0]).toMatchObject({ action: 'conflict' }); + expect(drifted.items[0]?.reason).toContain('changed after a create request became pending'); + }); +}); diff --git a/src/commands/suite.ts b/src/commands/suite.ts new file mode 100644 index 0000000..e20b2d3 --- /dev/null +++ b/src/commands/suite.ts @@ -0,0 +1,1237 @@ +import { createHash, randomUUID } from 'node:crypto'; +import { + existsSync, + readFileSync, + realpathSync, + renameSync, + statSync, + unlinkSync, + writeFileSync, +} from 'node:fs'; +import { basename, dirname, isAbsolute, relative, resolve } from 'node:path'; +import { Command } from 'commander'; +import { + makeHttpClient, + parseRequestTimeoutFlag, + type CommonOptions, +} from '../lib/client-factory.js'; +import { localValidationError } from '../lib/errors.js'; +import type { FetchImpl, HttpClient } from '../lib/http.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode, type OutputMode } from '../lib/output.js'; +import { paginate, type Page } from '../lib/pagination.js'; +import type { + CliCreateTestResponse, + CliPutTestCodeResponse, + CliTest, + CliTestCode, + CliUpdateTestResponse, +} from './test.js'; + +const SUITE_SCHEMA_VERSION = 1; +const LOCK_SCHEMA_VERSION = 1; +const MAX_SUITE_TESTS = 500; +const MAX_CODE_BYTES = 350 * 1024; +const PRIORITIES = ['p0', 'p1', 'p2', 'p3'] as const; +const KEY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._/-]*$/; +const MANIFEST_FIELDS = new Set(['schemaVersion', 'projectId', 'tests']); +const TEST_FIELDS = new Set([ + 'key', + 'testId', + 'name', + 'codeFile', + 'priority', + 'produces', + 'consumes', + 'category', +]); + +type Priority = (typeof PRIORITIES)[number]; +type SuitePlanAction = 'create' | 'update' | 'noop' | 'conflict'; + +export interface SuiteTestDefinition { + key: string; + testId?: string; + name: string; + codeFile: string; + priority?: Priority; + produces: string[]; + consumes: string[]; + category?: string; +} + +export interface SuiteManifest { + schemaVersion: typeof SUITE_SCHEMA_VERSION; + projectId: string; + tests: SuiteTestDefinition[]; +} + +export interface SuiteWave { + wave: number; + tests: string[]; +} + +export interface SuiteGraph { + projectId: string; + tests: number; + edges: Array<{ from: string; to: string; variable: string }>; + waves: SuiteWave[]; + producers: Record; +} + +interface SuiteLockEntry { + testId?: string; + codeVersion?: string | null; + desiredHash: string; + createKey?: string; + updatedAt: string; +} + +interface SuiteLock { + schemaVersion: typeof LOCK_SCHEMA_VERSION; + projectId: string; + entries: Record; +} + +export interface SuitePlanItem { + key: string; + testId?: string; + action: SuitePlanAction; + changes: string[]; + reason?: string; +} + +export interface SuitePlan { + schemaVersion: 1; + projectId: string; + manifest: string; + lockFile: string; + dryRun: boolean; + graph: SuiteGraph; + items: SuitePlanItem[]; + summary: Record; + unmanagedRemote: string[]; +} + +export interface SuiteApplyResult { + projectId: string; + lockFile: string; + applied: Array<{ key: string; testId: string; action: 'created' | 'updated' }>; + unchanged: string[]; + summary: { created: number; updated: number; unchanged: number }; +} + +export interface SuiteDeps { + env?: NodeJS.ProcessEnv; + credentialsPath?: string; + fetchImpl?: FetchImpl; + stdout?: (line: string) => void; + stderr?: (line: string) => void; + now?: () => Date; +} + +interface ManifestContext { + manifest: SuiteManifest; + manifestPath: string; + manifestDir: string; + lockPath: string; + codeByKey: Map; + desiredHashByKey: Map; + graph: SuiteGraph; +} + +interface ResolvedPlanItem extends SuitePlanItem { + spec: SuiteTestDefinition; + desiredCode: string; + desiredHash: string; + remote?: CliTest; + remoteCode?: CliTestCode; + createKey?: string; +} + +interface CalculatedPlan { + publicPlan: SuitePlan; + context: ManifestContext; + lock: SuiteLock; + items: ResolvedPlanItem[]; +} + +interface SuiteFileOptions extends CommonOptions { + manifestPath: string; + lockFile?: string; +} + +interface SuiteApplyOptions extends SuiteFileOptions { + confirm: boolean; +} + +/** Read and fully validate a backend Suitefile, including every referenced code file. */ +export function loadSuiteManifest(manifestPath: string, lockFile?: string): ManifestContext { + const absoluteManifest = resolve(manifestPath); + const manifestDir = dirname(absoluteManifest); + const lockPath = resolveLockPath(absoluteManifest, lockFile); + const raw = stripBom(readTextFile(absoluteManifest, 'manifest')); + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (error) { + throw localValidationError( + 'manifest', + `is not valid JSON: ${error instanceof Error ? error.message : String(error)}`, + ); + } + + const errors: string[] = []; + if (!isRecord(parsed)) { + throw localValidationError('manifest', 'must be a JSON object'); + } + rejectUnknownFields(parsed, MANIFEST_FIELDS, 'manifest', errors); + if (parsed.schemaVersion !== SUITE_SCHEMA_VERSION) { + errors.push(`schemaVersion must be ${SUITE_SCHEMA_VERSION}`); + } + const projectId = readRequiredString(parsed.projectId, 'projectId', errors, 200); + if (!Array.isArray(parsed.tests)) { + errors.push('tests must be an array'); + } else if (parsed.tests.length === 0 || parsed.tests.length > MAX_SUITE_TESTS) { + errors.push(`tests must contain between 1 and ${MAX_SUITE_TESTS} entries`); + } + + const tests: SuiteTestDefinition[] = []; + const seenKeys = new Set(); + const seenTestIds = new Set(); + if (Array.isArray(parsed.tests)) { + for (let index = 0; index < parsed.tests.length; index += 1) { + const value = parsed.tests[index]; + const prefix = `tests[${index}]`; + if (!isRecord(value)) { + errors.push(`${prefix} must be an object`); + continue; + } + rejectUnknownFields(value, TEST_FIELDS, prefix, errors); + const key = readRequiredString(value.key, `${prefix}.key`, errors, 128); + const name = readRequiredString(value.name, `${prefix}.name`, errors, 200); + const codeFile = readRequiredString(value.codeFile, `${prefix}.codeFile`, errors, 500); + const testId = readOptionalString(value.testId, `${prefix}.testId`, errors, 200); + const priority = readPriority(value.priority, `${prefix}.priority`, errors); + const produces = readStringArray(value.produces, `${prefix}.produces`, errors); + const consumes = readStringArray(value.consumes, `${prefix}.consumes`, errors); + const category = readOptionalString(value.category, `${prefix}.category`, errors, 100); + + if (key && !KEY_PATTERN.test(key)) { + errors.push(`${prefix}.key must match ${KEY_PATTERN.source}`); + } + if (key && seenKeys.has(key)) errors.push(`${prefix}.key duplicates "${key}"`); + if (key) seenKeys.add(key); + if (testId && seenTestIds.has(testId)) errors.push(`${prefix}.testId duplicates "${testId}"`); + if (testId) seenTestIds.add(testId); + + if (key && name && codeFile) { + tests.push({ + key, + ...(testId ? { testId } : {}), + name, + codeFile, + ...(priority ? { priority } : {}), + produces, + consumes, + ...(category ? { category } : {}), + }); + } + } + } + + if (errors.length > 0) throw manifestValidationError(errors); + + const codeByKey = new Map(); + const desiredHashByKey = new Map(); + for (const spec of tests) { + const codePath = resolveContainedCodePath(manifestDir, spec.codeFile, spec.key); + const code = readCodeFile(codePath, manifestDir, spec.key); + codeByKey.set(spec.key, code); + desiredHashByKey.set(spec.key, desiredHash(spec, code)); + } + + const manifest: SuiteManifest = { + schemaVersion: SUITE_SCHEMA_VERSION, + projectId, + tests, + }; + const graph = buildSuiteGraph(manifest); + return { + manifest, + manifestPath: absoluteManifest, + manifestDir, + lockPath, + codeByKey, + desiredHashByKey, + graph, + }; +} + +/** Compile produces/consumes declarations into deterministic execution waves. */ +export function buildSuiteGraph(manifest: SuiteManifest): SuiteGraph { + const errors: string[] = []; + const producers = new Map(); + for (const test of manifest.tests) { + for (const variable of test.produces) { + const existing = producers.get(variable); + if (existing !== undefined && existing !== test.key) { + errors.push(`variable "${variable}" has ambiguous producers: ${existing}, ${test.key}`); + } else { + producers.set(variable, test.key); + } + } + } + + const outgoing = new Map>(); + const indegree = new Map(); + const edges: Array<{ from: string; to: string; variable: string }> = []; + for (const test of manifest.tests) { + outgoing.set(test.key, new Set()); + indegree.set(test.key, 0); + } + for (const test of manifest.tests) { + for (const variable of test.consumes) { + const producer = producers.get(variable); + if (producer === undefined) { + errors.push(`${test.key} consumes "${variable}" but no suite test produces it`); + continue; + } + if (producer === test.key) { + errors.push(`${test.key} both produces and consumes "${variable}"`); + continue; + } + addGraphEdge(producer, test.key, variable, outgoing, indegree, edges); + } + } + + // Teardown/cleanup tests are compiled into the final wave, matching the + // backend wave planner's category contract even when they declare no inputs. + const teardownKeys = new Set( + manifest.tests + .filter(test => ['teardown', 'cleanup'].includes(test.category?.toLowerCase() ?? '')) + .map(test => test.key), + ); + for (const teardown of teardownKeys) { + for (const test of manifest.tests) { + if (test.key !== teardown && !teardownKeys.has(test.key)) { + addGraphEdge(test.key, teardown, '$teardown', outgoing, indegree, edges); + } + } + } + + if (errors.length > 0) throw manifestValidationError(errors); + + const waves: SuiteWave[] = []; + const remaining = new Map(indegree); + let ready = [...remaining.entries()] + .filter(([, degree]) => degree === 0) + .map(([key]) => key) + .sort(); + let visited = 0; + while (ready.length > 0) { + waves.push({ wave: waves.length + 1, tests: ready }); + visited += ready.length; + const next = new Set(); + for (const key of ready) { + for (const dependent of outgoing.get(key) ?? []) { + const degree = (remaining.get(dependent) ?? 0) - 1; + remaining.set(dependent, degree); + if (degree === 0) next.add(dependent); + } + } + ready = [...next].sort(); + } + if (visited !== manifest.tests.length) { + const cycle = [...remaining.entries()] + .filter(([, degree]) => degree > 0) + .map(([key]) => key) + .sort(); + throw manifestValidationError([`dependency cycle detected among: ${cycle.join(', ')}`]); + } + + return { + projectId: manifest.projectId, + tests: manifest.tests.length, + edges: edges.sort((a, b) => + `${a.from}:${a.to}:${a.variable}`.localeCompare(`${b.from}:${b.to}:${b.variable}`), + ), + waves, + producers: Object.fromEntries([...producers.entries()].sort(([a], [b]) => a.localeCompare(b))), + }; +} + +export async function runSuiteValidate( + opts: SuiteFileOptions, + deps: SuiteDeps = {}, +): Promise<{ valid: true; manifest: string; lockFile: string; graph: SuiteGraph }> { + const context = loadSuiteManifest(opts.manifestPath, opts.lockFile); + const result = { + valid: true as const, + manifest: context.manifestPath, + lockFile: context.lockPath, + graph: context.graph, + }; + makeOutput(opts.output, deps).print(result, () => renderValidationText(result)); + return result; +} + +export async function runSuiteGraph( + opts: SuiteFileOptions, + deps: SuiteDeps = {}, +): Promise { + const context = loadSuiteManifest(opts.manifestPath, opts.lockFile); + makeOutput(opts.output, deps).print(context.graph, data => renderGraphText(data as SuiteGraph)); + return context.graph; +} + +export async function runSuitePlan( + opts: SuiteFileOptions, + deps: SuiteDeps = {}, +): Promise { + const calculated = await calculateSuitePlan(opts, deps); + makeOutput(opts.output, deps).print(calculated.publicPlan, data => + renderPlanText(data as SuitePlan), + ); + return calculated.publicPlan; +} + +export async function runSuiteApply( + opts: SuiteApplyOptions, + deps: SuiteDeps = {}, +): Promise { + const calculated = await calculateSuitePlan(opts, deps); + const out = makeOutput(opts.output, deps); + if (opts.dryRun) { + out.print(calculated.publicPlan, data => renderPlanText(data as SuitePlan)); + return calculated.publicPlan; + } + const conflicts = calculated.items.filter(item => item.action === 'conflict'); + if (conflicts.length > 0) { + throw localValidationError( + 'suite', + `plan contains ${conflicts.length} conflict(s): ${conflicts.map(item => item.key).join(', ')}`, + ); + } + const mutations = calculated.items.filter( + item => item.action === 'create' || item.action === 'update', + ); + if (mutations.length > 0 && !opts.confirm) { + throw localValidationError( + 'confirm', + `required to apply ${mutations.length} suite mutation(s); inspect \`testsprite suite plan ${opts.manifestPath}\` first`, + ); + } + + const client = makeClient(opts, deps); + const lock = calculated.lock; + const applied: SuiteApplyResult['applied'] = []; + const unchanged: string[] = []; + for (const item of calculated.items) { + if (item.action === 'noop') { + unchanged.push(item.key); + if (item.testId) { + lock.entries[item.key] = makeCompletedLockEntry( + item.testId, + item.remoteCode?.codeVersion, + item.desiredHash, + deps, + ); + } + continue; + } + if (item.action === 'create') { + const createKey = + item.createKey ?? + suiteIdempotencyKey('create', calculated.context.manifest.projectId, item); + lock.entries[item.key] = { + desiredHash: item.desiredHash, + createKey, + updatedAt: nowIso(deps), + }; + writeSuiteLock(calculated.context.lockPath, lock); + const created = await client.post('/tests', { + body: createBody(calculated.context.manifest.projectId, item.spec, item.desiredCode), + headers: { 'idempotency-key': createKey }, + }); + lock.entries[item.key] = makeCompletedLockEntry( + created.testId, + created.codeVersion, + item.desiredHash, + deps, + ); + writeSuiteLock(calculated.context.lockPath, lock); + applied.push({ key: item.key, testId: created.testId, action: 'created' }); + continue; + } + if (item.action === 'update' && item.testId) { + let codeVersion = item.remoteCode?.codeVersion; + const metadataBody = updateMetadataBody(item); + if (Object.keys(metadataBody).length > 0) { + await client.put(`/tests/${encodeURIComponent(item.testId)}`, { + body: metadataBody, + headers: { + 'idempotency-key': suiteIdempotencyKey( + 'metadata', + calculated.context.manifest.projectId, + item, + ), + }, + }); + } + if (item.changes.includes('code')) { + const updatedCode = await client.put( + `/tests/${encodeURIComponent(item.testId)}/code`, + { + body: { code: item.desiredCode, language: 'python' }, + headers: { + 'idempotency-key': suiteIdempotencyKey( + 'code', + calculated.context.manifest.projectId, + item, + ), + 'if-match': codeVersion ?? '*', + }, + }, + ); + codeVersion = updatedCode.codeVersion; + } + lock.entries[item.key] = makeCompletedLockEntry( + item.testId, + codeVersion, + item.desiredHash, + deps, + ); + writeSuiteLock(calculated.context.lockPath, lock); + applied.push({ key: item.key, testId: item.testId, action: 'updated' }); + } + } + writeSuiteLock(calculated.context.lockPath, lock); + const result: SuiteApplyResult = { + projectId: calculated.context.manifest.projectId, + lockFile: calculated.context.lockPath, + applied, + unchanged, + summary: { + created: applied.filter(item => item.action === 'created').length, + updated: applied.filter(item => item.action === 'updated').length, + unchanged: unchanged.length, + }, + }; + out.print(result, data => renderApplyText(data as SuiteApplyResult)); + return result; +} + +export function createSuiteCommand(deps: SuiteDeps = {}): Command { + const suite = new Command('suite').description( + 'Validate, plan, and apply a declarative backend test suite from a versioned Suitefile', + ); + suite + .command('validate ') + .description( + 'Validate Suitefile structure, code paths, dependencies, and execution waves locally', + ) + .option('--lock-file ', 'override the adjacent *.lock.json path') + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (manifest: string, flags: { lockFile?: string }, command: Command) => { + await runSuiteValidate( + { ...resolveCommonOptions(command), manifestPath: manifest, lockFile: flags.lockFile }, + deps, + ); + }); + suite + .command('graph ') + .description('Compile produces/consumes declarations into deterministic execution waves') + .option('--lock-file ', 'override the adjacent *.lock.json path') + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (manifest: string, flags: { lockFile?: string }, command: Command) => { + await runSuiteGraph( + { ...resolveCommonOptions(command), manifestPath: manifest, lockFile: flags.lockFile }, + deps, + ); + }); + suite + .command('plan ') + .description('Compare the Suitefile with the remote project without changing either side') + .option('--lock-file ', 'override the adjacent *.lock.json path') + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (manifest: string, flags: { lockFile?: string }, command: Command) => { + await runSuitePlan( + { ...resolveCommonOptions(command), manifestPath: manifest, lockFile: flags.lockFile }, + deps, + ); + }); + suite + .command('apply ') + .description('Create and update backend tests from a previously reviewed Suitefile plan') + .option('--lock-file ', 'override the adjacent *.lock.json path') + .option('--confirm', 'required before remote mutations; deletions are never performed', false) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action( + async ( + manifest: string, + flags: { lockFile?: string; confirm?: boolean }, + command: Command, + ) => { + await runSuiteApply( + { + ...resolveCommonOptions(command), + manifestPath: manifest, + lockFile: flags.lockFile, + confirm: flags.confirm === true, + }, + deps, + ); + }, + ); + return suite; +} + +async function calculateSuitePlan( + opts: SuiteFileOptions, + deps: SuiteDeps, +): Promise { + const context = loadSuiteManifest(opts.manifestPath, opts.lockFile); + const lock = loadSuiteLock(context.lockPath, context.manifest.projectId); + assertManifestLockAgreement(context.manifest, lock); + + if (opts.dryRun) { + const items = context.manifest.tests.map((spec): ResolvedPlanItem => { + const entry = lock.entries[spec.key]; + const testId = spec.testId ?? entry?.testId; + const desiredHashValue = context.desiredHashByKey.get(spec.key)!; + return { + key: spec.key, + ...(testId ? { testId } : {}), + action: testId ? 'update' : 'create', + changes: testId ? ['remote state not inspected in --dry-run'] : ['new backend test'], + reason: '--dry-run validates local files but intentionally skips credentials and network', + spec, + desiredCode: context.codeByKey.get(spec.key)!, + desiredHash: desiredHashValue, + ...(entry?.createKey ? { createKey: entry.createKey } : {}), + }; + }); + return assemblePlan(context, lock, items, [], true); + } + + const client = makeClient(opts, deps); + const remotePage = await paginate( + ({ pageSize, cursor }) => + client.get>('/tests', { + query: { projectId: context.manifest.projectId, pageSize, cursor }, + }), + { pageSize: 100 }, + ); + const remoteById = new Map(remotePage.items.map(test => [test.id, test] as const)); + const remoteByName = new Map(); + for (const remote of remotePage.items) { + const key = remote.name.toLowerCase(); + remoteByName.set(key, [...(remoteByName.get(key) ?? []), remote]); + } + + const claimedRemoteIds = new Set(); + const items: ResolvedPlanItem[] = []; + for (const spec of context.manifest.tests) { + const desiredCode = context.codeByKey.get(spec.key)!; + const desiredHashValue = context.desiredHashByKey.get(spec.key)!; + const entry = lock.entries[spec.key]; + const testId = spec.testId ?? entry?.testId; + if (!testId) { + if (entry?.createKey) { + if (entry.desiredHash === desiredHashValue) { + items.push({ + key: spec.key, + action: 'create', + changes: ['resume pending idempotent create'], + reason: 'a prior apply recorded create intent but did not record the server response', + spec, + desiredCode, + desiredHash: desiredHashValue, + createKey: entry.createKey, + }); + } else { + items.push({ + key: spec.key, + action: 'conflict', + changes: [], + reason: + 'the Suitefile changed after a create request became pending; restore the previous definition or inspect and remove the pending lock entry explicitly', + spec, + desiredCode, + desiredHash: desiredHashValue, + }); + } + continue; + } + const sameName = remoteByName.get(spec.name.toLowerCase()) ?? []; + if (sameName.length > 0) { + items.push({ + key: spec.key, + action: 'conflict', + changes: [], + reason: `remote test name already exists (${sameName.map(test => test.id).join(', ')}); add testId or restore the lock entry to adopt it explicitly`, + spec, + desiredCode, + desiredHash: desiredHashValue, + }); + } else { + items.push({ + key: spec.key, + action: 'create', + changes: ['new backend test'], + spec, + desiredCode, + desiredHash: desiredHashValue, + }); + } + continue; + } + if (claimedRemoteIds.has(testId)) { + items.push({ + key: spec.key, + testId, + action: 'conflict', + changes: [], + reason: `remote test ${testId} is already claimed by another suite key`, + spec, + desiredCode, + desiredHash: desiredHashValue, + }); + continue; + } + claimedRemoteIds.add(testId); + const remote = remoteById.get(testId); + if (!remote) { + items.push({ + key: spec.key, + testId, + action: 'conflict', + changes: [], + reason: `locked remote test ${testId} was not found in project ${context.manifest.projectId}`, + spec, + desiredCode, + desiredHash: desiredHashValue, + }); + continue; + } + if (remote.type !== 'backend') { + items.push({ + key: spec.key, + testId, + action: 'conflict', + changes: [], + reason: `remote test ${testId} is ${remote.type}; Suitefile MVP manages backend tests only`, + spec, + desiredCode, + desiredHash: desiredHashValue, + remote, + }); + continue; + } + const remoteCode = await client.get(`/tests/${encodeURIComponent(testId)}/code`); + const remoteCodeBody = await resolveRemoteCode(remoteCode.code, deps.fetchImpl); + const changes = diffSuiteTest(spec, desiredCode, remote, remoteCodeBody); + items.push({ + key: spec.key, + testId, + action: changes.length > 0 ? 'update' : 'noop', + changes, + spec, + desiredCode, + desiredHash: desiredHashValue, + remote, + remoteCode, + }); + } + const unmanagedRemote = remotePage.items + .filter(test => !claimedRemoteIds.has(test.id)) + .map(test => test.id) + .sort(); + return assemblePlan(context, lock, items, unmanagedRemote, false); +} + +function assemblePlan( + context: ManifestContext, + lock: SuiteLock, + items: ResolvedPlanItem[], + unmanagedRemote: string[], + dryRun: boolean, +): CalculatedPlan { + const summary: Record = { create: 0, update: 0, noop: 0, conflict: 0 }; + for (const item of items) summary[item.action] += 1; + const publicPlan: SuitePlan = { + schemaVersion: 1, + projectId: context.manifest.projectId, + manifest: context.manifestPath, + lockFile: context.lockPath, + dryRun, + graph: context.graph, + items: items.map( + ({ + spec: _spec, + desiredCode: _code, + desiredHash: _hash, + remote: _remote, + remoteCode: _remoteCode, + createKey: _createKey, + ...item + }) => item, + ), + summary, + unmanagedRemote, + }; + return { publicPlan, context, lock, items }; +} + +function diffSuiteTest( + spec: SuiteTestDefinition, + desiredCode: string, + remote: CliTest, + remoteCode: string, +): string[] { + const changes: string[] = []; + if (remote.name !== spec.name) changes.push('name'); + if (spec.priority !== undefined && (remote.priority ?? null) !== spec.priority) + changes.push('priority'); + if (!sameStringSet(remote.produces ?? [], spec.produces)) changes.push('produces'); + if (!sameStringSet(remote.consumes ?? [], spec.consumes)) changes.push('consumes'); + if (spec.category !== undefined && (remote.category ?? null) !== spec.category) + changes.push('category'); + if (normalizeNewlines(remoteCode) !== normalizeNewlines(desiredCode)) changes.push('code'); + return changes; +} + +function updateMetadataBody(item: ResolvedPlanItem): Record { + const body: Record = {}; + if (item.changes.includes('name')) body.name = item.spec.name; + if (item.changes.includes('priority')) body.priority = item.spec.priority; + if (item.changes.includes('produces')) body.produces = item.spec.produces; + if (item.changes.includes('consumes')) body.consumes = item.spec.consumes; + if (item.changes.includes('category')) body.category = item.spec.category; + return body; +} + +function createBody( + projectId: string, + spec: SuiteTestDefinition, + code: string, +): Record { + return { + projectId, + type: 'backend', + name: spec.name, + code, + ...(spec.priority ? { priority: spec.priority } : {}), + ...(spec.produces.length > 0 ? { produces: spec.produces } : {}), + ...(spec.consumes.length > 0 ? { consumes: spec.consumes } : {}), + ...(spec.category ? { category: spec.category } : {}), + }; +} + +async function resolveRemoteCode(code: string, fetchImpl?: FetchImpl): Promise { + if (!code.startsWith('https://')) return code; + const response = await (fetchImpl ?? globalThis.fetch)(code); + if (!response.ok) { + throw localValidationError( + 'suite', + `failed to download remote test code (HTTP ${response.status})`, + ); + } + return response.text(); +} + +function loadSuiteLock(path: string, projectId: string): SuiteLock { + if (!existsSync(path)) return { schemaVersion: LOCK_SCHEMA_VERSION, projectId, entries: {} }; + const raw = readTextFile(path, 'lock-file'); + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (error) { + throw localValidationError( + 'lock-file', + `is not valid JSON: ${error instanceof Error ? error.message : String(error)}`, + ); + } + if ( + !isRecord(parsed) || + parsed.schemaVersion !== LOCK_SCHEMA_VERSION || + !isRecord(parsed.entries) + ) { + throw localValidationError( + 'lock-file', + `must use schemaVersion ${LOCK_SCHEMA_VERSION} and contain entries`, + ); + } + if (parsed.projectId !== projectId) { + throw localValidationError( + 'lock-file', + `belongs to project ${String(parsed.projectId)}, not ${projectId}`, + ); + } + const entries: Record = {}; + for (const [key, value] of Object.entries(parsed.entries)) { + if ( + !isRecord(value) || + typeof value.desiredHash !== 'string' || + typeof value.updatedAt !== 'string' + ) { + throw localValidationError('lock-file', `entry ${key} is malformed`); + } + entries[key] = { + desiredHash: value.desiredHash, + updatedAt: value.updatedAt, + ...(typeof value.testId === 'string' ? { testId: value.testId } : {}), + ...(typeof value.codeVersion === 'string' || value.codeVersion === null + ? { codeVersion: value.codeVersion } + : {}), + ...(typeof value.createKey === 'string' ? { createKey: value.createKey } : {}), + }; + } + return { schemaVersion: LOCK_SCHEMA_VERSION, projectId, entries }; +} + +function writeSuiteLock(path: string, lock: SuiteLock): void { + const parent = dirname(path); + if (!existsSync(parent) || !statSync(parent).isDirectory()) { + throw localValidationError('lock-file', `parent directory does not exist: ${parent}`); + } + const tmp = resolve(parent, `.${basename(path)}.tmp-${randomUUID()}`); + try { + writeFileSync(tmp, `${JSON.stringify(lock, null, 2)}\n`, { encoding: 'utf8', flag: 'wx' }); + renameSync(tmp, path); + } catch (error) { + if (existsSync(tmp)) unlinkSync(tmp); + throw localValidationError( + 'lock-file', + `cannot write ${path}: ${error instanceof Error ? error.message : String(error)}`, + ); + } +} + +function assertManifestLockAgreement(manifest: SuiteManifest, lock: SuiteLock): void { + const errors: string[] = []; + for (const spec of manifest.tests) { + const lockedId = lock.entries[spec.key]?.testId; + if (spec.testId && lockedId && spec.testId !== lockedId) { + errors.push( + `${spec.key} declares testId ${spec.testId}, but the lock file records ${lockedId}`, + ); + } + } + if (errors.length > 0) throw manifestValidationError(errors); +} + +function makeCompletedLockEntry( + testId: string, + codeVersion: string | null | undefined, + desiredHashValue: string, + deps: SuiteDeps, +): SuiteLockEntry { + return { + testId, + ...(codeVersion !== undefined ? { codeVersion } : {}), + desiredHash: desiredHashValue, + updatedAt: nowIso(deps), + }; +} + +function suiteIdempotencyKey(action: string, projectId: string, item: ResolvedPlanItem): string { + const material = [ + projectId, + item.key, + item.testId ?? '', + item.remoteCode?.codeVersion ?? '', + item.desiredHash, + ].join('\0'); + const digest = createHash('sha256').update(material).digest('hex').slice(0, 32); + return `cli-suite-v1-${action}-${digest}`; +} + +function desiredHash(spec: SuiteTestDefinition, code: string): string { + return createHash('sha256') + .update( + JSON.stringify({ + name: spec.name, + priority: spec.priority ?? null, + produces: [...spec.produces].sort(), + consumes: [...spec.consumes].sort(), + category: spec.category ?? null, + code: normalizeNewlines(code), + }), + ) + .digest('hex'); +} + +function addGraphEdge( + from: string, + to: string, + variable: string, + outgoing: Map>, + indegree: Map, + edges: Array<{ from: string; to: string; variable: string }>, +): void { + const targets = outgoing.get(from)!; + if (!targets.has(to)) { + targets.add(to); + indegree.set(to, (indegree.get(to) ?? 0) + 1); + } + edges.push({ from, to, variable }); +} + +function resolveLockPath(manifestPath: string, override?: string): string { + if (override) return resolve(override); + return manifestPath.toLowerCase().endsWith('.json') + ? `${manifestPath.slice(0, -5)}.lock.json` + : `${manifestPath}.lock.json`; +} + +function resolveContainedCodePath(manifestDir: string, codeFile: string, key: string): string { + if (isAbsolute(codeFile)) { + throw localValidationError('manifest', `${key}.codeFile must be relative to the Suitefile`); + } + const absolute = resolve(manifestDir, codeFile); + const rel = relative(manifestDir, absolute); + if (rel === '' || rel.startsWith('..') || isAbsolute(rel)) { + throw localValidationError('manifest', `${key}.codeFile escapes the Suitefile directory`); + } + if (!absolute.toLowerCase().endsWith('.py')) { + throw localValidationError('manifest', `${key}.codeFile must end in .py for a backend test`); + } + return absolute; +} + +function readCodeFile(path: string, manifestDir: string, key: string): string { + let stat; + try { + stat = statSync(path); + } catch (error) { + throw localValidationError( + 'manifest', + `${key}.codeFile cannot be read at ${path}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + if (!stat.isFile()) + throw localValidationError('manifest', `${key}.codeFile is not a file: ${path}`); + let realPath: string; + let realManifestDir: string; + try { + realPath = realpathSync(path); + realManifestDir = realpathSync(manifestDir); + } catch (error) { + throw localValidationError( + 'manifest', + `${key}.codeFile cannot be resolved safely: ${error instanceof Error ? error.message : String(error)}`, + ); + } + const realRelative = relative(realManifestDir, realPath); + if (realRelative === '' || realRelative.startsWith('..') || isAbsolute(realRelative)) { + throw localValidationError( + 'manifest', + `${key}.codeFile resolves outside the Suitefile directory (symlink escape)`, + ); + } + if (stat.size > MAX_CODE_BYTES) { + throw localValidationError( + 'manifest', + `${key}.codeFile exceeds the ${MAX_CODE_BYTES}-byte backend limit`, + ); + } + const code = stripBom(readTextFile(realPath, 'manifest')); + if (Buffer.byteLength(code, 'utf8') > MAX_CODE_BYTES) { + throw localValidationError( + 'manifest', + `${key}.codeFile exceeds the ${MAX_CODE_BYTES}-byte backend limit`, + ); + } + return code; +} + +function stripBom(value: string): string { + return value.charCodeAt(0) === 0xfeff ? value.slice(1) : value; +} + +function readTextFile(path: string, field: string): string { + try { + return readFileSync(path, 'utf8'); + } catch (error) { + throw localValidationError( + field, + `cannot read ${path}: ${error instanceof Error ? error.message : String(error)}`, + ); + } +} + +function readRequiredString( + value: unknown, + field: string, + errors: string[], + maxLength: number, +): string { + if (typeof value !== 'string' || value.trim().length === 0) { + errors.push(`${field} must be a non-empty string`); + return ''; + } + if (value.length > maxLength) errors.push(`${field} must be at most ${maxLength} characters`); + return value.trim(); +} + +function readOptionalString( + value: unknown, + field: string, + errors: string[], + maxLength: number, +): string | undefined { + if (value === undefined) return undefined; + return readRequiredString(value, field, errors, maxLength) || undefined; +} + +function readPriority(value: unknown, field: string, errors: string[]): Priority | undefined { + if (value === undefined) return undefined; + if (typeof value !== 'string' || !PRIORITIES.includes(value as Priority)) { + errors.push(`${field} must be one of: ${PRIORITIES.join(', ')}`); + return undefined; + } + return value as Priority; +} + +function readStringArray(value: unknown, field: string, errors: string[]): string[] { + if (value === undefined) return []; + if (!Array.isArray(value)) { + errors.push(`${field} must be an array of non-empty strings`); + return []; + } + const result: string[] = []; + const seen = new Set(); + for (let index = 0; index < value.length; index += 1) { + const item = value[index]; + if (typeof item !== 'string' || item.trim().length === 0) { + errors.push(`${field}[${index}] must be a non-empty string`); + continue; + } + const normalized = item.trim(); + if (normalized.length > 128) errors.push(`${field}[${index}] must be at most 128 characters`); + if (seen.has(normalized)) errors.push(`${field} contains duplicate "${normalized}"`); + seen.add(normalized); + result.push(normalized); + } + return result; +} + +function rejectUnknownFields( + value: Record, + allowed: ReadonlySet, + field: string, + errors: string[], +): void { + for (const key of Object.keys(value)) { + if (!allowed.has(key)) errors.push(`${field}.${key} is not supported`); + } +} + +function manifestValidationError(errors: string[]): Error { + return localValidationError('manifest', `${errors.length} problem(s): ${errors.join('; ')}`); +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function sameStringSet(left: readonly string[], right: readonly string[]): boolean { + if (left.length !== right.length) return false; + const a = [...left].sort(); + const b = [...right].sort(); + return a.every((value, index) => value === b[index]); +} + +function normalizeNewlines(value: string): string { + return value.replace(/\r\n/g, '\n'); +} + +function nowIso(deps: SuiteDeps): string { + return (deps.now?.() ?? new Date()).toISOString(); +} + +function resolveCommonOptions(command: Command): CommonOptions { + const globals = command.optsWithGlobals() as Partial & { requestTimeout?: string }; + return { + profile: globals.profile ?? 'default', + output: resolveOutputMode(globals.output), + dryRun: globals.dryRun ?? false, + endpointUrl: globals.endpointUrl, + debug: globals.debug ?? false, + verbose: globals.verbose ?? false, + requestTimeoutMs: parseRequestTimeoutFlag(globals.requestTimeout), + }; +} + +function makeClient(opts: CommonOptions, deps: SuiteDeps): HttpClient { + return makeHttpClient(opts, { + env: deps.env, + credentialsPath: deps.credentialsPath, + fetchImpl: deps.fetchImpl, + stderr: deps.stderr, + }); +} + +function makeOutput(mode: OutputMode, deps: SuiteDeps): Output { + return new Output(mode, { stdout: deps.stdout, stderr: deps.stderr }); +} + +function renderValidationText(result: { + manifest: string; + lockFile: string; + graph: SuiteGraph; +}): string { + return [ + 'Suitefile valid', + `manifest ${result.manifest}`, + `lockFile ${result.lockFile}`, + `project ${result.graph.projectId}`, + `tests ${result.graph.tests}`, + `waves ${result.graph.waves.length}`, + ].join('\n'); +} + +function renderGraphText(graph: SuiteGraph): string { + const lines = [ + `project ${graph.projectId}`, + `tests ${graph.tests}`, + `waves ${graph.waves.length}`, + ]; + for (const wave of graph.waves) + lines.push(`wave ${String(wave.wave).padStart(2)} ${wave.tests.join(', ')}`); + if (graph.edges.length > 0) { + lines.push('edges'); + for (const edge of graph.edges) lines.push(` ${edge.from} -> ${edge.to} [${edge.variable}]`); + } + return lines.join('\n'); +} + +function renderPlanText(plan: SuitePlan): string { + const lines = [ + `${plan.dryRun ? '[dry-run] ' : ''}Suite plan for ${plan.projectId}`, + `create ${plan.summary.create} update ${plan.summary.update} unchanged ${plan.summary.noop} conflicts ${plan.summary.conflict}`, + ]; + for (const item of plan.items) { + const detail = item.changes.length > 0 ? ` (${item.changes.join(', ')})` : ''; + lines.push( + `${item.action.padEnd(8)} ${item.key}${item.testId ? ` -> ${item.testId}` : ''}${detail}`, + ); + if (item.reason) lines.push(` ${item.reason}`); + } + if (plan.unmanagedRemote.length > 0) { + lines.push(`unmanaged remote tests: ${plan.unmanagedRemote.length} (never deleted)`); + } + return lines.join('\n'); +} + +function renderApplyText(result: SuiteApplyResult): string { + const lines = [ + `Suite applied to ${result.projectId}`, + `created ${result.summary.created} updated ${result.summary.updated} unchanged ${result.summary.unchanged}`, + `lockFile ${result.lockFile}`, + ]; + for (const item of result.applied) + lines.push(`${item.action.padEnd(8)} ${item.key} -> ${item.testId}`); + return lines.join('\n'); +} diff --git a/src/index.ts b/src/index.ts index 31d3399..155f155 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,6 +11,7 @@ import { runConfigureViaSetup, } from './commands/init.js'; import { createProjectCommand } from './commands/project.js'; +import { createSuiteCommand } from './commands/suite.js'; import { createTestCommand } from './commands/test.js'; import { createUsageCommand } from './commands/usage.js'; import { ApiError, CLIError, InterruptError, RequestTimeoutError } from './lib/errors.js'; @@ -53,7 +54,7 @@ program .option('--debug', 'Print HTTP method/path, request id, latency, retry decisions to stderr') .option( '--dry-run', - 'Skip the network and credentials; emit a canned sample matching the OpenAPI contract. Useful for learning the CLI surface without an API key. Note: file inputs you pass (--plan-from/--plans/--steps) are still read and validated locally; only --code-file uses a placeholder.', + 'Skip the network and credentials; emit a canned sample matching the OpenAPI contract. Useful for learning the CLI surface without an API key. Note: plan/step inputs and Suitefile manifests/code are still read and validated locally; only standalone test --code-file uses a placeholder.', ) .option( '--request-timeout ', @@ -90,6 +91,7 @@ program.addCommand(authCommand); program.addCommand(createProjectCommand({})); program.addCommand(createTestCommand()); +program.addCommand(createSuiteCommand()); program.addCommand(createAgentCommand({})); program.addCommand(createUsageCommand()); program.addCommand(createDoctorCommand()); diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 4724064..d66c731 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -209,6 +209,87 @@ Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeou " `; +exports[`--help snapshots > suite 1`] = ` +"Usage: testsprite suite [options] [command] + +Validate, plan, and apply a declarative backend test suite from a versioned +Suitefile + +Options: + -h, --help display help for command + +Commands: + validate [options] Validate Suitefile structure, code paths, + dependencies, and execution waves locally + graph [options] Compile produces/consumes declarations into + deterministic execution waves + plan [options] Compare the Suitefile with the remote project + without changing either side + apply [options] Create and update backend tests from a + previously reviewed Suitefile plan + help [command] display help for command +" +`; + +exports[`--help snapshots > suite apply 1`] = ` +"Usage: testsprite suite apply [options] + +Create and update backend tests from a previously reviewed Suitefile plan + +Options: + --lock-file override the adjacent *.lock.json path + --confirm required before remote mutations; deletions are never + performed (default: false) + -h, --help display help for command + +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): + testsprite --help +" +`; + +exports[`--help snapshots > suite graph 1`] = ` +"Usage: testsprite suite graph [options] + +Compile produces/consumes declarations into deterministic execution waves + +Options: + --lock-file override the adjacent *.lock.json path + -h, --help display help for command + +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): + testsprite --help +" +`; + +exports[`--help snapshots > suite plan 1`] = ` +"Usage: testsprite suite plan [options] + +Compare the Suitefile with the remote project without changing either side + +Options: + --lock-file override the adjacent *.lock.json path + -h, --help display help for command + +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): + testsprite --help +" +`; + +exports[`--help snapshots > suite validate 1`] = ` +"Usage: testsprite suite validate [options] + +Validate Suitefile structure, code paths, dependencies, and execution waves +locally + +Options: + --lock-file override the adjacent *.lock.json path + -h, --help display help for command + +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): + testsprite --help +" +`; + exports[`--help snapshots > test 1`] = ` "Usage: testsprite test [options] [command] @@ -738,9 +819,9 @@ Options: --dry-run Skip the network and credentials; emit a canned sample matching the OpenAPI contract. Useful for learning the CLI surface without an API key. - Note: file inputs you pass - (--plan-from/--plans/--steps) are still read and - validated locally; only --code-file uses a + Note: plan/step inputs and Suitefile + manifests/code are still read and validated + locally; only standalone test --code-file uses a placeholder. --request-timeout Client-side per-request timeout in seconds (default: 120). Aborts any single fetch that @@ -758,6 +839,8 @@ Commands: auth Manage TestSprite credentials project Manage TestSprite projects test Inspect TestSprite tests + suite Validate, plan, and apply a declarative backend + test suite from a versioned Suitefile agent Install TestSprite guidance into coding-agent config (Claude Code, Cursor, Cline, Antigravity, Kiro, Windsurf, Copilot, Codex) diff --git a/test/help.snapshot.test.ts b/test/help.snapshot.test.ts index b6c23b4..5f9734f 100644 --- a/test/help.snapshot.test.ts +++ b/test/help.snapshot.test.ts @@ -32,6 +32,11 @@ const cases: Array<[string, string[]]> = [ ['project', ['project', '--help']], ['project list', ['project', 'list', '--help']], ['project get', ['project', 'get', '--help']], + ['suite', ['suite', '--help']], + ['suite validate', ['suite', 'validate', '--help']], + ['suite graph', ['suite', 'graph', '--help']], + ['suite plan', ['suite', 'plan', '--help']], + ['suite apply', ['suite', 'apply', '--help']], ['test', ['test', '--help']], ['test list', ['test', 'list', '--help']], ['test get', ['test', 'get', '--help']], From 7cb64a666a82be940f41a414044aba437a0b3f26 Mon Sep 17 00:00:00 2001 From: cmdr-chara <249489759+cmdr-chara@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:37:59 +0200 Subject: [PATCH 2/2] fix(suite): harden no-op sync and remote downloads --- src/commands/suite.test.ts | 156 +++++++++++++++++++++++++++++++++++++ src/commands/suite.ts | 74 +++++++++++++----- src/lib/http.ts | 4 +- 3 files changed, 213 insertions(+), 21 deletions(-) diff --git a/src/commands/suite.test.ts b/src/commands/suite.test.ts index 62755c1..25e04e4 100644 --- a/src/commands/suite.test.ts +++ b/src/commands/suite.test.ts @@ -289,6 +289,60 @@ describe('suite plan', () => { expect(plan.items[0]).toMatchObject({ action: 'conflict' }); expect(plan.items[0]?.reason).toContain('add testId'); }); + + it('times out stalled presigned code downloads using the configured request deadline', async () => { + const { manifestPath } = writeSuite([ + { key: 'slow', testId: 'test_slow', name: 'Slow', codeFile: 'slow.py' }, + ]); + const fetchImpl: FetchImpl = async (input, init) => { + const url = new URL(String(input)); + if (url.pathname.endsWith('/tests')) { + return json({ + items: [ + { + id: 'test_slow', + projectId: 'proj_suite_1', + name: 'Slow', + type: 'backend', + createdFrom: 'cli', + status: 'ready', + produces: [], + consumes: [], + createdAt: '2026-01-01T00:00:00.000Z', + updatedAt: '2026-01-01T00:00:00.000Z', + }, + ], + nextToken: null, + }); + } + if (url.pathname.endsWith('/tests/test_slow/code')) { + return json({ + testId: 'test_slow', + language: 'python', + framework: 'pytest', + code: 'https://storage.example.test/slow.py', + codeVersion: 'v1', + }); + } + if (url.hostname === 'storage.example.test') { + return new Promise((_resolve, reject) => { + const signal = init?.signal; + if (!signal) { + reject(new Error('expected a request timeout signal')); + return; + } + const rejectFromAbort = () => reject(signal.reason); + if (signal.aborted) rejectFromAbort(); + else signal.addEventListener('abort', rejectFromAbort, { once: true }); + }); + } + throw new Error(`unexpected request: ${url}`); + }; + + await expect( + runSuitePlan({ ...common(fetchImpl), manifestPath, requestTimeoutMs: 1 }, common(fetchImpl)), + ).rejects.toMatchObject({ name: 'RequestTimeoutError', timeoutMs: 1_000 }); + }); }); describe('suite apply', () => { @@ -400,6 +454,108 @@ describe('suite apply', () => { }); }); + it('refuses to apply a plan containing conflicts without sending mutations', async () => { + const { manifestPath } = writeSuite([{ key: 'health', name: 'Health', codeFile: 'health.py' }]); + let mutations = 0; + const fetchImpl: FetchImpl = async (_input, init) => { + if ((init?.method ?? 'GET') !== 'GET') mutations += 1; + return json({ + items: [ + { + id: 'test_existing', + projectId: 'proj_suite_1', + name: 'Health', + type: 'backend', + createdFrom: 'portal', + status: 'ready', + createdAt: '2026-01-01T00:00:00.000Z', + updatedAt: '2026-01-01T00:00:00.000Z', + }, + ], + nextToken: null, + }); + }; + + await expect( + runSuiteApply({ ...common(fetchImpl), manifestPath, confirm: true }, common(fetchImpl)), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + nextAction: expect.stringContaining('plan contains 1 conflict'), + }); + expect(mutations).toBe(0); + }); + + it('does not churn an unchanged lock entry timestamp on repeated apply', async () => { + const { manifestPath, lockPath } = writeSuite([ + { key: 'health', name: 'Health', codeFile: 'health.py' }, + ]); + const createFetch: FetchImpl = async (input, init) => { + const url = new URL(String(input)); + if ((init?.method ?? 'GET') === 'GET' && url.pathname.endsWith('/tests')) { + return json({ items: [], nextToken: null }); + } + if ((init?.method ?? 'GET') === 'POST' && url.pathname.endsWith('/tests')) { + return json({ + testId: 'test_health', + type: 'backend', + codeVersion: 'v1', + createdAt: 'now', + }); + } + throw new Error(`unexpected request: ${init?.method ?? 'GET'} ${url.pathname}`); + }; + await runSuiteApply( + { ...common(createFetch), manifestPath, confirm: true }, + common(createFetch), + ); + const before = readFileSync(lockPath, 'utf8'); + + const noopFetch: FetchImpl = async input => { + const url = new URL(String(input)); + if (url.pathname.endsWith('/tests')) { + return json({ + items: [ + { + id: 'test_health', + projectId: 'proj_suite_1', + name: 'Health', + type: 'backend', + createdFrom: 'cli', + status: 'ready', + produces: [], + consumes: [], + createdAt: '2026-01-01T00:00:00.000Z', + updatedAt: '2026-01-01T00:00:00.000Z', + }, + ], + nextToken: null, + }); + } + if (url.pathname.endsWith('/tests/test_health/code')) { + return json({ + testId: 'test_health', + language: 'python', + framework: 'pytest', + code: 'def test_health():\n assert True\n', + codeVersion: 'v1', + }); + } + throw new Error(`unexpected request: ${url}`); + }; + const later = { + ...common(noopFetch), + now: () => new Date('2026-07-22T12:00:00.000Z'), + }; + const result = await runSuiteApply({ ...later, manifestPath, confirm: true }, later); + + expect('summary' in result && result.summary).toEqual({ + created: 0, + updated: 0, + unchanged: 1, + }); + expect(readFileSync(lockPath, 'utf8')).toBe(before); + }); + it('resumes an unchanged pending create and conflicts if its definition drifted', async () => { const { manifestPath, lockPath, dir } = writeSuite([ { key: 'health', name: 'Health', codeFile: 'health.py' }, diff --git a/src/commands/suite.ts b/src/commands/suite.ts index e20b2d3..45cbc65 100644 --- a/src/commands/suite.ts +++ b/src/commands/suite.ts @@ -13,10 +13,16 @@ import { Command } from 'commander'; import { makeHttpClient, parseRequestTimeoutFlag, + resolveRequestTimeoutMs, type CommonOptions, } from '../lib/client-factory.js'; -import { localValidationError } from '../lib/errors.js'; -import type { FetchImpl, HttpClient } from '../lib/http.js'; +import { + ApiError, + RequestTimeoutError, + TransportError, + localValidationError, +} from '../lib/errors.js'; +import { createRequestTimeout, type FetchImpl, type HttpClient } from '../lib/http.js'; import { GLOBAL_OPTS_HINT, Output, resolveOutputMode, type OutputMode } from '../lib/output.js'; import { paginate, type Page } from '../lib/pagination.js'; import type { @@ -430,12 +436,19 @@ export async function runSuiteApply( if (item.action === 'noop') { unchanged.push(item.key); if (item.testId) { - lock.entries[item.key] = makeCompletedLockEntry( - item.testId, - item.remoteCode?.codeVersion, - item.desiredHash, - deps, - ); + const existing = lock.entries[item.key]; + const meaningfulFieldsMatch = + existing?.testId === item.testId && + existing.codeVersion === item.remoteCode?.codeVersion && + existing.desiredHash === item.desiredHash; + if (!meaningfulFieldsMatch) { + lock.entries[item.key] = makeCompletedLockEntry( + item.testId, + item.remoteCode?.codeVersion, + item.desiredHash, + deps, + ); + } } continue; } @@ -631,6 +644,7 @@ async function calculateSuitePlan( const claimedRemoteIds = new Set(); const items: ResolvedPlanItem[] = []; + const requestTimeoutMs = resolveRequestTimeoutMs(opts, deps.env ?? process.env); for (const spec of context.manifest.tests) { const desiredCode = context.codeByKey.get(spec.key)!; const desiredHashValue = context.desiredHashByKey.get(spec.key)!; @@ -729,7 +743,11 @@ async function calculateSuitePlan( continue; } const remoteCode = await client.get(`/tests/${encodeURIComponent(testId)}/code`); - const remoteCodeBody = await resolveRemoteCode(remoteCode.code, deps.fetchImpl); + const remoteCodeBody = await resolveRemoteCode( + remoteCode.code, + deps.fetchImpl, + requestTimeoutMs, + ); const changes = diffSuiteTest(spec, desiredCode, remote, remoteCodeBody); items.push({ key: spec.key, @@ -828,16 +846,34 @@ function createBody( }; } -async function resolveRemoteCode(code: string, fetchImpl?: FetchImpl): Promise { +async function resolveRemoteCode( + code: string, + fetchImpl: FetchImpl | undefined, + requestTimeoutMs: number, +): Promise { if (!code.startsWith('https://')) return code; - const response = await (fetchImpl ?? globalThis.fetch)(code); - if (!response.ok) { - throw localValidationError( - 'suite', - `failed to download remote test code (HTTP ${response.status})`, - ); + const requestTimeout = createRequestTimeout(requestTimeoutMs); + try { + const response = await (fetchImpl ?? globalThis.fetch)(code, { + signal: requestTimeout.signal, + }); + if (!response.ok) { + throw localValidationError( + 'suite', + `failed to download remote test code (HTTP ${response.status})`, + ); + } + return await response.text(); + } catch (error) { + if (error instanceof ApiError || error instanceof RequestTimeoutError) throw error; + if (requestTimeout.signal.aborted) { + throw new RequestTimeoutError(requestTimeoutMs); + } + const message = error instanceof Error ? error.message : String(error); + throw new TransportError(`Failed to download remote test code: ${message}`); + } finally { + requestTimeout.clear(); } - return response.text(); } function loadSuiteLock(path: string, projectId: string): SuiteLock { @@ -878,13 +914,13 @@ function loadSuiteLock(path: string, projectId: string): SuiteLock { throw localValidationError('lock-file', `entry ${key} is malformed`); } entries[key] = { - desiredHash: value.desiredHash, - updatedAt: value.updatedAt, ...(typeof value.testId === 'string' ? { testId: value.testId } : {}), ...(typeof value.codeVersion === 'string' || value.codeVersion === null ? { codeVersion: value.codeVersion } : {}), + desiredHash: value.desiredHash, ...(typeof value.createKey === 'string' ? { createKey: value.createKey } : {}), + updatedAt: value.updatedAt, }; } return { schemaVersion: LOCK_SCHEMA_VERSION, projectId, entries }; diff --git a/src/lib/http.ts b/src/lib/http.ts index 21369de..65fadc9 100644 --- a/src/lib/http.ts +++ b/src/lib/http.ts @@ -828,12 +828,12 @@ function newRequestId(): string { return `cli_${randomUUID()}`; } -interface RequestTimeoutHandle { +export interface RequestTimeoutHandle { signal: AbortSignal; clear: () => void; } -function createRequestTimeout(timeoutMs: number): RequestTimeoutHandle { +export function createRequestTimeout(timeoutMs: number): RequestTimeoutHandle { const controller = new AbortController(); const timer = setTimeout(() => { controller.abort(makeTimeoutReason());