From a8432547f8dbc9f5e64d587db356fd3e246b32b7 Mon Sep 17 00:00:00 2001 From: Andy00L <89641810+Andy00L@users.noreply.github.com> Date: Tue, 21 Jul 2026 09:10:37 -0400 Subject: [PATCH] feat(test): add "test export" / "test import" to round-trip test definitions --- src/commands/test.test.ts | 151 ++++++++++ src/commands/test.ts | 270 ++++++++++++++++++ test/__snapshots__/help.snapshot.test.ts.snap | 11 + 3 files changed, 432 insertions(+) diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index c8fe5a5..4b2bf5d 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -31,6 +31,8 @@ import { runFailureGet, runFailureSummary, runGet, + runExport, + runImport, runLint, runList, runOpen, @@ -128,9 +130,11 @@ describe('createTestCommand — surface', () => { 'delete', 'delete-batch', 'diff', + 'export', 'failure', 'flaky', 'get', + 'import', 'lint', 'list', 'open', @@ -3258,6 +3262,153 @@ describe('runDiff', () => { }); }); +describe('runExport / runImport', () => { + const TEST_ROW = { + id: 'test_be', + projectId: 'project_alice', + projectName: 'Alice', + name: 'Health check', + type: 'backend', + createdFrom: 'cli', + status: 'ready', + createdAt: '2026-06-01T10:00:00.000Z', + updatedAt: '2026-06-01T10:00:00.000Z', + }; + const CODE_ROW = { + testId: 'test_be', + language: 'python', + framework: 'pytest', + code: 'import requests\n', + codeVersion: 'v3', + }; + + it('export composes metadata + code with codeVersion provenance (backend: lossless)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(url => ({ body: url.includes('/code') ? CODE_ROW : TEST_ROW })); + const out: string[] = []; + const definition = await runExport( + { profile: 'default', output: 'json', debug: false, testId: 'test_be', force: false }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + expect(definition).toMatchObject({ + schemaVersion: 1, + testId: 'test_be', + projectId: 'project_alice', + type: 'backend', + code: { language: 'python', body: 'import requests\n', codeVersion: 'v3' }, + }); + expect(definition.planUnavailable).toBeUndefined(); + expect(JSON.parse(out.join('')) as unknown).toEqual(definition); + }); + + it('a frontend export declares planUnavailable and warns on stderr', async () => { + const { credentialsPath } = makeCreds(); + const feRow = { ...TEST_ROW, id: 'test_fe2', type: 'frontend' }; + const fetchImpl = makeFetch(url => ({ body: url.includes('/code') ? CODE_ROW : feRow })); + const errs: string[] = []; + const definition = await runExport( + { profile: 'default', output: 'json', debug: false, testId: 'test_fe2', force: false }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: line => errs.push(line) }, + ); + expect(definition.planUnavailable).toBe(true); + expect(errs.join('\n')).toContain('write-only'); + }); + + it('import without testId creates via POST /tests with the definition body', async () => { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-import-')); + const file = join(dir, 'def.testsprite.json'); + writeFileSync( + file, + JSON.stringify({ + schemaVersion: 1, + projectId: 'project_alice', + type: 'backend', + name: 'Imported test', + code: { language: 'python', framework: 'pytest', body: 'print(1)\n', codeVersion: null }, + }), + 'utf8', + ); + const seen: Array<{ url: string; method: string; body: unknown }> = []; + const fetchImpl = (async (input: Parameters[0], init: RequestInit = {}) => { + seen.push({ + url: String(input), + method: init.method ?? 'GET', + body: init.body === undefined ? undefined : JSON.parse(init.body as string), + }); + return new Response(JSON.stringify({ testId: 'test_new' }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof fetch; + const result = await runImport( + { profile: 'default', output: 'json', debug: false, file }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + expect(result).toEqual({ testId: 'test_new', action: 'created' }); + expect(seen[0]!.method).toBe('POST'); + expect(seen[0]!.body).toMatchObject({ + projectId: 'project_alice', + type: 'backend', + name: 'Imported test', + code: 'print(1)\n', + }); + }); + + it('import with testId updates metadata then PUTs the code with If-Match provenance', async () => { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-import-upd-')); + const file = join(dir, 'def.testsprite.json'); + writeFileSync( + file, + JSON.stringify({ + schemaVersion: 1, + testId: 'test_be', + projectId: 'project_alice', + type: 'backend', + name: 'Renamed test', + code: { language: 'python', framework: 'pytest', body: 'print(2)\n', codeVersion: 'v3' }, + }), + 'utf8', + ); + const seen: Array<{ url: string; method: string; ifMatch: string | null }> = []; + const fetchImpl = (async (input: Parameters[0], init: RequestInit = {}) => { + const headers = new Headers(init.headers); + seen.push({ + url: String(input), + method: init.method ?? 'GET', + ifMatch: headers.get('if-match'), + }); + return new Response(JSON.stringify({ testId: 'test_be' }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof fetch; + const result = await runImport( + { profile: 'default', output: 'json', debug: false, file }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + expect(result).toEqual({ testId: 'test_be', action: 'updated' }); + expect(seen).toHaveLength(2); + expect(seen[0]!.method).toBe('PUT'); + expect(seen[0]!.url).toContain('/tests/test_be'); + expect(seen[1]!.url).toContain('/tests/test_be/code'); + expect(seen[1]!.ifMatch).toBe('v3'); + }); + + it('import rejects a wrong schemaVersion with a field-level VALIDATION_ERROR', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-import-bad-')); + const file = join(dir, 'def.json'); + writeFileSync(file, JSON.stringify({ schemaVersion: 2 }), 'utf8'); + await expect( + runImport( + { profile: 'default', output: 'json', debug: false, file }, + { stdout: () => undefined }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); +}); + describe('runLint', () => { const VALID_PLAN = JSON.stringify({ projectId: 'project_alice', diff --git a/src/commands/test.ts b/src/commands/test.ts index 7839ca7..6a7fdc5 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -4064,6 +4064,244 @@ export interface CliLintReport { issues: CliLintIssue[]; } +/** + * Deterministic on-disk test DEFINITION (issue #125): the round-trippable + * subset of a test (metadata + code) with provenance, so definitions can be + * version-controlled, reviewed, backed up, and migrated. DISTINCT from the + * JUnit RESULTS export: this is what the test IS, not what a run produced. + * Stable key order comes from constructing the object literal in one place. + */ +export interface CliTestDefinition { + schemaVersion: 1; + /** Present on exports; on import, its presence selects update-vs-create. */ + testId?: string; + projectId: string; + type: 'frontend' | 'backend'; + name: string; + description?: string; + code?: { + language: string; + framework: string; + body: string; + /** Optimistic-concurrency provenance; import replays it as If-Match. */ + codeVersion: string | null; + }; + /** + * Honest-limitation marker: a frontend test's authored planSteps[] is + * write-only on the wire (PUT exists, no GET), so an FE export carries + * metadata + generated code only. + */ + planUnavailable?: true; +} + +export interface ExportOptions extends CommonOptions { + testId: string; + out?: string; + force: boolean; +} + +/** `test export ` (issue #125): write the definition file. */ +export async function runExport( + opts: ExportOptions, + deps: TestDeps = {}, +): Promise { + const out = makeOutput(opts.output, deps); + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + if (opts.dryRun) { + emitDryRunBanner(stderrFn); + const sample: CliTestDefinition = { + schemaVersion: 1, + testId: opts.testId, + projectId: 'p_dryrun_2026', + type: 'backend', + name: 'Sample exported test', + code: { language: 'python', framework: 'pytest', body: '# dry-run', codeVersion: 'v1' }, + }; + out.print(sample, () => JSON.stringify(sample, null, 2)); + return sample; + } + + const client = makeClient(opts, deps); + const test = await client.get(`/tests/${encodeURIComponent(opts.testId)}`); + // The generated code may not exist yet (fresh FE test): NOT_FOUND means + // "no code portion", every other error propagates unchanged. + let code: CliTestCode | undefined; + try { + code = await client.get(`/tests/${encodeURIComponent(opts.testId)}/code`); + } catch (err) { + if (err instanceof ApiError && err.code === 'NOT_FOUND') code = undefined; + else throw err; + } + + const description = (test as { description?: string }).description; + const definition: CliTestDefinition = { + schemaVersion: 1, + testId: opts.testId, + projectId: test.projectId, + type: test.type === 'backend' ? 'backend' : 'frontend', + name: test.name, + ...(typeof description === 'string' && description.length > 0 ? { description } : {}), + ...(code !== undefined + ? { + code: { + language: code.language, + framework: code.framework, + body: code.code, + codeVersion: code.codeVersion, + }, + } + : {}), + ...(test.type === 'frontend' ? { planUnavailable: true as const } : {}), + }; + if (definition.planUnavailable === true) { + stderrFn( + 'note: frontend planSteps[] are write-only on the API; this export carries metadata + generated code only (planUnavailable: true)', + ); + } + + const fileBody = `${JSON.stringify(definition, null, 2)}\n`; + if (opts.out !== undefined) { + const resolved = isAbsolute(opts.out) ? opts.out : resolve(process.cwd(), opts.out); + if (!opts.force && existsSync(resolved)) { + throw localValidationError('out', `already exists: ${resolved}. Pass --force to overwrite`); + } + const sink = openOutputFile(opts.out); + const fileOut = makeFileOutput(opts.output, sink); + await fileOut.writeChunk(fileBody); + await closeOutputFile(sink, true); + stderrFn(`Definition written to ${resolved}`); + return definition; + } + out.print(definition, () => fileBody.trimEnd()); + return definition; +} + +export interface ImportOptions extends CommonOptions { + file: string; +} + +/** + * `test import ` (issue #125): create or update a test from a + * definition file. A `testId` in the file selects update (metadata PUT + + * code PUT with the recorded codeVersion as If-Match, so a drifted server + * copy fails loudly with the existing 412 contract); no `testId` creates + * (`POST /tests` with the same body shape `test create` sends). + */ +export async function runImport( + opts: ImportOptions, + deps: TestDeps = {}, +): Promise<{ testId: string; action: 'created' | 'updated' }> { + const out = makeOutput(opts.output, deps); + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + const absolute = resolveAbsolute(opts.file); + let parsed: unknown; + try { + parsed = JSON.parse(readFileSync(absolute, 'utf8')); + } catch { + throw localValidationError( + 'file', + `is not readable valid JSON: ${absolute}`, + undefined, + 'field', + ); + } + const def = parsed as Partial; + if (def.schemaVersion !== 1) { + throw localValidationError('schemaVersion', 'must be 1', [1], 'field'); + } + if (typeof def.projectId !== 'string' || def.projectId.length === 0) { + throw localValidationError('projectId', 'is required', undefined, 'field'); + } + if (def.type !== 'frontend' && def.type !== 'backend') { + throw localValidationError( + 'type', + "must be 'frontend' or 'backend'", + ['frontend', 'backend'], + 'field', + ); + } + if (typeof def.name !== 'string' || def.name.trim().length === 0) { + throw localValidationError('name', 'is required', undefined, 'field'); + } + + if (opts.dryRun) { + emitDryRunBanner(stderrFn); + const sample = { + testId: def.testId ?? 'test_dryrun_imported', + action: (def.testId !== undefined ? 'updated' : 'created') as 'created' | 'updated', + }; + out.print( + sample, + data => `${(data as { action: string }).action}: ${(data as { testId: string }).testId}`, + ); + return sample; + } + + const client = makeClient(opts, deps); + if (typeof def.testId === 'string' && def.testId.length > 0) { + const testId = def.testId; + await client.put(`/tests/${encodeURIComponent(testId)}`, { + body: { + name: def.name.trim(), + ...(def.description !== undefined ? { description: def.description } : {}), + }, + headers: { 'idempotency-key': `cli-import-meta-${randomUUID()}` }, + }); + if (def.code !== undefined && typeof def.code.body === 'string') { + await client.put(`/tests/${encodeURIComponent(testId)}/code`, { + body: { code: def.code.body }, + headers: { + 'idempotency-key': `cli-import-code-${randomUUID()}`, + // Replay the recorded provenance so a server copy that moved on + // fails with the existing 412 PRECONDITION_FAILED contract instead + // of being silently clobbered. + ...(def.code.codeVersion !== null && def.code.codeVersion !== undefined + ? { 'if-match': def.code.codeVersion } + : {}), + }, + }); + } + const result = { testId, action: 'updated' as const }; + out.print(result, () => `updated: ${testId}`); + return result; + } + + const created = await client.post<{ testId: string }>('/tests', { + body: { + projectId: def.projectId, + type: def.type, + name: def.name.trim(), + ...(def.description !== undefined ? { description: def.description } : {}), + ...(def.code !== undefined ? { code: def.code.body } : {}), + }, + headers: { 'idempotency-key': `cli-import-create-${randomUUID()}` }, + }); + const result = { testId: created.testId, action: 'created' as const }; + out.print(result, () => `created: ${created.testId}`); + return result; +} + +export interface LintOptions extends CommonOptions { + planFrom?: string; + planFromDir?: string; + plans?: string; + steps?: string; +} + +export interface CliLintIssue { + file: string; + field: string; + reason: string; +} + +export interface CliLintReport { + checked: number; + valid: number; + issues: CliLintIssue[]; +} + /** * `test lint` (issue #98): validate plan/steps files fully OFFLINE with the * SAME validators the create paths run, but collecting EVERY problem instead @@ -8846,6 +9084,38 @@ export function createTestCommand(deps: TestDeps = {}): Command { await runDiff({ ...resolveCommonOptions(command), runA, runB }, deps); }); + test + .command('export ') + .description( + 'Export the test DEFINITION (metadata + code, with codeVersion provenance) to a versionable JSON file. Frontend plans are write-only on the API, so FE exports carry planUnavailable: true.', + ) + .option('--out ', 'write to a file instead of stdout') + .option('--force', 'overwrite an existing --out file', false) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action( + async (testId: string, cmdOpts: { out?: string; force?: boolean }, command: Command) => { + await runExport( + { + ...resolveCommonOptions(command), + testId, + out: cmdOpts.out, + force: cmdOpts.force === true, + }, + deps, + ); + }, + ); + + test + .command('import ') + .description( + 'Create or update a test from a definition file produced by `test export`: a testId in the file updates (code PUT replays the recorded codeVersion as If-Match), no testId creates.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (file: string, _cmdOpts: unknown, command: Command) => { + await runImport({ ...resolveCommonOptions(command), file }, deps); + }); + test .command('lint') .description( diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 4724064..3394ad4 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -241,6 +241,17 @@ Commands: failedStepIndex, per-step status flips, codeVersion drift. Exit 0 when verdicts match, 1 when they differ. + export [options] Export the test DEFINITION (metadata + + code, with codeVersion provenance) to a + versionable JSON file. Frontend plans + are write-only on the API, so FE + exports carry planUnavailable: true. + import Create or update a test from a + definition file produced by \`test + export\`: a testId in the file updates + (code PUT replays the recorded + codeVersion as If-Match), no testId + creates. lint [options] Validate plan/steps files offline with the same validators \`create\` runs, collecting EVERY problem. No network,