From 995555a2eb3b188d9943f1af171969aecb819002 Mon Sep 17 00:00:00 2001 From: Andy00L <89641810+Andy00L@users.noreply.github.com> Date: Tue, 21 Jul 2026 08:03:48 -0400 Subject: [PATCH] feat(cli): add "testsprite ci" one-shot run + gate + GitHub-native output --- src/commands/ci.test.ts | 303 ++++++++++++++++++ src/commands/ci.ts | 273 ++++++++++++++++ src/index.ts | 2 + test/__snapshots__/help.snapshot.test.ts.snap | 4 + 4 files changed, 582 insertions(+) create mode 100644 src/commands/ci.test.ts create mode 100644 src/commands/ci.ts diff --git a/src/commands/ci.test.ts b/src/commands/ci.test.ts new file mode 100644 index 0000000..7865d84 --- /dev/null +++ b/src/commands/ci.test.ts @@ -0,0 +1,303 @@ +/** + * Unit tests for `testsprite ci`. The heavy lifting (trigger + poll) is the + * batch command's, already covered by its own suites; these tests cover the + * CI-facing composition seams: payload reduction, the job-summary Markdown, + * the GitHub gating, and the command wiring. + */ + +import { describe, expect, it, vi, beforeEach } from 'vitest'; +import { Command } from 'commander'; + +// runCi drives the real batch command; mock it so these tests exercise the +// CI presentation seam (capture → summarize → emit → re-throw the gate) with a +// canned payload instead of a live trigger+poll. +const runTestRunAllMock = vi.hoisted(() => vi.fn()); +vi.mock('./test.js', () => ({ runTestRunAll: runTestRunAllMock })); + +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { + createCiCommand, + emitGithubOutputs, + renderJobSummaryMarkdown, + runCi, + summarizeAcceptedPayload, + type CiSummary, +} from './ci.js'; +import { CLIError } from '../lib/errors.js'; + +const PAYLOAD = JSON.stringify({ + accepted: [ + { + testId: 'test_a', + runId: 'run_a', + status: 'passed', + dashboardUrl: 'https://portal.example.com/a', + }, + { + testId: 'test_b', + runId: 'run_b', + status: 'failed', + error: { code: 'INTERNAL', message: 'boom', exitCode: 1 }, + }, + { testId: 'test_c', runId: 'run_c', status: 'timeout' }, + ], + conflicts: [], +}); + +describe('summarizeAcceptedPayload', () => { + it('reduces accepted[] rows into counts and rows', () => { + const summary = summarizeAcceptedPayload(PAYLOAD); + expect(summary).toMatchObject({ total: 3, passed: 1, failed: 1, timedOut: 1 }); + expect(summary.runs[1]).toMatchObject({ testId: 'test_b', status: 'failed', error: 'boom' }); + }); + + it('unparseable or non-batch output reduces to an empty summary (never throws)', () => { + expect(summarizeAcceptedPayload('')).toMatchObject({ total: 0, passed: 0 }); + expect(summarizeAcceptedPayload('{"method":"POST"}')).toMatchObject({ total: 0 }); + expect(summarizeAcceptedPayload('not json')).toMatchObject({ total: 0 }); + }); +}); + +describe('renderJobSummaryMarkdown', () => { + it('renders the counts headline and one table row per run', () => { + const md = renderJobSummaryMarkdown(summarizeAcceptedPayload(PAYLOAD)); + expect(md).toContain('**1/3 passed** (1 failed, 1 timed out)'); + expect(md).toContain('| test_a | passed | [dashboard](https://portal.example.com/a) |'); + expect(md).toContain('| test_c | timeout | run_c |'); + }); +}); + +describe('emitGithubOutputs', () => { + const summary: CiSummary = summarizeAcceptedPayload(PAYLOAD); + + function makeSinks() { + const stdout: string[] = []; + const stderr: string[] = []; + const appended: Array<{ path: string; content: string }> = []; + return { + stdout, + stderr, + appended, + sinks: { + stdout: (line: string) => stdout.push(line), + stderr: (line: string) => stderr.push(line), + appendFile: (path: string, content: string) => appended.push({ path, content }), + }, + }; + } + + it('appends the job summary and annotates only non-passed runs under Actions', () => { + const { stdout, appended, sinks } = makeSinks(); + emitGithubOutputs( + summary, + { GITHUB_ACTIONS: 'true', GITHUB_STEP_SUMMARY: '/gh/summary.md' }, + sinks, + ); + expect(appended).toHaveLength(1); + expect(appended[0]!.path).toBe('/gh/summary.md'); + expect(appended[0]!.content).toContain('TestSprite results'); + const annotations = stdout.filter(line => line.startsWith('::error')); + expect(annotations).toHaveLength(2); + expect(annotations[0]).toContain('test_b'); + expect(annotations[0]).toContain('boom'); + expect(annotations[1]).toContain('test_c'); + }); + + it('emits nothing off-CI, and a broken summary file downgrades to stderr', () => { + const offCi = makeSinks(); + emitGithubOutputs(summary, {}, offCi.sinks); + expect(offCi.stdout).toHaveLength(0); + expect(offCi.appended).toHaveLength(0); + + const broken = makeSinks(); + emitGithubOutputs( + summary, + { GITHUB_STEP_SUMMARY: '/gh/summary.md' }, + { + ...broken.sinks, + appendFile: () => { + throw new Error('EROFS'); + }, + }, + ); + expect(broken.stderr.join('\n')).toContain('could not append'); + }); +}); + +describe('runCi', () => { + beforeEach(() => { + runTestRunAllMock.mockReset(); + }); + + const baseOpts = { + profile: 'default', + output: 'text' as const, + endpointUrl: undefined, + debug: false, + verbose: false, + dryRun: false, + requestTimeoutMs: undefined, + projectId: 'p1', + timeoutSeconds: 600, + maxConcurrency: 50, + }; + + it('prints the machine summary, writes --summary-file, and re-throws the gate error', async () => { + // The batch command prints its JSON payload to (captured) stdout, then + // throws its typed gate error for the failed run. + runTestRunAllMock.mockImplementation( + async (_opts: unknown, deps: { stdout?: (line: string) => void }) => { + deps.stdout?.(PAYLOAD); + throw new CLIError('1 run failed.', 1); + }, + ); + + const stdout: string[] = []; + const written: Array<{ path: string; content: string }> = []; + await expect( + runCi( + { ...baseOpts, summaryFile: '/out/ci.json' }, + { + env: {}, + stdout: (line: string) => stdout.push(line), + stderr: () => undefined, + writeFile: (path: string, content: string) => written.push({ path, content }), + }, + ), + ).rejects.toMatchObject({ exitCode: 1 }); + + const summary = JSON.parse(stdout[0]!) as CiSummary; + expect(summary).toMatchObject({ total: 3, passed: 1, failed: 1, timedOut: 1 }); + expect(written).toHaveLength(1); + expect(written[0]!.path).toBe('/out/ci.json'); + }); + + it('returns the summary and emits GitHub outputs when all pass under Actions', async () => { + const allPass = JSON.stringify({ + accepted: [{ testId: 'test_a', runId: 'run_a', status: 'passed' }], + }); + runTestRunAllMock.mockImplementation( + async (_opts: unknown, deps: { stdout?: (line: string) => void }) => { + deps.stdout?.(allPass); + }, + ); + + const stdout: string[] = []; + const appended: Array<{ path: string; content: string }> = []; + const summary = await runCi(baseOpts, { + env: { GITHUB_ACTIONS: 'true', GITHUB_STEP_SUMMARY: '/gh/summary.md' }, + stdout: (line: string) => stdout.push(line), + stderr: () => undefined, + appendFile: (path: string, content: string) => appended.push({ path, content }), + }); + expect(summary).toMatchObject({ total: 1, passed: 1, failed: 0 }); + expect(appended).toHaveLength(1); + // All passed → no ::error:: annotations. + expect(stdout.filter(line => line.startsWith('::error'))).toHaveLength(0); + }); + + it('uses the default fs writers for --summary-file and $GITHUB_STEP_SUMMARY', async () => { + runTestRunAllMock.mockImplementation( + async (_opts: unknown, deps: { stdout?: (line: string) => void }) => { + deps.stdout?.(JSON.stringify({ accepted: [{ testId: 'test_a', status: 'failed' }] })); + }, + ); + const dir = mkdtempSync(join(tmpdir(), 'ci-test-')); + const summaryFile = join(dir, 'ci.json'); + const stepSummary = join(dir, 'step-summary.md'); + const writeSpy = vi.spyOn(process.stdout, 'write').mockReturnValue(true); + try { + // No writeFile/appendFile injected → exercises the default fs writers. + await runCi( + { ...baseOpts, summaryFile }, + { + env: { GITHUB_ACTIONS: 'true', GITHUB_STEP_SUMMARY: stepSummary }, + stderr: () => undefined, + }, + ); + expect(JSON.parse(readFileSync(summaryFile, 'utf8'))).toMatchObject({ failed: 1 }); + expect(readFileSync(stepSummary, 'utf8')).toContain('TestSprite results'); + } finally { + writeSpy.mockRestore(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('falls back to real process stdout/stderr writers when no sinks are injected', async () => { + runTestRunAllMock.mockImplementation( + async (_opts: unknown, deps: { stdout?: (line: string) => void }) => { + deps.stdout?.(JSON.stringify({ accepted: [] })); + }, + ); + const writeSpy = vi.spyOn(process.stdout, 'write').mockReturnValue(true); + try { + // Only `env` is injected → exercises the default stdout/stderr arrows. + const summary = await runCi(baseOpts, { env: {} }); + expect(summary).toMatchObject({ total: 0 }); + expect(writeSpy).toHaveBeenCalled(); + } finally { + writeSpy.mockRestore(); + } + }); +}); + +describe('createCiCommand wiring', () => { + it('is named "ci" and validates --timeout / --max-concurrency bounds', async () => { + const cmd = createCiCommand({ stdout: () => undefined, stderr: () => undefined }); + expect(cmd.name()).toBe('ci'); + await expect( + cmd.parseAsync(['--project', 'p1', '--timeout', '0'], { from: 'user' }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + const cmd2 = createCiCommand({ stdout: () => undefined, stderr: () => undefined }); + await expect( + cmd2.parseAsync(['--project', 'p1', '--max-concurrency', '101'], { from: 'user' }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + it('drives runCi end-to-end through the parsed action (defaults + flags)', async () => { + runTestRunAllMock.mockReset(); + runTestRunAllMock.mockImplementation( + async (_opts: unknown, deps: { stdout?: (line: string) => void }) => { + deps.stdout?.(JSON.stringify({ accepted: [] })); + }, + ); + const stdout: string[] = []; + const cmd = createCiCommand({ + env: {}, + stdout: (line: string) => stdout.push(line), + stderr: () => undefined, + }); + await cmd.parseAsync(['--project', 'p1', '--timeout', '120', '--max-concurrency', '5'], { + from: 'user', + }); + expect(runTestRunAllMock).toHaveBeenCalledTimes(1); + const firstArg = runTestRunAllMock.mock.calls[0]![0] as { + timeoutSeconds: number; + maxConcurrency: number; + wait: boolean; + }; + expect(firstArg).toMatchObject({ timeoutSeconds: 120, maxConcurrency: 5, wait: true }); + expect(JSON.parse(stdout[0]!)).toMatchObject({ total: 0 }); + }); + + it('forwards the global --request-timeout (seconds) as requestTimeoutMs', async () => { + runTestRunAllMock.mockReset(); + runTestRunAllMock.mockImplementation( + async (_opts: unknown, deps: { stdout?: (line: string) => void }) => { + deps.stdout?.(JSON.stringify({ accepted: [] })); + }, + ); + // Mount `ci` under a parent that carries the global option, mirroring index.ts. + const program = new Command().option('--request-timeout ', 'per-request timeout (seconds)'); + program.addCommand( + createCiCommand({ env: {}, stdout: () => undefined, stderr: () => undefined }), + ); + await program.parseAsync(['--request-timeout', '30', 'ci', '--project', 'p1'], { + from: 'user', + }); + const firstArg = runTestRunAllMock.mock.calls[0]![0] as { requestTimeoutMs?: number }; + expect(firstArg.requestTimeoutMs).toBe(30000); + }); +}); diff --git a/src/commands/ci.ts b/src/commands/ci.ts new file mode 100644 index 0000000..0c6d3b2 --- /dev/null +++ b/src/commands/ci.ts @@ -0,0 +1,273 @@ +/** + * `testsprite ci` (issue #99) — one-shot run + gate + CI-native output. + * + * One command a team pastes into a single CI step. It drives the existing + * batch path (`test run --all --wait --output json`, the documented + * automation contract) rather than re-implementing it, then presents the + * result in the three formats CI consumes: + * (a) a stable machine summary `{total, passed, failed, timedOut, runs[]}` + * on stdout (and optionally `--summary-file `), + * (b) a Markdown results table appended to `$GITHUB_STEP_SUMMARY` when + * running under GitHub Actions, + * (c) one `::error::` workflow-command line per non-passed run so failures + * annotate the PR checks tab. + * The exit code is the batch gate's own (the underlying command's typed + * CLIError is re-thrown AFTER the CI artifacts are written), so + * `testsprite ci` gates the job exactly like `test run --all --wait`. + * Off-CI (no GitHub env), only the machine summary is emitted. + */ + +import { appendFileSync, writeFileSync } from 'node:fs'; +import { Command } from 'commander'; +import type { CommonOptions as FactoryCommonOptions } from '../lib/client-factory.js'; +import { GLOBAL_OPTS_HINT } from '../lib/output.js'; +import { localValidationError } from '../lib/errors.js'; +import type { TestDeps } from './test.js'; +import { runTestRunAll } from './test.js'; + +export interface CiRunRow { + testId: string; + runId?: string; + status: string; + dashboardUrl?: string; + error?: string; +} + +export interface CiSummary { + total: number; + passed: number; + failed: number; + timedOut: number; + runs: CiRunRow[]; +} + +export interface CiDeps extends TestDeps { + /** Appender for $GITHUB_STEP_SUMMARY. Defaults to fs.appendFileSync. */ + appendFile?: (path: string, content: string) => void; + /** Writer for --summary-file. Defaults to fs.writeFileSync. */ + writeFile?: (path: string, content: string) => void; +} + +interface CiOptions extends FactoryCommonOptions { + projectId?: string; + timeoutSeconds: number; + maxConcurrency: number; + summaryFile?: string; +} + +/** + * Reduce the batch command's printed JSON payload into the CI summary. The + * parse is defensive: `ci` reads the same `accepted[]` rows the automation + * contract documents, and anything unparseable (dry-run envelope, partial + * output after a timeout) reduces to an empty run list rather than a crash. + */ +export function summarizeAcceptedPayload(capturedJson: string): CiSummary { + let payload: { accepted?: unknown } = {}; + try { + payload = JSON.parse(capturedJson) as { accepted?: unknown }; + } catch { + // Not a JSON object (dry-run banner path or truncated output): no rows. + } + const rows: CiRunRow[] = Array.isArray(payload.accepted) + ? (payload.accepted as Array>).map(row => { + const errorMessage = + row.error !== null && typeof row.error === 'object' + ? (row.error as { message?: unknown }).message + : undefined; + return { + testId: String(row.testId ?? ''), + ...(typeof row.runId === 'string' ? { runId: row.runId } : {}), + status: String(row.status ?? 'unknown'), + ...(typeof row.dashboardUrl === 'string' ? { dashboardUrl: row.dashboardUrl } : {}), + ...(typeof errorMessage === 'string' ? { error: errorMessage } : {}), + }; + }) + : []; + const passed = rows.filter(row => row.status === 'passed').length; + const timedOut = rows.filter(row => row.status === 'timeout').length; + const failed = rows.length - passed - timedOut; + return { total: rows.length, passed, failed, timedOut, runs: rows }; +} + +/** Markdown table for the GitHub job summary. */ +export function renderJobSummaryMarkdown(summary: CiSummary): string { + return [ + '## TestSprite results', + '', + `**${summary.passed}/${summary.total} passed** (${summary.failed} failed, ${summary.timedOut} timed out)`, + '', + '| Test | Status | Run |', + '| --- | --- | --- |', + ...summary.runs.map( + row => + `| ${row.testId} | ${row.status} | ${ + row.dashboardUrl ? `[dashboard](${row.dashboardUrl})` : (row.runId ?? '') + } |`, + ), + '', + ].join('\n'); +} + +/** + * Emit the GitHub-native surfaces. Self-gating on the standard env vars: + * `$GITHUB_STEP_SUMMARY` (a file path Actions provides) receives the Markdown + * table; `GITHUB_ACTIONS=true` enables one `::error::` workflow command per + * non-passed run on stdout (Actions parses workflow commands from stdout). + * Both writes are best-effort: a broken summary file must not mask the gate. + */ +export function emitGithubOutputs( + summary: CiSummary, + env: NodeJS.ProcessEnv, + sinks: { + stdout: (line: string) => void; + stderr: (line: string) => void; + appendFile: (path: string, content: string) => void; + }, +): void { + const summaryPath = env.GITHUB_STEP_SUMMARY; + if (typeof summaryPath === 'string' && summaryPath.length > 0) { + try { + sinks.appendFile(summaryPath, renderJobSummaryMarkdown(summary)); + } catch { + sinks.stderr('[ci] could not append to GITHUB_STEP_SUMMARY; continuing'); + } + } + if (env.GITHUB_ACTIONS === 'true') { + for (const row of summary.runs) { + if (row.status === 'passed') continue; + const detail = row.error !== undefined ? ` ${row.error}` : ''; + const link = row.dashboardUrl !== undefined ? ` ${row.dashboardUrl}` : ''; + sinks.stdout(`::error title=TestSprite ${row.testId}::status=${row.status}${detail}${link}`); + } + } +} + +export async function runCi(opts: CiOptions, deps: CiDeps = {}): Promise { + const env = deps.env ?? process.env; + const stdoutFn = deps.stdout ?? ((line: string) => process.stdout.write(`${line}\n`)); + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + const appendFile = + deps.appendFile ?? ((path: string, content: string) => appendFileSync(path, content, 'utf8')); + const writeFile = + deps.writeFile ?? ((path: string, content: string) => writeFileSync(path, content, 'utf8')); + + // Drive the real batch command, capturing its machine payload instead of + // printing it (the summary below is `ci`'s stdout contract). Its typed + // gate error (exit 1/7/...) is re-thrown after the CI artifacts land. + const captured: string[] = []; + let gateError: unknown; + try { + await runTestRunAll( + { + profile: opts.profile, + output: 'json', + debug: opts.debug, + verbose: opts.verbose, + dryRun: opts.dryRun, + endpointUrl: opts.endpointUrl, + requestTimeoutMs: opts.requestTimeoutMs, + projectId: opts.projectId ?? '', + wait: true, + timeoutSeconds: opts.timeoutSeconds, + maxConcurrency: opts.maxConcurrency, + }, + { ...deps, stdout: (line: string) => captured.push(line), stderr: stderrFn }, + ); + } catch (err) { + gateError = err; + } + + const summary = summarizeAcceptedPayload(captured.join('\n')); + const summaryJson = JSON.stringify(summary, null, 2); + stdoutFn(summaryJson); + if (opts.summaryFile !== undefined) { + try { + writeFile(opts.summaryFile, `${summaryJson}\n`); + } catch { + stderrFn(`[ci] could not write --summary-file ${opts.summaryFile}; continuing`); + } + } + emitGithubOutputs(summary, env, { stdout: stdoutFn, stderr: stderrFn, appendFile }); + + if (gateError !== undefined) throw gateError; + return summary; +} + +/** Default overall deadline for the CI gate, in seconds (mirrors run --all). */ +const CI_DEFAULT_TIMEOUT_SECONDS = 600; +/** Default poll fan-out bound (mirrors run --all's --max-concurrency). */ +const CI_DEFAULT_MAX_CONCURRENCY = 50; + +export function createCiCommand(deps: CiDeps = {}): Command { + return new Command('ci') + .description( + 'One-shot CI gate: run the project suite, gate on the exit code, and emit CI-native output (machine summary; GitHub job summary + ::error:: PR annotations when on Actions)', + ) + .option('--project ', 'project id (or TESTSPRITE_PROJECT_ID / config project_id)') + .option( + '--timeout ', + `overall deadline in seconds (1-3600, default ${CI_DEFAULT_TIMEOUT_SECONDS})`, + ) + .option( + '--max-concurrency ', + `max concurrent run polls (1-100, default ${CI_DEFAULT_MAX_CONCURRENCY})`, + ) + .option('--summary-file ', 'also write the machine summary JSON to this file') + .addHelpText( + 'after', + '\nExamples:\n' + + ' testsprite ci --project proj_123 # gate a pipeline step\n' + + ' testsprite ci --summary-file testsprite.json # keep the summary as a build artifact', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action( + async ( + cmdOpts: { + project?: string; + timeout?: string; + maxConcurrency?: string; + summaryFile?: string; + }, + command: Command, + ) => { + const globals = command.optsWithGlobals() as Partial & { + requestTimeout?: string; + }; + const timeoutSeconds = + cmdOpts.timeout === undefined ? CI_DEFAULT_TIMEOUT_SECONDS : Number(cmdOpts.timeout); + if (!Number.isInteger(timeoutSeconds) || timeoutSeconds < 1 || timeoutSeconds > 3600) { + throw localValidationError('timeout', 'must be an integer between 1 and 3600'); + } + const maxConcurrency = + cmdOpts.maxConcurrency === undefined + ? CI_DEFAULT_MAX_CONCURRENCY + : Number(cmdOpts.maxConcurrency); + if (!Number.isInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 100) { + throw localValidationError('max-concurrency', 'must be an integer between 1 and 100'); + } + await runCi( + { + profile: globals.profile ?? 'default', + output: globals.output ?? 'text', + endpointUrl: globals.endpointUrl, + debug: globals.debug ?? false, + verbose: globals.verbose ?? false, + dryRun: globals.dryRun ?? false, + requestTimeoutMs: parseRequestTimeoutFlag(globals.requestTimeout), + projectId: cmdOpts.project, + timeoutSeconds, + maxConcurrency, + summaryFile: cmdOpts.summaryFile, + }, + deps, + ); + }, + ); +} + +function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { + if (raw === undefined) return undefined; + const seconds = Number(raw); + if (!Number.isFinite(seconds) || seconds <= 0) return undefined; + return Math.round(seconds * 1000); +} diff --git a/src/index.ts b/src/index.ts index 31d3399..c257ace 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,6 +3,7 @@ import { Command, CommanderError } from 'commander'; import { createAgentCommand } from './commands/agent.js'; import { createAuthCommand } from './commands/auth.js'; +import { createCiCommand } from './commands/ci.js'; import { createCompletionCommand, type CompletionSpec } from './commands/completion.js'; import { createDoctorCommand } from './commands/doctor.js'; import { @@ -93,6 +94,7 @@ program.addCommand(createTestCommand()); program.addCommand(createAgentCommand({})); program.addCommand(createUsageCommand()); program.addCommand(createDoctorCommand()); +program.addCommand(createCiCommand()); program.addCommand(createCompletionCommand(() => buildCompletionSpec())); // Derive the shell-completion spec from the fully-assembled command tree at call diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 4724064..ebc315d 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -765,6 +765,10 @@ Commands: (proactive pre-flight before a large test run) doctor Diagnose CLI setup: version, Node, profile, endpoint, credentials, connectivity, skill + ci [options] One-shot CI gate: run the project suite, gate on + the exit code, and emit CI-native output + (machine summary; GitHub job summary + ::error:: + PR annotations when on Actions) completion [shell] Print a shell completion script (bash|zsh|fish) help [command] display help for command "