diff --git a/tests/eval-host-config.test.ts b/tests/eval-host-config.test.ts index 06869e9..3fb8c5c 100644 --- a/tests/eval-host-config.test.ts +++ b/tests/eval-host-config.test.ts @@ -27,8 +27,8 @@ test("eval reviewer configuration keeps provenance and tuple options aligned", ( test("eval host writes reviewer model through native plugin tuple options", async () => { const repositoryRoot = join(import.meta.dir, ".."); - const scratch = await mkdtemp(join(tmpdir(), "flow-eval-host-config-test-")); const toolchain = currentBunToolchain(packageJson.packageManager); + const scratch = await mkdtemp(join(tmpdir(), "flow-eval-host-config-test-")); const previous = process.env.FLOW_EVAL_NO_AUTH_COPY; process.env.FLOW_EVAL_NO_AUTH_COPY = "1"; let host: EvalHost | null = null; @@ -41,6 +41,8 @@ test("eval host writes reviewer model through native plugin tuple options", asyn opencodeVersion: packageJson.devDependencies["@opencode-ai/plugin"], files: { "package.json": '{"name":"eval-host-config-test"}\n' }, reviewer: { model: "provider/reviewer", steps: 80 }, + // Let startup own cancellation/cleanup before the outer test expires. + signal: AbortSignal.timeout(180_000), }); expect( @@ -55,9 +57,13 @@ test("eval host writes reviewer model through native plugin tuple options", asyn ], }); } finally { - await host?.stop(); - if (previous === undefined) delete process.env.FLOW_EVAL_NO_AUTH_COPY; - else process.env.FLOW_EVAL_NO_AUTH_COPY = previous; - await rm(scratch, { recursive: true, force: true }); + try { + await host?.stop(); + } finally { + if (previous === undefined) delete process.env.FLOW_EVAL_NO_AUTH_COPY; + else process.env.FLOW_EVAL_NO_AUTH_COPY = previous; + await rm(scratch, { recursive: true, force: true }); + } } -}, 30_000); + // The host already permits 180s startup; allow packaging and cleanup as well. +}, 240_000); diff --git a/tests/eval-reporting.test.ts b/tests/eval-reporting.test.ts index a534f63..b5f2015 100644 --- a/tests/eval-reporting.test.ts +++ b/tests/eval-reporting.test.ts @@ -1187,7 +1187,12 @@ describe("eval campaign cancellation", () => { }); } - for (const phase of ["health", "cache", "cache-failure"] as const) { + for (const phase of [ + "health", + "readiness", + "cache", + "cache-failure", + ] as const) { test(`cleans startup scratch and temporary credentials on ${phase}`, async () => { const fixture = await mkdtemp(join(tmpdir(), "flow-eval-start-test-")); const source = join(fixture, "opencode", "auth.json"); @@ -1201,9 +1206,12 @@ describe("eval campaign cancellation", () => { process.env.XDG_DATA_HOME = fixture; delete process.env.FLOW_EVAL_NO_AUTH_COPY; const controller = new AbortController(); - const reason = new CampaignCancelled(130); + const reason = + phase === "readiness" + ? new DOMException("Startup deadline expired", "TimeoutError") + : new CampaignCancelled(130); let scratch = ""; - let healthSignal: AbortSignal | null | undefined; + let requestSignal: AbortSignal | null | undefined; const stop = EvalHost.prototype.stop; const stopping = spyOn(EvalHost.prototype, "stop").mockImplementation( function (this: EvalHost) { @@ -1211,8 +1219,14 @@ describe("eval campaign cancellation", () => { return stop.call(this); }, ); - const requests = mockFetch(async (_input, init) => { - healthSignal = init?.signal; + const requests = mockFetch(async (input, init) => { + if (phase === "readiness" && String(input).endsWith("/global/health")) + return Response.json({ healthy: true }); + if (phase === "readiness") { + expect(String(input)).toEndWith("/session"); + expect(init?.method).toBe("POST"); + } + requestSignal = init?.signal; queueMicrotask(() => controller.abort(reason)); return new Promise(() => {}); }); @@ -1236,12 +1250,16 @@ describe("eval campaign cancellation", () => { signal: controller.signal, }); if (phase === "cache-failure") await expect(starting).rejects.toThrow(); + else if (phase === "readiness") + await expect(starting).rejects.toThrow("Startup deadline expired"); else await expect(starting).rejects.toBe(reason); expect(scratch).not.toBe(""); await expect(readdir(scratch)).rejects.toThrow(); expect(await readFile(source, "utf8")).toBe(credentials); - if (phase === "health") expect(healthSignal?.aborted).toBe(true); + if (phase === "health" || phase === "readiness") + expect(requestSignal?.aborted).toBe(true); else expect(requests).not.toHaveBeenCalled(); + if (phase === "readiness") expect(requests).toHaveBeenCalledTimes(2); } finally { requests.mockRestore(); stopping.mockRestore();