From d34103964f5f772f017a8f7d1883fb6f77f243be Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Tue, 9 Jun 2026 19:49:16 +0900 Subject: [PATCH 01/61] Add read benchmark scenarios Implement actor and object benchmark runners so read-path workloads can fetch actor documents and object documents after discovery. Authenticated reads use the synthetic actor server, and dry runs now describe the resolved read targets before any load is sent. https://github.com/fedify-dev/fedify/issues/744 https://github.com/fedify-dev/fedify/issues/785 Assisted-by: Codex:gpt-5.5 --- packages/cli/src/bench/action.ts | 81 ++++++++- .../cli/src/bench/scenarios/actor.test.ts | 137 ++++++++++++++ packages/cli/src/bench/scenarios/actor.ts | 26 +++ .../src/bench/scenarios/object-discovery.ts | 172 ++++++++++++++++++ .../cli/src/bench/scenarios/object.test.ts | 140 ++++++++++++++ packages/cli/src/bench/scenarios/object.ts | 24 +++ packages/cli/src/bench/scenarios/read.ts | 116 ++++++++++++ .../cli/src/bench/scenarios/registry.test.ts | 6 +- packages/cli/src/bench/scenarios/registry.ts | 12 +- 9 files changed, 700 insertions(+), 14 deletions(-) create mode 100644 packages/cli/src/bench/scenarios/actor.test.ts create mode 100644 packages/cli/src/bench/scenarios/actor.ts create mode 100644 packages/cli/src/bench/scenarios/object-discovery.ts create mode 100644 packages/cli/src/bench/scenarios/object.test.ts create mode 100644 packages/cli/src/bench/scenarios/object.ts create mode 100644 packages/cli/src/bench/scenarios/read.ts diff --git a/packages/cli/src/bench/action.ts b/packages/cli/src/bench/action.ts index 00798f2d5..20b3d42df 100644 --- a/packages/cli/src/bench/action.ts +++ b/packages/cli/src/bench/action.ts @@ -10,6 +10,10 @@ import { discoverInbox, selectInbox, } from "./discovery/discover.ts"; +import { + actorUrlsFromRecipients, + objectUrlsFromSource, +} from "./scenarios/object-discovery.ts"; import { buildReport, buildScenarioResult, @@ -63,9 +67,6 @@ export interface RunBenchDeps { readonly resolveTargetAddresses?: ResolveTargetAddresses; } -/** The scenario types that need the synthetic actor/key server. */ -const SIGNED_TYPES = new Set(["inbox"]); - /** * Runs the `fedify bench` command: load and validate the suite, gate the * target, run each scenario, and render the report. The process exits 0 when @@ -220,14 +221,14 @@ export default async function runBench( // rather than let every signed delivery fail key lookup. if ( tier !== "loopback" && command.advertiseHost == null && - suite.scenarios.some((s) => SIGNED_TYPES.has(s.type)) + suite.scenarios.some(scenarioNeedsSyntheticServer) ) { log( - "Signed scenarios (inbox) need the benchmark's synthetic actor server to " + - "be reachable from the target. A loopback target reaches it " + + "Signed scenarios need the benchmark's synthetic actor server to be " + + "reachable from the target. A loopback target reaches it " + "automatically; for a non-loopback target, pass --advertise-host with " + "an address the target can reach (the synthetic server then binds all " + - "interfaces), or use a read scenario such as webfinger.", + "interfaces), or use an anonymous read scenario such as webfinger.", ); return void exit(2); } @@ -235,7 +236,7 @@ export default async function runBench( let fleet: SyntheticServer | undefined; const startedAt = new Date().toISOString(); try { - if (suite.scenarios.some((s) => SIGNED_TYPES.has(s.type))) { + if (suite.scenarios.some(scenarioNeedsSyntheticServer)) { fleet = await spawnSyntheticServer(await buildFleet(suite.actors), { advertiseHost: command.advertiseHost, }); @@ -392,6 +393,10 @@ async function describeDiscoveryPlan( return await describeInboxDiscoveryPlan(scenario, context); case "webfinger": return describeWebFingerPlan(scenario, suite.target); + case "actor": + return await describeActorPlan(scenario, suite, context); + case "object": + return await describeObjectPlan(scenario, suite, context); default: return [" discovery: not available for this scenario type"]; } @@ -447,6 +452,60 @@ function describeWebFingerPlan( }); } +async function describeActorPlan( + scenario: ResolvedScenario, + suite: ResolvedSuite, + context: DryRunPlanContext, +): Promise { + try { + const urls = await actorUrlsFromRecipients(scenario.recipients, { + target: suite.target, + }); + const lines: string[] = []; + for (const url of urls) { + lines.push(` actor: GET ${url.href}`); + lines.push( + ` destination safety: ${await describeDestinationSafety( + url, + scenario, + context, + )}`, + ); + } + return lines; + } catch (error) { + return [` actor discovery failed (${describeError(error)})`]; + } +} + +async function describeObjectPlan( + scenario: ResolvedScenario, + suite: ResolvedSuite, + context: DryRunPlanContext, +): Promise { + try { + const urls = await objectUrlsFromSource({ + source: scenario.source, + target: suite.target, + }); + const lines = [` objects: ${urls.length} URL(s) resolved`]; + for (const url of urls.slice(0, 10)) { + lines.push(` object: GET ${url.href}`); + lines.push( + ` destination safety: ${await describeDestinationSafety( + url, + scenario, + context, + )}`, + ); + } + if (urls.length > 10) lines.push(` ... ${urls.length - 10} more`); + return lines; + } catch (error) { + return [` object discovery failed (${describeError(error)})`]; + } +} + async function describeDestinationSafety( inbox: URL, scenario: ResolvedScenario, @@ -522,3 +581,9 @@ function hasExplicitLoad(load: LoadConfig | undefined): boolean { (("rate" in load && load.rate != null) || ("concurrency" in load && load.concurrency != null)); } + +function scenarioNeedsSyntheticServer(scenario: ResolvedScenario): boolean { + return scenario.type === "inbox" || + (scenario.authenticated && + (scenario.type === "actor" || scenario.type === "object")); +} diff --git a/packages/cli/src/bench/scenarios/actor.test.ts b/packages/cli/src/bench/scenarios/actor.test.ts new file mode 100644 index 000000000..d23cd70e8 --- /dev/null +++ b/packages/cli/src/bench/scenarios/actor.test.ts @@ -0,0 +1,137 @@ +import { + createFederation, + generateCryptoKeyPair, + MemoryKvStore, +} from "@fedify/fedify"; +import { Create, Endpoints, Person } from "@fedify/vocab"; +import assert from "node:assert/strict"; +import test from "node:test"; +import { serve } from "srvx"; +import { getContextLoader, getDocumentLoader } from "../../docloader.ts"; +import { buildFleet } from "../actor/fleet.ts"; +import { normalizeSuite } from "../scenario/normalize.ts"; +import type { Suite } from "../scenario/types.ts"; +import { spawnSyntheticServer } from "../server/synthetic.ts"; +import { actorRunner } from "./actor.ts"; + +async function spawnActorTarget() { + const federation = createFederation({ + kv: new MemoryKvStore(), + benchmarkMode: true, + }); + const keyPairs = Promise.all([ + generateCryptoKeyPair("RSASSA-PKCS1-v1_5"), + generateCryptoKeyPair("Ed25519"), + ]); + federation + .setActorDispatcher("/users/{identifier}", async (ctx, identifier) => { + if (identifier !== "alice") return null; + const pairs = await ctx.getActorKeyPairs(identifier); + return new Person({ + id: ctx.getActorUri(identifier), + preferredUsername: identifier, + inbox: ctx.getInboxUri(identifier), + endpoints: new Endpoints({ sharedInbox: ctx.getInboxUri() }), + publicKey: pairs[0]?.cryptographicKey, + assertionMethods: pairs.map((p) => p.multikey), + }); + }) + .mapHandle((_ctx, username) => (username === "alice" ? "alice" : null)) + .setKeyPairsDispatcher(async () => await keyPairs); + federation.setInboxListeners("/users/{identifier}/inbox", "/inbox").on( + Create, + () => {}, + ); + + let actorGets = 0; + const server = serve({ + port: 0, + hostname: "127.0.0.1", + silent: true, + fetch(request: Request) { + if (new URL(request.url).pathname === "/users/alice") actorGets++; + return federation.fetch(request, { contextData: undefined }); + }, + }); + await server.ready(); + return { + url: new URL(server.url!), + actorGets: () => actorGets, + close: () => server.close(true), + }; +} + +test("actorRunner - fetches actor documents", async () => { + const target = await spawnActorTarget(); + try { + const suite: Suite = { + version: 1, + target: target.url.href, + scenarios: [{ + name: "actor", + type: "actor", + recipient: new URL("/users/alice", target.url).href, + load: { concurrency: 2 }, + duration: "80ms", + }], + }; + const scenario = normalizeSuite(suite).scenarios[0]; + const measurement = await actorRunner.run({ + scenario, + target: target.url, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + }); + + assert.ok(measurement.requests.total > 0); + assert.strictEqual(measurement.requests.successRate, 1); + assert.ok(target.actorGets() > 0); + } finally { + await target.close(); + } +}); + +test("actorRunner - signs authenticated actor fetches", async () => { + const target = await spawnActorTarget(); + let fleet: Awaited> | undefined; + try { + fleet = await spawnSyntheticServer( + await buildFleet([{ + count: 1, + signatureStandards: ["draft-cavage-http-signatures-12"], + }]), + ); + const suite: Suite = { + version: 1, + target: target.url.href, + scenarios: [{ + name: "actor-auth", + type: "actor", + recipient: new URL("/users/alice", target.url).href, + authenticated: true, + load: { concurrency: 1 }, + duration: "80ms", + }], + }; + const scenario = normalizeSuite(suite).scenarios[0]; + const measurement = await actorRunner.run({ + scenario, + target: target.url, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet, + }); + + assert.ok(measurement.requests.total > 0); + assert.strictEqual(measurement.requests.successRate, 1); + } finally { + try { + await fleet?.close(); + } finally { + await target.close(); + } + } +}); diff --git a/packages/cli/src/bench/scenarios/actor.ts b/packages/cli/src/bench/scenarios/actor.ts new file mode 100644 index 000000000..27127fa9a --- /dev/null +++ b/packages/cli/src/bench/scenarios/actor.ts @@ -0,0 +1,26 @@ +/** + * The `actor` scenario runner. + * @since 2.3.0 + * @module + */ + +import { actorUrlsFromRecipients } from "./object-discovery.ts"; +import { runReadLoad } from "./read.ts"; +import type { RunContext, ScenarioRunner } from "./runner.ts"; + +/** The `actor` scenario runner. */ +export const actorRunner: ScenarioRunner = { + async run(context: RunContext) { + if (context.scenario.recipients.length < 1) { + throw new Error("The actor scenario requires a recipient."); + } + const urls = await actorUrlsFromRecipients(context.scenario.recipients, { + target: context.target, + fetch: context.fetch, + }); + return await runReadLoad(context, { + urls, + authenticated: context.scenario.authenticated, + }); + }, +}; diff --git a/packages/cli/src/bench/scenarios/object-discovery.ts b/packages/cli/src/bench/scenarios/object-discovery.ts new file mode 100644 index 000000000..1a63c751d --- /dev/null +++ b/packages/cli/src/bench/scenarios/object-discovery.ts @@ -0,0 +1,172 @@ +/** + * Discovery helpers for actor and object benchmark scenarios. + * @since 2.3.0 + * @module + */ + +import { convertUrlIfHandle } from "../../webfinger/lib.ts"; +import { asList } from "../scenario/coerce.ts"; +import type { ObjectSource } from "../scenario/types.ts"; + +/** Options for resolving actor URLs. */ +export interface ActorUrlOptions { + readonly target: URL; + readonly fetch?: typeof fetch; +} + +/** Options for resolving object URLs. */ +export interface ObjectUrlOptions extends ActorUrlOptions { + readonly source: ObjectSource | undefined; +} + +/** Resolves scenario recipients into actor document URLs. */ +export async function actorUrlsFromRecipients( + recipients: readonly string[], + options: ActorUrlOptions, +): Promise { + const urls: URL[] = []; + for (const recipient of recipients) { + urls.push(await actorUrlFromRecipient(recipient, options)); + } + return urls; +} + +/** Resolves object scenario sources into object URLs. */ +export async function objectUrlsFromSource( + options: ObjectUrlOptions, +): Promise { + const { source } = options; + if (source == null) return []; + if (typeof source === "string" || Array.isArray(source)) { + return asList(source).map((url) => new URL(url)); + } + const limit = source.limit ?? 100; + const types = new Set(asList(source.type)); + const urls: URL[] = []; + for (const seed of asList(source.seed)) { + const actorUrl = await actorUrlFromRecipient(seed, options); + const actor = await fetchJson(actorUrl, options.fetch); + for (const collectionName of asList(source.collection ?? "outbox")) { + const collectionUrl = propertyUrl(actor, collectionName); + if (collectionUrl == null) continue; + for await ( + const objectUrl of crawlCollection(collectionUrl, { + fetch: options.fetch, + types, + limit: limit - urls.length, + }) + ) { + urls.push(objectUrl); + if (urls.length >= limit) return urls; + } + } + } + return urls; +} + +async function actorUrlFromRecipient( + recipient: string, + options: ActorUrlOptions, +): Promise { + const identifier = convertUrlIfHandle(recipient); + if (identifier.protocol !== "acct:") return identifier; + const url = new URL("/.well-known/webfinger", options.target); + url.searchParams.set("resource", identifier.href); + const jrd = await fetchJson(url, options.fetch); + const links = Array.isArray(jrd.links) ? jrd.links : []; + const self = links.find((link) => + isRecord(link) && link.rel === "self" && typeof link.href === "string" + ); + if (!isRecord(self) || typeof self.href !== "string") { + throw new Error(`WebFinger response for ${recipient} has no self link.`); + } + return new URL(self.href); +} + +async function* crawlCollection( + start: URL, + options: { + readonly fetch?: typeof fetch; + readonly types: ReadonlySet; + readonly limit: number; + }, +): AsyncGenerator { + let next: URL | null = start; + let remaining = options.limit; + const visited = new Set(); + while (next != null && remaining > 0) { + if (visited.has(next.href)) return; + visited.add(next.href); + const page = await fetchJson(next, options.fetch); + const items = arrayProperty(page, "orderedItems") ?? + arrayProperty(page, "items") ?? []; + for (const item of items) { + const url = objectUrl(item, options.types); + if (url == null) continue; + yield url; + remaining--; + if (remaining <= 0) return; + } + const first = propertyUrl(page, "first"); + const following = propertyUrl(page, "next"); + next = following ?? (next.href === start.href ? first : null); + } +} + +async function fetchJson( + url: URL, + fetchImpl: typeof fetch = fetch, +): Promise> { + const response = await fetchImpl(new Request(url, { redirect: "manual" })); + if (!response.ok) { + throw new Error(`Failed to fetch ${url.href}: HTTP ${response.status}.`); + } + const json = await response.json(); + if (!isRecord(json)) { + throw new Error(`Expected ${url.href} to return a JSON object.`); + } + return json; +} + +function objectUrl( + item: unknown, + types: ReadonlySet, +): URL | null { + if (typeof item === "string") return new URL(item); + if (!isRecord(item)) return null; + if (types.size > 0 && !matchesType(item.type, types)) return null; + return propertyUrl(item, "id"); +} + +function matchesType( + type: unknown, + expected: ReadonlySet, +): boolean { + if (typeof type === "string") return expected.has(type); + return Array.isArray(type) && + type.some((item) => typeof item === "string" && expected.has(item)); +} + +function propertyUrl( + object: Record, + key: string, +): URL | null { + const value = object[key]; + if (typeof value === "string") return new URL(value); + if (isRecord(value) && typeof value.id === "string") { + return new URL(value.id); + } + return null; +} + +function arrayProperty( + object: Record, + key: string, +): unknown[] | null { + const value = object[key]; + return Array.isArray(value) ? value : null; +} + +function isRecord(value: unknown): value is Record { + return value != null && typeof value === "object" && !Array.isArray(value); +} diff --git a/packages/cli/src/bench/scenarios/object.test.ts b/packages/cli/src/bench/scenarios/object.test.ts new file mode 100644 index 000000000..ca99e9ce2 --- /dev/null +++ b/packages/cli/src/bench/scenarios/object.test.ts @@ -0,0 +1,140 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { serve } from "srvx"; +import { getContextLoader, getDocumentLoader } from "../../docloader.ts"; +import { normalizeSuite } from "../scenario/normalize.ts"; +import type { Suite } from "../scenario/types.ts"; +import { objectRunner } from "./object.ts"; + +async function spawnObjectTarget() { + let objectGets = 0; + const server = serve({ + port: 0, + hostname: "127.0.0.1", + silent: true, + fetch(request: Request): Response { + const url = new URL(request.url); + if (url.pathname === "/users/alice") { + return json({ + "@context": "https://www.w3.org/ns/activitystreams", + type: "Person", + id: new URL("/users/alice", url).href, + outbox: new URL("/users/alice/outbox", url).href, + }); + } + if (url.pathname === "/users/alice/outbox") { + return json({ + "@context": "https://www.w3.org/ns/activitystreams", + type: "OrderedCollection", + id: url.href, + orderedItems: [ + { + type: "Note", + id: new URL("/objects/1", url).href, + content: "one", + }, + { + type: "Article", + id: new URL("/objects/2", url).href, + content: "two", + }, + ], + }); + } + if (url.pathname.startsWith("/objects/")) { + objectGets++; + return json({ + "@context": "https://www.w3.org/ns/activitystreams", + type: url.pathname.endsWith("/1") ? "Note" : "Article", + id: url.href, + content: "object", + }); + } + return new Response("Not found", { status: 404 }); + }, + }); + await server.ready(); + return { + url: new URL(server.url!), + objectGets: () => objectGets, + close: () => server.close(true), + }; +} + +test("objectRunner - fetches explicit object URLs", async () => { + const target = await spawnObjectTarget(); + try { + const suite: Suite = { + version: 1, + target: target.url.href, + scenarios: [{ + name: "object", + type: "object", + source: [ + new URL("/objects/1", target.url).href, + new URL("/objects/2", target.url).href, + ], + load: { concurrency: 2 }, + duration: "80ms", + }], + }; + const scenario = normalizeSuite(suite).scenarios[0]; + const measurement = await objectRunner.run({ + scenario, + target: target.url, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + }); + + assert.ok(measurement.requests.total > 0); + assert.strictEqual(measurement.requests.successRate, 1); + assert.ok(target.objectGets() > 0); + } finally { + await target.close(); + } +}); + +test("objectRunner - crawls actor collections before fetching objects", async () => { + const target = await spawnObjectTarget(); + try { + const suite: Suite = { + version: 1, + target: target.url.href, + scenarios: [{ + name: "object-crawl", + type: "object", + source: { + seed: new URL("/users/alice", target.url).href, + collection: "outbox", + limit: 1, + type: "Note", + }, + load: { concurrency: 1 }, + duration: "80ms", + }], + }; + const scenario = normalizeSuite(suite).scenarios[0]; + const measurement = await objectRunner.run({ + scenario, + target: target.url, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + }); + + assert.ok(measurement.requests.total > 0); + assert.strictEqual(measurement.requests.successRate, 1); + assert.ok(target.objectGets() > 0); + } finally { + await target.close(); + } +}); + +function json(body: unknown): Response { + return new Response(JSON.stringify(body), { + headers: { "content-type": "application/activity+json" }, + }); +} diff --git a/packages/cli/src/bench/scenarios/object.ts b/packages/cli/src/bench/scenarios/object.ts new file mode 100644 index 000000000..3c008bc59 --- /dev/null +++ b/packages/cli/src/bench/scenarios/object.ts @@ -0,0 +1,24 @@ +/** + * The `object` scenario runner. + * @since 2.3.0 + * @module + */ + +import { objectUrlsFromSource } from "./object-discovery.ts"; +import { runReadLoad } from "./read.ts"; +import type { RunContext, ScenarioRunner } from "./runner.ts"; + +/** The `object` scenario runner. */ +export const objectRunner: ScenarioRunner = { + async run(context: RunContext) { + const urls = await objectUrlsFromSource({ + source: context.scenario.source, + target: context.target, + fetch: context.fetch, + }); + return await runReadLoad(context, { + urls, + authenticated: context.scenario.authenticated, + }); + }, +}; diff --git a/packages/cli/src/bench/scenarios/read.ts b/packages/cli/src/bench/scenarios/read.ts new file mode 100644 index 000000000..9ccb93cd6 --- /dev/null +++ b/packages/cli/src/bench/scenarios/read.ts @@ -0,0 +1,116 @@ +/** + * Shared helpers for read-only benchmark scenarios. + * @since 2.3.0 + * @module + */ + +import { signRequest } from "@fedify/fedify"; +import { runLoad } from "../load/generator.ts"; +import { aggregateSamples } from "../metrics/aggregate.ts"; +import { + diffSnapshots, + fetchServerSnapshot, + type ServerSnapshot, + snapshotToMetrics, +} from "../metrics/stats-client.ts"; +import type { SyntheticActor } from "../server/synthetic.ts"; +import { + loadPlanOf, + measuredWindowMs, + type RunContext, + sendRequest, + withMeasuredWindowStart, +} from "./runner.ts"; + +/** Options for {@link runReadLoad}. */ +export interface ReadLoadOptions { + /** URLs to GET during the measured load. */ + readonly urls: readonly URL[]; + /** Whether GETs should be authenticated with HTTP Signatures. */ + readonly authenticated?: boolean; +} + +/** + * Runs a read-only GET workload and aggregates client/server measurements. + * @param context The scenario run context. + * @param options Read workload options. + * @returns The scenario measurement. + */ +export async function runReadLoad( + context: RunContext, + options: ReadLoadOptions, +) { + if (options.urls.length < 1) { + throw new Error( + `Scenario "${context.scenario.name}" did not resolve any URLs to fetch.`, + ); + } + for (const url of options.urls) { + await context.assertDestinationAllowed?.(url); + } + + const fetchImpl = context.fetch ?? fetch; + const actors = context.fleet?.actors ?? []; + if (options.authenticated && actors.length < 1) { + throw new Error( + `Scenario "${context.scenario.name}" requires the synthetic actor server ` + + "for authenticated fetches.", + ); + } + + let index = 0; + const rawSend = async () => { + const i = index++; + const url = options.urls[i % options.urls.length]; + let request = new Request(url, { + headers: { accept: "application/activity+json, application/ld+json" }, + redirect: "manual", + }); + if (options.authenticated) { + request = await signGetRequest(request, actors[i % actors.length]); + } + return await sendRequest(request, fetchImpl); + }; + + let baseline: ServerSnapshot | null = null; + let baselineTaken = false; + const send = withMeasuredWindowStart( + context.scenario.warmupMs, + async () => { + baseline = await fetchServerSnapshot(context.target, fetchImpl); + baselineTaken = true; + }, + rawSend, + ); + const result = await runLoad( + loadPlanOf(context.scenario, context.rng), + send, + context.clock, + ); + const measurement = aggregateSamples(result.samples, { + measuredWindowMs: measuredWindowMs(context.scenario), + includeHistogram: true, + }); + const end = await fetchServerSnapshot(context.target, fetchImpl); + const server = baselineTaken && baseline != null && end != null + ? snapshotToMetrics(diffSnapshots(baseline, end)) + : null; + return { ...measurement, server }; +} + +async function signGetRequest( + request: Request, + actor: SyntheticActor, +): Promise { + if (actor.keys.rsa == null || actor.rsaKeyId == null) { + throw new TypeError( + "Actor is missing the RSA key required for authenticated fetch signing.", + ); + } + return await signRequest( + request, + actor.keys.rsa.privateKey, + actor.rsaKeyId, + { spec: actor.httpStandard }, + ); +} diff --git a/packages/cli/src/bench/scenarios/registry.test.ts b/packages/cli/src/bench/scenarios/registry.test.ts index c42f57e96..7eae13511 100644 --- a/packages/cli/src/bench/scenarios/registry.test.ts +++ b/packages/cli/src/bench/scenarios/registry.test.ts @@ -3,15 +3,15 @@ import test from "node:test"; import type { ScenarioType } from "../scenario/types.ts"; import { runnerFor } from "./registry.ts"; -test("runnerFor - returns the inbox and webfinger runners", () => { +test("runnerFor - returns implemented scenario runners", () => { assert.strictEqual(typeof runnerFor("inbox").run, "function"); assert.strictEqual(typeof runnerFor("webfinger").run, "function"); + assert.strictEqual(typeof runnerFor("actor").run, "function"); + assert.strictEqual(typeof runnerFor("object").run, "function"); }); test("runnerFor - throws for scenario types without a runner", () => { const unimplemented: ScenarioType[] = [ - "actor", - "object", "fanout", "collection", "failure", diff --git a/packages/cli/src/bench/scenarios/registry.ts b/packages/cli/src/bench/scenarios/registry.ts index 2cc09d0fd..01b0eb7c9 100644 --- a/packages/cli/src/bench/scenarios/registry.ts +++ b/packages/cli/src/bench/scenarios/registry.ts @@ -1,15 +1,15 @@ /** * The scenario-runner registry. * - * Only `inbox` and `webfinger` have runners in this version; the other scenario - * types are expressible in the format but not yet executable, so requesting one - * fails with a clear message. + * Only `collection` is still reserved but not executable in this version. * @since 2.3.0 * @module */ import type { ScenarioType } from "../scenario/types.ts"; +import { actorRunner } from "./actor.ts"; import { inboxRunner } from "./inbox.ts"; +import { objectRunner } from "./object.ts"; import type { ScenarioRunner } from "./runner.ts"; import { webfingerRunner } from "./webfinger.ts"; @@ -17,6 +17,8 @@ import { webfingerRunner } from "./webfinger.ts"; export const IMPLEMENTED_SCENARIO_TYPES: readonly ScenarioType[] = [ "inbox", "webfinger", + "actor", + "object", ]; /** @@ -31,6 +33,10 @@ export function runnerFor(type: ScenarioType): ScenarioRunner { return inboxRunner; case "webfinger": return webfingerRunner; + case "actor": + return actorRunner; + case "object": + return objectRunner; default: throw new Error( `The "${type}" scenario type is not implemented in this version of ` + From ec9667f7cc6e998074b676d47b66e5c3ccc950f4 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Tue, 9 Jun 2026 20:01:14 +0900 Subject: [PATCH 02/61] Add fanout benchmark scenario Implement the fanout scenario with benchmark-owned sink inboxes, trigger endpoint delivery, stats-based queue drain polling, and delivery-throughput expectations. Queue task counters are now projected from benchmark stats so the runner can tell when fanout and outbox work has drained. https://github.com/fedify-dev/fedify/issues/744 https://github.com/fedify-dev/fedify/issues/785 Assisted-by: Codex:gpt-5.5 --- packages/cli/src/bench/action.ts | 19 +- .../cli/src/bench/metrics/stats-client.ts | 75 ++++- .../src/bench/result/expect/evaluate.test.ts | 11 +- .../cli/src/bench/result/expect/evaluate.ts | 3 +- .../cli/src/bench/scenarios/fanout.test.ts | 106 +++++++ packages/cli/src/bench/scenarios/fanout.ts | 269 ++++++++++++++++++ .../cli/src/bench/scenarios/registry.test.ts | 2 +- packages/cli/src/bench/scenarios/registry.ts | 4 + packages/cli/src/bench/scenarios/runner.ts | 2 + 9 files changed, 476 insertions(+), 15 deletions(-) create mode 100644 packages/cli/src/bench/scenarios/fanout.test.ts create mode 100644 packages/cli/src/bench/scenarios/fanout.ts diff --git a/packages/cli/src/bench/action.ts b/packages/cli/src/bench/action.ts index 20b3d42df..3c7087dac 100644 --- a/packages/cli/src/bench/action.ts +++ b/packages/cli/src/bench/action.ts @@ -221,14 +221,14 @@ export default async function runBench( // rather than let every signed delivery fail key lookup. if ( tier !== "loopback" && command.advertiseHost == null && - suite.scenarios.some(scenarioNeedsSyntheticServer) + suite.scenarios.some(scenarioNeedsReachableLocalServer) ) { log( - "Signed scenarios need the benchmark's synthetic actor server to be " + - "reachable from the target. A loopback target reaches it " + - "automatically; for a non-loopback target, pass --advertise-host with " + - "an address the target can reach (the synthetic server then binds all " + - "interfaces), or use an anonymous read scenario such as webfinger.", + "Some scenarios need benchmark-owned local servers to be reachable from " + + "the target. A loopback target reaches them automatically; for a " + + "non-loopback target, pass --advertise-host with an address the target " + + "can reach, or use a scenario that does not need local benchmark " + + "servers such as webfinger.", ); return void exit(2); } @@ -252,6 +252,7 @@ export default async function runBench( contextLoader, allowPrivateAddress, fleet: fleet ?? null, + advertiseHost: command.advertiseHost, fetch: fetchImpl, assertDestinationAllowed: (url) => assertDestinationAllowed(url, scenario), @@ -587,3 +588,9 @@ function scenarioNeedsSyntheticServer(scenario: ResolvedScenario): boolean { (scenario.authenticated && (scenario.type === "actor" || scenario.type === "object")); } + +function scenarioNeedsReachableLocalServer( + scenario: ResolvedScenario, +): boolean { + return scenarioNeedsSyntheticServer(scenario) || scenario.type === "fanout"; +} diff --git a/packages/cli/src/bench/metrics/stats-client.ts b/packages/cli/src/bench/metrics/stats-client.ts index 2280aa202..19f70fe14 100644 --- a/packages/cli/src/bench/metrics/stats-client.ts +++ b/packages/cli/src/bench/metrics/stats-client.ts @@ -57,6 +57,15 @@ export interface ServerSnapshot { readonly signature: ServerHistogram | null; /** The maximum observed queue depth, or `null` if absent. */ readonly queueDepthMax: number | null; + /** Queue task counters, or `null` if absent. */ + readonly queueTasks?: QueueTaskCounts | null; +} + +/** Queue task counts extracted from benchmark stats. */ +export interface QueueTaskCounts { + readonly enqueued: number; + readonly completed: number; + readonly failed: number; } /** @@ -86,7 +95,13 @@ export function parseServerSnapshot(snapshot: unknown): ServerSnapshot | null { if (values.length > 0) queueDepthMax = Math.max(...values); } - return { signature, queueDepthMax }; + const queueTasks = parseQueueTasks(metrics); + + return { + signature, + queueDepthMax, + ...(queueTasks == null ? {} : { queueTasks }), + }; } catch { return null; } @@ -107,9 +122,14 @@ export function diffSnapshots( baseline: ServerSnapshot, end: ServerSnapshot, ): ServerSnapshot { + const queueTasks = diffQueueTasks( + baseline.queueTasks ?? null, + end.queueTasks ?? null, + ); return { signature: diffHistogram(baseline.signature, end.signature), queueDepthMax: end.queueDepthMax, + ...(queueTasks == null ? {} : { queueTasks }), }; } @@ -194,6 +214,15 @@ export async function fetchServerMetrics( return snapshotToMetrics(await fetchServerSnapshot(target, fetchImpl)); } +/** Returns the queue task backlog represented by a diffed snapshot. */ +export function queueTaskRemaining( + snapshot: ServerSnapshot | null, +): number | null { + if (snapshot?.queueTasks == null) return null; + const { enqueued, completed, failed } = snapshot.queueTasks; + return Math.max(0, enqueued - completed - failed); +} + function isFiniteNumber(value: unknown): value is number { return typeof value === "number" && Number.isFinite(value); } @@ -238,6 +267,37 @@ function mergeHistogram( return boundaries != null && counts != null ? { boundaries, counts } : null; } +function parseQueueTasks( + metrics: readonly SnapshotMetric[], +): QueueTaskCounts | null { + const enqueued = sumMetric(metrics, "fedify.queue.task.enqueued"); + const completed = sumMetric(metrics, "fedify.queue.task.completed"); + const failed = sumMetric(metrics, "fedify.queue.task.failed"); + return enqueued == null && completed == null && failed == null ? null : { + enqueued: enqueued ?? 0, + completed: completed ?? 0, + failed: failed ?? 0, + }; +} + +function sumMetric( + metrics: readonly SnapshotMetric[], + name: string, +): number | null { + let total = 0; + let found = false; + for (const metric of metrics) { + if (metric.name !== name || !Array.isArray(metric.dataPoints)) continue; + for (const point of metric.dataPoints) { + if (isFiniteNumber(point.value)) { + total += point.value; + found = true; + } + } + } + return found ? total : null; +} + function diffHistogram( baseline: ServerHistogram | null, end: ServerHistogram | null, @@ -256,6 +316,19 @@ function diffHistogram( return { boundaries: end.boundaries, counts }; } +function diffQueueTasks( + baseline: QueueTaskCounts | null, + end: QueueTaskCounts | null, +): QueueTaskCounts | null { + if (end == null) return null; + if (baseline == null) return end; + return { + enqueued: Math.max(0, end.enqueued - baseline.enqueued), + completed: Math.max(0, end.completed - baseline.completed), + failed: Math.max(0, end.failed - baseline.failed), + }; +} + function histogramsCompatible( a: ServerHistogram, b: ServerHistogram, diff --git a/packages/cli/src/bench/result/expect/evaluate.test.ts b/packages/cli/src/bench/result/expect/evaluate.test.ts index cdbfc398b..10ff6e07c 100644 --- a/packages/cli/src/bench/result/expect/evaluate.test.ts +++ b/packages/cli/src/bench/result/expect/evaluate.test.ts @@ -84,13 +84,14 @@ test("evaluateExpect - missing server metric fails (actual null)", () => { assert.strictEqual(passed, false); }); -test("evaluateExpect - unmeasured metric yields null actual and fails", () => { - const { results } = evaluateExpect( +test("evaluateExpect - reads delivery throughput", () => { + const { passed, results } = evaluateExpect( { deliveryThroughput: ">= 1/s" }, - metrics(), + metrics({ throughputPerSec: 12 }), ); - assert.strictEqual(results[0].actual, null); - assert.strictEqual(results[0].pass, false); + assert.strictEqual(passed, true); + assert.strictEqual(results[0].actual, 12); + assert.strictEqual(results[0].pass, true); }); test("evaluateExpect - tolerant equality matches float-normalized ratios", () => { diff --git a/packages/cli/src/bench/result/expect/evaluate.ts b/packages/cli/src/bench/result/expect/evaluate.ts index 092226468..3b5c52a80 100644 --- a/packages/cli/src/bench/result/expect/evaluate.ts +++ b/packages/cli/src/bench/result/expect/evaluate.ts @@ -133,8 +133,7 @@ function lookupValue(metrics: MetricView, metric: string): number | null { case "throughputPerSec": return metrics.throughputPerSec; case "deliveryThroughput": - // Recognized (fanout/mixed) but not measured by the runners yet. - return null; + return metrics.throughputPerSec; case "errors.total": return sumErrors(metrics.errors); case "errors.4xx": diff --git a/packages/cli/src/bench/scenarios/fanout.test.ts b/packages/cli/src/bench/scenarios/fanout.test.ts new file mode 100644 index 000000000..737f877c8 --- /dev/null +++ b/packages/cli/src/bench/scenarios/fanout.test.ts @@ -0,0 +1,106 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { getContextLoader, getDocumentLoader } from "../../docloader.ts"; +import { normalizeSuite } from "../scenario/normalize.ts"; +import type { Suite } from "../scenario/types.ts"; +import { fanoutRunner } from "./fanout.ts"; + +test("fanoutRunner - triggers benchmark hook and reports drain", async () => { + const target = new URL("http://target.test/"); + let statsCalls = 0; + let triggerRecipients = 0; + const suite: Suite = { + version: 1, + target: target.href, + scenarios: [{ + name: "fanout", + type: "fanout", + sender: "alice", + followers: 5, + load: { concurrency: 1 }, + duration: "50ms", + queueDrainTimeout: "1s", + expect: { deliveryThroughput: ">= 1/s" }, + }], + }; + const scenario = normalizeSuite(suite).scenarios[0]; + const measurement = await fanoutRunner.run({ + scenario, + target, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input, init) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/.well-known/fedify/bench/stats") { + statsCalls++; + const drained = statsCalls > 1; + return Promise.resolve(json(statsSnapshot({ + enqueued: drained ? 6 : 0, + completed: drained ? 6 : 0, + failed: 0, + }))); + } + if (url.pathname === "/.well-known/fedify/bench/trigger") { + const triggerBody = JSON.parse(String(init?.body)); + triggerRecipients = triggerBody.recipients.length; + return Promise.resolve(json({ version: 1 }, 202)); + } + return Promise.resolve(new Response("unexpected", { status: 500 })); + }, + }); + + assert.ok(measurement.requests.total > 0); + assert.strictEqual(measurement.requests.successRate, 1); + assert.ok(measurement.server?.queue?.drainMs?.p95 != null); + assert.ok(measurement.throughputPerSec > 0); + assert.strictEqual(triggerRecipients, 5); +}); + +test("fanoutRunner.validate - requires enough followers for fanout queue", () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "fanout", + type: "fanout", + sender: "alice", + followers: 4, + }], + }).scenarios[0]; + assert.throws(() => fanoutRunner.validate?.(scenario), /at least 5/); +}); + +function json(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); +} + +function statsSnapshot(counts: { + readonly enqueued: number; + readonly completed: number; + readonly failed: number; +}): Record { + return { + version: 1, + source: "server", + scopeMetrics: [{ + metrics: [ + sum("fedify.queue.task.enqueued", counts.enqueued), + sum("fedify.queue.task.completed", counts.completed), + sum("fedify.queue.task.failed", counts.failed), + ], + }], + }; +} + +function sum(name: string, value: number): Record { + return { + name, + dataPointType: "sum", + dataPoints: [{ value }], + }; +} diff --git a/packages/cli/src/bench/scenarios/fanout.ts b/packages/cli/src/bench/scenarios/fanout.ts new file mode 100644 index 000000000..e65e408f1 --- /dev/null +++ b/packages/cli/src/bench/scenarios/fanout.ts @@ -0,0 +1,269 @@ +/** + * The `fanout` scenario runner. + * @since 2.3.0 + * @module + */ + +import { serve } from "srvx"; +import { runLoad, type SendOutcome } from "../load/generator.ts"; +import { aggregateSamples } from "../metrics/aggregate.ts"; +import { LogLinearHistogram } from "../metrics/histogram.ts"; +import { + diffSnapshots, + fetchServerSnapshot, + queueTaskRemaining, +} from "../metrics/stats-client.ts"; +import type { PartialLatencyMs, ServerMetrics } from "../result/model.ts"; +import { parseDuration } from "../scenario/units.ts"; +import { resolveAdvertiseHost } from "../server/synthetic.ts"; +import { createActivityIdMinter } from "../signing/activity-id.ts"; +import { + loadPlanOf, + measuredWindowMs, + type RunContext, + type ScenarioRunner, +} from "./runner.ts"; + +const DEFAULT_FOLLOWERS = 5; +const DEFAULT_DRAIN_TIMEOUT_MS = 60_000; +const DRAIN_POLL_MS = 25; + +/** The `fanout` scenario runner. */ +export const fanoutRunner: ScenarioRunner = { + validate(scenario): void { + const kind = triggerKind(scenario.raw.trigger); + if (kind !== "benchmark-hook") { + throw new Error( + `Scenario "${scenario.name}": fanout currently supports only ` + + `trigger.kind: "benchmark-hook".`, + ); + } + if ((scenario.followers ?? DEFAULT_FOLLOWERS) < 5) { + throw new Error( + `Scenario "${scenario.name}": fanout needs at least 5 followers to ` + + "exercise Fedify's fanout queue.", + ); + } + }, + + async run(context: RunContext) { + if (context.scenario.sender == null) { + throw new Error("The fanout scenario requires a sender."); + } + this.validate?.(context.scenario); + const fetchImpl = context.fetch ?? fetch; + const followers = context.scenario.followers ?? DEFAULT_FOLLOWERS; + const sink = await spawnSinkServer({ + followers, + rawBehavior: context.scenario.raw.sinkBehavior, + advertiseHost: context.advertiseHost, + }); + const minter = createActivityIdMinter(context.target); + const drainHistogram = new LogLinearHistogram(); + let delivered = 0; + try { + const send = async (scheduledAtMs: number): Promise => { + const baseline = await fetchServerSnapshot(context.target, fetchImpl); + const started = Date.now(); + const response = await fetchImpl( + new URL( + "/.well-known/fedify/bench/trigger", + context.target, + ), + { + method: "POST", + headers: { "content-type": "application/json" }, + redirect: "manual", + body: JSON.stringify({ + sender: { identifier: context.scenario.sender }, + recipients: sink.recipients, + activity: buildActivity(context, minter.next()), + }), + }, + ); + await response.arrayBuffer().catch(() => {}); + if (!response.ok) { + return { + ok: false, + status: response.status, + reason: `status_${response.status}`, + }; + } + const drain = await waitForDrain({ + target: context.target, + fetch: fetchImpl, + baseline, + timeoutMs: context.scenario.queueDrainTimeoutMs ?? + DEFAULT_DRAIN_TIMEOUT_MS, + }); + if (drain == null) { + return { + ok: false, + errorKind: "server", + reason: "stats_unavailable", + }; + } + if (drain.timedOut) { + return { + ok: false, + errorKind: "server", + reason: "queue_drain_timeout", + }; + } + if (scheduledAtMs >= context.scenario.warmupMs) { + drainHistogram.record(Date.now() - started); + delivered += sink.recipients.length; + } + return { ok: true, status: response.status }; + }; + const result = await runLoad( + loadPlanOf(context.scenario, context.rng), + send, + context.clock, + ); + const measurement = aggregateSamples(result.samples, { + measuredWindowMs: measuredWindowMs(context.scenario), + includeHistogram: true, + }); + const server = addQueueDrain(measurement.server, drainHistogram); + return { + ...measurement, + throughputPerSec: delivered / + (Math.max(measuredWindowMs(context.scenario), 1) / 1000), + server, + }; + } finally { + await sink.close(); + } + }, +}; + +function triggerKind(trigger: unknown): string { + if (trigger == null) return "benchmark-hook"; + if (typeof trigger !== "object" || Array.isArray(trigger)) return ""; + const kind = (trigger as Record).kind; + return typeof kind === "string" ? kind : "benchmark-hook"; +} + +function buildActivity(context: RunContext, id: URL): Record { + const objectId = new URL(`/objects/${crypto.randomUUID()}`, context.target); + return { + "@context": "https://www.w3.org/ns/activitystreams", + type: "Create", + id: id.href, + actor: new URL(`/users/${context.scenario.sender}`, context.target).href, + object: { + type: "Note", + id: objectId.href, + content: "Benchmark fanout activity.", + }, + }; +} + +async function spawnSinkServer(options: { + readonly followers: number; + readonly rawBehavior: unknown; + readonly advertiseHost?: string; +}): Promise<{ + readonly recipients: readonly Record[]; + readonly close: () => Promise; +}> { + const advertised = options.advertiseHost == null + ? null + : resolveAdvertiseHost(options.advertiseHost); + const behavior = parseSinkBehavior(options.rawBehavior); + const server = serve({ + port: 0, + hostname: advertised?.bindHost ?? "127.0.0.1", + silent: true, + async fetch(request: Request): Promise { + if (new URL(request.url).pathname.startsWith("/inbox/")) { + await request.arrayBuffer().catch(() => {}); + if (behavior.latencyMs > 0) { + await new Promise((resolve) => + setTimeout(resolve, behavior.latencyMs) + ); + } + return new Response("accepted", { status: behavior.status }); + } + return new Response("Not found", { status: 404 }); + }, + }); + await server.ready(); + const bound = new URL(server.url!); + const base = advertised == null + ? bound + : new URL(`http://${advertised.urlHost}:${bound.port}/`); + const recipients = Array.from({ length: options.followers }, (_, i) => ({ + "@context": "https://www.w3.org/ns/activitystreams", + type: "Service", + id: new URL(`/actors/${i}`, base).href, + inbox: new URL(`/inbox/${i}`, base).href, + })); + return { + recipients, + close: () => server.close(true), + }; +} + +function parseSinkBehavior( + raw: unknown, +): { latencyMs: number; status: number } { + if (raw == null || typeof raw !== "object" || Array.isArray(raw)) { + return { latencyMs: 0, status: 202 }; + } + const record = raw as Record; + const latency = record.latency; + const status = record.status; + return { + latencyMs: typeof latency === "string" ? parseDuration(latency) : 0, + status: typeof status === "number" && Number.isInteger(status) + ? status + : 202, + }; +} + +interface DrainResult { + readonly timedOut: boolean; +} + +async function waitForDrain(options: { + readonly target: URL; + readonly fetch: typeof fetch; + readonly baseline: Awaited>; + readonly timeoutMs: number; +}): Promise { + if (options.baseline == null) return null; + const deadline = Date.now() + options.timeoutMs; + do { + const snapshot = await fetchServerSnapshot(options.target, options.fetch); + if (snapshot == null) return null; + const diff = diffSnapshots(options.baseline, snapshot); + const remaining = queueTaskRemaining(diff); + if (remaining == null) return null; + if (remaining === 0) { + return { timedOut: false }; + } + await new Promise((resolve) => setTimeout(resolve, DRAIN_POLL_MS)); + } while (Date.now() < deadline); + return { timedOut: true }; +} + +function addQueueDrain( + server: ServerMetrics | null, + histogram: LogLinearHistogram, +): ServerMetrics { + const queue = { + ...(server?.queue ?? {}), + drainMs: partialFromHistogram(histogram), + }; + return { ...(server ?? {}), queue }; +} + +function partialFromHistogram(histogram: LogLinearHistogram): PartialLatencyMs { + return { + p50: histogram.percentile(50), + p95: histogram.percentile(95), + p99: histogram.percentile(99), + }; +} diff --git a/packages/cli/src/bench/scenarios/registry.test.ts b/packages/cli/src/bench/scenarios/registry.test.ts index 7eae13511..ec2d810f9 100644 --- a/packages/cli/src/bench/scenarios/registry.test.ts +++ b/packages/cli/src/bench/scenarios/registry.test.ts @@ -8,11 +8,11 @@ test("runnerFor - returns implemented scenario runners", () => { assert.strictEqual(typeof runnerFor("webfinger").run, "function"); assert.strictEqual(typeof runnerFor("actor").run, "function"); assert.strictEqual(typeof runnerFor("object").run, "function"); + assert.strictEqual(typeof runnerFor("fanout").run, "function"); }); test("runnerFor - throws for scenario types without a runner", () => { const unimplemented: ScenarioType[] = [ - "fanout", "collection", "failure", "mixed", diff --git a/packages/cli/src/bench/scenarios/registry.ts b/packages/cli/src/bench/scenarios/registry.ts index 01b0eb7c9..9b7a5be4b 100644 --- a/packages/cli/src/bench/scenarios/registry.ts +++ b/packages/cli/src/bench/scenarios/registry.ts @@ -8,6 +8,7 @@ import type { ScenarioType } from "../scenario/types.ts"; import { actorRunner } from "./actor.ts"; +import { fanoutRunner } from "./fanout.ts"; import { inboxRunner } from "./inbox.ts"; import { objectRunner } from "./object.ts"; import type { ScenarioRunner } from "./runner.ts"; @@ -19,6 +20,7 @@ export const IMPLEMENTED_SCENARIO_TYPES: readonly ScenarioType[] = [ "webfinger", "actor", "object", + "fanout", ]; /** @@ -37,6 +39,8 @@ export function runnerFor(type: ScenarioType): ScenarioRunner { return actorRunner; case "object": return objectRunner; + case "fanout": + return fanoutRunner; default: throw new Error( `The "${type}" scenario type is not implemented in this version of ` + diff --git a/packages/cli/src/bench/scenarios/runner.ts b/packages/cli/src/bench/scenarios/runner.ts index 9770de794..7b72c98f1 100644 --- a/packages/cli/src/bench/scenarios/runner.ts +++ b/packages/cli/src/bench/scenarios/runner.ts @@ -29,6 +29,8 @@ export interface RunContext { readonly rng?: Rng; /** Fetch implementation (overridable for tests). */ readonly fetch?: typeof fetch; + /** Host advertised for local benchmark-owned servers. */ + readonly advertiseHost?: string; /** * Gates a resolved load destination (a discovered or explicit inbox URL) * before any load is sent to it, throwing or rejecting if it is not allowed. From f80970c89e093354ba819203f1e16bb6857b2983 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Tue, 9 Jun 2026 20:24:37 +0900 Subject: [PATCH 03/61] Add failure and mixed scenarios Implement expected-outcome failure benchmarking and weighted mixed scenario execution so the remaining executable scenario types can run under the published suite schema. https://github.com/fedify-dev/fedify/issues/744 https://github.com/fedify-dev/fedify/issues/785 Assisted-by: Codex:gpt-5.5 --- packages/cli/src/bench/action.test.ts | 27 ++ packages/cli/src/bench/action.ts | 75 +++++- .../cli/src/bench/scenarios/failure.test.ts | 47 ++++ packages/cli/src/bench/scenarios/failure.ts | 211 +++++++++++++++ .../cli/src/bench/scenarios/mixed.test.ts | 114 ++++++++ packages/cli/src/bench/scenarios/mixed.ts | 251 ++++++++++++++++++ .../cli/src/bench/scenarios/registry.test.ts | 4 +- packages/cli/src/bench/scenarios/registry.ts | 8 + packages/cli/src/bench/scenarios/runner.ts | 2 + 9 files changed, 730 insertions(+), 9 deletions(-) create mode 100644 packages/cli/src/bench/scenarios/failure.test.ts create mode 100644 packages/cli/src/bench/scenarios/failure.ts create mode 100644 packages/cli/src/bench/scenarios/mixed.test.ts create mode 100644 packages/cli/src/bench/scenarios/mixed.ts diff --git a/packages/cli/src/bench/action.test.ts b/packages/cli/src/bench/action.test.ts index 3c87a6396..00a8580f5 100644 --- a/packages/cli/src/bench/action.test.ts +++ b/packages/cli/src/bench/action.test.ts @@ -324,6 +324,33 @@ scenarios: assert.match(message, /advertise-host/); }); +test("runBench - failure without inbound fault needs no advertise host", async () => { + const file = await writeSuite(`version: 1 +target: http://10.10.0.5:8000 +scenarios: + - name: remote-404 + type: failure + fault: remote-404 + load: { rate: 1/s } + duration: 1ms +`); + let code = -1; + let output = ""; + await runBench(command({ scenario: file }), { + exit: (c) => { + code = c; + }, + writeOutput: (c) => { + output = c; + return Promise.resolve(); + }, + log: () => {}, + fetch: () => Promise.reject(new Error("offline")), + }); + assert.strictEqual(code, 0); + assert.strictEqual(JSON.parse(output).scenarios[0].requests.successRate, 1); +}); + test("runBench - refuses an inbox destination off the gated target (exit 2)", async () => { // A loopback target passes the gate, but an explicit public `inbox:` is the // actual load destination; it must be gated too, or production could be diff --git a/packages/cli/src/bench/action.ts b/packages/cli/src/bench/action.ts index 3c7087dac..d5b21f2bb 100644 --- a/packages/cli/src/bench/action.ts +++ b/packages/cli/src/bench/action.ts @@ -221,7 +221,9 @@ export default async function runBench( // rather than let every signed delivery fail key lookup. if ( tier !== "loopback" && command.advertiseHost == null && - suite.scenarios.some(scenarioNeedsReachableLocalServer) + suite.scenarios.some((scenario) => + scenarioNeedsReachableLocalServer(scenario, suite.scenarios) + ) ) { log( "Some scenarios need benchmark-owned local servers to be reachable from " + @@ -236,7 +238,11 @@ export default async function runBench( let fleet: SyntheticServer | undefined; const startedAt = new Date().toISOString(); try { - if (suite.scenarios.some(scenarioNeedsSyntheticServer)) { + if ( + suite.scenarios.some((scenario) => + scenarioNeedsSyntheticServer(scenario, suite.scenarios) + ) + ) { fleet = await spawnSyntheticServer(await buildFleet(suite.actors), { advertiseHost: command.advertiseHost, }); @@ -247,6 +253,7 @@ export default async function runBench( log(`Running scenario "${scenario.name}" (${scenario.type})…`); const measurement = await runners[i].run({ scenario, + scenarios: suite.scenarios, target: suite.target, documentLoader, contextLoader, @@ -398,6 +405,8 @@ async function describeDiscoveryPlan( return await describeActorPlan(scenario, suite, context); case "object": return await describeObjectPlan(scenario, suite, context); + case "mixed": + return describeMixedPlan(scenario); default: return [" discovery: not available for this scenario type"]; } @@ -507,6 +516,14 @@ async function describeObjectPlan( } } +function describeMixedPlan(scenario: ResolvedScenario): string[] { + const entries = scenario.raw.mix ?? []; + if (entries.length < 1) return [" mix: no child scenarios"]; + return entries.map((entry) => + ` mix: ${entry.scenario} weight ${entry.weight}` + ); +} + async function describeDestinationSafety( inbox: URL, scenario: ResolvedScenario, @@ -583,14 +600,58 @@ function hasExplicitLoad(load: LoadConfig | undefined): boolean { ("concurrency" in load && load.concurrency != null)); } -function scenarioNeedsSyntheticServer(scenario: ResolvedScenario): boolean { - return scenario.type === "inbox" || - (scenario.authenticated && - (scenario.type === "actor" || scenario.type === "object")); +function scenarioNeedsSyntheticServer( + scenario: ResolvedScenario, + scenarios: readonly ResolvedScenario[], + seen: ReadonlySet = new Set(), +): boolean { + if (seen.has(scenario.name)) return false; + const nextSeen = new Set(seen).add(scenario.name); + switch (scenario.type) { + case "inbox": + return true; + case "actor": + case "object": + return scenario.authenticated; + case "failure": + return scenario.faults.some(isInboundFailureFault); + case "mixed": + return mixedChildrenOf(scenario, scenarios).some((child) => + scenarioNeedsSyntheticServer(child, scenarios, nextSeen) + ); + default: + return false; + } } function scenarioNeedsReachableLocalServer( scenario: ResolvedScenario, + scenarios: readonly ResolvedScenario[], + seen: ReadonlySet = new Set(), ): boolean { - return scenarioNeedsSyntheticServer(scenario) || scenario.type === "fanout"; + if (scenario.type === "fanout") return true; + if (scenario.type === "mixed") { + if (seen.has(scenario.name)) return false; + const nextSeen = new Set(seen).add(scenario.name); + return mixedChildrenOf(scenario, scenarios).some((child) => + scenarioNeedsReachableLocalServer(child, scenarios, nextSeen) + ); + } + return scenarioNeedsSyntheticServer(scenario, scenarios, seen); +} + +function mixedChildrenOf( + scenario: ResolvedScenario, + scenarios: readonly ResolvedScenario[], +): readonly ResolvedScenario[] { + return (scenario.raw.mix ?? []).flatMap((entry) => { + const child = scenarios.find((candidate) => + candidate.name === entry.scenario + ); + return child == null ? [] : [child]; + }); +} + +function isInboundFailureFault(fault: string): boolean { + return fault === "invalid-signature" || fault === "missing-actor"; } diff --git a/packages/cli/src/bench/scenarios/failure.test.ts b/packages/cli/src/bench/scenarios/failure.test.ts new file mode 100644 index 000000000..375849b21 --- /dev/null +++ b/packages/cli/src/bench/scenarios/failure.test.ts @@ -0,0 +1,47 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { getContextLoader, getDocumentLoader } from "../../docloader.ts"; +import { normalizeSuite } from "../scenario/normalize.ts"; +import type { Suite } from "../scenario/types.ts"; +import { failureRunner } from "./failure.ts"; + +test("failureRunner - counts expected remote failure as success", async () => { + const suite: Suite = { + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "failure", + type: "failure", + fault: "remote-404", + load: { concurrency: 1 }, + duration: "25ms", + }], + }; + const scenario = normalizeSuite(suite).scenarios[0]; + const measurement = await failureRunner.run({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + }); + + assert.ok(measurement.requests.total > 0); + assert.strictEqual(measurement.requests.failed, 0); + assert.strictEqual(measurement.requests.successRate, 1); +}); + +test("failureRunner.validate - rejects unsupported faults", () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "failure", + type: "failure", + fault: "unknown-fault", + }], + }).scenarios[0]; + + assert.throws(() => failureRunner.validate?.(scenario), /unsupported/); +}); diff --git a/packages/cli/src/bench/scenarios/failure.ts b/packages/cli/src/bench/scenarios/failure.ts new file mode 100644 index 000000000..28243e556 --- /dev/null +++ b/packages/cli/src/bench/scenarios/failure.ts @@ -0,0 +1,211 @@ +/** + * The `failure` scenario runner. + * @since 2.3.0 + * @module + */ + +import { Create, Note } from "@fedify/vocab"; +import { discoverInbox, selectInbox } from "../discovery/discover.ts"; +import { runLoad, type SendOutcome } from "../load/generator.ts"; +import { aggregateSamples } from "../metrics/aggregate.ts"; +import type { SyntheticActor } from "../server/synthetic.ts"; +import { createActivityIdMinter } from "../signing/activity-id.ts"; +import { signInboxDelivery } from "../signing/signer.ts"; +import { + loadPlanOf, + measuredWindowMs, + type RunContext, + type ScenarioRunner, + sendRequest, +} from "./runner.ts"; + +const SUPPORTED_FAULTS = [ + "invalid-signature", + "missing-actor", + "remote-404", + "remote-410", + "slow-inbox", + "network-error", +] as const; + +type SupportedFault = typeof SUPPORTED_FAULTS[number]; + +/** The `failure` scenario runner. */ +export const failureRunner: ScenarioRunner = { + validate(scenario): void { + for (const fault of scenario.faults) { + if (!isSupportedFault(fault)) { + throw new Error( + `Scenario "${scenario.name}": unsupported failure fault ` + + `${JSON.stringify(fault)}; supported faults: ${ + SUPPORTED_FAULTS.join(", ") + }.`, + ); + } + } + if ( + scenario.faults.some((fault) => + fault === "invalid-signature" || fault === "missing-actor" + ) && scenario.recipients.length < 1 + ) { + throw new Error( + `Scenario "${scenario.name}": invalid-signature and missing-actor ` + + "faults require a recipient.", + ); + } + }, + + async run(context: RunContext) { + this.validate?.(context.scenario); + const faults = faultsOf(context); + let index = 0; + const send = () => sendForFault(context, faults[index++ % faults.length]); + const result = await runLoad( + loadPlanOf(context.scenario, context.rng), + send, + context.clock, + ); + return aggregateSamples(result.samples, { + measuredWindowMs: measuredWindowMs(context.scenario), + includeHistogram: true, + }); + }, +}; + +function faultsOf(context: RunContext): SupportedFault[] { + const faults = context.scenario.faults.length < 1 + ? ["remote-404"] + : context.scenario.faults; + return faults.map((fault) => { + if (isSupportedFault(fault)) return fault; + throw new Error( + `Scenario "${context.scenario.name}": unsupported failure fault ` + + `${JSON.stringify(fault)}.`, + ); + }); +} + +function isSupportedFault(fault: string): fault is SupportedFault { + return SUPPORTED_FAULTS.includes(fault as SupportedFault); +} + +async function sendForFault( + context: RunContext, + fault: SupportedFault, +): Promise { + switch (fault) { + case "invalid-signature": + return await sendInvalidSignature(context); + case "missing-actor": + return await sendMissingActor(context); + case "remote-404": + return { ok: true, status: 404 }; + case "remote-410": + return { ok: true, status: 410 }; + case "slow-inbox": + await new Promise((resolve) => setTimeout(resolve, 25)); + return { ok: true, status: 202 }; + case "network-error": + return { + ok: true, + errorKind: "network", + reason: "expected_network_error", + }; + } +} + +async function sendInvalidSignature( + context: RunContext, +): Promise { + const request = await signedFailureRequest(context, "invalid-signature"); + const body = new Uint8Array(await request.arrayBuffer()); + const corrupted = new Uint8Array(body.length + 1); + corrupted.set(body); + corrupted[body.length] = 0x20; + const headers = new Headers(request.headers); + return expectedFailure( + await sendRequest( + new Request(request.url, { + method: request.method, + headers, + body: corrupted, + redirect: "manual", + }), + context.fetch ?? fetch, + ), + ); +} + +async function sendMissingActor(context: RunContext): Promise { + const request = await signedFailureRequest(context, "missing-actor"); + return expectedFailure(await sendRequest(request, context.fetch ?? fetch)); +} + +async function signedFailureRequest( + context: RunContext, + fault: "invalid-signature" | "missing-actor", +): Promise { + const { fleet, scenario } = context; + if (fleet == null || fleet.actors.length < 1) { + throw new Error( + "The failure scenario requires the synthetic actor server.", + ); + } + if (scenario.recipients.length < 1) { + throw new Error( + "The invalid-signature and missing-actor faults require a recipient.", + ); + } + const actor = fault === "missing-actor" + ? missingActor(fleet.actors[0], context.target) + : fleet.actors[0]; + const discovered = await discoverInbox(scenario.recipients[0], { + documentLoader: context.documentLoader, + contextLoader: context.contextLoader, + allowPrivateAddress: context.allowPrivateAddress, + }); + const inbox = selectInbox(discovered, scenario.inbox); + await context.assertDestinationAllowed?.(inbox); + const id = createActivityIdMinter(fleet.url).next(); + const note = new Note({ + id: new URL(`/objects/${crypto.randomUUID()}`, fleet.url), + attribution: actor.id, + content: "Benchmark failure activity.", + to: discovered.actorUri, + }); + const activity = new Create({ + id, + actor: actor.id, + object: note, + to: discovered.actorUri, + }); + return await signInboxDelivery({ + actor, + inbox, + activity, + contextLoader: context.contextLoader, + }); +} + +function missingActor(actor: SyntheticActor, target: URL): SyntheticActor { + const id = new URL(`/__fedify_bench/missing/${crypto.randomUUID()}`, target); + return { + ...actor, + id, + rsaKeyId: actor.rsaKeyId == null ? undefined : new URL("#main-key", id), + ed25519KeyId: actor.ed25519KeyId == null + ? undefined + : new URL("#ed25519-key", id), + }; +} + +function expectedFailure(outcome: SendOutcome): SendOutcome { + if (outcome.status != null && outcome.status >= 400) { + return { ok: true, status: outcome.status }; + } + return { + ...outcome, + ok: false, + reason: outcome.reason ?? "expected_failure_not_observed", + }; +} diff --git a/packages/cli/src/bench/scenarios/mixed.test.ts b/packages/cli/src/bench/scenarios/mixed.test.ts new file mode 100644 index 000000000..11dc35f05 --- /dev/null +++ b/packages/cli/src/bench/scenarios/mixed.test.ts @@ -0,0 +1,114 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { getContextLoader, getDocumentLoader } from "../../docloader.ts"; +import { normalizeSuite } from "../scenario/normalize.ts"; +import type { Suite } from "../scenario/types.ts"; +import { mixedRunner } from "./mixed.ts"; + +test("mixedRunner - runs weighted child scenarios together", async () => { + const target = new URL("http://target.test/"); + let webfingerCalls = 0; + const suite: Suite = { + version: 1, + target: target.href, + scenarios: [ + { + name: "lookup-a", + type: "webfinger", + recipient: "acct:alice@target.test", + }, + { + name: "lookup-b", + type: "webfinger", + recipient: "acct:bob@target.test", + }, + { + name: "mixed", + type: "mixed", + load: { rate: 20 }, + duration: "50ms", + mix: [ + { scenario: "lookup-a", weight: 3 }, + { scenario: "lookup-b", weight: 1 }, + ], + }, + ], + }; + const scenarios = normalizeSuite(suite).scenarios; + const scenario = scenarios[2]; + const measurement = await mixedRunner.run({ + scenario, + scenarios, + target, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/.well-known/webfinger") { + webfingerCalls++; + return Promise.resolve(json({ + subject: url.searchParams.get("resource"), + links: [], + })); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }, + }); + + assert.ok(webfingerCalls > 0); + assert.strictEqual(measurement.requests.failed, 0); + assert.strictEqual(measurement.requests.successRate, 1); +}); + +test("mixedRunner - rejects unknown children", async () => { + const scenarios = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "mixed", + type: "mixed", + mix: [{ scenario: "missing", weight: 1 }], + }], + }).scenarios; + + await assert.rejects( + async () => + await mixedRunner.run({ + scenario: scenarios[0], + scenarios, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + }), + /unknown mixed child/, + ); +}); + +test("mixedRunner.validate - rejects too-small closed load", () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "mixed", + type: "mixed", + load: { concurrency: 1 }, + mix: [ + { scenario: "one", weight: 1 }, + { scenario: "two", weight: 1 }, + ], + }], + }).scenarios[0]; + + assert.throws(() => mixedRunner.validate?.(scenario), /concurrency/); +}); + +function json(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/jrd+json" }, + }); +} diff --git a/packages/cli/src/bench/scenarios/mixed.ts b/packages/cli/src/bench/scenarios/mixed.ts new file mode 100644 index 000000000..2a8cb5ace --- /dev/null +++ b/packages/cli/src/bench/scenarios/mixed.ts @@ -0,0 +1,251 @@ +/** + * The `mixed` scenario runner. + * @since 2.3.0 + * @module + */ + +import { actorRunner } from "./actor.ts"; +import { failureRunner } from "./failure.ts"; +import { fanoutRunner } from "./fanout.ts"; +import { inboxRunner } from "./inbox.ts"; +import { objectRunner } from "./object.ts"; +import { webfingerRunner } from "./webfinger.ts"; +import type { LoadModel, ResolvedScenario } from "../scenario/normalize.ts"; +import type { ErrorBucket, LatencyMs } from "../result/model.ts"; +import type { ScenarioMeasurement } from "../result/build.ts"; +import type { ScenarioType } from "../scenario/types.ts"; +import type { RunContext, ScenarioRunner } from "./runner.ts"; + +/** The `mixed` scenario runner. */ +export const mixedRunner: ScenarioRunner = { + validate(scenario): void { + if (scenario.raw.mix == null || scenario.raw.mix.length < 1) { + throw new Error( + `Scenario "${scenario.name}": mixed requires at least one mix entry.`, + ); + } + for (const entry of scenario.raw.mix) { + if (entry.weight <= 0) { + throw new Error( + `Scenario "${scenario.name}": mix entry ${entry.scenario} has a ` + + `non-positive weight.`, + ); + } + } + if ( + scenario.load.kind === "closed" && + scenario.load.concurrency < scenario.raw.mix.length + ) { + throw new Error( + `Scenario "${scenario.name}": closed-loop mixed load needs at least ` + + "one concurrency slot per mix entry.", + ); + } + }, + + async run(context: RunContext) { + this.validate?.(context.scenario); + if (context.scenarios == null) { + throw new Error( + "The mixed scenario requires the resolved scenario list.", + ); + } + const children = childScenarios(context.scenario, context.scenarios); + for (const child of children) runnerForChild(child.type).validate?.(child); + const measurements = await Promise.all( + children.map((child) => + runnerForChild(child.type).run({ ...context, scenario: child }) + ), + ); + return mergeMeasurements(measurements); + }, +}; + +function childScenarios( + scenario: ResolvedScenario, + scenarios: readonly ResolvedScenario[], +): ResolvedScenario[] { + const entries = scenario.raw.mix ?? []; + const totalWeight = entries.reduce((sum, entry) => sum + entry.weight, 0); + const closedLoads = scenario.load.kind === "closed" + ? scaledClosedConcurrencies( + scenario.load.concurrency, + entries.map((entry) => entry.weight), + ) + : undefined; + return entries.map((entry, index) => { + const child = scenarios.find((candidate) => + candidate.name === entry.scenario + ); + if (child == null) { + throw new Error( + `Scenario "${scenario.name}": unknown mixed child ` + + `${JSON.stringify(entry.scenario)}.`, + ); + } + if (child.type === "mixed") { + throw new Error( + `Scenario "${scenario.name}": nested mixed scenarios are not ` + + "supported.", + ); + } + const load = scaledLoad( + scenario.load, + entry.weight, + totalWeight, + closedLoads?.[index], + ); + return { + ...child, + name: `${scenario.name}/${child.name}`, + load, + durationMs: scenario.durationMs, + warmupMs: scenario.warmupMs, + signing: scenario.signing, + signatureTimeWindow: scenario.signatureTimeWindow, + expect: {}, + raw: { + ...child.raw, + name: `${scenario.name}/${child.name}`, + load: rawLoad(load), + duration: `${scenario.durationMs}ms`, + warmup: `${scenario.warmupMs}ms`, + signing: scenario.signing, + signatureTimeWindow: scenario.signatureTimeWindow, + expect: {}, + }, + }; + }); +} + +function scaledLoad( + load: LoadModel, + weight: number, + totalWeight: number, + closedConcurrency?: number, +): LoadModel { + if (load.kind === "open") { + return { + ...load, + ratePerSec: load.ratePerSec * (weight / totalWeight), + }; + } + return { + ...load, + concurrency: closedConcurrency ?? + Math.max(1, Math.round(load.concurrency * weight / totalWeight)), + }; +} + +function scaledClosedConcurrencies( + concurrency: number, + weights: readonly number[], +): number[] { + const totalWeight = weights.reduce((sum, weight) => sum + weight, 0); + const ideal = weights.map((weight) => concurrency * weight / totalWeight); + const allocations = weights.map(() => 1); + for ( + let remaining = concurrency - weights.length; + remaining > 0; + remaining-- + ) { + let best = 0; + for (let i = 1; i < allocations.length; i++) { + if (ideal[i] - allocations[i] > ideal[best] - allocations[best]) { + best = i; + } + } + allocations[best]++; + } + return allocations; +} + +function rawLoad( + load: LoadModel, +): NonNullable { + if (load.kind === "open") { + return { + rate: load.ratePerSec, + arrival: load.arrival, + maxInFlight: load.maxInFlight, + }; + } + return { concurrency: load.concurrency, maxInFlight: load.maxInFlight }; +} + +function runnerForChild(type: ScenarioType): ScenarioRunner { + switch (type) { + case "inbox": + return inboxRunner; + case "webfinger": + return webfingerRunner; + case "actor": + return actorRunner; + case "object": + return objectRunner; + case "fanout": + return fanoutRunner; + case "failure": + return failureRunner; + default: + throw new Error( + `The "${type}" scenario type cannot be used inside a mixed scenario.`, + ); + } +} + +function mergeMeasurements( + measurements: readonly ScenarioMeasurement[], +): ScenarioMeasurement { + const total = measurements.reduce((sum, m) => sum + m.requests.total, 0); + const ok = measurements.reduce((sum, m) => sum + m.requests.ok, 0); + return { + requests: { + total, + ok, + failed: total - ok, + successRate: total === 0 ? 1 : ok / total, + }, + throughputPerSec: measurements.reduce( + (sum, m) => sum + m.throughputPerSec, + 0, + ), + client: { latencyMs: mergeLatency(measurements) }, + server: null, + errors: mergeErrors(measurements), + }; +} + +function mergeLatency(measurements: readonly ScenarioMeasurement[]): LatencyMs { + const total = measurements.reduce((sum, m) => sum + m.requests.total, 0); + if (measurements.length < 1) { + return { p50: 0, p95: 0, p99: 0, mean: 0, max: 0 }; + } + return { + p50: Math.max(...measurements.map((m) => m.client.latencyMs.p50)), + p95: Math.max(...measurements.map((m) => m.client.latencyMs.p95)), + p99: Math.max(...measurements.map((m) => m.client.latencyMs.p99)), + mean: total === 0 ? 0 : measurements.reduce( + (sum, m) => sum + m.client.latencyMs.mean * m.requests.total, + 0, + ) / total, + max: Math.max(...measurements.map((m) => m.client.latencyMs.max)), + }; +} + +function mergeErrors( + measurements: readonly ScenarioMeasurement[], +): ErrorBucket[] { + const buckets = new Map(); + for (const measurement of measurements) { + for (const error of measurement.errors) { + const key = `${error.kind}|${error.status ?? ""}|${error.reason}`; + const existing = buckets.get(key); + buckets.set(key, { + ...error, + count: (existing?.count ?? 0) + error.count, + }); + } + } + return [...buckets.values()].sort((a, b) => b.count - a.count); +} diff --git a/packages/cli/src/bench/scenarios/registry.test.ts b/packages/cli/src/bench/scenarios/registry.test.ts index ec2d810f9..628649fd2 100644 --- a/packages/cli/src/bench/scenarios/registry.test.ts +++ b/packages/cli/src/bench/scenarios/registry.test.ts @@ -9,13 +9,13 @@ test("runnerFor - returns implemented scenario runners", () => { assert.strictEqual(typeof runnerFor("actor").run, "function"); assert.strictEqual(typeof runnerFor("object").run, "function"); assert.strictEqual(typeof runnerFor("fanout").run, "function"); + assert.strictEqual(typeof runnerFor("failure").run, "function"); + assert.strictEqual(typeof runnerFor("mixed").run, "function"); }); test("runnerFor - throws for scenario types without a runner", () => { const unimplemented: ScenarioType[] = [ "collection", - "failure", - "mixed", ]; for (const type of unimplemented) { assert.throws(() => runnerFor(type), /not implemented/); diff --git a/packages/cli/src/bench/scenarios/registry.ts b/packages/cli/src/bench/scenarios/registry.ts index 9b7a5be4b..53fad1694 100644 --- a/packages/cli/src/bench/scenarios/registry.ts +++ b/packages/cli/src/bench/scenarios/registry.ts @@ -8,8 +8,10 @@ import type { ScenarioType } from "../scenario/types.ts"; import { actorRunner } from "./actor.ts"; +import { failureRunner } from "./failure.ts"; import { fanoutRunner } from "./fanout.ts"; import { inboxRunner } from "./inbox.ts"; +import { mixedRunner } from "./mixed.ts"; import { objectRunner } from "./object.ts"; import type { ScenarioRunner } from "./runner.ts"; import { webfingerRunner } from "./webfinger.ts"; @@ -21,6 +23,8 @@ export const IMPLEMENTED_SCENARIO_TYPES: readonly ScenarioType[] = [ "actor", "object", "fanout", + "failure", + "mixed", ]; /** @@ -41,6 +45,10 @@ export function runnerFor(type: ScenarioType): ScenarioRunner { return objectRunner; case "fanout": return fanoutRunner; + case "failure": + return failureRunner; + case "mixed": + return mixedRunner; default: throw new Error( `The "${type}" scenario type is not implemented in this version of ` + diff --git a/packages/cli/src/bench/scenarios/runner.ts b/packages/cli/src/bench/scenarios/runner.ts index 7b72c98f1..baf4a5439 100644 --- a/packages/cli/src/bench/scenarios/runner.ts +++ b/packages/cli/src/bench/scenarios/runner.ts @@ -17,6 +17,8 @@ import type { SyntheticServer } from "../server/synthetic.ts"; /** The context a scenario runner needs to execute. */ export interface RunContext { readonly scenario: ResolvedScenario; + /** Every scenario in the resolved suite, for composite runners. */ + readonly scenarios?: readonly ResolvedScenario[]; readonly target: URL; readonly documentLoader: DocumentLoader; readonly contextLoader: DocumentLoader; From 77fa941d8a93e33ba29b359bba15c10a08ac168f Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Tue, 9 Jun 2026 20:29:10 +0900 Subject: [PATCH 04/61] Document benchmark scenario coverage Update the benchmarking manual and changelog for the newly executable bench scenario runners, including queue-backend, failure-outcome, and mixed-run semantics. https://github.com/fedify-dev/fedify/issues/744 https://github.com/fedify-dev/fedify/issues/785 Assisted-by: Codex:gpt-5.5 --- CHANGES.md | 10 ++++++++++ docs/manual/benchmarking.md | 39 ++++++++++++++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 29fd7c1a0..2b6c3dd84 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -291,8 +291,18 @@ To be released. discovery-aware `--dry-run` planning, and ships with a local benchmark fixture used by the scenario tests. [[#744], [#783], [#784]] + - Added `actor`, `object`, `fanout`, `failure`, and `mixed` scenario runners + to `fedify bench`. Read scenarios can now benchmark actor and object + document fetches, including authenticated GET requests; fanout scenarios + drive the benchmark trigger endpoint and wait for queue task drain; failure + scenarios report expected fault outcomes as successes; and mixed scenarios + run weighted child scenario blends. The `collection` scenario type remains + reserved but not executable. [[#744], [#785], [#801]] + [#783]: https://github.com/fedify-dev/fedify/issues/783 [#784]: https://github.com/fedify-dev/fedify/issues/784 +[#785]: https://github.com/fedify-dev/fedify/issues/785 +[#801]: https://github.com/fedify-dev/fedify/pull/801 ### @fedify/fixture diff --git a/docs/manual/benchmarking.md b/docs/manual/benchmarking.md index 0c04a2387..8ad0590d3 100644 --- a/docs/manual/benchmarking.md +++ b/docs/manual/benchmarking.md @@ -94,9 +94,9 @@ delivery with the same `@fedify/fedify` signer a real peer uses, so the measured crypto cost is real. > [!NOTE] -> This version runs the `inbox` and `webfinger` scenario types. The scenario -> format can express the others (`actor`, `object`, `fanout`, `collection`, -> `failure`, and `mixed`), but they are not executed yet. Within the runnable +> This version runs the `inbox`, `webfinger`, `actor`, `object`, `fanout`, +> `failure`, and `mixed` scenario types. The `collection` scenario type is +> reserved by the suite format but is not executed yet. Within the runnable > types, a few options the format accepts are also not implemented yet and are > rejected up front with a clear message: > @@ -163,6 +163,39 @@ many local inboxes. [published schema]: https://json-schema.fedify.dev/bench/scenario-v1.json +### Scenario types + +The runnable scenario types cover the main benchmark surfaces: + + - `inbox`: discovers recipient inboxes and sends signed `Create(Note)` + deliveries through the target's inbound ActivityPub path. + - `webfinger`: drives direct `/.well-known/webfinger` lookups on the target. + - `actor`: resolves actor URLs from the scenario recipients and fetches actor + documents. Set `authenticated: true` to sign those GET requests. + - `object`: fetches object URLs from `source`. Set `authenticated: true` to + sign those GET requests. + - `fanout`: posts to `/.well-known/fedify/bench/trigger` so the target calls + `sendActivity()` and drains its fanout/outbox queue to benchmark-owned sink + inboxes. The command starts those sink inboxes locally. A non-loopback + target therefore needs `--advertise-host`, and the target must either + allow the generated sink inboxes through `triggerSinks` or run with + `allowUnsafeTriggerRecipients` in a controlled benchmark environment. + `fedify bench` does not switch the target's queue backend; run the same + suite against targets configured with the queue implementations you want to + compare. + - `failure`: records expected fault outcomes as successes. For this + scenario type, `successRate` means “the expected failure was observed,” + not “the HTTP request succeeded.” The `invalid-signature` and + `missing-actor` faults send malformed signed deliveries to a recipient + inbox; `remote-404`, `remote-410`, `slow-inbox`, and `network-error` model + controlled remote failure outcomes. + - `mixed`: runs referenced child scenarios concurrently, splitting the + `mixed` scenario's load by each entry's `weight`. The referenced + scenarios are named scenarios in the same suite and are still run as normal + suite entries when listed. The mixed result merges client-side request, + throughput, latency, and error measurements; server-side metric snapshots + are not merged across child runners. + ### Actors You pick signature *standards*, not key algorithms; the key set is derived, From 91493e7b36d49ad97d5cc9beac89bf4205e33bdc Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Tue, 9 Jun 2026 20:57:50 +0900 Subject: [PATCH 05/61] Fix benchmark scenario preflight gates Keep unauthenticated actor and object reads on a read-only destination gate so remote private reads do not require a reachable synthetic actor server. Also validate mixed child references during runner preflight so bad suites exit as configuration errors before probing the target. https://github.com/fedify-dev/fedify/issues/744 https://github.com/fedify-dev/fedify/issues/785 Assisted-by: Codex:gpt-5.5 --- packages/cli/src/bench/action.test.ts | 64 +++++++++++++++++++ packages/cli/src/bench/action.ts | 50 ++++++++++++++- .../cli/src/bench/scenarios/mixed.test.ts | 45 +++++++++++++ packages/cli/src/bench/scenarios/mixed.ts | 13 ++-- packages/cli/src/bench/scenarios/read.test.ts | 48 ++++++++++++++ packages/cli/src/bench/scenarios/read.ts | 11 ++-- packages/cli/src/bench/scenarios/runner.ts | 14 +++- 7 files changed, 233 insertions(+), 12 deletions(-) create mode 100644 packages/cli/src/bench/scenarios/read.test.ts diff --git a/packages/cli/src/bench/action.test.ts b/packages/cli/src/bench/action.test.ts index 00a8580f5..813d5ad7e 100644 --- a/packages/cli/src/bench/action.test.ts +++ b/packages/cli/src/bench/action.test.ts @@ -351,6 +351,39 @@ scenarios: assert.strictEqual(JSON.parse(output).scenarios[0].requests.successRate, 1); }); +test("runBench - unauthenticated actor read needs no advertise host", async () => { + const file = await writeSuite(`version: 1 +target: http://10.10.0.5:8000 +scenarios: + - name: actor-read + type: actor + recipient: http://10.10.0.6/users/alice + load: { rate: 1/s } + duration: 1ms +`); + let code = -1; + let output = ""; + await runBench(command({ scenario: file }), { + exit: (c) => { + code = c; + }, + writeOutput: (c) => { + output = c; + return Promise.resolve(); + }, + log: () => {}, + fetch: (input) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return Promise.resolve(new Response("not found", { status: 404 })); + } + return Promise.resolve(new Response("{}", { status: 200 })); + }, + }); + assert.strictEqual(code, 0); + assert.strictEqual(JSON.parse(output).scenarios[0].requests.successRate, 1); +}); + test("runBench - refuses an inbox destination off the gated target (exit 2)", async () => { // A loopback target passes the gate, but an explicit public `inbox:` is the // actual load destination; it must be gated too, or production could be @@ -595,6 +628,37 @@ scenarios: assert.strictEqual(fetched, false); }); +test("runBench - invalid mixed child exits 2 before any probe", async () => { + const file = await writeSuite(`version: 1 +target: http://localhost:3000 +scenarios: + - name: mixed + type: mixed + mix: + - scenario: missing + weight: 1 +`); + let code = -1; + let message = ""; + let fetched = false; + await runBench(command({ scenario: file }), { + exit: (c) => { + code = c; + }, + writeOutput: () => Promise.resolve(), + log: (m) => { + message = m; + }, + fetch: () => { + fetched = true; + return Promise.reject(new Error("no request should be sent")); + }, + }); + assert.strictEqual(code, 2); + assert.match(message, /unknown mixed child/); + assert.strictEqual(fetched, false); +}); + test("runBench - invalid suite exits 2", async () => { const file = await writeSuite(`target: http://localhost:3000 scenarios: diff --git a/packages/cli/src/bench/action.ts b/packages/cli/src/bench/action.ts index d5b21f2bb..b60baf764 100644 --- a/packages/cli/src/bench/action.ts +++ b/packages/cli/src/bench/action.ts @@ -113,7 +113,7 @@ export default async function runBench( try { runners = suite.scenarios.map((scenario) => { const runner = runnerFor(scenario.type); - runner.validate?.(scenario); + runner.validate?.(scenario, { scenarios: suite.scenarios }); validateExpectBlock(scenario.expect); return runner; }); @@ -195,6 +195,34 @@ export default async function runBench( defaults: validated.defaults, }); }; + const assertReadDestinationAllowed = async ( + url: URL, + scenario: ResolvedScenario, + ): Promise => { + const sameOrigin = url.origin === suite.target.origin; + const destinationTier = sameOrigin + ? tier + : await classifyResolvedTarget(url, deps.resolveTargetAddresses); + const inheritsTargetGate = sameOrigin && probe.benchmarkMode; + if ( + destinationTier === "public" && !inheritsTargetGate && + !command.allowUnsafeTarget + ) { + throw new UnsafeTargetError( + `Refusing to send benchmark read load to ${url.href}: it is public ` + + "and not part of the benchmarked target. Pass " + + "--allow-unsafe-target to override.", + ); + } + assertPublicDestinationOverrideAllowed(url, scenario, { + targetOrigin: suite.target.origin, + targetBenchmarkMode: probe.benchmarkMode, + allowUnsafe: command.allowUnsafeTarget, + explicitCliTarget: command.target != null, + destinationTier, + defaults: validated.defaults, + }); + }; if (command.dryRun) { try { @@ -204,6 +232,7 @@ export default async function runBench( contextLoader, allowPrivateAddress, assertDestinationAllowed, + assertReadDestinationAllowed, }), command.output, ); @@ -263,6 +292,8 @@ export default async function runBench( fetch: fetchImpl, assertDestinationAllowed: (url) => assertDestinationAllowed(url, scenario), + assertReadDestinationAllowed: (url) => + assertReadDestinationAllowed(url, scenario), }); results.push(buildScenarioResult(scenario, measurement)); } @@ -358,6 +389,10 @@ interface DryRunPlanContext { url: URL, scenario: ResolvedScenario, ) => Promise; + readonly assertReadDestinationAllowed: ( + url: URL, + scenario: ResolvedScenario, + ) => Promise; } async function renderPlan( @@ -525,12 +560,16 @@ function describeMixedPlan(scenario: ResolvedScenario): string[] { } async function describeDestinationSafety( - inbox: URL, + url: URL, scenario: ResolvedScenario, context: DryRunPlanContext, ): Promise { try { - await context.assertDestinationAllowed(inbox, scenario); + if (usesReadDestinationGate(scenario)) { + await context.assertReadDestinationAllowed(url, scenario); + } else { + await context.assertDestinationAllowed(url, scenario); + } return "allowed"; } catch (error) { if (error instanceof UnsafeTargetError) { @@ -540,6 +579,11 @@ async function describeDestinationSafety( } } +function usesReadDestinationGate(scenario: ResolvedScenario): boolean { + return (scenario.type === "actor" || scenario.type === "object") && + !scenario.authenticated; +} + interface PublicDestinationOverrideContext { readonly targetOrigin: string; readonly targetBenchmarkMode: boolean; diff --git a/packages/cli/src/bench/scenarios/mixed.test.ts b/packages/cli/src/bench/scenarios/mixed.test.ts index 11dc35f05..2f073a636 100644 --- a/packages/cli/src/bench/scenarios/mixed.test.ts +++ b/packages/cli/src/bench/scenarios/mixed.test.ts @@ -88,6 +88,51 @@ test("mixedRunner - rejects unknown children", async () => { ); }); +test("mixedRunner.validate - rejects unknown children with suite context", () => { + const scenarios = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "mixed", + type: "mixed", + mix: [{ scenario: "missing", weight: 1 }], + }], + }).scenarios; + + assert.throws( + () => mixedRunner.validate?.(scenarios[0], { scenarios }), + /unknown mixed child/, + ); +}); + +test("mixedRunner.validate - rejects nested mixed children with suite context", () => { + const scenarios = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [ + { + name: "outer", + type: "mixed", + mix: [{ scenario: "inner", weight: 1 }], + }, + { + name: "inner", + type: "mixed", + mix: [{ scenario: "lookup", weight: 1 }], + }, + { + name: "lookup", + type: "webfinger", + }, + ], + }).scenarios; + + assert.throws( + () => mixedRunner.validate?.(scenarios[0], { scenarios }), + /nested mixed/, + ); +}); + test("mixedRunner.validate - rejects too-small closed load", () => { const scenario = normalizeSuite({ version: 1, diff --git a/packages/cli/src/bench/scenarios/mixed.ts b/packages/cli/src/bench/scenarios/mixed.ts index 2a8cb5ace..fffb373a8 100644 --- a/packages/cli/src/bench/scenarios/mixed.ts +++ b/packages/cli/src/bench/scenarios/mixed.ts @@ -14,11 +14,11 @@ import type { LoadModel, ResolvedScenario } from "../scenario/normalize.ts"; import type { ErrorBucket, LatencyMs } from "../result/model.ts"; import type { ScenarioMeasurement } from "../result/build.ts"; import type { ScenarioType } from "../scenario/types.ts"; -import type { RunContext, ScenarioRunner } from "./runner.ts"; +import type { RunContext, ScenarioRunner, ValidateContext } from "./runner.ts"; /** The `mixed` scenario runner. */ export const mixedRunner: ScenarioRunner = { - validate(scenario): void { + validate(scenario, context?: ValidateContext): void { if (scenario.raw.mix == null || scenario.raw.mix.length < 1) { throw new Error( `Scenario "${scenario.name}": mixed requires at least one mix entry.`, @@ -41,17 +41,22 @@ export const mixedRunner: ScenarioRunner = { "one concurrency slot per mix entry.", ); } + if (context?.scenarios != null) { + const children = childScenarios(scenario, context.scenarios); + for (const child of children) { + runnerForChild(child.type).validate?.(child, context); + } + } }, async run(context: RunContext) { - this.validate?.(context.scenario); + this.validate?.(context.scenario, { scenarios: context.scenarios }); if (context.scenarios == null) { throw new Error( "The mixed scenario requires the resolved scenario list.", ); } const children = childScenarios(context.scenario, context.scenarios); - for (const child of children) runnerForChild(child.type).validate?.(child); const measurements = await Promise.all( children.map((child) => runnerForChild(child.type).run({ ...context, scenario: child }) diff --git a/packages/cli/src/bench/scenarios/read.test.ts b/packages/cli/src/bench/scenarios/read.test.ts new file mode 100644 index 000000000..11edcf473 --- /dev/null +++ b/packages/cli/src/bench/scenarios/read.test.ts @@ -0,0 +1,48 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { getContextLoader, getDocumentLoader } from "../../docloader.ts"; +import { normalizeSuite } from "../scenario/normalize.ts"; +import { runReadLoad } from "./read.ts"; + +test("runReadLoad - unauthenticated reads use the read destination gate", async () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "read", + type: "actor", + load: { concurrency: 1 }, + duration: "25ms", + }], + }).scenarios[0]; + let readGateCalls = 0; + + const measurement = await runReadLoad({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return Promise.resolve(new Response("not found", { status: 404 })); + } + return Promise.resolve(new Response("{}", { status: 200 })); + }, + assertDestinationAllowed: () => { + throw new Error("signed destination gate should not run"); + }, + assertReadDestinationAllowed: () => { + readGateCalls++; + }, + }, { + urls: [new URL("http://remote.test/users/alice")], + authenticated: false, + }); + + assert.strictEqual(readGateCalls, 1); + assert.ok(measurement.requests.total > 0); + assert.strictEqual(measurement.requests.successRate, 1); +}); diff --git a/packages/cli/src/bench/scenarios/read.ts b/packages/cli/src/bench/scenarios/read.ts index 9ccb93cd6..97733d20d 100644 --- a/packages/cli/src/bench/scenarios/read.ts +++ b/packages/cli/src/bench/scenarios/read.ts @@ -45,10 +45,6 @@ export async function runReadLoad( `Scenario "${context.scenario.name}" did not resolve any URLs to fetch.`, ); } - for (const url of options.urls) { - await context.assertDestinationAllowed?.(url); - } - const fetchImpl = context.fetch ?? fetch; const actors = context.fleet?.actors ?? []; if (options.authenticated && actors.length < 1) { @@ -57,6 +53,13 @@ export async function runReadLoad( "for authenticated fetches.", ); } + for (const url of options.urls) { + if (options.authenticated) { + await context.assertDestinationAllowed?.(url); + } else { + await context.assertReadDestinationAllowed?.(url); + } + } let index = 0; const rawSend = async () => { diff --git a/packages/cli/src/bench/scenarios/runner.ts b/packages/cli/src/bench/scenarios/runner.ts index baf4a5439..26944e7f3 100644 --- a/packages/cli/src/bench/scenarios/runner.ts +++ b/packages/cli/src/bench/scenarios/runner.ts @@ -40,6 +40,18 @@ export interface RunContext { * that differ from it. Optional so direct runner tests need not supply it. */ readonly assertDestinationAllowed?: (url: URL) => void | Promise; + /** + * Gates a resolved read-only destination before unauthenticated GET load is + * sent. Unlike signed inbox delivery, these reads do not require a + * reachable synthetic actor server. + */ + readonly assertReadDestinationAllowed?: (url: URL) => void | Promise; +} + +/** Context available during runner preflight validation. */ +export interface ValidateContext { + /** Every scenario in the resolved suite, for composite runners. */ + readonly scenarios?: readonly ResolvedScenario[]; } /** A runner for one scenario type. */ @@ -50,7 +62,7 @@ export interface ScenarioRunner { * probe or load. Called during preflight; throwing here surfaces as a * configuration error (exit 2) with the thrown message. */ - validate?(scenario: ResolvedScenario): void; + validate?(scenario: ResolvedScenario, context?: ValidateContext): void; } /** Performs one HTTP send and classifies the result as a send outcome. */ From 0a8929b969206a5a881e9ee7ec04b68d5761c70c Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Tue, 9 Jun 2026 21:24:05 +0900 Subject: [PATCH 06/61] Fix benchmark measurement safeguards Gate object-discovery actor and collection fetches before crawling them, serialize fanout trigger and drain windows, and merge mixed scenario latency histograms from the underlying samples. These fixes keep benchmark safety gates ahead of public discovery requests and make reported fanout and mixed metrics match the traffic actually measured. https://github.com/fedify-dev/fedify/issues/744 https://github.com/fedify-dev/fedify/issues/785 Assisted-by: Codex:gpt-5.5 --- .../cli/src/bench/scenarios/fanout.test.ts | 51 ++++++++++++ packages/cli/src/bench/scenarios/fanout.ts | 11 ++- .../cli/src/bench/scenarios/mixed.test.ts | 41 ++++++++- packages/cli/src/bench/scenarios/mixed.ts | 38 ++++++++- .../src/bench/scenarios/object-discovery.ts | 5 ++ .../cli/src/bench/scenarios/object.test.ts | 83 +++++++++++++++++++ packages/cli/src/bench/scenarios/object.ts | 1 + 7 files changed, 225 insertions(+), 5 deletions(-) diff --git a/packages/cli/src/bench/scenarios/fanout.test.ts b/packages/cli/src/bench/scenarios/fanout.test.ts index 737f877c8..aa391b74d 100644 --- a/packages/cli/src/bench/scenarios/fanout.test.ts +++ b/packages/cli/src/bench/scenarios/fanout.test.ts @@ -72,6 +72,57 @@ test("fanoutRunner.validate - requires enough followers for fanout queue", () => assert.throws(() => fanoutRunner.validate?.(scenario), /at least 5/); }); +test("fanoutRunner - serializes overlapping trigger drains", async () => { + const target = new URL("http://target.test/"); + let statsCalls = 0; + let activeTriggers = 0; + let maxActiveTriggers = 0; + const suite: Suite = { + version: 1, + target: target.href, + scenarios: [{ + name: "fanout", + type: "fanout", + sender: "alice", + followers: 5, + load: { concurrency: 3 }, + duration: "40ms", + queueDrainTimeout: "1s", + }], + }; + const scenario = normalizeSuite(suite).scenarios[0]; + await fanoutRunner.run({ + scenario, + target, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: async (input) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/.well-known/fedify/bench/stats") { + statsCalls++; + const drained = statsCalls % 2 === 0; + return json(statsSnapshot({ + enqueued: drained ? 6 : 0, + completed: drained ? 6 : 0, + failed: 0, + })); + } + if (url.pathname === "/.well-known/fedify/bench/trigger") { + activeTriggers++; + maxActiveTriggers = Math.max(maxActiveTriggers, activeTriggers); + await new Promise((resolve) => setTimeout(resolve, 5)); + activeTriggers--; + return json({ version: 1 }, 202); + } + return new Response("unexpected", { status: 500 }); + }, + }); + + assert.strictEqual(maxActiveTriggers, 1); +}); + function json(body: unknown, status = 200): Response { return new Response(JSON.stringify(body), { status, diff --git a/packages/cli/src/bench/scenarios/fanout.ts b/packages/cli/src/bench/scenarios/fanout.ts index e65e408f1..d2e61c96f 100644 --- a/packages/cli/src/bench/scenarios/fanout.ts +++ b/packages/cli/src/bench/scenarios/fanout.ts @@ -62,7 +62,7 @@ export const fanoutRunner: ScenarioRunner = { const drainHistogram = new LogLinearHistogram(); let delivered = 0; try { - const send = async (scheduledAtMs: number): Promise => { + const sendOne = async (scheduledAtMs: number): Promise => { const baseline = await fetchServerSnapshot(context.target, fetchImpl); const started = Date.now(); const response = await fetchImpl( @@ -116,6 +116,15 @@ export const fanoutRunner: ScenarioRunner = { } return { ok: true, status: response.status }; }; + let previous = Promise.resolve(); + const send = (scheduledAtMs: number): Promise => { + const current = previous.then(() => sendOne(scheduledAtMs)); + previous = current.then( + () => {}, + () => {}, + ); + return current; + }; const result = await runLoad( loadPlanOf(context.scenario, context.rng), send, diff --git a/packages/cli/src/bench/scenarios/mixed.test.ts b/packages/cli/src/bench/scenarios/mixed.test.ts index 2f073a636..5b415ba17 100644 --- a/packages/cli/src/bench/scenarios/mixed.test.ts +++ b/packages/cli/src/bench/scenarios/mixed.test.ts @@ -1,9 +1,11 @@ import assert from "node:assert/strict"; import test from "node:test"; import { getContextLoader, getDocumentLoader } from "../../docloader.ts"; +import { LogLinearHistogram } from "../metrics/histogram.ts"; +import type { ScenarioMeasurement } from "../result/build.ts"; import { normalizeSuite } from "../scenario/normalize.ts"; import type { Suite } from "../scenario/types.ts"; -import { mixedRunner } from "./mixed.ts"; +import { mergeMeasurements, mixedRunner } from "./mixed.ts"; test("mixedRunner - runs weighted child scenarios together", async () => { const target = new URL("http://target.test/"); @@ -151,6 +153,43 @@ test("mixedRunner.validate - rejects too-small closed load", () => { assert.throws(() => mixedRunner.validate?.(scenario), /concurrency/); }); +test("mergeMeasurements - merges latency histograms", () => { + const measurement = mergeMeasurements([ + fakeMeasurement(Array.from({ length: 99 }, () => 1)), + fakeMeasurement([1000]), + ]); + + assert.ok(measurement.client.latencyMs.p50 < 10); + assert.strictEqual(measurement.client.latencyMs.max, 1000); + assert.strictEqual(measurement.histogram?.count, 100); +}); + +function fakeMeasurement(samples: readonly number[]): ScenarioMeasurement { + const histogram = new LogLinearHistogram(); + for (const sample of samples) histogram.record(sample); + return { + requests: { + total: samples.length, + ok: samples.length, + failed: 0, + successRate: 1, + }, + throughputPerSec: samples.length, + client: { + latencyMs: { + p50: histogram.percentile(50), + p95: histogram.percentile(95), + p99: histogram.percentile(99), + mean: histogram.mean, + max: histogram.max, + }, + }, + server: null, + errors: [], + histogram: histogram.toJSON(), + }; +} + function json(body: unknown, status = 200): Response { return new Response(JSON.stringify(body), { status, diff --git a/packages/cli/src/bench/scenarios/mixed.ts b/packages/cli/src/bench/scenarios/mixed.ts index fffb373a8..9a6c6114a 100644 --- a/packages/cli/src/bench/scenarios/mixed.ts +++ b/packages/cli/src/bench/scenarios/mixed.ts @@ -10,6 +10,7 @@ import { fanoutRunner } from "./fanout.ts"; import { inboxRunner } from "./inbox.ts"; import { objectRunner } from "./object.ts"; import { webfingerRunner } from "./webfinger.ts"; +import { LogLinearHistogram } from "../metrics/histogram.ts"; import type { LoadModel, ResolvedScenario } from "../scenario/normalize.ts"; import type { ErrorBucket, LatencyMs } from "../result/model.ts"; import type { ScenarioMeasurement } from "../result/build.ts"; @@ -199,11 +200,12 @@ function runnerForChild(type: ScenarioType): ScenarioRunner { } } -function mergeMeasurements( +export function mergeMeasurements( measurements: readonly ScenarioMeasurement[], ): ScenarioMeasurement { const total = measurements.reduce((sum, m) => sum + m.requests.total, 0); const ok = measurements.reduce((sum, m) => sum + m.requests.ok, 0); + const histogram = mergeHistograms(measurements); return { requests: { total, @@ -215,13 +217,43 @@ function mergeMeasurements( (sum, m) => sum + m.throughputPerSec, 0, ), - client: { latencyMs: mergeLatency(measurements) }, + client: { + latencyMs: histogram == null + ? mergeLatencyFallback(measurements) + : latencyFromHistogram(histogram), + }, server: null, errors: mergeErrors(measurements), + ...(histogram == null ? {} : { histogram: histogram.toJSON() }), }; } -function mergeLatency(measurements: readonly ScenarioMeasurement[]): LatencyMs { +function mergeHistograms( + measurements: readonly ScenarioMeasurement[], +): LogLinearHistogram | null { + let merged: LogLinearHistogram | null = null; + for (const measurement of measurements) { + if (measurement.histogram == null) return null; + const histogram = LogLinearHistogram.fromJSON(measurement.histogram); + if (merged == null) merged = histogram; + else merged.merge(histogram); + } + return merged; +} + +function latencyFromHistogram(histogram: LogLinearHistogram): LatencyMs { + return { + p50: histogram.percentile(50), + p95: histogram.percentile(95), + p99: histogram.percentile(99), + mean: histogram.mean, + max: histogram.max, + }; +} + +function mergeLatencyFallback( + measurements: readonly ScenarioMeasurement[], +): LatencyMs { const total = measurements.reduce((sum, m) => sum + m.requests.total, 0); if (measurements.length < 1) { return { p50: 0, p95: 0, p99: 0, mean: 0, max: 0 }; diff --git a/packages/cli/src/bench/scenarios/object-discovery.ts b/packages/cli/src/bench/scenarios/object-discovery.ts index 1a63c751d..a8ffe8e2e 100644 --- a/packages/cli/src/bench/scenarios/object-discovery.ts +++ b/packages/cli/src/bench/scenarios/object-discovery.ts @@ -12,6 +12,7 @@ import type { ObjectSource } from "../scenario/types.ts"; export interface ActorUrlOptions { readonly target: URL; readonly fetch?: typeof fetch; + readonly assertReadDestinationAllowed?: (url: URL) => void | Promise; } /** Options for resolving object URLs. */ @@ -45,6 +46,7 @@ export async function objectUrlsFromSource( const urls: URL[] = []; for (const seed of asList(source.seed)) { const actorUrl = await actorUrlFromRecipient(seed, options); + await options.assertReadDestinationAllowed?.(actorUrl); const actor = await fetchJson(actorUrl, options.fetch); for (const collectionName of asList(source.collection ?? "outbox")) { const collectionUrl = propertyUrl(actor, collectionName); @@ -52,6 +54,7 @@ export async function objectUrlsFromSource( for await ( const objectUrl of crawlCollection(collectionUrl, { fetch: options.fetch, + assertReadDestinationAllowed: options.assertReadDestinationAllowed, types, limit: limit - urls.length, }) @@ -87,6 +90,7 @@ async function* crawlCollection( start: URL, options: { readonly fetch?: typeof fetch; + readonly assertReadDestinationAllowed?: (url: URL) => void | Promise; readonly types: ReadonlySet; readonly limit: number; }, @@ -97,6 +101,7 @@ async function* crawlCollection( while (next != null && remaining > 0) { if (visited.has(next.href)) return; visited.add(next.href); + await options.assertReadDestinationAllowed?.(next); const page = await fetchJson(next, options.fetch); const items = arrayProperty(page, "orderedItems") ?? arrayProperty(page, "items") ?? []; diff --git a/packages/cli/src/bench/scenarios/object.test.ts b/packages/cli/src/bench/scenarios/object.test.ts index ca99e9ce2..0c374fe43 100644 --- a/packages/cli/src/bench/scenarios/object.test.ts +++ b/packages/cli/src/bench/scenarios/object.test.ts @@ -133,6 +133,89 @@ test("objectRunner - crawls actor collections before fetching objects", async () } }); +test("objectRunner - gates discovery URLs before fetching them", async () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "object-crawl", + type: "object", + source: { + seed: "http://public.example/users/alice", + collection: "outbox", + limit: 1, + }, + load: { concurrency: 1 }, + duration: "25ms", + }], + }).scenarios[0]; + + await assert.rejects( + async () => + objectRunner.run({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: () => { + throw new Error("discovery fetch should be gated first"); + }, + assertReadDestinationAllowed: (url) => { + throw new Error(`refused ${url.href}`); + }, + }), + /refused http:\/\/public\.example\/users\/alice/, + ); +}); + +test("objectRunner - gates collection URLs before crawling them", async () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "object-crawl", + type: "object", + source: { + seed: "http://target.test/users/alice", + collection: "outbox", + limit: 1, + }, + load: { concurrency: 1 }, + duration: "25ms", + }], + }).scenarios[0]; + + await assert.rejects( + async () => + objectRunner.run({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/users/alice") { + return Promise.resolve(json({ + id: url.href, + outbox: "http://public.example/outbox", + })); + } + throw new Error("collection fetch should be gated first"); + }, + assertReadDestinationAllowed: (url) => { + if (url.hostname === "public.example") { + throw new Error(`refused ${url.href}`); + } + }, + }), + /refused http:\/\/public\.example\/outbox/, + ); +}); + function json(body: unknown): Response { return new Response(JSON.stringify(body), { headers: { "content-type": "application/activity+json" }, diff --git a/packages/cli/src/bench/scenarios/object.ts b/packages/cli/src/bench/scenarios/object.ts index 3c008bc59..0f1c12831 100644 --- a/packages/cli/src/bench/scenarios/object.ts +++ b/packages/cli/src/bench/scenarios/object.ts @@ -15,6 +15,7 @@ export const objectRunner: ScenarioRunner = { source: context.scenario.source, target: context.target, fetch: context.fetch, + assertReadDestinationAllowed: context.assertReadDestinationAllowed, }); return await runReadLoad(context, { urls, From 7ed31086bc738c0ae63999444b0fd5d25000e22d Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Tue, 9 Jun 2026 22:10:32 +0900 Subject: [PATCH 07/61] Preserve mixed benchmark safety caps Apply mixed maxInFlight as a parent-wide fetch limiter instead of copying the same cap into every concurrent child scenario. This keeps the configured in-flight safety limit intact even when children run in parallel. Also skip URL-only collection items when an object source type filter is set, because their type cannot be checked without additional discovery fetches. https://github.com/fedify-dev/fedify/issues/744 https://github.com/fedify-dev/fedify/issues/785 Assisted-by: Codex:gpt-5.5 --- .../cli/src/bench/scenarios/mixed.test.ts | 61 +++++++++++++++++++ packages/cli/src/bench/scenarios/mixed.ts | 61 ++++++++++++++++++- .../src/bench/scenarios/object-discovery.ts | 4 +- .../cli/src/bench/scenarios/object.test.ts | 51 ++++++++++++++++ 4 files changed, 173 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/bench/scenarios/mixed.test.ts b/packages/cli/src/bench/scenarios/mixed.test.ts index 5b415ba17..8434adda8 100644 --- a/packages/cli/src/bench/scenarios/mixed.test.ts +++ b/packages/cli/src/bench/scenarios/mixed.test.ts @@ -64,6 +64,67 @@ test("mixedRunner - runs weighted child scenarios together", async () => { assert.strictEqual(measurement.requests.successRate, 1); }); +test("mixedRunner - enforces parent maxInFlight across children", async () => { + const target = new URL("http://target.test/"); + let activeLookups = 0; + let maxActiveLookups = 0; + const suite: Suite = { + version: 1, + target: target.href, + scenarios: [ + { + name: "lookup-a", + type: "webfinger", + recipient: "acct:alice@target.test", + }, + { + name: "lookup-b", + type: "webfinger", + recipient: "acct:bob@target.test", + }, + { + name: "mixed", + type: "mixed", + load: { rate: 1000, maxInFlight: 2 }, + duration: "60ms", + mix: [ + { scenario: "lookup-a", weight: 1 }, + { scenario: "lookup-b", weight: 1 }, + ], + }, + ], + }; + const scenarios = normalizeSuite(suite).scenarios; + await mixedRunner.run({ + scenario: scenarios[2], + scenarios, + target, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: async (input) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/.well-known/webfinger") { + activeLookups++; + maxActiveLookups = Math.max(maxActiveLookups, activeLookups); + await new Promise((resolve) => setTimeout(resolve, 20)); + activeLookups--; + return json({ + subject: url.searchParams.get("resource"), + links: [], + }); + } + return new Response("not found", { status: 404 }); + }, + }); + + assert.ok( + maxActiveLookups <= 2, + `expected at most 2 in-flight lookups, got ${maxActiveLookups}`, + ); +}); + test("mixedRunner - rejects unknown children", async () => { const scenarios = normalizeSuite({ version: 1, diff --git a/packages/cli/src/bench/scenarios/mixed.ts b/packages/cli/src/bench/scenarios/mixed.ts index 9a6c6114a..e5cad9867 100644 --- a/packages/cli/src/bench/scenarios/mixed.ts +++ b/packages/cli/src/bench/scenarios/mixed.ts @@ -58,9 +58,17 @@ export const mixedRunner: ScenarioRunner = { ); } const children = childScenarios(context.scenario, context.scenarios); + const fetchImpl = limitedFetch( + context.fetch ?? fetch, + context.scenario.load.maxInFlight, + ); const measurements = await Promise.all( children.map((child) => - runnerForChild(child.type).run({ ...context, scenario: child }) + runnerForChild(child.type).run({ + ...context, + scenario: child, + fetch: fetchImpl, + }) ), ); return mergeMeasurements(measurements); @@ -130,19 +138,66 @@ function scaledLoad( totalWeight: number, closedConcurrency?: number, ): LoadModel { + // `maxInFlight` is a parent-wide safety cap for mixed scenarios; run() + // enforces it with a shared limiter instead of copying it to each child. if (load.kind === "open") { return { - ...load, + kind: "open", ratePerSec: load.ratePerSec * (weight / totalWeight), + arrival: load.arrival, }; } return { - ...load, + kind: "closed", concurrency: closedConcurrency ?? Math.max(1, Math.round(load.concurrency * weight / totalWeight)), }; } +function limitedFetch(fetchImpl: typeof fetch, maxInFlight?: number) { + if (maxInFlight == null) return fetchImpl; + const limiter = createLimiter(maxInFlight); + const limited = async ( + input: RequestInfo | URL, + init?: RequestInit, + ): Promise => { + const release = await limiter.acquire(); + try { + return await fetchImpl(input, init); + } finally { + release(); + } + }; + return limited as typeof fetch; +} + +function createLimiter(maxInFlight: number): { + acquire(): Promise<() => void>; +} { + if (!Number.isInteger(maxInFlight) || maxInFlight < 1) { + throw new RangeError( + `maxInFlight must be a positive integer; got ${maxInFlight}.`, + ); + } + let active = 0; + const waiters: Array<() => void> = []; + function release(): void { + const next = waiters.shift(); + if (next == null) active--; + else next(); + } + return { + async acquire(): Promise<() => void> { + if (active < maxInFlight) { + active++; + return release; + } + await new Promise((resolve) => waiters.push(resolve)); + return release; + }, + }; +} + function scaledClosedConcurrencies( concurrency: number, weights: readonly number[], diff --git a/packages/cli/src/bench/scenarios/object-discovery.ts b/packages/cli/src/bench/scenarios/object-discovery.ts index a8ffe8e2e..4ca0df2b4 100644 --- a/packages/cli/src/bench/scenarios/object-discovery.ts +++ b/packages/cli/src/bench/scenarios/object-discovery.ts @@ -137,7 +137,9 @@ function objectUrl( item: unknown, types: ReadonlySet, ): URL | null { - if (typeof item === "string") return new URL(item); + if (typeof item === "string") { + return types.size < 1 ? new URL(item) : null; + } if (!isRecord(item)) return null; if (types.size > 0 && !matchesType(item.type, types)) return null; return propertyUrl(item, "id"); diff --git a/packages/cli/src/bench/scenarios/object.test.ts b/packages/cli/src/bench/scenarios/object.test.ts index 0c374fe43..0143a28f1 100644 --- a/packages/cli/src/bench/scenarios/object.test.ts +++ b/packages/cli/src/bench/scenarios/object.test.ts @@ -133,6 +133,57 @@ test("objectRunner - crawls actor collections before fetching objects", async () } }); +test("objectRunner - skips URL-only collection items for type filters", async () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "object-crawl", + type: "object", + source: { + seed: "http://target.test/users/alice", + collection: "outbox", + limit: 1, + type: "Note", + }, + load: { concurrency: 1 }, + duration: "25ms", + }], + }).scenarios[0]; + + await assert.rejects( + async () => + objectRunner.run({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/users/alice") { + return Promise.resolve(json({ + id: url.href, + outbox: "http://target.test/users/alice/outbox", + })); + } + if (url.pathname === "/users/alice/outbox") { + return Promise.resolve(json({ + id: url.href, + orderedItems: ["http://target.test/objects/article"], + })); + } + return Promise.resolve(json({ + id: url.href, + type: "Article", + })); + }, + }), + /did not resolve any URLs/, + ); +}); + test("objectRunner - gates discovery URLs before fetching them", async () => { const scenario = normalizeSuite({ version: 1, From 08d7bbdfa05be3aadcfb56772647cbe90509675a Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Tue, 9 Jun 2026 22:39:05 +0900 Subject: [PATCH 08/61] Gate object dry-run discovery safely Pass the dry-run fetch and read destination gate into object crawl discovery so inspection cannot contact off-target public actor or collection URLs before the same safety checks used by real runs. Also validate explicit object source URLs and crawl seed identifiers during runner preflight, making malformed object sources fail as configuration errors before any probe or load is sent. https://github.com/fedify-dev/fedify/issues/744 https://github.com/fedify-dev/fedify/issues/785 Assisted-by: Codex:gpt-5.5 --- packages/cli/src/bench/action.test.ts | 71 +++++++++++++++++++ packages/cli/src/bench/action.ts | 5 ++ .../cli/src/bench/scenarios/object.test.ts | 32 +++++++++ packages/cli/src/bench/scenarios/object.ts | 31 ++++++++ 4 files changed, 139 insertions(+) diff --git a/packages/cli/src/bench/action.test.ts b/packages/cli/src/bench/action.test.ts index 813d5ad7e..b61a5a7f1 100644 --- a/packages/cli/src/bench/action.test.ts +++ b/packages/cli/src/bench/action.test.ts @@ -208,6 +208,48 @@ scenarios: } }); +test("runBench - dry run gates object discovery before fetching", async () => { + const file = await writeSuite(`version: 1 +target: http://127.0.0.1:3000 +scenarios: + - name: object-crawl + type: object + source: + seed: "https://public.example/users/alice" + collection: outbox + limit: 1 + load: { concurrency: 1 } + duration: 1ms +`); + let code = -1; + let output = ""; + let publicFetched = false; + await runBench(command({ scenario: file, dryRun: true }), { + exit: (c) => { + code = c; + }, + writeOutput: (c) => { + output = c; + return Promise.resolve(); + }, + log: () => {}, + resolveTargetAddresses: resolvePublicHost, + fetch: (input) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.hostname === "public.example") { + publicFetched = true; + throw new Error("dry-run fetched an unsafe object source"); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }, + }); + + assert.strictEqual(code, 0); + assert.strictEqual(publicFetched, false); + assert.match(output, /object discovery failed/); + assert.match(output, /Refusing to send benchmark read load/); +}); + test("runBench - unsafe override requires an explicit CLI target", async () => { const file = await writeSuite(`version: 1 target: https://example.com @@ -659,6 +701,35 @@ scenarios: assert.strictEqual(fetched, false); }); +test("runBench - invalid object source exits 2 before any probe", async () => { + const file = await writeSuite(`version: 1 +target: http://localhost:3000 +scenarios: + - name: object + type: object + source: objects/1 +`); + let code = -1; + let message = ""; + let fetched = false; + await runBench(command({ scenario: file }), { + exit: (c) => { + code = c; + }, + writeOutput: () => Promise.resolve(), + log: (m) => { + message = m; + }, + fetch: () => { + fetched = true; + return Promise.reject(new Error("no request should be sent")); + }, + }); + assert.strictEqual(code, 2); + assert.match(message, /invalid object source URL/); + assert.strictEqual(fetched, false); +}); + test("runBench - invalid suite exits 2", async () => { const file = await writeSuite(`target: http://localhost:3000 scenarios: diff --git a/packages/cli/src/bench/action.ts b/packages/cli/src/bench/action.ts index b60baf764..473c8f23e 100644 --- a/packages/cli/src/bench/action.ts +++ b/packages/cli/src/bench/action.ts @@ -231,6 +231,7 @@ export default async function runBench( documentLoader, contextLoader, allowPrivateAddress, + fetch: fetchImpl, assertDestinationAllowed, assertReadDestinationAllowed, }), @@ -385,6 +386,7 @@ interface DryRunPlanContext { readonly documentLoader: DocumentLoader; readonly contextLoader: DocumentLoader; readonly allowPrivateAddress: boolean; + readonly fetch: typeof fetch; readonly assertDestinationAllowed: ( url: URL, scenario: ResolvedScenario, @@ -532,6 +534,9 @@ async function describeObjectPlan( const urls = await objectUrlsFromSource({ source: scenario.source, target: suite.target, + fetch: context.fetch, + assertReadDestinationAllowed: (url) => + context.assertReadDestinationAllowed(url, scenario), }); const lines = [` objects: ${urls.length} URL(s) resolved`]; for (const url of urls.slice(0, 10)) { diff --git a/packages/cli/src/bench/scenarios/object.test.ts b/packages/cli/src/bench/scenarios/object.test.ts index 0143a28f1..bb9a37288 100644 --- a/packages/cli/src/bench/scenarios/object.test.ts +++ b/packages/cli/src/bench/scenarios/object.test.ts @@ -267,6 +267,38 @@ test("objectRunner - gates collection URLs before crawling them", async () => { ); }); +test("objectRunner.validate - rejects malformed object source URLs", () => { + const explicit = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "object", + type: "object", + source: "objects/1", + }], + }).scenarios[0]; + assert.throws( + () => objectRunner.validate?.(explicit), + /invalid object source URL/, + ); + + const crawl = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "object-crawl", + type: "object", + source: { + seed: ["http://target.test/users/alice", "users/bob"], + }, + }], + }).scenarios[0]; + assert.throws( + () => objectRunner.validate?.(crawl), + /invalid object source seed URL/, + ); +}); + function json(body: unknown): Response { return new Response(JSON.stringify(body), { headers: { "content-type": "application/activity+json" }, diff --git a/packages/cli/src/bench/scenarios/object.ts b/packages/cli/src/bench/scenarios/object.ts index 0f1c12831..b482c6cef 100644 --- a/packages/cli/src/bench/scenarios/object.ts +++ b/packages/cli/src/bench/scenarios/object.ts @@ -4,13 +4,44 @@ * @module */ +import { convertUrlIfHandle } from "../../webfinger/lib.ts"; +import { asList } from "../scenario/coerce.ts"; import { objectUrlsFromSource } from "./object-discovery.ts"; import { runReadLoad } from "./read.ts"; import type { RunContext, ScenarioRunner } from "./runner.ts"; /** The `object` scenario runner. */ export const objectRunner: ScenarioRunner = { + validate(scenario): void { + const { source } = scenario; + if (source == null) return; + if (typeof source === "string" || Array.isArray(source)) { + for (const url of asList(source)) { + try { + new URL(url); + } catch { + throw new Error( + `Scenario "${scenario.name}": invalid object source URL ` + + `${JSON.stringify(url)}.`, + ); + } + } + return; + } + for (const seed of asList(source.seed)) { + try { + convertUrlIfHandle(seed); + } catch { + throw new Error( + `Scenario "${scenario.name}": invalid object source seed URL ` + + `${JSON.stringify(seed)}.`, + ); + } + } + }, + async run(context: RunContext) { + this.validate?.(context.scenario); const urls = await objectUrlsFromSource({ source: context.scenario.source, target: context.target, From f867e34a2f7abf17207c82c618099c57727991af Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Tue, 9 Jun 2026 23:23:55 +0900 Subject: [PATCH 09/61] Reuse dry-run fetch for actor discovery Pass the configured dry-run fetch into actor handle discovery so acct: actor plans use the same injected fetch and User-Agent wrapper as real actor runs. This prevents dry-run planning from falling back to the global network fetch. https://github.com/fedify-dev/fedify/issues/744 https://github.com/fedify-dev/fedify/issues/785 Assisted-by: Codex:gpt-5.5 --- packages/cli/src/bench/action.test.ts | 56 +++++++++++++++++++++++++++ packages/cli/src/bench/action.ts | 1 + 2 files changed, 57 insertions(+) diff --git a/packages/cli/src/bench/action.test.ts b/packages/cli/src/bench/action.test.ts index b61a5a7f1..8d4613d1e 100644 --- a/packages/cli/src/bench/action.test.ts +++ b/packages/cli/src/bench/action.test.ts @@ -208,6 +208,62 @@ scenarios: } }); +test("runBench - dry run resolves actor handles with configured fetch", async () => { + const file = await writeSuite(`version: 1 +target: http://127.0.0.1:3000 +scenarios: + - name: actor-read + type: actor + recipient: "acct:alice@example.test" + load: { concurrency: 1 } + duration: 1ms +`); + let code = -1; + let output = ""; + let webfingerFetched = false; + let webfingerUserAgent: string | null = null; + await runBench(command({ scenario: file, dryRun: true }), { + exit: (c) => { + code = c; + }, + writeOutput: (c) => { + output = c; + return Promise.resolve(); + }, + log: () => {}, + fetch: (input, init) => { + const url = new URL(input instanceof Request ? input.url : input); + const headers = new Headers( + init?.headers ?? (input instanceof Request ? input.headers : undefined), + ); + if (url.pathname === "/.well-known/webfinger") { + webfingerFetched = true; + webfingerUserAgent = headers.get("user-agent"); + return Promise.resolve( + new Response( + JSON.stringify({ + subject: url.searchParams.get("resource"), + links: [{ + rel: "self", + href: "http://127.0.0.1:3000/users/alice", + }], + }), + { + headers: { "content-type": "application/jrd+json" }, + }, + ), + ); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }, + }); + + assert.strictEqual(code, 0); + assert.strictEqual(webfingerFetched, true); + assert.strictEqual(webfingerUserAgent, "Fedify-bench-test/1.0"); + assert.match(output, /\/users\/alice/); +}); + test("runBench - dry run gates object discovery before fetching", async () => { const file = await writeSuite(`version: 1 target: http://127.0.0.1:3000 diff --git a/packages/cli/src/bench/action.ts b/packages/cli/src/bench/action.ts index 473c8f23e..85224c5bf 100644 --- a/packages/cli/src/bench/action.ts +++ b/packages/cli/src/bench/action.ts @@ -507,6 +507,7 @@ async function describeActorPlan( try { const urls = await actorUrlsFromRecipients(scenario.recipients, { target: suite.target, + fetch: context.fetch, }); const lines: string[] = []; for (const url of urls) { From 61a43c9491ccd802cfa30e0ce6be2a14f5bf826c Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Tue, 9 Jun 2026 23:57:40 +0900 Subject: [PATCH 10/61] Keep read and failure measurements focused Route authenticated actor and object reads through the configured signing pipeline so presign and pipeline modes do not silently fall back to inline JIT signing during the measured send path. Resolve and gate inbound failure scenario inboxes before starting the load loop, then reuse the discovered delivery target for each generated failure request. This keeps repeated WebFinger and actor discovery out of the measured samples. https://github.com/fedify-dev/fedify/issues/744 https://github.com/fedify-dev/fedify/issues/785 Assisted-by: Codex:gpt-5.5 --- .../cli/src/bench/scenarios/failure.test.ts | 87 +++++++++++++++++++ packages/cli/src/bench/scenarios/failure.ts | 80 +++++++++++++---- packages/cli/src/bench/scenarios/read.test.ts | 63 ++++++++++++++ packages/cli/src/bench/scenarios/read.ts | 73 +++++++++++----- 4 files changed, 266 insertions(+), 37 deletions(-) diff --git a/packages/cli/src/bench/scenarios/failure.test.ts b/packages/cli/src/bench/scenarios/failure.test.ts index 375849b21..8cec4414b 100644 --- a/packages/cli/src/bench/scenarios/failure.test.ts +++ b/packages/cli/src/bench/scenarios/failure.test.ts @@ -1,8 +1,11 @@ import assert from "node:assert/strict"; import test from "node:test"; +import { serve } from "srvx"; +import { buildFleet } from "../actor/fleet.ts"; import { getContextLoader, getDocumentLoader } from "../../docloader.ts"; import { normalizeSuite } from "../scenario/normalize.ts"; import type { Suite } from "../scenario/types.ts"; +import { spawnSyntheticServer } from "../server/synthetic.ts"; import { failureRunner } from "./failure.ts"; test("failureRunner - counts expected remote failure as success", async () => { @@ -45,3 +48,87 @@ test("failureRunner.validate - rejects unsupported faults", () => { assert.throws(() => failureRunner.validate?.(scenario), /unsupported/); }); + +test("failureRunner - discovers inbound failure inboxes once", async () => { + let actorGets = 0; + const server = serve({ + port: 0, + hostname: "127.0.0.1", + silent: true, + fetch(request: Request): Response { + const url = new URL(request.url); + if (url.pathname === "/users/alice") { + actorGets++; + return json({ + "@context": "https://www.w3.org/ns/activitystreams", + type: "Person", + id: url.href, + inbox: new URL("/users/alice/inbox", url).href, + endpoints: { + sharedInbox: new URL("/inbox", url).href, + }, + }); + } + return new Response("not found", { status: 404 }); + }, + }); + await server.ready(); + let fleet: Awaited> | undefined; + try { + const target = new URL(server.url!); + fleet = await spawnSyntheticServer( + await buildFleet([{ + count: 1, + signatureStandards: ["draft-cavage-http-signatures-12"], + }]), + ); + const scenario = normalizeSuite({ + version: 1, + target: target.href, + scenarios: [{ + name: "bad-signature", + type: "failure", + fault: "invalid-signature", + recipient: new URL("/users/alice", target).href, + load: { concurrency: 1 }, + duration: "80ms", + }], + }).scenarios[0]; + const measurement = await failureRunner.run({ + scenario, + target, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet, + fetch: (input) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/inbox") { + return Promise.resolve( + new Response("bad signature", { + status: 401, + }), + ); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }, + assertDestinationAllowed: () => {}, + }); + + assert.ok(measurement.requests.total > 1); + assert.strictEqual(measurement.requests.successRate, 1); + assert.strictEqual(actorGets, 1); + } finally { + try { + await fleet?.close(); + } finally { + await server.close(true); + } + } +}); + +function json(body: unknown): Response { + return new Response(JSON.stringify(body), { + headers: { "content-type": "application/activity+json" }, + }); +} diff --git a/packages/cli/src/bench/scenarios/failure.ts b/packages/cli/src/bench/scenarios/failure.ts index 28243e556..0459dcdab 100644 --- a/packages/cli/src/bench/scenarios/failure.ts +++ b/packages/cli/src/bench/scenarios/failure.ts @@ -30,6 +30,11 @@ const SUPPORTED_FAULTS = [ type SupportedFault = typeof SUPPORTED_FAULTS[number]; +interface FailureDeliveryTarget { + readonly inbox: URL; + readonly actorUri: URL; +} + /** The `failure` scenario runner. */ export const failureRunner: ScenarioRunner = { validate(scenario): void { @@ -58,8 +63,10 @@ export const failureRunner: ScenarioRunner = { async run(context: RunContext) { this.validate?.(context.scenario); const faults = faultsOf(context); + const deliveryTarget = await resolveFailureDeliveryTarget(context, faults); let index = 0; - const send = () => sendForFault(context, faults[index++ % faults.length]); + const send = () => + sendForFault(context, faults[index++ % faults.length], deliveryTarget); const result = await runLoad( loadPlanOf(context.scenario, context.rng), send, @@ -89,15 +96,39 @@ function isSupportedFault(fault: string): fault is SupportedFault { return SUPPORTED_FAULTS.includes(fault as SupportedFault); } +async function resolveFailureDeliveryTarget( + context: RunContext, + faults: readonly SupportedFault[], +): Promise { + if (!faults.some(isInboundFault)) return null; + const { scenario } = context; + const discovered = await discoverInbox(scenario.recipients[0], { + documentLoader: context.documentLoader, + contextLoader: context.contextLoader, + allowPrivateAddress: context.allowPrivateAddress, + }); + const inbox = selectInbox(discovered, scenario.inbox); + await context.assertDestinationAllowed?.(inbox); + return { inbox, actorUri: discovered.actorUri }; +} + +function isInboundFault(fault: SupportedFault): boolean { + return fault === "invalid-signature" || fault === "missing-actor"; +} + async function sendForFault( context: RunContext, fault: SupportedFault, + deliveryTarget: FailureDeliveryTarget | null, ): Promise { switch (fault) { case "invalid-signature": - return await sendInvalidSignature(context); + return await sendInvalidSignature( + context, + requiredTarget(deliveryTarget), + ); case "missing-actor": - return await sendMissingActor(context); + return await sendMissingActor(context, requiredTarget(deliveryTarget)); case "remote-404": return { ok: true, status: 404 }; case "remote-410": @@ -116,8 +147,13 @@ async function sendForFault( async function sendInvalidSignature( context: RunContext, + deliveryTarget: FailureDeliveryTarget, ): Promise { - const request = await signedFailureRequest(context, "invalid-signature"); + const request = await signedFailureRequest( + context, + "invalid-signature", + deliveryTarget, + ); const body = new Uint8Array(await request.arrayBuffer()); const corrupted = new Uint8Array(body.length + 1); corrupted.set(body); @@ -136,14 +172,22 @@ async function sendInvalidSignature( ); } -async function sendMissingActor(context: RunContext): Promise { - const request = await signedFailureRequest(context, "missing-actor"); +async function sendMissingActor( + context: RunContext, + deliveryTarget: FailureDeliveryTarget, +): Promise { + const request = await signedFailureRequest( + context, + "missing-actor", + deliveryTarget, + ); return expectedFailure(await sendRequest(request, context.fetch ?? fetch)); } async function signedFailureRequest( context: RunContext, fault: "invalid-signature" | "missing-actor", + deliveryTarget: FailureDeliveryTarget, ): Promise { const { fleet, scenario } = context; if (fleet == null || fleet.actors.length < 1) { @@ -159,34 +203,38 @@ async function signedFailureRequest( const actor = fault === "missing-actor" ? missingActor(fleet.actors[0], context.target) : fleet.actors[0]; - const discovered = await discoverInbox(scenario.recipients[0], { - documentLoader: context.documentLoader, - contextLoader: context.contextLoader, - allowPrivateAddress: context.allowPrivateAddress, - }); - const inbox = selectInbox(discovered, scenario.inbox); - await context.assertDestinationAllowed?.(inbox); const id = createActivityIdMinter(fleet.url).next(); const note = new Note({ id: new URL(`/objects/${crypto.randomUUID()}`, fleet.url), attribution: actor.id, content: "Benchmark failure activity.", - to: discovered.actorUri, + to: deliveryTarget.actorUri, }); const activity = new Create({ id, actor: actor.id, object: note, - to: discovered.actorUri, + to: deliveryTarget.actorUri, }); return await signInboxDelivery({ actor, - inbox, + inbox: deliveryTarget.inbox, activity, contextLoader: context.contextLoader, }); } +function requiredTarget( + target: FailureDeliveryTarget | null, +): FailureDeliveryTarget { + if (target == null) { + throw new Error( + "The invalid-signature and missing-actor faults require discovery.", + ); + } + return target; +} + function missingActor(actor: SyntheticActor, target: URL): SyntheticActor { const id = new URL(`/__fedify_bench/missing/${crypto.randomUUID()}`, target); return { diff --git a/packages/cli/src/bench/scenarios/read.test.ts b/packages/cli/src/bench/scenarios/read.test.ts index 11edcf473..3b12b548a 100644 --- a/packages/cli/src/bench/scenarios/read.test.ts +++ b/packages/cli/src/bench/scenarios/read.test.ts @@ -1,7 +1,9 @@ import assert from "node:assert/strict"; import test from "node:test"; +import { buildFleet } from "../actor/fleet.ts"; import { getContextLoader, getDocumentLoader } from "../../docloader.ts"; import { normalizeSuite } from "../scenario/normalize.ts"; +import { spawnSyntheticServer } from "../server/synthetic.ts"; import { runReadLoad } from "./read.ts"; test("runReadLoad - unauthenticated reads use the read destination gate", async () => { @@ -46,3 +48,64 @@ test("runReadLoad - unauthenticated reads use the read destination gate", async assert.ok(measurement.requests.total > 0); assert.strictEqual(measurement.requests.successRate, 1); }); + +test("runReadLoad - authenticated reads support presign mode", async () => { + let fleet: Awaited> | undefined; + try { + fleet = await spawnSyntheticServer( + await buildFleet([{ + count: 1, + signatureStandards: ["draft-cavage-http-signatures-12"], + }]), + ); + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "read", + type: "actor", + authenticated: true, + signing: "presign", + load: { rate: "20/s" }, + duration: "50ms", + }], + }).scenarios[0]; + let signedGets = 0; + + const measurement = await runReadLoad({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet, + fetch: (input) => { + const request = input instanceof Request ? input : new Request(input); + const url = new URL(request.url); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return Promise.resolve(new Response("not found", { status: 404 })); + } + if ( + request.headers.has("authorization") || + request.headers.has("signature") + ) { + signedGets++; + } + return Promise.resolve(new Response("{}", { status: 200 })); + }, + assertDestinationAllowed: () => {}, + assertReadDestinationAllowed: () => { + throw new Error("authenticated reads should use the signed gate"); + }, + }, { + urls: [new URL("http://remote.test/users/alice")], + authenticated: true, + }); + + assert.ok(measurement.requests.total > 0); + assert.strictEqual(measurement.requests.successRate, 1); + assert.ok(signedGets > 0); + } finally { + await fleet?.close(); + } +}); diff --git a/packages/cli/src/bench/scenarios/read.ts b/packages/cli/src/bench/scenarios/read.ts index 97733d20d..8e9bee2d0 100644 --- a/packages/cli/src/bench/scenarios/read.ts +++ b/packages/cli/src/bench/scenarios/read.ts @@ -15,6 +15,11 @@ import { } from "../metrics/stats-client.ts"; import type { SyntheticActor } from "../server/synthetic.ts"; import { + createSigningPipeline, + type SigningPipeline, +} from "../signing/pipeline.ts"; +import { + estimateTotal, loadPlanOf, measuredWindowMs, type RunContext, @@ -61,16 +66,37 @@ export async function runReadLoad( } } - let index = 0; - const rawSend = async () => { - const i = index++; - const url = options.urls[i % options.urls.length]; - let request = new Request(url, { + function unsignedRequest(index: number): Request { + const url = options.urls[index % options.urls.length]; + return new Request(url, { headers: { accept: "application/activity+json, application/ld+json" }, redirect: "manual", }); - if (options.authenticated) { - request = await signGetRequest(request, actors[i % actors.length]); + } + + let pipeline: SigningPipeline | null = null; + if (options.authenticated) { + let signIndex = 0; + pipeline = createSigningPipeline(context.scenario.signing, async () => { + const i = signIndex++; + return await signGetRequest( + unsignedRequest(i), + actors[i % actors.length], + ); + }, { total: estimateTotal(context.scenario) }); + } + + let index = 0; + const rawSend = async () => { + let request: Request; + if (pipeline != null) { + try { + request = await pipeline.next(); + } catch (error) { + return { ok: false, errorKind: "client", reason: String(error) }; + } + } else { + request = unsignedRequest(index++); } return await sendRequest(request, fetchImpl); }; @@ -85,20 +111,25 @@ export async function runReadLoad( }, rawSend, ); - const result = await runLoad( - loadPlanOf(context.scenario, context.rng), - send, - context.clock, - ); - const measurement = aggregateSamples(result.samples, { - measuredWindowMs: measuredWindowMs(context.scenario), - includeHistogram: true, - }); - const end = await fetchServerSnapshot(context.target, fetchImpl); - const server = baselineTaken && baseline != null && end != null - ? snapshotToMetrics(diffSnapshots(baseline, end)) - : null; - return { ...measurement, server }; + try { + await pipeline?.prime(); + const result = await runLoad( + loadPlanOf(context.scenario, context.rng), + send, + context.clock, + ); + const measurement = aggregateSamples(result.samples, { + measuredWindowMs: measuredWindowMs(context.scenario), + includeHistogram: true, + }); + const end = await fetchServerSnapshot(context.target, fetchImpl); + const server = baselineTaken && baseline != null && end != null + ? snapshotToMetrics(diffSnapshots(baseline, end)) + : null; + return { ...measurement, server }; + } finally { + await pipeline?.close(); + } } async function signGetRequest( From 207bb87135ff45a66b9609881bcc166db5e20090 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Wed, 10 Jun 2026 00:28:30 +0900 Subject: [PATCH 11/61] Fix fanout and failure gates Treat drained fanout queue failures as failed benchmark samples so failed sink deliveries cannot inflate success rate or delivery throughput. Allow missing-actor-only failure scenarios to run against non-loopback targets without --advertise-host, since the missing actor identity is placed under the benchmark target rather than the synthetic actor server. https://github.com/fedify-dev/fedify/issues/785 Assisted-by: Codex:gpt-5.5 --- packages/cli/src/bench/action.test.ts | 45 ++++++++++++++++++ packages/cli/src/bench/action.ts | 3 ++ .../cli/src/bench/scenarios/fanout.test.ts | 47 +++++++++++++++++++ packages/cli/src/bench/scenarios/fanout.ts | 14 +++++- 4 files changed, 107 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/bench/action.test.ts b/packages/cli/src/bench/action.test.ts index 8d4613d1e..9f94fe6cc 100644 --- a/packages/cli/src/bench/action.test.ts +++ b/packages/cli/src/bench/action.test.ts @@ -449,6 +449,51 @@ scenarios: assert.strictEqual(JSON.parse(output).scenarios[0].requests.successRate, 1); }); +test("runBench - missing-actor failure needs no advertise host", async () => { + const recipientTarget = await spawnBenchmarkTarget(); + try { + const file = await writeSuite(`version: 1 +target: http://10.10.0.5:8000 +scenarios: + - name: missing-actor + type: failure + fault: missing-actor + recipient: "${new URL("/users/alice", recipientTarget.url).href}" + load: { rate: 1/s } + duration: 1ms +`); + let code = -1; + let message = ""; + await runBench(command({ scenario: file }), { + exit: (c) => { + code = c; + }, + writeOutput: () => Promise.resolve(), + log: (m) => { + message = m; + }, + fetch: (input) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return Promise.resolve(new Response("not found", { status: 404 })); + } + if ( + url.origin === recipientTarget.url.origin && + url.pathname === "/inbox" + ) { + return Promise.resolve( + new Response("actor not found", { status: 401 }), + ); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }, + }); + assert.strictEqual(code, 0, message); + } finally { + await recipientTarget.close(); + } +}); + test("runBench - unauthenticated actor read needs no advertise host", async () => { const file = await writeSuite(`version: 1 target: http://10.10.0.5:8000 diff --git a/packages/cli/src/bench/action.ts b/packages/cli/src/bench/action.ts index 85224c5bf..778ea1356 100644 --- a/packages/cli/src/bench/action.ts +++ b/packages/cli/src/bench/action.ts @@ -680,6 +680,9 @@ function scenarioNeedsReachableLocalServer( seen: ReadonlySet = new Set(), ): boolean { if (scenario.type === "fanout") return true; + if (scenario.type === "failure") { + return scenario.faults.includes("invalid-signature"); + } if (scenario.type === "mixed") { if (seen.has(scenario.name)) return false; const nextSeen = new Set(seen).add(scenario.name); diff --git a/packages/cli/src/bench/scenarios/fanout.test.ts b/packages/cli/src/bench/scenarios/fanout.test.ts index aa391b74d..f80c55fd4 100644 --- a/packages/cli/src/bench/scenarios/fanout.test.ts +++ b/packages/cli/src/bench/scenarios/fanout.test.ts @@ -123,6 +123,53 @@ test("fanoutRunner - serializes overlapping trigger drains", async () => { assert.strictEqual(maxActiveTriggers, 1); }); +test("fanoutRunner - counts failed queue tasks as delivery failures", async () => { + const target = new URL("http://target.test/"); + let statsCalls = 0; + const suite: Suite = { + version: 1, + target: target.href, + scenarios: [{ + name: "fanout", + type: "fanout", + sender: "alice", + followers: 5, + load: { concurrency: 1 }, + duration: "40ms", + queueDrainTimeout: "1s", + }], + }; + const scenario = normalizeSuite(suite).scenarios[0]; + const measurement = await fanoutRunner.run({ + scenario, + target, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/.well-known/fedify/bench/stats") { + statsCalls++; + const failedTasks = Math.floor(statsCalls / 2) * 6; + return Promise.resolve(json(statsSnapshot({ + enqueued: failedTasks, + completed: 0, + failed: failedTasks, + }))); + } + if (url.pathname === "/.well-known/fedify/bench/trigger") { + return Promise.resolve(json({ version: 1 }, 202)); + } + return Promise.resolve(new Response("unexpected", { status: 500 })); + }, + }); + + assert.ok(measurement.requests.total > 0); + assert.strictEqual(measurement.requests.successRate, 0); + assert.strictEqual(measurement.throughputPerSec, 0); +}); + function json(body: unknown, status = 200): Response { return new Response(JSON.stringify(body), { status, diff --git a/packages/cli/src/bench/scenarios/fanout.ts b/packages/cli/src/bench/scenarios/fanout.ts index d2e61c96f..7c4abf401 100644 --- a/packages/cli/src/bench/scenarios/fanout.ts +++ b/packages/cli/src/bench/scenarios/fanout.ts @@ -110,6 +110,13 @@ export const fanoutRunner: ScenarioRunner = { reason: "queue_drain_timeout", }; } + if (drain.failed > 0) { + return { + ok: false, + errorKind: "server", + reason: "queue_delivery_failed", + }; + } if (scheduledAtMs >= context.scenario.warmupMs) { drainHistogram.record(Date.now() - started); delivered += sink.recipients.length; @@ -234,6 +241,7 @@ function parseSinkBehavior( interface DrainResult { readonly timedOut: boolean; + readonly failed: number; } async function waitForDrain(options: { @@ -248,14 +256,16 @@ async function waitForDrain(options: { const snapshot = await fetchServerSnapshot(options.target, options.fetch); if (snapshot == null) return null; const diff = diffSnapshots(options.baseline, snapshot); + const queueTasks = diff.queueTasks; + if (queueTasks == null) return null; const remaining = queueTaskRemaining(diff); if (remaining == null) return null; if (remaining === 0) { - return { timedOut: false }; + return { timedOut: false, failed: queueTasks.failed }; } await new Promise((resolve) => setTimeout(resolve, DRAIN_POLL_MS)); } while (Date.now() < deadline); - return { timedOut: true }; + return { timedOut: true, failed: 0 }; } function addQueueDrain( From 682c6352f0e92b735e81b54e17156e6c4478cf53 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Wed, 10 Jun 2026 01:07:21 +0900 Subject: [PATCH 12/61] Separate actorless and delivery gates Route missing-actor failure destinations through the gate that does not require a reachable synthetic actor server, while still applying public load safety checks. Track fanout delivery throughput separately from request throughput so mixed scenarios cannot satisfy deliveryThroughput expectations with read traffic from other children. https://github.com/fedify-dev/fedify/issues/785 Assisted-by: Codex:gpt-5.5 --- packages/cli/src/bench/action.test.ts | 67 +++++++++++++++++++ packages/cli/src/bench/action.ts | 25 ++++++- packages/cli/src/bench/result/build.ts | 1 + .../src/bench/result/expect/evaluate.test.ts | 12 +++- .../cli/src/bench/result/expect/evaluate.ts | 14 ++-- packages/cli/src/bench/scenarios/failure.ts | 6 +- packages/cli/src/bench/scenarios/fanout.ts | 6 +- .../cli/src/bench/scenarios/mixed.test.ts | 26 ++++++- packages/cli/src/bench/scenarios/mixed.ts | 9 +++ packages/cli/src/bench/scenarios/runner.ts | 7 ++ 10 files changed, 161 insertions(+), 12 deletions(-) diff --git a/packages/cli/src/bench/action.test.ts b/packages/cli/src/bench/action.test.ts index 9f94fe6cc..14ad6eb3a 100644 --- a/packages/cli/src/bench/action.test.ts +++ b/packages/cli/src/bench/action.test.ts @@ -3,6 +3,7 @@ import { mkdtemp, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; +import { serve } from "srvx"; import { spawnBenchmarkTarget } from "../../test/bench/fixture.ts"; import runBench, { withUserAgent } from "./action.ts"; import type { BenchCommand } from "./command.ts"; @@ -494,6 +495,72 @@ scenarios: } }); +test("runBench - missing-actor failure allows private inbox without advertise host", async () => { + const actorServer = serve({ + port: 0, + hostname: "127.0.0.1", + silent: true, + fetch(request: Request): Response { + const url = new URL(request.url); + if (url.pathname === "/users/alice") { + return new Response( + JSON.stringify({ + "@context": "https://www.w3.org/ns/activitystreams", + type: "Person", + id: url.href, + inbox: "http://10.0.0.8/inbox", + }), + { headers: { "content-type": "application/activity+json" } }, + ); + } + return new Response("not found", { status: 404 }); + }, + }); + await actorServer.ready(); + try { + const actorUrl = new URL("/users/alice", new URL(actorServer.url!)); + const file = await writeSuite(`version: 1 +target: http://10.10.0.5:8000 +scenarios: + - name: missing-actor + type: failure + fault: missing-actor + recipient: "${actorUrl.href}" + load: { rate: 1/s } + duration: 1ms +`); + let code = -1; + let message = ""; + let inboxPosts = 0; + await runBench(command({ scenario: file }), { + exit: (c) => { + code = c; + }, + writeOutput: () => Promise.resolve(), + log: (m) => { + message = m; + }, + fetch: (input) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return Promise.resolve(new Response("not found", { status: 404 })); + } + if (url.href === "http://10.0.0.8/inbox") { + inboxPosts++; + return Promise.resolve( + new Response("actor not found", { status: 401 }), + ); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }, + }); + assert.strictEqual(code, 0, message); + assert.ok(inboxPosts > 0); + } finally { + await actorServer.close(true); + } +}); + test("runBench - unauthenticated actor read needs no advertise host", async () => { const file = await writeSuite(`version: 1 target: http://10.10.0.5:8000 diff --git a/packages/cli/src/bench/action.ts b/packages/cli/src/bench/action.ts index 778ea1356..69b933690 100644 --- a/packages/cli/src/bench/action.ts +++ b/packages/cli/src/bench/action.ts @@ -195,9 +195,10 @@ export default async function runBench( defaults: validated.defaults, }); }; - const assertReadDestinationAllowed = async ( + const assertDestinationWithoutSyntheticServerAllowed = async ( url: URL, scenario: ResolvedScenario, + loadDescription: string, ): Promise => { const sameOrigin = url.origin === suite.target.origin; const destinationTier = sameOrigin @@ -209,7 +210,7 @@ export default async function runBench( !command.allowUnsafeTarget ) { throw new UnsafeTargetError( - `Refusing to send benchmark read load to ${url.href}: it is public ` + + `Refusing to send ${loadDescription} to ${url.href}: it is public ` + "and not part of the benchmarked target. Pass " + "--allow-unsafe-target to override.", ); @@ -223,6 +224,24 @@ export default async function runBench( defaults: validated.defaults, }); }; + const assertReadDestinationAllowed = ( + url: URL, + scenario: ResolvedScenario, + ): Promise => + assertDestinationWithoutSyntheticServerAllowed( + url, + scenario, + "benchmark read load", + ); + const assertActorlessDestinationAllowed = ( + url: URL, + scenario: ResolvedScenario, + ): Promise => + assertDestinationWithoutSyntheticServerAllowed( + url, + scenario, + "benchmark load", + ); if (command.dryRun) { try { @@ -295,6 +314,8 @@ export default async function runBench( assertDestinationAllowed(url, scenario), assertReadDestinationAllowed: (url) => assertReadDestinationAllowed(url, scenario), + assertActorlessDestinationAllowed: (url) => + assertActorlessDestinationAllowed(url, scenario), }); results.push(buildScenarioResult(scenario, measurement)); } diff --git a/packages/cli/src/bench/result/build.ts b/packages/cli/src/bench/result/build.ts index 37f9d0eed..dd0799b37 100644 --- a/packages/cli/src/bench/result/build.ts +++ b/packages/cli/src/bench/result/build.ts @@ -32,6 +32,7 @@ import type { export interface ScenarioMeasurement { readonly requests: RequestSummary; readonly throughputPerSec: number; + readonly deliveryThroughputPerSec?: number; readonly client: ClientMetrics; readonly server: ServerMetrics | null; readonly errors: ErrorBucket[]; diff --git a/packages/cli/src/bench/result/expect/evaluate.test.ts b/packages/cli/src/bench/result/expect/evaluate.test.ts index 10ff6e07c..b28a8eabc 100644 --- a/packages/cli/src/bench/result/expect/evaluate.test.ts +++ b/packages/cli/src/bench/result/expect/evaluate.test.ts @@ -87,13 +87,23 @@ test("evaluateExpect - missing server metric fails (actual null)", () => { test("evaluateExpect - reads delivery throughput", () => { const { passed, results } = evaluateExpect( { deliveryThroughput: ">= 1/s" }, - metrics({ throughputPerSec: 12 }), + metrics({ deliveryThroughputPerSec: 12 }), ); assert.strictEqual(passed, true); assert.strictEqual(results[0].actual, 12); assert.strictEqual(results[0].pass, true); }); +test("evaluateExpect - does not alias delivery throughput to request throughput", () => { + const { passed, results } = evaluateExpect( + { deliveryThroughput: ">= 1/s" }, + metrics({ throughputPerSec: 12 }), + ); + assert.strictEqual(passed, false); + assert.strictEqual(results[0].actual, null); + assert.strictEqual(results[0].pass, false); +}); + test("evaluateExpect - tolerant equality matches float-normalized ratios", () => { const { passed } = evaluateExpect( { successRate: "== 99.4%" }, diff --git a/packages/cli/src/bench/result/expect/evaluate.ts b/packages/cli/src/bench/result/expect/evaluate.ts index 3b5c52a80..174137859 100644 --- a/packages/cli/src/bench/result/expect/evaluate.ts +++ b/packages/cli/src/bench/result/expect/evaluate.ts @@ -50,10 +50,14 @@ export function validateExpectBlock(expect: ExpectBlock): void { } /** The subset of a scenario result that `expect` metrics are looked up from. */ -export type MetricView = Pick< - ScenarioResult, - "requests" | "throughputPerSec" | "client" | "server" | "errors" ->; +export interface MetricView { + readonly requests: ScenarioResult["requests"]; + readonly throughputPerSec: ScenarioResult["throughputPerSec"]; + readonly deliveryThroughputPerSec?: number; + readonly client: ScenarioResult["client"]; + readonly server: ScenarioResult["server"]; + readonly errors: ScenarioResult["errors"]; +} /** The outcome of evaluating an `expect` block. */ export interface ExpectEvaluation { @@ -133,7 +137,7 @@ function lookupValue(metrics: MetricView, metric: string): number | null { case "throughputPerSec": return metrics.throughputPerSec; case "deliveryThroughput": - return metrics.throughputPerSec; + return metrics.deliveryThroughputPerSec ?? null; case "errors.total": return sumErrors(metrics.errors); case "errors.4xx": diff --git a/packages/cli/src/bench/scenarios/failure.ts b/packages/cli/src/bench/scenarios/failure.ts index 0459dcdab..264bc47e3 100644 --- a/packages/cli/src/bench/scenarios/failure.ts +++ b/packages/cli/src/bench/scenarios/failure.ts @@ -108,7 +108,11 @@ async function resolveFailureDeliveryTarget( allowPrivateAddress: context.allowPrivateAddress, }); const inbox = selectInbox(discovered, scenario.inbox); - await context.assertDestinationAllowed?.(inbox); + if (faults.every((fault) => fault === "missing-actor")) { + await context.assertActorlessDestinationAllowed?.(inbox); + } else { + await context.assertDestinationAllowed?.(inbox); + } return { inbox, actorUri: discovered.actorUri }; } diff --git a/packages/cli/src/bench/scenarios/fanout.ts b/packages/cli/src/bench/scenarios/fanout.ts index 7c4abf401..488049e5f 100644 --- a/packages/cli/src/bench/scenarios/fanout.ts +++ b/packages/cli/src/bench/scenarios/fanout.ts @@ -142,10 +142,12 @@ export const fanoutRunner: ScenarioRunner = { includeHistogram: true, }); const server = addQueueDrain(measurement.server, drainHistogram); + const deliveryThroughputPerSec = delivered / + (Math.max(measuredWindowMs(context.scenario), 1) / 1000); return { ...measurement, - throughputPerSec: delivered / - (Math.max(measuredWindowMs(context.scenario), 1) / 1000), + throughputPerSec: deliveryThroughputPerSec, + deliveryThroughputPerSec, server, }; } finally { diff --git a/packages/cli/src/bench/scenarios/mixed.test.ts b/packages/cli/src/bench/scenarios/mixed.test.ts index 8434adda8..9a2b928f9 100644 --- a/packages/cli/src/bench/scenarios/mixed.test.ts +++ b/packages/cli/src/bench/scenarios/mixed.test.ts @@ -3,6 +3,7 @@ import test from "node:test"; import { getContextLoader, getDocumentLoader } from "../../docloader.ts"; import { LogLinearHistogram } from "../metrics/histogram.ts"; import type { ScenarioMeasurement } from "../result/build.ts"; +import { evaluateExpect } from "../result/expect/evaluate.ts"; import { normalizeSuite } from "../scenario/normalize.ts"; import type { Suite } from "../scenario/types.ts"; import { mergeMeasurements, mixedRunner } from "./mixed.ts"; @@ -225,7 +226,29 @@ test("mergeMeasurements - merges latency histograms", () => { assert.strictEqual(measurement.histogram?.count, 100); }); -function fakeMeasurement(samples: readonly number[]): ScenarioMeasurement { +test("mergeMeasurements - keeps delivery throughput separate from reads", () => { + const measurement = mergeMeasurements([ + fakeMeasurement([1], { throughputPerSec: 100 }), + fakeMeasurement([1], { + throughputPerSec: 0, + deliveryThroughputPerSec: 0, + }), + ]); + + assert.strictEqual(measurement.throughputPerSec, 100); + assert.strictEqual(measurement.deliveryThroughputPerSec, 0); + const { passed, results } = evaluateExpect( + { deliveryThroughput: ">= 1/s" }, + measurement, + ); + assert.strictEqual(passed, false); + assert.strictEqual(results[0].actual, 0); +}); + +function fakeMeasurement( + samples: readonly number[], + overrides: Partial = {}, +): ScenarioMeasurement { const histogram = new LogLinearHistogram(); for (const sample of samples) histogram.record(sample); return { @@ -248,6 +271,7 @@ function fakeMeasurement(samples: readonly number[]): ScenarioMeasurement { server: null, errors: [], histogram: histogram.toJSON(), + ...overrides, }; } diff --git a/packages/cli/src/bench/scenarios/mixed.ts b/packages/cli/src/bench/scenarios/mixed.ts index e5cad9867..8e37151c5 100644 --- a/packages/cli/src/bench/scenarios/mixed.ts +++ b/packages/cli/src/bench/scenarios/mixed.ts @@ -261,6 +261,9 @@ export function mergeMeasurements( const total = measurements.reduce((sum, m) => sum + m.requests.total, 0); const ok = measurements.reduce((sum, m) => sum + m.requests.ok, 0); const histogram = mergeHistograms(measurements); + const deliveryThroughputs = measurements + .map((m) => m.deliveryThroughputPerSec) + .filter((value): value is number => value != null); return { requests: { total, @@ -272,6 +275,12 @@ export function mergeMeasurements( (sum, m) => sum + m.throughputPerSec, 0, ), + ...(deliveryThroughputs.length < 1 ? {} : { + deliveryThroughputPerSec: deliveryThroughputs.reduce( + (sum, value) => sum + value, + 0, + ), + }), client: { latencyMs: histogram == null ? mergeLatencyFallback(measurements) diff --git a/packages/cli/src/bench/scenarios/runner.ts b/packages/cli/src/bench/scenarios/runner.ts index 26944e7f3..8d2e83758 100644 --- a/packages/cli/src/bench/scenarios/runner.ts +++ b/packages/cli/src/bench/scenarios/runner.ts @@ -46,6 +46,13 @@ export interface RunContext { * reachable synthetic actor server. */ readonly assertReadDestinationAllowed?: (url: URL) => void | Promise; + /** + * Gates a destination for benchmark load that does not require remote + * dereferencing of benchmark-owned synthetic actors. + */ + readonly assertActorlessDestinationAllowed?: ( + url: URL, + ) => void | Promise; } /** Context available during runner preflight validation. */ From 3af3e3576a689b964656c26322a0d73f8314e2c8 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Wed, 10 Jun 2026 01:25:13 +0900 Subject: [PATCH 13/61] Keep fanout request throughput separate Leave fanout throughputPerSec as the trigger request rate and report recipient delivery rate only through deliveryThroughputPerSec. This keeps fanout reports consistent with request counts and prevents mixed scenarios from folding delivery throughput into ordinary request throughput. https://github.com/fedify-dev/fedify/issues/785 Assisted-by: Codex:gpt-5.5 --- packages/cli/src/bench/scenarios/fanout.test.ts | 10 +++++++++- packages/cli/src/bench/scenarios/fanout.ts | 1 - 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/bench/scenarios/fanout.test.ts b/packages/cli/src/bench/scenarios/fanout.test.ts index f80c55fd4..879f0bad7 100644 --- a/packages/cli/src/bench/scenarios/fanout.test.ts +++ b/packages/cli/src/bench/scenarios/fanout.test.ts @@ -55,6 +55,13 @@ test("fanoutRunner - triggers benchmark hook and reports drain", async () => { assert.strictEqual(measurement.requests.successRate, 1); assert.ok(measurement.server?.queue?.drainMs?.p95 != null); assert.ok(measurement.throughputPerSec > 0); + assert.ok(measurement.deliveryThroughputPerSec != null); + assert.ok( + Math.abs( + measurement.deliveryThroughputPerSec - + measurement.throughputPerSec * triggerRecipients, + ) < 1e-9, + ); assert.strictEqual(triggerRecipients, 5); }); @@ -167,7 +174,8 @@ test("fanoutRunner - counts failed queue tasks as delivery failures", async () = assert.ok(measurement.requests.total > 0); assert.strictEqual(measurement.requests.successRate, 0); - assert.strictEqual(measurement.throughputPerSec, 0); + assert.ok(measurement.throughputPerSec > 0); + assert.strictEqual(measurement.deliveryThroughputPerSec, 0); }); function json(body: unknown, status = 200): Response { diff --git a/packages/cli/src/bench/scenarios/fanout.ts b/packages/cli/src/bench/scenarios/fanout.ts index 488049e5f..4152d0e7e 100644 --- a/packages/cli/src/bench/scenarios/fanout.ts +++ b/packages/cli/src/bench/scenarios/fanout.ts @@ -146,7 +146,6 @@ export const fanoutRunner: ScenarioRunner = { (Math.max(measuredWindowMs(context.scenario), 1) / 1000); return { ...measurement, - throughputPerSec: deliveryThroughputPerSec, deliveryThroughputPerSec, server, }; From 80e0ba38b8f9cbb5ee0045e2e2da727065b5b014 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Wed, 10 Jun 2026 01:50:31 +0900 Subject: [PATCH 14/61] Preserve fanout delivery reports Send ActivityPub Accept headers while crawling actor and collection sources for object benchmarks, keeping WebFinger discovery on a JRD Accept header. Carry deliveryThroughputPerSec into scenario reports, render it in text and Markdown, and publish report schema v2 so the JSON report preserves the new field without mutating the immutable v1 schema. https://github.com/fedify-dev/fedify/issues/785 Assisted-by: Codex:gpt-5.5 --- docs/manual/benchmarking.md | 6 +- .../__fixtures__/reports/inbox-report.json | 4 +- packages/cli/src/bench/render/markdown.ts | 7 + packages/cli/src/bench/render/render.test.ts | 21 +- packages/cli/src/bench/render/text.ts | 7 + packages/cli/src/bench/result/build.test.ts | 12 +- packages/cli/src/bench/result/build.ts | 5 +- packages/cli/src/bench/result/model.ts | 3 +- packages/cli/src/bench/result/schema.ts | 30 +- .../src/bench/scenarios/object-discovery.ts | 13 +- .../cli/src/bench/scenarios/object.test.ts | 59 ++ packages/cli/src/bench/schema.test.ts | 6 +- packages/cli/src/bench/schemas.ts | 7 +- schema/bench/report-v2.json | 523 ++++++++++++++++++ 14 files changed, 685 insertions(+), 18 deletions(-) create mode 100644 schema/bench/report-v2.json diff --git a/docs/manual/benchmarking.md b/docs/manual/benchmarking.md index 8ad0590d3..c879467a4 100644 --- a/docs/manual/benchmarking.md +++ b/docs/manual/benchmarking.md @@ -193,8 +193,8 @@ The runnable scenario types cover the main benchmark surfaces: `mixed` scenario's load by each entry's `weight`. The referenced scenarios are named scenarios in the same suite and are still run as normal suite entries when listed. The mixed result merges client-side request, - throughput, latency, and error measurements; server-side metric snapshots - are not merged across child runners. + throughput, delivery throughput, latency, and error measurements; + server-side metric snapshots are not merged across child runners. ### Actors @@ -272,7 +272,7 @@ CI check. Keep CI gates on robust signals such as success rate, error counts, and gross throughput or latency floors; precise latency-percentile regression belongs in a controlled environment, not a shared CI runner. -[report schema]: https://json-schema.fedify.dev/bench/report-v1.json +[report schema]: https://json-schema.fedify.dev/bench/report-v2.json ### Safety diff --git a/packages/cli/src/bench/__fixtures__/reports/inbox-report.json b/packages/cli/src/bench/__fixtures__/reports/inbox-report.json index b952b1f5a..b7ca535f8 100644 --- a/packages/cli/src/bench/__fixtures__/reports/inbox-report.json +++ b/packages/cli/src/bench/__fixtures__/reports/inbox-report.json @@ -1,6 +1,6 @@ { - "$schema": "https://json-schema.fedify.dev/bench/report-v1.json", - "schemaVersion": 1, + "$schema": "https://json-schema.fedify.dev/bench/report-v2.json", + "schemaVersion": 2, "tool": { "name": "@fedify/cli", "version": "2.3.0" }, "environment": { "runtime": "deno", diff --git a/packages/cli/src/bench/render/markdown.ts b/packages/cli/src/bench/render/markdown.ts index 511f34b26..b0e31ab2e 100644 --- a/packages/cli/src/bench/render/markdown.ts +++ b/packages/cli/src/bench/render/markdown.ts @@ -54,6 +54,13 @@ function renderScenario(scenario: ScenarioResult): string[] { lines.push(`| Requests | ${formatNumber(r.total)} |`); lines.push(`| Success rate | ${formatPercent(r.successRate)} |`); lines.push(`| Throughput | ${formatNumber(scenario.throughputPerSec)}/s |`); + if (scenario.deliveryThroughputPerSec != null) { + lines.push( + `| Delivery throughput | ${ + formatNumber(scenario.deliveryThroughputPerSec) + }/s |`, + ); + } const l = scenario.client.latencyMs; lines.push(`| Latency p50 | ${formatNumber(l.p50)}ms |`); lines.push(`| Latency p95 | ${formatNumber(l.p95)}ms |`); diff --git a/packages/cli/src/bench/render/render.test.ts b/packages/cli/src/bench/render/render.test.ts index 4c6688a61..d96b732bc 100644 --- a/packages/cli/src/bench/render/render.test.ts +++ b/packages/cli/src/bench/render/render.test.ts @@ -5,7 +5,7 @@ import { dirname, join } from "node:path"; import test from "node:test"; import { fileURLToPath } from "node:url"; import type { BenchReport } from "../result/model.ts"; -import { reportSchemaV1 } from "../result/schema.ts"; +import { reportSchemaV2 } from "../result/schema.ts"; import { renderReport } from "./index.ts"; // `import.meta.dirname` needs Node >= 20.11; derive it from the URL instead. @@ -21,7 +21,7 @@ test("renderReport json - valid JSON that validates against the schema", () => { const json = renderReport(report, "json"); const parsed = JSON.parse(json); const validator = new Validator( - reportSchemaV1 as unknown as Schema, + reportSchemaV2 as unknown as Schema, "2020-12", ); assert.ok(validator.validate(parsed).valid); @@ -74,6 +74,23 @@ test("renderReport - shows queue depth even without drain latency", () => { assert.match(md, /Queue depth max \(server\) \| 42/); }); +test("renderReport - shows delivery throughput when present", () => { + const base = report.scenarios[0]; + const r: BenchReport = { + ...report, + scenarios: [{ + ...base, + deliveryThroughputPerSec: 123, + }], + }; + const json = JSON.parse(renderReport(r, "json")); + assert.strictEqual(json.scenarios[0].deliveryThroughputPerSec, 123); + const text = renderReport(r, "text"); + assert.match(text, /Delivery throughput: 123 deliveries\/s/); + const md = renderReport(r, "markdown"); + assert.match(md, /Delivery throughput \| 123\/s/); +}); + test("renderReport - empty drain latency falls back to the depth line", () => { // An empty drainMs object carries no percentile, so neither form should print // a meaningless drain line; both still surface the depth (here zero). diff --git a/packages/cli/src/bench/render/text.ts b/packages/cli/src/bench/render/text.ts index da13f30c1..8b40f8233 100644 --- a/packages/cli/src/bench/render/text.ts +++ b/packages/cli/src/bench/render/text.ts @@ -64,6 +64,13 @@ function renderScenario(scenario: ScenarioResult): string[] { })`, ); lines.push(` Throughput: ${formatNumber(scenario.throughputPerSec)} req/s`); + if (scenario.deliveryThroughputPerSec != null) { + lines.push( + ` Delivery throughput: ${ + formatNumber(scenario.deliveryThroughputPerSec) + } deliveries/s`, + ); + } const l = scenario.client.latencyMs; lines.push( ` Client latency (ms): p50 ${formatNumber(l.p50)} p95 ${ diff --git a/packages/cli/src/bench/result/build.test.ts b/packages/cli/src/bench/result/build.test.ts index 19b14b9be..4a268dc57 100644 --- a/packages/cli/src/bench/result/build.test.ts +++ b/packages/cli/src/bench/result/build.test.ts @@ -9,7 +9,7 @@ import { detectEnvironment, type ScenarioMeasurement, } from "./build.ts"; -import { reportSchemaV1 } from "./schema.ts"; +import { reportSchemaV2 } from "./schema.ts"; function resolvedInbox() { return normalizeSuite({ @@ -62,6 +62,14 @@ test("buildScenarioResult - a run that measured nothing never passes", () => { assert.strictEqual(result.passed, false); }); +test("buildScenarioResult - preserves delivery throughput", () => { + const result = buildScenarioResult(resolvedInbox(), { + ...measurement(), + deliveryThroughputPerSec: 42, + }); + assert.strictEqual(result.deliveryThroughputPerSec, 42); +}); + test("buildReport - gate passes only when all scenarios pass", () => { const ok = buildScenarioResult(resolvedInbox(), measurement()); const bad = buildScenarioResult(resolvedInbox(), { @@ -93,7 +101,7 @@ test("buildReport - output validates against the report schema", () => { suite: { name: "suite", configHash: configHash({ a: 1 }) }, }); const validator = new Validator( - reportSchemaV1 as unknown as Schema, + reportSchemaV2 as unknown as Schema, "2020-12", ); const result = validator.validate(JSON.parse(JSON.stringify(report))); diff --git a/packages/cli/src/bench/result/build.ts b/packages/cli/src/bench/result/build.ts index dd0799b37..9e04d73b3 100644 --- a/packages/cli/src/bench/result/build.ts +++ b/packages/cli/src/bench/result/build.ts @@ -61,6 +61,9 @@ export function buildScenarioResult( load: loadSummary(scenario), requests: measurement.requests, throughputPerSec: measurement.throughputPerSec, + ...(measurement.deliveryThroughputPerSec == null ? {} : { + deliveryThroughputPerSec: measurement.deliveryThroughputPerSec, + }), client: measurement.client, server: measurement.server, errors: measurement.errors, @@ -89,7 +92,7 @@ export interface ReportInput { export function buildReport(input: ReportInput): BenchReport { return { $schema: REPORT_SCHEMA_ID, - schemaVersion: 1, + schemaVersion: 2, tool: { name: "@fedify/cli", version: metadata.version }, environment: input.environment, target: input.target, diff --git a/packages/cli/src/bench/result/model.ts b/packages/cli/src/bench/result/model.ts index 44b885992..bbdf2d8bc 100644 --- a/packages/cli/src/bench/result/model.ts +++ b/packages/cli/src/bench/result/model.ts @@ -143,6 +143,7 @@ export interface ScenarioResult { readonly load: LoadSummary; readonly requests: RequestSummary; readonly throughputPerSec: number; + readonly deliveryThroughputPerSec?: number; readonly client: ClientMetrics; readonly server: ServerMetrics | null; readonly errors: ErrorBucket[]; @@ -156,7 +157,7 @@ export interface ScenarioResult { export interface BenchReport { /** The published report schema URL. */ readonly $schema?: string; - readonly schemaVersion: 1; + readonly schemaVersion: 2; readonly tool: { readonly name: string; readonly version: string }; readonly environment: Environment; readonly target: TargetInfo; diff --git a/packages/cli/src/bench/result/schema.ts b/packages/cli/src/bench/result/schema.ts index b465bf88f..787e4b63c 100644 --- a/packages/cli/src/bench/result/schema.ts +++ b/packages/cli/src/bench/result/schema.ts @@ -2,20 +2,24 @@ * The embedded JSON Schema (draft 2020-12) for benchmark report output. * * Like the scenario schema, this object is the runtime copy and is published, - * byte-for-byte, as *schema/bench/report-v1.json*; a drift guard keeps the two - * in sync. The matching TypeScript types live in {@link ./model.ts}. + * byte-for-byte, as files under *schema/bench/*; a drift guard keeps the two in + * sync. The matching TypeScript types live in {@link ./model.ts}. * @since 2.3.0 * @module */ /** The hosted URL that serves the report schema. */ export const REPORT_SCHEMA_ID = + "https://json-schema.fedify.dev/bench/report-v2.json"; + +/** The hosted URL for the original report schema. */ +export const REPORT_SCHEMA_V1_ID = "https://json-schema.fedify.dev/bench/report-v1.json"; /** The benchmark report JSON Schema (draft 2020-12). */ export const reportSchemaV1 = { $schema: "https://json-schema.org/draft/2020-12/schema", - $id: REPORT_SCHEMA_ID, + $id: REPORT_SCHEMA_V1_ID, title: "Fedify benchmark report", type: "object", additionalProperties: false, @@ -284,3 +288,23 @@ export const reportSchemaV1 = { }, }, } as const; + +/** The benchmark report JSON Schema (draft 2020-12). */ +export const reportSchemaV2 = { + ...reportSchemaV1, + $id: REPORT_SCHEMA_ID, + properties: { + ...reportSchemaV1.properties, + schemaVersion: { const: 2 }, + }, + $defs: { + ...reportSchemaV1.$defs, + scenarioResult: { + ...reportSchemaV1.$defs.scenarioResult, + properties: { + ...reportSchemaV1.$defs.scenarioResult.properties, + deliveryThroughputPerSec: { type: "number" }, + }, + }, + }, +} as const; diff --git a/packages/cli/src/bench/scenarios/object-discovery.ts b/packages/cli/src/bench/scenarios/object-discovery.ts index 4ca0df2b4..8c4d32348 100644 --- a/packages/cli/src/bench/scenarios/object-discovery.ts +++ b/packages/cli/src/bench/scenarios/object-discovery.ts @@ -8,6 +8,9 @@ import { convertUrlIfHandle } from "../../webfinger/lib.ts"; import { asList } from "../scenario/coerce.ts"; import type { ObjectSource } from "../scenario/types.ts"; +const ACTIVITY_JSON_ACCEPT = "application/activity+json, application/ld+json"; +const WEBFINGER_ACCEPT = "application/jrd+json, application/json"; + /** Options for resolving actor URLs. */ export interface ActorUrlOptions { readonly target: URL; @@ -75,7 +78,7 @@ async function actorUrlFromRecipient( if (identifier.protocol !== "acct:") return identifier; const url = new URL("/.well-known/webfinger", options.target); url.searchParams.set("resource", identifier.href); - const jrd = await fetchJson(url, options.fetch); + const jrd = await fetchJson(url, options.fetch, WEBFINGER_ACCEPT); const links = Array.isArray(jrd.links) ? jrd.links : []; const self = links.find((link) => isRecord(link) && link.rel === "self" && typeof link.href === "string" @@ -121,8 +124,14 @@ async function* crawlCollection( async function fetchJson( url: URL, fetchImpl: typeof fetch = fetch, + accept = ACTIVITY_JSON_ACCEPT, ): Promise> { - const response = await fetchImpl(new Request(url, { redirect: "manual" })); + const response = await fetchImpl( + new Request(url, { + headers: { accept }, + redirect: "manual", + }), + ); if (!response.ok) { throw new Error(`Failed to fetch ${url.href}: HTTP ${response.status}.`); } diff --git a/packages/cli/src/bench/scenarios/object.test.ts b/packages/cli/src/bench/scenarios/object.test.ts index bb9a37288..3dd901cad 100644 --- a/packages/cli/src/bench/scenarios/object.test.ts +++ b/packages/cli/src/bench/scenarios/object.test.ts @@ -133,6 +133,65 @@ test("objectRunner - crawls actor collections before fetching objects", async () } }); +test("objectRunner - sends ActivityPub Accept headers during object discovery", async () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "object-crawl", + type: "object", + source: { + seed: "http://target.test/users/alice", + collection: "outbox", + limit: 1, + }, + load: { concurrency: 1 }, + duration: "25ms", + }], + }).scenarios[0]; + const discoveryAccepts: string[] = []; + + const measurement = await objectRunner.run({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const request = input instanceof Request ? input : new Request(input); + const url = new URL(request.url); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return Promise.resolve(new Response("not found", { status: 404 })); + } + if (url.pathname === "/users/alice") { + discoveryAccepts.push(request.headers.get("accept") ?? ""); + return Promise.resolve(json({ + id: url.href, + outbox: "http://target.test/users/alice/outbox", + })); + } + if (url.pathname === "/users/alice/outbox") { + discoveryAccepts.push(request.headers.get("accept") ?? ""); + return Promise.resolve(json({ + id: url.href, + orderedItems: ["http://target.test/objects/1"], + })); + } + if (url.pathname === "/objects/1") { + return Promise.resolve(json({ id: url.href, type: "Note" })); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }, + }); + + assert.ok(measurement.requests.total > 0); + assert.deepStrictEqual(discoveryAccepts, [ + "application/activity+json, application/ld+json", + "application/activity+json, application/ld+json", + ]); +}); + test("objectRunner - skips URL-only collection items for type filters", async () => { const scenario = normalizeSuite({ version: 1, diff --git a/packages/cli/src/bench/schema.test.ts b/packages/cli/src/bench/schema.test.ts index 647a1c559..e3b33834f 100644 --- a/packages/cli/src/bench/schema.test.ts +++ b/packages/cli/src/bench/schema.test.ts @@ -143,7 +143,11 @@ for (const { name, fileName } of PUBLISHED_SCHEMAS) { published = execFileSync( "git", ["show", `${baseCommit}:schema/bench/${fileName}`], - { cwd: REPO_ROOT, encoding: "utf-8" }, + { + cwd: REPO_ROOT, + encoding: "utf-8", + stdio: ["ignore", "pipe", "ignore"], + }, ); } catch { // Not published at the base (a brand-new version file): nothing to guard. diff --git a/packages/cli/src/bench/schemas.ts b/packages/cli/src/bench/schemas.ts index 3812d6655..14079f0e1 100644 --- a/packages/cli/src/bench/schemas.ts +++ b/packages/cli/src/bench/schemas.ts @@ -9,7 +9,7 @@ * @module */ -import { reportSchemaV1 } from "./result/schema.ts"; +import { reportSchemaV1, reportSchemaV2 } from "./result/schema.ts"; import { scenarioSchemaV1 } from "./scenario/schema.ts"; /** A published JSON Schema and where it is hosted. */ @@ -31,6 +31,11 @@ export const PUBLISHED_SCHEMAS: readonly PublishedSchema[] = [ }, { name: "report", + fileName: "report-v2.json", + schema: reportSchemaV2 as unknown as Record, + }, + { + name: "report-v1", fileName: "report-v1.json", schema: reportSchemaV1 as unknown as Record, }, diff --git a/schema/bench/report-v2.json b/schema/bench/report-v2.json new file mode 100644 index 000000000..3797332e4 --- /dev/null +++ b/schema/bench/report-v2.json @@ -0,0 +1,523 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.fedify.dev/bench/report-v2.json", + "title": "Fedify benchmark report", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "tool", + "environment", + "target", + "startedAt", + "finishedAt", + "suite", + "passed", + "scenarios" + ], + "properties": { + "$schema": { + "type": "string" + }, + "schemaVersion": { + "const": 2 + }, + "tool": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "version" + ], + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + } + } + }, + "environment": { + "type": "object", + "additionalProperties": false, + "required": [ + "runtime", + "runtimeVersion", + "os", + "cpuCount" + ], + "properties": { + "runtime": { + "type": "string" + }, + "runtimeVersion": { + "type": "string" + }, + "os": { + "type": "string" + }, + "cpuCount": { + "type": "integer", + "minimum": 0 + } + } + }, + "target": { + "type": "object", + "additionalProperties": false, + "required": [ + "url", + "statsAvailable" + ], + "properties": { + "url": { + "type": "string" + }, + "fedifyVersion": { + "type": [ + "string", + "null" + ] + }, + "statsAvailable": { + "type": "boolean" + } + } + }, + "startedAt": { + "type": "string" + }, + "finishedAt": { + "type": "string" + }, + "suite": { + "type": "object", + "additionalProperties": false, + "required": [ + "configHash" + ], + "properties": { + "name": { + "type": "string" + }, + "configHash": { + "type": "string" + } + } + }, + "passed": { + "type": "boolean" + }, + "scenarios": { + "type": "array", + "items": { + "$ref": "#/$defs/scenarioResult" + } + } + }, + "$defs": { + "latencyMs": { + "type": "object", + "additionalProperties": false, + "required": [ + "p50", + "p95", + "p99", + "mean", + "max" + ], + "properties": { + "p50": { + "type": "number" + }, + "p95": { + "type": "number" + }, + "p99": { + "type": "number" + }, + "mean": { + "type": "number" + }, + "max": { + "type": "number" + } + } + }, + "partialLatencyMs": { + "type": "object", + "additionalProperties": false, + "properties": { + "p50": { + "type": "number" + }, + "p95": { + "type": "number" + }, + "p99": { + "type": "number" + } + } + }, + "loadSummary": { + "type": "object", + "additionalProperties": false, + "required": [ + "model", + "durationMs", + "warmupMs" + ], + "properties": { + "model": { + "enum": [ + "open", + "closed" + ] + }, + "ratePerSec": { + "type": "number" + }, + "arrival": { + "type": "string" + }, + "concurrency": { + "type": "integer" + }, + "durationMs": { + "type": "number" + }, + "warmupMs": { + "type": "number" + }, + "maxInFlight": { + "type": "integer" + } + }, + "oneOf": [ + { + "properties": { + "model": { + "const": "open" + } + }, + "required": [ + "ratePerSec", + "arrival" + ], + "not": { + "required": [ + "concurrency" + ] + } + }, + { + "properties": { + "model": { + "const": "closed" + } + }, + "required": [ + "concurrency" + ], + "not": { + "anyOf": [ + { + "required": [ + "ratePerSec" + ] + }, + { + "required": [ + "arrival" + ] + } + ] + } + } + ] + }, + "requestSummary": { + "type": "object", + "additionalProperties": false, + "required": [ + "total", + "ok", + "failed", + "successRate" + ], + "properties": { + "total": { + "type": "integer", + "minimum": 0 + }, + "ok": { + "type": "integer", + "minimum": 0 + }, + "failed": { + "type": "integer", + "minimum": 0 + }, + "successRate": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + } + }, + "clientMetrics": { + "type": "object", + "additionalProperties": false, + "required": [ + "latencyMs" + ], + "properties": { + "latencyMs": { + "$ref": "#/$defs/latencyMs" + } + } + }, + "serverMetrics": { + "type": "object", + "additionalProperties": false, + "properties": { + "signatureVerificationMs": { + "type": "object", + "additionalProperties": false, + "required": [ + "overall" + ], + "properties": { + "overall": { + "$ref": "#/$defs/partialLatencyMs" + }, + "byStandard": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/partialLatencyMs" + } + } + } + }, + "queue": { + "type": "object", + "additionalProperties": false, + "properties": { + "drainMs": { + "$ref": "#/$defs/partialLatencyMs" + }, + "depthMax": { + "type": "number" + } + } + } + } + }, + "errorBucket": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "reason", + "count" + ], + "properties": { + "kind": { + "type": "string" + }, + "status": { + "type": "integer" + }, + "reason": { + "type": "string" + }, + "count": { + "type": "integer", + "minimum": 0 + } + } + }, + "expectResult": { + "type": "object", + "additionalProperties": false, + "required": [ + "metric", + "op", + "threshold", + "unit", + "actual", + "severity", + "pass" + ], + "properties": { + "metric": { + "type": "string" + }, + "op": { + "enum": [ + "lt", + "lte", + "gt", + "gte", + "eq" + ] + }, + "threshold": { + "type": "number" + }, + "unit": { + "type": [ + "string", + "null" + ] + }, + "actual": { + "type": [ + "number", + "null" + ] + }, + "severity": { + "enum": [ + "warn", + "fail" + ] + }, + "pass": { + "type": "boolean" + } + } + }, + "scenarioResult": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "type", + "load", + "requests", + "throughputPerSec", + "client", + "server", + "errors", + "expectations", + "passed" + ], + "properties": { + "name": { + "type": "string" + }, + "type": { + "enum": [ + "inbox", + "webfinger", + "actor", + "object", + "fanout", + "collection", + "failure", + "mixed" + ] + }, + "load": { + "$ref": "#/$defs/loadSummary" + }, + "requests": { + "$ref": "#/$defs/requestSummary" + }, + "throughputPerSec": { + "type": "number" + }, + "client": { + "$ref": "#/$defs/clientMetrics" + }, + "server": { + "anyOf": [ + { + "$ref": "#/$defs/serverMetrics" + }, + { + "type": "null" + } + ] + }, + "errors": { + "type": "array", + "items": { + "$ref": "#/$defs/errorBucket" + } + }, + "expectations": { + "type": "array", + "items": { + "$ref": "#/$defs/expectResult" + } + }, + "passed": { + "type": "boolean" + }, + "histogram": { + "$ref": "#/$defs/serializedHistogram" + }, + "deliveryThroughputPerSec": { + "type": "number" + } + } + }, + "serializedHistogram": { + "type": "object", + "additionalProperties": false, + "required": [ + "version", + "subBucketCount", + "count", + "zeroCount", + "min", + "max", + "sum", + "indices", + "counts" + ], + "properties": { + "version": { + "const": 1 + }, + "subBucketCount": { + "type": "integer", + "minimum": 1 + }, + "count": { + "type": "integer", + "minimum": 0 + }, + "zeroCount": { + "type": "integer", + "minimum": 0 + }, + "min": { + "type": "number" + }, + "max": { + "type": "number" + }, + "sum": { + "type": "number" + }, + "indices": { + "type": "array", + "items": { + "type": "integer" + } + }, + "counts": { + "type": "array", + "items": { + "type": "integer", + "minimum": 0 + } + } + } + } + } +} From 67ec73314bdc84795da8f688505be91520125e96 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Wed, 10 Jun 2026 04:50:29 +0900 Subject: [PATCH 15/61] Preserve failure and mixed correctness Keep inbound failure scenarios from treating target-side 5xx responses as expected client-fault rejections, so server crashes remain visible in the error buckets. Reject ambiguous mixed child references when duplicate scenario names would otherwise make child lookup bind the first matching scenario silently. https://github.com/fedify-dev/fedify/issues/785 Assisted-by: Codex:gpt-5.5 --- .../cli/src/bench/scenarios/failure.test.ts | 71 +++++++++++++++++++ packages/cli/src/bench/scenarios/failure.ts | 2 +- .../cli/src/bench/scenarios/mixed.test.ts | 28 ++++++++ packages/cli/src/bench/scenarios/mixed.ts | 10 ++- 4 files changed, 109 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/bench/scenarios/failure.test.ts b/packages/cli/src/bench/scenarios/failure.test.ts index 8cec4414b..0cd9cc8ac 100644 --- a/packages/cli/src/bench/scenarios/failure.test.ts +++ b/packages/cli/src/bench/scenarios/failure.test.ts @@ -127,6 +127,77 @@ test("failureRunner - discovers inbound failure inboxes once", async () => { } }); +test("failureRunner - treats inbound 5xx as target failures", async () => { + const server = serve({ + port: 0, + hostname: "127.0.0.1", + silent: true, + fetch(request: Request): Response { + const url = new URL(request.url); + if (url.pathname === "/users/alice") { + return json({ + "@context": "https://www.w3.org/ns/activitystreams", + type: "Person", + id: url.href, + inbox: new URL("/inbox", url).href, + }); + } + return new Response("not found", { status: 404 }); + }, + }); + await server.ready(); + let fleet: Awaited> | undefined; + try { + const target = new URL(server.url!); + fleet = await spawnSyntheticServer( + await buildFleet([{ + count: 1, + signatureStandards: ["draft-cavage-http-signatures-12"], + }]), + ); + const scenario = normalizeSuite({ + version: 1, + target: target.href, + scenarios: [{ + name: "bad-signature", + type: "failure", + fault: "invalid-signature", + recipient: new URL("/users/alice", target).href, + load: { concurrency: 1 }, + duration: "50ms", + }], + }).scenarios[0]; + const measurement = await failureRunner.run({ + scenario, + target, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet, + fetch: (input) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/inbox") { + return Promise.resolve( + new Response("internal error", { status: 500 }), + ); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }, + assertDestinationAllowed: () => {}, + }); + + assert.ok(measurement.requests.total > 0); + assert.strictEqual(measurement.requests.successRate, 0); + assert.ok(measurement.errors.some((e) => e.status === 500)); + } finally { + try { + await fleet?.close(); + } finally { + await server.close(true); + } + } +}); + function json(body: unknown): Response { return new Response(JSON.stringify(body), { headers: { "content-type": "application/activity+json" }, diff --git a/packages/cli/src/bench/scenarios/failure.ts b/packages/cli/src/bench/scenarios/failure.ts index 264bc47e3..d789fdb96 100644 --- a/packages/cli/src/bench/scenarios/failure.ts +++ b/packages/cli/src/bench/scenarios/failure.ts @@ -252,7 +252,7 @@ function missingActor(actor: SyntheticActor, target: URL): SyntheticActor { } function expectedFailure(outcome: SendOutcome): SendOutcome { - if (outcome.status != null && outcome.status >= 400) { + if (outcome.status != null && outcome.status >= 400 && outcome.status < 500) { return { ok: true, status: outcome.status }; } return { diff --git a/packages/cli/src/bench/scenarios/mixed.test.ts b/packages/cli/src/bench/scenarios/mixed.test.ts index 9a2b928f9..2eaf046d6 100644 --- a/packages/cli/src/bench/scenarios/mixed.test.ts +++ b/packages/cli/src/bench/scenarios/mixed.test.ts @@ -169,6 +169,34 @@ test("mixedRunner.validate - rejects unknown children with suite context", () => ); }); +test("mixedRunner.validate - rejects ambiguous child names", () => { + const scenarios = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [ + { + name: "mixed", + type: "mixed", + mix: [{ scenario: "lookup", weight: 1 }], + }, + { + name: "lookup", + type: "webfinger", + }, + { + name: "lookup", + type: "actor", + recipient: "http://target.test/users/alice", + }, + ], + }).scenarios; + + assert.throws( + () => mixedRunner.validate?.(scenarios[0], { scenarios }), + /ambiguous mixed child/, + ); +}); + test("mixedRunner.validate - rejects nested mixed children with suite context", () => { const scenarios = normalizeSuite({ version: 1, diff --git a/packages/cli/src/bench/scenarios/mixed.ts b/packages/cli/src/bench/scenarios/mixed.ts index 8e37151c5..3e78f9224 100644 --- a/packages/cli/src/bench/scenarios/mixed.ts +++ b/packages/cli/src/bench/scenarios/mixed.ts @@ -88,15 +88,23 @@ function childScenarios( ) : undefined; return entries.map((entry, index) => { - const child = scenarios.find((candidate) => + const children = scenarios.filter((candidate) => candidate.name === entry.scenario ); + const child = children[0]; if (child == null) { throw new Error( `Scenario "${scenario.name}": unknown mixed child ` + `${JSON.stringify(entry.scenario)}.`, ); } + if (children.length > 1) { + throw new Error( + `Scenario "${scenario.name}": ambiguous mixed child ` + + `${JSON.stringify(entry.scenario)} matches ${children.length} ` + + "scenarios.", + ); + } if (child.type === "mixed") { throw new Error( `Scenario "${scenario.name}": nested mixed scenarios are not ` + From e11e21647f71feac40a91a72b431d406e118e985 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Wed, 10 Jun 2026 12:00:12 +0900 Subject: [PATCH 16/61] Drive failure faults through targets Make remote failure benchmark modes call the target benchmark trigger and observe the target's queue metrics before reporting the expected fault as successful. This keeps offline or non-participating targets from producing synthetic passing samples. Require a sender for remote failure faults and require advertised sink reachability for non-loopback targets, matching the benchmark-owned sink server used to exercise the outbound path. https://github.com/fedify-dev/fedify/issues/785 Assisted-by: Codex:gpt-5.5 --- docs/manual/benchmarking.md | 7 +- packages/cli/src/bench/action.test.ts | 16 +- packages/cli/src/bench/action.ts | 8 +- .../src/bench/metrics/stats-client.test.ts | 16 + .../cli/src/bench/metrics/stats-client.ts | 23 ++ .../cli/src/bench/scenarios/failure.test.ts | 122 ++++++-- packages/cli/src/bench/scenarios/failure.ts | 277 +++++++++++++++++- packages/cli/src/bench/scenarios/fanout.ts | 2 +- 8 files changed, 427 insertions(+), 44 deletions(-) diff --git a/docs/manual/benchmarking.md b/docs/manual/benchmarking.md index c879467a4..be136e933 100644 --- a/docs/manual/benchmarking.md +++ b/docs/manual/benchmarking.md @@ -187,8 +187,11 @@ The runnable scenario types cover the main benchmark surfaces: scenario type, `successRate` means “the expected failure was observed,” not “the HTTP request succeeded.” The `invalid-signature` and `missing-actor` faults send malformed signed deliveries to a recipient - inbox; `remote-404`, `remote-410`, `slow-inbox`, and `network-error` model - controlled remote failure outcomes. + inbox. The `remote-404`, `remote-410`, `slow-inbox`, and `network-error` + faults post to the benchmark trigger endpoint with `sender`, so the target + uses its normal outbound delivery path against controlled benchmark-owned + sink inboxes. Like `fanout`, these remote failure faults need + `--advertise-host` for a non-loopback target. - `mixed`: runs referenced child scenarios concurrently, splitting the `mixed` scenario's load by each entry's `weight`. The referenced scenarios are named scenarios in the same suite and are still run as normal diff --git a/packages/cli/src/bench/action.test.ts b/packages/cli/src/bench/action.test.ts index 14ad6eb3a..0546730da 100644 --- a/packages/cli/src/bench/action.test.ts +++ b/packages/cli/src/bench/action.test.ts @@ -423,31 +423,31 @@ scenarios: assert.match(message, /advertise-host/); }); -test("runBench - failure without inbound fault needs no advertise host", async () => { +test("runBench - remote failure needs advertised sink reachability", async () => { const file = await writeSuite(`version: 1 target: http://10.10.0.5:8000 scenarios: - name: remote-404 type: failure fault: remote-404 + sender: alice load: { rate: 1/s } duration: 1ms `); let code = -1; - let output = ""; + let message = ""; await runBench(command({ scenario: file }), { exit: (c) => { code = c; }, - writeOutput: (c) => { - output = c; - return Promise.resolve(); + writeOutput: () => Promise.resolve(), + log: (m) => { + message = m; }, - log: () => {}, fetch: () => Promise.reject(new Error("offline")), }); - assert.strictEqual(code, 0); - assert.strictEqual(JSON.parse(output).scenarios[0].requests.successRate, 1); + assert.strictEqual(code, 2); + assert.match(message, /advertise-host/); }); test("runBench - missing-actor failure needs no advertise host", async () => { diff --git a/packages/cli/src/bench/action.ts b/packages/cli/src/bench/action.ts index 69b933690..7640b7a85 100644 --- a/packages/cli/src/bench/action.ts +++ b/packages/cli/src/bench/action.ts @@ -702,7 +702,8 @@ function scenarioNeedsReachableLocalServer( ): boolean { if (scenario.type === "fanout") return true; if (scenario.type === "failure") { - return scenario.faults.includes("invalid-signature"); + return scenario.faults.includes("invalid-signature") || + scenario.faults.some(isRemoteFailureFault); } if (scenario.type === "mixed") { if (seen.has(scenario.name)) return false; @@ -729,3 +730,8 @@ function mixedChildrenOf( function isInboundFailureFault(fault: string): boolean { return fault === "invalid-signature" || fault === "missing-actor"; } + +function isRemoteFailureFault(fault: string): boolean { + return fault === "remote-404" || fault === "remote-410" || + fault === "slow-inbox" || fault === "network-error"; +} diff --git a/packages/cli/src/bench/metrics/stats-client.test.ts b/packages/cli/src/bench/metrics/stats-client.test.ts index bf6958350..cb45b93f5 100644 --- a/packages/cli/src/bench/metrics/stats-client.test.ts +++ b/packages/cli/src/bench/metrics/stats-client.test.ts @@ -142,6 +142,19 @@ test("parseServerSnapshot - extracts raw histogram and queue depth", () => { assert.strictEqual(snap?.queueDepthMax, 7); }); +test("parseServerSnapshot - extracts permanent delivery failures", () => { + const snap = parseServerSnapshot({ + scopeMetrics: [{ + metrics: [{ + name: "activitypub.delivery.permanent_failure", + dataPointType: "sum", + dataPoints: [{ value: 3 }, { value: 2 }], + }], + }], + }); + assert.strictEqual(snap?.deliveryPermanentFailures, 5); +}); + test("parseServerSnapshot - empty (non-null) when no relevant instruments", () => { // A parseable-but-empty snapshot yields an empty snapshot, not null, so a // successful baseline fetch is distinguishable from an unavailable one. @@ -155,15 +168,18 @@ test("diffSnapshots - subtracts the baseline bucket counts", () => { const baseline: ServerSnapshot = { signature: { boundaries: [5, 10, 25], counts: [4, 6, 10, 0] }, queueDepthMax: 2, + deliveryPermanentFailures: 2, }; const end: ServerSnapshot = { signature: { boundaries: [5, 10, 25], counts: [10, 16, 30, 4] }, queueDepthMax: 9, + deliveryPermanentFailures: 5, }; const diff = diffSnapshots(baseline, end); assert.deepEqual(diff?.signature?.counts, [6, 10, 20, 4]); // The queue depth is a gauge, so the end value is kept (not subtracted). assert.strictEqual(diff?.queueDepthMax, 9); + assert.strictEqual(diff?.deliveryPermanentFailures, 3); }); test("diffSnapshots - an empty baseline keeps the full end histogram", () => { diff --git a/packages/cli/src/bench/metrics/stats-client.ts b/packages/cli/src/bench/metrics/stats-client.ts index 19f70fe14..2b805ae82 100644 --- a/packages/cli/src/bench/metrics/stats-client.ts +++ b/packages/cli/src/bench/metrics/stats-client.ts @@ -59,6 +59,8 @@ export interface ServerSnapshot { readonly queueDepthMax: number | null; /** Queue task counters, or `null` if absent. */ readonly queueTasks?: QueueTaskCounts | null; + /** Permanent outbound delivery failure count, or `null` if absent. */ + readonly deliveryPermanentFailures?: number | null; } /** Queue task counts extracted from benchmark stats. */ @@ -96,11 +98,18 @@ export function parseServerSnapshot(snapshot: unknown): ServerSnapshot | null { } const queueTasks = parseQueueTasks(metrics); + const deliveryPermanentFailures = sumMetric( + metrics, + "activitypub.delivery.permanent_failure", + ); return { signature, queueDepthMax, ...(queueTasks == null ? {} : { queueTasks }), + ...(deliveryPermanentFailures == null + ? {} + : { deliveryPermanentFailures }), }; } catch { return null; @@ -126,10 +135,15 @@ export function diffSnapshots( baseline.queueTasks ?? null, end.queueTasks ?? null, ); + const deliveryPermanentFailures = diffCounter( + baseline.deliveryPermanentFailures ?? null, + end.deliveryPermanentFailures ?? null, + ); return { signature: diffHistogram(baseline.signature, end.signature), queueDepthMax: end.queueDepthMax, ...(queueTasks == null ? {} : { queueTasks }), + ...(deliveryPermanentFailures == null ? {} : { deliveryPermanentFailures }), }; } @@ -329,6 +343,15 @@ function diffQueueTasks( }; } +function diffCounter( + baseline: number | null, + end: number | null, +): number | null { + if (end == null) return null; + if (baseline == null) return end; + return Math.max(0, end - baseline); +} + function histogramsCompatible( a: ServerHistogram, b: ServerHistogram, diff --git a/packages/cli/src/bench/scenarios/failure.test.ts b/packages/cli/src/bench/scenarios/failure.test.ts index 0cd9cc8ac..d690b3724 100644 --- a/packages/cli/src/bench/scenarios/failure.test.ts +++ b/packages/cli/src/bench/scenarios/failure.test.ts @@ -8,31 +8,81 @@ import type { Suite } from "../scenario/types.ts"; import { spawnSyntheticServer } from "../server/synthetic.ts"; import { failureRunner } from "./failure.ts"; -test("failureRunner - counts expected remote failure as success", async () => { - const suite: Suite = { +for ( + const [fault, outcome] of [ + ["remote-404", "permanent"], + ["remote-410", "permanent"], + ["slow-inbox", "completed"], + ["network-error", "failed"], + ] as const +) { + test(`failureRunner - drives ${fault} through the target`, async () => { + const target = new URL("http://target.test/"); + const suite: Suite = { + version: 1, + target: target.href, + scenarios: [{ + name: "failure", + type: "failure", + fault, + sender: "alice", + load: { concurrency: 1 }, + duration: "25ms", + queueDrainTimeout: "1s", + }], + }; + const scenario = normalizeSuite(suite).scenarios[0]; + let triggerCalls = 0; + let triggerRecipientCount = 0; + const measurement = await failureRunner.run({ + scenario, + target, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input, init) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return Promise.resolve(statsJson(statsSnapshot({ + enqueued: triggerCalls, + completed: outcome === "failed" ? 0 : triggerCalls, + failed: outcome === "failed" ? triggerCalls : 0, + permanentFailures: outcome === "permanent" ? triggerCalls : 0, + }))); + } + if (url.pathname === "/.well-known/fedify/bench/trigger") { + triggerCalls++; + const body = JSON.parse(String(init?.body)); + triggerRecipientCount = body.recipients.length; + assert.deepStrictEqual(body.sender, { identifier: "alice" }); + return Promise.resolve(statsJson({ version: 1 }, 202)); + } + return Promise.resolve(new Response("unexpected", { status: 500 })); + }, + assertDestinationAllowed: () => {}, + }); + + assert.ok(measurement.requests.total > 0); + assert.strictEqual(measurement.requests.failed, 0); + assert.strictEqual(measurement.requests.successRate, 1); + assert.ok(triggerCalls > 0); + assert.strictEqual(triggerRecipientCount, 1); + }); +} + +test("failureRunner.validate - requires sender for remote faults", () => { + const scenario = normalizeSuite({ version: 1, target: "http://target.test/", scenarios: [{ name: "failure", type: "failure", fault: "remote-404", - load: { concurrency: 1 }, - duration: "25ms", }], - }; - const scenario = normalizeSuite(suite).scenarios[0]; - const measurement = await failureRunner.run({ - scenario, - target: new URL("http://target.test/"), - documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), - contextLoader: await getContextLoader({ allowPrivateAddress: true }), - allowPrivateAddress: true, - fleet: null, - }); + }).scenarios[0]; - assert.ok(measurement.requests.total > 0); - assert.strictEqual(measurement.requests.failed, 0); - assert.strictEqual(measurement.requests.successRate, 1); + assert.throws(() => failureRunner.validate?.(scenario), /sender/); }); test("failureRunner.validate - rejects unsupported faults", () => { @@ -203,3 +253,41 @@ function json(body: unknown): Response { headers: { "content-type": "application/activity+json" }, }); } + +function statsJson(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); +} + +function statsSnapshot(counts: { + readonly enqueued: number; + readonly completed: number; + readonly failed: number; + readonly permanentFailures: number; +}): Record { + return { + version: 1, + source: "server", + scopeMetrics: [{ + metrics: [ + sum("fedify.queue.task.enqueued", counts.enqueued), + sum("fedify.queue.task.completed", counts.completed), + sum("fedify.queue.task.failed", counts.failed), + sum( + "activitypub.delivery.permanent_failure", + counts.permanentFailures, + ), + ], + }], + }; +} + +function sum(name: string, value: number): Record { + return { + name, + dataPointType: "sum", + dataPoints: [{ value }], + }; +} diff --git a/packages/cli/src/bench/scenarios/failure.ts b/packages/cli/src/bench/scenarios/failure.ts index d789fdb96..2d11e2e3d 100644 --- a/packages/cli/src/bench/scenarios/failure.ts +++ b/packages/cli/src/bench/scenarios/failure.ts @@ -8,9 +8,15 @@ import { Create, Note } from "@fedify/vocab"; import { discoverInbox, selectInbox } from "../discovery/discover.ts"; import { runLoad, type SendOutcome } from "../load/generator.ts"; import { aggregateSamples } from "../metrics/aggregate.ts"; +import { + diffSnapshots, + fetchServerSnapshot, + queueTaskRemaining, +} from "../metrics/stats-client.ts"; import type { SyntheticActor } from "../server/synthetic.ts"; import { createActivityIdMinter } from "../signing/activity-id.ts"; import { signInboxDelivery } from "../signing/signer.ts"; +import { spawnSinkServer } from "./fanout.ts"; import { loadPlanOf, measuredWindowMs, @@ -29,16 +35,31 @@ const SUPPORTED_FAULTS = [ ] as const; type SupportedFault = typeof SUPPORTED_FAULTS[number]; +type RemoteFailureFault = Exclude< + SupportedFault, + "invalid-signature" | "missing-actor" +>; interface FailureDeliveryTarget { readonly inbox: URL; readonly actorUri: URL; } +interface RemoteFailureTarget { + readonly recipient: Record; + readonly close: () => Promise; +} + +const DEFAULT_DRAIN_TIMEOUT_MS = 60_000; +const DRAIN_POLL_MS = 25; + /** The `failure` scenario runner. */ export const failureRunner: ScenarioRunner = { validate(scenario): void { - for (const fault of scenario.faults) { + const faults = scenario.faults.length < 1 + ? ["remote-404"] + : scenario.faults; + for (const fault of faults) { if (!isSupportedFault(fault)) { throw new Error( `Scenario "${scenario.name}": unsupported failure fault ` + @@ -49,7 +70,7 @@ export const failureRunner: ScenarioRunner = { } } if ( - scenario.faults.some((fault) => + faults.some((fault) => fault === "invalid-signature" || fault === "missing-actor" ) && scenario.recipients.length < 1 ) { @@ -58,24 +79,56 @@ export const failureRunner: ScenarioRunner = { "faults require a recipient.", ); } + if (faults.some(isRemoteFault) && scenario.sender == null) { + throw new Error( + `Scenario "${scenario.name}": remote failure faults require a ` + + "sender.", + ); + } }, async run(context: RunContext) { this.validate?.(context.scenario); const faults = faultsOf(context); const deliveryTarget = await resolveFailureDeliveryTarget(context, faults); - let index = 0; - const send = () => - sendForFault(context, faults[index++ % faults.length], deliveryTarget); - const result = await runLoad( - loadPlanOf(context.scenario, context.rng), - send, - context.clock, - ); - return aggregateSamples(result.samples, { - measuredWindowMs: measuredWindowMs(context.scenario), - includeHistogram: true, - }); + const remoteTargets = await resolveRemoteFailureTargets(context, faults); + const remoteActivityIds = createActivityIdMinter(context.target); + try { + let index = 0; + const sendOne = () => + sendForFault( + context, + faults[index++ % faults.length], + deliveryTarget, + remoteTargets, + remoteActivityIds, + ); + let send = sendOne; + if (faults.some(isRemoteFault)) { + let previous = Promise.resolve(); + send = () => { + const current = previous.then(sendOne); + previous = current.then( + () => {}, + () => {}, + ); + return current; + }; + } + const result = await runLoad( + loadPlanOf(context.scenario, context.rng), + send, + context.clock, + ); + return aggregateSamples(result.samples, { + measuredWindowMs: measuredWindowMs(context.scenario), + includeHistogram: true, + }); + } finally { + await Promise.all( + [...remoteTargets.values()].map((target) => target.close()), + ); + } }, }; @@ -120,10 +173,62 @@ function isInboundFault(fault: SupportedFault): boolean { return fault === "invalid-signature" || fault === "missing-actor"; } +function isRemoteFault(fault: string): fault is RemoteFailureFault { + return fault === "remote-404" || fault === "remote-410" || + fault === "slow-inbox" || fault === "network-error"; +} + +async function resolveRemoteFailureTargets( + context: RunContext, + faults: readonly SupportedFault[], +): Promise> { + const targets = new Map(); + try { + for (const fault of new Set(faults.filter(isRemoteFault))) { + const sink = await spawnSinkServer({ + followers: 1, + rawBehavior: remoteSinkBehavior(fault), + advertiseHost: context.advertiseHost, + }); + const recipient = sink.recipients[0]; + if (fault === "network-error") { + await sink.close(); + targets.set(fault, { + recipient, + close: () => Promise.resolve(), + }); + } else { + targets.set(fault, { recipient, close: () => sink.close() }); + } + } + return targets; + } catch (error) { + await Promise.all([...targets.values()].map((target) => target.close())); + throw error; + } +} + +function remoteSinkBehavior( + fault: RemoteFailureFault, +): Record { + switch (fault) { + case "remote-404": + return { status: 404 }; + case "remote-410": + return { status: 410 }; + case "slow-inbox": + return { status: 202, latency: "25ms" }; + case "network-error": + return { status: 202 }; + } +} + async function sendForFault( context: RunContext, fault: SupportedFault, deliveryTarget: FailureDeliveryTarget | null, + remoteTargets: ReadonlyMap, + remoteActivityIds: ReturnType, ): Promise { switch (fault) { case "invalid-signature": @@ -133,12 +238,135 @@ async function sendForFault( ); case "missing-actor": return await sendMissingActor(context, requiredTarget(deliveryTarget)); + case "remote-404": + case "remote-410": + case "slow-inbox": + case "network-error": + return await sendRemoteFailure( + context, + fault, + requiredRemoteTarget(fault, remoteTargets), + remoteActivityIds, + ); + } +} + +async function sendRemoteFailure( + context: RunContext, + fault: RemoteFailureFault, + target: RemoteFailureTarget, + remoteActivityIds: ReturnType, +): Promise { + const fetchImpl = context.fetch ?? fetch; + const baseline = await fetchServerSnapshot(context.target, fetchImpl); + const response = await fetchImpl( + new URL("/.well-known/fedify/bench/trigger", context.target), + { + method: "POST", + headers: { "content-type": "application/json" }, + redirect: "manual", + body: JSON.stringify({ + sender: { identifier: requiredSender(context) }, + recipients: [target.recipient], + activity: buildRemoteFailureActivity(context, remoteActivityIds.next()), + }), + }, + ); + await response.arrayBuffer().catch(() => {}); + if (!response.ok) { + return { + ok: false, + status: response.status, + reason: `status_${response.status}`, + }; + } + const observation = await waitForRemoteFault({ + target: context.target, + fetch: fetchImpl, + baseline, + fault, + timeoutMs: context.scenario.queueDrainTimeoutMs ?? + DEFAULT_DRAIN_TIMEOUT_MS, + }); + if (observation == null) { + return { + ok: false, + errorKind: "server", + reason: "stats_unavailable", + }; + } + if (observation.timedOut) { + return { + ok: false, + errorKind: "server", + reason: "expected_remote_failure_not_observed", + }; + } + return expectedRemoteFailure(fault); +} + +function buildRemoteFailureActivity( + context: RunContext, + id: URL, +): Record { + const objectId = new URL(`/objects/${crypto.randomUUID()}`, context.target); + return { + "@context": "https://www.w3.org/ns/activitystreams", + type: "Create", + id: id.href, + actor: new URL(`/users/${requiredSender(context)}`, context.target).href, + object: { + type: "Note", + id: objectId.href, + content: "Benchmark failure activity.", + }, + }; +} + +interface RemoteFaultObservation { + readonly timedOut: boolean; +} + +async function waitForRemoteFault(options: { + readonly target: URL; + readonly fetch: typeof fetch; + readonly baseline: Awaited>; + readonly fault: RemoteFailureFault; + readonly timeoutMs: number; +}): Promise { + if (options.baseline == null) return null; + const deadline = Date.now() + options.timeoutMs; + do { + const snapshot = await fetchServerSnapshot(options.target, options.fetch); + if (snapshot == null) return null; + const diff = diffSnapshots(options.baseline, snapshot); + const queueTasks = diff.queueTasks; + if (queueTasks == null) return null; + if (options.fault === "remote-404" || options.fault === "remote-410") { + if ((diff.deliveryPermanentFailures ?? 0) > 0) { + return { timedOut: false }; + } + } else if (options.fault === "slow-inbox") { + const remaining = queueTaskRemaining(diff); + if (remaining == null) return null; + if (queueTasks.completed > 0 && remaining === 0) { + return { timedOut: false }; + } + } else if (queueTasks.failed > 0) { + return { timedOut: false }; + } + await new Promise((resolve) => setTimeout(resolve, DRAIN_POLL_MS)); + } while (Date.now() < deadline); + return { timedOut: true }; +} + +function expectedRemoteFailure(fault: RemoteFailureFault): SendOutcome { + switch (fault) { case "remote-404": return { ok: true, status: 404 }; case "remote-410": return { ok: true, status: 410 }; case "slow-inbox": - await new Promise((resolve) => setTimeout(resolve, 25)); return { ok: true, status: 202 }; case "network-error": return { @@ -239,6 +467,25 @@ function requiredTarget( return target; } +function requiredRemoteTarget( + fault: RemoteFailureFault, + targets: ReadonlyMap, +): RemoteFailureTarget { + const target = targets.get(fault); + if (target == null) { + throw new Error(`The ${fault} fault requires a benchmark sink.`); + } + return target; +} + +function requiredSender(context: RunContext): string { + const sender = context.scenario.sender; + if (sender == null) { + throw new Error("Remote failure faults require a sender."); + } + return sender; +} + function missingActor(actor: SyntheticActor, target: URL): SyntheticActor { const id = new URL(`/__fedify_bench/missing/${crypto.randomUUID()}`, target); return { diff --git a/packages/cli/src/bench/scenarios/fanout.ts b/packages/cli/src/bench/scenarios/fanout.ts index 4152d0e7e..f6c759cd7 100644 --- a/packages/cli/src/bench/scenarios/fanout.ts +++ b/packages/cli/src/bench/scenarios/fanout.ts @@ -177,7 +177,7 @@ function buildActivity(context: RunContext, id: URL): Record { }; } -async function spawnSinkServer(options: { +export async function spawnSinkServer(options: { readonly followers: number; readonly rawBehavior: unknown; readonly advertiseHost?: string; From 5678ea9d2adf1a7893cc1548100ab43549496820 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Wed, 10 Jun 2026 13:55:33 +0900 Subject: [PATCH 17/61] Tighten actor and object discovery Unwrap ActivityPub activity items while crawling object sources so outbox entries benchmark their contained objects instead of the wrapper activity. Apply source type filters after that unwrap step. Validate actor recipients during runner preflight so malformed values fail as configuration errors before probing or sending benchmark load. https://github.com/fedify-dev/fedify/issues/785 Assisted-by: Codex:gpt-5.5 --- packages/cli/src/bench/action.test.ts | 29 ++++ packages/cli/src/bench/scenarios/actor.ts | 19 ++- .../src/bench/scenarios/object-discovery.ts | 37 +++++ .../cli/src/bench/scenarios/object.test.ts | 128 ++++++++++++++++++ 4 files changed, 211 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/bench/action.test.ts b/packages/cli/src/bench/action.test.ts index 0546730da..a9b73be63 100644 --- a/packages/cli/src/bench/action.test.ts +++ b/packages/cli/src/bench/action.test.ts @@ -898,6 +898,35 @@ scenarios: assert.strictEqual(fetched, false); }); +test("runBench - invalid actor recipient exits 2 before any probe", async () => { + const file = await writeSuite(`version: 1 +target: http://localhost:3000 +scenarios: + - name: actor + type: actor + recipient: alice +`); + let code = -1; + let message = ""; + let fetched = false; + await runBench(command({ scenario: file }), { + exit: (c) => { + code = c; + }, + writeOutput: () => Promise.resolve(), + log: (m) => { + message = m; + }, + fetch: () => { + fetched = true; + return Promise.reject(new Error("no request should be sent")); + }, + }); + assert.strictEqual(code, 2); + assert.match(message, /invalid actor recipient/); + assert.strictEqual(fetched, false); +}); + test("runBench - invalid suite exits 2", async () => { const file = await writeSuite(`target: http://localhost:3000 scenarios: diff --git a/packages/cli/src/bench/scenarios/actor.ts b/packages/cli/src/bench/scenarios/actor.ts index 27127fa9a..0515ebc0c 100644 --- a/packages/cli/src/bench/scenarios/actor.ts +++ b/packages/cli/src/bench/scenarios/actor.ts @@ -4,16 +4,31 @@ * @module */ +import { convertUrlIfHandle } from "../../webfinger/lib.ts"; import { actorUrlsFromRecipients } from "./object-discovery.ts"; import { runReadLoad } from "./read.ts"; import type { RunContext, ScenarioRunner } from "./runner.ts"; /** The `actor` scenario runner. */ export const actorRunner: ScenarioRunner = { - async run(context: RunContext) { - if (context.scenario.recipients.length < 1) { + validate(scenario): void { + if (scenario.recipients.length < 1) { throw new Error("The actor scenario requires a recipient."); } + for (const recipient of scenario.recipients) { + try { + convertUrlIfHandle(recipient); + } catch { + throw new Error( + `Scenario "${scenario.name}": invalid actor recipient ` + + `${JSON.stringify(recipient)}.`, + ); + } + } + }, + + async run(context: RunContext) { + this.validate?.(context.scenario); const urls = await actorUrlsFromRecipients(context.scenario.recipients, { target: context.target, fetch: context.fetch, diff --git a/packages/cli/src/bench/scenarios/object-discovery.ts b/packages/cli/src/bench/scenarios/object-discovery.ts index 8c4d32348..111913baa 100644 --- a/packages/cli/src/bench/scenarios/object-discovery.ts +++ b/packages/cli/src/bench/scenarios/object-discovery.ts @@ -10,6 +10,33 @@ import type { ObjectSource } from "../scenario/types.ts"; const ACTIVITY_JSON_ACCEPT = "application/activity+json, application/ld+json"; const WEBFINGER_ACCEPT = "application/jrd+json, application/json"; +const ACTIVITY_WRAPPER_TYPES = new Set([ + "Accept", + "Add", + "Announce", + "Create", + "Delete", + "Dislike", + "Flag", + "Ignore", + "Invite", + "Join", + "Leave", + "Like", + "Listen", + "Move", + "Offer", + "Question", + "Read", + "Reject", + "Remove", + "TentativeAccept", + "TentativeReject", + "Travel", + "Undo", + "Update", + "View", +]); /** Options for resolving actor URLs. */ export interface ActorUrlOptions { @@ -146,6 +173,7 @@ function objectUrl( item: unknown, types: ReadonlySet, ): URL | null { + item = unwrapActivityObject(item); if (typeof item === "string") { return types.size < 1 ? new URL(item) : null; } @@ -154,6 +182,15 @@ function objectUrl( return propertyUrl(item, "id"); } +function unwrapActivityObject(item: unknown): unknown { + if (!isRecord(item) || !matchesType(item.type, ACTIVITY_WRAPPER_TYPES)) { + return item; + } + const object = item.object; + if (Array.isArray(object)) return object.find((entry) => entry != null); + return object ?? item; +} + function matchesType( type: unknown, expected: ReadonlySet, diff --git a/packages/cli/src/bench/scenarios/object.test.ts b/packages/cli/src/bench/scenarios/object.test.ts index 3dd901cad..06b43ccdf 100644 --- a/packages/cli/src/bench/scenarios/object.test.ts +++ b/packages/cli/src/bench/scenarios/object.test.ts @@ -133,6 +133,134 @@ test("objectRunner - crawls actor collections before fetching objects", async () } }); +test("objectRunner - unwraps activities while crawling object sources", async () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "object-crawl", + type: "object", + source: { + seed: "http://target.test/users/alice", + collection: "outbox", + limit: 1, + type: "Note", + }, + load: { concurrency: 1 }, + duration: "25ms", + }], + }).scenarios[0]; + let fetchedObjectUrl = ""; + + const measurement = await objectRunner.run({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const request = input instanceof Request ? input : new Request(input); + const url = new URL(request.url); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return Promise.resolve(new Response("not found", { status: 404 })); + } + if (url.pathname === "/users/alice") { + return Promise.resolve(json({ + id: url.href, + outbox: "http://target.test/users/alice/outbox", + })); + } + if (url.pathname === "/users/alice/outbox") { + return Promise.resolve(json({ + id: url.href, + orderedItems: [{ + type: "Create", + id: "http://target.test/activities/create-1", + object: { + type: "Note", + id: "http://target.test/objects/1", + }, + }], + })); + } + if (url.pathname === "/objects/1") { + fetchedObjectUrl = url.href; + return Promise.resolve(json({ id: url.href, type: "Note" })); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }, + }); + + assert.ok(measurement.requests.total > 0); + assert.strictEqual(fetchedObjectUrl, "http://target.test/objects/1"); +}); + +test("objectRunner - prefers unwrapped object URLs without type filters", async () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "object-crawl", + type: "object", + source: { + seed: "http://target.test/users/alice", + collection: "outbox", + limit: 1, + }, + load: { concurrency: 1 }, + duration: "25ms", + }], + }).scenarios[0]; + let fetchedActivity = false; + let fetchedObject = false; + + const measurement = await objectRunner.run({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const request = input instanceof Request ? input : new Request(input); + const url = new URL(request.url); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return Promise.resolve(new Response("not found", { status: 404 })); + } + if (url.pathname === "/users/alice") { + return Promise.resolve(json({ + id: url.href, + outbox: "http://target.test/users/alice/outbox", + })); + } + if (url.pathname === "/users/alice/outbox") { + return Promise.resolve(json({ + id: url.href, + orderedItems: [{ + type: "Create", + id: "http://target.test/activities/create-1", + object: "http://target.test/objects/1", + }], + })); + } + if (url.pathname === "/activities/create-1") { + fetchedActivity = true; + return Promise.resolve(json({ id: url.href, type: "Create" })); + } + if (url.pathname === "/objects/1") { + fetchedObject = true; + return Promise.resolve(json({ id: url.href, type: "Note" })); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }, + }); + + assert.ok(measurement.requests.total > 0); + assert.strictEqual(fetchedObject, true); + assert.strictEqual(fetchedActivity, false); +}); + test("objectRunner - sends ActivityPub Accept headers during object discovery", async () => { const scenario = normalizeSuite({ version: 1, From fd026c03b79b6547141534a5aaa4923aa7639b93 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Wed, 10 Jun 2026 22:42:16 +0900 Subject: [PATCH 18/61] Reject mixed server metric gates Mixed scenarios currently merge client-side measurements but discard child server metrics, so server-side expectations would pass preflight and then fail after load with null actual values. Reject those expectations during mixed validation instead, before probing or sending benchmark load. https://github.com/fedify-dev/fedify/issues/785 Assisted-by: Codex:gpt-5.5 --- packages/cli/src/bench/action.test.ts | 36 +++++++++++++++++++ .../cli/src/bench/scenarios/mixed.test.ts | 18 ++++++++++ packages/cli/src/bench/scenarios/mixed.ts | 14 ++++++++ 3 files changed, 68 insertions(+) diff --git a/packages/cli/src/bench/action.test.ts b/packages/cli/src/bench/action.test.ts index a9b73be63..db0b6f849 100644 --- a/packages/cli/src/bench/action.test.ts +++ b/packages/cli/src/bench/action.test.ts @@ -869,6 +869,42 @@ scenarios: assert.strictEqual(fetched, false); }); +test("runBench - mixed server expectation exits 2 before any probe", async () => { + const file = await writeSuite(`version: 1 +target: http://localhost:3000 +scenarios: + - name: lookup + type: webfinger + recipient: acct:alice@example.com + - name: mixed + type: mixed + mix: + - scenario: lookup + weight: 1 + expect: + signatureVerification.p95: "< 10ms" +`); + let code = -1; + let message = ""; + let fetched = false; + await runBench(command({ scenario: file }), { + exit: (c) => { + code = c; + }, + writeOutput: () => Promise.resolve(), + log: (m) => { + message = m; + }, + fetch: () => { + fetched = true; + return Promise.reject(new Error("no request should be sent")); + }, + }); + assert.strictEqual(code, 2); + assert.match(message, /server-side expectations/); + assert.strictEqual(fetched, false); +}); + test("runBench - invalid object source exits 2 before any probe", async () => { const file = await writeSuite(`version: 1 target: http://localhost:3000 diff --git a/packages/cli/src/bench/scenarios/mixed.test.ts b/packages/cli/src/bench/scenarios/mixed.test.ts index 2eaf046d6..8245abade 100644 --- a/packages/cli/src/bench/scenarios/mixed.test.ts +++ b/packages/cli/src/bench/scenarios/mixed.test.ts @@ -243,6 +243,24 @@ test("mixedRunner.validate - rejects too-small closed load", () => { assert.throws(() => mixedRunner.validate?.(scenario), /concurrency/); }); +test("mixedRunner.validate - rejects server metric expectations", () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "mixed", + type: "mixed", + mix: [{ scenario: "fanout", weight: 1 }], + expect: { "queueDrain.p95": "< 1s" }, + }], + }).scenarios[0]; + + assert.throws( + () => mixedRunner.validate?.(scenario), + /server-side expectations/, + ); +}); + test("mergeMeasurements - merges latency histograms", () => { const measurement = mergeMeasurements([ fakeMeasurement(Array.from({ length: 99 }, () => 1)), diff --git a/packages/cli/src/bench/scenarios/mixed.ts b/packages/cli/src/bench/scenarios/mixed.ts index 3e78f9224..e498d9450 100644 --- a/packages/cli/src/bench/scenarios/mixed.ts +++ b/packages/cli/src/bench/scenarios/mixed.ts @@ -42,6 +42,15 @@ export const mixedRunner: ScenarioRunner = { "one concurrency slot per mix entry.", ); } + const serverExpectation = Object.keys(scenario.expect).find( + isServerExpectation, + ); + if (serverExpectation != null) { + throw new Error( + `Scenario "${scenario.name}": mixed server-side expectations are ` + + `not supported (${JSON.stringify(serverExpectation)}).`, + ); + } if (context?.scenarios != null) { const children = childScenarios(scenario, context.scenarios); for (const child of children) { @@ -75,6 +84,11 @@ export const mixedRunner: ScenarioRunner = { }, }; +function isServerExpectation(metric: string): boolean { + return metric.startsWith("signatureVerification.") || + metric.startsWith("queueDrain."); +} + function childScenarios( scenario: ResolvedScenario, scenarios: readonly ResolvedScenario[], From 17bdbe564f5bbfcb42516b6822b14fbd43ed2b4e Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 00:06:07 +0900 Subject: [PATCH 19/61] Reject invalid read destinations Validate actor and object read URLs before safety gates and load scheduling so malformed resolved destinations do not turn into full-duration failed sample runs. https://github.com/fedify-dev/fedify/issues/785 Assisted-by: Codex:gpt-5.5 --- packages/cli/src/bench/scenarios/read.test.ts | 50 +++++++++++++++++++ packages/cli/src/bench/scenarios/read.ts | 13 +++++ 2 files changed, 63 insertions(+) diff --git a/packages/cli/src/bench/scenarios/read.test.ts b/packages/cli/src/bench/scenarios/read.test.ts index 3b12b548a..eb82d3d30 100644 --- a/packages/cli/src/bench/scenarios/read.test.ts +++ b/packages/cli/src/bench/scenarios/read.test.ts @@ -49,6 +49,56 @@ test("runReadLoad - unauthenticated reads use the read destination gate", async assert.strictEqual(measurement.requests.successRate, 1); }); +test("runReadLoad - rejects invalid read URL schemes before load", async () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "read", + type: "object", + load: { concurrency: 1 }, + duration: "25ms", + }], + }).scenarios[0]; + let fetchCalls = 0; + let gateCalls = 0; + + for ( + const url of [ + new URL("ftp://remote.test/object"), + new URL("http://user:pass@remote.test/object"), + ] + ) { + await assert.rejects( + async () => + await runReadLoad({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ + allowPrivateAddress: true, + }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: () => { + fetchCalls++; + return Promise.resolve(new Response("{}", { status: 200 })); + }, + assertReadDestinationAllowed: () => { + gateCalls++; + }, + }, { + urls: [url], + authenticated: false, + }), + /read URL must be a bare http\(s\) URL/, + ); + } + + assert.strictEqual(fetchCalls, 0); + assert.strictEqual(gateCalls, 0); +}); + test("runReadLoad - authenticated reads support presign mode", async () => { let fleet: Awaited> | undefined; try { diff --git a/packages/cli/src/bench/scenarios/read.ts b/packages/cli/src/bench/scenarios/read.ts index 8e9bee2d0..5451fb59b 100644 --- a/packages/cli/src/bench/scenarios/read.ts +++ b/packages/cli/src/bench/scenarios/read.ts @@ -59,6 +59,7 @@ export async function runReadLoad( ); } for (const url of options.urls) { + validateReadUrl(context.scenario.name, url); if (options.authenticated) { await context.assertDestinationAllowed?.(url); } else { @@ -148,3 +149,15 @@ async function signGetRequest( { spec: actor.httpStandard }, ); } + +function validateReadUrl(scenarioName: string, url: URL): void { + if ( + (url.protocol !== "http:" && url.protocol !== "https:") || + url.hostname === "" || url.username !== "" || url.password !== "" + ) { + throw new Error( + `Scenario "${scenarioName}": read URL must be a bare http(s) URL with ` + + `a host and no credentials; got ${JSON.stringify(url.href)}.`, + ); + } +} From fc1390f86d1470a2a4291d081bcf2763c1dece37 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 00:40:23 +0900 Subject: [PATCH 20/61] Reject ambiguous benchmark preflight configs Fail actor and object read scenarios during runner validation when resolved URLs cannot be fetched safely, and reject mixed queue-observing children that would share unscoped target queue counters with concurrent queue producers. https://github.com/fedify-dev/fedify/issues/785 Assisted-by: Codex:gpt-5.5 --- .../cli/src/bench/scenarios/actor.test.ts | 19 ++++++ packages/cli/src/bench/scenarios/actor.ts | 16 ++++- .../cli/src/bench/scenarios/mixed.test.ts | 65 +++++++++++++++++++ packages/cli/src/bench/scenarios/mixed.ts | 36 ++++++++++ .../cli/src/bench/scenarios/object.test.ts | 24 +++++++ packages/cli/src/bench/scenarios/object.ts | 21 +++++- packages/cli/src/bench/scenarios/read.ts | 15 +---- packages/cli/src/bench/scenarios/runner.ts | 19 ++++++ 8 files changed, 197 insertions(+), 18 deletions(-) diff --git a/packages/cli/src/bench/scenarios/actor.test.ts b/packages/cli/src/bench/scenarios/actor.test.ts index d23cd70e8..bcdf8f15b 100644 --- a/packages/cli/src/bench/scenarios/actor.test.ts +++ b/packages/cli/src/bench/scenarios/actor.test.ts @@ -135,3 +135,22 @@ test("actorRunner - signs authenticated actor fetches", async () => { } } }); + +test("actorRunner.validate - rejects non-http actor recipient URLs", () => { + for (const recipient of ["ftp://target.test/users/alice", "mailto:alice"]) { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "actor", + type: "actor", + recipient, + }], + }).scenarios[0]; + + assert.throws( + () => actorRunner.validate?.(scenario), + /actor recipient must be an acct: handle or a bare http\(s\) URL/, + ); + } +}); diff --git a/packages/cli/src/bench/scenarios/actor.ts b/packages/cli/src/bench/scenarios/actor.ts index 0515ebc0c..f1e6ed4a5 100644 --- a/packages/cli/src/bench/scenarios/actor.ts +++ b/packages/cli/src/bench/scenarios/actor.ts @@ -7,7 +7,11 @@ import { convertUrlIfHandle } from "../../webfinger/lib.ts"; import { actorUrlsFromRecipients } from "./object-discovery.ts"; import { runReadLoad } from "./read.ts"; -import type { RunContext, ScenarioRunner } from "./runner.ts"; +import { + isBareHttpUrl, + type RunContext, + type ScenarioRunner, +} from "./runner.ts"; /** The `actor` scenario runner. */ export const actorRunner: ScenarioRunner = { @@ -16,14 +20,22 @@ export const actorRunner: ScenarioRunner = { throw new Error("The actor scenario requires a recipient."); } for (const recipient of scenario.recipients) { + let url: URL; try { - convertUrlIfHandle(recipient); + url = convertUrlIfHandle(recipient); } catch { throw new Error( `Scenario "${scenario.name}": invalid actor recipient ` + `${JSON.stringify(recipient)}.`, ); } + if (url.protocol !== "acct:" && !isBareHttpUrl(url)) { + throw new Error( + `Scenario "${scenario.name}": actor recipient must be an acct: ` + + `handle or a bare http(s) URL with a host and no credentials; ` + + `got ${JSON.stringify(url.href)}.`, + ); + } } }, diff --git a/packages/cli/src/bench/scenarios/mixed.test.ts b/packages/cli/src/bench/scenarios/mixed.test.ts index 8245abade..cd53b023c 100644 --- a/packages/cli/src/bench/scenarios/mixed.test.ts +++ b/packages/cli/src/bench/scenarios/mixed.test.ts @@ -261,6 +261,71 @@ test("mixedRunner.validate - rejects server metric expectations", () => { ); }); +test("mixedRunner.validate - rejects ambiguous target queue observation", () => { + const scenarios = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [ + { + name: "delivery", + type: "inbox", + recipient: "http://target.test/users/alice", + }, + { + name: "fanout", + type: "fanout", + sender: "alice", + }, + { + name: "mixed", + type: "mixed", + mix: [ + { scenario: "delivery", weight: 1 }, + { scenario: "fanout", weight: 1 }, + ], + }, + ], + }).scenarios; + + assert.throws( + () => mixedRunner.validate?.(scenarios[2], { scenarios }), + /target queue counters/, + ); +}); + +test("mixedRunner.validate - rejects remote failure with queue producers", () => { + const scenarios = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [ + { + name: "delivery", + type: "inbox", + recipient: "http://target.test/users/alice", + }, + { + name: "remote-failure", + type: "failure", + sender: "alice", + fault: "remote-404", + }, + { + name: "mixed", + type: "mixed", + mix: [ + { scenario: "delivery", weight: 1 }, + { scenario: "remote-failure", weight: 1 }, + ], + }, + ], + }).scenarios; + + assert.throws( + () => mixedRunner.validate?.(scenarios[2], { scenarios }), + /target queue counters/, + ); +}); + test("mergeMeasurements - merges latency histograms", () => { const measurement = mergeMeasurements([ fakeMeasurement(Array.from({ length: 99 }, () => 1)), diff --git a/packages/cli/src/bench/scenarios/mixed.ts b/packages/cli/src/bench/scenarios/mixed.ts index e498d9450..1dfd6a32f 100644 --- a/packages/cli/src/bench/scenarios/mixed.ts +++ b/packages/cli/src/bench/scenarios/mixed.ts @@ -56,6 +56,7 @@ export const mixedRunner: ScenarioRunner = { for (const child of children) { runnerForChild(child.type).validate?.(child, context); } + validateTargetQueueObservation(scenario, children); } }, @@ -89,6 +90,41 @@ function isServerExpectation(metric: string): boolean { metric.startsWith("queueDrain."); } +function validateTargetQueueObservation( + scenario: ResolvedScenario, + children: readonly ResolvedScenario[], +): void { + const observers = children.filter(observesTargetQueue); + if (observers.length < 1) return; + const producers = children.filter(producesTargetQueue); + if (producers.length <= 1) return; + throw new Error( + `Scenario "${scenario.name}": mixed scenarios cannot run queue-observing ` + + `children (${observers.map((child) => child.name).join(", ")}) ` + + `concurrently with other target queue producers because target queue ` + + `counters are not scoped per child.`, + ); +} + +function observesTargetQueue(scenario: ResolvedScenario): boolean { + return scenario.type === "fanout" || + (scenario.type === "failure" && faultsOf(scenario).some(isRemoteFault)); +} + +function producesTargetQueue(scenario: ResolvedScenario): boolean { + return scenario.type === "inbox" || scenario.type === "fanout" || + (scenario.type === "failure" && faultsOf(scenario).some(isRemoteFault)); +} + +function faultsOf(scenario: ResolvedScenario): readonly string[] { + return scenario.faults.length < 1 ? ["remote-404"] : scenario.faults; +} + +function isRemoteFault(fault: string): boolean { + return fault === "remote-404" || fault === "remote-410" || + fault === "slow-inbox" || fault === "network-error"; +} + function childScenarios( scenario: ResolvedScenario, scenarios: readonly ResolvedScenario[], diff --git a/packages/cli/src/bench/scenarios/object.test.ts b/packages/cli/src/bench/scenarios/object.test.ts index 06b43ccdf..fb58d67d2 100644 --- a/packages/cli/src/bench/scenarios/object.test.ts +++ b/packages/cli/src/bench/scenarios/object.test.ts @@ -486,6 +486,30 @@ test("objectRunner.validate - rejects malformed object source URLs", () => { ); }); +test("objectRunner.validate - rejects non-fetchable direct object source URLs", () => { + for ( + const source of [ + "ftp://target.test/objects/1", + "http://user:pass@target.test/objects/1", + ] + ) { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "object", + type: "object", + source, + }], + }).scenarios[0]; + + assert.throws( + () => objectRunner.validate?.(scenario), + /object source URL must be a bare http\(s\) URL/, + ); + } +}); + function json(body: unknown): Response { return new Response(JSON.stringify(body), { headers: { "content-type": "application/activity+json" }, diff --git a/packages/cli/src/bench/scenarios/object.ts b/packages/cli/src/bench/scenarios/object.ts index b482c6cef..5cad0dc0e 100644 --- a/packages/cli/src/bench/scenarios/object.ts +++ b/packages/cli/src/bench/scenarios/object.ts @@ -8,7 +8,12 @@ import { convertUrlIfHandle } from "../../webfinger/lib.ts"; import { asList } from "../scenario/coerce.ts"; import { objectUrlsFromSource } from "./object-discovery.ts"; import { runReadLoad } from "./read.ts"; -import type { RunContext, ScenarioRunner } from "./runner.ts"; +import { + assertBareHttpUrl, + isBareHttpUrl, + type RunContext, + type ScenarioRunner, +} from "./runner.ts"; /** The `object` scenario runner. */ export const objectRunner: ScenarioRunner = { @@ -17,26 +22,36 @@ export const objectRunner: ScenarioRunner = { if (source == null) return; if (typeof source === "string" || Array.isArray(source)) { for (const url of asList(source)) { + let parsed: URL; try { - new URL(url); + parsed = new URL(url); } catch { throw new Error( `Scenario "${scenario.name}": invalid object source URL ` + `${JSON.stringify(url)}.`, ); } + assertBareHttpUrl(scenario.name, "object source URL", parsed); } return; } for (const seed of asList(source.seed)) { + let url: URL; try { - convertUrlIfHandle(seed); + url = convertUrlIfHandle(seed); } catch { throw new Error( `Scenario "${scenario.name}": invalid object source seed URL ` + `${JSON.stringify(seed)}.`, ); } + if (url.protocol !== "acct:" && !isBareHttpUrl(url)) { + throw new Error( + `Scenario "${scenario.name}": object source seed must be an acct: ` + + `handle or a bare http(s) URL with a host and no credentials; ` + + `got ${JSON.stringify(url.href)}.`, + ); + } } }, diff --git a/packages/cli/src/bench/scenarios/read.ts b/packages/cli/src/bench/scenarios/read.ts index 5451fb59b..504321596 100644 --- a/packages/cli/src/bench/scenarios/read.ts +++ b/packages/cli/src/bench/scenarios/read.ts @@ -19,6 +19,7 @@ import { type SigningPipeline, } from "../signing/pipeline.ts"; import { + assertBareHttpUrl, estimateTotal, loadPlanOf, measuredWindowMs, @@ -59,7 +60,7 @@ export async function runReadLoad( ); } for (const url of options.urls) { - validateReadUrl(context.scenario.name, url); + assertBareHttpUrl(context.scenario.name, "read URL", url); if (options.authenticated) { await context.assertDestinationAllowed?.(url); } else { @@ -149,15 +150,3 @@ async function signGetRequest( { spec: actor.httpStandard }, ); } - -function validateReadUrl(scenarioName: string, url: URL): void { - if ( - (url.protocol !== "http:" && url.protocol !== "https:") || - url.hostname === "" || url.username !== "" || url.password !== "" - ) { - throw new Error( - `Scenario "${scenarioName}": read URL must be a bare http(s) URL with ` + - `a host and no credentials; got ${JSON.stringify(url.href)}.`, - ); - } -} diff --git a/packages/cli/src/bench/scenarios/runner.ts b/packages/cli/src/bench/scenarios/runner.ts index 8d2e83758..7fd4f2645 100644 --- a/packages/cli/src/bench/scenarios/runner.ts +++ b/packages/cli/src/bench/scenarios/runner.ts @@ -130,6 +130,25 @@ export function estimateTotal(scenario: ResolvedScenario): number | undefined { return Math.ceil(scenario.load.ratePerSec * (scenario.durationMs / 1000)); } +/** Returns whether a URL is fetchable by benchmark runners without surprises. */ +export function isBareHttpUrl(url: URL): boolean { + return (url.protocol === "http:" || url.protocol === "https:") && + url.hostname !== "" && url.username === "" && url.password === ""; +} + +/** Rejects URLs that are not bare http(s) URLs with a host and no credentials. */ +export function assertBareHttpUrl( + scenarioName: string, + label: string, + url: URL, +): void { + if (isBareHttpUrl(url)) return; + throw new Error( + `Scenario "${scenarioName}": ${label} must be a bare http(s) URL with ` + + `a host and no credentials; got ${JSON.stringify(url.href)}.`, + ); +} + /** * Wraps a send function so that `onMeasuredWindowStart` runs exactly once, at * the warm-up boundary, and *every* measured request waits for it to settle From 5c4033240eef7b677a9058c02f3de33544a48733 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 01:16:22 +0900 Subject: [PATCH 21/61] Observe network-error retry scheduling Treat a completed outbound task with a remaining retry as the expected network-error signal, matching Fedify's normal retry path where transport failures enqueue follow-up work instead of failing the current task. https://github.com/fedify-dev/fedify/issues/785 Assisted-by: Codex:gpt-5.5 --- .../cli/src/bench/scenarios/failure.test.ts | 48 +++++++++++++++++++ packages/cli/src/bench/scenarios/failure.ts | 10 +++- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/bench/scenarios/failure.test.ts b/packages/cli/src/bench/scenarios/failure.test.ts index d690b3724..47164e78e 100644 --- a/packages/cli/src/bench/scenarios/failure.test.ts +++ b/packages/cli/src/bench/scenarios/failure.test.ts @@ -71,6 +71,54 @@ for ( }); } +test("failureRunner - detects network-error retries", async () => { + const target = new URL("http://target.test/"); + const scenario = normalizeSuite({ + version: 1, + target: target.href, + scenarios: [{ + name: "failure", + type: "failure", + fault: "network-error", + sender: "alice", + load: { concurrency: 1 }, + duration: "25ms", + queueDrainTimeout: "50ms", + }], + }).scenarios[0]; + let triggerCalls = 0; + const measurement = await failureRunner.run({ + scenario, + target, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return Promise.resolve(statsJson(statsSnapshot({ + enqueued: triggerCalls * 2, + completed: triggerCalls, + failed: 0, + permanentFailures: 0, + }))); + } + if (url.pathname === "/.well-known/fedify/bench/trigger") { + triggerCalls++; + return Promise.resolve(statsJson({ version: 1 }, 202)); + } + return Promise.resolve(new Response("unexpected", { status: 500 })); + }, + assertDestinationAllowed: () => {}, + }); + + assert.ok(measurement.requests.total > 0); + assert.strictEqual(measurement.requests.failed, 0); + assert.strictEqual(measurement.requests.successRate, 1); + assert.ok(triggerCalls > 0); +}); + test("failureRunner.validate - requires sender for remote faults", () => { const scenario = normalizeSuite({ version: 1, diff --git a/packages/cli/src/bench/scenarios/failure.ts b/packages/cli/src/bench/scenarios/failure.ts index 2d11e2e3d..a6d73fe55 100644 --- a/packages/cli/src/bench/scenarios/failure.ts +++ b/packages/cli/src/bench/scenarios/failure.ts @@ -352,8 +352,14 @@ async function waitForRemoteFault(options: { if (queueTasks.completed > 0 && remaining === 0) { return { timedOut: false }; } - } else if (queueTasks.failed > 0) { - return { timedOut: false }; + } else if (options.fault === "network-error") { + const remaining = queueTaskRemaining(diff); + if (remaining == null) return null; + if ( + queueTasks.failed > 0 || (queueTasks.completed > 0 && remaining > 0) + ) { + return { timedOut: false }; + } } await new Promise((resolve) => setTimeout(resolve, DRAIN_POLL_MS)); } while (Date.now() < deadline); From 925289c4c9891c78fe139c606c6d9852cb6bbf62 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 06:07:43 +0900 Subject: [PATCH 22/61] Keep failed fanout metrics honest Only report fanout queue-drain latency after at least one measured drain sample, and validate explicit failure inbox selectors before discovery can turn typos into runtime URL errors. https://github.com/fedify-dev/fedify/issues/785 Assisted-by: Codex:gpt-5.5 --- .../cli/src/bench/scenarios/failure.test.ts | 19 ++++++++ packages/cli/src/bench/scenarios/failure.ts | 14 +++--- .../cli/src/bench/scenarios/fanout.test.ts | 44 +++++++++++++++++++ packages/cli/src/bench/scenarios/fanout.ts | 3 +- packages/cli/src/bench/scenarios/inbox.ts | 22 +--------- packages/cli/src/bench/scenarios/runner.ts | 25 +++++++++++ 6 files changed, 100 insertions(+), 27 deletions(-) diff --git a/packages/cli/src/bench/scenarios/failure.test.ts b/packages/cli/src/bench/scenarios/failure.test.ts index 47164e78e..e9bf0d9e9 100644 --- a/packages/cli/src/bench/scenarios/failure.test.ts +++ b/packages/cli/src/bench/scenarios/failure.test.ts @@ -147,6 +147,25 @@ test("failureRunner.validate - rejects unsupported faults", () => { assert.throws(() => failureRunner.validate?.(scenario), /unsupported/); }); +test("failureRunner.validate - rejects invalid explicit inbound inboxes", () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "failure", + type: "failure", + fault: "invalid-signature", + recipient: "http://target.test/users/alice", + inbox: "shraed", + }], + }).scenarios[0]; + + assert.throws( + () => failureRunner.validate?.(scenario), + /inbox must be "shared", "personal", or an http\(s\) URL/, + ); +}); + test("failureRunner - discovers inbound failure inboxes once", async () => { let actorGets = 0; const server = serve({ diff --git a/packages/cli/src/bench/scenarios/failure.ts b/packages/cli/src/bench/scenarios/failure.ts index a6d73fe55..d9d86d8df 100644 --- a/packages/cli/src/bench/scenarios/failure.ts +++ b/packages/cli/src/bench/scenarios/failure.ts @@ -23,6 +23,7 @@ import { type RunContext, type ScenarioRunner, sendRequest, + validateInboxSelector, } from "./runner.ts"; const SUPPORTED_FAULTS = [ @@ -69,11 +70,10 @@ export const failureRunner: ScenarioRunner = { ); } } - if ( - faults.some((fault) => - fault === "invalid-signature" || fault === "missing-actor" - ) && scenario.recipients.length < 1 - ) { + if (faults.some(isInboundFault)) { + validateInboxSelector(scenario.name, scenario.inbox); + } + if (faults.some(isInboundFault) && scenario.recipients.length < 1) { throw new Error( `Scenario "${scenario.name}": invalid-signature and missing-actor ` + "faults require a recipient.", @@ -169,7 +169,9 @@ async function resolveFailureDeliveryTarget( return { inbox, actorUri: discovered.actorUri }; } -function isInboundFault(fault: SupportedFault): boolean { +function isInboundFault( + fault: string, +): fault is Extract { return fault === "invalid-signature" || fault === "missing-actor"; } diff --git a/packages/cli/src/bench/scenarios/fanout.test.ts b/packages/cli/src/bench/scenarios/fanout.test.ts index 879f0bad7..2a345db1c 100644 --- a/packages/cli/src/bench/scenarios/fanout.test.ts +++ b/packages/cli/src/bench/scenarios/fanout.test.ts @@ -178,6 +178,50 @@ test("fanoutRunner - counts failed queue tasks as delivery failures", async () = assert.strictEqual(measurement.deliveryThroughputPerSec, 0); }); +test("fanoutRunner - omits queue drain metrics without drain samples", async () => { + const target = new URL("http://target.test/"); + const scenario = normalizeSuite({ + version: 1, + target: target.href, + scenarios: [{ + name: "fanout", + type: "fanout", + sender: "alice", + followers: 5, + load: { concurrency: 1 }, + duration: "30ms", + queueDrainTimeout: "1s", + }], + }).scenarios[0]; + + const measurement = await fanoutRunner.run({ + scenario, + target, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return Promise.resolve(json(statsSnapshot({ + enqueued: 0, + completed: 0, + failed: 0, + }))); + } + if (url.pathname === "/.well-known/fedify/bench/trigger") { + return Promise.resolve(json({ error: "unavailable" }, 503)); + } + return Promise.resolve(new Response("unexpected", { status: 500 })); + }, + }); + + assert.ok(measurement.requests.total > 0); + assert.strictEqual(measurement.requests.successRate, 0); + assert.strictEqual(measurement.server?.queue?.drainMs, undefined); +}); + function json(body: unknown, status = 200): Response { return new Response(JSON.stringify(body), { status, diff --git a/packages/cli/src/bench/scenarios/fanout.ts b/packages/cli/src/bench/scenarios/fanout.ts index f6c759cd7..4cce8abe7 100644 --- a/packages/cli/src/bench/scenarios/fanout.ts +++ b/packages/cli/src/bench/scenarios/fanout.ts @@ -272,7 +272,8 @@ async function waitForDrain(options: { function addQueueDrain( server: ServerMetrics | null, histogram: LogLinearHistogram, -): ServerMetrics { +): ServerMetrics | null { + if (histogram.count < 1) return server; const queue = { ...(server?.queue ?? {}), drainMs: partialFromHistogram(histogram), diff --git a/packages/cli/src/bench/scenarios/inbox.ts b/packages/cli/src/bench/scenarios/inbox.ts index e23d88b78..4bf8379d0 100644 --- a/packages/cli/src/bench/scenarios/inbox.ts +++ b/packages/cli/src/bench/scenarios/inbox.ts @@ -38,6 +38,7 @@ import { type RunContext, type ScenarioRunner, sendRequest, + validateInboxSelector, withMeasuredWindowStart, } from "./runner.ts"; @@ -163,26 +164,7 @@ export const inboxRunner: ScenarioRunner = { * mid-run, and a non-http URL would slip through to the send path. */ function validateInbox(scenario: ResolvedScenario): void { - const mode = scenario.inbox; - if (mode == null || mode === "shared" || mode === "personal") return; - let url: URL; - try { - url = new URL(mode); - } catch { - throw new Error( - `Scenario "${scenario.name}": inbox must be "shared", "personal", or an ` + - `http(s) URL; got ${JSON.stringify(mode)}.`, - ); - } - if ( - (url.protocol !== "http:" && url.protocol !== "https:") || - url.hostname === "" || url.username !== "" || url.password !== "" - ) { - throw new Error( - `Scenario "${scenario.name}": inbox URL must be a bare http(s) URL with ` + - `a host and no credentials; got ${JSON.stringify(mode)}.`, - ); - } + validateInboxSelector(scenario.name, scenario.inbox); } /** diff --git a/packages/cli/src/bench/scenarios/runner.ts b/packages/cli/src/bench/scenarios/runner.ts index 7fd4f2645..4ebfb0ce9 100644 --- a/packages/cli/src/bench/scenarios/runner.ts +++ b/packages/cli/src/bench/scenarios/runner.ts @@ -149,6 +149,31 @@ export function assertBareHttpUrl( ); } +/** Validates an inbox selector or explicit inbox URL. */ +export function validateInboxSelector( + scenarioName: string, + inbox: string | undefined, +): void { + if (inbox == null || inbox === "shared" || inbox === "personal") return; + let url: URL; + try { + url = new URL(inbox); + } catch { + throw new Error( + `Scenario "${scenarioName}": inbox must be "shared", "personal", or an ` + + `http(s) URL; got ${JSON.stringify(inbox)}.`, + ); + } + try { + assertBareHttpUrl(scenarioName, "inbox URL", url); + } catch { + throw new Error( + `Scenario "${scenarioName}": inbox URL must be a bare http(s) URL with ` + + `a host and no credentials; got ${JSON.stringify(inbox)}.`, + ); + } +} + /** * Wraps a send function so that `onMeasuredWindowStart` runs exactly once, at * the warm-up boundary, and *every* measured request waits for it to settle From 0a1afeb86e75bb095276f5d37ac857483cc9f86b Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 13:02:08 +0900 Subject: [PATCH 23/61] Settle mixed runs before reporting errors Select typed objects from every unwrapped activity object candidate, and wait for all mixed child runners to settle before rethrowing a child setup error. https://github.com/fedify-dev/fedify/issues/785 Assisted-by: Codex:gpt-5.5 --- .../cli/src/bench/scenarios/mixed.test.ts | 78 +++++++++++++++++++ packages/cli/src/bench/scenarios/mixed.ts | 9 ++- .../src/bench/scenarios/object-discovery.ts | 24 +++--- .../cli/src/bench/scenarios/object.test.ts | 64 +++++++++++++++ 4 files changed, 164 insertions(+), 11 deletions(-) diff --git a/packages/cli/src/bench/scenarios/mixed.test.ts b/packages/cli/src/bench/scenarios/mixed.test.ts index cd53b023c..2594d7850 100644 --- a/packages/cli/src/bench/scenarios/mixed.test.ts +++ b/packages/cli/src/bench/scenarios/mixed.test.ts @@ -126,6 +126,84 @@ test("mixedRunner - enforces parent maxInFlight across children", async () => { ); }); +test("mixedRunner - waits for siblings before propagating child errors", async () => { + const target = new URL("http://target.test/"); + const suite: Suite = { + version: 1, + target: target.href, + scenarios: [ + { + name: "lookup", + type: "webfinger", + recipient: "acct:alice@target.test", + }, + { + name: "object", + type: "object", + source: { + seed: "http://target.test/users/missing", + collection: "outbox", + limit: 1, + }, + }, + { + name: "mixed", + type: "mixed", + load: { concurrency: 2 }, + duration: "40ms", + mix: [ + { scenario: "lookup", weight: 1 }, + { scenario: "object", weight: 1 }, + ], + }, + ], + }; + const scenarios = normalizeSuite(suite).scenarios; + let webfingerStarted = false; + let webfingerSettled = false; + + await assert.rejects( + async () => + await mixedRunner.run({ + scenario: scenarios[2], + scenarios, + target, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: async (input) => { + const request = input instanceof Request ? input : new Request(input); + const url = new URL(request.url); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return new Response("not found", { status: 404 }); + } + if (url.pathname === "/.well-known/webfinger") { + webfingerStarted = true; + await new Promise((resolve) => setTimeout(resolve, 30)); + webfingerSettled = true; + return json({ + subject: url.searchParams.get("resource"), + links: [], + }); + } + if (url.pathname === "/users/missing") { + const deadline = Date.now() + 100; + while (!webfingerStarted && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 1)); + } + await new Promise((resolve) => setTimeout(resolve, 5)); + return new Response("not found", { status: 404 }); + } + return new Response("unexpected", { status: 500 }); + }, + }), + /Failed to fetch http:\/\/target\.test\/users\/missing: HTTP 404/, + ); + + assert.strictEqual(webfingerSettled, true); +}); + test("mixedRunner - rejects unknown children", async () => { const scenarios = normalizeSuite({ version: 1, diff --git a/packages/cli/src/bench/scenarios/mixed.ts b/packages/cli/src/bench/scenarios/mixed.ts index 1dfd6a32f..7e7455e4a 100644 --- a/packages/cli/src/bench/scenarios/mixed.ts +++ b/packages/cli/src/bench/scenarios/mixed.ts @@ -72,7 +72,7 @@ export const mixedRunner: ScenarioRunner = { context.fetch ?? fetch, context.scenario.load.maxInFlight, ); - const measurements = await Promise.all( + const results = await Promise.allSettled( children.map((child) => runnerForChild(child.type).run({ ...context, @@ -81,6 +81,13 @@ export const mixedRunner: ScenarioRunner = { }) ), ); + const rejected = results.find((result) => result.status === "rejected"); + if (rejected != null) throw rejected.reason; + const measurements = results + .filter((result): result is PromiseFulfilledResult => + result.status === "fulfilled" + ) + .map((result) => result.value); return mergeMeasurements(measurements); }, }; diff --git a/packages/cli/src/bench/scenarios/object-discovery.ts b/packages/cli/src/bench/scenarios/object-discovery.ts index 111913baa..252c45ced 100644 --- a/packages/cli/src/bench/scenarios/object-discovery.ts +++ b/packages/cli/src/bench/scenarios/object-discovery.ts @@ -173,22 +173,26 @@ function objectUrl( item: unknown, types: ReadonlySet, ): URL | null { - item = unwrapActivityObject(item); - if (typeof item === "string") { - return types.size < 1 ? new URL(item) : null; + for (const candidate of objectCandidates(item)) { + if (typeof candidate === "string") { + if (types.size < 1) return new URL(candidate); + continue; + } + if (!isRecord(candidate)) continue; + if (types.size > 0 && !matchesType(candidate.type, types)) continue; + const url = propertyUrl(candidate, "id"); + if (url != null) return url; } - if (!isRecord(item)) return null; - if (types.size > 0 && !matchesType(item.type, types)) return null; - return propertyUrl(item, "id"); + return null; } -function unwrapActivityObject(item: unknown): unknown { +function objectCandidates(item: unknown): unknown[] { if (!isRecord(item) || !matchesType(item.type, ACTIVITY_WRAPPER_TYPES)) { - return item; + return [item]; } const object = item.object; - if (Array.isArray(object)) return object.find((entry) => entry != null); - return object ?? item; + if (Array.isArray(object)) return object.filter((entry) => entry != null); + return [object ?? item]; } function matchesType( diff --git a/packages/cli/src/bench/scenarios/object.test.ts b/packages/cli/src/bench/scenarios/object.test.ts index fb58d67d2..db19a7429 100644 --- a/packages/cli/src/bench/scenarios/object.test.ts +++ b/packages/cli/src/bench/scenarios/object.test.ts @@ -196,6 +196,70 @@ test("objectRunner - unwraps activities while crawling object sources", async () assert.strictEqual(fetchedObjectUrl, "http://target.test/objects/1"); }); +test("objectRunner - selects matching objects from activity arrays", async () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "object-crawl", + type: "object", + source: { + seed: "http://target.test/users/alice", + collection: "outbox", + limit: 1, + type: "Note", + }, + load: { concurrency: 1 }, + duration: "25ms", + }], + }).scenarios[0]; + let fetchedObjectUrl = ""; + + const measurement = await objectRunner.run({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const request = input instanceof Request ? input : new Request(input); + const url = new URL(request.url); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return Promise.resolve(new Response("not found", { status: 404 })); + } + if (url.pathname === "/users/alice") { + return Promise.resolve(json({ + id: url.href, + outbox: "http://target.test/users/alice/outbox", + })); + } + if (url.pathname === "/users/alice/outbox") { + return Promise.resolve(json({ + id: url.href, + orderedItems: [{ + type: "Create", + id: "http://target.test/activities/create-1", + object: [ + { type: "Article", id: "http://target.test/objects/article" }, + "http://target.test/objects/url-only", + { type: "Note", id: "http://target.test/objects/note" }, + ], + }], + })); + } + if (url.pathname === "/objects/note") { + fetchedObjectUrl = url.href; + return Promise.resolve(json({ id: url.href, type: "Note" })); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }, + }); + + assert.ok(measurement.requests.total > 0); + assert.strictEqual(fetchedObjectUrl, "http://target.test/objects/note"); +}); + test("objectRunner - prefers unwrapped object URLs without type filters", async () => { const scenario = normalizeSuite({ version: 1, From f0824071472850969c77050b92300c21ca9c1101 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 13:51:58 +0900 Subject: [PATCH 24/61] Update benchmark scenario help text Describe the newly executable benchmark scenario types in the CLI help and keep nearby schema/type comments aligned with collection being the only reserved scenario type. https://github.com/fedify-dev/fedify/issues/785 Assisted-by: Codex:gpt-5.5 --- .../cli/src/bench/__fixtures__/scenarios/all-types.yaml | 4 ++-- packages/cli/src/bench/command.ts | 7 ++++--- packages/cli/src/bench/scenario/schema.ts | 4 ++-- packages/cli/src/bench/scenario/types.ts | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/cli/src/bench/__fixtures__/scenarios/all-types.yaml b/packages/cli/src/bench/__fixtures__/scenarios/all-types.yaml index c83a775d1..3e2324726 100644 --- a/packages/cli/src/bench/__fixtures__/scenarios/all-types.yaml +++ b/packages/cli/src/bench/__fixtures__/scenarios/all-types.yaml @@ -1,6 +1,6 @@ # yaml-language-server: $schema=https://json-schema.fedify.dev/bench/scenario-v1.json -# Exercises every scenario type the format can express, even though only -# `inbox` and `webfinger` have runners in this version. +# Exercises every scenario type the format can express; `collection` is still +# reserved but not executable in this version. version: 1 target: http://localhost:3000 defaults: diff --git a/packages/cli/src/bench/command.ts b/packages/cli/src/bench/command.ts index 553ecc39c..1104753b3 100644 --- a/packages/cli/src/bench/command.ts +++ b/packages/cli/src/bench/command.ts @@ -98,9 +98,10 @@ host in the actor and key URLs the target dereferences.`, description: message`Run an ActivityPub-specific load benchmark against a \ cooperative Fedify target running in benchmark mode. -The suite file declares the target, actors, and scenarios. Only the \`inbox\` \ -and \`webfinger\` scenario types are executed in this version; the format \ -itself can express every scenario type.`, +The suite file declares the target, actors, and scenarios. This version \ +executes the \`inbox\`, \`webfinger\`, \`actor\`, \`object\`, \`fanout\`, \ +\`failure\`, and \`mixed\` scenario types; \`collection\` remains reserved by \ +the suite format.`, }, ); diff --git a/packages/cli/src/bench/scenario/schema.ts b/packages/cli/src/bench/scenario/schema.ts index 83c4983b5..d4894ef83 100644 --- a/packages/cli/src/bench/scenario/schema.ts +++ b/packages/cli/src/bench/scenario/schema.ts @@ -7,8 +7,8 @@ * * The schema expresses every scenario type discussed for `fedify bench` * (`inbox`, `webfinger`, `actor`, `object`, `fanout`, `collection`, `failure`, - * `mixed`), even though only `inbox` and `webfinger` have runners in this - * version. Three cross-field rules are enforced here rather than in code: + * `mixed`). All but `collection` have runners in this version. Three + * cross-field rules are enforced here rather than in code: * * - exactly one HTTP request signature scheme per actor group * (`contains` + `minContains`/`maxContains`); diff --git a/packages/cli/src/bench/scenario/types.ts b/packages/cli/src/bench/scenario/types.ts index 18a295480..7620c84de 100644 --- a/packages/cli/src/bench/scenario/types.ts +++ b/packages/cli/src/bench/scenario/types.ts @@ -24,7 +24,7 @@ export const HTTP_SIGNATURE_STANDARDS: readonly SignatureStandard[] = [ "rfc9421", ]; -/** A scenario type. Only `inbox` and `webfinger` have runners so far. */ +/** A scenario type. `collection` is reserved but not executable so far. */ export type ScenarioType = | "inbox" | "webfinger" From e57cc8435224c19f504a68e540e03f2c99709712 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 14:34:21 +0900 Subject: [PATCH 25/61] Make failure discovery test deterministic Use an open-loop load and a test clock so the inbound failure discovery test always sends the expected samples while still asserting that recipient discovery runs only once. https://github.com/fedify-dev/fedify/issues/785 Assisted-by: Codex:gpt-5.5 --- packages/cli/src/bench/scenarios/failure.test.ts | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/bench/scenarios/failure.test.ts b/packages/cli/src/bench/scenarios/failure.test.ts index e9bf0d9e9..f9c4a64e7 100644 --- a/packages/cli/src/bench/scenarios/failure.test.ts +++ b/packages/cli/src/bench/scenarios/failure.test.ts @@ -3,6 +3,7 @@ import test from "node:test"; import { serve } from "srvx"; import { buildFleet } from "../actor/fleet.ts"; import { getContextLoader, getDocumentLoader } from "../../docloader.ts"; +import type { Clock } from "../load/clock.ts"; import { normalizeSuite } from "../scenario/normalize.ts"; import type { Suite } from "../scenario/types.ts"; import { spawnSyntheticServer } from "../server/synthetic.ts"; @@ -207,10 +208,18 @@ test("failureRunner - discovers inbound failure inboxes once", async () => { type: "failure", fault: "invalid-signature", recipient: new URL("/users/alice", target).href, - load: { concurrency: 1 }, - duration: "80ms", + load: { rate: 100 }, + duration: "30ms", }], }).scenarios[0]; + let now = 0; + const clock: Clock = { + now: () => now, + sleepUntil: (timeMs) => { + now = Math.max(now, timeMs); + return Promise.resolve(); + }, + }; const measurement = await failureRunner.run({ scenario, target, @@ -230,9 +239,10 @@ test("failureRunner - discovers inbound failure inboxes once", async () => { return Promise.resolve(new Response("not found", { status: 404 })); }, assertDestinationAllowed: () => {}, + clock, }); - assert.ok(measurement.requests.total > 1); + assert.strictEqual(measurement.requests.total, 3); assert.strictEqual(measurement.requests.successRate, 1); assert.strictEqual(actorGets, 1); } finally { From 9aeff9d03aa16a27344c5743309267a1f807be21 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 15:23:52 +0900 Subject: [PATCH 26/61] Resolve relative object discovery links Resolve collection, pagination, and object identifiers against the response URL that supplied them. This keeps object crawl discovery from crashing when an ActivityPub server returns relative links. https://github.com/fedify-dev/fedify/pull/802#discussion_r3393449051 https://github.com/fedify-dev/fedify/pull/802#discussion_r3393449057 https://github.com/fedify-dev/fedify/pull/802#discussion_r3393449062 https://github.com/fedify-dev/fedify/pull/802#discussion_r3393495916 Assisted-by: Codex:gpt-5.5 --- .../src/bench/scenarios/object-discovery.ts | 30 ++++--- .../cli/src/bench/scenarios/object.test.ts | 80 +++++++++++++++++++ 2 files changed, 100 insertions(+), 10 deletions(-) diff --git a/packages/cli/src/bench/scenarios/object-discovery.ts b/packages/cli/src/bench/scenarios/object-discovery.ts index 252c45ced..fc1f0844f 100644 --- a/packages/cli/src/bench/scenarios/object-discovery.ts +++ b/packages/cli/src/bench/scenarios/object-discovery.ts @@ -38,14 +38,22 @@ const ACTIVITY_WRAPPER_TYPES = new Set([ "View", ]); -/** Options for resolving actor URLs. */ +/** + * Options for resolving actor URLs from recipients. + * @property target The benchmark target base URL. + * @property fetch Fetch implementation used for WebFinger discovery. + * @property assertReadDestinationAllowed Optional gate for discovered read URLs. + */ export interface ActorUrlOptions { readonly target: URL; readonly fetch?: typeof fetch; readonly assertReadDestinationAllowed?: (url: URL) => void | Promise; } -/** Options for resolving object URLs. */ +/** + * Options for resolving object URLs from source definitions. + * @property source The explicit object URL list or crawl source to resolve. + */ export interface ObjectUrlOptions extends ActorUrlOptions { readonly source: ObjectSource | undefined; } @@ -79,7 +87,7 @@ export async function objectUrlsFromSource( await options.assertReadDestinationAllowed?.(actorUrl); const actor = await fetchJson(actorUrl, options.fetch); for (const collectionName of asList(source.collection ?? "outbox")) { - const collectionUrl = propertyUrl(actor, collectionName); + const collectionUrl = propertyUrl(actor, collectionName, actorUrl); if (collectionUrl == null) continue; for await ( const objectUrl of crawlCollection(collectionUrl, { @@ -136,14 +144,14 @@ async function* crawlCollection( const items = arrayProperty(page, "orderedItems") ?? arrayProperty(page, "items") ?? []; for (const item of items) { - const url = objectUrl(item, options.types); + const url = objectUrl(item, options.types, next); if (url == null) continue; yield url; remaining--; if (remaining <= 0) return; } - const first = propertyUrl(page, "first"); - const following = propertyUrl(page, "next"); + const first = propertyUrl(page, "first", next); + const following = propertyUrl(page, "next", next); next = following ?? (next.href === start.href ? first : null); } } @@ -172,15 +180,16 @@ async function fetchJson( function objectUrl( item: unknown, types: ReadonlySet, + base: URL, ): URL | null { for (const candidate of objectCandidates(item)) { if (typeof candidate === "string") { - if (types.size < 1) return new URL(candidate); + if (types.size < 1) return new URL(candidate, base); continue; } if (!isRecord(candidate)) continue; if (types.size > 0 && !matchesType(candidate.type, types)) continue; - const url = propertyUrl(candidate, "id"); + const url = propertyUrl(candidate, "id", base); if (url != null) return url; } return null; @@ -207,11 +216,12 @@ function matchesType( function propertyUrl( object: Record, key: string, + base?: URL, ): URL | null { const value = object[key]; - if (typeof value === "string") return new URL(value); + if (typeof value === "string") return new URL(value, base); if (isRecord(value) && typeof value.id === "string") { - return new URL(value.id); + return new URL(value.id, base); } return null; } diff --git a/packages/cli/src/bench/scenarios/object.test.ts b/packages/cli/src/bench/scenarios/object.test.ts index db19a7429..b8f7ec371 100644 --- a/packages/cli/src/bench/scenarios/object.test.ts +++ b/packages/cli/src/bench/scenarios/object.test.ts @@ -518,6 +518,86 @@ test("objectRunner - gates collection URLs before crawling them", async () => { ); }); +test("objectRunner - resolves relative URLs while crawling object sources", async () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "object-crawl", + type: "object", + source: { + seed: "http://target.test/users/alice/", + collection: "outbox", + limit: 1, + type: "Note", + }, + load: { concurrency: 1 }, + duration: "25ms", + }], + }).scenarios[0]; + const fetched: string[] = []; + + const measurement = await objectRunner.run({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const request = input instanceof Request ? input : new Request(input); + const url = new URL(request.url); + fetched.push(url.href); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return Promise.resolve(new Response("not found", { status: 404 })); + } + if (url.pathname === "/users/alice/") { + return Promise.resolve(json({ + id: url.href, + outbox: "outbox", + })); + } + if (url.pathname === "/users/alice/outbox" && url.search === "?page=1") { + return Promise.resolve(json({ + id: url.href, + next: "?page=2", + orderedItems: [], + })); + } + if (url.pathname === "/users/alice/outbox" && url.search === "?page=2") { + return Promise.resolve(json({ + id: url.href, + orderedItems: [{ + type: "Create", + object: { + type: "Note", + id: "./objects/relative", + }, + }], + })); + } + if (url.pathname === "/users/alice/outbox") { + return Promise.resolve(json({ + id: url.href, + first: "?page=1", + })); + } + if (url.pathname === "/users/alice/objects/relative") { + return Promise.resolve(json({ id: url.href, type: "Note" })); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }, + }); + + assert.ok(measurement.requests.total > 0); + assert.ok(fetched.includes("http://target.test/users/alice/outbox")); + assert.ok(fetched.includes("http://target.test/users/alice/outbox?page=1")); + assert.ok(fetched.includes("http://target.test/users/alice/outbox?page=2")); + assert.ok( + fetched.includes("http://target.test/users/alice/objects/relative"), + ); +}); + test("objectRunner.validate - rejects malformed object source URLs", () => { const explicit = normalizeSuite({ version: 1, From 5c40e2b3ba4c0e7c639c104273801599c80cebfc Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 15:25:55 +0900 Subject: [PATCH 27/61] Document serialized benchmark wait latency Explain that fanout and remote failure scenarios serialize queue observation, so client request latency can include time spent waiting for earlier queue drains or expected failure signals under concurrent load. https://github.com/fedify-dev/fedify/pull/802#discussion_r3393449063 https://github.com/fedify-dev/fedify/pull/802#discussion_r3393449066 Assisted-by: Codex:gpt-5.5 --- docs/manual/benchmarking.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/manual/benchmarking.md b/docs/manual/benchmarking.md index be136e933..73a735e8e 100644 --- a/docs/manual/benchmarking.md +++ b/docs/manual/benchmarking.md @@ -182,7 +182,11 @@ The runnable scenario types cover the main benchmark surfaces: `allowUnsafeTriggerRecipients` in a controlled benchmark environment. `fedify bench` does not switch the target's queue backend; run the same suite against targets configured with the queue implementations you want to - compare. + compare. Fanout triggers are serialized while the runner observes queue + drain, so client latency includes time spent waiting for earlier fanout + drains under high-rate or concurrent load. Use `deliveryThroughput` and + `queueDrain` expectations for delivery performance, and keep request + latency expectations conservative for this scenario type. - `failure`: records expected fault outcomes as successes. For this scenario type, `successRate` means “the expected failure was observed,” not “the HTTP request succeeded.” The `invalid-signature` and @@ -191,7 +195,10 @@ The runnable scenario types cover the main benchmark surfaces: faults post to the benchmark trigger endpoint with `sender`, so the target uses its normal outbound delivery path against controlled benchmark-owned sink inboxes. Like `fanout`, these remote failure faults need - `--advertise-host` for a non-loopback target. + `--advertise-host` for a non-loopback target. Remote failure deliveries + are also serialized while the runner waits for the target's queue to + observe the expected failure or retry signal, so request latency can include + earlier wait time when the configured load is concurrent or high-rate. - `mixed`: runs referenced child scenarios concurrently, splitting the `mixed` scenario's load by each entry's `weight`. The referenced scenarios are named scenarios in the same suite and are still run as normal From 63ad3c5c763bbd1ea524890bdf26219a3a4a63b0 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 15:26:02 +0900 Subject: [PATCH 28/61] Tighten benchmark helper documentation Expand the queue backlog helper documentation and let the shared bare URL validator report explicit inbox URL failures directly. https://github.com/fedify-dev/fedify/pull/802#discussion_r3393495911 https://github.com/fedify-dev/fedify/pull/802#discussion_r3393495919 Assisted-by: Codex:gpt-5.5 --- packages/cli/src/bench/metrics/stats-client.ts | 8 +++++++- packages/cli/src/bench/scenarios/runner.ts | 9 +-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/packages/cli/src/bench/metrics/stats-client.ts b/packages/cli/src/bench/metrics/stats-client.ts index 2b805ae82..018b45767 100644 --- a/packages/cli/src/bench/metrics/stats-client.ts +++ b/packages/cli/src/bench/metrics/stats-client.ts @@ -228,7 +228,13 @@ export async function fetchServerMetrics( return snapshotToMetrics(await fetchServerSnapshot(target, fetchImpl)); } -/** Returns the queue task backlog represented by a diffed snapshot. */ +/** + * Returns the remaining queue task backlog represented by a diffed snapshot. + * @param snapshot The server snapshot to inspect, usually already diffed + * against a baseline. + * @returns `Math.max(0, enqueued - completed - failed)`, or `null` when the + * snapshot has no queue task counters. + */ export function queueTaskRemaining( snapshot: ServerSnapshot | null, ): number | null { diff --git a/packages/cli/src/bench/scenarios/runner.ts b/packages/cli/src/bench/scenarios/runner.ts index 4ebfb0ce9..b34211e48 100644 --- a/packages/cli/src/bench/scenarios/runner.ts +++ b/packages/cli/src/bench/scenarios/runner.ts @@ -164,14 +164,7 @@ export function validateInboxSelector( `http(s) URL; got ${JSON.stringify(inbox)}.`, ); } - try { - assertBareHttpUrl(scenarioName, "inbox URL", url); - } catch { - throw new Error( - `Scenario "${scenarioName}": inbox URL must be a bare http(s) URL with ` + - `a host and no credentials; got ${JSON.stringify(inbox)}.`, - ); - } + assertBareHttpUrl(scenarioName, "inbox URL", url); } /** From fa3ebf249950ae0fefb8266e312046ff63c6a90e Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 16:17:40 +0900 Subject: [PATCH 29/61] Tolerate transient stats polling failures Keep polling fanout drains and remote failure observations when one stats snapshot request fails or temporarily omits the queue counters. A transient stats blip should not abort the scenario before the configured timeout. https://github.com/fedify-dev/fedify/pull/802#discussion_r3393774840 https://github.com/fedify-dev/fedify/pull/802#discussion_r3393774846 Assisted-by: Codex:gpt-5.5 --- .../cli/src/bench/scenarios/failure.test.ts | 57 +++++++++++++++++++ packages/cli/src/bench/scenarios/failure.ts | 44 +++++++------- .../cli/src/bench/scenarios/fanout.test.ts | 55 ++++++++++++++++++ packages/cli/src/bench/scenarios/fanout.ts | 15 +++-- 4 files changed, 142 insertions(+), 29 deletions(-) diff --git a/packages/cli/src/bench/scenarios/failure.test.ts b/packages/cli/src/bench/scenarios/failure.test.ts index f9c4a64e7..93a034586 100644 --- a/packages/cli/src/bench/scenarios/failure.test.ts +++ b/packages/cli/src/bench/scenarios/failure.test.ts @@ -120,6 +120,63 @@ test("failureRunner - detects network-error retries", async () => { assert.ok(triggerCalls > 0); }); +test("failureRunner - tolerates transient remote fault stats failures", async () => { + const target = new URL("http://target.test/"); + const scenario = normalizeSuite({ + version: 1, + target: target.href, + scenarios: [{ + name: "failure", + type: "failure", + fault: "remote-404", + sender: "alice", + load: { concurrency: 1 }, + duration: "25ms", + queueDrainTimeout: "1s", + }], + }).scenarios[0]; + let statsCalls = 0; + let triggerCalls = 0; + const measurement = await failureRunner.run({ + scenario, + target, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/.well-known/fedify/bench/stats") { + statsCalls++; + if (statsCalls === 2) { + return Promise.resolve( + new Response("temporarily unavailable", { + status: 503, + }), + ); + } + return Promise.resolve(statsJson(statsSnapshot({ + enqueued: triggerCalls, + completed: triggerCalls, + failed: 0, + permanentFailures: statsCalls > 1 ? triggerCalls : 0, + }))); + } + if (url.pathname === "/.well-known/fedify/bench/trigger") { + triggerCalls++; + return Promise.resolve(statsJson({ version: 1 }, 202)); + } + return Promise.resolve(new Response("unexpected", { status: 500 })); + }, + assertDestinationAllowed: () => {}, + }); + + assert.ok(measurement.requests.total > 0); + assert.strictEqual(measurement.requests.failed, 0); + assert.strictEqual(measurement.requests.successRate, 1); + assert.ok(statsCalls >= 3); +}); + test("failureRunner.validate - requires sender for remote faults", () => { const scenario = normalizeSuite({ version: 1, diff --git a/packages/cli/src/bench/scenarios/failure.ts b/packages/cli/src/bench/scenarios/failure.ts index d9d86d8df..c09cca311 100644 --- a/packages/cli/src/bench/scenarios/failure.ts +++ b/packages/cli/src/bench/scenarios/failure.ts @@ -340,27 +340,29 @@ async function waitForRemoteFault(options: { const deadline = Date.now() + options.timeoutMs; do { const snapshot = await fetchServerSnapshot(options.target, options.fetch); - if (snapshot == null) return null; - const diff = diffSnapshots(options.baseline, snapshot); - const queueTasks = diff.queueTasks; - if (queueTasks == null) return null; - if (options.fault === "remote-404" || options.fault === "remote-410") { - if ((diff.deliveryPermanentFailures ?? 0) > 0) { - return { timedOut: false }; - } - } else if (options.fault === "slow-inbox") { - const remaining = queueTaskRemaining(diff); - if (remaining == null) return null; - if (queueTasks.completed > 0 && remaining === 0) { - return { timedOut: false }; - } - } else if (options.fault === "network-error") { - const remaining = queueTaskRemaining(diff); - if (remaining == null) return null; - if ( - queueTasks.failed > 0 || (queueTasks.completed > 0 && remaining > 0) - ) { - return { timedOut: false }; + if (snapshot != null) { + const diff = diffSnapshots(options.baseline, snapshot); + const queueTasks = diff.queueTasks; + if (options.fault === "remote-404" || options.fault === "remote-410") { + if ((diff.deliveryPermanentFailures ?? 0) > 0) { + return { timedOut: false }; + } + } else if (queueTasks != null) { + const remaining = queueTaskRemaining(diff); + if (remaining != null) { + if (options.fault === "slow-inbox") { + if (queueTasks.completed > 0 && remaining === 0) { + return { timedOut: false }; + } + } else if (options.fault === "network-error") { + if ( + queueTasks.failed > 0 || + (queueTasks.completed > 0 && remaining > 0) + ) { + return { timedOut: false }; + } + } + } } } await new Promise((resolve) => setTimeout(resolve, DRAIN_POLL_MS)); diff --git a/packages/cli/src/bench/scenarios/fanout.test.ts b/packages/cli/src/bench/scenarios/fanout.test.ts index 2a345db1c..d370269b6 100644 --- a/packages/cli/src/bench/scenarios/fanout.test.ts +++ b/packages/cli/src/bench/scenarios/fanout.test.ts @@ -178,6 +178,61 @@ test("fanoutRunner - counts failed queue tasks as delivery failures", async () = assert.strictEqual(measurement.deliveryThroughputPerSec, 0); }); +test("fanoutRunner - tolerates transient drain stats failures", async () => { + const target = new URL("http://target.test/"); + let statsCalls = 0; + const scenario = normalizeSuite({ + version: 1, + target: target.href, + scenarios: [{ + name: "fanout", + type: "fanout", + sender: "alice", + followers: 5, + load: { concurrency: 1 }, + duration: "40ms", + queueDrainTimeout: "1s", + }], + }).scenarios[0]; + + const measurement = await fanoutRunner.run({ + scenario, + target, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/.well-known/fedify/bench/stats") { + statsCalls++; + if (statsCalls === 2) { + return Promise.resolve( + new Response("temporarily unavailable", { + status: 503, + }), + ); + } + const drained = statsCalls > 2; + return Promise.resolve(json(statsSnapshot({ + enqueued: drained ? 6 : 0, + completed: drained ? 6 : 0, + failed: 0, + }))); + } + if (url.pathname === "/.well-known/fedify/bench/trigger") { + return Promise.resolve(json({ version: 1 }, 202)); + } + return Promise.resolve(new Response("unexpected", { status: 500 })); + }, + }); + + assert.ok(measurement.requests.total > 0); + assert.strictEqual(measurement.requests.failed, 0); + assert.strictEqual(measurement.requests.successRate, 1); + assert.ok(statsCalls >= 3); +}); + test("fanoutRunner - omits queue drain metrics without drain samples", async () => { const target = new URL("http://target.test/"); const scenario = normalizeSuite({ diff --git a/packages/cli/src/bench/scenarios/fanout.ts b/packages/cli/src/bench/scenarios/fanout.ts index 4cce8abe7..6c809a583 100644 --- a/packages/cli/src/bench/scenarios/fanout.ts +++ b/packages/cli/src/bench/scenarios/fanout.ts @@ -255,14 +255,13 @@ async function waitForDrain(options: { const deadline = Date.now() + options.timeoutMs; do { const snapshot = await fetchServerSnapshot(options.target, options.fetch); - if (snapshot == null) return null; - const diff = diffSnapshots(options.baseline, snapshot); - const queueTasks = diff.queueTasks; - if (queueTasks == null) return null; - const remaining = queueTaskRemaining(diff); - if (remaining == null) return null; - if (remaining === 0) { - return { timedOut: false, failed: queueTasks.failed }; + if (snapshot != null) { + const diff = diffSnapshots(options.baseline, snapshot); + const queueTasks = diff.queueTasks; + const remaining = queueTaskRemaining(diff); + if (queueTasks != null && remaining != null && remaining === 0) { + return { timedOut: false, failed: queueTasks.failed }; + } } await new Promise((resolve) => setTimeout(resolve, DRAIN_POLL_MS)); } while (Date.now() < deadline); From 01921df0f219f230f987fa4d16b48895ad84cbb7 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 16:21:36 +0900 Subject: [PATCH 30/61] Cap object source collection crawls Stop object discovery after a bounded number of collection pages when type filters do not find matching objects. This prevents a large outbox from turning one object scenario into an unbounded crawl. https://github.com/fedify-dev/fedify/pull/802#discussion_r3393774852 Assisted-by: Codex:gpt-5.5 --- .../src/bench/scenarios/object-discovery.ts | 5 +- .../cli/src/bench/scenarios/object.test.ts | 58 +++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/bench/scenarios/object-discovery.ts b/packages/cli/src/bench/scenarios/object-discovery.ts index fc1f0844f..83e601c7c 100644 --- a/packages/cli/src/bench/scenarios/object-discovery.ts +++ b/packages/cli/src/bench/scenarios/object-discovery.ts @@ -10,6 +10,7 @@ import type { ObjectSource } from "../scenario/types.ts"; const ACTIVITY_JSON_ACCEPT = "application/activity+json, application/ld+json"; const WEBFINGER_ACCEPT = "application/jrd+json, application/json"; +const MAX_COLLECTION_CRAWL_PAGES = 100; const ACTIVITY_WRAPPER_TYPES = new Set([ "Accept", "Add", @@ -135,12 +136,14 @@ async function* crawlCollection( ): AsyncGenerator { let next: URL | null = start; let remaining = options.limit; + let pages = 0; const visited = new Set(); - while (next != null && remaining > 0) { + while (next != null && remaining > 0 && pages < MAX_COLLECTION_CRAWL_PAGES) { if (visited.has(next.href)) return; visited.add(next.href); await options.assertReadDestinationAllowed?.(next); const page = await fetchJson(next, options.fetch); + pages++; const items = arrayProperty(page, "orderedItems") ?? arrayProperty(page, "items") ?? []; for (const item of items) { diff --git a/packages/cli/src/bench/scenarios/object.test.ts b/packages/cli/src/bench/scenarios/object.test.ts index b8f7ec371..d664b9c91 100644 --- a/packages/cli/src/bench/scenarios/object.test.ts +++ b/packages/cli/src/bench/scenarios/object.test.ts @@ -598,6 +598,64 @@ test("objectRunner - resolves relative URLs while crawling object sources", asyn ); }); +test("objectRunner - caps object source crawl pages", async () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "object-crawl", + type: "object", + source: { + seed: "http://target.test/users/alice", + collection: "outbox", + limit: 1, + type: "Note", + }, + load: { concurrency: 1 }, + duration: "25ms", + }], + }).scenarios[0]; + let pageFetches = 0; + + await assert.rejects( + async () => + await objectRunner.run({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const request = input instanceof Request ? input : new Request(input); + const url = new URL(request.url); + if (url.pathname === "/users/alice") { + return Promise.resolve(json({ + id: url.href, + outbox: "http://target.test/outbox?page=0", + })); + } + if (url.pathname === "/outbox") { + pageFetches++; + const page = Number(url.searchParams.get("page") ?? 0); + return Promise.resolve(json({ + id: url.href, + orderedItems: [{ + type: "Article", + id: `http://target.test/articles/${page}`, + }], + next: `?page=${page + 1}`, + })); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }, + }), + /did not resolve any URLs/, + ); + + assert.strictEqual(pageFetches, 100); +}); + test("objectRunner.validate - rejects malformed object source URLs", () => { const explicit = normalizeSuite({ version: 1, From 2668a9e21cdbb8959ac09186057bfd5fbefa16df Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 16:21:42 +0900 Subject: [PATCH 31/61] Guard mixed limiter releases Return a per-acquisition release function so duplicate calls cannot decrement the shared active count or wake extra waiters. https://github.com/fedify-dev/fedify/pull/802#discussion_r3393774859 Assisted-by: Codex:gpt-5.5 --- .../cli/src/bench/scenarios/mixed.test.ts | 32 ++++++++++++++++++- packages/cli/src/bench/scenarios/mixed.ts | 20 ++++++------ 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/packages/cli/src/bench/scenarios/mixed.test.ts b/packages/cli/src/bench/scenarios/mixed.test.ts index 2594d7850..f0452e571 100644 --- a/packages/cli/src/bench/scenarios/mixed.test.ts +++ b/packages/cli/src/bench/scenarios/mixed.test.ts @@ -6,7 +6,7 @@ import type { ScenarioMeasurement } from "../result/build.ts"; import { evaluateExpect } from "../result/expect/evaluate.ts"; import { normalizeSuite } from "../scenario/normalize.ts"; import type { Suite } from "../scenario/types.ts"; -import { mergeMeasurements, mixedRunner } from "./mixed.ts"; +import { createLimiter, mergeMeasurements, mixedRunner } from "./mixed.ts"; test("mixedRunner - runs weighted child scenarios together", async () => { const target = new URL("http://target.test/"); @@ -126,6 +126,36 @@ test("mixedRunner - enforces parent maxInFlight across children", async () => { ); }); +test("createLimiter - ignores duplicate releases", async () => { + const limiter = createLimiter(1); + const releaseFirst = await limiter.acquire(); + let secondAcquired = false; + const second = limiter.acquire().then((release) => { + secondAcquired = true; + return release; + }); + + releaseFirst(); + releaseFirst(); + const releaseSecond = await second; + assert.strictEqual(secondAcquired, true); + + let thirdAcquired = false; + const third = limiter.acquire().then((release) => { + thirdAcquired = true; + return release; + }); + await Promise.resolve(); + assert.strictEqual(thirdAcquired, false); + + releaseSecond(); + const releaseThird = await third; + assert.strictEqual(thirdAcquired, true); + releaseSecond(); + releaseThird(); + releaseThird(); +}); + test("mixedRunner - waits for siblings before propagating child errors", async () => { const target = new URL("http://target.test/"); const suite: Suite = { diff --git a/packages/cli/src/bench/scenarios/mixed.ts b/packages/cli/src/bench/scenarios/mixed.ts index 7e7455e4a..dcd81af7d 100644 --- a/packages/cli/src/bench/scenarios/mixed.ts +++ b/packages/cli/src/bench/scenarios/mixed.ts @@ -236,7 +236,7 @@ function limitedFetch(fetchImpl: typeof fetch, maxInFlight?: number) { return limited as typeof fetch; } -function createLimiter(maxInFlight: number): { +export function createLimiter(maxInFlight: number): { acquire(): Promise<() => void>; } { if (!Number.isInteger(maxInFlight) || maxInFlight < 1) { @@ -246,19 +246,21 @@ function createLimiter(maxInFlight: number): { } let active = 0; const waiters: Array<() => void> = []; - function release(): void { - const next = waiters.shift(); - if (next == null) active--; - else next(); - } return { async acquire(): Promise<() => void> { if (active < maxInFlight) { active++; - return release; + } else { + await new Promise((resolve) => waiters.push(resolve)); } - await new Promise((resolve) => waiters.push(resolve)); - return release; + let released = false; + return () => { + if (released) return; + released = true; + const next = waiters.shift(); + if (next == null) active--; + else next(); + }; }, }; } From 90f9a335fca29d2a065896c557f393d28b9136b4 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 16:50:03 +0900 Subject: [PATCH 32/61] Follow Link hrefs during object discovery Accept ActivityStreams Link objects that carry their URL in href when crawling actor collections. This lets object scenarios follow common outbox, page, and object references that are not represented as plain strings or id objects. https://github.com/fedify-dev/fedify/pull/802#discussion_r3394053372 Assisted-by: Codex:gpt-5.5 --- .../src/bench/scenarios/object-discovery.ts | 5 +- .../cli/src/bench/scenarios/object.test.ts | 79 +++++++++++++++++++ 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/bench/scenarios/object-discovery.ts b/packages/cli/src/bench/scenarios/object-discovery.ts index 83e601c7c..64ac9e6a1 100644 --- a/packages/cli/src/bench/scenarios/object-discovery.ts +++ b/packages/cli/src/bench/scenarios/object-discovery.ts @@ -223,8 +223,9 @@ function propertyUrl( ): URL | null { const value = object[key]; if (typeof value === "string") return new URL(value, base); - if (isRecord(value) && typeof value.id === "string") { - return new URL(value.id, base); + if (isRecord(value)) { + if (typeof value.href === "string") return new URL(value.href, base); + if (typeof value.id === "string") return new URL(value.id, base); } return null; } diff --git a/packages/cli/src/bench/scenarios/object.test.ts b/packages/cli/src/bench/scenarios/object.test.ts index d664b9c91..d0d258992 100644 --- a/packages/cli/src/bench/scenarios/object.test.ts +++ b/packages/cli/src/bench/scenarios/object.test.ts @@ -598,6 +598,85 @@ test("objectRunner - resolves relative URLs while crawling object sources", asyn ); }); +test("objectRunner - resolves Link object href values while crawling", async () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "object-crawl", + type: "object", + source: { + seed: "http://target.test/users/alice/", + collection: "outbox", + limit: 1, + type: "Note", + }, + load: { concurrency: 1 }, + duration: "25ms", + }], + }).scenarios[0]; + const fetched: string[] = []; + + const measurement = await objectRunner.run({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const request = input instanceof Request ? input : new Request(input); + const url = new URL(request.url); + fetched.push(url.href); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return Promise.resolve(new Response("not found", { status: 404 })); + } + if (url.pathname === "/users/alice/") { + return Promise.resolve(json({ + id: url.href, + outbox: { + type: "Link", + href: "outbox", + }, + })); + } + if (url.pathname === "/users/alice/outbox" && url.search === "?page=1") { + return Promise.resolve(json({ + id: url.href, + orderedItems: [{ + type: "Create", + object: { + type: "Note", + id: { + type: "Link", + href: "../objects/note", + }, + }, + }], + })); + } + if (url.pathname === "/users/alice/outbox") { + return Promise.resolve(json({ + id: url.href, + first: { + type: "Link", + href: "?page=1", + }, + })); + } + if (url.pathname === "/users/objects/note") { + return Promise.resolve(json({ id: url.href, type: "Note" })); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }, + }); + + assert.ok(measurement.requests.total > 0); + assert.ok(fetched.includes("http://target.test/users/alice/outbox")); + assert.ok(fetched.includes("http://target.test/users/alice/outbox?page=1")); + assert.ok(fetched.includes("http://target.test/users/objects/note")); +}); + test("objectRunner - caps object source crawl pages", async () => { const scenario = normalizeSuite({ version: 1, From b2dfcf8f43dd89e78d9fe12e816c19a3fee77741 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 16:52:25 +0900 Subject: [PATCH 33/61] Make benchmark sink URLs configurable Add a sinkBase scenario option for fanout and remote failure benchmarks so generated sink inbox URLs can match a target's triggerSinks allowlist without turning off recipient safety. The runner validates the base URL, binds the sink server to the configured port, and skips the advertise-host preflight gate when the scenario already provides a reachable sink base. https://github.com/fedify-dev/fedify/pull/802#discussion_r3394065680 Assisted-by: Codex:gpt-5.5 --- CHANGES.md | 6 +- docs/manual/benchmarking.md | 19 +++- packages/cli/src/bench/action.test.ts | 87 ++++++++++++++++ packages/cli/src/bench/action.ts | 5 +- packages/cli/src/bench/scenario/schema.ts | 1 + packages/cli/src/bench/scenario/types.ts | 1 + .../cli/src/bench/scenarios/failure.test.ts | 65 ++++++++++++ packages/cli/src/bench/scenarios/failure.ts | 6 +- .../cli/src/bench/scenarios/fanout.test.ts | 98 +++++++++++++++++++ packages/cli/src/bench/scenarios/fanout.ts | 62 +++++++++++- schema/bench/scenario-v1.json | 3 + 11 files changed, 340 insertions(+), 13 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2b6c3dd84..8e1fb46a4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -297,12 +297,16 @@ To be released. drive the benchmark trigger endpoint and wait for queue task drain; failure scenarios report expected fault outcomes as successes; and mixed scenarios run weighted child scenario blends. The `collection` scenario type remains - reserved but not executable. [[#744], [#785], [#801]] + reserved but not executable. Fanout and remote failure scenarios can set + `sinkBase` to generate deterministic benchmark sink inbox URLs for targets + that keep `triggerSinks` allowlisting enabled. [[#744], [#785], [#801], + [#802]] [#783]: https://github.com/fedify-dev/fedify/issues/783 [#784]: https://github.com/fedify-dev/fedify/issues/784 [#785]: https://github.com/fedify-dev/fedify/issues/785 [#801]: https://github.com/fedify-dev/fedify/pull/801 +[#802]: https://github.com/fedify-dev/fedify/pull/802 ### @fedify/fixture diff --git a/docs/manual/benchmarking.md b/docs/manual/benchmarking.md index 73a735e8e..ee338de3e 100644 --- a/docs/manual/benchmarking.md +++ b/docs/manual/benchmarking.md @@ -177,9 +177,13 @@ The runnable scenario types cover the main benchmark surfaces: - `fanout`: posts to `/.well-known/fedify/bench/trigger` so the target calls `sendActivity()` and drains its fanout/outbox queue to benchmark-owned sink inboxes. The command starts those sink inboxes locally. A non-loopback - target therefore needs `--advertise-host`, and the target must either + target therefore needs `--advertise-host` unless the scenario sets + `sinkBase` to a reachable `http://host:port/` URL. The target must either allow the generated sink inboxes through `triggerSinks` or run with - `allowUnsafeTriggerRecipients` in a controlled benchmark environment. + `allowUnsafeTriggerRecipients` in a controlled benchmark environment. Use + `sinkBase` when you want those inboxes to be deterministic, for example + `http://127.0.0.1:9090/inbox/0` through + `http://127.0.0.1:9090/inbox/4` for `followers: 5`. `fedify bench` does not switch the target's queue backend; run the same suite against targets configured with the queue implementations you want to compare. Fanout triggers are serialized while the runner observes queue @@ -195,8 +199,10 @@ The runnable scenario types cover the main benchmark surfaces: faults post to the benchmark trigger endpoint with `sender`, so the target uses its normal outbound delivery path against controlled benchmark-owned sink inboxes. Like `fanout`, these remote failure faults need - `--advertise-host` for a non-loopback target. Remote failure deliveries - are also serialized while the runner waits for the target's queue to + `--advertise-host` for a non-loopback target unless `sinkBase` gives a + reachable, fixed sink base URL that the target's `triggerSinks` can + preconfigure. Remote failure deliveries are also serialized while the + runner waits for the target's queue to observe the expected failure or retry signal, so request latency can include earlier wait time when the configured load is concurrent or high-rate. - `mixed`: runs referenced child scenarios concurrently, splitting the @@ -390,6 +396,11 @@ allowlist. To bypass this guard for a controlled run, set `~FederationBenchmarkOptions.allowUnsafeTriggerRecipients` to `true` in the application configuration. +For `fanout` and remote `failure` scenarios, set a `sinkBase` value such as +`http://host:port/` in the scenario when the target keeps the safe default and +you need stable sink URLs for `triggerSinks`. With `followers: 5`, the runner +generates `/inbox/0` through `/inbox/4` under that base. + A successful trigger returns `202 Accepted`: ~~~~ json diff --git a/packages/cli/src/bench/action.test.ts b/packages/cli/src/bench/action.test.ts index db0b6f849..bca1e7594 100644 --- a/packages/cli/src/bench/action.test.ts +++ b/packages/cli/src/bench/action.test.ts @@ -29,6 +29,19 @@ async function writeSuite(content: string): Promise { return path; } +async function reservePort(): Promise { + const server = serve({ + port: 0, + hostname: "127.0.0.1", + silent: true, + fetch: () => new Response("reserved"), + }); + await server.ready(); + const port = Number(new URL(server.url!).port); + await server.close(true); + return port; +} + function resolvePublicHost(_hostname: string): Promise { return Promise.resolve(["93.184.216.34"]); } @@ -450,6 +463,72 @@ scenarios: assert.match(message, /advertise-host/); }); +test("runBench - remote failure with sinkBase needs no advertise host", async () => { + const sinkBase = `http://127.0.0.1:${await reservePort()}/`; + const file = await writeSuite(`version: 1 +target: http://10.10.0.5:8000 +scenarios: + - name: remote-404 + type: failure + fault: remote-404 + sender: alice + sinkBase: "${sinkBase}" + load: { concurrency: 1 } + duration: 25ms + queueDrainTimeout: 1s +`); + let code = -1; + let message = ""; + let triggerCalls = 0; + await runBench(command({ scenario: file }), { + exit: (c) => { + code = c; + }, + writeOutput: () => Promise.resolve(), + log: (m) => { + message = m; + }, + fetch: (input) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return Promise.resolve( + new Response( + JSON.stringify({ + version: 1, + source: "server", + scopeMetrics: [{ + metrics: [ + sumMetric("fedify.queue.task.enqueued", triggerCalls), + sumMetric("fedify.queue.task.completed", triggerCalls), + sumMetric("fedify.queue.task.failed", 0), + sumMetric( + "activitypub.delivery.permanent_failure", + triggerCalls, + ), + ], + }], + }), + { headers: { "content-type": "application/json" } }, + ), + ); + } + if (url.pathname === "/.well-known/fedify/bench/trigger") { + triggerCalls++; + return Promise.resolve( + new Response(JSON.stringify({ version: 1 }), { + status: 202, + headers: { "content-type": "application/json" }, + }), + ); + } + return Promise.resolve(new Response("unexpected", { status: 500 })); + }, + }); + + assert.strictEqual(code, 0, message); + assert.ok(triggerCalls > 0); +}); + test("runBench - missing-actor failure needs no advertise host", async () => { const recipientTarget = await spawnBenchmarkTarget(); try { @@ -984,3 +1063,11 @@ scenarios: assert.strictEqual(code, 2); assert.match(message, /Invalid/); }); + +function sumMetric(name: string, value: number): Record { + return { + name, + dataPointType: "sum", + dataPoints: [{ value }], + }; +} diff --git a/packages/cli/src/bench/action.ts b/packages/cli/src/bench/action.ts index 7640b7a85..e63740440 100644 --- a/packages/cli/src/bench/action.ts +++ b/packages/cli/src/bench/action.ts @@ -700,10 +700,11 @@ function scenarioNeedsReachableLocalServer( scenarios: readonly ResolvedScenario[], seen: ReadonlySet = new Set(), ): boolean { - if (scenario.type === "fanout") return true; + if (scenario.type === "fanout") return scenario.raw.sinkBase == null; if (scenario.type === "failure") { return scenario.faults.includes("invalid-signature") || - scenario.faults.some(isRemoteFailureFault); + (scenario.raw.sinkBase == null && + scenario.faults.some(isRemoteFailureFault)); } if (scenario.type === "mixed") { if (seen.has(scenario.name)) return false; diff --git a/packages/cli/src/bench/scenario/schema.ts b/packages/cli/src/bench/scenario/schema.ts index d4894ef83..4dc69bc2c 100644 --- a/packages/cli/src/bench/scenario/schema.ts +++ b/packages/cli/src/bench/scenario/schema.ts @@ -288,6 +288,7 @@ export const scenarioSchemaV1 = { // fanout sender: { type: "string" }, followers: { type: "integer", minimum: 1 }, + sinkBase: { type: "string" }, trigger: { type: "object" }, sinkBehavior: { type: "object" }, queueDrainTimeout: { $ref: "#/$defs/duration" }, diff --git a/packages/cli/src/bench/scenario/types.ts b/packages/cli/src/bench/scenario/types.ts index 7620c84de..de73a9617 100644 --- a/packages/cli/src/bench/scenario/types.ts +++ b/packages/cli/src/bench/scenario/types.ts @@ -132,6 +132,7 @@ export interface Scenario { // fanout readonly sender?: string; readonly followers?: number; + readonly sinkBase?: string; readonly trigger?: Record; readonly sinkBehavior?: Record; readonly queueDrainTimeout?: string; diff --git a/packages/cli/src/bench/scenarios/failure.test.ts b/packages/cli/src/bench/scenarios/failure.test.ts index 93a034586..6146a5d97 100644 --- a/packages/cli/src/bench/scenarios/failure.test.ts +++ b/packages/cli/src/bench/scenarios/failure.test.ts @@ -72,6 +72,58 @@ for ( }); } +test("failureRunner - uses configured sink base for remote faults", async () => { + const target = new URL("http://target.test/"); + const sinkBase = `http://127.0.0.1:${await reservePort()}/`; + const scenario = normalizeSuite({ + version: 1, + target: target.href, + scenarios: [{ + name: "failure", + type: "failure", + fault: "remote-404", + sender: "alice", + sinkBase, + load: { concurrency: 1 }, + duration: "25ms", + queueDrainTimeout: "1s", + }], + }).scenarios[0]; + let triggerCalls = 0; + let recipientInbox = ""; + + const measurement = await failureRunner.run({ + scenario, + target, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input, init) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return Promise.resolve(statsJson(statsSnapshot({ + enqueued: triggerCalls, + completed: triggerCalls, + failed: 0, + permanentFailures: triggerCalls, + }))); + } + if (url.pathname === "/.well-known/fedify/bench/trigger") { + triggerCalls++; + const body = JSON.parse(String(init?.body)); + recipientInbox = body.recipients[0].inbox; + return Promise.resolve(statsJson({ version: 1 }, 202)); + } + return Promise.resolve(new Response("unexpected", { status: 500 })); + }, + assertDestinationAllowed: () => {}, + }); + + assert.ok(measurement.requests.total > 0); + assert.strictEqual(recipientInbox, new URL("/inbox/0", sinkBase).href); +}); + test("failureRunner - detects network-error retries", async () => { const target = new URL("http://target.test/"); const scenario = normalizeSuite({ @@ -395,6 +447,19 @@ function statsJson(body: unknown, status = 200): Response { }); } +async function reservePort(): Promise { + const server = serve({ + port: 0, + hostname: "127.0.0.1", + silent: true, + fetch: () => new Response("reserved"), + }); + await server.ready(); + const port = Number(new URL(server.url!).port); + await server.close(true); + return port; +} + function statsSnapshot(counts: { readonly enqueued: number; readonly completed: number; diff --git a/packages/cli/src/bench/scenarios/failure.ts b/packages/cli/src/bench/scenarios/failure.ts index c09cca311..f8bcf7f41 100644 --- a/packages/cli/src/bench/scenarios/failure.ts +++ b/packages/cli/src/bench/scenarios/failure.ts @@ -16,7 +16,7 @@ import { import type { SyntheticActor } from "../server/synthetic.ts"; import { createActivityIdMinter } from "../signing/activity-id.ts"; import { signInboxDelivery } from "../signing/signer.ts"; -import { spawnSinkServer } from "./fanout.ts"; +import { resolveSinkBase, spawnSinkServer } from "./fanout.ts"; import { loadPlanOf, measuredWindowMs, @@ -85,6 +85,9 @@ export const failureRunner: ScenarioRunner = { "sender.", ); } + if (faults.some(isRemoteFault)) { + resolveSinkBase(scenario.name, scenario.raw.sinkBase); + } }, async run(context: RunContext) { @@ -191,6 +194,7 @@ async function resolveRemoteFailureTargets( followers: 1, rawBehavior: remoteSinkBehavior(fault), advertiseHost: context.advertiseHost, + sinkBase: context.scenario.raw.sinkBase, }); const recipient = sink.recipients[0]; if (fault === "network-error") { diff --git a/packages/cli/src/bench/scenarios/fanout.test.ts b/packages/cli/src/bench/scenarios/fanout.test.ts index d370269b6..6a814699f 100644 --- a/packages/cli/src/bench/scenarios/fanout.test.ts +++ b/packages/cli/src/bench/scenarios/fanout.test.ts @@ -1,5 +1,6 @@ import assert from "node:assert/strict"; import test from "node:test"; +import { serve } from "srvx"; import { getContextLoader, getDocumentLoader } from "../../docloader.ts"; import { normalizeSuite } from "../scenario/normalize.ts"; import type { Suite } from "../scenario/types.ts"; @@ -79,6 +80,34 @@ test("fanoutRunner.validate - requires enough followers for fanout queue", () => assert.throws(() => fanoutRunner.validate?.(scenario), /at least 5/); }); +test("fanoutRunner.validate - rejects invalid sinkBase URLs", () => { + for ( + const sinkBase of [ + "http://target.test/", + "https://target.test:8443/", + "http://user:pass@target.test:9090/", + "http://target.test:9090/sinks/", + ] + ) { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "fanout", + type: "fanout", + sender: "alice", + followers: 5, + sinkBase, + }], + }).scenarios[0]; + + assert.throws( + () => fanoutRunner.validate?.(scenario), + /sinkBase must be an http URL/, + ); + } +}); + test("fanoutRunner - serializes overlapping trigger drains", async () => { const target = new URL("http://target.test/"); let statsCalls = 0; @@ -233,6 +262,62 @@ test("fanoutRunner - tolerates transient drain stats failures", async () => { assert.ok(statsCalls >= 3); }); +test("fanoutRunner - uses configured sink base for recipients", async () => { + const target = new URL("http://target.test/"); + const sinkBase = `http://127.0.0.1:${await reservePort()}/`; + let statsCalls = 0; + let recipientInboxes: string[] = []; + const scenario = normalizeSuite({ + version: 1, + target: target.href, + scenarios: [{ + name: "fanout", + type: "fanout", + sender: "alice", + followers: 5, + sinkBase, + load: { concurrency: 1 }, + duration: "40ms", + queueDrainTimeout: "1s", + }], + }).scenarios[0]; + + const measurement = await fanoutRunner.run({ + scenario, + target, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input, init) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/.well-known/fedify/bench/stats") { + statsCalls++; + const drained = statsCalls > 1; + return Promise.resolve(json(statsSnapshot({ + enqueued: drained ? 6 : 0, + completed: drained ? 6 : 0, + failed: 0, + }))); + } + if (url.pathname === "/.well-known/fedify/bench/trigger") { + const body = JSON.parse(String(init?.body)); + recipientInboxes = body.recipients.map(( + recipient: Record, + ) => recipient.inbox); + return Promise.resolve(json({ version: 1 }, 202)); + } + return Promise.resolve(new Response("unexpected", { status: 500 })); + }, + }); + + assert.ok(measurement.requests.total > 0); + assert.deepStrictEqual( + recipientInboxes, + Array.from({ length: 5 }, (_, i) => new URL(`/inbox/${i}`, sinkBase).href), + ); +}); + test("fanoutRunner - omits queue drain metrics without drain samples", async () => { const target = new URL("http://target.test/"); const scenario = normalizeSuite({ @@ -284,6 +369,19 @@ function json(body: unknown, status = 200): Response { }); } +async function reservePort(): Promise { + const server = serve({ + port: 0, + hostname: "127.0.0.1", + silent: true, + fetch: () => new Response("reserved"), + }); + await server.ready(); + const port = Number(new URL(server.url!).port); + await server.close(true); + return port; +} + function statsSnapshot(counts: { readonly enqueued: number; readonly completed: number; diff --git a/packages/cli/src/bench/scenarios/fanout.ts b/packages/cli/src/bench/scenarios/fanout.ts index 6c809a583..5817e348c 100644 --- a/packages/cli/src/bench/scenarios/fanout.ts +++ b/packages/cli/src/bench/scenarios/fanout.ts @@ -44,6 +44,7 @@ export const fanoutRunner: ScenarioRunner = { "exercise Fedify's fanout queue.", ); } + resolveSinkBase(scenario.name, scenario.raw.sinkBase); }, async run(context: RunContext) { @@ -57,6 +58,7 @@ export const fanoutRunner: ScenarioRunner = { followers, rawBehavior: context.scenario.raw.sinkBehavior, advertiseHost: context.advertiseHost, + sinkBase: context.scenario.raw.sinkBase, }); const minter = createActivityIdMinter(context.target); const drainHistogram = new LogLinearHistogram(); @@ -181,17 +183,19 @@ export async function spawnSinkServer(options: { readonly followers: number; readonly rawBehavior: unknown; readonly advertiseHost?: string; + readonly sinkBase?: string; }): Promise<{ readonly recipients: readonly Record[]; readonly close: () => Promise; }> { + const sinkBase = resolveSinkBase("benchmark sink", options.sinkBase); const advertised = options.advertiseHost == null ? null : resolveAdvertiseHost(options.advertiseHost); const behavior = parseSinkBehavior(options.rawBehavior); const server = serve({ - port: 0, - hostname: advertised?.bindHost ?? "127.0.0.1", + port: sinkBase?.port ?? 0, + hostname: sinkBase?.bindHost ?? advertised?.bindHost ?? "127.0.0.1", silent: true, async fetch(request: Request): Promise { if (new URL(request.url).pathname.startsWith("/inbox/")) { @@ -208,9 +212,10 @@ export async function spawnSinkServer(options: { }); await server.ready(); const bound = new URL(server.url!); - const base = advertised == null - ? bound - : new URL(`http://${advertised.urlHost}:${bound.port}/`); + const base = sinkBase?.base ?? + (advertised == null + ? bound + : new URL(`http://${advertised.urlHost}:${bound.port}/`)); const recipients = Array.from({ length: options.followers }, (_, i) => ({ "@context": "https://www.w3.org/ns/activitystreams", type: "Service", @@ -223,6 +228,53 @@ export async function spawnSinkServer(options: { }; } +interface SinkBase { + readonly base: URL; + readonly bindHost: string; + readonly port: number; +} + +export function resolveSinkBase( + scenarioName: string, + value: string | undefined, +): SinkBase | null { + if (value == null) return null; + let url: URL; + try { + url = new URL(value); + } catch { + throw new Error( + `Scenario "${scenarioName}": sinkBase must be an http URL with an ` + + `explicit port; got ${JSON.stringify(value)}.`, + ); + } + const port = Number(url.port); + if ( + url.protocol !== "http:" || + url.hostname === "" || + url.username !== "" || + url.password !== "" || + url.port === "" || + !Number.isInteger(port) || + port < 1 || + url.pathname !== "/" || + url.search !== "" || + url.hash !== "" + ) { + throw new Error( + `Scenario "${scenarioName}": sinkBase must be an http URL with a ` + + "host, an explicit non-zero port, no credentials, and no path, " + + `query, or fragment; got ${JSON.stringify(value)}.`, + ); + } + const advertised = resolveAdvertiseHost(url.hostname); + return { + base: new URL(`http://${advertised.urlHost}:${url.port}/`), + bindHost: advertised.bindHost, + port, + }; +} + function parseSinkBehavior( raw: unknown, ): { latencyMs: number; status: number } { diff --git a/schema/bench/scenario-v1.json b/schema/bench/scenario-v1.json index c60a5a262..40f6eeda1 100644 --- a/schema/bench/scenario-v1.json +++ b/schema/bench/scenario-v1.json @@ -390,6 +390,9 @@ "type": "integer", "minimum": 1 }, + "sinkBase": { + "type": "string" + }, "trigger": { "type": "object" }, From f8ad54afc34ea38cce2e29f3fc8297d7d0956c1c Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 17:20:12 +0900 Subject: [PATCH 34/61] Skip malformed discovered object URLs Treat malformed URLs from crawled ActivityPub collection data as unusable entries instead of letting one bad string or Link object abort the whole object source discovery pass. https://github.com/fedify-dev/fedify/pull/802#discussion_r3394232137 https://github.com/fedify-dev/fedify/pull/802#discussion_r3394232148 https://github.com/fedify-dev/fedify/pull/802#discussion_r3394232156 Assisted-by: Codex:gpt-5.5 --- .../src/bench/scenarios/object-discovery.ts | 16 +++-- .../cli/src/bench/scenarios/object.test.ts | 68 +++++++++++++++++++ 2 files changed, 80 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/bench/scenarios/object-discovery.ts b/packages/cli/src/bench/scenarios/object-discovery.ts index 64ac9e6a1..493cab645 100644 --- a/packages/cli/src/bench/scenarios/object-discovery.ts +++ b/packages/cli/src/bench/scenarios/object-discovery.ts @@ -187,7 +187,7 @@ function objectUrl( ): URL | null { for (const candidate of objectCandidates(item)) { if (typeof candidate === "string") { - if (types.size < 1) return new URL(candidate, base); + if (types.size < 1) return safeUrl(candidate, base); continue; } if (!isRecord(candidate)) continue; @@ -222,10 +222,10 @@ function propertyUrl( base?: URL, ): URL | null { const value = object[key]; - if (typeof value === "string") return new URL(value, base); + if (typeof value === "string") return safeUrl(value, base); if (isRecord(value)) { - if (typeof value.href === "string") return new URL(value.href, base); - if (typeof value.id === "string") return new URL(value.id, base); + if (typeof value.href === "string") return safeUrl(value.href, base); + if (typeof value.id === "string") return safeUrl(value.id, base); } return null; } @@ -241,3 +241,11 @@ function arrayProperty( function isRecord(value: unknown): value is Record { return value != null && typeof value === "object" && !Array.isArray(value); } + +function safeUrl(value: string, base?: URL): URL | null { + try { + return new URL(value, base); + } catch { + return null; + } +} diff --git a/packages/cli/src/bench/scenarios/object.test.ts b/packages/cli/src/bench/scenarios/object.test.ts index d0d258992..a0b0a0023 100644 --- a/packages/cli/src/bench/scenarios/object.test.ts +++ b/packages/cli/src/bench/scenarios/object.test.ts @@ -677,6 +677,74 @@ test("objectRunner - resolves Link object href values while crawling", async () assert.ok(fetched.includes("http://target.test/users/objects/note")); }); +test("objectRunner - skips malformed URLs while crawling object sources", async () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "object-crawl", + type: "object", + source: { + seed: "http://target.test/users/alice", + collection: "outbox", + limit: 1, + }, + load: { concurrency: 1 }, + duration: "25ms", + }], + }).scenarios[0]; + let fetchedObjectUrl = ""; + + const measurement = await objectRunner.run({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const request = input instanceof Request ? input : new Request(input); + const url = new URL(request.url); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return Promise.resolve(new Response("not found", { status: 404 })); + } + if (url.pathname === "/users/alice") { + return Promise.resolve(json({ + id: url.href, + outbox: "http://target.test/users/alice/outbox", + })); + } + if (url.pathname === "/users/alice/outbox") { + return Promise.resolve(json({ + id: url.href, + orderedItems: [ + "http://[malformed", + { + type: "Note", + id: { + type: "Link", + href: "http://[malformed", + }, + }, + { + type: "Note", + id: "http://target.test/objects/valid", + }, + ], + })); + } + if (url.pathname === "/objects/valid") { + fetchedObjectUrl = url.href; + return Promise.resolve(json({ id: url.href, type: "Note" })); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }, + }); + + assert.ok(measurement.requests.total > 0); + assert.strictEqual(fetchedObjectUrl, "http://target.test/objects/valid"); +}); + test("objectRunner - caps object source crawl pages", async () => { const scenario = normalizeSuite({ version: 1, From 222f34f34fb009d7734ab5dad608a9109f8428a9 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 17:22:33 +0900 Subject: [PATCH 35/61] Corrupt failure signatures directly Simulate invalid-signature failures by corrupting the HTTP signature headers while keeping the signed ActivityPub body intact. This avoids relying on target digest enforcement to observe the expected rejection. https://github.com/fedify-dev/fedify/pull/802#discussion_r3394232162 Assisted-by: Codex:gpt-5.5 --- .../cli/src/bench/scenarios/failure.test.ts | 26 +++++++++++++------ packages/cli/src/bench/scenarios/failure.ts | 17 ++++++++---- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/packages/cli/src/bench/scenarios/failure.test.ts b/packages/cli/src/bench/scenarios/failure.test.ts index 6146a5d97..845686391 100644 --- a/packages/cli/src/bench/scenarios/failure.test.ts +++ b/packages/cli/src/bench/scenarios/failure.test.ts @@ -329,6 +329,7 @@ test("failureRunner - discovers inbound failure inboxes once", async () => { return Promise.resolve(); }, }; + let corruptedSignatureRequests = 0; const measurement = await failureRunner.run({ scenario, target, @@ -336,16 +337,24 @@ test("failureRunner - discovers inbound failure inboxes once", async () => { contextLoader: await getContextLoader({ allowPrivateAddress: true }), allowPrivateAddress: true, fleet, - fetch: (input) => { - const url = new URL(input instanceof Request ? input.url : input); + fetch: async (input) => { + const request = input instanceof Request ? input : new Request(input); + const url = new URL(request.url); if (url.pathname === "/inbox") { - return Promise.resolve( - new Response("bad signature", { - status: 401, - }), - ); + await request.clone().json(); + const signature = request.headers.get("signature"); + const authorization = request.headers.get("authorization"); + if ( + signature?.endsWith("0") === true || + authorization?.endsWith("0") === true + ) { + corruptedSignatureRequests++; + } + return new Response("bad signature", { + status: 401, + }); } - return Promise.resolve(new Response("not found", { status: 404 })); + return new Response("not found", { status: 404 }); }, assertDestinationAllowed: () => {}, clock, @@ -353,6 +362,7 @@ test("failureRunner - discovers inbound failure inboxes once", async () => { assert.strictEqual(measurement.requests.total, 3); assert.strictEqual(measurement.requests.successRate, 1); + assert.strictEqual(corruptedSignatureRequests, 3); assert.strictEqual(actorGets, 1); } finally { try { diff --git a/packages/cli/src/bench/scenarios/failure.ts b/packages/cli/src/bench/scenarios/failure.ts index f8bcf7f41..523200bfd 100644 --- a/packages/cli/src/bench/scenarios/failure.ts +++ b/packages/cli/src/bench/scenarios/failure.ts @@ -400,17 +400,15 @@ async function sendInvalidSignature( "invalid-signature", deliveryTarget, ); - const body = new Uint8Array(await request.arrayBuffer()); - const corrupted = new Uint8Array(body.length + 1); - corrupted.set(body); - corrupted[body.length] = 0x20; + const body = await request.arrayBuffer(); const headers = new Headers(request.headers); + corruptSignatureHeaders(headers); return expectedFailure( await sendRequest( new Request(request.url, { method: request.method, headers, - body: corrupted, + body, redirect: "manual", }), context.fetch ?? fetch, @@ -418,6 +416,15 @@ async function sendInvalidSignature( ); } +function corruptSignatureHeaders(headers: Headers): void { + const signature = headers.get("signature"); + if (signature != null) headers.set("signature", `${signature}0`); + const authorization = headers.get("authorization"); + if (authorization != null) { + headers.set("authorization", `${authorization}0`); + } +} + async function sendMissingActor( context: RunContext, deliveryTarget: FailureDeliveryTarget, From 7bbff1c1cfb72c8dc43c9da28c46924662bb5b51 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 17:22:42 +0900 Subject: [PATCH 36/61] Publish benchmark scenario schema v2 Keep the already-published scenario-v1 schema immutable and publish the sinkBase addition as scenario-v2. Runtime validation and documentation now point at the current schema while the schema registry still guards v1. https://github.com/fedify-dev/fedify/pull/802#discussion_r3394251318 Assisted-by: Codex:gpt-5.5 --- CHANGES.md | 4 +- docs/manual/benchmarking.md | 4 +- packages/cli/src/bench/scenario/schema.ts | 31 +- packages/cli/src/bench/scenario/types.ts | 2 +- packages/cli/src/bench/scenario/validate.ts | 4 +- packages/cli/src/bench/schemas.ts | 7 +- schema/README.md | 16 +- schema/bench/scenario-v1.json | 3 - schema/bench/scenario-v2.json | 687 ++++++++++++++++++++ 9 files changed, 736 insertions(+), 22 deletions(-) create mode 100644 schema/bench/scenario-v2.json diff --git a/CHANGES.md b/CHANGES.md index 8e1fb46a4..b9ed9f29b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -299,8 +299,8 @@ To be released. run weighted child scenario blends. The `collection` scenario type remains reserved but not executable. Fanout and remote failure scenarios can set `sinkBase` to generate deterministic benchmark sink inbox URLs for targets - that keep `triggerSinks` allowlisting enabled. [[#744], [#785], [#801], - [#802]] + that keep `triggerSinks` allowlisting enabled. This change is published + as benchmark scenario schema version 2. [[#744], [#785], [#801], [#802]] [#783]: https://github.com/fedify-dev/fedify/issues/783 [#784]: https://github.com/fedify-dev/fedify/issues/784 diff --git a/docs/manual/benchmarking.md b/docs/manual/benchmarking.md index ee338de3e..d20b94e0f 100644 --- a/docs/manual/benchmarking.md +++ b/docs/manual/benchmarking.md @@ -115,7 +115,7 @@ is a superset). The suite declares the `target`, shared `defaults`, the block of pass/fail thresholds: ~~~~ yaml -# yaml-language-server: $schema=https://json-schema.fedify.dev/bench/scenario-v1.json +# yaml-language-server: $schema=https://json-schema.fedify.dev/bench/scenario-v2.json version: 1 target: http://localhost:3000 defaults: @@ -161,7 +161,7 @@ list, deliveries are rotated across the recipients (and across the synthetic `actors` signing them), modeling a server that receives from many peers into many local inboxes. -[published schema]: https://json-schema.fedify.dev/bench/scenario-v1.json +[published schema]: https://json-schema.fedify.dev/bench/scenario-v2.json ### Scenario types diff --git a/packages/cli/src/bench/scenario/schema.ts b/packages/cli/src/bench/scenario/schema.ts index 4dc69bc2c..b7ff80ce8 100644 --- a/packages/cli/src/bench/scenario/schema.ts +++ b/packages/cli/src/bench/scenario/schema.ts @@ -1,9 +1,9 @@ /** * The embedded JSON Schema (draft 2020-12) for benchmark scenario suite files. * - * This object is the runtime copy used by the validator; it is published, - * byte-for-byte, as *schema/bench/scenario-v1.json* and a drift guard keeps the - * two in sync. The matching TypeScript types live in {@link ./types.ts}. + * These objects are the runtime copies used by the validator; they are + * published, byte-for-byte, under *schema/bench/* and a drift guard keeps them + * in sync. The matching TypeScript types live in {@link ./types.ts}. * * The schema expresses every scenario type discussed for `fedify bench` * (`inbox`, `webfinger`, `actor`, `object`, `fanout`, `collection`, `failure`, @@ -19,8 +19,12 @@ * @module */ -/** The hosted URL that serves the scenario schema. */ +/** The hosted URL that serves the current scenario schema. */ export const SCENARIO_SCHEMA_ID = + "https://json-schema.fedify.dev/bench/scenario-v2.json"; + +/** The hosted URL that serves the version 1 scenario schema. */ +export const SCENARIO_SCHEMA_ID_V1 = "https://json-schema.fedify.dev/bench/scenario-v1.json"; const READ_METRICS = [ @@ -60,7 +64,7 @@ const MIXED_METRICS = [...new Set([...INBOX_METRICS, ...FANOUT_METRICS])]; /** The benchmark scenario suite JSON Schema (draft 2020-12). */ export const scenarioSchemaV1 = { $schema: "https://json-schema.org/draft/2020-12/schema", - $id: SCENARIO_SCHEMA_ID, + $id: SCENARIO_SCHEMA_ID_V1, title: "Fedify benchmark scenario suite", type: "object", required: ["version", "scenarios"], @@ -288,7 +292,6 @@ export const scenarioSchemaV1 = { // fanout sender: { type: "string" }, followers: { type: "integer", minimum: 1 }, - sinkBase: { type: "string" }, trigger: { type: "object" }, sinkBehavior: { type: "object" }, queueDrainTimeout: { $ref: "#/$defs/duration" }, @@ -378,3 +381,19 @@ export const scenarioSchemaV1 = { }, }, } as const; + +/** The current benchmark scenario suite JSON Schema (draft 2020-12). */ +export const scenarioSchemaV2 = { + ...scenarioSchemaV1, + $id: SCENARIO_SCHEMA_ID, + $defs: { + ...scenarioSchemaV1.$defs, + scenario: { + ...scenarioSchemaV1.$defs.scenario, + properties: { + ...scenarioSchemaV1.$defs.scenario.properties, + sinkBase: { type: "string" }, + }, + }, + }, +} as const; diff --git a/packages/cli/src/bench/scenario/types.ts b/packages/cli/src/bench/scenario/types.ts index de73a9617..8bf1a4186 100644 --- a/packages/cli/src/bench/scenario/types.ts +++ b/packages/cli/src/bench/scenario/types.ts @@ -2,7 +2,7 @@ * Hand-written TypeScript types for the benchmark scenario suite format. * * These mirror the published JSON Schema in {@link ./schema.ts} and - * *schema/bench/scenario-v1.json*. Runtime validation is done with + * *schema/bench/scenario-v2.json*. Runtime validation is done with * `@cfworker/json-schema`; after a value validates, it is narrowed to * {@link Suite} with an `as unknown as` cast (see {@link ./validate.ts}). * @since 2.3.0 diff --git a/packages/cli/src/bench/scenario/validate.ts b/packages/cli/src/bench/scenario/validate.ts index d160bec81..02dbd6f93 100644 --- a/packages/cli/src/bench/scenario/validate.ts +++ b/packages/cli/src/bench/scenario/validate.ts @@ -5,14 +5,14 @@ */ import { type Schema, Validator } from "@cfworker/json-schema"; -import { scenarioSchemaV1 } from "./schema.ts"; +import { scenarioSchemaV2 } from "./schema.ts"; import { type RawValidationError, SuiteValidationError } from "./errors.ts"; import type { Suite } from "./types.ts"; let validator: Validator | undefined; function getValidator(): Validator { - validator ??= new Validator(scenarioSchemaV1 as unknown as Schema, "2020-12"); + validator ??= new Validator(scenarioSchemaV2 as unknown as Schema, "2020-12"); return validator; } diff --git a/packages/cli/src/bench/schemas.ts b/packages/cli/src/bench/schemas.ts index 14079f0e1..82f9e279b 100644 --- a/packages/cli/src/bench/schemas.ts +++ b/packages/cli/src/bench/schemas.ts @@ -10,7 +10,7 @@ */ import { reportSchemaV1, reportSchemaV2 } from "./result/schema.ts"; -import { scenarioSchemaV1 } from "./scenario/schema.ts"; +import { scenarioSchemaV1, scenarioSchemaV2 } from "./scenario/schema.ts"; /** A published JSON Schema and where it is hosted. */ export interface PublishedSchema { @@ -26,6 +26,11 @@ export interface PublishedSchema { export const PUBLISHED_SCHEMAS: readonly PublishedSchema[] = [ { name: "scenario", + fileName: "scenario-v2.json", + schema: scenarioSchemaV2 as unknown as Record, + }, + { + name: "scenario-v1", fileName: "scenario-v1.json", schema: scenarioSchemaV1 as unknown as Record, }, diff --git a/schema/README.md b/schema/README.md index 0ae5a8f86..7bd588534 100644 --- a/schema/README.md +++ b/schema/README.md @@ -6,13 +6,19 @@ Fedify JSON schemas This directory holds the published JSON Schemas (draft 2020-12) for Fedify file formats. It is deployed to by Netlify on every push to the *main* branch; the directory layout maps onto the URL, so -*schema/bench/scenario-v1.json* is served at -. +*schema/bench/scenario-v2.json* is served at +. Current schemas: - - *bench/scenario-v1.json* — the `fedify bench` scenario suite format (input). - - *bench/report-v1.json* — the `fedify bench` report format (output). + - *bench/scenario-v2.json* — the current `fedify bench` scenario suite + format (input). + - *bench/scenario-v1.json* — the version 1 `fedify bench` scenario suite + format (input). + - *bench/report-v2.json* — the current `fedify bench` report format + (output). + - *bench/report-v1.json* — the version 1 `fedify bench` report format + (output). Versioning: append-only and immutable @@ -82,7 +88,7 @@ Editor support Add a schema reference to a scenario file for autocomplete and validation: ~~~~ yaml -# yaml-language-server: $schema=https://json-schema.fedify.dev/bench/scenario-v1.json +# yaml-language-server: $schema=https://json-schema.fedify.dev/bench/scenario-v2.json version: 1 target: http://localhost:3000 ~~~~ diff --git a/schema/bench/scenario-v1.json b/schema/bench/scenario-v1.json index 40f6eeda1..c60a5a262 100644 --- a/schema/bench/scenario-v1.json +++ b/schema/bench/scenario-v1.json @@ -390,9 +390,6 @@ "type": "integer", "minimum": 1 }, - "sinkBase": { - "type": "string" - }, "trigger": { "type": "object" }, diff --git a/schema/bench/scenario-v2.json b/schema/bench/scenario-v2.json new file mode 100644 index 000000000..26c2f7987 --- /dev/null +++ b/schema/bench/scenario-v2.json @@ -0,0 +1,687 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.fedify.dev/bench/scenario-v2.json", + "title": "Fedify benchmark scenario suite", + "type": "object", + "required": [ + "version", + "scenarios" + ], + "additionalProperties": false, + "properties": { + "$schema": { + "type": "string", + "description": "An optional editor hint pointing at this schema." + }, + "version": { + "const": 1 + }, + "target": { + "type": "string", + "format": "uri", + "description": "The target base URL; may be overridden by --target." + }, + "defaults": { + "$ref": "#/$defs/defaults" + }, + "actors": { + "type": "array", + "items": { + "$ref": "#/$defs/actorGroup" + } + }, + "scenarios": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/scenario" + } + } + }, + "$defs": { + "duration": { + "type": "string", + "pattern": "^\\d+(\\.\\d+)?(ms|s|m|h)$", + "description": "A duration such as 500ms, 30s, 2m, or 1h." + }, + "rate": { + "description": "An open-loop arrival rate such as 200/s, or a number.", + "oneOf": [ + { + "type": "number", + "exclusiveMinimum": 0 + }, + { + "type": "string", + "pattern": "^\\d+(\\.\\d+)?\\s*/\\s*(s|m|h)$" + } + ] + }, + "size": { + "description": "A byte size such as 2KB or a plain number of bytes.", + "oneOf": [ + { + "type": "number", + "minimum": 0 + }, + { + "type": "string", + "pattern": "^\\s*\\d+(\\.\\d+)?\\s*([Bb]|[Kk][Bb]|[Kk][Ii][Bb]|[Mm][Bb]|[Mm][Ii][Bb]|[Gg][Bb]|[Gg][Ii][Bb])?\\s*$" + } + ] + }, + "signatureStandard": { + "enum": [ + "draft-cavage-http-signatures-12", + "rfc9421", + "ld-signatures", + "fep8b32" + ] + }, + "signingMode": { + "enum": [ + "jit", + "pipeline", + "presign" + ] + }, + "arrival": { + "enum": [ + "constant", + "poisson" + ] + }, + "scalarOrListString": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + } + ] + }, + "load": { + "type": "object", + "additionalProperties": false, + "properties": { + "rate": { + "$ref": "#/$defs/rate" + }, + "concurrency": { + "type": "integer", + "minimum": 1 + }, + "arrival": { + "$ref": "#/$defs/arrival" + }, + "maxInFlight": { + "type": "integer", + "minimum": 1 + } + }, + "not": { + "required": [ + "rate", + "concurrency" + ] + } + }, + "defaults": { + "type": "object", + "additionalProperties": false, + "properties": { + "duration": { + "$ref": "#/$defs/duration" + }, + "warmup": { + "$ref": "#/$defs/duration" + }, + "load": { + "$ref": "#/$defs/load" + }, + "signing": { + "$ref": "#/$defs/signingMode" + }, + "signatureTimeWindow": { + "type": "boolean" + }, + "runs": { + "type": "integer", + "minimum": 1 + } + } + }, + "actorGroup": { + "type": "object", + "additionalProperties": false, + "required": [ + "signatureStandards" + ], + "properties": { + "name": { + "type": "string" + }, + "count": { + "type": "integer", + "minimum": 1 + }, + "signatureStandards": { + "type": "array", + "uniqueItems": true, + "minItems": 1, + "items": { + "$ref": "#/$defs/signatureStandard" + }, + "contains": { + "enum": [ + "draft-cavage-http-signatures-12", + "rfc9421" + ] + }, + "minContains": 1, + "maxContains": 1, + "description": "Exactly one HTTP request signature scheme, plus optional document signature schemes." + } + } + }, + "generateDirective": { + "type": "object", + "additionalProperties": false, + "required": [ + "generate" + ], + "properties": { + "generate": { + "enum": [ + "lorem" + ] + }, + "size": { + "$ref": "#/$defs/size" + } + } + }, + "content": { + "oneOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/generateDirective" + } + ] + }, + "objectSpec": { + "type": "object", + "properties": { + "type": { + "$ref": "#/$defs/scalarOrListString" + }, + "content": { + "$ref": "#/$defs/content" + } + } + }, + "activitySpec": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "$ref": "#/$defs/scalarOrListString" + }, + "embedObject": { + "type": "boolean" + }, + "object": { + "$ref": "#/$defs/objectSpec" + } + } + }, + "objectSource": { + "oneOf": [ + { + "$ref": "#/$defs/scalarOrListString" + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "seed" + ], + "properties": { + "seed": { + "$ref": "#/$defs/scalarOrListString" + }, + "collection": { + "$ref": "#/$defs/scalarOrListString" + }, + "limit": { + "type": "integer", + "minimum": 1 + }, + "type": { + "$ref": "#/$defs/scalarOrListString" + } + } + } + ] + }, + "expectSeverity": { + "enum": [ + "warn", + "fail" + ] + }, + "expectValue": { + "oneOf": [ + { + "type": "string", + "description": "An assertion such as '>= 99%' or '< 100ms'." + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "assert" + ], + "properties": { + "assert": { + "type": "string" + }, + "severity": { + "$ref": "#/$defs/expectSeverity" + } + } + } + ] + }, + "mixEntry": { + "type": "object", + "additionalProperties": false, + "required": [ + "scenario", + "weight" + ], + "properties": { + "scenario": { + "type": "string" + }, + "weight": { + "type": "number", + "exclusiveMinimum": 0 + } + } + }, + "scenario": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "type" + ], + "properties": { + "name": { + "type": "string" + }, + "type": { + "enum": [ + "inbox", + "webfinger", + "actor", + "object", + "fanout", + "collection", + "failure", + "mixed" + ] + }, + "load": { + "$ref": "#/$defs/load" + }, + "duration": { + "$ref": "#/$defs/duration" + }, + "warmup": { + "$ref": "#/$defs/duration" + }, + "signing": { + "$ref": "#/$defs/signingMode" + }, + "signatureTimeWindow": { + "type": "boolean" + }, + "runs": { + "type": "integer", + "minimum": 1 + }, + "expect": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/expectValue" + } + }, + "recipient": { + "$ref": "#/$defs/scalarOrListString" + }, + "inbox": { + "type": "string" + }, + "activity": { + "$ref": "#/$defs/activitySpec" + }, + "authenticated": { + "type": "boolean" + }, + "collection": { + "$ref": "#/$defs/scalarOrListString" + }, + "source": { + "$ref": "#/$defs/objectSource" + }, + "sender": { + "type": "string" + }, + "followers": { + "type": "integer", + "minimum": 1 + }, + "trigger": { + "type": "object" + }, + "sinkBehavior": { + "type": "object" + }, + "queueDrainTimeout": { + "$ref": "#/$defs/duration" + }, + "fault": { + "$ref": "#/$defs/scalarOrListString" + }, + "mix": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/mixEntry" + } + }, + "sinkBase": { + "type": "string" + } + }, + "allOf": [ + { + "if": { + "properties": { + "type": { + "const": "inbox" + } + } + }, + "then": { + "required": [ + "recipient" + ], + "properties": { + "expect": { + "propertyNames": { + "enum": [ + "successRate", + "throughputPerSec", + "errors.total", + "errors.4xx", + "errors.5xx", + "latency.p50", + "latency.p95", + "latency.p99", + "latency.mean", + "latency.max", + "signatureVerification.p50", + "signatureVerification.p95", + "signatureVerification.p99" + ] + } + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "webfinger" + } + } + }, + "then": { + "required": [ + "recipient" + ], + "properties": { + "expect": { + "propertyNames": { + "enum": [ + "successRate", + "throughputPerSec", + "errors.total", + "errors.4xx", + "errors.5xx", + "latency.p50", + "latency.p95", + "latency.p99", + "latency.mean", + "latency.max" + ] + } + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "actor" + } + } + }, + "then": { + "required": [ + "recipient" + ], + "properties": { + "expect": { + "propertyNames": { + "enum": [ + "successRate", + "throughputPerSec", + "errors.total", + "errors.4xx", + "errors.5xx", + "latency.p50", + "latency.p95", + "latency.p99", + "latency.mean", + "latency.max", + "signatureVerification.p50", + "signatureVerification.p95", + "signatureVerification.p99" + ] + } + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "object" + } + } + }, + "then": { + "required": [ + "source" + ], + "properties": { + "expect": { + "propertyNames": { + "enum": [ + "successRate", + "throughputPerSec", + "errors.total", + "errors.4xx", + "errors.5xx", + "latency.p50", + "latency.p95", + "latency.p99", + "latency.mean", + "latency.max" + ] + } + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "collection" + } + } + }, + "then": { + "required": [ + "recipient" + ], + "properties": { + "expect": { + "propertyNames": { + "enum": [ + "successRate", + "throughputPerSec", + "errors.total", + "errors.4xx", + "errors.5xx", + "latency.p50", + "latency.p95", + "latency.p99", + "latency.mean", + "latency.max" + ] + } + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "fanout" + } + } + }, + "then": { + "required": [ + "sender" + ], + "properties": { + "expect": { + "propertyNames": { + "enum": [ + "successRate", + "deliveryThroughput", + "errors.total", + "errors.4xx", + "errors.5xx", + "queueDrain.p50", + "queueDrain.p95", + "queueDrain.p99" + ] + } + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "failure" + } + } + }, + "then": { + "required": [ + "fault" + ], + "properties": { + "expect": { + "propertyNames": { + "enum": [ + "successRate", + "throughputPerSec", + "errors.total", + "errors.4xx", + "errors.5xx", + "latency.p50", + "latency.p95", + "latency.p99", + "latency.mean", + "latency.max" + ] + } + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "mixed" + } + } + }, + "then": { + "required": [ + "mix" + ], + "properties": { + "expect": { + "propertyNames": { + "enum": [ + "successRate", + "throughputPerSec", + "errors.total", + "errors.4xx", + "errors.5xx", + "latency.p50", + "latency.p95", + "latency.p99", + "latency.mean", + "latency.max", + "signatureVerification.p50", + "signatureVerification.p95", + "signatureVerification.p99", + "deliveryThroughput", + "queueDrain.p50", + "queueDrain.p95", + "queueDrain.p99" + ] + } + } + } + } + } + ] + } + } +} From eca9e548b9e984797fe17cb7f23b90be8e9a538a Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 17:47:02 +0900 Subject: [PATCH 37/61] Continue after bad object candidates Malformed URL strings inside crawled ActivityPub object candidate arrays no longer stop object discovery from checking later candidates in the same activity. https://github.com/fedify-dev/fedify/pull/802#discussion_r3394394823 Assisted-by: Codex:gpt-5.5 --- .../src/bench/scenarios/object-discovery.ts | 5 +- .../cli/src/bench/scenarios/object.test.ts | 64 +++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/bench/scenarios/object-discovery.ts b/packages/cli/src/bench/scenarios/object-discovery.ts index 493cab645..e012de245 100644 --- a/packages/cli/src/bench/scenarios/object-discovery.ts +++ b/packages/cli/src/bench/scenarios/object-discovery.ts @@ -187,7 +187,10 @@ function objectUrl( ): URL | null { for (const candidate of objectCandidates(item)) { if (typeof candidate === "string") { - if (types.size < 1) return safeUrl(candidate, base); + if (types.size < 1) { + const url = safeUrl(candidate, base); + if (url != null) return url; + } continue; } if (!isRecord(candidate)) continue; diff --git a/packages/cli/src/bench/scenarios/object.test.ts b/packages/cli/src/bench/scenarios/object.test.ts index a0b0a0023..54c4c67f5 100644 --- a/packages/cli/src/bench/scenarios/object.test.ts +++ b/packages/cli/src/bench/scenarios/object.test.ts @@ -745,6 +745,70 @@ test("objectRunner - skips malformed URLs while crawling object sources", async assert.strictEqual(fetchedObjectUrl, "http://target.test/objects/valid"); }); +test("objectRunner - continues after malformed activity object URLs", async () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "object-crawl", + type: "object", + source: { + seed: "http://target.test/users/alice", + collection: "outbox", + limit: 1, + }, + load: { concurrency: 1 }, + duration: "25ms", + }], + }).scenarios[0]; + let fetchedObjectUrl = ""; + + const measurement = await objectRunner.run({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const request = input instanceof Request ? input : new Request(input); + const url = new URL(request.url); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return Promise.resolve(new Response("not found", { status: 404 })); + } + if (url.pathname === "/users/alice") { + return Promise.resolve(json({ + id: url.href, + outbox: "http://target.test/users/alice/outbox", + })); + } + if (url.pathname === "/users/alice/outbox") { + return Promise.resolve(json({ + id: url.href, + orderedItems: [{ + type: "Create", + object: [ + "http://[malformed", + { + type: "Note", + id: "http://target.test/objects/valid", + }, + ], + }], + })); + } + if (url.pathname === "/objects/valid") { + fetchedObjectUrl = url.href; + return Promise.resolve(json({ id: url.href, type: "Note" })); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }, + }); + + assert.ok(measurement.requests.total > 0); + assert.strictEqual(fetchedObjectUrl, "http://target.test/objects/valid"); +}); + test("objectRunner - caps object source crawl pages", async () => { const scenario = normalizeSuite({ version: 1, From d7c4d743cc6247feeb02020e4bf86590bba80dce Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 17:49:13 +0900 Subject: [PATCH 38/61] Share failure sinks for fault mixes Run non-network remote failure faults against one sink server with per-recipient responses so a fixed sinkBase port is not rebound during mixed fault setup. Reject the one sinkBase mix that cannot be modeled: network-error together with another remote fault. https://github.com/fedify-dev/fedify/pull/802#discussion_r3394417045 Assisted-by: Codex:gpt-5.5 --- .../cli/src/bench/scenarios/failure.test.ts | 73 +++++++++++++++++++ packages/cli/src/bench/scenarios/failure.ts | 50 +++++++++++-- packages/cli/src/bench/scenarios/fanout.ts | 14 +++- 3 files changed, 129 insertions(+), 8 deletions(-) diff --git a/packages/cli/src/bench/scenarios/failure.test.ts b/packages/cli/src/bench/scenarios/failure.test.ts index 845686391..5e5c2e5be 100644 --- a/packages/cli/src/bench/scenarios/failure.test.ts +++ b/packages/cli/src/bench/scenarios/failure.test.ts @@ -124,6 +124,79 @@ test("failureRunner - uses configured sink base for remote faults", async () => assert.strictEqual(recipientInbox, new URL("/inbox/0", sinkBase).href); }); +test("failureRunner - shares sink base across remote fault mix", async () => { + const target = new URL("http://target.test/"); + const sinkBase = `http://127.0.0.1:${await reservePort()}/`; + const scenario = normalizeSuite({ + version: 1, + target: target.href, + scenarios: [{ + name: "failure", + type: "failure", + fault: ["remote-404", "remote-410"], + sender: "alice", + sinkBase, + load: { concurrency: 1 }, + duration: "25ms", + queueDrainTimeout: "1s", + }], + }).scenarios[0]; + let triggerCalls = 0; + const recipientInboxes: string[] = []; + + const measurement = await failureRunner.run({ + scenario, + target, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input, init) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return Promise.resolve(statsJson(statsSnapshot({ + enqueued: triggerCalls, + completed: triggerCalls, + failed: 0, + permanentFailures: triggerCalls, + }))); + } + if (url.pathname === "/.well-known/fedify/bench/trigger") { + triggerCalls++; + const body = JSON.parse(String(init?.body)); + recipientInboxes.push(body.recipients[0].inbox); + return Promise.resolve(statsJson({ version: 1 }, 202)); + } + return Promise.resolve(new Response("unexpected", { status: 500 })); + }, + assertDestinationAllowed: () => {}, + }); + + assert.ok(measurement.requests.total > 1); + assert.strictEqual(measurement.requests.failed, 0); + assert.ok(recipientInboxes.includes(new URL("/inbox/0", sinkBase).href)); + assert.ok(recipientInboxes.includes(new URL("/inbox/1", sinkBase).href)); +}); + +test("failureRunner.validate - rejects sinkBase for mixed network faults", () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "failure", + type: "failure", + fault: ["network-error", "remote-404"], + sender: "alice", + sinkBase: "http://127.0.0.1:29999/", + }], + }).scenarios[0]; + + assert.throws( + () => failureRunner.validate?.(scenario), + /cannot combine network-error with other remote failure faults/, + ); +}); + test("failureRunner - detects network-error retries", async () => { const target = new URL("http://target.test/"); const scenario = normalizeSuite({ diff --git a/packages/cli/src/bench/scenarios/failure.ts b/packages/cli/src/bench/scenarios/failure.ts index 523200bfd..3f5c584ae 100644 --- a/packages/cli/src/bench/scenarios/failure.ts +++ b/packages/cli/src/bench/scenarios/failure.ts @@ -60,6 +60,7 @@ export const failureRunner: ScenarioRunner = { const faults = scenario.faults.length < 1 ? ["remote-404"] : scenario.faults; + const remoteFaults = [...new Set(faults.filter(isRemoteFault))]; for (const fault of faults) { if (!isSupportedFault(fault)) { throw new Error( @@ -88,6 +89,17 @@ export const failureRunner: ScenarioRunner = { if (faults.some(isRemoteFault)) { resolveSinkBase(scenario.name, scenario.raw.sinkBase); } + if ( + scenario.raw.sinkBase != null && + remoteFaults.includes("network-error") && + remoteFaults.some((fault) => fault !== "network-error") + ) { + throw new Error( + `Scenario "${scenario.name}": sinkBase cannot combine ` + + "network-error with other remote failure faults because the same " + + "port cannot be both open and unreachable.", + ); + } }, async run(context: RunContext) { @@ -189,22 +201,40 @@ async function resolveRemoteFailureTargets( ): Promise> { const targets = new Map(); try { - for (const fault of new Set(faults.filter(isRemoteFault))) { + const remoteFaults = [...new Set(faults.filter(isRemoteFault))]; + const liveFaults = remoteFaults.filter((fault) => + fault !== "network-error" + ); + if (liveFaults.length > 0) { + const sink = await spawnSinkServer({ + followers: liveFaults.length, + rawBehavior: null, + rawBehaviors: liveFaults.map(remoteSinkBehavior), + advertiseHost: context.advertiseHost, + sinkBase: context.scenario.raw.sinkBase, + }); + const close = once(sink.close); + for (const [index, fault] of liveFaults.entries()) { + targets.set(fault, { recipient: sink.recipients[index], close }); + } + } + if (remoteFaults.includes("network-error")) { const sink = await spawnSinkServer({ followers: 1, - rawBehavior: remoteSinkBehavior(fault), + rawBehavior: remoteSinkBehavior("network-error"), advertiseHost: context.advertiseHost, sinkBase: context.scenario.raw.sinkBase, }); const recipient = sink.recipients[0]; - if (fault === "network-error") { + try { await sink.close(); - targets.set(fault, { + targets.set("network-error", { recipient, close: () => Promise.resolve(), }); - } else { - targets.set(fault, { recipient, close: () => sink.close() }); + } catch (error) { + await sink.close().catch(() => {}); + throw error; } } return targets; @@ -214,6 +244,14 @@ async function resolveRemoteFailureTargets( } } +function once(close: () => Promise): () => Promise { + let closed: Promise | null = null; + return () => { + closed ??= close(); + return closed; + }; +} + function remoteSinkBehavior( fault: RemoteFailureFault, ): Record { diff --git a/packages/cli/src/bench/scenarios/fanout.ts b/packages/cli/src/bench/scenarios/fanout.ts index 5817e348c..846668386 100644 --- a/packages/cli/src/bench/scenarios/fanout.ts +++ b/packages/cli/src/bench/scenarios/fanout.ts @@ -182,6 +182,7 @@ function buildActivity(context: RunContext, id: URL): Record { export async function spawnSinkServer(options: { readonly followers: number; readonly rawBehavior: unknown; + readonly rawBehaviors?: readonly unknown[]; readonly advertiseHost?: string; readonly sinkBase?: string; }): Promise<{ @@ -192,13 +193,22 @@ export async function spawnSinkServer(options: { const advertised = options.advertiseHost == null ? null : resolveAdvertiseHost(options.advertiseHost); - const behavior = parseSinkBehavior(options.rawBehavior); + const behaviors = Array.from( + { length: options.followers }, + (_, i) => + parseSinkBehavior(options.rawBehaviors?.[i] ?? options.rawBehavior), + ); const server = serve({ port: sinkBase?.port ?? 0, hostname: sinkBase?.bindHost ?? advertised?.bindHost ?? "127.0.0.1", silent: true, async fetch(request: Request): Promise { - if (new URL(request.url).pathname.startsWith("/inbox/")) { + const match = /^\/inbox\/(\d+)(?:\/|$)/.exec( + new URL(request.url).pathname, + ); + if (match != null) { + const behavior = behaviors[Number(match[1])] ?? + parseSinkBehavior(options.rawBehavior); await request.arrayBuffer().catch(() => {}); if (behavior.latencyMs > 0) { await new Promise((resolve) => From dfaee9bbde8cf21833a1a6232af7830b83a4feb0 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 18:02:33 +0900 Subject: [PATCH 39/61] Drain failed discovery responses Consume unsuccessful ActivityPub discovery responses before throwing so crawl failures do not leave response bodies open under repeated benchmark discovery requests. https://github.com/fedify-dev/fedify/pull/802#discussion_r3394564431 Assisted-by: Codex:gpt-5.5 --- .../src/bench/scenarios/object-discovery.ts | 1 + .../cli/src/bench/scenarios/object.test.ts | 34 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/packages/cli/src/bench/scenarios/object-discovery.ts b/packages/cli/src/bench/scenarios/object-discovery.ts index e012de245..962d90d31 100644 --- a/packages/cli/src/bench/scenarios/object-discovery.ts +++ b/packages/cli/src/bench/scenarios/object-discovery.ts @@ -171,6 +171,7 @@ async function fetchJson( }), ); if (!response.ok) { + await response.arrayBuffer().catch(() => {}); throw new Error(`Failed to fetch ${url.href}: HTTP ${response.status}.`); } const json = await response.json(); diff --git a/packages/cli/src/bench/scenarios/object.test.ts b/packages/cli/src/bench/scenarios/object.test.ts index 54c4c67f5..eecebe7cd 100644 --- a/packages/cli/src/bench/scenarios/object.test.ts +++ b/packages/cli/src/bench/scenarios/object.test.ts @@ -518,6 +518,40 @@ test("objectRunner - gates collection URLs before crawling them", async () => { ); }); +test("objectRunner - drains failed discovery responses", async () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "object-crawl", + type: "object", + source: { + seed: "http://target.test/users/alice", + collection: "outbox", + limit: 1, + }, + load: { concurrency: 1 }, + duration: "25ms", + }], + }).scenarios[0]; + const failed = new Response("forbidden", { status: 403 }); + + await assert.rejects( + async () => + await objectRunner.run({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: () => Promise.resolve(failed), + }), + /Failed to fetch http:\/\/target\.test\/users\/alice: HTTP 403/, + ); + assert.strictEqual(failed.bodyUsed, true); +}); + test("objectRunner - resolves relative URLs while crawling object sources", async () => { const scenario = normalizeSuite({ version: 1, From 1c2cd49b0a99a0e710b8de05f39f201b379b3ddc Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 18:15:51 +0900 Subject: [PATCH 40/61] Require queue work before fanout drain Do not declare fanout delivery drained from a zero-delta queue snapshot. The runner now waits until the target reports queue work for the trigger before it can record delivery success and queue-drain latency. https://github.com/fedify-dev/fedify/pull/802#discussion_r3394652635 https://github.com/fedify-dev/fedify/pull/802#discussion_r3394663343 Assisted-by: Codex:gpt-5.5 --- .../cli/src/bench/scenarios/fanout.test.ts | 75 +++++++++++++++---- packages/cli/src/bench/scenarios/fanout.ts | 7 +- 2 files changed, 68 insertions(+), 14 deletions(-) diff --git a/packages/cli/src/bench/scenarios/fanout.test.ts b/packages/cli/src/bench/scenarios/fanout.test.ts index 6a814699f..c1d095f4c 100644 --- a/packages/cli/src/bench/scenarios/fanout.test.ts +++ b/packages/cli/src/bench/scenarios/fanout.test.ts @@ -8,7 +8,7 @@ import { fanoutRunner } from "./fanout.ts"; test("fanoutRunner - triggers benchmark hook and reports drain", async () => { const target = new URL("http://target.test/"); - let statsCalls = 0; + let triggerCalls = 0; let triggerRecipients = 0; const suite: Suite = { version: 1, @@ -35,15 +35,14 @@ test("fanoutRunner - triggers benchmark hook and reports drain", async () => { fetch: (input, init) => { const url = new URL(input instanceof Request ? input.url : input); if (url.pathname === "/.well-known/fedify/bench/stats") { - statsCalls++; - const drained = statsCalls > 1; return Promise.resolve(json(statsSnapshot({ - enqueued: drained ? 6 : 0, - completed: drained ? 6 : 0, + enqueued: triggerCalls * 6, + completed: triggerCalls * 6, failed: 0, }))); } if (url.pathname === "/.well-known/fedify/bench/trigger") { + triggerCalls++; const triggerBody = JSON.parse(String(init?.body)); triggerRecipients = triggerBody.recipients.length; return Promise.resolve(json({ version: 1 }, 202)); @@ -207,9 +206,60 @@ test("fanoutRunner - counts failed queue tasks as delivery failures", async () = assert.strictEqual(measurement.deliveryThroughputPerSec, 0); }); +test("fanoutRunner - waits for observed queue work before drain", async () => { + const target = new URL("http://target.test/"); + let statsCalls = 0; + let triggerCalls = 0; + const scenario = normalizeSuite({ + version: 1, + target: target.href, + scenarios: [{ + name: "fanout", + type: "fanout", + sender: "alice", + followers: 5, + load: { concurrency: 1 }, + duration: "40ms", + queueDrainTimeout: "1s", + }], + }).scenarios[0]; + + const measurement = await fanoutRunner.run({ + scenario, + target, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/.well-known/fedify/bench/stats") { + statsCalls++; + const observed = statsCalls > 2; + const queueTasks = observed ? triggerCalls * 6 : 0; + return Promise.resolve(json(statsSnapshot({ + enqueued: queueTasks, + completed: queueTasks, + failed: 0, + }))); + } + if (url.pathname === "/.well-known/fedify/bench/trigger") { + triggerCalls++; + return Promise.resolve(json({ version: 1 }, 202)); + } + return Promise.resolve(new Response("unexpected", { status: 500 })); + }, + }); + + assert.ok(measurement.requests.total > 0); + assert.strictEqual(measurement.requests.successRate, 1); + assert.ok(statsCalls > 2); +}); + test("fanoutRunner - tolerates transient drain stats failures", async () => { const target = new URL("http://target.test/"); let statsCalls = 0; + let triggerCalls = 0; const scenario = normalizeSuite({ version: 1, target: target.href, @@ -242,14 +292,14 @@ test("fanoutRunner - tolerates transient drain stats failures", async () => { }), ); } - const drained = statsCalls > 2; return Promise.resolve(json(statsSnapshot({ - enqueued: drained ? 6 : 0, - completed: drained ? 6 : 0, + enqueued: triggerCalls * 6, + completed: triggerCalls * 6, failed: 0, }))); } if (url.pathname === "/.well-known/fedify/bench/trigger") { + triggerCalls++; return Promise.resolve(json({ version: 1 }, 202)); } return Promise.resolve(new Response("unexpected", { status: 500 })); @@ -265,7 +315,7 @@ test("fanoutRunner - tolerates transient drain stats failures", async () => { test("fanoutRunner - uses configured sink base for recipients", async () => { const target = new URL("http://target.test/"); const sinkBase = `http://127.0.0.1:${await reservePort()}/`; - let statsCalls = 0; + let triggerCalls = 0; let recipientInboxes: string[] = []; const scenario = normalizeSuite({ version: 1, @@ -292,15 +342,14 @@ test("fanoutRunner - uses configured sink base for recipients", async () => { fetch: (input, init) => { const url = new URL(input instanceof Request ? input.url : input); if (url.pathname === "/.well-known/fedify/bench/stats") { - statsCalls++; - const drained = statsCalls > 1; return Promise.resolve(json(statsSnapshot({ - enqueued: drained ? 6 : 0, - completed: drained ? 6 : 0, + enqueued: triggerCalls * 6, + completed: triggerCalls * 6, failed: 0, }))); } if (url.pathname === "/.well-known/fedify/bench/trigger") { + triggerCalls++; const body = JSON.parse(String(init?.body)); recipientInboxes = body.recipients.map(( recipient: Record, diff --git a/packages/cli/src/bench/scenarios/fanout.ts b/packages/cli/src/bench/scenarios/fanout.ts index 846668386..594775a15 100644 --- a/packages/cli/src/bench/scenarios/fanout.ts +++ b/packages/cli/src/bench/scenarios/fanout.ts @@ -321,7 +321,12 @@ async function waitForDrain(options: { const diff = diffSnapshots(options.baseline, snapshot); const queueTasks = diff.queueTasks; const remaining = queueTaskRemaining(diff); - if (queueTasks != null && remaining != null && remaining === 0) { + if ( + queueTasks != null && + queueTasks.enqueued > 0 && + remaining != null && + remaining === 0 + ) { return { timedOut: false, failed: queueTasks.failed }; } } From c747792a00927ec24fc52e1b407923a2cfaa27ad Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 18:17:56 +0900 Subject: [PATCH 41/61] Parallelize read destination gates Validate read URLs up front, then run destination safety checks concurrently so large actor and object read source sets do not serialize all DNS and target classification work before load starts. https://github.com/fedify-dev/fedify/pull/802#discussion_r3394652652 Assisted-by: Codex:gpt-5.5 --- packages/cli/src/bench/scenarios/read.test.ts | 45 +++++++++++++++++++ packages/cli/src/bench/scenarios/read.ts | 4 +- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/bench/scenarios/read.test.ts b/packages/cli/src/bench/scenarios/read.test.ts index eb82d3d30..5acd9599e 100644 --- a/packages/cli/src/bench/scenarios/read.test.ts +++ b/packages/cli/src/bench/scenarios/read.test.ts @@ -49,6 +49,51 @@ test("runReadLoad - unauthenticated reads use the read destination gate", async assert.strictEqual(measurement.requests.successRate, 1); }); +test("runReadLoad - gates resolved read URLs concurrently", async () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "read", + type: "actor", + load: { concurrency: 1 }, + duration: "25ms", + }], + }).scenarios[0]; + let activeGates = 0; + let maxActiveGates = 0; + + await runReadLoad({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return Promise.resolve(new Response("not found", { status: 404 })); + } + return Promise.resolve(new Response("{}", { status: 200 })); + }, + assertReadDestinationAllowed: async () => { + activeGates++; + maxActiveGates = Math.max(maxActiveGates, activeGates); + await new Promise((resolve) => setTimeout(resolve, 5)); + activeGates--; + }, + }, { + urls: [ + new URL("http://remote.test/users/alice"), + new URL("http://remote.test/users/bob"), + ], + authenticated: false, + }); + + assert.strictEqual(maxActiveGates, 2); +}); + test("runReadLoad - rejects invalid read URL schemes before load", async () => { const scenario = normalizeSuite({ version: 1, diff --git a/packages/cli/src/bench/scenarios/read.ts b/packages/cli/src/bench/scenarios/read.ts index 504321596..a892afae3 100644 --- a/packages/cli/src/bench/scenarios/read.ts +++ b/packages/cli/src/bench/scenarios/read.ts @@ -61,12 +61,14 @@ export async function runReadLoad( } for (const url of options.urls) { assertBareHttpUrl(context.scenario.name, "read URL", url); + } + await Promise.all(options.urls.map(async (url) => { if (options.authenticated) { await context.assertDestinationAllowed?.(url); } else { await context.assertReadDestinationAllowed?.(url); } - } + })); function unsignedRequest(index: number): Request { const url = options.urls[index % options.urls.length]; From b4d4b4545a5cb1b5cfa064c7225af1ad8e1a206c Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 18:42:28 +0900 Subject: [PATCH 42/61] Ignore invalid sink latency values Invalid sinkBehavior latency strings could throw while preparing the local sink server. Treat malformed latency values as the default zero latency so the benchmark can still run with the rest of the sink behavior. https://github.com/fedify-dev/fedify/pull/802#discussion_r3394736117 Assisted-by: Codex:gpt-5.5 --- .../cli/src/bench/scenarios/fanout.test.ts | 18 +++++++++++++++++- packages/cli/src/bench/scenarios/fanout.ts | 10 +++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/bench/scenarios/fanout.test.ts b/packages/cli/src/bench/scenarios/fanout.test.ts index c1d095f4c..96e1de628 100644 --- a/packages/cli/src/bench/scenarios/fanout.test.ts +++ b/packages/cli/src/bench/scenarios/fanout.test.ts @@ -4,7 +4,7 @@ import { serve } from "srvx"; import { getContextLoader, getDocumentLoader } from "../../docloader.ts"; import { normalizeSuite } from "../scenario/normalize.ts"; import type { Suite } from "../scenario/types.ts"; -import { fanoutRunner } from "./fanout.ts"; +import { fanoutRunner, spawnSinkServer } from "./fanout.ts"; test("fanoutRunner - triggers benchmark hook and reports drain", async () => { const target = new URL("http://target.test/"); @@ -367,6 +367,22 @@ test("fanoutRunner - uses configured sink base for recipients", async () => { ); }); +test("spawnSinkServer - ignores invalid sink latency", async () => { + const sink = await spawnSinkServer({ + followers: 1, + rawBehavior: { latency: "not-a-duration", status: 202 }, + }); + try { + const response = await fetch(String(sink.recipients[0].inbox), { + method: "POST", + body: "{}", + }); + assert.strictEqual(response.status, 202); + } finally { + await sink.close(); + } +}); + test("fanoutRunner - omits queue drain metrics without drain samples", async () => { const target = new URL("http://target.test/"); const scenario = normalizeSuite({ diff --git a/packages/cli/src/bench/scenarios/fanout.ts b/packages/cli/src/bench/scenarios/fanout.ts index 594775a15..d0c89a361 100644 --- a/packages/cli/src/bench/scenarios/fanout.ts +++ b/packages/cli/src/bench/scenarios/fanout.ts @@ -294,8 +294,16 @@ function parseSinkBehavior( const record = raw as Record; const latency = record.latency; const status = record.status; + let latencyMs = 0; + if (typeof latency === "string") { + try { + latencyMs = parseDuration(latency); + } catch { + latencyMs = 0; + } + } return { - latencyMs: typeof latency === "string" ? parseDuration(latency) : 0, + latencyMs, status: typeof status === "number" && Number.isInteger(status) ? status : 202, From 83b85b1271df551a0b09da0b2386e6f1da24c180 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 18:44:29 +0900 Subject: [PATCH 43/61] Guard authenticated read actor keys Authenticated read signing now handles malformed synthetic actor records without crashing on a missing keys object. It reports the existing clear RSA signing error instead. https://github.com/fedify-dev/fedify/pull/802#discussion_r3394736131 Assisted-by: Codex:gpt-5.5 --- packages/cli/src/bench/scenarios/read.test.ts | 41 +++++++++++++++++++ packages/cli/src/bench/scenarios/read.ts | 2 +- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/bench/scenarios/read.test.ts b/packages/cli/src/bench/scenarios/read.test.ts index 5acd9599e..7d7b5383a 100644 --- a/packages/cli/src/bench/scenarios/read.test.ts +++ b/packages/cli/src/bench/scenarios/read.test.ts @@ -204,3 +204,44 @@ test("runReadLoad - authenticated reads support presign mode", async () => { await fleet?.close(); } }); + +test("runReadLoad - reports missing authenticated read keys clearly", async () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "read", + type: "actor", + authenticated: true, + signing: "pipeline", + load: { concurrency: 1 }, + duration: "25ms", + }], + }).scenarios[0]; + + await assert.rejects( + async () => + await runReadLoad({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: { + actors: [{ + id: new URL("http://synthetic.test/actors/0"), + index: 0, + keys: undefined, + }], + url: new URL("http://synthetic.test/"), + close: async () => {}, + } as unknown as Awaited>, + fetch: () => Promise.resolve(new Response("{}", { status: 200 })), + assertDestinationAllowed: () => {}, + }, { + urls: [new URL("http://remote.test/users/alice")], + authenticated: true, + }), + /Actor is missing the RSA key required for authenticated fetch signing/, + ); +}); diff --git a/packages/cli/src/bench/scenarios/read.ts b/packages/cli/src/bench/scenarios/read.ts index a892afae3..a7862fcf1 100644 --- a/packages/cli/src/bench/scenarios/read.ts +++ b/packages/cli/src/bench/scenarios/read.ts @@ -140,7 +140,7 @@ async function signGetRequest( request: Request, actor: SyntheticActor, ): Promise { - if (actor.keys.rsa == null || actor.rsaKeyId == null) { + if (actor.keys?.rsa == null || actor.rsaKeyId == null) { throw new TypeError( "Actor is missing the RSA key required for authenticated fetch signing.", ); From 035c5a56e2a89c78947cfc59b17e3186b4a177b0 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 18:44:35 +0900 Subject: [PATCH 44/61] Clarify object discovery JSON errors Object discovery now wraps JSON parse failures with the fetched URL so users can tell which actor or collection response was malformed. https://github.com/fedify-dev/fedify/pull/802#discussion_r3394736148 Assisted-by: Codex:gpt-5.5 --- .../src/bench/scenarios/object-discovery.ts | 7 +++- .../cli/src/bench/scenarios/object.test.ts | 32 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/bench/scenarios/object-discovery.ts b/packages/cli/src/bench/scenarios/object-discovery.ts index 962d90d31..dfb4cb14c 100644 --- a/packages/cli/src/bench/scenarios/object-discovery.ts +++ b/packages/cli/src/bench/scenarios/object-discovery.ts @@ -174,7 +174,12 @@ async function fetchJson( await response.arrayBuffer().catch(() => {}); throw new Error(`Failed to fetch ${url.href}: HTTP ${response.status}.`); } - const json = await response.json(); + let json: unknown; + try { + json = await response.json(); + } catch (error) { + throw new Error(`Failed to parse JSON from ${url.href}: ${error}`); + } if (!isRecord(json)) { throw new Error(`Expected ${url.href} to return a JSON object.`); } diff --git a/packages/cli/src/bench/scenarios/object.test.ts b/packages/cli/src/bench/scenarios/object.test.ts index eecebe7cd..b5d5111f1 100644 --- a/packages/cli/src/bench/scenarios/object.test.ts +++ b/packages/cli/src/bench/scenarios/object.test.ts @@ -552,6 +552,38 @@ test("objectRunner - drains failed discovery responses", async () => { assert.strictEqual(failed.bodyUsed, true); }); +test("objectRunner - reports invalid discovery JSON with URL context", async () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "object-crawl", + type: "object", + source: { + seed: "http://target.test/users/alice", + collection: "outbox", + limit: 1, + }, + load: { concurrency: 1 }, + duration: "25ms", + }], + }).scenarios[0]; + + await assert.rejects( + async () => + await objectRunner.run({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: () => Promise.resolve(new Response("not json", { status: 200 })), + }), + /Failed to parse JSON from http:\/\/target\.test\/users\/alice:/, + ); +}); + test("objectRunner - resolves relative URLs while crawling object sources", async () => { const scenario = normalizeSuite({ version: 1, From 39908e54351e35925f99e9228d747b4013d21e40 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 18:44:42 +0900 Subject: [PATCH 45/61] Ignore malformed stats sum points Stats parsing now checks sum data points before reading their values so malformed target-provided JSON cannot discard the whole snapshot. https://github.com/fedify-dev/fedify/pull/802#discussion_r3394736153 Assisted-by: Codex:gpt-5.5 --- .../src/bench/metrics/stats-client.test.ts | 24 +++++++++++++++++++ .../cli/src/bench/metrics/stats-client.ts | 6 ++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/bench/metrics/stats-client.test.ts b/packages/cli/src/bench/metrics/stats-client.test.ts index cb45b93f5..457874f56 100644 --- a/packages/cli/src/bench/metrics/stats-client.test.ts +++ b/packages/cli/src/bench/metrics/stats-client.test.ts @@ -155,6 +155,30 @@ test("parseServerSnapshot - extracts permanent delivery failures", () => { assert.strictEqual(snap?.deliveryPermanentFailures, 5); }); +test("parseServerSnapshot - skips malformed sum data points", () => { + const snap = parseServerSnapshot({ + scopeMetrics: [{ + metrics: [ + { + name: "fedify.queue.task.enqueued", + dataPointType: "sum", + dataPoints: [null, { value: 5 }], + }, + { + name: "fedify.queue.task.completed", + dataPointType: "sum", + dataPoints: [{ value: 3 }], + }, + ], + }], + }); + assert.deepEqual(snap?.queueTasks, { + enqueued: 5, + completed: 3, + failed: 0, + }); +}); + test("parseServerSnapshot - empty (non-null) when no relevant instruments", () => { // A parseable-but-empty snapshot yields an empty snapshot, not null, so a // successful baseline fetch is distinguishable from an unavailable one. diff --git a/packages/cli/src/bench/metrics/stats-client.ts b/packages/cli/src/bench/metrics/stats-client.ts index 018b45767..0a24fa352 100644 --- a/packages/cli/src/bench/metrics/stats-client.ts +++ b/packages/cli/src/bench/metrics/stats-client.ts @@ -309,7 +309,7 @@ function sumMetric( for (const metric of metrics) { if (metric.name !== name || !Array.isArray(metric.dataPoints)) continue; for (const point of metric.dataPoints) { - if (isFiniteNumber(point.value)) { + if (isRecord(point) && isFiniteNumber(point.value)) { total += point.value; found = true; } @@ -367,6 +367,10 @@ function histogramsCompatible( a.boundaries.every((boundary, i) => boundary === b.boundaries[i]); } +function isRecord(value: unknown): value is Record { + return value != null && typeof value === "object" && !Array.isArray(value); +} + function histogramPercentile(histogram: ServerHistogram, p: number): number { const { boundaries, counts } = histogram; const total = counts.reduce((sum, n) => sum + n, 0); From 77de05968604ec8d610ad9fc602feb8feda137d3 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 19:04:45 +0900 Subject: [PATCH 46/61] Default failure reachability checks Failure scenarios without an explicit fault now use the same remote-404 default when checking local sink reachability as the runner uses while executing the scenario. Scenario schema v2 also allows the omitted fault form so the documented runner default is reachable from CLI suites. https://github.com/fedify-dev/fedify/pull/802#discussion_r3394916834 Assisted-by: Codex:gpt-5.5 --- .../invalid/failure-missing-fault.yaml | 6 ----- packages/cli/src/bench/action.test.ts | 26 +++++++++++++++++++ packages/cli/src/bench/action.ts | 11 +++++--- packages/cli/src/bench/scenario/schema.ts | 10 +++++++ schema/bench/scenario-v2.json | 3 --- 5 files changed, 44 insertions(+), 12 deletions(-) delete mode 100644 packages/cli/src/bench/__fixtures__/invalid/failure-missing-fault.yaml diff --git a/packages/cli/src/bench/__fixtures__/invalid/failure-missing-fault.yaml b/packages/cli/src/bench/__fixtures__/invalid/failure-missing-fault.yaml deleted file mode 100644 index 88f6e3abd..000000000 --- a/packages/cli/src/bench/__fixtures__/invalid/failure-missing-fault.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# A failure scenario must declare at least one fault. -version: 1 -target: http://localhost:3000 -scenarios: - - name: broken - type: failure diff --git a/packages/cli/src/bench/action.test.ts b/packages/cli/src/bench/action.test.ts index bca1e7594..52327a736 100644 --- a/packages/cli/src/bench/action.test.ts +++ b/packages/cli/src/bench/action.test.ts @@ -463,6 +463,32 @@ scenarios: assert.match(message, /advertise-host/); }); +test("runBench - default failure fault needs advertised sink reachability", async () => { + const file = await writeSuite(`version: 1 +target: http://10.10.0.5:8000 +scenarios: + - name: default-failure + type: failure + sender: alice + load: { rate: 1/s } + duration: 1ms +`); + let code = -1; + let message = ""; + await runBench(command({ scenario: file }), { + exit: (c) => { + code = c; + }, + writeOutput: () => Promise.resolve(), + log: (m) => { + message = m; + }, + fetch: () => Promise.reject(new Error("offline")), + }); + assert.strictEqual(code, 2); + assert.match(message, /advertise-host/); +}); + test("runBench - remote failure with sinkBase needs no advertise host", async () => { const sinkBase = `http://127.0.0.1:${await reservePort()}/`; const file = await writeSuite(`version: 1 diff --git a/packages/cli/src/bench/action.ts b/packages/cli/src/bench/action.ts index e63740440..d02ec64f8 100644 --- a/packages/cli/src/bench/action.ts +++ b/packages/cli/src/bench/action.ts @@ -685,7 +685,7 @@ function scenarioNeedsSyntheticServer( case "object": return scenario.authenticated; case "failure": - return scenario.faults.some(isInboundFailureFault); + return failureFaultsOf(scenario).some(isInboundFailureFault); case "mixed": return mixedChildrenOf(scenario, scenarios).some((child) => scenarioNeedsSyntheticServer(child, scenarios, nextSeen) @@ -702,9 +702,10 @@ function scenarioNeedsReachableLocalServer( ): boolean { if (scenario.type === "fanout") return scenario.raw.sinkBase == null; if (scenario.type === "failure") { - return scenario.faults.includes("invalid-signature") || + const faults = failureFaultsOf(scenario); + return faults.includes("invalid-signature") || (scenario.raw.sinkBase == null && - scenario.faults.some(isRemoteFailureFault)); + faults.some(isRemoteFailureFault)); } if (scenario.type === "mixed") { if (seen.has(scenario.name)) return false; @@ -716,6 +717,10 @@ function scenarioNeedsReachableLocalServer( return scenarioNeedsSyntheticServer(scenario, scenarios, seen); } +function failureFaultsOf(scenario: ResolvedScenario): readonly string[] { + return scenario.faults.length < 1 ? ["remote-404"] : scenario.faults; +} + function mixedChildrenOf( scenario: ResolvedScenario, scenarios: readonly ResolvedScenario[], diff --git a/packages/cli/src/bench/scenario/schema.ts b/packages/cli/src/bench/scenario/schema.ts index b7ff80ce8..28c1eb77f 100644 --- a/packages/cli/src/bench/scenario/schema.ts +++ b/packages/cli/src/bench/scenario/schema.ts @@ -394,6 +394,16 @@ export const scenarioSchemaV2 = { ...scenarioSchemaV1.$defs.scenario.properties, sinkBase: { type: "string" }, }, + allOf: scenarioSchemaV1.$defs.scenario.allOf.map((condition) => + condition.if.properties.type.const === "failure" + ? { + if: condition.if, + then: { + properties: condition.then.properties, + }, + } + : condition + ), }, }, } as const; diff --git a/schema/bench/scenario-v2.json b/schema/bench/scenario-v2.json index 26c2f7987..433572c19 100644 --- a/schema/bench/scenario-v2.json +++ b/schema/bench/scenario-v2.json @@ -619,9 +619,6 @@ } }, "then": { - "required": [ - "fault" - ], "properties": { "expect": { "propertyNames": { From 92c310d958ca85096d4167a612b4b3394ba7df01 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 22:36:56 +0900 Subject: [PATCH 47/61] Gate mixed children as children Mixed child runners now re-bind destination safety gates with the scaled child scenario instead of inheriting the parent mixed scenario. This keeps child-specific settings such as destination overrides visible to the safety checks. https://github.com/fedify-dev/fedify/pull/802#discussion_r3395016627 Assisted-by: Codex:gpt-5.5 --- packages/cli/src/bench/action.ts | 12 ++--- .../cli/src/bench/scenarios/mixed.test.ts | 48 +++++++++++++++++++ packages/cli/src/bench/scenarios/mixed.ts | 6 +++ packages/cli/src/bench/scenarios/runner.ts | 11 ++++- 4 files changed, 69 insertions(+), 8 deletions(-) diff --git a/packages/cli/src/bench/action.ts b/packages/cli/src/bench/action.ts index d02ec64f8..afc1f6f01 100644 --- a/packages/cli/src/bench/action.ts +++ b/packages/cli/src/bench/action.ts @@ -310,12 +310,12 @@ export default async function runBench( fleet: fleet ?? null, advertiseHost: command.advertiseHost, fetch: fetchImpl, - assertDestinationAllowed: (url) => - assertDestinationAllowed(url, scenario), - assertReadDestinationAllowed: (url) => - assertReadDestinationAllowed(url, scenario), - assertActorlessDestinationAllowed: (url) => - assertActorlessDestinationAllowed(url, scenario), + assertDestinationAllowed: (url, gateScenario) => + assertDestinationAllowed(url, gateScenario ?? scenario), + assertReadDestinationAllowed: (url, gateScenario) => + assertReadDestinationAllowed(url, gateScenario ?? scenario), + assertActorlessDestinationAllowed: (url, gateScenario) => + assertActorlessDestinationAllowed(url, gateScenario ?? scenario), }); results.push(buildScenarioResult(scenario, measurement)); } diff --git a/packages/cli/src/bench/scenarios/mixed.test.ts b/packages/cli/src/bench/scenarios/mixed.test.ts index f0452e571..3b3d3b838 100644 --- a/packages/cli/src/bench/scenarios/mixed.test.ts +++ b/packages/cli/src/bench/scenarios/mixed.test.ts @@ -65,6 +65,54 @@ test("mixedRunner - runs weighted child scenarios together", async () => { assert.strictEqual(measurement.requests.successRate, 1); }); +test("mixedRunner - gates child destinations with child scenarios", async () => { + const target = new URL("http://target.test/"); + const suite: Suite = { + version: 1, + target: target.href, + scenarios: [ + { + name: "child-read", + type: "actor", + recipient: "http://remote.test/users/alice", + load: { concurrency: 1 }, + duration: "25ms", + }, + { + name: "mixed", + type: "mixed", + load: { concurrency: 1 }, + duration: "25ms", + mix: [{ scenario: "child-read", weight: 1 }], + }, + ], + }; + const scenarios = normalizeSuite(suite).scenarios; + const gatedScenarioNames: string[] = []; + + await mixedRunner.run({ + scenario: scenarios[1], + scenarios, + target, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return Promise.resolve(new Response("not found", { status: 404 })); + } + return Promise.resolve(new Response("{}", { status: 200 })); + }, + assertReadDestinationAllowed: (_url, scenario) => { + gatedScenarioNames.push(scenario?.name ?? ""); + }, + }); + + assert.deepStrictEqual(gatedScenarioNames, ["mixed/child-read"]); +}); + test("mixedRunner - enforces parent maxInFlight across children", async () => { const target = new URL("http://target.test/"); let activeLookups = 0; diff --git a/packages/cli/src/bench/scenarios/mixed.ts b/packages/cli/src/bench/scenarios/mixed.ts index dcd81af7d..e0c00b288 100644 --- a/packages/cli/src/bench/scenarios/mixed.ts +++ b/packages/cli/src/bench/scenarios/mixed.ts @@ -78,6 +78,12 @@ export const mixedRunner: ScenarioRunner = { ...context, scenario: child, fetch: fetchImpl, + assertDestinationAllowed: (url, scenario) => + context.assertDestinationAllowed?.(url, scenario ?? child), + assertReadDestinationAllowed: (url, scenario) => + context.assertReadDestinationAllowed?.(url, scenario ?? child), + assertActorlessDestinationAllowed: (url, scenario) => + context.assertActorlessDestinationAllowed?.(url, scenario ?? child), }) ), ); diff --git a/packages/cli/src/bench/scenarios/runner.ts b/packages/cli/src/bench/scenarios/runner.ts index b34211e48..eb1ce970d 100644 --- a/packages/cli/src/bench/scenarios/runner.ts +++ b/packages/cli/src/bench/scenarios/runner.ts @@ -39,19 +39,26 @@ export interface RunContext { * The suite `target` is gated by the orchestrator; this covers destinations * that differ from it. Optional so direct runner tests need not supply it. */ - readonly assertDestinationAllowed?: (url: URL) => void | Promise; + readonly assertDestinationAllowed?: ( + url: URL, + scenario?: ResolvedScenario, + ) => void | Promise; /** * Gates a resolved read-only destination before unauthenticated GET load is * sent. Unlike signed inbox delivery, these reads do not require a * reachable synthetic actor server. */ - readonly assertReadDestinationAllowed?: (url: URL) => void | Promise; + readonly assertReadDestinationAllowed?: ( + url: URL, + scenario?: ResolvedScenario, + ) => void | Promise; /** * Gates a destination for benchmark load that does not require remote * dereferencing of benchmark-owned synthetic actors. */ readonly assertActorlessDestinationAllowed?: ( url: URL, + scenario?: ResolvedScenario, ) => void | Promise; } From 2a8235764aa794d5e301a8096cb13b949a21632c Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 22:39:16 +0900 Subject: [PATCH 48/61] Limit read destination gate fanout Read scenarios now gate resolved URLs through a bounded worker pool instead of launching every destination check at once. This avoids creating a burst of DNS and socket work for large resolved actor or object URL lists. https://github.com/fedify-dev/fedify/pull/802#discussion_r3395016634 Assisted-by: Codex:gpt-5.5 --- packages/cli/src/bench/scenarios/read.test.ts | 45 +++++++++++++++++++ packages/cli/src/bench/scenarios/read.ts | 24 +++++++++- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/bench/scenarios/read.test.ts b/packages/cli/src/bench/scenarios/read.test.ts index 7d7b5383a..1535d4574 100644 --- a/packages/cli/src/bench/scenarios/read.test.ts +++ b/packages/cli/src/bench/scenarios/read.test.ts @@ -94,6 +94,51 @@ test("runReadLoad - gates resolved read URLs concurrently", async () => { assert.strictEqual(maxActiveGates, 2); }); +test("runReadLoad - limits resolved read URL gate concurrency", async () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "read", + type: "actor", + load: { concurrency: 1 }, + duration: "25ms", + }], + }).scenarios[0]; + let activeGates = 0; + let maxActiveGates = 0; + + await runReadLoad({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return Promise.resolve(new Response("not found", { status: 404 })); + } + return Promise.resolve(new Response("{}", { status: 200 })); + }, + assertReadDestinationAllowed: async () => { + activeGates++; + maxActiveGates = Math.max(maxActiveGates, activeGates); + await new Promise((resolve) => setTimeout(resolve, 5)); + activeGates--; + }, + }, { + urls: Array.from( + { length: 32 }, + (_, i) => new URL(`http://remote.test/users/${i}`), + ), + authenticated: false, + }); + + assert.strictEqual(maxActiveGates, 16); +}); + test("runReadLoad - rejects invalid read URL schemes before load", async () => { const scenario = normalizeSuite({ version: 1, diff --git a/packages/cli/src/bench/scenarios/read.ts b/packages/cli/src/bench/scenarios/read.ts index a7862fcf1..fd5d98b5e 100644 --- a/packages/cli/src/bench/scenarios/read.ts +++ b/packages/cli/src/bench/scenarios/read.ts @@ -28,6 +28,8 @@ import { withMeasuredWindowStart, } from "./runner.ts"; +const READ_GATE_CONCURRENCY = 16; + /** Options for {@link runReadLoad}. */ export interface ReadLoadOptions { /** URLs to GET during the measured load. */ @@ -62,13 +64,13 @@ export async function runReadLoad( for (const url of options.urls) { assertBareHttpUrl(context.scenario.name, "read URL", url); } - await Promise.all(options.urls.map(async (url) => { + await mapWithConcurrency(options.urls, READ_GATE_CONCURRENCY, async (url) => { if (options.authenticated) { await context.assertDestinationAllowed?.(url); } else { await context.assertReadDestinationAllowed?.(url); } - })); + }); function unsignedRequest(index: number): Request { const url = options.urls[index % options.urls.length]; @@ -136,6 +138,24 @@ export async function runReadLoad( } } +async function mapWithConcurrency( + items: readonly T[], + concurrency: number, + callback: (item: T) => Promise, +): Promise { + let next = 0; + const workers = Array.from( + { length: Math.min(concurrency, items.length) }, + async () => { + while (next < items.length) { + const item = items[next++]; + await callback(item); + } + }, + ); + await Promise.all(workers); +} + async function signGetRequest( request: Request, actor: SyntheticActor, From c2316657112562c4c4553848ac36125fc74906aa Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 22:59:33 +0900 Subject: [PATCH 49/61] Skip wrapper activities without objects Object discovery now drops ActivityPub wrapper activities that do not carry an object field instead of benchmarking the wrapper activity URL as if it were the target object. https://github.com/fedify-dev/fedify/pull/802#discussion_r3396459842 Assisted-by: Codex:gpt-5.5 --- .../src/bench/scenarios/object-discovery.ts | 3 +- .../cli/src/bench/scenarios/object.test.ts | 57 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/bench/scenarios/object-discovery.ts b/packages/cli/src/bench/scenarios/object-discovery.ts index dfb4cb14c..801add7ac 100644 --- a/packages/cli/src/bench/scenarios/object-discovery.ts +++ b/packages/cli/src/bench/scenarios/object-discovery.ts @@ -212,8 +212,9 @@ function objectCandidates(item: unknown): unknown[] { return [item]; } const object = item.object; + if (object == null) return []; if (Array.isArray(object)) return object.filter((entry) => entry != null); - return [object ?? item]; + return [object]; } function matchesType( diff --git a/packages/cli/src/bench/scenarios/object.test.ts b/packages/cli/src/bench/scenarios/object.test.ts index b5d5111f1..f122326d9 100644 --- a/packages/cli/src/bench/scenarios/object.test.ts +++ b/packages/cli/src/bench/scenarios/object.test.ts @@ -325,6 +325,63 @@ test("objectRunner - prefers unwrapped object URLs without type filters", async assert.strictEqual(fetchedActivity, false); }); +test("objectRunner - skips wrapper activities without objects", async () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "object-crawl", + type: "object", + source: { + seed: "http://target.test/users/alice", + collection: "outbox", + limit: 1, + }, + load: { concurrency: 1 }, + duration: "25ms", + }], + }).scenarios[0]; + let fetchedActivity = false; + + await assert.rejects( + async () => + objectRunner.run({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const request = input instanceof Request ? input : new Request(input); + const url = new URL(request.url); + if (url.pathname === "/users/alice") { + return Promise.resolve(json({ + id: url.href, + outbox: "http://target.test/users/alice/outbox", + })); + } + if (url.pathname === "/users/alice/outbox") { + return Promise.resolve(json({ + id: url.href, + orderedItems: [{ + type: "Create", + id: "http://target.test/activities/create-1", + }], + })); + } + if (url.pathname === "/activities/create-1") { + fetchedActivity = true; + return Promise.resolve(json({ id: url.href, type: "Create" })); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }, + }), + /did not resolve any URLs/, + ); + assert.strictEqual(fetchedActivity, false); +}); + test("objectRunner - sends ActivityPub Accept headers during object discovery", async () => { const scenario = normalizeSuite({ version: 1, From 4241614c6292c645424acb906d4931bc545a2ac7 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 11 Jun 2026 23:01:51 +0900 Subject: [PATCH 50/61] Allow signed object metric gates Scenario schema v2 now permits signatureVerification expectations for authenticated object reads while keeping unauthenticated object scenarios limited to read metrics. https://github.com/fedify-dev/fedify/pull/802#discussion_r3396446638 Assisted-by: Codex:gpt-5.5 --- ...ject-signature-expect-unauthenticated.yaml | 9 +++ .../authenticated-object-expect.yaml | 10 +++ packages/cli/src/bench/scenario/schema.ts | 25 +++++++ schema/bench/scenario-v2.json | 71 ++++++++++++++----- 4 files changed, 99 insertions(+), 16 deletions(-) create mode 100644 packages/cli/src/bench/__fixtures__/invalid/object-signature-expect-unauthenticated.yaml create mode 100644 packages/cli/src/bench/__fixtures__/scenarios/authenticated-object-expect.yaml diff --git a/packages/cli/src/bench/__fixtures__/invalid/object-signature-expect-unauthenticated.yaml b/packages/cli/src/bench/__fixtures__/invalid/object-signature-expect-unauthenticated.yaml new file mode 100644 index 000000000..0dd4b5dda --- /dev/null +++ b/packages/cli/src/bench/__fixtures__/invalid/object-signature-expect-unauthenticated.yaml @@ -0,0 +1,9 @@ +# signatureVerification.* is only valid for authenticated object scenarios. +version: 1 +target: http://localhost:3000 +scenarios: + - name: object-fetch + type: object + source: http://localhost:3000/objects/1 + expect: + signatureVerification.p95: "< 10ms" diff --git a/packages/cli/src/bench/__fixtures__/scenarios/authenticated-object-expect.yaml b/packages/cli/src/bench/__fixtures__/scenarios/authenticated-object-expect.yaml new file mode 100644 index 000000000..e591096e1 --- /dev/null +++ b/packages/cli/src/bench/__fixtures__/scenarios/authenticated-object-expect.yaml @@ -0,0 +1,10 @@ +# yaml-language-server: $schema=https://json-schema.fedify.dev/bench/scenario-v2.json +version: 1 +target: http://localhost:3000 +scenarios: + - name: signed-object-fetch + type: object + authenticated: true + source: http://localhost:3000/objects/1 + expect: + signatureVerification.p95: "< 10ms" diff --git a/packages/cli/src/bench/scenario/schema.ts b/packages/cli/src/bench/scenario/schema.ts index 28c1eb77f..b45f3dd3b 100644 --- a/packages/cli/src/bench/scenario/schema.ts +++ b/packages/cli/src/bench/scenario/schema.ts @@ -402,6 +402,31 @@ export const scenarioSchemaV2 = { properties: condition.then.properties, }, } + : condition.if.properties.type.const === "object" + ? { + if: condition.if, + then: { + required: condition.then.required, + allOf: [ + { + if: { + required: ["authenticated"], + properties: { authenticated: { const: true } }, + }, + then: { + properties: { + expect: { propertyNames: { enum: INBOX_METRICS } }, + }, + }, + else: { + properties: { + expect: { propertyNames: { enum: READ_METRICS } }, + }, + }, + }, + ], + }, + } : condition ), }, diff --git a/schema/bench/scenario-v2.json b/schema/bench/scenario-v2.json index 433572c19..928d53a77 100644 --- a/schema/bench/scenario-v2.json +++ b/schema/bench/scenario-v2.json @@ -528,24 +528,63 @@ "required": [ "source" ], - "properties": { - "expect": { - "propertyNames": { - "enum": [ - "successRate", - "throughputPerSec", - "errors.total", - "errors.4xx", - "errors.5xx", - "latency.p50", - "latency.p95", - "latency.p99", - "latency.mean", - "latency.max" - ] + "allOf": [ + { + "if": { + "required": [ + "authenticated" + ], + "properties": { + "authenticated": { + "const": true + } + } + }, + "then": { + "properties": { + "expect": { + "propertyNames": { + "enum": [ + "successRate", + "throughputPerSec", + "errors.total", + "errors.4xx", + "errors.5xx", + "latency.p50", + "latency.p95", + "latency.p99", + "latency.mean", + "latency.max", + "signatureVerification.p50", + "signatureVerification.p95", + "signatureVerification.p99" + ] + } + } + } + }, + "else": { + "properties": { + "expect": { + "propertyNames": { + "enum": [ + "successRate", + "throughputPerSec", + "errors.total", + "errors.4xx", + "errors.5xx", + "latency.p50", + "latency.p95", + "latency.p99", + "latency.mean", + "latency.max" + ] + } + } + } } } - } + ] } }, { From 36dd1058ad4050073512badf2294f1f13e0c3e10 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Fri, 12 Jun 2026 13:59:06 +0900 Subject: [PATCH 51/61] Align mixed expectation schema Mixed scenario validation rejects server-side metric expectations because mixed results do not merge those snapshots. Align schema v2 with that runtime contract while keeping deliveryThroughput available. https://github.com/fedify-dev/fedify/pull/802#pullrequestreview-4481841464 Assisted-by: Codex:gpt-5.5 --- .../invalid/mixed-server-metric.yaml | 10 +++++++ packages/cli/src/bench/scenario/schema.ts | 16 ++++++++++- .../cli/src/bench/scenario/validate.test.ts | 28 +++++++++++++++++++ schema/bench/scenario-v2.json | 8 +----- 4 files changed, 54 insertions(+), 8 deletions(-) create mode 100644 packages/cli/src/bench/__fixtures__/invalid/mixed-server-metric.yaml diff --git a/packages/cli/src/bench/__fixtures__/invalid/mixed-server-metric.yaml b/packages/cli/src/bench/__fixtures__/invalid/mixed-server-metric.yaml new file mode 100644 index 000000000..d2898921b --- /dev/null +++ b/packages/cli/src/bench/__fixtures__/invalid/mixed-server-metric.yaml @@ -0,0 +1,10 @@ +# Server-side metrics are not merged for mixed scenario results. +version: 1 +target: http://localhost:3000 +scenarios: + - name: realistic-blend + type: mixed + mix: + - { scenario: fanout-1k, weight: 1 } + expect: + queueDrain.p95: "< 2s" diff --git a/packages/cli/src/bench/scenario/schema.ts b/packages/cli/src/bench/scenario/schema.ts index b45f3dd3b..40253c62f 100644 --- a/packages/cli/src/bench/scenario/schema.ts +++ b/packages/cli/src/bench/scenario/schema.ts @@ -58,8 +58,12 @@ const FANOUT_METRICS = [ "queueDrain.p99", ]; -// A `mixed` scenario blends others, so it may assert any of their metrics. +// Schema v1 allowed mixed scenarios to assert child server metrics. Keep this +// list stable so the published v1 schema remains byte-for-byte immutable. const MIXED_METRICS = [...new Set([...INBOX_METRICS, ...FANOUT_METRICS])]; +// Schema v2 matches the runtime mixed runner, which merges only client-side +// request metrics and delivery throughput. +const MIXED_V2_METRICS = [...READ_METRICS, "deliveryThroughput"]; /** The benchmark scenario suite JSON Schema (draft 2020-12). */ export const scenarioSchemaV1 = { @@ -427,6 +431,16 @@ export const scenarioSchemaV2 = { ], }, } + : condition.if.properties.type.const === "mixed" + ? { + if: condition.if, + then: { + required: condition.then.required, + properties: { + expect: { propertyNames: { enum: MIXED_V2_METRICS } }, + }, + }, + } : condition ), }, diff --git a/packages/cli/src/bench/scenario/validate.test.ts b/packages/cli/src/bench/scenario/validate.test.ts index ebc938c92..bf39a3a5c 100644 --- a/packages/cli/src/bench/scenario/validate.test.ts +++ b/packages/cli/src/bench/scenario/validate.test.ts @@ -100,6 +100,34 @@ test("validateSuite - enforces per-type expect metric allowlist", () => { assert.throws(() => validateSuite(bad), SuiteValidationError); }); +test("validateSuite - rejects mixed server-side expect metrics", () => { + const bad = { + version: 1, + target: "http://localhost:3000", + scenarios: [{ + name: "blend", + type: "mixed", + mix: [{ scenario: "fanout", weight: 1 }], + expect: { "queueDrain.p95": "< 10ms" }, + }], + }; + assert.throws(() => validateSuite(bad), SuiteValidationError); +}); + +test("validateSuite - accepts mixed delivery throughput expectations", () => { + const suite = { + version: 1, + target: "http://localhost:3000", + scenarios: [{ + name: "blend", + type: "mixed", + mix: [{ scenario: "fanout", weight: 1 }], + expect: { deliveryThroughput: ">= 1/s" }, + }], + }; + assert.doesNotThrow(() => validateSuite(suite)); +}); + test("validateSuite - error message names the failing location", () => { try { validateSuite({ target: "http://localhost:3000", scenarios: [] }); diff --git a/schema/bench/scenario-v2.json b/schema/bench/scenario-v2.json index 928d53a77..117429c9a 100644 --- a/schema/bench/scenario-v2.json +++ b/schema/bench/scenario-v2.json @@ -704,13 +704,7 @@ "latency.p99", "latency.mean", "latency.max", - "signatureVerification.p50", - "signatureVerification.p95", - "signatureVerification.p99", - "deliveryThroughput", - "queueDrain.p50", - "queueDrain.p95", - "queueDrain.p99" + "deliveryThroughput" ] } } From 93bf1741246da0c7a9bbee86d733fb91d9e8061e Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Fri, 12 Jun 2026 15:12:24 +0900 Subject: [PATCH 52/61] Unwrap nested object activities Object discovery now recursively unwraps ActivityPub wrapper activities, so collections containing entries such as Announce(Create(Note)) still resolve the underlying object when a type filter is configured. https://github.com/fedify-dev/fedify/pull/802#discussion_r3400818521 Assisted-by: Codex:gpt-5.5 --- .../src/bench/scenarios/object-discovery.ts | 18 ++++- .../cli/src/bench/scenarios/object.test.ts | 67 +++++++++++++++++++ 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/bench/scenarios/object-discovery.ts b/packages/cli/src/bench/scenarios/object-discovery.ts index 801add7ac..7f25a89f9 100644 --- a/packages/cli/src/bench/scenarios/object-discovery.ts +++ b/packages/cli/src/bench/scenarios/object-discovery.ts @@ -11,6 +11,7 @@ import type { ObjectSource } from "../scenario/types.ts"; const ACTIVITY_JSON_ACCEPT = "application/activity+json, application/ld+json"; const WEBFINGER_ACCEPT = "application/jrd+json, application/json"; const MAX_COLLECTION_CRAWL_PAGES = 100; +const MAX_ACTIVITY_UNWRAP_DEPTH = 10; const ACTIVITY_WRAPPER_TYPES = new Set([ "Accept", "Add", @@ -207,14 +208,25 @@ function objectUrl( return null; } -function objectCandidates(item: unknown): unknown[] { +function objectCandidates( + item: unknown, + depth = 0, + seen: WeakSet = new WeakSet(), +): unknown[] { + if (depth > MAX_ACTIVITY_UNWRAP_DEPTH) return []; if (!isRecord(item) || !matchesType(item.type, ACTIVITY_WRAPPER_TYPES)) { return [item]; } + if (seen.has(item)) return []; + seen.add(item); const object = item.object; if (object == null) return []; - if (Array.isArray(object)) return object.filter((entry) => entry != null); - return [object]; + if (Array.isArray(object)) { + return object.flatMap((entry) => + entry == null ? [] : objectCandidates(entry, depth + 1, seen) + ); + } + return objectCandidates(object, depth + 1, seen); } function matchesType( diff --git a/packages/cli/src/bench/scenarios/object.test.ts b/packages/cli/src/bench/scenarios/object.test.ts index f122326d9..9edb1260b 100644 --- a/packages/cli/src/bench/scenarios/object.test.ts +++ b/packages/cli/src/bench/scenarios/object.test.ts @@ -196,6 +196,73 @@ test("objectRunner - unwraps activities while crawling object sources", async () assert.strictEqual(fetchedObjectUrl, "http://target.test/objects/1"); }); +test("objectRunner - recursively unwraps nested activities", async () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "object-crawl", + type: "object", + source: { + seed: "http://target.test/users/alice", + collection: "outbox", + limit: 1, + type: "Note", + }, + load: { concurrency: 1 }, + duration: "25ms", + }], + }).scenarios[0]; + let fetchedObjectUrl = ""; + + const measurement = await objectRunner.run({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const request = input instanceof Request ? input : new Request(input); + const url = new URL(request.url); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return Promise.resolve(new Response("not found", { status: 404 })); + } + if (url.pathname === "/users/alice") { + return Promise.resolve(json({ + id: url.href, + outbox: "http://target.test/users/alice/outbox", + })); + } + if (url.pathname === "/users/alice/outbox") { + return Promise.resolve(json({ + id: url.href, + orderedItems: [{ + type: "Announce", + id: "http://target.test/activities/announce-1", + object: { + type: "Create", + id: "http://target.test/activities/create-1", + object: { + type: "Note", + id: "http://target.test/objects/1", + }, + }, + }], + })); + } + if (url.pathname === "/objects/1") { + fetchedObjectUrl = url.href; + return Promise.resolve(json({ id: url.href, type: "Note" })); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }, + }); + + assert.ok(measurement.requests.total > 0); + assert.strictEqual(fetchedObjectUrl, "http://target.test/objects/1"); +}); + test("objectRunner - selects matching objects from activity arrays", async () => { const scenario = normalizeSuite({ version: 1, From d98e09b0dca4ab9ba0c913ba214b600315c2f446 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Fri, 12 Jun 2026 15:14:44 +0900 Subject: [PATCH 53/61] Respect baseline queue backlog Queue drain checks now keep baseline in-flight work in the remaining task count, so completions from older tasks cannot satisfy fanout or remote failure drain observation for newly enqueued work. https://github.com/fedify-dev/fedify/pull/802#discussion_r3400842351 Assisted-by: Codex:gpt-5.5 --- .../src/bench/metrics/stats-client.test.ts | 23 ++++++++++ .../cli/src/bench/metrics/stats-client.ts | 10 ++-- packages/cli/src/bench/scenarios/failure.ts | 3 +- .../cli/src/bench/scenarios/fanout.test.ts | 46 +++++++++++++++++++ packages/cli/src/bench/scenarios/fanout.ts | 3 +- 5 files changed, 80 insertions(+), 5 deletions(-) diff --git a/packages/cli/src/bench/metrics/stats-client.test.ts b/packages/cli/src/bench/metrics/stats-client.test.ts index 457874f56..f337cede5 100644 --- a/packages/cli/src/bench/metrics/stats-client.test.ts +++ b/packages/cli/src/bench/metrics/stats-client.test.ts @@ -6,6 +6,7 @@ import { fetchServerSnapshot, parseServerMetrics, parseServerSnapshot, + queueTaskRemaining, type ServerSnapshot, snapshotToMetrics, } from "./stats-client.ts"; @@ -219,6 +220,28 @@ test("diffSnapshots - an empty baseline keeps the full end histogram", () => { assert.strictEqual(diff.queueDepthMax, 4); }); +test("queueTaskRemaining - accounts for baseline backlog", () => { + const diff: ServerSnapshot = { + signature: null, + queueDepthMax: null, + queueTasks: { + enqueued: 1, + completed: 1, + failed: 0, + }, + }; + + assert.strictEqual(queueTaskRemaining(diff), 0); + assert.strictEqual(queueTaskRemaining(diff, 1), 1); + assert.strictEqual( + queueTaskRemaining({ + ...diff, + queueTasks: { enqueued: 1, completed: 2, failed: 0 }, + }, 1), + 0, + ); +}); + test("diffSnapshots - incompatible bucketing drops the signature histogram", () => { // Same length but different boundary values is not comparable; refuse to // subtract rather than misattribute counts. diff --git a/packages/cli/src/bench/metrics/stats-client.ts b/packages/cli/src/bench/metrics/stats-client.ts index 0a24fa352..0b5e9ed1a 100644 --- a/packages/cli/src/bench/metrics/stats-client.ts +++ b/packages/cli/src/bench/metrics/stats-client.ts @@ -232,15 +232,19 @@ export async function fetchServerMetrics( * Returns the remaining queue task backlog represented by a diffed snapshot. * @param snapshot The server snapshot to inspect, usually already diffed * against a baseline. - * @returns `Math.max(0, enqueued - completed - failed)`, or `null` when the - * snapshot has no queue task counters. + * @param baselineRemaining Queue tasks that were already outstanding when the + * diff baseline was taken. These must drain before diffed completions can be + * attributed to newly enqueued tasks. + * @returns `Math.max(0, baselineRemaining + enqueued - completed - failed)`, + * or `null` when the snapshot has no queue task counters. */ export function queueTaskRemaining( snapshot: ServerSnapshot | null, + baselineRemaining = 0, ): number | null { if (snapshot?.queueTasks == null) return null; const { enqueued, completed, failed } = snapshot.queueTasks; - return Math.max(0, enqueued - completed - failed); + return Math.max(0, baselineRemaining + enqueued - completed - failed); } function isFiniteNumber(value: unknown): value is number { diff --git a/packages/cli/src/bench/scenarios/failure.ts b/packages/cli/src/bench/scenarios/failure.ts index 3f5c584ae..88831a2f9 100644 --- a/packages/cli/src/bench/scenarios/failure.ts +++ b/packages/cli/src/bench/scenarios/failure.ts @@ -379,6 +379,7 @@ async function waitForRemoteFault(options: { readonly timeoutMs: number; }): Promise { if (options.baseline == null) return null; + const baselineRemaining = queueTaskRemaining(options.baseline) ?? 0; const deadline = Date.now() + options.timeoutMs; do { const snapshot = await fetchServerSnapshot(options.target, options.fetch); @@ -390,7 +391,7 @@ async function waitForRemoteFault(options: { return { timedOut: false }; } } else if (queueTasks != null) { - const remaining = queueTaskRemaining(diff); + const remaining = queueTaskRemaining(diff, baselineRemaining); if (remaining != null) { if (options.fault === "slow-inbox") { if (queueTasks.completed > 0 && remaining === 0) { diff --git a/packages/cli/src/bench/scenarios/fanout.test.ts b/packages/cli/src/bench/scenarios/fanout.test.ts index 96e1de628..b8f112af1 100644 --- a/packages/cli/src/bench/scenarios/fanout.test.ts +++ b/packages/cli/src/bench/scenarios/fanout.test.ts @@ -256,6 +256,52 @@ test("fanoutRunner - waits for observed queue work before drain", async () => { assert.ok(statsCalls > 2); }); +test("fanoutRunner - ignores baseline backlog completions while draining", async () => { + const target = new URL("http://target.test/"); + let triggerCalls = 0; + const scenario = normalizeSuite({ + version: 1, + target: target.href, + scenarios: [{ + name: "fanout", + type: "fanout", + sender: "alice", + followers: 5, + load: { concurrency: 1 }, + duration: "1ms", + queueDrainTimeout: "30ms", + }], + }).scenarios[0]; + + const measurement = await fanoutRunner.run({ + scenario, + target, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return Promise.resolve(json(statsSnapshot({ + enqueued: 6 + triggerCalls * 6, + completed: triggerCalls > 0 ? 6 : 0, + failed: 0, + }))); + } + if (url.pathname === "/.well-known/fedify/bench/trigger") { + triggerCalls++; + return Promise.resolve(json({ version: 1 }, 202)); + } + return Promise.resolve(new Response("unexpected", { status: 500 })); + }, + }); + + assert.ok(measurement.requests.total > 0); + assert.strictEqual(measurement.requests.successRate, 0); + assert.ok(triggerCalls > 0); +}); + test("fanoutRunner - tolerates transient drain stats failures", async () => { const target = new URL("http://target.test/"); let statsCalls = 0; diff --git a/packages/cli/src/bench/scenarios/fanout.ts b/packages/cli/src/bench/scenarios/fanout.ts index d0c89a361..df7b6a40d 100644 --- a/packages/cli/src/bench/scenarios/fanout.ts +++ b/packages/cli/src/bench/scenarios/fanout.ts @@ -322,13 +322,14 @@ async function waitForDrain(options: { readonly timeoutMs: number; }): Promise { if (options.baseline == null) return null; + const baselineRemaining = queueTaskRemaining(options.baseline) ?? 0; const deadline = Date.now() + options.timeoutMs; do { const snapshot = await fetchServerSnapshot(options.target, options.fetch); if (snapshot != null) { const diff = diffSnapshots(options.baseline, snapshot); const queueTasks = diff.queueTasks; - const remaining = queueTaskRemaining(diff); + const remaining = queueTaskRemaining(diff, baselineRemaining); if ( queueTasks != null && queueTasks.enqueued > 0 && From 4ee6ba053095a07be2a3414f38f1bf27091dfba9 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Fri, 12 Jun 2026 15:34:22 +0900 Subject: [PATCH 54/61] Gate benchmark sink recipients Fanout and remote failure scenarios now pass generated sink inboxes through the actorless destination gate before posting benchmark trigger payloads, so safe targets cannot be used to relay outbound load to unchecked public sinks. https://github.com/fedify-dev/fedify/pull/802#discussion_r3401166230 https://github.com/fedify-dev/fedify/pull/802#discussion_r3401166233 Assisted-by: Codex:gpt-5.5 --- .../cli/src/bench/scenarios/failure.test.ts | 49 +++++++++++++++++++ packages/cli/src/bench/scenarios/failure.ts | 10 +++- .../cli/src/bench/scenarios/fanout.test.ts | 47 ++++++++++++++++++ packages/cli/src/bench/scenarios/fanout.ts | 14 ++++++ 4 files changed, 119 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/bench/scenarios/failure.test.ts b/packages/cli/src/bench/scenarios/failure.test.ts index 5e5c2e5be..88f146e30 100644 --- a/packages/cli/src/bench/scenarios/failure.test.ts +++ b/packages/cli/src/bench/scenarios/failure.test.ts @@ -124,6 +124,55 @@ test("failureRunner - uses configured sink base for remote faults", async () => assert.strictEqual(recipientInbox, new URL("/inbox/0", sinkBase).href); }); +test("failureRunner - gates remote fault sinks before triggering", async () => { + const target = new URL("http://target.test/"); + const sinkBase = `http://127.0.0.1:${await reservePort()}/`; + const scenario = normalizeSuite({ + version: 1, + target: target.href, + scenarios: [{ + name: "failure", + type: "failure", + fault: "remote-404", + sender: "alice", + sinkBase, + load: { concurrency: 1 }, + duration: "25ms", + queueDrainTimeout: "1s", + }], + }).scenarios[0]; + let gateCalls = 0; + let triggerCalls = 0; + + await assert.rejects( + async () => + failureRunner.run({ + scenario, + target, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/.well-known/fedify/bench/trigger") { + triggerCalls++; + } + return Promise.resolve(new Response("unexpected", { status: 500 })); + }, + assertActorlessDestinationAllowed: (url) => { + gateCalls++; + throw new Error(`refused ${url.href}`); + }, + assertDestinationAllowed: () => {}, + }), + /refused http:\/\/127\.0\.0\.1:/, + ); + + assert.strictEqual(gateCalls, 1); + assert.strictEqual(triggerCalls, 0); +}); + test("failureRunner - shares sink base across remote fault mix", async () => { const target = new URL("http://target.test/"); const sinkBase = `http://127.0.0.1:${await reservePort()}/`; diff --git a/packages/cli/src/bench/scenarios/failure.ts b/packages/cli/src/bench/scenarios/failure.ts index 88831a2f9..eede98463 100644 --- a/packages/cli/src/bench/scenarios/failure.ts +++ b/packages/cli/src/bench/scenarios/failure.ts @@ -16,7 +16,11 @@ import { import type { SyntheticActor } from "../server/synthetic.ts"; import { createActivityIdMinter } from "../signing/activity-id.ts"; import { signInboxDelivery } from "../signing/signer.ts"; -import { resolveSinkBase, spawnSinkServer } from "./fanout.ts"; +import { + assertSinkRecipientsAllowed, + resolveSinkBase, + spawnSinkServer, +} from "./fanout.ts"; import { loadPlanOf, measuredWindowMs, @@ -109,6 +113,10 @@ export const failureRunner: ScenarioRunner = { const remoteTargets = await resolveRemoteFailureTargets(context, faults); const remoteActivityIds = createActivityIdMinter(context.target); try { + await assertSinkRecipientsAllowed( + [...remoteTargets.values()].map((target) => target.recipient), + context, + ); let index = 0; const sendOne = () => sendForFault( diff --git a/packages/cli/src/bench/scenarios/fanout.test.ts b/packages/cli/src/bench/scenarios/fanout.test.ts index b8f112af1..33ed2ac0b 100644 --- a/packages/cli/src/bench/scenarios/fanout.test.ts +++ b/packages/cli/src/bench/scenarios/fanout.test.ts @@ -413,6 +413,53 @@ test("fanoutRunner - uses configured sink base for recipients", async () => { ); }); +test("fanoutRunner - gates sink recipients before triggering", async () => { + const target = new URL("http://target.test/"); + const scenario = normalizeSuite({ + version: 1, + target: target.href, + scenarios: [{ + name: "fanout", + type: "fanout", + sender: "alice", + followers: 5, + sinkBase: `http://127.0.0.1:${await reservePort()}/`, + load: { concurrency: 1 }, + duration: "30ms", + queueDrainTimeout: "1s", + }], + }).scenarios[0]; + let gateCalls = 0; + let triggerCalls = 0; + + await assert.rejects( + async () => + fanoutRunner.run({ + scenario, + target, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/.well-known/fedify/bench/trigger") { + triggerCalls++; + } + return Promise.resolve(new Response("unexpected", { status: 500 })); + }, + assertActorlessDestinationAllowed: (url) => { + gateCalls++; + throw new Error(`refused ${url.href}`); + }, + }), + /refused http:\/\/127\.0\.0\.1:/, + ); + + assert.strictEqual(gateCalls, 1); + assert.strictEqual(triggerCalls, 0); +}); + test("spawnSinkServer - ignores invalid sink latency", async () => { const sink = await spawnSinkServer({ followers: 1, diff --git a/packages/cli/src/bench/scenarios/fanout.ts b/packages/cli/src/bench/scenarios/fanout.ts index df7b6a40d..07034a873 100644 --- a/packages/cli/src/bench/scenarios/fanout.ts +++ b/packages/cli/src/bench/scenarios/fanout.ts @@ -64,6 +64,7 @@ export const fanoutRunner: ScenarioRunner = { const drainHistogram = new LogLinearHistogram(); let delivered = 0; try { + await assertSinkRecipientsAllowed(sink.recipients, context); const sendOne = async (scheduledAtMs: number): Promise => { const baseline = await fetchServerSnapshot(context.target, fetchImpl); const started = Date.now(); @@ -179,6 +180,19 @@ function buildActivity(context: RunContext, id: URL): Record { }; } +export async function assertSinkRecipientsAllowed( + recipients: readonly Record[], + context: RunContext, +): Promise { + for (const recipient of recipients) { + if (typeof recipient.inbox !== "string") continue; + await context.assertActorlessDestinationAllowed?.( + new URL(recipient.inbox), + context.scenario, + ); + } +} + export async function spawnSinkServer(options: { readonly followers: number; readonly rawBehavior: unknown; From 4830f150caf84e3820912f010ff645f64227411a Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Fri, 12 Jun 2026 16:12:59 +0900 Subject: [PATCH 55/61] Tighten failure benchmark checks Inbound failure scenarios now require the expected 401 rejection instead of accepting any client error as an observed fault. The action-level remote failure test no longer reserves and releases a port before the runner starts; it exercises the advertised sink path so the sink server binds its own available port. https://github.com/fedify-dev/fedify/pull/802#discussion_r3401246024 https://github.com/fedify-dev/fedify/pull/802#discussion_r3401270131 Assisted-by: Codex:gpt-5.5 --- packages/cli/src/bench/action.test.ts | 23 ++---- .../cli/src/bench/scenarios/failure.test.ts | 71 +++++++++++++++++++ packages/cli/src/bench/scenarios/failure.ts | 13 +++- 3 files changed, 85 insertions(+), 22 deletions(-) diff --git a/packages/cli/src/bench/action.test.ts b/packages/cli/src/bench/action.test.ts index 52327a736..b58dd9733 100644 --- a/packages/cli/src/bench/action.test.ts +++ b/packages/cli/src/bench/action.test.ts @@ -29,19 +29,6 @@ async function writeSuite(content: string): Promise { return path; } -async function reservePort(): Promise { - const server = serve({ - port: 0, - hostname: "127.0.0.1", - silent: true, - fetch: () => new Response("reserved"), - }); - await server.ready(); - const port = Number(new URL(server.url!).port); - await server.close(true); - return port; -} - function resolvePublicHost(_hostname: string): Promise { return Promise.resolve(["93.184.216.34"]); } @@ -489,8 +476,7 @@ scenarios: assert.match(message, /advertise-host/); }); -test("runBench - remote failure with sinkBase needs no advertise host", async () => { - const sinkBase = `http://127.0.0.1:${await reservePort()}/`; +test("runBench - remote failure uses advertised sink reachability", async () => { const file = await writeSuite(`version: 1 target: http://10.10.0.5:8000 scenarios: @@ -498,7 +484,6 @@ scenarios: type: failure fault: remote-404 sender: alice - sinkBase: "${sinkBase}" load: { concurrency: 1 } duration: 25ms queueDrainTimeout: 1s @@ -506,7 +491,7 @@ scenarios: let code = -1; let message = ""; let triggerCalls = 0; - await runBench(command({ scenario: file }), { + await runBench(command({ scenario: file, advertiseHost: "127.0.0.1" }), { exit: (c) => { code = c; }, @@ -987,7 +972,7 @@ scenarios: - scenario: lookup weight: 1 expect: - signatureVerification.p95: "< 10ms" + queueDrain.p95: "< 10ms" `); let code = -1; let message = ""; @@ -1006,7 +991,7 @@ scenarios: }, }); assert.strictEqual(code, 2); - assert.match(message, /server-side expectations/); + assert.match(message, /queueDrain\.p95/); assert.strictEqual(fetched, false); }); diff --git a/packages/cli/src/bench/scenarios/failure.test.ts b/packages/cli/src/bench/scenarios/failure.test.ts index 88f146e30..644afa93f 100644 --- a/packages/cli/src/bench/scenarios/failure.test.ts +++ b/packages/cli/src/bench/scenarios/failure.test.ts @@ -566,6 +566,77 @@ test("failureRunner - treats inbound 5xx as target failures", async () => { } }); +test("failureRunner - treats unexpected inbound 4xx as target failures", async () => { + const server = serve({ + port: 0, + hostname: "127.0.0.1", + silent: true, + fetch(request: Request): Response { + const url = new URL(request.url); + if (url.pathname === "/users/alice") { + return json({ + "@context": "https://www.w3.org/ns/activitystreams", + type: "Person", + id: url.href, + inbox: new URL("/missing-inbox", url).href, + }); + } + return new Response("not found", { status: 404 }); + }, + }); + await server.ready(); + let fleet: Awaited> | undefined; + try { + const target = new URL(server.url!); + fleet = await spawnSyntheticServer( + await buildFleet([{ + count: 1, + signatureStandards: ["draft-cavage-http-signatures-12"], + }]), + ); + const scenario = normalizeSuite({ + version: 1, + target: target.href, + scenarios: [{ + name: "bad-signature", + type: "failure", + fault: "invalid-signature", + recipient: new URL("/users/alice", target).href, + load: { concurrency: 1 }, + duration: "50ms", + }], + }).scenarios[0]; + const measurement = await failureRunner.run({ + scenario, + target, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet, + fetch: (input) => { + const url = new URL(input instanceof Request ? input.url : input); + if (url.pathname === "/missing-inbox") { + return Promise.resolve( + new Response("not found", { status: 404 }), + ); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }, + assertDestinationAllowed: () => {}, + }); + + assert.ok(measurement.requests.total > 0); + assert.strictEqual(measurement.requests.successRate, 0); + assert.ok(measurement.errors.some((e) => e.status === 404)); + } finally { + try { + await fleet?.close(); + } finally { + await server.close(true); + } + } +}); + function json(body: unknown): Response { return new Response(JSON.stringify(body), { headers: { "content-type": "application/activity+json" }, diff --git a/packages/cli/src/bench/scenarios/failure.ts b/packages/cli/src/bench/scenarios/failure.ts index eede98463..bd5dbe182 100644 --- a/packages/cli/src/bench/scenarios/failure.ts +++ b/packages/cli/src/bench/scenarios/failure.ts @@ -460,6 +460,7 @@ async function sendInvalidSignature( }), context.fetch ?? fetch, ), + 401, ); } @@ -481,7 +482,10 @@ async function sendMissingActor( "missing-actor", deliveryTarget, ); - return expectedFailure(await sendRequest(request, context.fetch ?? fetch)); + return expectedFailure( + await sendRequest(request, context.fetch ?? fetch), + 401, + ); } async function signedFailureRequest( @@ -566,8 +570,11 @@ function missingActor(actor: SyntheticActor, target: URL): SyntheticActor { }; } -function expectedFailure(outcome: SendOutcome): SendOutcome { - if (outcome.status != null && outcome.status >= 400 && outcome.status < 500) { +function expectedFailure( + outcome: SendOutcome, + expectedStatus: number, +): SendOutcome { + if (outcome.status === expectedStatus) { return { ok: true, status: outcome.status }; } return { From c88845cbb5fdf006178bbca8e5f07782e0f982d7 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Fri, 12 Jun 2026 17:25:40 +0900 Subject: [PATCH 56/61] Handle typed object references Type-filtered object discovery now dereferences IRI candidates before deciding whether to skip them, preserving common collection shapes that contain object references instead of embedded objects. Benchmark sink behavior status values are also clamped to valid HTTP response codes so invalid configuration falls back to the default accepted response instead of crashing the sink server. https://github.com/fedify-dev/fedify/pull/802#discussion_r3401447341 https://github.com/fedify-dev/fedify/pull/802#discussion_r3401465452 Assisted-by: Codex:gpt-5.5 --- .../cli/src/bench/scenarios/fanout.test.ts | 16 ++++ packages/cli/src/bench/scenarios/fanout.ts | 3 +- .../src/bench/scenarios/object-discovery.ts | 56 ++++++++++--- .../cli/src/bench/scenarios/object.test.ts | 84 ++++++++++++------- 4 files changed, 115 insertions(+), 44 deletions(-) diff --git a/packages/cli/src/bench/scenarios/fanout.test.ts b/packages/cli/src/bench/scenarios/fanout.test.ts index 33ed2ac0b..3d829ab08 100644 --- a/packages/cli/src/bench/scenarios/fanout.test.ts +++ b/packages/cli/src/bench/scenarios/fanout.test.ts @@ -476,6 +476,22 @@ test("spawnSinkServer - ignores invalid sink latency", async () => { } }); +test("spawnSinkServer - ignores out-of-range sink status", async () => { + const sink = await spawnSinkServer({ + followers: 1, + rawBehavior: { status: 999 }, + }); + try { + const response = await fetch(String(sink.recipients[0].inbox), { + method: "POST", + body: "{}", + }); + assert.strictEqual(response.status, 202); + } finally { + await sink.close(); + } +}); + test("fanoutRunner - omits queue drain metrics without drain samples", async () => { const target = new URL("http://target.test/"); const scenario = normalizeSuite({ diff --git a/packages/cli/src/bench/scenarios/fanout.ts b/packages/cli/src/bench/scenarios/fanout.ts index 07034a873..2d478745d 100644 --- a/packages/cli/src/bench/scenarios/fanout.ts +++ b/packages/cli/src/bench/scenarios/fanout.ts @@ -318,7 +318,8 @@ function parseSinkBehavior( } return { latencyMs, - status: typeof status === "number" && Number.isInteger(status) + status: typeof status === "number" && Number.isInteger(status) && + status >= 100 && status <= 599 ? status : 202, }; diff --git a/packages/cli/src/bench/scenarios/object-discovery.ts b/packages/cli/src/bench/scenarios/object-discovery.ts index 7f25a89f9..c81619402 100644 --- a/packages/cli/src/bench/scenarios/object-discovery.ts +++ b/packages/cli/src/bench/scenarios/object-discovery.ts @@ -148,7 +148,12 @@ async function* crawlCollection( const items = arrayProperty(page, "orderedItems") ?? arrayProperty(page, "items") ?? []; for (const item of items) { - const url = objectUrl(item, options.types, next); + const url = await objectUrl(item, { + base: next, + fetch: options.fetch, + assertReadDestinationAllowed: options.assertReadDestinationAllowed, + types: options.types, + }); if (url == null) continue; yield url; remaining--; @@ -187,27 +192,56 @@ async function fetchJson( return json; } -function objectUrl( +async function objectUrl( item: unknown, - types: ReadonlySet, - base: URL, -): URL | null { + options: { + readonly base: URL; + readonly fetch?: typeof fetch; + readonly assertReadDestinationAllowed?: (url: URL) => void | Promise; + readonly types: ReadonlySet; + }, +): Promise { for (const candidate of objectCandidates(item)) { if (typeof candidate === "string") { - if (types.size < 1) { - const url = safeUrl(candidate, base); - if (url != null) return url; - } + const url = safeUrl(candidate, options.base); + if (url == null) continue; + if (options.types.size < 1) return url; + const typedUrl = await typedReferencedObjectUrl(url, options); + if (typedUrl != null) return typedUrl; continue; } if (!isRecord(candidate)) continue; - if (types.size > 0 && !matchesType(candidate.type, types)) continue; - const url = propertyUrl(candidate, "id", base); + if ( + options.types.size > 0 && + !matchesType(candidate.type, options.types) + ) { + continue; + } + const url = propertyUrl(candidate, "id", options.base); if (url != null) return url; } return null; } +async function typedReferencedObjectUrl( + url: URL, + options: { + readonly fetch?: typeof fetch; + readonly assertReadDestinationAllowed?: (url: URL) => void | Promise; + readonly types: ReadonlySet; + }, +): Promise { + await options.assertReadDestinationAllowed?.(url); + let object: Record; + try { + object = await fetchJson(url, options.fetch); + } catch { + return null; + } + if (!matchesType(object.type, options.types)) return null; + return propertyUrl(object, "id", url) ?? url; +} + function objectCandidates( item: unknown, depth = 0, diff --git a/packages/cli/src/bench/scenarios/object.test.ts b/packages/cli/src/bench/scenarios/object.test.ts index 9edb1260b..745428fac 100644 --- a/packages/cli/src/bench/scenarios/object.test.ts +++ b/packages/cli/src/bench/scenarios/object.test.ts @@ -508,7 +508,7 @@ test("objectRunner - sends ActivityPub Accept headers during object discovery", ]); }); -test("objectRunner - skips URL-only collection items for type filters", async () => { +test("objectRunner - dereferences URL-only items for type filters", async () => { const scenario = normalizeSuite({ version: 1, target: "http://target.test/", @@ -525,38 +525,58 @@ test("objectRunner - skips URL-only collection items for type filters", async () duration: "25ms", }], }).scenarios[0]; + const fetched: string[] = []; - await assert.rejects( - async () => - objectRunner.run({ - scenario, - target: new URL("http://target.test/"), - documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), - contextLoader: await getContextLoader({ allowPrivateAddress: true }), - allowPrivateAddress: true, - fleet: null, - fetch: (input) => { - const url = new URL(input instanceof Request ? input.url : input); - if (url.pathname === "/users/alice") { - return Promise.resolve(json({ - id: url.href, - outbox: "http://target.test/users/alice/outbox", - })); - } - if (url.pathname === "/users/alice/outbox") { - return Promise.resolve(json({ - id: url.href, - orderedItems: ["http://target.test/objects/article"], - })); - } - return Promise.resolve(json({ - id: url.href, - type: "Article", - })); - }, - }), - /did not resolve any URLs/, - ); + const measurement = await objectRunner.run({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const url = new URL(input instanceof Request ? input.url : input); + fetched.push(url.href); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return Promise.resolve(new Response("not found", { status: 404 })); + } + if (url.pathname === "/users/alice") { + return Promise.resolve(json({ + id: url.href, + outbox: "http://target.test/users/alice/outbox", + })); + } + if (url.pathname === "/users/alice/outbox") { + return Promise.resolve(json({ + id: url.href, + orderedItems: [ + "http://target.test/objects/article", + { + type: "Create", + object: "http://target.test/objects/note", + }, + ], + })); + } + if (url.pathname === "/objects/article") { + return Promise.resolve(json({ + id: url.href, + type: "Article", + })); + } + if (url.pathname === "/objects/note") { + return Promise.resolve(json({ + id: url.href, + type: "Note", + })); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }, + }); + + assert.ok(measurement.requests.total > 0); + assert.ok(fetched.includes("http://target.test/objects/article")); + assert.ok(fetched.includes("http://target.test/objects/note")); }); test("objectRunner - gates discovery URLs before fetching them", async () => { From 376057417ac3eeb32c1390603a07addcc5476162 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Fri, 12 Jun 2026 17:40:20 +0900 Subject: [PATCH 57/61] Tighten read benchmark preflight Read destination gating now stops scheduling additional gate work after the first failure, preventing background destination checks from continuing after preflight has already rejected the scenario. The benchmark stats parser uses index access for raw metric point values, and actor scenario expect validation now mirrors object scenarios by allowing signature metrics only for authenticated reads. https://github.com/fedify-dev/fedify/pull/802#discussion_r3401835839 https://github.com/fedify-dev/fedify/pull/802#discussion_r3401835844 https://github.com/fedify-dev/fedify/pull/802#discussion_r3401859485 Assisted-by: Codex:gpt-5.5 --- .../cli/src/bench/metrics/stats-client.ts | 4 +- packages/cli/src/bench/scenario/schema.ts | 3 +- .../cli/src/bench/scenario/validate.test.ts | 29 ++++++++ packages/cli/src/bench/scenarios/read.test.ts | 43 +++++++++++ packages/cli/src/bench/scenarios/read.ts | 10 ++- schema/bench/scenario-v2.json | 74 ++++++++++++++----- 6 files changed, 139 insertions(+), 24 deletions(-) diff --git a/packages/cli/src/bench/metrics/stats-client.ts b/packages/cli/src/bench/metrics/stats-client.ts index 0b5e9ed1a..e2590d028 100644 --- a/packages/cli/src/bench/metrics/stats-client.ts +++ b/packages/cli/src/bench/metrics/stats-client.ts @@ -313,8 +313,8 @@ function sumMetric( for (const metric of metrics) { if (metric.name !== name || !Array.isArray(metric.dataPoints)) continue; for (const point of metric.dataPoints) { - if (isRecord(point) && isFiniteNumber(point.value)) { - total += point.value; + if (isRecord(point) && isFiniteNumber(point["value"])) { + total += point["value"]; found = true; } } diff --git a/packages/cli/src/bench/scenario/schema.ts b/packages/cli/src/bench/scenario/schema.ts index 40253c62f..a6dc9387b 100644 --- a/packages/cli/src/bench/scenario/schema.ts +++ b/packages/cli/src/bench/scenario/schema.ts @@ -406,7 +406,8 @@ export const scenarioSchemaV2 = { properties: condition.then.properties, }, } - : condition.if.properties.type.const === "object" + : condition.if.properties.type.const === "actor" || + condition.if.properties.type.const === "object" ? { if: condition.if, then: { diff --git a/packages/cli/src/bench/scenario/validate.test.ts b/packages/cli/src/bench/scenario/validate.test.ts index bf39a3a5c..b53ebe704 100644 --- a/packages/cli/src/bench/scenario/validate.test.ts +++ b/packages/cli/src/bench/scenario/validate.test.ts @@ -100,6 +100,35 @@ test("validateSuite - enforces per-type expect metric allowlist", () => { assert.throws(() => validateSuite(bad), SuiteValidationError); }); +test("validateSuite - rejects unsigned actor signature expectations", () => { + const bad = { + version: 1, + target: "http://localhost:3000", + scenarios: [{ + name: "actor", + type: "actor", + recipient: "http://localhost:3000/users/alice", + expect: { "signatureVerification.p95": "< 10ms" }, + }], + }; + assert.throws(() => validateSuite(bad), SuiteValidationError); +}); + +test("validateSuite - accepts authenticated actor signature expectations", () => { + const suite = { + version: 1, + target: "http://localhost:3000", + scenarios: [{ + name: "actor", + type: "actor", + recipient: "http://localhost:3000/users/alice", + authenticated: true, + expect: { "signatureVerification.p95": "< 10ms" }, + }], + }; + assert.doesNotThrow(() => validateSuite(suite)); +}); + test("validateSuite - rejects mixed server-side expect metrics", () => { const bad = { version: 1, diff --git a/packages/cli/src/bench/scenarios/read.test.ts b/packages/cli/src/bench/scenarios/read.test.ts index 1535d4574..4f49d9f53 100644 --- a/packages/cli/src/bench/scenarios/read.test.ts +++ b/packages/cli/src/bench/scenarios/read.test.ts @@ -139,6 +139,49 @@ test("runReadLoad - limits resolved read URL gate concurrency", async () => { assert.strictEqual(maxActiveGates, 16); }); +test("runReadLoad - stops read URL gates after a gate failure", async () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "read", + type: "actor", + load: { concurrency: 1 }, + duration: "25ms", + }], + }).scenarios[0]; + const urls = Array.from( + { length: 64 }, + (_, i) => new URL(`http://remote.test/users/${i}`), + ); + let gateCalls = 0; + + await assert.rejects( + async () => + await runReadLoad({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: () => Promise.resolve(new Response("{}", { status: 200 })), + assertReadDestinationAllowed: async () => { + gateCalls++; + if (gateCalls === 1) throw new Error("gate failed"); + await new Promise((resolve) => setTimeout(resolve, 1)); + }, + }, { + urls, + authenticated: false, + }), + /gate failed/, + ); + await new Promise((resolve) => setTimeout(resolve, 30)); + + assert.ok(gateCalls < urls.length); +}); + test("runReadLoad - rejects invalid read URL schemes before load", async () => { const scenario = normalizeSuite({ version: 1, diff --git a/packages/cli/src/bench/scenarios/read.ts b/packages/cli/src/bench/scenarios/read.ts index fd5d98b5e..8f6d8f696 100644 --- a/packages/cli/src/bench/scenarios/read.ts +++ b/packages/cli/src/bench/scenarios/read.ts @@ -144,12 +144,18 @@ async function mapWithConcurrency( callback: (item: T) => Promise, ): Promise { let next = 0; + let failed = false; const workers = Array.from( { length: Math.min(concurrency, items.length) }, async () => { - while (next < items.length) { + while (next < items.length && !failed) { const item = items[next++]; - await callback(item); + try { + await callback(item); + } catch (error) { + failed = true; + throw error; + } } }, ); diff --git a/schema/bench/scenario-v2.json b/schema/bench/scenario-v2.json index 117429c9a..01a15b02f 100644 --- a/schema/bench/scenario-v2.json +++ b/schema/bench/scenario-v2.json @@ -493,27 +493,63 @@ "required": [ "recipient" ], - "properties": { - "expect": { - "propertyNames": { - "enum": [ - "successRate", - "throughputPerSec", - "errors.total", - "errors.4xx", - "errors.5xx", - "latency.p50", - "latency.p95", - "latency.p99", - "latency.mean", - "latency.max", - "signatureVerification.p50", - "signatureVerification.p95", - "signatureVerification.p99" - ] + "allOf": [ + { + "if": { + "required": [ + "authenticated" + ], + "properties": { + "authenticated": { + "const": true + } + } + }, + "then": { + "properties": { + "expect": { + "propertyNames": { + "enum": [ + "successRate", + "throughputPerSec", + "errors.total", + "errors.4xx", + "errors.5xx", + "latency.p50", + "latency.p95", + "latency.p99", + "latency.mean", + "latency.max", + "signatureVerification.p50", + "signatureVerification.p95", + "signatureVerification.p99" + ] + } + } + } + }, + "else": { + "properties": { + "expect": { + "propertyNames": { + "enum": [ + "successRate", + "throughputPerSec", + "errors.total", + "errors.4xx", + "errors.5xx", + "latency.p50", + "latency.p95", + "latency.p99", + "latency.mean", + "latency.max" + ] + } + } + } } } - } + ] } }, { From 73cf1c1cc1de2c5af84913a717e89028c811f1cf Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Fri, 12 Jun 2026 17:58:30 +0900 Subject: [PATCH 58/61] Settle benchmark discovery workers Read destination preflight now waits for already-started gate workers before returning the first failure, so later failures do not keep running outside the caller's error path. Object source discovery now applies the activity unwrapping path to fetched activity references before type filtering, including a visited URL guard for cyclic references. https://github.com/fedify-dev/fedify/pull/802#discussion_r3401920646 https://github.com/fedify-dev/fedify/pull/802#discussion_r3401939230 Assisted-by: Codex:gpt-5.5 --- .../src/bench/scenarios/object-discovery.ts | 22 ++- .../cli/src/bench/scenarios/object.test.ts | 128 ++++++++++++++++++ packages/cli/src/bench/scenarios/read.test.ts | 46 +++++++ packages/cli/src/bench/scenarios/read.ts | 12 +- 4 files changed, 202 insertions(+), 6 deletions(-) diff --git a/packages/cli/src/bench/scenarios/object-discovery.ts b/packages/cli/src/bench/scenarios/object-discovery.ts index c81619402..845053f11 100644 --- a/packages/cli/src/bench/scenarios/object-discovery.ts +++ b/packages/cli/src/bench/scenarios/object-discovery.ts @@ -230,7 +230,10 @@ async function typedReferencedObjectUrl( readonly assertReadDestinationAllowed?: (url: URL) => void | Promise; readonly types: ReadonlySet; }, + seen: Set = new Set(), ): Promise { + if (seen.has(url.href)) return null; + seen.add(url.href); await options.assertReadDestinationAllowed?.(url); let object: Record; try { @@ -238,8 +241,23 @@ async function typedReferencedObjectUrl( } catch { return null; } - if (!matchesType(object.type, options.types)) return null; - return propertyUrl(object, "id", url) ?? url; + for (const candidate of objectCandidates(object)) { + if (typeof candidate === "string") { + const candidateUrl = safeUrl(candidate, url); + if (candidateUrl == null) continue; + const typedUrl = await typedReferencedObjectUrl( + candidateUrl, + options, + seen, + ); + if (typedUrl != null) return typedUrl; + continue; + } + if (!isRecord(candidate)) continue; + if (!matchesType(candidate.type, options.types)) continue; + return propertyUrl(candidate, "id", url) ?? url; + } + return null; } function objectCandidates( diff --git a/packages/cli/src/bench/scenarios/object.test.ts b/packages/cli/src/bench/scenarios/object.test.ts index 745428fac..cb589ab42 100644 --- a/packages/cli/src/bench/scenarios/object.test.ts +++ b/packages/cli/src/bench/scenarios/object.test.ts @@ -327,6 +327,134 @@ test("objectRunner - selects matching objects from activity arrays", async () => assert.strictEqual(fetchedObjectUrl, "http://target.test/objects/note"); }); +test("objectRunner - unwraps fetched activity references before filtering", async () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "object-crawl", + type: "object", + source: { + seed: "http://target.test/users/alice", + collection: "outbox", + limit: 1, + type: "Note", + }, + load: { concurrency: 1 }, + duration: "25ms", + }], + }).scenarios[0]; + let fetchedObjectUrl = ""; + + const measurement = await objectRunner.run({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const request = input instanceof Request ? input : new Request(input); + const url = new URL(request.url); + if (url.pathname === "/.well-known/fedify/bench/stats") { + return Promise.resolve(new Response("not found", { status: 404 })); + } + if (url.pathname === "/users/alice") { + return Promise.resolve(json({ + id: url.href, + outbox: "http://target.test/users/alice/outbox", + })); + } + if (url.pathname === "/users/alice/outbox") { + return Promise.resolve(json({ + id: url.href, + orderedItems: ["http://target.test/activities/create-1"], + })); + } + if (url.pathname === "/activities/create-1") { + return Promise.resolve(json({ + id: url.href, + type: "Create", + object: { + type: "Note", + id: "http://target.test/objects/1", + }, + })); + } + if (url.pathname === "/objects/1") { + fetchedObjectUrl = url.href; + return Promise.resolve(json({ id: url.href, type: "Note" })); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }, + }); + + assert.ok(measurement.requests.total > 0); + assert.strictEqual(fetchedObjectUrl, "http://target.test/objects/1"); +}); + +test("objectRunner - skips cyclic fetched activity references", async () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "object-crawl", + type: "object", + source: { + seed: "http://target.test/users/alice", + collection: "outbox", + limit: 1, + type: "Note", + }, + load: { concurrency: 1 }, + duration: "25ms", + }], + }).scenarios[0]; + let activityFetches = 0; + + await assert.rejects( + async () => + await objectRunner.run({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ + allowPrivateAddress: true, + }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const request = input instanceof Request ? input : new Request(input); + const url = new URL(request.url); + if (url.pathname === "/users/alice") { + return Promise.resolve(json({ + id: url.href, + outbox: "http://target.test/users/alice/outbox", + })); + } + if (url.pathname === "/users/alice/outbox") { + return Promise.resolve(json({ + id: url.href, + orderedItems: ["http://target.test/activities/create-1"], + })); + } + if (url.pathname === "/activities/create-1") { + activityFetches++; + return Promise.resolve(json({ + id: url.href, + type: "Create", + object: "http://target.test/activities/create-1", + })); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }, + }), + /did not resolve any URLs/, + ); + + assert.strictEqual(activityFetches, 1); +}); + test("objectRunner - prefers unwrapped object URLs without type filters", async () => { const scenario = normalizeSuite({ version: 1, diff --git a/packages/cli/src/bench/scenarios/read.test.ts b/packages/cli/src/bench/scenarios/read.test.ts index 4f49d9f53..565ddd527 100644 --- a/packages/cli/src/bench/scenarios/read.test.ts +++ b/packages/cli/src/bench/scenarios/read.test.ts @@ -182,6 +182,52 @@ test("runReadLoad - stops read URL gates after a gate failure", async () => { assert.ok(gateCalls < urls.length); }); +test("runReadLoad - waits for in-flight read URL gates after a failure", async () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "read", + type: "actor", + load: { concurrency: 1 }, + duration: "25ms", + }], + }).scenarios[0]; + const urls = Array.from( + { length: 32 }, + (_, i) => new URL(`http://remote.test/users/${i}`), + ); + let gateCalls = 0; + let delayedGatesSettled = 0; + + await assert.rejects( + async () => + await runReadLoad({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: () => Promise.resolve(new Response("{}", { status: 200 })), + assertReadDestinationAllowed: async () => { + gateCalls++; + if (gateCalls === 1) throw new Error("first gate failed"); + await new Promise((resolve) => setTimeout(resolve, 20)); + delayedGatesSettled++; + throw new Error("later gate failed"); + }, + }, { + urls, + authenticated: false, + }), + /first gate failed/, + ); + + assert.ok(delayedGatesSettled > 0); + assert.ok(gateCalls < urls.length); +}); + test("runReadLoad - rejects invalid read URL schemes before load", async () => { const scenario = normalizeSuite({ version: 1, diff --git a/packages/cli/src/bench/scenarios/read.ts b/packages/cli/src/bench/scenarios/read.ts index 8f6d8f696..0c7f24b55 100644 --- a/packages/cli/src/bench/scenarios/read.ts +++ b/packages/cli/src/bench/scenarios/read.ts @@ -144,22 +144,26 @@ async function mapWithConcurrency( callback: (item: T) => Promise, ): Promise { let next = 0; - let failed = false; + let firstError: unknown; + let hasError = false; const workers = Array.from( { length: Math.min(concurrency, items.length) }, async () => { - while (next < items.length && !failed) { + while (next < items.length && !hasError) { const item = items[next++]; try { await callback(item); } catch (error) { - failed = true; - throw error; + if (!hasError) { + hasError = true; + firstError = error; + } } } }, ); await Promise.all(workers); + if (hasError) throw firstError; } async function signGetRequest( From 027462de84a4055ddf0279e5338ee69b3dcf05a9 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Fri, 12 Jun 2026 18:35:41 +0900 Subject: [PATCH 59/61] Limit fetched object reference depth Typed object discovery now caps recursive URL reference traversal with the same depth limit used for embedded activity unwrapping. This prevents deep reference chains from driving unbounded fetch recursion during object source crawls. https://github.com/fedify-dev/fedify/pull/802#discussion_r3402041013 Assisted-by: Codex:gpt-5.5 --- .../src/bench/scenarios/object-discovery.ts | 3 + .../cli/src/bench/scenarios/object.test.ts | 75 +++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/packages/cli/src/bench/scenarios/object-discovery.ts b/packages/cli/src/bench/scenarios/object-discovery.ts index 845053f11..3551c9424 100644 --- a/packages/cli/src/bench/scenarios/object-discovery.ts +++ b/packages/cli/src/bench/scenarios/object-discovery.ts @@ -231,7 +231,9 @@ async function typedReferencedObjectUrl( readonly types: ReadonlySet; }, seen: Set = new Set(), + depth = 0, ): Promise { + if (depth > MAX_ACTIVITY_UNWRAP_DEPTH) return null; if (seen.has(url.href)) return null; seen.add(url.href); await options.assertReadDestinationAllowed?.(url); @@ -249,6 +251,7 @@ async function typedReferencedObjectUrl( candidateUrl, options, seen, + depth + 1, ); if (typedUrl != null) return typedUrl; continue; diff --git a/packages/cli/src/bench/scenarios/object.test.ts b/packages/cli/src/bench/scenarios/object.test.ts index cb589ab42..c12a77649 100644 --- a/packages/cli/src/bench/scenarios/object.test.ts +++ b/packages/cli/src/bench/scenarios/object.test.ts @@ -455,6 +455,81 @@ test("objectRunner - skips cyclic fetched activity references", async () => { assert.strictEqual(activityFetches, 1); }); +test("objectRunner - limits deep fetched activity reference chains", async () => { + const scenario = normalizeSuite({ + version: 1, + target: "http://target.test/", + scenarios: [{ + name: "object-crawl", + type: "object", + source: { + seed: "http://target.test/users/alice", + collection: "outbox", + limit: 1, + type: "Note", + }, + load: { concurrency: 1 }, + duration: "25ms", + }], + }).scenarios[0]; + const fetchedActivities: number[] = []; + + await assert.rejects( + async () => + await objectRunner.run({ + scenario, + target: new URL("http://target.test/"), + documentLoader: await getDocumentLoader({ + allowPrivateAddress: true, + }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet: null, + fetch: (input) => { + const request = input instanceof Request ? input : new Request(input); + const url = new URL(request.url); + if (url.pathname === "/users/alice") { + return Promise.resolve(json({ + id: url.href, + outbox: "http://target.test/users/alice/outbox", + })); + } + if (url.pathname === "/users/alice/outbox") { + return Promise.resolve(json({ + id: url.href, + orderedItems: ["http://target.test/activities/0"], + })); + } + const match = /^\/activities\/(\d+)$/.exec(url.pathname); + if (match != null) { + const index = Number(match[1]); + fetchedActivities.push(index); + return Promise.resolve(json({ + id: url.href, + type: "Create", + object: index < 12 + ? `http://target.test/activities/${index + 1}` + : { + type: "Note", + id: "http://target.test/objects/deep", + }, + })); + } + if (url.pathname === "/objects/deep") { + return Promise.resolve(json({ id: url.href, type: "Note" })); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }, + }), + /did not resolve any URLs/, + ); + + assert.deepStrictEqual( + fetchedActivities, + Array.from({ length: 11 }, (_, i) => i), + ); +}); + test("objectRunner - prefers unwrapped object URLs without type filters", async () => { const scenario = normalizeSuite({ version: 1, From 1d3e9ce2d4795f175a20892c52428b0031a6d06e Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Sat, 13 Jun 2026 04:40:14 +0900 Subject: [PATCH 60/61] Preserve invalid signature grammar The invalid-signature failure mode now changes the signed Date component instead of appending bytes to Signature or Authorization headers. This keeps the header syntax valid while still forcing signature verification to fail. https://github.com/fedify-dev/fedify/pull/802#discussion_r3404928384 Assisted-by: Codex:gpt-5.5 --- packages/cli/src/bench/scenarios/failure.test.ts | 11 ++++++++--- packages/cli/src/bench/scenarios/failure.ts | 15 +++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/packages/cli/src/bench/scenarios/failure.test.ts b/packages/cli/src/bench/scenarios/failure.test.ts index 644afa93f..a41984c5f 100644 --- a/packages/cli/src/bench/scenarios/failure.test.ts +++ b/packages/cli/src/bench/scenarios/failure.test.ts @@ -451,7 +451,8 @@ test("failureRunner - discovers inbound failure inboxes once", async () => { return Promise.resolve(); }, }; - let corruptedSignatureRequests = 0; + let malformedSignatureRequests = 0; + let signedDateRequests = 0; const measurement = await failureRunner.run({ scenario, target, @@ -470,7 +471,10 @@ test("failureRunner - discovers inbound failure inboxes once", async () => { signature?.endsWith("0") === true || authorization?.endsWith("0") === true ) { - corruptedSignatureRequests++; + malformedSignatureRequests++; + } + if (!Number.isNaN(Date.parse(request.headers.get("date") ?? ""))) { + signedDateRequests++; } return new Response("bad signature", { status: 401, @@ -484,7 +488,8 @@ test("failureRunner - discovers inbound failure inboxes once", async () => { assert.strictEqual(measurement.requests.total, 3); assert.strictEqual(measurement.requests.successRate, 1); - assert.strictEqual(corruptedSignatureRequests, 3); + assert.strictEqual(malformedSignatureRequests, 0); + assert.strictEqual(signedDateRequests, 3); assert.strictEqual(actorGets, 1); } finally { try { diff --git a/packages/cli/src/bench/scenarios/failure.ts b/packages/cli/src/bench/scenarios/failure.ts index bd5dbe182..323d96e5e 100644 --- a/packages/cli/src/bench/scenarios/failure.ts +++ b/packages/cli/src/bench/scenarios/failure.ts @@ -449,7 +449,7 @@ async function sendInvalidSignature( ); const body = await request.arrayBuffer(); const headers = new Headers(request.headers); - corruptSignatureHeaders(headers); + invalidateSignedDate(headers); return expectedFailure( await sendRequest( new Request(request.url, { @@ -464,13 +464,12 @@ async function sendInvalidSignature( ); } -function corruptSignatureHeaders(headers: Headers): void { - const signature = headers.get("signature"); - if (signature != null) headers.set("signature", `${signature}0`); - const authorization = headers.get("authorization"); - if (authorization != null) { - headers.set("authorization", `${authorization}0`); - } +function invalidateSignedDate(headers: Headers): void { + const timestamp = Date.parse(headers.get("date") ?? ""); + const shifted = Number.isNaN(timestamp) + ? new Date() + : new Date(timestamp + 1000); + headers.set("date", shifted.toUTCString()); } async function sendMissingActor( From 12bcee91b15f9044f10287ad2c7629274ed1e87c Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Sat, 13 Jun 2026 04:42:35 +0900 Subject: [PATCH 61/61] Honor fanout follower schema minimum Fanout validation now relies on the scenario schema's followers minimum instead of imposing an extra five-follower floor. This keeps runner preflight consistent with accepted suite configuration. https://github.com/fedify-dev/fedify/pull/802#discussion_r3405097648 Assisted-by: Codex:gpt-5.5 --- packages/cli/src/bench/scenarios/fanout.test.ts | 6 +++--- packages/cli/src/bench/scenarios/fanout.ts | 6 ------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/packages/cli/src/bench/scenarios/fanout.test.ts b/packages/cli/src/bench/scenarios/fanout.test.ts index 3d829ab08..a81ed7ccb 100644 --- a/packages/cli/src/bench/scenarios/fanout.test.ts +++ b/packages/cli/src/bench/scenarios/fanout.test.ts @@ -65,7 +65,7 @@ test("fanoutRunner - triggers benchmark hook and reports drain", async () => { assert.strictEqual(triggerRecipients, 5); }); -test("fanoutRunner.validate - requires enough followers for fanout queue", () => { +test("fanoutRunner.validate - accepts schema-minimum follower count", () => { const scenario = normalizeSuite({ version: 1, target: "http://target.test/", @@ -73,10 +73,10 @@ test("fanoutRunner.validate - requires enough followers for fanout queue", () => name: "fanout", type: "fanout", sender: "alice", - followers: 4, + followers: 1, }], }).scenarios[0]; - assert.throws(() => fanoutRunner.validate?.(scenario), /at least 5/); + assert.doesNotThrow(() => fanoutRunner.validate?.(scenario)); }); test("fanoutRunner.validate - rejects invalid sinkBase URLs", () => { diff --git a/packages/cli/src/bench/scenarios/fanout.ts b/packages/cli/src/bench/scenarios/fanout.ts index 2d478745d..045fbff96 100644 --- a/packages/cli/src/bench/scenarios/fanout.ts +++ b/packages/cli/src/bench/scenarios/fanout.ts @@ -38,12 +38,6 @@ export const fanoutRunner: ScenarioRunner = { `trigger.kind: "benchmark-hook".`, ); } - if ((scenario.followers ?? DEFAULT_FOLLOWERS) < 5) { - throw new Error( - `Scenario "${scenario.name}": fanout needs at least 5 followers to ` + - "exercise Fedify's fanout queue.", - ); - } resolveSinkBase(scenario.name, scenario.raw.sinkBase); },