lib: add bench:plan event to node:bench

Signed-off-by: James M Snell <jasnell@gmail.com>
Assisted-by: Opencode
PR-URL: #65631
Reviewed-By: Filip Skokan <panva.ip@gmail.com>

Author: jasnell

doc/api/bench.md

@@ -594,6 +594,7 @@ both emitted as a named event and made available on the stream as
 
 The events are emitted in execution order:
 
+* `'bench:plan'`
 * `'bench:start'`
 * `'bench:sample'`
 * `'bench:complete'`
@@ -631,6 +632,33 @@ loading caused the declaration, while `file` identifies the source location of
 the declaration itself. `parentId` is based on the containing suite's source
 file and hierarchical name path.
 
+After asynchronous suite declarations settle, an in-process runner emits one
+`'bench:plan'` event for every benchmark it collected, in declaration order.
+All plans from that runner are emitted before its suite hooks or benchmark
+callbacks run. With process isolation, files run in separate children, so plans
+for a later file are emitted after an earlier child has completed. With no
+isolation, all files share one runner and their plans are emitted before any
+benchmark executes. Plan data contains the benchmark-scoped identity, location,
+tags, and parameters described in [benchmark result][], together with:
+
+* `samples` {number} The effective maximum number of measured callback
+  invocations after run-level overrides.
+* `warmup` {number} The effective number of unreported warmup callback
+  invocations after run-level overrides.
+* `timeout` {number|null} The timeout in milliseconds, or `null` when no timeout
+  is configured.
+* `yieldBetweenSamples` {boolean} Whether an event loop turn is scheduled between
+  sample callbacks.
+* `selected` {boolean} Whether the benchmark is eligible to run after applying
+  `skip`, `only`, and `namePattern` selection. Execution can still be prevented
+  by a duplicate declaration, suite build, hook, abort, or other runtime failure.
+* `skip` {boolean|string} When `selected` is `false`, the explicit skip value or
+  the selection reason, such as `'only'` or `'name pattern'`.
+
+The plan contains execution settings known to the runner. Runtime version,
+operating system, processor, and other environment metadata are intentionally
+left for reporters and higher-level tools to collect.
+
 `'bench:complete'` data contains a [benchmark result][]. A failed result has an
 additional `error` property and may contain samples recorded before the error.
 A skipped result has an additional `skip` property and an empty `samples`

lib/internal/bench_runner/benchmarks_stream.js

@@ -238,6 +238,10 @@ class BenchmarksStream extends Readable {
     return waiter.promise;
   }
 
+  plan(data) {
+    return this[kEmitMessage]('bench:plan', data);
+  }
+
   start(data) {
     return this[kEmitMessage]('bench:start', data);
   }

lib/internal/bench_runner/cli.js

@@ -10,8 +10,12 @@ const {
   ArrayPrototypePushApply,
   ArrayPrototypeSome,
   ArrayPrototypeSort,
+  NumberIsFinite,
   NumberIsSafeInteger,
   ObjectGetOwnPropertyDescriptor,
+  ObjectGetPrototypeOf,
+  ObjectPrototype,
+  ObjectValues,
   Promise,
   PromisePrototypeThen,
   PromiseReject,
@@ -46,6 +50,7 @@ const {
   },
 } = require('internal/errors');
 const { getOptionValue, getOptionsAsFlagsFromBinding } = require('internal/options');
+const { TIMEOUT_MAX } = require('internal/timers');
 const { kEmptyObject } = require('internal/util');
 const { validateUint32 } = require('internal/validators');
 const { pathToFileURL } = require('internal/url');
@@ -68,6 +73,7 @@ const kBuiltinReporters = new SafeMap([
 const kChildAckMessageType = 'node:bench:ack';
 const kChildMessageType = 'node:bench:record';
 const kEventTypes = new SafeSet([
+  'bench:plan',
   'bench:start',
   'bench:sample',
   'bench:complete',
@@ -256,6 +262,8 @@ async function finishReporters(state) {
 
 function emitRecord(stream, record) {
   switch (record.type) {
+    case 'bench:plan':
+      return stream.plan(record.data);
     case 'bench:start':
       return stream.start(record.data);
     case 'bench:sample':
@@ -311,6 +319,14 @@ function deserializeRecord(record) {
   };
 }
 
+function isStringArray(value) {
+  if (!ArrayIsArray(value)) return false;
+  for (let i = 0; i < value.length; i++) {
+    if (typeof value[i] !== 'string') return false;
+  }
+  return true;
+}
+
 function validateRecord(record) {
   if (record === null || typeof record !== 'object' ||
       !kEventTypes.has(record.type) || record.data === null ||
@@ -323,18 +339,52 @@ function validateRecord(record) {
     throw new ERR_INVALID_ARG_VALUE(
       'benchmark child message', record, 'is not a valid benchmark record');
   }
-  if ((record.type === 'bench:start' || record.type === 'bench:sample' ||
-       record.type === 'bench:complete') &&
-      (typeof record.data.benchId !== 'string' ||
+  if ((record.type === 'bench:plan' || record.type === 'bench:start' ||
+       record.type === 'bench:sample' || record.type === 'bench:complete') &&
+      (typeof record.data.fileRunId !== 'string' ||
+       typeof record.data.benchId !== 'string' ||
        (record.data.parentId !== null &&
-        typeof record.data.parentId !== 'string') ||
-       typeof record.data.name !== 'string' ||
-       !ArrayIsArray(record.data.namePath) ||
-       ArrayPrototypeSome(
-         record.data.namePath, (name) => typeof name !== 'string'))) {
+         typeof record.data.parentId !== 'string') ||
+        typeof record.data.name !== 'string' ||
+        !isStringArray(record.data.namePath))) {
     throw new ERR_INVALID_ARG_VALUE(
       'benchmark child message', record, 'is not a valid benchmark record');
   }
+  if (record.type === 'bench:plan') {
+    const {
+      samples,
+      selected,
+      skip,
+      timeout,
+      warmup,
+      yieldBetweenSamples,
+    } = record.data;
+    if (typeof record.data.file !== 'string' ||
+        !NumberIsSafeInteger(record.data.line) || record.data.line < 0 ||
+        !NumberIsSafeInteger(record.data.column) || record.data.column < 0 ||
+        !isStringArray(record.data.tags) ||
+        record.data.params === null ||
+        typeof record.data.params !== 'object' ||
+        ArrayIsArray(record.data.params) ||
+        (ObjectGetPrototypeOf(record.data.params) !== null &&
+         ObjectGetPrototypeOf(record.data.params) !== ObjectPrototype) ||
+        ArrayPrototypeSome(ObjectValues(record.data.params), (value) =>
+          typeof value !== 'string' && typeof value !== 'boolean' &&
+          (typeof value !== 'number' || !NumberIsFinite(value))) ||
+        !NumberIsSafeInteger(samples) || samples <= 0 ||
+        samples > 0xFFFFFFFF ||
+        !NumberIsSafeInteger(warmup) || warmup < 0 || warmup > 0xFFFFFFFF ||
+        (timeout !== null &&
+         (!NumberIsFinite(timeout) || timeout < 0 || timeout > TIMEOUT_MAX)) ||
+        typeof yieldBetweenSamples !== 'boolean' ||
+        typeof selected !== 'boolean' ||
+        (selected && skip !== undefined) ||
+        (!selected && skip !== true && typeof skip !== 'string')) {
+      throw new ERR_INVALID_ARG_VALUE(
+        'benchmark child plan', record.data,
+        'is not a valid benchmark plan');
+    }
+  }
   if (record.type === 'bench:summary') {
     const { counts, duration_ns, success } = record.data;
     if (typeof success !== 'boolean' || typeof duration_ns !== 'bigint' ||
@@ -552,7 +602,9 @@ async function runChild(path, options, scope, onRecord) {
     stdio: ['inherit', 'pipe', 'pipe', 'ipc'],
   });
   let protocolError;
+  let plansComplete = false;
   let recordPending = false;
+  let summaryReceived = false;
   const pendingRecords = new SafeSet();
   const handleRecord = (record) => {
     if (protocolError !== undefined) return;
@@ -571,7 +623,7 @@ async function runChild(path, options, scope, onRecord) {
       source?.resume();
     }, (error) => {
       pendingRecords.delete(tracked);
-      protocolError = error;
+      protocolError ??= error;
       child.kill();
     });
     pendingRecords.add(tracked);
@@ -610,10 +662,18 @@ async function runChild(path, options, scope, onRecord) {
       }
       recordPending = true;
       const record = deserializeRecord(validateRecord(message.record));
-      record.data.runId = options.runId;
-      if (record.data.fileRunId !== null) {
-        record.data.fileRunId = scope.fileRunId;
+      if (summaryReceived || (record.type === 'bench:plan' && plansComplete)) {
+        throw new ERR_INVALID_ARG_VALUE(
+          'benchmark child message', message,
+          'does not have a valid lifecycle sequence');
       }
+      if (record.type !== 'bench:plan' &&
+          record.type !== 'bench:diagnostic') {
+        plansComplete = true;
+      }
+      if (record.type === 'bench:summary') summaryReceived = true;
+      record.data.runId = options.runId;
+      record.data.fileRunId = scope.fileRunId;
       if (record.data.entryFile !== null) {
         record.data.entryFile = scope.entryFile;
       }
@@ -626,7 +686,7 @@ async function runChild(path, options, scope, onRecord) {
         }));
       }
     } catch (error) {
-      protocolError = error;
+      protocolError ??= error;
       child.kill();
     }
   });

lib/internal/bench_runner/harness.js

@@ -254,7 +254,7 @@ class Harness {
       if (typeof namePattern === 'string') {
         nextNamePattern = new RegExp(namePattern);
       } else if (isRegExp(namePattern)) {
-        nextNamePattern = namePattern;
+        nextNamePattern = new RegExp(namePattern);
       } else {
         throw new ERR_INVALID_ARG_TYPE(
           'options.namePattern', ['string', 'RegExp'], namePattern);
@@ -391,6 +391,37 @@ class Harness {
     });
   }
 
+  async #emitPlans() {
+    const benchmarks = [];
+    this.#walk(this.root, (node) => {
+      if (node instanceof Bench) ArrayPrototypePush(benchmarks, node);
+    });
+    for (let i = 0; i < benchmarks.length; i++) {
+      const benchmark = benchmarks[i];
+      const skip = this.#getSkip(benchmark);
+      const data = {
+        __proto__: null,
+        ...this.#getRecordScope(benchmark),
+        benchId: benchmark.benchId,
+        parentId: benchmark.parentId,
+        name: benchmark.name,
+        namePath: ArrayPrototypeSlice(benchmark.namePath),
+        file: benchmark.loc.file,
+        line: benchmark.loc.line,
+        column: benchmark.loc.column,
+        tags: ArrayPrototypeSlice(benchmark.tags),
+        params: benchmark.params,
+        samples: this.samples ?? benchmark.samples,
+        warmup: this.warmup ?? benchmark.warmup,
+        timeout: benchmark.timeout === Infinity ? null : benchmark.timeout,
+        yieldBetweenSamples: this.#yieldBetweenSamples,
+        selected: skip === null,
+      };
+      if (skip !== null) data.skip = skip;
+      await this.#waitForStream(this.stream.plan(data));
+    }
+  }
+
   #hasSelectedAncestor(benchmark) {
     for (let current = benchmark; current !== null; current = current.parent) {
       if (current.only) return true;
@@ -871,6 +902,7 @@ class Harness {
     this.#fileScopeStorage.disable();
     this.#prepare();
     this.state = 'running';
+    await this.#emitPlans();
     await this.#executeSuite(this.root);
     await this.#finish(startTime);
   }

test/fixtures/bench-runner/destroying-reporter.cjs

@@ -1,7 +1,7 @@
 'use strict';
 
 module.exports = async function* destroyingReporter(source) {
-  source.once('bench:start', () => {
+  source.once('bench:plan', () => {
     source.destroy(new Error('benchmark reporter closed the stream'));
   });
   yield* source;

test/fixtures/bench-runner/load-error-after-declaration.cjs

@@ -0,0 +1,11 @@
+'use strict';
+
+const { bench } = require('node:bench');
+
+bench('declared before load error', { samples: 1 }, (b) => {
+  b.start();
+  process.hrtime.bigint();
+  b.end(1);
+});
+
+throw new Error('load failed after declaration');

test/fixtures/bench-runner/malformed-plan-order.mjs

@@ -0,0 +1,34 @@
+import { bench } from 'node:bench';
+import common from '../../common/index.js';
+
+await new Promise((resolve, reject) => {
+  const timeout = setTimeout(() => {
+    reject(new Error('benchmark summary was not acknowledged'));
+  }, common.platformTimeout(10_000));
+  process.on('message', (message) => {
+    if (message?.type === 'node:bench:ack' && message.id === 0) {
+      clearTimeout(timeout);
+      resolve();
+    }
+  });
+  process.send?.({
+    id: 0,
+    type: 'node:bench:record',
+    record: {
+      type: 'bench:summary',
+      data: {
+        counts: { completed: 0, failed: 0, skipped: 0, total: 0 },
+        duration_ns: 1n,
+        entryFile: import.meta.filename,
+        fileRunId: process.env.NODE_BENCH_FILE_RUN_ID,
+        file: import.meta.filename,
+        runId: process.env.NODE_BENCH_RUN_ID,
+        success: true,
+      },
+    },
+  });
+});
+
+bench('late plan', { samples: 1 }, () => {
+  throw new Error('late plan benchmark ran');
+});

test/fixtures/bench-runner/malformed-record.cjs

@@ -4,6 +4,33 @@ const common = require('../../common');
 
 const kind = process.env.NODE_BENCH_MALFORMED_RECORD;
 const id = kind === 'sequence' ? null : 0;
+const benchmarkData = {
+  __proto__: null,
+  benchId: 'invalid plan',
+  column: 1,
+  entryFile: __filename,
+  file: __filename,
+  fileRunId: process.env.NODE_BENCH_FILE_RUN_ID,
+  line: 1,
+  name: 'invalid plan',
+  namePath: ['invalid plan'],
+  params: {},
+  parentId: null,
+  runId: process.env.NODE_BENCH_RUN_ID,
+  tags: [],
+};
+const plan = {
+  type: 'bench:plan',
+  data: {
+    __proto__: null,
+    ...benchmarkData,
+    samples: 1,
+    selected: true,
+    timeout: null,
+    warmup: 0,
+    yieldBetweenSamples: true,
+  },
+};
 const record = kind === 'summary' ? {
   type: 'bench:summary',
   data: {
@@ -14,6 +41,12 @@ const record = kind === 'summary' ? {
     runId: process.env.NODE_BENCH_RUN_ID,
     success: true,
   },
+} : kind === 'plan' ? {
+  ...plan,
+  data: {
+    ...plan.data,
+    samples: 0,
+  },
 } : kind === 'identity' ? {
   type: 'bench:complete',
   data: {

test/fixtures/bench-runner/slow-reporter.cjs

@@ -9,6 +9,7 @@ module.exports = async function* slowReporter(source) {
     if (++emitted === source.readableHighWaterMark) resolve();
   };
   for (const type of [
+    'bench:plan',
     'bench:start',
     'bench:sample',
     'bench:complete',