From 183b7597d86f4b2c28152f341f33774ada1a3a9c Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 4 Sep 2026 06:25:02 +0000 Subject: [PATCH 1/2] docs(evaluations): show a typed outcome grader (EvalGraderFunction / EvalGraderContext / EvalScriptOutcome) Adds "Writing an outcome grader" (en + zh) after the suite sample: the skills-starter release-result grader verbatim, the three EvalGraderContext fields, the EvalScriptOutcome shape, and what a grader defect does to the trial. Docs only. Fixes #489 --- .../docs/en/guide/development/evaluations.mdx | 47 +++++++++++++++++++ .../docs/zh/guide/development/evaluations.mdx | 44 +++++++++++++++++ 2 files changed, 91 insertions(+) diff --git a/website/docs/en/guide/development/evaluations.mdx b/website/docs/en/guide/development/evaluations.mdx index a2f9d1386..18c3cd2d7 100644 --- a/website/docs/en/guide/development/evaluations.mdx +++ b/website/docs/en/guide/development/evaluations.mdx @@ -45,6 +45,53 @@ npx agent-bundle eval --root . --suite mcp-app-status --trials 3 npx agent-bundle eval compare ``` +### Writing an outcome grader + +`expectOutcome({ script })` names a module, resolved relative to the suite file and required to +stay inside the suite directory, that default-exports one grader function. `agent-bundle/eval` +exports its contract: [`EvalGraderFunction`](../../api/types/eval.EvalGraderFunction.md) takes an +[`EvalGraderContext`](../../api/interfaces/eval.EvalGraderContext.md) and returns an +[`EvalScriptOutcome`](../../api/interfaces/eval.EvalScriptOutcome.md), synchronously or as a +promise. This is `examples/skills-starter/evals/graders/release-result.ts` verbatim; the suite +above names `./graders/status-result.ts`, which has the same shape in `examples/mcp-app`: + +```ts twoslash +// evals/graders/release-result.ts +import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import type { EvalGraderFunction } from 'agent-bundle/eval'; + +interface ReleaseResult { + readonly blockers?: unknown; + readonly verdict?: string; +} + +const grade: EvalGraderFunction = async ({ fixturePath }) => { + const result = JSON.parse(await readFile(join(fixturePath, 'result.json'), 'utf8')) as ReleaseResult; + return result.verdict === 'ready' && Array.isArray(result.blockers) && result.blockers.length === 0 + ? { detail: 'The release artifact is ready with no blockers.', outcome: 'pass' } + : { detail: 'The release artifact is not ready or has unresolved blockers.', outcome: 'fail' }; +}; + +export default grade; +``` + +The context is the trial the grader is judging: + +| Field | Meaning | +| --- | --- | +| `fixturePath` | The trial's own copy of the case `fixture`, after the agent ran against it. Read the agent's output from here; the checked-in fixture is never touched. | +| `artifactRoot` | The prepared artifact the trial ran against, for a grader that checks what shipped rather than what the agent wrote. | +| `prompt` | The case prompt the agent received. | + +The outcome is `{ outcome, detail }`: `outcome` is `pass`, `fail`, or `inconclusive` — the same +three verdicts as every other assertion — and `detail` is the sentence shown beside the result in +the trial record. A module that does not default-export a function, or returns anything but that +shape, is a grader defect, not plugin evidence: the trial is reported `EVAL_GRADER_FAILED` and its +result stays inconclusive. Graders run in the eval process with the host's file system, so read the +fixture and the artifact; do not reach into the agent's session. + ## Three outcomes, not two Every assertion resolves to `pass`, `fail`, or **`inconclusive`**, and declares the minimum diff --git a/website/docs/zh/guide/development/evaluations.mdx b/website/docs/zh/guide/development/evaluations.mdx index d8e228f15..7a7f1f36c 100644 --- a/website/docs/zh/guide/development/evaluations.mdx +++ b/website/docs/zh/guide/development/evaluations.mdx @@ -42,6 +42,50 @@ npx agent-bundle eval --root . --suite mcp-app-status --trials 3 npx agent-bundle eval compare ``` +### 编写一个结果 grader + +`expectOutcome({ script })` 指名一个模块——相对套件文件解析，并且必须留在套件目录之内——它默认导出一个 +grader 函数。`agent-bundle/eval` 导出了它的契约：[`EvalGraderFunction`](../../api/types/eval.EvalGraderFunction.md) +接收一个 [`EvalGraderContext`](../../api/interfaces/eval.EvalGraderContext.md)，并同步或以 promise 形式返回一个 +[`EvalScriptOutcome`](../../api/interfaces/eval.EvalScriptOutcome.md)。下面是 +`examples/skills-starter/evals/graders/release-result.ts` 的原文；上面的套件指名的 `./graders/status-result.ts` +在 `examples/mcp-app` 中具有同样的形状： + +```ts twoslash +// evals/graders/release-result.ts +import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import type { EvalGraderFunction } from 'agent-bundle/eval'; + +interface ReleaseResult { + readonly blockers?: unknown; + readonly verdict?: string; +} + +const grade: EvalGraderFunction = async ({ fixturePath }) => { + const result = JSON.parse(await readFile(join(fixturePath, 'result.json'), 'utf8')) as ReleaseResult; + return result.verdict === 'ready' && Array.isArray(result.blockers) && result.blockers.length === 0 + ? { detail: 'The release artifact is ready with no blockers.', outcome: 'pass' } + : { detail: 'The release artifact is not ready or has unresolved blockers.', outcome: 'fail' }; +}; + +export default grade; +``` + +上下文就是 grader 正在评判的那次试验： + +| 字段 | 含义 | +| --- | --- | +| `fixturePath` | 该次试验自己的 case `fixture` 副本，agent 已经在其上运行过。从这里读取 agent 的输出；签入的 fixture 永远不会被改动。 | +| `artifactRoot` | 该次试验所针对的已准备产物，供检查"实际发布了什么"而非"agent 写了什么"的 grader 使用。 | +| `prompt` | agent 收到的 case 提示词。 | + +结果是 `{ outcome, detail }`：`outcome` 是 `pass`、`fail` 或 `inconclusive`——与其他所有断言相同的三种裁定—— +`detail` 是在试验记录中显示于结果旁边的那句话。一个不默认导出函数、或返回了任何其他形状的模块，是 grader +的缺陷而不是插件的证据：该次试验会被报告为 `EVAL_GRADER_FAILED`，其结果保持 inconclusive。grader 在 eval +进程中以宿主的文件系统运行，因此请读取 fixture 与产物；不要触及 agent 的会话。 + ## 三种结果，而不是两种 每个断言都解析为 `pass`、`fail` 或 **`inconclusive`**，并声明它所接受的最低证据。当断言所需的证据强于 From eb71b219a44c226c635cde0193800ab733538b1f Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 4 Sep 2026 06:33:27 +0000 Subject: [PATCH 2/2] docs(evaluations): describe grader path containment as lexical, not physical --- website/docs/en/guide/development/evaluations.mdx | 6 ++++-- website/docs/zh/guide/development/evaluations.mdx | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/website/docs/en/guide/development/evaluations.mdx b/website/docs/en/guide/development/evaluations.mdx index 18c3cd2d7..84c960e61 100644 --- a/website/docs/en/guide/development/evaluations.mdx +++ b/website/docs/en/guide/development/evaluations.mdx @@ -47,8 +47,10 @@ npx agent-bundle eval compare ### Writing an outcome grader -`expectOutcome({ script })` names a module, resolved relative to the suite file and required to -stay inside the suite directory, that default-exports one grader function. `agent-bundle/eval` +`expectOutcome({ script })` names a module that default-exports one grader function. The path is +resolved relative to the suite file and must not climb out of the suite directory (`../` past it +is rejected; the check is on the path, so a symlink inside the suite is followed wherever it +points). `agent-bundle/eval` exports its contract: [`EvalGraderFunction`](../../api/types/eval.EvalGraderFunction.md) takes an [`EvalGraderContext`](../../api/interfaces/eval.EvalGraderContext.md) and returns an [`EvalScriptOutcome`](../../api/interfaces/eval.EvalScriptOutcome.md), synchronously or as a diff --git a/website/docs/zh/guide/development/evaluations.mdx b/website/docs/zh/guide/development/evaluations.mdx index 7a7f1f36c..405ff65ab 100644 --- a/website/docs/zh/guide/development/evaluations.mdx +++ b/website/docs/zh/guide/development/evaluations.mdx @@ -44,8 +44,8 @@ npx agent-bundle eval compare ### 编写一个结果 grader -`expectOutcome({ script })` 指名一个模块——相对套件文件解析，并且必须留在套件目录之内——它默认导出一个 -grader 函数。`agent-bundle/eval` 导出了它的契约：[`EvalGraderFunction`](../../api/types/eval.EvalGraderFunction.md) +`expectOutcome({ script })` 指名一个默认导出一个 grader 函数的模块。路径相对套件文件解析，且不得越出套件 +目录（越过它的 `../` 会被拒绝；这项检查针对的是路径本身，因此套件内的符号链接会被跟随到它所指向的任何位置）。`agent-bundle/eval` 导出了它的契约：[`EvalGraderFunction`](../../api/types/eval.EvalGraderFunction.md) 接收一个 [`EvalGraderContext`](../../api/interfaces/eval.EvalGraderContext.md)，并同步或以 promise 形式返回一个 [`EvalScriptOutcome`](../../api/interfaces/eval.EvalScriptOutcome.md)。下面是 `examples/skills-starter/evals/graders/release-result.ts` 的原文；上面的套件指名的 `./graders/status-result.ts`