From 7ec94d53ee8de91757e45ee6810d1f291d63d319 Mon Sep 17 00:00:00 2001 From: Dominic Nguyen Date: Wed, 12 Aug 2026 15:08:05 -0700 Subject: [PATCH 1/7] feat(benchmarks): gate empty-gold on utterance shape Reject questions, sibling commands, and refuse-then-alternate forms even when an LLM labels the empty as a pure refusal. --- .../synthesizer/emptyGoldUtterance.ts | 128 ++++++++++++++++++ .../src/translationBench/synthesizer/index.ts | 1 + ...ranslationBench.emptyGoldUtterance.spec.ts | 106 +++++++++++++++ 3 files changed, 235 insertions(+) create mode 100644 ts/packages/benchmarks/src/translationBench/synthesizer/emptyGoldUtterance.ts create mode 100644 ts/packages/benchmarks/test/translationBench.emptyGoldUtterance.spec.ts diff --git a/ts/packages/benchmarks/src/translationBench/synthesizer/emptyGoldUtterance.ts b/ts/packages/benchmarks/src/translationBench/synthesizer/emptyGoldUtterance.ts new file mode 100644 index 0000000000..f5023237d2 --- /dev/null +++ b/ts/packages/benchmarks/src/translationBench/synthesizer/emptyGoldUtterance.ts @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** ; | em-dash | en-dash | spaced hyphen — never bare `.` (schema.action / domains). */ +const CLAUSE_SEP = String.raw`(?:[;]|\u2014|\u2013|\s-\s)`; + +/** + * Clause separators for multi-part empties. Deliberately excludes `.` so + * schema.action tags, domains, and abbreviations do not false-split. + */ +const CLAUSE_SPLIT_RE = new RegExp(String.raw`\s*${CLAUSE_SEP}\s*`); + +/** + * Trailing clauses that still mean abstain (not a new tool request). + * Stripped before secondary-clause checks. + */ +const ABSTAIN_TRAIL_RE = new RegExp( + String.raw`${CLAUSE_SEP}\s*(?:I\b[\s\S]*|let\s+it\b[\s\S]*|leave\b[\s\S]*?\b(?:alone|unchanged|untouched)\b[\s\S]*|keep\b[\s\S]*|stay\b[\s\S]*|so\b[\s\S]*|because\b[\s\S]*|since\b[\s\S]*)$`, + "i", +); + +const OPENS_REFUSE_RE = /^(?:please\s+)?(?:do\s+not|don'?t|never)\b/i; + +const OPENS_LEAVE_ALONE_RE = /^(?:please\s+)?leave\b[\s\S]{0,48}\balone\b/i; + +const OPENS_OTHER_ABSTAIN_RE = + /^(?:please\s+)?(?:hands\s+off|do\s+nothing|refrain\s+from)\b/i; + +const OPENS_AVOID_DOING_RE = + /^(?:please\s+)?avoid\s+(?:doing|opening|closing|taking|capturing|running|starting|sending|changing|switching|deleting|creating|enabling|disabling)\b/i; + +/** Interrogative openers — exclude "do not" / "don't" (handled as refuse). */ +const INTERROGATIVE_OPENER_RE = + /^(?:what|why|how|when|where|who|which|is|are|can|could|would|should|does|did|will|have|has|was|were|what's|how's|who's|do(?!\s+not)\b)/i; + +const SOFT_SOLICIT_RE = + /\b(?:can you|could you|would you(?: mind)?|are you able|do you (?:know|support|handle)|is it possible|is there a way)\b/i; + +export interface TranslationBenchEmptyGoldUtteranceAssessment { + fair: boolean; + reason: string; +} + +/** + * Deterministic empty-gold utterance shape gate. + * + * LLM negativeAssessments alone are insufficient: the 1k-20260807-disambig set + * labeled ~100% of empties as review-approved while ~99% were contrastive + * sibling commands, how-to/status questions, or refuse-then-alternate forms + * (eval FPR ~97%). Labels may only approve pure_refusal when the utterance + * itself opens as a hard abstain and carries no toolable follow-on. + * + * Conservative by design — prefer false reject (regen) over false approve. + */ +export function assessEmptyGoldUtterance( + utterance: string, +): TranslationBenchEmptyGoldUtteranceAssessment { + const raw = String(utterance ?? "").trim(); + if (!raw) { + return { fair: false, reason: "empty utterance" }; + } + const t = raw.replace(/\s+/g, " "); + + if (/[?]/.test(t)) { + return { + fair: false, + reason: "question mark (invites chat/help/lookup)", + }; + } + // Check refuse openers before interrogative so "Do not …" is not + // misclassified as the bare auxiliary "Do …?". + const opensRefuse = + OPENS_REFUSE_RE.test(t) || + OPENS_LEAVE_ALONE_RE.test(t) || + OPENS_OTHER_ABSTAIN_RE.test(t) || + OPENS_AVOID_DOING_RE.test(t); + if (!opensRefuse) { + if (INTERROGATIVE_OPENER_RE.test(t)) { + return { fair: false, reason: "interrogative opener" }; + } + return { + fair: false, + reason: "does not open as pure refusal (need don't/do not/never/leave-alone)", + }; + } + if (SOFT_SOLICIT_RE.test(t)) { + return { fair: false, reason: "soft solicit or capability phrasing" }; + } + if (/\b(?:instead|rather\s+than)\b/i.test(t)) { + return { fair: false, reason: "contrastive instead/rather" }; + } + if (/\bjust\b/i.test(t)) { + return { + fair: false, + reason: "just-alternate (refuse-then-alternate or partial task)", + }; + } + if (/\b(?:tell|explain|describe|summarize)\b/i.test(t)) { + return { + fair: false, + reason: "requests explanation (chat/help under full catalog)", + }; + } + + // Strip a single allowed trailing abstain/reason clause, then reject any + // leftover secondary clause that is not itself abstain/reason. + const stripped = t.replace(ABSTAIN_TRAIL_RE, "").trim(); + const parts = stripped + .split(CLAUSE_SPLIT_RE) + .map((s) => s.trim()) + .filter(Boolean); + for (let i = 1; i < parts.length; i++) { + const p = parts[i]!; + if ( + /^(?:I\b|let\b|leave\b|keep\b|stay\b|so\b|because\b|since\b)/i.test( + p, + ) + ) { + continue; + } + return { + fair: false, + reason: `secondary clause not abstain/reason: "${p.slice(0, 80)}"`, + }; + } + + return { fair: true, reason: "pure refusal / leave-alone" }; +} diff --git a/ts/packages/benchmarks/src/translationBench/synthesizer/index.ts b/ts/packages/benchmarks/src/translationBench/synthesizer/index.ts index 4197f378d7..f1c7c94e29 100644 --- a/ts/packages/benchmarks/src/translationBench/synthesizer/index.ts +++ b/ts/packages/benchmarks/src/translationBench/synthesizer/index.ts @@ -15,4 +15,5 @@ export * from "./synthesizerPrompts.js"; export * from "./utteranceDisambiguation.js"; export * from "./catalogGenerator/index.js"; export { seedQaJsonlAdapter } from "./adapters/seedQaJsonlAdapter.js"; +export * from "./emptyGoldUtterance.js"; export * from "./goldParameterHygiene.js"; diff --git a/ts/packages/benchmarks/test/translationBench.emptyGoldUtterance.spec.ts b/ts/packages/benchmarks/test/translationBench.emptyGoldUtterance.spec.ts new file mode 100644 index 0000000000..690873a01f --- /dev/null +++ b/ts/packages/benchmarks/test/translationBench.emptyGoldUtterance.spec.ts @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, expect, it } from "@jest/globals"; + +import { assessEmptyGoldUtterance } from "../src/translationBench/synthesizer/emptyGoldUtterance.js"; + +describe("assessEmptyGoldUtterance deterministic shape gate", () => { + it("accepts start-anchored pure refusals and leave-alone forms", () => { + const fair = [ + "Don't take a screenshot of my online banking page.", + "Leave my tabs alone.", + "Do not open any websites right now.", + "Don't enable Game Mode; I need to compare performance with it off.", + "Don't pause the audiobook; let it keep playing.", + 'Don\'t deselect the photos in the "Graduation Ceremony" montage; leave the current selection unchanged.', + "Don't reload the concert ticket page; I haven't saved my details yet.", + "Don't cancel my passport renewal appointment on November 12.", + "Please don't pause the audiobook; let it keep playing.", + "Don't resume the podcast yet.", + "Don't go forward yet; stay on this checkout page.", + "Never open any websites right now.", + "Hands off my browser tabs.", + "Do nothing with my open tabs.", + // Periods in schema.action tags must not false-split clauses. + "Don't run browser.openWebPage right now; leave everything alone.", + "Don't run foo.bar.baz right now; leave everything alone (0).", + ]; + for (const u of fair) { + const r = assessEmptyGoldUtterance(u); + expect({ u, ...r }).toEqual({ + u, + fair: true, + reason: "pure refusal / leave-alone", + }); + } + }); + + it("rejects 1k-corpus unfair empties (questions, siblings, partials)", () => { + const unfair: Array<{ u: string; reasonSubstr: string }> = [ + { + u: "What keyboard shortcut can I use to take a screenshot of a webpage?", + reasonSubstr: "question", + }, + { + u: "Search Bing for Microsoft's current stock price.", + reasonSubstr: "does not open as pure refusal", + }, + { + u: "Close the fourth tab with the weather forecast.", + reasonSubstr: "does not open as pure refusal", + }, + { + u: "Don't close all tabs; just close this one.", + reasonSubstr: "just-alternate", + }, + { + u: "Can you open google.com for me?", + reasonSubstr: "question", + }, + { + u: "Don't open a website—just tell me whether the downtown library is open today.", + reasonSubstr: "just-alternate", + }, + { + u: "Build the current Visual Studio solution, but don't start debugging it.", + reasonSubstr: "does not open as pure refusal", + }, + { + u: "Don't change the editor layout; just increase the code font size.", + reasonSubstr: "just-alternate", + }, + { + u: "Turn on Night Light for this reading session only—don't schedule it.", + reasonSubstr: "does not open as pure refusal", + }, + { + u: "Don't list the scaffolding patterns; explain what a TypeAgent package manifest does.", + reasonSubstr: "explanation", + }, + { + u: "Stop reading the current webpage.", + reasonSubstr: "does not open as pure refusal", + }, + { + u: "Scroll up to the hotel comparison table near the top of the page.", + reasonSubstr: "does not open as pure refusal", + }, + { + u: "Is Bluetooth currently enabled?", + reasonSubstr: "question", + }, + { + u: "Keep my email tabs open, but close this webpage.", + reasonSubstr: "does not open as pure refusal", + }, + ]; + for (const { u, reasonSubstr } of unfair) { + const r = assessEmptyGoldUtterance(u); + expect(r.fair).toBe(false); + expect(r.reason.toLowerCase()).toContain( + reasonSubstr.toLowerCase(), + ); + } + }); +}); From d235685fbea49439f36019215f57458f9a6bb9fa Mon Sep 17 00:00:00 2001 From: typeagent-bot Date: Wed, 12 Aug 2026 22:17:55 +0000 Subject: [PATCH 2/7] docs: regenerate README.AUTOGEN.md, command reference, and action browser --- ts/packages/benchmarks/README.AUTOGEN.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ts/packages/benchmarks/README.AUTOGEN.md b/ts/packages/benchmarks/README.AUTOGEN.md index ecc64a825c..75d63ec343 100644 --- a/ts/packages/benchmarks/README.AUTOGEN.md +++ b/ts/packages/benchmarks/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # @typeagent/benchmarks — AI-generated documentation @@ -52,10 +52,10 @@ _None._ - [./src/core/types.ts](./src/core/types.ts) - [./src/translationBench/action-parameters-grader.generated.json](./src/translationBench/action-parameters-grader.generated.json) - [./src/translationBench/catalog.generated.json](./src/translationBench/catalog.generated.json) -- _…and 29 more under `./src/`._ +- _…and 30 more under `./src/`._ --- -_Auto-generated against commit `2f1ae13a34a138343a5b5113783950a8f1746724` on `2026-08-08T02:25:27.234Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ +_Auto-generated against commit `7ec94d53ee8de91757e45ee6810d1f291d63d319` on `2026-08-12T22:15:33.917Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ From e0141d90d812b22291264785035f0bae2fb91f9d Mon Sep 17 00:00:00 2001 From: Dominic Nguyen Date: Wed, 12 Aug 2026 15:43:17 -0700 Subject: [PATCH 3/7] chore: requeue CI after smoke install flake From b05505e10561939718dfda93da5c4d40b6a565ee Mon Sep 17 00:00:00 2001 From: Dominic Nguyen Date: Wed, 12 Aug 2026 15:49:44 -0700 Subject: [PATCH 4/7] chore: requeue CI after smoke install flake From 9062d672064583e8924ac7177bf6d01071f4d024 Mon Sep 17 00:00:00 2001 From: typeagent-bot Date: Thu, 13 Aug 2026 00:07:15 +0000 Subject: [PATCH 5/7] docs: regenerate README.AUTOGEN.md, command reference, and action browser --- ts/packages/benchmarks/README.AUTOGEN.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ts/packages/benchmarks/README.AUTOGEN.md b/ts/packages/benchmarks/README.AUTOGEN.md index 75d63ec343..5f876698c6 100644 --- a/ts/packages/benchmarks/README.AUTOGEN.md +++ b/ts/packages/benchmarks/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # @typeagent/benchmarks — AI-generated documentation @@ -52,10 +52,10 @@ _None._ - [./src/core/types.ts](./src/core/types.ts) - [./src/translationBench/action-parameters-grader.generated.json](./src/translationBench/action-parameters-grader.generated.json) - [./src/translationBench/catalog.generated.json](./src/translationBench/catalog.generated.json) -- _…and 30 more under `./src/`._ +- _…and 32 more under `./src/`._ --- -_Auto-generated against commit `7ec94d53ee8de91757e45ee6810d1f291d63d319` on `2026-08-12T22:15:33.917Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ +_Auto-generated against commit `db6cd0131f01e47a68bd85df790211cec36baf22` on `2026-08-13T00:05:00.987Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ From ee8cd4f54d48f39527b50c7f697fb7d3461d04fc Mon Sep 17 00:00:00 2001 From: typeagent-bot Date: Thu, 13 Aug 2026 01:29:05 +0000 Subject: [PATCH 6/7] docs: regenerate README.AUTOGEN.md, command reference, and action browser --- ts/packages/benchmarks/README.AUTOGEN.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ts/packages/benchmarks/README.AUTOGEN.md b/ts/packages/benchmarks/README.AUTOGEN.md index 5f876698c6..98ab93d8a5 100644 --- a/ts/packages/benchmarks/README.AUTOGEN.md +++ b/ts/packages/benchmarks/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # @typeagent/benchmarks — AI-generated documentation @@ -52,10 +52,10 @@ _None._ - [./src/core/types.ts](./src/core/types.ts) - [./src/translationBench/action-parameters-grader.generated.json](./src/translationBench/action-parameters-grader.generated.json) - [./src/translationBench/catalog.generated.json](./src/translationBench/catalog.generated.json) -- _…and 32 more under `./src/`._ +- _…and 33 more under `./src/`._ --- -_Auto-generated against commit `db6cd0131f01e47a68bd85df790211cec36baf22` on `2026-08-13T00:05:00.987Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ +_Auto-generated against commit `c97c0ab90b8bea3374463efebdb00d6d0ed4ff80` on `2026-08-13T01:26:47.872Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ From 561c87cbef6885274f1526330b94676f168e4acd Mon Sep 17 00:00:00 2001 From: typeagent-bot Date: Thu, 13 Aug 2026 03:37:02 +0000 Subject: [PATCH 7/7] docs: regenerate README.AUTOGEN.md, command reference, and action browser --- ts/packages/benchmarks/README.AUTOGEN.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ts/packages/benchmarks/README.AUTOGEN.md b/ts/packages/benchmarks/README.AUTOGEN.md index 98ab93d8a5..9ad6c8094f 100644 --- a/ts/packages/benchmarks/README.AUTOGEN.md +++ b/ts/packages/benchmarks/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # @typeagent/benchmarks — AI-generated documentation @@ -49,13 +49,13 @@ _None._ - [./src/core/model-prices.generated.json](./src/core/model-prices.generated.json) - [./src/core/paths.ts](./src/core/paths.ts) - [./src/core/prices.ts](./src/core/prices.ts) +- [./src/core/rateLimiter.ts](./src/core/rateLimiter.ts) - [./src/core/types.ts](./src/core/types.ts) - [./src/translationBench/action-parameters-grader.generated.json](./src/translationBench/action-parameters-grader.generated.json) -- [./src/translationBench/catalog.generated.json](./src/translationBench/catalog.generated.json) -- _…and 33 more under `./src/`._ +- _…and 35 more under `./src/`._ --- -_Auto-generated against commit `c97c0ab90b8bea3374463efebdb00d6d0ed4ff80` on `2026-08-13T01:26:47.872Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ +_Auto-generated against commit `01feca686ce14ae8b00182f75c305cf40e92c415` on `2026-08-13T03:34:32.783Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._