Gate empty-gold negatives on utterance shape

ts/packages/benchmarks/README.AUTOGEN.md

@@ -3,7 +3,7 @@
 
 <!-- AUTOGEN:DOCS:START -->
 
-<!-- AUTOGEN:DOCS:HASH:sha256=6add7220cd2366adea51b91f59e3a7c5a4cd7f6ba61ebef9308335d80c2e4027 -->
+<!-- AUTOGEN:DOCS:HASH:sha256=4789b0b546809d399518b84ad3e00f70824d9c558a19931aa499099859de8dd0 -->
 <!-- AUTOGEN:DOCS:SOURCE: ./README.md (hand-written documentation; this file is the AI-generated companion) -->
 
 # @typeagent/benchmarks — AI-generated documentation
@@ -52,10 +52,10 @@ _None._
 - [./src/core/rateLimiter.ts](./src/core/rateLimiter.ts)
 - [./src/core/types.ts](./src/core/types.ts)
 - [./src/translationBench/action-parameters-grader.generated.json](./src/translationBench/action-parameters-grader.generated.json)
-- _…and 34 more under `./src/`._
+- _…and 35 more under `./src/`._
 
 ---
 
-_Auto-generated against commit `d9cf714f7d151120013855a722e7583cbf2c30d7` on `2026-08-13T01:26:47.704Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._
+_Auto-generated against commit `01feca686ce14ae8b00182f75c305cf40e92c415` on `2026-08-13T03:34:32.783Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._
 
 <!-- AUTOGEN:DOCS:END -->

ts/packages/benchmarks/src/translationBench/synthesizer/emptyGoldUtterance.ts

@@ -0,0 +1,128 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+/** ; | em-dash | en-dash | spaced hyphen — never bare `.` (schema.action / domains). */
+const CLAUSE_SEP = String.raw`(?:[;]|\u2014|\u2013|\s-\s)`;
+
+/**
+ * Clause separators for multi-part empties. Deliberately excludes `.` so
+ * schema.action tags, domains, and abbreviations do not false-split.
+ */
+const CLAUSE_SPLIT_RE = new RegExp(String.raw`\s*${CLAUSE_SEP}\s*`);
+
+/**
+ * Trailing clauses that still mean abstain (not a new tool request).
+ * Stripped before secondary-clause checks.
+ */
+const ABSTAIN_TRAIL_RE = new RegExp(
+    String.raw`${CLAUSE_SEP}\s*(?:I\b[\s\S]*|let\s+it\b[\s\S]*|leave\b[\s\S]*?\b(?:alone|unchanged|untouched)\b[\s\S]*|keep\b[\s\S]*|stay\b[\s\S]*|so\b[\s\S]*|because\b[\s\S]*|since\b[\s\S]*)$`,
+    "i",
+);
+
+const OPENS_REFUSE_RE = /^(?:please\s+)?(?:do\s+not|don'?t|never)\b/i;
+
+const OPENS_LEAVE_ALONE_RE = /^(?:please\s+)?leave\b[\s\S]{0,48}\balone\b/i;
+
+const OPENS_OTHER_ABSTAIN_RE =
+    /^(?:please\s+)?(?:hands\s+off|do\s+nothing|refrain\s+from)\b/i;
+
+const OPENS_AVOID_DOING_RE =
+    /^(?:please\s+)?avoid\s+(?:doing|opening|closing|taking|capturing|running|starting|sending|changing|switching|deleting|creating|enabling|disabling)\b/i;
+
+/** Interrogative openers — exclude "do not" / "don't" (handled as refuse). */
+const INTERROGATIVE_OPENER_RE =
+    /^(?:what|why|how|when|where|who|which|is|are|can|could|would|should|does|did|will|have|has|was|were|what's|how's|who's|do(?!\s+not)\b)/i;
+
+const SOFT_SOLICIT_RE =
+    /\b(?:can you|could you|would you(?: mind)?|are you able|do you (?:know|support|handle)|is it possible|is there a way)\b/i;
+
+export interface TranslationBenchEmptyGoldUtteranceAssessment {
+    fair: boolean;
+    reason: string;
+}
+
+/**
+ * Deterministic empty-gold utterance shape gate.
+ *
+ * LLM negativeAssessments alone are insufficient: the 1k-20260807-disambig set
+ * labeled ~100% of empties as review-approved while ~99% were contrastive
+ * sibling commands, how-to/status questions, or refuse-then-alternate forms
+ * (eval FPR ~97%). Labels may only approve pure_refusal when the utterance
+ * itself opens as a hard abstain and carries no toolable follow-on.
+ *
+ * Conservative by design — prefer false reject (regen) over false approve.
+ */
+export function assessEmptyGoldUtterance(
+    utterance: string,
+): TranslationBenchEmptyGoldUtteranceAssessment {
+    const raw = String(utterance ?? "").trim();
+    if (!raw) {
+        return { fair: false, reason: "empty utterance" };
+    }
+    const t = raw.replace(/\s+/g, " ");
+
+    if (/[?]/.test(t)) {
+        return {
+            fair: false,
+            reason: "question mark (invites chat/help/lookup)",
+        };
+    }
+    // Check refuse openers before interrogative so "Do not …" is not
+    // misclassified as the bare auxiliary "Do …?".
+    const opensRefuse =
+        OPENS_REFUSE_RE.test(t) ||
+        OPENS_LEAVE_ALONE_RE.test(t) ||
+        OPENS_OTHER_ABSTAIN_RE.test(t) ||
+        OPENS_AVOID_DOING_RE.test(t);
+    if (!opensRefuse) {
+        if (INTERROGATIVE_OPENER_RE.test(t)) {
+            return { fair: false, reason: "interrogative opener" };
+        }
+        return {
+            fair: false,
+            reason: "does not open as pure refusal (need don't/do not/never/leave-alone)",
+        };
+    }
+    if (SOFT_SOLICIT_RE.test(t)) {
+        return { fair: false, reason: "soft solicit or capability phrasing" };
+    }
+    if (/\b(?:instead|rather\s+than)\b/i.test(t)) {
+        return { fair: false, reason: "contrastive instead/rather" };
+    }
+    if (/\bjust\b/i.test(t)) {
+        return {
+            fair: false,
+            reason: "just-alternate (refuse-then-alternate or partial task)",
+        };
+    }
+    if (/\b(?:tell|explain|describe|summarize)\b/i.test(t)) {
+        return {
+            fair: false,
+            reason: "requests explanation (chat/help under full catalog)",
+        };
+    }
+
+    // Strip a single allowed trailing abstain/reason clause, then reject any
+    // leftover secondary clause that is not itself abstain/reason.
+    const stripped = t.replace(ABSTAIN_TRAIL_RE, "").trim();
+    const parts = stripped
+        .split(CLAUSE_SPLIT_RE)
+        .map((s) => s.trim())
+        .filter(Boolean);
+    for (let i = 1; i < parts.length; i++) {
+        const p = parts[i]!;
+        if (
+            /^(?:I\b|let\b|leave\b|keep\b|stay\b|so\b|because\b|since\b)/i.test(
+                p,
+            )
+        ) {
+            continue;
+        }
+        return {
+            fair: false,
+            reason: `secondary clause not abstain/reason: "${p.slice(0, 80)}"`,
+        };
+    }
+
+    return { fair: true, reason: "pure refusal / leave-alone" };
+}

ts/packages/benchmarks/src/translationBench/synthesizer/index.ts

@@ -15,5 +15,6 @@ export * from "./synthesizerPrompts.js";
 export * from "./utteranceDisambiguation.js";
 export * from "./catalogGenerator/index.js";
 export { seedQaJsonlAdapter } from "./adapters/seedQaJsonlAdapter.js";
+export * from "./emptyGoldUtterance.js";
 export * from "./goldParameterHygiene.js";
 export * from "./actionValidation.js";

ts/packages/benchmarks/test/translationBench.emptyGoldUtterance.spec.ts

@@ -0,0 +1,106 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { describe, expect, it } from "@jest/globals";
+
+import { assessEmptyGoldUtterance } from "../src/translationBench/synthesizer/emptyGoldUtterance.js";
+
+describe("assessEmptyGoldUtterance deterministic shape gate", () => {
+    it("accepts start-anchored pure refusals and leave-alone forms", () => {
+        const fair = [
+            "Don't take a screenshot of my online banking page.",
+            "Leave my tabs alone.",
+            "Do not open any websites right now.",
+            "Don't enable Game Mode; I need to compare performance with it off.",
+            "Don't pause the audiobook; let it keep playing.",
+            'Don\'t deselect the photos in the "Graduation Ceremony" montage; leave the current selection unchanged.',
+            "Don't reload the concert ticket page; I haven't saved my details yet.",
+            "Don't cancel my passport renewal appointment on November 12.",
+            "Please don't pause the audiobook; let it keep playing.",
+            "Don't resume the podcast yet.",
+            "Don't go forward yet; stay on this checkout page.",
+            "Never open any websites right now.",
+            "Hands off my browser tabs.",
+            "Do nothing with my open tabs.",
+            // Periods in schema.action tags must not false-split clauses.
+            "Don't run browser.openWebPage right now; leave everything alone.",
+            "Don't run foo.bar.baz right now; leave everything alone (0).",
+        ];
+        for (const u of fair) {
+            const r = assessEmptyGoldUtterance(u);
+            expect({ u, ...r }).toEqual({
+                u,
+                fair: true,
+                reason: "pure refusal / leave-alone",
+            });
+        }
+    });
+
+    it("rejects 1k-corpus unfair empties (questions, siblings, partials)", () => {
+        const unfair: Array<{ u: string; reasonSubstr: string }> = [
+            {
+                u: "What keyboard shortcut can I use to take a screenshot of a webpage?",
+                reasonSubstr: "question",
+            },
+            {
+                u: "Search Bing for Microsoft's current stock price.",
+                reasonSubstr: "does not open as pure refusal",
+            },
+            {
+                u: "Close the fourth tab with the weather forecast.",
+                reasonSubstr: "does not open as pure refusal",
+            },
+            {
+                u: "Don't close all tabs; just close this one.",
+                reasonSubstr: "just-alternate",
+            },
+            {
+                u: "Can you open google.com for me?",
+                reasonSubstr: "question",
+            },
+            {
+                u: "Don't open a website—just tell me whether the downtown library is open today.",
+                reasonSubstr: "just-alternate",
+            },
+            {
+                u: "Build the current Visual Studio solution, but don't start debugging it.",
+                reasonSubstr: "does not open as pure refusal",
+            },
+            {
+                u: "Don't change the editor layout; just increase the code font size.",
+                reasonSubstr: "just-alternate",
+            },
+            {
+                u: "Turn on Night Light for this reading session only—don't schedule it.",
+                reasonSubstr: "does not open as pure refusal",
+            },
+            {
+                u: "Don't list the scaffolding patterns; explain what a TypeAgent package manifest does.",
+                reasonSubstr: "explanation",
+            },
+            {
+                u: "Stop reading the current webpage.",
+                reasonSubstr: "does not open as pure refusal",
+            },
+            {
+                u: "Scroll up to the hotel comparison table near the top of the page.",
+                reasonSubstr: "does not open as pure refusal",
+            },
+            {
+                u: "Is Bluetooth currently enabled?",
+                reasonSubstr: "question",
+            },
+            {
+                u: "Keep my email tabs open, but close this webpage.",
+                reasonSubstr: "does not open as pure refusal",
+            },
+        ];
+        for (const { u, reasonSubstr } of unfair) {
+            const r = assessEmptyGoldUtterance(u);
+            expect(r.fair).toBe(false);
+            expect(r.reason.toLowerCase()).toContain(
+                reasonSubstr.toLowerCase(),
+            );
+        }
+    });
+});