perf(server): avoid full patches for checkpoint summaries (#9694)

Author: t3dotgg

apps/server/package.json

@@ -28,7 +28,6 @@
     "@effect/sql-sqlite-bun": "catalog:",
     "@ff-labs/fff-node": "0.9.4",
     "@opencode-ai/sdk": "^1.3.15",
-    "@pierre/diffs": "catalog:",
     "effect": "catalog:",
     "msgpackr-extract": "3.0.4",
     "node-pty": "^1.1.0",

apps/server/src/checkpointing/CheckpointStore.test.ts

@@ -4,6 +4,7 @@ import * as NodePath from "node:path";
 import * as NodeServices from "@effect/platform-node/NodeServices";
 import { it } from "@effect/vitest";
 import { ThreadId, type VcsError } from "@t3tools/contracts";
+import { HostProcessPlatform } from "@t3tools/shared/hostProcess";
 import * as Effect from "effect/Effect";
 import * as FileSystem from "effect/FileSystem";
 import * as Layer from "effect/Layer";
@@ -12,6 +13,7 @@ import * as Scope from "effect/Scope";
 import { describe, expect } from "vite-plus/test";
 
 import { checkpointRefForThreadTurn } from "./Utils.ts";
+import { parseTurnDiffFilesFromNumstat } from "./Diffs.ts";
 import * as CheckpointStore from "./CheckpointStore.ts";
 import * as VcsDriverRegistry from "../vcs/VcsDriverRegistry.ts";
 import * as VcsProcess from "../vcs/VcsProcess.ts";
@@ -250,6 +252,182 @@ it.layer(TestLayer)("CheckpointStore.layer", (it) => {
         expect(whitespaceIgnoredDiff).toContain("+        <div>");
         expect(whitespaceIgnoredDiff).not.toContain("-      <h1>Title</h1>");
         expect(whitespaceIgnoredDiff).not.toContain("+          <h1>Title</h1>");
+
+        for (const ignoreWhitespace of [false, true]) {
+          const numstat = yield* checkpointStore.diffCheckpoints({
+            cwd: tmp,
+            fromCheckpointRef,
+            toCheckpointRef,
+            ignoreWhitespace,
+            format: "numstat",
+          });
+          expect(parseTurnDiffFilesFromNumstat(numstat)).toEqual([
+            {
+              path: "Component.tsx",
+              additions: ignoreWhitespace ? 4 : 6,
+              deletions: ignoreWhitespace ? 0 : 2,
+            },
+          ]);
+        }
+      }),
+    );
+  });
+
+  describe("checkpoint file summaries", () => {
+    it.effect("counts changes whose full patch exceeds the output limit", () =>
+      Effect.gen(function* () {
+        const tmp = yield* makeTmpDir();
+        yield* initRepoWithCommit(tmp);
+        const checkpointStore = yield* CheckpointStore.CheckpointStore;
+        const threadId = ThreadId.make("large-checkpoint-summary");
+        const fromCheckpointRef = checkpointRefForThreadTurn(threadId, 0);
+        const toCheckpointRef = checkpointRefForThreadTurn(threadId, 1);
+        const filePath = NodePath.join(tmp, "README.md");
+        const lineCount = 20_000;
+        yield* writeTextFile(filePath, `${"before".repeat(50)}\n`.repeat(lineCount));
+        yield* checkpointStore.captureCheckpoint({ cwd: tmp, checkpointRef: fromCheckpointRef });
+        yield* writeTextFile(filePath, `${"after".repeat(60)}\n`.repeat(lineCount));
+        yield* checkpointStore.captureCheckpoint({ cwd: tmp, checkpointRef: toCheckpointRef });
+
+        const numstat = yield* checkpointStore.diffCheckpoints({
+          cwd: tmp,
+          fromCheckpointRef,
+          toCheckpointRef,
+          ignoreWhitespace: false,
+          format: "numstat",
+        });
+
+        expect(parseTurnDiffFilesFromNumstat(numstat)).toEqual([
+          { path: "README.md", additions: lineCount, deletions: lineCount },
+        ]);
+        expect(numstat.length).toBeLessThan(100);
+      }),
+    );
+
+    it.effect("preserves file paths and turn ranges without changing the user index", () =>
+      Effect.gen(function* () {
+        const tmp = yield* makeTmpDir();
+        yield* initRepoWithCommit(tmp);
+        yield* git(tmp, ["config", "diff.renames", "copies"]);
+        const fileSystem = yield* FileSystem.FileSystem;
+        const checkpointStore = yield* CheckpointStore.CheckpointStore;
+        const threadId = ThreadId.make("checkpoint-summary-paths");
+        const baseline = checkpointRefForThreadTurn(threadId, 0);
+        const firstTurn = checkpointRefForThreadTurn(threadId, 1);
+        const secondTurn = checkpointRefForThreadTurn(threadId, 2);
+        const copiedText = Array.from({ length: 20 }, (_, index) => `copy line ${index}\n`).join(
+          "",
+        );
+        const platform = yield* HostProcessPlatform;
+        const renamedPath = platform === "win32" ? "renamed café.txt" : "renamed\tcafé\nname.txt";
+        const addedPath = platform === "win32" ? "new café.txt" : "new\tfile\n名.txt";
+        for (const [path, contents] of Object.entries({
+          "copy-source.txt": copiedText,
+          "deleted.txt": "delete me\n",
+          "rename-old.txt": "before\nkeep one\nkeep two\nkeep three\n",
+          "binary.bin": "\0before",
+        })) {
+          yield* writeTextFile(NodePath.join(tmp, path), contents);
+        }
+        yield* checkpointStore.captureCheckpoint({ cwd: tmp, checkpointRef: baseline });
+
+        yield* fileSystem.rename(
+          NodePath.join(tmp, "rename-old.txt"),
+          NodePath.join(tmp, renamedPath),
+        );
+        yield* fileSystem.remove(NodePath.join(tmp, "deleted.txt"));
+        for (const [path, contents] of Object.entries({
+          "copy-source.txt": `${copiedText}one more\n`,
+          "copied.txt": copiedText,
+          [renamedPath]: "after\nkeep one\nkeep two\nkeep three\n",
+          "binary.bin": "\0after",
+          "empty.txt": "",
+          [addedPath]: "first\nsecond\n",
+        })) {
+          yield* writeTextFile(NodePath.join(tmp, path), contents);
+        }
+        yield* checkpointStore.captureCheckpoint({ cwd: tmp, checkpointRef: firstTurn });
+        const userIndex = yield* fileSystem.readFile(NodePath.join(tmp, ".git/index"));
+        const input = {
+          cwd: tmp,
+          fromCheckpointRef: baseline,
+          toCheckpointRef: firstTurn,
+          ignoreWhitespace: false,
+          format: "numstat" as const,
+        };
+        const firstSummary = parseTurnDiffFilesFromNumstat(
+          yield* checkpointStore.diffCheckpoints(input),
+        );
+        const expectedFiles = [
+          { path: "binary.bin", additions: 0, deletions: 0 },
+          { path: "copied.txt", additions: 0, deletions: 0 },
+          { path: "copy-source.txt", additions: 1, deletions: 0 },
+          { path: "deleted.txt", additions: 0, deletions: 1 },
+          { path: "empty.txt", additions: 0, deletions: 0 },
+          { path: addedPath, additions: 2, deletions: 0 },
+          { path: renamedPath, additions: 1, deletions: 1 },
+        ].toSorted((left, right) => left.path.localeCompare(right.path));
+        expect(firstSummary).toEqual(expectedFiles);
+
+        yield* fileSystem.remove(NodePath.join(tmp, "empty.txt"));
+        yield* writeTextFile(NodePath.join(tmp, "copy-source.txt"), "replacement\n");
+        yield* checkpointStore.captureCheckpoint({ cwd: tmp, checkpointRef: secondTurn });
+        const secondSummary = parseTurnDiffFilesFromNumstat(
+          yield* checkpointStore.diffCheckpoints({
+            ...input,
+            fromCheckpointRef: firstTurn,
+            toCheckpointRef: secondTurn,
+          }),
+        );
+        expect(secondSummary).toEqual([
+          { path: "copy-source.txt", additions: 1, deletions: 21 },
+          { path: "empty.txt", additions: 0, deletions: 0 },
+        ]);
+
+        const inclusiveSummary = parseTurnDiffFilesFromNumstat(
+          yield* checkpointStore.diffCheckpoints({ ...input, toCheckpointRef: secondTurn }),
+        );
+        expect(inclusiveSummary).toEqual(
+          expectedFiles
+            .filter((file) => file.path !== "empty.txt")
+            .map((file) =>
+              file.path === "copy-source.txt" ? { ...file, additions: 1, deletions: 20 } : file,
+            ),
+        );
+        expect(
+          yield* checkpointStore.diffCheckpoints({ ...input, toCheckpointRef: baseline }),
+        ).toBe("");
+        expect(yield* fileSystem.readFile(NodePath.join(tmp, ".git/index"))).toEqual(userIndex);
+      }),
+    );
+
+    it.effect("uses HEAD for a missing baseline only when requested", () =>
+      Effect.gen(function* () {
+        const tmp = yield* makeTmpDir();
+        yield* initRepoWithCommit(tmp);
+        const checkpointStore = yield* CheckpointStore.CheckpointStore;
+        const threadId = ThreadId.make("checkpoint-summary-fallback");
+        const fromCheckpointRef = checkpointRefForThreadTurn(threadId, 0);
+        const toCheckpointRef = checkpointRefForThreadTurn(threadId, 1);
+        yield* writeTextFile(NodePath.join(tmp, "README.md"), "changed\n");
+        yield* checkpointStore.captureCheckpoint({ cwd: tmp, checkpointRef: toCheckpointRef });
+        const input = {
+          cwd: tmp,
+          fromCheckpointRef,
+          toCheckpointRef,
+          ignoreWhitespace: false,
+          format: "numstat" as const,
+        };
+
+        const error = yield* Effect.flip(checkpointStore.diffCheckpoints(input));
+        expect(error._tag).toBe("VcsProcessExitError");
+        const numstat = yield* checkpointStore.diffCheckpoints({
+          ...input,
+          fallbackFromToHead: true,
+        });
+        expect(parseTurnDiffFilesFromNumstat(numstat)).toEqual([
+          { path: "README.md", additions: 1, deletions: 1 },
+        ]);
       }),
     );
   });

apps/server/src/checkpointing/CheckpointStore.ts

@@ -39,6 +39,7 @@ export interface DiffCheckpointsInput {
   readonly toCheckpointRef: CheckpointRef;
   readonly fallbackFromToHead?: boolean;
   readonly ignoreWhitespace: boolean;
+  readonly format?: "patch" | "numstat";
 }
 
 export interface DeleteCheckpointRefsInput {
@@ -77,8 +78,9 @@ export class CheckpointStore extends Context.Service<
     ) => Effect.Effect<boolean, CheckpointStoreError>;
 
     /**
-     * Compute a patch diff between two checkpoint refs.
+     * Compute a diff between two checkpoint refs. Defaults to a full patch.
      *
+     * Numstat output has NUL-delimited paths for file summaries.
      * Can optionally treat a missing "from" ref as `HEAD`.
      */
     readonly diffCheckpoints: (

apps/server/src/checkpointing/Diffs.test.ts

@@ -1,68 +1,54 @@
 import { describe, expect, it } from "vite-plus/test";
 
-import { parseTurnDiffFilesFromUnifiedDiff } from "./Diffs.ts";
+import { parseTurnDiffFilesFromNumstat } from "./Diffs.ts";
 
-describe("parseTurnDiffFilesFromUnifiedDiff", () => {
-  it("returns empty list for empty diff", () => {
-    expect(parseTurnDiffFilesFromUnifiedDiff("")).toEqual([]);
+describe("parseTurnDiffFilesFromNumstat", () => {
+  it("returns an empty list when no files changed", () => {
+    expect(parseTurnDiffFilesFromNumstat("")).toEqual([]);
   });
 
-  it("parses per-file additions and deletions", () => {
-    const diff = [
-      "diff --git a/a.txt b/a.txt",
-      "index 1111111..2222222 100644",
-      "--- a/a.txt",
-      "+++ b/a.txt",
-      "@@ -1,2 +1,3 @@",
-      " one",
-      "-two",
-      "+two updated",
-      "+three",
-      "diff --git a/src/b.ts b/src/b.ts",
-      "index 3333333..4444444 100644",
-      "--- a/src/b.ts",
-      "+++ b/src/b.ts",
-      "@@ -3,2 +3,0 @@",
-      "-old",
-      "-stale",
-      "",
-    ].join("\n");
-
-    expect(parseTurnDiffFilesFromUnifiedDiff(diff)).toEqual([
+  it("sorts files and preserves addition and deletion counts", () => {
+    const numstat = ["0\t2\tsrc/b.ts", "2\t1\ta.txt", ""].join("\0");
+    expect(parseTurnDiffFilesFromNumstat(numstat)).toEqual([
       { path: "a.txt", additions: 2, deletions: 1 },
       { path: "src/b.ts", additions: 0, deletions: 2 },
     ]);
   });
 
-  it("parses rename-only diffs with zero line changes", () => {
-    const diff = [
-      "diff --git a/src/old.ts b/src/new.ts",
-      "similarity index 100%",
-      "rename from src/old.ts",
-      "rename to src/new.ts",
+  it("uses destination paths for renames and copies", () => {
+    const numstat = [
+      "0\t0\t",
+      "src/old.ts",
+      "src/new.ts",
+      "2\t1\t",
+      "src/source.ts",
+      "src/copied.ts",
+      "1\t0\tother.ts",
       "",
-    ].join("\n");
+    ].join("\0");
 
-    expect(parseTurnDiffFilesFromUnifiedDiff(diff)).toEqual([
+    expect(parseTurnDiffFilesFromNumstat(numstat)).toEqual([
+      { path: "other.ts", additions: 1, deletions: 0 },
+      { path: "src/copied.ts", additions: 2, deletions: 1 },
       { path: "src/new.ts", additions: 0, deletions: 0 },
     ]);
   });
 
-  it("normalizes CRLF input before parsing", () => {
-    const diff = [
-      "diff --git a/a.txt b/a.txt",
-      "index 1111111..2222222 100644",
-      "--- a/a.txt",
-      "+++ b/a.txt",
-      "@@ -1 +1,2 @@",
-      "-one",
-      "+one updated",
-      "+two",
-      "",
-    ].join("\r\n");
+  it("keeps binary files and empty files with zero line changes", () => {
+    const numstat = ["-\t-\timage.png", "0\t0\tempty.txt", ""].join("\0");
+    expect(parseTurnDiffFilesFromNumstat(numstat)).toEqual([
+      { path: "empty.txt", additions: 0, deletions: 0 },
+      { path: "image.png", additions: 0, deletions: 0 },
+    ]);
+  });
 
-    expect(parseTurnDiffFilesFromUnifiedDiff(diff)).toEqual([
-      { path: "a.txt", additions: 2, deletions: 1 },
+  it("preserves Unicode, tabs, line endings, and spaces in paths", () => {
+    const path = " café\tline\r\nname.txt ";
+    const numstat = `3\t2\t\0old\tname\n.txt\0${path}\0`;
+
+    expect(parseTurnDiffFilesFromNumstat(numstat)).toEqual([{ path, additions: 3, deletions: 2 }]);
+    expect(parseTurnDiffFilesFromNumstat(`1\t0\t${path}\0`)).toEqual([
+      { path, additions: 1, deletions: 0 },
     ]);
   });
 });

apps/server/src/checkpointing/Diffs.ts

@@ -1,27 +1,33 @@
-import { parsePatchFiles } from "@pierre/diffs/utils/parsePatchFiles";
-
 export interface TurnDiffFileSummary {
   readonly path: string;
   readonly additions: number;
   readonly deletions: number;
 }
 
-export function parseTurnDiffFilesFromUnifiedDiff(
-  diff: string,
-): ReadonlyArray<TurnDiffFileSummary> {
-  const normalized = diff.replace(/\r\n/g, "\n").trim();
-  if (normalized.length === 0) {
-    return [];
-  }
+/** Reads Git's NUL-delimited numstat output without decoding display paths. */
+export function parseTurnDiffFilesFromNumstat(numstat: string): ReadonlyArray<TurnDiffFileSummary> {
+  const records = numstat.split("\0");
+  const files: TurnDiffFileSummary[] = [];
+
+  for (let index = 0; index < records.length; index += 1) {
+    const record = records[index]!;
+    const counts = /^(\d+|-)\t(\d+|-)\t/.exec(record);
+    if (!counts) continue;
 
-  const parsedPatches = parsePatchFiles(normalized);
-  const files = parsedPatches.flatMap((patch) =>
-    patch.files.map((file) => ({
-      path: file.name,
-      additions: file.hunks.reduce((total, hunk) => total + hunk.additionLines, 0),
-      deletions: file.hunks.reduce((total, hunk) => total + hunk.deletionLines, 0),
-    })),
-  );
+    let path = record.slice(counts[0].length);
+    if (path.length === 0) {
+      // Renames and copies use two more records: the source and destination.
+      path = records[index + 2] ?? "";
+      index += 2;
+    }
+    if (path.length === 0) continue;
+
+    files.push({
+      path,
+      additions: counts[1] === "-" ? 0 : Number(counts[1]),
+      deletions: counts[2] === "-" ? 0 : Number(counts[2]),
+    });
+  }
 
   return files.toSorted((left, right) => left.path.localeCompare(right.path));
 }