From f929e9afaad2b4fc9a62ca2f419c57ec59699ad1 Mon Sep 17 00:00:00 2001
From: Jack Minnetian <270441393+BlueBottleLatte@users.noreply.github.com>
Date: Mon, 22 Jun 2026 16:51:26 -0700
Subject: [PATCH 1/8] feat(web): add layered prompt caching for the Ask agent

Adds a divergence-proof static front checkpoint (cross-chat reuse of
tool + static-system bytes) and an MCP activation-resilience breakpoint
on top of the existing moving tail marker, behind a provider-aware
resolver that is a no-op for non-Anthropic providers. Splits the system
prompt into static/dynamic blocks and hardens MCP ordering for byte
stability, all gated by new env flags.
---
 packages/shared/src/env.server.ts             |  12 +
 .../web/src/app/api/(server)/ee/chat/route.ts |  10 +
 .../web/src/ee/features/chat/agent.test.ts    | 119 +++++++++-
 packages/web/src/ee/features/chat/agent.ts    | 221 ++++++++++++------
 .../ee/features/chat/mcp/mcpClientFactory.ts  |   5 +
 .../ee/features/chat/mcp/mcpToolRegistry.ts   |  22 +-
 .../ee/features/chat/promptCaching.test.ts    |  93 ++++++++
 .../web/src/ee/features/chat/promptCaching.ts | 188 +++++++++++++++
 .../web/src/ee/features/mcp/askCodebase.ts    |   8 +
 9 files changed, 597 insertions(+), 81 deletions(-)
 create mode 100644 packages/web/src/ee/features/chat/promptCaching.test.ts
 create mode 100644 packages/web/src/ee/features/chat/promptCaching.ts

diff --git a/packages/shared/src/env.server.ts b/packages/shared/src/env.server.ts
index ee294b1db..179c3367f 100644
--- a/packages/shared/src/env.server.ts
+++ b/packages/shared/src/env.server.ts
@@ -311,6 +311,18 @@ const options = {
 
         SOURCEBOT_CHAT_MAX_STEP_COUNT: numberSchema.default(100),
         SOURCEBOT_CHAT_PROMPT_CACHING_ENABLED: booleanSchema.default('true'),
+        // Phased-rollout lever for the static front cache breakpoint (and the
+        // MCP activation-resilience breakpoint). Set to 'false' to fall back to
+        // the single moving tail marker. Only takes effect when prompt caching
+        // is enabled. See packages/web/src/ee/features/chat/promptCaching.ts.
+        SOURCEBOT_CHAT_PROMPT_CACHE_STATIC_PREFIX_ENABLED: booleanSchema.default('true'),
+        // TTL for the static front block (and MCP activation-resilience block).
+        // The moving tail marker always uses the 5m default.
+        SOURCEBOT_CHAT_PROMPT_CACHE_STATIC_TTL: z.enum(['5m', '1h']).default('5m'),
+        // Observability: when enabled, logs a warning on unexpected prompt-cache
+        // breaks (static-prefix signature changes, or zero cache reads on a
+        // continuation step). Does not affect request behavior.
+        SOURCEBOT_CHAT_PROMPT_CACHE_BREAK_DETECTION_ENABLED: booleanSchema.default('false'),
         SOURCEBOT_MCP_TOOL_CALL_TIMEOUT_MS: numberSchema.int().positive().max(maxTimerDelayMs).default(60000),
 
         DEBUG_WRITE_CHAT_MESSAGES_TO_FILE: booleanSchema.default('false'),
diff --git a/packages/web/src/app/api/(server)/ee/chat/route.ts b/packages/web/src/app/api/(server)/ee/chat/route.ts
index fcca63437..ff05467c6 100644
--- a/packages/web/src/app/api/(server)/ee/chat/route.ts
+++ b/packages/web/src/app/api/(server)/ee/chat/route.ts
@@ -1,6 +1,7 @@
 import { sew } from "@/middleware/sew";
 import { getAskMcpAvailabilityAnalytics, getAskMcpTurnCompletedAnalytics } from "@/ee/features/chat/askMcpAnalytics.server";
 import { createMessageStream } from "@/ee/features/chat/agent";
+import { getPromptCacheStrategy } from "@/ee/features/chat/promptCaching";
 import { additionalChatRequestParamsSchema } from "@/features/chat/types";
 import { getLanguageModelKey } from "@/features/chat/utils";
 import { checkAskEntitlement, getConfiguredLanguageModels, isOwnerOfChat, updateChatMessages } from "@/features/chat/utils.server";
@@ -88,6 +89,14 @@ export const POST = apiHandler(async (req: NextRequest) => {
 
             const { model, providerOptions, temperature } = await getAISDKLanguageModelAndOptions(languageModelConfig);
 
+            // Resolve provider-aware prompt caching. The strategy is a no-op for
+            // non-Anthropic providers or when caching is disabled, so threading it
+            // through never perturbs other providers' requests.
+            const promptCacheStrategy = getPromptCacheStrategy(
+                languageModelConfig.provider,
+                env.SOURCEBOT_CHAT_PROMPT_CACHING_ENABLED === 'true',
+            );
+
             const expandedRepos = (await Promise.all(selectedSearchScopes.map(async (scope) => {
                 if (scope.type === 'repo') return [scope.value];
                 if (scope.type === 'reposet') {
@@ -131,6 +140,7 @@ export const POST = apiHandler(async (req: NextRequest) => {
                 disabledMcpServerIds,
                 model,
                 modelName: languageModelConfig.displayName ?? languageModelConfig.model,
+                promptCacheStrategy,
                 modelProviderOptions: providerOptions,
                 modelTemperature: temperature,
                 userId: user?.id,
diff --git a/packages/web/src/ee/features/chat/agent.test.ts b/packages/web/src/ee/features/chat/agent.test.ts
index 5ac87c551..5bdf6bce7 100644
--- a/packages/web/src/ee/features/chat/agent.test.ts
+++ b/packages/web/src/ee/features/chat/agent.test.ts
@@ -1,6 +1,8 @@
 import { beforeEach, describe, expect, test, vi } from 'vitest';
 import type { ModelMessage } from 'ai';
+import type { ProviderOptions } from '@ai-sdk/provider-utils';
 import type { SBChatMessage, SBChatMessagePart } from '@/features/chat/types';
+import type { PromptCacheStrategy } from './promptCaching';
 
 const mockLogger = vi.hoisted(() => ({
     debug: vi.fn(),
@@ -25,6 +27,12 @@ vi.mock('@sourcebot/shared', () => ({
         SOURCEBOT_CHAT_MAX_STEP_COUNT: 8,
         SOURCEBOT_CHAT_MODEL_TEMPERATURE: 0,
         SOURCEBOT_TELEMETRY_PII_COLLECTION_ENABLED: 'false',
+        // Enable the static front checkpoint so marker placement is exercised;
+        // whether markers are actually emitted is then controlled by the
+        // PromptCacheStrategy passed into each call.
+        SOURCEBOT_CHAT_PROMPT_CACHE_STATIC_PREFIX_ENABLED: 'true',
+        SOURCEBOT_CHAT_PROMPT_CACHE_STATIC_TTL: '5m',
+        SOURCEBOT_CHAT_PROMPT_CACHE_BREAK_DETECTION_ENABLED: 'false',
     },
     getDBConnectionString: () => 'postgresql://sourcebot:sourcebot@db.example.com:5432/sourcebot',
 }));
@@ -94,6 +102,11 @@ vi.mock('ai', async (importOriginal) => {
 });
 
 const { createMessageStream } = await import('./agent');
+const { getPromptCacheStrategy } = await import('./promptCaching');
+
+// Strategies reused across the prompt-caching tests below.
+const anthropicStrategy = getPromptCacheStrategy('anthropic', true);
+const noopStrategy = getPromptCacheStrategy('openai', true);
 
 const listReposInput = {
     sort: 'name',
@@ -150,7 +163,19 @@ const createFakeStreamResult = () => ({
     }),
 });
 
-const runCreateMessageStream = async (messages: SBChatMessage[]) => {
+interface StreamTextArgs {
+    messages: ModelMessage[];
+    system: Array<{ role: 'system'; content: string; providerOptions?: ProviderOptions }>;
+    tools: Record<string, { providerOptions?: ProviderOptions }>;
+}
+
+const runCreateMessageStream = async (
+    messages: SBChatMessage[],
+    opts: {
+        promptCacheStrategy?: PromptCacheStrategy;
+        selectedRepos?: string[];
+    } = {},
+): Promise<StreamTextArgs> => {
     const convertedLastTurn: ModelMessage = {
         role: 'assistant',
         content: 'converted-last-turn',
@@ -161,10 +186,13 @@ const runCreateMessageStream = async (messages: SBChatMessage[]) => {
     const props = {
         chatId: 'chat-id',
         messages,
-        selectedRepos: [],
+        selectedRepos: opts.selectedRepos ?? [],
         prisma: {},
         model: {},
         modelName: 'test-model',
+        // Default to a no-op strategy so the approval-continuation tests below
+        // (which assert plain, unmarked messages) are unaffected by caching.
+        promptCacheStrategy: opts.promptCacheStrategy ?? noopStrategy,
         onFinish: vi.fn(),
         onError: () => 'error',
     } as unknown as Parameters<typeof createMessageStream>[0];
@@ -188,7 +216,7 @@ const runCreateMessageStream = async (messages: SBChatMessage[]) => {
         throw new Error('Expected streamText to be called with messages.');
     }
 
-    return streamTextArgs.messages as ModelMessage[];
+    return streamTextArgs as StreamTextArgs;
 };
 
 beforeEach(() => {
@@ -216,7 +244,7 @@ describe('createMessageStream approval continuation', () => {
             approvalPart,
         ]);
 
-        const streamTextMessages = await runCreateMessageStream([
+        const { messages: streamTextMessages } = await runCreateMessageStream([
             createUserMessage(),
             assistantMessage,
         ]);
@@ -249,7 +277,7 @@ describe('createMessageStream approval continuation', () => {
             dynamicApprovalRespondedPart,
         ]);
 
-        const streamTextMessages = await runCreateMessageStream([
+        const { messages: streamTextMessages } = await runCreateMessageStream([
             createUserMessage(),
             assistantMessage,
         ]);
@@ -265,3 +293,84 @@ describe('createMessageStream approval continuation', () => {
         });
     });
 });
+
+const EPHEMERAL = { type: 'ephemeral' };
+
+describe('createMessageStream prompt caching', () => {
+    test('marks the static system block and the last message for the Anthropic family', async () => {
+        const { system, messages } = await runCreateMessageStream([createUserMessage()], {
+            promptCacheStrategy: anthropicStrategy,
+        });
+
+        // No repos / files / MCP tools → only the static system block.
+        expect(system).toHaveLength(1);
+        expect(system[0].providerOptions?.anthropic?.cacheControl).toEqual(EPHEMERAL);
+
+        const lastMessage = messages.at(-1);
+        expect(lastMessage?.providerOptions?.anthropic?.cacheControl).toEqual(EPHEMERAL);
+    });
+
+    test('leaves the dynamic system block uncached', async () => {
+        const { system } = await runCreateMessageStream([createUserMessage()], {
+            promptCacheStrategy: anthropicStrategy,
+            selectedRepos: ['github.com/acme/repo'],
+        });
+
+        // Static checkpoint + dynamic (per-conversation) block.
+        expect(system).toHaveLength(2);
+        expect(system[0].providerOptions?.anthropic?.cacheControl).toEqual(EPHEMERAL);
+        expect(system[1].providerOptions).toBeUndefined();
+        expect(system[1].content).toContain('<selected_repositories>');
+    });
+
+    test('marks the tool_request_activation tool when MCP tools are present', async () => {
+        const { buildMcpToolRegistry } = await import('@/ee/features/chat/mcp/mcpToolRegistry');
+        vi.mocked(buildMcpToolRegistry).mockReturnValueOnce([
+            { name: 'mcp_linear__save_issue', description: 'Save an issue', serverName: 'linear' },
+        ]);
+
+        const { tools } = await runCreateMessageStream([createUserMessage()], {
+            promptCacheStrategy: anthropicStrategy,
+        });
+
+        expect(tools.tool_request_activation?.providerOptions?.anthropic?.cacheControl).toEqual(EPHEMERAL);
+    });
+
+    test.each([
+        ['non-Anthropic provider', () => getPromptCacheStrategy('openai', true)],
+        ['caching disabled', () => getPromptCacheStrategy('anthropic', false)],
+    ])('emits no cache markers for %s (multi-provider regression guard)', async (_label, makeStrategy) => {
+        const { buildMcpToolRegistry } = await import('@/ee/features/chat/mcp/mcpToolRegistry');
+        vi.mocked(buildMcpToolRegistry).mockReturnValueOnce([
+            { name: 'mcp_linear__save_issue', description: 'Save an issue', serverName: 'linear' },
+        ]);
+
+        const { system, messages, tools } = await runCreateMessageStream([createUserMessage()], {
+            promptCacheStrategy: makeStrategy(),
+            selectedRepos: ['github.com/acme/repo'],
+        });
+
+        for (const block of system) {
+            expect(block.providerOptions).toBeUndefined();
+        }
+        for (const message of messages) {
+            expect(message.providerOptions).toBeUndefined();
+        }
+        for (const tool of Object.values(tools)) {
+            expect(tool.providerOptions).toBeUndefined();
+        }
+    });
+
+    test('builds a byte-identical static prompt regardless of repos', async () => {
+        const first = await runCreateMessageStream([createUserMessage()], {
+            promptCacheStrategy: anthropicStrategy,
+            selectedRepos: ['github.com/acme/one'],
+        });
+        const second = await runCreateMessageStream([createUserMessage()], {
+            promptCacheStrategy: anthropicStrategy,
+            selectedRepos: ['github.com/acme/two', 'github.com/acme/three'],
+        });
+
+        expect(first.system[0].content).toBe(second.system[0].content);
+    });
+});
diff --git a/packages/web/src/ee/features/chat/agent.ts b/packages/web/src/ee/features/chat/agent.ts
index c415e80df..77b0a126b 100644
--- a/packages/web/src/ee/features/chat/agent.ts
+++ b/packages/web/src/ee/features/chat/agent.ts
@@ -9,6 +9,7 @@ import { createLogger, env } from "@sourcebot/shared";
 import {
     convertToModelMessages,
     createUIMessageStream, JSONValue, LanguageModel, ModelMessage, StopCondition, streamText, StreamTextResult,
+    SystemModelMessage,
     UIMessageStreamOnFinishCallback,
     UIMessageStreamOptions,
     UIMessageStreamWriter,
@@ -26,6 +27,7 @@ import { createTools } from "./tools";
 import { getConnectedMcpClients } from "@/ee/features/chat/mcp/mcpClientFactory";
 import { getMcpTools, McpToolsResult } from "@/ee/features/chat/mcp/mcpToolSets";
 import { buildMcpToolRegistry, McpToolRegistryEntry, searchMcpTools } from "@/ee/features/chat/mcp/mcpToolRegistry";
+import { PromptCacheStrategy, mergeProviderOptions, detectPromptCacheBreak, detectUnexpectedCacheMiss } from "./promptCaching";
 import { hasEntitlement } from '@/lib/entitlements';
 
 const dedent = _dedent.withOptions({ alignValues: true });
@@ -52,6 +54,7 @@ interface CreateMessageStreamResponseProps {
     disabledMcpServerIds?: string[];
     model: AISDKLanguageModelV3;
     modelName: string;
+    promptCacheStrategy: PromptCacheStrategy;
     onFinish: UIMessageStreamOnFinishCallback<SBChatMessage>;
     onError: (error: unknown) => string;
     modelProviderOptions?: Record<string, Record<string, JSONValue>>;
@@ -70,6 +73,7 @@ export const createMessageStream = async ({
     disabledMcpServerIds,
     model,
     modelName,
+    promptCacheStrategy,
     modelProviderOptions,
     modelTemperature,
     onFinish,
@@ -152,6 +156,7 @@ export const createMessageStream = async ({
 
             const researchStream = await createAgentStream({
                 model,
+                promptCacheStrategy,
                 providerOptions: modelProviderOptions,
                 temperature: modelTemperature,
                 inputMessages: messageHistory,
@@ -245,6 +250,20 @@ export const createMessageStream = async ({
                 stepTokenUsage[0].tools.unshift(...toolUsageByToolCallId.values());
             }
 
+            // Observability only (default off): warn when a continuation step
+            // reports zero cache reads while the provider supports breakpoints —
+            // a likely byte-stability regression in the cached prefix.
+            if (env.SOURCEBOT_CHAT_PROMPT_CACHE_BREAK_DETECTION_ENABLED === 'true') {
+                steps.forEach((step, stepIndex) => {
+                    detectUnexpectedCacheMiss({
+                        chatId,
+                        stepIndex,
+                        cacheReadTokens: step.usage.inputTokenDetails?.cacheReadTokens,
+                        supportsBreakpoints: promptCacheStrategy.supportsBreakpoints,
+                    });
+                });
+            }
+
             writer.write({
                 type: 'message-metadata',
                 messageMetadata: {
@@ -278,6 +297,7 @@ export const createMessageStream = async ({
 
 interface AgentOptions {
     model: LanguageModel;
+    promptCacheStrategy: PromptCacheStrategy;
     providerOptions?: ProviderOptions;
     temperature?: number;
     selectedRepos: string[];
@@ -296,6 +316,7 @@ interface AgentOptions {
 
 const createAgentStream = async ({
     model,
+    promptCacheStrategy,
     providerOptions,
     temperature,
     inputMessages,
@@ -311,6 +332,10 @@ const createAgentStream = async ({
     userId,
     orgId,
 }: AgentOptions) => {
+    // Normalize repo order so the dynamic system block's <selected_repositories>
+    // list is byte-stable across requests of the same chat, used for prompt caching.
+    const sortedRepos = [...selectedRepos].sort((a, b) => a.localeCompare(b));
+
     // For every file source, resolve the source code so that we can include it in the system prompt.
     const fileSources = inputSources.filter((source) => source.type === 'file');
     const resolvedFileSources = (
@@ -366,6 +391,22 @@ const createAgentStream = async ({
     const mcpRegistry = buildMcpToolRegistry(mcpToolSetsObj.tools);
     const hasMcpTools = mcpRegistry.length > 0;
 
+    // Phased-rollout lever for the static front checkpoint (markers 1 & 2). When
+    // disabled, only the moving tail marker is emitted and behavior collapses to
+    // the prior single-breakpoint scheme.
+    const useStaticPrefix = env.SOURCEBOT_CHAT_PROMPT_CACHE_STATIC_PREFIX_ENABLED === 'true';
+    const staticTtl = env.SOURCEBOT_CHAT_PROMPT_CACHE_STATIC_TTL;
+
+    // Marker 1 (MCP only): cache the byte-stable built-in tools + activation tool
+    // block on the `tool_request_activation` definition (the last always-active
+    // tool, serialized before the dynamic MCP tools). This survives the cache
+    // bust that mid-run `activeTools` growth causes for markers 2 & 3. When no
+    // MCP tools are present, marker 2 already covers the built-in tools, so this
+    // is skipped.
+    const activationToolMarker = (hasMcpTools && useStaticPrefix)
+        ? promptCacheStrategy.cacheControl({ ttl: staticTtl })
+        : undefined;
+
     const toolRequestActivation = tool({
         description: dedent`
         Activate an MCP tool by name so it becomes callable on your next step.
@@ -382,6 +423,7 @@ const createAgentStream = async ({
         inputSchema: z.object({
             tool_to_activate_name: z.string().describe('Exact tool name from the registry, e.g. "mcp_linear__save_comment"'),
         }),
+        providerOptions: activationToolMarker,
         execute: async ({ tool_to_activate_name }) => {
             const results = searchMcpTools(tool_to_activate_name, mcpRegistry);
             return {
@@ -390,56 +432,83 @@ const createAgentStream = async ({
         },
     });
 
-    const systemPrompt = createPrompt({
-        repos: selectedRepos,
+    const { staticPrompt, dynamicPrompt } = createPrompt({
+        repos: sortedRepos,
         files: resolvedFileSources,
         mcpToolRegistry: mcpRegistry,
     });
 
-    const builtinTools = createTools({ source: 'sourcebot-ask-agent', selectedRepos });
+    const builtinTools = createTools({ source: 'sourcebot-ask-agent', selectedRepos: sortedRepos });
     const builtinToolNames = Object.keys(builtinTools);
     const allTools: Record<string, Tool> = {
         ...builtinTools,
         ...(hasMcpTools ? { tool_request_activation: toolRequestActivation, ...mcpToolSetsObj.tools } : {}),
     };
 
-    // Anthropic prompt caching: mark the end of the prompt's static prefix —
-    // tool definitions, the system prompt (including any resolved file sources),
-    // and the conversation history — with an ephemeral (5m) cache breakpoint on
-    // the last input message. Anthropic caches everything up to and including
-    // this point, so the large prefix is written once (~1.25x) and read back at
-    // ~0.1x on every subsequent agent step and follow-up turn instead of being
-    // reprocessed in full. The `anthropic` provider-options namespace is ignored
-    // by non-Anthropic providers, so this is safe to apply unconditionally.
+    // Anthropic prompt caching uses up to three nested breakpoints over one
+    // cumulative prefix (render order: tools -> system -> messages):
+    //
+    //   Marker 1 (MCP only): the `tool_request_activation` tool definition (set
+    //     above) caches the byte-stable built-in + activation tool block.
+    //   Marker 2: the static system block below caches tools + the static system
+    //     instructions. This block is byte-identical across every chat and user,
+    //     so it is a divergence-proof checkpoint a brand-new chat can read from
+    //     instead of re-writing the large static prefix.
+    //   Marker 3: the last input message (below) caches tools + static + dynamic
+    //     system + the full conversation history — one conversation's growing
+    //     delta, re-warmed cheaply on every step and follow-up.
+    //
+    // The `anthropic` provider-options namespace is ignored by non-Anthropic
+    // providers, and a no-op strategy emits no markers at all, so this is safe
+    // for every provider. When the static prefix falls below the model's minimum
+    // cacheable size the marker is a harmless no-op.
     //
     // Caveat: when MCP tools are lazily activated mid-run via prepareStep, the
-    // tools section (which precedes everything else in the prefix) grows and
-    // invalidates the cache for that step; the cache re-warms on subsequent
-    // steps once the active tool set is stable.
-    const isPromptCachingEnabled = env.SOURCEBOT_CHAT_PROMPT_CACHING_ENABLED === 'true';
-    const messagesWithCachedPrefix: ModelMessage[] = inputMessages.map((message, index) => {
-        if (!isPromptCachingEnabled || index !== inputMessages.length - 1) {
-            return message;
-        }
+    // tools section grows and invalidates markers 2 & 3 for that step; marker 1
+    // preserves the built-in/activation block and the cache re-warms on
+    // subsequent steps once the active tool set is stable.
+    const staticMarker = useStaticPrefix
+        ? promptCacheStrategy.cacheControl({ ttl: staticTtl })
+        : undefined;
+    const systemMessages: SystemModelMessage[] = [
+        { role: 'system', content: staticPrompt, providerOptions: staticMarker },
+    ];
+    if (dynamicPrompt) {
+        systemMessages.push({ role: 'system', content: dynamicPrompt });
+    }
 
-        return {
-            ...message,
-            providerOptions: {
-                ...message.providerOptions,
-                anthropic: {
-                    ...message.providerOptions?.anthropic,
-                    cacheControl: { type: 'ephemeral' },
-                },
-            },
-        };
-    });
+    // Marker 3: ephemeral (5m) breakpoint on the last input message. Merged onto
+    // any existing providerOptions so sibling namespaces (e.g. anthropic.thinking)
+    // are preserved. A no-op strategy returns undefined and leaves messages as-is.
+    const tailMarker = promptCacheStrategy.cacheControl();
+    const messagesWithCachedPrefix: ModelMessage[] = tailMarker
+        ? inputMessages.map((message, index) =>
+            index === inputMessages.length - 1
+                ? { ...message, providerOptions: mergeProviderOptions(message.providerOptions, tailMarker) }
+                : message)
+        : inputMessages;
+
+    // Observability only (default off): warn when the cache-relevant static
+    // prefix (static system prompt, built-in tool definitions, model, or TTL)
+    // changes between requests for the same chat in a way that busts the cache.
+    if (env.SOURCEBOT_CHAT_PROMPT_CACHE_BREAK_DETECTION_ENABLED === 'true') {
+        detectPromptCacheBreak({
+            chatId,
+            staticPrompt,
+            toolSignature: Object.entries(builtinTools)
+                .map(([name, builtinTool]) => `${name}:${builtinTool.description ?? ''}`)
+                .join('|'),
+            model: typeof model === 'string' ? model : model.modelId,
+            staticTtl,
+        });
+    }
 
     try {
         const stream = streamText({
             model,
             providerOptions,
             messages: messagesWithCachedPrefix,
-            system: systemPrompt,
+            system: systemMessages,
             tools: allTools,
             activeTools: [
                 ...builtinToolNames,
@@ -544,8 +613,13 @@ const createPrompt = ({
     }[],
     repos: string[],
     mcpToolRegistry: McpToolRegistryEntry[],
-}) => {
-    return dedent`
+}): { staticPrompt: string; dynamicPrompt: string } => {
+    // Static prefix: byte-identical across every chat and user, so Anthropic can
+    // reuse its cache entry across chats (the divergence-proof checkpoint). It
+    // interpolates only module-level constants. Keep it free of any
+    // per-conversation data — repos, files, and MCP tools live in the dynamic
+    // block below so their volatility never busts the shared static cache.
+    const staticPrompt = dedent`
     You are a powerful agentic AI code assistant built into Sourcebot, the world's best code-intelligence platform. Your job is to help developers understand and navigate their large codebases.
 
     <workflow>
@@ -567,39 +641,6 @@ const createPrompt = ({
     During the research phase, use the tools available to you to gather comprehensive context before answering. Always explain why you're using each tool. Depending on the user's question, you may need to use multiple tools. If the question is vague, ask the user for more information.
     </research_phase_instructions>
 
-    ${repos.length > 0 ? dedent`
-        <selected_repositories>
-        The user has explicitly selected the following repositories for analysis:
-        ${repos.map(repo => `- ${repo}`).join('\n')}
-
-        When calling tools that accept a \`repo\` parameter (e.g. \`read_file\`, \`list_commits\`, \`list_tree\`, \`get_diff\`, \`grep\`), use these repository names exactly as listed above, including the full host prefix (e.g. \`github.com/org/repo\`).
-
-        When using \`grep\` to search across ALL selected repositories (e.g. "which repos have X?"), omit the \`repo\` parameter entirely — the tool will automatically search across all selected repositories in a single call. Do NOT call \`grep\` once per repository when a single broad search would suffice.
-        </selected_repositories>
-    ` : ''}
-
-    ${(files && files.length > 0) ? dedent`
-        <files>
-        The user has mentioned the following files, which are automatically included for analysis.
-
-        ${files?.map(file => `<file path="${file.path}" repository="${file.repo}" language="${file.language}" revision="${file.revision}">
-            ${addLineNumbers(file.source)}
-            </file>`).join('\n\n')}
-        </files>
-    `: ''}
-
-    ${(mcpToolRegistry.length > 0) ? dedent`
-        <mcp_tools>
-        External MCP tools are available but must first be activated via \`tool_request_activation\`.
-
-        **CRITICAL**: The list below is the complete and authoritative inventory of all tools available to you:
-        ${mcpToolRegistry.map(e => `- ${e.name}: ${e.description}`).join('\n')}
-
-        **How to use tool_request_activation**: Pass the exact tool name from the list above as the \`tool_to_activate_name\` parameter. Do NOT pass natural language descriptions or sentences. If you need multiple tools, call \`tool_request_activation\` once per tool.
-        Example: to activate the comment tool, call \`tool_request_activation\` with tool_to_activate_name="mcp_linear__save_comment", NOT tool_to_activate_name="save a comment on an issue".
-        </mcp_tools>
-    ` : ''}
-
     <answer_instructions>
     When you have sufficient context, output your answer as a structured markdown response.
 
@@ -630,7 +671,53 @@ const createPrompt = ({
     Authentication in Sourcebot is built on NextAuth.js with a session-based approach using JWT tokens and Prisma as the database adapter ${fileReferenceToString({ repo: 'github.com/sourcebot-dev/sourcebot', path: 'auth.ts', range: { startLine: 135, endLine: 140 } })}. The system supports multiple authentication providers and implements organization-based authorization with role-defined permissions.
     \`\`\`
     </answer_instructions>
-    `
+    `;
+
+    // Dynamic block: per-conversation context (selected repos, mentioned files,
+    // MCP tool registry). Placed after the static checkpoint so changes here
+    // never invalidate the cross-chat static cache. Empty string when none apply.
+    const dynamicSections: string[] = [];
+
+    if (repos.length > 0) {
+        dynamicSections.push(dedent`
+        <selected_repositories>
+        The user has explicitly selected the following repositories for analysis:
+        ${repos.map(repo => `- ${repo}`).join('\n')}
+
+        When calling tools that accept a \`repo\` parameter (e.g. \`read_file\`, \`list_commits\`, \`list_tree\`, \`get_diff\`, \`grep\`), use these repository names exactly as listed above, including the full host prefix (e.g. \`github.com/org/repo\`).
+
+        When using \`grep\` to search across ALL selected repositories (e.g. "which repos have X?"), omit the \`repo\` parameter entirely — the tool will automatically search across all selected repositories in a single call. Do NOT call \`grep\` once per repository when a single broad search would suffice.
+        </selected_repositories>
+        `);
+    }
+
+    if (files && files.length > 0) {
+        dynamicSections.push(dedent`
+        <files>
+        The user has mentioned the following files, which are automatically included for analysis.
+
+        ${files.map(file => `<file path="${file.path}" repository="${file.repo}" language="${file.language}" revision="${file.revision}">
+            ${addLineNumbers(file.source)}
+            </file>`).join('\n\n')}
+        </files>
+        `);
+    }
+
+    if (mcpToolRegistry.length > 0) {
+        dynamicSections.push(dedent`
+        <mcp_tools>
+        External MCP tools are available but must first be activated via \`tool_request_activation\`.
+
+        **CRITICAL**: The list below is the complete and authoritative inventory of all tools available to you:
+        ${mcpToolRegistry.map(e => `- ${e.name}: ${e.description}`).join('\n')}
+
+        **How to use tool_request_activation**: Pass the exact tool name from the list above as the \`tool_to_activate_name\` parameter. Do NOT pass natural language descriptions or sentences. If you need multiple tools, call \`tool_request_activation\` once per tool.
+        Example: to activate the comment tool, call \`tool_request_activation\` with tool_to_activate_name="mcp_linear__save_comment", NOT tool_to_activate_name="save a comment on an issue".
+        </mcp_tools>
+        `);
+    }
+
+    return { staticPrompt, dynamicPrompt: dynamicSections.join('\n\n') };
 }
 
 // If the agent exceeds the step count, then we will stop.
diff --git a/packages/web/src/ee/features/chat/mcp/mcpClientFactory.ts b/packages/web/src/ee/features/chat/mcp/mcpClientFactory.ts
index 58b813101..6dfdfcbfa 100644
--- a/packages/web/src/ee/features/chat/mcp/mcpClientFactory.ts
+++ b/packages/web/src/ee/features/chat/mcp/mcpClientFactory.ts
@@ -36,6 +36,11 @@ export async function getConnectedMcpClients(prisma: PrismaClient, userId: strin
                 clientInfo: { not: null },
             },
         },
+        // Deterministic ordering is required for prompt caching: the MCP server
+        // order flows into both the serialized tool definitions and the
+        // <mcp_tools> system-prompt block, and an unordered query would let the
+        // byte layout drift between requests and bust the cache.
+        orderBy: { serverId: 'asc' },
         select: {
             serverId: true,
             tokens: true,
diff --git a/packages/web/src/ee/features/chat/mcp/mcpToolRegistry.ts b/packages/web/src/ee/features/chat/mcp/mcpToolRegistry.ts
index 431710e9e..ead4f2be0 100644
--- a/packages/web/src/ee/features/chat/mcp/mcpToolRegistry.ts
+++ b/packages/web/src/ee/features/chat/mcp/mcpToolRegistry.ts
@@ -49,15 +49,19 @@ function expandTokens(tokens: string[]): string[] {
 }
 
 export function buildMcpToolRegistry(tools: McpToolRecord): McpToolRegistryEntry[] {
-    return Object.entries(tools).map(([name, tool]) => {
-        const match = name.match(/^mcp_(.+?)__/);
-        const serverName = match ? match[1] : '';
-        return {
-            name,
-            description: tool.description ?? '',
-            serverName,
-        };
-    });
+    return Object.entries(tools)
+        // Sort by tool name so the <mcp_tools> system-prompt block is byte-stable
+        // across requests regardless of upstream iteration order (prompt caching).
+        .sort(([nameA], [nameB]) => nameA.localeCompare(nameB))
+        .map(([name, tool]) => {
+            const match = name.match(/^mcp_(.+?)__/);
+            const serverName = match ? match[1] : '';
+            return {
+                name,
+                description: tool.description ?? '',
+                serverName,
+            };
+        });
 }
 
 export function searchMcpTools(
diff --git a/packages/web/src/ee/features/chat/promptCaching.test.ts b/packages/web/src/ee/features/chat/promptCaching.test.ts
new file mode 100644
index 000000000..69dd96ef4
--- /dev/null
+++ b/packages/web/src/ee/features/chat/promptCaching.test.ts
@@ -0,0 +1,93 @@
+import { describe, expect, test, vi } from 'vitest';
+
+vi.mock('@sourcebot/shared', () => ({
+    createLogger: () => ({
+        debug: vi.fn(),
+        error: vi.fn(),
+        info: vi.fn(),
+        warn: vi.fn(),
+    }),
+}));
+
+const { getPromptCacheStrategy, mergeProviderOptions } = await import('./promptCaching');
+
+describe('getPromptCacheStrategy', () => {
+    test.each(['anthropic', 'google-vertex-anthropic'] as const)(
+        'returns a 5m ephemeral marker (no ttl) for %s when enabled',
+        (provider) => {
+            const strategy = getPromptCacheStrategy(provider, true);
+            expect(strategy.supportsBreakpoints).toBe(true);
+            expect(strategy.cacheControl()).toEqual({
+                anthropic: { cacheControl: { type: 'ephemeral' } },
+            });
+        },
+    );
+
+    test('includes ttl only for the 1h bucket', () => {
+        const strategy = getPromptCacheStrategy('anthropic', true);
+        expect(strategy.cacheControl({ ttl: '1h' })).toEqual({
+            anthropic: { cacheControl: { type: 'ephemeral', ttl: '1h' } },
+        });
+        // 5m is the default — ttl is omitted so the request matches Anthropic's
+        // default and stays byte-stable.
+        expect(strategy.cacheControl({ ttl: '5m' })).toEqual({
+            anthropic: { cacheControl: { type: 'ephemeral' } },
+        });
+    });
+
+    test.each([
+        'amazon-bedrock',
+        'azure',
+        'deepseek',
+        'google-generative-ai',
+        'google-vertex',
+        'mistral',
+        'openai',
+        'openai-compatible',
+        'openrouter',
+        'xai',
+    ] as const)('is a no-op for non-Anthropic provider %s', (provider) => {
+        const strategy = getPromptCacheStrategy(provider, true);
+        expect(strategy.supportsBreakpoints).toBe(false);
+        expect(strategy.cacheControl()).toBeUndefined();
+        expect(strategy.cacheControl({ ttl: '1h' })).toBeUndefined();
+    });
+
+    test('is a no-op when caching is disabled, even for Anthropic', () => {
+        const strategy = getPromptCacheStrategy('anthropic', false);
+        expect(strategy.supportsBreakpoints).toBe(false);
+        expect(strategy.cacheControl()).toBeUndefined();
+    });
+});
+
+describe('mergeProviderOptions', () => {
+    test('returns the existing options unchanged when there is no marker', () => {
+        const existing = { anthropic: { thinking: { type: 'enabled' } } };
+        expect(mergeProviderOptions(existing, undefined)).toBe(existing);
+    });
+
+    test('returns the marker when there are no existing options', () => {
+        const marker = { anthropic: { cacheControl: { type: 'ephemeral' } } };
+        expect(mergeProviderOptions(undefined, marker)).toBe(marker);
+    });
+
+    test('deep-merges the cacheControl while preserving sibling options in the same namespace', () => {
+        const existing = { anthropic: { thinking: { type: 'enabled', budgetTokens: 1024 } } };
+        const marker = { anthropic: { cacheControl: { type: 'ephemeral' } } };
+        expect(mergeProviderOptions(existing, marker)).toEqual({
+            anthropic: {
+                thinking: { type: 'enabled', budgetTokens: 1024 },
+                cacheControl: { type: 'ephemeral' },
+            },
+        });
+    });
+
+    test('preserves sibling provider namespaces', () => {
+        const existing = { openai: { someOption: true } };
+        const marker = { anthropic: { cacheControl: { type: 'ephemeral' } } };
+        expect(mergeProviderOptions(existing, marker)).toEqual({
+            openai: { someOption: true },
+            anthropic: { cacheControl: { type: 'ephemeral' } },
+        });
+    });
+});
diff --git a/packages/web/src/ee/features/chat/promptCaching.ts b/packages/web/src/ee/features/chat/promptCaching.ts
new file mode 100644
index 000000000..83646a9a4
--- /dev/null
+++ b/packages/web/src/ee/features/chat/promptCaching.ts
@@ -0,0 +1,188 @@
+import { ProviderOptions } from "@ai-sdk/provider-utils";
+import { createLogger } from "@sourcebot/shared";
+import { LanguageModelProvider } from "@/features/chat/types";
+
+// @note: Prompt-cache plumbing for the Ask agent. Ask Sourcebot is an
+// enterprise feature, so this lives under `ee/`. It is the single place that
+// knows about provider-specific cache-control shapes — the agent loop stays
+// provider-agnostic by asking the strategy for a `providerOptions` blob at each
+// breakpoint and merging whatever it gets back (possibly `undefined`).
+
+const logger = createLogger('prompt-caching');
+
+export type CacheTtl = '5m' | '1h';
+
+export interface PromptCacheStrategy {
+    /**
+     * Whether the resolved provider supports explicit cache breakpoints. When
+     * false, every `cacheControl()` call is a no-op and the request is left
+     * untouched (non-Anthropic providers, or caching disabled).
+     */
+    readonly supportsBreakpoints: boolean;
+    /**
+     * Returns the `providerOptions` blob to merge onto a system block, tool
+     * definition, or message to mark a cache breakpoint — or `undefined` when
+     * caching is disabled or the provider does not support explicit breakpoints.
+     */
+    cacheControl: (opts?: { ttl?: CacheTtl }) => ProviderOptions | undefined;
+}
+
+const NOOP_STRATEGY: PromptCacheStrategy = {
+    supportsBreakpoints: false,
+    cacheControl: () => undefined,
+};
+
+// Providers whose `@ai-sdk/*` package consumes the `anthropic` providerOptions
+// namespace and honors ephemeral `cacheControl` breakpoints. `google-vertex-anthropic`
+// reuses the same namespace; if a given SDK version ignores it the marker is a
+// harmless no-op. Bedrock uses a different shape (`cachePoint`) and is intentionally
+// not covered here — adding it later is a single new branch.
+const ANTHROPIC_FAMILY_PROVIDERS: ReadonlySet<LanguageModelProvider> = new Set<LanguageModelProvider>([
+    'anthropic',
+    'google-vertex-anthropic',
+]);
+
+const anthropicCacheControl = ({ ttl }: { ttl?: CacheTtl } = {}): ProviderOptions => ({
+    anthropic: {
+        cacheControl: {
+            type: 'ephemeral',
+            // 5m is the Anthropic default; only emit `ttl` for the 1h bucket.
+            ...(ttl === '1h' ? { ttl: '1h' } : {}),
+        },
+    },
+});
+
+/**
+ * Resolves how (and whether) to emit prompt-cache breakpoints for a provider.
+ * Non-Anthropic providers, or a disabled master flag, yield a no-op strategy so
+ * their requests are never perturbed.
+ */
+export const getPromptCacheStrategy = (
+    provider: LanguageModelProvider,
+    enabled: boolean,
+): PromptCacheStrategy => {
+    if (!enabled || !ANTHROPIC_FAMILY_PROVIDERS.has(provider)) {
+        return NOOP_STRATEGY;
+    }
+
+    return {
+        supportsBreakpoints: true,
+        cacheControl: (opts) => anthropicCacheControl(opts),
+    };
+};
+
+/**
+ * Deep-merges a cache-control marker's `providerOptions` onto any existing
+ * `providerOptions`, preserving sibling provider namespaces and other options
+ * within the same namespace (e.g. `anthropic.thinking`). Returns the original
+ * value unchanged when there is no marker to apply.
+ */
+export const mergeProviderOptions = (
+    existing: ProviderOptions | undefined,
+    marker: ProviderOptions | undefined,
+): ProviderOptions | undefined => {
+    if (!marker) {
+        return existing;
+    }
+    if (!existing) {
+        return marker;
+    }
+
+    const merged: ProviderOptions = { ...existing };
+    for (const [namespace, options] of Object.entries(marker)) {
+        merged[namespace] = {
+            ...(existing[namespace] ?? {}),
+            ...options,
+        };
+    }
+    return merged;
+};
+
+// --- Cache-break detection (observability only; never throws) ---------------
+
+interface CacheBreakSnapshot {
+    signature: string;
+    requestCount: number;
+}
+
+// Keyed by chatId. In-memory, best-effort: survives within a server process and
+// is naturally bounded by the number of concurrent chats. Mirrors the in-memory
+// caching pattern used elsewhere (e.g. `anthropicThinkingConfigCache`).
+const cacheBreakSnapshots = new Map<string, CacheBreakSnapshot>();
+
+// Lightweight, stable, non-cryptographic hash (djb2). Observability only.
+const hashString = (input: string): string => {
+    let hash = 5381;
+    for (let i = 0; i < input.length; i++) {
+        hash = ((hash << 5) + hash + input.charCodeAt(i)) | 0;
+    }
+    return (hash >>> 0).toString(36);
+};
+
+/**
+ * Records the cache-relevant prefix signature for a chat and logs a warning when
+ * it changes between requests in a way that would invalidate the static prompt
+ * cache. Observability only — never affects request behavior.
+ */
+export const detectPromptCacheBreak = ({
+    chatId,
+    staticPrompt,
+    toolSignature,
+    model,
+    staticTtl,
+}: {
+    chatId: string;
+    staticPrompt: string;
+    toolSignature: string;
+    model: string;
+    staticTtl: CacheTtl;
+}): void => {
+    try {
+        const signature = hashString([
+            model,
+            staticTtl,
+            hashString(staticPrompt),
+            hashString(toolSignature),
+        ].join('|'));
+
+        const prev = cacheBreakSnapshots.get(chatId);
+        const requestCount = (prev?.requestCount ?? 0) + 1;
+        cacheBreakSnapshots.set(chatId, { signature, requestCount });
+
+        if (prev && prev.signature !== signature) {
+            logger.warn(
+                `Prompt cache break detected for chat ${chatId} (request #${requestCount}): the ` +
+                `static system prompt, built-in tool definitions, model, or TTL changed between ` +
+                `requests, invalidating the static cache prefix.`,
+            );
+        }
+    } catch (error) {
+        logger.debug(`detectPromptCacheBreak failed: ${error}`);
+    }
+};
+
+/**
+ * Logs a warning when a non-first agent step reports zero cache-read tokens while
+ * caching is supported — a likely byte-stability regression. Observability only.
+ */
+export const detectUnexpectedCacheMiss = ({
+    chatId,
+    stepIndex,
+    cacheReadTokens,
+    supportsBreakpoints,
+}: {
+    chatId: string;
+    stepIndex: number;
+    cacheReadTokens: number | undefined;
+    supportsBreakpoints: boolean;
+}): void => {
+    if (!supportsBreakpoints || stepIndex === 0) {
+        return;
+    }
+    if ((cacheReadTokens ?? 0) === 0) {
+        logger.warn(
+            `Prompt cache miss for chat ${chatId} at step ${stepIndex}: cacheReadTokens=0 on a ` +
+            `continuation step where a cache hit was expected (likely a byte-stability regression).`,
+        );
+    }
+};
diff --git a/packages/web/src/ee/features/mcp/askCodebase.ts b/packages/web/src/ee/features/mcp/askCodebase.ts
index 9eaaf3fc2..4b7cfb7b0 100644
--- a/packages/web/src/ee/features/mcp/askCodebase.ts
+++ b/packages/web/src/ee/features/mcp/askCodebase.ts
@@ -15,6 +15,7 @@ import { InferUIMessageChunk, UIDataTypes, UIMessage, UITools } from "ai";
 import { captureEvent } from "@/lib/posthog";
 import { createAudit } from "@/ee/features/audit/audit";
 import { createMessageStream } from "@/ee/features/chat/agent";
+import { getPromptCacheStrategy } from "@/ee/features/chat/promptCaching";
 
 const logger = createLogger('ask-codebase-api');
 
@@ -84,6 +85,12 @@ export const askCodebase = (params: AskCodebaseParams): Promise<AskCodebaseResul
             const { model, providerOptions, temperature } = await getAISDKLanguageModelAndOptions(languageModelConfig);
             const modelName = languageModelConfig.displayName ?? languageModelConfig.model;
 
+            // No-op for non-Anthropic providers / when caching is disabled.
+            const promptCacheStrategy = getPromptCacheStrategy(
+                languageModelConfig.provider,
+                env.SOURCEBOT_CHAT_PROMPT_CACHING_ENABLED === 'true',
+            );
+
             const chatVisibility = (requestedVisibility && user)
                 ? requestedVisibility
                 : (user ? ChatVisibility.PRIVATE : ChatVisibility.PUBLIC);
@@ -175,6 +182,7 @@ export const askCodebase = (params: AskCodebaseParams): Promise<AskCodebaseResul
                 prisma,
                 model,
                 modelName,
+                promptCacheStrategy,
                 modelProviderOptions: providerOptions,
                 modelTemperature: temperature,
                 onFinish: async ({ messages }) => {

From 8c9142ed9b7edc0837a281b58f14c42bd950d59d Mon Sep 17 00:00:00 2001
From: Jack Minnetian <270441393+BlueBottleLatte@users.noreply.github.com>
Date: Mon, 22 Jun 2026 18:27:16 -0700
Subject: [PATCH 2/8] feat(web): advance the prompt-cache tail marker per agent
 step

The moving tail breakpoint was set once on the last input message before
streamText's loop, so a turn's tool calls and outputs accumulated past it
and were reprocessed uncached on each later step. Apply it in prepareStep
to the last message of every step instead, caching the growing in-turn
delta incrementally. prepareStep now runs without MCP too, and stays a
no-op for non-Anthropic providers.
---
 .../web/src/ee/features/chat/agent.test.ts    | 79 ++++++++++++++++++-
 packages/web/src/ee/features/chat/agent.ts    | 52 ++++++++----
 2 files changed, 111 insertions(+), 20 deletions(-)

diff --git a/packages/web/src/ee/features/chat/agent.test.ts b/packages/web/src/ee/features/chat/agent.test.ts
index 5bdf6bce7..01dbfa8c0 100644
--- a/packages/web/src/ee/features/chat/agent.test.ts
+++ b/packages/web/src/ee/features/chat/agent.test.ts
@@ -163,10 +163,20 @@ const createFakeStreamResult = () => ({
     }),
 });
 
+type FakePrepareStep = (opts: {
+    steps: Array<{ toolResults: Array<{ toolName: string; output: unknown }> }>;
+    stepNumber: number;
+    model: unknown;
+    messages: ModelMessage[];
+}) =>
+    | { messages?: ModelMessage[]; activeTools?: string[] }
+    | Promise<{ messages?: ModelMessage[]; activeTools?: string[] }>;
+
 interface StreamTextArgs {
     messages: ModelMessage[];
     system: Array<{ role: 'system'; content: string; providerOptions?: ProviderOptions }>;
     tools: Record<string, { providerOptions?: ProviderOptions }>;
+    prepareStep?: FakePrepareStep;
 }
 
 const runCreateMessageStream = async (
@@ -297,7 +307,7 @@ describe('createMessageStream approval continuation', () => {
 const EPHEMERAL = { type: 'ephemeral' };
 
 describe('createMessageStream prompt caching', () => {
-    test('marks the static system block and the last message for the Anthropic family', async () => {
+    test('marks the static system block for the Anthropic family', async () => {
         const { system, messages } = await runCreateMessageStream([createUserMessage()], {
             promptCacheStrategy: anthropicStrategy,
         });
@@ -306,8 +316,71 @@ describe('createMessageStream prompt caching', () => {
         expect(system).toHaveLength(1);
         expect(system[0].providerOptions?.anthropic?.cacheControl).toEqual(EPHEMERAL);
 
-        const lastMessage = messages.at(-1);
-        expect(lastMessage?.providerOptions?.anthropic?.cacheControl).toEqual(EPHEMERAL);
+        // The tail marker is applied per-step in prepareStep, not on the messages
+        // handed to streamText — those stay unmarked.
+        for (const message of messages) {
+            expect(message.providerOptions).toBeUndefined();
+        }
+    });
+
+    test('moves the tail marker onto the last message of each step via prepareStep', async () => {
+        const { prepareStep } = await runCreateMessageStream([createUserMessage()], {
+            promptCacheStrategy: anthropicStrategy,
+        });
+        expect(prepareStep).toBeTypeOf('function');
+
+        // Step 0: a single input message → marker lands on it.
+        const step0 = await prepareStep!({
+            steps: [],
+            stepNumber: 0,
+            model: {},
+            messages: [{ role: 'user', content: 'q' }],
+        });
+        expect(step0.messages?.at(-1)?.providerOptions?.anthropic?.cacheControl).toEqual(EPHEMERAL);
+
+        // Continuation step: the marker rides the NEW last message, and only it —
+        // earlier messages (including the prior tail) carry no marker.
+        const stepN = await prepareStep!({
+            steps: [],
+            stepNumber: 1,
+            model: {},
+            messages: [
+                { role: 'user', content: 'q' },
+                { role: 'assistant', content: 'searching' },
+                { role: 'assistant', content: 'tool output' },
+            ],
+        });
+        const out = stepN.messages!;
+        expect(out[0].providerOptions?.anthropic?.cacheControl).toBeUndefined();
+        expect(out[1].providerOptions?.anthropic?.cacheControl).toBeUndefined();
+        expect(out.at(-1)?.providerOptions?.anthropic?.cacheControl).toEqual(EPHEMERAL);
+    });
+
+    test('prepareStep adds no tail marker for non-Anthropic providers', async () => {
+        // Force MCP so prepareStep exists even without a tail marker.
+        const { buildMcpToolRegistry } = await import('@/ee/features/chat/mcp/mcpToolRegistry');
+        vi.mocked(buildMcpToolRegistry).mockReturnValueOnce([
+            { name: 'mcp_linear__save_issue', description: 'Save an issue', serverName: 'linear' },
+        ]);
+
+        const { prepareStep } = await runCreateMessageStream([createUserMessage()], {
+            promptCacheStrategy: noopStrategy,
+        });
+        expect(prepareStep).toBeTypeOf('function');
+
+        const result = await prepareStep!({
+            steps: [],
+            stepNumber: 1,
+            model: {},
+            messages: [
+                { role: 'user', content: 'q' },
+                { role: 'assistant', content: 'a' },
+            ],
+        });
+
+        // activeTools still managed (MCP), but no message override / marker.
+        expect(result.messages).toBeUndefined();
+        expect(result.activeTools).toContain('tool_request_activation');
     });
 
     test('leaves the dynamic system block uncached', async () => {
diff --git a/packages/web/src/ee/features/chat/agent.ts b/packages/web/src/ee/features/chat/agent.ts
index 77b0a126b..dc2b335dc 100644
--- a/packages/web/src/ee/features/chat/agent.ts
+++ b/packages/web/src/ee/features/chat/agent.ts
@@ -454,9 +454,11 @@ const createAgentStream = async ({
     //     instructions. This block is byte-identical across every chat and user,
     //     so it is a divergence-proof checkpoint a brand-new chat can read from
     //     instead of re-writing the large static prefix.
-    //   Marker 3: the last input message (below) caches tools + static + dynamic
-    //     system + the full conversation history — one conversation's growing
-    //     delta, re-warmed cheaply on every step and follow-up.
+    //   Marker 3: a moving breakpoint on the last message of each step (applied
+    //     in `prepareStep` below) caches tools + static + dynamic system + the
+    //     full conversation so far. Because it advances to the new tail every
+    //     step, the turn's growing delta (assistant tool calls and their outputs)
+    //     is cached incrementally instead of reprocessed on each later step.
     //
     // The `anthropic` provider-options namespace is ignored by non-Anthropic
     // providers, and a no-op strategy emits no markers at all, so this is safe
@@ -477,16 +479,12 @@ const createAgentStream = async ({
         systemMessages.push({ role: 'system', content: dynamicPrompt });
     }
 
-    // Marker 3: ephemeral (5m) breakpoint on the last input message. Merged onto
-    // any existing providerOptions so sibling namespaces (e.g. anthropic.thinking)
-    // are preserved. A no-op strategy returns undefined and leaves messages as-is.
+    // Marker 3 (moving tail): an ephemeral (5m) breakpoint applied in
+    // `prepareStep` to the last message of each step, so it advances with the
+    // turn instead of staying pinned to the last input message. Merged onto any
+    // existing providerOptions so sibling namespaces (e.g. anthropic.thinking)
+    // are preserved. A no-op strategy leaves it undefined and messages untouched.
     const tailMarker = promptCacheStrategy.cacheControl();
-    const messagesWithCachedPrefix: ModelMessage[] = tailMarker
-        ? inputMessages.map((message, index) =>
-            index === inputMessages.length - 1
-                ? { ...message, providerOptions: mergeProviderOptions(message.providerOptions, tailMarker) }
-                : message)
-        : inputMessages;
 
     // Observability only (default off): warn when the cache-relevant static
     // prefix (static system prompt, built-in tool definitions, model, or TTL)
@@ -507,21 +505,39 @@ const createAgentStream = async ({
         const stream = streamText({
             model,
             providerOptions,
-            messages: messagesWithCachedPrefix,
+            messages: inputMessages,
             system: systemMessages,
             tools: allTools,
             activeTools: [
                 ...builtinToolNames,
                 ...(hasMcpTools ? ['tool_request_activation'] : []),
             ],
-            prepareStep: hasMcpTools ? ({ steps }) => {
+            // `prepareStep` runs before every step (including the first). The SDK
+            // rebuilds the step's messages each time as the original input plus
+            // its own accumulated response messages — neither carries our marker
+            // — and a returned `messages` override applies only to that step, so
+            // re-applying the moving tail marker to the new last message each step
+            // can't accumulate stale markers. When MCP tools are present we also
+            // expand `activeTools` with whatever has been activated so far.
+            prepareStep: (tailMarker || hasMcpTools) ? ({ steps, messages }) => {
+                const stepMessages = (tailMarker && messages.length > 0)
+                    ? messages.map((message, index) =>
+                        index === messages.length - 1
+                            ? { ...message, providerOptions: mergeProviderOptions(message.providerOptions, tailMarker) }
+                            : message)
+                    : undefined;
+
+                if (!hasMcpTools) {
+                    return stepMessages ? { messages: stepMessages } : {};
+                }
+
                 const activated = new Set<string>();
                 for (const step of steps) {
-                    for (const result of step.toolResults) {
-                        if (!result || result.toolName !== 'tool_request_activation') {
+                    for (const toolResult of step.toolResults) {
+                        if (!toolResult || toolResult.toolName !== 'tool_request_activation') {
                             continue;
                         }
-                        const output = result.output as { results?: Array<{ name: string }> };
+                        const output = toolResult.output as { results?: Array<{ name: string }> };
                         for (const { name } of output?.results ?? []) {
                             if (name in mcpToolSetsObj.tools) {
                                 activated.add(name);
@@ -529,7 +545,9 @@ const createAgentStream = async ({
                         }
                     }
                 }
+
                 return {
+                    ...(stepMessages ? { messages: stepMessages } : {}),
                     activeTools: [
                         ...builtinToolNames,
                         'tool_request_activation',

From 6d2ce960e56aaed0cb57a50ea5eb489084b43788 Mon Sep 17 00:00:00 2001
From: Jack Minnetian <270441393+BlueBottleLatte@users.noreply.github.com>
Date: Tue, 23 Jun 2026 09:56:33 -0700
Subject: [PATCH 3/8] fix(web): bound the cache-break snapshot map and use
 sha256 for its signature

cacheBreakSnapshots was keyed by chatId and never evicted, so with cache-break detection enabled it grew with the cumulative number of distinct chats served. Add a FIFO cap that drops the oldest entry on overflow, and replace the hand-rolled djb2 signature hash with a sha256 slice matching getOAuthScopeHash (observability-only and compared in-process, so determinism is all it needs).
---
 CHANGELOG.md                                  |  3 +++
 .../web/src/ee/features/chat/promptCaching.ts | 26 +++++++++++--------
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fa46b346e..92192956d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+- [EE] Improved Ask Sourcebot prompt caching by splitting static and dynamic prompt sections and advancing cache breakpoints after every agent step instead of only after each message. [#<id>](https://github.com/sourcebot-dev/sourcebot/pull/<id>)
+
 ### Added
 - Added per-step token cost tracking and estimated tool call token usage to Ask Sourcebot chat history. [#1353](https://github.com/sourcebot-dev/sourcebot/pull/1353)
 
diff --git a/packages/web/src/ee/features/chat/promptCaching.ts b/packages/web/src/ee/features/chat/promptCaching.ts
index 83646a9a4..3afb0e9e7 100644
--- a/packages/web/src/ee/features/chat/promptCaching.ts
+++ b/packages/web/src/ee/features/chat/promptCaching.ts
@@ -1,3 +1,4 @@
+import { createHash } from "crypto";
 import { ProviderOptions } from "@ai-sdk/provider-utils";
 import { createLogger } from "@sourcebot/shared";
 import { LanguageModelProvider } from "@/features/chat/types";
@@ -105,19 +106,14 @@ interface CacheBreakSnapshot {
     requestCount: number;
 }
 
-// Keyed by chatId. In-memory, best-effort: survives within a server process and
-// is naturally bounded by the number of concurrent chats. Mirrors the in-memory
-// caching pattern used elsewhere (e.g. `anthropicThinkingConfigCache`).
+// Keyed by chatId, in-memory, observability-only. Entries are never removed on chat
+// end, so the map is bounded by a FIFO cap (oldest insertion evicted first, below);
+// otherwise it grows with the cumulative count of distinct chats, not the concurrent count.
+const MAX_CACHE_BREAK_SNAPSHOTS = 10_000;
 const cacheBreakSnapshots = new Map<string, CacheBreakSnapshot>();
 
-// Lightweight, stable, non-cryptographic hash (djb2). Observability only.
-const hashString = (input: string): string => {
-    let hash = 5381;
-    for (let i = 0; i < input.length; i++) {
-        hash = ((hash << 5) + hash + input.charCodeAt(i)) | 0;
-    }
-    return (hash >>> 0).toString(36);
-};
+const hashString = (input: string): string =>
+    createHash('sha256').update(input).digest('hex').slice(0, 16);
 
 /**
  * Records the cache-relevant prefix signature for a chat and logs a warning when
@@ -149,6 +145,14 @@ export const detectPromptCacheBreak = ({
         const requestCount = (prev?.requestCount ?? 0) + 1;
         cacheBreakSnapshots.set(chatId, { signature, requestCount });
 
+        // FIFO eviction: once the map overflows, drop the oldest-inserted entry.
+        if (cacheBreakSnapshots.size > MAX_CACHE_BREAK_SNAPSHOTS) {
+            const oldestKey = cacheBreakSnapshots.keys().next().value;
+            if (oldestKey !== undefined) {
+                cacheBreakSnapshots.delete(oldestKey);
+            }
+        }
+
         if (prev && prev.signature !== signature) {
             logger.warn(
                 `Prompt cache break detected for chat ${chatId} (request #${requestCount}): the ` +

From f024e8cd5d4773fced77a7abfc5670d226d60c9b Mon Sep 17 00:00:00 2001
From: Jack Minnetian <270441393+BlueBottleLatte@users.noreply.github.com>
Date: Tue, 23 Jun 2026 11:58:50 -0700
Subject: [PATCH 4/8] docs: update changelog for prompt caching

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 92192956d..689718d36 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Changed
-- [EE] Improved Ask Sourcebot prompt caching by splitting static and dynamic prompt sections and advancing cache breakpoints after every agent step instead of only after each message. [#<id>](https://github.com/sourcebot-dev/sourcebot/pull/<id>)
+- [EE] Improved Ask Sourcebot prompt caching by splitting static and dynamic prompt sections and advancing cache breakpoints after every agent step instead of only after each message. [#1366](https://github.com/sourcebot-dev/sourcebot/pull/1366)
 
 ### Added
 - Added per-step token cost tracking and estimated tool call token usage to Ask Sourcebot chat history. [#1353](https://github.com/sourcebot-dev/sourcebot/pull/1353)

From 93ebced68bb6c9a49303dbb7cc89e7d46a32654e Mon Sep 17 00:00:00 2001
From: Jack Minnetian <270441393+BlueBottleLatte@users.noreply.github.com>
Date: Tue, 23 Jun 2026 16:27:35 -0700
Subject: [PATCH 5/8] refactor(web): drop the tools-block prompt-cache
 breakpoint

Marker 1 only saved re-writing the built-in tool schemas on mid-turn MCP
activation steps, and only when those schemas cleared the model's minimum
cacheable size. The static-system checkpoint and moving tail carry the
value, so this collapses the scheme to two breakpoints and removes the
activeTools insertion-order reasoning it required.
---
 .../web/src/ee/features/chat/agent.test.ts    |  8 +++-
 packages/web/src/ee/features/chat/agent.ts    | 42 +++++++------------
 2 files changed, 20 insertions(+), 30 deletions(-)

diff --git a/packages/web/src/ee/features/chat/agent.test.ts b/packages/web/src/ee/features/chat/agent.test.ts
index 01dbfa8c0..d7ff6ea81 100644
--- a/packages/web/src/ee/features/chat/agent.test.ts
+++ b/packages/web/src/ee/features/chat/agent.test.ts
@@ -396,7 +396,7 @@ describe('createMessageStream prompt caching', () => {
         expect(system[1].content).toContain('<selected_repositories>');
     });
 
-    test('marks the tool_request_activation tool when MCP tools are present', async () => {
+    test('does not mark the tools block, so mid-run activeTools growth never busts it', async () => {
         const { buildMcpToolRegistry } = await import('@/ee/features/chat/mcp/mcpToolRegistry');
         vi.mocked(buildMcpToolRegistry).mockReturnValueOnce([
             { name: 'mcp_linear__save_issue', description: 'Save an issue', serverName: 'linear' },
@@ -406,7 +406,11 @@ describe('createMessageStream prompt caching', () => {
             promptCacheStrategy: anthropicStrategy,
         });
 
-        expect(tools.tool_request_activation?.providerOptions?.anthropic?.cacheControl).toEqual(EPHEMERAL);
+        // The static checkpoint sits on the system block (after the full tools
+        // section in render order), so no tool definition carries a breakpoint.
+        for (const tool of Object.values(tools)) {
+            expect(tool.providerOptions?.anthropic?.cacheControl).toBeUndefined();
+        }
     });
 
     test.each([
diff --git a/packages/web/src/ee/features/chat/agent.ts b/packages/web/src/ee/features/chat/agent.ts
index dc2b335dc..4cca5327d 100644
--- a/packages/web/src/ee/features/chat/agent.ts
+++ b/packages/web/src/ee/features/chat/agent.ts
@@ -397,16 +397,6 @@ const createAgentStream = async ({
     const useStaticPrefix = env.SOURCEBOT_CHAT_PROMPT_CACHE_STATIC_PREFIX_ENABLED === 'true';
     const staticTtl = env.SOURCEBOT_CHAT_PROMPT_CACHE_STATIC_TTL;
 
-    // Marker 1 (MCP only): cache the byte-stable built-in tools + activation tool
-    // block on the `tool_request_activation` definition (the last always-active
-    // tool, serialized before the dynamic MCP tools). This survives the cache
-    // bust that mid-run `activeTools` growth causes for markers 2 & 3. When no
-    // MCP tools are present, marker 2 already covers the built-in tools, so this
-    // is skipped.
-    const activationToolMarker = (hasMcpTools && useStaticPrefix)
-        ? promptCacheStrategy.cacheControl({ ttl: staticTtl })
-        : undefined;
-
     const toolRequestActivation = tool({
         description: dedent`
         Activate an MCP tool by name so it becomes callable on your next step.
@@ -423,7 +413,6 @@ const createAgentStream = async ({
         inputSchema: z.object({
             tool_to_activate_name: z.string().describe('Exact tool name from the registry, e.g. "mcp_linear__save_comment"'),
         }),
-        providerOptions: activationToolMarker,
         execute: async ({ tool_to_activate_name }) => {
             const results = searchMcpTools(tool_to_activate_name, mcpRegistry);
             return {
@@ -445,20 +434,18 @@ const createAgentStream = async ({
         ...(hasMcpTools ? { tool_request_activation: toolRequestActivation, ...mcpToolSetsObj.tools } : {}),
     };
 
-    // Anthropic prompt caching uses up to three nested breakpoints over one
-    // cumulative prefix (render order: tools -> system -> messages):
+    // Anthropic prompt caching uses two nested breakpoints over one cumulative
+    // prefix (render order: tools -> system -> messages):
     //
-    //   Marker 1 (MCP only): the `tool_request_activation` tool definition (set
-    //     above) caches the byte-stable built-in + activation tool block.
-    //   Marker 2: the static system block below caches tools + the static system
-    //     instructions. This block is byte-identical across every chat and user,
-    //     so it is a divergence-proof checkpoint a brand-new chat can read from
-    //     instead of re-writing the large static prefix.
-    //   Marker 3: a moving breakpoint on the last message of each step (applied
-    //     in `prepareStep` below) caches tools + static + dynamic system + the
-    //     full conversation so far. Because it advances to the new tail every
-    //     step, the turn's growing delta (assistant tool calls and their outputs)
-    //     is cached incrementally instead of reprocessed on each later step.
+    //   Static checkpoint: the static system block below caches tools + the
+    //     static system instructions. This block is byte-identical across every
+    //     chat and user, so it is a divergence-proof checkpoint a brand-new chat
+    //     can read from instead of re-writing the large static prefix.
+    //   Moving tail: a breakpoint on the last message of each step (applied in
+    //     `prepareStep` below) caches tools + static + dynamic system + the full
+    //     conversation so far. Because it advances to the new tail every step,
+    //     the turn's growing delta (assistant tool calls and their outputs) is
+    //     cached incrementally instead of reprocessed on each later step.
     //
     // The `anthropic` provider-options namespace is ignored by non-Anthropic
     // providers, and a no-op strategy emits no markers at all, so this is safe
@@ -466,9 +453,8 @@ const createAgentStream = async ({
     // cacheable size the marker is a harmless no-op.
     //
     // Caveat: when MCP tools are lazily activated mid-run via prepareStep, the
-    // tools section grows and invalidates markers 2 & 3 for that step; marker 1
-    // preserves the built-in/activation block and the cache re-warms on
-    // subsequent steps once the active tool set is stable.
+    // tools section grows and invalidates both breakpoints for that step; the
+    // cache re-warms on subsequent steps once the active tool set is stable.
     const staticMarker = useStaticPrefix
         ? promptCacheStrategy.cacheControl({ ttl: staticTtl })
         : undefined;
@@ -479,7 +465,7 @@ const createAgentStream = async ({
         systemMessages.push({ role: 'system', content: dynamicPrompt });
     }
 
-    // Marker 3 (moving tail): an ephemeral (5m) breakpoint applied in
+    // Moving tail: an ephemeral (5m) breakpoint applied in
     // `prepareStep` to the last message of each step, so it advances with the
     // turn instead of staying pinned to the last input message. Merged onto any
     // existing providerOptions so sibling namespaces (e.g. anthropic.thinking)

From b8c8b2300a31cdafa11dedac1a83f963825e1ca1 Mon Sep 17 00:00:00 2001
From: Jack Minnetian <270441393+BlueBottleLatte@users.noreply.github.com>
Date: Tue, 23 Jun 2026 17:22:09 -0700
Subject: [PATCH 6/8] clean up comments

Remove stale references to the dropped tools-block breakpoint and tighten verbose prompt-caching comments. Comments only, no code changes.
---
 packages/shared/src/env.server.ts             | 10 ++++------
 .../web/src/app/api/(server)/ee/chat/route.ts |  5 ++---
 packages/web/src/ee/features/chat/agent.ts    | 19 +++++++++----------
 .../web/src/ee/features/chat/promptCaching.ts |  6 ++----
 4 files changed, 17 insertions(+), 23 deletions(-)

diff --git a/packages/shared/src/env.server.ts b/packages/shared/src/env.server.ts
index 179c3367f..b9b6c62d9 100644
--- a/packages/shared/src/env.server.ts
+++ b/packages/shared/src/env.server.ts
@@ -311,13 +311,11 @@ const options = {
 
         SOURCEBOT_CHAT_MAX_STEP_COUNT: numberSchema.default(100),
         SOURCEBOT_CHAT_PROMPT_CACHING_ENABLED: booleanSchema.default('true'),
-        // Phased-rollout lever for the static front cache breakpoint (and the
-        // MCP activation-resilience breakpoint). Set to 'false' to fall back to
-        // the single moving tail marker. Only takes effect when prompt caching
-        // is enabled. See packages/web/src/ee/features/chat/promptCaching.ts.
+        // Phased-rollout lever for the static checkpoint. Set to 'false' to fall
+        // back to the single moving tail marker. Only takes effect when prompt
+        // caching is enabled.
         SOURCEBOT_CHAT_PROMPT_CACHE_STATIC_PREFIX_ENABLED: booleanSchema.default('true'),
-        // TTL for the static front block (and MCP activation-resilience block).
-        // The moving tail marker always uses the 5m default.
+        // TTL for the static block. The moving tail marker always uses the 5m default.
         SOURCEBOT_CHAT_PROMPT_CACHE_STATIC_TTL: z.enum(['5m', '1h']).default('5m'),
         // Observability: when enabled, logs a warning on unexpected prompt-cache
         // breaks (static-prefix signature changes, or zero cache reads on a
diff --git a/packages/web/src/app/api/(server)/ee/chat/route.ts b/packages/web/src/app/api/(server)/ee/chat/route.ts
index ff05467c6..0f20ee8e3 100644
--- a/packages/web/src/app/api/(server)/ee/chat/route.ts
+++ b/packages/web/src/app/api/(server)/ee/chat/route.ts
@@ -89,9 +89,8 @@ export const POST = apiHandler(async (req: NextRequest) => {
 
             const { model, providerOptions, temperature } = await getAISDKLanguageModelAndOptions(languageModelConfig);
 
-            // Resolve provider-aware prompt caching. The strategy is a no-op for
-            // non-Anthropic providers or when caching is disabled, so threading it
-            // through never perturbs other providers' requests.
+            // No-op for non-Anthropic providers / when caching is disabled, so
+            // it never perturbs other providers' requests.
             const promptCacheStrategy = getPromptCacheStrategy(
                 languageModelConfig.provider,
                 env.SOURCEBOT_CHAT_PROMPT_CACHING_ENABLED === 'true',
diff --git a/packages/web/src/ee/features/chat/agent.ts b/packages/web/src/ee/features/chat/agent.ts
index 4cca5327d..6a3b21e36 100644
--- a/packages/web/src/ee/features/chat/agent.ts
+++ b/packages/web/src/ee/features/chat/agent.ts
@@ -332,8 +332,8 @@ const createAgentStream = async ({
     userId,
     orgId,
 }: AgentOptions) => {
-    // Normalize repo order so the dynamic system block's <selected_repositories>
-    // list is byte-stable across requests of the same chat, used for prompt caching.
+    // Sort repos so the dynamic <selected_repositories> block is byte-stable
+    // across a chat's requests (prompt caching).
     const sortedRepos = [...selectedRepos].sort((a, b) => a.localeCompare(b));
 
     // For every file source, resolve the source code so that we can include it in the system prompt.
@@ -391,9 +391,9 @@ const createAgentStream = async ({
     const mcpRegistry = buildMcpToolRegistry(mcpToolSetsObj.tools);
     const hasMcpTools = mcpRegistry.length > 0;
 
-    // Phased-rollout lever for the static front checkpoint (markers 1 & 2). When
-    // disabled, only the moving tail marker is emitted and behavior collapses to
-    // the prior single-breakpoint scheme.
+    // Phased-rollout lever for the static checkpoint. When disabled, only the
+    // moving tail marker is emitted and behavior collapses to the prior
+    // single-breakpoint scheme.
     const useStaticPrefix = env.SOURCEBOT_CHAT_PROMPT_CACHE_STATIC_PREFIX_ENABLED === 'true';
     const staticTtl = env.SOURCEBOT_CHAT_PROMPT_CACHE_STATIC_TTL;
 
@@ -465,11 +465,10 @@ const createAgentStream = async ({
         systemMessages.push({ role: 'system', content: dynamicPrompt });
     }
 
-    // Moving tail: an ephemeral (5m) breakpoint applied in
-    // `prepareStep` to the last message of each step, so it advances with the
-    // turn instead of staying pinned to the last input message. Merged onto any
-    // existing providerOptions so sibling namespaces (e.g. anthropic.thinking)
-    // are preserved. A no-op strategy leaves it undefined and messages untouched.
+    // The moving-tail marker (see above), resolved once here. `prepareStep`
+    // merges it onto the last message's existing providerOptions so sibling
+    // namespaces (e.g. anthropic.thinking) survive; a no-op strategy leaves it
+    // undefined and the messages untouched.
     const tailMarker = promptCacheStrategy.cacheControl();
 
     // Observability only (default off): warn when the cache-relevant static
diff --git a/packages/web/src/ee/features/chat/promptCaching.ts b/packages/web/src/ee/features/chat/promptCaching.ts
index 3afb0e9e7..f736f0b93 100644
--- a/packages/web/src/ee/features/chat/promptCaching.ts
+++ b/packages/web/src/ee/features/chat/promptCaching.ts
@@ -3,9 +3,8 @@ import { ProviderOptions } from "@ai-sdk/provider-utils";
 import { createLogger } from "@sourcebot/shared";
 import { LanguageModelProvider } from "@/features/chat/types";
 
-// @note: Prompt-cache plumbing for the Ask agent. Ask Sourcebot is an
-// enterprise feature, so this lives under `ee/`. It is the single place that
-// knows about provider-specific cache-control shapes — the agent loop stays
+// @note: Prompt-cache plumbing for the Ask agent — the single place that knows
+// about provider-specific cache-control shapes. The agent loop stays
 // provider-agnostic by asking the strategy for a `providerOptions` blob at each
 // breakpoint and merging whatever it gets back (possibly `undefined`).
 
@@ -145,7 +144,6 @@ export const detectPromptCacheBreak = ({
         const requestCount = (prev?.requestCount ?? 0) + 1;
         cacheBreakSnapshots.set(chatId, { signature, requestCount });
 
-        // FIFO eviction: once the map overflows, drop the oldest-inserted entry.
         if (cacheBreakSnapshots.size > MAX_CACHE_BREAK_SNAPSHOTS) {
             const oldestKey = cacheBreakSnapshots.keys().next().value;
             if (oldestKey !== undefined) {

From 8610781b8a402ef7578fe0a529a759a6db11e69e Mon Sep 17 00:00:00 2001
From: Jack Minnetian <270441393+BlueBottleLatte@users.noreply.github.com>
Date: Tue, 23 Jun 2026 18:17:11 -0700
Subject: [PATCH 7/8] trim comments further

---
 packages/web/src/ee/features/chat/agent.ts     | 18 +++++-------------
 .../web/src/ee/features/chat/promptCaching.ts  | 10 ++--------
 2 files changed, 7 insertions(+), 21 deletions(-)

diff --git a/packages/web/src/ee/features/chat/agent.ts b/packages/web/src/ee/features/chat/agent.ts
index 6a3b21e36..f600efdaf 100644
--- a/packages/web/src/ee/features/chat/agent.ts
+++ b/packages/web/src/ee/features/chat/agent.ts
@@ -471,9 +471,6 @@ const createAgentStream = async ({
     // undefined and the messages untouched.
     const tailMarker = promptCacheStrategy.cacheControl();
 
-    // Observability only (default off): warn when the cache-relevant static
-    // prefix (static system prompt, built-in tool definitions, model, or TTL)
-    // changes between requests for the same chat in a way that busts the cache.
     if (env.SOURCEBOT_CHAT_PROMPT_CACHE_BREAK_DETECTION_ENABLED === 'true') {
         detectPromptCacheBreak({
             chatId,
@@ -499,11 +496,8 @@ const createAgentStream = async ({
             ],
             // `prepareStep` runs before every step (including the first). The SDK
             // rebuilds the step's messages each time as the original input plus
-            // its own accumulated response messages — neither carries our marker
-            // — and a returned `messages` override applies only to that step, so
-            // re-applying the moving tail marker to the new last message each step
-            // can't accumulate stale markers. When MCP tools are present we also
-            // expand `activeTools` with whatever has been activated so far.
+            // its own accumulated response messages. Re-applying the moving tail marker
+            // to the new last message each step is safe and does not accumulate.
             prepareStep: (tailMarker || hasMcpTools) ? ({ steps, messages }) => {
                 const stepMessages = (tailMarker && messages.length > 0)
                     ? messages.map((message, index) =>
@@ -617,9 +611,8 @@ const createPrompt = ({
     repos: string[],
     mcpToolRegistry: McpToolRegistryEntry[],
 }): { staticPrompt: string; dynamicPrompt: string } => {
-    // Static prefix: byte-identical across every chat and user, so Anthropic can
-    // reuse its cache entry across chats (the divergence-proof checkpoint). It
-    // interpolates only module-level constants. Keep it free of any
+    // Static prefix: byte-identical across every chat and user.
+    // It interpolates only module-level constants. Keep it free of any
     // per-conversation data — repos, files, and MCP tools live in the dynamic
     // block below so their volatility never busts the shared static cache.
     const staticPrompt = dedent`
@@ -677,8 +670,7 @@ const createPrompt = ({
     `;
 
     // Dynamic block: per-conversation context (selected repos, mentioned files,
-    // MCP tool registry). Placed after the static checkpoint so changes here
-    // never invalidate the cross-chat static cache. Empty string when none apply.
+    // MCP tool registry).
     const dynamicSections: string[] = [];
 
     if (repos.length > 0) {
diff --git a/packages/web/src/ee/features/chat/promptCaching.ts b/packages/web/src/ee/features/chat/promptCaching.ts
index f736f0b93..828f48d4b 100644
--- a/packages/web/src/ee/features/chat/promptCaching.ts
+++ b/packages/web/src/ee/features/chat/promptCaching.ts
@@ -15,8 +15,7 @@ export type CacheTtl = '5m' | '1h';
 export interface PromptCacheStrategy {
     /**
      * Whether the resolved provider supports explicit cache breakpoints. When
-     * false, every `cacheControl()` call is a no-op and the request is left
-     * untouched (non-Anthropic providers, or caching disabled).
+     * false, every `cacheControl()` call is a no-op.
      */
     readonly supportsBreakpoints: boolean;
     /**
@@ -36,7 +35,7 @@ const NOOP_STRATEGY: PromptCacheStrategy = {
 // namespace and honors ephemeral `cacheControl` breakpoints. `google-vertex-anthropic`
 // reuses the same namespace; if a given SDK version ignores it the marker is a
 // harmless no-op. Bedrock uses a different shape (`cachePoint`) and is intentionally
-// not covered here — adding it later is a single new branch.
+// not covered here - adding it later is a single new branch.
 const ANTHROPIC_FAMILY_PROVIDERS: ReadonlySet<LanguageModelProvider> = new Set<LanguageModelProvider>([
     'anthropic',
     'google-vertex-anthropic',
@@ -54,8 +53,6 @@ const anthropicCacheControl = ({ ttl }: { ttl?: CacheTtl } = {}): ProviderOption
 
 /**
  * Resolves how (and whether) to emit prompt-cache breakpoints for a provider.
- * Non-Anthropic providers, or a disabled master flag, yield a no-op strategy so
- * their requests are never perturbed.
  */
 export const getPromptCacheStrategy = (
     provider: LanguageModelProvider,
@@ -98,8 +95,6 @@ export const mergeProviderOptions = (
     return merged;
 };
 
-// --- Cache-break detection (observability only; never throws) ---------------
-
 interface CacheBreakSnapshot {
     signature: string;
     requestCount: number;
@@ -107,7 +102,6 @@ interface CacheBreakSnapshot {
 
 // Keyed by chatId, in-memory, observability-only. Entries are never removed on chat
 // end, so the map is bounded by a FIFO cap (oldest insertion evicted first, below);
-// otherwise it grows with the cumulative count of distinct chats, not the concurrent count.
 const MAX_CACHE_BREAK_SNAPSHOTS = 10_000;
 const cacheBreakSnapshots = new Map<string, CacheBreakSnapshot>();
 

From cd7ae3e1b876d925357614da1b4ca4dfd75dc702 Mon Sep 17 00:00:00 2001
From: Jack Minnetian <270441393+BlueBottleLatte@users.noreply.github.com>
Date: Tue, 23 Jun 2026 20:29:23 -0700
Subject: [PATCH 8/8] refactor(web): always enable static-prefix prompt caching

Remove the SOURCEBOT_CHAT_PROMPT_CACHE_STATIC_PREFIX_ENABLED lever so the static checkpoint is always emitted, and switch the remaining cache env vars to JSDoc comments so their descriptions render on IDE hover.
---
 packages/shared/src/env.server.ts               | 14 ++++++--------
 packages/web/src/ee/features/chat/agent.test.ts |  4 ----
 packages/web/src/ee/features/chat/agent.ts      |  8 +-------
 3 files changed, 7 insertions(+), 19 deletions(-)

diff --git a/packages/shared/src/env.server.ts b/packages/shared/src/env.server.ts
index b9b6c62d9..57c83f9f0 100644
--- a/packages/shared/src/env.server.ts
+++ b/packages/shared/src/env.server.ts
@@ -311,15 +311,13 @@ const options = {
 
         SOURCEBOT_CHAT_MAX_STEP_COUNT: numberSchema.default(100),
         SOURCEBOT_CHAT_PROMPT_CACHING_ENABLED: booleanSchema.default('true'),
-        // Phased-rollout lever for the static checkpoint. Set to 'false' to fall
-        // back to the single moving tail marker. Only takes effect when prompt
-        // caching is enabled.
-        SOURCEBOT_CHAT_PROMPT_CACHE_STATIC_PREFIX_ENABLED: booleanSchema.default('true'),
-        // TTL for the static block. The moving tail marker always uses the 5m default.
+        /** TTL for the static block. The moving tail marker always uses the 5m default. */
         SOURCEBOT_CHAT_PROMPT_CACHE_STATIC_TTL: z.enum(['5m', '1h']).default('5m'),
-        // Observability: when enabled, logs a warning on unexpected prompt-cache
-        // breaks (static-prefix signature changes, or zero cache reads on a
-        // continuation step). Does not affect request behavior.
+        /**
+         * Observability: when enabled, logs a warning on unexpected prompt-cache
+         * breaks (static-prefix signature changes, or zero cache reads on a
+         * continuation step). Does not affect request behavior.
+         */
         SOURCEBOT_CHAT_PROMPT_CACHE_BREAK_DETECTION_ENABLED: booleanSchema.default('false'),
         SOURCEBOT_MCP_TOOL_CALL_TIMEOUT_MS: numberSchema.int().positive().max(maxTimerDelayMs).default(60000),
 
diff --git a/packages/web/src/ee/features/chat/agent.test.ts b/packages/web/src/ee/features/chat/agent.test.ts
index d7ff6ea81..a16c4041c 100644
--- a/packages/web/src/ee/features/chat/agent.test.ts
+++ b/packages/web/src/ee/features/chat/agent.test.ts
@@ -27,10 +27,6 @@ vi.mock('@sourcebot/shared', () => ({
         SOURCEBOT_CHAT_MAX_STEP_COUNT: 8,
         SOURCEBOT_CHAT_MODEL_TEMPERATURE: 0,
         SOURCEBOT_TELEMETRY_PII_COLLECTION_ENABLED: 'false',
-        // Enable the static front checkpoint so marker placement is exercised;
-        // whether markers are actually emitted is then controlled by the
-        // PromptCacheStrategy passed into each call.
-        SOURCEBOT_CHAT_PROMPT_CACHE_STATIC_PREFIX_ENABLED: 'true',
         SOURCEBOT_CHAT_PROMPT_CACHE_STATIC_TTL: '5m',
         SOURCEBOT_CHAT_PROMPT_CACHE_BREAK_DETECTION_ENABLED: 'false',
     },
diff --git a/packages/web/src/ee/features/chat/agent.ts b/packages/web/src/ee/features/chat/agent.ts
index f600efdaf..d2f3a4761 100644
--- a/packages/web/src/ee/features/chat/agent.ts
+++ b/packages/web/src/ee/features/chat/agent.ts
@@ -391,10 +391,6 @@ const createAgentStream = async ({
     const mcpRegistry = buildMcpToolRegistry(mcpToolSetsObj.tools);
     const hasMcpTools = mcpRegistry.length > 0;
 
-    // Phased-rollout lever for the static checkpoint. When disabled, only the
-    // moving tail marker is emitted and behavior collapses to the prior
-    // single-breakpoint scheme.
-    const useStaticPrefix = env.SOURCEBOT_CHAT_PROMPT_CACHE_STATIC_PREFIX_ENABLED === 'true';
     const staticTtl = env.SOURCEBOT_CHAT_PROMPT_CACHE_STATIC_TTL;
 
     const toolRequestActivation = tool({
@@ -455,9 +451,7 @@ const createAgentStream = async ({
     // Caveat: when MCP tools are lazily activated mid-run via prepareStep, the
     // tools section grows and invalidates both breakpoints for that step; the
     // cache re-warms on subsequent steps once the active tool set is stable.
-    const staticMarker = useStaticPrefix
-        ? promptCacheStrategy.cacheControl({ ttl: staticTtl })
-        : undefined;
+    const staticMarker = promptCacheStrategy.cacheControl({ ttl: staticTtl });
     const systemMessages: SystemModelMessage[] = [
         { role: 'system', content: staticPrompt, providerOptions: staticMarker },
     ];