fix(tables): allow url sandbox entries in the function-execute contract; key snapshot by column shape so schema edits invalidate it

Author: TheodoreSpeaks

apps/sim/lib/api/contracts/hotspots.ts

@@ -162,11 +162,20 @@ export const functionExecuteContract = defineRouteContract({
     isCustomTool: z.boolean().optional().default(false),
     _sandboxFiles: z
       .array(
-        z.object({
-          path: z.string(),
-          content: z.string(),
-          encoding: z.literal('base64').optional(),
-        })
+        z.union([
+          z.object({
+            type: z.literal('content').optional(),
+            path: z.string(),
+            content: z.string(),
+            encoding: z.literal('base64').optional(),
+          }),
+          // Mounted by reference: the sandbox fetches `url` itself (no bytes through the web tier).
+          z.object({
+            type: z.literal('url'),
+            path: z.string(),
+            url: z.string(),
+          }),
+        ])
       )
       .optional(),
   }),

apps/sim/lib/table/snapshot-cache.test.ts

@@ -28,11 +28,7 @@ vi.mock('@/lib/uploads/core/storage-service', () => ({
   deleteFile: mockDeleteFile,
 }))
 
-import {
-  getOrCreateTableSnapshot,
-  SNAPSHOT_MAX_BYTES,
-  TableSnapshotTooLargeError,
-} from '@/lib/table/snapshot-cache'
+import { getOrCreateTableSnapshot, TableSnapshotTooLargeError } from '@/lib/table/snapshot-cache'
 
 const table = {
   id: 'tbl_1',
@@ -86,7 +82,11 @@ describe('getOrCreateTableSnapshot', () => {
 
     const ref = await getOrCreateTableSnapshot(table, 'req')
 
-    expect(ref).toEqual({ key: 'table-snapshots/ws_1/tbl_1/v3.csv', size: 42, version: 3 })
+    expect(ref).toEqual({
+      key: expect.stringMatching(/^table-snapshots\/ws_1\/tbl_1\/v3-[0-9a-f]{12}\.csv$/),
+      size: 42,
+      version: 3,
+    })
     expect(mockCreateMultipartUpload).not.toHaveBeenCalled()
     expect(mockSelectExportRowPage).not.toHaveBeenCalled()
   })
@@ -98,25 +98,50 @@ describe('getOrCreateTableSnapshot', () => {
     const ref = await getOrCreateTableSnapshot(table, 'req')
 
     expect(mockCreateMultipartUpload).toHaveBeenCalledWith(
-      expect.objectContaining({ key: 'table-snapshots/ws_1/tbl_1/v3.csv', context: 'execution' })
+      expect.objectContaining({
+        key: expect.stringMatching(/^table-snapshots\/ws_1\/tbl_1\/v3-[0-9a-f]{12}\.csv$/),
+        context: 'execution',
+      })
     )
     expect(lastHandle?.content).toBe('name\nAda\n')
     expect(ref).toEqual({
-      key: 'table-snapshots/ws_1/tbl_1/v3.csv',
+      key: expect.stringMatching(/^table-snapshots\/ws_1\/tbl_1\/v3-[0-9a-f]{12}\.csv$/),
       size: Buffer.byteLength('name\nAda\n'),
       version: 3,
     })
     // Best-effort prune of v2.
     expect(mockDeleteFile).toHaveBeenCalledWith(
-      expect.objectContaining({ key: 'table-snapshots/ws_1/tbl_1/v2.csv', context: 'execution' })
+      expect.objectContaining({
+        key: expect.stringMatching(/^table-snapshots\/ws_1\/tbl_1\/v2-[0-9a-f]{12}\.csv$/),
+        context: 'execution',
+      })
     )
   })
 
   it('keys the snapshot by tenant — the same table id in another workspace gets a different key', async () => {
     versions(1)
     mockHeadObject.mockResolvedValue({ size: 1 })
     const ref = await getOrCreateTableSnapshot({ ...table, workspaceId: 'ws_2' }, 'req')
-    expect(ref.key).toBe('table-snapshots/ws_2/tbl_1/v1.csv')
+    expect(ref.key).toMatch(/^table-snapshots\/ws_2\/tbl_1\/v1-[0-9a-f]{12}\.csv$/)
+  })
+
+  it('changes the key when the column shape changes (schema edits invalidate the cache)', async () => {
+    versions(7, 7)
+    mockHeadObject.mockResolvedValue({ size: 1 })
+
+    const a = await getOrCreateTableSnapshot(table, 'req')
+    const b = await getOrCreateTableSnapshot(
+      {
+        ...table,
+        schema: { columns: [{ id: 'col_name', name: 'renamed', type: 'string' }] },
+      } as never,
+      'req'
+    )
+
+    // Same workspace/table/row-version, but a renamed column flips the shape hash → different key.
+    expect(a.key).not.toBe(b.key)
+    expect(a.key).toMatch(/\/v7-[0-9a-f]{12}\.csv$/)
+    expect(b.key).toMatch(/\/v7-[0-9a-f]{12}\.csv$/)
   })
 
   it('re-keys and rebuilds when rows_version advances mid-scan', async () => {
@@ -134,21 +159,25 @@ describe('getOrCreateTableSnapshot', () => {
     const ref = await getOrCreateTableSnapshot(table, 'req')
 
     expect(ref.version).toBe(4)
-    expect(ref.key).toBe('table-snapshots/ws_1/tbl_1/v4.csv')
+    expect(ref.key).toMatch(/^table-snapshots\/ws_1\/tbl_1\/v4-[0-9a-f]{12}\.csv$/)
     expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(2)
     // the stale v3 object is dropped
     expect(mockDeleteFile).toHaveBeenCalledWith(
-      expect.objectContaining({ key: 'table-snapshots/ws_1/tbl_1/v3.csv' })
+      expect.objectContaining({
+        key: expect.stringMatching(/^table-snapshots\/ws_1\/tbl_1\/v3-[0-9a-f]{12}\.csv$/),
+      })
     )
   })
 
   it('aborts and throws when the CSV exceeds the size cap', async () => {
     versions(1)
     mockHeadObject.mockResolvedValue(null)
     mockSelectExportRowPage.mockReset()
-    mockSelectExportRowPage.mockResolvedValueOnce([
-      { id: 'r1', data: { col_name: 'x'.repeat(SNAPSHOT_MAX_BYTES + 10) }, position: 0 },
-    ])
+    // A full batch of wide rows on every page → the materialize loop keeps paging until the running
+    // byte count crosses the cap, then aborts. Peak memory stays at one page (~MBs), not the cap.
+    const wideRow = { id: 'r', data: { col_name: 'x'.repeat(1000) }, position: 0 }
+    const fullPage = Array.from({ length: 10000 }, () => wideRow)
+    mockSelectExportRowPage.mockResolvedValue(fullPage)
 
     await expect(getOrCreateTableSnapshot(table, 'req')).rejects.toBeInstanceOf(
       TableSnapshotTooLargeError

apps/sim/lib/table/snapshot-cache.ts

@@ -11,6 +11,7 @@
  * contain — and be addressed by — its owning tenant.
  */
 
+import { createHash } from 'crypto'
 import { db } from '@sim/db'
 import { userTableDefinitions } from '@sim/db/schema'
 import { createLogger } from '@sim/logger'
@@ -52,19 +53,25 @@ export class TableSnapshotTooLargeError extends Error {
 }
 
 /**
- * Storage key for a table's snapshot at a given version.
- *
- * `projectionHash` is a forward seam: a future column-subset / filtered mount appends
- * `-{projectionHash}` so projections cache independently of the full snapshot.
+ * Fingerprint of the table's column shape (id + display name + order). `rows_version` only advances
+ * on row mutations (the trigger fires on `user_table_rows`), so without this a schema edit — rename,
+ * add, remove, or reorder a column — would change the CSV header/columns but keep the same key and
+ * serve a stale snapshot. Folding it into the key invalidates the cache on any schema change. This
+ * is also the seam for a future column-subset / filtered projection (mix it into the same hash).
  */
+function schemaFingerprint(table: TableDefinition): string {
+  const shape = table.schema.columns.map((c) => [getColumnId(c), c.name])
+  return createHash('sha1').update(JSON.stringify(shape)).digest('hex').slice(0, 12)
+}
+
+/** Storage key for a table's snapshot at a given row version + column shape. */
 function snapshotKey(
   workspaceId: string,
   tableId: string,
   version: number,
-  projectionHash?: string
+  shapeHash: string
 ): string {
-  const suffix = projectionHash ? `-${projectionHash}` : ''
-  return `table-snapshots/${workspaceId}/${tableId}/v${version}${suffix}.csv`
+  return `table-snapshots/${workspaceId}/${tableId}/v${version}-${shapeHash}.csv`
 }
 
 async function readRowsVersion(tableId: string): Promise<number> {
@@ -122,10 +129,14 @@ async function materialize(table: TableDefinition, key: string): Promise<number>
 }
 
 /** Best-effort removal of the immediately-prior version (the common single-mutation case). */
-async function deletePreviousVersion(table: TableDefinition, version: number): Promise<void> {
+async function deletePreviousVersion(
+  table: TableDefinition,
+  version: number,
+  shapeHash: string
+): Promise<void> {
   if (version <= 0) return
   await deleteFile({
-    key: snapshotKey(table.workspaceId, table.id, version - 1),
+    key: snapshotKey(table.workspaceId, table.id, version - 1, shapeHash),
     context: SNAPSHOT_STORAGE_CONTEXT,
   }).catch(() => {})
 }
@@ -142,8 +153,9 @@ export async function getOrCreateTableSnapshot(
   table: TableDefinition,
   requestId: string
 ): Promise<TableSnapshotRef> {
+  const shapeHash = schemaFingerprint(table)
   const version = await readRowsVersion(table.id)
-  const key = snapshotKey(table.workspaceId, table.id, version)
+  const key = snapshotKey(table.workspaceId, table.id, version, shapeHash)
 
   const head = await headObject(key, SNAPSHOT_STORAGE_CONTEXT)
   if (head) {
@@ -163,14 +175,14 @@ export async function getOrCreateTableSnapshot(
       from: version,
       to: after,
     })
-    const newKey = snapshotKey(table.workspaceId, table.id, after)
+    const newKey = snapshotKey(table.workspaceId, table.id, after, shapeHash)
     const newHead = await headObject(newKey, SNAPSHOT_STORAGE_CONTEXT)
     const newSize = newHead ? newHead.size : await materialize(table, newKey)
     await deleteFile({ key, context: SNAPSHOT_STORAGE_CONTEXT }).catch(() => {})
-    void deletePreviousVersion(table, after)
+    void deletePreviousVersion(table, after, shapeHash)
     return { key: newKey, size: newSize, version: after }
   }
 
-  void deletePreviousVersion(table, version)
+  void deletePreviousVersion(table, version, shapeHash)
   return { key, size, version }
 }