From 064f2540efaffc0a1e6b62343ee9ce36f02f3987 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Tue, 30 Jun 2026 12:41:01 -0700 Subject: [PATCH 1/5] feat(forking): resource copying UX to help with setup speed --- .../workspaces/[id]/background-work/route.ts | 12 +- .../api/workspaces/[id]/fork/diff/route.ts | 39 +- .../api/workspaces/[id]/fork/promote/route.ts | 3 +- .../sim/app/api/workspaces/[id]/fork/route.ts | 2 +- .../mention/use-editor-mentions.ts | 11 +- .../rich-markdown-editor.tsx | 8 +- .../rich-markdown-field.tsx | 5 +- .../components/version-description-modal.tsx | 1 + .../fork-activity-panel.tsx | 2 +- .../fork-file-tree/fork-file-tree.test.ts | 28 + .../fork-file-tree/fork-file-tree.tsx | 197 +++++++ .../fork-resource-picker.tsx | 207 +++++++ .../fork-workspace-modal.tsx | 288 +++++----- .../cleared-refs-list.test.ts | 126 +++++ .../cleared-refs-list.ts | 43 ++ .../copy-reconciliation.test.ts | 124 ++++ .../copy-reconciliation.ts | 85 +++ .../dependent-value.test.ts | 59 ++ .../promote-workspace-modal.tsx | 353 ++++++++++-- apps/sim/lib/api/contracts/workspace-fork.ts | 135 ++++- .../persistence/remap-internal-ids.test.ts | 75 +++ .../persistence/remap-internal-ids.ts | 19 +- .../fork/copy/cleanup-failed.test.ts | 432 ++++++++++++++ .../workspaces/fork/copy/cleanup-failed.ts | 352 ++++++++++++ .../fork/copy/content-copy-runner.test.ts | 36 ++ .../fork/copy/content-copy-runner.ts | 135 ++++- .../lib/workspaces/fork/copy/copy-files.ts | 95 +++- .../fork/copy/copy-resources.test.ts | 396 +++++++++++++ .../workspaces/fork/copy/copy-resources.ts | 535 ++++++++++++++++-- .../fork/copy/workflow-id-map.test.ts | 41 ++ .../workspaces/fork/copy/workflow-id-map.ts | 21 + apps/sim/lib/workspaces/fork/create-fork.ts | 112 ++-- .../workspaces/fork/mapping/cascade.test.ts | 1 + .../lib/workspaces/fork/mapping/cascade.ts | 8 +- .../fork/mapping/dependent-reconfigs.ts | 18 +- .../fork/mapping/mapping-service.test.ts | 66 +++ .../fork/mapping/mapping-service.ts | 29 +- .../workspaces/fork/mapping/mapping-store.ts | 19 +- .../workspaces/fork/mapping/resources.test.ts | 95 ++++ .../lib/workspaces/fork/mapping/resources.ts | 324 ++++++++--- .../fork/promote/cleared-refs.test.ts | 464 +++++++++++++++ .../workspaces/fork/promote/cleared-refs.ts | 211 +++++++ .../fork/promote/copy-unmapped.test.ts | 381 +++++++++++++ .../workspaces/fork/promote/copy-unmapped.ts | 335 +++++++++++ .../fork/promote/promote-plan.test.ts | 74 ++- .../workspaces/fork/promote/promote-plan.ts | 88 ++- .../lib/workspaces/fork/promote/promote.ts | 147 ++++- .../workspaces/fork/remap/reference-scan.ts | 57 ++ .../fork/remap/remap-content-refs.test.ts | 194 +++++++ .../fork/remap/remap-content-refs.ts | 127 +++++ .../lib/workspaces/fork/remap/remap-files.ts | 24 + .../fork/remap/remap-references.test.ts | 121 ++++ .../workspaces/fork/remap/remap-references.ts | 77 ++- 53 files changed, 6359 insertions(+), 478 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-file-tree/fork-file-tree.test.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-file-tree/fork-file-tree.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-resource-picker/fork-resource-picker.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/cleared-refs-list.test.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/cleared-refs-list.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/copy-reconciliation.test.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/copy-reconciliation.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/dependent-value.test.ts create mode 100644 apps/sim/lib/workspaces/fork/copy/cleanup-failed.test.ts create mode 100644 apps/sim/lib/workspaces/fork/copy/cleanup-failed.ts create mode 100644 apps/sim/lib/workspaces/fork/copy/content-copy-runner.test.ts create mode 100644 apps/sim/lib/workspaces/fork/copy/copy-resources.test.ts create mode 100644 apps/sim/lib/workspaces/fork/copy/workflow-id-map.test.ts create mode 100644 apps/sim/lib/workspaces/fork/copy/workflow-id-map.ts create mode 100644 apps/sim/lib/workspaces/fork/mapping/resources.test.ts create mode 100644 apps/sim/lib/workspaces/fork/promote/cleared-refs.test.ts create mode 100644 apps/sim/lib/workspaces/fork/promote/cleared-refs.ts create mode 100644 apps/sim/lib/workspaces/fork/promote/copy-unmapped.test.ts create mode 100644 apps/sim/lib/workspaces/fork/promote/copy-unmapped.ts create mode 100644 apps/sim/lib/workspaces/fork/remap/reference-scan.ts create mode 100644 apps/sim/lib/workspaces/fork/remap/remap-content-refs.test.ts create mode 100644 apps/sim/lib/workspaces/fork/remap/remap-content-refs.ts diff --git a/apps/sim/app/api/workspaces/[id]/background-work/route.ts b/apps/sim/app/api/workspaces/[id]/background-work/route.ts index 066ba5c4a8b..e7d2862ac47 100644 --- a/apps/sim/app/api/workspaces/[id]/background-work/route.ts +++ b/apps/sim/app/api/workspaces/[id]/background-work/route.ts @@ -5,7 +5,7 @@ import { parseRequest } from '@/lib/api/server' import { getSession } from '@/lib/auth' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { listSurfacedBackgroundWork } from '@/lib/workspaces/fork/background-work/store' -import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils' +import { assertWorkspaceAdminAccess } from '@/lib/workspaces/fork/lineage/authz' export const GET = withRouteHandler( async (req: NextRequest, context: { params: Promise<{ id: string }> }) => { @@ -18,13 +18,9 @@ export const GET = withRouteHandler( if (!parsed.success) return parsed.response const { id } = parsed.data.params - const access = await checkWorkspaceAccess(id, session.user.id) - if (!access.exists) { - return NextResponse.json({ error: 'Workspace not found' }, { status: 404 }) - } - if (!access.canAdmin) { - return NextResponse.json({ error: 'Forbidden' }, { status: 403 }) - } + // The fork Activity feed is a fork feature: gate it behind the same forking-enabled + + // workspace-admin check the other fork routes use, instead of a bare access check. + await assertWorkspaceAdminAccess(id, session.user.id) const rows = await listSurfacedBackgroundWork(db, id) return NextResponse.json({ diff --git a/apps/sim/app/api/workspaces/[id]/fork/diff/route.ts b/apps/sim/app/api/workspaces/[id]/fork/diff/route.ts index 4631e16fda2..e717958e081 100644 --- a/apps/sim/app/api/workspaces/[id]/fork/diff/route.ts +++ b/apps/sim/app/api/workspaces/[id]/fork/diff/route.ts @@ -1,4 +1,6 @@ import { db } from '@sim/db' +import { workflow } from '@sim/db/schema' +import { eq } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { getForkDiffContract } from '@/lib/api/contracts/workspace-fork' import { parseRequest } from '@/lib/api/server' @@ -16,6 +18,8 @@ import { forkDependentValueKey, loadForkDependentValues, } from '@/lib/workspaces/fork/mapping/dependent-value-store' +import { listForkResourceCandidates } from '@/lib/workspaces/fork/mapping/resources' +import { collectForkClearedRefCandidates } from '@/lib/workspaces/fork/promote/cleared-refs' import { computeForkPromotePlan } from '@/lib/workspaces/fork/promote/promote-plan' import { buildForkBlockIdResolver } from '@/lib/workspaces/fork/remap/block-identity' import { readTargetDraftDependentValue } from '@/lib/workspaces/fork/remap/remap-references' @@ -63,10 +67,17 @@ export const GET = withRouteHandler( const replaceTargetIds = plan.items .filter((item) => item.mode === 'replace') .map((item) => item.targetWorkflowId) - const [storedValues, targetDraftByWorkflow] = await Promise.all([ - loadForkDependentValues(db, auth.edge.childWorkspaceId, replaceTargetIds), - loadTargetDraftSubBlocks(db, replaceTargetIds), - ]) + const [storedValues, targetDraftByWorkflow, sourceCandidates, sourceWorkflowRows] = + await Promise.all([ + loadForkDependentValues(db, auth.edge.childWorkspaceId, replaceTargetIds), + loadTargetDraftSubBlocks(db, replaceTargetIds), + // Source resource labels (per kind) + workflow names, for the cleared-ref list's display. + listForkResourceCandidates(db, auth.sourceWorkspaceId), + db + .select({ id: workflow.id, name: workflow.name }) + .from(workflow) + .where(eq(workflow.workspaceId, auth.sourceWorkspaceId)), + ]) const storedByKey = new Map( storedValues.map((entry) => [ forkDependentValueKey(entry.targetWorkflowId, entry.targetBlockId, entry.subBlockKey), @@ -108,6 +119,24 @@ export const GET = withRouteHandler( ), })) + // References this sync will blank in the target (per block/field), for the pre-sync cleared-ref + // list. Labels resolve from the source candidate lists + workflow names loaded above. + const sourceLabels = new Map() + for (const [kind, candidates] of Object.entries(sourceCandidates)) { + for (const candidate of candidates) + sourceLabels.set(`${kind}:${candidate.id}`, candidate.label) + } + const sourceWorkflowNames = new Map(sourceWorkflowRows.map((row) => [row.id, row.name])) + const clearedRefs = collectForkClearedRefCandidates({ + items: plan.items, + sourceStates, + resolver: plan.resolver, + workflowIdMap: plan.workflowIdMap, + resolveBlockId, + sourceLabels, + sourceWorkflowNames, + }) + const toRef = (reference: (typeof plan.unmappedRequired)[number]) => ({ kind: reference.kind, sourceId: reference.sourceId, @@ -155,6 +184,8 @@ export const GET = withRouteHandler( inlineSecretSources: plan.inlineSecretSources, dependentReconfigs, resourceUsages: collectForkResourceUsages(plan.items, sourceStates), + copyableUnmapped: plan.copyableUnmapped, + clearedRefs, }) } ) diff --git a/apps/sim/app/api/workspaces/[id]/fork/promote/route.ts b/apps/sim/app/api/workspaces/[id]/fork/promote/route.ts index eba0e7b1c8f..6b6228620eb 100644 --- a/apps/sim/app/api/workspaces/[id]/fork/promote/route.ts +++ b/apps/sim/app/api/workspaces/[id]/fork/promote/route.ts @@ -25,7 +25,7 @@ export const POST = withRouteHandler( const parsed = await parseRequest(promoteForkContract, req, context) if (!parsed.success) return parsed.response const { id } = parsed.data.params - const { otherWorkspaceId, direction, dependentValues } = parsed.data.body + const { otherWorkspaceId, direction, dependentValues, copyResources } = parsed.data.body const auth = await assertCanPromote(id, otherWorkspaceId, direction, session.user.id) @@ -36,6 +36,7 @@ export const POST = withRouteHandler( direction, userId: session.user.id, dependentValues, + copyResources, requestId, }) diff --git a/apps/sim/app/api/workspaces/[id]/fork/route.ts b/apps/sim/app/api/workspaces/[id]/fork/route.ts index 27bea2f1b99..b3183465a6c 100644 --- a/apps/sim/app/api/workspaces/[id]/fork/route.ts +++ b/apps/sim/app/api/workspaces/[id]/fork/route.ts @@ -39,7 +39,7 @@ export const POST = withRouteHandler( knowledgeBases: copy?.knowledgeBases ?? [], customTools: copy?.customTools ?? [], skills: copy?.skills ?? [], - mcpServers: copy?.mcpServers ?? [], + workflowMcpServers: copy?.workflowMcpServers ?? [], }, requestId, }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-editor-mentions.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-editor-mentions.ts index edd533d1dd0..5ac2ce036fc 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-editor-mentions.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-editor-mentions.ts @@ -5,6 +5,8 @@ import { useMarkdownMentions } from './use-markdown-mentions' interface UseEditorMentionsOptions { /** Whether a chip can Cmd/Ctrl-click to its resource. On for the file viewer, off in modal fields. */ navigable?: boolean + /** Force the `@` insertion menu off even with a workspace; existing tags still render. */ + disableTagging?: boolean } /** @@ -20,17 +22,18 @@ export function useEditorMentions( const [active, setActive] = useState(false) const items = useMarkdownMentions(workspaceId, { enabled: active }) const navigable = options?.navigable ?? false + const disableTagging = options?.disableTagging ?? false useEffect(() => { if (!editor) return - const hasWorkspace = Boolean(workspaceId) - editor.storage.mention.enabled = hasWorkspace + const taggingOn = Boolean(workspaceId) && !disableTagging + editor.storage.mention.enabled = taggingOn editor.storage.mention.navigable = navigable - editor.storage.mention.onOpen = hasWorkspace ? () => setActive(true) : null + editor.storage.mention.onOpen = taggingOn ? () => setActive(true) : null return () => { editor.storage.mention.onOpen = null } - }, [editor, workspaceId, navigable]) + }, [editor, workspaceId, navigable, disableTagging]) useEffect(() => { editor?.storage.mention.store.set(items) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index f8a8286d221..e6993428142 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -55,6 +55,8 @@ interface RichMarkdownEditorProps { streamIsIncremental?: boolean disableStreamingAutoScroll?: boolean previewContextKey?: string + /** Disable the `@` tag-insertion menu (existing tags still render). Defaults off — the file editor keeps tagging. */ + disableTagging?: boolean } /** Inline WYSIWYG markdown editor: agent output streams in read-only, then the same instance becomes editable on settle. */ @@ -71,6 +73,7 @@ export const RichMarkdownEditor = memo(function RichMarkdownEditor({ streamIsIncremental, disableStreamingAutoScroll = false, previewContextKey, + disableTagging, }: RichMarkdownEditorProps) { const { content, @@ -112,6 +115,7 @@ export const RichMarkdownEditor = memo(function RichMarkdownEditor({ autoFocus={autoFocus} streamIsIncremental={streamIsIncremental} disableStreamingAutoScroll={disableStreamingAutoScroll} + disableTagging={disableTagging} onChange={setDraftContent} onSaveShortcut={saveImmediately} /> @@ -130,6 +134,7 @@ interface LoadedRichMarkdownEditorProps { /** See {@link RichMarkdownEditorProps.streamIsIncremental}. */ streamIsIncremental?: boolean disableStreamingAutoScroll?: boolean + disableTagging?: boolean onChange: (markdown: string) => void onSaveShortcut: () => Promise } @@ -154,6 +159,7 @@ export function LoadedRichMarkdownEditor({ autoFocus, streamIsIncremental, disableStreamingAutoScroll, + disableTagging, onChange, onSaveShortcut, }: LoadedRichMarkdownEditorProps) { @@ -338,7 +344,7 @@ export function LoadedRichMarkdownEditor({ } }, [editor]) - useEditorMentions(editor, workspaceId, { navigable: true }) + useEditorMentions(editor, workspaceId, { navigable: true, disableTagging }) const wasStreamingRef = useRef(streamingAtMountRef.current) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-field.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-field.tsx index 0df1030a7a8..202ed9b4651 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-field.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-field.tsx @@ -38,6 +38,8 @@ interface RichMarkdownFieldProps { error?: boolean /** Enables the `@` mention menu scoped to this workspace. Omit to disable mentions. */ workspaceId?: string + /** Force the `@` tag-insertion menu off even with a workspace set (existing tags still render). */ + disableTagging?: boolean /** * Intercepts a plain-text paste before the editor handles it. Return `true` to consume the paste * (e.g. a full document the host destructures elsewhere); `false` to fall through to normal @@ -62,6 +64,7 @@ function LoadedRichMarkdownField({ maxHeight = 360, error = false, workspaceId, + disableTagging, onPasteText, }: RichMarkdownFieldProps) { const containerRef = useRef(null) @@ -166,7 +169,7 @@ function LoadedRichMarkdownField({ if (editor.isEditable !== !disabled) editor.setEditable(!disabled) }, [editor, value, isStreaming, disabled]) - useEditorMentions(editor, workspaceId) + useEditorMentions(editor, workspaceId, { disableTagging }) return (
MAX_DESCRIPTION_LENGTH} workspaceId={workspaceId} + disableTagging /> diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-activity-panel/fork-activity-panel.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-activity-panel/fork-activity-panel.tsx index d4327c9d897..97bd5908068 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-activity-panel/fork-activity-panel.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-activity-panel/fork-activity-panel.tsx @@ -122,7 +122,7 @@ function jobReport(job: BackgroundWorkItem): JobReport { addGroup('Files', m.fileNames) addGroup('Custom tools', m.customToolNames) addGroup('Skills', m.skillNames) - addGroup('MCP servers', m.mcpServerNames) + addGroup('Workflow MCP servers', m.workflowMcpServerNames) // Pre-names entries fall back to the per-kind counts. if (groups.length === 0) { const counts = [ diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-file-tree/fork-file-tree.test.ts b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-file-tree/fork-file-tree.test.ts new file mode 100644 index 00000000000..39fa599a965 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-file-tree/fork-file-tree.test.ts @@ -0,0 +1,28 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { groupForkFilesIntoFolders } from '@/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-file-tree/fork-file-tree' + +describe('groupForkFilesIntoFolders', () => { + it('groups files under their folder and lifts un-foldered files to the root bucket', () => { + const { folders, rootFiles } = groupForkFilesIntoFolders([ + { id: 'f1', label: 'b.png', folderId: 'fld-1', folderName: 'Images' }, + { id: 'f2', label: 'a.png', folderId: 'fld-1', folderName: 'Images' }, + { id: 'f3', label: 'root.txt', folderId: null, folderName: null }, + { id: 'f4', label: 'doc.pdf', folderId: 'fld-2', folderName: 'Docs' }, + ]) + // Folders are sorted by name; each folder's files are sorted by label. + expect(folders.map((folder) => folder.name)).toEqual(['Docs', 'Images']) + expect(folders[1].files.map((file) => file.label)).toEqual(['a.png', 'b.png']) + expect(rootFiles.map((file) => file.label)).toEqual(['root.txt']) + }) + + it('treats a file whose folder was deleted (id set, name null) as a root file', () => { + const { folders, rootFiles } = groupForkFilesIntoFolders([ + { id: 'f1', label: 'orphan.png', folderId: 'fld-deleted', folderName: null }, + ]) + expect(folders).toEqual([]) + expect(rootFiles.map((file) => file.id)).toEqual(['f1']) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-file-tree/fork-file-tree.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-file-tree/fork-file-tree.tsx new file mode 100644 index 00000000000..76fa5a5e2b1 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-file-tree/fork-file-tree.tsx @@ -0,0 +1,197 @@ +'use client' + +import { useId, useState } from 'react' +import { Checkbox, ChevronDown, cn } from '@sim/emcn' + +export interface ForkFileTreeItem { + id: string + label: string +} + +export interface ForkFileTreeFolder { + id: string + name: string + files: ForkFileTreeItem[] +} + +/** A flat copyable file with its folder grouping, before {@link groupForkFilesIntoFolders}. */ +export interface ForkFlatFile { + id: string + label: string + folderId: string | null + folderName: string | null +} + +/** + * Group flat files into folders (sorted by name, each file sorted by label) plus a root bucket + * for files with no folder. A file whose folder id is set but whose name is null (its folder was + * deleted) falls into the root bucket, so it stays selectable rather than hiding under a phantom. + */ +export function groupForkFilesIntoFolders(files: ForkFlatFile[]): { + folders: ForkFileTreeFolder[] + rootFiles: ForkFileTreeItem[] +} { + const folderById = new Map() + const rootFiles: ForkFileTreeItem[] = [] + for (const file of files) { + const item: ForkFileTreeItem = { id: file.id, label: file.label } + if (file.folderId && file.folderName) { + let folder = folderById.get(file.folderId) + if (!folder) { + folder = { id: file.folderId, name: file.folderName, files: [] } + folderById.set(file.folderId, folder) + } + folder.files.push(item) + } else { + rootFiles.push(item) + } + } + const folders = Array.from(folderById.values()).sort((a, b) => a.name.localeCompare(b.name)) + for (const folder of folders) folder.files.sort((a, b) => a.label.localeCompare(b.label)) + rootFiles.sort((a, b) => a.label.localeCompare(b.label)) + return { folders, rootFiles } +} + +interface ForkFileTreeProps { + folders: ForkFileTreeFolder[] + rootFiles: ForkFileTreeItem[] + isSelected: (fileId: string) => boolean + onToggleFile: (fileId: string, checked: boolean) => void + /** Toggle every file in a folder at once (the folder-level select-all). */ + onToggleMany: (fileIds: string[], checked: boolean) => void + disabled?: boolean +} + +/** + * Folder ▸ file collapsible tree shared by the fork picker and the sync copy selector, so both + * surfaces group files identically. Each folder is a tri-state select-all header that expands to + * its files; files with no folder render at the top level. Files are the only copyable kind that + * nests - tables, knowledge bases, custom tools, and skills stay flat at the top level. + */ +export function ForkFileTree({ + folders, + rootFiles, + isSelected, + onToggleFile, + onToggleMany, + disabled = false, +}: ForkFileTreeProps) { + return ( +
+ {folders.map((folder) => ( + + ))} + {rootFiles.map((file) => ( + + ))} +
+ ) +} + +interface ForkFileFolderRowProps { + folder: ForkFileTreeFolder + isSelected: (fileId: string) => boolean + onToggleFile: (fileId: string, checked: boolean) => void + onToggleMany: (fileIds: string[], checked: boolean) => void + disabled: boolean +} + +function ForkFileFolderRow({ + folder, + isSelected, + onToggleFile, + onToggleMany, + disabled, +}: ForkFileFolderRowProps) { + const [expanded, setExpanded] = useState(false) + const fileIds = folder.files.map((file) => file.id) + const total = fileIds.length + const selectedCount = fileIds.filter(isSelected).length + const headerState = selectedCount === 0 ? false : selectedCount === total ? true : 'indeterminate' + + return ( +
+
+ onToggleMany(fileIds, headerState !== true)} + disabled={disabled} + /> + +
+ {expanded ? ( +
+ {folder.files.map((file) => ( + + ))} +
+ ) : null} +
+ ) +} + +interface ForkFileRowProps { + file: ForkFileTreeItem + checked: boolean + onToggle: (fileId: string, checked: boolean) => void + disabled: boolean +} + +function ForkFileRow({ file, checked, onToggle, disabled }: ForkFileRowProps) { + const itemId = useId() + return ( + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-resource-picker/fork-resource-picker.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-resource-picker/fork-resource-picker.tsx new file mode 100644 index 00000000000..bd847ee19a3 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-resource-picker/fork-resource-picker.tsx @@ -0,0 +1,207 @@ +'use client' + +import { useId, useMemo, useState } from 'react' +import { Checkbox, ChevronDown, ChipInput, cn } from '@sim/emcn' +import { Search } from 'lucide-react' +import { + ForkFileTree, + type ForkFlatFile, + groupForkFilesIntoFolders, +} from '@/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-file-tree/fork-file-tree' + +/** A flat copyable resource (table / KB / custom tool / skill / MCP server) in the picker. */ +export interface ForkResourcePickerItem { + id: string + label: string +} + +/** Show the inline search once a kind has more entries than fit comfortably. */ +const SEARCH_THRESHOLD = 8 + +interface ResourceKindRowProps { + label: string + items: ForkResourcePickerItem[] + selected: Set + onToggleAll: (selectAll: boolean) => void + onToggleItem: (id: string, checked: boolean) => void + disabled?: boolean +} + +/** + * One expandable resource kind in the fork / sync copy picker: a tri-state "select all" header + * (count of selected / total) plus, when expanded, a searchable scrollable list of individual + * resources so the user can copy a specific subset. Shared by the fork modal's "Copy resources" + * and the sync modal's "Copy resources" so the two surfaces stay identical. Files nest in a + * folder tree instead - use {@link FileKindRow}. + */ +export function ResourceKindRow({ + label, + items, + selected, + onToggleAll, + onToggleItem, + disabled = false, +}: ResourceKindRowProps) { + const [expanded, setExpanded] = useState(false) + const [query, setQuery] = useState('') + const fieldId = useId() + + const total = items.length + const selectedCount = selected.size + const headerState = selectedCount === 0 ? false : selectedCount === total ? true : 'indeterminate' + + const filtered = useMemo(() => { + const trimmed = query.trim().toLowerCase() + if (!trimmed) return items + return items.filter((item) => item.label.toLowerCase().includes(trimmed)) + }, [items, query]) + + return ( +
+
+ onToggleAll(headerState !== true)} + disabled={disabled} + /> + +
+ + {expanded ? ( +
+ {total > SEARCH_THRESHOLD ? ( + setQuery(event.target.value)} + placeholder={`Search ${label.toLowerCase()}`} + disabled={disabled} + /> + ) : null} +
+ {filtered.map((item) => { + const isChecked = selected.has(item.id) + const itemId = `${fieldId}-${item.id}` + return ( + + ) + })} + {filtered.length === 0 ? ( +

No matches

+ ) : null} +
+
+ ) : null} +
+ ) +} + +interface FileKindRowProps { + label: string + files: ForkFlatFile[] + selected: Set + onToggleAll: (selectAll: boolean) => void + onToggleItem: (id: string, checked: boolean) => void + onToggleMany: (ids: string[], checked: boolean) => void + disabled?: boolean +} + +/** + * The Files kind: a tri-state "select all" header (count selected / total) that expands to a + * folder ▸ file tree, so the user can copy a whole folder or a specific file. Files are the only + * copyable kind that nests; every other kind uses the flat {@link ResourceKindRow}. Shared by the + * fork and sync copy pickers so both group files identically. + */ +export function FileKindRow({ + label, + files, + selected, + onToggleAll, + onToggleItem, + onToggleMany, + disabled = false, +}: FileKindRowProps) { + const [expanded, setExpanded] = useState(false) + + const total = files.length + const selectedCount = files.filter((file) => selected.has(file.id)).length + const headerState = selectedCount === 0 ? false : selectedCount === total ? true : 'indeterminate' + + const { folders, rootFiles } = useMemo(() => groupForkFilesIntoFolders(files), [files]) + + return ( +
+
+ onToggleAll(headerState !== true)} + disabled={disabled} + /> + +
+ + {expanded ? ( +
+ selected.has(id)} + onToggleFile={onToggleItem} + onToggleMany={onToggleMany} + disabled={disabled} + /> +
+ ) : null} +
+ ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-workspace-modal/fork-workspace-modal.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-workspace-modal/fork-workspace-modal.tsx index 3ed9a6c7b64..3d2600544bc 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-workspace-modal/fork-workspace-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-workspace-modal/fork-workspace-modal.tsx @@ -1,9 +1,7 @@ 'use client' -import { useEffect, useId, useMemo, useState } from 'react' +import { useEffect, useMemo, useState } from 'react' import { - Checkbox, - ChevronDown, Chip, ChipConfirmModal, ChipCopyInput, @@ -15,19 +13,19 @@ import { type ChipModalFooterSlotAction, ChipModalHeader, ChipModalTabs, - cn, Tooltip, toast, } from '@sim/emcn' import { getErrorMessage } from '@sim/utils/errors' -import { Search } from 'lucide-react' +import { AlertTriangle } from 'lucide-react' import { useRouter } from 'next/navigation' -import type { - ForkCopyableResource, - GetForkResourcesResponse, -} from '@/lib/api/contracts/workspace-fork' +import type { GetForkResourcesResponse } from '@/lib/api/contracts/workspace-fork' import { SettingsSection } from '@/app/workspace/[workspaceId]/settings/components/settings-section/settings-section' import { ForkActivityPanel } from '@/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-activity-panel/fork-activity-panel' +import { + FileKindRow, + ResourceKindRow, +} from '@/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-resource-picker/fork-resource-picker' import { type ForkDirection, useForkResources, @@ -57,127 +55,24 @@ const RESOURCE_KINDS: ReadonlyArray<{ key: ResourceKey; label: string }> = [ { key: 'knowledgeBases', label: 'Knowledge bases' }, { key: 'customTools', label: 'Custom tools' }, { key: 'skills', label: 'Skills' }, - { key: 'mcpServers', label: 'MCP servers' }, + { key: 'workflowMcpServers', label: 'Workflow MCP servers' }, ] -/** Show the inline search once a kind has more entries than fit comfortably. */ -const SEARCH_THRESHOLD = 8 - const emptySelection = (): ResourceSelection => ({ files: new Set(), tables: new Set(), knowledgeBases: new Set(), customTools: new Set(), skills: new Set(), - mcpServers: new Set(), + workflowMcpServers: new Set(), }) -/** - * One expandable resource kind in the fork picker: a tri-state "select all" header - * (count of selected / total) plus, when expanded, a searchable scrollable list of - * individual resources so the user can copy a specific subset. - */ -function ResourceKindRow({ - label, - items, - selected, - onToggleAll, - onToggleItem, - disabled, -}: { - label: string - items: ForkCopyableResource[] - selected: Set - onToggleAll: (selectAll: boolean) => void - onToggleItem: (id: string, checked: boolean) => void - disabled: boolean -}) { - const [expanded, setExpanded] = useState(false) - const [query, setQuery] = useState('') - const fieldId = useId() - - const total = items.length - const selectedCount = selected.size - const headerState = selectedCount === 0 ? false : selectedCount === total ? true : 'indeterminate' - - const filtered = useMemo(() => { - const trimmed = query.trim().toLowerCase() - if (!trimmed) return items - return items.filter((item) => item.label.toLowerCase().includes(trimmed)) - }, [items, query]) - - return ( -
-
- onToggleAll(headerState !== true)} - disabled={disabled} - /> - -
- - {expanded ? ( -
- {total > SEARCH_THRESHOLD ? ( - setQuery(event.target.value)} - placeholder={`Search ${label.toLowerCase()}`} - disabled={disabled} - /> - ) : null} -
- {filtered.map((item) => { - const isChecked = selected.has(item.id) - const itemId = `${fieldId}-${item.id}` - return ( - - ) - })} - {filtered.length === 0 ? ( -

No matches

- ) : null} -
-
- ) : null} -
- ) +const fullSelection = (data: GetForkResourcesResponse): ResourceSelection => { + const selection = emptySelection() + for (const kind of RESOURCE_KINDS) { + selection[kind.key] = new Set((data[kind.key] ?? []).map((item) => item.id)) + } + return selection } /** @@ -200,6 +95,7 @@ export function ForkWorkspaceModal({ const resources = useForkResources(sourceWorkspaceId, open) const [name, setName] = useState('') const [selected, setSelected] = useState(emptySelection) + const [defaulted, setDefaulted] = useState(false) const [error, setError] = useState(null) const [activeTab, setActiveTab] = useState<'config' | 'activity'>('config') @@ -210,6 +106,7 @@ export function ForkWorkspaceModal({ if (open) { setName(`${sourceWorkspaceName} (fork)`) setSelected(emptySelection()) + setDefaulted(false) setError(null) setActiveTab('config') setForkedWorkspace(null) @@ -217,6 +114,12 @@ export function ForkWorkspaceModal({ } }, [open, sourceWorkspaceName]) + useEffect(() => { + if (!open || !resources.data || defaulted) return + setDefaulted(true) + setSelected(fullSelection(resources.data)) + }, [open, resources.data, defaulted]) + const isForking = forkWorkspace.isPending const availableKinds = useMemo( @@ -224,6 +127,15 @@ export function ForkWorkspaceModal({ [resources.data] ) + const hasDeselection = useMemo( + () => + defaulted && + availableKinds.some( + (kind) => selected[kind.key].size < (resources.data?.[kind.key]?.length ?? 0) + ), + [defaulted, availableKinds, selected, resources.data] + ) + // A fork always produces a usable workspace: deployed workflows are copied, and // when the source has none, create-fork seeds a blank starter workflow (plus any // selected resources). So forking is never blocked - we just set expectations when @@ -239,11 +151,14 @@ export function ForkWorkspaceModal({ return } const trimmed = name.trim() - if (!trimmed || isForking) return + // Block until the resources query resolves: building `copy` from an unloaded `resources.data` + // would send an empty selection and silently clear every reference in the fork. The Fork + // action is disabled in this state too; this is the defense-in-depth guard. + if (!trimmed || isForking || !resources.data) return setError(null) - const copy = resources.data - ? Object.fromEntries(RESOURCE_KINDS.map((kind) => [kind.key, Array.from(selected[kind.key])])) - : undefined + const copy = Object.fromEntries( + RESOURCE_KINDS.map((kind) => [kind.key, Array.from(selected[kind.key])]) + ) forkWorkspace.mutate( { workspaceId: sourceWorkspaceId, body: { name: trimmed, copy } }, { @@ -360,34 +275,83 @@ export function ForkWorkspaceModal({ {availableKinds.length > 0 ? (
- {availableKinds.map((kind) => ( - - setSelected((prev) => ({ - ...prev, - [kind.key]: selectAll - ? new Set((resources.data?.[kind.key] ?? []).map((item) => item.id)) - : new Set(), - })) - } - onToggleItem={(id, checked) => - setSelected((prev) => { - const next = new Set(prev[kind.key]) - if (checked) next.add(id) - else next.delete(id) - return { ...prev, [kind.key]: next } - }) - } - disabled={isForking} - /> - ))} -

- Unselected resources leave their workflow fields empty in the fork. -

+ {availableKinds.map((kind) => + kind.key === 'files' ? ( + + setSelected((prev) => ({ + ...prev, + files: selectAll + ? new Set((resources.data?.files ?? []).map((item) => item.id)) + : new Set(), + })) + } + onToggleItem={(id, checked) => + setSelected((prev) => { + const next = new Set(prev.files) + if (checked) next.add(id) + else next.delete(id) + return { ...prev, files: next } + }) + } + onToggleMany={(ids, checked) => + setSelected((prev) => { + const next = new Set(prev.files) + for (const id of ids) { + if (checked) next.add(id) + else next.delete(id) + } + return { ...prev, files: next } + }) + } + disabled={isForking} + /> + ) : ( + + setSelected((prev) => ({ + ...prev, + [kind.key]: selectAll + ? new Set( + (resources.data?.[kind.key] ?? []).map((item) => item.id) + ) + : new Set(), + })) + } + onToggleItem={(id, checked) => + setSelected((prev) => { + const next = new Set(prev[kind.key]) + if (checked) next.add(id) + else next.delete(id) + return { ...prev, [kind.key]: next } + }) + } + disabled={isForking} + /> + ) + )} + {hasDeselection ? ( +
+ + + Some resources are not selected — references to them in your workflows + will be cleared in the fork. + +
+ ) : ( +

+ Everything referenced by your workflows is copied. Deselect a resource to + skip it — its references will be cleared. +

+ )}
) : null} @@ -414,8 +378,14 @@ export function ForkWorkspaceModal({ : { label: isForking ? 'Forking...' : 'Fork', onClick: handleSubmit, - // At the cap the button stays clickable (no name needed) so it can route to upgrade. - disabled: isForking || (canFork && !name.trim()), + // At the cap the button stays clickable (no name needed) so it can route to + // upgrade. Otherwise it needs a name AND the resources query loaded - forking + // before `resources.data` arrives would clear every reference (P1-C). + disabled: isForking || (canFork && (!name.trim() || !resources.data)), + disabledTooltip: + canFork && name.trim() && !resources.data + ? 'Loading workspace resources…' + : undefined, } } /> @@ -437,7 +407,15 @@ export function ForkWorkspaceModal({ pending: rollback.isPending, pendingLabel: 'Rolling back...', }} - /> + > +
+ + + Resources copied into this workspace during syncs may remain afterward — rollback + restores workflows to their prior versions but does not remove copied resources. + +
+ ) } diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/cleared-refs-list.test.ts b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/cleared-refs-list.test.ts new file mode 100644 index 00000000000..f7ebed9a5da --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/cleared-refs-list.test.ts @@ -0,0 +1,126 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import type { ForkClearedRef } from '@/lib/api/contracts/workspace-fork' +import { selectVisibleClearedRefs } from '@/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/cleared-refs-list' + +const ref = (overrides: Partial): ForkClearedRef => ({ + targetWorkflowId: 'wf-tgt', + workflowName: 'Workflow', + blockId: 'block-1', + blockLabel: 'Block', + fieldLabel: 'Field', + kind: 'credential', + sourceId: 'src-1', + sourceLabel: 'Source', + cause: 'reference', + parentKind: null, + parentSourceId: null, + ...overrides, +}) + +// The modal's predicate is `mapped || copied`; here we model each disposition separately so the +// document-under-KB case is exercised for BOTH a copied parent and a mapped parent. +const resolvedKeys = (...keys: string[]) => { + const set = new Set(keys) + return (kind: string, sourceId: string) => set.has(`${kind}:${sourceId}`) +} + +const documentDependent = ref({ + cause: 'dependent', + fieldLabel: 'Document', + kind: 'knowledge-base', + sourceId: 'kb-1', + parentKind: 'knowledge-base', + parentSourceId: 'kb-1', +}) + +describe('selectVisibleClearedRefs', () => { + it('drops a document dependent when its parent KB is selected for copy', () => { + expect( + selectVisibleClearedRefs([documentDependent], resolvedKeys('knowledge-base:kb-1')) + ).toEqual([]) + }) + + it('drops a document dependent when its parent KB is mapped', () => { + // Map vs copy both resolve the parent through the same predicate; modeled identically here. + expect( + selectVisibleClearedRefs([documentDependent], resolvedKeys('knowledge-base:kb-1')) + ).toEqual([]) + }) + + it('keeps a document dependent while its parent KB is neither mapped nor copied', () => { + expect(selectVisibleClearedRefs([documentDependent], resolvedKeys())).toEqual([ + documentDependent, + ]) + }) + + it('keeps a credential-anchored dependent even when the credential is mapped (label still clears)', () => { + const labelDependent = ref({ + cause: 'dependent', + fieldLabel: 'Label', + kind: 'credential', + sourceId: 'cred-1', + parentKind: 'credential', + parentSourceId: 'cred-1', + }) + // A mapped credential remaps to a different account, so the account-scoped label is cleared + // regardless - the entry must stay even though the parent is "resolved". + expect(selectVisibleClearedRefs([labelDependent], resolvedKeys('credential:cred-1'))).toEqual([ + labelDependent, + ]) + expect(selectVisibleClearedRefs([labelDependent], resolvedKeys())).toEqual([labelDependent]) + }) + + it('keeps a table-anchored dependent even when the table is copied/mapped (column still clears)', () => { + const columnDependent = ref({ + cause: 'dependent', + fieldLabel: 'Column', + kind: 'table', + sourceId: 'tbl-1', + parentKind: 'table', + parentSourceId: 'tbl-1', + }) + expect(selectVisibleClearedRefs([columnDependent], resolvedKeys('table:tbl-1'))).toEqual([ + columnDependent, + ]) + }) + + it('drops the parent KB reference AND its child document together when the KB is resolved', () => { + const kbReference = ref({ + cause: 'reference', + fieldLabel: 'Knowledge Base', + kind: 'knowledge-base', + sourceId: 'kb-1', + }) + expect( + selectVisibleClearedRefs( + [kbReference, documentDependent], + resolvedKeys('knowledge-base:kb-1') + ) + ).toEqual([]) + }) + + it('applies the same predicate to a reference entry (drops resolved, keeps unresolved)', () => { + const credentialReference = ref({ cause: 'reference', kind: 'credential', sourceId: 'cred-1' }) + expect( + selectVisibleClearedRefs([credentialReference], resolvedKeys('credential:cred-1')) + ).toEqual([]) + expect(selectVisibleClearedRefs([credentialReference], resolvedKeys())).toEqual([ + credentialReference, + ]) + }) + + it('always keeps a workflow reference (it cannot be resolved in the modal)', () => { + const workflowReference = ref({ cause: 'workflow', kind: 'workflow', sourceId: 'wf-other' }) + expect( + selectVisibleClearedRefs([workflowReference], resolvedKeys('workflow:wf-other')) + ).toEqual([workflowReference]) + }) + + it('keeps a dependent missing its parent identity (defensive)', () => { + const orphanDependent = ref({ cause: 'dependent', parentKind: null, parentSourceId: null }) + expect(selectVisibleClearedRefs([orphanDependent], resolvedKeys())).toEqual([orphanDependent]) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/cleared-refs-list.ts b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/cleared-refs-list.ts new file mode 100644 index 00000000000..f877a779f99 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/cleared-refs-list.ts @@ -0,0 +1,43 @@ +import type { ForkClearedRef } from '@/lib/api/contracts/workspace-fork' + +/** Whether a resource is resolved by the current selection (mapped to a target OR selected for copy). */ +export type ClearedRefResolvedPredicate = (kind: string, sourceId: string) => boolean + +/** + * Parent kinds whose dependent child is itself a resource carried alongside the parent, so + * resolving the parent (mapping or copying it) PRESERVES the child rather than clearing it - + * currently just knowledge bases: a referenced document is copied with its KB, or auto-copied into + * a mapped KB, and remapped in place. Any other parent's child (a credential's label, a table's + * column) is account/table-scoped and cleared whenever the parent is remapped (the engine's + * `clearDependentsOnRemap`), so those entries stay regardless of the parent's disposition. + */ +const PARENT_KINDS_THAT_PRESERVE_CHILD: ReadonlySet = new Set(['knowledge-base']) + +/** + * Narrow the diff's cleared-ref candidates to those still cleared under the live selection: + * - `reference`: drops off once its own resource is resolved (mapped or copied). + * - `dependent`: drops off once its PARENT resource (`parentKind`/`parentSourceId`) is resolved - + * using the SAME predicate as `reference` - but ONLY when the child follows that parent (a + * document under a KB). A credential- or table-anchored dependent is cleared on any parent + * remap, so it stays even after the parent is mapped. A dependent missing its parent stays. + * - `workflow`: always stays - a cross-workflow reference cannot be resolved in the modal. + * + * Pure so the reactive list is unit-testable independent of the modal's selection state. + */ +export function selectVisibleClearedRefs( + clearedRefs: ForkClearedRef[], + isResolved: ClearedRefResolvedPredicate +): ForkClearedRef[] { + return clearedRefs.filter((ref) => { + if (ref.cause === 'reference') return !isResolved(ref.kind, ref.sourceId) + if ( + ref.cause === 'dependent' && + ref.parentKind && + ref.parentSourceId && + PARENT_KINDS_THAT_PRESERVE_CHILD.has(ref.parentKind) + ) { + return !isResolved(ref.parentKind, ref.parentSourceId) + } + return true + }) +} diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/copy-reconciliation.test.ts b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/copy-reconciliation.test.ts new file mode 100644 index 00000000000..04b164512c2 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/copy-reconciliation.test.ts @@ -0,0 +1,124 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import type { ForkCopyableUnmapped, ForkMappingEntry } from '@/lib/api/contracts/workspace-fork' +import { + effectiveForkTarget, + forkCopyingKeys, + forkMappedCopyableKeys, + forkRefKey, + forkRequiredPending, + forkVisibleCopyables, + isForkRequiredComplete, +} from '@/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/copy-reconciliation' + +const entry = (overrides: Partial): ForkMappingEntry => ({ + kind: 'credential', + resourceType: 'oauth_credential', + sourceId: 'src', + sourceLabel: 'Src', + targetId: null, + suggested: false, + required: false, + candidates: [], + candidatesTruncated: false, + ...overrides, +}) + +const copyable = (overrides: Partial): ForkCopyableUnmapped => ({ + kind: 'knowledge-base', + sourceId: 'kb', + label: 'KB', + parentId: null, + parentLabel: null, + ...overrides, +}) + +describe('forkRefKey / effectiveForkTarget', () => { + it('shares the kind:sourceId keyspace for entries and copyables', () => { + expect(forkRefKey({ kind: 'knowledge-base', sourceId: 'kb-1' })).toBe('knowledge-base:kb-1') + expect(forkRefKey(entry({ kind: 'table', sourceId: 't-1' }))).toBe('table:t-1') + }) + + it('prefers an in-session override, else the persisted target, else empty', () => { + const e = entry({ kind: 'table', sourceId: 't-1', targetId: 'persisted' }) + expect(effectiveForkTarget(e, { 'table:t-1': 'in-session' })).toBe('in-session') + expect(effectiveForkTarget(e, {})).toBe('persisted') + expect(effectiveForkTarget(entry({ kind: 'table', sourceId: 't-1' }), {})).toBe('') + }) +}) + +describe('copy-vs-map reconciliation', () => { + it('drops a mapped copyable from the visible copy list (maps win over copy)', () => { + const entries = [ + entry({ kind: 'knowledge-base', sourceId: 'kb-1', targetId: 'kb-tgt' }), + entry({ kind: 'table', sourceId: 'tbl-1' }), + ] + const candidates = [ + copyable({ kind: 'knowledge-base', sourceId: 'kb-1' }), + copyable({ kind: 'table', sourceId: 'tbl-1' }), + ] + const mapped = forkMappedCopyableKeys(entries, {}) + expect(mapped.has('knowledge-base:kb-1')).toBe(true) + expect(mapped.has('table:tbl-1')).toBe(false) + expect(forkVisibleCopyables(candidates, mapped).map(forkRefKey)).toEqual(['table:tbl-1']) + }) + + it('an in-session mapping target also drops the copyable', () => { + const entries = [entry({ kind: 'knowledge-base', sourceId: 'kb-1' })] + const mapped = forkMappedCopyableKeys(entries, { 'knowledge-base:kb-1': 'kb-tgt' }) + expect( + forkVisibleCopyables([copyable({ kind: 'knowledge-base', sourceId: 'kb-1' })], mapped) + ).toEqual([]) + }) + + it('copyingKeys = the visible candidates that are selected for copy', () => { + const visible = [ + copyable({ kind: 'table', sourceId: 'tbl-1' }), + copyable({ kind: 'skill', sourceId: 'sk-1' }), + ] + expect([...forkCopyingKeys(visible, new Set(['table:tbl-1']))]).toEqual(['table:tbl-1']) + }) +}) + +describe('isForkRequiredComplete', () => { + it('a required ref is satisfied by a mapping target', () => { + const entries = [ + entry({ kind: 'credential', sourceId: 'c1', required: true, targetId: 'c-tgt' }), + ] + expect(isForkRequiredComplete(entries, {}, new Set())).toBe(true) + }) + + it('a required ref is satisfied by a copy selection (server accepts copy as resolving it)', () => { + const entries = [entry({ kind: 'knowledge-base', sourceId: 'kb-1', required: true })] + expect(isForkRequiredComplete(entries, {}, new Set(['knowledge-base:kb-1']))).toBe(true) + }) + + it('a required ref neither mapped nor copied blocks', () => { + const entries = [entry({ kind: 'credential', sourceId: 'c1', required: true })] + expect(isForkRequiredComplete(entries, {}, new Set())).toBe(false) + }) + + it('optional refs never block', () => { + const entries = [entry({ kind: 'table', sourceId: 't1', required: false })] + expect(isForkRequiredComplete(entries, {}, new Set())).toBe(true) + }) +}) + +describe('forkRequiredPending', () => { + it('is true when a required ref is neither mapped nor selected for copy', () => { + const items = [entry({ kind: 'credential', sourceId: 'c1', required: true })] + expect(forkRequiredPending(items, {}, new Set())).toBe(true) + }) + + it('is false when the required ref is selected for copy', () => { + const items = [entry({ kind: 'knowledge-base', sourceId: 'kb-1', required: true })] + expect(forkRequiredPending(items, {}, new Set(['knowledge-base:kb-1']))).toBe(false) + }) + + it('is false when the required ref is mapped', () => { + const items = [entry({ kind: 'credential', sourceId: 'c1', required: true, targetId: 'c-tgt' })] + expect(forkRequiredPending(items, {}, new Set())).toBe(false) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/copy-reconciliation.ts b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/copy-reconciliation.ts new file mode 100644 index 00000000000..f1707ad1a3b --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/copy-reconciliation.ts @@ -0,0 +1,85 @@ +import type { ForkCopyableUnmapped, ForkMappingEntry } from '@/lib/api/contracts/workspace-fork' + +/** `${kind}:${sourceId}` - the shared key for a mapping entry and its copy candidate. */ +export const forkRefKey = (ref: { kind: string; sourceId: string }): string => + `${ref.kind}:${ref.sourceId}` + +/** Effective mapping target: the in-session override, else the persisted target, else ''. */ +export function effectiveForkTarget( + entry: ForkMappingEntry, + targets: Record +): string { + return targets[forkRefKey(entry)] ?? entry.targetId ?? '' +} + +/** + * Keys of copyable resources that already have a mapping target (in-session or persisted). Maps win + * over copy, so these drop out of the copy list - the copy-vs-map reconciliation. + */ +export function forkMappedCopyableKeys( + entries: ForkMappingEntry[], + targets: Record +): Set { + const keys = new Set() + for (const entry of entries) { + if (effectiveForkTarget(entry, targets) !== '') keys.add(forkRefKey(entry)) + } + return keys +} + +/** Copy candidates the user has not mapped (a mapped copyable is excluded - copy-vs-map reconcile). */ +export function forkVisibleCopyables( + copyableUnmapped: ForkCopyableUnmapped[], + mappedKeys: ReadonlySet +): ForkCopyableUnmapped[] { + return copyableUnmapped.filter((candidate) => !mappedKeys.has(forkRefKey(candidate))) +} + +/** Keys of the visible copy candidates actually selected for copy. */ +export function forkCopyingKeys( + visibleCopyables: ForkCopyableUnmapped[], + copySelected: ReadonlySet +): Set { + const keys = new Set() + for (const candidate of visibleCopyables) { + const key = forkRefKey(candidate) + if (copySelected.has(key)) keys.add(key) + } + return keys +} + +/** + * Whether every required reference is satisfied - it has a mapping target OR is selected for copy. + * The server accepts a copy as resolving a required ref (promote.ts `willResolve`), so the client + * gate must too. No double-count: a mapped copyable is excluded from the copy candidates, so the two + * branches are mutually exclusive. + */ +export function isForkRequiredComplete( + entries: ForkMappingEntry[], + targets: Record, + copyingKeys: ReadonlySet +): boolean { + return entries.every( + (entry) => + !entry.required || + effectiveForkTarget(entry, targets) !== '' || + copyingKeys.has(forkRefKey(entry)) + ) +} + +/** + * Whether any reference in a kind is required AND still unmapped AND not selected for copy - drives + * the overview's amber "pending" badge. Mirrors {@link isForkRequiredComplete}'s satisfied rule. + */ +export function forkRequiredPending( + items: ForkMappingEntry[], + targets: Record, + copyingKeys: ReadonlySet +): boolean { + return items.some( + (entry) => + entry.required && + effectiveForkTarget(entry, targets) === '' && + !copyingKeys.has(forkRefKey(entry)) + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/dependent-value.test.ts b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/dependent-value.test.ts new file mode 100644 index 00000000000..16f49d4de4a --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/dependent-value.test.ts @@ -0,0 +1,59 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import type { ForkDependentReconfig } from '@/lib/api/contracts/workspace-fork' +import { + dependentKey, + effectiveDependentValue, +} from '@/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/dependent-value' + +const field = (overrides: Partial = {}): ForkDependentReconfig => ({ + parentKind: 'credential', + parentSourceId: 'cred-1', + parentContextKey: 'oauthCredential', + targetWorkflowId: 'wf-1', + targetBlockId: 'blk-1', + blockName: 'Gmail', + subBlockKey: 'folder', + selectorKey: 'gmail.labels', + title: 'Label', + currentValue: 'INBOX', + required: false, + consumesContextKeys: [], + context: {}, + ...overrides, +}) + +describe('dependentKey', () => { + it('keys by target workflow + block + subblock', () => { + expect( + dependentKey(field({ targetWorkflowId: 'w', targetBlockId: 'b', subBlockKey: 's' })) + ).toBe('w:b:s') + }) +}) + +describe('effectiveDependentValue', () => { + it('returns the in-session re-pick when present', () => { + const f = field() + expect(effectiveDependentValue(f, { [dependentKey(f)]: 'Label_42' }, false)).toBe('Label_42') + }) + + it('returns the stored currentValue when no re-pick and the parent is unchanged', () => { + expect(effectiveDependentValue(field({ currentValue: 'INBOX' }), {}, false)).toBe('INBOX') + }) + + it('returns blank when the parent changed (the stored value no longer resolves)', () => { + expect(effectiveDependentValue(field({ currentValue: 'INBOX' }), {}, true)).toBe('') + }) + + it('an in-session re-pick wins even when the parent changed', () => { + const f = field({ currentValue: 'INBOX' }) + expect(effectiveDependentValue(f, { [dependentKey(f)]: 'Label_99' }, true)).toBe('Label_99') + }) + + it('an explicit empty re-pick is respected (not treated as absent)', () => { + const f = field({ currentValue: 'INBOX' }) + expect(effectiveDependentValue(f, { [dependentKey(f)]: '' }, false)).toBe('') + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/promote-workspace-modal.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/promote-workspace-modal.tsx index 0ffd90fb96c..8b74fc51ad5 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/promote-workspace-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/promote-workspace-modal.tsx @@ -16,6 +16,7 @@ import { import { getErrorMessage } from '@sim/utils/errors' import { ArrowRight } from 'lucide-react' import type { + ForkCopyableUnmapped, ForkDependentReconfig, ForkLineageNodeApi, ForkMappingEntry, @@ -23,7 +24,21 @@ import type { ForkWorkflowChange, } from '@/lib/api/contracts/workspace-fork' import { SettingsSection } from '@/app/workspace/[workspaceId]/settings/components/settings-section/settings-section' +import { + FileKindRow, + ResourceKindRow, +} from '@/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-resource-picker/fork-resource-picker' +import { selectVisibleClearedRefs } from '@/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/cleared-refs-list' import { ResourceReconfigure } from '@/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/components/resource-reconfigure' +import { + effectiveForkTarget, + forkCopyingKeys, + forkMappedCopyableKeys, + forkRefKey, + forkRequiredPending, + forkVisibleCopyables, + isForkRequiredComplete, +} from '@/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/promote-workspace-modal/copy-reconciliation' import { dependentKey, effectiveDependentValue, @@ -43,7 +58,7 @@ interface PromoteWorkspaceModalProps { parent: ForkLineageNodeApi | null } -const entryKey = (entry: ForkMappingEntry) => `${entry.kind}:${entry.sourceId}` +const entryKey = (entry: ForkMappingEntry) => forkRefKey(entry) /** * Whether a mapping entry needs an in-place reconfigure: its effective target was changed @@ -90,19 +105,43 @@ function takenTargetOwners( return owners } +/** + * The mapping kinds that can be a standalone mapping entry. `knowledge-document` is excluded: + * the mapping view (`getForkMappingView`) skips documents — they ride their parent KB via the + * reconfigure flow — so a `knowledge-document` mapping section is never reachable. + */ +type MappableMappingKind = Exclude + /** Section label + display order per mapping kind (one mapping step per kind). */ -const MAPPING_SECTION: Record = { +const MAPPING_SECTION: Record = { credential: { label: 'Credentials', order: 0 }, 'env-var': { label: 'Secrets', order: 1 }, table: { label: 'Tables', order: 2 }, 'knowledge-base': { label: 'Knowledge bases', order: 3 }, - 'knowledge-document': { label: 'Knowledge documents', order: 4 }, - file: { label: 'Files', order: 5 }, - 'mcp-server': { label: 'MCP servers', order: 6 }, - 'custom-tool': { label: 'Custom tools', order: 7 }, - skill: { label: 'Skills', order: 8 }, + file: { label: 'Files', order: 4 }, + 'mcp-server': { label: 'MCP servers', order: 5 }, + 'custom-tool': { label: 'Custom tools', order: 6 }, + skill: { label: 'Skills', order: 7 }, } +/** + * Copyable kinds as expandable sections in the sync "Copy resources" picker, ordered + labeled to + * match the fork modal's resource picker exactly. Files nest in a folder ▸ file tree; every other + * kind is a flat searchable list. + */ +const COPYABLE_KIND_SECTIONS: ReadonlyArray<{ + kind: ForkCopyableUnmapped['kind'] + label: string +}> = [ + { kind: 'file', label: 'Files' }, + { kind: 'table', label: 'Tables' }, + { kind: 'knowledge-base', label: 'Knowledge bases' }, + { kind: 'custom-tool', label: 'Custom tools' }, + { kind: 'skill', label: 'Skills' }, +] + +const copyableKey = (candidate: ForkCopyableUnmapped) => forkRefKey(candidate) + interface EdgeOption { value: string label: string @@ -157,6 +196,11 @@ export function PromoteWorkspaceModal({ // `dependentKey`. Folded into the full effective set sent on sync, which promote persists as // the stored mapping - so the selection survives every future sync without re-picking. const [reconfig, setReconfig] = useState>({}) + // Referenced-but-unmapped resources the user chose to copy into the target (keyed by + // `${kind}:${sourceId}`); default-selected once the diff loads. Selected ones are copied on + // sync so their references resolve to the copy instead of being cleared. + const [copySelected, setCopySelected] = useState>(new Set()) + const [copyDefaulted, setCopyDefaulted] = useState(false) // Wizard step: 0 is the overview; 1..N edit one resource kind each, entered via // "Edit mappings". Backing out of step 1 returns to the overview. const [step, setStep] = useState(0) @@ -176,6 +220,8 @@ export function PromoteWorkspaceModal({ setStep(0) setTargets({}) setReconfig({}) + setCopySelected(new Set()) + setCopyDefaulted(false) }, [open, selectedKey]) const selected = edgeOptions.find((option) => option.value === selectedKey) @@ -193,6 +239,59 @@ export function PromoteWorkspaceModal({ [diff.data?.dependentReconfigs] ) const resourceUsages = useMemo(() => diff.data?.resourceUsages ?? [], [diff.data?.resourceUsages]) + const copyableUnmapped = useMemo( + () => diff.data?.copyableUnmapped ?? [], + [diff.data?.copyableUnmapped] + ) + const clearedRefs = useMemo(() => diff.data?.clearedRefs ?? [], [diff.data?.clearedRefs]) + + // Copy-vs-map reconciliation: a copyable resource the user has given an effective (in-session + // or persisted) mapping target must NOT also appear in the copy list - the user picked map, not + // copy. `copyableKey` shares the `${kind}:${sourceId}` keyspace with `entryKey`, so a mapped + // entry's key directly excludes its copy candidate. The server enforces the same precedence: + // a mapped resource resolves != null, so it never reaches the plan's `copyableUnmapped`, and a + // copy request for it is dropped by `buildPromoteCopySelection`. + const mappedCopyableKeys = useMemo( + () => forkMappedCopyableKeys(entries, targets), + [entries, targets] + ) + + const visibleCopyables = useMemo( + () => forkVisibleCopyables(copyableUnmapped, mappedCopyableKeys), + [copyableUnmapped, mappedCopyableKeys] + ) + + // Copyables actually selected for copy (visible + checked), keyed for an O(1) lookup so a + // copyable mapping entry in the editor walk can show a "will be copied" note. + const copyingKeys = useMemo( + () => forkCopyingKeys(visibleCopyables, copySelected), + [visibleCopyables, copySelected] + ) + + // Group the visible copy candidates by kind so each renders as its own expandable section + // (chevron + tri-state select-all + count), matching the fork picker. Files nest in a folder ▸ + // file tree inside their section; every other kind is a flat searchable list. + const copyablesByKind = useMemo(() => { + const groups = new Map() + for (const candidate of visibleCopyables) { + const list = groups.get(candidate.kind) + if (list) list.push(candidate) + else groups.set(candidate.kind, [candidate]) + } + return groups + }, [visibleCopyables]) + + // Default every copyable referenced resource to "copy" once the diff loads, so the common case + // (bring the referenced resources along) needs no clicks; the user can deselect to clear instead. + // Seed ONLY from a settled diff for the current direction: on a direction switch the reset clears + // `copyDefaulted`, but `useForkDiff` keeps the previous direction's payload (placeholderData) until + // the new fetch resolves - seeding from it would latch against stale keys and leave the real + // copyables unchecked, clearing their references on Sync. + useEffect(() => { + if (!open || diff.isPlaceholderData || copyableUnmapped.length === 0 || copyDefaulted) return + setCopyDefaulted(true) + setCopySelected(new Set(copyableUnmapped.map(copyableKey))) + }, [open, diff.isPlaceholderData, copyableUnmapped, copyDefaulted]) // Group dependents by their parent (kind:sourceId) once, so each mapping entry below gets a // STABLE `dependents` array reference - a fresh `.filter` per render would defeat @@ -211,9 +310,11 @@ export function PromoteWorkspaceModal({ // Effective target for an entry: the user's in-session override if present, // else the persisted mapping from the server. Read directly from `entries` so // a reopened edge reflects stored mappings without a seeding effect. - const targetFor = (entry: ForkMappingEntry) => targets[entryKey(entry)] ?? entry.targetId ?? '' + const targetFor = (entry: ForkMappingEntry) => effectiveForkTarget(entry, targets) - const requiredComplete = entries.every((entry) => !entry.required || targetFor(entry) !== '') + // A required reference is satisfied when it has a mapping target OR the user selected it for copy + // (the server accepts a copy as resolving a required ref). See `isForkRequiredComplete`. + const requiredComplete = isForkRequiredComplete(entries, targets, copyingKeys) // Every workflow a mapping entry's resource is used in, for the always-on reconfigure // listing rendered beneath that mapping (so the credential/KB stays in context). @@ -228,8 +329,11 @@ export function PromoteWorkspaceModal({ // Group mappings by resource type - one step per kind, required types first. const groupedEntries = useMemo(() => { - const groups = new Map() + const groups = new Map() for (const entry of entries) { + // The mapping view never emits a document entry (it rides its KB), so the section is + // unreachable - skip defensively so the narrowed `MAPPING_SECTION` lookup stays sound. + if (entry.kind === 'knowledge-document') continue const list = groups.get(entry.kind) if (list) list.push(entry) else groups.set(entry.kind, [entry]) @@ -277,6 +381,32 @@ export function PromoteWorkspaceModal({ return dependentValueFor(field, parent) !== '' }) + // Kinds with a required DEPENDENT that still has no value (its parent is mapped): these block + // Sync via `reconfigComplete`, so the overview badge for that kind must not read "Fully mapped". + const reconfigPendingByKind = new Set() + for (const field of dependentReconfigs) { + if (!field.required) continue + const parent = entryForDependent(field) + if (!parent || targetFor(parent) === '') continue + if (dependentValueFor(field, parent) === '') { + reconfigPendingByKind.add(parent.kind as MappableMappingKind) + } + } + + // The references this sync will blank, reactively narrowed to the current selection. A resource + // is "resolved" once it has a mapping target OR is selected for copy - the same predicate drives + // a `reference` (its own resource) and a `dependent` (its PARENT resource), so mapping or copying + // a parent KB makes its child document drop off. `workflow` refs always clear (not resolvable here). + const clearedRefsToShow = useMemo(() => { + const isResolved = (kind: string, sourceId: string) => { + const key = `${kind}:${sourceId}` + const entry = entriesByParent.get(key) + const mapped = entry ? (targets[key] ?? entry.targetId ?? '') !== '' : false + return mapped || copyingKeys.has(key) + } + return selectVisibleClearedRefs(clearedRefs, isResolved) + }, [clearedRefs, entriesByParent, targets, copyingKeys]) + // Per-kind status for the overview listing: "Fully mapped" or "n/total mapped", // flagged when a REQUIRED target is still missing (which blocks Sync). Reads the // effective (override-or-persisted) target so it reflects both remembered mappings @@ -284,8 +414,10 @@ export function PromoteWorkspaceModal({ const kindSummaries = groupedEntries.map((group) => { const total = group.items.length const mapped = group.items.filter((entry) => targetFor(entry) !== '').length - const requiredPending = group.items.some((entry) => entry.required && targetFor(entry) === '') - return { kind: group.kind, label: group.label, total, mapped, requiredPending } + // Mirror the Sync gate: a required ref selected for copy is satisfied, so it is not "pending". + const requiredPending = forkRequiredPending(group.items, targets, copyingKeys) + const reconfigPending = reconfigPendingByKind.has(group.kind) + return { kind: group.kind, label: group.label, total, mapped, requiredPending, reconfigPending } }) // Step 0 is the overview; each subsequent step edits one resource kind, entered via @@ -296,16 +428,22 @@ export function PromoteWorkspaceModal({ const safeStep = Math.min(step, Math.max(0, stepCount - 1)) const isLastStep = safeStep >= stepCount - 1 const currentGroup = safeStep >= 1 ? (groupedEntries[safeStep - 1] ?? null) : null - // Gate Sync on the diff being loaded too, not just the mapping: until `diff.data` arrives - // `dependentReconfigs` is empty, so `reconfigComplete` is vacuously true and `runPromote` would - // omit `dependentValues` - i.e. Sync before the diff loads would bypass dependent gating. + // Gate Sync on BOTH queries being settled for the current direction. Beyond loading: a failed/ + // empty mapping (`!mapping.data`) must not read as "nothing required" (required-ref completeness + // can't be determined), and placeholder data after a direction switch is the PREVIOUS direction's + // payload - syncing on it would send stale mappings/copies and clear references. Until `diff.data` + // arrives `dependentReconfigs` is empty, so `reconfigComplete` is vacuously true and `runPromote` + // would bypass dependent gating. const syncDisabled = submitting || !otherWorkspaceId || !requiredComplete || !reconfigComplete || mapping.isLoading || - !diff.data + !mapping.data || + mapping.isPlaceholderData || + !diff.data || + diff.isPlaceholderData const headsUp = (diff.data?.mcpReauthServerIds.length ?? 0) > 0 || (diff.data?.inlineSecretSources.length ?? 0) > 0 @@ -344,6 +482,25 @@ export function PromoteWorkspaceModal({ ] }) + // Copy the referenced-but-unmapped resources the user kept selected, excluding any the user + // mapped in-session (reconciliation: maps win). The backend validates each id against the + // plan's copy candidates too, so a mapped/stale id is dropped server-side regardless. + const selectedCopyables = visibleCopyables.filter((candidate) => + copySelected.has(copyableKey(candidate)) + ) + const copyResources = { + knowledgeBases: selectedCopyables + .filter((c) => c.kind === 'knowledge-base') + .map((c) => c.sourceId), + tables: selectedCopyables.filter((c) => c.kind === 'table').map((c) => c.sourceId), + customTools: selectedCopyables + .filter((c) => c.kind === 'custom-tool') + .map((c) => c.sourceId), + skills: selectedCopyables.filter((c) => c.kind === 'skill').map((c) => c.sourceId), + // Files are identified by storage key (the copyable candidate's sourceId is the key). + files: selectedCopyables.filter((c) => c.kind === 'file').map((c) => c.sourceId), + } + const result = await promote.mutateAsync({ workspaceId, body: { @@ -354,12 +511,25 @@ export function PromoteWorkspaceModal({ // stored rows. Collapsing `[]` into omission would make the backend PRESERVE stale rows. // Only omit before the diff loads (set unknown), so the existing store is left untouched. ...(diff.data ? { dependentValues } : {}), + ...(selectedCopyables.length > 0 ? { copyResources } : {}), }, }) if (!result.promoteRunId) { if (result.unmappedRequired.length > 0) { - toast.error('Map all required credentials and secrets first') + // Name the actual blocking kinds rather than always blaming credentials: the server + // blocks on required REFERENCES (credentials and/or secrets); required dependents are + // gated client-side before this runs (see the Sync button's disabled tooltip). + const kinds = new Set(result.unmappedRequired.map((reference) => reference.kind)) + const what = + kinds.has('credential') && kinds.has('env-var') + ? 'credentials and secrets' + : kinds.has('credential') + ? 'credentials' + : kinds.has('env-var') + ? 'secrets' + : 'references' + toast.error(`Map all required ${what} first`) return } toast.error('Sync did not complete') @@ -447,6 +617,20 @@ export function PromoteWorkspaceModal({ /> + {/* Surface a failed/pending fetch so the modal never renders blank below the picker. */} + {mapping.isError || diff.isError ? ( + +
+ {getErrorMessage( + mapping.error ?? diff.error, + "Couldn't load sync details. Close and reopen to retry." + )} +
+ + ) : !diff.data ? ( +
Loading sync details…
+ ) : null} + {/* Always shown once the diff loads so the user sees the section even with nothing deployed - an empty change list means the source has no deployed workflows (every deployed workflow appears here, changed or not), so the muted state nudges a deploy. */} @@ -506,23 +690,122 @@ export function PromoteWorkspaceModal({ {kindSummaries.length > 0 ? (
- {kindSummaries.map(({ kind, label, total, mapped, requiredPending }) => { - const complete = mapped === total - return ( -
- {label} - - {complete ? 'Fully mapped' : `${mapped}/${total} mapped`} - -
+ {kindSummaries.map( + ({ kind, label, total, mapped, requiredPending, reconfigPending }) => { + const complete = mapped === total && !reconfigPending + return ( +
+ {label} + + {complete + ? 'Fully mapped' + : reconfigPending && mapped === total + ? 'Needs setup' + : `${mapped}/${total} mapped`} + +
+ ) + } + )} +
+ + ) : null} + + {clearedRefsToShow.length > 0 ? ( + +
+ {clearedRefsToShow.map((ref, index) => ( +
+ {ref.blockLabel} will lose{' '} + {ref.fieldLabel} in{' '} + {ref.workflowName} +
+ ))} +
+

+ Map or copy a reference to keep it. Fields that reference another workflow, or + that hang off a remapped credential or knowledge base, are cleared regardless. +

+ + ) : null} + + {visibleCopyables.length > 0 ? ( + +
+ {COPYABLE_KIND_SECTIONS.map((section) => { + const candidates = copyablesByKind.get(section.kind) + if (!candidates || candidates.length === 0) return null + // The picker rows track item ids; copy selection is keyed `${kind}:${id}` + // (matching `copyableKey`), so derive the per-kind selected-id subset and + // re-prefix on toggle. + const selectedIds = new Set( + candidates + .filter((candidate) => copySelected.has(copyableKey(candidate))) + .map((candidate) => candidate.sourceId) + ) + const toggleMany = (ids: string[], checked: boolean) => + setCopySelected((prev) => { + const next = new Set(prev) + for (const id of ids) { + const key = `${section.kind}:${id}` + if (checked) next.add(key) + else next.delete(key) + } + return next + }) + const toggleAll = (selectAll: boolean) => + toggleMany( + candidates.map((candidate) => candidate.sourceId), + selectAll + ) + return section.kind === 'file' ? ( + ({ + id: candidate.sourceId, + label: candidate.label, + folderId: candidate.parentId, + folderName: candidate.parentLabel, + }))} + selected={selectedIds} + onToggleAll={toggleAll} + onToggleItem={(id, checked) => toggleMany([id], checked)} + onToggleMany={toggleMany} + disabled={submitting} + /> + ) : ( + ({ + id: candidate.sourceId, + label: candidate.label, + }))} + selected={selectedIds} + onToggleAll={toggleAll} + onToggleItem={(id, checked) => toggleMany([id], checked)} + disabled={submitting} + /> ) })} +

+ These referenced resources aren't in the target yet. Selected ones are copied + during the sync; deselected ones have their references cleared. +

) : null} @@ -589,6 +872,12 @@ export function PromoteWorkspaceModal({ one, narrow it down by name.
) : null} + {copyingKeys.has(entryKey(entry)) ? ( +
+ Selected for copy in the overview — it'll be copied into the target. Pick a + target above to map it to an existing one instead. +
+ ) : null} {/* Always-on: every workflow this resource is used in, each expandable to its blocks + dependent selectors (greyed when nothing to configure). */} export const forkDirectionSchema = z.enum(['push', 'pull']) +/** + * The remappable, copyable resource kinds a sync can copy into the target when they are + * referenced but unmapped (the fork-style copy at promote time). Excludes credentials, env + * vars, and external MCP servers (never copied this way); documents are auto-copied with their + * parent knowledge base, not selected individually. Workspace `file` references are keyed by + * storage key (not `workspace_files.id`) and copied like fork does. + */ +export const forkCopyableKindSchema = z.enum([ + 'knowledge-base', + 'table', + 'custom-tool', + 'skill', + 'file', +]) +export type ForkCopyableKind = z.infer + export const forkLineageNodeSchema = z.object({ id: z.string(), name: z.string(), @@ -79,7 +100,9 @@ export const forkResourceSelectionSchema = z.object({ knowledgeBases: forkResourceIdList, customTools: forkResourceIdList, skills: forkResourceIdList, - mcpServers: forkResourceIdList, + // External MCP servers are never copied (they carry secrets / require re-auth); only + // workflow-publishing MCP servers are copyable, as config-only shells with no workflows. + workflowMcpServers: forkResourceIdList, }) export const forkWorkspaceBodySchema = z.object({ @@ -106,6 +129,18 @@ export type ForkWorkspaceResponse = z.output + +/** + * A copyable workspace file plus its folder grouping. `folderId`/`folderName` are null when + * the file sits at the workspace root (or its folder was deleted). Files are the only copyable + * kind that nests in the picker (folder ▸ file); every other kind stays flat at the top level. + */ +export const forkCopyableFileSchema = forkCopyableResourceSchema.extend({ + folderId: z.string().nullable(), + folderName: z.string().nullable(), +}) +export type ForkCopyableFile = z.output + export const getForkResourcesContract = defineRouteContract({ method: 'GET', path: '/api/workspaces/[id]/fork/resources', @@ -113,12 +148,12 @@ export const getForkResourcesContract = defineRouteContract({ response: { mode: 'json', schema: z.object({ - files: z.array(forkCopyableResourceSchema), + files: z.array(forkCopyableFileSchema), tables: z.array(forkCopyableResourceSchema), knowledgeBases: z.array(forkCopyableResourceSchema), customTools: z.array(forkCopyableResourceSchema), skills: z.array(forkCopyableResourceSchema), - mcpServers: z.array(forkCopyableResourceSchema), + workflowMcpServers: z.array(forkCopyableResourceSchema), deployedWorkflowCount: z.number().int(), }), }, @@ -284,6 +319,45 @@ export const forkResourceUsageSchema = z.object({ }) export type ForkResourceUsage = z.output +/** Cleared-ref kinds: every remappable kind plus `workflow` (a cross-workflow reference). */ +export const forkClearedRefKindSchema = z.enum([...forkRemapKindSchema.options, 'workflow']) +export type ForkClearedRefKind = z.infer + +/** + * A reference in a synced source workflow that WILL be blanked in the target by this sync, with + * the labels to phrase it as "{blockLabel} will lose {fieldLabel} in workflow {workflowName}". + * `cause` tells the client how the item resolves: + * - `reference`: an unmapped remappable resource - drops off once the user maps OR copies it + * (the only reactive kind; matched to a mapping entry by `${kind}:${sourceId}`). + * - `workflow`: a `workflow-selector`/`workflow_input` ref to a workflow not in the target - + * always cleared (cannot be fixed in the modal). + * - `dependent`: a create-target dependent selector the source configured that a remapped parent + * clears. Carries the controlling parent (`parentKind`/`parentSourceId`). When the child follows + * its parent (a document under a knowledge base, copied/auto-copied with it) the client drops the + * entry once that parent is mapped OR copied; a credential's label or a table's column is cleared + * on any parent remap, so it stays. + */ +export const forkClearedRefSchema = z.object({ + targetWorkflowId: z.string(), + workflowName: z.string(), + blockId: z.string(), + blockLabel: z.string(), + fieldLabel: z.string(), + kind: forkClearedRefKindSchema, + sourceId: z.string(), + sourceLabel: z.string(), + cause: z.enum(['reference', 'workflow', 'dependent']), + /** + * The dependsOn parent resource of a `dependent` entry (its KB / credential / table). When the + * child follows its parent (a document under a KB) the client drops the entry once this parent is + * mapped or copied; otherwise the child is cleared on any parent remap and the entry stays. Null + * for `reference`/`workflow`, whose own `kind`/`sourceId` are the reactive anchor. + */ + parentKind: forkRemapKindSchema.nullable(), + parentSourceId: z.string().nullable(), +}) +export type ForkClearedRef = z.output + export const getForkDiffQuerySchema = z.object({ otherWorkspaceId: workspaceIdSchema, direction: forkDirectionSchema, @@ -313,11 +387,35 @@ export const getForkDiffContract = defineRouteContract({ dependentReconfigs: z.array(forkDependentReconfigSchema), /** Every workflow each mapped resource is used in, for the always-on reconfigure listing. */ resourceUsages: z.array(forkResourceUsageSchema), + /** + * Referenced resources with no target mapping that the sync can copy into the target + * (fork-style), so the user can copy instead of mapping each one by hand. Default-selected + * in the modal; documents under a selected knowledge base are copied automatically. + * `parentId`/`parentLabel` carry the folder grouping for file entries (id + name); they + * are null for non-file kinds and for files at the workspace root. + */ + copyableUnmapped: z.array( + z.object({ + kind: forkCopyableKindSchema, + sourceId: z.string(), + label: z.string(), + parentId: z.string().nullable(), + parentLabel: z.string().nullable(), + }) + ), + /** + * References this sync will blank in the target, with labels for a pre-sync "what will be + * cleared" list. The client filters this against the current mapping/copy selection so a + * `reference` item disappears once mapped or selected for copy; `workflow`/`dependent` items + * always clear (informational). + */ + clearedRefs: z.array(forkClearedRefSchema), }), }, }) export type GetForkDiffResponse = z.output export type ForkWorkflowChange = z.output +export type ForkCopyableUnmapped = GetForkDiffResponse['copyableUnmapped'][number] /** * A workflow whose required dependent fields a sync cleared because their parent @@ -347,6 +445,21 @@ export const forkDependentValueEntrySchema = z.object({ }) export type ForkDependentValueEntry = z.input +/** + * Source resource ids (by kind) the user chose to copy into the target before the sync gate, + * for referenced-but-unmapped resources. Each kind's documents under a copied knowledge base + * are discovered + copied automatically (the user selects only the parent resources). + */ +export const promoteCopyResourcesSchema = z.object({ + knowledgeBases: forkResourceIdList, + tables: forkResourceIdList, + customTools: forkResourceIdList, + skills: forkResourceIdList, + /** Workspace files to copy, identified by storage key (not `workspace_files.id`). */ + files: forkResourceIdList, +}) +export type PromoteCopyResources = z.input + export const promoteForkBodySchema = z.object({ otherWorkspaceId: workspaceIdSchema, direction: forkDirectionSchema, @@ -357,6 +470,8 @@ export const promoteForkBodySchema = z.object({ * an explicit `[]` clears it for the written replace targets. */ dependentValues: z.array(forkDependentValueEntrySchema).max(2000).optional(), + /** Referenced-but-unmapped resources to copy into the target before the sync gate (U17). */ + copyResources: promoteCopyResourcesSchema.optional(), }) export const promoteForkContract = defineRouteContract({ method: 'POST', @@ -397,6 +512,8 @@ export const backgroundWorkMetadataSchema = z files: z.number().int().optional(), copied: z.number().int().optional(), failed: z.number().int().optional(), + /** Count of failed resources whose dangling references were cleared post-fork (U8). */ + clearedReferences: z.number().int().optional(), /** Names of the resources a fork copied, by kind, for the report breakdown. */ workflowNames: z.array(z.string()).optional(), tableNames: z.array(z.string()).optional(), @@ -404,7 +521,7 @@ export const backgroundWorkMetadataSchema = z fileNames: z.array(z.string()).optional(), customToolNames: z.array(z.string()).optional(), skillNames: z.array(z.string()).optional(), - mcpServerNames: z.array(z.string()).optional(), + workflowMcpServerNames: z.array(z.string()).optional(), // Sync / rollback otherWorkspaceName: z.string().optional(), direction: z.enum(['push', 'pull']).optional(), diff --git a/apps/sim/lib/workflows/persistence/remap-internal-ids.test.ts b/apps/sim/lib/workflows/persistence/remap-internal-ids.test.ts index e15c07a17ce..68de67a9bae 100644 --- a/apps/sim/lib/workflows/persistence/remap-internal-ids.test.ts +++ b/apps/sim/lib/workflows/persistence/remap-internal-ids.test.ts @@ -99,6 +99,26 @@ describe('remapWorkflowReferencesInSubBlocks', () => { expect(tools[0].params?.workflowId).toBe('sub-dst') }) + it('clears the sibling inputMapping when an unmapped workflow selector is cleared (U10)', () => { + const subBlocks: SubBlockRecord = { + workflowId: { id: 'workflowId', type: 'workflow-selector', value: 'wf-unknown' }, + inputMapping: { id: 'inputMapping', type: 'input-mapping', value: '{"a":"b"}' }, + } + const result = remapWorkflowReferencesInSubBlocks(subBlocks, map, { clearUnmapped: true }) + expect(result.workflowId.value).toBe('') + expect(result.inputMapping.value).toBe('') + }) + + it('keeps inputMapping when the workflow selector is remapped (not cleared)', () => { + const subBlocks: SubBlockRecord = { + workflowId: { id: 'workflowId', type: 'workflow-selector', value: 'wf-src' }, + inputMapping: { id: 'inputMapping', type: 'input-mapping', value: '{"a":"b"}' }, + } + const result = remapWorkflowReferencesInSubBlocks(subBlocks, map, { clearUnmapped: true }) + expect(result.workflowId.value).toBe('wf-dst') + expect(result.inputMapping.value).toBe('{"a":"b"}') + }) + it('remaps the advanced-mode manualWorkflowId override', () => { const subBlocks: SubBlockRecord = { manualWorkflowId: { id: 'manualWorkflowId', type: 'short-input', value: 'wf-src' }, @@ -134,6 +154,61 @@ describe('remapWorkflowReferencesInSubBlocks', () => { const result = remapWorkflowReferencesInSubBlocks(subBlocks, map) expect(result.workflowSelector.value).toEqual(['wf-dst', 'sub-dst']) }) + + // create-fork scopes its workflow id map to the workflows ACTUALLY copied (deployed state + // loaded). With BOTH the parent (`wf-src`) and child (`sub-src`) workflows copied, every + // reference variety must remap to the child id (NOT clear), even under fork-create's + // clearUnmapped policy - the explicit "both deployed and copied" guard. + it('remaps every reference variety when both referenced workflows are copied (clearUnmapped)', () => { + const subBlocks: SubBlockRecord = { + selector: { id: 'selector', type: 'workflow-selector', value: 'wf-src' }, + inputMapping: { id: 'inputMapping', type: 'input-mapping', value: '{"a":"b"}' }, + manualWorkflowId: { id: 'manualWorkflowId', type: 'short-input', value: 'sub-src' }, + manualWorkflowIds: { id: 'manualWorkflowIds', type: 'short-input', value: 'wf-src, sub-src' }, + workflowSelector: { id: 'workflowSelector', type: 'dropdown', value: ['wf-src', 'sub-src'] }, + tools: { + id: 'tools', + type: 'tool-input', + value: [ + { type: 'workflow_input', params: { workflowId: 'sub-src', inputMapping: '{}' } }, + { type: 'custom-tool', customToolId: 'ct-1' }, + ], + }, + } + const result = remapWorkflowReferencesInSubBlocks(subBlocks, map, { clearUnmapped: true }) + expect(result.selector.value).toBe('wf-dst') + expect(result.inputMapping.value).toBe('{"a":"b"}') + expect(result.manualWorkflowId.value).toBe('sub-dst') + expect(result.manualWorkflowIds.value).toBe('wf-dst,sub-dst') + expect(result.workflowSelector.value).toEqual(['wf-dst', 'sub-dst']) + const tools = result.tools.value as Array<{ type: string; params?: { workflowId?: string } }> + expect(tools[0].params?.workflowId).toBe('sub-dst') + expect(tools[1]).toEqual({ type: 'custom-tool', customToolId: 'ct-1' }) + }) + + // A deployed source workflow whose state failed to load is excluded from the scoped fork map, + // so a copied workflow's reference to it clears (never dangles at a never-created child id). + it('clears references to a deployed-but-uncopied workflow absent from the scoped map', () => { + const subBlocks: SubBlockRecord = { + selector: { id: 'selector', type: 'workflow-selector', value: 'wf-uncopied' }, + inputMapping: { id: 'inputMapping', type: 'input-mapping', value: '{"a":"b"}' }, + manualWorkflowIds: { + id: 'manualWorkflowIds', + type: 'short-input', + value: 'wf-src,wf-uncopied', + }, + tools: { + id: 'tools', + type: 'tool-input', + value: [{ type: 'workflow_input', params: { workflowId: 'wf-uncopied' } }], + }, + } + const result = remapWorkflowReferencesInSubBlocks(subBlocks, map, { clearUnmapped: true }) + expect(result.selector.value).toBe('') + expect(result.inputMapping.value).toBe('') + expect(result.manualWorkflowIds.value).toBe('wf-dst') + expect(result.tools.value as unknown[]).toHaveLength(0) + }) }) describe('coerceObjectArray', () => { diff --git a/apps/sim/lib/workflows/persistence/remap-internal-ids.ts b/apps/sim/lib/workflows/persistence/remap-internal-ids.ts index e71b9497367..5c887e7edff 100644 --- a/apps/sim/lib/workflows/persistence/remap-internal-ids.ts +++ b/apps/sim/lib/workflows/persistence/remap-internal-ids.ts @@ -144,10 +144,15 @@ export function remapWorkflowReferencesInSubBlocks( ): SubBlockRecord { if (!workflowIdMap?.size) return subBlocks const clearUnmapped = options?.clearUnmapped ?? false + let clearedWorkflowSelector = false const remapScalar = (value: string): string => { const mapped = workflowIdMap.get(value) if (mapped) return mapped - return clearUnmapped ? '' : value + if (clearUnmapped) { + clearedWorkflowSelector = true + return '' + } + return value } const updated: SubBlockRecord = {} for (const [key, subBlock] of Object.entries(subBlocks)) { @@ -178,6 +183,18 @@ export function remapWorkflowReferencesInSubBlocks( } updated[key] = subBlock } + // A cleared workflow selector (its target workflow wasn't copied) leaves the block's + // `inputMapping` pointing at a workflow that no longer exists; clear it too so no orphaned + // mapping survives. The nested `workflow_input` tool case drops the whole tool (with its + // inputMapping) above, so only the top-level block-level inputMapping needs this. + if (clearedWorkflowSelector) { + for (const [key, subBlock] of Object.entries(updated)) { + if (key.replace(/_\d+$/, '') !== 'inputMapping') continue + if (!subBlock || typeof subBlock !== 'object') continue + if (subBlock.value === '' || subBlock.value == null) continue + updated[key] = { ...subBlock, value: '' } + } + } return updated } diff --git a/apps/sim/lib/workspaces/fork/copy/cleanup-failed.test.ts b/apps/sim/lib/workspaces/fork/copy/cleanup-failed.test.ts new file mode 100644 index 00000000000..db1fbccf91b --- /dev/null +++ b/apps/sim/lib/workspaces/fork/copy/cleanup-failed.test.ts @@ -0,0 +1,432 @@ +/** + * @vitest-environment node + */ +import { knowledgeBase, workflow, workflowBlocks, workflowDeploymentVersion } from '@sim/db/schema' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const dbMock = vi.hoisted(() => { + const reads = new Map() + const updates: Array<{ table: unknown; values: Record }> = [] + const deletes: Array<{ table: unknown }> = [] + + const nextPage = (table: unknown): unknown[] => { + const pages = reads.get(table) + return pages && pages.length > 0 ? (pages.shift() as unknown[]) : [] + } + + // A drizzle-style read builder bound to one table: `.where`/`.orderBy`/`.limit` chain back to + // the same builder, and awaiting it (at `.where()` or `.limit()`) shifts that table's next page. + const makeReadBuilder = (table: unknown) => { + const builder = { + where: () => builder, + orderBy: () => builder, + limit: () => builder, + then: (onFulfilled: (rows: unknown[]) => unknown, onRejected?: (error: unknown) => unknown) => + Promise.resolve(nextPage(table)).then(onFulfilled, onRejected), + } + return builder + } + + const db = { + select: () => ({ from: (table: unknown) => makeReadBuilder(table) }), + update: (table: unknown) => ({ + set: (values: Record) => ({ + where: () => { + updates.push({ table, values }) + return Promise.resolve([]) + }, + }), + }), + delete: (table: unknown) => ({ + where: () => { + deletes.push({ table }) + return Promise.resolve([]) + }, + }), + } + + return { + db, + updates, + deletes, + queueRead: (table: unknown, ...pages: unknown[][]) => reads.set(table, pages), + reset: () => { + reads.clear() + updates.length = 0 + deletes.length = 0 + }, + } +}) + +const { mockInvalidateDeployedStateCache } = vi.hoisted(() => ({ + mockInvalidateDeployedStateCache: vi.fn(), +})) + +vi.mock('@sim/db', () => ({ + db: dbMock.db, + dbReplica: dbMock.db, + runOutsideTransactionContext: (fn: () => T): T => fn(), + instrumentPoolClient: (client: T): T => client, +})) + +vi.mock('@/lib/workflows/persistence/utils', () => ({ + invalidateDeployedStateCache: mockInvalidateDeployedStateCache, + CREDENTIAL_SUBBLOCK_IDS: new Set(['credential', 'manualCredential', 'triggerCredentials']), +})) + +// The reference indexer resolves a tool's params via the tool registry; stub it so loading the +// remap module never pulls the full registry (this file only exercises top-level selectors). +vi.mock('@/tools/params', () => ({ + getToolIdForOperation: () => undefined, + getToolParametersConfig: () => null, + getSubBlocksForToolInput: ( + _toolId: string, + _type: string, + _values: unknown, + _modes: unknown, + provided?: { subBlocks?: SubBlockConfig[] } + ) => ({ subBlocks: provided?.subBlocks ?? [] }), + formatParameterLabel: (label: string) => label, +})) + +import { + clearFailedForkResourceReferences, + clearFailedReferencesInDeploymentVersions, + clearFailedReferencesInWorkflows, + rewriteDeploymentVersionState, +} from '@/lib/workspaces/fork/copy/cleanup-failed' +import type { ForkCopyResolver } from '@/lib/workspaces/fork/remap/fork-bootstrap' +import type { ForkRemapKind } from '@/lib/workspaces/fork/remap/remap-references' +import { getBlock } from '@/blocks/registry' +import type { BlockConfig, SubBlockConfig } from '@/blocks/types' + +const blockWith = (subBlocks: SubBlockConfig[]): BlockConfig => + ({ name: 'Knowledge', description: '', subBlocks, outputs: {} }) as unknown as BlockConfig + +/** A KB block whose `documentId` (document-selector) hangs off `knowledgeBaseId` (kb-selector). */ +const kbBlockConfig = () => + blockWith([ + { id: 'knowledgeBaseId', title: 'KB', type: 'knowledge-base-selector' }, + { id: 'documentId', title: 'Doc', type: 'document-selector', dependsOn: ['knowledgeBaseId'] }, + ]) + +/** An agent block whose `tools` tool-input holds a KB tool with a nested `knowledgeBaseId` param. */ +const agentToolConfig = (type: string): BlockConfig => { + if (type === 'agent') return blockWith([{ id: 'tools', title: 'Tools', type: 'tool-input' }]) + if (type === 'kbtool') + return blockWith([{ id: 'knowledgeBaseId', title: 'KB', type: 'knowledge-base-selector' }]) + return undefined as unknown as BlockConfig +} + +/** A deployment-version state whose agent block references `kbId` inside a tool-input tool param. */ +const agentVersionState = (kbId: string) => ({ + blocks: { + 'agent-1': { + id: 'agent-1', + type: 'agent', + name: 'Agent', + subBlocks: { + tools: { + id: 'tools', + type: 'tool-input', + value: [{ type: 'kbtool', toolId: 'kbtool_search', params: { knowledgeBaseId: kbId } }], + }, + }, + }, + }, + edges: [], + loops: {}, + parallels: {}, +}) + +type AgentStateBlocks = { + blocks: { + 'agent-1': { subBlocks: { tools: { value: Array<{ params: { knowledgeBaseId: string } }> } } } + } +} +const nestedKbValue = (state: unknown) => + (state as AgentStateBlocks).blocks['agent-1'].subBlocks.tools.value[0].params.knowledgeBaseId + +/** A serialized deployment-version state whose single block points its KB selector at `kbId`. */ +const versionState = (kbId: string) => ({ + blocks: { + 'block-1': { + id: 'block-1', + type: 'knowledge', + name: 'KB Block', + subBlocks: { + knowledgeBaseId: { id: 'knowledgeBaseId', type: 'knowledge-base-selector', value: kbId }, + documentId: { id: 'documentId', type: 'document-selector', value: 'doc-keep' }, + }, + }, + }, + edges: [], + loops: {}, + parallels: {}, +}) + +const draftBlockRow = (kbId: string) => ({ + id: 'b-1', + workflowId: 'wf-1', + type: 'knowledge', + subBlocks: { + knowledgeBaseId: { id: 'knowledgeBaseId', type: 'knowledge-base-selector', value: kbId }, + documentId: { id: 'documentId', type: 'document-selector', value: 'doc-keep' }, + }, +}) + +/** A draft block whose `file-upload` subblock points at a copied workspace-file storage key. */ +const fileBlockRow = (fileKey: string) => ({ + id: 'b-file', + workflowId: 'wf-1', + type: 'agent', + subBlocks: { + file: { id: 'file', type: 'file-upload', value: { key: fileKey, name: 'a.png' } }, + }, +}) + +const failedKbResolver: ForkCopyResolver = (kind, id) => + kind === 'knowledge-base' && id === 'failed-kb' ? null : id + +const failedByKind = () => + new Map>([['knowledge-base', new Set(['failed-kb'])]]) + +type StateBlocks = { blocks: Record }> } +const kbValue = (state: unknown) => + (state as StateBlocks).blocks['block-1'].subBlocks.knowledgeBaseId.value +const docValue = (state: unknown) => + (state as StateBlocks).blocks['block-1'].subBlocks.documentId.value + +describe('cleanup-failed', () => { + beforeEach(() => { + vi.clearAllMocks() + dbMock.reset() + vi.mocked(getBlock).mockReturnValue(kbBlockConfig()) + }) + + describe('rewriteDeploymentVersionState', () => { + it('clears a block ref that resolves to a failed id and its dependents', () => { + const result = rewriteDeploymentVersionState(versionState('failed-kb'), failedKbResolver) + expect(result.changed).toBe(true) + expect(kbValue(result.state)).toBe('') + // documentId hangs off knowledgeBaseId, so the cleared parent clears it too. + expect(docValue(result.state)).toBe('') + }) + + it('leaves a state that references no failed id untouched (same reference, not changed)', () => { + const input = versionState('other-kb') + const result = rewriteDeploymentVersionState(input, failedKbResolver) + expect(result.changed).toBe(false) + expect(result.state).toBe(input) + }) + + it('is tolerant of a malformed state shape', () => { + const input = { not: 'a workflow state' } + const result = rewriteDeploymentVersionState(input, failedKbResolver) + expect(result.changed).toBe(false) + expect(result.state).toBe(input) + }) + + // The deployed-version sweep must clear EVERY subblock variety the draft sweep does - + // including a failed id nested in an agent block's `tool-input` tool params, not only + // top-level selectors - via the shared remapForkSubBlocks/clearFailedSubBlockReferences. + it('clears a failed id nested in an agent tool-input param inside a deployed version', () => { + vi.mocked(getBlock).mockImplementation((type) => agentToolConfig(type)) + const result = rewriteDeploymentVersionState(agentVersionState('failed-kb'), failedKbResolver) + expect(result.changed).toBe(true) + expect(nestedKbValue(result.state)).toBe('') + }) + + it('leaves an agent tool-input param that references no failed id untouched', () => { + vi.mocked(getBlock).mockImplementation((type) => agentToolConfig(type)) + const input = agentVersionState('other-kb') + const result = rewriteDeploymentVersionState(input, failedKbResolver) + expect(result.changed).toBe(false) + expect(result.state).toBe(input) + }) + }) + + describe('clearFailedReferencesInWorkflows', () => { + it('sweeps the draft blocks and returns the affected workflow ids', async () => { + dbMock.queueRead(workflow, [{ id: 'wf-1' }]) + dbMock.queueRead(workflowBlocks, [draftBlockRow('failed-kb')]) + + const affected = await clearFailedReferencesInWorkflows('child-ws', failedByKind(), 'test') + + expect([...affected]).toEqual(['wf-1']) + expect(dbMock.updates).toHaveLength(1) + expect(dbMock.updates[0].table).toBe(workflowBlocks) + const cleared = dbMock.updates[0].values.subBlocks as Record + expect(cleared.knowledgeBaseId.value).toBe('') + expect(cleared.documentId.value).toBe('') + }) + + it('returns an empty set and writes nothing when no block references a failed id', async () => { + dbMock.queueRead(workflow, [{ id: 'wf-1' }]) + dbMock.queueRead(workflowBlocks, [draftBlockRow('other-kb')]) + + const affected = await clearFailedReferencesInWorkflows('child-ws', failedByKind(), 'test') + + expect(affected.size).toBe(0) + expect(dbMock.updates).toHaveLength(0) + }) + }) + + describe('clearFailedReferencesInDeploymentVersions', () => { + it('rewrites a version referencing a failed id and invalidates its deployed-state cache', async () => { + dbMock.queueRead(workflowDeploymentVersion, [ + { id: 'dv-1', version: 5, state: versionState('failed-kb') }, + ]) + + await clearFailedReferencesInDeploymentVersions(new Set(['wf-1']), failedByKind(), 'test') + + expect(dbMock.updates).toHaveLength(1) + expect(dbMock.updates[0].table).toBe(workflowDeploymentVersion) + expect(kbValue(dbMock.updates[0].values.state)).toBe('') + expect(docValue(dbMock.updates[0].values.state)).toBe('') + expect(mockInvalidateDeployedStateCache).toHaveBeenCalledTimes(1) + expect(mockInvalidateDeployedStateCache).toHaveBeenCalledWith('dv-1') + }) + + it('leaves a version that does not reference a failed id unwritten and uncached', async () => { + dbMock.queueRead(workflowDeploymentVersion, [ + { id: 'dv-old', version: 3, state: versionState('other-kb') }, + ]) + + await clearFailedReferencesInDeploymentVersions(new Set(['wf-1']), failedByKind(), 'test') + + expect(dbMock.updates).toHaveLength(0) + expect(mockInvalidateDeployedStateCache).not.toHaveBeenCalled() + }) + + it('writes only the changed version when a workflow mixes referencing and non-referencing versions', async () => { + dbMock.queueRead(workflowDeploymentVersion, [ + { id: 'dv-active', version: 5, state: versionState('failed-kb') }, + { id: 'dv-old', version: 4, state: versionState('other-kb') }, + ]) + + await clearFailedReferencesInDeploymentVersions(new Set(['wf-1']), failedByKind(), 'test') + + expect(dbMock.updates).toHaveLength(1) + expect(mockInvalidateDeployedStateCache).toHaveBeenCalledTimes(1) + expect(mockInvalidateDeployedStateCache).toHaveBeenCalledWith('dv-active') + }) + + it('does nothing when no workflows were affected', async () => { + await clearFailedReferencesInDeploymentVersions(new Set(), failedByKind(), 'test') + expect(dbMock.updates).toHaveLength(0) + expect(mockInvalidateDeployedStateCache).not.toHaveBeenCalled() + }) + }) + + describe('clearFailedForkResourceReferences', () => { + it('threads the draft sweep into the deployed sweep, then drops the placeholder', async () => { + dbMock.queueRead(workflow, [{ id: 'wf-1' }]) + dbMock.queueRead(workflowBlocks, [draftBlockRow('failed-kb')]) + dbMock.queueRead(workflowDeploymentVersion, [ + { id: 'dv-active', version: 5, state: versionState('failed-kb') }, + { id: 'dv-old', version: 4, state: versionState('other-kb') }, + ]) + + const cleaned = await clearFailedForkResourceReferences({ + childWorkspaceId: 'child-ws', + failures: [{ kind: 'knowledge-base', childId: 'failed-kb', documentChildIds: [] }], + requestId: 'test', + }) + + expect(cleaned).toBe(1) + // One draft block update + one deployed version update (only the referencing version). + const updatedTables = dbMock.updates.map((u) => u.table) + expect(updatedTables).toEqual([workflowBlocks, workflowDeploymentVersion]) + expect(mockInvalidateDeployedStateCache).toHaveBeenCalledTimes(1) + expect(mockInvalidateDeployedStateCache).toHaveBeenCalledWith('dv-active') + // The orphaned KB placeholder is dropped after both sweeps. + expect(dbMock.deletes).toHaveLength(1) + expect(dbMock.deletes[0].table).toBe(knowledgeBase) + }) + + it('still drops the placeholder when no workflow referenced the failed resource', async () => { + dbMock.queueRead(workflow, [{ id: 'wf-1' }]) + dbMock.queueRead(workflowBlocks, [draftBlockRow('other-kb')]) + + const cleaned = await clearFailedForkResourceReferences({ + childWorkspaceId: 'child-ws', + failures: [{ kind: 'knowledge-base', childId: 'failed-kb', documentChildIds: [] }], + requestId: 'test', + }) + + expect(cleaned).toBe(1) + // No draft block referenced the failed id AND no deployed targets were threaded, so the + // deployed sweep is skipped entirely. + expect(dbMock.updates).toHaveLength(0) + expect(mockInvalidateDeployedStateCache).not.toHaveBeenCalled() + expect(dbMock.deletes).toHaveLength(1) + expect(dbMock.deletes[0].table).toBe(knowledgeBase) + }) + + it('sweeps a deployed target version even when no draft referenced the failed id', async () => { + // Draft is clean (other-kb), but a deployed target version still points at the dropped + // placeholder - the deployed-target scope (not draft divergence) catches it. + dbMock.queueRead(workflow, [{ id: 'wf-1' }]) + dbMock.queueRead(workflowBlocks, [draftBlockRow('other-kb')]) + dbMock.queueRead(workflowDeploymentVersion, [ + { id: 'dv-1', version: 5, state: versionState('failed-kb') }, + ]) + + const cleaned = await clearFailedForkResourceReferences({ + childWorkspaceId: 'child-ws', + failures: [{ kind: 'knowledge-base', childId: 'failed-kb', documentChildIds: [] }], + deployedTargetWorkflowIds: ['wf-deployed'], + requestId: 'test', + }) + + expect(cleaned).toBe(1) + expect(dbMock.updates.map((u) => u.table)).toContain(workflowDeploymentVersion) + expect(mockInvalidateDeployedStateCache).toHaveBeenCalledWith('dv-1') + // Clearing succeeded, so the placeholder is dropped. + expect(dbMock.deletes[0].table).toBe(knowledgeBase) + }) + + it('clears a file-upload reference to a failed copied blob and drops no row', async () => { + dbMock.queueRead(workflow, [{ id: 'wf-1' }]) + dbMock.queueRead(workflowBlocks, [fileBlockRow('workspace/child/failed.png')]) + + const cleaned = await clearFailedForkResourceReferences({ + childWorkspaceId: 'child-ws', + failures: [{ kind: 'file', childKey: 'workspace/child/failed.png' }], + requestId: 'test', + }) + + expect(cleaned).toBe(1) + expect(dbMock.updates).toHaveLength(1) + expect(dbMock.updates[0].table).toBe(workflowBlocks) + const cleared = dbMock.updates[0].values.subBlocks as Record + expect(cleared.file.value).toBe('') + // A failed file has no placeholder row to drop (the metadata row stays re-uploadable). + expect(dbMock.deletes).toHaveLength(0) + }) + + it('skips the placeholder drop when a reference-clear phase throws', async () => { + // A clear-phase failure must not drop the placeholder: that would turn an empty placeholder + // into a dangling reference to a deleted row. Make the draft block UPDATE throw. + dbMock.queueRead(workflow, [{ id: 'wf-1' }]) + dbMock.queueRead(workflowBlocks, [draftBlockRow('failed-kb')]) + const originalUpdate = dbMock.db.update + dbMock.db.update = () => { + throw new Error('update failed') + } + try { + const cleaned = await clearFailedForkResourceReferences({ + childWorkspaceId: 'child-ws', + failures: [{ kind: 'knowledge-base', childId: 'failed-kb', documentChildIds: [] }], + requestId: 'test', + }) + expect(cleaned).toBe(1) + } finally { + dbMock.db.update = originalUpdate + } + // The drop is skipped, so the placeholder row survives (no delete issued). + expect(dbMock.deletes).toHaveLength(0) + }) + }) +}) diff --git a/apps/sim/lib/workspaces/fork/copy/cleanup-failed.ts b/apps/sim/lib/workspaces/fork/copy/cleanup-failed.ts new file mode 100644 index 00000000000..7cc6b7676f2 --- /dev/null +++ b/apps/sim/lib/workspaces/fork/copy/cleanup-failed.ts @@ -0,0 +1,352 @@ +import { db } from '@sim/db' +import { + document, + knowledgeBase, + userTableDefinitions, + workflow, + workflowBlocks, + workflowDeploymentVersion, +} from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { and, asc, eq, gt, inArray } from 'drizzle-orm' +import { isRecord, type SubBlockRecord } from '@/lib/workflows/persistence/remap-internal-ids' +import { invalidateDeployedStateCache } from '@/lib/workflows/persistence/utils' +import type { ForkFailedResource } from '@/lib/workspaces/fork/copy/copy-resources' +import type { ForkCopyResolver } from '@/lib/workspaces/fork/remap/fork-bootstrap' +import { + clearDependentsOnRemap, + type ForkRemapKind, + remapForkSubBlocks, +} from '@/lib/workspaces/fork/remap/remap-references' + +const logger = createLogger('WorkspaceForkCleanupFailed') + +/** Child workflow ids loaded per page so the block sweep never materializes the whole workspace. */ +const WORKFLOW_PAGE = 200 + +/** Deployment versions loaded per page so a workflow with many versions never loads all at once. */ +const DEPLOYMENT_VERSION_PAGE = 100 + +/** Identity-or-clear resolver: a failed id resolves to null (cleared), any other id to itself. */ +function buildFailedResolver(failedByKind: Map>): ForkCopyResolver { + return (kind, id) => (failedByKind.get(kind)?.has(id) ? null : id) +} + +/** + * Apply the identity-or-clear resolver to one block's subBlocks: a value (top-level selector or + * nested tool param) that resolves to a failed id is cleared, and its `dependsOn` children with it; + * everything else is left untouched. Returns the rewritten record plus whether anything changed. + * Shared by the draft block sweep and the deployment-version state sweep so both clear identically. + */ +function clearFailedSubBlockReferences( + subBlocks: SubBlockRecord, + blockType: string, + resolve: ForkCopyResolver +): { subBlocks: SubBlockRecord; changed: boolean } { + const result = remapForkSubBlocks(subBlocks, resolve, 'create') + // remappedKeys is non-empty only when a failed id was actually cleared, so a block that + // referenced nothing failed is reported unchanged without a write. + if (result.remappedKeys.size === 0) return { subBlocks, changed: false } + return { + subBlocks: clearDependentsOnRemap(result.subBlocks, blockType, result.remappedKeys), + changed: true, + } +} + +/** + * Clean up after a resource whose post-commit content fill failed: clear every subblock + * reference in the child workspace's workflows that points at the failed resource (so no + * subblock keeps a dead id), then drop the orphaned placeholder rows (a KB cascade-drops its + * documents + embeddings; a table cascade-drops its rows). In-content references inside copied + * skill/markdown bodies are intentionally left as graceful broken links rather than mutated. + * + * The deployed-version sweep covers the draft-affected workflows UNION this sync's deployed + * target workflows (`deployedTargetWorkflowIds`): a deployed version can reference the dropped + * placeholder even when the draft no longer does (the user edited the empty-looking block in the + * fill window), so scoping to draft divergence alone would miss it. + * + * Best-effort and isolated: a failure cleaning one resource is logged and the rest continue, so + * a cleanup error never aborts the others. The placeholder drop is SKIPPED when a reference-clear + * phase threw - dropping it then would turn an empty placeholder into a dangling reference to a + * deleted row; leaving it keeps the reference resolvable (to empty content) until a later retry. + * Returns the number of failed resources cleaned, for the fork activity report. + */ +export async function clearFailedForkResourceReferences(params: { + childWorkspaceId: string + failures: ForkFailedResource[] + /** Target workflows this sync deployed; their deployed versions are swept regardless of draft. */ + deployedTargetWorkflowIds?: string[] + requestId?: string +}): Promise { + const { childWorkspaceId, failures, requestId = 'unknown' } = params + if (failures.length === 0) return 0 + + const failedByKind = new Map>() + const markFailed = (kind: ForkRemapKind, id: string) => { + const set = failedByKind.get(kind) + if (set) set.add(id) + else failedByKind.set(kind, new Set([id])) + } + const tableIds: string[] = [] + const kbIds: string[] = [] + // Standalone documents copied into an already-existing target KB (the doc-into-mapped-KB sync + // path) - dropped individually, since their KB is not ours to remove. + const docIds: string[] = [] + for (const failure of failures) { + if (failure.kind === 'table') { + markFailed('table', failure.childId) + tableIds.push(failure.childId) + } else if (failure.kind === 'knowledge-document') { + markFailed('knowledge-document', failure.childId) + docIds.push(failure.childId) + } else if (failure.kind === 'file') { + // A failed file blob: clear `file-upload` references to its copied storage key. No row to + // drop - the metadata row is left in place so the user can re-upload the missing blob. + markFailed('file', failure.childKey) + } else { + markFailed('knowledge-base', failure.childId) + for (const docId of failure.documentChildIds) markFailed('knowledge-document', docId) + kbIds.push(failure.childId) + } + } + + // Whether BOTH reference-clear phases completed without throwing. The placeholder drop below is + // gated on this: if clearing threw, a workflow (draft or deployed version) may still reference + // the failed id, so dropping its placeholder would create a dangling reference to a deleted row. + let clearingSucceeded = true + + let affectedWorkflowIds: Set = new Set() + try { + affectedWorkflowIds = await clearFailedReferencesInWorkflows( + childWorkspaceId, + failedByKind, + requestId + ) + } catch (error) { + clearingSucceeded = false + logger.error(`[${requestId}] Failed to clear references for failed fork resources`, { + childWorkspaceId, + error: getErrorMessage(error), + }) + } + + // The same dead id also lives in DEPLOYED version states (the active one re-cut from the + // placeholder-referencing draft at this sync's redeploy, plus any newer version from a rare + // race), not only the draft just swept above. Sweep those too so "Load deployment" can't + // re-poison the cleaned draft with a dropped id, and the execute path never runs against content + // that no longer exists. Scope is the draft-affected workflows UNION this sync's deployed target + // workflows: a deployed version may reference the placeholder even when the draft no longer does + // (edited in the fill window), so draft divergence alone is not a reliable scope. Isolated: a + // failure here never aborts a sibling resource's cleanup. + const deployedSweepIds = new Set(affectedWorkflowIds) + for (const id of params.deployedTargetWorkflowIds ?? []) deployedSweepIds.add(id) + try { + await clearFailedReferencesInDeploymentVersions(deployedSweepIds, failedByKind, requestId) + } catch (error) { + clearingSucceeded = false + logger.error(`[${requestId}] Failed to clear references in fork deployment versions`, { + childWorkspaceId, + error: getErrorMessage(error), + }) + } + + // Drop the orphaned placeholders. The KB delete cascades its documents + embeddings; the + // table delete cascades its rows; a standalone document delete cascades its embeddings. Done + // after the refs are cleared so a drop failure can't strand a workflow still pointing at the + // (now content-less) resource - and ONLY when clearing succeeded, so a clear failure never + // turns an empty placeholder into a dangling reference to a deleted row. + if (!clearingSucceeded) { + logger.warn( + `[${requestId}] Skipping fork resource placeholder drop after a reference-clear failure`, + { + childWorkspaceId, + tables: tableIds.length, + knowledgeBases: kbIds.length, + documents: docIds.length, + } + ) + return failures.length + } + try { + if (tableIds.length > 0) { + await db.delete(userTableDefinitions).where(inArray(userTableDefinitions.id, tableIds)) + } + if (kbIds.length > 0) { + await db.delete(knowledgeBase).where(inArray(knowledgeBase.id, kbIds)) + } + if (docIds.length > 0) { + await db.delete(document).where(inArray(document.id, docIds)) + } + } catch (error) { + logger.error(`[${requestId}] Failed to drop orphaned fork resource placeholders`, { + childWorkspaceId, + error: getErrorMessage(error), + }) + } + + return failures.length +} + +/** + * Sweep the child workspace's workflow blocks and clear any subblock value (top-level selector + * or nested tool param) that resolves to a failed child resource id. Reuses the create-mode + * remap with an identity-or-clear resolver: a non-failed id resolves to itself (left unchanged), + * a failed id resolves to null and is cleared, and its `dependsOn` children are cleared too. + * Returns the ids of the workflows whose blocks actually had a reference cleared, so the deployed + * version sweep can scope itself to exactly those workflows. + */ +export async function clearFailedReferencesInWorkflows( + childWorkspaceId: string, + failedByKind: Map>, + requestId: string +): Promise> { + const resolve = buildFailedResolver(failedByKind) + const affectedWorkflowIds = new Set() + + let afterWorkflowId: string | null = null + for (;;) { + const workflowRows = await db + .select({ id: workflow.id }) + .from(workflow) + .where( + afterWorkflowId === null + ? eq(workflow.workspaceId, childWorkspaceId) + : and(eq(workflow.workspaceId, childWorkspaceId), gt(workflow.id, afterWorkflowId)) + ) + .orderBy(asc(workflow.id)) + .limit(WORKFLOW_PAGE) + if (workflowRows.length === 0) break + + const workflowIds = workflowRows.map((row) => row.id) + const blocks = await db + .select({ + id: workflowBlocks.id, + workflowId: workflowBlocks.workflowId, + type: workflowBlocks.type, + subBlocks: workflowBlocks.subBlocks, + }) + .from(workflowBlocks) + .where(inArray(workflowBlocks.workflowId, workflowIds)) + + for (const block of blocks) { + const current = (block.subBlocks ?? {}) as SubBlockRecord + const { subBlocks: cleared, changed } = clearFailedSubBlockReferences( + current, + block.type, + resolve + ) + if (!changed) continue + await db + .update(workflowBlocks) + .set({ subBlocks: cleared }) + .where(eq(workflowBlocks.id, block.id)) + affectedWorkflowIds.add(block.workflowId) + } + + if (workflowRows.length < WORKFLOW_PAGE) break + afterWorkflowId = workflowIds[workflowIds.length - 1] + } + + return affectedWorkflowIds +} + +/** + * Rewrite a deployment version's serialized state in memory, clearing every block subblock that + * resolves to a failed id (and its `dependsOn` children). Returns `changed: false` with the + * original state when no block referenced a failed id - so a version cut before this sync, which + * cannot contain the new placeholder id, is a no-op and never written back. Tolerant of a + * malformed/legacy state shape (anything that is not `{ blocks: {...} }` is left untouched). + */ +export function rewriteDeploymentVersionState( + state: unknown, + resolve: ForkCopyResolver +): { state: unknown; changed: boolean } { + if (!isRecord(state) || !isRecord(state.blocks)) return { state, changed: false } + + let nextBlocks: Record | null = null + for (const [blockId, block] of Object.entries(state.blocks)) { + if (!isRecord(block)) continue + const blockType = typeof block.type === 'string' ? block.type : undefined + if (!blockType || !isRecord(block.subBlocks)) continue + const { subBlocks: cleared, changed } = clearFailedSubBlockReferences( + block.subBlocks as SubBlockRecord, + blockType, + resolve + ) + if (!changed) continue + nextBlocks ??= { ...state.blocks } + nextBlocks[blockId] = { ...block, subBlocks: cleared } + } + + if (!nextBlocks) return { state, changed: false } + return { state: { ...state, blocks: nextBlocks }, changed: true } +} + +/** + * Rewrite the DEPLOYED version states of the workflows whose draft blocks were just swept. The dead + * placeholder id lives in any version re-cut from the (placeholder-referencing) draft at this sync's + * redeploy - in practice the active one - so it must be cleared there too, not only in the draft. + * Every version of each affected workflow is examined, but only versions whose state actually + * changes are written: an older version predates the sync and cannot contain the new id, so it is a + * no-op. After a version is rewritten its cached deployed state is evicted so execute/serve rebuilds + * from the cleaned snapshot. Bounded work (no long transaction): per-version short UPDATEs, versions + * keyset-paginated, and a per-workflow failure is logged without aborting the other workflows. + */ +export async function clearFailedReferencesInDeploymentVersions( + workflowIds: ReadonlySet, + failedByKind: Map>, + requestId: string +): Promise { + if (workflowIds.size === 0) return + const resolve = buildFailedResolver(failedByKind) + + for (const workflowId of workflowIds) { + try { + let afterVersion: number | null = null + for (;;) { + const versions = await db + .select({ + id: workflowDeploymentVersion.id, + version: workflowDeploymentVersion.version, + state: workflowDeploymentVersion.state, + }) + .from(workflowDeploymentVersion) + .where( + afterVersion === null + ? eq(workflowDeploymentVersion.workflowId, workflowId) + : and( + eq(workflowDeploymentVersion.workflowId, workflowId), + gt(workflowDeploymentVersion.version, afterVersion) + ) + ) + .orderBy(asc(workflowDeploymentVersion.version)) + .limit(DEPLOYMENT_VERSION_PAGE) + if (versions.length === 0) break + + for (const version of versions) { + const { state: nextState, changed } = rewriteDeploymentVersionState( + version.state, + resolve + ) + if (!changed) continue + await db + .update(workflowDeploymentVersion) + .set({ state: nextState }) + .where(eq(workflowDeploymentVersion.id, version.id)) + // Evict the post-migration deployed state cached by this immutable version id so the + // execute/serve path rebuilds from the cleaned snapshot. + invalidateDeployedStateCache(version.id) + } + + if (versions.length < DEPLOYMENT_VERSION_PAGE) break + afterVersion = versions[versions.length - 1].version + } + } catch (error) { + logger.error(`[${requestId}] Failed to clear references in deployment versions`, { + workflowId, + error: getErrorMessage(error), + }) + } + } +} diff --git a/apps/sim/lib/workspaces/fork/copy/content-copy-runner.test.ts b/apps/sim/lib/workspaces/fork/copy/content-copy-runner.test.ts new file mode 100644 index 00000000000..0ee8e1b879d --- /dev/null +++ b/apps/sim/lib/workspaces/fork/copy/content-copy-runner.test.ts @@ -0,0 +1,36 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { serializeContentRefMaps } from '@/lib/workspaces/fork/copy/content-copy-runner' + +describe('serializeContentRefMaps', () => { + it('converts each map to a record and drops empty maps', () => { + const result = serializeContentRefMaps({ + workspaceId: { from: 'src', to: 'dst' }, + fileKeys: new Map([['k1', 'k2']]), + fileIds: new Map(), + workflows: new Map([['wf-src', 'wf-dst']]), + knowledgeBases: new Map([['kb-src', 'kb-dst']]), + skills: new Map([['sk-src', 'sk-dst']]), + }) + + expect(result.workspaceId).toEqual({ from: 'src', to: 'dst' }) + expect(result.fileKeys).toEqual({ k1: 'k2' }) + // An empty map is omitted rather than serialized to `{}`. + expect(result.fileIds).toBeUndefined() + expect(result.workflows).toEqual({ 'wf-src': 'wf-dst' }) + expect(result.knowledgeBases).toEqual({ 'kb-src': 'kb-dst' }) + expect(result.skills).toEqual({ 'sk-src': 'sk-dst' }) + // Maps not supplied stay undefined. + expect(result.tables).toBeUndefined() + expect(result.folders).toBeUndefined() + }) + + it('passes an undefined workspaceId through unchanged', () => { + const result = serializeContentRefMaps({}) + expect(result.workspaceId).toBeUndefined() + expect(result.fileKeys).toBeUndefined() + expect(result.skills).toBeUndefined() + }) +}) diff --git a/apps/sim/lib/workspaces/fork/copy/content-copy-runner.ts b/apps/sim/lib/workspaces/fork/copy/content-copy-runner.ts index 16f76dbf3d0..37c0479af90 100644 --- a/apps/sim/lib/workspaces/fork/copy/content-copy-runner.ts +++ b/apps/sim/lib/workspaces/fork/copy/content-copy-runner.ts @@ -1,10 +1,32 @@ import { db } from '@sim/db' +import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { isTriggerDevEnabled } from '@/lib/core/config/env-flags' +import { runDetached } from '@/lib/core/utils/background' import { finishBackgroundWork } from '@/lib/workspaces/fork/background-work/store' +import { clearFailedForkResourceReferences } from '@/lib/workspaces/fork/copy/cleanup-failed' import type { BlobCopyTask } from '@/lib/workspaces/fork/copy/copy-files' import { executeForkFileBlobCopies } from '@/lib/workspaces/fork/copy/copy-files' -import type { ForkContentPlan } from '@/lib/workspaces/fork/copy/copy-resources' +import type { ForkContentPlan, ForkFailedResource } from '@/lib/workspaces/fork/copy/copy-resources' import { copyForkResourceContent } from '@/lib/workspaces/fork/copy/copy-resources' +import type { ForkContentRefMaps } from '@/lib/workspaces/fork/remap/remap-content-refs' + +const logger = createLogger('WorkspaceForkContentCopy') + +/** + * JSON-serializable form of {@link ForkContentRefMaps} (Maps become Records) so the in-content + * reference maps survive the Trigger.dev payload boundary. Rehydrated to Maps in the runner. + */ +export interface SerializableForkContentRefMaps { + workspaceId?: { from: string; to: string } + fileKeys?: Record + fileIds?: Record + workflows?: Record + knowledgeBases?: Record + tables?: Record + skills?: Record + folders?: Record +} /** * Serializable payload for the post-fork heavy-content copy. Runs either as a @@ -14,15 +36,66 @@ import { copyForkResourceContent } from '@/lib/workspaces/fork/copy/copy-resourc export interface ForkContentCopyPayload { contentPlan: ForkContentPlan blobTasks: BlobCopyTask[] + /** In-content reference maps for rewriting copied markdown blobs (serialized form). */ + contentRefMaps?: SerializableForkContentRefMaps /** * `background_work_status` row to finish when the copy ends, so the source workspace's * Manage Forks -> Activity entry resolves (completed / warning / error). Started right * after the fork commits so it's visible immediately. */ statusId?: string + /** + * Target workflow ids this sync deployed (promote's deploy loop). When a copied resource's + * fill fails, its dropped placeholder must be cleared from these workflows' DEPLOYED version + * states too - a deployed version can reference the placeholder even when the draft no longer + * does (edited in the fill window), so the cleanup unions these with the draft-affected set + * rather than relying on draft divergence. Empty/omitted for fork-create (child is undeployed). + */ + deployedTargetWorkflowIds?: string[] requestId?: string } +const toRefMap = (record?: Record): Map | undefined => + record ? new Map(Object.entries(record)) : undefined + +const fromRefMap = (map?: ReadonlyMap): Record | undefined => + map && map.size > 0 ? Object.fromEntries(map) : undefined + +/** + * Convert the Map-based {@link ForkContentRefMaps} to its JSON-serializable form for the + * Trigger.dev payload. Empty maps are dropped (omitted). Single source of truth for the + * Map->Record direction, paired with {@link deserializeContentRefMaps}. + */ +export function serializeContentRefMaps(maps: ForkContentRefMaps): SerializableForkContentRefMaps { + return { + workspaceId: maps.workspaceId, + fileKeys: fromRefMap(maps.fileKeys), + fileIds: fromRefMap(maps.fileIds), + workflows: fromRefMap(maps.workflows), + knowledgeBases: fromRefMap(maps.knowledgeBases), + tables: fromRefMap(maps.tables), + skills: fromRefMap(maps.skills), + folders: fromRefMap(maps.folders), + } +} + +/** Rehydrate the serialized content-ref maps to the Map-based {@link ForkContentRefMaps}. */ +function deserializeContentRefMaps( + serialized?: SerializableForkContentRefMaps +): ForkContentRefMaps | undefined { + if (!serialized) return undefined + return { + workspaceId: serialized.workspaceId, + fileKeys: toRefMap(serialized.fileKeys), + fileIds: toRefMap(serialized.fileIds), + workflows: toRefMap(serialized.workflows), + knowledgeBases: toRefMap(serialized.knowledgeBases), + tables: toRefMap(serialized.tables), + skills: toRefMap(serialized.skills), + folders: toRefMap(serialized.folders), + } +} + /** * Copy the heavy fork content after the fork transaction has committed: table * rows, KB documents + embeddings (keyset-paginated), and file blobs. Best-effort @@ -33,8 +106,23 @@ export interface ForkContentCopyPayload { export async function runForkContentCopy(payload: ForkContentCopyPayload): Promise { const { contentPlan, blobTasks, statusId, requestId } = payload try { - const resourceCounts = await copyForkResourceContent({ contentPlan, requestId }) - const fileCounts = await executeForkFileBlobCopies(blobTasks, requestId) + const contentRefMaps = deserializeContentRefMaps(payload.contentRefMaps) + const resourceCounts = await copyForkResourceContent({ contentPlan, contentRefMaps, requestId }) + const fileCounts = await executeForkFileBlobCopies(blobTasks, requestId, contentRefMaps) + // A resource whose content fill failed leaves a dangling reference: a table/KB/doc placeholder + // its workflows still point at, or a `file-upload` whose copied blob is missing. Clear those + // references (draft + deployed versions) and drop the table/KB/doc placeholder so nothing + // dangles; a failed file leaves its metadata row (re-uploadable) but has its refs cleared. + const fileFailures: ForkFailedResource[] = fileCounts.failedTargetKeys.map((childKey) => ({ + kind: 'file', + childKey, + })) + const clearedReferences = await clearFailedForkResourceReferences({ + childWorkspaceId: contentPlan.childWorkspaceId, + failures: [...resourceCounts.failures, ...fileFailures], + deployedTargetWorkflowIds: payload.deployedTargetWorkflowIds, + requestId, + }) const copied = resourceCounts.copied + fileCounts.copied const failed = resourceCounts.failed + fileCounts.failed if (statusId) { @@ -44,7 +132,7 @@ export async function runForkContentCopy(payload: ForkContentCopyPayload): Promi failed > 0 ? `Copied ${copied} item${copied === 1 ? '' : 's'}; ${failed} could not be copied` : `Copied ${copied} item${copied === 1 ? '' : 's'}`, - metadata: { copied, failed }, + metadata: { copied, failed, clearedReferences }, }) } } catch (error) { @@ -57,3 +145,42 @@ export async function runForkContentCopy(payload: ForkContentCopyPayload): Promi throw error } } + +/** + * Schedule the post-commit heavy-content copy off the request path. Uses the Trigger.dev task + * when enabled (so it survives an app deploy), else `runDetached` inline best-effort. Shared by + * both fork and sync - only the `detachedLabel` (and so the inline job's name) differs. Never + * throws: a scheduling failure is logged and, when a status row exists, marks it failed, so a + * committed fork/sync is never turned into a 500 by a background-scheduling hiccup. + */ +export async function scheduleForkContentCopy( + payload: ForkContentCopyPayload, + options: { detachedLabel: string; requestId?: string } +): Promise { + const { detachedLabel, requestId = 'unknown' } = options + try { + if (isTriggerDevEnabled) { + const [{ forkContentCopyTask }, { tasks }, { resolveTriggerRegion }] = await Promise.all([ + import('@/background/fork-content-copy'), + import('@trigger.dev/sdk'), + import('@/lib/core/async-jobs/region'), + ]) + await tasks.trigger('fork-content-copy', payload, { + region: await resolveTriggerRegion(), + }) + } else { + runDetached(detachedLabel, () => runForkContentCopy(payload)) + } + } catch (error) { + logger.error(`[${requestId}] Failed to schedule fork content copy`, { + detachedLabel, + error: getErrorMessage(error), + }) + if (payload.statusId) { + await finishBackgroundWork(db, payload.statusId, { + status: 'failed', + error: getErrorMessage(error, 'Could not start the background copy'), + }).catch(() => {}) + } + } +} diff --git a/apps/sim/lib/workspaces/fork/copy/copy-files.ts b/apps/sim/lib/workspaces/fork/copy/copy-files.ts index 20bf90c59ad..e7e8f5080eb 100644 --- a/apps/sim/lib/workspaces/fork/copy/copy-files.ts +++ b/apps/sim/lib/workspaces/fork/copy/copy-files.ts @@ -2,15 +2,28 @@ import { workspaceFiles } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' -import { and, eq, inArray, isNull } from 'drizzle-orm' +import { and, eq, inArray, isNull, or } from 'drizzle-orm' import type { DbOrTx } from '@/lib/db/types' import { generateWorkspaceFileKey } from '@/lib/uploads/contexts/workspace/workspace-file-manager' import { downloadFile, uploadFile } from '@/lib/uploads/core/storage-service' import type { StorageContext } from '@/lib/uploads/shared/types' import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' +import { + type ForkContentRefMaps, + rewriteForkContentRefs, +} from '@/lib/workspaces/fork/remap/remap-content-refs' const logger = createLogger('WorkspaceForkCopyFiles') +const MARKDOWN_CONTENT_TYPES = new Set(['text/markdown', 'text/x-markdown']) + +/** Whether a copied blob is markdown text whose in-content references should be rewritten. */ +function isMarkdownBlob(task: Pick): boolean { + if (MARKDOWN_CONTENT_TYPES.has(task.contentType)) return true + const name = task.fileName.toLowerCase() + return name.endsWith('.md') || name.endsWith('.mdx') || name.endsWith('.markdown') +} + export interface BlobCopyTask { sourceKey: string targetKey: string @@ -25,10 +38,17 @@ export interface PlanForkFileCopiesResult { /** * source storage key -> child storage key. `file-upload` subblocks reference * files by storage key (not `workspace_files.id`), so the fork remap keys on the - * storage key. File identity is not persisted in the fork resource map - files - * are a fork-copy-only resource (not remapped on promote). + * storage key. At sync time this map is persisted in the fork resource map + * (`resourceType: 'file'`, keyed by storage key) so a re-sync resolves the copy + * instead of re-copying; at create-fork time it is not (the child is brand new). */ keyMap: Map + /** + * source `workspace_files.id` -> child id. Used to rewrite in-content file references + * that key on the file id (`sim:file/`, `/api/files/view/`, the in-app files + * path) inside copied skill/markdown content; not persisted in the fork resource map. + */ + idMap: Map /** Blob duplications to run after the fork transaction commits. */ blobTasks: BlobCopyTask[] } @@ -40,31 +60,44 @@ export interface PlanForkFileCopiesResult { * (its idempotent metadata insert reuses the row), and both must run after the * child workspace row exists (FK). Runs in the fork transaction; blob I/O is * deferred to {@link executeForkFileBlobCopies}. + * + * Files are selected EITHER by `workspace_files.id` (the fork modal's picker lists files + * by id) OR by storage `key` (sync references key files by their storage key, not id). At + * least one of the two must be non-empty; both may be supplied (their matched rows union). */ export async function planForkFileCopies(params: { tx: DbOrTx sourceWorkspaceId: string childWorkspaceId: string userId: string - fileIds: string[] + fileIds?: string[] + fileKeys?: string[] now: Date }): Promise { - const { tx, sourceWorkspaceId, childWorkspaceId, userId, fileIds, now } = params + const { tx, sourceWorkspaceId, childWorkspaceId, userId, now } = params + const fileIds = params.fileIds ?? [] + const fileKeys = params.fileKeys ?? [] const keyMap = new Map() + const idMap = new Map() const blobTasks: BlobCopyTask[] = [] - if (fileIds.length === 0) return { keyMap, blobTasks } + if (fileIds.length === 0 && fileKeys.length === 0) return { keyMap, idMap, blobTasks } - // Batch the metadata read (one query for all selected files) instead of a per-file - // lookup: non-deleted, scoped to the source workspace, and restricted to durable - // `workspace` files. Only workspace files are forkable - chat/copilot/mothership - // uploads are session-scoped and their chat-bound unique index can't be duplicated - - // so any non-workspace id passed here is ignored rather than copied. + // Match by id and/or storage key (OR'd) so either selection shape resolves to the same + // source rows. Batch the metadata read (one query for all selected files): non-deleted, + // scoped to the source workspace, and restricted to durable `workspace` files. Only + // workspace files are forkable - chat/copilot/mothership uploads are session-scoped and + // their chat-bound unique index can't be duplicated - so any non-workspace id/key passed + // here is ignored rather than copied. + const selectors = [ + fileIds.length > 0 ? inArray(workspaceFiles.id, fileIds) : undefined, + fileKeys.length > 0 ? inArray(workspaceFiles.key, fileKeys) : undefined, + ].filter((clause): clause is NonNullable => clause !== undefined) const metas = await tx .select() .from(workspaceFiles) .where( and( - inArray(workspaceFiles.id, fileIds), + selectors.length === 1 ? selectors[0] : or(...selectors), eq(workspaceFiles.workspaceId, sourceWorkspaceId), eq(workspaceFiles.context, 'workspace'), isNull(workspaceFiles.deletedAt) @@ -87,6 +120,7 @@ export async function planForkFileCopies(params: { uploadedAt: now, }) keyMap.set(meta.key, targetKey) + idMap.set(meta.id, childFileId) blobTasks.push({ sourceKey: meta.key, targetKey, @@ -98,21 +132,27 @@ export async function planForkFileCopies(params: { }) } - return { keyMap, blobTasks } + return { keyMap, idMap, blobTasks } } /** * Duplicate each planned file blob to its new key. `uploadFile`'s metadata insert * is idempotent on the key (the row was already created in the transaction), so - * this only copies bytes. Best-effort: a failed blob leaves the metadata row - * pointing at a missing object, which the user can re-upload. + * this only copies bytes. Markdown blobs additionally have their in-content references + * (`sim:` links, embedded file/image URLs) rewritten through `contentRefMaps` so they + * point at the copied resources (unmapped targets are left as graceful broken links). + * Best-effort: a content-rewrite failure falls back to copying the raw bytes. A failed + * blob's child storage key is returned in `failedTargetKeys` so the caller can clear the + * `file-upload` references pointing at the now-missing object (the metadata row is left in + * place, so the user can still re-upload the blob). */ export async function executeForkFileBlobCopies( blobTasks: BlobCopyTask[], - requestId = 'unknown' -): Promise<{ copied: number; failed: number }> { + requestId = 'unknown', + contentRefMaps?: ForkContentRefMaps +): Promise<{ copied: number; failed: number; failedTargetKeys: string[] }> { let copied = 0 - let failed = 0 + const failedTargetKeys: string[] = [] for (const task of blobTasks) { try { const buffer = await downloadFile({ @@ -120,8 +160,21 @@ export async function executeForkFileBlobCopies( context: task.context, maxBytes: MAX_FILE_SIZE, }) + let body: Buffer = buffer + if (contentRefMaps && isMarkdownBlob(task)) { + try { + const text = buffer.toString('utf8') + const rewritten = rewriteForkContentRefs(text, contentRefMaps) + if (rewritten !== text) body = Buffer.from(rewritten, 'utf8') + } catch (error) { + logger.warn(`[${requestId}] Failed to rewrite markdown blob content; copying raw bytes`, { + targetKey: task.targetKey, + error: getErrorMessage(error), + }) + } + } await uploadFile({ - file: buffer, + file: body, fileName: task.fileName, contentType: task.contentType, context: task.context, @@ -135,12 +188,12 @@ export async function executeForkFileBlobCopies( }) copied += 1 } catch (error) { - failed += 1 + failedTargetKeys.push(task.targetKey) logger.warn(`[${requestId}] Failed to copy file blob during fork`, { targetKey: task.targetKey, error: getErrorMessage(error), }) } } - return { copied, failed } + return { copied, failed: failedTargetKeys.length, failedTargetKeys } } diff --git a/apps/sim/lib/workspaces/fork/copy/copy-resources.test.ts b/apps/sim/lib/workspaces/fork/copy/copy-resources.test.ts new file mode 100644 index 00000000000..10626946ef3 --- /dev/null +++ b/apps/sim/lib/workspaces/fork/copy/copy-resources.test.ts @@ -0,0 +1,396 @@ +/** + * @vitest-environment node + */ +import { + dbChainMock, + dbChainMockFns, + resetDbChainMock, + storageServiceMock, + storageServiceMockFns, +} from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@sim/db', () => dbChainMock) +vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock) + +import type { DbOrTx } from '@/lib/db/types' +import { + copyForkResourceContainers, + copyForkResourceContent, + type ForkContentPlan, + planForkMappedKbDocumentCopies, +} from '@/lib/workspaces/fork/copy/copy-resources' +import type { ForkReferenceResolver } from '@/lib/workspaces/fork/remap/remap-references' + +function basePlan(overrides: Partial = {}): ForkContentPlan { + return { + sourceWorkspaceId: 'src-ws', + childWorkspaceId: 'child-ws', + userId: 'user-1', + tables: [], + knowledgeBases: [], + skills: [], + documents: [], + ...overrides, + } +} + +const sourceDoc = { + id: 'doc-1', + knowledgeBaseId: 'src-kb', + storageKey: 'kb/source-key', + fileUrl: '/api/files/serve/kb%2Fsource-key', + filename: 'report.pdf', + mimeType: 'application/pdf', +} + +describe('copyForkResourceContent', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + storageServiceMockFns.mockDownloadFile.mockResolvedValue(Buffer.from('blob-bytes')) + storageServiceMockFns.mockUploadFile.mockResolvedValue({ + key: 'kb/child-key', + path: '/api/files/serve/kb/child-key', + }) + }) + + it('rewrites in-workspace resource URLs nested in copied table cell data', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + id: 'r1', + tableId: 'src-tbl', + workspaceId: 'src-ws', + data: { + kb: '/workspace/src-ws/knowledge/kb-1', + nested: { wf: '/workspace/src-ws/w/wf-1' }, + plain: 'no url here', + }, + }, + ]) + + const result = await copyForkResourceContent({ + contentPlan: basePlan({ tables: [{ sourceId: 'src-tbl', childId: 'child-tbl' }] }), + contentRefMaps: { + workspaceId: { from: 'src-ws', to: 'child-ws' }, + knowledgeBases: new Map([['kb-1', 'kb-2']]), + workflows: new Map([['wf-1', 'wf-2']]), + }, + requestId: 'test', + }) + + expect(result.failed).toBe(0) + // The first insert is the table-rows copy (no KBs/docs/skills in this plan). + const inserted = dbChainMockFns.values.mock.calls[0][0] as Array<{ + data: { kb: string; nested: { wf: string }; plain: string } + }> + expect(inserted[0].data.kb).toBe('/workspace/child-ws/knowledge/kb-2') + expect(inserted[0].data.nested.wf).toBe('/workspace/child-ws/w/wf-2') + expect(inserted[0].data.plain).toBe('no url here') + }) + + it('#1 binds a copied KB document blob to the CHILD workspace + initiating user', async () => { + // One live document page, then the embeddings page resolves empty (default). + dbChainMockFns.limit.mockResolvedValueOnce([sourceDoc]) + + const result = await copyForkResourceContent({ + contentPlan: basePlan({ + knowledgeBases: [{ sourceId: 'src-kb', childId: 'child-kb', documentIdMap: {} }], + }), + requestId: 'test', + }) + + expect(result.failed).toBe(0) + expect(result.copied).toBe(1) + expect(storageServiceMockFns.mockUploadFile).toHaveBeenCalledTimes(1) + const uploadArg = storageServiceMockFns.mockUploadFile.mock.calls[0][0] + expect(uploadArg.context).toBe('knowledge-base') + expect(uploadArg.preserveKey).toBe(true) + // The ownership binding is what verifyKBFileAccess resolves the owning workspace from; + // it must name the CHILD workspace and the initiating user, or the copy is download-denied. + expect(uploadArg.metadata).toEqual({ + userId: 'user-1', + workspaceId: 'child-ws', + originalName: 'report.pdf', + }) + }) + + it('#4 rewrites a copied skill body post-commit via db.update using the content maps', async () => { + const result = await copyForkResourceContent({ + contentPlan: basePlan({ + skills: [{ childId: 'child-skill-1', content: 'see [K](sim:knowledge/src-kb)' }], + }), + contentRefMaps: { knowledgeBases: new Map([['src-kb', 'child-kb']]) }, + requestId: 'test', + }) + + expect(result.failed).toBe(0) + expect(dbChainMockFns.update).toHaveBeenCalledTimes(1) + expect(dbChainMockFns.set).toHaveBeenCalledWith({ + content: 'see [K](sim:knowledge/child-kb)', + }) + }) + + it('#4 leaves a skill untouched when nothing in its body remaps', async () => { + const result = await copyForkResourceContent({ + contentPlan: basePlan({ + skills: [{ childId: 'child-skill-1', content: 'no references here' }], + }), + contentRefMaps: { knowledgeBases: new Map([['src-kb', 'child-kb']]) }, + requestId: 'test', + }) + + expect(result.failed).toBe(0) + expect(dbChainMockFns.update).not.toHaveBeenCalled() + }) + + it('#4 skips the skill rewrite entirely when no content maps are supplied', async () => { + await copyForkResourceContent({ + contentPlan: basePlan({ + skills: [{ childId: 'child-skill-1', content: 'see [K](sim:knowledge/src-kb)' }], + }), + requestId: 'test', + }) + + expect(dbChainMockFns.update).not.toHaveBeenCalled() + }) + + it('#3 fails the whole KB (all-or-nothing) when one document copy throws', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([sourceDoc]) + // The document row insert throws; the blob copy is best-effort (never throws) so the + // failure must come from the persisted copy, marking the entire KB failed for cleanup. + dbChainMockFns.values.mockImplementationOnce(() => { + throw new Error('insert failed') + }) + + const result = await copyForkResourceContent({ + contentPlan: basePlan({ + knowledgeBases: [{ sourceId: 'src-kb', childId: 'child-kb', documentIdMap: {} }], + }), + requestId: 'test', + }) + + expect(result.copied).toBe(0) + expect(result.failed).toBe(1) + expect(result.failures).toEqual([ + { kind: 'knowledge-base', childId: 'child-kb', documentChildIds: [] }, + ]) + }) + + it('U-docs: fills a document copied into an existing target KB (blob re-key + placeholder update)', async () => { + const result = await copyForkResourceContent({ + contentPlan: basePlan({ + documents: [ + { + sourceDocId: 'doc-1', + childDocId: 'child-doc-1', + childKnowledgeBaseId: 'existing-target-kb', + storageKey: 'kb/source-key', + filename: 'report.pdf', + mimeType: 'application/pdf', + }, + ], + }), + requestId: 'test', + }) + + expect(result.failed).toBe(0) + expect(result.copied).toBe(1) + // The blob is re-keyed and the pre-created placeholder row's blob fields are updated. + expect(storageServiceMockFns.mockUploadFile).toHaveBeenCalledTimes(1) + expect(dbChainMockFns.update).toHaveBeenCalledTimes(1) + }) + + it('U-docs: a failed document fill is reported as a knowledge-document failure (for cleanup)', async () => { + // The placeholder blob update throws; the doc fails on its own without touching its KB. + dbChainMockFns.set.mockImplementationOnce(() => { + throw new Error('update failed') + }) + + const result = await copyForkResourceContent({ + contentPlan: basePlan({ + documents: [ + { + sourceDocId: 'doc-1', + childDocId: 'child-doc-1', + childKnowledgeBaseId: 'existing-target-kb', + storageKey: 'kb/source-key', + filename: 'report.pdf', + mimeType: 'application/pdf', + }, + ], + }), + requestId: 'test', + }) + + expect(result.copied).toBe(0) + expect(result.failed).toBe(1) + expect(result.failures).toEqual([{ kind: 'knowledge-document', childId: 'child-doc-1' }]) + }) +}) + +describe('copyForkResourceContainers custom-tool code env rewrite', () => { + function makeContainerTx(rows: Array>) { + const inserted: Array> = [] + const tx = { + select: () => ({ from: () => ({ where: () => Promise.resolve(rows) }) }), + insert: () => ({ + values: (values: Array>) => { + inserted.push(...values) + return Promise.resolve() + }, + }), + } + return { tx: tx as unknown as DbOrTx, inserted } + } + + const customToolSelection = { + customTools: ['ct-1'], + skills: [], + workflowMcpServers: [], + tables: [], + knowledgeBases: [], + } + + it('rewrites {{ENV}} refs in copied custom-tool code when a sync renames the env var', async () => { + const { tx, inserted } = makeContainerTx([ + { id: 'ct-1', title: 'Tool', code: 'fetch("{{SLACK_API_KEY}}", "{{KEEP}}")' }, + ]) + await copyForkResourceContainers({ + tx, + sourceWorkspaceId: 'src-ws', + childWorkspaceId: 'child-ws', + userId: 'user-1', + now: new Date(), + selection: customToolSelection, + workflowIdMap: new Map(), + resolveEnvName: (key) => (key === 'SLACK_API_KEY' ? 'SLACK_API_KEY_TEST' : key), + }) + expect(inserted).toHaveLength(1) + // The renamed key is rewritten; the same-name key is left verbatim. + expect(inserted[0].code).toBe('fetch("{{SLACK_API_KEY_TEST}}", "{{KEEP}}")') + expect(inserted[0].workspaceId).toBe('child-ws') + }) + + it('preserves custom-tool code verbatim when no env resolver is provided (fork-create)', async () => { + const { tx, inserted } = makeContainerTx([ + { id: 'ct-1', title: 'Tool', code: 'fetch("{{SLACK_API_KEY}}")' }, + ]) + await copyForkResourceContainers({ + tx, + sourceWorkspaceId: 'src-ws', + childWorkspaceId: 'child-ws', + userId: 'user-1', + now: new Date(), + selection: customToolSelection, + workflowIdMap: new Map(), + }) + expect(inserted[0].code).toBe('fetch("{{SLACK_API_KEY}}")') + }) +}) + +describe('planForkMappedKbDocumentCopies', () => { + const sourceRow = (id: string, knowledgeBaseId: string) => ({ + id, + knowledgeBaseId, + storageKey: `kb/${id}`, + filename: `${id}.pdf`, + mimeType: 'application/pdf', + connectorId: 'connector-1', + deletedAt: null, + archivedAt: null, + }) + + function makeTx(docs: ReturnType[]) { + const inserted: Array> = [] + let selectCalled = false + const tx = { + select: () => { + selectCalled = true + return { from: () => ({ where: () => Promise.resolve(docs) }) } + }, + insert: () => ({ + values: (rows: Array>) => { + inserted.push(...rows) + return Promise.resolve() + }, + }), + } + return { tx: tx as unknown as DbOrTx, inserted, wasSelectCalled: () => selectCalled } + } + + const mappedKbResolver: ForkReferenceResolver = (kind, id) => + kind === 'knowledge-base' && id === 'src-kb' ? 'target-kb' : null + + it('places a referenced doc into its already-mapped existing KB and returns the maps', async () => { + const { tx, inserted } = makeTx([sourceRow('doc-1', 'src-kb')]) + const result = await planForkMappedKbDocumentCopies({ + tx, + resolver: mappedKbResolver, + referencedDocumentIds: ['doc-1'], + alreadyCopiedSourceDocIds: new Set(), + }) + + const childId = result.docIdMap.get('doc-1') + expect(childId).toBeTruthy() + expect(inserted).toHaveLength(1) + expect(inserted[0]).toMatchObject({ + id: childId, + knowledgeBaseId: 'target-kb', + connectorId: null, + deletedAt: null, + archivedAt: null, + }) + expect(result.mappingEntries).toEqual([ + { resourceType: 'knowledge_document', parentResourceId: 'doc-1', childResourceId: childId }, + ]) + expect(result.documents).toEqual([ + { + sourceDocId: 'doc-1', + childDocId: childId, + childKnowledgeBaseId: 'target-kb', + storageKey: 'kb/doc-1', + filename: 'doc-1.pdf', + mimeType: 'application/pdf', + }, + ]) + }) + + it('skips a referenced doc whose parent KB is not mapped (reference is left to be cleared)', async () => { + const { tx, inserted } = makeTx([sourceRow('doc-1', 'unmapped-kb')]) + const result = await planForkMappedKbDocumentCopies({ + tx, + resolver: mappedKbResolver, + referencedDocumentIds: ['doc-1'], + alreadyCopiedSourceDocIds: new Set(), + }) + expect(inserted).toHaveLength(0) + expect(result.docIdMap.size).toBe(0) + expect(result.documents).toHaveLength(0) + }) + + it('skips a doc already placed under a copied KB this sync (no duplicate query)', async () => { + const { tx, wasSelectCalled } = makeTx([sourceRow('doc-1', 'src-kb')]) + const result = await planForkMappedKbDocumentCopies({ + tx, + resolver: mappedKbResolver, + referencedDocumentIds: ['doc-1'], + alreadyCopiedSourceDocIds: new Set(['doc-1']), + }) + expect(result.documents).toHaveLength(0) + expect(wasSelectCalled()).toBe(false) + }) + + it('skips a doc that already resolves (mapped by a prior sync)', async () => { + const { tx, wasSelectCalled } = makeTx([sourceRow('doc-1', 'src-kb')]) + const result = await planForkMappedKbDocumentCopies({ + tx, + resolver: (kind, id) => + kind === 'knowledge-document' && id === 'doc-1' ? 'existing-child-doc' : null, + referencedDocumentIds: ['doc-1'], + alreadyCopiedSourceDocIds: new Set(), + }) + expect(result.documents).toHaveLength(0) + expect(wasSelectCalled()).toBe(false) + }) +}) diff --git a/apps/sim/lib/workspaces/fork/copy/copy-resources.ts b/apps/sim/lib/workspaces/fork/copy/copy-resources.ts index 2fbde548ece..c7d064b39e3 100644 --- a/apps/sim/lib/workspaces/fork/copy/copy-resources.ts +++ b/apps/sim/lib/workspaces/fork/copy/copy-resources.ts @@ -4,22 +4,35 @@ import { document, embedding, knowledgeBase, - mcpServers, skill, userTableDefinitions, userTableRows, + workflowMcpServer, } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { and, asc, eq, gt, inArray, isNull, type SQL } from 'drizzle-orm' +import { mapWithConcurrency } from '@/lib/core/utils/concurrency' import type { DbOrTx } from '@/lib/db/types' -import { generateMcpServerId } from '@/lib/mcp/utils' import type { TableSchema } from '@/lib/table/types' +import { generateKnowledgeBaseFileKey } from '@/lib/uploads/contexts/knowledge-base/knowledge-base-file-manager' +import { downloadFile, uploadFile } from '@/lib/uploads/core/storage-service' +import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' +import { isRecord } from '@/lib/workflows/persistence/remap-internal-ids' import type { ForkMappingUpsert, ForkResourceType, } from '@/lib/workspaces/fork/mapping/mapping-store' +import { + type ForkContentRefMaps, + rewriteForkContentRefs, + rewriteForkResourceUrls, +} from '@/lib/workspaces/fork/remap/remap-content-refs' +import { + type ForkReferenceResolver, + rewriteEnvRefsInText, +} from '@/lib/workspaces/fork/remap/remap-references' import { remapForkTableWorkflowGroups } from '@/lib/workspaces/fork/remap/remap-table-groups' const logger = createLogger('WorkspaceForkCopyResources') @@ -27,6 +40,13 @@ const logger = createLogger('WorkspaceForkCopyResources') /** Page size for the post-transaction bulk content copy (keyset-paginated). */ const CONTENT_PAGE = 500 +/** + * Max documents copied concurrently within one KB page. Bounds fan-out (blob copy + per-doc + * embedding paging) so a large page doesn't issue every request at once; the keyset loop still + * processes one page at a time, so peak concurrency stays at this cap regardless of KB size. + */ +const KB_DOCUMENT_COPY_CONCURRENCY = 5 + export interface CopyResourcesParams { tx: DbOrTx sourceWorkspaceId: string @@ -37,12 +57,25 @@ export interface CopyResourcesParams { selection: { customTools: string[] skills: string[] - mcpServers: string[] + workflowMcpServers: string[] tables: string[] knowledgeBases: string[] } /** source workflow id -> child workflow id, for table workflow-group remap. */ workflowIdMap: Map + /** + * Source KB-document ids referenced by the copied workflows (document-selector values + + * nested `documentId` tool params). Documents in this set whose parent KB is being copied + * get a placeholder row + a persisted `knowledge_document` id map inside the transaction, so + * the reference remaps to the copied document instead of being cleared. Defaults to none. + */ + referencedDocumentIds?: string[] + /** + * Resolve a source env-var name to its target name, so a copied custom tool's `code` (which + * embeds `{{ENV}}` refs) is rewritten when a sync renames an env var. Provided by promote (the + * plan resolver); omitted by fork-create, which preserves env names verbatim (no rewrite). + */ + resolveEnvName?: (key: string) => string | null | undefined } export interface ForkContentPlanEntry { @@ -50,21 +83,82 @@ export interface ForkContentPlanEntry { childId: string } +/** + * A KB to copy post-commit, plus the source-document -> child-document id map for the + * documents that were pre-created as placeholders in the transaction (referenced by copied + * workflows). The content phase fills those exact child ids (and copies the rest fresh) so a + * remapped `document-selector` reference can never dangle. + */ +export interface ForkContentKbEntry extends ForkContentPlanEntry { + documentIdMap: Record +} + +/** + * A copied skill whose body's in-content references (`sim:` links + embedded file URLs) are + * rewritten post-commit. The child row is inserted in the fork tx with the SOURCE body; the + * content phase rewrites it best-effort so it points at the copied child resources (an unmapped + * target degrades to a graceful broken link, never an FK/subblock reference). + */ +export interface ForkContentSkillEntry { + childId: string + content: string +} + +/** + * A single source document copied into an EXISTING (already-mapped, not copied this sync) target + * KB - the sync-only "document into mapped KB" path (U-docs). The placeholder row is inserted in + * the promote tx at `childDocId` under `childKnowledgeBaseId`; the content phase fills its + * embeddings (and re-keys its blob) best-effort. Distinct from {@link ForkContentKbEntry}, which + * copies a whole KB's documents - here only the referenced documents are placed, since the KB + * itself already exists in the target with its own documents. + */ +export interface ForkContentDocumentEntry { + sourceDocId: string + childDocId: string + childKnowledgeBaseId: string + /** Source blob fields captured at placeholder time, for the post-commit blob re-key. */ + storageKey: string | null + filename: string + mimeType: string +} + /** Bulk content to copy AFTER the fork transaction commits (best-effort, batched). */ export interface ForkContentPlan { sourceWorkspaceId: string childWorkspaceId: string + /** Initiating user, recorded as the owner of copied KB-document blob bindings in the child. */ + userId: string tables: ForkContentPlanEntry[] - knowledgeBases: ForkContentPlanEntry[] + knowledgeBases: ForkContentKbEntry[] + skills: ForkContentSkillEntry[] + /** Documents copied into an already-existing target KB (sync-only; empty at fork create). */ + documents: ForkContentDocumentEntry[] } +/** + * A resource whose post-commit content fill failed, so every reference to it must be cleared. + * A table carries its child definition id; a KB carries its child id (dropping it cascade-removes + * its documents + embeddings) and the child ids of the document placeholders pre-created in the + * fork tx, so `document-selector` references clear too. A standalone `knowledge-document` (a doc + * copied into an existing target KB) carries just its child id - dropping that one row (its + * embeddings cascade) without touching the existing KB. A `file` carries the COPIED child storage + * key whose blob duplication failed: its `file-upload` references are cleared so no block points at + * a missing object. Unlike the others, a failed file drops no row - the metadata row is left so the + * user can re-upload the blob. + */ +export type ForkFailedResource = + | { kind: 'table'; childId: string } + | { kind: 'knowledge-base'; childId: string; documentChildIds: string[] } + | { kind: 'knowledge-document'; childId: string } + | { kind: 'file'; childKey: string } + /** Display names of the copied resources, by kind, for the fork activity report. */ export interface ForkCopiedResourceNames { tables: string[] knowledgeBases: string[] customTools: string[] skills: string[] - mcpServers: string[] + workflowMcpServers: string[] } export interface CopyResourcesResult { @@ -102,20 +196,25 @@ export async function copyForkResourceContainers( params: CopyResourcesParams ): Promise { const { tx, sourceWorkspaceId, childWorkspaceId, userId, now, selection, workflowIdMap } = params + const referencedDocumentIds = params.referencedDocumentIds ?? [] + const resolveEnvName = params.resolveEnvName const idMap = new Map>() const mappingEntries: ForkMappingUpsert[] = [] const contentPlan: ForkContentPlan = { sourceWorkspaceId, childWorkspaceId, + userId, tables: [], knowledgeBases: [], + skills: [], + documents: [], } const names: ForkCopiedResourceNames = { tables: [], knowledgeBases: [], customTools: [], skills: [], - mcpServers: [], + workflowMcpServers: [], } const record = (type: ForkResourceType, sourceId: string, childId: string) => { @@ -145,6 +244,13 @@ export async function copyForkResourceContainers( id: childId, workspaceId: childWorkspaceId, userId, + // The code column is copied verbatim, so a `{{ENV}}` ref in it would stay stale when a + // sync renames the env var. Rewrite those refs through the env-name resolver (subblock + // values are already remapped by the reference transform). Fork-create passes no resolver, + // preserving env names by default. + ...(resolveEnvName && typeof row.code === 'string' + ? { code: rewriteEnvRefsInText(row.code, resolveEnvName) } + : {}), createdAt: now, updatedAt: now, }) @@ -171,55 +277,43 @@ export async function copyForkResourceContainers( updatedAt: now, }) record('skill', row.id, childId) + // Rewritten post-commit (see copyForkResourceContent): the row is inserted with the + // source body here so the locked fork tx never runs the per-skill content UPDATE. + contentPlan.skills.push({ childId, content: row.content }) names.skills.push(row.name) } if (inserts.length > 0) await tx.insert(skill).values(inserts) } - if (selection.mcpServers.length > 0) { + if (selection.workflowMcpServers.length > 0) { const rows = await tx .select() - .from(mcpServers) + .from(workflowMcpServer) .where( and( - inArray(mcpServers.id, selection.mcpServers), - eq(mcpServers.workspaceId, sourceWorkspaceId), - isNull(mcpServers.deletedAt) + inArray(workflowMcpServer.id, selection.workflowMcpServers), + eq(workflowMcpServer.workspaceId, sourceWorkspaceId), + isNull(workflowMcpServer.deletedAt) ) ) - // `generateMcpServerId` is deterministic on (workspace, url), so two selected - // servers with the same normalized URL derive the same child id. Insert once - // and map both source ids to the surviving child rather than aborting the fork. - const insertsByChildId = new Map() + // Copy workflow-publishing MCP servers as config-only shells: the server definition + // (name/description/visibility) with NO `workflow_mcp_tool` rows attached, so the child + // re-registers its own workflows. These are fork-copy-only (not referenced by subblocks), + // so they are not recorded in the fork resource map. + const inserts: (typeof workflowMcpServer.$inferInsert)[] = [] for (const row of rows) { - const childId = row.url ? generateMcpServerId(childWorkspaceId, row.url) : generateId() - record('mcp_server', row.id, childId) - if (insertsByChildId.has(childId)) continue - names.mcpServers.push(row.name) - insertsByChildId.set(childId, { + inserts.push({ ...row, - id: childId, + id: generateId(), workspaceId: childWorkspaceId, createdBy: userId, - // Secrets are never copied across workspaces: drop the registered OAuth - // client + any auth headers so the child re-authenticates from scratch. - oauthClientId: null, - oauthClientSecret: null, - headers: {}, - connectionStatus: 'disconnected', - lastConnected: null, - lastError: null, deletedAt: null, createdAt: now, updatedAt: now, }) + names.workflowMcpServers.push(row.name) } - if (insertsByChildId.size > 0) { - await tx - .insert(mcpServers) - .values([...insertsByChildId.values()]) - .onConflictDoNothing() - } + if (inserts.length > 0) await tx.insert(workflowMcpServer).values(inserts) } if (selection.tables.length > 0) { @@ -273,6 +367,7 @@ export async function copyForkResourceContainers( ) ) const inserts: (typeof knowledgeBase.$inferInsert)[] = [] + const kbEntryBySourceId = new Map() for (const base of bases) { const childKbId = generateId() inserts.push({ @@ -285,32 +380,219 @@ export async function copyForkResourceContainers( updatedAt: now, }) record('knowledge_base', base.id, childKbId) - contentPlan.knowledgeBases.push({ sourceId: base.id, childId: childKbId }) + const entry: ForkContentKbEntry = { sourceId: base.id, childId: childKbId, documentIdMap: {} } + contentPlan.knowledgeBases.push(entry) + kbEntryBySourceId.set(base.id, entry) names.knowledgeBases.push(base.name) } if (inserts.length > 0) await tx.insert(knowledgeBase).values(inserts) + + // Pre-create placeholder document rows for the documents the copied workflows + // reference, at child ids generated inside the transaction, so each + // `document-selector` reference can be remapped to a valid copied document rather + // than cleared. Only documents whose parent KB is in this copy (FK-safe: the KB + // rows were just inserted above) are placed; the heavy content (embeddings + blob) + // is filled at these exact ids by the post-commit content phase. + await createForkDocumentPlaceholders({ + tx, + kbIdMap: idMap.get('knowledge_base') ?? new Map(), + kbEntryBySourceId, + referencedDocumentIds, + record, + }) } return { idMap, mappingEntries, contentPlan, names } } +/** + * Insert placeholder {@link document} rows in the child KBs for the referenced documents + * whose parent KB is being copied, recording the `knowledge_document` source->child id map. + * Each placeholder is the source document's metadata row at a fresh child id; its embeddings + * and (re-keyed) blob are filled by {@link copyForkResourceContent} after commit. Documents + * whose parent KB is not copied are skipped, leaving their references to be cleared. + */ +async function createForkDocumentPlaceholders(params: { + tx: DbOrTx + kbIdMap: Map + kbEntryBySourceId: Map + referencedDocumentIds: string[] + record: (type: ForkResourceType, sourceId: string, childId: string) => void +}): Promise { + const { tx, kbIdMap, kbEntryBySourceId, referencedDocumentIds, record } = params + if (referencedDocumentIds.length === 0 || kbIdMap.size === 0) return + + const docs = await tx + .select() + .from(document) + .where( + and( + inArray(document.id, referencedDocumentIds), + inArray(document.knowledgeBaseId, Array.from(kbIdMap.keys())), + isNull(document.deletedAt), + isNull(document.archivedAt) + ) + ) + + const inserts: (typeof document.$inferInsert)[] = [] + for (const doc of docs) { + const childKbId = kbIdMap.get(doc.knowledgeBaseId) + const kbEntry = kbEntryBySourceId.get(doc.knowledgeBaseId) + if (!childKbId || !kbEntry) continue + const childDocId = generateId() + inserts.push({ + ...doc, + id: childDocId, + knowledgeBaseId: childKbId, + connectorId: null, + deletedAt: null, + archivedAt: null, + }) + record('knowledge_document', doc.id, childDocId) + kbEntry.documentIdMap[doc.id] = childDocId + } + if (inserts.length > 0) await tx.insert(document).values(inserts) +} + +/** + * Plan the copy of documents referenced by the synced workflows whose parent knowledge base is + * ALREADY mapped to an existing target KB (not copied this sync) but the document itself is not + * mapped. For each, insert a placeholder document row into that existing target KB and return: + * - `documents`: content entries for the post-commit embeddings/blob fill, + * - `docIdMap`: source->child document id map (to augment the resolver so the `document-selector` + * reference remaps to the copy instead of being cleared), + * - `mappingEntries`: `knowledge_document` rows to persist (so a re-sync resolves the copy). + * + * FK-safe: the target KB is one the resolver returns (existence-checked via `validTargetIdsByKind`), + * and the placeholder insert is a bounded in-tx write with no object-storage I/O. Documents whose + * parent KB is being copied THIS sync are handled by {@link createForkDocumentPlaceholders} under + * that copied KB and are excluded here via `alreadyCopiedSourceDocIds`. A referenced document whose + * parent KB is not mapped at all is left untouched, so its reference is cleared as before. + */ +export async function planForkMappedKbDocumentCopies(params: { + tx: DbOrTx + resolver: ForkReferenceResolver + referencedDocumentIds: string[] + alreadyCopiedSourceDocIds: Set +}): Promise<{ + documents: ForkContentDocumentEntry[] + docIdMap: Map + mappingEntries: ForkMappingUpsert[] +}> { + const { tx, resolver, referencedDocumentIds, alreadyCopiedSourceDocIds } = params + const documents: ForkContentDocumentEntry[] = [] + const docIdMap = new Map() + const mappingEntries: ForkMappingUpsert[] = [] + + // A doc whose parent KB is being copied this sync is placed under that copied KB (skip via + // alreadyCopiedSourceDocIds); a doc that already resolves was mapped by a prior sync (skip via + // the resolver). Everything else is a candidate to copy into its (existing) mapped target KB. + const candidateIds = referencedDocumentIds.filter( + (id) => !alreadyCopiedSourceDocIds.has(id) && resolver('knowledge-document', id) == null + ) + if (candidateIds.length === 0) return { documents, docIdMap, mappingEntries } + + const docs = await tx + .select() + .from(document) + .where( + and( + inArray(document.id, candidateIds), + isNull(document.deletedAt), + isNull(document.archivedAt) + ) + ) + + const inserts: (typeof document.$inferInsert)[] = [] + for (const doc of docs) { + // The parent KB must already exist in the target. The resolver returns a target KB id only + // for a mapped, still-existing KB (validTargetIdsByKind), so this is FK-safe; a doc whose KB + // isn't mapped resolves null here and is left for its reference to be cleared. + const targetKbId = resolver('knowledge-base', doc.knowledgeBaseId) + if (targetKbId == null) continue + const childDocId = generateId() + inserts.push({ + ...doc, + id: childDocId, + knowledgeBaseId: targetKbId, + connectorId: null, + deletedAt: null, + archivedAt: null, + }) + docIdMap.set(doc.id, childDocId) + mappingEntries.push({ + resourceType: 'knowledge_document', + parentResourceId: doc.id, + childResourceId: childDocId, + }) + documents.push({ + sourceDocId: doc.id, + childDocId, + childKnowledgeBaseId: targetKbId, + storageKey: doc.storageKey, + filename: doc.filename, + mimeType: doc.mimeType, + }) + } + if (inserts.length > 0) await tx.insert(document).values(inserts) + return { documents, docIdMap, mappingEntries } +} + +/** + * Recursively rewrite the in-workspace resource URLs stored in a copied table row's `data` jsonb + * (column -> value), so resource-chip cells keep resolving after a cross-workspace copy. Fast-rejects + * a string without `/workspace/` before any regex, and returns the same reference when nothing + * changed (so only rows with a rewritten cell allocate a new `data` object). + */ +function remapTableRowResourceUrls(value: unknown, maps: ForkContentRefMaps): unknown { + if (typeof value === 'string') { + if (!value.includes('/workspace/')) return value + return rewriteForkResourceUrls(value, maps) + } + if (Array.isArray(value)) { + let changed = false + const next = value.map((item) => { + const remapped = remapTableRowResourceUrls(item, maps) + if (remapped !== item) changed = true + return remapped + }) + return changed ? next : value + } + if (isRecord(value)) { + let changed = false + const next: Record = {} + for (const [key, item] of Object.entries(value)) { + const remapped = remapTableRowResourceUrls(item, maps) + if (remapped !== item) changed = true + next[key] = remapped + } + return changed ? next : value + } + return value +} + /** * Copy the heavy resource content described by a {@link ForkContentPlan} AFTER the * fork transaction has committed: table rows, and KB documents + embeddings. Reads * and writes are keyset-paginated so peak memory is bounded to one page, and each * resource is copied in its own short statements (never one long transaction). * Best-effort: a failure on one resource is logged and the others continue - the - * fork itself (workflows + container rows) already succeeded. + * fork itself (workflows + container rows) already succeeded. Copied skill bodies are + * rewritten here too (`contentRefMaps`), out of the locked fork tx; that rewrite is an + * in-content link fixup, so a failure degrades to a broken link and never fails a resource. */ export async function copyForkResourceContent(params: { contentPlan: ForkContentPlan + /** In-content reference maps for rewriting copied skill bodies post-commit (best-effort). */ + contentRefMaps?: ForkContentRefMaps requestId?: string -}): Promise<{ copied: number; failed: number }> { - const { contentPlan, requestId = 'unknown' } = params - const { childWorkspaceId } = contentPlan +}): Promise<{ copied: number; failed: number; failures: ForkFailedResource[] }> { + const { contentPlan, contentRefMaps, requestId = 'unknown' } = params + const { childWorkspaceId, userId } = contentPlan let copiedResources = 0 let failedResources = 0 + const failures: ForkFailedResource[] = [] for (const table of contentPlan.tables) { try { @@ -334,6 +616,8 @@ export async function copyForkResourceContent(params: { id: generateId(), tableId: table.childId, workspaceId: childWorkspaceId, + // Repoint resource-chip URLs in cell data at the child copies (no-op when no maps). + data: contentRefMaps ? remapTableRowResourceUrls(row.data, contentRefMaps) : row.data, })) ) copied += rows.length @@ -347,6 +631,7 @@ export async function copyForkResourceContent(params: { copiedResources += 1 } catch (error) { failedResources += 1 + failures.push({ kind: 'table', childId: table.childId }) logger.warn(`[${requestId}] Failed to copy table rows during fork`, { sourceTableId: table.sourceId, error: getErrorMessage(error), @@ -375,24 +660,60 @@ export async function copyForkResourceContent(params: { .orderBy(asc(document.id)) .limit(CONTENT_PAGE) if (docs.length === 0) break - for (const doc of docs) { - const childDocId = generateId() - await db.insert(document).values({ - ...doc, - id: childDocId, - knowledgeBaseId: kb.childId, - connectorId: null, - deletedAt: null, - archivedAt: null, - }) - await copyDocumentEmbeddings(doc.id, childDocId, kb.childId) - } + // Copy the page's documents with bounded concurrency. The mapper never rejects + // (it captures its error), so all in-flight work settles before this resolves - no + // orphaned writes survive a failure - and a captured error is rethrown after to keep + // the KB ALL-OR-NOTHING (any failed doc fails the whole KB -> cleanup below). + const docErrors = await mapWithConcurrency( + docs, + KB_DOCUMENT_COPY_CONCURRENCY, + async (doc): Promise => { + try { + // Referenced documents were pre-created as placeholders in the fork tx at this + // exact child id, so a remapped `document-selector` reference can't dangle; the + // rest get a fresh id here. Copy the blob to a child-scoped KB key so the copy + // never shares the source's object (best-effort - keeps the source key on failure). + const placeholderId = kb.documentIdMap[doc.id] + const childDocId = placeholderId ?? generateId() + const blob = await copyKbDocumentBlob(doc, childWorkspaceId, userId, requestId) + if (placeholderId) { + if (blob) { + await db + .update(document) + .set({ storageKey: blob.storageKey, fileUrl: blob.fileUrl }) + .where(eq(document.id, childDocId)) + } + } else { + await db.insert(document).values({ + ...doc, + id: childDocId, + knowledgeBaseId: kb.childId, + connectorId: null, + deletedAt: null, + archivedAt: null, + ...(blob ? { storageKey: blob.storageKey, fileUrl: blob.fileUrl } : {}), + }) + } + await copyDocumentEmbeddings(doc.id, childDocId, kb.childId) + return null + } catch (error) { + return error + } + } + ) + const docError = docErrors.find((error) => error != null) + if (docError) throw docError afterDocId = docs[docs.length - 1].id if (docs.length < CONTENT_PAGE) break } copiedResources += 1 } catch (error) { failedResources += 1 + failures.push({ + kind: 'knowledge-base', + childId: kb.childId, + documentChildIds: Object.values(kb.documentIdMap), + }) logger.warn(`[${requestId}] Failed to copy knowledge base content during fork`, { sourceKnowledgeBaseId: kb.sourceId, error: getErrorMessage(error), @@ -400,7 +721,69 @@ export async function copyForkResourceContent(params: { } } - return { copied: copiedResources, failed: failedResources } + // Fill the documents copied into an already-existing target KB (sync-only U-docs path). The + // placeholder rows were inserted in the promote tx; here we re-key each blob and copy its + // embeddings into the existing KB. A per-document failure drops just that placeholder (its + // embeddings cascade) and clears its `document-selector` references - the existing KB and its + // own documents are never touched. + for (const docEntry of contentPlan.documents) { + try { + const blob = await copyKbDocumentBlob( + { + storageKey: docEntry.storageKey, + filename: docEntry.filename, + mimeType: docEntry.mimeType, + }, + childWorkspaceId, + userId, + requestId + ) + if (blob) { + await db + .update(document) + .set({ storageKey: blob.storageKey, fileUrl: blob.fileUrl }) + .where(eq(document.id, docEntry.childDocId)) + } + await copyDocumentEmbeddings( + docEntry.sourceDocId, + docEntry.childDocId, + docEntry.childKnowledgeBaseId + ) + copiedResources += 1 + } catch (error) { + failedResources += 1 + failures.push({ kind: 'knowledge-document', childId: docEntry.childDocId }) + logger.warn(`[${requestId}] Failed to copy document into mapped KB during sync`, { + sourceDocumentId: docEntry.sourceDocId, + error: getErrorMessage(error), + }) + } + } + + // Rewrite copied skill bodies out of the locked fork tx (the rows were inserted with the + // source body). Their `sim:` links + embedded file URLs are remapped to the child resources; + // this is an in-content link fixup, so a per-skill failure degrades to a broken link and is + // never counted as a failed resource (unmapped targets are left as graceful broken links). + if (contentRefMaps && contentPlan.skills.length > 0) { + for (const copiedSkill of contentPlan.skills) { + try { + const rewritten = rewriteForkContentRefs(copiedSkill.content, contentRefMaps) + if (rewritten !== copiedSkill.content) { + await db + .update(skill) + .set({ content: rewritten }) + .where(eq(skill.id, copiedSkill.childId)) + } + } catch (error) { + logger.warn(`[${requestId}] Failed to rewrite copied skill content; keeping source links`, { + childSkillId: copiedSkill.childId, + error: getErrorMessage(error), + }) + } + } + } + + return { copied: copiedResources, failed: failedResources, failures } } async function copyDocumentEmbeddings( @@ -433,3 +816,49 @@ async function copyDocumentEmbeddings( if (rows.length < CONTENT_PAGE) break } } + +/** + * Duplicate a KB document's stored blob to a fresh child-scoped KB storage key so the copied + * document never points at the source's object. The child key is written with a `file_metadata` + * ownership binding owned by the CHILD workspace (mirroring the canonical KB upload), so + * `verifyKBFileAccess` grants a child-workspace member - without it the copied object is + * download-denied (no binding = deny). Returns the new `storageKey` + serve `fileUrl`, or null + * when there is no internal blob to copy (external/`data:` docs have a null `storageKey`) or the + * copy fails - best-effort, so a single blob failure never aborts the KB. + */ +async function copyKbDocumentBlob( + doc: { storageKey: string | null; filename: string; mimeType: string }, + childWorkspaceId: string, + userId: string, + requestId: string +): Promise<{ storageKey: string; fileUrl: string } | null> { + if (!doc.storageKey) return null + try { + const buffer = await downloadFile({ + key: doc.storageKey, + context: 'knowledge-base', + maxBytes: MAX_FILE_SIZE, + }) + const targetKey = generateKnowledgeBaseFileKey(doc.filename) + await uploadFile({ + file: buffer, + fileName: doc.filename, + contentType: doc.mimeType, + context: 'knowledge-base', + customKey: targetKey, + preserveKey: true, + metadata: { + userId, + workspaceId: childWorkspaceId, + originalName: doc.filename, + }, + }) + return { storageKey: targetKey, fileUrl: `/api/files/serve/${encodeURIComponent(targetKey)}` } + } catch (error) { + logger.warn(`[${requestId}] Failed to copy KB document blob during fork; keeping source key`, { + sourceStorageKey: doc.storageKey, + error: getErrorMessage(error), + }) + return null + } +} diff --git a/apps/sim/lib/workspaces/fork/copy/workflow-id-map.test.ts b/apps/sim/lib/workspaces/fork/copy/workflow-id-map.test.ts new file mode 100644 index 00000000000..b0fb5f7a963 --- /dev/null +++ b/apps/sim/lib/workspaces/fork/copy/workflow-id-map.test.ts @@ -0,0 +1,41 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { buildForkWorkflowIdMap } from '@/lib/workspaces/fork/copy/workflow-id-map' + +describe('buildForkWorkflowIdMap', () => { + const sequentialIds = () => { + let n = 0 + return () => `child-${++n}` + } + + it('excludes a deployed source whose state failed to load from the map (and so the identity seed)', () => { + const deployed = [{ id: 'wf-a' }, { id: 'wf-b' }, { id: 'wf-c' }] + // wf-b's deployed state failed to load - the copy loop skips it. + const map = buildForkWorkflowIdMap(deployed, new Set(['wf-a', 'wf-c']), sequentialIds()) + // wf-b is absent, so a copied workflow's ref to it clears (not dangle) and the identity seed + // (derived from this map's entries) never gets an orphan row pointing at a never-created child. + expect([...map.keys()]).toEqual(['wf-a', 'wf-c']) + expect(map.has('wf-b')).toBe(false) + expect(map.get('wf-a')).toBe('child-1') + expect(map.get('wf-c')).toBe('child-2') + }) + + it('maps a both-deployed pair when both states loaded (refs remap, not clear)', () => { + const deployed = [{ id: 'parent-wf' }, { id: 'child-wf' }] + const map = buildForkWorkflowIdMap( + deployed, + new Set(['parent-wf', 'child-wf']), + sequentialIds() + ) + expect([...map.keys()]).toEqual(['parent-wf', 'child-wf']) + expect(map.get('parent-wf')).toBe('child-1') + expect(map.get('child-wf')).toBe('child-2') + }) + + it('returns an empty map when no states loaded', () => { + const map = buildForkWorkflowIdMap([{ id: 'wf-a' }], new Set(), () => 'x') + expect(map.size).toBe(0) + }) +}) diff --git a/apps/sim/lib/workspaces/fork/copy/workflow-id-map.ts b/apps/sim/lib/workspaces/fork/copy/workflow-id-map.ts new file mode 100644 index 00000000000..e85a4f1d753 --- /dev/null +++ b/apps/sim/lib/workspaces/fork/copy/workflow-id-map.ts @@ -0,0 +1,21 @@ +import { generateId } from '@sim/utils/id' + +/** + * Build the source->child workflow id map for a fork create, scoped to the deployed workflows whose + * state actually LOADED (the set the copy loop writes). A deployed source whose state failed to load + * is EXCLUDED, so a copied workflow's reference to it clears (clearUnmapped) instead of pointing at a + * never-created child, and no orphan `workspace_fork_resource_map` identity row is seeded for it (the + * identity seed is derived from this map). Mirrors promote's writtenItems-only scoping + * (`buildPromoteWorkflowIdMap`). `generateChildId` is injectable for deterministic tests. + */ +export function buildForkWorkflowIdMap( + deployedWorkflows: ReadonlyArray<{ id: string }>, + loadedStateWorkflowIds: ReadonlySet, + generateChildId: () => string = generateId +): Map { + const map = new Map() + for (const wf of deployedWorkflows) { + if (loadedStateWorkflowIds.has(wf.id)) map.set(wf.id, generateChildId()) + } + return map +} diff --git a/apps/sim/lib/workspaces/fork/create-fork.ts b/apps/sim/lib/workspaces/fork/create-fork.ts index 6cdf43dace7..4adbf0cea79 100644 --- a/apps/sim/lib/workspaces/fork/create-fork.ts +++ b/apps/sim/lib/workspaces/fork/create-fork.ts @@ -6,8 +6,6 @@ import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { and, eq } from 'drizzle-orm' import type { Workspace } from '@/lib/api/contracts/workspaces' -import { isTriggerDevEnabled } from '@/lib/core/config/env-flags' -import { runDetached } from '@/lib/core/utils/background' import { buildDefaultWorkflowArtifacts } from '@/lib/workflows/defaults' import { saveWorkflowToNormalizedTables } from '@/lib/workflows/persistence/utils' import { @@ -16,7 +14,8 @@ import { } from '@/lib/workspaces/fork/background-work/store' import { type ForkContentCopyPayload, - runForkContentCopy, + scheduleForkContentCopy, + serializeContentRefMaps, } from '@/lib/workspaces/fork/copy/content-copy-runner' import { planForkFileCopies } from '@/lib/workspaces/fork/copy/copy-files' import { @@ -29,6 +28,7 @@ import { resolveForkFolderMapping, } from '@/lib/workspaces/fork/copy/copy-workflows' import { loadSourceDeployedStates } from '@/lib/workspaces/fork/copy/deploy-bridge' +import { buildForkWorkflowIdMap } from '@/lib/workspaces/fork/copy/workflow-id-map' import { setForkLockTimeout } from '@/lib/workspaces/fork/lineage/lineage' import { type ForkBlockPair, @@ -41,6 +41,7 @@ import { seedEdgeMappings, } from '@/lib/workspaces/fork/mapping/mapping-store' import { createForkBootstrapTransform } from '@/lib/workspaces/fork/remap/fork-bootstrap' +import { collectReferencedDocumentIds } from '@/lib/workspaces/fork/remap/reference-scan' import type { ForkRemapKind } from '@/lib/workspaces/fork/remap/remap-references' import type { WorkspaceWithOwner } from '@/lib/workspaces/permissions/utils' import type { WorkspaceCreationPolicy } from '@/lib/workspaces/policy' @@ -55,7 +56,8 @@ export interface ForkResourceSelection { knowledgeBases: string[] customTools: string[] skills: string[] - mcpServers: string[] + /** Workflow-publishing MCP servers (copied as config-only shells); external MCP is never copied. */ + workflowMcpServers: string[] } const EMPTY_SELECTION: ForkResourceSelection = { @@ -64,7 +66,7 @@ const EMPTY_SELECTION: ForkResourceSelection = { knowledgeBases: [], customTools: [], skills: [], - mcpServers: [], + workflowMcpServers: [], } export interface CreateForkParams { @@ -84,12 +86,14 @@ export interface CreateForkResult { workflowsCopied: number } +// External MCP servers are intentionally absent: a fork never copies them, so their +// references resolve to null here and are cleared on remap (re-add + re-auth in the child). const FORK_KIND_TO_RESOURCE_TYPE: Partial> = { 'custom-tool': 'custom_tool', skill: 'skill', - 'mcp-server': 'mcp_server', table: 'table', 'knowledge-base': 'knowledge_base', + 'knowledge-document': 'knowledge_document', } /** @@ -111,15 +115,25 @@ export async function createFork(params: CreateForkParams): Promise { + const sourceState = sourceStates.get(wf.id) + return sourceState ? [sourceState] : [] + }) + ) + const forkedWorkflowNames: string[] = [] let forkedResourceNames: ForkCopiedResourceNames = { tables: [], knowledgeBases: [], customTools: [], skills: [], - mcpServers: [], + workflowMcpServers: [], } - const { result, blobTasks, contentPlan } = await db.transaction(async (tx) => { + const { result, blobTasks, contentPlan, contentRefMaps } = await db.transaction(async (tx) => { await setForkLockTimeout(tx) const now = new Date() const childWorkspaceId = generateId() @@ -167,8 +181,14 @@ export async function createFork(params: CreateForkParams): Promise() - for (const wf of deployedWorkflows) workflowIdMap.set(wf.id, generateId()) + // The id map (and the identity seed below) covers only the workflows ACTUALLY copied - + // those whose deployed state loaded. A deployed source whose state failed to load is + // skipped by the copy loop, so it must be excluded here too: keeping it would (1) remap a + // copied workflow's reference to a child id that is never created (a dangling ref) instead + // of clearing it, and (2) seed a `workspace_fork_resource_map` workflow row pointing at + // that never-created target, which a later push would treat as an orphan and archive the + // parent's real workflow. Mirrors promote's writtenItems-only identity seed. + const workflowIdMap = buildForkWorkflowIdMap(deployedWorkflows, new Set(sourceStates.keys())) const fileResult = await planForkFileCopies({ tx, @@ -179,6 +199,16 @@ export async function createFork(params: CreateForkParams): Promise child folder id map: remaps folder references in the copied workflows below and + // feeds the post-commit content-ref rewrite (`sim:folder/` mentions in skill/file bodies). + const folderIdMap = await resolveForkFolderMapping({ + tx, + sourceWorkspaceId: source.id, + targetWorkspaceId: childWorkspaceId, + userId, + now, + }) + const resourceResult = await copyForkResourceContainers({ tx, sourceWorkspaceId: source.id, @@ -188,11 +218,12 @@ export async function createFork(params: CreateForkParams): Promise task.fileName), customToolNames: forkedResourceNames.customTools, skillNames: forkedResourceNames.skills, - mcpServerNames: forkedResourceNames.mcpServers, + workflowMcpServerNames: forkedResourceNames.workflowMcpServers, }, }) } catch (error) { @@ -373,34 +411,14 @@ export async function createFork(params: CreateForkParams): Promise('fork-content-copy', payload, { - region: await resolveTriggerRegion(), - }) - } else { - runDetached('fork-content-copy', () => runForkContentCopy(payload)) - } - } catch (error) { - // The fork itself succeeded; only scheduling the background copy failed. Surface - // it on the status row instead of failing the (committed) fork response. - logger.error(`[${requestId}] Failed to schedule fork content copy`, { - childWorkspaceId: result.workspace.id, - error: getErrorMessage(error), - }) - if (statusId) { - await finishBackgroundWork(db, statusId, { - status: 'failed', - error: getErrorMessage(error, 'Could not start the background copy'), - }).catch(() => {}) - } + const payload: ForkContentCopyPayload = { + contentPlan, + blobTasks, + contentRefMaps, + statusId, + requestId, } + await scheduleForkContentCopy(payload, { detachedLabel: 'fork-content-copy', requestId }) return result } diff --git a/apps/sim/lib/workspaces/fork/mapping/cascade.test.ts b/apps/sim/lib/workspaces/fork/mapping/cascade.test.ts index f8f97f09c0d..9ed938ad3d9 100644 --- a/apps/sim/lib/workspaces/fork/mapping/cascade.test.ts +++ b/apps/sim/lib/workspaces/fork/mapping/cascade.test.ts @@ -14,6 +14,7 @@ function queuedExecutor(results: unknown[][]): DbOrTx { let index = 0 const builder = { from: () => builder, + innerJoin: () => builder, where: () => Promise.resolve(results[index++] ?? []), } return { select: () => builder } as unknown as DbOrTx diff --git a/apps/sim/lib/workspaces/fork/mapping/cascade.ts b/apps/sim/lib/workspaces/fork/mapping/cascade.ts index 27d15f9ac48..917760a081d 100644 --- a/apps/sim/lib/workspaces/fork/mapping/cascade.ts +++ b/apps/sim/lib/workspaces/fork/mapping/cascade.ts @@ -1,4 +1,4 @@ -import { customTools, knowledgeConnector, mcpServers } from '@sim/db/schema' +import { customTools, knowledgeBase, knowledgeConnector, mcpServers } from '@sim/db/schema' import { and, eq, inArray, isNull } from 'drizzle-orm' import type { DbOrTx } from '@/lib/db/types' import { @@ -148,6 +148,9 @@ export async function detectForkCascadeReferences(params: { } if (knowledgeBaseIds.size > 0) { + // Join to knowledgeBase + filter by the source workspace (defense-in-depth, matching the + // customTools/mcpServers reads above): a connector is only cascaded when its KB actually + // belongs to the source workspace, so a stale/crafted KB id can't pull in a foreign connector. const connectors = await executor .select({ id: knowledgeConnector.id, @@ -156,9 +159,12 @@ export async function detectForkCascadeReferences(params: { encryptedApiKey: knowledgeConnector.encryptedApiKey, }) .from(knowledgeConnector) + .innerJoin(knowledgeBase, eq(knowledgeConnector.knowledgeBaseId, knowledgeBase.id)) .where( and( inArray(knowledgeConnector.knowledgeBaseId, Array.from(knowledgeBaseIds)), + eq(knowledgeBase.workspaceId, sourceWorkspaceId), + isNull(knowledgeBase.deletedAt), isNull(knowledgeConnector.deletedAt) ) ) diff --git a/apps/sim/lib/workspaces/fork/mapping/dependent-reconfigs.ts b/apps/sim/lib/workspaces/fork/mapping/dependent-reconfigs.ts index fa54d294e2d..b617cec7ab9 100644 --- a/apps/sim/lib/workspaces/fork/mapping/dependent-reconfigs.ts +++ b/apps/sim/lib/workspaces/fork/mapping/dependent-reconfigs.ts @@ -11,6 +11,7 @@ import { evaluateSubBlockCondition, } from '@/lib/workflows/subblocks/visibility' import type { ForkBlockIdResolver } from '@/lib/workspaces/fork/remap/block-identity' +import { toScannerBlocks } from '@/lib/workspaces/fork/remap/reference-scan' import { isSubBlockRequired, scanWorkflowReferences, @@ -201,11 +202,17 @@ function emitAnchoredDependents(params: EmitAnchoredParams): void { export function collectForkDependentReconfigs( items: ReconfigItem[], sourceStates: Map, - resolveTargetBlockId: ForkBlockIdResolver + resolveTargetBlockId: ForkBlockIdResolver, + /** + * Which target mode to scan. Defaults to `replace` (the reconfigure UI, where the user re-picks + * a dependent against a swapped parent). The pre-sync cleared-ref list passes `create` to surface + * dependents a new target inherits that a remapped parent will clear (it can't be re-picked yet). + */ + mode: 'create' | 'replace' = 'replace' ): ForkDependentReconfig[] { const out: ForkDependentReconfig[] = [] for (const item of items) { - if (item.mode !== 'replace') continue + if (item.mode !== mode) continue const state = sourceStates.get(item.sourceWorkflowId) if (!state) continue for (const [sourceBlockId, block] of Object.entries(state.blocks)) { @@ -307,14 +314,9 @@ export function collectForkResourceUsages( if (item.mode !== 'replace') continue const state = sourceStates.get(item.sourceWorkflowId) if (!state) continue - const blocks = Object.values(state.blocks).map((block) => ({ - id: block.id, - name: block.name, - subBlocks: block.subBlocks as unknown, - })) // scanWorkflowReferences already dedups by `${kind}:${sourceId}` across the workflow, // so each resource appears once per workflow here. - for (const reference of scanWorkflowReferences(blocks, () => null).references) { + for (const reference of scanWorkflowReferences(toScannerBlocks(state), () => null).references) { const key = `${reference.kind}\u0000${reference.sourceId}` let usage = byResource.get(key) if (!usage) { diff --git a/apps/sim/lib/workspaces/fork/mapping/mapping-service.test.ts b/apps/sim/lib/workspaces/fork/mapping/mapping-service.test.ts index f349f3ea2c0..86570341867 100644 --- a/apps/sim/lib/workspaces/fork/mapping/mapping-service.test.ts +++ b/apps/sim/lib/workspaces/fork/mapping/mapping-service.test.ts @@ -22,8 +22,10 @@ vi.mock('@/lib/workspaces/fork/mapping/resources', () => ({ import { ForkError } from '@/lib/workspaces/fork/lineage/authz' import { findDuplicateTargetEntry, + suggestTarget, validateForkMappingTargets, } from '@/lib/workspaces/fork/mapping/mapping-service' +import type { ForkResourceCandidate } from '@/lib/workspaces/fork/mapping/resources' type ExistingByKind = Partial>> @@ -91,6 +93,34 @@ describe('validateForkMappingTargets', () => { ).rejects.toBeInstanceOf(ForkError) }) + it('accepts a file target whose storage key exists in the target workspace', async () => { + // Files are mappable like any other content kind; the target is the storage key, and + // filterExistingForkTargets resolves file existence by key in the target workspace. + mockFilterExisting.mockResolvedValue({ file: new Set(['workspace/DST/report.pdf']) }) + await expect( + validateForkMappingTargets('ws-source', 'ws-target', [ + { + resourceType: 'file', + sourceId: 'workspace/SRC/report.pdf', + targetId: 'workspace/DST/report.pdf', + }, + ]) + ).resolves.toBeUndefined() + }) + + it('rejects a file target whose storage key is missing in the target workspace', async () => { + mockFilterExisting.mockResolvedValue({ file: new Set() }) + await expect( + validateForkMappingTargets('ws-source', 'ws-target', [ + { + resourceType: 'file', + sourceId: 'workspace/SRC/report.pdf', + targetId: 'workspace/DST/gone.pdf', + }, + ]) + ).rejects.toBeInstanceOf(ForkError) + }) + it('rejects a credential whose target provider differs from the source provider', async () => { mockFilterExisting.mockResolvedValue({ credential: new Set(['cred-tgt']) }) mockGetCredentialProviders.mockImplementation(async (_db: unknown, workspaceId: string) => @@ -180,3 +210,39 @@ describe('findDuplicateTargetEntry', () => { ).toBeNull() }) }) + +describe('suggestTarget', () => { + const cand = (id: string, label: string, providerId?: string): ForkResourceCandidate => ({ + id, + label, + providerId, + }) + + it('disambiguates same-name credentials by matching the source provider', () => { + const target = suggestTarget('credential', 'Work', 'google-email', [ + cand('c1', 'Work', 'google-calendar'), + cand('c2', 'Work', 'google-email'), + ]) + expect(target).toBe('c2') + }) + + it('suggests a unique name match for a non-credential kind', () => { + expect( + suggestTarget('table', 'Orders', undefined, [cand('t1', 'Orders'), cand('t2', 'Other')]) + ).toBe('t1') + }) + + it('returns null when the name is ambiguous (two same-name candidates)', () => { + expect( + suggestTarget('table', 'Dup', undefined, [cand('t1', 'Dup'), cand('t2', 'Dup')]) + ).toBeNull() + }) + + it('returns null when no candidate name matches', () => { + expect(suggestTarget('table', 'Orders', undefined, [cand('t1', 'Other')])).toBeNull() + }) + + it('matches the name case- and whitespace-insensitively', () => { + expect(suggestTarget('table', ' Orders ', undefined, [cand('t1', 'orders')])).toBe('t1') + }) +}) diff --git a/apps/sim/lib/workspaces/fork/mapping/mapping-service.ts b/apps/sim/lib/workspaces/fork/mapping/mapping-service.ts index a7700aea2d5..7a9f4ad3115 100644 --- a/apps/sim/lib/workspaces/fork/mapping/mapping-service.ts +++ b/apps/sim/lib/workspaces/fork/mapping/mapping-service.ts @@ -1,5 +1,5 @@ import { db } from '@sim/db' -import type { ForkMappingEntry } from '@/lib/api/contracts/workspace-fork' +import type { ForkMappableResourceType, ForkMappingEntry } from '@/lib/api/contracts/workspace-fork' import type { DbOrTx } from '@/lib/db/types' import { listDeployedWorkflows, readDeployedState } from '@/lib/workspaces/fork/copy/deploy-bridge' import { ForkError } from '@/lib/workspaces/fork/lineage/authz' @@ -23,6 +23,7 @@ import { getWorkspaceEnvKeys, listForkResourceCandidates, } from '@/lib/workspaces/fork/mapping/resources' +import { toScannerBlocks } from '@/lib/workspaces/fork/remap/reference-scan' import { type ForkReference, type ForkRemapKind, @@ -35,7 +36,7 @@ interface ForkMappingViewParams { targetWorkspaceId: string } -function suggestTarget( +export function suggestTarget( kind: ForkRemapKind, sourceLabel: string, sourceProviderId: string | undefined, @@ -73,12 +74,12 @@ export async function getForkMappingView( const resolver = buildForkResolver(mappingRows, { sourceIsParent, targetEnvKeys, sourceEnvKeys }) - const resourceTypeBySourceId = new Map>() + const resourceTypeBySourceId = new Map() for (const row of mappingRows) { - // Workflow identity rows are system-managed, not user-mappable; skip them so a - // scanned reference can never be labeled `workflow` and the view stays within - // the mappable-type contract. - if (row.resourceType === 'workflow') continue + // Workflow identity rows are system-managed and document rows ride their parent KB - neither is + // user-mappable. Skip both so a scanned reference can never be labeled with a non-mappable type + // and the view stays within the mappable-type contract. + if (row.resourceType === 'workflow' || row.resourceType === 'knowledge_document') continue const key = sourceIsParent ? row.parentResourceId : row.childResourceId if (key) resourceTypeBySourceId.set(key, row.resourceType) } @@ -90,12 +91,7 @@ export async function getForkMappingView( for (const wf of deployedWorkflows) { const state = await readDeployedState(wf.id, sourceWorkspaceId) if (!state) continue - const blocks = Object.values(state.blocks).map((block) => ({ - id: block.id, - name: block.name, - subBlocks: block.subBlocks as unknown, - })) - for (const reference of scanWorkflowReferences(blocks, () => null).references) { + for (const reference of scanWorkflowReferences(toScannerBlocks(state), () => null).references) { referenceByKey.set(`${reference.kind}:${reference.sourceId}`, reference) } } @@ -116,7 +112,7 @@ export async function getForkMappingView( // valid mapping to a target past the display cap must be RETAINED, not shown unmapped. interface PendingEntry { reference: ForkReference - resourceType: Exclude + resourceType: ForkMappableResourceType sourceLabel: string sourceProviderId: string | undefined candidates: ForkResourceCandidate[] @@ -129,6 +125,11 @@ export async function getForkMappingView( // Only SOURCE workspace secrets are mappable; a `{{KEY}}` that isn't a source // workspace env var is a personal (user-scoped) secret - leave it as-is. if (reference.kind === 'env-var' && !sourceEnvKeys.has(reference.sourceId)) continue + // Knowledge documents are not a standalone mappable kind: a document is a dependent field + // of its knowledge base (the `document-selector` dependsOn the KB selector), re-picked in + // that KB's reconfigure flow and auto-remapped when the KB is copied. So a document never + // gets its own mapping entry - it follows its parent KB's target. + if (reference.kind === 'knowledge-document') continue let resourceType = resourceTypeBySourceId.get(reference.sourceId) if (!resourceType) { resourceType = diff --git a/apps/sim/lib/workspaces/fork/mapping/mapping-store.ts b/apps/sim/lib/workspaces/fork/mapping/mapping-store.ts index c37016584b0..fffff6a7a23 100644 --- a/apps/sim/lib/workspaces/fork/mapping/mapping-store.ts +++ b/apps/sim/lib/workspaces/fork/mapping/mapping-store.ts @@ -48,10 +48,10 @@ export function resourceTypeToForkKind(resourceType: ForkResourceType): ForkRema return RESOURCE_TYPE_TO_FORK_KIND[resourceType] } -const NON_CREDENTIAL_FORK_KIND_TO_RESOURCE_TYPE: Record< - Exclude, - Exclude -> = { +// `as const satisfies` (not a `Record` annotation) so each key keeps its precise literal +// value type - the generic accessor below then narrows its return per input kind (a uniform +// Record value type would collapse every key to the full value union). +const NON_CREDENTIAL_FORK_KIND_TO_RESOURCE_TYPE = { 'env-var': 'env_var', table: 'table', 'knowledge-base': 'knowledge_base', @@ -60,16 +60,19 @@ const NON_CREDENTIAL_FORK_KIND_TO_RESOURCE_TYPE: Record< 'mcp-server': 'mcp_server', 'custom-tool': 'custom_tool', skill: 'skill', -} +} as const satisfies Record< + Exclude, + Exclude +> /** * Stored resource type for a non-credential remap kind. Credentials are resolved * separately via `classifyCredentialResourceType` since the type (oauth vs * service account) depends on the credential row. */ -export function nonCredentialForkKindToResourceType( - kind: Exclude -): Exclude { +export function nonCredentialForkKindToResourceType>( + kind: K +): (typeof NON_CREDENTIAL_FORK_KIND_TO_RESOURCE_TYPE)[K] { return NON_CREDENTIAL_FORK_KIND_TO_RESOURCE_TYPE[kind] } diff --git a/apps/sim/lib/workspaces/fork/mapping/resources.test.ts b/apps/sim/lib/workspaces/fork/mapping/resources.test.ts new file mode 100644 index 00000000000..b67ce351f22 --- /dev/null +++ b/apps/sim/lib/workspaces/fork/mapping/resources.test.ts @@ -0,0 +1,95 @@ +/** + * @vitest-environment node + */ +import { dbChainMock, dbChainMockFns, resetDbChainMock } from '@sim/testing' +import { beforeEach, describe, expect, it } from 'vitest' +import type { DbOrTx } from '@/lib/db/types' +import { + listForkResourceCandidates, + loadForkCopyableResourceLabels, +} from '@/lib/workspaces/fork/mapping/resources' + +const executor = dbChainMock.db as unknown as DbOrTx + +describe('listForkResourceCandidates', () => { + beforeEach(() => { + resetDbChainMock() + }) + + it('populates file candidates keyed by storage key and leaves knowledge-document empty', async () => { + // The grouped queries resolve in Promise.all array order, each ending in `.limit()`: + // credentials, workspace env, tables, knowledge bases, MCP servers, custom tools, skills, + // files. Queue the eight results in that exact order. + dbChainMockFns.limit + .mockResolvedValueOnce([ + { id: 'cred-1', displayName: 'Cred One', providerId: 'google-email' }, + ]) + .mockResolvedValueOnce([{ variables: { API_KEY: 'secret' } }]) + .mockResolvedValueOnce([{ id: 'tbl-1', label: 'Table One' }]) + .mockResolvedValueOnce([{ id: 'kb-1', label: 'KB One' }]) + .mockResolvedValueOnce([{ id: 'mcp-1', label: 'MCP One' }]) + .mockResolvedValueOnce([{ id: 'ct-1', label: 'Tool One' }]) + .mockResolvedValueOnce([{ id: 'sk-1', label: 'Skill One' }]) + .mockResolvedValueOnce([ + { id: 'workspace/WS/report.pdf', label: 'report.pdf' }, + { id: 'workspace/WS/notes.md', label: 'notes.md' }, + ]) + + const result = await listForkResourceCandidates(executor, 'ws-1') + + // Files are mapping targets keyed by storage key (matching how `file-upload` references store + // them) - never a `workspace_files.id`. + expect(result.file).toEqual([ + { id: 'workspace/WS/report.pdf', label: 'report.pdf' }, + { id: 'workspace/WS/notes.md', label: 'notes.md' }, + ]) + // Documents are not a standalone mappable kind - they ride their KB via the reconfigure flow. + expect(result['knowledge-document']).toEqual([]) + expect(result['env-var']).toEqual([{ id: 'API_KEY', label: 'API_KEY' }]) + }) +}) + +describe('loadForkCopyableResourceLabels', () => { + beforeEach(() => { + resetDbChainMock() + }) + + it('carries the folder grouping for file entries (id + name, null at the root)', async () => { + // Only the file branch queries (no other kind has ids), so its terminal `.where()` is the + // single chain call. + dbChainMockFns.where.mockResolvedValueOnce([ + { key: 'workspace/SRC/a.png', label: 'a.png', folderId: 'fld-1', folderName: 'Images' }, + { key: 'workspace/SRC/root.txt', label: 'root.txt', folderId: null, folderName: null }, + ]) + + const labels = await loadForkCopyableResourceLabels(executor, 'ws-src', { + file: ['workspace/SRC/a.png', 'workspace/SRC/root.txt'], + }) + + expect(labels.get('file:workspace/SRC/a.png')).toEqual({ + label: 'a.png', + parentId: 'fld-1', + parentLabel: 'Images', + }) + // A file at the workspace root (or whose folder was deleted) carries null folder grouping. + expect(labels.get('file:workspace/SRC/root.txt')).toEqual({ + label: 'root.txt', + parentId: null, + parentLabel: null, + }) + }) + + it('returns null folder grouping for non-file kinds (they render flat)', async () => { + dbChainMockFns.where.mockResolvedValueOnce([{ id: 'kb-1', label: 'KB One' }]) + + const labels = await loadForkCopyableResourceLabels(executor, 'ws-src', { + 'knowledge-base': ['kb-1'], + }) + + expect(labels.get('knowledge-base:kb-1')).toEqual({ + label: 'KB One', + parentId: null, + parentLabel: null, + }) + }) +}) diff --git a/apps/sim/lib/workspaces/fork/mapping/resources.ts b/apps/sim/lib/workspaces/fork/mapping/resources.ts index 0fa56338d95..23b0bf1cd38 100644 --- a/apps/sim/lib/workspaces/fork/mapping/resources.ts +++ b/apps/sim/lib/workspaces/fork/mapping/resources.ts @@ -1,16 +1,20 @@ import { credential, customTools, + document, knowledgeBase, mcpServers, skill, userTableDefinitions, workflow, workflowDeploymentVersion, + workflowMcpServer, workspaceEnvironment, + workspaceFileFolder, workspaceFiles, } from '@sim/db/schema' import { and, count, eq, exists, inArray, isNull, sql } from 'drizzle-orm' +import type { ForkCopyableKind } from '@/lib/api/contracts/workspace-fork' import type { DbOrTx } from '@/lib/db/types' import type { ForkResourceType } from '@/lib/workspaces/fork/mapping/mapping-store' import type { ForkRemapKind } from '@/lib/workspaces/fork/remap/remap-references' @@ -40,59 +44,145 @@ export async function getWorkspaceEnvKeys( // Shared `{ id, label }` candidate queries for the content resource kinds that BOTH the // mapping-target picker and the fork-copy picker list - one source of the archived/deleted -// filters so the two pickers can never drift apart. Credentials, env vars (mapping-only), -// and files (copy-only) stay inline in their respective functions. -const tableCandidatesQuery = (executor: DbOrTx, workspaceId: string) => - executor +// filters so the two pickers can never drift apart, and one optional `ids` filter so the picker +// path (unfiltered, capped) and the existence/label path (exact ids) share a single definition +// per kind. When `ids` is given the query is filtered to those exact ids and is NOT capped, so a +// valid target sitting past the candidate cap is never wrongly dropped. Credentials, env vars +// (mapping-only), and files-with-folder (copy-only) keep their own helpers below. +const tableCandidatesQuery = (executor: DbOrTx, workspaceId: string, ids?: string[]) => { + const query = executor .select({ id: userTableDefinitions.id, label: userTableDefinitions.name }) .from(userTableDefinitions) .where( and( eq(userTableDefinitions.workspaceId, workspaceId), - isNull(userTableDefinitions.archivedAt) + isNull(userTableDefinitions.archivedAt), + ids ? inArray(userTableDefinitions.id, ids) : undefined ) ) - .limit(CANDIDATE_LIMIT) + return ids ? query : query.limit(CANDIDATE_LIMIT) +} -const knowledgeBaseCandidatesQuery = (executor: DbOrTx, workspaceId: string) => - executor +const knowledgeBaseCandidatesQuery = (executor: DbOrTx, workspaceId: string, ids?: string[]) => { + const query = executor .select({ id: knowledgeBase.id, label: knowledgeBase.name }) .from(knowledgeBase) - .where(and(eq(knowledgeBase.workspaceId, workspaceId), isNull(knowledgeBase.deletedAt))) - .limit(CANDIDATE_LIMIT) + .where( + and( + eq(knowledgeBase.workspaceId, workspaceId), + isNull(knowledgeBase.deletedAt), + ids ? inArray(knowledgeBase.id, ids) : undefined + ) + ) + return ids ? query : query.limit(CANDIDATE_LIMIT) +} -const customToolCandidatesQuery = (executor: DbOrTx, workspaceId: string) => - executor +const customToolCandidatesQuery = (executor: DbOrTx, workspaceId: string, ids?: string[]) => { + const query = executor .select({ id: customTools.id, label: customTools.title }) .from(customTools) - .where(eq(customTools.workspaceId, workspaceId)) - .limit(CANDIDATE_LIMIT) + .where( + and(eq(customTools.workspaceId, workspaceId), ids ? inArray(customTools.id, ids) : undefined) + ) + return ids ? query : query.limit(CANDIDATE_LIMIT) +} -const skillCandidatesQuery = (executor: DbOrTx, workspaceId: string) => - executor +const skillCandidatesQuery = (executor: DbOrTx, workspaceId: string, ids?: string[]) => { + const query = executor .select({ id: skill.id, label: skill.name }) .from(skill) - .where(eq(skill.workspaceId, workspaceId)) - .limit(CANDIDATE_LIMIT) + .where(and(eq(skill.workspaceId, workspaceId), ids ? inArray(skill.id, ids) : undefined)) + return ids ? query : query.limit(CANDIDATE_LIMIT) +} -const mcpServerCandidatesQuery = (executor: DbOrTx, workspaceId: string) => - executor +const mcpServerCandidatesQuery = (executor: DbOrTx, workspaceId: string, ids?: string[]) => { + const query = executor .select({ id: mcpServers.id, label: mcpServers.name }) .from(mcpServers) - .where(and(eq(mcpServers.workspaceId, workspaceId), isNull(mcpServers.deletedAt))) - .limit(CANDIDATE_LIMIT) + .where( + and( + eq(mcpServers.workspaceId, workspaceId), + isNull(mcpServers.deletedAt), + ids ? inArray(mcpServers.id, ids) : undefined + ) + ) + return ids ? query : query.limit(CANDIDATE_LIMIT) +} + +// Workspace-file mapping candidates are keyed by STORAGE KEY (not `workspace_files.id`): a +// `file-upload` reference stores the storage key, so a mapping target must be a key too. Only +// durable, non-deleted `workspace` files are mappable (chat/copilot uploads are session-scoped). +// An optional `keys` filter shares this definition between the mapping picker (unfiltered, capped) +// and the cap-free existence check. +const fileCandidatesQuery = (executor: DbOrTx, workspaceId: string, keys?: string[]) => { + const query = executor + .select({ + id: workspaceFiles.key, + label: sql`coalesce(${workspaceFiles.displayName}, ${workspaceFiles.originalName})`, + }) + .from(workspaceFiles) + .where( + and( + eq(workspaceFiles.workspaceId, workspaceId), + eq(workspaceFiles.context, 'workspace'), + isNull(workspaceFiles.deletedAt), + keys ? inArray(workspaceFiles.key, keys) : undefined + ) + ) + return keys ? query : query.limit(CANDIDATE_LIMIT) +} + +// Copyable workspace files WITH their folder grouping (LEFT JOIN gated on a live folder, so a file +// whose folder was deleted shows ungrouped). Shared by the fork-copy picker (unfiltered, capped) +// and the promote copyable-label lookup (filtered by exact storage keys, never capped), so the +// file+folder shape and its filters live in one place. Selects both the row id and the storage +// key; the copy picker reads `id`, the key-addressed label lookup reads `key`. +const fileCandidatesWithFolderQuery = ( + executor: DbOrTx, + workspaceId: string, + options: { keys?: string[] } = {} +) => { + const { keys } = options + const query = executor + .select({ + id: workspaceFiles.id, + key: workspaceFiles.key, + label: sql`coalesce(${workspaceFiles.displayName}, ${workspaceFiles.originalName})`, + folderId: workspaceFiles.folderId, + folderName: workspaceFileFolder.name, + }) + .from(workspaceFiles) + .leftJoin( + workspaceFileFolder, + and( + eq(workspaceFiles.folderId, workspaceFileFolder.id), + isNull(workspaceFileFolder.deletedAt) + ) + ) + .where( + and( + eq(workspaceFiles.workspaceId, workspaceId), + eq(workspaceFiles.context, 'workspace'), + isNull(workspaceFiles.deletedAt), + keys ? inArray(workspaceFiles.key, keys) : undefined + ) + ) + return keys ? query : query.limit(CANDIDATE_LIMIT) +} /** * List the resources in a workspace that can serve as mapping targets, grouped by * remap kind. Used to populate the mapping UI's target pickers and to label the - * source resources being mapped. `knowledge-document` and `file` are intentionally - * left empty for v1 (optional kinds resolved manually). + * source resources being mapped. `knowledge-document` is intentionally left empty: + * documents are not a standalone mappable kind - they are dependent fields of their + * knowledge base, re-picked in the per-KB reconfigure flow (and auto-remapped when + * their KB is copied). `file` candidates are keyed by storage key. */ export async function listForkResourceCandidates( executor: DbOrTx, workspaceId: string ): Promise> { - const [creds, wsEnvRows, tables, kbs, servers, tools, skills] = await Promise.all([ + const [creds, wsEnvRows, tables, kbs, servers, tools, skills, files] = await Promise.all([ executor .select({ id: credential.id, @@ -120,6 +210,7 @@ export async function listForkResourceCandidates( mcpServerCandidatesQuery(executor, workspaceId), customToolCandidatesQuery(executor, workspaceId), skillCandidatesQuery(executor, workspaceId), + fileCandidatesQuery(executor, workspaceId), ]) const envVariables = wsEnvRows[0]?.variables @@ -141,7 +232,7 @@ export async function listForkResourceCandidates( 'custom-tool': tools, skill: skills, 'knowledge-document': [], - file: [], + file: files, } } @@ -167,11 +258,15 @@ export async function filterExistingForkTargets( const credIds = ids('credential') const tableIds = ids('table') const kbIds = ids('knowledge-base') + const docIds = ids('knowledge-document') const mcpIds = ids('mcp-server') const toolIds = ids('custom-tool') const skillIds = ids('skill') + // Files are identified by storage key (not `workspace_files.id`); a copied file's mapping + // target is its child storage key, so existence is checked by key in the target workspace. + const fileKeys = ids('file') - const [creds, tables, kbs, servers, tools, skills] = await Promise.all([ + const [creds, tables, kbs, docs, servers, tools, skills, files] = await Promise.all([ credIds.length === 0 ? Promise.resolve([] as Array<{ id: string }>) : executor @@ -186,61 +281,51 @@ export async function filterExistingForkTargets( ), tableIds.length === 0 ? Promise.resolve([] as Array<{ id: string }>) - : executor - .select({ id: userTableDefinitions.id }) - .from(userTableDefinitions) - .where( - and( - eq(userTableDefinitions.workspaceId, workspaceId), - isNull(userTableDefinitions.archivedAt), - inArray(userTableDefinitions.id, tableIds) - ) - ), + : tableCandidatesQuery(executor, workspaceId, tableIds), kbIds.length === 0 + ? Promise.resolve([] as Array<{ id: string }>) + : knowledgeBaseCandidatesQuery(executor, workspaceId, kbIds), + // Documents are validated through a KB join (they are not a standalone candidate kind), so + // this existence check stays inline rather than sharing a per-kind candidate query. + docIds.length === 0 ? Promise.resolve([] as Array<{ id: string }>) : executor - .select({ id: knowledgeBase.id }) - .from(knowledgeBase) + .select({ id: document.id }) + .from(document) + .innerJoin(knowledgeBase, eq(document.knowledgeBaseId, knowledgeBase.id)) .where( and( eq(knowledgeBase.workspaceId, workspaceId), isNull(knowledgeBase.deletedAt), - inArray(knowledgeBase.id, kbIds) + isNull(document.deletedAt), + isNull(document.archivedAt), + inArray(document.id, docIds) ) ), mcpIds.length === 0 ? Promise.resolve([] as Array<{ id: string }>) - : executor - .select({ id: mcpServers.id }) - .from(mcpServers) - .where( - and( - eq(mcpServers.workspaceId, workspaceId), - isNull(mcpServers.deletedAt), - inArray(mcpServers.id, mcpIds) - ) - ), + : mcpServerCandidatesQuery(executor, workspaceId, mcpIds), toolIds.length === 0 ? Promise.resolve([] as Array<{ id: string }>) - : executor - .select({ id: customTools.id }) - .from(customTools) - .where(and(eq(customTools.workspaceId, workspaceId), inArray(customTools.id, toolIds))), + : customToolCandidatesQuery(executor, workspaceId, toolIds), skillIds.length === 0 ? Promise.resolve([] as Array<{ id: string }>) - : executor - .select({ id: skill.id }) - .from(skill) - .where(and(eq(skill.workspaceId, workspaceId), inArray(skill.id, skillIds))), + : skillCandidatesQuery(executor, workspaceId, skillIds), + fileKeys.length === 0 + ? Promise.resolve([] as Array<{ id: string }>) + : fileCandidatesQuery(executor, workspaceId, fileKeys), ]) const result: Partial>> = {} if (credIds.length > 0) result.credential = new Set(creds.map((r) => r.id)) if (tableIds.length > 0) result.table = new Set(tables.map((r) => r.id)) if (kbIds.length > 0) result['knowledge-base'] = new Set(kbs.map((r) => r.id)) + if (docIds.length > 0) result['knowledge-document'] = new Set(docs.map((r) => r.id)) if (mcpIds.length > 0) result['mcp-server'] = new Set(servers.map((r) => r.id)) if (toolIds.length > 0) result['custom-tool'] = new Set(tools.map((r) => r.id)) if (skillIds.length > 0) result.skill = new Set(skills.map((r) => r.id)) + // `fileCandidatesQuery` exposes the storage key under `id`, so file existence keys by `r.id`. + if (fileKeys.length > 0) result.file = new Set(files.map((r) => r.id)) return result } @@ -268,13 +353,20 @@ export async function getCredentialProvidersByIds( return new Map(rows.map((row) => [row.id, row.providerId ?? null])) } +/** A copyable workspace file plus its folder grouping (null folder = workspace root). */ +export interface ForkCopyableFileResource extends ForkResourceCandidate { + folderId: string | null + folderName: string | null +} + export interface ForkCopyableResources { - files: ForkResourceCandidate[] + files: ForkCopyableFileResource[] tables: ForkResourceCandidate[] knowledgeBases: ForkResourceCandidate[] customTools: ForkResourceCandidate[] skills: ForkResourceCandidate[] - mcpServers: ForkResourceCandidate[] + /** Workflow-publishing MCP servers, copied as config-only shells (external MCP is not copied). */ + workflowMcpServers: ForkResourceCandidate[] /** * Count of deployed workflows that the fork would copy. When 0, the fork modal shows an * informational note (forking is never blocked) - create-fork seeds a blank starter @@ -293,28 +385,18 @@ export async function listForkCopyableResources( workspaceId: string ): Promise { const [files, tables, kbs, tools, skills, servers, deployed] = await Promise.all([ - executor - .select({ - id: workspaceFiles.id, - // displayName is nullable; fall back to the (non-null) original name. - label: sql`coalesce(${workspaceFiles.displayName}, ${workspaceFiles.originalName})`, - }) - .from(workspaceFiles) - // Only durable workspace files are forkable - chat/copilot/mothership uploads are - // session-scoped attachments (and their chat-bound unique index can't be copied). - .where( - and( - eq(workspaceFiles.workspaceId, workspaceId), - eq(workspaceFiles.context, 'workspace'), - isNull(workspaceFiles.deletedAt) - ) - ) - .limit(CANDIDATE_LIMIT), + fileCandidatesWithFolderQuery(executor, workspaceId), tableCandidatesQuery(executor, workspaceId), knowledgeBaseCandidatesQuery(executor, workspaceId), customToolCandidatesQuery(executor, workspaceId), skillCandidatesQuery(executor, workspaceId), - mcpServerCandidatesQuery(executor, workspaceId), + executor + .select({ id: workflowMcpServer.id, label: workflowMcpServer.name }) + .from(workflowMcpServer) + .where( + and(eq(workflowMcpServer.workspaceId, workspaceId), isNull(workflowMcpServer.deletedAt)) + ) + .limit(CANDIDATE_LIMIT), executor .select({ value: count() }) .from(workflow) @@ -341,16 +423,98 @@ export async function listForkCopyableResources( ), ]) return { - files, + // The shared folder query also selects the storage key (for the label lookup); the copy + // picker addresses files by `workspace_files.id`, so drop the key here. + files: files.map((row) => ({ + id: row.id, + label: row.label, + folderId: row.folderId, + folderName: row.folderName, + })), tables, knowledgeBases: kbs, customTools: tools, skills, - mcpServers: servers, + workflowMcpServers: servers, deployedWorkflowCount: deployed[0]?.value ?? 0, } } +/** + * A copyable reference's display label plus its folder grouping. `parentId`/`parentLabel` are + * populated only for files (their folder id + name; null at the workspace root) and are null for + * every other copyable kind, which the picker renders flat. + */ +export interface ForkCopyableLabel { + label: string + parentId: string | null + parentLabel: string | null +} + +/** + * Labels (by exact id) for the copyable resource kinds referenced-but-unmapped at promote time, + * scoped to the source workspace and the same archived/deleted filters as the copy picker. A + * resource absent from the result no longer exists in the source, so it can't be copied and is + * dropped from the sync copy candidates. Keyed `${kind}:${id}` so callers can look a reference up + * directly; file entries additionally carry their folder grouping. Only kinds with ids are queried. + */ +export async function loadForkCopyableResourceLabels( + executor: DbOrTx, + sourceWorkspaceId: string, + idsByKind: Partial> +): Promise> { + const labels = new Map() + const ids = (kind: ForkCopyableKind): string[] => { + const list = idsByKind[kind] + return list && list.length > 0 ? list : [] + } + const kbIds = ids('knowledge-base') + const tableIds = ids('table') + const toolIds = ids('custom-tool') + const skillIds = ids('skill') + // Files are keyed by storage key (not `workspace_files.id`), so they label by key. + const fileKeys = ids('file') + + const [kbs, tables, tools, skills, files] = await Promise.all([ + kbIds.length === 0 + ? Promise.resolve([] as Array<{ id: string; label: string }>) + : knowledgeBaseCandidatesQuery(executor, sourceWorkspaceId, kbIds), + tableIds.length === 0 + ? Promise.resolve([] as Array<{ id: string; label: string }>) + : tableCandidatesQuery(executor, sourceWorkspaceId, tableIds), + toolIds.length === 0 + ? Promise.resolve([] as Array<{ id: string; label: string }>) + : customToolCandidatesQuery(executor, sourceWorkspaceId, toolIds), + skillIds.length === 0 + ? Promise.resolve([] as Array<{ id: string; label: string }>) + : skillCandidatesQuery(executor, sourceWorkspaceId, skillIds), + fileKeys.length === 0 + ? Promise.resolve( + [] as Array<{ + key: string + label: string + folderId: string | null + folderName: string | null + }> + ) + : fileCandidatesWithFolderQuery(executor, sourceWorkspaceId, { keys: fileKeys }), + ]) + + const flat = (label: string): ForkCopyableLabel => ({ label, parentId: null, parentLabel: null }) + for (const row of kbs) labels.set(`knowledge-base:${row.id}`, flat(row.label)) + for (const row of tables) labels.set(`table:${row.id}`, flat(row.label)) + for (const row of tools) labels.set(`custom-tool:${row.id}`, flat(row.label)) + for (const row of skills) labels.set(`skill:${row.id}`, flat(row.label)) + for (const row of files) { + labels.set(`file:${row.key}`, { + label: row.label, + parentId: row.folderId, + parentLabel: row.folderName, + }) + } + return labels +} + /** Resolve a credential id to its stored mapping resource type. */ export async function classifyCredentialResourceType( executor: DbOrTx, diff --git a/apps/sim/lib/workspaces/fork/promote/cleared-refs.test.ts b/apps/sim/lib/workspaces/fork/promote/cleared-refs.test.ts new file mode 100644 index 00000000000..23d34a1ddae --- /dev/null +++ b/apps/sim/lib/workspaces/fork/promote/cleared-refs.test.ts @@ -0,0 +1,464 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it, vi } from 'vitest' +import type { SubBlockConfig } from '@/blocks/types' + +// The reference indexer resolves a tool's params via the tool registry; stub it so loading the +// remap module never pulls the full registry (these cases use top-level selectors / dependents). +vi.mock('@/tools/params', () => ({ + getToolIdForOperation: () => undefined, + getToolParametersConfig: () => null, + getSubBlocksForToolInput: ( + _toolId: string, + _type: string, + _values: unknown, + _modes: unknown, + provided?: { subBlocks?: SubBlockConfig[] } + ) => ({ subBlocks: provided?.subBlocks ?? [] }), + formatParameterLabel: (label: string) => label, +})) + +import { collectForkClearedRefCandidates } from '@/lib/workspaces/fork/promote/cleared-refs' +import { + buildForkBlockIdResolver, + deriveForkBlockId, + EMPTY_FORK_BLOCK_MAP, +} from '@/lib/workspaces/fork/remap/block-identity' +import type { ForkReferenceResolver } from '@/lib/workspaces/fork/remap/remap-references' +import { getBlock } from '@/blocks/registry' +import type { BlockConfig } from '@/blocks/types' +import type { WorkflowState } from '@/stores/workflows/workflow/types' + +const blockWith = (subBlocks: SubBlockConfig[]): BlockConfig => + ({ name: 'Test', description: '', subBlocks, outputs: {} }) as unknown as BlockConfig + +// No persisted block map, so the resolver derives - matching the deriveForkBlockId expectations. +const resolveBlockId = buildForkBlockIdResolver(true, EMPTY_FORK_BLOCK_MAP) + +const stateWith = ( + blockType: string, + blockName: string, + subBlocks: Record +): WorkflowState => + ({ + blocks: { 'block-1': { id: 'block-1', type: blockType, name: blockName, subBlocks } }, + edges: [], + loops: {}, + parallels: {}, + variables: {}, + }) as unknown as WorkflowState + +interface ParamOverrides { + items?: Array<{ + sourceWorkflowId: string + targetWorkflowId: string + mode: 'create' | 'replace' + sourceMeta: { name: string } + }> + sourceStates?: Map + resolver?: ForkReferenceResolver + workflowIdMap?: Map + sourceLabels?: Map + sourceWorkflowNames?: Map +} + +const params = (overrides: ParamOverrides) => ({ + items: [], + sourceStates: new Map(), + resolver: (() => null) as ForkReferenceResolver, + workflowIdMap: new Map(), + resolveBlockId, + sourceLabels: new Map(), + sourceWorkflowNames: new Map(), + ...overrides, +}) + +const targetBlockId = deriveForkBlockId('wf-tgt', 'block-1') + +describe('collectForkClearedRefCandidates', () => { + it('emits an unmapped knowledge-base reference (cause reference) with block + field labels', () => { + vi.mocked(getBlock).mockReturnValue( + blockWith([{ id: 'kb', title: 'Knowledge Base', type: 'knowledge-base-selector' }]) + ) + const result = collectForkClearedRefCandidates( + params({ + items: [ + { + sourceWorkflowId: 'wf-src', + targetWorkflowId: 'wf-tgt', + mode: 'replace', + sourceMeta: { name: 'Search' }, + }, + ], + sourceStates: new Map([ + [ + 'wf-src', + stateWith('knowledge', 'KB Block', { + kb: { type: 'knowledge-base-selector', value: 'kb-src' }, + }), + ], + ]), + resolver: () => null, + sourceLabels: new Map([['knowledge-base:kb-src', 'Docs KB']]), + }) + ) + expect(result).toEqual([ + { + targetWorkflowId: 'wf-tgt', + workflowName: 'Search', + blockId: targetBlockId, + blockLabel: 'KB Block', + fieldLabel: 'Knowledge Base', + kind: 'knowledge-base', + sourceId: 'kb-src', + sourceLabel: 'Docs KB', + cause: 'reference', + parentKind: null, + parentSourceId: null, + }, + ]) + }) + + it('drops a reference once the resolver maps it (so a mapped resource is not "cleared")', () => { + vi.mocked(getBlock).mockReturnValue( + blockWith([{ id: 'kb', title: 'Knowledge Base', type: 'knowledge-base-selector' }]) + ) + const result = collectForkClearedRefCandidates( + params({ + items: [ + { + sourceWorkflowId: 'wf-src', + targetWorkflowId: 'wf-tgt', + mode: 'replace', + sourceMeta: { name: 'Search' }, + }, + ], + sourceStates: new Map([ + [ + 'wf-src', + stateWith('knowledge', 'KB Block', { + kb: { type: 'knowledge-base-selector', value: 'kb-src' }, + }), + ], + ]), + resolver: (kind, id) => (kind === 'knowledge-base' && id === 'kb-src' ? 'kb-tgt' : null), + }) + ) + expect(result).toEqual([]) + }) + + it('excludes a required credential reference (a blocker resolved by mapping, never cleared)', () => { + vi.mocked(getBlock).mockReturnValue( + blockWith([{ id: 'credential', title: 'Account', type: 'oauth-input' }]) + ) + const result = collectForkClearedRefCandidates( + params({ + items: [ + { + sourceWorkflowId: 'wf-src', + targetWorkflowId: 'wf-tgt', + mode: 'replace', + sourceMeta: { name: 'Send Email' }, + }, + ], + sourceStates: new Map([ + [ + 'wf-src', + stateWith('gmail', 'Gmail 1', { + credential: { type: 'oauth-input', value: 'cred-src' }, + }), + ], + ]), + // An unmapped required credential gates Sync; it must not appear as a "will be cleared" item. + resolver: () => null, + sourceLabels: new Map([['credential:cred-src', 'Work Gmail']]), + }) + ) + expect(result).toEqual([]) + }) + + it('never emits an env-var reference (env vars are preserved by name, not cleared)', () => { + vi.mocked(getBlock).mockReturnValue( + blockWith([{ id: 'url', title: 'URL', type: 'short-input' }]) + ) + const result = collectForkClearedRefCandidates( + params({ + items: [ + { + sourceWorkflowId: 'wf-src', + targetWorkflowId: 'wf-tgt', + mode: 'replace', + sourceMeta: { name: 'API' }, + }, + ], + sourceStates: new Map([ + [ + 'wf-src', + stateWith('api', 'API', { url: { type: 'short-input', value: '{{API_KEY}}' } }), + ], + ]), + resolver: () => null, + }) + ) + expect(result).toEqual([]) + }) + + it('emits a workflow reference to a workflow not carried into the target (cause workflow)', () => { + vi.mocked(getBlock).mockReturnValue( + blockWith([{ id: 'target', title: 'Workflow', type: 'workflow-selector' }]) + ) + const result = collectForkClearedRefCandidates( + params({ + items: [ + { + sourceWorkflowId: 'wf-src', + targetWorkflowId: 'wf-tgt', + mode: 'replace', + sourceMeta: { name: 'Caller' }, + }, + ], + sourceStates: new Map([ + [ + 'wf-src', + stateWith('workflow_caller', 'Run Subflow', { + target: { type: 'workflow-selector', value: 'wf-other' }, + }), + ], + ]), + workflowIdMap: new Map(), + sourceWorkflowNames: new Map([['wf-other', 'Other Workflow']]), + }) + ) + expect(result).toEqual([ + { + targetWorkflowId: 'wf-tgt', + workflowName: 'Caller', + blockId: targetBlockId, + blockLabel: 'Run Subflow', + fieldLabel: 'Workflow', + kind: 'workflow', + sourceId: 'wf-other', + sourceLabel: 'Other Workflow', + cause: 'workflow', + parentKind: null, + parentSourceId: null, + }, + ]) + }) + + it('does not emit a workflow reference when the workflow is carried into the target', () => { + vi.mocked(getBlock).mockReturnValue( + blockWith([{ id: 'target', title: 'Workflow', type: 'workflow-selector' }]) + ) + const result = collectForkClearedRefCandidates( + params({ + items: [ + { + sourceWorkflowId: 'wf-src', + targetWorkflowId: 'wf-tgt', + mode: 'replace', + sourceMeta: { name: 'Caller' }, + }, + ], + sourceStates: new Map([ + [ + 'wf-src', + stateWith('workflow_caller', 'Run Subflow', { + target: { type: 'workflow-selector', value: 'wf-other' }, + }), + ], + ]), + workflowIdMap: new Map([['wf-other', 'wf-other-child']]), + }) + ) + expect(result).toEqual([]) + }) + + it('emits a configured create-target dependent a remapped parent will clear (cause dependent)', () => { + vi.mocked(getBlock).mockReturnValue( + blockWith([ + { id: 'credential', title: 'Credential', type: 'oauth-input' }, + { + id: 'folder', + title: 'Label', + type: 'folder-selector', + dependsOn: ['credential'], + selectorKey: 'gmail.labels', + }, + ]) + ) + // Entries carry no `type`, so the credential is not a direct (category-A) reference here - + // isolating the create-dependent (category-C) path: the label hangs off the credential. + const result = collectForkClearedRefCandidates( + params({ + items: [ + { + sourceWorkflowId: 'wf-src', + targetWorkflowId: 'wf-tgt', + mode: 'create', + sourceMeta: { name: 'New Workflow' }, + }, + ], + sourceStates: new Map([ + [ + 'wf-src', + stateWith('gmail', 'Gmail', { + credential: { value: 'cred-src' }, + folder: { value: 'INBOX' }, + }), + ], + ]), + sourceLabels: new Map([['credential:cred-src', 'Work Gmail']]), + }) + ) + expect(result).toEqual([ + { + targetWorkflowId: 'wf-tgt', + workflowName: 'New Workflow', + blockId: targetBlockId, + blockLabel: 'Gmail', + fieldLabel: 'Label', + kind: 'credential', + sourceId: 'cred-src', + sourceLabel: 'Work Gmail', + cause: 'dependent', + parentKind: 'credential', + parentSourceId: 'cred-src', + }, + ]) + }) + + it('carries the knowledge-base parent on a document-selector dependent (so it can drop off)', () => { + vi.mocked(getBlock).mockReturnValue( + blockWith([ + { + id: 'knowledgeBaseSelector', + title: 'Knowledge Base', + type: 'knowledge-base-selector', + canonicalParamId: 'knowledgeBaseId', + }, + { + id: 'documentSelector', + title: 'Document', + type: 'document-selector', + canonicalParamId: 'documentId', + selectorKey: 'knowledge.documents', + dependsOn: ['knowledgeBaseSelector'], + }, + ]) + ) + const result = collectForkClearedRefCandidates( + params({ + items: [ + { + sourceWorkflowId: 'wf-src', + targetWorkflowId: 'wf-tgt', + mode: 'create', + sourceMeta: { name: 'New Workflow' }, + }, + ], + sourceStates: new Map([ + [ + 'wf-src', + stateWith('knowledge', 'Knowledge', { + knowledgeBaseSelector: { value: 'kb-src' }, + documentSelector: { value: 'doc-src' }, + }), + ], + ]), + sourceLabels: new Map([['knowledge-base:kb-src', 'Docs KB']]), + }) + ) + expect(result).toEqual([ + { + targetWorkflowId: 'wf-tgt', + workflowName: 'New Workflow', + blockId: targetBlockId, + blockLabel: 'Knowledge', + fieldLabel: 'Document', + kind: 'knowledge-base', + sourceId: 'kb-src', + sourceLabel: 'Docs KB', + cause: 'dependent', + parentKind: 'knowledge-base', + parentSourceId: 'kb-src', + }, + ]) + }) + + it('does not emit a create-target dependent the source left unset (nothing is lost)', () => { + vi.mocked(getBlock).mockReturnValue( + blockWith([ + { id: 'credential', title: 'Credential', type: 'oauth-input' }, + { + id: 'folder', + title: 'Label', + type: 'folder-selector', + dependsOn: ['credential'], + selectorKey: 'gmail.labels', + }, + ]) + ) + const result = collectForkClearedRefCandidates( + params({ + items: [ + { + sourceWorkflowId: 'wf-src', + targetWorkflowId: 'wf-tgt', + mode: 'create', + sourceMeta: { name: 'New Workflow' }, + }, + ], + sourceStates: new Map([ + [ + 'wf-src', + stateWith('gmail', 'Gmail', { + credential: { value: 'cred-src' }, + folder: { value: '' }, + }), + ], + ]), + }) + ) + expect(result).toEqual([]) + }) + + it('does not emit create-dependents for a replace target (handled by the reconfigure flow)', () => { + vi.mocked(getBlock).mockReturnValue( + blockWith([ + { id: 'credential', title: 'Credential', type: 'oauth-input' }, + { + id: 'folder', + title: 'Label', + type: 'folder-selector', + dependsOn: ['credential'], + selectorKey: 'gmail.labels', + }, + ]) + ) + const result = collectForkClearedRefCandidates( + params({ + items: [ + { + sourceWorkflowId: 'wf-src', + targetWorkflowId: 'wf-tgt', + mode: 'replace', + sourceMeta: { name: 'Existing' }, + }, + ], + sourceStates: new Map([ + [ + 'wf-src', + stateWith('gmail', 'Gmail', { + credential: { value: 'cred-src' }, + folder: { value: 'INBOX' }, + }), + ], + ]), + }) + ) + // No direct refs (untyped entries) and no create-dependents (replace mode) -> empty. + expect(result).toEqual([]) + }) +}) diff --git a/apps/sim/lib/workspaces/fork/promote/cleared-refs.ts b/apps/sim/lib/workspaces/fork/promote/cleared-refs.ts new file mode 100644 index 00000000000..6b57d3e45f5 --- /dev/null +++ b/apps/sim/lib/workspaces/fork/promote/cleared-refs.ts @@ -0,0 +1,211 @@ +import type { ForkClearedRef } from '@/lib/api/contracts/workspace-fork' +import { + coerceObjectArray, + isRecord, + type SubBlockRecord, +} from '@/lib/workflows/persistence/remap-internal-ids' +import { collectForkDependentReconfigs } from '@/lib/workspaces/fork/mapping/dependent-reconfigs' +import type { ForkBlockIdResolver } from '@/lib/workspaces/fork/remap/block-identity' +import { + type ForkReferenceResolver, + type ForkRemapKind, + REQUIRED_KINDS, + remapForkSubBlocks, +} from '@/lib/workspaces/fork/remap/remap-references' +import { getBlock } from '@/blocks/registry' +import type { WorkflowState } from '@/stores/workflows/workflow/types' + +/** + * Remappable kinds excluded from the `reference` cleared-ref list. REQUIRED kinds (credential, + * env-var) are BLOCKERS - they gate Sync and are resolved by mapping, never silently cleared - so + * they must not read as "will be cleared" (a credential is also preserved by name once mapped, an + * env-var always). `knowledge-document` follows its parent KB - a document under an unmapped KB is + * implied by the KB's own cleared-ref entry, and under a mapped/copied KB it is auto-copied. + */ +const CLEARED_REF_EXCLUDED_KINDS = new Set([...REQUIRED_KINDS, 'knowledge-document']) + +interface ClearedRefItem { + sourceWorkflowId: string + targetWorkflowId: string + mode: 'create' | 'replace' + sourceMeta: { name: string } +} + +export interface CollectForkClearedRefsParams { + items: ClearedRefItem[] + sourceStates: Map + /** Plan resolver (persisted mappings + env identity), to detect which refs are currently unmapped. */ + resolver: ForkReferenceResolver + /** Source workflow id -> target id for THIS sync; a ref to a workflow absent here is cleared. */ + workflowIdMap: Map + /** Same block-id resolver the sync uses, so a candidate's blockId matches the written block. */ + resolveBlockId: ForkBlockIdResolver + /** `${kind}:${sourceId}` -> source resource label, for the `sourceLabel` display. */ + sourceLabels: Map + /** Source workflow id -> name, for `workflow`-kind candidate labels. */ + sourceWorkflowNames: Map +} + +/** Strip an advanced-mode `_N` suffix so a subblock key matches its config id. */ +function baseSubBlockId(key: string): string { + return key.replace(/_\d+$/, '') +} + +/** + * Cross-workflow references (`workflow-selector`, advanced `manualWorkflowId(s)`, multi-select + * `workflowSelector`, nested `workflow_input` tools) in a block's subBlocks. Mirrors the detection + * in {@link remapWorkflowReferencesInSubBlocks} so the cleared-ref list flags exactly the refs that + * remap would clear. Returns one entry per referenced workflow id with its owning subblock key. + */ +function collectForkWorkflowReferences( + subBlocks: SubBlockRecord +): Array<{ workflowId: string; subBlockKey: string }> { + const out: Array<{ workflowId: string; subBlockKey: string }> = [] + for (const [key, subBlock] of Object.entries(subBlocks)) { + if (!subBlock || typeof subBlock !== 'object') continue + const baseKey = baseSubBlockId(key) + if ( + (subBlock.type === 'workflow-selector' || baseKey === 'manualWorkflowId') && + typeof subBlock.value === 'string' && + subBlock.value + ) { + out.push({ workflowId: subBlock.value, subBlockKey: key }) + } else if (baseKey === 'manualWorkflowIds' || baseKey === 'workflowSelector') { + const ids = Array.isArray(subBlock.value) + ? subBlock.value + : typeof subBlock.value === 'string' + ? subBlock.value.split(',').map((entry) => entry.trim()) + : [] + for (const id of ids) { + if (typeof id === 'string' && id) out.push({ workflowId: id, subBlockKey: key }) + } + } else if (subBlock.type === 'tool-input') { + const { array } = coerceObjectArray(subBlock.value) + if (!array) continue + for (const tool of array) { + if ( + isRecord(tool) && + tool.type === 'workflow_input' && + isRecord(tool.params) && + typeof tool.params.workflowId === 'string' && + tool.params.workflowId + ) { + out.push({ workflowId: tool.params.workflowId, subBlockKey: key }) + } + } + } + } + return out +} + +/** + * Compute the per-block/field references this sync WILL blank in the target, for the pre-sync + * "what will be cleared" list. Three causes (see {@link ForkClearedRef}): + * - `reference`: an unmapped remappable resource (credential / KB / table / file / MCP server / + * custom tool / skill). The client filters these against the live mapping + copy selection, so an + * item disappears once mapped or selected for copy. Env vars (preserved) and documents (follow + * their KB) are excluded. + * - `workflow`: a cross-workflow reference to a workflow not carried into the target - always cleared. + * - `dependent`: a create-target dependent selector the source configured that a remapped parent + * clears. Carries `parentKind`/`parentSourceId` so the client can drop it once a KB parent is + * mapped or copied (the document follows its KB); a credential's label or a table's column is + * cleared on any parent remap, so it stays. + * + * Pure (no DB): the caller supplies the plan, source states, resolver, block-id resolver, and the + * source label maps. Block + field labels come from the block registry / block state. + */ +export function collectForkClearedRefCandidates( + params: CollectForkClearedRefsParams +): ForkClearedRef[] { + const { items, sourceStates, resolver, workflowIdMap, resolveBlockId, sourceLabels } = params + const out: ForkClearedRef[] = [] + const labelFor = (kind: string, sourceId: string) => + sourceLabels.get(`${kind}:${sourceId}`) ?? sourceId + + for (const item of items) { + const state = sourceStates.get(item.sourceWorkflowId) + if (!state) continue + for (const [sourceBlockId, block] of Object.entries(state.blocks)) { + const config = getBlock(block.type) + const blockLabel = block.name + const targetBlockId = resolveBlockId(item.targetWorkflowId, sourceBlockId) + // double-cast-allowed: a WorkflowState block's SubBlockState entries are structurally + // SubBlockRecord entries but lack the open index signature SubBlockRecord declares + const subBlocks = (block.subBlocks ?? {}) as unknown as SubBlockRecord + const fieldLabel = (subBlockKey: string) => + config?.subBlocks.find((cfg) => cfg.id === baseSubBlockId(subBlockKey))?.title ?? + subBlockKey + + // Cause `reference`: unmapped remappable resource refs (per block/field). + const scan = remapForkSubBlocks(subBlocks, resolver, 'promote', { + blockId: targetBlockId, + blockName: blockLabel, + }) + for (const ref of scan.unmapped) { + if (CLEARED_REF_EXCLUDED_KINDS.has(ref.kind)) continue + out.push({ + targetWorkflowId: item.targetWorkflowId, + workflowName: item.sourceMeta.name, + blockId: targetBlockId, + blockLabel, + fieldLabel: fieldLabel(ref.subBlockKey), + kind: ref.kind, + sourceId: ref.sourceId, + sourceLabel: labelFor(ref.kind, ref.sourceId), + cause: 'reference', + parentKind: null, + parentSourceId: null, + }) + } + + // Cause `workflow`: refs to a workflow not carried into the target. + for (const wfRef of collectForkWorkflowReferences(subBlocks)) { + if (workflowIdMap.has(wfRef.workflowId)) continue + out.push({ + targetWorkflowId: item.targetWorkflowId, + workflowName: item.sourceMeta.name, + blockId: targetBlockId, + blockLabel, + fieldLabel: fieldLabel(wfRef.subBlockKey), + kind: 'workflow', + sourceId: wfRef.workflowId, + sourceLabel: params.sourceWorkflowNames.get(wfRef.workflowId) ?? wfRef.workflowId, + cause: 'workflow', + parentKind: null, + parentSourceId: null, + }) + } + } + } + + // Cause `dependent`: create-target dependent selectors the source configured that a remapped + // parent clears. Only `replace` targets get the in-place reconfigure flow; a created target has + // no draft to re-pick against, so these would clear silently - surface them here. + const workflowNameByTarget = new Map(items.map((i) => [i.targetWorkflowId, i.sourceMeta.name])) + for (const dependent of collectForkDependentReconfigs( + items, + sourceStates, + resolveBlockId, + 'create' + )) { + if (dependent.currentValue === '') continue + out.push({ + targetWorkflowId: dependent.targetWorkflowId, + workflowName: workflowNameByTarget.get(dependent.targetWorkflowId) ?? '', + blockId: dependent.targetBlockId, + blockLabel: dependent.blockName, + fieldLabel: dependent.title, + kind: dependent.parentKind, + sourceId: dependent.parentSourceId, + sourceLabel: labelFor(dependent.parentKind, dependent.parentSourceId), + cause: 'dependent', + // The dependsOn parent (its KB/credential/table). The client drops this entry once the parent + // is mapped or copied ONLY when the child follows it (a document under a KB); a credential's + // label or a table's column is cleared on any parent remap, so it stays. + parentKind: dependent.parentKind, + parentSourceId: dependent.parentSourceId, + }) + } + + return out +} diff --git a/apps/sim/lib/workspaces/fork/promote/copy-unmapped.test.ts b/apps/sim/lib/workspaces/fork/promote/copy-unmapped.test.ts new file mode 100644 index 00000000000..91e0e74341c --- /dev/null +++ b/apps/sim/lib/workspaces/fork/promote/copy-unmapped.test.ts @@ -0,0 +1,381 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + type ForkCopyableUnmapped, + forkCopyableKindSchema, +} from '@/lib/api/contracts/workspace-fork' +import type { DbOrTx } from '@/lib/db/types' + +const { + mockUpsertEdgeMappings, + mockDeleteEdgeMappingsByChildResources, + mockCopyForkResourceContainers, + mockPlanForkMappedKbDocumentCopies, + mockPlanForkFileCopies, +} = vi.hoisted(() => ({ + mockUpsertEdgeMappings: vi.fn(), + mockDeleteEdgeMappingsByChildResources: vi.fn(), + mockCopyForkResourceContainers: vi.fn(), + mockPlanForkMappedKbDocumentCopies: vi.fn(), + mockPlanForkFileCopies: vi.fn(), +})) + +vi.mock('@/lib/workspaces/fork/mapping/mapping-store', () => ({ + upsertEdgeMappings: mockUpsertEdgeMappings, + deleteEdgeMappingsByChildResources: mockDeleteEdgeMappingsByChildResources, + resourceTypeToForkKind: vi.fn(), +})) + +vi.mock('@/lib/workspaces/fork/copy/copy-resources', () => ({ + copyForkResourceContainers: mockCopyForkResourceContainers, + planForkMappedKbDocumentCopies: mockPlanForkMappedKbDocumentCopies, + copyForkResourceContent: vi.fn(), +})) + +vi.mock('@/lib/workspaces/fork/copy/copy-files', () => ({ + planForkFileCopies: mockPlanForkFileCopies, + executeForkFileBlobCopies: vi.fn(), +})) + +import type { ForkEdge } from '@/lib/workspaces/fork/lineage/lineage' +import type { ForkMappingUpsert } from '@/lib/workspaces/fork/mapping/mapping-store' +import { + augmentForkResolver, + buildPromoteCopySelection, + copyPromoteUnmappedResources, + FORK_COPYABLE_KIND_TO_SELECTION_KEY, + hasPromoteCopySelection, + persistPromoteCopiedMappings, +} from '@/lib/workspaces/fork/promote/copy-unmapped' +import { isForkCopyableKind } from '@/lib/workspaces/fork/promote/promote-plan' +import type { ForkRemapKind } from '@/lib/workspaces/fork/remap/remap-references' + +const candidates: ForkCopyableUnmapped[] = [ + { kind: 'knowledge-base', sourceId: 'kb-1', label: 'KB One', parentId: null, parentLabel: null }, + { kind: 'table', sourceId: 'tbl-1', label: 'Table One', parentId: null, parentLabel: null }, + { kind: 'custom-tool', sourceId: 'ct-1', label: 'Tool One', parentId: null, parentLabel: null }, + { kind: 'skill', sourceId: 'sk-1', label: 'Skill One', parentId: null, parentLabel: null }, + { + kind: 'file', + sourceId: 'workspace/SRC/a.png', + label: 'a.png', + parentId: 'fld-1', + parentLabel: 'Images', + }, +] + +describe('buildPromoteCopySelection', () => { + it('groups requested ids into the selection by kind and records willResolve keys', () => { + const { selection, willResolve } = buildPromoteCopySelection( + { knowledgeBases: ['kb-1'], tables: ['tbl-1'], customTools: ['ct-1'], skills: ['sk-1'] }, + candidates + ) + expect(selection.knowledgeBases).toEqual(['kb-1']) + expect(selection.tables).toEqual(['tbl-1']) + expect(selection.customTools).toEqual(['ct-1']) + expect(selection.skills).toEqual(['sk-1']) + expect(selection.workflowMcpServers).toEqual([]) + expect(willResolve.has('knowledge-base:kb-1')).toBe(true) + expect(willResolve.has('skill:sk-1')).toBe(true) + }) + + it('ignores a requested id that is not an actual copy candidate (security)', () => { + const { selection, willResolve } = buildPromoteCopySelection( + { knowledgeBases: ['kb-1', 'kb-not-a-candidate'] }, + candidates + ) + expect(selection.knowledgeBases).toEqual(['kb-1']) + expect(willResolve.has('knowledge-base:kb-not-a-candidate')).toBe(false) + }) + + it('groups requested file storage keys (security: only actual candidates)', () => { + const { selection, willResolve } = buildPromoteCopySelection( + { files: ['workspace/SRC/a.png', 'workspace/SRC/not-referenced.png'] }, + candidates + ) + expect(selection.files).toEqual(['workspace/SRC/a.png']) + expect(willResolve.has('file:workspace/SRC/a.png')).toBe(true) + expect(willResolve.has('file:workspace/SRC/not-referenced.png')).toBe(false) + }) + + it('returns an empty selection when nothing is requested', () => { + const { selection, willResolve } = buildPromoteCopySelection(undefined, candidates) + expect(hasPromoteCopySelection(selection)).toBe(false) + expect(willResolve.size).toBe(0) + }) + + it('copy-vs-map: maps win - a mapped resource is absent from the candidates, so a copy request for it is dropped', () => { + // Reconciliation precedence at the server boundary: a resource the user mapped resolves to a + // target, so the plan never lists it in `copyableUnmapped`. Even if a (stale) client still + // requests it for copy, only the genuinely-unmapped candidates survive - the map wins. + const onlyTableUnmapped: ForkCopyableUnmapped[] = [ + { kind: 'table', sourceId: 'tbl-1', label: 'Table One', parentId: null, parentLabel: null }, + ] + const { selection, willResolve } = buildPromoteCopySelection( + // kb-1 + the file were mapped (so absent from candidates); only the table remains copyable. + { + knowledgeBases: ['kb-1'], + tables: ['tbl-1'], + files: ['workspace/SRC/a.png'], + }, + onlyTableUnmapped + ) + expect(selection.knowledgeBases).toEqual([]) + expect(selection.files).toEqual([]) + expect(selection.tables).toEqual(['tbl-1']) + expect(willResolve.has('knowledge-base:kb-1')).toBe(false) + expect(willResolve.has('file:workspace/SRC/a.png')).toBe(false) + expect(willResolve.has('table:tbl-1')).toBe(true) + }) +}) + +describe('hasPromoteCopySelection', () => { + it('is true only when at least one copyable kind has ids', () => { + expect( + hasPromoteCopySelection({ + customTools: [], + skills: [], + workflowMcpServers: [], + tables: [], + knowledgeBases: ['kb-1'], + files: [], + }) + ).toBe(true) + expect( + hasPromoteCopySelection({ + customTools: [], + skills: [], + workflowMcpServers: [], + tables: [], + knowledgeBases: [], + files: [], + }) + ).toBe(false) + expect( + hasPromoteCopySelection({ + customTools: [], + skills: [], + workflowMcpServers: [], + tables: [], + knowledgeBases: [], + files: ['workspace/SRC/file.png'], + }) + ).toBe(true) + }) +}) + +describe('augmentForkResolver', () => { + it('resolves a just-copied resource via the extra map, else falls through to the base', () => { + const base = (kind: ForkRemapKind, id: string) => + kind === 'credential' && id === 'cred-src' ? 'cred-dst' : null + const extra = new Map>([ + ['knowledge-base', new Map([['kb-src', 'kb-dst']])], + ]) + const resolver = augmentForkResolver(base, extra) + expect(resolver('knowledge-base', 'kb-src')).toBe('kb-dst') + expect(resolver('credential', 'cred-src')).toBe('cred-dst') + expect(resolver('table', 'tbl-x')).toBeNull() + }) +}) + +describe('persistPromoteCopiedMappings', () => { + const tx = {} as DbOrTx + const entry: ForkMappingUpsert = { + resourceType: 'knowledge_base', + parentResourceId: 'src-kb', + childResourceId: 'dst-kb', + } + + beforeEach(() => { + vi.clearAllMocks() + }) + + it('pull keeps the source(parent)->target(child) orientation as-is', async () => { + await persistPromoteCopiedMappings(tx, 'edge-child', 'user-1', 'pull', [entry]) + expect(mockUpsertEdgeMappings).toHaveBeenCalledWith(tx, 'edge-child', 'user-1', [entry]) + expect(mockDeleteEdgeMappingsByChildResources).not.toHaveBeenCalled() + }) + + it('push swaps to target(parent)->source(child) and deletes the prior row keyed on the source child', async () => { + await persistPromoteCopiedMappings(tx, 'edge-child', 'user-1', 'push', [entry]) + // Delete keys on the source child resource (the swapped child id = the original parent id). + expect(mockDeleteEdgeMappingsByChildResources).toHaveBeenCalledWith(tx, 'edge-child', [ + { resourceType: 'knowledge_base', childResourceId: 'src-kb' }, + ]) + // The swap flips parent/child: the new copy (dst) becomes the parent side on push. + expect(mockUpsertEdgeMappings).toHaveBeenCalledWith(tx, 'edge-child', 'user-1', [ + { resourceType: 'knowledge_base', parentResourceId: 'dst-kb', childResourceId: 'src-kb' }, + ]) + }) + + it('push skips an entry with a null child id (the narrowing guard, no bogus mapping)', async () => { + await persistPromoteCopiedMappings(tx, 'edge-child', 'user-1', 'push', [ + { resourceType: 'knowledge_base', parentResourceId: 'src-kb', childResourceId: null }, + ]) + expect(mockDeleteEdgeMappingsByChildResources).not.toHaveBeenCalled() + expect(mockUpsertEdgeMappings).not.toHaveBeenCalled() + }) + + it('returns without writing when there are no entries', async () => { + await persistPromoteCopiedMappings(tx, 'edge-child', 'user-1', 'push', []) + expect(mockDeleteEdgeMappingsByChildResources).not.toHaveBeenCalled() + expect(mockUpsertEdgeMappings).not.toHaveBeenCalled() + }) +}) + +describe('copyPromoteUnmappedResources - files + folder content-refs', () => { + const tx = {} as DbOrTx + // Only edge.childWorkspaceId is read by the copy path. + const edge = { childWorkspaceId: 'edge-child' } as unknown as ForkEdge + + beforeEach(() => { + vi.clearAllMocks() + mockCopyForkResourceContainers.mockResolvedValue({ + idMap: new Map(), + mappingEntries: [], + contentPlan: { + sourceWorkspaceId: 'src-ws', + childWorkspaceId: 'target-ws', + userId: 'user-1', + tables: [], + knowledgeBases: [], + skills: [], + documents: [], + }, + names: { + tables: [], + knowledgeBases: [], + customTools: [], + skills: [], + workflowMcpServers: [], + }, + }) + mockPlanForkMappedKbDocumentCopies.mockResolvedValue({ + documents: [], + docIdMap: new Map(), + mappingEntries: [], + }) + }) + + it('copies selected files (keyMap + blobTasks), persists the file mapping, and threads file + folder content-ref maps', async () => { + mockPlanForkFileCopies.mockResolvedValue({ + keyMap: new Map([['workspace/SRC/a.png', 'workspace/DST/a.png']]), + idMap: new Map([['file-src', 'file-dst']]), + blobTasks: [ + { + sourceKey: 'workspace/SRC/a.png', + targetKey: 'workspace/DST/a.png', + context: 'workspace', + fileName: 'a.png', + contentType: 'image/png', + userId: 'user-1', + workspaceId: 'target-ws', + }, + ], + }) + + const result = await copyPromoteUnmappedResources({ + tx, + edge, + sourceWorkspaceId: 'src-ws', + targetWorkspaceId: 'target-ws', + direction: 'pull', + userId: 'user-1', + now: new Date(), + selection: { + customTools: [], + skills: [], + workflowMcpServers: [], + tables: [], + knowledgeBases: [], + files: ['workspace/SRC/a.png'], + }, + workflowIdMap: new Map(), + folderIdMap: new Map([['fld-src', 'fld-dst']]), + resolver: () => null, + referencedDocumentIds: [], + }) + + // planForkFileCopies is invoked by storage key (the sync references key files by key). + expect(mockPlanForkFileCopies).toHaveBeenCalledWith( + expect.objectContaining({ fileKeys: ['workspace/SRC/a.png'] }) + ) + // blobTasks bubble up for the post-commit blob duplication. + expect(result.blobTasks).toHaveLength(1) + // The copied file resolves by storage key for the subblock remap. + expect(result.copyIdMapByKind.get('file')).toEqual( + new Map([['workspace/SRC/a.png', 'workspace/DST/a.png']]) + ) + // The file mapping is persisted (pull keeps source(parent)->target(child) orientation) so a + // re-sync resolves the copy instead of re-copying it. + expect(mockUpsertEdgeMappings).toHaveBeenCalledWith(tx, 'edge-child', 'user-1', [ + { + resourceType: 'file', + parentResourceId: 'workspace/SRC/a.png', + childResourceId: 'workspace/DST/a.png', + }, + ]) + // The folder map AND the file key/id maps reach the in-content rewriter. + expect(result.contentRefMaps.folders).toEqual({ 'fld-src': 'fld-dst' }) + expect(result.contentRefMaps.fileKeys).toEqual({ 'workspace/SRC/a.png': 'workspace/DST/a.png' }) + expect(result.contentRefMaps.fileIds).toEqual({ 'file-src': 'file-dst' }) + }) + + it('threads the plan-provided referencedDocumentIds into both doc-copy paths (no in-tx re-scan)', async () => { + await copyPromoteUnmappedResources({ + tx, + edge, + sourceWorkspaceId: 'src-ws', + targetWorkspaceId: 'target-ws', + direction: 'pull', + userId: 'user-1', + now: new Date(), + selection: { + customTools: [], + skills: [], + workflowMcpServers: [], + tables: [], + knowledgeBases: ['kb-1'], + files: [], + }, + workflowIdMap: new Map(), + folderIdMap: new Map(), + resolver: () => null, + // The doc ids come straight from the promote plan's references; the copy must forward them, + // not re-scan every source workflow state inside the locked tx. + referencedDocumentIds: ['doc-1', 'doc-2'], + }) + + expect(mockCopyForkResourceContainers).toHaveBeenCalledWith( + expect.objectContaining({ referencedDocumentIds: ['doc-1', 'doc-2'] }) + ) + expect(mockPlanForkMappedKbDocumentCopies).toHaveBeenCalledWith( + expect.objectContaining({ referencedDocumentIds: ['doc-1', 'doc-2'] }) + ) + }) +}) + +describe('fork copyable kind drift', () => { + it('FORK_COPYABLE_KIND_TO_SELECTION_KEY covers exactly the contract copyable kinds', () => { + expect(Object.keys(FORK_COPYABLE_KIND_TO_SELECTION_KEY).sort()).toEqual( + [...forkCopyableKindSchema.options].sort() + ) + }) + + it('isForkCopyableKind matches the contract copyable kinds and excludes the rest', () => { + for (const kind of forkCopyableKindSchema.options) { + expect(isForkCopyableKind(kind)).toBe(true) + } + const nonCopyable: ForkRemapKind[] = [ + 'credential', + 'env-var', + 'knowledge-document', + 'mcp-server', + ] + for (const kind of nonCopyable) { + expect(isForkCopyableKind(kind)).toBe(false) + } + }) +}) diff --git a/apps/sim/lib/workspaces/fork/promote/copy-unmapped.ts b/apps/sim/lib/workspaces/fork/promote/copy-unmapped.ts new file mode 100644 index 00000000000..f0080b1d46c --- /dev/null +++ b/apps/sim/lib/workspaces/fork/promote/copy-unmapped.ts @@ -0,0 +1,335 @@ +import type { + ForkCopyableKind, + ForkCopyableUnmapped, + PromoteCopyResources, +} from '@/lib/api/contracts/workspace-fork' +import type { DbOrTx } from '@/lib/db/types' +import { + type SerializableForkContentRefMaps, + serializeContentRefMaps, +} from '@/lib/workspaces/fork/copy/content-copy-runner' +import { type BlobCopyTask, planForkFileCopies } from '@/lib/workspaces/fork/copy/copy-files' +import type { ForkContentPlan } from '@/lib/workspaces/fork/copy/copy-resources' +import { + copyForkResourceContainers, + planForkMappedKbDocumentCopies, +} from '@/lib/workspaces/fork/copy/copy-resources' +import type { ForkEdge } from '@/lib/workspaces/fork/lineage/lineage' +import { + deleteEdgeMappingsByChildResources, + type ForkMappingUpsert, + resourceTypeToForkKind, + upsertEdgeMappings, +} from '@/lib/workspaces/fork/mapping/mapping-store' +import type { + ForkReferenceResolver, + ForkRemapKind, +} from '@/lib/workspaces/fork/remap/remap-references' + +/** The source ids selected for copy at promote, validated against the plan's copyable candidates. */ +export interface PromoteCopySelection { + customTools: string[] + skills: string[] + workflowMcpServers: string[] + tables: string[] + knowledgeBases: string[] + /** Workspace files to copy, identified by storage key (not `workspace_files.id`). */ + files: string[] +} + +/** + * Each copyable kind to its key in {@link PromoteCopySelection}. Keyed on `ForkCopyableKind` + * (the wire contract enum) so TS fails to compile if the copyable enum grows a kind without a + * selection key here, keeping the two in lockstep. + */ +export const FORK_COPYABLE_KIND_TO_SELECTION_KEY: Record< + ForkCopyableKind, + keyof PromoteCopySelection +> = { + 'knowledge-base': 'knowledgeBases', + table: 'tables', + 'custom-tool': 'customTools', + skill: 'skills', + file: 'files', +} + +/** + * Intersect the user's requested copy with the plan's actual copyable candidates, so a sync can + * only copy resources that are genuinely referenced-but-unmapped + still exist in the source (a + * crafted request can never copy an arbitrary resource). Returns the validated selection plus the + * set of `${kind}:${sourceId}` references the copy will resolve, for the pre-copy sync gate. + */ +export function buildPromoteCopySelection( + requested: PromoteCopyResources | undefined, + copyableUnmapped: ForkCopyableUnmapped[] +): { selection: PromoteCopySelection; willResolve: Set } { + const allowed = new Map>() + for (const candidate of copyableUnmapped) { + const set = allowed.get(candidate.kind) + if (set) set.add(candidate.sourceId) + else allowed.set(candidate.kind, new Set([candidate.sourceId])) + } + const selection: PromoteCopySelection = { + customTools: [], + skills: [], + workflowMcpServers: [], + tables: [], + knowledgeBases: [], + files: [], + } + const willResolve = new Set() + const apply = ( + kind: keyof typeof FORK_COPYABLE_KIND_TO_SELECTION_KEY, + ids: string[] | undefined + ) => { + const allowedIds = allowed.get(kind) + if (!allowedIds || !ids) return + const key = FORK_COPYABLE_KIND_TO_SELECTION_KEY[kind] + for (const id of ids) { + if (!allowedIds.has(id)) continue + selection[key].push(id) + willResolve.add(`${kind}:${id}`) + } + } + apply('knowledge-base', requested?.knowledgeBases) + apply('table', requested?.tables) + apply('custom-tool', requested?.customTools) + apply('skill', requested?.skills) + apply('file', requested?.files) + return { selection, willResolve } +} + +/** Whether any resource is selected for copy. */ +export function hasPromoteCopySelection(selection: PromoteCopySelection): boolean { + return ( + selection.customTools.length > 0 || + selection.skills.length > 0 || + selection.tables.length > 0 || + selection.knowledgeBases.length > 0 || + selection.files.length > 0 + ) +} + +/** + * Layer the just-copied resources' source->target ids on top of the plan's resolver, so the + * synced workflows' references to those resources resolve to the new copies. The base resolver + * (persisted mappings + env identity) is consulted for everything else. + */ +export function augmentForkResolver( + base: ForkReferenceResolver, + extra: Map> +): ForkReferenceResolver { + return (kind, sourceId) => extra.get(kind)?.get(sourceId) ?? base(kind, sourceId) +} + +export interface PromoteCopyResult { + contentPlan: ForkContentPlan + /** Copied source resource id -> new target id, by remap kind, for resolver augmentation. */ + copyIdMapByKind: Map> + /** + * Serialized in-content reference maps for the post-commit copy to rewrite copied skill bodies + * (their `sim:` links + embedded URLs) off the locked promote tx - carried through the durable + * content-copy payload, mirroring fork. + */ + contentRefMaps: SerializableForkContentRefMaps + /** + * File blob duplications for copied workspace files, run post-commit by the durable content-copy + * runner (no object-storage I/O inside the locked promote tx). Empty when no files were copied. + */ + blobTasks: BlobCopyTask[] +} + +/** + * Copy the referenced-but-unmapped resources a sync brings into the target (reusing the fork copy + * pipeline), then persist the source<->target id map in the direction the edge expects: a pull + * fills the existing `(parent, child=null)` row (fill-null), a push replaces any prior + * `(parent, child)` row keyed on the source child resource (delete-then-insert). This covers: + * - the user-selected copyable containers (KB / table / custom-tool / skill) and workspace files, + * - documents referenced under a copied knowledge base (auto-placed under that copied KB), + * - documents referenced under an ALREADY-mapped (existing) KB - copied into that existing KB so + * the `document-selector` reference remaps instead of being cleared. + * + * The heavy content (table rows, KB documents + embeddings, file blobs) is returned as a content + * plan + blob tasks for a post-commit, best-effort fill; no object-storage I/O runs inside the + * locked promote tx. Always safe to call (a no-op when nothing is selected and nothing references + * a mapped-KB document). + */ +export async function copyPromoteUnmappedResources(params: { + tx: DbOrTx + edge: ForkEdge + sourceWorkspaceId: string + targetWorkspaceId: string + direction: 'push' | 'pull' + userId: string + now: Date + selection: PromoteCopySelection + workflowIdMap: Map + /** source folder id -> target folder id, so copied skill/markdown bodies rewrite `sim:folder/`. */ + folderIdMap: Map + /** Base resolver (persisted mappings + env identity), used to detect already-mapped KBs (U-docs). */ + resolver: ForkReferenceResolver + /** + * Knowledge-document ids the synced workflows reference, already scanned once in the promote + * plan and threaded in so the copy doesn't re-scan every source state inside the locked tx. + * `copyForkResourceContainers` / `planForkMappedKbDocumentCopies` place only those whose parent + * KB is in this copy (or already mapped), so an extra id is FK-safe and simply skipped. + */ + referencedDocumentIds: string[] +}): Promise { + const { + tx, + edge, + sourceWorkspaceId, + targetWorkspaceId, + direction, + userId, + now, + selection, + workflowIdMap, + folderIdMap, + resolver, + referencedDocumentIds, + } = params + + const result = await copyForkResourceContainers({ + tx, + sourceWorkspaceId, + childWorkspaceId: targetWorkspaceId, + userId, + now, + selection: { + customTools: selection.customTools, + skills: selection.skills, + workflowMcpServers: selection.workflowMcpServers, + tables: selection.tables, + knowledgeBases: selection.knowledgeBases, + }, + workflowIdMap, + referencedDocumentIds, + // A sync can rename env vars, so a copied custom tool's `code` must have its `{{ENV}}` refs + // rewritten through the same plan resolver that remaps subblock-value env refs. + resolveEnvName: (key) => resolver('env-var', key), + }) + + // Copy the selected workspace files (keyed by storage key) - metadata inserts in the tx, blob + // duplications deferred to the post-commit runner. + const fileResult = + selection.files.length > 0 + ? await planForkFileCopies({ + tx, + sourceWorkspaceId, + childWorkspaceId: targetWorkspaceId, + userId, + fileKeys: selection.files, + now, + }) + : { + keyMap: new Map(), + idMap: new Map(), + blobTasks: [] as BlobCopyTask[], + } + + // U-docs: documents referenced under an already-mapped (not copied this sync) KB. Skip any doc + // already placed under a copied KB above (its parent KB is in this copy), so a doc is never + // copied twice. + const containerDocMap = result.idMap.get('knowledge_document') ?? new Map() + const mappedKbDocs = await planForkMappedKbDocumentCopies({ + tx, + resolver, + referencedDocumentIds, + alreadyCopiedSourceDocIds: new Set(containerDocMap.keys()), + }) + result.contentPlan.documents.push(...mappedKbDocs.documents) + + // Persist every copied resource's mapping (containers + files + U-docs) so a re-sync resolves + // the copy instead of re-copying it. Files map by storage key; U-docs add knowledge_document rows. + const fileMappingEntries: ForkMappingUpsert[] = Array.from( + fileResult.keyMap, + ([source, child]) => ({ + resourceType: 'file' as const, + parentResourceId: source, + childResourceId: child, + }) + ) + await persistPromoteCopiedMappings(tx, edge.childWorkspaceId, userId, direction, [ + ...result.mappingEntries, + ...fileMappingEntries, + ...mappedKbDocs.mappingEntries, + ]) + + const copyIdMapByKind = new Map>() + for (const [resourceType, sourceToTarget] of result.idMap) { + const kind = resourceTypeToForkKind(resourceType) + if (!kind) continue + copyIdMapByKind.set(kind, sourceToTarget) + } + if (fileResult.keyMap.size > 0) copyIdMapByKind.set('file', fileResult.keyMap) + + // Merge the container's copied-KB document map with the U-docs map so every copied document + // (under a copied KB or into an existing one) remaps its `document-selector` reference. + const documentIdMap = new Map([...containerDocMap, ...mappedKbDocs.docIdMap]) + if (documentIdMap.size > 0) copyIdMapByKind.set('knowledge-document', documentIdMap) + + // Serialized maps for the post-commit content rewrite (run off the locked promote tx). Mirrors + // fork: workspace + workflow + folder ids plus this copy's own file/skill/table/KB maps, so a + // copied skill body / markdown blob's `sim:` links + embedded file URLs resolve to the new target + // copies instead of the source. + const contentRefMaps = serializeContentRefMaps({ + workspaceId: { from: sourceWorkspaceId, to: targetWorkspaceId }, + workflows: workflowIdMap, + folders: folderIdMap, + fileKeys: fileResult.keyMap, + fileIds: fileResult.idMap, + skills: result.idMap.get('skill'), + tables: result.idMap.get('table'), + knowledgeBases: result.idMap.get('knowledge_base'), + }) + + return { + contentPlan: result.contentPlan, + copyIdMapByKind, + contentRefMaps, + blobTasks: fileResult.blobTasks, + } +} + +/** + * Persist the copied resources' id mappings for the edge. The copy returns entries oriented + * source(parent)->target(child); a pull matches that orientation directly (fill-null upsert), a + * push swaps it (the parent side is the new TARGET) and first drops any prior row keyed on the + * source child resource so a changed target can't leak a second mapping. + */ +export async function persistPromoteCopiedMappings( + tx: DbOrTx, + childWorkspaceId: string, + userId: string, + direction: 'push' | 'pull', + entries: ForkMappingUpsert[] +): Promise { + if (entries.length === 0) return + if (direction === 'pull') { + await upsertEdgeMappings(tx, childWorkspaceId, userId, entries) + return + } + // Push: re-key on the source child resource. Skip any entry with a null child id (copy entries + // always carry one; the guard narrows the type so neither the swap nor the delete needs a cast). + // After the swap every childResourceId is the original (non-null) parent id, keyed for the + // delete-then-insert that prevents a changed target from leaking a second mapping. + const swapped: ForkMappingUpsert[] = [] + const deleteKeys: Array<{ + resourceType: ForkMappingUpsert['resourceType'] + childResourceId: string + }> = [] + for (const entry of entries) { + if (entry.childResourceId == null) continue + swapped.push({ + resourceType: entry.resourceType, + parentResourceId: entry.childResourceId, + childResourceId: entry.parentResourceId, + }) + deleteKeys.push({ resourceType: entry.resourceType, childResourceId: entry.parentResourceId }) + } + if (swapped.length === 0) return + await deleteEdgeMappingsByChildResources(tx, childWorkspaceId, deleteKeys) + await upsertEdgeMappings(tx, childWorkspaceId, userId, swapped) +} diff --git a/apps/sim/lib/workspaces/fork/promote/promote-plan.test.ts b/apps/sim/lib/workspaces/fork/promote/promote-plan.test.ts index d6e49f2c23d..bf3d07b1791 100644 --- a/apps/sim/lib/workspaces/fork/promote/promote-plan.test.ts +++ b/apps/sim/lib/workspaces/fork/promote/promote-plan.test.ts @@ -2,7 +2,20 @@ * @vitest-environment node */ import { describe, expect, it } from 'vitest' -import { buildPromoteWorkflowIdMap } from '@/lib/workspaces/fork/promote/promote-plan' +import type { ForkCopyableLabel } from '@/lib/workspaces/fork/mapping/resources' +import { + assembleForkCopyableUnmapped, + buildPromoteWorkflowIdMap, + collectForkCopyableIdsByKind, +} from '@/lib/workspaces/fork/promote/promote-plan' +import type { ForkReference } from '@/lib/workspaces/fork/remap/remap-references' + +const ref = (kind: ForkReference['kind'], sourceId: string): ForkReference => ({ + kind, + sourceId, + subBlockKey: 'sb', + required: false, +}) /** * `buildPromoteWorkflowIdMap` decides which cross-workflow references survive a @@ -80,3 +93,62 @@ describe('buildPromoteWorkflowIdMap', () => { expect(map.get('s')).toBe('t-new') }) }) + +describe('collectForkCopyableIdsByKind', () => { + it('groups copyable kinds and ignores non-copyable kinds (credential / env-var)', () => { + const byKind = collectForkCopyableIdsByKind([ + ref('knowledge-base', 'kb-1'), + ref('knowledge-base', 'kb-2'), + ref('table', 'tbl-1'), + ref('credential', 'cred-1'), + ref('env-var', 'API_KEY'), + ref('custom-tool', 'ct-1'), + ref('skill', 'sk-1'), + ref('file', 'fk-1'), + ]) + expect(byKind).toEqual({ + 'knowledge-base': ['kb-1', 'kb-2'], + table: ['tbl-1'], + 'custom-tool': ['ct-1'], + skill: ['sk-1'], + file: ['fk-1'], + }) + }) +}) + +describe('assembleForkCopyableUnmapped', () => { + const flat = (label: string): ForkCopyableLabel => ({ label, parentId: null, parentLabel: null }) + + it('emits a candidate per copyable ref whose label resolved, carrying its labels', () => { + const labels = new Map([ + ['knowledge-base:kb-1', flat('Docs KB')], + ['file:fk-1', { label: 'a.png', parentId: 'fld-1', parentLabel: 'Folder' }], + ]) + const result = assembleForkCopyableUnmapped( + [ref('knowledge-base', 'kb-1'), ref('file', 'fk-1'), ref('credential', 'cred-1')], + labels + ) + expect(result).toEqual([ + { + kind: 'knowledge-base', + sourceId: 'kb-1', + label: 'Docs KB', + parentId: null, + parentLabel: null, + }, + { kind: 'file', sourceId: 'fk-1', label: 'a.png', parentId: 'fld-1', parentLabel: 'Folder' }, + ]) + }) + + it('drops a copyable ref whose label is missing (no longer exists in the source)', () => { + expect(assembleForkCopyableUnmapped([ref('knowledge-base', 'kb-gone')], new Map())).toEqual([]) + }) + + it('ignores non-copyable kinds entirely (credential / env-var)', () => { + const result = assembleForkCopyableUnmapped( + [ref('credential', 'cred-1'), ref('env-var', 'API_KEY')], + new Map([['credential:cred-1', flat('X')]]) + ) + expect(result).toEqual([]) + }) +}) diff --git a/apps/sim/lib/workspaces/fork/promote/promote-plan.ts b/apps/sim/lib/workspaces/fork/promote/promote-plan.ts index e0dde4eb7db..e4d271b736a 100644 --- a/apps/sim/lib/workspaces/fork/promote/promote-plan.ts +++ b/apps/sim/lib/workspaces/fork/promote/promote-plan.ts @@ -1,6 +1,7 @@ import { workflow } from '@sim/db/schema' import { generateId } from '@sim/utils/id' import { and, eq, isNull } from 'drizzle-orm' +import { type ForkCopyableKind, forkCopyableKindSchema } from '@/lib/api/contracts/workspace-fork' import type { DbOrTx } from '@/lib/db/types' import type { DeployedWorkflowSummary } from '@/lib/workspaces/fork/copy/deploy-bridge' import type { ForkEdge } from '@/lib/workspaces/fork/lineage/lineage' @@ -11,9 +12,12 @@ import { resourceTypeToForkKind, } from '@/lib/workspaces/fork/mapping/mapping-store' import { + type ForkCopyableLabel, filterExistingForkTargets, getWorkspaceEnvKeys, + loadForkCopyableResourceLabels, } from '@/lib/workspaces/fork/mapping/resources' +import { toScannerBlocks } from '@/lib/workspaces/fork/remap/reference-scan' import { type ForkReference, type ForkReferenceResolver, @@ -56,11 +60,35 @@ export interface ForkPromotePlan { mcpReauthServerIds: string[] /** Review-only descriptions of inline secrets that cannot be id-mapped. */ inlineSecretSources: string[] + /** + * Referenced-but-unmapped resources of copyable kinds that still exist in the source, so a + * sync can copy them into the target instead of requiring a manual mapping (U15). Documents + * are auto-copied with their parent KB and are not listed here. `parentId`/`parentLabel` carry + * a file's folder grouping (null for non-file kinds and root files), for the nested picker. + */ + copyableUnmapped: Array<{ + kind: ForkCopyableKind + sourceId: string + label: string + parentId: string | null + parentLabel: string | null + }> willUpdate: number willCreate: number willArchive: number } +/** + * Copyable promote kinds, derived from the wire contract (`forkCopyableKindSchema`) so this + * guard can never drift from the single source of truth: growing the schema automatically + * grows the set. Typed as `ForkRemapKind` so `.has` accepts a broad scan-reference kind. + */ +const COPYABLE_PROMOTE_KINDS = new Set(forkCopyableKindSchema.options) + +export function isForkCopyableKind(kind: ForkRemapKind): kind is ForkCopyableKind { + return COPYABLE_PROMOTE_KINDS.has(kind) +} + /** * Build the cross-workflow reference map used to rewrite `workflow-selector`, * `manualWorkflowId`, and `workflow_input` references inside promoted workflows. @@ -92,6 +120,48 @@ export function buildPromoteWorkflowIdMap(params: { return workflowIdMap } +/** + * Collect the source ids of referenced-but-unmapped copyable resources, grouped by kind - the input + * to the source-label lookup that builds {@link ForkPromotePlan.copyableUnmapped}. Pure. + */ +export function collectForkCopyableIdsByKind( + unmappedReferences: ForkReference[] +): Partial> { + const byKind: Partial> = {} + for (const reference of unmappedReferences) { + if (!isForkCopyableKind(reference.kind)) continue + ;(byKind[reference.kind] ??= []).push(reference.sourceId) + } + return byKind +} + +/** + * Assemble {@link ForkPromotePlan.copyableUnmapped} from the unmapped references and the loaded + * source labels: each copyable reference whose label resolved becomes a copy candidate; one whose + * label is missing (the resource no longer exists in the source) is dropped. Pure - split from the + * DB label load so it is unit-testable. + */ +export function assembleForkCopyableUnmapped( + unmappedReferences: ForkReference[], + copyableLabels: Map +): ForkPromotePlan['copyableUnmapped'] { + return unmappedReferences.flatMap((reference) => { + if (!isForkCopyableKind(reference.kind)) return [] + const entry = copyableLabels.get(`${reference.kind}:${reference.sourceId}`) + return entry + ? [ + { + kind: reference.kind, + sourceId: reference.sourceId, + label: entry.label, + parentId: entry.parentId, + parentLabel: entry.parentLabel, + }, + ] + : [] + }) +} + /** * Compute everything a promote needs without mutating. Only the source's * **deployed** workflows participate; each plan item carries the source's active @@ -208,12 +278,8 @@ export async function computeForkPromotePlan(params: { }, }) - const blocks = Object.values(sourceState.blocks).map((block) => ({ - id: block.id, - name: block.name, - subBlocks: block.subBlocks as unknown, - })) - for (const reference of scanWorkflowReferences(blocks, resolver).references) { + for (const reference of scanWorkflowReferences(toScannerBlocks(sourceState), resolver) + .references) { referenceByKey.set(`${reference.kind}:${reference.sourceId}`, reference) } } @@ -257,6 +323,15 @@ export async function computeForkPromotePlan(params: { const unmappedRequired = allUnmapped.filter((reference) => reference.required) const unmappedOptional = allUnmapped.filter((reference) => !reference.required) + // Referenced-but-unmapped resources of copyable kinds that still exist in the source, so the + // sync modal can offer to copy them into the target (fork-style) instead of mapping by hand. + const copyableLabels = await loadForkCopyableResourceLabels( + executor, + sourceWorkspaceId, + collectForkCopyableIdsByKind(allUnmapped) + ) + const copyableUnmapped = assembleForkCopyableUnmapped(allUnmapped, copyableLabels) + const willUpdate = items.filter((i) => i.mode === 'replace').length const willCreate = items.filter((i) => i.mode === 'create').length @@ -275,6 +350,7 @@ export async function computeForkPromotePlan(params: { unmappedOptional, mcpReauthServerIds: cascade.mcpReauthServerIds, inlineSecretSources: cascade.inlineSecretSources, + copyableUnmapped, willUpdate, willCreate, willArchive: archivedTargetIds.length, diff --git a/apps/sim/lib/workspaces/fork/promote/promote.ts b/apps/sim/lib/workspaces/fork/promote/promote.ts index 9694f279bf5..2af3bd4c928 100644 --- a/apps/sim/lib/workspaces/fork/promote/promote.ts +++ b/apps/sim/lib/workspaces/fork/promote/promote.ts @@ -4,6 +4,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { and, eq, inArray } from 'drizzle-orm' +import type { PromoteCopyResources } from '@/lib/api/contracts/workspace-fork' import type { DbOrTx } from '@/lib/db/types' import { enqueueWorkflowUndeploySideEffects, @@ -11,6 +12,14 @@ import { } from '@/lib/workflows/deployment-outbox' import { performFullDeploy } from '@/lib/workflows/orchestration/deploy' import { undeployWorkflow } from '@/lib/workflows/persistence/utils' +import { startBackgroundWork } from '@/lib/workspaces/fork/background-work/store' +import { + type ForkContentCopyPayload, + type SerializableForkContentRefMaps, + scheduleForkContentCopy, +} from '@/lib/workspaces/fork/copy/content-copy-runner' +import type { BlobCopyTask } from '@/lib/workspaces/fork/copy/copy-files' +import type { ForkContentPlan } from '@/lib/workspaces/fork/copy/copy-resources' import { copyWorkflowStateIntoTarget, loadTargetDraftSubBlocks, @@ -43,6 +52,12 @@ import { type ForkMappingUpsert, upsertEdgeMappings, } from '@/lib/workspaces/fork/mapping/mapping-store' +import { + augmentForkResolver, + buildPromoteCopySelection, + copyPromoteUnmappedResources, + hasPromoteCopySelection, +} from '@/lib/workspaces/fork/promote/copy-unmapped' import { computeForkPromotePlan, type ForkPromotePlan, @@ -82,6 +97,12 @@ export interface PromoteForkParams { subBlockKey: string value: string }> + /** + * Referenced-but-unmapped resources (by source id) the caller chose to copy into the target + * before the sync gate. Validated against the plan's copyable candidates, so an arbitrary id is + * ignored. Each copied resource's references then resolve to the new copy instead of blocking. + */ + copyResources?: PromoteCopyResources requestId?: string } @@ -264,6 +285,12 @@ interface PromoteTxApplied { needsConfiguration: Array<{ workflowId: string; workflowName: string; blocks: string[] }> /** Per-workflow optional dependents a parent change cleared (surfaced, not gated). */ clearedOptional: Array<{ workflowName: string; blocks: string[] }> + /** Heavy content for resources copied into the target this sync, filled best-effort post-commit. */ + copyContentPlan: ForkContentPlan | null + /** Serialized in-content maps for the post-commit skill-body rewrite (paired with the plan). */ + copyContentRefMaps: SerializableForkContentRefMaps | null + /** File blob duplications for copied workspace files, run post-commit by the content-copy runner. */ + copyContentBlobTasks: BlobCopyTask[] } /** @@ -352,10 +379,25 @@ export async function promoteFork(params: PromoteForkParams): Promise 0) { + const now = new Date() + + // Copy the selected referenced-but-unmapped resources into the target BEFORE the gate, so a + // user can copy rather than map each one. The gate is evaluated against the post-copy state + // (the copy resolves the selected refs), so the copy only runs when the sync will actually + // proceed - if required refs remain unmapped, we block without copying anything. + const { selection: copySelection, willResolve } = buildPromoteCopySelection( + params.copyResources, + plan.copyableUnmapped + ) + // plan.unmappedRequired is already references.filter(resolver == null).filter(required), so + // subtracting the refs the copy will resolve is equivalent to re-scanning the predicate. + const postCopyUnmappedRequired = plan.unmappedRequired.filter( + (reference) => !willResolve.has(`${reference.kind}:${reference.sourceId}`) + ) + if (postCopyUnmappedRequired.length > 0) { return { blocked: 'unmapped', - unmappedRequired: plan.unmappedRequired.map((reference) => ({ + unmappedRequired: postCopyUnmappedRequired.map((reference) => ({ kind: reference.kind, sourceId: reference.sourceId, required: reference.required, @@ -364,8 +406,10 @@ export async function promoteFork(params: PromoteForkParams): Promisetarget folder map BEFORE the copy so the folders already exist in the + // target and the copy can rewrite `sim:folder/` references inside copied skill / markdown + // bodies (the post-commit content rewrite reads this map). Idempotent: it reuses target + // folders that already match by name within the same mapped parent. const folderIdMap = await resolveForkFolderMapping({ tx, sourceWorkspaceId, @@ -374,6 +418,42 @@ export async function promoteFork(params: PromoteForkParams): Promise reference.kind === 'knowledge-document') + .map((reference) => reference.sourceId) + // Run the copy when the user selected resources to copy OR any document is referenced (a + // referenced document under an already-mapped KB is auto-copied into that KB so its reference + // remaps instead of clearing). It runs only after the required-reference gate above, so a + // blocked sync copies nothing. + if (hasPromoteCopySelection(copySelection) || referencedDocumentIds.length > 0) { + const copyResult = await copyPromoteUnmappedResources({ + tx, + edge, + sourceWorkspaceId, + targetWorkspaceId, + direction, + userId, + now, + selection: copySelection, + workflowIdMap: plan.workflowIdMap, + folderIdMap, + resolver: plan.resolver, + referencedDocumentIds, + }) + resolver = augmentForkResolver(plan.resolver, copyResult.copyIdMapByKind) + copyContentPlan = copyResult.contentPlan + copyContentRefMaps = copyResult.contentRefMaps + copyContentBlobTasks = copyResult.blobTasks + } + + const transform = createForkSubBlockTransform(resolver) + // Batch every prior-version read (replace + archive targets) into one query before any // write, so the locked apply phase doesn't do N round-trips. Reads are pre-write, so // they still reflect the active version each target had before this sync. @@ -634,6 +714,9 @@ export async function promoteFork(params: PromoteForkParams): Promise 0 || + copyContentPlan.knowledgeBases.length > 0 || + copyContentPlan.skills.length > 0 || + copyContentPlan.documents.length > 0 || + copyBlobTasks.length > 0) + if (copyContentPlan && hasCopyContent) { + // Scope the durable record to the workspace whose Manage Forks -> Activity the user is + // viewing (the one the sync was initiated from), matching where the route records the sync. + const activityWorkspaceId = direction === 'push' ? sourceWorkspaceId : targetWorkspaceId + // The sync already committed; failing to record the tracking row must not turn it into a 500. + // The runner no-ops its status updates when statusId is absent, so the copy still runs. + let statusId: string | undefined + try { + statusId = await startBackgroundWork(db, { + workspaceId: activityWorkspaceId, + kind: 'fork_content_copy', + // Append-only: each sync's content fill is a distinct entry in the Activity history. + supersede: false, + message: 'Copying synced resources', + metadata: { + tables: copyContentPlan.tables.length, + knowledgeBases: copyContentPlan.knowledgeBases.length, + files: copyBlobTasks.length, + }, + }) + } catch (error) { + logger.error(`[${requestId}] Failed to record sync content-copy status`, { + targetWorkspaceId, + error: getErrorMessage(error), + }) + } + + const payload: ForkContentCopyPayload = { + contentPlan: copyContentPlan, + blobTasks: copyBlobTasks, + contentRefMaps: txResult.copyContentRefMaps ?? undefined, + statusId, + // The targets this sync wrote and deployed above, so a failed content fill can sweep the + // dropped placeholder from their DEPLOYED version states too, not just drafts. + deployedTargetWorkflowIds: txResult.deployTargetIds, + requestId, + } + await scheduleForkContentCopy(payload, { detachedLabel: 'fork-sync-content-copy', requestId }) + } + if (deployFailures.length > 0) { logger.warn(`[${requestId}] Promote wrote state but some targets failed to deploy`, { sourceWorkspaceId, diff --git a/apps/sim/lib/workspaces/fork/remap/reference-scan.ts b/apps/sim/lib/workspaces/fork/remap/reference-scan.ts new file mode 100644 index 00000000000..65f196413b1 --- /dev/null +++ b/apps/sim/lib/workspaces/fork/remap/reference-scan.ts @@ -0,0 +1,57 @@ +import { + type ForkRemapKind, + scanWorkflowReferences, +} from '@/lib/workspaces/fork/remap/remap-references' +import type { WorkflowState } from '@/stores/workflows/workflow/types' + +/** + * Unique source ids of one remap kind referenced across a set of blocks - both top-level + * selectors and nested tool params. Used at fork/promote time to decide which KB documents + * a copy must create placeholders for (so their references remap to the copied doc instead + * of being cleared). The resolver is irrelevant here (every reference is detected regardless + * of mapping), so a null resolver is passed. + */ +export function collectReferencedResourceIds( + blocks: Array<{ id: string; name: string; subBlocks: unknown }>, + kind: ForkRemapKind +): Set { + const ids = new Set() + for (const reference of scanWorkflowReferences(blocks, () => null).references) { + if (reference.kind === kind) ids.add(reference.sourceId) + } + return ids +} + +/** + * Map a workflow state's blocks to the `{ id, name, subBlocks }` shape the reference scanner + * consumes. Confines the `subBlocks as unknown` widening (the stored subblock record is opaque + * to the scanner, which re-narrows per subblock type) to one spot shared by every fork caller + * that scans a source workflow's references. + */ +export function toScannerBlocks( + state: WorkflowState +): Array<{ id: string; name: string; subBlocks: unknown }> { + return Object.values(state.blocks).map((block) => ({ + id: block.id, + name: block.name, + subBlocks: block.subBlocks as unknown, + })) +} + +/** + * Unique knowledge-document ids referenced across a set of source workflow states. Fork and + * promote use this to decide which KB documents a copy must pre-create placeholders for, so a + * `document-selector` reference remaps to the copied doc instead of being cleared. + */ +export function collectReferencedDocumentIds(states: Iterable): Set { + const ids = new Set() + for (const state of states) { + for (const docId of collectReferencedResourceIds( + toScannerBlocks(state), + 'knowledge-document' + )) { + ids.add(docId) + } + } + return ids +} diff --git a/apps/sim/lib/workspaces/fork/remap/remap-content-refs.test.ts b/apps/sim/lib/workspaces/fork/remap/remap-content-refs.test.ts new file mode 100644 index 00000000000..a6acca88bf7 --- /dev/null +++ b/apps/sim/lib/workspaces/fork/remap/remap-content-refs.test.ts @@ -0,0 +1,194 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + type ForkContentRefMaps, + rewriteForkContentRefs, + rewriteForkResourceUrls, +} from '@/lib/workspaces/fork/remap/remap-content-refs' + +const SRC_KEY = 'workspace/SRC/1700000000000-deadbeef-photo.png' +const DST_KEY = 'workspace/DST/1700000000001-cafebabe-photo.png' + +const maps = (): ForkContentRefMaps => ({ + workspaceId: { from: 'SRC', to: 'DST' }, + fileKeys: new Map([[SRC_KEY, DST_KEY]]), + fileIds: new Map([['file-src', 'file-dst']]), + workflows: new Map([['wf-src', 'wf-dst']]), + knowledgeBases: new Map([['kb-src', 'kb-dst']]), + tables: new Map([['tbl-src', 'tbl-dst']]), + skills: new Map([['skill-src', 'skill-dst']]), + folders: new Map([['fld-src', 'fld-dst']]), +}) + +describe('rewriteForkContentRefs - sim: links', () => { + it('remaps each mapped sim: link kind by its id map', () => { + const input = [ + 'see [F](sim:file/file-src)', + '[W](sim:workflow/wf-src)', + '[K](sim:knowledge/kb-src)', + '[T](sim:table/tbl-src)', + '[S](sim:skill/skill-src)', + '[D](sim:folder/fld-src)', + ].join(' ') + expect(rewriteForkContentRefs(input, maps())).toBe( + [ + 'see [F](sim:file/file-dst)', + '[W](sim:workflow/wf-dst)', + '[K](sim:knowledge/kb-dst)', + '[T](sim:table/tbl-dst)', + '[S](sim:skill/skill-dst)', + '[D](sim:folder/fld-dst)', + ].join(' ') + ) + }) + + it('leaves an unmapped id unchanged (graceful broken link)', () => { + const input = '[F](sim:file/unknown-file) and [I](sim:integration/gmail_v2)' + expect(rewriteForkContentRefs(input, maps())).toBe(input) + }) + + it('leaves a kind with no supplied map unchanged', () => { + const input = '[W](sim:workflow/wf-src)' + expect(rewriteForkContentRefs(input, { fileIds: new Map() })).toBe(input) + }) +}) + +describe('rewriteForkContentRefs - embedded urls', () => { + it('remaps a serve-url storage key (encoded form output)', () => { + const input = `![a](/api/files/serve/${encodeURIComponent(SRC_KEY)}?context=workspace)` + expect(rewriteForkContentRefs(input, maps())).toBe( + `![a](/api/files/serve/${encodeURIComponent(DST_KEY)}?context=workspace)` + ) + }) + + it('remaps a serve-url key given in raw (unencoded) form', () => { + const input = `![a](/api/files/serve/${SRC_KEY})` + expect(rewriteForkContentRefs(input, maps())).toBe( + `![a](/api/files/serve/${encodeURIComponent(DST_KEY)})` + ) + }) + + it('remaps an s3/blob-prefixed serve url, preserving the prefix', () => { + const input = `![a](/api/files/serve/s3/${encodeURIComponent(SRC_KEY)})` + expect(rewriteForkContentRefs(input, maps())).toBe( + `![a](/api/files/serve/s3/${encodeURIComponent(DST_KEY)})` + ) + }) + + it('remaps a view-url file id', () => { + const input = '![a](/api/files/view/file-src)' + expect(rewriteForkContentRefs(input, maps())).toBe('![a](/api/files/view/file-dst)') + }) + + it('remaps both the workspace id and file id in an in-app files path', () => { + const input = '![a](/workspace/SRC/files/file-src)' + expect(rewriteForkContentRefs(input, maps())).toBe('![a](/workspace/DST/files/file-dst)') + }) + + it('leaves a foreign-workspace in-app file path unchanged (both-or-nothing)', () => { + // The ws id is not the mapped source, so emitting the child file id under OTHER would 404. + const input = '![a](/workspace/OTHER/files/file-src)' + expect(rewriteForkContentRefs(input, maps())).toBe(input) + }) + + it('leaves an in-app file path unchanged when the file id is unmapped (both-or-nothing)', () => { + const input = '![a](/workspace/SRC/files/unknown-file)' + expect(rewriteForkContentRefs(input, maps())).toBe(input) + }) + + it('leaves an unmapped storage key / file id unchanged', () => { + const input = + '![a](/api/files/serve/workspace%2FSRC%2Funknown.png) ![b](/api/files/view/unknown-id)' + expect(rewriteForkContentRefs(input, maps())).toBe(input) + }) + + it('leaves an external / data url unchanged', () => { + const input = '![a](https://cdn.example.com/x.png) ![b](data:image/png;base64,AAAA)' + expect(rewriteForkContentRefs(input, maps())).toBe(input) + }) +}) + +describe('rewriteForkContentRefs - mixed and edge cases', () => { + it('rewrites multiple references of different shapes in one string', () => { + const input = [ + 'intro [S](sim:skill/skill-src)', + `![img](/api/files/serve/${encodeURIComponent(SRC_KEY)})`, + 'link [W](sim:workflow/wf-src)', + '![v](/api/files/view/file-src)', + ].join('\n') + const output = rewriteForkContentRefs(input, maps()) + expect(output).toContain('sim:skill/skill-dst') + expect(output).toContain('sim:workflow/wf-dst') + expect(output).toContain(encodeURIComponent(DST_KEY)) + expect(output).toContain('/api/files/view/file-dst') + }) + + it('returns the input unchanged when there are no references', () => { + const input = '# Heading\n\nNo references here, just text.' + expect(rewriteForkContentRefs(input, maps())).toBe(input) + }) + + it('returns the input unchanged for an empty string', () => { + expect(rewriteForkContentRefs('', maps())).toBe('') + }) + + it('leaves a malformed (un-decodable) serve key unchanged', () => { + const input = '![a](/api/files/serve/%E0%A4%A)' + expect(rewriteForkContentRefs(input, maps())).toBe(input) + }) + + it('does nothing when no maps are supplied', () => { + const input = `[S](sim:skill/skill-src) ![a](/api/files/serve/${SRC_KEY})` + expect(rewriteForkContentRefs(input, {})).toBe(input) + }) +}) + +describe('rewriteForkResourceUrls - table cell resource chip urls', () => { + it('rewrites the workspace id + resource id for each section when the resource is mapped', () => { + expect(rewriteForkResourceUrls('/workspace/SRC/w/wf-src', maps())).toBe( + '/workspace/DST/w/wf-dst' + ) + expect(rewriteForkResourceUrls('/workspace/SRC/tables/tbl-src', maps())).toBe( + '/workspace/DST/tables/tbl-dst' + ) + expect(rewriteForkResourceUrls('/workspace/SRC/knowledge/kb-src', maps())).toBe( + '/workspace/DST/knowledge/kb-dst' + ) + expect(rewriteForkResourceUrls('/workspace/SRC/files/file-src', maps())).toBe( + '/workspace/DST/files/file-dst' + ) + }) + + it('leaves an unmapped resource id unchanged (both-or-nothing -> graceful plain link)', () => { + expect(rewriteForkResourceUrls('/workspace/SRC/w/wf-unknown', maps())).toBe( + '/workspace/SRC/w/wf-unknown' + ) + }) + + it('leaves a foreign / unknown workspace id unchanged', () => { + expect(rewriteForkResourceUrls('/workspace/OTHER/w/wf-src', maps())).toBe( + '/workspace/OTHER/w/wf-src' + ) + }) + + it('leaves a non-matching string (unknown section / no resource path) unchanged', () => { + const input = 'text /workspace/ and /workspace/SRC/settings/x and /workspace/SRC/w/' + expect(rewriteForkResourceUrls(input, maps())).toBe(input) + }) + + it('does nothing without a workspaceId map', () => { + const input = '/workspace/SRC/w/wf-src' + expect(rewriteForkResourceUrls(input, { workflows: new Map([['wf-src', 'wf-dst']]) })).toBe( + input + ) + }) + + it('rewrites a URL embedded mid-text', () => { + const input = 'See [chip](/workspace/SRC/knowledge/kb-src) here' + expect(rewriteForkResourceUrls(input, maps())).toBe( + 'See [chip](/workspace/DST/knowledge/kb-dst) here' + ) + }) +}) diff --git a/apps/sim/lib/workspaces/fork/remap/remap-content-refs.ts b/apps/sim/lib/workspaces/fork/remap/remap-content-refs.ts new file mode 100644 index 00000000000..1f62e4057b0 --- /dev/null +++ b/apps/sim/lib/workspaces/fork/remap/remap-content-refs.ts @@ -0,0 +1,127 @@ +/** + * Pure rewriter for the in-content references embedded in copied free-text (skill bodies, + * markdown file blobs) at fork/sync time. It rewrites the two reference shapes a copy must + * keep pointing at the right workspace: + * + * - `sim:/` deep links (the `@`-mention / chip scheme) - the id is remapped through + * the matching fork id map by kind (file, folder, table, knowledge, workflow, skill). + * - Embedded file/image URLs - `/api/files/serve/` (workspace storage key), `/api/files/view/` + * (workspace file id), and the in-app `/workspace//files/` path - remapped through the file + * key / file id / workspace id maps. + * + * A reference whose target has no mapping is LEFT UNCHANGED (a graceful broken link), never deleted, + * so a copied document is never silently corrupted. Pure and isomorphic (no DOM/Node/DB). + */ + +/** Per-kind source->target id maps a content copy threads in (any subset may be supplied). */ +export interface ForkContentRefMaps { + /** Workspace id rewrite for the in-app `/workspace//files/...` path. */ + workspaceId?: { from: string; to: string } + /** source workspace-file storage key -> child storage key (serve-url embeds). */ + fileKeys?: ReadonlyMap + /** source workspace-file id -> child id (view-url + in-app-path embeds). */ + fileIds?: ReadonlyMap + /** source workflow id -> child id (`sim:workflow/`). */ + workflows?: ReadonlyMap + /** source knowledge-base id -> child id (`sim:knowledge/`). */ + knowledgeBases?: ReadonlyMap + /** source table id -> child id (`sim:table/`). */ + tables?: ReadonlyMap + /** source skill id -> child id (`sim:skill/`). */ + skills?: ReadonlyMap + /** source folder id -> child id (`sim:folder/`). */ + folders?: ReadonlyMap +} + +/** `sim:/` token; the id charset matches generateId/generateShortId so it stops at delimiters. */ +const SIM_LINK_RE = /sim:([a-z_]+)\/([A-Za-z0-9_-]+)/g +/** `/api/files/serve/[s3/|blob/]` (key may be raw or percent-encoded, ends at a delimiter). */ +const SERVE_URL_RE = /\/api\/files\/serve\/(s3\/|blob\/)?([^\s)"'<>?]+)/g +/** `/api/files/view/`. */ +const VIEW_URL_RE = /\/api\/files\/view\/([A-Za-z0-9_-]+)/g +/** In-app `/workspace//files/` embed path. */ +const INAPP_FILE_RE = /\/workspace\/([A-Za-z0-9-]+)\/files\/([A-Za-z0-9_-]+)/g + +export function rewriteForkContentRefs(content: string, maps: ForkContentRefMaps): string { + if (!content) return content + + const idMapForSimKind: Record | undefined> = { + file: maps.fileIds, + folder: maps.folders, + table: maps.tables, + knowledge: maps.knowledgeBases, + workflow: maps.workflows, + skill: maps.skills, + } + + let result = content.replace(SIM_LINK_RE, (full, kind: string, id: string) => { + const target = idMapForSimKind[kind]?.get(id) + return target ? `sim:${kind}/${target}` : full + }) + + result = result.replace(SERVE_URL_RE, (full, prefix: string | undefined, encodedKey: string) => { + if (!maps.fileKeys) return full + let key: string + try { + key = decodeURIComponent(encodedKey) + } catch { + return full + } + const target = maps.fileKeys.get(key) + return target ? `/api/files/serve/${prefix ?? ''}${encodeURIComponent(target)}` : full + }) + + result = result.replace(VIEW_URL_RE, (full, id: string) => { + const target = maps.fileIds?.get(id) + return target ? `/api/files/view/${target}` : full + }) + + // Both-or-nothing: rewrite only when the workspace id is the mapped source AND the file id + // resolves, so we never emit a child-workspace path with an unmapped (guaranteed-404) id - + // matching the serve/view branches and rewriteForkResourceUrls. A foreign workspace id or an + // unmapped file id leaves the original path untouched (graceful). + result = result.replace(INAPP_FILE_RE, (full, wsId: string, fileId: string) => { + if (maps.workspaceId?.from !== wsId) return full + const mappedFile = maps.fileIds?.get(fileId) + return mappedFile ? `/workspace/${maps.workspaceId.to}/files/${mappedFile}` : full + }) + + return result +} + +/** + * In-app `/workspace//(w|tables|knowledge|files)/` deep link - the form a TABLE + * CELL renders as a resource chip (`resolveSimResourceKind`), but ONLY when `wsId` matches the + * current workspace. The id charset stops at delimiters. + */ +const RESOURCE_URL_RE = + /\/workspace\/([A-Za-z0-9-]+)\/(w|tables|knowledge|files)\/([A-Za-z0-9_-]+)/g + +/** + * Rewrite the in-workspace resource deep links a copied table cell renders as a resource chip, so + * the chip keeps resolving after a cross-workspace copy (a cell chip renders only when the URL's + * workspace id is the current workspace, so a stale source id silently degrades to a plain link). + * Repoints both the workspace id and the resource id at the child copy, per section: `w` -> + * workflows, `tables` -> tables, `knowledge` -> knowledge bases, `files` -> file ids. + * + * Both-or-nothing: a match is rewritten ONLY when its workspace id is the mapped source AND its + * resource id is in that section's map - otherwise it is left UNCHANGED. Emitting a child-workspace + * URL with an unmapped id would render a "Not found" chip (worse than the graceful plain link an + * unchanged URL degrades to). Distinct from {@link rewriteForkContentRefs}: table cells do NOT + * specially render the `sim:` / serve / view forms that skill + markdown bodies do. + */ +export function rewriteForkResourceUrls(content: string, maps: ForkContentRefMaps): string { + if (!content || !maps.workspaceId) return content + const { from, to } = maps.workspaceId + const idMapForSection: Record | undefined> = { + w: maps.workflows, + tables: maps.tables, + knowledge: maps.knowledgeBases, + files: maps.fileIds, + } + return content.replace(RESOURCE_URL_RE, (full, wsId: string, section: string, id: string) => { + if (wsId !== from) return full + const mappedId = idMapForSection[section]?.get(id) + return mappedId ? `/workspace/${to}/${section}/${mappedId}` : full + }) +} diff --git a/apps/sim/lib/workspaces/fork/remap/remap-files.ts b/apps/sim/lib/workspaces/fork/remap/remap-files.ts index ea83a6d161c..a0442c352d0 100644 --- a/apps/sim/lib/workspaces/fork/remap/remap-files.ts +++ b/apps/sim/lib/workspaces/fork/remap/remap-files.ts @@ -36,6 +36,30 @@ function fileItemKeyField(item: unknown): { field: 'key' | 'path' | 'name'; key: return null } +/** + * Enumerate the workspace-file storage keys referenced by a `file-upload` subblock + * value (single object, array, or JSON-string form). Used at promote time to emit each + * workspace file as a `file` reference (keyed by storage key) so it surfaces in the + * scan / unmapped set and can be copied into the target. Deduplicated, order-preserving. + */ +export function collectForkFileUploadKeys(value: unknown): string[] { + const parsed = parseMaybeJson(value) + const items = Array.isArray(parsed.value) + ? (parsed.value as unknown[]) + : parsed.value + ? [parsed.value] + : [] + const keys: string[] = [] + const seen = new Set() + for (const item of items) { + const keyInfo = fileItemKeyField(item) + if (!keyInfo || seen.has(keyInfo.key)) continue + seen.add(keyInfo.key) + keys.push(keyInfo.key) + } + return keys +} + /** * Remap a `file-upload` subblock value. `resolveFileKey(sourceKey)` returns the * copied target storage key, or null when the file was not copied (drop the ref). diff --git a/apps/sim/lib/workspaces/fork/remap/remap-references.test.ts b/apps/sim/lib/workspaces/fork/remap/remap-references.test.ts index f607a1fb70f..a954b971388 100644 --- a/apps/sim/lib/workspaces/fork/remap/remap-references.test.ts +++ b/apps/sim/lib/workspaces/fork/remap/remap-references.test.ts @@ -20,6 +20,7 @@ vi.mock('@/tools/params', () => ({ })) import type { SubBlockRecord } from '@/lib/workflows/persistence/remap-internal-ids' +import { createForkBootstrapTransform } from '@/lib/workspaces/fork/remap/fork-bootstrap' import { applyDependentOverrides, collectClearedDependents, @@ -201,6 +202,38 @@ describe('remapToolBlockResources', () => { }) expect(result.params).toEqual({ knowledgeBaseId: 'kb-dst', documentId: '' }) }) + + it('remaps a nested documentId through the doc map when its document was copied', () => { + const tool = { + type: 'depblock', + toolId: 'depblock_run', + params: { knowledgeBaseId: 'kb-src', documentId: 'doc-src' }, + } + const map: Record = { + 'knowledge-base:kb-src': 'kb-dst', + 'knowledge-document:doc-src': 'doc-dst', + } + const result = remapToolBlockResources(tool, { + resolve: (kind, id) => map[`${kind}:${id}`] ?? null, + resolveFileKey: () => null, + clearUnresolved: true, + blockConfigs: { + depblock: { + subBlocks: [ + { id: 'knowledgeBaseId', title: 'KB', type: 'knowledge-base-selector' }, + { + id: 'documentId', + title: 'Doc', + type: 'document-selector', + dependsOn: ['knowledgeBaseId'], + }, + ], + }, + }, + }) + // documentId is remapped (not cleared as a dependent) because its document was copied. + expect(result.params).toEqual({ knowledgeBaseId: 'kb-dst', documentId: 'doc-dst' }) + }) }) describe('remapForkSubBlocks', () => { @@ -283,6 +316,52 @@ describe('remapForkSubBlocks', () => { const tools = result.subBlocks.tools.value as Array<{ params: { subject: string } }> expect(tools[0].params.subject).toBe('Hi {{NEW}}') }) + + const fileSubBlock = (): SubBlockRecord => ({ + file: { + id: 'file', + type: 'file-upload', + value: { key: 'workspace/SRC/a.png', name: 'a.png' }, + }, + }) + + it('promote mode: records an unmapped file-upload key as a file reference and clears it', () => { + const result = remapForkSubBlocks(fileSubBlock(), () => null, 'promote') + const keys = result.references.map((r) => `${r.kind}:${r.sourceId}`) + expect(keys).toContain('file:workspace/SRC/a.png') + // file refs are optional (not required), surfaced for the copy/clear decision. + expect(result.references.find((r) => r.kind === 'file')?.required).toBe(false) + expect(result.unmapped.map((r) => `${r.kind}:${r.sourceId}`)).toContain( + 'file:workspace/SRC/a.png' + ) + // An uncopied file key is dropped rather than carried cross-workspace. + expect(result.subBlocks.file.value).toBe('') + }) + + it('promote mode: remaps a file-upload key to the copied target and records it mapped', () => { + const result = remapForkSubBlocks( + fileSubBlock(), + (kind, id) => + kind === 'file' && id === 'workspace/SRC/a.png' ? 'workspace/DST/a.png' : null, + 'promote' + ) + expect(result.references.map((r) => `${r.kind}:${r.sourceId}`)).toContain( + 'file:workspace/SRC/a.png' + ) + expect(result.unmapped).toHaveLength(0) + expect((result.subBlocks.file.value as { key: string }).key).toBe('workspace/DST/a.png') + }) + + it('create mode: does not record file references but still remaps copied files', () => { + const result = remapForkSubBlocks( + fileSubBlock(), + (kind, id) => + kind === 'file' && id === 'workspace/SRC/a.png' ? 'workspace/DST/a.png' : null, + 'create' + ) + expect(result.references).toHaveLength(0) + expect((result.subBlocks.file.value as { key: string }).key).toBe('workspace/DST/a.png') + }) }) const blockWith = (subBlocks: SubBlockConfig[]): BlockConfig => @@ -290,6 +369,48 @@ const blockWith = (subBlocks: SubBlockConfig[]): BlockConfig => const entry = (id: string, type: string, value: unknown) => ({ id, type, value }) +describe('createForkBootstrapTransform document-selector remap', () => { + const docBlock = () => + blockWith([ + { id: 'knowledgeBaseId', title: 'KB', type: 'knowledge-base-selector' }, + { id: 'documentId', title: 'Doc', type: 'document-selector', dependsOn: ['knowledgeBaseId'] }, + ]) + const subBlocks = (): SubBlockRecord => ({ + knowledgeBaseId: { id: 'knowledgeBaseId', type: 'knowledge-base-selector', value: 'kb-src' }, + documentId: { id: 'documentId', type: 'document-selector', value: 'doc-src' }, + }) + + it('remaps documentId to the copied document (not cleared as a KB dependent)', () => { + vi.mocked(getBlock).mockReturnValue(docBlock()) + const map: Record = { + 'knowledge-base:kb-src': 'kb-dst', + 'knowledge-document:doc-src': 'doc-dst', + } + const transform = createForkBootstrapTransform((kind, id) => map[`${kind}:${id}`] ?? null) + const result = transform(subBlocks(), 'knowledge') + expect(result.knowledgeBaseId.value).toBe('kb-dst') + expect(result.documentId.value).toBe('doc-dst') + }) + + it('clears documentId when its parent KB was not copied', () => { + vi.mocked(getBlock).mockReturnValue(docBlock()) + const transform = createForkBootstrapTransform(() => null) + const result = transform(subBlocks(), 'knowledge') + expect(result.knowledgeBaseId.value).toBe('') + expect(result.documentId.value).toBe('') + }) + + it('clears documentId when its KB was copied but the document was not', () => { + vi.mocked(getBlock).mockReturnValue(docBlock()) + const transform = createForkBootstrapTransform((kind, id) => + kind === 'knowledge-base' && id === 'kb-src' ? 'kb-dst' : null + ) + const result = transform(subBlocks(), 'knowledge') + expect(result.knowledgeBaseId.value).toBe('kb-dst') + expect(result.documentId.value).toBe('') + }) +}) + describe('collectClearedDependents', () => { it('flags a required dependent the target had set but the merge left empty', () => { vi.mocked(getBlock).mockReturnValue( diff --git a/apps/sim/lib/workspaces/fork/remap/remap-references.ts b/apps/sim/lib/workspaces/fork/remap/remap-references.ts index 7df375febaa..f0db237ab42 100644 --- a/apps/sim/lib/workspaces/fork/remap/remap-references.ts +++ b/apps/sim/lib/workspaces/fork/remap/remap-references.ts @@ -23,7 +23,10 @@ import { isNonEmptyValue, } from '@/lib/workflows/subblocks/visibility' import type { ParsedStoredTool } from '@/lib/workflows/tool-input/types' -import { remapForkFileUploadValue } from '@/lib/workspaces/fork/remap/remap-files' +import { + collectForkFileUploadKeys, + remapForkFileUploadValue, +} from '@/lib/workspaces/fork/remap/remap-files' import { getBlock } from '@/blocks/registry' import type { SubBlockConfig } from '@/blocks/types' @@ -38,7 +41,12 @@ export type ForkRemapKind = z.infer const logger = createLogger('WorkspaceForkRemapReferences') -const REQUIRED_KINDS = new Set(['credential', 'env-var']) +/** + * Reference kinds whose absence BLOCKS a sync (they gate `requiredComplete` and are resolved by + * mapping), as opposed to optional kinds that silently clear. Exported so the cleared-ref preview + * can exclude them - a required ref is a blocker, never a silent "will be cleared" item. + */ +export const REQUIRED_KINDS = new Set(['credential', 'env-var']) /** * Id-based override kind for a TOOL param's credential, resolved by subblock id so a @@ -59,20 +67,43 @@ export const REGISTRY_KIND_TO_FORK_KIND: Partial< > = { 'oauth-credential': 'credential', 'knowledge-base': 'knowledge-base', + 'knowledge-document': 'knowledge-document', table: 'table', 'mcp-server': 'mcp-server', } -// `file` and `knowledge-document` are intentionally excluded from the generic -// registry path. `file-upload` (workspace files) is remapped by storage key via -// `remapForkFileUploadValue`; `file-selector` (external provider file ids, -// credential-scoped) carries over unchanged; `document-selector` is cleared by the -// `dependsOn` rule (clearDependentsOnRemap) when its parent knowledge base is remapped. -// `mcp-tool-selector` is likewise cleared by `dependsOn` when its `mcp-server-selector` -// parent is remapped - the tool list is server-scoped and may differ in the target. +// `file` is intentionally excluded from the generic registry path: `file-upload` +// (workspace files) is remapped by storage key via `remapForkFileUploadValue`, and +// `file-selector` (external provider file ids, credential-scoped) carries over +// unchanged. `document-selector` (`knowledge-document`) IS remapped through the doc-id +// map when its referenced document was copied into the fork; an unmapped document (its +// parent KB wasn't copied, or the doc wasn't copyable) resolves to null and is cleared, +// and `clearDependentsOnRemap` still clears it as a `knowledgeBaseId` dependent when the +// parent KB itself is unmapped. `mcp-tool-selector` is cleared by `dependsOn` when its +// `mcp-server-selector` parent is remapped - the tool list is server-scoped and may +// differ in the target. /** Matches `{{ENV_KEY}}` references inside subblock values; shared with cascade detection. */ export const ENV_REF_PATTERN = /\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g +/** + * Rewrite `{{ENV}}` references in free text (a copied custom tool's `code`, an MCP url/header) + * through an env-name resolver, so a promote that renames an env var (e.g. SLACK_API_KEY -> + * SLACK_API_KEY_TEST) keeps the copied text pointing at the right key. A key the resolver leaves + * unmapped (null/undefined) or maps to the same name is kept verbatim - a graceful no-op so an env + * that exists under the same name in the target still works. Pure; mirrors {@link remapEnvInValue}'s + * preserve-by-name policy for the string case. + */ +export function rewriteEnvRefsInText( + text: string, + resolveEnvName: (key: string) => string | null | undefined +): string { + if (!text) return text + return text.replace(ENV_REF_PATTERN, (full, key: string) => { + const target = resolveEnvName(key) + return target && target !== key ? `{{${target}}}` : full + }) +} + /** * A `credentialSet:` reference points at an ORG-scoped credential set. A fork * inherits its parent's org, so the set is already valid in the target — these refs @@ -232,8 +263,12 @@ export function remapToolBlockResources( if (definition.kind === 'file') { // file-upload (workspace file) remaps by storage key; file-selector (external - // provider id) carries over unchanged. + // provider id) carries over unchanged. Each key is recorded as a `file` reference so + // a nested tool's workspace file surfaces in the scan / unmapped set and can be copied. if (config.type !== 'file-upload') continue + for (const fileKey of collectForkFileUploadKeys(currentValue)) { + opts.record?.('file', fileKey, opts.resolveFileKey(fileKey) != null) + } const remapped = remapForkFileUploadValue(currentValue, opts.resolveFileKey) if (remapped !== currentValue) { setParam(paramId, remapped) @@ -485,9 +520,25 @@ export function remapForkSubBlocks( } if (subBlockType === 'file-upload') { - // Workspace-file refs don't sync on promote (the target lacks the source's - // blob); clear them rather than carry a cross-workspace key. On fork, the - // resolver returns the copied key. `file-selector` (external) is untouched. + // Each workspace-file key is a `file` reference (keyed by storage key). Recording it + // surfaces the file in the scan / unmapped set so a sync can copy it into the target, + // exactly like fork - rather than silently clearing it. The resolver returns the copied + // key once the file has been copied; an unmapped (uncopied) key is dropped by the remap + // below. `file-selector` (external provider ids) is untouched. + for (const fileKey of collectForkFileUploadKeys(value)) { + recordReference( + `file:${fileKey}`, + { + kind: 'file', + sourceId: fileKey, + blockId: context?.blockId, + blockName: context?.blockName, + subBlockKey, + required: false, + }, + resolve('file', fileKey) != null + ) + } value = remapForkFileUploadValue(value, (sourceKey) => resolve('file', sourceKey) ?? null) } else if (subBlockType === 'tool-input' || subBlockType === 'skill-input') { const record = (kind: ForkRemapKind, sourceId: string, mapped: boolean) => From 06343eb9690d69d1488c8b1062fe2f21a4e7bad1 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Tue, 30 Jun 2026 12:54:39 -0700 Subject: [PATCH 2/5] update UX nits --- .../fork-activity-panel.tsx | 4 +- .../fork-resource-picker.tsx | 22 ++- .../fork-workspace-modal.tsx | 18 +- .../cleared-refs-list.test.ts | 88 +++++----- .../cleared-refs-list.ts | 12 +- .../promote-workspace-modal.tsx | 160 ++++++++++++------ apps/sim/lib/api/contracts/workspace-fork.ts | 65 +++---- .../fork/promote/cleared-refs.test.ts | 4 - .../workspaces/fork/promote/cleared-refs.ts | 4 - 9 files changed, 216 insertions(+), 161 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-activity-panel/fork-activity-panel.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-activity-panel/fork-activity-panel.tsx index 97bd5908068..ec1208133e0 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-activity-panel/fork-activity-panel.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/fork-activity-panel/fork-activity-panel.tsx @@ -247,9 +247,9 @@ function ForkJobRow({ job }: { job: BackgroundWorkItem }) { {report.groups.map((group) => ( ))} - {report.notes.map((note) => ( + {report.notes.map((note, index) => ( - onToggleAll: (selectAll: boolean) => void + /** Toggle the given ids on/off. Used for select-all over the currently-VISIBLE (filtered) subset. */ + onToggleMany: (ids: string[], checked: boolean) => void onToggleItem: (id: string, checked: boolean) => void disabled?: boolean } @@ -38,7 +39,7 @@ export function ResourceKindRow({ label, items, selected, - onToggleAll, + onToggleMany, onToggleItem, disabled = false, }: ResourceKindRowProps) { @@ -46,16 +47,18 @@ export function ResourceKindRow({ const [query, setQuery] = useState('') const fieldId = useId() - const total = items.length - const selectedCount = selected.size - const headerState = selectedCount === 0 ? false : selectedCount === total ? true : 'indeterminate' - const filtered = useMemo(() => { const trimmed = query.trim().toLowerCase() if (!trimmed) return items return items.filter((item) => item.label.toLowerCase().includes(trimmed)) }, [items, query]) + // Count + header state + select-all are scoped to the VISIBLE (filtered) items so a search never + // selects or counts hidden ones. With no filter, `filtered === items`, so behavior is unchanged. + const total = filtered.length + const selectedCount = filtered.reduce((count, item) => count + (selected.has(item.id) ? 1 : 0), 0) + const headerState = selectedCount === 0 ? false : selectedCount === total ? true : 'indeterminate' + return (
@@ -63,7 +66,12 @@ export function ResourceKindRow({ size='sm' aria-label={`Copy all ${label}`} checked={headerState} - onCheckedChange={() => onToggleAll(headerState !== true)} + onCheckedChange={() => + onToggleMany( + filtered.map((item) => item.id), + headerState !== true + ) + } disabled={disabled} />