From 1fd00ede6a9c97144ca255576a89b7755ee10bf7 Mon Sep 17 00:00:00 2001 From: waleed Date: Mon, 6 Jul 2026 18:03:49 -0700 Subject: [PATCH 1/6] feat(textract): migrate to AWS SDK, add AnalyzeExpense and AnalyzeID - Replace hand-rolled AWS SigV4 signing with @aws-sdk/client-textract, matching sibling AWS integrations (secrets_manager, s3, sts) - Add Analyze Expense operation (invoice/receipt structured extraction) via AnalyzeExpense/StartExpenseAnalysis+GetExpenseAnalysis - Add Analyze Identity Document operation (AnalyzeID) with optional back-of-ID page - Add an operation selector to the textract_v2 block; defaults to the existing Analyze Document behavior for backward compatibility - Add tests for tool body/response mapping and route-level AWS response normalization --- .../textract/analyze-expense/route.test.ts | 81 +++ .../tools/textract/analyze-expense/route.ts | 217 ++++++ .../tools/textract/analyze-id/route.test.ts | 63 ++ .../api/tools/textract/analyze-id/route.ts | 150 +++++ .../sim/app/api/tools/textract/parse/route.ts | 634 +++--------------- .../sim/app/api/tools/textract/shared.test.ts | 133 ++++ apps/sim/app/api/tools/textract/shared.ts | 301 +++++++++ apps/sim/blocks/blocks/textract.ts | 274 ++++++-- .../contracts/tools/media/document-parse.ts | 73 ++ apps/sim/package.json | 1 + apps/sim/tools/registry.ts | 9 +- apps/sim/tools/textract/analyze-expense.ts | 219 ++++++ apps/sim/tools/textract/analyze-id.ts | 152 +++++ apps/sim/tools/textract/index.ts | 2 + apps/sim/tools/textract/textract.test.ts | 210 ++++++ apps/sim/tools/textract/types.ts | 124 +++- bun.lock | 3 + scripts/check-api-validation-contracts.ts | 4 +- 18 files changed, 2042 insertions(+), 608 deletions(-) create mode 100644 apps/sim/app/api/tools/textract/analyze-expense/route.test.ts create mode 100644 apps/sim/app/api/tools/textract/analyze-expense/route.ts create mode 100644 apps/sim/app/api/tools/textract/analyze-id/route.test.ts create mode 100644 apps/sim/app/api/tools/textract/analyze-id/route.ts create mode 100644 apps/sim/app/api/tools/textract/shared.test.ts create mode 100644 apps/sim/app/api/tools/textract/shared.ts create mode 100644 apps/sim/tools/textract/analyze-expense.ts create mode 100644 apps/sim/tools/textract/analyze-id.ts create mode 100644 apps/sim/tools/textract/textract.test.ts diff --git a/apps/sim/app/api/tools/textract/analyze-expense/route.test.ts b/apps/sim/app/api/tools/textract/analyze-expense/route.test.ts new file mode 100644 index 00000000000..76cb459128f --- /dev/null +++ b/apps/sim/app/api/tools/textract/analyze-expense/route.test.ts @@ -0,0 +1,81 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { normalizeExpenseDocuments } from '@/app/api/tools/textract/analyze-expense/route' + +describe('normalizeExpenseDocuments', () => { + it('maps a documented AWS AnalyzeExpense response shape', () => { + const result = normalizeExpenseDocuments([ + { + ExpenseIndex: 1, + SummaryFields: [ + { + Type: { Text: 'VENDOR_NAME', Confidence: 98.1 }, + ValueDetection: { Text: 'Acme Corp', Confidence: 97.5 }, + LabelDetection: { Text: 'Vendor', Confidence: 90 }, + PageNumber: 1, + Currency: { Code: 'USD', Confidence: 95 }, + GroupProperties: [{ Id: 'g1', Types: ['VENDOR'] }], + }, + ], + LineItemGroups: [ + { + LineItemGroupIndex: 1, + LineItems: [ + { + LineItemExpenseFields: [ + { + Type: { Text: 'ITEM', Confidence: 91 }, + ValueDetection: { Text: 'Widget', Confidence: 93 }, + }, + ], + }, + ], + }, + ], + }, + ]) + + expect(result).toEqual([ + { + expenseIndex: 1, + summaryFields: [ + { + type: { text: 'VENDOR_NAME', confidence: 98.1 }, + valueDetection: { text: 'Acme Corp', confidence: 97.5 }, + labelDetection: { text: 'Vendor', confidence: 90 }, + pageNumber: 1, + currency: { code: 'USD', confidence: 95 }, + groupProperties: [{ id: 'g1', types: ['VENDOR'] }], + }, + ], + lineItemGroups: [ + { + lineItemGroupIndex: 1, + lineItems: [ + { + lineItemExpenseFields: [ + { + type: { text: 'ITEM', confidence: 91 }, + valueDetection: { text: 'Widget', confidence: 93 }, + labelDetection: undefined, + pageNumber: undefined, + currency: undefined, + groupProperties: undefined, + }, + ], + }, + ], + }, + ], + }, + ]) + }) + + it('defaults missing arrays to empty arrays', () => { + expect(normalizeExpenseDocuments([{ ExpenseIndex: 0 }])).toEqual([ + { expenseIndex: 0, summaryFields: [], lineItemGroups: [] }, + ]) + }) +}) diff --git a/apps/sim/app/api/tools/textract/analyze-expense/route.ts b/apps/sim/app/api/tools/textract/analyze-expense/route.ts new file mode 100644 index 00000000000..bf04b26977a --- /dev/null +++ b/apps/sim/app/api/tools/textract/analyze-expense/route.ts @@ -0,0 +1,217 @@ +import { + AnalyzeExpenseCommand, + type ExpenseDocument, + GetExpenseAnalysisCommand, + StartExpenseAnalysisCommand, + TextractClient, +} from '@aws-sdk/client-textract' +import { createLogger } from '@sim/logger' +import type { NextRequest } from 'next/server' +import { NextResponse } from 'next/server' +import { textractAnalyzeExpenseContract } from '@/lib/api/contracts/tools/media/document-parse' +import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' +import { checkInternalAuth } from '@/lib/auth/hybrid' +import { generateRequestId } from '@/lib/core/utils/request' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { + mapTextractSdkError, + parseS3Uri, + pollTextractJob, + resolveDocumentInput, + textractErrorResponse, +} from '@/app/api/tools/textract/shared' + +export const dynamic = 'force-dynamic' +/** Mirrors maxDuration in ../parse/route.ts — see that file's TSDoc for details. */ +export const maxDuration = 5400 + +const logger = createLogger('TextractAnalyzeExpenseAPI') + +/** Response shape shared by AnalyzeExpense and its async Get* counterpart. */ +interface TextractExpenseResult { + JobStatus?: string + StatusMessage?: string + NextToken?: string + ExpenseDocuments?: ExpenseDocument[] + DocumentMetadata?: { Pages?: number } + AnalyzeExpenseModelVersion?: string +} + +export function normalizeExpenseField(field: { + Type?: { Text?: string; Confidence?: number } + ValueDetection?: { Text?: string; Confidence?: number } + LabelDetection?: { Text?: string; Confidence?: number } + PageNumber?: number + Currency?: { Code?: string; Confidence?: number } + GroupProperties?: { Id?: string; Types?: string[] }[] +}) { + return { + type: { text: field.Type?.Text, confidence: field.Type?.Confidence }, + valueDetection: { + text: field.ValueDetection?.Text, + confidence: field.ValueDetection?.Confidence, + }, + labelDetection: field.LabelDetection + ? { text: field.LabelDetection.Text, confidence: field.LabelDetection.Confidence } + : undefined, + pageNumber: field.PageNumber, + currency: field.Currency + ? { code: field.Currency.Code, confidence: field.Currency.Confidence } + : undefined, + groupProperties: field.GroupProperties?.map((group) => ({ + id: group.Id ?? '', + types: group.Types ?? [], + })), + } +} + +export function normalizeExpenseDocuments(documents: ExpenseDocument[]) { + return documents.map((doc) => ({ + expenseIndex: doc.ExpenseIndex, + summaryFields: (doc.SummaryFields ?? []).map(normalizeExpenseField), + lineItemGroups: (doc.LineItemGroups ?? []).map((group) => ({ + lineItemGroupIndex: group.LineItemGroupIndex, + lineItems: (group.LineItems ?? []).map((item) => ({ + lineItemExpenseFields: (item.LineItemExpenseFields ?? []).map(normalizeExpenseField), + })), + })), + })) +} + +export const POST = withRouteHandler(async (request: NextRequest) => { + const requestId = generateRequestId() + + try { + const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) + if (!authResult.success || !authResult.userId) { + logger.warn(`[${requestId}] Unauthorized Textract analyze-expense attempt`, { + error: authResult.error || 'Missing userId', + }) + return NextResponse.json( + { success: false, error: authResult.error || 'Unauthorized' }, + { status: 401 } + ) + } + const userId = authResult.userId + + const parsed = await parseRequest( + textractAnalyzeExpenseContract, + request, + {}, + { + validationErrorResponse: (error) => { + logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) + return NextResponse.json( + { + success: false, + error: getValidationErrorMessage(error, 'Invalid request data'), + details: error.issues, + }, + { status: 400 } + ) + }, + } + ) + if (!parsed.success) return parsed.response + + const validatedData = parsed.data.body + const processingMode = validatedData.processingMode || 'sync' + + logger.info(`[${requestId}] Textract analyze-expense request`, { + processingMode, + hasFile: Boolean(validatedData.file), + hasS3Uri: Boolean(validatedData.s3Uri), + userId, + }) + + const client = new TextractClient({ + region: validatedData.region, + credentials: { + accessKeyId: validatedData.accessKeyId, + secretAccessKey: validatedData.secretAccessKey, + }, + }) + + if (processingMode === 'async') { + if (!validatedData.s3Uri) { + return NextResponse.json( + { + success: false, + error: 'S3 URI is required for multi-page processing (s3://bucket/key)', + }, + { status: 400 } + ) + } + + const { bucket, key } = parseS3Uri(validatedData.s3Uri) + logger.info(`[${requestId}] Starting async Textract expense analysis job`, { + s3Bucket: bucket, + s3Key: key, + }) + + const { JobId: jobId } = await client.send( + new StartExpenseAnalysisCommand({ + DocumentLocation: { S3Object: { Bucket: bucket, Name: key } }, + }) + ) + if (!jobId) { + throw new Error('Failed to start Textract expense analysis job: No JobId returned') + } + logger.info(`[${requestId}] Async expense analysis job started`, { jobId }) + + const result = await pollTextractJob( + requestId, + logger, + (nextToken) => + client.send(new GetExpenseAnalysisCommand({ JobId: jobId, NextToken: nextToken })), + (accumulated, page) => ({ + ...page, + ExpenseDocuments: [ + ...(accumulated.ExpenseDocuments ?? []), + ...(page.ExpenseDocuments ?? []), + ], + }) + ) + + return NextResponse.json({ + success: true, + output: { + expenseDocuments: normalizeExpenseDocuments(result.ExpenseDocuments ?? []), + documentMetadata: { pages: result.DocumentMetadata?.Pages ?? 0 }, + modelVersion: result.AnalyzeExpenseModelVersion, + }, + }) + } + + const resolved = await resolveDocumentInput( + { file: validatedData.file, filePath: validatedData.filePath }, + userId, + requestId, + logger + ) + if (!resolved.ok) return resolved.response + const { bytes, isPdf } = resolved.document + + let result: TextractExpenseResult + try { + result = await client.send(new AnalyzeExpenseCommand({ Document: { Bytes: bytes } })) + } catch (error) { + throw mapTextractSdkError(error, isPdf) + } + + logger.info(`[${requestId}] Textract analyze-expense successful`, { + pageCount: result.DocumentMetadata?.Pages ?? 0, + expenseDocumentCount: result.ExpenseDocuments?.length ?? 0, + }) + + return NextResponse.json({ + success: true, + output: { + expenseDocuments: normalizeExpenseDocuments(result.ExpenseDocuments ?? []), + documentMetadata: { pages: result.DocumentMetadata?.Pages ?? 0 }, + }, + }) + } catch (error) { + return textractErrorResponse(error, requestId, logger) + } +}) diff --git a/apps/sim/app/api/tools/textract/analyze-id/route.test.ts b/apps/sim/app/api/tools/textract/analyze-id/route.test.ts new file mode 100644 index 00000000000..5798f2441fd --- /dev/null +++ b/apps/sim/app/api/tools/textract/analyze-id/route.test.ts @@ -0,0 +1,63 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { normalizeIdentityDocuments } from '@/app/api/tools/textract/analyze-id/route' + +describe('normalizeIdentityDocuments', () => { + it('maps a documented AWS AnalyzeID response shape', () => { + const result = normalizeIdentityDocuments([ + { + DocumentIndex: 1, + IdentityDocumentFields: [ + { + Type: { Text: 'FIRST_NAME', Confidence: 99 }, + ValueDetection: { Text: 'Jane', Confidence: 98 }, + }, + { + Type: { + Text: 'DATE_OF_BIRTH', + Confidence: 97, + NormalizedValue: { Value: '1990-01-01', ValueType: 'Date' }, + }, + ValueDetection: { + Text: '01/01/1990', + Confidence: 96, + NormalizedValue: { Value: '1990-01-01T00:00:00', ValueType: 'Date' }, + }, + }, + ], + }, + ]) + + expect(result).toEqual([ + { + documentIndex: 1, + identityDocumentFields: [ + { + type: { text: 'FIRST_NAME', confidence: 99, normalizedValue: undefined }, + valueDetection: { text: 'Jane', confidence: 98, normalizedValue: undefined }, + }, + { + type: { + text: 'DATE_OF_BIRTH', + confidence: 97, + normalizedValue: { value: '1990-01-01', valueType: 'Date' }, + }, + valueDetection: { + text: '01/01/1990', + confidence: 96, + normalizedValue: { value: '1990-01-01T00:00:00', valueType: 'Date' }, + }, + }, + ], + }, + ]) + }) + + it('defaults missing fields to an empty array', () => { + expect(normalizeIdentityDocuments([{ DocumentIndex: 0 }])).toEqual([ + { documentIndex: 0, identityDocumentFields: [] }, + ]) + }) +}) diff --git a/apps/sim/app/api/tools/textract/analyze-id/route.ts b/apps/sim/app/api/tools/textract/analyze-id/route.ts new file mode 100644 index 00000000000..9edf2ce97ed --- /dev/null +++ b/apps/sim/app/api/tools/textract/analyze-id/route.ts @@ -0,0 +1,150 @@ +import { AnalyzeIDCommand, type IdentityDocument, TextractClient } from '@aws-sdk/client-textract' +import { createLogger } from '@sim/logger' +import type { NextRequest } from 'next/server' +import { NextResponse } from 'next/server' +import { textractAnalyzeIdContract } from '@/lib/api/contracts/tools/media/document-parse' +import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' +import { checkInternalAuth } from '@/lib/auth/hybrid' +import { generateRequestId } from '@/lib/core/utils/request' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { + mapTextractSdkError, + resolveDocumentInput, + textractErrorResponse, +} from '@/app/api/tools/textract/shared' + +export const dynamic = 'force-dynamic' + +const logger = createLogger('TextractAnalyzeIdAPI') + +export function normalizeIdentityDocuments(documents: IdentityDocument[]) { + return documents.map((doc) => ({ + documentIndex: doc.DocumentIndex, + identityDocumentFields: (doc.IdentityDocumentFields ?? []).map((field) => ({ + type: { + text: field.Type?.Text, + confidence: field.Type?.Confidence, + normalizedValue: field.Type?.NormalizedValue + ? { + value: field.Type.NormalizedValue.Value, + valueType: field.Type.NormalizedValue.ValueType, + } + : undefined, + }, + valueDetection: { + text: field.ValueDetection?.Text, + confidence: field.ValueDetection?.Confidence, + normalizedValue: field.ValueDetection?.NormalizedValue + ? { + value: field.ValueDetection.NormalizedValue.Value, + valueType: field.ValueDetection.NormalizedValue.ValueType, + } + : undefined, + }, + })), + })) +} + +export const POST = withRouteHandler(async (request: NextRequest) => { + const requestId = generateRequestId() + + try { + const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) + if (!authResult.success || !authResult.userId) { + logger.warn(`[${requestId}] Unauthorized Textract analyze-id attempt`, { + error: authResult.error || 'Missing userId', + }) + return NextResponse.json( + { success: false, error: authResult.error || 'Unauthorized' }, + { status: 401 } + ) + } + const userId = authResult.userId + + const parsed = await parseRequest( + textractAnalyzeIdContract, + request, + {}, + { + validationErrorResponse: (error) => { + logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) + return NextResponse.json( + { + success: false, + error: getValidationErrorMessage(error, 'Invalid request data'), + details: error.issues, + }, + { status: 400 } + ) + }, + } + ) + if (!parsed.success) return parsed.response + + const validatedData = parsed.data.body + + logger.info(`[${requestId}] Textract analyze-id request`, { + hasFile: Boolean(validatedData.file), + hasBackFile: Boolean(validatedData.fileBack || validatedData.filePathBack), + userId, + }) + + const front = await resolveDocumentInput( + { file: validatedData.file, filePath: validatedData.filePath }, + userId, + requestId, + logger + ) + if (!front.ok) return front.response + + const documentPages = [{ Bytes: front.document.bytes }] + let isPdf = front.document.isPdf + + if (validatedData.fileBack || validatedData.filePathBack) { + const back = await resolveDocumentInput( + { file: validatedData.fileBack, filePath: validatedData.filePathBack }, + userId, + requestId, + logger + ) + if (!back.ok) return back.response + documentPages.push({ Bytes: back.document.bytes }) + isPdf = isPdf || back.document.isPdf + } + + const client = new TextractClient({ + region: validatedData.region, + credentials: { + accessKeyId: validatedData.accessKeyId, + secretAccessKey: validatedData.secretAccessKey, + }, + }) + + let result: { + AnalyzeIDModelVersion?: string + DocumentMetadata?: { Pages?: number } + IdentityDocuments?: IdentityDocument[] + } + try { + result = await client.send(new AnalyzeIDCommand({ DocumentPages: documentPages })) + } catch (error) { + throw mapTextractSdkError(error, isPdf, { hasAsyncMode: false }) + } + + logger.info(`[${requestId}] Textract analyze-id successful`, { + pageCount: result.DocumentMetadata?.Pages ?? 0, + documentCount: result.IdentityDocuments?.length ?? 0, + }) + + return NextResponse.json({ + success: true, + output: { + identityDocuments: normalizeIdentityDocuments(result.IdentityDocuments ?? []), + documentMetadata: { pages: result.DocumentMetadata?.Pages ?? 0 }, + modelVersion: result.AnalyzeIDModelVersion, + }, + }) + } catch (error) { + return textractErrorResponse(error, requestId, logger) + } +}) diff --git a/apps/sim/app/api/tools/textract/parse/route.ts b/apps/sim/app/api/tools/textract/parse/route.ts index 82eb65ff830..750aba8acc8 100644 --- a/apps/sim/app/api/tools/textract/parse/route.ts +++ b/apps/sim/app/api/tools/textract/parse/route.ts @@ -1,26 +1,28 @@ -import crypto from 'crypto' +import { + AnalyzeDocumentCommand, + DetectDocumentTextCommand, + type FeatureType, + GetDocumentAnalysisCommand, + GetDocumentTextDetectionCommand, + StartDocumentAnalysisCommand, + StartDocumentTextDetectionCommand, + TextractClient, +} from '@aws-sdk/client-textract' import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { sleep } from '@sim/utils/helpers' -import { type NextRequest, NextResponse } from 'next/server' +import type { NextRequest } from 'next/server' +import { NextResponse } from 'next/server' import { textractParseContract } from '@/lib/api/contracts/tools/media/document-parse' import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' -import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' -import { validateS3BucketName } from '@/lib/core/security/input-validation' -import { - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { isInternalFileUrl, processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' import { - downloadServableFileFromStorage, - resolveInternalFileUrl, -} from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' + mapTextractSdkError, + parseS3Uri, + pollTextractJob, + resolveDocumentInput, + textractErrorResponse, +} from '@/app/api/tools/textract/shared' export const dynamic = 'force-dynamic' /** @@ -32,244 +34,15 @@ export const maxDuration = 5400 const logger = createLogger('TextractParseAPI') -function getSignatureKey( - key: string, - dateStamp: string, - regionName: string, - serviceName: string -): Buffer { - const kDate = crypto.createHmac('sha256', `AWS4${key}`).update(dateStamp).digest() - const kRegion = crypto.createHmac('sha256', kDate).update(regionName).digest() - const kService = crypto.createHmac('sha256', kRegion).update(serviceName).digest() - const kSigning = crypto.createHmac('sha256', kService).update('aws4_request').digest() - return kSigning -} - -function signAwsRequest( - method: string, - host: string, - uri: string, - body: string, - accessKeyId: string, - secretAccessKey: string, - region: string, - service: string, - amzTarget: string -): Record { - const date = new Date() - const amzDate = date.toISOString().replace(/[:-]|\.\d{3}/g, '') - const dateStamp = amzDate.slice(0, 8) - - const payloadHash = crypto.createHash('sha256').update(body).digest('hex') - - const canonicalHeaders = - `content-type:application/x-amz-json-1.1\n` + - `host:${host}\n` + - `x-amz-date:${amzDate}\n` + - `x-amz-target:${amzTarget}\n` - - const signedHeaders = 'content-type;host;x-amz-date;x-amz-target' - - const canonicalRequest = `${method}\n${uri}\n\n${canonicalHeaders}\n${signedHeaders}\n${payloadHash}` - - const algorithm = 'AWS4-HMAC-SHA256' - const credentialScope = `${dateStamp}/${region}/${service}/aws4_request` - const stringToSign = `${algorithm}\n${amzDate}\n${credentialScope}\n${crypto.createHash('sha256').update(canonicalRequest).digest('hex')}` - - const signingKey = getSignatureKey(secretAccessKey, dateStamp, region, service) - const signature = crypto.createHmac('sha256', signingKey).update(stringToSign).digest('hex') - - const authorizationHeader = `${algorithm} Credential=${accessKeyId}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}` - - return { - 'Content-Type': 'application/x-amz-json-1.1', - Host: host, - 'X-Amz-Date': amzDate, - 'X-Amz-Target': amzTarget, - Authorization: authorizationHeader, - } -} - -async function fetchDocumentBytes(url: string): Promise<{ bytes: string; contentType: string }> { - const urlValidation = await validateUrlWithDNS(url, 'Document URL') - if (!urlValidation.isValid) { - throw new Error(urlValidation.error || 'Invalid document URL') - } - - const response = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP!, { - method: 'GET', - }) - if (!response.ok) { - await response.text().catch(() => {}) - throw new Error(`Failed to fetch document: ${response.statusText}`) - } - - const arrayBuffer = await response.arrayBuffer() - const bytes = Buffer.from(arrayBuffer).toString('base64') - const contentType = response.headers.get('content-type') || 'application/octet-stream' - - return { bytes, contentType } -} - -function parseS3Uri(s3Uri: string): { bucket: string; key: string } { - const match = s3Uri.match(/^s3:\/\/([^/]+)\/(.+)$/) - if (!match) { - throw new Error( - `Invalid S3 URI format: ${s3Uri}. Expected format: s3://bucket-name/path/to/object` - ) - } - - const bucket = match[1] - const key = match[2] - - const bucketValidation = validateS3BucketName(bucket, 'S3 bucket name') - if (!bucketValidation.isValid) { - throw new Error(bucketValidation.error) - } - - if (key.includes('..') || key.startsWith('/')) { - throw new Error('S3 key contains invalid path traversal sequences') - } - - return { bucket, key } -} - -async function callTextractAsync( - host: string, - amzTarget: string, - body: Record, - accessKeyId: string, - secretAccessKey: string, - region: string -): Promise> { - const bodyString = JSON.stringify(body) - const headers = signAwsRequest( - 'POST', - host, - '/', - bodyString, - accessKeyId, - secretAccessKey, - region, - 'textract', - amzTarget - ) - - const response = await fetch(`https://${host}/`, { - method: 'POST', - headers, - body: bodyString, - }) - - if (!response.ok) { - const errorText = await response.text() - let errorMessage = `Textract API error: ${response.statusText}` - try { - const errorJson = JSON.parse(errorText) - if (errorJson.Message) { - errorMessage = errorJson.Message - } else if (errorJson.__type) { - errorMessage = `${errorJson.__type}: ${errorJson.message || errorText}` - } - } catch { - // Use default error message - } - throw new Error(errorMessage) - } - - return response.json() -} - -async function pollForJobCompletion( - host: string, - jobId: string, - accessKeyId: string, - secretAccessKey: string, - region: string, - useAnalyzeDocument: boolean, - requestId: string -): Promise> { - const pollIntervalMs = 5000 - const maxPollTimeMs = getMaxExecutionTimeout() - const maxAttempts = Math.ceil(maxPollTimeMs / pollIntervalMs) - - const getTarget = useAnalyzeDocument - ? 'Textract.GetDocumentAnalysis' - : 'Textract.GetDocumentTextDetection' - - for (let attempt = 0; attempt < maxAttempts; attempt++) { - const result = await callTextractAsync( - host, - getTarget, - { JobId: jobId }, - accessKeyId, - secretAccessKey, - region - ) - - const jobStatus = result.JobStatus as string - - if (jobStatus === 'SUCCEEDED') { - logger.info(`[${requestId}] Async job completed successfully after ${attempt + 1} polls`) - - let allBlocks = (result.Blocks as unknown[]) || [] - let nextToken = result.NextToken as string | undefined - - while (nextToken) { - const nextResult = await callTextractAsync( - host, - getTarget, - { JobId: jobId, NextToken: nextToken }, - accessKeyId, - secretAccessKey, - region - ) - allBlocks = allBlocks.concat((nextResult.Blocks as unknown[]) || []) - nextToken = nextResult.NextToken as string | undefined - } - - return { - ...result, - Blocks: allBlocks, - } - } - - if (jobStatus === 'FAILED') { - throw new Error(`Textract job failed: ${result.StatusMessage || 'Unknown error'}`) - } - - if (jobStatus === 'PARTIAL_SUCCESS') { - logger.warn(`[${requestId}] Job completed with partial success: ${result.StatusMessage}`) - - let allBlocks = (result.Blocks as unknown[]) || [] - let nextToken = result.NextToken as string | undefined - - while (nextToken) { - const nextResult = await callTextractAsync( - host, - getTarget, - { JobId: jobId, NextToken: nextToken }, - accessKeyId, - secretAccessKey, - region - ) - allBlocks = allBlocks.concat((nextResult.Blocks as unknown[]) || []) - nextToken = nextResult.NextToken as string | undefined - } - - return { - ...result, - Blocks: allBlocks, - } - } - - logger.info(`[${requestId}] Job status: ${jobStatus}, attempt ${attempt + 1}/${maxAttempts}`) - await sleep(pollIntervalMs) - } - - throw new Error( - `Timeout waiting for Textract job to complete (max ${maxPollTimeMs / 1000} seconds)` - ) +/** Response shape shared by AnalyzeDocument/DetectDocumentText and their async Get* counterparts. */ +interface TextractDocumentResult { + JobStatus?: string + StatusMessage?: string + NextToken?: string + Blocks?: unknown[] + DocumentMetadata?: { Pages?: number } + AnalyzeDocumentModelVersion?: string + DetectDocumentTextModelVersion?: string } export const POST = withRouteHandler(async (request: NextRequest) => { @@ -277,20 +50,15 @@ export const POST = withRouteHandler(async (request: NextRequest) => { try { const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { logger.warn(`[${requestId}] Unauthorized Textract parse attempt`, { error: authResult.error || 'Missing userId', }) return NextResponse.json( - { - success: false, - error: authResult.error || 'Unauthorized', - }, + { success: false, error: authResult.error || 'Unauthorized' }, { status: 401 } ) } - const userId = authResult.userId const parsed = await parseRequest( @@ -314,11 +82,19 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (!parsed.success) return parsed.response const validatedData = parsed.data.body - const processingMode = validatedData.processingMode || 'sync' - const featureTypes = validatedData.featureTypes ?? [] + const featureTypes = (validatedData.featureTypes ?? []) as FeatureType[] const useAnalyzeDocument = featureTypes.length > 0 - const host = `textract.${validatedData.region}.amazonaws.com` + const queriesConfig = + validatedData.queries && validatedData.queries.length > 0 && featureTypes.includes('QUERIES') + ? { + Queries: validatedData.queries.map((q) => ({ + Text: q.Text, + Alias: q.Alias, + Pages: q.Pages, + })), + } + : undefined logger.info(`[${requestId}] Textract parse request`, { processingMode, @@ -328,6 +104,14 @@ export const POST = withRouteHandler(async (request: NextRequest) => { userId, }) + const client = new TextractClient({ + region: validatedData.region, + credentials: { + accessKeyId: validatedData.accessKeyId, + secretAccessKey: validatedData.secretAccessKey, + }, + }) + if (processingMode === 'async') { if (!validatedData.s3Uri) { return NextResponse.json( @@ -339,299 +123,97 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const { bucket: s3Bucket, key: s3Key } = parseS3Uri(validatedData.s3Uri) - - logger.info(`[${requestId}] Starting async Textract job`, { s3Bucket, s3Key }) - - const startTarget = useAnalyzeDocument - ? 'Textract.StartDocumentAnalysis' - : 'Textract.StartDocumentTextDetection' - - const startBody: Record = { - DocumentLocation: { - S3Object: { - Bucket: s3Bucket, - Name: s3Key, - }, - }, - } - - if (useAnalyzeDocument) { - startBody.FeatureTypes = featureTypes + const { bucket, key } = parseS3Uri(validatedData.s3Uri) + logger.info(`[${requestId}] Starting async Textract job`, { s3Bucket: bucket, s3Key: key }) - if ( - validatedData.queries && - validatedData.queries.length > 0 && - featureTypes.includes('QUERIES') - ) { - startBody.QueriesConfig = { - Queries: validatedData.queries.map((q) => ({ - Text: q.Text, - Alias: q.Alias, - Pages: q.Pages, - })), - } - } - } - - const startResult = await callTextractAsync( - host, - startTarget, - startBody, - validatedData.accessKeyId, - validatedData.secretAccessKey, - validatedData.region - ) - - const jobId = startResult.JobId as string + const { JobId: jobId } = useAnalyzeDocument + ? await client.send( + new StartDocumentAnalysisCommand({ + DocumentLocation: { S3Object: { Bucket: bucket, Name: key } }, + FeatureTypes: featureTypes, + QueriesConfig: queriesConfig, + }) + ) + : await client.send( + new StartDocumentTextDetectionCommand({ + DocumentLocation: { S3Object: { Bucket: bucket, Name: key } }, + }) + ) if (!jobId) { throw new Error('Failed to start Textract job: No JobId returned') } - logger.info(`[${requestId}] Async job started`, { jobId }) - const textractData = await pollForJobCompletion( - host, - jobId, - validatedData.accessKeyId, - validatedData.secretAccessKey, - validatedData.region, - useAnalyzeDocument, - requestId + const result = await pollTextractJob( + requestId, + logger, + async (nextToken) => + useAnalyzeDocument + ? await client.send( + new GetDocumentAnalysisCommand({ JobId: jobId, NextToken: nextToken }) + ) + : await client.send( + new GetDocumentTextDetectionCommand({ JobId: jobId, NextToken: nextToken }) + ), + (accumulated, page) => ({ + ...page, + Blocks: [...(accumulated.Blocks ?? []), ...(page.Blocks ?? [])], + }) ) logger.info(`[${requestId}] Textract async parse successful`, { - pageCount: (textractData.DocumentMetadata as { Pages?: number })?.Pages ?? 0, - blockCount: (textractData.Blocks as unknown[])?.length ?? 0, + pageCount: result.DocumentMetadata?.Pages ?? 0, + blockCount: result.Blocks?.length ?? 0, }) return NextResponse.json({ success: true, output: { - blocks: textractData.Blocks ?? [], - documentMetadata: { - pages: (textractData.DocumentMetadata as { Pages?: number })?.Pages ?? 0, - }, - modelVersion: (textractData.AnalyzeDocumentModelVersion ?? - textractData.DetectDocumentTextModelVersion) as string | undefined, + blocks: result.Blocks ?? [], + documentMetadata: { pages: result.DocumentMetadata?.Pages ?? 0 }, + modelVersion: result.AnalyzeDocumentModelVersion ?? result.DetectDocumentTextModelVersion, }, }) } - let bytes = '' - let contentType = 'application/octet-stream' - let isPdf = false - - if (validatedData.file) { - let userFile - try { - userFile = processSingleFileToUserFile(validatedData.file, requestId, logger) - } catch (error) { - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Failed to process file'), - }, - { status: 400 } - ) - } - - const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger) - if (denied) return denied - const { buffer, contentType: resolvedContentType } = await downloadServableFileFromStorage( - userFile, - requestId, - logger - ) - bytes = buffer.toString('base64') - contentType = resolvedContentType || userFile.type || 'application/octet-stream' - isPdf = contentType.includes('pdf') || userFile.name?.toLowerCase().endsWith('.pdf') - } else if (validatedData.filePath) { - let fileUrl = validatedData.filePath - - const isInternalFilePath = isInternalFileUrl(fileUrl) - - if (isInternalFilePath) { - const resolution = await resolveInternalFileUrl(fileUrl, userId, requestId, logger) - if (resolution.error) { - return NextResponse.json( - { - success: false, - error: resolution.error.message, - }, - { status: resolution.error.status } - ) - } - fileUrl = resolution.fileUrl || fileUrl - } else if (fileUrl.startsWith('/')) { - logger.warn(`[${requestId}] Invalid internal path`, { - userId, - path: fileUrl.substring(0, 50), - }) - return NextResponse.json( - { - success: false, - error: 'Invalid file path. Only uploaded files are supported for internal paths.', - }, - { status: 400 } - ) - } else { - const urlValidation = await validateUrlWithDNS(fileUrl, 'Document URL') - if (!urlValidation.isValid) { - logger.warn(`[${requestId}] SSRF attempt blocked`, { - userId, - url: fileUrl.substring(0, 100), - error: urlValidation.error, - }) - return NextResponse.json( - { - success: false, - error: urlValidation.error, - }, - { status: 400 } - ) - } - } - - const fetched = await fetchDocumentBytes(fileUrl) - bytes = fetched.bytes - contentType = fetched.contentType - isPdf = contentType.includes('pdf') || fileUrl.toLowerCase().endsWith('.pdf') - } else { - return NextResponse.json( - { - success: false, - error: 'File input is required for single-page processing', - }, - { status: 400 } - ) - } - - const uri = '/' - - let textractBody: Record - let amzTarget: string - - if (useAnalyzeDocument) { - amzTarget = 'Textract.AnalyzeDocument' - textractBody = { - Document: { - Bytes: bytes, - }, - FeatureTypes: featureTypes, - } - - if ( - validatedData.queries && - validatedData.queries.length > 0 && - featureTypes.includes('QUERIES') - ) { - textractBody.QueriesConfig = { - Queries: validatedData.queries.map((q) => ({ - Text: q.Text, - Alias: q.Alias, - Pages: q.Pages, - })), - } - } - } else { - amzTarget = 'Textract.DetectDocumentText' - textractBody = { - Document: { - Bytes: bytes, - }, - } - } - - const bodyString = JSON.stringify(textractBody) - - const headers = signAwsRequest( - 'POST', - host, - uri, - bodyString, - validatedData.accessKeyId, - validatedData.secretAccessKey, - validatedData.region, - 'textract', - amzTarget + const resolved = await resolveDocumentInput( + { file: validatedData.file, filePath: validatedData.filePath }, + userId, + requestId, + logger ) + if (!resolved.ok) return resolved.response + const { bytes, isPdf } = resolved.document - const textractResponse = await fetch(`https://${host}${uri}`, { - method: 'POST', - headers, - body: bodyString, - }) - - if (!textractResponse.ok) { - const errorText = await textractResponse.text() - logger.error(`[${requestId}] Textract API error:`, errorText) - - let errorMessage = `Textract API error: ${textractResponse.statusText}` - let isUnsupportedFormat = false - try { - const errorJson = JSON.parse(errorText) - if (errorJson.Message) { - errorMessage = errorJson.Message - } else if (errorJson.__type) { - errorMessage = `${errorJson.__type}: ${errorJson.message || errorText}` - } - // Check for unsupported document format error - isUnsupportedFormat = - errorJson.__type === 'UnsupportedDocumentException' || - errorJson.Message?.toLowerCase().includes('unsupported document') || - errorText.toLowerCase().includes('unsupported document') - } catch { - isUnsupportedFormat = errorText.toLowerCase().includes('unsupported document') - } - - // Provide helpful message for unsupported format (likely multi-page PDF) - if (isUnsupportedFormat && isPdf) { - errorMessage = - 'This document format is not supported in Single Page mode. If this is a multi-page PDF, please use "Multi-Page (PDF, TIFF via S3)" mode instead, which requires uploading your document to S3 first. Single Page mode only supports JPEG, PNG, and single-page PDF files.' - } - - return NextResponse.json( - { - success: false, - error: errorMessage, - }, - { status: textractResponse.status } - ) + let result: TextractDocumentResult + try { + result = useAnalyzeDocument + ? await client.send( + new AnalyzeDocumentCommand({ + Document: { Bytes: bytes }, + FeatureTypes: featureTypes, + QueriesConfig: queriesConfig, + }) + ) + : await client.send(new DetectDocumentTextCommand({ Document: { Bytes: bytes } })) + } catch (error) { + throw mapTextractSdkError(error, isPdf) } - const textractData = await textractResponse.json() - logger.info(`[${requestId}] Textract parse successful`, { - pageCount: textractData.DocumentMetadata?.Pages ?? 0, - blockCount: textractData.Blocks?.length ?? 0, + pageCount: result.DocumentMetadata?.Pages ?? 0, + blockCount: result.Blocks?.length ?? 0, }) return NextResponse.json({ success: true, output: { - blocks: textractData.Blocks ?? [], - documentMetadata: { - pages: textractData.DocumentMetadata?.Pages ?? 0, - }, - modelVersion: - textractData.AnalyzeDocumentModelVersion ?? - textractData.DetectDocumentTextModelVersion ?? - undefined, + blocks: result.Blocks ?? [], + documentMetadata: { pages: result.DocumentMetadata?.Pages ?? 0 }, + modelVersion: result.AnalyzeDocumentModelVersion ?? result.DetectDocumentTextModelVersion, }, }) } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - - logger.error(`[${requestId}] Error in Textract parse:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) + return textractErrorResponse(error, requestId, logger) } }) diff --git a/apps/sim/app/api/tools/textract/shared.test.ts b/apps/sim/app/api/tools/textract/shared.test.ts new file mode 100644 index 00000000000..3c503b08520 --- /dev/null +++ b/apps/sim/app/api/tools/textract/shared.test.ts @@ -0,0 +1,133 @@ +/** + * @vitest-environment node + */ +import { createLogger } from '@sim/logger' +import { describe, expect, it } from 'vitest' +import { + mapTextractSdkError, + parseS3Uri, + pollTextractJob, + TextractRouteError, +} from '@/app/api/tools/textract/shared' + +const logger = createLogger('TextractSharedTest') + +describe('parseS3Uri', () => { + it('parses a valid s3 URI', () => { + expect(parseS3Uri('s3://my-bucket/path/to/doc.pdf')).toEqual({ + bucket: 'my-bucket', + key: 'path/to/doc.pdf', + }) + }) + + it('rejects a malformed URI', () => { + expect(() => parseS3Uri('not-an-s3-uri')).toThrow(TextractRouteError) + }) + + it('rejects path traversal in the key', () => { + expect(() => parseS3Uri('s3://my-bucket/../secrets.pdf')).toThrow('path traversal') + }) +}) + +describe('mapTextractSdkError', () => { + it('gives a friendly hint for unsupported PDFs in single-page mode', () => { + const mapped = mapTextractSdkError( + { name: 'UnsupportedDocumentException', message: 'Unsupported document' }, + true + ) + expect(mapped.status).toBe(400) + expect(mapped.message).toContain('Multi-Page (PDF, TIFF via S3)') + }) + + it('omits the multi-page hint for operations without an async mode', () => { + const mapped = mapTextractSdkError( + { name: 'UnsupportedDocumentException', message: 'Unsupported document' }, + true, + { hasAsyncMode: false } + ) + expect(mapped.message).not.toContain('Multi-Page') + expect(mapped.message).toContain('Only JPEG, PNG, and single-page PDF files are supported') + }) + + it('does not rewrite the message for non-PDF unsupported documents', () => { + const mapped = mapTextractSdkError( + { name: 'UnsupportedDocumentException', message: 'Unsupported document' }, + false + ) + expect(mapped.message).toBe('Unsupported document') + }) + + it('uses the SDK http status when under 500', () => { + const mapped = mapTextractSdkError( + { + name: 'InvalidParameterException', + message: 'Bad param', + $metadata: { httpStatusCode: 400 }, + }, + false + ) + expect(mapped.status).toBe(400) + expect(mapped.message).toBe('Bad param') + }) + + it('passes through a 5xx SDK status so tool-execution retry logic still fires', () => { + const mapped = mapTextractSdkError( + { message: 'Internal failure', $metadata: { httpStatusCode: 500 } }, + false + ) + expect(mapped.status).toBe(500) + }) + + it('defaults to 400 when the SDK gives no http status', () => { + const mapped = mapTextractSdkError({ message: 'Unknown failure' }, false) + expect(mapped.status).toBe(400) + }) +}) + +describe('pollTextractJob', () => { + it('returns immediately on SUCCEEDED with no NextToken', async () => { + const result = await pollTextractJob( + 'req-1', + logger, + async () => ({ JobStatus: 'SUCCEEDED', Blocks: [{ Id: '1' }] }), + (accumulated, page) => ({ + ...page, + Blocks: [...(accumulated.Blocks ?? []), ...(page.Blocks ?? [])], + }) + ) + + expect(result.JobStatus).toBe('SUCCEEDED') + expect(result.Blocks).toHaveLength(1) + }) + + it('follows NextToken pagination and merges pages', async () => { + let calls = 0 + const result = await pollTextractJob( + 'req-2', + logger, + async (nextToken) => { + calls += 1 + if (!nextToken) return { JobStatus: 'SUCCEEDED', Blocks: [{ Id: '1' }], NextToken: 'next' } + return { JobStatus: 'SUCCEEDED', Blocks: [{ Id: '2' }] } + }, + (accumulated, page) => ({ + ...page, + Blocks: [...(accumulated.Blocks ?? []), ...(page.Blocks ?? [])], + }) + ) + + expect(calls).toBe(2) + expect(result.Blocks).toHaveLength(2) + }) + + it('throws a TextractRouteError when the job fails', async () => { + await expect( + pollTextractJob( + 'req-3', + logger, + async () => ({ JobStatus: 'FAILED', StatusMessage: 'boom' }), + (accumulated) => accumulated + ) + ).rejects.toThrow('Textract job failed: boom') + }) +}) diff --git a/apps/sim/app/api/tools/textract/shared.ts b/apps/sim/app/api/tools/textract/shared.ts new file mode 100644 index 00000000000..80363b388d0 --- /dev/null +++ b/apps/sim/app/api/tools/textract/shared.ts @@ -0,0 +1,301 @@ +import type { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { sleep } from '@sim/utils/helpers' +import { NextResponse } from 'next/server' +import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' +import { validateS3BucketName } from '@/lib/core/security/input-validation' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import type { RawFileInput } from '@/lib/uploads/utils/file-utils' +import { isInternalFileUrl, processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' +import { + downloadServableFileFromStorage, + resolveInternalFileUrl, +} from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' +import { assertToolFileAccess } from '@/app/api/files/authorization' + +type RouteLogger = ReturnType + +/** Thrown by AWS SDK call sites so route handlers can map failures to the right HTTP status. */ +export class TextractRouteError extends Error { + status: number + + constructor(message: string, status = 500) { + super(message) + this.name = 'TextractRouteError' + this.status = status + } +} + +export function textractErrorResponse( + error: unknown, + requestId: string, + logger: RouteLogger +): NextResponse { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + + logger.error(`[${requestId}] Error in Textract request:`, error) + const status = error instanceof TextractRouteError ? error.status : 500 + return NextResponse.json( + { success: false, error: getErrorMessage(error, 'Internal server error') }, + { status } + ) +} + +/** + * Maps an AWS SDK TextractClient rejection to a client-facing error, with a friendly hint for the + * common "PDF used in single-page mode" mistake. The real AWS HTTP status (including 5xx) is + * passed through so the tool-execution layer's retry logic can still treat throttling/internal + * errors as retryable, matching the pre-migration hand-rolled-signing behavior. + */ +export function mapTextractSdkError( + error: unknown, + isPdf: boolean, + options?: { hasAsyncMode?: boolean } +): TextractRouteError { + const err = error as { + name?: string + message?: string + $metadata?: { httpStatusCode?: number } + } + const hasAsyncMode = options?.hasAsyncMode ?? true + + const isUnsupportedFormat = + err.name === 'UnsupportedDocumentException' || + Boolean(err.message?.toLowerCase().includes('unsupported document')) + + if (isUnsupportedFormat && isPdf) { + const hint = hasAsyncMode + ? ' If this is a multi-page PDF, please use "Multi-Page (PDF, TIFF via S3)" mode instead, which requires uploading your document to S3 first. Single Page mode only supports JPEG, PNG, and single-page PDF files.' + : ' Only JPEG, PNG, and single-page PDF files are supported.' + return new TextractRouteError(`This document format is not supported.${hint}`, 400) + } + + const status = err.$metadata?.httpStatusCode || 400 + return new TextractRouteError(err.message || 'Textract API error', status) +} + +export interface ResolvedDocument { + bytes: Buffer + contentType: string + isPdf: boolean +} + +export type ResolveDocumentResult = + | { ok: true; document: ResolvedDocument } + | { ok: false; response: NextResponse } + +async function fetchDocumentBytes(url: string): Promise<{ bytes: Buffer; contentType: string }> { + const urlValidation = await validateUrlWithDNS(url, 'Document URL') + if (!urlValidation.isValid) { + throw new TextractRouteError(urlValidation.error || 'Invalid document URL', 400) + } + + const response = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP!, { + method: 'GET', + }) + if (!response.ok) { + await response.text().catch(() => {}) + throw new TextractRouteError(`Failed to fetch document: ${response.statusText}`, 400) + } + + const arrayBuffer = await response.arrayBuffer() + const contentType = response.headers.get('content-type') || 'application/octet-stream' + + return { bytes: Buffer.from(arrayBuffer), contentType } +} + +/** Resolves a document input (uploaded file reference or URL) to raw bytes for the Textract Document.Bytes field. */ +export async function resolveDocumentInput( + input: { file?: RawFileInput; filePath?: string }, + userId: string, + requestId: string, + logger: RouteLogger +): Promise { + if (input.file) { + let userFile: ReturnType + try { + userFile = processSingleFileToUserFile(input.file, requestId, logger) + } catch (error) { + return { + ok: false, + response: NextResponse.json( + { success: false, error: getErrorMessage(error, 'Failed to process file') }, + { status: 400 } + ), + } + } + + const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger) + if (denied) return { ok: false, response: denied } + + const { buffer, contentType } = await downloadServableFileFromStorage( + userFile, + requestId, + logger + ) + const resolvedContentType = contentType || userFile.type || 'application/octet-stream' + + return { + ok: true, + document: { + bytes: buffer, + contentType: resolvedContentType, + isPdf: + resolvedContentType.includes('pdf') || + Boolean(userFile.name?.toLowerCase().endsWith('.pdf')), + }, + } + } + + if (input.filePath) { + let fileUrl = input.filePath + const isInternalFilePath = isInternalFileUrl(fileUrl) + + if (isInternalFilePath) { + const resolution = await resolveInternalFileUrl(fileUrl, userId, requestId, logger) + if (resolution.error) { + return { + ok: false, + response: NextResponse.json( + { success: false, error: resolution.error.message }, + { status: resolution.error.status } + ), + } + } + fileUrl = resolution.fileUrl || fileUrl + } else if (fileUrl.startsWith('/')) { + logger.warn(`[${requestId}] Invalid internal path`, { + userId, + path: fileUrl.substring(0, 50), + }) + return { + ok: false, + response: NextResponse.json( + { + success: false, + error: 'Invalid file path. Only uploaded files are supported for internal paths.', + }, + { status: 400 } + ), + } + } else { + const urlValidation = await validateUrlWithDNS(fileUrl, 'Document URL') + if (!urlValidation.isValid) { + logger.warn(`[${requestId}] SSRF attempt blocked`, { + userId, + url: fileUrl.substring(0, 100), + error: urlValidation.error, + }) + return { + ok: false, + response: NextResponse.json( + { success: false, error: urlValidation.error }, + { status: 400 } + ), + } + } + } + + const fetched = await fetchDocumentBytes(fileUrl) + return { + ok: true, + document: { + bytes: fetched.bytes, + contentType: fetched.contentType, + isPdf: fetched.contentType.includes('pdf') || fileUrl.toLowerCase().endsWith('.pdf'), + }, + } + } + + return { + ok: false, + response: NextResponse.json( + { success: false, error: 'Document input is required' }, + { status: 400 } + ), + } +} + +export function parseS3Uri(s3Uri: string): { bucket: string; key: string } { + const match = s3Uri.match(/^s3:\/\/([^/]+)\/(.+)$/) + if (!match) { + throw new TextractRouteError( + `Invalid S3 URI format: ${s3Uri}. Expected format: s3://bucket-name/path/to/object`, + 400 + ) + } + + const bucket = match[1] + const key = match[2] + + const bucketValidation = validateS3BucketName(bucket, 'S3 bucket name') + if (!bucketValidation.isValid) { + throw new TextractRouteError(bucketValidation.error || 'Invalid S3 bucket name', 400) + } + + if (key.includes('..') || key.startsWith('/')) { + throw new TextractRouteError('S3 key contains invalid path traversal sequences', 400) + } + + return { bucket, key } +} + +interface PollableJobResult { + JobStatus?: string + StatusMessage?: string + NextToken?: string +} + +/** Polls a started async Textract job (StartDocumentAnalysis/StartDocumentTextDetection/StartExpenseAnalysis) until it completes, following NextToken pagination on success. */ +export async function pollTextractJob( + requestId: string, + logger: RouteLogger, + getPage: (nextToken?: string) => Promise, + mergePage: (accumulated: TResult, page: TResult) => TResult +): Promise { + const pollIntervalMs = 5000 + const maxPollTimeMs = getMaxExecutionTimeout() + const maxAttempts = Math.ceil(maxPollTimeMs / pollIntervalMs) + + for (let attempt = 0; attempt < maxAttempts; attempt++) { + const result = await getPage() + const jobStatus = result.JobStatus + + if (jobStatus === 'SUCCEEDED' || jobStatus === 'PARTIAL_SUCCESS') { + if (jobStatus === 'PARTIAL_SUCCESS') { + logger.warn(`[${requestId}] Job completed with partial success: ${result.StatusMessage}`) + } else { + logger.info(`[${requestId}] Async job completed successfully after ${attempt + 1} polls`) + } + + let merged = result + let nextToken = result.NextToken + while (nextToken) { + const page = await getPage(nextToken) + merged = mergePage(merged, page) + nextToken = page.NextToken + } + return merged + } + + if (jobStatus === 'FAILED') { + throw new TextractRouteError( + `Textract job failed: ${result.StatusMessage || 'Unknown error'}`, + 502 + ) + } + + logger.info(`[${requestId}] Job status: ${jobStatus}, attempt ${attempt + 1}/${maxAttempts}`) + await sleep(pollIntervalMs) + } + + throw new TextractRouteError( + `Timeout waiting for Textract job to complete (max ${maxPollTimeMs / 1000} seconds)`, + 504 + ) +} diff --git a/apps/sim/blocks/blocks/textract.ts b/apps/sim/blocks/blocks/textract.ts index 9a6baa033b8..5a6f8a22778 100644 --- a/apps/sim/blocks/blocks/textract.ts +++ b/apps/sim/blocks/blocks/textract.ts @@ -6,7 +6,7 @@ import { IntegrationType, type SubBlockType, } from '@/blocks/types' -import { createVersionedToolSelector, normalizeFileInput } from '@/blocks/utils' +import { normalizeFileInput } from '@/blocks/utils' import type { TextractParserOutput } from '@/tools/textract/types' export const TextractBlock: BlockConfig = { @@ -199,63 +199,176 @@ export const TextractBlock: BlockConfig = { }, } -const textractV2Inputs = TextractBlock.inputs ? { ...TextractBlock.inputs } : {} -const textractV2SubBlocks = (TextractBlock.subBlocks || []).flatMap((subBlock) => { - if (subBlock.id === 'filePath') { - return [] // Remove the old filePath subblock - } - if (subBlock.id === 'fileUpload') { - // Insert fileReference right after fileUpload - return [ - subBlock, - { - id: 'fileReference', - title: 'Document', - type: 'short-input' as SubBlockType, - canonicalParamId: 'document', - placeholder: 'File reference', - condition: { - field: 'processingMode', - value: 'async', - not: true, - }, - mode: 'advanced' as const, - }, - ] - } - return [subBlock] -}) +function requireAwsCredentials(params: Record) { + const accessKeyId = typeof params.accessKeyId === 'string' ? params.accessKeyId.trim() : '' + const secretAccessKey = + typeof params.secretAccessKey === 'string' ? params.secretAccessKey.trim() : '' + const region = typeof params.region === 'string' ? params.region.trim() : '' + + if (!accessKeyId) throw new Error('AWS Access Key ID is required') + if (!secretAccessKey) throw new Error('AWS Secret Access Key is required') + if (!region) throw new Error('AWS Region is required') + + return { accessKeyId, secretAccessKey, region } +} export const TextractV2Block: BlockConfig = { ...TextractBlock, type: 'textract_v2', name: 'AWS Textract', hideFromToolbar: false, - subBlocks: textractV2SubBlocks, + subBlocks: [ + { + id: 'operation', + title: 'Operation', + type: 'dropdown' as SubBlockType, + options: [ + { id: 'analyze_document', label: 'Analyze Document (Text, Tables, Forms)' }, + { id: 'analyze_expense', label: 'Analyze Expense (Invoices & Receipts)' }, + { id: 'analyze_id', label: 'Analyze Identity Document' }, + ], + value: () => 'analyze_document', + }, + { + id: 'processingMode', + title: 'Processing Mode', + type: 'dropdown' as SubBlockType, + options: [ + { id: 'sync', label: 'Single Page (JPEG, PNG, 1-page PDF)' }, + { id: 'async', label: 'Multi-Page (PDF, TIFF via S3)' }, + ], + tooltip: + 'Single Page uses synchronous API for JPEG, PNG, or single-page PDF. Multi-Page uses async API for multi-page PDF/TIFF stored in S3.', + condition: { field: 'operation', value: 'analyze_id', not: true }, + }, + { + id: 'fileUpload', + title: 'Document', + type: 'file-upload' as SubBlockType, + canonicalParamId: 'document', + acceptedTypes: 'image/jpeg,image/png,application/pdf', + placeholder: 'Upload JPEG, PNG, or single-page PDF (max 10MB)', + condition: { field: 'processingMode', value: 'async', not: true }, + mode: 'basic', + maxSize: 10, + }, + { + id: 'fileReference', + title: 'Document', + type: 'short-input' as SubBlockType, + canonicalParamId: 'document', + placeholder: 'File reference', + condition: { field: 'processingMode', value: 'async', not: true }, + mode: 'advanced' as const, + }, + { + id: 'fileUploadBack', + title: 'Back of ID (optional)', + type: 'file-upload' as SubBlockType, + canonicalParamId: 'documentBack', + acceptedTypes: 'image/jpeg,image/png,application/pdf', + placeholder: 'Upload the back of the ID, if it carries data', + condition: { field: 'operation', value: 'analyze_id' }, + mode: 'basic', + maxSize: 10, + }, + { + id: 'fileReferenceBack', + title: 'Back of ID (optional)', + type: 'short-input' as SubBlockType, + canonicalParamId: 'documentBack', + placeholder: 'File reference', + condition: { field: 'operation', value: 'analyze_id' }, + mode: 'advanced' as const, + }, + { + id: 's3Uri', + title: 'S3 URI', + type: 'short-input' as SubBlockType, + placeholder: 's3://bucket-name/path/to/document.pdf', + condition: { field: 'processingMode', value: 'async' }, + }, + { + id: 'region', + title: 'AWS Region', + type: 'short-input' as SubBlockType, + placeholder: 'e.g., us-east-1', + required: true, + }, + { + id: 'accessKeyId', + title: 'AWS Access Key ID', + type: 'short-input' as SubBlockType, + placeholder: 'Enter your AWS Access Key ID', + password: true, + required: true, + }, + { + id: 'secretAccessKey', + title: 'AWS Secret Access Key', + type: 'short-input' as SubBlockType, + placeholder: 'Enter your AWS Secret Access Key', + password: true, + required: true, + }, + { + id: 'extractTables', + title: 'Extract Tables', + type: 'switch' as SubBlockType, + condition: { field: 'operation', value: 'analyze_document' }, + }, + { + id: 'extractForms', + title: 'Extract Forms (Key-Value Pairs)', + type: 'switch' as SubBlockType, + condition: { field: 'operation', value: 'analyze_document' }, + }, + { + id: 'detectSignatures', + title: 'Detect Signatures', + type: 'switch' as SubBlockType, + condition: { field: 'operation', value: 'analyze_document' }, + }, + { + id: 'analyzeLayout', + title: 'Analyze Document Layout', + type: 'switch' as SubBlockType, + condition: { field: 'operation', value: 'analyze_document' }, + }, + ], tools: { - access: ['textract_parser_v2'], + access: ['textract_parser_v2', 'textract_analyze_expense', 'textract_analyze_id'], config: { - tool: createVersionedToolSelector({ - baseToolSelector: () => 'textract_parser', - suffix: '_v2', - fallbackToolId: 'textract_parser_v2', - }), + tool: (params) => { + const operation = params.operation || 'analyze_document' + if (operation === 'analyze_expense') return 'textract_analyze_expense' + if (operation === 'analyze_id') return 'textract_analyze_id' + return 'textract_parser_v2' + }, params: (params) => { - if (!params.accessKeyId || params.accessKeyId.trim() === '') { - throw new Error('AWS Access Key ID is required') - } - if (!params.secretAccessKey || params.secretAccessKey.trim() === '') { - throw new Error('AWS Secret Access Key is required') - } - if (!params.region || params.region.trim() === '') { - throw new Error('AWS Region is required') + const { accessKeyId, secretAccessKey, region } = requireAwsCredentials(params) + const operation = params.operation || 'analyze_document' + + if (operation === 'analyze_id') { + const file = normalizeFileInput(params.document, { single: true }) + if (!file) throw new Error('Identity document is required') + + const parameters: Record = { + accessKeyId, + secretAccessKey, + region, + file, + } + const fileBack = normalizeFileInput(params.documentBack, { single: true }) + if (fileBack) parameters.fileBack = fileBack + return parameters } const processingMode = params.processingMode || 'sync' const parameters: Record = { - accessKeyId: params.accessKeyId.trim(), - secretAccessKey: params.secretAccessKey.trim(), - region: params.region.trim(), + accessKeyId, + secretAccessKey, + region, processingMode, } @@ -265,29 +378,71 @@ export const TextractV2Block: BlockConfig = { } parameters.s3Uri = params.s3Uri.trim() } else { - // document is the canonical param for both basic (fileUpload) and advanced (fileReference) modes const file = normalizeFileInput(params.document, { single: true }) - if (!file) { - throw new Error('Document file is required') - } + if (!file) throw new Error('Document file is required') parameters.file = file } + if (operation === 'analyze_expense') return parameters + const featureTypes: string[] = [] if (params.extractTables) featureTypes.push('TABLES') if (params.extractForms) featureTypes.push('FORMS') if (params.detectSignatures) featureTypes.push('SIGNATURES') if (params.analyzeLayout) featureTypes.push('LAYOUT') - - if (featureTypes.length > 0) { - parameters.featureTypes = featureTypes - } + if (featureTypes.length > 0) parameters.featureTypes = featureTypes return parameters }, }, }, - inputs: textractV2Inputs, + inputs: { + operation: { + type: 'string', + description: 'Operation: analyze_document, analyze_expense, or analyze_id', + }, + processingMode: { type: 'string', description: 'Document type: single-page or multi-page' }, + document: { type: 'json', description: 'Document input (file upload or URL reference)' }, + documentBack: { + type: 'json', + description: 'Back-of-ID document input, for analyze_id (file upload or URL reference)', + }, + s3Uri: { type: 'string', description: 'S3 URI for multi-page processing (s3://bucket/key)' }, + extractTables: { type: 'boolean', description: 'Extract tables from document' }, + extractForms: { type: 'boolean', description: 'Extract form key-value pairs' }, + detectSignatures: { type: 'boolean', description: 'Detect signatures' }, + analyzeLayout: { type: 'boolean', description: 'Analyze document layout' }, + region: { type: 'string', description: 'AWS region' }, + accessKeyId: { type: 'string', description: 'AWS Access Key ID' }, + secretAccessKey: { type: 'string', description: 'AWS Secret Access Key' }, + }, + outputs: { + blocks: { + type: 'json', + description: 'Array of detected blocks (PAGE, LINE, WORD, TABLE, CELL, KEY_VALUE_SET, etc.)', + condition: { field: 'operation', value: 'analyze_document' }, + }, + expenseDocuments: { + type: 'json', + description: + '[{expenseIndex, summaryFields: [{type, valueDetection, currency}], lineItemGroups: [{lineItems}]}]', + condition: { field: 'operation', value: 'analyze_expense' }, + }, + identityDocuments: { + type: 'json', + description: + '[{documentIndex, identityDocumentFields: [{type: {text}, valueDetection: {text, normalizedValue}}]}]', + condition: { field: 'operation', value: 'analyze_id' }, + }, + documentMetadata: { + type: 'json', + description: 'Document metadata containing pages count', + }, + modelVersion: { + type: 'string', + description: 'Version of the Textract model used for processing', + }, + }, } export const TextractBlockMeta = { @@ -368,9 +523,9 @@ export const TextractBlockMeta = { { name: 'extract-invoice-fields', description: - 'Run an invoice or receipt through AWS Textract and return clean structured fields (vendor, date, totals, line items). Use when you need to digitize finance documents.', + 'Run an invoice or receipt through AWS Textract Analyze Expense and return clean structured fields (vendor, date, totals, line items). Use when you need to digitize finance documents.', content: - '# Extract Invoice Fields\n\nUse AWS Textract to pull structured data from an invoice or receipt image/PDF.\n\n## Steps\n1. Choose the processing mode: Single Page for JPEG, PNG, or one-page PDF; Multi-Page for multi-page PDF/TIFF staged in S3.\n2. Provide the document (upload, file reference, or S3 URI) and enable Extract Forms and Extract Tables so key-value pairs and line items are captured.\n3. Run the extraction and read the returned blocks (KEY_VALUE_SET, TABLE, CELL, LINE).\n4. Map key-value pairs to fields: vendor, invoice number, invoice date, due date, subtotal, tax, and total.\n5. Reconstruct line items from the TABLE/CELL blocks into rows with description, quantity, unit price, and amount.\n\n## Output\nReturn a clean JSON record with the header fields and a line-items array. Flag any field where Textract confidence is low or the totals do not reconcile so a human can review.', + '# Extract Invoice Fields\n\nUse AWS Textract Analyze Expense to pull structured data from an invoice or receipt image/PDF.\n\n## Steps\n1. Set Operation to "Analyze Expense". Choose Single Page for JPEG, PNG, or one-page PDF, or Multi-Page for a PDF/TIFF staged in S3.\n2. Provide the document (upload, file reference, or S3 URI) and run the extraction.\n3. Read `expenseDocuments[].summaryFields` for header data (vendor, invoice number, invoice date, due date, subtotal, tax, total) and `expenseDocuments[].lineItemGroups[].lineItems` for purchased items.\n4. Map each summary field\'s normalized `type.text` (e.g., VENDOR_NAME, TOTAL, INVOICE_DATE) to your target schema, and read `valueDetection.text` for the value.\n\n## Output\nReturn a clean JSON record with the header fields and a line-items array. Flag any field where confidence is low or the totals do not reconcile so a human can review.', }, { name: 'extract-form-key-values', @@ -386,5 +541,12 @@ export const TextractBlockMeta = { content: '# OCR Document To Text\n\nExtract readable text from a scanned or image-only document.\n\n## Steps\n1. Pick the processing mode that matches the file: Single Page for an image or one-page PDF, Multi-Page (S3) for multi-page documents.\n2. Provide the document and run the extraction. Plain OCR does not require the Forms or Tables features.\n3. Read the LINE and WORD blocks and join LINE blocks in reading order to reconstruct the text.\n4. Preserve paragraph and page breaks using the PAGE blocks.\n\n## Output\nReturn the full extracted text as clean Markdown, grouped by page. Include the page count from document metadata and surface any low-confidence lines so they can be reviewed before indexing.', }, + { + name: 'verify-identity-document', + description: + 'Run a government-issued ID through AWS Textract Analyze ID and return normalized identity fields (name, date of birth, ID number, expiration date). Use for KYC and identity-verification workflows.', + content: + '# Verify Identity Document\n\nUse AWS Textract Analyze ID to extract normalized fields from a driver\'s license, passport, or other identity document.\n\n## Steps\n1. Set Operation to "Analyze Identity Document".\n2. Upload the front of the ID. If the document carries data on both sides (e.g., a driver\'s license), also upload the back.\n3. Run the extraction and read `identityDocuments[].identityDocumentFields`, where each field has a normalized `type.text` (e.g., FIRST_NAME, LAST_NAME, DATE_OF_BIRTH, DOCUMENT_NUMBER, EXPIRATION_DATE) and a `valueDetection.text` (with `valueDetection.normalizedValue` for dates).\n4. Compare the extracted fields against the customer record you already have on file.\n\n## Output\nReturn a normalized identity record and flag any mismatch between the extracted fields and the existing customer record for human review.', + }, ], } as const satisfies BlockMeta diff --git a/apps/sim/lib/api/contracts/tools/media/document-parse.ts b/apps/sim/lib/api/contracts/tools/media/document-parse.ts index 8bafd437ef6..ecc4bd8e93c 100644 --- a/apps/sim/lib/api/contracts/tools/media/document-parse.ts +++ b/apps/sim/lib/api/contracts/tools/media/document-parse.ts @@ -46,6 +46,65 @@ export const textractParseBodySchema = z } }) +export const textractAnalyzeExpenseBodySchema = z + .object({ + accessKeyId: z.string().min(1, 'AWS Access Key ID is required'), + secretAccessKey: z.string().min(1, 'AWS Secret Access Key is required'), + region: z + .string() + .min(1, 'AWS region is required') + .regex( + AWS_REGION_PATTERN, + 'AWS region must be a valid AWS region (e.g., us-east-1, eu-west-2, us-gov-west-1)' + ), + processingMode: z.enum(['sync', 'async']).optional().default('sync'), + filePath: z.string().optional(), + file: RawFileInputSchema.optional(), + s3Uri: z.string().optional(), + }) + .superRefine((data, ctx) => { + if (data.processingMode === 'async' && !data.s3Uri) { + ctx.addIssue({ + code: 'custom', + message: 'S3 URI is required for multi-page processing (s3://bucket/key)', + path: ['s3Uri'], + }) + } + if (data.processingMode !== 'async' && !data.file && !data.filePath) { + ctx.addIssue({ + code: 'custom', + message: 'Document input is required for single-page processing', + path: ['filePath'], + }) + } + }) + +export const textractAnalyzeIdBodySchema = z + .object({ + accessKeyId: z.string().min(1, 'AWS Access Key ID is required'), + secretAccessKey: z.string().min(1, 'AWS Secret Access Key is required'), + region: z + .string() + .min(1, 'AWS region is required') + .regex( + AWS_REGION_PATTERN, + 'AWS region must be a valid AWS region (e.g., us-east-1, eu-west-2, us-gov-west-1)' + ), + filePath: z.string().optional(), + file: RawFileInputSchema.optional(), + filePathBack: z.string().optional(), + fileBack: RawFileInputSchema.optional(), + }) + .superRefine((data, ctx) => { + if (!data.file && !data.filePath) { + ctx.addIssue({ + code: 'custom', + message: 'Identity document is required', + path: ['filePath'], + }) + } + }) + export const reductoParseBodySchema = z.object({ apiKey: z.string().min(1, 'API key is required'), filePath: z.string().optional(), @@ -94,6 +153,20 @@ export const textractParseContract = defineRouteContract({ response: { mode: 'json', schema: toolJsonResponseSchema }, }) +export const textractAnalyzeExpenseContract = defineRouteContract({ + method: 'POST', + path: '/api/tools/textract/analyze-expense', + body: textractAnalyzeExpenseBodySchema, + response: { mode: 'json', schema: toolJsonResponseSchema }, +}) + +export const textractAnalyzeIdContract = defineRouteContract({ + method: 'POST', + path: '/api/tools/textract/analyze-id', + body: textractAnalyzeIdBodySchema, + response: { mode: 'json', schema: toolJsonResponseSchema }, +}) + export const reductoParseContract = defineRouteContract({ method: 'POST', path: '/api/tools/reducto/parse', diff --git a/apps/sim/package.json b/apps/sim/package.json index 1e824356698..31e66edbc00 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -54,6 +54,7 @@ "@aws-sdk/client-sqs": "3.1032.0", "@aws-sdk/client-sso-admin": "3.1032.0", "@aws-sdk/client-sts": "3.1032.0", + "@aws-sdk/client-textract": "3.1032.0", "@aws-sdk/lib-dynamodb": "3.1032.0", "@aws-sdk/s3-request-presigner": "3.1032.0", "@azure/communication-email": "1.0.0", diff --git a/apps/sim/tools/registry.ts b/apps/sim/tools/registry.ts index 20ac92282f3..f32edc49219 100644 --- a/apps/sim/tools/registry.ts +++ b/apps/sim/tools/registry.ts @@ -3901,7 +3901,12 @@ import { temporalUnpauseScheduleTool, temporalUpdateWorkflowTool, } from '@/tools/temporal' -import { textractParserTool, textractParserV2Tool } from '@/tools/textract' +import { + textractAnalyzeExpenseTool, + textractAnalyzeIdTool, + textractParserTool, + textractParserV2Tool, +} from '@/tools/textract' import { thinkingTool } from '@/tools/thinking' import { thriveAddAudienceManagersTool, @@ -6946,6 +6951,8 @@ export const tools: Record = { mistral_parser_v3: mistralParserV3Tool, reducto_parser: reductoParserTool, reducto_parser_v2: reductoParserV2Tool, + textract_analyze_expense: textractAnalyzeExpenseTool, + textract_analyze_id: textractAnalyzeIdTool, textract_parser: textractParserTool, textract_parser_v2: textractParserV2Tool, thinking_tool: thinkingTool, diff --git a/apps/sim/tools/textract/analyze-expense.ts b/apps/sim/tools/textract/analyze-expense.ts new file mode 100644 index 00000000000..5ce3a3ba458 --- /dev/null +++ b/apps/sim/tools/textract/analyze-expense.ts @@ -0,0 +1,219 @@ +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import type { + TextractAnalyzeExpenseOutput, + TextractAnalyzeExpenseV2Input, +} from '@/tools/textract/types' +import type { ToolConfig } from '@/tools/types' + +const logger = createLogger('TextractAnalyzeExpenseTool') + +/** Shared shape for AnalyzeExpense fields — used by both summaryFields and lineItemExpenseFields. */ +const expenseFieldOutputProperties = { + type: { + type: 'object', + description: 'Normalized field label (e.g., VENDOR_NAME, TOTAL, ITEM, QUANTITY, PRICE)', + properties: { + text: { type: 'string', description: 'Field label text' }, + confidence: { type: 'number', description: 'Confidence score (0-100)' }, + }, + }, + valueDetection: { + type: 'object', + description: 'Detected value for the field', + properties: { + text: { type: 'string', description: 'Field value text' }, + confidence: { type: 'number', description: 'Confidence score (0-100)' }, + }, + }, + labelDetection: { + type: 'object', + description: 'The printed label detected next to the value, if any', + optional: true, + properties: { + text: { type: 'string', description: 'Label text' }, + confidence: { type: 'number', description: 'Confidence score (0-100)' }, + }, + }, + pageNumber: { type: 'number', description: 'Page number the field was found on', optional: true }, + currency: { + type: 'object', + description: 'Currency of a monetary value, if detected', + optional: true, + properties: { + code: { type: 'string', description: 'ISO currency code (e.g., USD)' }, + confidence: { type: 'number', description: 'Confidence score (0-100)' }, + }, + }, + groupProperties: { + type: 'array', + description: 'Grouping metadata (e.g., distinguishes vendor vs. recipient address lines)', + optional: true, + items: { + type: 'object', + properties: { + id: { type: 'string', description: 'Group identifier' }, + types: { type: 'array', description: 'Group type tags', items: { type: 'string' } }, + }, + }, + }, +} as const + +export const textractAnalyzeExpenseTool: ToolConfig< + TextractAnalyzeExpenseV2Input, + TextractAnalyzeExpenseOutput +> = { + id: 'textract_analyze_expense', + name: 'AWS Textract Analyze Expense', + description: 'Extract structured invoice and receipt fields using AWS Textract AnalyzeExpense', + version: '1.0.0', + + params: { + accessKeyId: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'AWS Access Key ID', + }, + secretAccessKey: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'AWS Secret Access Key', + }, + region: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'AWS region for Textract service (e.g., us-east-1)', + }, + processingMode: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Document type: single-page or multi-page. Defaults to single-page.', + }, + file: { + type: 'file', + required: false, + visibility: 'hidden', + description: 'Invoice or receipt to be processed (JPEG, PNG, or single-page PDF).', + }, + s3Uri: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'S3 URI for multi-page processing (s3://bucket/key).', + }, + }, + + request: { + url: '/api/tools/textract/analyze-expense', + method: 'POST', + headers: () => ({ + 'Content-Type': 'application/json', + Accept: 'application/json', + }), + body: (params) => { + const processingMode = params.processingMode || 'sync' + const requestBody: Record = { + accessKeyId: params.accessKeyId?.trim(), + secretAccessKey: params.secretAccessKey?.trim(), + region: params.region?.trim(), + processingMode, + } + + if (processingMode === 'async') { + requestBody.s3Uri = params.s3Uri?.trim() + } else { + if (!params.file || typeof params.file !== 'object') { + throw new Error('Document file is required for single-page processing') + } + requestBody.file = params.file + } + + return requestBody + }, + }, + + transformResponse: async (response) => { + try { + const apiResult = await response.json() + + if (!apiResult || typeof apiResult !== 'object') { + throw new Error('Invalid response format from Textract API') + } + if (!apiResult.success) { + throw new Error(apiResult.error || 'Request failed') + } + + const data = apiResult.output ?? apiResult + + return { + success: true, + output: { + expenseDocuments: data.expenseDocuments ?? [], + documentMetadata: { pages: data.documentMetadata?.pages ?? 0 }, + modelVersion: data.modelVersion ?? undefined, + }, + } + } catch (error) { + logger.error('Error processing Textract AnalyzeExpense result:', toError(error)) + throw error + } + }, + + outputs: { + expenseDocuments: { + type: 'array', + description: 'Detected expense documents with summary fields and line items', + items: { + type: 'object', + properties: { + expenseIndex: { type: 'number', description: 'Index of the expense document' }, + summaryFields: { + type: 'array', + description: 'Header fields such as vendor name, invoice date, and totals', + items: { type: 'object', properties: expenseFieldOutputProperties }, + }, + lineItemGroups: { + type: 'array', + description: 'Groups of line items (e.g., purchased items and their prices)', + items: { + type: 'object', + properties: { + lineItemGroupIndex: { type: 'number', description: 'Index of the line item group' }, + lineItems: { + type: 'array', + description: 'Individual line items within the group', + items: { + type: 'object', + properties: { + lineItemExpenseFields: { + type: 'array', + description: 'Fields for a single line item (description, quantity, price)', + items: { type: 'object', properties: expenseFieldOutputProperties }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + documentMetadata: { + type: 'object', + description: 'Metadata about the analyzed document', + properties: { + pages: { type: 'number', description: 'Number of pages in the document' }, + }, + }, + modelVersion: { + type: 'string', + description: 'Version of the AnalyzeExpense model used (multi-page/async only)', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/textract/analyze-id.ts b/apps/sim/tools/textract/analyze-id.ts new file mode 100644 index 00000000000..e6700202eba --- /dev/null +++ b/apps/sim/tools/textract/analyze-id.ts @@ -0,0 +1,152 @@ +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import type { TextractAnalyzeIdOutput, TextractAnalyzeIdV2Input } from '@/tools/textract/types' +import type { ToolConfig } from '@/tools/types' + +const logger = createLogger('TextractAnalyzeIdTool') + +export const textractAnalyzeIdTool: ToolConfig = + { + id: 'textract_analyze_id', + name: 'AWS Textract Analyze ID', + description: 'Extract identity document fields using AWS Textract AnalyzeID', + version: '1.0.0', + + params: { + accessKeyId: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'AWS Access Key ID', + }, + secretAccessKey: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'AWS Secret Access Key', + }, + region: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'AWS region for Textract service (e.g., us-east-1)', + }, + file: { + type: 'file', + required: false, + visibility: 'hidden', + description: 'Front of the identity document (JPEG, PNG, or PDF).', + }, + fileBack: { + type: 'file', + required: false, + visibility: 'hidden', + description: 'Back of the identity document, if applicable (JPEG, PNG, or PDF).', + }, + }, + + request: { + url: '/api/tools/textract/analyze-id', + method: 'POST', + headers: () => ({ + 'Content-Type': 'application/json', + Accept: 'application/json', + }), + body: (params) => { + if (!params.file || typeof params.file !== 'object') { + throw new Error('Identity document is required') + } + + const requestBody: Record = { + accessKeyId: params.accessKeyId?.trim(), + secretAccessKey: params.secretAccessKey?.trim(), + region: params.region?.trim(), + file: params.file, + } + + if (params.fileBack && typeof params.fileBack === 'object') { + requestBody.fileBack = params.fileBack + } + + return requestBody + }, + }, + + transformResponse: async (response) => { + try { + const apiResult = await response.json() + + if (!apiResult || typeof apiResult !== 'object') { + throw new Error('Invalid response format from Textract API') + } + if (!apiResult.success) { + throw new Error(apiResult.error || 'Request failed') + } + + const data = apiResult.output ?? apiResult + + return { + success: true, + output: { + identityDocuments: data.identityDocuments ?? [], + documentMetadata: { pages: data.documentMetadata?.pages ?? 0 }, + modelVersion: data.modelVersion ?? undefined, + }, + } + } catch (error) { + logger.error('Error processing Textract AnalyzeID result:', toError(error)) + throw error + } + }, + + outputs: { + identityDocuments: { + type: 'array', + description: 'Detected identity documents with normalized fields', + items: { + type: 'object', + properties: { + documentIndex: { type: 'number', description: 'Index of the document page set' }, + identityDocumentFields: { + type: 'array', + description: + 'Normalized fields such as FIRST_NAME, LAST_NAME, DATE_OF_BIRTH, DOCUMENT_NUMBER, EXPIRATION_DATE', + items: { + type: 'object', + properties: { + type: { + type: 'object', + description: 'Normalized field label', + properties: { + text: { type: 'string', description: 'Field label text' }, + confidence: { type: 'number', description: 'Confidence score (0-100)' }, + }, + }, + valueDetection: { + type: 'object', + description: 'Detected value for the field, with a normalized value for dates', + properties: { + text: { type: 'string', description: 'Field value text' }, + confidence: { type: 'number', description: 'Confidence score (0-100)' }, + }, + }, + }, + }, + }, + }, + }, + }, + documentMetadata: { + type: 'object', + description: 'Metadata about the analyzed document', + properties: { + pages: { type: 'number', description: 'Number of pages analyzed' }, + }, + }, + modelVersion: { + type: 'string', + description: 'Version of the AnalyzeID model used for processing', + optional: true, + }, + }, + } diff --git a/apps/sim/tools/textract/index.ts b/apps/sim/tools/textract/index.ts index c47c5cfc557..aea5ba29d9b 100644 --- a/apps/sim/tools/textract/index.ts +++ b/apps/sim/tools/textract/index.ts @@ -1,2 +1,4 @@ +export { textractAnalyzeExpenseTool } from '@/tools/textract/analyze-expense' +export { textractAnalyzeIdTool } from '@/tools/textract/analyze-id' export { textractParserTool, textractParserV2Tool } from '@/tools/textract/parser' export * from '@/tools/textract/types' diff --git a/apps/sim/tools/textract/textract.test.ts b/apps/sim/tools/textract/textract.test.ts new file mode 100644 index 00000000000..520d6a4a0a5 --- /dev/null +++ b/apps/sim/tools/textract/textract.test.ts @@ -0,0 +1,210 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { textractAnalyzeExpenseTool } from '@/tools/textract/analyze-expense' +import { textractAnalyzeIdTool } from '@/tools/textract/analyze-id' +import { textractParserTool, textractParserV2Tool } from '@/tools/textract/parser' + +const respond = (body: unknown) => new Response(JSON.stringify(body)) + +describe('textract_parser', () => { + const body = textractParserTool.request.body! + + it('builds a sync body from filePath', () => { + expect( + body({ + accessKeyId: ' key ', + secretAccessKey: ' secret ', + region: ' us-east-1 ', + filePath: ' https://example.com/doc.pdf ', + featureTypes: ['TABLES'], + } as never) + ).toMatchObject({ + accessKeyId: 'key', + secretAccessKey: 'secret', + region: 'us-east-1', + processingMode: 'sync', + filePath: 'https://example.com/doc.pdf', + featureTypes: ['TABLES'], + }) + }) + + it('requires s3Uri for async mode', () => { + expect(() => + body({ + accessKeyId: 'key', + secretAccessKey: 'secret', + region: 'us-east-1', + processingMode: 'async', + } as never) + ).not.toThrow() + expect( + body({ + accessKeyId: 'key', + secretAccessKey: 'secret', + region: 'us-east-1', + processingMode: 'async', + s3Uri: 's3://bucket/key.pdf', + } as never) + ).toMatchObject({ processingMode: 'async', s3Uri: 's3://bucket/key.pdf' }) + }) + + it('throws when no document is provided for sync mode', () => { + expect(() => + body({ + accessKeyId: 'key', + secretAccessKey: 'secret', + region: 'us-east-1', + } as never) + ).toThrow('Document is required for single-page processing') + }) + + it('normalizes the documented response shape', async () => { + const result = await textractParserTool.transformResponse!( + respond({ + success: true, + output: { + blocks: [{ BlockType: 'LINE', Id: '1', Text: 'Hello' }], + documentMetadata: { pages: 2 }, + modelVersion: '1.0', + }, + }) + ) + + expect(result.success).toBe(true) + expect(result.output.blocks).toHaveLength(1) + expect(result.output.documentMetadata.pages).toBe(2) + expect(result.output.modelVersion).toBe('1.0') + }) + + it('surfaces the API error message on failure', async () => { + await expect( + textractParserTool.transformResponse!(respond({ success: false, error: 'Bad request' })) + ).rejects.toThrow('Bad request') + }) +}) + +describe('textract_parser_v2', () => { + const body = textractParserV2Tool.request.body! + + it('throws when no file is provided for sync mode', () => { + expect(() => + body({ + accessKeyId: 'key', + secretAccessKey: 'secret', + region: 'us-east-1', + } as never) + ).toThrow('Document file is required for single-page processing') + }) + + it('builds a sync body from a UserFile', () => { + const file = { key: 'file-key', name: 'doc.pdf' } as never + expect( + body({ + accessKeyId: 'key', + secretAccessKey: 'secret', + region: 'us-east-1', + file, + } as never) + ).toMatchObject({ processingMode: 'sync', file }) + }) +}) + +describe('textract_analyze_expense', () => { + const body = textractAnalyzeExpenseTool.request.body! + + it('throws when no file is provided for sync mode', () => { + expect(() => + body({ + accessKeyId: 'key', + secretAccessKey: 'secret', + region: 'us-east-1', + } as never) + ).toThrow('Document file is required for single-page processing') + }) + + it('builds an async body requiring s3Uri', () => { + expect( + body({ + accessKeyId: 'key', + secretAccessKey: 'secret', + region: 'us-east-1', + processingMode: 'async', + s3Uri: 's3://bucket/receipt.pdf', + } as never) + ).toMatchObject({ processingMode: 'async', s3Uri: 's3://bucket/receipt.pdf' }) + }) + + it('normalizes expense documents from the response', async () => { + const result = await textractAnalyzeExpenseTool.transformResponse!( + respond({ + success: true, + output: { + expenseDocuments: [{ expenseIndex: 0, summaryFields: [], lineItemGroups: [] }], + documentMetadata: { pages: 1 }, + }, + }) + ) + + expect(result.success).toBe(true) + expect(result.output.expenseDocuments).toHaveLength(1) + expect(result.output.documentMetadata.pages).toBe(1) + }) +}) + +describe('textract_analyze_id', () => { + const body = textractAnalyzeIdTool.request.body! + + it('throws when no front-of-ID file is provided', () => { + expect(() => + body({ accessKeyId: 'key', secretAccessKey: 'secret', region: 'us-east-1' } as never) + ).toThrow('Identity document is required') + }) + + it('includes fileBack only when provided', () => { + const file = { key: 'front-key' } as never + expect( + body({ accessKeyId: 'key', secretAccessKey: 'secret', region: 'us-east-1', file } as never) + ).toEqual({ + accessKeyId: 'key', + secretAccessKey: 'secret', + region: 'us-east-1', + file, + }) + + const fileBack = { key: 'back-key' } as never + expect( + body({ + accessKeyId: 'key', + secretAccessKey: 'secret', + region: 'us-east-1', + file, + fileBack, + } as never) + ).toEqual({ + accessKeyId: 'key', + secretAccessKey: 'secret', + region: 'us-east-1', + file, + fileBack, + }) + }) + + it('normalizes identity documents from the response', async () => { + const result = await textractAnalyzeIdTool.transformResponse!( + respond({ + success: true, + output: { + identityDocuments: [{ documentIndex: 0, identityDocumentFields: [] }], + documentMetadata: { pages: 1 }, + modelVersion: '1.0', + }, + }) + ) + + expect(result.success).toBe(true) + expect(result.output.identityDocuments).toHaveLength(1) + expect(result.output.modelVersion).toBe('1.0') + }) +}) diff --git a/apps/sim/tools/textract/types.ts b/apps/sim/tools/textract/types.ts index 813c613ad78..f8df613b473 100644 --- a/apps/sim/tools/textract/types.ts +++ b/apps/sim/tools/textract/types.ts @@ -81,41 +81,119 @@ interface TextractBlock { } } -interface TextractDocumentMetadataRaw { - Pages: number -} - interface TextractDocumentMetadata { pages: number } -interface TextractApiResponse { - Blocks: TextractBlock[] - DocumentMetadata: TextractDocumentMetadataRaw - AnalyzeDocumentModelVersion?: string - DetectDocumentTextModelVersion?: string -} - interface TextractNormalizedOutput { blocks: TextractBlock[] documentMetadata: TextractDocumentMetadata modelVersion?: string } -interface TextractAsyncJobResponse { - JobStatus: 'IN_PROGRESS' | 'SUCCEEDED' | 'FAILED' | 'PARTIAL_SUCCESS' - StatusMessage?: string - Blocks?: TextractBlock[] - DocumentMetadata?: TextractDocumentMetadataRaw - NextToken?: string - AnalyzeDocumentModelVersion?: string - DetectDocumentTextModelVersion?: string +export interface TextractParserOutput extends ToolResponse { + output: TextractNormalizedOutput } -interface TextractStartJobResponse { - JobId: string +export interface TextractAnalyzeExpenseInput { + accessKeyId: string + secretAccessKey: string + region: string + processingMode?: TextractProcessingMode + filePath?: string + file?: RawFileInput + s3Uri?: string } -export interface TextractParserOutput extends ToolResponse { - output: TextractNormalizedOutput +export interface TextractAnalyzeExpenseV2Input { + accessKeyId: string + secretAccessKey: string + region: string + processingMode?: TextractProcessingMode + file?: UserFile + s3Uri?: string +} + +interface TextractCurrency { + code?: string + confidence?: number +} + +interface TextractExpenseFieldValue { + text?: string + confidence?: number +} + +interface TextractExpenseField { + type?: TextractExpenseFieldValue + valueDetection?: TextractExpenseFieldValue + labelDetection?: TextractExpenseFieldValue + pageNumber?: number + currency?: TextractCurrency + groupProperties?: { id: string; types: string[] }[] +} + +interface TextractLineItem { + lineItemExpenseFields: TextractExpenseField[] +} + +interface TextractLineItemGroup { + lineItemGroupIndex?: number + lineItems: TextractLineItem[] +} + +interface TextractExpenseDocument { + expenseIndex?: number + summaryFields: TextractExpenseField[] + lineItemGroups: TextractLineItemGroup[] +} + +export interface TextractAnalyzeExpenseOutput extends ToolResponse { + output: { + expenseDocuments: TextractExpenseDocument[] + documentMetadata: TextractDocumentMetadata + modelVersion?: string + } +} + +export interface TextractAnalyzeIdInput { + accessKeyId: string + secretAccessKey: string + region: string + filePath?: string + file?: RawFileInput + filePathBack?: string + fileBack?: RawFileInput +} + +export interface TextractAnalyzeIdV2Input { + accessKeyId: string + secretAccessKey: string + region: string + file?: UserFile + fileBack?: UserFile +} + +interface TextractIdFieldValue { + text?: string + confidence?: number + normalizedValue?: { value?: string; valueType?: string } +} + +interface TextractIdentityDocumentField { + type?: TextractIdFieldValue + valueDetection?: TextractIdFieldValue +} + +interface TextractIdentityDocument { + documentIndex?: number + identityDocumentFields: TextractIdentityDocumentField[] +} + +export interface TextractAnalyzeIdOutput extends ToolResponse { + output: { + identityDocuments: TextractIdentityDocument[] + documentMetadata: TextractDocumentMetadata + modelVersion?: string + } } diff --git a/bun.lock b/bun.lock index ff44268a33c..b6ab6c68a15 100644 --- a/bun.lock +++ b/bun.lock @@ -122,6 +122,7 @@ "@aws-sdk/client-sqs": "3.1032.0", "@aws-sdk/client-sso-admin": "3.1032.0", "@aws-sdk/client-sts": "3.1032.0", + "@aws-sdk/client-textract": "3.1032.0", "@aws-sdk/lib-dynamodb": "3.1032.0", "@aws-sdk/s3-request-presigner": "3.1032.0", "@azure/communication-email": "1.0.0", @@ -700,6 +701,8 @@ "@aws-sdk/client-sts": ["@aws-sdk/client-sts@3.1032.0", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.1", "@aws-sdk/credential-provider-node": "^3.972.32", "@aws-sdk/middleware-host-header": "^3.972.10", "@aws-sdk/middleware-logger": "^3.972.10", "@aws-sdk/middleware-recursion-detection": "^3.972.11", "@aws-sdk/middleware-user-agent": "^3.972.31", "@aws-sdk/region-config-resolver": "^3.972.12", "@aws-sdk/signature-v4-multi-region": "^3.996.18", "@aws-sdk/types": "^3.973.8", "@aws-sdk/util-endpoints": "^3.996.7", "@aws-sdk/util-user-agent-browser": "^3.972.10", "@aws-sdk/util-user-agent-node": "^3.973.17", "@smithy/config-resolver": "^4.4.16", "@smithy/core": "^3.23.15", "@smithy/fetch-http-handler": "^5.3.17", "@smithy/hash-node": "^4.2.14", "@smithy/invalid-dependency": "^4.2.14", "@smithy/middleware-content-length": "^4.2.14", "@smithy/middleware-endpoint": "^4.4.30", "@smithy/middleware-retry": "^4.5.3", "@smithy/middleware-serde": "^4.2.18", "@smithy/middleware-stack": "^4.2.14", "@smithy/node-config-provider": "^4.3.14", "@smithy/node-http-handler": "^4.5.3", "@smithy/protocol-http": "^5.3.14", "@smithy/smithy-client": "^4.12.11", "@smithy/types": "^4.14.1", "@smithy/url-parser": "^4.2.14", "@smithy/util-base64": "^4.3.2", "@smithy/util-body-length-browser": "^4.2.2", "@smithy/util-body-length-node": "^4.2.3", "@smithy/util-defaults-mode-browser": "^4.3.47", "@smithy/util-defaults-mode-node": "^4.2.52", "@smithy/util-endpoints": "^3.4.1", "@smithy/util-middleware": "^4.2.14", "@smithy/util-retry": "^4.3.2", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-FCLc5VWb+yz1xb/Jv0sXFGqIIs+bHZQWBKbPQKCuypF3wU/7UFygXuSXo9uJfwISKNGVHJwp+0136f8mqmzRcA=="], + "@aws-sdk/client-textract": ["@aws-sdk/client-textract@3.1032.0", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.1", "@aws-sdk/credential-provider-node": "^3.972.32", "@aws-sdk/middleware-host-header": "^3.972.10", "@aws-sdk/middleware-logger": "^3.972.10", "@aws-sdk/middleware-recursion-detection": "^3.972.11", "@aws-sdk/middleware-user-agent": "^3.972.31", "@aws-sdk/region-config-resolver": "^3.972.12", "@aws-sdk/types": "^3.973.8", "@aws-sdk/util-endpoints": "^3.996.7", "@aws-sdk/util-user-agent-browser": "^3.972.10", "@aws-sdk/util-user-agent-node": "^3.973.17", "@smithy/config-resolver": "^4.4.16", "@smithy/core": "^3.23.15", "@smithy/fetch-http-handler": "^5.3.17", "@smithy/hash-node": "^4.2.14", "@smithy/invalid-dependency": "^4.2.14", "@smithy/middleware-content-length": "^4.2.14", "@smithy/middleware-endpoint": "^4.4.30", "@smithy/middleware-retry": "^4.5.3", "@smithy/middleware-serde": "^4.2.18", "@smithy/middleware-stack": "^4.2.14", "@smithy/node-config-provider": "^4.3.14", "@smithy/node-http-handler": "^4.5.3", "@smithy/protocol-http": "^5.3.14", "@smithy/smithy-client": "^4.12.11", "@smithy/types": "^4.14.1", "@smithy/url-parser": "^4.2.14", "@smithy/util-base64": "^4.3.2", "@smithy/util-body-length-browser": "^4.2.2", "@smithy/util-body-length-node": "^4.2.3", "@smithy/util-defaults-mode-browser": "^4.3.47", "@smithy/util-defaults-mode-node": "^4.2.52", "@smithy/util-endpoints": "^3.4.1", "@smithy/util-middleware": "^4.2.14", "@smithy/util-retry": "^4.3.2", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-aqOnp0aEiqCQQ/ceLTMAjtCsmIdWAeuOyjz9dIzwgcNZKqUPwf+rproTSl8XIHmorqaBb8donzPaLTcAkMr+Yw=="], + "@aws-sdk/core": ["@aws-sdk/core@3.974.21", "", { "dependencies": { "@aws-sdk/types": "^3.973.13", "@aws-sdk/xml-builder": "^3.972.30", "@aws/lambda-invoke-store": "^0.2.2", "@smithy/core": "^3.24.6", "@smithy/signature-v4": "^5.4.6", "@smithy/types": "^4.14.3", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "sha512-P5JAHvn4dTi96UsAGS67LVOqqpUNNRhnfFXqzCYtdBIGZtqBue4CXvRr9YenOO7PALj/Pn8uuyw53FBCiCYw8w=="], "@aws-sdk/credential-provider-env": ["@aws-sdk/credential-provider-env@3.972.47", "", { "dependencies": { "@aws-sdk/core": "^3.974.21", "@aws-sdk/types": "^3.973.13", "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-3YoPwJczcc+MtX2xxXaYaOOWO6xKUJr1ZIIDIFuninr51BYONVVcF/CP8K2xfVRC/PztJjqKWxNGFH7BWQAw1Q=="], diff --git a/scripts/check-api-validation-contracts.ts b/scripts/check-api-validation-contracts.ts index 677077f158e..2a55c58c4cb 100644 --- a/scripts/check-api-validation-contracts.ts +++ b/scripts/check-api-validation-contracts.ts @@ -9,8 +9,8 @@ const QUERY_HOOKS_DIR = path.join(ROOT, 'apps/sim/hooks/queries') const SELECTOR_HOOKS_DIR = path.join(ROOT, 'apps/sim/hooks/selectors') const BASELINE = { - totalRoutes: 904, - zodRoutes: 904, + totalRoutes: 906, + zodRoutes: 906, nonZodRoutes: 0, } as const From dbc9c0fdb4044583e8662ef9004a41d114ac320c Mon Sep 17 00:00:00 2001 From: waleed Date: Mon, 6 Jul 2026 18:16:12 -0700 Subject: [PATCH 2/6] fix(textract): forward URL documents and fix ambiguous error status - textract_analyze_expense/textract_analyze_id now fall back to filePath/filePathBack when the document input is a URL string rather than an uploaded file object, so advanced "File reference" URL inputs actually reach the API (Cursor Bugbot) - mapTextractSdkError defaults to 500 (not 400) when the AWS SDK error has no HTTP status, since that implies a server-side/network failure rather than a bad request (Greptile) --- .../sim/app/api/tools/textract/shared.test.ts | 4 +-- apps/sim/app/api/tools/textract/shared.ts | 4 ++- apps/sim/blocks/blocks/textract.ts | 29 +++++++++++++--- apps/sim/tools/textract/analyze-expense.ts | 15 ++++++--- apps/sim/tools/textract/analyze-id.ts | 27 ++++++++++++--- apps/sim/tools/textract/textract.test.ts | 33 +++++++++++++++++-- apps/sim/tools/textract/types.ts | 3 ++ 7 files changed, 96 insertions(+), 19 deletions(-) diff --git a/apps/sim/app/api/tools/textract/shared.test.ts b/apps/sim/app/api/tools/textract/shared.test.ts index 3c503b08520..08b6d4a3c2a 100644 --- a/apps/sim/app/api/tools/textract/shared.test.ts +++ b/apps/sim/app/api/tools/textract/shared.test.ts @@ -78,9 +78,9 @@ describe('mapTextractSdkError', () => { expect(mapped.status).toBe(500) }) - it('defaults to 400 when the SDK gives no http status', () => { + it('defaults to 500 when the SDK gives no http status, since that implies a server-side failure', () => { const mapped = mapTextractSdkError({ message: 'Unknown failure' }, false) - expect(mapped.status).toBe(400) + expect(mapped.status).toBe(500) }) }) diff --git a/apps/sim/app/api/tools/textract/shared.ts b/apps/sim/app/api/tools/textract/shared.ts index 80363b388d0..4680c86892b 100644 --- a/apps/sim/app/api/tools/textract/shared.ts +++ b/apps/sim/app/api/tools/textract/shared.ts @@ -75,7 +75,9 @@ export function mapTextractSdkError( return new TextractRouteError(`This document format is not supported.${hint}`, 400) } - const status = err.$metadata?.httpStatusCode || 400 + // No status at all (e.g. a network failure before AWS responded) is a server-side problem, not a + // bad request — default to 500 so tool-execution retry logic still treats it as retryable. + const status = err.$metadata?.httpStatusCode ?? 500 return new TextractRouteError(err.message || 'Textract API error', status) } diff --git a/apps/sim/blocks/blocks/textract.ts b/apps/sim/blocks/blocks/textract.ts index 5a6f8a22778..3d195ee36c4 100644 --- a/apps/sim/blocks/blocks/textract.ts +++ b/apps/sim/blocks/blocks/textract.ts @@ -199,6 +199,18 @@ export const TextractBlock: BlockConfig = { }, } +/** + * Resolves a canonical document input to either an uploaded file object or a plain URL string. + * `normalizeFileInput` only recognizes file objects (or JSON-serialized file references) — a raw + * URL typed into the "File reference" advanced field falls through to `filePath` instead. + */ +function resolveDocumentParam(value: unknown): { file?: object; filePath?: string } { + const file = normalizeFileInput(value, { single: true }) + if (file) return { file } + if (typeof value === 'string' && value.trim() !== '') return { filePath: value.trim() } + return {} +} + function requireAwsCredentials(params: Record) { const accessKeyId = typeof params.accessKeyId === 'string' ? params.accessKeyId.trim() : '' const secretAccessKey = @@ -350,17 +362,18 @@ export const TextractV2Block: BlockConfig = { const operation = params.operation || 'analyze_document' if (operation === 'analyze_id') { - const file = normalizeFileInput(params.document, { single: true }) - if (!file) throw new Error('Identity document is required') + const front = resolveDocumentParam(params.document) + if (!front.file && !front.filePath) throw new Error('Identity document is required') const parameters: Record = { accessKeyId, secretAccessKey, region, - file, + ...front, } - const fileBack = normalizeFileInput(params.documentBack, { single: true }) - if (fileBack) parameters.fileBack = fileBack + const back = resolveDocumentParam(params.documentBack) + if (back.file) parameters.fileBack = back.file + else if (back.filePath) parameters.filePathBack = back.filePath return parameters } @@ -377,7 +390,13 @@ export const TextractV2Block: BlockConfig = { throw new Error('S3 URI is required for multi-page processing') } parameters.s3Uri = params.s3Uri.trim() + } else if (operation === 'analyze_expense') { + // textract_analyze_expense accepts filePath as a fallback for URL-based documents. + const resolved = resolveDocumentParam(params.document) + if (!resolved.file && !resolved.filePath) throw new Error('Document file is required') + Object.assign(parameters, resolved) } else { + // textract_parser_v2 (analyze_document) has no filePath param — file only. const file = normalizeFileInput(params.document, { single: true }) if (!file) throw new Error('Document file is required') parameters.file = file diff --git a/apps/sim/tools/textract/analyze-expense.ts b/apps/sim/tools/textract/analyze-expense.ts index 5ce3a3ba458..2cc3da68ba9 100644 --- a/apps/sim/tools/textract/analyze-expense.ts +++ b/apps/sim/tools/textract/analyze-expense.ts @@ -99,6 +99,12 @@ export const textractAnalyzeExpenseTool: ToolConfig< visibility: 'hidden', description: 'Invoice or receipt to be processed (JPEG, PNG, or single-page PDF).', }, + filePath: { + type: 'string', + required: false, + visibility: 'hidden', + description: 'URL to an invoice or receipt to be processed, if not uploaded directly.', + }, s3Uri: { type: 'string', required: false, @@ -125,11 +131,12 @@ export const textractAnalyzeExpenseTool: ToolConfig< if (processingMode === 'async') { requestBody.s3Uri = params.s3Uri?.trim() - } else { - if (!params.file || typeof params.file !== 'object') { - throw new Error('Document file is required for single-page processing') - } + } else if (params.file && typeof params.file === 'object') { requestBody.file = params.file + } else if (params.filePath && params.filePath.trim() !== '') { + requestBody.filePath = params.filePath.trim() + } else { + throw new Error('Document is required for single-page processing') } return requestBody diff --git a/apps/sim/tools/textract/analyze-id.ts b/apps/sim/tools/textract/analyze-id.ts index e6700202eba..775283fa6df 100644 --- a/apps/sim/tools/textract/analyze-id.ts +++ b/apps/sim/tools/textract/analyze-id.ts @@ -37,12 +37,24 @@ export const textractAnalyzeIdTool: ToolConfig { - if (!params.file || typeof params.file !== 'object') { - throw new Error('Identity document is required') - } - const requestBody: Record = { accessKeyId: params.accessKeyId?.trim(), secretAccessKey: params.secretAccessKey?.trim(), region: params.region?.trim(), - file: params.file, + } + + if (params.file && typeof params.file === 'object') { + requestBody.file = params.file + } else if (params.filePath && params.filePath.trim() !== '') { + requestBody.filePath = params.filePath.trim() + } else { + throw new Error('Identity document is required') } if (params.fileBack && typeof params.fileBack === 'object') { requestBody.fileBack = params.fileBack + } else if (params.filePathBack && params.filePathBack.trim() !== '') { + requestBody.filePathBack = params.filePathBack.trim() } return requestBody diff --git a/apps/sim/tools/textract/textract.test.ts b/apps/sim/tools/textract/textract.test.ts index 520d6a4a0a5..1b5823c4ffc 100644 --- a/apps/sim/tools/textract/textract.test.ts +++ b/apps/sim/tools/textract/textract.test.ts @@ -114,14 +114,25 @@ describe('textract_parser_v2', () => { describe('textract_analyze_expense', () => { const body = textractAnalyzeExpenseTool.request.body! - it('throws when no file is provided for sync mode', () => { + it('throws when neither file nor filePath is provided for sync mode', () => { expect(() => body({ accessKeyId: 'key', secretAccessKey: 'secret', region: 'us-east-1', } as never) - ).toThrow('Document file is required for single-page processing') + ).toThrow('Document is required for single-page processing') + }) + + it('falls back to filePath when no file object is provided', () => { + expect( + body({ + accessKeyId: 'key', + secretAccessKey: 'secret', + region: 'us-east-1', + filePath: ' https://example.com/receipt.pdf ', + } as never) + ).toMatchObject({ processingMode: 'sync', filePath: 'https://example.com/receipt.pdf' }) }) it('builds an async body requiring s3Uri', () => { @@ -191,6 +202,24 @@ describe('textract_analyze_id', () => { }) }) + it('falls back to filePath/filePathBack when no file objects are provided', () => { + expect( + body({ + accessKeyId: 'key', + secretAccessKey: 'secret', + region: 'us-east-1', + filePath: ' https://example.com/id-front.png ', + filePathBack: ' https://example.com/id-back.png ', + } as never) + ).toEqual({ + accessKeyId: 'key', + secretAccessKey: 'secret', + region: 'us-east-1', + filePath: 'https://example.com/id-front.png', + filePathBack: 'https://example.com/id-back.png', + }) + }) + it('normalizes identity documents from the response', async () => { const result = await textractAnalyzeIdTool.transformResponse!( respond({ diff --git a/apps/sim/tools/textract/types.ts b/apps/sim/tools/textract/types.ts index f8df613b473..aea61f8b660 100644 --- a/apps/sim/tools/textract/types.ts +++ b/apps/sim/tools/textract/types.ts @@ -111,6 +111,7 @@ export interface TextractAnalyzeExpenseV2Input { region: string processingMode?: TextractProcessingMode file?: UserFile + filePath?: string s3Uri?: string } @@ -171,7 +172,9 @@ export interface TextractAnalyzeIdV2Input { secretAccessKey: string region: string file?: UserFile + filePath?: string fileBack?: UserFile + filePathBack?: string } interface TextractIdFieldValue { From dfe14bf446433f6f6b6967a6fefa60cc00fdbf7e Mon Sep 17 00:00:00 2001 From: waleed Date: Mon, 6 Jul 2026 18:41:21 -0700 Subject: [PATCH 3/6] fix(textract): stop stale processingMode from hiding ID document fields - Front-document fields (fileUpload/fileReference) are shared across all 3 operations; gate them with a values-aware condition so switching to Analyze Identity Document keeps them visible even if a stale processingMode='async' is left over from a previous operation - S3 URI field now also requires operation !== 'analyze_id', since that operation never supports S3 input --- apps/sim/blocks/blocks/textract.ts | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/apps/sim/blocks/blocks/textract.ts b/apps/sim/blocks/blocks/textract.ts index 3d195ee36c4..be0ec7b3e1c 100644 --- a/apps/sim/blocks/blocks/textract.ts +++ b/apps/sim/blocks/blocks/textract.ts @@ -199,6 +199,18 @@ export const TextractBlock: BlockConfig = { }, } +/** + * The front-document fields (fileUpload/fileReference) are shared by all three operations. + * Analyze Identity Document is always synchronous, so they must stay visible for it regardless + * of a stale `processingMode` value left over from switching away from another operation. + */ +function documentFieldCondition(values?: Record) { + if (values?.operation === 'analyze_id') { + return { field: 'operation', value: 'analyze_id' } as const + } + return { field: 'processingMode', value: 'async', not: true } as const +} + /** * Resolves a canonical document input to either an uploaded file object or a plain URL string. * `normalizeFileInput` only recognizes file objects (or JSON-serialized file references) — a raw @@ -260,7 +272,7 @@ export const TextractV2Block: BlockConfig = { canonicalParamId: 'document', acceptedTypes: 'image/jpeg,image/png,application/pdf', placeholder: 'Upload JPEG, PNG, or single-page PDF (max 10MB)', - condition: { field: 'processingMode', value: 'async', not: true }, + condition: documentFieldCondition, mode: 'basic', maxSize: 10, }, @@ -270,7 +282,7 @@ export const TextractV2Block: BlockConfig = { type: 'short-input' as SubBlockType, canonicalParamId: 'document', placeholder: 'File reference', - condition: { field: 'processingMode', value: 'async', not: true }, + condition: documentFieldCondition, mode: 'advanced' as const, }, { @@ -298,7 +310,11 @@ export const TextractV2Block: BlockConfig = { title: 'S3 URI', type: 'short-input' as SubBlockType, placeholder: 's3://bucket-name/path/to/document.pdf', - condition: { field: 'processingMode', value: 'async' }, + condition: { + field: 'processingMode', + value: 'async', + and: { field: 'operation', value: 'analyze_id', not: true }, + }, }, { id: 'region', From 790ef5a609f476a49d8a1a2e19beacfb2273737c Mon Sep 17 00:00:00 2001 From: waleed Date: Mon, 6 Jul 2026 18:52:29 -0700 Subject: [PATCH 4/6] fix(textract): preserve first-page metadata across async pagination pollTextractJob's merge callbacks spread only the latest page, dropping any field (DocumentMetadata, model version) the first page had but a follow-up NextToken page omits. Merge accumulated first so later pages only override fields they actually return. --- .../tools/textract/analyze-expense/route.ts | 1 + .../sim/app/api/tools/textract/parse/route.ts | 1 + .../sim/app/api/tools/textract/shared.test.ts | 33 +++++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/apps/sim/app/api/tools/textract/analyze-expense/route.ts b/apps/sim/app/api/tools/textract/analyze-expense/route.ts index bf04b26977a..c9009f89c2f 100644 --- a/apps/sim/app/api/tools/textract/analyze-expense/route.ts +++ b/apps/sim/app/api/tools/textract/analyze-expense/route.ts @@ -165,6 +165,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { (nextToken) => client.send(new GetExpenseAnalysisCommand({ JobId: jobId, NextToken: nextToken })), (accumulated, page) => ({ + ...accumulated, ...page, ExpenseDocuments: [ ...(accumulated.ExpenseDocuments ?? []), diff --git a/apps/sim/app/api/tools/textract/parse/route.ts b/apps/sim/app/api/tools/textract/parse/route.ts index 750aba8acc8..ec3a6d8e35f 100644 --- a/apps/sim/app/api/tools/textract/parse/route.ts +++ b/apps/sim/app/api/tools/textract/parse/route.ts @@ -156,6 +156,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { new GetDocumentTextDetectionCommand({ JobId: jobId, NextToken: nextToken }) ), (accumulated, page) => ({ + ...accumulated, ...page, Blocks: [...(accumulated.Blocks ?? []), ...(page.Blocks ?? [])], }) diff --git a/apps/sim/app/api/tools/textract/shared.test.ts b/apps/sim/app/api/tools/textract/shared.test.ts index 08b6d4a3c2a..e7af84d6893 100644 --- a/apps/sim/app/api/tools/textract/shared.test.ts +++ b/apps/sim/app/api/tools/textract/shared.test.ts @@ -111,6 +111,7 @@ describe('pollTextractJob', () => { return { JobStatus: 'SUCCEEDED', Blocks: [{ Id: '2' }] } }, (accumulated, page) => ({ + ...accumulated, ...page, Blocks: [...(accumulated.Blocks ?? []), ...(page.Blocks ?? [])], }) @@ -120,6 +121,38 @@ describe('pollTextractJob', () => { expect(result.Blocks).toHaveLength(2) }) + it('preserves fields the first page has but a later page omits (e.g. DocumentMetadata)', async () => { + const result = await pollTextractJob<{ + JobStatus?: string + NextToken?: string + Blocks?: unknown[] + DocumentMetadata?: { Pages?: number } + }>( + 'req-4', + logger, + async (nextToken) => { + if (!nextToken) { + return { + JobStatus: 'SUCCEEDED', + Blocks: [{ Id: '1' }], + DocumentMetadata: { Pages: 3 }, + NextToken: 'next', + } + } + // Follow-up pages from AWS often omit DocumentMetadata/model-version fields. + return { JobStatus: 'SUCCEEDED', Blocks: [{ Id: '2' }] } + }, + (accumulated, page) => ({ + ...accumulated, + ...page, + Blocks: [...(accumulated.Blocks ?? []), ...(page.Blocks ?? [])], + }) + ) + + expect(result.Blocks).toHaveLength(2) + expect(result.DocumentMetadata).toEqual({ Pages: 3 }) + }) + it('throws a TextractRouteError when the job fails', async () => { await expect( pollTextractJob( From dd92247687618f419ebf8986a6122ae03b048573 Mon Sep 17 00:00:00 2001 From: waleed Date: Mon, 6 Jul 2026 19:01:09 -0700 Subject: [PATCH 5/6] fix(textract): pass through the real upstream status for filePath fetch failures fetchDocumentBytes hardcoded 400 for any non-OK response from a document URL, masking transient 5xx failures from the document host as client errors and blocking tool-execution retries. Use the actual response status instead. --- apps/sim/app/api/tools/textract/shared.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/sim/app/api/tools/textract/shared.ts b/apps/sim/app/api/tools/textract/shared.ts index 4680c86892b..570049d6f33 100644 --- a/apps/sim/app/api/tools/textract/shared.ts +++ b/apps/sim/app/api/tools/textract/shared.ts @@ -102,7 +102,12 @@ async function fetchDocumentBytes(url: string): Promise<{ bytes: Buffer; content }) if (!response.ok) { await response.text().catch(() => {}) - throw new TextractRouteError(`Failed to fetch document: ${response.statusText}`, 400) + // Pass through the document host's real status (e.g. a transient 503) instead of a + // hardcoded 400, so tool-execution retry logic can still treat 5xx as retryable. + throw new TextractRouteError( + `Failed to fetch document: ${response.statusText}`, + response.status + ) } const arrayBuffer = await response.arrayBuffer() From 55cb4eb1740ea2d7e1c98d5be9e01971f84bb95f Mon Sep 17 00:00:00 2001 From: waleed Date: Mon, 6 Jul 2026 19:07:36 -0700 Subject: [PATCH 6/6] chore(textract): drop redundant inline comments --- apps/sim/app/api/tools/textract/shared.test.ts | 1 - apps/sim/app/api/tools/textract/shared.ts | 5 +---- apps/sim/blocks/blocks/textract.ts | 2 -- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/apps/sim/app/api/tools/textract/shared.test.ts b/apps/sim/app/api/tools/textract/shared.test.ts index e7af84d6893..b95b6753453 100644 --- a/apps/sim/app/api/tools/textract/shared.test.ts +++ b/apps/sim/app/api/tools/textract/shared.test.ts @@ -139,7 +139,6 @@ describe('pollTextractJob', () => { NextToken: 'next', } } - // Follow-up pages from AWS often omit DocumentMetadata/model-version fields. return { JobStatus: 'SUCCEEDED', Blocks: [{ Id: '2' }] } }, (accumulated, page) => ({ diff --git a/apps/sim/app/api/tools/textract/shared.ts b/apps/sim/app/api/tools/textract/shared.ts index 570049d6f33..90297f21498 100644 --- a/apps/sim/app/api/tools/textract/shared.ts +++ b/apps/sim/app/api/tools/textract/shared.ts @@ -75,8 +75,6 @@ export function mapTextractSdkError( return new TextractRouteError(`This document format is not supported.${hint}`, 400) } - // No status at all (e.g. a network failure before AWS responded) is a server-side problem, not a - // bad request — default to 500 so tool-execution retry logic still treats it as retryable. const status = err.$metadata?.httpStatusCode ?? 500 return new TextractRouteError(err.message || 'Textract API error', status) } @@ -91,6 +89,7 @@ export type ResolveDocumentResult = | { ok: true; document: ResolvedDocument } | { ok: false; response: NextResponse } +/** Passes through the document host's real HTTP status on failure, so tool-execution retry logic can still treat a transient 5xx as retryable. */ async function fetchDocumentBytes(url: string): Promise<{ bytes: Buffer; contentType: string }> { const urlValidation = await validateUrlWithDNS(url, 'Document URL') if (!urlValidation.isValid) { @@ -102,8 +101,6 @@ async function fetchDocumentBytes(url: string): Promise<{ bytes: Buffer; content }) if (!response.ok) { await response.text().catch(() => {}) - // Pass through the document host's real status (e.g. a transient 503) instead of a - // hardcoded 400, so tool-execution retry logic can still treat 5xx as retryable. throw new TextractRouteError( `Failed to fetch document: ${response.statusText}`, response.status diff --git a/apps/sim/blocks/blocks/textract.ts b/apps/sim/blocks/blocks/textract.ts index be0ec7b3e1c..c5a87c480ab 100644 --- a/apps/sim/blocks/blocks/textract.ts +++ b/apps/sim/blocks/blocks/textract.ts @@ -407,12 +407,10 @@ export const TextractV2Block: BlockConfig = { } parameters.s3Uri = params.s3Uri.trim() } else if (operation === 'analyze_expense') { - // textract_analyze_expense accepts filePath as a fallback for URL-based documents. const resolved = resolveDocumentParam(params.document) if (!resolved.file && !resolved.filePath) throw new Error('Document file is required') Object.assign(parameters, resolved) } else { - // textract_parser_v2 (analyze_document) has no filePath param — file only. const file = normalizeFileInput(params.document, { single: true }) if (!file) throw new Error('Document file is required') parameters.file = file