From 6f308d6a4e9457777ff9d5da7950c47c709bd7f8 Mon Sep 17 00:00:00 2001 From: waleed Date: Tue, 7 Jul 2026 08:13:22 -0700 Subject: [PATCH 1/2] fix(brightdata): align integration with live API docs, add markdown/mode/error-report support - fix Discover numResults docs (max 20, not 1000) and contentFormat value ("md" not "markdown", which the API always rejected) - add Discover mode param (standard/deep/fast/zeroRanking) - add markdown output support to Web Unlocker scrape_url via data_format - add include_errors support to scrape_dataset, matching sync_scrape - add .trim() on datasetId in scrape_dataset/sync_scrape URLs - add wandConfig on complex fields, tighten block output descriptions --- apps/sim/blocks/blocks/brightdata.ts | 86 ++++++++++++++++++++- apps/sim/tools/brightdata/discover.ts | 14 +++- apps/sim/tools/brightdata/scrape_dataset.ts | 9 ++- apps/sim/tools/brightdata/scrape_url.ts | 8 ++ apps/sim/tools/brightdata/sync_scrape.ts | 2 +- apps/sim/tools/brightdata/types.ts | 3 + 6 files changed, 114 insertions(+), 8 deletions(-) diff --git a/apps/sim/blocks/blocks/brightdata.ts b/apps/sim/blocks/blocks/brightdata.ts index 28cc33c067b..6d4e043e307 100644 --- a/apps/sim/blocks/blocks/brightdata.ts +++ b/apps/sim/blocks/blocks/brightdata.ts @@ -59,6 +59,18 @@ export const BrightDataBlock: BlockConfig = { value: () => 'raw', condition: { field: 'operation', value: 'scrape_url' }, }, + { + id: 'dataFormat', + title: 'Convert To', + type: 'dropdown', + options: [ + { label: 'None', id: '' }, + { label: 'Markdown', id: 'markdown' }, + ], + value: () => '', + mode: 'advanced', + condition: { field: 'operation', value: 'scrape_url' }, + }, { id: 'country', title: 'Country', @@ -119,6 +131,26 @@ export const BrightDataBlock: BlockConfig = { placeholder: 'Describe what you are looking for (e.g., "find official pricing pages and change notes")', condition: { field: 'operation', value: 'discover' }, + wandConfig: { + enabled: true, + prompt: + 'Generate a concise description of what the agent is trying to accomplish, to help rank web-discovery results by relevance (e.g., "find official pricing pages and recent change notes"). Return ONLY the intent description - no explanations, no extra text.', + placeholder: 'Describe what you are trying to accomplish...', + }, + }, + { + id: 'mode', + title: 'Search Mode', + type: 'dropdown', + options: [ + { label: 'Standard', id: 'standard' }, + { label: 'Deep', id: 'deep' }, + { label: 'Fast', id: 'fast' }, + { label: 'Zero Ranking', id: 'zeroRanking' }, + ], + value: () => 'standard', + mode: 'advanced', + condition: { field: 'operation', value: 'discover' }, }, { id: 'includeContent', @@ -133,7 +165,7 @@ export const BrightDataBlock: BlockConfig = { type: 'dropdown', options: [ { label: 'JSON', id: 'json' }, - { label: 'Markdown', id: 'markdown' }, + { label: 'Markdown', id: 'md' }, ], value: () => 'json', mode: 'advanced', @@ -154,6 +186,12 @@ export const BrightDataBlock: BlockConfig = { placeholder: '[{"url": "https://example.com/product"}]', condition: { field: 'operation', value: 'sync_scrape' }, required: { field: 'operation', value: 'sync_scrape' }, + wandConfig: { + enabled: true, + prompt: + 'Generate a JSON array of URL objects to scrape based on the user\'s description, in the form [{"url": "https://example.com/product"}]. Return ONLY the JSON array - no explanations, no extra text.', + placeholder: 'Describe the URLs to scrape...', + }, }, { id: 'syncFormat', @@ -167,6 +205,13 @@ export const BrightDataBlock: BlockConfig = { value: () => 'json', condition: { field: 'operation', value: 'sync_scrape' }, }, + { + id: 'syncIncludeErrors', + title: 'Include Errors', + type: 'switch', + mode: 'advanced', + condition: { field: 'operation', value: 'sync_scrape' }, + }, { id: 'datasetId', title: 'Dataset ID', @@ -182,6 +227,12 @@ export const BrightDataBlock: BlockConfig = { placeholder: '[{"url": "https://example.com/product"}]', condition: { field: 'operation', value: 'scrape_dataset' }, required: { field: 'operation', value: 'scrape_dataset' }, + wandConfig: { + enabled: true, + prompt: + 'Generate a JSON array of URL objects to scrape based on the user\'s description, in the form [{"url": "https://example.com/product"}]. Return ONLY the JSON array - no explanations, no extra text.', + placeholder: 'Describe the URLs to scrape...', + }, }, { id: 'datasetFormat', @@ -194,6 +245,13 @@ export const BrightDataBlock: BlockConfig = { value: () => 'json', condition: { field: 'operation', value: 'scrape_dataset' }, }, + { + id: 'datasetIncludeErrors', + title: 'Include Errors', + type: 'switch', + mode: 'advanced', + condition: { field: 'operation', value: 'scrape_dataset' }, + }, { id: 'snapshotId', title: 'Snapshot ID', @@ -251,6 +309,7 @@ export const BrightDataBlock: BlockConfig = { result.url = params.url if (params.format) result.format = params.format if (params.country) result.country = params.country + if (params.dataFormat) result.dataFormat = params.dataFormat break case 'serp_search': @@ -265,6 +324,7 @@ export const BrightDataBlock: BlockConfig = { case 'discover': result.query = params.discoverQuery if (params.numResults) result.numResults = Number(params.numResults) + if (params.mode) result.mode = params.mode if (params.intent) result.intent = params.intent if (params.includeContent != null) result.includeContent = params.includeContent if (params.contentFormat) result.format = params.contentFormat @@ -276,12 +336,15 @@ export const BrightDataBlock: BlockConfig = { result.datasetId = params.syncDatasetId result.urls = params.syncUrls if (params.syncFormat) result.format = params.syncFormat + if (params.syncIncludeErrors != null) result.includeErrors = params.syncIncludeErrors break case 'scrape_dataset': result.datasetId = params.datasetId result.urls = params.urls if (params.datasetFormat) result.format = params.datasetFormat + if (params.datasetIncludeErrors != null) + result.includeErrors = params.datasetIncludeErrors break case 'snapshot_status': @@ -308,6 +371,7 @@ export const BrightDataBlock: BlockConfig = { zone: { type: 'string', description: 'Bright Data zone name' }, url: { type: 'string', description: 'URL to scrape' }, format: { type: 'string', description: 'Response format' }, + dataFormat: { type: 'string', description: 'Convert scraped content to markdown' }, country: { type: 'string', description: 'Country code for geo-targeting' }, query: { type: 'string', description: 'Search query' }, searchEngine: { type: 'string', description: 'Search engine to use' }, @@ -315,14 +379,23 @@ export const BrightDataBlock: BlockConfig = { numResults: { type: 'number', description: 'Number of results' }, discoverQuery: { type: 'string', description: 'Discover search query' }, intent: { type: 'string', description: 'Intent for ranking results' }, + mode: { type: 'string', description: 'Search depth and ranking mode for discover' }, includeContent: { type: 'boolean', description: 'Include page content in discover results' }, contentFormat: { type: 'string', description: 'Content format for discover results' }, syncDatasetId: { type: 'string', description: 'Dataset scraper ID for sync scrape' }, syncUrls: { type: 'string', description: 'JSON array of URL objects for sync scrape' }, syncFormat: { type: 'string', description: 'Output format for sync scrape' }, + syncIncludeErrors: { + type: 'boolean', + description: 'Include error reports in sync scrape results', + }, datasetId: { type: 'string', description: 'Dataset scraper ID' }, urls: { type: 'string', description: 'JSON array of URL objects to scrape' }, datasetFormat: { type: 'string', description: 'Dataset output format' }, + datasetIncludeErrors: { + type: 'boolean', + description: 'Include error reports in scrape dataset results', + }, snapshotId: { type: 'string', description: 'Snapshot ID for status/download/cancel' }, downloadFormat: { type: 'string', description: 'Download output format' }, }, @@ -330,11 +403,18 @@ export const BrightDataBlock: BlockConfig = { content: { type: 'string', description: 'Scraped page content' }, url: { type: 'string', description: 'URL that was scraped' }, statusCode: { type: 'number', description: 'HTTP status code' }, - results: { type: 'json', description: 'Search or discover results array' }, + results: { + type: 'json', + description: + 'Search or discover results array: [{title, url, description, rank}] for SERP search, or [{url, title, description, relevanceScore, content}] for discover', + }, query: { type: 'string', description: 'Search query executed' }, searchEngine: { type: 'string', description: 'Search engine used' }, totalResults: { type: 'number', description: 'Total number of discover results' }, - data: { type: 'json', description: 'Scraped data records' }, + data: { + type: 'json', + description: 'Array of scraped result records with dataset-specific fields', + }, snapshotId: { type: 'string', description: 'Snapshot ID' }, isAsync: { type: 'boolean', description: 'Whether sync scrape fell back to async' }, status: { type: 'string', description: 'Job status' }, diff --git a/apps/sim/tools/brightdata/discover.ts b/apps/sim/tools/brightdata/discover.ts index 3a78c84084e..3a4ee9e315c 100644 --- a/apps/sim/tools/brightdata/discover.ts +++ b/apps/sim/tools/brightdata/discover.ts @@ -17,7 +17,7 @@ export const brightDataDiscoverTool: ToolConfig< id: 'brightdata_discover', name: 'Bright Data Discover', description: - 'AI-powered web discovery that finds and ranks results by intent. Returns up to 1,000 results with optional cleaned page content for RAG and verification.', + 'AI-powered web discovery that finds and ranks results by intent. Returns up to 20 results with optional cleaned page content for RAG and verification.', version: '1.0.0', params: { @@ -37,7 +37,14 @@ export const brightDataDiscoverTool: ToolConfig< type: 'number', required: false, visibility: 'user-or-llm', - description: 'Number of results to return, up to 1000. Defaults to 10', + description: 'Number of results to return (1-20). Defaults to 10', + }, + mode: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: + 'Search depth and ranking mode: "standard" (balanced), "deep" (exhaustive, broader search), "fast" (optimized for speed), or "zeroRanking" (raw volume without AI filtering). Defaults to "standard"', }, intent: { type: 'string', @@ -56,7 +63,7 @@ export const brightDataDiscoverTool: ToolConfig< type: 'string', required: false, visibility: 'user-or-llm', - description: 'Response format: "json" or "markdown". Defaults to "json"', + description: 'Response format: "json" or "md". Defaults to "json"', }, language: { type: 'string', @@ -84,6 +91,7 @@ export const brightDataDiscoverTool: ToolConfig< query: params.query, } if (params.numResults) body.num_results = params.numResults + if (params.mode) body.mode = params.mode if (params.intent) body.intent = params.intent if (params.includeContent != null) body.include_content = params.includeContent if (params.format) body.format = params.format diff --git a/apps/sim/tools/brightdata/scrape_dataset.ts b/apps/sim/tools/brightdata/scrape_dataset.ts index f53891d99f4..ea95a55ba60 100644 --- a/apps/sim/tools/brightdata/scrape_dataset.ts +++ b/apps/sim/tools/brightdata/scrape_dataset.ts @@ -41,14 +41,21 @@ export const brightDataScrapeDatasetTool: ToolConfig< visibility: 'user-or-llm', description: 'Output format: "json" or "csv". Defaults to "json"', }, + includeErrors: { + type: 'boolean', + required: false, + visibility: 'user-or-llm', + description: 'Whether to include a per-input error report in the results', + }, }, request: { method: 'POST', url: (params) => { const queryParams = new URLSearchParams() - queryParams.set('dataset_id', params.datasetId) + queryParams.set('dataset_id', params.datasetId.trim()) queryParams.set('format', params.format || 'json') + if (params.includeErrors) queryParams.set('include_errors', 'true') return `https://api.brightdata.com/datasets/v3/trigger?${queryParams.toString()}` }, headers: (params) => ({ diff --git a/apps/sim/tools/brightdata/scrape_url.ts b/apps/sim/tools/brightdata/scrape_url.ts index 1d62d4c6dfe..c183ec1fbee 100644 --- a/apps/sim/tools/brightdata/scrape_url.ts +++ b/apps/sim/tools/brightdata/scrape_url.ts @@ -46,6 +46,13 @@ export const brightDataScrapeUrlTool: ToolConfig< visibility: 'user-or-llm', description: 'Two-letter country code for geo-targeting (e.g., "us", "gb")', }, + dataFormat: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: + 'Convert the response to "markdown" instead of raw HTML, useful for feeding page content to an LLM. Omit for the default HTML/JSON response', + }, }, request: { @@ -62,6 +69,7 @@ export const brightDataScrapeUrlTool: ToolConfig< format: params.format || 'raw', } if (params.country) body.country = params.country + if (params.dataFormat) body.data_format = params.dataFormat return body }, }, diff --git a/apps/sim/tools/brightdata/sync_scrape.ts b/apps/sim/tools/brightdata/sync_scrape.ts index 8ac8d0108cc..af440ddd765 100644 --- a/apps/sim/tools/brightdata/sync_scrape.ts +++ b/apps/sim/tools/brightdata/sync_scrape.ts @@ -53,7 +53,7 @@ export const brightDataSyncScrapeTool: ToolConfig< method: 'POST', url: (params) => { const queryParams = new URLSearchParams() - queryParams.set('dataset_id', params.datasetId) + queryParams.set('dataset_id', params.datasetId.trim()) queryParams.set('format', params.format || 'json') if (params.includeErrors) queryParams.set('include_errors', 'true') return `https://api.brightdata.com/datasets/v3/scrape?${queryParams.toString()}` diff --git a/apps/sim/tools/brightdata/types.ts b/apps/sim/tools/brightdata/types.ts index d0a3e8aa39f..8df36c38267 100644 --- a/apps/sim/tools/brightdata/types.ts +++ b/apps/sim/tools/brightdata/types.ts @@ -6,6 +6,7 @@ export interface BrightDataScrapeUrlParams { url: string format?: string country?: string + dataFormat?: string } export interface BrightDataScrapeUrlResponse extends ToolResponse { @@ -44,6 +45,7 @@ export interface BrightDataScrapeDatasetParams { datasetId: string urls: string format?: string + includeErrors?: boolean } export interface BrightDataScrapeDatasetResponse extends ToolResponse { @@ -113,6 +115,7 @@ export interface BrightDataDiscoverParams { apiKey: string query: string numResults?: number + mode?: string intent?: string includeContent?: boolean format?: string From aef2da23eef04ec1267d673daad1c6509669eeb1 Mon Sep 17 00:00:00 2001 From: waleed Date: Tue, 7 Jul 2026 08:35:50 -0700 Subject: [PATCH 2/2] fix(brightdata): default mode field to empty so it's opt-in like other advanced params Greptile flagged that mode shipped with a non-empty default ('standard'), causing it to be sent on every Discover call unlike the sibling dataFormat field which uses an empty/None default and only sends when the user opts in. Aligns mode with that same pattern. --- apps/sim/blocks/blocks/brightdata.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/sim/blocks/blocks/brightdata.ts b/apps/sim/blocks/blocks/brightdata.ts index 6d4e043e307..b30af8a8052 100644 --- a/apps/sim/blocks/blocks/brightdata.ts +++ b/apps/sim/blocks/blocks/brightdata.ts @@ -143,12 +143,12 @@ export const BrightDataBlock: BlockConfig = { title: 'Search Mode', type: 'dropdown', options: [ - { label: 'Standard', id: 'standard' }, + { label: 'Standard (Default)', id: '' }, { label: 'Deep', id: 'deep' }, { label: 'Fast', id: 'fast' }, { label: 'Zero Ranking', id: 'zeroRanking' }, ], - value: () => 'standard', + value: () => '', mode: 'advanced', condition: { field: 'operation', value: 'discover' }, },