From 3ceac73ab790e66f5cf31ca9390e8aed943d5daa Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 8 Jul 2026 10:59:29 +0200 Subject: [PATCH] test(node-integration-tests): Type-check test suites in CI The `suites/**` test files were never type-checked: `yarn type-check` runs a bare `tsc` (whose config only includes `utils/**` + `src/**`) and vitest transpiles with esbuild, so type errors only ever showed up in the editor. This adds a CI gate that type-checks the suites and fixes the latent errors it surfaced. - Add `type-check:src` / `type-check:test` scripts (both `--noEmit`) and run `yarn type-check` for the package in the Lint CI job. - Relax `noUncheckedIndexedAccess` for test code and exclude the Prisma CLI config (its runtime shape intentionally differs from `@prisma/config`'s types). - Give `createTestServer` an explicit return type so chained `.get()` calls infer their `headers` callback parameter. - Add a `getStringAttributeValue` helper to narrow union-typed span attribute values in the AI tracing assertions. - Fix the vitest reporter config to the valid nested-tuple form. Relies on `attributes` being required on `SerializedStreamedSpan` (see the change this is stacked on). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../node-integration-tests/package.json | 4 ++- .../suites/tracing/anthropic/test.ts | 18 ++++++---- .../suites/tracing/google-genai/test.ts | 16 +++++---- .../suites/tracing/langchain/test.ts | 18 ++++++---- .../suites/tracing/langchain/v1/test.ts | 4 +-- .../suites/tracing/langgraph/test.ts | 23 ++++++++---- .../suites/tracing/openai/test.ts | 35 ++++++++++++------- .../vercelai/span-streaming-v4/test.ts | 16 +++++---- .../suites/tracing/vercelai/test.ts | 24 +++++++++---- .../suites/tracing/vercelai/v5/test.ts | 14 +++++--- .../node-integration-tests/tsconfig.test.json | 12 +++++-- .../node-integration-tests/utils/index.ts | 11 ++++++ .../utils/runner/createRunner.ts | 2 +- .../node-integration-tests/vite.config.ts | 2 +- dev-packages/test-utils/src/server.ts | 8 +++-- 15 files changed, 139 insertions(+), 68 deletions(-) diff --git a/dev-packages/node-integration-tests/package.json b/dev-packages/node-integration-tests/package.json index 90562d768dcd..ea930a64ab5f 100644 --- a/dev-packages/node-integration-tests/package.json +++ b/dev-packages/node-integration-tests/package.json @@ -18,7 +18,9 @@ "clean:script": "node scripts/clean.js", "lint": "OXLINT_TSGOLINT_DANGEROUSLY_SUPPRESS_PROGRAM_DIAGNOSTICS=true oxlint . --type-aware", "lint:fix": "OXLINT_TSGOLINT_DANGEROUSLY_SUPPRESS_PROGRAM_DIAGNOSTICS=true oxlint . --fix --type-aware", - "type-check": "tsc", + "type-check": "run-s type-check:src type-check:test", + "type-check:src": "tsc --noEmit", + "type-check:test": "tsc -p tsconfig.test.json --noEmit", "test": "vitest run", "test:orchestrion": "INJECT_ORCHESTRION=true yarn test", "test:watch": "yarn test --watch" diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts index 633ce2dbe892..a1aeb48f80e3 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts @@ -20,7 +20,7 @@ import { GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, } from '../../../../../packages/core/src/tracing/ai/gen-ai-attributes'; -import { isOrchestrionEnabled } from '../../../utils'; +import { getStringAttributeValue, isOrchestrionEnabled } from '../../../utils'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; describe('Anthropic integration', () => { @@ -580,7 +580,7 @@ describe('Anthropic integration', () => { { role: 'user', content: 'This is a small message that fits within the limit' }, ]); const truncatedSpan = container.items.find(span => - span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.match( + getStringAttributeValue(span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value)?.match( /^\[\{"role":"user","content":"C+"\}\]$/, ), ); @@ -750,7 +750,9 @@ describe('Anthropic integration', () => { const spans = container.items; const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.includes(streamingLongContent), + getStringAttributeValue(s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value)?.includes( + streamingLongContent, + ), ); expect(chatSpan).toBeDefined(); }, @@ -774,12 +776,14 @@ describe('Anthropic integration', () => { // With explicit enableTruncation: true, content should be truncated despite streaming. // Find the chat span by matching the start of the truncated content (the 'A' repeated messages). const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.startsWith('[{"role":"user","content":"AAAA'), + getStringAttributeValue(s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value)?.startsWith( + '[{"role":"user","content":"AAAA', + ), ); expect(chatSpan).toBeDefined(); - expect(chatSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value.length).toBeLessThan( - streamingLongContent.length, - ); + expect( + (getStringAttributeValue(chatSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value) ?? '').length, + ).toBeLessThan(streamingLongContent.length); }, }) .start() diff --git a/dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts b/dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts index 4476945a3e11..023c3eb2c855 100644 --- a/dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts @@ -22,7 +22,7 @@ import { GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, } from '../../../../../packages/core/src/tracing/ai/gen-ai-attributes'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; -import { isOrchestrionEnabled } from '../../../utils'; +import { getStringAttributeValue, isOrchestrionEnabled } from '../../../utils'; const EXPECTED_ORIGIN = isOrchestrionEnabled() ? 'auto.ai.orchestrion.google_genai' : 'auto.ai.google_genai'; @@ -360,7 +360,7 @@ describe('Google GenAI integration', () => { span: container => { expect(container.items).toHaveLength(2); const truncatedSpan = container.items.find(span => - span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.match( + getStringAttributeValue(span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value)?.match( /^\[\{"role":"user","parts":\[\{"text":"C+"\}\]\}\]$/, ), ); @@ -551,7 +551,9 @@ describe('Google GenAI integration', () => { const spans = container.items; const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.includes(streamingLongContent), + getStringAttributeValue(s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value)?.includes( + streamingLongContent, + ), ); expect(chatSpan).toBeDefined(); }, @@ -575,14 +577,14 @@ describe('Google GenAI integration', () => { // With explicit enableTruncation: true, content should be truncated despite streaming. // Find the chat span by matching the start of the truncated content (the 'A' repeated messages). const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.startsWith( + getStringAttributeValue(s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value)?.startsWith( '[{"role":"user","parts":[{"text":"AAAA', ), ); expect(chatSpan).toBeDefined(); - expect(chatSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value.length).toBeLessThan( - streamingLongContent.length, - ); + expect( + (getStringAttributeValue(chatSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value) ?? '').length, + ).toBeLessThan(streamingLongContent.length); }, }) .start() diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts index 0debcae19e1a..71267fabbe08 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts @@ -21,7 +21,7 @@ import { GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, } from '../../../../../packages/core/src/tracing/ai/gen-ai-attributes'; -import { isOrchestrionEnabled } from '../../../utils'; +import { getStringAttributeValue, isOrchestrionEnabled } from '../../../utils'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; import { createEsmTests } from '../../../utils/runner/createEsmAndCjsTests'; @@ -222,7 +222,7 @@ describe('LangChain integration', () => { const arrayInputSpan = container.items.find( span => span.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]?.value === 2 && - span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.match( + getStringAttributeValue(span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value)?.match( /^\[\{"role":"user","content":"C+"\}\]$/, ), ); @@ -495,7 +495,9 @@ describe('LangChain integration', () => { const spans = container.items; const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.includes(streamingLongContent), + getStringAttributeValue(s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value)?.includes( + streamingLongContent, + ), ); expect(chatSpan).toBeDefined(); }, @@ -518,12 +520,14 @@ describe('LangChain integration', () => { // With explicit enableTruncation: true, content should be truncated despite streaming. const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.startsWith('[{"role":"user","content":"AAAA'), + getStringAttributeValue(s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value)?.startsWith( + '[{"role":"user","content":"AAAA', + ), ); expect(chatSpan).toBeDefined(); - expect(chatSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value.length).toBeLessThan( - streamingLongContent.length, - ); + expect( + (getStringAttributeValue(chatSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value) ?? '').length, + ).toBeLessThan(streamingLongContent.length); }, }) .start() diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts index c3f48be36bc0..b555e48229e4 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts @@ -18,7 +18,7 @@ import { GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, } from '../../../../../../packages/core/src/tracing/ai/gen-ai-attributes'; -import { conditionalTest, isOrchestrionEnabled } from '../../../../utils'; +import { conditionalTest, getStringAttributeValue, isOrchestrionEnabled } from '../../../../utils'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; import { createEsmTests } from '../../../../utils/runner/createEsmAndCjsTests'; @@ -238,7 +238,7 @@ conditionalTest({ min: 20 })('LangChain integration (v1)', () => { const arrayInputSpan = container.items.find( span => span.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]?.value === 2 && - span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.match( + getStringAttributeValue(span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value)?.match( /^\[\{"role":"user","content":"C+"\}\]$/, ), ); diff --git a/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts b/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts index 68941c73f291..cb6920133d52 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts @@ -17,6 +17,7 @@ import { GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, } from '../../../../../packages/core/src/tracing/ai/gen-ai-attributes'; +import { getStringAttributeValue } from '../../../utils'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; describe('LangGraph integration', () => { @@ -77,7 +78,9 @@ describe('LangGraph integration', () => { expect(createAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.create_agent'); const weatherTodaySpan = container.items.find(span => - span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.includes('What is the weather today?'), + getStringAttributeValue(span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value)?.includes( + 'What is the weather today?', + ), ); expect(weatherTodaySpan).toBeDefined(); expect(weatherTodaySpan!.name).toBe('invoke_agent weather_assistant'); @@ -86,7 +89,9 @@ describe('LangGraph integration', () => { expect(weatherTodaySpan!.attributes['sentry.origin'].value).toBe('auto.ai.langgraph'); const weatherDetailsSpan = container.items.find(span => - span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.includes('Tell me about the weather'), + getStringAttributeValue(span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value)?.includes( + 'Tell me about the weather', + ), ); expect(weatherDetailsSpan).toBeDefined(); expect(weatherDetailsSpan!.name).toBe('invoke_agent weather_assistant'); @@ -319,7 +324,9 @@ describe('LangGraph integration', () => { const spans = container.items; const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.includes(streamingLongContent), + getStringAttributeValue(s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value)?.includes( + streamingLongContent, + ), ); expect(chatSpan).toBeDefined(); }, @@ -342,12 +349,14 @@ describe('LangGraph integration', () => { // With explicit enableTruncation: true, content should be truncated despite streaming. const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.startsWith('[{"role":"user","content":"AAAA'), + getStringAttributeValue(s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value)?.startsWith( + '[{"role":"user","content":"AAAA', + ), ); expect(chatSpan).toBeDefined(); - expect(chatSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value.length).toBeLessThan( - streamingLongContent.length, - ); + expect( + (getStringAttributeValue(chatSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value) ?? '').length, + ).toBeLessThan(streamingLongContent.length); }, }) .start() diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts index a3952271e472..d5cd027f78dc 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts @@ -22,7 +22,7 @@ import { GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, } from '../../../../../packages/core/src/tracing/ai/gen-ai-attributes'; -import { isOrchestrionEnabled } from '../../../utils'; +import { getStringAttributeValue, isOrchestrionEnabled } from '../../../utils'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; describe('OpenAI integration', () => { @@ -1193,7 +1193,7 @@ describe('OpenAI integration', () => { span: container => { expect(container.items).toHaveLength(2); const truncatedMessageSpan = container.items.find(span => - span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.match( + getStringAttributeValue(span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value)?.match( /^\[\{"role":"user","content":"C+"\}\]$/, ), ); @@ -1651,7 +1651,9 @@ describe('OpenAI integration', () => { span: container => { expect(container.items).toHaveLength(2); const multipleImagesSpan = container.items.find(span => - span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.includes('https://example.com/image.png'), + getStringAttributeValue(span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value)?.includes( + 'https://example.com/image.png', + ), ); expect(multipleImagesSpan).toBeDefined(); expect(multipleImagesSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toContain( @@ -1675,12 +1677,16 @@ describe('OpenAI integration', () => { const spans = container.items; const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.includes(streamingLongContent), + getStringAttributeValue(s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value)?.includes( + streamingLongContent, + ), ); expect(chatSpan).toBeDefined(); const responsesSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.includes(streamingLongString), + getStringAttributeValue(s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value)?.includes( + streamingLongString, + ), ); expect(responsesSpan).toBeDefined(); }, @@ -1704,21 +1710,24 @@ describe('OpenAI integration', () => { // With explicit enableTruncation: true, content should be truncated despite streaming. // Truncation keeps only the last message (50k 'A's) and crops it to the byte limit. const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.startsWith('[{"role":"user","content":"AAAA'), + getStringAttributeValue(s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value)?.startsWith( + '[{"role":"user","content":"AAAA', + ), ); expect(chatSpan).toBeDefined(); - expect(chatSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value.length).toBeLessThan( - streamingLongContent.length, - ); + expect( + (getStringAttributeValue(chatSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value) ?? '').length, + ).toBeLessThan(streamingLongContent.length); // The responses API string input (50k 'B's) should also be truncated. const responsesSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.startsWith('BBB'), + getStringAttributeValue(s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value)?.startsWith('BBB'), ); expect(responsesSpan).toBeDefined(); - expect(responsesSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value.length).toBeLessThan( - streamingLongString.length, - ); + expect( + (getStringAttributeValue(responsesSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value) ?? '') + .length, + ).toBeLessThan(streamingLongString.length); }, }) .start() diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts index a5219f76a88e..dc621a182bc6 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts @@ -19,7 +19,7 @@ import { GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, } from '../../../../../../packages/core/src/tracing/ai/gen-ai-attributes'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; -import { isOrchestrionEnabled } from '../../../../utils'; +import { getStringAttributeValue, isOrchestrionEnabled } from '../../../../utils'; /** * Helper to match a typed attribute value in a SerializedStreamedSpan. @@ -314,7 +314,9 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)', const spans = container.items; const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.includes(streamingLongContent), + getStringAttributeValue(s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value)?.includes( + streamingLongContent, + ), ); expect(chatSpan).toBeDefined(); }, @@ -333,12 +335,14 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)', // With explicit enableTruncation: true, content should be truncated despite streaming. const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.startsWith('[{"role":"user","content":"AAAA'), + getStringAttributeValue(s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value)?.startsWith( + '[{"role":"user","content":"AAAA', + ), ); expect(chatSpan).toBeDefined(); - expect(chatSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value.length).toBeLessThan( - streamingLongContent.length, - ); + expect( + (getStringAttributeValue(chatSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value) ?? '').length, + ).toBeLessThan(streamingLongContent.length); }, }) .start() diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts index b559610b81a9..e96c9dcf477a 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts @@ -22,7 +22,7 @@ import { GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, } from '../../../../../packages/core/src/tracing/ai/gen-ai-attributes'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; -import { isOrchestrionEnabled } from '../../../utils'; +import { getStringAttributeValue, isOrchestrionEnabled } from '../../../utils'; describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (v4)', () => { afterAll(() => { @@ -93,7 +93,9 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (v4)', () => { const secondGenerateContentSpan = container.items.find( span => span.name === 'generate_content mock-model-id' && - span.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value?.includes('Second span here!'), + getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value)?.includes( + 'Second span here!', + ), ); expect(secondGenerateContentSpan).toBeDefined(); expect(secondGenerateContentSpan!.name).toBe('generate_content mock-model-id'); @@ -170,7 +172,9 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (v4)', () => { const firstGenerateContentSpan = container.items.find( span => span.name === 'generate_content mock-model-id' && - span.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value?.includes('First span here!'), + getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value)?.includes( + 'First span here!', + ), ); expect(firstGenerateContentSpan).toBeDefined(); expect(firstGenerateContentSpan!.name).toBe('generate_content mock-model-id'); @@ -201,7 +205,9 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (v4)', () => { const secondGenerateContentSpan = container.items.find( span => span.name === 'generate_content mock-model-id' && - span.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value?.includes('Second span here!'), + getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value)?.includes( + 'Second span here!', + ), ); expect(secondGenerateContentSpan).toBeDefined(); expect(secondGenerateContentSpan!.name).toBe('generate_content mock-model-id'); @@ -226,7 +232,9 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (v4)', () => { const toolGenerateContentSpan = container.items.find( span => span.name === 'generate_content mock-model-id' && - span.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]?.value?.includes('getWeather'), + getStringAttributeValue(span.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]?.value)?.includes( + 'getWeather', + ), ); expect(toolGenerateContentSpan).toBeDefined(); expect(toolGenerateContentSpan!.name).toBe('generate_content mock-model-id'); @@ -459,7 +467,9 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (v4)', () => { const truncatedInvokeAgentSpan = container.items.find( span => span.name === 'invoke_agent' && - span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.match(/^\[.*"(?:text|content)":"C+".*\]$/), + getStringAttributeValue(span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value)?.match( + /^\[.*"(?:text|content)":"C+".*\]$/, + ), ); expect(truncatedInvokeAgentSpan).toBeDefined(); expect(truncatedInvokeAgentSpan!.name).toBe('invoke_agent'); @@ -474,7 +484,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (v4)', () => { const smallMessageInvokeAgentSpan = container.items.find( span => span.name === 'invoke_agent' && - span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.includes( + getStringAttributeValue(span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value)?.includes( 'This is a small message that fits within the limit', ), ); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts index 3fc8edf15566..87ca0c5861f3 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts @@ -18,7 +18,7 @@ import { GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, } from '../../../../../../packages/core/src/tracing/ai/gen-ai-attributes'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; -import { isOrchestrionEnabled } from '../../../../utils'; +import { getStringAttributeValue, isOrchestrionEnabled } from '../../../../utils'; const expectedOrigin = isOrchestrionEnabled() ? 'auto.vercelai.channel' : 'auto.vercelai.otel'; @@ -96,7 +96,9 @@ describe('Vercel AI integration (v5)', () => { const secondGenerateContentSpan = container.items.find( span => span.name === 'generate_content mock-model-id' && - span.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value?.includes('Second span here!'), + getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value)?.includes( + 'Second span here!', + ), ); expect(secondGenerateContentSpan).toBeDefined(); expect(secondGenerateContentSpan!.name).toBe('generate_content mock-model-id'); @@ -174,7 +176,9 @@ describe('Vercel AI integration (v5)', () => { const firstGenerateContentSpan = container.items.find( span => span.name === 'generate_content mock-model-id' && - span.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value?.includes('First span here!'), + getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value)?.includes( + 'First span here!', + ), ); expect(firstGenerateContentSpan).toBeDefined(); expect(firstGenerateContentSpan!.name).toBe('generate_content mock-model-id'); @@ -205,7 +209,9 @@ describe('Vercel AI integration (v5)', () => { const secondGenerateContentSpan = container.items.find( span => span.name === 'generate_content mock-model-id' && - span.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value?.includes('Second span here!'), + getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value)?.includes( + 'Second span here!', + ), ); expect(secondGenerateContentSpan).toBeDefined(); expect(secondGenerateContentSpan!.name).toBe('generate_content mock-model-id'); diff --git a/dev-packages/node-integration-tests/tsconfig.test.json b/dev-packages/node-integration-tests/tsconfig.test.json index 89f44d610021..b41c7de7464b 100644 --- a/dev-packages/node-integration-tests/tsconfig.test.json +++ b/dev-packages/node-integration-tests/tsconfig.test.json @@ -3,13 +3,21 @@ "include": ["suites/**/*.ts", "vite.config.ts"], + // `prisma.config.ts` files are consumed by the Prisma CLI, not the tests. Their runtime config + // shape (e.g. the `datasource` field) intentionally differs from `@prisma/config`'s exported + // types, so type-checking them here would produce false positives without protecting test code. + "exclude": ["suites/**/prisma.config.ts"], + "compilerOptions": { // Although this seems wrong to include `DOM` here, it's necessary to make // global fetch available in tests in lower Node versions. "lib": ["DOM", "es2020"], // should include all types from `./tsconfig.json` plus types for all test frameworks used - "types": ["node"] + "types": ["node"], - // other package-specific, test-specific options + // Test assertions routinely index into span attributes/arrays that are known to exist at that + // point (e.g. `span.attributes[KEY].value`); requiring `!` on every such access adds noise + // without catching real bugs in test code. + "noUncheckedIndexedAccess": false } } diff --git a/dev-packages/node-integration-tests/utils/index.ts b/dev-packages/node-integration-tests/utils/index.ts index c41f365ba8db..705921cd1b6e 100644 --- a/dev-packages/node-integration-tests/utils/index.ts +++ b/dev-packages/node-integration-tests/utils/index.ts @@ -60,3 +60,14 @@ export const parseEnvelope = (body: string): Array> => { export function isOrchestrionEnabled(): boolean { return process.env.INJECT_ORCHESTRION === 'true' || process.env.INJECT_ORCHESTRION === '1'; } + +/** + * Narrows a typed span attribute value to a string. + * + * Streamed span attribute values are a union (`string | number | boolean | string[] | ...`), so + * assertions that call string methods (`includes`, `startsWith`, `match`, `length`) need the value + * narrowed first. Returns `undefined` if the value is not a string. + */ +export function getStringAttributeValue(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined; +} diff --git a/dev-packages/node-integration-tests/utils/runner/createRunner.ts b/dev-packages/node-integration-tests/utils/runner/createRunner.ts index 2d450c10cd37..a164874883a3 100644 --- a/dev-packages/node-integration-tests/utils/runner/createRunner.ts +++ b/dev-packages/node-integration-tests/utils/runner/createRunner.ts @@ -48,7 +48,7 @@ interface DockerOptions { type VoidFunction = () => void; type ExpectedEvent = Partial | ((event: Event) => void); -type ExpectedTransaction = Partial | ((event: TransactionEvent) => void); +type ExpectedTransaction = DeepPartial | ((event: TransactionEvent) => void); type ExpectedSession = Partial | ((event: SerializedSession) => void); type ExpectedSessions = Partial | ((event: SessionAggregates) => void); type ExpectedCheckIn = Partial | ((event: SerializedCheckIn) => void); diff --git a/dev-packages/node-integration-tests/vite.config.ts b/dev-packages/node-integration-tests/vite.config.ts index 4b2c3b2a0a74..5a8b87af8dce 100644 --- a/dev-packages/node-integration-tests/vite.config.ts +++ b/dev-packages/node-integration-tests/vite.config.ts @@ -23,7 +23,7 @@ export default defineConfig({ // overhead is significantly less. pool: 'threads', reporters: process.env.DEBUG - ? ['default', { summary: false }] + ? [['default', { summary: false }]] : process.env.GITHUB_ACTIONS ? ['dot', 'github-actions'] : ['verbose'], diff --git a/dev-packages/test-utils/src/server.ts b/dev-packages/test-utils/src/server.ts index 178fa04375d7..3fef94be89a0 100644 --- a/dev-packages/test-utils/src/server.ts +++ b/dev-packages/test-utils/src/server.ts @@ -50,10 +50,10 @@ export function createTestServer(): TestServer { const gets: Array<[string, HeaderAssertCallback, number]> = []; let error: unknown | undefined; - return { - get: function (path: string, callback: HeaderAssertCallback, result = 200) { + const server: TestServer = { + get(path: string, callback: HeaderAssertCallback, result = 200): TestServer { gets.push([path, callback, result]); - return this; + return server; }, start: async (): Promise<[string, () => void]> => { const app = express(); @@ -86,4 +86,6 @@ export function createTestServer(): TestServer { }); }, }; + + return server; }