From 1f81937bd4071323fc3fc2838c7d12c2627a8778 Mon Sep 17 00:00:00 2001 From: Erica Pisani Date: Thu, 6 Aug 2026 13:31:05 -0400 Subject: [PATCH 1/2] feat(anthropic): Gate output collection on data_collection option Output data (text and tool calls) is now gated on the data_collection configuration, mirroring input collection behavior. When data_collection is configured, its gen_ai.outputs setting is respected. When not configured, falls back to the legacy send_default_pii and include_prompts behavior. Refs PY-2588 --- sentry_sdk/integrations/anthropic.py | 11 +- .../integrations/anthropic/test_anthropic.py | 903 +++++++++++++++++- 2 files changed, 910 insertions(+), 4 deletions(-) diff --git a/sentry_sdk/integrations/anthropic.py b/sentry_sdk/integrations/anthropic.py index f632227cde..cc1d468be9 100644 --- a/sentry_sdk/integrations/anthropic.py +++ b/sentry_sdk/integrations/anthropic.py @@ -606,7 +606,16 @@ def _set_output_data( set_on_span(SPANDATA.GEN_AI_RESPONSE_ID, response_id) if finish_reason is not None: set_on_span(SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS, [finish_reason]) - if should_send_default_pii() and integration.include_prompts: + + client = sentry_sdk.get_client() + record_outputs = False + if has_data_collection_enabled(client.options): + if client.options["data_collection"]["gen_ai"]["outputs"]: + record_outputs = True + elif should_send_default_pii() and integration.include_prompts: + record_outputs = True + + if record_outputs: output_messages: "dict[str, list[Any]]" = { "response": [], "tool": [], diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 009dd476ca..2871a0a1bd 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -106,6 +106,42 @@ async def __call__(self, *args, **kwargs): DATA_COLLECTION_INPUT_DATA_KEYS = list(DATA_COLLECTION_EXPECTED_INPUT_DATA.keys()) +DATA_COLLECTION_EXPECTED_RESPONSE_TEXT = "Let me check the weather." + +DATA_COLLECTION_EXPECTED_TOOL_CALLS = [ + { + "id": "toolu_01A09q90qw90lq917835lq9", + "input": {"location": "San Francisco, CA"}, + "name": "get_weather", + "type": "tool_use", + } +] + + +def data_collection_tool_use_message(): + """ + A response containing both a text block and a tool use block. + + Built lazily because ToolUseBlock is only importable on anthropic 0.27+. + """ + return Message( + id="msg_01XFDUDYJgAACzvnptvVoYEL", + model="model", + role="assistant", + content=[ + TextBlock(type="text", text=DATA_COLLECTION_EXPECTED_RESPONSE_TEXT), + ToolUseBlock( + id="toolu_01A09q90qw90lq917835lq9", + input={"location": "San Francisco, CA"}, + name="get_weather", + type="tool_use", + ), + ], + type="message", + stop_reason="tool_use", + usage=Usage(input_tokens=10, output_tokens=20), + ) + @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) @@ -601,6 +637,247 @@ async def test_nonstreaming_create_message_data_collection_async( assert key not in span_data +@pytest.mark.skipif( + ANTHROPIC_VERSION < (0, 27), + reason="anthropic.types.ToolUseBlock was added in 0.27.0. Before that, tool use was only available under the beta namespace and could not appear in a standard Message.", +) +@pytest.mark.parametrize("span_streaming", [True, False]) +@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) +@pytest.mark.parametrize( + "data_collection,send_default_pii,include_prompts,outputs_collected", + [ + pytest.param( + {"gen_ai": {"outputs": True}}, + False, + False, + True, + id="gen-ai-outputs-enabled-overrides-pii-and-include-prompts", + ), + pytest.param( + {"gen_ai": {"outputs": False}}, + True, + True, + False, + id="gen-ai-outputs-disabled-overrides-pii-and-include-prompts", + ), + pytest.param( + {"gen_ai": {}}, + False, + False, + True, + id="gen-ai-outputs-omitted-defaults-to-enabled", + ), + pytest.param( + None, + True, + True, + True, + id="legacy-pii-and-include-prompts-enabled", + ), + pytest.param( + None, + False, + True, + False, + id="legacy-pii-disabled", + ), + ], +) +def test_nonstreaming_create_message_data_collection_outputs( + sentry_init, + capture_events, + capture_items, + data_collection, + send_default_pii, + include_prompts, + outputs_collected, + stream_gen_ai_spans, + span_streaming, +): + sentry_init_kwargs = dict( + integrations=[AnthropicIntegration(include_prompts=include_prompts)], + disabled_integrations=[StdlibIntegration], + traces_sample_rate=1.0, + send_default_pii=send_default_pii, + stream_gen_ai_spans=stream_gen_ai_spans, + trace_lifecycle="stream" if span_streaming else "static", + ) + if data_collection is not None: + sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} + sentry_init(**sentry_init_kwargs) + + client = Anthropic(api_key="z") + client.messages._post = mock.Mock(return_value=data_collection_tool_use_message()) + + create_kwargs = dict( + max_tokens=1024, + model="model", + messages=[{"role": "user", "content": "What is the weather in San Francisco?"}], + tools=DATA_COLLECTION_EXAMPLE_TOOLS, + ) + + if span_streaming or stream_gen_ai_spans: + items = capture_items("transaction", "span") + + with start_transaction(name="anthropic"): + client.messages.create(**create_kwargs) + + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] + else: + events = capture_events() + + with start_transaction(name="anthropic"): + client.messages.create(**create_kwargs) + + (event,) = events + (span,) = event["spans"] + span_data = span["data"] + + # Output data that is not gated on data collection + assert span_data[SPANDATA.GEN_AI_RESPONSE_MODEL] == "model" + assert span_data[SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" + assert span_data[SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["tool_use"] + assert span_data[SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span_data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 + + if outputs_collected: + assert ( + span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] + == DATA_COLLECTION_EXPECTED_RESPONSE_TEXT + ) + assert ( + json.loads(span_data[SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]) + == DATA_COLLECTION_EXPECTED_TOOL_CALLS + ) + else: + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span_data + + +@pytest.mark.skipif( + ANTHROPIC_VERSION < (0, 27), + reason="anthropic.types.ToolUseBlock was added in 0.27.0. Before that, tool use was only available under the beta namespace and could not appear in a standard Message.", +) +@pytest.mark.parametrize("span_streaming", [True, False]) +@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "data_collection,send_default_pii,include_prompts,outputs_collected", + [ + pytest.param( + {"gen_ai": {"outputs": True}}, + False, + False, + True, + id="gen-ai-outputs-enabled-overrides-pii-and-include-prompts", + ), + pytest.param( + {"gen_ai": {"outputs": False}}, + True, + True, + False, + id="gen-ai-outputs-disabled-overrides-pii-and-include-prompts", + ), + pytest.param( + {"gen_ai": {}}, + False, + False, + True, + id="gen-ai-outputs-omitted-defaults-to-enabled", + ), + pytest.param( + None, + True, + True, + True, + id="legacy-pii-and-include-prompts-enabled", + ), + pytest.param( + None, + False, + True, + False, + id="legacy-pii-disabled", + ), + ], +) +async def test_nonstreaming_create_message_data_collection_outputs_async( + sentry_init, + capture_events, + capture_items, + data_collection, + send_default_pii, + include_prompts, + outputs_collected, + stream_gen_ai_spans, + span_streaming, +): + sentry_init_kwargs = dict( + integrations=[AnthropicIntegration(include_prompts=include_prompts)], + disabled_integrations=[StdlibIntegration], + traces_sample_rate=1.0, + send_default_pii=send_default_pii, + stream_gen_ai_spans=stream_gen_ai_spans, + trace_lifecycle="stream" if span_streaming else "static", + ) + if data_collection is not None: + sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} + sentry_init(**sentry_init_kwargs) + + client = AsyncAnthropic(api_key="z") + client.messages._post = AsyncMock(return_value=data_collection_tool_use_message()) + + create_kwargs = dict( + max_tokens=1024, + model="model", + messages=[{"role": "user", "content": "What is the weather in San Francisco?"}], + tools=DATA_COLLECTION_EXAMPLE_TOOLS, + ) + + if span_streaming or stream_gen_ai_spans: + items = capture_items("transaction", "span") + + with start_transaction(name="anthropic"): + await client.messages.create(**create_kwargs) + + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] + else: + events = capture_events() + + with start_transaction(name="anthropic"): + await client.messages.create(**create_kwargs) + + (event,) = events + (span,) = event["spans"] + span_data = span["data"] + + # Output data that is not gated on data collection + assert span_data[SPANDATA.GEN_AI_RESPONSE_MODEL] == "model" + assert span_data[SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" + assert span_data[SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["tool_use"] + assert span_data[SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span_data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 + + if outputs_collected: + assert ( + span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] + == DATA_COLLECTION_EXPECTED_RESPONSE_TEXT + ) + assert ( + json.loads(span_data[SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]) + == DATA_COLLECTION_EXPECTED_TOOL_CALLS + ) + else: + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span_data + + @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) @pytest.mark.asyncio @@ -1191,15 +1468,71 @@ def test_streaming_create_message_data_collection( @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) -def test_streaming_create_message_close( +@pytest.mark.parametrize( + "data_collection,send_default_pii,include_prompts,outputs_collected", + [ + pytest.param( + {"gen_ai": {"outputs": True}}, + False, + False, + True, + id="gen-ai-outputs-enabled-overrides-pii-and-include-prompts", + ), + pytest.param( + {"gen_ai": {"outputs": False}}, + True, + True, + False, + id="gen-ai-outputs-disabled-overrides-pii-and-include-prompts", + ), + pytest.param( + {"gen_ai": {}}, + False, + False, + True, + id="gen-ai-outputs-omitted-defaults-to-enabled", + ), + pytest.param( + None, + True, + True, + True, + id="legacy-pii-and-include-prompts-enabled", + ), + pytest.param( + None, + False, + True, + False, + id="legacy-pii-disabled", + ), + ], +) +def test_streaming_create_message_data_collection_outputs( sentry_init, capture_events, capture_items, + data_collection, + send_default_pii, + include_prompts, + outputs_collected, get_model_response, server_side_event_chunks, stream_gen_ai_spans, span_streaming, ): + sentry_init_kwargs = dict( + integrations=[AnthropicIntegration(include_prompts=include_prompts)], + disabled_integrations=[StdlibIntegration], + traces_sample_rate=1.0, + send_default_pii=send_default_pii, + stream_gen_ai_spans=stream_gen_ai_spans, + trace_lifecycle="stream" if span_streaming else "static", + ) + if data_collection is not None: + sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} + sentry_init(**sentry_init_kwargs) + client = Anthropic(api_key="z") response = get_model_response( @@ -1220,11 +1553,108 @@ def test_streaming_create_message_close( type="content_block_delta", ), ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), + delta=TextDelta(text="! I'm Claude!", type="text_delta"), index=0, type="content_block_delta", ), - ContentBlockDeltaEvent( + ContentBlockStopEvent(type="content_block_stop", index=0), + MessageDeltaEvent( + delta=Delta(stop_reason="max_tokens"), + usage=MessageDeltaUsage(output_tokens=10), + type="message_delta", + ), + ] + ) + ) + + create_kwargs = dict( + max_tokens=1024, + model="model", + messages=[{"role": "user", "content": "Hello, Claude"}], + stream=True, + ) + + if span_streaming or stream_gen_ai_spans: + items = capture_items("transaction", "span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ), start_transaction(name="anthropic"): + message = client.messages.create(**create_kwargs) + for _ in message: + pass + + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] + else: + events = capture_events() + + with mock.patch.object( + client._client, + "send", + return_value=response, + ), start_transaction(name="anthropic"): + message = client.messages.create(**create_kwargs) + for _ in message: + pass + + (event,) = events + span = next(s for s in event["spans"] if s["op"] == OP.GEN_AI_CHAT) + span_data = span["data"] + + # Output data that is not gated on data collection + assert span_data[SPANDATA.GEN_AI_RESPONSE_MODEL] == "model" + assert span_data[SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" + assert span_data[SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["max_tokens"] + assert span_data[SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span_data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 + + if outputs_collected: + assert span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" + else: + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + + +@pytest.mark.parametrize("span_streaming", [True, False]) +@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) +def test_streaming_create_message_close( + sentry_init, + capture_events, + capture_items, + get_model_response, + server_side_event_chunks, + stream_gen_ai_spans, + span_streaming, +): + client = Anthropic(api_key="z") + + response = get_model_response( + server_side_event_chunks( + [ + MessageStartEvent( + message=EXAMPLE_MESSAGE, + type="message_start", + ), + ContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=TextBlock(type="text", text=""), + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="Hi", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="!", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( delta=TextDelta(text=" I'm Claude!", type="text_delta"), index=0, type="content_block_delta", @@ -1843,6 +2273,160 @@ def test_stream_messages( assert span["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["max_tokens"] +@pytest.mark.parametrize("span_streaming", [True, False]) +@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) +@pytest.mark.parametrize( + "data_collection,send_default_pii,include_prompts,outputs_collected", + [ + pytest.param( + {"gen_ai": {"outputs": True}}, + False, + False, + True, + id="gen-ai-outputs-enabled-overrides-pii-and-include-prompts", + ), + pytest.param( + {"gen_ai": {"outputs": False}}, + True, + True, + False, + id="gen-ai-outputs-disabled-overrides-pii-and-include-prompts", + ), + pytest.param( + {"gen_ai": {}}, + False, + False, + True, + id="gen-ai-outputs-omitted-defaults-to-enabled", + ), + pytest.param( + None, + True, + True, + True, + id="legacy-pii-and-include-prompts-enabled", + ), + pytest.param( + None, + False, + True, + False, + id="legacy-pii-disabled", + ), + ], +) +def test_stream_messages_data_collection_outputs( + sentry_init, + capture_events, + capture_items, + data_collection, + send_default_pii, + include_prompts, + outputs_collected, + get_model_response, + server_side_event_chunks, + stream_gen_ai_spans, + span_streaming, +): + sentry_init_kwargs = dict( + integrations=[AnthropicIntegration(include_prompts=include_prompts)], + disabled_integrations=[StdlibIntegration], + traces_sample_rate=1.0, + send_default_pii=send_default_pii, + stream_gen_ai_spans=stream_gen_ai_spans, + trace_lifecycle="stream" if span_streaming else "static", + ) + if data_collection is not None: + sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} + sentry_init(**sentry_init_kwargs) + + client = Anthropic(api_key="z") + + response = get_model_response( + server_side_event_chunks( + [ + MessageStartEvent( + message=EXAMPLE_MESSAGE, + type="message_start", + ), + ContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=TextBlock(type="text", text=""), + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="Hi", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="! I'm Claude!", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockStopEvent(type="content_block_stop", index=0), + MessageDeltaEvent( + delta=Delta(stop_reason="max_tokens"), + usage=MessageDeltaUsage(output_tokens=10), + type="message_delta", + ), + ] + ) + ) + + stream_kwargs = dict( + max_tokens=1024, + model="model", + messages=[{"role": "user", "content": "Hello, Claude"}], + ) + + if span_streaming or stream_gen_ai_spans: + items = capture_items("transaction", "span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ), start_transaction(name="anthropic"), client.messages.stream( + **stream_kwargs + ) as stream: + for _ in stream: + pass + + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] + else: + events = capture_events() + + with mock.patch.object( + client._client, + "send", + return_value=response, + ), start_transaction(name="anthropic"), client.messages.stream( + **stream_kwargs + ) as stream: + for _ in stream: + pass + + (event,) = events + span = next(s for s in event["spans"] if s["op"] == OP.GEN_AI_CHAT) + span_data = span["data"] + + # Output data that is not gated on data collection + assert span_data[SPANDATA.GEN_AI_RESPONSE_MODEL] == "model" + assert span_data[SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" + assert span_data[SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["max_tokens"] + assert span_data[SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span_data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 + + if outputs_collected: + assert span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" + else: + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + + @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) def test_stream_messages_close( @@ -2516,6 +3100,163 @@ async def test_streaming_create_message_async( assert span["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["max_tokens"] +@pytest.mark.parametrize("span_streaming", [True, False]) +@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "data_collection,send_default_pii,include_prompts,outputs_collected", + [ + pytest.param( + {"gen_ai": {"outputs": True}}, + False, + False, + True, + id="gen-ai-outputs-enabled-overrides-pii-and-include-prompts", + ), + pytest.param( + {"gen_ai": {"outputs": False}}, + True, + True, + False, + id="gen-ai-outputs-disabled-overrides-pii-and-include-prompts", + ), + pytest.param( + {"gen_ai": {}}, + False, + False, + True, + id="gen-ai-outputs-omitted-defaults-to-enabled", + ), + pytest.param( + None, + True, + True, + True, + id="legacy-pii-and-include-prompts-enabled", + ), + pytest.param( + None, + False, + True, + False, + id="legacy-pii-disabled", + ), + ], +) +async def test_streaming_create_message_data_collection_outputs_async( + sentry_init, + capture_events, + capture_items, + data_collection, + send_default_pii, + include_prompts, + outputs_collected, + get_model_response, + async_iterator, + server_side_event_chunks, + stream_gen_ai_spans, + span_streaming, +): + sentry_init_kwargs = dict( + integrations=[AnthropicIntegration(include_prompts=include_prompts)], + disabled_integrations=[StdlibIntegration], + traces_sample_rate=1.0, + send_default_pii=send_default_pii, + stream_gen_ai_spans=stream_gen_ai_spans, + trace_lifecycle="stream" if span_streaming else "static", + ) + if data_collection is not None: + sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} + sentry_init(**sentry_init_kwargs) + + client = AsyncAnthropic(api_key="z") + + response = get_model_response( + async_iterator( + server_side_event_chunks( + [ + MessageStartEvent( + message=EXAMPLE_MESSAGE, + type="message_start", + ), + ContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=TextBlock(type="text", text=""), + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="Hi", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="! I'm Claude!", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockStopEvent(type="content_block_stop", index=0), + MessageDeltaEvent( + delta=Delta(stop_reason="max_tokens"), + usage=MessageDeltaUsage(output_tokens=10), + type="message_delta", + ), + ] + ) + ), + ) + + create_kwargs = dict( + max_tokens=1024, + model="model", + messages=[{"role": "user", "content": "Hello, Claude"}], + stream=True, + ) + + if span_streaming or stream_gen_ai_spans: + items = capture_items("transaction", "span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ), start_transaction(name="anthropic"): + message = await client.messages.create(**create_kwargs) + async for _ in message: + pass + + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] + else: + events = capture_events() + + with mock.patch.object( + client._client, + "send", + return_value=response, + ), start_transaction(name="anthropic"): + message = await client.messages.create(**create_kwargs) + async for _ in message: + pass + + (event,) = events + span = next(s for s in event["spans"] if s["op"] == OP.GEN_AI_CHAT) + span_data = span["data"] + + # Output data that is not gated on data collection + assert span_data[SPANDATA.GEN_AI_RESPONSE_MODEL] == "model" + assert span_data[SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" + assert span_data[SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["max_tokens"] + assert span_data[SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span_data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 + + if outputs_collected: + assert span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" + else: + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + + @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) @pytest.mark.asyncio @@ -3176,6 +3917,162 @@ async def test_stream_message_async( ) +@pytest.mark.parametrize("span_streaming", [True, False]) +@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "data_collection,send_default_pii,include_prompts,outputs_collected", + [ + pytest.param( + {"gen_ai": {"outputs": True}}, + False, + False, + True, + id="gen-ai-outputs-enabled-overrides-pii-and-include-prompts", + ), + pytest.param( + {"gen_ai": {"outputs": False}}, + True, + True, + False, + id="gen-ai-outputs-disabled-overrides-pii-and-include-prompts", + ), + pytest.param( + {"gen_ai": {}}, + False, + False, + True, + id="gen-ai-outputs-omitted-defaults-to-enabled", + ), + pytest.param( + None, + True, + True, + True, + id="legacy-pii-and-include-prompts-enabled", + ), + pytest.param( + None, + False, + True, + False, + id="legacy-pii-disabled", + ), + ], +) +async def test_stream_messages_data_collection_outputs_async( + sentry_init, + capture_events, + capture_items, + data_collection, + send_default_pii, + include_prompts, + outputs_collected, + get_model_response, + async_iterator, + server_side_event_chunks, + stream_gen_ai_spans, + span_streaming, +): + sentry_init_kwargs = dict( + integrations=[AnthropicIntegration(include_prompts=include_prompts)], + disabled_integrations=[StdlibIntegration], + traces_sample_rate=1.0, + send_default_pii=send_default_pii, + stream_gen_ai_spans=stream_gen_ai_spans, + trace_lifecycle="stream" if span_streaming else "static", + ) + if data_collection is not None: + sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} + sentry_init(**sentry_init_kwargs) + + client = AsyncAnthropic(api_key="z") + + response = get_model_response( + async_iterator( + server_side_event_chunks( + [ + MessageStartEvent( + message=EXAMPLE_MESSAGE, + type="message_start", + ), + ContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=TextBlock(type="text", text=""), + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="Hi", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="! I'm Claude!", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockStopEvent(type="content_block_stop", index=0), + MessageDeltaEvent( + delta=Delta(stop_reason="max_tokens"), + usage=MessageDeltaUsage(output_tokens=10), + type="message_delta", + ), + ] + ) + ), + ) + + stream_kwargs = dict( + max_tokens=1024, + model="model", + messages=[{"role": "user", "content": "Hello, Claude"}], + ) + + if span_streaming or stream_gen_ai_spans: + items = capture_items("transaction", "span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ), start_transaction(name="anthropic"): + async with client.messages.stream(**stream_kwargs) as stream: + async for _ in stream: + pass + + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] + else: + events = capture_events() + + with mock.patch.object( + client._client, + "send", + return_value=response, + ), start_transaction(name="anthropic"): + async with client.messages.stream(**stream_kwargs) as stream: + async for _ in stream: + pass + + (event,) = events + span = next(s for s in event["spans"] if s["op"] == OP.GEN_AI_CHAT) + span_data = span["data"] + + # Output data that is not gated on data collection + assert span_data[SPANDATA.GEN_AI_RESPONSE_MODEL] == "model" + assert span_data[SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" + assert span_data[SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["max_tokens"] + assert span_data[SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span_data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 + + if outputs_collected: + assert span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" + else: + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + + @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) @pytest.mark.skipif( From afdf68530f7842b73c4f578f94c9765aa82724c7 Mon Sep 17 00:00:00 2001 From: Erica Pisani Date: Thu, 6 Aug 2026 13:40:20 -0400 Subject: [PATCH 2/2] remove unnecessary comment --- tests/integrations/anthropic/test_anthropic.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 2871a0a1bd..e57afd67e6 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -119,11 +119,6 @@ async def __call__(self, *args, **kwargs): def data_collection_tool_use_message(): - """ - A response containing both a text block and a tool use block. - - Built lazily because ToolUseBlock is only importable on anthropic 0.27+. - """ return Message( id="msg_01XFDUDYJgAACzvnptvVoYEL", model="model",