From 4f25a56b1cb514a493c4eb6a278b2ff66cefe8e3 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 21 Jan 2026 18:40:02 +0100 Subject: [PATCH 01/22] feat(openai): Set system instruction attribute --- sentry_sdk/ai/_openai_completions_api.py | 20 +++++++++++ sentry_sdk/consts.py | 6 ++++ sentry_sdk/integrations/openai.py | 42 +++++++++++++++++++++++- 3 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 sentry_sdk/ai/_openai_completions_api.py diff --git a/sentry_sdk/ai/_openai_completions_api.py b/sentry_sdk/ai/_openai_completions_api.py new file mode 100644 index 0000000000..6697f285c6 --- /dev/null +++ b/sentry_sdk/ai/_openai_completions_api.py @@ -0,0 +1,20 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from openai.types.chat import ( + ChatCompletionMessageParam, + ChatCompletionSystemMessageParam, + ) + from typing import Iterable, Union + + +def _get_system_instructions( + messages: "Iterable[Union[ChatCompletionMessageParam, str]]", +) -> "list[ChatCompletionSystemMessageParam]": + system_messages = [] + + for message in messages: + if isinstance(message, dict) and message.get("role") == "system": + system_messages.append(message) + + return system_messages diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index 93fca6ba3e..4b61a317fb 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -542,6 +542,12 @@ class SPANDATA: Example: 2048 """ + GEN_AI_SYSTEM_INSTRUCTIONS = "gen_ai.system_instructions" + """ + The system instructions passed to the model. + Example: [{"type": "text", "text": "You are a helpful assistant."},{"type": "text", "text": "Be concise and clear."}] + """ + GEN_AI_REQUEST_MESSAGES = "gen_ai.request.messages" """ The messages passed to the model. The "content" can be a string or an array of objects. diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 553d93d195..6a269c40c7 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -9,6 +9,9 @@ normalize_message_roles, truncate_and_annotate_messages, ) +from sentry_sdk.ai._openai_completions_api import ( + _get_system_instructions as _get_system_instructions_completions, +) from sentry_sdk.consts import SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration from sentry_sdk.scope import should_send_default_pii @@ -35,7 +38,7 @@ ) from sentry_sdk.tracing import Span - from openai.types.responses import ResponseInputParam + from openai.types.responses import ResponseInputParam, ResponseInputItemParam try: try: @@ -193,6 +196,25 @@ def _calculate_token_usage( ) +def _get_system_instructions_responses( + input_items: "Union[ResponseInputParam, list[str]]", +) -> "list[ResponseInputItemParam]": + if isinstance(input_items, str): + return [] + + system_messages = [] + + for item in input_items: + if ( + isinstance(item, dict) + and item.get("type") == "message" + and item.get("role") == "system" + ): + system_messages.append(item) + + return system_messages + + def _get_input_messages( kwargs: "dict[str, Any]", ) -> "Optional[Iterable[Any] | list[str]]": @@ -245,6 +267,15 @@ def _set_responses_api_input_data( ): messages: "Optional[ResponseInputParam | list[str]]" = _get_input_messages(kwargs) # type: ignore + if messages is not None: + system_instructions = _get_system_instructions_responses(messages) + set_data_normalized( + span, + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, + system_instructions, + unpack=False, + ) + if ( messages is not None and len(messages) > 0 @@ -272,6 +303,15 @@ def _set_completions_api_input_data( _get_input_messages(kwargs) ) + if messages is not None: + system_instructions = _get_system_instructions_completions(messages) + set_data_normalized( + span, + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, + system_instructions, + unpack=False, + ) + if ( messages is not None and len(messages) > 0 # type: ignore From 9f5d8015f5ab0e222c784f1bee3ea05ef26b9f7e Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 21 Jan 2026 19:06:03 +0100 Subject: [PATCH 02/22] . --- sentry_sdk/ai/_openai_completions_api.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sentry_sdk/ai/_openai_completions_api.py b/sentry_sdk/ai/_openai_completions_api.py index 6697f285c6..8309c76c31 100644 --- a/sentry_sdk/ai/_openai_completions_api.py +++ b/sentry_sdk/ai/_openai_completions_api.py @@ -5,12 +5,14 @@ ChatCompletionMessageParam, ChatCompletionSystemMessageParam, ) - from typing import Iterable, Union + from typing import Iterable, Union, TypeVar + + T = TypeVar("T") def _get_system_instructions( - messages: "Iterable[Union[ChatCompletionMessageParam, str]]", -) -> "list[ChatCompletionSystemMessageParam]": + messages: "Iterable[Union[T, str]]", +) -> "list[T]": system_messages = [] for message in messages: From d66ffe12aee0f979d22140b3e4c05d4782127e37 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 21 Jan 2026 19:20:08 +0100 Subject: [PATCH 03/22] use specific openai types --- sentry_sdk/ai/_openai_completions_api.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/sentry_sdk/ai/_openai_completions_api.py b/sentry_sdk/ai/_openai_completions_api.py index 8309c76c31..6697f285c6 100644 --- a/sentry_sdk/ai/_openai_completions_api.py +++ b/sentry_sdk/ai/_openai_completions_api.py @@ -5,14 +5,12 @@ ChatCompletionMessageParam, ChatCompletionSystemMessageParam, ) - from typing import Iterable, Union, TypeVar - - T = TypeVar("T") + from typing import Iterable, Union def _get_system_instructions( - messages: "Iterable[Union[T, str]]", -) -> "list[T]": + messages: "Iterable[Union[ChatCompletionMessageParam, str]]", +) -> "list[ChatCompletionSystemMessageParam]": system_messages = [] for message in messages: From ac3ce00e26e6e86b5cec7ecb5170fad729f9a31d Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 22 Jan 2026 10:43:32 +0100 Subject: [PATCH 04/22] wip --- sentry_sdk/integrations/openai.py | 107 +++++++++++++++++------ tests/integrations/openai/test_openai.py | 73 +++++++++++++++- 2 files changed, 147 insertions(+), 33 deletions(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index b747596a61..cace34ebf1 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -11,6 +11,7 @@ ) from sentry_sdk.ai._openai_completions_api import ( _get_system_instructions as _get_system_instructions_completions, + _is_system_instruction as _is_system_instruction_completions, ) from sentry_sdk.consts import SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration @@ -196,23 +197,27 @@ def _calculate_token_usage( ) +def _is_system_instruction_responses(message: "ResponseInputItemParam"): + return ( + isinstance(message, dict) + and message.get("type") == "message" + and message.get("role") == "system" + ) + + def _get_system_instructions_responses( - input_items: "Union[ResponseInputParam, list[str]]", + messages: "Union[str, ResponseInputParam]", ) -> "list[ResponseInputItemParam]": - if isinstance(input_items, str): + if isinstance(messages, str): return [] - system_messages = [] + system_instructions = [] - for item in input_items: - if ( - isinstance(item, dict) - and item.get("type") == "message" - and item.get("role") == "system" - ): - system_messages.append(item) + for message in messages: + if _is_system_instruction_responses(message): + system_instructions.append(message) - return system_messages + return system_instructions def _get_input_messages( @@ -265,12 +270,19 @@ def _set_responses_api_input_data( kwargs: "dict[str, Any]", integration: "OpenAIIntegration", ) -> None: - messages: "Optional[Union[ResponseInputParam, list[str]]]" = _get_input_messages( - kwargs - ) + messages: "Optional[Union[str, ResponseInputParam]]" = kwargs.get("input") + + if messages is None: + set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "responses") + _commmon_set_input_data(span, kwargs) + return - if messages is not None: - system_instructions = _get_system_instructions_responses(messages) + system_instructions = _get_system_instructions_responses(messages) + if ( + len(system_instructions) > 0 + and should_send_default_pii() + and integration.include_prompts + ): set_data_normalized( span, SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, @@ -279,12 +291,11 @@ def _set_responses_api_input_data( ) if ( - messages is not None - and len(messages) > 0 + isinstance(messages, str) and should_send_default_pii() and integration.include_prompts ): - normalized_messages = normalize_message_roles(messages) # type: ignore + normalized_messages = normalize_message_roles([messages]) # type: ignore scope = sentry_sdk.get_current_scope() messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) if messages_data is not None: @@ -292,6 +303,23 @@ def _set_responses_api_input_data( span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False ) + elif should_send_default_pii() and integration.include_prompts: + non_system_messages = [ + message + for message in messages + if not _is_system_instruction_responses(message) + ] + if len(non_system_messages) > 0: + normalized_messages = normalize_message_roles(messages) # type: ignore + scope = sentry_sdk.get_current_scope() + messages_data = truncate_and_annotate_messages( + normalized_messages, span, scope + ) + if messages_data is not None: + set_data_normalized( + span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False + ) + set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "responses") _commmon_set_input_data(span, kwargs) @@ -301,12 +329,21 @@ def _set_completions_api_input_data( kwargs: "dict[str, Any]", integration: "OpenAIIntegration", ) -> None: - messages: "Optional[Union[Iterable[ChatCompletionMessageParam], list[str]]]" = ( - _get_input_messages(kwargs) + messages: "Optional[Union[str, Iterable[ChatCompletionMessageParam]]]" = kwargs.get( + "messages" ) - if messages is not None: - system_instructions = _get_system_instructions_completions(messages) + if messages is None: + set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "chat") + _commmon_set_input_data(span, kwargs) + return + + system_instructions = _get_system_instructions_completions(messages) + if ( + len(system_instructions) > 0 + and should_send_default_pii() + and integration.include_prompts + ): set_data_normalized( span, SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, @@ -315,21 +352,33 @@ def _set_completions_api_input_data( ) if ( - messages is not None - and len(messages) > 0 # type: ignore + isinstance(messages, str) and should_send_default_pii() and integration.include_prompts ): - normalized_messages = normalize_message_roles(messages) # type: ignore + normalized_messages = normalize_message_roles([messages]) # type: ignore scope = sentry_sdk.get_current_scope() messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) if messages_data is not None: set_data_normalized( span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False ) - - set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "chat") - _commmon_set_input_data(span, kwargs) + elif should_send_default_pii() and integration.include_prompts: + non_system_messages = [ + message + for message in messages + if not _is_system_instruction_completions(message) + ] + if len(non_system_messages) > 0: + normalized_messages = normalize_message_roles(non_system_messages) # type: ignore + scope = sentry_sdk.get_current_scope() + messages_data = truncate_and_annotate_messages( + normalized_messages, span, scope + ) + if messages_data is not None: + set_data_normalized( + span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False + ) def _set_embeddings_input_data( diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index 814289c887..c09ca61cb3 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -147,7 +147,11 @@ def test_nonstreaming_chat_completion( with start_transaction(name="openai tx"): response = ( client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], ) .choices[0] .message.content @@ -160,9 +164,17 @@ def test_nonstreaming_chat_completion( assert span["op"] == "gen_ai.chat" if send_default_pii and include_prompts: + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "content": "You are a helpful assistant.", + "role": "system", + } + ] + assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] assert "the model response" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] else: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] @@ -191,7 +203,11 @@ async def test_nonstreaming_chat_completion_async( with start_transaction(name="openai tx"): response = await client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], ) response = response.choices[0].message.content @@ -202,9 +218,17 @@ async def test_nonstreaming_chat_completion_async( assert span["op"] == "gen_ai.chat" if send_default_pii and include_prompts: + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "content": "You are a helpful assistant.", + "role": "system", + } + ] + assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] assert "the model response" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] else: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] @@ -283,7 +307,11 @@ def test_streaming_chat_completion( client.chat.completions._post = mock.Mock(return_value=returned_stream) with start_transaction(name="openai tx"): response_stream = client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], ) response_string = "".join( map(lambda x: x.choices[0].delta.content, response_stream) @@ -298,6 +326,7 @@ def test_streaming_chat_completion( assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] assert "hello world" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] else: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] @@ -377,7 +406,11 @@ async def test_streaming_chat_completion_async( client.chat.completions._post = AsyncMock(return_value=returned_stream) with start_transaction(name="openai tx"): response_stream = await client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], ) response_string = "" @@ -394,6 +427,7 @@ async def test_streaming_chat_completion_async( assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] assert "hello world" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] else: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] @@ -1427,6 +1461,37 @@ async def test_streaming_responses_api_async( assert span["data"]["gen_ai.usage.total_tokens"] == 30 +@pytest.mark.skipif( + OPENAI_VERSION <= (1, 1, 0), + reason="OpenAI versions <=1.1.0 do not support the tools parameter.", +) +@pytest.mark.parametrize( + "tools", + [[], None, NOT_GIVEN, omit], +) +def test_chat_completion_with_system_instruction(sentry_init, capture_events, tools): + sentry_init( + integrations=[OpenAIIntegration()], + traces_sample_rate=1.0, + ) + events = capture_events() + + client = OpenAI(api_key="z") + client.chat.completions._post = mock.Mock(return_value=EXAMPLE_CHAT_COMPLETION) + + with start_transaction(name="openai tx"): + client.chat.completions.create( + model="some-model", + messages=[{"role": "system", "content": "hello"}], + tools=tools, + ) + + (event,) = events + span = event["spans"][0] + + assert "gen_ai.request.available_tools" not in span["data"] + + @pytest.mark.skipif( OPENAI_VERSION <= (1, 1, 0), reason="OpenAI versions <=1.1.0 do not support the tools parameter.", From ef9fe6fee5c70cf2091effcbd190def07fbcc985 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 22 Jan 2026 10:44:10 +0100 Subject: [PATCH 05/22] . --- sentry_sdk/ai/_openai_completions_api.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/sentry_sdk/ai/_openai_completions_api.py b/sentry_sdk/ai/_openai_completions_api.py index 6697f285c6..d1c1c7d23b 100644 --- a/sentry_sdk/ai/_openai_completions_api.py +++ b/sentry_sdk/ai/_openai_completions_api.py @@ -5,16 +5,20 @@ ChatCompletionMessageParam, ChatCompletionSystemMessageParam, ) - from typing import Iterable, Union + from typing import Iterable + + +def _is_system_instruction(message: "ChatCompletionMessageParam"): + return isinstance(message, dict) and message.get("role") == "system" def _get_system_instructions( - messages: "Iterable[Union[ChatCompletionMessageParam, str]]", + messages: "Iterable[ChatCompletionMessageParam]", ) -> "list[ChatCompletionSystemMessageParam]": - system_messages = [] + system_instructions = [] for message in messages: - if isinstance(message, dict) and message.get("role") == "system": - system_messages.append(message) + if _is_system_instruction(message): + system_instructions.append(message) - return system_messages + return system_instructions From ce84a29356a29c4313b83bf4717e0dbbb64c9b32 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 22 Jan 2026 13:13:40 +0100 Subject: [PATCH 06/22] . --- sentry_sdk/_types.py | 4 + sentry_sdk/ai/_openai_completions_api.py | 26 +++ sentry_sdk/integrations/openai.py | 22 +- tests/integrations/openai/test_openai.py | 262 +++++++++++++++++++---- 4 files changed, 267 insertions(+), 47 deletions(-) diff --git a/sentry_sdk/_types.py b/sentry_sdk/_types.py index 7043bbc2ee..ecb8abcd10 100644 --- a/sentry_sdk/_types.py +++ b/sentry_sdk/_types.py @@ -359,3 +359,7 @@ class SDKInfo(TypedDict): ) HttpStatusCodeRange = Union[int, Container[int]] + + class TextPart(TypedDict): + type: Literal["text"] + content: str diff --git a/sentry_sdk/ai/_openai_completions_api.py b/sentry_sdk/ai/_openai_completions_api.py index d1c1c7d23b..3bb4c82448 100644 --- a/sentry_sdk/ai/_openai_completions_api.py +++ b/sentry_sdk/ai/_openai_completions_api.py @@ -7,6 +7,8 @@ ) from typing import Iterable + from sentry_sdk._types import TextPart + def _is_system_instruction(message: "ChatCompletionMessageParam"): return isinstance(message, dict) and message.get("role") == "system" @@ -22,3 +24,27 @@ def _get_system_instructions( system_instructions.append(message) return system_instructions + + +def _transform_system_instructions( + system_instructions: "list[ChatCompletionSystemMessageParam]", +) -> "list[TextPart]": + instruction_text_parts = [] + + for instruction in system_instructions: + if not isinstance(instruction, dict): + continue + + content = instruction.get("content") + + if isinstance(content, str): + instruction_text_parts.append({"type": "text", "content": content}) + + elif isinstance(content, list): + for part in content: + if isinstance(part, dict) and part.get("type") == "text": + text = part.get("text", "") + if text: + instruction_text_parts.append({"type": "text", "content": text}) + + return instruction_text_parts diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index cace34ebf1..1e587b3a3e 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -12,6 +12,7 @@ from sentry_sdk.ai._openai_completions_api import ( _get_system_instructions as _get_system_instructions_completions, _is_system_instruction as _is_system_instruction_completions, + _transform_system_instructions, ) from sentry_sdk.consts import SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration @@ -40,6 +41,7 @@ from sentry_sdk.tracing import Span from openai.types.responses import ResponseInputParam, ResponseInputItemParam + from openai import Omit try: try: @@ -277,16 +279,28 @@ def _set_responses_api_input_data( _commmon_set_input_data(span, kwargs) return + explicit_instructions: "Union[Optional[str], Omit]" = kwargs.get("instructions") system_instructions = _get_system_instructions_responses(messages) if ( - len(system_instructions) > 0 + (_is_given(explicit_instructions) or len(system_instructions) > 0) and should_send_default_pii() and integration.include_prompts ): + # Deliberate use of function accepting completions API type because + # of shared structure FOR THIS PURPOSE ONLY. + instructions_text_parts = _transform_system_instructions(system_instructions) # type: ignore + if _is_given(explicit_instructions): + instructions_text_parts.append( + { + "type": "text", + "content": explicit_instructions, + } + ) + set_data_normalized( span, SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, - system_instructions, + instructions_text_parts, unpack=False, ) @@ -310,7 +324,7 @@ def _set_responses_api_input_data( if not _is_system_instruction_responses(message) ] if len(non_system_messages) > 0: - normalized_messages = normalize_message_roles(messages) # type: ignore + normalized_messages = normalize_message_roles(non_system_messages) # type: ignore scope = sentry_sdk.get_current_scope() messages_data = truncate_and_annotate_messages( normalized_messages, span, scope @@ -347,7 +361,7 @@ def _set_completions_api_input_data( set_data_normalized( span, SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, - system_instructions, + _transform_system_instructions(system_instructions), unpack=False, ) diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index c09ca61cb3..a9bfa40fef 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -166,8 +166,8 @@ def test_nonstreaming_chat_completion( if send_default_pii and include_prompts: assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ { + "type": "text", "content": "You are a helpful assistant.", - "role": "system", } ] @@ -220,8 +220,8 @@ async def test_nonstreaming_chat_completion_async( if send_default_pii and include_prompts: assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ { + "type": "text", "content": "You are a helpful assistant.", - "role": "system", } ] @@ -251,8 +251,38 @@ def tiktoken_encoding_if_installed(): "send_default_pii, include_prompts", [(True, True), (True, False), (False, True), (False, False)], ) +@pytest.mark.parametrize( + "input", + [ + pytest.param( + [ + { + "type": "message", + "role": "system", + "content": "You are a helpful assistant.", + }, + {"type": "message", "role": "user", "content": "hello"}, + ], + id="blocks", + ), + pytest.param( + [ + { + "type": "message", + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful assistant."}, + {"type": "text", "text": "Be concise and clear."}, + ], + }, + {"type": "message", "role": "user", "content": "hello"}, + ], + id="parts", + ), + ], +) def test_streaming_chat_completion( - sentry_init, capture_events, send_default_pii, include_prompts + sentry_init, capture_events, send_default_pii, include_prompts, input, request ): sentry_init( integrations=[ @@ -308,10 +338,7 @@ def test_streaming_chat_completion( with start_transaction(name="openai tx"): response_stream = client.chat.completions.create( model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], + messages=input, ) response_string = "".join( map(lambda x: x.choices[0].delta.content, response_stream) @@ -322,7 +349,27 @@ def test_streaming_chat_completion( span = tx["spans"][0] assert span["op"] == "gen_ai.chat" + param_id = request.node.callspec.id if send_default_pii and include_prompts: + if "blocks" in param_id: + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "type": "text", + "content": "You are a helpful assistant.", + } + ] + else: + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "type": "text", + "content": "You are a helpful assistant.", + }, + { + "type": "text", + "content": "Be concise and clear.", + }, + ] + assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] assert "hello world" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] else: @@ -333,9 +380,14 @@ def test_streaming_chat_completion( try: import tiktoken # type: ignore # noqa # pylint: disable=unused-import - assert span["data"]["gen_ai.usage.output_tokens"] == 2 - assert span["data"]["gen_ai.usage.input_tokens"] == 1 - assert span["data"]["gen_ai.usage.total_tokens"] == 3 + if "blocks" in param_id: + assert span["data"]["gen_ai.usage.output_tokens"] == 2 + assert span["data"]["gen_ai.usage.input_tokens"] == 7 + assert span["data"]["gen_ai.usage.total_tokens"] == 9 + else: + assert span["data"]["gen_ai.usage.output_tokens"] == 2 + assert span["data"]["gen_ai.usage.input_tokens"] == 1 + assert span["data"]["gen_ai.usage.total_tokens"] == 3 except ImportError: pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly @@ -346,8 +398,38 @@ def test_streaming_chat_completion( "send_default_pii, include_prompts", [(True, True), (True, False), (False, True), (False, False)], ) +@pytest.mark.parametrize( + "input", + [ + pytest.param( + [ + { + "type": "message", + "role": "system", + "content": "You are a helpful assistant.", + }, + {"type": "message", "role": "user", "content": "hello"}, + ], + id="blocks", + ), + pytest.param( + [ + { + "type": "message", + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful assistant."}, + {"type": "text", "text": "Be concise and clear."}, + ], + }, + {"type": "message", "role": "user", "content": "hello"}, + ], + id="parts", + ), + ], +) async def test_streaming_chat_completion_async( - sentry_init, capture_events, send_default_pii, include_prompts + sentry_init, capture_events, send_default_pii, include_prompts, input, request ): sentry_init( integrations=[ @@ -407,10 +489,7 @@ async def test_streaming_chat_completion_async( with start_transaction(name="openai tx"): response_stream = await client.chat.completions.create( model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], + messages=input, ) response_string = "" @@ -423,7 +502,27 @@ async def test_streaming_chat_completion_async( span = tx["spans"][0] assert span["op"] == "gen_ai.chat" + param_id = request.node.callspec.id if send_default_pii and include_prompts: + if "blocks" in param_id: + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "type": "text", + "content": "You are a helpful assistant.", + } + ] + else: + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "type": "text", + "content": "You are a helpful assistant.", + }, + { + "type": "text", + "content": "Be concise and clear.", + }, + ] + assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] assert "hello world" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] else: @@ -434,9 +533,15 @@ async def test_streaming_chat_completion_async( try: import tiktoken # type: ignore # noqa # pylint: disable=unused-import - assert span["data"]["gen_ai.usage.output_tokens"] == 2 - assert span["data"]["gen_ai.usage.input_tokens"] == 1 - assert span["data"]["gen_ai.usage.total_tokens"] == 3 + if "blocks" in param_id: + assert span["data"]["gen_ai.usage.output_tokens"] == 2 + assert span["data"]["gen_ai.usage.input_tokens"] == 7 + assert span["data"]["gen_ai.usage.total_tokens"] == 9 + else: + assert span["data"]["gen_ai.usage.output_tokens"] == 2 + assert span["data"]["gen_ai.usage.input_tokens"] == 1 + assert span["data"]["gen_ai.usage.total_tokens"] == 3 + except ImportError: pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly @@ -1068,12 +1173,46 @@ def test_ai_client_span_responses_api_no_pii(sentry_init, capture_events): "thread.name": mock.ANY, } + assert "gen_ai.system_instructions" not in spans[0]["data"] assert "gen_ai.request.messages" not in spans[0]["data"] assert "gen_ai.response.text" not in spans[0]["data"] +@pytest.mark.parametrize( + "input", + [ + pytest.param( + "How do I check if a Python object is an instance of a class?", id="string" + ), + pytest.param( + [ + { + "type": "message", + "role": "system", + "content": "You are a helpful assistant.", + }, + {"type": "message", "role": "user", "content": "hello"}, + ], + id="blocks", + ), + pytest.param( + [ + { + "type": "message", + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful assistant."}, + {"type": "text", "text": "Be concise and clear."}, + ], + }, + {"type": "message", "role": "user", "content": "hello"}, + ], + id="parts", + ), + ], +) @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") -def test_ai_client_span_responses_api(sentry_init, capture_events): +def test_ai_client_span_responses_api(sentry_init, capture_events, input, request): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], traces_sample_rate=1.0, @@ -1088,7 +1227,7 @@ def test_ai_client_span_responses_api(sentry_init, capture_events): client.responses.create( model="gpt-4o", instructions="You are a coding assistant that talks like a pirate.", - input="How do I check if a Python object is an instance of a class?", + input=input, ) (transaction,) = events @@ -1097,21 +1236,59 @@ def test_ai_client_span_responses_api(sentry_init, capture_events): assert len(spans) == 1 assert spans[0]["op"] == "gen_ai.responses" assert spans[0]["origin"] == "auto.ai.openai" - assert spans[0]["data"] == { - "gen_ai.operation.name": "responses", - "gen_ai.request.messages": '["How do I check if a Python object is an instance of a class?"]', - "gen_ai.request.model": "gpt-4o", - "gen_ai.system": "openai", - "gen_ai.response.model": "response-model-id", - "gen_ai.usage.input_tokens": 20, - "gen_ai.usage.input_tokens.cached": 5, - "gen_ai.usage.output_tokens": 10, - "gen_ai.usage.output_tokens.reasoning": 8, - "gen_ai.usage.total_tokens": 30, - "gen_ai.response.text": "the model response", - "thread.id": mock.ANY, - "thread.name": mock.ANY, - } + + param_id = request.node.callspec.id + if param_id == "string": + assert spans[0]["data"] == { + "gen_ai.operation.name": "responses", + "gen_ai.request.messages": '["How do I check if a Python object is an instance of a class?"]', + "gen_ai.request.model": "gpt-4o", + "gen_ai.system": "openai", + "gen_ai.system_instructions": '[{"type": "text", "content": "You are a coding assistant that talks like a pirate."}]', + "gen_ai.response.model": "response-model-id", + "gen_ai.usage.input_tokens": 20, + "gen_ai.usage.input_tokens.cached": 5, + "gen_ai.usage.output_tokens": 10, + "gen_ai.usage.output_tokens.reasoning": 8, + "gen_ai.usage.total_tokens": 30, + "gen_ai.response.text": "the model response", + "thread.id": mock.ANY, + "thread.name": mock.ANY, + } + elif param_id == "blocks": + assert spans[0]["data"] == { + "gen_ai.operation.name": "responses", + "gen_ai.request.messages": '[{"type": "message", "role": "user", "content": "hello"}]', + "gen_ai.request.model": "gpt-4o", + "gen_ai.system": "openai", + "gen_ai.system_instructions": '[{"type": "text", "content": "You are a helpful assistant."}, {"type": "text", "content": "You are a coding assistant that talks like a pirate."}]', + "gen_ai.response.model": "response-model-id", + "gen_ai.usage.input_tokens": 20, + "gen_ai.usage.input_tokens.cached": 5, + "gen_ai.usage.output_tokens": 10, + "gen_ai.usage.output_tokens.reasoning": 8, + "gen_ai.usage.total_tokens": 30, + "gen_ai.response.text": "the model response", + "thread.id": mock.ANY, + "thread.name": mock.ANY, + } + else: + assert spans[0]["data"] == { + "gen_ai.operation.name": "responses", + "gen_ai.request.messages": '[{"type": "message", "role": "user", "content": "hello"}]', + "gen_ai.request.model": "gpt-4o", + "gen_ai.system": "openai", + "gen_ai.system_instructions": '[{"type": "text", "content": "You are a helpful assistant."}, {"type": "text", "content": "Be concise and clear."}, {"type": "text", "content": "You are a coding assistant that talks like a pirate."}]', + "gen_ai.response.model": "response-model-id", + "gen_ai.usage.input_tokens": 20, + "gen_ai.usage.input_tokens.cached": 5, + "gen_ai.usage.output_tokens": 10, + "gen_ai.usage.output_tokens.reasoning": 8, + "gen_ai.usage.total_tokens": 30, + "gen_ai.response.text": "the model response", + "thread.id": mock.ANY, + "thread.name": mock.ANY, + } @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") @@ -1183,6 +1360,7 @@ async def test_ai_client_span_responses_async_api(sentry_init, capture_events): "gen_ai.request.model": "gpt-4o", "gen_ai.response.model": "response-model-id", "gen_ai.system": "openai", + "gen_ai.system_instructions": '[{"type": "text", "content": "You are a coding assistant that talks like a pirate."}]', "gen_ai.usage.input_tokens": 20, "gen_ai.usage.input_tokens.cached": 5, "gen_ai.usage.output_tokens": 10, @@ -1230,6 +1408,7 @@ async def test_ai_client_span_streaming_responses_async_api( "gen_ai.response.model": "response-model-id", "gen_ai.response.streaming": True, "gen_ai.system": "openai", + "gen_ai.system_instructions": '[{"type": "text", "content": "You are a coding assistant that talks like a pirate."}]', "gen_ai.usage.input_tokens": 20, "gen_ai.usage.input_tokens.cached": 5, "gen_ai.usage.output_tokens": 10, @@ -1537,7 +1716,6 @@ def test_openai_message_role_mapping(sentry_init, capture_events): client.chat.completions._post = mock.Mock(return_value=EXAMPLE_CHAT_COMPLETION) # Test messages with mixed roles including "ai" that should be mapped to "assistant" test_messages = [ - {"role": "system", "content": "You are helpful."}, {"role": "user", "content": "Hello"}, {"role": "ai", "content": "Hi there!"}, # Should be mapped to "assistant" {"role": "assistant", "content": "How can I help?"}, # Should stay "assistant" @@ -1557,17 +1735,15 @@ def test_openai_message_role_mapping(sentry_init, capture_events): stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) # Verify that "ai" role was mapped to "assistant" - assert len(stored_messages) == 4 - assert stored_messages[0]["role"] == "system" - assert stored_messages[1]["role"] == "user" + assert len(stored_messages) == 3 assert ( - stored_messages[2]["role"] == "assistant" + stored_messages[1]["role"] == "assistant" ) # "ai" should be mapped to "assistant" - assert stored_messages[3]["role"] == "assistant" # should stay "assistant" + assert stored_messages[2]["role"] == "assistant" # should stay "assistant" # Verify content is preserved - assert stored_messages[2]["content"] == "Hi there!" - assert stored_messages[3]["content"] == "How can I help?" + assert stored_messages[1]["content"] == "Hi there!" + assert stored_messages[2]["content"] == "How can I help?" # Verify no "ai" roles remain roles = [msg["role"] for msg in stored_messages] From dee993063e181fef4359036e3f3ea3bf73740fca Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 22 Jan 2026 13:14:42 +0100 Subject: [PATCH 07/22] remove test --- tests/integrations/openai/test_openai.py | 31 ------------------------ 1 file changed, 31 deletions(-) diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index a9bfa40fef..49fc7baf1b 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -1640,37 +1640,6 @@ async def test_streaming_responses_api_async( assert span["data"]["gen_ai.usage.total_tokens"] == 30 -@pytest.mark.skipif( - OPENAI_VERSION <= (1, 1, 0), - reason="OpenAI versions <=1.1.0 do not support the tools parameter.", -) -@pytest.mark.parametrize( - "tools", - [[], None, NOT_GIVEN, omit], -) -def test_chat_completion_with_system_instruction(sentry_init, capture_events, tools): - sentry_init( - integrations=[OpenAIIntegration()], - traces_sample_rate=1.0, - ) - events = capture_events() - - client = OpenAI(api_key="z") - client.chat.completions._post = mock.Mock(return_value=EXAMPLE_CHAT_COMPLETION) - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=[{"role": "system", "content": "hello"}], - tools=tools, - ) - - (event,) = events - span = event["spans"][0] - - assert "gen_ai.request.available_tools" not in span["data"] - - @pytest.mark.skipif( OPENAI_VERSION <= (1, 1, 0), reason="OpenAI versions <=1.1.0 do not support the tools parameter.", From cb00ab3acd79431612b01f4edd1465b096ec8d75 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 22 Jan 2026 13:18:52 +0100 Subject: [PATCH 08/22] . --- sentry_sdk/ai/_openai_completions_api.py | 4 ++-- sentry_sdk/integrations/openai.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sentry_sdk/ai/_openai_completions_api.py b/sentry_sdk/ai/_openai_completions_api.py index 3bb4c82448..c77fdb82dc 100644 --- a/sentry_sdk/ai/_openai_completions_api.py +++ b/sentry_sdk/ai/_openai_completions_api.py @@ -10,7 +10,7 @@ from sentry_sdk._types import TextPart -def _is_system_instruction(message: "ChatCompletionMessageParam"): +def _is_system_instruction(message: "ChatCompletionMessageParam") -> bool: return isinstance(message, dict) and message.get("role") == "system" @@ -29,7 +29,7 @@ def _get_system_instructions( def _transform_system_instructions( system_instructions: "list[ChatCompletionSystemMessageParam]", ) -> "list[TextPart]": - instruction_text_parts = [] + instruction_text_parts: "list[TextPart]" = [] for instruction in system_instructions: if not isinstance(instruction, dict): diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 1e587b3a3e..538db1a603 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -199,7 +199,7 @@ def _calculate_token_usage( ) -def _is_system_instruction_responses(message: "ResponseInputItemParam"): +def _is_system_instruction_responses(message: "ResponseInputItemParam") -> bool: return ( isinstance(message, dict) and message.get("type") == "message" @@ -288,12 +288,12 @@ def _set_responses_api_input_data( ): # Deliberate use of function accepting completions API type because # of shared structure FOR THIS PURPOSE ONLY. - instructions_text_parts = _transform_system_instructions(system_instructions) # type: ignore + instructions_text_parts = _transform_system_instructions(system_instructions) if _is_given(explicit_instructions): instructions_text_parts.append( { "type": "text", - "content": explicit_instructions, + "content": explicit_instructions, # type: ignore } ) From 26b932bad88285d50a59c06a436d47ab2cb3d5c8 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 22 Jan 2026 13:40:58 +0100 Subject: [PATCH 09/22] . --- tests/integrations/openai/test_openai.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index 49fc7baf1b..33d476bb53 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -257,25 +257,23 @@ def tiktoken_encoding_if_installed(): pytest.param( [ { - "type": "message", "role": "system", "content": "You are a helpful assistant.", }, - {"type": "message", "role": "user", "content": "hello"}, + {"role": "user", "content": "hello"}, ], id="blocks", ), pytest.param( [ { - "type": "message", "role": "system", "content": [ {"type": "text", "text": "You are a helpful assistant."}, {"type": "text", "text": "Be concise and clear."}, ], }, - {"type": "message", "role": "user", "content": "hello"}, + {"role": "user", "content": "hello"}, ], id="parts", ), @@ -404,25 +402,23 @@ def test_streaming_chat_completion( pytest.param( [ { - "type": "message", "role": "system", "content": "You are a helpful assistant.", }, - {"type": "message", "role": "user", "content": "hello"}, + {"role": "user", "content": "hello"}, ], id="blocks", ), pytest.param( [ { - "type": "message", "role": "system", "content": [ {"type": "text", "text": "You are a helpful assistant."}, {"type": "text", "text": "Be concise and clear."}, ], }, - {"type": "message", "role": "user", "content": "hello"}, + {"role": "user", "content": "hello"}, ], id="parts", ), From 04dc92c064feffdacf708f2b229a4a48852c26c0 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 22 Jan 2026 14:40:10 +0100 Subject: [PATCH 10/22] . --- sentry_sdk/integrations/openai.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 538db1a603..2d6daf8822 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -282,7 +282,10 @@ def _set_responses_api_input_data( explicit_instructions: "Union[Optional[str], Omit]" = kwargs.get("instructions") system_instructions = _get_system_instructions_responses(messages) if ( - (_is_given(explicit_instructions) or len(system_instructions) > 0) + ( + (explicit_instructions is not None and _is_given(explicit_instructions)) + or len(system_instructions) > 0 + ) and should_send_default_pii() and integration.include_prompts ): @@ -394,6 +397,9 @@ def _set_completions_api_input_data( span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False ) + set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "chat") + _commmon_set_input_data(span, kwargs) + def _set_embeddings_input_data( span: "Span", From c7263ea349822c20960d53058ca57392440ed007 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 22 Jan 2026 15:45:59 +0100 Subject: [PATCH 11/22] edge case --- sentry_sdk/integrations/openai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 2d6daf8822..4520301df5 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -292,7 +292,7 @@ def _set_responses_api_input_data( # Deliberate use of function accepting completions API type because # of shared structure FOR THIS PURPOSE ONLY. instructions_text_parts = _transform_system_instructions(system_instructions) - if _is_given(explicit_instructions): + if explicit_instructions is not None and _is_given(explicit_instructions): instructions_text_parts.append( { "type": "text", From d947899bc3bab05dfce5bdecdabbae623930a548 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 22 Jan 2026 17:36:07 +0100 Subject: [PATCH 12/22] full responses api tests --- sentry_sdk/integrations/openai.py | 10 +- tests/integrations/openai/test_openai.py | 310 ++++++++++++++++++----- 2 files changed, 256 insertions(+), 64 deletions(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 4520301df5..676c8c8612 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -39,6 +39,7 @@ Union, ) from sentry_sdk.tracing import Span + from sentry_sdk._types import TextPart from openai.types.responses import ResponseInputParam, ResponseInputItemParam from openai import Omit @@ -289,16 +290,17 @@ def _set_responses_api_input_data( and should_send_default_pii() and integration.include_prompts ): - # Deliberate use of function accepting completions API type because - # of shared structure FOR THIS PURPOSE ONLY. - instructions_text_parts = _transform_system_instructions(system_instructions) + instructions_text_parts: "list[TextPart]" = [] if explicit_instructions is not None and _is_given(explicit_instructions): instructions_text_parts.append( { "type": "text", - "content": explicit_instructions, # type: ignore + "content": explicit_instructions, } ) + # Deliberate use of function accepting completions API type because + # of shared structure FOR THIS PURPOSE ONLY. + instructions_text_parts += _transform_system_instructions(system_instructions) set_data_normalized( span, diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index 33d476bb53..35a0b04f96 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -9,8 +9,10 @@ NOT_GIVEN = None try: from openai import omit + from openai import Omit except ImportError: omit = None + Omit = None from openai import AsyncOpenAI, OpenAI, AsyncStream, Stream, OpenAIError from openai.types import CompletionUsage, CreateEmbeddingResponse, Embedding @@ -47,6 +49,7 @@ from sentry_sdk.ai.utils import MAX_GEN_AI_MESSAGE_BYTES from sentry_sdk._types import AnnotatedValue from sentry_sdk.serializer import serialize +from sentry_sdk.utils import safe_serialize from unittest import mock # python 3.3 and above @@ -1174,6 +1177,13 @@ def test_ai_client_span_responses_api_no_pii(sentry_init, capture_events): assert "gen_ai.response.text" not in spans[0]["data"] +@pytest.mark.parametrize( + "instructions", + ( + omit, + "You are a coding assistant that talks like a pirate.", + ), +) @pytest.mark.parametrize( "input", [ @@ -1208,7 +1218,9 @@ def test_ai_client_span_responses_api_no_pii(sentry_init, capture_events): ], ) @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") -def test_ai_client_span_responses_api(sentry_init, capture_events, input, request): +def test_ai_client_span_responses_api( + sentry_init, capture_events, instructions, input, request +): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], traces_sample_rate=1.0, @@ -1222,7 +1234,7 @@ def test_ai_client_span_responses_api(sentry_init, capture_events, input, reques with start_transaction(name="openai tx"): client.responses.create( model="gpt-4o", - instructions="You are a coding assistant that talks like a pirate.", + instructions=instructions, input=input, ) @@ -1233,58 +1245,109 @@ def test_ai_client_span_responses_api(sentry_init, capture_events, input, reques assert spans[0]["op"] == "gen_ai.responses" assert spans[0]["origin"] == "auto.ai.openai" + expected_data = { + "gen_ai.operation.name": "responses", + "gen_ai.response.streaming": True, + "gen_ai.system": "openai", + "gen_ai.response.model": "response-model-id", + "gen_ai.usage.input_tokens": 20, + "gen_ai.usage.input_tokens.cached": 5, + "gen_ai.usage.output_tokens": 10, + "gen_ai.usage.output_tokens.reasoning": 8, + "gen_ai.usage.total_tokens": 30, + "gen_ai.request.model": "gpt-4o", + "gen_ai.response.text": "the model response", + "thread.id": mock.ANY, + "thread.name": mock.ANY, + } + param_id = request.node.callspec.id - if param_id == "string": - assert spans[0]["data"] == { - "gen_ai.operation.name": "responses", - "gen_ai.request.messages": '["How do I check if a Python object is an instance of a class?"]', - "gen_ai.request.model": "gpt-4o", - "gen_ai.system": "openai", - "gen_ai.system_instructions": '[{"type": "text", "content": "You are a coding assistant that talks like a pirate."}]', - "gen_ai.response.model": "response-model-id", - "gen_ai.usage.input_tokens": 20, - "gen_ai.usage.input_tokens.cached": 5, - "gen_ai.usage.output_tokens": 10, - "gen_ai.usage.output_tokens.reasoning": 8, - "gen_ai.usage.total_tokens": 30, - "gen_ai.response.text": "the model response", - "thread.id": mock.ANY, - "thread.name": mock.ANY, - } - elif param_id == "blocks": - assert spans[0]["data"] == { - "gen_ai.operation.name": "responses", - "gen_ai.request.messages": '[{"type": "message", "role": "user", "content": "hello"}]', - "gen_ai.request.model": "gpt-4o", - "gen_ai.system": "openai", - "gen_ai.system_instructions": '[{"type": "text", "content": "You are a helpful assistant."}, {"type": "text", "content": "You are a coding assistant that talks like a pirate."}]', - "gen_ai.response.model": "response-model-id", - "gen_ai.usage.input_tokens": 20, - "gen_ai.usage.input_tokens.cached": 5, - "gen_ai.usage.output_tokens": 10, - "gen_ai.usage.output_tokens.reasoning": 8, - "gen_ai.usage.total_tokens": 30, - "gen_ai.response.text": "the model response", - "thread.id": mock.ANY, - "thread.name": mock.ANY, - } + if "string" in param_id and isinstance(instructions, Omit): # type: ignore + expected_data.update( + { + "gen_ai.request.messages": safe_serialize( + ["How do I check if a Python object is an instance of a class?"] + ), + } + ) + elif "string" in param_id: + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [ + { + "type": "text", + "content": "You are a coding assistant that talks like a pirate.", + } + ] + ), + "gen_ai.request.messages": safe_serialize( + ["How do I check if a Python object is an instance of a class?"] + ), + } + ) + elif "blocks" in param_id and isinstance(instructions, Omit): # type: ignore + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [{"type": "text", "content": "You are a helpful assistant."}] + ), + "gen_ai.request.messages": safe_serialize( + [{"type": "message", "role": "user", "content": "hello"}] + ), + } + ) + elif "blocks" in param_id: + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [ + { + "type": "text", + "content": "You are a coding assistant that talks like a pirate.", + }, + {"type": "text", "content": "You are a helpful assistant."}, + ] + ), + "gen_ai.request.messages": safe_serialize( + [{"type": "message", "role": "user", "content": "hello"}] + ), + } + ) + elif isinstance(instructions, Omit): # type: ignore + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [ + {"type": "text", "content": "You are a helpful assistant."}, + {"type": "text", "content": "Be concise and clear."}, + ] + ), + "gen_ai.request.messages": safe_serialize( + [{"type": "message", "role": "user", "content": "hello"}] + ), + } + ) else: - assert spans[0]["data"] == { - "gen_ai.operation.name": "responses", - "gen_ai.request.messages": '[{"type": "message", "role": "user", "content": "hello"}]', - "gen_ai.request.model": "gpt-4o", - "gen_ai.system": "openai", - "gen_ai.system_instructions": '[{"type": "text", "content": "You are a helpful assistant."}, {"type": "text", "content": "Be concise and clear."}, {"type": "text", "content": "You are a coding assistant that talks like a pirate."}]', - "gen_ai.response.model": "response-model-id", - "gen_ai.usage.input_tokens": 20, - "gen_ai.usage.input_tokens.cached": 5, - "gen_ai.usage.output_tokens": 10, - "gen_ai.usage.output_tokens.reasoning": 8, - "gen_ai.usage.total_tokens": 30, - "gen_ai.response.text": "the model response", - "thread.id": mock.ANY, - "thread.name": mock.ANY, - } + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [ + { + "type": "text", + "content": "You are a coding assistant that talks like a pirate.", + }, + {"type": "text", "content": "You are a helpful assistant."}, + {"type": "text", "content": "Be concise and clear."}, + ] + ), + "gen_ai.request.messages": safe_serialize( + [{"type": "message", "role": "user", "content": "hello"}] + ), + } + ) + + assert spans[0]["data"] == expected_data @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") @@ -1369,9 +1432,49 @@ async def test_ai_client_span_responses_async_api(sentry_init, capture_events): @pytest.mark.asyncio +@pytest.mark.parametrize( + "instructions", + ( + omit, + "You are a coding assistant that talks like a pirate.", + ), +) +@pytest.mark.parametrize( + "input", + [ + pytest.param( + "How do I check if a Python object is an instance of a class?", id="string" + ), + pytest.param( + [ + { + "type": "message", + "role": "system", + "content": "You are a helpful assistant.", + }, + {"type": "message", "role": "user", "content": "hello"}, + ], + id="blocks", + ), + pytest.param( + [ + { + "type": "message", + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful assistant."}, + {"type": "text", "text": "Be concise and clear."}, + ], + }, + {"type": "message", "role": "user", "content": "hello"}, + ], + id="parts", + ), + ], +) @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") async def test_ai_client_span_streaming_responses_async_api( - sentry_init, capture_events + sentry_init, capture_events, instructions, input, request ): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], @@ -1386,8 +1489,8 @@ async def test_ai_client_span_streaming_responses_async_api( with start_transaction(name="openai tx"): await client.responses.create( model="gpt-4o", - instructions="You are a coding assistant that talks like a pirate.", - input="How do I check if a Python object is an instance of a class?", + instructions=instructions, + input=input, stream=True, ) @@ -1397,24 +1500,111 @@ async def test_ai_client_span_streaming_responses_async_api( assert len(spans) == 1 assert spans[0]["op"] == "gen_ai.responses" assert spans[0]["origin"] == "auto.ai.openai" - assert spans[0]["data"] == { + + expected_data = { "gen_ai.operation.name": "responses", - "gen_ai.request.messages": '["How do I check if a Python object is an instance of a class?"]', - "gen_ai.request.model": "gpt-4o", - "gen_ai.response.model": "response-model-id", "gen_ai.response.streaming": True, "gen_ai.system": "openai", - "gen_ai.system_instructions": '[{"type": "text", "content": "You are a coding assistant that talks like a pirate."}]', + "gen_ai.response.model": "response-model-id", "gen_ai.usage.input_tokens": 20, "gen_ai.usage.input_tokens.cached": 5, "gen_ai.usage.output_tokens": 10, "gen_ai.usage.output_tokens.reasoning": 8, "gen_ai.usage.total_tokens": 30, + "gen_ai.request.model": "gpt-4o", "gen_ai.response.text": "the model response", "thread.id": mock.ANY, "thread.name": mock.ANY, } + param_id = request.node.callspec.id + if "string" in param_id and isinstance(instructions, Omit): # type: ignore + expected_data.update( + { + "gen_ai.request.messages": safe_serialize( + ["How do I check if a Python object is an instance of a class?"] + ), + } + ) + elif "string" in param_id: + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [ + { + "type": "text", + "content": "You are a coding assistant that talks like a pirate.", + } + ] + ), + "gen_ai.request.messages": safe_serialize( + ["How do I check if a Python object is an instance of a class?"] + ), + } + ) + elif "blocks" in param_id and isinstance(instructions, Omit): # type: ignore + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [{"type": "text", "content": "You are a helpful assistant."}] + ), + "gen_ai.request.messages": safe_serialize( + [{"type": "message", "role": "user", "content": "hello"}] + ), + } + ) + elif "blocks" in param_id: + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [ + { + "type": "text", + "content": "You are a coding assistant that talks like a pirate.", + }, + {"type": "text", "content": "You are a helpful assistant."}, + ] + ), + "gen_ai.request.messages": safe_serialize( + [{"type": "message", "role": "user", "content": "hello"}] + ), + } + ) + elif isinstance(instructions, Omit): # type: ignore + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [ + {"type": "text", "content": "You are a helpful assistant."}, + {"type": "text", "content": "Be concise and clear."}, + ] + ), + "gen_ai.request.messages": safe_serialize( + [{"type": "message", "role": "user", "content": "hello"}] + ), + } + ) + else: + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [ + { + "type": "text", + "content": "You are a coding assistant that talks like a pirate.", + }, + {"type": "text", "content": "You are a helpful assistant."}, + {"type": "text", "content": "Be concise and clear."}, + ] + ), + "gen_ai.request.messages": safe_serialize( + [{"type": "message", "role": "user", "content": "hello"}] + ), + } + ) + + assert spans[0]["data"] == expected_data + @pytest.mark.asyncio @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") From 8cbeac119f9618475de35b0b04e78ffa1d4bd720 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 22 Jan 2026 17:41:22 +0100 Subject: [PATCH 13/22] remove sentry_sdk/ai/_openai_completions_api.py --- sentry_sdk/ai/_openai_completions_api.py | 50 ---------------------- sentry_sdk/integrations/openai.py | 53 ++++++++++++++++++++---- 2 files changed, 46 insertions(+), 57 deletions(-) delete mode 100644 sentry_sdk/ai/_openai_completions_api.py diff --git a/sentry_sdk/ai/_openai_completions_api.py b/sentry_sdk/ai/_openai_completions_api.py deleted file mode 100644 index c77fdb82dc..0000000000 --- a/sentry_sdk/ai/_openai_completions_api.py +++ /dev/null @@ -1,50 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from openai.types.chat import ( - ChatCompletionMessageParam, - ChatCompletionSystemMessageParam, - ) - from typing import Iterable - - from sentry_sdk._types import TextPart - - -def _is_system_instruction(message: "ChatCompletionMessageParam") -> bool: - return isinstance(message, dict) and message.get("role") == "system" - - -def _get_system_instructions( - messages: "Iterable[ChatCompletionMessageParam]", -) -> "list[ChatCompletionSystemMessageParam]": - system_instructions = [] - - for message in messages: - if _is_system_instruction(message): - system_instructions.append(message) - - return system_instructions - - -def _transform_system_instructions( - system_instructions: "list[ChatCompletionSystemMessageParam]", -) -> "list[TextPart]": - instruction_text_parts: "list[TextPart]" = [] - - for instruction in system_instructions: - if not isinstance(instruction, dict): - continue - - content = instruction.get("content") - - if isinstance(content, str): - instruction_text_parts.append({"type": "text", "content": content}) - - elif isinstance(content, list): - for part in content: - if isinstance(part, dict) and part.get("type") == "text": - text = part.get("text", "") - if text: - instruction_text_parts.append({"type": "text", "content": text}) - - return instruction_text_parts diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 676c8c8612..7a5d449e23 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -9,11 +9,6 @@ normalize_message_roles, truncate_and_annotate_messages, ) -from sentry_sdk.ai._openai_completions_api import ( - _get_system_instructions as _get_system_instructions_completions, - _is_system_instruction as _is_system_instruction_completions, - _transform_system_instructions, -) from sentry_sdk.consts import SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration from sentry_sdk.scope import should_send_default_pii @@ -30,13 +25,13 @@ if TYPE_CHECKING: from typing import ( Any, - Iterable, List, Optional, Callable, AsyncIterator, Iterator, Union, + Iterable, ) from sentry_sdk.tracing import Span from sentry_sdk._types import TextPart @@ -59,7 +54,11 @@ from openai.resources import Embeddings, AsyncEmbeddings if TYPE_CHECKING: - from openai.types.chat import ChatCompletionMessageParam, ChatCompletionChunk + from openai.types.chat import ( + ChatCompletionMessageParam, + ChatCompletionChunk, + ChatCompletionSystemMessageParam, + ) except ImportError: raise DidNotEnable("OpenAI not installed") @@ -200,6 +199,22 @@ def _calculate_token_usage( ) +def _is_system_instruction_completions(message: "ChatCompletionMessageParam") -> bool: + return isinstance(message, dict) and message.get("role") == "system" + + +def _get_system_instructions_completions( + messages: "Iterable[ChatCompletionMessageParam]", +) -> "list[ChatCompletionSystemMessageParam]": + system_instructions = [] + + for message in messages: + if _is_system_instruction_completions(message): + system_instructions.append(message) + + return system_instructions + + def _is_system_instruction_responses(message: "ResponseInputItemParam") -> bool: return ( isinstance(message, dict) @@ -223,6 +238,30 @@ def _get_system_instructions_responses( return system_instructions +def _transform_system_instructions( + system_instructions: "list[ChatCompletionSystemMessageParam]", +) -> "list[TextPart]": + instruction_text_parts: "list[TextPart]" = [] + + for instruction in system_instructions: + if not isinstance(instruction, dict): + continue + + content = instruction.get("content") + + if isinstance(content, str): + instruction_text_parts.append({"type": "text", "content": content}) + + elif isinstance(content, list): + for part in content: + if isinstance(part, dict) and part.get("type") == "text": + text = part.get("text", "") + if text: + instruction_text_parts.append({"type": "text", "content": text}) + + return instruction_text_parts + + def _get_input_messages( kwargs: "dict[str, Any]", ) -> "Optional[Union[Iterable[Any], list[str]]]": From bcebcc84857503fd0365dc44c0e282be2a2ab7ab Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 22 Jan 2026 17:44:28 +0100 Subject: [PATCH 14/22] fix test --- tests/integrations/openai/test_openai.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index 35a0b04f96..2deb1f6d5c 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -1247,7 +1247,6 @@ def test_ai_client_span_responses_api( expected_data = { "gen_ai.operation.name": "responses", - "gen_ai.response.streaming": True, "gen_ai.system": "openai", "gen_ai.response.model": "response-model-id", "gen_ai.usage.input_tokens": 20, From 3f5ad1152d9c1d94a30b7384dad216e4ab645bbd Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Fri, 23 Jan 2026 15:34:21 +0100 Subject: [PATCH 15/22] more defensive checks in case input is not iterable --- sentry_sdk/integrations/openai.py | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 7a5d449e23..f9a4dcef38 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -1,5 +1,6 @@ import sys from functools import wraps +from collections.abc import Iterable import sentry_sdk from sentry_sdk import consts @@ -31,7 +32,6 @@ AsyncIterator, Iterator, Union, - Iterable, ) from sentry_sdk.tracing import Span from sentry_sdk._types import TextPart @@ -205,14 +205,13 @@ def _is_system_instruction_completions(message: "ChatCompletionMessageParam") -> def _get_system_instructions_completions( messages: "Iterable[ChatCompletionMessageParam]", -) -> "list[ChatCompletionSystemMessageParam]": - system_instructions = [] - - for message in messages: - if _is_system_instruction_completions(message): - system_instructions.append(message) +) -> "list[ChatCompletionMessageParam]": + if not isinstance(messages, Iterable): + return [] - return system_instructions + return [ + message for message in messages if _is_system_instruction_completions(message) + ] def _is_system_instruction_responses(message: "ResponseInputItemParam") -> bool: @@ -226,20 +225,16 @@ def _is_system_instruction_responses(message: "ResponseInputItemParam") -> bool: def _get_system_instructions_responses( messages: "Union[str, ResponseInputParam]", ) -> "list[ResponseInputItemParam]": - if isinstance(messages, str): + if not isinstance(messages, list): return [] - system_instructions = [] - - for message in messages: - if _is_system_instruction_responses(message): - system_instructions.append(message) - - return system_instructions + return [ + message for message in messages if _is_system_instruction_responses(message) + ] def _transform_system_instructions( - system_instructions: "list[ChatCompletionSystemMessageParam]", + system_instructions: "list[ChatCompletionMessageParam]", ) -> "list[TextPart]": instruction_text_parts: "list[TextPart]" = [] From a8840d51858adc70edf60d462c927a4f0db905ec Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Mon, 26 Jan 2026 08:55:43 +0100 Subject: [PATCH 16/22] more early returns --- sentry_sdk/integrations/openai.py | 159 +-- tests/integrations/openai/test_openai.py | 1496 ++++++++++++++++------ 2 files changed, 1200 insertions(+), 455 deletions(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index f9a4dcef38..b8f94be995 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -1,6 +1,5 @@ import sys from functools import wraps -from collections.abc import Iterable import sentry_sdk from sentry_sdk import consts @@ -9,6 +8,7 @@ set_data_normalized, normalize_message_roles, truncate_and_annotate_messages, + truncate_and_annotate_embedding_inputs, ) from sentry_sdk.consts import SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration @@ -32,6 +32,7 @@ AsyncIterator, Iterator, Union, + Iterable, ) from sentry_sdk.tracing import Span from sentry_sdk._types import TextPart @@ -215,11 +216,10 @@ def _get_system_instructions_completions( def _is_system_instruction_responses(message: "ResponseInputItemParam") -> bool: - return ( - isinstance(message, dict) - and message.get("type") == "message" - and message.get("role") == "system" - ) + if not isinstance(message, dict) or not message.get("role") == "system": + return False + + return "type" not in message or message["type"] == "message" def _get_system_instructions_responses( @@ -234,7 +234,7 @@ def _get_system_instructions_responses( def _transform_system_instructions( - system_instructions: "list[ChatCompletionMessageParam]", + system_instructions: "list[ChatCompletionSystemMessageParam]", ) -> "list[TextPart]": instruction_text_parts: "list[TextPart]" = [] @@ -307,35 +307,53 @@ def _set_responses_api_input_data( kwargs: "dict[str, Any]", integration: "OpenAIIntegration", ) -> None: + explicit_instructions: "Union[Optional[str], Omit]" = kwargs.get("instructions") messages: "Optional[Union[str, ResponseInputParam]]" = kwargs.get("input") - if messages is None: + if not should_send_default_pii() or not integration.include_prompts: set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "responses") _commmon_set_input_data(span, kwargs) return - explicit_instructions: "Union[Optional[str], Omit]" = kwargs.get("instructions") - system_instructions = _get_system_instructions_responses(messages) if ( - ( - (explicit_instructions is not None and _is_given(explicit_instructions)) - or len(system_instructions) > 0 - ) - and should_send_default_pii() - and integration.include_prompts + messages is None + and explicit_instructions is not None + and _is_given(explicit_instructions) ): - instructions_text_parts: "list[TextPart]" = [] - if explicit_instructions is not None and _is_given(explicit_instructions): - instructions_text_parts.append( - { - "type": "text", - "content": explicit_instructions, - } - ) - # Deliberate use of function accepting completions API type because - # of shared structure FOR THIS PURPOSE ONLY. - instructions_text_parts += _transform_system_instructions(system_instructions) + set_data_normalized( + span, + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, + { + "type": "text", + "content": explicit_instructions, + }, + unpack=False, + ) + set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "responses") + _commmon_set_input_data(span, kwargs) + return + + if messages is None: + set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "responses") + _commmon_set_input_data(span, kwargs) + return + + instructions_text_parts: "list[TextPart]" = [] + if explicit_instructions is not None and _is_given(explicit_instructions): + instructions_text_parts.append( + { + "type": "text", + "content": explicit_instructions, + } + ) + + system_instructions = _get_system_instructions_responses(messages) + # Deliberate use of function accepting completions API type because + # of shared structure FOR THIS PURPOSE ONLY. + instructions_text_parts += _transform_system_instructions(system_instructions) + + if len(instructions_text_parts) > 0: set_data_normalized( span, SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, @@ -343,11 +361,7 @@ def _set_responses_api_input_data( unpack=False, ) - if ( - isinstance(messages, str) - and should_send_default_pii() - and integration.include_prompts - ): + if isinstance(messages, str): normalized_messages = normalize_message_roles([messages]) # type: ignore scope = sentry_sdk.get_current_scope() messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) @@ -356,22 +370,21 @@ def _set_responses_api_input_data( span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False ) - elif should_send_default_pii() and integration.include_prompts: - non_system_messages = [ - message - for message in messages - if not _is_system_instruction_responses(message) - ] - if len(non_system_messages) > 0: - normalized_messages = normalize_message_roles(non_system_messages) # type: ignore - scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages( - normalized_messages, span, scope + set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "responses") + _commmon_set_input_data(span, kwargs) + return + + non_system_messages = [ + message for message in messages if not _is_system_instruction_responses(message) + ] + if len(non_system_messages) > 0: + normalized_messages = normalize_message_roles(non_system_messages) # type: ignore + scope = sentry_sdk.get_current_scope() + messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) + if messages_data is not None: + set_data_normalized( + span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False ) - if messages_data is not None: - set_data_normalized( - span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False - ) set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "responses") _commmon_set_input_data(span, kwargs) @@ -386,17 +399,18 @@ def _set_completions_api_input_data( "messages" ) + if not should_send_default_pii() or not integration.include_prompts: + set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "responses") + _commmon_set_input_data(span, kwargs) + return + if messages is None: set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "chat") _commmon_set_input_data(span, kwargs) return system_instructions = _get_system_instructions_completions(messages) - if ( - len(system_instructions) > 0 - and should_send_default_pii() - and integration.include_prompts - ): + if len(system_instructions) > 0: set_data_normalized( span, SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, @@ -404,11 +418,7 @@ def _set_completions_api_input_data( unpack=False, ) - if ( - isinstance(messages, str) - and should_send_default_pii() - and integration.include_prompts - ): + if isinstance(messages, str): normalized_messages = normalize_message_roles([messages]) # type: ignore scope = sentry_sdk.get_current_scope() messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) @@ -416,22 +426,23 @@ def _set_completions_api_input_data( set_data_normalized( span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False ) - elif should_send_default_pii() and integration.include_prompts: - non_system_messages = [ - message - for message in messages - if not _is_system_instruction_completions(message) - ] - if len(non_system_messages) > 0: - normalized_messages = normalize_message_roles(non_system_messages) # type: ignore - scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages( - normalized_messages, span, scope + set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "chat") + _commmon_set_input_data(span, kwargs) + return + + non_system_messages = [ + message + for message in messages + if not _is_system_instruction_completions(message) + ] + if len(non_system_messages) > 0: + normalized_messages = normalize_message_roles(non_system_messages) # type: ignore + scope = sentry_sdk.get_current_scope() + messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) + if messages_data is not None: + set_data_normalized( + span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False ) - if messages_data is not None: - set_data_normalized( - span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False - ) set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "chat") _commmon_set_input_data(span, kwargs) @@ -452,7 +463,9 @@ def _set_embeddings_input_data( ): normalized_messages = normalize_message_roles(messages) # type: ignore scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) + messages_data = truncate_and_annotate_embedding_inputs( + normalized_messages, span, scope + ) if messages_data is not None: set_data_normalized( span, SPANDATA.GEN_AI_EMBEDDINGS_INPUT, messages_data, unpack=False diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index 2deb1f6d5c..4d15283ea9 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -46,7 +46,6 @@ OpenAIIntegration, _calculate_token_usage, ) -from sentry_sdk.ai.utils import MAX_GEN_AI_MESSAGE_BYTES from sentry_sdk._types import AnnotatedValue from sentry_sdk.serializer import serialize from sentry_sdk.utils import safe_serialize @@ -132,9 +131,13 @@ async def async_iterator(values): @pytest.mark.parametrize( "send_default_pii, include_prompts", - [(True, True), (True, False), (False, True), (False, False)], + [ + (True, False), + (False, True), + (False, False), + ], ) -def test_nonstreaming_chat_completion( +def test_nonstreaming_chat_completion_no_prompts( sentry_init, capture_events, send_default_pii, include_prompts ): sentry_init( @@ -166,20 +169,92 @@ def test_nonstreaming_chat_completion( span = tx["spans"][0] assert span["op"] == "gen_ai.chat" - if send_default_pii and include_prompts: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + + assert span["data"]["gen_ai.usage.output_tokens"] == 10 + assert span["data"]["gen_ai.usage.input_tokens"] == 20 + assert span["data"]["gen_ai.usage.total_tokens"] == 30 + + +@pytest.mark.parametrize( + "messages", + [ + pytest.param( + [ + { + "role": "system", + "content": "You are a helpful assistant.", + }, + {"role": "user", "content": "hello"}, + ], + id="blocks", + ), + pytest.param( + [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful assistant."}, + {"type": "text", "text": "Be concise and clear."}, + ], + }, + {"role": "user", "content": "hello"}, + ], + id="parts", + ), + ], +) +def test_nonstreaming_chat_completion(sentry_init, capture_events, messages, request): + sentry_init( + integrations=[OpenAIIntegration(include_prompts=True)], + traces_sample_rate=1.0, + send_default_pii=True, + ) + events = capture_events() + + client = OpenAI(api_key="z") + client.chat.completions._post = mock.Mock(return_value=EXAMPLE_CHAT_COMPLETION) + + with start_transaction(name="openai tx"): + response = ( + client.chat.completions.create( + model="some-model", + messages=messages, + ) + .choices[0] + .message.content + ) + + assert response == "the model response" + tx = events[0] + assert tx["type"] == "transaction" + span = tx["spans"][0] + assert span["op"] == "gen_ai.chat" + + param_id = request.node.callspec.id + if "blocks" in param_id: assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ { "type": "text", "content": "You are a helpful assistant.", } ] - - assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert "the model response" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "type": "text", + "content": "You are a helpful assistant.", + }, + { + "type": "text", + "content": "Be concise and clear.", + }, + ] + + assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + assert "the model response" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] assert span["data"]["gen_ai.usage.output_tokens"] == 10 assert span["data"]["gen_ai.usage.input_tokens"] == 20 @@ -189,9 +264,13 @@ def test_nonstreaming_chat_completion( @pytest.mark.asyncio @pytest.mark.parametrize( "send_default_pii, include_prompts", - [(True, True), (True, False), (False, True), (False, False)], + [ + (True, False), + (False, True), + (False, False), + ], ) -async def test_nonstreaming_chat_completion_async( +async def test_nonstreaming_chat_completion_async_no_prompts( sentry_init, capture_events, send_default_pii, include_prompts ): sentry_init( @@ -202,7 +281,7 @@ async def test_nonstreaming_chat_completion_async( events = capture_events() client = AsyncOpenAI(api_key="z") - client.chat.completions._post = AsyncMock(return_value=EXAMPLE_CHAT_COMPLETION) + client.chat.completions._post = mock.AsyncMock(return_value=EXAMPLE_CHAT_COMPLETION) with start_transaction(name="openai tx"): response = await client.chat.completions.create( @@ -220,42 +299,18 @@ async def test_nonstreaming_chat_completion_async( span = tx["spans"][0] assert span["op"] == "gen_ai.chat" - if send_default_pii and include_prompts: - assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ - { - "type": "text", - "content": "You are a helpful assistant.", - } - ] - - assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert "the model response" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] assert span["data"]["gen_ai.usage.output_tokens"] == 10 assert span["data"]["gen_ai.usage.input_tokens"] == 20 assert span["data"]["gen_ai.usage.total_tokens"] == 30 -def tiktoken_encoding_if_installed(): - try: - import tiktoken # type: ignore # noqa # pylint: disable=unused-import - - return "cl100k_base" - except ImportError: - return None - - -# noinspection PyTypeChecker -@pytest.mark.parametrize( - "send_default_pii, include_prompts", - [(True, True), (True, False), (False, True), (False, False)], -) +@pytest.mark.asyncio @pytest.mark.parametrize( - "input", + "messages", [ pytest.param( [ @@ -282,8 +337,80 @@ def tiktoken_encoding_if_installed(): ), ], ) -def test_streaming_chat_completion( - sentry_init, capture_events, send_default_pii, include_prompts, input, request +async def test_nonstreaming_chat_completion_async( + sentry_init, capture_events, messages, request +): + sentry_init( + integrations=[OpenAIIntegration(include_prompts=True)], + traces_sample_rate=1.0, + send_default_pii=True, + ) + events = capture_events() + + client = AsyncOpenAI(api_key="z") + client.chat.completions._post = AsyncMock(return_value=EXAMPLE_CHAT_COMPLETION) + + with start_transaction(name="openai tx"): + response = await client.chat.completions.create( + model="some-model", + messages=messages, + ) + response = response.choices[0].message.content + + assert response == "the model response" + tx = events[0] + assert tx["type"] == "transaction" + span = tx["spans"][0] + assert span["op"] == "gen_ai.chat" + + param_id = request.node.callspec.id + if "blocks" in param_id: + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "type": "text", + "content": "You are a helpful assistant.", + } + ] + else: + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "type": "text", + "content": "You are a helpful assistant.", + }, + { + "type": "text", + "content": "Be concise and clear.", + }, + ] + + assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + assert "the model response" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] + + assert span["data"]["gen_ai.usage.output_tokens"] == 10 + assert span["data"]["gen_ai.usage.input_tokens"] == 20 + assert span["data"]["gen_ai.usage.total_tokens"] == 30 + + +def tiktoken_encoding_if_installed(): + try: + import tiktoken # type: ignore # noqa # pylint: disable=unused-import + + return "cl100k_base" + except ImportError: + return None + + +# noinspection PyTypeChecker +@pytest.mark.parametrize( + "send_default_pii, include_prompts", + [ + (True, False), + (False, True), + (False, False), + ], +) +def test_streaming_chat_completion_no_prompts( + sentry_init, capture_events, send_default_pii, include_prompts ): sentry_init( integrations=[ @@ -339,7 +466,10 @@ def test_streaming_chat_completion( with start_transaction(name="openai tx"): response_stream = client.chat.completions.create( model="some-model", - messages=input, + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], ) response_string = "".join( map(lambda x: x.choices[0].delta.content, response_stream) @@ -350,57 +480,23 @@ def test_streaming_chat_completion( span = tx["spans"][0] assert span["op"] == "gen_ai.chat" - param_id = request.node.callspec.id - if send_default_pii and include_prompts: - if "blocks" in param_id: - assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ - { - "type": "text", - "content": "You are a helpful assistant.", - } - ] - else: - assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ - { - "type": "text", - "content": "You are a helpful assistant.", - }, - { - "type": "text", - "content": "Be concise and clear.", - }, - ] - - assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert "hello world" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] try: import tiktoken # type: ignore # noqa # pylint: disable=unused-import - if "blocks" in param_id: - assert span["data"]["gen_ai.usage.output_tokens"] == 2 - assert span["data"]["gen_ai.usage.input_tokens"] == 7 - assert span["data"]["gen_ai.usage.total_tokens"] == 9 - else: - assert span["data"]["gen_ai.usage.output_tokens"] == 2 - assert span["data"]["gen_ai.usage.input_tokens"] == 1 - assert span["data"]["gen_ai.usage.total_tokens"] == 3 + assert span["data"]["gen_ai.usage.output_tokens"] == 2 + assert span["data"]["gen_ai.usage.input_tokens"] == 7 + assert span["data"]["gen_ai.usage.total_tokens"] == 9 except ImportError: pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly # noinspection PyTypeChecker -@pytest.mark.asyncio -@pytest.mark.parametrize( - "send_default_pii, include_prompts", - [(True, True), (True, False), (False, True), (False, False)], -) @pytest.mark.parametrize( - "input", + "messages", [ pytest.param( [ @@ -427,74 +523,66 @@ def test_streaming_chat_completion( ), ], ) -async def test_streaming_chat_completion_async( - sentry_init, capture_events, send_default_pii, include_prompts, input, request -): +def test_streaming_chat_completion(sentry_init, capture_events, messages, request): sentry_init( integrations=[ OpenAIIntegration( - include_prompts=include_prompts, + include_prompts=True, tiktoken_encoding_name=tiktoken_encoding_if_installed(), ) ], traces_sample_rate=1.0, - send_default_pii=send_default_pii, + send_default_pii=True, ) events = capture_events() - client = AsyncOpenAI(api_key="z") - returned_stream = AsyncStream(cast_to=None, response=None, client=client) - returned_stream._iterator = async_iterator( - [ - ChatCompletionChunk( - id="1", - choices=[ - DeltaChoice( - index=0, delta=ChoiceDelta(content="hel"), finish_reason=None - ) - ], - created=100000, - model="model-id", - object="chat.completion.chunk", - ), - ChatCompletionChunk( - id="1", - choices=[ - DeltaChoice( - index=1, delta=ChoiceDelta(content="lo "), finish_reason=None - ) - ], - created=100000, - model="model-id", - object="chat.completion.chunk", - ), - ChatCompletionChunk( - id="1", - choices=[ - DeltaChoice( - index=2, - delta=ChoiceDelta(content="world"), - finish_reason="stop", - ) - ], - created=100000, - model="model-id", - object="chat.completion.chunk", - ), - ] - ) - - client.chat.completions._post = AsyncMock(return_value=returned_stream) - with start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", - messages=input, - ) - - response_string = "" - async for x in response_stream: - response_string += x.choices[0].delta.content + client = OpenAI(api_key="z") + returned_stream = Stream(cast_to=None, response=None, client=client) + returned_stream._iterator = [ + ChatCompletionChunk( + id="1", + choices=[ + DeltaChoice( + index=0, delta=ChoiceDelta(content="hel"), finish_reason=None + ) + ], + created=100000, + model="model-id", + object="chat.completion.chunk", + ), + ChatCompletionChunk( + id="1", + choices=[ + DeltaChoice( + index=1, delta=ChoiceDelta(content="lo "), finish_reason=None + ) + ], + created=100000, + model="model-id", + object="chat.completion.chunk", + ), + ChatCompletionChunk( + id="1", + choices=[ + DeltaChoice( + index=2, delta=ChoiceDelta(content="world"), finish_reason="stop" + ) + ], + created=100000, + model="model-id", + object="chat.completion.chunk", + ), + ] + client.chat.completions._post = mock.Mock(return_value=returned_stream) + with start_transaction(name="openai tx"): + response_stream = client.chat.completions.create( + model="some-model", + messages=messages, + ) + response_string = "".join( + map(lambda x: x.choices[0].delta.content, response_stream) + ) assert response_string == "hello world" tx = events[0] assert tx["type"] == "transaction" @@ -502,32 +590,27 @@ async def test_streaming_chat_completion_async( assert span["op"] == "gen_ai.chat" param_id = request.node.callspec.id - if send_default_pii and include_prompts: - if "blocks" in param_id: - assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ - { - "type": "text", - "content": "You are a helpful assistant.", - } - ] - else: - assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ - { - "type": "text", - "content": "You are a helpful assistant.", - }, - { - "type": "text", - "content": "Be concise and clear.", - }, - ] - - assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert "hello world" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] + if "blocks" in param_id: + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "type": "text", + "content": "You are a helpful assistant.", + } + ] else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "type": "text", + "content": "You are a helpful assistant.", + }, + { + "type": "text", + "content": "Be concise and clear.", + }, + ] + + assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + assert "hello world" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] try: import tiktoken # type: ignore # noqa # pylint: disable=unused-import @@ -540,127 +623,371 @@ async def test_streaming_chat_completion_async( assert span["data"]["gen_ai.usage.output_tokens"] == 2 assert span["data"]["gen_ai.usage.input_tokens"] == 1 assert span["data"]["gen_ai.usage.total_tokens"] == 3 - except ImportError: pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly -def test_bad_chat_completion(sentry_init, capture_events): - sentry_init(integrations=[OpenAIIntegration()], traces_sample_rate=1.0) - events = capture_events() - - client = OpenAI(api_key="z") - client.chat.completions._post = mock.Mock( - side_effect=OpenAIError("API rate limit reached") - ) - with pytest.raises(OpenAIError): - client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) - - (event,) = events - assert event["level"] == "error" - - -def test_span_status_error(sentry_init, capture_events): - sentry_init(integrations=[OpenAIIntegration()], traces_sample_rate=1.0) - events = capture_events() - - with start_transaction(name="test"): - client = OpenAI(api_key="z") - client.chat.completions._post = mock.Mock( - side_effect=OpenAIError("API rate limit reached") - ) - with pytest.raises(OpenAIError): - client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) - - (error, transaction) = events - assert error["level"] == "error" - assert transaction["spans"][0]["status"] == "internal_error" - assert transaction["spans"][0]["tags"]["status"] == "internal_error" - assert transaction["contexts"]["trace"]["status"] == "internal_error" - - +# noinspection PyTypeChecker @pytest.mark.asyncio -async def test_bad_chat_completion_async(sentry_init, capture_events): - sentry_init(integrations=[OpenAIIntegration()], traces_sample_rate=1.0) - events = capture_events() - - client = AsyncOpenAI(api_key="z") - client.chat.completions._post = AsyncMock( - side_effect=OpenAIError("API rate limit reached") - ) - with pytest.raises(OpenAIError): - await client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) - - (event,) = events - assert event["level"] == "error" - - @pytest.mark.parametrize( "send_default_pii, include_prompts", - [(True, True), (True, False), (False, True), (False, False)], + [ + (True, False), + (False, True), + (False, False), + ], ) -def test_embeddings_create( +async def test_streaming_chat_completion_async_no_prompts( sentry_init, capture_events, send_default_pii, include_prompts ): sentry_init( - integrations=[OpenAIIntegration(include_prompts=include_prompts)], + integrations=[ + OpenAIIntegration( + include_prompts=include_prompts, + tiktoken_encoding_name=tiktoken_encoding_if_installed(), + ) + ], traces_sample_rate=1.0, send_default_pii=send_default_pii, ) events = capture_events() - client = OpenAI(api_key="z") - - returned_embedding = CreateEmbeddingResponse( - data=[Embedding(object="embedding", index=0, embedding=[1.0, 2.0, 3.0])], - model="some-model", - object="list", - usage=EmbeddingTokenUsage( - prompt_tokens=20, - total_tokens=30, - ), + client = AsyncOpenAI(api_key="z") + returned_stream = AsyncStream(cast_to=None, response=None, client=client) + returned_stream._iterator = async_iterator( + [ + ChatCompletionChunk( + id="1", + choices=[ + DeltaChoice( + index=0, delta=ChoiceDelta(content="hel"), finish_reason=None + ) + ], + created=100000, + model="model-id", + object="chat.completion.chunk", + ), + ChatCompletionChunk( + id="1", + choices=[ + DeltaChoice( + index=1, delta=ChoiceDelta(content="lo "), finish_reason=None + ) + ], + created=100000, + model="model-id", + object="chat.completion.chunk", + ), + ChatCompletionChunk( + id="1", + choices=[ + DeltaChoice( + index=2, + delta=ChoiceDelta(content="world"), + finish_reason="stop", + ) + ], + created=100000, + model="model-id", + object="chat.completion.chunk", + ), + ] ) - client.embeddings._post = mock.Mock(return_value=returned_embedding) + client.chat.completions._post = AsyncMock(return_value=returned_stream) with start_transaction(name="openai tx"): - response = client.embeddings.create( - input="hello", model="text-embedding-3-large" + response_stream = await client.chat.completions.create( + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], ) - assert len(response.data[0].embedding) == 3 + response_string = "" + async for x in response_stream: + response_string += x.choices[0].delta.content + assert response_string == "hello world" tx = events[0] assert tx["type"] == "transaction" span = tx["spans"][0] - assert span["op"] == "gen_ai.embeddings" - if send_default_pii and include_prompts: - assert "hello" in span["data"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] - else: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in span["data"] + assert span["op"] == "gen_ai.chat" - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert span["data"]["gen_ai.usage.total_tokens"] == 30 + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + + try: + import tiktoken # type: ignore # noqa # pylint: disable=unused-import + + assert span["data"]["gen_ai.usage.output_tokens"] == 2 + assert span["data"]["gen_ai.usage.input_tokens"] == 7 + assert span["data"]["gen_ai.usage.total_tokens"] == 9 + + except ImportError: + pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly +# noinspection PyTypeChecker @pytest.mark.asyncio @pytest.mark.parametrize( - "send_default_pii, include_prompts", - [(True, True), (True, False), (False, True), (False, False)], + "messages", + [ + pytest.param( + [ + { + "role": "system", + "content": "You are a helpful assistant.", + }, + {"role": "user", "content": "hello"}, + ], + id="blocks", + ), + pytest.param( + [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful assistant."}, + {"type": "text", "text": "Be concise and clear."}, + ], + }, + {"role": "user", "content": "hello"}, + ], + id="parts", + ), + ], ) -async def test_embeddings_create_async( - sentry_init, capture_events, send_default_pii, include_prompts +async def test_streaming_chat_completion_async( + sentry_init, capture_events, messages, request ): sentry_init( - integrations=[OpenAIIntegration(include_prompts=include_prompts)], - traces_sample_rate=1.0, - send_default_pii=send_default_pii, - ) + integrations=[ + OpenAIIntegration( + include_prompts=True, + tiktoken_encoding_name=tiktoken_encoding_if_installed(), + ) + ], + traces_sample_rate=1.0, + send_default_pii=True, + ) + events = capture_events() + + client = AsyncOpenAI(api_key="z") + returned_stream = AsyncStream(cast_to=None, response=None, client=client) + returned_stream._iterator = async_iterator( + [ + ChatCompletionChunk( + id="1", + choices=[ + DeltaChoice( + index=0, delta=ChoiceDelta(content="hel"), finish_reason=None + ) + ], + created=100000, + model="model-id", + object="chat.completion.chunk", + ), + ChatCompletionChunk( + id="1", + choices=[ + DeltaChoice( + index=1, delta=ChoiceDelta(content="lo "), finish_reason=None + ) + ], + created=100000, + model="model-id", + object="chat.completion.chunk", + ), + ChatCompletionChunk( + id="1", + choices=[ + DeltaChoice( + index=2, + delta=ChoiceDelta(content="world"), + finish_reason="stop", + ) + ], + created=100000, + model="model-id", + object="chat.completion.chunk", + ), + ] + ) + + client.chat.completions._post = AsyncMock(return_value=returned_stream) + with start_transaction(name="openai tx"): + response_stream = await client.chat.completions.create( + model="some-model", + messages=messages, + ) + + response_string = "" + async for x in response_stream: + response_string += x.choices[0].delta.content + + assert response_string == "hello world" + tx = events[0] + assert tx["type"] == "transaction" + span = tx["spans"][0] + assert span["op"] == "gen_ai.chat" + + param_id = request.node.callspec.id + if "blocks" in param_id: + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "type": "text", + "content": "You are a helpful assistant.", + } + ] + else: + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "type": "text", + "content": "You are a helpful assistant.", + }, + { + "type": "text", + "content": "Be concise and clear.", + }, + ] + + assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + assert "hello world" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] + + try: + import tiktoken # type: ignore # noqa # pylint: disable=unused-import + + if "blocks" in param_id: + assert span["data"]["gen_ai.usage.output_tokens"] == 2 + assert span["data"]["gen_ai.usage.input_tokens"] == 7 + assert span["data"]["gen_ai.usage.total_tokens"] == 9 + else: + assert span["data"]["gen_ai.usage.output_tokens"] == 2 + assert span["data"]["gen_ai.usage.input_tokens"] == 1 + assert span["data"]["gen_ai.usage.total_tokens"] == 3 + + except ImportError: + pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly + + +def test_bad_chat_completion(sentry_init, capture_events): + sentry_init(integrations=[OpenAIIntegration()], traces_sample_rate=1.0) + events = capture_events() + + client = OpenAI(api_key="z") + client.chat.completions._post = mock.Mock( + side_effect=OpenAIError("API rate limit reached") + ) + with pytest.raises(OpenAIError): + client.chat.completions.create( + model="some-model", messages=[{"role": "system", "content": "hello"}] + ) + + (event,) = events + assert event["level"] == "error" + + +def test_span_status_error(sentry_init, capture_events): + sentry_init(integrations=[OpenAIIntegration()], traces_sample_rate=1.0) + events = capture_events() + + with start_transaction(name="test"): + client = OpenAI(api_key="z") + client.chat.completions._post = mock.Mock( + side_effect=OpenAIError("API rate limit reached") + ) + with pytest.raises(OpenAIError): + client.chat.completions.create( + model="some-model", messages=[{"role": "system", "content": "hello"}] + ) + + (error, transaction) = events + assert error["level"] == "error" + assert transaction["spans"][0]["status"] == "internal_error" + assert transaction["spans"][0]["tags"]["status"] == "internal_error" + assert transaction["contexts"]["trace"]["status"] == "internal_error" + + +@pytest.mark.asyncio +async def test_bad_chat_completion_async(sentry_init, capture_events): + sentry_init(integrations=[OpenAIIntegration()], traces_sample_rate=1.0) + events = capture_events() + + client = AsyncOpenAI(api_key="z") + client.chat.completions._post = AsyncMock( + side_effect=OpenAIError("API rate limit reached") + ) + with pytest.raises(OpenAIError): + await client.chat.completions.create( + model="some-model", messages=[{"role": "system", "content": "hello"}] + ) + + (event,) = events + assert event["level"] == "error" + + +@pytest.mark.parametrize( + "send_default_pii, include_prompts", + [(True, True), (True, False), (False, True), (False, False)], +) +def test_embeddings_create( + sentry_init, capture_events, send_default_pii, include_prompts +): + sentry_init( + integrations=[OpenAIIntegration(include_prompts=include_prompts)], + traces_sample_rate=1.0, + send_default_pii=send_default_pii, + ) + events = capture_events() + + client = OpenAI(api_key="z") + + returned_embedding = CreateEmbeddingResponse( + data=[Embedding(object="embedding", index=0, embedding=[1.0, 2.0, 3.0])], + model="some-model", + object="list", + usage=EmbeddingTokenUsage( + prompt_tokens=20, + total_tokens=30, + ), + ) + + client.embeddings._post = mock.Mock(return_value=returned_embedding) + with start_transaction(name="openai tx"): + response = client.embeddings.create( + input="hello", model="text-embedding-3-large" + ) + + assert len(response.data[0].embedding) == 3 + + tx = events[0] + assert tx["type"] == "transaction" + span = tx["spans"][0] + assert span["op"] == "gen_ai.embeddings" + if send_default_pii and include_prompts: + assert "hello" in span["data"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] + else: + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in span["data"] + + assert span["data"]["gen_ai.usage.input_tokens"] == 20 + assert span["data"]["gen_ai.usage.total_tokens"] == 30 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "send_default_pii, include_prompts", + [(True, True), (True, False), (False, True), (False, False)], +) +async def test_embeddings_create_async( + sentry_init, capture_events, send_default_pii, include_prompts +): + sentry_init( + integrations=[OpenAIIntegration(include_prompts=include_prompts)], + traces_sample_rate=1.0, + send_default_pii=send_default_pii, + ) events = capture_events() client = AsyncOpenAI(api_key="z") @@ -1181,6 +1508,311 @@ def test_ai_client_span_responses_api_no_pii(sentry_init, capture_events): "instructions", ( omit, + None, + "You are a coding assistant that talks like a pirate.", + ), +) +@pytest.mark.parametrize( + "input", + [ + pytest.param( + "How do I check if a Python object is an instance of a class?", id="string" + ), + pytest.param( + [ + { + "role": "system", + "content": "You are a helpful assistant.", + }, + {"role": "user", "content": "hello"}, + ], + id="blocks_no_type", + ), + pytest.param( + [ + { + "type": "message", + "role": "system", + "content": "You are a helpful assistant.", + }, + {"type": "message", "role": "user", "content": "hello"}, + ], + id="blocks", + ), + pytest.param( + [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful assistant."}, + {"type": "text", "text": "Be concise and clear."}, + ], + }, + {"role": "user", "content": "hello"}, + ], + id="parts_no_type", + ), + pytest.param( + [ + { + "type": "message", + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful assistant."}, + {"type": "text", "text": "Be concise and clear."}, + ], + }, + {"type": "message", "role": "user", "content": "hello"}, + ], + id="parts", + ), + ], +) +@pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") +def test_ai_client_span_responses_api( + sentry_init, capture_events, instructions, input, request +): + sentry_init( + integrations=[OpenAIIntegration(include_prompts=True)], + traces_sample_rate=1.0, + send_default_pii=True, + ) + events = capture_events() + + client = OpenAI(api_key="z") + client.responses._post = mock.Mock(return_value=EXAMPLE_RESPONSE) + + with start_transaction(name="openai tx"): + client.responses.create( + model="gpt-4o", + instructions=instructions, + input=input, + ) + + (transaction,) = events + spans = transaction["spans"] + + assert len(spans) == 1 + assert spans[0]["op"] == "gen_ai.responses" + assert spans[0]["origin"] == "auto.ai.openai" + + expected_data = { + "gen_ai.operation.name": "responses", + "gen_ai.system": "openai", + "gen_ai.response.model": "response-model-id", + "gen_ai.usage.input_tokens": 20, + "gen_ai.usage.input_tokens.cached": 5, + "gen_ai.usage.output_tokens": 10, + "gen_ai.usage.output_tokens.reasoning": 8, + "gen_ai.usage.total_tokens": 30, + "gen_ai.request.model": "gpt-4o", + "gen_ai.response.text": "the model response", + "sentry.sdk_meta.gen_ai.input.messages.original_length": 1, + "thread.id": mock.ANY, + "thread.name": mock.ANY, + } + + param_id = request.node.callspec.id + if "string" in param_id and ( + instructions is None or isinstance(instructions, Omit) + ): # type: ignore + expected_data.update( + { + "gen_ai.request.messages": safe_serialize( + ["How do I check if a Python object is an instance of a class?"] + ), + } + ) + elif "string" in param_id: + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [ + { + "type": "text", + "content": "You are a coding assistant that talks like a pirate.", + } + ] + ), + "gen_ai.request.messages": safe_serialize( + ["How do I check if a Python object is an instance of a class?"] + ), + } + ) + elif "blocks_no_type" in param_id and ( + instructions is None or isinstance(instructions, Omit) + ): # type: ignore + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [{"type": "text", "content": "You are a helpful assistant."}] + ), + "gen_ai.request.messages": safe_serialize( + [{"role": "user", "content": "hello"}] + ), + } + ) + elif "blocks_no_type" in param_id: + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [ + { + "type": "text", + "content": "You are a coding assistant that talks like a pirate.", + }, + {"type": "text", "content": "You are a helpful assistant."}, + ] + ), + "gen_ai.request.messages": safe_serialize( + [{"role": "user", "content": "hello"}] + ), + } + ) + elif "blocks" in param_id and ( + instructions is None or isinstance(instructions, Omit) + ): # type: ignore + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [{"type": "text", "content": "You are a helpful assistant."}] + ), + "gen_ai.request.messages": safe_serialize( + [{"type": "message", "role": "user", "content": "hello"}] + ), + } + ) + elif "blocks" in param_id: + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [ + { + "type": "text", + "content": "You are a coding assistant that talks like a pirate.", + }, + {"type": "text", "content": "You are a helpful assistant."}, + ] + ), + "gen_ai.request.messages": safe_serialize( + [{"type": "message", "role": "user", "content": "hello"}] + ), + } + ) + elif "parts_no_type" in param_id and ( + instructions is None or isinstance(instructions, Omit) + ): # type: ignore + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [ + {"type": "text", "content": "You are a helpful assistant."}, + {"type": "text", "content": "Be concise and clear."}, + ] + ), + "gen_ai.request.messages": safe_serialize( + [{"role": "user", "content": "hello"}] + ), + } + ) + elif "parts_no_type" in param_id: + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [ + { + "type": "text", + "content": "You are a coding assistant that talks like a pirate.", + }, + {"type": "text", "content": "You are a helpful assistant."}, + {"type": "text", "content": "Be concise and clear."}, + ] + ), + "gen_ai.request.messages": safe_serialize( + [{"role": "user", "content": "hello"}] + ), + } + ) + elif instructions is None or isinstance(instructions, Omit): # type: ignore + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [ + {"type": "text", "content": "You are a helpful assistant."}, + {"type": "text", "content": "Be concise and clear."}, + ] + ), + "gen_ai.request.messages": safe_serialize( + [{"type": "message", "role": "user", "content": "hello"}] + ), + } + ) + else: + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [ + { + "type": "text", + "content": "You are a coding assistant that talks like a pirate.", + }, + {"type": "text", "content": "You are a helpful assistant."}, + {"type": "text", "content": "Be concise and clear."}, + ] + ), + "gen_ai.request.messages": safe_serialize( + [{"type": "message", "role": "user", "content": "hello"}] + ), + } + ) + + assert spans[0]["data"] == expected_data + + +@pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") +def test_error_in_responses_api(sentry_init, capture_events): + sentry_init( + integrations=[OpenAIIntegration(include_prompts=True)], + traces_sample_rate=1.0, + send_default_pii=True, + ) + events = capture_events() + + client = OpenAI(api_key="z") + client.responses._post = mock.Mock( + side_effect=OpenAIError("API rate limit reached") + ) + + with start_transaction(name="openai tx"): + with pytest.raises(OpenAIError): + client.responses.create( + model="gpt-4o", + instructions="You are a coding assistant that talks like a pirate.", + input="How do I check if a Python object is an instance of a class?", + ) + + (error_event, transaction_event) = events + + assert transaction_event["type"] == "transaction" + # make sure the span where the error occurred is captured + assert transaction_event["spans"][0]["op"] == "gen_ai.responses" + + assert error_event["level"] == "error" + assert error_event["exception"]["values"][0]["type"] == "OpenAIError" + + assert ( + error_event["contexts"]["trace"]["trace_id"] + == transaction_event["contexts"]["trace"]["trace_id"] + ) + + +@pytest.mark.asyncio +@pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") +@pytest.mark.parametrize( + "instructions", + ( + omit, + None, "You are a coding assistant that talks like a pirate.", ), ) @@ -1190,6 +1822,16 @@ def test_ai_client_span_responses_api_no_pii(sentry_init, capture_events): pytest.param( "How do I check if a Python object is an instance of a class?", id="string" ), + pytest.param( + [ + { + "role": "system", + "content": "You are a helpful assistant.", + }, + {"role": "user", "content": "hello"}, + ], + id="blocks_no_type", + ), pytest.param( [ { @@ -1201,6 +1843,19 @@ def test_ai_client_span_responses_api_no_pii(sentry_init, capture_events): ], id="blocks", ), + pytest.param( + [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful assistant."}, + {"type": "text", "text": "Be concise and clear."}, + ], + }, + {"role": "user", "content": "hello"}, + ], + id="parts_no_type", + ), pytest.param( [ { @@ -1217,8 +1872,7 @@ def test_ai_client_span_responses_api_no_pii(sentry_init, capture_events): ), ], ) -@pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") -def test_ai_client_span_responses_api( +async def test_ai_client_span_responses_async_api( sentry_init, capture_events, instructions, input, request ): sentry_init( @@ -1228,11 +1882,11 @@ def test_ai_client_span_responses_api( ) events = capture_events() - client = OpenAI(api_key="z") - client.responses._post = mock.Mock(return_value=EXAMPLE_RESPONSE) + client = AsyncOpenAI(api_key="z") + client.responses._post = AsyncMock(return_value=EXAMPLE_RESPONSE) with start_transaction(name="openai tx"): - client.responses.create( + await client.responses.create( model="gpt-4o", instructions=instructions, input=input, @@ -1247,21 +1901,25 @@ def test_ai_client_span_responses_api( expected_data = { "gen_ai.operation.name": "responses", - "gen_ai.system": "openai", + "gen_ai.request.messages": '["How do I check if a Python object is an instance of a class?"]', + "gen_ai.request.model": "gpt-4o", "gen_ai.response.model": "response-model-id", + "gen_ai.system": "openai", "gen_ai.usage.input_tokens": 20, "gen_ai.usage.input_tokens.cached": 5, "gen_ai.usage.output_tokens": 10, "gen_ai.usage.output_tokens.reasoning": 8, "gen_ai.usage.total_tokens": 30, - "gen_ai.request.model": "gpt-4o", "gen_ai.response.text": "the model response", + "sentry.sdk_meta.gen_ai.input.messages.original_length": 1, "thread.id": mock.ANY, "thread.name": mock.ANY, } param_id = request.node.callspec.id - if "string" in param_id and isinstance(instructions, Omit): # type: ignore + if "string" in param_id and ( + instructions is None or isinstance(instructions, Omit) + ): # type: ignore expected_data.update( { "gen_ai.request.messages": safe_serialize( @@ -1285,7 +1943,39 @@ def test_ai_client_span_responses_api( ), } ) - elif "blocks" in param_id and isinstance(instructions, Omit): # type: ignore + elif "blocks_no_type" in param_id and ( + instructions is None or isinstance(instructions, Omit) + ): # type: ignore + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [{"type": "text", "content": "You are a helpful assistant."}] + ), + "gen_ai.request.messages": safe_serialize( + [{"role": "user", "content": "hello"}] + ), + } + ) + elif "blocks_no_type" in param_id: + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [ + { + "type": "text", + "content": "You are a coding assistant that talks like a pirate.", + }, + {"type": "text", "content": "You are a helpful assistant."}, + ] + ), + "gen_ai.request.messages": safe_serialize( + [{"role": "user", "content": "hello"}] + ), + } + ) + elif "blocks" in param_id and ( + instructions is None or isinstance(instructions, Omit) + ): # type: ignore expected_data.update( { "gen_ai.system_instructions": safe_serialize( @@ -1313,7 +2003,41 @@ def test_ai_client_span_responses_api( ), } ) - elif isinstance(instructions, Omit): # type: ignore + elif "parts_no_type" in param_id and ( + instructions is None or isinstance(instructions, Omit) + ): # type: ignore + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [ + {"type": "text", "content": "You are a helpful assistant."}, + {"type": "text", "content": "Be concise and clear."}, + ] + ), + "gen_ai.request.messages": safe_serialize( + [{"role": "user", "content": "hello"}] + ), + } + ) + elif "parts_no_type" in param_id: + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [ + { + "type": "text", + "content": "You are a coding assistant that talks like a pirate.", + }, + {"type": "text", "content": "You are a helpful assistant."}, + {"type": "text", "content": "Be concise and clear."}, + ] + ), + "gen_ai.request.messages": safe_serialize( + [{"role": "user", "content": "hello"}] + ), + } + ) + elif instructions is None or isinstance(instructions, Omit): # type: ignore expected_data.update( { "gen_ai.system_instructions": safe_serialize( @@ -1349,92 +2073,12 @@ def test_ai_client_span_responses_api( assert spans[0]["data"] == expected_data -@pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") -def test_error_in_responses_api(sentry_init, capture_events): - sentry_init( - integrations=[OpenAIIntegration(include_prompts=True)], - traces_sample_rate=1.0, - send_default_pii=True, - ) - events = capture_events() - - client = OpenAI(api_key="z") - client.responses._post = mock.Mock( - side_effect=OpenAIError("API rate limit reached") - ) - - with start_transaction(name="openai tx"): - with pytest.raises(OpenAIError): - client.responses.create( - model="gpt-4o", - instructions="You are a coding assistant that talks like a pirate.", - input="How do I check if a Python object is an instance of a class?", - ) - - (error_event, transaction_event) = events - - assert transaction_event["type"] == "transaction" - # make sure the span where the error occurred is captured - assert transaction_event["spans"][0]["op"] == "gen_ai.responses" - - assert error_event["level"] == "error" - assert error_event["exception"]["values"][0]["type"] == "OpenAIError" - - assert ( - error_event["contexts"]["trace"]["trace_id"] - == transaction_event["contexts"]["trace"]["trace_id"] - ) - - -@pytest.mark.asyncio -@pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") -async def test_ai_client_span_responses_async_api(sentry_init, capture_events): - sentry_init( - integrations=[OpenAIIntegration(include_prompts=True)], - traces_sample_rate=1.0, - send_default_pii=True, - ) - events = capture_events() - - client = AsyncOpenAI(api_key="z") - client.responses._post = AsyncMock(return_value=EXAMPLE_RESPONSE) - - with start_transaction(name="openai tx"): - await client.responses.create( - model="gpt-4o", - instructions="You are a coding assistant that talks like a pirate.", - input="How do I check if a Python object is an instance of a class?", - ) - - (transaction,) = events - spans = transaction["spans"] - - assert len(spans) == 1 - assert spans[0]["op"] == "gen_ai.responses" - assert spans[0]["origin"] == "auto.ai.openai" - assert spans[0]["data"] == { - "gen_ai.operation.name": "responses", - "gen_ai.request.messages": '["How do I check if a Python object is an instance of a class?"]', - "gen_ai.request.model": "gpt-4o", - "gen_ai.response.model": "response-model-id", - "gen_ai.system": "openai", - "gen_ai.system_instructions": '[{"type": "text", "content": "You are a coding assistant that talks like a pirate."}]', - "gen_ai.usage.input_tokens": 20, - "gen_ai.usage.input_tokens.cached": 5, - "gen_ai.usage.output_tokens": 10, - "gen_ai.usage.output_tokens.reasoning": 8, - "gen_ai.usage.total_tokens": 30, - "gen_ai.response.text": "the model response", - "thread.id": mock.ANY, - "thread.name": mock.ANY, - } - - @pytest.mark.asyncio @pytest.mark.parametrize( "instructions", ( omit, + None, "You are a coding assistant that talks like a pirate.", ), ) @@ -1444,6 +2088,16 @@ async def test_ai_client_span_responses_async_api(sentry_init, capture_events): pytest.param( "How do I check if a Python object is an instance of a class?", id="string" ), + pytest.param( + [ + { + "role": "system", + "content": "You are a helpful assistant.", + }, + {"role": "user", "content": "hello"}, + ], + id="blocks_no_type", + ), pytest.param( [ { @@ -1455,6 +2109,19 @@ async def test_ai_client_span_responses_async_api(sentry_init, capture_events): ], id="blocks", ), + pytest.param( + [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful assistant."}, + {"type": "text", "text": "Be concise and clear."}, + ], + }, + {"role": "user", "content": "hello"}, + ], + id="parts_no_type", + ), pytest.param( [ { @@ -1512,12 +2179,15 @@ async def test_ai_client_span_streaming_responses_async_api( "gen_ai.usage.total_tokens": 30, "gen_ai.request.model": "gpt-4o", "gen_ai.response.text": "the model response", + "sentry.sdk_meta.gen_ai.input.messages.original_length": 1, "thread.id": mock.ANY, "thread.name": mock.ANY, } param_id = request.node.callspec.id - if "string" in param_id and isinstance(instructions, Omit): # type: ignore + if "string" in param_id and ( + instructions is None or isinstance(instructions, Omit) + ): # type: ignore expected_data.update( { "gen_ai.request.messages": safe_serialize( @@ -1541,7 +2211,39 @@ async def test_ai_client_span_streaming_responses_async_api( ), } ) - elif "blocks" in param_id and isinstance(instructions, Omit): # type: ignore + elif "blocks_no_type" in param_id and ( + instructions is None or isinstance(instructions, Omit) + ): # type: ignore + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [{"type": "text", "content": "You are a helpful assistant."}] + ), + "gen_ai.request.messages": safe_serialize( + [{"role": "user", "content": "hello"}] + ), + } + ) + elif "blocks_no_type" in param_id: + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [ + { + "type": "text", + "content": "You are a coding assistant that talks like a pirate.", + }, + {"type": "text", "content": "You are a helpful assistant."}, + ] + ), + "gen_ai.request.messages": safe_serialize( + [{"role": "user", "content": "hello"}] + ), + } + ) + elif "blocks" in param_id and ( + instructions is None or isinstance(instructions, Omit) + ): # type: ignore expected_data.update( { "gen_ai.system_instructions": safe_serialize( @@ -1569,7 +2271,41 @@ async def test_ai_client_span_streaming_responses_async_api( ), } ) - elif isinstance(instructions, Omit): # type: ignore + elif "parts_no_type" in param_id and ( + instructions is None or isinstance(instructions, Omit) + ): # type: ignore + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [ + {"type": "text", "content": "You are a helpful assistant."}, + {"type": "text", "content": "Be concise and clear."}, + ] + ), + "gen_ai.request.messages": safe_serialize( + [{"role": "user", "content": "hello"}] + ), + } + ) + elif "parts_no_type" in param_id: + expected_data.update( + { + "gen_ai.system_instructions": safe_serialize( + [ + { + "type": "text", + "content": "You are a coding assistant that talks like a pirate.", + }, + {"type": "text", "content": "You are a helpful assistant."}, + {"type": "text", "content": "Be concise and clear."}, + ] + ), + "gen_ai.request.messages": safe_serialize( + [{"role": "user", "content": "hello"}] + ), + } + ) + elif instructions is None or isinstance(instructions, Omit): # type: ignore expected_data.update( { "gen_ai.system_instructions": safe_serialize( @@ -1856,7 +2592,24 @@ def test_empty_tools_in_chat_completion(sentry_init, capture_events, tools): assert "gen_ai.request.available_tools" not in span["data"] -def test_openai_message_role_mapping(sentry_init, capture_events): +# Test messages with mixed roles including "ai" that should be mapped to "assistant" +@pytest.mark.parametrize( + "test_message,expected_role", + [ + ({"role": "user", "content": "Hello"}, "user"), + ( + {"role": "ai", "content": "Hi there!"}, + "assistant", + ), # Should be mapped to "assistant" + ( + {"role": "assistant", "content": "How can I help?"}, + "assistant", + ), # Should stay "assistant" + ], +) +def test_openai_message_role_mapping( + sentry_init, capture_events, test_message, expected_role +): """Test that OpenAI integration properly maps message roles like 'ai' to 'assistant'""" sentry_init( @@ -1868,12 +2621,8 @@ def test_openai_message_role_mapping(sentry_init, capture_events): client = OpenAI(api_key="z") client.chat.completions._post = mock.Mock(return_value=EXAMPLE_CHAT_COMPLETION) - # Test messages with mixed roles including "ai" that should be mapped to "assistant" - test_messages = [ - {"role": "user", "content": "Hello"}, - {"role": "ai", "content": "Hi there!"}, # Should be mapped to "assistant" - {"role": "assistant", "content": "How can I help?"}, # Should stay "assistant" - ] + + test_messages = [test_message] with start_transaction(name="openai tx"): client.chat.completions.create(model="test-model", messages=test_messages) @@ -1888,20 +2637,8 @@ def test_openai_message_role_mapping(sentry_init, capture_events): stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) - # Verify that "ai" role was mapped to "assistant" - assert len(stored_messages) == 3 - assert ( - stored_messages[1]["role"] == "assistant" - ) # "ai" should be mapped to "assistant" - assert stored_messages[2]["role"] == "assistant" # should stay "assistant" - - # Verify content is preserved - assert stored_messages[1]["content"] == "Hi there!" - assert stored_messages[2]["content"] == "How can I help?" - - # Verify no "ai" roles remain - roles = [msg["role"] for msg in stored_messages] - assert "ai" not in roles + assert len(stored_messages) == 1 + assert stored_messages[0]["role"] == expected_role def test_openai_message_truncation(sentry_init, capture_events): @@ -1943,14 +2680,9 @@ def test_openai_message_truncation(sentry_init, capture_events): assert isinstance(parsed_messages, list) assert len(parsed_messages) <= len(large_messages) - if "_meta" in event and len(parsed_messages) < len(large_messages): - meta_path = event["_meta"] - if ( - "spans" in meta_path - and "0" in meta_path["spans"] - and "data" in meta_path["spans"]["0"] - ): - span_meta = meta_path["spans"]["0"]["data"] - if SPANDATA.GEN_AI_REQUEST_MESSAGES in span_meta: - messages_meta = span_meta[SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert "len" in messages_meta.get("", {}) + meta_path = event["_meta"] + span_meta = meta_path["spans"]["0"]["data"] + messages_meta = span_meta[SPANDATA.GEN_AI_REQUEST_MESSAGES] + + assert span["data"][SPANDATA.META_GEN_AI_ORIGINAL_INPUT_MESSAGES_LENGTH] == 3 + assert "len" in messages_meta.get("", {}) From 382e933e51305b122e86244fd6a0d66b5e559ad3 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Mon, 26 Jan 2026 08:58:09 +0100 Subject: [PATCH 17/22] revert unrelated tests --- tests/integrations/openai/test_openai.py | 63 +++++++++++++----------- 1 file changed, 35 insertions(+), 28 deletions(-) diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index 4d15283ea9..85acbb1486 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -2592,24 +2592,7 @@ def test_empty_tools_in_chat_completion(sentry_init, capture_events, tools): assert "gen_ai.request.available_tools" not in span["data"] -# Test messages with mixed roles including "ai" that should be mapped to "assistant" -@pytest.mark.parametrize( - "test_message,expected_role", - [ - ({"role": "user", "content": "Hello"}, "user"), - ( - {"role": "ai", "content": "Hi there!"}, - "assistant", - ), # Should be mapped to "assistant" - ( - {"role": "assistant", "content": "How can I help?"}, - "assistant", - ), # Should stay "assistant" - ], -) -def test_openai_message_role_mapping( - sentry_init, capture_events, test_message, expected_role -): +def test_openai_message_role_mapping(sentry_init, capture_events): """Test that OpenAI integration properly maps message roles like 'ai' to 'assistant'""" sentry_init( @@ -2621,8 +2604,13 @@ def test_openai_message_role_mapping( client = OpenAI(api_key="z") client.chat.completions._post = mock.Mock(return_value=EXAMPLE_CHAT_COMPLETION) - - test_messages = [test_message] + # Test messages with mixed roles including "ai" that should be mapped to "assistant" + test_messages = [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Hello"}, + {"role": "ai", "content": "Hi there!"}, # Should be mapped to "assistant" + {"role": "assistant", "content": "How can I help?"}, # Should stay "assistant" + ] with start_transaction(name="openai tx"): client.chat.completions.create(model="test-model", messages=test_messages) @@ -2637,8 +2625,22 @@ def test_openai_message_role_mapping( stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == expected_role + # Verify that "ai" role was mapped to "assistant" + assert len(stored_messages) == 4 + assert stored_messages[0]["role"] == "system" + assert stored_messages[1]["role"] == "user" + assert ( + stored_messages[2]["role"] == "assistant" + ) # "ai" should be mapped to "assistant" + assert stored_messages[3]["role"] == "assistant" # should stay "assistant" + + # Verify content is preserved + assert stored_messages[2]["content"] == "Hi there!" + assert stored_messages[3]["content"] == "How can I help?" + + # Verify no "ai" roles remain + roles = [msg["role"] for msg in stored_messages] + assert "ai" not in roles def test_openai_message_truncation(sentry_init, capture_events): @@ -2680,9 +2682,14 @@ def test_openai_message_truncation(sentry_init, capture_events): assert isinstance(parsed_messages, list) assert len(parsed_messages) <= len(large_messages) - meta_path = event["_meta"] - span_meta = meta_path["spans"]["0"]["data"] - messages_meta = span_meta[SPANDATA.GEN_AI_REQUEST_MESSAGES] - - assert span["data"][SPANDATA.META_GEN_AI_ORIGINAL_INPUT_MESSAGES_LENGTH] == 3 - assert "len" in messages_meta.get("", {}) + if "_meta" in event and len(parsed_messages) < len(large_messages): + meta_path = event["_meta"] + if ( + "spans" in meta_path + and "0" in meta_path["spans"] + and "data" in meta_path["spans"]["0"] + ): + span_meta = meta_path["spans"]["0"]["data"] + if SPANDATA.GEN_AI_REQUEST_MESSAGES in span_meta: + messages_meta = span_meta[SPANDATA.GEN_AI_REQUEST_MESSAGES] + assert "len" in messages_meta.get("", {}) From 179d59efd34e98bd553506482d0c888e094f67a9 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Mon, 26 Jan 2026 09:13:46 +0100 Subject: [PATCH 18/22] revert unrelated change --- sentry_sdk/integrations/openai.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index b8f94be995..bd84f3e63e 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -8,7 +8,6 @@ set_data_normalized, normalize_message_roles, truncate_and_annotate_messages, - truncate_and_annotate_embedding_inputs, ) from sentry_sdk.consts import SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration @@ -463,9 +462,7 @@ def _set_embeddings_input_data( ): normalized_messages = normalize_message_roles(messages) # type: ignore scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_embedding_inputs( - normalized_messages, span, scope - ) + messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) if messages_data is not None: set_data_normalized( span, SPANDATA.GEN_AI_EMBEDDINGS_INPUT, messages_data, unpack=False From 12cb21926816beb17537222e440d438543735b5b Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Mon, 26 Jan 2026 09:21:52 +0100 Subject: [PATCH 19/22] address comment --- sentry_sdk/integrations/openai.py | 12 +++++++----- tests/integrations/openai/test_openai.py | 17 ++++++----------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index bd84f3e63e..69d8d069fd 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -1,5 +1,6 @@ import sys from functools import wraps +from collections.abc import Iterable import sentry_sdk from sentry_sdk import consts @@ -31,7 +32,6 @@ AsyncIterator, Iterator, Union, - Iterable, ) from sentry_sdk.tracing import Span from sentry_sdk._types import TextPart @@ -322,10 +322,12 @@ def _set_responses_api_input_data( set_data_normalized( span, SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, - { - "type": "text", - "content": explicit_instructions, - }, + [ + { + "type": "text", + "content": explicit_instructions, + } + ], unpack=False, ) diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index 85acbb1486..813c2ab771 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -1607,7 +1607,6 @@ def test_ai_client_span_responses_api( "gen_ai.usage.total_tokens": 30, "gen_ai.request.model": "gpt-4o", "gen_ai.response.text": "the model response", - "sentry.sdk_meta.gen_ai.input.messages.original_length": 1, "thread.id": mock.ANY, "thread.name": mock.ANY, } @@ -1911,7 +1910,6 @@ async def test_ai_client_span_responses_async_api( "gen_ai.usage.output_tokens.reasoning": 8, "gen_ai.usage.total_tokens": 30, "gen_ai.response.text": "the model response", - "sentry.sdk_meta.gen_ai.input.messages.original_length": 1, "thread.id": mock.ANY, "thread.name": mock.ANY, } @@ -2179,7 +2177,6 @@ async def test_ai_client_span_streaming_responses_async_api( "gen_ai.usage.total_tokens": 30, "gen_ai.request.model": "gpt-4o", "gen_ai.response.text": "the model response", - "sentry.sdk_meta.gen_ai.input.messages.original_length": 1, "thread.id": mock.ANY, "thread.name": mock.ANY, } @@ -2606,7 +2603,6 @@ def test_openai_message_role_mapping(sentry_init, capture_events): client.chat.completions._post = mock.Mock(return_value=EXAMPLE_CHAT_COMPLETION) # Test messages with mixed roles including "ai" that should be mapped to "assistant" test_messages = [ - {"role": "system", "content": "You are helpful."}, {"role": "user", "content": "Hello"}, {"role": "ai", "content": "Hi there!"}, # Should be mapped to "assistant" {"role": "assistant", "content": "How can I help?"}, # Should stay "assistant" @@ -2626,17 +2622,16 @@ def test_openai_message_role_mapping(sentry_init, capture_events): stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) # Verify that "ai" role was mapped to "assistant" - assert len(stored_messages) == 4 - assert stored_messages[0]["role"] == "system" - assert stored_messages[1]["role"] == "user" + assert len(stored_messages) == 3 + assert stored_messages[0]["role"] == "user" assert ( - stored_messages[2]["role"] == "assistant" + stored_messages[1]["role"] == "assistant" ) # "ai" should be mapped to "assistant" - assert stored_messages[3]["role"] == "assistant" # should stay "assistant" + assert stored_messages[2]["role"] == "assistant" # should stay "assistant" # Verify content is preserved - assert stored_messages[2]["content"] == "Hi there!" - assert stored_messages[3]["content"] == "How can I help?" + assert stored_messages[1]["content"] == "Hi there!" + assert stored_messages[2]["content"] == "How can I help?" # Verify no "ai" roles remain roles = [msg["role"] for msg in stored_messages] From a6152fe7809e24a70e401a830189a66dd9cba5b5 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Mon, 26 Jan 2026 09:24:49 +0100 Subject: [PATCH 20/22] remove unused type ignore --- sentry_sdk/integrations/openai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 69d8d069fd..bdc60a0ce8 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -379,7 +379,7 @@ def _set_responses_api_input_data( message for message in messages if not _is_system_instruction_responses(message) ] if len(non_system_messages) > 0: - normalized_messages = normalize_message_roles(non_system_messages) # type: ignore + normalized_messages = normalize_message_roles(non_system_messages) scope = sentry_sdk.get_current_scope() messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) if messages_data is not None: From 5085fd8127c150573faf651699f920b4b0b499e1 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Mon, 26 Jan 2026 12:51:21 +0100 Subject: [PATCH 21/22] split off responses API changes --- sentry_sdk/integrations/openai.py | 79 +-- tests/integrations/openai/test_openai.py | 700 +---------------------- 2 files changed, 25 insertions(+), 754 deletions(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index bdc60a0ce8..391495f227 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -306,80 +306,17 @@ def _set_responses_api_input_data( kwargs: "dict[str, Any]", integration: "OpenAIIntegration", ) -> None: - explicit_instructions: "Union[Optional[str], Omit]" = kwargs.get("instructions") - messages: "Optional[Union[str, ResponseInputParam]]" = kwargs.get("input") - - if not should_send_default_pii() or not integration.include_prompts: - set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "responses") - _commmon_set_input_data(span, kwargs) - return + messages: "Optional[Union[ResponseInputParam, list[str]]]" = _get_input_messages( + kwargs + ) if ( - messages is None - and explicit_instructions is not None - and _is_given(explicit_instructions) + messages is not None + and len(messages) > 0 + and should_send_default_pii() + and integration.include_prompts ): - set_data_normalized( - span, - SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, - [ - { - "type": "text", - "content": explicit_instructions, - } - ], - unpack=False, - ) - - set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "responses") - _commmon_set_input_data(span, kwargs) - return - - if messages is None: - set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "responses") - _commmon_set_input_data(span, kwargs) - return - - instructions_text_parts: "list[TextPart]" = [] - if explicit_instructions is not None and _is_given(explicit_instructions): - instructions_text_parts.append( - { - "type": "text", - "content": explicit_instructions, - } - ) - - system_instructions = _get_system_instructions_responses(messages) - # Deliberate use of function accepting completions API type because - # of shared structure FOR THIS PURPOSE ONLY. - instructions_text_parts += _transform_system_instructions(system_instructions) - - if len(instructions_text_parts) > 0: - set_data_normalized( - span, - SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, - instructions_text_parts, - unpack=False, - ) - - if isinstance(messages, str): - normalized_messages = normalize_message_roles([messages]) # type: ignore - scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) - if messages_data is not None: - set_data_normalized( - span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False - ) - - set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "responses") - _commmon_set_input_data(span, kwargs) - return - - non_system_messages = [ - message for message in messages if not _is_system_instruction_responses(message) - ] - if len(non_system_messages) > 0: - normalized_messages = normalize_message_roles(non_system_messages) + normalized_messages = normalize_message_roles(messages) # type: ignore scope = sentry_sdk.get_current_scope() messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) if messages_data is not None: diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index 813c2ab771..247d0aadfd 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -1499,79 +1499,12 @@ def test_ai_client_span_responses_api_no_pii(sentry_init, capture_events): "thread.name": mock.ANY, } - assert "gen_ai.system_instructions" not in spans[0]["data"] assert "gen_ai.request.messages" not in spans[0]["data"] assert "gen_ai.response.text" not in spans[0]["data"] -@pytest.mark.parametrize( - "instructions", - ( - omit, - None, - "You are a coding assistant that talks like a pirate.", - ), -) -@pytest.mark.parametrize( - "input", - [ - pytest.param( - "How do I check if a Python object is an instance of a class?", id="string" - ), - pytest.param( - [ - { - "role": "system", - "content": "You are a helpful assistant.", - }, - {"role": "user", "content": "hello"}, - ], - id="blocks_no_type", - ), - pytest.param( - [ - { - "type": "message", - "role": "system", - "content": "You are a helpful assistant.", - }, - {"type": "message", "role": "user", "content": "hello"}, - ], - id="blocks", - ), - pytest.param( - [ - { - "role": "system", - "content": [ - {"type": "text", "text": "You are a helpful assistant."}, - {"type": "text", "text": "Be concise and clear."}, - ], - }, - {"role": "user", "content": "hello"}, - ], - id="parts_no_type", - ), - pytest.param( - [ - { - "type": "message", - "role": "system", - "content": [ - {"type": "text", "text": "You are a helpful assistant."}, - {"type": "text", "text": "Be concise and clear."}, - ], - }, - {"type": "message", "role": "user", "content": "hello"}, - ], - id="parts", - ), - ], -) @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") -def test_ai_client_span_responses_api( - sentry_init, capture_events, instructions, input, request -): +def test_ai_client_span_responses_api(sentry_init, capture_events): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], traces_sample_rate=1.0, @@ -1585,8 +1518,8 @@ def test_ai_client_span_responses_api( with start_transaction(name="openai tx"): client.responses.create( model="gpt-4o", - instructions=instructions, - input=input, + instructions="You are a coding assistant that talks like a pirate.", + input="How do I check if a Python object is an instance of a class?", ) (transaction,) = events @@ -1595,9 +1528,10 @@ def test_ai_client_span_responses_api( assert len(spans) == 1 assert spans[0]["op"] == "gen_ai.responses" assert spans[0]["origin"] == "auto.ai.openai" - - expected_data = { + assert spans[0]["data"] == { "gen_ai.operation.name": "responses", + "gen_ai.request.messages": '["How do I check if a Python object is an instance of a class?"]', + "gen_ai.request.model": "gpt-4o", "gen_ai.system": "openai", "gen_ai.response.model": "response-model-id", "gen_ai.usage.input_tokens": 20, @@ -1605,168 +1539,11 @@ def test_ai_client_span_responses_api( "gen_ai.usage.output_tokens": 10, "gen_ai.usage.output_tokens.reasoning": 8, "gen_ai.usage.total_tokens": 30, - "gen_ai.request.model": "gpt-4o", "gen_ai.response.text": "the model response", "thread.id": mock.ANY, "thread.name": mock.ANY, } - param_id = request.node.callspec.id - if "string" in param_id and ( - instructions is None or isinstance(instructions, Omit) - ): # type: ignore - expected_data.update( - { - "gen_ai.request.messages": safe_serialize( - ["How do I check if a Python object is an instance of a class?"] - ), - } - ) - elif "string" in param_id: - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [ - { - "type": "text", - "content": "You are a coding assistant that talks like a pirate.", - } - ] - ), - "gen_ai.request.messages": safe_serialize( - ["How do I check if a Python object is an instance of a class?"] - ), - } - ) - elif "blocks_no_type" in param_id and ( - instructions is None or isinstance(instructions, Omit) - ): # type: ignore - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [{"type": "text", "content": "You are a helpful assistant."}] - ), - "gen_ai.request.messages": safe_serialize( - [{"role": "user", "content": "hello"}] - ), - } - ) - elif "blocks_no_type" in param_id: - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [ - { - "type": "text", - "content": "You are a coding assistant that talks like a pirate.", - }, - {"type": "text", "content": "You are a helpful assistant."}, - ] - ), - "gen_ai.request.messages": safe_serialize( - [{"role": "user", "content": "hello"}] - ), - } - ) - elif "blocks" in param_id and ( - instructions is None or isinstance(instructions, Omit) - ): # type: ignore - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [{"type": "text", "content": "You are a helpful assistant."}] - ), - "gen_ai.request.messages": safe_serialize( - [{"type": "message", "role": "user", "content": "hello"}] - ), - } - ) - elif "blocks" in param_id: - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [ - { - "type": "text", - "content": "You are a coding assistant that talks like a pirate.", - }, - {"type": "text", "content": "You are a helpful assistant."}, - ] - ), - "gen_ai.request.messages": safe_serialize( - [{"type": "message", "role": "user", "content": "hello"}] - ), - } - ) - elif "parts_no_type" in param_id and ( - instructions is None or isinstance(instructions, Omit) - ): # type: ignore - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [ - {"type": "text", "content": "You are a helpful assistant."}, - {"type": "text", "content": "Be concise and clear."}, - ] - ), - "gen_ai.request.messages": safe_serialize( - [{"role": "user", "content": "hello"}] - ), - } - ) - elif "parts_no_type" in param_id: - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [ - { - "type": "text", - "content": "You are a coding assistant that talks like a pirate.", - }, - {"type": "text", "content": "You are a helpful assistant."}, - {"type": "text", "content": "Be concise and clear."}, - ] - ), - "gen_ai.request.messages": safe_serialize( - [{"role": "user", "content": "hello"}] - ), - } - ) - elif instructions is None or isinstance(instructions, Omit): # type: ignore - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [ - {"type": "text", "content": "You are a helpful assistant."}, - {"type": "text", "content": "Be concise and clear."}, - ] - ), - "gen_ai.request.messages": safe_serialize( - [{"type": "message", "role": "user", "content": "hello"}] - ), - } - ) - else: - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [ - { - "type": "text", - "content": "You are a coding assistant that talks like a pirate.", - }, - {"type": "text", "content": "You are a helpful assistant."}, - {"type": "text", "content": "Be concise and clear."}, - ] - ), - "gen_ai.request.messages": safe_serialize( - [{"type": "message", "role": "user", "content": "hello"}] - ), - } - ) - - assert spans[0]["data"] == expected_data - @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") def test_error_in_responses_api(sentry_init, capture_events): @@ -1807,73 +1584,7 @@ def test_error_in_responses_api(sentry_init, capture_events): @pytest.mark.asyncio @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") -@pytest.mark.parametrize( - "instructions", - ( - omit, - None, - "You are a coding assistant that talks like a pirate.", - ), -) -@pytest.mark.parametrize( - "input", - [ - pytest.param( - "How do I check if a Python object is an instance of a class?", id="string" - ), - pytest.param( - [ - { - "role": "system", - "content": "You are a helpful assistant.", - }, - {"role": "user", "content": "hello"}, - ], - id="blocks_no_type", - ), - pytest.param( - [ - { - "type": "message", - "role": "system", - "content": "You are a helpful assistant.", - }, - {"type": "message", "role": "user", "content": "hello"}, - ], - id="blocks", - ), - pytest.param( - [ - { - "role": "system", - "content": [ - {"type": "text", "text": "You are a helpful assistant."}, - {"type": "text", "text": "Be concise and clear."}, - ], - }, - {"role": "user", "content": "hello"}, - ], - id="parts_no_type", - ), - pytest.param( - [ - { - "type": "message", - "role": "system", - "content": [ - {"type": "text", "text": "You are a helpful assistant."}, - {"type": "text", "text": "Be concise and clear."}, - ], - }, - {"type": "message", "role": "user", "content": "hello"}, - ], - id="parts", - ), - ], -) -async def test_ai_client_span_responses_async_api( - sentry_init, capture_events, instructions, input, request -): +async def test_ai_client_span_responses_async_api(sentry_init, capture_events): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], traces_sample_rate=1.0, @@ -1887,8 +1598,8 @@ async def test_ai_client_span_responses_async_api( with start_transaction(name="openai tx"): await client.responses.create( model="gpt-4o", - instructions=instructions, - input=input, + instructions="You are a coding assistant that talks like a pirate.", + input="How do I check if a Python object is an instance of a class?", ) (transaction,) = events @@ -1897,8 +1608,7 @@ async def test_ai_client_span_responses_async_api( assert len(spans) == 1 assert spans[0]["op"] == "gen_ai.responses" assert spans[0]["origin"] == "auto.ai.openai" - - expected_data = { + assert spans[0]["data"] == { "gen_ai.operation.name": "responses", "gen_ai.request.messages": '["How do I check if a Python object is an instance of a class?"]', "gen_ai.request.model": "gpt-4o", @@ -1914,231 +1624,11 @@ async def test_ai_client_span_responses_async_api( "thread.name": mock.ANY, } - param_id = request.node.callspec.id - if "string" in param_id and ( - instructions is None or isinstance(instructions, Omit) - ): # type: ignore - expected_data.update( - { - "gen_ai.request.messages": safe_serialize( - ["How do I check if a Python object is an instance of a class?"] - ), - } - ) - elif "string" in param_id: - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [ - { - "type": "text", - "content": "You are a coding assistant that talks like a pirate.", - } - ] - ), - "gen_ai.request.messages": safe_serialize( - ["How do I check if a Python object is an instance of a class?"] - ), - } - ) - elif "blocks_no_type" in param_id and ( - instructions is None or isinstance(instructions, Omit) - ): # type: ignore - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [{"type": "text", "content": "You are a helpful assistant."}] - ), - "gen_ai.request.messages": safe_serialize( - [{"role": "user", "content": "hello"}] - ), - } - ) - elif "blocks_no_type" in param_id: - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [ - { - "type": "text", - "content": "You are a coding assistant that talks like a pirate.", - }, - {"type": "text", "content": "You are a helpful assistant."}, - ] - ), - "gen_ai.request.messages": safe_serialize( - [{"role": "user", "content": "hello"}] - ), - } - ) - elif "blocks" in param_id and ( - instructions is None or isinstance(instructions, Omit) - ): # type: ignore - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [{"type": "text", "content": "You are a helpful assistant."}] - ), - "gen_ai.request.messages": safe_serialize( - [{"type": "message", "role": "user", "content": "hello"}] - ), - } - ) - elif "blocks" in param_id: - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [ - { - "type": "text", - "content": "You are a coding assistant that talks like a pirate.", - }, - {"type": "text", "content": "You are a helpful assistant."}, - ] - ), - "gen_ai.request.messages": safe_serialize( - [{"type": "message", "role": "user", "content": "hello"}] - ), - } - ) - elif "parts_no_type" in param_id and ( - instructions is None or isinstance(instructions, Omit) - ): # type: ignore - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [ - {"type": "text", "content": "You are a helpful assistant."}, - {"type": "text", "content": "Be concise and clear."}, - ] - ), - "gen_ai.request.messages": safe_serialize( - [{"role": "user", "content": "hello"}] - ), - } - ) - elif "parts_no_type" in param_id: - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [ - { - "type": "text", - "content": "You are a coding assistant that talks like a pirate.", - }, - {"type": "text", "content": "You are a helpful assistant."}, - {"type": "text", "content": "Be concise and clear."}, - ] - ), - "gen_ai.request.messages": safe_serialize( - [{"role": "user", "content": "hello"}] - ), - } - ) - elif instructions is None or isinstance(instructions, Omit): # type: ignore - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [ - {"type": "text", "content": "You are a helpful assistant."}, - {"type": "text", "content": "Be concise and clear."}, - ] - ), - "gen_ai.request.messages": safe_serialize( - [{"type": "message", "role": "user", "content": "hello"}] - ), - } - ) - else: - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [ - { - "type": "text", - "content": "You are a coding assistant that talks like a pirate.", - }, - {"type": "text", "content": "You are a helpful assistant."}, - {"type": "text", "content": "Be concise and clear."}, - ] - ), - "gen_ai.request.messages": safe_serialize( - [{"type": "message", "role": "user", "content": "hello"}] - ), - } - ) - - assert spans[0]["data"] == expected_data - @pytest.mark.asyncio -@pytest.mark.parametrize( - "instructions", - ( - omit, - None, - "You are a coding assistant that talks like a pirate.", - ), -) -@pytest.mark.parametrize( - "input", - [ - pytest.param( - "How do I check if a Python object is an instance of a class?", id="string" - ), - pytest.param( - [ - { - "role": "system", - "content": "You are a helpful assistant.", - }, - {"role": "user", "content": "hello"}, - ], - id="blocks_no_type", - ), - pytest.param( - [ - { - "type": "message", - "role": "system", - "content": "You are a helpful assistant.", - }, - {"type": "message", "role": "user", "content": "hello"}, - ], - id="blocks", - ), - pytest.param( - [ - { - "role": "system", - "content": [ - {"type": "text", "text": "You are a helpful assistant."}, - {"type": "text", "text": "Be concise and clear."}, - ], - }, - {"role": "user", "content": "hello"}, - ], - id="parts_no_type", - ), - pytest.param( - [ - { - "type": "message", - "role": "system", - "content": [ - {"type": "text", "text": "You are a helpful assistant."}, - {"type": "text", "text": "Be concise and clear."}, - ], - }, - {"type": "message", "role": "user", "content": "hello"}, - ], - id="parts", - ), - ], -) @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") async def test_ai_client_span_streaming_responses_async_api( - sentry_init, capture_events, instructions, input, request + sentry_init, capture_events ): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], @@ -2153,8 +1643,8 @@ async def test_ai_client_span_streaming_responses_async_api( with start_transaction(name="openai tx"): await client.responses.create( model="gpt-4o", - instructions=instructions, - input=input, + instructions="You are a coding assistant that talks like a pirate.", + input="How do I check if a Python object is an instance of a class?", stream=True, ) @@ -2164,179 +1654,23 @@ async def test_ai_client_span_streaming_responses_async_api( assert len(spans) == 1 assert spans[0]["op"] == "gen_ai.responses" assert spans[0]["origin"] == "auto.ai.openai" - - expected_data = { + assert spans[0]["data"] == { "gen_ai.operation.name": "responses", + "gen_ai.request.messages": '["How do I check if a Python object is an instance of a class?"]', + "gen_ai.request.model": "gpt-4o", + "gen_ai.response.model": "response-model-id", "gen_ai.response.streaming": True, "gen_ai.system": "openai", - "gen_ai.response.model": "response-model-id", "gen_ai.usage.input_tokens": 20, "gen_ai.usage.input_tokens.cached": 5, "gen_ai.usage.output_tokens": 10, "gen_ai.usage.output_tokens.reasoning": 8, "gen_ai.usage.total_tokens": 30, - "gen_ai.request.model": "gpt-4o", "gen_ai.response.text": "the model response", "thread.id": mock.ANY, "thread.name": mock.ANY, } - param_id = request.node.callspec.id - if "string" in param_id and ( - instructions is None or isinstance(instructions, Omit) - ): # type: ignore - expected_data.update( - { - "gen_ai.request.messages": safe_serialize( - ["How do I check if a Python object is an instance of a class?"] - ), - } - ) - elif "string" in param_id: - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [ - { - "type": "text", - "content": "You are a coding assistant that talks like a pirate.", - } - ] - ), - "gen_ai.request.messages": safe_serialize( - ["How do I check if a Python object is an instance of a class?"] - ), - } - ) - elif "blocks_no_type" in param_id and ( - instructions is None or isinstance(instructions, Omit) - ): # type: ignore - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [{"type": "text", "content": "You are a helpful assistant."}] - ), - "gen_ai.request.messages": safe_serialize( - [{"role": "user", "content": "hello"}] - ), - } - ) - elif "blocks_no_type" in param_id: - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [ - { - "type": "text", - "content": "You are a coding assistant that talks like a pirate.", - }, - {"type": "text", "content": "You are a helpful assistant."}, - ] - ), - "gen_ai.request.messages": safe_serialize( - [{"role": "user", "content": "hello"}] - ), - } - ) - elif "blocks" in param_id and ( - instructions is None or isinstance(instructions, Omit) - ): # type: ignore - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [{"type": "text", "content": "You are a helpful assistant."}] - ), - "gen_ai.request.messages": safe_serialize( - [{"type": "message", "role": "user", "content": "hello"}] - ), - } - ) - elif "blocks" in param_id: - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [ - { - "type": "text", - "content": "You are a coding assistant that talks like a pirate.", - }, - {"type": "text", "content": "You are a helpful assistant."}, - ] - ), - "gen_ai.request.messages": safe_serialize( - [{"type": "message", "role": "user", "content": "hello"}] - ), - } - ) - elif "parts_no_type" in param_id and ( - instructions is None or isinstance(instructions, Omit) - ): # type: ignore - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [ - {"type": "text", "content": "You are a helpful assistant."}, - {"type": "text", "content": "Be concise and clear."}, - ] - ), - "gen_ai.request.messages": safe_serialize( - [{"role": "user", "content": "hello"}] - ), - } - ) - elif "parts_no_type" in param_id: - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [ - { - "type": "text", - "content": "You are a coding assistant that talks like a pirate.", - }, - {"type": "text", "content": "You are a helpful assistant."}, - {"type": "text", "content": "Be concise and clear."}, - ] - ), - "gen_ai.request.messages": safe_serialize( - [{"role": "user", "content": "hello"}] - ), - } - ) - elif instructions is None or isinstance(instructions, Omit): # type: ignore - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [ - {"type": "text", "content": "You are a helpful assistant."}, - {"type": "text", "content": "Be concise and clear."}, - ] - ), - "gen_ai.request.messages": safe_serialize( - [{"type": "message", "role": "user", "content": "hello"}] - ), - } - ) - else: - expected_data.update( - { - "gen_ai.system_instructions": safe_serialize( - [ - { - "type": "text", - "content": "You are a coding assistant that talks like a pirate.", - }, - {"type": "text", "content": "You are a helpful assistant."}, - {"type": "text", "content": "Be concise and clear."}, - ] - ), - "gen_ai.request.messages": safe_serialize( - [{"type": "message", "role": "user", "content": "hello"}] - ), - } - ) - - assert spans[0]["data"] == expected_data - @pytest.mark.asyncio @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") From 7fc64f0a9e699a4534506d0e99614178b710cfd3 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Mon, 26 Jan 2026 12:57:09 +0100 Subject: [PATCH 22/22] remove unused functions --- sentry_sdk/integrations/openai.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 391495f227..b659059b5d 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -214,24 +214,6 @@ def _get_system_instructions_completions( ] -def _is_system_instruction_responses(message: "ResponseInputItemParam") -> bool: - if not isinstance(message, dict) or not message.get("role") == "system": - return False - - return "type" not in message or message["type"] == "message" - - -def _get_system_instructions_responses( - messages: "Union[str, ResponseInputParam]", -) -> "list[ResponseInputItemParam]": - if not isinstance(messages, list): - return [] - - return [ - message for message in messages if _is_system_instruction_responses(message) - ] - - def _transform_system_instructions( system_instructions: "list[ChatCompletionSystemMessageParam]", ) -> "list[TextPart]":