From 085b4966f980b9ef731007d3ed7c55fa42d768e1 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Fri, 23 Jan 2026 15:14:09 +0100 Subject: [PATCH 1/6] feat(openai-agents): Set system instruction attribute on gen_ai.chat spans --- sentry_sdk/ai/_opanai_completions_api.py | 51 +++++ sentry_sdk/ai/_openai_responses_api.py | 29 +++ sentry_sdk/integrations/openai.py | 74 +------- .../integrations/openai_agents/utils.py | 44 ++++- .../openai_agents/test_openai_agents.py | 174 +++++++++++++++--- 5 files changed, 274 insertions(+), 98 deletions(-) create mode 100644 sentry_sdk/ai/_opanai_completions_api.py create mode 100644 sentry_sdk/ai/_openai_responses_api.py diff --git a/sentry_sdk/ai/_opanai_completions_api.py b/sentry_sdk/ai/_opanai_completions_api.py new file mode 100644 index 0000000000..7bf16a53ee --- /dev/null +++ b/sentry_sdk/ai/_opanai_completions_api.py @@ -0,0 +1,51 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Iterable + + from sentry_sdk._types import TextPart + + from openai.types.chat import ( + ChatCompletionMessageParam, + ChatCompletionSystemMessageParam, + ) + + +def _is_system_instruction(message: "ChatCompletionMessageParam") -> bool: + return isinstance(message, dict) and message.get("role") == "system" + + +def _get_system_instructions( + messages: "Iterable[ChatCompletionMessageParam]", +) -> "list[ChatCompletionSystemMessageParam]": + system_instructions = [] + + for message in messages: + if _is_system_instruction(message): + system_instructions.append(message) + + return system_instructions + + +def _transform_system_instructions( + system_instructions: "list[ChatCompletionSystemMessageParam]", +) -> "list[TextPart]": + instruction_text_parts: "list[TextPart]" = [] + + for instruction in system_instructions: + if not isinstance(instruction, dict): + continue + + content = instruction.get("content") + + if isinstance(content, str): + instruction_text_parts.append({"type": "text", "content": content}) + + elif isinstance(content, list): + for part in content: + if isinstance(part, dict) and part.get("type") == "text": + text = part.get("text", "") + if text: + instruction_text_parts.append({"type": "text", "content": text}) + + return instruction_text_parts diff --git a/sentry_sdk/ai/_openai_responses_api.py b/sentry_sdk/ai/_openai_responses_api.py new file mode 100644 index 0000000000..d766ac9869 --- /dev/null +++ b/sentry_sdk/ai/_openai_responses_api.py @@ -0,0 +1,29 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Union + + from openai.types.responses import ResponseInputParam, ResponseInputItemParam + + +def _is_system_instruction(message: "ResponseInputItemParam") -> bool: + return ( + isinstance(message, dict) + and message.get("type") == "message" + and message.get("role") == "system" + ) + + +def _get_system_instructions( + messages: "Union[str, ResponseInputParam]", +) -> "list[ResponseInputItemParam]": + if isinstance(messages, str): + return [] + + system_instructions = [] + + for message in messages: + if _is_system_instruction(message): + system_instructions.append(message) + + return system_instructions diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 7a5d449e23..215655ee39 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -9,6 +9,15 @@ normalize_message_roles, truncate_and_annotate_messages, ) +from sentry_sdk.ai._opanai_completions_api import ( + _is_system_instruction as _is_system_instruction_completions, + _get_system_instructions as _get_system_instructions_completions, + _transform_system_instructions, +) +from sentry_sdk.ai._openai_responses_api import ( + _is_system_instruction as _is_system_instruction_responses, + _get_system_instructions as _get_system_instructions_responses, +) from sentry_sdk.consts import SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration from sentry_sdk.scope import should_send_default_pii @@ -36,7 +45,7 @@ from sentry_sdk.tracing import Span from sentry_sdk._types import TextPart - from openai.types.responses import ResponseInputParam, ResponseInputItemParam + from openai.types.responses import ResponseInputParam from openai import Omit try: @@ -199,69 +208,6 @@ def _calculate_token_usage( ) -def _is_system_instruction_completions(message: "ChatCompletionMessageParam") -> bool: - return isinstance(message, dict) and message.get("role") == "system" - - -def _get_system_instructions_completions( - messages: "Iterable[ChatCompletionMessageParam]", -) -> "list[ChatCompletionSystemMessageParam]": - system_instructions = [] - - for message in messages: - if _is_system_instruction_completions(message): - system_instructions.append(message) - - return system_instructions - - -def _is_system_instruction_responses(message: "ResponseInputItemParam") -> bool: - return ( - isinstance(message, dict) - and message.get("type") == "message" - and message.get("role") == "system" - ) - - -def _get_system_instructions_responses( - messages: "Union[str, ResponseInputParam]", -) -> "list[ResponseInputItemParam]": - if isinstance(messages, str): - return [] - - system_instructions = [] - - for message in messages: - if _is_system_instruction_responses(message): - system_instructions.append(message) - - return system_instructions - - -def _transform_system_instructions( - system_instructions: "list[ChatCompletionSystemMessageParam]", -) -> "list[TextPart]": - instruction_text_parts: "list[TextPart]" = [] - - for instruction in system_instructions: - if not isinstance(instruction, dict): - continue - - content = instruction.get("content") - - if isinstance(content, str): - instruction_text_parts.append({"type": "text", "content": content}) - - elif isinstance(content, list): - for part in content: - if isinstance(part, dict) and part.get("type") == "text": - text = part.get("text", "") - if text: - instruction_text_parts.append({"type": "text", "content": text}) - - return instruction_text_parts - - def _get_input_messages( kwargs: "dict[str, Any]", ) -> "Optional[Union[Iterable[Any], list[str]]]": diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index a24d0e909d..e9494fd5da 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -11,14 +11,20 @@ from sentry_sdk.scope import should_send_default_pii from sentry_sdk.tracing_utils import set_span_errored from sentry_sdk.utils import event_from_exception, safe_serialize +from sentry_sdk.ai._opanai_completions_api import _transform_system_instructions +from sentry_sdk.ai._openai_responses_api import ( + _is_system_instruction, + _get_system_instructions, +) from typing import TYPE_CHECKING if TYPE_CHECKING: from typing import Any - from agents import Usage + from agents import Usage, TResponseInputItem from sentry_sdk.tracing import Span + from sentry_sdk._types import TextPart try: import agents @@ -115,16 +121,36 @@ def _set_input_data( return request_messages = [] - system_instructions = get_response_kwargs.get("system_instructions") - if system_instructions: - request_messages.append( - { - "role": GEN_AI_ALLOWED_MESSAGE_ROLES.SYSTEM, - "content": [{"type": "text", "text": system_instructions}], - } + messages: "str | list[TResponseInputItem]" = get_response_kwargs.get("input", []) + + explicit_instructions = get_response_kwargs.get("system_instructions") + system_instructions = _get_system_instructions(messages) + + if system_instructions is not None or len(system_instructions) > 0: + instructions_text_parts: "list[TextPart]" = [] + if explicit_instructions is not None: + instructions_text_parts.append( + { + "type": "text", + "content": explicit_instructions, + } + ) + + # Deliberate use of function accepting completions API type because + # of shared structure FOR THIS PURPOSE ONLY. + instructions_text_parts += _transform_system_instructions(system_instructions) + + set_data_normalized( + span, + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, + instructions_text_parts, + unpack=False, ) - for message in get_response_kwargs.get("input", []): + non_system_messages = [ + message for message in messages if not _is_system_instruction(message) + ] + for message in non_system_messages: if "role" in message: normalized_role = normalize_message_role(message.get("role")) content = message.get("content") diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index a3ae50d5f1..f09a52bbea 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -498,7 +498,43 @@ async def test_max_turns_before_handoff_span(sentry_init, capture_events, mock_u @pytest.mark.asyncio -async def test_tool_execution_span(sentry_init, capture_events, test_agent): +@pytest.mark.parametrize( + "input", + [ + pytest.param( + "Please use the simple test tool", + id="string", + ), + pytest.param( + [ + { + "type": "message", + "role": "system", + "content": "You are very powerful assistant, but don't know current events", + }, + {"role": "user", "content": "Please use the simple test tool"}, + ], + id="blocks", + ), + pytest.param( + [ + { + "type": "message", + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful assistant."}, + {"type": "text", "text": "Be concise and clear."}, + ], + }, + {"role": "user", "content": "Please use the simple test tool"}, + ], + id="parts", + ), + ], +) +async def test_tool_execution_span( + sentry_init, capture_events, test_agent, input, request +): """ Test tool execution span creation. """ @@ -569,7 +605,7 @@ def simple_test_tool(message: str) -> str: await agents.Runner.run( agent_with_tool, - "Please use the simple test tool", + input, run_config=test_run_config, ) @@ -625,20 +661,39 @@ def simple_test_tool(message: str) -> str: assert ai_client_span1["data"]["gen_ai.agent.name"] == "test_agent" assert ai_client_span1["data"]["gen_ai.request.available_tools"] == available_tools assert ai_client_span1["data"]["gen_ai.request.max_tokens"] == 100 + + param_id = request.node.callspec.id + if "string" in param_id: + assert ai_client_span1["data"]["gen_ai.system_instructions"] == safe_serialize( + [{"type": "text", "content": "You are a helpful test assistant."}] + ) + elif "blocks" in param_id: + assert ai_client_span1["data"]["gen_ai.system_instructions"] == safe_serialize( + [ + {"type": "text", "content": "You are a helpful test assistant."}, + { + "type": "text", + "content": "You are very powerful assistant, but don't know current events", + }, + ] + ) + else: + assert ai_client_span1["data"]["gen_ai.system_instructions"] == safe_serialize( + [ + {"type": "text", "content": "You are a helpful test assistant."}, + {"type": "text", "content": "You are a helpful assistant."}, + {"type": "text", "content": "Be concise and clear."}, + ] + ) + assert ai_client_span1["data"]["gen_ai.request.messages"] == safe_serialize( [ - { - "role": "system", - "content": [ - {"type": "text", "text": "You are a helpful test assistant."} - ], - }, { "role": "user", "content": [ {"type": "text", "text": "Please use the simple test tool"} ], - }, + } ] ) assert ai_client_span1["data"]["gen_ai.request.model"] == "gpt-4" @@ -696,14 +751,31 @@ def simple_test_tool(message: str) -> str: == available_tools ) assert ai_client_span2["data"]["gen_ai.request.max_tokens"] == 100 + if "string" in param_id: + assert ai_client_span2["data"]["gen_ai.system_instructions"] == safe_serialize( + [{"type": "text", "content": "You are a helpful test assistant."}] + ) + elif "blocks" in param_id: + assert ai_client_span1["data"]["gen_ai.system_instructions"] == safe_serialize( + [ + {"type": "text", "content": "You are a helpful test assistant."}, + { + "type": "text", + "content": "You are very powerful assistant, but don't know current events", + }, + ] + ) + else: + assert ai_client_span1["data"]["gen_ai.system_instructions"] == safe_serialize( + [ + {"type": "text", "content": "You are a helpful test assistant."}, + {"type": "text", "content": "You are a helpful assistant."}, + {"type": "text", "content": "Be concise and clear."}, + ] + ) + assert ai_client_span2["data"]["gen_ai.request.messages"] == safe_serialize( [ - { - "role": "system", - "content": [ - {"type": "text", "text": "You are a helpful test assistant."} - ], - }, { "role": "user", "content": [ @@ -950,7 +1022,43 @@ async def test_error_handling(sentry_init, capture_events, test_agent): @pytest.mark.asyncio -async def test_error_captures_input_data(sentry_init, capture_events, test_agent): +@pytest.mark.parametrize( + "input", + [ + pytest.param( + "Test input", + id="string", + ), + pytest.param( + [ + { + "type": "message", + "role": "system", + "content": "You are very powerful assistant, but don't know current events", + }, + {"role": "user", "content": "Test input"}, + ], + id="blocks", + ), + pytest.param( + [ + { + "type": "message", + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful assistant."}, + {"type": "text", "text": "Be concise and clear."}, + ], + }, + {"role": "user", "content": "Test input"}, + ], + id="parts", + ), + ], +) +async def test_error_captures_input_data( + sentry_init, capture_events, test_agent, input, request +): """ Test that input data is captured even when the API call raises an exception. This verifies that _set_input_data is called before the API call. @@ -970,9 +1078,7 @@ async def test_error_captures_input_data(sentry_init, capture_events, test_agent events = capture_events() with pytest.raises(Exception, match="API Error"): - await agents.Runner.run( - test_agent, "Test input", run_config=test_run_config - ) + await agents.Runner.run(test_agent, input, run_config=test_run_config) ( error_event, @@ -989,15 +1095,33 @@ async def test_error_captures_input_data(sentry_init, capture_events, test_agent assert ai_client_span["status"] == "internal_error" assert ai_client_span["tags"]["status"] == "internal_error" + param_id = request.node.callspec.id + if "string" in param_id: + assert ai_client_span["data"]["gen_ai.system_instructions"] == safe_serialize( + [{"type": "text", "content": "You are a helpful test assistant."}] + ) + elif "blocks" in param_id: + assert ai_client_span["data"]["gen_ai.system_instructions"] == safe_serialize( + [ + {"type": "text", "content": "You are a helpful test assistant."}, + { + "type": "text", + "content": "You are very powerful assistant, but don't know current events", + }, + ] + ) + else: + assert ai_client_span["data"]["gen_ai.system_instructions"] == safe_serialize( + [ + {"type": "text", "content": "You are a helpful test assistant."}, + {"type": "text", "content": "You are a helpful assistant."}, + {"type": "text", "content": "Be concise and clear."}, + ] + ) + assert "gen_ai.request.messages" in ai_client_span["data"] request_messages = safe_serialize( [ - { - "role": "system", - "content": [ - {"type": "text", "text": "You are a helpful test assistant."} - ], - }, {"role": "user", "content": [{"type": "text", "text": "Test input"}]}, ] ) From 48c7fbeee331266c5a97faf26dc7e5b0f445aa93 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Fri, 23 Jan 2026 15:18:18 +0100 Subject: [PATCH 2/6] add type ignores --- sentry_sdk/integrations/openai_agents/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index e9494fd5da..6048a7d1d9 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -152,8 +152,8 @@ def _set_input_data( ] for message in non_system_messages: if "role" in message: - normalized_role = normalize_message_role(message.get("role")) - content = message.get("content") + normalized_role = normalize_message_role(message.get("role")) # type: ignore + content = message.get("content") # type: ignore request_messages.append( { "role": normalized_role, @@ -165,14 +165,14 @@ def _set_input_data( } ) else: - if message.get("type") == "function_call": + if message.get("type") == "function_call": # type: ignore request_messages.append( { "role": GEN_AI_ALLOWED_MESSAGE_ROLES.ASSISTANT, "content": [message], } ) - elif message.get("type") == "function_call_output": + elif message.get("type") == "function_call_output": # type: ignore request_messages.append( { "role": GEN_AI_ALLOWED_MESSAGE_ROLES.TOOL, From fc9f1faebf2c6b00eaf3b035c49c23f14abff81f Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Fri, 23 Jan 2026 15:39:18 +0100 Subject: [PATCH 3/6] pick up changes to extraction functions --- sentry_sdk/ai/_opanai_completions_api.py | 13 +++++-------- sentry_sdk/ai/_openai_responses_api.py | 12 +++--------- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/sentry_sdk/ai/_opanai_completions_api.py b/sentry_sdk/ai/_opanai_completions_api.py index 7bf16a53ee..6e8c33378b 100644 --- a/sentry_sdk/ai/_opanai_completions_api.py +++ b/sentry_sdk/ai/_opanai_completions_api.py @@ -15,16 +15,13 @@ def _is_system_instruction(message: "ChatCompletionMessageParam") -> bool: return isinstance(message, dict) and message.get("role") == "system" -def _get_system_instructions( +def _get_system_instructions_completions( messages: "Iterable[ChatCompletionMessageParam]", -) -> "list[ChatCompletionSystemMessageParam]": - system_instructions = [] +) -> "list[ChatCompletionMessageParam]": + if not isinstance(messages, Iterable): + return [] - for message in messages: - if _is_system_instruction(message): - system_instructions.append(message) - - return system_instructions + return [message for message in messages if _is_system_instruction(message)] def _transform_system_instructions( diff --git a/sentry_sdk/ai/_openai_responses_api.py b/sentry_sdk/ai/_openai_responses_api.py index d766ac9869..2fb35cda77 100644 --- a/sentry_sdk/ai/_openai_responses_api.py +++ b/sentry_sdk/ai/_openai_responses_api.py @@ -14,16 +14,10 @@ def _is_system_instruction(message: "ResponseInputItemParam") -> bool: ) -def _get_system_instructions( +def _get_system_instructions_responses( messages: "Union[str, ResponseInputParam]", ) -> "list[ResponseInputItemParam]": - if isinstance(messages, str): + if not isinstance(messages, list): return [] - system_instructions = [] - - for message in messages: - if _is_system_instruction(message): - system_instructions.append(message) - - return system_instructions + return [message for message in messages if _is_system_instruction(message)] From bcdd87c86cd432157bf4bbda0fafedf2ac56af31 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Fri, 23 Jan 2026 15:41:48 +0100 Subject: [PATCH 4/6] fix func name --- sentry_sdk/ai/_opanai_completions_api.py | 2 +- sentry_sdk/ai/_openai_responses_api.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/ai/_opanai_completions_api.py b/sentry_sdk/ai/_opanai_completions_api.py index 6e8c33378b..fc60f8bf00 100644 --- a/sentry_sdk/ai/_opanai_completions_api.py +++ b/sentry_sdk/ai/_opanai_completions_api.py @@ -15,7 +15,7 @@ def _is_system_instruction(message: "ChatCompletionMessageParam") -> bool: return isinstance(message, dict) and message.get("role") == "system" -def _get_system_instructions_completions( +def _get_system_instructions( messages: "Iterable[ChatCompletionMessageParam]", ) -> "list[ChatCompletionMessageParam]": if not isinstance(messages, Iterable): diff --git a/sentry_sdk/ai/_openai_responses_api.py b/sentry_sdk/ai/_openai_responses_api.py index 2fb35cda77..b0cd8f768f 100644 --- a/sentry_sdk/ai/_openai_responses_api.py +++ b/sentry_sdk/ai/_openai_responses_api.py @@ -14,7 +14,7 @@ def _is_system_instruction(message: "ResponseInputItemParam") -> bool: ) -def _get_system_instructions_responses( +def _get_system_instructions( messages: "Union[str, ResponseInputParam]", ) -> "list[ResponseInputItemParam]": if not isinstance(messages, list): From 68b853fff94d9f2f4f3801cebf74312a1af51299 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Fri, 23 Jan 2026 16:00:38 +0100 Subject: [PATCH 5/6] fix Iterable import --- sentry_sdk/ai/_opanai_completions_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/ai/_opanai_completions_api.py b/sentry_sdk/ai/_opanai_completions_api.py index fc60f8bf00..a0f6e16a40 100644 --- a/sentry_sdk/ai/_opanai_completions_api.py +++ b/sentry_sdk/ai/_opanai_completions_api.py @@ -1,8 +1,8 @@ +from collections.abc import Iterable + from typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import Iterable - from sentry_sdk._types import TextPart from openai.types.chat import ( From 5ee52745a154def9a3a07ea51d68be45cdb60074 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Fri, 23 Jan 2026 16:07:18 +0100 Subject: [PATCH 6/6] remove runtime import --- sentry_sdk/integrations/openai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index dd2a6078bd..215655ee39 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -1,6 +1,5 @@ import sys from functools import wraps -from collections.abc import Iterable import sentry_sdk from sentry_sdk import consts @@ -41,6 +40,7 @@ AsyncIterator, Iterator, Union, + Iterable, ) from sentry_sdk.tracing import Span from sentry_sdk._types import TextPart