diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index a62fbf45d8..902db9bdc6 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -147,6 +147,7 @@ def _get_ai_system(all_params: "Dict[str, Any]") -> "Optional[str]": DATA_FIELDS = { "frequency_penalty": SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY, + # "function_call" is an OpenAI convention for the now-legacy Chat Completions API field "function_call": SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, "max_tokens": SPANDATA.GEN_AI_REQUEST_MAX_TOKENS, "presence_penalty": SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY, @@ -417,6 +418,12 @@ def on_llm_start( for key, attribute in DATA_FIELDS.items(): if key in all_params and all_params[key] is not None: + # This is correctly gated on "inputs" at the moment because the + # "on_llm_start" method is the start of a request. + # + # TODO: GEN_AI_RESPONSE_TOOL_CALLS will need to be + # transitioned to non-deprecated tool call attributes + if ( attribute == SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS and has_data_collection_enabled(client.options) @@ -516,6 +523,11 @@ def on_chat_model_start( for key, attribute in DATA_FIELDS.items(): if key in all_params and all_params[key] is not None: if ( + # This is correctly gated on "inputs" at the moment because the + # "on_chat_model_start" method is the start of a request. + # + # TODO: GEN_AI_RESPONSE_TOOL_CALLS will need to be + # transitioned to non-deprecated tool call attributes attribute == SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS and has_data_collection_enabled(client.options) and not client.options["data_collection"]["gen_ai"]["inputs"] @@ -615,14 +627,11 @@ def on_llm_end( client = sentry_sdk.get_client() - record_inputs = False record_outputs = False if has_data_collection_enabled(client.options): - record_inputs = client.options["data_collection"]["gen_ai"]["inputs"] record_outputs = client.options["data_collection"]["gen_ai"]["outputs"] elif should_send_default_pii() and self.include_prompts: # TODO: Remove this branch once `send_default_pii` is deprecated - record_inputs = True record_outputs = True try: @@ -647,7 +656,7 @@ def on_llm_end( if response_model is not None: set_on_span(SPANDATA.GEN_AI_RESPONSE_MODEL, response_model) - if record_inputs: + if record_outputs: tool_calls = getattr(generation.message, "tool_calls", None) if tool_calls is not None and tool_calls != []: set_data_normalized( @@ -1048,17 +1057,19 @@ def _set_tools_on_span(span: "Union[Span, StreamedSpan]", tools: "Any") -> None: return client = sentry_sdk.get_client() + attribute_name = SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS if has_data_collection_enabled(client.options): if not client.options["data_collection"]["gen_ai"]["inputs"]: return - + else: + attribute_name = SPANDATA.GEN_AI_TOOL_DEFINITIONS # Before data collection was introduced this was set unconditionally, so it # stays that way when data collection is not configured. simplified_tools = _simplify_langchain_tools(tools) if simplified_tools: set_data_normalized( span, - SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, + attribute_name, simplified_tools, unpack=False, ) diff --git a/tests/integrations/langchain/test_langchain.py b/tests/integrations/langchain/test_langchain.py index 353df628f1..92968e1915 100644 --- a/tests/integrations/langchain/test_langchain.py +++ b/tests/integrations/langchain/test_langchain.py @@ -6497,10 +6497,10 @@ def test_langchain_text_completion_data_collection( False, False, [ - SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, + SPANDATA.GEN_AI_TOOL_DEFINITIONS, SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, ], - [], + [SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS], id="gen-ai-inputs-and-outputs-enabled-tools-collected", ), pytest.param( @@ -6509,6 +6509,7 @@ def test_langchain_text_completion_data_collection( True, [], [ + SPANDATA.GEN_AI_TOOL_DEFINITIONS, SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, ], @@ -6518,33 +6519,34 @@ def test_langchain_text_completion_data_collection( {"gen_ai": {"inputs": True, "outputs": False}}, False, False, + [SPANDATA.GEN_AI_TOOL_DEFINITIONS], [ + # REQUEST_AVAILABLE_TOOLS is the legacy value set when data collection is not enabled SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, ], - [], - id="gen-ai-inputs-enabled-outputs-disabled-tools-collected", + id="gen-ai-inputs-enabled-outputs-disabled-only-tool-definitions-collected", ), pytest.param( {"gen_ai": {"inputs": False, "outputs": True}}, True, True, - [], + [SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS], [ + SPANDATA.GEN_AI_TOOL_DEFINITIONS, SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, - SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, ], - id="gen-ai-outputs-enabled-inputs-disabled-tools-not-collected", + id="gen-ai-outputs-enabled-inputs-disabled-only-response-tool-calls-collected", ), pytest.param( {"gen_ai": {}}, False, False, [ - SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, + SPANDATA.GEN_AI_TOOL_DEFINITIONS, SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, ], - [], + [SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS], id="gen-ai-inputs-and-outputs-omitted-default-to-enabled", ), pytest.param( @@ -6555,7 +6557,7 @@ def test_langchain_text_completion_data_collection( SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, ], - [], + [SPANDATA.GEN_AI_TOOL_DEFINITIONS], id="no-gen-ai-config-legacy-pii-and-include-prompts-enabled", ), pytest.param( @@ -6563,7 +6565,10 @@ def test_langchain_text_completion_data_collection( False, False, [SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS], - [SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS], + [ + SPANDATA.GEN_AI_TOOL_DEFINITIONS, + SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, + ], id="no-gen-ai-config-available-tools-collected-regardless-of-pii", ), ], @@ -6664,16 +6669,14 @@ def test_langchain_data_collection_tools( for key in expected_absent: assert key not in chat_spans[0], f"{key} should not have been collected" - # Available tools are gated the same way on every span they are set on - available_tools_collected = ( - SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS in expected_present - ) - assert ( - SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS in chat_spans[1] - ) is available_tools_collected - assert ( - SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS in invoke_agent_span - ) is available_tools_collected + # Tool definitions are gated the same way on every span they are set on + for key in ( + SPANDATA.GEN_AI_TOOL_DEFINITIONS, + SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, + ): + collected = key in expected_present + assert (key in chat_spans[1]) is collected + assert (key in invoke_agent_span) is collected @pytest.mark.parametrize("span_streaming", [True, False])