diff --git a/py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_parse_async.yaml b/py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_parse_async.yaml new file mode 100644 index 00000000..d71176bf --- /dev/null +++ b/py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_parse_async.yaml @@ -0,0 +1,107 @@ +interactions: +- request: + body: '{"max_tokens":128,"messages":[{"role":"user","content":"Return a JSON object + with answer=3 and label=\"async\"."}],"model":"claude-haiku-4-5","output_config":{"format":{"schema":{"type":"object","title":"ParsedAnswer","properties":{"answer":{"type":"integer","title":"Answer"},"label":{"type":"string","title":"Label"}},"additionalProperties":false,"required":["answer","label"]},"type":"json_schema"}}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '403' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 0.116.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-helper: + - messages.parse + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.116.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.3 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011Ccq6VRSBSPyrNEUcTfCuN","type":"message","role":"assistant","content":[{"type":"text","text":"{\"answer\":3,\"label\":\"async\"}"}],"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":210,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":12,"service_tier":"standard","inference_geo":"not_available"}}' + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-07-08T18:52:34Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-07-08T18:52:34Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-07-08T18:52:33Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-07-08T18:52:34Z' + cf-cache-status: + - DYNAMIC + cf-ray: + - a1814961defec48a-YYZ + connection: + - keep-alive + content-length: + - '492' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - application/json + date: + - Wed, 08 Jul 2026 18:52:34 GMT + request-id: + - req_011Ccq6VQ6ZdYURuNHeoyLen + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-d19eca3922a082a78c44c20b9b66b6d8-daed6abfc1d21be5-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_parse_sync.yaml b/py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_parse_sync.yaml new file mode 100644 index 00000000..3d248608 --- /dev/null +++ b/py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_parse_sync.yaml @@ -0,0 +1,110 @@ +interactions: +- request: + body: '{"max_tokens":128,"messages":[{"role":"user","content":"Return a JSON object + with answer=2 and label=\"ok\"."}],"model":"claude-haiku-4-5","output_config":{"format":{"schema":{"type":"object","title":"ParsedAnswer","properties":{"answer":{"type":"integer","title":"Answer"},"label":{"type":"string","title":"Label"}},"additionalProperties":false,"required":["answer","label"]},"type":"json_schema"}},"system":"Return + only the requested structured JSON object.","tool_choice":{"type":"none"},"tools":[{"name":"calculator","description":"Evaluate + simple arithmetic expressions.","input_schema":{"type":"object","properties":{"expression":{"type":"string"}},"required":["expression"]}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '684' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python 0.116.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-helper: + - messages.parse + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.116.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.3 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011Ccq6RSmCH5q9JEWYpLfqS","type":"message","role":"assistant","content":[{"type":"text","text":"{\"answer\": + 2, \"label\": \"ok\"}"}],"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":767,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":15,"service_tier":"standard","inference_geo":"not_available"}}' + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-07-08T18:51:39Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-07-08T18:51:39Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-07-08T18:51:37Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-07-08T18:51:39Z' + cf-cache-status: + - DYNAMIC + cf-ray: + - a1814807f983dde5-YYZ + connection: + - keep-alive + content-length: + - '492' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - application/json + date: + - Wed, 08 Jul 2026 18:51:40 GMT + request-id: + - req_011Ccq6RKXNVH5sQmXLTcj1o + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-81d6835151765e3c6b4845a036ff2a71-fff1ab70ce453822-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/anthropic/test_anthropic.py b/py/src/braintrust/integrations/anthropic/test_anthropic.py index 7865e85e..a2b7fca9 100644 --- a/py/src/braintrust/integrations/anthropic/test_anthropic.py +++ b/py/src/braintrust/integrations/anthropic/test_anthropic.py @@ -13,7 +13,7 @@ import pytest from braintrust import Attachment, logger from braintrust.integrations.anthropic import AnthropicIntegration, wrap_anthropic -from braintrust.integrations.anthropic._utils import extract_anthropic_usage +from braintrust.integrations.anthropic._utils import _try_to_dict, extract_anthropic_usage from braintrust.integrations.anthropic.tracing import ( _get_input_from_kwargs, _get_metadata_from_kwargs, @@ -22,6 +22,7 @@ from braintrust.integrations.test_utils import verify_autoinstrument_script from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import find_span_by_name, find_spans_by_type, init_test_logger +from pydantic import BaseModel PROJECT_NAME = "test-anthropic-app" @@ -39,6 +40,17 @@ "required": ["answer", "label"], "additionalProperties": False, } +STRUCTURED_OUTPUT_TOOLS = [ + { + "name": "calculator", + "description": "Evaluate simple arithmetic expressions.", + "input_schema": { + "type": "object", + "properties": {"expression": {"type": "string"}}, + "required": ["expression"], + }, + } +] PNG_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==" PDF_BASE64 = "JVBERi0xLjAKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PmVuZG9iagoyIDAgb2JqCjw8L1R5cGUvUGFnZXMvS2lkc1szIDAgUl0vQ291bnQgMT4+ZW5kb2JqCjMgMCBvYmoKPDwvVHlwZS9QYWdlL01lZGlhQm94WzAgMCA2MTIgNzkyXT4+ZW5kb2JqCnhyZWYKMCA0CjAwMDAwMDAwMDAgNjU1MzUgZg0KMDAwMDAwMDAxMCAwMDAwMCBuDQowMDAwMDAwMDUzIDAwMDAwIG4NCjAwMDAwMDAxMDIgMDAwMDAgbg0KdHJhaWxlcgo8PC9TaXplIDQvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgoxNDkKJUVPRg==" PROMPT_CACHE_TEST_TEXT = "\n".join(f"Cached geography fact {i}: Paris is the capital of France." for i in range(300)) @@ -52,6 +64,11 @@ def _get_async_client(): return anthropic.AsyncAnthropic() +class ParsedAnswer(BaseModel): + answer: int + label: str + + def _get_supported_structured_output_param(): parameter_names = inspect.signature(_get_client().messages.create).parameters if "output_config" in parameter_names: @@ -75,6 +92,28 @@ def _get_supported_structured_output_param(): pytest.skip("Installed anthropic SDK does not support structured outputs parameters") +def _skip_if_messages_parse_unsupported(): + if not hasattr(_get_client().messages, "parse"): + pytest.skip("Installed anthropic SDK does not support messages.parse") + + +def test_wrap_anthropic_preserves_messages_parse_feature_detection(): + class MessagesWithoutParse: + pass + + class FakeAnthropic: + messages = MessagesWithoutParse() + + class FakeAsyncAnthropic: + messages = MessagesWithoutParse() + + for client in (FakeAnthropic(), FakeAsyncAnthropic()): + messages = wrap_anthropic(client).messages + assert not hasattr(messages, "parse") + with pytest.raises(AttributeError): + getattr(messages, "parse") + + def _skip_if_server_tool_content_blocks_unsupported(): required_type_names = ("ServerToolUseBlock", "WebSearchToolResultBlock") if not all(hasattr(anthropic.types, type_name) for type_name in required_type_names): @@ -558,6 +597,93 @@ def test_anthropic_messages_create_tracks_structured_outputs_metadata(memory_log assert span["metadata"][structured_output_param_name] == structured_output_param_value +@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "body"]) +def test_anthropic_messages_parse_sync(memory_logger): + _skip_if_messages_parse_unsupported() + assert not memory_logger.pop() + + client = wrap_anthropic(_get_client()) + params = { + "model": STRUCTURED_OUTPUT_MODEL, + "max_tokens": 128, + "system": "Return only the requested structured JSON object.", + "messages": [ + { + "role": "user", + "content": 'Return a JSON object with answer=2 and label="ok".', + } + ], + "tools": STRUCTURED_OUTPUT_TOOLS, + "tool_choice": {"type": "none"}, + "output_format": ParsedAnswer, + } + + start = time.time() + response = client.messages.parse(**params) + end = time.time() + + assert _try_to_dict(response.parsed_output) == {"answer": 2, "label": "ok"} + + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + assert span["span_attributes"]["name"] == "anthropic.messages.parse" + assert span["span_attributes"]["type"] == "llm" + assert span["metadata"]["provider"] == "anthropic" + assert span["metadata"]["model"] == STRUCTURED_OUTPUT_MODEL + assert span["metadata"]["output_format"].endswith("ParsedAnswer") + assert span["metadata"]["tools"] == STRUCTURED_OUTPUT_TOOLS + assert span["input"] == [ + *params["messages"], + {"role": "system", "content": params["system"]}, + ] + assert span["output"]["parsed_output"] == {"answer": 2, "label": "ok"} + assert span["output"]["model"] == response.model + _assert_metrics_are_valid(span["metrics"], start, end) + assert span["metrics"]["prompt_tokens"] == response.usage.input_tokens + assert span["metrics"]["completion_tokens"] == response.usage.output_tokens + assert span["metrics"]["tokens"] == response.usage.input_tokens + response.usage.output_tokens + + +@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "body"]) +@pytest.mark.asyncio +async def test_anthropic_messages_parse_async(memory_logger): + _skip_if_messages_parse_unsupported() + assert not memory_logger.pop() + + client = wrap_anthropic(_get_async_client()) + params = { + "model": STRUCTURED_OUTPUT_MODEL, + "max_tokens": 128, + "messages": [ + { + "role": "user", + "content": 'Return a JSON object with answer=3 and label="async".', + } + ], + "output_format": ParsedAnswer, + } + + start = time.time() + response = await client.messages.parse(**params) + end = time.time() + + assert _try_to_dict(response.parsed_output) == {"answer": 3, "label": "async"} + + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + assert span["span_attributes"]["name"] == "anthropic.messages.parse" + assert span["metadata"]["provider"] == "anthropic" + assert span["metadata"]["model"] == STRUCTURED_OUTPUT_MODEL + assert span["metadata"]["output_format"].endswith("ParsedAnswer") + assert span["input"] == params["messages"] + assert span["output"]["parsed_output"] == {"answer": 3, "label": "async"} + _assert_metrics_are_valid(span["metrics"], start, end) + assert span["metrics"]["prompt_tokens"] == response.usage.input_tokens + assert span["metrics"]["completion_tokens"] == response.usage.output_tokens + + @pytest.mark.vcr def test_anthropic_messages_model_params_inputs(memory_logger): assert not memory_logger.pop() diff --git a/py/src/braintrust/integrations/anthropic/tracing.py b/py/src/braintrust/integrations/anthropic/tracing.py index a95eaa17..eacacb6e 100644 --- a/py/src/braintrust/integrations/anthropic/tracing.py +++ b/py/src/braintrust/integrations/anthropic/tracing.py @@ -65,6 +65,16 @@ def __init__(self, messages): super().__init__(messages) self.__messages = messages + def __getattr__(self, name: str) -> Any: + if name == "parse": + parse = getattr(self.__messages, "parse") + + async def traced_parse(*args, **kwargs): + return await _trace_async_parsed_message_call(parse, args, kwargs) + + return traced_parse + return super().__getattr__(name) + @property def batches(self): return AsyncBatches(self.__messages.batches) @@ -161,6 +171,16 @@ def __init__(self, messages): super().__init__(messages) self.__messages = messages + def __getattr__(self, name: str) -> Any: + if name == "parse": + parse = getattr(self.__messages, "parse") + + def traced_parse(*args, **kwargs): + return _trace_parsed_message_call(parse, args, kwargs) + + return traced_parse + return super().__getattr__(name) + @property def batches(self): return Batches(self.__messages.batches) @@ -1253,15 +1273,22 @@ def _get_input_from_kwargs(kwargs): system = bt_safe_deep_copy(kwargs.get("system", None)) if system: msgs.append({"role": "system", "content": _process_input_attachments(system)}) + return msgs +def _metadata_value_from_kwargs(key: str, value: Any) -> Any: + if key == "output_format" and isinstance(value, type): + return f"{value.__module__}.{value.__qualname__}" + return value + + def _get_metadata_from_kwargs(kwargs): metadata = {"provider": "anthropic"} for k in METADATA_PARAMS: v = kwargs.get(k, None) if v is not None: - metadata[k] = v + metadata[k] = _metadata_value_from_kwargs(k, v) return metadata @@ -1444,13 +1471,7 @@ def _log_server_tool_spans(content: Any, parent_span) -> None: _log_server_tool_span(parent_span, None, result_item) -def _log_message_to_span(message, span, time_to_first_token: float | None = None): - usage = getattr(message, "usage", {}) - metrics, metadata = extract_anthropic_usage(usage) - - if time_to_first_token is not None: - metrics["time_to_first_token"] = time_to_first_token - +def _message_output(message, *, include_parsed_output: bool = False): content = getattr(message, "content", None) output = { k: v @@ -1462,12 +1483,60 @@ def _log_message_to_span(message, span, time_to_first_token: float | None = None "stop_sequence": getattr(message, "stop_sequence", None), }.items() if v is not None - } or None + } + + if include_parsed_output: + parsed_output = getattr(message, "parsed_output", None) + if parsed_output is not None: + output["parsed_output"] = _normalize_anthropic_data(parsed_output) + + return output or None + + +def _log_message_to_span(message, span, time_to_first_token: float | None = None, *, include_parsed_output=False): + usage = getattr(message, "usage", {}) + metrics, metadata = extract_anthropic_usage(usage) + + if time_to_first_token is not None: + metrics["time_to_first_token"] = time_to_first_token + + content = getattr(message, "content", None) + output = _message_output(message, include_parsed_output=include_parsed_output) span.log(output=output, metrics=metrics, metadata=metadata) _log_server_tool_spans(content, span) +def _trace_parsed_message_call(method, args, kwargs): + span = _start_span("anthropic.messages.parse", kwargs) + request_start_time = time.time() + try: + msg = method(*args, **kwargs) + ttft = time.time() - request_start_time + _log_message_to_span(msg, span, time_to_first_token=ttft, include_parsed_output=True) + return msg + except Exception as e: + span.log(error=e) + raise + finally: + span.end() + + +async def _trace_async_parsed_message_call(method, args, kwargs): + span = _start_span("anthropic.messages.parse", kwargs) + request_start_time = time.time() + try: + msg = await method(*args, **kwargs) + ttft = time.time() - request_start_time + _log_message_to_span(msg, span, time_to_first_token=ttft, include_parsed_output=True) + return msg + except Exception as e: + span.log(error=e) + raise + finally: + span.end() + + _BRAINTRUST_TRACED = "__braintrust_traced__"