feat(anthropic): trace messages.parse outputs

py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_parse_async.yaml

@@ -0,0 +1,107 @@
+interactions:
+- request:
+    body: '{"max_tokens":128,"messages":[{"role":"user","content":"Return a JSON object
+      with answer=3 and label=\"async\"."}],"model":"claude-haiku-4-5","output_config":{"format":{"schema":{"type":"object","title":"ParsedAnswer","properties":{"answer":{"type":"integer","title":"Answer"},"label":{"type":"string","title":"Label"}},"additionalProperties":false,"required":["answer","label"]},"type":"json_schema"}}}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate
+      anthropic-version:
+      - '2023-06-01'
+      connection:
+      - keep-alive
+      content-length:
+      - '403'
+      content-type:
+      - application/json
+      host:
+      - api.anthropic.com
+      user-agent:
+      - AsyncAnthropic/Python 0.116.0
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - async:asyncio
+      x-stainless-helper:
+      - messages.parse
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 0.116.0
+      x-stainless-read-timeout:
+      - '600'
+      x-stainless-retry-count:
+      - '0'
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.14.3
+      x-stainless-timeout:
+      - '600'
+    method: POST
+    uri: https://api.anthropic.com/v1/messages
+  response:
+    body:
+      string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011Ccq6VRSBSPyrNEUcTfCuN","type":"message","role":"assistant","content":[{"type":"text","text":"{\"answer\":3,\"label\":\"async\"}"}],"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":210,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":12,"service_tier":"standard","inference_geo":"not_available"}}'
+    headers:
+      anthropic-organization-id:
+      - 27796668-7351-40ac-acc4-024aee8995a5
+      anthropic-ratelimit-input-tokens-limit:
+      - '10000000'
+      anthropic-ratelimit-input-tokens-remaining:
+      - '10000000'
+      anthropic-ratelimit-input-tokens-reset:
+      - '2026-07-08T18:52:34Z'
+      anthropic-ratelimit-output-tokens-limit:
+      - '2000000'
+      anthropic-ratelimit-output-tokens-remaining:
+      - '2000000'
+      anthropic-ratelimit-output-tokens-reset:
+      - '2026-07-08T18:52:34Z'
+      anthropic-ratelimit-requests-limit:
+      - '20000'
+      anthropic-ratelimit-requests-remaining:
+      - '19999'
+      anthropic-ratelimit-requests-reset:
+      - '2026-07-08T18:52:33Z'
+      anthropic-ratelimit-tokens-limit:
+      - '12000000'
+      anthropic-ratelimit-tokens-remaining:
+      - '12000000'
+      anthropic-ratelimit-tokens-reset:
+      - '2026-07-08T18:52:34Z'
+      cf-cache-status:
+      - DYNAMIC
+      cf-ray:
+      - a1814961defec48a-YYZ
+      connection:
+      - keep-alive
+      content-length:
+      - '492'
+      content-security-policy:
+      - default-src 'none'; frame-ancestors 'none'
+      content-type:
+      - application/json
+      date:
+      - Wed, 08 Jul 2026 18:52:34 GMT
+      request-id:
+      - req_011Ccq6VQ6ZdYURuNHeoyLen
+      server:
+      - cloudflare
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      traceresponse:
+      - 00-d19eca3922a082a78c44c20b9b66b6d8-daed6abfc1d21be5-01
+      transfer-encoding:
+      - chunked
+      vary:
+      - Accept-Encoding
+      x-robots-tag:
+      - none
+    status:
+      code: 200
+      message: OK
+version: 1

py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_parse_sync.yaml

@@ -0,0 +1,110 @@
+interactions:
+- request:
+    body: '{"max_tokens":128,"messages":[{"role":"user","content":"Return a JSON object
+      with answer=2 and label=\"ok\"."}],"model":"claude-haiku-4-5","output_config":{"format":{"schema":{"type":"object","title":"ParsedAnswer","properties":{"answer":{"type":"integer","title":"Answer"},"label":{"type":"string","title":"Label"}},"additionalProperties":false,"required":["answer","label"]},"type":"json_schema"}},"system":"Return
+      only the requested structured JSON object.","tool_choice":{"type":"none"},"tools":[{"name":"calculator","description":"Evaluate
+      simple arithmetic expressions.","input_schema":{"type":"object","properties":{"expression":{"type":"string"}},"required":["expression"]}}]}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate
+      anthropic-version:
+      - '2023-06-01'
+      connection:
+      - keep-alive
+      content-length:
+      - '684'
+      content-type:
+      - application/json
+      host:
+      - api.anthropic.com
+      user-agent:
+      - Anthropic/Python 0.116.0
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - 'false'
+      x-stainless-helper:
+      - messages.parse
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 0.116.0
+      x-stainless-read-timeout:
+      - '600'
+      x-stainless-retry-count:
+      - '0'
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.14.3
+      x-stainless-timeout:
+      - '600'
+    method: POST
+    uri: https://api.anthropic.com/v1/messages
+  response:
+    body:
+      string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011Ccq6RSmCH5q9JEWYpLfqS","type":"message","role":"assistant","content":[{"type":"text","text":"{\"answer\":
+        2, \"label\": \"ok\"}"}],"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":767,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":15,"service_tier":"standard","inference_geo":"not_available"}}'
+    headers:
+      anthropic-organization-id:
+      - 27796668-7351-40ac-acc4-024aee8995a5
+      anthropic-ratelimit-input-tokens-limit:
+      - '10000000'
+      anthropic-ratelimit-input-tokens-remaining:
+      - '10000000'
+      anthropic-ratelimit-input-tokens-reset:
+      - '2026-07-08T18:51:39Z'
+      anthropic-ratelimit-output-tokens-limit:
+      - '2000000'
+      anthropic-ratelimit-output-tokens-remaining:
+      - '2000000'
+      anthropic-ratelimit-output-tokens-reset:
+      - '2026-07-08T18:51:39Z'
+      anthropic-ratelimit-requests-limit:
+      - '20000'
+      anthropic-ratelimit-requests-remaining:
+      - '19999'
+      anthropic-ratelimit-requests-reset:
+      - '2026-07-08T18:51:37Z'
+      anthropic-ratelimit-tokens-limit:
+      - '12000000'
+      anthropic-ratelimit-tokens-remaining:
+      - '12000000'
+      anthropic-ratelimit-tokens-reset:
+      - '2026-07-08T18:51:39Z'
+      cf-cache-status:
+      - DYNAMIC
+      cf-ray:
+      - a1814807f983dde5-YYZ
+      connection:
+      - keep-alive
+      content-length:
+      - '492'
+      content-security-policy:
+      - default-src 'none'; frame-ancestors 'none'
+      content-type:
+      - application/json
+      date:
+      - Wed, 08 Jul 2026 18:51:40 GMT
+      request-id:
+      - req_011Ccq6RKXNVH5sQmXLTcj1o
+      server:
+      - cloudflare
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      traceresponse:
+      - 00-81d6835151765e3c6b4845a036ff2a71-fff1ab70ce453822-01
+      transfer-encoding:
+      - chunked
+      vary:
+      - Accept-Encoding
+      x-robots-tag:
+      - none
+    status:
+      code: 200
+      message: OK
+version: 1

py/src/braintrust/integrations/anthropic/test_anthropic.py

@@ -13,7 +13,7 @@
 import pytest
 from braintrust import Attachment, logger
 from braintrust.integrations.anthropic import AnthropicIntegration, wrap_anthropic
-from braintrust.integrations.anthropic._utils import extract_anthropic_usage
+from braintrust.integrations.anthropic._utils import _try_to_dict, extract_anthropic_usage
 from braintrust.integrations.anthropic.tracing import (
     _get_input_from_kwargs,
     _get_metadata_from_kwargs,
@@ -22,6 +22,7 @@
 from braintrust.integrations.test_utils import verify_autoinstrument_script
 from braintrust.span_types import SpanTypeAttribute
 from braintrust.test_helpers import find_span_by_name, find_spans_by_type, init_test_logger
+from pydantic import BaseModel
 
 
 PROJECT_NAME = "test-anthropic-app"
@@ -39,6 +40,17 @@
     "required": ["answer", "label"],
     "additionalProperties": False,
 }
+STRUCTURED_OUTPUT_TOOLS = [
+    {
+        "name": "calculator",
+        "description": "Evaluate simple arithmetic expressions.",
+        "input_schema": {
+            "type": "object",
+            "properties": {"expression": {"type": "string"}},
+            "required": ["expression"],
+        },
+    }
+]
 PNG_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="
 PDF_BASE64 = "JVBERi0xLjAKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PmVuZG9iagoyIDAgb2JqCjw8L1R5cGUvUGFnZXMvS2lkc1szIDAgUl0vQ291bnQgMT4+ZW5kb2JqCjMgMCBvYmoKPDwvVHlwZS9QYWdlL01lZGlhQm94WzAgMCA2MTIgNzkyXT4+ZW5kb2JqCnhyZWYKMCA0CjAwMDAwMDAwMDAgNjU1MzUgZg0KMDAwMDAwMDAxMCAwMDAwMCBuDQowMDAwMDAwMDUzIDAwMDAwIG4NCjAwMDAwMDAxMDIgMDAwMDAgbg0KdHJhaWxlcgo8PC9TaXplIDQvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgoxNDkKJUVPRg=="
 PROMPT_CACHE_TEST_TEXT = "\n".join(f"Cached geography fact {i}: Paris is the capital of France." for i in range(300))
@@ -52,6 +64,11 @@ def _get_async_client():
     return anthropic.AsyncAnthropic()
 
 
+class ParsedAnswer(BaseModel):
+    answer: int
+    label: str
+
+
 def _get_supported_structured_output_param():
     parameter_names = inspect.signature(_get_client().messages.create).parameters
     if "output_config" in parameter_names:
@@ -75,6 +92,28 @@ def _get_supported_structured_output_param():
     pytest.skip("Installed anthropic SDK does not support structured outputs parameters")
 
 
+def _skip_if_messages_parse_unsupported():
+    if not hasattr(_get_client().messages, "parse"):
+        pytest.skip("Installed anthropic SDK does not support messages.parse")
+
+
+def test_wrap_anthropic_preserves_messages_parse_feature_detection():
+    class MessagesWithoutParse:
+        pass
+
+    class FakeAnthropic:
+        messages = MessagesWithoutParse()
+
+    class FakeAsyncAnthropic:
+        messages = MessagesWithoutParse()
+
+    for client in (FakeAnthropic(), FakeAsyncAnthropic()):
+        messages = wrap_anthropic(client).messages
+        assert not hasattr(messages, "parse")
+        with pytest.raises(AttributeError):
+            getattr(messages, "parse")
+
+
 def _skip_if_server_tool_content_blocks_unsupported():
     required_type_names = ("ServerToolUseBlock", "WebSearchToolResultBlock")
     if not all(hasattr(anthropic.types, type_name) for type_name in required_type_names):
@@ -558,6 +597,93 @@ def test_anthropic_messages_create_tracks_structured_outputs_metadata(memory_log
     assert span["metadata"][structured_output_param_name] == structured_output_param_value
 
 
+@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "body"])
+def test_anthropic_messages_parse_sync(memory_logger):
+    _skip_if_messages_parse_unsupported()
+    assert not memory_logger.pop()
+
+    client = wrap_anthropic(_get_client())
+    params = {
+        "model": STRUCTURED_OUTPUT_MODEL,
+        "max_tokens": 128,
+        "system": "Return only the requested structured JSON object.",
+        "messages": [
+            {
+                "role": "user",
+                "content": 'Return a JSON object with answer=2 and label="ok".',
+            }
+        ],
+        "tools": STRUCTURED_OUTPUT_TOOLS,
+        "tool_choice": {"type": "none"},
+        "output_format": ParsedAnswer,
+    }
+
+    start = time.time()
+    response = client.messages.parse(**params)
+    end = time.time()
+
+    assert _try_to_dict(response.parsed_output) == {"answer": 2, "label": "ok"}
+
+    spans = memory_logger.pop()
+    assert len(spans) == 1
+    span = spans[0]
+    assert span["span_attributes"]["name"] == "anthropic.messages.parse"
+    assert span["span_attributes"]["type"] == "llm"
+    assert span["metadata"]["provider"] == "anthropic"
+    assert span["metadata"]["model"] == STRUCTURED_OUTPUT_MODEL
+    assert span["metadata"]["output_format"].endswith("ParsedAnswer")
+    assert span["metadata"]["tools"] == STRUCTURED_OUTPUT_TOOLS
+    assert span["input"] == [
+        *params["messages"],
+        {"role": "system", "content": params["system"]},
+    ]
+    assert span["output"]["parsed_output"] == {"answer": 2, "label": "ok"}
+    assert span["output"]["model"] == response.model
+    _assert_metrics_are_valid(span["metrics"], start, end)
+    assert span["metrics"]["prompt_tokens"] == response.usage.input_tokens
+    assert span["metrics"]["completion_tokens"] == response.usage.output_tokens
+    assert span["metrics"]["tokens"] == response.usage.input_tokens + response.usage.output_tokens
+
+
+@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "body"])
+@pytest.mark.asyncio
+async def test_anthropic_messages_parse_async(memory_logger):
+    _skip_if_messages_parse_unsupported()
+    assert not memory_logger.pop()
+
+    client = wrap_anthropic(_get_async_client())
+    params = {
+        "model": STRUCTURED_OUTPUT_MODEL,
+        "max_tokens": 128,
+        "messages": [
+            {
+                "role": "user",
+                "content": 'Return a JSON object with answer=3 and label="async".',
+            }
+        ],
+        "output_format": ParsedAnswer,
+    }
+
+    start = time.time()
+    response = await client.messages.parse(**params)
+    end = time.time()
+
+    assert _try_to_dict(response.parsed_output) == {"answer": 3, "label": "async"}
+
+    spans = memory_logger.pop()
+    assert len(spans) == 1
+    span = spans[0]
+    assert span["span_attributes"]["name"] == "anthropic.messages.parse"
+    assert span["metadata"]["provider"] == "anthropic"
+    assert span["metadata"]["model"] == STRUCTURED_OUTPUT_MODEL
+    assert span["metadata"]["output_format"].endswith("ParsedAnswer")
+    assert span["input"] == params["messages"]
+    assert span["output"]["parsed_output"] == {"answer": 3, "label": "async"}
+    _assert_metrics_are_valid(span["metrics"], start, end)
+    assert span["metrics"]["prompt_tokens"] == response.usage.input_tokens
+    assert span["metrics"]["completion_tokens"] == response.usage.output_tokens
+
+
 @pytest.mark.vcr
 def test_anthropic_messages_model_params_inputs(memory_logger):
     assert not memory_logger.pop()