diff --git a/.changeset/mastra-thread-view-llm-io.md b/.changeset/mastra-thread-view-llm-io.md new file mode 100644 index 000000000..3bd80f4ca --- /dev/null +++ b/.changeset/mastra-thread-view-llm-io.md @@ -0,0 +1,14 @@ +--- +"braintrust": patch +--- + +fix(mastra): Transform LLM input/output for the Braintrust Thread view + +`MODEL_GENERATION` spans from the Mastra observability exporter now log input +as a bare chat-message array (unwrapping Mastra's `{ messages: [...] }` +container, or wrapping a single `{ role, content }` object) and output as an +`{ role: 'assistant', content }` message (unwrapping Mastra's `{ text, ... }`), +so Braintrust's Thread view renders the conversation correctly. Non-model spans +are unchanged. + +Ports mastra-ai/mastra#10794 (fixes #9848). diff --git a/e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.log-payloads.json b/e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.log-payloads.json index 75d3077e7..ced8f093b 100644 --- a/e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.log-payloads.json +++ b/e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.log-payloads.json @@ -31,28 +31,26 @@ "type": "task" }, { - "input": { - "messages": [ - { - "content": "Answer weather questions with the provided mock forecast.", - "role": "system" - }, - { - "content": [ - { - "providerOptions": { - "mastra": { - "createdAt": 0 - } - }, - "text": "What is the weather in Paris?", - "type": "text" - } - ], - "role": "user" - } - ] - }, + "input": [ + { + "content": "Answer weather questions with the provided mock forecast.", + "role": "system" + }, + { + "content": [ + { + "providerOptions": { + "mastra": { + "createdAt": 0 + } + }, + "text": "What is the weather in Paris?", + "type": "text" + } + ], + "role": "user" + } + ], "metadata": { "entity_id": "weather-agent", "entity_name": "Weather Agent", @@ -67,10 +65,11 @@ ], "name": "llm: 'mock-model-id'", "output": { + "content": "The forecast is sunny.", "files": [], "reasoning": [], + "role": "assistant", "sources": [], - "text": "The forecast is sunny.", "warnings": [] }, "type": "llm" @@ -131,28 +130,26 @@ "type": "task" }, { - "input": { - "messages": [ - { - "content": "Answer weather questions with the provided mock forecast.", - "role": "system" - }, - { - "content": [ - { - "providerOptions": { - "mastra": { - "createdAt": 0 - } - }, - "text": "Stream the Paris forecast.", - "type": "text" - } - ], - "role": "user" - } - ] - }, + "input": [ + { + "content": "Answer weather questions with the provided mock forecast.", + "role": "system" + }, + { + "content": [ + { + "providerOptions": { + "mastra": { + "createdAt": 0 + } + }, + "text": "Stream the Paris forecast.", + "type": "text" + } + ], + "role": "user" + } + ], "metadata": { "entity_id": "weather-agent", "entity_name": "Weather Agent", @@ -167,10 +164,11 @@ ], "name": "llm: 'mock-model-id'", "output": { + "content": "The forecast is sunny.", "files": [], "reasoning": [], + "role": "assistant", "sources": [], - "text": "The forecast is sunny.", "warnings": [] }, "type": "llm" diff --git a/e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.span-tree.json b/e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.span-tree.json index e7ee35483..eede838c7 100644 --- a/e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.span-tree.json +++ b/e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.span-tree.json @@ -49,33 +49,32 @@ ] } ], - "input": { - "messages": [ - { - "content": "Answer weather questions with the provided mock forecast.", - "role": "system" - }, - { - "content": [ - { - "providerOptions": { - "mastra": { - "createdAt": 0 - } - }, - "text": "What is the weather in Paris?", - "type": "text" - } - ], - "role": "user" - } - ] - }, + "input": [ + { + "content": "Answer weather questions with the provided mock forecast.", + "role": "system" + }, + { + "content": [ + { + "providerOptions": { + "mastra": { + "createdAt": 0 + } + }, + "text": "What is the weather in Paris?", + "type": "text" + } + ], + "role": "user" + } + ], "output": { + "content": "The forecast is sunny.", "files": [], "reasoning": [], + "role": "assistant", "sources": [], - "text": "The forecast is sunny.", "warnings": [] }, "metadata": { @@ -145,33 +144,32 @@ ] } ], - "input": { - "messages": [ - { - "content": "Answer weather questions with the provided mock forecast.", - "role": "system" - }, - { - "content": [ - { - "providerOptions": { - "mastra": { - "createdAt": 0 - } - }, - "text": "Stream the Paris forecast.", - "type": "text" - } - ], - "role": "user" - } - ] - }, + "input": [ + { + "content": "Answer weather questions with the provided mock forecast.", + "role": "system" + }, + { + "content": [ + { + "providerOptions": { + "mastra": { + "createdAt": 0 + } + }, + "text": "Stream the Paris forecast.", + "type": "text" + } + ], + "role": "user" + } + ], "output": { + "content": "The forecast is sunny.", "files": [], "reasoning": [], + "role": "assistant", "sources": [], - "text": "The forecast is sunny.", "warnings": [] }, "metadata": { diff --git a/e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.span-tree.txt b/e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.span-tree.txt index e94ee3248..8bb120521 100644 --- a/e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.span-tree.txt +++ b/e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.span-tree.txt @@ -19,33 +19,32 @@ span_tree: │ │ "entity_type": "agent" │ │ } │ │ └── llm: 'mock-model-id' [llm] - │ │ input: { - │ │ "messages": [ - │ │ { - │ │ "content": "Answer weather questions with the provided mock forecast.", - │ │ "role": "system" - │ │ }, - │ │ { - │ │ "content": [ - │ │ { - │ │ "providerOptions": { - │ │ "mastra": { - │ │ "createdAt": 0 - │ │ } - │ │ }, - │ │ "text": "What is the weather in Paris?", - │ │ "type": "text" - │ │ } - │ │ ], - │ │ "role": "user" - │ │ } - │ │ ] - │ │ } + │ │ input: [ + │ │ { + │ │ "content": "Answer weather questions with the provided mock forecast.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerOptions": { + │ │ "mastra": { + │ │ "createdAt": 0 + │ │ } + │ │ }, + │ │ "text": "What is the weather in Paris?", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ } + │ │ ] │ │ output: { + │ │ "content": "The forecast is sunny.", │ │ "files": [], │ │ "reasoning": [], + │ │ "role": "assistant", │ │ "sources": [], - │ │ "text": "The forecast is sunny.", │ │ "warnings": [] │ │ } │ │ metadata: { @@ -96,33 +95,32 @@ span_tree: │ │ "entity_type": "agent" │ │ } │ │ └── llm: 'mock-model-id' [llm] - │ │ input: { - │ │ "messages": [ - │ │ { - │ │ "content": "Answer weather questions with the provided mock forecast.", - │ │ "role": "system" - │ │ }, - │ │ { - │ │ "content": [ - │ │ { - │ │ "providerOptions": { - │ │ "mastra": { - │ │ "createdAt": 0 - │ │ } - │ │ }, - │ │ "text": "Stream the Paris forecast.", - │ │ "type": "text" - │ │ } - │ │ ], - │ │ "role": "user" - │ │ } - │ │ ] - │ │ } + │ │ input: [ + │ │ { + │ │ "content": "Answer weather questions with the provided mock forecast.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerOptions": { + │ │ "mastra": { + │ │ "createdAt": 0 + │ │ } + │ │ }, + │ │ "text": "Stream the Paris forecast.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ } + │ │ ] │ │ output: { + │ │ "content": "The forecast is sunny.", │ │ "files": [], │ │ "reasoning": [], + │ │ "role": "assistant", │ │ "sources": [], - │ │ "text": "The forecast is sunny.", │ │ "warnings": [] │ │ } │ │ metadata: { diff --git a/js/src/wrappers/mastra.test.ts b/js/src/wrappers/mastra.test.ts new file mode 100644 index 000000000..5587ec8de --- /dev/null +++ b/js/src/wrappers/mastra.test.ts @@ -0,0 +1,104 @@ +import { + afterEach, + beforeAll, + beforeEach, + describe, + expect, + test, +} from "vitest"; +import { configureNode } from "../node/config"; +import { _exportsForTestingOnly, initLogger } from "../logger"; +import { BraintrustObservabilityExporter } from "./mastra"; + +try { + configureNode(); +} catch { + // Best-effort initialization for test environments. +} + +type MastraExportedSpan = Parameters< + BraintrustObservabilityExporter["exportTracingEvent"] +>[0]["exportedSpan"]; + +describe("mastra thread-view IO transform", () => { + let backgroundLogger: ReturnType< + typeof _exportsForTestingOnly.useTestBackgroundLogger + >; + + beforeAll(async () => { + await _exportsForTestingOnly.simulateLoginForTests(); + }); + + beforeEach(() => { + backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); + initLogger({ projectId: "test-project-id", projectName: "mastra.test.ts" }); + }); + + afterEach(() => { + _exportsForTestingOnly.clearTestBackgroundLogger(); + }); + + // Run a span through the exporter start->end and return the logged row. + async function logSpan(span: Partial): Promise { + const exporter = new BraintrustObservabilityExporter(); + const exportedSpan = { + id: "span-1", + traceId: "trace-1", + name: span.name ?? "llm: 'mock-model'", + type: "model_generation", + startTime: 1_000_000, + ...span, + } as MastraExportedSpan; + await exporter.exportTracingEvent({ type: "span_started", exportedSpan }); + await exporter.exportTracingEvent({ + type: "span_ended", + exportedSpan: { ...exportedSpan, endTime: 1_000_001 }, + }); + await backgroundLogger.flush(); + const events = (await backgroundLogger.drain()) as any[]; + return events.find((e) => e.span_attributes?.name === exportedSpan.name); + } + + test("unwraps model_generation input { messages } into a bare array", async () => { + const messages = [ + { role: "system", content: "You are helpful." }, + { role: "user", content: "Hi" }, + ]; + const row = await logSpan({ input: { messages }, output: { text: "hi" } }); + expect(row?.input).toEqual(messages); + }); + + test("wraps a single model_generation input object into an array", async () => { + const row = await logSpan({ + input: { role: "user", content: "Just this" }, + output: { text: "ok" }, + }); + expect(row?.input).toEqual([{ role: "user", content: "Just this" }]); + }); + + test("reshapes model_generation output text into an assistant message", async () => { + const row = await logSpan({ + input: { messages: [{ role: "user", content: "Hi" }] }, + output: { text: "Hello there!", files: [], warnings: [] }, + }); + expect(row?.output).toEqual({ + role: "assistant", + content: "Hello there!", + files: [], + warnings: [], + }); + }); + + test("leaves non-model spans untouched", async () => { + const input = { messages: [{ role: "user", content: "Hi" }] }; + const output = { text: "raw" }; + const row = await logSpan({ + name: "tool call", + type: "tool_call", + input, + output, + }); + expect(row?.input).toEqual(input); + expect(row?.output).toEqual(output); + }); +}); diff --git a/js/src/wrappers/mastra.ts b/js/src/wrappers/mastra.ts index 9946a9643..4baac4543 100644 --- a/js/src/wrappers/mastra.ts +++ b/js/src/wrappers/mastra.ts @@ -170,6 +170,43 @@ function modelMetrics( return Object.keys(out).length > 0 ? out : undefined; } +/** Mastra's `MODEL_GENERATION` span type string. Only these spans carry the + * chat input/output that Braintrust's Thread view renders, so the shape + * transforms below are scoped to them. */ +const MODEL_GENERATION_TYPE = "model_generation"; + +/** + * Unwrap a `MODEL_GENERATION` span's input into the message-array shape + * Braintrust's Thread view expects. Mastra logs LLM input as + * `{ messages: [...] }` (or occasionally a single `{ role, content }` + * object); Braintrust renders a bare array of chat messages. Non-model spans + * and unrecognized shapes pass through unchanged. + */ +function transformModelInput(input: unknown, mastraType: string): unknown { + if (mastraType !== MODEL_GENERATION_TYPE) return input; + if (isObject(input) && Array.isArray(input.messages)) { + return input.messages; + } + if (isObject(input) && "content" in input) { + return [{ role: input.role, content: input.content }]; + } + return input; +} + +/** + * Reshape a `MODEL_GENERATION` span's output into the assistant-message shape + * Braintrust's Thread view expects. Mastra logs LLM output as + * `{ text, ...rest }`; Braintrust renders `{ role: 'assistant', content }`. + * The remaining fields are preserved. Non-model spans and non-object outputs + * pass through unchanged. + */ +function transformModelOutput(output: unknown, mastraType: string): unknown { + if (mastraType !== MODEL_GENERATION_TYPE) return output; + if (!isObject(output)) return output; + const { text, ...rest } = output; + return { role: "assistant", content: text, ...rest }; +} + /** Build the metadata payload Braintrust shows on the span, merging * Mastra's own `metadata`, `attributes` (sans usage), and entity fields. */ function buildMetadata(exported: MastraExportedSpan): Record { @@ -334,11 +371,11 @@ export class BraintrustObservabilityExporter implements MastraObservabilityExpor const event: Record = {}; if (exported.input !== undefined) { - event.input = exported.input; + event.input = transformModelInput(exported.input, exported.type); record.hasLoggedInput = true; } if (exported.output !== undefined) { - event.output = exported.output; + event.output = transformModelOutput(exported.output, exported.type); } const metadata = buildMetadata(exported);