Skip to content

update eval span attributes to render input/output like we do for datasets #140

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Andrew Kent (realark) wants to merge 3 commits into
base: main
Choose a base branch
from
+49 −35
Draft

update eval span attributes to render input/output like we do for datasets #140

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5c669d6
wip2
realark Jul 7, 2026
cddc548
wip3
realark Jul 7, 2026
77f8766
wip4
realark Jul 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 8 additions & 9 deletions braintrust-sdk/src/main/java/dev/braintrust/eval/Eval.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,9 +120,7 @@ private void evalOne(String experimentId, DatasetCase<INPUT, OUTPUT> datasetCase
.setSpanKind(SpanKind.CLIENT)
.setAttribute(PARENT, "experiment_id:" + experimentId)
.setAttribute("braintrust.span_attributes", toJson(Map.of("type", "eval")))
.setAttribute(
"braintrust.input_json",
toJson(Map.of("input", datasetCase.input())))
.setAttribute("braintrust.input_json", toJson(datasetCase.input()))
.setAttribute("braintrust.expected", toJson(datasetCase.expected()))
.startSpan();
if (datasetCase.origin().isPresent()) {
Expand All @@ -146,22 +144,23 @@ private void evalOne(String experimentId, DatasetCase<INPUT, OUTPUT> datasetCase
.setAttribute(
"braintrust.span_attributes",
toJson(Map.of("type", "task")))
.setAttribute("braintrust.input_json", toJson(datasetCase.input()))
.setAttribute("braintrust.expected", toJson(datasetCase.expected()))
.startSpan();
taskSpanId = taskSpan.getSpanContext().getSpanId();
try (var unused =
BraintrustContext.ofExperiment(experimentId, taskSpan).makeCurrent()) {
taskResult = task.apply(datasetCase, parameters);
rootSpan.setAttribute(
"braintrust.output_json",
toJson(Map.of("output", taskResult.result())));
taskSpan.setAttribute("braintrust.output_json", toJson(taskResult.result()));
rootSpan.setAttribute("braintrust.output_json", toJson(taskResult.result()));
} catch (Exception e) {
taskSpan.setStatus(StatusCode.ERROR, e.getMessage());
taskSpan.recordException(e);
taskSpan.end();
rootSpan.setStatus(StatusCode.ERROR, e.getMessage());
rootSpan.setAttribute(
"braintrust.output_json",
toJson(Collections.singletonMap("output", null)));
// Matching the Go/Python SDKs: on task failure we do not write an
// output_json to the eval span at all (rather than a literal null),
// so the UI renders an empty output instead of the string "null".
log.debug("Task threw exception for input: " + datasetCase.input(), e);
// run scoreForTaskException on each scorer
for (var scorer : scorers) {
Expand Down
60 changes: 38 additions & 22 deletions braintrust-sdk/src/test/java/dev/braintrust/eval/EvalTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,12 +86,15 @@ public void evalOtelTraceWithProperAttributes() {
"all eval spans must set the parent to the experiment id");
if (span.getParentSpanId().equals(SpanId.getInvalid())) {
numRootSpans.incrementAndGet();
var inputJson =
// input_json is the bare value, not wrapped in {"input": ...}
var input =
fromJson(
span.getAttributes()
.get(AttributeKey.stringKey("braintrust.input_json")),
Map.class);
assertNotNull(inputJson.get("input"), "invlaid input: " + inputJson);
String.class);
assertTrue(
input.equals("strawberry") || input.equals("asparagus"),
"invalid input: " + input);

var expected =
fromJson(
Expand All @@ -100,14 +103,30 @@ public void evalOtelTraceWithProperAttributes() {
String.class);
assertTrue(isFruitOrVegetable(expected), "invalid expected: " + expected);

var outputJson =
// output_json is the bare value, not wrapped in {"output": ...}
var output =
fromJson(
span.getAttributes()
.get(AttributeKey.stringKey("braintrust.output_json")),
Map.class);
var output = outputJson.get("output");
assertNotNull(output, "invlaid output: " + outputJson);
assertTrue(isFruitOrVegetable(String.valueOf(output)), "invalid output: " + output);
String.class);
assertNotNull(output, "invalid output: " + output);
assertTrue(isFruitOrVegetable(output), "invalid output: " + output);
} else if ("task".equals(span.getName())) {
// task span carries its own bare input/output
var input =
fromJson(
span.getAttributes()
.get(AttributeKey.stringKey("braintrust.input_json")),
String.class);
assertTrue(
input.equals("strawberry") || input.equals("asparagus"),
"invalid task input: " + input);
var output =
fromJson(
span.getAttributes()
.get(AttributeKey.stringKey("braintrust.output_json")),
String.class);
assertTrue(isFruitOrVegetable(output), "invalid task output: " + output);
}
}
assertEquals(2, numRootSpans.get(), "each case should make a root span");
Expand Down Expand Up @@ -315,8 +334,7 @@ public void evalRootSpanPassesOriginIfPresent() {
var inputJson =
span.getAttributes().get(AttributeKey.stringKey("braintrust.input_json"));
assertNotNull(inputJson);
fromJson(inputJson, Map.class);
var input = (String) (fromJson(inputJson, Map.class)).get("input");
var input = fromJson(inputJson, String.class);
assertNotNull(input);
var origin = span.getAttributes().get(AttributeKey.stringKey("braintrust.origin"));
switch (input) {
Expand Down Expand Up @@ -481,14 +499,13 @@ void evalContinuesWhenTaskThrows() {
erroredRootSpan.getStatus().getDescription().contains("task failed on bad-input"),
"root span error should contain the exception message");

// The errored root span should have output: null
var erroredOutputJson =
fromJson(
erroredRootSpan
.getAttributes()
.get(AttributeKey.stringKey("braintrust.output_json")),
Map.class);
assertNull(erroredOutputJson.get("output"), "errored case output should be null");
// On task failure, matching the Go/Python SDKs, the eval span does not set
// output_json at all (rather than a literal null), so the UI shows an empty output.
assertNull(
erroredRootSpan
.getAttributes()
.get(AttributeKey.stringKey("braintrust.output_json")),
"errored case should not set output_json on the eval span");

// Find the task span for the errored case (child of errored root, type=task, status=ERROR)
var erroredTaskSpan =
Expand Down Expand Up @@ -560,14 +577,13 @@ void evalContinuesWhenTaskThrows() {
.filter(s -> s.getStatus().getStatusCode() != StatusCode.ERROR)
.findFirst()
.orElseThrow(() -> new AssertionError("expected a successful root span"));
var successOutputJson =
var successOutput =
fromJson(
successRootSpan
.getAttributes()
.get(AttributeKey.stringKey("braintrust.output_json")),
Map.class);
assertEquals(
"result", successOutputJson.get("output"), "successful case should have output");
String.class);
assertEquals("result", successOutput, "successful case should have output");
}

@Test
Expand Down
7 changes: 3 additions & 4 deletions examples/experiment/src/main/java/dev/braintrust/examples/ExperimentExample.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,8 @@
import com.openai.models.ChatModel;
import com.openai.models.chat.completions.ChatCompletionCreateParams;
import dev.braintrust.Braintrust;
import dev.braintrust.eval.DatasetCase;
import dev.braintrust.eval.Scorer;
import dev.braintrust.instrumentation.openai.BraintrustOpenAI;
import java.util.List;
import java.util.Map;
import java.util.function.Function;

public class ExperimentExample {
Expand Down Expand Up @@ -36,6 +33,7 @@ public static void main(String[] args) throws Exception {
.<String, String>evalBuilder()
// NOTE: pre-existing experiment names will append results
.name("java-eval-x-" + System.currentTimeMillis())
/*
.cases(
DatasetCase.of(
"strawberry",
Expand All @@ -47,8 +45,9 @@ public static void main(String[] args) throws Exception {
DatasetCase.of("asparagus", "vegetable"),
DatasetCase.of("apple", "fruit"),
DatasetCase.of("banana", "fruit"))
*/
// Or, to fetch a remote dataset:
// .dataset(braintrust.fetchDataset("my-dataset-name"))
.dataset(braintrust.fetchDataset("food"))
.taskFunction(getFoodType)
.scorers(
// to fetch a remote scorer:
Expand Down
Loading