diff --git a/braintrust-sdk/src/main/java/dev/braintrust/eval/Eval.java b/braintrust-sdk/src/main/java/dev/braintrust/eval/Eval.java index fc9e12c0..c162e360 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/eval/Eval.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/eval/Eval.java @@ -120,9 +120,7 @@ private void evalOne(String experimentId, DatasetCase datasetCase .setSpanKind(SpanKind.CLIENT) .setAttribute(PARENT, "experiment_id:" + experimentId) .setAttribute("braintrust.span_attributes", toJson(Map.of("type", "eval"))) - .setAttribute( - "braintrust.input_json", - toJson(Map.of("input", datasetCase.input()))) + .setAttribute("braintrust.input_json", toJson(datasetCase.input())) .setAttribute("braintrust.expected", toJson(datasetCase.expected())) .startSpan(); if (datasetCase.origin().isPresent()) { @@ -146,22 +144,23 @@ private void evalOne(String experimentId, DatasetCase datasetCase .setAttribute( "braintrust.span_attributes", toJson(Map.of("type", "task"))) + .setAttribute("braintrust.input_json", toJson(datasetCase.input())) + .setAttribute("braintrust.expected", toJson(datasetCase.expected())) .startSpan(); taskSpanId = taskSpan.getSpanContext().getSpanId(); try (var unused = BraintrustContext.ofExperiment(experimentId, taskSpan).makeCurrent()) { taskResult = task.apply(datasetCase, parameters); - rootSpan.setAttribute( - "braintrust.output_json", - toJson(Map.of("output", taskResult.result()))); + taskSpan.setAttribute("braintrust.output_json", toJson(taskResult.result())); + rootSpan.setAttribute("braintrust.output_json", toJson(taskResult.result())); } catch (Exception e) { taskSpan.setStatus(StatusCode.ERROR, e.getMessage()); taskSpan.recordException(e); taskSpan.end(); rootSpan.setStatus(StatusCode.ERROR, e.getMessage()); - rootSpan.setAttribute( - "braintrust.output_json", - toJson(Collections.singletonMap("output", null))); + // Matching the Go/Python SDKs: on task failure we do not write an + // output_json to the eval span at all (rather than a literal null), + // so the UI renders an empty output instead of the string "null". log.debug("Task threw exception for input: " + datasetCase.input(), e); // run scoreForTaskException on each scorer for (var scorer : scorers) { diff --git a/braintrust-sdk/src/test/java/dev/braintrust/eval/EvalTest.java b/braintrust-sdk/src/test/java/dev/braintrust/eval/EvalTest.java index c4cd1f00..447dfe8e 100644 --- a/braintrust-sdk/src/test/java/dev/braintrust/eval/EvalTest.java +++ b/braintrust-sdk/src/test/java/dev/braintrust/eval/EvalTest.java @@ -86,12 +86,15 @@ public void evalOtelTraceWithProperAttributes() { "all eval spans must set the parent to the experiment id"); if (span.getParentSpanId().equals(SpanId.getInvalid())) { numRootSpans.incrementAndGet(); - var inputJson = + // input_json is the bare value, not wrapped in {"input": ...} + var input = fromJson( span.getAttributes() .get(AttributeKey.stringKey("braintrust.input_json")), - Map.class); - assertNotNull(inputJson.get("input"), "invlaid input: " + inputJson); + String.class); + assertTrue( + input.equals("strawberry") || input.equals("asparagus"), + "invalid input: " + input); var expected = fromJson( @@ -100,14 +103,30 @@ public void evalOtelTraceWithProperAttributes() { String.class); assertTrue(isFruitOrVegetable(expected), "invalid expected: " + expected); - var outputJson = + // output_json is the bare value, not wrapped in {"output": ...} + var output = fromJson( span.getAttributes() .get(AttributeKey.stringKey("braintrust.output_json")), - Map.class); - var output = outputJson.get("output"); - assertNotNull(output, "invlaid output: " + outputJson); - assertTrue(isFruitOrVegetable(String.valueOf(output)), "invalid output: " + output); + String.class); + assertNotNull(output, "invalid output: " + output); + assertTrue(isFruitOrVegetable(output), "invalid output: " + output); + } else if ("task".equals(span.getName())) { + // task span carries its own bare input/output + var input = + fromJson( + span.getAttributes() + .get(AttributeKey.stringKey("braintrust.input_json")), + String.class); + assertTrue( + input.equals("strawberry") || input.equals("asparagus"), + "invalid task input: " + input); + var output = + fromJson( + span.getAttributes() + .get(AttributeKey.stringKey("braintrust.output_json")), + String.class); + assertTrue(isFruitOrVegetable(output), "invalid task output: " + output); } } assertEquals(2, numRootSpans.get(), "each case should make a root span"); @@ -315,8 +334,7 @@ public void evalRootSpanPassesOriginIfPresent() { var inputJson = span.getAttributes().get(AttributeKey.stringKey("braintrust.input_json")); assertNotNull(inputJson); - fromJson(inputJson, Map.class); - var input = (String) (fromJson(inputJson, Map.class)).get("input"); + var input = fromJson(inputJson, String.class); assertNotNull(input); var origin = span.getAttributes().get(AttributeKey.stringKey("braintrust.origin")); switch (input) { @@ -481,14 +499,13 @@ void evalContinuesWhenTaskThrows() { erroredRootSpan.getStatus().getDescription().contains("task failed on bad-input"), "root span error should contain the exception message"); - // The errored root span should have output: null - var erroredOutputJson = - fromJson( - erroredRootSpan - .getAttributes() - .get(AttributeKey.stringKey("braintrust.output_json")), - Map.class); - assertNull(erroredOutputJson.get("output"), "errored case output should be null"); + // On task failure, matching the Go/Python SDKs, the eval span does not set + // output_json at all (rather than a literal null), so the UI shows an empty output. + assertNull( + erroredRootSpan + .getAttributes() + .get(AttributeKey.stringKey("braintrust.output_json")), + "errored case should not set output_json on the eval span"); // Find the task span for the errored case (child of errored root, type=task, status=ERROR) var erroredTaskSpan = @@ -560,14 +577,13 @@ void evalContinuesWhenTaskThrows() { .filter(s -> s.getStatus().getStatusCode() != StatusCode.ERROR) .findFirst() .orElseThrow(() -> new AssertionError("expected a successful root span")); - var successOutputJson = + var successOutput = fromJson( successRootSpan .getAttributes() .get(AttributeKey.stringKey("braintrust.output_json")), - Map.class); - assertEquals( - "result", successOutputJson.get("output"), "successful case should have output"); + String.class); + assertEquals("result", successOutput, "successful case should have output"); } @Test diff --git a/examples/experiment/src/main/java/dev/braintrust/examples/ExperimentExample.java b/examples/experiment/src/main/java/dev/braintrust/examples/ExperimentExample.java index 09a33835..a7184392 100644 --- a/examples/experiment/src/main/java/dev/braintrust/examples/ExperimentExample.java +++ b/examples/experiment/src/main/java/dev/braintrust/examples/ExperimentExample.java @@ -4,11 +4,8 @@ import com.openai.models.ChatModel; import com.openai.models.chat.completions.ChatCompletionCreateParams; import dev.braintrust.Braintrust; -import dev.braintrust.eval.DatasetCase; import dev.braintrust.eval.Scorer; import dev.braintrust.instrumentation.openai.BraintrustOpenAI; -import java.util.List; -import java.util.Map; import java.util.function.Function; public class ExperimentExample { @@ -36,6 +33,7 @@ public static void main(String[] args) throws Exception { .evalBuilder() // NOTE: pre-existing experiment names will append results .name("java-eval-x-" + System.currentTimeMillis()) + /* .cases( DatasetCase.of( "strawberry", @@ -47,8 +45,9 @@ public static void main(String[] args) throws Exception { DatasetCase.of("asparagus", "vegetable"), DatasetCase.of("apple", "fruit"), DatasetCase.of("banana", "fruit")) + */ // Or, to fetch a remote dataset: - // .dataset(braintrust.fetchDataset("my-dataset-name")) + .dataset(braintrust.fetchDataset("food")) .taskFunction(getFoodType) .scorers( // to fetch a remote scorer: