Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,6 @@ class EvaluatorOutput(BaseModel):
def _require_label_or_error_code(self) -> "EvaluatorOutput":
if not self.errorCode and self.label is None:
raise ValueError(
"Either label, value, or errorCode must be set; "
"set errorCode to return an error response without a label"
"label is required for success responses; set errorCode to return an error response without a label"
)
return self
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,7 @@ evaluator = create_strands_evaluator("Builtin.Helpfulness")

# Run evaluations
experiment = Experiment(cases=cases, evaluators=[evaluator])
reports = experiment.run_evaluations(task_fn)
report = reports[0]
report = experiment.run_evaluations(task_fn)

# View results
print(f"Overall score: {report.overall_score:.2f}")
Expand Down Expand Up @@ -175,8 +174,7 @@ def task_fn(case):

evaluator = create_strands_evaluator("Builtin.Helpfulness")
experiment = Experiment(cases=cases, evaluators=[evaluator])
reports = experiment.run_evaluations(task_fn)
report = reports[0]
report = experiment.run_evaluations(task_fn)

print(f"Overall score: {report.overall_score:.2f}")
```
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,7 @@ def task_fn(case):
with patch("boto3.client", return_value=mock_boto_client):
evaluator = create_strands_evaluator("Builtin.Helpfulness")
experiment = Experiment(cases=cases, evaluators=[evaluator])
reports = experiment.run_evaluations(task_fn)
report = reports[0]
report = experiment.run_evaluations(task_fn)

# Should return 0 score for empty trajectory
assert report.overall_score == 0.0
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3722,9 +3722,13 @@ def test_async_mode_registers_multi_agent_callbacks(self, mock_memory_client):
registry = HookRegistry()
manager.register_hooks(registry)

for event_type in (MultiAgentInitializedEvent, AfterNodeCallEvent, AfterMultiAgentInvocationEvent):
callbacks = registry._registered_callbacks.get(event_type, [])
assert callbacks, f"No callbacks registered for {event_type.__name__}"
for event in (
MultiAgentInitializedEvent(source=Mock()),
AfterNodeCallEvent(source=Mock(), node_id="n1"),
AfterMultiAgentInvocationEvent(source=Mock()),
):
callbacks = list(registry.get_callbacks_for(event))
assert callbacks, f"No callbacks registered for {type(event).__name__}"
assert all(asyncio.iscoroutinefunction(cb) for cb in callbacks)

def test_async_mode_logs_sync_invocation_warning(self, mock_memory_client, caplog):
Expand All @@ -3746,15 +3750,18 @@ def test_async_mode_registers_bidi_agent_callbacks(self, mock_memory_client):
manager.register_hooks(registry)

# BidiAgentInitializedEvent dispatches via the sync hook path, so its callback must NOT be a coroutine.
init_callbacks = registry._registered_callbacks.get(BidiAgentInitializedEvent, [])
init_callbacks = list(registry.get_callbacks_for(BidiAgentInitializedEvent(agent=Mock())))
assert init_callbacks, "No callbacks registered for BidiAgentInitializedEvent"
assert not any(asyncio.iscoroutinefunction(cb) for cb in init_callbacks)

# BidiMessageAddedEvent and BidiAfterInvocationEvent dispatch via invoke_callbacks_async,
# so their callbacks should be async to keep the event loop unblocked.
for event_type in (BidiMessageAddedEvent, BidiAfterInvocationEvent):
callbacks = registry._registered_callbacks.get(event_type, [])
assert callbacks, f"No callbacks registered for {event_type.__name__}"
for event in (
BidiMessageAddedEvent(agent=Mock(), message={"role": "user", "content": [{"text": "x"}]}),
BidiAfterInvocationEvent(agent=Mock()),
):
callbacks = list(registry.get_callbacks_for(event))
assert callbacks, f"No callbacks registered for {type(event).__name__}"
assert all(asyncio.iscoroutinefunction(cb) for cb in callbacks)


Expand Down
Loading