diff --git a/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/models.py b/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/models.py index 7d17b1cf..c876b145 100644 --- a/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/models.py +++ b/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/models.py @@ -82,7 +82,6 @@ class EvaluatorOutput(BaseModel): def _require_label_or_error_code(self) -> "EvaluatorOutput": if not self.errorCode and self.label is None: raise ValueError( - "Either label, value, or errorCode must be set; " - "set errorCode to return an error response without a label" + "label is required for success responses; set errorCode to return an error response without a label" ) return self diff --git a/src/bedrock_agentcore/evaluation/integrations/strands_agents_evals/README.md b/src/bedrock_agentcore/evaluation/integrations/strands_agents_evals/README.md index 3f41931c..adf67656 100644 --- a/src/bedrock_agentcore/evaluation/integrations/strands_agents_evals/README.md +++ b/src/bedrock_agentcore/evaluation/integrations/strands_agents_evals/README.md @@ -83,8 +83,7 @@ evaluator = create_strands_evaluator("Builtin.Helpfulness") # Run evaluations experiment = Experiment(cases=cases, evaluators=[evaluator]) -reports = experiment.run_evaluations(task_fn) -report = reports[0] +report = experiment.run_evaluations(task_fn) # View results print(f"Overall score: {report.overall_score:.2f}") @@ -175,8 +174,7 @@ def task_fn(case): evaluator = create_strands_evaluator("Builtin.Helpfulness") experiment = Experiment(cases=cases, evaluators=[evaluator]) -reports = experiment.run_evaluations(task_fn) -report = reports[0] +report = experiment.run_evaluations(task_fn) print(f"Overall score: {report.overall_score:.2f}") ``` diff --git a/tests/bedrock_agentcore/evaluation/integrations/strands_agents_evals/test_end_to_end.py b/tests/bedrock_agentcore/evaluation/integrations/strands_agents_evals/test_end_to_end.py index 1da6ceb6..d426cffa 100644 --- a/tests/bedrock_agentcore/evaluation/integrations/strands_agents_evals/test_end_to_end.py +++ b/tests/bedrock_agentcore/evaluation/integrations/strands_agents_evals/test_end_to_end.py @@ -65,8 +65,7 @@ def task_fn(case): with patch("boto3.client", return_value=mock_boto_client): evaluator = create_strands_evaluator("Builtin.Helpfulness") experiment = Experiment(cases=cases, evaluators=[evaluator]) - reports = experiment.run_evaluations(task_fn) - report = reports[0] + report = experiment.run_evaluations(task_fn) # Should return 0 score for empty trajectory assert report.overall_score == 0.0 diff --git a/tests/bedrock_agentcore/memory/integrations/strands/test_agentcore_memory_session_manager.py b/tests/bedrock_agentcore/memory/integrations/strands/test_agentcore_memory_session_manager.py index fa0c4787..b225cfa9 100644 --- a/tests/bedrock_agentcore/memory/integrations/strands/test_agentcore_memory_session_manager.py +++ b/tests/bedrock_agentcore/memory/integrations/strands/test_agentcore_memory_session_manager.py @@ -3722,9 +3722,13 @@ def test_async_mode_registers_multi_agent_callbacks(self, mock_memory_client): registry = HookRegistry() manager.register_hooks(registry) - for event_type in (MultiAgentInitializedEvent, AfterNodeCallEvent, AfterMultiAgentInvocationEvent): - callbacks = registry._registered_callbacks.get(event_type, []) - assert callbacks, f"No callbacks registered for {event_type.__name__}" + for event in ( + MultiAgentInitializedEvent(source=Mock()), + AfterNodeCallEvent(source=Mock(), node_id="n1"), + AfterMultiAgentInvocationEvent(source=Mock()), + ): + callbacks = list(registry.get_callbacks_for(event)) + assert callbacks, f"No callbacks registered for {type(event).__name__}" assert all(asyncio.iscoroutinefunction(cb) for cb in callbacks) def test_async_mode_logs_sync_invocation_warning(self, mock_memory_client, caplog): @@ -3746,15 +3750,18 @@ def test_async_mode_registers_bidi_agent_callbacks(self, mock_memory_client): manager.register_hooks(registry) # BidiAgentInitializedEvent dispatches via the sync hook path, so its callback must NOT be a coroutine. - init_callbacks = registry._registered_callbacks.get(BidiAgentInitializedEvent, []) + init_callbacks = list(registry.get_callbacks_for(BidiAgentInitializedEvent(agent=Mock()))) assert init_callbacks, "No callbacks registered for BidiAgentInitializedEvent" assert not any(asyncio.iscoroutinefunction(cb) for cb in init_callbacks) # BidiMessageAddedEvent and BidiAfterInvocationEvent dispatch via invoke_callbacks_async, # so their callbacks should be async to keep the event loop unblocked. - for event_type in (BidiMessageAddedEvent, BidiAfterInvocationEvent): - callbacks = registry._registered_callbacks.get(event_type, []) - assert callbacks, f"No callbacks registered for {event_type.__name__}" + for event in ( + BidiMessageAddedEvent(agent=Mock(), message={"role": "user", "content": [{"text": "x"}]}), + BidiAfterInvocationEvent(agent=Mock()), + ): + callbacks = list(registry.get_callbacks_for(event)) + assert callbacks, f"No callbacks registered for {type(event).__name__}" assert all(asyncio.iscoroutinefunction(cb) for cb in callbacks)