From f7128fbd40c8b2c58081ff0166fc7578d8c2c6da Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 30 Jul 2026 23:05:33 +0000 Subject: [PATCH] fix(test): repair four unit tests drifting behind source and upstream APIs TestAsyncMode multi-agent/bidi callbacks: both tests read the private registry._registered_callbacks, which now holds strands' _CallbackEntry wrapper objects rather than raw functions, so iscoroutinefunction() was always False. Switched to the public registry.get_callbacks_for(event), matching the sibling async tests that already pass. This also makes the bidi init 'not any(iscoroutinefunction(...))' assertion meaningful -- it previously passed only because the wrapper is never a coroutine fn. EvaluatorOutput validator: the error message claimed 'Either label, value, or errorCode must be set', but the validator only accepts label or errorCode and never inspects value. Corrected the message to match the documented and implemented behaviour. test_evaluation_with_empty_trajectory: run_evaluations() returns a single EvaluationReport, not a list, so reports[0] raised TypeError. Dropped the index and fixed the same stale pattern in the README, which would have handed users the same TypeError. --- .../custom_code_based_evaluators/models.py | 3 +-- .../strands_agents_evals/README.md | 6 ++---- .../strands_agents_evals/test_end_to_end.py | 3 +-- .../test_agentcore_memory_session_manager.py | 21 ++++++++++++------- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/models.py b/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/models.py index 7d17b1cf..c876b145 100644 --- a/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/models.py +++ b/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/models.py @@ -82,7 +82,6 @@ class EvaluatorOutput(BaseModel): def _require_label_or_error_code(self) -> "EvaluatorOutput": if not self.errorCode and self.label is None: raise ValueError( - "Either label, value, or errorCode must be set; " - "set errorCode to return an error response without a label" + "label is required for success responses; set errorCode to return an error response without a label" ) return self diff --git a/src/bedrock_agentcore/evaluation/integrations/strands_agents_evals/README.md b/src/bedrock_agentcore/evaluation/integrations/strands_agents_evals/README.md index 3f41931c..adf67656 100644 --- a/src/bedrock_agentcore/evaluation/integrations/strands_agents_evals/README.md +++ b/src/bedrock_agentcore/evaluation/integrations/strands_agents_evals/README.md @@ -83,8 +83,7 @@ evaluator = create_strands_evaluator("Builtin.Helpfulness") # Run evaluations experiment = Experiment(cases=cases, evaluators=[evaluator]) -reports = experiment.run_evaluations(task_fn) -report = reports[0] +report = experiment.run_evaluations(task_fn) # View results print(f"Overall score: {report.overall_score:.2f}") @@ -175,8 +174,7 @@ def task_fn(case): evaluator = create_strands_evaluator("Builtin.Helpfulness") experiment = Experiment(cases=cases, evaluators=[evaluator]) -reports = experiment.run_evaluations(task_fn) -report = reports[0] +report = experiment.run_evaluations(task_fn) print(f"Overall score: {report.overall_score:.2f}") ``` diff --git a/tests/bedrock_agentcore/evaluation/integrations/strands_agents_evals/test_end_to_end.py b/tests/bedrock_agentcore/evaluation/integrations/strands_agents_evals/test_end_to_end.py index 1da6ceb6..d426cffa 100644 --- a/tests/bedrock_agentcore/evaluation/integrations/strands_agents_evals/test_end_to_end.py +++ b/tests/bedrock_agentcore/evaluation/integrations/strands_agents_evals/test_end_to_end.py @@ -65,8 +65,7 @@ def task_fn(case): with patch("boto3.client", return_value=mock_boto_client): evaluator = create_strands_evaluator("Builtin.Helpfulness") experiment = Experiment(cases=cases, evaluators=[evaluator]) - reports = experiment.run_evaluations(task_fn) - report = reports[0] + report = experiment.run_evaluations(task_fn) # Should return 0 score for empty trajectory assert report.overall_score == 0.0 diff --git a/tests/bedrock_agentcore/memory/integrations/strands/test_agentcore_memory_session_manager.py b/tests/bedrock_agentcore/memory/integrations/strands/test_agentcore_memory_session_manager.py index fa0c4787..b225cfa9 100644 --- a/tests/bedrock_agentcore/memory/integrations/strands/test_agentcore_memory_session_manager.py +++ b/tests/bedrock_agentcore/memory/integrations/strands/test_agentcore_memory_session_manager.py @@ -3722,9 +3722,13 @@ def test_async_mode_registers_multi_agent_callbacks(self, mock_memory_client): registry = HookRegistry() manager.register_hooks(registry) - for event_type in (MultiAgentInitializedEvent, AfterNodeCallEvent, AfterMultiAgentInvocationEvent): - callbacks = registry._registered_callbacks.get(event_type, []) - assert callbacks, f"No callbacks registered for {event_type.__name__}" + for event in ( + MultiAgentInitializedEvent(source=Mock()), + AfterNodeCallEvent(source=Mock(), node_id="n1"), + AfterMultiAgentInvocationEvent(source=Mock()), + ): + callbacks = list(registry.get_callbacks_for(event)) + assert callbacks, f"No callbacks registered for {type(event).__name__}" assert all(asyncio.iscoroutinefunction(cb) for cb in callbacks) def test_async_mode_logs_sync_invocation_warning(self, mock_memory_client, caplog): @@ -3746,15 +3750,18 @@ def test_async_mode_registers_bidi_agent_callbacks(self, mock_memory_client): manager.register_hooks(registry) # BidiAgentInitializedEvent dispatches via the sync hook path, so its callback must NOT be a coroutine. - init_callbacks = registry._registered_callbacks.get(BidiAgentInitializedEvent, []) + init_callbacks = list(registry.get_callbacks_for(BidiAgentInitializedEvent(agent=Mock()))) assert init_callbacks, "No callbacks registered for BidiAgentInitializedEvent" assert not any(asyncio.iscoroutinefunction(cb) for cb in init_callbacks) # BidiMessageAddedEvent and BidiAfterInvocationEvent dispatch via invoke_callbacks_async, # so their callbacks should be async to keep the event loop unblocked. - for event_type in (BidiMessageAddedEvent, BidiAfterInvocationEvent): - callbacks = registry._registered_callbacks.get(event_type, []) - assert callbacks, f"No callbacks registered for {event_type.__name__}" + for event in ( + BidiMessageAddedEvent(agent=Mock(), message={"role": "user", "content": [{"text": "x"}]}), + BidiAfterInvocationEvent(agent=Mock()), + ): + callbacks = list(registry.get_callbacks_for(event)) + assert callbacks, f"No callbacks registered for {type(event).__name__}" assert all(asyncio.iscoroutinefunction(cb) for cb in callbacks)