From fa9e48b370b6b8ef15cf740d404a6b2edbbac351 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Thu, 16 Jul 2026 08:12:28 -0300 Subject: [PATCH 1/2] fix(doctor): stop three readiness probes leaking non-fatal errors to console (#351) - LLM judge: normalize the Foundry project OpenAI client to the /openai/v1/ route (clearing the injected api-version) so chat completions stop returning HTTP 404. Reuses the same credential/token refresh; non-Foundry endpoints are left unchanged. - OpenAI data-plane RBAC check: send the supported assignedTo('') ARM filter instead of the combined atScopeAndAbove()+assignedTo() form that ARM rejects with UnsupportedQuery. assignedTo alone already covers scope plus ancestor scopes. - App Insights probe: catch OSError (covers socket.timeout, which is not a URLError) and raise the request timeout from 10s to 30s, so a slow App Insights degrades quietly instead of printing a non-fatal INFO line. Adds unit coverage for the OpenAI client normalization, the corrected RBAC filter, and the App Insights timeout/URLError degrade-to-None path. Copilot-Session: af408082-0971-42ea-828c-8155d4110101 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 23 ++++++ .../agent/checks/_rbac_authorization.py | 13 +-- src/agentops/agent/llm_assist/_client.py | 52 +++++++++++- src/agentops/agent/sources/azure_monitor.py | 10 ++- ...est_agent_checks_rbac_openai_data_plane.py | 8 +- tests/unit/test_llm_assist.py | 82 +++++++++++++++++++ tests/unit/test_telemetry.py | 40 +++++++++ 7 files changed, 216 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73f11f91..91b8b10b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,29 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres ## [Unreleased] +### Fixed +- **`agentops doctor` no longer prints noisy non-fatal errors to the console.** + Three low-level probes that already degrade gracefully were leaking their + transport errors to the screen during readiness checks: + - The **LLM judge** called the Foundry project OpenAI client on the legacy + `/openai/` route, which returns HTTP 404 for chat completions, printing + `WARNING: llm_assist: judge call failed: Error code: 404`. The client is now + normalized to the stable `/openai/v1/` route (with the injected + `api-version` query parameter cleared) using the same reused credential and + token refresh, so the judge runs instead of 404ing. Non-Foundry endpoints + are left untouched. + - The **OpenAI data-plane RBAC check** built an ARM role-assignment filter + (`atScopeAndAbove() and assignedTo('')`) that ARM rejects with + `UnsupportedQuery`. It now sends the supported `assignedTo('')` filter, + which already returns assignments at the target scope and every ancestor + scope, so the check runs instead of skipping with a console error. + - The **Application Insights** REST probe caught only `urllib.error.URLError`, + so a read timeout (a `socket.timeout`, which is an `OSError` but not a + `URLError`) escaped and surfaced + `INFO: Rate-limit App Insights probe failed (non-fatal): The read operation + timed out`. It now catches `OSError` (covering both) and the request timeout + was raised from 10s to 30s, so a slow App Insights degrades quietly. + ## [0.8.0] - 2026-07-14 ### Added diff --git a/src/agentops/agent/checks/_rbac_authorization.py b/src/agentops/agent/checks/_rbac_authorization.py index 77a250fb..7c580d83 100644 --- a/src/agentops/agent/checks/_rbac_authorization.py +++ b/src/agentops/agent/checks/_rbac_authorization.py @@ -60,8 +60,13 @@ def list_principal_role_definition_ids( """List role definition GUIDs assigned to the principal at/above scope. Uses ``RoleAssignmentsOperations.list_for_scope`` with the - ``atScopeAndAbove() and assignedTo('')`` filter so management-plane - inheritance (subscription, resource group, account) is honoured. + ``assignedTo('')`` filter. That filter already returns assignments at + the given scope and every ancestor scope (subscription, resource group, + account), including those inherited through group membership, so + management-plane inheritance is honoured. The ARM RBAC API rejects the + combined ``atScopeAndAbove() and assignedTo(...)`` form + (``UnsupportedQuery``); ``assignedTo(...)`` on its own is the supported + equivalent. """ try: from azure.mgmt.authorization import AuthorizationManagementClient @@ -95,9 +100,7 @@ def list_principal_role_definition_ids( assignments = list( client.role_assignments.list_for_scope( scope=scope, - filter=( - f"atScopeAndAbove() and assignedTo('{principal_object_id}')" - ), + filter=f"assignedTo('{principal_object_id}')", ) ) except Exception as exc: # noqa: BLE001 diff --git a/src/agentops/agent/llm_assist/_client.py b/src/agentops/agent/llm_assist/_client.py index 7cb130d2..049cc724 100644 --- a/src/agentops/agent/llm_assist/_client.py +++ b/src/agentops/agent/llm_assist/_client.py @@ -39,6 +39,51 @@ class JudgementMeta: model_deployment: Optional[str] = None +def _normalize_foundry_openai_client(client: Any) -> Any: + """Point a Foundry project OpenAI client at the ``/openai/v1/`` route. + + ``AIProjectClient.get_openai_client()`` returns a client whose + ``base_url`` ends in ``/openai/`` and which injects an ``api-version`` + query parameter. On that legacy route Foundry chat completions return + HTTP 404. The stable surface is the OpenAI v1 route (``/openai/v1/``) + served *without* an ``api-version`` parameter. + + We rewrite the client in place with the public ``with_options`` API so + the reused credential and its token refresh keep working (no second + credential, no endpoint string-munging). The client is returned + unchanged when its ``base_url`` is not the recognised Foundry project + shape, so custom or already-normalised endpoints are left untouched. + """ + try: + base_url = str(getattr(client, "base_url", "") or "") + except Exception: # pragma: no cover - defensive + return client + if not base_url: + return client + + normalized = base_url if base_url.endswith("/") else base_url + "/" + if normalized.endswith("/openai/v1/"): + # Already on the v1 surface; only ensure api-version is dropped. + pass + elif normalized.endswith("/openai/"): + normalized = normalized + "v1/" + else: + # Unknown shape (non-Foundry route). Leave routing untouched. + return client + + with_options = getattr(client, "with_options", None) + if with_options is None: # pragma: no cover - defensive + return client + try: + return with_options( + base_url=normalized, + default_query={"api-version": None}, + ) + except Exception as exc: # pragma: no cover - defensive + log.info("llm_assist: could not normalize OpenAI route: %s", exc) + return client + + class LLMJudge: """Thin wrapper around the Foundry project's OpenAI client.""" @@ -230,6 +275,7 @@ def _get_client(self) -> Optional[Any]: if not endpoint: log.info("llm_assist: project endpoint not configured") return None + try: from azure.ai.projects import AIProjectClient # type: ignore from azure.identity import DefaultAzureCredential # type: ignore @@ -240,7 +286,9 @@ def _get_client(self) -> Optional[Any]: try: project_client = AIProjectClient( endpoint=endpoint, - credential=DefaultAzureCredential(exclude_developer_cli_credential=True, process_timeout=30), + credential=DefaultAzureCredential( + exclude_developer_cli_credential=True, process_timeout=30 + ), ) # Foundry exposes get_openai_client without an api_version arg; # never pass one (the SDK picks the right version). @@ -251,7 +299,7 @@ def _get_client(self) -> Optional[Any]: if get_openai_client is None: log.info("llm_assist: AIProjectClient has no OpenAI client helper") return None - self._client = get_openai_client() + self._client = _normalize_foundry_openai_client(get_openai_client()) except Exception as exc: # pragma: no cover log.warning("llm_assist: openai client init failed: %s", exc) return None diff --git a/src/agentops/agent/sources/azure_monitor.py b/src/agentops/agent/sources/azure_monitor.py index dcac5236..42185e30 100644 --- a/src/agentops/agent/sources/azure_monitor.py +++ b/src/agentops/agent/sources/azure_monitor.py @@ -13,7 +13,7 @@ import re from dataclasses import dataclass, field from typing import Any, Dict, List, Optional -from urllib import error, request +from urllib import request from agentops.agent.config import AzureMonitorSourceConfig @@ -504,9 +504,13 @@ def _query_application_insights( method="POST", ) try: - with request.urlopen(req, timeout=10) as resp: # noqa: S310 + with request.urlopen(req, timeout=30) as resp: # noqa: S310 parsed = json.loads(resp.read()) - except (error.URLError, ValueError, KeyError) as exc: + except (OSError, ValueError, KeyError) as exc: + # OSError covers urllib.error.URLError *and* socket.timeout / + # TimeoutError. A read timeout is not a URLError, so catching only + # URLError here would let it propagate to the caller and surface a + # noisy non-fatal INFO line on the console. Degrade quietly instead. log.debug("App Insights REST query failed: %s", exc) return None diff --git a/tests/unit/test_agent_checks_rbac_openai_data_plane.py b/tests/unit/test_agent_checks_rbac_openai_data_plane.py index 2df70288..3f9c2b5a 100644 --- a/tests/unit/test_agent_checks_rbac_openai_data_plane.py +++ b/tests/unit/test_agent_checks_rbac_openai_data_plane.py @@ -310,8 +310,12 @@ def test_list_role_definition_ids_extracts_guid_suffix() -> None: role_assignments.list_for_scope.assert_called_once() kwargs = role_assignments.list_for_scope.call_args.kwargs assert kwargs["scope"] == _ACCOUNT_TARGET - assert _PRINCIPAL_OID in kwargs["filter"] - assert "atScopeAndAbove" in kwargs["filter"] + # The ARM RBAC API rejects the combined + # ``atScopeAndAbove() and assignedTo(...)`` filter with UnsupportedQuery. + # ``assignedTo(...)`` alone is the supported equivalent and already + # returns assignments at scope and every ancestor scope. + assert kwargs["filter"] == f"assignedTo('{_PRINCIPAL_OID}')" + assert "atScopeAndAbove" not in kwargs["filter"] def test_list_role_definition_ids_raises_when_sdk_missing() -> None: diff --git a/tests/unit/test_llm_assist.py b/tests/unit/test_llm_assist.py index b7d4449a..61822ec3 100644 --- a/tests/unit/test_llm_assist.py +++ b/tests/unit/test_llm_assist.py @@ -481,3 +481,85 @@ def __init__(self, **kwargs): judge = LLMJudge(config=_enabled_config(), workspace=tmp_path) assert judge._get_client() == "legacy-openai-client" + + +# --------------------------------------------------------------------------- +# _normalize_foundry_openai_client -- Foundry /openai/v1/ route fix +# --------------------------------------------------------------------------- + + +class _FakeOpenAIClient: + """Minimal stand-in exposing ``base_url`` and ``with_options``.""" + + def __init__(self, base_url: str) -> None: + self.base_url = base_url + self.with_options_kwargs: Optional[Dict[str, Any]] = None + + def with_options(self, **kwargs: Any) -> "_FakeOpenAIClient": + self.with_options_kwargs = kwargs + new = _FakeOpenAIClient(kwargs.get("base_url", self.base_url)) + new.with_options_kwargs = kwargs + return new + + +def test_normalize_rewrites_openai_route_to_v1_and_drops_api_version() -> None: + from agentops.agent.llm_assist._client import ( + _normalize_foundry_openai_client, + ) + + client = _FakeOpenAIClient( + "https://acct.services.ai.azure.com/api/projects/p/openai/" + ) + normalized = _normalize_foundry_openai_client(client) + + assert normalized.with_options_kwargs is not None + assert ( + normalized.with_options_kwargs["base_url"] + == "https://acct.services.ai.azure.com/api/projects/p/openai/v1/" + ) + # The injected ``api-version`` query param must be cleared, otherwise the + # Foundry v1 route returns HTTP 404. + assert normalized.with_options_kwargs["default_query"] == { + "api-version": None + } + + +def test_normalize_leaves_v1_route_but_still_drops_api_version() -> None: + from agentops.agent.llm_assist._client import ( + _normalize_foundry_openai_client, + ) + + client = _FakeOpenAIClient( + "https://acct.services.ai.azure.com/api/projects/p/openai/v1/" + ) + normalized = _normalize_foundry_openai_client(client) + + assert ( + normalized.with_options_kwargs["base_url"] + == "https://acct.services.ai.azure.com/api/projects/p/openai/v1/" + ) + assert normalized.with_options_kwargs["default_query"] == { + "api-version": None + } + + +def test_normalize_leaves_unknown_route_untouched() -> None: + from agentops.agent.llm_assist._client import ( + _normalize_foundry_openai_client, + ) + + client = _FakeOpenAIClient("https://custom.example.com/v1/") + normalized = _normalize_foundry_openai_client(client) + + # Non-Foundry routes are returned as-is; with_options is never called. + assert normalized is client + assert client.with_options_kwargs is None + + +def test_normalize_ignores_client_without_base_url() -> None: + from agentops.agent.llm_assist._client import ( + _normalize_foundry_openai_client, + ) + + sentinel = object() + assert _normalize_foundry_openai_client(sentinel) is sentinel diff --git a/tests/unit/test_telemetry.py b/tests/unit/test_telemetry.py index eb6af6f4..926717e5 100644 --- a/tests/unit/test_telemetry.py +++ b/tests/unit/test_telemetry.py @@ -708,6 +708,46 @@ def test_azure_monitor_skipped_when_connection_string_lacks_application_id( assert payload.diagnostics["discovery_reason"] +def test_query_application_insights_returns_none_on_read_timeout( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A read timeout must degrade quietly, not raise. + + ``socket.timeout`` (aka ``TimeoutError``) is an ``OSError`` but NOT a + ``urllib.error.URLError``, so the previous narrow ``except URLError`` + let it escape and surface a noisy non-fatal INFO line on the console. + The fix catches ``OSError`` and returns ``None``. + """ + import socket + + def _raise_timeout(*_args, **_kwargs): + raise socket.timeout("The read operation timed out") + + monkeypatch.setattr(azure_monitor.request, "urlopen", _raise_timeout) + + assert ( + azure_monitor._query_application_insights("app-id", "bearer", "kql") + is None + ) + + +def test_query_application_insights_returns_none_on_urlerror( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Transport failures (URLError) also degrade quietly to None.""" + from urllib.error import URLError + + def _raise_urlerror(*_args, **_kwargs): + raise URLError("connection refused") + + monkeypatch.setattr(azure_monitor.request, "urlopen", _raise_urlerror) + + assert ( + azure_monitor._query_application_insights("app-id", "bearer", "kql") + is None + ) + + def _app_insights_result(row: dict[str, object]) -> dict[str, object]: return { "tables": [ From 49d413ae67c8bbb56445d1897c311a9a966dd60a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 16 Jul 2026 11:13:24 +0000 Subject: [PATCH 2/2] chore: prepare release 0.8.1 --- .claude-plugin/marketplace.json | 2 +- .github/plugin/marketplace.json | 2 +- CHANGELOG.md | 2 ++ plugins/agentops/package.json | 2 +- plugins/agentops/plugin.json | 2 +- 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 42857b3f..62ae65f6 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -13,7 +13,7 @@ "name": "agentops-accelerator", "source": "../../plugins/agentops", "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Toolkit and Microsoft Foundry agents.", - "version": "0.8.0", + "version": "0.8.1", "keywords": [ "agentops", "evaluation", diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json index 42857b3f..62ae65f6 100644 --- a/.github/plugin/marketplace.json +++ b/.github/plugin/marketplace.json @@ -13,7 +13,7 @@ "name": "agentops-accelerator", "source": "../../plugins/agentops", "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Toolkit and Microsoft Foundry agents.", - "version": "0.8.0", + "version": "0.8.1", "keywords": [ "agentops", "evaluation", diff --git a/CHANGELOG.md b/CHANGELOG.md index 91b8b10b..6b854cbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres ## [Unreleased] +## [0.8.1] - 2026-07-16 + ### Fixed - **`agentops doctor` no longer prints noisy non-fatal errors to the console.** Three low-level probes that already degrade gracefully were leaking their diff --git a/plugins/agentops/package.json b/plugins/agentops/package.json index 2174b236..299397c9 100644 --- a/plugins/agentops/package.json +++ b/plugins/agentops/package.json @@ -2,7 +2,7 @@ "name": "agentops-accelerator", "displayName": "AgentOps Accelerator — Skills for GitHub Copilot", "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Accelerator and Microsoft Foundry agents.", - "version": "0.8.0", + "version": "0.8.1", "publisher": "AgentOpsAccelerator", "icon": "icon.png", "license": "MIT", diff --git a/plugins/agentops/plugin.json b/plugins/agentops/plugin.json index c1b7466e..0643b5ee 100644 --- a/plugins/agentops/plugin.json +++ b/plugins/agentops/plugin.json @@ -1,7 +1,7 @@ { "name": "agentops-accelerator", "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Accelerator and Microsoft Foundry agents.", - "version": "0.8.0", + "version": "0.8.1", "author": { "name": "AgentOps Accelerator", "url": "https://github.com/Azure/agentops"