diff --git a/app/en/build/tool-calling/error-handling/page.mdx b/app/en/build/tool-calling/error-handling/page.mdx index 0b0a7dfd3..241212a96 100644 --- a/app/en/build/tool-calling/error-handling/page.mdx +++ b/app/en/build/tool-calling/error-handling/page.mdx @@ -277,6 +277,10 @@ As of now, MCP Clients do not return structured error information, only an error 3. **Include additional context**: For `RetryableToolError` and `ContextRequiredToolError`, use the `additional_prompt_content` parameter to guide the LLM or user. +## Debugging a failed tool call after the fact + +Arcade records every tool run, so you don't have to reproduce a failure to inspect it. Open [Tool executions](/operate/governance/tool-executions) to find the run, read each attempt's error, and reveal the exact inputs the tool received and the outputs it returned. + ## Building tools with error handling To learn more about how to build tools with error handling, see the [Build Tools](/build/create-tools/error-handling/useful-tool-errors) section. diff --git a/app/en/operate/governance/_meta.tsx b/app/en/operate/governance/_meta.tsx index 39237a99b..ab6501e39 100644 --- a/app/en/operate/governance/_meta.tsx +++ b/app/en/operate/governance/_meta.tsx @@ -13,6 +13,9 @@ export const meta: MetaRecord = { "audit-logs": { title: "Audit Logs", }, + "tool-executions": { + title: "Tool Executions", + }, }; export default meta; diff --git a/app/en/operate/governance/page.mdx b/app/en/operate/governance/page.mdx index 73a0180bb..c820d3831 100644 --- a/app/en/operate/governance/page.mdx +++ b/app/en/operate/governance/page.mdx @@ -1,13 +1,13 @@ --- title: "Governance" -description: "Federate tools, enforce access policies, and audit administrative actions" +description: "Federate tools, enforce access policies, and review administrative actions and tool runs" --- import { Cards } from "nextra/components"; # Governance -Control which tools agents can use, enforce policies at runtime, and review administrative actions. +Control which tools agents can use, enforce policies at runtime, and review administrative actions and the tool runs they produced. @@ -16,4 +16,8 @@ Control which tools agents can use, enforce policies at runtime, and review admi href="/operate/governance/contextual-access" /> + diff --git a/app/en/operate/governance/tool-executions/page.mdx b/app/en/operate/governance/tool-executions/page.mdx new file mode 100644 index 000000000..451973f4a --- /dev/null +++ b/app/en/operate/governance/tool-executions/page.mdx @@ -0,0 +1,179 @@ +--- +title: "Tool Executions" +description: "Review every tool run in a project: what ran, for whom, and why it failed, with tool inputs and outputs held to project admins." +--- + +# Tool executions + +Arcade records every tool run in a project, including which tool ran, for which end user, when, and how it went, and exposes that history in the dashboard and over the API. + +This page serves two jobs: + +- **Agent and tool developers** debug a run that misbehaved. The exact inputs a tool received and the outputs it returned show whether your agent sent the wrong arguments or the tool or its provider failed. +- **Platform operators** review what a project has run, for compliance and to see which tools and toolkits people actually use. + +Tool executions cover runtime activity. For administrative changes such as creating an API key or adding a project member, see [Audit Logs](/operate/governance/audit-logs). + +## What Arcade records + +Each execution records: + +- The tool and toolkit that ran, including the toolkit version +- Whether the run was **immediate** (on demand) or **scheduled** (queued to run later) +- The end user it ran for +- Its status, plus the times Arcade created, scheduled, started, and finished it +- One entry per **attempt**, with that attempt's success or failure, timing, and error + +Arcade also records the **inputs** a tool received and the **outputs** it returned. Both can be large and can carry credentials or end-user data, so Arcade withholds them by default and holds them to project admins. See [Who can see what](#who-can-see-what). + +## Find a run in the dashboard + +Open your project in the dashboard and select **Tool Executions**. The newest runs appear first. + +![The Tool Executions page in the Arcade dashboard, listing recent runs with status, tool, toolkit, type, user, duration, and start time](/images/screenshots/tool-executions-list.png) + +Narrow the list with these filters: + +| Filter | Behavior | +| -- | -- | +| Search tool name | Case-insensitive substring match on the tool name | +| Toolkit | Exact toolkit name, picked from your project's catalog | +| Outcome | All, success, or failure | +| Time range | A preset from the last hour to the last 90 days, or a custom start and end | +| User ID | The end user Arcade ran the tool for | +| Error contains | Case-insensitive match against the error an attempt reported when it failed | + +Every filter lives in the URL, so a filtered view survives a reload and you can paste it to a teammate. + +Select a row to open the execution details panel. It shows the execution ID, the tool and its toolkit version, the run type, the end user, the full timeline, the duration, and each attempt with its status and error. + +![The execution details panel showing tool metadata and timing, with tool inputs and outputs hidden behind toggles](/images/screenshots/tool-executions-detail-hidden.png) + +## Debug a tool that isn't working + +When a tool call goes wrong, work from the record rather than from your agent's logs. The record is what Arcade actually ran. + +1. Set **Outcome** to failure, and narrow by tool name or by the affected user. +2. Open the run and read the attempt list. Each attempt shows whether it succeeded, how long it took, and the error it reported. Retries that eventually succeed point at a transient provider problem. A run that fails the same way on every attempt points at the request itself. +3. Turn on **Show inputs** to see the exact arguments the tool received. +4. Turn on **Show outputs** to see what the tool returned, including the full error a provider sent back. + +![The execution details panel with both toggles on, showing a warning that sensitive data is visible over the tool input and output JSON](/images/screenshots/tool-executions-detail-revealed.png) + +Read the two together to place the fault: + +| What you see | Where to look | +| -- | -- | +| Inputs don't match what you meant to send | Your agent, prompt, or harness built the wrong call | +| Inputs look right, output carries a provider error | The upstream provider, its permissions, or its rate limits | +| Inputs look right, output is wrong or empty | The tool's own logic | +| The run needed authorization | The end user hasn't connected the account the tool needs | + +Reading inputs or outputs takes project-admin authority. If the toggles won't turn on, ask a project admin, or read the attempt errors, which stay available to every reader. + +## Review what a project has run + +For compliance work, the list itself is the answer: every tool run in the project, with the end user it ran for and the outcome. Filter by **User ID** to produce the history for one person, or set a time range to cover an audit period. Because filters live in the URL, a scoped view is something you can share or store alongside your findings. + +To see which tools people actually use, filter by tool or toolkit and read the total off the list, or ask the API for a count without pulling a page: + +```bash +# How many times the project ran a GoogleDocs tool in the last week +curl -s "https://api.arcade.dev/v1/orgs/{org_id}/projects/{project_id}/tool_executions/count?toolkit_name=GoogleDocs&start_time=2026-08-08T00:00:00Z" \ + -H "Authorization: Bearer $ARCADE_API_KEY" +``` + +```json +{ "count": 412 } +``` + +Both surfaces stay scoped to a single project, and neither includes tool inputs or outputs. + +## Who can see what + +Anyone holding a role on the project can read the history of what that project ran. The recorded inputs and outputs take project-admin authority. + +| | Project member | Project admin / org admin | +| -- | -- | -- | +| List what the project ran | ✅ | ✅ | +| Open a single run, with its tool, user, status, timing, and attempts | ✅ | ✅ | +| See why a run failed | ✅ | ✅ | +| See the inputs a tool received | ❌ | ✅ | +| See the outputs it returned | ❌ | ✅ | + +Two consequences are worth knowing. + +**Diagnostics are not payload.** The error an attempt reported when it failed, and the system error message behind a scheduler or infrastructure failure, reach every reader. A project member can diagnose a failure without anyone revealing the payload. Free-form detail that a tool chose to attach to its error, such as a developer message, model-facing guidance, extra fields, and the stack trace, travels with the payload instead, and Arcade withholds it alongside the payload. + +**Revealing is deliberate.** A project member gets toggles they can't switch on, rather than a permission error. When an admin turns one on, the panel warns that sensitive data is visible, since anyone watching the screen or a recording can read it. The toggle state never lands in the URL, so a shared or bookmarked link never reveals another person's tool inputs on open. + +## Read executions from the API + +The same history is available over the Arcade API. The base URL is `https://api.arcade.dev`, and requests authenticate with a bearer token. + +List the runs that failed, 25 at a time: + +```bash +curl -s "https://api.arcade.dev/v1/orgs/{org_id}/projects/{project_id}/tool_executions?outcome=failure&limit=25" \ + -H "Authorization: Bearer $ARCADE_API_KEY" +``` + +```json +{ + "items": [ + { + "id": "te_3HvzkMFaDOAfGWLtqJq0nhoD9Qd", + "created_at": "2026-08-14T20:11:13Z", + "updated_at": "2026-08-14T20:11:13Z", + "execution_type": "immediate", + "execution_status": "failed", + "tool_name": "GetDocumentById", + "toolkit_name": "GoogleDocs", + "toolkit_version": "8.1.0", + "user_id": "sterling@arcade.dev", + "started_at": "2026-08-14T20:02:14Z", + "finished_at": "2026-08-14T20:11:13Z" + } + ], + "limit": 25, + "offset": 0, + "page_count": 1, + "total_count": 137 +} +``` + +The newest results come first, and pages advance by offset. `limit` defaults to 100. The list accepts `tool_name`, `toolkit_name`, `user_id`, `outcome`, `failure_reason`, `start_time`, and `end_time`, the same filters the dashboard exposes. Both timestamps take RFC 3339. + +Open one run, and ask for the arguments and results it recorded: + +```bash +curl -s "https://api.arcade.dev/v1/orgs/{org_id}/projects/{project_id}/tool_executions/{id}?include_inputs=true&include_outputs=true" \ + -H "Authorization: Bearer $ARCADE_API_KEY" +``` + +The detail response adds `input` and an `attempts` array to the fields the list returns. Each attempt carries its ID, timing, success, any output, and a system error message. + +| Endpoint | Method | Description | +| -- | -- | -- | +| `/v1/orgs/{org_id}/projects/{project_id}/tool_executions` | GET | List the project's executions, newest first | +| `/v1/orgs/{org_id}/projects/{project_id}/tool_executions/count` | GET | Count the executions matching the same filters | +| `/v1/orgs/{org_id}/projects/{project_id}/tool_executions/{id}` | GET | Read one execution and its attempts | + +Two behaviors trip up API callers: + +- **The payload flags change the authority a request needs.** Omit `include_inputs` and `include_outputs` and the request needs only a role on the project. Add either one and the request needs project-admin authority, so the same caller can receive `200` without the flags and `403` with them. +- **These endpoints serve operators and project API keys.** Arcade refuses a read-only project API key, and refuses any caller whose authority comes from a single end user, such as an MCP client. + +Find your organization and project IDs in the dashboard URL: `/orgs/{org_id}/projects/{project_id}`. For the full request and response schemas, see the [API reference](/references/api). + +## Retention + +Execution logging runs by default, and each project keeps its history for a bounded retention window: 7 days unless your organization sets a different default or a project overrides it. Past the window, Arcade deletes the records and they stop appearing in the history and the API. + +To change the retention window, or to turn off execution logging and delete the history for your organization, contact Arcade support. + +## Related content + +- [Audit Logs](/operate/governance/audit-logs): administrative actions across the platform +- [Contextual Access](/operate/governance/contextual-access): policies that decide whether a tool call runs at all +- [API reference](/references/api): full schemas for every endpoint diff --git a/app/en/operate/page.mdx b/app/en/operate/page.mdx index dce3ab2d8..47f921b25 100644 --- a/app/en/operate/page.mdx +++ b/app/en/operate/page.mdx @@ -52,12 +52,13 @@ Connect your end-user identity provider and configure how gateways authenticate ### Governance -Federate tools, enforce access policies, and audit administrative actions. +Federate tools, enforce access policies, and review administrative actions and the tool runs they produced. + ## Related references diff --git a/public/images/screenshots/tool-executions-detail-hidden.png b/public/images/screenshots/tool-executions-detail-hidden.png new file mode 100644 index 000000000..d9232c8b7 Binary files /dev/null and b/public/images/screenshots/tool-executions-detail-hidden.png differ diff --git a/public/images/screenshots/tool-executions-detail-revealed.png b/public/images/screenshots/tool-executions-detail-revealed.png new file mode 100644 index 000000000..7ab3c53ad Binary files /dev/null and b/public/images/screenshots/tool-executions-detail-revealed.png differ diff --git a/public/images/screenshots/tool-executions-list.png b/public/images/screenshots/tool-executions-list.png new file mode 100644 index 000000000..3e3922379 Binary files /dev/null and b/public/images/screenshots/tool-executions-list.png differ diff --git a/public/llms.txt b/public/llms.txt index d03cbda24..78fa318c7 100644 --- a/public/llms.txt +++ b/public/llms.txt @@ -1,4 +1,4 @@ - + # Arcade @@ -157,6 +157,7 @@ Arcade docs serve two audiences. Start with the path that matches your goal: - [Stytch](https://docs.arcade.dev/en/operate/identity/user-sources/stytch): Documentation page - [The Arcade Registry](https://docs.arcade.dev/en/resources/registry-early-access): The Arcade Registry documentation provides an overview of a platform where developers can share and monetize their tools for agentic applications, similar to HuggingFace or Pypi. It explains how the registry integrates runtime metrics and user feedback to enhance tool development and usage - [Tool error handling](https://docs.arcade.dev/en/build/tool-calling/error-handling): Documentation page +- [Tool executions](https://docs.arcade.dev/en/operate/governance/tool-executions): Documentation page - [Tools](https://docs.arcade.dev/en/resources/tools): This documentation page provides an overview of Arcade's ecosystem for AI tools, enabling users to explore a catalog of pre-built integrations, create custom tools, and contribute their own tools to the community. It outlines the benefits of using Arcade tools, such as built - [Types of Tools](https://docs.arcade.dev/en/build/create-tools/improve/types-of-tools): Documentation page - [Understanding `Context` and tools](https://docs.arcade.dev/en/build/create-tools/tool-basics/runtime-data-access): Documentation page