diff --git a/README.md b/README.md index ea0859d..c0fa7af 100644 --- a/README.md +++ b/README.md @@ -167,8 +167,12 @@ zenrows mcp config --client vscode # servers JSON zenrows mcp config --client generic --remote # hosted server at https://mcp.zenrows.com/mcp ``` -Local server: `npx -y @zenrows/mcp` (env `ZENROWS_API_KEY`). Remote server: -`https://mcp.zenrows.com/mcp`. +Local server: `npx -y @zenrows/mcp` (stdio; can auto-provision a Free plan key). +Remote server: `https://mcp.zenrows.com/mcp` (streamable-http; client OAuth). + +The portable Agent Plugins package at [`agent-plugin/`](agent-plugin/) declares +both servers in `mcp.json` (no secrets) and ships MCP-native skills under +`agent-plugin/skills/`. ## 12. Plugins @@ -177,6 +181,11 @@ zenrows plugin list zenrows plugin install claude-code # installs core skills + prints MCP config ``` +**Agent Plugins 1.0:** see [`agent-plugin/`](agent-plugin/) (`plugin.json` + +`mcp.json` + MCP-native skills). Legacy per-client snippets from +`zenrows plugin install` remain available. CLI-oriented skills stay under +repo-root `skills/`. + ## 13. Skills Agent-readable playbooks that teach agents how to choose primitives. diff --git a/agent-plugin/README.md b/agent-plugin/README.md new file mode 100644 index 0000000..2785032 --- /dev/null +++ b/agent-plugin/README.md @@ -0,0 +1,82 @@ +# Zenrows Agent Plugin + +Agent Plugins 1.0 package for [Zenrows](https://www.zenrows.com) — MCP servers + skills that teach agents to pick the cheapest reliable primitive for protected web data. + +```text +agent-plugin/ +├── plugin.json +├── mcp.json +├── README.md +└── skills/ + ├── zenrows/ + ├── cost-control/ + ├── protected-fetch/ + ├── extract/ + ├── batch-jobs/ + └── interact-browser/ +``` + +## Install + +Point your Agent Plugins–compatible client at this folder (or the `agent-plugin/` path in [`zenrows/cli`](https://github.com/zenrows/cli)). + +Clients discover: + +- `plugin.json` — manifest +- `mcp.json` — MCP servers (no secrets) +- `skills/*/SKILL.md` — playbooks + +## MCP servers (`mcp.json`) + +| Server | Transport | How auth works | +| --- | --- | --- | +| `zenrows` | stdio (`npx -y @zenrows/mcp`) | No key in the plugin. On first use the MCP server auto-provisions a Free plan account (unless disabled). Surface the claim URL when quota nears. | +| `zenrows-remote` | streamable-http `https://mcp.zenrows.com/mcp` | Client OAuth (Login or Create Free). No Bearer/API key in the plugin package. | + +This package never embeds API keys. Secrets stay in the client / local MCP runtime. + +## Skills (MCP-native) + +Use MCP tool names — not CLI `zenrows fetch`: + +| Skill | Primary tools | +| --- | --- | +| `zenrows` | Decision tree across all primitives | +| `protected-fetch` | `scrape` | +| `extract` | `extract` (fallback: `scrape` + autoparse / css_extractor) | +| `batch-jobs` | `batch_create`, `batch_status`, `batch_results`, `batch_cancel` | +| `interact-browser` | `browser_*` (escalation only) | +| `cost-control` | Cost multipliers + escalate-with-evidence | + +## Local MCP testing (before npm publish) + +Until `@zenrows/mcp` with extract/batch/auto-signup is published, point stdio at a local build: + +```bash +cd /path/to/zenrows-mcp +npm run build +# Inspector UI: +npm run inspect +# Or stdio with an existing key: +ZENROWS_API_KEY=… node dist/index.js +# Or local Streamable HTTP (Bearer still required — no anonymous remote signup): +ZENROWS_API_KEY=… npm run dev:http +``` + +In a client `mcp.json` / Cursor config, use: + +```json +{ + "command": "node", + "args": ["/absolute/path/to/zenrows-mcp/dist/index.js"], + "env": { "ZENROWS_API_KEY": "YOUR_KEY" } +} +``` + +Omit `ZENROWS_API_KEY` to exercise stdio auto-signup into `~/.zenrows/`. + +## Related + +- CLI product + installable CLI skills: repo root `skills/` +- MCP runtime: [`@zenrows/mcp`](https://www.npmjs.com/package/@zenrows/mcp) +- Spec: [Agent Plugins 1.0](https://agent-plugins.org/) diff --git a/agent-plugin/mcp.json b/agent-plugin/mcp.json new file mode 100644 index 0000000..ca0b07b --- /dev/null +++ b/agent-plugin/mcp.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://agent-plugins.org/schemas/1.0.0/mcp.schema.json", + "mcpServers": { + "zenrows": { + "type": "stdio", + "command": "npx", + "args": ["-y", "@zenrows/mcp"] + }, + "zenrows-remote": { + "type": "streamable-http", + "url": "https://mcp.zenrows.com/mcp" + } + } +} diff --git a/agent-plugin/plugin.json b/agent-plugin/plugin.json new file mode 100644 index 0000000..351af0c --- /dev/null +++ b/agent-plugin/plugin.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", + "name": "zenrows", + "version": "0.1.0", + "description": "Protected web data for AI agents via Zenrows MCP — scrape, extract, batch, and browser sessions with cost-aware escalation.", + "author": { + "name": "Zenrows", + "url": "https://www.zenrows.com" + }, + "homepage": "https://www.zenrows.com", + "repository": "https://github.com/zenrows/cli", + "license": "MIT", + "keywords": [ + "zenrows", + "mcp", + "scraping", + "extract", + "batch", + "browser", + "anti-bot", + "web-data" + ] +} diff --git a/agent-plugin/skills/batch-jobs/SKILL.md b/agent-plugin/skills/batch-jobs/SKILL.md new file mode 100644 index 0000000..daab22c --- /dev/null +++ b/agent-plugin/skills/batch-jobs/SKILL.md @@ -0,0 +1,47 @@ +--- +name: batch-jobs +description: Scale protected fetch/extract over many URLs via Zenrows MCP batch_* tools (beta). +version: 0.1.0 +--- + +# Batch Jobs (MCP) + +Process large workloads asynchronously. Batch is where Zenrows' high-scale +anti-bot advantage becomes obvious — thousands to millions of URLs. + +> Status: **beta**. Cloud Batch lives on `async.api.zenrows.com/v1`. Without +> beta access the API returns `BATCH_ACCESS_DENIED`. Do not fake a successful +> run. This is **not** `browser_batch` (that batches browser actions in one +> session). + +## MCP tools + +| Tool | Purpose | +| --- | --- | +| `batch_create` | Submit a job (list of URL tasks + optional scrape/extract params) | +| `batch_status` | Poll run status + progress stats | +| `batch_results` | Page results (cursor pagination; filter successful/failed/all) | +| `batch_cancel` | Stop an in-flight run | + +Optional if exposed by the server: `batch_wait` to block until terminal status. + +## Workflow + +1. Validate the workflow on **one** URL with `scrape` or `extract`. +2. `batch_create` with the full task list (each task: `url` + optional + overrides like `js_render`, `premium_proxy`, `proxy_country`, `mode`, + `autoparse`, `external_id`, `metadata`). +3. Poll `batch_status` until terminal (`completed` / `stopped` / …). +4. Collect with `batch_results`. +5. `batch_cancel` if the user aborts or the run should stop early. + +## Rules + +- **Never batch before validating** on a single page ([[extract]] / + [[protected-fetch]]). +- Do **not** fan out hundreds of `scrape`/`extract` calls when Batch is the + right primitive. +- Mind cost multipliers ([[cost-control]]). +- On Free-plan quota errors, surface the claim URL. +- Prefer these MCP tools over CLI `zenrows batch` when the MCP server is + connected. diff --git a/agent-plugin/skills/cost-control/SKILL.md b/agent-plugin/skills/cost-control/SKILL.md new file mode 100644 index 0000000..d74488d --- /dev/null +++ b/agent-plugin/skills/cost-control/SKILL.md @@ -0,0 +1,48 @@ +--- +name: cost-control +description: Pick the cheapest reliable Zenrows MCP configuration and understand cost multipliers. +version: 0.1.0 +--- + +# Cost control (MCP) + +Prefer the **smallest reliable** configuration for stealth flags; escalate only +with evidence. Do not assume `extract` is cheaper than `scrape`. + +## Cost multipliers (stealth / scrape options) + +These multipliers apply when you enable the flags (on `scrape` or `extract`): + +- Basic request: **1×** +- JS rendering (`js_render`): **5×** +- Premium proxies (`premium_proxy`): **10×** +- Both: **25×** + +`extract` `mode=auto` is **open beta** and currently free; billing may apply +later. Do not promise permanent free extract pricing. Autoparse / CSS still +follow the multipliers above when stealth flags are set. + +## Browser sessions (`browser_*`) + +Bill by **bandwidth + session time**. Sessions auto-terminate after ~15 minutes. +Always call `browser_close`. Prefer `scrape` / `extract` when they can do the job +(by output shape — not because extract is “cheaper”). + +## Batch + +Batch is usually better at **scale** (many URLs) than fan-out of per-URL tool +calls. Validate one URL with `scrape`/`extract` first, then `batch_create`. +Batch access may be beta-gated (`BATCH_ACCESS_DENIED`). + +## Tactics + +- Default to the smallest option set that works; add `js_render` / + `premium_proxy` only after a failed or empty result (or clear SPA/anti-bot + signals). +- Reduce payload size: `extract` / `css_extractor` / `outputs` / markdown + instead of full HTML. +- Validate on one page before `batch_create`. +- On Free-plan quota errors, surface the **claim URL** — do not invent billing + flows. +- Prefer MCP tools from this plugin; do not shell out to `zenrows fetch` as + the primary path. diff --git a/agent-plugin/skills/extract/SKILL.md b/agent-plugin/skills/extract/SKILL.md new file mode 100644 index 0000000..c4567b5 --- /dev/null +++ b/agent-plugin/skills/extract/SKILL.md @@ -0,0 +1,60 @@ +--- +name: extract +description: Turn protected pages into structured data with the Zenrows MCP extract tool (and scrape extraction params). +version: 0.1.0 +--- + +# Extract (MCP) + +Convert protected pages into structured data. Value is **protected page access ++ extraction**, not generic LLM parsing. + +## Primary tool: `extract` + +Use **`extract`** when you need structured JSON fields rather than a full page: + +``` +extract({ url }) # mode=auto (default) +extract({ url, mode: "autoparse" }) # general Autoparse (any domain) +extract({ url, mode: "css", css_extractor: '{"title":"h1","price":".price"}' }) +extract({ url, js_render: true, premium_proxy: true }) # stealth flags when needed +``` + +`mode=auto` (`extract=auto`) is **open beta**: richest on enabled domains, +currently free; billing may apply later. If the domain is not enabled (AUTH010), +the tool retries once with Autoparse by default (`fallback_autoparse`). + +## Fallback: `scrape` extraction params + +If `extract` is unavailable, similar outcomes are available on **`scrape`**: + +``` +scrape({ url, autoparse: true }) +scrape({ url, css_extractor: '{"title":"h1","price":".price"}' }) +scrape({ url, outputs: "emails,links" }) # or "*" for all built-ins +``` + +`outputs` types: `emails`, `headings`, `links`, `menus`, `images`, `videos`, +`audios`. + +## When to use which + +- **`mode=auto`**: site-tailored fields on beta-enabled domains. +- **`mode=autoparse`**: any website; also the automatic AUTH010 fallback. +- **`mode=css`**: known fields/selectors; deterministic. +- **Markdown via `scrape`**: feed clean content to an LLM yourself. + +Do **not** assume extract is cheaper than scrape — pick by output shape. +Stealth flags (`js_render`, `premium_proxy`) still apply the usual multipliers +([[cost-control]]). + +## Rules + +- Validate on a single page before `batch_create` ([[batch-jobs]]). +- If Autoparse misses fields, switch to `css_extractor`, or add `js_render` + for JS-heavy pages. +- Prefer MCP `extract` / `scrape` over CLI `zenrows extract` when the MCP + server is connected. + +See [[protected-fetch]] for retrieval-only semantics and [[cost-control]] for +multipliers. diff --git a/agent-plugin/skills/interact-browser/SKILL.md b/agent-plugin/skills/interact-browser/SKILL.md new file mode 100644 index 0000000..7c1c164 --- /dev/null +++ b/agent-plugin/skills/interact-browser/SKILL.md @@ -0,0 +1,45 @@ +--- +name: interact-browser +description: Escalate to Zenrows MCP browser_* sessions only when scrape/extract cannot do the job. +version: 0.1.0 +--- + +# Interact / Browser Sessions (MCP `browser_*`) + +Operate protected browser workflows (logins, clicks, forms, multi-step flows, +persistent cookies) on JS-heavy or interactive pages that `scrape` / `extract` +cannot handle. **Escalation only.** + +> Browser Sessions bill by **bandwidth + session time** and auto-terminate +> after ~15 minutes. Same backend as CLI Browser Sessions / hosted MCP. + +## Core flow + +1. `browser_navigate` → get `session_id` +2. Inspect with `browser_get_accessibility_tree` / `browser_get_text` / + `browser_screenshot` +3. Act: `browser_click`, `browser_fill`, `browser_type`, `browser_select_option`, + `browser_check` / `browser_uncheck`, `browser_press_key`, `browser_scroll`, + `browser_drag`, … +4. Wait: `browser_wait_for_selector`, `browser_wait_for_navigation`, `browser_wait` +5. Multi-step known sequences: `browser_batch` (actions against an existing + session — **not** URL Batch / `batch_create`) +6. Always `browser_close` when done + +Other useful tools: `browser_evaluate`, cookies (`browser_get_cookies` / +`browser_set_cookies` / `browser_clear_cookies`), `browser_local_storage`, +tabs (`browser_new_tab` / `browser_switch_tab`), `browser_generate_pdf`. + +## Rules + +- Try [[protected-fetch]] (`scrape`) / [[extract]] first; escalate only with + evidence — they cost less. +- Pass `session_id` on every call after navigate. +- Prefer `browser_batch` when you already know the full action sequence + (fewer round trips). +- Always `browser_close` interactive sessions. +- `browser_select_option` matches the option's `value` attribute (bare value + or a CSS selector like `option[value="2"]`). Label text / index are not + supported. +- Do not use CLI `zenrows browser` as the primary path when MCP `browser_*` + tools are available. diff --git a/agent-plugin/skills/protected-fetch/SKILL.md b/agent-plugin/skills/protected-fetch/SKILL.md new file mode 100644 index 0000000..c6cc969 --- /dev/null +++ b/agent-plugin/skills/protected-fetch/SKILL.md @@ -0,0 +1,42 @@ +--- +name: protected-fetch +description: Use the Zenrows MCP scrape tool for anti-bot-protected page retrieval. +version: 0.1.0 +--- + +# Protected Fetch (MCP `scrape`) + +Retrieve pages that normal fetch, generic scrapers, or naive browser tools +cannot. Primary MCP tool: **`scrape`**. + +## When to use + +- Known URL, want page content (markdown, HTML, text, PDF, or screenshot). +- Target has anti-bot protection, needs JS rendering, or geo-specific access. + +## How to call + +``` +scrape({ url }) +scrape({ url, response_type: "markdown" }) # default — ideal for LLMs +scrape({ url, js_render: true }) # SPAs / dynamic content +scrape({ url, js_render: true, premium_proxy: true }) # heavy anti-bot +scrape({ url, premium_proxy: true, proxy_country: "us" }) +scrape({ url, js_render: true, wait_for: ".price" }) +``` + +## Rules + +- Start minimal. Enable `js_render` when content is clearly JS-loaded or the + first result is empty/incomplete. Enable `premium_proxy` when you see + 403/blocked even with JS rendering. +- `proxy_country` requires `premium_proxy`. +- Prefer `response_type: "markdown"` for LLM consumption; use `html` only when + you need raw markup. +- For **structured** fields, prefer the `extract` tool (or `scrape` with + `autoparse` / `css_extractor` / `outputs`) — see [[extract]]. +- On failure, escalate with evidence before `browser_*` ([[interact-browser]]). +- Cost model: [[cost-control]]. + +Do **not** treat CLI `zenrows fetch` as the primary interface when this MCP +server is available. diff --git a/agent-plugin/skills/zenrows/SKILL.md b/agent-plugin/skills/zenrows/SKILL.md new file mode 100644 index 0000000..f33f994 --- /dev/null +++ b/agent-plugin/skills/zenrows/SKILL.md @@ -0,0 +1,80 @@ +--- +name: zenrows +description: Master playbook for choosing the right Zenrows MCP primitive for any web-data task, with cost and escalation rules. +version: 0.1.0 +--- + +# Zenrows (MCP): how to think about protected web data + +Zenrows is the **protected web data infrastructure layer**: strong anti-bot + +high scale + extraction + browser escalation. This plugin exposes that +infrastructure through the Zenrows MCP server. + +Your job as an agent is to pick a **reliable MCP tool** for the task (match +output shape; escalate only with evidence). Prefer MCP tools over shelling +out to the Zenrows CLI. For stealth flags, start simple — see [[cost-control]]. + +## Getting access (zero-config) + +You do **not** need an API key in the plugin package. + +- **stdio** (`npx -y @zenrows/mcp`): on first cloud call the server can + auto-create a **free, unclaimed** Zenrows Free plan account and persist the + key locally. When the Free plan nears its limit (or the user asks), surface + the **claim URL** so a human can claim the account (email + password). The + same key keeps working after claim. Never handle payment or upgrades — + only show the claim link. +- **Remote** (`https://mcp.zenrows.com/mcp`): the client runs **OAuth** + (Login or Create Free account → consent → token). Do not invent a parallel + auth flow and do not ask the user to paste keys into `mcp.json`. + +## The decision tree (follow this exactly) + +``` +If the user has a known URL and wants page content: + → Use Protected Fetch. (MCP tool: scrape) + +If the user has a known URL and wants structured data: + → Use Extract. (MCP tool: extract) + Fallback: scrape with autoparse / css_extractor / outputs + +If the user has many URLs (bulk): + → Use Batch. (validate one page with scrape/extract first, + then batch_create → batch_status → batch_results) + Prefer Batch over fan-out of scrape/extract + per URL for bulk work (may be beta-gated). + Cancel with batch_cancel when needed. + +If the user needs login, clicks, forms, sessions, or persistent state: + → Use Interact / Browser Sessions.(MCP tools: browser_*) [escalation-only] +``` + +## Preference rules + +- Match the tool to the output: `scrape` for page content, `extract` for + structured fields. Start simple; add `js_render` / `premium_proxy` only with + evidence of failure or dynamic/protected content. +- **Escalate only with evidence.** If `scrape`/`extract` fails or returns + empty/blocked content, diagnose before jumping to `browser_*`. +- **Do not use the browser unless required.** Browser bills by bandwidth + + session time and auto-terminates after ~15 minutes. Always `browser_close` + when done. +- **Do not scale before validating** the workflow on a small sample. + +## MCP tools available + +| Primitive | MCP tools | Notes | +| --- | --- | --- | +| Protected Fetch | `scrape` | Default markdown; HTML / plaintext / PDF / screenshot options | +| Extract | `extract` | Structured JSON (`extract=auto`, autoparse, css_extractor). `extract=auto` open beta (currently free; billing may apply later) | +| Batch | `batch_create`, `batch_status`, `batch_results`, `batch_cancel` | Beta; may return `BATCH_ACCESS_DENIED` without access | +| Browser | `browser_*` (start with `browser_navigate`) | Escalation-only; same backend as Browser Sessions | + +## Cost notes + +Stealth multipliers on requests: basic **1×**, JS rendering **5×**, premium +proxies **10×**, both **25×**. Keep responses small with `css_extractor`, +`outputs`, or `response_type: markdown`. Do not claim extract is cheaper than +scrape. See [[cost-control]]. + +See also: [[protected-fetch]], [[extract]], [[batch-jobs]], [[interact-browser]]. diff --git a/tests/agent-plugin.test.ts b/tests/agent-plugin.test.ts new file mode 100644 index 0000000..40c3e6d --- /dev/null +++ b/tests/agent-plugin.test.ts @@ -0,0 +1,67 @@ +import assert from "node:assert/strict"; +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { test } from "node:test"; + +const root = join(process.cwd(), "agent-plugin"); + +test("agent-plugin plugin.json is Agent Plugins 1.0", () => { + const plugin = JSON.parse(readFileSync(join(root, "plugin.json"), "utf8")) as Record; + assert.equal(plugin.$schema, "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json"); + assert.equal(plugin.name, "zenrows"); + assert.ok(typeof plugin.version === "string" && plugin.version.length > 0); + // closed schema: no unknown top-level keys beyond the known set + const allowed = new Set([ + "$schema", + "name", + "version", + "description", + "author", + "homepage", + "repository", + "license", + "keywords", + "extensions", + ]); + for (const key of Object.keys(plugin)) { + assert.ok(allowed.has(key), `unexpected plugin.json field: ${key}`); + } +}); + +test("agent-plugin mcp.json declares stdio and streamable-http without secrets", () => { + const mcp = JSON.parse(readFileSync(join(root, "mcp.json"), "utf8")) as { + $schema: string; + mcpServers: Record>; + }; + assert.equal(mcp.$schema, "https://agent-plugins.org/schemas/1.0.0/mcp.schema.json"); + const servers = mcp.mcpServers; + assert.ok(servers); + + const stdio = Object.values(servers).find((s) => s.type === "stdio"); + assert.ok(stdio, "expected a stdio server"); + assert.equal(stdio.command, "npx"); + assert.deepEqual(stdio.args, ["-y", "@zenrows/mcp"]); + assert.equal(stdio.env, undefined, "must not embed API keys in env"); + + const http = Object.values(servers).find((s) => s.type === "streamable-http"); + assert.ok(http, "expected a streamable-http server"); + assert.equal(http.url, "https://mcp.zenrows.com/mcp"); + assert.equal(http.headers, undefined, "must not embed secrets in headers"); +}); + +test("agent-plugin skills are MCP-native (not CLI-primary)", () => { + const skillsDir = join(root, "skills"); + const names = readdirSync(skillsDir).filter((n) => existsSync(join(skillsDir, n, "SKILL.md"))); + assert.ok(names.includes("zenrows")); + assert.ok(names.includes("extract")); + assert.ok(names.includes("batch-jobs")); + + const master = readFileSync(join(skillsDir, "zenrows", "SKILL.md"), "utf8"); + assert.match(master, /\bscrape\b/); + assert.match(master, /\bextract\b/); + assert.match(master, /batch_create|batch_\*/); + assert.match(master, /claim/i); + assert.match(master, /OAuth|stdio/i); + // should not instruct CLI as the primary path + assert.doesNotMatch(master, /→ Use Protected Fetch\.\s+\(zenrows fetch/); +});