diff --git a/README.md b/README.md index afe89cd..d0e64d9 100644 --- a/README.md +++ b/README.md @@ -63,19 +63,22 @@ metadata and are never recursively removed or rewritten by the CLI. ## Run idempotency -Use an explicit idempotency key when a run may need to be repeated without another charge: +The CLI generates a fresh random idempotency key for every logical invocation. Use an explicit +key when a run may need to be resumed manually without another charge: ```sh anyapi run reddit.search --input '{"query":"anyapi"}' --idempotency-key k1 ``` -Repeating the same request with the same key returns the original result without another charge once the gateway supports idempotency. Keys must be 1 to 255 visible ASCII characters. The CLI does not generate a key unless you explicitly opt in with `auto`: +Repeating the same request with the same key returns the original result without another charge. +Keys must be 1 to 255 visible ASCII characters. You can explicitly request another random key with +`auto`: ```sh anyapi run reddit.search --input '{"query":"anyapi"}' --idempotency-key auto ``` -`auto` derives a deterministic key from the SKU, canonical JSON input, and current UTC date. Equivalent JSON formatting and property order produce the same key and request body during that day. Use `auto` to protect against accidental same-day reruns, but omit it when repeated runs are intentional. +`auto` creates a random key for that invocation. It does not deduplicate a separate CLI command. If a key is already running, the CLI asks you to retry shortly with the same key. If a key was used with a different SKU or input, the CLI asks you to use a new key or retry the original request. diff --git a/__tests__/run.test.ts b/__tests__/run.test.ts index 159074d..a90cec2 100644 --- a/__tests__/run.test.ts +++ b/__tests__/run.test.ts @@ -1,40 +1,48 @@ -import { join } from 'node:path'; -import { PassThrough } from 'node:stream'; -import { describe, expect, it } from 'vitest'; -import { AnyApiClient } from '../src/api.js'; -import { runCommand } from '../src/commands.js'; -import { ApiError } from '../src/errors.js'; +import { join } from "node:path"; +import { PassThrough } from "node:stream"; +import { Command } from "commander"; +import { describe, expect, it } from "vitest"; +import { AnyApiClient } from "../src/api.js"; +import { runCommand } from "../src/commands.js"; +import { ApiError } from "../src/errors.js"; import { buildRunOutputPath, formatIdempotencyError, formatTrialCapMessage, isTrialCapReached, + normalizeRunCLIOptions, parseRunInput, prepareRunIdempotency, -} from '../src/run.js'; -import type { CommandContext } from '../src/io.js'; -import type { FetchLike } from '../src/types.js'; +} from "../src/run.js"; +import type { CommandContext } from "../src/io.js"; +import type { FetchLike } from "../src/types.js"; -describe('run idempotency', () => { - it('preserves customer output fields without recursive rewriting', async () => { +describe("run idempotency", () => { + it("maps Commander negated --no-wait into the transport option", () => { + const command = new Command().exitOverride().option("--no-wait"); + command.parse(["node", "test", "--no-wait"]); + expect(normalizeRunCLIOptions(command.opts()).noWait).toBe(true); + }); + + it("preserves customer output fields without recursive rewriting", async () => { const responseBody = { output: { found: true, data: { creditScore: 812, - provider: 'source named by the customer API', - providers: ['first source', 'second source'], - nested: { provider: { name: 'structured provider value' } }, + provider: "source named by the customer API", + providers: ["first source", "second source"], + nested: { provider: { name: "structured provider value" } }, }, }, - provider: 'AnyAPI', + provider: "AnyAPI", costUsd: 0.01, items: 1, }; const ctx = commandContext(async () => Response.json(responseBody)); - await runCommand(ctx, { apiKey: 'aa_live_test' }, 'finance.profile', { - input: '{}', + await runCommand(ctx, { apiKey: "aa_live_test" }, "finance.profile", { + input: "{}", json: true, }); @@ -42,83 +50,127 @@ describe('run idempotency', () => { expect(JSON.parse(stdout)).toEqual(responseBody); }); - it('preserves balance response fields without recursive rewriting', async () => { + it("preserves balance response fields without recursive rewriting", async () => { const responseBody = { balanceUsd: 1.25, creditScore: 812, - provider: 'account-data-source', - providers: ['account-data-source'], + provider: "account-data-source", + providers: ["account-data-source"], }; const client = new AnyApiClient({ - apiKey: 'aa_live_test', + apiKey: "aa_live_test", fetchImpl: async () => Response.json(responseBody), - restBaseUrl: 'https://example.test/v1', + restBaseUrl: "https://example.test/v1", }); await expect(client.balance()).resolves.toEqual(responseBody); }); - it('passes the command flag through to the idempotency key header', async () => { + it("passes the command flag through to the idempotency key header", async () => { let requestInit: RequestInit | undefined; const fetchImpl: FetchLike = async (_input, init) => { requestInit = init; - return Response.json({ output: {}, provider: 'AnyAPI', costUsd: 0.01, items: 1 }); + return Response.json({ + output: {}, + provider: "AnyAPI", + costUsd: 0.01, + items: 1, + }); }; - await runCommand(commandContext(fetchImpl), { apiKey: 'aa_live_test' }, 'reddit.search', { - input: '{"query":"anyapi"}', - idempotencyKey: 'k1', - json: true, - }); + await runCommand( + commandContext(fetchImpl), + { apiKey: "aa_live_test" }, + "reddit.search", + { + input: '{"query":"anyapi"}', + idempotencyKey: "k1", + json: true, + }, + ); - expect(new Headers(requestInit?.headers).get('Idempotency-Key')).toBe('k1'); + expect(new Headers(requestInit?.headers).get("Idempotency-Key")).toBe("k1"); }); - it('omits the idempotency key header when the flag is absent', async () => { + it("generates a fresh idempotency key when the flag is absent", async () => { let requestInit: RequestInit | undefined; const fetchImpl: FetchLike = async (_input, init) => { requestInit = init; - return Response.json({ output: {}, provider: 'AnyAPI', costUsd: 0.01, items: 1 }); + return Response.json({ + output: {}, + provider: "AnyAPI", + costUsd: 0.01, + items: 1, + }); }; - await runCommand(commandContext(fetchImpl), { apiKey: 'aa_live_test' }, 'reddit.search', { - input: '{"query":"anyapi"}', - json: true, - }); + await runCommand( + commandContext(fetchImpl), + { apiKey: "aa_live_test" }, + "reddit.search", + { + input: '{"query":"anyapi"}', + json: true, + }, + ); - expect(new Headers(requestInit?.headers).has('Idempotency-Key')).toBe(false); + expect(new Headers(requestInit?.headers).get("Idempotency-Key")).toMatch( + /^anyapi-auto-/, + ); }); - it('derives the same auto key and request body from equivalent JSON input', async () => { + it("derives a fresh auto key while keeping equivalent request bodies stable", async () => { const firstInput = await parseRunInput({ input: '{"query":"anyapi","filters":{"sort":"new","limit":5}}', }); const secondInput = await parseRunInput({ input: '{ "filters": { "limit": 5, "sort": "new" }, "query": "anyapi" }', }); - const date = new Date('2026-07-27T12:00:00Z'); + const date = new Date("2026-07-27T12:00:00Z"); - const first = prepareRunIdempotency('reddit.search', firstInput, 'auto', date); - const second = prepareRunIdempotency('reddit.search', secondInput, 'auto', date); + const first = prepareRunIdempotency( + "reddit.search", + firstInput, + "auto", + date, + ); + const second = prepareRunIdempotency( + "reddit.search", + secondInput, + "auto", + date, + ); - expect(second.idempotencyKey).toBe(first.idempotencyKey); + expect(second.idempotencyKey).not.toBe(first.idempotencyKey); expect(JSON.stringify(second.input)).toBe(JSON.stringify(first.input)); - expect(first.idempotencyKey).toMatch(/^anyapi-auto-[a-f0-9]{64}$/); + expect(first.idempotencyKey).toMatch(/^anyapi-auto-[a-f0-9-]{36}$/); + expect(second.idempotencyKey).toMatch(/^anyapi-auto-[a-f0-9-]{36}$/); }); - it('rejects explicit keys outside the gateway wire format', () => { - expect(() => prepareRunIdempotency('reddit.search', {}, '')).toThrow('1 to 255 visible ASCII'); - expect(() => prepareRunIdempotency('reddit.search', {}, 'contains space')).toThrow('1 to 255 visible ASCII'); - expect(() => prepareRunIdempotency('reddit.search', {}, 'ends-with-newline\n')).toThrow('1 to 255 visible ASCII'); - expect(() => prepareRunIdempotency('reddit.search', {}, 'x'.repeat(256))).toThrow('1 to 255 visible ASCII'); - expect(prepareRunIdempotency('reddit.search', {}, 'x'.repeat(255)).idempotencyKey).toHaveLength(255); + it("rejects explicit keys outside the gateway wire format", () => { + expect(() => prepareRunIdempotency("reddit.search", {}, "")).toThrow( + "1 to 255 visible ASCII", + ); + expect(() => + prepareRunIdempotency("reddit.search", {}, "contains space"), + ).toThrow("1 to 255 visible ASCII"); + expect(() => + prepareRunIdempotency("reddit.search", {}, "ends-with-newline\n"), + ).toThrow("1 to 255 visible ASCII"); + expect(() => + prepareRunIdempotency("reddit.search", {}, "x".repeat(256)), + ).toThrow("1 to 255 visible ASCII"); + expect( + prepareRunIdempotency("reddit.search", {}, "x".repeat(255)) + .idempotencyKey, + ).toHaveLength(255); }); }); function commandContext(fetchImpl: FetchLike): CommandContext { return { - cwd: '/tmp', - homeDir: '/tmp', + cwd: "/tmp", + homeDir: "/tmp", env: {}, stdin: new PassThrough(), stdout: new PassThrough(), @@ -127,68 +179,91 @@ function commandContext(fetchImpl: FetchLike): CommandContext { }; } -describe('run output paths', () => { - it('uses sku and a file-safe ISO timestamp under .anyapi', () => { - const path = buildRunOutputPath('reddit.search', new Date('2026-07-05T19:20:30.456Z'), '/tmp/project'); - expect(path).toBe(join('/tmp/project', '.anyapi', 'reddit.search-2026-07-05T19-20-30-456Z.json')); +describe("run output paths", () => { + it("uses sku and a file-safe ISO timestamp under .anyapi", () => { + const path = buildRunOutputPath( + "reddit.search", + new Date("2026-07-05T19:20:30.456Z"), + "/tmp/project", + ); + expect(path).toBe( + join( + "/tmp/project", + ".anyapi", + "reddit.search-2026-07-05T19-20-30-456Z.json", + ), + ); }); - it('replaces unsafe sku characters', () => { - const path = buildRunOutputPath('web/scrape test', new Date('2026-07-05T19:20:30.456Z'), '/tmp/project'); - expect(path.endsWith('web_scrape_test-2026-07-05T19-20-30-456Z.json')).toBe(true); + it("replaces unsafe sku characters", () => { + const path = buildRunOutputPath( + "web/scrape test", + new Date("2026-07-05T19:20:30.456Z"), + "/tmp/project", + ); + expect(path.endsWith("web_scrape_test-2026-07-05T19-20-30-456Z.json")).toBe( + true, + ); }); }); -describe('402 handling', () => { - it('detects trial cap errors and relays the server upgrade guidance', async () => { +describe("402 handling", () => { + it("detects trial cap errors and relays the server upgrade guidance", async () => { const fetchImpl: FetchLike = async () => new Response( - JSON.stringify({ error: 'trial_cap_reached', message: 'Trial budget used up; run anyapi connect.' }), - { status: 402, headers: { 'Content-Type': 'application/json' } }, + JSON.stringify({ + error: "trial_cap_reached", + message: "Trial budget used up; run anyapi connect.", + }), + { status: 402, headers: { "Content-Type": "application/json" } }, ); const client = new AnyApiClient({ - apiKey: 'aa_live_test', + apiKey: "aa_live_test", fetchImpl, - restBaseUrl: 'https://example.test/v1', + restBaseUrl: "https://example.test/v1", }); try { - await client.run('reddit.search', { query: 'anyapi' }); - throw new Error('Expected run to fail'); + await client.run("reddit.search", { query: "anyapi" }); + throw new Error("Expected run to fail"); } catch (error) { expect(error).toBeInstanceOf(ApiError); expect(isTrialCapReached(error)).toBe(true); - expect(formatTrialCapMessage(error)).toBe('Trial budget used up; run anyapi connect.'); + expect(formatTrialCapMessage(error)).toBe( + "Trial budget used up; run anyapi connect.", + ); } }); - it('falls back to a connect nudge when the 402 body has no message', () => { - const error = new ApiError('trial_cap_reached', 402, { error: 'trial_cap_reached' }); + it("falls back to a connect nudge when the 402 body has no message", () => { + const error = new ApiError("trial_cap_reached", 402, { + error: "trial_cap_reached", + }); expect(isTrialCapReached(error)).toBe(true); - expect(formatTrialCapMessage(error)).toContain('anyapi connect'); + expect(formatTrialCapMessage(error)).toContain("anyapi connect"); }); }); -describe('409 idempotency handling', () => { - it('explains when the key belongs to a different request using the error code', () => { - const error = new ApiError('Original request is still running.', 409, { - error: 'Original request is still running.', - code: 'idempotency_conflict', +describe("409 idempotency handling", () => { + it("explains when the key belongs to a different request using the error code", () => { + const error = new ApiError("Original request is still running.", 409, { + error: "Original request is still running.", + code: "idempotency_conflict", }); expect(formatIdempotencyError(error)).toBe( - 'This idempotency key was already used for a different request. Use a new key, or retry with the original SKU and input.', + "This idempotency key was already used for a different request. Use a new key, or retry with the original SKU and input.", ); }); - it('explains when the original request is still running using the error code', () => { - const error = new ApiError('This key belongs to another request.', 409, { - error: 'This key belongs to another request.', - code: 'idempotency_in_progress', + it("explains when the original request is still running using the error code", () => { + const error = new ApiError("This key belongs to another request.", 409, { + error: "This key belongs to another request.", + code: "idempotency_in_progress", }); expect(formatIdempotencyError(error)).toBe( - 'The original request for this idempotency key is still running. Retry shortly with the same key.', + "The original request for this idempotency key is still running. Retry shortly with the same key.", ); }); }); diff --git a/package-lock.json b/package-lock.json index 0a95453..4d13756 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "anyapi-cli", - "version": "0.5.0", + "version": "0.6.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "anyapi-cli", - "version": "0.5.0", + "version": "0.6.0", "license": "MIT", "dependencies": { "commander": "^12.1.0", diff --git a/package.json b/package.json index fcc6379..d98188f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "anyapi-cli", - "version": "0.5.0", + "version": "0.6.0", "description": "Official CLI for AnyAPI, a unified marketplace for scraping and data APIs.", "type": "module", "bin": { diff --git a/skills/anyapi-run/SKILL.md b/skills/anyapi-run/SKILL.md index b7504d7..eacc450 100644 --- a/skills/anyapi-run/SKILL.md +++ b/skills/anyapi-run/SKILL.md @@ -36,9 +36,14 @@ anyapi run reddit.search --input '{"query":"anyapi"}' --idempotency-key task-123 ``` Repeating the same request with the same key returns its original result without -another charge. Use `--idempotency-key auto` to derive an opt-in daily key from -the SKU and canonical JSON input. Omit the flag when repeated runs are -intentional. +another charge. The CLI generates a random key for every logical invocation. Pass +an explicit `--idempotency-key` only when you need to resume that invocation yourself. +`--idempotency-key auto` explicitly requests the same random per-invocation behavior. + +Provider-job APIs poll to completion by default. Use `--no-wait` to return the +durable request ID immediately, then run `anyapi requests get ` or +`anyapi requests wait `. Never repeat the paid `run`; resume by request ID. +On Ctrl-C the CLI prints the exact resume command. ## Run once, reshape forever at zero cost diff --git a/src/api.ts b/src/api.ts index 94b3e68..0262087 100644 --- a/src/api.ts +++ b/src/api.ts @@ -1,6 +1,6 @@ import { CATALOG_URL, REST_BASE_URL, SIGNUP_URL } from './constants.js'; import { readCatalogResponse, readDiscoveryApi, readSearchResponse } from './discovery.js'; -import { ApiError } from './errors.js'; +import { ApiError, CliError } from './errors.js'; import type { CatalogApi, CatalogResponse, @@ -9,6 +9,7 @@ import type { FetchLike, OAuthMetadata, RunResult, + RequestSnapshot, SearchResponse, SignupResponse, TokenResponse, @@ -28,6 +29,8 @@ export interface SignupOptions { export interface RunOptions { idempotencyKey?: string; + noWait?: boolean; + onAccepted?: (snapshot: RequestSnapshot) => void; } export class AnyApiClient { @@ -150,17 +153,43 @@ export class AnyApiClient { // run always fetches the FULL result. Response shaping (fields/max_items/summary/ // jq) is done locally by the CLI over the saved file, so re-slicing a paid run // costs nothing; no shape params are sent upstream. - async run(sku: string, input: unknown, options: RunOptions = {}): Promise { + async run(sku: string, input: unknown, options: RunOptions = {}): Promise { const url = new URL(`${this.restBaseUrl}/run/${encodeURIComponent(sku)}`); - return this.requestJson(url, { + const response = await this.requestJson(url, { method: 'POST', headers: { ...this.authHeaders(), 'Content-Type': 'application/json', ...(options.idempotencyKey ? { 'Idempotency-Key': options.idempotencyKey } : {}), + ...(options.noWait ? { Prefer: 'respond-async' } : {}), }, body: JSON.stringify(input), }); + if (!isRequestSnapshot(response)) return response; + options.onAccepted?.(response); + if (options.noWait) return response; + return this.waitRequest(response.requestId, response); + } + + async getRequest(requestId: string): Promise { + return this.requestJson(`${this.restBaseUrl}/requests/${encodeURIComponent(requestId)}`, { + headers: this.authHeaders(), + }); + } + + async waitRequest(requestId: string, initial?: RequestSnapshot): Promise { + let snapshot = initial ?? await this.getRequest(requestId); + const deadline = Date.now() + 300_000; + while (snapshot.status === 'queued' || snapshot.status === 'running') { + if (Date.now() >= deadline) { + throw new CliError(`Request ${requestId} is still running. Resume with: anyapi requests wait ${requestId}`); + } + await delay(Math.max(1, snapshot.retryAfterSeconds ?? 2) * 1000); + snapshot = await this.getRequest(requestId); + } + if (snapshot.status === 'succeeded' && snapshot.result) return snapshot.result; + if (snapshot.resultExpired) throw new CliError(`Request ${requestId} succeeded, but its result expired.`); + throw new CliError(`Request ${requestId} ended with ${snapshot.error?.code ?? snapshot.status}.`); } async balance(): Promise { @@ -186,6 +215,11 @@ export class AnyApiClient { } } +const delay = (milliseconds: number) => new Promise((resolve) => setTimeout(resolve, milliseconds)); +const isRequestSnapshot = (value: unknown): value is RequestSnapshot => + typeof value === 'object' && value !== null && typeof (value as RequestSnapshot).requestId === 'string' && + typeof (value as RequestSnapshot).status === 'string'; + function compactObject(input: Record): Record { return Object.fromEntries( Object.entries(input).filter((entry): entry is [string, string] => typeof entry[1] === 'string' && entry[1] !== ''), diff --git a/src/commands.ts b/src/commands.ts index 724ef9b..317ce12 100644 --- a/src/commands.ts +++ b/src/commands.ts @@ -24,7 +24,7 @@ import { resolveLastFile } from './view.js'; import { configureMcp, detectAgents, installSkillsForAgents, printAgentDetection } from './init.js'; import { promptYesNo, writeLine, type CommandContext } from './io.js'; import { deviceLoginCommand, type DeviceLoginDependencies } from './login.js'; -import type { AnyApiConfig, CatalogApi, RunResult, SignupResponse } from './types.js'; +import type { AnyApiConfig, CatalogApi, RequestSnapshot, RunResult, SignupResponse } from './types.js'; interface ShapeCliOptions { fields?: string; @@ -106,6 +106,7 @@ export async function runCommand( idempotencyKey?: string; output?: string; json?: boolean; + noWait?: boolean; }, ): Promise { const auth = await requireApiKey(ctx, global); @@ -114,11 +115,21 @@ export async function runCommand( const prepared = prepareRunIdempotency(sku, input, options.idempotencyKey); const shape = parseShapeRequest(options); - let result: RunResult; + let result: RunResult | RequestSnapshot; + let acceptedRequestId: string | undefined; + const onInterrupt = () => { + if (acceptedRequestId) writeLine(ctx.stderr, `Resume with: anyapi requests wait ${acceptedRequestId}`); + process.exit(130); + }; try { result = stripServerHint(await client.run(sku, prepared.input, { idempotencyKey: prepared.idempotencyKey, - })); + noWait: options.noWait, + onAccepted: (snapshot) => { + acceptedRequestId = snapshot.requestId; + process.once('SIGINT', onInterrupt); + }, + }) as RunResult); } catch (error) { const idempotencyMessage = formatIdempotencyError(error); if (idempotencyMessage) { @@ -129,6 +140,15 @@ export async function runCommand( throw new CliError('trial_cap_reached'); } throw error; + } finally { + process.removeListener('SIGINT', onInterrupt); + } + + if (isRequestSnapshot(result)) { + const rendered = JSON.stringify(result, null, options.json ? 0 : 2); + writeLine(ctx.stdout, rendered); + if (!options.json) writeLine(ctx.stdout, `Resume with: anyapi requests wait ${result.requestId}`); + return; } const shaped = hasShapeFlags(shape); @@ -159,6 +179,21 @@ export async function runCommand( } } +export async function getRequestCommand(ctx: CommandContext, global: GlobalOptions, requestId: string): Promise { + const auth = await requireApiKey(ctx, global); + const client = new AnyApiClient({ apiKey: auth.apiKey, fetchImpl: ctx.fetchImpl }); + writeLine(ctx.stdout, JSON.stringify(await client.getRequest(requestId), null, 2)); +} + +export async function waitRequestCommand(ctx: CommandContext, global: GlobalOptions, requestId: string): Promise { + const auth = await requireApiKey(ctx, global); + const client = new AnyApiClient({ apiKey: auth.apiKey, fetchImpl: ctx.fetchImpl }); + writeLine(ctx.stdout, JSON.stringify(await client.waitRequest(requestId), null, 2)); +} + +const isRequestSnapshot = (value: RunResult | RequestSnapshot): value is RequestSnapshot => + typeof value.requestId === 'string' && typeof value.status === 'string'; + export async function viewCommand( ctx: CommandContext, path: string | undefined, diff --git a/src/index.ts b/src/index.ts index bce73f0..fd3c40a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,9 +1,10 @@ #!/usr/bin/env node -import { Command } from 'commander'; +import { Command } from "commander"; import { balanceCommand, describeCommand, initCommand, + getRequestCommand, listCommand, loginCommand, runCommand, @@ -11,101 +12,174 @@ import { setupSkillsCommand, signupCommand, viewCommand, + waitRequestCommand, type GlobalOptions, -} from './commands.js'; -import { connectCommand } from './connect.js'; -import { CliError } from './errors.js'; -import { defaultContext } from './io.js'; +} from "./commands.js"; +import { connectCommand } from "./connect.js"; +import { CliError } from "./errors.js"; +import { defaultContext } from "./io.js"; +import { normalizeRunCLIOptions } from "./run.js"; const program = new Command(); const ctx = defaultContext(); program - .name('anyapi') - .description('Official CLI for AnyAPI.') - .option('--api-key ', 'AnyAPI API key. Overrides ANYAPI_API_KEY and local config.') - .version('0.5.0'); + .name("anyapi") + .description("Official CLI for AnyAPI.") + .option( + "--api-key ", + "AnyAPI API key. Overrides ANYAPI_API_KEY and local config.", + ) + .version("0.5.0"); program - .command('signup') - .description('Mint a free AnyAPI trial key and save it locally.') - .option('--label