diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 8a95939..c387345 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -156,11 +156,11 @@ The same option is available in `agent-docs.config.yml` as `options.urlPathPatte ### Request behavior -| Flag | Default | Description | -| -------------------------- | ------- | ------------------------------------------- | -| `--max-concurrency ` | `3` | Maximum concurrent HTTP requests | -| `--request-delay ` | `200` | Delay between requests in milliseconds | -| `--canonical-origin ` | | The production domain your content links to | +| Flag | Default | Description | +| -------------------------- | ------- | ------------------------------------------------------------------------------------ | +| `--max-concurrency ` | `3` | Maximum concurrent HTTP requests | +| `--request-delay ` | `200` | Delay between requests in milliseconds | +| `--canonical-origin ` | | The production base URL (origin, or origin plus a path prefix) your content links to | AFDocs enforces delays between requests and caps concurrent connections to avoid overloading your server. Adjust these if you need gentler or faster runs: @@ -172,11 +172,17 @@ afdocs check https://docs.example.com --request-delay 500 --max-concurrency 1 afdocs check https://docs.example.com --request-delay 50 --max-concurrency 10 ``` -Use `--canonical-origin` when your site's URLs in `sitemap.xml` and `llms.txt` don't match the domain you're testing, such as preview deployments or localhost. +Use `--canonical-origin` when your site's URLs in `sitemap.xml` and `llms.txt` don't match the base URL you're testing, such as preview deployments or localhost. It accepts either a bare origin or an origin plus a path prefix: + +- **Origin only** — rewrites every URL on that host, regardless of path. +- **Origin plus a path prefix** — rewrites only URLs under that prefix, remapping them to the base URL you pass to `check` (the prefixes need not match). ```bash -# Test a preview deployment +# Origin only: rewrite all example.com URLs to the preview host afdocs check https://preview-xyz-example.app/docs --canonical-origin https://example.com + +# Path prefix: production docs live under /docs, but your preview serves them under /preview +afdocs check http://localhost:3000/preview --canonical-origin https://example.com/docs ``` ### llms.txt selection diff --git a/docs/reference/config-file.md b/docs/reference/config-file.md index 19686c6..1ade92b 100644 --- a/docs/reference/config-file.md +++ b/docs/reference/config-file.md @@ -93,7 +93,7 @@ Override default runner options. All fields are optional: | `requestTimeout` | `30000` | Timeout for individual HTTP requests in milliseconds | | `preferredLocale` | auto-detect | Preferred locale for URL discovery (e.g. `en`, `fr`, `ja`) | | `preferredVersion` | auto-detect | Preferred version for URL discovery (e.g. `v3`, `2.x`) | -| `canonicalOrigin` | | The production domain your content links to | +| `canonicalOrigin` | | The production base URL (origin, or origin plus a path prefix) your content links to | | `llmsTxtUrl` | | Explicit llms.txt URL to use as canonical (overrides the discovery heuristic; see CLI docs) | | `thresholds.pass` | `50000` | Page size pass threshold in characters | | `thresholds.fail` | `100000` | Page size fail threshold in characters | diff --git a/docs/run-locally.md b/docs/run-locally.md index ae76ea5..286477c 100644 --- a/docs/run-locally.md +++ b/docs/run-locally.md @@ -101,7 +101,7 @@ Some checks may behave differently against a local server: When you build your site locally, generated files like `llms.txt` and `sitemap.xml` typically contain your production domain. AFDocs sees URLs pointing to `https://docs.example.com` but you're testing `http://localhost:3000`, so origin comparisons fail and checks like `llms-txt-coverage` report 0% coverage. -Use `--canonical-origin` to tell AFDocs which production domain to rewrite: +Use `--canonical-origin` to tell AFDocs which production base URL (origin, or origin plus a path prefix) to rewrite: ```bash npm run build diff --git a/src/cli/commands/check.ts b/src/cli/commands/check.ts index d1cf805..9e44fee 100644 --- a/src/cli/commands/check.ts +++ b/src/cli/commands/check.ts @@ -1,5 +1,5 @@ import type { Command } from 'commander'; -import { normalizeUrl, runChecks } from '../../runner.js'; +import { normalizeCanonical, normalizeUrl, runChecks } from '../../runner.js'; import { formatText } from '../formatters/text.js'; import { formatJson } from '../formatters/json.js'; import { formatScorecard } from '../formatters/scorecard.js'; @@ -73,7 +73,7 @@ export function registerCheckCommand(program: Command): void { ) .option( '--canonical-origin ', - 'The production domain your content links to (for preview/staging testing)', + 'The production base URL (origin, or origin plus a path prefix) your content links to, rewritten to the target for preview/staging testing', ) .option( '--llms-txt-url ', @@ -209,11 +209,20 @@ export function registerCheckCommand(program: Command): void { if (rawCanonical) { const normalized = normalizeUrl(rawCanonical); try { - canonicalOrigin = new URL(normalized).origin; - const targetOrigin = new URL(url).origin; - if (canonicalOrigin === targetOrigin) { + const parsedCanonical = new URL(normalized); + // Normalize identically to createContext so the warning reflects the value used. + canonicalOrigin = normalizeCanonical(normalized); + // The flag has no effect when the canonical resolves to what the rewrite would + // produce anyway: for a sub-path canonical that's the full target base; for an + // origin-only canonical it's just the target origin (the path is untouched). + const parsedTarget = new URL(url); + const canonicalHasSubPath = parsedCanonical.pathname !== '/'; + const noEffect = canonicalHasSubPath + ? canonicalOrigin === normalizeCanonical(url) + : parsedCanonical.origin === parsedTarget.origin; + if (noEffect) { process.stderr.write( - `Warning: --canonical-origin "${canonicalOrigin}" is the same as the target origin. The flag has no effect.\n`, + `Warning: --canonical-origin "${canonicalOrigin}" is the same as the target. The flag has no effect.\n`, ); canonicalOrigin = undefined; } diff --git a/src/http.ts b/src/http.ts index 7e3b7be..b49ad8e 100644 --- a/src/http.ts +++ b/src/http.ts @@ -11,7 +11,9 @@ interface RateLimitedHttpClientOptions { requestDelay: number; requestTimeout: number; maxConcurrency: number; + /** Canonical base URL to find in bodies (origin, or origin plus a path prefix). */ canonicalOrigin?: string; + /** Value to replace it with: the target origin, or the full target base for a path-prefix canonical. */ targetOrigin?: string; } @@ -24,9 +26,13 @@ function escapeRegExp(s: string): string { export function createHttpClient(options: RateLimitedHttpClientOptions): HttpClient { let lastRequestTime = 0; let activeRequests = 0; + // Match the canonical base only at a URL boundary: end-of-string or one of these + // delimiters. `)` and `,` are included so URLs inside markdown links `[x](url)` and + // prose `url, next` rewrite; the rare tradeoff is a path segment like `/docs,2024` + // being treated as the `/docs` prefix. const originPattern = options.canonicalOrigin && options.targetOrigin - ? new RegExp(escapeRegExp(options.canonicalOrigin) + '(?=[/\\s"\'\\]>]|$)', 'g') + ? new RegExp(escapeRegExp(options.canonicalOrigin) + '(?=[/?#\\s"\'\\]),>]|$)', 'g') : null; async function waitForSlot(): Promise { @@ -81,7 +87,11 @@ export function createHttpClient(options: RateLimitedHttpClientOptions): HttpCli if (/text|xml|json|markdown/.test(ct)) { const body = await response.text(); originPattern.lastIndex = 0; - const rewritten = body.replace(originPattern, options.targetOrigin); + // Use a function replacer so `$` in the target (e.g. a preview path + // containing `$'` or `$&`) is inserted literally, not interpreted as a + // String.replace replacement pattern. + const target = options.targetOrigin; + const rewritten = body.replace(originPattern, () => target); return { ok: response.ok, status: response.status, diff --git a/src/runner.ts b/src/runner.ts index 2e5a609..0089cd7 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -42,6 +42,18 @@ export function normalizeUrl(url: string): string { return url; } +/** + * Normalize a canonical/base URL for the http.ts rewrite regex, which is literal and + * case-sensitive: lowercase the host and drop default ports (via URL.origin) while + * preserving any sub-path, then strip the trailing slash so it matches path segments. + * Must be applied identically to the canonical value and the target base it is compared + * against. Assumes `raw` is already scheme-qualified (see normalizeUrl). + */ +export function normalizeCanonical(raw: string): string { + const parsed = new URL(raw); + return `${parsed.origin}${parsed.pathname}`.replace(/\/+$/, ''); +} + export function createContext(baseUrl: string, options?: Partial): CheckContext { if (options) { if (options.canonicalOrigin) { @@ -60,6 +72,18 @@ export function createContext(baseUrl: string, options?: Partial) const merged = { ...DEFAULT_OPTIONS, ...options }; baseUrl = normalizeUrl(baseUrl); const url = new URL(baseUrl); + const normalizedBaseUrl = baseUrl.replace(/\/$/, ''); + + // Normalize the canonical value once, here, so CLI and direct createContext() callers + // behave identically. Keep merged.canonicalOrigin in sync with the value wired below. + let canonicalOrigin: string | undefined; + if (merged.canonicalOrigin) { + canonicalOrigin = normalizeCanonical(merged.canonicalOrigin); + merged.canonicalOrigin = canonicalOrigin; + } + + // A sub-path canonical rewrites to the full preview base; origin-only swaps origins. + const canonicalHasSubPath = Boolean(canonicalOrigin && new URL(canonicalOrigin).pathname !== '/'); // Fail fast when the target port is on the WHATWG fetch bad port list: // undici would refuse every request, turning one port choice into a wall @@ -70,15 +94,19 @@ export function createContext(baseUrl: string, options?: Partial) } return { - baseUrl: baseUrl.replace(/\/$/, ''), + baseUrl: normalizedBaseUrl, origin: url.origin, previousResults: new Map(), http: createHttpClient({ requestDelay: merged.requestDelay, requestTimeout: merged.requestTimeout, maxConcurrency: merged.maxConcurrency, - canonicalOrigin: merged.canonicalOrigin, - targetOrigin: merged.canonicalOrigin ? url.origin : undefined, + canonicalOrigin, + targetOrigin: canonicalOrigin + ? canonicalHasSubPath + ? normalizedBaseUrl + : url.origin + : undefined, }), options: merged, pageCache: new Map(), diff --git a/src/types.ts b/src/types.ts index 54e0a99..4eded6a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -94,7 +94,12 @@ export interface CheckOptions { preferredLocale?: string; /** Preferred version for URL discovery (e.g. 'v3', '2.x', 'latest'). Overrides auto-detection from baseUrl. */ preferredVersion?: string; - /** Canonical origin to rewrite in fetched content (for preview/staging testing). */ + /** + * Canonical base URL to rewrite in fetched content (for preview/staging testing). + * Accepts an origin (`https://prod.example.com`) or an origin plus a path prefix + * (`https://prod.example.com/docs`); when a path prefix is given, matching URLs are + * rewritten to the full target base. + */ canonicalOrigin?: string; /** Pass threshold for llms-txt-coverage (0–100). Default 95. */ coveragePassThreshold?: number; diff --git a/test/unit/cli/check-command.test.ts b/test/unit/cli/check-command.test.ts index e76f336..5eec8bd 100644 --- a/test/unit/cli/check-command.test.ts +++ b/test/unit/cli/check-command.test.ts @@ -758,7 +758,115 @@ describe('check command config integration', () => { await new Promise((r) => setTimeout(r, 100)); const stderr = stderrSpy.mock.calls.map((c) => c[0]).join(''); - expect(stderr).toContain('same as the target origin'); + expect(stderr).toContain('same as the target'); + expect(stderr).toContain('no effect'); + + stdoutSpy.mockRestore(); + stderrSpy.mockRestore(); + }); + + it('warns when origin-only --canonical-origin matches target whose URL has a path', async () => { + server.use( + http.get('http://cmd-canon-path.local/docs/llms.txt', () => + HttpResponse.text(VALID_LLMS_TXT), + ), + http.get('http://cmd-canon-path.local/llms.txt', () => HttpResponse.text(VALID_LLMS_TXT)), + ); + + const stdoutSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + const stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + + const { run } = await import('../../../src/cli/index.js'); + await run([ + 'node', + 'afdocs', + 'check', + 'http://cmd-canon-path.local/docs', + '--canonical-origin', + 'http://cmd-canon-path.local', + '--checks', + 'llms-txt-exists', + '--request-delay', + '0', + ]); + await new Promise((r) => setTimeout(r, 100)); + + const stderr = stderrSpy.mock.calls.map((c) => c[0]).join(''); + // Origin-only canonical == target origin → no effect, even though target has a path. + expect(stderr).toContain('no effect'); + + stdoutSpy.mockRestore(); + stderrSpy.mockRestore(); + }); + + it('does not suppress --canonical-origin when same origin but different sub-path', async () => { + server.use( + http.get('http://cmd-canon-subpath.local/preview/llms.txt', () => + HttpResponse.text(VALID_LLMS_TXT), + ), + http.get( + 'http://cmd-canon-subpath.local/llms.txt', + () => new HttpResponse(null, { status: 404 }), + ), + http.get( + 'http://cmd-canon-subpath.local/docs/llms.txt', + () => new HttpResponse(null, { status: 404 }), + ), + ); + + const stdoutSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + const stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + + const { run } = await import('../../../src/cli/index.js'); + await run([ + 'node', + 'afdocs', + 'check', + 'http://cmd-canon-subpath.local/preview', + '--canonical-origin', + 'http://cmd-canon-subpath.local/aws/en', + '--checks', + 'llms-txt-exists', + '--request-delay', + '0', + ]); + await new Promise((r) => setTimeout(r, 100)); + + const stdout = stdoutSpy.mock.calls.map((c) => c[0]).join(''); + const stderr = stderrSpy.mock.calls.map((c) => c[0]).join(''); + expect(stderr).not.toContain('no effect'); + expect(stdout).toContain('llms-txt-exists'); + + stdoutSpy.mockRestore(); + stderrSpy.mockRestore(); + }); + + it('warns when a path-prefix --canonical-origin equals the target base', async () => { + server.use( + http.get('http://cmd-canon-eq.local/docs/llms.txt', () => HttpResponse.text(VALID_LLMS_TXT)), + http.get('http://cmd-canon-eq.local/llms.txt', () => HttpResponse.text(VALID_LLMS_TXT)), + ); + + const stdoutSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + const stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + + const { run } = await import('../../../src/cli/index.js'); + await run([ + 'node', + 'afdocs', + 'check', + 'http://cmd-canon-eq.local/docs', + '--canonical-origin', + 'http://cmd-canon-eq.local/docs', + '--checks', + 'llms-txt-exists', + '--request-delay', + '0', + ]); + await new Promise((r) => setTimeout(r, 100)); + + const stderr = stderrSpy.mock.calls.map((c) => c[0]).join(''); + // Path-prefix canonical resolves to the same base as the target → no-op rewrite. expect(stderr).toContain('no effect'); stdoutSpy.mockRestore(); diff --git a/test/unit/helpers/http.test.ts b/test/unit/helpers/http.test.ts index bdd2192..95ec3f1 100644 --- a/test/unit/helpers/http.test.ts +++ b/test/unit/helpers/http.test.ts @@ -191,6 +191,71 @@ describe('createHttpClient', () => { expect(text).toContain('https://prod.example.com.evil.com/phishing'); }); + it('does not match a longer sub-path that starts with the canonical base', async () => { + const body = [ + 'https://prod.example.com/docs/guide', + 'https://prod.example.com/docsearch/index', + ].join('\n'); + globalThis.fetch = vi.fn(async () => makeTextResponse(body, { contentType: 'text/plain' })); + + const client = createHttpClient({ + requestDelay: 0, + requestTimeout: 5000, + maxConcurrency: 10, + canonicalOrigin: 'https://prod.example.com/docs', + targetOrigin: 'https://preview.local/preview', + }); + const response = await client.fetch('http://preview.local/preview'); + const text = await response.text(); + + expect(text).toContain('https://preview.local/preview/guide'); + // /docsearch must NOT be rewritten by a /docs canonical. + expect(text).toContain('https://prod.example.com/docsearch/index'); + }); + + it('inserts a target containing $ literally (no replacement-pattern interpretation)', async () => { + const body = 'link https://prod.example.com/docs/guide tail'; + globalThis.fetch = vi.fn(async () => makeTextResponse(body, { contentType: 'text/plain' })); + + const client = createHttpClient({ + requestDelay: 0, + requestTimeout: 5000, + maxConcurrency: 10, + canonicalOrigin: 'https://prod.example.com/docs', + targetOrigin: "http://preview.local/a$'b$&c$`d", + }); + const text = await (await client.fetch('http://preview.local/x')).text(); + + expect(text).toBe("link http://preview.local/a$'b$&c$`d/guide tail"); + }); + + it('rewrites base URLs terminated by ) , ? or # (markdown links, prose)', async () => { + const body = [ + '[docs](https://prod.example.com)', + 'see https://prod.example.com, then', + 'query https://prod.example.com?a=1', + 'frag https://prod.example.com#top', + ].join('\n'); + globalThis.fetch = vi.fn(async () => + makeTextResponse(body, { contentType: 'text/markdown' }), + ); + + const client = createHttpClient({ + requestDelay: 0, + requestTimeout: 5000, + maxConcurrency: 10, + canonicalOrigin: 'https://prod.example.com', + targetOrigin: 'https://preview.local', + }); + const text = await (await client.fetch('http://preview.local/x')).text(); + + expect(text).not.toContain('prod.example.com'); + expect(text).toContain('[docs](https://preview.local)'); + expect(text).toContain('see https://preview.local, then'); + expect(text).toContain('query https://preview.local?a=1'); + expect(text).toContain('frag https://preview.local#top'); + }); + it('returns the same rewritten body on multiple text() calls', async () => { const body = 'Link: https://prod.example.com/page'; globalThis.fetch = vi.fn(async () => makeTextResponse(body, { contentType: 'text/plain' })); diff --git a/test/unit/runner.test.ts b/test/unit/runner.test.ts index 98a75a4..906e75d 100644 --- a/test/unit/runner.test.ts +++ b/test/unit/runner.test.ts @@ -1,7 +1,7 @@ import { describe, it, expect, beforeAll, vi } from 'vitest'; import { http, HttpResponse } from 'msw'; import { setupServer } from 'msw/node'; -import { createContext, normalizeUrl, runChecks } from '../../src/runner.js'; +import { createContext, normalizeCanonical, normalizeUrl, runChecks } from '../../src/runner.js'; import { registerCheck } from '../../src/checks/registry.js'; import '../../src/checks/index.js'; import { mockSitemapNotFound } from '../helpers/mock-sitemap-not-found.js'; @@ -36,6 +36,42 @@ describe('normalizeUrl', () => { }); }); +describe('normalizeCanonical', () => { + it('lowercases the host', () => { + expect(normalizeCanonical('https://PROD.Example.com')).toBe('https://prod.example.com'); + }); + + it('drops default ports', () => { + expect(normalizeCanonical('https://prod.example.com:443')).toBe('https://prod.example.com'); + expect(normalizeCanonical('http://prod.example.com:80')).toBe('http://prod.example.com'); + }); + + it('preserves a non-default port', () => { + expect(normalizeCanonical('https://prod.example.com:8080')).toBe( + 'https://prod.example.com:8080', + ); + }); + + it('preserves a sub-path but strips the trailing slash', () => { + expect(normalizeCanonical('https://prod.example.com/aws/en/')).toBe( + 'https://prod.example.com/aws/en', + ); + }); + + it('strips repeated trailing slashes', () => { + expect(normalizeCanonical('https://prod.example.com/docs//')).toBe( + 'https://prod.example.com/docs', + ); + expect(normalizeCanonical('https://prod.example.com//')).toBe('https://prod.example.com'); + }); + + it('drops query and fragment', () => { + expect(normalizeCanonical('https://prod.example.com/docs?x=1#h')).toBe( + 'https://prod.example.com/docs', + ); + }); +}); + describe('createContext URL normalization', () => { it('prepends https:// when no scheme is provided', () => { const ctx = createContext('example.com'); @@ -106,6 +142,105 @@ describe('createContext URL normalization', () => { }); }); +describe('createContext canonicalOrigin targetOrigin wiring', () => { + it('rewrites canonical origin when canonicalOrigin has no sub-path', async () => { + const body = 'See https://prod.example.com/docs/guide'; + const originalFetch = globalThis.fetch; + globalThis.fetch = vi.fn(async () => ({ + ok: true, + status: 200, + statusText: 'OK', + headers: new Headers({ 'content-type': 'text/plain' }), + url: 'http://preview.local/docs/guide', + redirected: false, + text: async () => body, + })) as unknown as typeof fetch; + + try { + const ctx = createContext('http://preview.local', { + canonicalOrigin: 'https://prod.example.com', + }); + const text = await (await ctx.http.fetch('http://preview.local/docs/guide')).text(); + expect(text).toBe('See http://preview.local/docs/guide'); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it('rewrites canonical base when canonicalOrigin has a sub-path', async () => { + const body = 'See https://prod.example.com/aws/en/docs/guide'; + const originalFetch = globalThis.fetch; + globalThis.fetch = vi.fn(async () => ({ + ok: true, + status: 200, + statusText: 'OK', + headers: new Headers({ 'content-type': 'text/plain' }), + url: 'http://preview.local/preview/docs/guide', + redirected: false, + text: async () => body, + })) as unknown as typeof fetch; + + try { + const ctx = createContext('http://preview.local/preview', { + canonicalOrigin: 'https://prod.example.com/aws/en', + }); + const text = await (await ctx.http.fetch('http://preview.local/preview/docs/guide')).text(); + expect(text).toBe('See http://preview.local/preview/docs/guide'); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it('rewrites when a sub-path canonicalOrigin has a trailing slash', async () => { + const body = 'See https://prod.example.com/aws/en/docs/guide'; + const originalFetch = globalThis.fetch; + globalThis.fetch = vi.fn(async () => ({ + ok: true, + status: 200, + statusText: 'OK', + headers: new Headers({ 'content-type': 'text/plain' }), + url: 'http://preview.local/preview/docs/guide', + redirected: false, + text: async () => body, + })) as unknown as typeof fetch; + + try { + const ctx = createContext('http://preview.local/preview', { + canonicalOrigin: 'https://prod.example.com/aws/en/', + }); + const text = await (await ctx.http.fetch('http://preview.local/preview/docs/guide')).text(); + expect(text).toBe('See http://preview.local/preview/docs/guide'); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it('rewrites when canonicalOrigin host is mixed-case (host is normalized)', async () => { + const body = 'See https://prod.example.com/docs/guide?x=1 and https://prod.example.com/a#h'; + const originalFetch = globalThis.fetch; + globalThis.fetch = vi.fn(async () => ({ + ok: true, + status: 200, + statusText: 'OK', + headers: new Headers({ 'content-type': 'text/plain' }), + url: 'http://preview.local/x', + redirected: false, + text: async () => body, + })) as unknown as typeof fetch; + + try { + const ctx = createContext('http://preview.local', { + canonicalOrigin: 'https://PROD.example.com', + }); + const text = await (await ctx.http.fetch('http://preview.local/x')).text(); + // Host normalized to lowercase so the rewrite matches; query/fragment boundaries match too. + expect(text).toBe('See http://preview.local/docs/guide?x=1 and http://preview.local/a#h'); + } finally { + globalThis.fetch = originalFetch; + } + }); +}); + describe('runner', () => { it('skips dependent checks when dependency fails', async () => { server.use(