diff --git a/scripts/migration/consume-standard-candidate.mjs b/scripts/migration/consume-standard-candidate.mjs index 41bd6da..2915d0d 100644 --- a/scripts/migration/consume-standard-candidate.mjs +++ b/scripts/migration/consume-standard-candidate.mjs @@ -1,6 +1,6 @@ #!/usr/bin/env node /** - * Standard-candidate consumption harness (META-235 Step 5 / I-7). + * Standard-candidate consumption harness (META-235 Step 5 / META-258 I-7). * * Packs the current repo as an npm tarball, installs it in a disposable * directory, starts the MCP server from the packed artifact (not from @@ -12,248 +12,314 @@ * the npm package and runs the server is doing exactly what this harness * does. * + * Every assertion lives in ./consumption-checks.mjs, which the harness calls + * and tests/migration/consumption-checks.test.ts exercises in both directions. + * Assertions are not written inline here: a check the tests cannot reach is a + * check nobody has watched go red (META-285, META-165 scope amendment). + * + * A verification script must not mutate the thing it verifies. Two structural + * rules enforce that here: + * - `run()` requires an explicit cwd. There is no repo-root default to + * forget, which is how `--help` came to run a real install into the source + * checkout (META-285 stop-now item). + * - The source tree is fingerprinted before and after, and the comparison is + * itself a recorded check. + * * Usage: * node scripts/migration/consume-standard-candidate.mjs * node scripts/migration/consume-standard-candidate.mjs --out + * node scripts/migration/consume-standard-candidate.mjs --help */ import { spawnSync } from "node:child_process"; -import { cpSync, existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { cpSync, existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { + checkAssessDecision, + checkCochangePartners, + checkFragilityEvidence, + checkFragilityTier, + checkHelpWroteNothing, + checkHookDenies, + checkHookOutputMentionsDeny, + checkInstallerHelp, + checkInstructionsMention, + checkPackContainsPath, + checkPackContainsPrefix, + checkPathExists, + checkProcessSucceeded, + checkTarballExists, + checkToolCallResponded, + checkToolsList, + checkTreeUnchanged, + createRecorder, +} from "./consumption-checks.mjs"; const here = dirname(fileURLToPath(import.meta.url)); const repoRoot = resolve(here, "..", ".."); const fixtureRoot = join(repoRoot, "fixture"); +const USAGE = [ + "Usage:", + " node scripts/migration/consume-standard-candidate.mjs [--out ]", + "", + "Packs this repository, installs the tarball into a disposable directory, and", + "verifies the packed artifact is consumable. Writes no files outside --out and", + "makes no change to this checkout.", + "", + "Options:", + " --out Write consumption-receipt.json into ", + " --help, -h Show this message and exit without running anything", +].join("\n"); + +/** + * `cwd` is required, not defaulted. + * + * The original helper defaulted to `repoRoot`, so any invocation that forgot + * to pass a cwd silently targeted the source checkout — which is how the + * installer `--help` probe came to run a real install against this repo. + * Making it required means that mistake cannot be made silently again. + */ function run(cmd, args, opts = {}) { - const result = spawnSync(cmd, args, { + if (!opts.cwd) throw new Error(`run(${cmd}) requires an explicit cwd — refusing to default to the source repo`); + return spawnSync(cmd, args, { encoding: "utf8", timeout: opts.timeout ?? 120000, - cwd: opts.cwd ?? repoRoot, + cwd: opts.cwd, stdio: opts.stdio ?? "pipe", + env: opts.env ?? process.env, }); - return result; } function runOrDie(cmd, args, opts = {}) { const result = run(cmd, args, opts); + if (result.error) throw new Error(`${cmd} ${args.join(" ")} failed to launch: ${result.error.message}`); if (result.status !== 0) { - console.error(`${cmd} ${args.join(" ")} failed (exit ${result.status})`); - if (result.stderr) console.error(result.stderr.slice(-2000)); - process.exit(1); + const detail = (result.stderr ?? "").slice(-2000); + throw new Error(`${cmd} ${args.join(" ")} failed (exit ${result.status})\n${detail}`); } return result; } +/** + * Absolute path to `git`, resolved from fixed system locations. + * + * The fingerprint below is the evidence that this harness did not mutate the + * source checkout. Resolving `git` through `PATH` would let a writable PATH + * entry decide what `git status` reports — the one thing that must not be + * forgeable is the tool asked to prove nothing changed. Fixed locations only. + * + * If none exists the fingerprint is unavailable, and `checkTreeUnchanged` + * fails on a non-string input rather than skipping to green. + */ +const GIT_BIN = ["/usr/bin/git", "/usr/local/bin/git", "/opt/homebrew/bin/git"].find((p) => existsSync(p)) ?? null; + +/** Fingerprint of the source checkout, used to prove the harness changed nothing. */ +function treeFingerprint() { + if (!GIT_BIN) return null; + const result = spawnSync(GIT_BIN, ["status", "--porcelain"], { encoding: "utf8", cwd: repoRoot, timeout: 30000 }); + if (result.error || result.status !== 0) return null; + return result.stdout; +} + +function parseArgs(argv) { + const opts = { outDir: null, help: false }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--help" || arg === "-h") { + opts.help = true; + } else if (arg === "--out") { + const value = argv[++i]; + if (!value) throw new Error("--out requires a directory argument"); + opts.outDir = resolve(value); + } else if (arg.startsWith("--out=")) { + const value = arg.slice("--out=".length); + if (!value) throw new Error("--out= requires a directory argument"); + opts.outDir = resolve(value); + } else { + throw new Error(`unknown argument: ${arg}\n\n${USAGE}`); + } + } + return opts; +} + async function main() { - const args = process.argv.slice(2); - let outDir = null; - for (let i = 0; i < args.length; i++) { - if (args[i] === "--out") outDir = resolve(args[++i]); + const opts = parseArgs(process.argv.slice(2)); + if (opts.help) { + console.log(USAGE); + return; } + const treeBefore = treeFingerprint(); const work = mkdtempSync(join(tmpdir(), "consume-std-")); const packDir = join(work, "pack"); const installDir = join(work, "install"); + // A sandbox that starts empty and must stay empty. Anything the installer + // writes while being asked for help lands here, where it is visible. + const helpSandbox = join(work, "help-sandbox"); mkdirSync(packDir, { recursive: true }); mkdirSync(installDir, { recursive: true }); + mkdirSync(helpSandbox, { recursive: true }); - let failures = 0; - const checks = []; + const recorder = createRecorder(); + let client = null; + let aborted = null; - function record(name, passed, detail) { - checks.push({ name, passed, detail }); - if (!passed) failures++; - console.log(`${passed ? "PASS" : "FAIL"} ${name}${!passed && detail ? ` -> ${detail}` : ""}`); + function record(id, outcome) { + const entry = recorder.record(id, outcome); + console.log(`${entry.passed ? "PASS" : "FAIL"} ${entry.name}${entry.detail ? ` -> ${entry.detail}` : ""}`); } try { // --- Step 1: Build and pack ------------------------------------------- console.log("\n=== Step 1: Build and pack ==="); - runOrDie("npm", ["ci"], { cwd: repoRoot }); + runOrDie("npm", ["ci"], { cwd: repoRoot, timeout: 300000 }); runOrDie("npm", ["run", "build"], { cwd: repoRoot }); - runOrDie("npm", ["run", "build:extension"], { cwd: repoRoot }); + runOrDie("npm", ["run", "build:extension"], { cwd: repoRoot, timeout: 600000 }); const packResult = runOrDie("npm", ["pack", "--json"], { cwd: repoRoot }); const packMeta = JSON.parse(packResult.stdout)[0]; const tarballPath = join(repoRoot, packMeta.filename); const tarballDest = join(packDir, packMeta.filename); cpSync(tarballPath, tarballDest); + // `npm pack` writes the tarball into the repo; removing it is what keeps + // the source tree unmodified, and repo.tree-unchanged proves it did. rmSync(tarballPath, { force: true }); - record("npm pack produces a tarball", existsSync(tarballDest), packMeta.filename); - record( - "tarball contains dist/index.js", - packMeta.files?.some((f) => f.path === "dist/index.js"), - JSON.stringify(packMeta.files?.map((f) => f.path)), - ); - record( - "tarball contains .mcp.json", - packMeta.files?.some((f) => f.path === ".mcp.json"), - ); - record( - "tarball contains hooks/", - packMeta.files?.some((f) => f.path.startsWith("hooks/")), - ); + record("pack.tarball", checkTarballExists(existsSync(tarballDest), packMeta.filename)); + record("pack.dist-index", checkPackContainsPath(packMeta, "dist/index.js")); + record("pack.mcp-json", checkPackContainsPath(packMeta, ".mcp.json")); + record("pack.hooks", checkPackContainsPrefix(packMeta, "hooks/")); // --- Step 2: Install in disposable dir -------------------------------- console.log("\n=== Step 2: Install from tarball ==="); - const installResult = run("npm", ["install", tarballDest], { cwd: installDir }); - record("npm install from tarball succeeds", installResult.status === 0, installResult.stderr?.slice(-500)); - - // Verify the installed package has the expected bin entry - const installedBin = join(installDir, "node_modules", "@workspacejson", "codex-mcp", "scripts", "install.mjs"); - record("installed package has scripts/install.mjs", existsSync(installedBin)); + const installResult = run("npm", ["install", tarballDest], { cwd: installDir, timeout: 300000 }); + record("install.from-tarball", checkProcessSucceeded(installResult, "npm install")); - const installedMain = join(installDir, "node_modules", "@workspacejson", "codex-mcp", "dist", "index.js"); - record("installed package has dist/index.js", existsSync(installedMain)); + const installedPkg = join(installDir, "node_modules", "@workspacejson", "codex-mcp"); + const installedBin = join(installedPkg, "scripts", "install.mjs"); + const installedMain = join(installedPkg, "dist", "index.js"); + record("install.has-install-script", checkPathExists(existsSync(installedBin), installedBin)); + record("install.has-dist-index", checkPathExists(existsSync(installedMain), installedMain)); // --- Step 3: Start MCP server from installed package ------------------ console.log("\n=== Step 3: MCP server from packed artifact ==="); - // Copy fixture into install dir so the server can find workspace.json const fixtureDest = join(installDir, "fixture"); cpSync(fixtureRoot, fixtureDest, { recursive: true }); - // Start the server using the installed package's main entry - const serverPath = join(installDir, "node_modules", "@workspacejson", "codex-mcp", "dist", "index.js"); - - // Use MCP client to test the server const { Client } = await import("@modelcontextprotocol/sdk/client/index.js"); const { StdioClientTransport } = await import("@modelcontextprotocol/sdk/client/stdio.js"); const transport = new StdioClientTransport({ - command: "node", - args: [serverPath], + command: process.execPath, + args: [installedMain], env: { ...process.env, WORKSPACE_JSON_ROOT: fixtureDest }, }); - const client = new Client({ name: "consume-harness", version: "0.0.0" }); + client = new Client({ name: "consume-harness", version: "0.0.0" }); await client.connect(transport); // --- Step 4: Smoke test the packed server ----------------------------- console.log("\n=== Step 4: Smoke test packed server ==="); - const instr = client.getInstructions(); - record("server instructions contain FRAGILE", instr?.includes("FRAGILE"), "missing instructions"); - - const toolResult = await client.callTool({ name: "workspace_list_fragile_files", arguments: {} }); - record("server responds to tool calls", !toolResult.isError); - - const { tools: availableTools } = await client.listTools(); - const toolNames = availableTools.map((t) => t.name).sort(); - record( - "tools/list returns 4 expected tools", - JSON.stringify(toolNames) === - JSON.stringify([ - "workspace_assess_change", - "workspace_get_cochange_partners", - "workspace_get_file_context", - "workspace_list_fragile_files", - ]), - toolNames.join(","), - ); + record("mcp.instructions-fragile", checkInstructionsMention(client.getInstructions(), "FRAGILE")); + + const listResult = await client.callTool({ name: "workspace_list_fragile_files", arguments: {} }); + record("mcp.tool-call-responds", checkToolCallResponded(listResult)); - // Test file context retrieval through the packed artifact - const r1 = await client.callTool({ + record("mcp.tools-list", checkToolsList(await client.listTools())); + + const fileContext = await client.callTool({ name: "workspace_get_file_context", arguments: { path: "src/routes/checkout.ts" }, }); - const s1 = r1.structuredContent; - record( - "file context returns fragility tier", - s1?.fragility?.tier !== undefined, - JSON.stringify(s1?.fragility?.tier), - ); - record( - "file context returns evidence", - Array.isArray(s1?.fragility?.evidence), - JSON.stringify(s1?.fragility?.evidence), - ); + record("mcp.file-context-tier", checkFragilityTier(fileContext)); + record("mcp.file-context-evidence", checkFragilityEvidence(fileContext)); - // Test co-change partners - const r2 = await client.callTool({ + const cochange = await client.callTool({ name: "workspace_get_cochange_partners", arguments: { path: "src/routes/checkout.ts" }, }); - const s2 = r2.structuredContent; - record("co-change partners returns array", Array.isArray(s2?.partners), JSON.stringify(s2?.partners)); + record("mcp.cochange-partners", checkCochangePartners(cochange)); - // Test assess change - const r3 = await client.callTool({ + const assess = await client.callTool({ name: "workspace_assess_change", arguments: { paths: ["src/routes/checkout.ts"] }, }); - const s3 = r3.structuredContent; - record( - "assess change returns decision", - s3?.action !== undefined || s3?.assessments !== undefined, - JSON.stringify(s3), - ); - - await client.close(); + record("mcp.assess-change", checkAssessDecision(assess)); // --- Step 5: Hook from packed artifact -------------------------------- console.log("\n=== Step 5: Hook from packed artifact ==="); - const hookPath = join(installDir, "node_modules", "@workspacejson", "codex-mcp", "hooks", "pre-edit-check.mjs"); - record("hook script exists in packed artifact", existsSync(hookPath)); + const hookPath = join(installedPkg, "hooks", "pre-edit-check.mjs"); + record("hook.exists", checkPathExists(existsSync(hookPath), hookPath)); - const hookResult = run("node", [hookPath, "--paths", "src/routes/checkout.ts"], { - cwd: fixtureDest, - }); - record( - "hook exits non-zero on evidenced-fragile without partners", - hookResult.status !== 0, - `exit ${hookResult.status}`, - ); - record( - "hook output mentions FRAGILE or deny", - /FRAGILE|deny|block/i.test(hookResult.stdout + hookResult.stderr), - (hookResult.stdout + hookResult.stderr).slice(-500), - ); + const hookResult = run(process.execPath, [hookPath, "--paths", "src/routes/checkout.ts"], { cwd: fixtureDest }); + record("hook.denies", checkHookDenies(hookResult)); + record("hook.output-mentions-deny", checkHookOutputMentionsDeny(hookResult)); // --- Step 6: Installer from packed artifact --------------------------- console.log("\n=== Step 6: Installer from packed artifact ==="); - const installScript = join(installDir, "node_modules", "@workspacejson", "codex-mcp", "scripts", "install.mjs"); - record("installer script exists in packed artifact", existsSync(installScript)); - - const installHelp = run("node", [installScript, "--help"]); - const installHelpOutput = (installHelp.stdout ?? "") + (installHelp.stderr ?? ""); - record( - "installer --help works from packed artifact", - installHelp.status === 0 && /Usage:/i.test(installHelpOutput), - installHelpOutput.slice(-500), - ); + const installScript = join(installedPkg, "scripts", "install.mjs"); + record("installer.exists", checkPathExists(existsSync(installScript), installScript)); + + // Help runs inside the disposable sandbox, never the source repo. If help + // ever regresses into an install again, it installs into a directory we + // then inspect — and installer.help-nondestructive goes red — instead of + // silently rewriting this checkout's .codex/config.toml. + const installHelp = run(process.execPath, [installScript, "--help"], { cwd: helpSandbox }); + record("installer.help-usage", checkInstallerHelp(installHelp)); + record("installer.help-nondestructive", checkHelpWroteNothing(readdirSync(helpSandbox))); + } catch (err) { + // The run is incomplete, not passing. Unreached checks stay `not_run` and + // the verdict cannot be CONSUMABLE. + aborted = { message: err instanceof Error ? err.message : String(err) }; + console.error(`\nHarness aborted: ${aborted.message}`); } finally { - // --- Write receipt ----------------------------------------------------- - const receipt = { - $comment: - "Standard-candidate consumption harness receipt. Generated by scripts/migration/consume-standard-candidate.mjs — do not hand-edit.", - generatedAt: new Date().toISOString(), - summary: { - total: checks.length, - passed: checks.filter((c) => c.passed).length, - failed: checks.filter((c) => !c.passed).length, - }, - verdict: failures === 0 ? "CONSUMABLE" : "NOT_CONSUMABLE", - checks: checks.map((c) => ({ name: c.name, status: c.passed ? "pass" : "fail", detail: c.detail ?? null })), - }; - - if (outDir) { - mkdirSync(outDir, { recursive: true }); - writeFileSync(join(outDir, "consumption-receipt.json"), `${JSON.stringify(receipt, null, 2)}\n`); - console.log(`\nReceipt: ${join(outDir, "consumption-receipt.json")}`); + // Close the client whatever happened. With the original happy-path-only + // close, any throw between connect and close leaked the spawned MCP server + // and hung CI instead of failing it. + if (client) { + try { + await client.close(); + } catch (closeErr) { + console.error(`Warning: MCP client close failed: ${closeErr.message}`); + } + } + + // --- Step 7: Source-tree safety --------------------------------------- + // Recorded last so it covers everything the harness did, and recorded even + // on abort — an aborted run is exactly when a stray mutation is likeliest. + try { + record("repo.tree-unchanged", checkTreeUnchanged(treeBefore, treeFingerprint())); + } catch (recordErr) { + console.error(`Warning: could not record tree check: ${recordErr.message}`); + } + + const receipt = recorder.buildReceipt({ aborted }); + + if (opts.outDir) { + mkdirSync(opts.outDir, { recursive: true }); + const receiptPath = join(opts.outDir, "consumption-receipt.json"); + writeFileSync(receiptPath, `${JSON.stringify(receipt, null, 2)}\n`); + console.log(`\nReceipt: ${receiptPath}`); } + const { total, passed, failed, notRun } = receipt.summary; console.log( - `\nConsumption verdict: ${receipt.verdict} — ${receipt.summary.passed}/${receipt.summary.total} passed, ${receipt.summary.failed} failed.`, + `\nConsumption verdict: ${receipt.verdict} — ${passed}/${total} passed, ${failed} failed, ${notRun} not run.`, ); + if (aborted) console.log(`Aborted before completion: ${aborted.message}`); + rmSync(work, { recursive: true, force: true }); - process.exitCode = failures > 0 ? 1 : 0; + process.exitCode = receipt.verdict === "CONSUMABLE" ? 0 : 1; } } -main().catch((err) => { - console.error("consume-standard-candidate crashed:", err); - process.exitCode = 2; -}); +const isDirectRun = process.argv[1] === fileURLToPath(import.meta.url); +if (isDirectRun) { + main().catch((err) => { + console.error("consume-standard-candidate crashed:", err); + process.exitCode = 2; + }); +} diff --git a/scripts/migration/consumption-checks.mjs b/scripts/migration/consumption-checks.mjs new file mode 100644 index 0000000..4d228d3 --- /dev/null +++ b/scripts/migration/consumption-checks.mjs @@ -0,0 +1,434 @@ +/** + * Check predicates and receipt construction for the standard-candidate + * consumption harness (META-285 / META-235 Step 5 / META-258 I-7). + * + * This module exists so that every check the harness runs has exactly ONE + * implementation, which the harness calls and the tests exercise. The + * alternative — the harness asserting inline while a test reimplements the + * same logic — is the one-concept-two-implementations class tracked in + * META-140, and it already bit `verify-receipt.test.ts`. + * + * Contract for every predicate here (the META-165 scope amendment): + * + * No verification check enters service until it has been demonstrated to + * FAIL when the property is broken and to PASS when it is not. + * + * Two consequences shape the code below: + * + * 1. Predicates take plain data, never live handles, so a test can hand them + * a deliberately broken input without standing up an MCP server. + * 2. Predicates never infer success from absence. A `spawnSync` result that + * never launched, a result object of the wrong shape, and a missing + * optional field are all explicit failures with a stated reason — not a + * falsy value that happens to land on the passing side of a comparison. + * + * Defect 2 in META-238 and defect 1 in META-285 are the same root cause read + * off a property nobody verified. There the silent `undefined` produced a + * vacuous pass; here it produced a guaranteed fail. The direction was luck. + * These predicates remove the luck by checking shape before value. + */ + +/** + * Every check the harness is expected to run, in execution order. + * + * The plan is declared up front rather than accumulated as checks execute. + * That is what lets the receipt distinguish "this check failed" from "this + * check never ran": a harness that throws midway leaves the unreached entries + * as `not_run` instead of silently shortening the list and reporting a clean + * sweep of whatever happened to complete first. + */ +export const CHECK_PLAN = [ + { id: "pack.tarball", step: 1, name: "npm pack produces a tarball" }, + { id: "pack.dist-index", step: 1, name: "tarball contains dist/index.js" }, + { id: "pack.mcp-json", step: 1, name: "tarball contains .mcp.json" }, + { id: "pack.hooks", step: 1, name: "tarball contains hooks/" }, + { id: "install.from-tarball", step: 2, name: "npm install from tarball succeeds" }, + { id: "install.has-install-script", step: 2, name: "installed package has scripts/install.mjs" }, + { id: "install.has-dist-index", step: 2, name: "installed package has dist/index.js" }, + { id: "mcp.instructions-fragile", step: 4, name: "server instructions contain FRAGILE" }, + { id: "mcp.tool-call-responds", step: 4, name: "server responds to tool calls" }, + { id: "mcp.tools-list", step: 4, name: "tools/list returns the 4 expected tools" }, + { id: "mcp.file-context-tier", step: 4, name: "file context returns fragility tier" }, + { id: "mcp.file-context-evidence", step: 4, name: "file context returns evidence" }, + { id: "mcp.cochange-partners", step: 4, name: "co-change partners returns array" }, + { id: "mcp.assess-change", step: 4, name: "assess change returns decision" }, + { id: "hook.exists", step: 5, name: "hook script exists in packed artifact" }, + { id: "hook.denies", step: 5, name: "hook exits non-zero on evidenced-fragile without partners" }, + { id: "hook.output-mentions-deny", step: 5, name: "hook output mentions FRAGILE or deny" }, + { id: "installer.exists", step: 6, name: "installer script exists in packed artifact" }, + { id: "installer.help-usage", step: 6, name: "installer --help works from packed artifact" }, + { id: "installer.help-nondestructive", step: 6, name: "installer --help writes no config into its cwd" }, + { id: "repo.tree-unchanged", step: 7, name: "harness leaves the source working tree unmodified" }, +]; + +export const EXPECTED_TOOLS = [ + "workspace_assess_change", + "workspace_get_cochange_partners", + "workspace_get_file_context", + "workspace_list_fragile_files", +]; + +const pass = (detail) => ({ passed: true, detail: detail ?? null }); +const fail = (detail) => ({ passed: false, detail: detail ?? null }); + +// ── Step 1: pack inventory ──────────────────────────────────────────────── + +/** + * `npm pack --json` reports the files it placed in the tarball. Shape is + * verified before use: a missing or non-array `files` means we learned + * nothing about the tarball, which is a failure, not an absent violation. + */ +function packFilePaths(packMeta) { + if (!packMeta || typeof packMeta !== "object") { + return { error: "pack metadata is not an object" }; + } + if (!Array.isArray(packMeta.files)) { + return { error: `pack metadata has no files[] array (got ${typeof packMeta.files})` }; + } + const paths = []; + for (const entry of packMeta.files) { + if (!entry || typeof entry.path !== "string") { + return { error: `pack metadata contains an entry without a string path: ${JSON.stringify(entry)}` }; + } + paths.push(entry.path); + } + return { paths }; +} + +export function checkTarballExists(tarballExists, filename) { + return tarballExists + ? pass(filename ?? null) + : fail(`tarball not found at destination (${filename ?? "no filename"})`); +} + +/** Exact-path membership, for files that ship at a known path. */ +export function checkPackContainsPath(packMeta, wanted) { + const { paths, error } = packFilePaths(packMeta); + if (error) return fail(error); + if (paths.includes(wanted)) return pass(wanted); + return fail(`'${wanted}' absent from ${paths.length} packed files`); +} + +/** + * Prefix membership, for directories. `npm pack --json` emits individual file + * paths and never a directory entry, so `path === "hooks/"` can never match — + * META-285 defect 2. A directory is present iff some packed file lives under it. + */ +export function checkPackContainsPrefix(packMeta, prefix) { + const { paths, error } = packFilePaths(packMeta); + if (error) return fail(error); + const matches = paths.filter((p) => p.startsWith(prefix)); + if (matches.length > 0) return pass(`${matches.length} file(s) under '${prefix}': ${matches.slice(0, 5).join(", ")}`); + return fail(`no packed file starts with '${prefix}' (${paths.length} packed files)`); +} + +// ── Process results ─────────────────────────────────────────────────────── + +/** + * Classify a `spawnSync` result before reading its status. + * + * `spawnSync` returns `status: null` when the process never exited normally — + * it failed to launch (ENOENT) or died on a signal. Any predicate that reads + * `status` without this guard is unsound in one direction or the other: + * `status === 0` turns a launch failure into a fail (survivable), while + * `status !== 0` turns a launch failure into a PASS (vacuous — META-285 + * defect, `hook.denies` at the original `:206`). + */ +function launched(result) { + if (!result || typeof result !== "object") return "no spawn result"; + if (result.error) return `process failed to launch: ${result.error.message}`; + if (result.status === null || result.status === undefined) { + return `process did not exit normally (signal ${result.signal ?? "unknown"})`; + } + return null; +} + +export function checkProcessSucceeded(result, what) { + const problem = launched(result); + if (problem) return fail(`${what}: ${problem}`); + if (result.status !== 0) return fail(`${what}: exit ${result.status} ${(result.stderr ?? "").slice(-300)}`.trim()); + return pass(`${what}: exit 0`); +} + +export function checkPathExists(exists, path) { + return exists ? pass(path) : fail(`not found: ${path}`); +} + +// ── Step 4: MCP server responses ────────────────────────────────────────── + +export function checkInstructionsMention(instructions, token) { + if (typeof instructions !== "string") { + return fail( + `server returned no instructions string (got ${instructions === undefined ? "undefined" : typeof instructions})`, + ); + } + if (!instructions.includes(token)) { + return fail(`instructions present (${instructions.length} chars) but do not mention '${token}'`); + } + return pass(`instructions mention '${token}'`); +} + +/** + * A tool call responded usefully. + * + * `isError` is optional on `CallToolResult`, so `!result.isError` alone passes + * for a malformed or empty result — it cannot distinguish "the server answered" + * from "we got something that isn't a tool result". `content` is required by + * the MCP spec, so its presence is what proves we are holding a real result. + */ +export function checkToolCallResponded(result) { + if (!result || typeof result !== "object") { + return fail(`tool result is not an object (got ${result === undefined ? "undefined" : typeof result})`); + } + if (result.isError === true) { + return fail(`server reported isError: ${JSON.stringify(result.content ?? null).slice(0, 300)}`); + } + if (!Array.isArray(result.content)) { + return fail(`tool result has no content[] array (got ${typeof result.content}) — result shape unverified`); + } + return pass(`content entries: ${result.content.length}`); +} + +export function checkToolsList(listResult, expected = EXPECTED_TOOLS) { + if (!listResult || typeof listResult !== "object") { + return fail( + `tools/list result is not an object (got ${listResult === undefined ? "undefined" : typeof listResult})`, + ); + } + if (!Array.isArray(listResult.tools)) { + return fail(`tools/list result has no tools[] array (got ${typeof listResult.tools})`); + } + for (const tool of listResult.tools) { + if (!tool || typeof tool.name !== "string") { + return fail(`tools/list contains an entry without a string name: ${JSON.stringify(tool)}`); + } + } + const byName = (a, b) => a.localeCompare(b); + const actual = listResult.tools.map((t) => t.name).sort(byName); + const want = [...expected].sort(byName); + if (JSON.stringify(actual) !== JSON.stringify(want)) { + return fail(`expected [${want.join(", ")}], got [${actual.join(", ")}]`); + } + return pass(actual.join(",")); +} + +function structuredContent(result) { + if (!result || typeof result !== "object") { + return { error: `tool result is not an object (got ${result === undefined ? "undefined" : typeof result})` }; + } + if (result.isError === true) return { error: "server reported isError" }; + if (!result.structuredContent || typeof result.structuredContent !== "object") { + return { error: `tool result has no structuredContent object (got ${typeof result.structuredContent})` }; + } + return { value: result.structuredContent }; +} + +export function checkFragilityTier(result) { + const { value, error } = structuredContent(result); + if (error) return fail(error); + const tier = value.fragility?.tier; + if (tier === undefined || tier === null) + return fail(`structuredContent.fragility.tier absent: ${JSON.stringify(value).slice(0, 300)}`); + return pass(`tier=${JSON.stringify(tier)}`); +} + +export function checkFragilityEvidence(result) { + const { value, error } = structuredContent(result); + if (error) return fail(error); + if (!Array.isArray(value.fragility?.evidence)) { + return fail(`structuredContent.fragility.evidence is not an array (got ${typeof value.fragility?.evidence})`); + } + return pass(`${value.fragility.evidence.length} evidence entries`); +} + +export function checkCochangePartners(result) { + const { value, error } = structuredContent(result); + if (error) return fail(error); + if (!Array.isArray(value.partners)) { + return fail(`structuredContent.partners is not an array (got ${typeof value.partners})`); + } + return pass(`${value.partners.length} partners`); +} + +export function checkAssessDecision(result) { + const { value, error } = structuredContent(result); + if (error) return fail(error); + if (value.action === undefined && value.assessments === undefined) { + return fail(`structuredContent has neither 'action' nor 'assessments': ${JSON.stringify(value).slice(0, 300)}`); + } + return pass(JSON.stringify({ action: value.action, assessments: value.assessments }).slice(0, 300)); +} + +// ── Step 5: hook behaviour ──────────────────────────────────────────────── + +/** + * Node exited non-zero because it could not run the script at all, rather than + * because the script decided something. + * + * `node ` launches successfully — the node binary + * exists — and exits 1 after failing to load. A predicate that only asks + * "non-zero?" reads that crash as a deny, so a candidate shipping NO hook at + * all scores a passing deny check. This is the same vacuous-pass shape as the + * original `status !== 0`, one level further in, and it was caught by running + * the harness against a candidate with `hooks/` removed from `files` rather + * than by any unit case. + */ +function crashedInsteadOfRunning(result) { + const output = `${result.stdout ?? ""}${result.stderr ?? ""}`; + if (/ERR_MODULE_NOT_FOUND|ERR_UNKNOWN_FILE_EXTENSION|Cannot find module/.test(output)) { + return "node could not load the hook script (module not found)"; + } + // Horizontal whitespace only: under /m, `\s*` also matches newlines, so the + // indent could be consumed across line boundaries — ambiguity that both + // backtracks super-linearly and lets a non-indented "at" line match. + if (/^[ \t]*at .*node:internal/m.test(output) && /Error\b/.test(output)) { + return "node exited on an unhandled error before the hook reached a decision"; + } + return null; +} + +/** + * The hook must actively deny — launch, run to a decision, and exit non-zero. + * A hook that cannot launch, or that crashes before deciding, is a failure of + * the packed artifact, not a denial. + */ +export function checkHookDenies(result) { + const problem = launched(result); + if (problem) return fail(`hook did not run: ${problem}`); + if (result.status === 0) return fail("hook exited 0; expected a non-zero deny on an evidenced-fragile path"); + const crash = crashedInsteadOfRunning(result); + if (crash) return fail(`${crash} — exit ${result.status} is a crash, not a deny`); + return pass(`exit ${result.status}`); +} + +export function checkHookOutputMentionsDeny(result) { + const problem = launched(result); + if (problem) return fail(`hook did not run: ${problem}`); + const output = `${result.stdout ?? ""}${result.stderr ?? ""}`; + if (output.trim() === "") return fail("hook produced no output"); + if (!/FRAGILE|deny|block/i.test(output)) return fail(`output mentions no denial: ${output.slice(-300)}`); + return pass(output.slice(-200).trim()); +} + +// ── Step 6: installer help ──────────────────────────────────────────────── + +/** + * `--help` must print usage and exit 0. + * + * Checked against combined stdout+stderr deliberately. The original harness + * read `stdout` only while the installer wrote `USAGE` to `console.error`, so + * the check could not pass even when help worked. Which stream carries usage + * is a presentation choice; that help is shown at all is the property. + */ +export function checkInstallerHelp(result) { + const problem = launched(result); + if (problem) return fail(`installer did not run: ${problem}`); + const output = `${result.stdout ?? ""}${result.stderr ?? ""}`; + if (result.status !== 0) return fail(`--help exited ${result.status}: ${output.slice(-300)}`); + if (!/Usage:/i.test(output)) return fail(`--help exited 0 but printed no usage: ${output.slice(-300)}`); + return pass(`exit 0, usage printed (${output.length} chars)`); +} + +/** + * `--help` must not install anything into the directory it runs in. + * + * This is the positive form of the META-285 stop-now item. Asserting only that + * the SOURCE tree is clean is weaker: it passes for the wrong reason whenever + * the source tree is already dirty, and it cannot see a destructive install + * aimed anywhere else. Running help inside a disposable sandbox and proving + * the sandbox stayed empty detects the destructive behaviour directly. + * + * `artifactsAfter` is the list of paths present in the sandbox after the run. + */ +export function checkHelpWroteNothing(artifactsAfter) { + if (!Array.isArray(artifactsAfter)) { + return fail(`sandbox listing unavailable (got ${typeof artifactsAfter}) — cannot prove --help was non-destructive`); + } + if (artifactsAfter.length > 0) { + return fail(`--help wrote into its cwd: ${artifactsAfter.join(", ")}`); + } + return pass("sandbox empty after --help"); +} + +// ── Step 7: source-tree safety ──────────────────────────────────────────── + +/** + * The harness must not modify the repository it verifies. + * + * Compares `git status --porcelain` before and after. Comparing before/after + * rather than requiring "clean" means the check still proves the harness made + * no change when started from an already-dirty checkout, and it cannot be + * satisfied by an unrelated pre-existing dirty state. + */ +export function checkTreeUnchanged(before, after) { + if (typeof before !== "string" || typeof after !== "string") { + return fail("git status unavailable before or after the run — tree safety not established"); + } + if (before !== after) { + const beforeSet = new Set(before.split("\n").filter(Boolean)); + const changed = after.split("\n").filter((line) => line && !beforeSet.has(line)); + return fail(`harness modified the source tree: ${changed.join(" | ") || "(entries removed)"}`); + } + return pass(before.trim() === "" ? "tree clean and unmodified" : "tree unmodified (pre-existing changes preserved)"); +} + +// ── Recorder / receipt ──────────────────────────────────────────────────── + +/** + * Records outcomes against the declared plan and builds the receipt. + * + * Rejects unknown and duplicate check ids. A harness that stops running a + * planned check then surfaces it as `not_run`; one that renames a check fails + * loudly here rather than quietly shrinking the evidence set. + */ +export function createRecorder(plan = CHECK_PLAN) { + const known = new Map(plan.map((c) => [c.id, c])); + const results = new Map(); + + return { + record(id, outcome) { + const spec = known.get(id); + if (!spec) throw new Error(`consumption-checks: unknown check id '${id}' — not in CHECK_PLAN`); + if (results.has(id)) throw new Error(`consumption-checks: check '${id}' recorded twice`); + if (!outcome || typeof outcome.passed !== "boolean") { + throw new Error(`consumption-checks: check '${id}' recorded a malformed outcome: ${JSON.stringify(outcome)}`); + } + results.set(id, outcome); + return { ...spec, ...outcome }; + }, + + buildReceipt({ generatedAt, aborted = null } = {}) { + const checks = plan.map(({ id, name, step }) => { + const outcome = results.get(id); + if (!outcome) return { id, name, step, status: "not_run", detail: null }; + return { id, name, step, status: outcome.passed ? "pass" : "fail", detail: outcome.detail ?? null }; + }); + + const count = (status) => checks.filter((c) => c.status === status).length; + const failed = count("fail"); + const notRun = count("not_run"); + + // Precedence is deliberate. A real failure outranks incompleteness, and + // CONSUMABLE requires that every planned check actually ran. The old + // harness computed `failures === 0 ? "CONSUMABLE" : ...` over only the + // checks that happened to execute, so a crash after seven passing checks + // produced "7/7 passed — CONSUMABLE" for a harness that never finished. + const verdict = failed > 0 ? "NOT_CONSUMABLE" : notRun > 0 ? "INCOMPLETE" : "CONSUMABLE"; + + return { + $comment: + "Standard-candidate consumption harness receipt. Generated by scripts/migration/consume-standard-candidate.mjs — do not hand-edit.", + generatedAt: generatedAt ?? new Date().toISOString(), + verdict, + aborted, + summary: { total: checks.length, passed: count("pass"), failed, notRun }, + checks, + }; + }, + }; +} + +/** A receipt is admissible evidence of consumability only when nothing failed AND nothing was skipped. */ +export function receiptIsClean(receipt) { + return receipt.verdict === "CONSUMABLE"; +} diff --git a/scripts/migration/verify-receipt.mjs b/scripts/migration/verify-receipt.mjs index 861d5fe..90cf975 100644 --- a/scripts/migration/verify-receipt.mjs +++ b/scripts/migration/verify-receipt.mjs @@ -24,12 +24,95 @@ * node scripts/migration/verify-receipt.mjs */ import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; -function readJson(path) { - return JSON.parse(readFileSync(path, "utf8")); +/** + * Read a receipt, failing with a message a CI reader can act on. + * + * A raw `ENOENT` or `SyntaxError` stack from a gate script says nothing about + * which of the two receipts was bad or what was expected of it. `label` names + * the role so the failure is legible without opening the script. + */ +export function readReceipt(path, label = "receipt") { + let raw; + try { + raw = readFileSync(path, "utf8"); + } catch (err) { + if (err.code === "ENOENT") { + throw new Error(`${label} not found: ${path}\nExpected a parity receipt JSON file at this path.`); + } + throw new Error(`${label} could not be read: ${path}\n${err.message}`); + } + + let parsed; + try { + parsed = JSON.parse(raw); + } catch (err) { + throw new Error(`${label} is not valid JSON: ${path}\n${err.message}`); + } + + if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) { + const kind = Array.isArray(parsed) ? "array" : parsed === null ? "null" : typeof parsed; + throw new Error(`${label} is not a receipt object: ${path}\nParsed a ${kind}.`); + } + return parsed; +} + +/** + * Reject a receipt missing the fields the comparator reads. + * + * Without this, a receipt lacking `summary` or `checks` throws a raw + * `TypeError` from inside the comparison, and a well-formed receipt with an + * empty check set compares clean against another empty one — a vacuous pass + * on the gate's own input. + * + * Duplicate ids are rejected for the same reason. The comparator projects + * `checks` into a `Map` keyed by id, which keeps only the last entry for a + * repeated id and silently discards the earlier one. A receipt carrying the + * same check twice with divergent statuses would then compare clean against + * the survivor — an internally inconsistent receipt reported as matching. + * `createRecorder` in ./consumption-checks.mjs already refuses to *emit* such + * a receipt; this refuses to *read* one, so neither end of the gate trusts it. + */ +function assertReceiptShape(receipt, label) { + const problems = []; + if (typeof receipt.verdict !== "string") problems.push("missing 'verdict' string"); + if (!receipt.summary || typeof receipt.summary !== "object") problems.push("missing 'summary' object"); + if (!Array.isArray(receipt.checks)) problems.push("missing 'checks' array"); + else if (receipt.checks.length === 0) problems.push("'checks' is empty — nothing to compare"); + else if (receipt.checks.some((c) => !c || typeof c.id !== "string")) { + problems.push("'checks' contains an entry without a string 'id'"); + } else { + const seen = new Set(); + const duplicated = new Set(); + for (const { id } of receipt.checks) { + if (seen.has(id)) duplicated.add(id); + seen.add(id); + } + if (duplicated.size > 0) { + const ids = [...duplicated].sort((a, b) => a.localeCompare(b)).join(", "); + problems.push(`'checks' contains duplicate ids: ${ids}`); + } + } + + if (problems.length > 0) { + throw new Error(`${label} is not a usable parity receipt:\n${problems.map((p) => ` - ${p}`).join("\n")}`); + } } -function compareReceipts(reference, candidate) { +/** + * The single comparator implementation. + * + * Exported so `tests/migration/verify-receipt.test.ts` exercises THIS function + * rather than a copy. The test previously reimplemented it inline, so the two + * drifted — the copy's violation messages already differed from these — and + * the tests could not have caught a regression in the comparator that actually + * runs in CI (META-140 defect class, recorded on META-285). + */ +export function compareReceipts(reference, candidate) { + assertReceiptShape(reference, "reference receipt"); + assertReceiptShape(candidate, "candidate receipt"); + const violations = []; if (reference.verdict !== candidate.verdict) { @@ -91,10 +174,19 @@ function main() { process.exit(2); } - const reference = readJson(refPath); - const candidate = readJson(candPath); - - const violations = compareReceipts(reference, candidate); + let reference; + let candidate; + let violations; + try { + reference = readReceipt(refPath, "reference receipt"); + candidate = readReceipt(candPath, "candidate receipt"); + violations = compareReceipts(reference, candidate); + } catch (err) { + // Exit 2 — a gate that could not read its inputs did not run. That is + // distinct from exit 1, "the receipts diverged", which is a real result. + console.error(`Receipt reproduction COULD NOT RUN:\n\n${err.message}\n`); + process.exit(2); + } if (violations.length > 0) { console.error("Receipt reproduction FAILED — committed receipt does not match CI-generated receipt:\n"); @@ -110,4 +202,7 @@ function main() { ); } -main(); +const isDirectRun = process.argv[1] === fileURLToPath(import.meta.url); +if (isDirectRun) { + main(); +} diff --git a/tests/migration/consumption-checks.test.ts b/tests/migration/consumption-checks.test.ts new file mode 100644 index 0000000..249ccfd --- /dev/null +++ b/tests/migration/consumption-checks.test.ts @@ -0,0 +1,528 @@ +import { describe, expect, it } from "vitest"; +import { + CHECK_PLAN, + EXPECTED_TOOLS, + checkAssessDecision, + checkCochangePartners, + checkFragilityEvidence, + checkFragilityTier, + checkHelpWroteNothing, + checkHookDenies, + checkHookOutputMentionsDeny, + checkInstallerHelp, + checkInstructionsMention, + checkPackContainsPath, + checkPackContainsPrefix, + checkPathExists, + checkProcessSucceeded, + checkTarballExists, + checkToolCallResponded, + checkToolsList, + checkTreeUnchanged, + createRecorder, +} from "../../scripts/migration/consumption-checks.mjs"; + +// Watched-red contract (META-165 scope amendment, recorded on META-285): +// +// No verification check enters service until it has been demonstrated to +// FAIL when the property is broken and to PASS when it is not. +// +// Every standing check in CHECK_PLAN therefore appears below at least twice: +// once green against a valid input and once red against a controlled broken +// one. `covers()` records which plan entry each block proves, and a final test +// asserts the plan is fully covered — so adding a check to the harness without +// watching it go red fails the suite rather than shipping unproven. + +const proven = new Set(); +function covers(id: string): string { + proven.add(id); + return id; +} + +const okProc = (over: Record = {}) => ({ + status: 0, + signal: null, + stdout: "", + stderr: "", + error: undefined, + ...over, +}); + +/** What spawnSync actually returns when the binary does not exist. */ +const failedToLaunch = () => ({ + status: null, + signal: null, + stdout: null, + stderr: null, + error: Object.assign(new Error("spawnSync node ENOENT"), { code: "ENOENT" }), +}); + +/** What spawnSync returns when the child is killed (e.g. timeout). */ +const killedBySignal = () => okProc({ status: null, signal: "SIGTERM" }); + +describe(covers("pack.tarball"), () => { + it("green: tarball present at destination", () => { + expect(checkTarballExists(true, "codex-mcp-0.1.9.tgz").passed).toBe(true); + }); + it("red: tarball absent", () => { + const r = checkTarballExists(false, "codex-mcp-0.1.9.tgz"); + expect(r.passed).toBe(false); + expect(r.detail).toMatch(/not found/); + }); +}); + +describe("pack inventory", () => { + const packMeta = { + filename: "codex-mcp-0.1.9.tgz", + files: [ + { path: "dist/index.js" }, + { path: "dist/index.d.ts" }, + { path: ".mcp.json" }, + { path: "hooks/pre-edit-check.mjs" }, + { path: "hooks/hooks.json" }, + { path: "scripts/install.mjs" }, + ], + }; + + it(`green: ${covers("pack.dist-index")} / ${covers("pack.mcp-json")} present`, () => { + expect(checkPackContainsPath(packMeta, "dist/index.js").passed).toBe(true); + expect(checkPackContainsPath(packMeta, ".mcp.json").passed).toBe(true); + }); + + it("red: a required file is missing from the tarball", () => { + const without = { ...packMeta, files: packMeta.files.filter((f) => f.path !== ".mcp.json") }; + const r = checkPackContainsPath(without, ".mcp.json"); + expect(r.passed).toBe(false); + expect(r.detail).toMatch(/absent from 5 packed files/); + }); + + it(`green: ${covers("pack.hooks")} matches files under the directory`, () => { + const r = checkPackContainsPrefix(packMeta, "hooks/"); + expect(r.passed).toBe(true); + expect(r.detail).toMatch(/2 file\(s\) under 'hooks\/'/); + }); + + it("red: no files under the directory", () => { + const without = { ...packMeta, files: packMeta.files.filter((f) => !f.path.startsWith("hooks/")) }; + expect(checkPackContainsPrefix(without, "hooks/").passed).toBe(false); + }); + + // META-285 defect 2 regression. `npm pack --json` emits file paths and never + // a directory entry, so an exact `path === "hooks/"` test could never pass. + // A prefix match must still succeed on that real-world inventory shape. + it("regression: passes on an inventory that contains no directory entries", () => { + expect(packMeta.files.some((f) => f.path === "hooks/")).toBe(false); + expect(checkPackContainsPrefix(packMeta, "hooks/").passed).toBe(true); + }); + + // Shape is verified before value: an inventory we could not read is a + // failure, not an absence of violations. + it("red: pack metadata without a files[] array fails rather than reporting absence", () => { + expect(checkPackContainsPath({ filename: "x.tgz" }, "dist/index.js").passed).toBe(false); + expect(checkPackContainsPrefix(undefined, "hooks/").passed).toBe(false); + expect(checkPackContainsPath({ files: [{ notAPath: 1 }] }, "dist/index.js").detail).toMatch( + /without a string path/, + ); + }); +}); + +describe(covers("install.from-tarball"), () => { + it("green: install exits 0", () => { + expect(checkProcessSucceeded(okProc(), "npm install").passed).toBe(true); + }); + it("red: install exits non-zero", () => { + const r = checkProcessSucceeded(okProc({ status: 1, stderr: "E404 not found" }), "npm install"); + expect(r.passed).toBe(false); + expect(r.detail).toMatch(/exit 1/); + }); + // Guards the direction the original harness got wrong elsewhere: a process + // that never launched must not be read as a result. + it("red: install never launched", () => { + const r = checkProcessSucceeded(failedToLaunch(), "npm install"); + expect(r.passed).toBe(false); + expect(r.detail).toMatch(/failed to launch/); + }); +}); + +describe(`${covers("install.has-install-script")} / ${covers("install.has-dist-index")} / ${covers("hook.exists")} / ${covers("installer.exists")}`, () => { + it("green: path exists", () => { + expect(checkPathExists(true, "/pkg/dist/index.js").passed).toBe(true); + }); + it("red: path missing", () => { + const r = checkPathExists(false, "/pkg/dist/index.js"); + expect(r.passed).toBe(false); + expect(r.detail).toMatch(/not found: \/pkg\/dist\/index\.js/); + }); +}); + +describe(covers("mcp.instructions-fragile"), () => { + it("green: instructions mention the token", () => { + expect(checkInstructionsMention("Treat FRAGILE files with care", "FRAGILE").passed).toBe(true); + }); + it("red: instructions present but do not mention the token", () => { + const r = checkInstructionsMention("Nothing relevant here", "FRAGILE"); + expect(r.passed).toBe(false); + expect(r.detail).toMatch(/do not mention 'FRAGILE'/); + }); + it("red: server returned no instructions at all", () => { + const r = checkInstructionsMention(undefined, "FRAGILE"); + expect(r.passed).toBe(false); + expect(r.detail).toMatch(/no instructions string/); + }); +}); + +describe(covers("mcp.tool-call-responds"), () => { + it("green: a real tool result with content", () => { + expect(checkToolCallResponded({ content: [{ type: "text", text: "ok" }] }).passed).toBe(true); + }); + it("red: server reported isError", () => { + expect(checkToolCallResponded({ content: [], isError: true }).passed).toBe(false); + }); + // The vacuous-pass direction. `!result.isError` alone passes for every one of + // these, because `isError` is optional on CallToolResult and absent here. + it("red: results that are not tool results do not pass merely by lacking isError", () => { + expect(checkToolCallResponded(undefined).passed).toBe(false); + expect(checkToolCallResponded({}).passed).toBe(false); + expect(checkToolCallResponded({ structuredContent: {} }).passed).toBe(false); + expect(checkToolCallResponded("ok").passed).toBe(false); + expect(checkToolCallResponded({}).detail).toMatch(/result shape unverified/); + }); +}); + +describe(covers("mcp.tools-list"), () => { + const listResult = { tools: EXPECTED_TOOLS.map((name) => ({ name })) }; + + it("green: exactly the expected tools, order-independent", () => { + expect(checkToolsList(listResult).passed).toBe(true); + expect(checkToolsList({ tools: [...listResult.tools].reverse() }).passed).toBe(true); + }); + it("red: a tool is missing", () => { + const r = checkToolsList({ tools: listResult.tools.slice(1) }); + expect(r.passed).toBe(false); + expect(r.detail).toMatch(/expected \[/); + }); + it("red: an unexpected tool appeared", () => { + expect(checkToolsList({ tools: [...listResult.tools, { name: "workspace_rm_rf" }] }).passed).toBe(false); + }); + // META-285 defect 1 regression. The original read `tools` off a callTool + // result, which has no such property, so this check could never pass. It must + // pass against a genuine ListToolsResult and fail when the property is absent. + it("red: a result with no tools[] array — the shape the original destructured", () => { + const callToolResult = { content: [], structuredContent: {}, isError: false }; + const r = checkToolsList(callToolResult); + expect(r.passed).toBe(false); + expect(r.detail).toMatch(/no tools\[\] array/); + }); +}); + +describe(`${covers("mcp.file-context-tier")} / ${covers("mcp.file-context-evidence")}`, () => { + const good = { + content: [], + structuredContent: { fragility: { tier: "evidenced-fragile", evidence: [{ kind: "co-change" }] } }, + }; + + it("green: tier and evidence present", () => { + expect(checkFragilityTier(good).passed).toBe(true); + expect(checkFragilityEvidence(good).passed).toBe(true); + }); + it("green: tier reported even when evidence is an empty array", () => { + const empty = { content: [], structuredContent: { fragility: { tier: "stable", evidence: [] } } }; + expect(checkFragilityEvidence(empty).passed).toBe(true); + }); + it("red: tier absent", () => { + expect(checkFragilityTier({ content: [], structuredContent: { fragility: {} } }).passed).toBe(false); + }); + it("red: evidence is not an array", () => { + const bad = { content: [], structuredContent: { fragility: { tier: "x", evidence: "some" } } }; + expect(checkFragilityEvidence(bad).passed).toBe(false); + }); + it("red: no structuredContent at all", () => { + expect(checkFragilityTier({ content: [] }).detail).toMatch(/no structuredContent/); + expect(checkFragilityEvidence({ content: [], isError: true }).passed).toBe(false); + }); +}); + +describe(covers("mcp.cochange-partners"), () => { + it("green: partners array present, including empty", () => { + expect(checkCochangePartners({ structuredContent: { partners: [{ path: "a" }] } }).passed).toBe(true); + expect(checkCochangePartners({ structuredContent: { partners: [] } }).passed).toBe(true); + }); + it("red: partners missing or wrong type", () => { + expect(checkCochangePartners({ structuredContent: {} }).passed).toBe(false); + expect(checkCochangePartners({ structuredContent: { partners: "none" } }).passed).toBe(false); + }); +}); + +describe(covers("mcp.assess-change"), () => { + it("green: either action or assessments present", () => { + expect(checkAssessDecision({ structuredContent: { action: "review" } }).passed).toBe(true); + expect(checkAssessDecision({ structuredContent: { assessments: [] } }).passed).toBe(true); + }); + it("red: neither field present", () => { + const r = checkAssessDecision({ structuredContent: { unrelated: 1 } }); + expect(r.passed).toBe(false); + expect(r.detail).toMatch(/neither 'action' nor 'assessments'/); + }); +}); + +describe(covers("hook.denies"), () => { + it("green: hook ran and denied with a non-zero exit", () => { + const r = checkHookDenies(okProc({ status: 2 })); + expect(r.passed).toBe(true); + expect(r.detail).toBe("exit 2"); + }); + it("red: hook allowed the edit (exit 0)", () => { + expect(checkHookDenies(okProc({ status: 0 })).passed).toBe(false); + }); + + // THE vacuous-pass defect this issue asked us to look for. + // + // The original predicate was `hookResult.status !== 0`. When spawnSync fails + // to launch, status is null, and `null !== 0` is true — so a hook that does + // not exist at all scored a PASS, and the receipt claimed the packed artifact + // enforced denial. Absence must never be read as a deny. + it("red: hook binary missing — must NOT pass on status === null", () => { + const result = failedToLaunch(); + expect(result.status !== 0).toBe(true); // what the old predicate saw + const r = checkHookDenies(result); + expect(r.passed).toBe(false); // what it must actually report + expect(r.detail).toMatch(/hook did not run/); + }); + it("red: hook killed by a signal rather than exiting", () => { + const r = checkHookDenies(killedBySignal()); + expect(r.passed).toBe(false); + expect(r.detail).toMatch(/did not exit normally/); + }); + + // The same vacuous pass one level further in, found by running the harness + // against a candidate built with `hooks/` removed from package.json files[]. + // `node ` launches fine and exits 1, so every "did it launch + // and exit non-zero" predicate reads a candidate that ships NO hook as a + // passing deny. Unit cases alone did not catch this; the end-to-end broken + // candidate did. + it("red: node exited 1 because the hook script was missing, not because it denied", () => { + const moduleNotFound = okProc({ + status: 1, + stderr: + "node:internal/modules/esm/resolve:275\n throw new ERR_MODULE_NOT_FOUND(\n ^\n" + + "Error [ERR_MODULE_NOT_FOUND]: Cannot find module '/pkg/hooks/pre-edit-check.mjs'\n" + + " at finalizeResolution (node:internal/modules/esm/resolve:275:11)\n", + }); + expect(moduleNotFound.status !== 0).toBe(true); // still non-zero + const r = checkHookDenies(moduleNotFound); + expect(r.passed).toBe(false); + expect(r.detail).toMatch(/module not found.*is a crash, not a deny/); + }); + + it("red: hook threw before reaching a decision", () => { + const threw = okProc({ + status: 1, + stderr: "Uncaught TypeError: cannot read properties of undefined\n at run (node:internal/main/run_main:1:1)\n", + }); + expect(checkHookDenies(threw).passed).toBe(false); + }); + + // The deny path must not be collateral damage: a real deny that happens to + // print a stack-shaped evidence line still passes. + it("green: a genuine deny whose message mentions a path is not mistaken for a crash", () => { + const deny = okProc({ + status: 2, + stdout: "DENY: src/routes/checkout.ts is FRAGILE at src/routes/checkout.ts:14 — include co-change partners", + }); + expect(checkHookDenies(deny).passed).toBe(true); + }); +}); + +describe(covers("hook.output-mentions-deny"), () => { + it("green: denial language on stdout or stderr", () => { + expect(checkHookOutputMentionsDeny(okProc({ status: 2, stdout: "FRAGILE: blocked" })).passed).toBe(true); + expect(checkHookOutputMentionsDeny(okProc({ status: 2, stderr: "deny" })).passed).toBe(true); + }); + it("red: ran but said nothing about a denial", () => { + expect(checkHookOutputMentionsDeny(okProc({ status: 2, stdout: "all good" })).passed).toBe(false); + }); + it("red: produced no output at all", () => { + const r = checkHookOutputMentionsDeny(okProc({ status: 2 })); + expect(r.passed).toBe(false); + expect(r.detail).toMatch(/no output/); + }); + it("red: never launched", () => { + expect(checkHookOutputMentionsDeny(failedToLaunch()).passed).toBe(false); + }); +}); + +describe(covers("installer.help-usage"), () => { + it("green: usage on stdout, exit 0", () => { + expect(checkInstallerHelp(okProc({ stdout: "Usage:\n install [--with-hook]" })).passed).toBe(true); + }); + + // META-285 defect 3, second half. The original asserted over `stdout` alone + // while the installer wrote USAGE to console.error, so the check could not + // pass even when help worked correctly. Which stream carries usage is a + // presentation choice; that help is shown is the property. + it("green: usage on stderr still counts as help being shown", () => { + expect(checkInstallerHelp(okProc({ stderr: "Usage:\n install" })).passed).toBe(true); + }); + + it("red: exited 0 but printed no usage", () => { + const r = checkInstallerHelp(okProc({ stdout: "installed!" })); + expect(r.passed).toBe(false); + expect(r.detail).toMatch(/printed no usage/); + }); + it("red: non-zero exit", () => { + expect(checkInstallerHelp(okProc({ status: 1, stderr: "Usage:" })).passed).toBe(false); + }); + it("red: installer never launched", () => { + expect(checkInstallerHelp(failedToLaunch()).passed).toBe(false); + }); +}); + +describe(covers("installer.help-nondestructive"), () => { + it("green: sandbox empty after --help", () => { + expect(checkHelpWroteNothing([]).passed).toBe(true); + }); + + // The stop-now item, asserted positively. If `--help` regresses into a real + // install, the installer writes .codex/config.toml into its cwd. Running help + // in a disposable sandbox turns that into a visible red instead of a silent + // rewrite of the source checkout. + it("red: --help wrote a config into its working directory", () => { + const r = checkHelpWroteNothing([".codex"]); + expect(r.passed).toBe(false); + expect(r.detail).toMatch(/--help wrote into its cwd: \.codex/); + }); + it("red: sandbox could not be listed — absence of a listing is not proof of safety", () => { + const r = checkHelpWroteNothing(undefined); + expect(r.passed).toBe(false); + expect(r.detail).toMatch(/cannot prove --help was non-destructive/); + }); +}); + +describe(covers("repo.tree-unchanged"), () => { + it("green: clean before, clean after", () => { + const r = checkTreeUnchanged("", ""); + expect(r.passed).toBe(true); + expect(r.detail).toMatch(/tree clean and unmodified/); + }); + + // Comparing before/after rather than requiring "clean" means the check still + // proves the harness changed nothing when it starts from a dirty checkout. + it("green: dirty before, identically dirty after", () => { + const dirty = " M src/index.ts\n?? scratch.txt\n"; + const r = checkTreeUnchanged(dirty, dirty); + expect(r.passed).toBe(true); + expect(r.detail).toMatch(/pre-existing changes preserved/); + }); + + // The exact signature of the destructive --help path recorded in the Aug 4 + // field observation: .codex/config.toml modified by the harness itself. + it("red: harness modified a tracked file", () => { + const r = checkTreeUnchanged("", " M .codex/config.toml\n"); + expect(r.passed).toBe(false); + expect(r.detail).toMatch(/modified the source tree.*\.codex\/config\.toml/); + }); + it("red: harness left a new untracked artifact behind", () => { + const r = checkTreeUnchanged("", "?? codex-mcp-0.1.9.tgz\n"); + expect(r.passed).toBe(false); + expect(r.detail).toMatch(/codex-mcp-0\.1\.9\.tgz/); + }); + it("red: git status unavailable — safety not established, not assumed", () => { + expect(checkTreeUnchanged(null, "").passed).toBe(false); + expect(checkTreeUnchanged("", null).detail).toMatch(/tree safety not established/); + }); +}); + +describe("receipt: failed vs not-run", () => { + const plan = [ + { id: "a", step: 1, name: "check a" }, + { id: "b", step: 1, name: "check b" }, + { id: "c", step: 2, name: "check c" }, + ]; + + it("green: all planned checks ran and passed → CONSUMABLE", () => { + const rec = createRecorder(plan); + for (const { id } of plan) rec.record(id, { passed: true, detail: "ok" }); + const receipt = rec.buildReceipt({ generatedAt: "2026-08-12T00:00:00.000Z" }); + expect(receipt.verdict).toBe("CONSUMABLE"); + expect(receipt.summary).toEqual({ total: 3, passed: 3, failed: 0, notRun: 0 }); + expect(receipt.aborted).toBeNull(); + }); + + it("red: a failure yields NOT_CONSUMABLE", () => { + const rec = createRecorder(plan); + rec.record("a", { passed: true }); + rec.record("b", { passed: false, detail: "broken" }); + rec.record("c", { passed: true }); + const receipt = rec.buildReceipt({}); + expect(receipt.verdict).toBe("NOT_CONSUMABLE"); + expect(receipt.summary.failed).toBe(1); + expect(receipt.checks.find((c) => c.id === "b")?.status).toBe("fail"); + }); + + // The receipt-level defect. The original computed the verdict from only the + // checks that happened to execute, so a harness that threw after two passing + // checks emitted "2/2 passed — CONSUMABLE". Unreached checks must be visible + // as not_run, and must make the run INCOMPLETE rather than clean. + it("red: an aborted run reports not_run and INCOMPLETE, never CONSUMABLE", () => { + const rec = createRecorder(plan); + rec.record("a", { passed: true }); + rec.record("b", { passed: true }); + const receipt = rec.buildReceipt({ aborted: { message: "MCP connect failed" } }); + + expect(receipt.verdict).toBe("INCOMPLETE"); + expect(receipt.verdict).not.toBe("CONSUMABLE"); + expect(receipt.summary).toEqual({ total: 3, passed: 2, failed: 0, notRun: 1 }); + expect(receipt.checks.find((c) => c.id === "c")).toMatchObject({ status: "not_run", detail: null }); + expect(receipt.aborted).toEqual({ message: "MCP connect failed" }); + }); + + it("not_run is a distinct status from fail", () => { + const rec = createRecorder(plan); + rec.record("a", { passed: false, detail: "broken" }); + const receipt = rec.buildReceipt({}); + const statuses = Object.fromEntries(receipt.checks.map((c) => [c.id, c.status])); + expect(statuses).toEqual({ a: "fail", b: "not_run", c: "not_run" }); + expect(receipt.summary.failed).toBe(1); + expect(receipt.summary.notRun).toBe(2); + }); + + it("a real failure outranks incompleteness in the verdict", () => { + const rec = createRecorder(plan); + rec.record("a", { passed: false }); + expect(rec.buildReceipt({}).verdict).toBe("NOT_CONSUMABLE"); + }); + + it("every check keeps its plan identity in the receipt", () => { + const receipt = createRecorder(plan).buildReceipt({}); + expect(receipt.checks.map((c) => c.id)).toEqual(["a", "b", "c"]); + expect(receipt.checks[0]).toMatchObject({ id: "a", name: "check a", step: 1 }); + }); + + // Guards against silent plan drift: a check that is renamed or recorded twice + // must fail loudly rather than quietly shrink the evidence set. + it("rejects unknown ids, duplicate records, and malformed outcomes", () => { + const rec = createRecorder(plan); + expect(() => rec.record("nonexistent", { passed: true })).toThrow(/unknown check id/); + rec.record("a", { passed: true }); + expect(() => rec.record("a", { passed: true })).toThrow(/recorded twice/); + expect(() => rec.record("b", { detail: "no verdict" } as never)).toThrow(/malformed outcome/); + }); +}); + +describe("watched-red coverage of the harness check plan", () => { + it("every check in CHECK_PLAN has been demonstrated red and green above", () => { + const planned = CHECK_PLAN.map((c) => c.id); + const unproven = planned.filter((id) => !proven.has(id)); + expect(unproven).toEqual([]); + }); + + it("no test claims coverage of a check the harness does not run", () => { + const planned = new Set(CHECK_PLAN.map((c) => c.id)); + expect([...proven].filter((id) => !planned.has(id))).toEqual([]); + }); + + it("check ids are unique", () => { + const ids = CHECK_PLAN.map((c) => c.id); + expect(ids.length).toBe(new Set(ids).size); + }); +}); diff --git a/tests/migration/verify-receipt.test.ts b/tests/migration/verify-receipt.test.ts index 6ecf90d..7083e40 100644 --- a/tests/migration/verify-receipt.test.ts +++ b/tests/migration/verify-receipt.test.ts @@ -1,4 +1,13 @@ -import { describe, expect, it } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, describe, expect, it } from "vitest"; +// The comparator under test is imported, never reimplemented. An earlier +// version of this file rebuilt `compareReceipts` inline, so the tests verified +// a copy: the real comparator could regress without a single test failing, and +// the copy's violation messages had already drifted from the source +// (META-140 one-concept-two-implementations class, recorded on META-285). +import { compareReceipts, readReceipt } from "../../scripts/migration/verify-receipt.mjs"; interface ReceiptCheck { id: string; @@ -15,63 +24,6 @@ interface Receipt { [key: string]: unknown; } -// We test the comparison logic by importing the module and calling its -// internal function. Since verify-receipt.mjs is a CLI script, we test -// the core comparator by reconstructing it inline from the same logic. -// This is a watched-red contract: the comparator must fail on perturbed input. - -function compareReceipts(reference: Receipt, candidate: Receipt): string[] { - const violations = []; - - if (reference.verdict !== candidate.verdict) { - violations.push(`verdict diverged: reference=${reference.verdict} candidate=${candidate.verdict}`); - } - - for (const field of ["total", "passed", "failed", "unsupported"] as const) { - if (reference.summary[field] !== candidate.summary[field]) { - violations.push( - `summary.${field} diverged: reference=${reference.summary[field]} candidate=${candidate.summary[field]}`, - ); - } - } - - const refChecks = new Map(reference.checks.map((c) => [c.id, c])); - const candChecks = new Map(candidate.checks.map((c) => [c.id, c])); - - const missingInCandidate = [...refChecks.keys()].filter((id) => !candChecks.has(id)); - const missingInReference = [...candChecks.keys()].filter((id) => !refChecks.has(id)); - - for (const id of missingInCandidate) { - violations.push(`check '${id}' present in reference but missing from candidate`); - } - for (const id of missingInReference) { - violations.push(`check '${id}' present in candidate but missing from reference`); - } - - for (const [id, refCheck] of refChecks) { - const candCheck = candChecks.get(id); - if (!candCheck) continue; - - if (refCheck.status !== candCheck.status) { - violations.push(`check '${id}' status diverged: reference=${refCheck.status} candidate=${candCheck.status}`); - } - - const refViolations = JSON.stringify(refCheck.violations ?? []); - const candViolations = JSON.stringify(candCheck.violations ?? []); - if (refViolations !== candViolations) { - violations.push(`check '${id}' violations diverged`); - } - } - - const refDiffs = JSON.stringify(reference.intentionalDifferences ?? []); - const candDiffs = JSON.stringify(candidate.intentionalDifferences ?? []); - if (refDiffs !== candDiffs) { - violations.push("intentionalDifferences diverged"); - } - - return violations; -} - const BASE_RECEIPT: Receipt = { verdict: "PARITY", summary: { total: 10, passed: 10, failed: 0, unsupported: 0 }, @@ -91,7 +43,7 @@ const BASE_RECEIPT: Receipt = { }; describe("compareReceipts", () => { - it("passes identical receipts ignoring non-deterministic fields", () => { + it("watched-green: passes identical receipts ignoring non-deterministic fields", () => { const candidate: Receipt = structuredClone(BASE_RECEIPT); candidate.generatedAt = "2026-08-01T12:00:00.000Z"; candidate.startedAt = "2026-08-01T11:59:30.000Z"; @@ -113,33 +65,40 @@ describe("compareReceipts", () => { expect(violations.some((v) => /verdict diverged/.test(v))).toBe(true); }); + it("watched-red: catches a summary count change", () => { + const candidate: Receipt = structuredClone(BASE_RECEIPT); + candidate.summary.unsupported = 2; + const violations = compareReceipts(BASE_RECEIPT, candidate); + expect(violations.some((v) => /summary\.unsupported diverged/.test(v))).toBe(true); + }); + it("watched-red: catches a check status flip", () => { const candidate: Receipt = structuredClone(BASE_RECEIPT); candidate.checks[4].status = "fail"; candidate.checks[4].violations = ["smoke failed"]; const violations = compareReceipts(BASE_RECEIPT, candidate); - expect(violations.some((v) => /check 'mcp.smoke' status diverged/.test(v))).toBe(true); + expect(violations.some((v) => /check 'mcp\.smoke' status diverged/.test(v))).toBe(true); }); it("watched-red: catches a missing check", () => { const candidate: Receipt = structuredClone(BASE_RECEIPT); candidate.checks = candidate.checks.slice(0, 9); const violations = compareReceipts(BASE_RECEIPT, candidate); - expect(violations.some((v) => /'generator.resolution' present in reference but missing/.test(v))).toBe(true); + expect(violations.some((v) => /'generator\.resolution' present in reference but missing/.test(v))).toBe(true); }); it("watched-red: catches an extra check", () => { const candidate: Receipt = structuredClone(BASE_RECEIPT); candidate.checks.push({ id: "evil.check", status: "pass", violations: [], evidence: {} }); const violations = compareReceipts(BASE_RECEIPT, candidate); - expect(violations.some((v) => /'evil.check' present in candidate but missing/.test(v))).toBe(true); + expect(violations.some((v) => /'evil\.check' present in candidate but missing/.test(v))).toBe(true); }); it("watched-red: catches new violations in a previously clean check", () => { const candidate: Receipt = structuredClone(BASE_RECEIPT); candidate.checks[2].violations = ["package.version diverged"]; const violations = compareReceipts(BASE_RECEIPT, candidate); - expect(violations.some((v) => /check 'pkg.identity' violations diverged/.test(v))).toBe(true); + expect(violations.some((v) => /check 'pkg\.identity' violations diverged/.test(v))).toBe(true); }); it("watched-red: catches changed intentionalDifferences", () => { @@ -147,4 +106,107 @@ describe("compareReceipts", () => { candidate.intentionalDifferences = [{ path: "README.md", justification: "changed" }]; expect(compareReceipts(BASE_RECEIPT, candidate).some((v) => /intentionalDifferences diverged/.test(v))).toBe(true); }); + + // Asserted against the real comparator's exact text. While the test owned a + // copy, these strings could drift from the shipped ones unnoticed — and had. + it("reports the actual reference and candidate values in the violation text", () => { + const candidate: Receipt = structuredClone(BASE_RECEIPT); + candidate.verdict = "DIVERGENT"; + const [violation] = compareReceipts(BASE_RECEIPT, candidate); + expect(violation).toBe("verdict diverged: reference=PARITY candidate=DIVERGENT"); + }); + + // A comparator that accepts a structurally empty receipt compares two empty + // check sets as identical — a pass that proves nothing about parity. + it("refuses a receipt with an empty check set rather than comparing it clean", () => { + const empty: Receipt = structuredClone(BASE_RECEIPT); + empty.checks = []; + expect(() => compareReceipts(empty, structuredClone(empty))).toThrow(/'checks' is empty/); + }); + + it("refuses a receipt missing the fields it reads", () => { + const noSummary = { verdict: "PARITY", checks: BASE_RECEIPT.checks } as unknown as Receipt; + expect(() => compareReceipts(noSummary, structuredClone(BASE_RECEIPT))).toThrow(/missing 'summary' object/); + + const noChecks = { verdict: "PARITY", summary: BASE_RECEIPT.summary } as unknown as Receipt; + expect(() => compareReceipts(structuredClone(BASE_RECEIPT), noChecks)).toThrow(/missing 'checks' array/); + }); + + it("names which side was malformed", () => { + const bad = { verdict: "PARITY", summary: {}, checks: [{ status: "pass" }] } as unknown as Receipt; + expect(() => compareReceipts(structuredClone(BASE_RECEIPT), bad)).toThrow(/candidate receipt is not a usable/); + expect(() => compareReceipts(bad, structuredClone(BASE_RECEIPT))).toThrow(/reference receipt is not a usable/); + }); + + // The comparator keys `checks` by id into a Map, which keeps only the last + // entry for a repeated id. Without this guard a receipt carrying the same + // check twice with divergent statuses loses the earlier one and compares + // clean against the survivor — inconsistency reported as parity. + it("refuses a receipt carrying the same check id twice", () => { + const duplicated: Receipt = structuredClone(BASE_RECEIPT); + duplicated.checks.push({ ...duplicated.checks[0], status: "fail", violations: ["tree differs"] }); + + expect(() => compareReceipts(duplicated, structuredClone(BASE_RECEIPT))).toThrow( + /'checks' contains duplicate ids: git\.tree-equality/, + ); + expect(() => compareReceipts(structuredClone(BASE_RECEIPT), duplicated)).toThrow( + /candidate receipt is not a usable/, + ); + }); + + // Guards the collapse directly: absent the duplicate check, the losing entry + // is discarded and this pair compares clean despite disagreeing. + it("does not let a duplicate id mask a status divergence", () => { + const reference: Receipt = structuredClone(BASE_RECEIPT); + const candidate: Receipt = structuredClone(BASE_RECEIPT); + candidate.checks[0].status = "fail"; + candidate.checks.push({ ...BASE_RECEIPT.checks[0] }); + + expect(() => compareReceipts(reference, candidate)).toThrow(/duplicate ids/); + }); + + it("names every duplicated id, not just the first", () => { + const duplicated: Receipt = structuredClone(BASE_RECEIPT); + duplicated.checks.push({ ...BASE_RECEIPT.checks[1] }, { ...BASE_RECEIPT.checks[0] }); + + expect(() => compareReceipts(duplicated, structuredClone(BASE_RECEIPT))).toThrow( + /duplicate ids: git\.tree-equality, pkg\.pack-inventory/, + ); + }); +}); + +describe("readReceipt", () => { + const dir = mkdtempSync(join(tmpdir(), "verify-receipt-test-")); + afterAll(() => rmSync(dir, { recursive: true, force: true })); + + it("reads a well-formed receipt", () => { + const path = join(dir, "good.json"); + writeFileSync(path, JSON.stringify(BASE_RECEIPT)); + expect(readReceipt(path, "reference receipt").verdict).toBe("PARITY"); + }); + + // A CI gate that dies on a raw ENOENT stack tells the reader nothing about + // which receipt was missing or what was expected there. + it("fails with a legible message on a missing file, naming the role and path", () => { + const path = join(dir, "absent.json"); + expect(() => readReceipt(path, "candidate receipt")).toThrow(/candidate receipt not found/); + expect(() => readReceipt(path, "candidate receipt")).toThrow(/Expected a parity receipt JSON file/); + expect(() => readReceipt(path, "candidate receipt")).not.toThrow(/ENOENT/); + }); + + it("fails with a legible message on malformed JSON", () => { + const path = join(dir, "malformed.json"); + writeFileSync(path, "{ not json at all "); + expect(() => readReceipt(path, "reference receipt")).toThrow(/reference receipt is not valid JSON/); + }); + + it("fails when the file parses but is not a receipt object", () => { + const arrayPath = join(dir, "array.json"); + writeFileSync(arrayPath, "[]"); + expect(() => readReceipt(arrayPath, "reference receipt")).toThrow(/is not a receipt object[\s\S]*array/); + + const scalarPath = join(dir, "scalar.json"); + writeFileSync(scalarPath, "42"); + expect(() => readReceipt(scalarPath, "candidate receipt")).toThrow(/is not a receipt object[\s\S]*number/); + }); });