diff --git a/scripts/migration/consume-standard-candidate.mjs b/scripts/migration/consume-standard-candidate.mjs
index 41bd6da..2915d0d 100644
--- a/scripts/migration/consume-standard-candidate.mjs
+++ b/scripts/migration/consume-standard-candidate.mjs
@@ -1,6 +1,6 @@
#!/usr/bin/env node
/**
- * Standard-candidate consumption harness (META-235 Step 5 / I-7).
+ * Standard-candidate consumption harness (META-235 Step 5 / META-258 I-7).
*
* Packs the current repo as an npm tarball, installs it in a disposable
* directory, starts the MCP server from the packed artifact (not from
@@ -12,248 +12,314 @@
* the npm package and runs the server is doing exactly what this harness
* does.
*
+ * Every assertion lives in ./consumption-checks.mjs, which the harness calls
+ * and tests/migration/consumption-checks.test.ts exercises in both directions.
+ * Assertions are not written inline here: a check the tests cannot reach is a
+ * check nobody has watched go red (META-285, META-165 scope amendment).
+ *
+ * A verification script must not mutate the thing it verifies. Two structural
+ * rules enforce that here:
+ * - `run()` requires an explicit cwd. There is no repo-root default to
+ * forget, which is how `--help` came to run a real install into the source
+ * checkout (META-285 stop-now item).
+ * - The source tree is fingerprinted before and after, and the comparison is
+ * itself a recorded check.
+ *
* Usage:
* node scripts/migration/consume-standard-candidate.mjs
* node scripts/migration/consume-standard-candidate.mjs --out
+ * node scripts/migration/consume-standard-candidate.mjs --help
*/
import { spawnSync } from "node:child_process";
-import { cpSync, existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { cpSync, existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
+import {
+ checkAssessDecision,
+ checkCochangePartners,
+ checkFragilityEvidence,
+ checkFragilityTier,
+ checkHelpWroteNothing,
+ checkHookDenies,
+ checkHookOutputMentionsDeny,
+ checkInstallerHelp,
+ checkInstructionsMention,
+ checkPackContainsPath,
+ checkPackContainsPrefix,
+ checkPathExists,
+ checkProcessSucceeded,
+ checkTarballExists,
+ checkToolCallResponded,
+ checkToolsList,
+ checkTreeUnchanged,
+ createRecorder,
+} from "./consumption-checks.mjs";
const here = dirname(fileURLToPath(import.meta.url));
const repoRoot = resolve(here, "..", "..");
const fixtureRoot = join(repoRoot, "fixture");
+const USAGE = [
+ "Usage:",
+ " node scripts/migration/consume-standard-candidate.mjs [--out ]",
+ "",
+ "Packs this repository, installs the tarball into a disposable directory, and",
+ "verifies the packed artifact is consumable. Writes no files outside --out and",
+ "makes no change to this checkout.",
+ "",
+ "Options:",
+ " --out Write consumption-receipt.json into ",
+ " --help, -h Show this message and exit without running anything",
+].join("\n");
+
+/**
+ * `cwd` is required, not defaulted.
+ *
+ * The original helper defaulted to `repoRoot`, so any invocation that forgot
+ * to pass a cwd silently targeted the source checkout — which is how the
+ * installer `--help` probe came to run a real install against this repo.
+ * Making it required means that mistake cannot be made silently again.
+ */
function run(cmd, args, opts = {}) {
- const result = spawnSync(cmd, args, {
+ if (!opts.cwd) throw new Error(`run(${cmd}) requires an explicit cwd — refusing to default to the source repo`);
+ return spawnSync(cmd, args, {
encoding: "utf8",
timeout: opts.timeout ?? 120000,
- cwd: opts.cwd ?? repoRoot,
+ cwd: opts.cwd,
stdio: opts.stdio ?? "pipe",
+ env: opts.env ?? process.env,
});
- return result;
}
function runOrDie(cmd, args, opts = {}) {
const result = run(cmd, args, opts);
+ if (result.error) throw new Error(`${cmd} ${args.join(" ")} failed to launch: ${result.error.message}`);
if (result.status !== 0) {
- console.error(`${cmd} ${args.join(" ")} failed (exit ${result.status})`);
- if (result.stderr) console.error(result.stderr.slice(-2000));
- process.exit(1);
+ const detail = (result.stderr ?? "").slice(-2000);
+ throw new Error(`${cmd} ${args.join(" ")} failed (exit ${result.status})\n${detail}`);
}
return result;
}
+/**
+ * Absolute path to `git`, resolved from fixed system locations.
+ *
+ * The fingerprint below is the evidence that this harness did not mutate the
+ * source checkout. Resolving `git` through `PATH` would let a writable PATH
+ * entry decide what `git status` reports — the one thing that must not be
+ * forgeable is the tool asked to prove nothing changed. Fixed locations only.
+ *
+ * If none exists the fingerprint is unavailable, and `checkTreeUnchanged`
+ * fails on a non-string input rather than skipping to green.
+ */
+const GIT_BIN = ["/usr/bin/git", "/usr/local/bin/git", "/opt/homebrew/bin/git"].find((p) => existsSync(p)) ?? null;
+
+/** Fingerprint of the source checkout, used to prove the harness changed nothing. */
+function treeFingerprint() {
+ if (!GIT_BIN) return null;
+ const result = spawnSync(GIT_BIN, ["status", "--porcelain"], { encoding: "utf8", cwd: repoRoot, timeout: 30000 });
+ if (result.error || result.status !== 0) return null;
+ return result.stdout;
+}
+
+function parseArgs(argv) {
+ const opts = { outDir: null, help: false };
+ for (let i = 0; i < argv.length; i++) {
+ const arg = argv[i];
+ if (arg === "--help" || arg === "-h") {
+ opts.help = true;
+ } else if (arg === "--out") {
+ const value = argv[++i];
+ if (!value) throw new Error("--out requires a directory argument");
+ opts.outDir = resolve(value);
+ } else if (arg.startsWith("--out=")) {
+ const value = arg.slice("--out=".length);
+ if (!value) throw new Error("--out= requires a directory argument");
+ opts.outDir = resolve(value);
+ } else {
+ throw new Error(`unknown argument: ${arg}\n\n${USAGE}`);
+ }
+ }
+ return opts;
+}
+
async function main() {
- const args = process.argv.slice(2);
- let outDir = null;
- for (let i = 0; i < args.length; i++) {
- if (args[i] === "--out") outDir = resolve(args[++i]);
+ const opts = parseArgs(process.argv.slice(2));
+ if (opts.help) {
+ console.log(USAGE);
+ return;
}
+ const treeBefore = treeFingerprint();
const work = mkdtempSync(join(tmpdir(), "consume-std-"));
const packDir = join(work, "pack");
const installDir = join(work, "install");
+ // A sandbox that starts empty and must stay empty. Anything the installer
+ // writes while being asked for help lands here, where it is visible.
+ const helpSandbox = join(work, "help-sandbox");
mkdirSync(packDir, { recursive: true });
mkdirSync(installDir, { recursive: true });
+ mkdirSync(helpSandbox, { recursive: true });
- let failures = 0;
- const checks = [];
+ const recorder = createRecorder();
+ let client = null;
+ let aborted = null;
- function record(name, passed, detail) {
- checks.push({ name, passed, detail });
- if (!passed) failures++;
- console.log(`${passed ? "PASS" : "FAIL"} ${name}${!passed && detail ? ` -> ${detail}` : ""}`);
+ function record(id, outcome) {
+ const entry = recorder.record(id, outcome);
+ console.log(`${entry.passed ? "PASS" : "FAIL"} ${entry.name}${entry.detail ? ` -> ${entry.detail}` : ""}`);
}
try {
// --- Step 1: Build and pack -------------------------------------------
console.log("\n=== Step 1: Build and pack ===");
- runOrDie("npm", ["ci"], { cwd: repoRoot });
+ runOrDie("npm", ["ci"], { cwd: repoRoot, timeout: 300000 });
runOrDie("npm", ["run", "build"], { cwd: repoRoot });
- runOrDie("npm", ["run", "build:extension"], { cwd: repoRoot });
+ runOrDie("npm", ["run", "build:extension"], { cwd: repoRoot, timeout: 600000 });
const packResult = runOrDie("npm", ["pack", "--json"], { cwd: repoRoot });
const packMeta = JSON.parse(packResult.stdout)[0];
const tarballPath = join(repoRoot, packMeta.filename);
const tarballDest = join(packDir, packMeta.filename);
cpSync(tarballPath, tarballDest);
+ // `npm pack` writes the tarball into the repo; removing it is what keeps
+ // the source tree unmodified, and repo.tree-unchanged proves it did.
rmSync(tarballPath, { force: true });
- record("npm pack produces a tarball", existsSync(tarballDest), packMeta.filename);
- record(
- "tarball contains dist/index.js",
- packMeta.files?.some((f) => f.path === "dist/index.js"),
- JSON.stringify(packMeta.files?.map((f) => f.path)),
- );
- record(
- "tarball contains .mcp.json",
- packMeta.files?.some((f) => f.path === ".mcp.json"),
- );
- record(
- "tarball contains hooks/",
- packMeta.files?.some((f) => f.path.startsWith("hooks/")),
- );
+ record("pack.tarball", checkTarballExists(existsSync(tarballDest), packMeta.filename));
+ record("pack.dist-index", checkPackContainsPath(packMeta, "dist/index.js"));
+ record("pack.mcp-json", checkPackContainsPath(packMeta, ".mcp.json"));
+ record("pack.hooks", checkPackContainsPrefix(packMeta, "hooks/"));
// --- Step 2: Install in disposable dir --------------------------------
console.log("\n=== Step 2: Install from tarball ===");
- const installResult = run("npm", ["install", tarballDest], { cwd: installDir });
- record("npm install from tarball succeeds", installResult.status === 0, installResult.stderr?.slice(-500));
-
- // Verify the installed package has the expected bin entry
- const installedBin = join(installDir, "node_modules", "@workspacejson", "codex-mcp", "scripts", "install.mjs");
- record("installed package has scripts/install.mjs", existsSync(installedBin));
+ const installResult = run("npm", ["install", tarballDest], { cwd: installDir, timeout: 300000 });
+ record("install.from-tarball", checkProcessSucceeded(installResult, "npm install"));
- const installedMain = join(installDir, "node_modules", "@workspacejson", "codex-mcp", "dist", "index.js");
- record("installed package has dist/index.js", existsSync(installedMain));
+ const installedPkg = join(installDir, "node_modules", "@workspacejson", "codex-mcp");
+ const installedBin = join(installedPkg, "scripts", "install.mjs");
+ const installedMain = join(installedPkg, "dist", "index.js");
+ record("install.has-install-script", checkPathExists(existsSync(installedBin), installedBin));
+ record("install.has-dist-index", checkPathExists(existsSync(installedMain), installedMain));
// --- Step 3: Start MCP server from installed package ------------------
console.log("\n=== Step 3: MCP server from packed artifact ===");
- // Copy fixture into install dir so the server can find workspace.json
const fixtureDest = join(installDir, "fixture");
cpSync(fixtureRoot, fixtureDest, { recursive: true });
- // Start the server using the installed package's main entry
- const serverPath = join(installDir, "node_modules", "@workspacejson", "codex-mcp", "dist", "index.js");
-
- // Use MCP client to test the server
const { Client } = await import("@modelcontextprotocol/sdk/client/index.js");
const { StdioClientTransport } = await import("@modelcontextprotocol/sdk/client/stdio.js");
const transport = new StdioClientTransport({
- command: "node",
- args: [serverPath],
+ command: process.execPath,
+ args: [installedMain],
env: { ...process.env, WORKSPACE_JSON_ROOT: fixtureDest },
});
- const client = new Client({ name: "consume-harness", version: "0.0.0" });
+ client = new Client({ name: "consume-harness", version: "0.0.0" });
await client.connect(transport);
// --- Step 4: Smoke test the packed server -----------------------------
console.log("\n=== Step 4: Smoke test packed server ===");
- const instr = client.getInstructions();
- record("server instructions contain FRAGILE", instr?.includes("FRAGILE"), "missing instructions");
-
- const toolResult = await client.callTool({ name: "workspace_list_fragile_files", arguments: {} });
- record("server responds to tool calls", !toolResult.isError);
-
- const { tools: availableTools } = await client.listTools();
- const toolNames = availableTools.map((t) => t.name).sort();
- record(
- "tools/list returns 4 expected tools",
- JSON.stringify(toolNames) ===
- JSON.stringify([
- "workspace_assess_change",
- "workspace_get_cochange_partners",
- "workspace_get_file_context",
- "workspace_list_fragile_files",
- ]),
- toolNames.join(","),
- );
+ record("mcp.instructions-fragile", checkInstructionsMention(client.getInstructions(), "FRAGILE"));
+
+ const listResult = await client.callTool({ name: "workspace_list_fragile_files", arguments: {} });
+ record("mcp.tool-call-responds", checkToolCallResponded(listResult));
- // Test file context retrieval through the packed artifact
- const r1 = await client.callTool({
+ record("mcp.tools-list", checkToolsList(await client.listTools()));
+
+ const fileContext = await client.callTool({
name: "workspace_get_file_context",
arguments: { path: "src/routes/checkout.ts" },
});
- const s1 = r1.structuredContent;
- record(
- "file context returns fragility tier",
- s1?.fragility?.tier !== undefined,
- JSON.stringify(s1?.fragility?.tier),
- );
- record(
- "file context returns evidence",
- Array.isArray(s1?.fragility?.evidence),
- JSON.stringify(s1?.fragility?.evidence),
- );
+ record("mcp.file-context-tier", checkFragilityTier(fileContext));
+ record("mcp.file-context-evidence", checkFragilityEvidence(fileContext));
- // Test co-change partners
- const r2 = await client.callTool({
+ const cochange = await client.callTool({
name: "workspace_get_cochange_partners",
arguments: { path: "src/routes/checkout.ts" },
});
- const s2 = r2.structuredContent;
- record("co-change partners returns array", Array.isArray(s2?.partners), JSON.stringify(s2?.partners));
+ record("mcp.cochange-partners", checkCochangePartners(cochange));
- // Test assess change
- const r3 = await client.callTool({
+ const assess = await client.callTool({
name: "workspace_assess_change",
arguments: { paths: ["src/routes/checkout.ts"] },
});
- const s3 = r3.structuredContent;
- record(
- "assess change returns decision",
- s3?.action !== undefined || s3?.assessments !== undefined,
- JSON.stringify(s3),
- );
-
- await client.close();
+ record("mcp.assess-change", checkAssessDecision(assess));
// --- Step 5: Hook from packed artifact --------------------------------
console.log("\n=== Step 5: Hook from packed artifact ===");
- const hookPath = join(installDir, "node_modules", "@workspacejson", "codex-mcp", "hooks", "pre-edit-check.mjs");
- record("hook script exists in packed artifact", existsSync(hookPath));
+ const hookPath = join(installedPkg, "hooks", "pre-edit-check.mjs");
+ record("hook.exists", checkPathExists(existsSync(hookPath), hookPath));
- const hookResult = run("node", [hookPath, "--paths", "src/routes/checkout.ts"], {
- cwd: fixtureDest,
- });
- record(
- "hook exits non-zero on evidenced-fragile without partners",
- hookResult.status !== 0,
- `exit ${hookResult.status}`,
- );
- record(
- "hook output mentions FRAGILE or deny",
- /FRAGILE|deny|block/i.test(hookResult.stdout + hookResult.stderr),
- (hookResult.stdout + hookResult.stderr).slice(-500),
- );
+ const hookResult = run(process.execPath, [hookPath, "--paths", "src/routes/checkout.ts"], { cwd: fixtureDest });
+ record("hook.denies", checkHookDenies(hookResult));
+ record("hook.output-mentions-deny", checkHookOutputMentionsDeny(hookResult));
// --- Step 6: Installer from packed artifact ---------------------------
console.log("\n=== Step 6: Installer from packed artifact ===");
- const installScript = join(installDir, "node_modules", "@workspacejson", "codex-mcp", "scripts", "install.mjs");
- record("installer script exists in packed artifact", existsSync(installScript));
-
- const installHelp = run("node", [installScript, "--help"]);
- const installHelpOutput = (installHelp.stdout ?? "") + (installHelp.stderr ?? "");
- record(
- "installer --help works from packed artifact",
- installHelp.status === 0 && /Usage:/i.test(installHelpOutput),
- installHelpOutput.slice(-500),
- );
+ const installScript = join(installedPkg, "scripts", "install.mjs");
+ record("installer.exists", checkPathExists(existsSync(installScript), installScript));
+
+ // Help runs inside the disposable sandbox, never the source repo. If help
+ // ever regresses into an install again, it installs into a directory we
+ // then inspect — and installer.help-nondestructive goes red — instead of
+ // silently rewriting this checkout's .codex/config.toml.
+ const installHelp = run(process.execPath, [installScript, "--help"], { cwd: helpSandbox });
+ record("installer.help-usage", checkInstallerHelp(installHelp));
+ record("installer.help-nondestructive", checkHelpWroteNothing(readdirSync(helpSandbox)));
+ } catch (err) {
+ // The run is incomplete, not passing. Unreached checks stay `not_run` and
+ // the verdict cannot be CONSUMABLE.
+ aborted = { message: err instanceof Error ? err.message : String(err) };
+ console.error(`\nHarness aborted: ${aborted.message}`);
} finally {
- // --- Write receipt -----------------------------------------------------
- const receipt = {
- $comment:
- "Standard-candidate consumption harness receipt. Generated by scripts/migration/consume-standard-candidate.mjs — do not hand-edit.",
- generatedAt: new Date().toISOString(),
- summary: {
- total: checks.length,
- passed: checks.filter((c) => c.passed).length,
- failed: checks.filter((c) => !c.passed).length,
- },
- verdict: failures === 0 ? "CONSUMABLE" : "NOT_CONSUMABLE",
- checks: checks.map((c) => ({ name: c.name, status: c.passed ? "pass" : "fail", detail: c.detail ?? null })),
- };
-
- if (outDir) {
- mkdirSync(outDir, { recursive: true });
- writeFileSync(join(outDir, "consumption-receipt.json"), `${JSON.stringify(receipt, null, 2)}\n`);
- console.log(`\nReceipt: ${join(outDir, "consumption-receipt.json")}`);
+ // Close the client whatever happened. With the original happy-path-only
+ // close, any throw between connect and close leaked the spawned MCP server
+ // and hung CI instead of failing it.
+ if (client) {
+ try {
+ await client.close();
+ } catch (closeErr) {
+ console.error(`Warning: MCP client close failed: ${closeErr.message}`);
+ }
+ }
+
+ // --- Step 7: Source-tree safety ---------------------------------------
+ // Recorded last so it covers everything the harness did, and recorded even
+ // on abort — an aborted run is exactly when a stray mutation is likeliest.
+ try {
+ record("repo.tree-unchanged", checkTreeUnchanged(treeBefore, treeFingerprint()));
+ } catch (recordErr) {
+ console.error(`Warning: could not record tree check: ${recordErr.message}`);
+ }
+
+ const receipt = recorder.buildReceipt({ aborted });
+
+ if (opts.outDir) {
+ mkdirSync(opts.outDir, { recursive: true });
+ const receiptPath = join(opts.outDir, "consumption-receipt.json");
+ writeFileSync(receiptPath, `${JSON.stringify(receipt, null, 2)}\n`);
+ console.log(`\nReceipt: ${receiptPath}`);
}
+ const { total, passed, failed, notRun } = receipt.summary;
console.log(
- `\nConsumption verdict: ${receipt.verdict} — ${receipt.summary.passed}/${receipt.summary.total} passed, ${receipt.summary.failed} failed.`,
+ `\nConsumption verdict: ${receipt.verdict} — ${passed}/${total} passed, ${failed} failed, ${notRun} not run.`,
);
+ if (aborted) console.log(`Aborted before completion: ${aborted.message}`);
+
rmSync(work, { recursive: true, force: true });
- process.exitCode = failures > 0 ? 1 : 0;
+ process.exitCode = receipt.verdict === "CONSUMABLE" ? 0 : 1;
}
}
-main().catch((err) => {
- console.error("consume-standard-candidate crashed:", err);
- process.exitCode = 2;
-});
+const isDirectRun = process.argv[1] === fileURLToPath(import.meta.url);
+if (isDirectRun) {
+ main().catch((err) => {
+ console.error("consume-standard-candidate crashed:", err);
+ process.exitCode = 2;
+ });
+}
diff --git a/scripts/migration/consumption-checks.mjs b/scripts/migration/consumption-checks.mjs
new file mode 100644
index 0000000..4d228d3
--- /dev/null
+++ b/scripts/migration/consumption-checks.mjs
@@ -0,0 +1,434 @@
+/**
+ * Check predicates and receipt construction for the standard-candidate
+ * consumption harness (META-285 / META-235 Step 5 / META-258 I-7).
+ *
+ * This module exists so that every check the harness runs has exactly ONE
+ * implementation, which the harness calls and the tests exercise. The
+ * alternative — the harness asserting inline while a test reimplements the
+ * same logic — is the one-concept-two-implementations class tracked in
+ * META-140, and it already bit `verify-receipt.test.ts`.
+ *
+ * Contract for every predicate here (the META-165 scope amendment):
+ *
+ * No verification check enters service until it has been demonstrated to
+ * FAIL when the property is broken and to PASS when it is not.
+ *
+ * Two consequences shape the code below:
+ *
+ * 1. Predicates take plain data, never live handles, so a test can hand them
+ * a deliberately broken input without standing up an MCP server.
+ * 2. Predicates never infer success from absence. A `spawnSync` result that
+ * never launched, a result object of the wrong shape, and a missing
+ * optional field are all explicit failures with a stated reason — not a
+ * falsy value that happens to land on the passing side of a comparison.
+ *
+ * Defect 2 in META-238 and defect 1 in META-285 are the same root cause read
+ * off a property nobody verified. There the silent `undefined` produced a
+ * vacuous pass; here it produced a guaranteed fail. The direction was luck.
+ * These predicates remove the luck by checking shape before value.
+ */
+
+/**
+ * Every check the harness is expected to run, in execution order.
+ *
+ * The plan is declared up front rather than accumulated as checks execute.
+ * That is what lets the receipt distinguish "this check failed" from "this
+ * check never ran": a harness that throws midway leaves the unreached entries
+ * as `not_run` instead of silently shortening the list and reporting a clean
+ * sweep of whatever happened to complete first.
+ */
+export const CHECK_PLAN = [
+ { id: "pack.tarball", step: 1, name: "npm pack produces a tarball" },
+ { id: "pack.dist-index", step: 1, name: "tarball contains dist/index.js" },
+ { id: "pack.mcp-json", step: 1, name: "tarball contains .mcp.json" },
+ { id: "pack.hooks", step: 1, name: "tarball contains hooks/" },
+ { id: "install.from-tarball", step: 2, name: "npm install from tarball succeeds" },
+ { id: "install.has-install-script", step: 2, name: "installed package has scripts/install.mjs" },
+ { id: "install.has-dist-index", step: 2, name: "installed package has dist/index.js" },
+ { id: "mcp.instructions-fragile", step: 4, name: "server instructions contain FRAGILE" },
+ { id: "mcp.tool-call-responds", step: 4, name: "server responds to tool calls" },
+ { id: "mcp.tools-list", step: 4, name: "tools/list returns the 4 expected tools" },
+ { id: "mcp.file-context-tier", step: 4, name: "file context returns fragility tier" },
+ { id: "mcp.file-context-evidence", step: 4, name: "file context returns evidence" },
+ { id: "mcp.cochange-partners", step: 4, name: "co-change partners returns array" },
+ { id: "mcp.assess-change", step: 4, name: "assess change returns decision" },
+ { id: "hook.exists", step: 5, name: "hook script exists in packed artifact" },
+ { id: "hook.denies", step: 5, name: "hook exits non-zero on evidenced-fragile without partners" },
+ { id: "hook.output-mentions-deny", step: 5, name: "hook output mentions FRAGILE or deny" },
+ { id: "installer.exists", step: 6, name: "installer script exists in packed artifact" },
+ { id: "installer.help-usage", step: 6, name: "installer --help works from packed artifact" },
+ { id: "installer.help-nondestructive", step: 6, name: "installer --help writes no config into its cwd" },
+ { id: "repo.tree-unchanged", step: 7, name: "harness leaves the source working tree unmodified" },
+];
+
+export const EXPECTED_TOOLS = [
+ "workspace_assess_change",
+ "workspace_get_cochange_partners",
+ "workspace_get_file_context",
+ "workspace_list_fragile_files",
+];
+
+const pass = (detail) => ({ passed: true, detail: detail ?? null });
+const fail = (detail) => ({ passed: false, detail: detail ?? null });
+
+// ── Step 1: pack inventory ────────────────────────────────────────────────
+
+/**
+ * `npm pack --json` reports the files it placed in the tarball. Shape is
+ * verified before use: a missing or non-array `files` means we learned
+ * nothing about the tarball, which is a failure, not an absent violation.
+ */
+function packFilePaths(packMeta) {
+ if (!packMeta || typeof packMeta !== "object") {
+ return { error: "pack metadata is not an object" };
+ }
+ if (!Array.isArray(packMeta.files)) {
+ return { error: `pack metadata has no files[] array (got ${typeof packMeta.files})` };
+ }
+ const paths = [];
+ for (const entry of packMeta.files) {
+ if (!entry || typeof entry.path !== "string") {
+ return { error: `pack metadata contains an entry without a string path: ${JSON.stringify(entry)}` };
+ }
+ paths.push(entry.path);
+ }
+ return { paths };
+}
+
+export function checkTarballExists(tarballExists, filename) {
+ return tarballExists
+ ? pass(filename ?? null)
+ : fail(`tarball not found at destination (${filename ?? "no filename"})`);
+}
+
+/** Exact-path membership, for files that ship at a known path. */
+export function checkPackContainsPath(packMeta, wanted) {
+ const { paths, error } = packFilePaths(packMeta);
+ if (error) return fail(error);
+ if (paths.includes(wanted)) return pass(wanted);
+ return fail(`'${wanted}' absent from ${paths.length} packed files`);
+}
+
+/**
+ * Prefix membership, for directories. `npm pack --json` emits individual file
+ * paths and never a directory entry, so `path === "hooks/"` can never match —
+ * META-285 defect 2. A directory is present iff some packed file lives under it.
+ */
+export function checkPackContainsPrefix(packMeta, prefix) {
+ const { paths, error } = packFilePaths(packMeta);
+ if (error) return fail(error);
+ const matches = paths.filter((p) => p.startsWith(prefix));
+ if (matches.length > 0) return pass(`${matches.length} file(s) under '${prefix}': ${matches.slice(0, 5).join(", ")}`);
+ return fail(`no packed file starts with '${prefix}' (${paths.length} packed files)`);
+}
+
+// ── Process results ───────────────────────────────────────────────────────
+
+/**
+ * Classify a `spawnSync` result before reading its status.
+ *
+ * `spawnSync` returns `status: null` when the process never exited normally —
+ * it failed to launch (ENOENT) or died on a signal. Any predicate that reads
+ * `status` without this guard is unsound in one direction or the other:
+ * `status === 0` turns a launch failure into a fail (survivable), while
+ * `status !== 0` turns a launch failure into a PASS (vacuous — META-285
+ * defect, `hook.denies` at the original `:206`).
+ */
+function launched(result) {
+ if (!result || typeof result !== "object") return "no spawn result";
+ if (result.error) return `process failed to launch: ${result.error.message}`;
+ if (result.status === null || result.status === undefined) {
+ return `process did not exit normally (signal ${result.signal ?? "unknown"})`;
+ }
+ return null;
+}
+
+export function checkProcessSucceeded(result, what) {
+ const problem = launched(result);
+ if (problem) return fail(`${what}: ${problem}`);
+ if (result.status !== 0) return fail(`${what}: exit ${result.status} ${(result.stderr ?? "").slice(-300)}`.trim());
+ return pass(`${what}: exit 0`);
+}
+
+export function checkPathExists(exists, path) {
+ return exists ? pass(path) : fail(`not found: ${path}`);
+}
+
+// ── Step 4: MCP server responses ──────────────────────────────────────────
+
+export function checkInstructionsMention(instructions, token) {
+ if (typeof instructions !== "string") {
+ return fail(
+ `server returned no instructions string (got ${instructions === undefined ? "undefined" : typeof instructions})`,
+ );
+ }
+ if (!instructions.includes(token)) {
+ return fail(`instructions present (${instructions.length} chars) but do not mention '${token}'`);
+ }
+ return pass(`instructions mention '${token}'`);
+}
+
+/**
+ * A tool call responded usefully.
+ *
+ * `isError` is optional on `CallToolResult`, so `!result.isError` alone passes
+ * for a malformed or empty result — it cannot distinguish "the server answered"
+ * from "we got something that isn't a tool result". `content` is required by
+ * the MCP spec, so its presence is what proves we are holding a real result.
+ */
+export function checkToolCallResponded(result) {
+ if (!result || typeof result !== "object") {
+ return fail(`tool result is not an object (got ${result === undefined ? "undefined" : typeof result})`);
+ }
+ if (result.isError === true) {
+ return fail(`server reported isError: ${JSON.stringify(result.content ?? null).slice(0, 300)}`);
+ }
+ if (!Array.isArray(result.content)) {
+ return fail(`tool result has no content[] array (got ${typeof result.content}) — result shape unverified`);
+ }
+ return pass(`content entries: ${result.content.length}`);
+}
+
+export function checkToolsList(listResult, expected = EXPECTED_TOOLS) {
+ if (!listResult || typeof listResult !== "object") {
+ return fail(
+ `tools/list result is not an object (got ${listResult === undefined ? "undefined" : typeof listResult})`,
+ );
+ }
+ if (!Array.isArray(listResult.tools)) {
+ return fail(`tools/list result has no tools[] array (got ${typeof listResult.tools})`);
+ }
+ for (const tool of listResult.tools) {
+ if (!tool || typeof tool.name !== "string") {
+ return fail(`tools/list contains an entry without a string name: ${JSON.stringify(tool)}`);
+ }
+ }
+ const byName = (a, b) => a.localeCompare(b);
+ const actual = listResult.tools.map((t) => t.name).sort(byName);
+ const want = [...expected].sort(byName);
+ if (JSON.stringify(actual) !== JSON.stringify(want)) {
+ return fail(`expected [${want.join(", ")}], got [${actual.join(", ")}]`);
+ }
+ return pass(actual.join(","));
+}
+
+function structuredContent(result) {
+ if (!result || typeof result !== "object") {
+ return { error: `tool result is not an object (got ${result === undefined ? "undefined" : typeof result})` };
+ }
+ if (result.isError === true) return { error: "server reported isError" };
+ if (!result.structuredContent || typeof result.structuredContent !== "object") {
+ return { error: `tool result has no structuredContent object (got ${typeof result.structuredContent})` };
+ }
+ return { value: result.structuredContent };
+}
+
+export function checkFragilityTier(result) {
+ const { value, error } = structuredContent(result);
+ if (error) return fail(error);
+ const tier = value.fragility?.tier;
+ if (tier === undefined || tier === null)
+ return fail(`structuredContent.fragility.tier absent: ${JSON.stringify(value).slice(0, 300)}`);
+ return pass(`tier=${JSON.stringify(tier)}`);
+}
+
+export function checkFragilityEvidence(result) {
+ const { value, error } = structuredContent(result);
+ if (error) return fail(error);
+ if (!Array.isArray(value.fragility?.evidence)) {
+ return fail(`structuredContent.fragility.evidence is not an array (got ${typeof value.fragility?.evidence})`);
+ }
+ return pass(`${value.fragility.evidence.length} evidence entries`);
+}
+
+export function checkCochangePartners(result) {
+ const { value, error } = structuredContent(result);
+ if (error) return fail(error);
+ if (!Array.isArray(value.partners)) {
+ return fail(`structuredContent.partners is not an array (got ${typeof value.partners})`);
+ }
+ return pass(`${value.partners.length} partners`);
+}
+
+export function checkAssessDecision(result) {
+ const { value, error } = structuredContent(result);
+ if (error) return fail(error);
+ if (value.action === undefined && value.assessments === undefined) {
+ return fail(`structuredContent has neither 'action' nor 'assessments': ${JSON.stringify(value).slice(0, 300)}`);
+ }
+ return pass(JSON.stringify({ action: value.action, assessments: value.assessments }).slice(0, 300));
+}
+
+// ── Step 5: hook behaviour ────────────────────────────────────────────────
+
+/**
+ * Node exited non-zero because it could not run the script at all, rather than
+ * because the script decided something.
+ *
+ * `node ` launches successfully — the node binary
+ * exists — and exits 1 after failing to load. A predicate that only asks
+ * "non-zero?" reads that crash as a deny, so a candidate shipping NO hook at
+ * all scores a passing deny check. This is the same vacuous-pass shape as the
+ * original `status !== 0`, one level further in, and it was caught by running
+ * the harness against a candidate with `hooks/` removed from `files` rather
+ * than by any unit case.
+ */
+function crashedInsteadOfRunning(result) {
+ const output = `${result.stdout ?? ""}${result.stderr ?? ""}`;
+ if (/ERR_MODULE_NOT_FOUND|ERR_UNKNOWN_FILE_EXTENSION|Cannot find module/.test(output)) {
+ return "node could not load the hook script (module not found)";
+ }
+ // Horizontal whitespace only: under /m, `\s*` also matches newlines, so the
+ // indent could be consumed across line boundaries — ambiguity that both
+ // backtracks super-linearly and lets a non-indented "at" line match.
+ if (/^[ \t]*at .*node:internal/m.test(output) && /Error\b/.test(output)) {
+ return "node exited on an unhandled error before the hook reached a decision";
+ }
+ return null;
+}
+
+/**
+ * The hook must actively deny — launch, run to a decision, and exit non-zero.
+ * A hook that cannot launch, or that crashes before deciding, is a failure of
+ * the packed artifact, not a denial.
+ */
+export function checkHookDenies(result) {
+ const problem = launched(result);
+ if (problem) return fail(`hook did not run: ${problem}`);
+ if (result.status === 0) return fail("hook exited 0; expected a non-zero deny on an evidenced-fragile path");
+ const crash = crashedInsteadOfRunning(result);
+ if (crash) return fail(`${crash} — exit ${result.status} is a crash, not a deny`);
+ return pass(`exit ${result.status}`);
+}
+
+export function checkHookOutputMentionsDeny(result) {
+ const problem = launched(result);
+ if (problem) return fail(`hook did not run: ${problem}`);
+ const output = `${result.stdout ?? ""}${result.stderr ?? ""}`;
+ if (output.trim() === "") return fail("hook produced no output");
+ if (!/FRAGILE|deny|block/i.test(output)) return fail(`output mentions no denial: ${output.slice(-300)}`);
+ return pass(output.slice(-200).trim());
+}
+
+// ── Step 6: installer help ────────────────────────────────────────────────
+
+/**
+ * `--help` must print usage and exit 0.
+ *
+ * Checked against combined stdout+stderr deliberately. The original harness
+ * read `stdout` only while the installer wrote `USAGE` to `console.error`, so
+ * the check could not pass even when help worked. Which stream carries usage
+ * is a presentation choice; that help is shown at all is the property.
+ */
+export function checkInstallerHelp(result) {
+ const problem = launched(result);
+ if (problem) return fail(`installer did not run: ${problem}`);
+ const output = `${result.stdout ?? ""}${result.stderr ?? ""}`;
+ if (result.status !== 0) return fail(`--help exited ${result.status}: ${output.slice(-300)}`);
+ if (!/Usage:/i.test(output)) return fail(`--help exited 0 but printed no usage: ${output.slice(-300)}`);
+ return pass(`exit 0, usage printed (${output.length} chars)`);
+}
+
+/**
+ * `--help` must not install anything into the directory it runs in.
+ *
+ * This is the positive form of the META-285 stop-now item. Asserting only that
+ * the SOURCE tree is clean is weaker: it passes for the wrong reason whenever
+ * the source tree is already dirty, and it cannot see a destructive install
+ * aimed anywhere else. Running help inside a disposable sandbox and proving
+ * the sandbox stayed empty detects the destructive behaviour directly.
+ *
+ * `artifactsAfter` is the list of paths present in the sandbox after the run.
+ */
+export function checkHelpWroteNothing(artifactsAfter) {
+ if (!Array.isArray(artifactsAfter)) {
+ return fail(`sandbox listing unavailable (got ${typeof artifactsAfter}) — cannot prove --help was non-destructive`);
+ }
+ if (artifactsAfter.length > 0) {
+ return fail(`--help wrote into its cwd: ${artifactsAfter.join(", ")}`);
+ }
+ return pass("sandbox empty after --help");
+}
+
+// ── Step 7: source-tree safety ────────────────────────────────────────────
+
+/**
+ * The harness must not modify the repository it verifies.
+ *
+ * Compares `git status --porcelain` before and after. Comparing before/after
+ * rather than requiring "clean" means the check still proves the harness made
+ * no change when started from an already-dirty checkout, and it cannot be
+ * satisfied by an unrelated pre-existing dirty state.
+ */
+export function checkTreeUnchanged(before, after) {
+ if (typeof before !== "string" || typeof after !== "string") {
+ return fail("git status unavailable before or after the run — tree safety not established");
+ }
+ if (before !== after) {
+ const beforeSet = new Set(before.split("\n").filter(Boolean));
+ const changed = after.split("\n").filter((line) => line && !beforeSet.has(line));
+ return fail(`harness modified the source tree: ${changed.join(" | ") || "(entries removed)"}`);
+ }
+ return pass(before.trim() === "" ? "tree clean and unmodified" : "tree unmodified (pre-existing changes preserved)");
+}
+
+// ── Recorder / receipt ────────────────────────────────────────────────────
+
+/**
+ * Records outcomes against the declared plan and builds the receipt.
+ *
+ * Rejects unknown and duplicate check ids. A harness that stops running a
+ * planned check then surfaces it as `not_run`; one that renames a check fails
+ * loudly here rather than quietly shrinking the evidence set.
+ */
+export function createRecorder(plan = CHECK_PLAN) {
+ const known = new Map(plan.map((c) => [c.id, c]));
+ const results = new Map();
+
+ return {
+ record(id, outcome) {
+ const spec = known.get(id);
+ if (!spec) throw new Error(`consumption-checks: unknown check id '${id}' — not in CHECK_PLAN`);
+ if (results.has(id)) throw new Error(`consumption-checks: check '${id}' recorded twice`);
+ if (!outcome || typeof outcome.passed !== "boolean") {
+ throw new Error(`consumption-checks: check '${id}' recorded a malformed outcome: ${JSON.stringify(outcome)}`);
+ }
+ results.set(id, outcome);
+ return { ...spec, ...outcome };
+ },
+
+ buildReceipt({ generatedAt, aborted = null } = {}) {
+ const checks = plan.map(({ id, name, step }) => {
+ const outcome = results.get(id);
+ if (!outcome) return { id, name, step, status: "not_run", detail: null };
+ return { id, name, step, status: outcome.passed ? "pass" : "fail", detail: outcome.detail ?? null };
+ });
+
+ const count = (status) => checks.filter((c) => c.status === status).length;
+ const failed = count("fail");
+ const notRun = count("not_run");
+
+ // Precedence is deliberate. A real failure outranks incompleteness, and
+ // CONSUMABLE requires that every planned check actually ran. The old
+ // harness computed `failures === 0 ? "CONSUMABLE" : ...` over only the
+ // checks that happened to execute, so a crash after seven passing checks
+ // produced "7/7 passed — CONSUMABLE" for a harness that never finished.
+ const verdict = failed > 0 ? "NOT_CONSUMABLE" : notRun > 0 ? "INCOMPLETE" : "CONSUMABLE";
+
+ return {
+ $comment:
+ "Standard-candidate consumption harness receipt. Generated by scripts/migration/consume-standard-candidate.mjs — do not hand-edit.",
+ generatedAt: generatedAt ?? new Date().toISOString(),
+ verdict,
+ aborted,
+ summary: { total: checks.length, passed: count("pass"), failed, notRun },
+ checks,
+ };
+ },
+ };
+}
+
+/** A receipt is admissible evidence of consumability only when nothing failed AND nothing was skipped. */
+export function receiptIsClean(receipt) {
+ return receipt.verdict === "CONSUMABLE";
+}
diff --git a/scripts/migration/verify-receipt.mjs b/scripts/migration/verify-receipt.mjs
index 861d5fe..90cf975 100644
--- a/scripts/migration/verify-receipt.mjs
+++ b/scripts/migration/verify-receipt.mjs
@@ -24,12 +24,95 @@
* node scripts/migration/verify-receipt.mjs
*/
import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
-function readJson(path) {
- return JSON.parse(readFileSync(path, "utf8"));
+/**
+ * Read a receipt, failing with a message a CI reader can act on.
+ *
+ * A raw `ENOENT` or `SyntaxError` stack from a gate script says nothing about
+ * which of the two receipts was bad or what was expected of it. `label` names
+ * the role so the failure is legible without opening the script.
+ */
+export function readReceipt(path, label = "receipt") {
+ let raw;
+ try {
+ raw = readFileSync(path, "utf8");
+ } catch (err) {
+ if (err.code === "ENOENT") {
+ throw new Error(`${label} not found: ${path}\nExpected a parity receipt JSON file at this path.`);
+ }
+ throw new Error(`${label} could not be read: ${path}\n${err.message}`);
+ }
+
+ let parsed;
+ try {
+ parsed = JSON.parse(raw);
+ } catch (err) {
+ throw new Error(`${label} is not valid JSON: ${path}\n${err.message}`);
+ }
+
+ if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
+ const kind = Array.isArray(parsed) ? "array" : parsed === null ? "null" : typeof parsed;
+ throw new Error(`${label} is not a receipt object: ${path}\nParsed a ${kind}.`);
+ }
+ return parsed;
+}
+
+/**
+ * Reject a receipt missing the fields the comparator reads.
+ *
+ * Without this, a receipt lacking `summary` or `checks` throws a raw
+ * `TypeError` from inside the comparison, and a well-formed receipt with an
+ * empty check set compares clean against another empty one — a vacuous pass
+ * on the gate's own input.
+ *
+ * Duplicate ids are rejected for the same reason. The comparator projects
+ * `checks` into a `Map` keyed by id, which keeps only the last entry for a
+ * repeated id and silently discards the earlier one. A receipt carrying the
+ * same check twice with divergent statuses would then compare clean against
+ * the survivor — an internally inconsistent receipt reported as matching.
+ * `createRecorder` in ./consumption-checks.mjs already refuses to *emit* such
+ * a receipt; this refuses to *read* one, so neither end of the gate trusts it.
+ */
+function assertReceiptShape(receipt, label) {
+ const problems = [];
+ if (typeof receipt.verdict !== "string") problems.push("missing 'verdict' string");
+ if (!receipt.summary || typeof receipt.summary !== "object") problems.push("missing 'summary' object");
+ if (!Array.isArray(receipt.checks)) problems.push("missing 'checks' array");
+ else if (receipt.checks.length === 0) problems.push("'checks' is empty — nothing to compare");
+ else if (receipt.checks.some((c) => !c || typeof c.id !== "string")) {
+ problems.push("'checks' contains an entry without a string 'id'");
+ } else {
+ const seen = new Set();
+ const duplicated = new Set();
+ for (const { id } of receipt.checks) {
+ if (seen.has(id)) duplicated.add(id);
+ seen.add(id);
+ }
+ if (duplicated.size > 0) {
+ const ids = [...duplicated].sort((a, b) => a.localeCompare(b)).join(", ");
+ problems.push(`'checks' contains duplicate ids: ${ids}`);
+ }
+ }
+
+ if (problems.length > 0) {
+ throw new Error(`${label} is not a usable parity receipt:\n${problems.map((p) => ` - ${p}`).join("\n")}`);
+ }
}
-function compareReceipts(reference, candidate) {
+/**
+ * The single comparator implementation.
+ *
+ * Exported so `tests/migration/verify-receipt.test.ts` exercises THIS function
+ * rather than a copy. The test previously reimplemented it inline, so the two
+ * drifted — the copy's violation messages already differed from these — and
+ * the tests could not have caught a regression in the comparator that actually
+ * runs in CI (META-140 defect class, recorded on META-285).
+ */
+export function compareReceipts(reference, candidate) {
+ assertReceiptShape(reference, "reference receipt");
+ assertReceiptShape(candidate, "candidate receipt");
+
const violations = [];
if (reference.verdict !== candidate.verdict) {
@@ -91,10 +174,19 @@ function main() {
process.exit(2);
}
- const reference = readJson(refPath);
- const candidate = readJson(candPath);
-
- const violations = compareReceipts(reference, candidate);
+ let reference;
+ let candidate;
+ let violations;
+ try {
+ reference = readReceipt(refPath, "reference receipt");
+ candidate = readReceipt(candPath, "candidate receipt");
+ violations = compareReceipts(reference, candidate);
+ } catch (err) {
+ // Exit 2 — a gate that could not read its inputs did not run. That is
+ // distinct from exit 1, "the receipts diverged", which is a real result.
+ console.error(`Receipt reproduction COULD NOT RUN:\n\n${err.message}\n`);
+ process.exit(2);
+ }
if (violations.length > 0) {
console.error("Receipt reproduction FAILED — committed receipt does not match CI-generated receipt:\n");
@@ -110,4 +202,7 @@ function main() {
);
}
-main();
+const isDirectRun = process.argv[1] === fileURLToPath(import.meta.url);
+if (isDirectRun) {
+ main();
+}
diff --git a/tests/migration/consumption-checks.test.ts b/tests/migration/consumption-checks.test.ts
new file mode 100644
index 0000000..249ccfd
--- /dev/null
+++ b/tests/migration/consumption-checks.test.ts
@@ -0,0 +1,528 @@
+import { describe, expect, it } from "vitest";
+import {
+ CHECK_PLAN,
+ EXPECTED_TOOLS,
+ checkAssessDecision,
+ checkCochangePartners,
+ checkFragilityEvidence,
+ checkFragilityTier,
+ checkHelpWroteNothing,
+ checkHookDenies,
+ checkHookOutputMentionsDeny,
+ checkInstallerHelp,
+ checkInstructionsMention,
+ checkPackContainsPath,
+ checkPackContainsPrefix,
+ checkPathExists,
+ checkProcessSucceeded,
+ checkTarballExists,
+ checkToolCallResponded,
+ checkToolsList,
+ checkTreeUnchanged,
+ createRecorder,
+} from "../../scripts/migration/consumption-checks.mjs";
+
+// Watched-red contract (META-165 scope amendment, recorded on META-285):
+//
+// No verification check enters service until it has been demonstrated to
+// FAIL when the property is broken and to PASS when it is not.
+//
+// Every standing check in CHECK_PLAN therefore appears below at least twice:
+// once green against a valid input and once red against a controlled broken
+// one. `covers()` records which plan entry each block proves, and a final test
+// asserts the plan is fully covered — so adding a check to the harness without
+// watching it go red fails the suite rather than shipping unproven.
+
+const proven = new Set();
+function covers(id: string): string {
+ proven.add(id);
+ return id;
+}
+
+const okProc = (over: Record = {}) => ({
+ status: 0,
+ signal: null,
+ stdout: "",
+ stderr: "",
+ error: undefined,
+ ...over,
+});
+
+/** What spawnSync actually returns when the binary does not exist. */
+const failedToLaunch = () => ({
+ status: null,
+ signal: null,
+ stdout: null,
+ stderr: null,
+ error: Object.assign(new Error("spawnSync node ENOENT"), { code: "ENOENT" }),
+});
+
+/** What spawnSync returns when the child is killed (e.g. timeout). */
+const killedBySignal = () => okProc({ status: null, signal: "SIGTERM" });
+
+describe(covers("pack.tarball"), () => {
+ it("green: tarball present at destination", () => {
+ expect(checkTarballExists(true, "codex-mcp-0.1.9.tgz").passed).toBe(true);
+ });
+ it("red: tarball absent", () => {
+ const r = checkTarballExists(false, "codex-mcp-0.1.9.tgz");
+ expect(r.passed).toBe(false);
+ expect(r.detail).toMatch(/not found/);
+ });
+});
+
+describe("pack inventory", () => {
+ const packMeta = {
+ filename: "codex-mcp-0.1.9.tgz",
+ files: [
+ { path: "dist/index.js" },
+ { path: "dist/index.d.ts" },
+ { path: ".mcp.json" },
+ { path: "hooks/pre-edit-check.mjs" },
+ { path: "hooks/hooks.json" },
+ { path: "scripts/install.mjs" },
+ ],
+ };
+
+ it(`green: ${covers("pack.dist-index")} / ${covers("pack.mcp-json")} present`, () => {
+ expect(checkPackContainsPath(packMeta, "dist/index.js").passed).toBe(true);
+ expect(checkPackContainsPath(packMeta, ".mcp.json").passed).toBe(true);
+ });
+
+ it("red: a required file is missing from the tarball", () => {
+ const without = { ...packMeta, files: packMeta.files.filter((f) => f.path !== ".mcp.json") };
+ const r = checkPackContainsPath(without, ".mcp.json");
+ expect(r.passed).toBe(false);
+ expect(r.detail).toMatch(/absent from 5 packed files/);
+ });
+
+ it(`green: ${covers("pack.hooks")} matches files under the directory`, () => {
+ const r = checkPackContainsPrefix(packMeta, "hooks/");
+ expect(r.passed).toBe(true);
+ expect(r.detail).toMatch(/2 file\(s\) under 'hooks\/'/);
+ });
+
+ it("red: no files under the directory", () => {
+ const without = { ...packMeta, files: packMeta.files.filter((f) => !f.path.startsWith("hooks/")) };
+ expect(checkPackContainsPrefix(without, "hooks/").passed).toBe(false);
+ });
+
+ // META-285 defect 2 regression. `npm pack --json` emits file paths and never
+ // a directory entry, so an exact `path === "hooks/"` test could never pass.
+ // A prefix match must still succeed on that real-world inventory shape.
+ it("regression: passes on an inventory that contains no directory entries", () => {
+ expect(packMeta.files.some((f) => f.path === "hooks/")).toBe(false);
+ expect(checkPackContainsPrefix(packMeta, "hooks/").passed).toBe(true);
+ });
+
+ // Shape is verified before value: an inventory we could not read is a
+ // failure, not an absence of violations.
+ it("red: pack metadata without a files[] array fails rather than reporting absence", () => {
+ expect(checkPackContainsPath({ filename: "x.tgz" }, "dist/index.js").passed).toBe(false);
+ expect(checkPackContainsPrefix(undefined, "hooks/").passed).toBe(false);
+ expect(checkPackContainsPath({ files: [{ notAPath: 1 }] }, "dist/index.js").detail).toMatch(
+ /without a string path/,
+ );
+ });
+});
+
+describe(covers("install.from-tarball"), () => {
+ it("green: install exits 0", () => {
+ expect(checkProcessSucceeded(okProc(), "npm install").passed).toBe(true);
+ });
+ it("red: install exits non-zero", () => {
+ const r = checkProcessSucceeded(okProc({ status: 1, stderr: "E404 not found" }), "npm install");
+ expect(r.passed).toBe(false);
+ expect(r.detail).toMatch(/exit 1/);
+ });
+ // Guards the direction the original harness got wrong elsewhere: a process
+ // that never launched must not be read as a result.
+ it("red: install never launched", () => {
+ const r = checkProcessSucceeded(failedToLaunch(), "npm install");
+ expect(r.passed).toBe(false);
+ expect(r.detail).toMatch(/failed to launch/);
+ });
+});
+
+describe(`${covers("install.has-install-script")} / ${covers("install.has-dist-index")} / ${covers("hook.exists")} / ${covers("installer.exists")}`, () => {
+ it("green: path exists", () => {
+ expect(checkPathExists(true, "/pkg/dist/index.js").passed).toBe(true);
+ });
+ it("red: path missing", () => {
+ const r = checkPathExists(false, "/pkg/dist/index.js");
+ expect(r.passed).toBe(false);
+ expect(r.detail).toMatch(/not found: \/pkg\/dist\/index\.js/);
+ });
+});
+
+describe(covers("mcp.instructions-fragile"), () => {
+ it("green: instructions mention the token", () => {
+ expect(checkInstructionsMention("Treat FRAGILE files with care", "FRAGILE").passed).toBe(true);
+ });
+ it("red: instructions present but do not mention the token", () => {
+ const r = checkInstructionsMention("Nothing relevant here", "FRAGILE");
+ expect(r.passed).toBe(false);
+ expect(r.detail).toMatch(/do not mention 'FRAGILE'/);
+ });
+ it("red: server returned no instructions at all", () => {
+ const r = checkInstructionsMention(undefined, "FRAGILE");
+ expect(r.passed).toBe(false);
+ expect(r.detail).toMatch(/no instructions string/);
+ });
+});
+
+describe(covers("mcp.tool-call-responds"), () => {
+ it("green: a real tool result with content", () => {
+ expect(checkToolCallResponded({ content: [{ type: "text", text: "ok" }] }).passed).toBe(true);
+ });
+ it("red: server reported isError", () => {
+ expect(checkToolCallResponded({ content: [], isError: true }).passed).toBe(false);
+ });
+ // The vacuous-pass direction. `!result.isError` alone passes for every one of
+ // these, because `isError` is optional on CallToolResult and absent here.
+ it("red: results that are not tool results do not pass merely by lacking isError", () => {
+ expect(checkToolCallResponded(undefined).passed).toBe(false);
+ expect(checkToolCallResponded({}).passed).toBe(false);
+ expect(checkToolCallResponded({ structuredContent: {} }).passed).toBe(false);
+ expect(checkToolCallResponded("ok").passed).toBe(false);
+ expect(checkToolCallResponded({}).detail).toMatch(/result shape unverified/);
+ });
+});
+
+describe(covers("mcp.tools-list"), () => {
+ const listResult = { tools: EXPECTED_TOOLS.map((name) => ({ name })) };
+
+ it("green: exactly the expected tools, order-independent", () => {
+ expect(checkToolsList(listResult).passed).toBe(true);
+ expect(checkToolsList({ tools: [...listResult.tools].reverse() }).passed).toBe(true);
+ });
+ it("red: a tool is missing", () => {
+ const r = checkToolsList({ tools: listResult.tools.slice(1) });
+ expect(r.passed).toBe(false);
+ expect(r.detail).toMatch(/expected \[/);
+ });
+ it("red: an unexpected tool appeared", () => {
+ expect(checkToolsList({ tools: [...listResult.tools, { name: "workspace_rm_rf" }] }).passed).toBe(false);
+ });
+ // META-285 defect 1 regression. The original read `tools` off a callTool
+ // result, which has no such property, so this check could never pass. It must
+ // pass against a genuine ListToolsResult and fail when the property is absent.
+ it("red: a result with no tools[] array — the shape the original destructured", () => {
+ const callToolResult = { content: [], structuredContent: {}, isError: false };
+ const r = checkToolsList(callToolResult);
+ expect(r.passed).toBe(false);
+ expect(r.detail).toMatch(/no tools\[\] array/);
+ });
+});
+
+describe(`${covers("mcp.file-context-tier")} / ${covers("mcp.file-context-evidence")}`, () => {
+ const good = {
+ content: [],
+ structuredContent: { fragility: { tier: "evidenced-fragile", evidence: [{ kind: "co-change" }] } },
+ };
+
+ it("green: tier and evidence present", () => {
+ expect(checkFragilityTier(good).passed).toBe(true);
+ expect(checkFragilityEvidence(good).passed).toBe(true);
+ });
+ it("green: tier reported even when evidence is an empty array", () => {
+ const empty = { content: [], structuredContent: { fragility: { tier: "stable", evidence: [] } } };
+ expect(checkFragilityEvidence(empty).passed).toBe(true);
+ });
+ it("red: tier absent", () => {
+ expect(checkFragilityTier({ content: [], structuredContent: { fragility: {} } }).passed).toBe(false);
+ });
+ it("red: evidence is not an array", () => {
+ const bad = { content: [], structuredContent: { fragility: { tier: "x", evidence: "some" } } };
+ expect(checkFragilityEvidence(bad).passed).toBe(false);
+ });
+ it("red: no structuredContent at all", () => {
+ expect(checkFragilityTier({ content: [] }).detail).toMatch(/no structuredContent/);
+ expect(checkFragilityEvidence({ content: [], isError: true }).passed).toBe(false);
+ });
+});
+
+describe(covers("mcp.cochange-partners"), () => {
+ it("green: partners array present, including empty", () => {
+ expect(checkCochangePartners({ structuredContent: { partners: [{ path: "a" }] } }).passed).toBe(true);
+ expect(checkCochangePartners({ structuredContent: { partners: [] } }).passed).toBe(true);
+ });
+ it("red: partners missing or wrong type", () => {
+ expect(checkCochangePartners({ structuredContent: {} }).passed).toBe(false);
+ expect(checkCochangePartners({ structuredContent: { partners: "none" } }).passed).toBe(false);
+ });
+});
+
+describe(covers("mcp.assess-change"), () => {
+ it("green: either action or assessments present", () => {
+ expect(checkAssessDecision({ structuredContent: { action: "review" } }).passed).toBe(true);
+ expect(checkAssessDecision({ structuredContent: { assessments: [] } }).passed).toBe(true);
+ });
+ it("red: neither field present", () => {
+ const r = checkAssessDecision({ structuredContent: { unrelated: 1 } });
+ expect(r.passed).toBe(false);
+ expect(r.detail).toMatch(/neither 'action' nor 'assessments'/);
+ });
+});
+
+describe(covers("hook.denies"), () => {
+ it("green: hook ran and denied with a non-zero exit", () => {
+ const r = checkHookDenies(okProc({ status: 2 }));
+ expect(r.passed).toBe(true);
+ expect(r.detail).toBe("exit 2");
+ });
+ it("red: hook allowed the edit (exit 0)", () => {
+ expect(checkHookDenies(okProc({ status: 0 })).passed).toBe(false);
+ });
+
+ // THE vacuous-pass defect this issue asked us to look for.
+ //
+ // The original predicate was `hookResult.status !== 0`. When spawnSync fails
+ // to launch, status is null, and `null !== 0` is true — so a hook that does
+ // not exist at all scored a PASS, and the receipt claimed the packed artifact
+ // enforced denial. Absence must never be read as a deny.
+ it("red: hook binary missing — must NOT pass on status === null", () => {
+ const result = failedToLaunch();
+ expect(result.status !== 0).toBe(true); // what the old predicate saw
+ const r = checkHookDenies(result);
+ expect(r.passed).toBe(false); // what it must actually report
+ expect(r.detail).toMatch(/hook did not run/);
+ });
+ it("red: hook killed by a signal rather than exiting", () => {
+ const r = checkHookDenies(killedBySignal());
+ expect(r.passed).toBe(false);
+ expect(r.detail).toMatch(/did not exit normally/);
+ });
+
+ // The same vacuous pass one level further in, found by running the harness
+ // against a candidate built with `hooks/` removed from package.json files[].
+ // `node ` launches fine and exits 1, so every "did it launch
+ // and exit non-zero" predicate reads a candidate that ships NO hook as a
+ // passing deny. Unit cases alone did not catch this; the end-to-end broken
+ // candidate did.
+ it("red: node exited 1 because the hook script was missing, not because it denied", () => {
+ const moduleNotFound = okProc({
+ status: 1,
+ stderr:
+ "node:internal/modules/esm/resolve:275\n throw new ERR_MODULE_NOT_FOUND(\n ^\n" +
+ "Error [ERR_MODULE_NOT_FOUND]: Cannot find module '/pkg/hooks/pre-edit-check.mjs'\n" +
+ " at finalizeResolution (node:internal/modules/esm/resolve:275:11)\n",
+ });
+ expect(moduleNotFound.status !== 0).toBe(true); // still non-zero
+ const r = checkHookDenies(moduleNotFound);
+ expect(r.passed).toBe(false);
+ expect(r.detail).toMatch(/module not found.*is a crash, not a deny/);
+ });
+
+ it("red: hook threw before reaching a decision", () => {
+ const threw = okProc({
+ status: 1,
+ stderr: "Uncaught TypeError: cannot read properties of undefined\n at run (node:internal/main/run_main:1:1)\n",
+ });
+ expect(checkHookDenies(threw).passed).toBe(false);
+ });
+
+ // The deny path must not be collateral damage: a real deny that happens to
+ // print a stack-shaped evidence line still passes.
+ it("green: a genuine deny whose message mentions a path is not mistaken for a crash", () => {
+ const deny = okProc({
+ status: 2,
+ stdout: "DENY: src/routes/checkout.ts is FRAGILE at src/routes/checkout.ts:14 — include co-change partners",
+ });
+ expect(checkHookDenies(deny).passed).toBe(true);
+ });
+});
+
+describe(covers("hook.output-mentions-deny"), () => {
+ it("green: denial language on stdout or stderr", () => {
+ expect(checkHookOutputMentionsDeny(okProc({ status: 2, stdout: "FRAGILE: blocked" })).passed).toBe(true);
+ expect(checkHookOutputMentionsDeny(okProc({ status: 2, stderr: "deny" })).passed).toBe(true);
+ });
+ it("red: ran but said nothing about a denial", () => {
+ expect(checkHookOutputMentionsDeny(okProc({ status: 2, stdout: "all good" })).passed).toBe(false);
+ });
+ it("red: produced no output at all", () => {
+ const r = checkHookOutputMentionsDeny(okProc({ status: 2 }));
+ expect(r.passed).toBe(false);
+ expect(r.detail).toMatch(/no output/);
+ });
+ it("red: never launched", () => {
+ expect(checkHookOutputMentionsDeny(failedToLaunch()).passed).toBe(false);
+ });
+});
+
+describe(covers("installer.help-usage"), () => {
+ it("green: usage on stdout, exit 0", () => {
+ expect(checkInstallerHelp(okProc({ stdout: "Usage:\n install [--with-hook]" })).passed).toBe(true);
+ });
+
+ // META-285 defect 3, second half. The original asserted over `stdout` alone
+ // while the installer wrote USAGE to console.error, so the check could not
+ // pass even when help worked correctly. Which stream carries usage is a
+ // presentation choice; that help is shown is the property.
+ it("green: usage on stderr still counts as help being shown", () => {
+ expect(checkInstallerHelp(okProc({ stderr: "Usage:\n install" })).passed).toBe(true);
+ });
+
+ it("red: exited 0 but printed no usage", () => {
+ const r = checkInstallerHelp(okProc({ stdout: "installed!" }));
+ expect(r.passed).toBe(false);
+ expect(r.detail).toMatch(/printed no usage/);
+ });
+ it("red: non-zero exit", () => {
+ expect(checkInstallerHelp(okProc({ status: 1, stderr: "Usage:" })).passed).toBe(false);
+ });
+ it("red: installer never launched", () => {
+ expect(checkInstallerHelp(failedToLaunch()).passed).toBe(false);
+ });
+});
+
+describe(covers("installer.help-nondestructive"), () => {
+ it("green: sandbox empty after --help", () => {
+ expect(checkHelpWroteNothing([]).passed).toBe(true);
+ });
+
+ // The stop-now item, asserted positively. If `--help` regresses into a real
+ // install, the installer writes .codex/config.toml into its cwd. Running help
+ // in a disposable sandbox turns that into a visible red instead of a silent
+ // rewrite of the source checkout.
+ it("red: --help wrote a config into its working directory", () => {
+ const r = checkHelpWroteNothing([".codex"]);
+ expect(r.passed).toBe(false);
+ expect(r.detail).toMatch(/--help wrote into its cwd: \.codex/);
+ });
+ it("red: sandbox could not be listed — absence of a listing is not proof of safety", () => {
+ const r = checkHelpWroteNothing(undefined);
+ expect(r.passed).toBe(false);
+ expect(r.detail).toMatch(/cannot prove --help was non-destructive/);
+ });
+});
+
+describe(covers("repo.tree-unchanged"), () => {
+ it("green: clean before, clean after", () => {
+ const r = checkTreeUnchanged("", "");
+ expect(r.passed).toBe(true);
+ expect(r.detail).toMatch(/tree clean and unmodified/);
+ });
+
+ // Comparing before/after rather than requiring "clean" means the check still
+ // proves the harness changed nothing when it starts from a dirty checkout.
+ it("green: dirty before, identically dirty after", () => {
+ const dirty = " M src/index.ts\n?? scratch.txt\n";
+ const r = checkTreeUnchanged(dirty, dirty);
+ expect(r.passed).toBe(true);
+ expect(r.detail).toMatch(/pre-existing changes preserved/);
+ });
+
+ // The exact signature of the destructive --help path recorded in the Aug 4
+ // field observation: .codex/config.toml modified by the harness itself.
+ it("red: harness modified a tracked file", () => {
+ const r = checkTreeUnchanged("", " M .codex/config.toml\n");
+ expect(r.passed).toBe(false);
+ expect(r.detail).toMatch(/modified the source tree.*\.codex\/config\.toml/);
+ });
+ it("red: harness left a new untracked artifact behind", () => {
+ const r = checkTreeUnchanged("", "?? codex-mcp-0.1.9.tgz\n");
+ expect(r.passed).toBe(false);
+ expect(r.detail).toMatch(/codex-mcp-0\.1\.9\.tgz/);
+ });
+ it("red: git status unavailable — safety not established, not assumed", () => {
+ expect(checkTreeUnchanged(null, "").passed).toBe(false);
+ expect(checkTreeUnchanged("", null).detail).toMatch(/tree safety not established/);
+ });
+});
+
+describe("receipt: failed vs not-run", () => {
+ const plan = [
+ { id: "a", step: 1, name: "check a" },
+ { id: "b", step: 1, name: "check b" },
+ { id: "c", step: 2, name: "check c" },
+ ];
+
+ it("green: all planned checks ran and passed → CONSUMABLE", () => {
+ const rec = createRecorder(plan);
+ for (const { id } of plan) rec.record(id, { passed: true, detail: "ok" });
+ const receipt = rec.buildReceipt({ generatedAt: "2026-08-12T00:00:00.000Z" });
+ expect(receipt.verdict).toBe("CONSUMABLE");
+ expect(receipt.summary).toEqual({ total: 3, passed: 3, failed: 0, notRun: 0 });
+ expect(receipt.aborted).toBeNull();
+ });
+
+ it("red: a failure yields NOT_CONSUMABLE", () => {
+ const rec = createRecorder(plan);
+ rec.record("a", { passed: true });
+ rec.record("b", { passed: false, detail: "broken" });
+ rec.record("c", { passed: true });
+ const receipt = rec.buildReceipt({});
+ expect(receipt.verdict).toBe("NOT_CONSUMABLE");
+ expect(receipt.summary.failed).toBe(1);
+ expect(receipt.checks.find((c) => c.id === "b")?.status).toBe("fail");
+ });
+
+ // The receipt-level defect. The original computed the verdict from only the
+ // checks that happened to execute, so a harness that threw after two passing
+ // checks emitted "2/2 passed — CONSUMABLE". Unreached checks must be visible
+ // as not_run, and must make the run INCOMPLETE rather than clean.
+ it("red: an aborted run reports not_run and INCOMPLETE, never CONSUMABLE", () => {
+ const rec = createRecorder(plan);
+ rec.record("a", { passed: true });
+ rec.record("b", { passed: true });
+ const receipt = rec.buildReceipt({ aborted: { message: "MCP connect failed" } });
+
+ expect(receipt.verdict).toBe("INCOMPLETE");
+ expect(receipt.verdict).not.toBe("CONSUMABLE");
+ expect(receipt.summary).toEqual({ total: 3, passed: 2, failed: 0, notRun: 1 });
+ expect(receipt.checks.find((c) => c.id === "c")).toMatchObject({ status: "not_run", detail: null });
+ expect(receipt.aborted).toEqual({ message: "MCP connect failed" });
+ });
+
+ it("not_run is a distinct status from fail", () => {
+ const rec = createRecorder(plan);
+ rec.record("a", { passed: false, detail: "broken" });
+ const receipt = rec.buildReceipt({});
+ const statuses = Object.fromEntries(receipt.checks.map((c) => [c.id, c.status]));
+ expect(statuses).toEqual({ a: "fail", b: "not_run", c: "not_run" });
+ expect(receipt.summary.failed).toBe(1);
+ expect(receipt.summary.notRun).toBe(2);
+ });
+
+ it("a real failure outranks incompleteness in the verdict", () => {
+ const rec = createRecorder(plan);
+ rec.record("a", { passed: false });
+ expect(rec.buildReceipt({}).verdict).toBe("NOT_CONSUMABLE");
+ });
+
+ it("every check keeps its plan identity in the receipt", () => {
+ const receipt = createRecorder(plan).buildReceipt({});
+ expect(receipt.checks.map((c) => c.id)).toEqual(["a", "b", "c"]);
+ expect(receipt.checks[0]).toMatchObject({ id: "a", name: "check a", step: 1 });
+ });
+
+ // Guards against silent plan drift: a check that is renamed or recorded twice
+ // must fail loudly rather than quietly shrink the evidence set.
+ it("rejects unknown ids, duplicate records, and malformed outcomes", () => {
+ const rec = createRecorder(plan);
+ expect(() => rec.record("nonexistent", { passed: true })).toThrow(/unknown check id/);
+ rec.record("a", { passed: true });
+ expect(() => rec.record("a", { passed: true })).toThrow(/recorded twice/);
+ expect(() => rec.record("b", { detail: "no verdict" } as never)).toThrow(/malformed outcome/);
+ });
+});
+
+describe("watched-red coverage of the harness check plan", () => {
+ it("every check in CHECK_PLAN has been demonstrated red and green above", () => {
+ const planned = CHECK_PLAN.map((c) => c.id);
+ const unproven = planned.filter((id) => !proven.has(id));
+ expect(unproven).toEqual([]);
+ });
+
+ it("no test claims coverage of a check the harness does not run", () => {
+ const planned = new Set(CHECK_PLAN.map((c) => c.id));
+ expect([...proven].filter((id) => !planned.has(id))).toEqual([]);
+ });
+
+ it("check ids are unique", () => {
+ const ids = CHECK_PLAN.map((c) => c.id);
+ expect(ids.length).toBe(new Set(ids).size);
+ });
+});
diff --git a/tests/migration/verify-receipt.test.ts b/tests/migration/verify-receipt.test.ts
index 6ecf90d..7083e40 100644
--- a/tests/migration/verify-receipt.test.ts
+++ b/tests/migration/verify-receipt.test.ts
@@ -1,4 +1,13 @@
-import { describe, expect, it } from "vitest";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, describe, expect, it } from "vitest";
+// The comparator under test is imported, never reimplemented. An earlier
+// version of this file rebuilt `compareReceipts` inline, so the tests verified
+// a copy: the real comparator could regress without a single test failing, and
+// the copy's violation messages had already drifted from the source
+// (META-140 one-concept-two-implementations class, recorded on META-285).
+import { compareReceipts, readReceipt } from "../../scripts/migration/verify-receipt.mjs";
interface ReceiptCheck {
id: string;
@@ -15,63 +24,6 @@ interface Receipt {
[key: string]: unknown;
}
-// We test the comparison logic by importing the module and calling its
-// internal function. Since verify-receipt.mjs is a CLI script, we test
-// the core comparator by reconstructing it inline from the same logic.
-// This is a watched-red contract: the comparator must fail on perturbed input.
-
-function compareReceipts(reference: Receipt, candidate: Receipt): string[] {
- const violations = [];
-
- if (reference.verdict !== candidate.verdict) {
- violations.push(`verdict diverged: reference=${reference.verdict} candidate=${candidate.verdict}`);
- }
-
- for (const field of ["total", "passed", "failed", "unsupported"] as const) {
- if (reference.summary[field] !== candidate.summary[field]) {
- violations.push(
- `summary.${field} diverged: reference=${reference.summary[field]} candidate=${candidate.summary[field]}`,
- );
- }
- }
-
- const refChecks = new Map(reference.checks.map((c) => [c.id, c]));
- const candChecks = new Map(candidate.checks.map((c) => [c.id, c]));
-
- const missingInCandidate = [...refChecks.keys()].filter((id) => !candChecks.has(id));
- const missingInReference = [...candChecks.keys()].filter((id) => !refChecks.has(id));
-
- for (const id of missingInCandidate) {
- violations.push(`check '${id}' present in reference but missing from candidate`);
- }
- for (const id of missingInReference) {
- violations.push(`check '${id}' present in candidate but missing from reference`);
- }
-
- for (const [id, refCheck] of refChecks) {
- const candCheck = candChecks.get(id);
- if (!candCheck) continue;
-
- if (refCheck.status !== candCheck.status) {
- violations.push(`check '${id}' status diverged: reference=${refCheck.status} candidate=${candCheck.status}`);
- }
-
- const refViolations = JSON.stringify(refCheck.violations ?? []);
- const candViolations = JSON.stringify(candCheck.violations ?? []);
- if (refViolations !== candViolations) {
- violations.push(`check '${id}' violations diverged`);
- }
- }
-
- const refDiffs = JSON.stringify(reference.intentionalDifferences ?? []);
- const candDiffs = JSON.stringify(candidate.intentionalDifferences ?? []);
- if (refDiffs !== candDiffs) {
- violations.push("intentionalDifferences diverged");
- }
-
- return violations;
-}
-
const BASE_RECEIPT: Receipt = {
verdict: "PARITY",
summary: { total: 10, passed: 10, failed: 0, unsupported: 0 },
@@ -91,7 +43,7 @@ const BASE_RECEIPT: Receipt = {
};
describe("compareReceipts", () => {
- it("passes identical receipts ignoring non-deterministic fields", () => {
+ it("watched-green: passes identical receipts ignoring non-deterministic fields", () => {
const candidate: Receipt = structuredClone(BASE_RECEIPT);
candidate.generatedAt = "2026-08-01T12:00:00.000Z";
candidate.startedAt = "2026-08-01T11:59:30.000Z";
@@ -113,33 +65,40 @@ describe("compareReceipts", () => {
expect(violations.some((v) => /verdict diverged/.test(v))).toBe(true);
});
+ it("watched-red: catches a summary count change", () => {
+ const candidate: Receipt = structuredClone(BASE_RECEIPT);
+ candidate.summary.unsupported = 2;
+ const violations = compareReceipts(BASE_RECEIPT, candidate);
+ expect(violations.some((v) => /summary\.unsupported diverged/.test(v))).toBe(true);
+ });
+
it("watched-red: catches a check status flip", () => {
const candidate: Receipt = structuredClone(BASE_RECEIPT);
candidate.checks[4].status = "fail";
candidate.checks[4].violations = ["smoke failed"];
const violations = compareReceipts(BASE_RECEIPT, candidate);
- expect(violations.some((v) => /check 'mcp.smoke' status diverged/.test(v))).toBe(true);
+ expect(violations.some((v) => /check 'mcp\.smoke' status diverged/.test(v))).toBe(true);
});
it("watched-red: catches a missing check", () => {
const candidate: Receipt = structuredClone(BASE_RECEIPT);
candidate.checks = candidate.checks.slice(0, 9);
const violations = compareReceipts(BASE_RECEIPT, candidate);
- expect(violations.some((v) => /'generator.resolution' present in reference but missing/.test(v))).toBe(true);
+ expect(violations.some((v) => /'generator\.resolution' present in reference but missing/.test(v))).toBe(true);
});
it("watched-red: catches an extra check", () => {
const candidate: Receipt = structuredClone(BASE_RECEIPT);
candidate.checks.push({ id: "evil.check", status: "pass", violations: [], evidence: {} });
const violations = compareReceipts(BASE_RECEIPT, candidate);
- expect(violations.some((v) => /'evil.check' present in candidate but missing/.test(v))).toBe(true);
+ expect(violations.some((v) => /'evil\.check' present in candidate but missing/.test(v))).toBe(true);
});
it("watched-red: catches new violations in a previously clean check", () => {
const candidate: Receipt = structuredClone(BASE_RECEIPT);
candidate.checks[2].violations = ["package.version diverged"];
const violations = compareReceipts(BASE_RECEIPT, candidate);
- expect(violations.some((v) => /check 'pkg.identity' violations diverged/.test(v))).toBe(true);
+ expect(violations.some((v) => /check 'pkg\.identity' violations diverged/.test(v))).toBe(true);
});
it("watched-red: catches changed intentionalDifferences", () => {
@@ -147,4 +106,107 @@ describe("compareReceipts", () => {
candidate.intentionalDifferences = [{ path: "README.md", justification: "changed" }];
expect(compareReceipts(BASE_RECEIPT, candidate).some((v) => /intentionalDifferences diverged/.test(v))).toBe(true);
});
+
+ // Asserted against the real comparator's exact text. While the test owned a
+ // copy, these strings could drift from the shipped ones unnoticed — and had.
+ it("reports the actual reference and candidate values in the violation text", () => {
+ const candidate: Receipt = structuredClone(BASE_RECEIPT);
+ candidate.verdict = "DIVERGENT";
+ const [violation] = compareReceipts(BASE_RECEIPT, candidate);
+ expect(violation).toBe("verdict diverged: reference=PARITY candidate=DIVERGENT");
+ });
+
+ // A comparator that accepts a structurally empty receipt compares two empty
+ // check sets as identical — a pass that proves nothing about parity.
+ it("refuses a receipt with an empty check set rather than comparing it clean", () => {
+ const empty: Receipt = structuredClone(BASE_RECEIPT);
+ empty.checks = [];
+ expect(() => compareReceipts(empty, structuredClone(empty))).toThrow(/'checks' is empty/);
+ });
+
+ it("refuses a receipt missing the fields it reads", () => {
+ const noSummary = { verdict: "PARITY", checks: BASE_RECEIPT.checks } as unknown as Receipt;
+ expect(() => compareReceipts(noSummary, structuredClone(BASE_RECEIPT))).toThrow(/missing 'summary' object/);
+
+ const noChecks = { verdict: "PARITY", summary: BASE_RECEIPT.summary } as unknown as Receipt;
+ expect(() => compareReceipts(structuredClone(BASE_RECEIPT), noChecks)).toThrow(/missing 'checks' array/);
+ });
+
+ it("names which side was malformed", () => {
+ const bad = { verdict: "PARITY", summary: {}, checks: [{ status: "pass" }] } as unknown as Receipt;
+ expect(() => compareReceipts(structuredClone(BASE_RECEIPT), bad)).toThrow(/candidate receipt is not a usable/);
+ expect(() => compareReceipts(bad, structuredClone(BASE_RECEIPT))).toThrow(/reference receipt is not a usable/);
+ });
+
+ // The comparator keys `checks` by id into a Map, which keeps only the last
+ // entry for a repeated id. Without this guard a receipt carrying the same
+ // check twice with divergent statuses loses the earlier one and compares
+ // clean against the survivor — inconsistency reported as parity.
+ it("refuses a receipt carrying the same check id twice", () => {
+ const duplicated: Receipt = structuredClone(BASE_RECEIPT);
+ duplicated.checks.push({ ...duplicated.checks[0], status: "fail", violations: ["tree differs"] });
+
+ expect(() => compareReceipts(duplicated, structuredClone(BASE_RECEIPT))).toThrow(
+ /'checks' contains duplicate ids: git\.tree-equality/,
+ );
+ expect(() => compareReceipts(structuredClone(BASE_RECEIPT), duplicated)).toThrow(
+ /candidate receipt is not a usable/,
+ );
+ });
+
+ // Guards the collapse directly: absent the duplicate check, the losing entry
+ // is discarded and this pair compares clean despite disagreeing.
+ it("does not let a duplicate id mask a status divergence", () => {
+ const reference: Receipt = structuredClone(BASE_RECEIPT);
+ const candidate: Receipt = structuredClone(BASE_RECEIPT);
+ candidate.checks[0].status = "fail";
+ candidate.checks.push({ ...BASE_RECEIPT.checks[0] });
+
+ expect(() => compareReceipts(reference, candidate)).toThrow(/duplicate ids/);
+ });
+
+ it("names every duplicated id, not just the first", () => {
+ const duplicated: Receipt = structuredClone(BASE_RECEIPT);
+ duplicated.checks.push({ ...BASE_RECEIPT.checks[1] }, { ...BASE_RECEIPT.checks[0] });
+
+ expect(() => compareReceipts(duplicated, structuredClone(BASE_RECEIPT))).toThrow(
+ /duplicate ids: git\.tree-equality, pkg\.pack-inventory/,
+ );
+ });
+});
+
+describe("readReceipt", () => {
+ const dir = mkdtempSync(join(tmpdir(), "verify-receipt-test-"));
+ afterAll(() => rmSync(dir, { recursive: true, force: true }));
+
+ it("reads a well-formed receipt", () => {
+ const path = join(dir, "good.json");
+ writeFileSync(path, JSON.stringify(BASE_RECEIPT));
+ expect(readReceipt(path, "reference receipt").verdict).toBe("PARITY");
+ });
+
+ // A CI gate that dies on a raw ENOENT stack tells the reader nothing about
+ // which receipt was missing or what was expected there.
+ it("fails with a legible message on a missing file, naming the role and path", () => {
+ const path = join(dir, "absent.json");
+ expect(() => readReceipt(path, "candidate receipt")).toThrow(/candidate receipt not found/);
+ expect(() => readReceipt(path, "candidate receipt")).toThrow(/Expected a parity receipt JSON file/);
+ expect(() => readReceipt(path, "candidate receipt")).not.toThrow(/ENOENT/);
+ });
+
+ it("fails with a legible message on malformed JSON", () => {
+ const path = join(dir, "malformed.json");
+ writeFileSync(path, "{ not json at all ");
+ expect(() => readReceipt(path, "reference receipt")).toThrow(/reference receipt is not valid JSON/);
+ });
+
+ it("fails when the file parses but is not a receipt object", () => {
+ const arrayPath = join(dir, "array.json");
+ writeFileSync(arrayPath, "[]");
+ expect(() => readReceipt(arrayPath, "reference receipt")).toThrow(/is not a receipt object[\s\S]*array/);
+
+ const scalarPath = join(dir, "scalar.json");
+ writeFileSync(scalarPath, "42");
+ expect(() => readReceipt(scalarPath, "candidate receipt")).toThrow(/is not a receipt object[\s\S]*number/);
+ });
});