From ccbf03986861388480e97f2502194130ea13851f Mon Sep 17 00:00:00 2001 From: UIOSUN <92213066+U1oSun@users.noreply.github.com> Date: Wed, 5 Aug 2026 14:15:21 +0800 Subject: [PATCH] feat(analysis): integrate native learning review into standard reports Route Task Episode learning candidates through the canonical source/brief/ report chain: episodes emit native learning review packets, verified decisions bind to packet and source digests, and confirmed candidates surface in learningCaptureDiagnostics. Keep review create/apply behind strict owner validation, refuse review-alias/evidence fabrication, and stay fully backward compatible when no candidates exist. Spec: docs/specs/2026-08-02-native-learning-report-integration.md Test: focused report-source-review/cli/contract 98 passed; learning-loop and task-loop suites 93 passed; doc-link 6/6; full suite 1266 passed with 1 existing Windows symlink skip; pack verification npm 450 / runtime zip 472. Co-authored-by: Codex (GPT 5.6 Sol) --- docs/better-harness-doc-links.mmd | 2 + ...8-02-native-learning-report-integration.md | 135 +++++++ scripts/better-harness-cli/cli.mjs | 9 +- scripts/better-harness-cli/registry.mjs | 7 + scripts/harness-analysis/evidence-brief.mjs | 6 + .../learning-loop-candidates.mjs | 215 +++++++++- .../learning-loop-review-packet.mjs | 172 +++++++- .../report-source/apply-review.mjs | 55 ++- .../harness-analysis/report-source/cli.mjs | 294 ++++++++++++++ .../report-source/review-packet.mjs | 163 +++++++- .../harness-analysis/report-source/source.mjs | 83 +++- scripts/harness-analysis/task-loop-source.mjs | 10 +- scripts/npm-package/verify-pack.mjs | 2 + skills/better-harness/SKILL.md | 6 +- .../references/report-source-review.md | 49 +++ test/better-harness-cli.test.mjs | 25 +- .../harness-help.txt | 2 + .../scripts-refactor-contract/root-help.txt | 2 +- test/report-source-review.test.mjs | 370 +++++++++++++++++- test/scripts-refactor-contract.test.mjs | 6 +- 20 files changed, 1566 insertions(+), 47 deletions(-) create mode 100644 docs/specs/2026-08-02-native-learning-report-integration.md create mode 100644 scripts/harness-analysis/report-source/cli.mjs create mode 100644 skills/better-harness/references/report-source-review.md diff --git a/docs/better-harness-doc-links.mmd b/docs/better-harness-doc-links.mmd index 0ce1c3e..e149375 100644 --- a/docs/better-harness-doc-links.mmd +++ b/docs/better-harness-doc-links.mmd @@ -59,6 +59,7 @@ flowchart LR n_skills_better_harness_references_findings_review_md["findings-review.md"] n_skills_better_harness_references_manual_direct_fix_md["manual-direct-fix.md"] n_skills_better_harness_references_project_harness_md["project-harness.md"] + n_skills_better_harness_references_report_source_review_md["report-source-review.md"] n_skills_better_harness_references_session_evidence_md["session-evidence.md"] n_skills_better_harness_references_session_repeated_workflows_md["session-repeated-workflows.md"] n_skills_better_harness_references_support_bootstrap_md["support-bootstrap.md"] @@ -79,6 +80,7 @@ flowchart LR n_skills_better_harness_SKILL_md --> n_skills_better_harness_references_findings_review_md n_skills_better_harness_SKILL_md --> n_skills_better_harness_references_manual_direct_fix_md n_skills_better_harness_SKILL_md --> n_skills_better_harness_references_project_harness_md + n_skills_better_harness_SKILL_md --> n_skills_better_harness_references_report_source_review_md n_skills_better_harness_SKILL_md --> n_skills_better_harness_references_session_evidence_md n_skills_better_harness_SKILL_md --> n_skills_better_harness_references_session_repeated_workflows_md n_skills_better_harness_SKILL_md --> n_skills_better_harness_references_support_bootstrap_md diff --git a/docs/specs/2026-08-02-native-learning-report-integration.md b/docs/specs/2026-08-02-native-learning-report-integration.md new file mode 100644 index 0000000..55fd27e --- /dev/null +++ b/docs/specs/2026-08-02-native-learning-report-integration.md @@ -0,0 +1,135 @@ +# Native Learning Review in Report Evidence + +## Traceability + +- Spec ID: `native-learning-report-integration` +- Roadmap: `LC-05` +- AI involvement: Codex (GPT 5.6 Sol) +- Status: Implemented + +## Intent + +Make the already validated native recurring-correction method usable through +the standard Harness review packet, lead-decision, and source-apply path. A +normal source must surface a bounded nested packet; the existing lead decision +may carry only its native decisions; source apply projects only validated +matches into canonical Learning Capture diagnostics and reader evidence. + +## Acceptance Scenarios + +- **AC-1: Reviewable source packet.** When ordinary report Task Episodes form + native candidate groups, generated diagnostics retain a privacy-safe packet + and declare that a decision is required. The packet is deterministic and + contains no raw session, prompt, command, transcript, credential, or path. +- **AC-2: Validated report projection.** Given exact native decisions in the + normal lead decision, source apply validates them and projects accepted signals, + candidates, match/abstain counts, and binding digests into canonical Learning + Capture diagnostics. +- **AC-3: Fail closed across the review lifecycle.** A pending source validates + its generated packet against current Episodes. Review compile/apply rejects + stale packet/source digests, invented aliases, and incomplete decisions. + A persisted reviewed source independently revalidates its stored packet, + decision allowlists, and result aggregates, so tampering fails validation + even after legitimate outer review fields have changed the current Episodes. +- **AC-4: No-group compatibility.** A source with no native groups has no new + native review diagnostic and preserves the existing Learning Loop output. +- **AC-5: Reader visibility.** The neutral evidence brief reports whether a + native review is pending or validated, and reports bounded group/match/ + abstain counts without exposing packet aliases or private source data. +- **AC-6: Standard review binding.** The outer Harness packet carries the + optional native subpacket but does not merge its opaque aliases into + `allowedEvidenceRefs`, including when another packet is built from a persisted + reviewed source. +- **AC-7: Public local review route.** A registered, parser-safe + `harness source-review` command exposes explicit `create`, `decision`, and + `apply` phases. `create` writes the bounded packet and a packet-bound decision + template, `decision` compiles only the caller-edited template, and `apply` + requires explicit non-interactive confirmation before atomically replacing + the selected source. The route does not call a model or author a decision. +- **AC-8: Strict persisted contract.** Stored Episode facts, groups, coverage, + decisions, matches, abstentions, and result metadata reject unknown fields, + unsafe values, cross-reference drift, and decision/result disagreement even + when packet digests are recomputed. The complete stored Learning Loop result + must equal deterministic reconstruction from the packet, review, final + Episodes, signals, interventions, and asset coverage. A packet with groups + requires complete native decisions before outer review apply; omission cannot + leave a stale pending packet after Episode review mutation. + +## Non-goals + +- Do not call an AI model or choose a provider/model. +- Do not create, alter, or apply a Skill, Rule, Memory, Hook, ledger entry, or + other Harness asset. +- Do not infer native `repeated-rediscovery` or another pattern. +- Do not claim intervention effectiveness, transfer, causality, or savings. +- Do not make unreviewed native candidates into canonical candidates. + +## Plan and Tasks + +1. Extend generated Learning Capture diagnostics with a tightly validated, + pending privacy-safe native review packet. +2. Add that packet as an independent nested contract in the Harness review + packet, without mixing its opaque aliases with source evidence references. +3. Require and compile native decisions when a nested packet is present, then + apply them only through `applyReportSourceReview` after both packet contracts + validate. +4. Surface aggregate native-review state in the neutral evidence brief. +5. Add the registered `harness source-review create|decision|apply` owner under + `report-source/`, with a packet-bound decision template, local JSON files, + parser-safe output, refusal to overwrite intermediate files, and explicit + confirmation for source apply. +6. Cover the standard outer packet -> lead decision -> apply path for positive, + abstain, stale/invented, dedupe, no-group, privacy, and stored-result + tampering behavior. + +## Test and Review Evidence + +- AC-1 to AC-8: native candidate, report-source review, public CLI, task-loop + source, analyzer, Learning Loop, Episode, and report-source contract tests. +- Public-path evidence invokes the root `scripts/better-harness.mjs` facade for + create, decision, refused unconfirmed apply, and confirmed atomic apply. +- Documentation and frozen-contract evidence covers Skill routing, generated + doc links, human help fixtures, command inventory, schema, and packaging. +- Regression and package evidence: `npm test` and `npm run pack:verify`. + +## Implementation Evidence + +- AC-1 and AC-4: `task-loop-source.mjs` emits the pending packet only when + bounded native groups exist; native and cross-module fixtures retain the + no-group behavior. +- AC-2, AC-3, AC-6, and AC-8: the outer packet -> lead decision -> apply test + rejects omitted decisions, stale digests, recomputed-digest nested packet + injection, invented or duplicate aliases, unknown nested result fields, and + decision/result drift. Synchronized edits to stored canonical candidates, + costs, or scores fail deterministic full-result reconstruction. Rebuilt outer + packets exclude every `native-learning-evidence` reference. +- AC-5: `evidence-brief.mjs` emits only pending/reviewed aggregate counts. +- AC-7: the registered root CLI E2E passed create, caller-authored decision, + unconfirmed-apply refusal, and confirmed atomic apply by editing the generated + template. Machine output and read errors omit absolute paths and raw validator + details, intermediate files use create-only writes, bare decision documents + are rejected without writing a review, and malformed help combinations fail + instead of hiding invalid arguments. +- Focused native/report-source run: 63/63. Cross-module source/analyzer/ + Learning Loop/Episode run: 81/81. +- CLI, Skill, frozen CLI, and doc-link contract run: 66 passed with one Windows + package-bin symlink test skipped. +- Final full regression: 1089 passed, 4 failed, and 6 skipped. All four failures + are Windows `EPERM` symlink-creation restrictions in analysis-scope, render, + and workspace-topology tests outside the changed modules. +- Package verification passed with an isolated npm cache: 385 npm entries and + 408 runtime ZIP entries. +- `git diff --check` passed. + +## Privacy and Risk + +- A review can become stale between runs. Rebuilding the packet and requiring + both digests fail closes stale input. +- Packets are source-visible data. Only the existing bounded, privacy-safe + packet projection is retained; the brief emits aggregate counts only. +- Existing unreviewed provider candidates remain unchanged. Native candidates + are added only after an exact validated match. +- The outer review packet schema advances from v2 to v3 because it gains an + optional nested field. Sources without native groups omit the field; v2 + packet fixtures remain valid only against their historical contract and are + not silently reinterpreted as v3 packets. diff --git a/scripts/better-harness-cli/cli.mjs b/scripts/better-harness-cli/cli.mjs index a4853ac..6ec0153 100644 --- a/scripts/better-harness-cli/cli.mjs +++ b/scripts/better-harness-cli/cli.mjs @@ -70,6 +70,7 @@ const GROUP_EXAMPLES = { { audience: "workflow", text: "better-harness harness checkup --phase scan --provider qoder --workspace . --json" }, { audience: "advanced", text: "better-harness harness workspace-topology --workspace . --json" }, { audience: "maintainer", text: "better-harness harness source --workspace . --source /report.source.json --language en" }, + { audience: "maintainer", text: "better-harness harness source-review create --source /report.source.json --packet /review.packet.json --decision /lead.decision.json --json" }, { audience: "advanced", text: "better-harness harness render --findings /findings.json --mode qoder-canvas --out .qoder/better-harness --target . --validate --json" }, { audience: "advanced", text: "better-harness harness preview-canvas /report.canvas.tsx --open" }, { audience: "advanced", text: "better-harness harness report-quality --report /report.md" }, @@ -261,6 +262,10 @@ function hasHelpFlag(argv) { return argv.some((value) => value === "--help" || value === "-h"); } +function requiresOwnerHelpValidation(command, subcommand) { + return command === "harness" && subcommand === "source-review"; +} + function hasJsonFlag(argv) { for (const value of argv) { if (value === "--") return false; @@ -426,7 +431,9 @@ export function resolveDispatch(argv = []) { return commandSchema(argv.slice(1)); } - if (hasHelpFlag(argv)) { + const ownerValidatedHelp = requiresOwnerHelpValidation(command, subcommand); + const canonicalOwnerHelp = ownerValidatedHelp && rest.length === 1 && hasHelpFlag(rest); + if (hasHelpFlag(argv) && (!ownerValidatedHelp || canonicalOwnerHelp)) { const direct = directDispatchFor(command, subcommand); if (direct) { const metadata = commandMetadata(command); diff --git a/scripts/better-harness-cli/registry.mjs b/scripts/better-harness-cli/registry.mjs index f0f0106..a8a14a3 100644 --- a/scripts/better-harness-cli/registry.mjs +++ b/scripts/better-harness-cli/registry.mjs @@ -227,6 +227,13 @@ const COMMANDS = [ summary: "Build the deterministic Agent Work Loop source envelope.", description: "Collect repository, practice, and session candidates into report.source.json while preserving evidence boundaries.", }, + { + name: "source-review", + audience: "maintainer", + script: "harness-analysis/report-source/cli.mjs", + summary: "Create, compile, and apply a bounded report-source review.", + description: "Expose an explicit local create, caller-authored decision, and confirmed apply lifecycle without calling a model or merging native evidence aliases into the outer evidence namespace.", + }, { name: "task-loop-report", audience: "maintainer", diff --git a/scripts/harness-analysis/evidence-brief.mjs b/scripts/harness-analysis/evidence-brief.mjs index 4942090..94cb786 100644 --- a/scripts/harness-analysis/evidence-brief.mjs +++ b/scripts/harness-analysis/evidence-brief.mjs @@ -170,6 +170,12 @@ function diagnosticsLines(repository = {}) { const friction = rows(signals.frictionSignals); if (Object.keys(learning).length > 0) { output.push(`- Learning observation coverage: ${rows(learning.episodeRecords).length} episode record(s), ${rows(learning.recurringIssueCandidates).length} recurring candidate(s), ${observedSkills.length} scoped Skill observation(s), and ${unscopedSkills.length} unscoped Skill observation(s).`); + const native = learning.nativeLearningReview ?? {}; + if (native.status === "review-required") { + output.push(` - Native recurring-correction review: ${rows(native.packet?.groups).length} bounded group(s) await a packet-bound decision.`); + } else if (native.status === "reviewed") { + output.push(` - Native recurring-correction review: validated ${rows(native.result?.matches).length} match(es) and ${rows(native.result?.abstentions).length} abstention(s).`); + } const frictionList = list(friction.map((row) => `${row?.name}: ${count(row?.count)}`), 6); if (frictionList.text) output.push(` - Aggregated friction signals: ${withOmitted(frictionList.text, frictionList.omitted)}.`); const coverage = list(Object.entries(learning.coverage ?? {}).map(([key, value]) => `${humanLabel(key)}=${cleanText(value, 100)}`), 9); diff --git a/scripts/harness-analysis/learning-loop-candidates.mjs b/scripts/harness-analysis/learning-loop-candidates.mjs index 1e73343..05bc837 100644 --- a/scripts/harness-analysis/learning-loop-candidates.mjs +++ b/scripts/harness-analysis/learning-loop-candidates.mjs @@ -6,11 +6,13 @@ import { NATIVE_LEARNING_PATTERN_IDS, NATIVE_LEARNING_REVIEW_PACKET_SCHEMA_VERSION, validateNativeLearningReviewPacket, + validateStoredNativeLearningReviewPacket, } from "./learning-loop-review-packet.mjs"; export { buildNativeLearningReviewPacket, nativeLearningReviewPacketDigest, + nativeLearningReviewSourceDigest, NATIVE_LEARNING_ABSTAIN_REASON_CODES, NATIVE_LEARNING_MATCH_REASON_CODES, NATIVE_LEARNING_PATTERN_IDS, @@ -18,6 +20,7 @@ export { NATIVE_LEARNING_REVIEW_KIND, NATIVE_LEARNING_REVIEW_PACKET_SCHEMA_VERSION, validateNativeLearningReviewPacket, + validateStoredNativeLearningReviewPacket, } from "./learning-loop-review-packet.mjs"; export const LEARNING_LOOP_REVIEW_SCHEMA_VERSION = 1; @@ -97,6 +100,12 @@ const CURRENT_TRUTH_ASSET_PATTERN_IDS = new Set([ "cross-asset-duplication-or-contradiction", "memory-skill-duplication", ]); +const LEARNING_LOOP_FIELD_EVIDENCE_FIELDS = new Set([ + "taskFamily", "repoArea", "changeType", "frictionType", "normalizedSignature", + "userCorrection", "asset", "assetLoaded", "assetRelevant", "requiredStepApplied", + "assetChanged", "validationResult", "deliveryResult", "elapsedMs", "toolCalls", + "tokens", "harnessVersion", +]); function rows(value) { return Array.isArray(value) ? value.filter(Boolean) : []; @@ -106,6 +115,16 @@ function isObject(value) { return Boolean(value) && typeof value === "object" && !Array.isArray(value); } +function canonicalValue(value) { + if (Array.isArray(value)) return value.map(canonicalValue); + if (!isObject(value)) return value; + return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonicalValue(value[key])])); +} + +function sameCanonicalValue(left, right) { + return JSON.stringify(canonicalValue(left)) === JSON.stringify(canonicalValue(right)); +} + function text(value, fallback = "") { const normalized = String(value ?? "").replace(/\s+/gu, " ").trim(); return normalized || fallback; @@ -565,8 +584,8 @@ function validateNativeDecision(decision, index, group, errors) { if (!hasCorrectionReason) errors.push(location + ".reasonCodes requires a supported recurring correction reason"); } -export function validateNativeLearningCandidateReview({ episodes = [], packet, review } = {}) { - const errors = [...validateNativeLearningReviewPacket({ episodes, packet })]; +export function validateStoredNativeLearningCandidateReview({ packet, review } = {}) { + const errors = [...validateStoredNativeLearningReviewPacket(packet)]; nativeReviewAllowedFields( review, new Set(["schemaVersion", "sourceDigest", "packetDigest", "decisions"]), @@ -609,6 +628,13 @@ export function validateNativeLearningCandidateReview({ episodes = [], packet, r return [...new Set(errors)].sort(); } +export function validateNativeLearningCandidateReview({ episodes = [], packet, review } = {}) { + return [...new Set([ + ...validateStoredNativeLearningCandidateReview({ packet, review }), + ...validateNativeLearningReviewPacket({ episodes, packet }), + ])].sort(); +} + function nativeSignalSignature(group) { return String(group?.patternSignature ?? ""); } @@ -697,17 +723,103 @@ function appliedNativeCoverage(packet, decisions) { }; } -export function applyNativeLearningCandidateReview({ +function expectedNativeAppliedRows({ episodes, packet, review }) { + const episodesById = new Map(rows(episodes).map((episode) => [String(episode?.id ?? ""), episode])); + const groupsById = new Map(rows(packet?.groups).map((group) => [group.groupRef, group])); + const matches = []; + const abstentions = []; + for (const decision of orderedByGroupRef(review?.decisions)) { + const group = groupsById.get(decision.groupRef); + if (!group) continue; + if (decision.decision === "abstain") { + abstentions.push({ groupRef: decision.groupRef, reasonCodes: [...decision.reasonCodes] }); + continue; + } + const sourceEpisodes = group.episodeRefs.map((episodeRef) => episodesById.get(episodeRef)); + const normalizedSignature = commonProviderSignature(sourceEpisodes, decision.patternId) + || nativeSignalSignature(group); + matches.push({ + groupRef: decision.groupRef, + patternId: decision.patternId, + sourceEpisodes: [...group.episodeRefs], + normalizedSignature, + evidenceRefs: decision.evidenceRefs.map((reference) => ({ + kind: "native-learning-evidence", + id: reference, + })), + reasonCodes: [...decision.reasonCodes], + claimType: "opportunity", + }); + } + return { matches, abstentions }; +} + +export function validateStoredNativeLearningAppliedReview({ episodes = [], packet, review, + result, signals = {}, interventions = [], assetCoverage = [], } = {}) { - const errors = validateNativeLearningCandidateReview({ episodes, packet, review }); - if (errors.length > 0) return { result: null, errors }; + const errors = [...validateStoredNativeLearningCandidateReview({ packet, review })]; + if (!isObject(result)) return [...new Set([...errors, "native learning applied result must be an object"])].sort(); + const allowedFields = new Set([ + "schemaVersion", "kind", "status", "sourceDigest", "packetDigest", + "matches", "abstentions", "coverage", "learningLoop", + ]); + for (const field of Object.keys(result)) { + if (!allowedFields.has(field)) errors.push(`native learning applied result has unsupported field: ${field}`); + } + if (result.schemaVersion !== NATIVE_LEARNING_APPLIED_REVIEW_SCHEMA_VERSION) { + errors.push(`native learning applied result schemaVersion must be ${NATIVE_LEARNING_APPLIED_REVIEW_SCHEMA_VERSION}`); + } + if (result.kind !== "native-learning-candidate-review") errors.push("native learning applied result kind is invalid"); + if (result.status !== "reviewed") errors.push("native learning applied result status must be reviewed"); + if (result.sourceDigest !== packet?.sourceDigest) errors.push("native learning applied result sourceDigest does not match the packet"); + if (result.packetDigest !== packet?.packetDigest) errors.push("native learning applied result packetDigest does not match the packet"); + if (!Array.isArray(result.matches)) errors.push("native learning applied result matches must be an array"); + if (!Array.isArray(result.abstentions)) errors.push("native learning applied result abstentions must be an array"); + if (!isObject(result.coverage)) errors.push("native learning applied result coverage must be an object"); + const expected = expectedNativeAppliedRows({ episodes, packet, review }); + if (!sameCanonicalValue(result.matches, expected.matches)) { + errors.push("native learning applied result matches do not match the packet-bound decisions"); + } + if (!sameCanonicalValue(result.abstentions, expected.abstentions)) { + errors.push("native learning applied result abstentions do not match the packet-bound decisions"); + } + const expectedCoverage = appliedNativeCoverage(packet, rows(review?.decisions)); + if (!sameCanonicalValue(result.coverage, expectedCoverage)) { + errors.push("native learning applied result coverage does not match the packet-bound decisions"); + } + errors.push(...validateLearningLoopReview(result.learningLoop, { + episodeIds: rows(episodes).map((episode) => String(episode?.id ?? "")), + }).map((error) => `native learning applied result ${error}`)); + if (validateStoredNativeLearningCandidateReview({ packet, review }).length === 0) { + const expectedLearningLoop = buildNativeLearningAppliedResult({ + episodes, + packet, + review, + signals, + interventions, + assetCoverage, + }).learningLoop; + if (!sameCanonicalValue(result.learningLoop, expectedLearningLoop)) { + errors.push("native learning applied result learningLoop does not match deterministic packet-bound reconstruction"); + } + } + return [...new Set(errors)].sort(); +} +function buildNativeLearningAppliedResult({ + episodes = [], + packet, + review, + signals = {}, + interventions = [], + assetCoverage = [], +} = {}) { const reviewedEpisodes = structuredClone(rows(episodes)); const episodesById = new Map(rows(reviewedEpisodes).map((episode) => [String(episode?.id ?? ""), episode])); const episodeFactsById = new Map(rows(packet?.episodeFacts).map((episode) => [episode.episodeRef, episode])); @@ -758,21 +870,30 @@ export function applyNativeLearningCandidateReview({ const learningLoop = buildLearningLoopReview({ episodes: reviewedEpisodes, signals, interventions, assetCoverage }); return { - result: { - schemaVersion: NATIVE_LEARNING_APPLIED_REVIEW_SCHEMA_VERSION, - kind: "native-learning-candidate-review", - status: "reviewed", - sourceDigest: packet.sourceDigest, - packetDigest: packet.packetDigest, - matches, - abstentions, - coverage: appliedNativeCoverage(packet, review.decisions), - learningLoop, - }, - errors: [], + schemaVersion: NATIVE_LEARNING_APPLIED_REVIEW_SCHEMA_VERSION, + kind: "native-learning-candidate-review", + status: "reviewed", + sourceDigest: packet.sourceDigest, + packetDigest: packet.packetDigest, + matches, + abstentions, + coverage: appliedNativeCoverage(packet, review.decisions), + learningLoop, }; } +export function applyNativeLearningCandidateReview(options = {}) { + const errors = validateNativeLearningCandidateReview(options); + if (errors.length > 0) return { result: null, errors }; + return { result: buildNativeLearningAppliedResult(options), errors: [] }; +} + +export function applyStoredNativeLearningCandidateReview(options = {}) { + const errors = validateStoredNativeLearningCandidateReview(options); + if (errors.length > 0) return { result: null, errors }; + return { result: buildNativeLearningAppliedResult(options), errors: [] }; +} + export function validateLearningLoopReview(value, { episodeIds = [] } = {}) { if (!isObject(value)) return ["learning loop review must be an object"]; const errors = []; @@ -784,8 +905,33 @@ export function validateLearningLoopReview(value, { episodeIds = [] } = {}) { if (!Array.isArray(value.candidates)) errors.push("learning loop review candidates must be an array"); if (!isObject(value.coverage)) errors.push("learning loop review coverage must be an object"); const knownEpisodes = new Set(episodeIds); + const rejectUnknownFields = (row, fields, at) => { + if (!isObject(row)) { + errors.push(`${at} must be an object`); + return false; + } + for (const field of Object.keys(row)) { + if (!fields.has(field)) errors.push(`${at} has unsupported field: ${field}`); + } + return true; + }; + const validateEvidenceRefs = (references, at) => { + if (!Array.isArray(references)) { + errors.push(`${at} must be an array`); + return; + } + for (const [index, reference] of references.entries()) { + rejectUnknownFields(reference, new Set(["kind", "id"]), `${at}[${index}]`); + } + }; for (const [index, candidate] of rows(value.candidates).entries()) { const at = `learning loop review candidates[${index}]`; + rejectUnknownFields(candidate, new Set([ + "id", "patternId", "claimType", "provenance", "sourceEpisodes", "taskFingerprint", + "normalizedSignature", "asset", "observedBehavior", "currentCost", "candidateCauses", + "brokenStage", "recommendedOwner", "intervention", "primaryMetric", "guardrails", + "stopOrRevert", "confidence", "priorityScore", "evidenceRefs", + ]), at); if (!PATTERN_SET.has(candidate?.patternId)) errors.push(`${at}.patternId is invalid`); if (!CLAIM_SET.has(candidate?.claimType)) errors.push(`${at}.claimType is invalid`); if (!FIELD_PROVENANCE.has(candidate?.provenance)) errors.push(`${at}.provenance is invalid`); @@ -805,10 +951,16 @@ export function validateLearningLoopReview(value, { episodeIds = [] } = {}) { if (!isObject(candidate?.taskFingerprint) || !text(candidate.taskFingerprint.family) || !text(candidate.taskFingerprint.repoArea)) errors.push(`${at}.taskFingerprint must name family and repoArea`); + else rejectUnknownFields(candidate.taskFingerprint, new Set(["family", "repoArea"]), `${at}.taskFingerprint`); if (!isObject(candidate?.currentCost) || !Number.isInteger(candidate.currentCost.episodeCount) || candidate.currentCost.episodeCount < 0) errors.push(`${at}.currentCost must contain a non-negative episodeCount`); - if (!Array.isArray(candidate?.evidenceRefs)) errors.push(`${at}.evidenceRefs must be an array`); + else rejectUnknownFields( + candidate.currentCost, + new Set(["episodeCount", "toolCalls", "elapsedMs", "tokens", "userCorrections"]), + `${at}.currentCost`, + ); + validateEvidenceRefs(candidate?.evidenceRefs, `${at}.evidenceRefs`); if (candidate?.asset !== undefined) { if (!isObject(candidate.asset) || !text(candidate.asset.kind) @@ -817,8 +969,11 @@ export function validateLearningLoopReview(value, { episodeIds = [] } = {}) { errors.push(`${at}.asset must name kind, ref, and scope`); } for (const field of ["currentTruthRefs", "requiredStepRefs", "updateEvidenceRefs", "outcomeEvidenceRefs"]) { - if (!Array.isArray(candidate?.asset?.[field])) errors.push(`${at}.asset.${field} must be an array`); + validateEvidenceRefs(candidate?.asset?.[field], `${at}.asset.${field}`); } + rejectUnknownFields(candidate.asset, new Set([ + "kind", "ref", "scope", "currentTruthRefs", "requiredStepRefs", "updateEvidenceRefs", "outcomeEvidenceRefs", + ]), `${at}.asset`); } if (value.status === "reviewed" && ASSET_SPECIFIC_PATTERN_IDS.has(candidate?.patternId) && candidate?.asset === undefined) { errors.push(`${at}.asset is required for a reviewed asset-specific candidate`); @@ -838,14 +993,36 @@ export function validateLearningLoopReview(value, { episodeIds = [] } = {}) { if (!Number.isInteger(candidate?.priorityScore) || candidate.priorityScore < 0) errors.push(`${at}.priorityScore must be a non-negative integer`); } for (const [episodeIndex, episode] of rows(value.episodeRecords).entries()) { + const episodeAt = `learning loop review episodeRecords[${episodeIndex}]`; + rejectUnknownFields(episode, new Set(["episodeId", "targetKeys", "signals"]), episodeAt); + if (!Array.isArray(episode?.targetKeys)) errors.push(`${episodeAt}.targetKeys must be an array`); + if (!Array.isArray(episode?.signals)) errors.push(`${episodeAt}.signals must be an array`); for (const [signalIndex, signal] of rows(episode?.signals).entries()) { const at = `learning loop review episodeRecords[${episodeIndex}].signals[${signalIndex}]`; + rejectUnknownFields(signal, new Set([ + "patternId", "normalizedSignature", "taskFamily", "repoArea", "changeType", "frictionType", + "userCorrection", "asset", "assetLoaded", "assetChanged", "assetRelevant", + "requiredStepApplied", "mandatory", "procedural", "validationResult", "deliveryResult", + "elapsedMs", "toolCalls", "tokens", "harnessVersion", "fieldEvidence", + "normalizationVersion", "evidenceRefs", + ]), at); if (signal?.normalizationVersion !== NORMALIZATION_VERSION) errors.push(`${at}.normalizationVersion is invalid`); if (!isObject(signal?.fieldEvidence)) errors.push(`${at}.fieldEvidence must be an object`); for (const [field, evidence] of Object.entries(signal?.fieldEvidence ?? {})) { + if (!LEARNING_LOOP_FIELD_EVIDENCE_FIELDS.has(field)) errors.push(`${at}.fieldEvidence has unsupported field: ${field}`); + rejectUnknownFields(evidence, new Set(["provenance", "coverage"]), `${at}.fieldEvidence.${field}`); if (!FIELD_PROVENANCE.has(evidence?.provenance)) errors.push(`${at}.fieldEvidence.${field}.provenance is invalid`); if (!FIELD_COVERAGE.has(evidence?.coverage)) errors.push(`${at}.fieldEvidence.${field}.coverage is invalid`); } + validateEvidenceRefs(signal?.evidenceRefs, `${at}.evidenceRefs`); + if (signal?.asset !== undefined) { + rejectUnknownFields(signal.asset, new Set([ + "kind", "ref", "scope", "currentTruthRefs", "requiredStepRefs", "updateEvidenceRefs", "outcomeEvidenceRefs", + ]), `${at}.asset`); + for (const field of ["currentTruthRefs", "requiredStepRefs", "updateEvidenceRefs", "outcomeEvidenceRefs"]) { + validateEvidenceRefs(signal.asset?.[field], `${at}.asset.${field}`); + } + } } } for (const field of COVERAGE_FIELDS) { diff --git a/scripts/harness-analysis/learning-loop-review-packet.mjs b/scripts/harness-analysis/learning-loop-review-packet.mjs index e619a12..180d80e 100644 --- a/scripts/harness-analysis/learning-loop-review-packet.mjs +++ b/scripts/harness-analysis/learning-loop-review-packet.mjs @@ -33,6 +33,8 @@ const SAFE_ROUTE_RE = /^(?:lifecycle:)?[a-z0-9][a-z0-9._:/-]{0,79}$/u; const SAFE_EVIDENCE_ID_RE = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,159}$/u; const SAFE_EVIDENCE_FINGERPRINT_RE = /^[a-f0-9]{16,128}$/u; const PRIVATE_EVIDENCE_ID_RE = /(?:^session(?:[-_:]|$)|@|:\/\/|^[A-Za-z]:[\\/]|^\\\\|^\/(?:Users|home)\/)/iu; +const SAFE_EVIDENCE_ALIAS_RE = /^evidence-ref-[a-f0-9]{20}$/u; +const SAFE_DIGEST_RE = /^[a-f0-9]{64}$/u; const FAILED_STATUS = new Set(["failed", "failure", "error", "errored"]); const PROTECTIVE_FRICTION = "protective-intervention"; const OBSERVED_PROVENANCE = new Set(["host-observed", "deterministic-derived", "ai-reviewed"]); @@ -400,6 +402,165 @@ export function nativeLearningReviewPacketDigest(packet) { return digest(packetPayload(packet)); } +export function nativeLearningReviewSourceDigest({ episodeFacts, groups, limits } = {}) { + return digest({ episodeFacts, groups, limits }); +} + +function storedObjectFields(value, allowedFields, location, errors) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + errors.push(`${location} must be an object`); + return false; + } + for (const field of Object.keys(value)) { + if (!allowedFields.has(field)) errors.push(`${location} has unsupported field: ${field}`); + } + return true; +} + +function storedSortedStrings(value, predicate, location, errors, { sorted = true } = {}) { + if (!Array.isArray(value)) { + errors.push(`${location} must be an array`); + return []; + } + if (value.some((item) => typeof item !== "string" || !predicate(item))) { + errors.push(`${location} contains an invalid value`); + } + const expected = sorted ? uniqueSorted(value) : [...new Set(value)]; + if (JSON.stringify(value) !== JSON.stringify(expected)) { + errors.push(`${location} must contain distinct${sorted ? " sorted" : ""} values`); + } + return value; +} + +function storedEpisodeFactErrors(fact, index, limits) { + const location = `native learning review packet episodeFacts[${index}]`; + const errors = []; + if (!storedObjectFields( + fact, + new Set(["episodeRef", "taskRoutes", "targetRefs", "checkRefs", "facts", "frictionKinds", "evidenceRefs", "evidenceCoverage"]), + location, + errors, + )) return errors; + if (!SAFE_EPISODE_ID_RE.test(String(fact.episodeRef ?? ""))) errors.push(`${location}.episodeRef is invalid`); + storedSortedStrings(fact.taskRoutes, (value) => safeRoute(value) === value, `${location}.taskRoutes`, errors); + storedSortedStrings(fact.targetRefs, (value) => safeTarget(value) === value, `${location}.targetRefs`, errors); + storedSortedStrings(fact.checkRefs, (value) => safeCheck(value) === value, `${location}.checkRefs`, errors); + const factStates = fact.facts; + if (storedObjectFields( + factStates, + new Set(["userCorrection", "protectiveIntervention", "sameCheckRepair", "validationFailure"]), + `${location}.facts`, + errors, + )) { + const observationStates = new Set(["true", "false", "unavailable"]); + if (!observationStates.has(factStates.userCorrection)) errors.push(`${location}.facts.userCorrection is invalid`); + if (!observationStates.has(factStates.protectiveIntervention)) errors.push(`${location}.facts.protectiveIntervention is invalid`); + if (!new Set(["true", "false"]).has(factStates.sameCheckRepair)) errors.push(`${location}.facts.sameCheckRepair is invalid`); + if (!new Set(["true", "false"]).has(factStates.validationFailure)) errors.push(`${location}.facts.validationFailure is invalid`); + } + const expectedFriction = uniqueSorted([ + factStates?.userCorrection === "true" ? "explicit-user-correction" : "", + factStates?.sameCheckRepair === "true" ? "same-check-repair" : "", + factStates?.validationFailure === "true" ? "validation-failure" : "", + ]); + storedSortedStrings( + fact.frictionKinds, + (value) => ["explicit-user-correction", "same-check-repair", "validation-failure"].includes(value), + `${location}.frictionKinds`, + errors, + ); + if (JSON.stringify(fact.frictionKinds) !== JSON.stringify(expectedFriction)) { + errors.push(`${location}.frictionKinds does not match observed facts`); + } + const evidenceRefs = storedSortedStrings( + fact.evidenceRefs, + (value) => SAFE_EVIDENCE_ALIAS_RE.test(value), + `${location}.evidenceRefs`, + errors, + { sorted: false }, + ); + if (evidenceRefs.length > limits.maxEvidenceRefsPerEpisode) { + errors.push(`${location}.evidenceRefs exceeds the packet limit`); + } + const coverage = fact.evidenceCoverage; + if (storedObjectFields( + coverage, + new Set(["status", "includedCount", "totalCount", "identitySetDigest"]), + `${location}.evidenceCoverage`, + errors, + )) { + const includedCount = Number(coverage.includedCount); + const totalCount = Number(coverage.totalCount); + if (!Number.isInteger(includedCount) || includedCount !== evidenceRefs.length) { + errors.push(`${location}.evidenceCoverage.includedCount does not match evidenceRefs`); + } + if (!Number.isInteger(totalCount) || totalCount < includedCount) { + errors.push(`${location}.evidenceCoverage.totalCount is invalid`); + } + const expectedStatus = totalCount === 0 ? "unavailable" : totalCount > includedCount ? "truncated" : "bounded"; + if (coverage.status !== expectedStatus) errors.push(`${location}.evidenceCoverage.status is invalid`); + if (!SAFE_DIGEST_RE.test(String(coverage.identitySetDigest ?? ""))) { + errors.push(`${location}.evidenceCoverage.identitySetDigest is invalid`); + } + } + return errors; +} + +export function validateStoredNativeLearningReviewPacket(packet) { + const errors = []; + const allowedFields = new Set(["schemaVersion", "kind", "sourceDigest", "limits", "allowed", "episodeFacts", "groups", "coverage", "packetDigest"]); + if (!packet || typeof packet !== "object" || Array.isArray(packet)) return ["native learning review packet must be an object"]; + for (const field of Object.keys(packet)) if (!allowedFields.has(field)) errors.push(`native learning review packet has unsupported field: ${field}`); + if (packet.schemaVersion !== NATIVE_LEARNING_REVIEW_PACKET_SCHEMA_VERSION) errors.push(`native learning review packet schemaVersion must be ${NATIVE_LEARNING_REVIEW_PACKET_SCHEMA_VERSION}`); + if (packet.kind !== NATIVE_LEARNING_REVIEW_KIND) errors.push("native learning review packet kind is invalid"); + if (packet.packetDigest !== nativeLearningReviewPacketDigest(packet)) errors.push("native learning review packet digest does not match its content"); + if (packet.sourceDigest !== nativeLearningReviewSourceDigest(packet)) errors.push("native learning review packet sourceDigest does not match episode facts, groups, and limits"); + const limitsAreObject = packet.limits && typeof packet.limits === "object" && !Array.isArray(packet.limits); + const limits = limitsAreObject ? normalizedLimits(packet.limits) : normalizedLimits(); + if (!limitsAreObject || JSON.stringify(packet.limits) !== JSON.stringify(limits)) errors.push("native learning review packet limits are invalid"); + const expectedAllowed = { + decisions: [...NATIVE_LEARNING_REVIEW_DECISIONS], + patternIds: [...NATIVE_LEARNING_PATTERN_IDS], + matchReasonCodes: [...NATIVE_LEARNING_MATCH_REASON_CODES], + abstainReasonCodes: [...NATIVE_LEARNING_ABSTAIN_REASON_CODES], + }; + if (JSON.stringify(canonicalValue(packet.allowed)) !== JSON.stringify(canonicalValue(expectedAllowed))) errors.push("native learning review packet allowed enums are invalid"); + if (!Array.isArray(packet.episodeFacts) || !Array.isArray(packet.groups) || !packet.coverage || typeof packet.coverage !== "object" || Array.isArray(packet.coverage)) { + errors.push("native learning review packet requires episodeFacts, groups, and coverage"); + return [...new Set(errors)].sort(); + } + if (packet.episodeFacts.length > limits.maxEpisodes) errors.push("native learning review packet episodeFacts exceeds the packet limit"); + for (const [index, fact] of packet.episodeFacts.entries()) { + errors.push(...storedEpisodeFactErrors(fact, index, limits)); + } + const episodeRefs = packet.episodeFacts.map((fact) => fact?.episodeRef); + if (JSON.stringify(episodeRefs) !== JSON.stringify(uniqueSorted(episodeRefs))) { + errors.push("native learning review packet episodeFacts must have distinct sorted Episode refs"); + } + if (errors.some((error) => error.includes("episodeFacts"))) return [...new Set(errors)].sort(); + const { groups: allGroups, subsumedCount } = screeningGroups(packet.episodeFacts); + const expectedGroups = allGroups.slice(0, limits.maxGroups); + if (JSON.stringify(canonicalValue(packet.groups)) !== JSON.stringify(canonicalValue(expectedGroups))) { + errors.push("native learning review packet groups do not match retained Episode facts"); + } + const inputEpisodeCount = Number(packet.coverage.inputEpisodeCount); + if (!Number.isInteger(inputEpisodeCount) || inputEpisodeCount < packet.episodeFacts.length) { + errors.push("native learning review packet coverage inputEpisodeCount is invalid"); + } else { + const expectedCoverage = coverageFor( + inputEpisodeCount, + packet.episodeFacts, + allGroups, + expectedGroups, + subsumedCount, + ); + if (JSON.stringify(canonicalValue(packet.coverage)) !== JSON.stringify(canonicalValue(expectedCoverage))) { + errors.push("native learning review packet coverage does not match retained groups and Episode facts"); + } + } + return [...new Set(errors)].sort(); +} + export function buildNativeLearningReviewPacket({ episodes = [], limits: suppliedLimits = {} } = {}) { const limits = normalizedLimits(suppliedLimits); const byId = new Map(); @@ -428,7 +589,7 @@ export function buildNativeLearningReviewPacket({ episodes = [], limits: supplie const packet = { schemaVersion: NATIVE_LEARNING_REVIEW_PACKET_SCHEMA_VERSION, kind: NATIVE_LEARNING_REVIEW_KIND, - sourceDigest: digest(safeSourceProjection), + sourceDigest: nativeLearningReviewSourceDigest(safeSourceProjection), limits, allowed: { decisions: [...NATIVE_LEARNING_REVIEW_DECISIONS], @@ -445,14 +606,7 @@ export function buildNativeLearningReviewPacket({ episodes = [], limits: supplie } export function validateNativeLearningReviewPacket({ episodes = [], packet } = {}) { - const errors = []; - if (packet?.schemaVersion !== NATIVE_LEARNING_REVIEW_PACKET_SCHEMA_VERSION) { - errors.push(`native learning review packet schemaVersion must be ${NATIVE_LEARNING_REVIEW_PACKET_SCHEMA_VERSION}`); - } - if (packet?.kind !== NATIVE_LEARNING_REVIEW_KIND) errors.push("native learning review packet kind is invalid"); - if (packet?.packetDigest !== nativeLearningReviewPacketDigest(packet)) { - errors.push("native learning review packet digest does not match its content"); - } + const errors = [...validateStoredNativeLearningReviewPacket(packet)]; let expected; try { expected = buildNativeLearningReviewPacket({ episodes, limits: packet?.limits }); diff --git a/scripts/harness-analysis/report-source/apply-review.mjs b/scripts/harness-analysis/report-source/apply-review.mjs index 11be346..61112a5 100644 --- a/scripts/harness-analysis/report-source/apply-review.mjs +++ b/scripts/harness-analysis/report-source/apply-review.mjs @@ -5,6 +5,10 @@ import { } from "../task-loop-report.mjs"; import { validateHarnessReviewBinding } from "./review-packet.mjs"; import { applyEpisodeReviews, normalizeDeliveryReviews } from "./episode-review.mjs"; +import { + applyStoredNativeLearningCandidateReview, + validateNativeLearningCandidateReview, +} from "../learning-loop-candidates.mjs"; function rows(value) { return Array.isArray(value) ? value : []; @@ -113,6 +117,26 @@ export function applyReportSourceReview(sourceInput, reviewInput, { packet } = { if (review.repositoryEvidence?.workflowDemandDiagnostics !== undefined) { throw new Error("repositoryEvidence.workflowDemandDiagnostics is generated and cannot be authored in review input"); } + const diagnostics = source.repositoryEvidence?.learningCaptureDiagnostics; + const native = diagnostics?.nativeLearningReview; + let pendingNativeReview = null; + if (native?.status === "review-required" && review.nativeLearningReview === undefined) { + throw new Error("nativeLearningReview is required for a generated pending native Learning Capture packet"); + } + if (review.nativeLearningReview !== undefined) { + if (native?.status !== "review-required" || !native.packet) { + throw new Error("nativeLearningReview requires a generated pending native Learning Capture packet"); + } + const nativeErrors = validateNativeLearningCandidateReview({ + episodes: source.taskEpisodes, + packet: native.packet, + review: review.nativeLearningReview, + }); + if (nativeErrors.length > 0) { + throw Object.assign(new Error(nativeErrors.join("; ")), { errors: nativeErrors }); + } + pendingNativeReview = { packet: native.packet, review: clone(review.nativeLearningReview) }; + } source.assessmentDecisions = rows(source.assessmentDecisions).map((decision) => { if (decision.kind === "source-candidate") { @@ -160,6 +184,35 @@ export function applyReportSourceReview(sourceInput, reviewInput, { packet } = { } : {}), }; if (review.episodeReviews !== undefined) applyEpisodeReviews(source, review.episodeReviews); + if (review.interventionLedger !== undefined) source.interventionLedger = clone(review.interventionLedger); + if (pendingNativeReview) { + const currentDiagnostics = source.repositoryEvidence.learningCaptureDiagnostics; + const applied = applyStoredNativeLearningCandidateReview({ + episodes: source.taskEpisodes, + packet: pendingNativeReview.packet, + review: pendingNativeReview.review, + signals: currentDiagnostics.signals, + interventions: source.interventionLedger, + assetCoverage: source.repositoryEvidence?.aiAgentPractice?.coverageRows, + }); + if (applied.errors.length > 0) { + throw Object.assign(new Error(applied.errors.join("; ")), { errors: applied.errors }); + } + source.repositoryEvidence.learningCaptureDiagnostics = { + ...currentDiagnostics, + learningCaptureSchemaVersion: applied.result.learningLoop.schemaVersion, + episodeRecords: applied.result.learningLoop.episodeRecords, + recurringIssueCandidates: applied.result.learningLoop.candidates, + coverage: applied.result.learningLoop.coverage, + nativeLearningReview: { + schemaVersion: 1, + status: "reviewed", + packet: pendingNativeReview.packet, + review: pendingNativeReview.review, + result: applied.result, + }, + }; + } const reviewedDelivery = review.deliveryReviews === undefined ? [] : normalizeDeliveryReviews(source, review.deliveryReviews); @@ -177,8 +230,6 @@ export function applyReportSourceReview(sourceInput, reviewInput, { packet } = { evidenceRefs: clone(episode.closure.evidenceRefs), })); source.deliveryEvidence = [...existingDelivery, ...reviewedFocusedChecks, ...reviewedDelivery]; - if (review.interventionLedger !== undefined) source.interventionLedger = clone(review.interventionLedger); - const sourceErrors = validateHarnessReportSource(source); if (sourceErrors.length > 0) throw Object.assign(new Error(sourceErrors.join("; ")), { errors: sourceErrors }); const projected = projectTaskLoopFindings(source); diff --git a/scripts/harness-analysis/report-source/cli.mjs b/scripts/harness-analysis/report-source/cli.mjs new file mode 100644 index 0000000..df26a63 --- /dev/null +++ b/scripts/harness-analysis/report-source/cli.mjs @@ -0,0 +1,294 @@ +#!/usr/bin/env node + +import { randomUUID } from "node:crypto"; +import { chmod, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { applyReportSourceReview } from "./apply-review.mjs"; +import { + buildHarnessLeadDecisionTemplate, + buildHarnessReviewPacket, + compileHarnessLeadDecision, + harnessLeadDecisionFromDocument, +} from "./review-packet.mjs"; +import { validateHarnessReportSource } from "./source.mjs"; + +export const SOURCE_REVIEW_HELP = `Usage: better-harness harness source-review [options] + +Run the local report-source review lifecycle without authoring a decision or +calling a model. Keep source, packet, decision template, and review files local. + +Phases: + create --source --packet + --decision + decision --source --packet + --decision --review + apply --source --packet + --review --yes + +Options: + --json Emit one parser-safe JSON document + --yes Confirm replacement of --source during apply + -h, --help +`; + +const VALUE_OPTIONS = new Set(["source", "packet", "decision", "review"]); +const MACHINE_ERROR_MESSAGES = new Map([ + ["UNKNOWN_ARGUMENT", "source-review received an unsupported argument"], + ["MISSING_ARGUMENT_VALUE", "a source-review option requires a value"], + ["DUPLICATE_ARGUMENT", "a source-review option was supplied more than once"], + ["INVALID_HELP_COMBINATION", "source-review help must be requested without other arguments"], + ["MISSING_ARGUMENT", "source-review is missing a required option"], + ["UNEXPECTED_ARGUMENT", "an option is not valid for the selected source-review phase"], + ["INVALID_REPORT_SOURCE", "the selected report source is not readable valid JSON or failed validation"], + ["INVALID_REVIEW_PACKET", "the selected review packet is not readable valid JSON or failed validation"], + ["INVALID_LEAD_DECISION_TEMPLATE", "the lead decision template does not match the selected review packet"], + ["INVALID_LEAD_DECISION", "the caller-authored lead decision failed validation"], + ["INVALID_COMPILED_REVIEW", "the compiled review failed validation"], + ["OUTPUT_PATH_CONFLICT", "source-review input and output roles require distinct files"], + ["INVALID_SOURCE_REVIEW_PHASE", "source-review phase must be create, decision, or apply"], + ["APPLY_CONFIRMATION_REQUIRED", "source-review apply requires explicit confirmation"], +]); + +function parseArgs(argv) { + const options = { json: false, yes: false }; + let phase = ""; + for (let index = 0; index < argv.length; index += 1) { + const value = argv[index]; + if (value === "-h" || value === "--help") options.help = true; + else if (value === "--json") options.json = true; + else if (value === "--yes") options.yes = true; + else if (value.startsWith("--")) { + const name = value.slice(2); + if (!VALUE_OPTIONS.has(name)) throw Object.assign(new Error(`unknown source-review argument: ${value}`), { code: "UNKNOWN_ARGUMENT" }); + const supplied = argv[index + 1]; + if (!supplied || supplied.startsWith("--")) throw Object.assign(new Error(`${value} requires a value`), { code: "MISSING_ARGUMENT_VALUE" }); + if (options[name] !== undefined) throw Object.assign(new Error(`${value} may be supplied only once`), { code: "DUPLICATE_ARGUMENT" }); + options[name] = supplied; + index += 1; + } else if (!phase) phase = value; + else throw Object.assign(new Error("source-review received an unexpected positional argument"), { code: "UNKNOWN_ARGUMENT" }); + } + if (options.help && argv.length !== 1) { + throw Object.assign(new Error("--help must be used without a phase or other arguments"), { + code: "INVALID_HELP_COMBINATION", + }); + } + return { phase, options }; +} + +function requireOptions(options, names) { + for (const name of names) { + if (!options[name]) throw Object.assign(new Error(`--${name} is required`), { code: "MISSING_ARGUMENT" }); + } +} + +function rejectOptions(options, names, phase) { + for (const name of names) { + if (options[name]) { + throw Object.assign(new Error(`--${name} is not valid for source-review ${phase}`), { + code: "UNEXPECTED_ARGUMENT", + }); + } + } +} + +async function readJson(filePath, label) { + let content; + try { + content = await readFile(filePath, "utf8"); + } catch { + throw Object.assign(new Error(`${label} is not a readable local JSON file`), { + code: `INVALID_${label.toUpperCase().replace(/[^A-Z0-9]+/gu, "_")}`, + }); + } + try { + return JSON.parse(content); + } catch { + throw Object.assign(new Error(`${label} is not valid JSON`), { + code: `INVALID_${label.toUpperCase().replace(/[^A-Z0-9]+/gu, "_")}`, + }); + } +} + +function assertDifferentPaths(left, right, message) { + if (path.resolve(left) === path.resolve(right)) throw Object.assign(new Error(message), { code: "OUTPUT_PATH_CONFLICT" }); +} + +async function writeNewJson(filePath, value) { + await writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`, { flag: "wx", mode: 0o600 }); +} + +async function writeNewReviewInputs(packetPath, packet, decisionPath, decisionTemplate) { + let packetCreated = false; + try { + await writeNewJson(packetPath, packet); + packetCreated = true; + await writeNewJson(decisionPath, decisionTemplate); + } catch (error) { + if (packetCreated) await rm(packetPath, { force: true }).catch(() => {}); + throw error; + } +} + +async function atomicReplaceJson(filePath, value) { + const metadata = await stat(filePath); + const temporaryPath = path.join(path.dirname(filePath), `.${path.basename(filePath)}.${process.pid}.${randomUUID()}.tmp`); + try { + await writeFile(temporaryPath, `${JSON.stringify(value, null, 2)}\n`, { flag: "wx" }); + await chmod(temporaryPath, metadata.mode); + await rename(temporaryPath, filePath); + } catch (error) { + await rm(temporaryPath, { force: true }).catch(() => {}); + throw error; + } +} + +function assertValidSource(source) { + const errors = validateHarnessReportSource(source); + if (errors.length > 0) { + throw Object.assign(new Error(errors.join("; ")), { code: "INVALID_REPORT_SOURCE", errors }); + } +} + +export async function runSourceReview({ phase, options }) { + if (!new Set(["create", "decision", "apply"]).has(phase)) { + throw Object.assign(new Error("source-review phase must be create, decision, or apply"), { + code: "INVALID_SOURCE_REVIEW_PHASE", + }); + } + requireOptions(options, ["source", "packet"]); + const sourcePath = path.resolve(options.source); + const packetPath = path.resolve(options.packet); + assertDifferentPaths(sourcePath, packetPath, "--packet must not name --source"); + const source = await readJson(sourcePath, "report source"); + assertValidSource(source); + + if (phase === "create") { + rejectOptions(options, ["review", "yes"], phase); + requireOptions(options, ["decision"]); + const decisionPath = path.resolve(options.decision); + assertDifferentPaths(sourcePath, decisionPath, "--decision must not name --source"); + assertDifferentPaths(packetPath, decisionPath, "--decision must not name --packet"); + const packet = buildHarnessReviewPacket(source); + const decisionTemplate = buildHarnessLeadDecisionTemplate(source, packet); + await writeNewReviewInputs(packetPath, packet, decisionPath, decisionTemplate); + return { + kind: "harness-source-review", + schemaVersion: 1, + status: "packet-created", + packetDigest: packet.packetDigest, + nativeGroupCount: packet.nativeLearningReview?.packet?.groups?.length ?? 0, + decisionRequired: Boolean(packet.nativeLearningReview?.packet), + decisionTemplateCreated: true, + }; + } + + const packet = await readJson(packetPath, "review packet"); + if (phase === "decision") { + rejectOptions(options, ["yes"], phase); + requireOptions(options, ["decision", "review"]); + const decisionPath = path.resolve(options.decision); + const reviewPath = path.resolve(options.review); + assertDifferentPaths(sourcePath, reviewPath, "--review must not replace --source"); + assertDifferentPaths(packetPath, reviewPath, "--review must not replace --packet"); + assertDifferentPaths(decisionPath, reviewPath, "--review must not replace --decision"); + const document = await readJson(decisionPath, "lead decision"); + if (document?.kind !== "harness-lead-decision-template") { + throw Object.assign(new Error("public source-review decision requires the template generated by create"), { + code: "INVALID_LEAD_DECISION_TEMPLATE", + }); + } + const resolved = harnessLeadDecisionFromDocument(source, packet, document); + if (resolved.errors.length > 0) { + throw Object.assign(new Error(resolved.errors.join("; ")), { + code: "INVALID_LEAD_DECISION_TEMPLATE", + }); + } + const compiled = compileHarnessLeadDecision(source, packet, resolved.decision); + if (compiled.errors.length > 0) { + throw Object.assign(new Error(compiled.errors.join("; ")), { + code: "INVALID_LEAD_DECISION", + errors: compiled.errors, + }); + } + await writeNewJson(reviewPath, compiled.review); + return { + kind: "harness-source-review", + schemaVersion: 1, + status: "decision-compiled", + packetDigest: packet.packetDigest, + }; + } + + rejectOptions(options, ["decision"], phase); + requireOptions(options, ["review"]); + if (!options.yes) { + throw Object.assign(new Error("source-review apply requires --yes before replacing --source"), { + code: "APPLY_CONFIRMATION_REQUIRED", + }); + } + const reviewPath = path.resolve(options.review); + assertDifferentPaths(sourcePath, reviewPath, "--review must not name --source"); + assertDifferentPaths(packetPath, reviewPath, "--review must not name --packet"); + const review = await readJson(reviewPath, "compiled review"); + let reviewed; + try { + reviewed = applyReportSourceReview(source, review, { packet }); + } catch (error) { + throw Object.assign(new Error(error.message), { + code: "INVALID_COMPILED_REVIEW", + ...(Array.isArray(error?.errors) ? { errors: error.errors } : {}), + }); + } + await atomicReplaceJson(sourcePath, reviewed); + return { + kind: "harness-source-review", + schemaVersion: 1, + status: "review-applied", + packetDigest: packet.packetDigest, + nativeStatus: reviewed.repositoryEvidence?.learningCaptureDiagnostics?.nativeLearningReview?.status ?? "absent", + }; +} + +function successText(payload) { + if (payload.status === "packet-created") return "source-review packet created"; + if (payload.status === "decision-compiled") return "source-review decision compiled"; + return "source-review applied"; +} + +export async function main(argv = process.argv.slice(2)) { + const machine = argv.includes("--json"); + let parsed; + try { + parsed = parseArgs(argv); + if (parsed.options.help) { + process.stdout.write(SOURCE_REVIEW_HELP); + return 0; + } + const payload = await runSourceReview(parsed); + process.stdout.write(parsed.options.json ? `${JSON.stringify(payload, null, 2)}\n` : `${successText(payload)}\n`); + return 0; + } catch (error) { + if (machine) { + const code = typeof error?.code === "string" ? error.code : "SOURCE_REVIEW_FAILED"; + process.stdout.write(`${JSON.stringify({ + kind: "harness-source-review", + schemaVersion: 1, + status: "error", + code, + message: MACHINE_ERROR_MESSAGES.get(code) + ?? "source-review failed while accessing a selected local file", + }, null, 2)}\n`); + } else { + process.stderr.write(`source-review failed: ${error.message}\n`); + } + return 1; + } +} + +const currentFile = fileURLToPath(import.meta.url); +if (process.argv[1] && path.resolve(process.argv[1]) === currentFile) { + process.exitCode = await main(); +} diff --git a/scripts/harness-analysis/report-source/review-packet.mjs b/scripts/harness-analysis/report-source/review-packet.mjs index b851ac2..41db282 100644 --- a/scripts/harness-analysis/report-source/review-packet.mjs +++ b/scripts/harness-analysis/report-source/review-packet.mjs @@ -1,7 +1,9 @@ import { createHash } from "node:crypto"; import { LEARNING_LOOP_CHECK_IDS } from "../learning-loop-contract.mjs"; +import { validateNativeLearningCandidateReview } from "../learning-loop-candidates.mjs"; -export const HARNESS_REVIEW_PACKET_SCHEMA_VERSION = 2; +export const HARNESS_REVIEW_PACKET_SCHEMA_VERSION = 3; +export const HARNESS_LEAD_DECISION_TEMPLATE_SCHEMA_VERSION = 1; const LEARNING_CAPTURE_CHECK_IDS = new Set(LEARNING_LOOP_CHECK_IDS); const EVIDENCE_ARRAY_FIELDS = new Set([ @@ -53,6 +55,7 @@ function evidenceKey(reference) { function publicEvidenceReference(reference) { const key = evidenceKey(reference); if (!key) return null; + if (String(reference.kind).trim() === "native-learning-evidence") return null; const value = { kind: String(reference.kind).trim(), id: String(reference.id).trim(), @@ -111,6 +114,7 @@ export function harnessReviewPacketDigest(packet) { } export function buildHarnessReviewPacket(source) { + const native = source?.repositoryEvidence?.learningCaptureDiagnostics?.nativeLearningReview; const packet = { schemaVersion: HARNESS_REVIEW_PACKET_SCHEMA_VERSION, kind: "harness-report-review-packet", @@ -122,11 +126,140 @@ export function buildHarnessReviewPacket(source) { taskUnderstandingState: ["Exercised", "Unobserved", "Not applicable"], }, allowedEvidenceRefs: collectSourceEvidenceIndex(source), + ...(native?.status === "review-required" ? { nativeLearningReview: { packet: clone(native.packet) } } : {}), }; packet.packetDigest = harnessReviewPacketDigest(packet); return packet; } +function decisionTemplateRow(id, extra = {}) { + return { id, summary: null, evidenceRefs: [], ...extra }; +} + +function decisionTemplateNativeGroups(packet) { + return rows(packet?.nativeLearningReview?.packet?.groups).map((group) => ({ + groupRef: group.groupRef, + episodeRefs: clone(group.episodeRefs), + evidenceRefs: clone(group.evidenceRefs), + allowedPatternIds: clone(group.allowedPatternIds), + supportedReasonCodes: clone(group.reasonCodes), + })); +} + +export function buildHarnessLeadDecisionTemplate(source, packet) { + const nativeGroups = decisionTemplateNativeGroups(packet); + return { + schemaVersion: HARNESS_LEAD_DECISION_TEMPLATE_SCHEMA_VERSION, + kind: "harness-lead-decision-template", + sourceDigest: packet.sourceDigest, + packetDigest: packet.packetDigest, + instructions: [ + "Edit only the decision object; replace nulls and add exact packet evidence references.", + "Retain every required id and every native groupRef exactly once.", + "For a native match, copy the exact group episodeRefs and selected evidenceRefs; for abstain, omit patternId and keep both arrays empty.", + "Add an optional shape only when the review changes Episode or delivery evidence.", + "The CLI validates and compiles this caller-authored judgment; it never selects values or calls a model.", + ], + required: clone(packet.required), + allowedEnums: clone(packet.allowedEnums), + nativeGroups, + optionalShapes: { + episodeReviews: { + item: { + episodeRef: null, + taskUnderstanding: ["goal-understanding", "relevant-context", "scope-boundary"] + .map((id) => decisionTemplateRow(id, { state: null })), + validationAssociations: [], + repairReview: { state: null }, + }, + }, + deliveryReviews: { + item: { + episodeRef: null, + provider: null, + kind: null, + level: null, + status: null, + summary: null, + evidenceRefs: [], + }, + }, + }, + decision: { + sourceCandidate: { evidenceRefs: [] }, + readerOverview: { text: null, evidenceRefs: [] }, + repositoryReview: { + reviewedFrameworks: rows(packet.required?.frameworks) + .map((id) => decisionTemplateRow(id)), + reviewedChecks: rows(packet.required?.checks).map((id) => decisionTemplateRow(id, { + ...(LEARNING_CAPTURE_CHECK_IDS.has(id) ? { state: null, findingRefs: [] } : {}), + ...(id === "loop-engineering" ? { mechanisms: [] } : {}), + })), + reviewedSoftwareFluencyCapabilities: rows(packet.required?.capabilities) + .map((id) => decisionTemplateRow(id)), + }, + repositoryEvidence: { diagnosticCoverageReviews: [] }, + scoreReview: { + dimensions: rows(packet.required?.dimensions).map((id) => ({ + id, + score: null, + confidence: null, + reason: null, + readerSummary: null, + evidenceRefs: [], + })), + }, + ...(nativeGroups.length > 0 ? { + nativeLearningDecisions: nativeGroups.map((group) => ({ + groupRef: group.groupRef, + decision: null, + patternId: null, + episodeRefs: [], + evidenceRefs: [], + reasonCodes: [], + })), + } : {}), + }, + }; +} + +export function validateHarnessLeadDecisionTemplate(source, packet, template) { + if (!template || typeof template !== "object" || Array.isArray(template)) { + return ["lead decision template must be one JSON object"]; + } + const errors = []; + const allowedFields = new Set([ + "schemaVersion", "kind", "sourceDigest", "packetDigest", "instructions", + "required", "allowedEnums", "nativeGroups", "optionalShapes", "decision", + ]); + for (const field of Object.keys(template)) { + if (!allowedFields.has(field)) errors.push(`lead decision template has unsupported field: ${field}`); + } + if (template.schemaVersion !== HARNESS_LEAD_DECISION_TEMPLATE_SCHEMA_VERSION) { + errors.push(`lead decision template schemaVersion must be ${HARNESS_LEAD_DECISION_TEMPLATE_SCHEMA_VERSION}`); + } + if (template.kind !== "harness-lead-decision-template") errors.push("lead decision template kind is invalid"); + if (!template.decision || typeof template.decision !== "object" || Array.isArray(template.decision)) { + errors.push("lead decision template decision must be one JSON object"); + } + const expected = buildHarnessLeadDecisionTemplate(source, packet); + const metadata = ({ decision: _decision, ...value }) => value; + if (digestJson(metadata(template)) !== digestJson(metadata(expected))) { + errors.push("lead decision template contract does not match the selected review packet"); + } + return [...new Set(errors)].sort(); +} + +export function harnessLeadDecisionFromDocument(source, packet, document) { + if (document?.kind !== "harness-lead-decision-template") { + return { decision: document, errors: [] }; + } + return { + decision: clone(document.decision), + errors: validateHarnessLeadDecisionTemplate(source, packet, document), + }; +} + export function validateHarnessReviewPacket(source, packet) { const errors = []; if (packet?.schemaVersion !== HARNESS_REVIEW_PACKET_SCHEMA_VERSION) { @@ -141,6 +274,9 @@ export function validateHarnessReviewPacket(source, packet) { if (digestJson(packet?.allowedEvidenceRefs) !== digestJson(expected.allowedEvidenceRefs)) { errors.push("review packet evidence index does not match the current source"); } + if (digestJson(packet?.nativeLearningReview ?? null) !== digestJson(expected.nativeLearningReview ?? null)) { + errors.push("review packet native Learning Capture subpacket does not match the current source"); + } return errors; } @@ -261,6 +397,26 @@ export function compileHarnessLeadDecision(source, packet, decisionInput) { ...(decision.deliveryReviews !== undefined ? { deliveryReviews: clone(decision.deliveryReviews) } : {}), ...(decision.interventionLedger !== undefined ? { interventionLedger: clone(decision.interventionLedger) } : {}), }; + const nativePacket = packet?.nativeLearningReview?.packet; + if (nativePacket && decision.nativeLearningDecisions === undefined) { + errors.push("nativeLearningDecisions is required for the native Learning Capture subpacket"); + } else if (decision.nativeLearningDecisions !== undefined) { + if (!nativePacket) { + errors.push("nativeLearningDecisions requires a native Learning Capture subpacket"); + } else { + review.nativeLearningReview = { + schemaVersion: nativePacket.schemaVersion, + sourceDigest: nativePacket.sourceDigest, + packetDigest: nativePacket.packetDigest, + decisions: clone(decision.nativeLearningDecisions), + }; + errors.push(...validateNativeLearningCandidateReview({ + episodes: source?.taskEpisodes, + packet: nativePacket, + review: review.nativeLearningReview, + })); + } + } errors.push(...validateHarnessReviewBinding(source, review, packet)); return { review, errors: [...new Set(errors)].sort() }; } @@ -274,7 +430,10 @@ function collectReviewEvidenceRefs(review) { return; } if (!value || typeof value !== "object") return; - for (const [field, child] of Object.entries(value)) visit(child, field); + for (const [field, child] of Object.entries(value)) { + if (field === "nativeLearningReview") continue; + visit(child, field); + } } visit(review); return references; diff --git a/scripts/harness-analysis/report-source/source.mjs b/scripts/harness-analysis/report-source/source.mjs index b4ec26a..3e5bf15 100644 --- a/scripts/harness-analysis/report-source/source.mjs +++ b/scripts/harness-analysis/report-source/source.mjs @@ -6,7 +6,7 @@ import { learningLoopStateErrors, projectLaterValidationState, } from "../learning-loop-contract.mjs"; -import { validateLearningLoopReview } from "../learning-loop-candidates.mjs"; +import { validateLearningLoopReview, validateNativeLearningReviewPacket, validateStoredNativeLearningAppliedReview } from "../learning-loop-candidates.mjs"; import { validateLearningCaptureEvidence } from "../learning-capture-evidence.mjs"; import { validateWorkflowDemandDiagnostics } from "../workflow-demand-diagnostics.mjs"; import { findingTargetErrors } from "../../workspace-topology/index.mjs"; @@ -89,7 +89,10 @@ const POSITIVE_RESULT_STATES = new Set([ "outcome-supported", "passed", "success", "succeeded", "unchanged", ]); const LEARNING_CAPTURE_DIAGNOSTIC_FIELDS = new Set([ - "signals", "learningCaptureSchemaVersion", "episodeRecords", "recurringIssueCandidates", "coverage", + "signals", "learningCaptureSchemaVersion", "episodeRecords", "recurringIssueCandidates", "coverage", "nativeLearningReview", +]); +const NATIVE_LEARNING_REVIEW_DIAGNOSTIC_FIELDS = new Set([ + "schemaVersion", "status", "packet", "review", "result", ]); const LEARNING_CAPTURE_SIGNAL_FIELDS = new Set(["observedSkills", "unscopedObservedSkills", "apparentSkillReads", "configuredSkills", "memories", "memoryActivity", "memoryScan", "frictionSignals", "priorInterventionCount"]); const MEMORY_SCAN_FIELDS = new Set(["status", "provider", "candidateCount", "contentPolicy"]); @@ -348,7 +351,69 @@ function validateStringArray(value, location) { : []; } -export function validateLearningCaptureDiagnostics(value) { +function nativeLearningReviewDiagnosticErrors(value, episodes, diagnostics, prefix, { + interventions = [], + assetCoverage = [], +} = {}) { + if (!isObject(value)) return [`${prefix}.nativeLearningReview must be an object`]; + const location = `${prefix}.nativeLearningReview`; + const errors = []; + for (const field of Object.keys(value)) { + if (!NATIVE_LEARNING_REVIEW_DIAGNOSTIC_FIELDS.has(field)) errors.push(`${location} has unsupported field: ${field}`); + } + if (value.schemaVersion !== 1) { + errors.push(`${location}.schemaVersion must be 1`); + } + if (!new Set(["review-required", "reviewed"]).has(value.status)) { + errors.push(`${location}.status must be review-required or reviewed`); + return errors; + } + if (value.status === "review-required") { + errors.push(...validateNativeLearningReviewPacket({ episodes, packet: value.packet }) + .map((error) => `${location}.packet: ${error}`)); + if (value.review !== undefined || value.result !== undefined) { + errors.push(`${location} review-required must omit review and result`); + } + return errors; + } + if (!isObject(value.packet) || value.review === undefined || value.result === undefined) { + errors.push(`${location} reviewed requires review and result`); + return errors; + } + if (value.review?.schemaVersion !== value.packet.schemaVersion + || value.review?.sourceDigest !== value.packet.sourceDigest + || value.review?.packetDigest !== value.packet.packetDigest + || !Array.isArray(value.review?.decisions)) { + errors.push(`${location}.review must retain the packet-bound native decision`); + } + errors.push(...validateStoredNativeLearningAppliedReview({ + episodes, + packet: value.packet, + review: value.review, + result: value.result, + signals: diagnostics.signals, + interventions, + assetCoverage, + }) + .map((error) => `${location}: ${error}`)); + const learningLoop = value.result?.learningLoop; + if (!isObject(value.result) || !isObject(learningLoop)) { + errors.push(`${location}.result must retain the applied native Learning Loop result`); + return errors; + } + if (JSON.stringify(diagnostics.episodeRecords) !== JSON.stringify(learningLoop.episodeRecords) + || JSON.stringify(diagnostics.recurringIssueCandidates) !== JSON.stringify(learningLoop.candidates) + || JSON.stringify(diagnostics.coverage) !== JSON.stringify(learningLoop.coverage)) { + errors.push(`${location} reviewed result must own the canonical Learning Capture projection`); + } + return errors; +} + +export function validateLearningCaptureDiagnostics(value, { + episodes = [], + interventions = [], + assetCoverage = [], +} = {}) { if (value === undefined) return []; if (!isObject(value)) return ["report source repositoryEvidence.learningCaptureDiagnostics must be an object"]; const prefix = "report source repositoryEvidence.learningCaptureDiagnostics"; @@ -404,6 +469,12 @@ export function validateLearningCaptureDiagnostics(value) { candidates: value.recurringIssueCandidates, coverage: value.coverage, }).map((error) => `${prefix}.${error}`)); + if (value.nativeLearningReview !== undefined) { + errors.push(...nativeLearningReviewDiagnosticErrors(value.nativeLearningReview, episodes, value, prefix, { + interventions, + assetCoverage, + })); + } return errors; } @@ -1065,7 +1136,11 @@ export function validateHarnessReportSource(source) { if (source.repositoryEvidence?.learningCaptureReview !== undefined) { errors.push("report source repositoryEvidence.learningCaptureReview is retired"); } - errors.push(...validateLearningCaptureDiagnostics(source.repositoryEvidence?.learningCaptureDiagnostics)); + errors.push(...validateLearningCaptureDiagnostics(source.repositoryEvidence?.learningCaptureDiagnostics, { + episodes: source.taskEpisodes, + interventions: source.interventionLedger, + assetCoverage: source.repositoryEvidence?.aiAgentPractice?.coverageRows, + })); if (source.repositoryEvidence?.workflowDemandDiagnostics !== undefined) { errors.push(...validateWorkflowDemandDiagnostics( source.repositoryEvidence.workflowDemandDiagnostics, diff --git a/scripts/harness-analysis/task-loop-source.mjs b/scripts/harness-analysis/task-loop-source.mjs index 778ae7e..1666e67 100644 --- a/scripts/harness-analysis/task-loop-source.mjs +++ b/scripts/harness-analysis/task-loop-source.mjs @@ -55,7 +55,7 @@ import { import { projectAgentLintPracticeEvidence } from "./practice-findings.mjs"; import { loadPriorLearningCaptureState } from "./learning-capture-state.mjs"; import { scanTaskLoopRepositoryEvidence } from "./task-loop-repository-evidence.mjs"; -import { buildLearningLoopReview } from "./learning-loop-candidates.mjs"; +import { buildLearningLoopReview, buildNativeLearningReviewPacket } from "./learning-loop-candidates.mjs"; import { buildWorkflowDemandDiagnostics } from "./workflow-demand-diagnostics.mjs"; import { findingTargetFromTopology } from "../workspace-topology/index.mjs"; @@ -314,12 +314,20 @@ function learningCaptureDiagnosticsCandidate(insights, repositoryEvidence, inter interventions, assetCoverage: repositoryEvidence?.aiAgentPractice?.coverageRows, }); + const nativePacket = buildNativeLearningReviewPacket({ episodes: taskEpisodes }); return { signals, learningCaptureSchemaVersion: learningLoop.schemaVersion, episodeRecords: learningLoop.episodeRecords, recurringIssueCandidates: learningLoop.candidates, coverage: learningLoop.coverage, + ...(nativePacket.groups.length > 0 ? { + nativeLearningReview: { + schemaVersion: 1, + status: "review-required", + packet: nativePacket, + }, + } : {}), }; } diff --git a/scripts/npm-package/verify-pack.mjs b/scripts/npm-package/verify-pack.mjs index b4f8f41..e6916e5 100644 --- a/scripts/npm-package/verify-pack.mjs +++ b/scripts/npm-package/verify-pack.mjs @@ -177,6 +177,7 @@ const required = [ "package/scripts/harness-analysis/evidence-bundle/project-harness.mjs", "package/scripts/harness-analysis/evidence-bundle/agent-customize.mjs", "package/scripts/harness-analysis/report-source/apply-review.mjs", + "package/scripts/harness-analysis/report-source/cli.mjs", "package/scripts/harness-analysis/report-source/episode-review.mjs", "package/scripts/harness-analysis/report-source/index.mjs", "package/scripts/harness-analysis/report-source/review-packet.mjs", @@ -306,6 +307,7 @@ const requiredBundleEntries = [ "scripts/harness-analysis/evidence-bundle/project-harness.mjs", "scripts/harness-analysis/evidence-bundle/agent-customize.mjs", "scripts/harness-analysis/report-source/apply-review.mjs", + "scripts/harness-analysis/report-source/cli.mjs", "scripts/harness-analysis/report-source/episode-review.mjs", "scripts/harness-analysis/report-source/index.mjs", "scripts/harness-analysis/report-source/review-packet.mjs", diff --git a/skills/better-harness/SKILL.md b/skills/better-harness/SKILL.md index 2720c11..971ab0f 100644 --- a/skills/better-harness/SKILL.md +++ b/skills/better-harness/SKILL.md @@ -203,10 +203,10 @@ bare directory, or an output-file inventory. - Usage/model questions use `session-analysis usage-summary` once. - Repeated work continues through [Loop Discovery](../../references/loop-engineering/loop-discovery.md). -- Detailed routes are available through - [Agent Customize](../../references/agent-customize/routing.md), +- Routes: [Agent Customize](../../references/agent-customize/routing.md), [Core Change Watch](../../references/project-harness/core-change-watch.md), - and [Report Routing](../../templates/reporting/routing.md). + [Report Routing](../../templates/reporting/routing.md), + [Source Review](references/report-source-review.md). The durable route authorizes only renderer-owned artifacts in its host root. Other creation, activation, mutation, cleanup, scheduling, external diff --git a/skills/better-harness/references/report-source-review.md b/skills/better-harness/references/report-source-review.md new file mode 100644 index 0000000..09223e3 --- /dev/null +++ b/skills/better-harness/references/report-source-review.md @@ -0,0 +1,49 @@ +# Report Source Review + +Use this maintainer route only when the task needs the deterministic Harness +report source to pass through a reviewable local decision boundary before +projection. It is not a substitute for the ordinary evidence-bundle and +renderer workflow. + +Create the source, then freeze its bounded review packet and create-only +decision template: + +```text + harness source --workspace --source /report.source.json --language + harness source-review create --source /report.source.json --packet /review.packet.json --decision /lead.decision.json --json +``` + +Edit only the generated template's `decision` object. Fill +`sourceCandidate.evidenceRefs` and `readerOverview`; retain every generated +framework, check, capability, and score row; add a summary and exact outer +packet evidence to each row; and assign each score, confidence, reason, and +reader summary. Keep packet aliases opaque and local. + +For every generated native group, choose exactly one `match` or `abstain`. +A match copies the group's exact Episode refs, selects evidence aliases owned +by every Episode, and uses only supported pattern/reason codes. An abstention +omits `patternId` and keeps Episode/evidence arrays empty. The template omits +optional Episode and delivery mutations; copy an item from `optionalShapes` +into `decision.episodeReviews` or `decision.deliveryReviews` only when that +evidence is being reviewed. The command does not call a model or choose a +decision. + +Compile the caller-authored decision, then explicitly confirm source +replacement: + +```text + harness source-review decision --source /report.source.json --packet /review.packet.json --decision /lead.decision.json --review /review.json --json + harness source-review apply --source /report.source.json --packet /review.packet.json --review /review.json --yes --json +``` + +Stop on any non-zero status. Do not repair a stale packet, missing native +decision, or digest mismatch by editing generated packet or review JSON. Re-run +`create` from the current source and author a fresh decision. The apply phase is +the only phase that replaces the source, and it requires `--yes`. + +After a successful apply, project the reviewed source through its existing +owner: + +```text + harness task-loop-report --source /report.source.json --findings /findings.json --json +``` diff --git a/test/better-harness-cli.test.mjs b/test/better-harness-cli.test.mjs index 788e03b..e60a6f9 100644 --- a/test/better-harness-cli.test.mjs +++ b/test/better-harness-cli.test.mjs @@ -335,6 +335,11 @@ test("better-harness CLI exposes command inventory as JSON", () => { ); assert.equal(harness.subcommands.find((subcommand) => subcommand.name === "render").audience, "advanced"); assert.equal(harness.subcommands.find((subcommand) => subcommand.name === "source").audience, "maintainer"); + assert.equal(harness.subcommands.find((subcommand) => subcommand.name === "source-review").audience, "maintainer"); + assert.equal( + harness.subcommands.find((subcommand) => subcommand.name === "source-review").script, + "scripts/harness-analysis/report-source/cli.mjs", + ); assert.equal(harness.subcommands.some((subcommand) => subcommand.name === "prepare"), false); assert.equal(harness.subcommands.some((subcommand) => subcommand.name === "finalize"), false); assert.equal(harness.subcommands.some((subcommand) => subcommand.name === "apply-review"), false); @@ -708,6 +713,8 @@ test("better-harness CLI short-circuits help for every registered terminal path" // same dispatch (identical script, `invalid-before-help` stripped, trailing // `--help`); an identical dispatch yields byte-identical execution, so a // per-variant guarded subprocess would only re-run canonical's own args. + if (pathSegments.join(" ") === "harness source-review") continue; + for (const helpFlag of ["--help", "-h"]) { const args = [...pathSegments, "invalid-before-help", helpFlag]; const dispatch = resolveDispatch(args); @@ -724,6 +731,21 @@ test("better-harness CLI short-circuits help for every registered terminal path" } }); +test("source-review help does not hide invalid phase or trailing arguments", () => { + for (const args of [ + ["harness", "source-review", "bogus", "--help"], + ["harness", "source-review", "--help", "trailing"], + ]) { + const dispatch = resolveDispatch(args); + assert.equal(dispatch.kind, "dispatch"); + assert.deepEqual(dispatch.args, args.slice(2)); + const result = runBetterHarness(args); + assert.equal(result.status, 1); + assert.equal(result.stdout, ""); + assert.match(result.stderr, /--help must be used without a phase or other arguments/u); + } +}); + test("better-harness CLI preserves built-in discovery help and literal positional help", () => { const commands = runBetterHarness(["commands", "--help", "--audience", "advanced"]); assert.equal(commands.status, 0, commands.stderr); @@ -767,6 +789,7 @@ test("better-harness CLI group help expands advanced and maintainer subcommands" assert.equal(maintainer.status, 0, maintainer.stderr); const maintainerCommands = listedSubcommands(maintainer.stdout); assert.equal(maintainerCommands.includes("source"), true); + assert.equal(maintainerCommands.includes("source-review"), true); assert.equal(maintainerCommands.includes("repair-findings"), true); }); @@ -836,7 +859,7 @@ test("better-harness CLI emits a machine-readable schema", () => { }); test("registered harness commands remain executable through the Node facade", () => { - for (const subcommand of ["evidence-bundle", "analyze", "selection-profile", "source", "task-loop-report", "render", "record-fix-output"]) { + for (const subcommand of ["evidence-bundle", "analyze", "selection-profile", "source", "source-review", "task-loop-report", "render", "record-fix-output"]) { const result = runBetterHarness(["harness", subcommand, "--help"]); assert.equal(result.status, 0, `${subcommand}: ${result.stderr}`); assert.match(result.stdout, /Usage:/, subcommand); diff --git a/test/fixtures/scripts-refactor-contract/harness-help.txt b/test/fixtures/scripts-refactor-contract/harness-help.txt index 3d1a468..1d0148b 100644 --- a/test/fixtures/scripts-refactor-contract/harness-help.txt +++ b/test/fixtures/scripts-refactor-contract/harness-help.txt @@ -14,6 +14,7 @@ Subcommands: checkup Scan agent customizations and produce a read-only cleanup plan selection-profile Build a privacy-safe session selection profile source Build the deterministic Agent Work Loop source envelope + source-review Create, compile, and apply a bounded report-source review task-loop-report Project deterministic task-loop findings from report.source.json render Render reviewed findings data into report artifacts preview-canvas Preview a Qoder Canvas report on this machine @@ -28,6 +29,7 @@ Examples: better-harness harness checkup --phase scan --provider qoder --workspace . --json better-harness harness workspace-topology --workspace . --json better-harness harness source --workspace . --source /report.source.json --language en + better-harness harness source-review create --source /report.source.json --packet /review.packet.json --decision /lead.decision.json --json better-harness harness render --findings /findings.json --mode qoder-canvas --out .qoder/better-harness --target . --validate --json better-harness harness preview-canvas /report.canvas.tsx --open better-harness harness report-quality --report /report.md diff --git a/test/fixtures/scripts-refactor-contract/root-help.txt b/test/fixtures/scripts-refactor-contract/root-help.txt index a3276ac..4171075 100644 --- a/test/fixtures/scripts-refactor-contract/root-help.txt +++ b/test/fixtures/scripts-refactor-contract/root-help.txt @@ -17,7 +17,7 @@ Commands: Workflows harness Check Better Harness readiness reports and Canvas outputs subcommands: evidence-bundle, workspace-topology, analyze, checkup, - selection-profile, source, task-loop-report, render, + selection-profile, source, source-review, task-loop-report, render, preview-canvas, report-quality, repair-findings, record-fix-output, validate-canvas diff --git a/test/report-source-review.test.mjs b/test/report-source-review.test.mjs index bfff13f..b9d3b67 100644 --- a/test/report-source-review.test.mjs +++ b/test/report-source-review.test.mjs @@ -1,7 +1,12 @@ import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; import test from "node:test"; import { applyReportSourceReview } from "../scripts/harness-analysis/apply-source-review.mjs"; +import { buildHarnessReviewPacket, compileHarnessLeadDecision } from "../scripts/harness-analysis/report-review-packet.mjs"; import { projectTaskLoopFindings } from "../scripts/harness-analysis/task-loop-report.mjs"; import { buildTaskLoopRepositoryEvidence } from "../scripts/harness-analysis/task-loop-repository-evidence.mjs"; import { buildTaskLoopSourceCandidate } from "../scripts/harness-analysis/task-loop-source.mjs"; @@ -11,7 +16,11 @@ import { validateHarnessReportSource, } from "../scripts/harness-analysis/report-source.mjs"; import { buildWorkflowDemandDiagnostics } from "../scripts/harness-analysis/workflow-demand-diagnostics.mjs"; -import { buildLearningLoopReview } from "../scripts/harness-analysis/learning-loop-candidates.mjs"; +import { + buildLearningLoopReview, + nativeLearningReviewPacketDigest, + nativeLearningReviewSourceDigest, +} from "../scripts/harness-analysis/learning-loop-candidates.mjs"; import { LEARNING_CAPTURE_REVIEWED_SCORE_FLOOR, } from "../scripts/harness-analysis/fluency-dimensions.mjs"; @@ -65,6 +74,29 @@ function sourceCandidate() { return source; } +function nativeCorrectionSource() { + const repairEvents = (sessionId, minute, suffix) => { + const base = `2026-08-02T10:${String(minute).padStart(2, "0")}`; + return [{ + sessionId, timestamp: `${base}:00.000Z`, type: "tool", toolName: "Bash", + commandText: "node --test test/shared.test.mjs", validationCategory: "node --test", + targetPaths: ["src/shared.mjs"], success: false, evidenceRef: { kind: "fixture", id: `${suffix}-failure` }, + }, { + sessionId, timestamp: `${base}:01.000Z`, type: "tool", toolName: "Edit", + filePath: "src/shared.mjs", evidenceRef: { kind: "fixture", id: `${suffix}-edit` }, + }, { + sessionId, timestamp: `${base}:02.000Z`, type: "tool", toolName: "Bash", + commandText: "node --test test/shared.test.mjs", validationCategory: "node --test", + targetPaths: ["src/shared.mjs"], success: true, evidenceRef: { kind: "fixture", id: `${suffix}-rerun` }, + }]; + }; + return buildTaskLoopSourceCandidate({ + scope: { platform: "qoder", workspace: "/tmp/project" }, + selection: { strategy: "latest-n", eligibleCount: 2, analyzedCount: 2, strata: [] }, + events: [...repairEvents("native-a", 0, "a"), ...repairEvents("native-b", 5, "b")], + }); +} + function attachRecurringWorkflowFriction(source, intent) { for (const episode of source.taskEpisodes) { episode.learningSignals = [{ @@ -329,6 +361,127 @@ function completeReview(source) { }; } +function completeNativeReview(source) { + const decision = completeReview(source); + const sourceRef = { kind: "session-selection", id: "bounded-selection" }; + decision.readerOverview.evidenceRefs = [sourceRef]; + for (const group of [ + decision.repositoryReview.reviewedFrameworks, + decision.repositoryReview.reviewedChecks, + decision.repositoryReview.reviewedSoftwareFluencyCapabilities, + decision.scoreReview.dimensions, + ]) for (const row of group) row.evidenceRefs = [sourceRef]; + const checks = new Map(decision.repositoryReview.reviewedChecks.map((row) => [row.id, row])); + Object.assign(checks.get("lifecycle-repeat-detection"), { + state: "Exercised", + evidenceRefs: source.taskEpisodes.map((episode) => ({ kind: "task-episode", id: episode.id })), + }); + Object.assign(checks.get("loop-engineering"), { + state: "Present", + mechanisms: ["skill"], + candidateOwner: "skill", + ownerSelectionEvidenceRefs: [sourceRef], + }); + decision.repositoryEvidence.diagnosticCoverageReviews = [{ + id: "core-diagnostic-coverage", + status: "covered", + affectedScope: "repository-wide", + summary: "Core diagnostics were reviewed in the bounded native fixture.", + evidenceRefs: [sourceRef], + }]; + const nativePacket = source.repositoryEvidence.learningCaptureDiagnostics.nativeLearningReview.packet; + decision.nativeLearningDecisions = nativePacket.groups.map((group) => ({ + groupRef: group.groupRef, + decision: "match", + patternId: "recurring-correction", + episodeRefs: group.episodeRefs, + evidenceRefs: group.evidenceRefs, + reasonCodes: group.reasonCodes, + })); + return decision; +} + +function fillNativeDecisionTemplate(template, source) { + const decision = template.decision; + const sourceRef = { kind: "session-selection", id: "bounded-selection" }; + decision.sourceCandidate.evidenceRefs = [sourceRef]; + decision.readerOverview.text = "The bounded recurring correction was reviewed without claiming later effectiveness."; + decision.readerOverview.evidenceRefs = [sourceRef]; + for (const group of [ + decision.repositoryReview.reviewedFrameworks, + decision.repositoryReview.reviewedChecks, + decision.repositoryReview.reviewedSoftwareFluencyCapabilities, + ]) for (const row of group) { + row.summary = `${row.id} was reviewed from the bounded decision template.`; + row.evidenceRefs = [sourceRef]; + } + const checks = new Map(decision.repositoryReview.reviewedChecks.map((row) => [row.id, row])); + Object.assign(checks.get("lifecycle-repeat-detection"), { + state: "Exercised", + evidenceRefs: source.taskEpisodes.map((episode) => ({ kind: "task-episode", id: episode.id })), + }); + Object.assign(checks.get("loop-engineering"), { + state: "Present", + mechanisms: ["skill"], + candidateOwner: "skill", + ownerSelectionEvidenceRefs: [sourceRef], + }); + decision.repositoryEvidence.diagnosticCoverageReviews = [{ + id: "core-diagnostic-coverage", + status: "covered", + affectedScope: "repository-wide", + summary: "Core diagnostics were reviewed from the generated decision template.", + evidenceRefs: [sourceRef], + }]; + for (const row of decision.scoreReview.dimensions) { + Object.assign(row, { + score: 59, + confidence: "medium", + reason: `${row.id} was judged from the reviewed evidence boundary.`, + readerSummary: "Reviewed evidence supports this judgment without claiming later effectiveness.", + evidenceRefs: [sourceRef], + }); + } + decision.episodeReviews = source.taskEpisodes.map((episode) => { + const episodeReview = structuredClone(template.optionalShapes.episodeReviews.item); + episodeReview.episodeRef = episode.id; + for (const row of episodeReview.taskUnderstanding) { + Object.assign(row, { + state: "Exercised", + summary: `${row.id} was reviewed for this bounded task episode`, + evidenceRefs: episode.evidenceRefs, + }); + } + episodeReview.validationAssociations = episode.changeSets.flatMap((change) => episode.validationSets + .filter((validation) => validation.status === "passed" + && validation.ordinal > change.lastOrdinal + && change.targetKeys.some((target) => validation.targetKeys.includes(target))) + .slice(0, 1) + .map((validation) => ({ + id: `${episode.id}:reviewed-association`, + changeSetRef: change.id, + validationSetRef: validation.id, + relation: "relevant-after-change", + summary: "The retained validation directly checks the bounded change set", + evidenceRefs: validation.evidenceRefs, + }))); + episodeReview.repairReview = { state: "Unobserved" }; + return episodeReview; + }); + const groups = new Map(template.nativeGroups.map((group) => [group.groupRef, group])); + for (const row of decision.nativeLearningDecisions) { + const group = groups.get(row.groupRef); + Object.assign(row, { + decision: "match", + patternId: "recurring-correction", + episodeRefs: group.episodeRefs, + evidenceRefs: group.evidenceRefs, + reasonCodes: group.supportedReasonCodes, + }); + } + return template; +} + function pendingIntervention() { const ref = (id) => ({ kind: "fixture", id }); return { @@ -448,6 +601,221 @@ test("source review owner merges a complete review without direct source editing assert.equal(reviewed.repositoryEvidence.readerOverview.text, completeReview(source).readerOverview.text); }); +test("native recurring-correction decisions travel through the standard review packet", () => { + const source = nativeCorrectionSource(); + const native = source.repositoryEvidence.learningCaptureDiagnostics.nativeLearningReview; + assert.equal(native.status, "review-required"); + assert.equal(native.packet.groups.length, 1); + + const packet = buildHarnessReviewPacket(source); + assert.equal(packet.schemaVersion, 3); + assert.deepEqual(packet.nativeLearningReview.packet, native.packet); + assert.equal(packet.allowedEvidenceRefs.some((ref) => ref.id === native.packet.groups[0].evidenceRefs[0]), false); + + const missingNativeDecision = completeNativeReview(source); + delete missingNativeDecision.nativeLearningDecisions; + const missingCompiled = compileHarnessLeadDecision(source, packet, missingNativeDecision); + assert.match(missingCompiled.errors.join("; "), /nativeLearningDecisions is required/u); + assert.throws( + () => applyReportSourceReview(source, missingCompiled.review, { packet }), + /nativeLearningReview is required/u, + ); + + const decision = completeNativeReview(source); + const compiled = compileHarnessLeadDecision(source, packet, decision); + assert.deepEqual(compiled.errors, []); + const reviewed = applyReportSourceReview(source, compiled.review, { packet }); + const applied = reviewed.repositoryEvidence.learningCaptureDiagnostics.nativeLearningReview; + assert.equal(applied.status, "reviewed"); + assert.equal(applied.result.matches.length, 1); + assert.equal(reviewed.repositoryEvidence.learningCaptureDiagnostics.recurringIssueCandidates[0].patternId, "recurring-correction"); + const replayPacket = buildHarnessReviewPacket(reviewed); + assert.equal(replayPacket.allowedEvidenceRefs.some((ref) => ref.kind === "native-learning-evidence"), false); + + const stale = structuredClone(compiled.review); + stale.nativeLearningReview.packetDigest = "0".repeat(64); + assert.throws(() => applyReportSourceReview(source, stale, { packet }), /packetDigest|does not match/u); + + const assertStoredTamper = (mutate, pattern) => { + const tampered = structuredClone(reviewed); + mutate(tampered.repositoryEvidence.learningCaptureDiagnostics.nativeLearningReview); + assert.ok(validateHarnessReportSource(tampered).some((error) => pattern.test(error))); + }; + assertStoredTamper((stored) => { + stored.packet.sourceDigest = "0".repeat(64); + stored.packet.packetDigest = nativeLearningReviewPacketDigest(stored.packet); + }, /sourceDigest/u); + assertStoredTamper((stored) => { + stored.packet.groups[0].reasonCodes = ["same-check"]; + stored.packet.packetDigest = nativeLearningReviewPacketDigest(stored.packet); + }, /sourceDigest/u); + assertStoredTamper((stored) => { + stored.packet.groups[0].privatePath = "C:\\Users\\Alice\\token.txt"; + stored.packet.sourceDigest = nativeLearningReviewSourceDigest(stored.packet); + stored.packet.packetDigest = nativeLearningReviewPacketDigest(stored.packet); + }, /groups do not match/u); + assertStoredTamper((stored) => { stored.review.decisions[0].reasonCodes = ["same-check"]; }, /correction reason/u); + assertStoredTamper((stored) => { + stored.review.decisions[0].reasonCodes = ["same-check", "same-check-repair"]; + }, /matches do not match/u); + assertStoredTamper((stored) => { stored.review.decisions[0].evidenceRefs.push(stored.review.decisions[0].evidenceRefs[0]); }, /distinct packet evidence refs/u); + assertStoredTamper((stored) => { stored.review.decisions[0].unexpected = true; }, /unsupported field/u); + assertStoredTamper((stored) => { stored.result.matches = []; }, /matches/u); + assertStoredTamper((stored) => { stored.result.matches[0].privatePath = "C:\\Users\\Alice\\token.txt"; }, /matches do not match/u); + assertStoredTamper((stored) => { + stored.result.learningLoop.candidates[0].privatePath = "C:\\Users\\Alice\\token.txt"; + }, /candidates\[0\] has unsupported field: privatePath/u); + assertStoredTamper((stored) => { + stored.result.learningLoop.candidates[0].priorityScore += 1; + stored.result.learningLoop.candidates[0].currentCost.toolCalls += 1; + }, /deterministic packet-bound reconstruction/u); + assertStoredTamper((stored) => { stored.result.status = "effectiveness-proven"; }, /status must be reviewed/u); +}); + +test("public source-review CLI completes create, decision, and confirmed apply", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "better-harness-source-review-")); + const sourcePath = path.join(root, "report.source.json"); + const packetPath = path.join(root, "review.packet.json"); + const decisionPath = path.join(root, "lead.decision.json"); + const reviewPath = path.join(root, "review.json"); + const cli = path.resolve("scripts/better-harness.mjs"); + const run = (args) => spawnSync(process.execPath, [cli, "harness", "source-review", ...args, "--json"], { + cwd: process.cwd(), + encoding: "utf8", + }); + try { + const source = nativeCorrectionSource(); + await writeFile(sourcePath, `${JSON.stringify(source, null, 2)}\n`); + + const created = run([ + "create", "--source", sourcePath, "--packet", packetPath, "--decision", decisionPath, + ]); + assert.equal(created.status, 0, created.stderr || created.stdout); + assert.equal(created.stderr, ""); + const createPayload = JSON.parse(created.stdout); + assert.equal(createPayload.status, "packet-created"); + assert.equal(createPayload.decisionRequired, true); + assert.equal(createPayload.decisionTemplateCreated, true); + assert.equal(created.stdout.includes(root), false, "machine output must not echo local absolute paths"); + const template = JSON.parse(await readFile(decisionPath, "utf8")); + assert.equal(template.kind, "harness-lead-decision-template"); + assert.deepEqual(template.required.dimensions, template.decision.scoreReview.dimensions.map((row) => row.id)); + assert.deepEqual(template.nativeGroups.map((group) => group.groupRef), template.decision.nativeLearningDecisions.map((row) => row.groupRef)); + assert.equal(template.decision.episodeReviews, undefined); + assert.equal(template.decision.deliveryReviews, undefined); + assert.ok(template.optionalShapes.episodeReviews.item); + const duplicateCreate = run([ + "create", "--source", sourcePath, "--packet", packetPath, "--decision", decisionPath, + ]); + assert.equal(duplicateCreate.status, 1); + assert.equal(JSON.parse(duplicateCreate.stdout).code, "EEXIST"); + assert.equal(duplicateCreate.stdout.includes(root), false, "machine errors must not echo local absolute paths"); + const rollbackPacketPath = path.join(root, "rollback.packet.json"); + const decisionConflict = run([ + "create", "--source", sourcePath, "--packet", rollbackPacketPath, "--decision", decisionPath, + ]); + assert.equal(decisionConflict.status, 1); + await assert.rejects(readFile(rollbackPacketPath, "utf8"), { code: "ENOENT" }); + + const rawDecision = completeNativeReview(source); + rawDecision.privatePath = "E:\\private\\alice\\raw-decision.json"; + await writeFile(decisionPath, `${JSON.stringify(rawDecision, null, 2)}\n`); + const rawDecisionResult = run([ + "decision", "--source", sourcePath, "--packet", packetPath, + "--decision", decisionPath, "--review", reviewPath, + ]); + assert.equal(rawDecisionResult.status, 1); + const rawDecisionPayload = JSON.parse(rawDecisionResult.stdout); + assert.equal(rawDecisionPayload.code, "INVALID_LEAD_DECISION_TEMPLATE"); + assert.equal(rawDecisionPayload.errors, undefined); + assert.equal(rawDecisionResult.stdout.includes("E:\\private\\alice"), false); + await assert.rejects(readFile(reviewPath, "utf8"), { code: "ENOENT" }); + + const privateDecision = structuredClone(template); + privateDecision.decision.repositoryReview.reviewedChecks[0].id = "E:\\private\\alice\\invented-check"; + await writeFile(decisionPath, `${JSON.stringify(privateDecision, null, 2)}\n`); + const privateDecisionResult = run([ + "decision", "--source", sourcePath, "--packet", packetPath, + "--decision", decisionPath, "--review", reviewPath, + ]); + assert.equal(privateDecisionResult.status, 1); + assert.equal(privateDecisionResult.stdout.includes("E:\\private\\alice"), false); + assert.equal(JSON.parse(privateDecisionResult.stdout).errors, undefined); + await writeFile(decisionPath, `${JSON.stringify(template, null, 2)}\n`); + const incomplete = run([ + "decision", "--source", sourcePath, "--packet", packetPath, + "--decision", decisionPath, "--review", reviewPath, + ]); + assert.equal(incomplete.status, 1); + const incompletePayload = JSON.parse(incomplete.stdout); + assert.equal(incompletePayload.code, "INVALID_LEAD_DECISION"); + assert.equal(incompletePayload.errors, undefined); + await writeFile(decisionPath, `${JSON.stringify(fillNativeDecisionTemplate(template, source), null, 2)}\n`); + + const compiled = run([ + "decision", "--source", sourcePath, "--packet", packetPath, + "--decision", decisionPath, "--review", reviewPath, + ]); + assert.equal(compiled.status, 0, compiled.stderr || compiled.stdout); + assert.equal(compiled.stderr, ""); + assert.equal(JSON.parse(compiled.stdout).status, "decision-compiled"); + assert.equal(compiled.stdout.includes(root), false, "machine output must not echo local absolute paths"); + + const unconfirmed = run([ + "apply", "--source", sourcePath, "--packet", packetPath, "--review", reviewPath, + ]); + assert.equal(unconfirmed.status, 1); + assert.equal(unconfirmed.stderr, ""); + assert.equal(JSON.parse(unconfirmed.stdout).code, "APPLY_CONFIRMATION_REQUIRED"); + assert.equal( + JSON.parse(await readFile(sourcePath, "utf8")).repositoryEvidence.learningCaptureDiagnostics.nativeLearningReview.status, + "review-required", + ); + + const applied = run([ + "apply", "--source", sourcePath, "--packet", packetPath, "--review", reviewPath, "--yes", + ]); + const applyDiagnostic = applied.status === 0 ? "" : spawnSync(process.execPath, [ + cli, "harness", "source-review", "apply", "--source", sourcePath, + "--packet", packetPath, "--review", reviewPath, "--yes", + ], { cwd: process.cwd(), encoding: "utf8" }).stderr; + assert.equal(applied.status, 0, applyDiagnostic || applied.stderr || applied.stdout); + assert.equal(applied.stderr, ""); + const applyPayload = JSON.parse(applied.stdout); + assert.equal(applyPayload.status, "review-applied"); + assert.equal(applyPayload.nativeStatus, "reviewed"); + assert.equal(applied.stdout.includes(root), false, "machine output must not echo local absolute paths"); + assert.equal( + JSON.parse(await readFile(sourcePath, "utf8")).repositoryEvidence.learningCaptureDiagnostics.nativeLearningReview.status, + "reviewed", + ); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("source-review read errors never echo selected absolute paths", () => { + const cli = path.resolve("scripts/better-harness.mjs"); + const privatePath = "E:\\private\\alice\\missing.source.json"; + const args = [ + "harness", "source-review", "create", "--source", privatePath, + "--packet", "E:\\private\\alice\\packet.json", + "--decision", "E:\\private\\alice\\decision.json", + ]; + const machine = spawnSync(process.execPath, [cli, ...args, "--json"], { encoding: "utf8" }); + assert.equal(machine.status, 1); + assert.equal(machine.stderr, ""); + assert.equal(JSON.parse(machine.stdout).code, "INVALID_REPORT_SOURCE"); + assert.equal(`${machine.stdout}${machine.stderr}`.includes("E:\\private\\alice"), false); + assert.equal(JSON.parse(machine.stdout).errors, undefined); + + const human = spawnSync(process.execPath, [cli, ...args], { encoding: "utf8" }); + assert.equal(human.status, 1); + assert.equal(human.stdout, ""); + assert.equal(`${human.stdout}${human.stderr}`.includes("E:\\private\\alice"), false); + assert.match(human.stderr, /report source is not a readable local JSON file/u); +}); + test("an adequate inspected window can retain an explicit no-candidate Learning Capture result", () => { const source = sourceWithAdequateNoCandidateWindow(); assert.equal(source.repositoryEvidence.workflowDemandDiagnostics.repeatedCandidates.length, 0); diff --git a/test/scripts-refactor-contract.test.mjs b/test/scripts-refactor-contract.test.mjs index d29dc53..96772a7 100644 --- a/test/scripts-refactor-contract.test.mjs +++ b/test/scripts-refactor-contract.test.mjs @@ -107,17 +107,17 @@ test("scripts refactor contract freezes machine-readable CLI output", () => { { label: "command inventory", args: ["commands", "--json"], - sha256: "fb100fe15aecbe194487db218dc3e65e0ec26be319a28f2975a1c0d4f6bb3610", + sha256: "c5fb49bd619ba8337b103b33db7f205592ca55b956686fe53466604f697087aa", }, { label: "OpenCLI schema", args: ["schema"], - sha256: "df90e94af6c86350331b3470c9dd9d6dd0b73456599302bd386af14e36088d1f", + sha256: "0283010f3ce112ab6b755ca1e0ea01cb76fc073d9fb2f8ddeeb4a64520459a2c", }, { label: "Harness command description", args: ["command", "describe", "harness", "--json"], - sha256: "a2562f1719b53a24c9e1a39d9fe32924ad5a1c38253177a2dc6f5970461b9303", + sha256: "aa7aeb8d28360da25a83dc65b9fea583b9abf7ae2e665c307cf6c838937dd1d4", }, ];