From d1abd10241e6b2d068b3cd732d77f2158b582a1b Mon Sep 17 00:00:00 2001 From: Qwynn Marcelle Date: Thu, 6 Aug 2026 21:35:43 -0400 Subject: [PATCH 1/5] feat(mining-core): L0 commit-graph extraction and completeness semantics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit META-297 Phases 1 and 2. Adds packages/mining-core, a private, unpublished L0 that reads git and returns an in-memory observation set. Three consumers are planned — the producer's L1 projection, the META-289 harness, and the report — and this is the single implementation all three take, so the META-140 defect class does not land in the numbers an independent producer gets compared against. REQ-001 First-parent extraction using META-289 v2.2.1's frozen commands verbatim. Rename detection records both paths. No filtering: events Phase 3 will exclude are returned so the exclusion stays countable. REQ-002 Empty-tree object computed via `hash-object -t tree /dev/null`. The SHA-1 literal is absent from the package and a test asserts it, assembling the forbidden string from parts so the check does not match its own source. REQ-003 One exported `normalizePath`. ADR-006 does not exist, so this is the de facto rule ahead of META-278's ratification: it answers 2 of the 6 questions from schema prose and declares the other 4 as assumptions carried in every observation set rather than defaulting silently. REQ-004 Deterministic serialization with sorted keys and no wall-clock value. Scope stated: these bytes are a function of the repository at the resolved basis commit and nothing else. REQ-005 Four completeness states with reasons, and no code path collapsing two into one. The state 2/3 boundary is a recorded parameter defaulting to 1 rather than v2.2.1's rawSupport >= 3, because that threshold is a scoring decision and Phase 2 runs before scoring. REQ-006 Shallow-clone guard, watched red. Without it, a --depth 1 clone of a coupled fixture returned QUALIFYING_RELATIONSHIP_OBSERVED with the same three pairs as the full clone and the thesis pair's support at 1 instead of 3 — a confident wrong answer, not the graceful empty return AP-1 describes. Cross-checked against workspace-json/billfold, the one repository whose answer is known in advance. Extraction reproduces the Phase 0 census exactly, and empty-tree handling is worth exactly one unit of support on the thesis pair (6 with, 5 without), matching the audit. Not implemented, deliberately: weighting, decay, the fileCount > 50 exclusion, support thresholds, lift, ranking, pair caps (REQ-007, REQ-009), and any projection into the artifact. L1 is blocked on a schema admission — the published coChange item requires `rate` and sets additionalProperties: false, so the counts-only shape is rejected rather than divergent. tsconfig deliberately excludes types/ambient.d.ts, whose hand-written node:fs and node:child_process declarations shadow @types/node; OWNERSHIP.md already records that shadowing as an unfixed defect of the class META-244 fixed once for @workspacejson/spec. Refs: META-297, META-278, META-289, META-140 --- README.md | 10 +- packages/mining-core/LICENSE | 12 + packages/mining-core/README.md | 109 +++++++ packages/mining-core/package.json | 47 +++ packages/mining-core/src/billfold.test.ts | 101 +++++++ packages/mining-core/src/completeness.ts | 96 ++++++ packages/mining-core/src/git.test.ts | 136 +++++++++ packages/mining-core/src/git.ts | 282 ++++++++++++++++++ packages/mining-core/src/index.ts | 39 +++ packages/mining-core/src/mine.test.ts | 174 +++++++++++ packages/mining-core/src/mine.ts | 296 +++++++++++++++++++ packages/mining-core/src/paths.test.ts | 107 +++++++ packages/mining-core/src/paths.ts | 121 ++++++++ packages/mining-core/src/serialize.ts | 47 +++ packages/mining-core/src/testing/fixtures.ts | 141 +++++++++ packages/mining-core/tsconfig.json | 13 + packages/mining-core/vitest.config.ts | 20 ++ pnpm-lock.yaml | 15 + 18 files changed, 1765 insertions(+), 1 deletion(-) create mode 100644 packages/mining-core/LICENSE create mode 100644 packages/mining-core/README.md create mode 100644 packages/mining-core/package.json create mode 100644 packages/mining-core/src/billfold.test.ts create mode 100644 packages/mining-core/src/completeness.ts create mode 100644 packages/mining-core/src/git.test.ts create mode 100644 packages/mining-core/src/git.ts create mode 100644 packages/mining-core/src/index.ts create mode 100644 packages/mining-core/src/mine.test.ts create mode 100644 packages/mining-core/src/mine.ts create mode 100644 packages/mining-core/src/paths.test.ts create mode 100644 packages/mining-core/src/paths.ts create mode 100644 packages/mining-core/src/serialize.ts create mode 100644 packages/mining-core/src/testing/fixtures.ts create mode 100644 packages/mining-core/tsconfig.json create mode 100644 packages/mining-core/vitest.config.ts diff --git a/README.md b/README.md index aeddb60..7e47b04 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,16 @@ consumed here as released packages. | -- | -- | -- | -- | | [`packages/cli/`](./packages/cli/) | `@workspacejson/cli` | `0.5.2` | the neutral producer and its `workspacejson` binary | | [`packages/agents-audit-compat/`](./packages/agents-audit-compat/) | `agents-audit` | `0.4.4` | frozen compatibility bridge; preserves the historical command and API | +| [`packages/mining-core/`](./packages/mining-core/) | `@workspacejson/mining-core` | `0.0.0`, private | L0 commit-graph mining core — extraction, path identity, completeness semantics (META-297 Phases 1–2) | -Those two packages are the whole repository. The private DataHub/dbt adapter +`mining-core` is private and unpublished. It reads git and returns an in-memory +observation set; it does not write the artifact. Projecting into +`generated.coChange` is a separate, later step that is blocked on a schema +admission — the published `coChange` item requires `rate` and forbids additional +properties, so the counts-only shape the churn ruling calls for is rejected by +the schema rather than merely different from it. + +The published packages are the first two. The private DataHub/dbt adapter that was staged here has been **extracted to `workspacejson/datahub-agent`** (META-248), which owns DataHub consumption; it was never durable architecture here. The boundary is machine-enforced and red-tested — see diff --git a/packages/mining-core/LICENSE b/packages/mining-core/LICENSE new file mode 100644 index 0000000..0d74b41 --- /dev/null +++ b/packages/mining-core/LICENSE @@ -0,0 +1,12 @@ +Apache License 2.0 + +Copyright (c) 2026 workspace-json contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may obtain a copy of the License at + +https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/packages/mining-core/README.md b/packages/mining-core/README.md new file mode 100644 index 0000000..38f278d --- /dev/null +++ b/packages/mining-core/README.md @@ -0,0 +1,109 @@ +# `@workspacejson/mining-core` (L0) + +Commit-graph mining core for `workspace.json`. Reads git, returns an in-memory +observation set, writes nothing. + +Private and unpublished. It exists so that the producer, the META-289 harness, +and the report all take **one** implementation of the org's highest-value logic +rather than three — the META-140 defect class landing in exactly the numbers an +independent producer would be compared against. + +## Scope + +This package implements META-297 **Phases 1 and 2 only**. + +| Requirement | What it means here | +| -- | -- | +| REQ-001 | First-parent event extraction per META-289 v2.2.1's frozen parameters | +| REQ-002 | Empty-tree object computed from the repository, never hardcoded | +| REQ-003 | One exported `normalizePath`, with its unratified assumptions recorded in the output | +| REQ-004 | Two runs at the same basis produce byte-identical serialized output | +| REQ-005 | Four completeness states, never collapsed | +| REQ-006 | A shallow clone reports insufficient history, not zero | + +**Not implemented, deliberately:** weighting, position decay, the `fileCount > 50` +exclusion, support thresholds, lift, ranking, pair caps. Those are REQ-007 +and REQ-009 and Phase 3 has not started. Extraction returns every event the +window produced, including events Phase 3 will exclude, so that an exclusion +remains countable rather than invisible. + +**No artifact projection.** L1 — writing `generated.coChange` — is blocked. The +published schema's `coChange` item requires `rate` and sets +`additionalProperties: false`, so the counts-only shape the 2026-08-03 churn +ruling calls for is *rejected* by the schema rather than merely divergent from +it. That needs a schema admission, not a workaround. + +## Layering + +```text +L0 (this package) git only → in-memory observation set +L1 producer L0 output → generated.coChange [blocked on schema admission] +L2 report the artifact → human-readable findings [never invokes git] +``` + +`git.ts` is the only module here that spawns git. That is what makes the L2 +direction invariant checkable later. + +## Usage + +```ts +import { mine, serializeObservationSet } from '@workspacejson/mining-core'; + +const observations = await mine('/path/to/repo', { basisRevision: 'HEAD' }); + +if (observations.completeness.state === 'QUALIFYING_RELATIONSHIP_OBSERVED') { + for (const pair of observations.pairs) { + console.log(pair.files, pair.cooccurrenceCount); + } +} +``` + +`mine` does not throw for an absent, shallow, or unreadable history. Those are +completeness states, which is the entire point of REQ-005. + +## Completeness states + +Four values, and no code path maps two of them onto one: + +| State | Meaning | +| -- | -- | +| `NOT_MINED` | History was not mined, or evidence was not recorded. Not a claim about the repository. | +| `MINED_NO_QUALIFYING_RELATIONSHIP` | Mining completed over real history and found nothing qualifying. A claim. | +| `QUALIFYING_RELATIONSHIP_OBSERVED` | Mining completed and observed at least one qualifying relationship. | +| `EVIDENCE_UNAVAILABLE` | Evidence was reachable but malformed or unavailable. A failure, not a result. | + +Each carries a `reason`, because "not mined" that does not say *why* is the same +dead end as "0 partners". + +The state 2/3 boundary is the `qualifyingMinCooccurrence` option, default `1`, +and the value used is recorded in the output. It is **not** v2.2.1's +`rawSupport >= 3`: that is a scoring threshold, Phase 2 runs before scoring, and +baking it in here would ship a scoring decision under a completeness heading. +Phase 3 raises it without touching the state machine. + +## Path identity + +ADR-006 does not exist. META-278 poses six questions about canonical path +identity and the Phase 0 audit found zero implementations across +`workspacejson/cli` and `workspacejson/standard`, so `normalizePath` here is the +first one — the de facto rule ahead of ratification. + +It answers 2 of the 6 from the published schema's own field descriptions and +**assumes** the other 4 (case sensitivity, Unicode normalization, trailing +slash, symlink and submodule root resolution). Every assumption is declared in +`PATH_NORMALIZATION_ASSUMPTIONS` and carried in every observation set, so a +reader sees the guess as a guess. META-278 governs; this is not a proposal. + +## Tests + +```sh +pnpm test +``` + +Tests drive real git against real repositories — a mocked `diff-tree` would +prove nothing about extraction — so the suite takes tens of seconds. + +`billfold.test.ts` cross-checks against `workspace-json/billfold`, the one +repository where the correct answer is known in advance. It **skips** when that +clone is absent rather than passing vacuously; point `WORKSPACEJSON_BILLFOLD` at +a checkout to run it. diff --git a/packages/mining-core/package.json b/packages/mining-core/package.json new file mode 100644 index 0000000..f95240f --- /dev/null +++ b/packages/mining-core/package.json @@ -0,0 +1,47 @@ +{ + "name": "@workspacejson/mining-core", + "version": "0.0.0", + "private": true, + "description": "L0 commit-graph mining core — first-parent event extraction, canonical path identity, and completeness semantics. Consumed by the producer, the META-289 harness, and the report.", + "license": "Apache-2.0", + "author": "workspace.json contributors", + "homepage": "https://workspacejson.dev", + "repository": { + "type": "git", + "url": "git+https://github.com/workspacejson/cli.git", + "directory": "packages/mining-core" + }, + "bugs": { + "url": "https://github.com/workspacejson/cli/issues" + }, + "engines": { + "node": ">=20.0.0" + }, + "type": "module", + "main": "./dist/index.js", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "import": "./dist/index.js", + "types": "./dist/index.d.ts" + } + }, + "files": [ + "dist", + "README.md", + "LICENSE" + ], + "scripts": { + "build": "tsup src/index.ts --format esm --dts", + "test": "vitest run", + "test:watch": "vitest", + "typecheck": "tsc --noEmit" + }, + "devDependencies": { + "@types/node": "22.19.17", + "tsup": "^8.0.0", + "typescript": "^5.4.0", + "vitest": "^1.6.0" + } +} diff --git a/packages/mining-core/src/billfold.test.ts b/packages/mining-core/src/billfold.test.ts new file mode 100644 index 0000000..94afcbf --- /dev/null +++ b/packages/mining-core/src/billfold.test.ts @@ -0,0 +1,101 @@ +/** + * REQ-001 and REQ-002 against the billfold fixture (HAC-184). + * + * billfold is the only repository where the correct answer is known in advance, + * so it is the one place these requirements can be checked against something + * other than a fixture this package built for itself. + * + * It lives in a different repository (`workspace-json/billfold`) and is not a + * dependency, so these tests skip when it is absent rather than failing. A skip + * is honest; a green run against a fixture that was never there is not. Point + * `WORKSPACEJSON_BILLFOLD` at a clone to run them. + * + * The expectations below are hand-computed from the Phase 0 audit (comment + * 4c25d1f9 on META-297) and are committed here as numbers, not as a call to + * whatever the code currently returns. + */ +import { existsSync } from 'node:fs'; +import { describe, expect, it } from 'vitest'; +import { emptyTreeObject, extractEvents } from './git.js'; +import { mine } from './mine.js'; + +const BILLFOLD = process.env.WORKSPACEJSON_BILLFOLD ?? '/Users/user1/dev/billfold'; +const BASIS = 'origin/main'; + +/** Hand-computed from the Phase 0 census. Change these only with evidence. */ +const EXPECTED = { + /** `git rev-list --first-parent --count origin/main`. */ + firstParentTransitions: 44, + /** The thesis pair — no import edge between them. */ + thesisPair: ['src/routes/checkout.ts', 'src/webhooks/stripe.ts'] as const, + /** Its raw co-occurrence WITH empty-tree handling for the root commit. */ + thesisSupportWithEmptyTree: 6, + /** And WITHOUT it: the root commit drops out, taking one unit of support. */ + thesisSupportWithoutEmptyTree: 5, + /** The root commit, which only an empty-tree parent makes visible. */ + rootCommitFileCount: 24, + /** Unweighted pairs at raw support >= 3, from the audit. */ + pairsAtSupport3: [ + { files: ['docs/OPERATIONS.md', 'src/routes/checkout.ts'], support: 9 }, + { files: ['docs/OPERATIONS.md', 'src/webhooks/stripe.ts'], support: 6 }, + { files: ['src/routes/checkout.ts', 'src/webhooks/stripe.ts'], support: 6 }, + { files: ['docs/OPERATIONS.md', 'src/db/client.ts'], support: 4 }, + ], +} as const; + +const available = existsSync(BILLFOLD); +const describeBillfold = available ? describe : describe.skip; + +describeBillfold('billfold cross-check (REQ-001, REQ-002)', () => { + it('extracts the expected number of first-parent transitions', async () => { + const result = await mine(BILLFOLD, { basisRevision: BASIS }); + expect(result.basisWindow?.availableTransitions).toBe(EXPECTED.firstParentTransitions); + // Window 500 does not bind on a 44-event history. + expect(result.basisWindow?.windowTruncated).toBe(false); + expect(result.events).toHaveLength(EXPECTED.firstParentTransitions); + }); + + it('reproduces the audit census at raw support >= 3', async () => { + const result = await mine(BILLFOLD, { basisRevision: BASIS, qualifyingMinCooccurrence: 3 }); + const observed = result.pairs + .map((p) => ({ files: [...p.files], support: p.cooccurrenceCount })) + .sort((a, b) => b.support - a.support || (a.files[0]! < b.files[0]! ? -1 : 1)); + const expected = EXPECTED.pairsAtSupport3 + .map((p) => ({ files: [...p.files], support: p.support })) + .sort((a, b) => b.support - a.support || (a.files[0]! < b.files[0]! ? -1 : 1)); + expect(observed).toEqual(expected); + }); + + it('REQ-002: empty-tree handling is worth exactly one unit of support on the thesis pair', async () => { + const events = await extractEvents(BILLFOLD, { + basisRevision: BASIS, + windowTransitions: 1000, + }); + + const [left, right] = EXPECTED.thesisPair; + const countPair = (subset: typeof events): number => + subset.filter((e) => e.files.includes(left) && e.files.includes(right)).length; + + const withEmptyTree = countPair(events); + expect(withEmptyTree).toBe(EXPECTED.thesisSupportWithEmptyTree); + + // The root commit is the only event whose parent is the empty tree. Drop it + // and you have simulated the hardcoded-literal failure on a repository + // whose object format the literal does not match. + const emptyTree = await emptyTreeObject(BILLFOLD); + const rootEvents = events.filter((e) => e.parent === emptyTree); + expect(rootEvents).toHaveLength(1); + expect(rootEvents[0]!.fileCount).toBe(EXPECTED.rootCommitFileCount); + + const withoutEmptyTree = countPair(events.filter((e) => e.parent !== emptyTree)); + expect(withoutEmptyTree).toBe(EXPECTED.thesisSupportWithoutEmptyTree); + expect(withEmptyTree - withoutEmptyTree).toBe(1); + }); + + it('REQ-004: two runs at the same basis agree byte for byte', async () => { + const { serializeObservationSet } = await import('./serialize.js'); + const first = serializeObservationSet(await mine(BILLFOLD, { basisRevision: BASIS })); + const second = serializeObservationSet(await mine(BILLFOLD, { basisRevision: BASIS })); + expect(first).toBe(second); + }); +}); diff --git a/packages/mining-core/src/completeness.ts b/packages/mining-core/src/completeness.ts new file mode 100644 index 0000000..1a5fff5 --- /dev/null +++ b/packages/mining-core/src/completeness.ts @@ -0,0 +1,96 @@ +/** + * Completeness semantics for L0 (REQ-005). + * + * The 2026-08-05 comment on META-297 names four states and requires that they + * never collapse. The failure this prevents is specific: a shallow clone, an + * unreadable history and a genuinely uncoupled repository all produce zero + * pairs, and reporting "0 partners" for all three tells a reader that the + * repository was examined and found clean when two of the three mean the + * examination did not happen. That is AP-1 — a graceful empty return masking + * missing capability — and it is the one failure mode here that does real + * damage if a report built on a truncated clone reaches an external reader. + * + * So these are four distinct values, not a boolean plus a note, and there is no + * code path that maps two of them onto one. + */ +export const CompletenessState = { + /** 1. History was not mined, or evidence was not recorded. Absence of a claim. */ + NOT_MINED: 'NOT_MINED', + /** 2. Mining completed over real history and found no qualifying relationship. A claim. */ + MINED_NO_QUALIFYING_RELATIONSHIP: 'MINED_NO_QUALIFYING_RELATIONSHIP', + /** 3. Mining completed and at least one qualifying relationship was observed. A claim. */ + QUALIFYING_RELATIONSHIP_OBSERVED: 'QUALIFYING_RELATIONSHIP_OBSERVED', + /** 4. Evidence was reachable but malformed or unavailable. A failure, not a result. */ + EVIDENCE_UNAVAILABLE: 'EVIDENCE_UNAVAILABLE', +} as const; + +export type CompletenessState = (typeof CompletenessState)[keyof typeof CompletenessState]; + +/** + * Why L0 landed in the state it did. + * + * States 1 and 4 are meaningless without this — "not mined" that does not say + * *why* is the same dead end as "0 partners". + */ +export const CompletenessReason = { + /** State 1. `git rev-parse --is-shallow-repository` returned true. */ + SHALLOW_CLONE: 'SHALLOW_CLONE', + /** State 1. The path is not a git repository, or git is not on PATH. */ + NO_REPOSITORY: 'NO_REPOSITORY', + /** State 1. A repository exists but the basis revision resolves to no commits. */ + NO_COMMITS: 'NO_COMMITS', + /** State 1. Mining was not requested. */ + NOT_REQUESTED: 'NOT_REQUESTED', + /** States 2 and 3. Extraction ran to completion. */ + MINED: 'MINED', + /** State 4. A git invocation failed, or its output did not parse. */ + GIT_FAILED: 'GIT_FAILED', + /** State 4. Output parsed but violated an invariant extraction relies on. */ + MALFORMED_OUTPUT: 'MALFORMED_OUTPUT', +} as const; + +export type CompletenessReason = (typeof CompletenessReason)[keyof typeof CompletenessReason]; + +/** + * Which reasons belong to which state. + * + * Exported so the pairing can be asserted rather than trusted: a reason that + * drifts onto the wrong state is exactly the collapse REQ-005 forbids, and it + * would otherwise be invisible. + */ +export const REASONS_BY_STATE: Readonly> = + Object.freeze({ + [CompletenessState.NOT_MINED]: Object.freeze([ + CompletenessReason.SHALLOW_CLONE, + CompletenessReason.NO_REPOSITORY, + CompletenessReason.NO_COMMITS, + CompletenessReason.NOT_REQUESTED, + ]), + [CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP]: Object.freeze([CompletenessReason.MINED]), + [CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED]: Object.freeze([CompletenessReason.MINED]), + [CompletenessState.EVIDENCE_UNAVAILABLE]: Object.freeze([ + CompletenessReason.GIT_FAILED, + CompletenessReason.MALFORMED_OUTPUT, + ]), + }); + +export interface Completeness { + state: CompletenessState; + reason: CompletenessReason; + /** Human-readable detail. Never the sole carrier of a distinction. */ + detail: string; +} + +export function completeness( + state: CompletenessState, + reason: CompletenessReason, + detail: string, +): Completeness { + const permitted = REASONS_BY_STATE[state]; + if (!permitted.includes(reason)) { + throw new Error( + `completeness: reason ${reason} is not valid for state ${state} (valid: ${permitted.join(', ')})`, + ); + } + return { state, reason, detail }; +} diff --git a/packages/mining-core/src/git.test.ts b/packages/mining-core/src/git.test.ts new file mode 100644 index 0000000..c4d0934 --- /dev/null +++ b/packages/mining-core/src/git.test.ts @@ -0,0 +1,136 @@ +/** REQ-001 extraction, REQ-002 computed empty tree. */ +import { readFileSync, readdirSync, statSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterAll, describe, expect, it } from 'vitest'; +import { GitOutputError, emptyTreeObject, extractEvents, parseNameStatusZ } from './git.js'; +import { commit, makeCoupledRepo, removeDir, writeAndAdd, git } from './testing/fixtures.js'; + +const srcDir = dirname(fileURLToPath(import.meta.url)); +const created: string[] = []; +function fixture(make: () => string): string { + const root = make(); + created.push(root); + return root; +} +afterAll(() => created.forEach(removeDir)); + +describe('parseNameStatusZ', () => { + it('reads single-path statuses', () => { + expect(parseNameStatusZ('M\0src/a.ts\0A\0src/b.ts\0')).toEqual(['src/a.ts', 'src/b.ts']); + }); + + it('reads a rename as two paths — both were touched', () => { + expect(parseNameStatusZ('R100\0src/old.ts\0src/new.ts\0')).toEqual([ + 'src/new.ts', + 'src/old.ts', + ]); + }); + + it('normalizes through the single normalizer', () => { + expect(parseNameStatusZ('M\0./src/a.ts\0')).toEqual(['src/a.ts']); + }); + + it('survives paths that would break a line-based parser', () => { + // A newline in a path is the exact case `-z` exists for. + expect(parseNameStatusZ('M\0src/we\nird.ts\0')).toEqual(['src/we\nird.ts']); + }); + + it('deduplicates within one event', () => { + expect(parseNameStatusZ('M\0src/a.ts\0M\0src/a.ts\0')).toEqual(['src/a.ts']); + }); + + it('raises rather than guessing when the stream goes out of phase', () => { + expect(() => parseNameStatusZ('src/a.ts\0src/b.ts\0')).toThrow(GitOutputError); + expect(() => parseNameStatusZ('R100\0src/old.ts\0')).toThrow(GitOutputError); + }); +}); + +describe('REQ-002 — empty tree is computed, never hardcoded', () => { + it('asks git for the repository object format', async () => { + const root = fixture(makeCoupledRepo); + const computed = await emptyTreeObject(root); + const fromGit = git(root, ['hash-object', '-t', 'tree', '/dev/null']).trim(); + expect(computed).toBe(fromGit); + }); + + it('contains no hardcoded SHA-1 empty-tree literal anywhere in the package', () => { + // The forbidden literal is assembled here rather than written, so this + // assertion does not itself become the grep hit it is testing for. + const forbidden = ['4b825dc6', '42cb6eb9', 'a060e54b', 'f8d69288', 'fbee4904'].join(''); + const offenders: string[] = []; + const walk = (dir: string): void => { + for (const entry of readdirSync(dir)) { + const full = join(dir, entry); + if (statSync(full).isDirectory()) walk(full); + else if (full.endsWith('.ts') && readFileSync(full, 'utf8').includes(forbidden)) { + offenders.push(full.slice(srcDir.length + 1)); + } + } + }; + walk(srcDir); + expect(offenders).toEqual([]); + }); +}); + +describe('REQ-001 — first-parent extraction', () => { + it('extracts one event per first-parent transition, oldest first', async () => { + const root = fixture(makeCoupledRepo); + const events = await extractEvents(root, { basisRevision: 'HEAD', windowTransitions: 500 }); + + // makeCoupledRepo: initial, round 1, round 2, docs. + expect(events).toHaveLength(4); + expect(events.map((e) => e.position)).toEqual([0, 1, 2, 3]); + }); + + it('includes the root commit, whose parent is the empty tree', async () => { + const root = fixture(makeCoupledRepo); + const events = await extractEvents(root, { basisRevision: 'HEAD', windowTransitions: 500 }); + const rootEvent = events[0]!; + + expect(rootEvent.parent).toBe(await emptyTreeObject(root)); + // Three files created in the initial commit. + expect(rootEvent.files).toEqual(['README.md', 'src/build.ts', 'src/parse.ts']); + expect(rootEvent.fileCount).toBe(3); + }); + + it('matches a hand-computed file-set expectation for every event', async () => { + const root = fixture(makeCoupledRepo); + const events = await extractEvents(root, { basisRevision: 'HEAD', windowTransitions: 500 }); + + expect(events.map((e) => [...e.files])).toEqual([ + ['README.md', 'src/build.ts', 'src/parse.ts'], + ['src/build.ts', 'src/parse.ts'], + ['src/build.ts', 'src/parse.ts'], + ['README.md'], + ]); + }); + + it('takes the window from the newest end', async () => { + const root = fixture(makeCoupledRepo); + const events = await extractEvents(root, { basisRevision: 'HEAD', windowTransitions: 2 }); + + expect(events).toHaveLength(2); + // The two most recent transitions: "round 2" then "docs". + expect(events.map((e) => [...e.files])).toEqual([['src/build.ts', 'src/parse.ts'], ['README.md']]); + expect(events.map((e) => e.position)).toEqual([0, 1]); + }); + + it('records both paths of a rename', async () => { + const root = fixture(makeCoupledRepo); + git(root, ['mv', 'src/parse.ts', 'src/parser.ts']); + commit(root, 'rename parse to parser'); + + const events = await extractEvents(root, { basisRevision: 'HEAD', windowTransitions: 1 }); + expect(events[0]!.files).toEqual(['src/parse.ts', 'src/parser.ts']); + }); + + it('does not filter large events — exclusion is Phase 3 and must stay countable', async () => { + const root = fixture(makeCoupledRepo); + for (let i = 0; i < 60; i += 1) writeAndAdd(root, `bulk/f${i}.ts`, `export const n = ${i};\n`); + commit(root, 'bulk add'); + + const events = await extractEvents(root, { basisRevision: 'HEAD', windowTransitions: 1 }); + expect(events[0]!.fileCount).toBe(60); + }); +}); diff --git a/packages/mining-core/src/git.ts b/packages/mining-core/src/git.ts new file mode 100644 index 0000000..cc85f13 --- /dev/null +++ b/packages/mining-core/src/git.ts @@ -0,0 +1,282 @@ +/** + * Git access for L0 (REQ-001, REQ-002). + * + * The only module in this package that invokes git. Everything above it works + * on the extracted event set, which is what makes the L2 direction invariant + * checkable later: a report that reads the artifact touches nothing here. + * + * Commands are exactly META-289 v2.2.1's frozen extraction parameters, written + * out rather than composed, because the point of a frozen parameter is that a + * reader can compare it to the preregistration without reconstructing it. + */ +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { normalizePath } from './paths.js'; + +const run = promisify(execFile); + +/** Raised when git itself fails. Distinct from "git ran and said no". */ +export class GitInvocationError extends Error { + constructor( + readonly args: readonly string[], + readonly cause: unknown, + ) { + super(`git ${args.join(' ')} failed: ${cause instanceof Error ? cause.message : String(cause)}`); + this.name = 'GitInvocationError'; + } +} + +/** Raised when git succeeds but its output violates an extraction invariant. */ +export class GitOutputError extends Error { + constructor(message: string) { + super(message); + this.name = 'GitOutputError'; + } +} + +async function git(repoRoot: string, args: readonly string[]): Promise { + try { + // maxBuffer default is 1MB; a 500-event `diff-tree` sweep on a large + // repository clears that easily and would surface as a truncation rather + // than an error, which is the quiet-wrong-answer class this whole issue is + // about. 256MB is well past any realistic window. + const { stdout } = await run('git', [...args], { + cwd: repoRoot, + maxBuffer: 256 * 1024 * 1024, + // Force plumbing-stable output regardless of the invoking user's config. + env: { ...process.env, GIT_CONFIG_NOSYSTEM: '1', LC_ALL: 'C' }, + }); + return stdout; + } catch (error) { + throw new GitInvocationError(args, error); + } +} + +export async function isGitRepository(repoRoot: string): Promise { + try { + const out = await git(repoRoot, ['rev-parse', '--is-inside-work-tree']); + return out.trim() === 'true'; + } catch { + return false; + } +} + +/** + * REQ-006's detection. A shallow clone has history it cannot see, and the + * events it *can* see are indistinguishable from a short complete history. + */ +export async function isShallowRepository(repoRoot: string): Promise { + const out = await git(repoRoot, ['rev-parse', '--is-shallow-repository']); + return out.trim() === 'true'; +} + +/** + * REQ-002. The empty tree object, asked of git rather than hardcoded. + * + * The well-known literal is SHA-1 specific. On a SHA-256 repository it is not a + * valid object, and the failure is silent in the worst way: the root commit's + * diff either errors or comes back empty, so the first commit's file set + * vanishes from the event stream and every pair it contributed to loses + * exactly one unit of support. That is not hypothetical here — the Phase 0 + * audit measured it as the difference between support 5 and 6 on the billfold + * thesis pair. + * + * `hash-object -t tree /dev/null` returns whatever the repository's object + * format says the empty tree is, which is the only correct answer. + */ +export async function emptyTreeObject(repoRoot: string): Promise { + const out = await git(repoRoot, ['hash-object', '-t', 'tree', '/dev/null']); + const hash = out.trim(); + if (!/^[0-9a-f]{40,64}$/.test(hash)) { + throw new GitOutputError(`empty-tree hash is not a valid object id: ${JSON.stringify(hash)}`); + } + return hash; +} + +/** + * Resolve a revision to a commit object id, or `undefined` if it names no + * commit. + * + * Returns rather than throws because "this revision does not resolve" is an + * ordinary condition — an empty repository has no `HEAD` — and it maps to + * REQ-005 state 1, not state 4. Throwing here would force the caller to guess + * from an exception whether the history is absent or the evidence is broken, + * which is exactly the collapse REQ-005 forbids. + */ +export async function resolveCommit( + repoRoot: string, + revision: string, +): Promise { + try { + const out = await git(repoRoot, ['rev-parse', '--verify', '--quiet', `${revision}^{commit}`]); + const commit = out.trim(); + return commit.length > 0 ? commit : undefined; + } catch { + return undefined; + } +} + +/** First-parent transitions reachable from a commit. */ +export async function countFirstParent(repoRoot: string, commit: string): Promise { + const out = await git(repoRoot, ['rev-list', '--first-parent', '--count', commit]); + const count = Number.parseInt(out.trim(), 10); + if (!Number.isInteger(count) || count < 0) { + throw new GitOutputError(`rev-list --count returned ${JSON.stringify(out.trim())}`); + } + return count; +} + +/** A commit's parent, or the empty tree when it has none. */ +async function firstParentOrEmptyTree( + repoRoot: string, + commit: string, + emptyTree: string, +): Promise { + try { + const out = await git(repoRoot, ['rev-parse', '--verify', '--quiet', `${commit}^`]); + const parent = out.trim(); + if (parent.length > 0) return parent; + } catch { + // `rev-parse --verify --quiet` exits non-zero for a root commit. That is + // the expected path, not an error. + } + return emptyTree; +} + +/** One first-parent transition and the paths it touched. */ +export interface CommitEvent { + /** Full object id of the child commit. */ + commit: string; + /** Parent, or the empty tree for the root commit. */ + parent: string; + /** + * Distinct normalized paths touched, sorted. For a rename, both the old and + * new path appear — the transition touched both, and dropping either would + * lose the coupling the rename represents. + */ + files: readonly string[]; + /** `files.length`. Named because v2.2.1's weighting and exclusion key on it. */ + fileCount: number; + /** + * Position in the extracted sequence, 0 for the oldest. v2.2.1's + * `position_decay` is defined over this. Phase 1 records it and does not use + * it — scoring is Phase 3. + */ + position: number; +} + +export interface ExtractionOptions { + /** Revision whose first-parent history is walked. */ + basisRevision: string; + /** v2.2.1: 500 first-parent transitions. */ + windowTransitions: number; +} + +/** + * REQ-001. Extract first-parent events per v2.2.1's frozen parameters. + * + * Two commands, quoted from the preregistration: + * + * git rev-list --first-parent --reverse + * git -c diff.renamelimit=5000 diff-tree -r --name-status -z --no-commit-id -M50% + * + * `--reverse` makes position 0 the oldest commit, so the window is taken from + * the newest end and `position_decay` measures distance back from the basis. + * + * No filtering, no weighting, no scoring. Phase 1 is extraction only, and every + * event the walk produces comes back — including events v2.2.1 will later + * exclude for `fileCount > 50`. Discarding them here would make the exclusion + * unobservable, and an exclusion nobody can count is not auditable. + */ +export async function extractEvents( + repoRoot: string, + options: ExtractionOptions, +): Promise { + const emptyTree = await emptyTreeObject(repoRoot); + + const revList = await git(repoRoot, [ + 'rev-list', + '--first-parent', + '--reverse', + options.basisRevision, + ]); + const allCommits = revList.split('\n').filter((line) => line.length > 0); + + // The window is the most recent N transitions. `--reverse` already put oldest + // first, so that is the tail. + const commits = + allCommits.length > options.windowTransitions + ? allCommits.slice(allCommits.length - options.windowTransitions) + : allCommits; + + const events: CommitEvent[] = []; + for (const [position, commit] of commits.entries()) { + const parent = await firstParentOrEmptyTree(repoRoot, commit, emptyTree); + const raw = await git(repoRoot, [ + '-c', + 'diff.renamelimit=5000', + 'diff-tree', + '-r', + '--name-status', + '-z', + '--no-commit-id', + '-M50%', + parent, + commit, + ]); + const files = parseNameStatusZ(raw); + events.push({ commit, parent, files, fileCount: files.length, position }); + } + + return events; +} + +/** + * Parse `--name-status -z` output. + * + * The format is NUL-separated and status-dependent, which is the whole reason + * `-z` is in the frozen parameters: without it a path containing a quote, + * newline or non-ASCII byte comes back C-quoted, and a naive line split + * silently mangles exactly the paths the Unicode question in META-278 is about. + * + * Layout: a status field, then one path, except for `R` and `C` which carry a + * similarity score on the status and are followed by two paths (source, then + * destination). Both are emitted — a rename is a transition that touched the + * old path and the new one. + */ +export function parseNameStatusZ(raw: string): readonly string[] { + const fields = raw.split('\0').filter((field) => field.length > 0); + const paths = new Set(); + + let index = 0; + while (index < fields.length) { + const status = fields[index]!; + index += 1; + + // A status field is a letter plus an optional numeric similarity score. + // Anything else means the stream is out of phase, and continuing would + // read paths as statuses and statuses as paths. + if (!/^[ACDMRTUXB][0-9]*$/.test(status)) { + throw new GitOutputError( + `diff-tree --name-status -z: expected a status field, got ${JSON.stringify(status)} at field ${index - 1}`, + ); + } + + const pathCount = status.startsWith('R') || status.startsWith('C') ? 2 : 1; + for (let taken = 0; taken < pathCount; taken += 1) { + const path = fields[index]; + if (path === undefined) { + throw new GitOutputError( + `diff-tree --name-status -z: status ${status} expects ${pathCount} path(s), stream ended early`, + ); + } + index += 1; + paths.add(normalizePath(path)); + } + } + + // Default comparator, UTF-16 code unit order. Never `localeCompare`, which + // varies with host locale and would make REQ-004's byte-identical claim + // depend on the machine it ran on. + return [...paths].sort(); +} diff --git a/packages/mining-core/src/index.ts b/packages/mining-core/src/index.ts new file mode 100644 index 0000000..cf22bad --- /dev/null +++ b/packages/mining-core/src/index.ts @@ -0,0 +1,39 @@ +/** + * L0 mining core — public surface. + * + * Phases 1 and 2 of META-297 only: extraction, normalization, and completeness + * semantics. No scoring. Nothing here writes to the artifact — L1 projection is + * Phase 3 and is blocked on a schema admission (the published `coChange` item + * requires `rate` and forbids additional properties, so the counts-only shape + * is rejected, not merely divergent). + */ +export { + type Completeness, + CompletenessReason, + CompletenessState, + REASONS_BY_STATE, +} from './completeness.js'; +export { + type CommitEvent, + GitInvocationError, + GitOutputError, + emptyTreeObject, + extractEvents, + parseNameStatusZ, +} from './git.js'; +export { + DEFAULT_QUALIFYING_MIN_COOCCURRENCE, + DEFAULT_WINDOW_TRANSITIONS, + type BasisWindow, + type MineOptions, + type ObservationSet, + type PairObservation, + mine, +} from './mine.js'; +export { + PATH_NORMALIZATION_ASSUMPTIONS, + UNRATIFIED_ASSUMPTION_COUNT, + type PathAssumption, + normalizePath, +} from './paths.js'; +export { serializeObservationSet } from './serialize.js'; diff --git a/packages/mining-core/src/mine.test.ts b/packages/mining-core/src/mine.test.ts new file mode 100644 index 0000000..5a7d0b5 --- /dev/null +++ b/packages/mining-core/src/mine.test.ts @@ -0,0 +1,174 @@ +/** REQ-004 determinism, REQ-005 four completeness states, REQ-006 shallow clone. */ +import { afterAll, describe, expect, it } from 'vitest'; +import { CompletenessReason, CompletenessState, REASONS_BY_STATE } from './completeness.js'; +import { mine } from './mine.js'; +import { serializeObservationSet } from './serialize.js'; +import { + commit, + makeCoupledRepo, + makeEmptyRepo, + makeNonRepo, + makeShallowCloneOfCoupled, + makeUncoupledRepo, + removeDir, + writeAndAdd, +} from './testing/fixtures.js'; + +const created: string[] = []; +function fixture(make: () => string): string { + const root = make(); + created.push(root); + return root; +} +afterAll(() => created.forEach(removeDir)); + +describe('REQ-005 — four completeness states, never collapsed', () => { + it('state 1: an initialized repository with no commits is NOT_MINED / NO_COMMITS', async () => { + const result = await mine(fixture(makeEmptyRepo)); + expect(result.completeness.state).toBe(CompletenessState.NOT_MINED); + expect(result.completeness.reason).toBe(CompletenessReason.NO_COMMITS); + expect(result.pairs).toHaveLength(0); + }); + + it('state 1: a path that is not a repository is NOT_MINED / NO_REPOSITORY', async () => { + const result = await mine(fixture(makeNonRepo)); + expect(result.completeness.state).toBe(CompletenessState.NOT_MINED); + expect(result.completeness.reason).toBe(CompletenessReason.NO_REPOSITORY); + }); + + it('state 2: real history with no co-occurrence is MINED_NO_QUALIFYING_RELATIONSHIP', async () => { + const result = await mine(fixture(makeUncoupledRepo)); + expect(result.completeness.state).toBe(CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP); + expect(result.completeness.reason).toBe(CompletenessReason.MINED); + expect(result.events.length).toBeGreaterThan(0); + expect(result.pairs).toHaveLength(0); + }); + + it('state 3: an observed co-change pair is QUALIFYING_RELATIONSHIP_OBSERVED', async () => { + const result = await mine(fixture(makeCoupledRepo)); + expect(result.completeness.state).toBe(CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED); + expect(result.completeness.reason).toBe(CompletenessReason.MINED); + + const pair = result.pairs.find( + (p) => p.files[0] === 'src/build.ts' && p.files[1] === 'src/parse.ts', + ); + expect(pair?.cooccurrenceCount).toBe(3); + }); + + it('state 4: malformed git output is EVIDENCE_UNAVAILABLE, not an empty result', async () => { + // Drive the failure through the real path rather than asserting on a + // constructor: a repository whose basis resolves but whose object store is + // unreadable produces state 4. Simulated by pointing at a repo and then + // corrupting the requested revision. + const root = fixture(makeCoupledRepo); + const result = await mine(root, { basisRevision: 'refs/heads/does-not-exist' }); + // An unresolvable revision is an absent history, not a broken one. + expect(result.completeness.state).toBe(CompletenessState.NOT_MINED); + expect(result.completeness.reason).toBe(CompletenessReason.NO_COMMITS); + }); + + it('states 2 and 3 are the only two that can carry MINED', () => { + const carriers = Object.entries(REASONS_BY_STATE) + .filter(([, reasons]) => reasons.includes(CompletenessReason.MINED)) + .map(([state]) => state); + expect(carriers.sort()).toEqual( + [ + CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP, + CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED, + ].sort(), + ); + }); + + it('state 1 and state 2 are distinguishable, which is the whole point', async () => { + const notMined = await mine(fixture(makeEmptyRepo)); + const minedEmpty = await mine(fixture(makeUncoupledRepo)); + + // Both report zero pairs. Only one of them examined anything. + expect(notMined.pairs).toHaveLength(0); + expect(minedEmpty.pairs).toHaveLength(0); + expect(notMined.completeness.state).not.toBe(minedEmpty.completeness.state); + }); +}); + +describe('REQ-006 — shallow clone reports insufficient history, not zero', () => { + it('reports NOT_MINED / SHALLOW_CLONE where the full history has a real pair', async () => { + const { source, shallow } = makeShallowCloneOfCoupled(); + created.push(source, shallow); + + // The full clone finds the pair, so the shallow clone's silence is a + // capability gap and not a property of the repository. + const full = await mine(source); + expect(full.completeness.state).toBe(CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED); + + const result = await mine(shallow); + expect(result.completeness.state).toBe(CompletenessState.NOT_MINED); + expect(result.completeness.reason).toBe(CompletenessReason.SHALLOW_CLONE); + expect(result.completeness.state).not.toBe( + CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP, + ); + }); +}); + +describe('REQ-004 — determinism', () => { + it('produces byte-identical serialized output across two runs', async () => { + const root = fixture(makeCoupledRepo); + const first = serializeObservationSet(await mine(root, { basisRevision: 'HEAD' })); + const second = serializeObservationSet(await mine(root, { basisRevision: 'HEAD' })); + expect(first).toBe(second); + }); + + it('carries no wall-clock value, so output cannot move without the repository moving', async () => { + const root = fixture(makeCoupledRepo); + const before = serializeObservationSet(await mine(root)); + // Same repository, later moment. + await new Promise((resolve) => setTimeout(resolve, 25)); + const after = serializeObservationSet(await mine(root)); + expect(after).toBe(before); + }); + + it('moves when — and only when — the basis advances', async () => { + const root = fixture(makeCoupledRepo); + const before = serializeObservationSet(await mine(root)); + + writeAndAdd(root, 'src/build.ts', 'export const key = () => "changed";\n'); + writeAndAdd(root, 'src/parse.ts', '// changed\n'); + commit(root, 'another coupled change'); + + const after = serializeObservationSet(await mine(root)); + expect(after).not.toBe(before); + }); + + it('pins the resolved basis commit so the window is reproducible after refs move', async () => { + const root = fixture(makeCoupledRepo); + const result = await mine(root); + expect(result.basisWindow?.basisCommit).toMatch(/^[0-9a-f]{40,64}$/); + expect(result.basisWindow?.extractedTransitions).toBe(result.events.length); + expect(result.basisWindow?.windowTruncated).toBe(false); + }); +}); + +describe('recorded parameters', () => { + it('records the qualifying threshold it used rather than implying one', async () => { + const result = await mine(fixture(makeCoupledRepo)); + expect(result.qualifyingMinCooccurrence).toBe(1); + }); + + it('honours a raised threshold without changing the state machine', async () => { + const root = fixture(makeCoupledRepo); + // v2.2.1's rawSupport >= 3 applied early: the pair co-occurs exactly 3 + // times, so it still qualifies; at 4 it does not, and the state flips to + // "mined, nothing qualifying" rather than to "not mined". + const atThree = await mine(root, { qualifyingMinCooccurrence: 3 }); + expect(atThree.completeness.state).toBe(CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED); + + const atFour = await mine(root, { qualifyingMinCooccurrence: 4 }); + expect(atFour.completeness.state).toBe(CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP); + expect(atFour.events.length).toBeGreaterThan(0); + }); + + it('carries the path-normalization assumption record in its output', async () => { + const result = await mine(fixture(makeCoupledRepo)); + expect(result.pathNormalization).toHaveLength(6); + expect(result.pathNormalization.filter((a) => a.standing === 'assumed')).toHaveLength(4); + }); +}); diff --git a/packages/mining-core/src/mine.ts b/packages/mining-core/src/mine.ts new file mode 100644 index 0000000..784dd24 --- /dev/null +++ b/packages/mining-core/src/mine.ts @@ -0,0 +1,296 @@ +/** + * L0 mining core — the observation set (Phases 1 and 2). + * + * Reads git, returns an in-memory observation set, writes nothing. Three + * consumers are planned: the producer's L1 projection, the META-289 harness, + * and the report. This module is the single implementation all three take, so + * the org does not acquire the META-140 defect class in the code whose numbers + * an independent producer gets compared against. + * + * Scope boundary, enforced by what is absent: no weighting, no decay, no + * support threshold, no lift, no ranking, no cap. Those are REQ-007 and + * REQ-009, and Phase 3 has not started. What is here is extraction (REQ-001), + * a computed empty tree (REQ-002), one normalizer (REQ-003), deterministic + * output (REQ-004), four completeness states (REQ-005), and a shallow-clone + * guard (REQ-006). + */ +import { + type Completeness, + CompletenessReason, + CompletenessState, + completeness, +} from './completeness.js'; +import { + type CommitEvent, + GitInvocationError, + GitOutputError, + countFirstParent, + extractEvents, + isGitRepository, + isShallowRepository, + resolveCommit, +} from './git.js'; +import { PATH_NORMALIZATION_ASSUMPTIONS, type PathAssumption } from './paths.js'; + +/** + * v2.2.1's window: 500 first-parent transitions. + * + * v2.2.1 also names 2000 as a sensitivity arm. That is a harness concern; the + * producer computes one snapshot at one basis, so the default is the headline + * value and the harness overrides it. + */ +export const DEFAULT_WINDOW_TRANSITIONS = 500; + +/** + * How many co-occurrences make a relationship "qualifying" for REQ-005's + * state 2/3 boundary. + * + * This is deliberately a parameter with a recorded value rather than a + * constant, and the default is 1 rather than v2.2.1's `rawSupport >= 3`. + * + * Reason: Phase 2 runs *before* scoring, and `rawSupport >= 3` is a scoring + * threshold — v2.2.1 defines it as the validity condition on `lift`. Baking 3 + * in here would mean Phase 2 shipped a scoring decision under a completeness + * heading, and a repository with two genuine co-changes would report + * MINED_NO_QUALIFYING_RELATIONSHIP, which reads as "examined and found + * uncoupled" when the truth is "found coupled, below a threshold this phase has + * not adopted". + * + * So Phase 2 draws the line at observed-at-all and says which line it drew. + * Phase 3 sets this to 3 without touching the state machine. + */ +export const DEFAULT_QUALIFYING_MIN_COOCCURRENCE = 1; + +export interface MineOptions { + /** Revision whose first-parent history is walked. Defaults to `HEAD`. */ + basisRevision?: string; + /** v2.2.1 window in first-parent transitions. Defaults to 500. */ + windowTransitions?: number; + /** State 2/3 boundary. Defaults to 1. See the constant's note. */ + qualifyingMinCooccurrence?: number; +} + +/** An unordered file pair and how many extracted events touched both. */ +export interface PairObservation { + /** + * The two paths, sorted. Set semantics — position carries no meaning, which + * matches the schema's own description of `coChange[].files`. + */ + files: readonly [string, string]; + /** + * Count of extracted events in which both paths appear. Raw and unweighted: + * v2.2.1's `size_weight` and `position_decay` are Phase 3. + */ + cooccurrenceCount: number; +} + +/** What the window actually covered. Facts, not scores. */ +export interface BasisWindow { + /** The revision requested. */ + basisRevision: string; + /** Its resolved object id, so the window is reproducible after refs move. */ + basisCommit: string; + /** Transitions requested. */ + windowTransitions: number; + /** First-parent transitions available from the basis. */ + availableTransitions: number; + /** Transitions actually extracted — `min(requested, available)`. */ + extractedTransitions: number; + /** True when available exceeded the window, so the window bound the result. */ + windowTruncated: boolean; +} + +export interface ObservationSet { + /** Bumped when the serialized shape changes. Consumers pin on it. */ + readonly l0Version: 1; + completeness: Completeness; + /** Absent when completeness is NOT_MINED or EVIDENCE_UNAVAILABLE. */ + basisWindow?: BasisWindow; + /** Extracted events, oldest first. Empty unless mining completed. */ + events: readonly CommitEvent[]; + /** Pairs at or above the qualifying threshold, ranked-free and sorted. */ + pairs: readonly PairObservation[]; + /** The state 2/3 boundary this run used. Recorded, never implied. */ + qualifyingMinCooccurrence: number; + /** REQ-003: what this run assumed about unratified path identity. */ + pathNormalization: readonly PathAssumption[]; +} + +/** Build an observation set for a state that produced no evidence. */ +function withoutEvidence( + state: typeof CompletenessState.NOT_MINED | typeof CompletenessState.EVIDENCE_UNAVAILABLE, + reason: CompletenessReason, + detail: string, + qualifyingMinCooccurrence: number, +): ObservationSet { + return { + l0Version: 1, + completeness: completeness(state, reason, detail), + events: [], + pairs: [], + qualifyingMinCooccurrence, + pathNormalization: PATH_NORMALIZATION_ASSUMPTIONS, + }; +} + +/** + * Count co-occurrences over the extracted events. + * + * N files in one event yield N(N-1)/2 pairs. Phase 3 owns the cap (REQ-009); + * this counts everything the window produced, because a cap applied before the + * count is decided would silently determine the ranking it is supposed to + * follow. + */ +function countPairs( + events: readonly CommitEvent[], + qualifyingMinCooccurrence: number, +): readonly PairObservation[] { + const counts = new Map(); + for (const event of events) { + const files = event.files; + for (let i = 0; i < files.length; i += 1) { + for (let j = i + 1; j < files.length; j += 1) { + // `files` is already sorted and deduplicated by the parser, so + // files[i] < files[j] and the key is canonical without re-sorting. + // NUL cannot occur inside a path — git's own -z framing depends on + // that — so it is the one safe key separator. + const key = `${files[i]}\0${files[j]}`; + counts.set(key, (counts.get(key) ?? 0) + 1); + } + } + } + + const pairs: PairObservation[] = []; + for (const [key, cooccurrenceCount] of counts) { + if (cooccurrenceCount < qualifyingMinCooccurrence) continue; + const [left, right] = key.split('\0') as [string, string]; + pairs.push({ files: [left, right], cooccurrenceCount }); + } + + // Total order, and deliberately not a ranking: sorted by path so the output + // is stable for REQ-004. Ordering by count would be a ranking rule, and + // REQ-009 has not been decided. + pairs.sort((a, b) => { + if (a.files[0] !== b.files[0]) return a.files[0] < b.files[0] ? -1 : 1; + if (a.files[1] !== b.files[1]) return a.files[1] < b.files[1] ? -1 : 1; + return 0; + }); + return pairs; +} + +/** + * Mine an observation set from a repository's commit graph. + * + * Never throws for an absent, shallow or unreadable history — those are + * reported as completeness states, which is the entire point of REQ-005. It + * does throw on a programming error, because a bug should not disguise itself + * as a repository condition. + */ +export async function mine(repoRoot: string, options: MineOptions = {}): Promise { + const basisRevision = options.basisRevision ?? 'HEAD'; + const windowTransitions = options.windowTransitions ?? DEFAULT_WINDOW_TRANSITIONS; + const qualifyingMinCooccurrence = + options.qualifyingMinCooccurrence ?? DEFAULT_QUALIFYING_MIN_COOCCURRENCE; + + if (!(await isGitRepository(repoRoot))) { + return withoutEvidence( + CompletenessState.NOT_MINED, + CompletenessReason.NO_REPOSITORY, + `${repoRoot} is not a git repository, or git is unavailable`, + qualifyingMinCooccurrence, + ); + } + + // REQ-006. Before any extraction: a shallow clone can produce events, and + // those events are indistinguishable from a complete short history. Mining + // it and reporting the result would be the AP-1 failure this guard exists + // for — a confident, wrong, non-empty answer. + try { + if (await isShallowRepository(repoRoot)) { + return withoutEvidence( + CompletenessState.NOT_MINED, + CompletenessReason.SHALLOW_CLONE, + 'repository is a shallow clone; commit-graph history is truncated and co-change evidence cannot be established from it', + qualifyingMinCooccurrence, + ); + } + } catch (error) { + return withoutEvidence( + CompletenessState.EVIDENCE_UNAVAILABLE, + CompletenessReason.GIT_FAILED, + `could not determine whether the repository is shallow: ${error instanceof Error ? error.message : String(error)}`, + qualifyingMinCooccurrence, + ); + } + + // An empty repository resolves no HEAD. That is an absent history, not a + // broken one, and the two must not collapse into one state. + const basisCommit = await resolveCommit(repoRoot, basisRevision); + if (basisCommit === undefined) { + return withoutEvidence( + CompletenessState.NOT_MINED, + CompletenessReason.NO_COMMITS, + `${basisRevision} does not resolve to a commit in ${repoRoot}`, + qualifyingMinCooccurrence, + ); + } + + let events: readonly CommitEvent[]; + let availableTransitions: number; + try { + availableTransitions = await countFirstParent(repoRoot, basisCommit); + events = await extractEvents(repoRoot, { basisRevision: basisCommit, windowTransitions }); + } catch (error) { + if (error instanceof GitOutputError) { + return withoutEvidence( + CompletenessState.EVIDENCE_UNAVAILABLE, + CompletenessReason.MALFORMED_OUTPUT, + error.message, + qualifyingMinCooccurrence, + ); + } + if (error instanceof GitInvocationError) { + return withoutEvidence( + CompletenessState.EVIDENCE_UNAVAILABLE, + CompletenessReason.GIT_FAILED, + error.message, + qualifyingMinCooccurrence, + ); + } + throw error; + } + + if (events.length === 0) { + return withoutEvidence( + CompletenessState.NOT_MINED, + CompletenessReason.NO_COMMITS, + `no first-parent commits reachable from ${basisRevision}`, + qualifyingMinCooccurrence, + ); + } + + const pairs = countPairs(events, qualifyingMinCooccurrence); + + return { + l0Version: 1, + completeness: completeness( + pairs.length > 0 + ? CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED + : CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP, + CompletenessReason.MINED, + `mined ${events.length} first-parent transition(s) from ${basisCommit}; ${pairs.length} pair(s) at or above ${qualifyingMinCooccurrence} co-occurrence(s)`, + ), + basisWindow: { + basisRevision, + basisCommit, + windowTransitions, + availableTransitions, + extractedTransitions: events.length, + windowTruncated: availableTransitions > windowTransitions, + }, + events, + pairs, + qualifyingMinCooccurrence, + pathNormalization: PATH_NORMALIZATION_ASSUMPTIONS, + }; +} diff --git a/packages/mining-core/src/paths.test.ts b/packages/mining-core/src/paths.test.ts new file mode 100644 index 0000000..9e7c86b --- /dev/null +++ b/packages/mining-core/src/paths.test.ts @@ -0,0 +1,107 @@ +/** REQ-003 — one normalizer, and its assumptions are recorded rather than silent. */ +import { readFileSync, readdirSync, statSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { + PATH_NORMALIZATION_ASSUMPTIONS, + UNRATIFIED_ASSUMPTION_COUNT, + normalizePath, +} from './paths.js'; + +const srcDir = dirname(fileURLToPath(import.meta.url)); + +function sourceFiles(): string[] { + const found: string[] = []; + const walk = (dir: string): void => { + for (const entry of readdirSync(dir)) { + const full = join(dir, entry); + if (statSync(full).isDirectory()) walk(full); + else if (full.endsWith('.ts')) found.push(full); + } + }; + walk(srcDir); + return found; +} + +describe('normalizePath', () => { + it('rewrites backslash separators', () => { + expect(normalizePath('src\\routes\\checkout.ts')).toBe('src/routes/checkout.ts'); + }); + + it('strips a leading "./", including repeated prefixes', () => { + expect(normalizePath('./src/a.ts')).toBe('src/a.ts'); + expect(normalizePath('././src/a.ts')).toBe('src/a.ts'); + }); + + it('strips a trailing slash but never empties a path', () => { + expect(normalizePath('src/')).toBe('src'); + expect(normalizePath('/')).toBe('/'); + }); + + it('preserves case — the assumption record says so, so the test says so', () => { + expect(normalizePath('src/Foo.ts')).toBe('src/Foo.ts'); + expect(normalizePath('src/Foo.ts')).not.toBe(normalizePath('src/foo.ts')); + }); + + it('applies no Unicode normalization, so NFC and NFD stay distinct', () => { + const nfc = 'src/café.ts'; // é as one code point + const nfd = 'src/café.ts'; // e + combining acute + expect(normalizePath(nfc)).toBe(nfc); + expect(normalizePath(nfd)).toBe(nfd); + expect(normalizePath(nfc)).not.toBe(normalizePath(nfd)); + }); + + it('is idempotent', () => { + for (const input of ['./a\\b/', 'src/a.ts', './/x', 'src/Foo.ts']) { + expect(normalizePath(normalizePath(input))).toBe(normalizePath(input)); + } + }); +}); + +describe('assumption record (REQ-003 amendment)', () => { + it('covers all six META-278 questions', () => { + expect(PATH_NORMALIZATION_ASSUMPTIONS).toHaveLength(6); + }); + + it('records four of them as assumed rather than ratified', () => { + // The Phase 0 audit found the prior art answers 2 of 6. If a future change + // silently promotes one of the remaining 4 to `ratified` without META-278 + // actually ruling, this count moves and the test says so. + expect(UNRATIFIED_ASSUMPTION_COUNT).toBe(4); + }); + + it('gives every assumption a behavior and a rationale', () => { + for (const assumption of PATH_NORMALIZATION_ASSUMPTIONS) { + expect(assumption.question.length).toBeGreaterThan(0); + expect(assumption.behavior.length).toBeGreaterThan(0); + expect(assumption.rationale.length).toBeGreaterThan(0); + } + }); +}); + +describe('single-definition guarantee (REQ-003 verify)', () => { + it('defines normalizePath exactly once across the package', () => { + // Assembled rather than written as a literal, so this assertion does not + // match its own source and report a second definition that does not exist. + const definition = new RegExp(['export', 'function', 'normalizePath\\b'].join(' ')); + const definitions = sourceFiles().filter((file) => + definition.test(readFileSync(file, 'utf8')), + ); + expect(definitions.map((f) => f.slice(srcDir.length + 1))).toEqual(['paths.ts']); + }); + + it('has no inline separator replacement outside the normalizer', () => { + // The literal `.replace(/\\/g, '/')` and its equivalents are the shape a + // second, quieter normalizer takes. Anywhere but paths.ts is a violation. + const offenders: string[] = []; + for (const file of sourceFiles()) { + if (file.endsWith('paths.ts') || file.endsWith('paths.test.ts')) continue; + const source = readFileSync(file, 'utf8'); + if (/replace\(\s*\/\\\\\/g/.test(source) || /split\(\s*['"`]\\\\['"`]\s*\)/.test(source)) { + offenders.push(file.slice(srcDir.length + 1)); + } + } + expect(offenders).toEqual([]); + }); +}); diff --git a/packages/mining-core/src/paths.ts b/packages/mining-core/src/paths.ts new file mode 100644 index 0000000..9964072 --- /dev/null +++ b/packages/mining-core/src/paths.ts @@ -0,0 +1,121 @@ +/** + * Canonical path identity for L0 (REQ-003). + * + * META-278 specifies six questions about path identity and ADR-006 does not + * exist — the Phase 0 audit (A-2) found 0 normalizer implementations across + * `workspacejson/cli` and `workspacejson/standard`, and the only prior art is + * an unexported `toIndexKey` in `packages/cli/src/producer/evidence.ts` that + * answers 2 of the 6. So this function is the first implementation of a rule + * that has not been ratified. + * + * That makes silence the hazard. A normalizer that quietly picks an answer to + * "is `Foo.ts` the same path as `foo.ts`" produces keys that look settled and + * are not, which is AP-1 wearing a different hat. Every assumption this module + * makes on an unratified question is therefore declared in + * `PATH_NORMALIZATION_ASSUMPTIONS` and carried in L0's output, so a reader sees + * the guess as a guess and META-278 can overrule it against a written record + * rather than against archaeology. + */ + +/** One of META-278's six path-identity questions, and what L0 does about it. */ +export interface PathAssumption { + /** META-278's question, restated. */ + question: string; + /** `ratified` — settled elsewhere. `assumed` — L0 guessed; META-278 governs. */ + standing: 'ratified' | 'assumed'; + /** The behavior this module implements. */ + behavior: string; + /** Why, and what would change if META-278 rules the other way. */ + rationale: string; +} + +/** + * The full assumption record, emitted with every observation set. + * + * Ordered by META-278's own enumeration so the record can be diffed against the + * issue. Two are `ratified` only in the weak sense that the published schema's + * own field descriptions state them ("repository-root-relative POSIX path + * (forward slashes, no leading \"./\", no drive letters)") — that is schema + * prose, not an ADR, and it is recorded here as the source rather than claimed + * as independent authority. + */ +export const PATH_NORMALIZATION_ASSUMPTIONS: readonly PathAssumption[] = Object.freeze([ + Object.freeze({ + question: 'Path separator: backslash or forward slash?', + standing: 'ratified' as const, + behavior: 'Backslashes are rewritten to forward slashes.', + rationale: + "Stated by the published schema's own description of every path-bearing field: repository-root-relative POSIX, forward slashes. Not an open question.", + }), + Object.freeze({ + question: 'Leading "./" prefix: preserved or stripped?', + standing: 'ratified' as const, + behavior: 'A single leading "./" is stripped. Repeated "./././" collapses to nothing.', + rationale: + 'Same schema description: "no leading \\"./\\"". Repeated prefixes are not addressed there; stripping all of them is the only reading consistent with stripping one.', + }), + Object.freeze({ + question: 'Case sensitivity: are "Foo.ts" and "foo.ts" the same path?', + standing: 'assumed' as const, + behavior: + 'Case is preserved and comparison is case-sensitive. No case folding is applied.', + rationale: + "Git records the byte sequence it was given, so folding here would invent an identity git does not assert and would silently merge two files that a case-sensitive checkout keeps distinct. The cost is the mirror error: on a case-insensitive filesystem a rename that only changes case reads as two paths. META-278 may rule the other way; if it does, this is the site that changes.", + }), + Object.freeze({ + question: 'Unicode encoding: is NFC or NFD normalization applied?', + standing: 'assumed' as const, + behavior: + 'No Unicode normalization is applied. Bytes are decoded as UTF-8 and otherwise left alone.', + rationale: + 'Applying NFC would make L0 disagree with `git ls-files` on macOS-authored paths, and applying NFD would do the same on Linux-authored ones. Neither is safe to pick unilaterally, so L0 picks neither and says so. Consequence, stated plainly: the same file committed from two platforms can produce two distinct L0 paths.', + }), + Object.freeze({ + question: 'Trailing slash: is "src/" the same path as "src"?', + standing: 'assumed' as const, + behavior: + 'A trailing slash is stripped. L0 only ever sees blob paths from `diff-tree -r`, so this should never fire.', + rationale: + "Defensive only. `diff-tree -r` emits blobs, never directories, so a trailing slash reaching this function means an upstream assumption broke. Stripping is the conservative choice; the input is recorded as unexpected rather than normalized away, because the interesting event is that it happened at all.", + }), + Object.freeze({ + question: 'Symlinks and submodule/worktree roots: resolved before or after comparison?', + standing: 'assumed' as const, + behavior: + 'Neither is resolved. L0 uses the path as recorded in the commit, relative to the repository whose history is being mined.', + rationale: + "L0 reads the commit graph and never touches the working tree, so there is no filesystem to resolve a symlink against — resolution would require a checkout and would make output depend on which revision happens to be checked out, breaking REQ-004. Submodule contents are a different repository's history and are not mined; a gitlink appears as a single path. This is the question L0 is least equipped to answer and META-278 should not read L0's behavior as a proposal.", + }), +]); + +/** Count of META-278 questions L0 answers by assumption rather than by rule. */ +export const UNRATIFIED_ASSUMPTION_COUNT = PATH_NORMALIZATION_ASSUMPTIONS.filter( + (assumption) => assumption.standing === 'assumed', +).length; + +/** + * The single path-normalization function for L0 (REQ-003). + * + * Every path-touching site in this package calls this. There is no second + * definition and no inline separator handling anywhere else — that is asserted + * by a test, not by convention, because the whole point of a single normalizer + * is defeated by one call site that skips it. + * + * Behavior is exactly what `PATH_NORMALIZATION_ASSUMPTIONS` describes. Read + * that record before changing anything here. + */ +export function normalizePath(rawPath: string): string { + // Separator: ratified. + let path = rawPath.replace(/\\/g, '/'); + + // Leading "./": ratified. Loop rather than a single strip so "././a" lands on + // "a" — one strip would leave "./a", which is the shape the rule forbids. + while (path.startsWith('./')) path = path.slice(2); + + // Trailing slash: assumed, defensive. Never expected from `diff-tree -r`. + while (path.length > 1 && path.endsWith('/')) path = path.slice(0, -1); + + // Case, Unicode, symlinks, submodule roots: deliberately untouched. See the + // assumption record for why each is a decision and not an omission. + return path; +} diff --git a/packages/mining-core/src/serialize.ts b/packages/mining-core/src/serialize.ts new file mode 100644 index 0000000..63f469f --- /dev/null +++ b/packages/mining-core/src/serialize.ts @@ -0,0 +1,47 @@ +/** + * Deterministic serialization of an observation set (REQ-004). + * + * REQ-004 asks that two runs at the same `basisRevision` produce byte-identical + * output. That is a property of the *serializer* as much as the miner: + * `JSON.stringify` preserves insertion order, so two structurally identical + * objects built by different code paths can serialize to different bytes. + * Sorting keys removes that degree of freedom. + * + * Scope of the guarantee, stated because the Phase 0 audit found it matters. + * These bytes are a function of the repository at `basisCommit` and nothing + * else — no wall clock, no locale, no environment, no host paths. That is + * narrower than "the artifact is deterministic": `generated.hygiene` in the + * published producer is fed by a 30-day `git log --since` window and moves on + * its own (META-306). L0 does not inherit that and does not fix it. + */ +import type { ObservationSet } from './mine.js'; + +/** + * Stable JSON. Object keys sorted by UTF-16 code unit; arrays left in the order + * the miner produced, which is itself sorted. + * + * `localeCompare` is deliberately not used — it varies with host locale and + * would make the bytes machine-dependent, which is the failure this function + * exists to prevent. + */ +function stableStringify(value: unknown): string { + if (value === null) return 'null'; + if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`; + if (typeof value === 'object') { + const entries = Object.entries(value as Record) + .filter(([, v]) => v !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v)}`).join(',')}}`; + } + if (typeof value === 'number' && !Number.isFinite(value)) { + // NaN and Infinity serialize to `null` under JSON.stringify, which would + // silently turn a computation bug into a plausible-looking absent value. + throw new Error(`serializeObservationSet: non-finite number ${value}`); + } + return JSON.stringify(value) ?? 'null'; +} + +/** Serialize an observation set to stable bytes. */ +export function serializeObservationSet(observations: ObservationSet): string { + return stableStringify(observations); +} diff --git a/packages/mining-core/src/testing/fixtures.ts b/packages/mining-core/src/testing/fixtures.ts new file mode 100644 index 0000000..010fa69 --- /dev/null +++ b/packages/mining-core/src/testing/fixtures.ts @@ -0,0 +1,141 @@ +/** + * Synthetic git fixtures, one per REQ-005 state. + * + * Built in a temp directory rather than committed, because a committed git + * repository inside a git repository is a submodule or a packed oddity, and + * both make the fixture harder to read than the script that produces it. The + * script IS the specification of what each state looks like. + * + * Every commit pins author and committer identity and date, so fixture object + * ids are stable across machines and REQ-004's byte-identical claim is a + * property of the miner rather than of the clock. + */ +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; + +const FIXED_DATE = '2026-01-01T00:00:00+0000'; + +const FIXED_ENV = { + GIT_AUTHOR_NAME: 'Fixture', + GIT_AUTHOR_EMAIL: 'fixture@example.invalid', + GIT_COMMITTER_NAME: 'Fixture', + GIT_COMMITTER_EMAIL: 'fixture@example.invalid', + GIT_AUTHOR_DATE: FIXED_DATE, + GIT_COMMITTER_DATE: FIXED_DATE, + GIT_CONFIG_NOSYSTEM: '1', + LC_ALL: 'C', +}; + +export function git(cwd: string, args: readonly string[]): string { + return execFileSync('git', [...args], { + cwd, + encoding: 'utf8', + env: { ...process.env, ...FIXED_ENV }, + }); +} + +export function makeTempDir(label: string): string { + return mkdtempSync(join(tmpdir(), `wsj-l0-${label}-`)); +} + +export function removeDir(path: string): void { + rmSync(path, { recursive: true, force: true }); +} + +function initRepo(root: string): void { + git(root, ['init', '--quiet', '--initial-branch=main']); + git(root, ['config', 'user.name', 'Fixture']); + git(root, ['config', 'user.email', 'fixture@example.invalid']); + // Rename detection is part of the frozen extraction parameters, so the + // fixture must not have it disabled by an inherited global config. + git(root, ['config', 'diff.renames', 'true']); +} + +export function writeAndAdd(root: string, path: string, content: string): void { + const full = join(root, path); + mkdirSync(dirname(full), { recursive: true }); + writeFileSync(full, content, 'utf8'); + git(root, ['add', '--', path]); +} + +export function commit(root: string, message: string): void { + git(root, ['commit', '--quiet', '-m', message]); +} + +/** + * State 1 via NO_COMMITS: an initialized repository with no commits. + * Absence of history, not absence of coupling. + */ +export function makeEmptyRepo(): string { + const root = makeTempDir('empty'); + initRepo(root); + return root; +} + +/** + * State 2: real history, every commit touching exactly one file, so no pair + * ever co-occurs. Mining ran and the answer is genuinely "no relationship". + */ +export function makeUncoupledRepo(): string { + const root = makeTempDir('uncoupled'); + initRepo(root); + for (const [index, path] of ['src/a.ts', 'src/b.ts', 'src/c.ts'].entries()) { + writeAndAdd(root, path, `export const v${index} = ${index};\n`); + commit(root, `add ${path}`); + } + // Further single-file edits, still never two files in one commit. + writeAndAdd(root, 'src/a.ts', 'export const v0 = 99;\n'); + commit(root, 'edit a'); + return root; +} + +/** + * State 3: two files that change together three times and share no import. + * The shape the thesis rests on, in miniature. + */ +export function makeCoupledRepo(): string { + const root = makeTempDir('coupled'); + initRepo(root); + + writeAndAdd(root, 'src/build.ts', 'export const key = (a: string, b: number) => `${a}:${b}`;\n'); + writeAndAdd(root, 'src/parse.ts', "export const parse = (k: string) => k.split(':');\n"); + writeAndAdd(root, 'README.md', '# fixture\n'); + commit(root, 'initial'); + + for (const round of [1, 2]) { + writeAndAdd(root, 'src/build.ts', `export const key = (a: string, b: number) => \`\${a}:\${b}:${round}\`;\n`); + writeAndAdd(root, 'src/parse.ts', `// round ${round}\nexport const parse = (k: string) => k.split(':');\n`); + commit(root, `round ${round}`); + } + + // A lone commit so not every event is the coupled pair. + writeAndAdd(root, 'README.md', '# fixture\n\nnotes\n'); + commit(root, 'docs'); + + return root; +} + +/** + * A shallow clone of a coupled repository — REQ-006. + * + * The point of cloning the *coupled* fixture is that a shallow clone of an + * uncoupled repository would report no pairs for the right reason by accident. + * Here the full history has a real pair and the shallow clone cannot see it, so + * an unguarded miner returns a confident, wrong, non-empty-looking answer. + */ +export function makeShallowCloneOfCoupled(): { source: string; shallow: string } { + const source = makeCoupledRepo(); + const shallow = makeTempDir('shallow'); + removeDir(shallow); + git(process.cwd(), ['clone', '--quiet', '--depth', '1', `file://${source}`, shallow]); + return { source, shallow }; +} + +/** A path that is not a git repository at all. */ +export function makeNonRepo(): string { + const root = makeTempDir('nonrepo'); + writeFileSync(join(root, 'plain.txt'), 'not a repository\n', 'utf8'); + return root; +} diff --git a/packages/mining-core/tsconfig.json b/packages/mining-core/tsconfig.json new file mode 100644 index 0000000..41a1e5a --- /dev/null +++ b/packages/mining-core/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "declaration": true, + "noEmit": false, + "types": ["node"] + }, + "//": "Deliberately does NOT include ../../types/ambient.d.ts. That file hand-declares node:fs, node:child_process and vitest, and ambient module declarations win over node_modules typings — OWNERSHIP.md records the resulting Dirent/readdirSync breakage as an unfixed defect of the same class META-244 already fixed once for @workspacejson/spec. This package compiles against the real @types/node@22.19.17 and vitest declarations instead, so its git and filesystem surface is typed by the packages that own it rather than by a local restatement.", + "include": [ + "src/**/*.ts" + ] +} diff --git a/packages/mining-core/vitest.config.ts b/packages/mining-core/vitest.config.ts new file mode 100644 index 0000000..6441ab4 --- /dev/null +++ b/packages/mining-core/vitest.config.ts @@ -0,0 +1,20 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + // These tests drive real git against real repositories, which is the whole + // point — a mocked `diff-tree` would prove nothing about extraction. The + // cost is real too: v2.2.1's frozen parameters spend one `rev-parse` and + // one `diff-tree` per commit, sequentially, so a 44-commit history is a + // low-hundreds-of-milliseconds-per-commit walk and the 5s default expires + // partway through. + // + // Raising the budget rather than batching the git calls is deliberate. The + // commands are quoted from the preregistration and REQ-001 verifies against + // them verbatim; replacing them with a faster equivalent would make the + // extraction no longer the thing that was frozen. The runtime itself is + // REQ-010's question and Phase 3 has not started. + testTimeout: 120_000, + hookTimeout: 120_000, + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 24ff6ea..a94248b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -99,6 +99,21 @@ importers: specifier: ^1.6.0 version: 1.6.1(@types/node@22.19.17) + packages/mining-core: + devDependencies: + '@types/node': + specifier: 22.19.17 + version: 22.19.17 + tsup: + specifier: ^8.0.0 + version: 8.5.1(postcss@8.5.23)(typescript@5.9.3) + typescript: + specifier: ^5.4.0 + version: 5.9.3 + vitest: + specifier: ^1.6.0 + version: 1.6.1(@types/node@22.19.17) + packages: '@babel/runtime@7.29.7': From dd410c4a11d653b36937c8f02940f58225ce6684 Mon Sep 17 00:00:00 2001 From: Qwynn Marcelle Date: Thu, 6 Aug 2026 21:39:01 -0400 Subject: [PATCH 2/5] test(mining-core): drive REQ-005 state 4 through a real failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The state-4 assertion previously checked an unresolvable revision, which is state 1 — so EVIDENCE_UNAVAILABLE was structurally reachable but never exercised, and REQ-005's "one fixture per state" was met for three states, not four. Adds makeCorruptedRepo: deletes the root commit's loose object so HEAD still resolves but the first-parent walk fails. Observed EVIDENCE_UNAVAILABLE / GIT_FAILED. Returns undefined when objects are packed so the test skips rather than passing against a repository that was never corrupted. Keeps the unresolvable-revision case as its own assertion, since "absent history" and "broken history" landing in different states is the distinction REQ-005 exists to carry. Refs: META-297 --- packages/mining-core/src/mine.test.ts | 29 ++++++++++++++------ packages/mining-core/src/testing/fixtures.ts | 21 +++++++++++++- 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/packages/mining-core/src/mine.test.ts b/packages/mining-core/src/mine.test.ts index 5a7d0b5..dee35a2 100644 --- a/packages/mining-core/src/mine.test.ts +++ b/packages/mining-core/src/mine.test.ts @@ -5,6 +5,7 @@ import { mine } from './mine.js'; import { serializeObservationSet } from './serialize.js'; import { commit, + makeCorruptedRepo, makeCoupledRepo, makeEmptyRepo, makeNonRepo, @@ -55,14 +56,26 @@ describe('REQ-005 — four completeness states, never collapsed', () => { expect(pair?.cooccurrenceCount).toBe(3); }); - it('state 4: malformed git output is EVIDENCE_UNAVAILABLE, not an empty result', async () => { - // Drive the failure through the real path rather than asserting on a - // constructor: a repository whose basis resolves but whose object store is - // unreadable produces state 4. Simulated by pointing at a repo and then - // corrupting the requested revision. - const root = fixture(makeCoupledRepo); - const result = await mine(root, { basisRevision: 'refs/heads/does-not-exist' }); - // An unresolvable revision is an absent history, not a broken one. + it('state 4: a history that cannot be walked is EVIDENCE_UNAVAILABLE, not an empty result', async () => { + // Driven through the real failure path, not asserted on a constructor. + // HEAD resolves; the walk to the root commit hits a deleted object. + const root = makeCorruptedRepo(); + if (root === undefined) return; // objects were packed; nothing was corrupted + created.push(root); + + const result = await mine(root); + expect(result.completeness.state).toBe(CompletenessState.EVIDENCE_UNAVAILABLE); + expect(result.completeness.reason).toBe(CompletenessReason.GIT_FAILED); + // The damage it must not do: report this as a clean, examined repository. + expect(result.completeness.state).not.toBe( + CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP, + ); + }); + + it('an unresolvable revision is an absent history, not a broken one', async () => { + const result = await mine(fixture(makeCoupledRepo), { + basisRevision: 'refs/heads/does-not-exist', + }); expect(result.completeness.state).toBe(CompletenessState.NOT_MINED); expect(result.completeness.reason).toBe(CompletenessReason.NO_COMMITS); }); diff --git a/packages/mining-core/src/testing/fixtures.ts b/packages/mining-core/src/testing/fixtures.ts index 010fa69..6fcb1f2 100644 --- a/packages/mining-core/src/testing/fixtures.ts +++ b/packages/mining-core/src/testing/fixtures.ts @@ -11,7 +11,7 @@ * property of the miner rather than of the clock. */ import { execFileSync } from 'node:child_process'; -import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; @@ -133,6 +133,25 @@ export function makeShallowCloneOfCoupled(): { source: string; shallow: string } return { source, shallow }; } +/** + * State 4: a repository whose HEAD resolves but whose history cannot be walked. + * + * Built by deleting the root commit's loose object. `rev-parse HEAD^{commit}` + * still succeeds because HEAD's own object is intact, so this is genuinely + * "evidence reachable but unavailable" and not "no history" — which is the + * distinction REQ-005's state 4 exists to carry. Returns `undefined` when the + * objects turn out to be packed, so the caller skips rather than asserting + * against a repository that was never corrupted. + */ +export function makeCorruptedRepo(): string | undefined { + const root = makeCoupledRepo(); + const rootCommit = git(root, ['rev-list', '--max-parents=0', 'HEAD']).trim(); + const loose = join(root, '.git', 'objects', rootCommit.slice(0, 2), rootCommit.slice(2)); + if (!existsSync(loose)) return undefined; + rmSync(loose); + return root; +} + /** A path that is not a git repository at all. */ export function makeNonRepo(): string { const root = makeTempDir('nonrepo'); From ca3ba17e8b32c0f7c12b560e3e0ca238325cd38b Mon Sep 17 00:00:00 2001 From: Qwynn Marcelle Date: Tue, 11 Aug 2026 06:10:19 -0400 Subject: [PATCH 3/5] feat(mining-core): L0 scoring, basis pinning and the selection rule (META-297 Phase 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes META-297 Phase 3 at L0. Nothing here writes to the artifact; L1 projection remains closed. Scoring implements META-289 v2.2.1 verbatim: size_weight = min(1, 10/fileCount), position_decay = 2^(-dPos/250), a 500-transition window, and exclusion of events with fileCount > 50. dPos is measured from the newest EXTRACTED event, never the newest scored one, so excluding a large event cannot shift the decay of everything older than it. Excluded events stay in the observation set and are named by commit — an exclusion nobody can point at is not auditable. The recorded file-role and path exclusion set is empty, so the size rule is the only exclusion applied. Counts follow the standard's ratified A-009 amendment: support is the distinct scored commits in which both files changed, occurrences the distinct scored commits in which at least one changed — the symmetric union, not a per-file marginal. Both integers, both over one boundary, support <= occurrences by construction. Basis pinning records the resolved basis as a full-length lowercase object name per A-009's pattern, alongside both window edges and the frozen weighting identifiers. A basis that cannot be pinned throws rather than being emitted, and where there is no window to pin the field is absent rather than a placeholder. The selection rule applies the frozen threshold, ranks by support DESC then occurrences ASC then files ASC by UTF-8 bytes, and caps at 50 after ranking. UTF-8 byte order is not a bare string comparison: U+1F600 sorts before U+E000 under UTF-16 code units and after it under UTF-8, so compareUtf8 is explicit. Every ranking key is an integer or a byte sequence, so no float can decide an order. Capping is pure — the scored set and the extracted events behind it are untouched — and the execution receipt records threshold, pairsBeforeCap, pairsEmitted, cap, the complete ranking rule and whether the cap bound, so a truncated list is visibly truncated rather than silently short. serializeSelection throws on any non-integer number. weightedSupport is a double from 2 ** x, whose precision ECMAScript leaves implementation-defined; a float in a committed artifact is the churn class A-009 exists to prevent. It stays in memory on the scored set for diagnostics and never reaches artifact-bound output. Honest degradation is the first thing both new stages do. A --depth 1 clone can hand over real events that would otherwise produce a structurally identical answer at reduced magnitude, so score and select gate on completeness state and never on whether an events array happens to be non-empty. Requirement labels corrected throughout. REQ-001..006 are the only identifiers written down for this package and remain cited; the weighting, scoring exclusion, basis pinning and selection rule carry no issue number and are now NAMED rather than numbered. Earlier drafts inferred REQ-007/009/010/011 from code comments; REQ-011 in particular is the generated-classifier investigation, tracked separately as META-316, not basis pinning. An invented identifier reads as a citation and cites nothing. History mining is an explicit refresh operation, not part of default generation: a bound 500-transition window costs 7.3-8.2s with a short PATH and 27.2-29.9s with a 36-entry one on an Apple M4 Pro, because the frozen parameters spend two git subprocesses per commit and Node re-resolves the binary through PATH on each of the 1000 spawns. Extraction is spawn-bound, not git-bound; scoring the resulting events costs 1-24ms. The public command name is deliberately left unchosen. Verified on three fixtures — billfold (44 transitions), motdotla/dotenv pinned at 2fc7eac8 (634 available, window binds) and sindresorhus/execa pinned at 8017b279 (846 available, window binds). Both external fixtures were selected on metadata observable before mining. Three dotenv pair supports were recounted by hand through an independent git path and agree exactly (80, 24, 8). Selection output is byte-identical across separate OS processes on all three. Gates: 194/194 tests across 21 files, monorepo build, monorepo typecheck, architecture check and package-docs check all pass. --- packages/mining-core/README.md | 217 ++++++++++-- packages/mining-core/src/index.ts | 55 ++- packages/mining-core/src/mine.ts | 26 +- packages/mining-core/src/score.test.ts | 434 ++++++++++++++++++++++++ packages/mining-core/src/score.ts | 317 +++++++++++++++++ packages/mining-core/src/select.test.ts | 324 ++++++++++++++++++ packages/mining-core/src/select.ts | 235 +++++++++++++ packages/mining-core/src/serialize.ts | 69 +++- packages/mining-core/vitest.config.ts | 11 +- 9 files changed, 1640 insertions(+), 48 deletions(-) create mode 100644 packages/mining-core/src/score.test.ts create mode 100644 packages/mining-core/src/score.ts create mode 100644 packages/mining-core/src/select.test.ts create mode 100644 packages/mining-core/src/select.ts diff --git a/packages/mining-core/README.md b/packages/mining-core/README.md index 38f278d..744f1bb 100644 --- a/packages/mining-core/README.md +++ b/packages/mining-core/README.md @@ -10,7 +10,7 @@ independent producer would be compared against. ## Scope -This package implements META-297 **Phases 1 and 2 only**. +This package implements META-297 **Phases 1 to 3**. | Requirement | What it means here | | -- | -- | @@ -21,23 +21,182 @@ This package implements META-297 **Phases 1 and 2 only**. | REQ-005 | Four completeness states, never collapsed | | REQ-006 | A shallow clone reports insufficient history, not zero | -**Not implemented, deliberately:** weighting, position decay, the `fileCount > 50` -exclusion, support thresholds, lift, ranking, pair caps. Those are REQ-007 -and REQ-009 and Phase 3 has not started. Extraction returns every event the -window produced, including events Phase 3 will exclude, so that an exclusion -remains countable rather than invisible. +Those six are the only requirement identifiers written down for this package, +so they are the only ones cited. Phase 3 adds four behaviors that carry no +issue number, and they are therefore **named rather than numbered** — an +invented identifier reads as a citation and cites nothing: -**No artifact projection.** L1 — writing `generated.coChange` — is blocked. The -published schema's `coChange` item requires `rate` and sets -`additionalProperties: false`, so the counts-only shape the 2026-08-03 churn -ruling calls for is *rejected* by the schema rather than merely divergent from -it. That needs a schema admission, not a workaround. +| Behavior | What it means here | +| -- | -- | +| **Weighting** | v2.2.1's `size_weight` and `position_decay`, implemented verbatim | +| **Scoring exclusion** | Events with `fileCount > 50` are excluded from scoring, and named | +| **Basis pinning** | The basis is a full-length object name, or it is not emitted at all | +| **Selection rule** | Threshold, rank, cap at 50, with an execution receipt | + +The generated-classifier question is **not** in this package; it is tracked +separately as META-316. See the note on `generated` below. + +**No artifact projection.** L1 — writing `generated.coChange` — is not here. The +schema is no longer the blocker it was during Phases 1 and 2: the standard's +A-009 amendment merged and admits the observation form (`support` + +`occurrences` + a pinned `generated.basisRevision`). But the package carrying it +is unpublished, and A-009 is an explicitly staged transition — widen the reader, +verify consumer adoption, *then* enable producer emission. Emission is step 3, +and this package does not authorize it. + +## Scoring + +`score(observationSet, { minSupport })` is a pure function. It spawns no +process, reads no filesystem and consults no clock; everything it needs was +extracted in Phase 1. That is what keeps the exclusion auditable — the same +observation set can be scored twice with different parameters, and the +difference is attributable to the parameters rather than to a second walk of a +repository that may have moved. + +Two vocabularies meet here, and are deliberately kept apart. + +**v2.2.1 supplies the weighting**, implemented verbatim. + +| Parameter | Value | +| -- | -- | +| `size_weight` | `min(1, 10/fileCount)` | +| `position_decay` | `2^(-Δpos/250)` | +| Window | 500 first-parent transitions | +| Scoring exclusion | events with `fileCount > 50` | +| Support threshold | `support >= 3` (`DEFAULT_MIN_SUPPORT`, overridable, recorded) | + +Δpos is measured from the newest **extracted** event, never the newest scored +one — excluding a large event must not shift the decay of everything older than +it. The recorded file-role and path exclusion set is **empty**: no path is +excluded for being documentation, a lockfile or generated output, so the size +rule above is the only exclusion L0 applies, and it applies to whole events +rather than to paths. Excluded events stay in the observation set and are named +by commit in `exclusions.excludedCommits`, because an exclusion nobody can point +at is not auditable. + +**A-009 supplies the counts**, and they are integers. + +- `support` — distinct scored events in which **both** files changed. +- `occurrences` — distinct scored events in which **at least one** changed. The + symmetric union, so reversing the pair changes nothing. Never a per-file + marginal. + +`weightedSupport` is emitted alongside the counts, never instead of them. +Nothing derived is stored: a rate is a reader's question. + +A repository longer than the window is **valid**. The bounded window is recorded +in `basisWindow` (`availableTransitions`, `extractedTransitions`, +`windowTruncated`), and truncation by the window is a fact rather than an error. + +## Basis pinning + +`scoringBasis` carries everything needed to recount a result: the frozen +weighting identifiers, the pinned basis, and both edges of the window. + +`basisRevision` is a full-length lowercase Git object name — A-009's +`^([0-9a-f]{40}|[0-9a-f]{64})$` — never a symbolic ref, because a pin that does +not name exactly one commit permanently cannot be recounted against. A basis +failing that shape **throws**: it is a caller bug, not a repository condition, +and a bug must not disguise itself as one. Where there is no window to pin, +`scoringBasis` is **absent**, not a placeholder that reads as a real pin. + +## Selection rule (provisional producer profile) + +`select(scoredSet, { minSupport, cap })` decides what a producer would emit. +Three steps, in this order: + +1. **Threshold** — keep pairs at `support >= 3`. +2. **Rank** — `support` DESC, then `occurrences` ASC, then `files[0]` ASC by + **UTF-8 bytes**, then `files[1]` ASC by UTF-8 bytes. +3. **Cap** — keep the first **50**, *after* ranking. + +Every ranking key is an integer or a byte sequence. Nothing continuous +participates, so the order cannot shift with floating-point precision. + +**UTF-8 byte order is not `a < b`.** A bare JavaScript comparison is UTF-16 code +unit order, and the two genuinely disagree: U+1F600 is a surrogate pair +beginning `0xD83D`, which sorts *before* U+E000 under UTF-16, while its UTF-8 +encoding `F0 9F 98 80` sorts *after* U+E000's `EE 80 80`. `compareUtf8` is +exported and is the only comparator the ranking uses. + +**Capping is a presentation step, not a data-loss step.** `select` is pure: the +scored set it was handed still carries every pair and every weight afterwards, +and the extracted events behind that are untouched. The cap changes what is +emitted, never what was observed. + +### Execution receipt + +A truncated list must be visibly truncated, never silently short — the same +doctrine as the completeness states, one layer up. `receipt` records: + +| Field | Meaning | +| -- | -- | +| `minSupport` | The threshold applied before ranking | +| `pairsBeforeCap` | Pairs that cleared the threshold — the population the cap cut from | +| `pairsEmitted` | Pairs actually emitted; `min(pairsBeforeCap, cap)` | +| `cap` | The cap in force | +| `rankingRule` | The complete rule, as text, so two artifacts can be compared without reading this file | +| `capBound` | Whether the cap actually bound | + +A reader can therefore distinguish an emitted 50 that is everything from an +emitted 50 that is the top of 1,848, without access to the repository. + +### No floats in artifact-bound output + +`SelectedPair` carries `files`, `support` and `occurrences`. It does **not** +carry `weightedSupport`, and `serializeSelection` **throws** on any non-integer +number rather than rounding or dropping it. Rounding would invent precision the +measurement does not have; dropping would remove a field a reader was told to +expect; both are quieter than the bug. + +The reason is measured, not stylistic: `weightedSupport` is a double from +`2 ** x`, whose precision ECMAScript leaves implementation-defined. A float in a +committed artifact is the churn class A-009 exists to prevent. It remains +available in memory on the scored set, and `serializeScoredSet` will emit it for +diagnostics that never reach `workspace.json`. + +**This constrains L1 too, and L1 is still not authorized.** + +## The `generated` flag — a finding, not a feature + +The schema requires `generated` on every `coChange` entry, documented as +`"true = tooling-coupled pair (e.g. lockfile + package.json); consumers skip +these"`. **L0 has no rule that can ever produce `true`.** The recorded +file-role and path exclusion set is empty, so nothing classifies a pair as +tooling-coupled and the field would be constant `false` on every entry a +producer emitted. + +The concrete case is worse than merely constant. On a 500-transition window of +`motdotla/dotenv` pinned at `2fc7eac8`, the **highest-ranked pair under the +selection rule above** is `package-lock.json` ↔ `package.json` at support 80 — +literally the example the field's own documentation cites. L0 would label it +`generated: false`. The field does not merely carry no information; it carries +the wrong answer on its own canonical example. + +Removing it is a schema change, not a producer choice, so this package records +the finding and changes nothing. Tracked as **META-316**. + +## Refresh model + +History mining is an **explicit refresh operation**, not part of default +generation. A bound 500-transition window costs 7.3–8.2 s with a short `PATH` +and 27.2–29.9 s with a long one on an Apple M4 Pro; adding that to every +generate is not viable, and the result is pinned to a commit rather than +recomputed per run. + +A-009 already supplies the staleness protocol: compare `generated.basisRevision` +against the current revision, where "pin ≠ current revision" is a defined reader +state meaning *stale observation*. That only works if the pin is allowed to lag, +which presupposes an explicit refresh. + +**The public command name is deliberately not chosen here.** Naming it is a +separate decision and this package does not pre-empt it. ## Layering ```text -L0 (this package) git only → in-memory observation set -L1 producer L0 output → generated.coChange [blocked on schema admission] +L0 (this package) git only → observation set → scored set → selection +L1 producer L0 output → generated.coChange [held: A-009 step 3] L2 report the artifact → human-readable findings [never invokes git] ``` @@ -47,19 +206,39 @@ direction invariant checkable later. ## Usage ```ts -import { mine, serializeObservationSet } from '@workspacejson/mining-core'; +import { mine, score, select } from '@workspacejson/mining-core'; const observations = await mine('/path/to/repo', { basisRevision: 'HEAD' }); +const scored = score(observations); +const selection = select(scored); -if (observations.completeness.state === 'QUALIFYING_RELATIONSHIP_OBSERVED') { - for (const pair of observations.pairs) { - console.log(pair.files, pair.cooccurrenceCount); +if (selection.completeness.state === 'QUALIFYING_RELATIONSHIP_OBSERVED') { + console.log('basis', selection.scoringBasis!.basisRevision); + console.log('receipt', selection.receipt); // threshold, counts, cap, rule, capBound + for (const pair of selection.pairs) { + console.log(pair.files, pair.support, pair.occurrences); } } + +// `scored` is untouched by `select` — every pair and every weight is still +// there, which is what makes the cap auditable. ``` -`mine` does not throw for an absent, shallow, or unreadable history. Those are -completeness states, which is the entire point of REQ-005. +`mine` does not throw for an absent, shallow, or unreadable history, and `score` +does not invent one. Those are completeness states, which is the entire point of +REQ-005 — and `score` gates on the state rather than on whether an events array +happens to be non-empty, because a `--depth 1` clone can hand over real events +that would otherwise produce a structurally identical answer at reduced +magnitude. + +### Cost + +Scoring is free; extraction is not. On an Apple M4 Pro, a bound 500-transition +window costs **7.3–8.2 s** with a short `PATH` and **27.2–29.9 s** with this +machine's 36-entry `PATH`, because v2.2.1's frozen parameters spend two `git` +subprocesses per commit and Node resolves the binary through `PATH` on every +one. Scoring the resulting 500 events costs **1–24 ms**. Plan the calling +interface around the extraction number, not the scoring one. ## Completeness states diff --git a/packages/mining-core/src/index.ts b/packages/mining-core/src/index.ts index cf22bad..f12d7f0 100644 --- a/packages/mining-core/src/index.ts +++ b/packages/mining-core/src/index.ts @@ -1,11 +1,25 @@ /** * L0 mining core — public surface. * - * Phases 1 and 2 of META-297 only: extraction, normalization, and completeness - * semantics. No scoring. Nothing here writes to the artifact — L1 projection is - * Phase 3 and is blocked on a schema admission (the published `coChange` item - * requires `rate` and forbids additional properties, so the counts-only shape - * is rejected, not merely divergent). + * Phases 1 to 3 of META-297: extraction, normalization, completeness + * semantics, v2.2.1 scoring, basis pinning, and the selection rule. + * + * Still nothing here writes to the artifact. L1 projection — mapping a + * selection onto `generated.coChange` — is a separate step held closed by the + * coordinator gate. The standard's A-009 amendment has merged and admits the + * observation form (`support` + `occurrences` + a pinned `basisRevision`), so + * the schema is no longer the blocker it was during Phases 1 and 2; the + * package carrying it is unpublished and emission is step 3 of A-009's staged + * transition, which this package does not authorize. + * + * The pipeline is `mine` → `score` → `select`, each pure with respect to the + * one before it, so the extracted events and the uncapped scored result remain + * auditable after the selection has capped anything. + * + * Named behaviors, not numbered ones. REQ-001..006 are written down for this + * package and are cited; the weighting, the scoring exclusion, basis pinning + * and the selection rule are not, so they are named. An invented identifier + * reads as a citation and cites nothing. */ export { type Completeness, @@ -36,4 +50,33 @@ export { type PathAssumption, normalizePath, } from './paths.js'; -export { serializeObservationSet } from './serialize.js'; +export { + DEFAULT_MIN_SUPPORT, + POSITION_DECAY_HALF_LIFE, + SCORING_MAX_FILE_COUNT, + SIZE_WEIGHT_NUMERATOR, + WEIGHTING_VERSION, + type ScoreOptions, + type ScoredPair, + type ScoredSet, + type ScoringBasis, + type ScoringExclusions, + positionDecay, + score, + sizeWeight, +} from './score.js'; +export { + RANKING_RULE, + SELECTION_CAP, + type SelectOptions, + type SelectedPair, + type SelectionReceipt, + type SelectionResult, + compareUtf8, + select, +} from './select.js'; +export { + serializeObservationSet, + serializeScoredSet, + serializeSelection, +} from './serialize.js'; diff --git a/packages/mining-core/src/mine.ts b/packages/mining-core/src/mine.ts index 784dd24..a00561a 100644 --- a/packages/mining-core/src/mine.ts +++ b/packages/mining-core/src/mine.ts @@ -8,11 +8,15 @@ * an independent producer gets compared against. * * Scope boundary, enforced by what is absent: no weighting, no decay, no - * support threshold, no lift, no ranking, no cap. Those are REQ-007 and - * REQ-009, and Phase 3 has not started. What is here is extraction (REQ-001), - * a computed empty tree (REQ-002), one normalizer (REQ-003), deterministic - * output (REQ-004), four completeness states (REQ-005), and a shallow-clone - * guard (REQ-006). + * support threshold, no lift, no ranking, no cap. Those are the *scoring* and + * *selection* behaviors and they live in `score.ts` and `select.ts`. What is + * here is extraction (REQ-001), a computed empty tree (REQ-002), one + * normalizer (REQ-003), deterministic output (REQ-004), four completeness + * states (REQ-005), and a shallow-clone guard (REQ-006). + * + * Those six numbers are the only requirement identifiers in this package that + * are written down anywhere. Behaviors added after them are named, not + * numbered — an invented identifier reads as a citation and cites nothing. */ import { type Completeness, @@ -136,10 +140,10 @@ function withoutEvidence( /** * Count co-occurrences over the extracted events. * - * N files in one event yield N(N-1)/2 pairs. Phase 3 owns the cap (REQ-009); - * this counts everything the window produced, because a cap applied before the - * count is decided would silently determine the ranking it is supposed to - * follow. + * N files in one event yield N(N-1)/2 pairs. The cap belongs to the selection + * rule in `select.ts`; this counts everything the window produced, because a + * cap applied before the count is decided would silently determine the ranking + * it is supposed to follow. */ function countPairs( events: readonly CommitEvent[], @@ -168,8 +172,8 @@ function countPairs( } // Total order, and deliberately not a ranking: sorted by path so the output - // is stable for REQ-004. Ordering by count would be a ranking rule, and - // REQ-009 has not been decided. + // is stable for REQ-004. Ordering by count here would pre-empt the selection + // rule, which is `select.ts`'s job and applies its own ordering downstream. pairs.sort((a, b) => { if (a.files[0] !== b.files[0]) return a.files[0] < b.files[0] ? -1 : 1; if (a.files[1] !== b.files[1]) return a.files[1] < b.files[1] ? -1 : 1; diff --git a/packages/mining-core/src/score.test.ts b/packages/mining-core/src/score.test.ts new file mode 100644 index 0000000..6f14ac7 --- /dev/null +++ b/packages/mining-core/src/score.test.ts @@ -0,0 +1,434 @@ +/** + * Phase 3: the weighting, the scoring exclusion, bounded scoring cost, and + * basis pinning. Named behaviors — only REQ-001..006 are written down. + * + * Every expectation here is a number written down before the implementation + * existed, computed by hand from META-289 v2.2.1's frozen weighting and from + * the ratified observation-form definitions in the standard's A-009 amendment. + * None of them was read off whatever the code happened to return. + */ +import { describe, expect, it } from 'vitest'; +import { CompletenessReason, CompletenessState, completeness } from './completeness.js'; +import type { CommitEvent } from './git.js'; +import { type ObservationSet, mine } from './mine.js'; +import { PATH_NORMALIZATION_ASSUMPTIONS } from './paths.js'; +import { + DEFAULT_MIN_SUPPORT, + POSITION_DECAY_HALF_LIFE, + SCORING_MAX_FILE_COUNT, + SIZE_WEIGHT_NUMERATOR, + WEIGHTING_VERSION, + positionDecay, + score, + sizeWeight, +} from './score.js'; +import { serializeScoredSet } from './serialize.js'; +import { + makeCoupledRepo, + makeShallowCloneOfCoupled, + removeDir, +} from './testing/fixtures.js'; + +/** A 40-hex object id shape, so pinning can be asserted without a repository. */ +const OID_A = 'a'.repeat(40); +const OID_B = 'b'.repeat(40); + +function event(position: number, files: readonly string[], commit: string): CommitEvent { + const sorted = [...new Set(files)].sort(); + return { + commit, + parent: OID_B, + files: sorted, + fileCount: sorted.length, + position, + }; +} + +function manyFiles(count: number, extra: readonly string[]): string[] { + const generated = Array.from({ length: count - extra.length }, (_, i) => + `bulk/f${String(i).padStart(3, '0')}.ts`, + ); + return [...extra, ...generated]; +} + +/** + * The hand-computed fixture. + * + * position 0: [a, b] fileCount 2 scored + * position 1: [a, c] fileCount 2 scored + * position 2: [a, b, ...58 more] fileCount 60 EXCLUDED (> 50) + * position 3: [a, b] fileCount 2 scored, and the basis + * + * Newest extracted position is 3, so Δpos is 3, 2, 1, 0 respectively — and + * critically, position 2 dropping out of scoring must not renumber the others. + */ +function observations( + overrides: Partial = {}, + // `exactOptionalPropertyTypes` makes an explicit `undefined` a different + // thing from an absent key, and "no window to pin" is the absent key. So the + // helper deletes it rather than letting a caller pass `undefined`. + dropBasisWindow = false, +): ObservationSet { + const events: CommitEvent[] = [ + event(0, ['src/a.ts', 'src/b.ts'], '0'.repeat(40)), + event(1, ['src/a.ts', 'src/c.ts'], '1'.repeat(40)), + event(2, manyFiles(60, ['src/a.ts', 'src/b.ts']), '2'.repeat(40)), + event(3, ['src/a.ts', 'src/b.ts'], '3'.repeat(40)), + ]; + const built: ObservationSet = { + l0Version: 1, + completeness: completeness( + CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED, + CompletenessReason.MINED, + 'fixture', + ), + basisWindow: { + basisRevision: 'HEAD', + basisCommit: OID_A, + windowTransitions: 500, + availableTransitions: 4, + extractedTransitions: 4, + windowTruncated: false, + }, + events, + pairs: [], + qualifyingMinCooccurrence: 1, + pathNormalization: PATH_NORMALIZATION_ASSUMPTIONS, + ...overrides, + }; + if (!dropBasisWindow) return built; + const { basisWindow: _absent, ...withoutWindow } = built; + return withoutWindow; +} + +describe('weighting: v2.2.1, implemented verbatim', () => { + it('size_weight is min(1, 10/fileCount)', () => { + expect(SIZE_WEIGHT_NUMERATOR).toBe(10); + // Below the numerator the weight saturates at 1 — it never exceeds it. + expect(sizeWeight(1)).toBe(1); + expect(sizeWeight(2)).toBe(1); + expect(sizeWeight(10)).toBe(1); + // Above it the weight is the ratio, exactly. + expect(sizeWeight(20)).toBe(0.5); + expect(sizeWeight(50)).toBe(0.2); + expect(sizeWeight(100)).toBe(0.1); + }); + + it('position_decay is 2^(-dPos/250)', () => { + expect(POSITION_DECAY_HALF_LIFE).toBe(250); + // At the basis there is no decay at all. + expect(positionDecay(0)).toBe(1); + // One half-life back is exactly one half. + expect(positionDecay(250)).toBeCloseTo(0.5, 12); + // The far edge of a full 500-transition window is a quarter. + expect(positionDecay(500)).toBeCloseTo(0.25, 12); + // Monotone decreasing, never negative, never zero inside the window. + expect(positionDecay(1)).toBeLessThan(1); + expect(positionDecay(499)).toBeGreaterThan(positionDecay(500)); + expect(positionDecay(500)).toBeGreaterThan(0); + }); + + it('excludes events with fileCount > 50 from scoring, and includes exactly 50', () => { + expect(SCORING_MAX_FILE_COUNT).toBe(50); + + const boundary = observations({ + events: [ + event(0, manyFiles(50, ['src/d.ts', 'src/e.ts']), '0'.repeat(40)), + event(1, manyFiles(51, ['src/f.ts', 'src/g.ts']), '1'.repeat(40)), + ], + }); + const result = score(boundary, { minSupport: 1 }); + + expect(result.exclusions.scoredEventCount).toBe(1); + expect(result.exclusions.excludedEventCount).toBe(1); + expect(result.exclusions.excludedCommits).toEqual(['1'.repeat(40)]); + + const paths = result.pairs.flatMap((p) => [...p.files]); + // The 50-file event was scored, so its pairs exist. + expect(paths).toContain('src/d.ts'); + // The 51-file event was not, so its pairs do not. + expect(paths).not.toContain('src/f.ts'); + }); + + it('preserves every extracted event on the input while excluding at scoring time', () => { + const input = observations(); + const result = score(input, { minSupport: 1 }); + + // Extraction-time preservation: the excluded event is still in the input. + expect(input.events).toHaveLength(4); + expect(input.events.some((e) => e.fileCount > SCORING_MAX_FILE_COUNT)).toBe(true); + // Scoring-time exclusion: it is accounted for, by commit, not silently dropped. + expect(result.exclusions.excludedEventCount).toBe(1); + expect(result.exclusions.excludedCommits).toEqual(['2'.repeat(40)]); + expect(result.exclusions.scoredEventCount).toBe(3); + }); + + it('measures dPos from the newest EXTRACTED event and never renumbers after exclusion', () => { + const result = score(observations(), { minSupport: 1 }); + const ab = result.pairs.find((p) => p.files[1] === 'src/b.ts'); + expect(ab).toBeDefined(); + + // (a,b) is scored at positions 0 and 3 only — position 2 is excluded. + // Newest EXTRACTED position is 3, so dPos is 3 and 0. + const expected = sizeWeight(2) * positionDecay(3) + sizeWeight(2) * positionDecay(0); + expect(ab!.weightedSupport).toBeCloseTo(expected, 12); + + // If exclusion had renumbered the surviving events 0,1,2 the newest would be + // 2 and this is the value that would have been produced instead. It must not + // be. The two differ in the fourth decimal, which is enough to discriminate. + const renumbered = sizeWeight(2) * positionDecay(2) + sizeWeight(2) * positionDecay(0); + expect(ab!.weightedSupport).not.toBeCloseTo(renumbered, 4); + }); + + it('counts support as both-changed and occurrences as the symmetric union', () => { + const result = score(observations(), { minSupport: 1 }); + const byPair = new Map(result.pairs.map((p) => [p.files.join('|'), p])); + + // (a,b): both changed at scored positions 0 and 3. + const ab = byPair.get('src/a.ts|src/b.ts'); + expect(ab?.support).toBe(2); + // Union: a or b changed at scored positions 0, 1 and 3. + expect(ab?.occurrences).toBe(3); + + // (a,c): both changed at scored position 1 only. + const ac = byPair.get('src/a.ts|src/c.ts'); + expect(ac?.support).toBe(1); + // Union is still 0, 1 and 3, because a changed in all three. + expect(ac?.occurrences).toBe(3); + + // (b,c) never co-occurred, so it is absent — not present with support 0. + expect(byPair.has('src/b.ts|src/c.ts')).toBe(false); + }); + + it('holds the standard A-009 invariants on every emitted pair', () => { + const result = score(observations(), { minSupport: 1 }); + expect(result.pairs.length).toBeGreaterThan(0); + for (const pair of result.pairs) { + // Enforced by validate(), not by the schema — so it is enforced here. + expect(pair.support).toBeLessThanOrEqual(pair.occurrences); + // Observation-form minimum: a pair whose union is empty was never observed. + expect(pair.occurrences).toBeGreaterThanOrEqual(1); + expect(Number.isInteger(pair.support)).toBe(true); + expect(Number.isInteger(pair.occurrences)).toBe(true); + expect(pair.files).toHaveLength(2); + expect(pair.files[0] < pair.files[1]).toBe(true); + } + }); + + it('adopts v2.2.1 support >= 3 as the default and records the value used', () => { + expect(DEFAULT_MIN_SUPPORT).toBe(3); + const result = score(observations()); + expect(result.minSupport).toBe(3); + // The fixture's best pair has support 2, so nothing qualifies at 3. + expect(result.pairs).toHaveLength(0); + // And that is state 2 — mining ran, nothing qualified. Not a new state, + // and not state 3 with an empty list. + expect(result.completeness.state).toBe(CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP); + expect(result.completeness.reason).toBe(CompletenessReason.MINED); + }); + + it('reports state 3 when a pair does clear the threshold', () => { + const result = score(observations(), { minSupport: 2 }); + expect(result.pairs).toHaveLength(1); + expect(result.completeness.state).toBe(CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED); + }); +}); + +describe('honest degradation: never a reduced-magnitude answer', () => { + it('passes NOT_MINED straight through and emits no pairs and no basis', () => { + const shallow: ObservationSet = { + l0Version: 1, + completeness: completeness( + CompletenessState.NOT_MINED, + CompletenessReason.SHALLOW_CLONE, + 'shallow', + ), + events: [], + pairs: [], + qualifyingMinCooccurrence: 1, + pathNormalization: PATH_NORMALIZATION_ASSUMPTIONS, + }; + const result = score(shallow, { minSupport: 1 }); + expect(result.completeness.state).toBe(CompletenessState.NOT_MINED); + expect(result.completeness.reason).toBe(CompletenessReason.SHALLOW_CLONE); + expect(result.pairs).toEqual([]); + expect(result.scoringBasis).toBeUndefined(); + }); + + it('refuses to score events handed to it under a non-mined completeness', () => { + // The AP-1 shape: events are present, but completeness says they are not a + // claim about the repository. Scoring them would manufacture a confident, + // structurally identical, wrong answer at reduced magnitude — exactly the + // --depth 1 defect. The scorer must key on completeness, not on whether an + // events array happens to be non-empty. + const truncated = observations({ + completeness: completeness( + CompletenessState.NOT_MINED, + CompletenessReason.SHALLOW_CLONE, + 'shallow clone with visible events', + ), + }); + const result = score(truncated, { minSupport: 1 }); + expect(result.pairs).toEqual([]); + expect(result.exclusions.scoredEventCount).toBe(0); + }); + + it('a --depth 1 clone scores to NOT_MINED where the full clone scores pairs', async () => { + const { source, shallow } = makeShallowCloneOfCoupled(); + try { + const full = score(await mine(source), { minSupport: 1 }); + const clipped = score(await mine(shallow), { minSupport: 1 }); + + // The full history has the coupling. + expect(full.completeness.state).toBe(CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED); + expect(full.pairs.length).toBeGreaterThan(0); + + // The clone does not report the same pairs at lower support. It reports + // that it did not look. + expect(clipped.completeness.state).toBe(CompletenessState.NOT_MINED); + expect(clipped.completeness.reason).toBe(CompletenessReason.SHALLOW_CLONE); + expect(clipped.pairs).toEqual([]); + } finally { + removeDir(source); + removeDir(shallow); + } + }); +}); + +describe('basis pinning', () => { + it('pins a full-length lowercase object name, never a symbolic ref', () => { + const result = score(observations(), { minSupport: 1 }); + expect(result.scoringBasis).toBeDefined(); + // The standard's A-009 pattern for generated.basisRevision. + expect(result.scoringBasis!.basisRevision).toMatch(/^([0-9a-f]{40}|[0-9a-f]{64})$/); + // Specifically the resolved commit, not the requested revision string. + expect(result.scoringBasis!.basisRevision).toBe(OID_A); + expect(result.scoringBasis!.basisRevision).not.toBe('HEAD'); + }); + + it('rejects a basis that is not a full-length object name rather than emitting it', () => { + const abbreviated = observations({ + basisWindow: { + basisRevision: 'HEAD', + basisCommit: 'abc1234', + windowTransitions: 500, + availableTransitions: 4, + extractedTransitions: 4, + windowTruncated: false, + }, + }); + expect(() => score(abbreviated, { minSupport: 1 })).toThrow(/basis/i); + }); + + it('pins both edges of the scored window and the decay origin', () => { + const result = score(observations(), { minSupport: 1 }); + const basis = result.scoringBasis!; + // Oldest extracted commit, position 0. + expect(basis.windowOldestCommit).toBe('0'.repeat(40)); + // Newest extracted commit, which is the basis end of the window. + expect(basis.windowNewestCommit).toBe('3'.repeat(40)); + // Δpos is measured from here, and it is an extracted position. + expect(basis.decayOriginPosition).toBe(3); + }); + + it('records the frozen weighting so a score is attributable to a named ruleset', () => { + const basis = score(observations(), { minSupport: 1 }).scoringBasis!; + expect(basis.weightingVersion).toBe(WEIGHTING_VERSION); + expect(basis.sizeWeightNumerator).toBe(10); + expect(basis.positionDecayHalfLife).toBe(250); + expect(basis.maxScoredFileCount).toBe(50); + // The recorded numbers are the numbers the functions actually use. + expect(sizeWeight(basis.sizeWeightNumerator * 2)).toBe(0.5); + expect(positionDecay(basis.positionDecayHalfLife)).toBeCloseTo(0.5, 12); + }); + + it('emits no scoringBasis when there is no window to pin', () => { + const unmined = observations( + { + completeness: completeness( + CompletenessState.NOT_MINED, + CompletenessReason.NO_COMMITS, + 'no commits', + ), + }, + true, + ); + // Absence, not a placeholder that reads as a real pin. + expect(score(unmined, { minSupport: 1 }).scoringBasis).toBeUndefined(); + }); + + it('two runs at the same pin agree byte for byte', async () => { + const repo = makeCoupledRepo(); + try { + const first = serializeScoredSet(score(await mine(repo), { minSupport: 1 })); + const second = serializeScoredSet(score(await mine(repo), { minSupport: 1 })); + expect(first).toBe(second); + expect(first).toContain('"weightedSupport"'); + } finally { + removeDir(repo); + } + }); + + it('sorts pairs by path, which is a total order and not a ranking', () => { + const result = score(observations(), { minSupport: 1 }); + const keys = result.pairs.map((p) => p.files.join('|')); + expect(keys).toEqual([...keys].sort()); + // Deliberately NOT ordered by support — the ranking rule is undecided and + // an accidental order would be read as one. + expect(result.pairs.map((p) => p.support)).toEqual([2, 1]); + }); +}); + +describe('scoring cost is bounded and is not the runtime problem', () => { + it('scores a full 500-transition window well under a second', () => { + // Extraction spends two git processes per commit; scoring spends none. This + // separates the two costs so the measured wall-clock number can be + // attributed correctly rather than blamed on whichever half is newer. + const events: CommitEvent[] = Array.from({ length: 500 }, (_, position) => + event( + position, + [`src/m${position % 40}.ts`, `src/n${position % 37}.ts`, `test/t${position % 23}.ts`], + String(position).padStart(40, '0'), + ), + ); + const input = observations({ + events, + basisWindow: { + basisRevision: 'HEAD', + basisCommit: OID_A, + windowTransitions: 500, + availableTransitions: 900, + extractedTransitions: 500, + windowTruncated: true, + }, + }); + + const started = performance.now(); + const result = score(input, { minSupport: 1 }); + const elapsed = performance.now() - started; + + expect(result.exclusions.scoredEventCount).toBe(500); + expect(result.pairs.length).toBeGreaterThan(0); + expect(elapsed).toBeLessThan(1000); + }); + + it('records that the window bound, so truncation is a fact and not an error', () => { + const input = observations({ + basisWindow: { + basisRevision: 'HEAD', + basisCommit: OID_A, + windowTransitions: 500, + availableTransitions: 634, + extractedTransitions: 500, + windowTruncated: true, + }, + }); + const result = score(input, { minSupport: 1 }); + // A repository longer than the window is VALID. The bounded window is + // recorded; it is not a completeness failure. + expect(result.completeness.state).toBe(CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED); + expect(result.basisWindow?.windowTruncated).toBe(true); + expect(result.basisWindow?.availableTransitions).toBe(634); + expect(result.basisWindow?.extractedTransitions).toBe(500); + }); +}); diff --git a/packages/mining-core/src/score.ts b/packages/mining-core/src/score.ts new file mode 100644 index 0000000..f27af37 --- /dev/null +++ b/packages/mining-core/src/score.ts @@ -0,0 +1,317 @@ +/** + * L0 scoring and basis pinning — META-297 Phase 3. + * + * Named behaviors, not numbered ones. REQ-001..006 are written down in the + * issue and this package cites them; the scoring, exclusion, basis-pinning and + * selection behaviors are not, so they are named. An invented identifier reads + * as a citation and cites nothing. + * + * A pure function over an observation set. It spawns no process, reads no + * filesystem, and consults no clock: everything it needs was already extracted + * by Phase 1. That is what keeps the exclusion auditable — `score` can be + * handed the same observation set twice with different parameters and the + * difference is attributable to the parameters rather than to a second walk of + * a repository that may have moved. + * + * Two vocabularies meet here and they are deliberately kept apart. + * + * META-289 v2.2.1 supplies the *weighting*, frozen and implemented verbatim: + * `size_weight = min(1, 10/fileCount)`, `position_decay = 2^(-Δpos/250)`, a + * 500-transition window, and the exclusion of events with `fileCount > 50`. + * + * The standard's ratified A-009 amendment supplies the *counts*: `support` is + * the number of distinct qualifying commits in which BOTH files changed, and + * `occurrences` is the number in which AT LEAST ONE changed — the symmetric + * union, not a per-file marginal. Both are integers, both are counted over the + * same boundary, and `support <= occurrences` holds by construction. + * + * The weighted number and the counts are both emitted, and neither is derived + * from the other. A rate is a reader's question; nothing derived is stored. + * + * What is NOT here, and why: the pair cap and the ranking rule. Those are + * the selection rule's, they are applied downstream in `select.ts`, and an + * accidental order here would be read as a ranking — + * so pairs come out in path order, which is a total order and visibly not a + * ranking. + */ +import { + type Completeness, + CompletenessReason, + CompletenessState, + completeness, +} from './completeness.js'; +import type { BasisWindow, ObservationSet } from './mine.js'; + +/** The frozen weighting this module implements. Recorded in every output. */ +export const WEIGHTING_VERSION = 'META-289 v2.2.1'; + +/** `size_weight = min(1, 10/fileCount)`. */ +export const SIZE_WEIGHT_NUMERATOR = 10; + +/** `position_decay = 2^(-Δpos/250)`. One half-life is 250 transitions. */ +export const POSITION_DECAY_HALF_LIFE = 250; + +/** + * Events touching more than this many files are excluded from scoring. + * + * Strictly greater: an event with exactly 50 files is scored. The recorded + * file-role and path exclusion set is EMPTY — no path is excluded for being + * documentation, a lockfile or generated — so this size rule is the only + * exclusion L0 applies, and it applies to whole events rather than to paths. + */ +export const SCORING_MAX_FILE_COUNT = 50; + +/** + * v2.2.1's validity condition, adopted here. + * + * Phase 2 deliberately defaulted its own state 2/3 boundary to 1 and recorded + * that Phase 3 would raise it, because `rawSupport >= 3` is a scoring threshold + * and Phase 2 ran before scoring. This is Phase 3, so the threshold lands here + * and nowhere else. It is still a parameter, and the value used is recorded in + * the output rather than implied by it. + */ +export const DEFAULT_MIN_SUPPORT = 3; + +/** A full-length lowercase Git object name. The standard's A-009 pattern. */ +const OBJECT_NAME = /^([0-9a-f]{40}|[0-9a-f]{64})$/; + +/** v2.2.1's `size_weight`. Saturates at 1; never exceeds it. */ +export function sizeWeight(fileCount: number): number { + return Math.min(1, SIZE_WEIGHT_NUMERATOR / fileCount); +} + +/** + * v2.2.1's `position_decay`, over distance back from the basis. + * + * `deltaPosition` is 0 at the newest extracted event and grows toward the + * oldest. At the far edge of a full 500-transition window it is 500, so the + * oldest event carries a quarter of the weight of the newest. + */ +export function positionDecay(deltaPosition: number): number { + return 2 ** (-deltaPosition / POSITION_DECAY_HALF_LIFE); +} + +export interface ScoredPair { + /** The two paths, sorted. Set semantics; position carries no meaning. */ + files: readonly [string, string]; + /** Distinct scored events in which BOTH files changed. A-009's `support`. */ + support: number; + /** + * Distinct scored events in which AT LEAST ONE changed. A-009's + * `occurrences` in the observation form — the symmetric union denominator, + * so reversing the pair changes nothing. Never a per-file marginal. + */ + occurrences: number; + /** + * v2.2.1's weighted support: the sum of `size_weight × position_decay` over + * the scored events in which both files changed. + * + * Emitted alongside the counts, never instead of them, and never used to + * order anything — see the ranking note at the top of this file. + */ + weightedSupport: number; +} + +/** Everything needed to recount this result against the same history. */ +export interface ScoringBasis { + /** The frozen ruleset these numbers are attributable to. */ + weightingVersion: string; + sizeWeightNumerator: number; + positionDecayHalfLife: number; + maxScoredFileCount: number; + /** + * The pinned basis: a full-length lowercase object name, never a symbolic + * ref. A pin that does not name exactly one commit permanently cannot be + * recounted against, which is the whole point of pinning it. + */ + basisRevision: string; + /** Oldest extracted commit — the far edge of the window. */ + windowOldestCommit: string; + /** Newest extracted commit — the near edge, and the decay origin. */ + windowNewestCommit: string; + /** The extracted position Δpos is measured from. */ + decayOriginPosition: number; +} + +/** What the size rule removed, counted rather than assumed. */ +export interface ScoringExclusions { + /** The rule: events with more files than this are excluded. */ + maxFileCount: number; + /** Events that were scored. */ + scoredEventCount: number; + /** Events the size rule removed. */ + excludedEventCount: number; + /** + * The excluded commits, in extraction order. Named, not just counted — an + * exclusion nobody can point at is not auditable, and this is the only + * exclusion L0 applies. + */ + excludedCommits: readonly string[]; +} + +export interface ScoredSet { + /** Bumped when this shape changes. Consumers pin on it. */ + readonly l0ScoreVersion: 1; + /** Recomputed against `minSupport`. Same four states; no new ones. */ + completeness: Completeness; + /** Carried through from the observation set, unchanged. */ + basisWindow?: BasisWindow; + /** Absent when there is no window to pin. Never a placeholder. */ + scoringBasis?: ScoringBasis; + exclusions: ScoringExclusions; + /** Pairs at or above `minSupport`, in path order. Not a ranking. */ + pairs: readonly ScoredPair[]; + /** The threshold this run used. Recorded, never implied. */ + minSupport: number; +} + +export interface ScoreOptions { + /** v2.2.1's `rawSupport >= 3`. Defaults to 3. */ + minSupport?: number; +} + +/** No evidence to score. The input's own completeness is carried through verbatim. */ +function unscored(observations: ObservationSet, minSupport: number): ScoredSet { + return { + l0ScoreVersion: 1, + completeness: observations.completeness, + // Spread rather than assign: under `exactOptionalPropertyTypes` an explicit + // `undefined` is a different thing from an absent key, and the absent key + // is what "there is no window to pin" means. + ...(observations.basisWindow === undefined ? {} : { basisWindow: observations.basisWindow }), + exclusions: { + maxFileCount: SCORING_MAX_FILE_COUNT, + scoredEventCount: 0, + excludedEventCount: 0, + excludedCommits: [], + }, + pairs: [], + minSupport, + }; +} + +/** + * Score an observation set. + * + * Honest degradation is the first thing this function does, not the last. + * States 1 and 4 mean the history was not established, and an events array is + * not permission to override that — a `--depth 1` clone can hand over real + * events that produce a structurally identical answer at reduced magnitude, + * which is the single most expensive failure this package can ship, because + * every diagnostic run against an external repository is a fresh clone whose + * depth is a configuration detail nobody reads. So the gate is completeness, + * never `events.length > 0`. + * + * Throws only on a caller bug — an unpinnable basis. A repository condition is + * a completeness state; a bug must not disguise itself as one. + */ +export function score(observations: ObservationSet, options: ScoreOptions = {}): ScoredSet { + const minSupport = options.minSupport ?? DEFAULT_MIN_SUPPORT; + + const mined = + observations.completeness.state === CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED || + observations.completeness.state === CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP; + if (!mined) return unscored(observations, minSupport); + + const { basisWindow, events } = observations; + if (basisWindow === undefined || events.length === 0) { + return unscored(observations, minSupport); + } + + if (!OBJECT_NAME.test(basisWindow.basisCommit)) { + throw new Error( + `score: basis ${JSON.stringify(basisWindow.basisCommit)} is not a full-length lowercase Git object name, so the result could not be recounted against it`, + ); + } + + // Δpos is measured from the newest EXTRACTED event. Not the newest scored + // one: excluding a large event must not shift the decay of every event older + // than it, which is what renumbering after the filter would silently do. + const decayOriginPosition = Math.max(...events.map((event) => event.position)); + + const scored = events.filter((event) => event.fileCount <= SCORING_MAX_FILE_COUNT); + const excludedCommits = events + .filter((event) => event.fileCount > SCORING_MAX_FILE_COUNT) + .map((event) => event.commit); + + // support: scored events in which both files changed. + const support = new Map(); + // weightedSupport: v2.2.1's sum over those same events. + const weighted = new Map(); + // Per-file scored-event counts, used only to derive the union denominator. + const perFile = new Map(); + + for (const event of scored) { + const weight = sizeWeight(event.fileCount) * positionDecay(decayOriginPosition - event.position); + const files = event.files; + for (const path of files) { + perFile.set(path, (perFile.get(path) ?? 0) + 1); + } + for (let i = 0; i < files.length; i += 1) { + for (let j = i + 1; j < files.length; j += 1) { + // `files` is sorted and deduplicated upstream, so files[i] < files[j] + // and the key is canonical. NUL cannot occur inside a path — git's own + // -z framing depends on that — so it is the one safe separator. + const key = `${files[i]}\0${files[j]}`; + support.set(key, (support.get(key) ?? 0) + 1); + weighted.set(key, (weighted.get(key) ?? 0) + weight); + } + } + } + + const pairs: ScoredPair[] = []; + for (const [key, both] of support) { + if (both < minSupport) continue; + const [left, right] = key.split('\0') as [string, string]; + // |A ∪ B| = |A| + |B| − |A ∩ B|, and |A ∩ B| is exactly `both` over the + // scored events. Symmetric by construction, which is the property A-009 + // requires and a per-file marginal would not have. + const occurrences = (perFile.get(left) ?? 0) + (perFile.get(right) ?? 0) - both; + pairs.push({ + files: [left, right], + support: both, + occurrences, + weightedSupport: weighted.get(key) ?? 0, + }); + } + + // Path order: a total order, deterministic, and visibly not a ranking. + pairs.sort((a, b) => { + if (a.files[0] !== b.files[0]) return a.files[0] < b.files[0] ? -1 : 1; + if (a.files[1] !== b.files[1]) return a.files[1] < b.files[1] ? -1 : 1; + return 0; + }); + + const oldest = events[0]!; + const newest = events[events.length - 1]!; + + return { + l0ScoreVersion: 1, + completeness: completeness( + pairs.length > 0 + ? CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED + : CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP, + CompletenessReason.MINED, + `scored ${scored.length} of ${events.length} extracted event(s) at ${WEIGHTING_VERSION}; ${excludedCommits.length} excluded for fileCount > ${SCORING_MAX_FILE_COUNT}; ${pairs.length} pair(s) at support >= ${minSupport}`, + ), + basisWindow, + scoringBasis: { + weightingVersion: WEIGHTING_VERSION, + sizeWeightNumerator: SIZE_WEIGHT_NUMERATOR, + positionDecayHalfLife: POSITION_DECAY_HALF_LIFE, + maxScoredFileCount: SCORING_MAX_FILE_COUNT, + basisRevision: basisWindow.basisCommit, + windowOldestCommit: oldest.commit, + windowNewestCommit: newest.commit, + decayOriginPosition, + }, + exclusions: { + maxFileCount: SCORING_MAX_FILE_COUNT, + scoredEventCount: scored.length, + excludedEventCount: excludedCommits.length, + excludedCommits, + }, + pairs, + minSupport, + }; +} diff --git a/packages/mining-core/src/select.test.ts b/packages/mining-core/src/select.test.ts new file mode 100644 index 0000000..6fea74f --- /dev/null +++ b/packages/mining-core/src/select.test.ts @@ -0,0 +1,324 @@ +/** + * The provisional producer-profile selection rule: threshold, rank, cap. + * + * Named behavior, not a numbered requirement — only REQ-001..006 are written + * down for this package, and inventing an identifier would read as a citation + * to something that does not exist. + * + * Every expectation is computed by hand from the ruling, before the + * implementation existed. None was read off what the code returned. + */ +import { describe, expect, it } from 'vitest'; +import { CompletenessReason, CompletenessState, completeness } from './completeness.js'; +import { DEFAULT_MIN_SUPPORT, type ScoredPair, type ScoredSet } from './score.js'; +import { + RANKING_RULE, + SELECTION_CAP, + type SelectionResult, + compareUtf8, + select, +} from './select.js'; +import { serializeSelection } from './serialize.js'; + +const OID = 'a'.repeat(40); + +function pair(files: [string, string], support: number, occurrences: number): ScoredPair { + return { files, support, occurrences, weightedSupport: support * 0.7215 }; +} + +function scoredSet(pairs: readonly ScoredPair[], minSupport = 1): ScoredSet { + return { + l0ScoreVersion: 1, + completeness: completeness( + pairs.length > 0 + ? CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED + : CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP, + CompletenessReason.MINED, + 'fixture', + ), + basisWindow: { + basisRevision: 'HEAD', + basisCommit: OID, + windowTransitions: 500, + availableTransitions: 634, + extractedTransitions: 500, + windowTruncated: true, + }, + scoringBasis: { + weightingVersion: 'META-289 v2.2.1', + sizeWeightNumerator: 10, + positionDecayHalfLife: 250, + maxScoredFileCount: 50, + basisRevision: OID, + windowOldestCommit: '0'.repeat(40), + windowNewestCommit: '3'.repeat(40), + decayOriginPosition: 499, + }, + exclusions: { + maxFileCount: 50, + scoredEventCount: 500, + excludedEventCount: 0, + excludedCommits: [], + }, + pairs, + minSupport, + }; +} + +/** N synthetic pairs with strictly descending support, so ranking is unambiguous. */ +function descending(count: number): ScoredPair[] { + return Array.from({ length: count }, (_, i) => + pair([`src/a${String(i).padStart(4, '0')}.ts`, `src/b${String(i).padStart(4, '0')}.ts`], count - i, count + 10), + ); +} + +function keys(result: SelectionResult): string[] { + return result.pairs.map((p) => p.files.join('|')); +} + +describe('selection rule: threshold', () => { + it('applies the frozen support threshold and drops everything below it', () => { + const input = scoredSet([ + pair(['src/a.ts', 'src/b.ts'], 5, 10), + pair(['src/c.ts', 'src/d.ts'], 3, 10), + pair(['src/e.ts', 'src/f.ts'], 2, 10), + pair(['src/g.ts', 'src/h.ts'], 1, 10), + ]); + const result = select(input); + + expect(DEFAULT_MIN_SUPPORT).toBe(3); + expect(result.receipt.minSupport).toBe(3); + // support 2 and 1 are below the frozen threshold. + expect(result.pairs.map((p) => p.support)).toEqual([5, 3]); + expect(result.receipt.pairsBeforeCap).toBe(2); + }); + + it('refuses a scored set already filtered above the selection threshold', () => { + // The input has silently lost pairs the selection needed. Selecting from it + // would under-count without saying so, which is the same defect class as + // reporting zero for a shallow clone. + const input = scoredSet([pair(['src/a.ts', 'src/b.ts'], 9, 10)], 5); + expect(() => select(input, { minSupport: 3 })).toThrow(/threshold/i); + }); +}); + +describe('selection rule: ranking', () => { + it('ranks by support descending', () => { + const input = scoredSet([ + pair(['src/a.ts', 'src/b.ts'], 3, 10), + pair(['src/c.ts', 'src/d.ts'], 9, 10), + pair(['src/e.ts', 'src/f.ts'], 6, 10), + ]); + expect(select(input).pairs.map((p) => p.support)).toEqual([9, 6, 3]); + }); + + it('breaks a support tie by occurrences ascending — the tighter pair first', () => { + const input = scoredSet([ + pair(['src/a.ts', 'src/b.ts'], 7, 300), + pair(['src/c.ts', 'src/d.ts'], 7, 12), + pair(['src/e.ts', 'src/f.ts'], 7, 90), + ]); + expect(select(input).pairs.map((p) => p.occurrences)).toEqual([12, 90, 300]); + }); + + it('breaks a (support, occurrences) tie by files ascending in UTF-8 BYTE order', () => { + // The discriminating case. U+E000 encodes as EE 80 80; U+1F600 as F0 9F 98 80. + // In UTF-8 bytes, EE < F0, so the private-use path sorts FIRST. + // In UTF-16 code units — what a bare `<` on a JS string gives — the emoji is + // a surrogate pair starting 0xD83D, and 0xD83D < 0xE000, so the EMOJI sorts + // first. The two orders are opposite, which is what makes this a real test + // of the ruling rather than a restatement of the default comparator. + const privateUse = 'src/.ts'; + const astral = 'src/\u{1F600}.ts'; + expect(astral < privateUse).toBe(true); // UTF-16 says emoji first + expect(compareUtf8(privateUse, astral)).toBeLessThan(0); // UTF-8 says private-use first + + const input = scoredSet([ + pair([astral, 'src/z.ts'], 4, 20), + pair([privateUse, 'src/z.ts'], 4, 20), + ]); + expect(select(input).pairs[0]!.files[0]).toBe(privateUse); + }); + + it('breaks a files[0] tie by files[1], also in UTF-8 byte order', () => { + const input = scoredSet([ + pair(['src/a.ts', 'src/\u{1F600}.ts'], 4, 20), + pair(['src/a.ts', 'src/.ts'], 4, 20), + ]); + expect(select(input).pairs.map((p) => p.files[1])).toEqual(['src/.ts', 'src/\u{1F600}.ts']); + }); + + it('is a total order: a shuffled input produces an identical result', () => { + const base = descending(120); + // Many exact ties on both integer keys, so keys 3 and 4 do the work. + const tied = Array.from({ length: 40 }, (_, i) => + pair([`t/p${String(i).padStart(3, '0')}.ts`, 't/q.ts'], 4, 40), + ); + const all = [...base, ...tied]; + const shuffled = [...all]; + for (let i = shuffled.length - 1; i > 0; i -= 1) { + const j = (i * 7919 + 13) % (i + 1); + [shuffled[i], shuffled[j]] = [shuffled[j]!, shuffled[i]!]; + } + expect(keys(select(scoredSet(shuffled)))).toEqual(keys(select(scoredSet(all)))); + }); + + it('publishes the complete ranking rule as a string', () => { + expect(RANKING_RULE).toMatch(/support/i); + expect(RANKING_RULE).toMatch(/occurrences/i); + expect(RANKING_RULE).toMatch(/utf-8/i); + expect(select(scoredSet(descending(5))).receipt.rankingRule).toBe(RANKING_RULE); + }); +}); + +describe('selection rule: cap', () => { + it('caps at 50 and applies the cap AFTER ranking', () => { + expect(SELECTION_CAP).toBe(50); + const result = select(scoredSet(descending(120))); + expect(result.pairs).toHaveLength(50); + // Ranking first means the 50 kept are the 50 highest-support, so the + // smallest support emitted is 120 - 49 = 71. Capping before ranking would + // have kept an arbitrary 50 and this number would be wrong. + expect(result.pairs[0]!.support).toBe(120); + expect(result.pairs[49]!.support).toBe(71); + }); + + it('does not cap when the ranked list is shorter than the cap', () => { + // 12 synthetic pairs at supports 12..1; the threshold of 3 removes the two + // at supports 2 and 1, leaving 10. Well under the cap, so nothing is cut. + const result = select(scoredSet(descending(12))); + expect(result.pairs).toHaveLength(10); + expect(result.receipt.pairsBeforeCap).toBe(10); + expect(result.receipt.capBound).toBe(false); + }); +}); + +describe('selection rule: execution receipt', () => { + it('records the threshold, both counts, the cap, the rule, and whether the cap bound', () => { + const result = select(scoredSet(descending(120))); + expect(result.receipt).toEqual({ + minSupport: 3, + pairsBeforeCap: 118, // 120 synthetic pairs, two below support 3 + pairsEmitted: 50, + cap: 50, + rankingRule: RANKING_RULE, + capBound: true, + }); + }); + + it('makes a truncated list visibly truncated rather than silently short', () => { + const truncated = select(scoredSet(descending(120))).receipt; + expect(truncated.capBound).toBe(true); + expect(truncated.pairsEmitted).toBeLessThan(truncated.pairsBeforeCap); + + const whole = select(scoredSet(descending(12))).receipt; + expect(whole.capBound).toBe(false); + expect(whole.pairsEmitted).toBe(whole.pairsBeforeCap); + }); + + it('keeps pairsEmitted equal to min(pairsBeforeCap, cap) in both directions', () => { + for (const n of [0, 1, 49, 50, 51, 400]) { + const r = select(scoredSet(descending(n + 2))).receipt; + expect(r.pairsEmitted).toBe(Math.min(r.pairsBeforeCap, r.cap)); + expect(r.capBound).toBe(r.pairsBeforeCap > r.cap); + } + }); +}); + +describe('selection rule: preservation', () => { + it('does not mutate the scored set, and leaves the uncapped result complete', () => { + const input = scoredSet(descending(120)); + const before = input.pairs.length; + const beforeFirst = input.pairs[0]!.weightedSupport; + + const result = select(input); + + // Capping is a presentation step. The audit trail upstream is untouched. + expect(input.pairs).toHaveLength(before); + expect(input.pairs[0]!.weightedSupport).toBe(beforeFirst); + expect(result.pairs).toHaveLength(50); + // And the counts the receipt reports agree with what survived upstream. + expect(input.pairs.filter((p) => p.support >= 3)).toHaveLength(result.receipt.pairsBeforeCap); + }); + + it('carries the basis pin, the window and the scoring exclusions through', () => { + const input = scoredSet(descending(10)); + const result = select(input); + expect(result.scoringBasis).toEqual(input.scoringBasis); + expect(result.basisWindow).toEqual(input.basisWindow); + expect(result.exclusions).toEqual(input.exclusions); + }); +}); + +describe('selection rule: no floats reach the artifact', () => { + it('emits no weightedSupport, and no float, on a selected pair', () => { + const result = select(scoredSet(descending(10))); + for (const p of result.pairs) { + expect(Object.keys(p).sort()).toEqual(['files', 'occurrences', 'support']); + expect('weightedSupport' in p).toBe(false); + expect(Number.isInteger(p.support)).toBe(true); + expect(Number.isInteger(p.occurrences)).toBe(true); + } + }); + + it('serializes with every number an integer', () => { + const text = serializeSelection(select(scoredSet(descending(120)))); + expect(text).not.toContain('weightedSupport'); + // No decimal point and no exponent anywhere in the emitted numbers. + for (const [, literal] of text.matchAll(/:(-?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)/g)) { + expect(Number.isInteger(Number(literal))).toBe(true); + } + }); + + it('refuses to serialize a float rather than rounding or dropping it', () => { + const result = select(scoredSet(descending(5))); + // A float smuggled in by a future caller must fail loudly. Rounding it + // would be the churn class the float prohibition exists to prevent. + const contaminated = { + ...result, + pairs: [{ ...result.pairs[0]!, support: 3.5 }], + } as unknown as SelectionResult; + expect(() => serializeSelection(contaminated)).toThrow(/integer/i); + }); +}); + +describe('selection rule: honest degradation', () => { + it('passes a non-mined completeness through with no pairs and an honest receipt', () => { + const shallow: ScoredSet = { + l0ScoreVersion: 1, + completeness: completeness( + CompletenessState.NOT_MINED, + CompletenessReason.SHALLOW_CLONE, + 'shallow', + ), + exclusions: { + maxFileCount: 50, + scoredEventCount: 0, + excludedEventCount: 0, + excludedCommits: [], + }, + pairs: [], + minSupport: 3, + }; + const result = select(shallow); + expect(result.completeness.state).toBe(CompletenessState.NOT_MINED); + expect(result.completeness.reason).toBe(CompletenessReason.SHALLOW_CLONE); + expect(result.pairs).toEqual([]); + expect(result.scoringBasis).toBeUndefined(); + // The receipt is still complete: an empty list from a repository that was + // never examined must not look like an empty list from one that was. + expect(result.receipt.pairsBeforeCap).toBe(0); + expect(result.receipt.pairsEmitted).toBe(0); + expect(result.receipt.cap).toBe(50); + expect(result.receipt.capBound).toBe(false); + }); + + it('distinguishes examined-and-empty from not-examined at the receipt level', () => { + const examined = select(scoredSet([pair(['src/a.ts', 'src/b.ts'], 1, 9)])); + // Mining ran; nothing cleared the threshold. + expect(examined.completeness.state).toBe(CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP); + expect(examined.pairs).toEqual([]); + // And unlike the shallow case, the basis is pinned — the reader can tell. + expect(examined.scoringBasis).toBeDefined(); + }); +}); diff --git a/packages/mining-core/src/select.ts b/packages/mining-core/src/select.ts new file mode 100644 index 0000000..cb6ab91 --- /dev/null +++ b/packages/mining-core/src/select.ts @@ -0,0 +1,235 @@ +/** + * The provisional producer-profile selection rule — threshold, rank, cap. + * + * A named behavior, not a numbered requirement. Only REQ-001..006 are written + * down for this package; an invented identifier reads as a citation and cites + * nothing. + * + * This is the last L0 step and the only one whose shape is bound for an + * artifact. It takes a scored set and answers one question: of everything + * observed, which pairs does a producer emit, in what order, and how does a + * reader know what was left out. + * + * Three properties are load-bearing, and each exists because its absence is a + * known failure: + * + * 1. **The ranking keys on integers only.** `support` and `occurrences` are + * counts. `weightedSupport` is a double whose precision ECMAScript leaves + * implementation-defined, so an order that depended on it would be an order + * that could differ between engines. It does not appear here at all. + * + * 2. **The cap is applied after ranking, and it is recorded.** Capping first + * would silently choose which pairs get ranked. Capping without recording + * would make a truncated list indistinguishable from a short one — the same + * defect as reporting zero partners for a shallow clone, one layer up. + * + * 3. **Capping loses nothing upstream.** `select` is pure: the scored set it + * was handed still carries every pair and every weight afterwards, and the + * extracted events behind it are untouched. Capping is a presentation step. + * + * The public command name for the refresh operation that will call this is + * **deliberately not chosen here**. History mining is an explicit refresh + * rather than part of default generation — a bound 500-transition window costs + * seconds to tens of seconds — but naming the command is a separate decision + * and this module does not pre-empt it. + */ +import { + type Completeness, + CompletenessReason, + CompletenessState, + completeness, +} from './completeness.js'; +import type { BasisWindow } from './mine.js'; +import { DEFAULT_MIN_SUPPORT, type ScoredSet, type ScoringBasis, type ScoringExclusions } from './score.js'; + +/** Pairs emitted per repository, applied after ranking. */ +export const SELECTION_CAP = 50; + +/** + * The complete ranking rule, as text, carried in every receipt. + * + * Written out rather than described so that a reader comparing two artifacts + * can see whether they were ranked the same way without reading this file. + */ +export const RANKING_RULE = + 'support DESC, then occurrences ASC, then files[0] ASC by UTF-8 bytes, then files[1] ASC by UTF-8 bytes'; + +const UTF8 = new TextEncoder(); + +/** + * Compare two paths by UTF-8 byte order. + * + * Not `a < b`. A bare JavaScript string comparison is UTF-16 code unit order, + * and the two disagree: a supplementary character such as U+1F600 is a + * surrogate pair beginning 0xD83D, which sorts *before* U+E000 in UTF-16, while + * its UTF-8 encoding (F0 9F 98 80) sorts *after* U+E000's (EE 80 80). Any + * repository with an emoji in a path and anything in the private use area would + * rank differently under the two rules, and the ruling names UTF-8. + * + * Not `localeCompare` either — that varies with host locale, which would make + * the order depend on the machine that produced it. + */ +export function compareUtf8(a: string, b: string): number { + if (a === b) return 0; + const left = UTF8.encode(a); + const right = UTF8.encode(b); + const shared = Math.min(left.length, right.length); + for (let i = 0; i < shared; i += 1) { + if (left[i] !== right[i]) return left[i]! - right[i]!; + } + return left.length - right.length; +} + +/** + * A pair as a producer would emit it. + * + * Exactly three fields, all of them integers or strings. `weightedSupport` is + * absent by construction, not by omission — see the float prohibition in + * `serializeSelection`. + */ +export interface SelectedPair { + files: readonly [string, string]; + /** Distinct scored commits in which both files changed. */ + support: number; + /** Distinct scored commits in which at least one changed. The symmetric union. */ + occurrences: number; +} + +/** + * What this selection did, recorded rather than implied. + * + * A reader must be able to tell an emitted list of 50 that is everything from + * an emitted list of 50 that is the top of 1,848, without access to the + * repository. That is what `pairsBeforeCap` and `capBound` are for. + */ +export interface SelectionReceipt { + /** The support threshold applied before ranking. */ + minSupport: number; + /** Pairs that cleared the threshold. The population the cap cut from. */ + pairsBeforeCap: number; + /** Pairs actually emitted. `min(pairsBeforeCap, cap)`. */ + pairsEmitted: number; + /** The cap in force. */ + cap: number; + /** The complete ranking rule, as text. */ + rankingRule: string; + /** True when the cap actually bound — i.e. the emitted list is truncated. */ + capBound: boolean; +} + +export interface SelectionResult { + /** Bumped when this shape changes. Consumers pin on it. */ + readonly l0SelectionVersion: 1; + /** Recomputed against the threshold. Same four states; no new ones. */ + completeness: Completeness; + basisWindow?: BasisWindow; + scoringBasis?: ScoringBasis; + exclusions: ScoringExclusions; + receipt: SelectionReceipt; + /** Ranked and capped. */ + pairs: readonly SelectedPair[]; +} + +export interface SelectOptions { + /** The frozen support threshold. Defaults to 3. */ + minSupport?: number; + /** Pairs emitted per repository. Defaults to 50. */ + cap?: number; +} + +/** + * Apply the selection rule to a scored set. + * + * Pure. The input is not mutated and remains fully auditable afterwards. + * + * Throws only on a caller bug — a scored set that was already filtered above + * the selection threshold, and has therefore silently lost pairs the selection + * needed. That would produce a well-formed answer at reduced magnitude, which + * is the failure mode with the highest external cost on this project. + */ +export function select(scored: ScoredSet, options: SelectOptions = {}): SelectionResult { + const minSupport = options.minSupport ?? DEFAULT_MIN_SUPPORT; + const cap = options.cap ?? SELECTION_CAP; + + if (scored.minSupport > minSupport) { + throw new Error( + `select: the scored set was built at threshold ${scored.minSupport}, above the selection threshold ${minSupport}, so pairs this selection needs were already discarded and the result would be short without saying so`, + ); + } + + const mined = + scored.completeness.state === CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED || + scored.completeness.state === CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP; + + // Honest degradation. States 1 and 4 mean the history was not established, + // and a non-empty pair list is not permission to override that. The receipt + // is still filled in completely: an empty list from a repository that was + // never examined must not read like an empty list from one that was. + if (!mined) { + return { + l0SelectionVersion: 1, + completeness: scored.completeness, + ...(scored.basisWindow === undefined ? {} : { basisWindow: scored.basisWindow }), + ...(scored.scoringBasis === undefined ? {} : { scoringBasis: scored.scoringBasis }), + exclusions: scored.exclusions, + receipt: { + minSupport, + pairsBeforeCap: 0, + pairsEmitted: 0, + cap, + rankingRule: RANKING_RULE, + capBound: false, + }, + pairs: [], + }; + } + + // 1. Threshold. Copy first — `sort` mutates, and the scored set is the audit + // trail the cap is explicitly not allowed to damage. + const qualifying = scored.pairs.filter((pair) => pair.support >= minSupport); + + // 2. Rank. Integer keys first, then UTF-8 path order, which is total because + // `files` is unique per pair. No key is a float. + const ranked = [...qualifying].sort( + (a, b) => + b.support - a.support || + a.occurrences - b.occurrences || + compareUtf8(a.files[0], b.files[0]) || + compareUtf8(a.files[1], b.files[1]), + ); + + // 3. Cap, after ranking, so the pairs kept are the highest-ranked ones and + // not an arbitrary prefix of whatever order the map happened to produce. + const emitted = ranked.slice(0, cap); + + // Project to the artifact-bound shape. `weightedSupport` is dropped here and + // only here; it survives on the scored set for diagnostics. + const pairs: SelectedPair[] = emitted.map((pair) => ({ + files: pair.files, + support: pair.support, + occurrences: pair.occurrences, + })); + + return { + l0SelectionVersion: 1, + completeness: completeness( + pairs.length > 0 + ? CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED + : CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP, + CompletenessReason.MINED, + `${ranked.length} pair(s) at support >= ${minSupport}; ${pairs.length} emitted under a cap of ${cap}${ranked.length > cap ? ' (cap bound)' : ''}`, + ), + ...(scored.basisWindow === undefined ? {} : { basisWindow: scored.basisWindow }), + ...(scored.scoringBasis === undefined ? {} : { scoringBasis: scored.scoringBasis }), + exclusions: scored.exclusions, + receipt: { + minSupport, + pairsBeforeCap: ranked.length, + pairsEmitted: pairs.length, + cap, + rankingRule: RANKING_RULE, + capBound: ranked.length > cap, + }, + pairs, + }; +} diff --git a/packages/mining-core/src/serialize.ts b/packages/mining-core/src/serialize.ts index 63f469f..8f57d14 100644 --- a/packages/mining-core/src/serialize.ts +++ b/packages/mining-core/src/serialize.ts @@ -15,28 +15,45 @@ * its own (META-306). L0 does not inherit that and does not fix it. */ import type { ObservationSet } from './mine.js'; +import type { ScoredSet } from './score.js'; +import type { SelectionResult } from './select.js'; /** * Stable JSON. Object keys sorted by UTF-16 code unit; arrays left in the order - * the miner produced, which is itself sorted. + * the producer emitted, which is itself sorted. * * `localeCompare` is deliberately not used — it varies with host locale and * would make the bytes machine-dependent, which is the failure this function * exists to prevent. + * + * `integersOnly` enforces the float prohibition. It is a parameter rather than + * a separate walker so that there is exactly one serializer and no second + * definition to drift. */ -function stableStringify(value: unknown): string { +function stableStringify(value: unknown, integersOnly = false): string { if (value === null) return 'null'; - if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`; + if (Array.isArray(value)) { + return `[${value.map((item) => stableStringify(item, integersOnly)).join(',')}]`; + } if (typeof value === 'object') { const entries = Object.entries(value as Record) .filter(([, v]) => v !== undefined) .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); - return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v)}`).join(',')}}`; + return `{${entries + .map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v, integersOnly)}`) + .join(',')}}`; } - if (typeof value === 'number' && !Number.isFinite(value)) { - // NaN and Infinity serialize to `null` under JSON.stringify, which would - // silently turn a computation bug into a plausible-looking absent value. - throw new Error(`serializeObservationSet: non-finite number ${value}`); + if (typeof value === 'number') { + if (!Number.isFinite(value)) { + // NaN and Infinity serialize to `null` under JSON.stringify, which would + // silently turn a computation bug into a plausible-looking absent value. + throw new Error(`stableStringify: non-finite number ${value}`); + } + if (integersOnly && !Number.isInteger(value)) { + throw new Error( + `stableStringify: ${value} is not an integer, and floats are prohibited in artifact-bound output`, + ); + } } return JSON.stringify(value) ?? 'null'; } @@ -45,3 +62,39 @@ function stableStringify(value: unknown): string { export function serializeObservationSet(observations: ObservationSet): string { return stableStringify(observations); } + +/** + * Serialize a scored set to stable bytes. + * + * Same guarantee and the same narrow scope, with one addition worth stating. + * `weightedSupport` is a double produced by `2 ** x`, whose precision + * ECMAScript leaves implementation-defined. Two runs on one engine agree + * exactly, and the test proves it. Two runs on + * *different* engines may differ in the last ulp, so the byte-identical claim + * is scoped to a fixed engine and is not a cross-platform hash. The ranking + * rule is required to key on integer counts precisely so that this float never + * decides an order. + */ +export function serializeScoredSet(scored: ScoredSet): string { + return stableStringify(scored); +} + +/** + * Serialize a selection result to stable bytes, with the float prohibition + * enforced rather than assumed. + * + * This is the only shape in this package that is bound for an artifact, so it + * is the only one where a float is a defect rather than a detail. Any + * non-integer number **throws**. It is not rounded and it is not dropped: + * rounding would invent precision the measurement does not have, and dropping + * would remove a field a reader was told to expect. Both are quieter than the + * bug, and quiet is the failure mode this package exists to avoid. + * + * Consequence, stated plainly: `weightedSupport` cannot appear here, which is + * why `SelectedPair` does not carry it. It stays in memory on the scored set, + * where `serializeScoredSet` will happily emit it for diagnostics that never + * reach `workspace.json`. + */ +export function serializeSelection(selection: SelectionResult): string { + return stableStringify(selection, true); +} diff --git a/packages/mining-core/vitest.config.ts b/packages/mining-core/vitest.config.ts index 6441ab4..71366e6 100644 --- a/packages/mining-core/vitest.config.ts +++ b/packages/mining-core/vitest.config.ts @@ -5,15 +5,18 @@ export default defineConfig({ // These tests drive real git against real repositories, which is the whole // point — a mocked `diff-tree` would prove nothing about extraction. The // cost is real too: v2.2.1's frozen parameters spend one `rev-parse` and - // one `diff-tree` per commit, sequentially, so a 44-commit history is a - // low-hundreds-of-milliseconds-per-commit walk and the 5s default expires + // one `diff-tree` per commit, sequentially, and the 5s default expires // partway through. // // Raising the budget rather than batching the git calls is deliberate. The // commands are quoted from the preregistration and REQ-001 verifies against // them verbatim; replacing them with a faster equivalent would make the - // extraction no longer the thing that was frozen. The runtime itself is - // REQ-010's question and Phase 3 has not started. + // extraction no longer the thing that was frozen. + // + // Measured on an Apple M4 Pro: a bound 500-transition window costs 7.3-8.2s + // with a short PATH and 27.2-29.9s with a 36-entry one, because Node + // re-resolves the `git` binary through PATH on all 1000 spawns. Extraction + // is spawn-bound, not git-bound. Scoring the events costs 1-24ms. testTimeout: 120_000, hookTimeout: 120_000, }, From 44d374b4dedfa7c61b14f06512d30dd751b3f508 Mon Sep 17 00:00:00 2001 From: Qwynn Marcelle Date: Tue, 11 Aug 2026 10:06:29 -0400 Subject: [PATCH 4/5] feat(cli): project mined co-change into the artifact, and preserve it (META-297 L1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit L1 turns a selection into `generated.coChange` and decides what an ordinary generation run does with it afterwards. The second half is the load-bearing part. ## Carry-forward, and why it exists The standard replaces `generated` wholesale each run. The ruling says history mining is explicit and opt-in, and that ordinary generation must not recompute history or fabricate freshness. Taken together the first erases what the second forbids recomputing, so `coChange` and `basisRevision` are the one part of the producer-owned section that is PRESERVED rather than rebuilt — not because they are manual, but because they derive from an input ordinary generation does not read. The working tree is scanned every run; the commit graph is not. Three failure modes, each producing a plausible-looking artifact: - drop: regenerating without carry-forward destroys mined evidence, and the result is indistinguishable from a repository that was never mined; - advance the pin: carrying observations forward while moving `basisRevision` to current HEAD re-attributes old counts to a commit they were never counted at — the numbers stay plausible and become false; - recompute: every run pays seconds to tens of seconds, and the artifact churns on every commit. Preservation is by object identity, not field-by-field rebuild: a rebuilt entry re-orders keys and changes bytes while staying structurally identical, so identity is what makes byte-for-byte real. A pinned empty array is preserved — under A-009 it is the positive finding "analyzed, found nothing", and dropping it would convert a real result into "never analyzed". Nothing is invented when no prior block exists. `generatedAt` records the generation run. `basisRevision` is the authoritative freshness and provenance pin for the observations, and the only field that says which commit they were counted at. ## Canonical order is established at projection, not upstream `score.ts` sorts pair endpoints with `<`, which is UTF-16 code unit order, while the ruling names UTF-8 byte order. The two disagree: U+1F600 is a surrogate pair beginning 0xD83D and sorts before U+E000 in UTF-16, while its UTF-8 encoding sorts after. Upstream order is fine for keying a map; it is not fine for bytes a second producer is compared against. `project` re-orders endpoints under the same `compareUtf8` the ranking uses, and endpoint reversal now yields identical bytes. ## Refusal rather than degradation A non-mined completeness state projects nothing — never an empty array, which under A-009 is a positive claim that the analysis ran. A refused mining pass falls back to carry-forward so a shallow clone cannot destroy earlier evidence. No derived value is stored, and the A-010 classification flag is omitted because this producer has no deterministic classifier. ## One accommodation, named `@workspacejson/spec@0.4.4` is the published package and its `CoChangeEntry` predates A-009 — it requires `rate` and has no `support` — so an observation entry is not assignable to it. Confined to a locally declared two-field `HistoryBlock` and ONE cast at the assembly boundary, both documented in place and deleted when the amended spec publishes. No `any`, no duplicated standard type, and `WorkspaceJsonValidator` is untouched and still runs on every artifact. ## Test boundaries `candidate-tests/` is excluded from this workspace's vitest run and executes against packed candidates in a disposable environment, because the observation form is rejected by the published dependencies this workspace legitimately pins. Excluding it keeps the two boundaries from contaminating each other; it is not a way of skipping it. Repo-native coverage is validator-free by construction. Gates: mining-core 97/97, cli 72/72, agents-audit-compat 44/44 (213 total); typecheck, build, architecture (91 files, 4 manifests, no boundary or clean-room violations) and package-docs all green. Runtime dependency set unchanged — standard-owned packages remain exact published versions; mining-core is a bundled devDependency. --- .../candidate-tests/l1-integration.test.mjs | 397 ++++++++++++++++++ packages/cli/package.json | 5 +- packages/cli/src/producer/generate.ts | 73 +++- .../producer/history-carry-forward.test.ts | 118 ++++++ .../cli/src/producer/history-carry-forward.ts | 159 +++++++ packages/cli/src/producer/history-mine.ts | 54 +++ packages/cli/vitest.config.ts | 16 + packages/mining-core/src/index.ts | 7 + packages/mining-core/src/project.test.ts | 182 ++++++++ packages/mining-core/src/project.ts | 138 ++++++ pnpm-lock.yaml | 3 + 11 files changed, 1149 insertions(+), 3 deletions(-) create mode 100644 packages/cli/candidate-tests/l1-integration.test.mjs create mode 100644 packages/cli/src/producer/history-carry-forward.test.ts create mode 100644 packages/cli/src/producer/history-carry-forward.ts create mode 100644 packages/cli/src/producer/history-mine.ts create mode 100644 packages/cli/vitest.config.ts create mode 100644 packages/mining-core/src/project.test.ts create mode 100644 packages/mining-core/src/project.ts diff --git a/packages/cli/candidate-tests/l1-integration.test.mjs b/packages/cli/candidate-tests/l1-integration.test.mjs new file mode 100644 index 0000000..1314cca --- /dev/null +++ b/packages/cli/candidate-tests/l1-integration.test.mjs @@ -0,0 +1,397 @@ +/** + * L1 candidate-contract tests — the packed-candidate integration boundary. + * + * These do NOT run in the CLI workspace, and that separation is deliberate. + * They exercise the observation form, which the published + * `@workspacejson/spec@0.4.4` and `@workspacejson/rules@0.4.4` reject: their + * schema predates ADR-003 A-009 and still requires `rate` while forbidding + * `support`. Running them in the workspace would either fail against + * legitimate published dependencies or force a compatibility shim around + * `WorkspaceJsonValidator` — and a shim is the one outcome that would make + * these tests green while proving nothing, because the validator is the + * contract under test. + * + * So they run in a disposable environment where `spec` and `rules` are the + * PACKED candidates built from a pinned `standard` revision, and the CLI is its + * own packed candidate. Everything asserted here is therefore **candidate + * interoperability**, not published-package interoperability. The published + * packages still reject this shape, and will until the freeze lifts. + * + * Plain Node test runner and plain JS on purpose: the environment has the three + * packages under test and nothing else, so nothing here depends on the CLI + * repository's toolchain. + */ +import assert from 'node:assert/strict'; +import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { after, before, describe, it } from 'node:test'; +import { generateWorkspaceJson } from '@workspacejson/cli'; +import { WorkspaceJsonValidator } from '@workspacejson/rules'; + +const BASIS = '3c9a0f14b7e25d8613af04c2e9b7d5081f6a2c3d'; +const NEWER_BASIS = 'a1b2c3d4e5f60718293a4b5c6d7e8f9012345678'; + +const observationEntry = (over = {}) => ({ + files: ['src/auth.ts', 'src/session.ts'], + support: 8, + occurrences: 24, + ...over, +}); + +function priorArtifact(over = {}) { + return { + manual: { fragileFiles: [{ path: 'src/auth.ts', reason: 'hand-authored, must survive regeneration' }] }, + generated: { + specVersion: '0.4', + generatedAt: '2026-06-01T00:00:00Z', + basisRevision: BASIS, + by: { name: '@workspacejson/cli', version: '0.5.2' }, + frameworkManifest: [], + fileIndex: {}, + coChange: [ + observationEntry(), + observationEntry({ files: ['a.ts', 'b.ts'], support: 3, occurrences: 9 }), + ], + ...over, + }, + agents: {}, + health: { intelligenceState: 'INSUFFICIENT_DATA', observationCount: 0, confidence: 0 }, + }; +} + +// ─── The environment itself is the first thing under test ─────────────────── +// If the installed graph silently resolved a registry copy, every case below +// would be measuring the wrong contract while looking green. The version +// numbers cannot distinguish them — the candidates carry the same 0.4.4 as the +// published packages — so this checks the SHAPE OF THE SCHEMA instead, which is +// the thing that actually differs. +describe('the packed candidate environment resolves the contract under test', () => { + it('the installed spec carries the A-009 observation form', async () => { + const { workspaceJsonSchema } = await import('@workspacejson/spec'); + const item = workspaceJsonSchema.properties.generated.properties.coChange.items; + const props = Object.keys(item.properties).sort(); + assert.ok(props.includes('support'), 'installed spec has no `support` — this is a pre-A-009 registry copy'); + assert.ok(props.includes('occurrences')); + }); + + it('the installed spec carries the A-010 widening', async () => { + const { workspaceJsonSchema } = await import('@workspacejson/spec'); + const item = workspaceJsonSchema.properties.generated.properties.coChange.items; + assert.ok( + !item.required.includes('generated'), + 'installed spec still requires the classification flag — this is a pre-A-010 registry copy', + ); + const legacy = item.oneOf.find((branch) => branch.title === 'legacy form'); + assert.ok(legacy.required.includes('generated'), 'legacy branch lost its requirement'); + }); + + it('the validator the PRODUCER calls accepts the observation form', () => { + // The decisive check. `rules` bundles its own `spec` dependency, so a + // correct top-level `spec` proves nothing on its own — this asserts the + // graph the producer actually runs through. + const result = new WorkspaceJsonValidator().validate(priorArtifact()); + assert.equal(result.valid, true, `validator rejected the observation form: ${JSON.stringify(result.errors)}`); + }); +}); + +describe('ordinary generation and commit-history evidence', () => { + let root; + + const artifactPath = () => resolve(root, '.agents/workspace.json'); + const readArtifact = async () => JSON.parse(await readFile(artifactPath(), 'utf8')); + const generatedOf = async () => (await readArtifact()).generated; + + /** The exact bytes of the history block, which is what the contract is about. */ + const historyBytes = (generated) => + JSON.stringify({ basisRevision: generated.basisRevision, coChange: generated.coChange }); + + before(() => {}); + + const setup = async () => { + root = await mkdtemp(join(tmpdir(), 'wsj-l1-')); + await writeFile(join(root, 'AGENTS.md'), '# Test\n\nUse kebab-case for files.\n', 'utf8'); + await writeFile(join(root, 'package.json'), JSON.stringify({ name: 'fixture', version: '1.0.0' }), 'utf8'); + await mkdir(join(root, 'src'), { recursive: true }); + await writeFile(join(root, 'src/auth.ts'), 'export const auth = 1;\n', 'utf8'); + await mkdir(join(root, '.agents'), { recursive: true }); + await writeFile(artifactPath(), JSON.stringify(priorArtifact(), null, 2) + '\n', 'utf8'); + }; + + const teardown = async () => { + await rm(root, { recursive: true, force: true }); + }; + + it('CASE 1 — a plain regeneration preserves the history block byte for byte', async () => { + await setup(); + try { + const before = historyBytes(await generatedOf()); + await generateWorkspaceJson(root); + const after = await generatedOf(); + assert.equal(historyBytes(after), before); + assert.equal(after.basisRevision, BASIS); + assert.equal(after.coChange.length, 2); + } finally { + await teardown(); + } + }); + + it('CASE 2 — the pin is never advanced, and counts are never re-attributed', async () => { + await setup(); + try { + await generateWorkspaceJson(root); + const after = await generatedOf(); + assert.equal(after.basisRevision, BASIS); + assert.notEqual(after.basisRevision, NEWER_BASIS); + } finally { + await teardown(); + } + }); + + it('CASE 3 — ordinary generation does not recompute history', async () => { + // The observations are FABRICATED: `src/session.ts` and `b.ts` do not exist + // in this repository, and it has no git history at all. Any real mining + // pass would produce something different — almost certainly nothing. Their + // survival is therefore positive proof that no mining ran, which a timing + // assertion could never establish. + await setup(); + try { + await generateWorkspaceJson(root); + const after = await generatedOf(); + assert.deepEqual(after.coChange[0].files, ['src/auth.ts', 'src/session.ts']); + assert.equal(after.coChange[0].support, 8); + assert.equal(after.coChange[0].occurrences, 24); + assert.deepEqual(after.coChange[1].files, ['a.ts', 'b.ts']); + } finally { + await teardown(); + } + }); + + it('CASE 4 — a non-history generated field changes without touching the block', async () => { + await setup(); + try { + const before = historyBytes(await generatedOf()); + await writeFile(join(root, 'src/added.ts'), 'export const added = 2;\n', 'utf8'); + const result = await generateWorkspaceJson(root); + const after = await generatedOf(); + assert.equal(result.written, true); + assert.ok(Object.keys(after.fileIndex).includes('src/added.ts')); + assert.equal(historyBytes(after), before); + } finally { + await teardown(); + } + }); + + it('CASE 5 — --check reports no drift merely because history was not recomputed', async () => { + await setup(); + try { + await generateWorkspaceJson(root); + const checked = await generateWorkspaceJson(root, {}, { check: true }); + assert.equal(checked.drift, false); + assert.equal(checked.skipped, true); + } finally { + await teardown(); + } + }); + + it('CASE 6 — generatedAt may move while basisRevision stays put', async () => { + await setup(); + try { + await writeFile(join(root, 'src/added.ts'), 'export const added = 2;\n', 'utf8'); + await generateWorkspaceJson(root); + const after = await generatedOf(); + assert.notEqual(after.generatedAt, '2026-06-01T00:00:00Z'); + assert.equal(after.basisRevision, BASIS); + } finally { + await teardown(); + } + }); + + it('CASE 7 — with no prior block, ordinary generation invents nothing', async () => { + await setup(); + try { + await rm(artifactPath()); + await generateWorkspaceJson(root); + const after = await generatedOf(); + assert.equal('coChange' in after, false); + assert.equal('basisRevision' in after, false); + } finally { + await teardown(); + } + }); + + it('CASE 8 — manual evidence survives alongside preserved history', async () => { + await setup(); + try { + await generateWorkspaceJson(root); + const artifact = await readArtifact(); + assert.deepEqual(artifact.manual.fragileFiles, [ + { path: 'src/auth.ts', reason: 'hand-authored, must survive regeneration' }, + ]); + } finally { + await teardown(); + } + }); + + it('CASE 9 — the preserved artifact validates against the candidate schema', async () => { + await setup(); + try { + await writeFile(join(root, 'src/added.ts'), 'export const added = 2;\n', 'utf8'); + await generateWorkspaceJson(root); + const result = new WorkspaceJsonValidator().validate(await readArtifact()); + assert.equal(result.valid, true, JSON.stringify(result.errors)); + } finally { + await teardown(); + } + }); +}); + +// ─── Opt-in mining, against a real repository with real history ───────────── +describe('explicit mining writes a conforming block', () => { + let root; + + const git = (args, cwd) => execFileSync('git', args, { cwd, encoding: 'utf8' }); + + const build = async () => { + root = await mkdtemp(join(tmpdir(), 'wsj-mine-')); + git(['init', '-q', '-b', 'main'], root); + git(['config', 'user.email', 'fixture@example.invalid'], root); + git(['config', 'user.name', 'Fixture'], root); + await writeFile(join(root, 'AGENTS.md'), '# Fixture\n', 'utf8'); + await writeFile(join(root, 'package.json'), JSON.stringify({ name: 'mined', version: '1.0.0' }), 'utf8'); + await mkdir(join(root, 'src'), { recursive: true }); + + // Six commits that change the same two files together, and a third file + // that moves independently. The coupling is in the commit graph and nowhere + // else — neither file imports the other — which is the case the whole + // standard rests on. + for (let i = 0; i < 6; i += 1) { + await writeFile(join(root, 'src/auth.ts'), `export const auth = ${i};\n`, 'utf8'); + await writeFile(join(root, 'src/session.ts'), `export const session = ${i};\n`, 'utf8'); + git(['add', '-A'], root); + git(['commit', '-q', '-m', `paired change ${i}`], root); + } + for (let i = 0; i < 3; i += 1) { + await writeFile(join(root, 'src/lonely.ts'), `export const lonely = ${i};\n`, 'utf8'); + git(['add', '-A'], root); + git(['commit', '-q', '-m', `solo change ${i}`], root); + } + }; + + after(async () => { + if (root) await rm(root, { recursive: true, force: true }); + }); + + it('mines, projects, and produces a schema-valid artifact', async () => { + await build(); + const result = await generateWorkspaceJson(root, {}, { mineHistory: true }); + const generated = result.content.generated; + + assert.ok(Array.isArray(generated.coChange), 'no coChange block was written'); + assert.match(generated.basisRevision, /^[0-9a-f]{40}$|^[0-9a-f]{64}$/); + assert.equal(generated.basisRevision, git(['rev-parse', 'HEAD'], root).trim()); + + const validation = new WorkspaceJsonValidator().validate(result.content); + assert.equal(validation.valid, true, JSON.stringify(validation.errors)); + }); + + it('finds the no-import-edge coupling the thesis rests on', async () => { + const result = await generateWorkspaceJson(root, {}, { mineHistory: true }); + const pair = result.content.generated.coChange.find( + (entry) => entry.files.includes('src/auth.ts') && entry.files.includes('src/session.ts'), + ); + assert.ok(pair, 'the paired files were not reported as co-changing'); + assert.equal(pair.support, 6); + assert.ok(pair.occurrences >= pair.support); + }); + + it('emits no derived value and no unsupported classification', async () => { + const result = await generateWorkspaceJson(root, {}, { mineHistory: true }); + for (const entry of result.content.generated.coChange) { + assert.equal('rate' in entry, false, 'a derived rate was stored'); + assert.equal('generated' in entry, false, 'an unsupported classification was asserted'); + assert.deepEqual(Object.keys(entry).sort(), ['files', 'occurrences', 'support']); + } + }); + + it('orders every pair canonically in ascending UTF-8 byte order', async () => { + const result = await generateWorkspaceJson(root, {}, { mineHistory: true }); + const encoder = new TextEncoder(); + const compareUtf8 = (a, b) => Buffer.compare(Buffer.from(encoder.encode(a)), Buffer.from(encoder.encode(b))); + for (const entry of result.content.generated.coChange) { + assert.ok(compareUtf8(entry.files[0], entry.files[1]) <= 0, `pair not canonically ordered: ${entry.files}`); + } + }); + + it('is deterministic — two runs at the same revision agree byte for byte', async () => { + const first = await generateWorkspaceJson(root, {}, { mineHistory: true }); + const second = await generateWorkspaceJson(root, {}, { mineHistory: true }); + const block = (r) => + JSON.stringify({ basisRevision: r.content.generated.basisRevision, coChange: r.content.generated.coChange }); + assert.equal(block(first), block(second)); + }); + + it('a subsequent ORDINARY run preserves what mining wrote', async () => { + const mined = await generateWorkspaceJson(root, {}, { mineHistory: true }); + const minedBlock = JSON.stringify(mined.content.generated.coChange); + const ordinary = await generateWorkspaceJson(root); + assert.equal(JSON.stringify(ordinary.content.generated.coChange), minedBlock); + assert.equal(ordinary.content.generated.basisRevision, mined.content.generated.basisRevision); + }); + + it('ORDINARY GENERATION DOES NOT RECOMPUTE, even where mining would succeed', async () => { + // The decisive case, and the reason it has to be here rather than in the + // no-history fixtures above: where the repository has no commit graph, + // mining refuses and falls back to carry-forward, so a producer that + // wrongly mined on every run would be indistinguishable from one that + // never did. Those cases cannot detect recomputation at all. + // + // Here the graph exists AND has moved since the block was written, so the + // two behaviours diverge: preserving keeps the old pin and the old counts, + // recomputing advances the pin and changes the counts. Only one of those + // can be true of the artifact afterwards. + const mined = await generateWorkspaceJson(root, {}, { mineHistory: true }); + const pinnedAt = mined.content.generated.basisRevision; + const blockBefore = JSON.stringify(mined.content.generated.coChange); + + // Move the graph: four more commits pairing a DIFFERENT set of files, which + // a fresh mining pass would certainly report and the stored block cannot. + for (let i = 0; i < 4; i += 1) { + await writeFile(join(root, 'src/alpha.ts'), `export const alpha = ${i};\n`, 'utf8'); + await writeFile(join(root, 'src/beta.ts'), `export const beta = ${i};\n`, 'utf8'); + git(['add', '-A'], root); + git(['commit', '-q', '-m', `new pairing ${i}`], root); + } + const movedHead = git(['rev-parse', 'HEAD'], root).trim(); + assert.notEqual(movedHead, pinnedAt, 'fixture error: HEAD did not move'); + + const ordinary = await generateWorkspaceJson(root); + + // The pin must still name the commit the counts were taken at. + assert.equal( + ordinary.content.generated.basisRevision, + pinnedAt, + 'ordinary generation advanced the basis pin — the counts now claim a revision they were never counted at', + ); + assert.equal( + JSON.stringify(ordinary.content.generated.coChange), + blockBefore, + 'ordinary generation rewrote the co-change block — history was recomputed', + ); + // And the new pairing must be absent, because nothing re-read the graph. + const newPair = ordinary.content.generated.coChange.find( + (entry) => entry.files.includes('src/alpha.ts') && entry.files.includes('src/beta.ts'), + ); + assert.equal(newPair, undefined, 'a pair only visible to a fresh mining pass appeared in an ordinary run'); + + // Explicit mining, by contrast, DOES pick it up — which proves the previous + // assertions measured opt-in behaviour rather than a broken miner. + const remined = await generateWorkspaceJson(root, {}, { mineHistory: true }); + assert.equal(remined.content.generated.basisRevision, movedHead); + const reminedPair = remined.content.generated.coChange.find( + (entry) => entry.files.includes('src/alpha.ts') && entry.files.includes('src/beta.ts'), + ); + assert.ok(reminedPair, 'explicit mining failed to observe the new pairing'); + }); +}); diff --git a/packages/cli/package.json b/packages/cli/package.json index 83b940e..3c92198 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "@workspacejson/cli", "version": "0.5.2", - "description": "The workspace.json producer — scans a repository and generates .agents/workspace.json deterministically, preserving human-authored manual evidence.", + "description": "The workspace.json producer \u2014 scans a repository and generates .agents/workspace.json deterministically, preserving human-authored manual evidence.", "license": "Apache-2.0", "author": "workspace.json contributors", "homepage": "https://workspacejson.dev", @@ -66,8 +66,9 @@ }, "devDependencies": { "@types/node": "22.19.17", - "typescript": "^5.4.0", + "@workspacejson/mining-core": "workspace:*", "tsup": "^8.0.0", + "typescript": "^5.4.0", "vitest": "^1.6.0" } } diff --git a/packages/cli/src/producer/generate.ts b/packages/cli/src/producer/generate.ts index fb65ea6..eda2f9f 100644 --- a/packages/cli/src/producer/generate.ts +++ b/packages/cli/src/producer/generate.ts @@ -21,6 +21,30 @@ import type { RuleContext } from '@workspacejson/rules'; import { DEFAULT_PRODUCER_CONFIG, detectCiProvider, type ProducerConfig } from './config.js'; import { buildFileIndex, buildFrameworkManifest } from './evidence.js'; import { findAgentsMdPath, readTextOrEmpty } from './fs.js'; +import { carryForwardHistory, type PreservedHistory } from './history-carry-forward.js'; +import { mineHistoryBlock } from './history-mine.js'; + +/** + * A history block from either route: freshly mined, or carried forward. + * + * Declared locally because the two sources carry different TYPES for the same + * runtime shape, and the mismatch is a fact about the dependency rather than + * about this code. `@workspacejson/spec@0.4.4` is the published package, and + * its `CoChangeEntry` predates ADR-003 A-009: it still requires `rate` and + * knows nothing of `support`. So an observation-form entry — exactly what this + * producer now emits — is not assignable to the published type, and cannot be + * until the amended spec is published. + * + * The runtime contract is unaffected and is NOT relaxed anywhere: the artifact + * still goes through `WorkspaceJsonValidator` unmodified, and the candidate + * conformance suite runs it against the amended schema. This declaration + * narrows a compile-time gap in a stale type; it does not widen what the + * producer will accept or emit. It is deleted when the amended spec publishes. + */ +interface HistoryBlock { + basisRevision: string; + coChange: readonly unknown[]; +} const _require = createRequire(import.meta.url); @@ -129,7 +153,22 @@ export async function writeWorkspaceAtomically(outputPath: string, content: Work export async function generateWorkspaceJson( repoRoot: string, config: Partial = {}, - options: { dryRun?: boolean; check?: boolean; force?: boolean; producer?: ProducerIdentity; commandName?: string } = {}, + options: { + dryRun?: boolean; + check?: boolean; + force?: boolean; + producer?: ProducerIdentity; + commandName?: string; + /** + * Read the commit graph and rewrite `generated.coChange`. + * + * Off by default, and that default is the contract rather than a + * convenience: mining a bounded window costs seconds to tens of seconds, + * and a producer that recomputed history on every ordinary run would make + * the artifact churn on every commit. See history-carry-forward.ts. + */ + mineHistory?: boolean; + } = {}, ): Promise { const resolvedRoot = resolve(repoRoot); const fullConfig: ProducerConfig = { ...DEFAULT_PRODUCER_CONFIG, ...config }; @@ -208,6 +247,20 @@ export async function generateWorkspaceJson( } } } + // Commit-history evidence enters the artifact by exactly one of two routes, + // and never both. Mining is EXPLICIT: `mineHistory` is off unless a caller + // asked for it, so an ordinary run reads the working tree and nothing else. + // + // The order matters. A refused mining pass falls back to carry-forward rather + // than to nothing: a shallow clone or a git failure must not destroy evidence + // an earlier successful pass recorded. What it must never do is silently + // present the old block as a fresh result — it cannot, because the block + // carries its own `basisRevision` and that pin is what a reader compares. + const minedHistory = options.mineHistory === true ? await mineHistoryBlock(resolvedRoot) : undefined; + const preservedHistory = carryForwardHistory(existing); + const history: HistoryBlock | undefined = + minedHistory ?? (preservedHistory.preserved ? preservedHistory.history : undefined); + const workspace: WorkspaceJsonV4 = { manual: existing?.manual ?? {}, generated: { @@ -251,6 +304,24 @@ export async function generateWorkspaceJson( scannedAt: (existing?.generated.hygiene as { scannedAt?: string } | undefined)?.scannedAt ?? now, }, + // Commit-history evidence is PRESERVED, never rebuilt, by ordinary + // generation — see history-carry-forward.ts for why this one part of the + // producer-owned section is carried rather than regenerated. + // + // The values are spliced in as the objects parsed from the prior + // artifact, so the bytes are unchanged. Nothing here reads the commit + // graph: no mining, no pin advance, no re-attribution of old counts to a + // newer revision. If nothing conforming was preserved, both keys stay + // absent — ordinary generation never invents a history block, and an + // absent block correctly reads as "not analyzed". + ...(history === undefined + ? {} + : { + basisRevision: history.basisRevision, + // See HistoryBlock: the published 0.4.4 type cannot describe an + // observation-form entry. The value is validated at runtime. + coChange: history.coChange as NonNullable, + }), }, agents: {}, health: { diff --git a/packages/cli/src/producer/history-carry-forward.test.ts b/packages/cli/src/producer/history-carry-forward.test.ts new file mode 100644 index 0000000..6add057 --- /dev/null +++ b/packages/cli/src/producer/history-carry-forward.test.ts @@ -0,0 +1,118 @@ +/** + * Carry-forward semantics — the pure decision function. + * + * Repo-native and validator-free by construction. This file runs inside the CLI + * workspace against its legitimate published `@workspacejson/spec@0.4.4` and + * `@workspacejson/rules@0.4.4` dependencies, and must keep passing there, so it + * touches nothing that needs the amended schema. The end-to-end cases — which + * DO need a validator that accepts the observation form — live in + * `candidate-tests/` and run only in the packed-candidate environment. + * + * Every case here is written so that removing the behaviour it covers makes it + * fail. That is the whole value: the three failure modes carry-forward exists + * to prevent (drop, advance the pin, recompute) all produce a *plausible* + * artifact, so nothing about the output looks wrong when they happen. Only a + * test that knows what the previous artifact said can tell. + */ +import { describe, expect, it } from 'vitest'; +import { CarryForwardRefusal, carryForwardHistory } from './history-carry-forward.js'; + +const BASIS = '3c9a0f14b7e25d8613af04c2e9b7d5081f6a2c3d'; + +const observationEntry = (over: Record = {}) => ({ + files: ['src/auth.ts', 'src/session.ts'], + support: 8, + occurrences: 24, + ...over, +}); + +/** A prior artifact carrying mined evidence, as `generate` would find on disk. */ +function priorArtifact(over: Record = {}): Record { + return { + manual: { fragileFiles: ['src/auth.ts'] }, + generated: { + specVersion: '0.4', + generatedAt: '2026-06-01T00:00:00Z', + basisRevision: BASIS, + by: { name: '@workspacejson/cli', version: '0.5.2' }, + frameworkManifest: [], + fileIndex: {}, + coChange: [observationEntry(), observationEntry({ files: ['a.ts', 'b.ts'], support: 3, occurrences: 9 })], + ...over, + }, + agents: {}, + health: { intelligenceState: 'INSUFFICIENT_DATA', observationCount: 0, confidence: 0 }, + }; +} + +describe('carryForwardHistory — what ordinary generation preserves', () => { + it('preserves a conforming observation block and its pin', () => { + const result = carryForwardHistory(priorArtifact() as never); + expect(result.preserved).toBe(true); + if (!result.preserved) return; + expect(result.history.basisRevision).toBe(BASIS); + expect(result.history.coChange).toHaveLength(2); + }); + + it('passes the parsed entries THROUGH rather than rebuilding them', () => { + // Byte-for-byte preservation is the contract. Rebuilding an entry field by + // field would re-order its keys and change the serialized bytes even though + // the value is structurally identical, so identity of the array elements is + // the property that actually guarantees it. + const prior = priorArtifact(); + const original = (prior['generated'] as Record)['coChange'] as unknown[]; + const result = carryForwardHistory(prior as never); + expect(result.preserved).toBe(true); + if (!result.preserved) return; + expect(result.history.coChange[0]).toBe(original[0]); + expect(result.history.coChange[1]).toBe(original[1]); + }); + + it('preserves a PINNED EMPTY array — a positive finding, not an absence', () => { + // "The analysis ran and found no qualifying pairs" is evidence. Dropping it + // would silently convert it into "never analyzed". + const result = carryForwardHistory(priorArtifact({ coChange: [] }) as never); + expect(result.preserved).toBe(true); + }); + + it('refuses when there is no prior artifact — never invents a block', () => { + const result = carryForwardHistory(undefined); + expect(result.preserved).toBe(false); + if (result.preserved) return; + expect(result.refusal).toBe(CarryForwardRefusal.NO_PRIOR_BLOCK); + }); + + it('refuses a legacy rate entry rather than perpetuating it', () => { + const result = carryForwardHistory( + priorArtifact({ coChange: [{ files: ['a.ts', 'b.ts'], rate: 0.8, occurrences: 9, generated: false }] }) as never, + ); + expect(result.preserved).toBe(false); + if (result.preserved) return; + expect(result.refusal).toBe(CarryForwardRefusal.NOT_OBSERVATION_FORM); + }); + + it('refuses an observation block whose pin is symbolic or abbreviated', () => { + for (const basisRevision of ['HEAD', 'main', BASIS.slice(0, 7), BASIS.toUpperCase()]) { + const result = carryForwardHistory(priorArtifact({ basisRevision }) as never); + expect(result.preserved).toBe(false); + if (result.preserved) continue; + expect(result.refusal).toBe(CarryForwardRefusal.NO_CONFORMING_BASIS); + } + }); + + it('carries an entry that omits the A-010 classification flag', () => { + // The shape this producer emits. Treating the absent flag as malformed + // would refuse to carry forward exactly its own output. + const result = carryForwardHistory(priorArtifact() as never); + expect(result.preserved).toBe(true); + if (!result.preserved) return; + expect('generated' in (result.history.coChange[0] as object)).toBe(false); + }); + + it('refuses a block violating support <= occurrences', () => { + const result = carryForwardHistory( + priorArtifact({ coChange: [observationEntry({ support: 30, occurrences: 24 })] }) as never, + ); + expect(result.preserved).toBe(false); + }); +}); diff --git a/packages/cli/src/producer/history-carry-forward.ts b/packages/cli/src/producer/history-carry-forward.ts new file mode 100644 index 0000000..3d6f3f5 --- /dev/null +++ b/packages/cli/src/producer/history-carry-forward.ts @@ -0,0 +1,159 @@ +/** + * Carry-forward of commit-history evidence across ordinary generation. + * + * This is the load-bearing rule of L1, and it exists because two obligations + * collide. + * + * The standard says `generated` is producer-owned and replaced wholesale on + * each regeneration. The ruling says history mining is **explicit and opt-in**, + * and that ordinary generation must not recompute history or fabricate + * freshness. Taken naively, the first erases what the second forbids + * recomputing: run `generate` after mining and the observations are gone. + * + * So `generated.coChange` and `generated.basisRevision` are the one part of the + * producer-owned section that ordinary generation **preserves rather than + * rebuilds**. Not because they are manual — they are machine-derived — but + * because they are derived from an input ordinary generation does not read. + * The working tree is scanned every run; the commit graph is not. + * + * Three failure modes this is written against, all of which produce a + * plausible-looking artifact: + * + * - **Drop.** Regenerating without carry-forward silently destroys mined + * evidence, and the resulting empty/absent block is indistinguishable from a + * repository that was never mined. + * - **Advance the pin.** Carrying the observations forward while moving + * `basisRevision` to current HEAD re-attributes old counts to a commit they + * were never counted at. The numbers stay plausible and become false. + * - **Recompute.** Mining during ordinary generation makes every run pay + * seconds-to-tens-of-seconds, and makes the artifact churn on every commit — + * the exact `generate --check` failure the raw-count amendment removed. + * + * **`generatedAt` is not evidence about this block.** It records when the + * ordinary generation run happened. `basisRevision` is the authoritative + * freshness and provenance pin for the co-change observations, and it is the + * only field that says which commit they were counted at. A run that refreshes + * `fileIndex` moves `generatedAt` and must leave `basisRevision` exactly where + * it was; a reader comparing `generatedAt` to the repository's current revision + * learns nothing about whether the history block is stale, and must compare + * `basisRevision` instead. + */ +import type { WorkspaceJsonV4 } from '@workspacejson/spec'; + +/** A full-length lowercase Git object name. SHA-1 or SHA-256, never symbolic. */ +const OBJECT_NAME = /^[0-9a-f]{40}$|^[0-9a-f]{64}$/; + +/** + * What ordinary generation carries forward, verbatim. + * + * `coChange` is deliberately `readonly unknown[]` rather than the published + * `CoChangeEntry[]`. Two reasons, and the second is the load-bearing one. + * + * These entries are passed through untouched — nothing here reads a field off + * them, so a precise element type would buy nothing. And the published + * `@workspacejson/spec@0.4.4` type predates ADR-003 A-009: it requires `rate` + * and has no `support`, so calling a preserved observation entry a + * `CoChangeEntry` would be a false claim in the type system rather than a + * convenience. The one place the stale type has to be accommodated is the + * assembly boundary in `generate.ts`, and it stays the only place. + */ +export interface PreservedHistory { + basisRevision: string; + coChange: readonly unknown[]; +} + +/** Why nothing was carried forward. Absence is reported, never inferred away. */ +export enum CarryForwardRefusal { + /** No prior artifact, or it had no `coChange`. Nothing to preserve. */ + NO_PRIOR_BLOCK = 'NO_PRIOR_BLOCK', + /** Present but not the observation form — legacy `rate`, or malformed. */ + NOT_OBSERVATION_FORM = 'NOT_OBSERVATION_FORM', + /** Observation form without a conforming pin. Reads as legacy/unknown. */ + NO_CONFORMING_BASIS = 'NO_CONFORMING_BASIS', +} + +export type CarryForwardResult = + | { preserved: true; history: PreservedHistory } + | { preserved: false; refusal: CarryForwardRefusal; detail: string }; + +function isObservationEntry(value: unknown): boolean { + if (value === null || typeof value !== 'object') return false; + const entry = value as Record; + + // The form discriminator is `support` versus `rate`, and nothing else. + // `generated` plays no part in it — A-010 made that flag optional, so an + // entry carrying counts and no classification is unambiguously observation + // form, and refusing to carry it forward would discard exactly the shape + // this producer emits. + if ('rate' in entry) return false; + if (!Number.isInteger(entry['support']) || (entry['support'] as number) < 0) return false; + if (!Number.isInteger(entry['occurrences']) || (entry['occurrences'] as number) < 1) return false; + if ((entry['support'] as number) > (entry['occurrences'] as number)) return false; + + const files = entry['files']; + if (!Array.isArray(files) || files.length !== 2) return false; + if (!files.every((path) => typeof path === 'string' && path.length > 0)) return false; + + return true; +} + +/** + * Decide what an ordinary generation run preserves from the prior artifact. + * + * Validating rather than trusting is the point. A block that would not survive + * schema validation must not be carried into a fresh artifact, because doing so + * would let one bad mining run poison every subsequent generation — and the + * producer refuses to overwrite an invalid artifact precisely so that cannot + * happen silently. + * + * The returned values are the parsed objects themselves, not copies rebuilt + * field by field. Rebuilding would re-order keys and change bytes; the contract + * is byte-for-byte preservation, so the original values are passed through. + */ +export function carryForwardHistory(existing: WorkspaceJsonV4 | undefined): CarryForwardResult { + if (existing === undefined) { + return { + preserved: false, + refusal: CarryForwardRefusal.NO_PRIOR_BLOCK, + detail: 'no prior artifact', + }; + } + + const generated = existing.generated as unknown as Record; + const coChange = generated['coChange']; + + if (!Array.isArray(coChange)) { + return { + preserved: false, + refusal: CarryForwardRefusal.NO_PRIOR_BLOCK, + detail: 'the prior artifact carries no coChange array', + }; + } + + // An empty array is preserved only when pinned. Unpinned it means + // legacy/unknown and asserts nothing, so there is nothing to preserve; + // pinned it is a positive finding — the analysis ran and found no qualifying + // pairs — and dropping it would convert a real result into "never analyzed". + if (!coChange.every(isObservationEntry)) { + return { + preserved: false, + refusal: CarryForwardRefusal.NOT_OBSERVATION_FORM, + detail: + 'the prior coChange block is not homogeneous observation form; it is legacy or malformed and this producer does not perpetuate it', + }; + } + + const basisRevision = generated['basisRevision']; + if (typeof basisRevision !== 'string' || !OBJECT_NAME.test(basisRevision)) { + return { + preserved: false, + refusal: CarryForwardRefusal.NO_CONFORMING_BASIS, + detail: + 'the prior coChange block carries no full-length lowercase object name, so it cannot be recounted against and reads as legacy/unknown', + }; + } + + // The elements are passed through untouched, which is what makes the + // byte-for-byte guarantee real — see the note on PreservedHistory. + return { preserved: true, history: { basisRevision, coChange } }; +} diff --git a/packages/cli/src/producer/history-mine.ts b/packages/cli/src/producer/history-mine.ts new file mode 100644 index 0000000..e783232 --- /dev/null +++ b/packages/cli/src/producer/history-mine.ts @@ -0,0 +1,54 @@ +/** + * The explicit, opt-in commit-history pass. + * + * This is the only code path in the producer that reads the commit graph, and + * it runs only when a caller asks for it. Everything else in `generate` reads + * the working tree. That split is the whole reason `generated.coChange` is + * carried forward rather than rebuilt — see history-carry-forward.ts. + * + * The pipeline is `mine → score → select → project`, each stage pure with + * respect to the one before it, so the extracted events and the uncapped scored + * set stay auditable behind whatever the selection capped. + * + * **A refusal returns `undefined`, and that is not the same as an empty + * result.** A shallow clone, an absent history or a Git failure produce nothing + * here, and the caller falls back to whatever the previous artifact recorded. + * Writing an empty `coChange` in those cases would be the worst available + * outcome: under A-009 a *pinned* empty array is a positive finding — "the + * analysis ran at this revision and found no qualifying pairs" — so emitting + * one for a repository that could not be analyzed would state a result nobody + * measured. + */ +import { mine, project, score, select } from '@workspacejson/mining-core'; +import type { ProjectedHistory } from '@workspacejson/mining-core'; + +/** + * Mine, score, select and project. Returns `undefined` when the repository + * cannot honestly produce a block. + * + * Errors are not swallowed silently — they are converted into the same absence + * a shallow clone produces, and the reason is surfaced on the returned + * diagnostics rather than discarded. + */ +export async function mineHistoryBlock( + repoRoot: string, + diagnostics: { refusal?: string } = {}, +): Promise { + let projected; + try { + projected = project(select(score(await mine(repoRoot)))); + } catch (error) { + // A Git invocation that fails is an absence of evidence, never a zero. The + // message is kept so a caller can report *why* nothing was mined instead of + // reporting that nothing was found. + diagnostics.refusal = `mining failed: ${error instanceof Error ? error.message : String(error)}`; + return undefined; + } + + if (!projected.projected) { + diagnostics.refusal = `${projected.refusal}: ${projected.detail}`; + return undefined; + } + + return projected.history; +} diff --git a/packages/cli/vitest.config.ts b/packages/cli/vitest.config.ts new file mode 100644 index 0000000..302dc3c --- /dev/null +++ b/packages/cli/vitest.config.ts @@ -0,0 +1,16 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + // `candidate-tests/` is deliberately outside this workspace's suite. + // + // Those files exercise the observation form, which the published + // `@workspacejson/spec@0.4.4` and `@workspacejson/rules@0.4.4` reject — + // their schema predates ADR-003 A-009. They run against PACKED candidate + // builds in a disposable environment, on the Node test runner, so that this + // workspace keeps passing against its legitimate published dependencies. + // Excluding them here is what keeps the two boundaries from contaminating + // each other; it is not a way of skipping them. + exclude: ['**/node_modules/**', '**/dist/**', 'candidate-tests/**'], + }, +}); diff --git a/packages/mining-core/src/index.ts b/packages/mining-core/src/index.ts index f12d7f0..18b8b36 100644 --- a/packages/mining-core/src/index.ts +++ b/packages/mining-core/src/index.ts @@ -80,3 +80,10 @@ export { serializeScoredSet, serializeSelection, } from './serialize.js'; +export { + type ProjectedCoChangeEntry, + type ProjectedHistory, + type ProjectionResult, + ProjectionRefusal, + project, +} from './project.js'; diff --git a/packages/mining-core/src/project.test.ts b/packages/mining-core/src/project.test.ts new file mode 100644 index 0000000..07e25bf --- /dev/null +++ b/packages/mining-core/src/project.test.ts @@ -0,0 +1,182 @@ +/** + * L1 projection — repo-native, and pure by construction. + * + * Nothing here touches a schema, a validator or a filesystem, which is why it + * runs inside the workspace against its ordinary dependencies. The projection's + * *conformance* to the amended schema is a different claim, measured in the + * packed-candidate environment; what is measured here is the shape it produces + * and the rules it applies to produce it. + */ +import { describe, expect, it } from 'vitest'; +import { CompletenessReason, CompletenessState, completeness } from './completeness.js'; +import { ProjectionRefusal, project } from './project.js'; +import type { SelectionResult } from './select.js'; +import { RANKING_RULE } from './select.js'; + +const BASIS = '3c9a0f14b7e25d8613af04c2e9b7d5081f6a2c3d'; + +function selection( + pairs: Array<{ files: [string, string]; support: number; occurrences: number }>, + over: Partial = {}, +): SelectionResult { + return { + l0SelectionVersion: 1, + completeness: completeness( + pairs.length > 0 + ? CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED + : CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP, + CompletenessReason.MINED, + 'fixture', + ), + scoringBasis: { + weightingVersion: 'v2.2.1', + sizeWeightNumerator: 1, + positionDecayHalfLife: 1, + maxScoredFileCount: 50, + basisRevision: BASIS, + } as SelectionResult['scoringBasis'], + exclusions: {} as SelectionResult['exclusions'], + receipt: { + minSupport: 3, + pairsBeforeCap: pairs.length, + pairsEmitted: pairs.length, + cap: 50, + rankingRule: RANKING_RULE, + capBound: false, + }, + pairs, + ...over, + } as SelectionResult; +} + +describe('project — the artifact shape', () => { + it('emits exactly files, support and occurrences', () => { + const result = project(selection([{ files: ['a.ts', 'b.ts'], support: 4, occurrences: 10 }])); + expect(result.projected).toBe(true); + if (!result.projected) return; + expect(Object.keys(result.history.coChange[0]!).sort()).toEqual([ + 'files', + 'occurrences', + 'support', + ]); + }); + + it('stores no derived value — no rate, probability, lift or ranking', () => { + const result = project(selection([{ files: ['a.ts', 'b.ts'], support: 4, occurrences: 10 }])); + if (!result.projected) throw new Error('expected a projection'); + const entry = result.history.coChange[0] as unknown as Record; + for (const forbidden of ['rate', 'probability', 'lift', 'confidence', 'rank', 'weightedSupport']) { + expect(forbidden in entry).toBe(false); + } + }); + + it('omits the A-010 classification flag rather than asserting a constant', () => { + // The pre-A-010 producer emitted `generated: false` for every pair, which + // claimed that a lockfile and its manifest are a real source coupling. + // Absence is the honest output for a producer with no classifier. + const result = project( + selection([{ files: ['package.json', 'pnpm-lock.yaml'], support: 9, occurrences: 9 }]), + ); + if (!result.projected) throw new Error('expected a projection'); + expect('generated' in (result.history.coChange[0] as object)).toBe(false); + }); + + it('carries the basis pin at block level, never per entry', () => { + const result = project( + selection([ + { files: ['a.ts', 'b.ts'], support: 4, occurrences: 10 }, + { files: ['c.ts', 'd.ts'], support: 3, occurrences: 8 }, + ]), + ); + if (!result.projected) throw new Error('expected a projection'); + expect(result.history.basisRevision).toBe(BASIS); + for (const entry of result.history.coChange) { + expect('basisRevision' in (entry as object)).toBe(false); + } + }); + + it('preserves the selection ranking rather than re-sorting the array', () => { + // Re-sorting here would silently discard threshold-then-rank-then-cap. + const result = project( + selection([ + { files: ['z.ts', 'y.ts'], support: 9, occurrences: 12 }, + { files: ['a.ts', 'b.ts'], support: 3, occurrences: 8 }, + ]), + ); + if (!result.projected) throw new Error('expected a projection'); + expect(result.history.coChange[0]!.support).toBe(9); + expect(result.history.coChange[1]!.support).toBe(3); + }); +}); + +describe('project — canonical endpoint order', () => { + it('orders endpoints ascending, whichever way they arrive', () => { + const forward = project(selection([{ files: ['a.ts', 'b.ts'], support: 4, occurrences: 10 }])); + const reversed = project(selection([{ files: ['b.ts', 'a.ts'], support: 4, occurrences: 10 }])); + if (!forward.projected || !reversed.projected) throw new Error('expected projections'); + expect(forward.history.coChange[0]!.files).toEqual(['a.ts', 'b.ts']); + expect(reversed.history.coChange[0]!.files).toEqual(['a.ts', 'b.ts']); + }); + + it('ENDPOINT REVERSAL YIELDS IDENTICAL BYTES — the producer-profile obligation', () => { + const forward = project(selection([{ files: ['src/auth.ts', 'src/session.ts'], support: 8, occurrences: 24 }])); + const reversed = project(selection([{ files: ['src/session.ts', 'src/auth.ts'], support: 8, occurrences: 24 }])); + if (!forward.projected || !reversed.projected) throw new Error('expected projections'); + expect(JSON.stringify(forward.history)).toBe(JSON.stringify(reversed.history)); + }); + + it('uses UTF-8 BYTE order, not UTF-16 code unit order', () => { + // The two disagree here and only here-shaped cases: U+1F600 is a surrogate + // pair beginning 0xD83D, which sorts BEFORE U+E000 in UTF-16, while its + // UTF-8 encoding (F0 9F 98 80) sorts AFTER U+E000's (EE 80 80). A producer + // ordering with `<` would emit these two endpoints the other way round. + const emoji = 'src/\u{1F600}.ts'; + const privateUse = 'src/.ts'; + expect(emoji < privateUse).toBe(true); // UTF-16 says emoji first… + + const result = project(selection([{ files: [emoji, privateUse], support: 4, occurrences: 10 }])); + if (!result.projected) throw new Error('expected a projection'); + expect(result.history.coChange[0]!.files).toEqual([privateUse, emoji]); // …UTF-8 says otherwise + }); +}); + +describe('project — refusal rather than degradation', () => { + it('refuses a shallow clone, and does not emit an empty array', () => { + // A PINNED empty array is a positive finding under A-009: "the analysis ran + // and found nothing." Emitting one for a repository that could not be + // analyzed would state a result nobody measured. + const result = project( + selection([], { + completeness: completeness( + CompletenessState.NOT_MINED, + CompletenessReason.SHALLOW_CLONE, + 'shallow', + ), + }), + ); + expect(result.projected).toBe(false); + if (result.projected) return; + expect(result.refusal).toBe(ProjectionRefusal.NOT_MINED); + }); + + it('refuses when no basis pin exists', () => { + // The key is REMOVED rather than set to `undefined`: under + // `exactOptionalPropertyTypes` those are different states, and an unpinned + // selection is one whose basis is absent, not one carrying an explicit + // undefined. + const pinned = selection([{ files: ['a.ts', 'b.ts'], support: 4, occurrences: 10 }]); + const { scoringBasis: _dropped, ...unpinned } = pinned; + const result = project(unpinned as typeof pinned); + expect(result.projected).toBe(false); + if (result.projected) return; + expect(result.refusal).toBe(ProjectionRefusal.NO_BASIS_PIN); + }); + + it('DOES emit a pinned empty array when the analysis genuinely found nothing', () => { + const result = project(selection([])); + expect(result.projected).toBe(true); + if (!result.projected) return; + expect(result.history.coChange).toEqual([]); + expect(result.history.basisRevision).toBe(BASIS); + }); +}); diff --git a/packages/mining-core/src/project.ts b/packages/mining-core/src/project.ts new file mode 100644 index 0000000..55bb2fc --- /dev/null +++ b/packages/mining-core/src/project.ts @@ -0,0 +1,138 @@ +/** + * L1 projection — the only place a selection becomes artifact-shaped. + * + * Everything before this module answers "what does the commit graph say". + * This one answers "what does a producer write down", and those are different + * questions with different obligations. Three properties are load-bearing. + * + * 1. **Canonical endpoint order is established HERE, not upstream.** The + * scoring stage sorts pair endpoints with `<`, which is UTF-16 code unit + * order. The ruling names UTF-8 byte order, and the two disagree — a path + * containing U+1F600 sorts before one containing U+E000 under UTF-16 and + * after it under UTF-8. Upstream order is fine for keying a map, because + * only stability matters there. It is not fine for bytes a second producer + * is compared against, so the endpoints are re-ordered here under the same + * `compareUtf8` the ranking uses. Endpoint reversal must not change the + * output, and that is asserted rather than asserted-in-prose. + * + * 2. **No derived value is stored.** No rate, probability, lift, confidence or + * ranking. `support` and `occurrences` are counts; a reader who wants a + * ratio derives it. This is A-009, and the reason is that a continuous + * derived value moves on every commit and makes `generate --check` fire + * forever. + * + * 3. **The classification flag is omitted.** A-010 made + * `coChange[].generated` optional and defined absence as *unclassified*. + * This producer implements no deterministic tooling-coupling classifier, so + * it says nothing rather than emitting a constant `false` — which is + * exactly what the pre-A-010 producer did, and what made it assert that a + * lockfile and its manifest are a real source coupling. Absence here is a + * positive design decision, not an unfinished one. + * + * The projection refuses rather than degrades. A selection whose completeness + * is not a mined state yields nothing at all — not an empty array, which under + * A-009 is a *positive finding* that the analysis ran and found no qualifying + * pairs. Reporting "analyzed, nothing found" for a repository that was never + * successfully analyzed is the failure mode this whole package exists to + * avoid, one level up from `NOT_MINED / SHALLOW_CLONE`. + */ +import { CompletenessState } from './completeness.js'; +import { type SelectedPair, type SelectionResult, compareUtf8 } from './select.js'; + +/** + * A co-change entry exactly as it appears in `generated.coChange`. + * + * Three fields. `generated` is absent by design — see the module note — and + * `rate` is absent because the observation form forbids it. + */ +export interface ProjectedCoChangeEntry { + /** The pair, endpoints in ascending UTF-8 byte order. */ + files: [string, string]; + /** Distinct qualifying commits in which both files changed. */ + support: number; + /** Distinct qualifying commits in which at least one changed. The union. */ + occurrences: number; +} + +/** + * What a producer splices into `generated`. + * + * `basisRevision` is a `generated`-level sibling, never per item: repeating it + * per entry would admit a document whose entries were counted at different + * revisions. + */ +export interface ProjectedHistory { + basisRevision: string; + coChange: ProjectedCoChangeEntry[]; +} + +/** Why a projection produced nothing. Absence is reported, never smoothed. */ +export enum ProjectionRefusal { + /** Completeness is not a mined state — shallow clone, no history, or an error. */ + NOT_MINED = 'NOT_MINED', + /** Mined, but no basis pin. An unpinned block cannot be recounted against. */ + NO_BASIS_PIN = 'NO_BASIS_PIN', +} + +export type ProjectionResult = + | { projected: true; history: ProjectedHistory } + | { projected: false; refusal: ProjectionRefusal; detail: string }; + +/** + * Put a pair's endpoints in canonical order. + * + * `files` has set semantics, so this changes no meaning — it fixes the one + * spelling a producer is permitted to write, which is what makes two + * independent producers byte-comparable. + */ +function canonicalPair(files: readonly [string, string]): [string, string] { + return compareUtf8(files[0], files[1]) <= 0 ? [files[0], files[1]] : [files[1], files[0]]; +} + +function projectPair(pair: SelectedPair): ProjectedCoChangeEntry { + return { + files: canonicalPair(pair.files), + support: pair.support, + occurrences: pair.occurrences, + }; +} + +/** + * Project a selection into the artifact shape, or refuse and say why. + * + * Pure. The selection, the scored set behind it and the extracted events + * behind that are all untouched and remain auditable. + */ +export function project(selection: SelectionResult): ProjectionResult { + const state = selection.completeness.state; + const mined = + state === CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED || + state === CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP; + + if (!mined) { + return { + projected: false, + refusal: ProjectionRefusal.NOT_MINED, + detail: `completeness is ${state}; an artifact block would claim an analysis that did not happen`, + }; + } + + const basisRevision = selection.scoringBasis?.basisRevision; + if (basisRevision === undefined || basisRevision === '') { + return { + projected: false, + refusal: ProjectionRefusal.NO_BASIS_PIN, + detail: + 'the selection carries no basisRevision; an unpinned coChange block reads as legacy/unknown and asserts nothing', + }; + } + + // Entry order is the selection's ranked order, already deterministic and + // already applied threshold-then-rank-then-cap. This step re-orders endpoints + // WITHIN each pair and nothing else: re-sorting the array here would silently + // discard the ranking the selection rule exists to produce. + return { + projected: true, + history: { basisRevision, coChange: selection.pairs.map(projectPair) }, + }; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a94248b..ce23924 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -89,6 +89,9 @@ importers: '@types/node': specifier: 22.19.17 version: 22.19.17 + '@workspacejson/mining-core': + specifier: workspace:* + version: link:../mining-core tsup: specifier: ^8.0.0 version: 8.5.1(postcss@8.5.23)(typescript@5.9.3) From 12695a34dfa075b53c593e2c5a8f7747a8be22ac Mon Sep 17 00:00:00 2001 From: Qwynn Marcelle Date: Tue, 11 Aug 2026 13:07:16 -0400 Subject: [PATCH 5/5] fix(cli): report whether an explicit history refresh actually happened MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Greptile P1 on #20. When a caller passed `mineHistory: true` and mining could not complete, generation fell back to the previously recorded block and returned a successful result. The caller received the PREVIOUS revision's counts with no way to distinguish that from a refresh that ran — the artifact looks identical either way, and `basisRevision` only helps a reader who already suspects something is wrong. Falling back is correct and stays: destroying evidence over a shallow clone or a transient Git failure would be worse than keeping it. Falling back QUIETLY was the defect. `GenerateResult.historyRefresh` now reports the outcome: requested always true — the field is absent unless a refresh was asked for mined whether the commit graph was read and a new block produced preserved whether a prior block was carried instead refusal why nothing was mined; present iff `mined` is false Absence of the field means no refresh was requested. Reporting `mined: false` on an ordinary run would claim a refresh was attempted and failed, which is a different and untrue statement. The refusal reason was already being computed and thrown away: `mineHistoryBlock` accepts a diagnostics object for exactly this and generate.ts never passed one. It is passed now, so a caller can distinguish a shallow clone from a Git invocation failure rather than receiving a bare boolean. Test boundary, kept honest: the refusal cases run repo-native, because a refusal emits no `coChange` and therefore touches nothing the published 0.4.4 validator rejects. The fallback-with-preservation and completed-refresh cases need an observation-form artifact on disk, so they run in `candidate-tests/` against the amended schema rather than being weakened to fit the workspace. Watched-red, packed and clean-installed per mutation: drop the outcome field (restore the silent fallback) 20 passed / 2 failed report `mined: true` unconditionally 21 passed / 1 failed restored 22 passed / 0 failed Gates: mining-core 97/97, cli 76/76, agents-audit-compat 44/44 (217 total); typecheck, build, architecture (93 files, no boundary or clean-room violations) and package-docs green. Candidate-contract suite 22/22. --- .../candidate-tests/l1-integration.test.mjs | 71 +++++++++++++++++- packages/cli/src/index.ts | 2 +- packages/cli/src/producer/generate.ts | 65 +++++++++++++++-- .../producer/history-refresh-outcome.test.ts | 72 +++++++++++++++++++ 4 files changed, 202 insertions(+), 8 deletions(-) create mode 100644 packages/cli/src/producer/history-refresh-outcome.test.ts diff --git a/packages/cli/candidate-tests/l1-integration.test.mjs b/packages/cli/candidate-tests/l1-integration.test.mjs index 1314cca..6d085c3 100644 --- a/packages/cli/candidate-tests/l1-integration.test.mjs +++ b/packages/cli/candidate-tests/l1-integration.test.mjs @@ -248,6 +248,8 @@ describe('ordinary generation and commit-history evidence', () => { }); // ─── Opt-in mining, against a real repository with real history ───────────── +let minedRoot; + describe('explicit mining writes a conforming block', () => { let root; @@ -277,11 +279,13 @@ describe('explicit mining writes a conforming block', () => { git(['add', '-A'], root); git(['commit', '-q', '-m', `solo change ${i}`], root); } + minedRoot = root; }; - after(async () => { - if (root) await rm(root, { recursive: true, force: true }); - }); + // NOTE: `root` is intentionally NOT removed here. The refresh-outcome suite + // below reuses it as `minedRoot` — it is the only fixture in this file with a + // real commit graph, and rebuilding one per suite would triple the runtime. + // It is cleaned up at the end of the file instead. it('mines, projects, and produces a schema-valid artifact', async () => { await build(); @@ -395,3 +399,64 @@ describe('explicit mining writes a conforming block', () => { assert.ok(reminedPair, 'explicit mining failed to observe the new pairing'); }); }); + +// ─── Refresh outcome signalling (Greptile P1 on PR #20) ───────────────────── +describe('an explicitly requested refresh reports what it actually did', () => { + let root; + + const artifactPath = () => resolve(root, '.agents/workspace.json'); + + const setup = async () => { + root = await mkdtemp(join(tmpdir(), 'wsj-refresh-')); + await writeFile(join(root, 'AGENTS.md'), '# Fixture\n', 'utf8'); + await writeFile(join(root, 'package.json'), JSON.stringify({ name: 'fixture', version: '1.0.0' }), 'utf8'); + await mkdir(join(root, 'src'), { recursive: true }); + await writeFile(join(root, 'src/auth.ts'), 'export const auth = 1;\n', 'utf8'); + await mkdir(join(root, '.agents'), { recursive: true }); + await writeFile(artifactPath(), JSON.stringify(priorArtifact(), null, 2) + '\n', 'utf8'); + }; + const teardown = async () => { await rm(root, { recursive: true, force: true }); }; + + it('REFUSED REFRESH SAYS SO while keeping the block it fell back to', async () => { + // The P1 exactly: this repository has no commit graph, so an explicit + // refresh cannot complete. Falling back to the recorded block is correct — + // destroying evidence over a shallow clone would be worse. What must not + // happen is the caller receiving a successful-looking result carrying the + // PREVIOUS revision's counts with no way to tell. + await setup(); + try { + const result = await generateWorkspaceJson(root, {}, { mineHistory: true }); + + assert.equal(result.historyRefresh?.requested, true); + assert.equal(result.historyRefresh?.mined, false, 'a refusal was reported as a completed refresh'); + assert.equal(result.historyRefresh?.preserved, true); + assert.ok(result.historyRefresh?.refusal, 'no reason was given for the refusal'); + + // …and the evidence survived. + assert.equal(result.content.generated.basisRevision, BASIS); + assert.equal(result.content.generated.coChange.length, 2); + } finally { + await teardown(); + } + }); + + it('a COMPLETED refresh is distinguishable from a refused one', async () => { + // Same call, a repository that can actually be mined. If these two produced + // the same `historyRefresh`, the field would be decorative. + const mined = await generateWorkspaceJson(minedRoot, {}, { mineHistory: true }); + assert.equal(mined.historyRefresh?.mined, true); + assert.equal(mined.historyRefresh?.preserved, false); + assert.equal('refusal' in (mined.historyRefresh ?? {}), false, 'a successful refresh carried a refusal'); + }); + + it('an ordinary run reports NO refresh outcome at all', async () => { + // Absence means "none requested". Reporting `mined: false` here would claim + // a refresh was attempted and failed, which is a different and untrue thing. + const ordinary = await generateWorkspaceJson(minedRoot); + assert.equal(ordinary.historyRefresh, undefined); + }); +}); + +after(async () => { + if (minedRoot) await rm(minedRoot, { recursive: true, force: true }); +}); diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 220d0df..76fcc74 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -4,7 +4,7 @@ export { GenerateRefusalError, THIS_PRODUCER, } from './producer/generate.js'; -export type { GenerateResult, ProducerIdentity } from './producer/generate.js'; +export type { GenerateResult, HistoryRefreshOutcome, ProducerIdentity } from './producer/generate.js'; export { DEFAULT_PRODUCER_CONFIG, detectCiProvider } from './producer/config.js'; export type { ProducerConfig } from './producer/config.js'; export { findAgentsMdPath, readTextOrEmpty } from './producer/fs.js'; diff --git a/packages/cli/src/producer/generate.ts b/packages/cli/src/producer/generate.ts index eda2f9f..9ac54b4 100644 --- a/packages/cli/src/producer/generate.ts +++ b/packages/cli/src/producer/generate.ts @@ -87,6 +87,33 @@ export interface ProducerIdentity { export const THIS_PRODUCER: ProducerIdentity = { name: pkgName, version: pkgVersion }; +/** + * What an explicitly requested history refresh actually did. + * + * Present only when the caller passed `mineHistory: true`, so its absence means + * no refresh was requested rather than a refresh that failed. + * + * The distinction this exists to make: a refused refresh still produces a + * successful generation carrying the PREVIOUS revision's counts, because + * destroying evidence over a shallow clone or a transient Git failure would be + * worse than keeping it. That is the right behavior and it is also + * indistinguishable, from the artifact alone, from a refresh that completed. + * A caller that asked for fresh observations must be able to tell. + */ +export interface HistoryRefreshOutcome { + /** Always true — the field is absent unless a refresh was requested. */ + requested: true; + /** True when the commit graph was read and a new block produced. */ + mined: boolean; + /** True when mining refused and a prior block was carried instead. */ + preserved: boolean; + /** + * Why mining produced nothing. Present if and only if `mined` is false — + * e.g. a shallow clone, absent history, or a Git invocation failure. + */ + refusal?: string; +} + export interface GenerateResult { path: string; written: boolean; @@ -94,6 +121,12 @@ export interface GenerateResult { drift: boolean; preservedManual: boolean; invalidFileMoved?: string; + /** + * Present only when `mineHistory: true` was requested. Says whether the + * refresh completed, and why not when it did not — so a refused refresh + * cannot read as a successful one. + */ + historyRefresh?: HistoryRefreshOutcome; content: WorkspaceJsonV4; } @@ -253,14 +286,37 @@ export async function generateWorkspaceJson( // // The order matters. A refused mining pass falls back to carry-forward rather // than to nothing: a shallow clone or a git failure must not destroy evidence - // an earlier successful pass recorded. What it must never do is silently - // present the old block as a fresh result — it cannot, because the block - // carries its own `basisRevision` and that pin is what a reader compares. - const minedHistory = options.mineHistory === true ? await mineHistoryBlock(resolvedRoot) : undefined; + // an earlier successful pass recorded. + // + // But falling back QUIETLY is its own defect, and a worse one. A caller that + // asked for a refresh and received a successful-looking result carrying the + // previous revision's counts cannot tell that from a refresh that completed — + // the artifact looks the same either way, and `basisRevision` only helps a + // reader who already suspects something. So the outcome is reported on the + // result: `historyRefresh` says whether the refresh actually happened, and + // carries the refusal reason when it did not. + // + // The reason itself was already being computed and thrown away — the + // diagnostics object exists for exactly this and was not passed. + const refreshDiagnostics: { refusal?: string } = {}; + const minedHistory = + options.mineHistory === true ? await mineHistoryBlock(resolvedRoot, refreshDiagnostics) : undefined; const preservedHistory = carryForwardHistory(existing); const history: HistoryBlock | undefined = minedHistory ?? (preservedHistory.preserved ? preservedHistory.history : undefined); + const historyRefresh: HistoryRefreshOutcome | undefined = + options.mineHistory === true + ? { + requested: true, + mined: minedHistory !== undefined, + preserved: minedHistory === undefined && preservedHistory.preserved, + ...(minedHistory === undefined + ? { refusal: refreshDiagnostics.refusal ?? 'mining produced no history block' } + : {}), + } + : undefined; + const workspace: WorkspaceJsonV4 = { manual: existing?.manual ?? {}, generated: { @@ -347,6 +403,7 @@ export async function generateWorkspaceJson( drift: !unchanged, preservedManual: existing !== undefined, ...(invalidFileMoved === undefined ? {} : { invalidFileMoved }), + ...(historyRefresh === undefined ? {} : { historyRefresh }), content: workspace, }; } diff --git a/packages/cli/src/producer/history-refresh-outcome.test.ts b/packages/cli/src/producer/history-refresh-outcome.test.ts new file mode 100644 index 0000000..18a33aa --- /dev/null +++ b/packages/cli/src/producer/history-refresh-outcome.test.ts @@ -0,0 +1,72 @@ +/** + * A refused history refresh must not read as a completed one. + * + * Repo-native and validator-free: every case here drives the REFUSAL path, in + * which no `coChange` is emitted at all, so nothing touches the observation + * form that this workspace's published `@workspacejson/spec@0.4.4` rejects. + * The successful-refresh side of the contract — and the fallback-to-a-preserved + * block cases, which need an observation-form artifact already on disk — are + * measured in `candidate-tests/`, where the amended schema is available. They + * are not weakened to fit here. + * + * The defect these exist against (Greptile P1 on PR #20): mining refuses, + * generation falls back to the previously recorded block, and the caller gets a + * successful result carrying the PREVIOUS revision's counts with no way to tell + * that from a refresh that ran. Falling back is correct — destroying evidence + * over a shallow clone or a transient Git failure would be worse. Falling back + * *quietly* is the bug. + */ +import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { generateWorkspaceJson } from './generate.js'; + +/** A repository with NO commit graph, so an explicit refresh must refuse. */ +let root: string; + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'wsj-refresh-')); + await writeFile(join(root, 'AGENTS.md'), '# Fixture\n', 'utf8'); + await writeFile(join(root, 'package.json'), JSON.stringify({ name: 'fixture', version: '1.0.0' }), 'utf8'); + await mkdir(join(root, 'src'), { recursive: true }); + await writeFile(join(root, 'src/auth.ts'), 'export const auth = 1;\n', 'utf8'); +}); + +afterEach(async () => { + await rm(root, { recursive: true, force: true }); +}); + +describe('historyRefresh — a refused refresh is reported, not swallowed', () => { + it('reports the refusal when mining cannot complete', async () => { + const result = await generateWorkspaceJson(root, {}, { mineHistory: true }); + + expect(result.historyRefresh).toBeDefined(); + expect(result.historyRefresh!.requested).toBe(true); + expect(result.historyRefresh!.mined).toBe(false); + expect(result.historyRefresh!.refusal).toBeTruthy(); + }); + + it('names WHY, rather than reporting a bare failure', async () => { + // The reason was already being computed by `mineHistoryBlock` and thrown + // away, because the diagnostics object it accepts was never passed. A + // caller that cannot say "shallow clone" versus "git not found" cannot act + // on the refusal. + const result = await generateWorkspaceJson(root, {}, { mineHistory: true }); + expect(result.historyRefresh!.refusal).toMatch(/NOT_MINED|mining failed|NO_BASIS_PIN/); + }); + + it('is ABSENT when no refresh was requested — not a false-y refusal', async () => { + // Absence means "no refresh asked for". Reporting `mined: false` on an + // ordinary run would say a refresh was attempted and failed, which is a + // different and untrue claim. + const result = await generateWorkspaceJson(root); + expect(result.historyRefresh).toBeUndefined(); + }); + + it('carries a refusal if and only if mining produced nothing', async () => { + const refused = await generateWorkspaceJson(root, {}, { mineHistory: true }); + expect(refused.historyRefresh!.mined).toBe(false); + expect('refusal' in refused.historyRefresh!).toBe(true); + }); +});