diff --git a/README.md b/README.md index aeddb60..7e47b04 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,16 @@ consumed here as released packages. | -- | -- | -- | -- | | [`packages/cli/`](./packages/cli/) | `@workspacejson/cli` | `0.5.2` | the neutral producer and its `workspacejson` binary | | [`packages/agents-audit-compat/`](./packages/agents-audit-compat/) | `agents-audit` | `0.4.4` | frozen compatibility bridge; preserves the historical command and API | +| [`packages/mining-core/`](./packages/mining-core/) | `@workspacejson/mining-core` | `0.0.0`, private | L0 commit-graph mining core — extraction, path identity, completeness semantics (META-297 Phases 1–2) | -Those two packages are the whole repository. The private DataHub/dbt adapter +`mining-core` is private and unpublished. It reads git and returns an in-memory +observation set; it does not write the artifact. Projecting into +`generated.coChange` is a separate, later step that is blocked on a schema +admission — the published `coChange` item requires `rate` and forbids additional +properties, so the counts-only shape the churn ruling calls for is rejected by +the schema rather than merely different from it. + +The published packages are the first two. The private DataHub/dbt adapter that was staged here has been **extracted to `workspacejson/datahub-agent`** (META-248), which owns DataHub consumption; it was never durable architecture here. The boundary is machine-enforced and red-tested — see diff --git a/packages/cli/candidate-tests/l1-integration.test.mjs b/packages/cli/candidate-tests/l1-integration.test.mjs new file mode 100644 index 0000000..6d085c3 --- /dev/null +++ b/packages/cli/candidate-tests/l1-integration.test.mjs @@ -0,0 +1,462 @@ +/** + * L1 candidate-contract tests — the packed-candidate integration boundary. + * + * These do NOT run in the CLI workspace, and that separation is deliberate. + * They exercise the observation form, which the published + * `@workspacejson/spec@0.4.4` and `@workspacejson/rules@0.4.4` reject: their + * schema predates ADR-003 A-009 and still requires `rate` while forbidding + * `support`. Running them in the workspace would either fail against + * legitimate published dependencies or force a compatibility shim around + * `WorkspaceJsonValidator` — and a shim is the one outcome that would make + * these tests green while proving nothing, because the validator is the + * contract under test. + * + * So they run in a disposable environment where `spec` and `rules` are the + * PACKED candidates built from a pinned `standard` revision, and the CLI is its + * own packed candidate. Everything asserted here is therefore **candidate + * interoperability**, not published-package interoperability. The published + * packages still reject this shape, and will until the freeze lifts. + * + * Plain Node test runner and plain JS on purpose: the environment has the three + * packages under test and nothing else, so nothing here depends on the CLI + * repository's toolchain. + */ +import assert from 'node:assert/strict'; +import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { after, before, describe, it } from 'node:test'; +import { generateWorkspaceJson } from '@workspacejson/cli'; +import { WorkspaceJsonValidator } from '@workspacejson/rules'; + +const BASIS = '3c9a0f14b7e25d8613af04c2e9b7d5081f6a2c3d'; +const NEWER_BASIS = 'a1b2c3d4e5f60718293a4b5c6d7e8f9012345678'; + +const observationEntry = (over = {}) => ({ + files: ['src/auth.ts', 'src/session.ts'], + support: 8, + occurrences: 24, + ...over, +}); + +function priorArtifact(over = {}) { + return { + manual: { fragileFiles: [{ path: 'src/auth.ts', reason: 'hand-authored, must survive regeneration' }] }, + generated: { + specVersion: '0.4', + generatedAt: '2026-06-01T00:00:00Z', + basisRevision: BASIS, + by: { name: '@workspacejson/cli', version: '0.5.2' }, + frameworkManifest: [], + fileIndex: {}, + coChange: [ + observationEntry(), + observationEntry({ files: ['a.ts', 'b.ts'], support: 3, occurrences: 9 }), + ], + ...over, + }, + agents: {}, + health: { intelligenceState: 'INSUFFICIENT_DATA', observationCount: 0, confidence: 0 }, + }; +} + +// ─── The environment itself is the first thing under test ─────────────────── +// If the installed graph silently resolved a registry copy, every case below +// would be measuring the wrong contract while looking green. The version +// numbers cannot distinguish them — the candidates carry the same 0.4.4 as the +// published packages — so this checks the SHAPE OF THE SCHEMA instead, which is +// the thing that actually differs. +describe('the packed candidate environment resolves the contract under test', () => { + it('the installed spec carries the A-009 observation form', async () => { + const { workspaceJsonSchema } = await import('@workspacejson/spec'); + const item = workspaceJsonSchema.properties.generated.properties.coChange.items; + const props = Object.keys(item.properties).sort(); + assert.ok(props.includes('support'), 'installed spec has no `support` — this is a pre-A-009 registry copy'); + assert.ok(props.includes('occurrences')); + }); + + it('the installed spec carries the A-010 widening', async () => { + const { workspaceJsonSchema } = await import('@workspacejson/spec'); + const item = workspaceJsonSchema.properties.generated.properties.coChange.items; + assert.ok( + !item.required.includes('generated'), + 'installed spec still requires the classification flag — this is a pre-A-010 registry copy', + ); + const legacy = item.oneOf.find((branch) => branch.title === 'legacy form'); + assert.ok(legacy.required.includes('generated'), 'legacy branch lost its requirement'); + }); + + it('the validator the PRODUCER calls accepts the observation form', () => { + // The decisive check. `rules` bundles its own `spec` dependency, so a + // correct top-level `spec` proves nothing on its own — this asserts the + // graph the producer actually runs through. + const result = new WorkspaceJsonValidator().validate(priorArtifact()); + assert.equal(result.valid, true, `validator rejected the observation form: ${JSON.stringify(result.errors)}`); + }); +}); + +describe('ordinary generation and commit-history evidence', () => { + let root; + + const artifactPath = () => resolve(root, '.agents/workspace.json'); + const readArtifact = async () => JSON.parse(await readFile(artifactPath(), 'utf8')); + const generatedOf = async () => (await readArtifact()).generated; + + /** The exact bytes of the history block, which is what the contract is about. */ + const historyBytes = (generated) => + JSON.stringify({ basisRevision: generated.basisRevision, coChange: generated.coChange }); + + before(() => {}); + + const setup = async () => { + root = await mkdtemp(join(tmpdir(), 'wsj-l1-')); + await writeFile(join(root, 'AGENTS.md'), '# Test\n\nUse kebab-case for files.\n', 'utf8'); + await writeFile(join(root, 'package.json'), JSON.stringify({ name: 'fixture', version: '1.0.0' }), 'utf8'); + await mkdir(join(root, 'src'), { recursive: true }); + await writeFile(join(root, 'src/auth.ts'), 'export const auth = 1;\n', 'utf8'); + await mkdir(join(root, '.agents'), { recursive: true }); + await writeFile(artifactPath(), JSON.stringify(priorArtifact(), null, 2) + '\n', 'utf8'); + }; + + const teardown = async () => { + await rm(root, { recursive: true, force: true }); + }; + + it('CASE 1 — a plain regeneration preserves the history block byte for byte', async () => { + await setup(); + try { + const before = historyBytes(await generatedOf()); + await generateWorkspaceJson(root); + const after = await generatedOf(); + assert.equal(historyBytes(after), before); + assert.equal(after.basisRevision, BASIS); + assert.equal(after.coChange.length, 2); + } finally { + await teardown(); + } + }); + + it('CASE 2 — the pin is never advanced, and counts are never re-attributed', async () => { + await setup(); + try { + await generateWorkspaceJson(root); + const after = await generatedOf(); + assert.equal(after.basisRevision, BASIS); + assert.notEqual(after.basisRevision, NEWER_BASIS); + } finally { + await teardown(); + } + }); + + it('CASE 3 — ordinary generation does not recompute history', async () => { + // The observations are FABRICATED: `src/session.ts` and `b.ts` do not exist + // in this repository, and it has no git history at all. Any real mining + // pass would produce something different — almost certainly nothing. Their + // survival is therefore positive proof that no mining ran, which a timing + // assertion could never establish. + await setup(); + try { + await generateWorkspaceJson(root); + const after = await generatedOf(); + assert.deepEqual(after.coChange[0].files, ['src/auth.ts', 'src/session.ts']); + assert.equal(after.coChange[0].support, 8); + assert.equal(after.coChange[0].occurrences, 24); + assert.deepEqual(after.coChange[1].files, ['a.ts', 'b.ts']); + } finally { + await teardown(); + } + }); + + it('CASE 4 — a non-history generated field changes without touching the block', async () => { + await setup(); + try { + const before = historyBytes(await generatedOf()); + await writeFile(join(root, 'src/added.ts'), 'export const added = 2;\n', 'utf8'); + const result = await generateWorkspaceJson(root); + const after = await generatedOf(); + assert.equal(result.written, true); + assert.ok(Object.keys(after.fileIndex).includes('src/added.ts')); + assert.equal(historyBytes(after), before); + } finally { + await teardown(); + } + }); + + it('CASE 5 — --check reports no drift merely because history was not recomputed', async () => { + await setup(); + try { + await generateWorkspaceJson(root); + const checked = await generateWorkspaceJson(root, {}, { check: true }); + assert.equal(checked.drift, false); + assert.equal(checked.skipped, true); + } finally { + await teardown(); + } + }); + + it('CASE 6 — generatedAt may move while basisRevision stays put', async () => { + await setup(); + try { + await writeFile(join(root, 'src/added.ts'), 'export const added = 2;\n', 'utf8'); + await generateWorkspaceJson(root); + const after = await generatedOf(); + assert.notEqual(after.generatedAt, '2026-06-01T00:00:00Z'); + assert.equal(after.basisRevision, BASIS); + } finally { + await teardown(); + } + }); + + it('CASE 7 — with no prior block, ordinary generation invents nothing', async () => { + await setup(); + try { + await rm(artifactPath()); + await generateWorkspaceJson(root); + const after = await generatedOf(); + assert.equal('coChange' in after, false); + assert.equal('basisRevision' in after, false); + } finally { + await teardown(); + } + }); + + it('CASE 8 — manual evidence survives alongside preserved history', async () => { + await setup(); + try { + await generateWorkspaceJson(root); + const artifact = await readArtifact(); + assert.deepEqual(artifact.manual.fragileFiles, [ + { path: 'src/auth.ts', reason: 'hand-authored, must survive regeneration' }, + ]); + } finally { + await teardown(); + } + }); + + it('CASE 9 — the preserved artifact validates against the candidate schema', async () => { + await setup(); + try { + await writeFile(join(root, 'src/added.ts'), 'export const added = 2;\n', 'utf8'); + await generateWorkspaceJson(root); + const result = new WorkspaceJsonValidator().validate(await readArtifact()); + assert.equal(result.valid, true, JSON.stringify(result.errors)); + } finally { + await teardown(); + } + }); +}); + +// ─── Opt-in mining, against a real repository with real history ───────────── +let minedRoot; + +describe('explicit mining writes a conforming block', () => { + let root; + + const git = (args, cwd) => execFileSync('git', args, { cwd, encoding: 'utf8' }); + + const build = async () => { + root = await mkdtemp(join(tmpdir(), 'wsj-mine-')); + git(['init', '-q', '-b', 'main'], root); + git(['config', 'user.email', 'fixture@example.invalid'], root); + git(['config', 'user.name', 'Fixture'], root); + await writeFile(join(root, 'AGENTS.md'), '# Fixture\n', 'utf8'); + await writeFile(join(root, 'package.json'), JSON.stringify({ name: 'mined', version: '1.0.0' }), 'utf8'); + await mkdir(join(root, 'src'), { recursive: true }); + + // Six commits that change the same two files together, and a third file + // that moves independently. The coupling is in the commit graph and nowhere + // else — neither file imports the other — which is the case the whole + // standard rests on. + for (let i = 0; i < 6; i += 1) { + await writeFile(join(root, 'src/auth.ts'), `export const auth = ${i};\n`, 'utf8'); + await writeFile(join(root, 'src/session.ts'), `export const session = ${i};\n`, 'utf8'); + git(['add', '-A'], root); + git(['commit', '-q', '-m', `paired change ${i}`], root); + } + for (let i = 0; i < 3; i += 1) { + await writeFile(join(root, 'src/lonely.ts'), `export const lonely = ${i};\n`, 'utf8'); + git(['add', '-A'], root); + git(['commit', '-q', '-m', `solo change ${i}`], root); + } + minedRoot = root; + }; + + // NOTE: `root` is intentionally NOT removed here. The refresh-outcome suite + // below reuses it as `minedRoot` — it is the only fixture in this file with a + // real commit graph, and rebuilding one per suite would triple the runtime. + // It is cleaned up at the end of the file instead. + + it('mines, projects, and produces a schema-valid artifact', async () => { + await build(); + const result = await generateWorkspaceJson(root, {}, { mineHistory: true }); + const generated = result.content.generated; + + assert.ok(Array.isArray(generated.coChange), 'no coChange block was written'); + assert.match(generated.basisRevision, /^[0-9a-f]{40}$|^[0-9a-f]{64}$/); + assert.equal(generated.basisRevision, git(['rev-parse', 'HEAD'], root).trim()); + + const validation = new WorkspaceJsonValidator().validate(result.content); + assert.equal(validation.valid, true, JSON.stringify(validation.errors)); + }); + + it('finds the no-import-edge coupling the thesis rests on', async () => { + const result = await generateWorkspaceJson(root, {}, { mineHistory: true }); + const pair = result.content.generated.coChange.find( + (entry) => entry.files.includes('src/auth.ts') && entry.files.includes('src/session.ts'), + ); + assert.ok(pair, 'the paired files were not reported as co-changing'); + assert.equal(pair.support, 6); + assert.ok(pair.occurrences >= pair.support); + }); + + it('emits no derived value and no unsupported classification', async () => { + const result = await generateWorkspaceJson(root, {}, { mineHistory: true }); + for (const entry of result.content.generated.coChange) { + assert.equal('rate' in entry, false, 'a derived rate was stored'); + assert.equal('generated' in entry, false, 'an unsupported classification was asserted'); + assert.deepEqual(Object.keys(entry).sort(), ['files', 'occurrences', 'support']); + } + }); + + it('orders every pair canonically in ascending UTF-8 byte order', async () => { + const result = await generateWorkspaceJson(root, {}, { mineHistory: true }); + const encoder = new TextEncoder(); + const compareUtf8 = (a, b) => Buffer.compare(Buffer.from(encoder.encode(a)), Buffer.from(encoder.encode(b))); + for (const entry of result.content.generated.coChange) { + assert.ok(compareUtf8(entry.files[0], entry.files[1]) <= 0, `pair not canonically ordered: ${entry.files}`); + } + }); + + it('is deterministic — two runs at the same revision agree byte for byte', async () => { + const first = await generateWorkspaceJson(root, {}, { mineHistory: true }); + const second = await generateWorkspaceJson(root, {}, { mineHistory: true }); + const block = (r) => + JSON.stringify({ basisRevision: r.content.generated.basisRevision, coChange: r.content.generated.coChange }); + assert.equal(block(first), block(second)); + }); + + it('a subsequent ORDINARY run preserves what mining wrote', async () => { + const mined = await generateWorkspaceJson(root, {}, { mineHistory: true }); + const minedBlock = JSON.stringify(mined.content.generated.coChange); + const ordinary = await generateWorkspaceJson(root); + assert.equal(JSON.stringify(ordinary.content.generated.coChange), minedBlock); + assert.equal(ordinary.content.generated.basisRevision, mined.content.generated.basisRevision); + }); + + it('ORDINARY GENERATION DOES NOT RECOMPUTE, even where mining would succeed', async () => { + // The decisive case, and the reason it has to be here rather than in the + // no-history fixtures above: where the repository has no commit graph, + // mining refuses and falls back to carry-forward, so a producer that + // wrongly mined on every run would be indistinguishable from one that + // never did. Those cases cannot detect recomputation at all. + // + // Here the graph exists AND has moved since the block was written, so the + // two behaviours diverge: preserving keeps the old pin and the old counts, + // recomputing advances the pin and changes the counts. Only one of those + // can be true of the artifact afterwards. + const mined = await generateWorkspaceJson(root, {}, { mineHistory: true }); + const pinnedAt = mined.content.generated.basisRevision; + const blockBefore = JSON.stringify(mined.content.generated.coChange); + + // Move the graph: four more commits pairing a DIFFERENT set of files, which + // a fresh mining pass would certainly report and the stored block cannot. + for (let i = 0; i < 4; i += 1) { + await writeFile(join(root, 'src/alpha.ts'), `export const alpha = ${i};\n`, 'utf8'); + await writeFile(join(root, 'src/beta.ts'), `export const beta = ${i};\n`, 'utf8'); + git(['add', '-A'], root); + git(['commit', '-q', '-m', `new pairing ${i}`], root); + } + const movedHead = git(['rev-parse', 'HEAD'], root).trim(); + assert.notEqual(movedHead, pinnedAt, 'fixture error: HEAD did not move'); + + const ordinary = await generateWorkspaceJson(root); + + // The pin must still name the commit the counts were taken at. + assert.equal( + ordinary.content.generated.basisRevision, + pinnedAt, + 'ordinary generation advanced the basis pin — the counts now claim a revision they were never counted at', + ); + assert.equal( + JSON.stringify(ordinary.content.generated.coChange), + blockBefore, + 'ordinary generation rewrote the co-change block — history was recomputed', + ); + // And the new pairing must be absent, because nothing re-read the graph. + const newPair = ordinary.content.generated.coChange.find( + (entry) => entry.files.includes('src/alpha.ts') && entry.files.includes('src/beta.ts'), + ); + assert.equal(newPair, undefined, 'a pair only visible to a fresh mining pass appeared in an ordinary run'); + + // Explicit mining, by contrast, DOES pick it up — which proves the previous + // assertions measured opt-in behaviour rather than a broken miner. + const remined = await generateWorkspaceJson(root, {}, { mineHistory: true }); + assert.equal(remined.content.generated.basisRevision, movedHead); + const reminedPair = remined.content.generated.coChange.find( + (entry) => entry.files.includes('src/alpha.ts') && entry.files.includes('src/beta.ts'), + ); + assert.ok(reminedPair, 'explicit mining failed to observe the new pairing'); + }); +}); + +// ─── Refresh outcome signalling (Greptile P1 on PR #20) ───────────────────── +describe('an explicitly requested refresh reports what it actually did', () => { + let root; + + const artifactPath = () => resolve(root, '.agents/workspace.json'); + + const setup = async () => { + root = await mkdtemp(join(tmpdir(), 'wsj-refresh-')); + await writeFile(join(root, 'AGENTS.md'), '# Fixture\n', 'utf8'); + await writeFile(join(root, 'package.json'), JSON.stringify({ name: 'fixture', version: '1.0.0' }), 'utf8'); + await mkdir(join(root, 'src'), { recursive: true }); + await writeFile(join(root, 'src/auth.ts'), 'export const auth = 1;\n', 'utf8'); + await mkdir(join(root, '.agents'), { recursive: true }); + await writeFile(artifactPath(), JSON.stringify(priorArtifact(), null, 2) + '\n', 'utf8'); + }; + const teardown = async () => { await rm(root, { recursive: true, force: true }); }; + + it('REFUSED REFRESH SAYS SO while keeping the block it fell back to', async () => { + // The P1 exactly: this repository has no commit graph, so an explicit + // refresh cannot complete. Falling back to the recorded block is correct — + // destroying evidence over a shallow clone would be worse. What must not + // happen is the caller receiving a successful-looking result carrying the + // PREVIOUS revision's counts with no way to tell. + await setup(); + try { + const result = await generateWorkspaceJson(root, {}, { mineHistory: true }); + + assert.equal(result.historyRefresh?.requested, true); + assert.equal(result.historyRefresh?.mined, false, 'a refusal was reported as a completed refresh'); + assert.equal(result.historyRefresh?.preserved, true); + assert.ok(result.historyRefresh?.refusal, 'no reason was given for the refusal'); + + // …and the evidence survived. + assert.equal(result.content.generated.basisRevision, BASIS); + assert.equal(result.content.generated.coChange.length, 2); + } finally { + await teardown(); + } + }); + + it('a COMPLETED refresh is distinguishable from a refused one', async () => { + // Same call, a repository that can actually be mined. If these two produced + // the same `historyRefresh`, the field would be decorative. + const mined = await generateWorkspaceJson(minedRoot, {}, { mineHistory: true }); + assert.equal(mined.historyRefresh?.mined, true); + assert.equal(mined.historyRefresh?.preserved, false); + assert.equal('refusal' in (mined.historyRefresh ?? {}), false, 'a successful refresh carried a refusal'); + }); + + it('an ordinary run reports NO refresh outcome at all', async () => { + // Absence means "none requested". Reporting `mined: false` here would claim + // a refresh was attempted and failed, which is a different and untrue thing. + const ordinary = await generateWorkspaceJson(minedRoot); + assert.equal(ordinary.historyRefresh, undefined); + }); +}); + +after(async () => { + if (minedRoot) await rm(minedRoot, { recursive: true, force: true }); +}); diff --git a/packages/cli/package.json b/packages/cli/package.json index 83b940e..3c92198 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "@workspacejson/cli", "version": "0.5.2", - "description": "The workspace.json producer — scans a repository and generates .agents/workspace.json deterministically, preserving human-authored manual evidence.", + "description": "The workspace.json producer \u2014 scans a repository and generates .agents/workspace.json deterministically, preserving human-authored manual evidence.", "license": "Apache-2.0", "author": "workspace.json contributors", "homepage": "https://workspacejson.dev", @@ -66,8 +66,9 @@ }, "devDependencies": { "@types/node": "22.19.17", - "typescript": "^5.4.0", + "@workspacejson/mining-core": "workspace:*", "tsup": "^8.0.0", + "typescript": "^5.4.0", "vitest": "^1.6.0" } } diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 220d0df..76fcc74 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -4,7 +4,7 @@ export { GenerateRefusalError, THIS_PRODUCER, } from './producer/generate.js'; -export type { GenerateResult, ProducerIdentity } from './producer/generate.js'; +export type { GenerateResult, HistoryRefreshOutcome, ProducerIdentity } from './producer/generate.js'; export { DEFAULT_PRODUCER_CONFIG, detectCiProvider } from './producer/config.js'; export type { ProducerConfig } from './producer/config.js'; export { findAgentsMdPath, readTextOrEmpty } from './producer/fs.js'; diff --git a/packages/cli/src/producer/generate.ts b/packages/cli/src/producer/generate.ts index fb65ea6..9ac54b4 100644 --- a/packages/cli/src/producer/generate.ts +++ b/packages/cli/src/producer/generate.ts @@ -21,6 +21,30 @@ import type { RuleContext } from '@workspacejson/rules'; import { DEFAULT_PRODUCER_CONFIG, detectCiProvider, type ProducerConfig } from './config.js'; import { buildFileIndex, buildFrameworkManifest } from './evidence.js'; import { findAgentsMdPath, readTextOrEmpty } from './fs.js'; +import { carryForwardHistory, type PreservedHistory } from './history-carry-forward.js'; +import { mineHistoryBlock } from './history-mine.js'; + +/** + * A history block from either route: freshly mined, or carried forward. + * + * Declared locally because the two sources carry different TYPES for the same + * runtime shape, and the mismatch is a fact about the dependency rather than + * about this code. `@workspacejson/spec@0.4.4` is the published package, and + * its `CoChangeEntry` predates ADR-003 A-009: it still requires `rate` and + * knows nothing of `support`. So an observation-form entry — exactly what this + * producer now emits — is not assignable to the published type, and cannot be + * until the amended spec is published. + * + * The runtime contract is unaffected and is NOT relaxed anywhere: the artifact + * still goes through `WorkspaceJsonValidator` unmodified, and the candidate + * conformance suite runs it against the amended schema. This declaration + * narrows a compile-time gap in a stale type; it does not widen what the + * producer will accept or emit. It is deleted when the amended spec publishes. + */ +interface HistoryBlock { + basisRevision: string; + coChange: readonly unknown[]; +} const _require = createRequire(import.meta.url); @@ -63,6 +87,33 @@ export interface ProducerIdentity { export const THIS_PRODUCER: ProducerIdentity = { name: pkgName, version: pkgVersion }; +/** + * What an explicitly requested history refresh actually did. + * + * Present only when the caller passed `mineHistory: true`, so its absence means + * no refresh was requested rather than a refresh that failed. + * + * The distinction this exists to make: a refused refresh still produces a + * successful generation carrying the PREVIOUS revision's counts, because + * destroying evidence over a shallow clone or a transient Git failure would be + * worse than keeping it. That is the right behavior and it is also + * indistinguishable, from the artifact alone, from a refresh that completed. + * A caller that asked for fresh observations must be able to tell. + */ +export interface HistoryRefreshOutcome { + /** Always true — the field is absent unless a refresh was requested. */ + requested: true; + /** True when the commit graph was read and a new block produced. */ + mined: boolean; + /** True when mining refused and a prior block was carried instead. */ + preserved: boolean; + /** + * Why mining produced nothing. Present if and only if `mined` is false — + * e.g. a shallow clone, absent history, or a Git invocation failure. + */ + refusal?: string; +} + export interface GenerateResult { path: string; written: boolean; @@ -70,6 +121,12 @@ export interface GenerateResult { drift: boolean; preservedManual: boolean; invalidFileMoved?: string; + /** + * Present only when `mineHistory: true` was requested. Says whether the + * refresh completed, and why not when it did not — so a refused refresh + * cannot read as a successful one. + */ + historyRefresh?: HistoryRefreshOutcome; content: WorkspaceJsonV4; } @@ -129,7 +186,22 @@ export async function writeWorkspaceAtomically(outputPath: string, content: Work export async function generateWorkspaceJson( repoRoot: string, config: Partial = {}, - options: { dryRun?: boolean; check?: boolean; force?: boolean; producer?: ProducerIdentity; commandName?: string } = {}, + options: { + dryRun?: boolean; + check?: boolean; + force?: boolean; + producer?: ProducerIdentity; + commandName?: string; + /** + * Read the commit graph and rewrite `generated.coChange`. + * + * Off by default, and that default is the contract rather than a + * convenience: mining a bounded window costs seconds to tens of seconds, + * and a producer that recomputed history on every ordinary run would make + * the artifact churn on every commit. See history-carry-forward.ts. + */ + mineHistory?: boolean; + } = {}, ): Promise { const resolvedRoot = resolve(repoRoot); const fullConfig: ProducerConfig = { ...DEFAULT_PRODUCER_CONFIG, ...config }; @@ -208,6 +280,43 @@ export async function generateWorkspaceJson( } } } + // Commit-history evidence enters the artifact by exactly one of two routes, + // and never both. Mining is EXPLICIT: `mineHistory` is off unless a caller + // asked for it, so an ordinary run reads the working tree and nothing else. + // + // The order matters. A refused mining pass falls back to carry-forward rather + // than to nothing: a shallow clone or a git failure must not destroy evidence + // an earlier successful pass recorded. + // + // But falling back QUIETLY is its own defect, and a worse one. A caller that + // asked for a refresh and received a successful-looking result carrying the + // previous revision's counts cannot tell that from a refresh that completed — + // the artifact looks the same either way, and `basisRevision` only helps a + // reader who already suspects something. So the outcome is reported on the + // result: `historyRefresh` says whether the refresh actually happened, and + // carries the refusal reason when it did not. + // + // The reason itself was already being computed and thrown away — the + // diagnostics object exists for exactly this and was not passed. + const refreshDiagnostics: { refusal?: string } = {}; + const minedHistory = + options.mineHistory === true ? await mineHistoryBlock(resolvedRoot, refreshDiagnostics) : undefined; + const preservedHistory = carryForwardHistory(existing); + const history: HistoryBlock | undefined = + minedHistory ?? (preservedHistory.preserved ? preservedHistory.history : undefined); + + const historyRefresh: HistoryRefreshOutcome | undefined = + options.mineHistory === true + ? { + requested: true, + mined: minedHistory !== undefined, + preserved: minedHistory === undefined && preservedHistory.preserved, + ...(minedHistory === undefined + ? { refusal: refreshDiagnostics.refusal ?? 'mining produced no history block' } + : {}), + } + : undefined; + const workspace: WorkspaceJsonV4 = { manual: existing?.manual ?? {}, generated: { @@ -251,6 +360,24 @@ export async function generateWorkspaceJson( scannedAt: (existing?.generated.hygiene as { scannedAt?: string } | undefined)?.scannedAt ?? now, }, + // Commit-history evidence is PRESERVED, never rebuilt, by ordinary + // generation — see history-carry-forward.ts for why this one part of the + // producer-owned section is carried rather than regenerated. + // + // The values are spliced in as the objects parsed from the prior + // artifact, so the bytes are unchanged. Nothing here reads the commit + // graph: no mining, no pin advance, no re-attribution of old counts to a + // newer revision. If nothing conforming was preserved, both keys stay + // absent — ordinary generation never invents a history block, and an + // absent block correctly reads as "not analyzed". + ...(history === undefined + ? {} + : { + basisRevision: history.basisRevision, + // See HistoryBlock: the published 0.4.4 type cannot describe an + // observation-form entry. The value is validated at runtime. + coChange: history.coChange as NonNullable, + }), }, agents: {}, health: { @@ -276,6 +403,7 @@ export async function generateWorkspaceJson( drift: !unchanged, preservedManual: existing !== undefined, ...(invalidFileMoved === undefined ? {} : { invalidFileMoved }), + ...(historyRefresh === undefined ? {} : { historyRefresh }), content: workspace, }; } diff --git a/packages/cli/src/producer/history-carry-forward.test.ts b/packages/cli/src/producer/history-carry-forward.test.ts new file mode 100644 index 0000000..6add057 --- /dev/null +++ b/packages/cli/src/producer/history-carry-forward.test.ts @@ -0,0 +1,118 @@ +/** + * Carry-forward semantics — the pure decision function. + * + * Repo-native and validator-free by construction. This file runs inside the CLI + * workspace against its legitimate published `@workspacejson/spec@0.4.4` and + * `@workspacejson/rules@0.4.4` dependencies, and must keep passing there, so it + * touches nothing that needs the amended schema. The end-to-end cases — which + * DO need a validator that accepts the observation form — live in + * `candidate-tests/` and run only in the packed-candidate environment. + * + * Every case here is written so that removing the behaviour it covers makes it + * fail. That is the whole value: the three failure modes carry-forward exists + * to prevent (drop, advance the pin, recompute) all produce a *plausible* + * artifact, so nothing about the output looks wrong when they happen. Only a + * test that knows what the previous artifact said can tell. + */ +import { describe, expect, it } from 'vitest'; +import { CarryForwardRefusal, carryForwardHistory } from './history-carry-forward.js'; + +const BASIS = '3c9a0f14b7e25d8613af04c2e9b7d5081f6a2c3d'; + +const observationEntry = (over: Record = {}) => ({ + files: ['src/auth.ts', 'src/session.ts'], + support: 8, + occurrences: 24, + ...over, +}); + +/** A prior artifact carrying mined evidence, as `generate` would find on disk. */ +function priorArtifact(over: Record = {}): Record { + return { + manual: { fragileFiles: ['src/auth.ts'] }, + generated: { + specVersion: '0.4', + generatedAt: '2026-06-01T00:00:00Z', + basisRevision: BASIS, + by: { name: '@workspacejson/cli', version: '0.5.2' }, + frameworkManifest: [], + fileIndex: {}, + coChange: [observationEntry(), observationEntry({ files: ['a.ts', 'b.ts'], support: 3, occurrences: 9 })], + ...over, + }, + agents: {}, + health: { intelligenceState: 'INSUFFICIENT_DATA', observationCount: 0, confidence: 0 }, + }; +} + +describe('carryForwardHistory — what ordinary generation preserves', () => { + it('preserves a conforming observation block and its pin', () => { + const result = carryForwardHistory(priorArtifact() as never); + expect(result.preserved).toBe(true); + if (!result.preserved) return; + expect(result.history.basisRevision).toBe(BASIS); + expect(result.history.coChange).toHaveLength(2); + }); + + it('passes the parsed entries THROUGH rather than rebuilding them', () => { + // Byte-for-byte preservation is the contract. Rebuilding an entry field by + // field would re-order its keys and change the serialized bytes even though + // the value is structurally identical, so identity of the array elements is + // the property that actually guarantees it. + const prior = priorArtifact(); + const original = (prior['generated'] as Record)['coChange'] as unknown[]; + const result = carryForwardHistory(prior as never); + expect(result.preserved).toBe(true); + if (!result.preserved) return; + expect(result.history.coChange[0]).toBe(original[0]); + expect(result.history.coChange[1]).toBe(original[1]); + }); + + it('preserves a PINNED EMPTY array — a positive finding, not an absence', () => { + // "The analysis ran and found no qualifying pairs" is evidence. Dropping it + // would silently convert it into "never analyzed". + const result = carryForwardHistory(priorArtifact({ coChange: [] }) as never); + expect(result.preserved).toBe(true); + }); + + it('refuses when there is no prior artifact — never invents a block', () => { + const result = carryForwardHistory(undefined); + expect(result.preserved).toBe(false); + if (result.preserved) return; + expect(result.refusal).toBe(CarryForwardRefusal.NO_PRIOR_BLOCK); + }); + + it('refuses a legacy rate entry rather than perpetuating it', () => { + const result = carryForwardHistory( + priorArtifact({ coChange: [{ files: ['a.ts', 'b.ts'], rate: 0.8, occurrences: 9, generated: false }] }) as never, + ); + expect(result.preserved).toBe(false); + if (result.preserved) return; + expect(result.refusal).toBe(CarryForwardRefusal.NOT_OBSERVATION_FORM); + }); + + it('refuses an observation block whose pin is symbolic or abbreviated', () => { + for (const basisRevision of ['HEAD', 'main', BASIS.slice(0, 7), BASIS.toUpperCase()]) { + const result = carryForwardHistory(priorArtifact({ basisRevision }) as never); + expect(result.preserved).toBe(false); + if (result.preserved) continue; + expect(result.refusal).toBe(CarryForwardRefusal.NO_CONFORMING_BASIS); + } + }); + + it('carries an entry that omits the A-010 classification flag', () => { + // The shape this producer emits. Treating the absent flag as malformed + // would refuse to carry forward exactly its own output. + const result = carryForwardHistory(priorArtifact() as never); + expect(result.preserved).toBe(true); + if (!result.preserved) return; + expect('generated' in (result.history.coChange[0] as object)).toBe(false); + }); + + it('refuses a block violating support <= occurrences', () => { + const result = carryForwardHistory( + priorArtifact({ coChange: [observationEntry({ support: 30, occurrences: 24 })] }) as never, + ); + expect(result.preserved).toBe(false); + }); +}); diff --git a/packages/cli/src/producer/history-carry-forward.ts b/packages/cli/src/producer/history-carry-forward.ts new file mode 100644 index 0000000..3d6f3f5 --- /dev/null +++ b/packages/cli/src/producer/history-carry-forward.ts @@ -0,0 +1,159 @@ +/** + * Carry-forward of commit-history evidence across ordinary generation. + * + * This is the load-bearing rule of L1, and it exists because two obligations + * collide. + * + * The standard says `generated` is producer-owned and replaced wholesale on + * each regeneration. The ruling says history mining is **explicit and opt-in**, + * and that ordinary generation must not recompute history or fabricate + * freshness. Taken naively, the first erases what the second forbids + * recomputing: run `generate` after mining and the observations are gone. + * + * So `generated.coChange` and `generated.basisRevision` are the one part of the + * producer-owned section that ordinary generation **preserves rather than + * rebuilds**. Not because they are manual — they are machine-derived — but + * because they are derived from an input ordinary generation does not read. + * The working tree is scanned every run; the commit graph is not. + * + * Three failure modes this is written against, all of which produce a + * plausible-looking artifact: + * + * - **Drop.** Regenerating without carry-forward silently destroys mined + * evidence, and the resulting empty/absent block is indistinguishable from a + * repository that was never mined. + * - **Advance the pin.** Carrying the observations forward while moving + * `basisRevision` to current HEAD re-attributes old counts to a commit they + * were never counted at. The numbers stay plausible and become false. + * - **Recompute.** Mining during ordinary generation makes every run pay + * seconds-to-tens-of-seconds, and makes the artifact churn on every commit — + * the exact `generate --check` failure the raw-count amendment removed. + * + * **`generatedAt` is not evidence about this block.** It records when the + * ordinary generation run happened. `basisRevision` is the authoritative + * freshness and provenance pin for the co-change observations, and it is the + * only field that says which commit they were counted at. A run that refreshes + * `fileIndex` moves `generatedAt` and must leave `basisRevision` exactly where + * it was; a reader comparing `generatedAt` to the repository's current revision + * learns nothing about whether the history block is stale, and must compare + * `basisRevision` instead. + */ +import type { WorkspaceJsonV4 } from '@workspacejson/spec'; + +/** A full-length lowercase Git object name. SHA-1 or SHA-256, never symbolic. */ +const OBJECT_NAME = /^[0-9a-f]{40}$|^[0-9a-f]{64}$/; + +/** + * What ordinary generation carries forward, verbatim. + * + * `coChange` is deliberately `readonly unknown[]` rather than the published + * `CoChangeEntry[]`. Two reasons, and the second is the load-bearing one. + * + * These entries are passed through untouched — nothing here reads a field off + * them, so a precise element type would buy nothing. And the published + * `@workspacejson/spec@0.4.4` type predates ADR-003 A-009: it requires `rate` + * and has no `support`, so calling a preserved observation entry a + * `CoChangeEntry` would be a false claim in the type system rather than a + * convenience. The one place the stale type has to be accommodated is the + * assembly boundary in `generate.ts`, and it stays the only place. + */ +export interface PreservedHistory { + basisRevision: string; + coChange: readonly unknown[]; +} + +/** Why nothing was carried forward. Absence is reported, never inferred away. */ +export enum CarryForwardRefusal { + /** No prior artifact, or it had no `coChange`. Nothing to preserve. */ + NO_PRIOR_BLOCK = 'NO_PRIOR_BLOCK', + /** Present but not the observation form — legacy `rate`, or malformed. */ + NOT_OBSERVATION_FORM = 'NOT_OBSERVATION_FORM', + /** Observation form without a conforming pin. Reads as legacy/unknown. */ + NO_CONFORMING_BASIS = 'NO_CONFORMING_BASIS', +} + +export type CarryForwardResult = + | { preserved: true; history: PreservedHistory } + | { preserved: false; refusal: CarryForwardRefusal; detail: string }; + +function isObservationEntry(value: unknown): boolean { + if (value === null || typeof value !== 'object') return false; + const entry = value as Record; + + // The form discriminator is `support` versus `rate`, and nothing else. + // `generated` plays no part in it — A-010 made that flag optional, so an + // entry carrying counts and no classification is unambiguously observation + // form, and refusing to carry it forward would discard exactly the shape + // this producer emits. + if ('rate' in entry) return false; + if (!Number.isInteger(entry['support']) || (entry['support'] as number) < 0) return false; + if (!Number.isInteger(entry['occurrences']) || (entry['occurrences'] as number) < 1) return false; + if ((entry['support'] as number) > (entry['occurrences'] as number)) return false; + + const files = entry['files']; + if (!Array.isArray(files) || files.length !== 2) return false; + if (!files.every((path) => typeof path === 'string' && path.length > 0)) return false; + + return true; +} + +/** + * Decide what an ordinary generation run preserves from the prior artifact. + * + * Validating rather than trusting is the point. A block that would not survive + * schema validation must not be carried into a fresh artifact, because doing so + * would let one bad mining run poison every subsequent generation — and the + * producer refuses to overwrite an invalid artifact precisely so that cannot + * happen silently. + * + * The returned values are the parsed objects themselves, not copies rebuilt + * field by field. Rebuilding would re-order keys and change bytes; the contract + * is byte-for-byte preservation, so the original values are passed through. + */ +export function carryForwardHistory(existing: WorkspaceJsonV4 | undefined): CarryForwardResult { + if (existing === undefined) { + return { + preserved: false, + refusal: CarryForwardRefusal.NO_PRIOR_BLOCK, + detail: 'no prior artifact', + }; + } + + const generated = existing.generated as unknown as Record; + const coChange = generated['coChange']; + + if (!Array.isArray(coChange)) { + return { + preserved: false, + refusal: CarryForwardRefusal.NO_PRIOR_BLOCK, + detail: 'the prior artifact carries no coChange array', + }; + } + + // An empty array is preserved only when pinned. Unpinned it means + // legacy/unknown and asserts nothing, so there is nothing to preserve; + // pinned it is a positive finding — the analysis ran and found no qualifying + // pairs — and dropping it would convert a real result into "never analyzed". + if (!coChange.every(isObservationEntry)) { + return { + preserved: false, + refusal: CarryForwardRefusal.NOT_OBSERVATION_FORM, + detail: + 'the prior coChange block is not homogeneous observation form; it is legacy or malformed and this producer does not perpetuate it', + }; + } + + const basisRevision = generated['basisRevision']; + if (typeof basisRevision !== 'string' || !OBJECT_NAME.test(basisRevision)) { + return { + preserved: false, + refusal: CarryForwardRefusal.NO_CONFORMING_BASIS, + detail: + 'the prior coChange block carries no full-length lowercase object name, so it cannot be recounted against and reads as legacy/unknown', + }; + } + + // The elements are passed through untouched, which is what makes the + // byte-for-byte guarantee real — see the note on PreservedHistory. + return { preserved: true, history: { basisRevision, coChange } }; +} diff --git a/packages/cli/src/producer/history-mine.ts b/packages/cli/src/producer/history-mine.ts new file mode 100644 index 0000000..e783232 --- /dev/null +++ b/packages/cli/src/producer/history-mine.ts @@ -0,0 +1,54 @@ +/** + * The explicit, opt-in commit-history pass. + * + * This is the only code path in the producer that reads the commit graph, and + * it runs only when a caller asks for it. Everything else in `generate` reads + * the working tree. That split is the whole reason `generated.coChange` is + * carried forward rather than rebuilt — see history-carry-forward.ts. + * + * The pipeline is `mine → score → select → project`, each stage pure with + * respect to the one before it, so the extracted events and the uncapped scored + * set stay auditable behind whatever the selection capped. + * + * **A refusal returns `undefined`, and that is not the same as an empty + * result.** A shallow clone, an absent history or a Git failure produce nothing + * here, and the caller falls back to whatever the previous artifact recorded. + * Writing an empty `coChange` in those cases would be the worst available + * outcome: under A-009 a *pinned* empty array is a positive finding — "the + * analysis ran at this revision and found no qualifying pairs" — so emitting + * one for a repository that could not be analyzed would state a result nobody + * measured. + */ +import { mine, project, score, select } from '@workspacejson/mining-core'; +import type { ProjectedHistory } from '@workspacejson/mining-core'; + +/** + * Mine, score, select and project. Returns `undefined` when the repository + * cannot honestly produce a block. + * + * Errors are not swallowed silently — they are converted into the same absence + * a shallow clone produces, and the reason is surfaced on the returned + * diagnostics rather than discarded. + */ +export async function mineHistoryBlock( + repoRoot: string, + diagnostics: { refusal?: string } = {}, +): Promise { + let projected; + try { + projected = project(select(score(await mine(repoRoot)))); + } catch (error) { + // A Git invocation that fails is an absence of evidence, never a zero. The + // message is kept so a caller can report *why* nothing was mined instead of + // reporting that nothing was found. + diagnostics.refusal = `mining failed: ${error instanceof Error ? error.message : String(error)}`; + return undefined; + } + + if (!projected.projected) { + diagnostics.refusal = `${projected.refusal}: ${projected.detail}`; + return undefined; + } + + return projected.history; +} diff --git a/packages/cli/src/producer/history-refresh-outcome.test.ts b/packages/cli/src/producer/history-refresh-outcome.test.ts new file mode 100644 index 0000000..18a33aa --- /dev/null +++ b/packages/cli/src/producer/history-refresh-outcome.test.ts @@ -0,0 +1,72 @@ +/** + * A refused history refresh must not read as a completed one. + * + * Repo-native and validator-free: every case here drives the REFUSAL path, in + * which no `coChange` is emitted at all, so nothing touches the observation + * form that this workspace's published `@workspacejson/spec@0.4.4` rejects. + * The successful-refresh side of the contract — and the fallback-to-a-preserved + * block cases, which need an observation-form artifact already on disk — are + * measured in `candidate-tests/`, where the amended schema is available. They + * are not weakened to fit here. + * + * The defect these exist against (Greptile P1 on PR #20): mining refuses, + * generation falls back to the previously recorded block, and the caller gets a + * successful result carrying the PREVIOUS revision's counts with no way to tell + * that from a refresh that ran. Falling back is correct — destroying evidence + * over a shallow clone or a transient Git failure would be worse. Falling back + * *quietly* is the bug. + */ +import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { generateWorkspaceJson } from './generate.js'; + +/** A repository with NO commit graph, so an explicit refresh must refuse. */ +let root: string; + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'wsj-refresh-')); + await writeFile(join(root, 'AGENTS.md'), '# Fixture\n', 'utf8'); + await writeFile(join(root, 'package.json'), JSON.stringify({ name: 'fixture', version: '1.0.0' }), 'utf8'); + await mkdir(join(root, 'src'), { recursive: true }); + await writeFile(join(root, 'src/auth.ts'), 'export const auth = 1;\n', 'utf8'); +}); + +afterEach(async () => { + await rm(root, { recursive: true, force: true }); +}); + +describe('historyRefresh — a refused refresh is reported, not swallowed', () => { + it('reports the refusal when mining cannot complete', async () => { + const result = await generateWorkspaceJson(root, {}, { mineHistory: true }); + + expect(result.historyRefresh).toBeDefined(); + expect(result.historyRefresh!.requested).toBe(true); + expect(result.historyRefresh!.mined).toBe(false); + expect(result.historyRefresh!.refusal).toBeTruthy(); + }); + + it('names WHY, rather than reporting a bare failure', async () => { + // The reason was already being computed by `mineHistoryBlock` and thrown + // away, because the diagnostics object it accepts was never passed. A + // caller that cannot say "shallow clone" versus "git not found" cannot act + // on the refusal. + const result = await generateWorkspaceJson(root, {}, { mineHistory: true }); + expect(result.historyRefresh!.refusal).toMatch(/NOT_MINED|mining failed|NO_BASIS_PIN/); + }); + + it('is ABSENT when no refresh was requested — not a false-y refusal', async () => { + // Absence means "no refresh asked for". Reporting `mined: false` on an + // ordinary run would say a refresh was attempted and failed, which is a + // different and untrue claim. + const result = await generateWorkspaceJson(root); + expect(result.historyRefresh).toBeUndefined(); + }); + + it('carries a refusal if and only if mining produced nothing', async () => { + const refused = await generateWorkspaceJson(root, {}, { mineHistory: true }); + expect(refused.historyRefresh!.mined).toBe(false); + expect('refusal' in refused.historyRefresh!).toBe(true); + }); +}); diff --git a/packages/cli/vitest.config.ts b/packages/cli/vitest.config.ts new file mode 100644 index 0000000..302dc3c --- /dev/null +++ b/packages/cli/vitest.config.ts @@ -0,0 +1,16 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + // `candidate-tests/` is deliberately outside this workspace's suite. + // + // Those files exercise the observation form, which the published + // `@workspacejson/spec@0.4.4` and `@workspacejson/rules@0.4.4` reject — + // their schema predates ADR-003 A-009. They run against PACKED candidate + // builds in a disposable environment, on the Node test runner, so that this + // workspace keeps passing against its legitimate published dependencies. + // Excluding them here is what keeps the two boundaries from contaminating + // each other; it is not a way of skipping them. + exclude: ['**/node_modules/**', '**/dist/**', 'candidate-tests/**'], + }, +}); diff --git a/packages/mining-core/LICENSE b/packages/mining-core/LICENSE new file mode 100644 index 0000000..0d74b41 --- /dev/null +++ b/packages/mining-core/LICENSE @@ -0,0 +1,12 @@ +Apache License 2.0 + +Copyright (c) 2026 workspace-json contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may obtain a copy of the License at + +https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/packages/mining-core/README.md b/packages/mining-core/README.md new file mode 100644 index 0000000..744f1bb --- /dev/null +++ b/packages/mining-core/README.md @@ -0,0 +1,288 @@ +# `@workspacejson/mining-core` (L0) + +Commit-graph mining core for `workspace.json`. Reads git, returns an in-memory +observation set, writes nothing. + +Private and unpublished. It exists so that the producer, the META-289 harness, +and the report all take **one** implementation of the org's highest-value logic +rather than three — the META-140 defect class landing in exactly the numbers an +independent producer would be compared against. + +## Scope + +This package implements META-297 **Phases 1 to 3**. + +| Requirement | What it means here | +| -- | -- | +| REQ-001 | First-parent event extraction per META-289 v2.2.1's frozen parameters | +| REQ-002 | Empty-tree object computed from the repository, never hardcoded | +| REQ-003 | One exported `normalizePath`, with its unratified assumptions recorded in the output | +| REQ-004 | Two runs at the same basis produce byte-identical serialized output | +| REQ-005 | Four completeness states, never collapsed | +| REQ-006 | A shallow clone reports insufficient history, not zero | + +Those six are the only requirement identifiers written down for this package, +so they are the only ones cited. Phase 3 adds four behaviors that carry no +issue number, and they are therefore **named rather than numbered** — an +invented identifier reads as a citation and cites nothing: + +| Behavior | What it means here | +| -- | -- | +| **Weighting** | v2.2.1's `size_weight` and `position_decay`, implemented verbatim | +| **Scoring exclusion** | Events with `fileCount > 50` are excluded from scoring, and named | +| **Basis pinning** | The basis is a full-length object name, or it is not emitted at all | +| **Selection rule** | Threshold, rank, cap at 50, with an execution receipt | + +The generated-classifier question is **not** in this package; it is tracked +separately as META-316. See the note on `generated` below. + +**No artifact projection.** L1 — writing `generated.coChange` — is not here. The +schema is no longer the blocker it was during Phases 1 and 2: the standard's +A-009 amendment merged and admits the observation form (`support` + +`occurrences` + a pinned `generated.basisRevision`). But the package carrying it +is unpublished, and A-009 is an explicitly staged transition — widen the reader, +verify consumer adoption, *then* enable producer emission. Emission is step 3, +and this package does not authorize it. + +## Scoring + +`score(observationSet, { minSupport })` is a pure function. It spawns no +process, reads no filesystem and consults no clock; everything it needs was +extracted in Phase 1. That is what keeps the exclusion auditable — the same +observation set can be scored twice with different parameters, and the +difference is attributable to the parameters rather than to a second walk of a +repository that may have moved. + +Two vocabularies meet here, and are deliberately kept apart. + +**v2.2.1 supplies the weighting**, implemented verbatim. + +| Parameter | Value | +| -- | -- | +| `size_weight` | `min(1, 10/fileCount)` | +| `position_decay` | `2^(-Δpos/250)` | +| Window | 500 first-parent transitions | +| Scoring exclusion | events with `fileCount > 50` | +| Support threshold | `support >= 3` (`DEFAULT_MIN_SUPPORT`, overridable, recorded) | + +Δpos is measured from the newest **extracted** event, never the newest scored +one — excluding a large event must not shift the decay of everything older than +it. The recorded file-role and path exclusion set is **empty**: no path is +excluded for being documentation, a lockfile or generated output, so the size +rule above is the only exclusion L0 applies, and it applies to whole events +rather than to paths. Excluded events stay in the observation set and are named +by commit in `exclusions.excludedCommits`, because an exclusion nobody can point +at is not auditable. + +**A-009 supplies the counts**, and they are integers. + +- `support` — distinct scored events in which **both** files changed. +- `occurrences` — distinct scored events in which **at least one** changed. The + symmetric union, so reversing the pair changes nothing. Never a per-file + marginal. + +`weightedSupport` is emitted alongside the counts, never instead of them. +Nothing derived is stored: a rate is a reader's question. + +A repository longer than the window is **valid**. The bounded window is recorded +in `basisWindow` (`availableTransitions`, `extractedTransitions`, +`windowTruncated`), and truncation by the window is a fact rather than an error. + +## Basis pinning + +`scoringBasis` carries everything needed to recount a result: the frozen +weighting identifiers, the pinned basis, and both edges of the window. + +`basisRevision` is a full-length lowercase Git object name — A-009's +`^([0-9a-f]{40}|[0-9a-f]{64})$` — never a symbolic ref, because a pin that does +not name exactly one commit permanently cannot be recounted against. A basis +failing that shape **throws**: it is a caller bug, not a repository condition, +and a bug must not disguise itself as one. Where there is no window to pin, +`scoringBasis` is **absent**, not a placeholder that reads as a real pin. + +## Selection rule (provisional producer profile) + +`select(scoredSet, { minSupport, cap })` decides what a producer would emit. +Three steps, in this order: + +1. **Threshold** — keep pairs at `support >= 3`. +2. **Rank** — `support` DESC, then `occurrences` ASC, then `files[0]` ASC by + **UTF-8 bytes**, then `files[1]` ASC by UTF-8 bytes. +3. **Cap** — keep the first **50**, *after* ranking. + +Every ranking key is an integer or a byte sequence. Nothing continuous +participates, so the order cannot shift with floating-point precision. + +**UTF-8 byte order is not `a < b`.** A bare JavaScript comparison is UTF-16 code +unit order, and the two genuinely disagree: U+1F600 is a surrogate pair +beginning `0xD83D`, which sorts *before* U+E000 under UTF-16, while its UTF-8 +encoding `F0 9F 98 80` sorts *after* U+E000's `EE 80 80`. `compareUtf8` is +exported and is the only comparator the ranking uses. + +**Capping is a presentation step, not a data-loss step.** `select` is pure: the +scored set it was handed still carries every pair and every weight afterwards, +and the extracted events behind that are untouched. The cap changes what is +emitted, never what was observed. + +### Execution receipt + +A truncated list must be visibly truncated, never silently short — the same +doctrine as the completeness states, one layer up. `receipt` records: + +| Field | Meaning | +| -- | -- | +| `minSupport` | The threshold applied before ranking | +| `pairsBeforeCap` | Pairs that cleared the threshold — the population the cap cut from | +| `pairsEmitted` | Pairs actually emitted; `min(pairsBeforeCap, cap)` | +| `cap` | The cap in force | +| `rankingRule` | The complete rule, as text, so two artifacts can be compared without reading this file | +| `capBound` | Whether the cap actually bound | + +A reader can therefore distinguish an emitted 50 that is everything from an +emitted 50 that is the top of 1,848, without access to the repository. + +### No floats in artifact-bound output + +`SelectedPair` carries `files`, `support` and `occurrences`. It does **not** +carry `weightedSupport`, and `serializeSelection` **throws** on any non-integer +number rather than rounding or dropping it. Rounding would invent precision the +measurement does not have; dropping would remove a field a reader was told to +expect; both are quieter than the bug. + +The reason is measured, not stylistic: `weightedSupport` is a double from +`2 ** x`, whose precision ECMAScript leaves implementation-defined. A float in a +committed artifact is the churn class A-009 exists to prevent. It remains +available in memory on the scored set, and `serializeScoredSet` will emit it for +diagnostics that never reach `workspace.json`. + +**This constrains L1 too, and L1 is still not authorized.** + +## The `generated` flag — a finding, not a feature + +The schema requires `generated` on every `coChange` entry, documented as +`"true = tooling-coupled pair (e.g. lockfile + package.json); consumers skip +these"`. **L0 has no rule that can ever produce `true`.** The recorded +file-role and path exclusion set is empty, so nothing classifies a pair as +tooling-coupled and the field would be constant `false` on every entry a +producer emitted. + +The concrete case is worse than merely constant. On a 500-transition window of +`motdotla/dotenv` pinned at `2fc7eac8`, the **highest-ranked pair under the +selection rule above** is `package-lock.json` ↔ `package.json` at support 80 — +literally the example the field's own documentation cites. L0 would label it +`generated: false`. The field does not merely carry no information; it carries +the wrong answer on its own canonical example. + +Removing it is a schema change, not a producer choice, so this package records +the finding and changes nothing. Tracked as **META-316**. + +## Refresh model + +History mining is an **explicit refresh operation**, not part of default +generation. A bound 500-transition window costs 7.3–8.2 s with a short `PATH` +and 27.2–29.9 s with a long one on an Apple M4 Pro; adding that to every +generate is not viable, and the result is pinned to a commit rather than +recomputed per run. + +A-009 already supplies the staleness protocol: compare `generated.basisRevision` +against the current revision, where "pin ≠ current revision" is a defined reader +state meaning *stale observation*. That only works if the pin is allowed to lag, +which presupposes an explicit refresh. + +**The public command name is deliberately not chosen here.** Naming it is a +separate decision and this package does not pre-empt it. + +## Layering + +```text +L0 (this package) git only → observation set → scored set → selection +L1 producer L0 output → generated.coChange [held: A-009 step 3] +L2 report the artifact → human-readable findings [never invokes git] +``` + +`git.ts` is the only module here that spawns git. That is what makes the L2 +direction invariant checkable later. + +## Usage + +```ts +import { mine, score, select } from '@workspacejson/mining-core'; + +const observations = await mine('/path/to/repo', { basisRevision: 'HEAD' }); +const scored = score(observations); +const selection = select(scored); + +if (selection.completeness.state === 'QUALIFYING_RELATIONSHIP_OBSERVED') { + console.log('basis', selection.scoringBasis!.basisRevision); + console.log('receipt', selection.receipt); // threshold, counts, cap, rule, capBound + for (const pair of selection.pairs) { + console.log(pair.files, pair.support, pair.occurrences); + } +} + +// `scored` is untouched by `select` — every pair and every weight is still +// there, which is what makes the cap auditable. +``` + +`mine` does not throw for an absent, shallow, or unreadable history, and `score` +does not invent one. Those are completeness states, which is the entire point of +REQ-005 — and `score` gates on the state rather than on whether an events array +happens to be non-empty, because a `--depth 1` clone can hand over real events +that would otherwise produce a structurally identical answer at reduced +magnitude. + +### Cost + +Scoring is free; extraction is not. On an Apple M4 Pro, a bound 500-transition +window costs **7.3–8.2 s** with a short `PATH` and **27.2–29.9 s** with this +machine's 36-entry `PATH`, because v2.2.1's frozen parameters spend two `git` +subprocesses per commit and Node resolves the binary through `PATH` on every +one. Scoring the resulting 500 events costs **1–24 ms**. Plan the calling +interface around the extraction number, not the scoring one. + +## Completeness states + +Four values, and no code path maps two of them onto one: + +| State | Meaning | +| -- | -- | +| `NOT_MINED` | History was not mined, or evidence was not recorded. Not a claim about the repository. | +| `MINED_NO_QUALIFYING_RELATIONSHIP` | Mining completed over real history and found nothing qualifying. A claim. | +| `QUALIFYING_RELATIONSHIP_OBSERVED` | Mining completed and observed at least one qualifying relationship. | +| `EVIDENCE_UNAVAILABLE` | Evidence was reachable but malformed or unavailable. A failure, not a result. | + +Each carries a `reason`, because "not mined" that does not say *why* is the same +dead end as "0 partners". + +The state 2/3 boundary is the `qualifyingMinCooccurrence` option, default `1`, +and the value used is recorded in the output. It is **not** v2.2.1's +`rawSupport >= 3`: that is a scoring threshold, Phase 2 runs before scoring, and +baking it in here would ship a scoring decision under a completeness heading. +Phase 3 raises it without touching the state machine. + +## Path identity + +ADR-006 does not exist. META-278 poses six questions about canonical path +identity and the Phase 0 audit found zero implementations across +`workspacejson/cli` and `workspacejson/standard`, so `normalizePath` here is the +first one — the de facto rule ahead of ratification. + +It answers 2 of the 6 from the published schema's own field descriptions and +**assumes** the other 4 (case sensitivity, Unicode normalization, trailing +slash, symlink and submodule root resolution). Every assumption is declared in +`PATH_NORMALIZATION_ASSUMPTIONS` and carried in every observation set, so a +reader sees the guess as a guess. META-278 governs; this is not a proposal. + +## Tests + +```sh +pnpm test +``` + +Tests drive real git against real repositories — a mocked `diff-tree` would +prove nothing about extraction — so the suite takes tens of seconds. + +`billfold.test.ts` cross-checks against `workspace-json/billfold`, the one +repository where the correct answer is known in advance. It **skips** when that +clone is absent rather than passing vacuously; point `WORKSPACEJSON_BILLFOLD` at +a checkout to run it. diff --git a/packages/mining-core/package.json b/packages/mining-core/package.json new file mode 100644 index 0000000..f95240f --- /dev/null +++ b/packages/mining-core/package.json @@ -0,0 +1,47 @@ +{ + "name": "@workspacejson/mining-core", + "version": "0.0.0", + "private": true, + "description": "L0 commit-graph mining core — first-parent event extraction, canonical path identity, and completeness semantics. Consumed by the producer, the META-289 harness, and the report.", + "license": "Apache-2.0", + "author": "workspace.json contributors", + "homepage": "https://workspacejson.dev", + "repository": { + "type": "git", + "url": "git+https://github.com/workspacejson/cli.git", + "directory": "packages/mining-core" + }, + "bugs": { + "url": "https://github.com/workspacejson/cli/issues" + }, + "engines": { + "node": ">=20.0.0" + }, + "type": "module", + "main": "./dist/index.js", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "import": "./dist/index.js", + "types": "./dist/index.d.ts" + } + }, + "files": [ + "dist", + "README.md", + "LICENSE" + ], + "scripts": { + "build": "tsup src/index.ts --format esm --dts", + "test": "vitest run", + "test:watch": "vitest", + "typecheck": "tsc --noEmit" + }, + "devDependencies": { + "@types/node": "22.19.17", + "tsup": "^8.0.0", + "typescript": "^5.4.0", + "vitest": "^1.6.0" + } +} diff --git a/packages/mining-core/src/billfold.test.ts b/packages/mining-core/src/billfold.test.ts new file mode 100644 index 0000000..94afcbf --- /dev/null +++ b/packages/mining-core/src/billfold.test.ts @@ -0,0 +1,101 @@ +/** + * REQ-001 and REQ-002 against the billfold fixture (HAC-184). + * + * billfold is the only repository where the correct answer is known in advance, + * so it is the one place these requirements can be checked against something + * other than a fixture this package built for itself. + * + * It lives in a different repository (`workspace-json/billfold`) and is not a + * dependency, so these tests skip when it is absent rather than failing. A skip + * is honest; a green run against a fixture that was never there is not. Point + * `WORKSPACEJSON_BILLFOLD` at a clone to run them. + * + * The expectations below are hand-computed from the Phase 0 audit (comment + * 4c25d1f9 on META-297) and are committed here as numbers, not as a call to + * whatever the code currently returns. + */ +import { existsSync } from 'node:fs'; +import { describe, expect, it } from 'vitest'; +import { emptyTreeObject, extractEvents } from './git.js'; +import { mine } from './mine.js'; + +const BILLFOLD = process.env.WORKSPACEJSON_BILLFOLD ?? '/Users/user1/dev/billfold'; +const BASIS = 'origin/main'; + +/** Hand-computed from the Phase 0 census. Change these only with evidence. */ +const EXPECTED = { + /** `git rev-list --first-parent --count origin/main`. */ + firstParentTransitions: 44, + /** The thesis pair — no import edge between them. */ + thesisPair: ['src/routes/checkout.ts', 'src/webhooks/stripe.ts'] as const, + /** Its raw co-occurrence WITH empty-tree handling for the root commit. */ + thesisSupportWithEmptyTree: 6, + /** And WITHOUT it: the root commit drops out, taking one unit of support. */ + thesisSupportWithoutEmptyTree: 5, + /** The root commit, which only an empty-tree parent makes visible. */ + rootCommitFileCount: 24, + /** Unweighted pairs at raw support >= 3, from the audit. */ + pairsAtSupport3: [ + { files: ['docs/OPERATIONS.md', 'src/routes/checkout.ts'], support: 9 }, + { files: ['docs/OPERATIONS.md', 'src/webhooks/stripe.ts'], support: 6 }, + { files: ['src/routes/checkout.ts', 'src/webhooks/stripe.ts'], support: 6 }, + { files: ['docs/OPERATIONS.md', 'src/db/client.ts'], support: 4 }, + ], +} as const; + +const available = existsSync(BILLFOLD); +const describeBillfold = available ? describe : describe.skip; + +describeBillfold('billfold cross-check (REQ-001, REQ-002)', () => { + it('extracts the expected number of first-parent transitions', async () => { + const result = await mine(BILLFOLD, { basisRevision: BASIS }); + expect(result.basisWindow?.availableTransitions).toBe(EXPECTED.firstParentTransitions); + // Window 500 does not bind on a 44-event history. + expect(result.basisWindow?.windowTruncated).toBe(false); + expect(result.events).toHaveLength(EXPECTED.firstParentTransitions); + }); + + it('reproduces the audit census at raw support >= 3', async () => { + const result = await mine(BILLFOLD, { basisRevision: BASIS, qualifyingMinCooccurrence: 3 }); + const observed = result.pairs + .map((p) => ({ files: [...p.files], support: p.cooccurrenceCount })) + .sort((a, b) => b.support - a.support || (a.files[0]! < b.files[0]! ? -1 : 1)); + const expected = EXPECTED.pairsAtSupport3 + .map((p) => ({ files: [...p.files], support: p.support })) + .sort((a, b) => b.support - a.support || (a.files[0]! < b.files[0]! ? -1 : 1)); + expect(observed).toEqual(expected); + }); + + it('REQ-002: empty-tree handling is worth exactly one unit of support on the thesis pair', async () => { + const events = await extractEvents(BILLFOLD, { + basisRevision: BASIS, + windowTransitions: 1000, + }); + + const [left, right] = EXPECTED.thesisPair; + const countPair = (subset: typeof events): number => + subset.filter((e) => e.files.includes(left) && e.files.includes(right)).length; + + const withEmptyTree = countPair(events); + expect(withEmptyTree).toBe(EXPECTED.thesisSupportWithEmptyTree); + + // The root commit is the only event whose parent is the empty tree. Drop it + // and you have simulated the hardcoded-literal failure on a repository + // whose object format the literal does not match. + const emptyTree = await emptyTreeObject(BILLFOLD); + const rootEvents = events.filter((e) => e.parent === emptyTree); + expect(rootEvents).toHaveLength(1); + expect(rootEvents[0]!.fileCount).toBe(EXPECTED.rootCommitFileCount); + + const withoutEmptyTree = countPair(events.filter((e) => e.parent !== emptyTree)); + expect(withoutEmptyTree).toBe(EXPECTED.thesisSupportWithoutEmptyTree); + expect(withEmptyTree - withoutEmptyTree).toBe(1); + }); + + it('REQ-004: two runs at the same basis agree byte for byte', async () => { + const { serializeObservationSet } = await import('./serialize.js'); + const first = serializeObservationSet(await mine(BILLFOLD, { basisRevision: BASIS })); + const second = serializeObservationSet(await mine(BILLFOLD, { basisRevision: BASIS })); + expect(first).toBe(second); + }); +}); diff --git a/packages/mining-core/src/completeness.ts b/packages/mining-core/src/completeness.ts new file mode 100644 index 0000000..1a5fff5 --- /dev/null +++ b/packages/mining-core/src/completeness.ts @@ -0,0 +1,96 @@ +/** + * Completeness semantics for L0 (REQ-005). + * + * The 2026-08-05 comment on META-297 names four states and requires that they + * never collapse. The failure this prevents is specific: a shallow clone, an + * unreadable history and a genuinely uncoupled repository all produce zero + * pairs, and reporting "0 partners" for all three tells a reader that the + * repository was examined and found clean when two of the three mean the + * examination did not happen. That is AP-1 — a graceful empty return masking + * missing capability — and it is the one failure mode here that does real + * damage if a report built on a truncated clone reaches an external reader. + * + * So these are four distinct values, not a boolean plus a note, and there is no + * code path that maps two of them onto one. + */ +export const CompletenessState = { + /** 1. History was not mined, or evidence was not recorded. Absence of a claim. */ + NOT_MINED: 'NOT_MINED', + /** 2. Mining completed over real history and found no qualifying relationship. A claim. */ + MINED_NO_QUALIFYING_RELATIONSHIP: 'MINED_NO_QUALIFYING_RELATIONSHIP', + /** 3. Mining completed and at least one qualifying relationship was observed. A claim. */ + QUALIFYING_RELATIONSHIP_OBSERVED: 'QUALIFYING_RELATIONSHIP_OBSERVED', + /** 4. Evidence was reachable but malformed or unavailable. A failure, not a result. */ + EVIDENCE_UNAVAILABLE: 'EVIDENCE_UNAVAILABLE', +} as const; + +export type CompletenessState = (typeof CompletenessState)[keyof typeof CompletenessState]; + +/** + * Why L0 landed in the state it did. + * + * States 1 and 4 are meaningless without this — "not mined" that does not say + * *why* is the same dead end as "0 partners". + */ +export const CompletenessReason = { + /** State 1. `git rev-parse --is-shallow-repository` returned true. */ + SHALLOW_CLONE: 'SHALLOW_CLONE', + /** State 1. The path is not a git repository, or git is not on PATH. */ + NO_REPOSITORY: 'NO_REPOSITORY', + /** State 1. A repository exists but the basis revision resolves to no commits. */ + NO_COMMITS: 'NO_COMMITS', + /** State 1. Mining was not requested. */ + NOT_REQUESTED: 'NOT_REQUESTED', + /** States 2 and 3. Extraction ran to completion. */ + MINED: 'MINED', + /** State 4. A git invocation failed, or its output did not parse. */ + GIT_FAILED: 'GIT_FAILED', + /** State 4. Output parsed but violated an invariant extraction relies on. */ + MALFORMED_OUTPUT: 'MALFORMED_OUTPUT', +} as const; + +export type CompletenessReason = (typeof CompletenessReason)[keyof typeof CompletenessReason]; + +/** + * Which reasons belong to which state. + * + * Exported so the pairing can be asserted rather than trusted: a reason that + * drifts onto the wrong state is exactly the collapse REQ-005 forbids, and it + * would otherwise be invisible. + */ +export const REASONS_BY_STATE: Readonly> = + Object.freeze({ + [CompletenessState.NOT_MINED]: Object.freeze([ + CompletenessReason.SHALLOW_CLONE, + CompletenessReason.NO_REPOSITORY, + CompletenessReason.NO_COMMITS, + CompletenessReason.NOT_REQUESTED, + ]), + [CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP]: Object.freeze([CompletenessReason.MINED]), + [CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED]: Object.freeze([CompletenessReason.MINED]), + [CompletenessState.EVIDENCE_UNAVAILABLE]: Object.freeze([ + CompletenessReason.GIT_FAILED, + CompletenessReason.MALFORMED_OUTPUT, + ]), + }); + +export interface Completeness { + state: CompletenessState; + reason: CompletenessReason; + /** Human-readable detail. Never the sole carrier of a distinction. */ + detail: string; +} + +export function completeness( + state: CompletenessState, + reason: CompletenessReason, + detail: string, +): Completeness { + const permitted = REASONS_BY_STATE[state]; + if (!permitted.includes(reason)) { + throw new Error( + `completeness: reason ${reason} is not valid for state ${state} (valid: ${permitted.join(', ')})`, + ); + } + return { state, reason, detail }; +} diff --git a/packages/mining-core/src/git.test.ts b/packages/mining-core/src/git.test.ts new file mode 100644 index 0000000..c4d0934 --- /dev/null +++ b/packages/mining-core/src/git.test.ts @@ -0,0 +1,136 @@ +/** REQ-001 extraction, REQ-002 computed empty tree. */ +import { readFileSync, readdirSync, statSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterAll, describe, expect, it } from 'vitest'; +import { GitOutputError, emptyTreeObject, extractEvents, parseNameStatusZ } from './git.js'; +import { commit, makeCoupledRepo, removeDir, writeAndAdd, git } from './testing/fixtures.js'; + +const srcDir = dirname(fileURLToPath(import.meta.url)); +const created: string[] = []; +function fixture(make: () => string): string { + const root = make(); + created.push(root); + return root; +} +afterAll(() => created.forEach(removeDir)); + +describe('parseNameStatusZ', () => { + it('reads single-path statuses', () => { + expect(parseNameStatusZ('M\0src/a.ts\0A\0src/b.ts\0')).toEqual(['src/a.ts', 'src/b.ts']); + }); + + it('reads a rename as two paths — both were touched', () => { + expect(parseNameStatusZ('R100\0src/old.ts\0src/new.ts\0')).toEqual([ + 'src/new.ts', + 'src/old.ts', + ]); + }); + + it('normalizes through the single normalizer', () => { + expect(parseNameStatusZ('M\0./src/a.ts\0')).toEqual(['src/a.ts']); + }); + + it('survives paths that would break a line-based parser', () => { + // A newline in a path is the exact case `-z` exists for. + expect(parseNameStatusZ('M\0src/we\nird.ts\0')).toEqual(['src/we\nird.ts']); + }); + + it('deduplicates within one event', () => { + expect(parseNameStatusZ('M\0src/a.ts\0M\0src/a.ts\0')).toEqual(['src/a.ts']); + }); + + it('raises rather than guessing when the stream goes out of phase', () => { + expect(() => parseNameStatusZ('src/a.ts\0src/b.ts\0')).toThrow(GitOutputError); + expect(() => parseNameStatusZ('R100\0src/old.ts\0')).toThrow(GitOutputError); + }); +}); + +describe('REQ-002 — empty tree is computed, never hardcoded', () => { + it('asks git for the repository object format', async () => { + const root = fixture(makeCoupledRepo); + const computed = await emptyTreeObject(root); + const fromGit = git(root, ['hash-object', '-t', 'tree', '/dev/null']).trim(); + expect(computed).toBe(fromGit); + }); + + it('contains no hardcoded SHA-1 empty-tree literal anywhere in the package', () => { + // The forbidden literal is assembled here rather than written, so this + // assertion does not itself become the grep hit it is testing for. + const forbidden = ['4b825dc6', '42cb6eb9', 'a060e54b', 'f8d69288', 'fbee4904'].join(''); + const offenders: string[] = []; + const walk = (dir: string): void => { + for (const entry of readdirSync(dir)) { + const full = join(dir, entry); + if (statSync(full).isDirectory()) walk(full); + else if (full.endsWith('.ts') && readFileSync(full, 'utf8').includes(forbidden)) { + offenders.push(full.slice(srcDir.length + 1)); + } + } + }; + walk(srcDir); + expect(offenders).toEqual([]); + }); +}); + +describe('REQ-001 — first-parent extraction', () => { + it('extracts one event per first-parent transition, oldest first', async () => { + const root = fixture(makeCoupledRepo); + const events = await extractEvents(root, { basisRevision: 'HEAD', windowTransitions: 500 }); + + // makeCoupledRepo: initial, round 1, round 2, docs. + expect(events).toHaveLength(4); + expect(events.map((e) => e.position)).toEqual([0, 1, 2, 3]); + }); + + it('includes the root commit, whose parent is the empty tree', async () => { + const root = fixture(makeCoupledRepo); + const events = await extractEvents(root, { basisRevision: 'HEAD', windowTransitions: 500 }); + const rootEvent = events[0]!; + + expect(rootEvent.parent).toBe(await emptyTreeObject(root)); + // Three files created in the initial commit. + expect(rootEvent.files).toEqual(['README.md', 'src/build.ts', 'src/parse.ts']); + expect(rootEvent.fileCount).toBe(3); + }); + + it('matches a hand-computed file-set expectation for every event', async () => { + const root = fixture(makeCoupledRepo); + const events = await extractEvents(root, { basisRevision: 'HEAD', windowTransitions: 500 }); + + expect(events.map((e) => [...e.files])).toEqual([ + ['README.md', 'src/build.ts', 'src/parse.ts'], + ['src/build.ts', 'src/parse.ts'], + ['src/build.ts', 'src/parse.ts'], + ['README.md'], + ]); + }); + + it('takes the window from the newest end', async () => { + const root = fixture(makeCoupledRepo); + const events = await extractEvents(root, { basisRevision: 'HEAD', windowTransitions: 2 }); + + expect(events).toHaveLength(2); + // The two most recent transitions: "round 2" then "docs". + expect(events.map((e) => [...e.files])).toEqual([['src/build.ts', 'src/parse.ts'], ['README.md']]); + expect(events.map((e) => e.position)).toEqual([0, 1]); + }); + + it('records both paths of a rename', async () => { + const root = fixture(makeCoupledRepo); + git(root, ['mv', 'src/parse.ts', 'src/parser.ts']); + commit(root, 'rename parse to parser'); + + const events = await extractEvents(root, { basisRevision: 'HEAD', windowTransitions: 1 }); + expect(events[0]!.files).toEqual(['src/parse.ts', 'src/parser.ts']); + }); + + it('does not filter large events — exclusion is Phase 3 and must stay countable', async () => { + const root = fixture(makeCoupledRepo); + for (let i = 0; i < 60; i += 1) writeAndAdd(root, `bulk/f${i}.ts`, `export const n = ${i};\n`); + commit(root, 'bulk add'); + + const events = await extractEvents(root, { basisRevision: 'HEAD', windowTransitions: 1 }); + expect(events[0]!.fileCount).toBe(60); + }); +}); diff --git a/packages/mining-core/src/git.ts b/packages/mining-core/src/git.ts new file mode 100644 index 0000000..cc85f13 --- /dev/null +++ b/packages/mining-core/src/git.ts @@ -0,0 +1,282 @@ +/** + * Git access for L0 (REQ-001, REQ-002). + * + * The only module in this package that invokes git. Everything above it works + * on the extracted event set, which is what makes the L2 direction invariant + * checkable later: a report that reads the artifact touches nothing here. + * + * Commands are exactly META-289 v2.2.1's frozen extraction parameters, written + * out rather than composed, because the point of a frozen parameter is that a + * reader can compare it to the preregistration without reconstructing it. + */ +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { normalizePath } from './paths.js'; + +const run = promisify(execFile); + +/** Raised when git itself fails. Distinct from "git ran and said no". */ +export class GitInvocationError extends Error { + constructor( + readonly args: readonly string[], + readonly cause: unknown, + ) { + super(`git ${args.join(' ')} failed: ${cause instanceof Error ? cause.message : String(cause)}`); + this.name = 'GitInvocationError'; + } +} + +/** Raised when git succeeds but its output violates an extraction invariant. */ +export class GitOutputError extends Error { + constructor(message: string) { + super(message); + this.name = 'GitOutputError'; + } +} + +async function git(repoRoot: string, args: readonly string[]): Promise { + try { + // maxBuffer default is 1MB; a 500-event `diff-tree` sweep on a large + // repository clears that easily and would surface as a truncation rather + // than an error, which is the quiet-wrong-answer class this whole issue is + // about. 256MB is well past any realistic window. + const { stdout } = await run('git', [...args], { + cwd: repoRoot, + maxBuffer: 256 * 1024 * 1024, + // Force plumbing-stable output regardless of the invoking user's config. + env: { ...process.env, GIT_CONFIG_NOSYSTEM: '1', LC_ALL: 'C' }, + }); + return stdout; + } catch (error) { + throw new GitInvocationError(args, error); + } +} + +export async function isGitRepository(repoRoot: string): Promise { + try { + const out = await git(repoRoot, ['rev-parse', '--is-inside-work-tree']); + return out.trim() === 'true'; + } catch { + return false; + } +} + +/** + * REQ-006's detection. A shallow clone has history it cannot see, and the + * events it *can* see are indistinguishable from a short complete history. + */ +export async function isShallowRepository(repoRoot: string): Promise { + const out = await git(repoRoot, ['rev-parse', '--is-shallow-repository']); + return out.trim() === 'true'; +} + +/** + * REQ-002. The empty tree object, asked of git rather than hardcoded. + * + * The well-known literal is SHA-1 specific. On a SHA-256 repository it is not a + * valid object, and the failure is silent in the worst way: the root commit's + * diff either errors or comes back empty, so the first commit's file set + * vanishes from the event stream and every pair it contributed to loses + * exactly one unit of support. That is not hypothetical here — the Phase 0 + * audit measured it as the difference between support 5 and 6 on the billfold + * thesis pair. + * + * `hash-object -t tree /dev/null` returns whatever the repository's object + * format says the empty tree is, which is the only correct answer. + */ +export async function emptyTreeObject(repoRoot: string): Promise { + const out = await git(repoRoot, ['hash-object', '-t', 'tree', '/dev/null']); + const hash = out.trim(); + if (!/^[0-9a-f]{40,64}$/.test(hash)) { + throw new GitOutputError(`empty-tree hash is not a valid object id: ${JSON.stringify(hash)}`); + } + return hash; +} + +/** + * Resolve a revision to a commit object id, or `undefined` if it names no + * commit. + * + * Returns rather than throws because "this revision does not resolve" is an + * ordinary condition — an empty repository has no `HEAD` — and it maps to + * REQ-005 state 1, not state 4. Throwing here would force the caller to guess + * from an exception whether the history is absent or the evidence is broken, + * which is exactly the collapse REQ-005 forbids. + */ +export async function resolveCommit( + repoRoot: string, + revision: string, +): Promise { + try { + const out = await git(repoRoot, ['rev-parse', '--verify', '--quiet', `${revision}^{commit}`]); + const commit = out.trim(); + return commit.length > 0 ? commit : undefined; + } catch { + return undefined; + } +} + +/** First-parent transitions reachable from a commit. */ +export async function countFirstParent(repoRoot: string, commit: string): Promise { + const out = await git(repoRoot, ['rev-list', '--first-parent', '--count', commit]); + const count = Number.parseInt(out.trim(), 10); + if (!Number.isInteger(count) || count < 0) { + throw new GitOutputError(`rev-list --count returned ${JSON.stringify(out.trim())}`); + } + return count; +} + +/** A commit's parent, or the empty tree when it has none. */ +async function firstParentOrEmptyTree( + repoRoot: string, + commit: string, + emptyTree: string, +): Promise { + try { + const out = await git(repoRoot, ['rev-parse', '--verify', '--quiet', `${commit}^`]); + const parent = out.trim(); + if (parent.length > 0) return parent; + } catch { + // `rev-parse --verify --quiet` exits non-zero for a root commit. That is + // the expected path, not an error. + } + return emptyTree; +} + +/** One first-parent transition and the paths it touched. */ +export interface CommitEvent { + /** Full object id of the child commit. */ + commit: string; + /** Parent, or the empty tree for the root commit. */ + parent: string; + /** + * Distinct normalized paths touched, sorted. For a rename, both the old and + * new path appear — the transition touched both, and dropping either would + * lose the coupling the rename represents. + */ + files: readonly string[]; + /** `files.length`. Named because v2.2.1's weighting and exclusion key on it. */ + fileCount: number; + /** + * Position in the extracted sequence, 0 for the oldest. v2.2.1's + * `position_decay` is defined over this. Phase 1 records it and does not use + * it — scoring is Phase 3. + */ + position: number; +} + +export interface ExtractionOptions { + /** Revision whose first-parent history is walked. */ + basisRevision: string; + /** v2.2.1: 500 first-parent transitions. */ + windowTransitions: number; +} + +/** + * REQ-001. Extract first-parent events per v2.2.1's frozen parameters. + * + * Two commands, quoted from the preregistration: + * + * git rev-list --first-parent --reverse + * git -c diff.renamelimit=5000 diff-tree -r --name-status -z --no-commit-id -M50% + * + * `--reverse` makes position 0 the oldest commit, so the window is taken from + * the newest end and `position_decay` measures distance back from the basis. + * + * No filtering, no weighting, no scoring. Phase 1 is extraction only, and every + * event the walk produces comes back — including events v2.2.1 will later + * exclude for `fileCount > 50`. Discarding them here would make the exclusion + * unobservable, and an exclusion nobody can count is not auditable. + */ +export async function extractEvents( + repoRoot: string, + options: ExtractionOptions, +): Promise { + const emptyTree = await emptyTreeObject(repoRoot); + + const revList = await git(repoRoot, [ + 'rev-list', + '--first-parent', + '--reverse', + options.basisRevision, + ]); + const allCommits = revList.split('\n').filter((line) => line.length > 0); + + // The window is the most recent N transitions. `--reverse` already put oldest + // first, so that is the tail. + const commits = + allCommits.length > options.windowTransitions + ? allCommits.slice(allCommits.length - options.windowTransitions) + : allCommits; + + const events: CommitEvent[] = []; + for (const [position, commit] of commits.entries()) { + const parent = await firstParentOrEmptyTree(repoRoot, commit, emptyTree); + const raw = await git(repoRoot, [ + '-c', + 'diff.renamelimit=5000', + 'diff-tree', + '-r', + '--name-status', + '-z', + '--no-commit-id', + '-M50%', + parent, + commit, + ]); + const files = parseNameStatusZ(raw); + events.push({ commit, parent, files, fileCount: files.length, position }); + } + + return events; +} + +/** + * Parse `--name-status -z` output. + * + * The format is NUL-separated and status-dependent, which is the whole reason + * `-z` is in the frozen parameters: without it a path containing a quote, + * newline or non-ASCII byte comes back C-quoted, and a naive line split + * silently mangles exactly the paths the Unicode question in META-278 is about. + * + * Layout: a status field, then one path, except for `R` and `C` which carry a + * similarity score on the status and are followed by two paths (source, then + * destination). Both are emitted — a rename is a transition that touched the + * old path and the new one. + */ +export function parseNameStatusZ(raw: string): readonly string[] { + const fields = raw.split('\0').filter((field) => field.length > 0); + const paths = new Set(); + + let index = 0; + while (index < fields.length) { + const status = fields[index]!; + index += 1; + + // A status field is a letter plus an optional numeric similarity score. + // Anything else means the stream is out of phase, and continuing would + // read paths as statuses and statuses as paths. + if (!/^[ACDMRTUXB][0-9]*$/.test(status)) { + throw new GitOutputError( + `diff-tree --name-status -z: expected a status field, got ${JSON.stringify(status)} at field ${index - 1}`, + ); + } + + const pathCount = status.startsWith('R') || status.startsWith('C') ? 2 : 1; + for (let taken = 0; taken < pathCount; taken += 1) { + const path = fields[index]; + if (path === undefined) { + throw new GitOutputError( + `diff-tree --name-status -z: status ${status} expects ${pathCount} path(s), stream ended early`, + ); + } + index += 1; + paths.add(normalizePath(path)); + } + } + + // Default comparator, UTF-16 code unit order. Never `localeCompare`, which + // varies with host locale and would make REQ-004's byte-identical claim + // depend on the machine it ran on. + return [...paths].sort(); +} diff --git a/packages/mining-core/src/index.ts b/packages/mining-core/src/index.ts new file mode 100644 index 0000000..18b8b36 --- /dev/null +++ b/packages/mining-core/src/index.ts @@ -0,0 +1,89 @@ +/** + * L0 mining core — public surface. + * + * Phases 1 to 3 of META-297: extraction, normalization, completeness + * semantics, v2.2.1 scoring, basis pinning, and the selection rule. + * + * Still nothing here writes to the artifact. L1 projection — mapping a + * selection onto `generated.coChange` — is a separate step held closed by the + * coordinator gate. The standard's A-009 amendment has merged and admits the + * observation form (`support` + `occurrences` + a pinned `basisRevision`), so + * the schema is no longer the blocker it was during Phases 1 and 2; the + * package carrying it is unpublished and emission is step 3 of A-009's staged + * transition, which this package does not authorize. + * + * The pipeline is `mine` → `score` → `select`, each pure with respect to the + * one before it, so the extracted events and the uncapped scored result remain + * auditable after the selection has capped anything. + * + * Named behaviors, not numbered ones. REQ-001..006 are written down for this + * package and are cited; the weighting, the scoring exclusion, basis pinning + * and the selection rule are not, so they are named. An invented identifier + * reads as a citation and cites nothing. + */ +export { + type Completeness, + CompletenessReason, + CompletenessState, + REASONS_BY_STATE, +} from './completeness.js'; +export { + type CommitEvent, + GitInvocationError, + GitOutputError, + emptyTreeObject, + extractEvents, + parseNameStatusZ, +} from './git.js'; +export { + DEFAULT_QUALIFYING_MIN_COOCCURRENCE, + DEFAULT_WINDOW_TRANSITIONS, + type BasisWindow, + type MineOptions, + type ObservationSet, + type PairObservation, + mine, +} from './mine.js'; +export { + PATH_NORMALIZATION_ASSUMPTIONS, + UNRATIFIED_ASSUMPTION_COUNT, + type PathAssumption, + normalizePath, +} from './paths.js'; +export { + DEFAULT_MIN_SUPPORT, + POSITION_DECAY_HALF_LIFE, + SCORING_MAX_FILE_COUNT, + SIZE_WEIGHT_NUMERATOR, + WEIGHTING_VERSION, + type ScoreOptions, + type ScoredPair, + type ScoredSet, + type ScoringBasis, + type ScoringExclusions, + positionDecay, + score, + sizeWeight, +} from './score.js'; +export { + RANKING_RULE, + SELECTION_CAP, + type SelectOptions, + type SelectedPair, + type SelectionReceipt, + type SelectionResult, + compareUtf8, + select, +} from './select.js'; +export { + serializeObservationSet, + serializeScoredSet, + serializeSelection, +} from './serialize.js'; +export { + type ProjectedCoChangeEntry, + type ProjectedHistory, + type ProjectionResult, + ProjectionRefusal, + project, +} from './project.js'; diff --git a/packages/mining-core/src/mine.test.ts b/packages/mining-core/src/mine.test.ts new file mode 100644 index 0000000..dee35a2 --- /dev/null +++ b/packages/mining-core/src/mine.test.ts @@ -0,0 +1,187 @@ +/** REQ-004 determinism, REQ-005 four completeness states, REQ-006 shallow clone. */ +import { afterAll, describe, expect, it } from 'vitest'; +import { CompletenessReason, CompletenessState, REASONS_BY_STATE } from './completeness.js'; +import { mine } from './mine.js'; +import { serializeObservationSet } from './serialize.js'; +import { + commit, + makeCorruptedRepo, + makeCoupledRepo, + makeEmptyRepo, + makeNonRepo, + makeShallowCloneOfCoupled, + makeUncoupledRepo, + removeDir, + writeAndAdd, +} from './testing/fixtures.js'; + +const created: string[] = []; +function fixture(make: () => string): string { + const root = make(); + created.push(root); + return root; +} +afterAll(() => created.forEach(removeDir)); + +describe('REQ-005 — four completeness states, never collapsed', () => { + it('state 1: an initialized repository with no commits is NOT_MINED / NO_COMMITS', async () => { + const result = await mine(fixture(makeEmptyRepo)); + expect(result.completeness.state).toBe(CompletenessState.NOT_MINED); + expect(result.completeness.reason).toBe(CompletenessReason.NO_COMMITS); + expect(result.pairs).toHaveLength(0); + }); + + it('state 1: a path that is not a repository is NOT_MINED / NO_REPOSITORY', async () => { + const result = await mine(fixture(makeNonRepo)); + expect(result.completeness.state).toBe(CompletenessState.NOT_MINED); + expect(result.completeness.reason).toBe(CompletenessReason.NO_REPOSITORY); + }); + + it('state 2: real history with no co-occurrence is MINED_NO_QUALIFYING_RELATIONSHIP', async () => { + const result = await mine(fixture(makeUncoupledRepo)); + expect(result.completeness.state).toBe(CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP); + expect(result.completeness.reason).toBe(CompletenessReason.MINED); + expect(result.events.length).toBeGreaterThan(0); + expect(result.pairs).toHaveLength(0); + }); + + it('state 3: an observed co-change pair is QUALIFYING_RELATIONSHIP_OBSERVED', async () => { + const result = await mine(fixture(makeCoupledRepo)); + expect(result.completeness.state).toBe(CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED); + expect(result.completeness.reason).toBe(CompletenessReason.MINED); + + const pair = result.pairs.find( + (p) => p.files[0] === 'src/build.ts' && p.files[1] === 'src/parse.ts', + ); + expect(pair?.cooccurrenceCount).toBe(3); + }); + + it('state 4: a history that cannot be walked is EVIDENCE_UNAVAILABLE, not an empty result', async () => { + // Driven through the real failure path, not asserted on a constructor. + // HEAD resolves; the walk to the root commit hits a deleted object. + const root = makeCorruptedRepo(); + if (root === undefined) return; // objects were packed; nothing was corrupted + created.push(root); + + const result = await mine(root); + expect(result.completeness.state).toBe(CompletenessState.EVIDENCE_UNAVAILABLE); + expect(result.completeness.reason).toBe(CompletenessReason.GIT_FAILED); + // The damage it must not do: report this as a clean, examined repository. + expect(result.completeness.state).not.toBe( + CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP, + ); + }); + + it('an unresolvable revision is an absent history, not a broken one', async () => { + const result = await mine(fixture(makeCoupledRepo), { + basisRevision: 'refs/heads/does-not-exist', + }); + expect(result.completeness.state).toBe(CompletenessState.NOT_MINED); + expect(result.completeness.reason).toBe(CompletenessReason.NO_COMMITS); + }); + + it('states 2 and 3 are the only two that can carry MINED', () => { + const carriers = Object.entries(REASONS_BY_STATE) + .filter(([, reasons]) => reasons.includes(CompletenessReason.MINED)) + .map(([state]) => state); + expect(carriers.sort()).toEqual( + [ + CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP, + CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED, + ].sort(), + ); + }); + + it('state 1 and state 2 are distinguishable, which is the whole point', async () => { + const notMined = await mine(fixture(makeEmptyRepo)); + const minedEmpty = await mine(fixture(makeUncoupledRepo)); + + // Both report zero pairs. Only one of them examined anything. + expect(notMined.pairs).toHaveLength(0); + expect(minedEmpty.pairs).toHaveLength(0); + expect(notMined.completeness.state).not.toBe(minedEmpty.completeness.state); + }); +}); + +describe('REQ-006 — shallow clone reports insufficient history, not zero', () => { + it('reports NOT_MINED / SHALLOW_CLONE where the full history has a real pair', async () => { + const { source, shallow } = makeShallowCloneOfCoupled(); + created.push(source, shallow); + + // The full clone finds the pair, so the shallow clone's silence is a + // capability gap and not a property of the repository. + const full = await mine(source); + expect(full.completeness.state).toBe(CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED); + + const result = await mine(shallow); + expect(result.completeness.state).toBe(CompletenessState.NOT_MINED); + expect(result.completeness.reason).toBe(CompletenessReason.SHALLOW_CLONE); + expect(result.completeness.state).not.toBe( + CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP, + ); + }); +}); + +describe('REQ-004 — determinism', () => { + it('produces byte-identical serialized output across two runs', async () => { + const root = fixture(makeCoupledRepo); + const first = serializeObservationSet(await mine(root, { basisRevision: 'HEAD' })); + const second = serializeObservationSet(await mine(root, { basisRevision: 'HEAD' })); + expect(first).toBe(second); + }); + + it('carries no wall-clock value, so output cannot move without the repository moving', async () => { + const root = fixture(makeCoupledRepo); + const before = serializeObservationSet(await mine(root)); + // Same repository, later moment. + await new Promise((resolve) => setTimeout(resolve, 25)); + const after = serializeObservationSet(await mine(root)); + expect(after).toBe(before); + }); + + it('moves when — and only when — the basis advances', async () => { + const root = fixture(makeCoupledRepo); + const before = serializeObservationSet(await mine(root)); + + writeAndAdd(root, 'src/build.ts', 'export const key = () => "changed";\n'); + writeAndAdd(root, 'src/parse.ts', '// changed\n'); + commit(root, 'another coupled change'); + + const after = serializeObservationSet(await mine(root)); + expect(after).not.toBe(before); + }); + + it('pins the resolved basis commit so the window is reproducible after refs move', async () => { + const root = fixture(makeCoupledRepo); + const result = await mine(root); + expect(result.basisWindow?.basisCommit).toMatch(/^[0-9a-f]{40,64}$/); + expect(result.basisWindow?.extractedTransitions).toBe(result.events.length); + expect(result.basisWindow?.windowTruncated).toBe(false); + }); +}); + +describe('recorded parameters', () => { + it('records the qualifying threshold it used rather than implying one', async () => { + const result = await mine(fixture(makeCoupledRepo)); + expect(result.qualifyingMinCooccurrence).toBe(1); + }); + + it('honours a raised threshold without changing the state machine', async () => { + const root = fixture(makeCoupledRepo); + // v2.2.1's rawSupport >= 3 applied early: the pair co-occurs exactly 3 + // times, so it still qualifies; at 4 it does not, and the state flips to + // "mined, nothing qualifying" rather than to "not mined". + const atThree = await mine(root, { qualifyingMinCooccurrence: 3 }); + expect(atThree.completeness.state).toBe(CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED); + + const atFour = await mine(root, { qualifyingMinCooccurrence: 4 }); + expect(atFour.completeness.state).toBe(CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP); + expect(atFour.events.length).toBeGreaterThan(0); + }); + + it('carries the path-normalization assumption record in its output', async () => { + const result = await mine(fixture(makeCoupledRepo)); + expect(result.pathNormalization).toHaveLength(6); + expect(result.pathNormalization.filter((a) => a.standing === 'assumed')).toHaveLength(4); + }); +}); diff --git a/packages/mining-core/src/mine.ts b/packages/mining-core/src/mine.ts new file mode 100644 index 0000000..a00561a --- /dev/null +++ b/packages/mining-core/src/mine.ts @@ -0,0 +1,300 @@ +/** + * L0 mining core — the observation set (Phases 1 and 2). + * + * Reads git, returns an in-memory observation set, writes nothing. Three + * consumers are planned: the producer's L1 projection, the META-289 harness, + * and the report. This module is the single implementation all three take, so + * the org does not acquire the META-140 defect class in the code whose numbers + * an independent producer gets compared against. + * + * Scope boundary, enforced by what is absent: no weighting, no decay, no + * support threshold, no lift, no ranking, no cap. Those are the *scoring* and + * *selection* behaviors and they live in `score.ts` and `select.ts`. What is + * here is extraction (REQ-001), a computed empty tree (REQ-002), one + * normalizer (REQ-003), deterministic output (REQ-004), four completeness + * states (REQ-005), and a shallow-clone guard (REQ-006). + * + * Those six numbers are the only requirement identifiers in this package that + * are written down anywhere. Behaviors added after them are named, not + * numbered — an invented identifier reads as a citation and cites nothing. + */ +import { + type Completeness, + CompletenessReason, + CompletenessState, + completeness, +} from './completeness.js'; +import { + type CommitEvent, + GitInvocationError, + GitOutputError, + countFirstParent, + extractEvents, + isGitRepository, + isShallowRepository, + resolveCommit, +} from './git.js'; +import { PATH_NORMALIZATION_ASSUMPTIONS, type PathAssumption } from './paths.js'; + +/** + * v2.2.1's window: 500 first-parent transitions. + * + * v2.2.1 also names 2000 as a sensitivity arm. That is a harness concern; the + * producer computes one snapshot at one basis, so the default is the headline + * value and the harness overrides it. + */ +export const DEFAULT_WINDOW_TRANSITIONS = 500; + +/** + * How many co-occurrences make a relationship "qualifying" for REQ-005's + * state 2/3 boundary. + * + * This is deliberately a parameter with a recorded value rather than a + * constant, and the default is 1 rather than v2.2.1's `rawSupport >= 3`. + * + * Reason: Phase 2 runs *before* scoring, and `rawSupport >= 3` is a scoring + * threshold — v2.2.1 defines it as the validity condition on `lift`. Baking 3 + * in here would mean Phase 2 shipped a scoring decision under a completeness + * heading, and a repository with two genuine co-changes would report + * MINED_NO_QUALIFYING_RELATIONSHIP, which reads as "examined and found + * uncoupled" when the truth is "found coupled, below a threshold this phase has + * not adopted". + * + * So Phase 2 draws the line at observed-at-all and says which line it drew. + * Phase 3 sets this to 3 without touching the state machine. + */ +export const DEFAULT_QUALIFYING_MIN_COOCCURRENCE = 1; + +export interface MineOptions { + /** Revision whose first-parent history is walked. Defaults to `HEAD`. */ + basisRevision?: string; + /** v2.2.1 window in first-parent transitions. Defaults to 500. */ + windowTransitions?: number; + /** State 2/3 boundary. Defaults to 1. See the constant's note. */ + qualifyingMinCooccurrence?: number; +} + +/** An unordered file pair and how many extracted events touched both. */ +export interface PairObservation { + /** + * The two paths, sorted. Set semantics — position carries no meaning, which + * matches the schema's own description of `coChange[].files`. + */ + files: readonly [string, string]; + /** + * Count of extracted events in which both paths appear. Raw and unweighted: + * v2.2.1's `size_weight` and `position_decay` are Phase 3. + */ + cooccurrenceCount: number; +} + +/** What the window actually covered. Facts, not scores. */ +export interface BasisWindow { + /** The revision requested. */ + basisRevision: string; + /** Its resolved object id, so the window is reproducible after refs move. */ + basisCommit: string; + /** Transitions requested. */ + windowTransitions: number; + /** First-parent transitions available from the basis. */ + availableTransitions: number; + /** Transitions actually extracted — `min(requested, available)`. */ + extractedTransitions: number; + /** True when available exceeded the window, so the window bound the result. */ + windowTruncated: boolean; +} + +export interface ObservationSet { + /** Bumped when the serialized shape changes. Consumers pin on it. */ + readonly l0Version: 1; + completeness: Completeness; + /** Absent when completeness is NOT_MINED or EVIDENCE_UNAVAILABLE. */ + basisWindow?: BasisWindow; + /** Extracted events, oldest first. Empty unless mining completed. */ + events: readonly CommitEvent[]; + /** Pairs at or above the qualifying threshold, ranked-free and sorted. */ + pairs: readonly PairObservation[]; + /** The state 2/3 boundary this run used. Recorded, never implied. */ + qualifyingMinCooccurrence: number; + /** REQ-003: what this run assumed about unratified path identity. */ + pathNormalization: readonly PathAssumption[]; +} + +/** Build an observation set for a state that produced no evidence. */ +function withoutEvidence( + state: typeof CompletenessState.NOT_MINED | typeof CompletenessState.EVIDENCE_UNAVAILABLE, + reason: CompletenessReason, + detail: string, + qualifyingMinCooccurrence: number, +): ObservationSet { + return { + l0Version: 1, + completeness: completeness(state, reason, detail), + events: [], + pairs: [], + qualifyingMinCooccurrence, + pathNormalization: PATH_NORMALIZATION_ASSUMPTIONS, + }; +} + +/** + * Count co-occurrences over the extracted events. + * + * N files in one event yield N(N-1)/2 pairs. The cap belongs to the selection + * rule in `select.ts`; this counts everything the window produced, because a + * cap applied before the count is decided would silently determine the ranking + * it is supposed to follow. + */ +function countPairs( + events: readonly CommitEvent[], + qualifyingMinCooccurrence: number, +): readonly PairObservation[] { + const counts = new Map(); + for (const event of events) { + const files = event.files; + for (let i = 0; i < files.length; i += 1) { + for (let j = i + 1; j < files.length; j += 1) { + // `files` is already sorted and deduplicated by the parser, so + // files[i] < files[j] and the key is canonical without re-sorting. + // NUL cannot occur inside a path — git's own -z framing depends on + // that — so it is the one safe key separator. + const key = `${files[i]}\0${files[j]}`; + counts.set(key, (counts.get(key) ?? 0) + 1); + } + } + } + + const pairs: PairObservation[] = []; + for (const [key, cooccurrenceCount] of counts) { + if (cooccurrenceCount < qualifyingMinCooccurrence) continue; + const [left, right] = key.split('\0') as [string, string]; + pairs.push({ files: [left, right], cooccurrenceCount }); + } + + // Total order, and deliberately not a ranking: sorted by path so the output + // is stable for REQ-004. Ordering by count here would pre-empt the selection + // rule, which is `select.ts`'s job and applies its own ordering downstream. + pairs.sort((a, b) => { + if (a.files[0] !== b.files[0]) return a.files[0] < b.files[0] ? -1 : 1; + if (a.files[1] !== b.files[1]) return a.files[1] < b.files[1] ? -1 : 1; + return 0; + }); + return pairs; +} + +/** + * Mine an observation set from a repository's commit graph. + * + * Never throws for an absent, shallow or unreadable history — those are + * reported as completeness states, which is the entire point of REQ-005. It + * does throw on a programming error, because a bug should not disguise itself + * as a repository condition. + */ +export async function mine(repoRoot: string, options: MineOptions = {}): Promise { + const basisRevision = options.basisRevision ?? 'HEAD'; + const windowTransitions = options.windowTransitions ?? DEFAULT_WINDOW_TRANSITIONS; + const qualifyingMinCooccurrence = + options.qualifyingMinCooccurrence ?? DEFAULT_QUALIFYING_MIN_COOCCURRENCE; + + if (!(await isGitRepository(repoRoot))) { + return withoutEvidence( + CompletenessState.NOT_MINED, + CompletenessReason.NO_REPOSITORY, + `${repoRoot} is not a git repository, or git is unavailable`, + qualifyingMinCooccurrence, + ); + } + + // REQ-006. Before any extraction: a shallow clone can produce events, and + // those events are indistinguishable from a complete short history. Mining + // it and reporting the result would be the AP-1 failure this guard exists + // for — a confident, wrong, non-empty answer. + try { + if (await isShallowRepository(repoRoot)) { + return withoutEvidence( + CompletenessState.NOT_MINED, + CompletenessReason.SHALLOW_CLONE, + 'repository is a shallow clone; commit-graph history is truncated and co-change evidence cannot be established from it', + qualifyingMinCooccurrence, + ); + } + } catch (error) { + return withoutEvidence( + CompletenessState.EVIDENCE_UNAVAILABLE, + CompletenessReason.GIT_FAILED, + `could not determine whether the repository is shallow: ${error instanceof Error ? error.message : String(error)}`, + qualifyingMinCooccurrence, + ); + } + + // An empty repository resolves no HEAD. That is an absent history, not a + // broken one, and the two must not collapse into one state. + const basisCommit = await resolveCommit(repoRoot, basisRevision); + if (basisCommit === undefined) { + return withoutEvidence( + CompletenessState.NOT_MINED, + CompletenessReason.NO_COMMITS, + `${basisRevision} does not resolve to a commit in ${repoRoot}`, + qualifyingMinCooccurrence, + ); + } + + let events: readonly CommitEvent[]; + let availableTransitions: number; + try { + availableTransitions = await countFirstParent(repoRoot, basisCommit); + events = await extractEvents(repoRoot, { basisRevision: basisCommit, windowTransitions }); + } catch (error) { + if (error instanceof GitOutputError) { + return withoutEvidence( + CompletenessState.EVIDENCE_UNAVAILABLE, + CompletenessReason.MALFORMED_OUTPUT, + error.message, + qualifyingMinCooccurrence, + ); + } + if (error instanceof GitInvocationError) { + return withoutEvidence( + CompletenessState.EVIDENCE_UNAVAILABLE, + CompletenessReason.GIT_FAILED, + error.message, + qualifyingMinCooccurrence, + ); + } + throw error; + } + + if (events.length === 0) { + return withoutEvidence( + CompletenessState.NOT_MINED, + CompletenessReason.NO_COMMITS, + `no first-parent commits reachable from ${basisRevision}`, + qualifyingMinCooccurrence, + ); + } + + const pairs = countPairs(events, qualifyingMinCooccurrence); + + return { + l0Version: 1, + completeness: completeness( + pairs.length > 0 + ? CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED + : CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP, + CompletenessReason.MINED, + `mined ${events.length} first-parent transition(s) from ${basisCommit}; ${pairs.length} pair(s) at or above ${qualifyingMinCooccurrence} co-occurrence(s)`, + ), + basisWindow: { + basisRevision, + basisCommit, + windowTransitions, + availableTransitions, + extractedTransitions: events.length, + windowTruncated: availableTransitions > windowTransitions, + }, + events, + pairs, + qualifyingMinCooccurrence, + pathNormalization: PATH_NORMALIZATION_ASSUMPTIONS, + }; +} diff --git a/packages/mining-core/src/paths.test.ts b/packages/mining-core/src/paths.test.ts new file mode 100644 index 0000000..9e7c86b --- /dev/null +++ b/packages/mining-core/src/paths.test.ts @@ -0,0 +1,107 @@ +/** REQ-003 — one normalizer, and its assumptions are recorded rather than silent. */ +import { readFileSync, readdirSync, statSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { + PATH_NORMALIZATION_ASSUMPTIONS, + UNRATIFIED_ASSUMPTION_COUNT, + normalizePath, +} from './paths.js'; + +const srcDir = dirname(fileURLToPath(import.meta.url)); + +function sourceFiles(): string[] { + const found: string[] = []; + const walk = (dir: string): void => { + for (const entry of readdirSync(dir)) { + const full = join(dir, entry); + if (statSync(full).isDirectory()) walk(full); + else if (full.endsWith('.ts')) found.push(full); + } + }; + walk(srcDir); + return found; +} + +describe('normalizePath', () => { + it('rewrites backslash separators', () => { + expect(normalizePath('src\\routes\\checkout.ts')).toBe('src/routes/checkout.ts'); + }); + + it('strips a leading "./", including repeated prefixes', () => { + expect(normalizePath('./src/a.ts')).toBe('src/a.ts'); + expect(normalizePath('././src/a.ts')).toBe('src/a.ts'); + }); + + it('strips a trailing slash but never empties a path', () => { + expect(normalizePath('src/')).toBe('src'); + expect(normalizePath('/')).toBe('/'); + }); + + it('preserves case — the assumption record says so, so the test says so', () => { + expect(normalizePath('src/Foo.ts')).toBe('src/Foo.ts'); + expect(normalizePath('src/Foo.ts')).not.toBe(normalizePath('src/foo.ts')); + }); + + it('applies no Unicode normalization, so NFC and NFD stay distinct', () => { + const nfc = 'src/café.ts'; // é as one code point + const nfd = 'src/café.ts'; // e + combining acute + expect(normalizePath(nfc)).toBe(nfc); + expect(normalizePath(nfd)).toBe(nfd); + expect(normalizePath(nfc)).not.toBe(normalizePath(nfd)); + }); + + it('is idempotent', () => { + for (const input of ['./a\\b/', 'src/a.ts', './/x', 'src/Foo.ts']) { + expect(normalizePath(normalizePath(input))).toBe(normalizePath(input)); + } + }); +}); + +describe('assumption record (REQ-003 amendment)', () => { + it('covers all six META-278 questions', () => { + expect(PATH_NORMALIZATION_ASSUMPTIONS).toHaveLength(6); + }); + + it('records four of them as assumed rather than ratified', () => { + // The Phase 0 audit found the prior art answers 2 of 6. If a future change + // silently promotes one of the remaining 4 to `ratified` without META-278 + // actually ruling, this count moves and the test says so. + expect(UNRATIFIED_ASSUMPTION_COUNT).toBe(4); + }); + + it('gives every assumption a behavior and a rationale', () => { + for (const assumption of PATH_NORMALIZATION_ASSUMPTIONS) { + expect(assumption.question.length).toBeGreaterThan(0); + expect(assumption.behavior.length).toBeGreaterThan(0); + expect(assumption.rationale.length).toBeGreaterThan(0); + } + }); +}); + +describe('single-definition guarantee (REQ-003 verify)', () => { + it('defines normalizePath exactly once across the package', () => { + // Assembled rather than written as a literal, so this assertion does not + // match its own source and report a second definition that does not exist. + const definition = new RegExp(['export', 'function', 'normalizePath\\b'].join(' ')); + const definitions = sourceFiles().filter((file) => + definition.test(readFileSync(file, 'utf8')), + ); + expect(definitions.map((f) => f.slice(srcDir.length + 1))).toEqual(['paths.ts']); + }); + + it('has no inline separator replacement outside the normalizer', () => { + // The literal `.replace(/\\/g, '/')` and its equivalents are the shape a + // second, quieter normalizer takes. Anywhere but paths.ts is a violation. + const offenders: string[] = []; + for (const file of sourceFiles()) { + if (file.endsWith('paths.ts') || file.endsWith('paths.test.ts')) continue; + const source = readFileSync(file, 'utf8'); + if (/replace\(\s*\/\\\\\/g/.test(source) || /split\(\s*['"`]\\\\['"`]\s*\)/.test(source)) { + offenders.push(file.slice(srcDir.length + 1)); + } + } + expect(offenders).toEqual([]); + }); +}); diff --git a/packages/mining-core/src/paths.ts b/packages/mining-core/src/paths.ts new file mode 100644 index 0000000..9964072 --- /dev/null +++ b/packages/mining-core/src/paths.ts @@ -0,0 +1,121 @@ +/** + * Canonical path identity for L0 (REQ-003). + * + * META-278 specifies six questions about path identity and ADR-006 does not + * exist — the Phase 0 audit (A-2) found 0 normalizer implementations across + * `workspacejson/cli` and `workspacejson/standard`, and the only prior art is + * an unexported `toIndexKey` in `packages/cli/src/producer/evidence.ts` that + * answers 2 of the 6. So this function is the first implementation of a rule + * that has not been ratified. + * + * That makes silence the hazard. A normalizer that quietly picks an answer to + * "is `Foo.ts` the same path as `foo.ts`" produces keys that look settled and + * are not, which is AP-1 wearing a different hat. Every assumption this module + * makes on an unratified question is therefore declared in + * `PATH_NORMALIZATION_ASSUMPTIONS` and carried in L0's output, so a reader sees + * the guess as a guess and META-278 can overrule it against a written record + * rather than against archaeology. + */ + +/** One of META-278's six path-identity questions, and what L0 does about it. */ +export interface PathAssumption { + /** META-278's question, restated. */ + question: string; + /** `ratified` — settled elsewhere. `assumed` — L0 guessed; META-278 governs. */ + standing: 'ratified' | 'assumed'; + /** The behavior this module implements. */ + behavior: string; + /** Why, and what would change if META-278 rules the other way. */ + rationale: string; +} + +/** + * The full assumption record, emitted with every observation set. + * + * Ordered by META-278's own enumeration so the record can be diffed against the + * issue. Two are `ratified` only in the weak sense that the published schema's + * own field descriptions state them ("repository-root-relative POSIX path + * (forward slashes, no leading \"./\", no drive letters)") — that is schema + * prose, not an ADR, and it is recorded here as the source rather than claimed + * as independent authority. + */ +export const PATH_NORMALIZATION_ASSUMPTIONS: readonly PathAssumption[] = Object.freeze([ + Object.freeze({ + question: 'Path separator: backslash or forward slash?', + standing: 'ratified' as const, + behavior: 'Backslashes are rewritten to forward slashes.', + rationale: + "Stated by the published schema's own description of every path-bearing field: repository-root-relative POSIX, forward slashes. Not an open question.", + }), + Object.freeze({ + question: 'Leading "./" prefix: preserved or stripped?', + standing: 'ratified' as const, + behavior: 'A single leading "./" is stripped. Repeated "./././" collapses to nothing.', + rationale: + 'Same schema description: "no leading \\"./\\"". Repeated prefixes are not addressed there; stripping all of them is the only reading consistent with stripping one.', + }), + Object.freeze({ + question: 'Case sensitivity: are "Foo.ts" and "foo.ts" the same path?', + standing: 'assumed' as const, + behavior: + 'Case is preserved and comparison is case-sensitive. No case folding is applied.', + rationale: + "Git records the byte sequence it was given, so folding here would invent an identity git does not assert and would silently merge two files that a case-sensitive checkout keeps distinct. The cost is the mirror error: on a case-insensitive filesystem a rename that only changes case reads as two paths. META-278 may rule the other way; if it does, this is the site that changes.", + }), + Object.freeze({ + question: 'Unicode encoding: is NFC or NFD normalization applied?', + standing: 'assumed' as const, + behavior: + 'No Unicode normalization is applied. Bytes are decoded as UTF-8 and otherwise left alone.', + rationale: + 'Applying NFC would make L0 disagree with `git ls-files` on macOS-authored paths, and applying NFD would do the same on Linux-authored ones. Neither is safe to pick unilaterally, so L0 picks neither and says so. Consequence, stated plainly: the same file committed from two platforms can produce two distinct L0 paths.', + }), + Object.freeze({ + question: 'Trailing slash: is "src/" the same path as "src"?', + standing: 'assumed' as const, + behavior: + 'A trailing slash is stripped. L0 only ever sees blob paths from `diff-tree -r`, so this should never fire.', + rationale: + "Defensive only. `diff-tree -r` emits blobs, never directories, so a trailing slash reaching this function means an upstream assumption broke. Stripping is the conservative choice; the input is recorded as unexpected rather than normalized away, because the interesting event is that it happened at all.", + }), + Object.freeze({ + question: 'Symlinks and submodule/worktree roots: resolved before or after comparison?', + standing: 'assumed' as const, + behavior: + 'Neither is resolved. L0 uses the path as recorded in the commit, relative to the repository whose history is being mined.', + rationale: + "L0 reads the commit graph and never touches the working tree, so there is no filesystem to resolve a symlink against — resolution would require a checkout and would make output depend on which revision happens to be checked out, breaking REQ-004. Submodule contents are a different repository's history and are not mined; a gitlink appears as a single path. This is the question L0 is least equipped to answer and META-278 should not read L0's behavior as a proposal.", + }), +]); + +/** Count of META-278 questions L0 answers by assumption rather than by rule. */ +export const UNRATIFIED_ASSUMPTION_COUNT = PATH_NORMALIZATION_ASSUMPTIONS.filter( + (assumption) => assumption.standing === 'assumed', +).length; + +/** + * The single path-normalization function for L0 (REQ-003). + * + * Every path-touching site in this package calls this. There is no second + * definition and no inline separator handling anywhere else — that is asserted + * by a test, not by convention, because the whole point of a single normalizer + * is defeated by one call site that skips it. + * + * Behavior is exactly what `PATH_NORMALIZATION_ASSUMPTIONS` describes. Read + * that record before changing anything here. + */ +export function normalizePath(rawPath: string): string { + // Separator: ratified. + let path = rawPath.replace(/\\/g, '/'); + + // Leading "./": ratified. Loop rather than a single strip so "././a" lands on + // "a" — one strip would leave "./a", which is the shape the rule forbids. + while (path.startsWith('./')) path = path.slice(2); + + // Trailing slash: assumed, defensive. Never expected from `diff-tree -r`. + while (path.length > 1 && path.endsWith('/')) path = path.slice(0, -1); + + // Case, Unicode, symlinks, submodule roots: deliberately untouched. See the + // assumption record for why each is a decision and not an omission. + return path; +} diff --git a/packages/mining-core/src/project.test.ts b/packages/mining-core/src/project.test.ts new file mode 100644 index 0000000..07e25bf --- /dev/null +++ b/packages/mining-core/src/project.test.ts @@ -0,0 +1,182 @@ +/** + * L1 projection — repo-native, and pure by construction. + * + * Nothing here touches a schema, a validator or a filesystem, which is why it + * runs inside the workspace against its ordinary dependencies. The projection's + * *conformance* to the amended schema is a different claim, measured in the + * packed-candidate environment; what is measured here is the shape it produces + * and the rules it applies to produce it. + */ +import { describe, expect, it } from 'vitest'; +import { CompletenessReason, CompletenessState, completeness } from './completeness.js'; +import { ProjectionRefusal, project } from './project.js'; +import type { SelectionResult } from './select.js'; +import { RANKING_RULE } from './select.js'; + +const BASIS = '3c9a0f14b7e25d8613af04c2e9b7d5081f6a2c3d'; + +function selection( + pairs: Array<{ files: [string, string]; support: number; occurrences: number }>, + over: Partial = {}, +): SelectionResult { + return { + l0SelectionVersion: 1, + completeness: completeness( + pairs.length > 0 + ? CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED + : CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP, + CompletenessReason.MINED, + 'fixture', + ), + scoringBasis: { + weightingVersion: 'v2.2.1', + sizeWeightNumerator: 1, + positionDecayHalfLife: 1, + maxScoredFileCount: 50, + basisRevision: BASIS, + } as SelectionResult['scoringBasis'], + exclusions: {} as SelectionResult['exclusions'], + receipt: { + minSupport: 3, + pairsBeforeCap: pairs.length, + pairsEmitted: pairs.length, + cap: 50, + rankingRule: RANKING_RULE, + capBound: false, + }, + pairs, + ...over, + } as SelectionResult; +} + +describe('project — the artifact shape', () => { + it('emits exactly files, support and occurrences', () => { + const result = project(selection([{ files: ['a.ts', 'b.ts'], support: 4, occurrences: 10 }])); + expect(result.projected).toBe(true); + if (!result.projected) return; + expect(Object.keys(result.history.coChange[0]!).sort()).toEqual([ + 'files', + 'occurrences', + 'support', + ]); + }); + + it('stores no derived value — no rate, probability, lift or ranking', () => { + const result = project(selection([{ files: ['a.ts', 'b.ts'], support: 4, occurrences: 10 }])); + if (!result.projected) throw new Error('expected a projection'); + const entry = result.history.coChange[0] as unknown as Record; + for (const forbidden of ['rate', 'probability', 'lift', 'confidence', 'rank', 'weightedSupport']) { + expect(forbidden in entry).toBe(false); + } + }); + + it('omits the A-010 classification flag rather than asserting a constant', () => { + // The pre-A-010 producer emitted `generated: false` for every pair, which + // claimed that a lockfile and its manifest are a real source coupling. + // Absence is the honest output for a producer with no classifier. + const result = project( + selection([{ files: ['package.json', 'pnpm-lock.yaml'], support: 9, occurrences: 9 }]), + ); + if (!result.projected) throw new Error('expected a projection'); + expect('generated' in (result.history.coChange[0] as object)).toBe(false); + }); + + it('carries the basis pin at block level, never per entry', () => { + const result = project( + selection([ + { files: ['a.ts', 'b.ts'], support: 4, occurrences: 10 }, + { files: ['c.ts', 'd.ts'], support: 3, occurrences: 8 }, + ]), + ); + if (!result.projected) throw new Error('expected a projection'); + expect(result.history.basisRevision).toBe(BASIS); + for (const entry of result.history.coChange) { + expect('basisRevision' in (entry as object)).toBe(false); + } + }); + + it('preserves the selection ranking rather than re-sorting the array', () => { + // Re-sorting here would silently discard threshold-then-rank-then-cap. + const result = project( + selection([ + { files: ['z.ts', 'y.ts'], support: 9, occurrences: 12 }, + { files: ['a.ts', 'b.ts'], support: 3, occurrences: 8 }, + ]), + ); + if (!result.projected) throw new Error('expected a projection'); + expect(result.history.coChange[0]!.support).toBe(9); + expect(result.history.coChange[1]!.support).toBe(3); + }); +}); + +describe('project — canonical endpoint order', () => { + it('orders endpoints ascending, whichever way they arrive', () => { + const forward = project(selection([{ files: ['a.ts', 'b.ts'], support: 4, occurrences: 10 }])); + const reversed = project(selection([{ files: ['b.ts', 'a.ts'], support: 4, occurrences: 10 }])); + if (!forward.projected || !reversed.projected) throw new Error('expected projections'); + expect(forward.history.coChange[0]!.files).toEqual(['a.ts', 'b.ts']); + expect(reversed.history.coChange[0]!.files).toEqual(['a.ts', 'b.ts']); + }); + + it('ENDPOINT REVERSAL YIELDS IDENTICAL BYTES — the producer-profile obligation', () => { + const forward = project(selection([{ files: ['src/auth.ts', 'src/session.ts'], support: 8, occurrences: 24 }])); + const reversed = project(selection([{ files: ['src/session.ts', 'src/auth.ts'], support: 8, occurrences: 24 }])); + if (!forward.projected || !reversed.projected) throw new Error('expected projections'); + expect(JSON.stringify(forward.history)).toBe(JSON.stringify(reversed.history)); + }); + + it('uses UTF-8 BYTE order, not UTF-16 code unit order', () => { + // The two disagree here and only here-shaped cases: U+1F600 is a surrogate + // pair beginning 0xD83D, which sorts BEFORE U+E000 in UTF-16, while its + // UTF-8 encoding (F0 9F 98 80) sorts AFTER U+E000's (EE 80 80). A producer + // ordering with `<` would emit these two endpoints the other way round. + const emoji = 'src/\u{1F600}.ts'; + const privateUse = 'src/.ts'; + expect(emoji < privateUse).toBe(true); // UTF-16 says emoji first… + + const result = project(selection([{ files: [emoji, privateUse], support: 4, occurrences: 10 }])); + if (!result.projected) throw new Error('expected a projection'); + expect(result.history.coChange[0]!.files).toEqual([privateUse, emoji]); // …UTF-8 says otherwise + }); +}); + +describe('project — refusal rather than degradation', () => { + it('refuses a shallow clone, and does not emit an empty array', () => { + // A PINNED empty array is a positive finding under A-009: "the analysis ran + // and found nothing." Emitting one for a repository that could not be + // analyzed would state a result nobody measured. + const result = project( + selection([], { + completeness: completeness( + CompletenessState.NOT_MINED, + CompletenessReason.SHALLOW_CLONE, + 'shallow', + ), + }), + ); + expect(result.projected).toBe(false); + if (result.projected) return; + expect(result.refusal).toBe(ProjectionRefusal.NOT_MINED); + }); + + it('refuses when no basis pin exists', () => { + // The key is REMOVED rather than set to `undefined`: under + // `exactOptionalPropertyTypes` those are different states, and an unpinned + // selection is one whose basis is absent, not one carrying an explicit + // undefined. + const pinned = selection([{ files: ['a.ts', 'b.ts'], support: 4, occurrences: 10 }]); + const { scoringBasis: _dropped, ...unpinned } = pinned; + const result = project(unpinned as typeof pinned); + expect(result.projected).toBe(false); + if (result.projected) return; + expect(result.refusal).toBe(ProjectionRefusal.NO_BASIS_PIN); + }); + + it('DOES emit a pinned empty array when the analysis genuinely found nothing', () => { + const result = project(selection([])); + expect(result.projected).toBe(true); + if (!result.projected) return; + expect(result.history.coChange).toEqual([]); + expect(result.history.basisRevision).toBe(BASIS); + }); +}); diff --git a/packages/mining-core/src/project.ts b/packages/mining-core/src/project.ts new file mode 100644 index 0000000..55bb2fc --- /dev/null +++ b/packages/mining-core/src/project.ts @@ -0,0 +1,138 @@ +/** + * L1 projection — the only place a selection becomes artifact-shaped. + * + * Everything before this module answers "what does the commit graph say". + * This one answers "what does a producer write down", and those are different + * questions with different obligations. Three properties are load-bearing. + * + * 1. **Canonical endpoint order is established HERE, not upstream.** The + * scoring stage sorts pair endpoints with `<`, which is UTF-16 code unit + * order. The ruling names UTF-8 byte order, and the two disagree — a path + * containing U+1F600 sorts before one containing U+E000 under UTF-16 and + * after it under UTF-8. Upstream order is fine for keying a map, because + * only stability matters there. It is not fine for bytes a second producer + * is compared against, so the endpoints are re-ordered here under the same + * `compareUtf8` the ranking uses. Endpoint reversal must not change the + * output, and that is asserted rather than asserted-in-prose. + * + * 2. **No derived value is stored.** No rate, probability, lift, confidence or + * ranking. `support` and `occurrences` are counts; a reader who wants a + * ratio derives it. This is A-009, and the reason is that a continuous + * derived value moves on every commit and makes `generate --check` fire + * forever. + * + * 3. **The classification flag is omitted.** A-010 made + * `coChange[].generated` optional and defined absence as *unclassified*. + * This producer implements no deterministic tooling-coupling classifier, so + * it says nothing rather than emitting a constant `false` — which is + * exactly what the pre-A-010 producer did, and what made it assert that a + * lockfile and its manifest are a real source coupling. Absence here is a + * positive design decision, not an unfinished one. + * + * The projection refuses rather than degrades. A selection whose completeness + * is not a mined state yields nothing at all — not an empty array, which under + * A-009 is a *positive finding* that the analysis ran and found no qualifying + * pairs. Reporting "analyzed, nothing found" for a repository that was never + * successfully analyzed is the failure mode this whole package exists to + * avoid, one level up from `NOT_MINED / SHALLOW_CLONE`. + */ +import { CompletenessState } from './completeness.js'; +import { type SelectedPair, type SelectionResult, compareUtf8 } from './select.js'; + +/** + * A co-change entry exactly as it appears in `generated.coChange`. + * + * Three fields. `generated` is absent by design — see the module note — and + * `rate` is absent because the observation form forbids it. + */ +export interface ProjectedCoChangeEntry { + /** The pair, endpoints in ascending UTF-8 byte order. */ + files: [string, string]; + /** Distinct qualifying commits in which both files changed. */ + support: number; + /** Distinct qualifying commits in which at least one changed. The union. */ + occurrences: number; +} + +/** + * What a producer splices into `generated`. + * + * `basisRevision` is a `generated`-level sibling, never per item: repeating it + * per entry would admit a document whose entries were counted at different + * revisions. + */ +export interface ProjectedHistory { + basisRevision: string; + coChange: ProjectedCoChangeEntry[]; +} + +/** Why a projection produced nothing. Absence is reported, never smoothed. */ +export enum ProjectionRefusal { + /** Completeness is not a mined state — shallow clone, no history, or an error. */ + NOT_MINED = 'NOT_MINED', + /** Mined, but no basis pin. An unpinned block cannot be recounted against. */ + NO_BASIS_PIN = 'NO_BASIS_PIN', +} + +export type ProjectionResult = + | { projected: true; history: ProjectedHistory } + | { projected: false; refusal: ProjectionRefusal; detail: string }; + +/** + * Put a pair's endpoints in canonical order. + * + * `files` has set semantics, so this changes no meaning — it fixes the one + * spelling a producer is permitted to write, which is what makes two + * independent producers byte-comparable. + */ +function canonicalPair(files: readonly [string, string]): [string, string] { + return compareUtf8(files[0], files[1]) <= 0 ? [files[0], files[1]] : [files[1], files[0]]; +} + +function projectPair(pair: SelectedPair): ProjectedCoChangeEntry { + return { + files: canonicalPair(pair.files), + support: pair.support, + occurrences: pair.occurrences, + }; +} + +/** + * Project a selection into the artifact shape, or refuse and say why. + * + * Pure. The selection, the scored set behind it and the extracted events + * behind that are all untouched and remain auditable. + */ +export function project(selection: SelectionResult): ProjectionResult { + const state = selection.completeness.state; + const mined = + state === CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED || + state === CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP; + + if (!mined) { + return { + projected: false, + refusal: ProjectionRefusal.NOT_MINED, + detail: `completeness is ${state}; an artifact block would claim an analysis that did not happen`, + }; + } + + const basisRevision = selection.scoringBasis?.basisRevision; + if (basisRevision === undefined || basisRevision === '') { + return { + projected: false, + refusal: ProjectionRefusal.NO_BASIS_PIN, + detail: + 'the selection carries no basisRevision; an unpinned coChange block reads as legacy/unknown and asserts nothing', + }; + } + + // Entry order is the selection's ranked order, already deterministic and + // already applied threshold-then-rank-then-cap. This step re-orders endpoints + // WITHIN each pair and nothing else: re-sorting the array here would silently + // discard the ranking the selection rule exists to produce. + return { + projected: true, + history: { basisRevision, coChange: selection.pairs.map(projectPair) }, + }; +} diff --git a/packages/mining-core/src/score.test.ts b/packages/mining-core/src/score.test.ts new file mode 100644 index 0000000..6f14ac7 --- /dev/null +++ b/packages/mining-core/src/score.test.ts @@ -0,0 +1,434 @@ +/** + * Phase 3: the weighting, the scoring exclusion, bounded scoring cost, and + * basis pinning. Named behaviors — only REQ-001..006 are written down. + * + * Every expectation here is a number written down before the implementation + * existed, computed by hand from META-289 v2.2.1's frozen weighting and from + * the ratified observation-form definitions in the standard's A-009 amendment. + * None of them was read off whatever the code happened to return. + */ +import { describe, expect, it } from 'vitest'; +import { CompletenessReason, CompletenessState, completeness } from './completeness.js'; +import type { CommitEvent } from './git.js'; +import { type ObservationSet, mine } from './mine.js'; +import { PATH_NORMALIZATION_ASSUMPTIONS } from './paths.js'; +import { + DEFAULT_MIN_SUPPORT, + POSITION_DECAY_HALF_LIFE, + SCORING_MAX_FILE_COUNT, + SIZE_WEIGHT_NUMERATOR, + WEIGHTING_VERSION, + positionDecay, + score, + sizeWeight, +} from './score.js'; +import { serializeScoredSet } from './serialize.js'; +import { + makeCoupledRepo, + makeShallowCloneOfCoupled, + removeDir, +} from './testing/fixtures.js'; + +/** A 40-hex object id shape, so pinning can be asserted without a repository. */ +const OID_A = 'a'.repeat(40); +const OID_B = 'b'.repeat(40); + +function event(position: number, files: readonly string[], commit: string): CommitEvent { + const sorted = [...new Set(files)].sort(); + return { + commit, + parent: OID_B, + files: sorted, + fileCount: sorted.length, + position, + }; +} + +function manyFiles(count: number, extra: readonly string[]): string[] { + const generated = Array.from({ length: count - extra.length }, (_, i) => + `bulk/f${String(i).padStart(3, '0')}.ts`, + ); + return [...extra, ...generated]; +} + +/** + * The hand-computed fixture. + * + * position 0: [a, b] fileCount 2 scored + * position 1: [a, c] fileCount 2 scored + * position 2: [a, b, ...58 more] fileCount 60 EXCLUDED (> 50) + * position 3: [a, b] fileCount 2 scored, and the basis + * + * Newest extracted position is 3, so Δpos is 3, 2, 1, 0 respectively — and + * critically, position 2 dropping out of scoring must not renumber the others. + */ +function observations( + overrides: Partial = {}, + // `exactOptionalPropertyTypes` makes an explicit `undefined` a different + // thing from an absent key, and "no window to pin" is the absent key. So the + // helper deletes it rather than letting a caller pass `undefined`. + dropBasisWindow = false, +): ObservationSet { + const events: CommitEvent[] = [ + event(0, ['src/a.ts', 'src/b.ts'], '0'.repeat(40)), + event(1, ['src/a.ts', 'src/c.ts'], '1'.repeat(40)), + event(2, manyFiles(60, ['src/a.ts', 'src/b.ts']), '2'.repeat(40)), + event(3, ['src/a.ts', 'src/b.ts'], '3'.repeat(40)), + ]; + const built: ObservationSet = { + l0Version: 1, + completeness: completeness( + CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED, + CompletenessReason.MINED, + 'fixture', + ), + basisWindow: { + basisRevision: 'HEAD', + basisCommit: OID_A, + windowTransitions: 500, + availableTransitions: 4, + extractedTransitions: 4, + windowTruncated: false, + }, + events, + pairs: [], + qualifyingMinCooccurrence: 1, + pathNormalization: PATH_NORMALIZATION_ASSUMPTIONS, + ...overrides, + }; + if (!dropBasisWindow) return built; + const { basisWindow: _absent, ...withoutWindow } = built; + return withoutWindow; +} + +describe('weighting: v2.2.1, implemented verbatim', () => { + it('size_weight is min(1, 10/fileCount)', () => { + expect(SIZE_WEIGHT_NUMERATOR).toBe(10); + // Below the numerator the weight saturates at 1 — it never exceeds it. + expect(sizeWeight(1)).toBe(1); + expect(sizeWeight(2)).toBe(1); + expect(sizeWeight(10)).toBe(1); + // Above it the weight is the ratio, exactly. + expect(sizeWeight(20)).toBe(0.5); + expect(sizeWeight(50)).toBe(0.2); + expect(sizeWeight(100)).toBe(0.1); + }); + + it('position_decay is 2^(-dPos/250)', () => { + expect(POSITION_DECAY_HALF_LIFE).toBe(250); + // At the basis there is no decay at all. + expect(positionDecay(0)).toBe(1); + // One half-life back is exactly one half. + expect(positionDecay(250)).toBeCloseTo(0.5, 12); + // The far edge of a full 500-transition window is a quarter. + expect(positionDecay(500)).toBeCloseTo(0.25, 12); + // Monotone decreasing, never negative, never zero inside the window. + expect(positionDecay(1)).toBeLessThan(1); + expect(positionDecay(499)).toBeGreaterThan(positionDecay(500)); + expect(positionDecay(500)).toBeGreaterThan(0); + }); + + it('excludes events with fileCount > 50 from scoring, and includes exactly 50', () => { + expect(SCORING_MAX_FILE_COUNT).toBe(50); + + const boundary = observations({ + events: [ + event(0, manyFiles(50, ['src/d.ts', 'src/e.ts']), '0'.repeat(40)), + event(1, manyFiles(51, ['src/f.ts', 'src/g.ts']), '1'.repeat(40)), + ], + }); + const result = score(boundary, { minSupport: 1 }); + + expect(result.exclusions.scoredEventCount).toBe(1); + expect(result.exclusions.excludedEventCount).toBe(1); + expect(result.exclusions.excludedCommits).toEqual(['1'.repeat(40)]); + + const paths = result.pairs.flatMap((p) => [...p.files]); + // The 50-file event was scored, so its pairs exist. + expect(paths).toContain('src/d.ts'); + // The 51-file event was not, so its pairs do not. + expect(paths).not.toContain('src/f.ts'); + }); + + it('preserves every extracted event on the input while excluding at scoring time', () => { + const input = observations(); + const result = score(input, { minSupport: 1 }); + + // Extraction-time preservation: the excluded event is still in the input. + expect(input.events).toHaveLength(4); + expect(input.events.some((e) => e.fileCount > SCORING_MAX_FILE_COUNT)).toBe(true); + // Scoring-time exclusion: it is accounted for, by commit, not silently dropped. + expect(result.exclusions.excludedEventCount).toBe(1); + expect(result.exclusions.excludedCommits).toEqual(['2'.repeat(40)]); + expect(result.exclusions.scoredEventCount).toBe(3); + }); + + it('measures dPos from the newest EXTRACTED event and never renumbers after exclusion', () => { + const result = score(observations(), { minSupport: 1 }); + const ab = result.pairs.find((p) => p.files[1] === 'src/b.ts'); + expect(ab).toBeDefined(); + + // (a,b) is scored at positions 0 and 3 only — position 2 is excluded. + // Newest EXTRACTED position is 3, so dPos is 3 and 0. + const expected = sizeWeight(2) * positionDecay(3) + sizeWeight(2) * positionDecay(0); + expect(ab!.weightedSupport).toBeCloseTo(expected, 12); + + // If exclusion had renumbered the surviving events 0,1,2 the newest would be + // 2 and this is the value that would have been produced instead. It must not + // be. The two differ in the fourth decimal, which is enough to discriminate. + const renumbered = sizeWeight(2) * positionDecay(2) + sizeWeight(2) * positionDecay(0); + expect(ab!.weightedSupport).not.toBeCloseTo(renumbered, 4); + }); + + it('counts support as both-changed and occurrences as the symmetric union', () => { + const result = score(observations(), { minSupport: 1 }); + const byPair = new Map(result.pairs.map((p) => [p.files.join('|'), p])); + + // (a,b): both changed at scored positions 0 and 3. + const ab = byPair.get('src/a.ts|src/b.ts'); + expect(ab?.support).toBe(2); + // Union: a or b changed at scored positions 0, 1 and 3. + expect(ab?.occurrences).toBe(3); + + // (a,c): both changed at scored position 1 only. + const ac = byPair.get('src/a.ts|src/c.ts'); + expect(ac?.support).toBe(1); + // Union is still 0, 1 and 3, because a changed in all three. + expect(ac?.occurrences).toBe(3); + + // (b,c) never co-occurred, so it is absent — not present with support 0. + expect(byPair.has('src/b.ts|src/c.ts')).toBe(false); + }); + + it('holds the standard A-009 invariants on every emitted pair', () => { + const result = score(observations(), { minSupport: 1 }); + expect(result.pairs.length).toBeGreaterThan(0); + for (const pair of result.pairs) { + // Enforced by validate(), not by the schema — so it is enforced here. + expect(pair.support).toBeLessThanOrEqual(pair.occurrences); + // Observation-form minimum: a pair whose union is empty was never observed. + expect(pair.occurrences).toBeGreaterThanOrEqual(1); + expect(Number.isInteger(pair.support)).toBe(true); + expect(Number.isInteger(pair.occurrences)).toBe(true); + expect(pair.files).toHaveLength(2); + expect(pair.files[0] < pair.files[1]).toBe(true); + } + }); + + it('adopts v2.2.1 support >= 3 as the default and records the value used', () => { + expect(DEFAULT_MIN_SUPPORT).toBe(3); + const result = score(observations()); + expect(result.minSupport).toBe(3); + // The fixture's best pair has support 2, so nothing qualifies at 3. + expect(result.pairs).toHaveLength(0); + // And that is state 2 — mining ran, nothing qualified. Not a new state, + // and not state 3 with an empty list. + expect(result.completeness.state).toBe(CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP); + expect(result.completeness.reason).toBe(CompletenessReason.MINED); + }); + + it('reports state 3 when a pair does clear the threshold', () => { + const result = score(observations(), { minSupport: 2 }); + expect(result.pairs).toHaveLength(1); + expect(result.completeness.state).toBe(CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED); + }); +}); + +describe('honest degradation: never a reduced-magnitude answer', () => { + it('passes NOT_MINED straight through and emits no pairs and no basis', () => { + const shallow: ObservationSet = { + l0Version: 1, + completeness: completeness( + CompletenessState.NOT_MINED, + CompletenessReason.SHALLOW_CLONE, + 'shallow', + ), + events: [], + pairs: [], + qualifyingMinCooccurrence: 1, + pathNormalization: PATH_NORMALIZATION_ASSUMPTIONS, + }; + const result = score(shallow, { minSupport: 1 }); + expect(result.completeness.state).toBe(CompletenessState.NOT_MINED); + expect(result.completeness.reason).toBe(CompletenessReason.SHALLOW_CLONE); + expect(result.pairs).toEqual([]); + expect(result.scoringBasis).toBeUndefined(); + }); + + it('refuses to score events handed to it under a non-mined completeness', () => { + // The AP-1 shape: events are present, but completeness says they are not a + // claim about the repository. Scoring them would manufacture a confident, + // structurally identical, wrong answer at reduced magnitude — exactly the + // --depth 1 defect. The scorer must key on completeness, not on whether an + // events array happens to be non-empty. + const truncated = observations({ + completeness: completeness( + CompletenessState.NOT_MINED, + CompletenessReason.SHALLOW_CLONE, + 'shallow clone with visible events', + ), + }); + const result = score(truncated, { minSupport: 1 }); + expect(result.pairs).toEqual([]); + expect(result.exclusions.scoredEventCount).toBe(0); + }); + + it('a --depth 1 clone scores to NOT_MINED where the full clone scores pairs', async () => { + const { source, shallow } = makeShallowCloneOfCoupled(); + try { + const full = score(await mine(source), { minSupport: 1 }); + const clipped = score(await mine(shallow), { minSupport: 1 }); + + // The full history has the coupling. + expect(full.completeness.state).toBe(CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED); + expect(full.pairs.length).toBeGreaterThan(0); + + // The clone does not report the same pairs at lower support. It reports + // that it did not look. + expect(clipped.completeness.state).toBe(CompletenessState.NOT_MINED); + expect(clipped.completeness.reason).toBe(CompletenessReason.SHALLOW_CLONE); + expect(clipped.pairs).toEqual([]); + } finally { + removeDir(source); + removeDir(shallow); + } + }); +}); + +describe('basis pinning', () => { + it('pins a full-length lowercase object name, never a symbolic ref', () => { + const result = score(observations(), { minSupport: 1 }); + expect(result.scoringBasis).toBeDefined(); + // The standard's A-009 pattern for generated.basisRevision. + expect(result.scoringBasis!.basisRevision).toMatch(/^([0-9a-f]{40}|[0-9a-f]{64})$/); + // Specifically the resolved commit, not the requested revision string. + expect(result.scoringBasis!.basisRevision).toBe(OID_A); + expect(result.scoringBasis!.basisRevision).not.toBe('HEAD'); + }); + + it('rejects a basis that is not a full-length object name rather than emitting it', () => { + const abbreviated = observations({ + basisWindow: { + basisRevision: 'HEAD', + basisCommit: 'abc1234', + windowTransitions: 500, + availableTransitions: 4, + extractedTransitions: 4, + windowTruncated: false, + }, + }); + expect(() => score(abbreviated, { minSupport: 1 })).toThrow(/basis/i); + }); + + it('pins both edges of the scored window and the decay origin', () => { + const result = score(observations(), { minSupport: 1 }); + const basis = result.scoringBasis!; + // Oldest extracted commit, position 0. + expect(basis.windowOldestCommit).toBe('0'.repeat(40)); + // Newest extracted commit, which is the basis end of the window. + expect(basis.windowNewestCommit).toBe('3'.repeat(40)); + // Δpos is measured from here, and it is an extracted position. + expect(basis.decayOriginPosition).toBe(3); + }); + + it('records the frozen weighting so a score is attributable to a named ruleset', () => { + const basis = score(observations(), { minSupport: 1 }).scoringBasis!; + expect(basis.weightingVersion).toBe(WEIGHTING_VERSION); + expect(basis.sizeWeightNumerator).toBe(10); + expect(basis.positionDecayHalfLife).toBe(250); + expect(basis.maxScoredFileCount).toBe(50); + // The recorded numbers are the numbers the functions actually use. + expect(sizeWeight(basis.sizeWeightNumerator * 2)).toBe(0.5); + expect(positionDecay(basis.positionDecayHalfLife)).toBeCloseTo(0.5, 12); + }); + + it('emits no scoringBasis when there is no window to pin', () => { + const unmined = observations( + { + completeness: completeness( + CompletenessState.NOT_MINED, + CompletenessReason.NO_COMMITS, + 'no commits', + ), + }, + true, + ); + // Absence, not a placeholder that reads as a real pin. + expect(score(unmined, { minSupport: 1 }).scoringBasis).toBeUndefined(); + }); + + it('two runs at the same pin agree byte for byte', async () => { + const repo = makeCoupledRepo(); + try { + const first = serializeScoredSet(score(await mine(repo), { minSupport: 1 })); + const second = serializeScoredSet(score(await mine(repo), { minSupport: 1 })); + expect(first).toBe(second); + expect(first).toContain('"weightedSupport"'); + } finally { + removeDir(repo); + } + }); + + it('sorts pairs by path, which is a total order and not a ranking', () => { + const result = score(observations(), { minSupport: 1 }); + const keys = result.pairs.map((p) => p.files.join('|')); + expect(keys).toEqual([...keys].sort()); + // Deliberately NOT ordered by support — the ranking rule is undecided and + // an accidental order would be read as one. + expect(result.pairs.map((p) => p.support)).toEqual([2, 1]); + }); +}); + +describe('scoring cost is bounded and is not the runtime problem', () => { + it('scores a full 500-transition window well under a second', () => { + // Extraction spends two git processes per commit; scoring spends none. This + // separates the two costs so the measured wall-clock number can be + // attributed correctly rather than blamed on whichever half is newer. + const events: CommitEvent[] = Array.from({ length: 500 }, (_, position) => + event( + position, + [`src/m${position % 40}.ts`, `src/n${position % 37}.ts`, `test/t${position % 23}.ts`], + String(position).padStart(40, '0'), + ), + ); + const input = observations({ + events, + basisWindow: { + basisRevision: 'HEAD', + basisCommit: OID_A, + windowTransitions: 500, + availableTransitions: 900, + extractedTransitions: 500, + windowTruncated: true, + }, + }); + + const started = performance.now(); + const result = score(input, { minSupport: 1 }); + const elapsed = performance.now() - started; + + expect(result.exclusions.scoredEventCount).toBe(500); + expect(result.pairs.length).toBeGreaterThan(0); + expect(elapsed).toBeLessThan(1000); + }); + + it('records that the window bound, so truncation is a fact and not an error', () => { + const input = observations({ + basisWindow: { + basisRevision: 'HEAD', + basisCommit: OID_A, + windowTransitions: 500, + availableTransitions: 634, + extractedTransitions: 500, + windowTruncated: true, + }, + }); + const result = score(input, { minSupport: 1 }); + // A repository longer than the window is VALID. The bounded window is + // recorded; it is not a completeness failure. + expect(result.completeness.state).toBe(CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED); + expect(result.basisWindow?.windowTruncated).toBe(true); + expect(result.basisWindow?.availableTransitions).toBe(634); + expect(result.basisWindow?.extractedTransitions).toBe(500); + }); +}); diff --git a/packages/mining-core/src/score.ts b/packages/mining-core/src/score.ts new file mode 100644 index 0000000..f27af37 --- /dev/null +++ b/packages/mining-core/src/score.ts @@ -0,0 +1,317 @@ +/** + * L0 scoring and basis pinning — META-297 Phase 3. + * + * Named behaviors, not numbered ones. REQ-001..006 are written down in the + * issue and this package cites them; the scoring, exclusion, basis-pinning and + * selection behaviors are not, so they are named. An invented identifier reads + * as a citation and cites nothing. + * + * A pure function over an observation set. It spawns no process, reads no + * filesystem, and consults no clock: everything it needs was already extracted + * by Phase 1. That is what keeps the exclusion auditable — `score` can be + * handed the same observation set twice with different parameters and the + * difference is attributable to the parameters rather than to a second walk of + * a repository that may have moved. + * + * Two vocabularies meet here and they are deliberately kept apart. + * + * META-289 v2.2.1 supplies the *weighting*, frozen and implemented verbatim: + * `size_weight = min(1, 10/fileCount)`, `position_decay = 2^(-Δpos/250)`, a + * 500-transition window, and the exclusion of events with `fileCount > 50`. + * + * The standard's ratified A-009 amendment supplies the *counts*: `support` is + * the number of distinct qualifying commits in which BOTH files changed, and + * `occurrences` is the number in which AT LEAST ONE changed — the symmetric + * union, not a per-file marginal. Both are integers, both are counted over the + * same boundary, and `support <= occurrences` holds by construction. + * + * The weighted number and the counts are both emitted, and neither is derived + * from the other. A rate is a reader's question; nothing derived is stored. + * + * What is NOT here, and why: the pair cap and the ranking rule. Those are + * the selection rule's, they are applied downstream in `select.ts`, and an + * accidental order here would be read as a ranking — + * so pairs come out in path order, which is a total order and visibly not a + * ranking. + */ +import { + type Completeness, + CompletenessReason, + CompletenessState, + completeness, +} from './completeness.js'; +import type { BasisWindow, ObservationSet } from './mine.js'; + +/** The frozen weighting this module implements. Recorded in every output. */ +export const WEIGHTING_VERSION = 'META-289 v2.2.1'; + +/** `size_weight = min(1, 10/fileCount)`. */ +export const SIZE_WEIGHT_NUMERATOR = 10; + +/** `position_decay = 2^(-Δpos/250)`. One half-life is 250 transitions. */ +export const POSITION_DECAY_HALF_LIFE = 250; + +/** + * Events touching more than this many files are excluded from scoring. + * + * Strictly greater: an event with exactly 50 files is scored. The recorded + * file-role and path exclusion set is EMPTY — no path is excluded for being + * documentation, a lockfile or generated — so this size rule is the only + * exclusion L0 applies, and it applies to whole events rather than to paths. + */ +export const SCORING_MAX_FILE_COUNT = 50; + +/** + * v2.2.1's validity condition, adopted here. + * + * Phase 2 deliberately defaulted its own state 2/3 boundary to 1 and recorded + * that Phase 3 would raise it, because `rawSupport >= 3` is a scoring threshold + * and Phase 2 ran before scoring. This is Phase 3, so the threshold lands here + * and nowhere else. It is still a parameter, and the value used is recorded in + * the output rather than implied by it. + */ +export const DEFAULT_MIN_SUPPORT = 3; + +/** A full-length lowercase Git object name. The standard's A-009 pattern. */ +const OBJECT_NAME = /^([0-9a-f]{40}|[0-9a-f]{64})$/; + +/** v2.2.1's `size_weight`. Saturates at 1; never exceeds it. */ +export function sizeWeight(fileCount: number): number { + return Math.min(1, SIZE_WEIGHT_NUMERATOR / fileCount); +} + +/** + * v2.2.1's `position_decay`, over distance back from the basis. + * + * `deltaPosition` is 0 at the newest extracted event and grows toward the + * oldest. At the far edge of a full 500-transition window it is 500, so the + * oldest event carries a quarter of the weight of the newest. + */ +export function positionDecay(deltaPosition: number): number { + return 2 ** (-deltaPosition / POSITION_DECAY_HALF_LIFE); +} + +export interface ScoredPair { + /** The two paths, sorted. Set semantics; position carries no meaning. */ + files: readonly [string, string]; + /** Distinct scored events in which BOTH files changed. A-009's `support`. */ + support: number; + /** + * Distinct scored events in which AT LEAST ONE changed. A-009's + * `occurrences` in the observation form — the symmetric union denominator, + * so reversing the pair changes nothing. Never a per-file marginal. + */ + occurrences: number; + /** + * v2.2.1's weighted support: the sum of `size_weight × position_decay` over + * the scored events in which both files changed. + * + * Emitted alongside the counts, never instead of them, and never used to + * order anything — see the ranking note at the top of this file. + */ + weightedSupport: number; +} + +/** Everything needed to recount this result against the same history. */ +export interface ScoringBasis { + /** The frozen ruleset these numbers are attributable to. */ + weightingVersion: string; + sizeWeightNumerator: number; + positionDecayHalfLife: number; + maxScoredFileCount: number; + /** + * The pinned basis: a full-length lowercase object name, never a symbolic + * ref. A pin that does not name exactly one commit permanently cannot be + * recounted against, which is the whole point of pinning it. + */ + basisRevision: string; + /** Oldest extracted commit — the far edge of the window. */ + windowOldestCommit: string; + /** Newest extracted commit — the near edge, and the decay origin. */ + windowNewestCommit: string; + /** The extracted position Δpos is measured from. */ + decayOriginPosition: number; +} + +/** What the size rule removed, counted rather than assumed. */ +export interface ScoringExclusions { + /** The rule: events with more files than this are excluded. */ + maxFileCount: number; + /** Events that were scored. */ + scoredEventCount: number; + /** Events the size rule removed. */ + excludedEventCount: number; + /** + * The excluded commits, in extraction order. Named, not just counted — an + * exclusion nobody can point at is not auditable, and this is the only + * exclusion L0 applies. + */ + excludedCommits: readonly string[]; +} + +export interface ScoredSet { + /** Bumped when this shape changes. Consumers pin on it. */ + readonly l0ScoreVersion: 1; + /** Recomputed against `minSupport`. Same four states; no new ones. */ + completeness: Completeness; + /** Carried through from the observation set, unchanged. */ + basisWindow?: BasisWindow; + /** Absent when there is no window to pin. Never a placeholder. */ + scoringBasis?: ScoringBasis; + exclusions: ScoringExclusions; + /** Pairs at or above `minSupport`, in path order. Not a ranking. */ + pairs: readonly ScoredPair[]; + /** The threshold this run used. Recorded, never implied. */ + minSupport: number; +} + +export interface ScoreOptions { + /** v2.2.1's `rawSupport >= 3`. Defaults to 3. */ + minSupport?: number; +} + +/** No evidence to score. The input's own completeness is carried through verbatim. */ +function unscored(observations: ObservationSet, minSupport: number): ScoredSet { + return { + l0ScoreVersion: 1, + completeness: observations.completeness, + // Spread rather than assign: under `exactOptionalPropertyTypes` an explicit + // `undefined` is a different thing from an absent key, and the absent key + // is what "there is no window to pin" means. + ...(observations.basisWindow === undefined ? {} : { basisWindow: observations.basisWindow }), + exclusions: { + maxFileCount: SCORING_MAX_FILE_COUNT, + scoredEventCount: 0, + excludedEventCount: 0, + excludedCommits: [], + }, + pairs: [], + minSupport, + }; +} + +/** + * Score an observation set. + * + * Honest degradation is the first thing this function does, not the last. + * States 1 and 4 mean the history was not established, and an events array is + * not permission to override that — a `--depth 1` clone can hand over real + * events that produce a structurally identical answer at reduced magnitude, + * which is the single most expensive failure this package can ship, because + * every diagnostic run against an external repository is a fresh clone whose + * depth is a configuration detail nobody reads. So the gate is completeness, + * never `events.length > 0`. + * + * Throws only on a caller bug — an unpinnable basis. A repository condition is + * a completeness state; a bug must not disguise itself as one. + */ +export function score(observations: ObservationSet, options: ScoreOptions = {}): ScoredSet { + const minSupport = options.minSupport ?? DEFAULT_MIN_SUPPORT; + + const mined = + observations.completeness.state === CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED || + observations.completeness.state === CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP; + if (!mined) return unscored(observations, minSupport); + + const { basisWindow, events } = observations; + if (basisWindow === undefined || events.length === 0) { + return unscored(observations, minSupport); + } + + if (!OBJECT_NAME.test(basisWindow.basisCommit)) { + throw new Error( + `score: basis ${JSON.stringify(basisWindow.basisCommit)} is not a full-length lowercase Git object name, so the result could not be recounted against it`, + ); + } + + // Δpos is measured from the newest EXTRACTED event. Not the newest scored + // one: excluding a large event must not shift the decay of every event older + // than it, which is what renumbering after the filter would silently do. + const decayOriginPosition = Math.max(...events.map((event) => event.position)); + + const scored = events.filter((event) => event.fileCount <= SCORING_MAX_FILE_COUNT); + const excludedCommits = events + .filter((event) => event.fileCount > SCORING_MAX_FILE_COUNT) + .map((event) => event.commit); + + // support: scored events in which both files changed. + const support = new Map(); + // weightedSupport: v2.2.1's sum over those same events. + const weighted = new Map(); + // Per-file scored-event counts, used only to derive the union denominator. + const perFile = new Map(); + + for (const event of scored) { + const weight = sizeWeight(event.fileCount) * positionDecay(decayOriginPosition - event.position); + const files = event.files; + for (const path of files) { + perFile.set(path, (perFile.get(path) ?? 0) + 1); + } + for (let i = 0; i < files.length; i += 1) { + for (let j = i + 1; j < files.length; j += 1) { + // `files` is sorted and deduplicated upstream, so files[i] < files[j] + // and the key is canonical. NUL cannot occur inside a path — git's own + // -z framing depends on that — so it is the one safe separator. + const key = `${files[i]}\0${files[j]}`; + support.set(key, (support.get(key) ?? 0) + 1); + weighted.set(key, (weighted.get(key) ?? 0) + weight); + } + } + } + + const pairs: ScoredPair[] = []; + for (const [key, both] of support) { + if (both < minSupport) continue; + const [left, right] = key.split('\0') as [string, string]; + // |A ∪ B| = |A| + |B| − |A ∩ B|, and |A ∩ B| is exactly `both` over the + // scored events. Symmetric by construction, which is the property A-009 + // requires and a per-file marginal would not have. + const occurrences = (perFile.get(left) ?? 0) + (perFile.get(right) ?? 0) - both; + pairs.push({ + files: [left, right], + support: both, + occurrences, + weightedSupport: weighted.get(key) ?? 0, + }); + } + + // Path order: a total order, deterministic, and visibly not a ranking. + pairs.sort((a, b) => { + if (a.files[0] !== b.files[0]) return a.files[0] < b.files[0] ? -1 : 1; + if (a.files[1] !== b.files[1]) return a.files[1] < b.files[1] ? -1 : 1; + return 0; + }); + + const oldest = events[0]!; + const newest = events[events.length - 1]!; + + return { + l0ScoreVersion: 1, + completeness: completeness( + pairs.length > 0 + ? CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED + : CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP, + CompletenessReason.MINED, + `scored ${scored.length} of ${events.length} extracted event(s) at ${WEIGHTING_VERSION}; ${excludedCommits.length} excluded for fileCount > ${SCORING_MAX_FILE_COUNT}; ${pairs.length} pair(s) at support >= ${minSupport}`, + ), + basisWindow, + scoringBasis: { + weightingVersion: WEIGHTING_VERSION, + sizeWeightNumerator: SIZE_WEIGHT_NUMERATOR, + positionDecayHalfLife: POSITION_DECAY_HALF_LIFE, + maxScoredFileCount: SCORING_MAX_FILE_COUNT, + basisRevision: basisWindow.basisCommit, + windowOldestCommit: oldest.commit, + windowNewestCommit: newest.commit, + decayOriginPosition, + }, + exclusions: { + maxFileCount: SCORING_MAX_FILE_COUNT, + scoredEventCount: scored.length, + excludedEventCount: excludedCommits.length, + excludedCommits, + }, + pairs, + minSupport, + }; +} diff --git a/packages/mining-core/src/select.test.ts b/packages/mining-core/src/select.test.ts new file mode 100644 index 0000000..6fea74f --- /dev/null +++ b/packages/mining-core/src/select.test.ts @@ -0,0 +1,324 @@ +/** + * The provisional producer-profile selection rule: threshold, rank, cap. + * + * Named behavior, not a numbered requirement — only REQ-001..006 are written + * down for this package, and inventing an identifier would read as a citation + * to something that does not exist. + * + * Every expectation is computed by hand from the ruling, before the + * implementation existed. None was read off what the code returned. + */ +import { describe, expect, it } from 'vitest'; +import { CompletenessReason, CompletenessState, completeness } from './completeness.js'; +import { DEFAULT_MIN_SUPPORT, type ScoredPair, type ScoredSet } from './score.js'; +import { + RANKING_RULE, + SELECTION_CAP, + type SelectionResult, + compareUtf8, + select, +} from './select.js'; +import { serializeSelection } from './serialize.js'; + +const OID = 'a'.repeat(40); + +function pair(files: [string, string], support: number, occurrences: number): ScoredPair { + return { files, support, occurrences, weightedSupport: support * 0.7215 }; +} + +function scoredSet(pairs: readonly ScoredPair[], minSupport = 1): ScoredSet { + return { + l0ScoreVersion: 1, + completeness: completeness( + pairs.length > 0 + ? CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED + : CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP, + CompletenessReason.MINED, + 'fixture', + ), + basisWindow: { + basisRevision: 'HEAD', + basisCommit: OID, + windowTransitions: 500, + availableTransitions: 634, + extractedTransitions: 500, + windowTruncated: true, + }, + scoringBasis: { + weightingVersion: 'META-289 v2.2.1', + sizeWeightNumerator: 10, + positionDecayHalfLife: 250, + maxScoredFileCount: 50, + basisRevision: OID, + windowOldestCommit: '0'.repeat(40), + windowNewestCommit: '3'.repeat(40), + decayOriginPosition: 499, + }, + exclusions: { + maxFileCount: 50, + scoredEventCount: 500, + excludedEventCount: 0, + excludedCommits: [], + }, + pairs, + minSupport, + }; +} + +/** N synthetic pairs with strictly descending support, so ranking is unambiguous. */ +function descending(count: number): ScoredPair[] { + return Array.from({ length: count }, (_, i) => + pair([`src/a${String(i).padStart(4, '0')}.ts`, `src/b${String(i).padStart(4, '0')}.ts`], count - i, count + 10), + ); +} + +function keys(result: SelectionResult): string[] { + return result.pairs.map((p) => p.files.join('|')); +} + +describe('selection rule: threshold', () => { + it('applies the frozen support threshold and drops everything below it', () => { + const input = scoredSet([ + pair(['src/a.ts', 'src/b.ts'], 5, 10), + pair(['src/c.ts', 'src/d.ts'], 3, 10), + pair(['src/e.ts', 'src/f.ts'], 2, 10), + pair(['src/g.ts', 'src/h.ts'], 1, 10), + ]); + const result = select(input); + + expect(DEFAULT_MIN_SUPPORT).toBe(3); + expect(result.receipt.minSupport).toBe(3); + // support 2 and 1 are below the frozen threshold. + expect(result.pairs.map((p) => p.support)).toEqual([5, 3]); + expect(result.receipt.pairsBeforeCap).toBe(2); + }); + + it('refuses a scored set already filtered above the selection threshold', () => { + // The input has silently lost pairs the selection needed. Selecting from it + // would under-count without saying so, which is the same defect class as + // reporting zero for a shallow clone. + const input = scoredSet([pair(['src/a.ts', 'src/b.ts'], 9, 10)], 5); + expect(() => select(input, { minSupport: 3 })).toThrow(/threshold/i); + }); +}); + +describe('selection rule: ranking', () => { + it('ranks by support descending', () => { + const input = scoredSet([ + pair(['src/a.ts', 'src/b.ts'], 3, 10), + pair(['src/c.ts', 'src/d.ts'], 9, 10), + pair(['src/e.ts', 'src/f.ts'], 6, 10), + ]); + expect(select(input).pairs.map((p) => p.support)).toEqual([9, 6, 3]); + }); + + it('breaks a support tie by occurrences ascending — the tighter pair first', () => { + const input = scoredSet([ + pair(['src/a.ts', 'src/b.ts'], 7, 300), + pair(['src/c.ts', 'src/d.ts'], 7, 12), + pair(['src/e.ts', 'src/f.ts'], 7, 90), + ]); + expect(select(input).pairs.map((p) => p.occurrences)).toEqual([12, 90, 300]); + }); + + it('breaks a (support, occurrences) tie by files ascending in UTF-8 BYTE order', () => { + // The discriminating case. U+E000 encodes as EE 80 80; U+1F600 as F0 9F 98 80. + // In UTF-8 bytes, EE < F0, so the private-use path sorts FIRST. + // In UTF-16 code units — what a bare `<` on a JS string gives — the emoji is + // a surrogate pair starting 0xD83D, and 0xD83D < 0xE000, so the EMOJI sorts + // first. The two orders are opposite, which is what makes this a real test + // of the ruling rather than a restatement of the default comparator. + const privateUse = 'src/.ts'; + const astral = 'src/\u{1F600}.ts'; + expect(astral < privateUse).toBe(true); // UTF-16 says emoji first + expect(compareUtf8(privateUse, astral)).toBeLessThan(0); // UTF-8 says private-use first + + const input = scoredSet([ + pair([astral, 'src/z.ts'], 4, 20), + pair([privateUse, 'src/z.ts'], 4, 20), + ]); + expect(select(input).pairs[0]!.files[0]).toBe(privateUse); + }); + + it('breaks a files[0] tie by files[1], also in UTF-8 byte order', () => { + const input = scoredSet([ + pair(['src/a.ts', 'src/\u{1F600}.ts'], 4, 20), + pair(['src/a.ts', 'src/.ts'], 4, 20), + ]); + expect(select(input).pairs.map((p) => p.files[1])).toEqual(['src/.ts', 'src/\u{1F600}.ts']); + }); + + it('is a total order: a shuffled input produces an identical result', () => { + const base = descending(120); + // Many exact ties on both integer keys, so keys 3 and 4 do the work. + const tied = Array.from({ length: 40 }, (_, i) => + pair([`t/p${String(i).padStart(3, '0')}.ts`, 't/q.ts'], 4, 40), + ); + const all = [...base, ...tied]; + const shuffled = [...all]; + for (let i = shuffled.length - 1; i > 0; i -= 1) { + const j = (i * 7919 + 13) % (i + 1); + [shuffled[i], shuffled[j]] = [shuffled[j]!, shuffled[i]!]; + } + expect(keys(select(scoredSet(shuffled)))).toEqual(keys(select(scoredSet(all)))); + }); + + it('publishes the complete ranking rule as a string', () => { + expect(RANKING_RULE).toMatch(/support/i); + expect(RANKING_RULE).toMatch(/occurrences/i); + expect(RANKING_RULE).toMatch(/utf-8/i); + expect(select(scoredSet(descending(5))).receipt.rankingRule).toBe(RANKING_RULE); + }); +}); + +describe('selection rule: cap', () => { + it('caps at 50 and applies the cap AFTER ranking', () => { + expect(SELECTION_CAP).toBe(50); + const result = select(scoredSet(descending(120))); + expect(result.pairs).toHaveLength(50); + // Ranking first means the 50 kept are the 50 highest-support, so the + // smallest support emitted is 120 - 49 = 71. Capping before ranking would + // have kept an arbitrary 50 and this number would be wrong. + expect(result.pairs[0]!.support).toBe(120); + expect(result.pairs[49]!.support).toBe(71); + }); + + it('does not cap when the ranked list is shorter than the cap', () => { + // 12 synthetic pairs at supports 12..1; the threshold of 3 removes the two + // at supports 2 and 1, leaving 10. Well under the cap, so nothing is cut. + const result = select(scoredSet(descending(12))); + expect(result.pairs).toHaveLength(10); + expect(result.receipt.pairsBeforeCap).toBe(10); + expect(result.receipt.capBound).toBe(false); + }); +}); + +describe('selection rule: execution receipt', () => { + it('records the threshold, both counts, the cap, the rule, and whether the cap bound', () => { + const result = select(scoredSet(descending(120))); + expect(result.receipt).toEqual({ + minSupport: 3, + pairsBeforeCap: 118, // 120 synthetic pairs, two below support 3 + pairsEmitted: 50, + cap: 50, + rankingRule: RANKING_RULE, + capBound: true, + }); + }); + + it('makes a truncated list visibly truncated rather than silently short', () => { + const truncated = select(scoredSet(descending(120))).receipt; + expect(truncated.capBound).toBe(true); + expect(truncated.pairsEmitted).toBeLessThan(truncated.pairsBeforeCap); + + const whole = select(scoredSet(descending(12))).receipt; + expect(whole.capBound).toBe(false); + expect(whole.pairsEmitted).toBe(whole.pairsBeforeCap); + }); + + it('keeps pairsEmitted equal to min(pairsBeforeCap, cap) in both directions', () => { + for (const n of [0, 1, 49, 50, 51, 400]) { + const r = select(scoredSet(descending(n + 2))).receipt; + expect(r.pairsEmitted).toBe(Math.min(r.pairsBeforeCap, r.cap)); + expect(r.capBound).toBe(r.pairsBeforeCap > r.cap); + } + }); +}); + +describe('selection rule: preservation', () => { + it('does not mutate the scored set, and leaves the uncapped result complete', () => { + const input = scoredSet(descending(120)); + const before = input.pairs.length; + const beforeFirst = input.pairs[0]!.weightedSupport; + + const result = select(input); + + // Capping is a presentation step. The audit trail upstream is untouched. + expect(input.pairs).toHaveLength(before); + expect(input.pairs[0]!.weightedSupport).toBe(beforeFirst); + expect(result.pairs).toHaveLength(50); + // And the counts the receipt reports agree with what survived upstream. + expect(input.pairs.filter((p) => p.support >= 3)).toHaveLength(result.receipt.pairsBeforeCap); + }); + + it('carries the basis pin, the window and the scoring exclusions through', () => { + const input = scoredSet(descending(10)); + const result = select(input); + expect(result.scoringBasis).toEqual(input.scoringBasis); + expect(result.basisWindow).toEqual(input.basisWindow); + expect(result.exclusions).toEqual(input.exclusions); + }); +}); + +describe('selection rule: no floats reach the artifact', () => { + it('emits no weightedSupport, and no float, on a selected pair', () => { + const result = select(scoredSet(descending(10))); + for (const p of result.pairs) { + expect(Object.keys(p).sort()).toEqual(['files', 'occurrences', 'support']); + expect('weightedSupport' in p).toBe(false); + expect(Number.isInteger(p.support)).toBe(true); + expect(Number.isInteger(p.occurrences)).toBe(true); + } + }); + + it('serializes with every number an integer', () => { + const text = serializeSelection(select(scoredSet(descending(120)))); + expect(text).not.toContain('weightedSupport'); + // No decimal point and no exponent anywhere in the emitted numbers. + for (const [, literal] of text.matchAll(/:(-?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)/g)) { + expect(Number.isInteger(Number(literal))).toBe(true); + } + }); + + it('refuses to serialize a float rather than rounding or dropping it', () => { + const result = select(scoredSet(descending(5))); + // A float smuggled in by a future caller must fail loudly. Rounding it + // would be the churn class the float prohibition exists to prevent. + const contaminated = { + ...result, + pairs: [{ ...result.pairs[0]!, support: 3.5 }], + } as unknown as SelectionResult; + expect(() => serializeSelection(contaminated)).toThrow(/integer/i); + }); +}); + +describe('selection rule: honest degradation', () => { + it('passes a non-mined completeness through with no pairs and an honest receipt', () => { + const shallow: ScoredSet = { + l0ScoreVersion: 1, + completeness: completeness( + CompletenessState.NOT_MINED, + CompletenessReason.SHALLOW_CLONE, + 'shallow', + ), + exclusions: { + maxFileCount: 50, + scoredEventCount: 0, + excludedEventCount: 0, + excludedCommits: [], + }, + pairs: [], + minSupport: 3, + }; + const result = select(shallow); + expect(result.completeness.state).toBe(CompletenessState.NOT_MINED); + expect(result.completeness.reason).toBe(CompletenessReason.SHALLOW_CLONE); + expect(result.pairs).toEqual([]); + expect(result.scoringBasis).toBeUndefined(); + // The receipt is still complete: an empty list from a repository that was + // never examined must not look like an empty list from one that was. + expect(result.receipt.pairsBeforeCap).toBe(0); + expect(result.receipt.pairsEmitted).toBe(0); + expect(result.receipt.cap).toBe(50); + expect(result.receipt.capBound).toBe(false); + }); + + it('distinguishes examined-and-empty from not-examined at the receipt level', () => { + const examined = select(scoredSet([pair(['src/a.ts', 'src/b.ts'], 1, 9)])); + // Mining ran; nothing cleared the threshold. + expect(examined.completeness.state).toBe(CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP); + expect(examined.pairs).toEqual([]); + // And unlike the shallow case, the basis is pinned — the reader can tell. + expect(examined.scoringBasis).toBeDefined(); + }); +}); diff --git a/packages/mining-core/src/select.ts b/packages/mining-core/src/select.ts new file mode 100644 index 0000000..cb6ab91 --- /dev/null +++ b/packages/mining-core/src/select.ts @@ -0,0 +1,235 @@ +/** + * The provisional producer-profile selection rule — threshold, rank, cap. + * + * A named behavior, not a numbered requirement. Only REQ-001..006 are written + * down for this package; an invented identifier reads as a citation and cites + * nothing. + * + * This is the last L0 step and the only one whose shape is bound for an + * artifact. It takes a scored set and answers one question: of everything + * observed, which pairs does a producer emit, in what order, and how does a + * reader know what was left out. + * + * Three properties are load-bearing, and each exists because its absence is a + * known failure: + * + * 1. **The ranking keys on integers only.** `support` and `occurrences` are + * counts. `weightedSupport` is a double whose precision ECMAScript leaves + * implementation-defined, so an order that depended on it would be an order + * that could differ between engines. It does not appear here at all. + * + * 2. **The cap is applied after ranking, and it is recorded.** Capping first + * would silently choose which pairs get ranked. Capping without recording + * would make a truncated list indistinguishable from a short one — the same + * defect as reporting zero partners for a shallow clone, one layer up. + * + * 3. **Capping loses nothing upstream.** `select` is pure: the scored set it + * was handed still carries every pair and every weight afterwards, and the + * extracted events behind it are untouched. Capping is a presentation step. + * + * The public command name for the refresh operation that will call this is + * **deliberately not chosen here**. History mining is an explicit refresh + * rather than part of default generation — a bound 500-transition window costs + * seconds to tens of seconds — but naming the command is a separate decision + * and this module does not pre-empt it. + */ +import { + type Completeness, + CompletenessReason, + CompletenessState, + completeness, +} from './completeness.js'; +import type { BasisWindow } from './mine.js'; +import { DEFAULT_MIN_SUPPORT, type ScoredSet, type ScoringBasis, type ScoringExclusions } from './score.js'; + +/** Pairs emitted per repository, applied after ranking. */ +export const SELECTION_CAP = 50; + +/** + * The complete ranking rule, as text, carried in every receipt. + * + * Written out rather than described so that a reader comparing two artifacts + * can see whether they were ranked the same way without reading this file. + */ +export const RANKING_RULE = + 'support DESC, then occurrences ASC, then files[0] ASC by UTF-8 bytes, then files[1] ASC by UTF-8 bytes'; + +const UTF8 = new TextEncoder(); + +/** + * Compare two paths by UTF-8 byte order. + * + * Not `a < b`. A bare JavaScript string comparison is UTF-16 code unit order, + * and the two disagree: a supplementary character such as U+1F600 is a + * surrogate pair beginning 0xD83D, which sorts *before* U+E000 in UTF-16, while + * its UTF-8 encoding (F0 9F 98 80) sorts *after* U+E000's (EE 80 80). Any + * repository with an emoji in a path and anything in the private use area would + * rank differently under the two rules, and the ruling names UTF-8. + * + * Not `localeCompare` either — that varies with host locale, which would make + * the order depend on the machine that produced it. + */ +export function compareUtf8(a: string, b: string): number { + if (a === b) return 0; + const left = UTF8.encode(a); + const right = UTF8.encode(b); + const shared = Math.min(left.length, right.length); + for (let i = 0; i < shared; i += 1) { + if (left[i] !== right[i]) return left[i]! - right[i]!; + } + return left.length - right.length; +} + +/** + * A pair as a producer would emit it. + * + * Exactly three fields, all of them integers or strings. `weightedSupport` is + * absent by construction, not by omission — see the float prohibition in + * `serializeSelection`. + */ +export interface SelectedPair { + files: readonly [string, string]; + /** Distinct scored commits in which both files changed. */ + support: number; + /** Distinct scored commits in which at least one changed. The symmetric union. */ + occurrences: number; +} + +/** + * What this selection did, recorded rather than implied. + * + * A reader must be able to tell an emitted list of 50 that is everything from + * an emitted list of 50 that is the top of 1,848, without access to the + * repository. That is what `pairsBeforeCap` and `capBound` are for. + */ +export interface SelectionReceipt { + /** The support threshold applied before ranking. */ + minSupport: number; + /** Pairs that cleared the threshold. The population the cap cut from. */ + pairsBeforeCap: number; + /** Pairs actually emitted. `min(pairsBeforeCap, cap)`. */ + pairsEmitted: number; + /** The cap in force. */ + cap: number; + /** The complete ranking rule, as text. */ + rankingRule: string; + /** True when the cap actually bound — i.e. the emitted list is truncated. */ + capBound: boolean; +} + +export interface SelectionResult { + /** Bumped when this shape changes. Consumers pin on it. */ + readonly l0SelectionVersion: 1; + /** Recomputed against the threshold. Same four states; no new ones. */ + completeness: Completeness; + basisWindow?: BasisWindow; + scoringBasis?: ScoringBasis; + exclusions: ScoringExclusions; + receipt: SelectionReceipt; + /** Ranked and capped. */ + pairs: readonly SelectedPair[]; +} + +export interface SelectOptions { + /** The frozen support threshold. Defaults to 3. */ + minSupport?: number; + /** Pairs emitted per repository. Defaults to 50. */ + cap?: number; +} + +/** + * Apply the selection rule to a scored set. + * + * Pure. The input is not mutated and remains fully auditable afterwards. + * + * Throws only on a caller bug — a scored set that was already filtered above + * the selection threshold, and has therefore silently lost pairs the selection + * needed. That would produce a well-formed answer at reduced magnitude, which + * is the failure mode with the highest external cost on this project. + */ +export function select(scored: ScoredSet, options: SelectOptions = {}): SelectionResult { + const minSupport = options.minSupport ?? DEFAULT_MIN_SUPPORT; + const cap = options.cap ?? SELECTION_CAP; + + if (scored.minSupport > minSupport) { + throw new Error( + `select: the scored set was built at threshold ${scored.minSupport}, above the selection threshold ${minSupport}, so pairs this selection needs were already discarded and the result would be short without saying so`, + ); + } + + const mined = + scored.completeness.state === CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED || + scored.completeness.state === CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP; + + // Honest degradation. States 1 and 4 mean the history was not established, + // and a non-empty pair list is not permission to override that. The receipt + // is still filled in completely: an empty list from a repository that was + // never examined must not read like an empty list from one that was. + if (!mined) { + return { + l0SelectionVersion: 1, + completeness: scored.completeness, + ...(scored.basisWindow === undefined ? {} : { basisWindow: scored.basisWindow }), + ...(scored.scoringBasis === undefined ? {} : { scoringBasis: scored.scoringBasis }), + exclusions: scored.exclusions, + receipt: { + minSupport, + pairsBeforeCap: 0, + pairsEmitted: 0, + cap, + rankingRule: RANKING_RULE, + capBound: false, + }, + pairs: [], + }; + } + + // 1. Threshold. Copy first — `sort` mutates, and the scored set is the audit + // trail the cap is explicitly not allowed to damage. + const qualifying = scored.pairs.filter((pair) => pair.support >= minSupport); + + // 2. Rank. Integer keys first, then UTF-8 path order, which is total because + // `files` is unique per pair. No key is a float. + const ranked = [...qualifying].sort( + (a, b) => + b.support - a.support || + a.occurrences - b.occurrences || + compareUtf8(a.files[0], b.files[0]) || + compareUtf8(a.files[1], b.files[1]), + ); + + // 3. Cap, after ranking, so the pairs kept are the highest-ranked ones and + // not an arbitrary prefix of whatever order the map happened to produce. + const emitted = ranked.slice(0, cap); + + // Project to the artifact-bound shape. `weightedSupport` is dropped here and + // only here; it survives on the scored set for diagnostics. + const pairs: SelectedPair[] = emitted.map((pair) => ({ + files: pair.files, + support: pair.support, + occurrences: pair.occurrences, + })); + + return { + l0SelectionVersion: 1, + completeness: completeness( + pairs.length > 0 + ? CompletenessState.QUALIFYING_RELATIONSHIP_OBSERVED + : CompletenessState.MINED_NO_QUALIFYING_RELATIONSHIP, + CompletenessReason.MINED, + `${ranked.length} pair(s) at support >= ${minSupport}; ${pairs.length} emitted under a cap of ${cap}${ranked.length > cap ? ' (cap bound)' : ''}`, + ), + ...(scored.basisWindow === undefined ? {} : { basisWindow: scored.basisWindow }), + ...(scored.scoringBasis === undefined ? {} : { scoringBasis: scored.scoringBasis }), + exclusions: scored.exclusions, + receipt: { + minSupport, + pairsBeforeCap: ranked.length, + pairsEmitted: pairs.length, + cap, + rankingRule: RANKING_RULE, + capBound: ranked.length > cap, + }, + pairs, + }; +} diff --git a/packages/mining-core/src/serialize.ts b/packages/mining-core/src/serialize.ts new file mode 100644 index 0000000..8f57d14 --- /dev/null +++ b/packages/mining-core/src/serialize.ts @@ -0,0 +1,100 @@ +/** + * Deterministic serialization of an observation set (REQ-004). + * + * REQ-004 asks that two runs at the same `basisRevision` produce byte-identical + * output. That is a property of the *serializer* as much as the miner: + * `JSON.stringify` preserves insertion order, so two structurally identical + * objects built by different code paths can serialize to different bytes. + * Sorting keys removes that degree of freedom. + * + * Scope of the guarantee, stated because the Phase 0 audit found it matters. + * These bytes are a function of the repository at `basisCommit` and nothing + * else — no wall clock, no locale, no environment, no host paths. That is + * narrower than "the artifact is deterministic": `generated.hygiene` in the + * published producer is fed by a 30-day `git log --since` window and moves on + * its own (META-306). L0 does not inherit that and does not fix it. + */ +import type { ObservationSet } from './mine.js'; +import type { ScoredSet } from './score.js'; +import type { SelectionResult } from './select.js'; + +/** + * Stable JSON. Object keys sorted by UTF-16 code unit; arrays left in the order + * the producer emitted, which is itself sorted. + * + * `localeCompare` is deliberately not used — it varies with host locale and + * would make the bytes machine-dependent, which is the failure this function + * exists to prevent. + * + * `integersOnly` enforces the float prohibition. It is a parameter rather than + * a separate walker so that there is exactly one serializer and no second + * definition to drift. + */ +function stableStringify(value: unknown, integersOnly = false): string { + if (value === null) return 'null'; + if (Array.isArray(value)) { + return `[${value.map((item) => stableStringify(item, integersOnly)).join(',')}]`; + } + if (typeof value === 'object') { + const entries = Object.entries(value as Record) + .filter(([, v]) => v !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + return `{${entries + .map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v, integersOnly)}`) + .join(',')}}`; + } + if (typeof value === 'number') { + if (!Number.isFinite(value)) { + // NaN and Infinity serialize to `null` under JSON.stringify, which would + // silently turn a computation bug into a plausible-looking absent value. + throw new Error(`stableStringify: non-finite number ${value}`); + } + if (integersOnly && !Number.isInteger(value)) { + throw new Error( + `stableStringify: ${value} is not an integer, and floats are prohibited in artifact-bound output`, + ); + } + } + return JSON.stringify(value) ?? 'null'; +} + +/** Serialize an observation set to stable bytes. */ +export function serializeObservationSet(observations: ObservationSet): string { + return stableStringify(observations); +} + +/** + * Serialize a scored set to stable bytes. + * + * Same guarantee and the same narrow scope, with one addition worth stating. + * `weightedSupport` is a double produced by `2 ** x`, whose precision + * ECMAScript leaves implementation-defined. Two runs on one engine agree + * exactly, and the test proves it. Two runs on + * *different* engines may differ in the last ulp, so the byte-identical claim + * is scoped to a fixed engine and is not a cross-platform hash. The ranking + * rule is required to key on integer counts precisely so that this float never + * decides an order. + */ +export function serializeScoredSet(scored: ScoredSet): string { + return stableStringify(scored); +} + +/** + * Serialize a selection result to stable bytes, with the float prohibition + * enforced rather than assumed. + * + * This is the only shape in this package that is bound for an artifact, so it + * is the only one where a float is a defect rather than a detail. Any + * non-integer number **throws**. It is not rounded and it is not dropped: + * rounding would invent precision the measurement does not have, and dropping + * would remove a field a reader was told to expect. Both are quieter than the + * bug, and quiet is the failure mode this package exists to avoid. + * + * Consequence, stated plainly: `weightedSupport` cannot appear here, which is + * why `SelectedPair` does not carry it. It stays in memory on the scored set, + * where `serializeScoredSet` will happily emit it for diagnostics that never + * reach `workspace.json`. + */ +export function serializeSelection(selection: SelectionResult): string { + return stableStringify(selection, true); +} diff --git a/packages/mining-core/src/testing/fixtures.ts b/packages/mining-core/src/testing/fixtures.ts new file mode 100644 index 0000000..6fcb1f2 --- /dev/null +++ b/packages/mining-core/src/testing/fixtures.ts @@ -0,0 +1,160 @@ +/** + * Synthetic git fixtures, one per REQ-005 state. + * + * Built in a temp directory rather than committed, because a committed git + * repository inside a git repository is a submodule or a packed oddity, and + * both make the fixture harder to read than the script that produces it. The + * script IS the specification of what each state looks like. + * + * Every commit pins author and committer identity and date, so fixture object + * ids are stable across machines and REQ-004's byte-identical claim is a + * property of the miner rather than of the clock. + */ +import { execFileSync } from 'node:child_process'; +import { existsSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; + +const FIXED_DATE = '2026-01-01T00:00:00+0000'; + +const FIXED_ENV = { + GIT_AUTHOR_NAME: 'Fixture', + GIT_AUTHOR_EMAIL: 'fixture@example.invalid', + GIT_COMMITTER_NAME: 'Fixture', + GIT_COMMITTER_EMAIL: 'fixture@example.invalid', + GIT_AUTHOR_DATE: FIXED_DATE, + GIT_COMMITTER_DATE: FIXED_DATE, + GIT_CONFIG_NOSYSTEM: '1', + LC_ALL: 'C', +}; + +export function git(cwd: string, args: readonly string[]): string { + return execFileSync('git', [...args], { + cwd, + encoding: 'utf8', + env: { ...process.env, ...FIXED_ENV }, + }); +} + +export function makeTempDir(label: string): string { + return mkdtempSync(join(tmpdir(), `wsj-l0-${label}-`)); +} + +export function removeDir(path: string): void { + rmSync(path, { recursive: true, force: true }); +} + +function initRepo(root: string): void { + git(root, ['init', '--quiet', '--initial-branch=main']); + git(root, ['config', 'user.name', 'Fixture']); + git(root, ['config', 'user.email', 'fixture@example.invalid']); + // Rename detection is part of the frozen extraction parameters, so the + // fixture must not have it disabled by an inherited global config. + git(root, ['config', 'diff.renames', 'true']); +} + +export function writeAndAdd(root: string, path: string, content: string): void { + const full = join(root, path); + mkdirSync(dirname(full), { recursive: true }); + writeFileSync(full, content, 'utf8'); + git(root, ['add', '--', path]); +} + +export function commit(root: string, message: string): void { + git(root, ['commit', '--quiet', '-m', message]); +} + +/** + * State 1 via NO_COMMITS: an initialized repository with no commits. + * Absence of history, not absence of coupling. + */ +export function makeEmptyRepo(): string { + const root = makeTempDir('empty'); + initRepo(root); + return root; +} + +/** + * State 2: real history, every commit touching exactly one file, so no pair + * ever co-occurs. Mining ran and the answer is genuinely "no relationship". + */ +export function makeUncoupledRepo(): string { + const root = makeTempDir('uncoupled'); + initRepo(root); + for (const [index, path] of ['src/a.ts', 'src/b.ts', 'src/c.ts'].entries()) { + writeAndAdd(root, path, `export const v${index} = ${index};\n`); + commit(root, `add ${path}`); + } + // Further single-file edits, still never two files in one commit. + writeAndAdd(root, 'src/a.ts', 'export const v0 = 99;\n'); + commit(root, 'edit a'); + return root; +} + +/** + * State 3: two files that change together three times and share no import. + * The shape the thesis rests on, in miniature. + */ +export function makeCoupledRepo(): string { + const root = makeTempDir('coupled'); + initRepo(root); + + writeAndAdd(root, 'src/build.ts', 'export const key = (a: string, b: number) => `${a}:${b}`;\n'); + writeAndAdd(root, 'src/parse.ts', "export const parse = (k: string) => k.split(':');\n"); + writeAndAdd(root, 'README.md', '# fixture\n'); + commit(root, 'initial'); + + for (const round of [1, 2]) { + writeAndAdd(root, 'src/build.ts', `export const key = (a: string, b: number) => \`\${a}:\${b}:${round}\`;\n`); + writeAndAdd(root, 'src/parse.ts', `// round ${round}\nexport const parse = (k: string) => k.split(':');\n`); + commit(root, `round ${round}`); + } + + // A lone commit so not every event is the coupled pair. + writeAndAdd(root, 'README.md', '# fixture\n\nnotes\n'); + commit(root, 'docs'); + + return root; +} + +/** + * A shallow clone of a coupled repository — REQ-006. + * + * The point of cloning the *coupled* fixture is that a shallow clone of an + * uncoupled repository would report no pairs for the right reason by accident. + * Here the full history has a real pair and the shallow clone cannot see it, so + * an unguarded miner returns a confident, wrong, non-empty-looking answer. + */ +export function makeShallowCloneOfCoupled(): { source: string; shallow: string } { + const source = makeCoupledRepo(); + const shallow = makeTempDir('shallow'); + removeDir(shallow); + git(process.cwd(), ['clone', '--quiet', '--depth', '1', `file://${source}`, shallow]); + return { source, shallow }; +} + +/** + * State 4: a repository whose HEAD resolves but whose history cannot be walked. + * + * Built by deleting the root commit's loose object. `rev-parse HEAD^{commit}` + * still succeeds because HEAD's own object is intact, so this is genuinely + * "evidence reachable but unavailable" and not "no history" — which is the + * distinction REQ-005's state 4 exists to carry. Returns `undefined` when the + * objects turn out to be packed, so the caller skips rather than asserting + * against a repository that was never corrupted. + */ +export function makeCorruptedRepo(): string | undefined { + const root = makeCoupledRepo(); + const rootCommit = git(root, ['rev-list', '--max-parents=0', 'HEAD']).trim(); + const loose = join(root, '.git', 'objects', rootCommit.slice(0, 2), rootCommit.slice(2)); + if (!existsSync(loose)) return undefined; + rmSync(loose); + return root; +} + +/** A path that is not a git repository at all. */ +export function makeNonRepo(): string { + const root = makeTempDir('nonrepo'); + writeFileSync(join(root, 'plain.txt'), 'not a repository\n', 'utf8'); + return root; +} diff --git a/packages/mining-core/tsconfig.json b/packages/mining-core/tsconfig.json new file mode 100644 index 0000000..41a1e5a --- /dev/null +++ b/packages/mining-core/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "declaration": true, + "noEmit": false, + "types": ["node"] + }, + "//": "Deliberately does NOT include ../../types/ambient.d.ts. That file hand-declares node:fs, node:child_process and vitest, and ambient module declarations win over node_modules typings — OWNERSHIP.md records the resulting Dirent/readdirSync breakage as an unfixed defect of the same class META-244 already fixed once for @workspacejson/spec. This package compiles against the real @types/node@22.19.17 and vitest declarations instead, so its git and filesystem surface is typed by the packages that own it rather than by a local restatement.", + "include": [ + "src/**/*.ts" + ] +} diff --git a/packages/mining-core/vitest.config.ts b/packages/mining-core/vitest.config.ts new file mode 100644 index 0000000..71366e6 --- /dev/null +++ b/packages/mining-core/vitest.config.ts @@ -0,0 +1,23 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + // These tests drive real git against real repositories, which is the whole + // point — a mocked `diff-tree` would prove nothing about extraction. The + // cost is real too: v2.2.1's frozen parameters spend one `rev-parse` and + // one `diff-tree` per commit, sequentially, and the 5s default expires + // partway through. + // + // Raising the budget rather than batching the git calls is deliberate. The + // commands are quoted from the preregistration and REQ-001 verifies against + // them verbatim; replacing them with a faster equivalent would make the + // extraction no longer the thing that was frozen. + // + // Measured on an Apple M4 Pro: a bound 500-transition window costs 7.3-8.2s + // with a short PATH and 27.2-29.9s with a 36-entry one, because Node + // re-resolves the `git` binary through PATH on all 1000 spawns. Extraction + // is spawn-bound, not git-bound. Scoring the events costs 1-24ms. + testTimeout: 120_000, + hookTimeout: 120_000, + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 24ff6ea..ce23924 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -85,6 +85,24 @@ importers: picocolors: specifier: ^1.0.1 version: 1.1.1 + devDependencies: + '@types/node': + specifier: 22.19.17 + version: 22.19.17 + '@workspacejson/mining-core': + specifier: workspace:* + version: link:../mining-core + tsup: + specifier: ^8.0.0 + version: 8.5.1(postcss@8.5.23)(typescript@5.9.3) + typescript: + specifier: ^5.4.0 + version: 5.9.3 + vitest: + specifier: ^1.6.0 + version: 1.6.1(@types/node@22.19.17) + + packages/mining-core: devDependencies: '@types/node': specifier: 22.19.17