feat(memory): provenance metadata and origin filtering

CLAUDE.md

@@ -67,13 +67,13 @@ Auth uses timing-safe token comparison. Origin allowlisting supports `"*"` wildc
 
 | Tool | Purpose |
 |------|---------|
-| `remember_context` | Single write tool for chat and admin clients: save durable memory with optional topic, draft flag or explicit branch state, image input, and artifact refs |
-| `search_context` | Query → embedding → Firestore vector similarity search (cosine, top-K) with metadata filters |
+| `remember_context` | Single write tool for chat and admin clients: save durable memory with optional topic, draft flag or explicit branch state, image input, artifact refs, and provenance metadata |
+| `search_context` | Query → embedding → Firestore vector similarity search (cosine, top-K) with metadata and provenance origin filters |
 | `fetch_context` | Retrieve one stored memory by document ID after search |
 | `deprecate_context` | Soft-delete: mark document as deprecated, record superseding document ID, and record supersession_reason ("changed" sets valid_until, "corrected" does not) |
 | `consolidate_context` | Merge N related memories into one canonical active memory via LLM; deprecates all sources with `superseded_by` pointing to the merged result. Defaults to WIP queue for a topic; accepts explicit `source_ids` for targeted consolidation |
 
-`remember_context` also accepts optional `valid_from`/`valid_until` (epoch-ms numbers) so a write can carry its temporal validity window from creation.
+`remember_context` also accepts optional `valid_from`/`valid_until` (epoch-ms numbers) so a write can carry its temporal validity window from creation. It also accepts optional `origin`, `source_session`, `derived_from`, and `confidence` for provenance tracking (`origin` defaults to `agent_inferred` when omitted). `search_context` accepts optional `filter_origin` to filter search results by provenance origin as a post-filter (no new Firestore index).
 
 ### MCP Prompts
 
@@ -91,19 +91,19 @@ Auth uses timing-safe token comparison. Origin allowlisting supports `"*"` wildc
 | `errors.ts` | ~9 | `HttpError` exception with `statusCode` field |
 | `merging.ts` | ~74 | `LlmMergeClient` interface + `GeminiMergeClient` — calls Gemini to merge N memory contents into one |
 | `runtime.ts` | ~107 | Dependency injection: `createRuntime()` lazily creates and caches Gemini clients, Firestore repo, service |
-| `service.ts` | ~401 | `MetaCortexService` — remember/store/search/fetch/deprecate/consolidate flows |
+| `service.ts` | ~468 | `MetaCortexService` — remember/store/search/fetch/deprecate/consolidate flows |
 | `observability.ts` | ~150 | Structured tool-event and request-event logging plus Firestore-backed `memory_events` audit trail |
 | `embeddings.ts` | ~195 | `GeminiEmbeddingClient` + `GeminiMultimodalPreparer` (image→text normalization for retrieval) |
 | `normalize.ts` | ~8 | Shared text-normalization helper |
 | `memoryRepository.ts` | ~228 | Firestore CRUD: `store()`, `search()` (findNearest + cosine), `deprecate()`, `getConsolidationQueue()` |
-| `types.ts` | ~138 | Enums (`BRANCH_STATES`, `MEMORY_MODALITIES`, `MCP_TOOL_NAMES`) and interfaces |
-| `mcpServer.ts` | ~639 | MCP tool registration with Zod schemas, filtered by client's `allowedTools` and `allowedFilterStates`; also registers the `correct_memory` prompt unconditionally |
+| `types.ts` | ~175 | Enums (`BRANCH_STATES`, `MEMORY_MODALITIES`, `MCP_TOOL_NAMES`, `PROVENANCE_ORIGINS`) and interfaces |
+| `mcpServer.ts` | ~674 | MCP tool registration with Zod schemas, filtered by client's `allowedTools` and `allowedFilterStates`; also registers the `correct_memory` prompt unconditionally |
 
 ### Data Flow
 
-**remember_context**: Chat/admin input → server defaults/inference for metadata and lifecycle state → Gemini multimodal normalization (if image) → canonical `content` + internal `retrieval_text` → Gemini embedding (deployment currently pinned to 768-dim) → Firestore document with vector + metadata
+**remember_context**: Chat/admin input → server defaults/inference for metadata (including provenance, where origin defaults to agent_inferred) and lifecycle state → Gemini multimodal normalization (if image) → canonical `content` + internal `retrieval_text` → Gemini embedding (deployment currently pinned to 768-dim) → Firestore document with vector + metadata (including provenance origin)
 
-**search_context**: Query text → Gemini embedding → Firestore `findNearest()` (cosine distance, top-K) with required `branch_state` and optional topic filter; optional `valid_at` post-filters results in the service layer by temporal validity window
+**search_context**: Query text → Gemini embedding → Firestore `findNearest()` (cosine distance, top-K) with required `branch_state` and optional topic filter; optional `valid_at` post-filters results in the service layer by temporal validity window, and optional `filter_origin` post-filters by provenance origin
 
 **fetch_context**: Document ID → direct Firestore read of one stored memory
 

functions/package.json

@@ -11,6 +11,7 @@
     "build": "tsc -p tsconfig.json",
     "clean": "node -e \"const fs=require('fs'); fs.rmSync('lib',{recursive:true,force:true}); fs.rmSync('coverage',{recursive:true,force:true});\"",
     "backfill:ttl": "node scripts/backfill-firestore-ttl.mjs",
+    "backfill:provenance": "node scripts/backfill-firestore-provenance.mjs",
     "eval:generate": "tsx scripts/retrieval-eval.ts generate-isolated",
     "eval:import": "tsx scripts/retrieval-eval.ts import-production",
     "eval:run": "tsx scripts/retrieval-eval.ts run",

functions/scripts/backfill-firestore-provenance.mjs

@@ -0,0 +1,172 @@
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { getApps, initializeApp } from "firebase-admin/app";
+import { getFirestore } from "firebase-admin/firestore";
+
+const scriptDir = path.dirname(fileURLToPath(import.meta.url));
+const functionsDir = path.resolve(scriptDir, "..");
+const repoRoot = path.resolve(functionsDir, "..");
+const explicitEnvKeys = new Set(Object.keys(process.env));
+const loadedEnv = {};
+
+for (const fileName of [".env", ".env.prod"]) {
+  loadEnvFile(path.join(functionsDir, fileName), loadedEnv);
+}
+
+for (const [key, value] of Object.entries(loadedEnv)) {
+  if (!explicitEnvKeys.has(key)) {
+    process.env[key] = value;
+  }
+}
+
+const args = process.argv.slice(2);
+const write = args.includes("--write");
+const projectId =
+  readArg("project") ??
+  process.env.GOOGLE_CLOUD_PROJECT ??
+  process.env.GCLOUD_PROJECT ??
+  readFirebaseProject(repoRoot) ??
+  "my-brain-88870";
+const memoryCollection =
+  readArg("memory-collection") ??
+  process.env.MEMORY_COLLECTION?.trim() ??
+  "memory_vectors";
+const batchSize = positiveInt(readArg("batch-size"), 250, "batch-size");
+
+if (getApps().length === 0) {
+  initializeApp({ projectId });
+}
+
+const firestore = getFirestore();
+
+console.log(`project: ${projectId}`);
+console.log(`mode: ${write ? "write" : "dry-run"}`);
+console.log(`collection: ${memoryCollection}`);
+
+const result = await backfillProvenance(
+  firestore.collection(memoryCollection),
+  batchSize,
+  write
+);
+
+console.log(`${memoryCollection}:`, result);
+
+if (!write) {
+  console.log("Dry run complete. Re-run with --write to apply updates.");
+}
+
+async function backfillProvenance(collection, batchLimit, shouldWrite) {
+  // This legacy backfill reads the collection in one pass; for collections above
+  // roughly 100k docs, switch to paginated reads with limit/startAfter.
+  const snapshot = await collection.get();
+  const updates = [];
+  let skipped = 0;
+
+  for (const doc of snapshot.docs) {
+    const data = doc.data();
+
+    if (data.metadata && typeof data.metadata.provenance !== "undefined") {
+      skipped += 1;
+    } else if (data.metadata && typeof data.metadata.provenance === "undefined") {
+      updates.push({
+        ref: doc.ref,
+        update: {
+          "metadata.provenance": { origin: "legacy_import" }
+        }
+      });
+    }
+  }
+
+  if (shouldWrite && updates.length > 0) {
+    await commitUpdates(updates, batchLimit);
+  }
+
+  return {
+    scanned: snapshot.size,
+    update_count: updates.length,
+    skipped
+  };
+}
+
+async function commitUpdates(updates, batchLimit) {
+  for (let index = 0; index < updates.length; index += batchLimit) {
+    const batch = firestore.batch();
+
+    for (const { ref, update } of updates.slice(index, index + batchLimit)) {
+      batch.update(ref, update);
+    }
+
+    await batch.commit();
+  }
+}
+
+function readArg(name) {
+  const index = args.findIndex(arg => arg === `--${name}`);
+
+  if (index === -1) {
+    return undefined;
+  }
+
+  return args[index + 1];
+}
+
+function positiveInt(value, fallback, key) {
+  if (!value) {
+    return fallback;
+  }
+
+  const parsed = Number.parseInt(value, 10);
+
+  if (!Number.isInteger(parsed) || parsed <= 0) {
+    throw new Error(`${key} must be a positive integer`);
+  }
+
+  return parsed;
+}
+
+function loadEnvFile(filePath, target) {
+  if (!fs.existsSync(filePath)) {
+    return;
+  }
+
+  for (const rawLine of fs.readFileSync(filePath, "utf8").split(/\r?\n/)) {
+    const line = rawLine.trim();
+
+    if (!line || line.startsWith("#")) {
+      continue;
+    }
+
+    const separatorIndex = line.indexOf("=");
+
+    if (separatorIndex === -1) {
+      continue;
+    }
+
+    const key = line.slice(0, separatorIndex).trim();
+    let value = line.slice(separatorIndex + 1).trim();
+
+    if (
+      (value.startsWith("\"") && value.endsWith("\"")) ||
+      (value.startsWith("'") && value.endsWith("'"))
+    ) {
+      value = value.slice(1, -1);
+    }
+
+    target[key] = value;
+  }
+}
+
+function readFirebaseProject(rootDir) {
+  const firebaseRcPath = path.join(rootDir, ".firebaserc");
+
+  if (!fs.existsSync(firebaseRcPath)) {
+    return undefined;
+  }
+
+  const firebaseRc = JSON.parse(fs.readFileSync(firebaseRcPath, "utf8"));
+  const project = firebaseRc.projects?.prod ?? firebaseRc.projects?.default;
+
+  return typeof project === "string" && project.trim() ? project.trim() : undefined;
+}

functions/src/mcpServer.ts

@@ -18,6 +18,7 @@ import {
 } from "./service.js";
 import {
   BRANCH_STATES,
+  PROVENANCE_ORIGINS,
   SUPERSESSION_REASONS,
   type BranchState,
   type McpToolName
@@ -76,6 +77,32 @@ export function createMetaCortexMcpServer(
         .describe(
           "Optional epoch-ms timestamp marking when this fact stops being valid. Omit for facts with no known end."
         ),
+      origin: z
+        .enum(PROVENANCE_ORIGINS)
+        .optional()
+        .describe(
+          "Optional provenance origin for this write. Defaults to agent_inferred when omitted. Only claim user_asserted when the user explicitly stated this fact themselves."
+        ),
+      source_session: z
+        .string()
+        .optional()
+        .describe(
+          "Optional identifier for the session or conversation this memory was derived from."
+        ),
+      derived_from: z
+        .array(z.string())
+        .optional()
+        .describe(
+          "Optional list of memory ids that this inference was derived from."
+        ),
+      confidence: z
+        .number()
+        .min(0)
+        .max(1)
+        .optional()
+        .describe(
+          "Optional confidence score between 0 and 1 for agent-inferred memories."
+        ),
       image_base64: z
         .string()
         .optional()
@@ -243,6 +270,7 @@ export function createMetaCortexMcpServer(
         const requestSummary = {
           topic: normalizeOptionalText(args.topic) ?? "general",
           branch_state: requestedBranchState,
+          origin: args.origin,
           draft: args.draft,
           content_length: args.content?.trim().length ?? 0,
           image_present: Boolean(args.image_base64),
@@ -300,7 +328,13 @@ export function createMetaCortexMcpServer(
           valid_at: z
             .number()
             .optional()
-            .describe("Optional epoch-ms timestamp. When provided, only returns memories valid at that point in time (valid_from <= valid_at < valid_until, excluding corrected records).")
+            .describe("Optional epoch-ms timestamp. When provided, only returns memories valid at that point in time (valid_from <= valid_at < valid_until, excluding corrected records)."),
+          filter_origin: z
+            .enum(PROVENANCE_ORIGINS)
+            .optional()
+            .describe(
+              "Optional provenance origin filter. Only returns memories whose provenance.origin matches exactly; memories without provenance metadata are excluded when this filter is set."
+            )
         }
       },
       async args => {
@@ -313,7 +347,8 @@ export function createMetaCortexMcpServer(
           filter_topic: normalizedFilterTopic,
           filter_state: requestedFilterState,
           limit: args.limit,
-          valid_at: args.valid_at
+          valid_at: args.valid_at,
+          filter_origin: args.filter_origin
         };
         const result = await observeToolCall(
           "search_context",

functions/src/service.ts

@@ -56,6 +56,13 @@ export class MetaCortexService {
     });
     const now = Date.now();
 
+    const provenance = {
+      origin: input.origin ?? "agent_inferred",
+      ...(input.source_session ? { source_session: input.source_session } : {}),
+      ...(input.derived_from && input.derived_from.length > 0 ? { derived_from: input.derived_from } : {}),
+      ...(typeof input.confidence === "number" ? { confidence: input.confidence } : {})
+    };
+
     const metadata = {
       module_name: normalizedModule,
       branch_state: input.branch_state,
@@ -70,7 +77,8 @@ export class MetaCortexService {
         : {}),
       ...(typeof input.valid_until === "number"
         ? { valid_until: input.valid_until }
-        : {})
+        : {}),
+      provenance
     } as const;
 
     const result = await this.repository.store({
@@ -111,7 +119,11 @@ export class MetaCortexService {
       image_base64: normalizeOptionalText(input.image_base64),
       image_mime_type: normalizeOptionalText(input.image_mime_type),
       valid_from: input.valid_from,
-      valid_until: input.valid_until
+      valid_until: input.valid_until,
+      origin: input.origin,
+      source_session: input.source_session,
+      derived_from: input.derived_from,
+      confidence: input.confidence
     });
   }
 
@@ -134,6 +146,10 @@ export class MetaCortexService {
       matches = matches.filter(match => matchesValidAt(match.metadata, input.valid_at!));
     }
 
+    if (input.filter_origin) {
+      matches = matches.filter(match => match.metadata.provenance?.origin === input.filter_origin);
+    }
+
     return {
       matches,
       appliedFilters: {
@@ -408,7 +424,22 @@ function buildPublicMetadata(match: Pick<MemoryDocument, "metadata">): Record<st
       ? { artifact_refs: match.metadata.artifact_refs }
       : {}),
     created_at: new Date(match.metadata.created_at).toISOString(),
-    updated_at: new Date(match.metadata.updated_at).toISOString()
+    updated_at: new Date(match.metadata.updated_at).toISOString(),
+    ...(typeof match.metadata.valid_from === "number"
+      ? { valid_from: new Date(match.metadata.valid_from).toISOString() }
+      : {}),
+    ...(typeof match.metadata.valid_until === "number"
+      ? { valid_until: new Date(match.metadata.valid_until).toISOString() }
+      : {}),
+    ...(match.metadata.supersession_reason
+      ? { supersession_reason: match.metadata.supersession_reason }
+      : {}),
+    ...(match.metadata.initiator
+      ? { initiator: match.metadata.initiator }
+      : {}),
+    ...(match.metadata.provenance
+      ? { provenance: match.metadata.provenance }
+      : {})
   };
 }