Forge: persist sanitized fingerprint signatures

plugins/repo-dreamer/skills/repository-skill-forge/SKILL.md

@@ -71,6 +71,7 @@ repository-skill-forge-state:v2:begin
     "cursor": null,
     "updatedAt": "<windowEnd>",
     "observations": [],
+    "fingerprintCatalog": {},
     "proposalQueue": [],
     "proposalHistory": {}
 }
@@ -210,7 +211,11 @@ evidence into candidate generation.
 Raw commands and source content remain ephemeral. Persistent observations keep
 only evidence/fingerprint hashes, repository-salted session and branch hashes,
 timestamps, outcome, surface, kind, bounded path families, and repository
-references.
+references. Durable state also keeps a bounded catalog of versioned, sanitized
+structural signatures keyed by fingerprint. This allows later runs to explain
+and aggregate historical patterns without retaining raw commands or source
+content. Legacy v2 states without `fingerprintCatalog` migrate as an empty
+catalog on their next parse and render.
 
 ### 6. Build compact state and candidates
 

plugins/repo-dreamer/skills/repository-skill-forge/assets/schemas.json

@@ -259,6 +259,22 @@
           "type": "array",
           "items": { "$ref": "#/$defs/compactObservation" }
         },
+        "fingerprintCatalog": {
+          "type": "object",
+          "maxProperties": 64,
+          "propertyNames": { "pattern": "^[0-9a-f]{16}$" },
+          "additionalProperties": {
+            "type": "object",
+            "required": ["kind", "signatureVersion", "signature", "lastSeenAt"],
+            "properties": {
+              "kind": { "enum": ["command", "script"] },
+              "signatureVersion": { "const": 1 },
+              "signature": { "type": "object", "minProperties": 1 },
+              "lastSeenAt": { "type": "string" }
+            },
+            "additionalProperties": false
+          }
+        },
         "proposalQueue": {
           "type": "array",
           "items": { "$ref": "#/$defs/proposalQueueEntry" }

plugins/repo-dreamer/skills/repository-skill-forge/scripts/aggregate-primitives.py

@@ -6,13 +6,37 @@
 from __future__ import annotations
 
 import argparse
+import importlib.util
+import json
 import re
 from collections import defaultdict
 from datetime import timedelta
+from pathlib import Path
 from typing import Any
 
 from forge_common import parse_timestamp, read_json, stable_hash, timestamp_text, write_json
 
+SIGNATURE_VERSION = 1
+MAX_FINGERPRINT_CATALOG_ENTRIES = 64
+MAX_FINGERPRINT_CATALOG_BYTES = 12_000
+MAX_SIGNATURE_BYTES = 2_000
+
+
+def validate_next_state(
+    state: dict[str, Any],
+    repository: str,
+) -> dict[str, Any]:
+    validator_path = Path(__file__).with_name("issue-state.py")
+    spec = importlib.util.spec_from_file_location(
+        "repository_skill_forge_issue_state",
+        validator_path,
+    )
+    if spec is None or spec.loader is None:
+        raise ValueError("unable to load issue state validator")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module.validate_state(state, repository)
+
 
 def reference_values(primitive: dict[str, Any], ref_type: str) -> set[str]:
     refs = primitive.get("refs")
@@ -34,10 +58,34 @@ def merge_evidence(
     if isinstance(previous_scope, dict) and previous_scope.get("repository") != repository:
         raise ValueError("state repository does not match the current repository")
     evidence_by_key: dict[str, dict[str, Any]] = {}
+    fingerprint_catalog = (
+        state.get("fingerprintCatalog", {})
+        if isinstance(state, dict)
+        else {}
+    )
+    fingerprint_catalog = (
+        fingerprint_catalog if isinstance(fingerprint_catalog, dict) else {}
+    )
     if isinstance(state, dict):
         for item in state.get("observations", state.get("evidenceLedger", [])):
             if isinstance(item, dict) and item.get("evidenceKey"):
-                evidence_by_key[str(item["evidenceKey"])] = item
+                restored = dict(item)
+                catalog_entry = fingerprint_catalog.get(str(item.get("fingerprint")))
+                if (
+                    isinstance(catalog_entry, dict)
+                    and catalog_entry.get("kind") == item.get("kind")
+                    and catalog_entry.get("signatureVersion") == SIGNATURE_VERSION
+                    and isinstance(catalog_entry.get("signature"), dict)
+                    and stable_hash(
+                        {
+                            "kind": item.get("kind"),
+                            "signature": catalog_entry["signature"],
+                        }
+                    )
+                    == item.get("fingerprint")
+                ):
+                    restored["signature"] = catalog_entry["signature"]
+                evidence_by_key[str(item["evidenceKey"])] = restored
     for item in document.get("primitives", []):
         if isinstance(item, dict) and item.get("evidenceKey"):
             evidence_by_key[str(item["evidenceKey"])] = compact_observation(
@@ -51,6 +99,63 @@ def merge_evidence(
     return list(evidence_by_key.values()), history if isinstance(history, dict) else {}
 
 
+def build_fingerprint_catalog(
+    evidence: list[dict[str, Any]],
+) -> dict[str, dict[str, Any]]:
+    entries: dict[str, dict[str, Any]] = {}
+    for item in evidence:
+        fingerprint = item.get("fingerprint")
+        kind = item.get("kind")
+        signature = item.get("signature")
+        completed_at = item.get("completedAt")
+        if (
+            not isinstance(fingerprint, str)
+            or kind not in {"command", "script"}
+            or not isinstance(signature, dict)
+            or not signature
+            or not isinstance(completed_at, str)
+            or stable_hash({"kind": kind, "signature": signature}) != fingerprint
+        ):
+            continue
+        signature_bytes = len(
+            json.dumps(signature, sort_keys=True, separators=(",", ":")).encode()
+        )
+        if signature_bytes > MAX_SIGNATURE_BYTES:
+            continue
+        current = entries.get(fingerprint)
+        if current is None or parse_timestamp(completed_at) > parse_timestamp(
+            current["lastSeenAt"]
+        ):
+            entries[fingerprint] = {
+                "kind": kind,
+                "signatureVersion": SIGNATURE_VERSION,
+                "signature": signature,
+                "lastSeenAt": completed_at,
+            }
+
+    retained: dict[str, dict[str, Any]] = {}
+    retained_bytes = 2
+    ordered = sorted(
+        entries.items(),
+        key=lambda item: (-parse_timestamp(item[1]["lastSeenAt"]).timestamp(), item[0]),
+    )
+    for fingerprint, entry in ordered:
+        if len(retained) >= MAX_FINGERPRINT_CATALOG_ENTRIES:
+            break
+        entry_bytes = len(
+            json.dumps(
+                {fingerprint: entry},
+                sort_keys=True,
+                separators=(",", ":"),
+            ).encode()
+        )
+        if retained_bytes + entry_bytes > MAX_FINGERPRINT_CATALOG_BYTES:
+            continue
+        retained[fingerprint] = entry
+        retained_bytes += entry_bytes
+    return dict(sorted(retained.items()))
+
+
 def compact_observation(item: dict[str, Any], repository: str) -> dict[str, Any]:
     session_hash = (
         str(item["sessionHash"])
@@ -347,11 +452,13 @@ def main() -> None:
             retained_observations,
             key=lambda item: str(item.get("evidenceKey")),
         ),
+        "fingerprintCatalog": build_fingerprint_catalog(retained_evidence),
         "proposalQueue": proposal_queue if isinstance(proposal_queue, list) else [],
         "proposalHistory": proposal_history,
     }
+    validated_state = validate_next_state(next_state, args.repository)
     write_json(args.output, result)
-    write_json(args.state_out, next_state)
+    write_json(args.state_out, validated_state)
 
 
 if __name__ == "__main__":

plugins/repo-dreamer/skills/repository-skill-forge/scripts/issue-state.py

@@ -12,7 +12,7 @@
 from pathlib import Path
 from typing import Any
 
-from forge_common import parse_timestamp, read_json, write_json
+from forge_common import parse_timestamp, read_json, stable_hash, write_json
 
 MARKER = "repository-skill-forge-state"
 BLOCK_RE = re.compile(
@@ -45,9 +45,11 @@
     "cursor",
     "updatedAt",
     "observations",
+    "fingerprintCatalog",
     "proposalQueue",
     "proposalHistory",
 }
+LEGACY_TOP_LEVEL = ALLOWED_TOP_LEVEL - {"fingerprintCatalog"}
 OBSERVATION_KEYS = {
     "evidenceKey",
     "fingerprint",
@@ -111,8 +113,16 @@
 FORBIDDEN_VALUE_RE = re.compile(
     r"(?:/(?:Users|home)/[^/\s]+|[A-Za-z]:\\Users\\|"
     r"\b(?:gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})\b|"
-    r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----)"
+    r"\bAKIA[0-9A-Z]{16}\b|"
+    r"\bBearer\s+[A-Za-z0-9._~+/=-]{20,}|"
+    r"\b(?:password|passwd|token|secret|api[_-]?key)\s*[:=]\s*"
+    r"['\"]?(?!<|\$\{|\$[A-Z_]+)[^\s'\"]{12,}|"
+    r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----)",
+    re.IGNORECASE,
 )
+MAX_FINGERPRINT_CATALOG_ENTRIES = 64
+MAX_FINGERPRINT_CATALOG_BYTES = 12_000
+MAX_SIGNATURE_BYTES = 2_000
 
 
 def validate_timestamp(value: Any, field: str) -> None:
@@ -196,6 +206,93 @@ def validate_observations(observations: list[Any]) -> None:
                 raise ValueError("issue state observation contains an invalid ref")
 
 
+def validate_fingerprint_catalog(catalog: Any) -> None:
+    if not isinstance(catalog, dict):
+        raise ValueError("issue state fingerprintCatalog must be an object")
+    if len(catalog) > MAX_FINGERPRINT_CATALOG_ENTRIES:
+        raise ValueError("issue state fingerprintCatalog exceeds its entry limit")
+    serialized_bytes = len(
+        json.dumps(catalog, sort_keys=True, separators=(",", ":")).encode()
+    )
+    if serialized_bytes > MAX_FINGERPRINT_CATALOG_BYTES:
+        raise ValueError("issue state fingerprintCatalog exceeds its size limit")
+    for fingerprint, entry in catalog.items():
+        if not isinstance(fingerprint, str) or not re.fullmatch(
+            r"[0-9a-f]{16}", fingerprint
+        ):
+            raise ValueError("issue state fingerprintCatalog contains an invalid key")
+        if not isinstance(entry, dict) or set(entry) != {
+            "kind",
+            "signatureVersion",
+            "signature",
+            "lastSeenAt",
+        }:
+            raise ValueError(
+                "issue state fingerprintCatalog contains an unsupported entry"
+            )
+        kind = entry.get("kind")
+        signature = entry.get("signature")
+        if kind not in {"command", "script"}:
+            raise ValueError("issue state fingerprintCatalog contains an invalid kind")
+        if (
+            isinstance(entry.get("signatureVersion"), bool)
+            or entry.get("signatureVersion") != 1
+        ):
+            raise ValueError(
+                "issue state fingerprintCatalog contains an unsupported signature version"
+            )
+        if not isinstance(signature, dict) or not signature:
+            raise ValueError(
+                "issue state fingerprintCatalog requires a nonempty signature"
+            )
+        signature_bytes = len(
+            json.dumps(signature, sort_keys=True, separators=(",", ":")).encode()
+        )
+        if signature_bytes > MAX_SIGNATURE_BYTES:
+            raise ValueError("issue state fingerprintCatalog signature is too large")
+        if kind == "command":
+            if set(signature) != {"tokens"}:
+                raise ValueError(
+                    "issue state command fingerprint requires a token signature"
+                )
+            tokens = signature.get("tokens")
+            if (
+                not isinstance(tokens, list)
+                or not tokens
+                or len(tokens) > 40
+                or any(
+                    not isinstance(token, str) or not token or len(token) > 80
+                    for token in tokens
+                )
+            ):
+                raise ValueError(
+                    "issue state command fingerprint has invalid tokens"
+                )
+        else:
+            if set(signature) != {"imports", "calls", "fileExtensions"}:
+                raise ValueError(
+                    "issue state script fingerprint has an invalid signature"
+                )
+            for key in ("imports", "calls", "fileExtensions"):
+                values = signature.get(key)
+                if (
+                    not isinstance(values, list)
+                    or len(values) > 64
+                    or any(
+                        not isinstance(value, str) or not value or len(value) > 200
+                        for value in values
+                    )
+                ):
+                    raise ValueError(
+                        f"issue state script fingerprint has invalid {key}"
+                    )
+        if stable_hash({"kind": kind, "signature": signature}) != fingerprint:
+            raise ValueError(
+                "issue state fingerprintCatalog signature does not match its key"
+            )
+        validate_timestamp(entry.get("lastSeenAt"), "fingerprint lastSeenAt")
+
+
 def validate_reconciliation(value: Any) -> None:
     if value is None:
         return
@@ -335,8 +432,13 @@ def validate_sanitized_value(value: Any, key: str | None = None) -> None:
 def validate_state(state: Any, repository: str) -> dict[str, Any]:
     if not isinstance(state, dict):
         raise ValueError("issue state must be a JSON object")
-    if set(state) != ALLOWED_TOP_LEVEL:
+    if frozenset(state) not in {
+        frozenset(ALLOWED_TOP_LEVEL),
+        frozenset(LEGACY_TOP_LEVEL),
+    }:
         raise ValueError("issue state has an unsupported top-level shape")
+    state = dict(state)
+    state.setdefault("fingerprintCatalog", {})
     if state.get("schemaVersion") != 2:
         raise ValueError("unsupported issue state schema version")
     if not isinstance(state.get("stateVersion"), int) or state["stateVersion"] < 1:
@@ -354,6 +456,7 @@ def validate_state(state: Any, repository: str) -> dict[str, Any]:
     if not isinstance(state.get("proposalHistory"), dict):
         raise ValueError("issue state proposalHistory must be an object")
     validate_observations(state["observations"])
+    validate_fingerprint_catalog(state["fingerprintCatalog"])
     validate_proposal_queue(state["proposalQueue"])
     validate_proposal_history(state["proposalHistory"])
     validate_sanitized_value(state)