diff --git a/python/scripts/reconciliation_record_contract.py b/python/scripts/reconciliation_record_contract.py new file mode 100644 index 0000000..21a4720 --- /dev/null +++ b/python/scripts/reconciliation_record_contract.py @@ -0,0 +1,566 @@ +#!/usr/bin/env python3 +"""Validate the contract-only QSL ReconciliationRecord v1 consumer contract.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import re +import sys +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Mapping + +from activation_contract import ActivationValidationError, parse_activation_json, validate_activation +from deployment_bundle_contract import BundleValidationError, parse_bundle_json, validate_bundle + +SCHEMA_ID = "qsl.reconciliation_record.v1" +BUNDLE_SCHEMA_ID = "qsl.deployment_bundle.v1" +ACTIVATION_SCHEMA_ID = "qsl.activation.v1" +OBSERVER_SCHEMA_ID = "qsl.reconciliation_observer_receipt.v1" +RECONCILIATION_STATUSES = ("MISSING", "MATCHED", "MISMATCHED") +COMPARISON_FIELDS = ( + "deployment_bundle_sha256", + "activation_id", + "activation_sha256", + "platform", + "repository", + "revision", + "environment", + "account_alias", + "account_digest_sha256", +) + +_IDENTITY_PATTERN = re.compile(r"^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*$") +_REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$") +_SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") +_TIMESTAMP_PATTERN = re.compile(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$") +_REPOSITORY_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*$") +_FORBIDDEN_KEY_PATTERN = re.compile( + r"credential|secret|token|password|cookie|jwt|private(?:[_-]?key)?|api[_-]?key|access[_-]?key|" + r"provider[_-]?rows?|raw[_-]?provider|account[_-]?(?:number|id|balance)|balance|positions?|orders?|" + r"fills?|capital(?:[_-]?(?:amount|balance|value))?", + re.IGNORECASE, +) +_ALLOWED_ASSERTION_KEYS = {"fills_verified", "capital_use_verified"} +_URL_PATTERN = re.compile(r"[a-z][a-z0-9+.-]*://", re.IGNORECASE) +_JWT_PATTERN = re.compile(r"(?:^|\s)[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}(?:$|\s)") +_BEARER_PATTERN = re.compile(r"(?:^|\s)bearer\s+\S+", re.IGNORECASE) + +_COMMON_FIELDS = { + "schema", + "reconciliation_id", + "created_at", + "expires_at", + "digest_algorithm", + "contract_only", + "deployment_bundle", + "activation", + "target", + "expected_identity", + "status", + "assertions", + "reconciliation_sha256", +} +_OBSERVATION_FIELDS = {"observed_identity", "observer_receipt", "comparison"} +_BUNDLE_REFERENCE_FIELDS = {"schema", "bundle_id", "bundle_sha256"} +_ACTIVATION_REFERENCE_FIELDS = {"schema", "activation_id", "activation_sha256"} +_TARGET_FIELDS = { + "platform", + "repository", + "revision", + "environment", + "account_alias", + "account_digest_sha256", +} +_EXPECTED_IDENTITY_FIELDS = set(COMPARISON_FIELDS) | {"expected_identity_sha256"} +_OBSERVED_IDENTITY_FIELDS = set(COMPARISON_FIELDS) | { + "producer_id", + "producer_revision", + "artifact_sha256", + "observed_at", + "observed_identity_sha256", +} +_OBSERVER_RECEIPT_FIELDS = { + "schema", + "observer_id", + "observer_revision", + "created_at", + "observed_identity", + "observer_receipt_sha256", +} +_ASSERTION_FIELDS = { + "apply_performed", + "config_sync_performed", + "runtime_mutation_performed", + "runtime_active", + "fills_verified", + "capital_use_verified", +} + + +class ReconciliationValidationError(ValueError): + """Raised when an input is not a valid consumer-side reconciliation record.""" + + +def _fail(message: str) -> None: + raise ReconciliationValidationError(message) + + +def _reject_non_finite_or_null(value: Any, path: str = "reconciliation") -> None: + if value is None: + _fail(f"{path} must not be null") + if isinstance(value, float) and not math.isfinite(value): + _fail(f"{path} contains a non-finite number") + if isinstance(value, Mapping): + for key, child in value.items(): + if not isinstance(key, str): + _fail(f"{path} contains a non-string key") + _reject_non_finite_or_null(child, f"{path}.{key}") + elif isinstance(value, list): + for index, child in enumerate(value): + _reject_non_finite_or_null(child, f"{path}[{index}]") + + +def _reject_forbidden_material(value: Any, path: str = "reconciliation") -> None: + if isinstance(value, Mapping): + for key, child in value.items(): + if key not in _ALLOWED_ASSERTION_KEYS and _FORBIDDEN_KEY_PATTERN.search(key): + _fail(f"{path}.{key} is forbidden in a reconciliation contract") + _reject_forbidden_material(child, f"{path}.{key}") + elif isinstance(value, list): + for index, child in enumerate(value): + _reject_forbidden_material(child, f"{path}[{index}]") + elif isinstance(value, str): + if _URL_PATTERN.search(value): + _fail(f"{path} contains a forbidden credential-capable URL") + if _JWT_PATTERN.search(value) or _BEARER_PATTERN.search(value): + _fail(f"{path} contains forbidden credential material") + + +def _expect_object(value: Any, path: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + _fail(f"{path} must be an object") + return value + + +def _expect_exact_keys(value: Mapping[str, Any], expected: set[str], path: str) -> None: + missing = sorted(expected - set(value)) + unknown = sorted(set(value) - expected) + if missing: + _fail(f"{path} missing required field(s): {', '.join(missing)}") + if unknown: + _fail(f"{path} has unknown field(s): {', '.join(unknown)}") + + +def _expect_identity(value: Any, path: str, *, allow_numeric_only: bool = True) -> str: + if not isinstance(value, str) or not _IDENTITY_PATTERN.fullmatch(value): + _fail(f"{path} must be a lowercase immutable identity") + if not allow_numeric_only and value.isdigit(): + _fail(f"{path} must be a non-sensitive alias, not a numeric account identifier") + return value + + +def _expect_revision(value: Any, path: str) -> str: + if not isinstance(value, str) or not _REVISION_PATTERN.fullmatch(value): + _fail(f"{path} must be a lowercase 40-character revision") + return value + + +def _expect_sha256(value: Any, path: str) -> str: + if not isinstance(value, str) or not _SHA256_PATTERN.fullmatch(value): + _fail(f"{path} must be a lowercase SHA-256 digest") + return value + + +def _expect_repository(value: Any, path: str) -> str: + if not isinstance(value, str) or not _REPOSITORY_PATTERN.fullmatch(value): + _fail(f"{path} must be an owner/repository identity, not a URL") + return value + + +def _parse_timestamp(value: Any, path: str) -> datetime: + if not isinstance(value, str) or not _TIMESTAMP_PATTERN.fullmatch(value): + _fail(f"{path} must be an RFC3339 UTC timestamp with whole seconds") + try: + parsed = datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ") + except ValueError as exc: + raise ReconciliationValidationError(f"{path} must be a valid calendar timestamp") from exc + return parsed.replace(tzinfo=UTC) + + +def _canonical_without(value: Mapping[str, Any], excluded_field: str, path: str) -> str: + if not isinstance(value, Mapping): + _fail(f"{path} must be an object") + content = dict(value) + content.pop(excluded_field, None) + try: + return json.dumps(content, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False) + except (TypeError, ValueError) as exc: + raise ReconciliationValidationError(f"{path} cannot be represented as canonical JSON") from exc + + +def canonical_json(record: Mapping[str, Any]) -> str: + """Return deterministic JSON with only the record's self hash omitted.""" + return _canonical_without(record, "reconciliation_sha256", "reconciliation") + + +def calculate_reconciliation_sha256(record: Mapping[str, Any]) -> str: + return hashlib.sha256(canonical_json(record).encode("utf-8")).hexdigest() + + +def calculate_expected_identity_sha256(identity: Mapping[str, Any]) -> str: + canonical = _canonical_without(identity, "expected_identity_sha256", "expected_identity") + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def calculate_observed_identity_sha256(identity: Mapping[str, Any]) -> str: + canonical = _canonical_without(identity, "observed_identity_sha256", "observed_identity") + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def calculate_observer_receipt_sha256(receipt: Mapping[str, Any]) -> str: + canonical = _canonical_without(receipt, "observer_receipt_sha256", "observer_receipt") + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def _validate_target(value: Any, path: str = "target") -> Mapping[str, Any]: + target = _expect_object(value, path) + _expect_exact_keys(target, _TARGET_FIELDS, path) + _expect_identity(target["platform"], f"{path}.platform") + _expect_repository(target["repository"], f"{path}.repository") + _expect_revision(target["revision"], f"{path}.revision") + _expect_identity(target["environment"], f"{path}.environment") + _expect_identity(target["account_alias"], f"{path}.account_alias", allow_numeric_only=False) + _expect_sha256(target["account_digest_sha256"], f"{path}.account_digest_sha256") + return target + + +def _expected_identity(bundle: Mapping[str, Any], activation: Mapping[str, Any]) -> dict[str, Any]: + target = activation["target"] + identity = { + "deployment_bundle_sha256": bundle["bundle_sha256"], + "activation_id": activation["activation_id"], + "activation_sha256": activation["activation_sha256"], + "platform": target["platform"], + "repository": target["repository"], + "revision": target["revision"], + "environment": target["environment"], + "account_alias": target["account_alias"], + "account_digest_sha256": target["account_digest_sha256"], + } + identity["expected_identity_sha256"] = calculate_expected_identity_sha256(identity) + return identity + + +def _validate_expected_identity(value: Any) -> Mapping[str, Any]: + identity = _expect_object(value, "expected_identity") + _expect_exact_keys(identity, _EXPECTED_IDENTITY_FIELDS, "expected_identity") + _expect_sha256(identity["deployment_bundle_sha256"], "expected_identity.deployment_bundle_sha256") + _expect_identity(identity["activation_id"], "expected_identity.activation_id") + _expect_sha256(identity["activation_sha256"], "expected_identity.activation_sha256") + _validate_target({field: identity[field] for field in _TARGET_FIELDS}, "expected_identity") + _expect_sha256(identity["expected_identity_sha256"], "expected_identity.expected_identity_sha256") + if identity["expected_identity_sha256"] != calculate_expected_identity_sha256(identity): + _fail("expected_identity_sha256 mismatch") + return identity + + +def _validate_observed_identity(value: Any) -> tuple[Mapping[str, Any], datetime]: + identity = _expect_object(value, "observed_identity") + _expect_exact_keys(identity, _OBSERVED_IDENTITY_FIELDS, "observed_identity") + _expect_sha256(identity["deployment_bundle_sha256"], "observed_identity.deployment_bundle_sha256") + _expect_identity(identity["activation_id"], "observed_identity.activation_id") + _expect_sha256(identity["activation_sha256"], "observed_identity.activation_sha256") + _validate_target({field: identity[field] for field in _TARGET_FIELDS}, "observed_identity") + _expect_identity(identity["producer_id"], "observed_identity.producer_id") + _expect_revision(identity["producer_revision"], "observed_identity.producer_revision") + _expect_sha256(identity["artifact_sha256"], "observed_identity.artifact_sha256") + observed_at = _parse_timestamp(identity["observed_at"], "observed_identity.observed_at") + _expect_sha256(identity["observed_identity_sha256"], "observed_identity.observed_identity_sha256") + if identity["observed_identity_sha256"] != calculate_observed_identity_sha256(identity): + _fail("observed_identity_sha256 mismatch") + return identity, observed_at + + +def _validate_observer_receipt(value: Any) -> tuple[Mapping[str, Any], Mapping[str, Any], datetime]: + receipt = _expect_object(value, "observer_receipt") + _expect_exact_keys(receipt, _OBSERVER_RECEIPT_FIELDS, "observer_receipt") + if receipt["schema"] != OBSERVER_SCHEMA_ID: + _fail(f"observer_receipt.schema must be {OBSERVER_SCHEMA_ID}") + _expect_identity(receipt["observer_id"], "observer_receipt.observer_id") + _expect_revision(receipt["observer_revision"], "observer_receipt.observer_revision") + created_at = _parse_timestamp(receipt["created_at"], "observer_receipt.created_at") + observed, _ = _validate_observed_identity(receipt["observed_identity"]) + if receipt["observer_id"] == observed["producer_id"]: + _fail("observer receipt must be produced by an identity separate from the platform producer") + _expect_sha256(receipt["observer_receipt_sha256"], "observer_receipt.observer_receipt_sha256") + if receipt["observer_receipt_sha256"] != calculate_observer_receipt_sha256(receipt): + _fail("observer_receipt_sha256 mismatch") + return receipt, observed, created_at + + +def _validate_comparison( + value: Any, + expected: Mapping[str, Any], + observed: Mapping[str, Any], +) -> list[str]: + comparison = _expect_object(value, "comparison") + _expect_exact_keys(comparison, {"complete", "fields"}, "comparison") + if comparison["complete"] is not True: + _fail("comparison.complete must be true for an observed reconciliation") + fields = _expect_object(comparison["fields"], "comparison.fields") + _expect_exact_keys(fields, set(COMPARISON_FIELDS), "comparison.fields") + differences = [] + for field in COMPARISON_FIELDS: + entry = _expect_object(fields[field], f"comparison.fields.{field}") + _expect_exact_keys(entry, {"expected", "observed", "equal"}, f"comparison.fields.{field}") + if entry["expected"] != expected[field] or entry["observed"] != observed[field]: + _fail(f"comparison.fields.{field} does not bind the exact expected and observed values") + equality = expected[field] == observed[field] + if entry["equal"] is not equality: + _fail(f"comparison.fields.{field}.equal is inconsistent") + if not equality: + differences.append(field) + return differences + + +def _validate_references( + root: Mapping[str, Any], + bundle: Mapping[str, Any], + activation: Mapping[str, Any], +) -> Mapping[str, Any]: + bundle_reference = _expect_object(root["deployment_bundle"], "deployment_bundle") + _expect_exact_keys(bundle_reference, _BUNDLE_REFERENCE_FIELDS, "deployment_bundle") + expected_bundle_reference = { + "schema": BUNDLE_SCHEMA_ID, + "bundle_id": bundle["bundle_id"], + "bundle_sha256": bundle["bundle_sha256"], + } + if bundle_reference != expected_bundle_reference: + _fail("deployment bundle reference does not match the exact expected bundle identity") + + activation_reference = _expect_object(root["activation"], "activation") + _expect_exact_keys(activation_reference, _ACTIVATION_REFERENCE_FIELDS, "activation") + expected_activation_reference = { + "schema": ACTIVATION_SCHEMA_ID, + "activation_id": activation["activation_id"], + "activation_sha256": activation["activation_sha256"], + } + if activation_reference != expected_activation_reference: + _fail("activation reference does not match the exact expected activation identity") + + target = _validate_target(root["target"]) + if target != activation["target"]: + _fail("target does not match the exact activation target") + expected_identity = _validate_expected_identity(root["expected_identity"]) + if expected_identity != _expected_identity(bundle, activation): + _fail("expected_identity does not match the exact bundle, activation, and target identity") + return expected_identity + + +def _validate_assertions(value: Any) -> None: + assertions = _expect_object(value, "assertions") + _expect_exact_keys(assertions, _ASSERTION_FIELDS, "assertions") + for field in _ASSERTION_FIELDS: + if assertions[field] is not False: + _fail(f"assertions.{field} must be false for a contract-only record") + + +def validate_reconciliation_record( + record: Any, + *, + expected_bundle: Any, + expected_activation: Any, + as_of: str | None = None, +) -> Mapping[str, Any]: + """Validate exact expected identity and optional independently observed truth, fail closed.""" + _reject_non_finite_or_null(record) + _reject_forbidden_material(record) + root = _expect_object(record, "reconciliation") + status = root.get("status") + if status not in RECONCILIATION_STATUSES: + _fail("status must be one of MISSING, MATCHED, MISMATCHED") + expected_fields = _COMMON_FIELDS if status == "MISSING" else _COMMON_FIELDS | _OBSERVATION_FIELDS + _expect_exact_keys(root, expected_fields, "reconciliation") + if root["schema"] != SCHEMA_ID: + _fail(f"schema must be {SCHEMA_ID}") + _expect_identity(root["reconciliation_id"], "reconciliation_id") + created_at = _parse_timestamp(root["created_at"], "created_at") + expires_at = _parse_timestamp(root["expires_at"], "expires_at") + if expires_at <= created_at: + _fail("expires_at must be after created_at") + if root["digest_algorithm"] != "sha256": + _fail("digest_algorithm must be sha256") + if root["contract_only"] is not True: + _fail("contract_only must be true") + _expect_sha256(root["reconciliation_sha256"], "reconciliation_sha256") + if root["reconciliation_sha256"] != calculate_reconciliation_sha256(root): + _fail("reconciliation_sha256 mismatch") + _validate_assertions(root["assertions"]) + + if expected_bundle is None or expected_activation is None: + _fail("exact expected bundle and activation inputs are required") + try: + bundle = validate_bundle(expected_bundle) + activation = validate_activation( + expected_activation, + as_of=as_of, + expected_bundle=bundle, + ) + except (BundleValidationError, ActivationValidationError) as exc: + raise ReconciliationValidationError(f"expected bundle or activation is invalid: {exc}") from exc + expected = _validate_references(root, bundle, activation) + + validation_time = datetime.now(UTC).replace(microsecond=0) if as_of is None else _parse_timestamp(as_of, "as_of") + if created_at > validation_time: + _fail("reconciliation record was created in the future") + if validation_time >= expires_at: + _fail("reconciliation record is stale or expired") + activation_effective = _parse_timestamp(activation["effective_at"], "expected_activation.effective_at") + activation_expires = _parse_timestamp(activation["expires_at"], "expected_activation.expires_at") + if created_at < activation_effective or expires_at > activation_expires: + _fail("reconciliation validity window must remain within the activation window") + + if status == "MISSING": + return root + + observed, observed_at = _validate_observed_identity(root["observed_identity"]) + _, receipt_observed, receipt_created_at = _validate_observer_receipt(root["observer_receipt"]) + if receipt_observed != observed: + _fail("observer_receipt does not bind the exact platform-produced observed identity") + if observed_at < activation_effective or observed_at > created_at: + _fail("observed identity is stale or was produced after the reconciliation record") + if receipt_created_at < observed_at or receipt_created_at > created_at: + _fail("observer receipt time must be between observation and reconciliation creation") + differences = _validate_comparison(root["comparison"], expected, observed) + if status == "MATCHED" and differences: + _fail("MATCHED requires every expected and observed identity field to be exactly equal") + if status == "MISMATCHED" and not differences: + _fail("MISMATCHED requires at least one explicit required-field difference") + return root + + +def build_missing_record( + *, + record_id: str, + created_at: str, + expires_at: str, + expected_bundle: Any, + expected_activation: Any, + as_of: str | None = None, +) -> dict[str, Any]: + """Build only the safe default MISSING record; never synthesize observed platform truth.""" + try: + bundle = validate_bundle(expected_bundle) + activation = validate_activation(expected_activation, as_of=as_of, expected_bundle=bundle) + except (BundleValidationError, ActivationValidationError) as exc: + raise ReconciliationValidationError(f"expected bundle or activation is invalid: {exc}") from exc + record = { + "schema": SCHEMA_ID, + "reconciliation_id": record_id, + "created_at": created_at, + "expires_at": expires_at, + "digest_algorithm": "sha256", + "contract_only": True, + "deployment_bundle": { + "schema": BUNDLE_SCHEMA_ID, + "bundle_id": bundle["bundle_id"], + "bundle_sha256": bundle["bundle_sha256"], + }, + "activation": { + "schema": ACTIVATION_SCHEMA_ID, + "activation_id": activation["activation_id"], + "activation_sha256": activation["activation_sha256"], + }, + "target": dict(activation["target"]), + "expected_identity": _expected_identity(bundle, activation), + "status": "MISSING", + "assertions": {field: False for field in sorted(_ASSERTION_FIELDS)}, + } + record["reconciliation_sha256"] = calculate_reconciliation_sha256(record) + validate_reconciliation_record( + record, + expected_bundle=bundle, + expected_activation=activation, + as_of=as_of, + ) + return record + + +def _reject_duplicate_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + _fail(f"duplicate JSON key: {key}") + result[key] = value + return result + + +def _load_json(text: str) -> Any: + try: + return json.loads( + text, + object_pairs_hook=_reject_duplicate_pairs, + parse_constant=lambda _: _fail("non-finite JSON value"), + ) + except json.JSONDecodeError as exc: + raise ReconciliationValidationError("invalid JSON") from exc + + +def parse_reconciliation_json( + text: str, + *, + expected_bundle: Any, + expected_activation: Any, + as_of: str | None = None, +) -> Mapping[str, Any]: + return validate_reconciliation_record( + _load_json(text), + expected_bundle=expected_bundle, + expected_activation=expected_activation, + as_of=as_of, + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input", type=Path, required=True, help="contract-only ReconciliationRecord JSON") + parser.add_argument("--bundle", type=Path, required=True, help="exact DeploymentBundle JSON") + parser.add_argument("--activation", type=Path, required=True, help="exact Activation JSON") + parser.add_argument("--as-of", help="inject canonical UTC validation time; defaults to current UTC") + args = parser.parse_args(argv) + try: + bundle = parse_bundle_json(args.bundle.read_text(encoding="utf-8")) + activation = parse_activation_json( + args.activation.read_text(encoding="utf-8"), + as_of=args.as_of, + expected_bundle=bundle, + ) + record = parse_reconciliation_json( + args.input.read_text(encoding="utf-8"), + expected_bundle=bundle, + expected_activation=activation, + as_of=args.as_of, + ) + except (OSError, ReconciliationValidationError, ActivationValidationError, BundleValidationError) as exc: + print(f"reconciliation validation failed: {exc}", file=sys.stderr) + return 1 + print( + json.dumps( + { + "contract_only": True, + "reconciliation_sha256": record["reconciliation_sha256"], + "schema": record["schema"], + "status": record["status"], + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/tests/test_reconciliation_record_contract.py b/python/tests/test_reconciliation_record_contract.py new file mode 100644 index 0000000..01d97be --- /dev/null +++ b/python/tests/test_reconciliation_record_contract.py @@ -0,0 +1,378 @@ +from __future__ import annotations + +import copy +import importlib.util +import json +import math +import sys +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" + + +def _load_module(name: str): + spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py") + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +deployment_bundle_contract = _load_module("deployment_bundle_contract") +activation_contract = _load_module("activation_contract") +reconciliation_record_contract = _load_module("reconciliation_record_contract") + + +class ReconciliationRecordContractTest(unittest.TestCase): + @staticmethod + def _sha(character: str) -> str: + return character * 64 + + @staticmethod + def _revision(character: str) -> str: + return character * 40 + + def _bundle(self) -> dict[str, object]: + bundle: dict[str, object] = { + "schema": "qsl.deployment_bundle.v1", + "bundle_id": "bundle.soxl-signal.ibkr-us.20260805", + "created_at": "2026-08-05T08:00:00Z", + "digest_algorithm": "sha256", + "strategy": { + "id": "soxl-signal", + "source_id": "us-equity-strategies", + "revision": self._revision("a"), + "artifact_sha256": self._sha("b"), + }, + "profile": { + "id": "research-profile", + "revision": self._revision("c"), + "artifact_sha256": self._sha("d"), + }, + "config": { + "id": "ibkr-us-config", + "revision": self._revision("e"), + "artifact_sha256": self._sha("f"), + }, + "evidence": { + "id": "soxl-evidence", + "revision": self._revision("1"), + "artifact_sha256": self._sha("2"), + }, + "target": {"id": "ibkr-us", "platform_id": "interactive-brokers"}, + "dependencies": { + "qpk": { + "id": "quant-platform-kit", + "revision": self._revision("3"), + "artifact_sha256": self._sha("4"), + }, + "strategy": { + "id": "us-equity-strategies", + "revision": self._revision("a"), + "artifact_sha256": self._sha("5"), + }, + "pipeline": { + "id": "crypto-live-pool-pipelines", + "revision": self._revision("6"), + "artifact_sha256": self._sha("7"), + }, + "platform": { + "id": "interactive-brokers", + "revision": self._revision("8"), + "artifact_sha256": self._sha("9"), + }, + }, + } + bundle["bundle_sha256"] = deployment_bundle_contract.calculate_bundle_sha256(bundle) + return bundle + + def _activation(self) -> dict[str, object]: + bundle = self._bundle() + activation: dict[str, object] = { + "schema": "qsl.activation.v1", + "activation_id": "activation.soxl-signal.ibkr-us.paper.20260805", + "created_at": "2026-08-05T09:00:00Z", + "digest_algorithm": "sha256", + "contract_only": True, + "deployment_bundle": { + "schema": bundle["schema"], + "bundle_id": bundle["bundle_id"], + "bundle_sha256": bundle["bundle_sha256"], + }, + "stage": "PAPER_DRY_RUN", + "effective_at": "2026-08-05T10:00:00Z", + "expires_at": "2026-08-05T18:00:00Z", + "human_authority": { + "stage": "PAPER_DRY_RUN", + "authority_id": "human-authority.paper-dry-run.20260805", + "authority_version": "v1", + "authority_receipt_sha256": self._sha("c"), + }, + "target": { + "platform": "interactive-brokers", + "repository": "QuantStrategyLab/InteractiveBrokersPlatform", + "revision": self._revision("8"), + "environment": "ibkr-paper", + "account_alias": "ibkr-research", + "account_digest_sha256": self._sha("d"), + }, + } + activation["activation_sha256"] = activation_contract.calculate_activation_sha256(activation) + return activation + + def _missing(self) -> dict[str, object]: + return reconciliation_record_contract.build_missing_record( + record_id="reconciliation.soxl-signal.ibkr-us.20260805", + created_at="2026-08-05T12:00:00Z", + expires_at="2026-08-05T13:00:00Z", + expected_bundle=self._bundle(), + expected_activation=self._activation(), + as_of="2026-08-05T12:00:00Z", + ) + + def _observed(self, expected: dict[str, object]) -> dict[str, object]: + observed = { + **{key: value for key, value in expected.items() if key != "expected_identity_sha256"}, + "producer_id": "interactive-brokers.runtime-report", + "producer_revision": self._revision("8"), + "artifact_sha256": self._sha("e"), + "observed_at": "2026-08-05T11:55:00Z", + } + observed["observed_identity_sha256"] = ( + reconciliation_record_contract.calculate_observed_identity_sha256(observed) + ) + return observed + + def _observer_receipt(self, observed: dict[str, object]) -> dict[str, object]: + receipt = { + "schema": "qsl.reconciliation_observer_receipt.v1", + "observer_id": "qsl-independent-observer", + "observer_revision": self._revision("f"), + "created_at": "2026-08-05T11:58:00Z", + "observed_identity": copy.deepcopy(observed), + } + receipt["observer_receipt_sha256"] = ( + reconciliation_record_contract.calculate_observer_receipt_sha256(receipt) + ) + return receipt + + def _comparison(self, expected: dict[str, object], observed: dict[str, object]) -> dict[str, object]: + fields = {} + for field in reconciliation_record_contract.COMPARISON_FIELDS: + fields[field] = { + "expected": expected[field], + "observed": observed[field], + "equal": expected[field] == observed[field], + } + return {"complete": True, "fields": fields} + + def _record_with_observation(self, status: str = "MATCHED") -> dict[str, object]: + record = self._missing() + expected = record["expected_identity"] + observed = self._observed(expected) + if status == "MISMATCHED": + observed["revision"] = self._revision("0") + observed["observed_identity_sha256"] = ( + reconciliation_record_contract.calculate_observed_identity_sha256(observed) + ) + record["status"] = status + record["observed_identity"] = observed + record["observer_receipt"] = self._observer_receipt(observed) + record["comparison"] = self._comparison(expected, observed) + record["reconciliation_sha256"] = reconciliation_record_contract.calculate_reconciliation_sha256(record) + return record + + def _validate(self, record: dict[str, object], *, as_of: str = "2026-08-05T12:00:00Z"): + return reconciliation_record_contract.validate_reconciliation_record( + record, + expected_bundle=self._bundle(), + expected_activation=self._activation(), + as_of=as_of, + ) + + def test_schema_is_closed_contract_only_and_uses_canonical_statuses(self): + schema = json.loads((ROOT.parent / "schemas" / "qsl-reconciliation-record.v1.schema.json").read_text()) + self.assertEqual(schema["$id"], "qsl.reconciliation_record.v1") + self.assertFalse(schema["additionalProperties"]) + self.assertEqual(schema["properties"]["contract_only"], {"const": True}) + self.assertEqual(schema["properties"]["status"]["enum"], ["MISSING", "MATCHED", "MISMATCHED"]) + self.assertIn("does not perform apply", schema["description"]) + + def test_valid_missing_is_default_deterministic_and_binds_exact_inputs(self): + record = self._missing() + validated = self._validate(record) + self.assertEqual(validated["status"], "MISSING") + self.assertNotIn("observed_identity", record) + self.assertEqual( + reconciliation_record_contract.canonical_json(record), + reconciliation_record_contract.canonical_json(dict(reversed(record.items()))), + ) + self.assertEqual( + record["reconciliation_sha256"], + reconciliation_record_contract.calculate_reconciliation_sha256(record), + ) + + def test_missing_rejects_fake_observed_or_matched_claims(self): + for field, value in ( + ("observed_identity", self._observed(self._missing()["expected_identity"])), + ("comparison", {"complete": False}), + ("runtime_active", True), + ): + with self.subTest(field=field): + record = self._missing() + record[field] = value + record["reconciliation_sha256"] = reconciliation_record_contract.calculate_reconciliation_sha256(record) + with self.assertRaises(reconciliation_record_contract.ReconciliationValidationError): + self._validate(record) + + def test_matched_requires_separate_immutable_observer_receipt(self): + record = self._record_with_observation() + record.pop("observer_receipt") + record["reconciliation_sha256"] = reconciliation_record_contract.calculate_reconciliation_sha256(record) + with self.assertRaisesRegex(reconciliation_record_contract.ReconciliationValidationError, "observer_receipt"): + self._validate(record) + + def test_same_identity_matched_fixture_validates_without_creating_runtime_truth(self): + record = self._record_with_observation() + validated = self._validate(record) + self.assertEqual(validated["status"], "MATCHED") + self.assertFalse(validated["assertions"]["runtime_active"]) + self.assertFalse(validated["assertions"]["fills_verified"]) + self.assertFalse(validated["assertions"]["capital_use_verified"]) + + def test_mismatched_requires_at_least_one_explicit_difference(self): + record = self._record_with_observation("MISMATCHED") + self._validate(record) + matched = self._record_with_observation() + matched["status"] = "MISMATCHED" + matched["reconciliation_sha256"] = reconciliation_record_contract.calculate_reconciliation_sha256(matched) + with self.assertRaisesRegex(reconciliation_record_contract.ReconciliationValidationError, "at least one"): + self._validate(matched) + + def test_required_mismatch_cannot_be_labeled_matched(self): + record = self._record_with_observation() + record["observed_identity"]["revision"] = self._revision("0") + record["observed_identity"]["observed_identity_sha256"] = ( + reconciliation_record_contract.calculate_observed_identity_sha256(record["observed_identity"]) + ) + record["observer_receipt"] = self._observer_receipt(record["observed_identity"]) + record["comparison"] = self._comparison(record["expected_identity"], record["observed_identity"]) + record["reconciliation_sha256"] = reconciliation_record_contract.calculate_reconciliation_sha256(record) + with self.assertRaisesRegex(reconciliation_record_contract.ReconciliationValidationError, "MATCHED"): + self._validate(record) + + def test_stale_expired_future_and_invalid_calendar_times_fail_closed(self): + with self.assertRaisesRegex(reconciliation_record_contract.ReconciliationValidationError, "expired"): + self._validate(self._missing(), as_of="2026-08-05T13:00:00Z") + + future = self._missing() + future["created_at"] = "2026-08-05T12:01:00Z" + future["reconciliation_sha256"] = reconciliation_record_contract.calculate_reconciliation_sha256(future) + with self.assertRaisesRegex(reconciliation_record_contract.ReconciliationValidationError, "future"): + self._validate(future) + + stale_observation = self._record_with_observation() + stale_observation["observed_identity"]["observed_at"] = "2026-08-05T09:59:59Z" + stale_observation["observed_identity"]["observed_identity_sha256"] = ( + reconciliation_record_contract.calculate_observed_identity_sha256(stale_observation["observed_identity"]) + ) + stale_observation["observer_receipt"] = self._observer_receipt(stale_observation["observed_identity"]) + stale_observation["reconciliation_sha256"] = ( + reconciliation_record_contract.calculate_reconciliation_sha256(stale_observation) + ) + with self.assertRaisesRegex(reconciliation_record_contract.ReconciliationValidationError, "stale"): + self._validate(stale_observation) + + for field, timestamp in ( + ("created_at", "2026-02-30T12:00:00Z"), + ("expires_at", "2026-08-05T25:00:00Z"), + ): + with self.subTest(field=field): + record = self._missing() + record[field] = timestamp + record["reconciliation_sha256"] = reconciliation_record_contract.calculate_reconciliation_sha256(record) + with self.assertRaisesRegex(reconciliation_record_contract.ReconciliationValidationError, "timestamp"): + self._validate(record) + + def test_cross_bundle_activation_platform_and_observer_splices_fail_closed(self): + mutations = ( + lambda value: value["deployment_bundle"].update({"bundle_sha256": self._sha("0")}), + lambda value: value["activation"].update({"activation_id": "activation.other"}), + lambda value: value["target"].update({"platform": "binance-platform"}), + lambda value: value["observer_receipt"]["observed_identity"].update( + {"artifact_sha256": self._sha("0")} + ), + ) + for mutate in mutations: + record = self._record_with_observation() + mutate(record) + if "observer_receipt" in record: + record["observer_receipt"]["observer_receipt_sha256"] = ( + reconciliation_record_contract.calculate_observer_receipt_sha256(record["observer_receipt"]) + ) + record["reconciliation_sha256"] = reconciliation_record_contract.calculate_reconciliation_sha256(record) + with self.assertRaises(reconciliation_record_contract.ReconciliationValidationError): + self._validate(record) + + def test_mutation_and_nested_digest_mismatch_fail_closed(self): + record = self._missing() + record["target"]["environment"] = "ibkr-shadow" + with self.assertRaisesRegex(reconciliation_record_contract.ReconciliationValidationError, "reconciliation_sha256"): + self._validate(record) + + record = self._record_with_observation() + record["observed_identity"]["artifact_sha256"] = self._sha("0") + record["reconciliation_sha256"] = reconciliation_record_contract.calculate_reconciliation_sha256(record) + with self.assertRaisesRegex(reconciliation_record_contract.ReconciliationValidationError, "observed_identity_sha256"): + self._validate(record) + + def test_secret_provider_and_financial_material_is_rejected_recursively(self): + for key, value in ( + ("api_token", "not-a-real-token"), + ("credential_url", "https://user:pass@example.invalid"), + ("provider_rows", [{"close": 1.0}]), + ("account_number", "00000000"), + ("balance", 100.0), + ("positions", [{"symbol": "SPY"}]), + ("orders", []), + ("fills", []), + ("capital_value", 1.0), + ): + with self.subTest(key=key): + record = self._missing() + record["expected_identity"]["nested"] = {key: value} + record["reconciliation_sha256"] = reconciliation_record_contract.calculate_reconciliation_sha256(record) + with self.assertRaisesRegex(reconciliation_record_contract.ReconciliationValidationError, "forbidden"): + self._validate(record) + + def test_unknown_null_non_finite_duplicate_and_malformed_json_fail_closed(self): + for mutate in ( + lambda value: value.update({"unknown": "x"}), + lambda value: value.update({"status": None}), + ): + record = self._missing() + mutate(record) + record["reconciliation_sha256"] = reconciliation_record_contract.calculate_reconciliation_sha256(record) + with self.assertRaises(reconciliation_record_contract.ReconciliationValidationError): + self._validate(record) + + record = self._missing() + record["assertions"]["runtime_active"] = math.nan + with self.assertRaises(reconciliation_record_contract.ReconciliationValidationError): + self._validate(record) + + duplicate = json.dumps(self._missing()).replace('"status": "MISSING"', '"status": "MISSING", "status": "MATCHED"') + with self.assertRaisesRegex(reconciliation_record_contract.ReconciliationValidationError, "duplicate JSON key"): + reconciliation_record_contract.parse_reconciliation_json( + duplicate, + expected_bundle=self._bundle(), + expected_activation=self._activation(), + as_of="2026-08-05T12:00:00Z", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/schemas/qsl-reconciliation-record.v1.schema.json b/schemas/qsl-reconciliation-record.v1.schema.json new file mode 100644 index 0000000..d9384d7 --- /dev/null +++ b/schemas/qsl-reconciliation-record.v1.schema.json @@ -0,0 +1,227 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "qsl.reconciliation_record.v1", + "title": "QSL ReconciliationRecord v1", + "description": "Contract-only consumer record for exact expected identity and separately platform-produced observed identity. It does not perform apply, config sync, runtime mutation, or prove active runtime, fills, or capital use.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "reconciliation_id", + "created_at", + "expires_at", + "digest_algorithm", + "contract_only", + "deployment_bundle", + "activation", + "target", + "expected_identity", + "status", + "assertions", + "reconciliation_sha256" + ], + "properties": { + "schema": {"const": "qsl.reconciliation_record.v1"}, + "reconciliation_id": {"$ref": "#/$defs/identity"}, + "created_at": {"$ref": "#/$defs/timestamp"}, + "expires_at": {"$ref": "#/$defs/timestamp"}, + "digest_algorithm": {"const": "sha256"}, + "contract_only": {"const": true}, + "deployment_bundle": {"$ref": "#/$defs/deploymentBundleReference"}, + "activation": {"$ref": "#/$defs/activationReference"}, + "target": {"$ref": "#/$defs/target"}, + "expected_identity": {"$ref": "#/$defs/expectedIdentity"}, + "status": {"enum": ["MISSING", "MATCHED", "MISMATCHED"]}, + "observed_identity": {"$ref": "#/$defs/observedIdentity"}, + "observer_receipt": {"$ref": "#/$defs/observerReceipt"}, + "comparison": {"$ref": "#/$defs/comparison"}, + "assertions": {"$ref": "#/$defs/assertions"}, + "reconciliation_sha256": {"$ref": "#/$defs/sha256"} + }, + "oneOf": [ + { + "properties": {"status": {"const": "MISSING"}}, + "not": { + "anyOf": [ + {"required": ["observed_identity"]}, + {"required": ["observer_receipt"]}, + {"required": ["comparison"]} + ] + } + }, + { + "properties": {"status": {"enum": ["MATCHED", "MISMATCHED"]}}, + "required": ["observed_identity", "observer_receipt", "comparison"] + } + ], + "$defs": { + "identity": {"type": "string", "pattern": "^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*$"}, + "revision": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "timestamp": { + "type": "string", + "format": "date-time", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + }, + "repository": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*$" + }, + "deploymentBundleReference": { + "type": "object", + "additionalProperties": false, + "required": ["schema", "bundle_id", "bundle_sha256"], + "properties": { + "schema": {"const": "qsl.deployment_bundle.v1"}, + "bundle_id": {"$ref": "#/$defs/identity"}, + "bundle_sha256": {"$ref": "#/$defs/sha256"} + } + }, + "activationReference": { + "type": "object", + "additionalProperties": false, + "required": ["schema", "activation_id", "activation_sha256"], + "properties": { + "schema": {"const": "qsl.activation.v1"}, + "activation_id": {"$ref": "#/$defs/identity"}, + "activation_sha256": {"$ref": "#/$defs/sha256"} + } + }, + "target": { + "type": "object", + "additionalProperties": false, + "required": ["platform", "repository", "revision", "environment", "account_alias", "account_digest_sha256"], + "properties": { + "platform": {"$ref": "#/$defs/identity"}, + "repository": {"$ref": "#/$defs/repository"}, + "revision": {"$ref": "#/$defs/revision"}, + "environment": {"$ref": "#/$defs/identity"}, + "account_alias": {"$ref": "#/$defs/identity"}, + "account_digest_sha256": {"$ref": "#/$defs/sha256"} + } + }, + "expectedIdentity": { + "allOf": [ + {"$ref": "#/$defs/comparableIdentity"}, + { + "type": "object", + "required": ["expected_identity_sha256"], + "properties": {"expected_identity_sha256": {"$ref": "#/$defs/sha256"}} + } + ], + "unevaluatedProperties": false + }, + "comparableIdentity": { + "type": "object", + "required": [ + "deployment_bundle_sha256", + "activation_id", + "activation_sha256", + "platform", + "repository", + "revision", + "environment", + "account_alias", + "account_digest_sha256" + ], + "properties": { + "deployment_bundle_sha256": {"$ref": "#/$defs/sha256"}, + "activation_id": {"$ref": "#/$defs/identity"}, + "activation_sha256": {"$ref": "#/$defs/sha256"}, + "platform": {"$ref": "#/$defs/identity"}, + "repository": {"$ref": "#/$defs/repository"}, + "revision": {"$ref": "#/$defs/revision"}, + "environment": {"$ref": "#/$defs/identity"}, + "account_alias": {"$ref": "#/$defs/identity"}, + "account_digest_sha256": {"$ref": "#/$defs/sha256"} + } + }, + "observedIdentity": { + "allOf": [ + {"$ref": "#/$defs/comparableIdentity"}, + { + "type": "object", + "required": ["producer_id", "producer_revision", "artifact_sha256", "observed_at", "observed_identity_sha256"], + "properties": { + "producer_id": {"$ref": "#/$defs/identity"}, + "producer_revision": {"$ref": "#/$defs/revision"}, + "artifact_sha256": {"$ref": "#/$defs/sha256"}, + "observed_at": {"$ref": "#/$defs/timestamp"}, + "observed_identity_sha256": {"$ref": "#/$defs/sha256"} + } + } + ], + "unevaluatedProperties": false + }, + "observerReceipt": { + "type": "object", + "additionalProperties": false, + "required": ["schema", "observer_id", "observer_revision", "created_at", "observed_identity", "observer_receipt_sha256"], + "properties": { + "schema": {"const": "qsl.reconciliation_observer_receipt.v1"}, + "observer_id": {"$ref": "#/$defs/identity"}, + "observer_revision": {"$ref": "#/$defs/revision"}, + "created_at": {"$ref": "#/$defs/timestamp"}, + "observed_identity": {"$ref": "#/$defs/observedIdentity"}, + "observer_receipt_sha256": {"$ref": "#/$defs/sha256"} + } + }, + "comparisonEntry": { + "type": "object", + "additionalProperties": false, + "required": ["expected", "observed", "equal"], + "properties": { + "expected": {"type": "string", "minLength": 1}, + "observed": {"type": "string", "minLength": 1}, + "equal": {"type": "boolean"} + } + }, + "comparison": { + "type": "object", + "additionalProperties": false, + "required": ["complete", "fields"], + "properties": { + "complete": {"const": true}, + "fields": { + "type": "object", + "additionalProperties": false, + "required": [ + "deployment_bundle_sha256", + "activation_id", + "activation_sha256", + "platform", + "repository", + "revision", + "environment", + "account_alias", + "account_digest_sha256" + ], + "properties": { + "deployment_bundle_sha256": {"$ref": "#/$defs/comparisonEntry"}, + "activation_id": {"$ref": "#/$defs/comparisonEntry"}, + "activation_sha256": {"$ref": "#/$defs/comparisonEntry"}, + "platform": {"$ref": "#/$defs/comparisonEntry"}, + "repository": {"$ref": "#/$defs/comparisonEntry"}, + "revision": {"$ref": "#/$defs/comparisonEntry"}, + "environment": {"$ref": "#/$defs/comparisonEntry"}, + "account_alias": {"$ref": "#/$defs/comparisonEntry"}, + "account_digest_sha256": {"$ref": "#/$defs/comparisonEntry"} + } + } + } + }, + "assertions": { + "type": "object", + "additionalProperties": false, + "required": ["apply_performed", "config_sync_performed", "runtime_mutation_performed", "runtime_active", "fills_verified", "capital_use_verified"], + "properties": { + "apply_performed": {"const": false}, + "config_sync_performed": {"const": false}, + "runtime_mutation_performed": {"const": false}, + "runtime_active": {"const": false}, + "fills_verified": {"const": false}, + "capital_use_verified": {"const": false} + } + } + } +}