diff --git a/crates/qbit-prism/sql/001_share_ledger.sql b/crates/qbit-prism/sql/001_share_ledger.sql index a3ba15c..2e1ff6c 100644 --- a/crates/qbit-prism/sql/001_share_ledger.sql +++ b/crates/qbit-prism/sql/001_share_ledger.sql @@ -137,6 +137,7 @@ ALTER TABLE qbit_payout_carry_forward CREATE TABLE IF NOT EXISTS qbit_pool_blocks ( block_hash text PRIMARY KEY, + audit_publication_sequence bigint, block_height bigint NOT NULL CHECK (block_height >= 0), parent_hash text NOT NULL, coinbase_txid text NOT NULL, @@ -170,6 +171,565 @@ ALTER TABLE qbit_pool_blocks ADD CONSTRAINT qbit_pool_blocks_chain_state_check CHECK (chain_state IN ('prepared', 'confirmed', 'inactive', 'rejected', 'reversed')); +-- Artifact order is allocated at the durable prepared -> confirmed boundary. +-- Exact confirmed replay and a later inactive -> confirmed transition reuse it, +-- independent of block height. Upgrade existing confirmed and inactive rows +-- deterministically and advance the sequence beyond any value already installed +-- by a partial migration. +BEGIN; +SELECT pg_advisory_xact_lock( + hashtext('qbit_audit_publication_sequence_migration') +); +DO $$ +DECLARE + table_namespace text := current_schema(); + table_oid oid; +BEGIN + SELECT pool_blocks.oid + INTO table_oid + FROM pg_class pool_blocks + JOIN pg_namespace namespace ON namespace.oid = pool_blocks.relnamespace + WHERE namespace.nspname = table_namespace + AND pool_blocks.relname = 'qbit_pool_blocks' + AND pool_blocks.relkind = 'r'; + IF table_oid IS NULL THEN + RAISE EXCEPTION 'missing qbit_pool_blocks in current schema'; + END IF; + IF NOT EXISTS ( + SELECT 1 + FROM pg_class relation + JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace + WHERE namespace.nspname = table_namespace + AND relation.relname = 'qbit_audit_publication_sequence_seq' + ) THEN + EXECUTE format( + 'CREATE SEQUENCE %I.qbit_audit_publication_sequence_seq', + table_namespace + ); + END IF; +END; +$$; + +DO $$ +DECLARE + sequence_definition record; +BEGIN + SELECT + sequence.seqtypid, + sequence.seqstart, + sequence.seqincrement, + sequence.seqmax, + sequence.seqmin, + sequence.seqcache, + sequence.seqcycle, + relation.relnamespace AS sequence_namespace, + relation.relpersistence AS sequence_persistence, + relation.relowner AS sequence_owner, + pool_blocks.relnamespace AS table_namespace, + pool_blocks.relowner AS table_owner, + EXISTS ( + SELECT 1 + FROM pg_depend dependency + WHERE dependency.classid = 'pg_class'::regclass + AND dependency.objid = relation.oid + AND dependency.refclassid = 'pg_class'::regclass + AND dependency.refobjsubid > 0 + AND dependency.deptype IN ('a', 'i') + ) AS owned_by_column + INTO sequence_definition + FROM pg_class pool_blocks + JOIN pg_namespace table_namespace + ON table_namespace.oid = pool_blocks.relnamespace + JOIN pg_class relation + ON relation.relnamespace = pool_blocks.relnamespace + AND relation.relname = 'qbit_audit_publication_sequence_seq' + JOIN pg_sequence sequence ON sequence.seqrelid = relation.oid + WHERE relation.relkind = 'S' + AND table_namespace.nspname = current_schema() + AND pool_blocks.relname = 'qbit_pool_blocks' + AND pool_blocks.relkind = 'r'; + IF NOT FOUND + OR sequence_definition.seqtypid <> 'bigint'::regtype + OR sequence_definition.seqstart <> 1 + OR sequence_definition.seqincrement <> 1 + OR sequence_definition.seqmax <> 9223372036854775807 + OR sequence_definition.seqmin <> 1 + OR sequence_definition.seqcache <> 1 + OR sequence_definition.seqcycle + OR sequence_definition.sequence_namespace <> + sequence_definition.table_namespace + OR sequence_definition.sequence_persistence <> 'p' + OR sequence_definition.sequence_owner <> sequence_definition.table_owner + OR sequence_definition.owned_by_column THEN + RAISE EXCEPTION 'invalid audit publication sequence definition'; + END IF; +END; +$$; + +DO $$ +DECLARE + table_namespace text := current_schema(); +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_class pool_blocks + JOIN pg_namespace namespace + ON namespace.oid = pool_blocks.relnamespace + WHERE namespace.nspname = table_namespace + AND pool_blocks.relname = 'qbit_pool_blocks' + AND pool_blocks.relkind = 'r' + ) THEN + RAISE EXCEPTION 'missing qbit_pool_blocks in current schema'; + END IF; + EXECUTE format( + 'ALTER TABLE %I.qbit_pool_blocks ' + 'ADD COLUMN IF NOT EXISTS audit_publication_sequence bigint', + table_namespace + ); +END; +$$; + +DO $$ +DECLARE + table_oid oid; +BEGIN + SELECT pool_blocks.oid + INTO table_oid + FROM pg_class pool_blocks + JOIN pg_namespace namespace + ON namespace.oid = pool_blocks.relnamespace + WHERE namespace.nspname = current_schema() + AND pool_blocks.relname = 'qbit_pool_blocks' + AND pool_blocks.relkind = 'r'; + IF NOT EXISTS ( + SELECT 1 + FROM pg_attribute + WHERE attrelid = table_oid + AND attname = 'audit_publication_sequence' + AND attnum > 0 + AND NOT attisdropped + AND atttypid = 'bigint'::regtype + AND NOT attnotnull + AND NOT atthasdef + AND attidentity = '' + AND attgenerated = '' + AND attcollation = 0 + ) THEN + RAISE EXCEPTION 'invalid audit publication sequence column definition'; + END IF; +END; +$$; + +DO $$ +DECLARE + pending record; + assigned_sequence numeric; + assignment_start numeric; + invalid_sequence boolean; + duplicate_sequence boolean; + maximum_sequence bigint; + pending_count bigint; + raw_next_sequence numeric; + sequence_last bigint; + sequence_called boolean; + sequence_relation regclass; + table_namespace text := current_schema(); + table_oid oid; +BEGIN + -- Serialize partial/concurrent schema initialization and exclude live + -- confirmation updates while setval/backfill establish the ordinal floor. + SELECT pool_blocks.oid + INTO table_oid + FROM pg_class pool_blocks + JOIN pg_namespace namespace + ON namespace.oid = pool_blocks.relnamespace + WHERE namespace.nspname = table_namespace + AND pool_blocks.relname = 'qbit_pool_blocks' + AND pool_blocks.relkind = 'r'; + IF table_oid IS NULL THEN + RAISE EXCEPTION 'missing qbit_pool_blocks in current schema'; + END IF; + EXECUTE format( + 'LOCK TABLE %I.qbit_pool_blocks IN SHARE ROW EXCLUSIVE MODE', + table_namespace + ); + EXECUTE format( + 'SELECT EXISTS (' + 'SELECT 1 FROM %I.qbit_pool_blocks ' + 'WHERE audit_publication_sequence IS NOT NULL ' + 'AND audit_publication_sequence <= 0)', + table_namespace + ) INTO invalid_sequence; + IF invalid_sequence THEN + RAISE EXCEPTION 'invalid non-positive audit publication sequence'; + END IF; + EXECUTE format( + 'SELECT EXISTS (' + 'SELECT audit_publication_sequence ' + 'FROM %I.qbit_pool_blocks ' + 'WHERE audit_publication_sequence IS NOT NULL ' + 'GROUP BY audit_publication_sequence HAVING count(*) > 1)', + table_namespace + ) INTO duplicate_sequence; + IF duplicate_sequence THEN + RAISE EXCEPTION 'duplicate audit publication sequence'; + END IF; + EXECUTE format( + 'SELECT COALESCE(MAX(audit_publication_sequence), 0) ' + 'FROM %I.qbit_pool_blocks', + table_namespace + ) INTO maximum_sequence; + SELECT sequence.oid::regclass + INTO sequence_relation + FROM pg_class sequence + WHERE sequence.relnamespace = ( + SELECT oid FROM pg_namespace + WHERE nspname = table_namespace + ) + AND sequence.relname = 'qbit_audit_publication_sequence_seq' + AND sequence.relkind = 'S'; + EXECUTE format( + 'SELECT last_value, is_called ' + 'FROM %I.qbit_audit_publication_sequence_seq', + table_namespace + ) INTO sequence_last, sequence_called; + raw_next_sequence := sequence_last::numeric + + CASE WHEN sequence_called THEN 1 ELSE 0 END; + EXECUTE format( + 'SELECT count(*) FROM %I.qbit_pool_blocks ' + 'WHERE chain_state IN (''confirmed'', ''inactive'') ' + 'AND audit_publication_sequence IS NULL', + table_namespace + ) INTO pending_count; + assignment_start := GREATEST( + raw_next_sequence, + maximum_sequence::numeric + 1 + ); + IF pending_count > 0 + AND ( + assignment_start < 1 + OR assignment_start + pending_count::numeric - 1 + > 9223372036854775807::numeric + ) THEN + RAISE EXCEPTION 'audit publication sequence exhausted'; + END IF; + assigned_sequence := assignment_start; + FOR pending IN EXECUTE format( + 'SELECT block_hash FROM %I.qbit_pool_blocks ' + 'WHERE chain_state IN (''confirmed'', ''inactive'') ' + 'AND audit_publication_sequence IS NULL ' + 'ORDER BY found_at, block_hash', + table_namespace + ) + LOOP + EXECUTE format( + 'UPDATE %I.qbit_pool_blocks ' + 'SET audit_publication_sequence = $1 ' + 'WHERE block_hash = $2 ' + 'AND audit_publication_sequence IS NULL', + table_namespace + ) USING assigned_sequence::bigint, pending.block_hash; + assigned_sequence := assigned_sequence + 1; + END LOOP; +END; +$$; + +DO $$ +DECLARE + index_name constant text := + 'qbit_pool_blocks_audit_publication_sequence_idx'; + canonical_index_oid oid; + table_namespace text := current_schema(); + table_oid oid; +BEGIN + SELECT pool_blocks.oid + INTO table_oid + FROM pg_class pool_blocks + JOIN pg_namespace namespace + ON namespace.oid = pool_blocks.relnamespace + WHERE namespace.nspname = table_namespace + AND pool_blocks.relname = 'qbit_pool_blocks' + AND pool_blocks.relkind = 'r'; + IF table_oid IS NULL THEN + RAISE EXCEPTION 'missing qbit_pool_blocks in current schema'; + END IF; + SELECT index_relation.oid + INTO canonical_index_oid + FROM pg_class index_relation + WHERE index_relation.relnamespace = ( + SELECT oid FROM pg_namespace + WHERE nspname = table_namespace + ) + AND index_relation.relname = index_name; + IF canonical_index_oid IS NULL THEN + EXECUTE format( + 'CREATE UNIQUE INDEX %I ' + 'ON %I.qbit_pool_blocks (audit_publication_sequence)', + index_name, + table_namespace + ); + SELECT index_relation.oid + INTO canonical_index_oid + FROM pg_class index_relation + WHERE index_relation.relnamespace = ( + SELECT oid FROM pg_namespace + WHERE nspname = table_namespace + ) + AND index_relation.relname = index_name; + END IF; + IF NOT EXISTS ( + SELECT 1 + FROM pg_index index_definition + JOIN pg_class index_relation + ON index_relation.oid = index_definition.indexrelid + JOIN pg_am access_method + ON access_method.oid = index_relation.relam + JOIN pg_attribute ordinal_attribute + ON ordinal_attribute.attrelid = index_definition.indrelid + AND ordinal_attribute.attname = 'audit_publication_sequence' + AND NOT ordinal_attribute.attisdropped + JOIN pg_opclass operator_class + ON index_definition.indclass::text = operator_class.oid::text + WHERE index_definition.indexrelid = canonical_index_oid + AND index_definition.indrelid = table_oid + AND access_method.amname = 'btree' + AND index_relation.relkind = 'i' + AND index_relation.relnamespace = ( + SELECT relnamespace + FROM pg_class + WHERE oid = index_definition.indrelid + ) + AND index_relation.relpersistence = 'p' + AND index_relation.relowner = ( + SELECT relowner + FROM pg_class + WHERE oid = index_definition.indrelid + ) + AND index_definition.indisunique + AND index_definition.indisvalid + AND index_definition.indisready + AND index_definition.indislive + AND index_definition.indimmediate + AND NOT index_definition.indisprimary + AND NOT index_definition.indisexclusion + AND NOT index_definition.indisclustered + AND NOT index_definition.indisreplident + AND NOT index_definition.indnullsnotdistinct + AND index_definition.indnkeyatts = 1 + AND index_definition.indnatts = 1 + AND index_definition.indkey::text = ordinal_attribute.attnum::text + AND index_definition.indcollation::text = '0' + AND index_definition.indoption::text = '0' + AND operator_class.opcname = 'int8_ops' + AND operator_class.opcmethod = index_relation.relam + AND operator_class.opcnamespace = 'pg_catalog'::regnamespace + AND operator_class.opcintype = 'bigint'::regtype + AND operator_class.opcdefault + AND index_definition.indexprs IS NULL + AND index_definition.indpred IS NULL + ) THEN + RAISE EXCEPTION 'invalid audit publication sequence index definition'; + END IF; + IF EXISTS ( + SELECT 1 + FROM pg_index index_definition + JOIN pg_class index_relation + ON index_relation.oid = index_definition.indexrelid + JOIN pg_am access_method + ON access_method.oid = index_relation.relam + JOIN pg_attribute ordinal_attribute + ON ordinal_attribute.attrelid = index_definition.indrelid + AND ordinal_attribute.attname = 'audit_publication_sequence' + AND NOT ordinal_attribute.attisdropped + JOIN pg_opclass operator_class + ON index_definition.indclass::text = operator_class.oid::text + WHERE index_definition.indrelid = table_oid + AND index_definition.indexrelid <> canonical_index_oid + AND access_method.amname = 'btree' + AND index_relation.relkind = 'i' + AND index_relation.relnamespace = ( + SELECT relnamespace + FROM pg_class + WHERE oid = index_definition.indrelid + ) + AND index_relation.relpersistence = 'p' + AND index_relation.relowner = ( + SELECT relowner + FROM pg_class + WHERE oid = index_definition.indrelid + ) + AND index_definition.indisunique + AND index_definition.indisvalid + AND index_definition.indisready + AND index_definition.indnkeyatts = 1 + AND index_definition.indnatts = 1 + AND index_definition.indkey::text = ordinal_attribute.attnum::text + AND index_definition.indcollation::text = '0' + AND index_definition.indoption::text = '0' + AND operator_class.opcname = 'int8_ops' + AND operator_class.opcmethod = index_relation.relam + AND operator_class.opcnamespace = 'pg_catalog'::regnamespace + AND operator_class.opcintype = 'bigint'::regtype + AND operator_class.opcdefault + AND index_definition.indexprs IS NULL + AND index_definition.indpred IS NULL + ) THEN + RAISE EXCEPTION 'duplicate audit publication sequence index definition'; + END IF; +END; +$$; + +DO $$ +DECLARE + constraint_definition text; + constraint_validated boolean; + table_namespace text := current_schema(); + table_oid oid; +BEGIN + SELECT pool_blocks.oid + INTO table_oid + FROM pg_class pool_blocks + JOIN pg_namespace namespace + ON namespace.oid = pool_blocks.relnamespace + WHERE namespace.nspname = table_namespace + AND pool_blocks.relname = 'qbit_pool_blocks' + AND pool_blocks.relkind = 'r'; + IF table_oid IS NULL THEN + RAISE EXCEPTION 'missing qbit_pool_blocks in current schema'; + END IF; + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conrelid = table_oid + AND conname = 'qbit_pool_blocks_audit_publication_sequence_check' + ) THEN + EXECUTE format( + 'ALTER TABLE %I.qbit_pool_blocks ' + 'ADD CONSTRAINT ' + 'qbit_pool_blocks_audit_publication_sequence_check ' + 'CHECK ((audit_publication_sequence IS NULL ' + 'OR audit_publication_sequence > 0) ' + 'AND (chain_state <> ''confirmed'' ' + 'OR audit_publication_sequence IS NOT NULL))', + table_namespace + ); + END IF; + SELECT + regexp_replace( + regexp_replace( + pg_get_constraintdef(oid, true), + '[[:space:]]+', + ' ', + 'g' + ), + ' NOT VALID$', + '' + ), + convalidated + INTO constraint_definition, constraint_validated + FROM pg_constraint + WHERE conrelid = table_oid + AND conname = 'qbit_pool_blocks_audit_publication_sequence_check' + AND contype = 'c' + AND NOT condeferrable + AND NOT condeferred + AND NOT connoinherit + AND conislocal + AND coninhcount = 0 + AND cardinality(conkey) = 2 + AND conkey @> ARRAY[ + ( + SELECT attnum::smallint + FROM pg_attribute + WHERE attrelid = table_oid + AND attname = 'audit_publication_sequence' + AND NOT attisdropped + ), + ( + SELECT attnum::smallint + FROM pg_attribute + WHERE attrelid = table_oid + AND attname = 'chain_state' + AND NOT attisdropped + ) + ]::smallint[]; + IF constraint_definition IS NULL + OR constraint_definition <> + 'CHECK ((audit_publication_sequence IS NULL OR audit_publication_sequence > 0) AND (chain_state <> ''confirmed''::text OR audit_publication_sequence IS NOT NULL))' THEN + RAISE EXCEPTION 'invalid audit publication sequence constraint definition'; + END IF; + IF EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conrelid = table_oid + AND conname <> 'qbit_pool_blocks_audit_publication_sequence_check' + AND contype = 'c' + AND regexp_replace( + regexp_replace( + pg_get_constraintdef(oid, true), + '[[:space:]]+', + ' ', + 'g' + ), + ' NOT VALID$', + '' + ) = constraint_definition + ) THEN + RAISE EXCEPTION 'duplicate audit publication sequence constraint definition'; + END IF; + IF NOT constraint_validated THEN + EXECUTE format( + 'ALTER TABLE %I.qbit_pool_blocks ' + 'VALIDATE CONSTRAINT ' + 'qbit_pool_blocks_audit_publication_sequence_check', + table_namespace + ); + END IF; +END; +$$; + +-- Sequence operations are nontransactional in PostgreSQL. Keep the sole +-- allocator mutation after every row and catalog validation so any earlier +-- rejection leaves an existing sequence's exact state untouched. +DO $$ +DECLARE + maximum_sequence bigint; + raw_next_sequence numeric; + sequence_called boolean; + sequence_last bigint; + sequence_relation regclass; + table_namespace text := current_schema(); +BEGIN + SELECT sequence.oid::regclass + INTO sequence_relation + FROM pg_class sequence + JOIN pg_namespace namespace + ON namespace.oid = sequence.relnamespace + WHERE namespace.nspname = table_namespace + AND sequence.relname = 'qbit_audit_publication_sequence_seq' + AND sequence.relkind = 'S'; + IF sequence_relation IS NULL THEN + RAISE EXCEPTION 'missing audit publication sequence'; + END IF; + EXECUTE format( + 'SELECT COALESCE(MAX(audit_publication_sequence), 0) ' + 'FROM %I.qbit_pool_blocks', + table_namespace + ) INTO maximum_sequence; + EXECUTE format( + 'SELECT last_value, is_called ' + 'FROM %I.qbit_audit_publication_sequence_seq', + table_namespace + ) INTO sequence_last, sequence_called; + raw_next_sequence := sequence_last::numeric + + CASE WHEN sequence_called THEN 1 ELSE 0 END; + IF maximum_sequence::numeric >= raw_next_sequence THEN + PERFORM setval(sequence_relation, maximum_sequence, true); + END IF; +END; +$$; +COMMIT; + CREATE TABLE IF NOT EXISTS qbit_pool_audit_bundles ( block_hash text PRIMARY KEY REFERENCES qbit_pool_blocks(block_hash), audit_bundle jsonb NOT NULL, @@ -1084,7 +1644,20 @@ AS $$ DECLARE lease_count integer; confirmed_count integer; + publication_sequence pg_catalog.regclass; BEGIN + SELECT sequence.oid::pg_catalog.regclass + INTO publication_sequence + FROM pg_catalog.pg_class pool_blocks + JOIN pg_catalog.pg_class sequence + ON sequence.relnamespace = pool_blocks.relnamespace + AND sequence.relname = 'qbit_audit_publication_sequence_seq' + AND sequence.relkind = 'S' + WHERE pool_blocks.oid = 'qbit_pool_blocks'::pg_catalog.regclass + AND pool_blocks.relkind = 'r'; + IF publication_sequence IS NULL THEN + RAISE EXCEPTION 'missing audit publication sequence'; + END IF; UPDATE qbit_ledger_writer_lease SET lease_expires_at = clock_timestamp() + lease_duration, updated_at = clock_timestamp() @@ -1099,7 +1672,8 @@ BEGIN END IF; UPDATE qbit_pool_blocks - SET chain_state = 'confirmed' + SET chain_state = 'confirmed', + audit_publication_sequence = pg_catalog.nextval(publication_sequence) WHERE block_hash = confirmed_block_hash AND block_height = active_tip_height AND chain_state = 'prepared' @@ -1215,6 +1789,34 @@ BEGIN END; $$; +-- PL/pgSQL plans relation references on first execution. Pin confirmation +-- ordinal allocation and reactivation to their installation schema, and list +-- pg_temp last so a caller +-- cannot redirect the lease, pool-block, or sequence names through its own +-- search path or a temporary relation. +DO $$ +DECLARE + installation_schema pg_catalog.text := pg_catalog.current_schema(); +BEGIN + EXECUTE pg_catalog.format( + 'ALTER FUNCTION %I.qbit_confirm_pool_block(' + 'pg_catalog.text, pg_catalog.int8, pg_catalog.text, ' + 'pg_catalog.int8, pg_catalog.text, pg_catalog.interval) ' + 'SET search_path TO pg_catalog, %I, pg_temp', + installation_schema, + installation_schema + ); + EXECUTE pg_catalog.format( + 'ALTER FUNCTION %I.qbit_reactivate_pool_block(' + 'pg_catalog.text, pg_catalog.int8, pg_catalog.text, ' + 'pg_catalog.int8, pg_catalog.text, pg_catalog.interval) ' + 'SET search_path TO pg_catalog, %I, pg_temp', + installation_schema, + installation_schema + ); +END; +$$; + DROP FUNCTION IF EXISTS qbit_reject_prepared_pool_block(text, bigint, text, bigint, text); CREATE OR REPLACE FUNCTION qbit_reject_prepared_pool_block( diff --git a/crates/qbit-prism/tests/audit_cli.rs b/crates/qbit-prism/tests/audit_cli.rs index 89ac118..d0a90c9 100644 --- a/crates/qbit-prism/tests/audit_cli.rs +++ b/crates/qbit-prism/tests/audit_cli.rs @@ -85,6 +85,27 @@ fn canonical_share_segment_bytes( .into_bytes() } +fn canonicalize_cli_value(value: &serde_json::Value, label: &str) -> Vec { + let bundle_path = std::env::temp_dir().join(format!( + "qbit-prism-audit-canonicalize-{label}-{}.json", + std::process::id() + )); + fs::write(&bundle_path, serde_json::to_vec_pretty(value).unwrap()).unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_qbit-prism-audit-canonicalize")) + .arg("--input") + .arg(&bundle_path) + .output() + .unwrap(); + let _ = fs::remove_file(&bundle_path); + assert!( + output.status.success(), + "stdout: {}\nstderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + output.stdout +} + #[test] fn verifier_cli_accepts_exported_power_law_bundle() { let fixture: Fixture = serde_json::from_str(include_str!( @@ -153,6 +174,43 @@ fn verifier_cli_accepts_exported_power_law_bundle() { ); } +#[test] +fn canonicalizer_restores_typed_bytes_for_reordered_unicode_and_optional_input() { + let fixture: Fixture = serde_json::from_str(include_str!( + "../fixtures/power-law-accrual.prism-fixture.json" + )) + .unwrap(); + let bundle = build_audit_bundle( + fixture.shares, + fixture.found_block, + power_law_prior_balances(), + PayoutPolicy::day_one_default(), + &manifest_signing_key(), + &ledger_signing_key(), + ) + .unwrap(); + let canonical = canonical_audit_bundle_bytes(&bundle).unwrap(); + + // Value serialization uses map order rather than AuditBundle's typed field + // order. The adapter must restore the typed canonical representation. + let mut reordered = serde_json::to_value(&bundle).unwrap(); + assert_eq!(canonicalize_cli_value(&reordered, "reordered"), canonical); + + reordered["shares"][0]["credit_policy"] = serde_json::Value::Null; + assert_eq!( + canonicalize_cli_value(&reordered, "explicit-optional-null"), + canonical + ); + + reordered["shares"][0]["miner_id"] = serde_json::Value::String("miner-é".into()); + let unicode_bundle: AuditBundle = serde_json::from_value(reordered.clone()).unwrap(); + let unicode_canonical = canonical_audit_bundle_bytes(&unicode_bundle).unwrap(); + let unicode_output = canonicalize_cli_value(&reordered, "unicode"); + assert_eq!(unicode_output, unicode_canonical); + assert!(unicode_output.windows("miner-é".len()).any(|window| window == "miner-é".as_bytes())); + +} + #[test] fn verifier_cli_accepts_live_testnet_scale_legacy_bundle() { let bundle = live_testnet_scale_bundle(); @@ -173,8 +231,6 @@ fn verifier_cli_accepts_live_testnet_scale_legacy_bundle() { .arg(report.coinbase_value_sats.to_string()) .output() .unwrap(); - let _ = fs::remove_file(&bundle_path); - assert!( output.status.success(), "stdout: {}\nstderr: {}", @@ -184,6 +240,18 @@ fn verifier_cli_accepts_live_testnet_scale_legacy_bundle() { assert!( String::from_utf8_lossy(&output.stdout).contains("qbit.prism.audit-verification-report.v1") ); + let canonical_output = Command::new(env!("CARGO_BIN_EXE_qbit-prism-audit-canonicalize")) + .arg("--input") + .arg(&bundle_path) + .output() + .unwrap(); + let _ = fs::remove_file(&bundle_path); + assert!(canonical_output.status.success()); + assert_eq!( + canonical_output.stdout, + canonical_audit_bundle_bytes(&bundle).unwrap() + ); + assert!(bundle.found_block.network_difficulty > u64::MAX as u128); } #[test] diff --git a/lab/prism/audit_artifacts.py b/lab/prism/audit_artifacts.py new file mode 100644 index 0000000..f98a41b --- /dev/null +++ b/lab/prism/audit_artifacts.py @@ -0,0 +1,4392 @@ +#!/usr/bin/env python3 +"""Filesystem ownership, verification, and publication for PRISM audit data. + +This module deliberately has no coordinator dependency. The coordinator owns +block-finalization sequencing and the ledger owns database authorization; this +store is the sole authority for paths and filesystem mutation below the audit +root. +""" + +from __future__ import annotations + +import copy +from contextlib import contextmanager, nullcontext +import fcntl +import hashlib +import hmac +import json +import os +import re +import selectors +import signal +import stat +import subprocess +import threading +import time +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Iterator, Mapping + +from lab.prism.prism_tools import prism_tool_command + + +AUDIT_BODY_REF_SCHEMA = "qbit.prism.audit-body-ref.v1" +AUDIT_BUNDLE_V2_SCHEMA = "qbit.prism.audit-bundle.v2" +AUDIT_SHARE_SEGMENT_SCHEMA = "qbit.prism.audit-share-segment.v1" +AUDIT_WINDOW_COMPLETENESS_PROOF_SCHEMA = ( + "qbit.prism.window-completeness-proof.v1" +) +LIVE_ENVELOPE_SCHEMA = "qbit.prism.live-audit-bundle-envelope.v1" +LIVE_EVIDENCE_SCHEMA = "qbit.prism.live-stratum-evidence.v1" +DEFAULT_AUDIT_SHARE_SEGMENT_SIZE = 10_000 +DEFAULT_VERIFIER_TIMEOUT_SECONDS = 60.0 +MAX_VERIFIER_OUTPUT_BYTES = 1024 * 1024 +VERIFICATION_REPORT_SCHEMA = "qbit.prism.audit-verification-report.v1" +VERIFICATION_IDENTITY_SCHEMA = "qbit.prism.audit-verification-identity.v1" +LEGACY_VERIFICATION_UNAVAILABLE_SCHEMA = ( + "qbit.prism.legacy-verification-unavailable.v1" +) + +_HEX_64 = r"[0-9a-f]{64}" +_TOKEN_32 = r"[0-9a-f]{32}" +_BODY_RE = re.compile( + rf"\Aprism-audit-bundle-body-(?P{_HEX_64})-(?P{_HEX_64})\.json\Z" +) +_SHARE_CONTENT_RE = re.compile( + rf"\Aprism-audit-share-segment-(?P[1-9][0-9]*)-" + rf"(?P[1-9][0-9]*)-(?P{_HEX_64})\.json\Z" +) +_SHARE_SLOT_RE = re.compile( + r"\Aprism-audit-share-segment-slot-(?P[1-9][0-9]*)-" + r"(?P[1-9][0-9]*)\.json\Z" +) +_LIVE_RE = re.compile( + rf"\Aprism-live-audit-bundle-(?P0|[1-9][0-9]*)-" + rf"(?P{_HEX_64})\.json\Z" +) +_CANDIDATE_RE = re.compile( + rf"\A\.prism-live-audit-bundle-candidate-(?P{_HEX_64})-" + rf"(?P{_TOKEN_32})\.json\.tmp\Z" +) +_LEGACY_CANDIDATE_RE = re.compile( + rf"\Aprism-live-audit-bundle-candidate-(?P{_HEX_64})\.json\Z" +) +_LEGACY_HIDDEN_CANDIDATE_RE = re.compile( + rf"\A\.prism-live-audit-bundle-candidate-(?P{_HEX_64})\.json\.tmp\Z" +) +_PUBLICATION_LOCK_NAME = ".prism-audit-publication.lock" +_PUBLICATION_THREAD_LOCKS_GUARD = threading.Lock() +_PUBLICATION_THREAD_LOCKS: dict[tuple[int, int], threading.RLock] = {} +_PUBLICATION_LOCAL_OWNERS: dict[tuple[int, int], tuple[int, int]] = {} + + +def _publication_thread_lock(identity: tuple[int, int]) -> threading.RLock: + """Share a local lock across store instances opened on the same inode.""" + + with _PUBLICATION_THREAD_LOCKS_GUARD: + lock = _PUBLICATION_THREAD_LOCKS.get(identity) + if lock is None: + lock = threading.RLock() + _PUBLICATION_THREAD_LOCKS[identity] = lock + return lock + + +def _reject_cross_store_local_owner( + identity: tuple[int, int], + *, + thread_id: int, + store_id: int, +) -> None: + """Reject same-thread nesting through a second store before it can block.""" + + with _PUBLICATION_THREAD_LOCKS_GUARD: + local_owner = _PUBLICATION_LOCAL_OWNERS.get(identity) + if local_owner is not None: + owner_thread, owner_store = local_owner + if owner_thread == thread_id and owner_store != store_id: + raise RuntimeError( + "audit publication guard is already held by another store " + "on this thread" + ) + + +def _canonical_hex(value: object, *, name: str, expected_bytes: int = 32) -> str: + text = str(value) + if len(text) != expected_bytes * 2: + raise ValueError(f"{name} must be {expected_bytes} bytes of hex") + try: + bytes.fromhex(text) + except ValueError as exc: + raise ValueError(f"{name} must be hexadecimal") from exc + return text.lower() + + +def _sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _json_bytes(payload: Mapping[str, Any], *, sort_keys: bool = False) -> bytes: + return json.dumps( + payload, + separators=(",", ":"), + sort_keys=sort_keys, + ).encode("utf-8") + + +def canonical_audit_bundle_bytes( + final_bundle: dict[str, Any], + canonicalizer: Callable[[dict[str, Any]], bytes] | None = None, +) -> bytes: + if canonicalizer is None: + raise RuntimeError("J1 canonical bundle capability is required") + canonical = canonicalizer(final_bundle) + return canonical.encode() if isinstance(canonical, str) else bytes(canonical) + + +@dataclass(frozen=True) +class AuditArtifactConfig: + root: Path + evidence_path: Path + live_bundle_retention: int = 5 + candidate_retention_seconds: int = 24 * 60 * 60 + share_segment_size: int = 0 + verifier_timeout_seconds: float = DEFAULT_VERIFIER_TIMEOUT_SECONDS + + +@dataclass(frozen=True) +class AuditPublicationIdentity: + """Ordering token assigned while P1's balance serializer is held.""" + + sequence: int + block_height: int + block_hash: str + + def __post_init__(self) -> None: + if isinstance(self.sequence, bool) or not isinstance(self.sequence, int): + raise ValueError("publication sequence must be an integer") + if self.sequence < 0: + raise ValueError("publication sequence must be non-negative") + if isinstance(self.block_height, bool) or not isinstance( + self.block_height, + int, + ): + raise ValueError("publication block height must be an integer") + if self.block_height < 0: + raise ValueError("publication block height must be non-negative") + canonical_block_hash = _canonical_hex( + self.block_hash, + name="publication block hash", + ) + if self.block_hash != canonical_block_hash: + raise ValueError("publication block hash must be canonical") + + def to_json(self) -> dict[str, object]: + return { + "sequence": self.sequence, + "block_height": self.block_height, + "block_hash": self.block_hash, + } + + +@dataclass(frozen=True) +class OwnedCandidateArtifact: + path: Path + block_hash: str + token: str + store_token: str + + +@dataclass(frozen=True) +class VerifiedAuditBundle: + candidate: OwnedCandidateArtifact + report: Mapping[str, Any] + literal_sha256: str + byte_length: int + device: int + inode: int + mtime_ns: int + canonical_copy_eligible: bool + verification_identity: Mapping[str, Any] + + +@dataclass(frozen=True) +class PublishedAuditBodyRef: + path: Path + body_uri: str + audit_bundle_sha256: str + + +@dataclass(frozen=True) +class AuditPublication: + identity: AuditPublicationIdentity + envelope_path: Path + evidence: Mapping[str, Any] + published: bool + + +@dataclass(frozen=True) +class RetentionResult: + live_removed: int = 0 + candidate_removed: int = 0 + errors: int = 0 + + +@dataclass(frozen=True) +class _FileIdentity: + device: int + inode: int + mode: int + size: int + mtime_ns: int + + @classmethod + def from_stat(cls, value: os.stat_result) -> _FileIdentity: + return cls( + value.st_dev, + value.st_ino, + value.st_mode, + value.st_size, + value.st_mtime_ns, + ) + + def matches(self, value: os.stat_result) -> bool: + return ( + self.device == value.st_dev + and self.inode == value.st_ino + and self.mode == value.st_mode + and self.size == value.st_size + and self.mtime_ns == value.st_mtime_ns + and stat.S_ISREG(value.st_mode) + ) + + +@dataclass(frozen=True) +class _LegacyProofToken: + identity: AuditPublicationIdentity + evidence_file: _FileIdentity + evidence_sha256: str + envelope_file: _FileIdentity + envelope_sha256: str + + +class AuditArtifactStore: + """Own every audit artifact path and filesystem mutation for one root.""" + + def __init__( + self, + config: AuditArtifactConfig, + *, + canonicalizer: Callable[[dict[str, Any]], bytes] | None = None, + verifier: Callable[..., dict[str, Any]] | None = None, + wall_time: Callable[[], float] = time.time, + ) -> None: + live_bundle_retention = int(config.live_bundle_retention) + candidate_retention_seconds = int(config.candidate_retention_seconds) + share_segment_size = int(config.share_segment_size) + verifier_timeout_seconds = float(config.verifier_timeout_seconds) + if share_segment_size < 0: + raise ValueError("audit share segment size must be non-negative") + if verifier_timeout_seconds <= 0: + raise ValueError("verifier timeout must be positive") + configured_root = Path(config.root).expanduser().absolute() + configured_root.mkdir(parents=True, exist_ok=True) + root_stat = configured_root.lstat() + if not stat.S_ISDIR(root_stat.st_mode) or stat.S_ISLNK(root_stat.st_mode): + raise RuntimeError("audit artifact root must be a non-symlink directory") + root = configured_root.resolve(strict=True) + configured_evidence = Path(config.evidence_path).expanduser().absolute() + configured_evidence.parent.mkdir(parents=True, exist_ok=True) + evidence_path = configured_evidence.parent.resolve(strict=True) / configured_evidence.name + self._root = root + self._evidence_path = evidence_path + self._root_fd, self._root_identity = self._open_directory_authority(root) + try: + ( + self._publication_lock_fd, + self._publication_lock_identity, + ) = self._open_publication_lock_authority( + root, + self._root_fd, + self._root_identity, + ) + self._publication_lifecycle_lock = threading.RLock() + self._publication_inode_thread_lock = _publication_thread_lock( + ( + self._publication_lock_identity.device, + self._publication_lock_identity.inode, + ) + ) + self._publication_guard_owner: int | None = None + self._publication_guard_depth = 0 + ( + self._evidence_parent_fd, + self._evidence_parent_identity, + ) = self._open_directory_authority(evidence_path.parent) + except BaseException: + publication_lock_fd = getattr(self, "_publication_lock_fd", -1) + if publication_lock_fd >= 0: + os.close(publication_lock_fd) + self._publication_lock_fd = -1 + os.close(self._root_fd) + self._root_fd = -1 + raise + self._live_bundle_retention = live_bundle_retention + self._candidate_retention_seconds = candidate_retention_seconds + self._share_segment_size = share_segment_size + self._verifier_timeout_seconds = verifier_timeout_seconds + self._canonicalizer = canonicalizer + self._verifier = verifier + self._wall_time = wall_time + self._lock = threading.RLock() + self._closed = False + self._store_token = uuid.uuid4().hex + self._active_candidates: dict[str, tuple[Path, _FileIdentity | None]] = {} + self._latest_evidence: dict[str, Any] | None = None + self._current_envelope: Path | None = None + self._current_identity: AuditPublicationIdentity | None = None + self._evidence_state = "absent" + self._compatibility_evidence_override = False + self._invalidated_legacy_identity: AuditPublicationIdentity | None = None + self._legacy_proof_token: _LegacyProofToken | None = None + self._load_current_evidence() + + @property + def root(self) -> Path: + return self._root + + @property + def evidence_path(self) -> Path: + return self._evidence_path + + @property + def share_segment_size(self) -> int: + return self._share_segment_size + + @property + def live_bundle_retention(self) -> int: + return self._live_bundle_retention + + @property + def candidate_retention_seconds(self) -> int: + return self._candidate_retention_seconds + + @staticmethod + def _directory_identity(path: Path) -> tuple[int, int]: + value = path.lstat() + if not stat.S_ISDIR(value.st_mode) or stat.S_ISLNK(value.st_mode): + raise RuntimeError("audit artifact parent must be a non-symlink directory") + return value.st_dev, value.st_ino + + @staticmethod + def _open_directory_authority(path: Path) -> tuple[int, tuple[int, int]]: + fd = os.open( + path, + os.O_RDONLY + | getattr(os, "O_DIRECTORY", 0) + | getattr(os, "O_NOFOLLOW", 0), + ) + try: + value = os.fstat(fd) + if not stat.S_ISDIR(value.st_mode): + raise RuntimeError("audit artifact parent is not a directory") + identity = (value.st_dev, value.st_ino) + if AuditArtifactStore._directory_identity(path) != identity: + raise RuntimeError("audit artifact parent identity changed") + return fd, identity + except RuntimeError: + os.close(fd) + raise + except OSError as exc: + os.close(fd) + raise RuntimeError( + "audit artifact parent authority is invalid" + ) from exc + except BaseException: + os.close(fd) + raise + + @staticmethod + def _open_publication_lock_authority( + root: Path, + root_fd: int, + root_identity: tuple[int, int], + ) -> tuple[int, _FileIdentity]: + flags = os.O_RDWR | getattr(os, "O_NOFOLLOW", 0) + created = False + try: + try: + fd = os.open( + _PUBLICATION_LOCK_NAME, + flags | os.O_CREAT | os.O_EXCL, + 0o600, + dir_fd=root_fd, + ) + created = True + except FileExistsError: + fd = os.open(_PUBLICATION_LOCK_NAME, flags, dir_fd=root_fd) + except OSError as exc: + raise RuntimeError( + "audit publication lock cannot be opened safely" + ) from exc + try: + value = os.fstat(fd) + identity = _FileIdentity.from_stat(value) + if not stat.S_ISREG(value.st_mode): + raise RuntimeError("audit publication lock is not a regular file") + linked = os.stat( + _PUBLICATION_LOCK_NAME, + dir_fd=root_fd, + follow_symlinks=False, + ) + if not identity.matches(linked): + raise RuntimeError("audit publication lock identity changed") + root_value = os.fstat(root_fd) + if (root_value.st_dev, root_value.st_ino) != root_identity: + raise RuntimeError("audit artifact root authority is invalid") + if AuditArtifactStore._directory_identity(root) != root_identity: + raise RuntimeError("audit artifact root identity changed") + if created: + os.fsync(fd) + os.fsync(root_fd) + return fd, identity + except RuntimeError: + os.close(fd) + raise + except OSError as exc: + os.close(fd) + raise RuntimeError( + "audit publication lock authority is invalid" + ) from exc + except BaseException: + os.close(fd) + raise + + def close(self) -> None: + lock = getattr(self, "_lock", None) + publication_lock = getattr(self, "_publication_lifecycle_lock", None) + if lock is None: + self._close_unlocked() + return + if publication_lock is None: + with lock: + self._close_unlocked() + return + with publication_lock: + if ( + self._publication_guard_owner == threading.get_ident() + and self._publication_guard_depth > 0 + ): + raise RuntimeError( + "cannot close audit artifact store inside publication guard" + ) + with lock: + self._close_unlocked() + + def _close_unlocked(self) -> None: + if getattr(self, "_closed", False): + return + self._closed = True + for field in ( + "_publication_lock_fd", + "_root_fd", + "_evidence_parent_fd", + ): + fd = getattr(self, field, -1) + if isinstance(fd, int) and fd >= 0: + try: + os.close(fd) + except OSError: + pass + setattr(self, field, -1) + + def __del__(self) -> None: + try: + self.close() + except Exception: + pass + + def _validate_publication_lock_identity( + self, + *, + root: Path | None = None, + root_fd: int | None = None, + root_identity: tuple[int, int] | None = None, + lock_fd: int | None = None, + lock_identity: _FileIdentity | None = None, + ) -> None: + root = self._root if root is None else root + root_fd = self._root_fd if root_fd is None else root_fd + root_identity = ( + self._root_identity if root_identity is None else root_identity + ) + lock_fd = self._publication_lock_fd if lock_fd is None else lock_fd + lock_identity = ( + self._publication_lock_identity + if lock_identity is None + else lock_identity + ) + if getattr(self, "_closed", False) or root_fd < 0 or lock_fd < 0: + raise RuntimeError("audit artifact store is closed") + try: + if self._directory_identity(root) != root_identity: + raise RuntimeError("audit artifact root identity changed") + root_value = os.fstat(root_fd) + if (root_value.st_dev, root_value.st_ino) != root_identity: + raise RuntimeError("audit artifact root authority is invalid") + value = os.fstat(lock_fd) + if not lock_identity.matches(value): + raise RuntimeError("audit publication lock authority is invalid") + linked = os.stat( + _PUBLICATION_LOCK_NAME, + dir_fd=root_fd, + follow_symlinks=False, + ) + except OSError as exc: + raise RuntimeError("audit publication lock identity changed") from exc + if not lock_identity.matches(linked): + raise RuntimeError("audit publication lock identity changed") + + @contextmanager + def publication_order_guard(self) -> Iterator[None]: + """Serialize ordinal allocation/publication across threads/processes.""" + + with self._publication_order_guard(validate_on_exit=True): + yield + + @contextmanager + def _publication_order_guard( + self, + *, + validate_on_exit: bool, + ) -> Iterator[None]: + thread_id = threading.get_ident() + initial_identity = self._publication_lock_identity + _reject_cross_store_local_owner( + (initial_identity.device, initial_identity.inode), + thread_id=thread_id, + store_id=id(self), + ) + with self._publication_lifecycle_lock: + stable_identity = self._publication_lock_identity + _reject_cross_store_local_owner( + (stable_identity.device, stable_identity.inode), + thread_id=thread_id, + store_id=id(self), + ) + process_lock = self._publication_inode_thread_lock + with process_lock: + with self._publication_order_guard_locked( + validate_on_exit=validate_on_exit + ): + yield + + @contextmanager + def _publication_order_guard_locked( + self, + *, + validate_on_exit: bool, + ) -> Iterator[None]: + """Guard body with lifecycle and current inode thread locks held.""" + + thread_id = threading.get_ident() + if self._publication_guard_owner == thread_id: + self._publication_guard_depth += 1 + try: + self._validate_publication_lock_identity() + yield + if validate_on_exit: + self._validate_publication_lock_identity() + finally: + self._publication_guard_depth -= 1 + return + if self._publication_guard_owner is not None: + raise RuntimeError("audit publication guard ownership is inconsistent") + self._validate_publication_lock_identity() + lock_fd = self._publication_lock_fd + root = self._root + root_fd = self._root_fd + root_identity = self._root_identity + lock_identity = self._publication_lock_identity + local_key = (lock_identity.device, lock_identity.inode) + with _PUBLICATION_THREAD_LOCKS_GUARD: + local_owner = _PUBLICATION_LOCAL_OWNERS.get(local_key) + if local_owner is not None: + owner_thread, owner_store = local_owner + if owner_thread == thread_id and owner_store != id(self): + raise RuntimeError( + "audit publication guard is already held by another store " + "on this thread" + ) + raise RuntimeError("audit publication guard local ownership is inconsistent") + fcntl.flock(lock_fd, fcntl.LOCK_EX) + registered = False + try: + with _PUBLICATION_THREAD_LOCKS_GUARD: + local_owner = _PUBLICATION_LOCAL_OWNERS.get(local_key) + if local_owner is not None: + raise RuntimeError( + "audit publication guard local ownership is inconsistent" + ) + _PUBLICATION_LOCAL_OWNERS[local_key] = (thread_id, id(self)) + registered = True + self._publication_guard_owner = thread_id + self._publication_guard_depth = 1 + self._validate_publication_lock_identity( + root=root, + root_fd=root_fd, + root_identity=root_identity, + lock_fd=lock_fd, + lock_identity=lock_identity, + ) + yield + if validate_on_exit: + self._validate_publication_lock_identity( + root=root, + root_fd=root_fd, + root_identity=root_identity, + lock_fd=lock_fd, + lock_identity=lock_identity, + ) + finally: + self._publication_guard_depth = 0 + self._publication_guard_owner = None + if registered: + with _PUBLICATION_THREAD_LOCKS_GUARD: + if _PUBLICATION_LOCAL_OWNERS.get(local_key) == ( + thread_id, + id(self), + ): + del _PUBLICATION_LOCAL_OWNERS[local_key] + fcntl.flock(lock_fd, fcntl.LOCK_UN) + + def _require_publication_order_guard(self) -> None: + if ( + self._publication_guard_owner != threading.get_ident() + or self._publication_guard_depth <= 0 + ): + raise RuntimeError("audit publication order guard is required") + self._validate_publication_lock_identity() + + @contextmanager + def _prepared_publication_order_guard( + self, + *, + root: Path, + root_fd: int, + root_identity: tuple[int, int], + lock_fd: int, + lock_identity: _FileIdentity, + ) -> Iterator[None]: + current_identity = self._publication_lock_identity + if ( + lock_identity.device == current_identity.device + and lock_identity.inode == current_identity.inode + ): + self._validate_publication_lock_identity( + root=root, + root_fd=root_fd, + root_identity=root_identity, + lock_fd=lock_fd, + lock_identity=lock_identity, + ) + yield + self._validate_publication_lock_identity( + root=root, + root_fd=root_fd, + root_identity=root_identity, + lock_fd=lock_fd, + lock_identity=lock_identity, + ) + return + if root_identity == self._root_identity: + raise RuntimeError( + "audit publication lock changed within the current root" + ) + process_lock = _publication_thread_lock( + (lock_identity.device, lock_identity.inode) + ) + if not process_lock.acquire(blocking=False): + raise RuntimeError("new audit publication guard is busy") + flocked = False + try: + try: + fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as exc: + raise RuntimeError("new audit publication guard is busy") from exc + flocked = True + self._validate_publication_lock_identity( + root=root, + root_fd=root_fd, + root_identity=root_identity, + lock_fd=lock_fd, + lock_identity=lock_identity, + ) + yield + self._validate_publication_lock_identity( + root=root, + root_fd=root_fd, + root_identity=root_identity, + lock_fd=lock_fd, + lock_identity=lock_identity, + ) + finally: + if flocked: + fcntl.flock(lock_fd, fcntl.LOCK_UN) + process_lock.release() + + def _validate_root_identity(self) -> None: + if getattr(self, "_closed", False) or self._root_fd < 0: + raise RuntimeError("audit artifact store is closed") + try: + current = self._directory_identity(self._root) + except (FileNotFoundError, OSError) as exc: + raise RuntimeError("audit artifact root identity changed") from exc + if current != self._root_identity: + raise RuntimeError("audit artifact root identity changed") + value = os.fstat(self._root_fd) + if (value.st_dev, value.st_ino) != self._root_identity: + raise RuntimeError("audit artifact root authority is invalid") + + def _validate_evidence_parent_identity(self) -> None: + if getattr(self, "_closed", False) or self._evidence_parent_fd < 0: + raise RuntimeError("audit artifact store is closed") + try: + current = self._directory_identity(self._evidence_path.parent) + except (FileNotFoundError, OSError) as exc: + raise RuntimeError("audit evidence parent identity changed") from exc + if current != self._evidence_parent_identity: + raise RuntimeError("audit evidence parent identity changed") + value = os.fstat(self._evidence_parent_fd) + if (value.st_dev, value.st_ino) != self._evidence_parent_identity: + raise RuntimeError("audit evidence parent authority is invalid") + + def _owned_parent_fd(self, path: Path) -> int | None: + path = Path(path).absolute() + try: + parent = path.parent.resolve(strict=True) + except OSError: + parent = path.parent + if parent == self._root: + return self._root_fd + if parent == self._evidence_path.parent: + return self._evidence_parent_fd + return None + + def duplicate_root_directory_fd(self) -> int: + with self._lock: + self._validate_root_identity() + return os.dup(self._root_fd) + + def _owned_lstat(self, path: Path) -> os.stat_result: + fd = self._owned_parent_fd(path) + if fd is None: + raise RuntimeError("audit stat target has no directory authority") + return os.stat(Path(path).name, dir_fd=fd, follow_symlinks=False) + + def _owned_open(self, path: Path, flags: int, mode: int = 0o777) -> int: + fd = self._owned_parent_fd(path) + if fd is None: + mutation_flags = ( + os.O_WRONLY + | os.O_RDWR + | os.O_CREAT + | os.O_EXCL + | getattr(os, "O_TRUNC", 0) + ) + if flags & mutation_flags: + raise RuntimeError("audit mutation target has no directory authority") + raise RuntimeError("audit read target has no directory authority") + return os.open(Path(path).name, flags, mode, dir_fd=fd) + + def _owned_unlink(self, path: Path) -> None: + fd = self._owned_parent_fd(path) + if fd is None: + raise RuntimeError("audit unlink target has no directory authority") + os.unlink(Path(path).name, dir_fd=fd) + + def _owned_replace(self, source: Path, target: Path) -> None: + source_fd = self._owned_parent_fd(source) + target_fd = self._owned_parent_fd(target) + if source_fd is None or target_fd is None: + raise RuntimeError("audit replace target has no directory authority") + os.replace( + Path(source).name, + Path(target).name, + src_dir_fd=source_fd, + dst_dir_fd=target_fd, + ) + + def _owned_link(self, source: Path, target: Path) -> None: + source_fd = self._owned_parent_fd(source) + target_fd = self._owned_parent_fd(target) + if source_fd is None or target_fd is None: + raise RuntimeError("audit link target has no directory authority") + os.link( + Path(source).name, + Path(target).name, + src_dir_fd=source_fd, + dst_dir_fd=target_fd, + follow_symlinks=False, + ) + + def _read_owned_regular_bytes( + self, + path: Path, + ) -> tuple[bytes, os.stat_result]: + parent_fd = self._owned_parent_fd(path) + if parent_fd is not None: + self._validate_owned_parent(path) + fd = self._owned_open( + path, + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), + ) + result = self._read_regular_bytes_fd(fd) + if parent_fd is not None: + self._validate_owned_parent(path) + return result + + def _validate_owned_parent(self, path: Path) -> None: + path = Path(path).absolute() + try: + parent = path.parent.resolve(strict=True) + except OSError: + parent = path.parent + if parent == self._root: + self._validate_root_identity() + elif parent == self._evidence_path.parent: + self._validate_evidence_parent_identity() + else: + raise RuntimeError("audit target has no directory authority") + + def reconfigure( + self, + *, + root: Path | None = None, + evidence_path: Path | None = None, + live_bundle_retention: int | None = None, + candidate_retention_seconds: int | None = None, + share_segment_size: int | None = None, + canonicalizer: Callable[[dict[str, Any]], bytes] | None = None, + ) -> None: + if ( + self._publication_guard_owner == threading.get_ident() + and self._publication_guard_depth > 0 + ): + raise RuntimeError( + "cannot reconfigure audit artifact store inside publication guard" + ) + retired_root_fd: int | None = None + retired_publication_lock_fd: int | None = None + with self._publication_order_guard(validate_on_exit=False): + ( + retired_root_fd, + retired_publication_lock_fd, + ) = self._reconfigure_under_publication_guard( + root=root, + evidence_path=evidence_path, + live_bundle_retention=live_bundle_retention, + candidate_retention_seconds=candidate_retention_seconds, + share_segment_size=share_segment_size, + canonicalizer=canonicalizer, + ) + for retired_fd in (retired_publication_lock_fd, retired_root_fd): + if retired_fd is not None: + try: + os.close(retired_fd) + except OSError: + pass + + def _reconfigure_under_publication_guard( + self, + *, + root: Path | None, + evidence_path: Path | None, + live_bundle_retention: int | None, + candidate_retention_seconds: int | None, + share_segment_size: int | None, + canonicalizer: Callable[[dict[str, Any]], bytes] | None, + ) -> tuple[int | None, int | None]: + self._require_publication_order_guard() + with self._lock: + if self._closed: + raise RuntimeError("audit artifact store is closed") + if root is not None and self._active_candidates: + raise RuntimeError( + "cannot reconfigure audit root while candidates are active" + ) + size = self._share_segment_size + if share_segment_size is not None: + size = int(share_segment_size) + if size < 0: + raise ValueError( + "audit share segment size must be non-negative" + ) + new_root = self._root + new_root_fd: int | None = None + new_root_identity = self._root_identity + new_publication_lock_fd: int | None = None + new_publication_lock_identity = self._publication_lock_identity + new_evidence_path = self._evidence_path + new_evidence_parent_fd: int | None = None + new_evidence_parent_identity = self._evidence_parent_identity + try: + if root is not None: + candidate_root = Path(root).expanduser().absolute() + candidate_root.mkdir(parents=True, exist_ok=True) + new_root = candidate_root.resolve(strict=True) + new_root_fd, new_root_identity = ( + self._open_directory_authority(new_root) + ) + ( + new_publication_lock_fd, + new_publication_lock_identity, + ) = self._open_publication_lock_authority( + new_root, + new_root_fd, + new_root_identity, + ) + if evidence_path is not None: + candidate_evidence = Path(evidence_path).expanduser().absolute() + candidate_evidence.parent.mkdir(parents=True, exist_ok=True) + new_evidence_path = ( + candidate_evidence.parent.resolve(strict=True) + / candidate_evidence.name + ) + ( + new_evidence_parent_fd, + new_evidence_parent_identity, + ) = self._open_directory_authority( + new_evidence_path.parent + ) + except BaseException: + for prepared_fd in ( + new_publication_lock_fd, + new_root_fd, + new_evidence_parent_fd, + ): + if prepared_fd is not None: + os.close(prepared_fd) + raise + new_live_bundle_retention = ( + self._live_bundle_retention + if live_bundle_retention is None + else int(live_bundle_retention) + ) + new_candidate_retention_seconds = ( + self._candidate_retention_seconds + if candidate_retention_seconds is None + else int(candidate_retention_seconds) + ) + prepared_guard = ( + self._prepared_publication_order_guard( + root=new_root, + root_fd=new_root_fd, + root_identity=new_root_identity, + lock_fd=new_publication_lock_fd, + lock_identity=new_publication_lock_identity, + ) + if new_root_fd is not None + and new_publication_lock_fd is not None + else nullcontext() + ) + old_root = self._root + old_root_fd = self._root_fd + old_root_identity = self._root_identity + old_publication_lock_fd = self._publication_lock_fd + old_publication_lock_identity = self._publication_lock_identity + old_publication_inode_thread_lock = ( + self._publication_inode_thread_lock + ) + old_evidence_path = self._evidence_path + old_evidence_parent_fd = self._evidence_parent_fd + old_evidence_parent_identity = self._evidence_parent_identity + old_latest_evidence = self._latest_evidence + old_current_envelope = self._current_envelope + old_current_identity = self._current_identity + old_evidence_state = self._evidence_state + old_compatibility_override = self._compatibility_evidence_override + old_invalidated_legacy_identity = self._invalidated_legacy_identity + old_legacy_proof_token = self._legacy_proof_token + old_live_retention = self._live_bundle_retention + old_candidate_retention = self._candidate_retention_seconds + old_share_segment_size = self._share_segment_size + old_canonicalizer = self._canonicalizer + try: + with prepared_guard: + # Final old-authority validation is the linearization + # boundary. After this succeeds, only the prepared new + # authority may influence commit or rollback. + self._validate_publication_lock_identity( + root=old_root, + root_fd=old_root_fd, + root_identity=old_root_identity, + lock_fd=old_publication_lock_fd, + lock_identity=old_publication_lock_identity, + ) + if new_root_fd is not None: + self._root = new_root + self._root_fd = new_root_fd + self._root_identity = new_root_identity + assert new_publication_lock_fd is not None + self._publication_lock_fd = new_publication_lock_fd + self._publication_lock_identity = ( + new_publication_lock_identity + ) + self._publication_inode_thread_lock = _publication_thread_lock( + ( + new_publication_lock_identity.device, + new_publication_lock_identity.inode, + ) + ) + if new_evidence_parent_fd is not None: + self._evidence_path = new_evidence_path + self._evidence_parent_fd = new_evidence_parent_fd + self._evidence_parent_identity = ( + new_evidence_parent_identity + ) + if root is not None or evidence_path is not None: + self._latest_evidence = None + self._current_envelope = None + self._current_identity = None + self._evidence_state = "absent" + self._compatibility_evidence_override = False + self._invalidated_legacy_identity = None + self._legacy_proof_token = None + self._live_bundle_retention = new_live_bundle_retention + self._candidate_retention_seconds = ( + new_candidate_retention_seconds + ) + self._share_segment_size = size + if canonicalizer is not None: + self._canonicalizer = canonicalizer + if root is not None or evidence_path is not None: + # Readers must see either the complete old authority/cache + # or the complete new authority/cache. Both old and new + # process guards remain held across switch and reload. + self._reload_current_evidence_locked() + if new_evidence_parent_fd is not None: + try: + os.close(old_evidence_parent_fd) + except OSError: + pass + return ( + old_root_fd if new_root_fd is not None else None, + ( + old_publication_lock_fd + if new_publication_lock_fd is not None + else None + ), + ) + except BaseException: + self._root = old_root + self._root_fd = old_root_fd + self._root_identity = old_root_identity + self._publication_lock_fd = old_publication_lock_fd + self._publication_lock_identity = old_publication_lock_identity + self._publication_inode_thread_lock = ( + old_publication_inode_thread_lock + ) + self._evidence_path = old_evidence_path + self._evidence_parent_fd = old_evidence_parent_fd + self._evidence_parent_identity = old_evidence_parent_identity + self._latest_evidence = old_latest_evidence + self._current_envelope = old_current_envelope + self._current_identity = old_current_identity + self._evidence_state = old_evidence_state + self._compatibility_evidence_override = ( + old_compatibility_override + ) + self._invalidated_legacy_identity = ( + old_invalidated_legacy_identity + ) + self._legacy_proof_token = old_legacy_proof_token + self._live_bundle_retention = old_live_retention + self._candidate_retention_seconds = old_candidate_retention + self._share_segment_size = old_share_segment_size + self._canonicalizer = old_canonicalizer + for prepared_fd in ( + new_publication_lock_fd, + new_root_fd, + new_evidence_parent_fd, + ): + if prepared_fd is not None: + try: + os.close(prepared_fd) + except OSError: + pass + raise + + def publication_sequence_floor(self) -> int: + with self._lock: + if not self._directory_authority_is_current(): + return 0 + return ( + self._current_identity.sequence + if self._current_identity is not None + and self._evidence_state in {"valid", "legacy_proven"} + else 0 + ) + + def legacy_evidence_identity(self) -> AuditPublicationIdentity | None: + with self._lock: + if not self._directory_authority_is_current(): + return None + if ( + self._publication_guard_owner == threading.get_ident() + and self._publication_guard_depth > 0 + and not self._compatibility_evidence_override + ): + self._reload_current_evidence_locked() + if self._evidence_state not in { + "legacy", + "legacy_unproven", + } or self._current_identity is None: + return None + return self._current_identity + + def _directory_authority_is_current(self) -> bool: + try: + self._validate_root_identity() + self._validate_evidence_parent_identity() + return True + except RuntimeError: + # A transient pathname replacement revokes authority while it is + # present, but the pinned fd/cache can safely recover if the exact + # original directory inode is restored. Malformed evidence uses + # the separate sticky `invalid` state. + return False + + @staticmethod + def artifact_kind(name: str) -> str: + if _BODY_RE.fullmatch(name): + return "body" + segment = _SHARE_CONTENT_RE.fullmatch(name) or _SHARE_SLOT_RE.fullmatch(name) + if segment: + return ( + "share_segment" + if int(segment.group("first")) <= int(segment.group("last")) + else "other" + ) + if ( + _CANDIDATE_RE.fullmatch(name) + or _LEGACY_CANDIDATE_RE.fullmatch(name) + or _LEGACY_HIDDEN_CANDIDATE_RE.fullmatch(name) + ): + return "candidate" + if _LIVE_RE.fullmatch(name): + return "live_bundle" + return "other" + + def metrics_snapshot(self) -> dict[str, dict[str, int] | int]: + metrics: dict[str, dict[str, int] | int] = { + kind: {"files": 0, "bytes": 0} + for kind in ( + "body", + "share_segment", + "live_bundle", + "candidate", + "other", + ) + } + metrics["scan_error"] = 0 + try: + self._validate_root_identity() + paths = [self._root / name for name in os.listdir(self._root_fd)] + except (OSError, RuntimeError): + metrics["scan_error"] = 1 + return metrics + for path in paths: + if path.name == _PUBLICATION_LOCK_NAME: + continue + try: + value = self._owned_lstat(path) + if not stat.S_ISREG(value.st_mode): + continue + except RuntimeError: + metrics["scan_error"] = 1 + break + except OSError: + metrics["scan_error"] = 1 + continue + kind = self.artifact_kind(path.name) + bucket = metrics[kind] + assert isinstance(bucket, dict) + bucket["files"] += 1 + bucket["bytes"] += value.st_size + try: + self._validate_root_identity() + except RuntimeError: + metrics["scan_error"] = 1 + return metrics + + def issue_candidate(self, *, block_hash: str) -> OwnedCandidateArtifact: + block_hash = _canonical_hex(block_hash, name="block_hash") + with self._lock: + self._validate_root_identity() + for _attempt in range(16): + token = uuid.uuid4().hex + path = self._root / ( + f".prism-live-audit-bundle-candidate-{block_hash}-{token}.json.tmp" + ) + try: + self._owned_lstat(path) + except FileNotFoundError: + candidate = OwnedCandidateArtifact( + path=path, + block_hash=block_hash, + token=token, + store_token=self._store_token, + ) + self._active_candidates[token] = (path, None) + return candidate + raise RuntimeError("could not allocate an absent audit candidate path") + + def _require_candidate( + self, + candidate: OwnedCandidateArtifact, + ) -> tuple[Path, _FileIdentity | None]: + self._validate_root_identity() + if candidate.store_token != self._store_token: + raise RuntimeError("candidate does not belong to this audit store") + if not _CANDIDATE_RE.fullmatch(candidate.path.name): + raise RuntimeError("candidate has an invalid owned filename") + if candidate.path.parent != self._root: + raise RuntimeError("candidate escapes the audit artifact root") + current = self._active_candidates.get(candidate.token) + if current is None or current[0] != candidate.path: + raise RuntimeError("candidate is no longer active") + return current + + def adopt_created_candidate( + self, + candidate: OwnedCandidateArtifact, + ) -> None: + raise RuntimeError( + "pathname adoption is forbidden; transfer the exact open compiler inode" + ) + + def adopt_compiler_candidate( + self, + candidate: OwnedCandidateArtifact, + *, + path: Path, + value: os.stat_result, + ) -> None: + """Transfer the exact still-open inode created exclusively by J1.""" + + with self._lock: + expected_path, identity = self._require_candidate(candidate) + if identity is not None or Path(path) != expected_path: + raise RuntimeError("compiler candidate transfer is invalid") + transferred = _FileIdentity.from_stat(value) + current = self._owned_lstat(expected_path) + if not transferred.matches(current): + raise RuntimeError("compiler candidate identity changed before transfer") + self._active_candidates[candidate.token] = (expected_path, transferred) + + def write_compatibility_candidate( + self, + candidate: OwnedCandidateArtifact, + bundle: Mapping[str, Any], + ) -> Path: + payload = _json_bytes(bundle) + with self._lock: + path, identity = self._require_candidate(candidate) + if identity is not None: + raise RuntimeError("candidate already exists") + created = False + try: + fd = self._owned_open( + path, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + created = True + opened = os.fstat(fd) + if not stat.S_ISREG(opened.st_mode): + raise RuntimeError("candidate is not a regular file") + candidate_identity = _FileIdentity.from_stat(opened) + with self._lock: + self._require_candidate(candidate) + self._active_candidates[candidate.token] = ( + path, + candidate_identity, + ) + with os.fdopen(fd, "wb") as handle: + try: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + finally: + finalized_identity = _FileIdentity.from_stat( + os.fstat(handle.fileno()) + ) + with self._lock: + current = self._active_candidates.get(candidate.token) + if current == (path, candidate_identity): + self._active_candidates[candidate.token] = ( + path, + finalized_identity, + ) + self._validate_root_identity() + return path + except BaseException: + if created: + self._unlink_candidate_if_same(candidate, allow_unadopted=True) + else: + self.release_candidate(candidate) + raise + + def release_candidate(self, candidate: OwnedCandidateArtifact) -> None: + """Release an uncreated reservation without acquiring delete authority.""" + + with self._lock: + self._require_candidate(candidate) + self._active_candidates.pop(candidate.token, None) + + def _unlink_candidate_if_same( + self, + candidate: OwnedCandidateArtifact, + *, + allow_unadopted: bool = False, + ) -> None: + with self._lock: + self._validate_root_identity() + current = self._active_candidates.get(candidate.token) + if current is None or current[0] != candidate.path: + return + expected = current[1] + try: + value = self._owned_lstat(candidate.path) + except FileNotFoundError: + self._active_candidates.pop(candidate.token, None) + return + if not stat.S_ISREG(value.st_mode): + self._active_candidates.pop(candidate.token, None) + return + if expected is not None and not expected.matches(value): + self._active_candidates.pop(candidate.token, None) + return + if expected is None and not allow_unadopted: + return + self._remove_identity_safe( + candidate.path, + _FileIdentity.from_stat(value), + ) + self._active_candidates.pop(candidate.token, None) + + def discard_candidate(self, candidate: OwnedCandidateArtifact) -> None: + with self._lock: + current = self._active_candidates.get(candidate.token) + if current is not None and current[1] is None: + # Reservation alone never grants deletion authority. J1 must + # return and transfer the exact created inode first. + self._active_candidates.pop(candidate.token, None) + return + self._unlink_candidate_if_same(candidate) + + @staticmethod + def verified_canonical_bundle_path( + candidate_bundle_path: Path, + report: Mapping[str, Any], + ) -> Path | None: + expected = str(report["audit_bundle_sha256_hex"]).lower() + payload, _value = AuditArtifactStore.read_regular_bytes( + Path(candidate_bundle_path) + ) + return ( + Path(candidate_bundle_path) + if _sha256_bytes(payload) == expected + else None + ) + + def verify_candidate( + self, + candidate: OwnedCandidateArtifact, + *, + coinbase_tx_hex: str, + expected_coinbase_value_sats: int, + trusted_writer_public_key_hex: str, + trust_source: str = "configured", + expected_block_height: int | None = None, + verifier: Callable[..., dict[str, Any]] | None = None, + ) -> VerifiedAuditBundle: + key = _canonical_hex( + trusted_writer_public_key_hex, + name="ledger writer public key", + ) + coinbase_tx_hex = _canonical_hex_bytes( + coinbase_tx_hex, + name="coinbase_tx_hex", + ) + with self._lock: + path, identity = self._require_candidate(candidate) + if identity is None: + raise RuntimeError( + "candidate compiler inode was not transferred before verification" + ) + before = self._owned_lstat(path) + if not identity.matches(before): + raise RuntimeError("candidate identity changed before verification") + before_bytes, before_descriptor_stat = self._read_owned_regular_bytes(path) + if not identity.matches(before_descriptor_stat): + raise RuntimeError("candidate identity changed before verification") + literal_before = _sha256_bytes(before_bytes) + verify = verifier or self._verifier or self.verify_bundle + snapshot_fd, verification_path = self._open_verification_snapshot( + before_bytes + ) + try: + report = verify( + verification_path, + coinbase_tx_hex, + key, + expected_coinbase_value_sats=expected_coinbase_value_sats, + expected_block_height=expected_block_height, + ) + finally: + os.close(snapshot_fd) + after_bytes, after = self._read_owned_regular_bytes(path) + literal_after = _sha256_bytes(after_bytes) + if ( + not identity.matches(after) + or before.st_size != after.st_size + or before.st_mtime_ns != after.st_mtime_ns + or not hmac.compare_digest(literal_before, literal_after) + ): + raise RuntimeError("candidate changed during verification") + normalized = self._validate_verifier_report( + report, + coinbase_tx_hex=coinbase_tx_hex, + expected_coinbase_value_sats=expected_coinbase_value_sats, + expected_block_height=expected_block_height, + ) + expected_digest = str(normalized["audit_bundle_sha256_hex"]) + verification_identity = self.build_verification_identity( + trust_source=trust_source, + trusted_writer_public_key_hex=key, + literal_sha256=literal_after, + literal_byte_len=after.st_size, + report=normalized, + ) + verified = VerifiedAuditBundle( + candidate=candidate, + report=copy.deepcopy(normalized), + literal_sha256=literal_after, + byte_length=after.st_size, + device=after.st_dev, + inode=after.st_ino, + mtime_ns=after.st_mtime_ns, + canonical_copy_eligible=hmac.compare_digest( + literal_after, + expected_digest, + ), + verification_identity=verification_identity, + ) + self._validate_root_identity() + return verified + + def require_current_verified_candidate( + self, + verified: VerifiedAuditBundle, + candidate: OwnedCandidateArtifact, + ) -> None: + """Reject reuse of a successful result across ephemeral candidates.""" + + with self._lock: + path, identity = self._require_candidate(candidate) + if verified.candidate != candidate or identity is None: + raise RuntimeError("verified audit result belongs to another candidate") + if ( + identity.device != verified.device + or identity.inode != verified.inode + or path != candidate.path + ): + raise RuntimeError("verified audit result candidate identity changed") + + @staticmethod + def build_verification_identity( + *, + trust_source: str, + trusted_writer_public_key_hex: str, + literal_sha256: str, + literal_byte_len: int, + report: Mapping[str, Any], + ) -> dict[str, Any]: + if trust_source not in {"configured", "embedded_test_only"}: + raise RuntimeError("audit verifier trust source is invalid") + normalized_report = AuditArtifactStore._normalize_report_identity(report) + byte_len = literal_byte_len + if isinstance(byte_len, bool) or not isinstance(byte_len, int) or byte_len < 0: + raise RuntimeError("audit verifier literal byte length is invalid") + base = { + "schema": VERIFICATION_IDENTITY_SCHEMA, + "trust_source": trust_source, + "ledger_writer_public_key_hex": _canonical_hex( + trusted_writer_public_key_hex, + name="ledger writer public key", + ), + "literal_sha256_hex": _canonical_hex( + literal_sha256, + name="verified literal sha256", + ), + "literal_byte_len": byte_len, + "report": normalized_report, + } + return { + **base, + "identity_sha256_hex": _sha256_bytes( + _json_bytes(base, sort_keys=True) + ), + } + + @staticmethod + def _normalize_verification_identity( + value: object, + *, + report: Mapping[str, Any], + ) -> dict[str, Any]: + if not isinstance(value, dict): + raise RuntimeError("audit verification identity is required") + normalized = AuditArtifactStore.build_verification_identity( + trust_source=str(value.get("trust_source") or ""), + trusted_writer_public_key_hex=str( + value.get("ledger_writer_public_key_hex") or "" + ), + literal_sha256=str(value.get("literal_sha256_hex") or ""), + literal_byte_len=value.get("literal_byte_len"), + report=report, + ) + if value != normalized: + raise RuntimeError("audit verification identity mismatch") + return normalized + + @staticmethod + def _legacy_verification_marker( + *, + identity: AuditPublicationIdentity, + report: Mapping[str, Any], + ) -> dict[str, Any]: + normalized_report = AuditArtifactStore._normalize_report_identity(report) + return { + "schema": LEGACY_VERIFICATION_UNAVAILABLE_SCHEMA, + "reason": "evidence-predates-deterministic-verification-identity", + "block_hash": identity.block_hash, + "block_height": identity.block_height, + "audit_publication_sequence": identity.sequence, + "report_identity_sha256_hex": _sha256_bytes( + _json_bytes(normalized_report, sort_keys=True) + ), + } + + @staticmethod + def _normalize_legacy_verification_marker( + value: object, + *, + identity: AuditPublicationIdentity, + report: Mapping[str, Any], + ) -> dict[str, Any]: + if not isinstance(value, dict): + raise RuntimeError("legacy audit verification marker is required") + expected = AuditArtifactStore._legacy_verification_marker( + identity=identity, + report=report, + ) + if value != expected: + raise RuntimeError("legacy audit verification marker mismatch") + return expected + + @staticmethod + def _loaded_evidence_state( + payload: Mapping[str, Any], + identity: AuditPublicationIdentity, + ) -> str: + if identity.sequence == 0: + return "legacy" + verification = payload.get("audit_verification_identity") + if ( + isinstance(verification, dict) + and verification.get("schema") + == LEGACY_VERIFICATION_UNAVAILABLE_SCHEMA + ): + # The on-disk ledger proof is a hint only. The coordinator must + # revalidate exact active ledger state on every process start. + return "legacy_unproven" + return "valid" + + def _open_verification_snapshot(self, payload: bytes) -> tuple[int, Path]: + path = self._root / f".prism-audit-verification-{uuid.uuid4().hex}.tmp" + identity: _FileIdentity | None = None + read_fd: int | None = None + try: + fd = self._owned_open( + path, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + identity = _FileIdentity.from_stat(os.fstat(fd)) + with os.fdopen(fd, "wb") as handle: + try: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + finally: + identity = _FileIdentity.from_stat(os.fstat(handle.fileno())) + read_fd = self._owned_open( + path, + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), + ) + if not identity.matches(os.fstat(read_fd)): + raise RuntimeError("verification snapshot identity changed") + if not self._remove_identity_safe(path, identity): + raise RuntimeError("verification snapshot path was replaced") + self._fsync_directory(self._root) + return read_fd, Path(f"/dev/fd/{read_fd}") + except BaseException: + if read_fd is not None: + os.close(read_fd) + if identity is not None: + self._remove_identity_safe(path, identity) + raise + + @staticmethod + def _validate_verifier_report( + report: object, + *, + coinbase_tx_hex: str, + expected_coinbase_value_sats: int, + expected_block_height: int | None, + ) -> dict[str, Any]: + if not isinstance(report, dict): + raise RuntimeError("audit verifier returned a non-object report") + result = AuditArtifactStore._normalize_report_identity(report) + if not hmac.compare_digest(result["coinbase_tx_hex"], coinbase_tx_hex): + raise RuntimeError("audit verifier coinbase does not match submitted coinbase") + if int(result["coinbase_value_sats"]) != int(expected_coinbase_value_sats): + raise RuntimeError("audit verifier coinbase value does not match expected value") + if ( + expected_block_height is not None + and result["block_height"] != int(expected_block_height) + ): + raise RuntimeError("audit verifier block height does not match expected height") + return result + + @staticmethod + def _normalize_report_identity(report: Mapping[str, Any]) -> dict[str, Any]: + result = dict(report) + if result.get("schema") != VERIFICATION_REPORT_SCHEMA: + raise RuntimeError("audit verifier report schema is invalid") + block_height = result.get("block_height") + if isinstance(block_height, bool) or not isinstance(block_height, int): + raise RuntimeError("audit verifier block height is invalid") + if block_height < 0: + raise RuntimeError("audit verifier block height is invalid") + result["audit_bundle_sha256_hex"] = _canonical_hex( + result.get("audit_bundle_sha256_hex"), + name="audit bundle sha256", + ) + result["coinbase_tx_hex"] = _canonical_hex_bytes( + result.get("coinbase_tx_hex"), + name="report coinbase_tx_hex", + ) + report_value = result.get("coinbase_value_sats") + if isinstance(report_value, bool) or not isinstance(report_value, int): + raise RuntimeError("audit verifier coinbase value is required") + if result["coinbase_value_sats"] < 0: + raise RuntimeError("audit verifier coinbase value is invalid") + for key in ( + "reward_manifest_sha256_hex", + "payout_policy_manifest_sha256_hex", + "prism_audit_commitment_leaf_hex", + "audit_commitment_root_hex", + "coinbase_txid", + "coinbase_wtxid", + "coinbase_manifest_sha256_hex", + ): + result[key] = _canonical_hex(result.get(key), name=key) + for key in ( + "min_output_sats", + "onchain_output_count", + "accrued_account_count", + ): + value = result.get(key) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise RuntimeError(f"audit verifier {key} is invalid") + return result + + def verify_bundle( + self, + bundle_path: Path, + coinbase_tx_hex: str, + ledger_writer_public_key_hex: str, + *, + expected_coinbase_value_sats: int, + expected_block_height: int | None = None, + ) -> dict[str, Any]: + del expected_block_height + inherited_fds: tuple[int, ...] = () + match = re.fullmatch(r"/dev/fd/(?P[0-9]+)", str(bundle_path)) + if match is not None: + inherited_fds = (int(match.group("fd")),) + process = subprocess.Popen( + prism_tool_command("qbit-prism-audit-verify") + + [ + str(bundle_path), + "--coinbase-tx-hex", + coinbase_tx_hex, + "--ledger-writer-public-key-hex", + ledger_writer_public_key_hex, + "--expected-coinbase-value-sats", + str(expected_coinbase_value_sats), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + pass_fds=inherited_fds, + start_new_session=True, + ) + outputs = {"stdout": bytearray(), "stderr": bytearray()} + selector = selectors.DefaultSelector() + assert process.stdout is not None + assert process.stderr is not None + for name, stream in ( + ("stdout", process.stdout), + ("stderr", process.stderr), + ): + os.set_blocking(stream.fileno(), False) + selector.register(stream, selectors.EVENT_READ, name) + deadline = time.monotonic() + self._verifier_timeout_seconds + try: + while selector.get_map() or process.poll() is None: + remaining_seconds = deadline - time.monotonic() + if remaining_seconds <= 0: + self._kill_verifier_group(process) + process.wait() + raise RuntimeError("qbit-prism-audit-verify timed out") + events = selector.select(min(0.05, remaining_seconds)) + for key, _mask in events: + try: + chunk = os.read(key.fileobj.fileno(), 64 * 1024) + except (BlockingIOError, InterruptedError): + continue + if not chunk: + selector.unregister(key.fileobj) + continue + output = outputs[str(key.data)] + remaining = MAX_VERIFIER_OUTPUT_BYTES + 1 - len(output) + if remaining > 0: + output.extend(chunk[:remaining]) + if len(output) > MAX_VERIFIER_OUTPUT_BYTES: + self._kill_verifier_group(process) + process.wait() + raise RuntimeError( + "qbit-prism-audit-verify output exceeded limit" + ) + process.wait() + except BaseException: + if process.poll() is None: + self._kill_verifier_group(process) + process.wait() + raise + finally: + selector.close() + process.stdout.close() + process.stderr.close() + stdout = bytes(outputs["stdout"]) + stderr = bytes(outputs["stderr"]) + if process.returncode != 0: + raise RuntimeError( + "qbit-prism-audit-verify failed: " + + stderr.decode(errors="replace").strip() + ) + try: + report = json.loads(stdout) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError("qbit-prism-audit-verify returned invalid JSON") from exc + if not isinstance(report, dict): + raise RuntimeError("qbit-prism-audit-verify returned a non-object report") + return report + + @staticmethod + def _kill_verifier_group(process: subprocess.Popen[bytes]) -> None: + try: + os.killpg(process.pid, signal.SIGKILL) + except (ProcessLookupError, PermissionError, AttributeError): + try: + process.kill() + except (ProcessLookupError, PermissionError): + pass + + @staticmethod + def trusted_writer_key( + configured_key: str | None, + bundle: Mapping[str, Any], + *, + allow_embedded_test_key: bool, + ) -> str: + if configured_key is not None: + return _canonical_hex( + configured_key, + name="configured ledger writer public key", + ) + if not allow_embedded_test_key: + raise RuntimeError("configured ledger writer public key is required") + try: + embedded = bundle["ledger_window_attestation"]["signature"][ + "public_key_hex" + ] + except (KeyError, TypeError) as exc: + raise RuntimeError("audit bundle has no embedded test writer key") from exc + return _canonical_hex(embedded, name="bundle ledger public key") + + def live_envelope_path(self, *, block_height: int, block_hash: str) -> Path: + block_hash = _canonical_hex(block_hash, name="block_hash") + if isinstance(block_height, bool) or not isinstance(block_height, int): + raise ValueError("block_height must be an integer") + if block_height < 0: + raise ValueError("block_height must be non-negative") + return self._root / ( + f"prism-live-audit-bundle-{block_height}-{block_hash}.json" + ) + + def body_path(self, block_hash: str, audit_bundle_sha256: str) -> Path: + block_hash = _canonical_hex(block_hash, name="block_hash") + digest = _canonical_hex( + audit_bundle_sha256, + name="audit_bundle_sha256", + ) + return self._root / f"prism-audit-bundle-body-{block_hash}-{digest}.json" + + def resolve_owned_path(self, body_uri: object) -> Path: + self._validate_root_identity() + raw = Path(str(body_uri)).expanduser() + if not raw.is_absolute() and ( + len(raw.parts) != 1 or raw.name in {".", ".."} + ): + raise RuntimeError( + f"audit bundle body path escapes audit body store: {body_uri}" + ) + path = raw if raw.is_absolute() else self._root / raw + path = path.absolute() + try: + path = path.parent.resolve(strict=True) / path.name + except OSError as exc: + raise RuntimeError( + f"audit bundle body path is not resolvable: {body_uri}" + ) from exc + if path.parent != self._root: + raise RuntimeError( + f"audit bundle body path escapes audit body store: {body_uri}" + ) + if self.artifact_kind(path.name) not in {"body", "share_segment"}: + raise RuntimeError(f"audit bundle body path is not owned: {body_uri}") + self._validate_root_identity() + return path + + def publish_success( + self, + *, + identity: AuditPublicationIdentity, + publication_floor_sequence: int, + report: Mapping[str, Any], + persistence: Mapping[str, Any], + evidence: Mapping[str, Any], + verification_identity: Mapping[str, Any], + created_at: str, + ) -> AuditPublication: + self._require_publication_order_guard() + block_hash = _canonical_hex(identity.block_hash, name="block_hash") + if block_hash != identity.block_hash: + raise ValueError("publication identity block_hash must be canonical") + if identity.sequence <= 0 or identity.block_height < 0: + raise ValueError("publication identity is invalid") + if ( + isinstance(publication_floor_sequence, bool) + or not isinstance(publication_floor_sequence, int) + or publication_floor_sequence < 0 + ): + raise ValueError("publication floor sequence must be a non-negative integer") + if identity.sequence > publication_floor_sequence: + raise RuntimeError( + "audit publication sequence exceeds the durable ledger floor" + ) + envelope_path = self.live_envelope_path( + block_height=identity.block_height, + block_hash=block_hash, + ) + with self._lock: + self._validate_root_identity() + self._validate_evidence_parent_identity() + if not self._compatibility_evidence_override: + invalidated_legacy = self._invalidated_legacy_identity + self._reload_current_evidence_locked() + if ( + invalidated_legacy is not None + and self._evidence_state in {"legacy", "legacy_unproven"} + and self._current_identity == invalidated_legacy + ): + # Explicit invalidation remains repairable only while disk + # still names the same unproven legacy publication. A peer + # valid publication always wins this reconciliation. + self._latest_evidence = None + self._current_envelope = None + self._current_identity = None + self._evidence_state = "invalid" + else: + self._invalidated_legacy_identity = None + current = self._current_identity + if self._evidence_state in {"legacy", "legacy_unproven"}: + raise RuntimeError( + "current audit evidence lacks a validated publication identity" + ) + if current is not None: + exact = current == identity + if ( + self._evidence_state == "legacy_proven" + and identity.sequence <= current.sequence + ): + raise RuntimeError( + "legacy audit evidence is never exact-replay-equivalent" + ) + if identity.sequence == current.sequence and not exact: + raise RuntimeError("audit publication identity conflict") + if identity.sequence < current.sequence: + return AuditPublication( + identity=identity, + envelope_path=envelope_path, + evidence=copy.deepcopy(dict(evidence)), + published=False, + ) + if exact and self._latest_evidence is not None: + expected_envelope = self._build_live_envelope( + identity=identity, + report=report, + persistence=persistence, + created_at=created_at, + ) + replay_annotations = copy.deepcopy(dict(evidence)) + # These global counters are observational annotations, not + # immutable block identity. Reuse the originally durable + # values so an outbox replay after more shares arrive does + # not strand an otherwise exact publication. + for annotation in ( + "accepted_share_count", + "distinct_miner_count", + ): + if annotation in self._latest_evidence: + replay_annotations[annotation] = ( + self._latest_evidence[annotation] + ) + replay_persistence = copy.deepcopy(dict(persistence)) + current_persistence = self._latest_evidence.get( + "persistence" + ) + if isinstance(current_persistence, dict) and ( + "share_count" in current_persistence + ): + # persist_accepted_block reports the current global + # accepted-share count. It is observational and may + # advance between a committed block and outbox replay. + replay_persistence["share_count"] = ( + current_persistence["share_count"] + ) + replay_evidence = self._normalized_durable_evidence( + identity=identity, + envelope_path=envelope_path, + report=report, + persistence=replay_persistence, + evidence=replay_annotations, + verification_identity=verification_identity, + ) + if self._latest_evidence != replay_evidence: + raise RuntimeError( + "exact audit publication replay payload conflict" + ) + envelope_present = True + evidence_present = True + try: + disk_envelope_bytes, _value = ( + self._read_owned_regular_bytes(envelope_path) + ) + except FileNotFoundError: + envelope_present = False + else: + try: + disk_envelope = json.loads(disk_envelope_bytes) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError( + "existing live audit envelope is invalid" + ) from exc + if not isinstance(disk_envelope, dict): + raise RuntimeError( + "existing live audit envelope is invalid" + ) + if { + key: value + for key, value in disk_envelope.items() + if key != "created_at" + } != { + key: value + for key, value in expected_envelope.items() + if key != "created_at" + }: + raise RuntimeError( + "existing live audit envelope conflicts" + ) + try: + disk_evidence_bytes, _value = ( + self._read_owned_regular_bytes(self._evidence_path) + ) + except FileNotFoundError: + evidence_present = False + else: + try: + disk_evidence = json.loads(disk_evidence_bytes) + except (UnicodeDecodeError, json.JSONDecodeError): + # The evidence path is mutable publication state, + # not operator-owned preservation authority. A + # ledger-proven exact replay may repair malformed + # bytes while retaining the exact immutable + # envelope checked above. + evidence_present = False + else: + if disk_evidence != replay_evidence: + try: + self._validate_evidence(disk_evidence) + except ( + OSError, + RuntimeError, + TypeError, + ValueError, + ): + evidence_present = False + else: + raise RuntimeError( + "existing live audit evidence conflicts" + ) + if envelope_present and evidence_present: + self._fsync_directory(envelope_path.parent) + self._fsync_directory(self._evidence_path.parent) + return AuditPublication( + identity=identity, + envelope_path=envelope_path, + evidence=copy.deepcopy(self._latest_evidence), + published=False, + ) + if identity.sequence != publication_floor_sequence: + raise RuntimeError( + "audit publication repair is behind the durable ledger floor" + ) + # A missing exact entry is repaired by the same atomic + # primitives as a first publication. Preserve the durable + # observational annotations while falling through. + persistence = replay_persistence + evidence = replay_annotations + if identity.sequence != publication_floor_sequence: + raise RuntimeError( + "audit publication is behind the durable ledger floor" + ) + envelope = self._build_live_envelope( + identity=identity, + report=report, + persistence=persistence, + created_at=created_at, + ) + try: + old_envelope, _old_envelope_stat = self._read_owned_regular_bytes( + envelope_path + ) + except FileNotFoundError: + old_envelope = None + reuse_envelope = False + if old_envelope is not None: + try: + existing_envelope = json.loads(old_envelope) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError( + "existing live audit envelope is invalid" + ) from exc + if not isinstance(existing_envelope, dict): + raise RuntimeError("existing live audit envelope is invalid") + immutable_existing = { + key: value + for key, value in existing_envelope.items() + if key != "created_at" + } + immutable_new = { + key: value for key, value in envelope.items() if key != "created_at" + } + if immutable_existing != immutable_new: + raise RuntimeError("existing live audit envelope conflicts") + reuse_envelope = True + durable_evidence = self._normalized_durable_evidence( + identity=identity, + envelope_path=envelope_path, + report=report, + persistence=persistence, + evidence=evidence, + verification_identity=verification_identity, + ) + try: + old_evidence, _old_evidence_stat = ( + self._read_owned_regular_bytes(self._evidence_path) + ) + except FileNotFoundError: + old_evidence = None + reuse_evidence = False + if old_evidence is not None: + try: + existing_evidence = json.loads(old_evidence) + except (UnicodeDecodeError, json.JSONDecodeError): + # Invalid bytes at the mutable evidence path may be + # repaired by this ledger-proven publication. + pass + else: + if existing_evidence == durable_evidence: + reuse_evidence = True + else: + try: + _payload, _path, existing_identity = ( + self._validate_evidence(existing_evidence) + ) + except (OSError, RuntimeError, TypeError, ValueError): + # Canonical JSON can still be structurally or + # cryptographically invalid. It has the same + # repair policy as malformed bytes. + pass + else: + if identity.sequence <= existing_identity.sequence: + raise RuntimeError( + "existing live audit evidence conflicts" + ) + if not reuse_envelope: + published_envelope_identity = self._write_mutable_json( + envelope_path, + envelope, + sort_keys=True, + ) + else: + published_envelope_identity = None + self._fsync_directory(envelope_path.parent) + published_evidence_identity: _FileIdentity | None = None + try: + if reuse_evidence: + self._fsync_directory(self._evidence_path.parent) + else: + published_evidence_identity = self._write_mutable_json( + self._evidence_path, + durable_evidence, + sort_keys=True, + ) + self._validate_root_identity() + self._validate_evidence_parent_identity() + self._require_publication_order_guard() + except BaseException: + if published_evidence_identity is not None: + try: + self._rollback_mutable_replace( + self._evidence_path, + old_evidence, + published_evidence_identity, + allow_detached_authority=True, + ) + except BaseException: + pass + if reuse_envelope: + pass + elif old_envelope is None: + assert published_envelope_identity is not None + self._remove_identity_safe( + envelope_path, + published_envelope_identity, + allow_detached_authority=True, + ) + self._fsync_pinned_directory(envelope_path.parent) + else: + assert published_envelope_identity is not None + self._rollback_mutable_replace( + envelope_path, + old_envelope, + published_envelope_identity, + allow_detached_authority=True, + ) + raise + self._require_publication_order_guard() + self._latest_evidence = copy.deepcopy(durable_evidence) + self._current_envelope = envelope_path + self._current_identity = identity + self._evidence_state = "valid" + self._compatibility_evidence_override = False + self._invalidated_legacy_identity = None + self._legacy_proof_token = None + try: + self.prune_best_effort() + except Exception: + # Retention is maintenance after the evidence reference is durable. + pass + return AuditPublication( + identity=identity, + envelope_path=envelope_path, + evidence=copy.deepcopy(durable_evidence), + published=True, + ) + + def _build_live_envelope( + self, + *, + identity: AuditPublicationIdentity, + report: Mapping[str, Any], + persistence: Mapping[str, Any], + created_at: str, + ) -> dict[str, Any]: + normalized_report = self._normalize_report_identity(report) + if normalized_report["block_height"] != identity.block_height: + raise RuntimeError("audit report block height does not match publication") + report_digest = _canonical_hex( + normalized_report.get("audit_bundle_sha256_hex"), + name="audit report digest", + ) + persistence_digest = _canonical_hex( + persistence.get("audit_bundle_sha256"), + name="persistence digest", + ) + if not hmac.compare_digest(report_digest, persistence_digest): + raise RuntimeError("audit report and persistence digests differ") + body_uri = str(persistence.get("body_uri") or "") + if body_uri: + body_path = self.resolve_owned_path(body_uri) + expected_body_path = self.body_path(identity.block_hash, persistence_digest) + if body_path != expected_body_path: + raise RuntimeError( + "persistence body URI does not match publication identity" + ) + body_uri = str(body_path) + return { + "schema": LIVE_ENVELOPE_SCHEMA, + "block_hash": identity.block_hash, + "block_height": identity.block_height, + "audit_bundle_sha256": persistence_digest, + "body_uri": body_uri, + "body_filename": Path(body_uri).name if body_uri else None, + "coinbase_txid": normalized_report.get("coinbase_txid"), + "coinbase_manifest_sha256": normalized_report.get( + "coinbase_manifest_sha256_hex" + ), + "coinbase_tx_hex": normalized_report.get("coinbase_tx_hex"), + "coinbase_value_sats": normalized_report.get("coinbase_value_sats"), + "created_at": created_at, + } + + def _normalized_durable_evidence( + self, + *, + identity: AuditPublicationIdentity, + envelope_path: Path, + report: Mapping[str, Any], + persistence: Mapping[str, Any], + evidence: Mapping[str, Any], + verification_identity: Mapping[str, Any], + ) -> dict[str, Any]: + normalized_report = self._normalize_report_identity(report) + normalized_verification = self._normalize_verification_identity( + dict(verification_identity), + report=normalized_report, + ) + normalized_persistence = copy.deepcopy(dict(persistence)) + normalized_persistence["audit_bundle_sha256"] = _canonical_hex( + normalized_persistence.get("audit_bundle_sha256"), + name="persistence digest", + ) + body_uri = str(normalized_persistence.get("body_uri") or "") + if body_uri: + normalized_persistence["body_uri"] = str(self.resolve_owned_path(body_uri)) + durable = copy.deepcopy(dict(evidence)) + durable.update( + { + "schema": LIVE_EVIDENCE_SCHEMA, + "block_hash": identity.block_hash, + "block_height": identity.block_height, + "audit_bundle_path": str(envelope_path), + "audit_publication_identity": identity.to_json(), + "audit_report": normalized_report, + "audit_verification_identity": normalized_verification, + "persistence": normalized_persistence, + "coinbase_txid": normalized_report["coinbase_txid"], + "coinbase_manifest_sha256_hex": normalized_report[ + "coinbase_manifest_sha256_hex" + ], + "coinbase_tx_hex": normalized_report["coinbase_tx_hex"], + "coinbase_value_sats": normalized_report["coinbase_value_sats"], + } + ) + return durable + + def latest_evidence(self) -> dict[str, Any] | None: + with self._lock: + if not self._directory_authority_is_current(): + return None + if self._compatibility_evidence_override: + return copy.deepcopy(self._latest_evidence) + if self._evidence_state == "legacy_unproven": + return None + if self._latest_evidence is not None: + return copy.deepcopy(self._latest_evidence) + if self._evidence_state == "invalid": + return None + self._load_current_evidence() + with self._lock: + if not self._directory_authority_is_current(): + return None + return copy.deepcopy(self._latest_evidence) + + def set_latest_evidence_for_compatibility( + self, + payload: Mapping[str, Any] | None, + ) -> None: + with self._lock: + self._compatibility_evidence_override = True + self._invalidated_legacy_identity = None + self._legacy_proof_token = None + if payload is None: + self._latest_evidence = None + self._current_envelope = None + self._current_identity = None + self._evidence_state = "absent" + return + copied = copy.deepcopy(dict(payload)) + try: + latest, envelope, identity = self._validate_evidence(copied) + except (OSError, RuntimeError, TypeError, ValueError): + self._latest_evidence = copied + self._current_envelope = None + self._current_identity = None + self._evidence_state = "invalid" + return + self._latest_evidence = latest + self._current_envelope = envelope + self._current_identity = identity + self._evidence_state = self._loaded_evidence_state(latest, identity) + + def _load_current_evidence(self) -> None: + with self._lock: + self._load_current_evidence_locked() + + def _load_current_evidence_locked(self) -> None: + try: + self._validate_evidence_parent_identity() + evidence_bytes, _value = self._read_owned_regular_bytes(self._evidence_path) + except FileNotFoundError: + self._latest_evidence = None + self._current_envelope = None + self._current_identity = None + self._evidence_state = "absent" + return + except (OSError, RuntimeError): + self._latest_evidence = None + self._current_envelope = None + self._current_identity = None + self._evidence_state = "invalid" + return + try: + payload = json.loads(evidence_bytes.decode("utf-8")) + parsed = self._validate_evidence(payload) + except (OSError, UnicodeDecodeError, json.JSONDecodeError, RuntimeError, TypeError, ValueError): + self._latest_evidence = None + self._current_envelope = None + self._current_identity = None + self._evidence_state = "invalid" + return + self._latest_evidence, self._current_envelope, self._current_identity = parsed + self._evidence_state = self._loaded_evidence_state( + self._latest_evidence, + self._current_identity, + ) + + def _reload_current_evidence_locked(self) -> None: + """Reload peer state while retaining an exact in-process legacy proof.""" + + prior_token = self._legacy_proof_token + self._load_current_evidence_locked() + if ( + prior_token is not None + and self._evidence_state == "legacy_unproven" + and self._current_identity == prior_token.identity + ): + try: + current_token = self._capture_legacy_proof_token( + prior_token.identity + ) + except (OSError, RuntimeError): + current_token = None + if current_token == prior_token: + self._evidence_state = "legacy_proven" + return + self._legacy_proof_token = None + + def _capture_legacy_proof_token( + self, + identity: AuditPublicationIdentity, + ) -> _LegacyProofToken: + if self._current_envelope is None: + raise RuntimeError("legacy audit envelope is unavailable") + evidence_bytes, evidence_stat = self._read_owned_regular_bytes( + self._evidence_path + ) + envelope_bytes, envelope_stat = self._read_owned_regular_bytes( + self._current_envelope + ) + return _LegacyProofToken( + identity=identity, + evidence_file=_FileIdentity.from_stat(evidence_stat), + evidence_sha256=_sha256_bytes(evidence_bytes), + envelope_file=_FileIdentity.from_stat(envelope_stat), + envelope_sha256=_sha256_bytes(envelope_bytes), + ) + + def _validate_evidence( + self, + payload: object, + ) -> tuple[dict[str, Any], Path, AuditPublicationIdentity]: + if not isinstance(payload, dict) or payload.get("schema") != LIVE_EVIDENCE_SCHEMA: + raise RuntimeError("invalid evidence schema") + self._validate_root_identity() + raw_block_hash = payload.get("block_hash") + block_hash = _canonical_hex(raw_block_hash, name="block_hash") + if raw_block_hash != block_hash: + raise RuntimeError("evidence block hash is not canonical") + raw_block_height = payload.get("block_height") + if ( + isinstance(raw_block_height, bool) + or not isinstance(raw_block_height, int) + or raw_block_height < 0 + ): + raise RuntimeError("evidence block height is invalid") + block_height = raw_block_height + raw_envelope_path = payload.get("audit_bundle_path") + if not isinstance(raw_envelope_path, str): + raise RuntimeError("evidence envelope path is not canonical") + envelope_path = Path(raw_envelope_path).absolute() + expected_path = self.live_envelope_path( + block_height=block_height, + block_hash=block_hash, + ) + if envelope_path != expected_path or raw_envelope_path != str(expected_path): + raise RuntimeError("evidence envelope path is not canonical") + envelope_bytes, _envelope_stat = self._read_owned_regular_bytes(envelope_path) + envelope = json.loads(envelope_bytes.decode("utf-8")) + if ( + not isinstance(envelope, dict) + or envelope.get("schema") != LIVE_ENVELOPE_SCHEMA + or envelope.get("block_hash") != block_hash + or isinstance(envelope.get("block_height"), bool) + or not isinstance(envelope.get("block_height"), int) + or envelope.get("block_height") != block_height + ): + raise RuntimeError("evidence envelope identity mismatch") + report = payload.get("audit_report") + persistence = payload.get("persistence") + if not isinstance(report, dict) or not isinstance(persistence, dict): + raise RuntimeError("evidence report and persistence are required") + normalized_report = self._normalize_report_identity(report) + if report != normalized_report: + raise RuntimeError("evidence audit report is not canonical") + if normalized_report["block_height"] != block_height: + raise RuntimeError("evidence report block height mismatch") + report_digest = _canonical_hex( + normalized_report.get("audit_bundle_sha256_hex"), + name="audit report digest", + ) + raw_persistence_digest = persistence.get("audit_bundle_sha256") + persistence_digest = _canonical_hex( + raw_persistence_digest, + name="persistence digest", + ) + raw_envelope_digest = envelope.get("audit_bundle_sha256") + envelope_digest = _canonical_hex( + raw_envelope_digest, + name="envelope digest", + ) + if ( + raw_persistence_digest != persistence_digest + or raw_envelope_digest != envelope_digest + ): + raise RuntimeError("evidence digest is not canonical") + if not ( + hmac.compare_digest(report_digest, persistence_digest) + and hmac.compare_digest(report_digest, envelope_digest) + ): + raise RuntimeError("evidence digest mismatch") + raw_persistence_body_uri = persistence.get("body_uri") + raw_envelope_body_uri = envelope.get("body_uri") + if not isinstance(raw_persistence_body_uri, str) or not isinstance( + raw_envelope_body_uri, + str, + ): + raise RuntimeError("evidence body URI is not canonical") + persistence_body_uri = raw_persistence_body_uri + envelope_body_uri = raw_envelope_body_uri + if persistence_body_uri: + body_path = self.resolve_owned_path(persistence_body_uri) + expected_body_path = self.body_path(block_hash, persistence_digest) + if body_path != expected_body_path: + raise RuntimeError("evidence body URI identity mismatch") + canonical_body_uri = str(body_path) + if persistence_body_uri != canonical_body_uri: + raise RuntimeError("evidence body URI is not canonical") + persistence_body_uri = canonical_body_uri + if envelope_body_uri: + canonical_envelope_body_uri = str( + self.resolve_owned_path(envelope_body_uri) + ) + if envelope_body_uri != canonical_envelope_body_uri: + raise RuntimeError("envelope body URI is not canonical") + envelope_body_uri = canonical_envelope_body_uri + if ( + envelope_body_uri != persistence_body_uri + or envelope.get("body_filename") + != (Path(persistence_body_uri).name if persistence_body_uri else None) + ): + raise RuntimeError("evidence and envelope body URIs differ") + coinbase_bindings = ( + ("coinbase_txid", "coinbase_txid"), + ("coinbase_manifest_sha256_hex", "coinbase_manifest_sha256"), + ("coinbase_tx_hex", "coinbase_tx_hex"), + ("coinbase_value_sats", "coinbase_value_sats"), + ) + for report_key, envelope_key in coinbase_bindings: + expected_value = normalized_report[report_key] + evidence_value = payload.get(report_key) + envelope_value = envelope.get(envelope_key) + if report_key == "coinbase_value_sats": + if ( + isinstance(evidence_value, bool) + or not isinstance(evidence_value, int) + or isinstance(envelope_value, bool) + or not isinstance(envelope_value, int) + ): + raise RuntimeError("evidence coinbase value is invalid") + elif report_key == "coinbase_tx_hex": + canonical_evidence_value = _canonical_hex_bytes( + evidence_value, + name="evidence coinbase_tx_hex", + ) + canonical_envelope_value = _canonical_hex_bytes( + envelope_value, + name="envelope coinbase_tx_hex", + ) + if ( + evidence_value != canonical_evidence_value + or envelope_value != canonical_envelope_value + ): + raise RuntimeError("evidence coinbase identity is not canonical") + evidence_value = canonical_evidence_value + envelope_value = canonical_envelope_value + else: + canonical_evidence_value = _canonical_hex( + evidence_value, + name=report_key, + ) + canonical_envelope_value = _canonical_hex( + envelope_value, + name=envelope_key, + ) + if ( + evidence_value != canonical_evidence_value + or envelope_value != canonical_envelope_value + ): + raise RuntimeError("evidence coinbase identity is not canonical") + evidence_value = canonical_evidence_value + envelope_value = canonical_envelope_value + if evidence_value != expected_value or envelope_value != expected_value: + raise RuntimeError("evidence coinbase identity mismatch") + identity_payload = payload.get("audit_publication_identity") + if not isinstance(identity_payload, dict): + # Older durable evidence remains readable but starts at sequence 0. + identity = AuditPublicationIdentity(0, block_height, block_hash) + else: + raw_sequence = identity_payload.get("sequence") + raw_identity_height = identity_payload.get("block_height") + if ( + isinstance(raw_sequence, bool) + or not isinstance(raw_sequence, int) + or isinstance(raw_identity_height, bool) + or not isinstance(raw_identity_height, int) + ): + raise RuntimeError("invalid evidence publication identity") + identity = AuditPublicationIdentity( + raw_sequence, + raw_identity_height, + _canonical_hex( + identity_payload.get("block_hash"), + name="publication block hash", + ), + ) + if ( + identity.sequence < 0 + or identity.block_height != block_height + or identity.block_hash != block_hash + or identity_payload != identity.to_json() + ): + raise RuntimeError("invalid evidence publication identity") + normalized_payload = copy.deepcopy(payload) + verification_payload = payload.get("audit_verification_identity") + if identity.sequence > 0: + if ( + isinstance(verification_payload, dict) + and verification_payload.get("schema") + == LEGACY_VERIFICATION_UNAVAILABLE_SCHEMA + ): + normalized_verification = ( + self._normalize_legacy_verification_marker( + verification_payload, + identity=identity, + report=normalized_report, + ) + ) + else: + normalized_verification = self._normalize_verification_identity( + verification_payload, + report=normalized_report, + ) + normalized_payload["audit_verification_identity"] = ( + normalized_verification + ) + elif verification_payload is not None: + # Sequence-zero evidence is the only supported legacy form. If a + # producer supplied a verification identity anyway, validate it + # rather than allowing an unchecked field to become durable state. + normalized_payload["audit_verification_identity"] = ( + self._normalize_verification_identity( + verification_payload, + report=normalized_report, + ) + ) + self._validate_root_identity() + self._validate_evidence_parent_identity() + return normalized_payload, envelope_path, identity + + def adopt_legacy_publication_identity( + self, + identity: AuditPublicationIdentity, + *, + publication_floor_sequence: int, + ) -> None: + """Record a ledger proof that must be revalidated after every restart.""" + + self._require_publication_order_guard() + if ( + isinstance(publication_floor_sequence, bool) + or not isinstance(publication_floor_sequence, int) + or publication_floor_sequence < 0 + ): + raise ValueError("publication floor sequence must be a non-negative integer") + with self._lock: + self._validate_root_identity() + self._validate_evidence_parent_identity() + if not self._compatibility_evidence_override: + self._load_current_evidence_locked() + if self._evidence_state not in { + "legacy", + "legacy_unproven", + } or self._latest_evidence is None: + return + if identity.sequence > publication_floor_sequence: + raise RuntimeError( + "legacy audit publication sequence exceeds the durable ledger floor" + ) + if identity.sequence < publication_floor_sequence: + self.invalidate_unprovable_legacy_evidence() + return + if ( + identity.sequence <= 0 + or identity.block_hash != self._latest_evidence.get("block_hash") + or identity.block_height + != int(self._latest_evidence.get("block_height", -1)) + ): + raise RuntimeError("legacy evidence ledger identity mismatch") + upgraded = copy.deepcopy(self._latest_evidence) + upgraded["audit_publication_identity"] = identity.to_json() + report = upgraded.get("audit_report") + if not isinstance(report, dict): + raise RuntimeError("legacy audit report is unavailable") + upgraded["audit_verification_identity"] = ( + self._legacy_verification_marker( + identity=identity, + report=report, + ) + ) + old_evidence, _old_stat = self._read_owned_regular_bytes( + self._evidence_path + ) + published_identity = self._write_mutable_json( + self._evidence_path, + upgraded, + sort_keys=True, + ) + try: + self._validate_root_identity() + self._validate_evidence_parent_identity() + except BaseException: + self._rollback_mutable_replace( + self._evidence_path, + old_evidence, + published_identity, + allow_detached_authority=True, + ) + raise + self._latest_evidence = upgraded + self._current_identity = identity + self._evidence_state = "legacy_proven" + self._invalidated_legacy_identity = None + self._legacy_proof_token = self._capture_legacy_proof_token(identity) + + def invalidate_unprovable_legacy_evidence(self) -> None: + """Drop ordering/pin authority while allowing a new durable repair.""" + + with self._lock: + if self._evidence_state not in {"legacy", "legacy_unproven"}: + return + self._invalidated_legacy_identity = self._current_identity + self._legacy_proof_token = None + self._latest_evidence = None + self._current_envelope = None + self._current_identity = None + self._evidence_state = "invalid" + + def prune_best_effort( + self, + *, + keep_live_path: Path | None = None, + ) -> RetentionResult: + try: + with self.publication_order_guard(): + with self._lock: + if not self._compatibility_evidence_override: + # A different coordinator process may have published a + # newer reference. Reconcile it under the same flock + # immediately before deriving live-deletion pins. + self._reload_current_evidence_locked() + return self._prune_best_effort_guarded( + keep_live_path=keep_live_path, + ) + except (OSError, RuntimeError): + return RetentionResult(errors=1) + + def _prune_best_effort_guarded( + self, + *, + keep_live_path: Path | None = None, + ) -> RetentionResult: + self._require_publication_order_guard() + live_removed = 0 + candidate_removed = 0 + errors = 0 + with self._lock: + live_authorized = self._directory_authority_is_current() + pins = ( + { + path + for path in (self._current_envelope,) + if path is not None + } + if live_authorized + and self._evidence_state in {"valid", "legacy_proven"} + else set() + ) + if keep_live_path is not None: + raw_keep = Path(keep_live_path).absolute() + candidate_keep = raw_keep.parent.resolve() / raw_keep.name + if ( + candidate_keep.parent == self._root + and _LIVE_RE.fullmatch(candidate_keep.name) + ): + pins.add(candidate_keep) + active_paths = {entry[0] for entry in self._active_candidates.values()} + live_retention = self._live_bundle_retention + candidate_retention = self._candidate_retention_seconds + evidence_state = self._evidence_state if live_authorized else "invalid" + try: + self._validate_root_identity() + entries = sorted( + (self._root / name for name in os.listdir(self._root_fd)), + key=lambda value: value.name, + ) + except (OSError, RuntimeError): + return RetentionResult(errors=1) + live: list[tuple[int, int, str, Path, _FileIdentity]] = [] + now = self._wall_time() + for path in entries: + try: + value = self._owned_lstat(path) + except RuntimeError: + errors += 1 + return RetentionResult(0, candidate_removed, errors) + except (FileNotFoundError, OSError): + errors += 1 + continue + if not stat.S_ISREG(value.st_mode): + continue + live_match = _LIVE_RE.fullmatch(path.name) + if live_match: + live.append( + ( + value.st_mtime_ns, + int(live_match.group("height")), + live_match.group("block"), + path, + _FileIdentity.from_stat(value), + ) + ) + continue + if not ( + _CANDIDATE_RE.fullmatch(path.name) + or _LEGACY_CANDIDATE_RE.fullmatch(path.name) + or _LEGACY_HIDDEN_CANDIDATE_RE.fullmatch(path.name) + ): + continue + if path in active_paths: + continue + if candidate_retention != 0 and now - value.st_mtime <= candidate_retention: + continue + try: + with self._lock: + if not self._directory_authority_is_current(): + errors += 1 + break + current_active_paths = { + entry[0] for entry in self._active_candidates.values() + } + if path in current_active_paths: + continue + if self._unlink_scanned_owned( + path, + _FileIdentity.from_stat(value), + ): + candidate_removed += 1 + except (OSError, RuntimeError): + errors += 1 + if evidence_state not in {"valid", "legacy_proven"}: + return RetentionResult(0, candidate_removed, errors) + live.sort(key=lambda item: (item[0], item[1], item[2], item[3].name), reverse=True) + retained = 0 + for _mtime, _height, _block, path, identity in live: + if path in pins: + continue + if live_retention < 0 or retained < max(live_retention - len(pins), 0): + retained += 1 + continue + try: + with self._lock: + self._require_publication_order_guard() + if not self._directory_authority_is_current(): + errors += 1 + break + if path == self._current_envelope: + continue + if self._unlink_scanned_owned( + path, + identity, + require_all_authorities=True, + ): + live_removed += 1 + except (OSError, RuntimeError): + errors += 1 + try: + self._validate_root_identity() + self._validate_evidence_parent_identity() + except RuntimeError: + errors += 1 + return RetentionResult(live_removed, candidate_removed, errors) + + def _unlink_scanned_owned( + self, + path: Path, + identity: _FileIdentity, + *, + require_all_authorities: bool = False, + ) -> bool: + if require_all_authorities and not self._directory_authority_is_current(): + raise RuntimeError("audit directory authority changed before live prune") + return self._remove_identity_safe(path, identity) + + def _remove_identity_safe( + self, + path: Path, + identity: _FileIdentity, + *, + allow_detached_authority: bool = False, + ) -> bool: + if not allow_detached_authority: + self._validate_owned_parent(path) + try: + current = self._owned_lstat(path) + except FileNotFoundError: + return False + if not stat.S_ISREG(current.st_mode): + # Hard-link restoration is unavailable for directories and is not + # portable for symlinks. Never relocate an unowned nonregular + # replacement merely to discover that cleanup lacks authority. + return False + if not identity.matches(current): + # A replacement that was already present at the precheck is not + # ours to relocate, even temporarily. + return False + quarantine = path.with_name(f".{path.name}.{uuid.uuid4().hex}.cleanup") + try: + self._owned_replace(path, quarantine) + except FileNotFoundError: + return False + try: + moved = self._owned_lstat(quarantine) + if identity.matches(moved): + self._owned_unlink(quarantine) + return True + # A replacement won the race. Restore it when the original name is + # free; otherwise preserve it under the quarantine name. + try: + self._owned_link(quarantine, path) + except OSError: + return False + self._owned_unlink(quarantine) + return False + except BaseException: + # Never turn an uncertain identity into deletion authority. + raise + + def _write_mutable_json( + self, + path: Path, + payload: Mapping[str, Any], + *, + sort_keys: bool, + ) -> _FileIdentity: + body = _json_bytes(payload, sort_keys=sort_keys) + return self._write_mutable_bytes(path, body) + + def _write_mutable_bytes(self, path: Path, payload: bytes) -> _FileIdentity: + path = Path(path).absolute() + resolved_parent = path.parent.resolve(strict=True) + if not ( + resolved_parent == self._root + or path == self._evidence_path + ): + raise RuntimeError("mutable audit target is outside the owned store") + with self._lock: + if path == self._evidence_path: + self._validate_evidence_parent_identity() + else: + self._validate_root_identity() + old_bytes: bytes | None + try: + old_bytes, _old_stat = self._read_owned_regular_bytes(path) + except FileNotFoundError: + old_bytes = None + tmp_path = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + temp_identity: _FileIdentity | None = None + published = False + try: + fd = self._owned_open( + tmp_path, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + temp_identity = _FileIdentity.from_stat(os.fstat(fd)) + with os.fdopen(fd, "wb") as handle: + try: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + finally: + temp_identity = _FileIdentity.from_stat( + os.fstat(handle.fileno()) + ) + self._owned_replace(tmp_path, path) + published = True + try: + self._fsync_directory(path.parent) + if path == self._evidence_path: + self._validate_evidence_parent_identity() + else: + self._validate_root_identity() + except BaseException: + self._rollback_mutable_replace( + path, + old_bytes, + temp_identity, + allow_detached_authority=True, + ) + raise + return temp_identity + finally: + if not published and temp_identity is not None: + self._remove_identity_safe( + tmp_path, + temp_identity, + allow_detached_authority=True, + ) + + def _rollback_mutable_replace( + self, + path: Path, + old_bytes: bytes | None, + published_identity: _FileIdentity, + *, + allow_detached_authority: bool = False, + ) -> None: + removed = self._remove_identity_safe( + path, + published_identity, + allow_detached_authority=allow_detached_authority, + ) + if not removed: + # A replacement won the race. Preserve it and do not restore stale + # bytes over an inode this operation never owned. + return + if old_bytes is None: + if allow_detached_authority: + self._fsync_pinned_directory(path.parent) + else: + self._fsync_directory(path.parent) + return + rollback = path.with_name(f".{path.name}.{uuid.uuid4().hex}.rollback") + rollback_identity: _FileIdentity | None = None + try: + fd = self._owned_open( + rollback, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + rollback_identity = _FileIdentity.from_stat(os.fstat(fd)) + with os.fdopen(fd, "wb") as handle: + try: + handle.write(old_bytes) + handle.flush() + os.fsync(handle.fileno()) + finally: + rollback_identity = _FileIdentity.from_stat( + os.fstat(handle.fileno()) + ) + try: + self._owned_link(rollback, path) + except FileExistsError: + # Preserve a concurrent replacement rather than overwriting it. + return + if allow_detached_authority: + self._fsync_pinned_directory(path.parent) + else: + self._fsync_directory(path.parent) + finally: + if rollback_identity is not None: + self._remove_identity_safe( + rollback, + rollback_identity, + allow_detached_authority=allow_detached_authority, + ) + + def _write_immutable_bytes(self, path: Path, payload: bytes) -> None: + path = Path(path).absolute() + if ( + path.parent.resolve(strict=True) != self._root + or self.artifact_kind(path.name) not in {"body", "share_segment"} + ): + raise RuntimeError("immutable audit target is not an owned body or segment") + with self._lock: + self._validate_root_identity() + try: + existing = self._owned_lstat(path) + except FileNotFoundError: + existing = None + if existing is not None: + if not stat.S_ISREG(existing.st_mode) or not self.file_matches_bytes( + path, + payload, + ): + raise RuntimeError( + f"existing audit artifact does not match payload at {path}" + ) + # The prior writer may have failed between link/rename and its + # directory fsync. An idempotent retry is the durability + # repair boundary, even though no bytes need to change. + self._fsync_directory(path.parent) + return + tmp_path = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + temp_identity: _FileIdentity | None = None + linked_by_this_call = False + try: + fd = self._owned_open( + tmp_path, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + temp_identity = _FileIdentity.from_stat(os.fstat(fd)) + with os.fdopen(fd, "wb") as handle: + try: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + finally: + temp_identity = _FileIdentity.from_stat( + os.fstat(handle.fileno()) + ) + try: + self._owned_link(tmp_path, path) + linked_by_this_call = True + except FileExistsError: + if not self.file_matches_bytes(path, payload): + raise RuntimeError( + f"existing audit artifact does not match payload at {path}" + ) + try: + self._fsync_directory(path.parent) + except BaseException: + if linked_by_this_call: + # Revoke only the inode linked by this call. A racing + # replacement is never deleted or overwritten. + self._remove_identity_safe( + path, + temp_identity, + allow_detached_authority=True, + ) + try: + self._fsync_directory(path.parent) + except BaseException: + # Preserve the original durability failure; the + # retry path always fsyncs an equal existing file. + pass + raise + self._validate_root_identity() + finally: + if temp_identity is not None: + self._remove_identity_safe( + tmp_path, + temp_identity, + allow_detached_authority=True, + ) + + def _fsync_directory(self, path: Path) -> None: + path = Path(path).absolute() + if path == self._root: + self._validate_root_identity() + os.fsync(self._root_fd) + self._validate_root_identity() + return + if path == self._evidence_path.parent: + self._validate_evidence_parent_identity() + os.fsync(self._evidence_parent_fd) + self._validate_evidence_parent_identity() + return + raise RuntimeError("audit fsync target has no pinned authority") + + def _fsync_pinned_directory(self, path: Path) -> None: + path = Path(path).absolute() + if path == self._root: + os.fsync(self._root_fd) + return + if path == self._evidence_path.parent: + os.fsync(self._evidence_parent_fd) + return + raise RuntimeError("audit fsync target has no pinned authority") + + def file_matches_bytes(self, path: Path, expected: bytes) -> bool: + try: + payload, value = self._read_owned_regular_bytes(path) + if value.st_size != len(expected): + return False + return hmac.compare_digest(payload, expected) + except OSError: + return False + + @staticmethod + def file_sha256_hex(path: Path) -> str: + payload, _value = AuditArtifactStore.read_regular_bytes(path) + return _sha256_bytes(payload) + + @staticmethod + def read_regular_bytes(path: Path) -> tuple[bytes, os.stat_result]: + fd = os.open( + path, + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), + ) + return AuditArtifactStore._read_regular_bytes_fd(fd) + + @staticmethod + def _read_regular_bytes_fd(fd: int) -> tuple[bytes, os.stat_result]: + try: + before = os.fstat(fd) + if not stat.S_ISREG(before.st_mode): + raise OSError("audit artifact is not a regular file") + chunks: list[bytes] = [] + while True: + chunk = os.read(fd, 1024 * 1024) + if not chunk: + break + chunks.append(chunk) + after = os.fstat(fd) + if ( + before.st_dev != after.st_dev + or before.st_ino != after.st_ino + or before.st_size != after.st_size + or before.st_mtime_ns != after.st_mtime_ns + or before.st_ctime_ns != after.st_ctime_ns + ): + raise OSError("audit artifact changed while reading") + return b"".join(chunks), after + finally: + os.close(fd) + + # External body and share-segment capability used by the ledger. Database + # lease checks remain in the ledger; paths, encodings, and bytes stay here. + + def canonical_audit_bundle_bytes(self, final_bundle: dict[str, Any]) -> bytes: + return canonical_audit_bundle_bytes(final_bundle, self._canonicalizer) + + def canonical_audit_body_bytes_for_sha( + self, + final_bundle: dict[str, Any], + audit_bundle_sha256: str, + ) -> bytes: + expected = _canonical_hex( + audit_bundle_sha256, + name="audit_bundle_sha256", + ) + body = self.canonical_audit_bundle_bytes(final_bundle) + actual = _sha256_bytes(body) + if not hmac.compare_digest(actual, expected): + raise RuntimeError( + f"audit bundle sha256 mismatch: expected {expected}, got {actual}" + ) + return body + + @staticmethod + def storage_json_bytes(payload: Mapping[str, Any]) -> bytes: + return _json_bytes(payload) + + def audit_share_segment_payload( + self, + *, + first_share_seq: int, + last_share_seq: int, + shares: list[Any], + ) -> dict[str, Any]: + return { + "schema": AUDIT_SHARE_SEGMENT_SCHEMA, + "first_share_seq": first_share_seq, + "last_share_seq": last_share_seq, + "share_count": len(shares), + "shares": shares, + } + + def write_audit_share_segment( + self, + *, + first_share_seq: int, + last_share_seq: int, + shares: list[Any], + ) -> tuple[str, str]: + self._validate_share_range(first_share_seq, last_share_seq, shares) + segment = self.audit_share_segment_payload( + first_share_seq=first_share_seq, + last_share_seq=last_share_seq, + shares=shares, + ) + payload = self.storage_json_bytes(segment) + digest = _sha256_bytes(payload) + path = self._root / ( + f"prism-audit-share-segment-{first_share_seq}-{last_share_seq}-{digest}.json" + ) + if not _SHARE_CONTENT_RE.fullmatch(path.name): + raise RuntimeError("invalid audit share segment bounds") + self._write_immutable_bytes(path, payload) + self._validate_root_identity() + return str(path), digest + + def write_audit_share_segment_range( + self, + *, + segment_first_share_seq: int, + segment_last_share_seq: int, + first_share_seq: int, + last_share_seq: int, + shares: list[Any], + ) -> tuple[str, str]: + if not shares: + raise RuntimeError("audit share segment range cannot be empty") + if ( + segment_first_share_seq <= 0 + or segment_last_share_seq < segment_first_share_seq + or first_share_seq < segment_first_share_seq + or last_share_seq > segment_last_share_seq + ): + raise RuntimeError("audit share range is outside its segment slot") + self._validate_share_range(first_share_seq, last_share_seq, shares) + path = self._root / ( + "prism-audit-share-segment-slot-" + f"{segment_first_share_seq}-{segment_last_share_seq}.json" + ) + if not _SHARE_SLOT_RE.fullmatch(path.name): + raise RuntimeError("invalid audit share segment slot bounds") + incoming = self.audit_share_segment_payload( + first_share_seq=first_share_seq, + last_share_seq=last_share_seq, + shares=shares, + ) + incoming_bytes = self.storage_json_bytes(incoming) + range_digest = _sha256_bytes(incoming_bytes) + with self._lock: + self._validate_root_identity() + if self.file_matches_bytes(path, incoming_bytes): + self._fsync_directory(path.parent) + return str(path), range_digest + existing_bytes: bytes | None = None + merged = shares + try: + value = self._owned_lstat(path) + except FileNotFoundError: + value = None + if value is not None: + if not stat.S_ISREG(value.st_mode): + raise RuntimeError( + f"existing audit share segment is not regular at {path}" + ) + try: + existing_bytes, existing_descriptor = self._read_owned_regular_bytes( + path + ) + if ( + existing_descriptor.st_dev != value.st_dev + or existing_descriptor.st_ino != value.st_ino + or existing_descriptor.st_mtime_ns != value.st_mtime_ns + ): + raise OSError("share segment identity changed") + existing = json.loads(existing_bytes) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError( + f"existing audit share segment is not valid JSON at {path}" + ) from exc + if ( + not isinstance(existing, dict) + or existing.get("schema") != AUDIT_SHARE_SEGMENT_SCHEMA + or not isinstance(existing.get("shares"), list) + ): + raise RuntimeError( + f"existing audit share segment has invalid schema at {path}" + ) + merged = self.merge_audit_share_ranges( + existing["shares"], + shares, + segment_path=path, + ) + segment = self.audit_share_segment_payload( + first_share_seq=int(merged[0]["share_seq"]), + last_share_seq=int(merged[-1]["share_seq"]), + shares=merged, + ) + if ( + int(merged[0]["share_seq"]) < segment_first_share_seq + or int(merged[-1]["share_seq"]) > segment_last_share_seq + ): + raise RuntimeError("existing audit share range escapes its slot") + segment_bytes = self.storage_json_bytes(segment) + if existing_bytes != segment_bytes: + self._write_mutable_bytes(path, segment_bytes) + else: + self._fsync_directory(path.parent) + self._validate_root_identity() + return str(path), range_digest + + @staticmethod + def _validate_share_range( + first_share_seq: int, + last_share_seq: int, + shares: list[Any], + ) -> None: + if first_share_seq <= 0 or last_share_seq < first_share_seq or not shares: + raise RuntimeError("audit share range bounds are invalid") + try: + sequences = [int(share["share_seq"]) for share in shares] + except (KeyError, TypeError, ValueError) as exc: + raise RuntimeError("audit share range has invalid share_seq") from exc + if ( + sequences[0] != first_share_seq + or sequences[-1] != last_share_seq + or any(current + 1 != nxt for current, nxt in zip(sequences, sequences[1:])) + ): + raise RuntimeError("audit share range is not exactly contiguous") + + def merge_audit_share_ranges( + self, + existing_shares: list[Any], + incoming_shares: list[Any], + *, + segment_path: Path, + ) -> list[Any]: + if not existing_shares: + return list(incoming_shares) + existing = self.audit_shares_by_seq(existing_shares, segment_path=segment_path) + incoming = self.audit_shares_by_seq(incoming_shares, segment_path=segment_path) + for share_seq, value in incoming.items(): + old = existing.get(share_seq) + if old is not None and old != value: + raise RuntimeError( + "existing audit share segment conflicts at share_seq " + f"{share_seq} in {segment_path}" + ) + merged = {**existing, **incoming} + ordered = sorted(merged) + if any(current + 1 != nxt for current, nxt in zip(ordered, ordered[1:])): + raise RuntimeError( + f"existing audit share segment would become non-contiguous at {segment_path}" + ) + return [merged[share_seq] for share_seq in ordered] + + @staticmethod + def audit_shares_by_seq( + shares: list[Any], + *, + segment_path: Path, + ) -> dict[int, Any]: + result: dict[int, Any] = {} + for share in shares: + if not isinstance(share, dict): + raise RuntimeError( + f"audit share segment has invalid share payload at {segment_path}" + ) + try: + share_seq = int(share["share_seq"]) + except (KeyError, TypeError, ValueError) as exc: + raise RuntimeError( + f"audit share segment has invalid share_seq at {segment_path}" + ) from exc + old = result.get(share_seq) + if old is not None and old != share: + raise RuntimeError( + "audit share segment has duplicate conflicting share_seq " + f"{share_seq} at {segment_path}" + ) + result[share_seq] = share + ordered = sorted(result) + if any(current + 1 != nxt for current, nxt in zip(ordered, ordered[1:])): + raise RuntimeError( + f"audit share segment has non-contiguous share_seq values at {segment_path}" + ) + return result + + def audit_share_range_parts(self, shares: list[Any]) -> list[dict[str, Any]] | None: + if self._share_segment_size <= 0: + return None + try: + share_seqs = [int(share["share_seq"]) for share in shares] + except (KeyError, TypeError, ValueError): + return None + if any(current + 1 != nxt for current, nxt in zip(share_seqs, share_seqs[1:])): + return None + parts: list[dict[str, Any]] = [] + index = 0 + while index < len(shares): + first = share_seqs[index] + segment_start = ((first - 1) // self._share_segment_size) * self._share_segment_size + 1 + segment_end = segment_start + self._share_segment_size - 1 + end = index + while end < len(shares) and share_seqs[end] <= segment_end: + end += 1 + chunk = shares[index:end] + first_chunk = share_seqs[index] + last_chunk = share_seqs[end - 1] + uri, digest = self.write_audit_share_segment_range( + segment_first_share_seq=segment_start, + segment_last_share_seq=segment_end, + first_share_seq=first_chunk, + last_share_seq=last_chunk, + shares=chunk, + ) + parts.append( + { + "kind": "segment_range", + "segment_first_share_seq": segment_start, + "segment_last_share_seq": segment_end, + "first_share_seq": first_chunk, + "last_share_seq": last_chunk, + "share_count": len(chunk), + "range_sha256": digest, + "body_uri": uri, + } + ) + index = end + return parts + + def audit_share_parts(self, shares: list[Any]) -> list[dict[str, Any]] | None: + if self._share_segment_size <= 0: + return None + try: + share_seqs = [int(share["share_seq"]) for share in shares] + except (KeyError, TypeError, ValueError): + return None + if any(current + 1 != nxt for current, nxt in zip(share_seqs, share_seqs[1:])): + return None + parts: list[dict[str, Any]] = [] + index = 0 + while index < len(shares): + first = share_seqs[index] + segment_start = ((first - 1) // self._share_segment_size) * self._share_segment_size + 1 + segment_end = segment_start + self._share_segment_size - 1 + end = index + while end < len(shares) and share_seqs[end] <= segment_end: + end += 1 + chunk = shares[index:end] + chunk_seqs = share_seqs[index:end] + if ( + len(chunk) == self._share_segment_size + and chunk_seqs[0] == segment_start + and chunk_seqs[-1] == segment_end + ): + uri, digest = self.write_audit_share_segment( + first_share_seq=segment_start, + last_share_seq=segment_end, + shares=chunk, + ) + parts.append( + { + "kind": "segment", + "first_share_seq": segment_start, + "last_share_seq": segment_end, + "share_count": len(chunk), + "sha256": digest, + "body_uri": uri, + } + ) + else: + parts.append( + { + "kind": "inline", + "first_share_seq": chunk_seqs[0], + "last_share_seq": chunk_seqs[-1], + "share_count": len(chunk), + "shares": chunk, + } + ) + index = end + return parts + + def audit_body_ref( + self, + *, + block_hash: str, + audit_bundle_sha256: str, + final_bundle: dict[str, Any], + ) -> dict[str, Any] | None: + if self._share_segment_size <= 0: + return None + shares = final_bundle.get("shares") + if not isinstance(shares, list) or not shares: + return None + parts = self.audit_share_parts(shares) + if parts is None or not any(part.get("kind") == "segment" for part in parts): + return None + without_shares = {key: value for key, value in final_bundle.items() if key != "shares"} + return { + "schema": AUDIT_BODY_REF_SCHEMA, + "block_hash": block_hash, + "audit_bundle_sha256": audit_bundle_sha256, + "audit_bundle_schema": str(final_bundle.get("schema") or ""), + "share_count": len(shares), + "share_segment_size": self._share_segment_size, + "shares_key_index": list(final_bundle).index("shares"), + "bundle_without_shares": without_shares, + "share_parts": parts, + } + + def audit_bundle_v2( + self, + *, + block_hash: str, + audit_bundle_sha256: str, + final_bundle: dict[str, Any], + ) -> dict[str, Any] | None: + if self._share_segment_size <= 0: + return None + shares = final_bundle.get("shares") + if not isinstance(shares, list) or not shares: + return None + parts = self.audit_share_range_parts(shares) + if parts is None: + return None + proof: dict[str, Any] = { + "schema": AUDIT_WINDOW_COMPLETENESS_PROOF_SCHEMA, + "share_segment_size": self._share_segment_size, + "first_share_seq": int(shares[0]["share_seq"]), + "last_share_seq": int(shares[-1]["share_seq"]), + "share_count": len(shares), + "share_parts_digest_hex": _sha256_bytes( + self.storage_json_bytes({"share_parts": parts}) + ), + "share_parts": parts, + } + reward = final_bundle.get("reward_manifest") + if isinstance(reward, dict): + for key in ( + "anchor_job_issued_at_ms", + "anchor_share_seq", + "newest_share_seq", + "oldest_share_seq", + "included_share_count", + "requested_window_weight", + "counted_window_weight", + "share_slice_digest_hex", + ): + if key in reward: + proof[key] = reward[key] + return { + "schema": AUDIT_BUNDLE_V2_SCHEMA, + "block_hash": block_hash, + "audit_bundle_sha256": audit_bundle_sha256, + "logical_audit_bundle_schema": str(final_bundle.get("schema") or ""), + "share_count": len(shares), + "shares_key_index": list(final_bundle).index("shares"), + "bundle_without_shares": { + key: value for key, value in final_bundle.items() if key != "shares" + }, + "share_window_proof": proof, + } + + def externalize_audit_body( + self, + block_hash: str, + audit_bundle_sha256: str, + final_bundle: dict[str, Any], + ) -> str: + block_hash = _canonical_hex(block_hash, name="block_hash") + digest = _canonical_hex( + audit_bundle_sha256, + name="audit_bundle_sha256", + ) + body = self.canonical_audit_body_bytes_for_sha(final_bundle, digest) + path = self.body_path(block_hash, digest) + self._write_immutable_bytes(path, body) + self._validate_root_identity() + return str(path) + + def prepare_external_audit_body( + self, + payload: Mapping[str, Any], + final_bundle: dict[str, Any], + *, + body_uri: str | None, + canonical_bundle_path: Path | None = None, + ) -> str | None: + block_hash = _canonical_hex(payload["block_hash"], name="block_hash") + expected = _canonical_hex( + payload["audit_bundle_sha256"], + name="audit_bundle_sha256", + ) + if body_uri is None: + return None + body_path = self.resolve_owned_path(body_uri) + canonical_path = self.body_path(block_hash, expected) + if body_path != canonical_path: + raise RuntimeError( + "existing audit bundle body pointer does not match canonical external path: " + f"{body_uri}" + ) + literal: bytes | None = None + source_identity: _FileIdentity | None = None + if canonical_bundle_path is not None: + source = Path(canonical_bundle_path) + try: + literal, before = self._read_owned_regular_bytes(source) + except OSError as exc: + raise RuntimeError( + f"canonical audit bundle is not retrievable at {source}: {exc}" + ) from exc + source_identity = _FileIdentity.from_stat(before) + actual = _sha256_bytes(literal) + if not hmac.compare_digest(actual, expected): + raise RuntimeError( + f"audit bundle sha256 mismatch: expected {expected}, got {actual}" + ) + # Compact storage is derived from final_bundle, not necessarily from the + # supplied canonical path. Bind those logical inputs independently. + if literal is not None: + try: + canonical_logical = json.loads(literal) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError("canonical audit candidate is not valid JSON") from exc + if canonical_logical != final_bundle: + raise RuntimeError( + "canonical audit candidate does not match logical bundle" + ) + else: + self.canonical_audit_body_bytes_for_sha(final_bundle, expected) + storage = self.audit_bundle_v2( + block_hash=block_hash, + audit_bundle_sha256=expected, + final_bundle=final_bundle, + ) + if storage is None: + storage = self.audit_body_ref( + block_hash=block_hash, + audit_bundle_sha256=expected, + final_bundle=final_bundle, + ) + if storage is not None: + body_bytes = self.storage_json_bytes(storage) + elif literal is not None: + body_bytes = literal + else: + body_bytes = self.canonical_audit_body_bytes_for_sha( + final_bundle, + expected, + ) + try: + existing = self._owned_lstat(body_path) + except FileNotFoundError: + existing = None + if existing is not None: + if not stat.S_ISREG(existing.st_mode): + raise RuntimeError( + f"existing audit bundle body is not regular at {body_path}" + ) + if storage is not None and self.file_matches_bytes(body_path, body_bytes): + if self._compact_body_reconstructs_to( + body_path, + expected=expected, + final_bundle=final_bundle, + ): + self._fsync_directory(body_path.parent) + return str(body_path) + raise RuntimeError( + f"existing audit bundle body does not match payload at {body_path}" + ) + if storage is None and self.external_body_matches_sha(body_path, expected): + self._fsync_directory(body_path.parent) + return str(body_path) + # Layout upgrades may represent the same canonical logical bundle + # with different storage bytes. Verify by reconstruction. + if not self.external_body_matches_sha(body_path, expected): + raise RuntimeError( + f"existing audit bundle body does not match payload at {body_path}" + ) + self._fsync_directory(body_path.parent) + return str(body_path) + self._write_immutable_bytes(body_path, body_bytes) + if canonical_bundle_path is not None and source_identity is not None: + source_after = self._owned_lstat(Path(canonical_bundle_path)) + if not source_identity.matches(source_after): + raise RuntimeError("canonical audit bundle identity changed after publication") + # The compact storage bytes were derived from the already verified + # logical bundle. Validate exact destination bytes here; expensive + # reconstruction remains only the cross-version mismatch path. + if storage is not None: + valid = self._compact_body_reconstructs_to( + body_path, + expected=expected, + final_bundle=final_bundle, + ) + else: + valid = self.external_body_matches_sha(body_path, expected) + if not valid: + raise RuntimeError("published audit bundle body failed digest verification") + self._validate_root_identity() + return str(body_path) + + def _compact_body_reconstructs_to( + self, + body_path: Path, + *, + expected: str, + final_bundle: Mapping[str, Any], + ) -> bool: + try: + body_bytes, _value = self._read_owned_regular_bytes(body_path) + body = json.loads(body_bytes) + if not isinstance(body, dict): + return False + if body.get("schema") == AUDIT_BODY_REF_SCHEMA: + reconstructed = self.resolve_audit_body_ref( + body, + expected_sha256=expected, + body_uri=str(body_path), + verify_digest=False, + ) + elif body.get("schema") == AUDIT_BUNDLE_V2_SCHEMA: + reconstructed = self.resolve_audit_bundle_v2( + body, + expected_sha256=expected, + body_uri=str(body_path), + verify_digest=False, + ) + else: + return False + return reconstructed == final_bundle + except ( + OSError, + RuntimeError, + TypeError, + ValueError, + UnicodeDecodeError, + json.JSONDecodeError, + ): + return False + + def validate_canonical_source( + self, + path: Path, + expected_sha256: str, + final_bundle: Mapping[str, Any] | None = None, + ) -> None: + expected = _canonical_hex( + expected_sha256, + name="audit_bundle_sha256", + ) + try: + payload, _value = self._read_owned_regular_bytes(Path(path)) + except OSError as exc: + raise RuntimeError( + f"canonical audit bundle is not retrievable at {path}: {exc}" + ) from exc + actual = _sha256_bytes(payload) + if not hmac.compare_digest(actual, expected): + raise RuntimeError( + f"audit bundle sha256 mismatch: expected {expected}, got {actual}" + ) + if final_bundle is not None: + try: + logical = json.loads(payload) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError("canonical audit candidate is not valid JSON") from exc + if logical != final_bundle: + raise RuntimeError( + "canonical audit candidate does not match logical bundle" + ) + self._validate_root_identity() + + def audit_body_byte_len( + self, + body_uri: object | None, + final_bundle: dict[str, Any], + canonical_bundle_path: Path | None = None, + ) -> int: + if body_uri: + _payload, value = self._read_owned_regular_bytes( + self.resolve_owned_path(body_uri) + ) + self._validate_root_identity() + return value.st_size + if canonical_bundle_path is not None: + _payload, value = self._read_owned_regular_bytes(Path(canonical_bundle_path)) + self._validate_root_identity() + return value.st_size + value = len(self.canonical_audit_bundle_bytes(final_bundle)) + self._validate_root_identity() + return value + + def read_external_body( + self, + body_uri: object, + *, + expected_sha256: object | None = None, + ) -> dict[str, object] | None: + if not body_uri: + return None + try: + path = self.resolve_owned_path(body_uri) + body_match = _BODY_RE.fullmatch(path.name) + if body_match is None: + raise RuntimeError("body URI is not an owned canonical body") + if expected_sha256 is not None and body_match.group("digest") != _canonical_hex( + expected_sha256, + name="audit_bundle_sha256", + ): + raise RuntimeError("body URI digest does not match expected digest") + body_bytes, _value = self._read_owned_regular_bytes(path) + except (OSError, RuntimeError, TypeError, ValueError, OverflowError) as exc: + raise RuntimeError( + f"audit bundle body is not retrievable at {body_uri}: {exc}" + ) from exc + try: + body = json.loads(body_bytes.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: {exc}" + ) from exc + try: + if isinstance(body, dict) and body.get("schema") == AUDIT_BODY_REF_SCHEMA: + resolved = self.resolve_audit_body_ref( + body, + expected_sha256=expected_sha256, + body_uri=body_uri, + ) + self._validate_root_identity() + return resolved + if isinstance(body, dict) and body.get("schema") == AUDIT_BUNDLE_V2_SCHEMA: + resolved = self.resolve_audit_bundle_v2( + body, + expected_sha256=expected_sha256, + body_uri=body_uri, + ) + self._validate_root_identity() + return resolved + if expected_sha256: + expected = _canonical_hex( + expected_sha256, + name="audit_bundle_sha256", + ) + actual = _sha256_bytes(body_bytes) + if not hmac.compare_digest(actual, expected): + raise RuntimeError( + f"audit bundle body hash mismatch at {body_uri}: " + f"expected {expected}, got {actual}" + ) + self._validate_root_identity() + return body + except RuntimeError: + raise + except (TypeError, ValueError, OverflowError) as exc: + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: {exc}" + ) from exc + + def external_body_matches_sha(self, body_path: Path, expected: str) -> bool: + try: + self.read_external_body(str(body_path), expected_sha256=expected) + except ( + OSError, + RuntimeError, + TypeError, + ValueError, + OverflowError, + UnicodeDecodeError, + json.JSONDecodeError, + ): + return False + return True + + def external_body_available_for_sha(self, body_uri: object, expected: str) -> bool: + try: + expected = _canonical_hex(expected, name="audit_bundle_sha256") + self.read_external_body(body_uri, expected_sha256=expected) + except ( + OSError, + RuntimeError, + TypeError, + ValueError, + OverflowError, + UnicodeDecodeError, + json.JSONDecodeError, + ): + return False + return True + + def resolve_audit_body_ref( + self, + body_ref: Mapping[str, Any], + *, + expected_sha256: object | None, + body_uri: object, + verify_digest: bool = True, + ) -> dict[str, object]: + expected = ( + _canonical_hex(expected_sha256, name="audit_bundle_sha256") + if expected_sha256 + else None + ) + declared = _canonical_hex( + body_ref.get("audit_bundle_sha256"), + name="audit_bundle_sha256", + ) + self._validate_body_wrapper_identity(body_ref, body_uri, declared) + if expected and not hmac.compare_digest(declared, expected): + raise RuntimeError( + f"audit bundle body hash mismatch at {body_uri}: expected {expected}, got {declared}" + ) + without_shares = body_ref.get("bundle_without_shares") + parts = body_ref.get("share_parts") + if not isinstance(without_shares, dict) or not isinstance(parts, list): + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: invalid body reference" + ) + shares: list[Any] = [] + previous_last_share_seq: int | None = None + for part in parts: + if not isinstance(part, dict): + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: invalid share part" + ) + kind = part.get("kind") + if kind in {"segment", "segment_range", "segment_prefix"}: + part_shares = self.read_audit_share_segment( + part, + parent_body_uri=body_uri, + ) + elif kind == "inline" and isinstance(part.get("shares"), list): + inline = part["shares"] + if len(inline) != int(part.get("share_count") or 0): + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: inline share count mismatch" + ) + self._validate_share_range( + int(part.get("first_share_seq") or 0), + int(part.get("last_share_seq") or 0), + inline, + ) + part_shares = inline + else: + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: invalid share part kind" + ) + first_share_seq = int(part.get("first_share_seq") or 0) + last_share_seq = int(part.get("last_share_seq") or 0) + if ( + previous_last_share_seq is not None + and first_share_seq <= previous_last_share_seq + ): + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: share parts overlap or are out of order" + ) + previous_last_share_seq = last_share_seq + shares.extend(part_shares) + if len(shares) != int(body_ref.get("share_count") or 0): + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: share count mismatch" + ) + bundle = self._insert_shares( + without_shares, + shares, + int(body_ref.get("shares_key_index", len(without_shares))), + ) + if verify_digest: + actual = _sha256_bytes(self.canonical_audit_bundle_bytes(bundle)) + if not hmac.compare_digest(actual, declared): + raise RuntimeError( + f"audit bundle body hash mismatch at {body_uri}: expected {declared}, got {actual}" + ) + return bundle + + def resolve_audit_bundle_v2( + self, + body: Mapping[str, Any], + *, + expected_sha256: object | None, + body_uri: object, + verify_digest: bool = True, + ) -> dict[str, object]: + expected = ( + _canonical_hex(expected_sha256, name="audit_bundle_sha256") + if expected_sha256 + else None + ) + declared = _canonical_hex( + body.get("audit_bundle_sha256"), + name="audit_bundle_sha256", + ) + self._validate_body_wrapper_identity(body, body_uri, declared) + if expected and not hmac.compare_digest(declared, expected): + raise RuntimeError( + f"audit bundle body hash mismatch at {body_uri}: expected {expected}, got {declared}" + ) + without_shares = body.get("bundle_without_shares") + proof = body.get("share_window_proof") + if not isinstance(without_shares, dict) or not isinstance(proof, dict): + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: invalid v2 body" + ) + if proof.get("schema") != AUDIT_WINDOW_COMPLETENESS_PROOF_SCHEMA: + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: invalid proof schema" + ) + if int(body.get("share_count") or 0) != int(proof.get("share_count") or 0): + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: proof share count mismatch" + ) + parts = proof.get("share_parts") + if not isinstance(parts, list): + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: missing share parts" + ) + expected_parts_digest = str(proof.get("share_parts_digest_hex") or "").lower() + actual_parts_digest = _sha256_bytes( + self.storage_json_bytes({"share_parts": parts}) + ) + if expected_parts_digest != actual_parts_digest: + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: share parts digest mismatch" + ) + shares: list[Any] = [] + for part in parts: + if not isinstance(part, dict): + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: invalid share part" + ) + shares.extend(self.read_audit_share_segment(part, parent_body_uri=body_uri)) + if len(shares) != int(proof.get("share_count") or 0): + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: share count mismatch" + ) + if shares and ( + int(shares[0].get("share_seq") or 0) + != int(proof.get("first_share_seq") or 0) + or int(shares[-1].get("share_seq") or 0) + != int(proof.get("last_share_seq") or 0) + ): + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: proof range mismatch" + ) + bundle = self._insert_shares( + without_shares, + shares, + int(body.get("shares_key_index", len(without_shares))), + ) + reward_manifest = bundle.get("reward_manifest") + copied_proof_fields = ( + "anchor_job_issued_at_ms", + "anchor_share_seq", + "newest_share_seq", + "oldest_share_seq", + "included_share_count", + "requested_window_weight", + "counted_window_weight", + "share_slice_digest_hex", + ) + if isinstance(reward_manifest, dict): + for field in copied_proof_fields: + if (field in proof) != (field in reward_manifest) or proof.get( + field + ) != reward_manifest.get(field): + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: proof {field} mismatch" + ) + elif any(field in proof for field in copied_proof_fields): + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: proof has no reward manifest" + ) + if verify_digest: + actual = _sha256_bytes(self.canonical_audit_bundle_bytes(bundle)) + if not hmac.compare_digest(actual, declared): + raise RuntimeError( + f"audit bundle body hash mismatch at {body_uri}: expected {declared}, got {actual}" + ) + return bundle + + def _validate_body_wrapper_identity( + self, + body: Mapping[str, Any], + body_uri: object, + declared_digest: str, + ) -> None: + path = self.resolve_owned_path(body_uri) + match = _BODY_RE.fullmatch(path.name) + if match is None: + raise RuntimeError("compact audit body path is invalid") + block_hash = _canonical_hex(body.get("block_hash"), name="block_hash") + if ( + block_hash != match.group("block") + or declared_digest != match.group("digest") + ): + raise RuntimeError("compact audit body identity mismatch") + + @staticmethod + def _insert_shares( + without_shares: Mapping[str, Any], + shares: list[Any], + shares_key_index: int, + ) -> dict[str, object]: + bundle: dict[str, object] = {} + inserted = False + for index, (key, value) in enumerate(without_shares.items()): + if index == shares_key_index: + bundle["shares"] = shares + inserted = True + bundle[str(key)] = value + if not inserted: + bundle["shares"] = shares + return bundle + + def read_audit_share_segment( + self, + part: Mapping[str, Any], + *, + parent_body_uri: object, + ) -> list[Any]: + body_uri = part.get("body_uri") + kind = str(part.get("kind") or "") + if kind not in {"segment", "segment_range", "segment_prefix"}: + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: " + "invalid share part kind" + ) + try: + path = self.resolve_owned_path(body_uri) + if self.artifact_kind(path.name) != "share_segment": + raise RuntimeError("not an owned share segment") + if kind == "segment": + match = _SHARE_CONTENT_RE.fullmatch(path.name) + if match is None: + raise RuntimeError("immutable share segment path is invalid") + if ( + int(match.group("first")) != int(part.get("first_share_seq") or 0) + or int(match.group("last")) != int(part.get("last_share_seq") or 0) + ): + raise RuntimeError("immutable share segment path bounds mismatch") + elif kind in {"segment_range", "segment_prefix"}: + match = _SHARE_SLOT_RE.fullmatch(path.name) + if match is None: + raise RuntimeError("share segment slot path is invalid") + declared_slot_first = int( + part.get("segment_first_share_seq") + or match.group("first") + ) + declared_slot_last = int( + part.get("segment_last_share_seq") + or match.group("last") + ) + if ( + declared_slot_first != int(match.group("first")) + or declared_slot_last != int(match.group("last")) + ): + raise RuntimeError("share segment slot path bounds mismatch") + segment_bytes, _value = self._read_owned_regular_bytes(path) + except (OSError, RuntimeError) as exc: + raise RuntimeError( + f"audit bundle body is not retrievable at {parent_body_uri}: " + f"share segment {body_uri}: {exc}" + ) from exc + if kind == "segment": + expected = str(part.get("sha256") or "").lower() + actual = _sha256_bytes(segment_bytes) + if not hmac.compare_digest(actual, expected): + raise RuntimeError( + f"audit bundle body hash mismatch at {parent_body_uri}: " + f"share segment {body_uri} expected {expected}, got {actual}" + ) + try: + segment = json.loads(segment_bytes) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: " + f"share segment {body_uri}: {exc}" + ) from exc + if ( + not isinstance(segment, dict) + or segment.get("schema") != AUDIT_SHARE_SEGMENT_SCHEMA + or not isinstance(segment.get("shares"), list) + ): + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: " + f"invalid share segment {body_uri}" + ) + segment_shares = segment["shares"] + try: + segment_first = int(segment.get("first_share_seq")) + segment_last = int(segment.get("last_share_seq")) + segment_count = int(segment.get("share_count")) + self._validate_share_range( + segment_first, + segment_last, + segment_shares, + ) + except (TypeError, ValueError, RuntimeError) as exc: + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: " + f"share segment {body_uri} header mismatch" + ) from exc + if segment_count != len(segment_shares): + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: " + f"share segment {body_uri} share count mismatch" + ) + if kind == "segment": + assert match is not None + if ( + segment_first != int(match.group("first")) + or segment_last != int(match.group("last")) + ): + raise RuntimeError("immutable share segment header bounds mismatch") + else: + assert match is not None + if ( + segment_first < int(match.group("first")) + or segment_last > int(match.group("last")) + ): + raise RuntimeError("share segment header escapes slot bounds") + selected = self.select_audit_share_segment_range( + segment_shares, + first_share_seq=int(part.get("first_share_seq") or 0), + last_share_seq=int(part.get("last_share_seq") or 0), + parent_body_uri=parent_body_uri, + body_uri=body_uri, + ) + if len(selected) != int(part.get("share_count") or 0): + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: " + f"share segment {body_uri} share count mismatch" + ) + if kind in {"segment_range", "segment_prefix"}: + key = "range_sha256" if kind == "segment_range" else "prefix_sha256" + actual = _sha256_bytes( + self.storage_json_bytes( + self.audit_share_segment_payload( + first_share_seq=int(part.get("first_share_seq") or 0), + last_share_seq=int(part.get("last_share_seq") or 0), + shares=selected, + ) + ) + ) + expected = str(part.get(key) or "").lower() + if not hmac.compare_digest(actual, expected): + raise RuntimeError( + f"audit bundle body hash mismatch at {parent_body_uri}: " + f"share segment range {body_uri} expected {expected}, got {actual}" + ) + elif kind != "segment": + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: invalid share part kind" + ) + self._validate_root_identity() + return selected + + @staticmethod + def select_audit_share_segment_range( + shares: list[Any], + *, + first_share_seq: int, + last_share_seq: int, + parent_body_uri: object, + body_uri: object, + ) -> list[Any]: + selected: list[Any] = [] + previous: int | None = None + for share in shares: + if not isinstance(share, dict): + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: " + f"share segment {body_uri} has invalid share" + ) + share_seq = int(share.get("share_seq") or 0) + if previous is not None and previous + 1 != share_seq: + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: " + f"share segment {body_uri} is not contiguous" + ) + previous = share_seq + if first_share_seq <= share_seq <= last_share_seq: + selected.append(share) + if ( + not selected + or int(selected[0].get("share_seq") or 0) != first_share_seq + or int(selected[-1].get("share_seq") or 0) != last_share_seq + ): + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: " + f"share segment {body_uri} does not contain requested range" + ) + return selected + + +def _canonical_hex_bytes(value: object, *, name: str) -> str: + text = str(value) + if len(text) % 2: + raise ValueError(f"{name} must contain complete hexadecimal bytes") + try: + bytes.fromhex(text) + except ValueError as exc: + raise ValueError(f"{name} must be hexadecimal") from exc + return text.lower() diff --git a/lab/prism/bundle_compiler.py b/lab/prism/bundle_compiler.py index 1012a2a..6010e04 100644 --- a/lab/prism/bundle_compiler.py +++ b/lab/prism/bundle_compiler.py @@ -7,9 +7,11 @@ import json import os from pathlib import Path +import stat import subprocess import tempfile import time +import uuid from typing import Any, Callable, Protocol from lab.prism.prism_tools import prism_tool_command @@ -19,6 +21,26 @@ PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS = 0.05 +def canonical_bundle_bytes(bundle: dict[str, Any]) -> bytes: + """Use J1's typed Rust adapter for canonical audit-bundle bytes. + + Generic compact JSON is not equivalent: serde restores typed field order, + emits UTF-8, and omits optional defaults. Keep those digest semantics in J1. + """ + + completed = subprocess.run( + prism_tool_command("qbit-prism-audit-canonicalize") + ["--input", "-"], + input=json.dumps(bundle).encode("utf-8"), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if completed.returncode != 0: + stderr = completed.stderr.decode(errors="replace").strip() + raise RuntimeError(f"qbit-prism-audit-canonicalize failed: {stderr}") + return completed.stdout + + class CancellationPort(Protocol): def is_set(self) -> bool: ... @@ -73,6 +95,8 @@ def build_audit_bundle( witness_merkle_leaves_hex: list[str] | None = None, ctv_fee_parent_hash: str | None = None, canonical_output_path: Path | None = None, + canonical_output_parent_fd: int | None = None, + canonical_output_adopter: Callable[[Path, os.stat_result], None] | None = None, summary_only: bool = False, payout_policy: dict[str, object] | None = None, ctv_settlement: dict[str, object] | None = None, @@ -148,10 +172,14 @@ def build_audit_bundle( command.append("--job-summary-output" if summary_only else "--canonical-output") if record_phase_metrics: command.append("--phase-metrics") - if canonical_output_path is not None: + if ( + canonical_output_path is not None + and canonical_output_parent_fd is None + ): canonical_output_path.parent.mkdir(parents=True, exist_ok=True) succeeded = False created_output = False + created_output_fd: int | None = None try: with ExitStack() as stack: if canonical_output_path is None: @@ -159,10 +187,28 @@ def build_audit_bundle( tempfile.TemporaryFile(mode="w+", encoding="utf-8") ) else: - output = stack.enter_context( - canonical_output_path.open("x+", encoding="utf-8") - ) + if canonical_output_parent_fd is None: + output = stack.enter_context( + canonical_output_path.open("x+", encoding="utf-8") + ) + else: + output_fd = os.open( + canonical_output_path.name, + os.O_RDWR + | os.O_CREAT + | os.O_EXCL + | getattr(os, "O_NOFOLLOW", 0), + 0o666, + dir_fd=canonical_output_parent_fd, + ) + output = stack.enter_context( + os.fdopen(output_fd, "w+", encoding="utf-8") + ) created_output = True + # Keep the created inode allocated until cleanup compares + # it with the published name. This closes the Linux inode- + # reuse window after an adopter replaces the path. + created_output_fd = os.dup(output.fileno()) stderr = stack.enter_context( tempfile.TemporaryFile(mode="w+", encoding="utf-8") ) @@ -382,14 +428,119 @@ def write(self, value: str) -> int: ) if cancellation is not None: cancellation.raise_if_cancelled("builder verification") + if canonical_output_path is not None and canonical_output_adopter is not None: + canonical_output_adopter( + canonical_output_path, + os.fstat(output.fileno()), + ) succeeded = True return bundle finally: - if canonical_output_path is not None and created_output and not succeeded: - try: - canonical_output_path.unlink() - except FileNotFoundError: - pass + try: + if ( + canonical_output_path is not None + and created_output + and not succeeded + ): + created_output_identity = ( + None + if created_output_fd is None + else os.fstat(created_output_fd) + ) + self._remove_created_output_if_same( + canonical_output_path, + created_output_identity, + parent_fd=canonical_output_parent_fd, + ) + finally: + if created_output_fd is not None: + os.close(created_output_fd) + + @staticmethod + def _remove_created_output_if_same( + path: Path, + identity: os.stat_result | None, + *, + parent_fd: int | None = None, + ) -> None: + if identity is None: + return + try: + current = ( + path.lstat() + if parent_fd is None + else os.stat(path.name, dir_fd=parent_fd, follow_symlinks=False) + ) + except FileNotFoundError: + return + if not stat.S_ISREG(current.st_mode): + # Cleanup has no portable no-replace restore primitive for a + # directory/symlink. Reject it before moving an unowned path. + return + if not BundleCompiler._output_identity_matches(identity, current): + # Preserve a replacement already visible at the canonical name. + return + quarantine = path.with_name(f".{path.name}.{uuid.uuid4().hex}.cleanup") + try: + if parent_fd is None: + os.replace(path, quarantine) + else: + os.replace( + path.name, + quarantine.name, + src_dir_fd=parent_fd, + dst_dir_fd=parent_fd, + ) + except FileNotFoundError: + return + moved = ( + quarantine.lstat() + if parent_fd is None + else os.stat( + quarantine.name, + dir_fd=parent_fd, + follow_symlinks=False, + ) + ) + if BundleCompiler._output_identity_matches(identity, moved): + if parent_fd is None: + quarantine.unlink() + else: + os.unlink(quarantine.name, dir_fd=parent_fd) + return + try: + if parent_fd is None: + os.link(quarantine, path, follow_symlinks=False) + else: + os.link( + quarantine.name, + path.name, + src_dir_fd=parent_fd, + dst_dir_fd=parent_fd, + follow_symlinks=False, + ) + except OSError: + # A second replacement won after quarantine. Keep both artifacts; + # an unowned non-regular replacement is likewise preserved there. + return + if parent_fd is None: + quarantine.unlink() + else: + os.unlink(quarantine.name, dir_fd=parent_fd) + + @staticmethod + def _output_identity_matches( + identity: os.stat_result, + value: os.stat_result, + ) -> bool: + return bool( + value.st_dev == identity.st_dev + and value.st_ino == identity.st_ino + and value.st_mode == identity.st_mode + and value.st_size == identity.st_size + and value.st_mtime_ns == identity.st_mtime_ns + and stat.S_ISREG(value.st_mode) + ) def _record_phase_metrics( self, diff --git a/lab/prism/prism_coordinator.py b/lab/prism/prism_coordinator.py index b3c0348..bb5052a 100644 --- a/lab/prism/prism_coordinator.py +++ b/lab/prism/prism_coordinator.py @@ -6,6 +6,7 @@ from collections import OrderedDict from concurrent.futures import Future, ThreadPoolExecutor from contextlib import ExitStack, contextmanager +import copy import dataclasses import hashlib import json @@ -20,7 +21,6 @@ import traceback import urllib.parse import urllib.request -import uuid from types import SimpleNamespace from dataclasses import dataclass, replace as dataclass_replace from decimal import Decimal, ROUND_CEILING @@ -45,7 +45,12 @@ _BoundedPriorityExecutor, # noqa: F401 - compatibility re-export _DeliveryQueueFull, # noqa: F401 - compatibility re-export ) -from lab.prism.prism_tools import prism_tool_command +from lab.prism.audit_artifacts import ( + AuditArtifactConfig, + AuditArtifactStore, + AuditPublicationIdentity, +) +from lab.prism.bundle_compiler import canonical_bundle_bytes from lab.prism.ctv_broadcaster import CtvFanoutBroadcaster from lab.prism.coordinator_config import ( CoordinatorConfig, @@ -1721,7 +1726,6 @@ def __init__(self, config: CoordinatorConfig | None = None) -> None: self.ledger_writer_public_key_hex = ledger_config.writer_public_key_hex self.evidence_path = audit_config.evidence_path self.audit_dir = audit_config.directory - self.audit_dir.mkdir(parents=True, exist_ok=True) self.audit_share_segment_size = audit_config.share_segment_size self.audit_live_bundle_retention = audit_config.live_bundle_retention self.audit_candidate_retention_seconds = audit_config.candidate_retention_seconds @@ -1739,6 +1743,7 @@ def __init__(self, config: CoordinatorConfig | None = None) -> None: lifecycle_config.writer_quiescence_timeout_seconds ) self.ledger = self.make_ledger() + self._upgrade_legacy_audit_evidence() self._ctv_fanout_market_fee_rate_cache: dict[tuple[int | None, str | None], int] = {} self.lock = _ObservedRLock() self.clients: set[ClientState] = set() @@ -1855,7 +1860,6 @@ def __init__(self, config: CoordinatorConfig | None = None) -> None: self.reorg_reconcile_skip_count = 0 self.reorg_reconcile_error_count = 0 self.matured_payout_count = 0 - self.latest_evidence: dict[str, Any] | None = None # The full accepted-block bundle is durable in the audit store. Keeping # it here only to derive one metric pinned the complete share window for # the lifetime of the coordinator. @@ -2032,6 +2036,166 @@ def share_weight_for_worker(self, worker: WorkerIdentity) -> int: self.share_weights_by_username.get(worker.payout_address, self.default_share_weight), ) + def _ensure_audit_artifact_store(self) -> AuditArtifactStore: + init_lock = self.__dict__.setdefault( + "_audit_artifact_store_init_lock", + threading.Lock(), + ) + assert isinstance(init_lock, type(threading.Lock())) + with init_lock: + audit_dir = Path( + self.__dict__.get("audit_dir", Path("prism-audit")) + ) + evidence_path = Path( + self.__dict__.get( + "evidence_path", + audit_dir / "prism-live-stratum-evidence.json", + ) + ) + live_retention = int( + self.__dict__.get("audit_live_bundle_retention", 5) + ) + candidate_retention = int( + self.__dict__.get( + "audit_candidate_retention_seconds", + 24 * 60 * 60, + ) + ) + share_segment_size = int( + self.__dict__.get( + "audit_share_segment_size", + DEFAULT_AUDIT_SHARE_SEGMENT_SIZE, + ) + ) + store = self.__dict__.get("_audit_artifact_store") + if not isinstance(store, AuditArtifactStore): + store = AuditArtifactStore( + AuditArtifactConfig( + root=audit_dir, + evidence_path=evidence_path, + live_bundle_retention=live_retention, + candidate_retention_seconds=candidate_retention, + share_segment_size=share_segment_size, + ), + canonicalizer=canonical_bundle_bytes, + ) + self.__dict__["_audit_artifact_store"] = store + if "_audit_latest_evidence_seed" in self.__dict__: + store.set_latest_evidence_for_compatibility( + self.__dict__.pop("_audit_latest_evidence_seed") + ) + else: + updates: dict[str, Any] = {} + if store.root != audit_dir.expanduser().absolute().resolve(): + updates["root"] = audit_dir + expected_evidence = ( + evidence_path.expanduser().absolute().parent.resolve() + / evidence_path.name + ) + if store.evidence_path != expected_evidence: + updates["evidence_path"] = evidence_path + if store.live_bundle_retention != live_retention: + updates["live_bundle_retention"] = live_retention + if store.candidate_retention_seconds != candidate_retention: + updates["candidate_retention_seconds"] = candidate_retention + if store.share_segment_size != share_segment_size: + updates["share_segment_size"] = share_segment_size + if updates: + store.reconfigure(**updates) + return store + + def _upgrade_legacy_audit_evidence(self) -> None: + store = self._ensure_audit_artifact_store() + with self._ensure_payout_state_service().balance_mutation_lock: + with store.publication_order_guard(): + legacy = store.legacy_evidence_identity() + if legacy is None: + return + reader = getattr(self.ledger, "pool_block_state", None) + floor_reader = getattr( + self.ledger, + "audit_publication_sequence_floor", + None, + ) + if not callable(reader) or not callable(floor_reader): + store.invalidate_unprovable_legacy_evidence() + return + state = reader(block_hash=legacy.block_hash) + if not isinstance(state, dict): + store.invalidate_unprovable_legacy_evidence() + return + sequence = state.get("audit_publication_sequence") + state_block_hash = state.get("block_hash") + state_block_height = state.get("block_height") + if ( + sequence is None + or isinstance(sequence, bool) + or not isinstance(sequence, int) + or sequence <= 0 + or not isinstance(state_block_hash, str) + or state_block_hash != legacy.block_hash + or isinstance(state_block_height, bool) + or not isinstance(state_block_height, int) + or state_block_height != legacy.block_height + or str(state.get("chain_state") or "") != "confirmed" + or str(state.get("maturity_state") or "") + not in {"immature", "mature"} + ): + store.invalidate_unprovable_legacy_evidence() + return + publication_floor_sequence = floor_reader() + store.adopt_legacy_publication_identity( + AuditPublicationIdentity( + int(sequence), + legacy.block_height, + legacy.block_hash, + ), + publication_floor_sequence=publication_floor_sequence, + ) + + def _audit_publication_identity( + self, + *, + block_hash: str, + block_height: int, + confirmation: Mapping[str, Any], + ) -> AuditPublicationIdentity: + sequence = confirmation.get("audit_publication_sequence") + if sequence is None: + # Compatibility-only fake ledgers in unit tests predate the durable + # ordinal. Production Postgres and the memory ledger always return + # it; never synthesize for an identified durable backend. + if str(confirmation.get("backend") or "") not in {"", "fake"}: + raise RuntimeError( + "ledger confirmation omitted audit publication sequence" + ) + sequences = self.__dict__.setdefault( + "_compat_audit_publication_sequences", + {}, + ) + assert isinstance(sequences, dict) + sequence = sequences.get(block_hash) + if sequence is None: + sequence = max( + [ + self._ensure_audit_artifact_store().publication_sequence_floor(), + *(int(value) for value in sequences.values()), + ] + ) + 1 + sequences[block_hash] = sequence + if isinstance(sequence, bool) or not isinstance(sequence, int): + raise RuntimeError("ledger confirmation returned invalid publication sequence") + if isinstance(block_height, bool) or not isinstance(block_height, int): + raise RuntimeError("ledger confirmation returned invalid block height") + canonical_block_hash = str(block_hash).lower() + if canonical_block_hash != block_hash: + raise RuntimeError("ledger confirmation returned non-canonical block hash") + return AuditPublicationIdentity( + sequence, + block_height, + canonical_block_hash, + ) + def make_ledger(self) -> SingleWriterShareLedger | PsqlShareLedger: config = getattr(self, "config", None) ledger_config = config.ledger if config is not None else None @@ -2084,7 +2248,7 @@ def make_ledger(self) -> SingleWriterShareLedger | PsqlShareLedger: "PRISM_LEDGER_WRITER_SESSION_TOKEN requires " "PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN=1 for local tests" ) - audit_body_dir = getattr(self, "audit_dir", None) + audit_store = self._ensure_audit_artifact_store() return PsqlShareLedger( psql_command=psql_command, database_url=database_url or None, @@ -2132,8 +2296,7 @@ def make_ledger(self) -> SingleWriterShareLedger | PsqlShareLedger: 30.0, ) ), - audit_body_dir=str(audit_body_dir) if audit_body_dir is not None else None, - audit_share_segment_size=getattr(self, "audit_share_segment_size", DEFAULT_AUDIT_SHARE_SEGMENT_SIZE), + audit_artifact_store=audit_store, ctv_broadcast_attempt_detail_limit=getattr( self, "ctv_broadcast_attempt_detail_limit", @@ -6584,12 +6747,11 @@ def retry_superseded_candidate() -> bool: on_active_chain and chain_state == "inactive" ): - reactivated = ( - self.ledger.reactivate_pool_block( + with self._ensure_audit_artifact_store().publication_order_guard(): + reactivated = self.ledger.reactivate_pool_block( block_hash=block_hash, active_tip_height=active_tip_height, ) - ) reactivated_count = int( reactivated.get( "reactivated_count", @@ -7039,6 +7201,8 @@ def build_audit_bundle( witness_merkle_leaves_hex: list[str] | None = None, ctv_fee_parent_hash: str | None = None, canonical_output_path: Path | None = None, + canonical_output_parent_fd: int | None = None, + canonical_output_adopter: Callable[[Path, os.stat_result], None] | None = None, summary_only: bool = False, payout_policy: dict[str, object] | None = None, ctv_settlement: dict[str, object] | None = None, @@ -7053,6 +7217,8 @@ def build_audit_bundle( witness_merkle_leaves_hex=witness_merkle_leaves_hex, ctv_fee_parent_hash=ctv_fee_parent_hash, canonical_output_path=canonical_output_path, + canonical_output_parent_fd=canonical_output_parent_fd, + canonical_output_adopter=canonical_output_adopter, summary_only=summary_only, payout_policy=payout_policy, ctv_settlement=ctv_settlement, @@ -9268,6 +9434,8 @@ def _land_and_confirm_block_candidate( dict[str, Any], dict[str, Any], dict[str, Any], + AuditPublicationIdentity, + dict[str, Any], ] | None: """Land, verify, publish, persist, and confirm one candidate. @@ -9475,35 +9643,62 @@ def _land_and_confirm_block_candidate( self._publish_accepted_block_payout_preview(block_hash, preview) self._record_heartbeat("block_submitter") - candidate_bundle_path = self.temporary_audit_bundle_path( + audit_store = self._ensure_audit_artifact_store() + candidate_artifact = audit_store.issue_candidate( block_hash=submission.block_hash_hex ) - final_bundle = self.build_audit_bundle( - shares=context.shares_json, - found_block=context.found_block, - prior_balances=context.prior_balances, - coinbase_script_sig_suffix_hex=self.coinbase_script_sig_suffix_hex( - candidate.extranonce1_hex, - candidate.extranonce2_hex, - ), - witness_merkle_leaves_hex=list( - getattr(context.job, "witness_merkle_leaves_hex", ()) + candidate_bundle_path = candidate_artifact.path + compiler_transferred_candidate = False + + def adopt_compiler_output(path: Path, value: os.stat_result) -> None: + nonlocal compiler_transferred_candidate + audit_store.adopt_compiler_candidate( + candidate_artifact, + path=path, + value=value, ) - or direct_stratum.witness_merkle_leaves_hex( - getattr(context.job, "transaction_hexes", ()) - ), - ctv_fee_parent_hash=parent_hash, - canonical_output_path=candidate_bundle_path, - ) + compiler_transferred_candidate = True + + compiler_parent_fd = audit_store.duplicate_root_directory_fd() + try: + final_bundle = self.build_audit_bundle( + shares=context.shares_json, + found_block=context.found_block, + prior_balances=context.prior_balances, + coinbase_script_sig_suffix_hex=self.coinbase_script_sig_suffix_hex( + candidate.extranonce1_hex, + candidate.extranonce2_hex, + ), + witness_merkle_leaves_hex=list( + getattr(context.job, "witness_merkle_leaves_hex", ()) + ) + or direct_stratum.witness_merkle_leaves_hex( + getattr(context.job, "transaction_hexes", ()) + ), + ctv_fee_parent_hash=parent_hash, + canonical_output_path=candidate_bundle_path, + canonical_output_parent_fd=compiler_parent_fd, + canonical_output_adopter=adopt_compiler_output, + ) + except BaseException: + audit_store.discard_candidate(candidate_artifact) + raise + finally: + os.close(compiler_parent_fd) # Compatibility builders used by tests and older integrations may # ignore canonical_output_path. Persist their logical bundle via # the normal canonicalization fallback without mislabeling bytes. - if not candidate_bundle_path.exists(): - candidate_bundle_path = self.write_temporary_audit_bundle( - final_bundle, - block_hash=submission.block_hash_hex, - ) try: + if not candidate_bundle_path.exists(): + candidate_bundle_path = audit_store.write_compatibility_candidate( + candidate_artifact, + final_bundle, + ) + else: + if not compiler_transferred_candidate: + raise RuntimeError( + "audit builder created an output path without exact inode transfer" + ) final_manifest = final_bundle["signed_coinbase_manifest"]["manifest"] final_coinbase_tx_hex_raw = final_manifest["coinbase_tx_hex"] if not isinstance(final_coinbase_tx_hex_raw, str): @@ -9512,16 +9707,10 @@ def _land_and_confirm_block_candidate( ) final_coinbase_tx_hex = final_coinbase_tx_hex_raw.lower() except BaseException: - try: - candidate_bundle_path.unlink() - except FileNotFoundError: - pass + audit_store.discard_candidate(candidate_artifact) raise if final_coinbase_tx_hex != submission.coinbase_tx_hex.lower(): - try: - candidate_bundle_path.unlink() - except FileNotFoundError: - pass + audit_store.discard_candidate(candidate_artifact) self.request_shutdown() self._clear_accepted_block_payout_preview( block_hash, @@ -9536,17 +9725,40 @@ def _land_and_confirm_block_candidate( payout_commit_started: float | None = None payout_commit_source: int | None = None try: - report = self.verify_bundle( - candidate_bundle_path, - submission.coinbase_tx_hex, - self.trusted_ledger_writer_public_key_hex(final_bundle), + verifier_override = self.__dict__.get("verify_bundle") + configured_writer_key = getattr( + self, + "ledger_writer_public_key_hex", + None, + ) + verified_audit = audit_store.verify_candidate( + candidate_artifact, + coinbase_tx_hex=submission.coinbase_tx_hex, expected_coinbase_value_sats=int(context.template["coinbasevalue"]), + expected_block_height=expected_height, + trusted_writer_public_key_hex=( + self.trusted_ledger_writer_public_key_hex(final_bundle) + ), + trust_source=( + "configured" + if configured_writer_key is not None + else "embedded_test_only" + ), + verifier=( + verifier_override + if callable(verifier_override) + else None + ), + ) + audit_store.require_current_verified_candidate( + verified_audit, + candidate_artifact, ) + report = dict(verified_audit.report) persistence_canonical_bundle_path = ( - self.verified_canonical_bundle_path( - candidate_bundle_path, - report, - ) + candidate_bundle_path + if verified_audit.canonical_copy_eligible + else None ) self._record_heartbeat("block_submitter") verified_preview = self._accepted_block_payout_preview_from_bundle( @@ -9633,15 +9845,24 @@ def _land_and_confirm_block_candidate( worker=worker, ) return None - confirmation = self.ledger.confirm_accepted_block( - block_hash=block_hash, - # The ledger confirmation function matches this value - # against the candidate row's own height. An accepted - # ancestor can be finalized after newer blocks arrive. - active_tip_height=expected_height, - ) - confirmed_count = int(confirmation.get("confirmed_count", 0)) - if confirmed_count not in {0, 1}: + with audit_store.publication_order_guard(): + confirmation = self.ledger.confirm_accepted_block( + block_hash=block_hash, + # The ledger confirmation function matches this value + # against the candidate row's own height. An accepted + # ancestor can be finalized after newer blocks arrive. + active_tip_height=expected_height, + ) + confirmed_count = int(confirmation.get("confirmed_count", 0)) + if confirmed_count == 1: + audit_publication_identity = ( + self._audit_publication_identity( + block_hash=block_hash, + block_height=expected_height, + confirmation=confirmation, + ) + ) + if confirmed_count != 1: self.request_shutdown() self._clear_accepted_block_payout_preview( block_hash, @@ -9748,7 +9969,14 @@ def _land_and_confirm_block_candidate( f"scheduled refresh hash={block_hash}", flush=True, ) - return final_bundle, report, persistence, confirmation + return ( + final_bundle, + report, + persistence, + confirmation, + audit_publication_identity, + dict(verified_audit.verification_identity), + ) except Exception: if payout_commit_started is not None and payout_commit_source is not None: # Persistence/confirmation can report failure after a @@ -9770,10 +9998,7 @@ def _land_and_confirm_block_candidate( "preparation", max(0.0, time.monotonic() - payout_commit_started), ) - try: - candidate_bundle_path.unlink() - except FileNotFoundError: - pass + audit_store.discard_candidate(candidate_artifact) @ledger_writer_operation("accepted_block_handling") def submit_block_candidate(self, candidate: PrismBlockCandidate) -> bool: @@ -9868,7 +10093,14 @@ def submit_block_candidate(self, candidate: PrismBlockCandidate) -> bool: ) if landed is None: return False - final_bundle, report, persistence, confirmation = landed + ( + final_bundle, + report, + persistence, + confirmation, + audit_publication_identity, + audit_verification_identity, + ) = landed with self.lock: already_accounted = block_hash in self._accounted_accepted_block_hashes if already_accounted: @@ -9885,19 +10117,6 @@ def submit_block_candidate(self, candidate: PrismBlockCandidate) -> bool: manifest_set=ctv_manifest_set, manifest_set_sha256=sha256_json_hex(ctv_manifest_set), ) - final_bundle_path = ( - self.audit_dir - / f"prism-live-audit-bundle-{expected_height}-{block_hash}.json" - ) - self.write_audit_bundle_envelope( - final_bundle_path, - block_hash=block_hash, - block_height=expected_height, - report=report, - persistence=persistence, - ) - self.prune_audit_artifacts(keep_live_path=final_bundle_path) - bundle_path = final_bundle_path if candidate.credit_share_on_accept: self.append_accepted_share( candidate.client, @@ -9915,17 +10134,56 @@ def submit_block_candidate(self, candidate: PrismBlockCandidate) -> bool: "block_hash": block_hash, "block_height": expected_height, "coinbase_tx_hex": submission.coinbase_tx_hex, - "audit_bundle_path": str(bundle_path), "audit_report": report, "ledger_backend": self.ledger.backend_name, "persistence": persistence, "confirmation": confirmation, + "audit_verification_identity": audit_verification_identity, "ctv_persistence": ctv_persistence, "accepted_share_count": evidence_share_count, "distinct_miner_count": evidence_distinct_miners, "job_share_count": len(context.shares_json), } - self.evidence_path.write_text(json.dumps(evidence, indent=2), encoding="utf-8") + publication_persistence = dict(persistence) + publication_persistence.setdefault( + "audit_bundle_sha256", + report.get("audit_bundle_sha256_hex"), + ) + publication_persistence.setdefault("body_uri", "") + evidence["persistence"] = publication_persistence + audit_store = self._ensure_audit_artifact_store() + with self._ensure_payout_state_service().balance_mutation_lock: + with audit_store.publication_order_guard(): + publication_floor_reader = getattr( + self.ledger, + "audit_publication_sequence_floor", + None, + ) + if callable(publication_floor_reader): + # This is deliberately a fresh durable-row read immediately + # before A1 publication. Confirmation-time state or a raw + # sequence value cannot fence rollback gaps and restart + # replays. P1's local serializer plus A1's process guard + # prevent another confirmation/reactivation from allocating + # between this read and the durable publication decision. + publication_floor_sequence = publication_floor_reader() + else: + # Compatibility-only ledgers used by legacy embeddings/tests + # do not own durable ordinal state. Production memory/Postgres + # backends implement the reader above. + publication_floor_sequence = ( + audit_publication_identity.sequence + ) + publication = audit_store.publish_success( + identity=audit_publication_identity, + publication_floor_sequence=publication_floor_sequence, + report=report, + persistence=publication_persistence, + evidence=evidence, + verification_identity=audit_verification_identity, + created_at=public_api.utc_now_iso(), + ) + evidence = dict(publication.evidence) with self.lock: newly_accounted = block_hash not in self._accounted_accepted_block_hashes if newly_accounted: @@ -9938,7 +10196,6 @@ def submit_block_candidate(self, candidate: PrismBlockCandidate) -> bool: ] ) ) // 2 - self.latest_evidence = evidence should_stop = ( newly_accounted and (self.stop_after_block or self.accepted_block_count >= self.max_blocks) @@ -9973,119 +10230,20 @@ def reject_prepared_block(self, *, block_hash: str, active_tip_height: int) -> d active_tip_height=active_tip_height, ) - def temporary_audit_bundle_path(self, *, block_hash: str) -> Path: - self.audit_dir.mkdir(parents=True, exist_ok=True) - return self.audit_dir / ( - f".prism-live-audit-bundle-candidate-{block_hash}-{uuid.uuid4().hex}.json.tmp" - ) - @staticmethod def verified_canonical_bundle_path( candidate_bundle_path: Path, report: dict[str, Any], ) -> Path | None: - expected_sha256 = str(report["audit_bundle_sha256_hex"]).lower() - digest = hashlib.sha256() - with candidate_bundle_path.open("rb") as handle: - while chunk := handle.read(1024 * 1024): - digest.update(chunk) - if digest.hexdigest() != expected_sha256: - return None - return candidate_bundle_path - - def write_temporary_audit_bundle(self, bundle: dict[str, Any], *, block_hash: str) -> Path: - path = self.temporary_audit_bundle_path(block_hash=block_hash) - with path.open("x", encoding="utf-8") as handle: - json.dump(bundle, handle, separators=(",", ":")) - handle.flush() - os.fsync(handle.fileno()) - return path - - def write_audit_bundle_envelope( - self, - path: Path, - *, - block_hash: str, - block_height: int, - report: dict[str, Any], - persistence: dict[str, Any], - ) -> None: - audit_bundle_sha256 = str( - persistence.get("audit_bundle_sha256") - or report.get("audit_bundle_sha256_hex") - or "" - ).lower() - body_uri = str(persistence.get("body_uri") or "") - envelope = { - "schema": "qbit.prism.live-audit-bundle-envelope.v1", - "block_hash": block_hash, - "block_height": block_height, - "audit_bundle_sha256": audit_bundle_sha256, - "body_uri": body_uri, - "body_filename": Path(body_uri).name if body_uri else None, - "coinbase_txid": report.get("coinbase_txid"), - "coinbase_manifest_sha256": report.get("coinbase_manifest_sha256_hex"), - "coinbase_tx_hex": report.get("coinbase_tx_hex"), - "created_at": public_api.utc_now_iso(), - } - self.write_json_atomically(path, envelope) - - def write_json_atomically(self, path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - body = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8") - tmp_path = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") - try: - with tmp_path.open("xb") as handle: - handle.write(body) - handle.flush() - os.fsync(handle.fileno()) - tmp_path.replace(path) - finally: - try: - tmp_path.unlink() - except FileNotFoundError: - pass + return AuditArtifactStore.verified_canonical_bundle_path( + candidate_bundle_path, + report, + ) def prune_audit_artifacts(self, *, keep_live_path: Path | None = None) -> None: - self.prune_live_audit_envelopes(keep_path=keep_live_path) - self.prune_candidate_audit_bundles() - - def prune_live_audit_envelopes(self, *, keep_path: Path | None = None) -> None: - retention = int(getattr(self, "audit_live_bundle_retention", 5)) - if retention < 0: - return - keep_resolved = keep_path.resolve() if keep_path is not None else None - retained_non_keep = max(retention - 1, 0) if keep_resolved is not None else retention - paths = sorted( - self.audit_dir.glob("prism-live-audit-bundle-[0-9]*.json"), - key=lambda path: path.stat().st_mtime if path.exists() else 0, - reverse=True, - ) - retained_count = 0 - for path in paths: - if keep_resolved is not None and path.resolve() == keep_resolved: - continue - if retained_count < retained_non_keep: - retained_count += 1 - continue - try: - path.unlink() - except FileNotFoundError: - pass - - def prune_candidate_audit_bundles(self) -> None: - retention_seconds = int(getattr(self, "audit_candidate_retention_seconds", 24 * 60 * 60)) - now = time.time() - for pattern in ( - "prism-live-audit-bundle-candidate-*.json", - ".prism-live-audit-bundle-candidate-*.json.tmp", - ): - for path in self.audit_dir.glob(pattern): - try: - if retention_seconds == 0 or now - path.stat().st_mtime > retention_seconds: - path.unlink() - except FileNotFoundError: - pass + self._ensure_audit_artifact_store().prune_best_effort( + keep_live_path=keep_live_path + ) def verify_bundle( self, @@ -10094,34 +10252,23 @@ def verify_bundle( ledger_writer_public_key_hex: str, *, expected_coinbase_value_sats: int, + expected_block_height: int | None = None, ) -> dict[str, Any]: - completed = subprocess.run( - prism_tool_command("qbit-prism-audit-verify") - + [ - str(bundle_path), - "--coinbase-tx-hex", - coinbase_tx_hex, - "--ledger-writer-public-key-hex", - ledger_writer_public_key_hex, - "--expected-coinbase-value-sats", - str(expected_coinbase_value_sats), - ], - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=False, + return self._ensure_audit_artifact_store().verify_bundle( + bundle_path, + coinbase_tx_hex, + ledger_writer_public_key_hex, + expected_coinbase_value_sats=expected_coinbase_value_sats, + expected_block_height=expected_block_height, ) - if completed.returncode != 0: - raise RuntimeError(f"qbit-prism-audit-verify failed: {completed.stderr}") - return json.loads(completed.stdout) def trusted_ledger_writer_public_key_hex(self, bundle: dict[str, Any]) -> str: - if self.ledger_writer_public_key_hex is not None: - return self.ledger_writer_public_key_hex - return validate_hex( - str(bundle["ledger_window_attestation"]["signature"]["public_key_hex"]), - name="bundle ledger public key", - expected_bytes=32, + return AuditArtifactStore.trusted_writer_key( + getattr(self, "ledger_writer_public_key_hex", None), + bundle, + allow_embedded_test_key=( + getattr(self, "ledger_writer_public_key_hex", None) is None + ), ) @staticmethod @@ -10702,13 +10849,34 @@ def start_health_snapshot_refresher(self) -> None: registry.register(self._health_snapshot_service_spec()) self._start_background_service("health_snapshot_refresher") + @property + def latest_evidence(self) -> dict[str, Any] | None: + if ( + "_audit_artifact_store" not in self.__dict__ + and "audit_dir" not in self.__dict__ + and "evidence_path" not in self.__dict__ + ): + value = self.__dict__.get("_audit_latest_evidence_seed") + return copy.deepcopy(value) if isinstance(value, dict) else None + return self._ensure_audit_artifact_store().latest_evidence() + + @latest_evidence.setter + def latest_evidence(self, payload: Mapping[str, Any] | None) -> None: + if ( + "_audit_artifact_store" not in self.__dict__ + and "audit_dir" not in self.__dict__ + and "evidence_path" not in self.__dict__ + ): + self.__dict__["_audit_latest_evidence_seed"] = ( + copy.deepcopy(dict(payload)) if payload is not None else None + ) + return + self._ensure_audit_artifact_store().set_latest_evidence_for_compatibility( + payload + ) + def latest_evidence_payload(self) -> dict[str, object] | None: - with self.lock: - if self.latest_evidence is not None: - return dict(self.latest_evidence) - if self.evidence_path.exists(): - return json.loads(self.evidence_path.read_text(encoding="utf-8")) - return None + return self._ensure_audit_artifact_store().latest_evidence() def owed_balances_payload(self) -> dict[str, object]: return { @@ -11260,48 +11428,11 @@ def shutdown_metrics_lines(self) -> list[str]: ] def audit_artifact_metrics(self) -> dict[str, dict[str, int] | int]: - metrics: dict[str, dict[str, int] | int] = { - kind: {"files": 0, "bytes": 0} - for kind in ("body", "share_segment", "live_bundle", "candidate", "other") - } - metrics["scan_error"] = 0 - audit_dir = getattr(self, "audit_dir", None) - if audit_dir is None: - metrics["scan_error"] = 1 - return metrics - try: - paths = list(Path(audit_dir).iterdir()) - except OSError: - metrics["scan_error"] = 1 - return metrics - for path in paths: - try: - if not path.is_file(): - continue - size = path.stat().st_size - except OSError: - metrics["scan_error"] = 1 - continue - kind = self.audit_artifact_kind(path.name) - bucket = metrics[kind] - assert isinstance(bucket, dict) - bucket["files"] += 1 - bucket["bytes"] += size - return metrics + return self._ensure_audit_artifact_store().metrics_snapshot() @staticmethod def audit_artifact_kind(name: str) -> str: - if name.startswith("prism-audit-bundle-body-") and name.endswith(".json"): - return "body" - if name.startswith("prism-audit-share-segment-") and name.endswith(".json"): - return "share_segment" - if name.startswith("prism-live-audit-bundle-candidate-") or name.startswith( - ".prism-live-audit-bundle-candidate-" - ): - return "candidate" - if name.startswith("prism-live-audit-bundle-") and name.endswith(".json"): - return "live_bundle" - return "other" + return AuditArtifactStore.artifact_kind(name) def ctv_fanout_broadcaster_metrics_lines(self) -> list[str]: return self._ensure_ctv_runtime().metrics_lines() diff --git a/lab/prism/share_ledger.py b/lab/prism/share_ledger.py index e793956..dfbd323 100644 --- a/lab/prism/share_ledger.py +++ b/lab/prism/share_ledger.py @@ -6,7 +6,6 @@ import json import copy import hashlib -import hmac import os import math import shlex @@ -21,7 +20,12 @@ from threading import BoundedSemaphore, Lock from typing import Any, Callable, Iterator -from lab.prism.prism_tools import prism_tool_command +from lab.prism.audit_artifacts import ( + AuditArtifactConfig, + AuditArtifactStore, + canonical_audit_bundle_bytes, +) +from lab.prism.bundle_compiler import canonical_bundle_bytes AUDIT_BODY_REF_SCHEMA = "qbit.prism.audit-body-ref.v1" AUDIT_BUNDLE_V2_SCHEMA = "qbit.prism.audit-bundle.v2" @@ -144,6 +148,10 @@ def __init__( self._ctv_fanout_sets: dict[str, dict[str, Any]] = {} self._ctv_fanout_statuses: dict[str, dict[str, Any]] = {} self._ctv_fanout_attempts: dict[str, list[dict[str, Any]]] = {} + self._audit_publication_sequences: dict[str, int | None] = {} + self._next_audit_publication_sequence = 1 + self._inactive_audit_publications: set[str] = set() + self._memory_pool_blocks: dict[str, tuple[int, str, str]] = {} self._lock = Lock() def append(self, pending: PendingShare) -> AcceptedShareRecord: @@ -1049,47 +1057,192 @@ def persist_accepted_block( audit_report: dict[str, Any], canonical_bundle_path: Path | None = None, ) -> dict[str, int | str]: + block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) + with self._lock: + previous = self._memory_pool_blocks.get(block_hash) + if previous is not None and previous[0] != int(block_height): + raise RuntimeError("memory pool block height conflicts") + if previous is None: + self._memory_pool_blocks[block_hash] = ( + int(block_height), + "prepared", + str(parent_hash), + ) + self._audit_publication_sequences.setdefault(block_hash, None) + share_count = len(self._shares) return { "backend": "memory", - "share_count": len(self), + "share_count": share_count, "block_count": 0, "payout_entry_count": 0, "carry_forward_count": 0, } def reverse_immature_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, int | str]: + block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) + with self._lock: + block = self._memory_pool_blocks.get(block_hash) + if ( + block is not None + and block[1] in {"confirmed", "inactive"} + and int(active_tip_height) >= block[0] + 1000 + ): + raise RuntimeError( + f"refusing to reverse mature pool block {block_hash}" + ) + if block is None or block[1] not in { + "prepared", + "confirmed", + "inactive", + }: + count = 0 + else: + self._memory_pool_blocks[block_hash] = ( + block[0], + "reversed", + block[2], + ) + self._inactive_audit_publications.discard(block_hash) + count = 1 return { "backend": "memory", - "reversed_count": 0, + "reversed_count": count, } def reject_prepared_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, int | str]: + block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) + with self._lock: + block = self._memory_pool_blocks.get(block_hash) + if block is None or block[1] != "prepared": + count = 0 + else: + self._memory_pool_blocks[block_hash] = ( + block[0], + "rejected", + block[2], + ) + count = 1 return { "backend": "memory", - "rejected_count": 0, + "rejected_count": count, } def confirm_accepted_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, int | str]: + block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) + with self._lock: + block = self._memory_pool_blocks.get(block_hash) + if ( + block is None + or block[0] != int(active_tip_height) + or block[1] not in {"prepared", "confirmed"} + ): + return {"backend": "memory", "confirmed_count": 0} + publication_sequence = self._audit_publication_sequences.get(block_hash) + if publication_sequence is None: + publication_sequence = self._next_audit_publication_sequence + self._next_audit_publication_sequence += 1 + self._audit_publication_sequences[block_hash] = publication_sequence + self._memory_pool_blocks[block_hash] = ( + block[0], + "confirmed", + block[2], + ) return { "backend": "memory", "confirmed_count": 1, + "audit_publication_sequence": publication_sequence, } def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: return [] def mark_pool_block_inactive(self, *, block_hash: str, active_tip_height: int) -> dict[str, int | str]: + block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) + with self._lock: + block = self._memory_pool_blocks.get(block_hash) + if block is None or block[1] != "confirmed": + count = 0 + else: + self._inactive_audit_publications.add(block_hash) + self._memory_pool_blocks[block_hash] = ( + block[0], + "inactive", + block[2], + ) + count = 1 return { "backend": "memory", - "inactive_count": 0, + "inactive_count": count, } def reactivate_pool_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, int | str]: + block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) + with self._lock: + block = self._memory_pool_blocks.get(block_hash) + if ( + block is None + or block[0] > int(active_tip_height) + or block[1] != "inactive" + or block_hash not in self._inactive_audit_publications + ): + count = 0 + sequence = None + else: + sequence = self._audit_publication_sequences.get(block_hash) + if sequence is None: + raise RuntimeError( + "inactive pool block has no audit publication sequence" + ) + self._inactive_audit_publications.remove(block_hash) + self._memory_pool_blocks[block_hash] = ( + block[0], + "confirmed", + block[2], + ) + count = 1 return { "backend": "memory", - "reactivated_count": 0, + "reactivated_count": count, + **( + {"audit_publication_sequence": int(sequence)} + if sequence is not None + else {} + ), } + def pool_block_state(self, *, block_hash: str) -> dict[str, object] | None: + block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) + with self._lock: + block = self._memory_pool_blocks.get(block_hash) + if block is None: + return None + publication_sequence = self._audit_publication_sequences.get(block_hash) + return { + "block_hash": block_hash, + "block_height": block[0], + "parent_hash": block[2], + "chain_state": block[1], + "maturity_state": ( + "reversed" + if block[1] in {"rejected", "reversed"} + else "immature" + ), + "audit_publication_sequence": publication_sequence, + } + + def audit_publication_sequence_floor(self) -> int: + """Return the newest ordinal attached to any durable pool-block row.""" + + with self._lock: + return max( + ( + sequence + for sequence in self._audit_publication_sequences.values() + if sequence is not None + ), + default=0, + ) + def mark_mature_pool_payouts(self, *, active_tip_height: int) -> dict[str, int | str]: return { "backend": "memory", @@ -1370,6 +1523,7 @@ def __init__( audit_body_dir: str | Path | None = None, audit_bundle_canonicalizer: Callable[[dict[str, Any]], bytes] | None = None, audit_share_segment_size: int = 0, + audit_artifact_store: AuditArtifactStore | None = None, ctv_broadcast_attempt_detail_limit: int = DEFAULT_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT, ctv_broadcast_retry_backoff_seconds: int = DEFAULT_CTV_BROADCAST_RETRY_BACKOFF_SECONDS, ): @@ -1408,12 +1562,27 @@ def __init__( self._lease_retry_min_sleep_seconds = min(0.25, self._lease_retry_max_sleep_seconds) self._lock = Lock() self._read_semaphore = BoundedSemaphore(read_concurrency) - self._audit_body_dir = Path(audit_body_dir) if audit_body_dir else None - self._audit_bundle_canonicalizer = audit_bundle_canonicalizer audit_share_segment_size = int(audit_share_segment_size) if audit_share_segment_size < 0: raise ValueError("audit_share_segment_size must be non-negative") - self._audit_share_segment_size = audit_share_segment_size + if audit_artifact_store is None and audit_body_dir is not None: + body_root = Path(audit_body_dir) + audit_artifact_store = AuditArtifactStore( + AuditArtifactConfig( + root=body_root, + evidence_path=body_root / "prism-live-stratum-evidence.json", + share_segment_size=audit_share_segment_size, + ), + # Legacy direct-ledger construction is an explicit adapter. + # Coordinator production wiring injects the shared A1 store. + canonicalizer=( + audit_bundle_canonicalizer or canonical_bundle_bytes + ), + ) + self._audit_artifact_store = audit_artifact_store + self._audit_bundle_canonicalizer = ( + audit_bundle_canonicalizer or canonical_bundle_bytes + ) ctv_broadcast_attempt_detail_limit = int(ctv_broadcast_attempt_detail_limit) if ctv_broadcast_attempt_detail_limit < 0: raise ValueError("ctv_broadcast_attempt_detail_limit must be non-negative") @@ -4513,549 +4682,95 @@ def dashboard_hashrate_series( for row in rows ] - def _externalize_audit_body( - self, - block_hash: str, - audit_bundle_sha256: str, - final_bundle: dict[str, Any], - ) -> str | None: - """Write the audit-bundle body to the external store and return its path. - - Returns None when no body store is configured, in which case the caller - keeps the body inline in Postgres (legacy behavior). Externalizing the - body is what stops the per-block audit_bundle JSONB from growing with the - full accepted-share history. - """ - if self._audit_body_dir is None: - return None - block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) - audit_bundle_sha256 = canonical_hex( - str(audit_bundle_sha256), - name="audit_bundle_sha256", - expected_bytes=32, - ) - body_bytes = self._canonical_audit_body_bytes_for_sha(final_bundle, audit_bundle_sha256) - return self._write_external_audit_body(block_hash, audit_bundle_sha256, body_bytes) - - def _canonical_audit_body_bytes_for_sha( - self, - final_bundle: dict[str, Any], - audit_bundle_sha256: str, - ) -> bytes: - body_bytes = self._canonical_audit_bundle_bytes(final_bundle) - actual_sha256 = sha256_bytes_hex(body_bytes) - if actual_sha256 != str(audit_bundle_sha256).lower(): - raise RuntimeError( - "audit bundle sha256 mismatch: " - f"expected {str(audit_bundle_sha256).lower()}, got {actual_sha256}" + def _audit_store(self) -> AuditArtifactStore: + store = getattr(self, "_audit_artifact_store", None) + if store is None: + raise RuntimeError("audit body store is not configured") + return store + + def _audit_reader(self, body_uri: object) -> AuditArtifactStore: + del body_uri + store = getattr(self, "_audit_artifact_store", None) + if store is not None: + return store + # Compatibility-only dashboard resolvers historically initialized a + # read-only PsqlShareLedger subclass with just `_audit_body_dir`. Keep + # that explicit adapter working while routing every read through A1. + legacy_root = getattr(self, "_audit_body_dir", None) + if legacy_root is not None: + root = Path(legacy_root) + store = AuditArtifactStore( + AuditArtifactConfig( + root=root, + evidence_path=root / "prism-live-stratum-evidence.json", + ), + canonicalizer=getattr( + self, + "_audit_bundle_canonicalizer", + canonical_bundle_bytes, + ), ) - return body_bytes - - def _external_audit_storage_bytes( - self, - *, - block_hash: str, - audit_bundle_sha256: str, - final_bundle: dict[str, Any], - canonical_body_bytes: bytes, - ) -> bytes: - v2_bundle = self._audit_bundle_v2( - block_hash=block_hash, - audit_bundle_sha256=audit_bundle_sha256, - final_bundle=final_bundle, - ) - if v2_bundle is not None: - return self._storage_json_bytes(v2_bundle) - body_ref = self._audit_body_ref( - block_hash=block_hash, - audit_bundle_sha256=audit_bundle_sha256, - final_bundle=final_bundle, + self._audit_artifact_store = store + return store + raise RuntimeError( + "audit bundle body is not retrievable: audit body store is not configured" ) - if body_ref is None: - return canonical_body_bytes - return self._storage_json_bytes(body_ref) - def _audit_body_ref( - self, - *, - block_hash: str, - audit_bundle_sha256: str, - final_bundle: dict[str, Any], - ) -> dict[str, Any] | None: - if self._audit_body_dir is None or self._audit_share_segment_size <= 0: - return None - shares = final_bundle.get("shares") - if not isinstance(shares, list) or not shares: + def _externalize_audit_body(self, block_hash: str, audit_bundle_sha256: str, final_bundle: dict[str, Any]) -> str | None: + if self._audit_artifact_store is None: return None - share_parts = self._audit_share_parts(shares) - if share_parts is None or not any(part.get("kind") == "segment" for part in share_parts): - return None - shares_key_index = list(final_bundle).index("shares") - # Persistence only reads this immutable build result. A shallow outer - # mapping avoids recursively copying both the top-level shares and the - # reward-manifest share window merely to remove one key. - bundle_without_shares = { - key: value for key, value in final_bundle.items() if key != "shares" - } - return { - "schema": AUDIT_BODY_REF_SCHEMA, - "block_hash": block_hash, - "audit_bundle_sha256": audit_bundle_sha256, - "audit_bundle_schema": str(final_bundle.get("schema") or ""), - "share_count": len(shares), - "share_segment_size": self._audit_share_segment_size, - "shares_key_index": shares_key_index, - "bundle_without_shares": bundle_without_shares, - "share_parts": share_parts, - } + return self._audit_store().externalize_audit_body(block_hash, audit_bundle_sha256, final_bundle) - def _audit_bundle_v2( - self, - *, - block_hash: str, - audit_bundle_sha256: str, - final_bundle: dict[str, Any], - ) -> dict[str, Any] | None: - if self._audit_body_dir is None or self._audit_share_segment_size <= 0: - return None - shares = final_bundle.get("shares") - if not isinstance(shares, list) or not shares: - return None - share_parts = self._audit_share_range_parts(shares) - if share_parts is None: - return None - shares_key_index = list(final_bundle).index("shares") - bundle_without_shares = { - key: value for key, value in final_bundle.items() if key != "shares" - } - reward_manifest = final_bundle.get("reward_manifest") - proof: dict[str, Any] = { - "schema": AUDIT_WINDOW_COMPLETENESS_PROOF_SCHEMA, - "share_segment_size": self._audit_share_segment_size, - "first_share_seq": int(shares[0]["share_seq"]), - "last_share_seq": int(shares[-1]["share_seq"]), - "share_count": len(shares), - "share_parts_digest_hex": sha256_bytes_hex( - self._storage_json_bytes({"share_parts": share_parts}) - ), - "share_parts": share_parts, - } - if isinstance(reward_manifest, dict): - for key in ( - "anchor_job_issued_at_ms", - "anchor_share_seq", - "newest_share_seq", - "oldest_share_seq", - "included_share_count", - "requested_window_weight", - "counted_window_weight", - "share_slice_digest_hex", - ): - if key in reward_manifest: - proof[key] = reward_manifest[key] - return { - "schema": AUDIT_BUNDLE_V2_SCHEMA, - "block_hash": block_hash, - "audit_bundle_sha256": audit_bundle_sha256, - "logical_audit_bundle_schema": str(final_bundle.get("schema") or ""), - "share_count": len(shares), - "shares_key_index": shares_key_index, - "bundle_without_shares": bundle_without_shares, - "share_window_proof": proof, - } + def _canonical_audit_body_bytes_for_sha(self, final_bundle: dict[str, Any], audit_bundle_sha256: str) -> bytes: + return self._audit_store().canonical_audit_body_bytes_for_sha(final_bundle, audit_bundle_sha256) - def _audit_share_parts(self, shares: list[Any]) -> list[dict[str, Any]] | None: - share_seqs: list[int] = [] - for share in shares: - if not isinstance(share, dict): - return None - try: - share_seq = int(share["share_seq"]) - except (KeyError, TypeError, ValueError): - return None - share_seqs.append(share_seq) - if any(current + 1 != nxt for current, nxt in zip(share_seqs, share_seqs[1:])): - return None + def _audit_body_ref(self, **kwargs: Any) -> dict[str, Any] | None: + return self._audit_store().audit_body_ref(**kwargs) - parts: list[dict[str, Any]] = [] - index = 0 - segment_size = self._audit_share_segment_size - while index < len(shares): - first_seq = share_seqs[index] - segment_start = ((first_seq - 1) // segment_size) * segment_size + 1 - segment_end = segment_start + segment_size - 1 - end = index - while end < len(shares) and share_seqs[end] <= segment_end: - end += 1 - chunk = shares[index:end] - chunk_seqs = share_seqs[index:end] - if len(chunk) == segment_size and chunk_seqs[0] == segment_start and chunk_seqs[-1] == segment_end: - segment_uri, segment_sha256 = self._write_audit_share_segment( - first_share_seq=segment_start, - last_share_seq=segment_end, - shares=chunk, - ) - parts.append( - { - "kind": "segment", - "first_share_seq": segment_start, - "last_share_seq": segment_end, - "share_count": len(chunk), - "sha256": segment_sha256, - "body_uri": segment_uri, - } - ) - else: - parts.append( - { - "kind": "inline", - "first_share_seq": chunk_seqs[0], - "last_share_seq": chunk_seqs[-1], - "share_count": len(chunk), - "shares": chunk, - } - ) - index = end - return parts + def _audit_bundle_v2(self, **kwargs: Any) -> dict[str, Any] | None: + return self._audit_store().audit_bundle_v2(**kwargs) + + def _audit_share_parts(self, shares: list[Any]) -> list[dict[str, Any]] | None: + return self._audit_store().audit_share_parts(shares) def _audit_share_range_parts(self, shares: list[Any]) -> list[dict[str, Any]] | None: - share_seqs: list[int] = [] - for share in shares: - if not isinstance(share, dict): - return None - try: - share_seq = int(share["share_seq"]) - except (KeyError, TypeError, ValueError): - return None - share_seqs.append(share_seq) - if any(current + 1 != nxt for current, nxt in zip(share_seqs, share_seqs[1:])): - return None + return self._audit_store().audit_share_range_parts(shares) - parts: list[dict[str, Any]] = [] - index = 0 - segment_size = self._audit_share_segment_size - while index < len(shares): - first_seq = share_seqs[index] - segment_start = ((first_seq - 1) // segment_size) * segment_size + 1 - segment_end = segment_start + segment_size - 1 - end = index - while end < len(shares) and share_seqs[end] <= segment_end: - end += 1 - chunk = shares[index:end] - chunk_seqs = share_seqs[index:end] - segment_uri, range_sha256 = self._write_audit_share_segment_range( - segment_first_share_seq=segment_start, - segment_last_share_seq=segment_end, - first_share_seq=chunk_seqs[0], - last_share_seq=chunk_seqs[-1], - shares=chunk, - ) - parts.append( - { - "kind": "segment_range", - "segment_first_share_seq": segment_start, - "segment_last_share_seq": segment_end, - "first_share_seq": chunk_seqs[0], - "last_share_seq": chunk_seqs[-1], - "share_count": len(chunk), - "range_sha256": range_sha256, - "body_uri": segment_uri, - } - ) - index = end - return parts + def _audit_share_segment_payload(self, **kwargs: Any) -> dict[str, Any]: + return self._audit_store().audit_share_segment_payload(**kwargs) - def _audit_share_segment_payload(self, *, first_share_seq: int, last_share_seq: int, shares: list[Any]) -> dict[str, Any]: - return { - "schema": AUDIT_SHARE_SEGMENT_SCHEMA, - "first_share_seq": first_share_seq, - "last_share_seq": last_share_seq, - "share_count": len(shares), - "shares": shares, - } + def _write_audit_share_segment(self, **kwargs: Any) -> tuple[str, str]: + return self._audit_store().write_audit_share_segment(**kwargs) - def _write_audit_share_segment( - self, - *, - first_share_seq: int, - last_share_seq: int, - shares: list[Any], - ) -> tuple[str, str]: - if self._audit_body_dir is None: - raise RuntimeError("audit body store is not configured") - segment = self._audit_share_segment_payload( - first_share_seq=first_share_seq, - last_share_seq=last_share_seq, - shares=shares, - ) - segment_bytes = self._storage_json_bytes(segment) - segment_sha256 = sha256_bytes_hex(segment_bytes) - segment_path = self._audit_body_dir.resolve() / ( - f"prism-audit-share-segment-{first_share_seq}-{last_share_seq}-{segment_sha256}.json" - ) - if segment_path.exists(): - if not self._file_matches_bytes(segment_path, segment_bytes): - raise RuntimeError(f"existing audit share segment does not match payload at {segment_path}") - else: - self._write_bytes_atomically(segment_path, segment_bytes) - return str(segment_path), segment_sha256 - - def _write_audit_share_segment_range( - self, - *, - segment_first_share_seq: int, - segment_last_share_seq: int, - first_share_seq: int, - last_share_seq: int, - shares: list[Any], - ) -> tuple[str, str]: - if self._audit_body_dir is None: - raise RuntimeError("audit body store is not configured") - if not shares: - raise RuntimeError("audit share segment range cannot be empty") - segment_path = self._audit_body_dir.resolve() / ( - f"prism-audit-share-segment-slot-{segment_first_share_seq}-{segment_last_share_seq}.json" - ) - range_payload = self._audit_share_segment_payload( - first_share_seq=first_share_seq, - last_share_seq=last_share_seq, - shares=shares, - ) - # Encode the incoming range once. Completed 10k slots dominate normal - # block persistence; byte equality lets them return without parsing, - # merging, deep-copying, or serializing the existing slot tree. - range_bytes = self._storage_json_bytes(range_payload) - range_sha256 = sha256_bytes_hex(range_bytes) - if segment_path.exists() and self._file_matches_bytes(segment_path, range_bytes): - return str(segment_path), range_sha256 - - merged_shares = shares - existing_bytes: bytes | None = None - if segment_path.exists(): - try: - existing_bytes = segment_path.read_bytes() - existing = json.loads(existing_bytes) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise RuntimeError(f"existing audit share segment is not valid JSON at {segment_path}") from exc - if not isinstance(existing, dict) or existing.get("schema") != AUDIT_SHARE_SEGMENT_SCHEMA: - raise RuntimeError(f"existing audit share segment has invalid schema at {segment_path}") - existing_shares = existing.get("shares") - if not isinstance(existing_shares, list): - raise RuntimeError(f"existing audit share segment has no shares at {segment_path}") - merged_shares = self._merge_audit_share_ranges( - existing_shares, - shares, - segment_path=segment_path, - ) - segment_first = int(merged_shares[0]["share_seq"]) - segment_last = int(merged_shares[-1]["share_seq"]) - if ( - segment_first == first_share_seq - and segment_last == last_share_seq - and len(merged_shares) == len(shares) - ): - segment_bytes = range_bytes - else: - segment = self._audit_share_segment_payload( - first_share_seq=segment_first, - last_share_seq=segment_last, - shares=merged_shares, - ) - segment_bytes = self._storage_json_bytes(segment) - if existing_bytes != segment_bytes: - self._write_bytes_atomically(segment_path, segment_bytes) - return str(segment_path), range_sha256 + def _write_audit_share_segment_range(self, **kwargs: Any) -> tuple[str, str]: + return self._audit_store().write_audit_share_segment_range(**kwargs) - def _merge_audit_share_ranges( - self, - existing_shares: list[Any], - incoming_shares: list[Any], - *, - segment_path: Path, - ) -> list[Any]: - if not existing_shares: - return list(incoming_shares) - existing_by_seq = self._audit_shares_by_seq(existing_shares, segment_path=segment_path) - incoming_by_seq = self._audit_shares_by_seq(incoming_shares, segment_path=segment_path) - for share_seq, incoming in incoming_by_seq.items(): - existing = existing_by_seq.get(share_seq) - if existing is not None and existing != incoming: - raise RuntimeError(f"existing audit share segment conflicts at share_seq {share_seq} in {segment_path}") - merged_by_seq = {**existing_by_seq, **incoming_by_seq} - ordered_seqs = sorted(merged_by_seq) - if any(current + 1 != nxt for current, nxt in zip(ordered_seqs, ordered_seqs[1:])): - raise RuntimeError(f"existing audit share segment would become non-contiguous at {segment_path}") - return [merged_by_seq[share_seq] for share_seq in ordered_seqs] + def _merge_audit_share_ranges(self, existing_shares: list[Any], incoming_shares: list[Any], *, segment_path: Path) -> list[Any]: + return self._audit_store().merge_audit_share_ranges(existing_shares, incoming_shares, segment_path=segment_path) def _audit_shares_by_seq(self, shares: list[Any], *, segment_path: Path) -> dict[int, Any]: - by_seq: dict[int, Any] = {} - for share in shares: - if not isinstance(share, dict): - raise RuntimeError(f"audit share segment has invalid share payload at {segment_path}") - try: - share_seq = int(share["share_seq"]) - except (KeyError, TypeError, ValueError) as exc: - raise RuntimeError(f"audit share segment has invalid share_seq at {segment_path}") from exc - existing = by_seq.get(share_seq) - if existing is not None and existing != share: - raise RuntimeError(f"audit share segment has duplicate conflicting share_seq {share_seq} at {segment_path}") - by_seq[share_seq] = share - ordered = sorted(by_seq) - if any(current + 1 != nxt for current, nxt in zip(ordered, ordered[1:])): - raise RuntimeError(f"audit share segment has non-contiguous share_seq values at {segment_path}") - return by_seq + return self._audit_store().audit_shares_by_seq(shares, segment_path=segment_path) def _storage_json_bytes(self, payload: dict[str, Any]) -> bytes: - return json.dumps(payload, separators=(",", ":")).encode("utf-8") - - @staticmethod - def _file_matches_bytes(path: Path, expected: bytes) -> bool: - try: - if path.stat().st_size != len(expected): - return False - offset = 0 - view = memoryview(expected) - with path.open("rb") as handle: - while offset < len(expected): - chunk = handle.read(min(1024 * 1024, len(expected) - offset)) - if not chunk or chunk != view[offset : offset + len(chunk)]: - return False - offset += len(chunk) - return not handle.read(1) - except OSError: - return False - - def _write_bytes_atomically(self, path: Path, payload: bytes) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - tmp_path = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") - try: - with tmp_path.open("xb") as handle: - handle.write(payload) - handle.flush() - os.fsync(handle.fileno()) - tmp_path.replace(path) - finally: - try: - tmp_path.unlink() - except FileNotFoundError: - pass - - def _write_json_atomically(self, path: Path, payload: dict[str, Any]) -> None: - """Stream compact JSON through the existing fsync-and-rename boundary.""" - path.parent.mkdir(parents=True, exist_ok=True) - tmp_path = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") - encoder = json.JSONEncoder(separators=(",", ":")) - try: - with tmp_path.open("xb") as handle: - for chunk in encoder.iterencode(payload): - handle.write(chunk.encode("utf-8")) - handle.flush() - os.fsync(handle.fileno()) - tmp_path.replace(path) - finally: - try: - tmp_path.unlink() - except FileNotFoundError: - pass - - def _file_matches_json_payload(self, path: Path, payload: dict[str, Any]) -> bool: - """Compare compact JSON exactly without materializing expected bytes.""" - expected_digest = hashlib.sha256() - expected_length = 0 - encoder = json.JSONEncoder(separators=(",", ":")) - for chunk in encoder.iterencode(payload): - encoded = chunk.encode("utf-8") - expected_digest.update(encoded) - expected_length += len(encoded) - try: - return ( - path.stat().st_size == expected_length - and hmac.compare_digest( - self._file_sha256_hex(path), - expected_digest.hexdigest(), - ) - ) - except OSError: - return False - - def _copy_file_atomically(self, path: Path, source: Path) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - tmp_path = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") - try: - with source.open("rb") as input_handle, tmp_path.open("xb") as output_handle: - while chunk := input_handle.read(1024 * 1024): - output_handle.write(chunk) - output_handle.flush() - os.fsync(output_handle.fileno()) - tmp_path.replace(path) - finally: - try: - tmp_path.unlink() - except FileNotFoundError: - pass + return self._audit_store().storage_json_bytes(payload) @staticmethod def _file_sha256_hex(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - while chunk := handle.read(1024 * 1024): - digest.update(chunk) - return digest.hexdigest() - - def _write_external_audit_body( - self, - block_hash: str, - audit_bundle_sha256: str, - body_bytes: bytes, - ) -> str | None: - if self._audit_body_dir is None: - return None - self._audit_body_dir.mkdir(parents=True, exist_ok=True) - body_path = self._audit_body_path(block_hash, audit_bundle_sha256) - if body_path.exists(): - existing = body_path.read_bytes() - if existing != body_bytes: - raise RuntimeError(f"existing audit bundle body does not match payload at {body_path}") - return str(body_path) - self._write_bytes_atomically(body_path, body_bytes) - return str(body_path) + return AuditArtifactStore.file_sha256_hex(path) def _canonical_audit_bundle_bytes(self, final_bundle: dict[str, Any]) -> bytes: - if self._audit_bundle_canonicalizer is not None: - canonical = self._audit_bundle_canonicalizer(final_bundle) - return canonical.encode() if isinstance(canonical, str) else bytes(canonical) - completed = subprocess.run( - prism_tool_command("qbit-prism-audit-canonicalize") - + [ - "--input", - "-", - ], - input=json.dumps(final_bundle).encode(), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=False, - ) - if completed.returncode != 0: - stderr = completed.stderr.decode(errors="replace").strip() - raise RuntimeError(f"qbit-prism-audit-canonicalize failed: {stderr}") - return completed.stdout + if self._audit_artifact_store is None: + return canonical_audit_bundle_bytes( + final_bundle, + self._audit_bundle_canonicalizer, + ) + return self._audit_store().canonical_audit_bundle_bytes(final_bundle) def _audit_body_path(self, block_hash: str, audit_bundle_sha256: str) -> Path: - if self._audit_body_dir is None: - raise RuntimeError("audit body store is not configured") - root = self._audit_body_dir.resolve() - body_path = root / f"prism-audit-bundle-body-{block_hash}-{audit_bundle_sha256}.json" - return self._resolve_audit_body_path(body_path) + return self._audit_store().body_path(block_hash, audit_bundle_sha256) def _resolve_audit_body_path(self, body_uri: object) -> Path: - body_path = Path(str(body_uri)).expanduser().resolve() - if self._audit_body_dir is not None: - root = self._audit_body_dir.resolve() - try: - body_path.relative_to(root) - except ValueError as exc: - raise RuntimeError(f"audit bundle body path escapes audit body store: {body_uri}") from exc - return body_path + return self._audit_store().resolve_owned_path(body_uri) def _external_audit_body_write_plan(self, payload: dict[str, Any]) -> str | None: """Refresh the writer lease and decide whether this persist may write a body. @@ -5065,7 +4780,7 @@ def _external_audit_body_write_plan(self, payload: dict[str, Any]) -> str | None writers from creating artifacts by requiring the DB lease to be current before any filesystem side effect. """ - if self._audit_body_dir is None: + if self._audit_artifact_store is None: return None sql = f""" WITH payload AS ( @@ -5137,450 +4852,70 @@ def _external_audit_body_write_plan(self, payload: dict[str, Any]) -> str | None return None return str(self._audit_body_path(payload["block_hash"], payload["audit_bundle_sha256"])) - def _audit_body_byte_len( - self, - body_uri: object | None, - final_bundle: dict[str, Any], - canonical_bundle_path: Path | None = None, - ) -> int: - if body_uri: - return self._resolve_audit_body_path(body_uri).stat().st_size - if canonical_bundle_path is not None: - return canonical_bundle_path.stat().st_size - return len(self._canonical_audit_bundle_bytes(final_bundle)) + def _audit_body_byte_len(self, body_uri: object | None, final_bundle: dict[str, Any], canonical_bundle_path: Path | None = None) -> int: + if self._audit_artifact_store is None: + return len(self._canonical_audit_bundle_bytes(final_bundle)) + return self._audit_store().audit_body_byte_len(body_uri, final_bundle, canonical_bundle_path) - def _prepare_external_audit_body( - self, - payload: dict[str, Any], - final_bundle: dict[str, Any], - *, - canonical_bundle_path: Path | None = None, - ) -> str | None: - if self._audit_body_dir is None: + def _prepare_external_audit_body(self, payload: dict[str, Any], final_bundle: dict[str, Any], *, canonical_bundle_path: Path | None = None) -> str | None: + if self._audit_artifact_store is None: return None - payload = { + normalized = { **payload, "block_hash": canonical_hex(str(payload["block_hash"]), name="block_hash", expected_bytes=32), - "audit_bundle_sha256": canonical_hex( - str(payload["audit_bundle_sha256"]), - name="audit_bundle_sha256", - expected_bytes=32, - ), + "audit_bundle_sha256": canonical_hex(str(payload["audit_bundle_sha256"]), name="audit_bundle_sha256", expected_bytes=32), } - expected_sha256 = str(payload["audit_bundle_sha256"]) - body_bytes: bytes | None = None if canonical_bundle_path is not None: - canonical_bundle_path = canonical_bundle_path.resolve() - try: - actual_sha256 = self._file_sha256_hex(canonical_bundle_path) - except OSError as exc: - raise RuntimeError( - f"canonical audit bundle is not retrievable at {canonical_bundle_path}: {exc}" - ) from exc - if actual_sha256 != expected_sha256: - raise RuntimeError( - "audit bundle sha256 mismatch: " - f"expected {expected_sha256}, got {actual_sha256}" - ) - else: - body_bytes = self._canonical_audit_body_bytes_for_sha( + self._audit_store().validate_canonical_source( + canonical_bundle_path, + str(normalized["audit_bundle_sha256"]), final_bundle, - expected_sha256, ) - body_uri = self._external_audit_body_write_plan(payload) - if body_uri is None: - return None - body_path = self._resolve_audit_body_path(body_uri) - storage_payload = self._audit_bundle_v2( - block_hash=str(payload["block_hash"]), - audit_bundle_sha256=expected_sha256, - final_bundle=final_bundle, - ) - if storage_payload is None: - storage_payload = self._audit_body_ref( - block_hash=str(payload["block_hash"]), - audit_bundle_sha256=expected_sha256, - final_bundle=final_bundle, - ) - if body_path.exists(): - if storage_payload is not None and self._file_matches_json_payload( - body_path, storage_payload - ): - return str(body_path) - if ( - storage_payload is None - and canonical_bundle_path is not None - and self._file_sha256_hex(body_path) == expected_sha256 - ): - return str(body_path) - # Preserve compatibility with bodies written by an older storage - # layout. This expensive reconstruction is only the mismatch path; - # same-version crash retries take the bounded exact-match path. - if not self._external_body_matches_sha(body_path, expected_sha256): - raise RuntimeError(f"existing audit bundle body does not match payload at {body_path}") - return str(body_path) - canonical_body_path = self._audit_body_path( - str(payload["block_hash"]), - str(payload["audit_bundle_sha256"]), + body_uri = self._external_audit_body_write_plan(normalized) + return self._audit_store().prepare_external_audit_body( + normalized, + final_bundle, + body_uri=body_uri, + canonical_bundle_path=canonical_bundle_path, ) - if body_path != canonical_body_path: - raise RuntimeError( - "existing audit bundle body pointer does not match canonical external path: " - f"{body_uri}" - ) - if storage_payload is not None: - self._write_json_atomically(body_path, storage_payload) - elif canonical_bundle_path is not None: - self._copy_file_atomically(body_path, canonical_bundle_path) - else: - assert body_bytes is not None - self._write_bytes_atomically(body_path, body_bytes) - return str(body_path) - def _read_external_body( - self, - body_uri: object, - *, - expected_sha256: object | None = None, - ) -> dict[str, object] | None: - if not body_uri: - return None - try: - body_path = self._resolve_audit_body_path(body_uri) - body_bytes = body_path.read_bytes() - except OSError as exc: - raise RuntimeError( - f"audit bundle body is not retrievable at {body_uri}: {exc}" - ) from exc - try: - body = json.loads(body_bytes.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: {exc}") from exc - if isinstance(body, dict) and body.get("schema") == AUDIT_BODY_REF_SCHEMA: - return self._resolve_audit_body_ref(body, expected_sha256=expected_sha256, body_uri=body_uri) - if isinstance(body, dict) and body.get("schema") == AUDIT_BUNDLE_V2_SCHEMA: - return self._resolve_audit_bundle_v2(body, expected_sha256=expected_sha256, body_uri=body_uri) - if expected_sha256: - expected = str(expected_sha256).lower() - actual = sha256_bytes_hex(body_bytes) - if actual != expected: - raise RuntimeError( - f"audit bundle body hash mismatch at {body_uri}: expected {expected}, got {actual}" - ) - return body + def _read_external_body(self, body_uri: object, *, expected_sha256: object | None = None) -> dict[str, object] | None: + return self._audit_reader(body_uri).read_external_body(body_uri, expected_sha256=expected_sha256) def _external_body_matches_sha(self, body_path: Path, expected_sha256: str) -> bool: - try: - self._read_external_body(str(body_path), expected_sha256=expected_sha256) - except RuntimeError: - return False - return True + return self._audit_reader(body_path).external_body_matches_sha(body_path, expected_sha256) def _external_body_available_for_sha(self, body_uri: object, expected_sha256: str) -> bool: try: - body_path = self._resolve_audit_body_path(body_uri) - body_bytes = body_path.read_bytes() - except (OSError, RuntimeError): + reader = self._audit_reader(body_uri) + except RuntimeError: return False - try: - body = json.loads(body_bytes.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError): - body = None - if isinstance(body, dict) and body.get("schema") == AUDIT_BODY_REF_SCHEMA: - if str(body.get("audit_bundle_sha256") or "").lower() != expected_sha256: - return False - bundle_without_shares = body.get("bundle_without_shares") - share_parts = body.get("share_parts") - if not isinstance(bundle_without_shares, dict) or not isinstance(share_parts, list): - return False - expected_share_count = int(body.get("share_count") or 0) - actual_share_count = 0 - for part in share_parts: - if not isinstance(part, dict): - return False - kind = part.get("kind") - if kind == "segment": - if not self._audit_share_segment_available(part, parent_body_uri=body_uri): - return False - elif kind in {"segment_range", "segment_prefix"}: - if not self._audit_share_segment_available(part, parent_body_uri=body_uri): - return False - elif kind == "inline": - inline_shares = part.get("shares") - if not isinstance(inline_shares, list) or len(inline_shares) != int(part.get("share_count") or 0): - return False - else: - return False - actual_share_count += int(part.get("share_count") or 0) - return actual_share_count == expected_share_count - if isinstance(body, dict) and body.get("schema") == AUDIT_BUNDLE_V2_SCHEMA: - try: - self._resolve_audit_bundle_v2(body, expected_sha256=expected_sha256, body_uri=body_uri) - except RuntimeError: - return False - return True - return sha256_bytes_hex(body_bytes) == expected_sha256 + return reader.external_body_available_for_sha(body_uri, expected_sha256) def _audit_share_segment_available(self, part: dict[str, Any], *, parent_body_uri: object) -> bool: try: - self._read_audit_share_segment(part, parent_body_uri=parent_body_uri) - except RuntimeError: + self._audit_store().read_audit_share_segment(part, parent_body_uri=parent_body_uri) + except (OSError, RuntimeError, TypeError, ValueError): return False return True - def _resolve_audit_body_ref( - self, - body_ref: dict[str, Any], - *, - expected_sha256: object | None, - body_uri: object, - ) -> dict[str, object]: - expected = str(expected_sha256).lower() if expected_sha256 else None - declared_sha256 = str(body_ref.get("audit_bundle_sha256") or "").lower() - if expected and declared_sha256 != expected: - raise RuntimeError( - f"audit bundle body hash mismatch at {body_uri}: expected {expected}, got {declared_sha256}" - ) - bundle_without_shares = body_ref.get("bundle_without_shares") - if not isinstance(bundle_without_shares, dict): - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: missing bundle_without_shares") - share_parts = body_ref.get("share_parts") - if not isinstance(share_parts, list): - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: missing share_parts") - shares: list[Any] = [] - for part in share_parts: - if not isinstance(part, dict): - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: invalid share part") - kind = part.get("kind") - if kind == "segment": - shares.extend(self._read_audit_share_segment(part, parent_body_uri=body_uri)) - elif kind in {"segment_range", "segment_prefix"}: - shares.extend(self._read_audit_share_segment(part, parent_body_uri=body_uri)) - elif kind == "inline": - inline_shares = part.get("shares") - if not isinstance(inline_shares, list): - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: invalid inline shares") - if len(inline_shares) != int(part.get("share_count") or 0): - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: inline share count mismatch") - # The body was freshly parsed for this request; transfer its - # share objects into the reconstructed bundle without cloning. - shares.extend(inline_shares) - else: - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: invalid share part kind") - expected_share_count = int(body_ref.get("share_count") or 0) - if len(shares) != expected_share_count: - raise RuntimeError( - f"audit bundle body is not valid JSON at {body_uri}: expected " - f"{expected_share_count} shares, reconstructed {len(shares)}" - ) - shares_key_index_raw = body_ref.get("shares_key_index") - shares_key_index = len(bundle_without_shares) if shares_key_index_raw is None else int(shares_key_index_raw) - bundle: dict[str, object] = {} - shares_inserted = False - for index, (key, value) in enumerate(bundle_without_shares.items()): - if index == shares_key_index: - bundle["shares"] = shares - shares_inserted = True - bundle[str(key)] = value - if not shares_inserted: - bundle["shares"] = shares - if expected: - actual = sha256_bytes_hex(self._canonical_audit_bundle_bytes(bundle)) - if actual != expected: - raise RuntimeError( - f"audit bundle body hash mismatch at {body_uri}: expected {expected}, got {actual}" - ) - return bundle + def _resolve_audit_body_ref(self, body_ref: dict[str, Any], *, expected_sha256: object | None, body_uri: object) -> dict[str, object]: + return self._audit_store().resolve_audit_body_ref(body_ref, expected_sha256=expected_sha256, body_uri=body_uri) - def _resolve_audit_bundle_v2( - self, - body: dict[str, Any], - *, - expected_sha256: object | None, - body_uri: object, - ) -> dict[str, object]: - expected = str(expected_sha256).lower() if expected_sha256 else None - declared_sha256 = str(body.get("audit_bundle_sha256") or "").lower() - if expected and declared_sha256 != expected: - raise RuntimeError( - f"audit bundle body hash mismatch at {body_uri}: expected {expected}, got {declared_sha256}" - ) - bundle_without_shares = body.get("bundle_without_shares") - if not isinstance(bundle_without_shares, dict): - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: missing bundle_without_shares") - proof = body.get("share_window_proof") - if not isinstance(proof, dict) or proof.get("schema") != AUDIT_WINDOW_COMPLETENESS_PROOF_SCHEMA: - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: missing share_window_proof") - share_parts = proof.get("share_parts") - if not isinstance(share_parts, list): - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: missing share_parts") - expected_parts_digest = str(proof.get("share_parts_digest_hex") or "").lower() - if expected_parts_digest: - actual_parts_digest = sha256_bytes_hex(self._storage_json_bytes({"share_parts": share_parts})) - if actual_parts_digest != expected_parts_digest: - raise RuntimeError( - f"audit bundle body is not valid JSON at {body_uri}: share_parts_digest_hex mismatch" - ) - shares: list[Any] = [] - for part in share_parts: - if not isinstance(part, dict): - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: invalid share part") - shares.extend(self._read_audit_share_segment(part, parent_body_uri=body_uri)) - expected_share_count = int(body.get("share_count") or proof.get("share_count") or 0) - if len(shares) != expected_share_count: - raise RuntimeError( - f"audit bundle body is not valid JSON at {body_uri}: expected " - f"{expected_share_count} shares, reconstructed {len(shares)}" - ) - if shares: - first_share_seq = int(shares[0].get("share_seq")) if isinstance(shares[0], dict) else None - last_share_seq = int(shares[-1].get("share_seq")) if isinstance(shares[-1], dict) else None - if int(proof.get("first_share_seq") or 0) != first_share_seq: - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: proof first_share_seq mismatch") - if int(proof.get("last_share_seq") or 0) != last_share_seq: - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: proof last_share_seq mismatch") - reward_manifest = bundle_without_shares.get("reward_manifest") - proof_share_digest = str(proof.get("share_slice_digest_hex") or "") - if proof_share_digest and isinstance(reward_manifest, dict): - reward_share_digest = str(reward_manifest.get("share_slice_digest_hex") or "") - if not proof_share_digest.lower() == reward_share_digest.lower(): - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: proof share digest mismatch") - shares_key_index_raw = body.get("shares_key_index") - shares_key_index = len(bundle_without_shares) if shares_key_index_raw is None else int(shares_key_index_raw) - bundle: dict[str, object] = {} - shares_inserted = False - for index, (key, value) in enumerate(bundle_without_shares.items()): - if index == shares_key_index: - bundle["shares"] = shares - shares_inserted = True - bundle[str(key)] = value - if not shares_inserted: - bundle["shares"] = shares - actual = sha256_bytes_hex(self._canonical_audit_bundle_bytes(bundle)) - if declared_sha256 and actual != declared_sha256: - raise RuntimeError( - f"audit bundle body hash mismatch at {body_uri}: expected {declared_sha256}, got {actual}" - ) - return bundle + def _resolve_audit_bundle_v2(self, body: dict[str, Any], *, expected_sha256: object | None, body_uri: object) -> dict[str, object]: + return self._audit_store().resolve_audit_bundle_v2(body, expected_sha256=expected_sha256, body_uri=body_uri) def _read_audit_share_segment(self, part: dict[str, Any], *, parent_body_uri: object) -> list[Any]: - body_uri = part.get("body_uri") - kind = str(part.get("kind") or "") - try: - body_path = self._resolve_audit_body_path(body_uri) - segment_bytes = body_path.read_bytes() - except OSError as exc: - raise RuntimeError( - f"audit bundle body is not retrievable at {parent_body_uri}: share segment {body_uri}: {exc}" - ) from exc - expected_sha256 = str(part.get("sha256") or "").lower() - if kind == "segment" and sha256_bytes_hex(segment_bytes) != expected_sha256: - raise RuntimeError( - f"audit bundle body hash mismatch at {parent_body_uri}: " - f"share segment {body_uri} expected {expected_sha256}, got {sha256_bytes_hex(segment_bytes)}" - ) - try: - segment = json.loads(segment_bytes.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: share segment {body_uri}: {exc}" - ) from exc - if not isinstance(segment, dict) or segment.get("schema") != AUDIT_SHARE_SEGMENT_SCHEMA: - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: invalid share segment {body_uri}" - ) - shares = segment.get("shares") - if not isinstance(shares, list): - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: share segment {body_uri} has no shares" - ) - expected_count = int(part.get("share_count") or 0) - first_share_seq = int(part.get("first_share_seq") or 0) - last_share_seq = int(part.get("last_share_seq") or 0) - selected_shares = self._select_audit_share_segment_range( + return self._audit_store().read_audit_share_segment(part, parent_body_uri=parent_body_uri) + + def _select_audit_share_segment_range(self, shares: list[Any], *, first_share_seq: int, last_share_seq: int, parent_body_uri: object, body_uri: object) -> list[Any]: + return self._audit_store().select_audit_share_segment_range( shares, first_share_seq=first_share_seq, last_share_seq=last_share_seq, parent_body_uri=parent_body_uri, body_uri=body_uri, ) - if len(selected_shares) != expected_count: - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: share segment {body_uri} " - f"expected {expected_count} shares, found {len(selected_shares)}" - ) - if kind == "segment_range": - expected_range_sha256 = str(part.get("range_sha256") or "").lower() - actual_range_sha256 = sha256_bytes_hex( - self._storage_json_bytes( - self._audit_share_segment_payload( - first_share_seq=first_share_seq, - last_share_seq=last_share_seq, - shares=selected_shares, - ) - ) - ) - if actual_range_sha256 != expected_range_sha256: - raise RuntimeError( - f"audit bundle body hash mismatch at {parent_body_uri}: " - f"share segment range {body_uri} expected {expected_range_sha256}, got {actual_range_sha256}" - ) - elif kind == "segment_prefix": - expected_prefix_sha256 = str(part.get("prefix_sha256") or "").lower() - actual_prefix_sha256 = sha256_bytes_hex( - self._storage_json_bytes( - self._audit_share_segment_payload( - first_share_seq=first_share_seq, - last_share_seq=last_share_seq, - shares=selected_shares, - ) - ) - ) - if actual_prefix_sha256 != expected_prefix_sha256: - raise RuntimeError( - f"audit bundle body hash mismatch at {parent_body_uri}: " - f"share segment prefix {body_uri} expected {expected_prefix_sha256}, got {actual_prefix_sha256}" - ) - elif kind != "segment": - raise RuntimeError(f"audit bundle body is not valid JSON at {parent_body_uri}: invalid share part kind") - # selected_shares references a freshly parsed, request-local segment. - return selected_shares - - def _select_audit_share_segment_range( - self, - shares: list[Any], - *, - first_share_seq: int, - last_share_seq: int, - parent_body_uri: object, - body_uri: object, - ) -> list[Any]: - selected: list[Any] = [] - previous_seq: int | None = None - for share in shares: - if not isinstance(share, dict): - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: share segment {body_uri} has invalid share" - ) - share_seq = int(share.get("share_seq") or 0) - if previous_seq is not None and previous_seq + 1 != share_seq: - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: share segment {body_uri} is not contiguous" - ) - previous_seq = share_seq - if first_share_seq <= share_seq <= last_share_seq: - selected.append(share) - if selected: - if int(selected[0].get("share_seq") or 0) != first_share_seq: - selected = [] - elif int(selected[-1].get("share_seq") or 0) != last_share_seq: - selected = [] - if not selected and first_share_seq <= last_share_seq: - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: " - f"share segment {body_uri} does not contain requested range" - ) - return selected def _resolve_audit_bundle_row(self, row: object) -> dict[str, object] | None: """Return an audit-bundle row with its body resolved inline. @@ -6023,13 +5358,43 @@ def confirm_accepted_block(self, *, block_hash: str, active_tip_height: int) -> ) ); """ - result = self._run_fenced_json(sql) - if "error" in result: - raise RuntimeError(str(result["error"])) - return { + with self._lock: + result = self._run_json(sql) + if "error" in result: + raise RuntimeError(str(result["error"])) + confirmed_count = int(result["confirmed_count"]) + publication_sequence: object | None = None + if confirmed_count == 1: + # A data-modifying PL/pgSQL function runs under the statement's + # command snapshot, so a join in that same statement cannot see + # the freshly assigned ordinal. Read it in the next statement + # while retaining the ledger writer lock. + state = self._run_retry_safe_read_json( + f""" +SELECT json_build_object( + 'audit_publication_sequence', ( + SELECT audit_publication_sequence + FROM qbit_pool_blocks + WHERE block_hash = {self._text_literal(block_hash)} + AND block_height = {int(active_tip_height)} + AND chain_state = 'confirmed' + AND maturity_state <> 'reversed' + ) +); +""" + ) + publication_sequence = state.get("audit_publication_sequence") + if publication_sequence is None: + raise RuntimeError( + "confirmed pool block has no audit publication sequence" + ) + response: dict[str, int | str] = { "backend": str(result["backend"]), - "confirmed_count": int(result["confirmed_count"]), + "confirmed_count": confirmed_count, } + if publication_sequence is not None: + response["audit_publication_sequence"] = int(publication_sequence) + return response def pool_block_state(self, *, block_hash: str) -> dict[str, object] | None: block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) @@ -6041,7 +5406,8 @@ def pool_block_state(self, *, block_hash: str) -> dict[str, object] | None: 'block_height', block_height, 'parent_hash', parent_hash, 'chain_state', chain_state, - 'maturity_state', maturity_state + 'maturity_state', maturity_state, + 'audit_publication_sequence', audit_publication_sequence ) FROM qbit_pool_blocks WHERE block_hash = {self._text_literal(block_hash)} @@ -6060,6 +5426,27 @@ def pool_block_state(self, *, block_hash: str) -> dict[str, object] | None: state["block_height"] = int(state["block_height"]) return state + def audit_publication_sequence_floor(self) -> int: + """Return MAX durable pool-block ordinal, excluding sequence gaps.""" + + sql = """ +SELECT json_build_object( + 'audit_publication_sequence_floor', + COALESCE(MAX(audit_publication_sequence), 0) +) +FROM qbit_pool_blocks; +""" + with self._lock: + result = self._run_retry_safe_read_json(sql) + if not isinstance(result, dict): + raise RuntimeError( + "audit publication sequence floor query returned non-object JSON" + ) + value = result.get("audit_publication_sequence_floor") + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise RuntimeError("audit publication sequence floor is invalid") + return value + def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: sql = """ SELECT COALESCE(json_agg(json_build_object( @@ -6116,13 +5503,39 @@ def reactivate_pool_block(self, *, block_hash: str, active_tip_height: int) -> d ) ); """ - result = self._run_fenced_json(sql) - if "error" in result: - raise RuntimeError(str(result["error"])) - return { + with self._lock: + result = self._run_json(sql) + if "error" in result: + raise RuntimeError(str(result["error"])) + reactivated_count = int(result["reactivated_count"]) + publication_sequence: object | None = None + if reactivated_count == 1: + state = self._run_retry_safe_read_json( + f""" +SELECT json_build_object( + 'audit_publication_sequence', ( + SELECT audit_publication_sequence + FROM qbit_pool_blocks + WHERE block_hash = {self._text_literal(block_hash)} + AND block_height <= {int(active_tip_height)} + AND chain_state = 'confirmed' + AND maturity_state = 'immature' + ) +); +""" + ) + publication_sequence = state.get("audit_publication_sequence") + if publication_sequence is None: + raise RuntimeError( + "reactivated pool block has no audit publication sequence" + ) + response: dict[str, int | str] = { "backend": str(result["backend"]), - "reactivated_count": int(result["reactivated_count"]), + "reactivated_count": reactivated_count, } + if publication_sequence is not None: + response["audit_publication_sequence"] = int(publication_sequence) + return response def mark_mature_pool_payouts(self, *, active_tip_height: int) -> dict[str, int | str]: sql = f""" diff --git a/test/test-prism-postgres-ledger.sh b/test/test-prism-postgres-ledger.sh index eb8990c..b0d9088 100644 --- a/test/test-prism-postgres-ledger.sh +++ b/test/test-prism-postgres-ledger.sh @@ -48,7 +48,12 @@ else "${POSTGRES_IMAGE}" >/dev/null deadline=$((SECONDS + 60)) - until docker exec "${POSTGRES_CONTAINER}" pg_isready -U qbit -d qbit >/dev/null 2>&1; do + # A fresh official image briefly accepts connections on its bootstrap server + # before restarting into the durable server. Require readiness across that + # handoff so schema initialization cannot race the bootstrap shutdown. + until docker exec "${POSTGRES_CONTAINER}" pg_isready -U qbit -d qbit >/dev/null 2>&1 \ + && sleep 1 \ + && docker exec "${POSTGRES_CONTAINER}" pg_isready -U qbit -d qbit >/dev/null 2>&1; do if [[ "${SECONDS}" -ge "${deadline}" ]]; then echo "timed out waiting for PRISM Postgres container" >&2 docker logs "${POSTGRES_CONTAINER}" >&2 || true @@ -59,6 +64,14 @@ else PSQL_COMMAND="docker exec -i ${POSTGRES_CONTAINER} psql -U qbit -d qbit" fi +if [[ -n "${EXTERNAL_PSQL}" ]]; then + GATE_IMAGE="external-postgres" + GATE_IMAGE_DIGEST="not-applicable" +else + GATE_IMAGE="$(docker inspect --format '{{.Config.Image}}' "${POSTGRES_CONTAINER}")" + GATE_IMAGE_DIGEST="$(docker inspect --format '{{.Image}}' "${POSTGRES_CONTAINER}")" +fi + ( cd "${ROOT_DIR}" PRISM_PSQL_COMMAND="${PSQL_COMMAND}" \ @@ -72,7 +85,12 @@ import os import tempfile from pathlib import Path -from lab.prism.share_ledger import PendingShare, PsqlShareLedger, ShareReplayConflict +from lab.prism.share_ledger import ( + PendingShare, + PsqlShareLedger, + ShareReplayConflict, + SingleWriterShareLedger, +) def pending( @@ -732,8 +750,39 @@ WHERE miner_id LIKE 'miner-unanchored-%'; """ ) assert_equal(unanchored_balances, [], "unanchored carry rows require a confirmed pool block") +unknown_confirmation = replacement.confirm_accepted_block( + block_hash="ff" * 32, + active_tip_height=7, +) +assert_equal( + unknown_confirmation, + {"backend": "postgres-psql", "confirmed_count": 0}, + "unknown block confirmation exposes no publication ordinal", +) +wrong_height_confirmation = replacement.confirm_accepted_block( + block_hash="44" * 32, + active_tip_height=8, +) +assert_equal( + wrong_height_confirmation, + {"backend": "postgres-psql", "confirmed_count": 0}, + "wrong-height confirmation exposes no publication ordinal", +) confirmed = replacement.confirm_accepted_block(block_hash="44" * 32, active_tip_height=7) assert_equal(confirmed["confirmed_count"], 1, "confirmed block count") +first_publication_sequence = int(confirmed["audit_publication_sequence"]) +if first_publication_sequence <= 0: + raise SystemExit("first confirmed block received a non-positive publication ordinal") +confirmed_replay = replacement.confirm_accepted_block( + block_hash="44" * 32, + active_tip_height=7, +) +assert_equal(confirmed_replay["confirmed_count"], 1, "exact confirmation replay count") +assert_equal( + int(confirmed_replay["audit_publication_sequence"]), + first_publication_sequence, + "exact confirmation replay preserves publication ordinal", +) assert_equal( replacement.dashboard_miner_pending_maturity_bits(recipient_id="miner-b"), 49500, @@ -757,14 +806,16 @@ INSERT INTO qbit_pool_blocks ( parent_hash, coinbase_txid, payout_manifest_sha256, - chain_state + chain_state, + audit_publication_sequence ) VALUES ( '""" + alias_block_hash + """', 72, '""" + "44" * 32 + """', '""" + "46" * 32 + """', '""" + "47" * 32 + """', - 'confirmed' + 'confirmed', + nextval('qbit_audit_publication_sequence_seq') ); INSERT INTO qbit_payout_carry_forward ( @@ -865,7 +916,13 @@ replacement.persist_accepted_block( audit_report=zero_net_report, ) force_expired_idle_lease(replacement) -replacement.confirm_accepted_block(block_hash="45" * 32, active_tip_height=8) +second_confirmation = replacement.confirm_accepted_block( + block_hash="45" * 32, + active_tip_height=8, +) +second_publication_sequence = int(second_confirmation["audit_publication_sequence"]) +if second_publication_sequence == first_publication_sequence: + raise SystemExit("distinct confirmed blocks shared a publication ordinal") zero_net_balances = replacement._run_json( """ SELECT COALESCE(json_agg(json_build_object( @@ -915,9 +972,31 @@ else: inactive_count = replacement.mark_pool_block_inactive(block_hash="44" * 32, active_tip_height=7)["inactive_count"] assert_equal(inactive_count, 1, "inactive block quarantine count") +assert_equal( + replacement.mark_pool_block_inactive( + block_hash="44" * 32, + active_tip_height=7, + )["inactive_count"], + 0, + "repeated inactive transition is count-zero", +) +assert_equal( + replacement.confirm_accepted_block( + block_hash="44" * 32, + active_tip_height=7, + ), + {"backend": "postgres-psql", "confirmed_count": 0}, + "inactive block confirmation exposes no publication ordinal", +) assert_equal(replacement.current_owed_balances(), [], "inactive owed balances are excluded") -reactivated_count = replacement.reactivate_pool_block(block_hash="44" * 32, active_tip_height=7)["reactivated_count"] -assert_equal(reactivated_count, 1, "inactive block reactivation count") +reactivated = replacement.reactivate_pool_block(block_hash="44" * 32, active_tip_height=7) +assert_equal(reactivated["reactivated_count"], 1, "inactive block reactivation count") +reactivated_publication_sequence = int(reactivated["audit_publication_sequence"]) +assert_equal( + reactivated_publication_sequence, + first_publication_sequence, + "reactivated block preserves its published ordinal", +) if not replacement.current_owed_balances(): raise SystemExit("reactivated block did not restore owed balances") inactive_count = replacement.mark_pool_block_inactive(block_hash="44" * 32, active_tip_height=7)["inactive_count"] @@ -1356,6 +1435,19 @@ WHERE block_hash = '""" + external_block_hash + """'; "externalized conflicting duplicate does not write an orphan body file", ) +external_successor.release_writer_lease() + print("prism postgres ledger PASS shares=14 lease=replay startup-retry persist-fence sql-window maturity=reorg carry-replay integrity") PY + + PRISM_PSQL_COMMAND="${PSQL_COMMAND}" \ + python3 -m tests.prism_postgres_a1_gate + + PRISM_PSQL_COMMAND="${PSQL_COMMAND}" \ + python3 -m tests.prism_postgres_a1_migration_gate + + PRISM_PSQL_COMMAND="${PSQL_COMMAND}" \ + QBIT_PRISM_GATE_IMAGE="${GATE_IMAGE}" \ + QBIT_PRISM_GATE_IMAGE_DIGEST="${GATE_IMAGE_DIGEST}" \ + python3 -m tests.prism_postgres_a1_process_gate ) diff --git a/tests/prism_postgres_a1_gate.py b/tests/prism_postgres_a1_gate.py new file mode 100644 index 0000000..d96918f --- /dev/null +++ b/tests/prism_postgres_a1_gate.py @@ -0,0 +1,1413 @@ +"""Non-discovered PostgreSQL/A1 integration gate. + +This helper is invoked by ``test/test-prism-postgres-ledger.sh``. It lives +outside unittest discovery because every scenario requires an explicitly +provisioned PostgreSQL target. +""" + +from __future__ import annotations + +import atexit +import hashlib +import json +import os +import re +import shlex +import signal +import subprocess +import tempfile +import threading +from typing import Any + +from lab.prism.share_ledger import PsqlShareLedger, SingleWriterShareLedger + + +PSQL_TIMEOUT_SECONDS = 30.0 +PSQL_OUTPUT_LIMIT_BYTES = 1 << 20 +SCHEMA_PATTERN = re.compile(r"qbit_a1_[a-z0-9_]+") +BASE_PSQL_COMMAND = os.environ.get("PRISM_PSQL_COMMAND", "") +if not BASE_PSQL_COMMAND: + raise SystemExit("PRISM_PSQL_COMMAND is required") +BASE_PSQL_ARGV = shlex.split(BASE_PSQL_COMMAND) +if not BASE_PSQL_ARGV: + raise SystemExit("PRISM_PSQL_COMMAND is empty") + +RUN_TOKEN = os.urandom(16).hex() +RUN_MARKER = f"qbit-a1-test:{RUN_TOKEN}" +OWNED_SCHEMAS: list[tuple[str, str]] = [] +ACTIVE_CHILDREN: set[subprocess.Popen[str]] = set() +ACTIVE_CHILDREN_LOCK = threading.Lock() + + +def fake_bundle_bytes(payload: dict[str, object]) -> bytes: + return json.dumps(payload, separators=(",", ":")).encode() + + +PERSIST_BUNDLE: dict[str, object] = { + "signed_coinbase_manifest": {"manifest": {"payout_count": 1}}, + "payout_policy_manifest": { + "accounts": [ + { + "recipient_id": "a1-parity-miner", + "order_key": "a1-parity-order", + "p2mr_program_hex": "42" * 32, + "gross_amount_sats": 1000, + "prior_balance_sats": 0, + "candidate_balance_sats": 1000, + "onchain_amount_sats": 0, + "carry_forward_balance_sats": 1000, + "action": "accrued", + } + ] + }, +} +PERSIST_BUNDLE_SHA256 = hashlib.sha256( + fake_bundle_bytes(PERSIST_BUNDLE) +).hexdigest() +PERSIST_REPORT: dict[str, object] = { + "coinbase_txid": "20" * 32, + "coinbase_manifest_sha256_hex": "30" * 32, + "audit_bundle_sha256_hex": PERSIST_BUNDLE_SHA256, + "coinbase_tx_hex": "00", +} + + +class GateFailure(RuntimeError): + pass + + +def assert_equal(actual: object, expected: object, message: str) -> None: + if actual != expected: + raise GateFailure(f"{message}: expected {expected!r}, got {actual!r}") + + +def _terminate_and_reap(process: subprocess.Popen[str]) -> None: + if process.poll() is not None: + process.wait() + return + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + process.wait(timeout=2.0) + return + except subprocess.TimeoutExpired: + pass + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=2.0) + + +def cleanup_active_children() -> None: + with ACTIVE_CHILDREN_LOCK: + children = list(ACTIVE_CHILDREN) + for process in children: + _terminate_and_reap(process) + for stream in (process.stdin, process.stdout, process.stderr): + if stream is not None and not stream.closed: + stream.close() + with ACTIVE_CHILDREN_LOCK: + ACTIVE_CHILDREN.discard(process) + + +def run_psql(sql: str, *, schema: str | None = None) -> str: + if schema is not None and SCHEMA_PATTERN.fullmatch(schema) is None: + raise GateFailure(f"invalid scoped schema: {schema!r}") + scoped_sql = sql + if schema is not None: + scoped_sql = ( + "SET statement_timeout = '20s';\n" + "SET lock_timeout = '10s';\n" + f'SET search_path TO "{schema}", pg_catalog;\n' + + sql + ) + command = [ + *BASE_PSQL_ARGV, + "--no-psqlrc", + "--set", + "ON_ERROR_STOP=1", + "--tuples-only", + "--no-align", + "--quiet", + ] + with ( + tempfile.TemporaryFile(mode="w+t", encoding="utf-8") as stdout_file, + tempfile.TemporaryFile(mode="w+t", encoding="utf-8") as stderr_file, + ): + process = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=stdout_file, + stderr=stderr_file, + text=True, + start_new_session=True, + ) + with ACTIVE_CHILDREN_LOCK: + ACTIVE_CHILDREN.add(process) + try: + process.communicate( + scoped_sql, + timeout=PSQL_TIMEOUT_SECONDS, + ) + except BaseException: + _terminate_and_reap(process) + raise + finally: + if process.poll() is None: + _terminate_and_reap(process) + if process.stdin is not None and not process.stdin.closed: + process.stdin.close() + with ACTIVE_CHILDREN_LOCK: + ACTIVE_CHILDREN.discard(process) + stdout_file.seek(0) + stderr_file.seek(0) + stdout = stdout_file.read(PSQL_OUTPUT_LIMIT_BYTES + 1) + stderr = stderr_file.read(PSQL_OUTPUT_LIMIT_BYTES + 1) + if ( + len(stdout.encode("utf-8")) > PSQL_OUTPUT_LIMIT_BYTES + or len(stderr.encode("utf-8")) > PSQL_OUTPUT_LIMIT_BYTES + ): + raise GateFailure("psql output exceeded 1 MiB limit") + if process.returncode != 0: + raise GateFailure( + f"psql failed with exit {process.returncode}: {stderr.strip()}" + ) + return stdout + + +def run_json(sql: str, *, schema: str | None = None) -> Any: + output = run_psql(sql, schema=schema).strip() + if not output: + raise GateFailure("psql query returned no JSON") + return json.loads(output.splitlines()[-1]) + + +def _schema_registered(schema: str) -> bool: + return any(name == schema for name, _marker in OWNED_SCHEMAS) + + +def create_owned_schema(label: str) -> str: + if re.fullmatch(r"[a-z0-9_]+", label) is None: + raise GateFailure(f"invalid schema label: {label!r}") + schema = f"qbit_a1_{label}_{os.getpid()}_{os.urandom(5).hex()}" + if SCHEMA_PATTERN.fullmatch(schema) is None or len(schema) > 63: + raise GateFailure(f"invalid generated schema: {schema!r}") + # Register authority before the transaction starts. If this call is + # interrupted, run_psql kills/reaps its child before atexit examines the + # durable marker. + OWNED_SCHEMAS.append((schema, RUN_MARKER)) + run_psql( + f""" +BEGIN; +CREATE SCHEMA "{schema}"; +COMMENT ON SCHEMA "{schema}" IS '{RUN_MARKER}'; +COMMIT; +""" + ) + return schema + + +def cleanup_owned_schemas() -> None: + while OWNED_SCHEMAS: + schema, expected_marker = OWNED_SCHEMAS[-1] + if SCHEMA_PATTERN.fullmatch(schema) is None: + raise GateFailure(f"refusing to drop invalid schema: {schema!r}") + marker = run_json( + f""" +SELECT json_build_object( + 'marker', ( + SELECT obj_description(oid, 'pg_namespace') + FROM pg_namespace + WHERE nspname = '{schema}' + ) +); +""" + )["marker"] + if marker is None: + OWNED_SCHEMAS.pop() + continue + if marker != expected_marker: + raise GateFailure( + f"refusing to drop schema with wrong marker: {schema!r}" + ) + run_psql(f'DROP SCHEMA "{schema}" CASCADE;') + OWNED_SCHEMAS.pop() + + +def _signal_exit(signum: int, _frame: object) -> None: + # Raising first lets the active run_psql frame kill and reap its exact + # process group. Registered atexit cleanup runs only after that unwind. + raise SystemExit(128 + signum) + + +signal.signal(signal.SIGTERM, _signal_exit) +signal.signal(signal.SIGINT, _signal_exit) +atexit.register(cleanup_owned_schemas) +atexit.register(cleanup_active_children) + + +class ScopedPsqlLedger(PsqlShareLedger): + def __init__(self, *, test_schema: str, **kwargs: object) -> None: + if not _schema_registered(test_schema): + raise GateFailure("ledger requires a registered owned schema") + self._test_schema = test_schema + kwargs["psql_command"] = BASE_PSQL_COMMAND + kwargs["native_client_mode"] = "psql" + super().__init__(**kwargs) # type: ignore[arg-type] + + def _run_sql(self, sql: str) -> str: + if not _schema_registered(self._test_schema): + raise GateFailure("ledger schema authority was revoked") + return run_psql(sql, schema=self._test_schema) + + +def public_sentinel() -> dict[str, object]: + return run_json( + """ +SELECT json_build_object( + 'pool_oid', 'public.qbit_pool_blocks'::regclass::oid, + 'pool_rows', ( + SELECT md5(COALESCE( + string_agg(row_to_json(row_value)::text, ',' ORDER BY block_hash), + '' + )) + FROM public.qbit_pool_blocks row_value + ), + 'sequence_oid', + 'public.qbit_audit_publication_sequence_seq'::regclass::oid, + 'sequence_definition', ( + SELECT json_build_object( + 'type_oid', sequence.seqtypid, + 'start', sequence.seqstart, + 'increment', sequence.seqincrement, + 'max', sequence.seqmax, + 'min', sequence.seqmin, + 'cache', sequence.seqcache, + 'cycle', sequence.seqcycle, + 'persistence', relation.relpersistence, + 'owner', relation.relowner + ) + FROM pg_catalog.pg_sequence sequence + JOIN pg_catalog.pg_class relation + ON relation.oid = sequence.seqrelid + WHERE sequence.seqrelid = + 'public.qbit_audit_publication_sequence_seq'::regclass + ), + 'sequence_state', ( + SELECT row_to_json(state) + FROM ( + SELECT last_value, is_called + FROM public.qbit_audit_publication_sequence_seq + ) state + ), + 'index_oid', + 'public.qbit_pool_blocks_audit_publication_sequence_idx'::regclass::oid, + 'index_filenode', pg_relation_filenode( + 'public.qbit_pool_blocks_audit_publication_sequence_idx'::regclass + ), + 'index_definition', pg_get_indexdef( + 'public.qbit_pool_blocks_audit_publication_sequence_idx'::regclass + ), + 'index_catalog', ( + SELECT json_build_object( + 'kind', relation.relkind, + 'persistence', relation.relpersistence, + 'owner', relation.relowner, + 'same_owner_as_table', relation.relowner = ( + SELECT pool_blocks.relowner + FROM pg_catalog.pg_class pool_blocks + JOIN pg_catalog.pg_namespace pool_namespace + ON pool_namespace.oid = pool_blocks.relnamespace + WHERE pool_namespace.nspname = 'public' + AND pool_blocks.relname = 'qbit_pool_blocks' + ), + 'unique', index_definition.indisunique, + 'valid', index_definition.indisvalid, + 'ready', index_definition.indisready, + 'live', index_definition.indislive, + 'immediate', index_definition.indimmediate, + 'primary', index_definition.indisprimary, + 'exclusion', index_definition.indisexclusion, + 'clustered', index_definition.indisclustered, + 'replica_identity', index_definition.indisreplident, + 'nulls_not_distinct', index_definition.indnullsnotdistinct, + 'key_count', index_definition.indnkeyatts, + 'attribute_count', index_definition.indnatts, + 'keys', index_definition.indkey::text, + 'classes', index_definition.indclass::text, + 'collations', index_definition.indcollation::text, + 'options', index_definition.indoption::text, + 'expressions', index_definition.indexprs::text, + 'predicate', index_definition.indpred::text + ) + FROM pg_catalog.pg_index index_definition + JOIN pg_catalog.pg_class relation + ON relation.oid = index_definition.indexrelid + WHERE index_definition.indexrelid = + 'public.qbit_pool_blocks_audit_publication_sequence_idx'::regclass + ), + 'constraint', ( + SELECT json_build_object( + 'oid', oid, + 'conbin', conbin::text, + 'definition', pg_get_constraintdef(oid, true), + 'validated', convalidated, + 'name', conname, + 'type', contype, + 'keys', conkey::text, + 'local', conislocal, + 'inherited_count', coninhcount, + 'no_inherit', connoinherit, + 'deferrable', condeferrable, + 'deferred', condeferred + ) + FROM pg_constraint + WHERE conrelid = 'public.qbit_pool_blocks'::regclass + AND conname = + 'qbit_pool_blocks_audit_publication_sequence_check' + ) +); +""" + ) + + +def marker_schema_count() -> int: + value = run_json( + f""" +SELECT json_build_object( + 'count', count(*) +) +FROM pg_namespace +WHERE obj_description(oid, 'pg_namespace') = '{RUN_MARKER}'; +""" + )["count"] + return int(value) + + +def allocator_state(ledger: ScopedPsqlLedger) -> dict[str, object]: + return ledger._run_json( + """ +SELECT json_build_object( + 'last_value', last_value, + 'is_called', is_called +) +FROM qbit_audit_publication_sequence_seq; +""" + ) + + +def seed_prepared_direct( + postgres: ScopedPsqlLedger, + memory: SingleWriterShareLedger, + *, + block_hash: str, + block_height: int, +) -> None: + postgres._run_sql( + f""" +INSERT INTO qbit_pool_blocks ( + block_hash, + block_height, + parent_hash, + coinbase_txid, + payout_manifest_sha256, + chain_state, + maturity_state +) VALUES ( + '{block_hash}', + {block_height}, + '{'10' * 32}', + '{'20' * 32}', + '{'30' * 32}', + 'prepared', + 'immature' +) +ON CONFLICT (block_hash) DO NOTHING; +""" + ) + memory.persist_accepted_block( + block_hash=block_hash, + block_height=block_height, + parent_hash="10" * 32, + final_bundle={}, + audit_report={}, + ) + + +def normalized_response(payload: dict[str, int | str]) -> dict[str, int | str]: + return {**payload, "backend": "ledger"} + + +def assert_states_equal( + postgres: ScopedPsqlLedger, + memory: SingleWriterShareLedger, + block_hashes: list[str], + message: str, +) -> None: + for block_hash in block_hashes: + assert_equal( + postgres.pool_block_state(block_hash=block_hash), + memory.pool_block_state(block_hash=block_hash), + f"{message} state {block_hash}", + ) + assert_equal( + postgres.audit_publication_sequence_floor(), + memory.audit_publication_sequence_floor(), + f"{message} durable floor", + ) + + +def call_both( + postgres: ScopedPsqlLedger, + memory: SingleWriterShareLedger, + method: str, + *, + block_hash: str, + active_tip_height: int, + known_hashes: list[str], + message: str, +) -> dict[str, int | str]: + postgres_result = getattr(postgres, method)( + block_hash=block_hash, + active_tip_height=active_tip_height, + ) + memory_result = getattr(memory, method)( + block_hash=block_hash, + active_tip_height=active_tip_height, + ) + assert_equal( + normalized_response(postgres_result), + normalized_response(memory_result), + f"{message} response", + ) + assert_states_equal(postgres, memory, known_hashes, message) + return postgres_result + + +def test_exact_transition_parity() -> None: + schema = create_owned_schema("parity") + postgres = ScopedPsqlLedger( + test_schema=schema, + writer_id="a1-parity", + writer_epoch=1, + initialize_schema=True, + audit_bundle_canonicalizer=fake_bundle_bytes, + ) + memory = SingleWriterShareLedger() + try: + assert_equal( + postgres._run_json( + "SELECT json_build_object('schema', current_schema());" + )["schema"], + schema, + "parity current schema", + ) + assert_equal(allocator_state(postgres), {"last_value": 1, "is_called": False}, "parity fresh allocator") + block_a = "a1" * 32 + block_b = "b1" * 32 + block_c = "c1" * 32 + known = [block_a] + seed_prepared_direct(postgres, memory, block_hash=block_a, block_height=10) + seed_prepared_direct(postgres, memory, block_hash=block_a, block_height=10) + assert_states_equal(postgres, memory, known, "duplicate direct prepared seed") + + api_hash = "d1" * 32 + expected_inline_body_bytes = fake_bundle_bytes(PERSIST_BUNDLE) + expected_inline_body_byte_len = len(expected_inline_body_bytes) + postgres_first_persist = postgres.persist_accepted_block( + block_hash=api_hash, + block_height=30, + parent_hash="10" * 32, + final_bundle=PERSIST_BUNDLE, + audit_report=PERSIST_REPORT, + ) + memory_first_persist = memory.persist_accepted_block( + block_hash=api_hash, + block_height=30, + parent_hash="10" * 32, + final_bundle=PERSIST_BUNDLE, + audit_report=PERSIST_REPORT, + ) + assert_equal( + postgres_first_persist, + { + "backend": "postgres-psql", + "share_count": 0, + "block_count": 1, + "bundle_count": 1, + "payout_entry_count": 1, + "carry_forward_count": 1, + "onchain_output_count": 1, + "audit_bundle_sha256": PERSIST_BUNDLE_SHA256, + "body_uri": "", + "audit_body_byte_len": expected_inline_body_byte_len, + }, + "PostgreSQL first persist API response", + ) + persisted_inline_body = postgres._run_json( + f""" +SELECT json_build_object( + 'audit_bundle_present', audit_bundle IS NOT NULL, + 'body_uri_is_null', body_uri IS NULL, + 'audit_body_byte_len', audit_body_byte_len, + 'audit_bundle_sha256', audit_bundle_sha256, + 'audit_bundle', audit_bundle +) +FROM qbit_pool_audit_bundles +WHERE block_hash = '{api_hash}'; +""" + ) + assert_equal( + persisted_inline_body, + { + "audit_bundle_present": True, + "body_uri_is_null": True, + "audit_body_byte_len": expected_inline_body_byte_len, + "audit_bundle_sha256": hashlib.sha256( + expected_inline_body_bytes + ).hexdigest(), + "audit_bundle": PERSIST_BUNDLE, + }, + "PostgreSQL inline audit body row", + ) + assert_equal( + memory_first_persist, + { + "backend": "memory", + "share_count": 0, + "block_count": 0, + "payout_entry_count": 0, + "carry_forward_count": 0, + }, + "memory first persist API response", + ) + postgres_duplicate_persist = postgres.persist_accepted_block( + block_hash=api_hash, + block_height=30, + parent_hash="10" * 32, + final_bundle=PERSIST_BUNDLE, + audit_report=PERSIST_REPORT, + ) + memory_duplicate_persist = memory.persist_accepted_block( + block_hash=api_hash, + block_height=30, + parent_hash="10" * 32, + final_bundle=PERSIST_BUNDLE, + audit_report=PERSIST_REPORT, + ) + assert_equal( + postgres_duplicate_persist, + postgres_first_persist, + "PostgreSQL duplicate persist exact response", + ) + assert_equal( + memory_duplicate_persist, + memory_first_persist, + "memory duplicate persist exact response", + ) + known.append(api_hash) + assert_states_equal(postgres, memory, known, "public persist API parity") + + before = allocator_state(postgres) + call_both(postgres, memory, "confirm_accepted_block", block_hash=block_a, active_tip_height=9, known_hashes=known, message="wrong-height confirmation") + assert_equal(allocator_state(postgres), before, "wrong-height confirmation allocator immobility") + confirmed = call_both(postgres, memory, "confirm_accepted_block", block_hash=block_a, active_tip_height=10, known_hashes=known, message="first confirmation") + assert_equal(confirmed["audit_publication_sequence"], 1, "first event ordinal") + before = allocator_state(postgres) + replay = call_both(postgres, memory, "confirm_accepted_block", block_hash=block_a, active_tip_height=10, known_hashes=known, message="exact confirmation replay") + assert_equal(replay["audit_publication_sequence"], 1, "replay ordinal") + for replay_round in range(2, 4): + replay = call_both( + postgres, + memory, + "confirm_accepted_block", + block_hash=block_a, + active_tip_height=10, + known_hashes=known, + message=f"exact confirmation replay round {replay_round}", + ) + assert_equal( + replay["audit_publication_sequence"], + 1, + f"replay round {replay_round} ordinal", + ) + assert_equal(allocator_state(postgres), before, "exact replay allocator immobility") + before = allocator_state(postgres) + call_both(postgres, memory, "reactivate_pool_block", block_hash=block_a, active_tip_height=10, known_hashes=known, message="confirmed count-zero reactivation") + assert_equal(allocator_state(postgres), before, "count-zero reactivation allocator immobility") + call_both(postgres, memory, "mark_pool_block_inactive", block_hash=block_a, active_tip_height=10, known_hashes=known, message="inactive transition") + before = allocator_state(postgres) + call_both( + postgres, + memory, + "confirm_accepted_block", + block_hash=block_a, + active_tip_height=10, + known_hashes=known, + message="inactive confirmation", + ) + assert_equal( + allocator_state(postgres), + before, + "inactive confirmation allocator immobility", + ) + call_both(postgres, memory, "reactivate_pool_block", block_hash=block_a, active_tip_height=9, known_hashes=known, message="wrong-height reactivation") + assert_equal(allocator_state(postgres), before, "wrong-height reactivation allocator immobility") + postgres.release_writer_lease() + postgres.close() + postgres = ScopedPsqlLedger( + test_schema=schema, + writer_id="a1-parity-reactivation-restart", + writer_epoch=1, + audit_bundle_canonicalizer=fake_bundle_bytes, + ) + assert_states_equal( + postgres, + memory, + known, + "inactive restart serializer", + ) + assert_equal( + allocator_state(postgres), + before, + "inactive restart allocator preservation", + ) + reactivated = call_both(postgres, memory, "reactivate_pool_block", block_hash=block_a, active_tip_height=10, known_hashes=known, message="reactivation") + assert_equal(reactivated["audit_publication_sequence"], 1, "reactivation preserves published ordinal") + assert_equal(allocator_state(postgres), before, "reactivation allocator immobility") + before = allocator_state(postgres) + reactivated_confirmation = call_both( + postgres, + memory, + "confirm_accepted_block", + block_hash=block_a, + active_tip_height=10, + known_hashes=known, + message="reactivated confirmation replay", + ) + assert_equal( + reactivated_confirmation["audit_publication_sequence"], + 1, + "reactivated confirmation replay ordinal", + ) + call_both(postgres, memory, "reactivate_pool_block", block_hash=block_a, active_tip_height=10, known_hashes=known, message="reactivation replay") + assert_equal(allocator_state(postgres), before, "reactivation replay allocator immobility") + call_both(postgres, memory, "mark_pool_block_inactive", block_hash=block_a, active_tip_height=10, known_hashes=known, message="second inactive transition") + call_both(postgres, memory, "reverse_immature_block", block_hash=block_a, active_tip_height=10, known_hashes=known, message="terminal reversal") + before = allocator_state(postgres) + postgres.release_writer_lease() + postgres.close() + postgres = ScopedPsqlLedger( + test_schema=schema, + writer_id="a1-parity-reversed-restart", + writer_epoch=1, + audit_bundle_canonicalizer=fake_bundle_bytes, + ) + assert_states_equal(postgres, memory, known, "reversed restart serializer") + assert_equal( + allocator_state(postgres), + before, + "reversed restart allocator preservation", + ) + call_both(postgres, memory, "confirm_accepted_block", block_hash=block_a, active_tip_height=10, known_hashes=known, message="reversed confirmation") + call_both(postgres, memory, "reactivate_pool_block", block_hash=block_a, active_tip_height=10, known_hashes=known, message="reversed reactivation") + assert_equal(allocator_state(postgres), before, "terminal transition allocator immobility") + + known.append(block_b) + seed_prepared_direct(postgres, memory, block_hash=block_b, block_height=1) + lower = call_both(postgres, memory, "confirm_accepted_block", block_hash=block_b, active_tip_height=1, known_hashes=known, message="later lower-height confirmation") + assert_equal(lower["audit_publication_sequence"], 2, "event order dominates block height") + + known.append(block_c) + seed_prepared_direct(postgres, memory, block_hash=block_c, block_height=20) + before = allocator_state(postgres) + call_both(postgres, memory, "reject_prepared_block", block_hash=block_c, active_tip_height=20, known_hashes=known, message="prepared rejection") + call_both(postgres, memory, "reject_prepared_block", block_hash=block_c, active_tip_height=20, known_hashes=known, message="rejection replay") + assert_equal(allocator_state(postgres), before, "rejection allocator immobility") + block_d = "e1" * 32 + known.append(block_d) + seed_prepared_direct(postgres, memory, block_hash=block_d, block_height=0) + final_confirmation = call_both( + postgres, + memory, + "confirm_accepted_block", + block_hash=block_d, + active_tip_height=0, + known_hashes=known, + message="post-replay fresh confirmation", + ) + assert_equal( + final_confirmation["audit_publication_sequence"], + 3, + "post-replay exact next ordinal", + ) + assert_equal(postgres.audit_publication_sequence_floor(), 3, "parity final durable floor") + finally: + postgres.release_writer_lease() + postgres.close() + + +LEGACY_POOL_BLOCKS_SQL = """ +CREATE TABLE qbit_pool_blocks ( + block_hash text PRIMARY KEY, + block_height bigint NOT NULL CHECK (block_height >= 0), + parent_hash text NOT NULL, + coinbase_txid text NOT NULL, + payout_manifest_sha256 text NOT NULL, + found_at timestamptz NOT NULL DEFAULT clock_timestamp(), + chain_state text NOT NULL DEFAULT 'prepared' + CHECK (chain_state IN ('prepared', 'confirmed', 'inactive', 'rejected', 'reversed')), + maturity_state text NOT NULL DEFAULT 'immature' + CHECK (maturity_state IN ('immature', 'mature', 'reversed')), + matured_at timestamptz, + disconnected_at timestamptz, + CHECK ((maturity_state = 'mature') = (matured_at IS NOT NULL)), + CHECK ((maturity_state = 'reversed') = (disconnected_at IS NOT NULL)) +); +""" + + +def assert_empty_serializer_case(*, legacy: bool) -> None: + label = "empty_legacy" if legacy else "empty_fresh" + schema = create_owned_schema(label) + if legacy: + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=schema) + first = ScopedPsqlLedger( + test_schema=schema, + writer_id=f"a1-{label}-first", + writer_epoch=1, + initialize_schema=True, + ) + try: + assert_equal(first.audit_publication_sequence_floor(), 0, f"{label} floor") + assert_equal(allocator_state(first), {"last_value": 1, "is_called": False}, f"{label} allocator") + finally: + first.release_writer_lease() + first.close() + second = ScopedPsqlLedger( + test_schema=schema, + writer_id=f"a1-{label}-second", + writer_epoch=1, + initialize_schema=True, + ) + try: + assert_equal(second.audit_publication_sequence_floor(), 0, f"{label} rerun floor") + assert_equal(allocator_state(second), {"last_value": 1, "is_called": False}, f"{label} rerun allocator") + block_hash = hashlib.sha256(label.encode()).hexdigest() + memory = SingleWriterShareLedger() + before = allocator_state(second) + for method, operation in ( + ("confirm_accepted_block", "confirmation"), + ("reactivate_pool_block", "reactivation"), + ("mark_pool_block_inactive", "inactive transition"), + ("reverse_immature_block", "reversal"), + ("reject_prepared_block", "rejection"), + ): + call_both( + second, + memory, + method, + block_hash=block_hash, + active_tip_height=1, + known_hashes=[block_hash], + message=f"{label} missing {operation}", + ) + assert_equal( + second.pool_block_state(block_hash=block_hash), + None, + f"{label} missing state serializer", + ) + assert_equal( + allocator_state(second), + before, + f"{label} missing-row allocator immobility", + ) + seed_prepared_direct(second, memory, block_hash=block_hash, block_height=1) + seed_prepared_direct(second, memory, block_hash=block_hash, block_height=1) + assert_states_equal( + second, + memory, + [block_hash], + f"{label} nullable prepared serializer", + ) + assert_equal( + second.pool_block_state(block_hash=block_hash)[ + "audit_publication_sequence" + ], + None, + f"{label} explicit prepared nullable ordinal", + ) + before = allocator_state(second) + call_both( + second, + memory, + "confirm_accepted_block", + block_hash=block_hash, + active_tip_height=0, + known_hashes=[block_hash], + message=f"{label} wrong-height confirmation", + ) + call_both( + second, + memory, + "reactivate_pool_block", + block_hash=block_hash, + active_tip_height=1, + known_hashes=[block_hash], + message=f"{label} prepared count-zero reactivation", + ) + assert_equal( + allocator_state(second), + before, + f"{label} nullable zero-op allocator immobility", + ) + result = call_both( + second, + memory, + "confirm_accepted_block", + block_hash=block_hash, + active_tip_height=1, + known_hashes=[block_hash], + message=f"{label} first confirmation", + ) + assert_equal(result["audit_publication_sequence"], 1, f"{label} first ordinal") + assert_equal(second.audit_publication_sequence_floor(), 1, f"{label} confirmed floor") + finally: + second.release_writer_lease() + second.close() + + +def transition_binding_state_sql( + *, + target_schema: str, + decoy_schema: str, +) -> str: + return f""" +SELECT json_build_object( + 'target_sequence', ( + SELECT json_build_object('last_value', last_value, 'is_called', is_called) + FROM "{target_schema}".qbit_audit_publication_sequence_seq + ), + 'decoy_sequence', ( + SELECT json_build_object('last_value', last_value, 'is_called', is_called) + FROM {decoy_schema}.qbit_audit_publication_sequence_seq + ), + 'target_rows', ( + SELECT json_object_agg( + block_hash, + json_build_object( + 'chain_state', chain_state, + 'audit_publication_sequence', audit_publication_sequence + ) + ) + FROM "{target_schema}".qbit_pool_blocks + WHERE block_hash IN ('{'71' * 32}', '{'72' * 32}') + ), + 'decoy_rows', ( + SELECT json_object_agg( + block_hash, + json_build_object( + 'chain_state', chain_state, + 'audit_publication_sequence', audit_publication_sequence + ) + ) + FROM {decoy_schema}.qbit_pool_blocks + ), + 'target_lease_changed', ( + SELECT before.value <> + row_to_json(current_lease)::text + FROM a1_target_lease_before before, + "{target_schema}".qbit_ledger_writer_lease current_lease + ), + 'target_lease', ( + SELECT json_build_object( + 'writer_id', writer_id, + 'writer_epoch', writer_epoch, + 'writer_session_token', writer_session_token, + 'active', lease_expires_at > clock_timestamp(), + 'expiry_after_update', lease_expires_at > updated_at + ) + FROM "{target_schema}".qbit_ledger_writer_lease + ), + 'decoy_lease_unchanged', ( + SELECT before.value = row_to_json(current_lease)::text + FROM a1_decoy_lease_before before, + {decoy_schema}.qbit_ledger_writer_lease current_lease + ), + 'decoy_lease', ( + SELECT json_build_object( + 'writer_id', writer_id, + 'writer_epoch', writer_epoch, + 'writer_session_token', writer_session_token + ) + FROM {decoy_schema}.qbit_ledger_writer_lease + ), + 'proconfig', ( + SELECT json_object_agg(procedure.proname, procedure.proconfig[1]) + FROM pg_catalog.pg_proc procedure + JOIN pg_catalog.pg_namespace namespace + ON namespace.oid = procedure.pronamespace + WHERE procedure.oid IN ( + pg_catalog.to_regprocedure( + '"{target_schema}".qbit_confirm_pool_block(' + 'text,bigint,text,bigint,text,interval)' + ), + pg_catalog.to_regprocedure( + '"{target_schema}".qbit_reactivate_pool_block(' + 'text,bigint,text,bigint,text,interval)' + ) + ) + ) +); +""" + + +def seed_transition_binding_target(ledger: ScopedPsqlLedger) -> None: + ledger._run_sql( + f""" +INSERT INTO qbit_pool_blocks ( + block_hash, + audit_publication_sequence, + block_height, + parent_hash, + coinbase_txid, + payout_manifest_sha256, + chain_state, + maturity_state +) VALUES + ('{'71' * 32}', NULL, 10, '{'10' * 32}', '{'20' * 32}', + '{'30' * 32}', 'prepared', 'immature'), + ('{'72' * 32}', NULL, 20, '{'10' * 32}', '{'20' * 32}', + '{'30' * 32}', 'prepared', 'immature'); +""" + ) + confirmed = ledger.confirm_accepted_block( + block_hash="72" * 32, + active_tip_height=20, + ) + assert_equal( + confirmed["audit_publication_sequence"], + 1, + "binding seed confirmation ordinal", + ) + assert_equal( + ledger.mark_pool_block_inactive( + block_hash="72" * 32, + active_tip_height=20, + )["inactive_count"], + 1, + "binding seed inactive transition", + ) + + +def decoy_objects_sql(*, temporary: bool) -> str: + temporary_keyword = "TEMPORARY " if temporary else "" + return f""" +CREATE {temporary_keyword}TABLE qbit_ledger_writer_lease ( + singleton boolean PRIMARY KEY, + writer_id text NOT NULL, + writer_epoch bigint NOT NULL, + writer_session_token text NOT NULL, + lease_expires_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL +); +CREATE {temporary_keyword}TABLE qbit_pool_blocks ( + block_hash text PRIMARY KEY, + audit_publication_sequence bigint, + block_height bigint NOT NULL, + chain_state text NOT NULL, + maturity_state text NOT NULL +); +CREATE {temporary_keyword}SEQUENCE qbit_audit_publication_sequence_seq + AS bigint START WITH 700; +INSERT INTO qbit_ledger_writer_lease VALUES ( + true, + 'a1-binding', + 1, + 'a1-binding-token', + '2040-01-01T00:00:00Z', + '2000-01-01T00:00:00Z' +); +INSERT INTO qbit_pool_blocks VALUES + ('{'71' * 32}', NULL, 10, 'prepared', 'immature'), + ('{'72' * 32}', 700, 20, 'inactive', 'immature'); +""" + + +def assert_transition_binding_case(*, temporary: bool) -> None: + label = "binding_temp" if temporary else "binding_ordinary" + target_schema = create_owned_schema(label) + target = ScopedPsqlLedger( + test_schema=target_schema, + writer_id="a1-binding", + writer_epoch=1, + writer_session_token="a1-binding-token", + initialize_schema=True, + ) + try: + seed_transition_binding_target(target) + if temporary: + decoy_schema = "pg_temp" + caller_setup = decoy_objects_sql(temporary=True) + caller_path = f'pg_temp, "{target_schema}", pg_catalog' + else: + ordinary_decoy = create_owned_schema("binding_decoy") + run_psql(decoy_objects_sql(temporary=False), schema=ordinary_decoy) + decoy_schema = f'"{ordinary_decoy}"' + caller_setup = "" + caller_path = f'"{ordinary_decoy}", "{target_schema}", pg_catalog' + expected_config = f"search_path=pg_catalog, {target_schema}, pg_temp" + result = run_json( + f""" +SET statement_timeout = '20s'; +{caller_setup} +SET search_path TO {caller_path}; +CREATE TEMPORARY TABLE a1_target_lease_before (value text NOT NULL); +INSERT INTO a1_target_lease_before +SELECT row_to_json(target_lease)::text +FROM "{target_schema}".qbit_ledger_writer_lease target_lease; +CREATE TEMPORARY TABLE a1_decoy_lease_before (value text NOT NULL); +INSERT INTO a1_decoy_lease_before +SELECT row_to_json(decoy_lease)::text +FROM {decoy_schema}.qbit_ledger_writer_lease decoy_lease; +SELECT "{target_schema}".qbit_confirm_pool_block( + '{'71' * 32}', 10, 'a1-binding', 1, 'a1-binding-token', interval '5 minutes' +); +SELECT "{target_schema}".qbit_reactivate_pool_block( + '{'72' * 32}', 20, 'a1-binding', 1, 'a1-binding-token', interval '5 minutes' +); +{transition_binding_state_sql(target_schema=target_schema, decoy_schema=decoy_schema)} +""" + ) + assert_equal( + result["proconfig"], + { + "qbit_confirm_pool_block": expected_config, + "qbit_reactivate_pool_block": expected_config, + }, + f"{label} exact transition proconfig", + ) + assert_equal( + result["target_sequence"], + {"last_value": 2, "is_called": True}, + f"{label} target allocator", + ) + assert_equal( + result["decoy_sequence"], + {"last_value": 700, "is_called": False}, + f"{label} decoy allocator preservation", + ) + assert_equal( + result["target_rows"], + { + "71" * 32: { + "chain_state": "confirmed", + "audit_publication_sequence": 2, + }, + "72" * 32: { + "chain_state": "confirmed", + "audit_publication_sequence": 1, + }, + }, + f"{label} target row transitions", + ) + assert_equal( + result["decoy_rows"], + { + "71" * 32: { + "chain_state": "prepared", + "audit_publication_sequence": None, + }, + "72" * 32: { + "chain_state": "inactive", + "audit_publication_sequence": 700, + }, + }, + f"{label} decoy row preservation", + ) + assert_equal( + result["target_lease_changed"], + True, + f"{label} target lease renewal", + ) + assert_equal( + result["target_lease"], + { + "writer_id": "a1-binding", + "writer_epoch": 1, + "writer_session_token": "a1-binding-token", + "active": True, + "expiry_after_update": True, + }, + f"{label} target lease identity", + ) + assert_equal( + result["decoy_lease_unchanged"], + True, + f"{label} decoy lease byte preservation", + ) + assert_equal( + result["decoy_lease"], + { + "writer_id": "a1-binding", + "writer_epoch": 1, + "writer_session_token": "a1-binding-token", + }, + f"{label} decoy lease identity", + ) + finally: + target.release_writer_lease() + target.close() + + +def test_durable_floor_ignores_allocator_gaps() -> None: + schema = create_owned_schema("durable_floor") + ledger = ScopedPsqlLedger( + test_schema=schema, + writer_id="a1-durable-floor", + writer_epoch=1, + initialize_schema=True, + ) + memory_seed = SingleWriterShareLedger() + block_a = "61" * 32 + block_b = "62" * 32 + rejected = "63" * 32 + try: + assert_equal(ledger.audit_publication_sequence_floor(), 0, "empty floor") + seed_prepared_direct( + ledger, + memory_seed, + block_hash=block_a, + block_height=10, + ) + assert_equal( + ledger.pool_block_state(block_hash=block_a)[ + "audit_publication_sequence" + ], + None, + "prepared floor row is nullable", + ) + committed_gap = ledger._run_json( + """ +SELECT json_build_object( + 'value', nextval('qbit_audit_publication_sequence_seq') +); +""" + )["value"] + assert_equal(committed_gap, 1, "committed unattached allocator gap") + assert_equal(ledger.audit_publication_sequence_floor(), 0, "gap floor") + ledger._run_sql( + """ +BEGIN; +SELECT nextval('qbit_audit_publication_sequence_seq'); +ROLLBACK; +""" + ) + assert_equal( + allocator_state(ledger), + {"last_value": 2, "is_called": True}, + "rolled-back allocator gap persists", + ) + assert_equal( + ledger.audit_publication_sequence_floor(), + 0, + "rolled-back gap is not authority", + ) + confirmed = ledger.confirm_accepted_block( + block_hash=block_a, + active_tip_height=10, + ) + assert_equal( + confirmed["audit_publication_sequence"], + 3, + "confirmation follows both allocator gaps", + ) + before = allocator_state(ledger) + for _round in range(3): + replay = ledger.confirm_accepted_block( + block_hash=block_a, + active_tip_height=10, + ) + assert_equal( + replay["audit_publication_sequence"], + 3, + "gap-vector confirmation replay", + ) + assert_equal(allocator_state(ledger), before, "gap replay immobility") + ledger.mark_pool_block_inactive( + block_hash=block_a, + active_tip_height=10, + ) + assert_equal(ledger.audit_publication_sequence_floor(), 3, "inactive floor") + reactivated = ledger.reactivate_pool_block( + block_hash=block_a, + active_tip_height=10, + ) + assert_equal( + reactivated["audit_publication_sequence"], + 3, + "reactivation preserves ordinal after gaps", + ) + assert_equal( + allocator_state(ledger), + before, + "gap-vector reactivation allocator immobility", + ) + before = allocator_state(ledger) + assert_equal( + ledger.reactivate_pool_block( + block_hash=block_a, + active_tip_height=10, + ), + {"backend": "postgres-psql", "reactivated_count": 0}, + "gap-vector reactivation replay", + ) + assert_equal(allocator_state(ledger), before, "reactivation replay immobility") + ledger.mark_pool_block_inactive( + block_hash=block_a, + active_tip_height=10, + ) + ledger.reverse_immature_block( + block_hash=block_a, + active_tip_height=10, + ) + assert_equal(ledger.audit_publication_sequence_floor(), 3, "reversed floor") + seed_prepared_direct( + ledger, + memory_seed, + block_hash=rejected, + block_height=20, + ) + before = allocator_state(ledger) + assert_equal( + ledger.reject_prepared_block( + block_hash=rejected, + active_tip_height=20, + )["rejected_count"], + 1, + "rejected nullable floor row", + ) + assert_equal(allocator_state(ledger), before, "rejection allocator immobility") + assert_equal(ledger.audit_publication_sequence_floor(), 3, "rejected floor") + assert_equal( + ledger._run_json( + """ +SELECT json_build_object( + 'value', nextval('qbit_audit_publication_sequence_seq') +); +""" + )["value"], + 4, + "second unattached allocator gap", + ) + seed_prepared_direct( + ledger, + memory_seed, + block_hash=block_b, + block_height=1, + ) + assert_equal( + ledger.confirm_accepted_block( + block_hash=block_b, + active_tip_height=1, + )["audit_publication_sequence"], + 5, + "post-gap event ordinal", + ) + ledger._run_sql( + f""" +INSERT INTO qbit_pool_blocks ( + block_hash, audit_publication_sequence, block_height, parent_hash, + coinbase_txid, payout_manifest_sha256, chain_state, maturity_state +) VALUES + ('{'64' * 32}', 9000000000, 30, '{'10' * 32}', '{'20' * 32}', + '{'30' * 32}', 'prepared', 'immature'), + ('{'65' * 32}', 9000000001, 31, '{'10' * 32}', '{'20' * 32}', + '{'30' * 32}', 'rejected', 'immature'); +""" + ) + assert_equal( + ledger.audit_publication_sequence_floor(), + 9_000_000_001, + "partial-state bigint durable floor", + ) + finally: + ledger.release_writer_lease() + ledger.close() + restart = ScopedPsqlLedger( + test_schema=schema, + writer_id="a1-durable-floor-restart", + writer_epoch=1, + initialize_schema=True, + ) + try: + assert_equal( + restart.audit_publication_sequence_floor(), + 9_000_000_001, + "restart bigint durable floor", + ) + assert_equal( + restart.pool_block_state(block_hash=block_a), + { + "block_hash": block_a, + "block_height": 10, + "parent_hash": "10" * 32, + "chain_state": "reversed", + "maturity_state": "reversed", + "audit_publication_sequence": 3, + }, + "restart exact reversed serializer", + ) + assert_equal( + restart.pool_block_state(block_hash=rejected)[ + "audit_publication_sequence" + ], + None, + "restart exact rejected nullable serializer", + ) + finally: + restart.release_writer_lease() + restart.close() + + +def main() -> None: + public_before = public_sentinel() + failure: BaseException | None = None + try: + test_exact_transition_parity() + assert_empty_serializer_case(legacy=False) + assert_empty_serializer_case(legacy=True) + assert_transition_binding_case(temporary=False) + assert_transition_binding_case(temporary=True) + test_durable_floor_ignores_allocator_gaps() + except BaseException as error: + failure = error + try: + cleanup_active_children() + cleanup_owned_schemas() + assert_equal(marker_schema_count(), 0, "owned schema marker cleanup") + assert_equal(public_sentinel(), public_before, "public sentinel preservation") + except BaseException as cleanup_error: + if failure is None: + raise + raise GateFailure( + f"scenario failed with {failure!r}; cleanup also failed with " + f"{cleanup_error!r}" + ) from cleanup_error + else: + atexit.unregister(cleanup_active_children) + atexit.unregister(cleanup_owned_schemas) + if failure is not None: + raise failure + print( + "prism postgres A1 gate PASS " + "exact-transition-parity empty-fresh empty-legacy " + "ordinary-decoy-binding temporary-decoy-binding durable-floor-gaps" + ) + + +if __name__ == "__main__": + main() diff --git a/tests/prism_postgres_a1_migration_gate.py b/tests/prism_postgres_a1_migration_gate.py new file mode 100644 index 0000000..949e75f --- /dev/null +++ b/tests/prism_postgres_a1_migration_gate.py @@ -0,0 +1,1782 @@ +"""Non-discovered PostgreSQL ordinal migration integration gate.""" + +from __future__ import annotations + +import hashlib +import subprocess +import tempfile +import time +from pathlib import Path +from typing import Any + +from tests import prism_postgres_a1_gate as support +from tests.prism_postgres_a1_gate import ( + ACTIVE_CHILDREN, + ACTIVE_CHILDREN_LOCK, + BASE_PSQL_ARGV, + GateFailure, + LEGACY_POOL_BLOCKS_SQL, + PSQL_OUTPUT_LIMIT_BYTES, + PSQL_TIMEOUT_SECONDS, + RUN_TOKEN, + SCHEMA_PATTERN, + ScopedPsqlLedger, + _schema_registered, + _terminate_and_reap, + assert_equal, + create_owned_schema, + run_json, + run_psql, +) + +MIGRATION_HASH_A = "81" * 32 +MIGRATION_HASH_B = "82" * 32 +MIGRATION_HASH_C = "83" * 32 +MIGRATION_HASH_OTHER = "84" * 32 +BIGINT_MAX = 9_223_372_036_854_775_807 + + +def add_ordinal_column(schema: str) -> None: + run_psql( + "ALTER TABLE qbit_pool_blocks " + "ADD COLUMN audit_publication_sequence bigint;", + schema=schema, + ) + + +def create_ordinal_sequence( + schema: str, + *, + last_value: int = 1, + is_called: bool = False, +) -> None: + run_psql( + f""" +CREATE SEQUENCE qbit_audit_publication_sequence_seq AS bigint; +SELECT setval( + 'qbit_audit_publication_sequence_seq'::regclass, + {last_value}, + {'true' if is_called else 'false'} +); +""", + schema=schema, + ) + + +def seed_migration_rows( + schema: str, + *, + ordinals: tuple[int | None, int | None, int | None] | None, + include_other_states: bool = True, +) -> None: + ordinal_column = ( + ", audit_publication_sequence" if ordinals is not None else "" + ) + ordinal_values = ( + ["NULL" if value is None else str(value) for value in ordinals] + if ordinals is not None + else [None, None, None] + ) + + def row( + block_hash: str, + found_at: str, + ordinal: str | None, + state: str, + maturity: str = "immature", + ) -> str: + ordinal_sql = f", {ordinal}" if ordinal is not None else "" + disconnected = ( + ", '2020-01-02T00:00:00Z'" if maturity == "reversed" else ", NULL" + ) + return ( + f"('{block_hash}', 10, '{'10' * 32}', '{'20' * 32}', " + f"'{'30' * 32}', '{found_at}', '{state}', '{maturity}', " + f"NULL{disconnected}{ordinal_sql})" + ) + + rows = [ + # Insertion order intentionally differs from deterministic migration + # order. A/B share a timestamp and sort by block hash. + row( + MIGRATION_HASH_C, + "2020-01-01T00:00:02Z", + ordinal_values[2], + "confirmed", + ), + row( + MIGRATION_HASH_B, + "2020-01-01T00:00:01Z", + ordinal_values[1], + "confirmed", + ), + row( + MIGRATION_HASH_A, + "2020-01-01T00:00:01Z", + ordinal_values[0], + "confirmed", + ), + ] + if include_other_states: + rows.extend( + [ + row( + MIGRATION_HASH_OTHER, + "2020-01-01T00:00:03Z", + "NULL" if ordinals is not None else None, + "prepared", + ), + row( + "85" * 32, + "2020-01-01T00:00:04Z", + "NULL" if ordinals is not None else None, + "inactive", + ), + row( + "86" * 32, + "2020-01-01T00:00:05Z", + "NULL" if ordinals is not None else None, + "rejected", + "reversed", + ), + row( + "87" * 32, + "2020-01-01T00:00:06Z", + "NULL" if ordinals is not None else None, + "reversed", + "reversed", + ), + ] + ) + run_psql( + f""" +INSERT INTO qbit_pool_blocks ( + block_hash, + block_height, + parent_hash, + coinbase_txid, + payout_manifest_sha256, + found_at, + chain_state, + maturity_state, + matured_at, + disconnected_at + {ordinal_column} +) VALUES +{', '.join(rows)}; +""", + schema=schema, + ) + + +def migration_snapshot(schema: str) -> dict[str, object]: + if SCHEMA_PATTERN.fullmatch(schema) is None: + raise GateFailure(f"invalid migration snapshot schema: {schema!r}") + snapshot = run_json( + f""" +SELECT json_build_object( + 'rows', ( + SELECT COALESCE( + json_agg(to_jsonb(block) ORDER BY block_hash), + '[]'::json + ) + FROM "{schema}".qbit_pool_blocks block + ), + 'column', ( + SELECT json_build_object( + 'attnum', attribute.attnum, + 'type_oid', attribute.atttypid::pg_catalog.int8, + 'nullable', NOT attribute.attnotnull, + 'has_default', attribute.atthasdef, + 'identity', attribute.attidentity, + 'generated', attribute.attgenerated, + 'collation', attribute.attcollation::pg_catalog.int8 + ) + FROM pg_catalog.pg_attribute attribute + JOIN pg_catalog.pg_class relation + ON relation.oid = attribute.attrelid + JOIN pg_catalog.pg_namespace namespace + ON namespace.oid = relation.relnamespace + WHERE namespace.nspname = '{schema}' + AND relation.relname = 'qbit_pool_blocks' + AND attribute.attname = 'audit_publication_sequence' + AND attribute.attnum > 0 + AND NOT attribute.attisdropped + ), + 'sequence_catalog', ( + SELECT json_build_object( + 'oid', relation.oid, + 'kind', relation.relkind, + 'persistence', relation.relpersistence, + 'owner', relation.relowner, + 'type_oid', sequence.seqtypid::pg_catalog.int8, + 'start', sequence.seqstart, + 'increment', sequence.seqincrement, + 'max', sequence.seqmax, + 'min', sequence.seqmin, + 'cache', sequence.seqcache, + 'cycle', sequence.seqcycle, + 'owned_dependencies', ( + SELECT count(*) + FROM pg_catalog.pg_depend dependency + WHERE dependency.classid = 'pg_catalog.pg_class'::regclass + AND dependency.objid = relation.oid + AND dependency.refclassid = 'pg_catalog.pg_class'::regclass + AND dependency.refobjsubid > 0 + AND dependency.deptype IN ('a', 'i') + ), + 'same_owner_as_table', relation.relowner = ( + SELECT pool_blocks.relowner + FROM pg_catalog.pg_class pool_blocks + JOIN pg_catalog.pg_namespace pool_namespace + ON pool_namespace.oid = pool_blocks.relnamespace + WHERE pool_namespace.nspname = '{schema}' + AND pool_blocks.relname = 'qbit_pool_blocks' + ) + ) + FROM pg_catalog.pg_class relation + JOIN pg_catalog.pg_namespace namespace + ON namespace.oid = relation.relnamespace + LEFT JOIN pg_catalog.pg_sequence sequence + ON sequence.seqrelid = relation.oid + WHERE namespace.nspname = '{schema}' + AND relation.relname = 'qbit_audit_publication_sequence_seq' + ), + 'indexes', ( + SELECT COALESCE(json_agg(index_row ORDER BY index_row.name), '[]'::json) + FROM ( + SELECT + index_relation.relname AS name, + index_relation.oid, + pg_catalog.pg_relation_filenode(index_relation.oid) AS filenode, + index_relation.relkind AS kind, + index_relation.relpersistence AS persistence, + index_relation.relowner AS owner, + pg_catalog.pg_get_indexdef(index_relation.oid) AS definition, + index_definition.indisunique AS unique, + index_definition.indisvalid AS valid, + index_definition.indisready AS ready, + index_definition.indislive AS live, + index_definition.indimmediate AS immediate, + index_definition.indisprimary AS primary, + index_definition.indisexclusion AS exclusion, + index_definition.indisclustered AS clustered, + index_definition.indisreplident AS replica_identity, + index_definition.indnullsnotdistinct AS nulls_not_distinct, + index_definition.indnkeyatts AS key_count, + index_definition.indnatts AS attribute_count, + index_definition.indkey::text AS keys, + index_definition.indclass::text AS classes, + index_definition.indcollation::text AS collations, + index_definition.indoption::text AS options, + index_definition.indexprs::text AS expressions, + index_definition.indpred::text AS predicate + FROM pg_catalog.pg_index index_definition + JOIN pg_catalog.pg_class index_relation + ON index_relation.oid = index_definition.indexrelid + JOIN pg_catalog.pg_class table_relation + ON table_relation.oid = index_definition.indrelid + JOIN pg_catalog.pg_namespace namespace + ON namespace.oid = table_relation.relnamespace + WHERE namespace.nspname = '{schema}' + AND table_relation.relname = 'qbit_pool_blocks' + AND EXISTS ( + SELECT 1 + FROM unnest(index_definition.indkey) AS key(attnum) + JOIN pg_catalog.pg_attribute attribute + ON attribute.attrelid = table_relation.oid + AND attribute.attnum = key.attnum + WHERE attribute.attname = 'audit_publication_sequence' + ) + ) index_row + ), + 'named_index_relation', ( + SELECT json_build_object( + 'oid', relation.oid, + 'kind', relation.relkind, + 'persistence', relation.relpersistence, + 'owner', relation.relowner, + 'filenode', pg_catalog.pg_relation_filenode(relation.oid) + ) + FROM pg_catalog.pg_class relation + JOIN pg_catalog.pg_namespace namespace + ON namespace.oid = relation.relnamespace + WHERE namespace.nspname = '{schema}' + AND relation.relname = + 'qbit_pool_blocks_audit_publication_sequence_idx' + ), + 'constraints', ( + SELECT COALESCE( + json_agg(constraint_row ORDER BY constraint_row.name), + '[]'::json + ) + FROM ( + SELECT + constraint_definition.conname AS name, + constraint_definition.oid, + constraint_definition.contype AS type, + constraint_definition.conbin::text AS expression_tree, + pg_catalog.pg_get_constraintdef( + constraint_definition.oid, + true + ) AS definition, + constraint_definition.convalidated AS validated, + constraint_definition.conkey::text AS keys, + constraint_definition.conislocal AS local, + constraint_definition.coninhcount AS inherited_count, + constraint_definition.connoinherit AS no_inherit, + constraint_definition.condeferrable AS deferrable, + constraint_definition.condeferred AS deferred + FROM pg_catalog.pg_constraint constraint_definition + JOIN pg_catalog.pg_class table_relation + ON table_relation.oid = constraint_definition.conrelid + JOIN pg_catalog.pg_namespace namespace + ON namespace.oid = table_relation.relnamespace + WHERE namespace.nspname = '{schema}' + AND table_relation.relname = 'qbit_pool_blocks' + AND ( + constraint_definition.conname = + 'qbit_pool_blocks_audit_publication_sequence_check' + OR pg_catalog.pg_get_constraintdef( + constraint_definition.oid, + true + ) LIKE '%audit_publication_sequence%' + ) + ) constraint_row + ) +); +""" + ) + sequence_catalog = snapshot["sequence_catalog"] + if sequence_catalog is None or sequence_catalog["kind"] != "S": + snapshot["sequence_state"] = None + else: + snapshot["sequence_state"] = run_json( + f""" +SELECT json_build_object('last_value', last_value, 'is_called', is_called) +FROM "{schema}".qbit_audit_publication_sequence_seq; +""" + ) + return snapshot + + +def assert_advisory_lock_available(message: str) -> None: + result = run_json( + """ +WITH acquired AS ( + SELECT pg_catalog.pg_try_advisory_lock( + pg_catalog.hashtext('qbit_audit_publication_sequence_migration') + ) AS ok +) +SELECT json_build_object( + 'acquired', (SELECT ok FROM acquired), + 'released', CASE + WHEN (SELECT ok FROM acquired) THEN pg_catalog.pg_advisory_unlock( + pg_catalog.hashtext('qbit_audit_publication_sequence_migration') + ) + ELSE false + END +); +""" + ) + assert_equal( + result, + {"acquired": True, "released": True}, + message, + ) + + +def initialize_migration_schema( + schema: str, + *, + writer_id: str, +) -> ScopedPsqlLedger: + return ScopedPsqlLedger( + test_schema=schema, + writer_id=writer_id, + writer_epoch=1, + initialize_schema=True, + ) + + +def assert_valid_migration_case( + *, + schema: str, + label: str, + expected_ordinals: dict[str, int | None], + expected_next_ordinal: int | None, + expected_sequence_state: dict[str, object] | None = None, +) -> dict[str, object]: + if not _schema_registered(schema): + raise GateFailure(f"{label} requires an explicit registered schema") + ledger = initialize_migration_schema(schema, writer_id=f"a1-{label}-first") + try: + first = migration_snapshot(schema) + rows = { + str(row["block_hash"]): row.get("audit_publication_sequence") + for row in first["rows"] # type: ignore[union-attr] + } + for block_hash, expected in expected_ordinals.items(): + assert_equal(rows[block_hash], expected, f"{label} ordinal {block_hash}") + if expected_sequence_state is not None: + assert_equal( + first["sequence_state"], + expected_sequence_state, + f"{label} exact allocator state", + ) + assert_equal(len(first["indexes"]), 1, f"{label} canonical index count") + assert_equal(len(first["constraints"]), 1, f"{label} canonical constraint count") + assert_equal( + first["column"], + { + "attnum": first["column"]["attnum"], # type: ignore[index] + "type_oid": 20, + "nullable": True, + "has_default": False, + "identity": "", + "generated": "", + "collation": 0, + }, + f"{label} exact ordinal column catalog", + ) + sequence_catalog = first["sequence_catalog"] + assert sequence_catalog is not None + assert_equal( + { + key: sequence_catalog[key] # type: ignore[index] + for key in ( + "kind", + "persistence", + "type_oid", + "start", + "increment", + "max", + "min", + "cache", + "cycle", + "owned_dependencies", + "same_owner_as_table", + ) + }, + { + "kind": "S", + "persistence": "p", + "type_oid": 20, + "start": 1, + "increment": 1, + "max": BIGINT_MAX, + "min": 1, + "cache": 1, + "cycle": False, + "owned_dependencies": 0, + "same_owner_as_table": True, + }, + f"{label} exact ordinal sequence catalog", + ) + index = first["indexes"][0] # type: ignore[index] + assert_equal( + { + key: index[key] + for key in ( + "name", + "kind", + "persistence", + "unique", + "valid", + "ready", + "live", + "immediate", + "primary", + "exclusion", + "clustered", + "replica_identity", + "nulls_not_distinct", + "key_count", + "attribute_count", + "collations", + "options", + "expressions", + "predicate", + ) + }, + { + "name": "qbit_pool_blocks_audit_publication_sequence_idx", + "kind": "i", + "persistence": "p", + "unique": True, + "valid": True, + "ready": True, + "live": True, + "immediate": True, + "primary": False, + "exclusion": False, + "clustered": False, + "replica_identity": False, + "nulls_not_distinct": False, + "key_count": 1, + "attribute_count": 1, + "collations": "0", + "options": "0", + "expressions": None, + "predicate": None, + }, + f"{label} exact ordinal index catalog", + ) + assert_equal( + index["definition"], + "CREATE UNIQUE INDEX " + "qbit_pool_blocks_audit_publication_sequence_idx ON " + f"{schema}.qbit_pool_blocks USING btree " + "(audit_publication_sequence)", + f"{label} exact ordinal index definition", + ) + constraint = first["constraints"][0] # type: ignore[index] + assert_equal( + { + key: constraint[key] + for key in ( + "name", + "type", + "definition", + "validated", + "local", + "inherited_count", + "no_inherit", + "deferrable", + "deferred", + ) + }, + { + "name": "qbit_pool_blocks_audit_publication_sequence_check", + "type": "c", + "definition": "CHECK ((audit_publication_sequence IS NULL OR " + "audit_publication_sequence > 0) AND (chain_state <> " + "'confirmed'::text OR audit_publication_sequence IS NOT NULL))", + "validated": True, + "local": True, + "inherited_count": 0, + "no_inherit": False, + "deferrable": False, + "deferred": False, + }, + f"{label} exact ordinal constraint catalog", + ) + finally: + ledger.release_writer_lease() + ledger.close() + for rerun_round in (1, 2): + rerun = initialize_migration_schema( + schema, + writer_id=f"a1-{label}-rerun-{rerun_round}", + ) + try: + assert_equal( + migration_snapshot(schema), + first, + f"{label} idempotent catalog rerun {rerun_round}", + ) + finally: + rerun.release_writer_lease() + rerun.close() + legacy_inactive_hash = "85" * 32 + legacy_inactive_sequence = expected_ordinals.get(legacy_inactive_hash) + if legacy_inactive_sequence is not None: + reactivator = ScopedPsqlLedger( + test_schema=schema, + writer_id=f"a1-{label}-legacy-reactivation", + writer_epoch=1, + ) + try: + reactivated = reactivator.reactivate_pool_block( + block_hash=legacy_inactive_hash, + active_tip_height=10, + ) + assert_equal( + reactivated["audit_publication_sequence"], + legacy_inactive_sequence, + f"{label} legacy inactive reactivation preserves backfilled ordinal", + ) + finally: + reactivator.release_writer_lease() + reactivator.close() + if expected_next_ordinal is not None: + final = ScopedPsqlLedger( + test_schema=schema, + writer_id=f"a1-{label}-next", + writer_epoch=1, + ) + try: + block_hash = hashlib.sha256(f"{label}-next".encode()).hexdigest() + final._run_sql( + f""" +INSERT INTO qbit_pool_blocks ( + block_hash, block_height, parent_hash, coinbase_txid, + payout_manifest_sha256, chain_state, maturity_state +) VALUES ( + '{block_hash}', 99, '{'10' * 32}', '{'20' * 32}', + '{'30' * 32}', 'prepared', 'immature' +); +""" + ) + confirmation = final.confirm_accepted_block( + block_hash=block_hash, + active_tip_height=99, + ) + assert_equal( + confirmation["audit_publication_sequence"], + expected_next_ordinal, + f"{label} next production ordinal", + ) + finally: + final.release_writer_lease() + final.close() + assert_advisory_lock_available(f"{label} advisory lock released") + return first + + +def insert_migration_row( + schema: str, + *, + block_hash: str, + ordinal: int | None, + found_at: str, + chain_state: str = "confirmed", +) -> None: + ordinal_sql = "NULL" if ordinal is None else str(ordinal) + run_psql( + f""" +INSERT INTO qbit_pool_blocks ( + block_hash, + audit_publication_sequence, + block_height, + parent_hash, + coinbase_txid, + payout_manifest_sha256, + found_at, + chain_state, + maturity_state +) VALUES ( + '{block_hash}', + {ordinal_sql}, + 10, + '{'10' * 32}', + '{'20' * 32}', + '{'30' * 32}', + '{found_at}', + '{chain_state}', + 'immature' +); +""", + schema=schema, + ) + + +def assert_initializer_rejects_unchanged( + schema: str, + *, + label: str, + error_fragment: str, +) -> dict[str, object]: + before = migration_snapshot(schema) + try: + initialize_migration_schema(schema, writer_id=f"a1-{label}-reject") + except GateFailure as error: + if error_fragment not in str(error): + raise GateFailure( + f"{label} wrong failure: expected {error_fragment!r}, got {error!r}" + ) from error + else: + raise GateFailure(f"{label} initializer unexpectedly succeeded") + assert_equal( + migration_snapshot(schema), + before, + f"{label} transactional rollback snapshot", + ) + assert_advisory_lock_available(f"{label} rejecting advisory lock released") + return before + + +def create_canonical_ordinal_constraint(schema: str, *, not_valid: bool = False) -> None: + run_psql( + """ +ALTER TABLE qbit_pool_blocks +ADD CONSTRAINT qbit_pool_blocks_audit_publication_sequence_check +CHECK ( + (audit_publication_sequence IS NULL OR audit_publication_sequence > 0) + AND (chain_state <> 'confirmed' OR audit_publication_sequence IS NOT NULL) +) +""" + + (" NOT VALID;" if not_valid else ";"), + schema=schema, + ) + + +def test_m0_m11_migration_matrix() -> None: + other_null_expected = { + MIGRATION_HASH_OTHER: None, + "85" * 32: 4, + "86" * 32: None, + "87" * 32: None, + } + common_null_expected = { + MIGRATION_HASH_A: 1, + MIGRATION_HASH_B: 2, + MIGRATION_HASH_C: 3, + **other_null_expected, + } + + m0 = create_owned_schema("m0") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m0) + seed_migration_rows(m0, ordinals=None) + assert_valid_migration_case( + schema=m0, + label="m0", + expected_ordinals=common_null_expected, + expected_next_ordinal=5, + ) + + m1 = create_owned_schema("m1") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m1) + add_ordinal_column(m1) + seed_migration_rows(m1, ordinals=(None, None, None)) + assert_valid_migration_case( + schema=m1, + label="m1", + expected_ordinals=common_null_expected, + expected_next_ordinal=5, + ) + + m2 = create_owned_schema("m2") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m2) + create_ordinal_sequence(m2) + seed_migration_rows(m2, ordinals=None) + assert_valid_migration_case( + schema=m2, + label="m2", + expected_ordinals=common_null_expected, + expected_next_ordinal=5, + ) + + m3 = create_owned_schema("m3") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m3) + add_ordinal_column(m3) + create_ordinal_sequence(m3) + seed_migration_rows(m3, ordinals=(None, None, None)) + assert_valid_migration_case( + schema=m3, + label="m3", + expected_ordinals=common_null_expected, + expected_next_ordinal=5, + ) + + m4 = create_owned_schema("m4") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m4) + add_ordinal_column(m4) + create_ordinal_sequence(m4) + seed_migration_rows(m4, ordinals=(None, None, None)) + run_psql( + "CREATE UNIQUE INDEX " + "qbit_pool_blocks_audit_publication_sequence_idx " + "ON qbit_pool_blocks (audit_publication_sequence);", + schema=m4, + ) + m4_before = migration_snapshot(m4) + m4_first = assert_valid_migration_case( + schema=m4, + label="m4", + expected_ordinals=common_null_expected, + expected_next_ordinal=5, + ) + assert_equal( + m4_first["indexes"], + m4_before["indexes"], + "M4 first migration preserves correct index OID/filenode/catalog", + ) + assert_equal( + m4_first["named_index_relation"], + m4_before["named_index_relation"], + "M4 first migration preserves correct named index relation", + ) + + m5 = create_owned_schema("m5") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m5) + add_ordinal_column(m5) + create_ordinal_sequence(m5, last_value=3, is_called=True) + seed_migration_rows(m5, ordinals=(1, 2, 3)) + create_canonical_ordinal_constraint(m5) + m5_before = migration_snapshot(m5) + m5_first = assert_valid_migration_case( + schema=m5, + label="m5", + expected_ordinals={ + MIGRATION_HASH_A: 1, + MIGRATION_HASH_B: 2, + MIGRATION_HASH_C: 3, + **other_null_expected, + }, + expected_next_ordinal=5, + ) + assert_equal( + m5_first["constraints"], + m5_before["constraints"], + "M5 first migration preserves correct constraint OID/conbin/catalog", + ) + + m6 = create_owned_schema("m6") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m6) + add_ordinal_column(m6) + create_ordinal_sequence(m6) + seed_migration_rows(m6, ordinals=(9002, None, None)) + assert_valid_migration_case( + schema=m6, + label="m6", + expected_ordinals={ + MIGRATION_HASH_A: 9002, + MIGRATION_HASH_B: 9003, + MIGRATION_HASH_C: 9004, + MIGRATION_HASH_OTHER: None, + "85" * 32: 9005, + "86" * 32: None, + "87" * 32: None, + }, + expected_next_ordinal=9006, + expected_sequence_state={"last_value": 9005, "is_called": True}, + ) + + for label, called, inactive_ordinal, expected_next in ( + ("m7a", True, 12, 13), + ("m8a", False, 11, 12), + ): + schema = create_owned_schema(label) + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=schema) + add_ordinal_column(schema) + create_ordinal_sequence(schema, last_value=11, is_called=called) + seed_migration_rows(schema, ordinals=(5, 6, 7)) + assert_valid_migration_case( + schema=schema, + label=label, + expected_ordinals={ + MIGRATION_HASH_A: 5, + MIGRATION_HASH_B: 6, + MIGRATION_HASH_C: 7, + MIGRATION_HASH_OTHER: None, + "85" * 32: inactive_ordinal, + "86" * 32: None, + "87" * 32: None, + }, + expected_next_ordinal=expected_next, + expected_sequence_state={ + "last_value": inactive_ordinal, + "is_called": True, + }, + ) + + for label, called, expected, inactive_ordinal, expected_next in ( + ("m7b", True, (12, 13, 14), 15, 16), + ("m8b", False, (11, 12, 13), 14, 15), + ): + schema = create_owned_schema(label) + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=schema) + add_ordinal_column(schema) + create_ordinal_sequence(schema, last_value=11, is_called=called) + seed_migration_rows(schema, ordinals=(None, None, None)) + run_psql( + f""" +UPDATE qbit_pool_blocks +SET audit_publication_sequence = 7 +WHERE block_hash = '{MIGRATION_HASH_OTHER}'; +""", + schema=schema, + ) + assert_valid_migration_case( + schema=schema, + label=label, + expected_ordinals={ + MIGRATION_HASH_A: expected[0], + MIGRATION_HASH_B: expected[1], + MIGRATION_HASH_C: expected[2], + MIGRATION_HASH_OTHER: 7, + "85" * 32: inactive_ordinal, + "86" * 32: None, + "87" * 32: None, + }, + expected_next_ordinal=expected_next, + expected_sequence_state={ + "last_value": inactive_ordinal, + "is_called": True, + }, + ) + + m9 = create_owned_schema("m9") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m9) + add_ordinal_column(m9) + seed_migration_rows(m9, ordinals=(0, None, None)) + assert_initializer_rejects_unchanged( + m9, + label="m9", + error_fragment="invalid non-positive audit publication sequence", + ) + + m10 = create_owned_schema("m10") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m10) + add_ordinal_column(m10) + seed_migration_rows(m10, ordinals=(4, 4, None)) + assert_initializer_rejects_unchanged( + m10, + label="m10", + error_fragment="duplicate audit publication sequence", + ) + + m11 = create_owned_schema("m11") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m11) + create_ordinal_sequence(m11, last_value=19, is_called=True) + seed_migration_rows(m11, ordinals=None) + run_psql( + """ +ALTER TABLE qbit_pool_blocks +ADD CONSTRAINT qbit_pool_blocks_audit_publication_sequence_check +CHECK (block_height < 1000); +""", + schema=m11, + ) + assert_initializer_rejects_unchanged( + m11, + label="m11_wrong_constraint", + error_fragment="invalid audit publication sequence constraint definition", + ) + run_psql( + "ALTER TABLE qbit_pool_blocks DROP CONSTRAINT " + "qbit_pool_blocks_audit_publication_sequence_check;", + schema=m11, + ) + assert_valid_migration_case( + schema=m11, + label="m11", + expected_ordinals={ + MIGRATION_HASH_A: 20, + MIGRATION_HASH_B: 21, + MIGRATION_HASH_C: 22, + "85" * 32: 23, + }, + expected_next_ordinal=24, + ) + + m11_alt = create_owned_schema("m11_alt") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m11_alt) + add_ordinal_column(m11_alt) + create_ordinal_sequence(m11_alt, last_value=3, is_called=True) + seed_migration_rows(m11_alt, ordinals=(1, 2, 3)) + run_psql( + "CREATE UNIQUE INDEX qbit_audit_publication_sequence_alternate " + "ON qbit_pool_blocks (audit_publication_sequence);", + schema=m11_alt, + ) + assert_initializer_rejects_unchanged( + m11_alt, + label="m11_alternate_index", + error_fragment="duplicate audit publication sequence index definition", + ) + run_psql( + "DROP INDEX qbit_audit_publication_sequence_alternate;", + schema=m11_alt, + ) + assert_valid_migration_case( + schema=m11_alt, + label="m11_alt", + expected_ordinals={ + MIGRATION_HASH_A: 1, + MIGRATION_HASH_B: 2, + MIGRATION_HASH_C: 3, + "85" * 32: 4, + }, + expected_next_ordinal=5, + ) + + m11_named = create_owned_schema("m11_named") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m11_named) + add_ordinal_column(m11_named) + create_ordinal_sequence(m11_named, last_value=3, is_called=True) + seed_migration_rows(m11_named, ordinals=(1, 2, 3)) + run_psql( + "CREATE INDEX qbit_pool_blocks_audit_publication_sequence_idx " + "ON qbit_pool_blocks (audit_publication_sequence);", + schema=m11_named, + ) + assert_initializer_rejects_unchanged( + m11_named, + label="m11_wrong_named_index", + error_fragment="invalid audit publication sequence index definition", + ) + run_psql( + "DROP INDEX qbit_pool_blocks_audit_publication_sequence_idx;", + schema=m11_named, + ) + assert_valid_migration_case( + schema=m11_named, + label="m11_named", + expected_ordinals={ + MIGRATION_HASH_A: 1, + MIGRATION_HASH_B: 2, + MIGRATION_HASH_C: 3, + "85" * 32: 4, + }, + expected_next_ordinal=5, + ) + + m11_sequence = create_owned_schema("m11_sequence") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m11_sequence) + add_ordinal_column(m11_sequence) + run_psql( + "CREATE SEQUENCE qbit_audit_publication_sequence_seq AS integer;", + schema=m11_sequence, + ) + seed_migration_rows(m11_sequence, ordinals=(None, None, None)) + assert_initializer_rejects_unchanged( + m11_sequence, + label="m11_wrong_sequence", + error_fragment="invalid audit publication sequence definition", + ) + run_psql( + "DROP SEQUENCE qbit_audit_publication_sequence_seq;", + schema=m11_sequence, + ) + create_ordinal_sequence(m11_sequence) + assert_valid_migration_case( + schema=m11_sequence, + label="m11_sequence", + expected_ordinals=common_null_expected, + expected_next_ordinal=5, + ) + + m11_column = create_owned_schema("m11_column") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m11_column) + run_psql( + "ALTER TABLE qbit_pool_blocks " + "ADD COLUMN audit_publication_sequence integer;", + schema=m11_column, + ) + seed_migration_rows(m11_column, ordinals=(None, None, None)) + assert_initializer_rejects_unchanged( + m11_column, + label="m11_wrong_column", + error_fragment="invalid audit publication sequence column definition", + ) + run_psql( + "ALTER TABLE qbit_pool_blocks ALTER COLUMN " + "audit_publication_sequence TYPE bigint;", + schema=m11_column, + ) + assert_valid_migration_case( + schema=m11_column, + label="m11_column", + expected_ordinals=common_null_expected, + expected_next_ordinal=5, + ) + + m11_not_valid = create_owned_schema("m11_not_valid") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m11_not_valid) + add_ordinal_column(m11_not_valid) + create_ordinal_sequence(m11_not_valid, last_value=3, is_called=True) + seed_migration_rows(m11_not_valid, ordinals=(1, 2, 3)) + create_canonical_ordinal_constraint(m11_not_valid, not_valid=True) + before_not_valid = migration_snapshot(m11_not_valid) + not_valid_constraint = before_not_valid["constraints"][0] # type: ignore[index] + assert_equal( + not_valid_constraint["validated"], + False, + "M11 preexisting canonical constraint starts NOT VALID", + ) + normalized = assert_valid_migration_case( + schema=m11_not_valid, + label="m11_not_valid", + expected_ordinals={ + MIGRATION_HASH_A: 1, + MIGRATION_HASH_B: 2, + MIGRATION_HASH_C: 3, + **other_null_expected, + }, + expected_next_ordinal=5, + ) + assert_equal( + normalized["constraints"][0]["oid"], # type: ignore[index] + not_valid_constraint["oid"], + "M11 NOT VALID normalization preserves constraint OID", + ) + + m11_alt_constraint = create_owned_schema("m11_alt_constraint") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m11_alt_constraint) + add_ordinal_column(m11_alt_constraint) + create_ordinal_sequence(m11_alt_constraint, last_value=3, is_called=True) + seed_migration_rows(m11_alt_constraint, ordinals=(1, 2, 3)) + run_psql( + """ +ALTER TABLE qbit_pool_blocks +ADD CONSTRAINT qbit_audit_publication_sequence_alternate_check +CHECK ( + (audit_publication_sequence IS NULL OR audit_publication_sequence > 0) + AND (chain_state <> 'confirmed' OR audit_publication_sequence IS NOT NULL) +); +""", + schema=m11_alt_constraint, + ) + assert_initializer_rejects_unchanged( + m11_alt_constraint, + label="m11_alternate_constraint", + error_fragment="duplicate audit publication sequence constraint definition", + ) + run_psql( + "ALTER TABLE qbit_pool_blocks DROP CONSTRAINT " + "qbit_audit_publication_sequence_alternate_check;", + schema=m11_alt_constraint, + ) + assert_valid_migration_case( + schema=m11_alt_constraint, + label="m11_alt_constraint", + expected_ordinals={ + MIGRATION_HASH_A: 1, + MIGRATION_HASH_B: 2, + MIGRATION_HASH_C: 3, + **other_null_expected, + }, + expected_next_ordinal=5, + ) + + m11_relkind = create_owned_schema("m11_relkind") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m11_relkind) + add_ordinal_column(m11_relkind) + create_ordinal_sequence(m11_relkind, last_value=3, is_called=True) + seed_migration_rows(m11_relkind, ordinals=(1, 2, 3)) + run_psql( + "CREATE TABLE qbit_pool_blocks_audit_publication_sequence_idx " + "(sentinel integer);", + schema=m11_relkind, + ) + assert_initializer_rejects_unchanged( + m11_relkind, + label="m11_wrong_index_relkind", + error_fragment="invalid audit publication sequence index definition", + ) + run_psql( + "DROP TABLE qbit_pool_blocks_audit_publication_sequence_idx;", + schema=m11_relkind, + ) + assert_valid_migration_case( + schema=m11_relkind, + label="m11_relkind", + expected_ordinals={ + MIGRATION_HASH_A: 1, + MIGRATION_HASH_B: 2, + MIGRATION_HASH_C: 3, + **other_null_expected, + }, + expected_next_ordinal=5, + ) + + +def prepare_ordinal_migration_schema( + label: str, + *, + sequence_last: int, + sequence_called: bool, +) -> str: + schema = create_owned_schema(label) + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=schema) + add_ordinal_column(schema) + create_ordinal_sequence( + schema, + last_value=sequence_last, + is_called=sequence_called, + ) + return schema + + +def assert_next_allocation_exhausted(schema: str, *, label: str) -> None: + block_hash = hashlib.sha256(f"{label}-exhausted".encode()).hexdigest() + run_psql( + f""" +INSERT INTO qbit_pool_blocks ( + block_hash, block_height, parent_hash, coinbase_txid, + payout_manifest_sha256, chain_state, maturity_state +) VALUES ( + '{block_hash}', 101, '{'10' * 32}', '{'20' * 32}', + '{'30' * 32}', 'prepared', 'immature' +); +""", + schema=schema, + ) + ledger = ScopedPsqlLedger( + test_schema=schema, + writer_id=f"a1-{label}-exhausted", + writer_epoch=1, + ) + try: + before = migration_snapshot(schema) + try: + ledger.confirm_accepted_block( + block_hash=block_hash, + active_tip_height=101, + ) + except GateFailure as error: + if "reached maximum value" not in str(error): + raise GateFailure( + f"{label} wrong allocator exhaustion: {error!r}" + ) from error + else: + raise GateFailure(f"{label} exhausted allocator unexpectedly advanced") + assert_equal( + migration_snapshot(schema), + before, + f"{label} exhaustion leaves rows/catalog/allocator unchanged", + ) + finally: + ledger.release_writer_lease() + ledger.close() + + +def test_migration_bigint_boundaries() -> None: + max_called = prepare_ordinal_migration_schema( + "bound_max_called", + sequence_last=BIGINT_MAX, + sequence_called=True, + ) + for offset, block_hash in zip( + (-2, -1, 0), + (MIGRATION_HASH_A, MIGRATION_HASH_B, MIGRATION_HASH_C), + strict=True, + ): + insert_migration_row( + max_called, + block_hash=block_hash, + ordinal=BIGINT_MAX + offset, + found_at=f"2020-01-01T00:00:0{offset + 3}Z", + ) + assert_valid_migration_case( + schema=max_called, + label="bound_max_called", + expected_ordinals={ + MIGRATION_HASH_A: BIGINT_MAX - 2, + MIGRATION_HASH_B: BIGINT_MAX - 1, + MIGRATION_HASH_C: BIGINT_MAX, + }, + expected_next_ordinal=None, + expected_sequence_state={"last_value": BIGINT_MAX, "is_called": True}, + ) + assert_next_allocation_exhausted( + max_called, + label="bound_max_called", + ) + + max_uncalled_one = prepare_ordinal_migration_schema( + "bound_max_one", + sequence_last=BIGINT_MAX, + sequence_called=False, + ) + insert_migration_row( + max_uncalled_one, + block_hash=MIGRATION_HASH_A, + ordinal=None, + found_at="2020-01-01T00:00:01Z", + ) + assert_valid_migration_case( + schema=max_uncalled_one, + label="bound_max_one", + expected_ordinals={MIGRATION_HASH_A: BIGINT_MAX}, + expected_next_ordinal=None, + expected_sequence_state={"last_value": BIGINT_MAX, "is_called": True}, + ) + assert_next_allocation_exhausted( + max_uncalled_one, + label="bound_max_one", + ) + + max_uncalled_two = prepare_ordinal_migration_schema( + "bound_max_two", + sequence_last=BIGINT_MAX, + sequence_called=False, + ) + for block_hash, found_at in ( + (MIGRATION_HASH_A, "2020-01-01T00:00:01Z"), + (MIGRATION_HASH_B, "2020-01-01T00:00:02Z"), + ): + insert_migration_row( + max_uncalled_two, + block_hash=block_hash, + ordinal=None, + found_at=found_at, + ) + assert_initializer_rejects_unchanged( + max_uncalled_two, + label="bound_max_two", + error_fragment="audit publication sequence exhausted", + ) + + durable_max_pending = prepare_ordinal_migration_schema( + "bound_durable_max", + sequence_last=1, + sequence_called=False, + ) + insert_migration_row( + durable_max_pending, + block_hash=MIGRATION_HASH_A, + ordinal=BIGINT_MAX, + found_at="2020-01-01T00:00:01Z", + ) + insert_migration_row( + durable_max_pending, + block_hash=MIGRATION_HASH_B, + ordinal=None, + found_at="2020-01-01T00:00:02Z", + ) + assert_initializer_rejects_unchanged( + durable_max_pending, + label="bound_durable_max", + error_fragment="audit publication sequence exhausted", + ) + + near_max_three = prepare_ordinal_migration_schema( + "bound_near_three", + sequence_last=BIGINT_MAX - 2, + sequence_called=False, + ) + for block_hash, second in ( + (MIGRATION_HASH_C, 2), + (MIGRATION_HASH_B, 1), + (MIGRATION_HASH_A, 1), + ): + insert_migration_row( + near_max_three, + block_hash=block_hash, + ordinal=None, + found_at=f"2020-01-01T00:00:0{second}Z", + ) + assert_valid_migration_case( + schema=near_max_three, + label="bound_near_three", + expected_ordinals={ + MIGRATION_HASH_A: BIGINT_MAX - 2, + MIGRATION_HASH_B: BIGINT_MAX - 1, + MIGRATION_HASH_C: BIGINT_MAX, + }, + expected_next_ordinal=None, + expected_sequence_state={"last_value": BIGINT_MAX, "is_called": True}, + ) + + near_max_four = prepare_ordinal_migration_schema( + "bound_near_four", + sequence_last=BIGINT_MAX - 2, + sequence_called=False, + ) + for index, block_hash in enumerate( + (MIGRATION_HASH_A, MIGRATION_HASH_B, MIGRATION_HASH_C, "88" * 32), + start=1, + ): + insert_migration_row( + near_max_four, + block_hash=block_hash, + ordinal=None, + found_at=f"2020-01-01T00:00:0{index}Z", + ) + assert_initializer_rejects_unchanged( + near_max_four, + label="bound_near_four", + error_fragment="audit publication sequence exhausted", + ) + + +def test_invalid_sequence_and_column_definitions() -> None: + sequence_definitions = { + "seq_start": "CREATE SEQUENCE qbit_audit_publication_sequence_seq AS bigint START WITH 2", + "seq_increment": "CREATE SEQUENCE qbit_audit_publication_sequence_seq AS bigint INCREMENT BY 2", + "seq_min": "CREATE SEQUENCE qbit_audit_publication_sequence_seq AS bigint MINVALUE 2 START WITH 2", + "seq_max": "CREATE SEQUENCE qbit_audit_publication_sequence_seq AS bigint MAXVALUE 100", + "seq_cache": "CREATE SEQUENCE qbit_audit_publication_sequence_seq AS bigint CACHE 2", + "seq_cycle": "CREATE SEQUENCE qbit_audit_publication_sequence_seq AS bigint CYCLE", + "seq_unlogged": "CREATE UNLOGGED SEQUENCE qbit_audit_publication_sequence_seq AS bigint", + } + for label, definition in sequence_definitions.items(): + schema = create_owned_schema(label) + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=schema) + add_ordinal_column(schema) + run_psql(definition + ";", schema=schema) + seed_migration_rows( + schema, + ordinals=(1, 2, 3), + include_other_states=False, + ) + assert_initializer_rejects_unchanged( + schema, + label=label, + error_fragment="invalid audit publication sequence definition", + ) + + owned = create_owned_schema("seq_owned") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=owned) + add_ordinal_column(owned) + create_ordinal_sequence(owned) + run_psql( + "ALTER SEQUENCE qbit_audit_publication_sequence_seq " + "OWNED BY qbit_pool_blocks.audit_publication_sequence;", + schema=owned, + ) + seed_migration_rows(owned, ordinals=(1, 2, 3)) + assert_initializer_rejects_unchanged( + owned, + label="seq_owned", + error_fragment="invalid audit publication sequence definition", + ) + + wrong_kind = create_owned_schema("seq_wrong_kind") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=wrong_kind) + add_ordinal_column(wrong_kind) + run_psql( + "CREATE TABLE qbit_audit_publication_sequence_seq (sentinel integer);", + schema=wrong_kind, + ) + seed_migration_rows(wrong_kind, ordinals=(1, 2, 3)) + assert_initializer_rejects_unchanged( + wrong_kind, + label="seq_wrong_kind", + error_fragment="invalid audit publication sequence definition", + ) + + column_definitions = { + "column_not_null": "bigint NOT NULL", + "column_default": "bigint DEFAULT 1", + "column_identity": "bigint GENERATED BY DEFAULT AS IDENTITY", + } + for label, definition in column_definitions.items(): + schema = create_owned_schema(label) + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=schema) + run_psql( + "ALTER TABLE qbit_pool_blocks ADD COLUMN " + f"audit_publication_sequence {definition};", + schema=schema, + ) + seed_migration_rows( + schema, + ordinals=(1, 2, 3), + include_other_states=False, + ) + assert_initializer_rejects_unchanged( + schema, + label=label, + error_fragment="invalid audit publication sequence column definition", + ) + + generated = create_owned_schema("column_generated") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=generated) + seed_migration_rows(generated, ordinals=None) + run_psql( + "ALTER TABLE qbit_pool_blocks ADD COLUMN " + "audit_publication_sequence bigint " + "GENERATED ALWAYS AS (block_height + 1) STORED;", + schema=generated, + ) + assert_initializer_rejects_unchanged( + generated, + label="column_generated", + error_fragment="invalid audit publication sequence column definition", + ) + # Owner mismatch needs CREATE/SET ROLE authority that the supported + # external-psql mode does not promise. Production still validates exact + # same-owner identity, and every portable sequence property is exercised. + + +def _psql_process_command() -> list[str]: + return [ + *BASE_PSQL_ARGV, + "--no-psqlrc", + "--set", + "ON_ERROR_STOP=1", + "--tuples-only", + "--no-align", + "--quiet", + ] + + +def _register_process(process: subprocess.Popen[str]) -> None: + with ACTIVE_CHILDREN_LOCK: + ACTIVE_CHILDREN.add(process) + + +def _forget_process(process: subprocess.Popen[str]) -> None: + with ACTIVE_CHILDREN_LOCK: + ACTIVE_CHILDREN.discard(process) + + +def _wait_for_advisory_holder( + process: subprocess.Popen[str], + *, + application_name: str, + deadline: float, +) -> None: + while time.monotonic() < deadline: + state = run_json( + f""" +SELECT json_build_object( + 'ready', EXISTS ( + SELECT 1 + FROM pg_catalog.pg_stat_activity activity + JOIN pg_catalog.pg_locks lock_state + ON lock_state.pid = activity.pid + WHERE activity.application_name = '{application_name}' + AND lock_state.locktype = 'advisory' + AND lock_state.granted + ) +); +""" + ) + if state["ready"]: + return + if process.poll() is not None: + break + time.sleep(0.05) + raise GateFailure( + f"advisory holder {application_name!r} not observed; exit={process.poll()}" + ) + + +def _wait_file_process( + process: subprocess.Popen[str], + *, + stdout_file: Any, + stderr_file: Any, + expected_success: bool, + error_fragment: str | None, +) -> None: + try: + process.wait(timeout=PSQL_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired as error: + _terminate_and_reap(process) + raise GateFailure("tagged psql worker timed out") from error + finally: + _forget_process(process) + stdout_file.seek(0) + stderr_file.seek(0) + stdout = stdout_file.read(PSQL_OUTPUT_LIMIT_BYTES + 1) + stderr = stderr_file.read(PSQL_OUTPUT_LIMIT_BYTES + 1) + if ( + len(stdout.encode("utf-8")) > PSQL_OUTPUT_LIMIT_BYTES + or len(stderr.encode("utf-8")) > PSQL_OUTPUT_LIMIT_BYTES + ): + raise GateFailure("tagged psql worker output exceeded 1 MiB") + if expected_success and process.returncode != 0: + raise GateFailure( + f"tagged psql worker failed with {process.returncode}: {stderr.strip()}" + ) + if not expected_success: + if process.returncode == 0: + raise GateFailure("rejecting tagged psql worker unexpectedly succeeded") + if error_fragment is not None and error_fragment not in stderr: + raise GateFailure( + f"tagged psql worker wrong failure: {stderr.strip()}" + ) + + +def test_migration_advisory_waiter() -> None: + schema_sql = ( + Path(__file__).resolve().parents[1] + / "crates/qbit-prism/sql/001_share_ledger.sql" + ).read_text(encoding="utf-8") + for outcome, rejecting in (("commit", False), ("rollback", True)): + schema = create_owned_schema(f"migration_wait_{outcome}") + rejection_before: dict[str, object] | None = None + if rejecting: + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=schema) + add_ordinal_column(schema) + seed_migration_rows(schema, ordinals=(0, None, None)) + rejection_before = migration_snapshot(schema) + # PostgreSQL truncates application_name to NAMEDATALEN - 1 bytes. + # Keep the tags below that limit so catalog coordination uses the + # exact values the clients set. + holder_tag = f"a1_mig_holder_{RUN_TOKEN}_{outcome}" + waiter_tag = f"a1_mig_waiter_{RUN_TOKEN}_{outcome}" + holder_input = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") + holder_stderr = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") + waiter_input = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") + waiter_stdout = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") + waiter_stderr = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") + holder: subprocess.Popen[str] | None = None + waiter: subprocess.Popen[str] | None = None + try: + holder_input.write( + f""" +SET application_name = '{holder_tag}'; +BEGIN; +SELECT pg_catalog.pg_advisory_xact_lock( + pg_catalog.hashtext('qbit_audit_publication_sequence_migration') +); +DO $qbit_a1_wait$ +DECLARE + waiter_observed boolean := false; +BEGIN + FOR attempt IN 1..600 LOOP + PERFORM pg_catalog.pg_stat_clear_snapshot(); + SELECT EXISTS ( + SELECT 1 + FROM pg_catalog.pg_stat_activity waiter + JOIN pg_catalog.pg_locks lock_state + ON lock_state.pid = waiter.pid + WHERE waiter.application_name = '{waiter_tag}' + AND lock_state.locktype = 'advisory' + AND NOT lock_state.granted + AND pg_catalog.pg_backend_pid() = ANY( + pg_catalog.pg_blocking_pids(waiter.pid) + ) + ) + INTO waiter_observed; + EXIT WHEN waiter_observed; + PERFORM pg_catalog.pg_sleep(0.05); + END LOOP; + IF NOT waiter_observed THEN + RAISE EXCEPTION 'migration advisory waiter not observed'; + END IF; +END; +$qbit_a1_wait$; +{outcome.upper()}; +""" + ) + holder_input.seek(0) + holder = subprocess.Popen( + _psql_process_command(), + stdin=holder_input, + stdout=subprocess.DEVNULL, + stderr=holder_stderr, + text=True, + bufsize=1, + start_new_session=True, + ) + _register_process(holder) + _wait_for_advisory_holder( + holder, + application_name=holder_tag, + deadline=time.monotonic() + PSQL_TIMEOUT_SECONDS, + ) + + waiter_input.write( + f""" +SET application_name = '{waiter_tag}'; +SET statement_timeout = '20s'; +SET lock_timeout = '20s'; +SET search_path TO "{schema}", pg_catalog; +{schema_sql} +""" + ) + waiter_input.seek(0) + waiter = subprocess.Popen( + _psql_process_command(), + stdin=waiter_input, + stdout=waiter_stdout, + stderr=waiter_stderr, + text=True, + start_new_session=True, + ) + _register_process(waiter) + try: + holder.wait(timeout=PSQL_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired as error: + _terminate_and_reap(holder) + raise GateFailure("migration advisory holder timed out") from error + finally: + _forget_process(holder) + if holder.returncode != 0: + holder_stderr.seek(0) + raise GateFailure( + "migration advisory holder failed: " + + holder_stderr.read(PSQL_OUTPUT_LIMIT_BYTES + 1).strip() + ) + + _wait_file_process( + waiter, + stdout_file=waiter_stdout, + stderr_file=waiter_stderr, + expected_success=not rejecting, + error_fragment=( + "invalid non-positive audit publication sequence" + if rejecting + else None + ), + ) + if rejecting: + assert rejection_before is not None + assert_equal( + migration_snapshot(schema), + rejection_before, + "rejecting advisory waiter exact rollback", + ) + else: + success_snapshot = migration_snapshot(schema) + assert_equal( + success_snapshot["sequence_state"], + {"last_value": 1, "is_called": False}, + "successful advisory waiter empty allocator", + ) + assert_advisory_lock_available( + f"migration {outcome} advisory lock no leak" + ) + remaining = run_json( + f""" +SELECT json_build_object('count', count(*)) +FROM pg_catalog.pg_stat_activity +WHERE application_name IN ('{holder_tag}', '{waiter_tag}'); +""" + )["count"] + assert_equal( + remaining, + 0, + f"migration {outcome} tagged backend cleanup", + ) + finally: + for process in (waiter, holder): + if process is not None: + if process.poll() is None: + _terminate_and_reap(process) + _forget_process(process) + for stream in (process.stdin, process.stdout, process.stderr): + if stream is not None and not stream.closed: + stream.close() + for stream in ( + holder_input, + holder_stderr, + waiter_input, + waiter_stdout, + waiter_stderr, + ): + if not stream.closed: + stream.close() + + +def main() -> None: + public_before = support.public_sentinel() + failure: BaseException | None = None + try: + test_m0_m11_migration_matrix() + test_migration_bigint_boundaries() + test_invalid_sequence_and_column_definitions() + test_migration_advisory_waiter() + except BaseException as error: + failure = error + try: + support.cleanup_active_children() + support.cleanup_owned_schemas() + support.assert_equal(support.marker_schema_count(), 0, "migration marker cleanup") + support.assert_equal(support.public_sentinel(), public_before, "migration public preservation") + except BaseException as cleanup_error: + if failure is None: + raise + raise GateFailure( + f"migration scenario failed with {failure!r}; cleanup also failed with {cleanup_error!r}" + ) from cleanup_error + else: + support.atexit.unregister(support.cleanup_active_children) + support.atexit.unregister(support.cleanup_owned_schemas) + if failure is not None: + raise failure + print( + "prism postgres A1 migration gate PASS " + "M0-M11 bigint-bounds invalid-definitions migration-advisory-waiter" + ) + + +if __name__ == "__main__": + main() diff --git a/tests/prism_postgres_a1_process_gate.py b/tests/prism_postgres_a1_process_gate.py new file mode 100644 index 0000000..9ff8536 --- /dev/null +++ b/tests/prism_postgres_a1_process_gate.py @@ -0,0 +1,2624 @@ +"""Non-discovered PostgreSQL/A1 cross-process integration gate.""" + +from __future__ import annotations + +from contextlib import ExitStack +import fcntl +import hashlib +import json +import os +from pathlib import Path +import re +import select +import subprocess +import sys +import tempfile +import time +import traceback +from typing import Any + +from lab.prism import audit_artifacts as audit_artifacts_module +from lab.prism.audit_artifacts import ( + AuditArtifactConfig, + AuditArtifactStore, + AuditPublicationIdentity, +) +from lab.prism.share_ledger import PsqlShareLedger +from tests import prism_postgres_a1_gate as support +from tests import prism_postgres_a1_migration_gate as migration + + +_PROCESS_GATE_MODULE = "tests.prism_postgres_a1_process_gate" +_C2_DIGEST = "11" * 32 + + +class _WorkerScopedPsqlLedger(PsqlShareLedger): + """Child-process ledger that consumes, but never owns, the parent schema.""" + + def __init__( + self, + *, + test_schema: str, + application_name: str, + **kwargs: object, + ) -> None: + if support.SCHEMA_PATTERN.fullmatch(test_schema) is None: + raise support.GateFailure(f"invalid worker schema: {test_schema!r}") + if ( + re.fullmatch(r"qbit_a1_c[23]_[a-z0-9_]+", application_name) is None + or len(application_name) > 63 + ): + raise support.GateFailure( + f"invalid process worker application name: {application_name!r}" + ) + self._worker_schema = test_schema + self._application_name = application_name + kwargs["psql_command"] = support.BASE_PSQL_COMMAND + kwargs["native_client_mode"] = "psql" + super().__init__(**kwargs) # type: ignore[arg-type] + + def _run_sql(self, sql: str) -> str: + return support.run_psql( + f"SET application_name = '{self._application_name}';\n" + sql, + schema=self._worker_schema, + ) + + +def _c2_report(*, block_height: int) -> dict[str, object]: + return { + "schema": "qbit.prism.audit-verification-report.v1", + "block_height": block_height, + "audit_bundle_sha256_hex": _C2_DIGEST, + "reward_manifest_sha256_hex": "44" * 32, + "payout_policy_manifest_sha256_hex": "55" * 32, + "prism_audit_commitment_leaf_hex": "66" * 32, + "audit_commitment_root_hex": "77" * 32, + "coinbase_txid": "22" * 32, + "coinbase_wtxid": "88" * 32, + "coinbase_manifest_sha256_hex": "33" * 32, + "coinbase_tx_hex": "00", + "coinbase_value_sats": 1, + "min_output_sats": 1, + "onchain_output_count": 0, + "accrued_account_count": 0, + } + + +def _c2_store(root: Path, evidence_path: Path) -> AuditArtifactStore: + return AuditArtifactStore( + AuditArtifactConfig( + root=root, + evidence_path=evidence_path, + live_bundle_retention=1, + candidate_retention_seconds=60, + share_segment_size=0, + ) + ) + + +def _c2_publish( + store: AuditArtifactStore, + *, + identity: AuditPublicationIdentity, + publication_floor_sequence: int, + created_at: str, +) -> object: + report = _c2_report(block_height=identity.block_height) + verification_identity = store.build_verification_identity( + trust_source="configured", + trusted_writer_public_key_hex="44" * 32, + literal_sha256=_C2_DIGEST, + literal_byte_len=123, + report=report, + ) + return store.publish_success( + identity=identity, + publication_floor_sequence=publication_floor_sequence, + report=report, + persistence={ + "audit_bundle_sha256": _C2_DIGEST, + "body_uri": "", + }, + evidence={ + "accepted_share_count": 0, + "distinct_miner_count": 0, + }, + verification_identity=verification_identity, + created_at=created_at, + ) + + +def _c2_authority(store: AuditArtifactStore) -> dict[str, list[int]]: + root_value = os.fstat(store._root_fd) + lock_value = os.fstat(store._publication_lock_fd) + return { + "root": [root_value.st_dev, root_value.st_ino], + "lock": [lock_value.st_dev, lock_value.st_ino], + } + + +def _c2_path_snapshot(path: Path) -> dict[str, object]: + try: + value = path.lstat() + except FileNotFoundError: + return {"exists": False} + payload: dict[str, object] = { + "exists": True, + "device": value.st_dev, + "inode": value.st_ino, + "mode": value.st_mode, + "size": value.st_size, + "mtime_ns": value.st_mtime_ns, + } + if path.is_file() and not path.is_symlink(): + payload["sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + return payload + + +def _c2_filesystem_snapshot( + *, + root: Path, + evidence_path: Path, +) -> dict[str, object]: + return { + "root": _c2_path_snapshot(root), + "entries": { + path.name: _c2_path_snapshot(path) + for path in sorted(root.iterdir(), key=lambda value: value.name) + }, + "evidence": _c2_path_snapshot(evidence_path), + } + + +def _c2_database_snapshot( + ledger: PsqlShareLedger, + *, + block_hashes: tuple[str, ...], +) -> dict[str, object]: + quoted = ", ".join(f"'{block_hash}'" for block_hash in block_hashes) + return ledger._run_json( + f""" +SELECT json_build_object( + 'rows', COALESCE(( + SELECT json_agg(json_build_object( + 'block_hash', block_hash, + 'block_height', block_height, + 'chain_state', chain_state, + 'maturity_state', maturity_state, + 'audit_publication_sequence', audit_publication_sequence + ) ORDER BY block_hash) + FROM qbit_pool_blocks + WHERE block_hash IN ({quoted}) + ), '[]'::json), + 'floor', COALESCE(( + SELECT MAX(audit_publication_sequence) + FROM qbit_pool_blocks + ), 0), + 'allocator', ( + SELECT json_build_object( + 'last_value', last_value, + 'is_called', is_called + ) + FROM qbit_audit_publication_sequence_seq + ) +); +""" + ) + + +def _emit_worker_event(event: str, **payload: object) -> None: + print( + json.dumps( + {"event": event, **payload}, + separators=(",", ":"), + sort_keys=True, + ), + flush=True, + ) + + +def _read_worker_command(expected: str) -> dict[str, object]: + line = sys.stdin.readline(support.PSQL_OUTPUT_LIMIT_BYTES + 2) + if not line: + raise support.GateFailure(f"worker command {expected!r} was not received") + if len(line.encode("utf-8")) > support.PSQL_OUTPUT_LIMIT_BYTES: + raise support.GateFailure("worker command exceeded 1 MiB") + try: + payload = json.loads(line) + except json.JSONDecodeError as error: + raise support.GateFailure("worker command is not JSON") from error + if not isinstance(payload, dict) or payload.get("command") != expected: + raise support.GateFailure( + f"expected worker command {expected!r}, got {payload!r}" + ) + return payload + + +class _JsonWorker: + def __init__(self, config_path: Path) -> None: + self._stderr = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") + self._captured = 0 + self._event_buffer = bytearray() + self.process = subprocess.Popen( + [ + sys.executable, + "-m", + _PROCESS_GATE_MODULE, + "--c2-worker", + str(config_path), + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=self._stderr, + text=True, + bufsize=1, + start_new_session=True, + ) + migration._register_process(self.process) + + def _stderr_text(self) -> str: + raw = os.pread( + self._stderr.fileno(), + support.PSQL_OUTPUT_LIMIT_BYTES + 1, + 0, + ) + if len(raw) > support.PSQL_OUTPUT_LIMIT_BYTES: + return "worker stderr exceeded 1 MiB" + return raw.decode("utf-8", errors="replace").strip() + + def read_event(self, expected: str) -> dict[str, object]: + if self.process.stdout is None: + raise support.GateFailure("JSON worker has no stdout") + deadline = time.monotonic() + support.PSQL_TIMEOUT_SECONDS + while time.monotonic() < deadline: + newline = self._event_buffer.find(b"\n") + if newline >= 0: + raw_line = bytes(self._event_buffer[:newline]) + del self._event_buffer[: newline + 1] + try: + line = raw_line.decode("utf-8") + except UnicodeDecodeError as error: + raise support.GateFailure( + "JSON worker emitted non-UTF-8 protocol bytes" + ) from error + try: + payload = json.loads(line) + except json.JSONDecodeError as error: + raise support.GateFailure( + f"JSON worker emitted invalid protocol line: {line!r}" + ) from error + if not isinstance(payload, dict): + raise support.GateFailure("JSON worker event is not an object") + if payload.get("event") == "error": + raise support.GateFailure( + f"JSON worker failed: {payload.get('message')}; " + f"stderr={self._stderr_text()}" + ) + if payload.get("event") != expected: + raise support.GateFailure( + f"expected JSON worker event {expected!r}, got {payload!r}" + ) + return payload + remaining = max(0.0, deadline - time.monotonic()) + ready, _, _ = select.select([self.process.stdout], [], [], remaining) + if not ready: + break + chunk = os.read(self.process.stdout.fileno(), 64 * 1024) + if not chunk: + break + self._captured += len(chunk) + if self._captured > support.PSQL_OUTPUT_LIMIT_BYTES: + raise support.GateFailure("JSON worker stdout exceeded 1 MiB") + self._event_buffer.extend(chunk) + if ( + b"\n" not in self._event_buffer + and len(self._event_buffer) > support.PSQL_OUTPUT_LIMIT_BYTES + ): + raise support.GateFailure( + "JSON worker event line exceeded 1 MiB" + ) + raise support.GateFailure( + f"JSON worker event {expected!r} timed out; " + f"exit={self.process.poll()} stderr={self._stderr_text()}" + ) + + def send(self, command: str) -> None: + if self.process.stdin is None or self.process.stdin.closed: + raise support.GateFailure("JSON worker has no writable stdin") + self.process.stdin.write( + json.dumps( + {"command": command}, + separators=(",", ":"), + sort_keys=True, + ) + + "\n" + ) + self.process.stdin.flush() + + def wait_success(self) -> None: + if self.process.stdin is not None and not self.process.stdin.closed: + self.process.stdin.close() + try: + self.process.wait(timeout=support.PSQL_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired as error: + support._terminate_and_reap(self.process) + raise support.GateFailure("JSON worker timed out during exit") from error + finally: + migration._forget_process(self.process) + residual = bytes(self._event_buffer) + self._event_buffer.clear() + if self.process.stdout is not None: + while True: + chunk = os.read(self.process.stdout.fileno(), 64 * 1024) + if not chunk: + break + residual += chunk + self._captured += len(chunk) + if self._captured > support.PSQL_OUTPUT_LIMIT_BYTES: + raise support.GateFailure("JSON worker stdout exceeded 1 MiB") + stderr = self._stderr_text() + if self.process.returncode != 0: + raise support.GateFailure( + f"JSON worker exited {self.process.returncode}: {stderr}" + ) + if residual.strip(): + raise support.GateFailure( + "JSON worker emitted unexpected residual output: " + + residual.decode("utf-8", errors="replace").strip() + ) + if stderr: + raise support.GateFailure( + f"JSON worker emitted unexpected stderr: {stderr}" + ) + + def close(self) -> None: + if self.process.poll() is None: + support._terminate_and_reap(self.process) + migration._forget_process(self.process) + for stream in (self.process.stdin, self.process.stdout, self.process.stderr): + if stream is not None and not stream.closed: + stream.close() + if not self._stderr.closed: + self._stderr.close() + + +def _c2_worker_ledger(config: dict[str, object]) -> _WorkerScopedPsqlLedger: + writer = config.get("writer") + if not isinstance(writer, dict): + raise support.GateFailure("process worker writer configuration is invalid") + return _WorkerScopedPsqlLedger( + test_schema=str(config["schema"]), + application_name=str(config["application_name"]), + writer_id=str(writer["id"]), + writer_epoch=int(writer["epoch"]), + writer_session_token=str(writer["token"]), + initialize_schema=False, + ) + + +def _c2_worker_store(config: dict[str, object]) -> AuditArtifactStore: + return _c2_store( + Path(str(config["root"])), + Path(str(config["evidence_path"])), + ) + + +def _c2_worker_identity( + config: dict[str, object], + key: str, + *, + sequence: int, +) -> AuditPublicationIdentity: + value = config.get(key) + if not isinstance(value, dict): + raise support.GateFailure(f"process worker {key} identity is invalid") + return AuditPublicationIdentity( + sequence, + int(value["height"]), + str(value["hash"]), + ) + + +def _run_c2_a_worker(config: dict[str, object]) -> None: + ledger: _WorkerScopedPsqlLedger | None = None + store: AuditArtifactStore | None = None + try: + ledger = _c2_worker_ledger(config) + store = _c2_worker_store(config) + initial = _c2_worker_identity( + config, + "initial", + sequence=int(config["initial_sequence"]), + ) + with store.publication_order_guard(): + floor = ledger.audit_publication_sequence_floor() + _emit_worker_event( + "guard-acquired", + authority=_c2_authority(store), + floor=floor, + pid=os.getpid(), + ) + _read_worker_command("repair") + publication = _c2_publish( + store, + identity=initial, + publication_floor_sequence=floor, + created_at=f"c2-{config['variant']}-a-repair", + ) + _emit_worker_event( + "repaired", + published=publication.published, # type: ignore[attr-defined] + identity=publication.identity.to_json(), # type: ignore[attr-defined] + latest=store.latest_evidence(), + authority=_c2_authority(store), + ) + _emit_worker_event("guard-released", authority=_c2_authority(store)) + _read_worker_command("stale-check") + stale_hashes = tuple( + str(config[key]["hash"]) # type: ignore[index] + for key in ("stale_confirmation", "stale_reactivation") + ) + before = _c2_database_snapshot(ledger, block_hashes=stale_hashes) + errors: dict[str, str] = {} + stale_confirmation = config["stale_confirmation"] + stale_reactivation = config["stale_reactivation"] + assert isinstance(stale_confirmation, dict) + assert isinstance(stale_reactivation, dict) + operations = ( + ( + "confirmation", + ledger.confirm_accepted_block, + stale_confirmation, + ), + ( + "reactivation", + ledger.reactivate_pool_block, + stale_reactivation, + ), + ) + for label, operation, target in operations: + try: + operation( + block_hash=str(target["hash"]), + active_tip_height=int(target["height"]), + ) + except Exception as error: + message = str(error) + if "writer lease is not active" not in message: + raise support.GateFailure( + f"stale A {label} wrong failure: {message}" + ) from error + errors[label] = message + else: + raise support.GateFailure( + f"stale A {label} unexpectedly succeeded" + ) + after = _c2_database_snapshot(ledger, block_hashes=stale_hashes) + _emit_worker_event( + "stale-result", + errors=errors, + before=before, + after=after, + authority=_c2_authority(store), + ) + _read_worker_command("finish") + store.close() + store = None + ledger.close() + ledger = None + _emit_worker_event("closed") + finally: + if store is not None: + store.close() + if ledger is not None: + ledger.close() + + +def _run_c2_b_worker(config: dict[str, object]) -> None: + ledger: _WorkerScopedPsqlLedger | None = None + store: AuditArtifactStore | None = None + original_flock = audit_artifacts_module.fcntl.flock + try: + ledger = _c2_worker_ledger(config) + store = _c2_worker_store(config) + writer = config["writer"] + assert isinstance(writer, dict) + ledger._run_sql( + f""" +INSERT INTO qbit_a1_c2_attempts (variant, transition, writer_id) +VALUES ( + '{config['variant']}', + '{config['transition']}', + '{writer['id']}' +); +""" + ) + attempted = False + + def observed_flock(fd: int, operation: int) -> object: + nonlocal attempted + if ( + not attempted + and fd == store._publication_lock_fd # type: ignore[union-attr] + and operation & fcntl.LOCK_EX + ): + attempted = True + _emit_worker_event( + "flock-attempt", + authority=_c2_authority(store), # type: ignore[arg-type] + pid=os.getpid(), + ) + return original_flock(fd, operation) + + audit_artifacts_module.fcntl.flock = observed_flock + try: + with store.publication_order_guard(): + if str(config["transition"]) == "confirmation": + target = config["target"] + assert isinstance(target, dict) + result = ledger.confirm_accepted_block( + block_hash=str(target["hash"]), + active_tip_height=int(target["height"]), + ) + count = int(result["confirmed_count"]) + elif str(config["transition"]) == "reactivation": + target = config["target"] + assert isinstance(target, dict) + result = ledger.reactivate_pool_block( + block_hash=str(target["hash"]), + active_tip_height=int(target["height"]), + ) + count = int(result["reactivated_count"]) + else: + raise support.GateFailure("invalid C2 B transition") + support.assert_equal(count, 1, "C2 B transition count") + sequence = int(result["audit_publication_sequence"]) + identity = _c2_worker_identity( + config, + "target", + sequence=sequence, + ) + floor = ledger.audit_publication_sequence_floor() + publication = _c2_publish( + store, + identity=identity, + publication_floor_sequence=floor, + created_at=f"c2-{config['variant']}-b-publication", + ) + _emit_worker_event( + "transition-published", + transition_result=result, + floor=floor, + published=publication.published, # type: ignore[attr-defined] + identity=identity.to_json(), + latest=store.latest_evidence(), + authority=_c2_authority(store), + ) + finally: + audit_artifacts_module.fcntl.flock = original_flock + if not attempted: + raise support.GateFailure("C2 B did not attempt the publication flock") + _read_worker_command("finish") + released = ledger.release_writer_lease() + support.assert_equal(released, True, "C2 B writer lease release") + store.close() + store = None + ledger.close() + ledger = None + _emit_worker_event("closed", lease_released=released) + finally: + audit_artifacts_module.fcntl.flock = original_flock + if store is not None: + store.close() + if ledger is not None: + try: + ledger.release_writer_lease() + except BaseException: + pass + ledger.close() + + +def _c3_stale_publication_attempt( + *, + ledger: _WorkerScopedPsqlLedger, + store: AuditArtifactStore, + config: dict[str, object], +) -> tuple[int, dict[str, object]]: + floor = ledger.audit_publication_sequence_floor() + identity = _c2_worker_identity( + config, + "initial", + sequence=int(config["initial_sequence"]), + ) + try: + publication = _c2_publish( + store, + identity=identity, + publication_floor_sequence=floor, + created_at=f"c3-{config['variant']}-late-a", + ) + except RuntimeError as error: + message = str(error) + if "behind" not in message: + raise support.GateFailure( + f"C3 stale A publication wrong failure: {message}" + ) from error + outcome = {"kind": "behind-error", "message": message} + else: + if publication.published: + raise support.GateFailure("C3 stale A publication regressed evidence") + outcome = { + "kind": "returned", + "published": publication.published, + "identity": publication.identity.to_json(), + } + return floor, outcome + + +def _c3_stale_transition_check( + ledger: _WorkerScopedPsqlLedger, + config: dict[str, object], +) -> dict[str, object]: + stale_hashes = tuple( + str(config[key]["hash"]) # type: ignore[index] + for key in ("stale_confirmation", "stale_reactivation") + ) + before = _c2_database_snapshot(ledger, block_hashes=stale_hashes) + errors: dict[str, str] = {} + stale_confirmation = config["stale_confirmation"] + stale_reactivation = config["stale_reactivation"] + assert isinstance(stale_confirmation, dict) + assert isinstance(stale_reactivation, dict) + operations = ( + ("confirmation", ledger.confirm_accepted_block, stale_confirmation), + ("reactivation", ledger.reactivate_pool_block, stale_reactivation), + ) + for label, operation, target in operations: + try: + operation( + block_hash=str(target["hash"]), + active_tip_height=int(target["height"]), + ) + except Exception as error: + message = str(error) + if "writer lease is not active" not in message: + raise support.GateFailure( + f"C3 stale A {label} wrong failure: {message}" + ) from error + errors[label] = message + else: + raise support.GateFailure(f"C3 stale A {label} unexpectedly succeeded") + after = _c2_database_snapshot(ledger, block_hashes=stale_hashes) + return {"errors": errors, "before": before, "after": after} + + +def _run_c3_primary_a_worker(config: dict[str, object]) -> None: + ledger: _WorkerScopedPsqlLedger | None = None + store: AuditArtifactStore | None = None + try: + ledger = _c2_worker_ledger(config) + store = _c2_worker_store(config) + _emit_worker_event( + "parked", + authority=_c2_authority(store), + pid=os.getpid(), + guard="outside", + ) + _read_worker_command("attempt") + with store.publication_order_guard(): + floor, outcome = _c3_stale_publication_attempt( + ledger=ledger, + store=store, + config=config, + ) + _emit_worker_event( + "stale-publication", + authority=_c2_authority(store), + floor=floor, + outcome=outcome, + latest=store.latest_evidence(), + ) + transition_check = _c3_stale_transition_check(ledger, config) + retention: list[dict[str, int]] = [] + for live_retention in (0, 1): + store.reconfigure(live_bundle_retention=live_retention) + result = store.prune_best_effort() + retention.append( + { + "retention": live_retention, + "live_removed": result.live_removed, + "candidate_removed": result.candidate_removed, + "errors": result.errors, + } + ) + _emit_worker_event( + "post-check", + authority=_c2_authority(store), + transition_check=transition_check, + retention=retention, + latest=store.latest_evidence(), + ) + _read_worker_command("finish") + store.close() + store = None + ledger.close() + ledger = None + _emit_worker_event("closed") + finally: + if store is not None: + store.close() + if ledger is not None: + ledger.close() + + +def _run_c3_late_a_worker(config: dict[str, object]) -> None: + ledger: _WorkerScopedPsqlLedger | None = None + store: AuditArtifactStore | None = None + try: + ledger = _c2_worker_ledger(config) + store = _c2_worker_store(config) + initial = config["initial"] + assert isinstance(initial, dict) + with store.publication_order_guard(): + result = ledger.confirm_accepted_block( + block_hash=str(initial["hash"]), + active_tip_height=int(initial["height"]), + ) + sequence = int(result["audit_publication_sequence"]) + support.assert_equal(sequence, 1, "C3 late A confirmation ordinal N") + floor = ledger.audit_publication_sequence_floor() + support.assert_equal(floor, sequence, "C3 late A confirmed floor N") + _emit_worker_event( + "confirmed", + authority=_c2_authority(store), + pid=os.getpid(), + identity=_c2_worker_identity( + config, + "initial", + sequence=sequence, + ).to_json(), + floor=floor, + ) + _emit_worker_event( + "parked", + authority=_c2_authority(store), + pid=os.getpid(), + guard="outside", + ) + _read_worker_command("attempt") + with store.publication_order_guard(): + floor, outcome = _c3_stale_publication_attempt( + ledger=ledger, + store=store, + config=config, + ) + _emit_worker_event( + "stale-publication", + authority=_c2_authority(store), + floor=floor, + outcome=outcome, + latest=store.latest_evidence(), + ) + _read_worker_command("finish") + store.close() + store = None + ledger.close() + ledger = None + _emit_worker_event("closed") + finally: + if store is not None: + store.close() + if ledger is not None: + ledger.close() + + +def _run_c3_b_worker(config: dict[str, object]) -> None: + ledger: _WorkerScopedPsqlLedger | None = None + store: AuditArtifactStore | None = None + original_flock = audit_artifacts_module.fcntl.flock + try: + ledger = _c2_worker_ledger(config) + store = _c2_worker_store(config) + writer = config["writer"] + assert isinstance(writer, dict) + ledger._run_sql( + f""" +INSERT INTO qbit_a1_c3_attempts (variant, writer_id) +VALUES ('{config['variant']}', '{writer['id']}'); +""" + ) + attempted = False + + def observed_flock(fd: int, operation: int) -> object: + nonlocal attempted + if ( + not attempted + and fd == store._publication_lock_fd # type: ignore[union-attr] + and operation & fcntl.LOCK_EX + ): + attempted = True + _emit_worker_event( + "flock-attempt", + authority=_c2_authority(store), # type: ignore[arg-type] + pid=os.getpid(), + ) + return original_flock(fd, operation) + + audit_artifacts_module.fcntl.flock = observed_flock + try: + with store.publication_order_guard(): + target = config["target"] + assert isinstance(target, dict) + result = ledger.confirm_accepted_block( + block_hash=str(target["hash"]), + active_tip_height=int(target["height"]), + ) + support.assert_equal( + result["confirmed_count"], + 1, + "C3 B confirmation count", + ) + sequence = int(result["audit_publication_sequence"]) + identity = _c2_worker_identity( + config, + "target", + sequence=sequence, + ) + floor = ledger.audit_publication_sequence_floor() + publication = _c2_publish( + store, + identity=identity, + publication_floor_sequence=floor, + created_at=f"c3-{config['variant']}-b-publication", + ) + _emit_worker_event( + "transition-published", + authority=_c2_authority(store), + transition_result=result, + floor=floor, + identity=identity.to_json(), + published=publication.published, + latest=store.latest_evidence(), + ) + finally: + audit_artifacts_module.fcntl.flock = original_flock + if not attempted: + raise support.GateFailure("C3 B did not attempt the publication flock") + _read_worker_command("finish") + released = ledger.release_writer_lease() + support.assert_equal(released, True, "C3 B writer lease release") + store.close() + store = None + ledger.close() + ledger = None + _emit_worker_event("closed", lease_released=released) + finally: + audit_artifacts_module.fcntl.flock = original_flock + if store is not None: + store.close() + if ledger is not None: + try: + ledger.release_writer_lease() + except BaseException: + pass + ledger.close() + + +def _run_c2_worker(config_path: Path) -> int: + try: + raw = config_path.read_bytes() + if len(raw) > support.PSQL_OUTPUT_LIMIT_BYTES: + raise support.GateFailure("process worker configuration exceeded 1 MiB") + config = json.loads(raw) + if not isinstance(config, dict): + raise support.GateFailure("process worker configuration is not an object") + role = config.get("role") + if role == "a": + _run_c2_a_worker(config) + elif role == "b": + _run_c2_b_worker(config) + elif role == "c3-primary-a": + _run_c3_primary_a_worker(config) + elif role == "c3-late-a": + _run_c3_late_a_worker(config) + elif role == "c3-b": + _run_c3_b_worker(config) + else: + raise support.GateFailure(f"invalid C2 worker role: {role!r}") + except BaseException as error: + _emit_worker_event( + "error", + error_type=type(error).__name__, + message=str(error), + ) + traceback.print_exc(file=sys.stderr) + return 1 + return 0 + + +def _start_file_worker( + sql: str, +) -> tuple[subprocess.Popen[str], Any, Any, Any]: + input_file = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") + stdout_file = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") + stderr_file = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") + input_file.write(sql) + input_file.seek(0) + process = subprocess.Popen( + migration._psql_process_command(), + stdin=input_file, + stdout=stdout_file, + stderr=stderr_file, + text=True, + start_new_session=True, + ) + migration._register_process(process) + return process, input_file, stdout_file, stderr_file + + +def _close_file_worker( + worker: tuple[subprocess.Popen[str], Any, Any, Any] | None, +) -> None: + if worker is None: + return + process, input_file, stdout_file, stderr_file = worker + if process.poll() is None: + support._terminate_and_reap(process) + migration._forget_process(process) + for stream in ( + process.stdin, + process.stdout, + process.stderr, + input_file, + stdout_file, + stderr_file, + ): + if stream is not None and not stream.closed: + stream.close() + + +def _wait_success( + worker: tuple[subprocess.Popen[str], Any, Any, Any], +) -> None: + process, _input_file, stdout_file, stderr_file = worker + migration._wait_file_process( + process, + stdout_file=stdout_file, + stderr_file=stderr_file, + expected_success=True, + error_fragment=None, + ) + + +def _wait_holder_success( + holder: subprocess.Popen[str], + *, + stderr_file: Any, + label: str, +) -> None: + try: + holder.wait(timeout=support.PSQL_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired as error: + support._terminate_and_reap(holder) + raise support.GateFailure(f"{label} timed out") from error + finally: + migration._forget_process(holder) + stdout = "" + if holder.stdout is not None: + stdout = holder.stdout.read(support.PSQL_OUTPUT_LIMIT_BYTES + 1) + stderr_file.seek(0) + stderr = stderr_file.read(support.PSQL_OUTPUT_LIMIT_BYTES + 1) + if ( + len(stdout.encode("utf-8")) > support.PSQL_OUTPUT_LIMIT_BYTES + or len(stderr.encode("utf-8")) > support.PSQL_OUTPUT_LIMIT_BYTES + ): + raise support.GateFailure(f"{label} output exceeded 1 MiB") + if holder.returncode != 0: + diagnostics = stderr.strip() or stdout.strip() or "no diagnostic output" + raise support.GateFailure( + f"{label} failed with {holder.returncode}: {diagnostics}" + ) + + +def _prepared_rows_sql(block_hashes: tuple[str, ...]) -> str: + values = ",\n".join( + ( + f"('{block_hash}', {height}, '{'10' * 32}', '{'20' * 32}', " + f"'{'30' * 32}', 'prepared', 'immature')" + ) + for height, block_hash in enumerate(block_hashes, start=10) + ) + return f""" +INSERT INTO qbit_pool_blocks ( + block_hash, block_height, parent_hash, coinbase_txid, + payout_manifest_sha256, chain_state, maturity_state +) VALUES +{values}; +""" + + +def _confirmation_worker_sql( + *, + schema: str, + application_name: str, + round_name: str, + block_hash: str, + height: int, + writer_id: str, + writer_epoch: int, + writer_token: str, + gate_a: int, + gate_b: int, +) -> str: + return f""" +SET application_name = '{application_name}'; +SET statement_timeout = '20s'; +SET lock_timeout = '20s'; +SET search_path TO "{schema}", pg_catalog; +BEGIN; +SELECT pg_catalog.pg_advisory_xact_lock_shared({gate_a}, {gate_b}); +DO $qbit_a1_confirm$ +DECLARE + confirmed_count integer; +BEGIN + SELECT "{schema}".qbit_confirm_pool_block( + '{block_hash}', {height}, '{writer_id}', {writer_epoch}, + '{writer_token}', interval '5 minutes' + ) + INTO confirmed_count; + IF confirmed_count <> 1 THEN + RAISE EXCEPTION 'expected one confirmed pool block, got %', confirmed_count; + END IF; +END; +$qbit_a1_confirm$; +INSERT INTO "{schema}".qbit_a1_confirmation_results ( + round_name, + block_hash, + audit_publication_sequence +) +SELECT + '{round_name}', + block_hash, + audit_publication_sequence +FROM "{schema}".qbit_pool_blocks +WHERE block_hash = '{block_hash}' + AND chain_state = 'confirmed' + AND audit_publication_sequence IS NOT NULL; +COMMIT; +""" + + +def _assert_no_tagged_backends(tags: tuple[str, ...], message: str) -> None: + quoted = ", ".join(f"'{tag}'" for tag in tags) + count = int( + support.run_json( + f""" +SELECT json_build_object('count', count(*)) +FROM pg_catalog.pg_stat_activity +WHERE application_name IN ({quoted}); +""" + )["count"] + ) + support.assert_equal(count, 0, message) + + +def test_database_observed_confirmation_order() -> None: + for round_index, reverse_launch in enumerate((False, True), start=1): + round_name = f"c1-round-{round_index}" + schema = support.create_owned_schema(f"c1_round_{round_index}") + writer_id = f"a1-c1-writer-{round_index}" + writer_token = f"a1-c1-token-{round_index}" + ledger = support.ScopedPsqlLedger( + test_schema=schema, + writer_id=writer_id, + writer_epoch=1, + writer_session_token=writer_token, + initialize_schema=True, + ) + block_hashes = ("91" * 32, "92" * 32) + heights = {block_hashes[0]: 10, block_hashes[1]: 11} + gate_a = 41_000 + round_index + gate_b = 51_000 + round_index + holder_tag = f"qbit_a1_c1_holder_{support.RUN_TOKEN}_{round_index}" + worker_tags = ( + f"qbit_a1_c1_worker_a_{support.RUN_TOKEN}_{round_index}", + f"qbit_a1_c1_worker_b_{support.RUN_TOKEN}_{round_index}", + ) + holder_input = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") + holder_stderr = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") + holder: subprocess.Popen[str] | None = None + workers: list[tuple[subprocess.Popen[str], Any, Any, Any]] = [] + try: + ledger._run_sql( + """ +CREATE TABLE qbit_a1_confirmation_results ( + completion_id bigserial PRIMARY KEY, + round_name text NOT NULL, + block_hash text NOT NULL UNIQUE, + audit_publication_sequence bigint NOT NULL +); +""" + + _prepared_rows_sql(block_hashes) + ) + holder_input.write( + f""" +SET application_name = '{holder_tag}'; +BEGIN; +SELECT pg_catalog.pg_advisory_xact_lock({gate_a}, {gate_b}); +DO $qbit_a1_wait$ +DECLARE + both_waiting boolean := false; +BEGIN + FOR attempt IN 1..600 LOOP + PERFORM pg_catalog.pg_stat_clear_snapshot(); + SELECT count(DISTINCT worker.pid) = 2 + INTO both_waiting + FROM pg_catalog.pg_stat_activity worker + WHERE worker.application_name IN ( + '{worker_tags[0]}', + '{worker_tags[1]}' + ) + AND pg_catalog.pg_backend_pid() = ANY( + pg_catalog.pg_blocking_pids(worker.pid) + ) + AND EXISTS ( + SELECT 1 + FROM pg_catalog.pg_locks lock_state + WHERE lock_state.pid = worker.pid + AND lock_state.locktype = 'advisory' + AND NOT lock_state.granted + ); + EXIT WHEN both_waiting; + PERFORM pg_catalog.pg_sleep(0.05); + END LOOP; + IF NOT both_waiting THEN + RAISE EXCEPTION 'two confirmation advisory waiters were not observed'; + END IF; +END; +$qbit_a1_wait$; +COMMIT; +""" + ) + holder_input.seek(0) + holder = subprocess.Popen( + migration._psql_process_command(), + stdin=holder_input, + stdout=subprocess.DEVNULL, + stderr=holder_stderr, + text=True, + bufsize=1, + start_new_session=True, + ) + migration._register_process(holder) + migration._wait_for_advisory_holder( + holder, + application_name=holder_tag, + deadline=time.monotonic() + support.PSQL_TIMEOUT_SECONDS, + ) + launch = [0, 1] + if reverse_launch: + launch.reverse() + for worker_index in launch: + block_hash = block_hashes[worker_index] + workers.append( + _start_file_worker( + _confirmation_worker_sql( + schema=schema, + application_name=worker_tags[worker_index], + round_name=round_name, + block_hash=block_hash, + height=heights[block_hash], + writer_id=writer_id, + writer_epoch=1, + writer_token=writer_token, + gate_a=gate_a, + gate_b=gate_b, + ) + ) + ) + _wait_holder_success( + holder, + stderr_file=holder_stderr, + label="C1 holder", + ) + for worker in workers: + _wait_success(worker) + results = ledger._run_json( + f""" +SELECT json_build_object( + 'rows', COALESCE(json_agg(json_build_object( + 'completion_id', completion_id, + 'block_hash', block_hash, + 'audit_publication_sequence', audit_publication_sequence + ) ORDER BY completion_id), '[]'::json) +) +FROM qbit_a1_confirmation_results +WHERE round_name = '{round_name}'; +""" + )["rows"] + support.assert_equal(len(results), 2, f"{round_name} completion count") + support.assert_equal( + [row["completion_id"] for row in results], + [1, 2], + f"{round_name} exact completion identifiers", + ) + support.assert_equal( + [row["audit_publication_sequence"] for row in results], + [1, 2], + f"{round_name} completion order matches ordinal order", + ) + support.assert_equal( + {row["block_hash"] for row in results}, + set(block_hashes), + f"{round_name} distinct completed blocks", + ) + before_replay = support.allocator_state(ledger) + for _replay_round in range(3): + for block_hash in block_hashes: + state = ledger.pool_block_state(block_hash=block_hash) + replay = ledger.confirm_accepted_block( + block_hash=block_hash, + active_tip_height=heights[block_hash], + ) + support.assert_equal( + replay["audit_publication_sequence"], + state["audit_publication_sequence"], # type: ignore[index] + f"{round_name} exact replay ordinal", + ) + support.assert_equal( + support.allocator_state(ledger), + before_replay, + f"{round_name} replay allocator immobility", + ) + fresh_hash = "93" * 32 + ledger._run_sql(_prepared_rows_sql((fresh_hash,))) + support.assert_equal( + ledger.confirm_accepted_block( + block_hash=fresh_hash, + active_tip_height=10, + )["audit_publication_sequence"], + 3, + f"{round_name} exact next ordinal", + ) + _assert_no_tagged_backends( + (holder_tag, *worker_tags), + f"{round_name} tagged backend cleanup", + ) + finally: + for worker in workers: + _close_file_worker(worker) + if holder is not None: + if holder.poll() is None: + support._terminate_and_reap(holder) + migration._forget_process(holder) + for stream in (holder.stdin, holder.stdout, holder.stderr): + if stream is not None and not stream.closed: + stream.close() + if not holder_stderr.closed: + holder_stderr.close() + if not holder_input.closed: + holder_input.close() + ledger.release_writer_lease() + ledger.close() + + +def _c2_seed_rows( + ledger: support.ScopedPsqlLedger, + *, + initial: dict[str, object], + target: dict[str, object], + stale_confirmation: dict[str, object], + stale_reactivation: dict[str, object], +) -> None: + rows = ( + (initial, "prepared"), + (target, "prepared"), + (stale_confirmation, "prepared"), + (stale_reactivation, "inactive"), + ) + values = ",\n".join( + ( + f"('{row['hash']}', {int(row['height'])}, '{'10' * 32}', " + f"'{'20' * 32}', '{'30' * 32}', '{chain_state}', 'immature')" + ) + for row, chain_state in rows + ) + ledger._run_sql( + f""" +CREATE TABLE qbit_a1_c2_attempts ( + attempt_id bigserial PRIMARY KEY, + variant text NOT NULL, + transition text NOT NULL, + writer_id text NOT NULL, + attempted_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +INSERT INTO qbit_pool_blocks ( + block_hash, + block_height, + parent_hash, + coinbase_txid, + payout_manifest_sha256, + chain_state, + maturity_state +) VALUES +{values}; +""" + ) + + +def _c2_attempt_and_lease( + ledger: support.ScopedPsqlLedger, + *, + variant: str, +) -> dict[str, object]: + return ledger._run_json( + f""" +SELECT json_build_object( + 'attempts', COALESCE(( + SELECT json_agg(json_build_object( + 'variant', variant, + 'transition', transition, + 'writer_id', writer_id + ) ORDER BY attempt_id) + FROM qbit_a1_c2_attempts + WHERE variant = '{variant}' + ), '[]'::json), + 'lease', ( + SELECT json_build_object( + 'writer_id', writer_id, + 'writer_epoch', writer_epoch, + 'writer_session_token', writer_session_token + ) + FROM qbit_ledger_writer_lease + WHERE singleton + ) +); +""" + ) + + +def _c2_expire_writer( + ledger: support.ScopedPsqlLedger, + *, + writer_id: str, + writer_epoch: int, + writer_token: str, +) -> None: + expired = ledger._run_json( + f""" +WITH expired AS ( + UPDATE qbit_ledger_writer_lease + SET updated_at = clock_timestamp() - interval '6 minutes', + lease_expires_at = clock_timestamp() - interval '1 minute' + WHERE singleton + AND writer_id = '{writer_id}' + AND writer_epoch = {writer_epoch} + AND writer_session_token = '{writer_token}' + RETURNING writer_id +) +SELECT json_build_object('count', count(*)) +FROM expired; +""" + )["count"] + support.assert_equal(expired, 1, "C2 exact A lease expiry") + + +def _c2_write_config(path: Path, config: dict[str, object]) -> None: + path.write_text( + json.dumps(config, separators=(",", ":"), sort_keys=True), + encoding="utf-8", + ) + + +def _c2_expected_authority(root: Path) -> dict[str, list[int]]: + root_value = root.lstat() + lock_value = (root / ".prism-audit-publication.lock").lstat() + return { + "root": [root_value.st_dev, root_value.st_ino], + "lock": [lock_value.st_dev, lock_value.st_ino], + } + + +def _assert_c2_final_database( + snapshot: dict[str, object], + *, + initial: dict[str, object], + target: dict[str, object], + stale_confirmation: dict[str, object], + stale_reactivation: dict[str, object], + initial_sequence: int, + target_sequence: int, + variant: str, +) -> None: + rows = snapshot["rows"] + assert isinstance(rows, list) + by_hash = {str(row["block_hash"]): row for row in rows} + support.assert_equal( + by_hash[str(initial["hash"])], + { + "block_hash": initial["hash"], + "block_height": initial["height"], + "chain_state": "confirmed", + "maturity_state": "immature", + "audit_publication_sequence": initial_sequence, + }, + f"C2 {variant} initial exact row", + ) + support.assert_equal( + by_hash[str(target["hash"])], + { + "block_hash": target["hash"], + "block_height": target["height"], + "chain_state": "confirmed", + "maturity_state": "immature", + "audit_publication_sequence": target_sequence, + }, + f"C2 {variant} B exact row", + ) + support.assert_equal( + by_hash[str(stale_confirmation["hash"])], + { + "block_hash": stale_confirmation["hash"], + "block_height": stale_confirmation["height"], + "chain_state": "prepared", + "maturity_state": "immature", + "audit_publication_sequence": None, + }, + f"C2 {variant} stale prepared C row", + ) + support.assert_equal( + by_hash[str(stale_reactivation["hash"])], + { + "block_hash": stale_reactivation["hash"], + "block_height": stale_reactivation["height"], + "chain_state": "inactive", + "maturity_state": "immature", + "audit_publication_sequence": None, + }, + f"C2 {variant} stale inactive R row", + ) + support.assert_equal( + snapshot["floor"], + target_sequence, + f"C2 {variant} final floor", + ) + support.assert_equal( + snapshot["allocator"], + {"last_value": target_sequence, "is_called": True}, + f"C2 {variant} exact allocator", + ) + ordinals = sorted( + int(row["audit_publication_sequence"]) + for row in rows + if row["audit_publication_sequence"] is not None + ) + support.assert_equal( + len(ordinals), + len(set(ordinals)), + f"C2 {variant} unique durable ordinals", + ) + + +def _test_c2_a_wins(transition: str) -> None: + if transition not in {"confirmation", "reactivation"}: + raise support.GateFailure(f"invalid C2 transition: {transition!r}") + variant_index = 1 if transition == "confirmation" else 2 + variant = transition + schema = support.create_owned_schema(f"c2_{variant}") + writer_a = { + "id": f"a1-c2-{variant}-writer-a", + "epoch": 1, + "token": f"a1-c2-{variant}-token-a", + } + writer_b = { + "id": f"a1-c2-{variant}-writer-b", + "epoch": 2, + "token": f"a1-c2-{variant}-token-b", + } + initial = {"hash": f"a{variant_index}" * 32, "height": 10} + target = {"hash": f"b{variant_index}" * 32, "height": 20} + stale_confirmation = {"hash": f"c{variant_index}" * 32, "height": 30} + stale_reactivation = {"hash": f"d{variant_index}" * 32, "height": 31} + all_hashes = tuple( + str(value["hash"]) + for value in ( + initial, + target, + stale_confirmation, + stale_reactivation, + ) + ) + ledger = support.ScopedPsqlLedger( + test_schema=schema, + writer_id=str(writer_a["id"]), + writer_epoch=int(writer_a["epoch"]), + writer_session_token=str(writer_a["token"]), + initialize_schema=True, + ) + worker_a: _JsonWorker | None = None + worker_b: _JsonWorker | None = None + setup_store: AuditArtifactStore | None = None + try: + _c2_seed_rows( + ledger, + initial=initial, + target=target, + stale_confirmation=stale_confirmation, + stale_reactivation=stale_reactivation, + ) + initial_sequence = 1 + if transition == "reactivation": + target_prior = ledger.confirm_accepted_block( + block_hash=str(target["hash"]), + active_tip_height=int(target["height"]), + ) + support.assert_equal( + target_prior["audit_publication_sequence"], + 1, + "C2 reactivation target prior confirmation ordinal", + ) + support.assert_equal( + ledger.mark_pool_block_inactive( + block_hash=str(target["hash"]), + active_tip_height=int(target["height"]), + )["inactive_count"], + 1, + "C2 reactivation target prior inactive transition", + ) + initial_sequence = 2 + initial_result = ledger.confirm_accepted_block( + block_hash=str(initial["hash"]), + active_tip_height=int(initial["height"]), + ) + support.assert_equal( + initial_result["audit_publication_sequence"], + initial_sequence, + f"C2 {variant} initial ordinal", + ) + support.assert_equal( + ledger.audit_publication_sequence_floor(), + initial_sequence, + f"C2 {variant} initial floor", + ) + support.assert_equal( + support.allocator_state(ledger), + {"last_value": initial_sequence, "is_called": True}, + f"C2 {variant} initial allocator", + ) + target_sequence = initial_sequence + 1 + with tempfile.TemporaryDirectory() as tmp, ExitStack() as temp_cleanup: + base = Path(tmp) + root = base / "audit" + evidence_path = base / "state" / "evidence.json" + setup_store = _c2_store(root, evidence_path) + temp_cleanup.callback(setup_store.close) + initial_identity = AuditPublicationIdentity( + initial_sequence, + int(initial["height"]), + str(initial["hash"]), + ) + with setup_store.publication_order_guard(): + initial_publication = _c2_publish( + setup_store, + identity=initial_identity, + publication_floor_sequence=initial_sequence, + created_at=f"c2-{variant}-initial", + ) + support.assert_equal( + initial_publication.published, # type: ignore[attr-defined] + True, + f"C2 {variant} initial publication", + ) + initial_envelope = setup_store.live_envelope_path( + block_height=int(initial["height"]), + block_hash=str(initial["hash"]), + ) + setup_store.close() + setup_store = None + evidence_path.write_bytes(b"{c2-damaged-evidence") + expected_authority = _c2_expected_authority(root) + common_config: dict[str, object] = { + "schema": schema, + "root": str(root), + "evidence_path": str(evidence_path), + "variant": variant, + "transition": transition, + "initial_sequence": initial_sequence, + "initial": initial, + "target": target, + "stale_confirmation": stale_confirmation, + "stale_reactivation": stale_reactivation, + } + config_a = base / "worker-a.json" + config_b = base / "worker-b.json" + tag_suffix = f"{variant[0]}_{support.RUN_TOKEN[:8]}" + tag_a = f"qbit_a1_c2_a_{tag_suffix}" + tag_b = f"qbit_a1_c2_b_{tag_suffix}" + _c2_write_config( + config_a, + { + **common_config, + "role": "a", + "writer": writer_a, + "application_name": tag_a, + }, + ) + _c2_write_config( + config_b, + { + **common_config, + "role": "b", + "writer": writer_b, + "application_name": tag_b, + }, + ) + worker_a = _JsonWorker(config_a) + temp_cleanup.callback(worker_a.close) + a_guard = worker_a.read_event("guard-acquired") + support.assert_equal( + a_guard["authority"], + expected_authority, + f"C2 {variant} A exact root/lock inode", + ) + support.assert_equal( + a_guard["floor"], + initial_sequence, + f"C2 {variant} A floor N", + ) + blocked_filesystem = _c2_filesystem_snapshot( + root=root, + evidence_path=evidence_path, + ) + blocked_database = _c2_database_snapshot( + ledger, + block_hashes=all_hashes, + ) + _c2_expire_writer( + ledger, + writer_id=str(writer_a["id"]), + writer_epoch=int(writer_a["epoch"]), + writer_token=str(writer_a["token"]), + ) + worker_b = _JsonWorker(config_b) + temp_cleanup.callback(worker_b.close) + b_attempt = worker_b.read_event("flock-attempt") + support.assert_equal( + b_attempt["authority"], + expected_authority, + f"C2 {variant} B exact root/lock inode", + ) + support.assert_equal( + worker_b.process.poll(), + None, + f"C2 {variant} B remains alive while flock-blocked", + ) + child_pids = {int(a_guard["pid"]), int(b_attempt["pid"])} + support.assert_equal( + len(child_pids), + 2, + f"C2 {variant} distinct A/B OS processes", + ) + if os.getpid() in child_pids: + raise support.GateFailure( + f"C2 {variant} worker reused the parent process" + ) + support.assert_equal( + _c2_attempt_and_lease(ledger, variant=variant), + { + "attempts": [ + { + "variant": variant, + "transition": transition, + "writer_id": writer_b["id"], + } + ], + "lease": { + "writer_id": writer_b["id"], + "writer_epoch": writer_b["epoch"], + "writer_session_token": writer_b["token"], + }, + }, + f"C2 {variant} DB-visible B attempt and replacement lease", + ) + support.assert_equal( + _c2_database_snapshot(ledger, block_hashes=all_hashes), + blocked_database, + f"C2 {variant} B row/floor/allocator unchanged while blocked", + ) + support.assert_equal( + _c2_filesystem_snapshot(root=root, evidence_path=evidence_path), + blocked_filesystem, + f"C2 {variant} filesystem unchanged while B blocked", + ) + worker_a.send("repair") + repaired = worker_a.read_event("repaired") + support.assert_equal( + repaired["published"], + True, + f"C2 {variant} A exact repair published", + ) + support.assert_equal( + repaired["identity"], + initial_identity.to_json(), + f"C2 {variant} A repair identity N", + ) + support.assert_equal( + repaired["authority"], + expected_authority, + f"C2 {variant} A repair authority", + ) + released = worker_a.read_event("guard-released") + support.assert_equal( + released["authority"], + expected_authority, + f"C2 {variant} A guard release authority", + ) + b_done = worker_b.read_event("transition-published") + expected_b_identity = AuditPublicationIdentity( + target_sequence, + int(target["height"]), + str(target["hash"]), + ) + support.assert_equal( + b_done["identity"], + expected_b_identity.to_json(), + f"C2 {variant} B identity N+1", + ) + support.assert_equal( + b_done["floor"], + target_sequence, + f"C2 {variant} B fresh floor", + ) + support.assert_equal( + b_done["published"], + True, + f"C2 {variant} B publication", + ) + support.assert_equal( + b_done["authority"], + expected_authority, + f"C2 {variant} B publication authority", + ) + final_database = _c2_database_snapshot( + ledger, + block_hashes=all_hashes, + ) + _assert_c2_final_database( + final_database, + initial=initial, + target=target, + stale_confirmation=stale_confirmation, + stale_reactivation=stale_reactivation, + initial_sequence=initial_sequence, + target_sequence=target_sequence, + variant=variant, + ) + latest = json.loads(evidence_path.read_text(encoding="utf-8")) + support.assert_equal( + latest["audit_publication_identity"], + expected_b_identity.to_json(), + f"C2 {variant} final evidence B/N+1", + ) + support.assert_equal( + latest["block_hash"], + target["hash"], + f"C2 {variant} final evidence B hash", + ) + target_envelope = root / ( + f"prism-live-audit-bundle-{target['height']}-{target['hash']}.json" + ) + support.assert_equal( + initial_envelope.exists(), + False, + f"C2 {variant} retention removes unpinned A envelope", + ) + support.assert_equal( + target_envelope.exists(), + True, + f"C2 {variant} retention keeps B envelope", + ) + support.assert_equal( + sorted(path.name for path in root.iterdir()), + [ + ".prism-audit-publication.lock", + target_envelope.name, + ], + f"C2 {variant} exact retained root entries", + ) + before_stale_database = _c2_database_snapshot( + ledger, + block_hashes=all_hashes, + ) + before_stale_filesystem = _c2_filesystem_snapshot( + root=root, + evidence_path=evidence_path, + ) + worker_a.send("stale-check") + stale_result = worker_a.read_event("stale-result") + stale_errors = stale_result["errors"] + assert isinstance(stale_errors, dict) + support.assert_equal( + set(stale_errors), + {"confirmation", "reactivation"}, + f"C2 {variant} exact stale A failure labels", + ) + for label in ("confirmation", "reactivation"): + if "writer lease is not active" not in str(stale_errors[label]): + raise support.GateFailure( + f"C2 {variant} stale A {label} wrong error: " + f"{stale_errors[label]!r}" + ) + support.assert_equal( + stale_result["before"], + stale_result["after"], + f"C2 {variant} child-observed stale transition immobility", + ) + support.assert_equal( + stale_result["authority"], + expected_authority, + f"C2 {variant} stale A authority", + ) + support.assert_equal( + _c2_database_snapshot(ledger, block_hashes=all_hashes), + before_stale_database, + f"C2 {variant} stale A row/floor/allocator immobility", + ) + support.assert_equal( + _c2_filesystem_snapshot(root=root, evidence_path=evidence_path), + before_stale_filesystem, + f"C2 {variant} stale A filesystem immobility", + ) + worker_a.send("finish") + worker_a.read_event("closed") + worker_a.wait_success() + worker_b.send("finish") + b_closed = worker_b.read_event("closed") + support.assert_equal( + b_closed["lease_released"], + True, + f"C2 {variant} B lease released", + ) + worker_b.wait_success() + _assert_no_tagged_backends( + (tag_a, tag_b), + f"C2 {variant} tagged child backend cleanup", + ) + finally: + if setup_store is not None: + setup_store.close() + if worker_a is not None: + worker_a.close() + if worker_b is not None: + worker_b.close() + ledger.close() + + +def test_c2_a_wins_confirmation_publication() -> None: + _test_c2_a_wins("confirmation") + + +def _c3_seed_rows( + ledger: support.ScopedPsqlLedger, + *, + initial: dict[str, object], + target: dict[str, object], + stale_confirmation: dict[str, object], + stale_reactivation: dict[str, object], +) -> None: + rows = ( + (initial, "prepared"), + (target, "prepared"), + (stale_confirmation, "prepared"), + (stale_reactivation, "inactive"), + ) + values = ",\n".join( + ( + f"('{row['hash']}', {int(row['height'])}, '{'10' * 32}', " + f"'{'20' * 32}', '{'30' * 32}', '{chain_state}', 'immature')" + ) + for row, chain_state in rows + ) + ledger._run_sql( + f""" +CREATE TABLE qbit_a1_c3_attempts ( + attempt_id bigserial PRIMARY KEY, + variant text NOT NULL, + writer_id text NOT NULL, + attempted_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +INSERT INTO qbit_pool_blocks ( + block_hash, + block_height, + parent_hash, + coinbase_txid, + payout_manifest_sha256, + chain_state, + maturity_state +) VALUES +{values}; +""" + ) + + +def _c3_attempt_and_lease( + ledger: support.ScopedPsqlLedger, + *, + variant: str, +) -> dict[str, object]: + return ledger._run_json( + f""" +SELECT json_build_object( + 'attempts', COALESCE(( + SELECT json_agg(json_build_object( + 'variant', variant, + 'writer_id', writer_id + ) ORDER BY attempt_id) + FROM qbit_a1_c3_attempts + WHERE variant = '{variant}' + ), '[]'::json), + 'lease', ( + SELECT json_build_object( + 'writer_id', writer_id, + 'writer_epoch', writer_epoch, + 'writer_session_token', writer_session_token + ) + FROM qbit_ledger_writer_lease + WHERE singleton + ) +); +""" + ) + + +def _c3_expected_database( + *, + initial: dict[str, object], + target: dict[str, object], + stale_confirmation: dict[str, object], + stale_reactivation: dict[str, object], +) -> dict[str, object]: + return { + "rows": [ + { + "block_hash": initial["hash"], + "block_height": initial["height"], + "chain_state": "confirmed", + "maturity_state": "immature", + "audit_publication_sequence": 1, + }, + { + "block_hash": target["hash"], + "block_height": target["height"], + "chain_state": "confirmed", + "maturity_state": "immature", + "audit_publication_sequence": 2, + }, + { + "block_hash": stale_confirmation["hash"], + "block_height": stale_confirmation["height"], + "chain_state": "prepared", + "maturity_state": "immature", + "audit_publication_sequence": None, + }, + { + "block_hash": stale_reactivation["hash"], + "block_height": stale_reactivation["height"], + "chain_state": "inactive", + "maturity_state": "immature", + "audit_publication_sequence": None, + }, + ], + "floor": 2, + "allocator": {"last_value": 2, "is_called": True}, + } + + +def _assert_c3_stale_outcome(outcome: object, *, label: str) -> None: + if not isinstance(outcome, dict): + raise support.GateFailure(f"{label} outcome is not an object") + if outcome.get("kind") == "returned": + support.assert_equal( + outcome.get("published"), + False, + f"{label} returns published false", + ) + return + if outcome.get("kind") == "behind-error" and "behind" in str( + outcome.get("message") + ): + return + raise support.GateFailure(f"{label} unexpected outcome: {outcome!r}") + + +def _test_c3_interleaving(variant: str) -> None: + if variant not in {"primary", "late"}: + raise support.GateFailure(f"invalid C3 variant: {variant!r}") + variant_index = 3 if variant == "primary" else 4 + schema = support.create_owned_schema(f"c3_{variant}") + writer_a = { + "id": f"a1-c3-{variant}-writer-a", + "epoch": 1, + "token": f"a1-c3-{variant}-token-a", + } + writer_b = { + "id": f"a1-c3-{variant}-writer-b", + "epoch": 2, + "token": f"a1-c3-{variant}-token-b", + } + initial = {"hash": f"a{variant_index}" * 32, "height": 10} + target = {"hash": f"b{variant_index}" * 32, "height": 20} + stale_confirmation = {"hash": f"c{variant_index}" * 32, "height": 30} + stale_reactivation = {"hash": f"d{variant_index}" * 32, "height": 31} + all_hashes = tuple( + str(value["hash"]) + for value in ( + initial, + target, + stale_confirmation, + stale_reactivation, + ) + ) + ledger = support.ScopedPsqlLedger( + test_schema=schema, + writer_id=str(writer_a["id"]), + writer_epoch=int(writer_a["epoch"]), + writer_session_token=str(writer_a["token"]), + initialize_schema=True, + ) + worker_a: _JsonWorker | None = None + worker_b: _JsonWorker | None = None + setup_store: AuditArtifactStore | None = None + try: + _c3_seed_rows( + ledger, + initial=initial, + target=target, + stale_confirmation=stale_confirmation, + stale_reactivation=stale_reactivation, + ) + if variant == "primary": + initial_result = ledger.confirm_accepted_block( + block_hash=str(initial["hash"]), + active_tip_height=int(initial["height"]), + ) + support.assert_equal( + initial_result["audit_publication_sequence"], + 1, + "C3 primary initial ordinal N", + ) + with tempfile.TemporaryDirectory() as tmp, ExitStack() as temp_cleanup: + base = Path(tmp) + root = base / "audit" + evidence_path = base / "state" / "evidence.json" + setup_store = _c2_store(root, evidence_path) + temp_cleanup.callback(setup_store.close) + initial_identity = AuditPublicationIdentity( + 1, + int(initial["height"]), + str(initial["hash"]), + ) + initial_envelope = setup_store.live_envelope_path( + block_height=int(initial["height"]), + block_hash=str(initial["hash"]), + ) + if variant == "primary": + with setup_store.publication_order_guard(): + initial_publication = _c2_publish( + setup_store, + identity=initial_identity, + publication_floor_sequence=1, + created_at="c3-primary-initial", + ) + support.assert_equal( + initial_publication.published, + True, + "C3 primary initial publication N", + ) + setup_store.close() + setup_store = None + if variant == "primary": + initial_envelope.unlink() + support.assert_equal( + evidence_path.exists(), + True, + "C3 primary damaged N retains evidence", + ) + support.assert_equal( + initial_envelope.exists(), + False, + f"C3 {variant} N envelope starts absent", + ) + expected_authority = _c2_expected_authority(root) + tag_suffix = f"{variant[0]}_{support.RUN_TOKEN[:8]}" + tag_a = f"qbit_a1_c3_a_{tag_suffix}" + tag_b = f"qbit_a1_c3_b_{tag_suffix}" + common_config: dict[str, object] = { + "schema": schema, + "root": str(root), + "evidence_path": str(evidence_path), + "variant": variant, + "transition": "confirmation", + "initial_sequence": 1, + "initial": initial, + "target": target, + "stale_confirmation": stale_confirmation, + "stale_reactivation": stale_reactivation, + } + config_a = base / "worker-a.json" + config_b = base / "worker-b.json" + _c2_write_config( + config_a, + { + **common_config, + "role": ( + "c3-primary-a" if variant == "primary" else "c3-late-a" + ), + "writer": writer_a, + "application_name": tag_a, + }, + ) + _c2_write_config( + config_b, + { + **common_config, + "role": "c3-b", + "writer": writer_b, + "application_name": tag_b, + }, + ) + worker_a = _JsonWorker(config_a) + temp_cleanup.callback(worker_a.close) + if variant == "late": + confirmed = worker_a.read_event("confirmed") + support.assert_equal( + confirmed["authority"], + expected_authority, + "C3 late A confirmation guard authority", + ) + support.assert_equal( + confirmed["identity"], + initial_identity.to_json(), + "C3 late A confirmed identity N", + ) + support.assert_equal( + confirmed["floor"], + 1, + "C3 late A confirmed floor N", + ) + parked = worker_a.read_event("parked") + support.assert_equal( + parked["authority"], + expected_authority, + f"C3 {variant} A exact root/lock inode", + ) + support.assert_equal( + parked["guard"], + "outside", + f"C3 {variant} A parks outside guard", + ) + if variant == "late": + support.assert_equal( + parked["pid"], + confirmed["pid"], + "C3 late A exits guard in same child", + ) + support.assert_equal( + worker_a.process.poll(), + None, + f"C3 {variant} parked A remains alive", + ) + initial_database = _c2_database_snapshot( + ledger, + block_hashes=all_hashes, + ) + support.assert_equal( + initial_database["floor"], + 1, + f"C3 {variant} durable floor N before replacement", + ) + support.assert_equal( + initial_database["allocator"], + {"last_value": 1, "is_called": True}, + f"C3 {variant} allocator N before replacement", + ) + support.assert_equal( + initial_database["rows"], + [ + { + "block_hash": initial["hash"], + "block_height": initial["height"], + "chain_state": "confirmed", + "maturity_state": "immature", + "audit_publication_sequence": 1, + }, + { + "block_hash": target["hash"], + "block_height": target["height"], + "chain_state": "prepared", + "maturity_state": "immature", + "audit_publication_sequence": None, + }, + { + "block_hash": stale_confirmation["hash"], + "block_height": stale_confirmation["height"], + "chain_state": "prepared", + "maturity_state": "immature", + "audit_publication_sequence": None, + }, + { + "block_hash": stale_reactivation["hash"], + "block_height": stale_reactivation["height"], + "chain_state": "inactive", + "maturity_state": "immature", + "audit_publication_sequence": None, + }, + ], + f"C3 {variant} exact rows before replacement", + ) + damaged_snapshot = _c2_filesystem_snapshot( + root=root, + evidence_path=evidence_path, + ) + if variant == "primary": + damaged_evidence = json.loads( + evidence_path.read_text(encoding="utf-8") + ) + support.assert_equal( + damaged_evidence["audit_publication_identity"], + initial_identity.to_json(), + "C3 primary damaged evidence still names N", + ) + _c2_expire_writer( + ledger, + writer_id=str(writer_a["id"]), + writer_epoch=int(writer_a["epoch"]), + writer_token=str(writer_a["token"]), + ) + worker_b = _JsonWorker(config_b) + temp_cleanup.callback(worker_b.close) + b_attempt = worker_b.read_event("flock-attempt") + support.assert_equal( + b_attempt["authority"], + expected_authority, + f"C3 {variant} B exact root/lock inode", + ) + child_pids = {int(parked["pid"]), int(b_attempt["pid"])} + support.assert_equal( + len(child_pids), + 2, + f"C3 {variant} distinct A/B OS processes", + ) + if os.getpid() in child_pids: + raise support.GateFailure( + f"C3 {variant} worker reused the parent process" + ) + b_done = worker_b.read_event("transition-published") + expected_b_identity = AuditPublicationIdentity( + 2, + int(target["height"]), + str(target["hash"]), + ) + support.assert_equal( + b_done["authority"], + expected_authority, + f"C3 {variant} B publication authority", + ) + support.assert_equal( + b_done["identity"], + expected_b_identity.to_json(), + f"C3 {variant} B identity N+1", + ) + support.assert_equal(b_done["floor"], 2, f"C3 {variant} B floor N+1") + support.assert_equal( + b_done["published"], + True, + f"C3 {variant} B publishes N+1", + ) + support.assert_equal( + _c3_attempt_and_lease(ledger, variant=variant), + { + "attempts": [ + {"variant": variant, "writer_id": writer_b["id"]} + ], + "lease": { + "writer_id": writer_b["id"], + "writer_epoch": writer_b["epoch"], + "writer_session_token": writer_b["token"], + }, + }, + f"C3 {variant} B attempt and active replacement lease", + ) + expected_database = _c3_expected_database( + initial=initial, + target=target, + stale_confirmation=stale_confirmation, + stale_reactivation=stale_reactivation, + ) + support.assert_equal( + _c2_database_snapshot(ledger, block_hashes=all_hashes), + expected_database, + f"C3 {variant} exact database after B", + ) + target_envelope = root / ( + f"prism-live-audit-bundle-{target['height']}-{target['hash']}.json" + ) + b_evidence_bytes = evidence_path.read_bytes() + b_envelope_bytes = target_envelope.read_bytes() + durable_b_evidence = json.loads(b_evidence_bytes) + durable_b_envelope = json.loads(b_envelope_bytes) + support.assert_equal( + durable_b_evidence["audit_publication_identity"], + expected_b_identity.to_json(), + f"C3 {variant} durable B evidence identity N+1", + ) + support.assert_equal( + durable_b_evidence["block_hash"], + target["hash"], + f"C3 {variant} durable B evidence hash", + ) + support.assert_equal( + { + "block_hash": durable_b_envelope["block_hash"], + "block_height": durable_b_envelope["block_height"], + "audit_bundle_sha256": durable_b_envelope[ + "audit_bundle_sha256" + ], + }, + { + "block_hash": target["hash"], + "block_height": target["height"], + "audit_bundle_sha256": _C2_DIGEST, + }, + f"C3 {variant} durable B envelope identity", + ) + b_evidence_stat = evidence_path.lstat() + b_envelope_stat = target_envelope.lstat() + b_filesystem = _c2_filesystem_snapshot( + root=root, + evidence_path=evidence_path, + ) + support.assert_equal( + sorted(path.name for path in root.iterdir()), + [".prism-audit-publication.lock", target_envelope.name], + f"C3 {variant} exact B root entries", + ) + support.assert_equal( + initial_envelope.exists(), + False, + f"C3 {variant} N remains absent after B", + ) + worker_a.send("attempt") + stale_publication = worker_a.read_event("stale-publication") + support.assert_equal( + stale_publication["authority"], + expected_authority, + f"C3 {variant} late A same lock inode", + ) + support.assert_equal( + stale_publication["floor"], + 2, + f"C3 {variant} late A reads fresh N+1 floor", + ) + _assert_c3_stale_outcome( + stale_publication["outcome"], + label=f"C3 {variant} stale N publication", + ) + stale_latest = stale_publication["latest"] + assert isinstance(stale_latest, dict) + support.assert_equal( + stale_latest["audit_publication_identity"], + expected_b_identity.to_json(), + f"C3 {variant} stale A reconciles B evidence", + ) + if variant == "primary": + post_check = worker_a.read_event("post-check") + transition_check = post_check["transition_check"] + assert isinstance(transition_check, dict) + errors = transition_check["errors"] + assert isinstance(errors, dict) + support.assert_equal( + set(errors), + {"confirmation", "reactivation"}, + "C3 primary exact stale lease-failure labels", + ) + for label in ("confirmation", "reactivation"): + if "writer lease is not active" not in str(errors[label]): + raise support.GateFailure( + f"C3 primary stale {label} wrong error: {errors[label]!r}" + ) + support.assert_equal( + transition_check["before"], + transition_check["after"], + "C3 primary child-observed stale transition immobility", + ) + support.assert_equal( + post_check["retention"], + [ + { + "retention": 0, + "live_removed": 0, + "candidate_removed": 0, + "errors": 0, + }, + { + "retention": 1, + "live_removed": 0, + "candidate_removed": 0, + "errors": 0, + }, + ], + "C3 primary stale A retention 0/1 pins B", + ) + support.assert_equal( + post_check["authority"], + expected_authority, + "C3 primary post-check authority", + ) + support.assert_equal( + _c2_database_snapshot(ledger, block_hashes=all_hashes), + expected_database, + f"C3 {variant} late A row/floor/allocator immobility", + ) + support.assert_equal( + _c3_attempt_and_lease(ledger, variant=variant)["lease"], + { + "writer_id": writer_b["id"], + "writer_epoch": writer_b["epoch"], + "writer_session_token": writer_b["token"], + }, + f"C3 {variant} B lease remains active through A checks", + ) + support.assert_equal( + _c2_filesystem_snapshot(root=root, evidence_path=evidence_path), + b_filesystem, + f"C3 {variant} exact B filesystem survives late A", + ) + support.assert_equal( + evidence_path.read_bytes(), + b_evidence_bytes, + f"C3 {variant} B evidence bytes unchanged", + ) + support.assert_equal( + target_envelope.read_bytes(), + b_envelope_bytes, + f"C3 {variant} B envelope bytes unchanged", + ) + evidence_after = evidence_path.lstat() + envelope_after = target_envelope.lstat() + support.assert_equal( + (evidence_after.st_dev, evidence_after.st_ino), + (b_evidence_stat.st_dev, b_evidence_stat.st_ino), + f"C3 {variant} B evidence inode unchanged", + ) + support.assert_equal( + (envelope_after.st_dev, envelope_after.st_ino), + (b_envelope_stat.st_dev, b_envelope_stat.st_ino), + f"C3 {variant} B envelope inode unchanged", + ) + support.assert_equal( + initial_envelope.exists(), + False, + f"C3 {variant} stale A never repairs N", + ) + if variant == "primary": + damaged_entries = damaged_snapshot["entries"] + assert isinstance(damaged_entries, dict) + support.assert_equal( + sorted(damaged_entries), + [".prism-audit-publication.lock"], + "C3 primary damaged snapshot has no N envelope", + ) + worker_a.send("finish") + worker_a.read_event("closed") + worker_a.wait_success() + worker_b.send("finish") + b_closed = worker_b.read_event("closed") + support.assert_equal( + b_closed["lease_released"], + True, + f"C3 {variant} B lease release", + ) + worker_b.wait_success() + _assert_no_tagged_backends( + (tag_a, tag_b), + f"C3 {variant} tagged child backend cleanup", + ) + with support.ACTIVE_CHILDREN_LOCK: + support.assert_equal( + len(support.ACTIVE_CHILDREN), + 0, + f"C3 {variant} active child registry cleanup", + ) + finally: + if setup_store is not None: + setup_store.close() + if worker_a is not None: + worker_a.close() + if worker_b is not None: + worker_b.close() + ledger.close() + + +def test_c3_b_wins_and_late_a_interleavings() -> None: + for variant in ("primary", "late"): + _test_c3_interleaving(variant) + + +def server_evidence() -> dict[str, object]: + evidence = support.run_json( + """ +SELECT json_build_object( + 'server_version', current_setting('server_version'), + 'server_version_num', current_setting('server_version_num') +); +""" + ) + configured_image = os.environ.get("QBIT_PRISM_GATE_IMAGE", "").strip() + provisioned_image_digest = os.environ.get( + "QBIT_PRISM_GATE_IMAGE_DIGEST", + "", + ).strip() + if not configured_image or configured_image.casefold() == "unreported": + raise support.GateFailure("configured PostgreSQL image evidence is required") + if ( + not provisioned_image_digest + or provisioned_image_digest.casefold() == "unreported" + ): + raise support.GateFailure( + "provisioned PostgreSQL image digest evidence is required" + ) + evidence["configured_image"] = configured_image + evidence["provisioned_image_digest"] = provisioned_image_digest + return evidence + + +def main() -> None: + public_before = support.public_sentinel() + failure: BaseException | None = None + try: + test_database_observed_confirmation_order() + test_c2_a_wins_confirmation_publication() + test_c3_b_wins_and_late_a_interleavings() + except BaseException as error: + failure = error + try: + support.cleanup_active_children() + support.cleanup_owned_schemas() + support.assert_equal( + support.marker_schema_count(), + 0, + "process gate marker cleanup", + ) + support.assert_equal( + support.public_sentinel(), + public_before, + "process gate public preservation", + ) + except BaseException as cleanup_error: + if failure is None: + raise + raise support.GateFailure( + f"process scenario failed with {failure!r}; cleanup also failed " + f"with {cleanup_error!r}" + ) from cleanup_error + else: + support.atexit.unregister(support.cleanup_active_children) + support.atexit.unregister(support.cleanup_owned_schemas) + if failure is not None: + raise failure + print("prism postgres A1 process gate evidence " + json.dumps(server_evidence())) + print( + "prism postgres A1 process gate PASS " + "C1-confirmation-order C2-A-wins-confirmation-publication " + "C3-B-wins-late-A" + ) + + +if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "--c2-worker": + raise SystemExit(_run_c2_worker(Path(sys.argv[2]))) + if len(sys.argv) != 1: + raise SystemExit("unexpected process-gate arguments") + main() diff --git a/tests/prism_vardiff_test_support.py b/tests/prism_vardiff_test_support.py index 0b21f10..4e85d6e 100644 --- a/tests/prism_vardiff_test_support.py +++ b/tests/prism_vardiff_test_support.py @@ -189,6 +189,7 @@ def __init__(self) -> None: self.reversed: list[dict[str, object]] = [] self.rejected: list[dict[str, object]] = [] self.submit_seen = False + self._audit_publication_sequences: dict[str, int] = {} def append(self, pending: object) -> object: self.pending.append(pending) @@ -229,7 +230,19 @@ def reject_prepared_block(self, **kwargs: object) -> dict[str, object]: def confirm_accepted_block(self, **kwargs: object) -> dict[str, object]: self.confirmed.append({**kwargs, "submit_seen_at_confirm": self.submit_seen}) - return {"backend": "fake", "confirmed_count": 1} + block_hash = str(kwargs.get("block_hash") or "") + sequence = self._audit_publication_sequences.get(block_hash) + if sequence is None: + sequence = len(self._audit_publication_sequences) + 1 + self._audit_publication_sequences[block_hash] = sequence + return { + "backend": "fake", + "confirmed_count": 1, + "audit_publication_sequence": sequence, + } + + def audit_publication_sequence_floor(self) -> int: + return max(self._audit_publication_sequences.values(), default=0) def all_shares(self) -> list[object]: return [ @@ -725,10 +738,18 @@ def verified_block_bundle(coinbase_tx_hex: str = "c0ffee") -> dict[str, object]: def verified_audit_report(coinbase_tx_hex: str = "c0ffee") -> dict[str, object]: return { + "reward_manifest_sha256_hex": "44" * 32, + "payout_policy_manifest_sha256_hex": "55" * 32, + "prism_audit_commitment_leaf_hex": "66" * 32, + "audit_commitment_root_hex": "77" * 32, "coinbase_txid": "11" * 32, + "coinbase_wtxid": "88" * 32, "coinbase_manifest_sha256_hex": "22" * 32, "audit_bundle_sha256_hex": "33" * 32, "coinbase_tx_hex": coinbase_tx_hex, + "min_output_sats": 1, + "onchain_output_count": 0, + "accrued_account_count": 0, } diff --git a/tests/test_prism_audit_api.py b/tests/test_prism_audit_api.py index 1c63d45..6a73895 100644 --- a/tests/test_prism_audit_api.py +++ b/tests/test_prism_audit_api.py @@ -3,13 +3,15 @@ from __future__ import annotations import json +import tempfile import threading import unittest import urllib.error import urllib.request from http.server import ThreadingHTTPServer +from pathlib import Path -from lab.prism.prism_coordinator import make_audit_handler +from lab.prism.prism_coordinator import PrismCoordinator, make_audit_handler class FakeLedger: @@ -272,6 +274,45 @@ def test_health_latest_owed_and_metrics_endpoints(self) -> None: self.assertIn("qbit_prism_ctv_fanouts_broadcastable 1", metrics) self.assertIn("qbit_prism_ctv_fanouts_failed 0", metrics) + def test_coordinator_latest_audit_endpoint_preserves_none_and_snapshot_payload(self) -> None: + coordinator = PrismCoordinator.__new__(PrismCoordinator) + coordinator.latest_evidence = None + with tempfile.TemporaryDirectory() as audit_root: + coordinator.audit_dir = Path(audit_root) / "artifacts" + coordinator.evidence_path = Path(audit_root) / "evidence.json" + handler = make_audit_handler(coordinator) + server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + base_url = f"http://127.0.0.1:{server.server_port}" + try: + with self.assertRaises(urllib.error.HTTPError) as raised: + urllib.request.urlopen(base_url + "/audit/latest", timeout=5) + self.assertEqual(raised.exception.code, 404) + raised.exception.close() + + seed = {"schema": "test", "nested": {"value": 1}} + coordinator.latest_evidence = seed + seed["nested"]["value"] = 2 + with urllib.request.urlopen( + base_url + "/audit/latest", + timeout=5, + ) as response: + payload = json.loads(response.read()) + self.assertEqual(payload, {"schema": "test", "nested": {"value": 1}}) + payload["nested"]["value"] = 3 + self.assertEqual( + coordinator.latest_evidence_payload(), + {"schema": "test", "nested": {"value": 1}}, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + store = coordinator.__dict__.get("_audit_artifact_store") + if store is not None: + store.close() + def test_unhealthy_progress_returns_http_503_and_bounded_reason(self) -> None: handler = make_audit_handler(UnhealthyCoordinator()) # type: ignore[arg-type] server = ThreadingHTTPServer(("127.0.0.1", 0), handler) diff --git a/tests/test_prism_audit_artifacts.py b/tests/test_prism_audit_artifacts.py new file mode 100644 index 0000000..7aa6836 --- /dev/null +++ b/tests/test_prism_audit_artifacts.py @@ -0,0 +1,3593 @@ +#!/usr/bin/env python3 + +import hashlib +import json +import os +from dataclasses import replace as dataclass_replace +from pathlib import Path +import select +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from unittest import mock + +from lab.prism.audit_artifacts import ( + AuditArtifactConfig, + AuditArtifactStore, + AuditPublicationIdentity, + LIVE_ENVELOPE_SCHEMA, + LIVE_EVIDENCE_SCHEMA, + OwnedCandidateArtifact, + _FileIdentity, + canonical_audit_bundle_bytes, +) +from lab.prism.bundle_compiler import canonical_bundle_bytes +from lab.prism.share_ledger import SingleWriterShareLedger + + +BLOCK_A = "aa" * 32 +BLOCK_B = "bb" * 32 +DIGEST = "11" * 32 + + +class _TestAuditArtifactStore(AuditArtifactStore): + """Keep publication fixtures terse while supplying a real verifier identity.""" + + def publish_success(self, **kwargs: object): # type: ignore[no-untyped-def] + report = kwargs.get("report") + identity = kwargs.get("identity") + if "publication_floor_sequence" not in kwargs: + if not isinstance(identity, AuditPublicationIdentity): + raise AssertionError("publication test identity is required") + kwargs["publication_floor_sequence"] = identity.sequence + if "verification_identity" not in kwargs: + if not isinstance(report, dict): + raise AssertionError("publication test report is required") + kwargs["verification_identity"] = self.build_verification_identity( + trust_source="configured", + trusted_writer_public_key_hex="44" * 32, + literal_sha256=DIGEST, + literal_byte_len=123, + report=report, + ) + with self.publication_order_guard(): + return super().publish_success(**kwargs) # type: ignore[arg-type] + + +class AuditArtifactStoreTest(unittest.TestCase): + def make_store(self, root: Path, **kwargs: object) -> AuditArtifactStore: + evidence_path = Path(kwargs.pop("evidence_path", root / "evidence.json")) + return _TestAuditArtifactStore( + AuditArtifactConfig( + root=root, + evidence_path=evidence_path, + live_bundle_retention=int(kwargs.pop("live_bundle_retention", 5)), + candidate_retention_seconds=int( + kwargs.pop("candidate_retention_seconds", 86_400) + ), + share_segment_size=int(kwargs.pop("share_segment_size", 0)), + verifier_timeout_seconds=float( + kwargs.pop("verifier_timeout_seconds", 60.0) + ), + ), + **kwargs, + ) + + @staticmethod + def transfer_candidate( + store: AuditArtifactStore, + candidate: OwnedCandidateArtifact, + ) -> None: + path = candidate.path + with path.open("rb") as handle: + store.adopt_compiler_candidate( + candidate, + path=path, + value=os.fstat(handle.fileno()), + ) + + @staticmethod + def report( + digest: str = DIGEST, + *, + block_height: int = 1, + ) -> dict[str, object]: + return { + "schema": "qbit.prism.audit-verification-report.v1", + "block_height": block_height, + "audit_bundle_sha256_hex": digest, + "reward_manifest_sha256_hex": "44" * 32, + "payout_policy_manifest_sha256_hex": "55" * 32, + "prism_audit_commitment_leaf_hex": "66" * 32, + "audit_commitment_root_hex": "77" * 32, + "coinbase_txid": "22" * 32, + "coinbase_wtxid": "88" * 32, + "coinbase_manifest_sha256_hex": "33" * 32, + "coinbase_tx_hex": "00", + "coinbase_value_sats": 1, + "min_output_sats": 1, + "onchain_output_count": 0, + "accrued_account_count": 0, + } + + @staticmethod + def persistence(digest: str = DIGEST) -> dict[str, object]: + return {"audit_bundle_sha256": digest, "body_uri": ""} + + def test_paths_reject_untrusted_hashes_and_stay_in_resolved_root(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) / "audit" + store = self.make_store(root) + candidate = store.issue_candidate(block_hash=BLOCK_A) + self.assertEqual(candidate.path.parent, root.resolve()) + self.assertEqual( + store.live_envelope_path(block_height=0, block_hash=BLOCK_A).parent, + root.resolve(), + ) + for invalid in ("a", "gg" * 32, "../" + BLOCK_A, BLOCK_A + "/x"): + with self.subTest(invalid=invalid), self.assertRaises(ValueError): + store.issue_candidate(block_hash=invalid) + + def test_publication_identity_and_live_path_reject_lossy_types(self) -> None: + invalid_identities = ( + (True, 1, BLOCK_A), + ("1", 1, BLOCK_A), + (-1, 1, BLOCK_A), + (1, False, BLOCK_A), + (1, "1", BLOCK_A), + (1, -1, BLOCK_A), + (1, 1, BLOCK_A.upper()), + ) + for sequence, height, block_hash in invalid_identities: + with self.subTest( + sequence=sequence, + height=height, + block_hash=block_hash, + ), self.assertRaises(ValueError): + AuditPublicationIdentity(sequence, height, block_hash) # type: ignore[arg-type] + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + for height in (True, "1"): + with self.subTest(path_height=height), self.assertRaises(ValueError): + store.live_envelope_path( # type: ignore[arg-type] + block_height=height, + block_hash=BLOCK_A, + ) + identity = AuditPublicationIdentity(1, 1, BLOCK_A) + for floor in (True, "1", -1): + with self.subTest(publication_floor=floor), self.assertRaises( + ValueError + ): + store.publish_success( + identity=identity, + publication_floor_sequence=floor, # type: ignore[arg-type] + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + with self.assertRaisesRegex(RuntimeError, "exceeds"): + store.publish_success( + identity=identity, + publication_floor_sequence=0, + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + with self.assertRaisesRegex(RuntimeError, "behind"): + store.publish_success( + identity=identity, + publication_floor_sequence=2, + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + + def test_live_publication_canonicalizes_persistence_digest_for_restart(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + publication = store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(DIGEST.upper()), + evidence={}, + created_at="now", + ) + self.assertEqual( + publication.evidence["persistence"]["audit_bundle_sha256"], # type: ignore[index] + DIGEST, + ) + store.close() + restarted = self.make_store(root) + latest = restarted.latest_evidence() + self.assertIsNotNone(latest) + assert latest is not None + self.assertEqual(latest["persistence"]["audit_bundle_sha256"], DIGEST) + + def test_root_final_symlink_is_rejected_and_ancestor_is_resolved(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + real = base / "real" + real.mkdir() + ancestor = base / "ancestor" + ancestor.symlink_to(real, target_is_directory=True) + store = self.make_store(ancestor / "audit") + self.assertEqual(store.root, (real / "audit").resolve()) + target = base / "target" + target.mkdir() + final_link = base / "final-link" + final_link.symlink_to(target, target_is_directory=True) + with self.assertRaisesRegex(RuntimeError, "non-symlink"): + self.make_store(final_link) + + def test_root_directory_authority_swap_matrix_fails_closed_and_recovers(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + root = base / "audit" + evidence_path = base / "state" / "evidence.json" + store = self.make_store(root, evidence_path=evidence_path) + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + candidate = store.issue_candidate(block_hash=BLOCK_B) + store.write_compatibility_candidate(candidate, {"candidate": True}) + body = store.body_path(BLOCK_A, DIGEST) + store._write_immutable_bytes(body, b"body") + + pinned_root = base / "audit-pinned" + root.rename(pinned_root) + root.mkdir() + sentinel = root / "operator-sentinel" + sentinel.write_bytes(b"operator") + + self.assertIsNone(store.latest_evidence()) + self.assertEqual(store.publication_sequence_floor(), 0) + self.assertEqual(store.metrics_snapshot()["scan_error"], 1) + self.assertEqual(store.prune_best_effort().errors, 1) + with self.assertRaisesRegex(RuntimeError, "root identity"): + store._write_immutable_bytes(body, b"replacement") + with self.assertRaisesRegex(RuntimeError, "root identity"): + store._read_owned_regular_bytes(body) + with self.assertRaisesRegex(RuntimeError, "root identity"): + store.discard_candidate(candidate) + with self.assertRaisesRegex(RuntimeError, "root identity"): + store.publish_success( + identity=AuditPublicationIdentity(2, 2, BLOCK_B), + report=self.report(block_height=2), + persistence=self.persistence(), + evidence={}, + created_at="blocked", + ) + self.assertEqual(sentinel.read_bytes(), b"operator") + self.assertEqual(sorted(path.name for path in root.iterdir()), [sentinel.name]) + + replacement = base / "audit-replacement" + root.rename(replacement) + pinned_root.rename(root) + self.assertEqual(store.publication_sequence_floor(), 1) + self.assertEqual(store.latest_evidence()["block_hash"], BLOCK_A) # type: ignore[index] + self.assertEqual(store._read_owned_regular_bytes(body)[0], b"body") + store.discard_candidate(candidate) + self.assertFalse(candidate.path.exists()) + self.assertEqual((replacement / sentinel.name).read_bytes(), b"operator") + + def test_external_evidence_parent_swap_rolls_back_publication_and_recovers(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + root = base / "audit" + state = base / "state" + evidence_path = state / "evidence.json" + store = self.make_store(root, evidence_path=evidence_path) + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + original_write = store._write_mutable_json + pinned_state = base / "state-pinned" + replacement_state = base / "state-replacement" + + def swap_before_evidence( + path: Path, + *args: object, + **kwargs: object, + ) -> object: + if path == store.evidence_path: + state.rename(pinned_state) + state.mkdir() + (state / "operator-sentinel").write_bytes(b"operator") + return original_write(path, *args, **kwargs) + + new_envelope = store.live_envelope_path( + block_height=2, + block_hash=BLOCK_B, + ) + with mock.patch.object( + store, + "_write_mutable_json", + side_effect=swap_before_evidence, + ), self.assertRaisesRegex(RuntimeError, "evidence parent identity"): + store.publish_success( + identity=AuditPublicationIdentity(2, 2, BLOCK_B), + report=self.report(block_height=2), + persistence=self.persistence(), + evidence={}, + created_at="blocked", + ) + self.assertFalse(new_envelope.exists()) + self.assertEqual( + (state / "operator-sentinel").read_bytes(), + b"operator", + ) + self.assertIsNone(store.latest_evidence()) + self.assertEqual(store.publication_sequence_floor(), 0) + + state.rename(replacement_state) + pinned_state.rename(state) + self.assertEqual(store.publication_sequence_floor(), 1) + self.assertEqual(store.latest_evidence()["block_hash"], BLOCK_A) # type: ignore[index] + + def test_live_prune_rechecks_evidence_authority_before_removal(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + root = base / "audit" + state = base / "state" + pinned_state = base / "state-pinned" + store = self.make_store( + root, + evidence_path=state / "evidence.json", + live_bundle_retention=0, + ) + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + stale = store.live_envelope_path(block_height=2, block_hash=BLOCK_B) + stale.write_text("stale", encoding="utf-8") + real_unlink = store._unlink_scanned_owned + swapped = False + + def swap_after_scan( + path: Path, + identity: _FileIdentity, + *, + require_all_authorities: bool = False, + ) -> bool: + nonlocal swapped + if require_all_authorities and not swapped: + swapped = True + state.rename(pinned_state) + state.mkdir() + (state / "operator-sentinel").write_bytes(b"operator") + return real_unlink( + path, + identity, + require_all_authorities=require_all_authorities, + ) + + with mock.patch.object( + store, + "_unlink_scanned_owned", + side_effect=swap_after_scan, + ): + result = store.prune_best_effort() + self.assertTrue(swapped) + self.assertEqual(result.live_removed, 0) + self.assertGreaterEqual(result.errors, 1) + self.assertTrue(stale.exists()) + self.assertEqual( + (state / "operator-sentinel").read_bytes(), + b"operator", + ) + + def test_candidate_prune_rechecks_reservations_after_active_snapshot(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store( + Path(tmp), + candidate_retention_seconds=0, + ) + scan_waiting = threading.Event() + release_scan = threading.Event() + result: list[object] = [] + errors: list[BaseException] = [] + real_listdir = os.listdir + + def pause_before_scan(fd: int) -> list[str]: + if fd == store._root_fd: + scan_waiting.set() + if not release_scan.wait(5): + raise AssertionError("timed out waiting to release prune scan") + return real_listdir(fd) + + def prune() -> None: + try: + result.append(store.prune_best_effort()) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + with mock.patch( + "lab.prism.audit_artifacts.os.listdir", + side_effect=pause_before_scan, + ): + thread = threading.Thread(target=prune) + thread.start() + self.assertTrue(scan_waiting.wait(5)) + candidate = store.issue_candidate(block_hash=BLOCK_A) + store.write_compatibility_candidate(candidate, {"reserved": True}) + expected = candidate.path.read_bytes() + release_scan.set() + thread.join(timeout=5) + + self.assertFalse(errors) + self.assertFalse(thread.is_alive()) + self.assertEqual(len(result), 1) + self.assertEqual(result[0].candidate_removed, 0) # type: ignore[union-attr] + self.assertEqual(candidate.path.read_bytes(), expected) + store.discard_candidate(candidate) + + def test_observer_scans_fail_closed_on_midscan_authority_change(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + root = base / "audit" + store = self.make_store( + root, + candidate_retention_seconds=0, + live_bundle_retention=0, + ) + candidate = root / f"prism-live-audit-bundle-candidate-{BLOCK_A}.json" + candidate.write_bytes(b"candidate") + real_lstat = store._owned_lstat + + def run_with_swap(operation: object, label: str) -> object: + pinned = base / f"{label}-pinned" + replacement = base / f"{label}-replacement" + swapped = False + + def swap_then_reject(path: Path) -> os.stat_result: + nonlocal swapped + if not swapped: + swapped = True + root.rename(pinned) + root.mkdir() + (root / "operator-sentinel").write_bytes(b"operator") + raise RuntimeError("authority changed") + return real_lstat(path) + + with mock.patch.object( + store, + "_owned_lstat", + side_effect=swap_then_reject, + ): + result = operation() # type: ignore[operator] + self.assertTrue(swapped) + self.assertEqual( + (root / "operator-sentinel").read_bytes(), + b"operator", + ) + root.rename(replacement) + pinned.rename(root) + return result + + metrics = run_with_swap(store.metrics_snapshot, "metrics") + self.assertEqual(metrics["scan_error"], 1) # type: ignore[index] + self.assertTrue(candidate.exists()) + retained = run_with_swap(store.prune_best_effort, "prune") + self.assertGreaterEqual(retained.errors, 1) # type: ignore[union-attr] + self.assertEqual(retained.candidate_removed, 0) # type: ignore[union-attr] + self.assertTrue(candidate.exists()) + + def test_directory_authority_reconfigure_and_close_are_atomic_and_fd_clean(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + old_root = base / "old-root" + old_evidence = base / "old-state" / "evidence.json" + store = self.make_store(old_root, evidence_path=old_evidence) + old_root_fd = store._root_fd + old_publication_lock_fd = store._publication_lock_fd + old_evidence_fd = store._evidence_parent_fd + real_open = store._open_directory_authority + calls = 0 + prepared_root_fd: int | None = None + + def fail_second(path: Path) -> tuple[int, tuple[int, int]]: + nonlocal calls, prepared_root_fd + calls += 1 + if calls == 2: + raise OSError("second authority") + result = real_open(path) + prepared_root_fd = result[0] + return result + + new_root = base / "new-root" + new_evidence = base / "new-state" / "evidence.json" + with mock.patch.object( + store, + "_open_directory_authority", + side_effect=fail_second, + ), self.assertRaisesRegex(OSError, "second authority"): + store.reconfigure(root=new_root, evidence_path=new_evidence) + self.assertEqual(store.root, old_root.resolve()) + self.assertEqual(store.evidence_path, old_evidence.resolve()) + os.fstat(old_root_fd) + os.fstat(old_evidence_fd) + self.assertIsNotNone(prepared_root_fd) + assert prepared_root_fd is not None + with self.assertRaises(OSError): + os.fstat(prepared_root_fd) + + real_os_close = os.close + old_fds = { + old_root_fd, + old_publication_lock_fd, + old_evidence_fd, + } + + def close_then_report_error(fd: int) -> None: + real_os_close(fd) + if fd in old_fds: + raise OSError("close status unavailable") + + with mock.patch( + "lab.prism.audit_artifacts.os.close", + side_effect=close_then_report_error, + ): + store.reconfigure(root=new_root, evidence_path=new_evidence) + new_root_fd = store._root_fd + new_publication_lock_fd = store._publication_lock_fd + new_evidence_fd = store._evidence_parent_fd + for closed_fd in ( + old_root_fd, + old_publication_lock_fd, + old_evidence_fd, + ): + with self.assertRaises(OSError): + os.fstat(closed_fd) + store.close() + store.close() + self.assertIsNone(store.latest_evidence()) + self.assertEqual(store.publication_sequence_floor(), 0) + metrics = store.metrics_snapshot() + self.assertEqual(metrics["scan_error"], 1) + for kind in ( + "body", + "share_segment", + "live_bundle", + "candidate", + "other", + ): + self.assertEqual(metrics[kind], {"files": 0, "bytes": 0}) + retention = store.prune_best_effort() + self.assertEqual(retention.live_removed, 0) + self.assertEqual(retention.candidate_removed, 0) + self.assertEqual(retention.errors, 1) + with self.assertRaisesRegex(RuntimeError, "closed"): + store.issue_candidate(block_hash=BLOCK_A) + for closed_fd in ( + new_root_fd, + new_publication_lock_fd, + new_evidence_fd, + ): + with self.assertRaises(OSError): + os.fstat(closed_fd) + + def test_reconfigure_hides_transient_new_authority_state_from_readers(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + old_store = self.make_store( + base / "old-root", + evidence_path=base / "old-state" / "evidence.json", + ) + old_store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="old", + ) + new_root = base / "new-root" + new_evidence = base / "new-state" / "evidence.json" + prepared = self.make_store(new_root, evidence_path=new_evidence) + prepared.publish_success( + identity=AuditPublicationIdentity(2, 2, BLOCK_B), + report=self.report(block_height=2), + persistence=self.persistence(), + evidence={}, + created_at="new", + ) + prepared.close() + + real_load = old_store._load_current_evidence_locked + load_entered = threading.Event() + release_load = threading.Event() + reader_started = threading.Event() + reader_values: list[dict[str, object] | None] = [] + errors: list[BaseException] = [] + + def blocked_load() -> None: + load_entered.set() + if not release_load.wait(5): + raise AssertionError("timed out waiting to release evidence load") + real_load() + + def reconfigure() -> None: + try: + old_store.reconfigure( + root=new_root, + evidence_path=new_evidence, + ) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + def read_latest() -> None: + reader_started.set() + try: + reader_values.append(old_store.latest_evidence()) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + with mock.patch.object( + old_store, + "_load_current_evidence_locked", + side_effect=blocked_load, + ): + reconfigure_thread = threading.Thread(target=reconfigure) + reconfigure_thread.start() + self.assertTrue(load_entered.wait(5)) + reader_thread = threading.Thread(target=read_latest) + reader_thread.start() + self.assertTrue(reader_started.wait(5)) + reader_thread.join(timeout=0.05) + self.assertTrue(reader_thread.is_alive()) + release_load.set() + reconfigure_thread.join(timeout=5) + reader_thread.join(timeout=5) + + self.assertFalse(errors) + self.assertFalse(reconfigure_thread.is_alive()) + self.assertFalse(reader_thread.is_alive()) + self.assertEqual(len(reader_values), 1) + assert reader_values[0] is not None + self.assertEqual(reader_values[0]["block_hash"], BLOCK_B) + self.assertEqual(old_store.publication_sequence_floor(), 2) + + def test_publication_guard_is_required_and_internal_file_is_hidden(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = AuditArtifactStore( + AuditArtifactConfig( + root=root, + evidence_path=root / "evidence.json", + ), + canonicalizer=canonical_bundle_bytes, + ) + identity = AuditPublicationIdentity(1, 1, BLOCK_A) + with self.assertRaisesRegex(RuntimeError, "guard is required"): + store.publish_success( + identity=identity, + publication_floor_sequence=1, + report=self.report(), + persistence=self.persistence(), + evidence={}, + verification_identity={}, + created_at="now", + ) + with self.assertRaisesRegex(RuntimeError, "guard is required"): + store.adopt_legacy_publication_identity( + identity, + publication_floor_sequence=1, + ) + self.assertFalse(store.evidence_path.exists()) + metrics = store.metrics_snapshot() + self.assertEqual(metrics["scan_error"], 0) + for kind in ( + "body", + "share_segment", + "live_bundle", + "candidate", + "other", + ): + self.assertEqual(metrics[kind], {"files": 0, "bytes": 0}) + + lock_path = root / ".prism-audit-publication.lock" + parked = root / ".prism-audit-publication.lock.parked" + lock_path.rename(parked) + lock_path.write_bytes(b"replacement") + with self.assertRaisesRegex(RuntimeError, "lock identity changed"): + with store.publication_order_guard(): + pass + self.assertEqual(store.prune_best_effort().errors, 1) + lock_path.unlink() + parked.rename(lock_path) + with store.publication_order_guard(): + with store.publication_order_guard(): + pass + old_authority = ( + store.root, + store.evidence_path, + store._root_fd, + store._publication_lock_fd, + store._evidence_parent_fd, + ) + next_root = root / "nested-reconfigure" + with self.assertRaisesRegex(RuntimeError, "inside publication guard"): + store.reconfigure( + root=next_root, + evidence_path=next_root / "evidence.json", + ) + self.assertEqual( + ( + store.root, + store.evidence_path, + store._root_fd, + store._publication_lock_fd, + store._evidence_parent_fd, + ), + old_authority, + ) + self.assertFalse(next_root.exists()) + for fd in old_authority[2:]: + os.fstat(fd) + + def test_close_waits_for_guard_and_is_rejected_inside_owned_guard(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + guard_entered = threading.Event() + release_guard = threading.Event() + close_attempting = threading.Event() + close_finished = threading.Event() + + def hold_guard() -> None: + with store.publication_order_guard(): + guard_entered.set() + release_guard.wait(5) + + def close_store() -> None: + close_attempting.set() + store.close() + close_finished.set() + + guard_thread = threading.Thread(target=hold_guard) + guard_thread.start() + self.assertTrue(guard_entered.wait(5)) + close_thread = threading.Thread(target=close_store) + close_thread.start() + self.assertTrue(close_attempting.wait(5)) + self.assertFalse(close_finished.wait(0.05)) + release_guard.set() + guard_thread.join(timeout=5) + close_thread.join(timeout=5) + self.assertFalse(guard_thread.is_alive()) + self.assertFalse(close_thread.is_alive()) + self.assertTrue(close_finished.is_set()) + with self.assertRaisesRegex(RuntimeError, "closed"): + with store.publication_order_guard(): + pass + + second = self.make_store(root / "second") + old_fds = ( + second._root_fd, + second._publication_lock_fd, + second._evidence_parent_fd, + ) + with second.publication_order_guard(): + with self.assertRaisesRegex(RuntimeError, "inside publication guard"): + second.close() + for fd in old_fds: + os.fstat(fd) + second.close() + + def test_publication_lock_symlink_is_rejected_on_open_and_reconfigure(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + outside = base / "outside" + outside.write_bytes(b"outside") + bad_root = base / "bad-root" + bad_root.mkdir() + (bad_root / ".prism-audit-publication.lock").symlink_to(outside) + with self.assertRaisesRegex(RuntimeError, "cannot be opened safely"): + AuditArtifactStore( + AuditArtifactConfig( + root=bad_root, + evidence_path=bad_root / "evidence.json", + ) + ) + + good_root = base / "good-root" + store = self.make_store(good_root) + old_state = ( + store.root, + store._root_fd, + store._publication_lock_fd, + store._evidence_parent_fd, + ) + with self.assertRaisesRegex(RuntimeError, "cannot be opened safely"): + store.reconfigure(root=bad_root) + self.assertEqual(store.root, old_state[0]) + for fd in old_state[1:]: + os.fstat(fd) + self.assertEqual(outside.read_bytes(), b"outside") + + def test_two_store_and_subprocess_publication_guards_serialize(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + first = self.make_store(root) + second = self.make_store(root) + attempting = threading.Event() + entered = threading.Event() + release = threading.Event() + + def hold_second() -> None: + attempting.set() + with second.publication_order_guard(): + entered.set() + release.wait(5) + + with first.publication_order_guard(): + thread = threading.Thread(target=hold_second) + thread.start() + self.assertTrue(attempting.wait(5)) + self.assertFalse(entered.wait(0.05)) + with self.assertRaisesRegex(RuntimeError, "another store"): + with second.publication_order_guard(): + pass + self.assertTrue(entered.wait(5)) + release.set() + thread.join(timeout=5) + self.assertFalse(thread.is_alive()) + + script = """ +import sys +from pathlib import Path +from lab.prism.audit_artifacts import AuditArtifactConfig, AuditArtifactStore +root = Path(sys.argv[1]) +store = AuditArtifactStore(AuditArtifactConfig(root=root, evidence_path=root / 'evidence.json')) +with store.publication_order_guard(): + print('locked', flush=True) + sys.stdin.readline() +""" + child = subprocess.Popen( + [sys.executable, "-c", script, str(root)], + cwd=Path(__file__).resolve().parents[1], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + try: + assert child.stdin is not None + assert child.stdout is not None + assert child.stderr is not None + ready, _writable, _exceptional = select.select( + [child.stdout], + [], + [], + 5, + ) + self.assertEqual(ready, [child.stdout], "child guard handshake timed out") + self.assertEqual(child.stdout.readline().strip(), "locked") + parent_attempting = threading.Event() + parent_entered = threading.Event() + + def enter_parent() -> None: + parent_attempting.set() + with first.publication_order_guard(): + parent_entered.set() + + parent_thread = threading.Thread(target=enter_parent) + parent_thread.start() + self.assertTrue(parent_attempting.wait(5)) + self.assertFalse(parent_entered.wait(0.05)) + child.stdin.write("\n") + child.stdin.flush() + child.wait(timeout=5) + parent_thread.join(timeout=5) + self.assertEqual(child.returncode, 0, child.stderr.read()) + self.assertTrue(parent_entered.is_set()) + self.assertFalse(parent_thread.is_alive()) + finally: + if child.poll() is None: + try: + assert child.stdin is not None + child.stdin.write("\n") + child.stdin.flush() + child.wait(timeout=2) + except (BrokenPipeError, subprocess.TimeoutExpired): + child.kill() + child.wait(timeout=2) + for stream in (child.stdin, child.stdout, child.stderr): + if stream is not None: + stream.close() + + def test_peer_publication_reconciles_stale_replay_repair_and_prune(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + stale = self.make_store(root, live_bundle_retention=2) + first = stale.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="first", + ) + peer = self.make_store(root, live_bundle_retention=2) + second = peer.publish_success( + identity=AuditPublicationIdentity(2, 2, BLOCK_B), + report=self.report(block_height=2), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="second", + ) + + replay = stale.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + publication_floor_sequence=2, + report=self.report(), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="stale", + ) + self.assertFalse(replay.published) + self.assertEqual(stale.latest_evidence()["block_hash"], BLOCK_B) # type: ignore[index] + + stale._invalidated_legacy_identity = AuditPublicationIdentity( + 1, + 1, + BLOCK_A, + ) + with self.assertRaisesRegex(RuntimeError, "identity conflict"): + stale.publish_success( + identity=AuditPublicationIdentity(2, 2, BLOCK_A), + publication_floor_sequence=2, + report=self.report(block_height=2), + persistence=self.persistence(), + evidence={}, + created_at="conflict", + ) + stale.reconfigure(live_bundle_retention=0) + result = stale.prune_best_effort() + self.assertTrue(second.envelope_path.exists()) + self.assertFalse(first.envelope_path.exists()) + self.assertEqual(result.live_removed, 1) + + def test_floor_publication_fence_blocks_allocator_and_stale_inverse_repair( + self, + ) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root, live_bundle_retention=2) + ledger = SingleWriterShareLedger() + balance_lock = threading.RLock() + + def persist(block_hash: str, height: int) -> None: + ledger.persist_accepted_block( + block_hash=block_hash, + block_height=height, + parent_hash="00" * 32, + final_bundle={}, + audit_report={}, + ) + + persist(BLOCK_A, 1) + confirmed_a = ledger.confirm_accepted_block( + block_hash=BLOCK_A, + active_tip_height=1, + ) + identity_a = AuditPublicationIdentity( + int(confirmed_a["audit_publication_sequence"]), + 1, + BLOCK_A, + ) + floor_read = threading.Event() + release_publisher = threading.Event() + allocator_attempting = threading.Event() + allocator_allocated = threading.Event() + errors: list[BaseException] = [] + publications: list[object] = [] + + def publish_a() -> None: + try: + with balance_lock: + with store.publication_order_guard(): + floor = ledger.audit_publication_sequence_floor() + floor_read.set() + if not release_publisher.wait(5): + raise AssertionError("publisher release timed out") + publications.append( + store.publish_success( + identity=identity_a, + publication_floor_sequence=floor, + report=self.report(block_height=1), + persistence=self.persistence(), + evidence={"confirmation": confirmed_a}, + created_at="a", + ) + ) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + def allocate_and_publish_b() -> None: + try: + allocator_attempting.set() + with balance_lock: + with store.publication_order_guard(): + persist(BLOCK_B, 2) + confirmed_b = ledger.confirm_accepted_block( + block_hash=BLOCK_B, + active_tip_height=2, + ) + allocator_allocated.set() + identity_b = AuditPublicationIdentity( + int(confirmed_b["audit_publication_sequence"]), + 2, + BLOCK_B, + ) + publications.append( + store.publish_success( + identity=identity_b, + publication_floor_sequence=( + ledger.audit_publication_sequence_floor() + ), + report=self.report(block_height=2), + persistence=self.persistence(), + evidence={"confirmation": confirmed_b}, + created_at="b", + ) + ) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + publisher = threading.Thread(target=publish_a) + publisher.start() + allocator: threading.Thread | None = None + try: + self.assertTrue(floor_read.wait(5)) + allocator = threading.Thread(target=allocate_and_publish_b) + allocator.start() + self.assertTrue(allocator_attempting.wait(5)) + unexpectedly_acquired = balance_lock.acquire(blocking=False) + if unexpectedly_acquired: + balance_lock.release() + self.assertFalse(unexpectedly_acquired) + self.assertFalse(allocator_allocated.is_set()) + finally: + release_publisher.set() + publisher.join(timeout=5) + if allocator is not None: + allocator.join(timeout=5) + self.assertFalse(publisher.is_alive()) + assert allocator is not None + self.assertFalse(allocator.is_alive()) + self.assertFalse(errors) + self.assertTrue(allocator_allocated.is_set()) + self.assertEqual(len(publications), 2) + self.assertEqual(ledger.audit_publication_sequence_floor(), 2) + latest = store.latest_evidence() + assert latest is not None + self.assertEqual(latest["block_hash"], BLOCK_B) + + stale_envelope = store.live_envelope_path( + block_height=1, + block_hash=BLOCK_A, + ) + stale_envelope.unlink() + durable_before = store.evidence_path.read_bytes() + durable_identity = _FileIdentity.from_stat(store.evidence_path.stat()) + with balance_lock: + with store.publication_order_guard(): + stale = store.publish_success( + identity=identity_a, + publication_floor_sequence=( + ledger.audit_publication_sequence_floor() + ), + report=self.report(block_height=1), + persistence=self.persistence(), + evidence={"confirmation": confirmed_a}, + created_at="stale", + ) + self.assertFalse(stale.published) + self.assertFalse(stale_envelope.exists()) + self.assertEqual(store.evidence_path.read_bytes(), durable_before) + self.assertTrue(durable_identity.matches(store.evidence_path.stat())) + + def test_reconfigure_busy_and_post_swap_failure_are_atomic_and_fd_clean(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + old_root = base / "old-root" + old_evidence = base / "old-state" / "evidence.json" + store = self.make_store(old_root, evidence_path=old_evidence) + new_root = base / "new-root" + blocker = self.make_store( + new_root, + evidence_path=base / "new-state" / "evidence.json", + ) + blocker_entered = threading.Event() + release_blocker = threading.Event() + + def hold_new_root() -> None: + with blocker.publication_order_guard(): + blocker_entered.set() + release_blocker.wait(5) + + blocker_thread = threading.Thread(target=hold_new_root) + blocker_thread.start() + self.assertTrue(blocker_entered.wait(5)) + old_fds = ( + store._root_fd, + store._publication_lock_fd, + store._evidence_parent_fd, + ) + with self.assertRaisesRegex(RuntimeError, "guard is busy"): + store.reconfigure( + root=new_root, + evidence_path=base / "new-state" / "evidence.json", + ) + self.assertEqual(store.root, old_root.resolve()) + for fd in old_fds: + os.fstat(fd) + release_blocker.set() + blocker_thread.join(timeout=5) + self.assertFalse(blocker_thread.is_alive()) + + target_root = base / "target-root" + target_evidence = base / "target-state" / "evidence.json" + prepared_fds: list[int] = [] + real_open_directory = store._open_directory_authority + real_open_lock = store._open_publication_lock_authority + + def record_directory(path: Path) -> tuple[int, tuple[int, int]]: + result = real_open_directory(path) + prepared_fds.append(result[0]) + return result + + def record_lock( + root: Path, + root_fd: int, + root_identity: tuple[int, int], + ) -> tuple[int, _FileIdentity]: + result = real_open_lock(root, root_fd, root_identity) + prepared_fds.append(result[0]) + return result + + with mock.patch.object( + store, + "_open_directory_authority", + side_effect=record_directory, + ), mock.patch.object( + store, + "_open_publication_lock_authority", + side_effect=record_lock, + ), mock.patch.object( + store, + "_reload_current_evidence_locked", + side_effect=RuntimeError("injected post-swap load failure"), + ), self.assertRaisesRegex(RuntimeError, "post-swap load"): + store.reconfigure( + root=target_root, + evidence_path=target_evidence, + ) + self.assertEqual(store.root, old_root.resolve()) + self.assertEqual(store.evidence_path, old_evidence.resolve()) + self.assertEqual( + ( + store._root_fd, + store._publication_lock_fd, + store._evidence_parent_fd, + ), + old_fds, + ) + for fd in old_fds: + os.fstat(fd) + for fd in prepared_fds: + with self.assertRaises(OSError): + os.fstat(fd) + + boundary_root = base / "boundary-root" + boundary_evidence = base / "boundary-state" / "evidence.json" + boundary_prepared_fds: list[int] = [] + explicit_old_validations = 0 + real_validate = store._validate_publication_lock_identity + + def record_boundary_directory( + path: Path, + ) -> tuple[int, tuple[int, int]]: + result = real_open_directory(path) + boundary_prepared_fds.append(result[0]) + return result + + def record_boundary_lock( + root: Path, + root_fd: int, + root_identity: tuple[int, int], + ) -> tuple[int, _FileIdentity]: + result = real_open_lock(root, root_fd, root_identity) + boundary_prepared_fds.append(result[0]) + return result + + def fail_final_old_validation(**kwargs: object) -> None: + nonlocal explicit_old_validations + if kwargs.get("root_fd") == old_fds[0]: + explicit_old_validations += 1 + if explicit_old_validations == 2: + raise RuntimeError("injected final old-authority loss") + real_validate(**kwargs) # type: ignore[arg-type] + + with mock.patch.object( + store, + "_open_directory_authority", + side_effect=record_boundary_directory, + ), mock.patch.object( + store, + "_open_publication_lock_authority", + side_effect=record_boundary_lock, + ), mock.patch.object( + store, + "_validate_publication_lock_identity", + side_effect=fail_final_old_validation, + ), self.assertRaisesRegex(RuntimeError, "final old-authority"): + store.reconfigure( + root=boundary_root, + evidence_path=boundary_evidence, + ) + self.assertEqual(store.root, old_root.resolve()) + self.assertEqual(store.evidence_path, old_evidence.resolve()) + self.assertEqual( + ( + store._root_fd, + store._publication_lock_fd, + store._evidence_parent_fd, + ), + old_fds, + ) + for fd in old_fds: + os.fstat(fd) + for fd in boundary_prepared_fds: + with self.assertRaises(OSError): + os.fstat(fd) + + store.reconfigure( + root=boundary_root, + evidence_path=boundary_evidence, + ) + self.assertEqual(store.root, boundary_root.resolve()) + self.assertEqual(store.evidence_path, boundary_evidence.resolve()) + for fd in old_fds: + with self.assertRaises(OSError): + os.fstat(fd) + + def test_strict_artifact_classification_rejects_lookalikes(self) -> None: + self.assertEqual( + AuditArtifactStore.artifact_kind( + f"prism-audit-bundle-body-{BLOCK_A}-{DIGEST}.json" + ), + "body", + ) + self.assertEqual( + AuditArtifactStore.artifact_kind( + f"prism-live-audit-bundle-2-{BLOCK_A}.json" + ), + "live_bundle", + ) + self.assertEqual( + AuditArtifactStore.artifact_kind( + f".prism-live-audit-bundle-candidate-{BLOCK_A}.json.tmp" + ), + "candidate", + ) + for name in ( + f"prism-live-audit-bundle-02-{BLOCK_A}.json", + f"prism-live-audit-bundle-2-{BLOCK_A}.json.bak", + "prism-live-audit-bundle-candidate-operator.json", + ): + self.assertEqual(AuditArtifactStore.artifact_kind(name), "other") + + def test_a1_only_consumes_an_injected_j1_canonical_capability(self) -> None: + with mock.patch( + "lab.prism.audit_artifacts.subprocess.run", + side_effect=AssertionError("A1 must not own canonicalizer subprocesses"), + ): + self.assertEqual( + canonical_audit_bundle_bytes({"x": 1}, lambda _value: b"typed"), + b"typed", + ) + with self.assertRaisesRegex(RuntimeError, "J1 canonical"): + canonical_audit_bundle_bytes({"x": 1}) + + completed = mock.Mock(returncode=0, stdout=b"rust-typed", stderr=b"") + with mock.patch( + "lab.prism.bundle_compiler.subprocess.run", + return_value=completed, + ) as run: + self.assertEqual(canonical_bundle_bytes({"x": "miner-é"}), b"rust-typed") + self.assertIn("qbit-prism-audit-canonicalize", " ".join(run.call_args.args[0])) + + def test_candidate_cleanup_removes_only_its_adopted_inode(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + candidate = store.issue_candidate(block_hash=BLOCK_A) + candidate.path.write_bytes(b"first") + self.transfer_candidate(store, candidate) + candidate.path.unlink() + candidate.path.write_bytes(b"replacement") + store.discard_candidate(candidate) + self.assertEqual(candidate.path.read_bytes(), b"replacement") + + def test_unadopted_candidate_collision_is_preserved_and_released(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + candidate = store.issue_candidate(block_hash=BLOCK_A) + candidate.path.write_bytes(b"competitor") + with self.assertRaises(FileExistsError): + store.write_compatibility_candidate(candidate, {"x": 1}) + self.assertEqual(candidate.path.read_bytes(), b"competitor") + store.reconfigure(root=root / "next") + + def test_unadopted_discard_never_deletes_a_created_path(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + candidate = store.issue_candidate(block_hash=BLOCK_A) + candidate.path.write_bytes(b"not-transferred") + store.discard_candidate(candidate) + self.assertEqual(candidate.path.read_bytes(), b"not-transferred") + + def test_reconfigure_rejects_active_candidate(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root / "a") + store.issue_candidate(block_hash=BLOCK_A) + with self.assertRaisesRegex(RuntimeError, "candidates are active"): + store.reconfigure(root=root / "b") + + def test_compatibility_candidate_failure_cleans_owned_temp(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + candidate = store.issue_candidate(block_hash=BLOCK_A) + with mock.patch("os.fsync", side_effect=OSError("fsync")): + with self.assertRaises(OSError): + store.write_compatibility_candidate(candidate, {"x": 1}) + self.assertFalse(candidate.path.exists()) + + def test_verify_candidate_binds_literal_identity_and_report(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + candidate = store.issue_candidate(block_hash=BLOCK_A) + candidate.path.write_bytes(b"canonical") + self.transfer_candidate(store, candidate) + digest = hashlib.sha256(b"canonical").hexdigest() + verified = store.verify_candidate( + candidate, + coinbase_tx_hex="00", + expected_coinbase_value_sats=1, + trusted_writer_public_key_hex="44" * 32, + verifier=lambda *_args, **_kwargs: self.report(digest), + ) + self.assertTrue(verified.canonical_copy_eligible) + self.assertEqual(verified.literal_sha256, digest) + self.assertEqual( + verified.verification_identity, + AuditArtifactStore.build_verification_identity( + trust_source="configured", + trusted_writer_public_key_hex="44" * 32, + literal_sha256=digest, + literal_byte_len=len(b"canonical"), + report=self.report(digest), + ), + ) + + def test_verifier_retry_identity_never_reuses_prior_attempt(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + candidate = store.issue_candidate(block_hash=BLOCK_A) + candidate.path.write_bytes(b"canonical") + self.transfer_candidate(store, candidate) + digest = hashlib.sha256(b"canonical").hexdigest() + + with self.assertRaisesRegex(RuntimeError, "failed attempt"): + store.verify_candidate( + candidate, + coinbase_tx_hex="00", + expected_coinbase_value_sats=1, + trusted_writer_public_key_hex="44" * 32, + verifier=lambda *_args, **_kwargs: (_ for _ in ()).throw( + RuntimeError("failed attempt") + ), + ) + verified = store.verify_candidate( + candidate, + coinbase_tx_hex="00", + expected_coinbase_value_sats=1, + trusted_writer_public_key_hex="44" * 32, + verifier=lambda *_args, **_kwargs: self.report(digest), + ) + store.require_current_verified_candidate(verified, candidate) + + store.discard_candidate(candidate) + replacement = store.issue_candidate(block_hash=BLOCK_A) + replacement.path.write_bytes(b"canonical") + self.transfer_candidate(store, replacement) + with self.assertRaisesRegex(RuntimeError, "another candidate"): + store.require_current_verified_candidate(verified, replacement) + with self.assertRaisesRegex(RuntimeError, "later failure"): + store.verify_candidate( + replacement, + coinbase_tx_hex="00", + expected_coinbase_value_sats=1, + trusted_writer_public_key_hex="44" * 32, + verifier=lambda *_args, **_kwargs: (_ for _ in ()).throw( + RuntimeError("later failure") + ), + ) + + def test_verify_candidate_rejects_post_verifier_swap(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + candidate = store.issue_candidate(block_hash=BLOCK_A) + candidate.path.write_bytes(b"canonical") + self.transfer_candidate(store, candidate) + + def verifier(_path: Path, *_args: object, **_kwargs: object) -> dict[str, object]: + candidate.path.unlink() + candidate.path.write_bytes(b"replacement") + return self.report(hashlib.sha256(b"replacement").hexdigest()) + + with self.assertRaisesRegex(RuntimeError, "changed during"): + store.verify_candidate( + candidate, + coinbase_tx_hex="00", + expected_coinbase_value_sats=1, + trusted_writer_public_key_hex="44" * 32, + verifier=verifier, + ) + + def test_verifier_report_requires_complete_coinbase_identity(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + for missing in ( + "audit_bundle_sha256_hex", + "reward_manifest_sha256_hex", + "payout_policy_manifest_sha256_hex", + "prism_audit_commitment_leaf_hex", + "audit_commitment_root_hex", + "coinbase_txid", + "coinbase_wtxid", + "coinbase_manifest_sha256_hex", + "coinbase_tx_hex", + "coinbase_value_sats", + "min_output_sats", + "onchain_output_count", + "accrued_account_count", + ): + with self.subTest(missing=missing): + store = self.make_store(Path(tmp) / missing) + candidate = store.issue_candidate(block_hash=BLOCK_A) + candidate.path.write_bytes(b"canonical") + self.transfer_candidate(store, candidate) + report = self.report(hashlib.sha256(b"canonical").hexdigest()) + report.pop(missing) + with self.assertRaises((RuntimeError, ValueError)): + store.verify_candidate( + candidate, + coinbase_tx_hex="00", + expected_coinbase_value_sats=1, + trusted_writer_public_key_hex="44" * 32, + verifier=lambda *_args, _report=report, **_kwargs: _report, + ) + store.discard_candidate(candidate) + + def test_verifier_report_schema_types_and_height_fail_closed(self) -> None: + cases = ( + ("schema", "wrong"), + ("block_height", "1"), + ("block_height", 2), + ("coinbase_value_sats", 1.5), + ("min_output_sats", True), + ("onchain_output_count", -1), + ("accrued_account_count", 1.5), + ) + for field, value in cases: + with self.subTest(field=field, value=value): + report = self.report() + report[field] = value + with self.assertRaises(RuntimeError): + AuditArtifactStore._validate_verifier_report( + report, + coinbase_tx_hex="00", + expected_coinbase_value_sats=1, + expected_block_height=1, + ) + + def test_trust_precedence_and_embedded_test_key_gate(self) -> None: + bundle = { + "ledger_window_attestation": { + "signature": {"public_key_hex": "55" * 32} + } + } + self.assertEqual( + AuditArtifactStore.trusted_writer_key( + "44" * 32, + bundle, + allow_embedded_test_key=True, + ), + "44" * 32, + ) + with self.assertRaisesRegex(RuntimeError, "configured"): + AuditArtifactStore.trusted_writer_key( + None, + bundle, + allow_embedded_test_key=False, + ) + self.assertEqual( + AuditArtifactStore.trusted_writer_key( + None, + bundle, + allow_embedded_test_key=True, + ), + "55" * 32, + ) + + def test_verifier_reads_unlinked_snapshot_during_candidate_aba(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + candidate = store.issue_candidate(block_hash=BLOCK_A) + candidate.path.write_bytes(b"canonical") + self.transfer_candidate(store, candidate) + original = candidate.path.stat() + digest = hashlib.sha256(b"canonical").hexdigest() + + def verifier(path: Path, *_args: object, **_kwargs: object) -> dict[str, object]: + self.assertEqual(path.read_bytes(), b"canonical") + candidate.path.write_bytes(b"replacement") + candidate.path.write_bytes(b"canonical") + os.utime( + candidate.path, + ns=(original.st_atime_ns, original.st_mtime_ns), + ) + return self.report(digest) + + verified = store.verify_candidate( + candidate, + coinbase_tx_hex="00", + expected_coinbase_value_sats=1, + trusted_writer_public_key_hex="44" * 32, + expected_block_height=1, + verifier=verifier, + ) + self.assertEqual(verified.literal_sha256, digest) + + def test_verifier_subprocess_timeout_nonzero_malformed_and_oversize(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + bundle = root / "bundle.json" + bundle.write_text("{}", encoding="utf-8") + cases = ( + ( + [sys.executable, "-c", "import time; time.sleep(2)"], + "timed out", + 0.05, + ), + ([sys.executable, "-c", "import sys; sys.stderr.write('bad'); sys.exit(2)"], "failed: bad", 1.0), + ([sys.executable, "-c", "print('not-json')"], "invalid JSON", 1.0), + ([sys.executable, "-c", "import sys; sys.stdout.write('x' * 1100000)"], "output exceeded", 1.0), + ) + for command, message, timeout in cases: + with self.subTest(message=message): + store = self.make_store( + root / message.replace(" ", "-"), + verifier_timeout_seconds=timeout, + ) + with mock.patch( + "lab.prism.audit_artifacts.prism_tool_command", + return_value=command, + ), self.assertRaisesRegex(RuntimeError, message): + store.verify_bundle( + bundle, + "00", + "44" * 32, + expected_coinbase_value_sats=1, + ) + + def test_verifier_timeout_kills_descendants_that_inherit_output_pipes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + bundle = root / "bundle.json" + marker = root / "descendant-survived" + bundle.write_text("{}", encoding="utf-8") + descendant = ( + "import pathlib,time; time.sleep(0.4); " + f"pathlib.Path({str(marker)!r}).write_text('survived')" + ) + parent = ( + "import subprocess,sys,time; " + f"subprocess.Popen([sys.executable,'-c',{descendant!r}]," + "stdout=sys.stdout,stderr=sys.stderr); time.sleep(5)" + ) + store = self.make_store( + root / "audit", + verifier_timeout_seconds=0.05, + ) + with mock.patch( + "lab.prism.audit_artifacts.prism_tool_command", + return_value=[sys.executable, "-c", parent], + ), self.assertRaisesRegex(RuntimeError, "timed out"): + store.verify_bundle( + bundle, + "00", + "44" * 32, + expected_coinbase_value_sats=1, + ) + time.sleep(0.5) + self.assertFalse(marker.exists()) + + def test_publication_uses_durable_ordinal_not_height(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp), live_bundle_retention=0) + newer = store.publish_success( + identity=AuditPublicationIdentity(8, 101, BLOCK_B), + report=self.report(block_height=101), + persistence=self.persistence(), + evidence={"audit_report": self.report(), "persistence": self.persistence()}, + created_at="now", + ) + stale = store.publish_success( + identity=AuditPublicationIdentity(7, 100, BLOCK_A), + report=self.report(block_height=100), + persistence=self.persistence(), + evidence={"audit_report": self.report(), "persistence": self.persistence()}, + created_at="later", + ) + self.assertTrue(newer.published) + self.assertFalse(stale.published) + self.assertEqual(store.latest_evidence()["block_hash"], BLOCK_B) # type: ignore[index] + deep_reorg = store.publish_success( + identity=AuditPublicationIdentity(9, 99, BLOCK_A), + report=self.report(block_height=99), + persistence=self.persistence(), + evidence={"audit_report": self.report(), "persistence": self.persistence()}, + created_at="latest", + ) + self.assertTrue(deep_reorg.published) + self.assertEqual(store.latest_evidence()["block_hash"], BLOCK_A) # type: ignore[index] + + def test_publication_and_startup_bind_report_height(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + with self.assertRaisesRegex(RuntimeError, "block height"): + store.publish_success( + identity=AuditPublicationIdentity(1, 2, BLOCK_A), + report=self.report(block_height=1), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + store.publish_success( + identity=AuditPublicationIdentity(1, 2, BLOCK_A), + report=self.report(block_height=2), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + payload = json.loads(store.evidence_path.read_text(encoding="utf-8")) + payload["audit_report"]["block_height"] = 1 + store.evidence_path.write_text(json.dumps(payload), encoding="utf-8") + self.assertIsNone(self.make_store(root).latest_evidence()) + + def test_equal_ordinal_conflict_fails_closed(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(block_height=1), + persistence=self.persistence(), + evidence={"audit_report": self.report(), "persistence": self.persistence()}, + created_at="now", + ) + with self.assertRaisesRegex(RuntimeError, "conflict"): + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_B), + report=self.report(block_height=1), + persistence=self.persistence(), + evidence={"audit_report": self.report(), "persistence": self.persistence()}, + created_at="later", + ) + + def test_exact_replay_rejects_changed_coinbase_identity(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + identity = AuditPublicationIdentity(1, 1, BLOCK_A) + store.publish_success( + identity=identity, + report=self.report(block_height=1), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + changed = self.report(block_height=1) + changed["coinbase_txid"] = "99" * 32 + with self.assertRaisesRegex(RuntimeError, "replay payload conflict"): + store.publish_success( + identity=identity, + report=changed, + persistence=self.persistence(), + evidence={}, + created_at="later", + ) + + def test_exact_replay_is_stable_but_changed_evidence_conflicts(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + identity = AuditPublicationIdentity(3, 4, BLOCK_A) + first = store.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="now", + ) + replay = store.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="later", + ) + self.assertTrue(first.published) + self.assertFalse(replay.published) + advanced_floor_replay = store.publish_success( + identity=identity, + publication_floor_sequence=4, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="later-still", + ) + self.assertFalse(advanced_floor_replay.published) + with self.assertRaisesRegex(RuntimeError, "replay payload conflict"): + store.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 2}}, + created_at="later", + ) + + def test_exact_replay_repairs_invalid_mutable_evidence_but_rejects_envelope_replacement( + self, + ) -> None: + identity = AuditPublicationIdentity(3, 4, BLOCK_A) + for missing in ("evidence", "envelope", "both"): + with self.subTest(missing=missing), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + first = store.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="now", + ) + if missing in {"evidence", "both"}: + store.evidence_path.unlink() + if missing in {"envelope", "both"}: + first.envelope_path.unlink() + before_advanced_floor_attempt = ( + first.envelope_path.read_bytes() + if first.envelope_path.exists() + else None, + store.evidence_path.read_bytes() + if store.evidence_path.exists() + else None, + ) + with self.assertRaisesRegex(RuntimeError, "behind"): + store.publish_success( + identity=identity, + publication_floor_sequence=4, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="advanced-floor", + ) + self.assertEqual( + ( + first.envelope_path.read_bytes() + if first.envelope_path.exists() + else None, + store.evidence_path.read_bytes() + if store.evidence_path.exists() + else None, + ), + before_advanced_floor_attempt, + ) + repaired = store.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="later", + ) + self.assertTrue(repaired.published) + self.assertTrue(store.evidence_path.exists()) + self.assertTrue(first.envelope_path.exists()) + store.close() + restarted = self.make_store(root) + self.assertEqual( + restarted.latest_evidence()["block_hash"], # type: ignore[index] + BLOCK_A, + ) + + for replacement_bytes in ( + b"not-json", + b'{"operator":"replacement"}', + ): + for restart in (False, True): + with self.subTest( + evidence_replacement=replacement_bytes, + restart=restart, + ), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + first = store.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="now", + ) + envelope_bytes = first.envelope_path.read_bytes() + store.evidence_path.write_bytes(replacement_bytes) + if restart: + store.close() + store = self.make_store(root) + self.assertIsNone(store.latest_evidence()) + with self.assertRaisesRegex(RuntimeError, "behind"): + store.publish_success( + identity=identity, + publication_floor_sequence=4, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="advanced-floor", + ) + self.assertEqual( + store.evidence_path.read_bytes(), + replacement_bytes, + ) + repaired = store.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="later", + ) + self.assertTrue(repaired.published) + self.assertEqual(first.envelope_path.read_bytes(), envelope_bytes) + self.assertEqual( + json.loads(store.evidence_path.read_text(encoding="utf-8"))[ + "block_hash" + ], + BLOCK_A, + ) + + for restart in (False, True): + with self.subTest( + envelope_replacement=True, + restart=restart, + ), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + first = store.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="now", + ) + first.envelope_path.write_bytes(b'{"operator":"replacement"}') + replacement_bytes = first.envelope_path.read_bytes() + if restart: + store.close() + store = self.make_store(root) + self.assertIsNone(store.latest_evidence()) + with self.assertRaisesRegex(RuntimeError, "(conflicts|invalid)"): + store.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="later", + ) + self.assertEqual(first.envelope_path.read_bytes(), replacement_bytes) + + def test_newer_publication_repairs_invalid_evidence_and_repoints_it(self) -> None: + for next_block_hash, next_height in ((BLOCK_A, 4), (BLOCK_B, 5)): + for restart in (False, True): + with self.subTest( + next_block_hash=next_block_hash, + restart=restart, + ), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + first = store.publish_success( + identity=AuditPublicationIdentity(3, 4, BLOCK_A), + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="now", + ) + first_envelope_bytes = first.envelope_path.read_bytes() + store.evidence_path.write_bytes(b'{"operator":"replacement"}') + if restart: + store.close() + store = self.make_store(root) + self.assertIsNone(store.latest_evidence()) + second = store.publish_success( + identity=AuditPublicationIdentity( + 4, + next_height, + next_block_hash, + ), + publication_floor_sequence=4, + report=self.report(block_height=next_height), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="later", + ) + self.assertTrue(second.published) + self.assertEqual( + first.envelope_path.read_bytes(), + first_envelope_bytes, + ) + self.assertTrue(second.envelope_path.exists()) + self.assertEqual( + json.loads(store.evidence_path.read_text(encoding="utf-8"))[ + "block_hash" + ], + next_block_hash, + ) + + def test_restart_repair_requires_the_fresh_durable_publication_floor(self) -> None: + identity = AuditPublicationIdentity(3, 4, BLOCK_A) + for damage in ("missing_envelope", "malformed_evidence", "both_missing"): + with self.subTest(damage=damage), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + first = store.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="now", + ) + if damage in {"missing_envelope", "both_missing"}: + first.envelope_path.unlink() + if damage == "malformed_evidence": + store.evidence_path.write_bytes(b"not-json") + elif damage == "both_missing": + store.evidence_path.unlink() + store.close() + + restarted = self.make_store(root) + self.assertIsNone(restarted.latest_evidence()) + evidence_before = ( + restarted.evidence_path.read_bytes() + if restarted.evidence_path.exists() + else None + ) + with self.assertRaisesRegex(RuntimeError, "behind"): + restarted.publish_success( + identity=AuditPublicationIdentity(2, 5, BLOCK_B), + publication_floor_sequence=3, + report=self.report(block_height=5), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="stale", + ) + self.assertFalse( + restarted.live_envelope_path( + block_height=5, + block_hash=BLOCK_B, + ).exists() + ) + self.assertEqual( + restarted.evidence_path.read_bytes() + if restarted.evidence_path.exists() + else None, + evidence_before, + ) + repaired = restarted.publish_success( + identity=identity, + publication_floor_sequence=3, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="repair", + ) + self.assertTrue(repaired.published) + self.assertTrue(first.envelope_path.exists()) + self.assertEqual( + json.loads(restarted.evidence_path.read_text(encoding="utf-8"))[ + "block_hash" + ], + BLOCK_A, + ) + + def test_failed_invalid_evidence_repair_preserves_bytes_and_revokes_stale_pin(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + identity = AuditPublicationIdentity(3, 4, BLOCK_A) + first = store.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="now", + ) + latest_before = store.latest_evidence() + envelope_before = first.envelope_path.read_bytes() + corrupt_evidence = b'{"operator":"replacement"}' + store.evidence_path.write_bytes(corrupt_evidence) + original_fsync = store._fsync_directory + failed = False + + def fail_evidence_parent(parent: Path) -> None: + nonlocal failed + if parent == store.evidence_path.parent and not failed: + failed = True + raise OSError("injected evidence durability failure") + original_fsync(parent) + + with mock.patch.object( + store, + "_fsync_directory", + side_effect=fail_evidence_parent, + ), self.assertRaisesRegex(OSError, "evidence durability"): + store.publish_success( + identity=identity, + publication_floor_sequence=3, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="later", + ) + + self.assertTrue(failed) + self.assertEqual(store.evidence_path.read_bytes(), corrupt_evidence) + self.assertEqual(first.envelope_path.read_bytes(), envelope_before) + self.assertIsNotNone(latest_before) + self.assertIsNone(store.latest_evidence()) + self.assertIsNone(store._current_envelope) + self.assertIsNone(store._current_identity) + + def test_exact_replay_reuses_nonidentity_global_stats_after_restart(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + identity = AuditPublicationIdentity(3, 4, BLOCK_A) + store = self.make_store(root) + store.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence={**self.persistence(), "share_count": 10}, + evidence={ + "confirmation": {"confirmed_count": 1}, + "accepted_share_count": 10, + "distinct_miner_count": 2, + "job_share_count": 3, + }, + created_at="now", + ) + restarted = self.make_store(root) + replay = restarted.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence={**self.persistence(), "share_count": 99}, + evidence={ + "confirmation": {"confirmed_count": 1}, + "accepted_share_count": 99, + "distinct_miner_count": 20, + "job_share_count": 3, + }, + created_at="later", + ) + self.assertFalse(replay.published) + self.assertEqual(replay.evidence["accepted_share_count"], 10) + self.assertEqual(replay.evidence["distinct_miner_count"], 2) + self.assertEqual(replay.evidence["job_share_count"], 3) + self.assertEqual(replay.evidence["persistence"]["share_count"], 10) # type: ignore[index] + with self.assertRaisesRegex(RuntimeError, "replay payload conflict"): + restarted.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence={**self.persistence(), "share_count": 100}, + evidence={ + "confirmation": {"confirmed_count": 1}, + "accepted_share_count": 100, + "distinct_miner_count": 21, + "job_share_count": 4, + }, + created_at="later", + ) + + def test_verification_identity_is_restart_stable_and_tamper_evident(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + report = self.report(block_height=4) + verification = AuditArtifactStore.build_verification_identity( + trust_source="configured", + trusted_writer_public_key_hex="44" * 32, + literal_sha256="99" * 32, + literal_byte_len=456, + report=report, + ) + identity = AuditPublicationIdentity(3, 4, BLOCK_A) + store.publish_success( + identity=identity, + report=report, + persistence=self.persistence(), + evidence={}, + verification_identity=verification, + created_at="now", + ) + restarted = self.make_store(root) + latest = restarted.latest_evidence() + assert latest is not None + self.assertEqual(latest["audit_verification_identity"], verification) + replay = restarted.publish_success( + identity=identity, + report=report, + persistence=self.persistence(), + evidence={}, + verification_identity=verification, + created_at="later", + ) + self.assertFalse(replay.published) + + changed_report = dict(report) + changed_report["coinbase_wtxid"] = "aa" * 32 + variants = ( + ( + report, + AuditArtifactStore.build_verification_identity( + trust_source="configured", + trusted_writer_public_key_hex="55" * 32, + literal_sha256="99" * 32, + literal_byte_len=456, + report=report, + ), + ), + ( + report, + AuditArtifactStore.build_verification_identity( + trust_source="embedded_test_only", + trusted_writer_public_key_hex="44" * 32, + literal_sha256="99" * 32, + literal_byte_len=456, + report=report, + ), + ), + ( + report, + AuditArtifactStore.build_verification_identity( + trust_source="configured", + trusted_writer_public_key_hex="44" * 32, + literal_sha256="aa" * 32, + literal_byte_len=456, + report=report, + ), + ), + ( + report, + AuditArtifactStore.build_verification_identity( + trust_source="configured", + trusted_writer_public_key_hex="44" * 32, + literal_sha256="99" * 32, + literal_byte_len=457, + report=report, + ), + ), + ( + changed_report, + AuditArtifactStore.build_verification_identity( + trust_source="configured", + trusted_writer_public_key_hex="44" * 32, + literal_sha256="99" * 32, + literal_byte_len=456, + report=changed_report, + ), + ), + ) + for variant_report, changed in variants: + with self.subTest(changed=changed), self.assertRaisesRegex( + RuntimeError, + "replay payload conflict", + ): + restarted.publish_success( + identity=identity, + report=variant_report, + persistence=self.persistence(), + evidence={}, + verification_identity=changed, + created_at="later", + ) + + def test_startup_rejects_each_verification_identity_tamper(self) -> None: + cases = ( + ("trust_source", "embedded_test_only"), + ("ledger_writer_public_key_hex", "55" * 32), + ("literal_sha256_hex", "99" * 32), + ("literal_byte_len", 999), + ("identity_sha256_hex", "88" * 32), + ) + for field, value in cases: + with self.subTest(field=field), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + payload = json.loads(store.evidence_path.read_text(encoding="utf-8")) + payload["audit_verification_identity"][field] = value + store.evidence_path.write_text(json.dumps(payload), encoding="utf-8") + self.assertIsNone(self.make_store(root).latest_evidence()) + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + payload = json.loads(store.evidence_path.read_text(encoding="utf-8")) + payload["audit_verification_identity"]["report"]["coinbase_txid"] = ( + "99" * 32 + ) + store.evidence_path.write_text(json.dumps(payload), encoding="utf-8") + self.assertIsNone(self.make_store(root).latest_evidence()) + + def test_higher_ordinal_replaces_same_height_block(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + store.publish_success( + identity=AuditPublicationIdentity(1, 10, BLOCK_A), + report=self.report(block_height=10), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + replacement = store.publish_success( + identity=AuditPublicationIdentity(2, 10, BLOCK_B), + report=self.report(block_height=10), + persistence=self.persistence(), + evidence={}, + created_at="later", + ) + self.assertTrue(replacement.published) + self.assertEqual(store.latest_evidence()["block_hash"], BLOCK_B) # type: ignore[index] + + def test_publication_rejects_body_for_wrong_block_or_digest(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + identity = AuditPublicationIdentity(1, 1, BLOCK_A) + for body in ( + store.body_path(BLOCK_B, DIGEST), + store.body_path(BLOCK_A, "77" * 32), + ): + with self.subTest(body=body), self.assertRaisesRegex( + RuntimeError, + "body URI does not match", + ): + store.publish_success( + identity=identity, + report=self.report(block_height=1), + persistence={ + "audit_bundle_sha256": DIGEST, + "body_uri": str(body), + }, + evidence={}, + created_at="now", + ) + + def test_restart_preserves_publication_ordinal_fence(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + store.publish_success( + identity=AuditPublicationIdentity(5, 5, BLOCK_B), + report=self.report(block_height=5), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + restarted = self.make_store(root) + stale = restarted.publish_success( + identity=AuditPublicationIdentity(4, 6, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="later", + ) + self.assertFalse(stale.published) + self.assertEqual(restarted.latest_evidence()["block_hash"], BLOCK_B) # type: ignore[index] + + def test_invalid_evidence_can_be_repaired_by_durable_publication(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "evidence.json").write_text("broken", encoding="utf-8") + store = self.make_store(root) + publication = store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + self.assertTrue(publication.published) + self.assertEqual(self.make_store(root).latest_evidence()["block_hash"], BLOCK_A) # type: ignore[index] + + def test_startup_rejects_coinbase_tampered_evidence(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + payload = json.loads(store.evidence_path.read_text(encoding="utf-8")) + payload["coinbase_txid"] = "99" * 32 + store.evidence_path.write_text(json.dumps(payload), encoding="utf-8") + self.assertIsNone(self.make_store(root).latest_evidence()) + + def test_startup_rejects_noncanonical_durable_identity_fields(self) -> None: + canonical_digest = "ab" * 32 + + def publish_fixture(root: Path, *, with_body: bool = False) -> AuditArtifactStore: + store = self.make_store(root) + persistence = self.persistence(canonical_digest) + if with_body: + persistence["body_uri"] = str( + store.body_path(BLOCK_A, canonical_digest) + ) + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(canonical_digest), + persistence=persistence, + evidence={}, + created_at="now", + ) + return store + + cases = ( + ("height-string", lambda evidence, _envelope: evidence.__setitem__("block_height", "1")), + ("height-bool", lambda evidence, _envelope: evidence.__setitem__("block_height", True)), + ("sequence-string", lambda evidence, _envelope: evidence["audit_publication_identity"].__setitem__("sequence", "1")), + ("sequence-bool", lambda evidence, _envelope: evidence["audit_publication_identity"].__setitem__("sequence", True)), + ("coinbase-value-string", lambda evidence, _envelope: evidence.__setitem__("coinbase_value_sats", "1")), + ("block-hash-uppercase", lambda evidence, _envelope: evidence.__setitem__("block_hash", BLOCK_A.upper())), + ("persistence-digest-uppercase", lambda evidence, _envelope: evidence["persistence"].__setitem__("audit_bundle_sha256", canonical_digest.upper())), + ("report-hash-uppercase", lambda evidence, _envelope: evidence["audit_report"].__setitem__("audit_bundle_sha256_hex", canonical_digest.upper())), + ("envelope-height-string", lambda _evidence, envelope: envelope.__setitem__("block_height", "1")), + ("envelope-value-bool", lambda _evidence, envelope: envelope.__setitem__("coinbase_value_sats", True)), + ) + for name, mutate in cases: + with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = publish_fixture(root) + evidence = json.loads(store.evidence_path.read_text(encoding="utf-8")) + envelope_path = Path(evidence["audit_bundle_path"]) + envelope = json.loads(envelope_path.read_text(encoding="utf-8")) + mutate(evidence, envelope) + store.evidence_path.write_text(json.dumps(evidence), encoding="utf-8") + envelope_path.write_text(json.dumps(envelope), encoding="utf-8") + self.assertIsNone(self.make_store(root).latest_evidence()) + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = publish_fixture(root, with_body=True) + evidence = json.loads(store.evidence_path.read_text(encoding="utf-8")) + envelope_path = Path(evidence["audit_bundle_path"]) + envelope = json.loads(envelope_path.read_text(encoding="utf-8")) + (root / "sub").mkdir() + body_name = Path(evidence["persistence"]["body_uri"]).name + noncanonical = str(root / "sub" / ".." / body_name) + evidence["persistence"]["body_uri"] = noncanonical + envelope["body_uri"] = noncanonical + store.evidence_path.write_text(json.dumps(evidence), encoding="utf-8") + envelope_path.write_text(json.dumps(envelope), encoding="utf-8") + self.assertIsNone(self.make_store(root).latest_evidence()) + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = publish_fixture(root) + evidence = json.loads(store.evidence_path.read_text(encoding="utf-8")) + original_cwd = Path.cwd() + try: + os.chdir(root) + evidence["audit_bundle_path"] = Path( + evidence["audit_bundle_path"] + ).name + store.evidence_path.write_text(json.dumps(evidence), encoding="utf-8") + self.assertIsNone(self.make_store(root).latest_evidence()) + finally: + os.chdir(original_cwd) + + def test_prune_failure_does_not_undo_durable_publication(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + with mock.patch.object(store, "prune_best_effort", side_effect=OSError("prune")): + result = store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(block_height=1), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + self.assertTrue(result.published) + self.assertEqual(store.latest_evidence()["block_hash"], BLOCK_A) # type: ignore[index] + + def test_evidence_failure_preserves_old_reference_and_skips_prune(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp), live_bundle_retention=0) + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={"audit_report": self.report(), "persistence": self.persistence()}, + created_at="old", + ) + old = store.evidence_path.read_bytes() + original = store._write_mutable_json + + def fail_evidence(path: Path, *args: object, **kwargs: object) -> object: + if path == store.evidence_path: + raise OSError("evidence") + return original(path, *args, **kwargs) + + with mock.patch.object(store, "_write_mutable_json", side_effect=fail_evidence): + with self.assertRaises(OSError): + store.publish_success( + identity=AuditPublicationIdentity(2, 2, BLOCK_B), + report=self.report(block_height=2), + persistence=self.persistence(), + evidence={"audit_report": self.report(), "persistence": self.persistence()}, + created_at="new", + ) + self.assertEqual(store.evidence_path.read_bytes(), old) + self.assertTrue( + store.live_envelope_path(block_height=1, block_hash=BLOCK_A).exists() + ) + + def test_publication_fsyncs_each_parent_in_commit_order(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + root = base / "artifacts" + evidence_path = base / "state" / "evidence.json" + store = self.make_store(root, evidence_path=evidence_path) + fsync_parents: list[Path] = [] + real_fsync = store._fsync_directory + + def record(parent: Path) -> None: + fsync_parents.append(parent) + real_fsync(parent) + + with mock.patch.object( + store, + "_fsync_directory", + side_effect=record, + ): + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + + self.assertGreaterEqual(len(fsync_parents), 2) + self.assertEqual(fsync_parents[:2], [root.resolve(), evidence_path.parent.resolve()]) + + def test_second_parent_fsync_failure_preserves_recoverable_state(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + root = base / "artifacts" + evidence_path = base / "state" / "evidence.json" + store = self.make_store(root, evidence_path=evidence_path) + fsync_parents: list[Path] = [] + real_fsync = store._fsync_directory + failed = False + + def fail_evidence_parent_once(parent: Path) -> None: + nonlocal failed + fsync_parents.append(parent) + if parent == evidence_path.parent.resolve() and not failed: + failed = True + raise OSError("evidence parent fsync") + real_fsync(parent) + + envelope = store.live_envelope_path( + block_height=1, + block_hash=BLOCK_A, + ) + with mock.patch.object( + store, + "_fsync_directory", + side_effect=fail_evidence_parent_once, + ), self.assertRaisesRegex(OSError, "evidence parent fsync"): + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + + self.assertEqual( + fsync_parents[:2], + [root.resolve(), evidence_path.parent.resolve()], + ) + self.assertFalse(envelope.exists()) + self.assertFalse(evidence_path.exists()) + restarted = self.make_store(root, evidence_path=evidence_path) + self.assertIsNone(restarted.latest_evidence()) + + def test_evidence_failure_preserves_competing_envelope_replacement(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp), live_bundle_retention=0) + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="old", + ) + competitor_path = store.live_envelope_path( + block_height=2, + block_hash=BLOCK_B, + ) + original = store._write_mutable_json + + def replace_then_fail( + path: Path, + *args: object, + **kwargs: object, + ) -> object: + if path == store.evidence_path: + competitor_path.unlink() + competitor_path.write_bytes(b"competitor") + raise OSError("evidence") + return original(path, *args, **kwargs) + + with mock.patch.object( + store, + "_write_mutable_json", + side_effect=replace_then_fail, + ), self.assertRaises(OSError): + store.publish_success( + identity=AuditPublicationIdentity(2, 2, BLOCK_B), + report=self.report(block_height=2), + persistence=self.persistence(), + evidence={}, + created_at="new", + ) + + self.assertEqual(competitor_path.read_bytes(), b"competitor") + self.assertEqual(store.latest_evidence()["block_hash"], BLOCK_A) # type: ignore[index] + + def test_directory_fsync_failure_rolls_back_previous_bytes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + path = store.root / "operator.json" + path.write_bytes(b"old") + real = store._fsync_directory + calls = 0 + + def fail_once(parent: Path) -> None: + nonlocal calls + calls += 1 + if calls == 1: + raise OSError("dir fsync") + real(parent) + + with mock.patch.object(store, "_fsync_directory", side_effect=fail_once): + with self.assertRaises(OSError): + store._write_mutable_bytes(path, b"new") + self.assertEqual(path.read_bytes(), b"old") + + def test_directory_fsync_rollback_preserves_competing_replacement(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + path = store.root / "mutable.json" + path.write_bytes(b"old") + real = store._fsync_directory + calls = 0 + + def swap_then_fail(parent: Path) -> None: + nonlocal calls + calls += 1 + if calls == 1: + path.unlink() + path.write_bytes(b"competitor") + raise OSError("dir fsync") + real(parent) + + with mock.patch.object(store, "_fsync_directory", side_effect=swap_then_fail): + with self.assertRaises(OSError): + store._write_mutable_bytes(path, b"new") + self.assertEqual(path.read_bytes(), b"competitor") + + def test_identity_cleanup_never_moves_an_existing_replacement(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + path = root / "owned.json" + path.write_bytes(b"owned-a") + owned = _FileIdentity.from_stat(path.stat()) + path.unlink() + path.write_bytes(b"competitor-b") + replacement = path.stat() + # Force the Linux ABA case even on filesystems that do not + # immediately reuse the unlinked inode in this test. + owned = dataclass_replace( + owned, + device=replacement.st_dev, + inode=replacement.st_ino, + ) + with mock.patch.object( + store, + "_owned_replace", + side_effect=AssertionError("replacement must not move"), + ): + self.assertFalse(store._remove_identity_safe(path, owned)) + + self.assertEqual(path.read_bytes(), b"competitor-b") + quarantines = list(root.glob(".owned.json.*.cleanup")) + self.assertEqual(quarantines, []) + + def test_identity_cleanup_never_relocates_nonregular_replacement(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + path = root / "owned.json" + path.write_bytes(b"owned") + owned = _FileIdentity.from_stat(path.stat()) + path.unlink() + path.mkdir() + + self.assertFalse(store._remove_identity_safe(path, owned)) + self.assertTrue(path.is_dir()) + self.assertEqual(list(root.glob(".owned.json.*.cleanup")), []) + + path.rmdir() + target = root / "operator" + target.write_bytes(b"operator") + path.symlink_to(target) + self.assertFalse(store._remove_identity_safe(path, owned)) + self.assertTrue(path.is_symlink()) + self.assertEqual(path.read_bytes(), b"operator") + + def test_atomic_stage_faults_preserve_target_and_clean_owned_temps(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + path = store.root / "mutable.json" + path.write_bytes(b"old") + with mock.patch("os.fsync", side_effect=OSError("file fsync")): + with self.assertRaises(OSError): + store._write_mutable_bytes(path, b"new") + self.assertEqual(path.read_bytes(), b"old") + self.assertEqual(list(store.root.glob(".*.tmp")), []) + + real_replace = os.replace + + def fail_publish( + source: object, + target: object, + **kwargs: object, + ) -> None: + target_path = Path(target) + if kwargs.get("dst_dir_fd") == store._root_fd: + target_path = store.root / target_path + if target_path == path: + raise OSError("replace") + real_replace(source, target, **kwargs) + + with mock.patch("os.replace", side_effect=fail_publish): + with self.assertRaises(OSError): + store._write_mutable_bytes(path, b"new") + self.assertEqual(path.read_bytes(), b"old") + self.assertEqual(list(store.root.glob(".*.tmp")), []) + + token = "66" * 16 + stage = store.root / f".{path.name}.{token}.tmp" + stage.write_bytes(b"preexisting") + with mock.patch( + "lab.prism.audit_artifacts.uuid.uuid4", + return_value=mock.Mock(hex=token), + ): + with self.assertRaises(FileExistsError): + store._write_mutable_bytes(path, b"new") + self.assertEqual(stage.read_bytes(), b"preexisting") + self.assertEqual(path.read_bytes(), b"old") + + def test_immutable_primitive_rejects_arbitrary_owned_root_targets(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + target = store.root / "operator-selected.json" + with self.assertRaisesRegex(RuntimeError, "owned body or segment"): + store._write_immutable_bytes(target, b"payload") + self.assertFalse(target.exists()) + + def test_immutable_publish_fsync_failure_is_recoverable_and_idempotent(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + target = store.body_path(BLOCK_A, DIGEST) + real_fsync = store._fsync_directory + calls = 0 + + def fail_once(parent: Path) -> None: + nonlocal calls + calls += 1 + if calls == 1: + raise OSError("dir fsync") + real_fsync(parent) + + with mock.patch.object( + store, + "_fsync_directory", + side_effect=fail_once, + ), self.assertRaisesRegex(OSError, "dir fsync"): + store._write_immutable_bytes(target, b"payload") + self.assertFalse(target.exists()) + self.assertEqual(list(store.root.glob(".*.tmp")), []) + + store._write_immutable_bytes(target, b"payload") + with mock.patch.object( + store, + "_fsync_directory", + wraps=real_fsync, + ) as fsync: + store._write_immutable_bytes(target, b"payload") + fsync.assert_called_once_with(target.parent) + with self.assertRaisesRegex(RuntimeError, "does not match"): + store._write_immutable_bytes(target, b"different") + self.assertEqual(target.read_bytes(), b"payload") + + def test_compatibility_none_override_is_stable_and_defensive(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={"nested": {"value": 1}}, + created_at="now", + ) + store.set_latest_evidence_for_compatibility(None) + self.assertIsNone(store.latest_evidence()) + seed = {"nested": {"value": 2}} + store.set_latest_evidence_for_compatibility(seed) + seed["nested"]["value"] = 3 + first = store.latest_evidence() + assert first is not None + self.assertEqual(first["nested"]["value"], 2) + first["nested"]["value"] = 4 + self.assertEqual(store.latest_evidence()["nested"]["value"], 2) # type: ignore[index] + + def test_compact_body_and_segment_headers_are_identity_bound(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store( + Path(tmp), + canonicalizer=lambda value: json.dumps( + value, + separators=(",", ":"), + ).encode(), + share_segment_size=2, + ) + logical = {"schema": "example", "shares": [{"share_seq": 1}]} + digest = hashlib.sha256( + json.dumps(logical, separators=(",", ":")).encode() + ).hexdigest() + body_path = store.body_path(BLOCK_A, digest) + wrapper = { + "schema": "qbit.prism.audit-body-ref.v1", + "block_hash": BLOCK_B, + "audit_bundle_sha256": digest, + "audit_bundle_schema": "example", + "share_count": 1, + "share_segment_size": 2, + "shares_key_index": 1, + "bundle_without_shares": {"schema": "example"}, + "share_parts": [{ + "kind": "inline", + "first_share_seq": 1, + "last_share_seq": 1, + "share_count": 1, + "shares": [{"share_seq": 1}], + }], + } + body_path.write_bytes(store.storage_json_bytes(wrapper)) + with self.assertRaisesRegex(RuntimeError, "identity mismatch"): + store.read_external_body(str(body_path), expected_sha256=digest) + + segment = { + "schema": "qbit.prism.audit-share-segment.v1", + "first_share_seq": 1, + "last_share_seq": 2, + "share_count": 2, + "shares": [{"share_seq": 1}], + } + segment_bytes = store.storage_json_bytes(segment) + segment_digest = hashlib.sha256(segment_bytes).hexdigest() + segment_path = store.root / ( + f"prism-audit-share-segment-1-2-{segment_digest}.json" + ) + segment_path.write_bytes(segment_bytes) + with self.assertRaisesRegex(RuntimeError, "header mismatch"): + store.read_audit_share_segment( + { + "kind": "segment", + "first_share_seq": 1, + "last_share_seq": 2, + "share_count": 2, + "sha256": segment_digest, + "body_uri": str(segment_path), + }, + parent_body_uri=str(body_path), + ) + + def test_memory_ledger_reactivation_preserves_publication_ordinal(self) -> None: + ledger = SingleWriterShareLedger() + self.assertEqual(ledger.audit_publication_sequence_floor(), 0) + + def persist(block_hash: str, height: int) -> None: + ledger.persist_accepted_block( + block_hash=block_hash, + block_height=height, + parent_hash=BLOCK_B, + final_bundle={}, + audit_report={}, + ) + + persist(BLOCK_A, 1) + self.assertEqual( + ledger.pool_block_state(block_hash=BLOCK_A)["chain_state"], # type: ignore[index] + "prepared", + ) + first = ledger.confirm_accepted_block(block_hash=BLOCK_A, active_tip_height=1) + replay = ledger.confirm_accepted_block(block_hash=BLOCK_A, active_tip_height=1) + self.assertEqual(ledger.audit_publication_sequence_floor(), 1) + self.assertEqual( + first["audit_publication_sequence"], + replay["audit_publication_sequence"], + ) + persist(BLOCK_A, 1) + self.assertEqual( + ledger.pool_block_state(block_hash=BLOCK_A)["chain_state"], # type: ignore[index] + "confirmed", + ) + self.assertEqual( + ledger.mark_pool_block_inactive(block_hash=BLOCK_A, active_tip_height=2)[ + "inactive_count" + ], + 1, + ) + persist(BLOCK_A, 1) + inactive_state = ledger.pool_block_state(block_hash=BLOCK_A) + self.assertEqual(inactive_state["chain_state"], "inactive") # type: ignore[index] + self.assertEqual( + inactive_state["audit_publication_sequence"], # type: ignore[index] + first["audit_publication_sequence"], + ) + wrong_height = ledger.reactivate_pool_block( + block_hash=BLOCK_A, + active_tip_height=0, + ) + self.assertEqual(wrong_height["reactivated_count"], 0) + self.assertNotIn("audit_publication_sequence", wrong_height) + reactivated = ledger.reactivate_pool_block( + block_hash=BLOCK_A, + active_tip_height=1, + ) + self.assertEqual( + reactivated["audit_publication_sequence"], + first["audit_publication_sequence"], + ) + self.assertEqual(ledger.audit_publication_sequence_floor(), 1) + replay_reactivation = ledger.reactivate_pool_block( + block_hash=BLOCK_A, + active_tip_height=1, + ) + self.assertEqual(replay_reactivation["reactivated_count"], 0) + self.assertNotIn("audit_publication_sequence", replay_reactivation) + self.assertEqual( + ledger.mark_pool_block_inactive( + block_hash=BLOCK_A, + active_tip_height=2, + )["inactive_count"], + 1, + ) + self.assertEqual( + ledger.reverse_immature_block( + block_hash=BLOCK_A, + active_tip_height=2, + )["reversed_count"], + 1, + ) + persist(BLOCK_A, 1) + reversed_state = ledger.pool_block_state(block_hash=BLOCK_A) + self.assertEqual(reversed_state["chain_state"], "reversed") # type: ignore[index] + self.assertEqual(reversed_state["maturity_state"], "reversed") # type: ignore[index] + self.assertEqual( + reversed_state["audit_publication_sequence"], # type: ignore[index] + reactivated["audit_publication_sequence"], + ) + self.assertEqual( + ledger.confirm_accepted_block( + block_hash=BLOCK_A, + active_tip_height=1, + ), + {"backend": "memory", "confirmed_count": 0}, + ) + self.assertEqual( + ledger.reactivate_pool_block( + block_hash=BLOCK_A, + active_tip_height=1, + ), + {"backend": "memory", "reactivated_count": 0}, + ) + persist(BLOCK_B, 2) + next_publication = ledger.confirm_accepted_block( + block_hash=BLOCK_B, + active_tip_height=2, + ) + self.assertEqual(next_publication["audit_publication_sequence"], 2) + self.assertEqual(ledger.audit_publication_sequence_floor(), 2) + with self.assertRaisesRegex(RuntimeError, "mature pool block"): + ledger.reverse_immature_block( + block_hash=BLOCK_B, + active_tip_height=1002, + ) + rejected_hash = "cc" * 32 + persist(rejected_hash, 3) + self.assertEqual( + ledger.reject_prepared_block( + block_hash=rejected_hash, + active_tip_height=3, + )["rejected_count"], + 1, + ) + self.assertEqual( + ledger.reject_prepared_block( + block_hash=rejected_hash, + active_tip_height=3, + )["rejected_count"], + 0, + ) + rejected_state = ledger.pool_block_state(block_hash=rejected_hash) + self.assertEqual(rejected_state["chain_state"], "rejected") # type: ignore[index] + self.assertEqual(rejected_state["maturity_state"], "reversed") # type: ignore[index] + + def test_memory_ledger_rejects_unknown_wrong_height_and_inactive_confirmation(self) -> None: + ledger = SingleWriterShareLedger() + self.assertEqual( + ledger.confirm_accepted_block( + block_hash=BLOCK_A, + active_tip_height=1, + )["confirmed_count"], + 0, + ) + ledger.persist_accepted_block( + block_hash=BLOCK_A, + block_height=1, + parent_hash=BLOCK_B, + final_bundle={}, + audit_report={}, + ) + self.assertEqual( + ledger.confirm_accepted_block( + block_hash=BLOCK_A, + active_tip_height=2, + )["confirmed_count"], + 0, + ) + ledger.confirm_accepted_block(block_hash=BLOCK_A, active_tip_height=1) + self.assertEqual( + ledger.mark_pool_block_inactive( + block_hash=BLOCK_A, + active_tip_height=2, + )["inactive_count"], + 1, + ) + self.assertEqual( + ledger.mark_pool_block_inactive( + block_hash=BLOCK_A, + active_tip_height=2, + )["inactive_count"], + 0, + ) + inactive_confirmation = ledger.confirm_accepted_block( + block_hash=BLOCK_A, + active_tip_height=1, + ) + self.assertEqual(inactive_confirmation["confirmed_count"], 0) + self.assertNotIn("audit_publication_sequence", inactive_confirmation) + + def test_invalid_evidence_disables_live_prune_but_not_candidate_prune(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "evidence.json").write_text("not-json", encoding="utf-8") + live = root / f"prism-live-audit-bundle-1-{BLOCK_A}.json" + live.write_text("{}", encoding="utf-8") + candidate = root / f"prism-live-audit-bundle-candidate-{BLOCK_A}.json" + candidate.write_text("{}", encoding="utf-8") + store = self.make_store( + root, + live_bundle_retention=0, + candidate_retention_seconds=0, + ) + result = store.prune_best_effort() + self.assertTrue(live.exists()) + self.assertFalse(candidate.exists()) + self.assertEqual(result.live_removed, 0) + self.assertEqual(result.candidate_removed, 1) + + def test_legacy_evidence_requires_durable_identity_upgrade(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + envelope = store.live_envelope_path(block_height=2, block_hash=BLOCK_B) + envelope.write_text( + json.dumps( + { + "schema": LIVE_ENVELOPE_SCHEMA, + "block_hash": BLOCK_B, + "block_height": 2, + "audit_bundle_sha256": DIGEST, + "body_uri": "", + "body_filename": None, + "coinbase_txid": "22" * 32, + "coinbase_manifest_sha256": "33" * 32, + "coinbase_tx_hex": "00", + "coinbase_value_sats": 1, + } + ), + encoding="utf-8", + ) + legacy = { + "schema": LIVE_EVIDENCE_SCHEMA, + "block_hash": BLOCK_B, + "block_height": 2, + "audit_bundle_path": str(envelope), + "audit_report": self.report(block_height=2), + "persistence": self.persistence(), + "coinbase_txid": "22" * 32, + "coinbase_manifest_sha256_hex": "33" * 32, + "coinbase_tx_hex": "00", + "coinbase_value_sats": 1, + } + store.evidence_path.write_text(json.dumps(legacy), encoding="utf-8") + restarted = self.make_store(root) + with self.assertRaisesRegex(RuntimeError, "validated publication identity"): + restarted.publish_success( + identity=AuditPublicationIdentity(10, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={"audit_report": self.report(), "persistence": self.persistence()}, + created_at="later", + ) + with restarted.publication_order_guard(): + restarted.adopt_legacy_publication_identity( + AuditPublicationIdentity(20, 2, BLOCK_B), + publication_floor_sequence=20, + ) + with self.assertRaisesRegex(RuntimeError, "never exact-replay"): + restarted.publish_success( + identity=AuditPublicationIdentity(10, 1, BLOCK_A), + report=self.report(block_height=1), + persistence=self.persistence(), + evidence={ + "audit_report": self.report(), + "persistence": self.persistence(), + }, + created_at="later", + ) + + # The disk marker never grants order or pin authority by itself. + second_process = self.make_store(root) + self.assertIsNone(second_process.latest_evidence()) + self.assertEqual(second_process.publication_sequence_floor(), 0) + self.assertEqual( + second_process.legacy_evidence_identity(), + AuditPublicationIdentity(20, 2, BLOCK_B), + ) + with second_process.publication_order_guard(): + second_process.adopt_legacy_publication_identity( + AuditPublicationIdentity(20, 2, BLOCK_B), + publication_floor_sequence=20, + ) + self.assertEqual(second_process.publication_sequence_floor(), 20) + self.assertEqual( + second_process.latest_evidence()["block_hash"], # type: ignore[index] + BLOCK_B, + ) + for stale_identity in ( + AuditPublicationIdentity(20, 2, BLOCK_B), + AuditPublicationIdentity(19, 3, BLOCK_A), + ): + with self.subTest(stale_identity=stale_identity), self.assertRaisesRegex( + RuntimeError, + "never exact-replay", + ): + second_process.publish_success( + identity=stale_identity, + report=self.report(block_height=stale_identity.block_height), + persistence=self.persistence(), + evidence={}, + created_at="stale", + ) + repair = second_process.publish_success( + identity=AuditPublicationIdentity(21, 3, BLOCK_A), + report=self.report(block_height=3), + persistence=self.persistence(), + evidence={}, + created_at="repair", + ) + self.assertTrue(repair.published) + + def test_legacy_proof_token_is_revoked_by_peer_inode_replacement(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + seed = self.make_store(root) + envelope = seed.live_envelope_path( + block_height=2, + block_hash=BLOCK_B, + ) + envelope.write_text( + json.dumps( + { + "schema": LIVE_ENVELOPE_SCHEMA, + "block_hash": BLOCK_B, + "block_height": 2, + "audit_bundle_sha256": DIGEST, + "body_uri": "", + "body_filename": None, + "coinbase_txid": "22" * 32, + "coinbase_manifest_sha256": "33" * 32, + "coinbase_tx_hex": "00", + "coinbase_value_sats": 1, + } + ), + encoding="utf-8", + ) + seed.evidence_path.write_text( + json.dumps( + { + "schema": LIVE_EVIDENCE_SCHEMA, + "block_hash": BLOCK_B, + "block_height": 2, + "audit_bundle_path": str(envelope), + "audit_report": self.report(block_height=2), + "persistence": self.persistence(), + "coinbase_txid": "22" * 32, + "coinbase_manifest_sha256_hex": "33" * 32, + "coinbase_tx_hex": "00", + "coinbase_value_sats": 1, + } + ), + encoding="utf-8", + ) + seed.close() + identity = AuditPublicationIdentity(20, 2, BLOCK_B) + proven = self.make_store(root) + with proven.publication_order_guard(): + proven.adopt_legacy_publication_identity( + identity, + publication_floor_sequence=20, + ) + self.assertIsNotNone(proven._legacy_proof_token) + self.assertEqual(proven.publication_sequence_floor(), 20) + original_evidence_inode = _FileIdentity.from_stat( + proven.evidence_path.stat() + ) + + peer = self.make_store(root) + with peer.publication_order_guard(): + peer.adopt_legacy_publication_identity( + identity, + publication_floor_sequence=20, + ) + self.assertFalse( + original_evidence_inode.matches(proven.evidence_path.stat()) + ) + + proven.prune_best_effort() + self.assertIsNone(proven._legacy_proof_token) + self.assertIsNone(proven.latest_evidence()) + self.assertEqual(proven.publication_sequence_floor(), 0) + self.assertEqual(proven.legacy_evidence_identity(), identity) + + def test_unprovable_legacy_evidence_is_repairable(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + store._evidence_state = "legacy" + store._latest_evidence = {"block_hash": BLOCK_A, "block_height": 1} + store._current_identity = AuditPublicationIdentity(0, 1, BLOCK_A) + store.invalidate_unprovable_legacy_evidence() + repaired = store.publish_success( + identity=AuditPublicationIdentity(2, 2, BLOCK_B), + report=self.report(block_height=2), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + self.assertTrue(repaired.published) + + def test_disk_legacy_invalidation_is_sticky_until_durable_repair(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root, live_bundle_retention=0) + envelope = store.live_envelope_path(block_height=2, block_hash=BLOCK_B) + envelope.write_text( + json.dumps( + { + "schema": LIVE_ENVELOPE_SCHEMA, + "block_hash": BLOCK_B, + "block_height": 2, + "audit_bundle_sha256": DIGEST, + "body_uri": "", + "body_filename": None, + "coinbase_txid": "22" * 32, + "coinbase_manifest_sha256": "33" * 32, + "coinbase_tx_hex": "00", + "coinbase_value_sats": 1, + } + ), + encoding="utf-8", + ) + legacy = { + "schema": LIVE_EVIDENCE_SCHEMA, + "block_hash": BLOCK_B, + "block_height": 2, + "audit_bundle_path": str(envelope), + "audit_report": self.report(block_height=2), + "persistence": self.persistence(), + "coinbase_txid": "22" * 32, + "coinbase_manifest_sha256_hex": "33" * 32, + "coinbase_tx_hex": "00", + "coinbase_value_sats": 1, + } + store.evidence_path.write_text(json.dumps(legacy), encoding="utf-8") + restarted = self.make_store(root, live_bundle_retention=0) + restarted.invalidate_unprovable_legacy_evidence() + self.assertIsNone(restarted.latest_evidence()) + self.assertEqual(restarted.prune_best_effort().live_removed, 0) + repaired = restarted.publish_success( + identity=AuditPublicationIdentity(21, 3, BLOCK_A), + report=self.report(block_height=3), + persistence=self.persistence(), + evidence={}, + created_at="repair", + ) + self.assertTrue(repaired.published) + self.assertEqual(restarted.latest_evidence()["block_hash"], BLOCK_A) # type: ignore[index] + + def test_retention_preserves_current_and_unowned_entries(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root, live_bundle_retention=0) + publication = store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={"audit_report": self.report(), "persistence": self.persistence()}, + created_at="now", + ) + lookalike = root / f"prism-live-audit-bundle-1-{BLOCK_B}.json.bak" + lookalike.write_text("operator", encoding="utf-8") + link = root / f"prism-live-audit-bundle-2-{BLOCK_B}.json" + link.symlink_to(lookalike) + store.prune_best_effort() + self.assertTrue(publication.envelope_path.exists()) + self.assertTrue(lookalike.exists()) + self.assertTrue(link.is_symlink()) + + def test_retention_ties_pin_active_candidate_and_never_remove_bodies(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store( + root, + live_bundle_retention=2, + candidate_retention_seconds=0, + ) + current = store.publish_success( + identity=AuditPublicationIdentity(1, 0, BLOCK_A), + report=self.report(block_height=0), + persistence=self.persistence(), + evidence={}, + created_at="current", + ) + first = store.live_envelope_path(block_height=1, block_hash=BLOCK_A) + second = store.live_envelope_path(block_height=2, block_hash=BLOCK_B) + first.write_text("{}", encoding="utf-8") + second.write_text("{}", encoding="utf-8") + os.utime(first, ns=(1, 1)) + os.utime(second, ns=(1, 1)) + active = store.issue_candidate(block_hash=BLOCK_A) + active.path.write_bytes(b"active") + self.transfer_candidate(store, active) + body = store.body_path(BLOCK_A, DIGEST) + body.write_bytes(b"body") + result = store.prune_best_effort() + self.assertEqual(result.live_removed, 1) + self.assertTrue(current.envelope_path.exists()) + self.assertTrue(active.path.exists()) + self.assertTrue(body.exists()) + store.discard_candidate(active) + + def test_body_segment_and_evidence_symlinks_fail_no_follow(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + outside = root / "outside" + outside.write_text("{}", encoding="utf-8") + body = store.body_path(BLOCK_A, DIGEST) + body.symlink_to(outside) + with self.assertRaisesRegex(RuntimeError, "not retrievable"): + store.read_external_body(str(body), expected_sha256=DIGEST) + store.evidence_path.symlink_to(outside) + self.assertIsNone(self.make_store(root).latest_evidence()) + + def test_metrics_ignore_symlinks_and_classify_malformed_as_other(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + body = root / f"prism-audit-bundle-body-{BLOCK_A}-{DIGEST}.json" + body.write_bytes(b"body") + malformed = root / "prism-audit-bundle-body-operator.json" + malformed.write_bytes(b"x") + (root / f"prism-live-audit-bundle-1-{BLOCK_A}.json").symlink_to(body) + metrics = store.metrics_snapshot() + self.assertEqual(metrics["body"], {"files": 1, "bytes": 4}) + self.assertEqual(metrics["other"], {"files": 1, "bytes": 1}) + + def test_mutable_share_slot_merge_preserves_prior_range(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp), share_segment_size=4) + uri, _digest = store.write_audit_share_segment_range( + segment_first_share_seq=1, + segment_last_share_seq=4, + first_share_seq=1, + last_share_seq=2, + shares=[{"share_seq": 1}, {"share_seq": 2}], + ) + store.write_audit_share_segment_range( + segment_first_share_seq=1, + segment_last_share_seq=4, + first_share_seq=3, + last_share_seq=4, + shares=[{"share_seq": 3}, {"share_seq": 4}], + ) + payload = json.loads(Path(uri).read_text(encoding="utf-8")) + self.assertEqual([row["share_seq"] for row in payload["shares"]], [1, 2, 3, 4]) + + def test_concurrent_mutable_share_slot_merge_is_serialized_and_lossless( + self, + ) -> None: + rows = [ + { + "share_seq": share_seq, + "worker": f"miner-{share_seq}", + "accepted": True, + } + for share_seq in range(1, 5) + ] + scenarios = ( + ("disjoint", rows[:2], rows[2:]), + ("identical_overlap", rows[:3], rows[1:]), + ) + + for scenario, first_shares, second_shares in scenarios: + with self.subTest(scenario=scenario), tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp), share_segment_size=4) + first_name = f"audit-slot-{scenario}-first" + second_name = f"audit-slot-{scenario}-second" + first_at_write = threading.Event() + release_first = threading.Event() + second_lock_attempt = threading.Event() + second_lock_acquired = threading.Event() + second_at_write = threading.Event() + write_call_lock = threading.Lock() + write_call_threads: list[str] = [] + results: dict[str, tuple[str, str]] = {} + errors: dict[str, BaseException] = {} + original_lock = store._lock + original_write_mutable_bytes = store._write_mutable_bytes + + class ObservedLock: + def __enter__(self) -> "ObservedLock": + if threading.current_thread().name == second_name: + second_lock_attempt.set() + original_lock.acquire() + if threading.current_thread().name == second_name: + second_lock_acquired.set() + return self + + def __exit__( + self, + _exc_type: object, + _exc_value: object, + _traceback: object, + ) -> None: + original_lock.release() + + def gated_write(path: Path, payload: bytes) -> None: + thread_name = threading.current_thread().name + with write_call_lock: + write_call_threads.append(thread_name) + if thread_name == first_name: + first_at_write.set() + if not release_first.wait(timeout=5.0): + raise AssertionError("first share-slot writer was not released") + elif thread_name == second_name: + second_at_write.set() + original_write_mutable_bytes(path, payload) + + def write_range(label: str, shares: list[dict[str, object]]) -> None: + try: + results[label] = store.write_audit_share_segment_range( + segment_first_share_seq=1, + segment_last_share_seq=4, + first_share_seq=int(shares[0]["share_seq"]), + last_share_seq=int(shares[-1]["share_seq"]), + shares=shares, + ) + except BaseException as exc: + errors[label] = exc + + first_thread = threading.Thread( + target=write_range, + args=("first", first_shares), + name=first_name, + ) + second_thread = threading.Thread( + target=write_range, + args=("second", second_shares), + name=second_name, + ) + second_started = False + with mock.patch.object( + store, + "_lock", + ObservedLock(), + ), mock.patch.object( + store, + "_write_mutable_bytes", + side_effect=gated_write, + ): + first_thread.start() + try: + self.assertTrue( + first_at_write.wait(timeout=5.0), + "first writer did not reach the gated mutable write", + ) + second_thread.start() + second_started = True + self.assertTrue( + second_lock_attempt.wait(timeout=5.0), + "second writer did not attempt the store lock", + ) + self.assertTrue(first_thread.is_alive()) + self.assertTrue(second_thread.is_alive()) + self.assertFalse(second_lock_acquired.is_set()) + self.assertFalse(second_at_write.is_set()) + finally: + release_first.set() + first_thread.join(timeout=5.0) + if second_started: + second_thread.join(timeout=5.0) + + self.assertFalse(first_thread.is_alive()) + self.assertFalse(second_thread.is_alive()) + active_names = {thread.name for thread in threading.enumerate()} + self.assertNotIn(first_name, active_names) + self.assertNotIn(second_name, active_names) + self.assertEqual(errors, {}) + self.assertTrue(second_lock_acquired.is_set()) + self.assertTrue(second_at_write.is_set()) + self.assertEqual(write_call_threads, [first_name, second_name]) + + self.assertEqual(results["first"][0], results["second"][0]) + slot_path = Path(results["first"][0]) + expected_payload = store.audit_share_segment_payload( + first_share_seq=1, + last_share_seq=4, + shares=rows, + ) + expected_bytes = store.storage_json_bytes(expected_payload) + stored_bytes = slot_path.read_bytes() + self.assertEqual(stored_bytes, expected_bytes) + stored = json.loads(stored_bytes) + sequences = [int(row["share_seq"]) for row in stored["shares"]] + self.assertEqual(sequences, [1, 2, 3, 4]) + self.assertEqual(len(sequences), len(set(sequences))) + self.assertEqual(stored["shares"], rows) + + inputs = {"first": first_shares, "second": second_shares} + for label, shares in inputs.items(): + incoming_payload = store.audit_share_segment_payload( + first_share_seq=int(shares[0]["share_seq"]), + last_share_seq=int(shares[-1]["share_seq"]), + shares=shares, + ) + expected_digest = hashlib.sha256( + store.storage_json_bytes(incoming_payload) + ).hexdigest() + self.assertEqual(results[label][1], expected_digest) + + before_retry = slot_path.stat() + for label, shares in inputs.items(): + retry = store.write_audit_share_segment_range( + segment_first_share_seq=1, + segment_last_share_seq=4, + first_share_seq=int(shares[0]["share_seq"]), + last_share_seq=int(shares[-1]["share_seq"]), + shares=shares, + ) + self.assertEqual(retry, results[label]) + after_retry = slot_path.stat() + self.assertEqual(slot_path.read_bytes(), expected_bytes) + self.assertEqual( + (after_retry.st_dev, after_retry.st_ino, after_retry.st_mtime_ns), + (before_retry.st_dev, before_retry.st_ino, before_retry.st_mtime_ns), + ) + + conflicting_share = {**rows[1], "worker": "conflicting-miner"} + with self.assertRaisesRegex( + RuntimeError, + "conflicts at share_seq 2", + ): + store.write_audit_share_segment_range( + segment_first_share_seq=1, + segment_last_share_seq=4, + first_share_seq=2, + last_share_seq=2, + shares=[conflicting_share], + ) + after_conflict = slot_path.stat() + self.assertEqual(slot_path.read_bytes(), expected_bytes) + self.assertEqual( + ( + after_conflict.st_dev, + after_conflict.st_ino, + after_conflict.st_mtime_ns, + ), + (after_retry.st_dev, after_retry.st_ino, after_retry.st_mtime_ns), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_block_candidates.py b/tests/test_prism_block_candidates.py index 0c3d62e..962dd62 100644 --- a/tests/test_prism_block_candidates.py +++ b/tests/test_prism_block_candidates.py @@ -7,10 +7,639 @@ import contextlib from dataclasses import replace as dataclass_replace import unittest +from unittest import mock from tests.prism_vardiff_test_support import * +from lab.prism.audit_artifacts import ( + AuditArtifactStore, + AuditPublicationIdentity, + RetentionResult, +) +from lab.prism.bundle_compiler import BundleCompiler +from lab.prism.prism_coordinator import PrismCoordinator + + +_compat_verified_audit_report = verified_audit_report + + +def configure_temporary_audit_root( + test_case: unittest.TestCase, + server: PrismCoordinator, +) -> None: + temporary = tempfile.TemporaryDirectory() + server.audit_dir = Path(temporary.name) / "audit" + server.evidence_path = Path(temporary.name) / "state" / "evidence.json" + + def cleanup() -> None: + store = server.__dict__.get("_audit_artifact_store") + if isinstance(store, AuditArtifactStore): + store.close() + temporary.cleanup() + + test_case.addCleanup(cleanup) + + +def verified_audit_report( + coinbase_tx_hex: str = "c0ffee", + block_height: int = 10, +) -> dict[str, object]: + report = _compat_verified_audit_report(coinbase_tx_hex) + report["schema"] = "qbit.prism.audit-verification-report.v1" + report["block_height"] = block_height + report["coinbase_value_sats"] = 50_00000000 + return report class PrismCoordinatorVardiffTests(unittest.TestCase): + def test_audit_store_lazy_adopts_compatibility_fields_before_and_after_construction(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + server = PrismCoordinator.__new__(PrismCoordinator) + server.audit_dir = base / "audit-a" + server.evidence_path = base / "state-a" / "evidence.json" + server.audit_live_bundle_retention = 3 + server.audit_candidate_retention_seconds = 7 + server.audit_share_segment_size = 11 + first = server._ensure_audit_artifact_store() + self.assertEqual(first.root, server.audit_dir.resolve()) + self.assertEqual(first.evidence_path, server.evidence_path.resolve()) + self.assertEqual(first.live_bundle_retention, 3) + self.assertEqual(first.candidate_retention_seconds, 7) + self.assertEqual(first.share_segment_size, 11) + + server.audit_dir = base / "audit-b" + server.evidence_path = base / "state-b" / "evidence.json" + server.audit_live_bundle_retention = 5 + server.audit_candidate_retention_seconds = 13 + server.audit_share_segment_size = 17 + second = server._ensure_audit_artifact_store() + self.assertIs(second, first) + self.assertEqual(second.root, server.audit_dir.resolve()) + self.assertEqual(second.evidence_path, server.evidence_path.resolve()) + self.assertEqual(second.live_bundle_retention, 5) + self.assertEqual(second.candidate_retention_seconds, 13) + self.assertEqual(second.share_segment_size, 17) + + candidate = second.issue_candidate(block_hash="aa" * 32) + server.audit_dir = base / "audit-c" + with self.assertRaisesRegex(RuntimeError, "candidates are active"): + server._ensure_audit_artifact_store() + self.assertIs(server.__dict__["_audit_artifact_store"], first) + self.assertEqual(first.root, (base / "audit-b").resolve()) + first.discard_candidate(candidate) + + def test_coordinator_latest_evidence_seed_is_stable_and_defensive_before_and_after_store(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + server = PrismCoordinator.__new__(PrismCoordinator) + seed = {"nested": {"value": 1}} + server.latest_evidence = seed + seed["nested"]["value"] = 2 + before = server.latest_evidence + self.assertEqual(before, {"nested": {"value": 1}}) + assert before is not None + before["nested"]["value"] = 3 + self.assertEqual(server.latest_evidence, {"nested": {"value": 1}}) + + root = Path(tmp) + server.audit_dir = root / "audit" + server.evidence_path = root / "state" / "evidence.json" + store = server._ensure_audit_artifact_store() + self.assertNotIn("_audit_latest_evidence_seed", server.__dict__) + self.assertEqual(server.latest_evidence_payload(), {"nested": {"value": 1}}) + snapshot = server.latest_evidence_payload() + assert snapshot is not None + snapshot["nested"]["value"] = 4 # type: ignore[index] + self.assertEqual(server.latest_evidence_payload(), {"nested": {"value": 1}}) + server.latest_evidence = None + self.assertIsNone(server.latest_evidence_payload()) + server.latest_evidence = {"after": {"value": 5}} + self.assertEqual(server.latest_evidence_payload(), {"after": {"value": 5}}) + self.assertIs(server._ensure_audit_artifact_store(), store) + + def test_coordinator_audit_static_and_instance_facades_preserve_contract(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + sentinel = mock.Mock() + sentinel.metrics_snapshot.return_value = {"scan_error": 0} + sentinel.prune_best_effort.return_value = RetentionResult(live_removed=1) + sentinel.verify_bundle.return_value = {"verified": True} + server._ensure_audit_artifact_store = lambda: sentinel # type: ignore[method-assign] + + self.assertEqual( + PrismCoordinator.audit_artifact_kind( + f"prism-live-audit-bundle-1-{'aa' * 32}.json" + ), + "live_bundle", + ) + self.assertEqual(server.audit_artifact_metrics(), {"scan_error": 0}) + keep = Path("keep.json") + server.prune_audit_artifacts(keep_live_path=keep) + sentinel.prune_best_effort.assert_called_once_with(keep_live_path=keep) + result = PrismCoordinator.verify_bundle( + server, + Path("bundle.json"), + "00", + "11" * 32, + expected_coinbase_value_sats=5, + expected_block_height=6, + ) + self.assertEqual(result, {"verified": True}) + sentinel.verify_bundle.assert_called_once_with( + Path("bundle.json"), + "00", + "11" * 32, + expected_coinbase_value_sats=5, + expected_block_height=6, + ) + server.ledger_writer_public_key_hex = "22" * 32 + self.assertEqual( + server.trusted_ledger_writer_public_key_hex({}), + "22" * 32, + ) + + with tempfile.TemporaryDirectory() as tmp: + candidate = Path(tmp) / "candidate.json" + candidate.write_bytes(b"canonical") + digest = hashlib.sha256(b"canonical").hexdigest() + self.assertEqual( + PrismCoordinator.verified_canonical_bundle_path( + candidate, + {"audit_bundle_sha256_hex": digest.upper()}, + ), + candidate, + ) + + def test_candidate_verifier_override_is_resolved_after_store_construction(self) -> None: + server, state, ledger = submit_coordinator() + server.max_blocks = 10 + server.stop_after_block = False + with tempfile.TemporaryDirectory() as tmp: + server.audit_dir = Path(tmp) + server.evidence_path = Path(tmp) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + store = server._ensure_audit_artifact_store() + calls: list[str] = [] + + def verifier_a(*_args: object, **_kwargs: object) -> dict[str, object]: + calls.append("a") + return verified_audit_report() + + def verifier_b(*_args: object, **_kwargs: object) -> dict[str, object]: + calls.append("b") + return verified_audit_report() + + server.verify_bundle = verifier_a # type: ignore[method-assign] + server.verify_bundle = verifier_b # type: ignore[method-assign] + block_hash = "91" * 32 + server.rpc = SubmitRpc( + tip="00" * 32, + block_hash=block_hash, + ledger=ledger, + ) + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() + ) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + ) + self.assertTrue( + server.submit_block_candidate( + block_candidate(server, state, submission) + ) + ) + self.assertIs(server._ensure_audit_artifact_store(), store) + self.assertEqual(calls, ["b"]) + + direct_server, direct_state, direct_ledger = submit_coordinator() + direct_server.max_blocks = 10 + direct_server.stop_after_block = False + direct_server.audit_dir = Path(tmp) / "direct-a1" + direct_server.evidence_path = Path(tmp) / "direct-a1-evidence.json" + direct_server.ledger_writer_public_key_hex = "aa" * 32 + direct_store = direct_server._ensure_audit_artifact_store() + direct_calls: list[str] = [] + + def direct_a1_verifier( + *_args: object, + **_kwargs: object, + ) -> dict[str, object]: + direct_calls.append("a1") + return verified_audit_report() + + direct_store.verify_bundle = direct_a1_verifier # type: ignore[method-assign] + self.assertNotIn("verify_bundle", direct_server.__dict__) + direct_hash = "93" * 32 + direct_server.rpc = SubmitRpc( + tip="00" * 32, + block_hash=direct_hash, + ledger=direct_ledger, + ) + direct_server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() + ) + direct_submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=direct_hash, + block_hex="00", + ) + self.assertTrue( + direct_server.submit_block_candidate( + block_candidate( + direct_server, + direct_state, + direct_submission, + ) + ) + ) + self.assertEqual(direct_calls, ["a1"]) + + def test_audit_publication_occurs_after_durable_confirm_and_before_success_tail(self) -> None: + server, state, ledger = submit_coordinator() + events: list[str] = [] + with tempfile.TemporaryDirectory() as tmp: + server.audit_dir = Path(tmp) + server.evidence_path = Path(tmp) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + store = server._ensure_audit_artifact_store() + payout_service = server._ensure_payout_state_service() + real_persist = ledger.persist_accepted_block + real_confirm = ledger.confirm_accepted_block + real_floor = ledger.audit_publication_sequence_floor + real_identity = server._audit_publication_identity + real_publish = store.publish_success + real_shutdown = server.request_shutdown + + def verify(*_args: object, **_kwargs: object) -> dict[str, object]: + events.append("verify") + return verified_audit_report() + + def persist(**kwargs: object) -> dict[str, object]: + events.append("persist") + return real_persist(**kwargs) + + def confirm(**kwargs: object) -> dict[str, object]: + self.assertTrue( + payout_service.balance_mutation_lock._is_owned(), # type: ignore[attr-defined] + ) + self.assertEqual(store._publication_guard_owner, threading.get_ident()) + events.append("confirm") + return real_confirm(**kwargs) + + def publication_identity(**kwargs: object) -> AuditPublicationIdentity: + self.assertTrue( + payout_service.balance_mutation_lock._is_owned(), # type: ignore[attr-defined] + ) + self.assertEqual(store._publication_guard_owner, threading.get_ident()) + events.append("publication_identity") + return real_identity(**kwargs) + + def publication_floor() -> int: + self.assertTrue( + payout_service.balance_mutation_lock._is_owned(), # type: ignore[attr-defined] + ) + self.assertEqual(store._publication_guard_owner, threading.get_ident()) + events.append("publication_floor") + return real_floor() + + def publish(**kwargs: object) -> object: + self.assertTrue( + payout_service.balance_mutation_lock._is_owned(), # type: ignore[attr-defined] + ) + self.assertEqual(store._publication_guard_owner, threading.get_ident()) + events.append("publish_success") + return real_publish(**kwargs) + + def terminal_success() -> None: + events.append("terminal_success") + real_shutdown() + + server.verify_bundle = verify # type: ignore[method-assign] + ledger.persist_accepted_block = persist # type: ignore[method-assign] + ledger.confirm_accepted_block = confirm # type: ignore[method-assign] + ledger.audit_publication_sequence_floor = publication_floor # type: ignore[method-assign] + server._audit_publication_identity = publication_identity # type: ignore[method-assign] + store.publish_success = publish # type: ignore[method-assign] + server.request_shutdown = terminal_success # type: ignore[method-assign] + block_hash = "92" * 32 + server.rpc = SubmitRpc( + tip="00" * 32, + block_hash=block_hash, + ledger=ledger, + ) + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() + ) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + ) + self.assertTrue( + server.submit_block_candidate( + block_candidate(server, state, submission) + ) + ) + self.assertEqual( + events, + [ + "verify", + "persist", + "confirm", + "publication_identity", + "publication_floor", + "publish_success", + "terminal_success", + ], + ) + + def test_audit_publication_failure_after_confirm_retries_same_ordinal_once(self) -> None: + server, state, ledger = submit_coordinator() + block_hash = "94" * 32 + with tempfile.TemporaryDirectory() as tmp: + server.audit_dir = Path(tmp) + server.evidence_path = Path(tmp) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + store = server._ensure_audit_artifact_store() + server.rpc = SubmitRpc( + tip="00" * 32, + block_hash=block_hash, + ledger=ledger, + ) + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() + ) + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: verified_audit_report() + ) + real_confirm = ledger.confirm_accepted_block + real_publish = store.publish_success + real_shutdown = server.request_shutdown + confirmation_sequences: list[int] = [] + publication_sequences: list[int] = [] + terminal_successes: list[str] = [] + publication_attempts = 0 + + def confirm(**kwargs: object) -> dict[str, object]: + result = real_confirm(**kwargs) + result["audit_publication_sequence"] = 7 + confirmation_sequences.append(7) + return result + + def publish(**kwargs: object) -> object: + nonlocal publication_attempts + publication_attempts += 1 + identity = kwargs["identity"] + assert isinstance(identity, AuditPublicationIdentity) + publication_sequences.append(identity.sequence) + if publication_attempts == 1: + raise RuntimeError("injected evidence publication failure") + return real_publish(**kwargs) + + def terminal_success() -> None: + terminal_successes.append("success") + real_shutdown() + + ledger.confirm_accepted_block = confirm # type: ignore[method-assign] + ledger.audit_publication_sequence_floor = lambda: 7 # type: ignore[method-assign] + store.publish_success = publish # type: ignore[method-assign] + server.request_shutdown = terminal_success # type: ignore[method-assign] + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + ) + candidate = block_candidate(server, state, submission) + + with self.assertRaisesRegex( + RuntimeError, + "injected evidence publication failure", + ): + server.submit_block_candidate(candidate) + self.assertEqual(confirmation_sequences, [7]) + self.assertEqual(publication_sequences, [7]) + self.assertIsNone(store.latest_evidence()) + self.assertEqual(server.accepted_block_count, 0) + self.assertEqual(terminal_successes, []) + + assert isinstance(server.rpc, SubmitRpc) + server.rpc.tip = block_hash + self.assertTrue(server.submit_block_candidate(candidate)) + self.assertEqual(confirmation_sequences, [7, 7]) + self.assertEqual(publication_sequences, [7, 7]) + self.assertEqual(server.accepted_block_count, 1) + self.assertEqual(terminal_successes, ["success"]) + latest = store.latest_evidence() + self.assertIsNotNone(latest) + assert latest is not None + self.assertEqual( + latest["audit_publication_identity"]["sequence"], # type: ignore[index] + 7, + ) + + def test_publication_floor_failure_or_mismatch_never_publishes_evidence(self) -> None: + cases = ( + ("query_failure", RuntimeError("floor query failed"), "floor query failed"), + ("below_identity", 0, "exceeds"), + ("above_identity", 2, "behind"), + ("invalid_bool", True, "floor sequence"), + ) + for name, floor_result, error_pattern in cases: + with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp: + server, state, ledger = submit_coordinator() + block_hash = "95" * 32 + server.audit_dir = Path(tmp) + server.evidence_path = Path(tmp) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + store = server._ensure_audit_artifact_store() + server.rpc = SubmitRpc( + tip="00" * 32, + block_hash=block_hash, + ledger=ledger, + ) + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() + ) + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: verified_audit_report() + ) + publish_calls = 0 + real_publish = store.publish_success + + def publication_floor() -> int: + if isinstance(floor_result, BaseException): + raise floor_result + return floor_result # type: ignore[return-value] + + def publish(**kwargs: object) -> object: + nonlocal publish_calls + publish_calls += 1 + return real_publish(**kwargs) + + ledger.audit_publication_sequence_floor = publication_floor # type: ignore[method-assign] + store.publish_success = publish # type: ignore[method-assign] + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + ) + with self.assertRaisesRegex( + (RuntimeError, ValueError), + error_pattern, + ): + server.submit_block_candidate( + block_candidate(server, state, submission) + ) + self.assertEqual( + publish_calls, + 0 if name == "query_failure" else 1, + ) + self.assertIsNone(store.latest_evidence()) + self.assertEqual(server.accepted_block_count, 0) + + def test_make_ledger_preserves_single_a1_store_identity(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + server = PrismCoordinator.__new__(PrismCoordinator) + server.audit_dir = Path(tmp) / "audit-a" + server.evidence_path = Path(tmp) / "state-a" / "evidence.json" + store = server._ensure_audit_artifact_store() + ledger = SimpleNamespace() + with mock.patch.dict( + os.environ, + {"PRISM_POSTGRES_PSQL_COMMAND": "psql example"}, + clear=True, + ), mock.patch( + "lab.prism.prism_coordinator.PsqlShareLedger", + return_value=ledger, + ) as constructor: + self.assertIs(server.make_ledger(), ledger) + self.assertIs( + constructor.call_args.kwargs["audit_artifact_store"], + store, + ) + server.audit_dir = Path(tmp) / "audit-b" + server.evidence_path = Path(tmp) / "state-b" / "evidence.json" + self.assertIs(server._ensure_audit_artifact_store(), store) + self.assertEqual(store.root, server.audit_dir.resolve()) + + memory_server = PrismCoordinator.__new__(PrismCoordinator) + with mock.patch.dict( + os.environ, + {"PRISM_ALLOW_MEMORY_LEDGER": "1"}, + clear=True, + ): + self.assertIsInstance( + memory_server.make_ledger(), + SingleWriterShareLedger, + ) + self.assertNotIn("_audit_artifact_store", memory_server.__dict__) + + def test_legacy_evidence_upgrade_requires_exact_durable_pool_block_proof(self) -> None: + legacy_identity = AuditPublicationIdentity(0, 10, "aa" * 32) + + for maturity_state in ("immature", "mature"): + with self.subTest(valid_maturity=maturity_state): + server = coordinator() + store = mock.MagicMock() + store.legacy_evidence_identity.return_value = legacy_identity + server._ensure_audit_artifact_store = lambda: store # type: ignore[method-assign] + server.ledger = SimpleNamespace( + audit_publication_sequence_floor=lambda: 7, + pool_block_state=lambda **_kwargs: { + "block_hash": legacy_identity.block_hash, + "block_height": legacy_identity.block_height, + "chain_state": "confirmed", + "maturity_state": maturity_state, + "audit_publication_sequence": 7, + } + ) + server._upgrade_legacy_audit_evidence() + store.adopt_legacy_publication_identity.assert_called_once_with( + AuditPublicationIdentity(7, 10, legacy_identity.block_hash), + publication_floor_sequence=7, + ) + store.invalidate_unprovable_legacy_evidence.assert_not_called() + + invalid_states = ( + None, + { + "block_height": 10, + "chain_state": "confirmed", + "maturity_state": "immature", + "audit_publication_sequence": 7, + }, + { + "block_hash": legacy_identity.block_hash.upper(), + "block_height": 10, + "chain_state": "confirmed", + "maturity_state": "immature", + "audit_publication_sequence": 7, + }, + { + "block_hash": legacy_identity.block_hash, + "block_height": "10", + "chain_state": "confirmed", + "maturity_state": "immature", + "audit_publication_sequence": 7, + }, + { + "block_hash": legacy_identity.block_hash, + "block_height": True, + "chain_state": "confirmed", + "maturity_state": "immature", + "audit_publication_sequence": 7, + }, + { + "block_height": 10, + "chain_state": "inactive", + "maturity_state": "immature", + "audit_publication_sequence": 7, + }, + { + "block_height": 10, + "chain_state": "confirmed", + "maturity_state": "reversed", + "audit_publication_sequence": 7, + }, + { + "block_height": 10, + "chain_state": "confirmed", + "maturity_state": "unknown", + "audit_publication_sequence": 7, + }, + { + "block_height": 11, + "chain_state": "confirmed", + "maturity_state": "immature", + "audit_publication_sequence": 7, + }, + { + "block_height": 10, + "chain_state": "confirmed", + "maturity_state": "immature", + "audit_publication_sequence": True, + }, + ) + for state in invalid_states: + with self.subTest(invalid_state=state): + server = coordinator() + store = mock.MagicMock() + store.legacy_evidence_identity.return_value = legacy_identity + server._ensure_audit_artifact_store = lambda: store # type: ignore[method-assign] + server.ledger = SimpleNamespace( + pool_block_state=lambda **_kwargs: state + ) + server._upgrade_legacy_audit_evidence() + store.invalidate_unprovable_legacy_evidence.assert_called_once_with() + store.adopt_legacy_publication_identity.assert_not_called() + + server = coordinator() + store = mock.MagicMock() + store.legacy_evidence_identity.return_value = legacy_identity + server._ensure_audit_artifact_store = lambda: store # type: ignore[method-assign] + server.ledger = SingleWriterShareLedger() + server._upgrade_legacy_audit_evidence() + store.invalidate_unprovable_legacy_evidence.assert_called_once_with() + def test_build_audit_bundle_passes_pool_fee_policy_to_cli_payload(self) -> None: server = coordinator() server.rpc = AddressRpc(valid_address="tq1fee", script_byte="99") @@ -116,6 +745,163 @@ def test_build_audit_bundle_preserves_exact_canonical_output_file(self) -> None: self.assertEqual(output_path.read_bytes(), canonical_bytes) self.assertIn("--canonical-output", captured["cmd"]) self.assertEqual(captured["payload"]["shares"], []) + + def test_build_audit_bundle_transfers_the_exact_open_output_inode(self) -> None: + server = coordinator() + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + captured: dict[str, object] = {} + adopted: list[tuple[Path, int, int]] = [] + + def adopt(path: Path, value: os.stat_result) -> None: + current = path.stat() + self.assertEqual((value.st_dev, value.st_ino), (current.st_dev, current.st_ino)) + adopted.append((path, value.st_dev, value.st_ino)) + + with tempfile.TemporaryDirectory() as tmp, patch( + "lab.prism.bundle_compiler.subprocess.Popen", + fake_audit_bundle_popen(captured), + ): + output_path = Path(tmp) / "candidate.audit.json" + server.build_audit_bundle( + shares=[], + found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + canonical_output_path=output_path, + canonical_output_adopter=adopt, + ) + + self.assertEqual(len(adopted), 1) + self.assertEqual(adopted[0][0], output_path) + self.assertEqual(adopted[0][1:], (output_path.stat().st_dev, output_path.stat().st_ino)) + + def test_build_audit_bundle_pinned_parent_never_recreates_stale_path(self) -> None: + server = coordinator() + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + captured: dict[str, object] = {} + + with tempfile.TemporaryDirectory() as tmp, patch( + "lab.prism.bundle_compiler.subprocess.Popen", + fake_audit_bundle_popen(captured), + ): + base = Path(tmp) + root = base / "audit" + root.mkdir() + pinned = base / "audit-pinned" + output_path = root / "candidate.audit.json" + parent_fd = os.open( + root, + os.O_RDONLY | getattr(os, "O_DIRECTORY", 0), + ) + root.rename(pinned) + + def reject_stale_authority( + path: Path, + value: os.stat_result, + ) -> None: + self.assertEqual(path, output_path) + self.assertFalse(root.exists()) + pinned_output = pinned / output_path.name + self.assertTrue(pinned_output.exists()) + self.assertEqual( + (value.st_dev, value.st_ino), + (pinned_output.stat().st_dev, pinned_output.stat().st_ino), + ) + raise RuntimeError("stale A1 authority") + + try: + with self.assertRaisesRegex(RuntimeError, "stale A1 authority"): + server.build_audit_bundle( + shares=[], + found_block={ + "block_height": 10, + "coinbase_value_sats": 50_00000000, + }, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + canonical_output_path=output_path, + canonical_output_parent_fd=parent_fd, + canonical_output_adopter=reject_stale_authority, + ) + finally: + os.close(parent_fd) + + self.assertFalse(root.exists()) + self.assertFalse((pinned / output_path.name).exists()) + + def test_build_audit_bundle_adopter_failure_preserves_path_replacement(self) -> None: + server = coordinator() + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + captured: dict[str, object] = {} + + def replace_then_reject(path: Path, _value: os.stat_result) -> None: + path.unlink() + path.write_bytes(b"competitor") + raise RuntimeError("transfer rejected") + + with tempfile.TemporaryDirectory() as tmp, patch( + "lab.prism.bundle_compiler.subprocess.Popen", + fake_audit_bundle_popen(captured), + ): + output_path = Path(tmp) / "candidate.audit.json" + with self.assertRaisesRegex(RuntimeError, "transfer rejected"): + server.build_audit_bundle( + shares=[], + found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + canonical_output_path=output_path, + canonical_output_adopter=replace_then_reject, + ) + + self.assertEqual(output_path.read_bytes(), b"competitor") + + def test_builder_cleanup_never_moves_an_existing_replacement(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + path = root / "candidate.audit.json" + path.write_bytes(b"owned-a") + identity = path.stat() + path.unlink() + path.write_bytes(b"competitor-b") + with patch( + "os.replace", + side_effect=AssertionError("replacement must not move"), + ): + BundleCompiler._remove_created_output_if_same(path, identity) + + self.assertEqual(path.read_bytes(), b"competitor-b") + quarantines = list(root.glob(".candidate.audit.json.*.cleanup")) + self.assertEqual(quarantines, []) + + def test_builder_cleanup_never_relocates_nonregular_replacement(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + path = root / "candidate.audit.json" + path.write_bytes(b"owned") + identity = path.stat() + path.unlink() + path.mkdir() + + BundleCompiler._remove_created_output_if_same(path, identity) + + self.assertTrue(path.is_dir()) + self.assertEqual( + list(root.glob(".candidate.audit.json.*.cleanup")), + [], + ) + + path.rmdir() + target = root / "operator" + target.write_bytes(b"operator") + path.symlink_to(target) + BundleCompiler._remove_created_output_if_same(path, identity) + self.assertTrue(path.is_symlink()) + self.assertEqual(path.read_bytes(), b"operator") + def test_build_audit_bundle_summary_only_requests_and_parses_job_summary(self) -> None: server = coordinator() server.signing_seed_hex = "42" * 32 @@ -243,6 +1029,28 @@ def test_build_audit_bundle_removes_partial_output_after_builder_failure(self) - self.assertEqual(recovered, {"ok": True}) self.assertEqual(server._ensure_job_bundle_service()._worker_counts["restarts"], 1) + + def test_build_audit_bundle_removes_owned_output_after_parse_failure(self) -> None: + server = coordinator() + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + captured: dict[str, object] = {} + + with tempfile.TemporaryDirectory() as tmp, patch( + "lab.prism.bundle_compiler.subprocess.Popen", + fake_audit_bundle_popen(captured, output_text='{"partial":'), + ): + output_path = Path(tmp) / "candidate.audit.json" + with self.assertRaises(json.JSONDecodeError): + server.build_audit_bundle( + shares=[], + found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + canonical_output_path=output_path, + ) + self.assertFalse(output_path.exists()) + def test_build_audit_bundle_recovers_after_cancelled_worker_timeout(self) -> None: server = coordinator() server.signing_seed_hex = "42" * 32 @@ -641,6 +1449,7 @@ def test_orphaned_block_candidate_keeps_share_credit(self) -> None: self.assertEqual(ledger.persisted, []) def test_block_candidate_queue_overflow_coalesces_wakeup_without_drop(self) -> None: server, state, _ledger = submit_coordinator() + configure_temporary_audit_root(self, server) server.block_candidate_queue = queue.Queue(maxsize=2) def candidate(tag: str) -> PrismBlockCandidate: @@ -748,6 +1557,7 @@ def test_candidate_intent_avoids_duplicate_template_transaction_bodies(self) -> ) def test_transient_candidate_failure_remains_pending_for_retry(self) -> None: server, state, _recording = submit_coordinator() + configure_temporary_audit_root(self, server) ledger = SingleWriterShareLedger() server.ledger = ledger pending = PendingShare( @@ -1217,6 +2027,7 @@ def test_invalid_durable_candidate_is_quarantined_by_outbox_row_key(self) -> Non for payload_hash in (None, "ff" * 32): with self.subTest(payload_hash=payload_hash): server, _state, _recording = submit_coordinator() + configure_temporary_audit_root(self, server) ledger = SingleWriterShareLedger() server.ledger = ledger durable_hash = "de" * 32 @@ -1485,10 +2296,13 @@ def fake_build_audit_bundle(**kwargs: object) -> dict[str, object]: build_kwargs.append(kwargs) output_path = kwargs["canonical_output_path"] assert isinstance(output_path, Path) - output_path.write_text( - json.dumps(alternate_bundle, indent=2), - encoding="utf-8", - ) + adopter = kwargs["canonical_output_adopter"] + assert callable(adopter) + with output_path.open("x+", encoding="utf-8") as output: + output.write(json.dumps(alternate_bundle, indent=2)) + output.flush() + os.fsync(output.fileno()) + adopter(output_path, os.fstat(output.fileno())) return alternate_bundle def fake_verify_bundle(bundle_path: Path, *_args: object, **_kwargs: object) -> dict[str, object]: @@ -1500,10 +2314,8 @@ def fake_verify_bundle(bundle_path: Path, *_args: object, **_kwargs: object) -> canonical_sha256, ) return { - "coinbase_txid": "11" * 32, - "coinbase_manifest_sha256_hex": "22" * 32, + **verified_audit_report(), "audit_bundle_sha256_hex": canonical_sha256, - "coinbase_tx_hex": "c0ffee", } persist_accepted_block = ledger.persist_accepted_block @@ -1544,7 +2356,10 @@ def persist_with_canonicalization(**kwargs: object) -> dict[str, object]: self.assertEqual(envelope["block_height"], 10) self.assertEqual(envelope["audit_bundle_sha256"], canonical_sha256) self.assertNotIn("signed_coinbase_manifest", envelope) - self.assertEqual(server.latest_evidence["audit_bundle_path"], str(live_files[0])) + self.assertEqual( + Path(server.latest_evidence["audit_bundle_path"]), + live_files[0].resolve(), + ) self.assertTrue(rpc.submitted) self.assertEqual( @@ -1875,7 +2690,9 @@ def call(self, method: str, params: object = None) -> object: server.rpc = rpc server.build_audit_bundle = blocking_payout_bundle # type: ignore[method-assign] server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() + lambda *_args, **kwargs: verified_audit_report( + block_height=int(kwargs["expected_block_height"]) + ) ) sent: list[dict[str, object]] = [] state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] @@ -2322,10 +3139,9 @@ def test_idempotent_direct_block_replay_skips_publication(self) -> None: # Replay the durable candidate after its block landed, its # confirmation committed, and the network built on top of it. - # qbit_confirm_pool_block reports confirmed_count=0 for an - # already-confirmed block whose height no longer matches the - # active tip — the sustained replay state of the post-block - # livelock. The published payout state already covers the + # Exact confirmed replay reports confirmed_count=1 at the block's + # own expected height even though the active chain tip is now a + # child. The published payout state already covers the # candidate, so the replay must not reserve a source, bump the # generation, wipe the job-bundle cache, or schedule refresh # churn. @@ -2358,9 +3174,12 @@ def call(self, method: str, params: object = None) -> object: "maturity_state": "immature", } ) - ledger.confirm_accepted_block = ( # type: ignore[method-assign] - lambda **_kwargs: {"backend": "fake", "confirmed_count": 0} - ) + def confirm_exact_ancestor(**kwargs: object) -> dict[str, object]: + self.assertEqual(kwargs["block_hash"], block_hash) + self.assertEqual(kwargs["active_tip_height"], 10) + return {"backend": "fake", "confirmed_count": 1} + + ledger.confirm_accepted_block = confirm_exact_ancestor # type: ignore[method-assign] retry_calls = 0 def count_retry() -> None: @@ -2433,8 +3252,8 @@ def test_leaked_publication_fence_replay_republishes(self) -> None: # force-blocked delivery while its source generation already # matched the published source, then failed before republishing. # The leaked fence blocks every job build until a publication - # lands, so a confirmed_count=0 replay must not take the covered - # skip here. + # lands, so an exact confirmed replay must heal the leaked fence + # before taking the covered-state skip. server._block_payout_state_publication(force=True) self.assertTrue(server._payout_state_service._publication_blocked) self.assertEqual( @@ -2470,9 +3289,12 @@ def call(self, method: str, params: object = None) -> object: "maturity_state": "immature", } ) - ledger.confirm_accepted_block = ( # type: ignore[method-assign] - lambda **_kwargs: {"backend": "fake", "confirmed_count": 0} - ) + def confirm_exact_ancestor(**kwargs: object) -> dict[str, object]: + self.assertEqual(kwargs["block_hash"], block_hash) + self.assertEqual(kwargs["active_tip_height"], 10) + return {"backend": "fake", "confirmed_count": 1} + + ledger.confirm_accepted_block = confirm_exact_ancestor # type: ignore[method-assign] self.assertTrue( server.submit_block_candidate( @@ -2545,13 +3367,14 @@ def test_untrusted_direct_block_reconcile_publishes_newer_source_once(self) -> N server.qbit_chain_view_untrusted = lambda: True # type: ignore[method-assign] block_hash = "d1" * 32 newer_tip = "d2" * 32 + real_confirm = ledger.confirm_accepted_block - def superseding_noop_confirmation(**_kwargs: object) -> dict[str, object]: + def superseding_noop_confirmation(**kwargs: object) -> dict[str, object]: server._reserve_payout_state_source( "external_tip", tip_hash=newer_tip, ) - return {"backend": "fake", "confirmed_count": 0} + return real_confirm(**kwargs) ledger.confirm_accepted_block = superseding_noop_confirmation # type: ignore[method-assign] with tempfile.TemporaryDirectory() as tempdir: @@ -2655,12 +3478,9 @@ def assert_candidate(params: object) -> None: "manifest": {"coinbase_tx_hex": "c0ffee", "payout_count": 1} }, } - server.verify_bundle = lambda *_args, **_kwargs: { # type: ignore[method-assign] - "coinbase_txid": "11" * 32, - "coinbase_manifest_sha256_hex": "22" * 32, - "audit_bundle_sha256_hex": "33" * 32, - "coinbase_tx_hex": "c0ffee", - } + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: verified_audit_report() + ) submission = SimpleNamespace( coinbase_tx_hex="c0ffee", block_hash_hex=block_hash, @@ -2702,6 +3522,7 @@ def assert_candidate(params: object) -> None: lambda **_kwargs: [] ) self.assertTrue(server.submit_block_candidate(candidate)) + latest_evidence = server.latest_evidence self.assertEqual([row["block_hash"] for row in ledger.persisted], [block_hash] * 2) # This compatibility builder ignores canonical_output_path, so its @@ -2717,13 +3538,15 @@ def assert_candidate(params: object) -> None: self.assertFalse(server.stop_event.is_set()) self.assertEqual(server.accepted_block_count, 1) self.assertEqual(refreshes, [block_hash]) - self.assertEqual(server.latest_evidence["persistence"]["block_count"], 1) - self.assertEqual(server.latest_evidence["confirmation"]["confirmed_count"], 1) + self.assertIsNotNone(latest_evidence) + assert latest_evidence is not None + self.assertEqual(latest_evidence["persistence"]["block_count"], 1) + self.assertEqual(latest_evidence["confirmation"]["confirmed_count"], 1) # Evidence carries an aggregate miner count, not a materialized list of # every miner id (which scanned the whole ledger twice under the lock). - self.assertEqual(server.latest_evidence["accepted_share_count"], 0) - self.assertEqual(server.latest_evidence["distinct_miner_count"], 0) - self.assertNotIn("distinct_miners", server.latest_evidence) + self.assertEqual(latest_evidence["accepted_share_count"], 0) + self.assertEqual(latest_evidence["distinct_miner_count"], 0) + self.assertNotIn("distinct_miners", latest_evidence) def test_audit_retention_prunes_only_live_and_candidate_files(self) -> None: server = coordinator() with tempfile.TemporaryDirectory() as tempdir: @@ -2743,6 +3566,13 @@ def test_audit_retention_prunes_only_live_and_candidate_files(self) -> None: segment = Path(tempdir) / f"prism-audit-share-segment-1-2-{'ee' * 32}.json" segment.write_text("{}", encoding="utf-8") + # Live deletion is fail-closed unless durable evidence has supplied + # publication authority. This facade test supplies that authority + # explicitly while keeping its hand-written retention fixtures. + store = server._ensure_audit_artifact_store() + store._compatibility_evidence_override = True + store._evidence_state = "valid" + server.prune_audit_artifacts() live_names = sorted(path.name for path in Path(tempdir).glob("prism-live-audit-bundle-[0-9]*.json")) @@ -2767,6 +3597,11 @@ def test_audit_retention_zero_preserves_current_live_envelope(self) -> None: current = Path(tempdir) / f"prism-live-audit-bundle-2-{'bb' * 32}.json" current.write_text("{}", encoding="utf-8") + store = server._ensure_audit_artifact_store() + store._compatibility_evidence_override = True + store._evidence_state = "valid" + store._current_envelope = current.resolve() + server.prune_audit_artifacts(keep_live_path=current) self.assertFalse(old.exists()) @@ -3036,12 +3871,9 @@ def test_rejected_candidate_never_creates_prepared_payout_state(self) -> None: } }, } - server.verify_bundle = lambda *_args, **_kwargs: { # type: ignore[method-assign] - "coinbase_txid": "11" * 32, - "coinbase_manifest_sha256_hex": "22" * 32, - "audit_bundle_sha256_hex": "33" * 32, - "coinbase_tx_hex": "c0ffee", - } + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: verified_audit_report() + ) submission = SimpleNamespace( coinbase_tx_hex="c0ffee", block_hash_hex=block_hash, diff --git a/tests/test_prism_coordinator_metrics.py b/tests/test_prism_coordinator_metrics.py index 144dfb9..b6ec256 100644 --- a/tests/test_prism_coordinator_metrics.py +++ b/tests/test_prism_coordinator_metrics.py @@ -4,16 +4,45 @@ from __future__ import annotations +from pathlib import Path +import tempfile import unittest +from unittest import mock from tests import prism_coordinator_test_support as _job_support from tests import prism_vardiff_test_support as _vardiff_support +def install_isolated_coordinator( + test_case: unittest.TestCase, + factory: object, +) -> None: + assert callable(factory) + + def isolated(*args: object, **kwargs: object) -> object: + result = factory(*args, **kwargs) + server = result[0] if isinstance(result, tuple) else result + temporary = tempfile.TemporaryDirectory() + server.audit_dir = Path(temporary.name) / "audit" + server.evidence_path = Path(temporary.name) / "state" / "evidence.json" + + def cleanup() -> None: + store = server.__dict__.get("_audit_artifact_store") + if store is not None: + store.close() + temporary.cleanup() + + test_case.addCleanup(cleanup) + return result + + globals()["coordinator"] = isolated + + class _VardiffSupportTestCase(unittest.TestCase): def setUp(self) -> None: globals().update( {name: getattr(_vardiff_support, name) for name in _vardiff_support.__all__} ) + install_isolated_coordinator(self, _vardiff_support.coordinator) class _JobSupportTestCase(unittest.TestCase): @@ -21,6 +50,7 @@ def setUp(self) -> None: globals().update( {name: getattr(_job_support, name) for name in _job_support.__all__} ) + install_isolated_coordinator(self, _job_support.coordinator) class PrismCoordinatorVardiffTests(_VardiffSupportTestCase): @@ -261,6 +291,25 @@ def test_metrics_include_audit_artifact_storage_gauges(self) -> None: self.assertIn('qbit_prism_audit_artifact_bytes{kind="other"} 6', metrics) self.assertIn('qbit_prism_audit_artifact_files{kind="other"} 1', metrics) self.assertIn("qbit_prism_audit_artifact_scan_error 0", metrics) + def test_metrics_payload_reads_one_a1_snapshot(self) -> None: + server = coordinator() + snapshot = { + "body": {"files": 1, "bytes": 2}, + "share_segment": {"files": 3, "bytes": 4}, + "live_bundle": {"files": 5, "bytes": 6}, + "candidate": {"files": 7, "bytes": 8}, + "other": {"files": 9, "bytes": 10}, + "scan_error": 0, + } + reader = mock.Mock(return_value=snapshot) + server.audit_artifact_metrics = reader # type: ignore[method-assign] + + metrics = server.metrics_payload() + + reader.assert_called_once_with() + self.assertIn('qbit_prism_audit_artifact_files{kind="body"} 1', metrics) + self.assertIn('qbit_prism_audit_artifact_bytes{kind="other"} 10', metrics) + self.assertIn("qbit_prism_audit_artifact_scan_error 0", metrics) def _pending_append(self, tag: str, accepted_at_ms: int = 2) -> PendingShareAppend: from lab.prism.share_ledger import PendingShare diff --git a/tests/test_prism_share_ledger.py b/tests/test_prism_share_ledger.py index 40dc576..b815dec 100644 --- a/tests/test_prism_share_ledger.py +++ b/tests/test_prism_share_ledger.py @@ -216,6 +216,178 @@ def release_reads(self) -> None: class PrismShareLedgerTests(unittest.TestCase): + def test_psql_private_audit_delegates_and_subclass_override_route_through_a1(self) -> None: + ledger = PsqlShareLedger.__new__(PsqlShareLedger) + store = unittest.mock.Mock() + ledger._audit_artifact_store = store + ledger._audit_bundle_canonicalizer = lambda _bundle: b"legacy" + bundle = {"schema": "test"} + body_path = Path("body.json") + part = {"kind": "segment"} + cases = ( + ( + "_externalize_audit_body", + ("aa" * 32, "bb" * 32, bundle), + {}, + "externalize_audit_body", + ), + ( + "_canonical_audit_body_bytes_for_sha", + (bundle, "bb" * 32), + {}, + "canonical_audit_body_bytes_for_sha", + ), + ("_audit_body_ref", (), {"payload": bundle}, "audit_body_ref"), + ("_audit_bundle_v2", (), {"payload": bundle}, "audit_bundle_v2"), + ("_audit_share_parts", ([{"share_seq": 1}],), {}, "audit_share_parts"), + ( + "_audit_share_range_parts", + ([{"share_seq": 1}],), + {}, + "audit_share_range_parts", + ), + ( + "_audit_share_segment_payload", + (), + {"shares": [{"share_seq": 1}]}, + "audit_share_segment_payload", + ), + ( + "_write_audit_share_segment", + (), + {"shares": [{"share_seq": 1}]}, + "write_audit_share_segment", + ), + ( + "_write_audit_share_segment_range", + (), + {"shares": [{"share_seq": 1}]}, + "write_audit_share_segment_range", + ), + ( + "_merge_audit_share_ranges", + ([{"share_seq": 1}], [{"share_seq": 2}]), + {"segment_path": body_path}, + "merge_audit_share_ranges", + ), + ( + "_audit_shares_by_seq", + ([{"share_seq": 1}],), + {"segment_path": body_path}, + "audit_shares_by_seq", + ), + ("_storage_json_bytes", (bundle,), {}, "storage_json_bytes"), + ( + "_canonical_audit_bundle_bytes", + (bundle,), + {}, + "canonical_audit_bundle_bytes", + ), + ( + "_audit_body_path", + ("aa" * 32, "bb" * 32), + {}, + "body_path", + ), + ("_resolve_audit_body_path", (body_path,), {}, "resolve_owned_path"), + ( + "_audit_body_byte_len", + (body_path, bundle, body_path), + {}, + "audit_body_byte_len", + ), + ( + "_read_external_body", + (body_path,), + {"expected_sha256": "bb" * 32}, + "read_external_body", + ), + ( + "_external_body_matches_sha", + (body_path, "bb" * 32), + {}, + "external_body_matches_sha", + ), + ( + "_external_body_available_for_sha", + (body_path, "bb" * 32), + {}, + "external_body_available_for_sha", + ), + ( + "_resolve_audit_body_ref", + (bundle,), + {"expected_sha256": "bb" * 32, "body_uri": body_path}, + "resolve_audit_body_ref", + ), + ( + "_resolve_audit_bundle_v2", + (bundle,), + {"expected_sha256": "bb" * 32, "body_uri": body_path}, + "resolve_audit_bundle_v2", + ), + ( + "_read_audit_share_segment", + (part,), + {"parent_body_uri": body_path}, + "read_audit_share_segment", + ), + ( + "_select_audit_share_segment_range", + ([{"share_seq": 1}],), + { + "first_share_seq": 1, + "last_share_seq": 1, + "parent_body_uri": body_path, + "body_uri": body_path, + }, + "select_audit_share_segment_range", + ), + ) + token = object() + for wrapper_name, args, kwargs, target_name in cases: + with self.subTest(wrapper=wrapper_name): + target = getattr(store, target_name) + target.reset_mock() + target.return_value = token + self.assertIs(getattr(ledger, wrapper_name)(*args, **kwargs), token) + target.assert_called_once_with(*args, **kwargs) + self.assertIs(ledger._audit_store(), store) + + store.read_audit_share_segment.reset_mock() + self.assertTrue( + ledger._audit_share_segment_available( + part, + parent_body_uri=body_path, + ) + ) + store.read_audit_share_segment.assert_called_once_with( + part, + parent_body_uri=body_path, + ) + + override_calls: list[tuple[object, object]] = [] + ledger._read_external_body = ( # type: ignore[method-assign] + lambda body_uri, *, expected_sha256=None: ( + override_calls.append((body_uri, expected_sha256)) + or {"resolved": True} + ) + ) + self.assertEqual( + ledger._resolve_audit_bundle_row( + { + "audit_bundle": None, + "body_uri": "owned-body", + "audit_bundle_sha256": "bb" * 32, + } + ), + { + "audit_bundle": {"resolved": True}, + "audit_bundle_sha256": "bb" * 32, + }, + ) + self.assertEqual(override_calls, [("owned-body", "bb" * 32)]) + def test_memory_recovery_append_distinguishes_insert_exact_and_conflict(self) -> None: ledger = SingleWriterShareLedger() share = pending_share(1) @@ -648,6 +820,7 @@ def test_pool_block_state_wraps_nullable_row_in_json(self) -> None: "parent_hash": "bb" * 32, "chain_state": "confirmed", "maturity_state": "immature", + "audit_publication_sequence": None, } }, ] @@ -662,10 +835,37 @@ def test_pool_block_state_wraps_nullable_row_in_json(self) -> None: "parent_hash": "bb" * 32, "chain_state": "confirmed", "maturity_state": "immature", + "audit_publication_sequence": None, }, ) self.assertIn("SELECT json_build_object(\n 'state'", ledger.queries[1]) + def test_postgres_publication_floor_reads_max_durable_row_ordinal(self) -> None: + ledger = CannedQueryPsqlShareLedger( + [ + acquired_lease_result(), + {"audit_publication_sequence_floor": 7}, + ] + ) + + self.assertEqual(ledger.audit_publication_sequence_floor(), 7) + query = ledger.queries[1] + self.assertIn("MAX(audit_publication_sequence)", query) + self.assertIn("COALESCE", query) + self.assertIn("FROM qbit_pool_blocks", query) + self.assertNotIn("last_value", query) + + for invalid in (None, True, "7", -1): + with self.subTest(invalid=invalid): + invalid_ledger = CannedQueryPsqlShareLedger( + [ + acquired_lease_result(), + {"audit_publication_sequence_floor": invalid}, + ] + ) + with self.assertRaisesRegex(RuntimeError, "floor is invalid"): + invalid_ledger.audit_publication_sequence_floor() + def test_prior_balances_after_pool_block_is_height_bounded(self) -> None: balance = { "recipient_id": "miner-a", @@ -1103,15 +1303,30 @@ def test_block_state_functions_refresh_configured_lease_after_sql_function(self) ) for method_name, result, function_name, count_key in cases: with self.subTest(method_name=method_name): + canned = [acquired_lease(), {"backend": "postgres-psql", **result}] + if method_name in { + "confirm_accepted_block", + "reactivate_pool_block", + }: + canned.append({"audit_publication_sequence": 7}) ledger = FakeLeasePsqlShareLedger( - [acquired_lease(), {"backend": "postgres-psql", **result}], + canned, lease_ttl_seconds=42, ) payload = getattr(ledger, method_name)(block_hash="aa" * 32, active_tip_height=10) - query = ledger.lease_queries[-1] + query = next( + query + for query in ledger.lease_queries + if function_name in query + ) self.assertEqual(payload[count_key], 1) + if method_name in { + "confirm_accepted_block", + "reactivate_pool_block", + }: + self.assertEqual(payload["audit_publication_sequence"], 7) self.assertIn(function_name, query) self.assertNotIn("lease_refresh AS", query) self.assertIn("make_interval(secs => 42.0)", query) @@ -2009,18 +2224,20 @@ def test_psql_reuses_complete_10k_share_slot_without_parse_merge_or_memory_ampli expected_bytes = segment_path.read_bytes() expected_file_sha256 = hashlib.sha256(expected_bytes).hexdigest() expected_mtime_ns = segment_path.stat().st_mtime_ns + store = ledger._audit_artifact_store + assert store is not None gc.collect() tracemalloc.start() try: with ( unittest.mock.patch( - "lab.prism.share_ledger.json.loads", + "lab.prism.audit_artifacts.json.loads", side_effect=AssertionError("complete share slot must not be parsed"), ), unittest.mock.patch.object( - ledger, - "_merge_audit_share_ranges", + store, + "merge_audit_share_ranges", side_effect=AssertionError("complete share slot must not be merged"), ), ): @@ -2098,7 +2315,8 @@ def test_psql_canonical_bundle_path_skips_canonicalizer_and_is_retry_safe(self) with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) body_dir = root / "body-store" - candidate_path = root / "canonical-candidate.json" + body_dir.mkdir() + candidate_path = body_dir / "canonical-candidate.json" bundle = { "schema": "qbit.prism.audit-bundle.v1", "shares": [{"share_seq": 1, "share_id": "s1"}], @@ -2159,11 +2377,12 @@ def test_psql_canonical_bundle_path_skips_canonicalizer_and_is_retry_safe(self) canonicalizer.assert_not_called() self.assertEqual(list(body_dir.glob(".*.tmp")), []) - def test_psql_compact_v2_retry_stays_bounded_and_skips_reconstruction(self) -> None: + def test_psql_compact_v2_retry_reconstructs_without_recanonicalizing(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) body_dir = root / "body-store" - candidate_path = root / "canonical-candidate.json" + body_dir.mkdir() + candidate_path = body_dir / "canonical-candidate.json" bundle = { "schema": "qbit.prism.audit-bundle.v1", "shares": [ @@ -2215,18 +2434,20 @@ def test_psql_compact_v2_retry_stays_bounded_and_skips_reconstruction(self) -> N json.loads(body_path.read_text(encoding="utf-8"))["schema"], "qbit.prism.audit-bundle.v2", ) + store = ledger._audit_artifact_store + assert store is not None with ( unittest.mock.patch.object( - ledger, - "_external_body_matches_sha", + store, + "external_body_matches_sha", side_effect=AssertionError("same-version compact retry must compare bounded storage"), ), unittest.mock.patch.object( - ledger, - "_resolve_audit_bundle_v2", - side_effect=AssertionError("compact retry must not reconstruct every segment"), - ), + store, + "resolve_audit_bundle_v2", + wraps=store.resolve_audit_bundle_v2, + ) as reconstruct, ): second_uri = ledger._prepare_external_audit_body( payload, @@ -2242,13 +2463,72 @@ def test_psql_compact_v2_retry_stays_bounded_and_skips_reconstruction(self) -> N self.assertEqual(path.stat().st_ino, first_stat.st_ino) self.assertEqual(path.stat().st_mtime_ns, first_stat.st_mtime_ns) canonicalizer.assert_not_called() + reconstruct.assert_called_once() + self.assertFalse(reconstruct.call_args.kwargs["verify_digest"]) self.assertEqual(list(body_dir.glob(".*.tmp")), []) + def test_psql_compact_body_rejects_canonical_and_logical_bundle_mismatch(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + body_dir = root / "body-store" + body_dir.mkdir() + candidate_path = body_dir / "canonical-candidate.json" + canonical_bundle = { + "schema": "qbit.prism.audit-bundle.v1", + "shares": [ + {"share_seq": 1, "share_id": "canonical-a"}, + {"share_seq": 2, "share_id": "canonical-a-2"}, + ], + } + logical_bundle = { + "schema": "qbit.prism.audit-bundle.v1", + "shares": [ + {"share_seq": 1, "share_id": "logical-b"}, + {"share_seq": 2, "share_id": "logical-b-2"}, + ], + } + canonical_bytes = fake_audit_bundle_bytes(canonical_bundle) + candidate_path.write_bytes(canonical_bytes) + body_sha = hashlib.sha256(canonical_bytes).hexdigest() + ledger = FakeLeasePsqlShareLedger( + [acquired_lease()], + audit_body_dir=body_dir, + audit_bundle_canonicalizer=unittest.mock.Mock( + side_effect=AssertionError("literal candidate should be authoritative") + ), + audit_share_segment_size=1, + ) + payload = { + "block_hash": "aa" * 32, + "audit_bundle_sha256": body_sha, + "body_uri": str( + body_dir + / f"prism-audit-bundle-body-{'aa' * 32}-{body_sha}.json" + ), + } + + with self.assertRaisesRegex(RuntimeError, "does not match logical bundle"): + ledger._prepare_external_audit_body( + payload, + logical_bundle, + canonical_bundle_path=candidate_path, + ) + + self.assertEqual( + [ + path + for path in body_dir.iterdir() + if path.name != ".prism-audit-publication.lock" + ], + [candidate_path], + ) + def test_psql_canonical_bundle_path_hash_mismatch_publishes_nothing(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) body_dir = root / "body-store" - candidate_path = root / "canonical-candidate.json" + body_dir.mkdir() + candidate_path = body_dir / "canonical-candidate.json" bundle = {"schema": "qbit.prism.audit-bundle.v1", "shares": []} candidate_path.write_bytes(fake_audit_bundle_bytes(bundle)) canonicalizer = unittest.mock.Mock( @@ -2285,7 +2565,14 @@ def test_psql_canonical_bundle_path_hash_mismatch_publishes_nothing(self) -> Non canonicalizer.assert_not_called() self.assertEqual(len(ledger.lease_results), 1) - self.assertFalse(body_dir.exists()) + self.assertEqual( + [ + path + for path in body_dir.iterdir() + if path.name != ".prism-audit-publication.lock" + ], + [candidate_path], + ) def test_psql_compact_audit_body_writes_v2_range_proof_and_resolves_v1_bundle(self) -> None: with tempfile.TemporaryDirectory() as tmp: @@ -2344,6 +2631,193 @@ def test_psql_compact_audit_body_writes_v2_range_proof_and_resolves_v1_bundle(se resolved = reader._read_external_body(body_uri, expected_sha256=body_sha) self.assertEqual(resolved, bundle) + def test_psql_v1_and_v2_body_tamper_fail_read_and_availability(self) -> None: + def assert_tamper_rejected( + ledger: PsqlShareLedger, + body_path: Path, + digest: str, + original: dict[str, Any], + mutate: Any, + ) -> None: + tampered = json.loads(json.dumps(original)) + mutate(tampered) + body_path.write_text( + json.dumps(tampered, separators=(",", ":")), + encoding="utf-8", + ) + self.assertFalse( + ledger._external_body_available_for_sha(str(body_path), digest) + ) + with self.assertRaises(RuntimeError): + ledger._read_external_body( + str(body_path), + expected_sha256=digest, + ) + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + ledger = FakeLeasePsqlShareLedger( + [acquired_lease()], + audit_body_dir=root, + audit_bundle_canonicalizer=fake_audit_bundle_bytes, + ) + v1_bundle = { + "schema": "qbit.prism.audit-bundle.v1", + "shares": [ + {"share_seq": 1, "share_id": "s1"}, + {"share_seq": 2, "share_id": "s2"}, + ], + "found_block": {"bits": "207fffff"}, + } + v1_digest = fake_audit_bundle_sha256(v1_bundle) + v1_body = { + "schema": AUDIT_BODY_REF_SCHEMA, + "block_hash": "aa" * 32, + "audit_bundle_sha256": v1_digest, + "bundle_without_shares": { + "schema": "qbit.prism.audit-bundle.v1", + "found_block": {"bits": "207fffff"}, + }, + "share_count": 2, + "shares_key_index": 1, + "share_parts": [ + { + "kind": "inline", + "first_share_seq": 1, + "last_share_seq": 1, + "share_count": 1, + "shares": [{"share_seq": 1, "share_id": "s1"}], + }, + { + "kind": "inline", + "first_share_seq": 2, + "last_share_seq": 2, + "share_count": 1, + "shares": [{"share_seq": 2, "share_id": "s2"}], + }, + ], + } + v1_path = root / ( + f"prism-audit-bundle-body-{'aa' * 32}-{v1_digest}.json" + ) + v1_path.write_text( + json.dumps(v1_body, separators=(",", ":")), + encoding="utf-8", + ) + self.assertEqual( + ledger._read_external_body( + str(v1_path), + expected_sha256=v1_digest, + ), + v1_bundle, + ) + v1_mutations = ( + lambda body: body.__setitem__("block_hash", "bb" * 32), + lambda body: body["bundle_without_shares"]["found_block"].__setitem__( + "bits", + "1d00ffff", + ), + lambda body: body.__setitem__("shares_key_index", 0), + lambda body: body.__setitem__( + "share_parts", + list(reversed(body["share_parts"])), + ), + lambda body: body["share_parts"][0].__setitem__( + "first_share_seq", + 2, + ), + ) + for index, mutate in enumerate(v1_mutations): + with self.subTest(schema="v1", mutation=index): + assert_tamper_rejected( + ledger, + v1_path, + v1_digest, + v1_body, + mutate, + ) + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + v2_bundle = { + "schema": "qbit.prism.audit-bundle.v1", + "shares": [ + {"share_seq": 1, "share_id": "s1"}, + {"share_seq": 2, "share_id": "s2"}, + ], + "reward_manifest": { + "anchor_job_issued_at_ms": 100, + "anchor_share_seq": 1, + "newest_share_seq": 2, + "oldest_share_seq": 1, + "included_share_count": 2, + "requested_window_weight": 20, + "counted_window_weight": 20, + "share_slice_digest_hex": "44" * 32, + }, + } + v2_digest = fake_audit_bundle_sha256(v2_bundle) + writer = FakeLeasePsqlShareLedger( + [ + acquired_lease(), + {"existing_block": False, "existing_body_uri": None}, + ], + audit_body_dir=root, + audit_bundle_canonicalizer=fake_audit_bundle_bytes, + audit_share_segment_size=1, + ) + body_uri = writer._prepare_external_audit_body( + { + "block_hash": "cc" * 32, + "audit_bundle_sha256": v2_digest, + "coinbase_tx_hex": "00", + "coinbase_txid": "11" * 32, + "payout_manifest_sha256": "22" * 32, + "block_height": 10, + "parent_hash": "bb" * 32, + "writer_id": writer._writer_id, + "writer_epoch": writer._writer_epoch, + "writer_session_token": writer._writer_session_token, + }, + v2_bundle, + ) + assert body_uri is not None + v2_path = Path(body_uri) + v2_body = json.loads(v2_path.read_text(encoding="utf-8")) + self.assertEqual(v2_body["schema"], AUDIT_BUNDLE_V2_SCHEMA) + self.assertEqual( + writer._read_external_body( + body_uri, + expected_sha256=v2_digest, + ), + v2_bundle, + ) + v2_mutations = ( + lambda body: body.__setitem__("block_hash", "dd" * 32), + lambda body: body["bundle_without_shares"]["reward_manifest"].__setitem__( + "included_share_count", + 3, + ), + lambda body: body.__setitem__("shares_key_index", 0), + lambda body: body["share_window_proof"].__setitem__( + "share_parts", + list(reversed(body["share_window_proof"]["share_parts"])), + ), + lambda body: body["share_window_proof"].__setitem__( + "included_share_count", + 3, + ), + ) + for index, mutate in enumerate(v2_mutations): + with self.subTest(schema="v2", mutation=index): + assert_tamper_rejected( + writer, + v2_path, + v2_digest, + v2_body, + mutate, + ) + def test_psql_v2_range_segments_grow_without_breaking_old_refs(self) -> None: with tempfile.TemporaryDirectory() as tmp: first_bundle = { @@ -2445,7 +2919,9 @@ def test_psql_public_artifact_resolves_external_audit_bundle(self) -> None: "has_audit_row": True, "fallback": None, }, - ] + ], + audit_body_dir=tmp, + audit_bundle_canonicalizer=fake_audit_bundle_bytes, ) self.assertEqual(ledger.dashboard_public_artifact(sha256=body_sha), bundle) @@ -2490,6 +2966,11 @@ def test_psql_public_artifact_exists_rejects_missing_external_body(self) -> None def test_psql_public_artifact_exists_validates_compact_body_segments(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) + logical_bundle = { + "schema": "qbit.prism.audit-bundle.v1", + "shares": [{"share_seq": 1, "share_id": "s1"}], + } + body_sha = fake_audit_bundle_sha256(logical_bundle) segment = { "schema": "qbit.prism.audit-share-segment.v1", "first_share_seq": 1, @@ -2503,9 +2984,11 @@ def test_psql_public_artifact_exists_validates_compact_body_segments(self) -> No segment_path.write_bytes(segment_bytes) body_ref = { "schema": AUDIT_BODY_REF_SCHEMA, - "audit_bundle_sha256": "aa" * 32, + "block_hash": "bb" * 32, + "audit_bundle_sha256": body_sha, "bundle_without_shares": {"schema": "qbit.prism.audit-bundle.v1"}, "share_count": 1, + "shares_key_index": 1, "share_parts": [ { "kind": "segment", @@ -2517,7 +3000,7 @@ def test_psql_public_artifact_exists_validates_compact_body_segments(self) -> No } ], } - body_path = root / f"prism-audit-bundle-body-{'bb' * 32}-{'aa' * 32}.json" + body_path = root / f"prism-audit-bundle-body-{'bb' * 32}-{body_sha}.json" body_path.write_text(json.dumps(body_ref, separators=(",", ":")), encoding="utf-8") ledger = FakeLeasePsqlShareLedger( [ @@ -2530,9 +3013,10 @@ def test_psql_public_artifact_exists_validates_compact_body_segments(self) -> No }, ], audit_body_dir=tmp, + audit_bundle_canonicalizer=fake_audit_bundle_bytes, ) - self.assertTrue(ledger.dashboard_public_artifact_exists(sha256="aa" * 32)) + self.assertTrue(ledger.dashboard_public_artifact_exists(sha256=body_sha)) segment_path.unlink() ledger = FakeLeasePsqlShareLedger( [ @@ -2545,8 +3029,9 @@ def test_psql_public_artifact_exists_validates_compact_body_segments(self) -> No }, ], audit_body_dir=tmp, + audit_bundle_canonicalizer=fake_audit_bundle_bytes, ) - self.assertFalse(ledger.dashboard_public_artifact_exists(sha256="aa" * 32)) + self.assertFalse(ledger.dashboard_public_artifact_exists(sha256=body_sha)) def test_psql_public_artifact_exists_rejects_overstated_inline_share_count(self) -> None: body_ref = { @@ -2565,7 +3050,9 @@ def test_psql_public_artifact_exists_rejects_overstated_inline_share_count(self) ], } with tempfile.TemporaryDirectory() as tmp: - body_path = Path(tmp) / "body-ref.json" + body_path = Path(tmp) / ( + f"prism-audit-bundle-body-{'11' * 32}-{'aa' * 32}.json" + ) body_path.write_text(json.dumps(body_ref, separators=(",", ":")), encoding="utf-8") ledger = FakeLeasePsqlShareLedger([acquired_lease()], audit_body_dir=tmp) @@ -2580,6 +3067,7 @@ def test_psql_body_ref_respects_zero_shares_key_index(self) -> None: body_sha = fake_audit_bundle_sha256(bundle) body_ref = { "schema": AUDIT_BODY_REF_SCHEMA, + "block_hash": "11" * 32, "audit_bundle_sha256": body_sha, "share_count": 1, "shares_key_index": 0, @@ -2598,7 +3086,9 @@ def test_psql_body_ref_respects_zero_shares_key_index(self) -> None: ], } with tempfile.TemporaryDirectory() as tmp: - body_path = Path(tmp) / "body-ref.json" + body_path = Path(tmp) / ( + f"prism-audit-bundle-body-{'11' * 32}-{body_sha}.json" + ) body_path.write_text(json.dumps(body_ref, separators=(",", ":")), encoding="utf-8") ledger = FakeLeasePsqlShareLedger( [acquired_lease()], @@ -2635,7 +3125,9 @@ def test_psql_external_body_hash_mismatch_fails_readers(self) -> None: acquired_lease(), audit_row, {**audit_row, "audit_commitment_leaf_hex": "ab" * 32}, - ] + ], + audit_body_dir=tmp, + audit_bundle_canonicalizer=fake_audit_bundle_bytes, ) with self.assertRaisesRegex(RuntimeError, "hash mismatch"): ledger.audit_bundle(block_hash="aa" * 32) @@ -2652,7 +3144,9 @@ def test_psql_external_body_hash_mismatch_fails_readers(self) -> None: "has_audit_row": True, "fallback": None, }, - ] + ], + audit_body_dir=tmp, + audit_bundle_canonicalizer=fake_audit_bundle_bytes, ) with self.assertRaisesRegex(RuntimeError, "hash mismatch"): public_ledger.dashboard_public_artifact(sha256=body_sha) diff --git a/tests/test_prism_tip_refresh_delivery.py b/tests/test_prism_tip_refresh_delivery.py index 746b297..9e30e0e 100644 --- a/tests/test_prism_tip_refresh_delivery.py +++ b/tests/test_prism_tip_refresh_delivery.py @@ -4,6 +4,9 @@ from __future__ import annotations +from pathlib import Path +import tempfile +import threading import unittest from tests import prism_coordinator_test_support as _job_support from tests import prism_vardiff_test_support as _vardiff_support @@ -599,6 +602,29 @@ def test_reconciliation_reactivates_inactive_block_that_returns_to_active_chain( ] ) server.ledger = ledger + temporary_audit = tempfile.TemporaryDirectory() + self.addCleanup(temporary_audit.cleanup) + server.audit_dir = Path(temporary_audit.name) / "audit" + server.evidence_path = ( + Path(temporary_audit.name) / "state" / "evidence.json" + ) + audit_store = server._ensure_audit_artifact_store() + self.addCleanup(audit_store.close) + payout_service = server._ensure_payout_state_service() + real_reactivate = ledger.reactivate_pool_block + + def guarded_reactivate(**kwargs: object) -> dict[str, object]: + self.assertTrue( + payout_service.balance_mutation_lock._is_owned(), # type: ignore[attr-defined] + "P1 balance lock must be outermost during reactivation", + ) + self.assertEqual( + audit_store._publication_guard_owner, + threading.get_ident(), + ) + return real_reactivate(**kwargs) + + ledger.reactivate_pool_block = guarded_reactivate # type: ignore[method-assign] server.rpc = ReorgRpc( tip=pool_block_hash, template=gbt_template(pool_block_hash, height=13),