backport: assumeutxo M4 — evo snapshot format v3 and LLMQ reconstruction

src/Makefile.am

@@ -231,9 +231,11 @@ BITCOIN_CORE_H = \
   evo/mnauth.h \
   evo/mnhftx.h \
   evo/netinfo.h \
+  evo/snapshot_types.h \
   evo/providertx.h \
   evo/simplifiedmns.h \
   evo/smldiff.h \
+  evo/snapshot.h \
   evo/specialtx.h \
   evo/specialtx_filter.h \
   evo/specialtxman.h \
@@ -538,6 +540,8 @@ libbitcoin_node_a_SOURCES = \
   evo/evodb.cpp \
   evo/mnauth.cpp \
   evo/mnhftx.cpp \
+  evo/snapshot.cpp \
+  evo/snapshot_load.cpp \
   evo/providertx.cpp \
   evo/simplifiedmns.cpp \
   evo/smldiff.cpp \
@@ -1275,6 +1279,7 @@ libdashkernel_la_SOURCES = \
   evo/providertx_util.cpp \
   evo/simplifiedmns.cpp \
   evo/smldiff.cpp \
+  evo/snapshot.cpp \
   evo/specialtx.cpp \
   evo/specialtx_filter.cpp \
   evo/specialtxman.cpp \
@@ -1379,6 +1384,7 @@ libdashkernel_la_SOURCES = \
   util/threadnames.cpp \
   util/time.cpp \
   util/tokenpipe.cpp \
+  evo/snapshot_load.cpp \
   validation.cpp \
   validationinterface.cpp \
   versionbits.cpp \

src/Makefile.test.include

@@ -118,6 +118,7 @@ BITCOIN_TESTS =\
   test/evo_mnhf_tests.cpp \
   test/evo_netinfo_tests.cpp \
   test/evo_simplifiedmns_tests.cpp \
+  test/evo_snapshot_tests.cpp \
   test/evo_trivialvalidation.cpp \
   test/evo_utils_tests.cpp \
   test/flatfile_tests.cpp \

src/chainparams.cpp

@@ -882,11 +882,11 @@ class CRegTestParams : public CChainParams {
         m_assumeutxo_data = MapAssumeutxo{
             {
                 110,
-                {AssumeutxoHash{uint256S("0x9b2a277a3e3b979f1a539d57e949495d7f8247312dbc32bce6619128c192b44b")}, 110},
+                {AssumeutxoHash{uint256S("0x9b2a277a3e3b979f1a539d57e949495d7f8247312dbc32bce6619128c192b44b")}, EvoSnapshotHash{uint256{}}, 110},
             },
             {
                 200,
-                {AssumeutxoHash{uint256S("0x8a5bdd92252fc6b24663244bbe958c947bb036dc1f94ccd15439f48d8d1cb4e3")}, 200},
+                {AssumeutxoHash{uint256S("0x8a5bdd92252fc6b24663244bbe958c947bb036dc1f94ccd15439f48d8d1cb4e3")}, EvoSnapshotHash{uint256{}}, 200},
             },
         };
 

src/chainparams.h

@@ -34,6 +34,10 @@ struct AssumeutxoHash : public BaseHash<uint256> {
     explicit AssumeutxoHash(const uint256& hash) : BaseHash(hash) {}
 };
 
+struct EvoSnapshotHash : public BaseHash<uint256> {
+    explicit EvoSnapshotHash(const uint256& hash) : BaseHash(hash) {}
+};
+
 /**
  * Holds configuration for use during UTXO snapshot load and validation. The contents
  * here are security critical, since they dictate which UTXO snapshots are recognized
@@ -43,6 +47,9 @@ struct AssumeutxoData {
     //! The expected hash of the deserialized UTXO set.
     const AssumeutxoHash hash_serialized;
 
+    //! The expected single-SHA256 hash of the canonical Dash evo section.
+    const EvoSnapshotHash evo_hash;
+
     //! Used to populate the nChainTx value, which is used during BlockManager::LoadBlockIndex().
     //!
     //! We need to hardcode the value here because this is computed cumulatively using block data,

src/evo/chainhelper.cpp

@@ -9,6 +9,7 @@
 #include <evo/creditpool.h>
 #include <evo/deterministicmns.h>
 #include <evo/mnhftx.h>
+#include <evo/snapshot.h>
 #include <evo/specialtxman.h>
 #include <governance/superblock.h>
 #include <hash.h>
@@ -27,6 +28,8 @@ CChainstateHelper::CChainstateHelper(CEvoDB& evodb, CDeterministicMNManager& dmn
     isman{isman},
     mn_sync{mn_sync},
     m_dmnman{dmnman},
+    m_qblockman{qblockman},
+    m_qsnapman{qsnapman},
     credit_pool_manager{std::make_unique<CCreditPoolManager>(evodb, chainman)},
     m_chainlocks{chainlocks},
     ehf_manager{std::make_unique<CMNHFManager>(evodb, chainman)},
@@ -66,7 +69,12 @@ int32_t CChainstateHelper::GetBestChainLockHeight() const { return m_chainlocks.
 
 uint256 CChainstateHelper::GetDeterministicMNListHash(const CBlockIndex* pindex) const
 {
-    return SerializeHash(m_dmnman.GetListForBlock(Assert(pindex)));
+    const CBlockIndex* index{Assert(pindex)};
+    CDeterministicMNList list{m_dmnman.GetListForBlock(index)};
+    if (list.GetBlockHash().IsNull()) {
+        list = CDeterministicMNList{index->GetBlockHash(), index->nHeight, 0};
+    }
+    return evo::CanonicalMNListHash(list);
 }
 
 /** Passthrough functions to CCreditPoolManager */

src/evo/chainhelper.h

@@ -43,6 +43,8 @@ class CChainstateHelper
     llmq::CInstantSendManager& isman;
     const CMasternodeSync& mn_sync;
     CDeterministicMNManager& m_dmnman;
+    llmq::CQuorumBlockProcessor& m_qblockman;
+    llmq::CQuorumSnapshotManager& m_qsnapman;
 
 public:
     const std::unique_ptr<CCreditPoolManager> credit_pool_manager;
@@ -72,6 +74,9 @@ class CChainstateHelper
 
     /** Return a canonical hash of the deterministic MN list derived at a block. */
     uint256 GetDeterministicMNListHash(const CBlockIndex* pindex) const;
+    CDeterministicMNManager& DeterministicMNManager() { return m_dmnman; }
+    llmq::CQuorumBlockProcessor& QuorumBlockProcessor() { return m_qblockman; }
+    llmq::CQuorumSnapshotManager& QuorumSnapshotManager() { return m_qsnapman; }
 
     /** Passthrough functions to CCreditPoolManager */
     CCreditPool GetCreditPool(const CBlockIndex* const pindex);

src/evo/creditpool.cpp

@@ -125,12 +125,12 @@ std::optional<CCreditPool> CCreditPoolManager::GetFromCache(const CBlockIndex& b
             return pool;
         }
     }
-    if (block_index.nHeight % DISK_SNAPSHOT_PERIOD == 0) {
-        if (evoDb.Read(std::make_pair(DB_CREDITPOOL_SNAPSHOT, block_hash), pool)) {
-            LOCK(cache_mutex);
-            creditPoolCache.insert(block_hash, pool);
-            return pool;
-        }
+    // Snapshot activation may deliberately seed a full state at a height that
+    // is not one of the normal periodic checkpoints.
+    if (evoDb.Read(std::make_pair(DB_CREDITPOOL_SNAPSHOT, block_hash), pool)) {
+        LOCK(cache_mutex);
+        creditPoolCache.insert(block_hash, pool);
+        return pool;
     }
     return std::nullopt;
 }
@@ -155,6 +155,12 @@ void CCreditPoolManager::AddToCache(const uint256& block_hash, int height, const
     }
 }
 
+bool CCreditPoolManager::SeedSnapshot(const CBlockIndex* block, const CCreditPool& pool)
+{
+    assert(block != nullptr);
+    return evoDb.WriteDerived(std::make_pair(DB_CREDITPOOL_SNAPSHOT, block->GetBlockHash()), pool);
+}
+
 CCreditPool CCreditPoolManager::ConstructCreditPool(const gsl::not_null<const CBlockIndex*> block_index, CCreditPool prev)
 {
     std::optional<CreditPoolDataPerBlock> opt_block_data = GetCreditDataFromBlock(block_index, m_chainman.GetConsensus());

src/evo/creditpool.h

@@ -134,6 +134,8 @@ class CCreditPoolManager
       * it can happen if there limits of withdrawal (unlock) exceed
       */
     CCreditPool GetCreditPool(const CBlockIndex* block) EXCLUSIVE_LOCKS_REQUIRED(!cache_mutex);
+    /** Seed a full pool snapshot in the current EvoDB transaction. */
+    bool SeedSnapshot(const CBlockIndex* block, const CCreditPool& pool) EXCLUSIVE_LOCKS_REQUIRED(!cache_mutex);
 
 private:
     std::optional<CCreditPool> GetFromCache(const CBlockIndex& block_index) EXCLUSIVE_LOCKS_REQUIRED(!cache_mutex);

src/evo/deterministicmns.cpp

@@ -374,6 +374,27 @@ void CDeterministicMNList::ApplyDiff(gsl::not_null<const CBlockIndex*> pindex, c
     blockHash = pindex->GetBlockHash();
     nHeight = pindex->nHeight;
 
+    for (const auto& id : diff.removedMns) {
+        auto dmn = GetMNByInternalId(id);
+        if (!dmn) throw std::runtime_error(strprintf("%s: can't find a removed masternode, id=%d", __func__, id));
+        RemoveMN(dmn->proTxHash);
+    }
+    for (const auto& dmn : diff.addedMNs) AddMN(dmn);
+    for (const auto& p : diff.updatedMNs) {
+        auto dmn = GetMNByInternalId(p.first);
+        if (!dmn) throw std::runtime_error(strprintf("%s: can't find an updated masternode, id=%d", __func__, p.first));
+        UpdateMN(*dmn, p.second);
+    }
+}
+
+void CDeterministicMNList::ApplyDiffForSnapshot(const uint256& block_hash, int height,
+                                                uint32_t total_registered_count,
+                                                const CDeterministicMNListDiff& diff)
+{
+    if (height < 0) throw std::runtime_error("negative historical MN-list height");
+    blockHash = block_hash;
+    nHeight = height;
+
     for (const auto& id : diff.removedMns) {
         auto dmn = GetMNByInternalId(id);
         if (!dmn) {
@@ -382,7 +403,7 @@ void CDeterministicMNList::ApplyDiff(gsl::not_null<const CBlockIndex*> pindex, c
         RemoveMN(dmn->proTxHash);
     }
     for (const auto& dmn : diff.addedMNs) {
-        AddMN(dmn);
+        AddMN(dmn, /*fBumpTotalCount=*/false);
     }
     for (const auto& p : diff.updatedMNs) {
         auto dmn = GetMNByInternalId(p.first);
@@ -391,6 +412,7 @@ void CDeterministicMNList::ApplyDiff(gsl::not_null<const CBlockIndex*> pindex, c
         }
         UpdateMN(*dmn, p.second);
     }
+    nTotalRegisteredCount = total_registered_count;
 }
 
 void CDeterministicMNList::AddMN(const CDeterministicMNCPtr& dmn, bool fBumpTotalCount)
@@ -622,6 +644,18 @@ CDeterministicMNManager::CDeterministicMNManager(CEvoDB& evoDb, CMasternodeMetaM
 
 CDeterministicMNManager::~CDeterministicMNManager() = default;
 
+bool CDeterministicMNManager::SeedListForBlock(const CDeterministicMNList& list)
+{
+    return m_evoDb.WriteDerived(std::make_pair(DB_LIST_SNAPSHOT, list.GetBlockHash()), list);
+}
+
+void CDeterministicMNManager::InvalidateListCacheForBlock(const uint256& block_hash)
+{
+    LOCK(cs);
+    mnListsCache.erase(block_hash);
+    mnListDiffsCache.erase(block_hash);
+}
+
 bool CDeterministicMNManager::ProcessBlock(const CBlock& block, gsl::not_null<const CBlockIndex*> pindex,
                                            BlockValidationState& state, const CDeterministicMNList& newList,
                                            std::optional<MNListUpdates>& updatesRet)
@@ -789,6 +823,7 @@ CDeterministicMNList CDeterministicMNManager::GetListForBlockInternal(gsl::not_n
             mnListsCache.emplace(pindex->GetBlockHash(), snapshot);
             break;
         }
+        if (m_list_snapshot_miss_hook) m_list_snapshot_miss_hook(pindex);
 
         // no snapshot found yet, check diffs
         auto itDiffs = mnListDiffsCache.find(pindex->GetBlockHash());

src/evo/deterministicmns.h

@@ -23,6 +23,7 @@
 
 #include <algorithm>
 #include <atomic>
+#include <functional>
 #include <limits>
 #include <numeric>
 #include <stdexcept>
@@ -339,6 +340,8 @@ class CDeterministicMNList
         assert(nHeight >= 0);
         return nHeight;
     }
+    /** Snapshot hashing also covers the pre-DIP3 default list (height -1). */
+    [[nodiscard]] int GetHeightForSnapshotCodec() const noexcept { return nHeight; }
     void SetHeight(int _height)
     {
         assert(_height >= 0);
@@ -422,6 +425,10 @@ class CDeterministicMNList
      */
     void ApplyDiff(gsl::not_null<const CBlockIndex*> pindex, const CDeterministicMNListDiff& diff)
         EXCLUSIVE_LOCKS_REQUIRED(!m_cached_sml_mutex);
+    /** Apply a snapshot-local historical diff without dereferencing block data. */
+    void ApplyDiffForSnapshot(const uint256& block_hash, int height, uint32_t total_registered_count,
+                              const CDeterministicMNListDiff& diff)
+        EXCLUSIVE_LOCKS_REQUIRED(!m_cached_sml_mutex);
 
     void AddMN(const CDeterministicMNCPtr& dmn, bool fBumpTotalCount = true) EXCLUSIVE_LOCKS_REQUIRED(!m_cached_sml_mutex);
     void UpdateMN(const CDeterministicMN& oldDmn, const std::shared_ptr<const CDeterministicMNState>& pdmnState)
@@ -766,6 +773,7 @@ class CDeterministicMNManager
 
     Uint256HashMap<CDeterministicMNList> mnListsCache GUARDED_BY(cs);
     Uint256HashMap<CDeterministicMNListDiff> mnListDiffsCache GUARDED_BY(cs);
+    std::function<void(const CBlockIndex*)> m_list_snapshot_miss_hook GUARDED_BY(cs);
     const CBlockIndex* tipIndex GUARDED_BY(cs) {nullptr};
     const CBlockIndex* m_initial_snapshot_index GUARDED_BY(cs) {nullptr};
 
@@ -789,6 +797,21 @@ class CDeterministicMNManager
     };
     CDeterministicMNList GetListAtChainTip() EXCLUSIVE_LOCKS_REQUIRED(!cs);
 
+    /** Seed a canonical full-list snapshot in the current EvoDB transaction. */
+    bool SeedListForBlock(const CDeterministicMNList& list) EXCLUSIVE_LOCKS_REQUIRED(!cs);
+
+    /** Invalidate cached list data so the next lookup reloads it from EvoDB. */
+    void InvalidateListCacheForBlock(const uint256& block_hash) EXCLUSIVE_LOCKS_REQUIRED(!cs);
+
+    /** Test-only guard invoked after a full-list cache/EvoDB miss, before
+     * ordinary diff-chain reconstruction can access earlier NORMAL state. */
+    void SetListSnapshotMissHookForTesting(std::function<void(const CBlockIndex*)> hook)
+        EXCLUSIVE_LOCKS_REQUIRED(!cs)
+    {
+        LOCK(cs);
+        m_list_snapshot_miss_hook = std::move(hook);
+    }
+
     void SetListForBlockForTesting(const CDeterministicMNList& list) EXCLUSIVE_LOCKS_REQUIRED(!cs)
     {
         LOCK(cs);

src/evo/evodb.cpp

@@ -42,6 +42,12 @@ CEvoDB::CEvoDB(const util::DbWrapperParams& db_params) :
 
 CEvoDB::~CEvoDB() = default;
 
+bool CEvoDB::HasActiveTransaction()
+{
+    LOCK(cs);
+    return active_transaction.has_value();
+}
+
 CEvoDB::TransactionContext& CEvoDB::GetContext(EvoDbIdentity identity)
 {
     auto it = transaction_contexts.find(identity);
@@ -182,6 +188,37 @@ bool CEvoDB::ReadBackgroundMNListHash(uint256& block_hash, uint256& mn_list_hash
     return true;
 }
 
+void CEvoDB::WriteRequiredWorkMNListHashes(const std::vector<uint256>& block_hashes)
+{
+    Write(EVODB_REQUIRED_WORK_MNLISTS, block_hashes);
+}
+
+bool CEvoDB::ReadRequiredWorkMNListHashes(std::vector<uint256>& block_hashes)
+{
+    return Read(EVODB_REQUIRED_WORK_MNLISTS, block_hashes);
+}
+
+void CEvoDB::WriteBackgroundWorkMNListHash(const uint256& block_hash, const uint256& mn_list_hash)
+{
+    Write(std::make_pair(EVODB_BACKGROUND_WORK_MNLIST_HASH, block_hash), mn_list_hash);
+}
+
+bool CEvoDB::ReadBackgroundWorkMNListHash(const uint256& block_hash, uint256& mn_list_hash)
+{
+    return Read(std::make_pair(EVODB_BACKGROUND_WORK_MNLIST_HASH, block_hash), mn_list_hash);
+}
+
+static void EraseHistoricalMNListMarkers(CDBWrapper& db, CDBBatch& batch)
+{
+    std::vector<uint256> required;
+    if (db.Read(EVODB_REQUIRED_WORK_MNLISTS, required)) {
+        for (const auto& block_hash : required) {
+            batch.Erase(std::make_pair(EVODB_BACKGROUND_WORK_MNLIST_HASH, block_hash));
+        }
+    }
+    batch.Erase(EVODB_REQUIRED_WORK_MNLISTS);
+}
+
 bool CEvoDB::PromoteSnapshotMarkers(const uint256& expected_snapshot_tip)
 {
     LOCK(cs);
@@ -198,7 +235,9 @@ bool CEvoDB::PromoteSnapshotMarkers(const uint256& expected_snapshot_tip)
         uint256 normal_tip;
         const bool already_promoted = db->Read(EVODB_BEST_BLOCK, normal_tip) && normal_tip == expected_snapshot_tip &&
                                       !db->Exists(EVODB_DUAL_CHAINSTATE) && !db->Exists(EVODB_SNAPSHOT_MNLIST_HASH) &&
-                                      !db->Exists(EVODB_BACKGROUND_MNLIST_HASH);
+                                      !db->Exists(EVODB_BACKGROUND_MNLIST_HASH) &&
+                                      !db->Exists(EVODB_REQUIRED_WORK_MNLISTS) &&
+                                      !db->Exists(EVODB_SNAPSHOT_EVO_SECTION);
         if (already_promoted) m_default_identity = EvoDbIdentity::NORMAL;
         return already_promoted;
     }
@@ -209,6 +248,8 @@ bool CEvoDB::PromoteSnapshotMarkers(const uint256& expected_snapshot_tip)
     batch.Erase(snapshot_key);
     batch.Erase(EVODB_SNAPSHOT_MNLIST_HASH);
     batch.Erase(EVODB_BACKGROUND_MNLIST_HASH);
+    EraseHistoricalMNListMarkers(*db, batch);
+    batch.Erase(EVODB_SNAPSHOT_EVO_SECTION);
     batch.Erase(EVODB_DUAL_CHAINSTATE);
     if (!db->WriteBatch(batch, /*fSync=*/true)) return false;
     // The dual-chainstate run is over: the promoted state is the NORMAL
@@ -231,6 +272,8 @@ bool CEvoDB::DiscardSnapshotMarkers()
     batch.Erase(std::make_pair(EVODB_BEST_BLOCK, uint8_t{1}));
     batch.Erase(EVODB_SNAPSHOT_MNLIST_HASH);
     batch.Erase(EVODB_BACKGROUND_MNLIST_HASH);
+    EraseHistoricalMNListMarkers(*db, batch);
+    batch.Erase(EVODB_SNAPSHOT_EVO_SECTION);
     batch.Erase(EVODB_DUAL_CHAINSTATE);
     if (!db->WriteBatch(batch, /*fSync=*/true)) return false;
     // The snapshot chainstate is gone; transaction-less access must resolve