fix: bound DMN list/diff caches to stop getmnlistd memory exhaustion

Keep a hard cap on recent masternode lists and diffs so unauthenticated historical GETMNLISTDIFF traffic cannot grow memory without bound between blocks.

Add a small LRU tier for stale mini-snapshots so repeated requests for heights older than the recency window stay cheap after first warm-up, instead of re-walking up to ~575 diffs under cs_main on every call.

Extend the unit test past the recency and diff-cap windows, and document the change in release notes.

Author: PastaPastaPasta

doc/release-notes-7485.md

@@ -0,0 +1,8 @@
+P2P and network
+---------------
+
+- Bound in-memory masternode list caches so unauthenticated historical
+  `GETMNLISTDIFF` requests can no longer grow memory without limit between
+  blocks. Recent lists are capped by height-aware eviction; stale historical
+  mini-snapshots are kept in a small LRU cache so repeated requests do not
+  re-read large on-disk snapshots on every call. (#7485)

src/evo/deterministicmns.cpp

@@ -742,6 +742,7 @@ bool CDeterministicMNManager::UndoBlock(gsl::not_null<const CBlockIndex*> pindex
 
         mnListsCache.erase(blockHash);
         mnListDiffsCache.erase(blockHash);
+        EraseStaleList(blockHash);
     }
     if (diff.HasChanges()) {
         CDeterministicMNList curList{prevList};
@@ -826,6 +827,7 @@ void CDeterministicMNManager::CacheMNList(const uint256& block_hash, const CDete
 {
     AssertLockHeld(cs);
     if (!ShouldRetainCacheHeight(list.GetHeight())) {
+        CacheStaleMNList(block_hash, list);
         return;
     }
     // Prefer emplace over assign: CDeterministicMNList::operator= locks m_cached_sml_mutex
@@ -836,6 +838,46 @@ void CDeterministicMNManager::CacheMNList(const uint256& block_hash, const CDete
     }
 }
 
+void CDeterministicMNManager::CacheStaleMNList(const uint256& block_hash, const CDeterministicMNList& list)
+{
+    AssertLockHeld(cs);
+    if (auto it = mnStaleListsCache.find(block_hash); it != mnStaleListsCache.end()) {
+        mnStaleListsLru.splice(mnStaleListsLru.begin(), mnStaleListsLru, it->second.lru_it);
+        return;
+    }
+    mnStaleListsLru.push_front(block_hash);
+    mnStaleListsCache.emplace(block_hash, StaleListEntry{list, mnStaleListsLru.begin()});
+    while (mnStaleListsCache.size() > MAX_STALE_CACHE_LISTS) {
+        const uint256& evict_hash = mnStaleListsLru.back();
+        if (auto evict_it = mnStaleListsCache.find(evict_hash); evict_it != mnStaleListsCache.end()) {
+            mnStaleListsCache.erase(evict_it);
+        }
+        mnStaleListsLru.pop_back();
+    }
+}
+
+std::optional<CDeterministicMNList> CDeterministicMNManager::GetStaleList(const uint256& block_hash)
+{
+    AssertLockHeld(cs);
+    const auto it = mnStaleListsCache.find(block_hash);
+    if (it == mnStaleListsCache.end()) {
+        return std::nullopt;
+    }
+    mnStaleListsLru.splice(mnStaleListsLru.begin(), mnStaleListsLru, it->second.lru_it);
+    return it->second.list;
+}
+
+void CDeterministicMNManager::EraseStaleList(const uint256& block_hash)
+{
+    AssertLockHeld(cs);
+    const auto it = mnStaleListsCache.find(block_hash);
+    if (it == mnStaleListsCache.end()) {
+        return;
+    }
+    mnStaleListsLru.erase(it->second.lru_it);
+    mnStaleListsCache.erase(it);
+}
+
 void CDeterministicMNManager::CacheMNListDiff(const uint256& block_hash, CDeterministicMNListDiff diff)
 {
     AssertLockHeld(cs);
@@ -868,6 +910,11 @@ CDeterministicMNList CDeterministicMNManager::GetListForBlockInternal(gsl::not_n
             break;
         }
 
+        if (auto stale = GetStaleList(pindex->GetBlockHash())) {
+            snapshot = std::move(*stale);
+            break;
+        }
+
         if (m_evoDb.Read(std::make_pair(DB_LIST_SNAPSHOT, pindex->GetBlockHash()), snapshot)) {
             // Use the list; only retain it in the cache if it is tip-recent.
             CacheMNList(pindex->GetBlockHash(), snapshot);

src/evo/deterministicmns.h

@@ -24,7 +24,9 @@
 #include <algorithm>
 #include <atomic>
 #include <limits>
+#include <list>
 #include <numeric>
+#include <optional>
 #include <stdexcept>
 #include <unordered_map>
 #include <utility>
@@ -746,21 +748,30 @@ class CDeterministicMNManager
     // Hard caps on the in-memory caches. CleanupCache() alone is not enough: it only
     // runs once a new block has arrived, so between blocks an unauthenticated peer
     // spamming getmnlistd for historical blocks could append entries without bound
-    // (a full mainnet list is several MB). Admission is bounded two ways: entries
-    // older than the window CleanupCache would drop anyway are not retained at all,
-    // and these caps evict the lowest-height entries when exceeded.
+    // (a full mainnet list is several MB). List admission uses two tiers:
+    //   - Recent tier (mnListsCache): tip-recent heights only, lowest-height-first
+    //     eviction, cap MAX_CACHE_LISTS. Attacker stale traffic never enters here.
+    //   - Stale tier (mnStaleListsCache): LRU, cap MAX_STALE_CACHE_LISTS. Keeps a
+    //     bounded working set of historical mini-snapshots so repeated stale
+    //     getmnlistd requests do not re-read a multi-MB disk snapshot on every call.
     // Lists are rebuilt by applying up to DISK_SNAPSHOT_PERIOD - 1 diffs from the
     // previous on-disk snapshot, so block validation / invalidation spanning a
     // snapshot boundary can legitimately keep up to two snapshot periods of lists
-    // (per-block lists plus mini-snapshots) resident. Size the cap to hold that
-    // whole window so bounding admission never slows the (dis)connect hot path;
+    // (per-block lists plus mini-snapshots) resident. Size the recent cap to hold
+    // that whole window so bounding admission never slows the (dis)connect hot path;
     // it is well above honest steady-state usage (tip + live quorum bases +
     // mini-snapshots within LIST_DIFFS_CACHE_SIZE of the tip).
     static constexpr size_t MAX_CACHE_LISTS = static_cast<size_t>(DISK_SNAPSHOT_PERIOD) * 2;
+    // One 576-block snapshot interval of mini-snapshots (18) plus margin.
+    static constexpr size_t MAX_STALE_CACHE_LISTS = 32;
     // Diffs are small; allow a full recency window plus a margin for one rebuild walk.
     static constexpr size_t MAX_CACHE_DIFFS = static_cast<size_t>(LIST_DIFFS_CACHE_SIZE) + 64;
 
 private:
+    struct StaleListEntry {
+        CDeterministicMNList list;
+        std::list<uint256>::iterator lru_it;
+    };
     Mutex cs;
     Mutex cs_cleanup;
     // We have performed CleanupCache() on this height.
@@ -774,6 +785,8 @@ class CDeterministicMNManager
 
     Uint256HashMap<CDeterministicMNList> mnListsCache GUARDED_BY(cs);
     Uint256HashMap<CDeterministicMNListDiff> mnListDiffsCache GUARDED_BY(cs);
+    std::list<uint256> mnStaleListsLru GUARDED_BY(cs);
+    Uint256HashMap<StaleListEntry> mnStaleListsCache GUARDED_BY(cs);
     const CBlockIndex* tipIndex GUARDED_BY(cs) {nullptr};
     const CBlockIndex* m_initial_snapshot_index GUARDED_BY(cs) {nullptr};
 
@@ -808,6 +821,11 @@ class CDeterministicMNManager
         LOCK(cs);
         return mnListDiffsCache.size();
     }
+    size_t GetStaleListCacheSize() EXCLUSIVE_LOCKS_REQUIRED(!cs)
+    {
+        LOCK(cs);
+        return mnStaleListsCache.size();
+    }
 
     // Test if given TX is a ProRegTx which also contains the collateral at index n
     static bool IsProTxWithCollateral(const CTransactionRef& tx, uint32_t n);
@@ -852,6 +870,9 @@ class CDeterministicMNManager
     [[nodiscard]] bool ShouldRetainCacheHeight(int height) EXCLUSIVE_LOCKS_REQUIRED(cs);
     void CacheMNList(const uint256& block_hash, const CDeterministicMNList& list) EXCLUSIVE_LOCKS_REQUIRED(cs);
     void CacheMNListDiff(const uint256& block_hash, CDeterministicMNListDiff diff) EXCLUSIVE_LOCKS_REQUIRED(cs);
+    void CacheStaleMNList(const uint256& block_hash, const CDeterministicMNList& list) EXCLUSIVE_LOCKS_REQUIRED(cs);
+    [[nodiscard]] std::optional<CDeterministicMNList> GetStaleList(const uint256& block_hash) EXCLUSIVE_LOCKS_REQUIRED(cs);
+    void EraseStaleList(const uint256& block_hash) EXCLUSIVE_LOCKS_REQUIRED(cs);
     void EnforceListsCacheLimit() EXCLUSIVE_LOCKS_REQUIRED(cs);
     void EnforceDiffsCacheLimit() EXCLUSIVE_LOCKS_REQUIRED(cs);
 

src/test/evo_deterministicmns_tests.cpp

@@ -3102,20 +3102,18 @@ BOOST_AUTO_TEST_CASE(mn_lists_cache_bounded)
     dmnman.UpdatedBlockTip(tip_index());
     dmnman.DoMaintenance();
 
-    // Mine more than the hard cap without running cleanup — mirrors the
-    // attacker window between blocks when getmnlistd populates the cache.
-    constexpr size_t n_blocks = CDeterministicMNManager::MAX_CACHE_LISTS + 64;
+    // Mine past the recency window and diff cap without running cleanup — mirrors
+    // the attacker window between blocks when getmnlistd populates the cache.
+    constexpr size_t recency_window = CDeterministicMNManager::MAX_CACHE_DIFFS - 64;
+    constexpr size_t n_blocks = recency_window + (40 * 32);
     for (size_t i = 0; i < n_blocks; ++i) {
         setup.CreateAndProcessBlock({}, coinbase_pk);
         dmnman.UpdatedBlockTip(tip_index());
     }
 
     // Record the expected list for a spread of historical heights, and for every
-    // height in the lowest 64 of the range. Eviction is lowest-height-first, so
-    // after the descending sweep below (which touches MAX_CACHE_LISTS newer
-    // heights after each of these), the lowest-64 entries are guaranteed to have
-    // been evicted; re-querying them proves eviction never changes what is
-    // returned.
+    // height in the lowest 64 of the range. Eviction is lowest-height-first in
+    // the recent tier; stale heights land in the LRU stale tier instead.
     const CBlockIndex* tip = tip_index();
     BOOST_REQUIRE(tip != nullptr);
     const int lowest_height = tip->nHeight - static_cast<int>(n_blocks) + 1;
@@ -3133,30 +3131,50 @@ BOOST_AUTO_TEST_CASE(mn_lists_cache_bounded)
     }
     BOOST_REQUIRE(expected.size() > 64);
 
-    // Now exercise GetListForBlock over every distinct historical height — the
+    // Exercise GetListForBlock over every distinct historical height — the
     // getmnlistd / BuildSimplifiedMNListDiff path an unauthenticated peer drives.
     for (int h = tip->nHeight; h >= 0 && h > tip->nHeight - static_cast<int>(n_blocks); --h) {
         const CBlockIndex* pindex = tip->GetAncestor(h);
         BOOST_REQUIRE(pindex != nullptr);
         (void)dmnman.GetListForBlock(pindex);
     }
 
-    // Pre-fix: each ProcessBlock / historical load appends freely → size > MAX.
-    // Post-fix: insert-time retention + hard cap keep the cache bounded.
-    // (The diffs cache stays far below its cap here — its bound only bites on
-    // walks that read stale diffs back from disk — so only the lists cap is
-    // driven past its limit by this test.)
     const size_t list_cache_size = dmnman.GetListCacheSize();
+    const size_t diff_cache_size = dmnman.GetListDiffsCacheSize();
+    const size_t stale_cache_size = dmnman.GetStaleListCacheSize();
     BOOST_TEST_MESSAGE("mnListsCache size after sweep: " << list_cache_size);
+    BOOST_TEST_MESSAGE("mnListDiffsCache size after sweep: " << diff_cache_size);
+    BOOST_TEST_MESSAGE("mnStaleListsCache size after sweep: " << stale_cache_size);
+
     BOOST_CHECK_MESSAGE(list_cache_size <= CDeterministicMNManager::MAX_CACHE_LISTS,
                         strprintf("mnListsCache size %zu exceeds hard cap %zu", list_cache_size,
                                   CDeterministicMNManager::MAX_CACHE_LISTS));
-    BOOST_CHECK_LE(dmnman.GetListDiffsCacheSize(), CDeterministicMNManager::MAX_CACHE_DIFFS);
-
-    // The cache is pure memoisation: bounding it must not change any result. The
-    // lowest-64 entries recorded above have certainly been evicted by now, so
-    // they are recomputed from disk here — the values must still match what the
-    // cache returned above.
+    BOOST_CHECK_LE(diff_cache_size, CDeterministicMNManager::MAX_CACHE_DIFFS);
+    // n_blocks ProcessBlock admissions exceed MAX_CACHE_DIFFS; the cap must have trimmed.
+    BOOST_CHECK_MESSAGE(diff_cache_size >= CDeterministicMNManager::MAX_CACHE_DIFFS - 64,
+                        strprintf("mnListDiffsCache size %zu never approached cap %zu", diff_cache_size,
+                                  CDeterministicMNManager::MAX_CACHE_DIFFS));
+
+    // Stale heights from the sweep must have been routed to the LRU stale tier.
+    BOOST_CHECK_GT(stale_cache_size, 0U);
+    BOOST_CHECK_LE(stale_cache_size, CDeterministicMNManager::MAX_STALE_CACHE_LISTS);
+
+    // Repeat access to one stale interval must be cache-served, not a full re-walk.
+    const int stale_target_height = tip->nHeight - static_cast<int>(recency_window) - 100;
+    BOOST_REQUIRE(stale_target_height >= 0);
+    const CBlockIndex* stale_pindex = tip->GetAncestor(stale_target_height);
+    BOOST_REQUIRE(stale_pindex != nullptr);
+    const auto stale_first = dmnman.GetListForBlock(stale_pindex);
+    const size_t stale_size_after_first = dmnman.GetStaleListCacheSize();
+    const CBlockIndex* stale_neighbor = tip->GetAncestor(stale_target_height + 16);
+    BOOST_REQUIRE(stale_neighbor != nullptr);
+    const auto stale_neighbor_result = dmnman.GetListForBlock(stale_neighbor);
+    const auto stale_second = dmnman.GetListForBlock(stale_pindex);
+    BOOST_CHECK(stale_first == stale_second);
+    BOOST_CHECK(stale_neighbor_result == dmnman.GetListForBlock(stale_neighbor));
+    BOOST_CHECK_LE(dmnman.GetStaleListCacheSize(), stale_size_after_first + 1);
+
+    // The cache is pure memoisation: bounding it must not change any result.
     for (const auto& [pindex, want] : expected) {
         const auto got = dmnman.GetListForBlock(pindex);
         BOOST_CHECK_MESSAGE(got == want,