Feature/keep previous substates flag

Dockerfile

@@ -50,7 +50,7 @@ ENV VERSION_LAST_TAG=$VERSION_LAST_TAG
 RUN apt-get update \
   && apt-get install -y --no-install-recommends \
     docker.io=20.10.24+dfsg1-1+deb12u1+b3 \
-    libssl-dev=3.0.18-1~deb12u2 \
+    libssl-dev=3.0.19-1~deb12u2 \
     pkg-config=1.8.1-1 \
     unzip=6.0-28 \
     wget=${WGET_VERSION} \
@@ -131,7 +131,7 @@ RUN apt-get update \
     g++-x86-64-linux-gnu \
     libc6-dev-arm64-cross=2.36-8cross1 \
     libclang-dev=1:14.0-55.7~deb12u1 \
-    libssl-dev=3.0.18-1~deb12u2 \
+    libssl-dev=3.0.19-1~deb12u2 \
     pkg-config=1.8.1-1 \
   && rm -rf /var/lib/apt/lists/*
 

core-rust-bridge/src/main/java/com/radixdlt/environment/DatabaseConfig.java

@@ -72,7 +72,8 @@ public record DatabaseConfig(
     boolean enableLocalTransactionExecutionIndex,
     boolean enableAccountChangeIndex,
     boolean enableHistoricalSubstateValues,
-    boolean enableEntityListingIndices) {
+    boolean enableEntityListingIndices,
+    boolean keepPreviousSubstateValues) {
   public static void registerCodec(CodecMap codecMap) {
     codecMap.register(
         DatabaseConfig.class,
@@ -81,6 +82,6 @@ public static void registerCodec(CodecMap codecMap) {
 
   public static DatabaseConfig forTesting() {
     // Many test assert on transaction execution details, so we keep this one on by default:
-    return new DatabaseConfig(true, false, false, false);
+    return new DatabaseConfig(true, false, false, false, true);
   }
 }

core-rust-bridge/src/test/java/com/radixdlt/helper/NodeRustEnvironmentBuilder.java

@@ -90,7 +90,7 @@ public static NodeRustEnvironment createNodeRustEnvironment(String dbPath) {
                 new RustMempoolConfig(mempoolMaxTotalTransactionsSize, mempoolMaxTransactionCount)),
             Option.none(),
             stateManagerDbConfig,
-            new DatabaseConfig(false, false, false, false),
+            new DatabaseConfig(false, false, false, false, true),
             LoggingConfig.getDefault(),
             StateTreeGcConfig.forTesting(),
             LedgerProofsGcConfig.forTesting(),

core-rust/core-api-server/src/core_api/conversions/receipt.rs

@@ -379,7 +379,9 @@ pub fn to_api_state_updates(
                         &typed_substate_key,
                         new,
                     )?),
-                    previous_value: if context.substate_options.include_previous {
+                    previous_value: if context.substate_options.include_previous
+                        && !previous.is_empty()
+                    {
                         Some(Box::new(to_api_substate_value(
                             context,
                             &state_mapping_lookups,
@@ -395,7 +397,9 @@ pub fn to_api_state_updates(
             SubstateChangeAction::Delete { previous } => {
                 deleted_substates.push(models::DeletedSubstate {
                     substate_id,
-                    previous_value: if context.substate_options.include_previous {
+                    previous_value: if context.substate_options.include_previous
+                        && !previous.is_empty()
+                    {
                         Some(Box::new(to_api_substate_value(
                             context,
                             &state_mapping_lookups,

core-rust/core-api-server/src/core_api/handlers/lts/transaction_status.rs

@@ -36,13 +36,13 @@ pub(crate) async fn handle_lts_transaction_status(
         .filter_map(|p| p.1.intent_invalid_from_epoch)
         .next();
 
-    let intent_is_permanently_rejected = invalid_from_epoch.map_or(false, |invalid_from_epoch| {
-        current_epoch >= invalid_from_epoch
-    }) || known_pending_payloads.iter().any(|p| {
-        p.1.earliest_permanent_rejection
-            .as_ref()
-            .map_or(false, |r| r.marks_permanent_rejection_for_intent())
-    });
+    let intent_is_permanently_rejected = invalid_from_epoch
+        .is_some_and(|invalid_from_epoch| current_epoch >= invalid_from_epoch)
+        || known_pending_payloads.iter().any(|p| {
+            p.1.earliest_permanent_rejection
+                .as_ref()
+                .is_some_and(|r| r.marks_permanent_rejection_for_intent())
+        });
 
     if let Some(txn_state_version) = txn_state_version_opt {
         let hashes = database

core-rust/core-api-server/src/core_api/handlers/transaction_status.rs

@@ -36,13 +36,13 @@ pub(crate) async fn handle_transaction_status(
         .filter_map(|p| p.1.intent_invalid_from_epoch)
         .next();
 
-    let intent_is_permanently_rejected = invalid_from_epoch.map_or(false, |invalid_from_epoch| {
-        current_epoch >= invalid_from_epoch
-    }) || known_pending_payloads.iter().any(|p| {
-        p.1.earliest_permanent_rejection
-            .as_ref()
-            .map_or(false, |r| r.marks_permanent_rejection_for_intent())
-    });
+    let intent_is_permanently_rejected = invalid_from_epoch
+        .is_some_and(|invalid_from_epoch| current_epoch >= invalid_from_epoch)
+        || known_pending_payloads.iter().any(|p| {
+            p.1.earliest_permanent_rejection
+                .as_ref()
+                .is_some_and(|r| r.marks_permanent_rejection_for_intent())
+        });
 
     if let Some(txn_state_version) = txn_state_version_opt {
         let hashes = database

core-rust/engine-state-api-server/src/engine_state_api/readers.rs

@@ -667,7 +667,7 @@ pub struct BlueprintFieldMeta {
 
 impl BlueprintFieldMeta {
     /// Post-processes and returns the [`Self::transience_default_value_bytes`] (see the note there).
-    pub fn transience(&self) -> Option<FieldTransienceMeta> {
+    pub fn transience(&self) -> Option<FieldTransienceMeta<'_>> {
         self.transience_default_value_bytes
             .as_ref()
             .map(|default_value_bytes| FieldTransienceMeta {
@@ -1212,7 +1212,7 @@ impl<'s, S: SubstateDatabase> EngineStateDataLoader<'s, S> {
         module_id: ModuleId,
         collection_meta: &'s ObjectCollectionMeta,
         from_key: Option<&RawCollectionKey>,
-    ) -> Result<impl Iterator<Item = SborCollectionKey> + '_, EngineStateBrowsingError> {
+    ) -> Result<impl Iterator<Item = SborCollectionKey<'_>> + '_, EngineStateBrowsingError> {
         // From performance PoV, there is no way to iterate over keys without iterating over values
         // too. The cost of the (discarded) `SborData` wrapper construction is negligible, hence:
         Ok(self
@@ -1230,7 +1230,7 @@ impl<'s, S: SubstateDatabase> EngineStateDataLoader<'s, S> {
         collection_meta: &'s ObjectCollectionMeta,
         from_key: Option<&RawCollectionKey>,
     ) -> Result<
-        impl Iterator<Item = (SborCollectionKey, SborData<'s>)> + '_,
+        impl Iterator<Item = (SborCollectionKey<'_>, SborData<'s>)> + '_,
         EngineStateBrowsingError,
     > {
         let collection_index = collection_meta.index.number;
@@ -1262,7 +1262,7 @@ impl<'s, S: SubstateDatabase> EngineStateDataLoader<'s, S> {
         node_id: &NodeId,
         kv_store_meta: &'s KeyValueStoreMeta,
         from_key: Option<&MapKey>,
-    ) -> Result<impl Iterator<Item = SborData> + '_, EngineStateBrowsingError> {
+    ) -> Result<impl Iterator<Item = SborData<'_>> + '_, EngineStateBrowsingError> {
         Ok(self
             .reader
             .key_value_store_iter(node_id, from_key)
@@ -1280,7 +1280,7 @@ impl<'s, S: SubstateDatabase> EngineStateDataLoader<'s, S> {
     pub fn load_schema(
         &self,
         reference: &SchemaReference,
-    ) -> Result<SborData, EngineStateBrowsingError> {
+    ) -> Result<SborData<'_>, EngineStateBrowsingError> {
         let versioned_schema = self
             .reader
             .get_schema(&reference.node_id, &reference.schema_hash)
@@ -1305,7 +1305,7 @@ impl<'s, S: SubstateDatabase> EngineStateDataLoader<'s, S> {
     fn to_object_collection_key(
         substate_key: SubstateKey,
         collection_meta: &ObjectCollectionMeta,
-    ) -> SborCollectionKey {
+    ) -> SborCollectionKey<'_> {
         match (&collection_meta.kind, substate_key) {
             (ObjectCollectionKind::KeyValueStore, SubstateKey::Map(key)) => {
                 SborCollectionKey::KeyValueStore(SborData::new(

core-rust/node-common/src/locks.rs

@@ -309,7 +309,7 @@ impl<'s, D: Snapshottable<'s>> DbLock<D> {
     /// do not leave database inconsistent.
     // TODO(future enhancement): also provide something like `lock_snapshot_unlock()`, which would
     // not need the "writes are atomic+consistent" assumption?
-    pub fn snapshot(&'s self) -> impl Deref<Target = D::Snapshot> + '_ {
+    pub fn snapshot(&'s self) -> impl Deref<Target = D::Snapshot> + 's {
         LockGuard::new(
             || OwnedDeref {
                 owned: self.database.snapshot(),

core-rust/node-common/src/store/rocks_db.rs

@@ -99,7 +99,7 @@ pub trait ReadableRocks {
         &self,
         cf: &impl AsColumnFamilyRef,
         key: impl AsRef<[u8]>,
-    ) -> Option<DBPinnableSlice>;
+    ) -> Option<DBPinnableSlice<'_>>;
 
     /// Gets multiple values by keys.
     ///
@@ -121,7 +121,7 @@ pub trait WriteableRocks: ReadableRocks {
     fn write(&self, batch: WriteBatch);
 
     /// Returns a snapshot of the current state.
-    fn snapshot(&self) -> SnapshotRocks;
+    fn snapshot(&self) -> SnapshotRocks<'_>;
 }
 
 /// A [`ReadableRocks`] instance opened as secondary instance.
@@ -162,7 +162,7 @@ impl ReadableRocks for DirectRocks {
         &self,
         cf: &impl AsColumnFamilyRef,
         key: impl AsRef<[u8]>,
-    ) -> Option<DBPinnableSlice> {
+    ) -> Option<DBPinnableSlice<'_>> {
         self.db.get_pinned_cf(cf, key).expect("DB get by key")
     }
 
@@ -183,7 +183,7 @@ impl WriteableRocks for DirectRocks {
         self.db.write(batch).expect("DB write batch");
     }
 
-    fn snapshot(&self) -> SnapshotRocks {
+    fn snapshot(&self) -> SnapshotRocks<'_> {
         SnapshotRocks {
             db: &self.db,
             snapshot: self.db.snapshot(),
@@ -248,7 +248,7 @@ impl<'db> ReadableRocks for SnapshotRocks<'db> {
         &self,
         cf: &impl AsColumnFamilyRef,
         key: impl AsRef<[u8]>,
-    ) -> Option<DBPinnableSlice> {
+    ) -> Option<DBPinnableSlice<'_>> {
         self.snapshot
             .get_pinned_cf(cf, key)
             .expect("snapshot DB get by key")

core-rust/node-common/src/utils.rs

@@ -65,18 +65,14 @@
 use crate::prelude::*;
 
 /// An implementation helper for a runtime-safe "capture a single expected value" functionality.
+#[derive(Default)]
 pub enum CaptureSupport<T> {
+    #[default]
     NotExpecting,
     Expecting,
     Captured(T),
 }
 
-impl<T> Default for CaptureSupport<T> {
-    fn default() -> Self {
-        Self::NotExpecting
-    }
-}
-
 impl<T> CaptureSupport<T> {
     /// Configures this instance to expect a single [`Self::capture_value()`] call.
     pub fn expect_capture(&mut self) {

core-rust/p2p/src/rocks_db.rs

@@ -127,7 +127,9 @@ fn new_rocks_db(root_path: PathBuf, column_families: &[&str]) -> DB {
     DB::open_cf_descriptors(&db_opts, root_path.as_path(), column_families).unwrap()
 }
 
-fn open_rw_context<R: WriteableRocks>(db: &R) -> TypedDbContext<R, BufferedWriteSupport<R>> {
+fn open_rw_context<R: WriteableRocks>(
+    db: &R,
+) -> TypedDbContext<'_, R, BufferedWriteSupport<'_, R>> {
     TypedDbContext::new(db, BufferedWriteSupport::new(db))
 }
 

core-rust/state-manager/src/accumulator_tree/slice_merger.rs

@@ -109,7 +109,7 @@ impl<N> AccuTreeSliceMerger<N> {
                 break;
             }
             let mut appended_nodes = appended_level.unwrap().nodes;
-            merged_to = (merged_to + 1) / 2;
+            merged_to = merged_to.div_ceil(2);
             appended_from /= 2;
             merged_nodes.truncate(merged_nodes.len() + appended_from - merged_to);
             merged_nodes.append(&mut appended_nodes);

core-rust/state-manager/src/accumulator_tree/tree_builder.rs

@@ -124,7 +124,7 @@ impl<'s, S: AccuTreeStore<usize, M>, M: Merklizable> AccuTree<'s, S, M> {
         let target_height = usize::BITS - (target_length - 1).leading_zeros();
         for _ in 0..target_height {
             let previous_slice_level = previous_slice_levels.next();
-            let left_sibling_cache = if from % 2 == 0 {
+            let left_sibling_cache = if from.is_multiple_of(2) {
                 None
             } else {
                 let previous_slice_level = previous_slice_level.unwrap();
@@ -138,7 +138,7 @@ impl<'s, S: AccuTreeStore<usize, M>, M: Merklizable> AccuTree<'s, S, M> {
                 TreeSliceLevel::new(left_sibling_cache, higher_level_nodes),
             );
 
-            let to = (from + lower_level_access.slice_level.nodes.len() + 1) / 2;
+            let to = (from + lower_level_access.slice_level.nodes.len()).div_ceil(2);
             from /= 2;
             higher_level_nodes = (from..to)
                 .map(|level_index| level_index * 2)

core-rust/state-manager/src/protocol/protocol_config.rs

@@ -259,8 +259,7 @@ impl ProtocolVersionName {
     pub fn padded_len_16_version_name_for_readiness_signal(&self) -> String {
         self.validate()
             .expect("Must be valid before extracting readiness signal name");
-        std::iter::repeat('0')
-            .take(16 - self.0.len())
+        std::iter::repeat_n('0', 16 - self.0.len())
             .chain(self.0.chars())
             .collect()
     }

core-rust/state-manager/src/protocol/protocol_updates/definitions/custom_definition.rs

@@ -77,7 +77,7 @@ impl NodeProtocolUpdateGenerator for ArbitraryNodeBatchGenerator {
         false
     }
 
-    fn batch_groups(&self) -> Vec<Box<dyn NodeProtocolUpdateBatchGroupGenerator + '_>> {
+    fn batch_groups(&self) -> Vec<Box<dyn NodeProtocolUpdateBatchGroupGenerator<'_> + '_>> {
         let mut batch_group = NodeFixedBatchGroupGenerator::named(Self::BATCH_GROUP_DESCRIPTOR);
         for (index, batch) in self.batches.iter().enumerate() {
             batch_group = batch_group.add_batch(format!("batch-{index:02}"), |_| batch.clone())

core-rust/state-manager/src/protocol/protocol_updates/protocol_content_overrides.rs

@@ -124,7 +124,7 @@ impl RawProtocolUpdateContentOverrides {
         Default::default()
     }
 
-    pub fn iter(&self) -> hash_map::Iter<ProtocolVersionName, Vec<u8>> {
+    pub fn iter(&self) -> hash_map::Iter<'_, ProtocolVersionName, Vec<u8>> {
         self.0.iter()
     }
 

core-rust/state-manager/src/protocol/protocol_updates/protocol_updaters.rs

@@ -36,7 +36,7 @@ pub trait NodeProtocolUpdateGenerator {
     /// Return the list of batch groups for the protocol update.
     ///
     /// Each should be a fixed, conceptual step in the update process.
-    fn batch_groups(&self) -> Vec<Box<dyn NodeProtocolUpdateBatchGroupGenerator + '_>>;
+    fn batch_groups(&self) -> Vec<Box<dyn NodeProtocolUpdateBatchGroupGenerator<'_> + '_>>;
 }
 
 /// Each batch group is a logical grouping of batches.
@@ -107,7 +107,7 @@ impl NodeProtocolUpdateGenerator for WrappedProtocolUpdateGenerator {
             .insert_status_tracking_flash_transactions()
     }
 
-    fn batch_groups(&self) -> Vec<Box<dyn NodeProtocolUpdateBatchGroupGenerator + '_>> {
+    fn batch_groups(&self) -> Vec<Box<dyn NodeProtocolUpdateBatchGroupGenerator<'_> + '_>> {
         self.engine_generator
             .batch_groups()
             .into_iter()
@@ -236,7 +236,7 @@ impl<B: NodeProtocolUpdateGenerator> NodeProtocolUpdateGenerator
         })
     }
 
-    fn batch_groups(&self) -> Vec<Box<dyn NodeProtocolUpdateBatchGroupGenerator + '_>> {
+    fn batch_groups(&self) -> Vec<Box<dyn NodeProtocolUpdateBatchGroupGenerator<'_> + '_>> {
         let mut batch_groups = self.base_batch_generator.batch_groups();
         if !self.scenario_names.is_empty() {
             batch_groups.insert(

core-rust/state-manager/src/receipt.rs

@@ -594,6 +594,29 @@ impl<T> BySubstate<T> {
             })
     }
 
+    pub fn iter_mut(&mut self) -> impl Iterator<Item = (SubstateReference, &mut T)> + '_ {
+        self.by_node_id
+            .iter_mut()
+            .flat_map(|(node_id, by_partition_num)| {
+                by_partition_num
+                    .iter_mut()
+                    .flat_map(move |(partition_num, by_substate_key)| {
+                        by_substate_key
+                            .iter_mut()
+                            .map(move |(substate_key, element)| {
+                                (
+                                    SubstateReference(
+                                        *node_id,
+                                        *partition_num,
+                                        substate_key.clone(),
+                                    ),
+                                    element,
+                                )
+                            })
+                    })
+            })
+    }
+
     pub fn iter_node_ids(&self) -> impl Iterator<Item = &NodeId> + '_ {
         self.by_node_id.keys()
     }

core-rust/state-manager/src/staging/cache.rs

@@ -106,13 +106,10 @@ impl ExecutionCacheManager {
         let execution_cache = self.execution_cache.lock();
         let mut transaction_root = parent_transaction_root;
         for transaction in transactions {
-            transaction_root = match execution_cache.get_cached_transaction_root(
+            transaction_root = execution_cache.get_cached_transaction_root(
                 transaction_root,
                 &transaction.ledger_transaction_hash(),
-            ) {
-                Some(cached) => cached,
-                None => return None,
-            }
+            )?
         }
         Some(*transaction_root)
     }

core-rust/state-manager/src/staging/epoch_handling.rs

@@ -96,7 +96,7 @@ impl EpochAwareAccuTreeFactory {
         &'s self,
         previous_epoch_root: N,
         store: &'s mut S,
-    ) -> EpochAccuTreeBuilder<S, N> {
+    ) -> EpochAccuTreeBuilder<'s, S, N> {
         EpochAccuTreeBuilder::new(
             store,
             self.epoch_start_state_version,

core-rust/state-manager/src/store/historical_state.rs

@@ -264,10 +264,10 @@ impl<'s, R: ReadableRocks + 's, DS: Deref<Target = StateManagerDatabase<R>>>
             return Err(StateHistoryError::StateVersionInFuture { current_version });
         }
 
-        return Ok(Self::Historical(StateTreeBasedSubstateDatabase::new(
+        Ok(Self::Historical(StateTreeBasedSubstateDatabase::new(
             database,
             requested_version,
-        )));
+        )))
     }
 
     /// Returns the summary of the ledger's state at which this store is scoped.