Skip to content

Commit ca9aba7

Browse files
committed
CORE-285 reworking storage and protocol upgrade
- **Disk-backed indexed.** `block_height_index` / `switch_block_era_id_index` / `transaction_hash_index` are now LMDB tables. If a node boots against a store where they're missing or empty, they're rebuilt from a one-off full header scan - **`IndexedLmdbBlockStore` merged into `LmdbBlockStore`.** Eliminated the separate wrapper type and the largely pass-through `DataReader`/`DataWriter` duplication; index-aware logic now lives directly on `LmdbBlockStore`. Also removed the now-unused in-memory `temp_map`. - **Protocol upgrade commit restructured.** Dropped `ReactorState::Upgrading` / `upgrading_instruction.rs`. The commit now happens synchronously in `MainReactor::new`, before the event loop starts, for a node restarting with its tip already at the pre-activation switch block; `CatchUp` just finishes (signs + gossips) the resulting immediate switch block. Also removed a redundant in-`CatchUp` commit path for nodes syncing through a _historical_ activation point — verified empirically against `dev` that this case never needs it; those nodes just fetch the post-upgrade chain normally, like any other historical data. - **New test coverage**: `emergency_upgrade.rs` (hard-reset upgrade with block peeling) and `legacy_storage_reindex_and_upgrade.rs`, which boots a 4-node network from a committed pre-refactor block store (real blocks/transactions across 3 eras, no persisted indexes), confirms the indexes rebuild and the network stays live, then drives it through an ordinary protocol upgrade. - **Test harness additions** in `fixture.rs`: `new_with_keys_and_storage_dirs` (boot nodes from pre-existing storage) and a generalized `schedule_upgrade` (arbitrary era/version, not just era 2).
1 parent 8177511 commit ca9aba7

35 files changed

Lines changed: 2793 additions & 1962 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎executor/evm/src/block_hash.rs‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,7 @@
33
use std::sync::Arc;
44

55
use casper_storage::block_store::{
6-
lmdb::IndexedLmdbBlockStore, types::BlockHeight, BlockStoreError, BlockStoreProvider,
7-
DataReader,
6+
lmdb::LmdbBlockStore, types::BlockHeight, BlockStoreError, BlockStoreProvider, DataReader,
87
};
98
use casper_types::{BlockHash, BlockHeader};
109

@@ -42,12 +41,12 @@ impl BlockHashProvider for NoBlockHashProvider {
4241
/// Block hash provider backed by Casper's indexed LMDB block store.
4342
#[derive(Clone, Debug)]
4443
pub struct IndexedLmdbBlockHashProvider {
45-
block_store: Arc<IndexedLmdbBlockStore>,
44+
block_store: Arc<LmdbBlockStore>,
4645
}
4746

4847
impl IndexedLmdbBlockHashProvider {
4948
/// Creates a block hash provider backed by `block_store`.
50-
pub fn new(block_store: Arc<IndexedLmdbBlockStore>) -> Self {
49+
pub fn new(block_store: Arc<LmdbBlockStore>) -> Self {
5150
Self { block_store }
5251
}
5352
}

‎node/src/components/block_accumulator/tests.rs‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -187,10 +187,20 @@ impl Reactor for MockReactor {
187187
)
188188
.unwrap();
189189

190-
let storage = Storage::new(
190+
let protocol_version = ProtocolVersion::from_parts(1, 0, 0);
191+
let (storage_root, mut storage_block_store) =
192+
storage::open_block_store(&storage_withdir, "test").unwrap();
193+
storage::prune_block_store(
194+
&mut storage_block_store,
195+
chainspec.hard_reset_to_start_of_era(),
196+
protocol_version,
197+
)
198+
.unwrap();
199+
let mut storage = Storage::new(
191200
&storage_withdir,
192-
None,
193-
ProtocolVersion::from_parts(1, 0, 0),
201+
storage_root,
202+
storage_block_store,
203+
protocol_version,
194204
EraId::default(),
195205
"test",
196206
chainspec.transaction_config.max_ttl.into(),
@@ -200,6 +210,7 @@ impl Reactor for MockReactor {
200210
TransactionConfig::default(),
201211
)
202212
.unwrap();
213+
storage.initialize_for_test();
203214

204215
let reactor = MockReactor {
205216
storage,

‎node/src/components/contract_runtime.rs‎

Lines changed: 52 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ mod utils;
1414

1515
use std::{
1616
cmp::Ordering,
17-
collections::BTreeMap,
1817
convert::TryInto,
1918
fmt::{self, Debug, Formatter},
2019
path::Path,
@@ -33,7 +32,7 @@ use casper_storage::{
3332
data_access_layer::{
3433
AddressableEntityRequest, AddressableEntityResult, BlockStore, DataAccessLayer,
3534
EntryPointExistsRequest, ExecutionResultsChecksumRequest, FlushRequest, FlushResult,
36-
GenesisRequest, GenesisResult, TrieRequest,
35+
GenesisRequest, GenesisResult, ProtocolUpgradeRequest, ProtocolUpgradeResult, TrieRequest,
3736
},
3837
global_state::{
3938
state::{lmdb::LmdbGlobalState, CommitProvider, StateProvider},
@@ -44,13 +43,13 @@ use casper_storage::{
4443
tracking_copy::TrackingCopyError,
4544
};
4645
use casper_types::{
47-
account::AccountHash, ActivationPoint, Chainspec, ChainspecRawBytes, ChainspecRegistry,
48-
EntityAddr, EraId, Key, PublicKey,
46+
account::AccountHash, ActivationPoint, Chainspec, ChainspecRawBytes, ChainspecRegistry, Digest,
47+
EntityAddr, EraId, Key, ProtocolUpgradeConfig,
4948
};
5049

5150
use crate::{
5251
components::{fetcher::FetchResponse, Component, ComponentState},
53-
contract_runtime::{types::EraPrice, utils::handle_protocol_upgrade},
52+
contract_runtime::types::EraPrice,
5453
effect::{
5554
announcements::{
5655
ContractRuntimeAnnouncement, FatalAnnouncement, MetaBlockAnnouncement,
@@ -62,10 +61,7 @@ use crate::{
6261
},
6362
fatal,
6463
protocol::Message,
65-
types::{
66-
BlockPayload, ExecutableBlock, FinalizedBlock, InternalEraReport, MetaBlockState,
67-
TrieOrChunk, TrieOrChunkId,
68-
},
64+
types::{TrieOrChunk, TrieOrChunkId},
6965
NodeRng,
7066
};
7167
pub(crate) use config::Config;
@@ -307,6 +303,47 @@ impl ContractRuntime {
307303
result
308304
}
309305

306+
/// Commits a protocol upgrade against global state and flushes it to disk.
307+
///
308+
/// The commit itself runs on the blocking thread-pool (via `run_intensive_task`), since it
309+
/// can take a long time against a large global state; this lets the caller bound the wait
310+
/// with a timeout instead of stalling its task indefinitely.
311+
pub(crate) async fn commit_protocol_upgrade(
312+
&self,
313+
upgrade_config: ProtocolUpgradeConfig,
314+
) -> Result<Digest, String> {
315+
debug!(?upgrade_config, "upgrade");
316+
let start = Instant::now();
317+
let upgrade_request = ProtocolUpgradeRequest::new(upgrade_config);
318+
319+
let data_access_layer = Arc::clone(&self.data_access_layer);
320+
let metrics = Arc::clone(&self.metrics);
321+
run_intensive_task(move || {
322+
let result = data_access_layer.protocol_upgrade(upgrade_request);
323+
if result.is_success() {
324+
info!("committed upgrade");
325+
metrics
326+
.commit_upgrade
327+
.observe(start.elapsed().as_secs_f64());
328+
let flush_req = FlushRequest::new();
329+
if let FlushResult::Failure(err) = data_access_layer.flush(flush_req) {
330+
return Err(format!("{:?}", err));
331+
}
332+
}
333+
334+
match result {
335+
ProtocolUpgradeResult::RootNotFound => {
336+
Err("Root not found for protocol upgrade".to_string())
337+
}
338+
ProtocolUpgradeResult::Failure(err) => Err(format!("{:?}", err)),
339+
ProtocolUpgradeResult::Success {
340+
post_state_hash, ..
341+
} => Ok(post_state_hash),
342+
}
343+
})
344+
.await
345+
}
346+
310347
/// Handles a contract runtime request.
311348
fn handle_contract_runtime_request<REv>(
312349
&mut self,
@@ -546,91 +583,6 @@ impl ContractRuntime {
546583
}
547584
.ignore()
548585
}
549-
ContractRuntimeRequest::UpdatePreState { new_pre_state } => {
550-
let next_block_height = new_pre_state.next_block_height();
551-
self.set_execution_pre_state(new_pre_state);
552-
let current_price = self.current_gas_price.gas_price();
553-
async move {
554-
let block_header = match effect_builder
555-
.get_highest_complete_block_header_from_storage()
556-
.await
557-
{
558-
Some(header)
559-
if header.is_switch_block()
560-
&& (header.height() + 1 == next_block_height) =>
561-
{
562-
header
563-
}
564-
Some(_) => {
565-
return fatal!(
566-
effect_builder,
567-
"Latest complete block is not a switch block to update state"
568-
)
569-
.await;
570-
}
571-
None => {
572-
return fatal!(
573-
effect_builder,
574-
"No complete block header found to update post upgrade state"
575-
)
576-
.await;
577-
}
578-
};
579-
580-
let payload = BlockPayload::new(
581-
BTreeMap::new(),
582-
vec![],
583-
Default::default(),
584-
false,
585-
current_price,
586-
);
587-
588-
let finalized_block = FinalizedBlock::new(
589-
payload,
590-
Some(InternalEraReport::default()),
591-
block_header.timestamp(),
592-
block_header.next_block_era_id(),
593-
next_block_height,
594-
PublicKey::System,
595-
);
596-
597-
info!("Enqueuing block for execution post state refresh");
598-
599-
effect_builder
600-
.enqueue_block_for_execution(
601-
ExecutableBlock::from_finalized_block_and_transactions(
602-
finalized_block,
603-
vec![],
604-
),
605-
MetaBlockState::new_not_to_be_gossiped(),
606-
)
607-
.await;
608-
}
609-
.ignore()
610-
}
611-
ContractRuntimeRequest::DoProtocolUpgrade {
612-
protocol_upgrade_config,
613-
next_block_height,
614-
parent_hash,
615-
parent_seed,
616-
} => {
617-
let mut effects = Effects::new();
618-
let data_access_layer = Arc::clone(&self.data_access_layer);
619-
let metrics = Arc::clone(&self.metrics);
620-
effects.extend(
621-
handle_protocol_upgrade(
622-
effect_builder,
623-
data_access_layer,
624-
metrics,
625-
*protocol_upgrade_config,
626-
next_block_height,
627-
parent_hash,
628-
parent_seed,
629-
)
630-
.ignore(),
631-
);
632-
effects
633-
}
634586
ContractRuntimeRequest::EnqueueBlockForExecution {
635587
executable_block,
636588
key_block_height_for_activation_point,
@@ -847,6 +799,12 @@ impl ContractRuntime {
847799
pub(crate) fn current_era_price(&self) -> EraPrice {
848800
self.current_gas_price
849801
}
802+
803+
/// Returns the current gas price, for use when building the deterministic block payload of
804+
/// a post-upgrade (or post-genesis) immediate switch block.
805+
pub(crate) fn current_gas_price(&self) -> u8 {
806+
self.current_gas_price.gas_price()
807+
}
850808
}
851809

852810
impl<REv> Component<REv> for ContractRuntime

‎node/src/components/contract_runtime/tests.rs‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -124,9 +124,18 @@ impl reactor::Reactor for Reactor {
124124
}
125125

126126
let storage_withdir = WithDir::new(storage_tempdir.path(), storage_config);
127-
let storage = Storage::new(
127+
let (storage_root, mut storage_block_store) =
128+
storage::open_block_store(&storage_withdir, "test").unwrap();
129+
storage::prune_block_store(
130+
&mut storage_block_store,
131+
chainspec.hard_reset_to_start_of_era(),
132+
chainspec.protocol_version(),
133+
)
134+
.unwrap();
135+
let mut storage = Storage::new(
128136
&storage_withdir,
129-
None,
137+
storage_root,
138+
storage_block_store,
130139
chainspec.protocol_version(),
131140
EraId::default(),
132141
"test",
@@ -137,6 +146,7 @@ impl reactor::Reactor for Reactor {
137146
TransactionConfig::default(),
138147
)
139148
.unwrap();
149+
storage.initialize_for_test();
140150

141151
let contract_runtime =
142152
ContractRuntime::new(storage.root_path(), &config.config, chainspec, registry)?;

‎node/src/components/contract_runtime/utils.rs‎

Lines changed: 3 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ use std::{
1010
fmt::Debug,
1111
ops::Range,
1212
sync::{Arc, Mutex},
13-
time::Instant,
1413
};
1514
use tracing::{debug, error, info};
1615

@@ -36,15 +35,10 @@ use crate::{
3635
use casper_binary_port::SpeculativeExecutionResult;
3736
use casper_execution_engine::engine_state::{ExecutionEngineV1, WasmV1Result};
3837
use casper_storage::{
39-
data_access_layer::{
40-
DataAccessLayer, FlushRequest, FlushResult, ProtocolUpgradeRequest, ProtocolUpgradeResult,
41-
TransferResult,
42-
},
43-
global_state::state::{lmdb::LmdbGlobalState, CommitProvider, StateProvider},
44-
};
45-
use casper_types::{
46-
BlockHash, Chainspec, Digest, EraId, Gas, Key, ProtocolUpgradeConfig, Transaction,
38+
data_access_layer::{DataAccessLayer, TransferResult},
39+
global_state::state::lmdb::LmdbGlobalState,
4740
};
41+
use casper_types::{BlockHash, Chainspec, EraId, Gas, Key, Transaction};
4842

4943
/// Maximum number of resource intensive tasks that can be run in parallel.
5044
///
@@ -493,75 +487,6 @@ pub(super) async fn exec_and_check_next<REv>(
493487
}
494488
}
495489

496-
pub(super) async fn handle_protocol_upgrade<REv>(
497-
effect_builder: EffectBuilder<REv>,
498-
data_access_layer: Arc<DataAccessLayer<LmdbGlobalState>>,
499-
metrics: Arc<Metrics>,
500-
upgrade_config: ProtocolUpgradeConfig,
501-
next_block_height: u64,
502-
parent_hash: BlockHash,
503-
parent_seed: Digest,
504-
) where
505-
REv: From<ContractRuntimeRequest>
506-
+ From<ContractRuntimeAnnouncement>
507-
+ From<StorageRequest>
508-
+ From<MetaBlockAnnouncement>
509-
+ From<FatalAnnouncement>
510-
+ Send,
511-
{
512-
debug!(?upgrade_config, "upgrade");
513-
let start = Instant::now();
514-
let upgrade_request = ProtocolUpgradeRequest::new(upgrade_config);
515-
516-
let result = run_intensive_task(move || {
517-
let result = data_access_layer.protocol_upgrade(upgrade_request);
518-
if result.is_success() {
519-
info!("committed upgrade");
520-
metrics
521-
.commit_upgrade
522-
.observe(start.elapsed().as_secs_f64());
523-
let flush_req = FlushRequest::new();
524-
if let FlushResult::Failure(err) = data_access_layer.flush(flush_req) {
525-
return Err(format!("{:?}", err));
526-
}
527-
}
528-
529-
Ok(result)
530-
})
531-
.await;
532-
533-
match result {
534-
Err(error_msg) => {
535-
// The only way this happens is if there is a problem in the flushing.
536-
error!(%error_msg, ":Error in post upgrade flush");
537-
fatal!(effect_builder, "{}", error_msg).await;
538-
}
539-
Ok(result) => match result {
540-
ProtocolUpgradeResult::RootNotFound => {
541-
let error_msg = "Root not found for protocol upgrade";
542-
fatal!(effect_builder, "{}", error_msg).await;
543-
}
544-
ProtocolUpgradeResult::Failure(err) => {
545-
fatal!(effect_builder, "{:?}", err).await;
546-
}
547-
ProtocolUpgradeResult::Success {
548-
post_state_hash, ..
549-
} => {
550-
let post_upgrade_state = ExecutionPreState::new(
551-
next_block_height,
552-
post_state_hash,
553-
parent_hash,
554-
parent_seed,
555-
);
556-
557-
effect_builder
558-
.update_contract_runtime_state(post_upgrade_state)
559-
.await
560-
}
561-
},
562-
}
563-
}
564-
565490
fn generate_range_by_index(
566491
highest_era: u64,
567492
batch_size: u64,

0 commit comments

Comments
 (0)