Skip to content
Open
Show file tree
Hide file tree
Changes from 18 commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
1036ca6
db: migrate from lmdb-zero to heed
ardocrat Apr 16, 2026
02cce56
fix: check resizing operation and wait to avoid crash with multiple b…
ardocrat Apr 16, 2026
593f4c4
build: fix missing deps at Cargo.lock
ardocrat Apr 16, 2026
fcf0884
lmdb: single environment, migrate existing databases with provided no…
ardocrat Apr 16, 2026
08e95ce
fix: revert chunk size to 128mb
ardocrat Apr 16, 2026
f9a04ff
lmdb: ability to use multiple shared environments
ardocrat Apr 20, 2026
5adec7a
build: remove unused dependency
ardocrat Apr 20, 2026
b5eeb37
fix: resize to have correct multiplier of the system page size
ardocrat Apr 21, 2026
b18f453
lmdb: speed up prefix iter by storing keys
ardocrat Apr 22, 2026
eef000d
lmdb: default env name
ardocrat Apr 22, 2026
70040d2
lmdb: wait db resize before read, reduce timeout before resizing
ardocrat Apr 24, 2026
deb5b49
lmdb: use static reader for iterator, count existing batches for stab…
ardocrat Apr 24, 2026
e22f5ac
fix: check batches count on resize waiting
ardocrat Apr 24, 2026
9ca157d
lmdb: use separate databases instead of prefixes, use default db for …
ardocrat Apr 27, 2026
4cadce1
fix: pop pos key
ardocrat Apr 27, 2026
f41d188
lmdb: count all open transactions to finish before resizing
ardocrat Apr 29, 2026
d52cfe1
lmdb: immediate resize if there are no open transactions
ardocrat Apr 29, 2026
9d0925a
lmdb: remove env state when there are no more stores
ardocrat Apr 30, 2026
f921f87
lmdb: use atomic for resize and resize checking flags
ardocrat May 15, 2026
22bc944
lmdb: sleep 10ms when waiting all opened txs to be closed
ardocrat May 15, 2026
aa28adc
lmdb: use atomic open txs and stores count
ardocrat May 15, 2026
7d251d2
lmdb: use index to detect separator, ignore unknown db key to not hav…
ardocrat May 15, 2026
d217bda
lmdb: store max 10k keys in the iterator
ardocrat May 16, 2026
418d865
lmdb: check iter result on getting total
ardocrat May 16, 2026
fc66b73
lmdb: handle errors at iterator
ardocrat May 17, 2026
630bb2e
lmdb: handle an error when db with provided key not found
ardocrat May 17, 2026
cbf2caf
lmdb: fix iterate over 10k keys
ardocrat May 17, 2026
b97fa28
lmdb: document migration resize safety
ardocrat May 17, 2026
17cd0b9
lmdb: fix iter test
ardocrat May 17, 2026
7bf4609
lmdb: clear new db after unsuccessful migration, handle read error on…
ardocrat May 17, 2026
9f29af8
store: bring back old key methods to reproduce data migration
ardocrat May 17, 2026
2dd726f
lmdb: return an error on unsuccessful migration
ardocrat May 18, 2026
7a387d4
lmdb: migration test, clean data after allocate test
ardocrat May 18, 2026
a24f30e
Merge branch 'staging' into lmdb_update
ardocrat May 18, 2026
dedaea2
lmdb: info migration log
ardocrat May 19, 2026
e9e38bd
fix: move iterator before handling an error to allow skip bad value
ardocrat May 25, 2026
aca71f7
lmdb: return an error if removal of old DB file failed after migration
ardocrat May 25, 2026
6d10bea
lmdb: lifetime for iterator, use write transaction at batch iterator
ardocrat May 25, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
245 changes: 187 additions & 58 deletions Cargo.lock

Large diffs are not rendered by default.

49 changes: 25 additions & 24 deletions chain/src/chain.rs
Original file line number Diff line number Diff line change
Expand Up @@ -150,8 +150,8 @@ pub struct Chain {
store: Arc<store::ChainStore>,
adapter: Arc<dyn ChainAdapter + Send + Sync>,
orphans: Arc<OrphanBlockPool>,
txhashset: Arc<RwLock<txhashset::TxHashSet>>,
header_pmmr: Arc<RwLock<txhashset::PMMRHandle<BlockHeader>>>,
txhashset: Arc<RwLock<TxHashSet>>,
header_pmmr: Arc<RwLock<PMMRHandle<BlockHeader>>>,
pibd_segmenter: Arc<RwLock<Option<Segmenter>>>,
pibd_desegmenter: Arc<RwLock<Option<Desegmenter>>>,
// POW verification function
Expand Down Expand Up @@ -189,9 +189,9 @@ impl Chain {
// Initialize the output_pos index based on UTXO set
// and NRD kernel_pos index based recent kernel history.
{
let batch = store.batch()?;
txhashset.init_output_pos_index(&header_pmmr, &batch)?;
txhashset.init_recent_kernel_pos_index(&header_pmmr, &batch)?;
let mut batch = store.batch()?;
txhashset.init_output_pos_index(&header_pmmr, &mut batch)?;
txhashset.init_recent_kernel_pos_index(&header_pmmr, &mut batch)?;
batch.commit()?;
}

Expand Down Expand Up @@ -275,7 +275,7 @@ impl Chain {
pub fn reset_chain_head_to_genesis(&self) -> Result<(), Error> {
let mut header_pmmr = self.header_pmmr.write();
let mut txhashset = self.txhashset.write();
let batch = self.store.batch()?;
let mut batch = self.store.batch()?;

// Change head back to genesis
{
Expand Down Expand Up @@ -314,7 +314,7 @@ impl Chain {

/// Reset PIBD head
pub fn reset_pibd_head(&self) -> Result<(), Error> {
let batch = self.store.batch()?;
let mut batch = self.store.batch()?;
batch.save_pibd_head(&self.genesis().into())?;
Ok(())
}
Expand Down Expand Up @@ -530,9 +530,9 @@ impl Chain {
pub fn new_ctx<'a>(
&self,
opts: Options,
batch: store::Batch<'a>,
header_pmmr: &'a mut txhashset::PMMRHandle<BlockHeader>,
txhashset: &'a mut txhashset::TxHashSet,
batch: Batch<'a>,
header_pmmr: &'a mut PMMRHandle<BlockHeader>,
txhashset: &'a mut TxHashSet,
) -> Result<pipe::BlockContext<'a>, Error> {
let denylist = self.denylist.read().clone();
Ok(pipe::BlockContext {
Expand Down Expand Up @@ -832,7 +832,7 @@ impl Chain {
&self,
header: &BlockHeader,
ext: &mut ExtensionPair,
batch: &Batch,
batch: &mut Batch,
) -> Result<BlockHeader, Error> {
let denylist = self.denylist.read().clone();
pipe::rewind_and_apply_fork(header, ext, batch, &|header| {
Expand All @@ -846,7 +846,7 @@ impl Chain {
&self,
header: &BlockHeader,
ext: &mut HeaderExtension,
batch: &Batch,
batch: &mut Batch,
) -> Result<(), Error> {
let denylist = self.denylist.read().clone();
pipe::rewind_and_apply_header_fork(header, ext, batch, &|header| {
Expand Down Expand Up @@ -1015,7 +1015,7 @@ impl Chain {
fn validate_kernel_history(
&self,
header: &BlockHeader,
txhashset: &txhashset::TxHashSet,
txhashset: &TxHashSet,
) -> Result<(), Error> {
debug!("validate_kernel_history: rewinding and validating kernel history (readonly)");

Expand Down Expand Up @@ -1151,11 +1151,11 @@ impl Chain {
self.validate_kernel_history(&header, &txhashset)?;

let header_pmmr = self.header_pmmr.read();
let batch = self.store.batch()?;
let mut batch = self.store.batch()?;
txhashset.verify_kernel_pos_index(
&self.genesis.header,
&header_pmmr,
&batch,
&mut batch,
None,
None,
)?;
Expand Down Expand Up @@ -1213,10 +1213,10 @@ impl Chain {
}

// Rebuild our output_pos index in the db based on fresh UTXO set.
txhashset.init_output_pos_index(&header_pmmr, &batch)?;
txhashset.init_output_pos_index(&header_pmmr, &mut batch)?;

// Rebuild our NRD kernel_pos index based on recent kernel history.
txhashset.init_recent_kernel_pos_index(&header_pmmr, &batch)?;
txhashset.init_recent_kernel_pos_index(&header_pmmr, &mut batch)?;

// Commit all the changes to the db.
batch.commit()?;
Expand Down Expand Up @@ -1257,9 +1257,9 @@ impl Chain {
/// *Only* runs if we are not in archive mode.
fn remove_historical_blocks(
&self,
header_pmmr: &txhashset::PMMRHandle<BlockHeader>,
header_pmmr: &PMMRHandle<BlockHeader>,
archive_header: BlockHeader,
batch: &store::Batch<'_>,
batch: &mut Batch<'_>,
) -> Result<(), Error> {
if self.archive_mode() {
return Ok(());
Expand Down Expand Up @@ -1345,7 +1345,7 @@ impl Chain {
// Take a write lock on the txhashet and start a new writeable db batch.
let header_pmmr = self.header_pmmr.read();
let mut txhashset = self.txhashset.write();
let batch = self.store.batch()?;
let mut batch = self.store.batch()?;

// Compact the txhashset itself (rewriting the pruned backend files).
{
Expand All @@ -1361,15 +1361,15 @@ impl Chain {

// If we are not in archival mode remove historical blocks from the db.
if !self.archive_mode() {
self.remove_historical_blocks(&header_pmmr, archive_header, &batch)?;
self.remove_historical_blocks(&header_pmmr, archive_header, &mut batch)?;
}

// Make sure our output_pos index is consistent with the UTXO set.
txhashset.init_output_pos_index(&header_pmmr, &batch)?;
txhashset.init_output_pos_index(&header_pmmr, &mut batch)?;

// TODO - Why is this part of chain compaction?
// Rebuild our NRD kernel_pos index based on recent kernel history.
txhashset.init_recent_kernel_pos_index(&header_pmmr, &batch)?;
txhashset.init_recent_kernel_pos_index(&header_pmmr, &mut batch)?;

// Commit all the above db changes.
batch.commit()?;
Expand Down Expand Up @@ -1439,7 +1439,8 @@ impl Chain {
0
} else {
self.get_header_by_height(start_block_height - 1)?
.output_mmr_size + 1
.output_mmr_size
+ 1
};
let end_mmr_size = self.get_header_by_height(end_block_height)?.output_mmr_size;
Ok((start_mmr_size, end_mmr_size))
Expand Down
Loading
Loading