From 55d465beb295b5e1216a17558e11b09d245631ee Mon Sep 17 00:00:00 2001 From: ardocrat Date: Thu, 14 May 2026 13:15:44 +0300 Subject: [PATCH 01/17] Update rust edition to 2021 (#3834) * build: update rust edition 2021 * fix: add missed build to grin_p2p from staging branch * cargo: format all --- Cargo.toml | 2 +- api/Cargo.toml | 2 +- chain/Cargo.toml | 2 +- chain/src/chain.rs | 3 ++- config/Cargo.toml | 2 +- core/Cargo.toml | 2 +- core/fuzz/Cargo.toml | 2 +- core/src/pow/cuckaroo.rs | 2 +- core/src/pow/cuckarood.rs | 2 +- core/src/pow/cuckaroom.rs | 2 +- core/src/pow/cuckarooz.rs | 2 +- core/tests/core.rs | 37 +++++++++---------------------------- etc/gen_gen/_Cargo.toml | 2 +- keychain/Cargo.toml | 2 +- p2p/Cargo.toml | 2 +- pool/Cargo.toml | 2 +- pool/fuzz/Cargo.toml | 2 +- pool/src/pool.rs | 4 +--- rustfmt.toml | 2 +- servers/Cargo.toml | 2 +- store/Cargo.toml | 2 +- util/Cargo.toml | 2 +- util/src/zip.rs | 20 ++++++++++---------- 23 files changed, 41 insertions(+), 61 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b4b26e16..c8e43720 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ keywords = [ "crypto", "grin", "mimblewimble" ] readme = "README.md" exclude = ["**/*.grin", "**/*.grin2"] build = "src/build/build.rs" -edition = "2018" +edition = "2021" [workspace] members = ["api", "chain", "config", "core", "keychain", "p2p", "servers", "store", "util", "pool"] diff --git a/api/Cargo.toml b/api/Cargo.toml index 88af1575..e87d0b4f 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -7,7 +7,7 @@ license = "Apache-2.0" repository = "https://github.com/mimblewimble/grin" keywords = [ "crypto", "grin", "mimblewimble" ] workspace = ".." -edition = "2018" +edition = "2021" [dependencies] easy-jsonrpc-mw = "0.5.4" diff --git a/chain/Cargo.toml b/chain/Cargo.toml index c9fab243..3b930231 100644 --- a/chain/Cargo.toml +++ b/chain/Cargo.toml @@ -7,7 +7,7 @@ license = "Apache-2.0" repository = "https://github.com/mimblewimble/grin" keywords = [ "crypto", "grin", "mimblewimble" ] workspace = ".." -edition = "2018" +edition = "2021" [dependencies] bit-vec = "0.6" diff --git a/chain/src/chain.rs b/chain/src/chain.rs index 273a4415..e566d1bc 100644 --- a/chain/src/chain.rs +++ b/chain/src/chain.rs @@ -1439,7 +1439,8 @@ impl Chain { 0 } else { self.get_header_by_height(start_block_height - 1)? - .output_mmr_size + 1 + .output_mmr_size + + 1 }; let end_mmr_size = self.get_header_by_height(end_block_height)?.output_mmr_size; Ok((start_mmr_size, end_mmr_size)) diff --git a/config/Cargo.toml b/config/Cargo.toml index 5700ea18..c7ea4743 100644 --- a/config/Cargo.toml +++ b/config/Cargo.toml @@ -7,7 +7,7 @@ license = "Apache-2.0" repository = "https://github.com/mimblewimble/grin" keywords = [ "crypto", "grin", "mimblewimble" ] workspace = ".." -edition = "2018" +edition = "2021" [dependencies] rand = "0.6" diff --git a/core/Cargo.toml b/core/Cargo.toml index e9229e2f..40798834 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -7,7 +7,7 @@ license = "Apache-2.0" repository = "https://github.com/mimblewimble/grin" keywords = [ "crypto", "grin", "mimblewimble" ] workspace = ".." -edition = "2018" +edition = "2021" [dependencies] blake2 = { package = "blake2-rfc", version = "0.2"} diff --git a/core/fuzz/Cargo.toml b/core/fuzz/Cargo.toml index 158391ad..1cf4ffdb 100644 --- a/core/fuzz/Cargo.toml +++ b/core/fuzz/Cargo.toml @@ -3,7 +3,7 @@ name = "grin_core-fuzz" version = "0.0.3" authors = ["Grin Developers "] publish = false -edition = "2018" +edition = "2021" [dependencies] libfuzzer-sys = "0.4.0" diff --git a/core/src/pow/cuckaroo.rs b/core/src/pow/cuckaroo.rs index 05e42f36..1cb62d7c 100644 --- a/core/src/pow/cuckaroo.rs +++ b/core/src/pow/cuckaroo.rs @@ -71,7 +71,7 @@ impl PoWContext for CuckarooContext { let mut xor0: u64 = 0; let mut xor1: u64 = 0; let mask = u64::MAX >> (size as u64).leading_zeros(); // round size up to 2-power - 1 - // the next three arrays form a linked list of nodes with matching bits 6..1 + // the next three arrays form a linked list of nodes with matching bits 6..1 let mut headu = vec![2 * size; 1 + mask as usize]; let mut headv = vec![2 * size; 1 + mask as usize]; let mut prev = vec![0usize; 2 * size]; diff --git a/core/src/pow/cuckarood.rs b/core/src/pow/cuckarood.rs index ed4c73f1..1e6e5b68 100644 --- a/core/src/pow/cuckarood.rs +++ b/core/src/pow/cuckarood.rs @@ -66,7 +66,7 @@ impl PoWContext for CuckaroodContext { let mut xor0: u64 = 0; let mut xor1: u64 = 0; let mask = u64::MAX >> (size as u64).leading_zeros(); // round size up to 2-power - 1 - // the next two arrays form a linked list of nodes with matching bits 4..0|dir + // the next two arrays form a linked list of nodes with matching bits 4..0|dir let mut headu = vec![2 * size; 1 + mask as usize]; let mut headv = vec![2 * size; 1 + mask as usize]; let mut prev = vec![0usize; 2 * size]; diff --git a/core/src/pow/cuckaroom.rs b/core/src/pow/cuckaroom.rs index 82fc031d..28fe1d56 100644 --- a/core/src/pow/cuckaroom.rs +++ b/core/src/pow/cuckaroom.rs @@ -65,7 +65,7 @@ impl PoWContext for CuckaroomContext { let mut xor_from: u64 = 0; let mut xor_to: u64 = 0; let mask = u64::MAX >> (size as u64).leading_zeros(); // round size up to 2-power - 1 - // the next two arrays form a linked list of nodes with matching bits 6..1 + // the next two arrays form a linked list of nodes with matching bits 6..1 let mut head = vec![size; 1 + mask as usize]; let mut prev = vec![0usize; size]; diff --git a/core/src/pow/cuckarooz.rs b/core/src/pow/cuckarooz.rs index 5705a530..2a179627 100644 --- a/core/src/pow/cuckarooz.rs +++ b/core/src/pow/cuckarooz.rs @@ -64,7 +64,7 @@ impl PoWContext for CuckaroozContext { let mut uvs = vec![0u64; 2 * size]; let mut xoruv: u64 = 0; let mask = u64::MAX >> (size as u64).leading_zeros(); // round size up to 2-power - 1 - // the next two arrays form a linked list of nodes with matching bits 6..1 + // the next two arrays form a linked list of nodes with matching bits 6..1 let mut head = vec![2 * size; 1 + mask as usize]; let mut prev = vec![0usize; 2 * size]; diff --git a/core/tests/core.rs b/core/tests/core.rs index f2d6d32d..b0e35ace 100644 --- a/core/tests/core.rs +++ b/core/tests/core.rs @@ -244,7 +244,6 @@ fn transaction_cut_through() { let tx1 = tx1i2o(); let tx2 = tx2i1o(); - assert!(tx1.validate(Weighting::AsTransaction).is_ok()); assert!(tx2.validate(Weighting::AsTransaction).is_ok()); @@ -277,17 +276,12 @@ fn multi_kernel_transaction_deaggregation() { assert!(tx34.validate(Weighting::AsTransaction).is_ok()); let deaggregated_tx34 = deaggregate(tx1234.clone(), &[tx12.clone()]).unwrap(); - assert!(deaggregated_tx34 - .validate(Weighting::AsTransaction) - - .is_ok()); + assert!(deaggregated_tx34.validate(Weighting::AsTransaction).is_ok()); assert_eq!(tx34, deaggregated_tx34); let deaggregated_tx12 = deaggregate(tx1234, &[tx34]).unwrap(); - assert!(deaggregated_tx12 - .validate(Weighting::AsTransaction) - .is_ok()); + assert!(deaggregated_tx12.validate(Weighting::AsTransaction).is_ok()); assert_eq!(tx12, deaggregated_tx12); } @@ -309,9 +303,7 @@ fn multi_kernel_transaction_deaggregation_2() { assert!(tx12.validate(Weighting::AsTransaction).is_ok()); let deaggregated_tx3 = deaggregate(tx123, &[tx12]).unwrap(); - assert!(deaggregated_tx3 - .validate(Weighting::AsTransaction) - .is_ok()); + assert!(deaggregated_tx3.validate(Weighting::AsTransaction).is_ok()); assert_eq!(tx3, deaggregated_tx3); } @@ -334,9 +326,7 @@ fn multi_kernel_transaction_deaggregation_3() { assert!(tx2.validate(Weighting::AsTransaction).is_ok()); let deaggregated_tx13 = deaggregate(tx123, &[tx2]).unwrap(); - assert!(deaggregated_tx13 - .validate(Weighting::AsTransaction) - .is_ok()); + assert!(deaggregated_tx13.validate(Weighting::AsTransaction).is_ok()); assert_eq!(tx13, deaggregated_tx13); } @@ -366,9 +356,7 @@ fn multi_kernel_transaction_deaggregation_4() { assert!(tx12345.validate(Weighting::AsTransaction).is_ok()); let deaggregated_tx5 = deaggregate(tx12345, &[tx1, tx2, tx3, tx4]).unwrap(); - assert!(deaggregated_tx5 - .validate(Weighting::AsTransaction) - .is_ok()); + assert!(deaggregated_tx5.validate(Weighting::AsTransaction).is_ok()); assert_eq!(tx5, deaggregated_tx5); } @@ -401,9 +389,7 @@ fn multi_kernel_transaction_deaggregation_5() { assert!(tx12345.validate(Weighting::AsTransaction).is_ok()); let deaggregated_tx5 = deaggregate(tx12345, &[tx12, tx34]).unwrap(); - assert!(deaggregated_tx5 - .validate(Weighting::AsTransaction) - .is_ok()); + assert!(deaggregated_tx5.validate(Weighting::AsTransaction).is_ok()); assert_eq!(tx5, deaggregated_tx5); } @@ -424,16 +410,12 @@ fn basic_transaction_deaggregation() { let deaggregated_tx1 = deaggregate(tx3.clone(), &[tx2.clone()]).unwrap(); - assert!(deaggregated_tx1 - .validate(Weighting::AsTransaction) - .is_ok()); + assert!(deaggregated_tx1.validate(Weighting::AsTransaction).is_ok()); assert_eq!(tx1, deaggregated_tx1); let deaggregated_tx2 = deaggregate(tx3, &[tx1]).unwrap(); - assert!(deaggregated_tx2 - .validate(Weighting::AsTransaction) - .is_ok()); + assert!(deaggregated_tx2.validate(Weighting::AsTransaction).is_ok()); assert_eq!(tx2, deaggregated_tx2); } @@ -558,8 +540,7 @@ fn reward_with_tx_block() { let tx1 = tx2i1o(); let previous_header = BlockHeader::default(); - tx1.validate(Weighting::AsTransaction) - .unwrap(); + tx1.validate(Weighting::AsTransaction).unwrap(); let block = new_block(&[tx1], &keychain, &builder, &previous_header, &key_id); block.validate(&BlindingFactor::zero()).unwrap(); diff --git a/etc/gen_gen/_Cargo.toml b/etc/gen_gen/_Cargo.toml index f053cda1..ba53139b 100644 --- a/etc/gen_gen/_Cargo.toml +++ b/etc/gen_gen/_Cargo.toml @@ -1,7 +1,7 @@ [package] name = "grin_gen_gen" version = "0.0.1" -edition = "2018" +edition = "2021" authors = ["Grin Developers "] description = "Utility to automate the generation of Grin's genesis block" license = "Apache-2.0" diff --git a/keychain/Cargo.toml b/keychain/Cargo.toml index 2d46b63d..1a7d1950 100644 --- a/keychain/Cargo.toml +++ b/keychain/Cargo.toml @@ -7,7 +7,7 @@ license = "Apache-2.0" repository = "https://github.com/mimblewimble/grin" keywords = [ "crypto", "grin", "mimblewimble" ] workspace = '..' -edition = "2018" +edition = "2021" [dependencies] byteorder = "1" diff --git a/p2p/Cargo.toml b/p2p/Cargo.toml index 539feb39..69a2ca43 100644 --- a/p2p/Cargo.toml +++ b/p2p/Cargo.toml @@ -7,7 +7,7 @@ license = "Apache-2.0" repository = "https://github.com/mimblewimble/grin" keywords = [ "crypto", "grin", "mimblewimble" ] workspace = ".." -edition = "2018" +edition = "2021" build = "src/build/build.rs" [dependencies] diff --git a/pool/Cargo.toml b/pool/Cargo.toml index 3b0f77ad..8ff37d24 100644 --- a/pool/Cargo.toml +++ b/pool/Cargo.toml @@ -7,7 +7,7 @@ license = "Apache-2.0" repository = "https://github.com/mimblewimble/grin" keywords = [ "crypto", "grin", "mimblewimble" ] workspace = '..' -edition = "2018" +edition = "2021" [dependencies] blake2-rfc = "0.2" diff --git a/pool/fuzz/Cargo.toml b/pool/fuzz/Cargo.toml index a10d61a7..c0aacd1f 100644 --- a/pool/fuzz/Cargo.toml +++ b/pool/fuzz/Cargo.toml @@ -3,7 +3,7 @@ name = "grin_pool_fuzz" version = "0.0.0" authors = ["Automatically generated"] publish = false -edition = "2018" +edition = "2021" [dependencies] chrono = "0.4.11" diff --git a/pool/src/pool.rs b/pool/src/pool.rs index 3e470b16..f0dbe389 100644 --- a/pool/src/pool.rs +++ b/pool/src/pool.rs @@ -402,9 +402,7 @@ where // Otherwise discard and let the next block pick this tx up. let bucket = &tx_buckets[pos]; - if let Ok(new_bucket) = - bucket.aggregate_with_tx(entry.tx.clone(), weighting) - { + if let Ok(new_bucket) = bucket.aggregate_with_tx(entry.tx.clone(), weighting) { if new_bucket.fee_rate >= bucket.fee_rate { // Only aggregate if it would not reduce the fee_rate ratio. tx_buckets[pos] = new_bucket; diff --git a/rustfmt.toml b/rustfmt.toml index b2b24311..c691d12a 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -1,2 +1,2 @@ hard_tabs = true -edition = "2018" +edition = "2021" diff --git a/servers/Cargo.toml b/servers/Cargo.toml index cb162771..16d8f775 100644 --- a/servers/Cargo.toml +++ b/servers/Cargo.toml @@ -7,7 +7,7 @@ license = "Apache-2.0" repository = "https://github.com/mimblewimble/grin" keywords = [ "crypto", "grin", "mimblewimble" ] workspace = ".." -edition = "2018" +edition = "2021" [dependencies] hyper = { version = "0.14", features = ["full"] } diff --git a/store/Cargo.toml b/store/Cargo.toml index 72f30ca8..9974a4c3 100644 --- a/store/Cargo.toml +++ b/store/Cargo.toml @@ -7,7 +7,7 @@ license = "Apache-2.0" repository = "https://github.com/mimblewimble/grin" keywords = [ "crypto", "grin", "mimblewimble" ] workspace = ".." -edition = "2018" +edition = "2021" [dependencies] byteorder = "1" diff --git a/util/Cargo.toml b/util/Cargo.toml index 978b5568..a20a9759 100644 --- a/util/Cargo.toml +++ b/util/Cargo.toml @@ -7,7 +7,7 @@ license = "Apache-2.0" repository = "https://github.com/mimblewimble/grin" keywords = [ "crypto", "grin", "mimblewimble" ] workspace = ".." -edition = "2018" +edition = "2021" [dependencies] anyhow = "1.0" diff --git a/util/src/zip.rs b/util/src/zip.rs index 967738ef..7e436c1e 100644 --- a/util/src/zip.rs +++ b/util/src/zip.rs @@ -24,16 +24,16 @@ use zip as zip_rs; // Sanitize file path for normal components, excluding '/', '..', and '.' // From private function in zip crate fn path_to_string(path: &std::path::Path) -> String { - let mut path_str = String::new(); - for component in path.components() { - if let std::path::Component::Normal(os_str) = component { - if !path_str.is_empty() { - path_str.push('/'); - } - path_str.push_str(&*os_str.to_string_lossy()); - } - } - path_str + let mut path_str = String::new(); + for component in path.components() { + if let std::path::Component::Normal(os_str) = component { + if !path_str.is_empty() { + path_str.push('/'); + } + path_str.push_str(&*os_str.to_string_lossy()); + } + } + path_str } /// Create a zip archive from source dir and list of relative file paths. From f921f8758cf2084f76bce9957d44bc1cf14e9fc1 Mon Sep 17 00:00:00 2001 From: ardocrat Date: Fri, 15 May 2026 10:52:09 +0300 Subject: [PATCH 02/17] lmdb: use atomic for resize and resize checking flags --- store/src/lmdb.rs | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/store/src/lmdb.rs b/store/src/lmdb.rs index 6d0b9405..ea07c1c4 100644 --- a/store/src/lmdb.rs +++ b/store/src/lmdb.rs @@ -18,6 +18,7 @@ use heed::types::Bytes; use heed::{Database, Env, EnvOpenOptions, RoTxn, RwTxn, WithoutTls}; use std::collections::HashMap; use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, OnceLock}; use std::time::Duration; use std::{fs, thread}; @@ -96,8 +97,8 @@ static ENV_MAP: OnceLock>> = OnceLock::new(); struct EnvState { env: Env, open_txs_count: u32, - resizing: bool, - resize_checking: bool, + resizing: AtomicBool, + resize_checking: AtomicBool, stores_count: u32, } @@ -194,8 +195,8 @@ impl Store { EnvState { env, open_txs_count: 0, - resizing: false, - resize_checking: false, + resizing: AtomicBool::new(false), + resize_checking: AtomicBool::new(false), stores_count: 1, }, ); @@ -310,6 +311,7 @@ impl Store { .get(&self.env_path) .unwrap() .resize_checking + .load(Ordering::Relaxed) } /// Set flag if requirement for environment resize is checking. @@ -320,7 +322,8 @@ impl Store { .write() .get_mut(&self.env_path) .unwrap() - .resize_checking = resize_checking; + .resize_checking + .store(resize_checking, Ordering::Relaxed); } /// Wait while database is resizing. @@ -333,6 +336,7 @@ impl Store { .get(&self.env_path) .unwrap() .resizing + .load(Ordering::Relaxed) { break; } @@ -377,7 +381,8 @@ impl Store { .write() .get_mut(&env_path) .unwrap() - .resizing = true; + .resizing + .store(true, Ordering::Relaxed); // Wait to make sure there are no more active txs left. thread::sleep(Duration::from_millis(1000)); break; @@ -393,8 +398,8 @@ impl Store { let mut w_env_map = ENV_MAP.get().unwrap().write(); let env_state = w_env_map.get_mut(&env_path).unwrap(); - env_state.resizing = false; - env_state.resize_checking = false; + env_state.resizing.store(false, Ordering::Relaxed); + env_state.resize_checking.store(false, Ordering::Relaxed); }); return; } else { @@ -402,14 +407,14 @@ impl Store { let env_state = w_env_map.get_mut(&env_path).unwrap(); debug!("Start immediate resizing DB {}", env_path); - env_state.resizing = true; + env_state.resizing.store(true, Ordering::Relaxed); unsafe { match env.resize(new_size) { Ok(_) => debug!("End resizing DB {}", env_path), Err(e) => error!("Resize DB {} error: {:?}", env_path, e), } } - env_state.resizing = false; + env_state.resizing.store(false, Ordering::Relaxed); } } From 22bc94491832157d907702f32221fcba1781da2e Mon Sep 17 00:00:00 2001 From: ardocrat Date: Fri, 15 May 2026 10:53:20 +0300 Subject: [PATCH 03/17] lmdb: sleep 10ms when waiting all opened txs to be closed --- store/src/lmdb.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/store/src/lmdb.rs b/store/src/lmdb.rs index ea07c1c4..d18dd6a0 100644 --- a/store/src/lmdb.rs +++ b/store/src/lmdb.rs @@ -387,6 +387,7 @@ impl Store { thread::sleep(Duration::from_millis(1000)); break; } + thread::sleep(Duration::from_millis(10)); } unsafe { From aa28adc6bd3f1523bff4586b2179295c72bf16cc Mon Sep 17 00:00:00 2001 From: ardocrat Date: Fri, 15 May 2026 18:20:13 +0300 Subject: [PATCH 04/17] lmdb: use atomic open txs and stores count --- store/src/lmdb.rs | 46 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/store/src/lmdb.rs b/store/src/lmdb.rs index d18dd6a0..c40f8021 100644 --- a/store/src/lmdb.rs +++ b/store/src/lmdb.rs @@ -18,7 +18,7 @@ use heed::types::Bytes; use heed::{Database, Env, EnvOpenOptions, RoTxn, RwTxn, WithoutTls}; use std::collections::HashMap; use std::path::Path; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::sync::{Arc, OnceLock}; use std::time::Duration; use std::{fs, thread}; @@ -96,10 +96,10 @@ static ENV_MAP: OnceLock>> = OnceLock::new(); /// State of active database environment. struct EnvState { env: Env, - open_txs_count: u32, + open_txs_count: AtomicU32, resizing: AtomicBool, resize_checking: AtomicBool, - stores_count: u32, + stores_count: AtomicU32, } /// LMDB-backed store facilitating data access and serialization. All writes @@ -117,7 +117,16 @@ impl Drop for Store { fn drop(&mut self) { { let mut w_map = ENV_MAP.get().unwrap().write(); - w_map.get_mut(&self.env_path).unwrap().stores_count -= 1; + let stores_count = w_map + .get(&self.env_path) + .unwrap() + .stores_count + .load(Ordering::Relaxed); + w_map + .get_mut(&self.env_path) + .unwrap() + .stores_count + .store(stores_count - 1, Ordering::Relaxed); } let no_stores = { ENV_MAP @@ -127,6 +136,7 @@ impl Drop for Store { .get(&self.env_path) .unwrap() .stores_count + .load(Ordering::Relaxed) == 0 }; if no_stores { @@ -194,15 +204,24 @@ impl Store { full_path.clone(), EnvState { env, - open_txs_count: 0, + open_txs_count: AtomicU32::new(0), resizing: AtomicBool::new(false), resize_checking: AtomicBool::new(false), - stores_count: 1, + stores_count: AtomicU32::new(1), }, ); } else { let mut w_env_map = env_map.write(); - w_env_map.get_mut(&full_path).unwrap().stores_count += 1; + let stores_count = w_env_map + .get(&full_path) + .unwrap() + .stores_count + .load(Ordering::Relaxed); + w_env_map + .get_mut(&full_path) + .unwrap() + .stores_count + .store(stores_count + 1, Ordering::Relaxed); } // Database setup. @@ -300,6 +319,7 @@ impl Store { .get(&self.env_path) .unwrap() .open_txs_count + .load(Ordering::Relaxed) } /// Check if requirement for environment resize is checking. @@ -372,7 +392,8 @@ impl Store { .read() .get(&env_path) .unwrap() - .open_txs_count; + .open_txs_count + .load(Ordering::Relaxed); if txs_count == 0 { debug!("Start resizing DB {}", env_path); ENV_MAP @@ -564,10 +585,15 @@ impl TxCounter { fn on_change_tx_count(env_path: &String, inc: bool) { let mut w_env_map = ENV_MAP.get().unwrap().write(); let env_state = w_env_map.get_mut(env_path).unwrap(); + let open_txs_count = env_state.open_txs_count.load(Ordering::Relaxed); if inc { - env_state.open_txs_count += 1; + env_state + .open_txs_count + .store(open_txs_count + 1, Ordering::Relaxed); } else { - env_state.open_txs_count -= 1; + env_state + .open_txs_count + .store(open_txs_count - 1, Ordering::Relaxed); } } } From 7d251d2267508b2eedc9d90d3eb732f859d469b8 Mon Sep 17 00:00:00 2001 From: ardocrat Date: Fri, 15 May 2026 18:38:25 +0300 Subject: [PATCH 05/17] lmdb: use index to detect separator, ignore unknown db key to not have a panic --- store/src/lmdb.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/store/src/lmdb.rs b/store/src/lmdb.rs index c40f8021..1c186a6a 100644 --- a/store/src/lmdb.rs +++ b/store/src/lmdb.rs @@ -280,7 +280,7 @@ impl Store { unsafe { from_env.resize(new_size)?; self.env.resize(new_size)?; - }; + } } let db_from = { let mut write = from_env.write_txn()?; @@ -293,12 +293,15 @@ impl Store { let mut count = 0; for kv in db_from.iter(&read_from)? { if let Ok((k, v)) = kv { - if k.contains(&PREFIX_KEY_SEPARATOR) { + if k.len() > 1 && k[1] == PREFIX_KEY_SEPARATOR { let db_name = k.split_at(1).0; - let db = self.pre_dbs.get(&db_name[0]).unwrap(); - let key = k.split_at(2).1; - db.put(&mut write_to, key, &v)?; - count += 1; + if let Some(db) = self.pre_dbs.get(&db_name[0]) { + let key = k.split_at(2).1; + db.put(&mut write_to, key, &v)?; + count += 1; + } else { + error!("Migration: unknown db key: {}", db_name[0]); + } } else { self.def_db.put(&mut write_to, k, &v)?; count += 1; From d217bda12ed385a72ce65b3b73f4a680d15a9665 Mon Sep 17 00:00:00 2001 From: ardocrat Date: Sat, 16 May 2026 12:44:21 +0300 Subject: [PATCH 06/17] lmdb: store max 10k keys in the iterator --- store/src/lmdb.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/store/src/lmdb.rs b/store/src/lmdb.rs index 1c186a6a..6e3196a4 100644 --- a/store/src/lmdb.rs +++ b/store/src/lmdb.rs @@ -759,6 +759,8 @@ where read: Arc>, keys: Vec>, skip: usize, + total_keys: usize, + skip_keys: usize, deserialize: F, #[allow(dead_code)] tx_counter: TxCounter, @@ -777,11 +779,26 @@ where return match (self.deserialize)(k, v) { Ok(v) => { self.skip += 1; + self.skip_keys += 1; Some(v) } Err(_) => None, }; } + } else if self.total_keys > self.skip_keys { + let keys = if let Ok(iter) = self.db.iter(&self.read) { + iter.move_between_keys() + .skip(self.skip_keys) + .take(10000) + .filter(|kv| kv.is_ok()) + .map(|kv| kv.unwrap().0.to_vec()) + .collect::>>() + } else { + vec![] + }; + self.skip = 0; + self.keys = keys; + return self.next(); } None } @@ -798,8 +815,10 @@ where read: RoTxn<'static, WithoutTls>, deserialize: F, ) -> DatabaseIterator { + let total_keys = db.iter(&read).unwrap().count(); let keys = if let Ok(iter) = db.iter(&read) { iter.move_between_keys() + .take(10000) .filter(|kv| kv.is_ok()) .map(|kv| kv.unwrap().0.to_vec()) .collect::>>() @@ -810,6 +829,8 @@ where db, read: Arc::new(read), keys, + total_keys, + skip_keys: 0, skip: 0, deserialize, tx_counter: TxCounter { From 418d865c01f55fcdea3de9e91338757d226aa59a Mon Sep 17 00:00:00 2001 From: ardocrat Date: Sat, 16 May 2026 12:51:25 +0300 Subject: [PATCH 07/17] lmdb: check iter result on getting total --- store/src/lmdb.rs | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/store/src/lmdb.rs b/store/src/lmdb.rs index 6e3196a4..f0b63aff 100644 --- a/store/src/lmdb.rs +++ b/store/src/lmdb.rs @@ -815,15 +815,20 @@ where read: RoTxn<'static, WithoutTls>, deserialize: F, ) -> DatabaseIterator { - let total_keys = db.iter(&read).unwrap().count(); - let keys = if let Ok(iter) = db.iter(&read) { - iter.move_between_keys() - .take(10000) - .filter(|kv| kv.is_ok()) - .map(|kv| kv.unwrap().0.to_vec()) - .collect::>>() + let (keys, total_keys) = if let Ok(iter) = db.iter(&read) { + let total = iter.move_between_keys().count(); + let keys = if let Ok(iter) = db.iter(&read) { + iter.move_between_keys() + .take(10000) + .filter(|kv| kv.is_ok()) + .map(|kv| kv.unwrap().0.to_vec()) + .collect::>>() + } else { + vec![] + }; + (keys, total) } else { - vec![] + (vec![], 0) }; DatabaseIterator { db, From fc66b73e5b448baa2c0a70cb3d409391d7b7c8f7 Mon Sep 17 00:00:00 2001 From: ardocrat Date: Sun, 17 May 2026 12:36:31 +0300 Subject: [PATCH 08/17] lmdb: handle errors at iterator --- chain/src/chain.rs | 8 +++--- chain/src/linked_list.rs | 12 ++++++--- chain/src/store.rs | 10 ++++--- chain/src/txhashset/txhashset.rs | 26 +++++++++--------- p2p/src/peers.rs | 5 +++- p2p/src/store.rs | 16 ++++++++--- store/src/lmdb.rs | 46 +++++++++++++++----------------- 7 files changed, 72 insertions(+), 51 deletions(-) diff --git a/chain/src/chain.rs b/chain/src/chain.rs index aabadc32..13325fad 100644 --- a/chain/src/chain.rs +++ b/chain/src/chain.rs @@ -1299,9 +1299,11 @@ impl Chain { // Remove old blocks (including short lived fork blocks) which height < tail.height for block in batch.blocks_iter()? { - if block.header.height < tail.height { - let _ = batch.delete_block(&block.hash()); - count += 1; + if let Ok(block) = block { + if block.header.height < tail.height { + let _ = batch.delete_block(&block.hash()); + count += 1; + } } } diff --git a/chain/src/linked_list.rs b/chain/src/linked_list.rs index 289ffd10..11b0a114 100644 --- a/chain/src/linked_list.rs +++ b/chain/src/linked_list.rs @@ -409,13 +409,17 @@ impl PruneableListIndex for MultiIndex { let mut entry_count = 0; let list_db_key = Some(self.list_prefix); for key in batch.db.iter(list_db_key, |k, _| Ok(k.to_vec()))? { - let _ = batch.delete(list_db_key, &key); - list_count += 1; + if let Ok(key) = key { + let _ = batch.delete(list_db_key, &key); + list_count += 1; + } } let entry_db_key = Some(self.entry_prefix); for key in batch.db.iter(entry_db_key, |k, _| Ok(k.to_vec()))? { - let _ = batch.delete(entry_db_key, &key); - entry_count += 1; + if let Ok(key) = key { + let _ = batch.delete(entry_db_key, &key); + entry_count += 1; + } } debug!( "clear: lists deleted: {}, entries deleted: {}", diff --git a/chain/src/store.rs b/chain/src/store.rs index 3a3df575..e5ed2c60 100644 --- a/chain/src/store.rs +++ b/chain/src/store.rs @@ -339,7 +339,9 @@ impl<'a> Batch<'a> { } /// Iterator over the output_pos index. - pub fn output_pos_iter(&self) -> Result, CommitPos)>, Error> { + pub fn output_pos_iter( + &self, + ) -> Result, CommitPos), Error>>, Error> { let protocol_version = self.db.protocol_version(); self.db.iter(Some(OUTPUT_POS_PREFIX), move |k, mut v| { ser::deserialize(&mut v, protocol_version, DeserializationMode::default()) @@ -459,7 +461,7 @@ impl<'a> Batch<'a> { /// Iterator over all full blocks in the db. /// Uses default db serialization strategy via db protocol version. - pub fn blocks_iter(&self) -> Result + 'a, Error> { + pub fn blocks_iter(&self) -> Result> + 'a, Error> { let protocol_version = self.db.protocol_version(); self.db.iter(Some(BLOCK_PREFIX), move |_, mut v| { ser::deserialize(&mut v, protocol_version, DeserializationMode::default()) @@ -469,7 +471,9 @@ impl<'a> Batch<'a> { /// Iterator over raw data for full blocks in the db. /// Used during block migration (we need flexibility around deserialization). - pub fn blocks_raw_iter(&self) -> Result, Vec)>, Error> { + pub fn blocks_raw_iter( + &self, + ) -> Result, Vec), Error>>, Error> { self.db .iter(Some(BLOCK_PREFIX), |k, v| Ok((k.to_vec(), v.to_vec()))) } diff --git a/chain/src/txhashset/txhashset.rs b/chain/src/txhashset/txhashset.rs index c9c55a70..8155a547 100644 --- a/chain/src/txhashset/txhashset.rs +++ b/chain/src/txhashset/txhashset.rs @@ -644,21 +644,23 @@ impl TxHashSet { // Iterate over the current output_pos index, removing any entries that // do not point to to the expected output. let mut removed_count = 0; - for (key, pos1) in batch.output_pos_iter()? { - let pos0 = pos1.pos - 1; - if let Some(out) = output_pmmr.get_data(pos0) { - if let Ok(pos0_via_mmr) = batch.get_output_pos(&out.commitment()) { - // If the pos matches and the index key matches the commitment - // then keep the entry, other we want to clean it up. - if pos0 == pos0_via_mmr - && batch.is_match_output_pos_key(&key, &out.commitment()) - { - continue; + for kp in batch.output_pos_iter()? { + if let Ok((key, pos1)) = kp { + let pos0 = pos1.pos - 1; + if let Some(out) = output_pmmr.get_data(pos0) { + if let Ok(pos0_via_mmr) = batch.get_output_pos(&out.commitment()) { + // If the pos matches and the index key matches the commitment + // then keep the entry, other we want to clean it up. + if pos0 == pos0_via_mmr + && batch.is_match_output_pos_key(&key, &out.commitment()) + { + continue; + } } } + batch.delete(Some(store::OUTPUT_POS_PREFIX), &key)?; + removed_count += 1; } - batch.delete(Some(store::OUTPUT_POS_PREFIX), &key)?; - removed_count += 1; } debug!( "init_output_pos_index: removed {} stale index entries", diff --git a/p2p/src/peers.rs b/p2p/src/peers.rs index 21340b10..23746477 100644 --- a/p2p/src/peers.rs +++ b/p2p/src/peers.rs @@ -279,7 +279,10 @@ impl Peers { pub fn all_peer_data(&self) -> Vec { match self.store.iter_batch() { Ok(batch) => match batch.peers_iter() { - Ok(iter) => iter.collect(), + Ok(iter) => iter + .filter(|p| p.is_ok()) + .map(|p| p.ok().unwrap()) + .collect(), Err(e) => { error!("failed to get all peer data: {:?}", e); vec![] diff --git a/p2p/src/store.rs b/p2p/src/store.rs index e0f32e9f..fe404e05 100644 --- a/p2p/src/store.rs +++ b/p2p/src/store.rs @@ -195,7 +195,7 @@ pub struct PeersIterBatch<'a> { impl<'a> PeersIterBatch<'a> { /// Iterator over all known peers. - pub fn peers_iter(&self) -> Result, Error> { + pub fn peers_iter(&self) -> Result>, Error> { let protocol_version = self.db.protocol_version(); self.db.iter(Some(PEER_PREFIX), move |_, mut v| { ser::deserialize(&mut v, protocol_version, DeserializationMode::default()) @@ -212,6 +212,8 @@ impl<'a> PeersIterBatch<'a> { ) -> Result, Error> { let peers = self .peers_iter()? + .filter(|p| p.is_ok()) + .map(|p| p.ok().unwrap()) .filter(|p| p.flags == state && p.capabilities.contains(cap)) .choose_multiple(&mut thread_rng(), count); Ok(peers) @@ -220,7 +222,11 @@ impl<'a> PeersIterBatch<'a> { /// List all known peers /// Used for /v1/peers/all api endpoint pub fn all_peers(&self) -> Result, Error> { - let peers: Vec = self.peers_iter()?.collect(); + let peers: Vec = self + .peers_iter()? + .filter(|p| p.is_ok()) + .map(|p| p.ok().unwrap()) + .collect(); Ok(peers) } @@ -232,8 +238,10 @@ impl<'a> PeersIterBatch<'a> { let mut to_remove = vec![]; for x in self.peers_iter()? { - if predicate(&x) { - to_remove.push(x) + if let Ok(x) = x { + if predicate(&x) { + to_remove.push(x) + } } } diff --git a/store/src/lmdb.rs b/store/src/lmdb.rs index f0b63aff..0ed76c11 100644 --- a/store/src/lmdb.rs +++ b/store/src/lmdb.rs @@ -770,35 +770,33 @@ impl Iterator for DatabaseIterator where F: Fn(&[u8], &[u8]) -> Result, { - type Item = T; + type Item = Result; fn next(&mut self) -> Option { if let Some(k) = self.keys.iter().skip(self.skip).next() { - let v = self.db.get(&self.read, k).unwrap_or(None); - if let Some(v) = v { - return match (self.deserialize)(k, v) { - Ok(v) => { - self.skip += 1; - self.skip_keys += 1; - Some(v) + match self.db.get(&self.read, k) { + Ok(v) => { + if let Some(v) = v { + return match (self.deserialize)(k, v) { + Ok(v) => { + self.skip += 1; + self.skip_keys += 1; + Some(Ok(v)) + } + Err(e) => { + error!("db iter: error deserializing: {}", e); + Some(Err(Error::from(e))) + } + }; } - Err(_) => None, - }; + } + Err(e) => { + return { + error!("db iter: error read value: {}", e); + Some(Err(Error::from(e))) + } + } } - } else if self.total_keys > self.skip_keys { - let keys = if let Ok(iter) = self.db.iter(&self.read) { - iter.move_between_keys() - .skip(self.skip_keys) - .take(10000) - .filter(|kv| kv.is_ok()) - .map(|kv| kv.unwrap().0.to_vec()) - .collect::>>() - } else { - vec![] - }; - self.skip = 0; - self.keys = keys; - return self.next(); } None } From 630bb2e5652da52784b9c90f4843986664b19ffb Mon Sep 17 00:00:00 2001 From: ardocrat Date: Sun, 17 May 2026 12:51:42 +0300 Subject: [PATCH 09/17] lmdb: handle an error when db with provided key not found --- store/src/lmdb.rs | 53 +++++++++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/store/src/lmdb.rs b/store/src/lmdb.rs index 0ed76c11..6db682b3 100644 --- a/store/src/lmdb.rs +++ b/store/src/lmdb.rs @@ -452,10 +452,16 @@ impl Store { } /// Get database from provided key or return default. - fn get_db(&self, db_key: Option) -> &Database { + fn get_db(&self, db_key: Option) -> Result<&Database, Error> { match db_key { - Some(db) => self.pre_dbs.get(&db).unwrap(), - None => &self.def_db, + Some(db) => { + if let Some(db) = self.pre_dbs.get(&db) { + Ok(db) + } else { + Err(Error::OtherErr("db for provided key not found".to_string())) + } + } + None => Ok(&self.def_db), } } @@ -471,7 +477,7 @@ impl Store { where F: Fn(&[u8], &[u8]) -> Result, { - let db = self.get_db(db_key); + let db = self.get_db(db_key)?; let res: Option<&[u8]> = db.get(read, key)?; match res { None => Ok(None), @@ -514,10 +520,15 @@ impl Store { let res = { match self.env.read_txn() { Ok(read) => { - let db = self.get_db(db_key); - let res = db.get(&read, key); - match res { - Ok(r) => Ok(r.is_some()), + let db_res = self.get_db(db_key); + match db_res { + Ok(db) => { + let res = db.get(&read, key); + match res { + Ok(r) => Ok(r.is_some()), + Err(e) => Err(Error::from(e)), + } + } Err(e) => Err(Error::from(e)), } } @@ -542,13 +553,19 @@ impl Store { TxCounter::on_change_tx_count(&self.env_path, true); match self.env.clone().static_read_txn() { Ok(read) => { - let db = self.get_db(db_key); - Ok(DatabaseIterator::new( - self, - Arc::new(db.clone()), - read, - deserialize, - )) + let db_res = self.get_db(db_key); + match db_res { + Ok(db) => Ok(DatabaseIterator::new( + self, + Arc::new(db.clone()), + read, + deserialize, + )), + Err(e) => { + TxCounter::on_change_tx_count(&self.env_path, false); + Err(Error::from(e)) + } + } } Err(e) => { TxCounter::on_change_tx_count(&self.env_path, false); @@ -624,7 +641,7 @@ impl<'a> Batch<'a> { /// Writes a single key/value pair to the provided database key. pub fn put(&mut self, db_key: Option, key: &[u8], value: &[u8]) -> Result<(), Error> { - let db = self.store.get_db(db_key); + let db = self.store.get_db(db_key)?; db.put(&mut self.write, key, value)?; Ok(()) } @@ -680,7 +697,7 @@ impl<'a> Batch<'a> { /// This is in the context of the current write transaction. pub fn exists(&self, db_key: Option, key: &[u8]) -> Result { let read = self.write.nested_read_txn()?; - let db = self.store.get_db(db_key); + let db = self.store.get_db(db_key)?; let res = db.get(&read, key)?; Ok(res.is_some()) } @@ -718,7 +735,7 @@ impl<'a> Batch<'a> { /// Deletes a key/value pair from the database. pub fn delete(&mut self, db_key: Option, key: &[u8]) -> Result<(), Error> { - let db = self.store.get_db(db_key); + let db = self.store.get_db(db_key)?; db.delete(&mut self.write, key)?; Ok(()) } From cbf2cafd69c2ed7cb821fdabac81ecb0583cc371 Mon Sep 17 00:00:00 2001 From: ardocrat Date: Sun, 17 May 2026 13:09:53 +0300 Subject: [PATCH 10/17] lmdb: fix iterate over 10k keys --- store/src/lmdb.rs | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/store/src/lmdb.rs b/store/src/lmdb.rs index 6db682b3..f66b90e8 100644 --- a/store/src/lmdb.rs +++ b/store/src/lmdb.rs @@ -775,9 +775,9 @@ where db: Arc>, read: Arc>, keys: Vec>, - skip: usize, total_keys: usize, - skip_keys: usize, + skip_cur: usize, + skip_total: usize, deserialize: F, #[allow(dead_code)] tx_counter: TxCounter, @@ -790,14 +790,14 @@ where type Item = Result; fn next(&mut self) -> Option { - if let Some(k) = self.keys.iter().skip(self.skip).next() { + if let Some(k) = self.keys.iter().skip(self.skip_cur).next() { match self.db.get(&self.read, k) { Ok(v) => { if let Some(v) = v { return match (self.deserialize)(k, v) { Ok(v) => { - self.skip += 1; - self.skip_keys += 1; + self.skip_total += 1; + self.skip_cur += 1; Some(Ok(v)) } Err(e) => { @@ -814,6 +814,20 @@ where } } } + } else if self.total_keys > self.skip_total { + let keys = if let Ok(iter) = self.db.iter(&self.read) { + iter.move_between_keys() + .skip(self.skip_total) + .take(10000) + .filter(|kv| kv.is_ok()) + .map(|kv| kv.unwrap().0.to_vec()) + .collect::>>() + } else { + vec![] + }; + self.skip_cur = 0; + self.keys = keys; + return self.next(); } None } @@ -850,8 +864,8 @@ where read: Arc::new(read), keys, total_keys, - skip_keys: 0, - skip: 0, + skip_cur: 0, + skip_total: 0, deserialize, tx_counter: TxCounter { env_path: store.env_path.clone(), From b97fa28aa49468ec0cfc4a765c48a5604c1bc458 Mon Sep 17 00:00:00 2001 From: ardocrat Date: Sun, 17 May 2026 13:14:36 +0300 Subject: [PATCH 11/17] lmdb: document migration resize safety --- store/src/lmdb.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/store/src/lmdb.rs b/store/src/lmdb.rs index f66b90e8..75cb6d88 100644 --- a/store/src/lmdb.rs +++ b/store/src/lmdb.rs @@ -277,6 +277,7 @@ impl Store { }; let (resize, new_size) = needs_resize(&from_env, self.alloc_chunk_size); if resize { + // We are sure there are no active txs, cause migration is called on database creation. unsafe { from_env.resize(new_size)?; self.env.resize(new_size)?; From 17cd0b9ed4415dc5b9a78682c705e6607d2d8bf2 Mon Sep 17 00:00:00 2001 From: ardocrat Date: Sun, 17 May 2026 13:25:15 +0300 Subject: [PATCH 12/17] lmdb: fix iter test --- store/tests/lmdb.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/tests/lmdb.rs b/store/tests/lmdb.rs index f8c68386..ac0b56f9 100644 --- a/store/tests/lmdb.rs +++ b/store/tests/lmdb.rs @@ -126,7 +126,7 @@ fn test_iter() -> Result<(), store::Error> { // Check we can see the new entry via an iterator after committing the batch. let mut iter = store.iter(Some(prefix), |_, v| Ok(v.to_vec()))?; - assert_eq!(iter.next(), Some(value.to_vec())); + assert_eq!(iter.next(), Some(Ok(value.to_vec()))); assert_eq!(iter.next(), None); clean_output_dir(test_dir); From 7bf460992d39f052647585f48d5b4307c9f9ad94 Mon Sep 17 00:00:00 2001 From: ardocrat Date: Sun, 17 May 2026 13:35:32 +0300 Subject: [PATCH 13/17] lmdb: clear new db after unsuccessful migration, handle read error on migration to interrupt process --- store/src/lmdb.rs | 46 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/store/src/lmdb.rs b/store/src/lmdb.rs index 75cb6d88..b0526ae2 100644 --- a/store/src/lmdb.rs +++ b/store/src/lmdb.rs @@ -255,7 +255,15 @@ impl Store { Ok(_) => { let _ = fs::remove_dir_all(&migrate_from); } - Err(e) => error!("DB {} migration error: {:?}", env_name, e), + Err(e) => { + error!("DB {} migration error: {:?}", env_name, e); + match s.clear() { + Ok(_) => {} + Err(e) => { + error!("Can not clear new DB after unsuccessful migration: {:?}", e) + } + } + } } } } @@ -293,24 +301,23 @@ impl Store { let read_from = from_env.read_txn()?; let mut count = 0; for kv in db_from.iter(&read_from)? { - if let Ok((k, v)) = kv { - if k.len() > 1 && k[1] == PREFIX_KEY_SEPARATOR { - let db_name = k.split_at(1).0; - if let Some(db) = self.pre_dbs.get(&db_name[0]) { - let key = k.split_at(2).1; - db.put(&mut write_to, key, &v)?; - count += 1; - } else { - error!("Migration: unknown db key: {}", db_name[0]); - } - } else { - self.def_db.put(&mut write_to, k, &v)?; + let (k, v) = kv?; + if k.len() > 1 && k[1] == PREFIX_KEY_SEPARATOR { + let db_name = k.split_at(1).0; + if let Some(db) = self.pre_dbs.get(&db_name[0]) { + let key = k.split_at(2).1; + db.put(&mut write_to, key, &v)?; count += 1; + } else { + error!("Migration: unknown DB key: {}", db_name[0]); } + } else { + self.def_db.put(&mut write_to, k, &v)?; + count += 1; } } write_to.commit()?; - debug!("Migrated {} records from DB {:?}", count, from_path); + debug!("Migrated {} records from {:?}", count, from_path); Ok(()) } @@ -447,6 +454,17 @@ impl Store { self.set_resize_checking(false); } + /// Clear all data from database environment. + fn clear(&self) -> Result<(), Error> { + let mut w = self.env.write_txn()?; + self.def_db.clear(&mut w)?; + for db in self.pre_dbs.values() { + db.clear(&mut w)?; + } + w.commit()?; + Ok(()) + } + /// Protocol version for the store. pub fn protocol_version(&self) -> ProtocolVersion { self.version From 9f29af8e14fa39d36973bdeadb7f58a1da69e655 Mon Sep 17 00:00:00 2001 From: ardocrat Date: Sun, 17 May 2026 13:43:27 +0300 Subject: [PATCH 14/17] store: bring back old key methods to reproduce data migration --- store/src/lib.rs | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/store/src/lib.rs b/store/src/lib.rs index 5814bf6c..b2408ca5 100644 --- a/store/src/lib.rs +++ b/store/src/lib.rs @@ -32,6 +32,39 @@ pub mod pmmr; pub mod prune_list; pub mod types; +const SEP: u8 = b':'; + +use byteorder::{BigEndian, WriteBytesExt}; + +/// Build a db key from a prefix and a byte vector identifier. +pub fn to_key>(prefix: u8, k: K) -> Vec { + let k = k.as_ref(); + let mut res = Vec::with_capacity(k.len() + 2); + res.push(prefix); + res.push(SEP); + res.extend_from_slice(k); + res +} + +/// Build a db key from a prefix and a byte vector identifier and numeric identifier +pub fn to_key_u64>(prefix: u8, k: K, val: u64) -> Vec { + let k = k.as_ref(); + let mut res = Vec::with_capacity(k.len() + 10); + res.push(prefix); + res.push(SEP); + res.extend_from_slice(k); + res.write_u64::(val).unwrap(); + res +} +/// Build a db key from a prefix and a numeric identifier. +pub fn u64_to_key(prefix: u8, val: u64) -> Vec { + let mut res = Vec::with_capacity(10); + res.push(prefix); + res.push(SEP); + res.write_u64::(val).unwrap(); + res +} + pub use crate::lmdb::*; use std::ffi::OsStr; From 2dd726f17b07b348060131e63d1a1bd605bc759c Mon Sep 17 00:00:00 2001 From: ardocrat Date: Mon, 18 May 2026 12:48:17 +0300 Subject: [PATCH 15/17] lmdb: return an error on unsuccessful migration --- store/src/lmdb.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/store/src/lmdb.rs b/store/src/lmdb.rs index b0526ae2..2f0452e4 100644 --- a/store/src/lmdb.rs +++ b/store/src/lmdb.rs @@ -84,7 +84,7 @@ where const DEFAULT_DB_VERSION: ProtocolVersion = ProtocolVersion(3); /// Default environment. -const DEFAULT_ENV_NAME: &'static str = "lmdb"; +pub const DEFAULT_ENV_NAME: &'static str = "lmdb"; /// Default multi-database environment without prefixes. const DEFAULT_MULTI_DB_ENV_NAME: &'static str = "multi_lmdb"; /// Prefix key separator. @@ -263,6 +263,7 @@ impl Store { error!("Can not clear new DB after unsuccessful migration: {:?}", e) } } + return Err(e); } } } From 7a387d434b31b19e3e43efaabceddcda9699cf3b Mon Sep 17 00:00:00 2001 From: ardocrat Date: Mon, 18 May 2026 13:22:24 +0300 Subject: [PATCH 16/17] lmdb: migration test, clean data after allocate test --- store/tests/lmdb.rs | 117 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 115 insertions(+), 2 deletions(-) diff --git a/store/tests/lmdb.rs b/store/tests/lmdb.rs index ac0b56f9..797614db 100644 --- a/store/tests/lmdb.rs +++ b/store/tests/lmdb.rs @@ -16,9 +16,17 @@ use grin_core as core; use grin_store as store; use grin_util as util; -use crate::core::global; -use crate::core::ser::{self, Readable, Reader, Writeable, Writer}; +use core::global; +use core::ser::{self, Readable, Reader, Writeable, Writer}; +use store::{ + needs_resize, to_key, to_key_u64, Store, ALLOC_CHUNK_SIZE_DEFAULT_TEST, DEFAULT_ENV_NAME, +}; + +use byteorder::WriteBytesExt; +use heed::types::Bytes; +use heed::{Database, Env, EnvOpenOptions, WithoutTls}; use std::fs; +use std::path::Path; const WRITE_CHUNK_SIZE: usize = 20; const TEST_ALLOC_SIZE: usize = store::lmdb::ALLOC_CHUNK_SIZE_DEFAULT / 8 / WRITE_CHUNK_SIZE; @@ -168,5 +176,110 @@ fn lmdb_allocate() -> Result<(), store::Error> { } } + clean_output_dir(test_dir); + Ok(()) +} + +fn create_old_db( + test_dir: &str, +) -> Result<(Database, Env), store::Error> { + let env_name = DEFAULT_ENV_NAME; + let alloc_chunk_size = ALLOC_CHUNK_SIZE_DEFAULT_TEST; + let full_path = Path::new(test_dir) + .join(env_name) + .to_str() + .unwrap() + .to_string(); + let _ = fs::create_dir_all(&full_path); + + let env = unsafe { + let mut options = EnvOpenOptions::new().read_txn_without_tls(); + let env_options = options.map_size(alloc_chunk_size).max_dbs(1); + env_options.open(&full_path)? + }; + let (resize, new_size) = needs_resize(&env, alloc_chunk_size); + if resize { + unsafe { + env.resize(new_size)?; + }; + } + + let mut write = env.write_txn()?; + let db: Database = env.create_database(&mut write, Some(env_name))?; + write.commit()?; + + Ok((db, env)) +} + +#[test] +fn test_migration() -> Result<(), store::Error> { + let test_dir = "target/test_migration"; + setup(test_dir); + + let test_prefix_1 = b'H'; + let test_key_1 = [0, 1, 2, 4]; + let test_data_1 = [1, 2, 3, 4]; + + let test_prefix_2 = b'G'; + let test_key_2 = [3, 4, 5, 6]; + let test_key_64_2 = 65480464; + let test_data_2 = [4, 5, 6, 7]; + + let test_key_3 = [6, 7, 8, 9]; + let test_data_3 = [7, 8, 9, 10]; + + // Create old db and fill the data. + { + let (old_db, old_env) = create_old_db(test_dir)?; + let mut w = old_env.write_txn()?; + + // Create old format key value. + let key_1 = to_key(test_prefix_1, test_key_1); + old_db.put(&mut w, key_1.as_slice(), test_data_1.as_slice())?; + + // Create old format 64 key value. + let key_2 = to_key_u64(test_prefix_2, test_key_2, test_key_64_2); + old_db.put(&mut w, key_2.as_slice(), test_data_2.as_slice())?; + + // Create key value without prefix. + old_db.put(&mut w, test_key_3.as_slice(), test_data_3.as_slice())?; + + w.commit()?; + } + + // Create new store to migrate data. + let store = Store::new( + test_dir, + None, + Some(DEFAULT_ENV_NAME), + vec![test_prefix_1, test_prefix_2], + None, + )?; + + // Check we can see key value. + { + assert!(store.exists(Some(test_prefix_1), &test_key_1)?); + let data = store.get_ser::>(Some(test_prefix_1), &test_key_1, None)?; + assert_eq!(data, Some(test_data_1.to_vec())); + } + + // Check we can see key 64 value. + { + let mut key = test_key_2.to_vec(); + key.write_u64::(test_key_64_2) + .unwrap(); + assert!(store.exists(Some(test_prefix_2), &key)?); + let data = store.get_ser::>(Some(test_prefix_2), &key, None)?; + assert_eq!(data, Some(test_data_2.to_vec())); + } + + // Check we can see key value without prefix. + { + assert!(store.exists(None, &test_key_3)?); + let data = store.get_ser::>(None, &test_key_3, None)?; + assert_eq!(data, Some(test_data_3.to_vec())); + } + + clean_output_dir(test_dir); Ok(()) } From dedaea205f455fd910e7e1f702a89f87283f8abf Mon Sep 17 00:00:00 2001 From: ardocrat Date: Tue, 19 May 2026 21:50:37 +0300 Subject: [PATCH 17/17] lmdb: info migration log --- store/src/lmdb.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/store/src/lmdb.rs b/store/src/lmdb.rs index 2f0452e4..ee4c977f 100644 --- a/store/src/lmdb.rs +++ b/store/src/lmdb.rs @@ -278,7 +278,7 @@ impl Store { from_name: Option<&str>, from_path: &Path, ) -> Result<(), Error> { - debug!("Migrating DB {:?}", from_path); + info!("Migrating DB {:?}, please wait...", from_path); let from_env = unsafe { let mut options = EnvOpenOptions::new().read_txn_without_tls(); let env_options = options.map_size(self.alloc_chunk_size).max_dbs(24); @@ -318,7 +318,7 @@ impl Store { } } write_to.commit()?; - debug!("Migrated {} records from {:?}", count, from_path); + info!("Migrated {} records from {:?}", count, from_path); Ok(()) }