From 244fc8d32ee1f2b1129ca8c77d8a5e16d530f7a2 Mon Sep 17 00:00:00 2001 From: Yeastplume Date: Thu, 15 Mar 2018 17:12:30 +0000 Subject: [PATCH] PMMR data compaction (#784) * fix and re-introduce data file compaction * rustfmt * remove commented line * test cleanup * remove println * fix to core test --- core/src/core/pmmr.rs | 16 ++++++- store/src/pmmr.rs | 104 ++++++++++++++++++++---------------------- store/tests/pmmr.rs | 8 ++-- 3 files changed, 68 insertions(+), 60 deletions(-) diff --git a/core/src/core/pmmr.rs b/core/src/core/pmmr.rs index db45de36..61d58d4f 100644 --- a/core/src/core/pmmr.rs +++ b/core/src/core/pmmr.rs @@ -63,14 +63,18 @@ where /// occurred (see remove). fn rewind(&mut self, position: u64, index: u32) -> Result<(), String>; - /// Get a Hash/Element by insertion position. If include_data is true, will + /// Get a Hash by insertion position. If include_data is true, will /// also return the associated data element fn get(&self, position: u64, include_data: bool) -> Option<(Hash, Option)>; - /// Get a Hash/Element by original insertion position (ignoring the remove + /// Get a Hash by original insertion position (ignoring the remove /// list). fn get_from_file(&self, position: u64) -> Option; + /// Get a Data Element by original insertion position (ignoring the remove + /// list). + fn get_data_from_file(&self, position: u64) -> Option; + /// Remove HashSums by insertion position. An index is also provided so the /// underlying backend can implement some rollback of positions up to a /// given index (practically the index is the height of a block that @@ -1027,6 +1031,14 @@ mod test { } } + fn get_data_from_file(&self, position: u64) -> Option { + if let Some(ref x) = self.elems[(position - 1) as usize] { + x.1.clone() + } else { + None + } + } + fn remove(&mut self, positions: Vec, _index: u32) -> Result<(), String> { for n in positions { self.remove_list.push(n) diff --git a/store/src/pmmr.rs b/store/src/pmmr.rs index 81f7cf45..875866c2 100644 --- a/store/src/pmmr.rs +++ b/store/src/pmmr.rs @@ -137,6 +137,30 @@ where } } + fn get_data_from_file(&self, position: u64) -> Option { + let shift = self.pruned_nodes.get_leaf_shift(position); + if let None = shift { + return None; + } + + let pos = pmmr::n_leaves(position) - 1; + + // Must be on disk, doing a read at the correct position + let record_len = T::len(); + let file_offset = ((pos - shift.unwrap()) as usize) * record_len; + let data = self.data_file.read(file_offset, record_len); + match ser::deserialize(&mut &data[..]) { + Ok(h) => Some(h), + Err(e) => { + error!( + LOGGER, + "Corrupted storage, could not read an entry from data store: {:?}", e + ); + return None; + } + } + } + /// Get a Hash by insertion position fn get(&self, position: u64, include_data: bool) -> Option<(Hash, Option)> { // Check if this position has been pruned in the remove log... @@ -144,12 +168,6 @@ where return None; } - // ... or in the prune list - // let prune_shift = match self.pruned_nodes.get_leaf_shift(position) { - // Some(shift) => shift, - // None => return None, - // }; - let hash_val = self.get_from_file(position); if !include_data { return hash_val.map(|hash| (hash, None)); @@ -160,24 +178,7 @@ where return hash_val.map(|hash| (hash, None)); } - // Optionally read flatfile storage to get data element - // let flatfile_pos = pmmr::n_leaves(position) - 1 - prune_shift; - let flatfile_pos = pmmr::n_leaves(position) - 1; - - let record_len = T::len(); - let file_offset = flatfile_pos as usize * T::len(); - let data = self.data_file.read(file_offset, record_len); - let data = match ser::deserialize(&mut &data[..]) { - Ok(elem) => Some(elem), - Err(e) => { - error!( - LOGGER, - "Corrupted storage, could not read an entry from backend flatfile store: {:?}", - e - ); - None - } - }; + let data = self.get_data_from_file(position); hash_val.map(|x| (x, data)) } @@ -351,8 +352,7 @@ where // Paths for tmp hash and data files. let tmp_prune_file_hash = format!("{}/{}.hashprune", self.data_dir, PMMR_HASH_FILE); - // let tmp_prune_file_data = format!("{}/{}.dataprune", self.data_dir, - // PMMR_DATA_FILE); + let tmp_prune_file_data = format!("{}/{}.dataprune", self.data_dir, PMMR_DATA_FILE); // Pos we want to get rid of. // Filtered by cutoff index. @@ -360,7 +360,7 @@ where // Filtered to exclude the subtree "roots". let pos_to_rm = removed_excl_roots(rm_pre_cutoff.clone()); // Filtered for leaves only. - // let leaf_pos_to_rm = removed_leaves(pos_to_rm.clone()); + let leaf_pos_to_rm = removed_leaves(pos_to_rm.clone()); // 1. Save compact copy of the hash file, skipping removed data. { @@ -383,27 +383,24 @@ where } // 2. Save compact copy of the data file, skipping removed leaves. - // { - // let record_len = T::len() as u64; - // - // let off_to_rm = leaf_pos_to_rm - // .iter() - // .map(|pos| { - // let shift = self.pruned_nodes.get_leaf_shift(*pos); - // (pos - 1 - shift.unwrap()) * record_len - // }) - // .collect::>(); - // - // println!("compacting the data file: pos {:?}, offs {:?}", leaf_pos_to_rm, - // off_to_rm); - // - // self.data_file.save_prune( - // tmp_prune_file_data.clone(), - // off_to_rm, - // record_len, - // prune_cb, - // )?; - // } + { + let record_len = T::len() as u64; + + let off_to_rm = leaf_pos_to_rm + .iter() + .map(|pos| { + let shift = self.pruned_nodes.get_leaf_shift(*pos); + (pmmr::n_leaves(pos - shift.unwrap()) - 1) * record_len + }) + .collect::>(); + + self.data_file.save_prune( + tmp_prune_file_data.clone(), + off_to_rm, + record_len, + prune_cb, + )?; + } // 3. Update the prune list and save it in place. { @@ -425,12 +422,11 @@ where self.hash_file = AppendOnlyFile::open(format!("{}/{}", self.data_dir, PMMR_HASH_FILE), 0)?; // 5. Rename the compact copy of the data file and reopen it. - // fs::rename( - // tmp_prune_file_data.clone(), - // format!("{}/{}", self.data_dir, PMMR_DATA_FILE), - // )?; - // self.data_file = AppendOnlyFile::open(format!("{}/{}", self.data_dir, - // PMMR_DATA_FILE), 0)?; + fs::rename( + tmp_prune_file_data.clone(), + format!("{}/{}", self.data_dir, PMMR_DATA_FILE), + )?; + self.data_file = AppendOnlyFile::open(format!("{}/{}", self.data_dir, PMMR_DATA_FILE), 0)?; // 6. Truncate the rm log based on pos removed. // Excluding roots which remain in rm log. diff --git a/store/tests/pmmr.rs b/store/tests/pmmr.rs index 32877d0c..0ebb03bc 100644 --- a/store/tests/pmmr.rs +++ b/store/tests/pmmr.rs @@ -122,6 +122,8 @@ fn pmmr_compact_leaf_sibling() { // Check we can still retrieve the "removed" hash at pos 1 from the hash file. // It should still be available even after pruning and compacting. assert_eq!(backend.get_from_file(1).unwrap(), pos_1_hash); + + teardown(data_dir); } #[test] @@ -408,8 +410,6 @@ fn pmmr_compact_horizon() { // 0010012001001230 // 9 leaves - // data file compaction commented out for now - // assert_eq!(backend.data_size().unwrap(), 9); assert_eq!(backend.data_size().unwrap(), 19); assert_eq!(backend.hash_size().unwrap(), 35); @@ -488,7 +488,7 @@ fn pmmr_compact_horizon() { let backend = store::pmmr::PMMRBackend::::new(data_dir.to_string(), None).unwrap(); - assert_eq!(backend.data_size().unwrap(), 19); + assert_eq!(backend.data_size().unwrap(), 17); assert_eq!(backend.hash_size().unwrap(), 33); // check we can read a hash by pos correctly from recreated backend @@ -522,7 +522,7 @@ fn pmmr_compact_horizon() { // 0010012001001230 - assert_eq!(backend.data_size().unwrap(), 19); + assert_eq!(backend.data_size().unwrap(), 15); assert_eq!(backend.hash_size().unwrap(), 29); // check we can read a hash by pos correctly from recreated backend