GRIN/grin-node
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PMMR data compaction (#784)

Browse Source 
* fix and re-introduce data file compaction

* rustfmt

* remove commented line

* test cleanup

* remove println

* fix to core test
This commit is contained in:
Yeastplume
2018-03-15 17:12:30 +00:00
committed by GitHub
parent 00bfc4ec38
commit 244fc8d32e
3 changed files with 68 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/core/pmmr.rs
+14 -2
View File
@@ -63,14 +63,18 @@ where
/// occurred (see remove).
fn rewind(&mut self, position: u64, index: u32) -> Result<(), String>;
/// Get a Hash/Element by insertion position. If include_data is true, will
/// Get a Hash by insertion position. If include_data is true, will
/// also return the associated data element
fn get(&self, position: u64, include_data: bool) -> Option<(Hash, Option<T>)>;
/// Get a Hash/Element by original insertion position (ignoring the remove
/// Get a Hash by original insertion position (ignoring the remove
/// list).
fn get_from_file(&self, position: u64) -> Option<Hash>;
/// Get a Data Element by original insertion position (ignoring the remove
/// list).
fn get_data_from_file(&self, position: u64) -> Option<T>;
/// Remove HashSums by insertion position. An index is also provided so the
/// underlying backend can implement some rollback of positions up to a
/// given index (practically the index is the height of a block that
@@ -1027,6 +1031,14 @@ mod test {
}
}
fn get_data_from_file(&self, position: u64) -> Option<T> {
if let Some(ref x) = self.elems[(position - 1) as usize] {
x.1.clone()
} else {
None
}
}
fn remove(&mut self, positions: Vec<u64>, _index: u32) -> Result<(), String> {
for n in positions {
self.remove_list.push(n)
store/src/pmmr.rs
+50 -54
View File
@@ -137,6 +137,30 @@ where
}
}
fn get_data_from_file(&self, position: u64) -> Option<T> {
let shift = self.pruned_nodes.get_leaf_shift(position);
if let None = shift {
return None;
}
let pos = pmmr::n_leaves(position) - 1;
// Must be on disk, doing a read at the correct position
let record_len = T::len();
let file_offset = ((pos - shift.unwrap()) as usize) * record_len;
let data = self.data_file.read(file_offset, record_len);
match ser::deserialize(&mut &data[..]) {
Ok(h) => Some(h),
Err(e) => {
error!(
LOGGER,
"Corrupted storage, could not read an entry from data store: {:?}", e
);
return None;
}
}
}
/// Get a Hash by insertion position
fn get(&self, position: u64, include_data: bool) -> Option<(Hash, Option<T>)> {
// Check if this position has been pruned in the remove log...
@@ -144,12 +168,6 @@ where
return None;
}
// ... or in the prune list
// let prune_shift = match self.pruned_nodes.get_leaf_shift(position) {
// Some(shift) => shift,
// None => return None,
// };
let hash_val = self.get_from_file(position);
if !include_data {
return hash_val.map(|hash| (hash, None));
@@ -160,24 +178,7 @@ where
return hash_val.map(|hash| (hash, None));
}
// Optionally read flatfile storage to get data element
// let flatfile_pos = pmmr::n_leaves(position) - 1 - prune_shift;
let flatfile_pos = pmmr::n_leaves(position) - 1;
let record_len = T::len();
let file_offset = flatfile_pos as usize * T::len();
let data = self.data_file.read(file_offset, record_len);
let data = match ser::deserialize(&mut &data[..]) {
Ok(elem) => Some(elem),
Err(e) => {
error!(
LOGGER,
"Corrupted storage, could not read an entry from backend flatfile store: {:?}",
e
);
None
}
};
let data = self.get_data_from_file(position);
hash_val.map(|x| (x, data))
}
@@ -351,8 +352,7 @@ where
// Paths for tmp hash and data files.
let tmp_prune_file_hash = format!("{}/{}.hashprune", self.data_dir, PMMR_HASH_FILE);
// let tmp_prune_file_data = format!("{}/{}.dataprune", self.data_dir,
// PMMR_DATA_FILE);
let tmp_prune_file_data = format!("{}/{}.dataprune", self.data_dir, PMMR_DATA_FILE);
// Pos we want to get rid of.
// Filtered by cutoff index.
@@ -360,7 +360,7 @@ where
// Filtered to exclude the subtree "roots".
let pos_to_rm = removed_excl_roots(rm_pre_cutoff.clone());
// Filtered for leaves only.
// let leaf_pos_to_rm = removed_leaves(pos_to_rm.clone());
let leaf_pos_to_rm = removed_leaves(pos_to_rm.clone());
// 1. Save compact copy of the hash file, skipping removed data.
{
@@ -383,27 +383,24 @@ where
}
// 2. Save compact copy of the data file, skipping removed leaves.
// {
// let record_len = T::len() as u64;
//
// let off_to_rm = leaf_pos_to_rm
// .iter()
// .map(|pos| {
// let shift = self.pruned_nodes.get_leaf_shift(*pos);
// (pos - 1 - shift.unwrap()) * record_len
// })
// .collect::<Vec<_>>();
//
// println!("compacting the data file: pos {:?}, offs {:?}", leaf_pos_to_rm,
// off_to_rm);
//
// self.data_file.save_prune(
// tmp_prune_file_data.clone(),
// off_to_rm,
// record_len,
// prune_cb,
// )?;
// }
{
let record_len = T::len() as u64;
let off_to_rm = leaf_pos_to_rm
.iter()
.map(|pos| {
let shift = self.pruned_nodes.get_leaf_shift(*pos);
(pmmr::n_leaves(pos - shift.unwrap()) - 1) * record_len
})
.collect::<Vec<_>>();
self.data_file.save_prune(
tmp_prune_file_data.clone(),
off_to_rm,
record_len,
prune_cb,
)?;
}
// 3. Update the prune list and save it in place.
{
@@ -425,12 +422,11 @@ where
self.hash_file = AppendOnlyFile::open(format!("{}/{}", self.data_dir, PMMR_HASH_FILE), 0)?;
// 5. Rename the compact copy of the data file and reopen it.
// fs::rename(
// tmp_prune_file_data.clone(),
// format!("{}/{}", self.data_dir, PMMR_DATA_FILE),
// )?;
// self.data_file = AppendOnlyFile::open(format!("{}/{}", self.data_dir,
// PMMR_DATA_FILE), 0)?;
fs::rename(
tmp_prune_file_data.clone(),
format!("{}/{}", self.data_dir, PMMR_DATA_FILE),
)?;
self.data_file = AppendOnlyFile::open(format!("{}/{}", self.data_dir, PMMR_DATA_FILE), 0)?;
// 6. Truncate the rm log based on pos removed.
// Excluding roots which remain in rm log.
store/tests/pmmr.rs
+4 -4
View File
@@ -122,6 +122,8 @@ fn pmmr_compact_leaf_sibling() {
// Check we can still retrieve the "removed" hash at pos 1 from the hash file.
// It should still be available even after pruning and compacting.
assert_eq!(backend.get_from_file(1).unwrap(), pos_1_hash);
teardown(data_dir);
}
#[test]
@@ -408,8 +410,6 @@ fn pmmr_compact_horizon() {
// 0010012001001230
// 9 leaves
// data file compaction commented out for now
// assert_eq!(backend.data_size().unwrap(), 9);
assert_eq!(backend.data_size().unwrap(), 19);
assert_eq!(backend.hash_size().unwrap(), 35);
@@ -488,7 +488,7 @@ fn pmmr_compact_horizon() {
let backend =
store::pmmr::PMMRBackend::<TestElem>::new(data_dir.to_string(), None).unwrap();
assert_eq!(backend.data_size().unwrap(), 19);
assert_eq!(backend.data_size().unwrap(), 17);
assert_eq!(backend.hash_size().unwrap(), 33);
// check we can read a hash by pos correctly from recreated backend
@@ -522,7 +522,7 @@ fn pmmr_compact_horizon() {
// 0010012001001230
assert_eq!(backend.data_size().unwrap(), 19);
assert_eq!(backend.data_size().unwrap(), 15);
assert_eq!(backend.hash_size().unwrap(), 29);
// check we can read a hash by pos correctly from recreated backend