* implement prune_list as a bitmap and simplify * cleanup prune_list, use maximum() * handle migration of prune_list to new bitmap prun file * legacy filename consts * cleanup and docs
This commit is contained in:
Antioch Peverell
@@ -20,7 +20,6 @@ pub mod hash;
|
||||
pub mod id;
|
||||
pub mod merkle_proof;
|
||||
pub mod pmmr;
|
||||
pub mod prune_list;
|
||||
pub mod target;
|
||||
pub mod transaction;
|
||||
|
||||
|
||||
@@ -722,6 +722,18 @@ pub fn path(pos: u64, last_pos: u64) -> Vec<u64> {
|
||||
path
|
||||
}
|
||||
|
||||
// TODO - this is simpler, test it is actually correct?
|
||||
// pub fn path(pos: u64, last_pos: u64) -> Vec<u64> {
|
||||
// let mut path = vec![];
|
||||
// let mut current = pos;
|
||||
// while current <= last_pos {
|
||||
// path.push(current);
|
||||
// let (parent, _) = family(current);
|
||||
// current = parent;
|
||||
// }
|
||||
// path
|
||||
// }
|
||||
|
||||
/// For a given starting position calculate the parent and sibling positions
|
||||
/// for the branch/path from that position to the peak of the tree.
|
||||
/// We will use the sibling positions to generate the "path" of a Merkle proof.
|
||||
|
||||
@@ -1,173 +0,0 @@
|
||||
// Copyright 2018 The Grin Developers
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The Grin "Prune List" implementation.
|
||||
//! Currently implemented as a vec of u64 positions.
|
||||
//! *Soon* to be implemented as a compact bitmap.
|
||||
//!
|
||||
//! Maintains a set of pruned root node positions that define the pruned
|
||||
//! and compacted "gaps" in the MMR data and hash files.
|
||||
//! The root itself is maintained in the hash file, but all positions beneath
|
||||
//! the root are compacted away. All positions to the right of a pruned node
|
||||
//! must be shifted the appropriate amount when reading from the hash and data
|
||||
//! files.
|
||||
|
||||
use core::pmmr::{bintree_postorder_height, family};
|
||||
|
||||
/// Maintains a list of previously pruned nodes in PMMR, compacting the list as
|
||||
/// parents get pruned and allowing checking whether a leaf is pruned. Given
|
||||
/// a node's position, computes how much it should get shifted given the
|
||||
/// subtrees that have been pruned before.
|
||||
///
|
||||
/// The PruneList is useful when implementing compact backends for a PMMR (for
|
||||
/// example a single large byte array or a file). As nodes get pruned and
|
||||
/// removed from the backend to free space, the backend will get more compact
|
||||
/// but positions of a node within the PMMR will not match positions in the
|
||||
/// backend storage anymore. The PruneList accounts for that mismatch and does
|
||||
/// the position translation.
|
||||
#[derive(Default)]
|
||||
pub struct PruneList {
|
||||
/// Vector of pruned nodes positions
|
||||
pub pruned_nodes: Vec<u64>,
|
||||
}
|
||||
|
||||
impl PruneList {
|
||||
/// Instantiate a new empty prune list
|
||||
pub fn new() -> PruneList {
|
||||
PruneList {
|
||||
pruned_nodes: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
/// Computes by how many positions a node at pos should be shifted given the
|
||||
/// number of nodes that have already been pruned before it. Returns None if
|
||||
/// the position has already been pruned.
|
||||
pub fn get_shift(&self, pos: u64) -> Option<u64> {
|
||||
// get the position where the node at pos would fit in the pruned list, if
|
||||
// it's already pruned, nothing to skip
|
||||
|
||||
let pruned_idx = self.next_pruned_idx(pos);
|
||||
let next_idx = self.pruned_nodes.binary_search(&pos).map(|x| x + 1).ok();
|
||||
match pruned_idx.or(next_idx) {
|
||||
None => None,
|
||||
Some(idx) => {
|
||||
// skip by the number of elements pruned in the preceding subtrees,
|
||||
// which is the sum of the size of each subtree
|
||||
Some(
|
||||
self.pruned_nodes[0..(idx as usize)]
|
||||
.iter()
|
||||
.map(|n| {
|
||||
let height = bintree_postorder_height(*n);
|
||||
// height 0, 1 node, offset 0 = 0 + 0
|
||||
// height 1, 3 nodes, offset 2 = 1 + 1
|
||||
// height 2, 7 nodes, offset 6 = 3 + 3
|
||||
// height 3, 15 nodes, offset 14 = 7 + 7
|
||||
2 * ((1 << height) - 1)
|
||||
})
|
||||
.sum(),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// As above, but only returning the number of leaf nodes to skip for a
|
||||
/// given leaf. Helpful if, for instance, data for each leaf is being stored
|
||||
/// separately in a continuous flat-file. Returns None if the position has
|
||||
/// already been pruned.
|
||||
pub fn get_leaf_shift(&self, pos: u64) -> Option<u64> {
|
||||
// get the position where the node at pos would fit in the pruned list, if
|
||||
// it's already pruned, nothing to skip
|
||||
|
||||
let pruned_idx = self.next_pruned_idx(pos);
|
||||
let next_idx = self.pruned_nodes.binary_search(&pos).map(|x| x + 1).ok();
|
||||
|
||||
let idx = pruned_idx.or(next_idx)?;
|
||||
Some(
|
||||
// skip by the number of leaf nodes pruned in the preceeding subtrees
|
||||
// which just 2^height
|
||||
// except in the case of height==0
|
||||
// (where we want to treat the pruned tree as 0 leaves)
|
||||
self.pruned_nodes[0..(idx as usize)]
|
||||
.iter()
|
||||
.map(|n| {
|
||||
let height = bintree_postorder_height(*n);
|
||||
if height == 0 {
|
||||
0
|
||||
} else {
|
||||
(1 << height)
|
||||
}
|
||||
})
|
||||
.sum(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Push the node at the provided position in the prune list. Compacts the
|
||||
/// list if pruning the additional node means a parent can get pruned as
|
||||
/// well.
|
||||
pub fn add(&mut self, pos: u64) {
|
||||
let mut current = pos;
|
||||
loop {
|
||||
let (parent, sibling) = family(current);
|
||||
|
||||
match self.pruned_nodes.binary_search(&sibling) {
|
||||
Ok(idx) => {
|
||||
self.pruned_nodes.remove(idx);
|
||||
current = parent;
|
||||
}
|
||||
Err(_) => {
|
||||
if let Some(idx) = self.next_pruned_idx(current) {
|
||||
self.pruned_nodes.insert(idx, current);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Checks if the specified position has been pruned,
|
||||
/// either directly (pos contained in the prune list itself)
|
||||
/// or indirectly (pos is beneath a pruned root).
|
||||
pub fn is_pruned(&self, pos: u64) -> bool {
|
||||
self.next_pruned_idx(pos).is_none()
|
||||
}
|
||||
|
||||
/// Gets the index a new pruned node should take in the prune list.
|
||||
/// If the node has already been pruned, either directly or through one of
|
||||
/// its parents contained in the prune list, returns None.
|
||||
pub fn next_pruned_idx(&self, pos: u64) -> Option<usize> {
|
||||
match self.pruned_nodes.binary_search(&pos) {
|
||||
Ok(_) => None,
|
||||
Err(idx) => {
|
||||
if self.pruned_nodes.len() > idx {
|
||||
// the node at pos can't be a child of lower position nodes by MMR
|
||||
// construction but can be a child of the next node, going up parents
|
||||
// from pos to make sure it's not the case
|
||||
let next_peak_pos = self.pruned_nodes[idx];
|
||||
let mut cursor = pos;
|
||||
loop {
|
||||
let (parent, _) = family(cursor);
|
||||
if next_peak_pos == parent {
|
||||
return None;
|
||||
}
|
||||
if next_peak_pos < parent {
|
||||
break;
|
||||
}
|
||||
cursor = parent;
|
||||
}
|
||||
}
|
||||
Some(idx)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -21,7 +21,6 @@ mod vec_backend;
|
||||
|
||||
use core::core::hash::Hash;
|
||||
use core::core::pmmr::{self, PMMR};
|
||||
use core::core::prune_list::PruneList;
|
||||
use core::ser::PMMRIndexHashable;
|
||||
use vec_backend::{TestElem, VecBackend};
|
||||
|
||||
@@ -450,266 +449,6 @@ fn pmmr_prune() {
|
||||
assert_eq!(ba.remove_list.len(), 9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pmmr_next_pruned_idx() {
|
||||
let mut pl = PruneList::new();
|
||||
|
||||
assert_eq!(pl.pruned_nodes.len(), 0);
|
||||
assert_eq!(pl.next_pruned_idx(1), Some(0));
|
||||
assert_eq!(pl.next_pruned_idx(2), Some(0));
|
||||
assert_eq!(pl.next_pruned_idx(3), Some(0));
|
||||
|
||||
pl.add(2);
|
||||
assert_eq!(pl.pruned_nodes.len(), 1);
|
||||
assert_eq!(pl.pruned_nodes, [2]);
|
||||
assert_eq!(pl.next_pruned_idx(1), Some(0));
|
||||
assert_eq!(pl.next_pruned_idx(2), None);
|
||||
assert_eq!(pl.next_pruned_idx(3), Some(1));
|
||||
assert_eq!(pl.next_pruned_idx(4), Some(1));
|
||||
|
||||
pl.add(1);
|
||||
assert_eq!(pl.pruned_nodes.len(), 1);
|
||||
assert_eq!(pl.pruned_nodes, [3]);
|
||||
assert_eq!(pl.next_pruned_idx(1), None);
|
||||
assert_eq!(pl.next_pruned_idx(2), None);
|
||||
assert_eq!(pl.next_pruned_idx(3), None);
|
||||
assert_eq!(pl.next_pruned_idx(4), Some(1));
|
||||
assert_eq!(pl.next_pruned_idx(5), Some(1));
|
||||
|
||||
pl.add(3);
|
||||
assert_eq!(pl.pruned_nodes.len(), 1);
|
||||
assert_eq!(pl.pruned_nodes, [3]);
|
||||
assert_eq!(pl.next_pruned_idx(1), None);
|
||||
assert_eq!(pl.next_pruned_idx(2), None);
|
||||
assert_eq!(pl.next_pruned_idx(3), None);
|
||||
assert_eq!(pl.next_pruned_idx(4), Some(1));
|
||||
assert_eq!(pl.next_pruned_idx(5), Some(1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pmmr_prune_leaf_shift() {
|
||||
let mut pl = PruneList::new();
|
||||
|
||||
// start with an empty prune list (nothing shifted)
|
||||
assert_eq!(pl.pruned_nodes.len(), 0);
|
||||
assert_eq!(pl.get_leaf_shift(1), Some(0));
|
||||
assert_eq!(pl.get_leaf_shift(2), Some(0));
|
||||
assert_eq!(pl.get_leaf_shift(4), Some(0));
|
||||
|
||||
// now add a single leaf pos to the prune list
|
||||
// note this does not shift anything (we only start shifting after pruning a
|
||||
// parent)
|
||||
pl.add(1);
|
||||
assert_eq!(pl.pruned_nodes.len(), 1);
|
||||
assert_eq!(pl.pruned_nodes, [1]);
|
||||
assert_eq!(pl.get_leaf_shift(1), Some(0));
|
||||
assert_eq!(pl.get_leaf_shift(2), Some(0));
|
||||
assert_eq!(pl.get_leaf_shift(3), Some(0));
|
||||
assert_eq!(pl.get_leaf_shift(4), Some(0));
|
||||
|
||||
// now add the sibling leaf pos (pos 1 and pos 2) which will prune the parent
|
||||
// at pos 3 this in turn will "leaf shift" the leaf at pos 3 by 2
|
||||
pl.add(2);
|
||||
assert_eq!(pl.pruned_nodes.len(), 1);
|
||||
assert_eq!(pl.pruned_nodes, [3]);
|
||||
assert_eq!(pl.get_leaf_shift(1), None);
|
||||
assert_eq!(pl.get_leaf_shift(2), None);
|
||||
assert_eq!(pl.get_leaf_shift(3), Some(2));
|
||||
assert_eq!(pl.get_leaf_shift(4), Some(2));
|
||||
assert_eq!(pl.get_leaf_shift(5), Some(2));
|
||||
|
||||
// now prune an additional leaf at pos 4
|
||||
// leaf offset of subsequent pos will be 2
|
||||
// 00100120
|
||||
pl.add(4);
|
||||
assert_eq!(pl.pruned_nodes, [3, 4]);
|
||||
assert_eq!(pl.get_leaf_shift(1), None);
|
||||
assert_eq!(pl.get_leaf_shift(2), None);
|
||||
assert_eq!(pl.get_leaf_shift(3), Some(2));
|
||||
assert_eq!(pl.get_leaf_shift(4), Some(2));
|
||||
assert_eq!(pl.get_leaf_shift(5), Some(2));
|
||||
assert_eq!(pl.get_leaf_shift(6), Some(2));
|
||||
assert_eq!(pl.get_leaf_shift(7), Some(2));
|
||||
assert_eq!(pl.get_leaf_shift(8), Some(2));
|
||||
|
||||
// now prune the sibling at pos 5
|
||||
// the two smaller subtrees (pos 3 and pos 6) are rolled up to larger subtree
|
||||
// (pos 7) the leaf offset is now 4 to cover entire subtree containing first
|
||||
// 4 leaves 00100120
|
||||
pl.add(5);
|
||||
assert_eq!(pl.pruned_nodes, [7]);
|
||||
assert_eq!(pl.get_leaf_shift(1), None);
|
||||
assert_eq!(pl.get_leaf_shift(2), None);
|
||||
assert_eq!(pl.get_leaf_shift(3), None);
|
||||
assert_eq!(pl.get_leaf_shift(4), None);
|
||||
assert_eq!(pl.get_leaf_shift(5), None);
|
||||
assert_eq!(pl.get_leaf_shift(6), None);
|
||||
assert_eq!(pl.get_leaf_shift(7), Some(4));
|
||||
assert_eq!(pl.get_leaf_shift(8), Some(4));
|
||||
assert_eq!(pl.get_leaf_shift(9), Some(4));
|
||||
|
||||
// now check we can prune some of these in an arbitrary order
|
||||
// final result is one leaf (pos 2) and one small subtree (pos 6) pruned
|
||||
// with leaf offset of 2 to account for the pruned subtree
|
||||
let mut pl = PruneList::new();
|
||||
pl.add(2);
|
||||
pl.add(5);
|
||||
pl.add(4);
|
||||
assert_eq!(pl.pruned_nodes, [2, 6]);
|
||||
assert_eq!(pl.get_leaf_shift(1), Some(0));
|
||||
assert_eq!(pl.get_leaf_shift(2), Some(0));
|
||||
assert_eq!(pl.get_leaf_shift(3), Some(0));
|
||||
assert_eq!(pl.get_leaf_shift(4), None);
|
||||
assert_eq!(pl.get_leaf_shift(5), None);
|
||||
assert_eq!(pl.get_leaf_shift(6), Some(2));
|
||||
assert_eq!(pl.get_leaf_shift(7), Some(2));
|
||||
assert_eq!(pl.get_leaf_shift(8), Some(2));
|
||||
assert_eq!(pl.get_leaf_shift(9), Some(2));
|
||||
|
||||
pl.add(1);
|
||||
assert_eq!(pl.pruned_nodes, [7]);
|
||||
assert_eq!(pl.get_leaf_shift(1), None);
|
||||
assert_eq!(pl.get_leaf_shift(2), None);
|
||||
assert_eq!(pl.get_leaf_shift(3), None);
|
||||
assert_eq!(pl.get_leaf_shift(4), None);
|
||||
assert_eq!(pl.get_leaf_shift(5), None);
|
||||
assert_eq!(pl.get_leaf_shift(6), None);
|
||||
assert_eq!(pl.get_leaf_shift(7), Some(4));
|
||||
assert_eq!(pl.get_leaf_shift(8), Some(4));
|
||||
assert_eq!(pl.get_leaf_shift(9), Some(4));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pmmr_prune_shift() {
|
||||
let mut pl = PruneList::new();
|
||||
assert!(pl.pruned_nodes.is_empty());
|
||||
assert_eq!(pl.get_shift(1), Some(0));
|
||||
assert_eq!(pl.get_shift(2), Some(0));
|
||||
assert_eq!(pl.get_shift(3), Some(0));
|
||||
|
||||
// prune a single leaf node
|
||||
// pruning only a leaf node does not shift any subsequent pos
|
||||
// we will only start shifting when a parent can be pruned
|
||||
pl.add(1);
|
||||
assert_eq!(pl.pruned_nodes, [1]);
|
||||
assert_eq!(pl.get_shift(1), Some(0));
|
||||
assert_eq!(pl.get_shift(2), Some(0));
|
||||
assert_eq!(pl.get_shift(3), Some(0));
|
||||
|
||||
pl.add(2);
|
||||
assert_eq!(pl.pruned_nodes, [3]);
|
||||
assert_eq!(pl.get_shift(1), None);
|
||||
assert_eq!(pl.get_shift(2), None);
|
||||
// pos 3 is in the prune list, so removed but not compacted, but still shifted
|
||||
assert_eq!(pl.get_shift(3), Some(2));
|
||||
assert_eq!(pl.get_shift(4), Some(2));
|
||||
assert_eq!(pl.get_shift(5), Some(2));
|
||||
assert_eq!(pl.get_shift(6), Some(2));
|
||||
|
||||
// pos 3 is not a leaf and is already in prune list
|
||||
// prune it and check we are still consistent
|
||||
pl.add(3);
|
||||
assert_eq!(pl.pruned_nodes, [3]);
|
||||
assert_eq!(pl.get_shift(1), None);
|
||||
assert_eq!(pl.get_shift(2), None);
|
||||
// pos 3 is in the prune list, so removed but not compacted, but still shifted
|
||||
assert_eq!(pl.get_shift(3), Some(2));
|
||||
assert_eq!(pl.get_shift(4), Some(2));
|
||||
assert_eq!(pl.get_shift(5), Some(2));
|
||||
assert_eq!(pl.get_shift(6), Some(2));
|
||||
|
||||
pl.add(4);
|
||||
assert_eq!(pl.pruned_nodes, [3, 4]);
|
||||
assert_eq!(pl.get_shift(1), None);
|
||||
assert_eq!(pl.get_shift(2), None);
|
||||
// pos 3 is in the prune list, so removed but not compacted, but still shifted
|
||||
assert_eq!(pl.get_shift(3), Some(2));
|
||||
// pos 4 is also in the prune list and also shifted by same amount
|
||||
assert_eq!(pl.get_shift(4), Some(2));
|
||||
// subsequent nodes also shifted consistently
|
||||
assert_eq!(pl.get_shift(5), Some(2));
|
||||
assert_eq!(pl.get_shift(6), Some(2));
|
||||
|
||||
pl.add(5);
|
||||
assert_eq!(pl.pruned_nodes, [7]);
|
||||
assert_eq!(pl.get_shift(1), None);
|
||||
assert_eq!(pl.get_shift(2), None);
|
||||
assert_eq!(pl.get_shift(3), None);
|
||||
assert_eq!(pl.get_shift(4), None);
|
||||
assert_eq!(pl.get_shift(5), None);
|
||||
assert_eq!(pl.get_shift(6), None);
|
||||
// everything prior to pos 7 is compacted away
|
||||
// pos 7 is shifted by 6 to account for this
|
||||
assert_eq!(pl.get_shift(7), Some(6));
|
||||
assert_eq!(pl.get_shift(8), Some(6));
|
||||
assert_eq!(pl.get_shift(9), Some(6));
|
||||
|
||||
// prune a bunch more
|
||||
for x in 6..1000 {
|
||||
pl.add(x);
|
||||
}
|
||||
// and check we shift by a large number (hopefully the correct number...)
|
||||
assert_eq!(pl.get_shift(1010), Some(996));
|
||||
|
||||
let mut pl = PruneList::new();
|
||||
pl.add(2);
|
||||
pl.add(5);
|
||||
pl.add(4);
|
||||
assert_eq!(pl.pruned_nodes, [2, 6]);
|
||||
assert_eq!(pl.get_shift(1), Some(0));
|
||||
assert_eq!(pl.get_shift(2), Some(0));
|
||||
assert_eq!(pl.get_shift(3), Some(0));
|
||||
assert_eq!(pl.get_shift(4), None);
|
||||
assert_eq!(pl.get_shift(5), None);
|
||||
assert_eq!(pl.get_shift(6), Some(2));
|
||||
assert_eq!(pl.get_shift(7), Some(2));
|
||||
assert_eq!(pl.get_shift(8), Some(2));
|
||||
assert_eq!(pl.get_shift(9), Some(2));
|
||||
|
||||
// TODO - put some of these tests back in place for completeness
|
||||
|
||||
//
|
||||
// let mut pl = PruneList::new();
|
||||
// pl.add(4);
|
||||
// assert_eq!(pl.pruned_nodes.len(), 1);
|
||||
// assert_eq!(pl.pruned_nodes, [4]);
|
||||
// assert_eq!(pl.get_shift(1), Some(0));
|
||||
// assert_eq!(pl.get_shift(2), Some(0));
|
||||
// assert_eq!(pl.get_shift(3), Some(0));
|
||||
// assert_eq!(pl.get_shift(4), None);
|
||||
// assert_eq!(pl.get_shift(5), Some(1));
|
||||
// assert_eq!(pl.get_shift(6), Some(1));
|
||||
//
|
||||
//
|
||||
// pl.add(5);
|
||||
// assert_eq!(pl.pruned_nodes.len(), 1);
|
||||
// assert_eq!(pl.pruned_nodes[0], 6);
|
||||
// assert_eq!(pl.get_shift(8), Some(3));
|
||||
// assert_eq!(pl.get_shift(2), Some(0));
|
||||
// assert_eq!(pl.get_shift(5), None);
|
||||
//
|
||||
// pl.add(2);
|
||||
// assert_eq!(pl.pruned_nodes.len(), 2);
|
||||
// assert_eq!(pl.pruned_nodes[0], 2);
|
||||
// assert_eq!(pl.get_shift(8), Some(4));
|
||||
// assert_eq!(pl.get_shift(1), Some(0));
|
||||
//
|
||||
// pl.add(8);
|
||||
// pl.add(11);
|
||||
// assert_eq!(pl.pruned_nodes.len(), 4);
|
||||
//
|
||||
// pl.add(1);
|
||||
// assert_eq!(pl.pruned_nodes.len(), 3);
|
||||
// assert_eq!(pl.pruned_nodes[0], 7);
|
||||
// assert_eq!(pl.get_shift(12), Some(9));
|
||||
//
|
||||
// pl.add(12);
|
||||
// assert_eq!(pl.pruned_nodes.len(), 3);
|
||||
// assert_eq!(pl.get_shift(12), None);
|
||||
// assert_eq!(pl.get_shift(9), Some(8));
|
||||
// assert_eq!(pl.get_shift(17), Some(11));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn check_all_ones() {
|
||||
for i in 0..1000000 {
|
||||
|
||||
Reference in New Issue
Block a user