GRIN/grin-node
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

implement prune_list as a bitmap (#1179) (#1206)

Browse Source 
* implement prune_list as a bitmap
and simplify

* cleanup prune_list, use maximum()

* handle migration of prune_list to new bitmap prun file

* legacy filename consts

* cleanup and docs
This commit is contained in:
Antioch Peverell
2018-06-28 20:53:00 -04:00
committed by GitHub
parent 5ac61b0bc8
commit d0f8d325f2
11 changed files with 556 additions and 503 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/tests/pmmr.rs
-261
View File
@@ -21,7 +21,6 @@ mod vec_backend;
use core::core::hash::Hash;
use core::core::pmmr::{self, PMMR};
use core::core::prune_list::PruneList;
use core::ser::PMMRIndexHashable;
use vec_backend::{TestElem, VecBackend};
@@ -450,266 +449,6 @@ fn pmmr_prune() {
assert_eq!(ba.remove_list.len(), 9);
}
#[test]
fn pmmr_next_pruned_idx() {
let mut pl = PruneList::new();
assert_eq!(pl.pruned_nodes.len(), 0);
assert_eq!(pl.next_pruned_idx(1), Some(0));
assert_eq!(pl.next_pruned_idx(2), Some(0));
assert_eq!(pl.next_pruned_idx(3), Some(0));
pl.add(2);
assert_eq!(pl.pruned_nodes.len(), 1);
assert_eq!(pl.pruned_nodes, [2]);
assert_eq!(pl.next_pruned_idx(1), Some(0));
assert_eq!(pl.next_pruned_idx(2), None);
assert_eq!(pl.next_pruned_idx(3), Some(1));
assert_eq!(pl.next_pruned_idx(4), Some(1));
pl.add(1);
assert_eq!(pl.pruned_nodes.len(), 1);
assert_eq!(pl.pruned_nodes, [3]);
assert_eq!(pl.next_pruned_idx(1), None);
assert_eq!(pl.next_pruned_idx(2), None);
assert_eq!(pl.next_pruned_idx(3), None);
assert_eq!(pl.next_pruned_idx(4), Some(1));
assert_eq!(pl.next_pruned_idx(5), Some(1));
pl.add(3);
assert_eq!(pl.pruned_nodes.len(), 1);
assert_eq!(pl.pruned_nodes, [3]);
assert_eq!(pl.next_pruned_idx(1), None);
assert_eq!(pl.next_pruned_idx(2), None);
assert_eq!(pl.next_pruned_idx(3), None);
assert_eq!(pl.next_pruned_idx(4), Some(1));
assert_eq!(pl.next_pruned_idx(5), Some(1));
}
#[test]
fn pmmr_prune_leaf_shift() {
let mut pl = PruneList::new();
// start with an empty prune list (nothing shifted)
assert_eq!(pl.pruned_nodes.len(), 0);
assert_eq!(pl.get_leaf_shift(1), Some(0));
assert_eq!(pl.get_leaf_shift(2), Some(0));
assert_eq!(pl.get_leaf_shift(4), Some(0));
// now add a single leaf pos to the prune list
// note this does not shift anything (we only start shifting after pruning a
// parent)
pl.add(1);
assert_eq!(pl.pruned_nodes.len(), 1);
assert_eq!(pl.pruned_nodes, [1]);
assert_eq!(pl.get_leaf_shift(1), Some(0));
assert_eq!(pl.get_leaf_shift(2), Some(0));
assert_eq!(pl.get_leaf_shift(3), Some(0));
assert_eq!(pl.get_leaf_shift(4), Some(0));
// now add the sibling leaf pos (pos 1 and pos 2) which will prune the parent
// at pos 3 this in turn will "leaf shift" the leaf at pos 3 by 2
pl.add(2);
assert_eq!(pl.pruned_nodes.len(), 1);
assert_eq!(pl.pruned_nodes, [3]);
assert_eq!(pl.get_leaf_shift(1), None);
assert_eq!(pl.get_leaf_shift(2), None);
assert_eq!(pl.get_leaf_shift(3), Some(2));
assert_eq!(pl.get_leaf_shift(4), Some(2));
assert_eq!(pl.get_leaf_shift(5), Some(2));
// now prune an additional leaf at pos 4
// leaf offset of subsequent pos will be 2
// 00100120
pl.add(4);
assert_eq!(pl.pruned_nodes, [3, 4]);
assert_eq!(pl.get_leaf_shift(1), None);
assert_eq!(pl.get_leaf_shift(2), None);
assert_eq!(pl.get_leaf_shift(3), Some(2));
assert_eq!(pl.get_leaf_shift(4), Some(2));
assert_eq!(pl.get_leaf_shift(5), Some(2));
assert_eq!(pl.get_leaf_shift(6), Some(2));
assert_eq!(pl.get_leaf_shift(7), Some(2));
assert_eq!(pl.get_leaf_shift(8), Some(2));
// now prune the sibling at pos 5
// the two smaller subtrees (pos 3 and pos 6) are rolled up to larger subtree
// (pos 7) the leaf offset is now 4 to cover entire subtree containing first
// 4 leaves 00100120
pl.add(5);
assert_eq!(pl.pruned_nodes, [7]);
assert_eq!(pl.get_leaf_shift(1), None);
assert_eq!(pl.get_leaf_shift(2), None);
assert_eq!(pl.get_leaf_shift(3), None);
assert_eq!(pl.get_leaf_shift(4), None);
assert_eq!(pl.get_leaf_shift(5), None);
assert_eq!(pl.get_leaf_shift(6), None);
assert_eq!(pl.get_leaf_shift(7), Some(4));
assert_eq!(pl.get_leaf_shift(8), Some(4));
assert_eq!(pl.get_leaf_shift(9), Some(4));
// now check we can prune some of these in an arbitrary order
// final result is one leaf (pos 2) and one small subtree (pos 6) pruned
// with leaf offset of 2 to account for the pruned subtree
let mut pl = PruneList::new();
pl.add(2);
pl.add(5);
pl.add(4);
assert_eq!(pl.pruned_nodes, [2, 6]);
assert_eq!(pl.get_leaf_shift(1), Some(0));
assert_eq!(pl.get_leaf_shift(2), Some(0));
assert_eq!(pl.get_leaf_shift(3), Some(0));
assert_eq!(pl.get_leaf_shift(4), None);
assert_eq!(pl.get_leaf_shift(5), None);
assert_eq!(pl.get_leaf_shift(6), Some(2));
assert_eq!(pl.get_leaf_shift(7), Some(2));
assert_eq!(pl.get_leaf_shift(8), Some(2));
assert_eq!(pl.get_leaf_shift(9), Some(2));
pl.add(1);
assert_eq!(pl.pruned_nodes, [7]);
assert_eq!(pl.get_leaf_shift(1), None);
assert_eq!(pl.get_leaf_shift(2), None);
assert_eq!(pl.get_leaf_shift(3), None);
assert_eq!(pl.get_leaf_shift(4), None);
assert_eq!(pl.get_leaf_shift(5), None);
assert_eq!(pl.get_leaf_shift(6), None);
assert_eq!(pl.get_leaf_shift(7), Some(4));
assert_eq!(pl.get_leaf_shift(8), Some(4));
assert_eq!(pl.get_leaf_shift(9), Some(4));
}
#[test]
fn pmmr_prune_shift() {
let mut pl = PruneList::new();
assert!(pl.pruned_nodes.is_empty());
assert_eq!(pl.get_shift(1), Some(0));
assert_eq!(pl.get_shift(2), Some(0));
assert_eq!(pl.get_shift(3), Some(0));
// prune a single leaf node
// pruning only a leaf node does not shift any subsequent pos
// we will only start shifting when a parent can be pruned
pl.add(1);
assert_eq!(pl.pruned_nodes, [1]);
assert_eq!(pl.get_shift(1), Some(0));
assert_eq!(pl.get_shift(2), Some(0));
assert_eq!(pl.get_shift(3), Some(0));
pl.add(2);
assert_eq!(pl.pruned_nodes, [3]);
assert_eq!(pl.get_shift(1), None);
assert_eq!(pl.get_shift(2), None);
// pos 3 is in the prune list, so removed but not compacted, but still shifted
assert_eq!(pl.get_shift(3), Some(2));
assert_eq!(pl.get_shift(4), Some(2));
assert_eq!(pl.get_shift(5), Some(2));
assert_eq!(pl.get_shift(6), Some(2));
// pos 3 is not a leaf and is already in prune list
// prune it and check we are still consistent
pl.add(3);
assert_eq!(pl.pruned_nodes, [3]);
assert_eq!(pl.get_shift(1), None);
assert_eq!(pl.get_shift(2), None);
// pos 3 is in the prune list, so removed but not compacted, but still shifted
assert_eq!(pl.get_shift(3), Some(2));
assert_eq!(pl.get_shift(4), Some(2));
assert_eq!(pl.get_shift(5), Some(2));
assert_eq!(pl.get_shift(6), Some(2));
pl.add(4);
assert_eq!(pl.pruned_nodes, [3, 4]);
assert_eq!(pl.get_shift(1), None);
assert_eq!(pl.get_shift(2), None);
// pos 3 is in the prune list, so removed but not compacted, but still shifted
assert_eq!(pl.get_shift(3), Some(2));
// pos 4 is also in the prune list and also shifted by same amount
assert_eq!(pl.get_shift(4), Some(2));
// subsequent nodes also shifted consistently
assert_eq!(pl.get_shift(5), Some(2));
assert_eq!(pl.get_shift(6), Some(2));
pl.add(5);
assert_eq!(pl.pruned_nodes, [7]);
assert_eq!(pl.get_shift(1), None);
assert_eq!(pl.get_shift(2), None);
assert_eq!(pl.get_shift(3), None);
assert_eq!(pl.get_shift(4), None);
assert_eq!(pl.get_shift(5), None);
assert_eq!(pl.get_shift(6), None);
// everything prior to pos 7 is compacted away
// pos 7 is shifted by 6 to account for this
assert_eq!(pl.get_shift(7), Some(6));
assert_eq!(pl.get_shift(8), Some(6));
assert_eq!(pl.get_shift(9), Some(6));
// prune a bunch more
for x in 6..1000 {
pl.add(x);
}
// and check we shift by a large number (hopefully the correct number...)
assert_eq!(pl.get_shift(1010), Some(996));
let mut pl = PruneList::new();
pl.add(2);
pl.add(5);
pl.add(4);
assert_eq!(pl.pruned_nodes, [2, 6]);
assert_eq!(pl.get_shift(1), Some(0));
assert_eq!(pl.get_shift(2), Some(0));
assert_eq!(pl.get_shift(3), Some(0));
assert_eq!(pl.get_shift(4), None);
assert_eq!(pl.get_shift(5), None);
assert_eq!(pl.get_shift(6), Some(2));
assert_eq!(pl.get_shift(7), Some(2));
assert_eq!(pl.get_shift(8), Some(2));
assert_eq!(pl.get_shift(9), Some(2));
// TODO - put some of these tests back in place for completeness
//
// let mut pl = PruneList::new();
// pl.add(4);
// assert_eq!(pl.pruned_nodes.len(), 1);
// assert_eq!(pl.pruned_nodes, [4]);
// assert_eq!(pl.get_shift(1), Some(0));
// assert_eq!(pl.get_shift(2), Some(0));
// assert_eq!(pl.get_shift(3), Some(0));
// assert_eq!(pl.get_shift(4), None);
// assert_eq!(pl.get_shift(5), Some(1));
// assert_eq!(pl.get_shift(6), Some(1));
//
//
// pl.add(5);
// assert_eq!(pl.pruned_nodes.len(), 1);
// assert_eq!(pl.pruned_nodes[0], 6);
// assert_eq!(pl.get_shift(8), Some(3));
// assert_eq!(pl.get_shift(2), Some(0));
// assert_eq!(pl.get_shift(5), None);
//
// pl.add(2);
// assert_eq!(pl.pruned_nodes.len(), 2);
// assert_eq!(pl.pruned_nodes[0], 2);
// assert_eq!(pl.get_shift(8), Some(4));
// assert_eq!(pl.get_shift(1), Some(0));
//
// pl.add(8);
// pl.add(11);
// assert_eq!(pl.pruned_nodes.len(), 4);
//
// pl.add(1);
// assert_eq!(pl.pruned_nodes.len(), 3);
// assert_eq!(pl.pruned_nodes[0], 7);
// assert_eq!(pl.get_shift(12), Some(9));
//
// pl.add(12);
// assert_eq!(pl.pruned_nodes.len(), 3);
// assert_eq!(pl.get_shift(12), None);
// assert_eq!(pl.get_shift(9), Some(8));
// assert_eq!(pl.get_shift(17), Some(11));
}
#[test]
fn check_all_ones() {
for i in 0..1000000 {