The Header MMR (One MMR To Rule Them All) (#1716) (#1747)

* header MMR in use within txhashset itself
works with fast sync
not yet in place for initial header sync

* add the (currently unused) sync_head mmr

* use sync MMR during fast sync
rebuild header MMR after we validate full txhashset after download

* support missing header MMR (rebuild as necessary) for legacy nodes

* rename to HashOnly

* cleanup backend.append()

* simplify vec_backend to match simpler append api

core/tests/pmmr.rs

@@ -438,7 +438,7 @@ fn pmmr_prune() {
 	}
 
 	// First check the initial numbers of elements.
-	assert_eq!(ba.elems.len(), 16);
+	assert_eq!(ba.hashes.len(), 16);
 	assert_eq!(ba.remove_list.len(), 0);
 
 	// pruning a leaf with no parent should do nothing
@@ -447,7 +447,7 @@ fn pmmr_prune() {
 		pmmr.prune(16).unwrap();
 		assert_eq!(orig_root, pmmr.root());
 	}
-	assert_eq!(ba.elems.len(), 16);
+	assert_eq!(ba.hashes.len(), 16);
 	assert_eq!(ba.remove_list.len(), 1);
 
 	// pruning leaves with no shared parent just removes 1 element
@@ -456,7 +456,7 @@ fn pmmr_prune() {
 		pmmr.prune(2).unwrap();
 		assert_eq!(orig_root, pmmr.root());
 	}
-	assert_eq!(ba.elems.len(), 16);
+	assert_eq!(ba.hashes.len(), 16);
 	assert_eq!(ba.remove_list.len(), 2);
 
 	{
@@ -464,7 +464,7 @@ fn pmmr_prune() {
 		pmmr.prune(4).unwrap();
 		assert_eq!(orig_root, pmmr.root());
 	}
-	assert_eq!(ba.elems.len(), 16);
+	assert_eq!(ba.hashes.len(), 16);
 	assert_eq!(ba.remove_list.len(), 3);
 
 	// pruning a non-leaf node has no effect
@@ -473,7 +473,7 @@ fn pmmr_prune() {
 		pmmr.prune(3).unwrap_err();
 		assert_eq!(orig_root, pmmr.root());
 	}
-	assert_eq!(ba.elems.len(), 16);
+	assert_eq!(ba.hashes.len(), 16);
 	assert_eq!(ba.remove_list.len(), 3);
 
 	// TODO - no longer true (leaves only now) - pruning sibling removes subtree
@@ -482,7 +482,7 @@ fn pmmr_prune() {
 		pmmr.prune(5).unwrap();
 		assert_eq!(orig_root, pmmr.root());
 	}
-	assert_eq!(ba.elems.len(), 16);
+	assert_eq!(ba.hashes.len(), 16);
 	assert_eq!(ba.remove_list.len(), 4);
 
 	// TODO - no longer true (leaves only now) - pruning all leaves under level >1
@@ -492,7 +492,7 @@ fn pmmr_prune() {
 		pmmr.prune(1).unwrap();
 		assert_eq!(orig_root, pmmr.root());
 	}
-	assert_eq!(ba.elems.len(), 16);
+	assert_eq!(ba.hashes.len(), 16);
 	assert_eq!(ba.remove_list.len(), 5);
 
 	// pruning everything should only leave us with a single peak
@@ -503,7 +503,7 @@ fn pmmr_prune() {
 		}
 		assert_eq!(orig_root, pmmr.root());
 	}
-	assert_eq!(ba.elems.len(), 16);
+	assert_eq!(ba.hashes.len(), 16);
 	assert_eq!(ba.remove_list.len(), 9);
 }
 

core/tests/vec_backend/mod.rs

@@ -17,7 +17,7 @@ extern crate croaring;
 use croaring::Bitmap;
 
 use core::core::hash::Hash;
-use core::core::pmmr::Backend;
+use core::core::pmmr::{self, Backend};
 use core::core::BlockHeader;
 use core::ser;
 use core::ser::{PMMRable, Readable, Reader, Writeable, Writer};
@@ -59,7 +59,8 @@ where
 	T: PMMRable,
 {
 	/// Backend elements
-	pub elems: Vec<Option<(Hash, Option<T>)>>,
+	pub data: Vec<T>,
+	pub hashes: Vec<Hash>,
 	/// Positions of removed elements
 	pub remove_list: Vec<u64>,
 }
@@ -68,8 +69,9 @@ impl<T> Backend<T> for VecBackend<T>
 where
 	T: PMMRable,
 {
-	fn append(&mut self, _position: u64, data: Vec<(Hash, Option<T>)>) -> Result<(), String> {
-		self.elems.append(&mut map_vec!(data, |d| Some(d.clone())));
+	fn append(&mut self, data: T, hashes: Vec<Hash>) -> Result<(), String> {
+		self.data.push(data);
+		self.hashes.append(&mut hashes.clone());
 		Ok(())
 	}
 
@@ -77,11 +79,7 @@ where
 		if self.remove_list.contains(&position) {
 			None
 		} else {
-			if let Some(ref elem) = self.elems[(position - 1) as usize] {
-				Some(elem.0)
-			} else {
-				None
-			}
+			self.get_from_file(position)
 		}
 	}
 
@@ -89,28 +87,19 @@ where
 		if self.remove_list.contains(&position) {
 			None
 		} else {
-			if let Some(ref elem) = self.elems[(position - 1) as usize] {
-				elem.1.clone()
-			} else {
-				None
-			}
+			self.get_data_from_file(position)
 		}
 	}
 
 	fn get_from_file(&self, position: u64) -> Option<Hash> {
-		if let Some(ref x) = self.elems[(position - 1) as usize] {
-			Some(x.0)
-		} else {
-			None
-		}
+		let hash = &self.hashes[(position - 1) as usize];
+		Some(hash.clone())
 	}
 
 	fn get_data_from_file(&self, position: u64) -> Option<T> {
-		if let Some(ref x) = self.elems[(position - 1) as usize] {
-			x.1.clone()
-		} else {
-			None
-		}
+		let idx = pmmr::n_leaves(position);
+		let data = &self.data[(idx - 1) as usize];
+		Some(data.clone())
 	}
 
 	fn remove(&mut self, position: u64) -> Result<(), String> {
@@ -119,7 +108,9 @@ where
 	}
 
 	fn rewind(&mut self, position: u64, _rewind_rm_pos: &Bitmap) -> Result<(), String> {
-		self.elems = self.elems[0..(position as usize) + 1].to_vec();
+		let idx = pmmr::n_leaves(position);
+		self.data = self.data[0..(idx as usize) + 1].to_vec();
+		self.hashes = self.hashes[0..(position as usize) + 1].to_vec();
 		Ok(())
 	}
 
@@ -141,20 +132,9 @@ where
 	/// Instantiates a new VecBackend<T>
 	pub fn new() -> VecBackend<T> {
 		VecBackend {
-			elems: vec![],
+			data: vec![],
+			hashes: vec![],
 			remove_list: vec![],
 		}
 	}
-
-	// /// Current number of elements in the underlying Vec.
-	// pub fn used_size(&self) -> usize {
-	// 	let mut usz = self.elems.len();
-	// 	for (idx, _) in self.elems.iter().enumerate() {
-	// 		let idx = idx as u64;
-	// 		if self.remove_list.contains(&idx) {
-	// 			usz -= 1;
-	// 		}
-	// 	}
-	// 	usz
-	// }
 }