The Header MMR (One MMR To Rule Them All) (#1716) (#1747)

* header MMR in use within txhashset itself
works with fast sync
not yet in place for initial header sync

* add the (currently unused) sync_head mmr

* use sync MMR during fast sync
rebuild header MMR after we validate full txhashset after download

* support missing header MMR (rebuild as necessary) for legacy nodes

* rename to HashOnly

* cleanup backend.append()

* simplify vec_backend to match simpler append api

core/src/core/block.rs

@@ -34,7 +34,7 @@ use core::{
 use global;
 use keychain::{self, BlindingFactor};
 use pow::{Difficulty, Proof, ProofOfWork};
-use ser::{self, Readable, Reader, Writeable, Writer};
+use ser::{self, PMMRable, Readable, Reader, Writeable, Writer};
 use util::{secp, static_secp_instance, LOGGER};
 
 /// Errors thrown by Block validation
@@ -189,6 +189,14 @@ impl Default for BlockHeader {
 	}
 }
 
+/// Block header hashes are maintained in the header MMR
+/// but we store the data itself in the db.
+impl PMMRable for BlockHeader {
+	fn len() -> usize {
+		0
+	}
+}
+
 /// Serialization of a block header
 impl Writeable for BlockHeader {
 	fn write<W: Writer>(&self, writer: &mut W) -> Result<(), ser::Error> {

core/src/core/hash.rs

@@ -53,11 +53,13 @@ impl fmt::Display for Hash {
 }
 
 impl Hash {
+	pub const SIZE: usize = 32;
+
 	/// Builds a Hash from a byte vector. If the vector is too short, it will be
 	/// completed by zeroes. If it's too long, it will be truncated.
 	pub fn from_vec(v: &[u8]) -> Hash {
-		let mut h = [0; 32];
-		let copy_size = min(v.len(), 32);
+		let mut h = [0; Hash::SIZE];
+		let copy_size = min(v.len(), Hash::SIZE);
 		h[..copy_size].copy_from_slice(&v[..copy_size]);
 		Hash(h)
 	}

core/src/core/pmmr/backend.rs

@@ -18,6 +18,14 @@ use core::hash::Hash;
 use core::BlockHeader;
 use ser::PMMRable;
 
+pub trait HashOnlyBackend {
+	fn append(&mut self, data: Vec<Hash>) -> Result<(), String>;
+
+	fn rewind(&mut self, position: u64) -> Result<(), String>;
+
+	fn get_hash(&self, position: u64) -> Option<Hash>;
+}
+
 /// Storage backend for the MMR, just needs to be indexed by order of insertion.
 /// The PMMR itself does not need the Backend to be accurate on the existence
 /// of an element (i.e. remove could be a no-op) but layers above can
@@ -30,7 +38,7 @@ where
 	/// associated data element to flatfile storage (for leaf nodes only). The
 	/// position of the first element of the Vec in the MMR is provided to
 	/// help the implementation.
-	fn append(&mut self, position: u64, data: Vec<(Hash, Option<T>)>) -> Result<(), String>;
+	fn append(&mut self, data: T, hashes: Vec<Hash>) -> Result<(), String>;
 
 	/// Rewind the backend state to a previous position, as if all append
 	/// operations after that had been canceled. Expects a position in the PMMR

core/src/core/pmmr/db_pmmr.rs

@@ -0,0 +1,173 @@
+// Copyright 2018 The Grin Developers
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Database backed MMR.
+
+use std::marker;
+
+use core::hash::Hash;
+use core::pmmr::{bintree_postorder_height, is_leaf, peak_map_height, peaks, HashOnlyBackend};
+use ser::{PMMRIndexHashable, PMMRable};
+
+/// Database backed MMR.
+pub struct DBPMMR<'a, T, B>
+where
+	T: PMMRable,
+	B: 'a + HashOnlyBackend,
+{
+	/// The last position in the PMMR
+	last_pos: u64,
+	/// The backend for this readonly PMMR
+	backend: &'a mut B,
+	// only needed to parameterise Backend
+	_marker: marker::PhantomData<T>,
+}
+
+impl<'a, T, B> DBPMMR<'a, T, B>
+where
+	T: PMMRable + ::std::fmt::Debug,
+	B: 'a + HashOnlyBackend,
+{
+	/// Build a new db backed MMR.
+	pub fn new(backend: &'a mut B) -> DBPMMR<T, B> {
+		DBPMMR {
+			last_pos: 0,
+			backend: backend,
+			_marker: marker::PhantomData,
+		}
+	}
+
+	/// Build a new db backed MMR initialized to
+	/// last_pos with the provided db backend.
+	pub fn at(backend: &'a mut B, last_pos: u64) -> DBPMMR<T, B> {
+		DBPMMR {
+			last_pos: last_pos,
+			backend: backend,
+			_marker: marker::PhantomData,
+		}
+	}
+
+	pub fn unpruned_size(&self) -> u64 {
+		self.last_pos
+	}
+
+	pub fn is_empty(&self) -> bool {
+		self.last_pos == 0
+	}
+
+	pub fn rewind(&mut self, position: u64) -> Result<(), String> {
+		// Identify which actual position we should rewind to as the provided
+		// position is a leaf. We traverse the MMR to include any parent(s) that
+		// need to be included for the MMR to be valid.
+		let mut pos = position;
+		while bintree_postorder_height(pos + 1) > 0 {
+			pos += 1;
+		}
+		self.backend.rewind(pos)?;
+		self.last_pos = pos;
+		Ok(())
+	}
+
+	/// Get the hash element at provided position in the MMR.
+	pub fn get_hash(&self, pos: u64) -> Option<Hash> {
+		if pos > self.last_pos {
+			// If we are beyond the rhs of the MMR return None.
+			None
+		} else if is_leaf(pos) {
+			// If we are a leaf then get data from the backend.
+			self.backend.get_hash(pos)
+		} else {
+			// If we are not a leaf then return None as only leaves have data.
+			None
+		}
+	}
+
+	/// Push a new element into the MMR. Computes new related peaks at
+	/// the same time if applicable.
+	pub fn push(&mut self, elmt: T) -> Result<u64, String> {
+		let elmt_pos = self.last_pos + 1;
+		let mut current_hash = elmt.hash_with_index(elmt_pos - 1);
+
+		let mut to_append = vec![current_hash];
+		let mut pos = elmt_pos;
+
+		let (peak_map, height) = peak_map_height(pos - 1);
+		if height != 0 {
+			return Err(format!("bad mmr size {}", pos - 1));
+		}
+		// hash with all immediately preceding peaks, as indicated by peak map
+		let mut peak = 1;
+		while (peak_map & peak) != 0 {
+			let left_sibling = pos + 1 - 2 * peak;
+			let left_hash = self
+				.backend
+				.get_hash(left_sibling)
+				.ok_or("missing left sibling in tree, should not have been pruned")?;
+			peak *= 2;
+			pos += 1;
+			current_hash = (left_hash, current_hash).hash_with_index(pos - 1);
+			to_append.push(current_hash);
+		}
+
+		// append all the new nodes and update the MMR index
+		self.backend.append(to_append)?;
+		self.last_pos = pos;
+		Ok(elmt_pos)
+	}
+
+	pub fn peaks(&self) -> Vec<Hash> {
+		let peaks_pos = peaks(self.last_pos);
+		peaks_pos
+			.into_iter()
+			.filter_map(|pi| self.backend.get_hash(pi))
+			.collect()
+	}
+
+	pub fn root(&self) -> Hash {
+		let mut res = None;
+		for peak in self.peaks().iter().rev() {
+			res = match res {
+				None => Some(*peak),
+				Some(rhash) => Some((*peak, rhash).hash_with_index(self.unpruned_size())),
+			}
+		}
+		res.expect("no root, invalid tree")
+	}
+
+	pub fn validate(&self) -> Result<(), String> {
+		// iterate on all parent nodes
+		for n in 1..(self.last_pos + 1) {
+			let height = bintree_postorder_height(n);
+			if height > 0 {
+				if let Some(hash) = self.get_hash(n) {
+					let left_pos = n - (1 << height);
+					let right_pos = n - 1;
+					if let Some(left_child_hs) = self.get_hash(left_pos) {
+						if let Some(right_child_hs) = self.get_hash(right_pos) {
+							// hash the two child nodes together with parent_pos and compare
+							if (left_child_hs, right_child_hs).hash_with_index(n - 1) != hash {
+								return Err(format!(
+									"Invalid MMR, hash of parent at {} does \
+									 not match children.",
+									n
+								));
+							}
+						}
+					}
+				}
+			}
+		}
+		Ok(())
+	}
+}

core/src/core/pmmr/mod.rs

@@ -37,11 +37,13 @@
 //! either be a simple Vec or a database.
 
 mod backend;
+mod db_pmmr;
 mod pmmr;
 mod readonly_pmmr;
 mod rewindable_pmmr;
 
 pub use self::backend::*;
+pub use self::db_pmmr::*;
 pub use self::pmmr::*;
 pub use self::readonly_pmmr::*;
 pub use self::rewindable_pmmr::*;

core/src/core/pmmr/pmmr.rs

@@ -84,8 +84,7 @@ where
 				// here we want to get from underlying hash file
 				// as the pos *may* have been "removed"
 				self.backend.get_from_file(pi)
-			})
-			.collect()
+			}).collect()
 	}
 
 	fn peak_path(&self, peak_pos: u64) -> Vec<Hash> {
@@ -174,7 +173,7 @@ where
 		let elmt_pos = self.last_pos + 1;
 		let mut current_hash = elmt.hash_with_index(elmt_pos - 1);
 
-		let mut to_append = vec![(current_hash, Some(elmt))];
+		let mut to_append = vec![current_hash];
 		let mut pos = elmt_pos;
 
 		let (peak_map, height) = peak_map_height(pos - 1);
@@ -192,11 +191,11 @@ where
 			peak *= 2;
 			pos += 1;
 			current_hash = (left_hash, current_hash).hash_with_index(pos - 1);
-			to_append.push((current_hash, None));
+			to_append.push(current_hash);
 		}
 
 		// append all the new nodes and update the MMR index
-		self.backend.append(elmt_pos, to_append)?;
+		self.backend.append(elmt, to_append)?;
 		self.last_pos = pos;
 		Ok(elmt_pos)
 	}
@@ -464,6 +463,9 @@ pub fn n_leaves(size: u64) -> u64 {
 
 /// Returns the pmmr index of the nth inserted element
 pub fn insertion_to_pmmr_index(mut sz: u64) -> u64 {
+	if sz == 0 {
+		return 0;
+	}
 	// 1 based pmmrs
 	sz -= 1;
 	2 * sz - sz.count_ones() as u64 + 1

core/src/core/pmmr/rewindable_pmmr.rs

@@ -110,8 +110,7 @@ where
 				// here we want to get from underlying hash file
 				// as the pos *may* have been "removed"
 				self.backend.get_from_file(pi)
-			})
-			.collect()
+			}).collect()
 	}
 
 	/// Total size of the tree, including intermediary nodes and ignoring any

core/tests/pmmr.rs

@@ -438,7 +438,7 @@ fn pmmr_prune() {
 	}
 
 	// First check the initial numbers of elements.
-	assert_eq!(ba.elems.len(), 16);
+	assert_eq!(ba.hashes.len(), 16);
 	assert_eq!(ba.remove_list.len(), 0);
 
 	// pruning a leaf with no parent should do nothing
@@ -447,7 +447,7 @@ fn pmmr_prune() {
 		pmmr.prune(16).unwrap();
 		assert_eq!(orig_root, pmmr.root());
 	}
-	assert_eq!(ba.elems.len(), 16);
+	assert_eq!(ba.hashes.len(), 16);
 	assert_eq!(ba.remove_list.len(), 1);
 
 	// pruning leaves with no shared parent just removes 1 element
@@ -456,7 +456,7 @@ fn pmmr_prune() {
 		pmmr.prune(2).unwrap();
 		assert_eq!(orig_root, pmmr.root());
 	}
-	assert_eq!(ba.elems.len(), 16);
+	assert_eq!(ba.hashes.len(), 16);
 	assert_eq!(ba.remove_list.len(), 2);
 
 	{
@@ -464,7 +464,7 @@ fn pmmr_prune() {
 		pmmr.prune(4).unwrap();
 		assert_eq!(orig_root, pmmr.root());
 	}
-	assert_eq!(ba.elems.len(), 16);
+	assert_eq!(ba.hashes.len(), 16);
 	assert_eq!(ba.remove_list.len(), 3);
 
 	// pruning a non-leaf node has no effect
@@ -473,7 +473,7 @@ fn pmmr_prune() {
 		pmmr.prune(3).unwrap_err();
 		assert_eq!(orig_root, pmmr.root());
 	}
-	assert_eq!(ba.elems.len(), 16);
+	assert_eq!(ba.hashes.len(), 16);
 	assert_eq!(ba.remove_list.len(), 3);
 
 	// TODO - no longer true (leaves only now) - pruning sibling removes subtree
@@ -482,7 +482,7 @@ fn pmmr_prune() {
 		pmmr.prune(5).unwrap();
 		assert_eq!(orig_root, pmmr.root());
 	}
-	assert_eq!(ba.elems.len(), 16);
+	assert_eq!(ba.hashes.len(), 16);
 	assert_eq!(ba.remove_list.len(), 4);
 
 	// TODO - no longer true (leaves only now) - pruning all leaves under level >1
@@ -492,7 +492,7 @@ fn pmmr_prune() {
 		pmmr.prune(1).unwrap();
 		assert_eq!(orig_root, pmmr.root());
 	}
-	assert_eq!(ba.elems.len(), 16);
+	assert_eq!(ba.hashes.len(), 16);
 	assert_eq!(ba.remove_list.len(), 5);
 
 	// pruning everything should only leave us with a single peak
@@ -503,7 +503,7 @@ fn pmmr_prune() {
 		}
 		assert_eq!(orig_root, pmmr.root());
 	}
-	assert_eq!(ba.elems.len(), 16);
+	assert_eq!(ba.hashes.len(), 16);
 	assert_eq!(ba.remove_list.len(), 9);
 }
 

core/tests/vec_backend/mod.rs

@@ -17,7 +17,7 @@ extern crate croaring;
 use croaring::Bitmap;
 
 use core::core::hash::Hash;
-use core::core::pmmr::Backend;
+use core::core::pmmr::{self, Backend};
 use core::core::BlockHeader;
 use core::ser;
 use core::ser::{PMMRable, Readable, Reader, Writeable, Writer};
@@ -59,7 +59,8 @@ where
 	T: PMMRable,
 {
 	/// Backend elements
-	pub elems: Vec<Option<(Hash, Option<T>)>>,
+	pub data: Vec<T>,
+	pub hashes: Vec<Hash>,
 	/// Positions of removed elements
 	pub remove_list: Vec<u64>,
 }
@@ -68,8 +69,9 @@ impl<T> Backend<T> for VecBackend<T>
 where
 	T: PMMRable,
 {
-	fn append(&mut self, _position: u64, data: Vec<(Hash, Option<T>)>) -> Result<(), String> {
-		self.elems.append(&mut map_vec!(data, |d| Some(d.clone())));
+	fn append(&mut self, data: T, hashes: Vec<Hash>) -> Result<(), String> {
+		self.data.push(data);
+		self.hashes.append(&mut hashes.clone());
 		Ok(())
 	}
 
@@ -77,11 +79,7 @@ where
 		if self.remove_list.contains(&position) {
 			None
 		} else {
-			if let Some(ref elem) = self.elems[(position - 1) as usize] {
-				Some(elem.0)
-			} else {
-				None
-			}
+			self.get_from_file(position)
 		}
 	}
 
@@ -89,28 +87,19 @@ where
 		if self.remove_list.contains(&position) {
 			None
 		} else {
-			if let Some(ref elem) = self.elems[(position - 1) as usize] {
-				elem.1.clone()
-			} else {
-				None
-			}
+			self.get_data_from_file(position)
 		}
 	}
 
 	fn get_from_file(&self, position: u64) -> Option<Hash> {
-		if let Some(ref x) = self.elems[(position - 1) as usize] {
-			Some(x.0)
-		} else {
-			None
-		}
+		let hash = &self.hashes[(position - 1) as usize];
+		Some(hash.clone())
 	}
 
 	fn get_data_from_file(&self, position: u64) -> Option<T> {
-		if let Some(ref x) = self.elems[(position - 1) as usize] {
-			x.1.clone()
-		} else {
-			None
-		}
+		let idx = pmmr::n_leaves(position);
+		let data = &self.data[(idx - 1) as usize];
+		Some(data.clone())
 	}
 
 	fn remove(&mut self, position: u64) -> Result<(), String> {
@@ -119,7 +108,9 @@ where
 	}
 
 	fn rewind(&mut self, position: u64, _rewind_rm_pos: &Bitmap) -> Result<(), String> {
-		self.elems = self.elems[0..(position as usize) + 1].to_vec();
+		let idx = pmmr::n_leaves(position);
+		self.data = self.data[0..(idx as usize) + 1].to_vec();
+		self.hashes = self.hashes[0..(position as usize) + 1].to_vec();
 		Ok(())
 	}
 
@@ -141,20 +132,9 @@ where
 	/// Instantiates a new VecBackend<T>
 	pub fn new() -> VecBackend<T> {
 		VecBackend {
-			elems: vec![],
+			data: vec![],
+			hashes: vec![],
 			remove_list: vec![],
 		}
 	}
-
-	// /// Current number of elements in the underlying Vec.
-	// pub fn used_size(&self) -> usize {
-	// 	let mut usz = self.elems.len();
-	// 	for (idx, _) in self.elems.iter().enumerate() {
-	// 		let idx = idx as u64;
-	// 		if self.remove_list.contains(&idx) {
-	// 			usz -= 1;
-	// 		}
-	// 	}
-	// 	usz
-	// }
 }