check_compact retains leaves and roots until parents are pruned (#753)

* wip

* failing test for being too eager when pruning a sibling

* commit

* rustfmt

* [WIP] modified get_shift and get_leaf_shift to account for leaving "pruned but not compacted" leaves in place
Note: this currently breaks check_compact as nothing else is aware of the modified behavior

* rustfmt

* commit

* rustfmt

* basic prune/compact/shift working

* rustfmt

* commit

* rustfmt

* next_pruned_idx working (I think)

* commit

* horizon test uncovered some subtle issues - wip

* rustfmt

* cleanup

* rustfmt

* commit

* cleanup

* cleanup

* commit

* rustfmt

* contains -> binary_search

* rustfmt

* no need for height==0 special case

* wip - works for single compact, 2nd one breaks the mmr hashes

* commit

* rustfmt

* fixed it (needs a lot of cleanup)
we were not traversing all the way up to the peak if we pruned an entire tree
so rm_log and prune list were inconsistent

* multiple compact steps are working
data file not being copmacted currently (still to investigate)

* cleanup store tests

* cleanup

* cleanup up debug

* rustfmt

* take kernel offsets into account when summing kernels and outputs for full txhashset validation
validate chain state pre and post compaction

* rustfmt

* fix wallet refresh (we need block height to be refreshed on non-coinbase outputs)
otherwise we cannot spend them...

* rustfmt

chain/src/chain.rs

@@ -556,11 +556,36 @@ impl Chain {
 	/// Meanwhile, the chain will not be able to accept new blocks. It should
 	/// therefore be called judiciously.
 	pub fn compact(&self) -> Result<(), Error> {
-		let mut sumtrees = self.txhashset.write().unwrap();
-		sumtrees.compact()?;
+		// First check we can successfully validate the full chain state.
+		// If we cannot then do not attempt to compact.
+		// This should not be required long term - but doing this for debug purposes.
+		self.validate()?;
 
+		// Now compact the txhashset via the extension.
+		{
+			let mut txhashes = self.txhashset.write().unwrap();
+			txhashes.compact()?;
+
+			// print out useful debug info after compaction
+			txhashset::extending(&mut txhashes, |extension| {
+				extension.dump_output_pmmr();
+				Ok(())
+			})?;
+		}
+
+		// Now check we can still successfully validate the chain state after
+		// compacting.
+		self.validate()?;
+
+		// we need to be careful here in testing as 20 blocks is not that long
+		// in wall clock time
 		let horizon = global::cut_through_horizon() as u64;
 		let head = self.head()?;
+
+		if head.height <= horizon {
+			return Ok(());
+		}
+
 		let mut current = self.store.get_header_by_height(head.height - horizon - 1)?;
 		loop {
 			match self.store.get_block(&current.hash()) {

chain/src/lib.rs

@@ -31,6 +31,7 @@ extern crate slog;
 extern crate time;
 
 extern crate grin_core as core;
+extern crate grin_keychain as keychain;
 extern crate grin_store;
 extern crate grin_util as util;
 

chain/src/txhashset.rs

@@ -35,6 +35,7 @@ use core::ser::{self, PMMRIndexHashable, PMMRable};
 use grin_store;
 use grin_store::pmmr::{PMMRBackend, PMMRFileMetadata};
 use grin_store::types::prune_noop;
+use keychain::BlindingFactor;
 use types::{ChainStore, Error, PMMRFileMetadataCollection, TxHashSetRoots};
 use util::{zip, LOGGER};
 
@@ -54,7 +55,7 @@ where
 
 impl<T> PMMRHandle<T>
 where
-	T: PMMRable,
+	T: PMMRable + ::std::fmt::Debug,
 {
 	fn new(
 		root_dir: String,
@@ -210,15 +211,24 @@ impl TxHashSet {
 
 	/// Compact the MMR data files and flush the rm logs
 	pub fn compact(&mut self) -> Result<(), Error> {
-		let horizon = global::cut_through_horizon();
 		let commit_index = self.commit_index.clone();
+		let head = commit_index.head()?;
+		let current_height = head.height;
+
+		// horizon for compacting is based on current_height
+		let horizon = (current_height as u32).saturating_sub(global::cut_through_horizon());
+
 		let clean_output_index = |commit: &[u8]| {
+			// do we care if this fails?
 			let _ = commit_index.delete_output_pos(commit);
 		};
+
 		let min_rm = (horizon / 10) as usize;
+
 		self.output_pmmr_h
 			.backend
 			.check_compact(min_rm, horizon, clean_output_index)?;
+
 		self.rproof_pmmr_h
 			.backend
 			.check_compact(min_rm, horizon, &prune_noop)?;
@@ -381,6 +391,7 @@ impl<'a> Extension<'a> {
 				// check hash from pmmr matches hash from input (or corresponding output)
 				// if not then the input is not being honest about
 				// what it is attempting to spend...
+
 				if output_id_hash != read_hash
 					|| output_id_hash
 						!= read_elem
@@ -562,14 +573,19 @@ impl<'a> Extension<'a> {
 
 		// the real magicking: the sum of all kernel excess should equal the sum
 		// of all Output commitments, minus the total supply
-		let (kernel_sum, fees) = self.sum_kernels()?;
+		let kernel_offset = self.sum_kernel_offsets(&header)?;
+		let kernel_sum = self.sum_kernels(kernel_offset)?;
 		let output_sum = self.sum_outputs()?;
+
+		// supply is the sum of the coinbase outputs from all the block headers
+		let supply = header.height * REWARD;
+
 		{
 			let secp = static_secp_instance();
 			let secp = secp.lock().unwrap();
-			let over_commit = secp.commit_value(header.height * REWARD)?;
-			let adjusted_sum_output = secp.commit_sum(vec![output_sum], vec![over_commit])?;
 
+			let over_commit = secp.commit_value(supply)?;
+			let adjusted_sum_output = secp.commit_sum(vec![output_sum], vec![over_commit])?;
 			if adjusted_sum_output != kernel_sum {
 				return Err(Error::InvalidTxHashSet(
 					"Differing Output commitment and kernel excess sums.".to_owned(),
@@ -601,6 +617,14 @@ impl<'a> Extension<'a> {
 		self.rollback = true;
 	}
 
+	/// Dumps the output MMR.
+	/// We use this after compacting for visual confirmation that it worked.
+	pub fn dump_output_pmmr(&self) {
+		debug!(LOGGER, "-- outputs --");
+		self.output_pmmr.dump_from_file(false);
+		debug!(LOGGER, "-- end of outputs --");
+	}
+
 	/// Dumps the state of the 3 sum trees to stdout for debugging. Short
 	/// version only prints the Output tree.
 	pub fn dump(&self, short: bool) {
@@ -623,9 +647,46 @@ impl<'a> Extension<'a> {
 		)
 	}
 
+	/// TODO - Just use total_offset from latest header once this is available.
+	/// So we do not need to iterate over all the headers to calculate it.
+	fn sum_kernel_offsets(&self, header: &BlockHeader) -> Result<Option<Commitment>, Error> {
+		let mut kernel_offsets = vec![];
+
+		// iterate back up the chain collecting the kernel offset for each block header
+		let mut current = header.clone();
+		while current.height > 0 {
+			kernel_offsets.push(current.kernel_offset);
+			current = self.commit_index.get_block_header(&current.previous)?;
+		}
+
+		// now sum the kernel_offset from each block header
+		// to give us an aggregate offset for the entire
+		// blockchain
+		let secp = static_secp_instance();
+		let secp = secp.lock().unwrap();
+
+		let keys = kernel_offsets
+			.iter()
+			.cloned()
+			.filter(|x| *x != BlindingFactor::zero())
+			.filter_map(|x| x.secret_key(&secp).ok())
+			.collect::<Vec<_>>();
+
+		let offset = if keys.is_empty() {
+			None
+		} else {
+			let sum = secp.blind_sum(keys, vec![])?;
+			let offset = BlindingFactor::from_secret_key(sum);
+			let skey = offset.secret_key(&secp)?;
+			Some(secp.commit(0, skey)?)
+		};
+
+		Ok(offset)
+	}
+
 	/// Sums the excess of all our kernels, validating their signatures on the
 	/// way
-	fn sum_kernels(&self) -> Result<(Commitment, u64), Error> {
+	fn sum_kernels(&self, kernel_offset: Option<Commitment>) -> Result<Commitment, Error> {
 		// make sure we have the right count of kernels using the MMR, the storage
 		// file may have a few more
 		let mmr_sz = self.kernel_pmmr.unpruned_size();
@@ -635,7 +696,6 @@ impl<'a> Extension<'a> {
 		let first: TxKernel = ser::deserialize(&mut kernel_file)?;
 		first.verify()?;
 		let mut sum_kernel = first.excess;
-		let mut fees = first.fee;
 
 		let secp = static_secp_instance();
 		let mut kern_count = 1;
@@ -645,7 +705,6 @@ impl<'a> Extension<'a> {
 					kernel.verify()?;
 					let secp = secp.lock().unwrap();
 					sum_kernel = secp.commit_sum(vec![sum_kernel, kernel.excess], vec![])?;
-					fees += kernel.fee;
 					kern_count += 1;
 					if kern_count == count {
 						break;
@@ -654,8 +713,20 @@ impl<'a> Extension<'a> {
 				Err(_) => break,
 			}
 		}
-		debug!(LOGGER, "Validated and summed {} kernels", kern_count);
-		Ok((sum_kernel, fees))
+
+		// now apply the kernel offset of we have one
+		{
+			let secp = secp.lock().unwrap();
+			if let Some(kernel_offset) = kernel_offset {
+				sum_kernel = secp.commit_sum(vec![sum_kernel, kernel_offset], vec![])?;
+			}
+		}
+
+		debug!(
+			LOGGER,
+			"Validated, summed (and offset) {} kernels", kern_count
+		);
+		Ok(sum_kernel)
 	}
 
 	/// Sums all our Output commitments, checking range proofs at the same time
@@ -664,7 +735,7 @@ impl<'a> Extension<'a> {
 		let mut output_count = 0;
 		let secp = static_secp_instance();
 		for n in 1..self.output_pmmr.unpruned_size() + 1 {
-			if pmmr::bintree_postorder_height(n) == 0 {
+			if pmmr::is_leaf(n) {
 				if let Some((_, output)) = self.output_pmmr.get(n, true) {
 					let out = output.expect("not a leaf node");
 					let commit = out.commit.clone();

chain/src/types.rs

@@ -24,6 +24,7 @@ use core::core::{block, transaction, Block, BlockHeader};
 use core::core::hash::{Hash, Hashed};
 use core::core::target::Difficulty;
 use core::ser::{self, Readable, Reader, Writeable, Writer};
+use keychain;
 use grin_store;
 use grin_store::pmmr::PMMRFileMetadata;
 
@@ -75,6 +76,10 @@ pub enum Error {
 	InvalidRoot,
 	/// Something does not look right with the switch commitment
 	InvalidSwitchCommit,
+	/// Error from underlying keychain impl
+	Keychain(keychain::Error),
+	/// Error from underlying secp lib
+	Secp(secp::Error),
 	/// One of the inputs in the block has already been spent
 	AlreadySpent(Commitment),
 	/// An output with that commitment already exists (should be unique)
@@ -108,19 +113,28 @@ impl From<grin_store::Error> for Error {
 		Error::StoreErr(e, "wrapped".to_owned())
 	}
 }
+
 impl From<ser::Error> for Error {
 	fn from(e: ser::Error) -> Error {
 		Error::SerErr(e)
 	}
 }
+
 impl From<io::Error> for Error {
 	fn from(e: io::Error) -> Error {
 		Error::TxHashSetErr(e.to_string())
 	}
 }
+
+impl From<keychain::Error> for Error {
+	fn from(e: keychain::Error) -> Error {
+		Error::Keychain(e)
+	}
+}
+
 impl From<secp::Error> for Error {
 	fn from(e: secp::Error) -> Error {
-		Error::TxHashSetErr(format!("Sum validation error: {}", e.to_string()))
+		Error::Secp(e)
 	}
 }