[testnet2] Store output sum (#1043)

* block sums and reworked block validation
read and write block_sums
refactor validate on both block and txhashset
write block_sum on fast sync
we store the kernel_sum (need to account for the offset)

* block_sums

* rustfmt

* cleanup

chain/src/txhashset.rs

@@ -35,9 +35,8 @@ use core::ser::{PMMRIndexHashable, PMMRable};
 use grin_store;
 use grin_store::pmmr::PMMRBackend;
 use grin_store::types::prune_noop;
-use keychain::BlindingFactor;
-use types::{BlockMarker, ChainStore, Error, TxHashSetRoots};
-use util::{zip, LOGGER};
+use types::{BlockMarker, BlockSums, ChainStore, Error, TxHashSetRoots};
+use util::{secp_static, zip, LOGGER};
 
 const TXHASHSET_SUBDIR: &'static str = "txhashset";
 const OUTPUT_SUBDIR: &'static str = "output";
@@ -607,8 +606,9 @@ impl<'a> Extension<'a> {
 		Ok(())
 	}
 
-	/// Validate the txhashset state against the provided block header.
-	pub fn validate(&mut self, header: &BlockHeader, skip_rproofs: bool) -> Result<(), Error> {
+	fn validate_mmrs(&self) -> Result<(), Error> {
+		let now = Instant::now();
+
 		// validate all hashes and sums within the trees
 		if let Err(e) = self.output_pmmr.validate() {
 			return Err(Error::InvalidTxHashSet(e));
@@ -620,40 +620,106 @@ impl<'a> Extension<'a> {
 			return Err(Error::InvalidTxHashSet(e));
 		}
 
+		debug!(
+			LOGGER,
+			"txhashset: validated the output|rproof|kernel mmrs, took {}s",
+			now.elapsed().as_secs(),
+		);
+
+		Ok(())
+	}
+
+	/// The real magicking: the sum of all kernel excess should equal the sum
+	/// of all output commitments, minus the total supply.
+	pub fn validate_sums(&self, header: &BlockHeader) -> Result<((Commitment, Commitment)), Error> {
+		let now = Instant::now();
+
+		// supply is the sum of the coinbase outputs from each block header
+		let supply_commit = {
+			let secp = static_secp_instance();
+			let secp = secp.lock().unwrap();
+			let supply = header.height * REWARD;
+			secp.commit_value(supply)?
+		};
+		let output_sum = self.sum_outputs(supply_commit)?;
+		let kernel_sum = self.sum_kernels()?;
+
+		let zero_commit = secp_static::commit_to_zero_value();
+
+		let offset = {
+			let secp = static_secp_instance();
+			let secp = secp.lock().unwrap();
+			let key = header.total_kernel_offset.secret_key(&secp)?;
+			secp.commit(0, key)?
+		};
+
+		let mut excesses = vec![kernel_sum, offset];
+		excesses.retain(|x| *x != zero_commit);
+
+		let kernel_sum_plus_offset = {
+			let secp = static_secp_instance();
+			let secp = secp.lock().unwrap();
+			secp.commit_sum(excesses, vec![])?
+		};
+
+		if output_sum != kernel_sum_plus_offset {
+			return Err(Error::InvalidTxHashSet(
+				"Differing Output commitment and kernel excess sums.".to_owned(),
+			));
+		}
+
+		debug!(
+			LOGGER,
+			"txhashset: validated sums, took (total) {}s",
+			now.elapsed().as_secs(),
+		);
+
+		Ok((output_sum, kernel_sum))
+	}
+
+	/// Validate the txhashset state against the provided block header.
+	pub fn validate(
+		&mut self,
+		header: &BlockHeader,
+		skip_rproofs: bool,
+	) -> Result<((Commitment, Commitment)), Error> {
+		self.validate_mmrs()?;
 		self.validate_roots(header)?;
 
 		if header.height == 0 {
-			return Ok(());
+			let zero_commit = secp_static::commit_to_zero_value();
+			return Ok((zero_commit.clone(), zero_commit.clone()));
 		}
 
-		// the real magicking: the sum of all kernel excess should equal the sum
-		// of all Output commitments, minus the total supply
-		let kernel_offset = self.sum_kernel_offsets(&header)?;
-		let kernel_sum = self.sum_kernels(kernel_offset)?;
-		let output_sum = self.sum_outputs()?;
+		let (output_sum, kernel_sum) = self.validate_sums(header)?;
 
-		// supply is the sum of the coinbase outputs from all the block headers
-		let supply = header.height * REWARD;
+		// this is a relatively expensive verification step
+		self.verify_kernel_signatures()?;
 
-		{
-			let secp = static_secp_instance();
-			let secp = secp.lock().unwrap();
-
-			let over_commit = secp.commit_value(supply)?;
-			let adjusted_sum_output = secp.commit_sum(vec![output_sum], vec![over_commit])?;
-			if adjusted_sum_output != kernel_sum {
-				return Err(Error::InvalidTxHashSet(
-					"Differing Output commitment and kernel excess sums.".to_owned(),
-				));
-			}
-		}
-
-		// now verify the rangeproof for each output in the sum above
+		// verify the rangeproof for each output in the sum above
 		// this is an expensive operation (only verified if requested)
 		if !skip_rproofs {
 			self.verify_rangeproofs()?;
 		}
 
+		Ok((output_sum, kernel_sum))
+	}
+
+	/// Save blocks sums (the output_sum and kernel_sum) for the given block
+	/// header.
+	pub fn save_latest_block_sums(
+		&self,
+		header: &BlockHeader,
+		output_sum: Commitment,
+		kernel_sum: Commitment,
+	) -> Result<(), Error> {
+		self.commit_index.save_block_sums(
+			&header.hash(),
+			&BlockSums {
+				output_sum,
+				kernel_sum,
+			},
+		)?;
 		Ok(())
 	}
 
@@ -709,53 +775,59 @@ impl<'a> Extension<'a> {
 		)
 	}
 
-	// We maintain the total accumulated kernel offset in each block header.
-	// So "summing" is just a case of taking the total kernel offset
-	// directly from the current block header.
-	fn sum_kernel_offsets(&self, header: &BlockHeader) -> Result<Option<Commitment>, Error> {
-		let offset = if header.total_kernel_offset == BlindingFactor::zero() {
-			None
-		} else {
-			let secp = static_secp_instance();
-			let secp = secp.lock().unwrap();
-			let skey = header.total_kernel_offset.secret_key(&secp)?;
-			Some(secp.commit(0, skey)?)
-		};
-		Ok(offset)
-	}
-
-	/// Sums the excess of all our kernels, validating their signatures on the
-	/// way
-	fn sum_kernels(&self, kernel_offset: Option<Commitment>) -> Result<Commitment, Error> {
+	/// Sums the excess of all our kernels.
+	fn sum_kernels(&self) -> Result<Commitment, Error> {
 		let now = Instant::now();
 
 		let mut commitments = vec![];
-		if let Some(offset) = kernel_offset {
-			commitments.push(offset);
-		}
-
 		for n in 1..self.kernel_pmmr.unpruned_size() + 1 {
 			if pmmr::is_leaf(n) {
 				if let Some(kernel) = self.kernel_pmmr.get_data(n) {
-					kernel.verify()?;
 					commitments.push(kernel.excess);
 				}
 			}
 		}
 
-		let secp = static_secp_instance();
-		let secp = secp.lock().unwrap();
 		let kern_count = commitments.len();
-		let sum_kernel = secp.commit_sum(commitments, vec![])?;
+
+		let kernel_sum = {
+			let secp = static_secp_instance();
+			let secp = secp.lock().unwrap();
+			secp.commit_sum(commitments, vec![])?
+		};
 
 		debug!(
 			LOGGER,
-			"Validated, summed (and offset) {} kernels, pmmr size {}, took {}s",
+			"txhashset: summed {} kernels, pmmr size {}, took {}s",
 			kern_count,
 			self.kernel_pmmr.unpruned_size(),
 			now.elapsed().as_secs(),
 		);
-		Ok(sum_kernel)
+		Ok(kernel_sum)
+	}
+
+	fn verify_kernel_signatures(&self) -> Result<(), Error> {
+		let now = Instant::now();
+
+		let mut kern_count = 0;
+		for n in 1..self.kernel_pmmr.unpruned_size() + 1 {
+			if pmmr::is_leaf(n) {
+				if let Some(kernel) = self.kernel_pmmr.get_data(n) {
+					kernel.verify()?;
+					kern_count += 1;
+				}
+			}
+		}
+
+		debug!(
+			LOGGER,
+			"txhashset: verified {} kernel signatures, pmmr size {}, took {}s",
+			kern_count,
+			self.kernel_pmmr.unpruned_size(),
+			now.elapsed().as_secs(),
+		);
+
+		Ok(())
 	}
 
 	fn verify_rangeproofs(&self) -> Result<(), Error> {
@@ -784,7 +856,7 @@ impl<'a> Extension<'a> {
 		}
 		debug!(
 			LOGGER,
-			"Verified {} Rangeproofs, pmmr size {}, took {}s",
+			"txhashset: verified {} rangeproofs, pmmr size {}, took {}s",
 			proof_count,
 			self.rproof_pmmr.unpruned_size(),
 			now.elapsed().as_secs(),
@@ -792,8 +864,8 @@ impl<'a> Extension<'a> {
 		Ok(())
 	}
 
-	/// Sums all our Output commitments, checking range proofs at the same time
-	fn sum_outputs(&self) -> Result<Commitment, Error> {
+	/// Sums all our unspent output commitments.
+	fn sum_outputs(&self, supply_commit: Commitment) -> Result<Commitment, Error> {
 		let now = Instant::now();
 
 		let mut commitments = vec![];
@@ -805,20 +877,23 @@ impl<'a> Extension<'a> {
 			}
 		}
 
-		let secp = static_secp_instance();
-		let secp = secp.lock().unwrap();
 		let commit_count = commitments.len();
-		let sum_output = secp.commit_sum(commitments, vec![])?;
+
+		let output_sum = {
+			let secp = static_secp_instance();
+			let secp = secp.lock().unwrap();
+			secp.commit_sum(commitments, vec![supply_commit])?
+		};
 
 		debug!(
 			LOGGER,
-			"Summed {} Outputs, pmmr size {}, took {}s",
+			"txhashset: summed {} outputs, pmmr size {}, took {}s",
 			commit_count,
 			self.output_pmmr.unpruned_size(),
 			now.elapsed().as_secs(),
 		);
 
-		Ok(sum_output)
+		Ok(output_sum)
 	}
 }