[WIP] Abridged sync (#440)

* Util to zip and unzip directories
* First pass at sumtree request/response. Add message types, implement the exchange in the protocol, zip up the sumtree directory and stream the file over, with necessary adapter hooks.
* Implement the sumtree archive receive logicGets the sumtree archive data stream from the network and write it to a file. Unzip the file, place it at the right spot and reconstruct the sumtree data structure, rewinding where to the right spot.
* Sumtree hash structure validation
* Simplify sumtree backend buffering logic. The backend for a sumtree has to implement some in-memory buffering logic to provide a commit/rollback interface. The backend itself is an aggregate of 3 underlying storages (an append only file, a remove log and a skip list). The buffering was previously implemented both by the backend and some of the underlying storages. Now pushing back all buffering logic to the storages to keep the backend simpler.
* Add kernel append only store file to sumtrees. The chain sumtrees structure now also saves all kernels to a dedicated file. As that storage is implemented by the append only file wrapper, it's also rewind-aware.
* Full state validation. Checks that:
    - MMRs are sane (hash and sum each node)
    - Tree roots match the corresponding header
    - Kernel signatures are valid
    - Sum of all kernel excesses equals the sum of UTXO commitments
minus the supply
* Fast sync handoff to body sync. Once the fast-sync state is fully setup, get bacj in body sync
mode to get the full bodies of the last blocks we're missing.
* First fully working fast sync
* Facility in p2p conn to deal with attachments (raw binary after message).
* Re-introduced sumtree send and receive message handling using the above.
* Fixed test and finished updating all required db state after sumtree validation.
* Massaged a little bit the pipeline orphan check to still work after the new sumtrees have been setup.
* Various cleanup. Consolidated fast sync and full sync into a single function as they're very similar. Proper conditions to trigger a sumtree request and some checks on receiving it.

chain/src/chain.rs

@@ -16,13 +16,16 @@
 //! and mostly the chain pipeline.
 
 use std::collections::HashMap;
+use std::fs::File;
 use std::sync::{Arc, Mutex, RwLock};
 use std::time::{Duration, Instant};
 
 use util::secp::pedersen::RangeProof;
 
 use core::core::{Input, OutputIdentifier, SumCommit};
+use core::core::hash::Hashed;
 use core::core::pmmr::{HashSum, NoSum};
+use core::global;
 
 use core::core::{Block, BlockHeader, TxKernel};
 use core::core::target::Difficulty;
@@ -112,6 +115,7 @@ impl OrphanBlockPool {
 /// the current view of the UTXO set according to the chain state. Also
 /// maintains locking for the pipeline to avoid conflicting processing.
 pub struct Chain {
+	db_root: String,
 	store: Arc<ChainStore>,
 	adapter: Arc<ChainAdapter>,
 
@@ -183,9 +187,10 @@ impl Chain {
 		);
 
 		let store = Arc::new(chain_store);
-		let sumtrees = sumtree::SumTrees::open(db_root, store.clone())?;
+		let sumtrees = sumtree::SumTrees::open(db_root.clone(), store.clone())?;
 
 		Ok(Chain {
+			db_root: db_root,
 			store: store,
 			adapter: adapter,
 			head: Arc::new(Mutex::new(head)),
@@ -194,25 +199,26 @@ impl Chain {
 			pow_verifier: pow_verifier,
 		})
 	}
-/// Processes a single block, then checks for orphans, processing
-/// those as well if they're found
-pub fn process_block(&self, b: Block, opts: Options)
--> Result<(Option<Tip>, Option<Block>), Error>
-{
-	let res = self.process_block_no_orphans(b, opts);
-	match res {
-		Ok((t, b)) => {
-			// We accepted a block, so see if we can accept any orphans
-			if b.is_some() {
-				self.check_orphans(&b.clone().unwrap());
+
+	/// Processes a single block, then checks for orphans, processing
+	/// those as well if they're found
+	pub fn process_block(&self, b: Block, opts: Options)
+		-> Result<(Option<Tip>, Option<Block>), Error>
+		{
+			let res = self.process_block_no_orphans(b, opts);
+			match res {
+				Ok((t, b)) => {
+					// We accepted a block, so see if we can accept any orphans
+					if let Some(ref b) = b {
+						self.check_orphans(b.hash());
+					}
+					Ok((t, b))
+				},
+				Err(e) => {
+					Err(e)
+				}
 			}
-			Ok((t, b))
-		},
-		Err(e) => {
-			Err(e)
 		}
-	}
-}
 
 	/// Attempt to add a new block to the chain. Returns the new chain tip if it
 	/// has been added to the longest chain, None if it's added to an (as of
@@ -348,13 +354,12 @@ pub fn process_block(&self, b: Block, opts: Options)
 
 
 	/// Check for orphans, once a block is successfully added
-	pub fn check_orphans(&self, block: &Block) {
+	pub fn check_orphans(&self, mut last_block_hash: Hash) {
 		debug!(
 			LOGGER,
 			"chain: check_orphans: # orphans {}",
 			self.orphans.len(),
 		);
-		let mut last_block_hash = block.hash();
 		// Is there an orphan in our orphans that we can now process?
 		// We just processed the given block, are there any orphans that have this block
 		// as their "previous" block?
@@ -390,6 +395,14 @@ pub fn process_block(&self, b: Block, opts: Options)
 		sumtrees.is_unspent(output_ref)
 	}
 
+	pub fn validate(&self) -> Result<(), Error> {
+		let header = self.store.head_header()?;
+		let mut sumtrees = self.sumtrees.write().unwrap();
+		sumtree::extending(&mut sumtrees, |extension| {
+			extension.validate(&header)
+		})
+	}
+
 	/// Check if the input has matured sufficiently for the given block height.
 	/// This only applies to inputs spending coinbase outputs.
 	/// An input spending a non-coinbase output will always pass this check.
@@ -432,6 +445,76 @@ pub fn process_block(&self, b: Block, opts: Options)
 		sumtrees.roots()
 	}
 
+	/// Provides a reading view into the current sumtree state as well as
+	/// the required indexes for a consumer to rewind to a consistent state
+	/// at the provided block hash.
+	pub fn sumtrees_read(&self, h: Hash) -> Result<(u64, u64, File), Error> {
+		let b = self.get_block(&h)?;
+
+		// get the indexes for the block
+		let out_index: u64;
+		let kernel_index: u64;
+		{
+			let sumtrees = self.sumtrees.read().unwrap();
+			let (oi, ki) = sumtrees.indexes_at(&b)?;
+			out_index = oi;
+			kernel_index = ki;
+		}
+
+		// prepares the zip and return the corresponding Read
+		let sumtree_reader = sumtree::zip_read(self.db_root.clone())?;
+		Ok((out_index, kernel_index, sumtree_reader))
+	}
+
+	/// Writes a reading view on a sumtree state that's been provided to us.
+	/// If we're willing to accept that new state, the data stream will be
+	/// read as a zip file, unzipped and the resulting state files should be
+	/// rewound to the provided indexes.
+	pub fn sumtrees_write(
+		&self,
+		h: Hash,
+		rewind_to_output: u64,
+		rewind_to_kernel: u64,
+		sumtree_data: File
+	) -> Result<(), Error> {
+
+		let head = self.head().unwrap();
+		let header_head = self.get_header_head().unwrap();
+		if header_head.height - head.height < global::cut_through_horizon() as u64 {
+			return Err(Error::InvalidSumtree("not needed".to_owned()));
+		}	
+
+		let header = self.store.get_block_header(&h)?;
+		sumtree::zip_write(self.db_root.clone(), sumtree_data)?;
+
+		let mut sumtrees = sumtree::SumTrees::open(self.db_root.clone(), self.store.clone())?;
+		sumtree::extending(&mut sumtrees, |extension| {
+			extension.rewind_pos(header.height, rewind_to_output, rewind_to_kernel)?;
+			extension.validate(&header)?;
+			// TODO validate kernels and their sums with UTXOs
+			extension.rebuild_index()?;
+			Ok(())
+		})?;
+
+		// replace the chain sumtrees with the newly built one
+		{
+			let mut sumtrees_ref = self.sumtrees.write().unwrap();
+			*sumtrees_ref = sumtrees;
+		}
+
+		// setup new head
+		{
+			let mut head = self.head.lock().unwrap();
+			*head = Tip::from_block(&header);
+			self.store.save_body_head(&head);
+			self.store.save_header_height(&header)?;
+		}
+
+		self.check_orphans(header.hash());
+
+		Ok(())
+	}
+
 	/// returns the last n nodes inserted into the utxo sum tree
 	pub fn get_last_n_utxo(&self, distance: u64) -> Vec<HashSum<SumCommit>> {
 		let mut sumtrees = self.sumtrees.write().unwrap();
@@ -455,6 +538,11 @@ pub fn process_block(&self, b: Block, opts: Options)
 		self.head.lock().unwrap().clone().total_difficulty
 	}
 
+	/// Total difficulty at the head of the header chain
+	pub fn total_header_difficulty(&self) -> Result<Difficulty, Error> {
+		Ok(self.store.get_header_head()?.total_difficulty)
+	}
+
 	/// Reset header_head and sync_head to head of current body chain
 	pub fn reset_head(&self) -> Result<(), Error> {
 		self.store

chain/src/pipe.rs

@@ -64,19 +64,23 @@ pub fn process_block(b: &Block, mut ctx: BlockContext) -> Result<Option<Tip>, Er
 
 	validate_header(&b.header, &mut ctx)?;
 
-	// valid header, now check we actually have the previous block in the store
+	// valid header, now check we actually have the previous block in the store	
 	// not just the header but the block itself
-	// we cannot assume we can use the chain head for this as we may be dealing with a fork
-	// we cannot use heights here as the fork may have jumped in height
-	match ctx.store.get_block(&b.header.previous) {
-		Ok(_) => {},
-		Err(grin_store::Error::NotFoundErr) => {
-			return Err(Error::Orphan);
-		},
-		Err(e) => {
-			return Err(Error::StoreErr(e, "pipe get previous".to_owned()));
+	// short circuit the test first both for performance (in-mem vs db access)
+	// but also for the specific case of the first fast sync full block
+	if b.header.previous != ctx.head.last_block_h {
+		// we cannot assume we can use the chain head for this as we may be dealing with a fork
+		// we cannot use heights here as the fork may have jumped in height
+		match ctx.store.block_exists(&b.header.previous) {
+			Ok(true) => {},
+			Ok(false) => {
+				return Err(Error::Orphan);
+			},
+			Err(e) => {
+				return Err(Error::StoreErr(e, "pipe get previous".to_owned()));
+			}
 		}
-	};
+	}
 
 	// valid header and we have a previous block, time to take the lock on the sum trees
 	let local_sumtrees = ctx.sumtrees.clone();

chain/src/store.rs

@@ -24,6 +24,7 @@ use core::core::{Block, BlockHeader};
 use core::consensus::TargetError;
 use core::core::target::Difficulty;
 use grin_store::{self, option_to_not_found, to_key, Error, u64_to_key};
+use util::LOGGER;
 
 const STORE_SUBPATH: &'static str = "chain";
 
@@ -98,6 +99,10 @@ impl ChainStore for ChainKVStore {
 		option_to_not_found(self.db.get_ser(&to_key(BLOCK_PREFIX, &mut h.to_vec())))
 	}
 
+	fn block_exists(&self, h: &Hash) -> Result<bool, Error> {
+		self.db.exists(&to_key(BLOCK_PREFIX, &mut h.to_vec()))
+	}
+
 	fn get_block_header(&self, h: &Hash) -> Result<BlockHeader, Error> {
 		option_to_not_found(
 			self.db.get_ser(&to_key(BLOCK_HEADER_PREFIX, &mut h.to_vec())),
@@ -139,6 +144,11 @@ impl ChainStore for ChainKVStore {
 		option_to_not_found(self.db.get_ser(&u64_to_key(HEADER_HEIGHT_PREFIX, height)))
 	}
 
+	fn save_header_height(&self, bh: &BlockHeader) -> Result<(), Error> {
+		self.db
+			.put_ser(&u64_to_key(HEADER_HEIGHT_PREFIX, bh.height), bh)
+	}
+
 	fn delete_header_by_height(&self, height: u64) -> Result<(), Error> {
 		self.db.delete(&u64_to_key(HEADER_HEIGHT_PREFIX, height))
 	}

chain/src/sumtree.rs

@@ -17,23 +17,31 @@
 
 use std::fs;
 use std::collections::HashMap;
-use std::path::Path;
+use std::fs::File;
+use std::ops::Deref;
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
-use core::core::{Block, SumCommit, Input, Output, OutputIdentifier, TxKernel, OutputFeatures};
-use core::core::pmmr::{HashSum, NoSum, Summable, PMMR};
+use util::{secp, static_secp_instance};
+use util::secp::pedersen::{RangeProof, Commitment};
+
+use core::consensus::reward;
+use core::core::{Block, BlockHeader, SumCommit, Input, Output, OutputIdentifier, OutputFeatures, TxKernel};
+use core::core::pmmr::{self, HashSum, NoSum, Summable, PMMR};
 use core::core::hash::Hashed;
+use core::ser::{self, Readable};
 use grin_store;
-use grin_store::sumtree::PMMRBackend;
+use grin_store::sumtree::{PMMRBackend, AppendOnlyFile};
 use types::ChainStore;
 use types::Error;
-use util::LOGGER;
-use util::secp::pedersen::{RangeProof, Commitment};
+use util::{LOGGER, zip};
 
 const SUMTREES_SUBDIR: &'static str = "sumtrees";
 const UTXO_SUBDIR: &'static str = "utxo";
 const RANGE_PROOF_SUBDIR: &'static str = "rangeproof";
 const KERNEL_SUBDIR: &'static str = "kernel";
+const KERNEL_FILE: &'static str = "kernel_full_data.bin";
+const SUMTREES_ZIP: &'static str = "sumtrees_snapshot.zip";
 
 struct PMMRHandle<T>
 where
@@ -68,10 +76,14 @@ where
 /// guaranteed to indicate whether an output is spent or not. The index
 /// may have commitments that have already been spent, even with
 /// pruning enabled.
+///
+/// In addition of the sumtrees, this maintains the full list of kernel
+/// data so it can be easily packaged for sync or validation.
 pub struct SumTrees {
 	output_pmmr_h: PMMRHandle<SumCommit>,
 	rproof_pmmr_h: PMMRHandle<NoSum<RangeProof>>,
 	kernel_pmmr_h: PMMRHandle<NoSum<TxKernel>>,
+	kernel_file: AppendOnlyFile,
 
 	// chain store used as index of commitments to MMR positions
 	commit_index: Arc<ChainStore>,
@@ -80,10 +92,16 @@ pub struct SumTrees {
 impl SumTrees {
 	/// Open an existing or new set of backends for the SumTrees
 	pub fn open(root_dir: String, commit_index: Arc<ChainStore>) -> Result<SumTrees, Error> {
+		let mut kernel_file_path: PathBuf = [&root_dir, SUMTREES_SUBDIR, KERNEL_SUBDIR].iter().collect();
+		fs::create_dir_all(kernel_file_path.clone())?;
+		kernel_file_path.push(KERNEL_FILE);
+		let kernel_file = AppendOnlyFile::open(kernel_file_path.to_str().unwrap().to_owned())?;
+
 		Ok(SumTrees {
 			output_pmmr_h: PMMRHandle::new(root_dir.clone(), UTXO_SUBDIR)?,
 			rproof_pmmr_h: PMMRHandle::new(root_dir.clone(), RANGE_PROOF_SUBDIR)?,
 			kernel_pmmr_h: PMMRHandle::new(root_dir.clone(), KERNEL_SUBDIR)?,
+			kernel_file: kernel_file,
 			commit_index: commit_index,
 		})
 	}
@@ -163,6 +181,11 @@ impl SumTrees {
 		kernel_pmmr.get_last_n_insertions(distance)
 	}
 
+	/// Output and kernel MMR indexes at the end of the provided block
+	pub fn indexes_at(&self, block: &Block) -> Result<(u64, u64), Error> {
+		indexes_at(block, self.commit_index.deref())
+	}
+
 	/// Get sum tree roots
 	pub fn roots(
 		&mut self,
@@ -193,10 +216,12 @@ where
 	let res: Result<T, Error>;
 	let rollback: bool;
 	{
-		debug!(LOGGER, "Starting new sumtree extension.");
 		let commit_index = trees.commit_index.clone();
+
+		debug!(LOGGER, "Starting new sumtree extension.");
 		let mut extension = Extension::new(trees, commit_index);
 		res = inner(&mut extension);
+
 		rollback = extension.rollback;
 		if res.is_ok() && !rollback {
 			extension.save_pos_index()?;
@@ -209,6 +234,7 @@ where
 			trees.output_pmmr_h.backend.discard();
 			trees.rproof_pmmr_h.backend.discard();
 			trees.kernel_pmmr_h.backend.discard();
+			trees.kernel_file.discard();
 			Err(e)
 		}
 		Ok(r) => {
@@ -217,11 +243,13 @@ where
 				trees.output_pmmr_h.backend.discard();
 				trees.rproof_pmmr_h.backend.discard();
 				trees.kernel_pmmr_h.backend.discard();
+				trees.kernel_file.discard();
 			} else {
 				debug!(LOGGER, "Committing sumtree extension.");
 				trees.output_pmmr_h.backend.sync()?;
 				trees.rproof_pmmr_h.backend.sync()?;
 				trees.kernel_pmmr_h.backend.sync()?;
+				trees.kernel_file.flush()?;
 				trees.output_pmmr_h.last_pos = sizes.0;
 				trees.rproof_pmmr_h.last_pos = sizes.1;
 				trees.kernel_pmmr_h.last_pos = sizes.2;
@@ -241,6 +269,7 @@ pub struct Extension<'a> {
 	rproof_pmmr: PMMR<'a, NoSum<RangeProof>, PMMRBackend<NoSum<RangeProof>>>,
 	kernel_pmmr: PMMR<'a, NoSum<TxKernel>, PMMRBackend<NoSum<TxKernel>>>,
 
+	kernel_file: &'a mut AppendOnlyFile,
 	commit_index: Arc<ChainStore>,
 	new_output_commits: HashMap<Commitment, u64>,
 	new_kernel_excesses: HashMap<Commitment, u64>,
@@ -249,7 +278,11 @@ pub struct Extension<'a> {
 
 impl<'a> Extension<'a> {
 	// constructor
-	fn new(trees: &'a mut SumTrees, commit_index: Arc<ChainStore>) -> Extension<'a> {
+	fn new(
+		trees: &'a mut SumTrees,
+		commit_index: Arc<ChainStore>,
+	) -> Extension<'a> {
+
 		Extension {
 			output_pmmr: PMMR::at(
 				&mut trees.output_pmmr_h.backend,
@@ -263,6 +296,7 @@ impl<'a> Extension<'a> {
 				&mut trees.kernel_pmmr_h.backend,
 				trees.kernel_pmmr_h.last_pos,
 			),
+			kernel_file: &mut trees.kernel_file,
 			commit_index: commit_index,
 			new_output_commits: HashMap::new(),
 			new_kernel_excesses: HashMap::new(),
@@ -407,15 +441,18 @@ impl<'a> Extension<'a> {
 				}
 			}
 		}
-		// push kernels in their MMR
+
+		// push kernels in their MMR and file
 		let pos = self.kernel_pmmr
 			.push(NoSum(kernel.clone()))
 			.map_err(&Error::SumTreeErr)?;
 		self.new_kernel_excesses.insert(kernel.excess, pos);
+		self.kernel_file.append(&mut ser::ser_vec(&kernel).unwrap());
+
 		Ok(())
 	}
 
-	/// Rewinds the MMRs to the provided position, given the last output and
+	/// Rewinds the MMRs to the provided block, using the last output and
 	/// last kernel of the block we want to rewind to.
 	pub fn rewind(&mut self, block: &Block) -> Result<(), Error> {
 		debug!(
@@ -425,22 +462,23 @@ impl<'a> Extension<'a> {
 			block.header.height,
 		);
 
-		let out_pos_rew = match block.outputs.last() {
-			Some(output) => self.get_output_pos(&output.commitment())
-				.map_err(|e| {
-					Error::StoreErr(e, format!("missing output pos for known block"))
-				})?,
-			None => 0,
-		};
+		// rewind each MMR
+		let (out_pos_rew, kern_pos_rew) = indexes_at(block, self.commit_index.deref())?;
+		self.rewind_pos(block.header.height, out_pos_rew, kern_pos_rew)?;
+		
+		// rewind the kernel file store, the position is the number of kernels
+		// multiplied by their size
+		// the number of kernels is the number of leaves in the MMR, which is the
+		// sum of the number of leaf nodes under each peak in the MMR
+		let pos: u64 = pmmr::peaks(kern_pos_rew).iter().map(|n| (1 << n) as u64).sum();
+		self.kernel_file.rewind(pos * (TxKernel::size() as u64));
 
-		let kern_pos_rew = match block.kernels.last() {
-			Some(kernel) => self.get_kernel_pos(&kernel.excess)
-				.map_err(|e| {
-					Error::StoreErr(e, format!("missing kernel pos for known block"))
-				})?,
-			None => 0,
-		};
+		Ok(())
+	}
 
+	/// Rewinds the MMRs to the provided positions, given the output and
+	/// kernel we want to rewind to.
+	pub fn rewind_pos(&mut self, height: u64, out_pos_rew: u64, kern_pos_rew: u64) -> Result<(), Error> {
 		debug!(
 			LOGGER,
 			"Rewind sumtrees to output pos: {}, kernel pos: {}",
@@ -448,8 +486,6 @@ impl<'a> Extension<'a> {
 			kern_pos_rew,
 		);
 
-		let height = block.header.height;
-
 		self.output_pmmr
 			.rewind(out_pos_rew, height as u32)
 			.map_err(&Error::SumTreeErr)?;
@@ -496,14 +532,67 @@ impl<'a> Extension<'a> {
 		)
 	}
 
+	/// Validate the current sumtree state against a block header
+	pub fn validate(&self, header: &BlockHeader) -> Result<(), Error> {
+		// validate all hashes and sums within the trees
+		if let Err(e) = self.output_pmmr.validate() {
+			return Err(Error::InvalidSumtree(e));
+		}
+		if let Err(e) = self.rproof_pmmr.validate() {
+			return Err(Error::InvalidSumtree(e));
+		}
+		if let Err(e) = self.kernel_pmmr.validate() {
+			return Err(Error::InvalidSumtree(e));
+		}
+
+		// validate the tree roots against the block header
+		let (utxo_root, rproof_root, kernel_root) = self.roots();
+		if utxo_root.hash != header.utxo_root || rproof_root.hash != header.range_proof_root
+			|| kernel_root.hash != header.kernel_root
+		{
+			return Err(Error::InvalidRoot);
+		}
+
+		// the real magicking: the sum of all kernel excess should equal the sum
+		// of all UTXO commitments, minus the total supply
+		let (kernel_sum, fees) = self.sum_kernels()?;
+		let utxo_sum = self.sum_utxos()?;
+		{
+			let secp = static_secp_instance();
+			let secp = secp.lock().unwrap();
+			let over_commit = secp.commit_value(header.height * reward(0) - fees / 2)?;
+			let adjusted_sum_utxo = secp.commit_sum(vec![utxo_sum], vec![over_commit])?;
+
+			if adjusted_sum_utxo != kernel_sum {
+				return Err(Error::InvalidSumtree("Differing UTXO commitment and kernel excess sums.".to_owned()));
+			}
+		}
+
+		Ok(())
+	}
+
+	/// Rebuild the index of MMR positions to the corresponding UTXO and kernel
+	/// by iterating over the whole MMR data. This is a costly operation
+	/// performed only when we receive a full new chain state.
+	pub fn rebuild_index(&self) -> Result<(), Error> {
+		for n in 1..self.output_pmmr.unpruned_size()+1 {
+			// non-pruned leaves only
+			if pmmr::bintree_postorder_height(n) == 0 {
+				if let Some(hs) = self.output_pmmr.get(n) {
+					self.commit_index.save_output_pos(&hs.sum.commit, n)?;
+				}
+			}
+		}
+		Ok(())
+	}
+
 	/// Force the rollback of this extension, no matter the result
 	pub fn force_rollback(&mut self) {
 		self.rollback = true;
 	}
 
 	/// Dumps the state of the 3 sum trees to stdout for debugging. Short
-	/// version
-	/// only prints the UTXO tree.
+	/// version only prints the UTXO tree.
 	pub fn dump(&self, short: bool) {
 		debug!(LOGGER, "-- outputs --");
 		self.output_pmmr.dump(short);
@@ -523,4 +612,109 @@ impl<'a> Extension<'a> {
 			self.kernel_pmmr.unpruned_size(),
 		)
 	}
+
+	/// Sums the excess of all our kernels, validating their signatures on the way
+	fn sum_kernels(&self) -> Result<(Commitment, u64), Error> {
+		// make sure we have the right count of kernels using the MMR, the storage
+		// file may have a few more
+		let mmr_sz = self.kernel_pmmr.unpruned_size();
+		let count: u64 = pmmr::peaks(mmr_sz).iter().map(|n| {
+			(1 << pmmr::bintree_postorder_height(*n)) as u64
+		}).sum();
+
+		let mut kernel_file = File::open(self.kernel_file.path())?;
+		let first: TxKernel = ser::deserialize(&mut kernel_file)?;
+		first.verify()?;
+		let mut sum_kernel = first.excess;
+		let mut fees = first.fee;
+
+		let secp = static_secp_instance();
+		let mut kern_count = 1;
+		loop {
+			match ser::deserialize::<TxKernel>(&mut kernel_file) {
+				Ok(kernel) => {
+					kernel.verify()?;
+					let secp = secp.lock().unwrap();
+					sum_kernel = secp.commit_sum(vec![sum_kernel, kernel.excess], vec![])?;
+					fees += kernel.fee;
+					kern_count += 1;
+					if kern_count == count {
+						break;
+					}
+				}
+				Err(_) => break,
+			}
+		}
+		debug!(LOGGER, "Validated and summed {} kernels", kern_count);
+		Ok((sum_kernel, fees))
+	}
+
+	/// Sums all our UTXO commitments
+	fn sum_utxos(&self) -> Result<Commitment, Error> {
+		let mut sum_utxo = None;
+		let mut utxo_count = 0;
+		let secp = static_secp_instance();
+		for n in 1..self.output_pmmr.unpruned_size()+1 {
+			if pmmr::bintree_postorder_height(n) == 0 {
+				if let Some(hs) = self.output_pmmr.get(n) {
+					if n == 1 {
+						sum_utxo = Some(hs.sum.commit);
+					} else {
+						let secp = secp.lock().unwrap();
+						sum_utxo = Some(secp.commit_sum(vec![sum_utxo.unwrap(), hs.sum.commit], vec![])?);
+					}
+					utxo_count += 1;
+				}
+			}
+		}
+		debug!(LOGGER, "Summed {} UTXOs", utxo_count);
+		Ok(sum_utxo.unwrap())
+	}
+}
+
+/// Output and kernel MMR indexes at the end of the provided block
+fn indexes_at(block: &Block, commit_index: &ChainStore) -> Result<(u64, u64), Error> {
+	let out_idx = match block.outputs.last() {
+		Some(output) => commit_index.get_output_pos(&output.commitment())
+			.map_err(|e| {
+				Error::StoreErr(e, format!("missing output pos for known block"))
+			})?,
+		None => 0,
+	};
+
+	let kern_idx = match block.kernels.last() {
+		Some(kernel) => commit_index.get_kernel_pos(&kernel.excess)
+			.map_err(|e| {
+				Error::StoreErr(e, format!("missing kernel pos for known block"))
+			})?,
+		None => 0,
+	};
+	Ok((out_idx, kern_idx))
+}
+
+/// Packages the sumtree data files into a zip and returns a Read to the
+/// resulting file
+pub fn zip_read(root_dir: String) -> Result<File, Error> {
+	let sumtrees_path = Path::new(&root_dir).join(SUMTREES_SUBDIR);
+	let zip_path = Path::new(&root_dir).join(SUMTREES_ZIP);
+
+	// create the zip archive
+	{
+		zip::compress(&sumtrees_path, &File::create(zip_path.clone())?)
+			.map_err(|ze| Error::Other(ze.to_string()))?;
+	}
+
+	// open it again to read it back
+	let zip_file = File::open(zip_path)?;
+	Ok(zip_file)
+}
+
+/// Extract the sumtree data from a zip file and writes the content into the
+/// sumtree storage dir
+pub fn zip_write(root_dir: String, sumtree_data: File) -> Result<(), Error> {
+	let sumtrees_path = Path::new(&root_dir).join(SUMTREES_SUBDIR);
+
+	fs::create_dir_all(sumtrees_path.clone())?;
+	zip::decompress(sumtree_data, &sumtrees_path)
+			.map_err(|ze| Error::Other(ze.to_string()))
 }

chain/src/types.rs

@@ -16,6 +16,7 @@
 
 use std::io;
 
+use util::secp;
 use util::secp::pedersen::Commitment;
 
 use grin_store as store;
@@ -76,6 +77,8 @@ pub enum Error {
 	OutputSpent,
 	/// Invalid block version, either a mistake or outdated software
 	InvalidBlockVersion(u16),
+	/// We've been provided a bad sumtree
+	InvalidSumtree(String),
 	/// Internal issue when trying to save or load data from store
 	StoreErr(grin_store::Error, String),
 	/// Error serializing or deserializing a type
@@ -105,10 +108,15 @@ impl From<io::Error> for Error {
 		Error::SumTreeErr(e.to_string())
 	}
 }
+impl From<secp::Error> for Error {
+	fn from(e: secp::Error) -> Error {
+		Error::SumTreeErr(format!("Sum validation error: {}", e.to_string()))
+	}
+}
 
 impl Error {
 	/// Whether the error is due to a block that was intrinsically wrong
-	pub fn is_bad_block(&self) -> bool {
+	pub fn is_bad_data(&self) -> bool {
 		// shorter to match on all the "not the block's fault" errors
 		match *self {
 			Error::Unfit(_) |
@@ -211,6 +219,9 @@ pub trait ChainStore: Send + Sync {
 	/// Gets a block header by hash
 	fn get_block(&self, h: &Hash) -> Result<Block, store::Error>;
 
+	/// Check whether we have a block without reading it
+	fn block_exists(&self, h: &Hash) -> Result<bool, store::Error>;
+
 	/// Gets a block header by hash
 	fn get_block_header(&self, h: &Hash) -> Result<BlockHeader, store::Error>;
 
@@ -238,6 +249,9 @@ pub trait ChainStore: Send + Sync {
 	/// Gets the block header at the provided height
 	fn get_header_by_height(&self, height: u64) -> Result<BlockHeader, store::Error>;
 
+	/// Save a header as associated with its height
+	fn save_header_height(&self, header: &BlockHeader) -> Result<(), store::Error>;
+
 	/// Delete the block header at the height
 	fn delete_header_by_height(&self, height: u64) -> Result<(), store::Error>;