[WIP] Abridged sync (#440)

* Util to zip and unzip directories
* First pass at sumtree request/response. Add message types, implement the exchange in the protocol, zip up the sumtree directory and stream the file over, with necessary adapter hooks.
* Implement the sumtree archive receive logicGets the sumtree archive data stream from the network and write it to a file. Unzip the file, place it at the right spot and reconstruct the sumtree data structure, rewinding where to the right spot.
* Sumtree hash structure validation
* Simplify sumtree backend buffering logic. The backend for a sumtree has to implement some in-memory buffering logic to provide a commit/rollback interface. The backend itself is an aggregate of 3 underlying storages (an append only file, a remove log and a skip list). The buffering was previously implemented both by the backend and some of the underlying storages. Now pushing back all buffering logic to the storages to keep the backend simpler.
* Add kernel append only store file to sumtrees. The chain sumtrees structure now also saves all kernels to a dedicated file. As that storage is implemented by the append only file wrapper, it's also rewind-aware.
* Full state validation. Checks that:
    - MMRs are sane (hash and sum each node)
    - Tree roots match the corresponding header
    - Kernel signatures are valid
    - Sum of all kernel excesses equals the sum of UTXO commitments
minus the supply
* Fast sync handoff to body sync. Once the fast-sync state is fully setup, get bacj in body sync
mode to get the full bodies of the last blocks we're missing.
* First fully working fast sync
* Facility in p2p conn to deal with attachments (raw binary after message).
* Re-introduced sumtree send and receive message handling using the above.
* Fixed test and finished updating all required db state after sumtree validation.
* Massaged a little bit the pipeline orphan check to still work after the new sumtrees have been setup.
* Various cleanup. Consolidated fast sync and full sync into a single function as they're very similar. Proper conditions to trigger a sumtree request and some checks on receiving it.

core/src/core/pmmr.rs

@@ -47,7 +47,7 @@ use util::LOGGER;
 /// the tree can sum over
 pub trait Summable {
 	/// The type of the sum
-	type Sum: Clone + ops::Add<Output = Self::Sum> + Readable + Writeable;
+	type Sum: Clone + ops::Add<Output = Self::Sum> + Readable + Writeable + PartialEq;
 
 	/// Obtain the sum of the element
 	fn sum(&self) -> Self::Sum;
@@ -80,6 +80,12 @@ impl Writeable for NullSum {
 	}
 }
 
+impl PartialEq for NullSum {
+	fn eq(&self, _other: &NullSum) -> bool {
+		true
+	}
+}
+
 /// Wrapper for a type that allows it to be inserted in a tree without summing
 #[derive(Clone, Debug)]
 pub struct NoSum<T>(pub T);
@@ -103,7 +109,7 @@ where
 
 /// A utility type to handle (Hash, Sum) pairs more conveniently. The addition
 /// of two HashSums is the (Hash(h1|h2), h1 + h2) HashSum.
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, Eq)]
 pub struct HashSum<T>
 where
 	T: Summable,
@@ -130,6 +136,15 @@ where
 	}
 }
 
+impl<T> PartialEq for HashSum<T>
+where
+	T: Summable,
+{
+	fn eq(&self, other: &HashSum<T>) -> bool {
+		self.hash == other.hash && self.sum == other.sum
+	}
+}
+
 impl<T> Readable for HashSum<T>
 where
 	T: Summable,
@@ -382,6 +397,32 @@ where
 		return_vec
 	}
 
+	/// Walks all unpruned nodes in the MMR and revalidate all parent hashes
+	/// and sums.
+	pub fn validate(&self) -> Result<(), String> {
+		// iterate on all parent nodes
+		for n in 1..(self.last_pos + 1) {
+			if bintree_postorder_height(n) > 0 {
+				if let Some(hs) = self.get(n) {
+					// take the left and right children, if they exist
+					let left_pos = bintree_move_down_left(n).unwrap();
+					let right_pos = bintree_jump_right_sibling(left_pos);
+
+					if let Some(left_child_hs) = self.get(left_pos) {
+						if let Some(right_child_hs) = self.get(right_pos) {
+							// sum and compare
+							if left_child_hs + right_child_hs != hs {
+								return Err(format!("Invalid MMR, hashsum of parent at {} does \
+																	 not match children.", n));
+							}
+						}
+					}
+				}
+			}
+		}
+		Ok(())
+	}
+
 	/// Total size of the tree, including intermediary nodes an ignoring any
 	/// pruning.
 	pub fn unpruned_size(&self) -> u64 {
@@ -583,7 +624,7 @@ impl PruneList {
 /// node's position. Starts with the top peak, which is always on the left
 /// side of the range, and navigates toward lower siblings toward the right
 /// of the range.
-fn peaks(num: u64) -> Vec<u64> {
+pub fn peaks(num: u64) -> Vec<u64> {
 	// detecting an invalid mountain range, when siblings exist but no parent
 	// exists
 	if bintree_postorder_height(num + 1) > bintree_postorder_height(num) {

core/src/core/transaction.rs

@@ -167,6 +167,13 @@ impl TxKernel {
 		}
 		Ok(())
 	}
+
+	/// Size in bytes of a kernel, necessary for binary storage
+	pub fn size() -> usize {
+		17 + // features plus fee and lock_height
+			secp::constants::PEDERSEN_COMMITMENT_SIZE +
+			secp::constants::AGG_SIGNATURE_SIZE
+	}
 }
 
 /// A transaction
@@ -770,7 +777,7 @@ impl Readable for OutputIdentifier {
 }
 
 /// Wrapper to Output commitments to provide the Summable trait.
-#[derive(Serialize, Deserialize, Debug, Clone)]
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
 pub struct SumCommit {
 	/// Output features (coinbase vs. regular transaction output)
 	/// We need to include this when hashing to ensure coinbase maturity can be enforced.

core/src/global.rs

@@ -26,7 +26,7 @@ use consensus::PROOFSIZE;
 use consensus::DEFAULT_SIZESHIFT;
 use consensus::COINBASE_MATURITY;
 use consensus::{MEDIAN_TIME_WINDOW, INITIAL_DIFFICULTY, 
-	BLOCK_TIME_SEC, DIFFICULTY_ADJUST_WINDOW};
+	BLOCK_TIME_SEC, DIFFICULTY_ADJUST_WINDOW, CUT_THROUGH_HORIZON};
 use core::target::Difficulty;
 use consensus::TargetError;
 
@@ -51,6 +51,9 @@ pub const AUTOMATED_TESTING_COINBASE_MATURITY: u64 = 3;
 /// User testing coinbase maturity
 pub const USER_TESTING_COINBASE_MATURITY: u64 = 3;
 
+/// Testing cut through horizon in blocks
+pub const TESTING_CUT_THROUGH_HORIZON: u32 = 20;
+
 /// Testing initial block difficulty
 pub const TESTING_INITIAL_DIFFICULTY: u64 = 1;
 
@@ -163,6 +166,18 @@ pub fn initial_block_difficulty() -> u64 {
 	}
 }
 
+/// Horizon at which we can cut-through and do full local pruning
+pub fn cut_through_horizon() -> u32 {
+	let param_ref = CHAIN_TYPE.read().unwrap();
+	match *param_ref {
+		ChainTypes::AutomatedTesting => TESTING_CUT_THROUGH_HORIZON,
+		ChainTypes::UserTesting => TESTING_CUT_THROUGH_HORIZON,
+		ChainTypes::Testnet1 => CUT_THROUGH_HORIZON,
+		ChainTypes::Testnet2 => CUT_THROUGH_HORIZON,
+		ChainTypes::Mainnet => CUT_THROUGH_HORIZON,
+	}
+}
+
 /// Are we in automated testing mode?
 pub fn is_automated_testing_mode() -> bool {
 	let param_ref = CHAIN_TYPE.read().unwrap();