refactor the state_sync to handle the long fork (#1902)

* split horizon into two explicit values for cut through and txhashset request

* let node which has 2-7 days of history be able to handle forks larger than 2 days

* add test simulate_long_fork

* add pause/resume feature on p2p for tests

* refactor the state_sync

* ignore the test case simulate_long_fork for normal Travis-CI

* refactor function check_txhashset_needed to be shared with body_sync

* fix: state TxHashsetDone should allow header sync

chain/src/chain.rs

@@ -25,7 +25,7 @@ use util::RwLock;
 use lmdb;
 use lru_cache::LruCache;
 
-use core::core::hash::{Hash, Hashed};
+use core::core::hash::{Hash, Hashed, ZERO_HASH};
 use core::core::merkle_proof::MerkleProof;
 use core::core::verifier_cache::VerifierCache;
 use core::core::{Block, BlockHeader, BlockSums, Output, OutputIdentifier, Transaction, TxKernel};
@@ -700,6 +700,73 @@ impl Chain {
 		Ok(())
 	}
 
+	/// Check chain status whether a txhashset downloading is needed
+	pub fn check_txhashset_needed(&self, caller: String, hashes: &mut Option<Vec<Hash>>) -> bool {
+		let horizon = global::cut_through_horizon() as u64;
+		let body_head = self.head().unwrap();
+		let header_head = self.header_head().unwrap();
+		let sync_head = self.get_sync_head().unwrap();
+
+		debug!(
+			"{}: body_head - {}, {}, header_head - {}, {}, sync_head - {}, {}",
+			caller,
+			body_head.last_block_h,
+			body_head.height,
+			header_head.last_block_h,
+			header_head.height,
+			sync_head.last_block_h,
+			sync_head.height,
+		);
+
+		if body_head.total_difficulty >= header_head.total_difficulty {
+			debug!(
+				"{}: no need. header_head.total_difficulty: {} <= body_head.total_difficulty: {}",
+				caller, header_head.total_difficulty, body_head.total_difficulty,
+			);
+			return false;
+		}
+
+		let mut oldest_height = 0;
+		let mut oldest_hash = ZERO_HASH;
+
+		let mut current = self.get_block_header(&header_head.last_block_h);
+		if current.is_err() {
+			error!(
+				"{}: header_head not found in chain db: {} at {}",
+				caller, header_head.last_block_h, header_head.height,
+			);
+		}
+
+		while let Ok(header) = current {
+			// break out of the while loop when we find a header common
+			// between the header chain and the current body chain
+			if let Ok(_) = self.is_on_current_chain(&header) {
+				break;
+			}
+
+			oldest_height = header.height;
+			oldest_hash = header.hash();
+			if let Some(hs) = hashes {
+				hs.push(oldest_hash);
+			}
+			current = self.get_previous_header(&header);
+		}
+
+		if oldest_height < header_head.height.saturating_sub(horizon) {
+			if oldest_height > 0 {
+				debug!(
+					"{}: oldest block which is not on local chain: {} at {}",
+					caller, oldest_hash, oldest_height,
+				);
+				return true;
+			} else {
+				error!("{}: something is wrong! oldest_height is 0", caller);
+				return false;
+			};
+		}
+		return false;
+	}
+
 	/// Writes a reading view on a txhashset state that's been provided to us.
 	/// If we're willing to accept that new state, the data stream will be
 	/// read as a zip file, unzipped and the resulting state files should be
@@ -712,13 +779,11 @@ impl Chain {
 	) -> Result<(), Error> {
 		status.on_setup();
 
-		// Initial check based on relative heights of current head and header_head.
-		{
-			let head = self.head().unwrap();
-			let header_head = self.header_head().unwrap();
-			if header_head.height - head.height < global::cut_through_horizon() as u64 {
-				return Err(ErrorKind::InvalidTxHashSet("not needed".to_owned()).into());
-			}
+		// Initial check whether this txhashset is needed or not
+		let mut hashes: Option<Vec<Hash>> = None;
+		if !self.check_txhashset_needed("txhashset_write".to_owned(), &mut hashes) {
+			warn!("txhashset_write: txhashset received but it's not needed! ignored.");
+			return Err(ErrorKind::InvalidTxHashSet("not needed".to_owned()).into());
 		}
 
 		let header = self.get_block_header(&h)?;
@@ -769,6 +834,9 @@ impl Chain {
 			batch.save_body_head(&tip)?;
 			batch.save_header_height(&header)?;
 			batch.build_by_height_index(&header, true)?;
+
+			// Reset the body tail to the body head after a txhashset write
+			batch.save_body_tail(&tip)?;
 		}
 
 		// Commit all the changes to the db.
@@ -819,12 +887,13 @@ impl Chain {
 
 		let horizon = global::cut_through_horizon() as u64;
 		let head = self.head()?;
+		let tail = self.tail()?;
 
 		let cutoff = head.height.saturating_sub(horizon);
 
 		debug!(
-			"compact_blocks_db: head height: {}, horizon: {}, cutoff: {}",
-			head.height, horizon, cutoff,
+			"compact_blocks_db: head height: {}, tail height: {}, horizon: {}, cutoff: {}",
+			head.height, tail.height, horizon, cutoff,
 		);
 
 		if cutoff == 0 {
@@ -859,8 +928,13 @@ impl Chain {
 				Err(e) => return Err(From::from(e)),
 			}
 		}
+		let tail = batch.get_header_by_height(head.height - horizon)?;
+		batch.save_body_tail(&Tip::from_header(&tail))?;
 		batch.commit()?;
-		debug!("compact_blocks_db: removed {} blocks.", count);
+		debug!(
+			"compact_blocks_db: removed {} blocks. tail height: {}",
+			count, tail.height
+		);
 		Ok(())
 	}
 
@@ -943,6 +1017,13 @@ impl Chain {
 			.map_err(|e| ErrorKind::StoreErr(e, "chain head".to_owned()).into())
 	}
 
+	/// Tail of the block chain in this node after compact (cross-block cut-through)
+	pub fn tail(&self) -> Result<Tip, Error> {
+		self.store
+			.tail()
+			.map_err(|e| ErrorKind::StoreErr(e, "chain tail".to_owned()).into())
+	}
+
 	/// Tip (head) of the header chain.
 	pub fn header_head(&self) -> Result<Tip, Error> {
 		self.store

chain/src/pipe.rs

@@ -199,6 +199,10 @@ pub fn process_block(b: &Block, ctx: &mut BlockContext) -> Result<Option<Tip>, E
 	// so we can maintain multiple (in progress) forks.
 	add_block(b, &ctx.batch)?;
 
+	if ctx.batch.tail().is_err() {
+		update_body_tail(&b.header, &ctx.batch)?;
+	}
+
 	// Update the chain head if total work is increased.
 	let res = update_head(b, ctx)?;
 	Ok(res)
@@ -552,6 +556,16 @@ fn add_block(b: &Block, batch: &store::Batch) -> Result<(), Error> {
 	Ok(())
 }
 
+/// Update the block chain tail so we can know the exact tail of full blocks in this node
+fn update_body_tail(bh: &BlockHeader, batch: &store::Batch) -> Result<(), Error> {
+	let tip = Tip::from_header(bh);
+	batch
+		.save_body_tail(&tip)
+		.map_err(|e| ErrorKind::StoreErr(e, "pipe save body tail".to_owned()))?;
+	debug!("body tail {} @ {}", bh.hash(), bh.height);
+	Ok(())
+}
+
 /// Officially adds the block header to our header chain.
 fn add_block_header(bh: &BlockHeader, batch: &store::Batch) -> Result<(), Error> {
 	batch

chain/src/store.rs

@@ -36,6 +36,7 @@ const STORE_SUBPATH: &'static str = "chain";
 const BLOCK_HEADER_PREFIX: u8 = 'h' as u8;
 const BLOCK_PREFIX: u8 = 'b' as u8;
 const HEAD_PREFIX: u8 = 'H' as u8;
+const TAIL_PREFIX: u8 = 'T' as u8;
 const HEADER_HEAD_PREFIX: u8 = 'I' as u8;
 const SYNC_HEAD_PREFIX: u8 = 's' as u8;
 const HEADER_HEIGHT_PREFIX: u8 = '8' as u8;
@@ -70,6 +71,10 @@ impl ChainStore {
 		option_to_not_found(self.db.get_ser(&vec![HEAD_PREFIX]), "HEAD")
 	}
 
+	pub fn tail(&self) -> Result<Tip, Error> {
+		option_to_not_found(self.db.get_ser(&vec![TAIL_PREFIX]), "TAIL")
+	}
+
 	/// Header of the block at the head of the block chain (not the same thing as header_head).
 	pub fn head_header(&self) -> Result<BlockHeader, Error> {
 		self.get_block_header(&self.head()?.last_block_h)
@@ -225,6 +230,10 @@ impl<'a> Batch<'a> {
 		option_to_not_found(self.db.get_ser(&vec![HEAD_PREFIX]), "HEAD")
 	}
 
+	pub fn tail(&self) -> Result<Tip, Error> {
+		option_to_not_found(self.db.get_ser(&vec![TAIL_PREFIX]), "TAIL")
+	}
+
 	/// Header of the block at the head of the block chain (not the same thing as header_head).
 	pub fn head_header(&self) -> Result<BlockHeader, Error> {
 		self.get_block_header(&self.head()?.last_block_h)
@@ -248,6 +257,10 @@ impl<'a> Batch<'a> {
 		self.db.put_ser(&vec![HEAD_PREFIX], t)
 	}
 
+	pub fn save_body_tail(&self, t: &Tip) -> Result<(), Error> {
+		self.db.put_ser(&vec![TAIL_PREFIX], t)
+	}
+
 	pub fn save_header_head(&self, t: &Tip) -> Result<(), Error> {
 		self.db.put_ser(&vec![HEADER_HEAD_PREFIX], t)
 	}