[T4] Secondary proof of work difficulty adjustments (#1709)

* First pass at secondary proof of work difficulty adjustments
* Core and chain test fixes
* Next difficulty calc now needs a height. Scaling calculation fixes. Setting scaling on mined block.
* Change factor to u32 instead of u64.
* Cleanup structs used by next_difficulty
* Fix header size calc with u32 scaling

core/src/consensus.rs

@@ -51,6 +51,14 @@ pub const BLOCK_TIME_SEC: u64 = 60;
 /// set to nominal number of block in one day (1440 with 1-minute blocks)
 pub const COINBASE_MATURITY: u64 = 24 * 60 * 60 / BLOCK_TIME_SEC;
 
+/// Ratio the secondary proof of work should take over the primary, as a
+/// function of block height (time). Starts at 90% losing a percent
+/// approximately every week (10000 blocks). Represented as an integer
+/// between 0 and 100.
+pub fn secondary_pow_ratio(height: u64) -> u64 {
+	90u64.saturating_sub(height / 10000)
+}
+
 /// Cuckoo-cycle proof size (cycle length)
 pub const PROOFSIZE: usize = 42;
 
@@ -108,15 +116,15 @@ pub const HARD_FORK_INTERVAL: u64 = 250_000;
 /// 6 months interval scheduled hard forks for the first 2 years.
 pub fn valid_header_version(height: u64, version: u16) -> bool {
 	// uncomment below as we go from hard fork to hard fork
-	if height < HEADER_V2_HARD_FORK {
+	if height < HARD_FORK_INTERVAL {
 		version == 1
-	} else if height < HARD_FORK_INTERVAL {
+	/* } else if height < 2 * HARD_FORK_INTERVAL {
 		version == 2
-	} else if height < 2 * HARD_FORK_INTERVAL {
+	} else if height < 3 * HARD_FORK_INTERVAL {
 		version == 3
-	/* } else if height < 3 * HARD_FORK_INTERVAL {
-		version == 4 */
-	/* } else if height >= 4 * HARD_FORK_INTERVAL {
+	} else if height < 4 * HARD_FORK_INTERVAL {
+		version == 4 
+	} else if height >= 5 * HARD_FORK_INTERVAL {
 		version > 4 */
 	} else {
 		false
@@ -164,20 +172,62 @@ impl fmt::Display for Error {
 	}
 }
 
-/// Error when computing the next difficulty adjustment.
-#[derive(Debug, Clone, Fail)]
-pub struct TargetError(pub String);
+/// Minimal header information required for the Difficulty calculation to
+/// take place
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct HeaderInfo {
+	/// Timestamp of the header, 1 when not used (returned info)
+	pub timestamp: u64,
+	/// Network difficulty or next difficulty to use
+	pub difficulty: Difficulty,
+	/// Network secondary PoW factor or factor to use
+	pub secondary_scaling: u32,
+	/// Whether the header is a secondary proof of work
+	pub is_secondary: bool,
+}
 
-impl fmt::Display for TargetError {
-	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-		write!(f, "Error computing new difficulty: {}", self.0)
+impl HeaderInfo {
+	/// Default constructor
+	pub fn new(
+		timestamp: u64,
+		difficulty: Difficulty,
+		secondary_scaling: u32,
+		is_secondary: bool,
+	) -> HeaderInfo {
+		HeaderInfo {
+			timestamp,
+			difficulty,
+			secondary_scaling,
+			is_secondary,
+		}
+	}
+
+	/// Constructor from a timestamp and difficulty, setting a default secondary
+	/// PoW factor
+	pub fn from_ts_diff(timestamp: u64, difficulty: Difficulty) -> HeaderInfo {
+		HeaderInfo {
+			timestamp,
+			difficulty,
+			secondary_scaling: 1,
+			is_secondary: false,
+		}
+	}
+
+	/// Constructor from a difficulty and secondary factor, setting a default
+	/// timestamp
+	pub fn from_diff_scaling(difficulty: Difficulty, secondary_scaling: u32) -> HeaderInfo {
+		HeaderInfo {
+			timestamp: 1,
+			difficulty,
+			secondary_scaling,
+			is_secondary: false,
+		}
 	}
 }
 
 /// Computes the proof-of-work difficulty that the next block should comply
-/// with. Takes an iterator over past blocks, from latest (highest height) to
-/// oldest (lowest height). The iterator produces pairs of timestamp and
-/// difficulty for each block.
+/// with. Takes an iterator over past block headers information, from latest
+/// (highest height) to oldest (lowest height).
 ///
 /// The difficulty calculation is based on both Digishield and GravityWave
 /// family of difficulty computation, coming to something very close to Zcash.
@@ -185,9 +235,12 @@ impl fmt::Display for TargetError {
 /// DIFFICULTY_ADJUST_WINDOW blocks. The corresponding timespan is calculated
 /// by using the difference between the median timestamps at the beginning
 /// and the end of the window.
-pub fn next_difficulty<T>(cursor: T) -> Result<Difficulty, TargetError>
+///
+/// The secondary proof-of-work factor is calculated along the same lines, as
+/// an adjustment on the deviation against the ideal value.
+pub fn next_difficulty<T>(height: u64, cursor: T) -> HeaderInfo
 where
-	T: IntoIterator<Item = Result<(u64, Difficulty), TargetError>>,
+	T: IntoIterator<Item = HeaderInfo>,
 {
 	// Create vector of difficulty data running from earliest
 	// to latest, and pad with simulated pre-genesis data to allow earlier
@@ -195,27 +248,20 @@ where
 	// length will be DIFFICULTY_ADJUST_WINDOW+MEDIAN_TIME_WINDOW
 	let diff_data = global::difficulty_data_to_vector(cursor);
 
+	// First, get the ratio of secondary PoW vs primary
+	let sec_pow_scaling = secondary_pow_scaling(height, &diff_data);
+
 	// Obtain the median window for the earlier time period
 	// the first MEDIAN_TIME_WINDOW elements
-	let mut window_earliest: Vec<u64> = diff_data
-		.iter()
-		.take(MEDIAN_TIME_WINDOW as usize)
-		.map(|n| n.clone().unwrap().0)
-		.collect();
-	// pick median
-	window_earliest.sort();
-	let earliest_ts = window_earliest[MEDIAN_TIME_INDEX as usize];
+	let earliest_ts = time_window_median(&diff_data, 0, MEDIAN_TIME_WINDOW as usize);
 
 	// Obtain the median window for the latest time period
 	// i.e. the last MEDIAN_TIME_WINDOW elements
-	let mut window_latest: Vec<u64> = diff_data
-		.iter()
-		.skip(DIFFICULTY_ADJUST_WINDOW as usize)
-		.map(|n| n.clone().unwrap().0)
-		.collect();
-	// pick median
-	window_latest.sort();
-	let latest_ts = window_latest[MEDIAN_TIME_INDEX as usize];
+	let latest_ts = time_window_median(
+		&diff_data,
+		DIFFICULTY_ADJUST_WINDOW as usize,
+		MEDIAN_TIME_WINDOW as usize,
+	);
 
 	// median time delta
 	let ts_delta = latest_ts - earliest_ts;
@@ -224,7 +270,7 @@ where
 	let diff_sum = diff_data
 		.iter()
 		.skip(MEDIAN_TIME_WINDOW as usize)
-		.fold(0, |sum, d| sum + d.clone().unwrap().1.to_num());
+		.fold(0, |sum, d| sum + d.difficulty.to_num());
 
 	// Apply dampening except when difficulty is near 1
 	let ts_damp = if diff_sum < DAMP_FACTOR * DIFFICULTY_ADJUST_WINDOW {
@@ -242,9 +288,49 @@ where
 		ts_damp
 	};
 
-	let difficulty = diff_sum * BLOCK_TIME_SEC / adj_ts;
+	let difficulty = max(diff_sum * BLOCK_TIME_SEC / adj_ts, 1);
 
-	Ok(Difficulty::from_num(max(difficulty, 1)))
+	HeaderInfo::from_diff_scaling(Difficulty::from_num(difficulty), sec_pow_scaling)
+}
+
+/// Factor by which the secondary proof of work difficulty will be adjusted
+fn secondary_pow_scaling(height: u64, diff_data: &Vec<HeaderInfo>) -> u32 {
+	// median of past scaling factors, scaling is 1 if none found
+	let mut scalings = diff_data
+		.iter()
+		.map(|n| n.secondary_scaling)
+		.collect::<Vec<_>>();
+	if scalings.len() == 0 {
+		return 1;
+	}
+	scalings.sort();
+	let scaling_median = scalings[scalings.len() / 2] as u64;
+	let secondary_count = diff_data.iter().filter(|n| n.is_secondary).count() as u64;
+
+	// what's the ideal ratio at the current height
+	let ratio = secondary_pow_ratio(height);
+
+	// adjust the past median based on ideal ratio vs actual ratio
+	let scaling = scaling_median * secondary_count * 100 / ratio / diff_data.len() as u64;
+	if scaling == 0 {
+		1
+	} else {
+		scaling as u32
+	}
+}
+
+/// Median timestamp within the time window starting at `from` with the
+/// provided `length`.
+fn time_window_median(diff_data: &Vec<HeaderInfo>, from: usize, length: usize) -> u64 {
+	let mut window_latest: Vec<u64> = diff_data
+		.iter()
+		.skip(from)
+		.take(length)
+		.map(|n| n.timestamp)
+		.collect();
+	// pick median
+	window_latest.sort();
+	window_latest[MEDIAN_TIME_INDEX as usize]
 }
 
 /// Consensus rule that collections of items are sorted lexicographically.
@@ -252,6 +338,3 @@ pub trait VerifySortOrder<T> {
 	/// Verify a collection of items is sorted as required.
 	fn verify_sort_order(&self) -> Result<(), Error>;
 }
-
-/// Height for the v2 headers hard fork, with extended proof of work in header
-pub const HEADER_V2_HARD_FORK: u64 = 95_000;

core/src/core/block.rs

@@ -139,7 +139,7 @@ pub struct BlockHeader {
 }
 
 /// Serialized size of fixed part of a BlockHeader, i.e. without pow
-fn fixed_size_of_serialized_header(version: u16) -> usize {
+fn fixed_size_of_serialized_header(_version: u16) -> usize {
 	let mut size: usize = 0;
 	size += mem::size_of::<u16>(); // version
 	size += mem::size_of::<u64>(); // height
@@ -152,9 +152,7 @@ fn fixed_size_of_serialized_header(version: u16) -> usize {
 	size += mem::size_of::<u64>(); // output_mmr_size
 	size += mem::size_of::<u64>(); // kernel_mmr_size
 	size += mem::size_of::<Difficulty>(); // total_difficulty
-	if version >= 2 {
-		size += mem::size_of::<u64>(); // scaling_difficulty
-	}
+	size += mem::size_of::<u32>(); // scaling_difficulty
 	size += mem::size_of::<u64>(); // nonce
 	size
 }
@@ -208,19 +206,12 @@ impl Readable for BlockHeader {
 		let (version, height) = ser_multiread!(reader, read_u16, read_u64);
 		let previous = Hash::read(reader)?;
 		let timestamp = reader.read_i64()?;
-		let mut total_difficulty = None;
-		if version == 1 {
-			total_difficulty = Some(Difficulty::read(reader)?);
-		}
 		let output_root = Hash::read(reader)?;
 		let range_proof_root = Hash::read(reader)?;
 		let kernel_root = Hash::read(reader)?;
 		let total_kernel_offset = BlindingFactor::read(reader)?;
 		let (output_mmr_size, kernel_mmr_size) = ser_multiread!(reader, read_u64, read_u64);
-		let mut pow = ProofOfWork::read(version, reader)?;
-		if version == 1 {
-			pow.total_difficulty = total_difficulty.unwrap();
-		}
+		let pow = ProofOfWork::read(version, reader)?;
 
 		if timestamp > MAX_DATE.and_hms(0, 0, 0).timestamp()
 			|| timestamp < MIN_DATE.and_hms(0, 0, 0).timestamp()
@@ -254,10 +245,6 @@ impl BlockHeader {
 			[write_fixed_bytes, &self.previous],
 			[write_i64, self.timestamp.timestamp()]
 		);
-		if self.version == 1 {
-			// written as part of the ProofOfWork in later versions
-			writer.write_u64(self.pow.total_difficulty.to_num())?;
-		}
 		ser_multiwrite!(
 			writer,
 			[write_fixed_bytes, &self.output_root],
@@ -501,18 +488,11 @@ impl Block {
 		let now = Utc::now().timestamp();
 		let timestamp = DateTime::<Utc>::from_utc(NaiveDateTime::from_timestamp(now, 0), Utc);
 
-		let version = if prev.height + 1 < consensus::HEADER_V2_HARD_FORK {
-			1
-		} else {
-			2
-		};
-
 		// Now build the block with all the above information.
 		// Note: We have not validated the block here.
 		// Caller must validate the block as necessary.
 		Block {
 			header: BlockHeader {
-				version,
 				height: prev.height + 1,
 				timestamp,
 				previous: prev.hash(),

core/src/global.rs

@@ -16,13 +16,14 @@
 //! having to pass them all over the place, but aren't consensus values.
 //! should be used sparingly.
 
-use consensus::TargetError;
+use consensus::HeaderInfo;
 use consensus::{
 	BLOCK_TIME_SEC, COINBASE_MATURITY, CUT_THROUGH_HORIZON, DEFAULT_MIN_SIZESHIFT,
 	DIFFICULTY_ADJUST_WINDOW, EASINESS, INITIAL_DIFFICULTY, MEDIAN_TIME_WINDOW, PROOFSIZE,
 	REFERENCE_SIZESHIFT,
 };
-use pow::{self, CuckatooContext, Difficulty, EdgeType, PoWContext};
+use pow::{self, CuckatooContext, EdgeType, PoWContext};
+
 /// An enum collecting sets of parameters used throughout the
 /// code wherever mining is needed. This should allow for
 /// different sets of parameters for different purposes,
@@ -260,14 +261,13 @@ pub fn get_genesis_nonce() -> u64 {
 /// vector and pads if needed (which will) only be needed for the first few
 /// blocks after genesis
 
-pub fn difficulty_data_to_vector<T>(cursor: T) -> Vec<Result<(u64, Difficulty), TargetError>>
+pub fn difficulty_data_to_vector<T>(cursor: T) -> Vec<HeaderInfo>
 where
-	T: IntoIterator<Item = Result<(u64, Difficulty), TargetError>>,
+	T: IntoIterator<Item = HeaderInfo>,
 {
 	// Convert iterator to vector, so we can append to it if necessary
 	let needed_block_count = (MEDIAN_TIME_WINDOW + DIFFICULTY_ADJUST_WINDOW) as usize;
-	let mut last_n: Vec<Result<(u64, Difficulty), TargetError>> =
-		cursor.into_iter().take(needed_block_count).collect();
+	let mut last_n: Vec<HeaderInfo> = cursor.into_iter().take(needed_block_count).collect();
 
 	// Sort blocks from earliest to latest (to keep conceptually easier)
 	last_n.reverse();
@@ -277,16 +277,17 @@ where
 	let block_count_difference = needed_block_count - last_n.len();
 	if block_count_difference > 0 {
 		// Collect any real data we have
-		let mut live_intervals: Vec<(u64, Difficulty)> = last_n
+		let mut live_intervals: Vec<HeaderInfo> = last_n
 			.iter()
-			.map(|b| (b.clone().unwrap().0, b.clone().unwrap().1))
+			.map(|b| HeaderInfo::from_ts_diff(b.timestamp, b.difficulty))
 			.collect();
 		for i in (1..live_intervals.len()).rev() {
 			// prevents issues with very fast automated test chains
-			if live_intervals[i - 1].0 > live_intervals[i].0 {
-				live_intervals[i].0 = 0;
+			if live_intervals[i - 1].timestamp > live_intervals[i].timestamp {
+				live_intervals[i].timestamp = 0;
 			} else {
-				live_intervals[i].0 = live_intervals[i].0 - live_intervals[i - 1].0;
+				live_intervals[i].timestamp =
+					live_intervals[i].timestamp - live_intervals[i - 1].timestamp;
 			}
 		}
 		// Remove genesis "interval"
@@ -294,16 +295,16 @@ where
 			live_intervals.remove(0);
 		} else {
 			//if it's just genesis, adjust the interval
-			live_intervals[0].0 = BLOCK_TIME_SEC;
+			live_intervals[0].timestamp = BLOCK_TIME_SEC;
 		}
 		let mut interval_index = live_intervals.len() - 1;
-		let mut last_ts = last_n.first().as_ref().unwrap().as_ref().unwrap().0;
-		let last_diff = live_intervals[live_intervals.len() - 1].1;
+		let mut last_ts = last_n.first().unwrap().timestamp;
+		let last_diff = live_intervals[live_intervals.len() - 1].difficulty;
 		// fill in simulated blocks with values from the previous real block
 
 		for _ in 0..block_count_difference {
-			last_ts = last_ts.saturating_sub(live_intervals[live_intervals.len() - 1].0);
-			last_n.insert(0, Ok((last_ts, last_diff.clone())));
+			last_ts = last_ts.saturating_sub(live_intervals[live_intervals.len() - 1].timestamp);
+			last_n.insert(0, HeaderInfo::from_ts_diff(last_ts, last_diff.clone()));
 			interval_index = match interval_index {
 				0 => live_intervals.len() - 1,
 				_ => interval_index - 1,

core/src/pow/lean.rs

@@ -88,8 +88,6 @@ impl Lean {
 #[cfg(test)]
 mod test {
 	use super::*;
-	use pow::common;
-	use pow::cuckatoo::*;
 	use pow::types::PoWContext;
 
 	#[test]

core/src/pow/types.rs

@@ -84,7 +84,7 @@ impl Difficulty {
 	/// Computes the difficulty from a hash. Divides the maximum target by the
 	/// provided hash and applies the Cuckoo sizeshift adjustment factor (see
 	/// https://lists.launchpad.net/mimblewimble/msg00494.html).
-	pub fn from_proof_adjusted(proof: &Proof) -> Difficulty {
+	fn from_proof_adjusted(proof: &Proof) -> Difficulty {
 		// Adjust the difficulty based on a 2^(N-M)*(N-1) factor, with M being
 		// the minimum sizeshift and N the provided sizeshift
 		let shift = proof.cuckoo_sizeshift;
@@ -96,9 +96,9 @@ impl Difficulty {
 	/// Same as `from_proof_adjusted` but instead of an adjustment based on
 	/// cycle size, scales based on a provided factor. Used by dual PoW system
 	/// to scale one PoW against the other.
-	pub fn from_proof_scaled(proof: &Proof, scaling: u64) -> Difficulty {
+	fn from_proof_scaled(proof: &Proof, scaling: u32) -> Difficulty {
 		// Scaling between 2 proof of work algos
-		Difficulty::from_num(proof.raw_difficulty() * scaling)
+		Difficulty::from_num(proof.raw_difficulty() * scaling as u64)
 	}
 
 	/// Converts the difficulty into a u64
@@ -219,7 +219,7 @@ pub struct ProofOfWork {
 	/// Total accumulated difficulty since genesis block
 	pub total_difficulty: Difficulty,
 	/// Difficulty scaling factor between the different proofs of work
-	pub scaling_difficulty: u64,
+	pub scaling_difficulty: u32,
 	/// Nonce increment used to mine this block.
 	pub nonce: u64,
 	/// Proof of work data.
@@ -240,13 +240,9 @@ impl Default for ProofOfWork {
 
 impl ProofOfWork {
 	/// Read implementation, can't define as trait impl as we need a version
-	pub fn read(ver: u16, reader: &mut Reader) -> Result<ProofOfWork, ser::Error> {
-		let (total_difficulty, scaling_difficulty) = if ver == 1 {
-			// read earlier in the header on older versions
-			(Difficulty::one(), 1)
-		} else {
-			(Difficulty::read(reader)?, reader.read_u64()?)
-		};
+	pub fn read(_ver: u16, reader: &mut Reader) -> Result<ProofOfWork, ser::Error> {
+		let total_difficulty = Difficulty::read(reader)?;
+		let scaling_difficulty = reader.read_u32()?;
 		let nonce = reader.read_u64()?;
 		let proof = Proof::read(reader)?;
 		Ok(ProofOfWork {
@@ -269,14 +265,12 @@ impl ProofOfWork {
 	}
 
 	/// Write the pre-hash portion of the header
-	pub fn write_pre_pow<W: Writer>(&self, ver: u16, writer: &mut W) -> Result<(), ser::Error> {
-		if ver > 1 {
-			ser_multiwrite!(
-				writer,
-				[write_u64, self.total_difficulty.to_num()],
-				[write_u64, self.scaling_difficulty]
-			);
-		}
+	pub fn write_pre_pow<W: Writer>(&self, _ver: u16, writer: &mut W) -> Result<(), ser::Error> {
+		ser_multiwrite!(
+			writer,
+			[write_u64, self.total_difficulty.to_num()],
+			[write_u32, self.scaling_difficulty]
+		);
 		Ok(())
 	}
 
@@ -295,6 +289,21 @@ impl ProofOfWork {
 	pub fn cuckoo_sizeshift(&self) -> u8 {
 		self.proof.cuckoo_sizeshift
 	}
+
+	/// Whether this proof of work is for the primary algorithm (as opposed
+	/// to secondary). Only depends on the size shift at this time.
+	pub fn is_primary(&self) -> bool {
+		// 2 conditions are redundant right now but not necessarily in
+		// the future
+		self.proof.cuckoo_sizeshift != SECOND_POW_SIZESHIFT
+			&& self.proof.cuckoo_sizeshift >= global::min_sizeshift()
+	}
+
+	/// Whether this proof of work is for the secondary algorithm (as opposed
+	/// to primary). Only depends on the size shift at this time.
+	pub fn is_secondary(&self) -> bool {
+		self.proof.cuckoo_sizeshift == SECOND_POW_SIZESHIFT
+	}
 }
 
 /// A Cuckoo Cycle proof of work, consisting of the shift to get the graph