NRD rules and "recent" kernel pos index (#3302)

* variants for output_pos linked list entries (head/tail/middle/unique)
next and prev and Vec<u8> lmdb keys

* get_pos on enum

* break list and list entries out into separate enums

* track output features in the new output_pos index, so we can determine coinbase maturity

* push entry impl for none and unique

* some test coverage for output_pos_list

* commit

* wip - FooListEntry

* use instance of the index

* linked list of output_pos and commit_pos both now supported

* linked_list

* cleanup and rename

* rename

* peek_pos

* push some, peek some, pop some

* cleanup

* commit pos
cleanup

* split list and entry out into separate db prefixes

* cleanup and add placeholder for pop_back

* pop_pos_back (for popping off the back of the linked list)
test coverage for pop_pos_back

* wip

* placeholder for prune via a trait
pos must always increase in the index

* rewind kernel_pos_idx when calling rewind_single_block

* RewindableListIndex with rewind support.

* test coverage for rewindable list index

* test coverage for rewind back to 0

* rewind past end of list

* add tests for kernel_pos_idx with multiple commits

* commit

* cleanup

* hook NRD relative lock height validation into block processing and tx validation

* cleanup

* set local chain type for kernel_idx tests

* add test coverage for NRD rules in block processing

* NRD test coverage and cleanup

* NRD relative height 1 test

* test coverage for NRD kernels in block processing

* cleanup

* start of test coverage for txpool NRD kernel rules

* wip

* rework pool tests to use real chain (was mock chain) to better reflect reality (tx/block validation rules etc.)

* cleanup

* cleanup pruneable trait for kernel pos index

* add clear() to kernel_pos idx and test coverage

* hook kernel_pos rebuild into node startup, compaction and fast sync

* verify full NRD history on fast sync

* return early if nrd disabled

* fix header sync issue

pool/src/pool.rs

@@ -29,6 +29,7 @@ use grin_util as util;
 use std::cmp::Reverse;
 use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
+use util::secp::pedersen::Commitment;
 use util::static_secp_instance;
 
 pub struct Pool<B, V>
@@ -70,6 +71,19 @@ where
 			.map(|x| x.tx.clone())
 	}
 
+	/// Query the tx pool for an individual tx matching the given public excess.
+	/// Used for checking for duplicate NRD kernels in the txpool.
+	pub fn retrieve_tx_by_kernel_excess(&self, excess: Commitment) -> Option<Transaction> {
+		for x in &self.entries {
+			for k in x.tx.kernels() {
+				if k.excess() == excess {
+					return Some(x.tx.clone());
+				}
+			}
+		}
+		None
+	}
+
 	/// Query the tx pool for an individual tx matching the given kernel hash.
 	pub fn retrieve_tx_by_kernel_hash(&self, hash: Hash) -> Option<Transaction> {
 		for x in &self.entries {
@@ -197,7 +211,6 @@ where
 		// Validate aggregated tx (existing pool + new tx), ignoring tx weight limits.
 		// Validate against known chain state at the provided header.
 		self.validate_raw_tx(&agg_tx, header, Weighting::NoLimit)?;
-
 		// If we get here successfully then we can safely add the entry to the pool.
 		self.log_pool_add(&entry, header);
 		self.entries.push(entry);
@@ -425,6 +438,7 @@ where
 		tx_buckets.into_iter().flat_map(|x| x.raw_txs).collect()
 	}
 
+	/// TODO - This is kernel based. How does this interact with NRD?
 	pub fn find_matching_transactions(&self, kernels: &[TxKernel]) -> Vec<Transaction> {
 		// While the inputs outputs can be cut-through the kernel will stay intact
 		// In order to deaggregate tx we look for tx with the same kernel

pool/src/transaction_pool.rs

@@ -160,13 +160,18 @@ where
 		stem: bool,
 		header: &BlockHeader,
 	) -> Result<(), PoolError> {
-		// Quick check to deal with common case of seeing the *same* tx
-		// broadcast from multiple peers simultaneously.
-		if !stem && self.txpool.contains_tx(tx.hash()) {
+		// Quick check for duplicate txs.
+		// Our stempool is private and we do not want to reveal anything about the txs contained.
+		// If this is a stem tx and is already present in stempool then fluff by adding to txpool.
+		// Otherwise if already present in txpool return a "duplicate tx" error.
+		if stem && self.stempool.contains_tx(tx.hash()) {
+			return self.add_to_pool(src, tx, false, header);
+		} else if self.txpool.contains_tx(tx.hash()) {
 			return Err(PoolError::DuplicateTx);
 		}
 
 		// Check this tx is valid based on current header version.
+		// NRD kernels only valid post HF3 and if NRD feature enabled.
 		self.verify_kernel_variants(&tx, header)?;
 
 		// Do we have the capacity to accept this transaction?
@@ -195,20 +200,19 @@ where
 			tx,
 		};
 
-		// If not stem then we are fluff.
-		// If this is a stem tx then attempt to stem.
-		// Any problems during stem, fallback to fluff.
-		if !stem
-			|| self
-				.add_to_stempool(entry.clone(), header)
-				.and_then(|_| self.adapter.stem_tx_accepted(&entry))
-				.is_err()
-		{
-			self.add_to_txpool(entry.clone(), header)?;
-			self.add_to_reorg_cache(entry.clone());
-			self.adapter.tx_accepted(&entry);
+		// If this is a stem tx then attempt to add it to stempool.
+		// If the adapter fails to accept the new stem tx then fallback to fluff via txpool.
+		if stem {
+			self.add_to_stempool(entry.clone(), header)?;
+			if self.adapter.stem_tx_accepted(&entry).is_ok() {
+				return Ok(());
+			}
 		}
 
+		self.add_to_txpool(entry.clone(), header)?;
+		self.add_to_reorg_cache(entry.clone());
+		self.adapter.tx_accepted(&entry);
+
 		// Transaction passed all the checks but we have to make space for it
 		if evict {
 			self.evict_from_txpool();

pool/src/types.rs

@@ -227,6 +227,9 @@ pub enum PoolError {
 	/// NRD kernels are not valid if disabled locally via "feature flag".
 	#[fail(display = "NRD kernel not enabled")]
 	NRDKernelNotEnabled,
+	/// NRD kernels are not valid if relative_height rule not met.
+	#[fail(display = "NRD kernel relative height")]
+	NRDKernelRelativeHeight,
 	/// Other kinds of error (not yet pulled out into meaningful errors).
 	#[fail(display = "General pool error {}", _0)]
 	Other(String),
@@ -234,7 +237,10 @@ pub enum PoolError {
 
 impl From<transaction::Error> for PoolError {
 	fn from(e: transaction::Error) -> PoolError {
-		PoolError::InvalidTx(e)
+		match e {
+			transaction::Error::InvalidNRDRelativeHeight => PoolError::NRDKernelRelativeHeight,
+			e @ _ => PoolError::InvalidTx(e),
+		}
 	}
 }
 

pool/tests/common.rs

@@ -19,12 +19,12 @@ use self::chain::Chain;
 use self::core::consensus;
 use self::core::core::hash::Hash;
 use self::core::core::verifier_cache::{LruVerifierCache, VerifierCache};
-use self::core::core::{Block, BlockHeader, BlockSums, KernelFeatures, Transaction};
+use self::core::core::{Block, BlockHeader, BlockSums, KernelFeatures, Transaction, TxKernel};
 use self::core::genesis;
 use self::core::global;
 use self::core::libtx::{build, reward, ProofBuilder};
 use self::core::pow;
-use self::keychain::{ExtKeychain, ExtKeychainPath, Keychain};
+use self::keychain::{BlindingFactor, ExtKeychain, ExtKeychainPath, Keychain};
 use self::pool::types::*;
 use self::pool::TransactionPool;
 use self::util::RwLock;
@@ -127,9 +127,11 @@ impl BlockChain for ChainAdapter {
 	}
 
 	fn validate_tx(&self, tx: &Transaction) -> Result<(), pool::PoolError> {
-		self.chain
-			.validate_tx(tx)
-			.map_err(|_| PoolError::Other("failed to validate tx".into()))
+		self.chain.validate_tx(tx).map_err(|e| match e.kind() {
+			chain::ErrorKind::Transaction(txe) => txe.into(),
+			chain::ErrorKind::NRDRelativeHeight => PoolError::NRDKernelRelativeHeight,
+			_ => PoolError::Other("failed to validate tx".into()),
+		})
 	}
 
 	fn verify_coinbase_maturity(&self, tx: &Transaction) -> Result<(), PoolError> {
@@ -254,6 +256,38 @@ where
 	.unwrap()
 }
 
+pub fn test_transaction_with_kernel<K>(
+	keychain: &K,
+	input_values: Vec<u64>,
+	output_values: Vec<u64>,
+	kernel: TxKernel,
+	excess: BlindingFactor,
+) -> Transaction
+where
+	K: Keychain,
+{
+	let mut tx_elements = Vec::new();
+
+	for input_value in input_values {
+		let key_id = ExtKeychain::derive_key_id(1, input_value as u32, 0, 0, 0);
+		tx_elements.push(build::input(input_value, key_id));
+	}
+
+	for output_value in output_values {
+		let key_id = ExtKeychain::derive_key_id(1, output_value as u32, 0, 0, 0);
+		tx_elements.push(build::output(output_value, key_id));
+	}
+
+	build::transaction_with_kernel(
+		tx_elements,
+		kernel,
+		excess,
+		keychain,
+		&ProofBuilder::new(keychain),
+	)
+	.unwrap()
+}
+
 pub fn test_source() -> TxSource {
 	TxSource::Broadcast
 }

pool/tests/nrd_kernel_relative_height.rs

@@ -0,0 +1,265 @@
+// Copyright 2020 The Grin Developers
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod common;
+
+use self::core::consensus;
+use self::core::core::hash::Hashed;
+use self::core::core::verifier_cache::LruVerifierCache;
+use self::core::core::{HeaderVersion, KernelFeatures, NRDRelativeHeight, TxKernel};
+use self::core::global;
+use self::core::libtx::aggsig;
+use self::keychain::{BlindingFactor, ExtKeychain, Keychain};
+use self::pool::types::PoolError;
+use self::util::RwLock;
+use crate::common::*;
+use grin_core as core;
+use grin_keychain as keychain;
+use grin_pool as pool;
+use grin_util as util;
+use std::sync::Arc;
+
+#[test]
+fn test_nrd_kernel_relative_height() -> Result<(), PoolError> {
+	util::init_test_logger();
+	global::set_local_chain_type(global::ChainTypes::AutomatedTesting);
+	global::set_local_nrd_enabled(true);
+
+	let keychain: ExtKeychain = Keychain::from_random_seed(false).unwrap();
+
+	let db_root = "target/.nrd_kernel_relative_height";
+	clean_output_dir(db_root.into());
+
+	let genesis = genesis_block(&keychain);
+	let chain = Arc::new(init_chain(db_root, genesis));
+	let verifier_cache = Arc::new(RwLock::new(LruVerifierCache::new()));
+
+	// Initialize a new pool with our chain adapter.
+	let mut pool = init_transaction_pool(
+		Arc::new(ChainAdapter {
+			chain: chain.clone(),
+		}),
+		verifier_cache,
+	);
+
+	add_some_blocks(&chain, 3, &keychain);
+
+	let header_1 = chain.get_header_by_height(1).unwrap();
+
+	// Now create tx to spend an early coinbase (now matured).
+	// Provides us with some useful outputs to test with.
+	let initial_tx = test_transaction_spending_coinbase(&keychain, &header_1, vec![10, 20, 30, 40]);
+
+	// Mine that initial tx so we can spend it with multiple txs.
+	add_block(&chain, vec![initial_tx], &keychain);
+
+	add_some_blocks(&chain, 5, &keychain);
+
+	let header = chain.head_header().unwrap();
+
+	assert_eq!(header.height, consensus::TESTING_THIRD_HARD_FORK);
+	assert_eq!(header.version, HeaderVersion(4));
+
+	let (tx1, tx2, tx3) = {
+		let mut kernel = TxKernel::with_features(KernelFeatures::NoRecentDuplicate {
+			fee: 6,
+			relative_height: NRDRelativeHeight::new(2)?,
+		});
+		let msg = kernel.msg_to_sign().unwrap();
+
+		// Generate a kernel with public excess and associated signature.
+		let excess = BlindingFactor::rand(&keychain.secp());
+		let skey = excess.secret_key(&keychain.secp()).unwrap();
+		kernel.excess = keychain.secp().commit(0, skey).unwrap();
+		let pubkey = &kernel.excess.to_pubkey(&keychain.secp()).unwrap();
+		kernel.excess_sig =
+			aggsig::sign_with_blinding(&keychain.secp(), &msg, &excess, Some(&pubkey)).unwrap();
+		kernel.verify().unwrap();
+
+		let tx1 = test_transaction_with_kernel(
+			&keychain,
+			vec![10, 20],
+			vec![24],
+			kernel.clone(),
+			excess.clone(),
+		);
+
+		let tx2 = test_transaction_with_kernel(
+			&keychain,
+			vec![24],
+			vec![18],
+			kernel.clone(),
+			excess.clone(),
+		);
+
+		// Now reuse kernel excess for tx3 but with NRD relative_height=1 (and different fee).
+		let mut kernel_short = TxKernel::with_features(KernelFeatures::NoRecentDuplicate {
+			fee: 3,
+			relative_height: NRDRelativeHeight::new(1)?,
+		});
+		let msg_short = kernel_short.msg_to_sign().unwrap();
+		kernel_short.excess = kernel.excess;
+		kernel_short.excess_sig =
+			aggsig::sign_with_blinding(&keychain.secp(), &msg_short, &excess, Some(&pubkey))
+				.unwrap();
+		kernel_short.verify().unwrap();
+
+		let tx3 = test_transaction_with_kernel(
+			&keychain,
+			vec![18],
+			vec![15],
+			kernel_short.clone(),
+			excess.clone(),
+		);
+
+		(tx1, tx2, tx3)
+	};
+
+	// Confirm we can successfully add tx1 with NRD kernel to stempool.
+	assert_eq!(
+		pool.add_to_pool(test_source(), tx1.clone(), true, &header),
+		Ok(()),
+	);
+	assert_eq!(pool.stempool.size(), 1);
+
+	// Confirm we cannot add tx2 to stempool while tx1 is in there (duplicate NRD kernels).
+	assert_eq!(
+		pool.add_to_pool(test_source(), tx2.clone(), true, &header),
+		Err(PoolError::NRDKernelRelativeHeight)
+	);
+
+	// Confirm we can successfully add tx1 with NRD kernel to txpool,
+	// removing existing instance of tx1 from stempool in the process.
+	assert_eq!(
+		pool.add_to_pool(test_source(), tx1.clone(), false, &header),
+		Ok(()),
+	);
+	assert_eq!(pool.txpool.size(), 1);
+	assert_eq!(pool.stempool.size(), 0);
+
+	// Confirm we cannot add tx2 to stempool while tx1 is in txpool (duplicate NRD kernels).
+	assert_eq!(
+		pool.add_to_pool(test_source(), tx2.clone(), true, &header),
+		Err(PoolError::NRDKernelRelativeHeight)
+	);
+
+	// Confirm we cannot add tx2 to txpool while tx1 is in there (duplicate NRD kernels).
+	assert_eq!(
+		pool.add_to_pool(test_source(), tx2.clone(), false, &header),
+		Err(PoolError::NRDKernelRelativeHeight)
+	);
+
+	assert_eq!(pool.total_size(), 1);
+	assert_eq!(pool.txpool.size(), 1);
+	assert_eq!(pool.stempool.size(), 0);
+
+	let txs = pool.prepare_mineable_transactions().unwrap();
+	assert_eq!(txs.len(), 1);
+
+	// Mine block containing tx1 from the txpool.
+	add_block(&chain, txs, &keychain);
+	let header = chain.head_header().unwrap();
+	let block = chain.get_block(&header.hash()).unwrap();
+
+	// Confirm the stempool/txpool is empty after reconciling the new block.
+	pool.reconcile_block(&block)?;
+	assert_eq!(pool.total_size(), 0);
+	assert_eq!(pool.txpool.size(), 0);
+	assert_eq!(pool.stempool.size(), 0);
+
+	// Confirm we cannot add tx2 to stempool with tx1 in previous block (NRD relative_height=2)
+	assert_eq!(
+		pool.add_to_pool(test_source(), tx2.clone(), true, &header),
+		Err(PoolError::NRDKernelRelativeHeight)
+	);
+
+	// Confirm we cannot add tx2 to txpool with tx1 in previous block (NRD relative_height=2)
+	assert_eq!(
+		pool.add_to_pool(test_source(), tx2.clone(), false, &header),
+		Err(PoolError::NRDKernelRelativeHeight)
+	);
+
+	// Add another block so NRD relative_height rule is now met.
+	add_block(&chain, vec![], &keychain);
+	let header = chain.head_header().unwrap();
+
+	// Confirm we can now add tx2 to stempool with NRD relative_height rule met.
+	assert_eq!(
+		pool.add_to_pool(test_source(), tx2.clone(), true, &header),
+		Ok(())
+	);
+	assert_eq!(pool.total_size(), 0);
+	assert_eq!(pool.txpool.size(), 0);
+	assert_eq!(pool.stempool.size(), 1);
+
+	// Confirm we cannot yet add tx3 to stempool (NRD relative_height=1)
+	assert_eq!(
+		pool.add_to_pool(test_source(), tx3.clone(), true, &header),
+		Err(PoolError::NRDKernelRelativeHeight)
+	);
+
+	// Confirm we can now add tx2 to txpool with NRD relative_height rule met.
+	assert_eq!(
+		pool.add_to_pool(test_source(), tx2.clone(), false, &header),
+		Ok(())
+	);
+
+	// Confirm we cannot yet add tx3 to txpool (NRD relative_height=1)
+	assert_eq!(
+		pool.add_to_pool(test_source(), tx3.clone(), false, &header),
+		Err(PoolError::NRDKernelRelativeHeight)
+	);
+
+	assert_eq!(pool.total_size(), 1);
+	assert_eq!(pool.txpool.size(), 1);
+	assert_eq!(pool.stempool.size(), 0);
+
+	let txs = pool.prepare_mineable_transactions().unwrap();
+	assert_eq!(txs.len(), 1);
+
+	// Mine block containing tx2 from the txpool.
+	add_block(&chain, txs, &keychain);
+	let header = chain.head_header().unwrap();
+	let block = chain.get_block(&header.hash()).unwrap();
+	pool.reconcile_block(&block)?;
+
+	assert_eq!(pool.total_size(), 0);
+	assert_eq!(pool.txpool.size(), 0);
+	assert_eq!(pool.stempool.size(), 0);
+
+	// Confirm we can now add tx3 to stempool with tx2 in immediate previous block (NRD relative_height=1)
+	assert_eq!(
+		pool.add_to_pool(test_source(), tx3.clone(), true, &header),
+		Ok(())
+	);
+
+	assert_eq!(pool.total_size(), 0);
+	assert_eq!(pool.txpool.size(), 0);
+	assert_eq!(pool.stempool.size(), 1);
+
+	// Confirm we can now add tx3 to txpool with tx2 in immediate previous block (NRD relative_height=1)
+	assert_eq!(
+		pool.add_to_pool(test_source(), tx3.clone(), false, &header),
+		Ok(())
+	);
+
+	assert_eq!(pool.total_size(), 1);
+	assert_eq!(pool.txpool.size(), 1);
+	assert_eq!(pool.stempool.size(), 0);
+
+	// Cleanup db directory
+	clean_output_dir(db_root.into());
+
+	Ok(())
+}

pool/tests/transaction_pool.rs

@@ -165,12 +165,14 @@ fn test_the_transaction_pool() {
 		pool.add_to_pool(test_source(), tx.clone(), true, &header)
 			.unwrap();
 		assert_eq!(pool.total_size(), 4);
+		assert_eq!(pool.txpool.size(), 4);
 		assert_eq!(pool.stempool.size(), 1);
 
 		// Duplicate stem tx so fluff, adding it to txpool and removing it from stempool.
 		pool.add_to_pool(test_source(), tx.clone(), true, &header)
 			.unwrap();
 		assert_eq!(pool.total_size(), 5);
+		assert_eq!(pool.txpool.size(), 5);
 		assert!(pool.stempool.is_empty());
 	}