NRD rules and "recent" kernel pos index (#3302)

* variants for output_pos linked list entries (head/tail/middle/unique)
next and prev and Vec<u8> lmdb keys

* get_pos on enum

* break list and list entries out into separate enums

* track output features in the new output_pos index, so we can determine coinbase maturity

* push entry impl for none and unique

* some test coverage for output_pos_list

* commit

* wip - FooListEntry

* use instance of the index

* linked list of output_pos and commit_pos both now supported

* linked_list

* cleanup and rename

* rename

* peek_pos

* push some, peek some, pop some

* cleanup

* commit pos
cleanup

* split list and entry out into separate db prefixes

* cleanup and add placeholder for pop_back

* pop_pos_back (for popping off the back of the linked list)
test coverage for pop_pos_back

* wip

* placeholder for prune via a trait
pos must always increase in the index

* rewind kernel_pos_idx when calling rewind_single_block

* RewindableListIndex with rewind support.

* test coverage for rewindable list index

* test coverage for rewind back to 0

* rewind past end of list

* add tests for kernel_pos_idx with multiple commits

* commit

* cleanup

* hook NRD relative lock height validation into block processing and tx validation

* cleanup

* set local chain type for kernel_idx tests

* add test coverage for NRD rules in block processing

* NRD test coverage and cleanup

* NRD relative height 1 test

* test coverage for NRD kernels in block processing

* cleanup

* start of test coverage for txpool NRD kernel rules

* wip

* rework pool tests to use real chain (was mock chain) to better reflect reality (tx/block validation rules etc.)

* cleanup

* cleanup pruneable trait for kernel pos index

* add clear() to kernel_pos idx and test coverage

* hook kernel_pos rebuild into node startup, compaction and fast sync

* verify full NRD history on fast sync

* return early if nrd disabled

* fix header sync issue

pool/src/pool.rs

@@ -29,6 +29,7 @@ use grin_util as util;
 use std::cmp::Reverse;
 use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
+use util::secp::pedersen::Commitment;
 use util::static_secp_instance;
 
 pub struct Pool<B, V>
@@ -70,6 +71,19 @@ where
 			.map(|x| x.tx.clone())
 	}
 
+	/// Query the tx pool for an individual tx matching the given public excess.
+	/// Used for checking for duplicate NRD kernels in the txpool.
+	pub fn retrieve_tx_by_kernel_excess(&self, excess: Commitment) -> Option<Transaction> {
+		for x in &self.entries {
+			for k in x.tx.kernels() {
+				if k.excess() == excess {
+					return Some(x.tx.clone());
+				}
+			}
+		}
+		None
+	}
+
 	/// Query the tx pool for an individual tx matching the given kernel hash.
 	pub fn retrieve_tx_by_kernel_hash(&self, hash: Hash) -> Option<Transaction> {
 		for x in &self.entries {
@@ -197,7 +211,6 @@ where
 		// Validate aggregated tx (existing pool + new tx), ignoring tx weight limits.
 		// Validate against known chain state at the provided header.
 		self.validate_raw_tx(&agg_tx, header, Weighting::NoLimit)?;
-
 		// If we get here successfully then we can safely add the entry to the pool.
 		self.log_pool_add(&entry, header);
 		self.entries.push(entry);
@@ -425,6 +438,7 @@ where
 		tx_buckets.into_iter().flat_map(|x| x.raw_txs).collect()
 	}
 
+	/// TODO - This is kernel based. How does this interact with NRD?
 	pub fn find_matching_transactions(&self, kernels: &[TxKernel]) -> Vec<Transaction> {
 		// While the inputs outputs can be cut-through the kernel will stay intact
 		// In order to deaggregate tx we look for tx with the same kernel

pool/src/transaction_pool.rs

@@ -160,13 +160,18 @@ where
 		stem: bool,
 		header: &BlockHeader,
 	) -> Result<(), PoolError> {
-		// Quick check to deal with common case of seeing the *same* tx
-		// broadcast from multiple peers simultaneously.
-		if !stem && self.txpool.contains_tx(tx.hash()) {
+		// Quick check for duplicate txs.
+		// Our stempool is private and we do not want to reveal anything about the txs contained.
+		// If this is a stem tx and is already present in stempool then fluff by adding to txpool.
+		// Otherwise if already present in txpool return a "duplicate tx" error.
+		if stem && self.stempool.contains_tx(tx.hash()) {
+			return self.add_to_pool(src, tx, false, header);
+		} else if self.txpool.contains_tx(tx.hash()) {
 			return Err(PoolError::DuplicateTx);
 		}
 
 		// Check this tx is valid based on current header version.
+		// NRD kernels only valid post HF3 and if NRD feature enabled.
 		self.verify_kernel_variants(&tx, header)?;
 
 		// Do we have the capacity to accept this transaction?
@@ -195,20 +200,19 @@ where
 			tx,
 		};
 
-		// If not stem then we are fluff.
-		// If this is a stem tx then attempt to stem.
-		// Any problems during stem, fallback to fluff.
-		if !stem
-			|| self
-				.add_to_stempool(entry.clone(), header)
-				.and_then(|_| self.adapter.stem_tx_accepted(&entry))
-				.is_err()
-		{
-			self.add_to_txpool(entry.clone(), header)?;
-			self.add_to_reorg_cache(entry.clone());
-			self.adapter.tx_accepted(&entry);
+		// If this is a stem tx then attempt to add it to stempool.
+		// If the adapter fails to accept the new stem tx then fallback to fluff via txpool.
+		if stem {
+			self.add_to_stempool(entry.clone(), header)?;
+			if self.adapter.stem_tx_accepted(&entry).is_ok() {
+				return Ok(());
+			}
 		}
 
+		self.add_to_txpool(entry.clone(), header)?;
+		self.add_to_reorg_cache(entry.clone());
+		self.adapter.tx_accepted(&entry);
+
 		// Transaction passed all the checks but we have to make space for it
 		if evict {
 			self.evict_from_txpool();

pool/src/types.rs

@@ -227,6 +227,9 @@ pub enum PoolError {
 	/// NRD kernels are not valid if disabled locally via "feature flag".
 	#[fail(display = "NRD kernel not enabled")]
 	NRDKernelNotEnabled,
+	/// NRD kernels are not valid if relative_height rule not met.
+	#[fail(display = "NRD kernel relative height")]
+	NRDKernelRelativeHeight,
 	/// Other kinds of error (not yet pulled out into meaningful errors).
 	#[fail(display = "General pool error {}", _0)]
 	Other(String),
@@ -234,7 +237,10 @@ pub enum PoolError {
 
 impl From<transaction::Error> for PoolError {
 	fn from(e: transaction::Error) -> PoolError {
-		PoolError::InvalidTx(e)
+		match e {
+			transaction::Error::InvalidNRDRelativeHeight => PoolError::NRDKernelRelativeHeight,
+			e @ _ => PoolError::InvalidTx(e),
+		}
 	}
 }