NRD rules and "recent" kernel pos index (#3302)

* variants for output_pos linked list entries (head/tail/middle/unique)
next and prev and Vec<u8> lmdb keys

* get_pos on enum

* break list and list entries out into separate enums

* track output features in the new output_pos index, so we can determine coinbase maturity

* push entry impl for none and unique

* some test coverage for output_pos_list

* commit

* wip - FooListEntry

* use instance of the index

* linked list of output_pos and commit_pos both now supported

* linked_list

* cleanup and rename

* rename

* peek_pos

* push some, peek some, pop some

* cleanup

* commit pos
cleanup

* split list and entry out into separate db prefixes

* cleanup and add placeholder for pop_back

* pop_pos_back (for popping off the back of the linked list)
test coverage for pop_pos_back

* wip

* placeholder for prune via a trait
pos must always increase in the index

* rewind kernel_pos_idx when calling rewind_single_block

* RewindableListIndex with rewind support.

* test coverage for rewindable list index

* test coverage for rewind back to 0

* rewind past end of list

* add tests for kernel_pos_idx with multiple commits

* commit

* cleanup

* hook NRD relative lock height validation into block processing and tx validation

* cleanup

* set local chain type for kernel_idx tests

* add test coverage for NRD rules in block processing

* NRD test coverage and cleanup

* NRD relative height 1 test

* test coverage for NRD kernels in block processing

* cleanup

* start of test coverage for txpool NRD kernel rules

* wip

* rework pool tests to use real chain (was mock chain) to better reflect reality (tx/block validation rules etc.)

* cleanup

* cleanup pruneable trait for kernel pos index

* add clear() to kernel_pos idx and test coverage

* hook kernel_pos rebuild into node startup, compaction and fast sync

* verify full NRD history on fast sync

* return early if nrd disabled

* fix header sync issue

chain/src/chain.rs

@@ -19,7 +19,8 @@ use crate::core::core::hash::{Hash, Hashed, ZERO_HASH};
 use crate::core::core::merkle_proof::MerkleProof;
 use crate::core::core::verifier_cache::VerifierCache;
 use crate::core::core::{
-	Block, BlockHeader, BlockSums, Committed, Output, OutputIdentifier, Transaction, TxKernel,
+	Block, BlockHeader, BlockSums, Committed, KernelFeatures, Output, OutputIdentifier,
+	Transaction, TxKernel,
 };
 use crate::core::global;
 use crate::core::pow;
@@ -195,12 +196,12 @@ impl Chain {
 			&mut txhashset,
 		)?;
 
-		// Initialize the output_pos index based on UTXO set.
-		// This is fast as we only look for stale and missing entries
-		// and do not need to rebuild the entire index.
+		// Initialize the output_pos index based on UTXO set
+		// and NRD kernel_pos index based recent kernel history.
 		{
 			let batch = store.batch()?;
 			txhashset.init_output_pos_index(&header_pmmr, &batch)?;
+			txhashset.init_recent_kernel_pos_index(&header_pmmr, &batch)?;
 			batch.commit()?;
 		}
 
@@ -296,6 +297,11 @@ impl Chain {
 	/// Returns true if it has been added to the longest chain
 	/// or false if it has added to a fork (or orphan?).
 	fn process_block_single(&self, b: Block, opts: Options) -> Result<Option<Tip>, Error> {
+		// Process the header first.
+		// If invalid then fail early.
+		// If valid then continue with block processing with header_head committed to db etc.
+		self.process_block_header(&b.header, opts)?;
+
 		let (maybe_new_head, prev_head) = {
 			let mut header_pmmr = self.header_pmmr.write();
 			let mut txhashset = self.txhashset.write();
@@ -513,13 +519,38 @@ impl Chain {
 		})
 	}
 
-	/// Validate the tx against the current UTXO set.
+	/// Validate the tx against the current UTXO set and recent kernels (NRD relative lock heights).
 	pub fn validate_tx(&self, tx: &Transaction) -> Result<(), Error> {
+		self.validate_tx_against_utxo(tx)?;
+		self.validate_tx_kernels(tx)?;
+		Ok(())
+	}
+
+	/// Validates NRD relative height locks against "recent" kernel history.
+	/// Applies the kernels to the current kernel MMR in a readonly extension.
+	/// The extension and the db batch are discarded.
+	/// The batch ensures duplicate NRD kernels within the tx are handled correctly.
+	fn validate_tx_kernels(&self, tx: &Transaction) -> Result<(), Error> {
+		let has_nrd_kernel = tx.kernels().iter().any(|k| match k.features {
+			KernelFeatures::NoRecentDuplicate { .. } => true,
+			_ => false,
+		});
+		if !has_nrd_kernel {
+			return Ok(());
+		}
+		let mut header_pmmr = self.header_pmmr.write();
+		let mut txhashset = self.txhashset.write();
+		txhashset::extending_readonly(&mut header_pmmr, &mut txhashset, |ext, batch| {
+			let height = self.next_block_height()?;
+			ext.extension.apply_kernels(tx.kernels(), height, batch)
+		})
+	}
+
+	fn validate_tx_against_utxo(&self, tx: &Transaction) -> Result<(), Error> {
 		let header_pmmr = self.header_pmmr.read();
 		let txhashset = self.txhashset.read();
 		txhashset::utxo_view(&header_pmmr, &txhashset, |utxo, batch| {
-			utxo.validate_tx(tx, batch)?;
-			Ok(())
+			utxo.validate_tx(tx, batch)
 		})
 	}
 
@@ -929,8 +960,16 @@ impl Chain {
 			Some(&header),
 		)?;
 
-		// Validate the full kernel history (kernel MMR root for every block header).
-		self.validate_kernel_history(&header, &txhashset)?;
+		// Validate the full kernel history.
+		// Check kernel MMR root for every block header.
+		// Check NRD relative height rules for full kernel history.
+		{
+			self.validate_kernel_history(&header, &txhashset)?;
+
+			let header_pmmr = self.header_pmmr.read();
+			let batch = self.store.batch()?;
+			txhashset.verify_kernel_pos_index(&self.genesis, &header_pmmr, &batch)?;
+		}
 
 		// all good, prepare a new batch and update all the required records
 		debug!("txhashset_write: rewinding a 2nd time (writeable)");
@@ -979,6 +1018,9 @@ impl Chain {
 		// Rebuild our output_pos index in the db based on fresh UTXO set.
 		txhashset.init_output_pos_index(&header_pmmr, &batch)?;
 
+		// Rebuild our NRD kernel_pos index based on recent kernel history.
+		txhashset.init_recent_kernel_pos_index(&header_pmmr, &batch)?;
+
 		// Commit all the changes to the db.
 		batch.commit()?;
 
@@ -1115,6 +1157,9 @@ impl Chain {
 		// Make sure our output_pos index is consistent with the UTXO set.
 		txhashset.init_output_pos_index(&header_pmmr, &batch)?;
 
+		// Rebuild our NRD kernel_pos index based on recent kernel history.
+		txhashset.init_recent_kernel_pos_index(&header_pmmr, &batch)?;
+
 		// Commit all the above db changes.
 		batch.commit()?;
 

chain/src/error.rs

@@ -122,6 +122,9 @@ pub enum ErrorKind {
 	/// Tx not valid based on lock_height.
 	#[fail(display = "Transaction Lock Height")]
 	TxLockHeight,
+	/// Tx is not valid due to NRD relative_height restriction.
+	#[fail(display = "NRD Relative Height")]
+	NRDRelativeHeight,
 	/// No chain exists and genesis block is required
 	#[fail(display = "Genesis Block Required")]
 	GenesisBlockRequired,

chain/src/lib.rs

@@ -23,6 +23,9 @@
 #[macro_use]
 extern crate bitflags;
 
+#[macro_use]
+extern crate enum_primitive;
+
 #[macro_use]
 extern crate serde_derive;
 #[macro_use]
@@ -35,6 +38,7 @@ use grin_util as util;
 
 mod chain;
 mod error;
+pub mod linked_list;
 pub mod pipe;
 pub mod store;
 pub mod txhashset;

chain/src/linked_list.rs

@@ -0,0 +1,582 @@
+// Copyright 2020 The Grin Developers
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Implements "linked list" storage primitive for lmdb index supporting multiple entries.
+
+use crate::core::ser::{self, Readable, Reader, Writeable, Writer};
+use crate::store::Batch;
+use crate::types::CommitPos;
+use crate::util::secp::pedersen::Commitment;
+use enum_primitive::FromPrimitive;
+use grin_store as store;
+use std::marker::PhantomData;
+use store::{to_key, to_key_u64, Error};
+
+enum_from_primitive! {
+	#[derive(Copy, Clone, Debug, PartialEq)]
+	enum ListWrapperVariant {
+		Single = 0,
+		Multi = 1,
+	}
+}
+
+impl Writeable for ListWrapperVariant {
+	fn write<W: Writer>(&self, writer: &mut W) -> Result<(), ser::Error> {
+		writer.write_u8(*self as u8)
+	}
+}
+
+impl Readable for ListWrapperVariant {
+	fn read<R: Reader>(reader: &mut R) -> Result<ListWrapperVariant, ser::Error> {
+		ListWrapperVariant::from_u8(reader.read_u8()?).ok_or(ser::Error::CorruptedData)
+	}
+}
+
+enum_from_primitive! {
+	#[derive(Copy, Clone, Debug, PartialEq)]
+	enum ListEntryVariant {
+		// Start at 2 here to differentiate from ListWrapperVariant above.
+		Head = 2,
+		Tail = 3,
+		Middle = 4,
+	}
+}
+
+impl Writeable for ListEntryVariant {
+	fn write<W: Writer>(&self, writer: &mut W) -> Result<(), ser::Error> {
+		writer.write_u8(*self as u8)
+	}
+}
+
+impl Readable for ListEntryVariant {
+	fn read<R: Reader>(reader: &mut R) -> Result<ListEntryVariant, ser::Error> {
+		ListEntryVariant::from_u8(reader.read_u8()?).ok_or(ser::Error::CorruptedData)
+	}
+}
+
+/// Index supporting a list of (duplicate) entries per commitment.
+/// Each entry will be at a unique MMR pos.
+pub trait ListIndex {
+	/// List type
+	type List: Readable + Writeable;
+
+	/// List entry type
+	type Entry: ListIndexEntry;
+
+	/// Construct a key for the list.
+	fn list_key(&self, commit: Commitment) -> Vec<u8>;
+
+	/// Construct a key for an individual entry in the list.
+	fn entry_key(&self, commit: Commitment, pos: u64) -> Vec<u8>;
+
+	/// Returns either a "Single" with embedded "pos" or a "list" with "head" and "tail".
+	/// Key is "prefix|commit".
+	/// Note the key for an individual entry in the list is "prefix|commit|pos".
+	fn get_list(&self, batch: &Batch<'_>, commit: Commitment) -> Result<Option<Self::List>, Error> {
+		batch.db.get_ser(&self.list_key(commit))
+	}
+
+	/// Returns one of "head", "tail" or "middle" entry variants.
+	/// Key is "prefix|commit|pos".
+	fn get_entry(
+		&self,
+		batch: &Batch<'_>,
+		commit: Commitment,
+		pos: u64,
+	) -> Result<Option<Self::Entry>, Error> {
+		batch.db.get_ser(&self.entry_key(commit, pos))
+	}
+
+	/// Peek the head of the list for the specified commitment.
+	fn peek_pos(
+		&self,
+		batch: &Batch<'_>,
+		commit: Commitment,
+	) -> Result<Option<<Self::Entry as ListIndexEntry>::Pos>, Error>;
+
+	/// Push a pos onto the list for the specified commitment.
+	fn push_pos(
+		&self,
+		batch: &Batch<'_>,
+		commit: Commitment,
+		new_pos: <Self::Entry as ListIndexEntry>::Pos,
+	) -> Result<(), Error>;
+
+	/// Pop a pos off the list for the specified commitment.
+	fn pop_pos(
+		&self,
+		batch: &Batch<'_>,
+		commit: Commitment,
+	) -> Result<Option<<Self::Entry as ListIndexEntry>::Pos>, Error>;
+}
+
+/// Supports "rewind" given the provided commit and a pos to rewind back to.
+pub trait RewindableListIndex {
+	/// Rewind the index for the given commitment to the specified position.
+	fn rewind(&self, batch: &Batch<'_>, commit: Commitment, rewind_pos: u64) -> Result<(), Error>;
+}
+
+/// A pruneable list index supports pruning of old data from the index lists.
+/// This allows us to efficiently maintain an index of "recent" kernel data.
+/// We can maintain a window of 2 weeks of recent data, discarding anything older than this.
+pub trait PruneableListIndex: ListIndex {
+	/// Clear all data from the index.
+	/// Used when rebuilding the index.
+	fn clear(&self, batch: &Batch<'_>) -> Result<(), Error>;
+
+	/// Prune old data.
+	fn prune(&self, batch: &Batch<'_>, commit: Commitment, cutoff_pos: u64) -> Result<(), Error>;
+
+	/// Pop a pos off the back of the list (used for pruning old data).
+	fn pop_pos_back(
+		&self,
+		batch: &Batch<'_>,
+		commit: Commitment,
+	) -> Result<Option<<Self::Entry as ListIndexEntry>::Pos>, Error>;
+}
+
+/// Wrapper for the list to handle either `Single` or `Multi` entries.
+/// Optimized for the common case where we have a single entry in the list.
+#[derive(Copy, Clone, Debug, PartialEq)]
+pub enum ListWrapper<T> {
+	/// List with a single entry.
+	/// Allows direct access to the pos.
+	Single {
+		/// The MMR pos where this single entry is located.
+		pos: T,
+	},
+	/// List with multiple entries.
+	/// Maintains head and tail of the underlying linked list.
+	Multi {
+		/// Head of the linked list.
+		head: u64,
+		/// Tail of the linked list.
+		tail: u64,
+	},
+}
+
+impl<T> Writeable for ListWrapper<T>
+where
+	T: Writeable,
+{
+	/// Write first byte representing the variant, followed by variant specific data.
+	/// "Single" is optimized with embedded "pos".
+	/// "Multi" has references to "head" and "tail".
+	fn write<W: Writer>(&self, writer: &mut W) -> Result<(), ser::Error> {
+		match self {
+			ListWrapper::Single { pos } => {
+				ListWrapperVariant::Single.write(writer)?;
+				pos.write(writer)?;
+			}
+			ListWrapper::Multi { head, tail } => {
+				ListWrapperVariant::Multi.write(writer)?;
+				writer.write_u64(*head)?;
+				writer.write_u64(*tail)?;
+			}
+		}
+		Ok(())
+	}
+}
+
+impl<T> Readable for ListWrapper<T>
+where
+	T: Readable,
+{
+	/// Read the first byte to determine what needs to be read beyond that.
+	fn read<R: Reader>(reader: &mut R) -> Result<ListWrapper<T>, ser::Error> {
+		let entry = match ListWrapperVariant::read(reader)? {
+			ListWrapperVariant::Single => ListWrapper::Single {
+				pos: T::read(reader)?,
+			},
+			ListWrapperVariant::Multi => ListWrapper::Multi {
+				head: reader.read_u64()?,
+				tail: reader.read_u64()?,
+			},
+		};
+		Ok(entry)
+	}
+}
+
+/// Index supporting multiple duplicate entries.
+pub struct MultiIndex<T> {
+	phantom: PhantomData<*const T>,
+	list_prefix: u8,
+	entry_prefix: u8,
+}
+
+impl<T> MultiIndex<T> {
+	/// Initialize a new multi index with the specified list and entry prefixes.
+	pub fn init(list_prefix: u8, entry_prefix: u8) -> MultiIndex<T> {
+		MultiIndex {
+			phantom: PhantomData,
+			list_prefix,
+			entry_prefix,
+		}
+	}
+}
+
+impl<T> ListIndex for MultiIndex<T>
+where
+	T: PosEntry,
+{
+	type List = ListWrapper<T>;
+	type Entry = ListEntry<T>;
+
+	fn list_key(&self, commit: Commitment) -> Vec<u8> {
+		to_key(self.list_prefix, &mut commit.as_ref().to_vec())
+	}
+
+	fn entry_key(&self, commit: Commitment, pos: u64) -> Vec<u8> {
+		to_key_u64(self.entry_prefix, &mut commit.as_ref().to_vec(), pos)
+	}
+
+	fn peek_pos(&self, batch: &Batch<'_>, commit: Commitment) -> Result<Option<T>, Error> {
+		match self.get_list(batch, commit)? {
+			None => Ok(None),
+			Some(ListWrapper::Single { pos }) => Ok(Some(pos)),
+			Some(ListWrapper::Multi { head, .. }) => {
+				if let Some(ListEntry::Head { pos, .. }) = self.get_entry(batch, commit, head)? {
+					Ok(Some(pos))
+				} else {
+					Err(Error::OtherErr("expected head to be head variant".into()))
+				}
+			}
+		}
+	}
+
+	fn push_pos(&self, batch: &Batch<'_>, commit: Commitment, new_pos: T) -> Result<(), Error> {
+		match self.get_list(batch, commit)? {
+			None => {
+				let list = ListWrapper::Single { pos: new_pos };
+				batch.db.put_ser(&self.list_key(commit), &list)?;
+			}
+			Some(ListWrapper::Single { pos: current_pos }) => {
+				if new_pos.pos() <= current_pos.pos() {
+					return Err(Error::OtherErr("pos must be increasing".into()));
+				}
+
+				let head = ListEntry::Head {
+					pos: new_pos,
+					next: current_pos.pos(),
+				};
+				let tail = ListEntry::Tail {
+					pos: current_pos,
+					prev: new_pos.pos(),
+				};
+				let list: ListWrapper<T> = ListWrapper::Multi {
+					head: new_pos.pos(),
+					tail: current_pos.pos(),
+				};
+				batch
+					.db
+					.put_ser(&self.entry_key(commit, new_pos.pos()), &head)?;
+				batch
+					.db
+					.put_ser(&self.entry_key(commit, current_pos.pos()), &tail)?;
+				batch.db.put_ser(&self.list_key(commit), &list)?;
+			}
+			Some(ListWrapper::Multi { head, tail }) => {
+				if new_pos.pos() <= head {
+					return Err(Error::OtherErr("pos must be increasing".into()));
+				}
+
+				if let Some(ListEntry::Head {
+					pos: current_pos,
+					next: current_next,
+				}) = self.get_entry(batch, commit, head)?
+				{
+					let head = ListEntry::Head {
+						pos: new_pos,
+						next: current_pos.pos(),
+					};
+					let middle = ListEntry::Middle {
+						pos: current_pos,
+						next: current_next,
+						prev: new_pos.pos(),
+					};
+					let list: ListWrapper<T> = ListWrapper::Multi {
+						head: new_pos.pos(),
+						tail,
+					};
+					batch
+						.db
+						.put_ser(&self.entry_key(commit, new_pos.pos()), &head)?;
+					batch
+						.db
+						.put_ser(&self.entry_key(commit, current_pos.pos()), &middle)?;
+					batch.db.put_ser(&self.list_key(commit), &list)?;
+				} else {
+					return Err(Error::OtherErr("expected head to be head variant".into()));
+				}
+			}
+		}
+		Ok(())
+	}
+
+	/// Pop the head of the list.
+	/// Returns the output_pos.
+	/// Returns None if list was empty.
+	fn pop_pos(&self, batch: &Batch<'_>, commit: Commitment) -> Result<Option<T>, Error> {
+		match self.get_list(batch, commit)? {
+			None => Ok(None),
+			Some(ListWrapper::Single { pos }) => {
+				batch.delete(&self.list_key(commit))?;
+				Ok(Some(pos))
+			}
+			Some(ListWrapper::Multi { head, tail }) => {
+				if let Some(ListEntry::Head {
+					pos: current_pos,
+					next: current_next,
+				}) = self.get_entry(batch, commit, head)?
+				{
+					match self.get_entry(batch, commit, current_next)? {
+						Some(ListEntry::Middle { pos, next, .. }) => {
+							let head = ListEntry::Head { pos, next };
+							let list: ListWrapper<T> = ListWrapper::Multi {
+								head: pos.pos(),
+								tail,
+							};
+							batch.delete(&self.entry_key(commit, current_pos.pos()))?;
+							batch
+								.db
+								.put_ser(&self.entry_key(commit, pos.pos()), &head)?;
+							batch.db.put_ser(&self.list_key(commit), &list)?;
+							Ok(Some(current_pos))
+						}
+						Some(ListEntry::Tail { pos, .. }) => {
+							let list = ListWrapper::Single { pos };
+							batch.delete(&self.entry_key(commit, current_pos.pos()))?;
+							batch.db.put_ser(&self.list_key(commit), &list)?;
+							Ok(Some(current_pos))
+						}
+						Some(_) => Err(Error::OtherErr("next was unexpected".into())),
+						None => Err(Error::OtherErr("next missing".into())),
+					}
+				} else {
+					Err(Error::OtherErr("expected head to be head variant".into()))
+				}
+			}
+		}
+	}
+}
+
+/// List index that supports rewind.
+impl<T: PosEntry> RewindableListIndex for MultiIndex<T> {
+	fn rewind(&self, batch: &Batch<'_>, commit: Commitment, rewind_pos: u64) -> Result<(), Error> {
+		while self
+			.peek_pos(batch, commit)?
+			.map(|x| x.pos() > rewind_pos)
+			.unwrap_or(false)
+		{
+			self.pop_pos(batch, commit)?;
+		}
+		Ok(())
+	}
+}
+
+impl<T: PosEntry> PruneableListIndex for MultiIndex<T> {
+	fn clear(&self, batch: &Batch<'_>) -> Result<(), Error> {
+		let mut list_count = 0;
+		let mut entry_count = 0;
+		let prefix = to_key(self.list_prefix, "");
+		for (key, _) in batch.db.iter::<ListWrapper<T>>(&prefix)? {
+			let _ = batch.delete(&key);
+			list_count += 1;
+		}
+		let prefix = to_key(self.entry_prefix, "");
+		for (key, _) in batch.db.iter::<ListEntry<T>>(&prefix)? {
+			let _ = batch.delete(&key);
+			entry_count += 1;
+		}
+		debug!(
+			"clear: lists deleted: {}, entries deleted: {}",
+			list_count, entry_count
+		);
+		Ok(())
+	}
+
+	/// Pruning will be more performant than full rebuild but not yet necessary.
+	fn prune(
+		&self,
+		_batch: &Batch<'_>,
+		_commit: Commitment,
+		_cutoff_pos: u64,
+	) -> Result<(), Error> {
+		unimplemented!(
+			"we currently rebuild index on startup/compaction, pruning not yet implemented"
+		);
+	}
+
+	/// Pop off the back/tail of the linked list.
+	/// Used when pruning old data.
+	fn pop_pos_back(&self, batch: &Batch<'_>, commit: Commitment) -> Result<Option<T>, Error> {
+		match self.get_list(batch, commit)? {
+			None => Ok(None),
+			Some(ListWrapper::Single { pos }) => {
+				batch.delete(&self.list_key(commit))?;
+				Ok(Some(pos))
+			}
+			Some(ListWrapper::Multi { head, tail }) => {
+				if let Some(ListEntry::Tail {
+					pos: current_pos,
+					prev: current_prev,
+				}) = self.get_entry(batch, commit, tail)?
+				{
+					match self.get_entry(batch, commit, current_prev)? {
+						Some(ListEntry::Middle { pos, prev, .. }) => {
+							let tail = ListEntry::Tail { pos, prev };
+							let list: ListWrapper<T> = ListWrapper::Multi {
+								head,
+								tail: pos.pos(),
+							};
+							batch.delete(&self.entry_key(commit, current_pos.pos()))?;
+							batch
+								.db
+								.put_ser(&self.entry_key(commit, pos.pos()), &tail)?;
+							batch.db.put_ser(&self.list_key(commit), &list)?;
+							Ok(Some(current_pos))
+						}
+						Some(ListEntry::Head { pos, .. }) => {
+							let list = ListWrapper::Single { pos };
+							batch.delete(&self.entry_key(commit, current_pos.pos()))?;
+							batch.db.put_ser(&self.list_key(commit), &list)?;
+							Ok(Some(current_pos))
+						}
+						Some(_) => Err(Error::OtherErr("prev was unexpected".into())),
+						None => Err(Error::OtherErr("prev missing".into())),
+					}
+				} else {
+					Err(Error::OtherErr("expected tail to be tail variant".into()))
+				}
+			}
+		}
+	}
+}
+
+/// Something that tracks pos (in an MMR).
+pub trait PosEntry: Readable + Writeable + Copy {
+	/// Accessor for the underlying (MMR) pos.
+	fn pos(&self) -> u64;
+}
+
+impl PosEntry for CommitPos {
+	fn pos(&self) -> u64 {
+		self.pos
+	}
+}
+
+/// Entry maintained in the list index.
+pub trait ListIndexEntry: Readable + Writeable {
+	/// Type of the underlying pos indexed in the list.
+	type Pos: PosEntry;
+
+	/// Accessor for the underlying pos.
+	fn get_pos(&self) -> Self::Pos;
+}
+
+impl<T> ListIndexEntry for ListEntry<T>
+where
+	T: PosEntry,
+{
+	type Pos = T;
+
+	/// Read the common pos from the various enum variants.
+	fn get_pos(&self) -> Self::Pos {
+		match self {
+			Self::Head { pos, .. } => *pos,
+			Self::Tail { pos, .. } => *pos,
+			Self::Middle { pos, .. } => *pos,
+		}
+	}
+}
+
+/// Head|Middle|Tail variants for the linked list entries.
+pub enum ListEntry<T> {
+	/// Head of ther list.
+	Head {
+		/// The thing in the list.
+		pos: T,
+		/// The next entry in the list.
+		next: u64,
+	},
+	/// Tail of the list.
+	Tail {
+		/// The thing in the list.
+		pos: T,
+		/// The previous entry in the list.
+		prev: u64,
+	},
+	/// An entry in the middle of the list.
+	Middle {
+		/// The thing in the list.
+		pos: T,
+		/// The next entry in the list.
+		next: u64,
+		/// The previous entry in the list.
+		prev: u64,
+	},
+}
+
+impl<T> Writeable for ListEntry<T>
+where
+	T: Writeable,
+{
+	/// Write first byte representing the variant, followed by variant specific data.
+	fn write<W: Writer>(&self, writer: &mut W) -> Result<(), ser::Error> {
+		match self {
+			ListEntry::Head { pos, next } => {
+				ListEntryVariant::Head.write(writer)?;
+				pos.write(writer)?;
+				writer.write_u64(*next)?;
+			}
+			ListEntry::Tail { pos, prev } => {
+				ListEntryVariant::Tail.write(writer)?;
+				pos.write(writer)?;
+				writer.write_u64(*prev)?;
+			}
+			ListEntry::Middle { pos, next, prev } => {
+				ListEntryVariant::Middle.write(writer)?;
+				pos.write(writer)?;
+				writer.write_u64(*next)?;
+				writer.write_u64(*prev)?;
+			}
+		}
+		Ok(())
+	}
+}
+
+impl<T> Readable for ListEntry<T>
+where
+	T: Readable,
+{
+	/// Read the first byte to determine what needs to be read beyond that.
+	fn read<R: Reader>(reader: &mut R) -> Result<ListEntry<T>, ser::Error> {
+		let entry = match ListEntryVariant::read(reader)? {
+			ListEntryVariant::Head => ListEntry::Head {
+				pos: T::read(reader)?,
+				next: reader.read_u64()?,
+			},
+			ListEntryVariant::Tail => ListEntry::Tail {
+				pos: T::read(reader)?,
+				prev: reader.read_u64()?,
+			},
+			ListEntryVariant::Middle => ListEntry::Middle {
+				pos: T::read(reader)?,
+				next: reader.read_u64()?,
+				prev: reader.read_u64()?,
+			},
+		};
+		Ok(entry)
+	}
+}

chain/src/pipe.rs

@@ -227,9 +227,6 @@ pub fn sync_block_headers(
 /// Note: In contrast to processing a full block we treat "already known" as success
 /// to allow processing to continue (for header itself).
 pub fn process_block_header(header: &BlockHeader, ctx: &mut BlockContext<'_>) -> Result<(), Error> {
-	// Check this header is not an orphan, we must know about the previous header to continue.
-	let prev_header = ctx.batch.get_previous_header(&header)?;
-
 	// If we have already processed the full block for this header then done.
 	// Note: "already known" in this context is success so subsequent processing can continue.
 	{
@@ -239,6 +236,9 @@ pub fn process_block_header(header: &BlockHeader, ctx: &mut BlockContext<'_>) ->
 		}
 	}
 
+	// Check this header is not an orphan, we must know about the previous header to continue.
+	let prev_header = ctx.batch.get_previous_header(&header)?;
+
 	// If we have not yet seen the full block then check if we have seen this header.
 	// If it does not increase total_difficulty beyond our current header_head
 	// then we can (re)accept this header and process the full block (or request it).

chain/src/store.rs

@@ -19,6 +19,7 @@ use crate::core::core::hash::{Hash, Hashed};
 use crate::core::core::{Block, BlockHeader, BlockSums};
 use crate::core::pow::Difficulty;
 use crate::core::ser::ProtocolVersion;
+use crate::linked_list::MultiIndex;
 use crate::types::{CommitPos, Tip};
 use crate::util::secp::pedersen::Commitment;
 use croaring::Bitmap;
@@ -35,6 +36,12 @@ const HEAD_PREFIX: u8 = b'H';
 const TAIL_PREFIX: u8 = b'T';
 const HEADER_HEAD_PREFIX: u8 = b'G';
 const OUTPUT_POS_PREFIX: u8 = b'p';
+
+/// Prefix for NRD kernel pos index lists.
+pub const NRD_KERNEL_LIST_PREFIX: u8 = b'K';
+/// Prefix for NRD kernel pos index entries.
+pub const NRD_KERNEL_ENTRY_PREFIX: u8 = b'k';
+
 const BLOCK_INPUT_BITMAP_PREFIX: u8 = b'B';
 const BLOCK_SUMS_PREFIX: u8 = b'M';
 const BLOCK_SPENT_PREFIX: u8 = b'S';
@@ -61,9 +68,7 @@ impl ChainStore {
 			db: db_with_version,
 		}
 	}
-}
 
-impl ChainStore {
 	/// The current chain head.
 	pub fn head(&self) -> Result<Tip, Error> {
 		option_to_not_found(self.db.get_ser(&[HEAD_PREFIX]), || "HEAD".to_owned())
@@ -144,7 +149,8 @@ impl ChainStore {
 /// An atomic batch in which all changes can be committed all at once or
 /// discarded on error.
 pub struct Batch<'a> {
-	db: store::Batch<'a>,
+	/// The underlying db instance.
+	pub db: store::Batch<'a>,
 }
 
 impl<'a> Batch<'a> {
@@ -475,3 +481,10 @@ impl<'a> Iterator for DifficultyIter<'a> {
 		}
 	}
 }
+
+/// Init the NRD "recent history" kernel index backed by the underlying db.
+/// List index supports multiple entries per key, maintaining insertion order.
+/// Allows for fast lookup of the most recent entry per excess commitment.
+pub fn nrd_recent_kernel_index() -> MultiIndex<CommitPos> {
+	MultiIndex::init(NRD_KERNEL_LIST_PREFIX, NRD_KERNEL_ENTRY_PREFIX)
+}

chain/src/txhashset/txhashset.rs

@@ -15,14 +15,19 @@
 //! Utility structs to handle the 3 MMRs (output, rangeproof,
 //! kernel) along the overall header MMR conveniently and transactionally.
 
+use crate::core::consensus::WEEK_HEIGHT;
 use crate::core::core::committed::Committed;
 use crate::core::core::hash::{Hash, Hashed};
 use crate::core::core::merkle_proof::MerkleProof;
 use crate::core::core::pmmr::{self, Backend, ReadonlyPMMR, RewindablePMMR, PMMR};
-use crate::core::core::{Block, BlockHeader, Input, Output, OutputIdentifier, TxKernel};
+use crate::core::core::{
+	Block, BlockHeader, Input, KernelFeatures, Output, OutputIdentifier, TxKernel,
+};
+use crate::core::global;
 use crate::core::ser::{PMMRable, ProtocolVersion};
 use crate::error::{Error, ErrorKind};
-use crate::store::{Batch, ChainStore};
+use crate::linked_list::{ListIndex, PruneableListIndex, RewindableListIndex};
+use crate::store::{self, Batch, ChainStore};
 use crate::txhashset::bitmap_accumulator::BitmapAccumulator;
 use crate::txhashset::{RewindableKernelView, UTXOView};
 use crate::types::{CommitPos, OutputRoots, Tip, TxHashSetRoots, TxHashsetWriteStatus};
@@ -375,6 +380,86 @@ impl TxHashSet {
 		Ok(())
 	}
 
+	/// (Re)build the NRD kernel_pos index based on 2 weeks of recent kernel history.
+	pub fn init_recent_kernel_pos_index(
+		&self,
+		header_pmmr: &PMMRHandle<BlockHeader>,
+		batch: &Batch<'_>,
+	) -> Result<(), Error> {
+		let head = batch.head()?;
+		let cutoff = head.height.saturating_sub(WEEK_HEIGHT * 2);
+		let cutoff_hash = header_pmmr.get_header_hash_by_height(cutoff)?;
+		let cutoff_header = batch.get_block_header(&cutoff_hash)?;
+		self.verify_kernel_pos_index(&cutoff_header, header_pmmr, batch)
+	}
+
+	/// Verify and (re)build the NRD kernel_pos index from the provided header onwards.
+	pub fn verify_kernel_pos_index(
+		&self,
+		from_header: &BlockHeader,
+		header_pmmr: &PMMRHandle<BlockHeader>,
+		batch: &Batch<'_>,
+	) -> Result<(), Error> {
+		if !global::is_nrd_enabled() {
+			return Ok(());
+		}
+
+		let now = Instant::now();
+		let kernel_index = store::nrd_recent_kernel_index();
+		kernel_index.clear(batch)?;
+
+		let prev_size = if from_header.height == 0 {
+			0
+		} else {
+			let prev_header = batch.get_previous_header(&from_header)?;
+			prev_header.kernel_mmr_size
+		};
+
+		debug!(
+			"verify_kernel_pos_index: header: {} at {}, prev kernel_mmr_size: {}",
+			from_header.hash(),
+			from_header.height,
+			prev_size,
+		);
+
+		let kernel_pmmr =
+			ReadonlyPMMR::at(&self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos);
+
+		let mut current_pos = prev_size + 1;
+		let mut current_header = from_header.clone();
+		let mut count = 0;
+		while current_pos <= self.kernel_pmmr_h.last_pos {
+			if pmmr::is_leaf(current_pos) {
+				if let Some(kernel) = kernel_pmmr.get_data(current_pos) {
+					match kernel.features {
+						KernelFeatures::NoRecentDuplicate { .. } => {
+							while current_pos > current_header.kernel_mmr_size {
+								let hash = header_pmmr
+									.get_header_hash_by_height(current_header.height + 1)?;
+								current_header = batch.get_block_header(&hash)?;
+							}
+							let new_pos = CommitPos {
+								pos: current_pos,
+								height: current_header.height,
+							};
+							apply_kernel_rules(&kernel, new_pos, batch)?;
+							count += 1;
+						}
+						_ => {}
+					}
+				}
+			}
+			current_pos += 1;
+		}
+
+		debug!(
+			"verify_kernel_pos_index: pushed {} entries to the index, took {}s",
+			count,
+			now.elapsed().as_secs(),
+		);
+		Ok(())
+	}
+
 	/// (Re)build the output_pos index to be consistent with the current UTXO set.
 	/// Remove any "stale" index entries that do not correspond to outputs in the UTXO set.
 	/// Add any missing index entries based on UTXO set.
@@ -453,7 +538,7 @@ impl TxHashSet {
 			}
 		}
 		debug!(
-			"init_height_pos_index: added entries for {} utxos, took {}s",
+			"init_output_pos_index: added entries for {} utxos, took {}s",
 			total_outputs,
 			now.elapsed().as_secs(),
 		);
@@ -933,16 +1018,16 @@ impl<'a> Extension<'a> {
 		// Remove the spent output from the output_pos index.
 		let mut spent = vec![];
 		for input in b.inputs() {
-			let spent_pos = self.apply_input(input, batch)?;
-			affected_pos.push(spent_pos.pos);
+			let pos = self.apply_input(input, batch)?;
+			affected_pos.push(pos.pos);
 			batch.delete_output_pos_height(&input.commitment())?;
-			spent.push(spent_pos);
+			spent.push(pos);
 		}
 		batch.save_spent_index(&b.hash(), &spent)?;
 
-		for kernel in b.kernels() {
-			self.apply_kernel(kernel)?;
-		}
+		// Apply the kernels to the kernel MMR.
+		// Note: This validates and NRD relative height locks via the "recent" kernel index.
+		self.apply_kernels(b.kernels(), b.header.height, batch)?;
 
 		// Update our BitmapAccumulator based on affected outputs (both spent and created).
 		self.apply_to_bitmap_accumulator(&affected_pos)?;
@@ -1037,12 +1122,32 @@ impl<'a> Extension<'a> {
 		Ok(output_pos)
 	}
 
+	/// Apply kernels to the kernel MMR.
+	/// Validate any NRD relative height locks via the "recent" kernel index.
+	/// Note: This is used for both block processing and tx validation.
+	/// In the block processing case we use the block height.
+	/// In the tx validation case we use the "next" block height based on current chain head.
+	pub fn apply_kernels(
+		&mut self,
+		kernels: &[TxKernel],
+		height: u64,
+		batch: &Batch<'_>,
+	) -> Result<(), Error> {
+		for kernel in kernels {
+			let pos = self.apply_kernel(kernel)?;
+			let commit_pos = CommitPos { pos, height };
+			apply_kernel_rules(kernel, commit_pos, batch)?;
+		}
+		Ok(())
+	}
+
 	/// Push kernel onto MMR (hash and data files).
-	fn apply_kernel(&mut self, kernel: &TxKernel) -> Result<(), Error> {
-		self.kernel_pmmr
+	fn apply_kernel(&mut self, kernel: &TxKernel) -> Result<u64, Error> {
+		let pos = self
+			.kernel_pmmr
 			.push(kernel)
 			.map_err(&ErrorKind::TxHashSetErr)?;
-		Ok(())
+		Ok(pos)
 	}
 
 	/// Build a Merkle proof for the given output and the block
@@ -1109,7 +1214,8 @@ impl<'a> Extension<'a> {
 			let mut affected_pos = vec![];
 			let mut current = head_header;
 			while header.height < current.height {
-				let mut affected_pos_single_block = self.rewind_single_block(&current, batch)?;
+				let block = batch.get_block(&current.hash())?;
+				let mut affected_pos_single_block = self.rewind_single_block(&block, batch)?;
 				affected_pos.append(&mut affected_pos_single_block);
 				current = batch.get_previous_header(&current)?;
 			}
@@ -1126,11 +1232,10 @@ impl<'a> Extension<'a> {
 	// Rewind the MMRs and the output_pos index.
 	// Returns a vec of "affected_pos" so we can apply the necessary updates to the bitmap
 	// accumulator in a single pass for all rewound blocks.
-	fn rewind_single_block(
-		&mut self,
-		header: &BlockHeader,
-		batch: &Batch<'_>,
-	) -> Result<Vec<u64>, Error> {
+	fn rewind_single_block(&mut self, block: &Block, batch: &Batch<'_>) -> Result<Vec<u64>, Error> {
+		let header = &block.header;
+		let prev_header = batch.get_previous_header(&header)?;
+
 		// The spent index allows us to conveniently "unspend" everything in a block.
 		let spent = batch.get_spent_index(&header.hash());
 
@@ -1149,7 +1254,7 @@ impl<'a> Extension<'a> {
 		if header.height == 0 {
 			self.rewind_mmrs_to_pos(0, 0, &spent_pos)?;
 		} else {
-			let prev = batch.get_previous_header(&header)?;
+			let prev = batch.get_previous_header(header)?;
 			self.rewind_mmrs_to_pos(prev.output_mmr_size, prev.kernel_mmr_size, &spent_pos)?;
 		}
 
@@ -1160,7 +1265,6 @@ impl<'a> Extension<'a> {
 		affected_pos.push(self.output_pmmr.last_pos);
 
 		// Remove any entries from the output_pos created by the block being rewound.
-		let block = batch.get_block(&header.hash())?;
 		let mut missing_count = 0;
 		for out in block.outputs() {
 			if batch.delete_output_pos_height(&out.commitment()).is_err() {
@@ -1176,6 +1280,17 @@ impl<'a> Extension<'a> {
 			);
 		}
 
+		// If NRD feature flag is enabled rewind the kernel_pos index
+		// for any NRD kernels in the block being rewound.
+		if global::is_nrd_enabled() {
+			let kernel_index = store::nrd_recent_kernel_index();
+			for kernel in block.kernels() {
+				if let KernelFeatures::NoRecentDuplicate { .. } = kernel.features {
+					kernel_index.rewind(batch, kernel.excess(), prev_header.kernel_mmr_size)?;
+				}
+			}
+		}
+
 		// Update output_pos based on "unspending" all spent pos from this block.
 		// This is necessary to ensure the output_pos index correclty reflects a
 		// reused output commitment. For example an output at pos 1, spent, reused at pos 2.
@@ -1649,3 +1764,36 @@ fn input_pos_to_rewind(
 	}
 	Ok(bitmap)
 }
+
+/// If NRD enabled then enforce NRD relative height rules.
+fn apply_kernel_rules(kernel: &TxKernel, pos: CommitPos, batch: &Batch<'_>) -> Result<(), Error> {
+	if !global::is_nrd_enabled() {
+		return Ok(());
+	}
+	match kernel.features {
+		KernelFeatures::NoRecentDuplicate {
+			relative_height, ..
+		} => {
+			let kernel_index = store::nrd_recent_kernel_index();
+			debug!("checking NRD index: {:?}", kernel.excess());
+			if let Some(prev) = kernel_index.peek_pos(batch, kernel.excess())? {
+				let diff = pos.height.saturating_sub(prev.height);
+				debug!(
+					"NRD check: {}, {:?}, {:?}",
+					pos.height, prev, relative_height
+				);
+				if diff < relative_height.into() {
+					return Err(ErrorKind::NRDRelativeHeight.into());
+				}
+			}
+			debug!(
+				"pushing entry to NRD index: {:?}: {:?}",
+				kernel.excess(),
+				pos,
+			);
+			kernel_index.push_pos(batch, kernel.excess(), pos)?;
+		}
+		_ => {}
+	}
+	Ok(())
+}

chain/src/types.rs

@@ -303,7 +303,7 @@ impl OutputRoots {
 }
 
 /// Minimal struct representing a known MMR position and associated block height.
-#[derive(Debug)]
+#[derive(Clone, Copy, Debug, PartialEq)]
 pub struct CommitPos {
 	/// MMR position
 	pub pos: u64,