Dandelion++ Rewrite (#2628)

* reworked the dandelion rewrite (dandelion++)

* fallback to fluff/broadcast if we cannot stem the tx for any reason

* rework stem vs fluff logic during accepting tx

* cleanup docs

* add is_stem to logging

* cleanup

* rustfmt

* cleanup monitor and logging

* rework dandelion monitor to use simple cutoff for aggregation

* transition to next epoch *after* processing tx
so we fluff final outstanding txs

* fluff all txs in stempool if any are older than 30s
aggressively aggregate when we can

* fix rebase onto 1.1.0

* default config comments for Dandelion

* fix code to reflect our tests - fallback to txpool on stempool error

* log fluff and expire errors in dandelion monitor

* cleanup

* fix off by one

* cleanup

* cleanup

* various fixes

* one less clone

* cleanup

pool/src/lib.rs

@@ -34,7 +34,8 @@ mod pool;
 pub mod transaction_pool;
 pub mod types;
 
+pub use crate::pool::Pool;
 pub use crate::transaction_pool::TransactionPool;
 pub use crate::types::{
-	BlockChain, DandelionConfig, PoolAdapter, PoolConfig, PoolEntryState, PoolError, TxSource,
+	BlockChain, DandelionConfig, PoolAdapter, PoolConfig, PoolEntry, PoolError, TxSource,
 };

pool/src/pool.rs

@@ -23,7 +23,7 @@ use self::core::core::{
 	Block, BlockHeader, BlockSums, Committed, Transaction, TxKernel, Weighting,
 };
 use self::util::RwLock;
-use crate::types::{BlockChain, PoolEntry, PoolEntryState, PoolError};
+use crate::types::{BlockChain, PoolEntry, PoolError};
 use grin_core as core;
 use grin_util as util;
 use std::collections::{HashMap, HashSet};
@@ -139,7 +139,7 @@ impl Pool {
 		// Verify these txs produce an aggregated tx below max tx weight.
 		// Return a vec of all the valid txs.
 		let txs = self.validate_raw_txs(
-			tx_buckets,
+			&tx_buckets,
 			None,
 			&header,
 			Weighting::AsLimitedTransaction { max_weight },
@@ -167,33 +167,6 @@ impl Pool {
 		Ok(Some(tx))
 	}
 
-	pub fn select_valid_transactions(
-		&self,
-		txs: Vec<Transaction>,
-		extra_tx: Option<Transaction>,
-		header: &BlockHeader,
-	) -> Result<Vec<Transaction>, PoolError> {
-		let valid_txs = self.validate_raw_txs(txs, extra_tx, header, Weighting::NoLimit)?;
-		Ok(valid_txs)
-	}
-
-	pub fn get_transactions_in_state(&self, state: PoolEntryState) -> Vec<Transaction> {
-		self.entries
-			.iter()
-			.filter(|x| x.state == state)
-			.map(|x| x.tx.clone())
-			.collect::<Vec<_>>()
-	}
-
-	// Transition the specified pool entries to the new state.
-	pub fn transition_to_state(&mut self, txs: &[Transaction], state: PoolEntryState) {
-		for x in &mut self.entries {
-			if txs.contains(&x.tx) {
-				x.state = state;
-			}
-		}
-	}
-
 	// Aggregate this new tx with all existing txs in the pool.
 	// If we can validate the aggregated tx against the current chain state
 	// then we can safely add the tx to the pool.
@@ -267,9 +240,9 @@ impl Pool {
 		Ok(new_sums)
 	}
 
-	fn validate_raw_txs(
+	pub fn validate_raw_txs(
 		&self,
-		txs: Vec<Transaction>,
+		txs: &[Transaction],
 		extra_tx: Option<Transaction>,
 		header: &BlockHeader,
 		weighting: Weighting,
@@ -289,7 +262,7 @@ impl Pool {
 
 			// We know the tx is valid if the entire aggregate tx is valid.
 			if self.validate_raw_tx(&agg_tx, header, weighting).is_ok() {
-				valid_txs.push(tx);
+				valid_txs.push(tx.clone());
 			}
 		}
 

pool/src/transaction_pool.rs

@@ -23,9 +23,7 @@ use self::core::core::verifier_cache::VerifierCache;
 use self::core::core::{transaction, Block, BlockHeader, Transaction, Weighting};
 use self::util::RwLock;
 use crate::pool::Pool;
-use crate::types::{
-	BlockChain, PoolAdapter, PoolConfig, PoolEntry, PoolEntryState, PoolError, TxSource,
-};
+use crate::types::{BlockChain, PoolAdapter, PoolConfig, PoolEntry, PoolError, TxSource};
 use chrono::prelude::*;
 use grin_core as core;
 use grin_util as util;
@@ -76,13 +74,10 @@ impl TransactionPool {
 		self.blockchain.chain_head()
 	}
 
+	// Add tx to stempool (passing in all txs from txpool to validate against).
 	fn add_to_stempool(&mut self, entry: PoolEntry, header: &BlockHeader) -> Result<(), PoolError> {
-		// Add tx to stempool (passing in all txs from txpool to validate against).
 		self.stempool
 			.add_to_pool(entry, self.txpool.all_transactions(), header)?;
-
-		// Note: we do not notify the adapter here,
-		// we let the dandelion monitor handle this.
 		Ok(())
 	}
 
@@ -124,8 +119,6 @@ impl TransactionPool {
 			let txpool_tx = self.txpool.all_transactions_aggregate()?;
 			self.stempool.reconcile(txpool_tx, header)?;
 		}
-
-		self.adapter.tx_accepted(&entry.tx);
 		Ok(())
 	}
 
@@ -159,28 +152,25 @@ impl TransactionPool {
 		self.blockchain.verify_coinbase_maturity(&tx)?;
 
 		let entry = PoolEntry {
-			state: PoolEntryState::Fresh,
 			src,
 			tx_at: Utc::now(),
 			tx,
 		};
 
-		// If we are in "stem" mode then check if this is a new tx or if we have seen it before.
-		// If new tx - add it to our stempool.
-		// If we have seen any of the kernels before then fallback to fluff,
-		// adding directly to txpool.
-		if stem
-			&& self
-				.stempool
-				.find_matching_transactions(entry.tx.kernels())
-				.is_empty()
+		// If not stem then we are fluff.
+		// If this is a stem tx then attempt to stem.
+		// Any problems during stem, fallback to fluff.
+		if !stem
+			|| self
+				.add_to_stempool(entry.clone(), header)
+				.and_then(|_| self.adapter.stem_tx_accepted(&entry.tx))
+				.is_err()
 		{
-			self.add_to_stempool(entry, header)?;
-			return Ok(());
+			self.add_to_txpool(entry.clone(), header)?;
+			self.add_to_reorg_cache(entry.clone());
+			self.adapter.tx_accepted(&entry.tx);
 		}
 
-		self.add_to_txpool(entry.clone(), header)?;
-		self.add_to_reorg_cache(entry);
 		Ok(())
 	}
 

pool/src/types.rs

@@ -27,62 +27,61 @@ use failure::Fail;
 use grin_core as core;
 use grin_keychain as keychain;
 
-/// Dandelion relay timer
-const DANDELION_RELAY_SECS: u64 = 600;
+/// Dandelion "epoch" length.
+const DANDELION_EPOCH_SECS: u16 = 600;
 
-/// Dandelion embargo timer
-const DANDELION_EMBARGO_SECS: u64 = 180;
+/// Dandelion embargo timer.
+const DANDELION_EMBARGO_SECS: u16 = 180;
 
-/// Dandelion patience timer
-const DANDELION_PATIENCE_SECS: u64 = 10;
+/// Dandelion aggregation timer.
+const DANDELION_AGGREGATION_SECS: u16 = 30;
 
 /// Dandelion stem probability (stem 90% of the time, fluff 10%).
-const DANDELION_STEM_PROBABILITY: usize = 90;
+const DANDELION_STEM_PROBABILITY: u8 = 90;
 
 /// Configuration for "Dandelion".
 /// Note: shared between p2p and pool.
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
 pub struct DandelionConfig {
-	/// Choose new Dandelion relay peer every n secs.
-	#[serde = "default_dandelion_relay_secs"]
-	pub relay_secs: Option<u64>,
-	/// Dandelion embargo, fluff and broadcast tx if not seen on network before
-	/// embargo expires.
-	#[serde = "default_dandelion_embargo_secs"]
-	pub embargo_secs: Option<u64>,
-	/// Dandelion patience timer, fluff/stem processing runs every n secs.
-	/// Tx aggregation happens on stem txs received within this window.
-	#[serde = "default_dandelion_patience_secs"]
-	pub patience_secs: Option<u64>,
+	/// Length of each "epoch".
+	#[serde(default = "default_dandelion_epoch_secs")]
+	pub epoch_secs: Option<u16>,
+	/// Dandelion embargo timer. Fluff and broadcast individual txs if not seen
+	/// on network before embargo expires.
+	#[serde(default = "default_dandelion_embargo_secs")]
+	pub embargo_secs: Option<u16>,
+	/// Dandelion aggregation timer.
+	#[serde(default = "default_dandelion_aggregation_secs")]
+	pub aggregation_secs: Option<u16>,
 	/// Dandelion stem probability (stem 90% of the time, fluff 10% etc.)
-	#[serde = "default_dandelion_stem_probability"]
-	pub stem_probability: Option<usize>,
+	#[serde(default = "default_dandelion_stem_probability")]
+	pub stem_probability: Option<u8>,
 }
 
 impl Default for DandelionConfig {
 	fn default() -> DandelionConfig {
 		DandelionConfig {
-			relay_secs: default_dandelion_relay_secs(),
+			epoch_secs: default_dandelion_epoch_secs(),
 			embargo_secs: default_dandelion_embargo_secs(),
-			patience_secs: default_dandelion_patience_secs(),
+			aggregation_secs: default_dandelion_aggregation_secs(),
 			stem_probability: default_dandelion_stem_probability(),
 		}
 	}
 }
 
-fn default_dandelion_relay_secs() -> Option<u64> {
-	Some(DANDELION_RELAY_SECS)
+fn default_dandelion_epoch_secs() -> Option<u16> {
+	Some(DANDELION_EPOCH_SECS)
 }
 
-fn default_dandelion_embargo_secs() -> Option<u64> {
+fn default_dandelion_embargo_secs() -> Option<u16> {
 	Some(DANDELION_EMBARGO_SECS)
 }
 
-fn default_dandelion_patience_secs() -> Option<u64> {
-	Some(DANDELION_PATIENCE_SECS)
+fn default_dandelion_aggregation_secs() -> Option<u16> {
+	Some(DANDELION_AGGREGATION_SECS)
 }
 
-fn default_dandelion_stem_probability() -> Option<usize> {
+fn default_dandelion_stem_probability() -> Option<u8> {
 	Some(DANDELION_STEM_PROBABILITY)
 }
 
@@ -138,8 +137,6 @@ fn default_mineable_max_weight() -> usize {
 /// A single (possibly aggregated) transaction.
 #[derive(Clone, Debug)]
 pub struct PoolEntry {
-	/// The state of the pool entry.
-	pub state: PoolEntryState,
 	/// Info on where this tx originated from.
 	pub src: TxSource,
 	/// Timestamp of when this tx was originally added to the pool.
@@ -148,21 +145,6 @@ pub struct PoolEntry {
 	pub tx: Transaction,
 }
 
-/// The possible states a pool entry can be in.
-#[derive(Clone, Copy, Debug, PartialEq)]
-pub enum PoolEntryState {
-	/// A new entry, not yet processed.
-	Fresh,
-	/// Tx to be included in the next "stem" run.
-	ToStem,
-	/// Tx previously "stemmed" and propagated.
-	Stemmed,
-	/// Tx to be included in the next "fluff" run.
-	ToFluff,
-	/// Tx previously "fluffed" and broadcast.
-	Fluffed,
-}
-
 /// Placeholder: the data representing where we heard about a tx from.
 ///
 /// Used to make decisions based on transaction acceptance priority from
@@ -267,12 +249,10 @@ pub trait BlockChain: Sync + Send {
 /// downstream processing of valid transactions by the rest of the system, most
 /// importantly the broadcasting of transactions to our peers.
 pub trait PoolAdapter: Send + Sync {
-	/// The transaction pool has accepted this transactions as valid and added
-	/// it to its internal cache.
+	/// The transaction pool has accepted this transaction as valid.
 	fn tx_accepted(&self, tx: &transaction::Transaction);
-	/// The stem transaction pool has accepted this transactions as valid and
-	/// added it to its internal cache, we have waited for the "patience" timer
-	/// to fire and we now want to propagate the tx to the next Dandelion relay.
+
+	/// The stem transaction pool has accepted this transactions as valid.
 	fn stem_tx_accepted(&self, tx: &transaction::Transaction) -> Result<(), PoolError>;
 }
 
@@ -281,9 +261,8 @@ pub trait PoolAdapter: Send + Sync {
 pub struct NoopAdapter {}
 
 impl PoolAdapter for NoopAdapter {
-	fn tx_accepted(&self, _: &transaction::Transaction) {}
-
-	fn stem_tx_accepted(&self, _: &transaction::Transaction) -> Result<(), PoolError> {
+	fn tx_accepted(&self, _tx: &transaction::Transaction) {}
+	fn stem_tx_accepted(&self, _tx: &transaction::Transaction) -> Result<(), PoolError> {
 		Ok(())
 	}
 }