Minimal Transaction Pool (#1067)

* verify a tx like we verify a block (experimental) * first minimal_pool test up and running but not testing what we need to * rework tx_pool validation to use txhashset extension * minimal tx pool wired up but rough * works locally (rough statew though) delete "legacy" pool and graph code * rework the new pool into TransactionPool and Pool impls * rework pool to store pool entries with associated timer and source etc. * all_transactions * extra_txs so we can validate stempool against existing txpool * rework reconcile_block * txhashset apply_raw_tx can now rewind to a checkpoint (prev raw tx) * wip - txhashset tx tests * more flexible rewind on MMRs * add tests to cover apply_raw_txs on txhashset extension * add_to_stempool and add_to_txpool * deaggregate multi kernel tx when adding to txpoool * handle freshness in stempool handle propagation of stempool txs via dandelion monitor * patience timer and fluff if we cannot propagate to next relay * aggregate and fluff stempool is we have no relay * refactor coinbase maturity * rewrote basic tx pool tests to use a real txhashset via chain adapter * rework dandelion monitor to reflect recent discussion works locally but needs a cleanup * refactor dandelion_monitor - split out phases * more pool test coverage * remove old test code from pool (still wip) * block_building and block_reconciliation tests * tracked down chain test failure... * fix test_coinbase_maturity * dandelion_monitor now runs... * refactor dandelion config, shared across p2p and pool components * fix pool tests with new config * fix p2p tests * rework tx pool to deal with duplicate commitments (testnet2 limitation) * cleanup and address some PR feedback * add big comment about pre_tx...
2018-05-30 16:57:13 -04:00
parent b6c31ebc78
commit 4fda7a6899
61 changed files with 2265 additions and 3569 deletions
@@ -15,28 +15,28 @@
 //! Adapters connecting new block, new transaction, and accepted transaction
 //! events to consumers of those events.

+use rand;
+use rand::Rng;
 use std::fs::File;
 use std::net::SocketAddr;
 use std::ops::Deref;
-use std::sync::{Arc, RwLock, Weak};
 use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, RwLock, Weak};
 use std::thread;
 use std::time::Instant;
-use rand;
-use rand::Rng;

 use chain::{self, ChainAdapter, Options, Tip};
+use common::types::{ChainValidationMode, ServerConfig};
 use core::core;
 use core::core::block::BlockHeader;
 use core::core::hash::{Hash, Hashed};
 use core::core::target::Difficulty;
-use core::core::transaction::{Input, OutputIdentifier};
+use core::core::transaction::Transaction;
 use p2p;
 use pool;
-use util::OneTime;
 use store;
-use common::types::{ChainValidationMode, ServerConfig};
 use util::LOGGER;
+use util::OneTime;

 // All adapters use `Weak` references instead of `Arc` to avoid cycles that
 // can never be destroyed. These 2 functions are simple helpers to reduce the
@@ -75,35 +75,23 @@ impl p2p::ChainAdapter for NetToChainAdapter {
 			debug_name: "p2p".to_string(),
 			identifier: "?.?.?.?".to_string(),
 		};
+
 		debug!(
 			LOGGER,
-			"Received tx {} from {}, going to process.",
+			"Received tx {} from {:?}, going to process.",
 			tx.hash(),
-			source.identifier,
+			source,
 		);

 		let h = tx.hash();

-		if !stem && tx.kernels.len() != 1 {
-			debug!(
-				LOGGER,
-				"Received regular multi-kernel transaction will attempt to deaggregate"
-			);
-			if let Err(e) = self.tx_pool
-				.write()
-				.unwrap()
-				.deaggregate_and_add_to_memory_pool(source, tx, stem)
-			{
-				debug!(LOGGER, "Transaction {} rejected: {:?}", h, e);
-			}
-		} else {
-			if let Err(e) = self.tx_pool
-				.write()
-				.unwrap()
-				.add_to_memory_pool(source, tx, stem)
-			{
-				debug!(LOGGER, "Transaction {} rejected: {:?}", h, e);
-			}
+		let res = {
+			let mut tx_pool = self.tx_pool.write().unwrap();
+			tx_pool.add_to_pool(source, tx, stem)
+		};
+
+		if let Err(e) = res {
+			debug!(LOGGER, "Transaction {} rejected: {:?}", h, e);
 		}
 	}

@@ -635,8 +623,8 @@ impl ChainToPoolAndNetAdapter {
 		}
 	}

-	/// Initialize a ChainToPoolAndNetAdapter instance with hanlde to a Peers object.
-	/// Should only be called once.
+	/// Initialize a ChainToPoolAndNetAdapter instance with hanlde to a Peers
+	/// object. Should only be called once.
 	pub fn init(&self, peers: Weak<p2p::Peers>) {
 		self.peers.init(peers);
 	}
@@ -649,8 +637,11 @@ pub struct PoolToNetAdapter {
 }

 impl pool::PoolAdapter for PoolToNetAdapter {
-	fn stem_tx_accepted(&self, tx: &core::Transaction) {
-		wo(&self.peers).broadcast_stem_transaction(tx);
+	fn stem_tx_accepted(&self, tx: &core::Transaction) -> Result<(), pool::PoolError> {
+		wo(&self.peers)
+			.broadcast_stem_transaction(tx)
+			.map_err(|_| pool::PoolError::DandelionError)?;
+		Ok(())
 	}
 	fn tx_accepted(&self, tx: &core::Transaction) {
 		wo(&self.peers).broadcast_transaction(tx);
@@ -694,26 +685,25 @@ impl PoolToChainAdapter {
 }

 impl pool::BlockChain for PoolToChainAdapter {
-	fn is_unspent(&self, output_ref: &OutputIdentifier) -> Result<Hash, pool::PoolError> {
-		wo(&self.chain).is_unspent(output_ref).map_err(|e| match e {
-			chain::types::Error::OutputNotFound => pool::PoolError::OutputNotFound,
-			chain::types::Error::OutputSpent => pool::PoolError::OutputSpent,
-			_ => pool::PoolError::GenericPoolError,
+	fn validate_raw_txs(
+		&self,
+		txs: Vec<Transaction>,
+		pre_tx: Option<Transaction>,
+	) -> Result<(Vec<Transaction>), pool::PoolError> {
+		wo(&self.chain).validate_raw_txs(txs, pre_tx).map_err(|_| {
+			pool::PoolError::Other("Chain adapter failed to validate_raw_txs.".to_string())
 		})
 	}

-	fn is_matured(&self, input: &Input, height: u64) -> Result<(), pool::PoolError> {
+	fn verify_coinbase_maturity(&self, tx: &Transaction) -> Result<(), pool::PoolError> {
 		wo(&self.chain)
-			.is_matured(input, height)
-			.map_err(|e| match e {
-				chain::types::Error::OutputNotFound => pool::PoolError::OutputNotFound,
-				_ => pool::PoolError::GenericPoolError,
-			})
+			.verify_coinbase_maturity(tx)
+			.map_err(|_| pool::PoolError::ImmatureCoinbase)
 	}

-	fn head_header(&self) -> Result<BlockHeader, pool::PoolError> {
+	fn verify_tx_lock_height(&self, tx: &Transaction) -> Result<(), pool::PoolError> {
 		wo(&self.chain)
-			.head_header()
-			.map_err(|_| pool::PoolError::GenericPoolError)
+			.verify_tx_lock_height(tx)
+			.map_err(|_| pool::PoolError::ImmatureTransaction)
 	}
 }
@@ -19,12 +19,24 @@ use std::convert::From;
 use api;
 use chain;
 use core::core;
+use core::global::ChainTypes;
+use core::pow;
 use p2p;
 use pool;
 use store;
 use wallet;
-use core::global::ChainTypes;
-use core::pow;
+
+/// Dandelion relay timer
+const DANDELION_RELAY_SECS: u64 = 600;
+
+/// Dandelion emabargo timer
+const DANDELION_EMBARGO_SECS: u64 = 180;
+
+/// Dandelion patience timer
+const DANDELION_PATIENCE_SECS: u64 = 10;
+
+/// Dandelion stem probability (stem 90% of the time, fluff 10%).
+const DANDELION_STEM_PROBABILITY: usize = 90;

 /// Error type wrapping underlying module errors.
 #[derive(Debug)]
@@ -123,6 +135,54 @@ impl Default for Seeding {
 	}
 }

+fn default_dandelion_stem_probability() -> usize {
+	DANDELION_STEM_PROBABILITY
+}
+
+fn default_dandelion_relay_secs() -> u64 {
+	DANDELION_RELAY_SECS
+}
+
+fn default_dandelion_embargo_secs() -> u64 {
+	DANDELION_EMBARGO_SECS
+}
+
+fn default_dandelion_patience_secs() -> u64 {
+	DANDELION_PATIENCE_SECS
+}
+
+/// Dandelion config.
+/// Note: Used by both p2p and pool components.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct DandelionConfig {
+	/// Choose new Dandelion relay peer every n secs.
+	#[serde = "default_dandelion_relay_secs"]
+	pub relay_secs: u64,
+	/// Dandelion embargo, fluff and broadcast tx if not seen on network before
+	/// embargo expires.
+	#[serde = "default_dandelion_embargo_secs"]
+	pub embargo_secs: u64,
+	/// Dandelion patience timer, fluff/stem processing runs every n secs.
+	/// Tx aggregation happens on stem txs received within this window.
+	#[serde = "default_dandelion_patience_secs"]
+	pub patience_secs: u64,
+	/// Dandelion stem probability.
+	#[serde = "default_dandelion_stem_probability"]
+	pub stem_probability: usize,
+}
+
+/// Default address for peer-to-peer connections.
+impl Default for DandelionConfig {
+	fn default() -> DandelionConfig {
+		DandelionConfig {
+			relay_secs: default_dandelion_relay_secs(),
+			embargo_secs: default_dandelion_embargo_secs(),
+			patience_secs: default_dandelion_patience_secs(),
+			stem_probability: default_dandelion_stem_probability(),
+		}
+	}
+}
+
 /// Full server configuration, aggregating configurations required for the
 /// different components.
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -167,6 +227,10 @@ pub struct ServerConfig {
 	#[serde(default)]
 	pub pool_config: pool::PoolConfig,

+	/// Dandelion configuration
+	#[serde(default)]
+	pub dandelion_config: DandelionConfig,
+
 	/// Whether to skip the sync timeout on startup
 	/// (To assist testing on solo chains)
 	pub skip_sync_wait: Option<bool>,
@@ -185,6 +249,28 @@ pub struct ServerConfig {
 	pub test_miner_wallet_url: Option<String>,
 }

+impl ServerConfig {
+	/// Adapter for configuring Dandelion on the pool component.
+	pub fn pool_dandelion_config(&self) -> pool::DandelionConfig {
+		pool::DandelionConfig {
+			relay_secs: self.dandelion_config.relay_secs,
+			embargo_secs: self.dandelion_config.embargo_secs,
+			patience_secs: self.dandelion_config.patience_secs,
+			stem_probability: self.dandelion_config.stem_probability,
+		}
+	}
+
+	/// Adapter for configuring Dandelion on the p2p component.
+	pub fn p2p_dandelion_config(&self) -> p2p::DandelionConfig {
+		p2p::DandelionConfig {
+			relay_secs: self.dandelion_config.relay_secs,
+			embargo_secs: self.dandelion_config.embargo_secs,
+			patience_secs: self.dandelion_config.patience_secs,
+			stem_probability: self.dandelion_config.stem_probability,
+		}
+	}
+}
+
 impl Default for ServerConfig {
 	fn default() -> ServerConfig {
 		ServerConfig {
@@ -194,6 +280,7 @@ impl Default for ServerConfig {
 			seeding_type: Seeding::default(),
 			seeds: None,
 			p2p_config: p2p::P2PConfig::default(),
+			dandelion_config: DandelionConfig::default(),
 			stratum_mining_config: Some(StratumServerConfig::default()),
 			chain_type: ChainTypes::default(),
 			archive_mode: None,