Dandelion++ Rewrite (#2628)

* reworked the dandelion rewrite (dandelion++)

* fallback to fluff/broadcast if we cannot stem the tx for any reason

* rework stem vs fluff logic during accepting tx

* cleanup docs

* add is_stem to logging

* cleanup

* rustfmt

* cleanup monitor and logging

* rework dandelion monitor to use simple cutoff for aggregation

* transition to next epoch *after* processing tx
so we fluff final outstanding txs

* fluff all txs in stempool if any are older than 30s
aggressively aggregate when we can

* fix rebase onto 1.1.0

* default config comments for Dandelion

* fix code to reflect our tests - fallback to txpool on stempool error

* log fluff and expire errors in dandelion monitor

* cleanup

* fix off by one

* cleanup

* cleanup

* various fixes

* one less clone

* cleanup
This commit is contained in:
Antioch Peverell
2019-03-20 13:08:56 +00:00
committed by GitHub
parent 16487a3eb7
commit a2adf2dfe8
12 changed files with 335 additions and 374 deletions
+63 -9
View File
@@ -23,7 +23,9 @@ use std::time::Instant;
use crate::chain::{self, BlockStatus, ChainAdapter, Options};
use crate::common::hooks::{ChainEvents, NetEvents};
use crate::common::types::{self, ChainValidationMode, ServerConfig, SyncState, SyncStatus};
use crate::common::types::{
self, ChainValidationMode, DandelionEpoch, ServerConfig, SyncState, SyncStatus,
};
use crate::core::core::hash::{Hash, Hashed};
use crate::core::core::transaction::Transaction;
use crate::core::core::verifier_cache::VerifierCache;
@@ -33,6 +35,7 @@ use crate::core::{core, global};
use crate::p2p;
use crate::p2p::types::PeerAddr;
use crate::pool;
use crate::pool::types::DandelionConfig;
use crate::util::OneTime;
use chrono::prelude::*;
use chrono::Duration;
@@ -685,26 +688,77 @@ impl ChainToPoolAndNetAdapter {
/// transactions that have been accepted.
pub struct PoolToNetAdapter {
peers: OneTime<Weak<p2p::Peers>>,
dandelion_epoch: Arc<RwLock<DandelionEpoch>>,
}
/// Adapter between the Dandelion monitor and the current Dandelion "epoch".
pub trait DandelionAdapter: Send + Sync {
/// Is the node stemming (or fluffing) transactions in the current epoch?
fn is_stem(&self) -> bool;
/// Is the current Dandelion epoch expired?
fn is_expired(&self) -> bool;
/// Transition to the next Dandelion epoch (new stem/fluff state, select new relay peer).
fn next_epoch(&self);
}
impl DandelionAdapter for PoolToNetAdapter {
fn is_stem(&self) -> bool {
self.dandelion_epoch.read().is_stem()
}
fn is_expired(&self) -> bool {
self.dandelion_epoch.read().is_expired()
}
fn next_epoch(&self) {
self.dandelion_epoch.write().next_epoch(&self.peers());
}
}
impl pool::PoolAdapter for PoolToNetAdapter {
fn stem_tx_accepted(&self, tx: &core::Transaction) -> Result<(), pool::PoolError> {
self.peers()
.relay_stem_transaction(tx)
.map_err(|_| pool::PoolError::DandelionError)?;
Ok(())
}
fn tx_accepted(&self, tx: &core::Transaction) {
self.peers().broadcast_transaction(tx);
}
fn stem_tx_accepted(&self, tx: &core::Transaction) -> Result<(), pool::PoolError> {
// Take write lock on the current epoch.
// We need to be able to update the current relay peer if not currently connected.
let mut epoch = self.dandelion_epoch.write();
// If "stem" epoch attempt to relay the tx to the next Dandelion relay.
// Fallback to immediately fluffing the tx if we cannot stem for any reason.
// If "fluff" epoch then nothing to do right now (fluff via Dandelion monitor).
if epoch.is_stem() {
if let Some(peer) = epoch.relay_peer(&self.peers()) {
match peer.send_stem_transaction(tx) {
Ok(_) => {
info!("Stemming this epoch, relaying to next peer.");
Ok(())
}
Err(e) => {
error!("Stemming tx failed. Fluffing. {:?}", e);
Err(pool::PoolError::DandelionError)
}
}
} else {
error!("No relay peer. Fluffing.");
Err(pool::PoolError::DandelionError)
}
} else {
info!("Fluff epoch. Aggregating stem tx(s). Will fluff via Dandelion monitor.");
Ok(())
}
}
}
impl PoolToNetAdapter {
/// Create a new pool to net adapter
pub fn new() -> PoolToNetAdapter {
pub fn new(config: DandelionConfig) -> PoolToNetAdapter {
PoolToNetAdapter {
peers: OneTime::new(),
dandelion_epoch: Arc::new(RwLock::new(DandelionEpoch::new(config))),
}
}
+96 -2
View File
@@ -13,18 +13,21 @@
// limitations under the License.
//! Server types
use crate::util::RwLock;
use std::convert::From;
use std::sync::Arc;
use chrono::prelude::{DateTime, Utc};
use rand::prelude::*;
use crate::api;
use crate::chain;
use crate::core::global::ChainTypes;
use crate::core::{core, pow};
use crate::p2p;
use crate::pool;
use crate::pool::types::DandelionConfig;
use crate::store;
use chrono::prelude::{DateTime, Utc};
use crate::util::RwLock;
/// Error type wrapping underlying module errors.
#[derive(Debug)]
@@ -437,3 +440,94 @@ impl chain::TxHashsetWriteStatus for SyncState {
self.update(SyncStatus::TxHashsetDone);
}
}
/// A node is either "stem" of "fluff" for the duration of a single epoch.
/// A node also maintains an outbound relay peer for the epoch.
#[derive(Debug)]
pub struct DandelionEpoch {
config: DandelionConfig,
// When did this epoch start?
start_time: Option<i64>,
// Are we in "stem" mode or "fluff" mode for this epoch?
is_stem: bool,
// Our current Dandelion relay peer (effective for this epoch).
relay_peer: Option<Arc<p2p::Peer>>,
}
impl DandelionEpoch {
/// Create a new Dandelion epoch, defaulting to "stem" and no outbound relay peer.
pub fn new(config: DandelionConfig) -> DandelionEpoch {
DandelionEpoch {
config,
start_time: None,
is_stem: true,
relay_peer: None,
}
}
/// Is the current Dandelion epoch expired?
/// It is expired if start_time is older than the configured epoch_secs.
pub fn is_expired(&self) -> bool {
match self.start_time {
None => true,
Some(start_time) => {
let epoch_secs = self.config.epoch_secs.expect("epoch_secs config missing") as i64;
Utc::now().timestamp().saturating_sub(start_time) > epoch_secs
}
}
}
/// Transition to next Dandelion epoch.
/// Select stem/fluff based on configured stem_probability.
/// Choose a new outbound stem relay peer.
pub fn next_epoch(&mut self, peers: &Arc<p2p::Peers>) {
self.start_time = Some(Utc::now().timestamp());
self.relay_peer = peers.outgoing_connected_peers().first().cloned();
// If stem_probability == 90 then we stem 90% of the time.
let mut rng = rand::thread_rng();
let stem_probability = self
.config
.stem_probability
.expect("stem_probability config missing");
self.is_stem = rng.gen_range(0, 100) < stem_probability;
let addr = self.relay_peer.clone().map(|p| p.info.addr);
info!(
"DandelionEpoch: next_epoch: is_stem: {} ({}%), relay: {:?}",
self.is_stem, stem_probability, addr
);
}
/// Are we stemming (or fluffing) transactions in this epoch?
pub fn is_stem(&self) -> bool {
self.is_stem
}
/// What is our current relay peer?
/// If it is not connected then choose a new one.
pub fn relay_peer(&mut self, peers: &Arc<p2p::Peers>) -> Option<Arc<p2p::Peer>> {
let mut update_relay = false;
if let Some(peer) = &self.relay_peer {
if !peer.is_connected() {
info!(
"DandelionEpoch: relay_peer: {:?} not connected, choosing a new one.",
peer.info.addr
);
update_relay = true;
}
} else {
update_relay = true;
}
if update_relay {
self.relay_peer = peers.outgoing_connected_peers().first().cloned();
info!(
"DandelionEpoch: relay_peer: new peer chosen: {:?}",
self.relay_peer.clone().map(|p| p.info.addr)
);
}
self.relay_peer.clone()
}
}
+103 -169
View File
@@ -12,17 +12,18 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::util::{Mutex, RwLock, StopState};
use chrono::prelude::Utc;
use rand::{thread_rng, Rng};
use std::sync::Arc;
use std::thread;
use std::time::Duration;
use crate::common::adapters::DandelionAdapter;
use crate::core::core::hash::Hashed;
use crate::core::core::transaction;
use crate::core::core::verifier_cache::VerifierCache;
use crate::pool::{DandelionConfig, PoolEntryState, PoolError, TransactionPool, TxSource};
use crate::pool::{DandelionConfig, Pool, PoolEntry, PoolError, TransactionPool, TxSource};
use crate::util::{Mutex, RwLock, StopState};
/// A process to monitor transactions in the stempool.
/// With Dandelion, transaction can be broadcasted in stem or fluff phase.
@@ -35,6 +36,7 @@ use crate::pool::{DandelionConfig, PoolEntryState, PoolError, TransactionPool, T
pub fn monitor_transactions(
dandelion_config: DandelionConfig,
tx_pool: Arc<RwLock<TransactionPool>>,
adapter: Arc<DandelionAdapter>,
verifier_cache: Arc<RwLock<dyn VerifierCache>>,
stop_state: Arc<Mutex<StopState>>,
) {
@@ -44,211 +46,143 @@ pub fn monitor_transactions(
.name("dandelion".to_string())
.spawn(move || {
loop {
// Halt Dandelion monitor if we have been notified that we are stopping.
if stop_state.lock().is_stopped() {
break;
}
// This is the patience timer, we loop every n secs.
let patience_secs = dandelion_config.patience_secs.unwrap();
thread::sleep(Duration::from_secs(patience_secs));
// Step 1: find all "ToStem" entries in stempool from last run.
// Aggregate them up to give a single (valid) aggregated tx and propagate it
// to the next Dandelion relay along the stem.
if process_stem_phase(tx_pool.clone(), verifier_cache.clone()).is_err() {
error!("dand_mon: Problem with stem phase.");
if !adapter.is_stem() {
let _ =
process_fluff_phase(&dandelion_config, &tx_pool, &adapter, &verifier_cache)
.map_err(|e| {
error!("dand_mon: Problem processing fluff phase. {:?}", e);
});
}
// Step 2: find all "ToFluff" entries in stempool from last run.
// Aggregate them up to give a single (valid) aggregated tx and (re)add it
// to our pool with stem=false (which will then broadcast it).
if process_fluff_phase(tx_pool.clone(), verifier_cache.clone()).is_err() {
error!("dand_mon: Problem with fluff phase.");
// Now find all expired entries based on embargo timer.
let _ = process_expired_entries(&dandelion_config, &tx_pool).map_err(|e| {
error!("dand_mon: Problem processing expired entries. {:?}", e);
});
// Handle the tx above *before* we transition to next epoch.
// This gives us an opportunity to do the final "fluff" before we start
// stemming on the subsequent epoch.
if adapter.is_expired() {
adapter.next_epoch();
}
// Step 3: now find all "Fresh" entries in stempool since last run.
// Coin flip for each (90/10) and label them as either "ToStem" or "ToFluff".
// We will process these in the next run (waiting patience secs).
if process_fresh_entries(dandelion_config.clone(), tx_pool.clone()).is_err() {
error!("dand_mon: Problem processing fresh pool entries.");
}
// Step 4: now find all expired entries based on embargo timer.
if process_expired_entries(dandelion_config.clone(), tx_pool.clone()).is_err() {
error!("dand_mon: Problem processing fresh pool entries.");
}
// Monitor loops every 10s.
thread::sleep(Duration::from_secs(10));
}
});
}
fn process_stem_phase(
tx_pool: Arc<RwLock<TransactionPool>>,
verifier_cache: Arc<RwLock<dyn VerifierCache>>,
) -> Result<(), PoolError> {
let mut tx_pool = tx_pool.write();
let header = tx_pool.chain_head()?;
let stem_txs = tx_pool
.stempool
.get_transactions_in_state(PoolEntryState::ToStem);
if stem_txs.is_empty() {
return Ok(());
}
// Get the aggregate tx representing the entire txpool.
let txpool_tx = tx_pool.txpool.all_transactions_aggregate()?;
let stem_txs = tx_pool
.stempool
.select_valid_transactions(stem_txs, txpool_tx, &header)?;
tx_pool
.stempool
.transition_to_state(&stem_txs, PoolEntryState::Stemmed);
if stem_txs.len() > 0 {
debug!("dand_mon: Found {} txs for stemming.", stem_txs.len());
let agg_tx = transaction::aggregate(stem_txs)?;
agg_tx.validate(
transaction::Weighting::AsTransaction,
verifier_cache.clone(),
)?;
let res = tx_pool.adapter.stem_tx_accepted(&agg_tx);
if res.is_err() {
debug!("dand_mon: Unable to propagate stem tx. No relay, fluffing instead.");
let src = TxSource {
debug_name: "no_relay".to_string(),
identifier: "?.?.?.?".to_string(),
};
tx_pool.add_to_pool(src, agg_tx, false, &header)?;
}
}
Ok(())
// Query the pool for transactions older than the cutoff.
// Used for both periodic fluffing and handling expired embargo timer.
fn select_txs_cutoff(pool: &Pool, cutoff_secs: u16) -> Vec<PoolEntry> {
let cutoff = Utc::now().timestamp() - cutoff_secs as i64;
pool.entries
.iter()
.filter(|x| x.tx_at.timestamp() < cutoff)
.cloned()
.collect()
}
fn process_fluff_phase(
tx_pool: Arc<RwLock<TransactionPool>>,
verifier_cache: Arc<RwLock<dyn VerifierCache>>,
dandelion_config: &DandelionConfig,
tx_pool: &Arc<RwLock<TransactionPool>>,
adapter: &Arc<DandelionAdapter>,
verifier_cache: &Arc<RwLock<dyn VerifierCache>>,
) -> Result<(), PoolError> {
// Take a write lock on the txpool for the duration of this processing.
let mut tx_pool = tx_pool.write();
let header = tx_pool.chain_head()?;
let stem_txs = tx_pool
.stempool
.get_transactions_in_state(PoolEntryState::ToFluff);
if stem_txs.is_empty() {
let all_entries = tx_pool.stempool.entries.clone();
if all_entries.is_empty() {
return Ok(());
}
// Get the aggregate tx representing the entire txpool.
let txpool_tx = tx_pool.txpool.all_transactions_aggregate()?;
let cutoff_secs = dandelion_config
.aggregation_secs
.expect("aggregation secs config missing");
let cutoff_entries = select_txs_cutoff(&tx_pool.stempool, cutoff_secs);
let stem_txs = tx_pool
.stempool
.select_valid_transactions(stem_txs, txpool_tx, &header)?;
tx_pool
.stempool
.transition_to_state(&stem_txs, PoolEntryState::Fluffed);
if stem_txs.len() > 0 {
debug!("dand_mon: Found {} txs for fluffing.", stem_txs.len());
let agg_tx = transaction::aggregate(stem_txs)?;
agg_tx.validate(
transaction::Weighting::AsTransaction,
verifier_cache.clone(),
)?;
let src = TxSource {
debug_name: "fluff".to_string(),
identifier: "?.?.?.?".to_string(),
};
tx_pool.add_to_pool(src, agg_tx, false, &header)?;
// If epoch is expired, fluff *all* outstanding entries in stempool.
// If *any* entry older than aggregation_secs (30s) then fluff *all* entries.
// Otherwise we are done for now and we can give txs more time to aggregate.
if !adapter.is_expired() && cutoff_entries.is_empty() {
return Ok(());
}
Ok(())
}
fn process_fresh_entries(
dandelion_config: DandelionConfig,
tx_pool: Arc<RwLock<TransactionPool>>,
) -> Result<(), PoolError> {
let mut tx_pool = tx_pool.write();
let header = tx_pool.chain_head()?;
let mut rng = thread_rng();
let fluffable_txs = {
let txpool_tx = tx_pool.txpool.all_transactions_aggregate()?;
let txs: Vec<_> = all_entries.into_iter().map(|x| x.tx).collect();
tx_pool.stempool.validate_raw_txs(
&txs,
txpool_tx,
&header,
transaction::Weighting::NoLimit,
)?
};
let fresh_entries = &mut tx_pool
.stempool
.entries
.iter_mut()
.filter(|x| x.state == PoolEntryState::Fresh)
.collect::<Vec<_>>();
debug!(
"dand_mon: Found {} txs in local stempool to fluff",
fluffable_txs.len()
);
if fresh_entries.len() > 0 {
debug!(
"dand_mon: Found {} fresh entries in stempool.",
fresh_entries.len()
);
let agg_tx = transaction::aggregate(fluffable_txs)?;
agg_tx.validate(
transaction::Weighting::AsTransaction,
verifier_cache.clone(),
)?;
for x in &mut fresh_entries.iter_mut() {
let random = rng.gen_range(0, 101);
if random <= dandelion_config.stem_probability.unwrap() {
x.state = PoolEntryState::ToStem;
} else {
x.state = PoolEntryState::ToFluff;
}
}
}
let src = TxSource {
debug_name: "fluff".to_string(),
identifier: "?.?.?.?".to_string(),
};
tx_pool.add_to_pool(src, agg_tx, false, &header)?;
Ok(())
}
fn process_expired_entries(
dandelion_config: DandelionConfig,
tx_pool: Arc<RwLock<TransactionPool>>,
dandelion_config: &DandelionConfig,
tx_pool: &Arc<RwLock<TransactionPool>>,
) -> Result<(), PoolError> {
let now = Utc::now().timestamp();
let embargo_sec = dandelion_config.embargo_secs.unwrap() + thread_rng().gen_range(0, 31);
let cutoff = now - embargo_sec as i64;
// Take a write lock on the txpool for the duration of this processing.
let mut tx_pool = tx_pool.write();
let mut expired_entries = vec![];
{
let tx_pool = tx_pool.read();
for entry in tx_pool
.stempool
.entries
.iter()
.filter(|x| x.tx_at.timestamp() < cutoff)
{
debug!("dand_mon: Embargo timer expired for {:?}", entry.tx.hash());
expired_entries.push(entry.clone());
}
let embargo_secs = dandelion_config
.embargo_secs
.expect("embargo_secs config missing")
+ thread_rng().gen_range(0, 31);
let expired_entries = select_txs_cutoff(&tx_pool.stempool, embargo_secs);
if expired_entries.is_empty() {
return Ok(());
}
if expired_entries.len() > 0 {
debug!("dand_mon: Found {} expired txs.", expired_entries.len());
debug!("dand_mon: Found {} expired txs.", expired_entries.len());
{
let mut tx_pool = tx_pool.write();
let header = tx_pool.chain_head()?;
let header = tx_pool.chain_head()?;
for entry in expired_entries {
let src = TxSource {
debug_name: "embargo_expired".to_string(),
identifier: "?.?.?.?".to_string(),
};
match tx_pool.add_to_pool(src, entry.tx, false, &header) {
Ok(_) => debug!("dand_mon: embargo expired, fluffed tx successfully."),
Err(e) => debug!("dand_mon: Failed to fluff expired tx - {:?}", e),
};
}
}
let src = TxSource {
debug_name: "embargo_expired".to_string(),
identifier: "?.?.?.?".to_string(),
};
for entry in expired_entries {
let txhash = entry.tx.hash();
match tx_pool.add_to_pool(src.clone(), entry.tx, false, &header) {
Ok(_) => info!(
"dand_mon: embargo expired for {}, fluffed successfully.",
txhash
),
Err(e) => warn!("dand_mon: failed to fluff expired tx {}, {:?}", txhash, e),
};
}
Ok(())
}
-19
View File
@@ -29,7 +29,6 @@ use crate::core::global;
use crate::p2p;
use crate::p2p::types::PeerAddr;
use crate::p2p::ChainAdapter;
use crate::pool::DandelionConfig;
use crate::util::{Mutex, StopState};
// DNS Seeds with contact email associated
@@ -52,7 +51,6 @@ const FLOONET_DNS_SEEDS: &'static [&'static str] = &[
pub fn connect_and_monitor(
p2p_server: Arc<p2p::Server>,
capabilities: p2p::Capabilities,
dandelion_config: DandelionConfig,
seed_list: Box<dyn Fn() -> Vec<PeerAddr> + Send>,
preferred_peers: Option<Vec<PeerAddr>>,
stop_state: Arc<Mutex<StopState>>,
@@ -119,8 +117,6 @@ pub fn connect_and_monitor(
preferred_peers.clone(),
);
update_dandelion_relay(peers.clone(), dandelion_config.clone());
prev = Utc::now();
start_attempt = cmp::min(6, start_attempt + 1);
}
@@ -244,21 +240,6 @@ fn monitor_peers(
}
}
fn update_dandelion_relay(peers: Arc<p2p::Peers>, dandelion_config: DandelionConfig) {
// Dandelion Relay Updater
let dandelion_relay = peers.get_dandelion_relay();
if let Some((last_added, _)) = dandelion_relay {
let dandelion_interval = Utc::now().timestamp() - last_added;
if dandelion_interval >= dandelion_config.relay_secs.unwrap() as i64 {
debug!("monitor_peers: updating expired dandelion relay");
peers.update_dandelion_relay();
}
} else {
debug!("monitor_peers: no dandelion relay updating");
peers.update_dandelion_relay();
}
}
// Check if we have any pre-existing peer in db. If so, start with those,
// otherwise use the seeds provided.
fn connect_to_seeds_and_preferred_peers(
+4 -2
View File
@@ -154,7 +154,7 @@ impl Server {
let verifier_cache = Arc::new(RwLock::new(LruVerifierCache::new()));
let pool_adapter = Arc::new(PoolToChainAdapter::new());
let pool_net_adapter = Arc::new(PoolToNetAdapter::new());
let pool_net_adapter = Arc::new(PoolToNetAdapter::new(config.dandelion_config.clone()));
let tx_pool = Arc::new(RwLock::new(pool::TransactionPool::new(
config.pool_config.clone(),
pool_adapter.clone(),
@@ -207,6 +207,8 @@ impl Server {
genesis.hash(),
stop_state.clone(),
)?);
// Initialize various adapters with our dynamic set of connected peers.
chain_adapter.init(p2p_server.peers.clone());
pool_net_adapter.init(p2p_server.peers.clone());
net_adapter.init(p2p_server.peers.clone());
@@ -227,7 +229,6 @@ impl Server {
seed::connect_and_monitor(
p2p_server.clone(),
config.p2p_config.capabilities,
config.dandelion_config.clone(),
seeder,
config.p2p_config.peers_preferred.clone(),
stop_state.clone(),
@@ -281,6 +282,7 @@ impl Server {
dandelion_monitor::monitor_transactions(
config.dandelion_config.clone(),
tx_pool.clone(),
pool_net_adapter.clone(),
verifier_cache.clone(),
stop_state.clone(),
);