Minimal Transaction Pool (#1067)

* verify a tx like we verify a block (experimental)

* first minimal_pool test up and running but not testing what we need to

* rework tx_pool validation to use txhashset extension

* minimal tx pool wired up but rough

* works locally (rough statew though)
delete "legacy" pool and graph code

* rework the new pool into TransactionPool and Pool impls

* rework pool to store pool entries
with associated timer and source etc.

* all_transactions

* extra_txs so we can validate stempool against existing txpool

* rework reconcile_block

* txhashset apply_raw_tx can now rewind to a checkpoint (prev raw tx)

* wip - txhashset tx tests

* more flexible rewind on MMRs

* add tests to cover apply_raw_txs on txhashset extension

* add_to_stempool and add_to_txpool

* deaggregate multi kernel tx when adding to txpoool

* handle freshness in stempool
handle propagation of stempool txs via dandelion monitor

* patience timer and fluff if we cannot propagate
to next relay

* aggregate and fluff stempool is we have no relay

* refactor coinbase maturity

* rewrote basic tx pool tests to use a real txhashset via chain adapter

* rework dandelion monitor to reflect recent discussion
works locally but needs a cleanup

* refactor dandelion_monitor - split out phases

* more pool test coverage

* remove old test code from pool (still wip)

* block_building and block_reconciliation tests

* tracked down chain test failure...

* fix test_coinbase_maturity

* dandelion_monitor now runs...

* refactor dandelion config, shared across p2p and pool components

* fix pool tests with new config

* fix p2p tests

* rework tx pool to deal with duplicate commitments (testnet2 limitation)

* cleanup and address some PR feedback

* add big comment about pre_tx...

store/src/pmmr.rs

@@ -17,8 +17,8 @@ use std::fs;
 use std::io;
 use std::marker;
 
-use core::core::pmmr::{self, family, Backend};
 use core::core::hash::Hash;
+use core::core::pmmr::{self, family, Backend};
 use core::ser;
 use core::ser::PMMRable;
 use types::*;
@@ -272,15 +272,6 @@ where
 		self.get_data_file_path()
 	}
 
-	/// Return last written buffer positions for the hash file and the data file
-	// pub fn last_file_positions(&self) -> PMMRFileMetadata {
-	// 	PMMRFileMetadata {
-	// 		block_height: 0,
-	// 		last_hash_file_pos: self.hash_file.last_buffer_pos() as u64,
-	// 		last_data_file_pos: self.data_file.last_buffer_pos() as u64,
-	// 	}
-	// }
-
 	/// Checks the length of the remove log to see if it should get compacted.
 	/// If so, the remove log is flushed into the pruned list, which itself gets
 	/// saved, and the hash and data files are rewritten, cutting the removed

store/src/types.rs

@@ -16,15 +16,15 @@ use memmap;
 
 use std::cmp;
 use std::fs::{self, File, OpenOptions};
+use std::io::Read;
 use std::io::{self, BufRead, BufReader, BufWriter, ErrorKind, Write};
 use std::os::unix::io::AsRawFd;
-use std::io::Read;
 use std::path::Path;
 
-#[cfg(any(target_os = "linux"))]
-use libc::{ftruncate64, off64_t};
 #[cfg(not(any(target_os = "linux", target_os = "android")))]
 use libc::{ftruncate as ftruncate64, off_t as off64_t};
+#[cfg(any(target_os = "linux"))]
+use libc::{ftruncate64, off64_t};
 
 use core::ser;
 
@@ -82,12 +82,25 @@ impl AppendOnlyFile {
 
 	/// Rewinds the data file back to a lower position. The new position needs
 	/// to be the one of the first byte the next time data is appended.
-	pub fn rewind(&mut self, pos: u64) {
-		if self.buffer_start_bak > 0 || self.buffer.len() > 0 {
-			panic!("Can't rewind on a dirty state.");
+	/// Supports two scenarios currently -
+	///   * rewind from a clean state (rewinding to handle a forked block)
+	///   * rewind within the buffer itself (raw_tx fails to validate)
+	/// Note: we do not currently support a rewind() that
+	/// crosses the buffer boundary.
+	pub fn rewind(&mut self, file_pos: u64) {
+		if self.buffer.is_empty() {
+			// rewinding from clean state, no buffer, not already rewound anything
+			self.buffer_start_bak = self.buffer_start;
+			self.buffer_start = file_pos as usize;
+		} else {
+			// rewinding (within) the buffer
+			if self.buffer_start as u64 > file_pos {
+				panic!("cannot rewind buffer beyond buffer_start");
+			} else {
+				let buffer_len = file_pos - self.buffer_start as u64;
+				self.buffer.truncate(buffer_len as usize);
+			}
 		}
-		self.buffer_start_bak = self.buffer_start;
-		self.buffer_start = pos as usize;
 	}
 
 	/// Syncs all writes (fsync), reallocating the memory map to make the newly
@@ -263,16 +276,18 @@ impl RemoveLog {
 	/// In practice the index is a block height, so we rewind back to that block
 	/// keeping everything in the rm_log up to and including that block.
 	pub fn rewind(&mut self, idx: u32) -> io::Result<()> {
-		// simplifying assumption: we always remove older than what's in tmp
-		self.removed_tmp = vec![];
-		// backing it up before truncating
-		self.removed_bak = self.removed.clone();
+		// backing it up before truncating (unless we already have a backup)
+		if self.removed_bak.is_empty() {
+			self.removed_bak = self.removed.clone();
+		}
 
 		if idx == 0 {
 			self.removed = vec![];
+			self.removed_tmp = vec![];
 		} else {
 			// retain rm_log entries up to and including those at the provided index
 			self.removed.retain(|&(_, x)| x <= idx);
+			self.removed_tmp.retain(|&(_, x)| x <= idx);
 		}
 		Ok(())
 	}