GRIN/grin-node
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[WIP] txpool (tx validation) using block sums for full validation (#1567)

Browse Source 
tx validation (txpool) rework/simplify
This commit is contained in:
Antioch Peverell
2018-09-24 09:24:10 +01:00
committed by GitHub
parent 82b785282c
commit e72d8b58e4
22 changed files with 521 additions and 712 deletions
Download Patch File Download Diff File Expand all files Collapse all files
chain/src/chain.rs
+17 -22
View File
@@ -431,6 +431,7 @@ impl Chain {
}
}
/// TODO - where do we call this from? And do we need a rewind first?
/// For the given commitment find the unspent output and return the
/// associated Return an error if the output does not exist or has been
/// spent. This querying is done in a way that is consistent with the
@@ -445,33 +446,20 @@ impl Chain {
}
}
pub fn validate_tx(&self, tx: &Transaction, header: &BlockHeader) -> Result<(), Error> {
let mut txhashset = self.txhashset.write().unwrap();
txhashset::extending_readonly(&mut txhashset, |extension| {
extension.rewind(header)?;
extension.validate_utxo_fast(tx.inputs(), tx.outputs())?;
Ok(())
})
}
fn next_block_height(&self) -> Result<u64, Error> {
let bh = self.head_header()?;
Ok(bh.height + 1)
}
/// Validate a vec of "raw" transactions against a known chain state
/// at the block with the specified block hash.
/// Specifying a "pre_tx" if we need to adjust the state, for example when
/// validating the txs in the stempool we adjust the state based on the
/// txpool.
pub fn validate_raw_txs(
&self,
txs: Vec<Transaction>,
pre_tx: Option<Transaction>,
block_hash: &Hash,
) -> Result<Vec<Transaction>, Error> {
// Get header so we can rewind chain state correctly.
let header = self.store.get_block_header(block_hash)?;
let mut txhashset = self.txhashset.write().unwrap();
txhashset::extending_readonly(&mut txhashset, |extension| {
extension.rewind(&header)?;
let valid_txs = extension.validate_raw_txs(txs, pre_tx)?;
Ok(valid_txs)
})
}
/// Verify we are not attempting to spend a coinbase output
/// that has not yet sufficiently matured.
pub fn verify_coinbase_maturity(&self, tx: &Transaction) -> Result<(), Error> {
@@ -876,6 +864,13 @@ impl Chain {
.map_err(|e| ErrorKind::StoreErr(e, "chain get header".to_owned()).into())
}
/// Get block_sums by header hash.
pub fn get_block_sums(&self, h: &Hash) -> Result<BlockSums, Error> {
self.store
.get_block_sums(h)
.map_err(|e| ErrorKind::StoreErr(e, "chain get block_sums".to_owned()).into())
}
/// Gets the block header at the provided height
pub fn get_header_by_height(&self, height: u64) -> Result<BlockHeader, Error> {
self.store
chain/src/pipe.rs
+4
View File
@@ -519,6 +519,10 @@ fn verify_coinbase_maturity(block: &Block, ext: &mut txhashset::Extension) -> Re
/// based on block_sums of previous block, accounting for the inputs|outputs|kernels
/// of the new block.
fn verify_block_sums(b: &Block, ext: &mut txhashset::Extension) -> Result<(), Error> {
// First check all our inputs exist in the current UTXO set.
// And that we are not introducing any duplicate outputs in the UTXO set.
ext.validate_utxo_fast(b.inputs(), b.outputs())?;
// Retrieve the block_sums for the previous block.
let block_sums = ext.batch.get_block_sums(&b.header.previous)?;
chain/src/store.rs
+8
View File
@@ -90,6 +90,14 @@ impl ChainStore {
self.db.exists(&to_key(BLOCK_PREFIX, &mut h.to_vec()))
}
pub fn get_block_sums(&self, bh: &Hash) -> Result<BlockSums, Error> {
option_to_not_found(
self.db
.get_ser(&to_key(BLOCK_SUMS_PREFIX, &mut bh.to_vec())),
&format!("Block sums for block: {}", bh),
)
}
pub fn get_block_header(&self, h: &Hash) -> Result<BlockHeader, Error> {
{
let mut header_cache = self.header_cache.write().unwrap();
chain/src/txhashset.rs
+45 -115
View File
@@ -29,9 +29,7 @@ use core::core::committed::Committed;
use core::core::hash::{Hash, Hashed};
use core::core::merkle_proof::MerkleProof;
use core::core::pmmr::{self, PMMR};
use core::core::{
Block, BlockHeader, Input, Output, OutputFeatures, OutputIdentifier, Transaction, TxKernel,
};
use core::core::{Block, BlockHeader, Input, Output, OutputFeatures, OutputIdentifier, TxKernel};
use core::global;
use core::ser::{PMMRIndexHashable, PMMRable};
@@ -439,115 +437,6 @@ impl<'a> Extension<'a> {
}
}
// Rewind the MMR backend to undo applying a raw tx to the txhashset extension.
// This is used during txpool validation to undo an invalid tx.
fn rewind_raw_tx(
&mut self,
output_pos: u64,
kernel_pos: u64,
rewind_rm_pos: &Bitmap,
) -> Result<(), Error> {
self.rewind_to_pos(output_pos, kernel_pos, rewind_rm_pos)?;
Ok(())
}
/// Apply a "raw" transaction to the txhashset.
/// We will never commit a txhashset extension that includes raw txs.
/// But we can use this when validating txs in the tx pool.
/// If we can add a tx to the tx pool and then successfully add the
/// aggregated tx from the tx pool to the current chain state (via a
/// txhashset extension) then we know the tx pool is valid (including the
/// new tx).
pub fn apply_raw_tx(&mut self, tx: &Transaction) -> Result<(), Error> {
// This should *never* be called on a writeable extension...
assert!(
self.rollback,
"applied raw_tx to writeable txhashset extension"
);
// Checkpoint the MMR positions before we apply the new tx,
// anything goes wrong we will rewind to these positions.
let output_pos = self.output_pmmr.unpruned_size();
let kernel_pos = self.kernel_pmmr.unpruned_size();
// Build bitmap of output pos spent (as inputs) by this tx for rewind.
let rewind_rm_pos = tx
.inputs()
.iter()
.filter_map(|x| self.batch.get_output_pos(&x.commitment()).ok())
.map(|x| x as u32)
.collect();
for ref output in tx.outputs() {
match self.apply_output(output) {
Ok(pos) => {
self.batch.save_output_pos(&output.commitment(), pos)?;
}
Err(e) => {
self.rewind_raw_tx(output_pos, kernel_pos, &rewind_rm_pos)?;
return Err(e);
}
}
}
for ref input in tx.inputs() {
if let Err(e) = self.apply_input(input) {
self.rewind_raw_tx(output_pos, kernel_pos, &rewind_rm_pos)?;
return Err(e);
}
}
for ref kernel in tx.kernels() {
if let Err(e) = self.apply_kernel(kernel) {
self.rewind_raw_tx(output_pos, kernel_pos, &rewind_rm_pos)?;
return Err(e);
}
}
Ok(())
}
/// Validate a vector of "raw" transactions against the current chain state.
/// We support rewind on a "dirty" txhashset - so we can apply each tx in
/// turn, rewinding if any particular tx is not valid and continuing
/// through the vec of txs provided. This allows us to efficiently apply
/// all the txs, filtering out those that are not valid and returning the
/// final vec of txs that were successfully validated against the txhashset.
///
/// Note: We also pass in a "pre_tx". This tx is applied to and validated
/// before we start applying the vec of txs. We use this when validating
/// txs in the stempool as we need to account for txs in the txpool as
/// well (new_tx + stempool + txpool + txhashset). So we aggregate the
/// contents of the txpool into a single aggregated tx and pass it in here
/// as the "pre_tx" so we apply it to the txhashset before we start
/// validating the stempool txs.
/// This is optional and we pass in None when validating the txpool txs
/// themselves.
///
pub fn validate_raw_txs(
&mut self,
txs: Vec<Transaction>,
pre_tx: Option<Transaction>,
) -> Result<Vec<Transaction>, Error> {
let mut valid_txs = vec![];
// First apply the "pre_tx" to account for any state that need adding to
// the chain state before we can validate our vec of txs.
// This is the aggregate tx from the txpool if we are validating the stempool.
if let Some(tx) = pre_tx {
self.apply_raw_tx(&tx)?;
}
// Now validate each tx, rewinding any tx (and only that tx)
// if it fails to validate successfully.
for tx in txs {
if self.apply_raw_tx(&tx).is_ok() {
valid_txs.push(tx);
}
}
Ok(valid_txs)
}
/// Verify we are not attempting to spend any coinbase outputs
/// that have not sufficiently matured.
pub fn verify_coinbase_maturity(
@@ -589,9 +478,25 @@ impl<'a> Extension<'a> {
Ok(())
}
/// Apply a new set of blocks on top the existing sum trees. Blocks are
/// applied in order of the provided Vec. If pruning is enabled, inputs also
/// prune MMR data.
// Inputs _must_ spend unspent outputs.
// Outputs _must not_ introduce duplicate commitments.
pub fn validate_utxo_fast(
&mut self,
inputs: &Vec<Input>,
outputs: &Vec<Output>,
) -> Result<(), Error> {
for out in outputs {
self.validate_utxo_output(out)?;
}
for input in inputs {
self.validate_utxo_input(input)?;
}
Ok(())
}
/// Apply a new block to the existing state.
pub fn apply_block(&mut self, b: &Block) -> Result<(), Error> {
for out in b.outputs() {
let pos = self.apply_output(out)?;
@@ -610,6 +515,18 @@ impl<'a> Extension<'a> {
Ok(())
}
// TODO - Is this sufficient?
fn validate_utxo_input(&mut self, input: &Input) -> Result<(), Error> {
let commit = input.commitment();
let pos_res = self.batch.get_output_pos(&commit);
if let Ok(pos) = pos_res {
if let Some(_) = self.output_pmmr.get_data(pos) {
return Ok(());
}
}
Err(ErrorKind::AlreadySpent(commit).into())
}
fn apply_input(&mut self, input: &Input) -> Result<(), Error> {
let commit = input.commitment();
let pos_res = self.batch.get_output_pos(&commit);
@@ -648,6 +565,19 @@ impl<'a> Extension<'a> {
Ok(())
}
/// TODO - Is this sufficient?
fn validate_utxo_output(&mut self, out: &Output) -> Result<(), Error> {
let commit = out.commitment();
if let Ok(pos) = self.batch.get_output_pos(&commit) {
if let Some(out_mmr) = self.output_pmmr.get_data(pos) {
if out_mmr.commitment() == commit {
return Err(ErrorKind::DuplicateCommitment(commit).into());
}
}
}
Ok(())
}
fn apply_output(&mut self, out: &Output) -> Result<(u64), Error> {
let commit = out.commitment();
chain/tests/test_txhashset.rs
+4 -108
View File
@@ -17,7 +17,6 @@ extern crate grin_core as core;
extern crate grin_keychain as keychain;
extern crate grin_store as store;
extern crate grin_util as util;
extern crate grin_wallet as wallet;
use std::collections::HashSet;
use std::fs::{self, File, OpenOptions};
@@ -27,113 +26,13 @@ use std::sync::Arc;
use chain::store::ChainStore;
use chain::txhashset;
use chain::types::Tip;
use core::core::{Block, BlockHeader};
use core::pow::Difficulty;
use keychain::{ExtKeychain, Keychain};
use core::core::BlockHeader;
use util::file;
use wallet::libtx::{build, reward};
fn clean_output_dir(dir_name: &str) {
let _ = fs::remove_dir_all(dir_name);
}
#[test]
fn test_some_raw_txs() {
let db_root = format!(".grin_txhashset_raw_txs");
clean_output_dir(&db_root);
let db_env = Arc::new(store::new_env(db_root.clone()));
let chain_store = ChainStore::new(db_env).unwrap();
let store = Arc::new(chain_store);
// open the txhashset, creating a new one if necessary
let mut txhashset = txhashset::TxHashSet::open(db_root.clone(), store.clone(), None).unwrap();
let keychain = ExtKeychain::from_random_seed().unwrap();
let key_id1 = keychain.derive_key_id(1).unwrap();
let key_id2 = keychain.derive_key_id(2).unwrap();
let key_id3 = keychain.derive_key_id(3).unwrap();
let key_id4 = keychain.derive_key_id(4).unwrap();
let key_id5 = keychain.derive_key_id(5).unwrap();
let key_id6 = keychain.derive_key_id(6).unwrap();
// Create a simple block with a single coinbase output
// so we have something to spend.
let prev_header = BlockHeader::default();
let reward_out = reward::output(&keychain, &key_id1, 0, prev_header.height).unwrap();
let block = Block::new(&prev_header, vec![], Difficulty::one(), reward_out).unwrap();
// Now apply this initial block to the (currently empty) MMRs.
// Note: this results in an output MMR with a single leaf node.
// We need to be careful with pruning while processing the txs below
// as we cannot prune a tree with a single node in it (no sibling or parent).
let mut batch = store.batch().unwrap();
txhashset::extending(&mut txhashset, &mut batch, |extension| {
extension.apply_block(&block)
}).unwrap();
// Make sure we setup the head in the store based on block we just accepted.
let head = Tip::from_block(&block.header);
batch.save_head(&head).unwrap();
batch.commit().unwrap();
let coinbase_reward = 60_000_000_000;
// tx1 spends the original coinbase output from the block
let tx1 = build::transaction(
vec![
build::coinbase_input(coinbase_reward, key_id1.clone()),
build::output(100, key_id2.clone()),
build::output(150, key_id3.clone()),
],
&keychain,
).unwrap();
// tx2 attempts to "double spend" the coinbase output from the block (conflicts
// with tx1)
let tx2 = build::transaction(
vec![
build::coinbase_input(coinbase_reward, key_id1.clone()),
build::output(100, key_id4.clone()),
],
&keychain,
).unwrap();
// tx3 spends one output from tx1
let tx3 = build::transaction(
vec![
build::input(100, key_id2.clone()),
build::output(90, key_id5.clone()),
],
&keychain,
).unwrap();
// tx4 spends the other output from tx1 and the output from tx3
let tx4 = build::transaction(
vec![
build::input(150, key_id3.clone()),
build::input(90, key_id5.clone()),
build::output(220, key_id6.clone()),
],
&keychain,
).unwrap();
// Now validate the txs against the txhashset (via a readonly extension).
// Note: we use a single txhashset extension and we can continue to
// apply txs successfully after a failure.
let _ = txhashset::extending_readonly(&mut txhashset, |extension| {
// Note: we pass in an increasing "height" here so we can rollback
// each tx individually as necessary, while maintaining a long lived
// txhashset extension.
assert!(extension.apply_raw_tx(&tx1).is_ok());
assert!(extension.apply_raw_tx(&tx2).is_err());
assert!(extension.apply_raw_tx(&tx3).is_ok());
assert!(extension.apply_raw_tx(&tx4).is_ok());
Ok(())
});
}
#[test]
fn test_unexpected_zip() {
let db_root = format!(".grin_txhashset_zip");
@@ -180,8 +79,7 @@ fn write_file(db_root: String) {
.join("txhashset")
.join("kernel")
.join("strange0"),
)
.unwrap();
).unwrap();
OpenOptions::new()
.create(true)
.write(true)
@@ -196,8 +94,7 @@ fn write_file(db_root: String) {
.join("txhashset")
.join("strange_dir")
.join("strange2"),
)
.unwrap();
).unwrap();
fs::create_dir(
Path::new(&db_root)
.join("txhashset")
@@ -213,8 +110,7 @@ fn write_file(db_root: String) {
.join("strange_dir")
.join("strange_subdir")
.join("strange3"),
)
.unwrap();
).unwrap();
}
fn txhashset_contains_expected_files(dirname: String, path_buf: PathBuf) -> bool {