GRIN/grin-node
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rework "bucket transactions" logic (buckets are now weight limited) (#2487)

Browse Source 
* rework bucket txs

* introduce max_block_weight to global (based on chain_type)
not yet in use anywhere...

* now using global::max_block_weight() everywhere

* testing max_block_weight needs to be big enough to cover existing chain tests...

* add some test coverage for prepare mineable transactions at the block weight limit
introduce AsLimitedTx{max_weight} so we can build artifically small blocks (per mining config)

* cleanup

* default_mineable_max_weight is just the default max_block_weight
we do not need to account for coinbase reward here (tx vs block)

* 75 change outputs in a test is not valid now that we have a low block weight limit...
This commit is contained in:
Antioch Peverell
2019-02-01 10:44:04 +00:00
committed by GitHub
parent 062d41766e
commit a82e2a0126
8 changed files with 290 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pool/src/pool.rs
+88 -41
View File
@@ -123,35 +123,27 @@ impl Pool {
max_weight: usize,
) -> Result<Vec<Transaction>, PoolError> {
let header = self.blockchain.chain_head()?;
let tx_buckets = self.bucket_transactions();
let mut tx_buckets = self.bucket_transactions(max_weight);
// flatten buckets using aggregate (with cut-through)
let mut flat_txs: Vec<Transaction> = tx_buckets
.into_iter()
.filter_map(|bucket| transaction::aggregate(bucket).ok())
.filter(|x| {
// Here we validate the tx, subject to regular tx weight limits.
x.validate(Weighting::AsTransaction, self.verifier_cache.clone())
.is_ok()
})
.collect();
// At this point we know that all "buckets" are valid and that
// there are no dependencies between them.
// This allows us to arbitrarily sort them and filter them safely.
// sort by fees over weight, multiplying by 1000 to keep some precision
// don't think we'll ever see a >max_u64/1000 fee transaction
flat_txs.sort_unstable_by_key(|tx| tx.fee() * 1000 / tx.tx_weight() as u64);
// accumulate as long as we're not above the block weight
let mut weight = 0;
flat_txs.retain(|tx| {
weight += tx.tx_weight_as_block() as usize;
weight < max_weight
});
// Sort them by fees over weight, multiplying by 1000 to keep some precision
// don't think we'll ever see a >max_u64/1000 fee transaction.
// We want to select the txs with highest fee per unit of weight first.
tx_buckets.sort_unstable_by_key(|tx| tx.fee() * 1000 / tx.tx_weight() as u64);
// Iteratively apply the txs to the current chain state,
// rejecting any that do not result in a valid state.
// Verify these txs produce an aggregated tx below max tx weight.
// Return a vec of all the valid txs.
let txs = self.validate_raw_txs(flat_txs, None, &header, Weighting::AsTransaction)?;
let txs = self.validate_raw_txs(
tx_buckets,
None,
&header,
Weighting::AsLimitedTransaction { max_weight },
)?;
Ok(txs)
}
@@ -345,36 +337,91 @@ impl Pool {
Ok(())
}
// Group dependent transactions in buckets (vectors), each bucket
// is therefore independent from the others. Relies on the entries
// Vec having parent transactions first (should always be the case)
fn bucket_transactions(&self) -> Vec<Vec<Transaction>> {
// Group dependent transactions in buckets (aggregated txs).
// Each bucket is independent from the others. Relies on the entries
// vector having parent transactions first (should always be the case).
fn bucket_transactions(&self, max_weight: usize) -> Vec<Transaction> {
let mut tx_buckets = vec![];
let mut output_commits = HashMap::new();
let mut rejected = HashSet::new();
for entry in &self.entries {
// check the commits index to find parents and their position
// picking the last one for bucket (so all parents come first)
let mut insert_pos: i32 = -1;
// if single parent then we are good, we can bucket it with its parent
// if multiple parents then we need to combine buckets, but for now simply reject it (rare case)
let mut insert_pos = None;
let mut is_rejected = false;
for input in entry.tx.inputs() {
if let Some(pos) = output_commits.get(&input.commitment()) {
if *pos > insert_pos {
insert_pos = *pos;
if rejected.contains(&input.commitment()) {
// Depends on a rejected tx, so reject this one.
is_rejected = true;
continue;
} else if let Some(pos) = output_commits.get(&input.commitment()) {
if insert_pos.is_some() {
// Multiple dependencies so reject this tx (pick it up in next block).
is_rejected = true;
continue;
} else {
// Track the pos of the bucket we fall into.
insert_pos = Some(*pos);
}
}
}
if insert_pos == -1 {
// no parent, just add to the end in its own bucket
insert_pos = tx_buckets.len() as i32;
tx_buckets.push(vec![entry.tx.clone()]);
} else {
// parent found, add to its bucket
tx_buckets[insert_pos as usize].push(entry.tx.clone());
// If this tx is rejected then store all output commitments in our rejected set.
if is_rejected {
for out in entry.tx.outputs() {
rejected.insert(out.commitment());
}
// Done with this entry (rejected), continue to next entry.
continue;
}
// update the commits index
for out in entry.tx.outputs() {
output_commits.insert(out.commitment(), insert_pos);
match insert_pos {
None => {
// No parent tx, just add to the end in its own bucket.
// This is the common case for non 0-conf txs in the txpool.
// We assume the tx is valid here as we validated it on the way into the txpool.
insert_pos = Some(tx_buckets.len());
tx_buckets.push(entry.tx.clone());
}
Some(pos) => {
// We found a single parent tx, so aggregate in the bucket
// if the aggregate tx is a valid tx.
// Otherwise discard and let the next block pick this tx up.
let current = tx_buckets[pos].clone();
if let Ok(agg_tx) = transaction::aggregate(vec![current, entry.tx.clone()]) {
if agg_tx
.validate(
Weighting::AsLimitedTransaction { max_weight },
self.verifier_cache.clone(),
)
.is_ok()
{
tx_buckets[pos] = agg_tx;
} else {
// Aggregated tx is not valid so discard this new tx.
is_rejected = true;
}
} else {
// Aggregation failed so discard this new tx.
is_rejected = true;
}
}
}
if is_rejected {
for out in entry.tx.outputs() {
rejected.insert(out.commitment());
}
} else if let Some(insert_pos) = insert_pos {
// We successfully added this tx to our set of buckets.
// Update commits index for subsequent txs.
for out in entry.tx.outputs() {
output_commits.insert(out.commitment(), insert_pos);
}
}
}
tx_buckets
pool/src/types.rs
+2 -2
View File
@@ -17,12 +17,12 @@
use chrono::prelude::{DateTime, Utc};
use self::core::consensus;
use self::core::core::block;
use self::core::core::committed;
use self::core::core::hash::Hash;
use self::core::core::transaction::{self, Transaction};
use self::core::core::{BlockHeader, BlockSums};
use self::core::{consensus, global};
use grin_core as core;
use grin_keychain as keychain;
@@ -130,7 +130,7 @@ fn default_max_stempool_size() -> usize {
50_000
}
fn default_mineable_max_weight() -> usize {
consensus::MAX_BLOCK_WEIGHT - consensus::BLOCK_OUTPUT_WEIGHT - consensus::BLOCK_KERNEL_WEIGHT
global::max_block_weight()
}
/// Represents a single entry in the pool.
pool/tests/block_max_weight.rs
+143
View File
@@ -0,0 +1,143 @@
// Copyright 2018 The Grin Developers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Test coverage for block building at the limit of max_block_weight.
pub mod common;
use self::core::core::hash::Hashed;
use self::core::core::verifier_cache::LruVerifierCache;
use self::core::core::{Block, BlockHeader, Transaction};
use self::core::global;
use self::core::libtx;
use self::core::pow::Difficulty;
use self::keychain::{ExtKeychain, Keychain};
use self::util::RwLock;
use crate::common::*;
use grin_core as core;
use grin_keychain as keychain;
use grin_util as util;
use std::sync::Arc;
#[test]
fn test_block_building_max_weight() {
util::init_test_logger();
global::set_mining_mode(global::ChainTypes::AutomatedTesting);
let keychain: ExtKeychain = Keychain::from_random_seed(false).unwrap();
let db_root = ".grin_block_building_max_weight".to_string();
clean_output_dir(db_root.clone());
let mut chain = ChainAdapter::init(db_root.clone()).unwrap();
let verifier_cache = Arc::new(RwLock::new(LruVerifierCache::new()));
// Convenient was to add a new block to the chain.
let add_block = |prev_header: BlockHeader, txs: Vec<Transaction>, chain: &mut ChainAdapter| {
let height = prev_header.height + 1;
let key_id = ExtKeychain::derive_key_id(1, height as u32, 0, 0, 0);
let fee = txs.iter().map(|x| x.fee()).sum();
let reward = libtx::reward::output(&keychain, &key_id, fee).unwrap();
let mut block = Block::new(&prev_header, txs, Difficulty::min(), reward).unwrap();
// Set the prev_root to the prev hash for testing purposes (no MMR to obtain a root from).
block.header.prev_root = prev_header.hash();
chain.update_db_for_block(&block);
block
};
// Initialize the chain/txhashset with an initial block
// so we have a non-empty UTXO set.
let block = add_block(BlockHeader::default(), vec![], &mut chain);
let header = block.header;
// Now create tx to spend that first coinbase (now matured).
// Provides us with some useful outputs to test with.
let initial_tx = test_transaction_spending_coinbase(&keychain, &header, vec![100, 200, 300]);
// Mine that initial tx so we can spend it with multiple txs
let block = add_block(header, vec![initial_tx], &mut chain);
let header = block.header;
// Initialize a new pool with our chain adapter.
let pool = RwLock::new(test_setup(Arc::new(chain.clone()), verifier_cache));
// Build some dependent txs to add to the txpool.
// We will build a block from a subset of these.
let txs = vec![
test_transaction(&keychain, vec![100], vec![90, 1]),
test_transaction(&keychain, vec![90], vec![80, 2]),
test_transaction(&keychain, vec![200], vec![199]),
test_transaction(&keychain, vec![300], vec![290, 3]),
test_transaction(&keychain, vec![290], vec![280, 4]),
];
// Populate our txpool with the txs.
{
let mut write_pool = pool.write();
for tx in txs {
write_pool
.add_to_pool(test_source(), tx, false, &header)
.unwrap();
}
}
// Check we added them all to the txpool successfully.
assert_eq!(pool.read().total_size(), 5);
// Prepare some "mineable txs" from the txpool.
// Note: We cannot fit all the txs from the txpool into a block.
let txs = pool.read().prepare_mineable_transactions().unwrap();
// Check resulting tx aggregation is what we expect.
// We expect to produce 2 aggregated txs based on txpool contents.
assert_eq!(txs.len(), 2);
// Check the tx we built is the aggregation of the correct set of underlying txs.
// We included 4 out of the 5 txs here.
assert_eq!(txs[0].kernels().len(), 1);
assert_eq!(txs[1].kernels().len(), 2);
// Check our weights after aggregation.
assert_eq!(txs[0].inputs().len(), 1);
assert_eq!(txs[0].outputs().len(), 1);
assert_eq!(txs[0].kernels().len(), 1);
assert_eq!(txs[0].tx_weight_as_block(), 25);
assert_eq!(txs[1].inputs().len(), 1);
assert_eq!(txs[1].outputs().len(), 3);
assert_eq!(txs[1].kernels().len(), 2);
assert_eq!(txs[1].tx_weight_as_block(), 70);
let block = add_block(header, txs, &mut chain);
// Check contents of the block itself (including coinbase reward).
assert_eq!(block.inputs().len(), 2);
assert_eq!(block.outputs().len(), 5);
assert_eq!(block.kernels().len(), 4);
// Now reconcile the transaction pool with the new block
// and check the resulting contents of the pool are what we expect.
{
let mut write_pool = pool.write();
write_pool.reconcile_block(&block).unwrap();
// We should still have 2 tx in the pool after accepting the new block.
// This one exceeded the max block weight when building the block so
// remained in the txpool.
assert_eq!(write_pool.total_size(), 2);
}
}