From a0b9aa5967294859985cd14009699d1d9cfabf8d Mon Sep 17 00:00:00 2001 From: Antioch Peverell Date: Wed, 12 Dec 2018 09:19:36 +0000 Subject: [PATCH 01/16] Sign kernel features (#2104) * include kernel features in msg being signed hash the msg before signing it (for consistent 32 bytes) * rustfmt * fix various tests * no HF for this (mainnet only) --- .gitignore | 1 + core/src/core/hash.rs | 7 +++- core/src/core/id.rs | 4 +-- core/src/core/transaction.rs | 69 +++++++++++++++++++----------------- core/src/libtx/aggsig.rs | 8 ++--- core/src/libtx/reward.rs | 2 +- core/src/libtx/slate.rs | 9 +++-- core/tests/block.rs | 6 +++- wallet/tests/libwallet.rs | 34 ++++++++++-------- 9 files changed, 81 insertions(+), 59 deletions(-) diff --git a/.gitignore b/.gitignore index 44cdbf52..c2a6954d 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ grin.log wallet.seed test_output wallet_data +wallet/db diff --git a/core/src/core/hash.rs b/core/src/core/hash.rs index 318963db..4f38c403 100644 --- a/core/src/core/hash.rs +++ b/core/src/core/hash.rs @@ -34,7 +34,7 @@ pub const ZERO_HASH: Hash = Hash([0; 32]); /// A hash to uniquely (or close enough) identify one of the main blockchain /// constructs. Used pervasively for blocks, transactions and outputs. #[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Hash, Serialize, Deserialize)] -pub struct Hash(pub [u8; 32]); +pub struct Hash([u8; 32]); impl fmt::Debug for Hash { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -71,6 +71,11 @@ impl Hash { self.0.to_vec() } + /// Returns a byte slice of the hash contents. + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } + /// Convert a hash to hex string format. pub fn to_hex(&self) -> String { util::to_hex(self.to_vec()) diff --git a/core/src/core/id.rs b/core/src/core/id.rs index b4ab8be9..99ea8643 100644 --- a/core/src/core/id.rs +++ b/core/src/core/id.rs @@ -51,8 +51,8 @@ impl ShortIdentifiable for H { use std::hash::Hasher; // extract k0/k1 from the block_hash - let k0 = LittleEndian::read_u64(&hash_with_nonce.0[0..8]); - let k1 = LittleEndian::read_u64(&hash_with_nonce.0[8..16]); + let k0 = LittleEndian::read_u64(&hash_with_nonce.as_bytes()[0..8]); + let k1 = LittleEndian::read_u64(&hash_with_nonce.as_bytes()[8..16]); // initialize a siphasher24 with k0/k1 let mut sip_hasher = SipHasher24::new_with_keys(k0, k1); diff --git a/core/src/core/transaction.rs b/core/src/core/transaction.rs index ffd539c4..b046453f 100644 --- a/core/src/core/transaction.rs +++ b/core/src/core/transaction.rs @@ -18,6 +18,7 @@ use crate::consensus; use crate::core::hash::Hashed; use crate::core::verifier_cache::VerifierCache; use crate::core::{committed, Committed}; +use crate::global; use crate::keychain::{self, BlindingFactor}; use crate::ser::{ self, read_multi, FixedLength, PMMRable, Readable, Reader, VerifySortedAndUnique, Writeable, @@ -46,6 +47,13 @@ bitflags! { } } +impl Writeable for KernelFeatures { + fn write(&self, writer: &mut W) -> Result<(), ser::Error> { + writer.write_u8(self.bits())?; + Ok(()) + } +} + /// Errors thrown by Transaction validation #[derive(Clone, Eq, Debug, PartialEq)] pub enum Error { @@ -163,13 +171,9 @@ impl ::std::hash::Hash for TxKernel { impl Writeable for TxKernel { fn write(&self, writer: &mut W) -> Result<(), ser::Error> { - ser_multiwrite!( - writer, - [write_u8, self.features.bits()], - [write_u64, self.fee], - [write_u64, self.lock_height], - [write_fixed_bytes, &self.excess] - ); + self.features.write(writer)?; + ser_multiwrite!(writer, [write_u64, self.fee], [write_u64, self.lock_height]); + self.excess.write(writer)?; self.excess_sig.write(writer)?; Ok(()) } @@ -207,7 +211,7 @@ impl TxKernel { /// The msg signed as part of the tx kernel. /// Consists of the fee and the lock_height. pub fn msg_to_sign(&self) -> Result { - let msg = kernel_sig_msg(self.fee, self.lock_height)?; + let msg = kernel_sig_msg(self.fee, self.lock_height, self.features)?; Ok(msg) } @@ -272,29 +276,14 @@ pub struct TxKernelEntry { impl Writeable for TxKernelEntry { fn write(&self, writer: &mut W) -> Result<(), ser::Error> { - ser_multiwrite!( - writer, - [write_u8, self.kernel.features.bits()], - [write_u64, self.kernel.fee], - [write_u64, self.kernel.lock_height], - [write_fixed_bytes, &self.kernel.excess] - ); - self.kernel.excess_sig.write(writer)?; + self.kernel.write(writer)?; Ok(()) } } impl Readable for TxKernelEntry { - fn read(reader: &mut dyn Reader) -> Result { - let features = - KernelFeatures::from_bits(reader.read_u8()?).ok_or(ser::Error::CorruptedData)?; - let kernel = TxKernel { - features: features, - fee: reader.read_u64()?, - lock_height: reader.read_u64()?, - excess: Commitment::read(reader)?, - excess_sig: secp::Signature::read(reader)?, - }; + fn read(reader: &mut Reader) -> Result { + let kernel = TxKernel::read(reader)?; Ok(TxKernelEntry { kernel }) } } @@ -1293,12 +1282,28 @@ impl From for OutputIdentifier { } } -/// Construct msg from tx fee and lock_height. -pub fn kernel_sig_msg(fee: u64, lock_height: u64) -> Result { - let mut bytes = [0; 32]; - BigEndian::write_u64(&mut bytes[16..24], fee); - BigEndian::write_u64(&mut bytes[24..], lock_height); - let msg = secp::Message::from_slice(&bytes)?; +/// Construct msg from tx fee, lock_height and kernel features. +/// In testnet4 we did not include the kernel features in the message being signed. +/// In mainnet we changed this to include features and we hash (fee || lock_height || features) +/// to produce a 32 byte message to sign. +/// +/// testnet4: msg = (fee || lock_height) +/// mainnet: msg = hash(fee || lock_height || features) +/// +pub fn kernel_sig_msg( + fee: u64, + lock_height: u64, + features: KernelFeatures, +) -> Result { + let msg = if global::is_mainnet() { + let hash = (fee, lock_height, features).hash(); + secp::Message::from_slice(&hash.as_bytes())? + } else { + let mut bytes = [0; 32]; + BigEndian::write_u64(&mut bytes[16..24], fee); + BigEndian::write_u64(&mut bytes[24..], lock_height); + secp::Message::from_slice(&bytes)? + }; Ok(msg) } diff --git a/core/src/libtx/aggsig.rs b/core/src/libtx/aggsig.rs index 6baf3c8f..6815e834 100644 --- a/core/src/libtx/aggsig.rs +++ b/core/src/libtx/aggsig.rs @@ -229,7 +229,7 @@ pub fn verify_partial_sig( /// use util::secp::key::{PublicKey, SecretKey}; /// use util::secp::{ContextFlag, Secp256k1}; /// use core::libtx::{aggsig, proof}; -/// use core::core::transaction::kernel_sig_msg; +/// use core::core::transaction::{kernel_sig_msg, KernelFeatures}; /// use core::core::{Output, OutputFeatures}; /// use keychain::{Keychain, ExtKeychain}; /// @@ -248,7 +248,7 @@ pub fn verify_partial_sig( /// let height = 20; /// let over_commit = secp.commit_value(reward(fees)).unwrap(); /// let out_commit = output.commitment(); -/// let msg = kernel_sig_msg(0, height).unwrap(); +/// let msg = kernel_sig_msg(0, height, KernelFeatures::DEFAULT_KERNEL).unwrap(); /// let excess = secp.commit_sum(vec![out_commit], vec![over_commit]).unwrap(); /// let pubkey = excess.to_pubkey(&secp).unwrap(); /// let sig = aggsig::sign_from_key_id(&secp, &keychain, &msg, &key_id, Some(&pubkey)).unwrap(); @@ -301,7 +301,7 @@ where /// use core::libtx::{aggsig, proof}; /// use util::secp::key::{PublicKey, SecretKey}; /// use util::secp::{ContextFlag, Secp256k1}; -/// use core::core::transaction::kernel_sig_msg; +/// use core::core::transaction::{kernel_sig_msg, KernelFeatures}; /// use core::core::{Output, OutputFeatures}; /// use keychain::{Keychain, ExtKeychain}; /// @@ -321,7 +321,7 @@ where /// let height = 20; /// let over_commit = secp.commit_value(reward(fees)).unwrap(); /// let out_commit = output.commitment(); -/// let msg = kernel_sig_msg(0, height).unwrap(); +/// let msg = kernel_sig_msg(0, height, KernelFeatures::DEFAULT_KERNEL).unwrap(); /// let excess = secp.commit_sum(vec![out_commit], vec![over_commit]).unwrap(); /// let pubkey = excess.to_pubkey(&secp).unwrap(); /// let sig = aggsig::sign_from_key_id(&secp, &keychain, &msg, &key_id, Some(&pubkey)).unwrap(); diff --git a/core/src/libtx/reward.rs b/core/src/libtx/reward.rs index 8daea68f..094696ab 100644 --- a/core/src/libtx/reward.rs +++ b/core/src/libtx/reward.rs @@ -58,7 +58,7 @@ where // not the lock_height of the tx (there is no tx for a coinbase output). // This output will not be spendable earlier than lock_height (and we sign this // here). - let msg = kernel_sig_msg(0, height)?; + let msg = kernel_sig_msg(0, height, KernelFeatures::COINBASE_KERNEL)?; let sig = aggsig::sign_from_key_id(&secp, keychain, &msg, &key_id, Some(&pubkey))?; let proof = TxKernel { diff --git a/core/src/libtx/slate.rs b/core/src/libtx/slate.rs index 77893d23..fd9551da 100644 --- a/core/src/libtx/slate.rs +++ b/core/src/libtx/slate.rs @@ -17,9 +17,9 @@ use crate::blake2::blake2b::blake2b; use crate::core::committed::Committed; -use crate::core::transaction::kernel_sig_msg; +use crate::core::transaction::{kernel_sig_msg, KernelFeatures, Transaction}; use crate::core::verifier_cache::LruVerifierCache; -use crate::core::{amount_to_hr_string, Transaction}; +use crate::core::amount_to_hr_string; use crate::keychain::{BlindSum, BlindingFactor, Keychain}; use crate::libtx::error::{Error, ErrorKind}; use crate::libtx::{aggsig, build, tx_fee}; @@ -159,7 +159,10 @@ impl Slate { // This is the msg that we will sign as part of the tx kernel. // Currently includes the fee and the lock_height. fn msg_to_sign(&self) -> Result { - let msg = kernel_sig_msg(self.fee, self.lock_height)?; + // Currently we only support interactively creating a tx with a "default" kernel. + let features = KernelFeatures::DEFAULT_KERNEL; + + let msg = kernel_sig_msg(self.fee, self.lock_height, features)?; Ok(msg) } diff --git a/core/tests/block.rs b/core/tests/block.rs index b0bb3755..9766a341 100644 --- a/core/tests/block.rs +++ b/core/tests/block.rs @@ -18,6 +18,7 @@ use crate::core::consensus::{BLOCK_OUTPUT_WEIGHT, MAX_BLOCK_WEIGHT}; use crate::core::core::block::Error; use crate::core::core::hash::Hashed; use crate::core::core::id::ShortIdentifiable; +use crate::core::core::transaction; use crate::core::core::verifier_cache::{LruVerifierCache, VerifierCache}; use crate::core::core::Committed; use crate::core::core::{Block, BlockHeader, CompactBlock, KernelFeatures, OutputFeatures}; @@ -193,14 +194,17 @@ fn remove_coinbase_kernel_flag() { .features .remove(KernelFeatures::COINBASE_KERNEL); + // Flipping the coinbase flag results in kernels not summing correctly. assert_eq!( b.verify_coinbase(), Err(Error::Secp(secp::Error::IncorrectCommitSum)) ); + // Also results in the block no longer validating correctly + // because the message being signed on each tx kernel includes the kernel features. assert_eq!( b.validate(&BlindingFactor::zero(), verifier_cache()), - Err(Error::Secp(secp::Error::IncorrectCommitSum)) + Err(Error::Transaction(transaction::Error::IncorrectSignature)) ); } diff --git a/wallet/tests/libwallet.rs b/wallet/tests/libwallet.rs index 5c7eda71..0fb73e6f 100644 --- a/wallet/tests/libwallet.rs +++ b/wallet/tests/libwallet.rs @@ -12,7 +12,7 @@ // limitations under the License. //! core::libtx specific tests -use self::core::core::transaction::kernel_sig_msg; +use self::core::core::transaction; use self::core::libtx::{aggsig, proof}; use self::keychain::{BlindSum, BlindingFactor, ExtKeychain, Keychain}; use self::util::secp; @@ -25,6 +25,10 @@ use grin_util as util; use grin_wallet as wallet; use rand::thread_rng; +fn kernel_sig_msg() -> secp::Message { + transaction::kernel_sig_msg(0, 0, transaction::KernelFeatures::DEFAULT_KERNEL).unwrap() +} + #[test] fn aggsig_sender_receiver_interaction() { let sender_keychain = ExtKeychain::from_random_seed().unwrap(); @@ -105,7 +109,7 @@ fn aggsig_sender_receiver_interaction() { ) .unwrap(); - let msg = kernel_sig_msg(0, 0).unwrap(); + let msg = kernel_sig_msg(); let sig_part = aggsig::calculate_partial_sig( &keychain.secp(), &rx_cx.sec_key, @@ -122,7 +126,7 @@ fn aggsig_sender_receiver_interaction() { // received in the response back from the receiver { let keychain = sender_keychain.clone(); - let msg = kernel_sig_msg(0, 0).unwrap(); + let msg = kernel_sig_msg(); let sig_verifies = aggsig::verify_partial_sig( &keychain.secp(), &rx_sig_part, @@ -137,7 +141,7 @@ fn aggsig_sender_receiver_interaction() { // now sender signs with their key let sender_sig_part = { let keychain = sender_keychain.clone(); - let msg = kernel_sig_msg(0, 0).unwrap(); + let msg = kernel_sig_msg(); let sig_part = aggsig::calculate_partial_sig( &keychain.secp(), &s_cx.sec_key, @@ -154,7 +158,7 @@ fn aggsig_sender_receiver_interaction() { // received by the sender { let keychain = receiver_keychain.clone(); - let msg = kernel_sig_msg(0, 0).unwrap(); + let msg = kernel_sig_msg(); let sig_verifies = aggsig::verify_partial_sig( &keychain.secp(), &sender_sig_part, @@ -170,7 +174,7 @@ fn aggsig_sender_receiver_interaction() { let (final_sig, final_pubkey) = { let keychain = receiver_keychain.clone(); - let msg = kernel_sig_msg(0, 0).unwrap(); + let msg = kernel_sig_msg(); let our_sig_part = aggsig::calculate_partial_sig( &keychain.secp(), &rx_cx.sec_key, @@ -205,7 +209,7 @@ fn aggsig_sender_receiver_interaction() { // Receiver checks the final signature verifies { let keychain = receiver_keychain.clone(); - let msg = kernel_sig_msg(0, 0).unwrap(); + let msg = kernel_sig_msg(); // Receiver check the final signature verifies let sig_verifies = aggsig::verify_completed_sig( @@ -221,7 +225,7 @@ fn aggsig_sender_receiver_interaction() { // Check we can verify the sig using the kernel excess { let keychain = ExtKeychain::from_random_seed().unwrap(); - let msg = kernel_sig_msg(0, 0).unwrap(); + let msg = kernel_sig_msg(); let sig_verifies = aggsig::verify_single_from_commit(&keychain.secp(), &final_sig, &msg, &kernel_excess); @@ -321,7 +325,7 @@ fn aggsig_sender_receiver_interaction_offset() { ) .unwrap(); - let msg = kernel_sig_msg(0, 0).unwrap(); + let msg = kernel_sig_msg(); let sig_part = aggsig::calculate_partial_sig( &keychain.secp(), &rx_cx.sec_key, @@ -338,7 +342,7 @@ fn aggsig_sender_receiver_interaction_offset() { // received in the response back from the receiver { let keychain = sender_keychain.clone(); - let msg = kernel_sig_msg(0, 0).unwrap(); + let msg = kernel_sig_msg(); let sig_verifies = aggsig::verify_partial_sig( &keychain.secp(), &sig_part, @@ -353,7 +357,7 @@ fn aggsig_sender_receiver_interaction_offset() { // now sender signs with their key let sender_sig_part = { let keychain = sender_keychain.clone(); - let msg = kernel_sig_msg(0, 0).unwrap(); + let msg = kernel_sig_msg(); let sig_part = aggsig::calculate_partial_sig( &keychain.secp(), &s_cx.sec_key, @@ -370,7 +374,7 @@ fn aggsig_sender_receiver_interaction_offset() { // received by the sender { let keychain = receiver_keychain.clone(); - let msg = kernel_sig_msg(0, 0).unwrap(); + let msg = kernel_sig_msg(); let sig_verifies = aggsig::verify_partial_sig( &keychain.secp(), &sender_sig_part, @@ -385,7 +389,7 @@ fn aggsig_sender_receiver_interaction_offset() { // Receiver now builds final signature from sender and receiver parts let (final_sig, final_pubkey) = { let keychain = receiver_keychain.clone(); - let msg = kernel_sig_msg(0, 0).unwrap(); + let msg = kernel_sig_msg(); let our_sig_part = aggsig::calculate_partial_sig( &keychain.secp(), &rx_cx.sec_key, @@ -420,7 +424,7 @@ fn aggsig_sender_receiver_interaction_offset() { // Receiver checks the final signature verifies { let keychain = receiver_keychain.clone(); - let msg = kernel_sig_msg(0, 0).unwrap(); + let msg = kernel_sig_msg(); // Receiver check the final signature verifies let sig_verifies = aggsig::verify_completed_sig( @@ -436,7 +440,7 @@ fn aggsig_sender_receiver_interaction_offset() { // Check we can verify the sig using the kernel excess { let keychain = ExtKeychain::from_random_seed().unwrap(); - let msg = kernel_sig_msg(0, 0).unwrap(); + let msg = kernel_sig_msg(); let sig_verifies = aggsig::verify_single_from_commit(&keychain.secp(), &final_sig, &msg, &kernel_excess); From 901c665ba7ce94c91966b6b6e6ba3c394018827b Mon Sep 17 00:00:00 2001 From: Yeastplume Date: Wed, 12 Dec 2018 13:38:29 +0000 Subject: [PATCH 02/16] Wallet tests, bug fixes (#2138) * More tests for smallest selection, resulting bug fixes, remove unneeded TXLogEntryTypes * rustfmt --- src/bin/cmd/wallet_tests.rs | 111 ++++++++++++++++++++- wallet/src/command.rs | 2 +- wallet/src/libwallet/api.rs | 13 +-- wallet/src/libwallet/internal/selection.rs | 8 -- wallet/src/libwallet/internal/tx.rs | 31 +++--- wallet/src/libwallet/types.rs | 6 -- 6 files changed, 134 insertions(+), 37 deletions(-) diff --git a/src/bin/cmd/wallet_tests.rs b/src/bin/cmd/wallet_tests.rs index 386299a6..4ab313d0 100644 --- a/src/bin/cmd/wallet_tests.rs +++ b/src/bin/cmd/wallet_tests.rs @@ -310,6 +310,17 @@ mod wallet_tests { execute_command(&app, test_dir, "wallet1", &client1, arg_vec)?; bh += 1; + let wallet1 = instantiate_wallet(config1.clone(), client1.clone(), "password", "default")?; + + // Check our transaction log, should have 10 entries + grin_wallet::controller::owner_single_use(wallet1.clone(), |api| { + api.set_active_account("mining")?; + let (refreshed, txs) = api.retrieve_txs(true, None, None)?; + assert!(refreshed); + assert_eq!(txs.len(), bh as usize); + Ok(()) + })?; + let _ = test_framework::award_blocks_to_wallet(&chain, wallet1.clone(), 10); bh += 10; @@ -338,10 +349,108 @@ mod wallet_tests { Ok(()) })?; - // txs + // Self-send to same account, using smallest strategy + let arg_vec = vec![ + "grin", + "wallet", + "-p", + "password", + "-a", + "mining", + "send", + "-m", + "file", + "-d", + &file_name, + "-g", + "Love, Yeast, Smallest", + "-s", + "smallest", + "10", + ]; + execute_command(&app, test_dir, "wallet1", &client1, arg_vec)?; + + let arg_vec = vec![ + "grin", + "wallet", + "-p", + "password", + "-a", + "mining", + "receive", + "-i", + &file_name, + "-g", + "Thanks, Yeast!", + ]; + execute_command(&app, test_dir, "wallet1", &client1, arg_vec.clone())?; + + let arg_vec = vec![ + "grin", + "wallet", + "-p", + "password", + "finalize", + "-i", + &response_file_name, + ]; + execute_command(&app, test_dir, "wallet1", &client1, arg_vec)?; + bh += 1; + + // Check our transaction log, should have bh entries + one for the self receive + let wallet1 = instantiate_wallet(config1.clone(), client1.clone(), "password", "default")?; + + grin_wallet::controller::owner_single_use(wallet1.clone(), |api| { + api.set_active_account("mining")?; + let (refreshed, txs) = api.retrieve_txs(true, None, None)?; + assert!(refreshed); + assert_eq!(txs.len(), bh as usize + 1); + Ok(()) + })?; + + // Try using the self-send method + let arg_vec = vec![ + "grin", + "wallet", + "-p", + "password", + "-a", + "mining", + "send", + "-m", + "self", + "-d", + "mining", + "-g", + "Self love", + "-s", + "smallest", + "10", + ]; + execute_command(&app, test_dir, "wallet1", &client1, arg_vec)?; + bh += 1; + + // Check our transaction log, should have bh entries + 2 for the self receives + let wallet1 = instantiate_wallet(config1.clone(), client1.clone(), "password", "default")?; + + grin_wallet::controller::owner_single_use(wallet1.clone(), |api| { + api.set_active_account("mining")?; + let (refreshed, txs) = api.retrieve_txs(true, None, None)?; + assert!(refreshed); + assert_eq!(txs.len(), bh as usize + 2); + Ok(()) + })?; + + // txs and outputs (mostly spit out for a visual in test logs) let arg_vec = vec!["grin", "wallet", "-p", "password", "-a", "mining", "txs"]; execute_command(&app, test_dir, "wallet1", &client1, arg_vec)?; + // txs and outputs (mostly spit out for a visual in test logs) + let arg_vec = vec![ + "grin", "wallet", "-p", "password", "-a", "mining", "outputs", + ]; + execute_command(&app, test_dir, "wallet1", &client1, arg_vec)?; + // let logging finish thread::sleep(Duration::from_millis(200)); Ok(()) diff --git a/wallet/src/command.rs b/wallet/src/command.rs index cf4ac442..d1e1529e 100644 --- a/wallet/src/command.rs +++ b/wallet/src/command.rs @@ -234,13 +234,13 @@ pub fn send( }; if adapter.supports_sync() { slate = adapter.send_tx_sync(&args.dest, &slate)?; + api.tx_lock_outputs(&slate, lock_fn)?; if args.method == "self" { controller::foreign_single_use(wallet, |api| { api.receive_tx(&mut slate, Some(&args.dest), None)?; Ok(()) })?; } - api.tx_lock_outputs(&slate, lock_fn)?; if let Err(e) = api.verify_slate_messages(&slate) { error!("Error validating participant messages: {}", e); return Err(e); diff --git a/wallet/src/libwallet/api.rs b/wallet/src/libwallet/api.rs index 459d9407..3447d41d 100644 --- a/wallet/src/libwallet/api.rs +++ b/wallet/src/libwallet/api.rs @@ -41,8 +41,8 @@ use crate::core::ser; use crate::keychain::{Identifier, Keychain}; use crate::libwallet::internal::{keys, tx, updater}; use crate::libwallet::types::{ - AcctPathMapping, BlockFees, CbData, NodeClient, OutputData, TxLogEntry, TxWrapper, - WalletBackend, WalletInfo, + AcctPathMapping, BlockFees, CbData, NodeClient, OutputData, TxLogEntry, TxLogEntryType, + TxWrapper, WalletBackend, WalletInfo, }; use crate::libwallet::{Error, ErrorKind}; use crate::util; @@ -634,7 +634,6 @@ where num_change_outputs, selection_strategy_is_use_all, &parent_key_id, - false, message, )?; @@ -834,10 +833,12 @@ where }; // Don't do this multiple times let tx = updater::retrieve_txs(&mut *w, None, Some(slate.id), Some(&parent_key_id))?; - if tx.len() > 0 { - return Err(ErrorKind::TransactionAlreadyReceived(slate.id.to_string()).into()); + for t in &tx { + if t.tx_type == TxLogEntryType::TxReceived { + return Err(ErrorKind::TransactionAlreadyReceived(slate.id.to_string()).into()); + } } - let res = tx::receive_tx(&mut *w, slate, &parent_key_id, false, message); + let res = tx::receive_tx(&mut *w, slate, &parent_key_id, message); w.close()?; if let Err(e) = res { diff --git a/wallet/src/libwallet/internal/selection.rs b/wallet/src/libwallet/internal/selection.rs index 50568d29..6a7344fe 100644 --- a/wallet/src/libwallet/internal/selection.rs +++ b/wallet/src/libwallet/internal/selection.rs @@ -36,7 +36,6 @@ pub fn build_send_tx_slate( change_outputs: usize, selection_strategy_is_use_all: bool, parent_key_id: Identifier, - is_self: bool, ) -> Result< ( Slate, @@ -99,9 +98,6 @@ where let mut batch = wallet.batch()?; let log_id = batch.next_tx_log_id(&parent_key_id)?; let mut t = TxLogEntry::new(parent_key_id.clone(), TxLogEntryType::TxSent, log_id); - if is_self { - t.tx_type = TxLogEntryType::TxSentSelf; - } t.tx_slate_id = Some(slate_id); t.fee = Some(fee); t.tx_hex = Some(tx_hex.to_owned()); @@ -148,7 +144,6 @@ pub fn build_recipient_output_with_slate( wallet: &mut T, slate: &mut Slate, parent_key_id: Identifier, - is_self: bool, ) -> Result< ( Identifier, @@ -190,9 +185,6 @@ where let mut batch = wallet.batch()?; let log_id = batch.next_tx_log_id(&parent_key_id)?; let mut t = TxLogEntry::new(parent_key_id.clone(), TxLogEntryType::TxReceived, log_id); - if is_self { - t.tx_type = TxLogEntryType::TxReceivedSelf; - } t.tx_slate_id = Some(slate_id); t.amount_credited = amount; t.num_outputs = 1; diff --git a/wallet/src/libwallet/internal/tx.rs b/wallet/src/libwallet/internal/tx.rs index 8529d3f6..acbca767 100644 --- a/wallet/src/libwallet/internal/tx.rs +++ b/wallet/src/libwallet/internal/tx.rs @@ -30,7 +30,6 @@ pub fn receive_tx( wallet: &mut T, slate: &mut Slate, parent_key_id: &Identifier, - is_self: bool, message: Option, ) -> Result<(), Error> where @@ -39,12 +38,8 @@ where K: Keychain, { // create an output using the amount in the slate - let (_, mut context, receiver_create_fn) = selection::build_recipient_output_with_slate( - wallet, - slate, - parent_key_id.clone(), - is_self, - )?; + let (_, mut context, receiver_create_fn) = + selection::build_recipient_output_with_slate(wallet, slate, parent_key_id.clone())?; // fill public keys let _ = slate.fill_round_1( @@ -74,7 +69,6 @@ pub fn create_send_tx( num_change_outputs: usize, selection_strategy_is_use_all: bool, parent_key_id: &Identifier, - is_self: bool, message: Option, ) -> Result< ( @@ -114,7 +108,6 @@ where num_change_outputs, selection_strategy_is_use_all, parent_key_id.clone(), - is_self, )?; // Generate a kernel offset and subtract from our context's secret key. Store @@ -210,7 +203,7 @@ where /// Update the stored hex transaction (this update needs to happen when the TX is finalised) pub fn update_tx_hex( wallet: &mut T, - parent_key_id: &Identifier, + _parent_key_id: &Identifier, slate: &Slate, ) -> Result<(), Error> where @@ -220,15 +213,23 @@ where { let tx_hex = util::to_hex(ser::ser_vec(&slate.tx).unwrap()); // This will ignore the parent key, so no need to specify account on the - // finalise command + // finalize command let tx_vec = updater::retrieve_txs(wallet, None, Some(slate.id), None)?; - if tx_vec.len() != 1 { - return Err(ErrorKind::TransactionDoesntExist(slate.id.to_string()))?; + let mut tx = None; + // don't want to assume this is the right tx, in case of self-sending + for t in tx_vec { + if t.tx_type == TxLogEntryType::TxSent { + tx = Some(t.clone()); + break; + } } - let mut tx = tx_vec[0].clone(); + let mut tx = match tx { + Some(t) => t, + None => return Err(ErrorKind::TransactionDoesntExist(slate.id.to_string()))?, + }; tx.tx_hex = Some(tx_hex); let batch = wallet.batch()?; - batch.save_tx_log_entry(tx, &parent_key_id)?; + batch.save_tx_log_entry(tx.clone(), &tx.parent_key_id)?; batch.commit()?; Ok(()) } diff --git a/wallet/src/libwallet/types.rs b/wallet/src/libwallet/types.rs index d17b8da1..88978cde 100644 --- a/wallet/src/libwallet/types.rs +++ b/wallet/src/libwallet/types.rs @@ -544,10 +544,6 @@ pub enum TxLogEntryType { TxReceived, /// Inputs locked + change outputs when a transaction is created TxSent, - /// As above, but self-transaction - TxReceivedSelf, - /// As Above - TxSentSelf, /// Received transaction that was rolled back by user TxReceivedCancelled, /// Sent transaction that was rolled back by user @@ -560,8 +556,6 @@ impl fmt::Display for TxLogEntryType { TxLogEntryType::ConfirmedCoinbase => write!(f, "Confirmed \nCoinbase"), TxLogEntryType::TxReceived => write!(f, "Received Tx"), TxLogEntryType::TxSent => write!(f, "Sent Tx"), - TxLogEntryType::TxReceivedSelf => write!(f, "Received Tx (Self)"), - TxLogEntryType::TxSentSelf => write!(f, "Sent Tx (Self)"), TxLogEntryType::TxReceivedCancelled => write!(f, "Received Tx\n- Cancelled"), TxLogEntryType::TxSentCancelled => write!(f, "Send Tx\n- Cancelled"), } From ed9c1acec12438f299d479e74949e9670c4fc6b9 Mon Sep 17 00:00:00 2001 From: Mark Renten <42224876+rentenmark@users.noreply.github.com> Date: Wed, 12 Dec 2018 10:33:30 -0500 Subject: [PATCH 03/16] offline build not possible right now closes #2139 (#2141) * remove (almost) all unwrap calls * replace with ? operator * build script will now work offline and in many other scenarios where the web wallet may not be downloaded --- src/build/build.rs | 86 ++++++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 41 deletions(-) diff --git a/src/build/build.rs b/src/build/build.rs index 8b64ef0d..09acc4f2 100644 --- a/src/build/build.rs +++ b/src/build/build.rs @@ -59,59 +59,61 @@ fn main() { format!("{}{}", env::var("OUT_DIR").unwrap(), "/built.rs"), ); - install_web_wallet(); + let web_wallet_install = install_web_wallet(); + match web_wallet_install { + Ok(true) => {} + _ => println!( + "WARNING : Web wallet could not be installed due to {:?}", + web_wallet_install + ), + } } -fn download_and_decompress(target_file: &str) { +fn download_and_decompress(target_file: &str) -> Result> { let req_path = format!("https://github.com/mimblewimble/grin-web-wallet/releases/download/{}/grin-web-wallet.tar.gz", WEB_WALLET_TAG); - let resp = reqwest::get(&req_path); + let mut resp = reqwest::get(&req_path)?; - // don't interfere if this doesn't work - if resp.is_err() { - println!("Warning: Failed to download grin-web-wallet. Web wallet will not be available"); - return; + if !resp.status().is_success() { + return Ok(false); } - let mut resp = resp.unwrap(); - if resp.status().is_success() { - // read response - let mut out: Vec = vec![]; - let r2 = resp.read_to_end(&mut out); - if r2.is_err() { - println!( - "Warning: Failed to download grin-web-wallet. Web wallet will not be available" - ); - return; - } + // read response + let mut out: Vec = vec![]; + resp.read_to_end(&mut out)?; - // Gunzip - let mut d = GzDecoder::new(&out[..]); - let mut decomp: Vec = vec![]; - d.read_to_end(&mut decomp).unwrap(); + // Gunzip + let mut d = GzDecoder::new(&out[..]); + let mut decomp: Vec = vec![]; + d.read_to_end(&mut decomp)?; - // write temp file - let mut buffer = File::create(target_file.clone()).unwrap(); - buffer.write_all(&decomp).unwrap(); - buffer.flush().unwrap(); - } + // write temp file + let mut buffer = File::create(target_file.clone())?; + buffer.write_all(&decomp)?; + buffer.flush()?; + + Ok(true) } /// Download and unzip tagged web-wallet build -fn install_web_wallet() { +fn install_web_wallet() -> Result> { let target_file = format!( "{}/grin-web-wallet-{}.tar", - env::var("OUT_DIR").unwrap(), + env::var("OUT_DIR")?, WEB_WALLET_TAG ); - let out_dir = env::var("OUT_DIR").unwrap(); + let out_dir = env::var("OUT_DIR")?; let mut out_path = PathBuf::from(&out_dir); out_path.pop(); out_path.pop(); out_path.pop(); // only re-download if needed + println!("{}", target_file); if !Path::new(&target_file).is_file() { - download_and_decompress(&target_file); + let success = download_and_decompress(&target_file)?; + if !success { + return Ok(false); // could not download and decompress + } } // remove old version @@ -120,26 +122,28 @@ fn install_web_wallet() { let _ = fs::remove_dir_all(remove_path); // Untar - let file = File::open(target_file).unwrap(); + let file = File::open(target_file)?; let mut a = Archive::new(file); - for file in a.entries().unwrap() { - let mut file = file.unwrap(); + for file in a.entries()? { + let mut file = file?; let h = file.header().clone(); - let path = h.path().unwrap().clone().into_owned(); + let path = h.path()?.clone().into_owned(); let is_dir = path.to_str().unwrap().ends_with(path::MAIN_SEPARATOR); - let path = path.strip_prefix("dist").unwrap(); + let path = path.strip_prefix("dist")?; let mut final_path = out_path.clone(); final_path.push(path); let mut tmp: Vec = vec![]; - file.read_to_end(&mut tmp).unwrap(); + file.read_to_end(&mut tmp)?; if is_dir { - fs::create_dir_all(final_path).unwrap(); + fs::create_dir_all(final_path)?; } else { - let mut buffer = File::create(final_path).unwrap(); - buffer.write_all(&tmp).unwrap(); - buffer.flush().unwrap(); + let mut buffer = File::create(final_path)?; + buffer.write_all(&tmp)?; + buffer.flush()?; } } + + Ok(true) } From 305b36dccee3527efead3a9d3b6d88001e801ca9 Mon Sep 17 00:00:00 2001 From: Antioch Peverell Date: Wed, 12 Dec 2018 16:49:35 +0000 Subject: [PATCH 04/16] replace is_mainnet() with !is_testnet() (#2125) * replace is_mainnet() with !is_testnet() makes testing "mainnet" code significantly easier (its the default) * fix next_target_adjustment test based on tromp analysis * rustfmt * cleanup wallet db files and add to gitignore * fix --- core/src/consensus.rs | 17 ++++++++++------- core/src/core/transaction.rs | 8 ++++---- core/src/global.rs | 12 ++++++++---- core/tests/consensus.rs | 21 +++++++++++++++------ 4 files changed, 37 insertions(+), 21 deletions(-) diff --git a/core/src/consensus.rs b/core/src/consensus.rs index 4cae47dc..804e5cd0 100644 --- a/core/src/consensus.rs +++ b/core/src/consensus.rs @@ -62,29 +62,31 @@ pub const COINBASE_MATURITY: u64 = DAY_HEIGHT; /// function of block height (time). Starts at 90% losing a percent /// approximately every week. Represented as an integer between 0 and 100. pub fn secondary_pow_ratio(height: u64) -> u64 { - if global::is_mainnet() { - 90u64.saturating_sub(height / (2 * YEAR_HEIGHT / 90)) - } else { + if global::is_testnet() { if height < T4_CUCKAROO_HARDFORK { // Maintaining pre hardfork testnet4 behavior 90u64.saturating_sub(height / WEEK_HEIGHT) } else { 90u64.saturating_sub(height / (2 * YEAR_HEIGHT / 90)) } + } else { + // Mainnet (or testing mainnet code). + 90u64.saturating_sub(height / (2 * YEAR_HEIGHT / 90)) } } /// The AR scale damping factor to use. Dependent on block height /// to account for pre HF behavior on testnet4. fn ar_scale_damp_factor(height: u64) -> u64 { - if global::is_mainnet() { - AR_SCALE_DAMP_FACTOR - } else { + if global::is_testnet() { if height < T4_CUCKAROO_HARDFORK { DIFFICULTY_DAMP_FACTOR } else { AR_SCALE_DAMP_FACTOR } + } else { + // Mainnet (or testing mainnet code). + AR_SCALE_DAMP_FACTOR } } @@ -332,7 +334,8 @@ where /// the hardfork. fn ar_count(height: u64, diff_data: &[HeaderInfo]) -> u64 { let mut to_skip = 1; - if !global::is_mainnet() && height < T4_CUCKAROO_HARDFORK { + if global::is_testnet() && height < T4_CUCKAROO_HARDFORK { + // Maintain behavior of testnet4 pre-HF. to_skip = 0; } 100 * diff_data diff --git a/core/src/core/transaction.rs b/core/src/core/transaction.rs index b046453f..10ba740d 100644 --- a/core/src/core/transaction.rs +++ b/core/src/core/transaction.rs @@ -1295,14 +1295,14 @@ pub fn kernel_sig_msg( lock_height: u64, features: KernelFeatures, ) -> Result { - let msg = if global::is_mainnet() { - let hash = (fee, lock_height, features).hash(); - secp::Message::from_slice(&hash.as_bytes())? - } else { + let msg = if global::is_testnet() { let mut bytes = [0; 32]; BigEndian::write_u64(&mut bytes[16..24], fee); BigEndian::write_u64(&mut bytes[24..], lock_height); secp::Message::from_slice(&bytes)? + } else { + let hash = (fee, lock_height, features).hash(); + secp::Message::from_slice(&hash.as_bytes())? }; Ok(msg) } diff --git a/core/src/global.rs b/core/src/global.rs index 30a38374..153bea40 100644 --- a/core/src/global.rs +++ b/core/src/global.rs @@ -284,7 +284,8 @@ pub fn is_user_testing_mode() -> bool { ChainTypes::UserTesting == *param_ref } -/// Are we in production mode (a live public network)? +/// Are we in production mode? +/// Production defined as a live public network, testnet[n] or mainnet. pub fn is_production_mode() -> bool { let param_ref = CHAIN_TYPE.read(); ChainTypes::Testnet1 == *param_ref @@ -294,10 +295,13 @@ pub fn is_production_mode() -> bool { || ChainTypes::Mainnet == *param_ref } -/// Are we in mainnet? -pub fn is_mainnet() -> bool { +/// Are we in one of our (many) testnets? +pub fn is_testnet() -> bool { let param_ref = CHAIN_TYPE.read(); - ChainTypes::Mainnet == *param_ref + ChainTypes::Testnet1 == *param_ref + || ChainTypes::Testnet2 == *param_ref + || ChainTypes::Testnet3 == *param_ref + || ChainTypes::Testnet4 == *param_ref } /// Helper function to get a nonce known to create a valid POW on diff --git a/core/tests/consensus.rs b/core/tests/consensus.rs index 1dc59a94..8178c456 100644 --- a/core/tests/consensus.rs +++ b/core/tests/consensus.rs @@ -384,9 +384,18 @@ fn next_target_adjustment() { let diff_min = Difficulty::min(); // Check we don't get stuck on difficulty <= MIN_DIFFICULTY (at 4x faster blocks at least) - let mut hi = HeaderInfo::from_diff_scaling(diff_min, MIN_DIFFICULTY as u32); + let mut hi = HeaderInfo::from_diff_scaling(diff_min, AR_SCALE_DAMP_FACTOR as u32); hi.is_secondary = false; - let hinext = next_difficulty(1, repeat(15, hi.clone(), DIFFICULTY_ADJUST_WINDOW, None)); + let hinext = next_difficulty( + 1, + repeat( + BLOCK_TIME_SEC / 4, + hi.clone(), + DIFFICULTY_ADJUST_WINDOW, + None, + ), + ); + assert_ne!(hinext.difficulty, diff_min); // Check we don't get stuck on scale MIN_DIFFICULTY, when primary frequency is too high @@ -476,7 +485,7 @@ fn test_secondary_pow_ratio() { // Tests for mainnet chain type. { global::set_mining_mode(global::ChainTypes::Mainnet); - assert_eq!(global::is_mainnet(), true); + assert_eq!(global::is_testnet(), false); assert_eq!(secondary_pow_ratio(1), 90); assert_eq!(secondary_pow_ratio(89), 90); @@ -518,7 +527,7 @@ fn test_secondary_pow_ratio() { // Tests for testnet4 chain type (covers pre and post hardfork). { global::set_mining_mode(global::ChainTypes::Testnet4); - assert_eq!(global::is_mainnet(), false); + assert_eq!(global::is_testnet(), true); assert_eq!(secondary_pow_ratio(1), 90); assert_eq!(secondary_pow_ratio(89), 90); @@ -566,7 +575,7 @@ fn test_secondary_pow_scale() { // testnet4 testing { global::set_mining_mode(global::ChainTypes::Testnet4); - assert_eq!(global::is_mainnet(), false); + assert_eq!(global::is_testnet(), true); // all primary, factor should increase so it becomes easier to find a high // difficulty block @@ -640,7 +649,7 @@ fn test_secondary_pow_scale() { // mainnet testing { global::set_mining_mode(global::ChainTypes::Mainnet); - assert_eq!(global::is_mainnet(), true); + assert_eq!(global::is_testnet(), false); // all primary, factor should increase so it becomes easier to find a high // difficulty block From 675edb4a19f99c9045dd73c47177f59cdaeeb8fd Mon Sep 17 00:00:00 2001 From: Antioch Peverell Date: Wed, 12 Dec 2018 19:41:42 +0000 Subject: [PATCH 05/16] TUI Mining Screen (block hash and block height) (#2142) * add block hash to tui mining diff screen include current header in there also * rework the mining screen on the tui show block hash as well as block height fix the "off by one" error --- servers/src/common/stats.rs | 7 ++++-- servers/src/grin/server.rs | 47 +++++++++++++++++++++++-------------- src/bin/tui/mining.rs | 17 ++++++++------ 3 files changed, 44 insertions(+), 27 deletions(-) diff --git a/servers/src/common/stats.rs b/servers/src/common/stats.rs index 2fa74147..02a96688 100644 --- a/servers/src/common/stats.rs +++ b/servers/src/common/stats.rs @@ -20,6 +20,7 @@ use std::sync::Arc; use std::time::SystemTime; use crate::core::consensus::graph_weight; +use crate::core::core::hash::Hash; use chrono::prelude::*; @@ -118,8 +119,10 @@ pub struct DiffStats { /// Last n blocks for difficulty calculation purposes #[derive(Clone, Debug)] pub struct DiffBlock { - /// Block number (can be negative for a new chain) - pub block_number: i64, + /// Block height (can be negative for a new chain) + pub block_height: i64, + /// Block hash (may be synthetic for a new chain) + pub block_hash: Hash, /// Block network difficulty pub difficulty: u64, /// Time block was found (epoch seconds) diff --git a/servers/src/grin/server.rs b/servers/src/grin/server.rs index 976933a6..6f267de4 100644 --- a/servers/src/grin/server.rs +++ b/servers/src/grin/server.rs @@ -27,6 +27,7 @@ use crate::common::adapters::{ }; use crate::common::stats::{DiffBlock, DiffStats, PeerStats, ServerStateInfo, ServerStats}; use crate::common::types::{Error, ServerConfig, StratumServerConfig, SyncState, SyncStatus}; +use crate::core::core::hash::{Hashed, ZERO_HASH}; use crate::core::core::verifier_cache::{LruVerifierCache, VerifierCache}; use crate::core::{consensus, genesis, global, pow}; use crate::grin::{dandelion_monitor, seed, sync}; @@ -368,29 +369,39 @@ impl Server { let last_blocks: Vec = global::difficulty_data_to_vector(self.chain.difficulty_iter()) .into_iter() - .take(consensus::DIFFICULTY_ADJUST_WINDOW as usize) .collect(); - let mut last_time = last_blocks[0].timestamp; let tip_height = self.chain.head().unwrap().height as i64; - let earliest_block_height = tip_height as i64 - last_blocks.len() as i64; - let mut i = 1; + let mut height = tip_height as i64 - last_blocks.len() as i64 + 1; let diff_entries: Vec = last_blocks - .iter() - .skip(1) - .map(|n| { - let dur = n.timestamp - last_time; - let height = earliest_block_height + i; - i += 1; - last_time = n.timestamp; + .windows(2) + .map(|pair| { + let prev = &pair[0]; + let next = &pair[1]; + + height += 1; + + // Use header hash if real header. + // Default to "zero" hash if synthetic header_info. + let hash = if height >= 0 { + if let Ok(header) = self.chain.get_header_by_height(height as u64) { + header.hash() + } else { + ZERO_HASH + } + } else { + ZERO_HASH + }; + DiffBlock { - block_number: height, - difficulty: n.difficulty.to_num(), - time: n.timestamp, - duration: dur, - secondary_scaling: n.secondary_scaling, - is_secondary: n.is_secondary, + block_height: height, + block_hash: hash, + difficulty: next.difficulty.to_num(), + time: next.timestamp, + duration: next.timestamp - prev.timestamp, + secondary_scaling: next.secondary_scaling, + is_secondary: next.is_secondary, } }) .collect(); @@ -398,7 +409,7 @@ impl Server { let block_time_sum = diff_entries.iter().fold(0, |sum, t| sum + t.duration); let block_diff_sum = diff_entries.iter().fold(0, |sum, d| sum + d.difficulty); DiffStats { - height: tip_height as u64, + height: height as u64, last_blocks: diff_entries, average_block_time: block_time_sum / (consensus::DIFFICULTY_ADJUST_WINDOW - 1), average_difficulty: block_diff_sum / (consensus::DIFFICULTY_ADJUST_WINDOW - 1), diff --git a/src/bin/tui/mining.rs b/src/bin/tui/mining.rs index 01b8c5c7..74135d3c 100644 --- a/src/bin/tui/mining.rs +++ b/src/bin/tui/mining.rs @@ -99,7 +99,8 @@ impl TableViewItem for WorkerStats { } #[derive(Copy, Clone, PartialEq, Eq, Hash)] enum DiffColumn { - BlockNumber, + Height, + Hash, PoWType, Difficulty, SecondaryScaling, @@ -110,7 +111,8 @@ enum DiffColumn { impl DiffColumn { fn _as_str(&self) -> &str { match *self { - DiffColumn::BlockNumber => "Block Number", + DiffColumn::Height => "Height", + DiffColumn::Hash => "Hash", DiffColumn::PoWType => "Type", DiffColumn::Difficulty => "Network Difficulty", DiffColumn::SecondaryScaling => "Sec. Scaling", @@ -130,7 +132,8 @@ impl TableViewItem for DiffBlock { }; match column { - DiffColumn::BlockNumber => self.block_number.to_string(), + DiffColumn::Height => self.block_height.to_string(), + DiffColumn::Hash => self.block_hash.to_string(), DiffColumn::PoWType => pow_type, DiffColumn::Difficulty => self.difficulty.to_string(), DiffColumn::SecondaryScaling => self.secondary_scaling.to_string(), @@ -144,7 +147,8 @@ impl TableViewItem for DiffBlock { Self: Sized, { match column { - DiffColumn::BlockNumber => Ordering::Equal, + DiffColumn::Height => Ordering::Equal, + DiffColumn::Hash => Ordering::Equal, DiffColumn::PoWType => Ordering::Equal, DiffColumn::Difficulty => Ordering::Equal, DiffColumn::SecondaryScaling => Ordering::Equal, @@ -264,9 +268,8 @@ impl TUIStatusListener for TUIMiningView { ); let diff_table_view = TableView::::new() - .column(DiffColumn::BlockNumber, "Block Number", |c| { - c.width_percent(15) - }) + .column(DiffColumn::Height, "Height", |c| c.width_percent(10)) + .column(DiffColumn::Hash, "Hash", |c| c.width_percent(10)) .column(DiffColumn::PoWType, "Type", |c| c.width_percent(10)) .column(DiffColumn::Difficulty, "Network Difficulty", |c| { c.width_percent(15) From 793e3843f02b69bfb50202949f0943dd1b84723d Mon Sep 17 00:00:00 2001 From: Antioch Peverell Date: Thu, 13 Dec 2018 09:57:24 +0000 Subject: [PATCH 06/16] HeaderEntry for storing subset of header info in the header MMR (#2137) * HeaderEntry for storing subset of header info in the header MMR * cleanup * cleanup --- chain/src/txhashset/txhashset.rs | 8 ++-- core/src/core/block.rs | 71 ++++++++++++++++++++++++++++++-- 2 files changed, 72 insertions(+), 7 deletions(-) diff --git a/chain/src/txhashset/txhashset.rs b/chain/src/txhashset/txhashset.rs index 1f45e680..b711caf1 100644 --- a/chain/src/txhashset/txhashset.rs +++ b/chain/src/txhashset/txhashset.rs @@ -204,8 +204,8 @@ impl TxHashSet { let pos = pmmr::insertion_to_pmmr_index(height + 1); let header_pmmr = ReadonlyPMMR::at(&self.header_pmmr_h.backend, self.header_pmmr_h.last_pos); - if let Some(hash) = header_pmmr.get_data(pos) { - let header = self.commit_index.get_block_header(&hash)?; + if let Some(entry) = header_pmmr.get_data(pos) { + let header = self.commit_index.get_block_header(&entry.hash())?; Ok(header) } else { Err(ErrorKind::Other(format!("get header by height")).into()) @@ -613,7 +613,7 @@ impl<'a> HeaderExtension<'a> { /// Get the header hash for the specified pos from the underlying MMR backend. fn get_header_hash(&self, pos: u64) -> Option { - self.pmmr.get_data(pos) + self.pmmr.get_data(pos).map(|x| x.hash()) } /// Get the header at the specified height based on the current state of the header extension. @@ -989,7 +989,7 @@ impl<'a> Extension<'a> { /// Get the header hash for the specified pos from the underlying MMR backend. fn get_header_hash(&self, pos: u64) -> Option { - self.header_pmmr.get_data(pos) + self.header_pmmr.get_data(pos).map(|x| x.hash()) } /// Get the header at the specified height based on the current state of the extension. diff --git a/core/src/core/block.rs b/core/src/core/block.rs index 220d88bc..abe6c1d7 100644 --- a/core/src/core/block.rs +++ b/core/src/core/block.rs @@ -34,7 +34,7 @@ use crate::core::{ use crate::global; use crate::keychain::{self, BlindingFactor}; use crate::pow::{Difficulty, Proof, ProofOfWork}; -use crate::ser::{self, PMMRable, Readable, Reader, Writeable, Writer}; +use crate::ser::{self, FixedLength, PMMRable, Readable, Reader, Writeable, Writer}; use crate::util::{secp, static_secp_instance}; /// Errors thrown by Block validation @@ -109,6 +109,65 @@ impl fmt::Display for Error { } } +/// Header entry for storing in the header MMR. +/// Note: we hash the block header itself and maintain the hash in the entry. +/// This allows us to lookup the original header from the db as necessary. +pub struct HeaderEntry { + hash: Hash, + timestamp: u64, + total_difficulty: Difficulty, + secondary_scaling: u32, + is_secondary: bool, +} + +impl Readable for HeaderEntry { + fn read(reader: &mut Reader) -> Result { + let hash = Hash::read(reader)?; + let timestamp = reader.read_u64()?; + let total_difficulty = Difficulty::read(reader)?; + let secondary_scaling = reader.read_u32()?; + + // Using a full byte to represent the bool for now. + let is_secondary = reader.read_u8()? != 0; + + Ok(HeaderEntry { + hash, + timestamp, + total_difficulty, + secondary_scaling, + is_secondary, + }) + } +} + +impl Writeable for HeaderEntry { + fn write(&self, writer: &mut W) -> Result<(), ser::Error> { + self.hash.write(writer)?; + writer.write_u64(self.timestamp)?; + self.total_difficulty.write(writer)?; + writer.write_u32(self.secondary_scaling)?; + + // Using a full byte to represent the bool for now. + if self.is_secondary { + writer.write_u8(1)?; + } else { + writer.write_u8(0)?; + } + Ok(()) + } +} + +impl FixedLength for HeaderEntry { + const LEN: usize = Hash::LEN + 8 + Difficulty::LEN + 4 + 1; +} + +impl HeaderEntry { + /// The hash of the underlying block. + pub fn hash(&self) -> Hash { + self.hash + } +} + /// Block header, fairly standard compared to other blockchains. #[derive(Clone, Debug, PartialEq)] pub struct BlockHeader { @@ -160,10 +219,16 @@ impl Default for BlockHeader { } impl PMMRable for BlockHeader { - type E = Hash; + type E = HeaderEntry; fn as_elmt(&self) -> Self::E { - self.hash() + HeaderEntry { + hash: self.hash(), + timestamp: self.timestamp.timestamp() as u64, + total_difficulty: self.total_difficulty(), + secondary_scaling: self.pow.secondary_scaling, + is_secondary: self.pow.is_secondary(), + } } } From df62fd6d95d81ca87867d5dc6f4dc5041594cc4d Mon Sep 17 00:00:00 2001 From: Yeastplume Date: Thu, 13 Dec 2018 11:19:46 +0000 Subject: [PATCH 07/16] Coin selection with small amounts fix (#2144) * coin selection fix for sending with an amount <= smallest output * rustfmt --- wallet/src/adapters/http.rs | 5 +++-- wallet/src/libwallet/internal/selection.rs | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/wallet/src/adapters/http.rs b/wallet/src/adapters/http.rs index 9c172f3d..941cc0cd 100644 --- a/wallet/src/adapters/http.rs +++ b/wallet/src/adapters/http.rs @@ -48,8 +48,9 @@ impl WalletCommAdapter for HTTPWalletCommAdapter { let url = format!("{}/v1/wallet/foreign/receive_tx", dest); debug!("Posting transaction slate to {}", url); - let res = api::client::post(url.as_str(), None, slate) - .context(ErrorKind::ClientCallback("Posting transaction slate"))?; + let res = api::client::post(url.as_str(), None, slate).context( + ErrorKind::ClientCallback("Posting transaction slate (is recipient listening?)"), + )?; Ok(res) } diff --git a/wallet/src/libwallet/internal/selection.rs b/wallet/src/libwallet/internal/selection.rs index 6a7344fe..2a747dc8 100644 --- a/wallet/src/libwallet/internal/selection.rs +++ b/wallet/src/libwallet/internal/selection.rs @@ -235,7 +235,7 @@ where K: Keychain, { // select some spendable coins from the wallet - let (max_outputs, coins) = select_coins( + let (max_outputs, mut coins) = select_coins( wallet, amount, current_height, @@ -291,7 +291,7 @@ where } // select some spendable coins from the wallet - let (_, coins) = select_coins( + coins = select_coins( wallet, amount_with_fee, current_height, @@ -299,7 +299,8 @@ where max_outputs, selection_strategy_is_use_all, parent_key_id, - ); + ) + .1; fee = tx_fee(coins.len(), num_outputs, 1, None); total = coins.iter().map(|c| c.value).sum(); amount_with_fee = amount + fee; From 7ae22eff2ba38338633ae72a5f817efba9b1158c Mon Sep 17 00:00:00 2001 From: Antioch Peverell Date: Thu, 13 Dec 2018 13:44:50 +0000 Subject: [PATCH 08/16] add hash() to tip for safety (#2145) --- chain/src/types.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/chain/src/types.rs b/chain/src/types.rs index 9a6ced64..7e784b80 100644 --- a/chain/src/types.rs +++ b/chain/src/types.rs @@ -64,8 +64,7 @@ pub struct Tip { } impl Tip { - /// TODO - why do we have Tip when we could just use a block header? - /// Creates a new tip based on header. + /// Creates a new tip based on provided header. pub fn from_header(header: &BlockHeader) -> Tip { Tip { height: header.height, @@ -74,6 +73,12 @@ impl Tip { total_difficulty: header.total_difficulty(), } } + + /// *Really* easy to accidentally call hash() on a tip (thinking its a header). + /// So lets make hash() do the right thing here. + pub fn hash(&self) -> Hash { + self.last_block_h + } } impl Default for Tip { From e6bc6e060e12d7761201cf790fc136349708d524 Mon Sep 17 00:00:00 2001 From: Gary Yu Date: Fri, 14 Dec 2018 22:37:45 +0800 Subject: [PATCH 09/16] fix travis-ci test suite for wallet_command_line test (#2155) --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index a28731a9..eb862a8d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,14 +40,12 @@ env: matrix: include: - - os: linux - env: TEST_SUITE=. - os: linux env: TEST_SUITE=servers - os: linux env: TEST_SUITE=chain-core - os: linux - env: TEST_SUITE=pool-p2p + env: TEST_SUITE=pool-p2p-src - os: linux env: TEST_SUITE=keychain-wallet - os: linux @@ -71,7 +69,7 @@ script: fi; before_deploy: - - if [[ "$TEST_SUITE" == "pool-p2p" ]]; then + - if [[ "$TEST_SUITE" == "pool-p2p-src" ]]; then cargo clean && cargo build --release && ./.auto-release.sh; fi - if [[ "$TEST_SUITE" == "release" ]] && [[ "$TRAVIS_OS_NAME" == "osx" ]]; then From 32a7c309e7fe8260286f1823fec99683e2565719 Mon Sep 17 00:00:00 2001 From: Gary Yu Date: Fri, 14 Dec 2018 22:38:05 +0800 Subject: [PATCH 10/16] TUI basis status text align (#2150) * tui basic status text align * rustfmt --- src/bin/tui/status.rs | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/bin/tui/status.rs b/src/bin/tui/status.rs index f93c1606..45d81e03 100644 --- a/src/bin/tui/status.rs +++ b/src/bin/tui/status.rs @@ -38,26 +38,27 @@ impl TUIStatusListener for TUIStatusView { LinearLayout::new(Orientation::Vertical) .child( LinearLayout::new(Orientation::Horizontal) - .child(TextView::new("Current Status: ")) + .child(TextView::new("Current Status: ")) .child(TextView::new("Starting").with_id("basic_current_status")), ) .child( LinearLayout::new(Orientation::Horizontal) - .child(TextView::new("Connected Peers: ")) + .child(TextView::new("Connected Peers: ")) .child(TextView::new("0").with_id("connected_peers")), ) .child( - LinearLayout::new(Orientation::Horizontal) - .child(TextView::new("------------------------")), + LinearLayout::new(Orientation::Horizontal).child(TextView::new( + "------------------------------------------------", + )), ) .child( LinearLayout::new(Orientation::Horizontal) - .child(TextView::new("Header Tip Hash: ")) + .child(TextView::new("Header Tip Hash: ")) .child(TextView::new(" ").with_id("basic_header_tip_hash")), ) .child( LinearLayout::new(Orientation::Horizontal) - .child(TextView::new("Header Chain Height: ")) + .child(TextView::new("Header Chain Height: ")) .child(TextView::new(" ").with_id("basic_header_chain_height")), ) .child( @@ -66,27 +67,29 @@ impl TUIStatusListener for TUIStatusView { .child(TextView::new(" ").with_id("basic_header_total_difficulty")), ) .child( - LinearLayout::new(Orientation::Horizontal) - .child(TextView::new("------------------------")), + LinearLayout::new(Orientation::Horizontal).child(TextView::new( + "------------------------------------------------", + )), ) .child( LinearLayout::new(Orientation::Horizontal) - .child(TextView::new("Tip Hash: ")) + .child(TextView::new("Chain Tip Hash: ")) .child(TextView::new(" ").with_id("tip_hash")), ) .child( LinearLayout::new(Orientation::Horizontal) - .child(TextView::new("Chain Height: ")) + .child(TextView::new("Chain Height: ")) .child(TextView::new(" ").with_id("chain_height")), ) .child( LinearLayout::new(Orientation::Horizontal) - .child(TextView::new("Cumulative Difficulty: ")) + .child(TextView::new("Chain Cumulative Difficulty: ")) .child(TextView::new(" ").with_id("basic_total_difficulty")), ) .child( - LinearLayout::new(Orientation::Horizontal) - .child(TextView::new("------------------------")), + LinearLayout::new(Orientation::Horizontal).child(TextView::new( + "------------------------------------------------", + )), ) .child( LinearLayout::new(Orientation::Horizontal) From 8e678058f12d5136b13b35e70a8ff2dfc3178f4e Mon Sep 17 00:00:00 2001 From: Yeastplume Date: Fri, 14 Dec 2018 16:24:53 +0000 Subject: [PATCH 11/16] [WIP] Store completed transactions in files instead of DB (#2148) Store completed transactions in files instead of DB --- src/bin/cmd/wallet_tests.rs | 4 +- wallet/src/command.rs | 2 +- wallet/src/controller.rs | 5 +- wallet/src/display.rs | 2 +- wallet/src/libwallet/api.rs | 22 +++++-- wallet/src/libwallet/internal/selection.rs | 76 ++++++++++++---------- wallet/src/libwallet/internal/tx.rs | 22 ++----- wallet/src/libwallet/types.rs | 21 +++--- wallet/src/lmdb_wallet.rs | 58 ++++++++++++++++- wallet/tests/repost.rs | 6 +- wallet/tests/transaction.rs | 3 +- 11 files changed, 141 insertions(+), 80 deletions(-) diff --git a/src/bin/cmd/wallet_tests.rs b/src/bin/cmd/wallet_tests.rs index 4ab313d0..cc3bc49f 100644 --- a/src/bin/cmd/wallet_tests.rs +++ b/src/bin/cmd/wallet_tests.rs @@ -408,7 +408,7 @@ mod wallet_tests { Ok(()) })?; - // Try using the self-send method + // Try using the self-send method, splitting up outputs for the fun of it let arg_vec = vec![ "grin", "wallet", @@ -423,6 +423,8 @@ mod wallet_tests { "mining", "-g", "Self love", + "-o", + "75", "-s", "smallest", "10", diff --git a/wallet/src/command.rs b/wallet/src/command.rs index d1e1529e..35d9b9fc 100644 --- a/wallet/src/command.rs +++ b/wallet/src/command.rs @@ -409,7 +409,7 @@ pub fn repost( ) -> Result<(), Error> { controller::owner_single_use(wallet.clone(), |api| { let (_, txs) = api.retrieve_txs(true, Some(args.id), None)?; - let stored_tx = txs[0].get_stored_tx(); + let stored_tx = api.get_stored_tx(&txs[0])?; if stored_tx.is_none() { error!( "Transaction with id {} does not have transaction data. Not reposting.", diff --git a/wallet/src/controller.rs b/wallet/src/controller.rs index ed91f6e9..b19014a4 100644 --- a/wallet/src/controller.rs +++ b/wallet/src/controller.rs @@ -231,7 +231,10 @@ where if let Some(id_string) = params.get("id") { match id_string[0].parse() { Ok(id) => match api.retrieve_txs(true, Some(id), None) { - Ok((_, txs)) => Ok((txs[0].confirmed, txs[0].get_stored_tx())), + Ok((_, txs)) => { + let stored_tx = api.get_stored_tx(&txs[0])?; + Ok((txs[0].confirmed, stored_tx)) + } Err(e) => { error!("retrieve_stored_tx: failed with error: {}", e); Err(e) diff --git a/wallet/src/display.rs b/wallet/src/display.rs index 86e1e331..f5e24dba 100644 --- a/wallet/src/display.rs +++ b/wallet/src/display.rs @@ -178,7 +178,7 @@ pub fn txs( ) }; let tx_data = match t.tx_hex { - Some(_) => format!("Exists"), + Some(t) => format!("{}", t), None => "None".to_owned(), }; if dark_background_color_scheme { diff --git a/wallet/src/libwallet/api.rs b/wallet/src/libwallet/api.rs index 3447d41d..2dd9f88d 100644 --- a/wallet/src/libwallet/api.rs +++ b/wallet/src/libwallet/api.rs @@ -612,7 +612,13 @@ where num_change_outputs: usize, selection_strategy_is_use_all: bool, message: Option, - ) -> Result<(Slate, impl FnOnce(&mut W, &str) -> Result<(), Error>), Error> { + ) -> Result< + ( + Slate, + impl FnOnce(&mut W, &Transaction) -> Result<(), Error>, + ), + Error, + > { let mut w = self.wallet.lock(); w.open_with_credentials()?; let parent_key_id = match src_acct_name { @@ -653,12 +659,11 @@ where pub fn tx_lock_outputs( &mut self, slate: &Slate, - lock_fn: impl FnOnce(&mut W, &str) -> Result<(), Error>, + lock_fn: impl FnOnce(&mut W, &Transaction) -> Result<(), Error>, ) -> Result<(), Error> { let mut w = self.wallet.lock(); w.open_with_credentials()?; - let tx_hex = util::to_hex(ser::ser_vec(&slate.tx).unwrap()); - lock_fn(&mut *w, &tx_hex)?; + lock_fn(&mut *w, &slate.tx)?; Ok(()) } @@ -668,11 +673,10 @@ where /// propagation. pub fn finalize_tx(&mut self, slate: &mut Slate) -> Result<(), Error> { let mut w = self.wallet.lock(); - let parent_key_id = w.parent_key_id(); w.open_with_credentials()?; let context = w.get_private_context(slate.id.as_bytes())?; tx::complete_tx(&mut *w, slate, &context)?; - tx::update_tx_hex(&mut *w, &parent_key_id, slate)?; + tx::update_stored_tx(&mut *w, slate)?; { let mut batch = w.batch()?; batch.delete_private_context(slate.id.as_bytes())?; @@ -706,6 +710,12 @@ where Ok(()) } + /// Retrieves a stored transaction from a TxLogEntry + pub fn get_stored_tx(&self, entry: &TxLogEntry) -> Result, Error> { + let w = self.wallet.lock(); + w.get_stored_tx(entry) + } + /// Posts a transaction to the chain pub fn post_tx(&self, tx: &Transaction, fluff: bool) -> Result<(), Error> { let tx_hex = util::to_hex(ser::ser_vec(tx).unwrap()); diff --git a/wallet/src/libwallet/internal/selection.rs b/wallet/src/libwallet/internal/selection.rs index 2a747dc8..be2160ec 100644 --- a/wallet/src/libwallet/internal/selection.rs +++ b/wallet/src/libwallet/internal/selection.rs @@ -14,6 +14,7 @@ //! Selection of inputs for building transactions +use crate::core::core::Transaction; use crate::core::libtx::{build, slate::Slate, tx_fee}; use crate::keychain::{Identifier, Keychain}; use crate::libwallet::error::{Error, ErrorKind}; @@ -40,7 +41,7 @@ pub fn build_send_tx_slate( ( Slate, Context, - impl FnOnce(&mut T, &str) -> Result<(), Error>, + impl FnOnce(&mut T, &Transaction) -> Result<(), Error>, ), Error, > @@ -94,42 +95,47 @@ where // Return a closure to acquire wallet lock and lock the coins being spent // so we avoid accidental double spend attempt. - let update_sender_wallet_fn = move |wallet: &mut T, tx_hex: &str| { - let mut batch = wallet.batch()?; - let log_id = batch.next_tx_log_id(&parent_key_id)?; - let mut t = TxLogEntry::new(parent_key_id.clone(), TxLogEntryType::TxSent, log_id); - t.tx_slate_id = Some(slate_id); - t.fee = Some(fee); - t.tx_hex = Some(tx_hex.to_owned()); - let mut amount_debited = 0; - t.num_inputs = lock_inputs.len(); - for id in lock_inputs { - let mut coin = batch.get(&id).unwrap(); - coin.tx_log_entry = Some(log_id); - amount_debited = amount_debited + coin.value; - batch.lock_output(&mut coin)?; - } + let update_sender_wallet_fn = move |wallet: &mut T, tx: &Transaction| { + let tx_entry = { + let mut batch = wallet.batch()?; + let log_id = batch.next_tx_log_id(&parent_key_id)?; + let mut t = TxLogEntry::new(parent_key_id.clone(), TxLogEntryType::TxSent, log_id); + t.tx_slate_id = Some(slate_id); + let filename = format!("{}.grintx", slate_id); + t.tx_hex = Some(filename); + t.fee = Some(fee); + let mut amount_debited = 0; + t.num_inputs = lock_inputs.len(); + for id in lock_inputs { + let mut coin = batch.get(&id).unwrap(); + coin.tx_log_entry = Some(log_id); + amount_debited = amount_debited + coin.value; + batch.lock_output(&mut coin)?; + } - t.amount_debited = amount_debited; + t.amount_debited = amount_debited; - // write the output representing our change - for (change_amount, id) in &change_amounts_derivations { - t.num_outputs += 1; - t.amount_credited += change_amount; - batch.save(OutputData { - root_key_id: parent_key_id.clone(), - key_id: id.clone(), - n_child: id.to_path().last_path_index(), - value: change_amount.clone(), - status: OutputStatus::Unconfirmed, - height: current_height, - lock_height: 0, - is_coinbase: false, - tx_log_entry: Some(log_id), - })?; - } - batch.save_tx_log_entry(t, &parent_key_id)?; - batch.commit()?; + // write the output representing our change + for (change_amount, id) in &change_amounts_derivations { + t.num_outputs += 1; + t.amount_credited += change_amount; + batch.save(OutputData { + root_key_id: parent_key_id.clone(), + key_id: id.clone(), + n_child: id.to_path().last_path_index(), + value: change_amount.clone(), + status: OutputStatus::Unconfirmed, + height: current_height, + lock_height: 0, + is_coinbase: false, + tx_log_entry: Some(log_id), + })?; + } + batch.save_tx_log_entry(t.clone(), &parent_key_id)?; + batch.commit()?; + t + }; + wallet.store_tx(&format!("{}", tx_entry.tx_slate_id.unwrap()), tx)?; Ok(()) }; diff --git a/wallet/src/libwallet/internal/tx.rs b/wallet/src/libwallet/internal/tx.rs index acbca767..caac3359 100644 --- a/wallet/src/libwallet/internal/tx.rs +++ b/wallet/src/libwallet/internal/tx.rs @@ -14,11 +14,10 @@ //! Transaction building functions -use crate::util; use uuid::Uuid; +use crate::core::core::Transaction; use crate::core::libtx::slate::Slate; -use crate::core::ser; use crate::keychain::{Identifier, Keychain}; use crate::libwallet::internal::{selection, updater}; use crate::libwallet::types::{Context, NodeClient, TxLogEntryType, WalletBackend}; @@ -74,7 +73,7 @@ pub fn create_send_tx( ( Slate, Context, - impl FnOnce(&mut T, &str) -> Result<(), Error>, + impl FnOnce(&mut T, &Transaction) -> Result<(), Error>, ), Error, > @@ -200,19 +199,13 @@ where Ok((tx.confirmed, tx.tx_hex)) } -/// Update the stored hex transaction (this update needs to happen when the TX is finalised) -pub fn update_tx_hex( - wallet: &mut T, - _parent_key_id: &Identifier, - slate: &Slate, -) -> Result<(), Error> +/// Update the stored transaction (this update needs to happen when the TX is finalised) +pub fn update_stored_tx(wallet: &mut T, slate: &Slate) -> Result<(), Error> where T: WalletBackend, C: NodeClient, K: Keychain, { - let tx_hex = util::to_hex(ser::ser_vec(&slate.tx).unwrap()); - // This will ignore the parent key, so no need to specify account on the // finalize command let tx_vec = updater::retrieve_txs(wallet, None, Some(slate.id), None)?; let mut tx = None; @@ -223,14 +216,11 @@ where break; } } - let mut tx = match tx { + let tx = match tx { Some(t) => t, None => return Err(ErrorKind::TransactionDoesntExist(slate.id.to_string()))?, }; - tx.tx_hex = Some(tx_hex); - let batch = wallet.batch()?; - batch.save_tx_log_entry(tx.clone(), &tx.parent_key_id)?; - batch.commit()?; + wallet.store_tx(&format!("{}", tx.tx_slate_id.unwrap()), &slate.tx)?; Ok(()) } diff --git a/wallet/src/libwallet/types.rs b/wallet/src/libwallet/types.rs index 88978cde..e4999811 100644 --- a/wallet/src/libwallet/types.rs +++ b/wallet/src/libwallet/types.rs @@ -21,7 +21,6 @@ use crate::core::libtx::aggsig; use crate::core::ser; use crate::keychain::{Identifier, Keychain}; use crate::libwallet::error::{Error, ErrorKind}; -use crate::util; use crate::util::secp::key::{PublicKey, SecretKey}; use crate::util::secp::{self, pedersen, Secp256k1}; use chrono::prelude::*; @@ -102,6 +101,12 @@ where /// Gets an account path for a given label fn get_acct_path(&self, label: String) -> Result, Error>; + /// Stores a transaction + fn store_tx(&self, uuid: &str, tx: &Transaction) -> Result<(), Error>; + + /// Retrieves a stored transaction from a TxLogEntry + fn get_stored_tx(&self, entry: &TxLogEntry) -> Result, Error>; + /// Create a new write batch to update or remove output data fn batch<'a>(&'a mut self) -> Result + 'a>, Error>; @@ -156,7 +161,7 @@ where fn tx_log_iter(&self) -> Box>; /// save a tx log entry - fn save_tx_log_entry(&self, t: TxLogEntry, parent_id: &Identifier) -> Result<(), Error>; + fn save_tx_log_entry(&mut self, t: TxLogEntry, parent_id: &Identifier) -> Result<(), Error>; /// save an account label -> path mapping fn save_acct_path(&mut self, mapping: AcctPathMapping) -> Result<(), Error>; @@ -595,6 +600,7 @@ pub struct TxLogEntry { pub amount_debited: u64, /// Fee pub fee: Option, + // TODO: rename this to 'stored_tx_file' or something for mainnet /// The transaction json itself, stored for reference or resending pub tx_hex: Option, } @@ -636,17 +642,6 @@ impl TxLogEntry { pub fn update_confirmation_ts(&mut self) { self.confirmation_ts = Some(Utc::now()); } - - /// Retrieve the stored transaction, if any - pub fn get_stored_tx(&self) -> Option { - match self.tx_hex.as_ref() { - None => None, - Some(t) => { - let tx_bin = util::from_hex(t.clone()).unwrap(); - Some(ser::deserialize::(&mut &tx_bin[..]).unwrap()) - } - } - } } /// Map of named accounts to BIP32 paths diff --git a/wallet/src/lmdb_wallet.rs b/wallet/src/lmdb_wallet.rs index 306136e2..acadf365 100644 --- a/wallet/src/lmdb_wallet.rs +++ b/wallet/src/lmdb_wallet.rs @@ -16,18 +16,29 @@ use std::cell::RefCell; use std::sync::Arc; use std::{fs, path}; +// for writing storedtransaction files +use std::fs::File; +use std::io::{Read, Write}; +use std::path::Path; + +use serde_json; + use failure::ResultExt; use uuid::Uuid; use crate::keychain::{ChildNumber, ExtKeychain, Identifier, Keychain}; use crate::store::{self, option_to_not_found, to_key, to_key_u64}; +use crate::core::core::Transaction; +use crate::core::ser; use crate::libwallet::types::*; use crate::libwallet::{internal, Error, ErrorKind}; use crate::types::{WalletConfig, WalletSeed}; +use crate::util; use crate::util::secp::pedersen; pub const DB_DIR: &'static str = "db"; +pub const TX_SAVE_DIR: &'static str = "saved_txs"; const COMMITMENT_PREFIX: u8 = 'C' as u8; const OUTPUT_PREFIX: u8 = 'o' as u8; @@ -69,10 +80,16 @@ impl LMDBBackend { let db_path = path::Path::new(&config.data_file_dir).join(DB_DIR); fs::create_dir_all(&db_path).expect("Couldn't create wallet backend directory!"); + let stored_tx_path = path::Path::new(&config.data_file_dir).join(TX_SAVE_DIR); + fs::create_dir_all(&stored_tx_path) + .expect("Couldn't create wallet backend tx storage directory!"); + let lmdb_env = Arc::new(store::new_env(db_path.to_str().unwrap().to_string())); let store = store::Store::open(lmdb_env, DB_DIR); // Make sure default wallet derivation path always exists + // as well as path (so it can be retrieved by batches to know where to store + // completed transactions, for reference let default_account = AcctPathMapping { label: "default".to_owned(), path: LMDBBackend::::default_path(), @@ -228,6 +245,37 @@ where self.db.get_ser(&acct_key).map_err(|e| e.into()) } + fn store_tx(&self, uuid: &str, tx: &Transaction) -> Result<(), Error> { + let filename = format!("{}.grintx", uuid); + let path = path::Path::new(&self.config.data_file_dir) + .join(TX_SAVE_DIR) + .join(filename); + let path_buf = Path::new(&path).to_path_buf(); + let mut stored_tx = File::create(path_buf)?; + let tx_hex = util::to_hex(ser::ser_vec(tx).unwrap());; + stored_tx.write_all(&tx_hex.as_bytes())?; + stored_tx.sync_all()?; + Ok(()) + } + + fn get_stored_tx(&self, entry: &TxLogEntry) -> Result, Error> { + let filename = match entry.tx_hex.clone() { + Some(f) => f, + None => return Ok(None), + }; + let path = path::Path::new(&self.config.data_file_dir) + .join(TX_SAVE_DIR) + .join(filename); + let tx_file = Path::new(&path).to_path_buf(); + let mut tx_f = File::open(tx_file)?; + let mut content = String::new(); + tx_f.read_to_string(&mut content)?; + let tx_bin = util::from_hex(content).unwrap(); + Ok(Some( + ser::deserialize::(&mut &tx_bin[..]).unwrap(), + )) + } + fn batch<'a>(&'a mut self) -> Result + 'a>, Error> { Ok(Box::new(Batch { _store: self, @@ -403,17 +451,21 @@ where Ok(()) } - fn save_tx_log_entry(&self, t: TxLogEntry, parent_id: &Identifier) -> Result<(), Error> { + fn save_tx_log_entry( + &mut self, + tx_in: TxLogEntry, + parent_id: &Identifier, + ) -> Result<(), Error> { let tx_log_key = to_key_u64( TX_LOG_ENTRY_PREFIX, &mut parent_id.to_bytes().to_vec(), - t.id as u64, + tx_in.id as u64, ); self.db .borrow() .as_ref() .unwrap() - .put_ser(&tx_log_key, &t)?; + .put_ser(&tx_log_key, &tx_in)?; Ok(()) } diff --git a/wallet/tests/repost.rs b/wallet/tests/repost.rs index 69f0820f..c2cc8ad8 100644 --- a/wallet/tests/repost.rs +++ b/wallet/tests/repost.rs @@ -148,7 +148,8 @@ fn file_repost_test_impl(test_dir: &str) -> Result<(), libwallet::Error> { // Now repost from cached wallet::controller::owner_single_use(wallet1.clone(), |api| { let (_, txs) = api.retrieve_txs(true, None, Some(slate.id))?; - api.post_tx(&txs[0].get_stored_tx().unwrap(), false)?; + let stored_tx = api.get_stored_tx(&txs[0])?; + api.post_tx(&stored_tx.unwrap(), false)?; bh += 1; Ok(()) })?; @@ -214,7 +215,8 @@ fn file_repost_test_impl(test_dir: &str) -> Result<(), libwallet::Error> { // Now repost from cached wallet::controller::owner_single_use(wallet1.clone(), |api| { let (_, txs) = api.retrieve_txs(true, None, Some(slate.id))?; - api.post_tx(&txs[0].get_stored_tx().unwrap(), false)?; + let stored_tx = api.get_stored_tx(&txs[0])?; + api.post_tx(&stored_tx.unwrap(), false)?; bh += 1; Ok(()) })?; diff --git a/wallet/tests/transaction.rs b/wallet/tests/transaction.rs index 8e725d4f..9ad880ad 100644 --- a/wallet/tests/transaction.rs +++ b/wallet/tests/transaction.rs @@ -252,7 +252,8 @@ fn basic_transaction_api(test_dir: &str) -> Result<(), libwallet::Error> { .iter() .find(|t| t.tx_slate_id == Some(slate.id)) .unwrap(); - sender_api.post_tx(&tx.get_stored_tx().unwrap(), false)?; + let stored_tx = sender_api.get_stored_tx(&tx)?; + sender_api.post_tx(&stored_tx.unwrap(), false)?; let (_, wallet1_info) = sender_api.retrieve_summary_info(true, 1)?; // should be mined now assert_eq!( From 197d4f9575d7f97b080ff7eea3b34a0c81c469dc Mon Sep 17 00:00:00 2001 From: Gary Yu Date: Sat, 15 Dec 2018 10:10:33 +0800 Subject: [PATCH 12/16] TUI mining server status: add blocks found statistics (#2151) * TUI mining server status: add solutions found statistics * rustfmt * modify 'solutions found' as 'blocks found' --- servers/src/common/stats.rs | 3 +++ servers/src/mining/stratumserver.rs | 8 ++++++-- src/bin/tui/mining.rs | 29 ++++++++++++++++++----------- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/servers/src/common/stats.rs b/servers/src/common/stats.rs index 02a96688..47befa75 100644 --- a/servers/src/common/stats.rs +++ b/servers/src/common/stats.rs @@ -80,6 +80,8 @@ pub struct WorkerStats { pub num_rejected: u64, /// number of shares submitted too late pub num_stale: u64, + /// number of valid blocks found + pub num_blocks_found: u64, } /// Struct to return relevant information about the stratum server @@ -209,6 +211,7 @@ impl Default for WorkerStats { num_accepted: 0, num_rejected: 0, num_stale: 0, + num_blocks_found: 0, } } } diff --git a/servers/src/mining/stratumserver.rs b/servers/src/mining/stratumserver.rs index 7a700999..4db3a192 100644 --- a/servers/src/mining/stratumserver.rs +++ b/servers/src/mining/stratumserver.rs @@ -512,10 +512,14 @@ impl StratumServer { return Err(serde_json::to_value(e).unwrap()); } share_is_block = true; + worker_stats.num_blocks_found += 1; // Log message to make it obvious we found a block warn!( - "(Server ID: {}) Solution Found for block {} - Yay!!!", - self.id, params.height + "(Server ID: {}) Solution Found for block {} - Yay!!! Worker ID: {}, blocks found: {}, shares: {}", + self.id, params.height, + worker_stats.id, + worker_stats.num_blocks_found, + worker_stats.num_accepted, ); } else { // Do some validation but dont submit diff --git a/src/bin/tui/mining.rs b/src/bin/tui/mining.rs index 74135d3c..5cf6f1d4 100644 --- a/src/bin/tui/mining.rs +++ b/src/bin/tui/mining.rs @@ -44,6 +44,7 @@ enum StratumWorkerColumn { NumAccepted, NumRejected, NumStale, + NumBlocksFound, } impl StratumWorkerColumn { @@ -56,6 +57,7 @@ impl StratumWorkerColumn { StratumWorkerColumn::NumAccepted => "Num Accepted", StratumWorkerColumn::NumRejected => "Num Rejected", StratumWorkerColumn::NumStale => "Num Stale", + StratumWorkerColumn::NumBlocksFound => "Blocks Found", } } } @@ -79,6 +81,7 @@ impl TableViewItem for WorkerStats { StratumWorkerColumn::NumAccepted => self.num_accepted.to_string(), StratumWorkerColumn::NumRejected => self.num_rejected.to_string(), StratumWorkerColumn::NumStale => self.num_stale.to_string(), + StratumWorkerColumn::NumBlocksFound => self.num_blocks_found.to_string(), } } @@ -94,6 +97,7 @@ impl TableViewItem for WorkerStats { StratumWorkerColumn::NumAccepted => Ordering::Equal, StratumWorkerColumn::NumRejected => Ordering::Equal, StratumWorkerColumn::NumStale => Ordering::Equal, + StratumWorkerColumn::NumBlocksFound => Ordering::Equal, } } } @@ -185,17 +189,15 @@ impl TUIStatusListener for TUIMiningView { }); let table_view = TableView::::new() - .column(StratumWorkerColumn::Id, "Worker ID", |c| { - c.width_percent(10) - }) + .column(StratumWorkerColumn::Id, "Worker ID", |c| c.width_percent(8)) .column(StratumWorkerColumn::IsConnected, "Connected", |c| { - c.width_percent(10) + c.width_percent(8) }) .column(StratumWorkerColumn::LastSeen, "Last Seen", |c| { c.width_percent(16) }) .column(StratumWorkerColumn::PowDifficulty, "Pow Difficulty", |c| { - c.width_percent(14) + c.width_percent(12) }) .column(StratumWorkerColumn::NumAccepted, "Num Accepted", |c| { c.width_percent(10) @@ -205,6 +207,9 @@ impl TUIStatusListener for TUIMiningView { }) .column(StratumWorkerColumn::NumStale, "Num Stale", |c| { c.width_percent(10) + }) + .column(StratumWorkerColumn::NumBlocksFound, "Blocks Found", |c| { + c.width_percent(10) }); let status_view = LinearLayout::new(Orientation::Vertical) @@ -330,18 +335,20 @@ impl TUIStatusListener for TUIMiningView { ); let stratum_stats = stats.stratum_stats.clone(); let stratum_network_hashrate = format!( - "Network Hashrate: {:.*}", + "Network Hashrate: {:.*}", 2, stratum_stats.network_hashrate(stratum_stats.block_height) ); let worker_stats = stratum_stats.worker_stats; let stratum_enabled = format!("Mining server enabled: {}", stratum_stats.is_enabled); let stratum_is_running = format!("Mining server running: {}", stratum_stats.is_running); - let stratum_num_workers = format!("Number of workers: {}", stratum_stats.num_workers); - let stratum_block_height = format!("Solving Block Height: {}", stratum_stats.block_height); - let stratum_network_difficulty = - format!("Network Difficulty: {}", stratum_stats.network_difficulty); - let stratum_edge_bits = format!("Cuckoo Size: {}", stratum_stats.edge_bits); + let stratum_num_workers = format!("Number of workers: {}", stratum_stats.num_workers); + let stratum_block_height = format!("Solving Block Height: {}", stratum_stats.block_height); + let stratum_network_difficulty = format!( + "Network Difficulty: {}", + stratum_stats.network_difficulty + ); + let stratum_edge_bits = format!("Cuckoo Size: {}", stratum_stats.edge_bits); c.call_on_id("stratum_config_status", |t: &mut TextView| { t.set_content(stratum_enabled); From 85074f83562b58aa14a6aac257d1049c2f6d3317 Mon Sep 17 00:00:00 2001 From: Michalis Kargakis Date: Sat, 15 Dec 2018 20:39:33 +0100 Subject: [PATCH 13/16] Wallet docs markdown fixes --- doc/wallet/tls-setup.md | 38 ++++++++++---------- doc/wallet/usage.md | 80 +++++++++++++++++++++-------------------- 2 files changed, 61 insertions(+), 57 deletions(-) diff --git a/doc/wallet/tls-setup.md b/doc/wallet/tls-setup.md index 1d354fb2..b73afbc7 100644 --- a/doc/wallet/tls-setup.md +++ b/doc/wallet/tls-setup.md @@ -12,14 +12,14 @@ If you don't have a domain name there is a possibility to get a TLS certificate ## I have a TLS certificate already Uncomment and update the following lines in wallet config (by default `~/.grin/grin-wallet.toml`): -``` +```toml tls_certificate_file = "/path/to/my/cerificate/fullchain.pem" tls_certificate_key = "/path/to/my/cerificate/privkey.pem" ``` If you have Stratum server enabled (you run a miner) make sure that wallet listener URL starts with `https` in node config (by default `~/.grin/grin-server.toml`): -``` +```toml wallet_listener_url = "https://grin1.example.com:13415" ``` @@ -34,32 +34,32 @@ Go to [Certbot home page](https://certbot.eff.org/), choose I'm using `None of t ### Obtain certificate If you have experince with `certboot` feel free to use any type of challenge. This guide covers the simplest case of HTTP challenge. For this you need to have a web server listening on port `80`, which requires running it as root in the simplest case. We will use the server provided by certbot. **Make sure you have port 80 open** -``` - sudo certbot certonly --standalone -d grin1.example.com +```sh +sudo certbot certonly --standalone -d grin1.example.com ``` It will ask you some questions, as result you should see something like: ``` - Congratulations! Your certificate and chain have been saved at: - /etc/letsencrypt/live/grin1.example.com/fullchain.pem - Your key file has been saved at: - /etc/letsencrypt/live/grin1.example.com/privkey.pem - Your cert will expire on 2019-01-16. To obtain a new or tweaked - version of this certificate in the future, simply run certbot - again. To non-interactively renew *all* of your certificates, run - "certbot renew" +Congratulations! Your certificate and chain have been saved at: + /etc/letsencrypt/live/grin1.example.com/fullchain.pem + Your key file has been saved at: + /etc/letsencrypt/live/grin1.example.com/privkey.pem + Your cert will expire on 2019-01-16. To obtain a new or tweaked + version of this certificate in the future, simply run certbot + again. To non-interactively renew *all* of your certificates, run +"certbot renew" ``` ### Change permissions Now you have the certificate files but only root user can read it. We run grin as `ubuntu` user. There are different scenarios how to fix it, the simplest one is to create a group which will have access to `/etc/letsencrypt` directory and add our user to this group. -``` -$ sudo groupadd tls-cert` -$ sudo usermod -a -G tls-cert ubuntu` -$ chgrp -R tls-cert /etc/letsencrypt` -$ chmod -R g=rX /etc/letsencrypt` -$ sudo chmod 2755 /etc/letsencrypt` +```sh +sudo groupadd tls-cert +sudo usermod -a -G tls-cert ubuntu +chgrp -R tls-cert /etc/letsencrypt +chmod -R g=rX /etc/letsencrypt +sudo chmod 2755 /etc/letsencrypt ``` The last step is needed for renewal, it makes sure that all new files will have the same group ownership. @@ -67,7 +67,7 @@ The last step is needed for renewal, it makes sure that all new files will have ### Update wallet config Refer to `I have a TLS certificate already` because you have it now. Use the folowing values: -``` +```toml tls_certificate_file = "/etc/letsencrypt/live/grin1.example.com/fullchain.pem" tls_certificate_key = "/etc/letsencrypt/live/grin1.example.com/privkey.pem" ``` diff --git a/doc/wallet/usage.md b/doc/wallet/usage.md index a88bf542..a7f6de9e 100644 --- a/doc/wallet/usage.md +++ b/doc/wallet/usage.md @@ -5,7 +5,7 @@ A Grin wallet maintains its state in an LMDB database, with the master seed stored in a separate file. When creating a new wallet, the file structure should be: -```sh +``` ~/[Wallet Directory] -wallet_data/ -db/ @@ -38,8 +38,8 @@ Logging configuration for the wallet is read from `grin-wallet.toml`. The wallet supports multiple accounts. To set the active account for a wallet command, use the '-a' switch, e.g: -``` -[host]$ grin wallet -a account_1 info +```sh +grin wallet -a account_1 info ``` All output creation, transaction building, and querying is done against a particular account in the wallet. @@ -52,7 +52,7 @@ tries to contact a node at `127.0.0.1:13413`. To change this, modify the value i you can provide the `-r` (seRver) switch to the wallet command, e.g.: ```sh -[host]$ grin wallet -a "http://192.168.0.2:1341" info +grin wallet -a "http://192.168.0.2:1341" info ``` If commands that need to update from a grin node can't find one, they will generally inform you that the node couldn't be reached @@ -65,7 +65,7 @@ at wallet creation time, and must be provided for any wallet operation. You will you can also specify it on the command line by providing the `-p`argument. ```sh -[host]$ grin wallet -p mypass info +grin wallet -p mypass info ``` ## Basic Wallet Commands @@ -79,14 +79,14 @@ Before using a wallet a new `grin-wallet.toml` configuration file, master seed c to be generated via the init command as follows: ```sh -[host]$ grin wallet init +grin wallet init ``` You will be prompted to enter a password for the new wallet. By default, your wallet files will be placed into `~/.grin`. Alternatively, if you'd like to run a wallet in a directory other than the default, you can run: ```sh -[host]$ grin wallet -p mypass init -h +grin wallet -p mypass init -h ``` This will create a `grin-wallet.toml` file in the current directory configured to use the data files in the current directory, @@ -110,11 +110,11 @@ the correct recovery phrase,) your wallet contents should again be usable. To recover your wallet seed, delete (or backup) the wallet's `wallet_data/wallet.seed` file, then run: ```sh -[host]$ grin wallet recover -p "[12 or 24 word passphrase separated by spaces" - +grin wallet recover -p "[12 or 24 word passphrase separated by spaces" +``` e.g: - -[host]$ grin wallet recover -p "shiver alarm excuse turtle absorb surface lunch virtual want remind hard slow vacuum park silver asthma engage library battle jelly buffalo female inquiry wire" +```sh +grin wallet recover -p "shiver alarm excuse turtle absorb surface lunch virtual want remind hard slow vacuum park silver asthma engage library battle jelly buffalo female inquiry wire" ``` If you're restoring a wallet from scratch, you'll then need to use the `grin wallet restore` command to scan the chain @@ -124,7 +124,7 @@ You can also view your recovery phrase with your password by running the recover ```sh -[host]$ grin wallet recover +grin wallet recover Password: Your recovery phrase is: shiver alarm excuse turtle absorb surface lunch virtual want remind hard slow vacuum park silver asthma engage library battle jelly buffalo female inquiry wire @@ -135,21 +135,21 @@ Please back-up these words in a non-digital format. To create a new account, use the 'grin wallet account' command with the argument '-c', e.g.: -``` -[host]$ grin wallet account -c my_account +```sh +grin wallet account -c my_account ``` This will create a new account called 'my_account'. To use this account in subsequent commands, provide the '-a' flag to all wallet commands: -``` -[host]$ grin wallet -a my_account info +```sh +grin wallet -a my_account info ``` To display a list of created accounts in the wallet, use the 'account' command with no flags: -``` -[host]$ grin wallet account +```sh +grin wallet account ``` This will print out the following. ```sh @@ -191,13 +191,13 @@ By default the `listen` commands runs in a manner that only allows access from t to other machines, use the `-e` switch: ```sh -[host]$ grin wallet -e listen +grin wallet -e listen ``` To change the port on which the wallet is listening, either configure `grin-wallet.toml` or use the `-l` flag, e.g: ```sh -[host]$ grin wallet -l 14000 listen +grin wallet -l 14000 listen ``` The wallet will listen for requests until the process is cancelled with ``. Note that external ports/firewalls need to be configured @@ -211,7 +211,7 @@ this is how you send Grins to another party. The most important fields here are the destination (`-d`) and the amount itself. To send an amount to another listening wallet: ```sh -[host]$ grin wallet send -d "http://192.168.0.10:13415" 60.00 +grin wallet send -d "http://192.168.0.10:13415" 60.00 ``` This will create a transaction with the other wallet listening at 192.168.0.10, port 13415 which credits the other wallet 60 grins @@ -231,13 +231,13 @@ Outputs in your wallet will appear as unconfirmed or locked until the transactio You can also create a transaction entirely within your own wallet by specifying the method 'self'. Using the 'self' method, you can send yourself money in a single command (for testing purposes,) or distribute funds between accounts within your wallet without having to run a listener or manipulate files. For instance, to send funds from your wallet's 'default' account to an account called 'account1', use: ```sh -[host]$ grin wallet send -m self -d "account1" 60 +grin wallet send -m self -d "account1" 60 ``` or, to send between accounts, use the -a flag to specify the source account: ```sh -[host]$ grin wallet -a "my_source_account" send -m self -d "my_dest_account" 60 +grin wallet -a "my_source_account" send -m self -d "my_dest_account" 60 ``` When sending to self, the transaction will be created and posted to the chain in the same operation. @@ -247,7 +247,7 @@ Other flags here are: * `-m` 'Method', which can be 'http', 'file' or 'self' (described above). If 'http' is specified (default), the transaction will be sent to the IP address which follows the `-d` flag. If 'file' is specified, Grin wallet will generate a partial transaction file under the file name specified in the `-d` flag. This file needs to be signed by the recipient using the `grin wallet receive -i filename` command and finalized by the sender using the `grin wallet finalize -i filename.response` command. To create a partial transaction file, use: ```sh -[host]$ grin wallet send -d "transaction" -m file 60.00 +grin wallet send -d "transaction" -m file 60.00 ``` * `-s` 'Selection strategy', which can be 'all' or 'smallest'. Since it's advantageous for outputs to be removed from the Grin chain, @@ -257,7 +257,7 @@ Other flags here are: cover the amount you want to send + fees, use: ```sh -[host]$ grin wallet send -d "http://192.168.0.10:13415" -s smallest 60.00 +grin wallet send -d "http://192.168.0.10:13415" -s smallest 60.00 ``` * `-f` 'Fluff' Grin uses a protocol called 'Dandelion' which bounces your transaction directly through several listening nodes in a @@ -265,7 +265,7 @@ Other flags here are: the time before your transaction appears on the chain. To ignore the stem phase and broadcast immediately: ```sh -[host]$ grin wallet send -f -d "http://192.168.0.10:13415" 60.00 +grin wallet send -f -d "http://192.168.0.10:13415" 60.00 ``` * `-g` 'Message' - You can specify an optional message to include alongside your transaction data. This message is purely for informational @@ -274,7 +274,7 @@ a signature that can be verified with the participant's public key. A message ca command. ```sh -[host]$ grin wallet send -f -d "http://192.168.0.10:13415" -g "This is from Dave" 60.00 +grin wallet send -f -d "http://192.168.0.10:13415" -g "This is from Dave" 60.00 ``` ### outputs @@ -282,7 +282,7 @@ command. Simply displays all the the outputs in your wallet: e.g: ```sh -[host]$ grin wallet outputs +grin wallet outputs Wallet Outputs - Account 'default' - Block Height: 49 ------------------------------------------------------------------------------------------------------------------------------------------------ Key Id Child Key Index Block Height Locked Until Status Is Coinbase? Num. of Confirmations Value Transaction @@ -302,8 +302,8 @@ Wallet Outputs - Account 'default' - Block Height: 49 Spent outputs are not shown by default. To show them, provide the `-s` flag. - ```sh -[host]$ grin wallet -s outputs +```sh +grin wallet -s outputs ``` ### txs @@ -314,7 +314,7 @@ this transaction log is necessary in order to allow your wallet to keep track of transaction log, use the `txs` ```sh -[host]$ grin wallet txs +grin wallet txs Transaction Log - Account 'default' - Block Height: 49 ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Id Type Shared Transaction Id Creation Time Confirmed? Confirmation Time Num. Inputs Num. Outputs Amount Credited Amount Debited Fee Net Difference @@ -331,11 +331,11 @@ Transaction Log - Account 'default' - Block Height: 49 ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 6 Received Tx 03715cf6-f29b-4a3a-bda5-b02cba6bf0d9 2018-07-20 19:46:46.120244904 UTC false None 0 1 60.000000000 0.000000000 None 60.000000000 ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ - +``` To see the inputs/outputs associated with a particular transaction, use the `-i` switch providing the Id of the given transaction, e.g: ```sh -[host]$ grin wallet txs -i 6 +grin wallet txs -i 6 Transaction Log - Account 'default' - Block Height: 49 ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Id Type Shared Transaction Id Creation Time Confirmed? Confirmation Time Num. Inputs Num. Outputs Amount Credited Amount Debited Fee Net Difference @@ -365,8 +365,8 @@ outputs can be unlocked and associate unconfirmed outputs removed with the `canc Running against the data above: ```sh -[host]$ grin wallet cancel -i 6 -[host]$ grin wallet txs -i 6 +grin wallet cancel -i 6 +grin wallet txs -i 6 Transaction Log - Account 'default' - Block Height: 49 ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Id Type Shared Transaction Id Creation Time Confirmed? Confirmation Time Num. Inputs Num. Outputs Amount Credited Amount Debited Fee Net Difference @@ -389,13 +389,17 @@ If you're the sender of a posted transaction that doesn't confirm on the chain ( To do this, look up the transaction id using the `grin wallet txs` command, and using the id (say 3 in this example,) enter: -`grin wallet repost -i 3` +```sh +grin wallet repost -i 3 +``` This will attempt to repost the transaction to the chain. Note this won't attempt to send if the transaction is already marked as 'confirmed' within the wallet. You can also use the `repost` command to dump the transaction in a raw json format with the `-m` (duMp) switch, e.g: -`grin wallet repost -i 3 -m tx_3.json` +```sh +grin wallet repost -i 3 -m tx_3.json +``` This will create a file called tx_3.json containing your raw transaction data. Note that this formatting in the file isn't yet very user-readable. @@ -416,7 +420,7 @@ Delete the newly generated wallet data directory and seed file: [host@new_wallet_dir]# rm wallet_data/wallet.seed ``` -If you need to recover your wallet seed from a recovery phrase, use the `grin wallet recover -p "[recovery phrase]" command +If you need to recover your wallet seed from a recovery phrase, use the `grin wallet recover -p "[recovery phrase]"` command as outlined above. Otherwise, if you're restoring from a backed-up seed file, simply copy your backed up `wallet.seed` file into the new `wallet_data` directory, ensuring it's called `wallet.seed` From 9a4895c86a67b69e99dbcdb9c53d0d4d7035be44 Mon Sep 17 00:00:00 2001 From: Antioch Peverell Date: Sat, 15 Dec 2018 20:44:33 +0000 Subject: [PATCH 14/16] take lock once in get_header_for_output (#2161) --- chain/src/chain.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/chain/src/chain.rs b/chain/src/chain.rs index d845841f..be4dd05e 100644 --- a/chain/src/chain.rs +++ b/chain/src/chain.rs @@ -1123,11 +1123,9 @@ impl Chain { &self, output_ref: &OutputIdentifier, ) -> Result { - let pos = { - let txhashset = self.txhashset.read(); - let (_, pos) = txhashset.is_unspent(output_ref)?; - pos - }; + let txhashset = self.txhashset.read(); + + let (_, pos) = txhashset.is_unspent(output_ref)?; let mut min = 1; let mut max = { @@ -1137,8 +1135,8 @@ impl Chain { loop { let search_height = max - (max - min) / 2; - let h = self.get_header_by_height(search_height)?; - let h_prev = self.get_header_by_height(search_height - 1)?; + let h = txhashset.get_header_by_height(search_height)?; + let h_prev = txhashset.get_header_by_height(search_height - 1)?; if pos > h.output_mmr_size { min = search_height; } else if pos < h_prev.output_mmr_size { From a50dcbfaa5ca655a3f5a8df41411b68d01c8e6b8 Mon Sep 17 00:00:00 2001 From: Antioch Peverell Date: Sun, 16 Dec 2018 09:26:17 +0000 Subject: [PATCH 15/16] less txhashset locking (#2163) * less txhashset locking * rework server stats a bit --- chain/src/chain.rs | 21 ++++++++++++--------- servers/src/common/adapters.rs | 10 ++++++++-- servers/src/grin/server.rs | 5 ++++- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/chain/src/chain.rs b/chain/src/chain.rs index be4dd05e..e19298b5 100644 --- a/chain/src/chain.rs +++ b/chain/src/chain.rs @@ -28,6 +28,7 @@ use crate::lmdb; use crate::pipe; use crate::store; use crate::txhashset; +use crate::txhashset::TxHashSet; use crate::types::{ BlockStatus, ChainAdapter, NoStatus, Options, Tip, TxHashSetRoots, TxHashsetWriteStatus, }; @@ -207,6 +208,11 @@ impl Chain { Ok(chain) } + /// Return our shared txhashset instance. + pub fn txhashset(&self) -> Arc> { + self.txhashset.clone() + } + fn log_heads(store: &store::ChainStore) -> Result<(), Error> { let head = store.head()?; debug!( @@ -1106,19 +1112,16 @@ impl Chain { .map_err(|e| ErrorKind::StoreErr(e, "chain get block_sums".to_owned()).into()) } - /// Gets the block header at the provided height + /// Gets the block header at the provided height. + /// Note: This takes a read lock on the txhashset. + /// Take care not to call this repeatedly in a tight loop. pub fn get_header_by_height(&self, height: u64) -> Result { - let mut txhashset = self.txhashset.write(); - let mut batch = self.store.batch()?; - let header = txhashset::header_extending(&mut txhashset, &mut batch, |extension| { - let header = extension.get_header_by_height(height)?; - Ok(header) - })?; - + let txhashset = self.txhashset.read(); + let header = txhashset.get_header_by_height(height)?; Ok(header) } - /// Gets the block header in which a given output appears in the txhashset + /// Gets the block header in which a given output appears in the txhashset. pub fn get_header_for_output( &self, output_ref: &OutputIdentifier, diff --git a/servers/src/common/adapters.rs b/servers/src/common/adapters.rs index e5e23175..3fe0a6bd 100644 --- a/servers/src/common/adapters.rs +++ b/servers/src/common/adapters.rs @@ -260,6 +260,9 @@ impl p2p::ChainAdapter for NetToChainAdapter { let max_height = self.chain().header_head().unwrap().height; + let txhashset = self.chain().txhashset(); + let txhashset = txhashset.read(); + // looks like we know one, getting as many following headers as allowed let hh = header.height; let mut headers = vec![]; @@ -268,7 +271,7 @@ impl p2p::ChainAdapter for NetToChainAdapter { break; } - if let Ok(header) = self.chain().get_header_by_height(h) { + if let Ok(header) = txhashset.get_header_by_height(h) { headers.push(header); } else { error!("Failed to locate headers successfully."); @@ -399,9 +402,12 @@ impl NetToChainAdapter { // Find the first locator hash that refers to a known header on our main chain. fn find_common_header(&self, locator: &[Hash]) -> Option { + let txhashset = self.chain().txhashset(); + let txhashset = txhashset.read(); + for hash in locator { if let Ok(header) = self.chain().get_block_header(&hash) { - if let Ok(header_at_height) = self.chain().get_header_by_height(header.height) { + if let Ok(header_at_height) = txhashset.get_header_by_height(header.height) { if header.hash() == header_at_height.hash() { return Some(header); } diff --git a/servers/src/grin/server.rs b/servers/src/grin/server.rs index 6f267de4..89b1e3f9 100644 --- a/servers/src/grin/server.rs +++ b/servers/src/grin/server.rs @@ -374,6 +374,9 @@ impl Server { let tip_height = self.chain.head().unwrap().height as i64; let mut height = tip_height as i64 - last_blocks.len() as i64 + 1; + let txhashset = self.chain.txhashset(); + let txhashset = txhashset.read(); + let diff_entries: Vec = last_blocks .windows(2) .map(|pair| { @@ -385,7 +388,7 @@ impl Server { // Use header hash if real header. // Default to "zero" hash if synthetic header_info. let hash = if height >= 0 { - if let Ok(header) = self.chain.get_header_by_height(height as u64) { + if let Ok(header) = txhashset.get_header_by_height(height as u64) { header.hash() } else { ZERO_HASH From c188b60a38833abcbdeaa24c48eddff660af83aa Mon Sep 17 00:00:00 2001 From: Antioch Peverell Date: Sun, 16 Dec 2018 09:26:39 +0000 Subject: [PATCH 16/16] readonly verify_coinbase_maturity (#2164) * move verify_coinbase_maturity into utxo_view no longer need a write lock on txhashset * rustfmt --- chain/src/chain.rs | 6 +-- chain/src/pipe.rs | 18 +++----- chain/src/txhashset/txhashset.rs | 51 ++++------------------ chain/src/txhashset/utxo_view.rs | 75 +++++++++++++++++++++++++++++--- 4 files changed, 85 insertions(+), 65 deletions(-) diff --git a/chain/src/chain.rs b/chain/src/chain.rs index e19298b5..70ab100e 100644 --- a/chain/src/chain.rs +++ b/chain/src/chain.rs @@ -506,9 +506,9 @@ impl Chain { /// that has not yet sufficiently matured. pub fn verify_coinbase_maturity(&self, tx: &Transaction) -> Result<(), Error> { let height = self.next_block_height()?; - let mut txhashset = self.txhashset.write(); - txhashset::extending_readonly(&mut txhashset, |extension| { - extension.verify_coinbase_maturity(&tx.inputs(), height)?; + let txhashset = self.txhashset.read(); + txhashset::utxo_view(&txhashset, |utxo| { + utxo.verify_coinbase_maturity(&tx.inputs(), height)?; Ok(()) }) } diff --git a/chain/src/pipe.rs b/chain/src/pipe.rs index 3f949869..3fcc5583 100644 --- a/chain/src/pipe.rs +++ b/chain/src/pipe.rs @@ -434,16 +434,10 @@ fn validate_block(block: &Block, ctx: &mut BlockContext<'_>) -> Result<(), Error Ok(()) } -/// TODO - This can move into the utxo_view. -/// Verify the block is not attempting to spend coinbase outputs -/// before they have sufficiently matured. -/// Note: requires a txhashset extension. -fn verify_coinbase_maturity( - block: &Block, - ext: &mut txhashset::Extension<'_>, -) -> Result<(), Error> { - ext.verify_coinbase_maturity(&block.inputs(), block.header.height)?; - Ok(()) +/// Verify the block is not spending coinbase outputs before they have sufficiently matured. +fn verify_coinbase_maturity(block: &Block, ext: &txhashset::Extension<'_>) -> Result<(), Error> { + ext.utxo_view() + .verify_coinbase_maturity(&block.inputs(), block.header.height) } /// Some "real magick" verification logic. @@ -649,7 +643,5 @@ pub fn rewind_and_apply_fork(b: &Block, ext: &mut txhashset::Extension<'_>) -> R } fn validate_utxo(block: &Block, ext: &txhashset::Extension<'_>) -> Result<(), Error> { - let utxo = ext.utxo_view(); - utxo.validate_block(block)?; - Ok(()) + ext.utxo_view().validate_block(block) } diff --git a/chain/src/txhashset/txhashset.rs b/chain/src/txhashset/txhashset.rs index b711caf1..0ed33e08 100644 --- a/chain/src/txhashset/txhashset.rs +++ b/chain/src/txhashset/txhashset.rs @@ -20,7 +20,7 @@ use crate::core::core::hash::{Hash, Hashed}; use crate::core::core::merkle_proof::MerkleProof; use crate::core::core::pmmr::{self, ReadonlyPMMR, RewindablePMMR, PMMR}; use crate::core::core::{ - Block, BlockHeader, Input, Output, OutputFeatures, OutputIdentifier, TxKernel, TxKernelEntry, + Block, BlockHeader, Input, Output, OutputIdentifier, TxKernel, TxKernelEntry, }; use crate::core::global; use crate::core::ser::{PMMRIndexHashable, PMMRable}; @@ -353,11 +353,13 @@ where { let output_pmmr = ReadonlyPMMR::at(&trees.output_pmmr_h.backend, trees.output_pmmr_h.last_pos); + let header_pmmr = + ReadonlyPMMR::at(&trees.header_pmmr_h.backend, trees.header_pmmr_h.last_pos); // Create a new batch here to pass into the utxo_view. // Discard it (rollback) after we finish with the utxo_view. let batch = trees.commit_index.batch()?; - let utxo = UTXOView::new(output_pmmr, &batch); + let utxo = UTXOView::new(output_pmmr, header_pmmr, &batch); res = inner(&utxo); } res @@ -829,46 +831,11 @@ impl<'a> Extension<'a> { /// Build a view of the current UTXO set based on the output PMMR. pub fn utxo_view(&'a self) -> UTXOView<'a> { - UTXOView::new(self.output_pmmr.readonly_pmmr(), self.batch) - } - - /// Verify we are not attempting to spend any coinbase outputs - /// that have not sufficiently matured. - pub fn verify_coinbase_maturity( - &mut self, - inputs: &Vec, - height: u64, - ) -> Result<(), Error> { - // Find the greatest output pos of any coinbase - // outputs we are attempting to spend. - let pos = inputs - .iter() - .filter(|x| x.features.contains(OutputFeatures::COINBASE_OUTPUT)) - .filter_map(|x| self.batch.get_output_pos(&x.commitment()).ok()) - .max() - .unwrap_or(0); - - if pos > 0 { - // If we have not yet reached 1,000 / 1,440 blocks then - // we can fail immediately as coinbase cannot be mature. - if height < global::coinbase_maturity() { - return Err(ErrorKind::ImmatureCoinbase.into()); - } - - // Find the "cutoff" pos in the output MMR based on the - // header from 1,000 blocks ago. - let cutoff_height = height.checked_sub(global::coinbase_maturity()).unwrap_or(0); - let cutoff_header = self.get_header_by_height(cutoff_height)?; - let cutoff_pos = cutoff_header.output_mmr_size; - - // If any output pos exceed the cutoff_pos - // we know they have not yet sufficiently matured. - if pos > cutoff_pos { - return Err(ErrorKind::ImmatureCoinbase.into()); - } - } - - Ok(()) + UTXOView::new( + self.output_pmmr.readonly_pmmr(), + self.header_pmmr.readonly_pmmr(), + self.batch, + ) } /// Apply a new block to the existing state. diff --git a/chain/src/txhashset/utxo_view.rs b/chain/src/txhashset/utxo_view.rs index cd4c2d98..586ea108 100644 --- a/chain/src/txhashset/utxo_view.rs +++ b/chain/src/txhashset/utxo_view.rs @@ -14,8 +14,10 @@ //! Lightweight readonly view into output MMR for convenience. -use crate::core::core::pmmr::ReadonlyPMMR; -use crate::core::core::{Block, Input, Output, Transaction}; +use crate::core::core::hash::Hash; +use crate::core::core::pmmr::{self, ReadonlyPMMR}; +use crate::core::core::{Block, BlockHeader, Input, Output, OutputFeatures, Transaction}; +use crate::core::global; use crate::core::ser::PMMRIndexHashable; use crate::error::{Error, ErrorKind}; use crate::store::Batch; @@ -23,17 +25,23 @@ use grin_store::pmmr::PMMRBackend; /// Readonly view of the UTXO set (based on output MMR). pub struct UTXOView<'a> { - pmmr: ReadonlyPMMR<'a, Output, PMMRBackend>, + output_pmmr: ReadonlyPMMR<'a, Output, PMMRBackend>, + header_pmmr: ReadonlyPMMR<'a, BlockHeader, PMMRBackend>, batch: &'a Batch<'a>, } impl<'a> UTXOView<'a> { /// Build a new UTXO view. pub fn new( - pmmr: ReadonlyPMMR<'a, Output, PMMRBackend>, + output_pmmr: ReadonlyPMMR<'a, Output, PMMRBackend>, + header_pmmr: ReadonlyPMMR<'a, BlockHeader, PMMRBackend>, batch: &'a Batch<'_>, ) -> UTXOView<'a> { - UTXOView { pmmr, batch } + UTXOView { + output_pmmr, + header_pmmr, + batch, + } } /// Validate a block against the current UTXO set. @@ -69,7 +77,7 @@ impl<'a> UTXOView<'a> { // Compare the hash in the output MMR at the expected pos. fn validate_input(&self, input: &Input) -> Result<(), Error> { if let Ok(pos) = self.batch.get_output_pos(&input.commitment()) { - if let Some(hash) = self.pmmr.get_hash(pos) { + if let Some(hash) = self.output_pmmr.get_hash(pos) { if hash == input.hash_with_index(pos - 1) { return Ok(()); } @@ -81,7 +89,7 @@ impl<'a> UTXOView<'a> { // Output is valid if it would not result in a duplicate commitment in the output MMR. fn validate_output(&self, output: &Output) -> Result<(), Error> { if let Ok(pos) = self.batch.get_output_pos(&output.commitment()) { - if let Some(out_mmr) = self.pmmr.get_data(pos) { + if let Some(out_mmr) = self.output_pmmr.get_data(pos) { if out_mmr.commitment() == output.commitment() { return Err(ErrorKind::DuplicateCommitment(output.commitment()).into()); } @@ -89,4 +97,57 @@ impl<'a> UTXOView<'a> { } Ok(()) } + + /// Verify we are not attempting to spend any coinbase outputs + /// that have not sufficiently matured. + pub fn verify_coinbase_maturity(&self, inputs: &Vec, height: u64) -> Result<(), Error> { + // Find the greatest output pos of any coinbase + // outputs we are attempting to spend. + let pos = inputs + .iter() + .filter(|x| x.features.contains(OutputFeatures::COINBASE_OUTPUT)) + .filter_map(|x| self.batch.get_output_pos(&x.commitment()).ok()) + .max() + .unwrap_or(0); + + if pos > 0 { + // If we have not yet reached 1,000 / 1,440 blocks then + // we can fail immediately as coinbase cannot be mature. + if height < global::coinbase_maturity() { + return Err(ErrorKind::ImmatureCoinbase.into()); + } + + // Find the "cutoff" pos in the output MMR based on the + // header from 1,000 blocks ago. + let cutoff_height = height.checked_sub(global::coinbase_maturity()).unwrap_or(0); + let cutoff_header = self.get_header_by_height(cutoff_height)?; + let cutoff_pos = cutoff_header.output_mmr_size; + + // If any output pos exceed the cutoff_pos + // we know they have not yet sufficiently matured. + if pos > cutoff_pos { + return Err(ErrorKind::ImmatureCoinbase.into()); + } + } + + Ok(()) + } + + /// Get the header hash for the specified pos from the underlying MMR backend. + fn get_header_hash(&self, pos: u64) -> Option { + self.header_pmmr.get_data(pos).map(|x| x.hash()) + } + + /// Get the header at the specified height based on the current state of the extension. + /// Derives the MMR pos from the height (insertion index) and retrieves the header hash. + /// Looks the header up in the db by hash. + pub fn get_header_by_height(&self, height: u64) -> Result { + let pos = pmmr::insertion_to_pmmr_index(height + 1); + if let Some(hash) = self.get_header_hash(pos) { + let header = self.batch.get_block_header(&hash)?; + Ok(header) + } else { + Err(ErrorKind::Other(format!("get header by height")).into()) + } + } }