hash (features|commitment) in output mmr (#615)

* experiment with lock_heights on outputs * playing around with lock_height as part of the switch commitment hash * cleanup * include features in the switch commit hash key * commit * rebase off master * commit * cleanup * missing docs * rework coinbase maturity test to build valid tx * pool and chain tests passing (inputs have switch commitments) * commit * cleanup * check inputs spending coinbase outputs have valid lock_heights * wip - got it building (tests still failing) * use zero key for non coinbase switch commit hash * fees and height wrong order... * send output lock_height over to wallet via api * no more header by height index workaround this for wallet refresh and wallet restore * refresh heights for unspent wallet outputs where missing * TODO - might be slow? * simplify - do not pass around lock_height for non coinbase outputs * commit * fix tests after merge * build input vs coinbase_input switch commit hash key encodes lock_height cleanup output by commit index (currently broken...) * is_unspent and get_unspent cleanup - we have no outputs, only switch_commit_hashes * separate concept of utxo vs output in the api utxos come from the sumtrees (and only the sumtrees, limited info) outputs come from blocks (and we need to look them up via block height) * cleanup * better api support for block outputs with range proofs * basic wallet operations appear to work restore is not working fully refresh refreshes heights correctly (at least appears to) * wallet refresh and wallet restore appear to be working now * fix core tests * fix some mine_simple_chain tests * fixup chain tests * rework so pool tests pass * wallet restore now safely habndles duplicate commitments (reused wallet keys) for coinbase outputs where lock_height is _very_ important * wip * validate_coinbase_maturity got things building tests are failing * lite vs full versions of is_unspent * builds and working locally zero-conf - what to do here? * handle zero-conf edge case (use latest block) * introduce OutputIdentifier, avoid leaking SumCommit everywhere * fix the bad merge * pool verifies coinbase maturity via is_matured this uses sumtree in a consistent way * cleanup * add docs, cleanup build warnings * fix core tests * fix chain tests * fix pool tests * cleanup debug logging that we no longer need * make out_block optional on an input (only care about it for spending coinbase outputs) * cleanup * bump the build
2018-01-16 22:03:40 -05:00
parent 7e7c8e157e
commit cbd3b2ff87
31 changed files with 1345 additions and 901 deletions
@@ -1,4 +1,4 @@
-// Copyright 2016 The Grin Developers
+// Copyright 2017 The Grin Developers
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -19,22 +19,18 @@ use std::collections::hash_map::Entry;
 use std::collections::HashMap;

 use api;
+use core::core::hash::Hash;
 use types::*;
 use keychain::{Identifier, Keychain};
 use util::secp::pedersen;
 use util;
 use util::LOGGER;

-// Transitions a local wallet output from Unconfirmed -> Unspent.
-// Also updates the height and lock_height based on latest from the api.
-fn refresh_output(out: &mut OutputData, api_out: &api::Output) {
-	out.height = api_out.height;
-	out.lock_height = api_out.lock_height;

+// Transitions a local wallet output from Unconfirmed -> Unspent.
+fn mark_unspent_output(out: &mut OutputData) {
 	match out.status {
-		OutputStatus::Unconfirmed => {
-			out.status = OutputStatus::Unspent;
-		}
+		OutputStatus::Unconfirmed => out.status = OutputStatus::Unspent,
 		_ => (),
 	}
 }
@@ -45,79 +41,92 @@ fn refresh_output(out: &mut OutputData, api_out: &api::Output) {
 // Locked -> Spent
 fn mark_spent_output(out: &mut OutputData) {
 	match out.status {
-		OutputStatus::Unspent | OutputStatus::Locked => out.status = OutputStatus::Spent,
+		OutputStatus::Unspent => out.status = OutputStatus::Spent,
+		OutputStatus::Locked => out.status = OutputStatus::Spent,
 		_ => (),
 	}
 }

-/// Builds multiple api queries to retrieve the latest output data from the node.
-/// So we can refresh the local wallet outputs.
 pub fn refresh_outputs(config: &WalletConfig, keychain: &Keychain) -> Result<(), Error> {
-	debug!(LOGGER, "Refreshing wallet outputs");
-	let mut wallet_outputs: HashMap<pedersen::Commitment, Identifier> = HashMap::new();
-	let mut commits: Vec<pedersen::Commitment> = vec![];
+	refresh_output_state(config, keychain)?;
+	refresh_missing_block_hashes(config, keychain)?;
+	Ok(())
+}

-	// build a local map of wallet outputs by commits
+// TODO - this might be slow if we have really old outputs that have never been refreshed
+fn refresh_missing_block_hashes(config: &WalletConfig, keychain: &Keychain) -> Result<(), Error> {
+	// build a local map of wallet outputs keyed by commit
 	// and a list of outputs we want to query the node for
+	let mut wallet_outputs: HashMap<pedersen::Commitment, Identifier> = HashMap::new();
 	let _ = WalletData::read_wallet(&config.data_file_dir, |wallet_data| {
 		for out in wallet_data
 			.outputs
 			.values()
-			.filter(|out| out.root_key_id == keychain.root_key_id())
-			.filter(|out| out.status != OutputStatus::Spent)
+			.filter(|x| {
+				x.root_key_id == keychain.root_key_id() &&
+				x.block_hash == Hash::zero() &&
+				x.status == OutputStatus::Unspent
+			})
 		{
-			let commit = keychain
-				.commit_with_key_index(out.value, out.n_child)
-				.unwrap();
-			commits.push(commit);
+			let commit = keychain.commit_with_key_index(out.value, out.n_child).unwrap();
 			wallet_outputs.insert(commit, out.key_id.clone());
 		}
 	});

-	// build the necessary query params -
-	// ?id=xxx&id=yyy&id=zzz
-	let query_params: Vec<String> = commits
-		.iter()
+	// nothing to do so return (otherwise we hit the api with a monster query...)
+	if wallet_outputs.is_empty() {
+		return Ok(());
+	}
+
+	debug!(
+		LOGGER,
+		"Refreshing missing block hashes (and heights) for {} outputs",
+		wallet_outputs.len(),
+	);
+
+	let mut id_params: Vec<String> = wallet_outputs
+		.keys()
 		.map(|commit| {
 			let id = util::to_hex(commit.as_ref().to_vec());
 			format!("id={}", id)
 		})
 		.collect();

-	// build a map of api outputs by commit so we can look them up efficiently
-	let mut api_outputs: HashMap<pedersen::Commitment, api::Output> = HashMap::new();
+	let tip = get_tip_from_node(config)?;

-	// size of the batch size for the utxo query
-	let batch_query_size = 500;
+	let height_params = format!(
+		"start_height={}&end_height={}",
+		0,
+		tip.height,
+	);
+	let mut query_params = vec![height_params];
+	query_params.append(&mut id_params);

-	let mut index_id = 0;
-	while index_id < query_params.len() {
-		let batch_query: Vec<String>;
-		if index_id + batch_query_size > query_params.len() {
-			batch_query = query_params[index_id..query_params.len()].to_vec();
-			index_id = query_params.len();
-		} else {
-			batch_query = query_params[index_id..index_id + batch_query_size].to_vec();
-			index_id = index_id + batch_query_size;
-		}
+	let url =
+		format!(
+		"{}/v1/chain/utxos/byheight?{}",
+		config.check_node_api_http_addr,
+		query_params.join("&"),
+	);
+	debug!(LOGGER, "{:?}", url);

-		let query_string = batch_query.join("&");
-
-		let url = format!(
-			"{}/v1/chain/utxos/byids?{}",
-			config.check_node_api_http_addr, query_string,
-		);
-
-		match api::client::get::<Vec<api::Output>>(url.as_str()) {
-			Ok(outputs) => for out in outputs {
-				api_outputs.insert(out.commit, out);
-			},
-			Err(e) => {
-				// if we got anything other than 200 back from server, don't attempt to refresh
-				// the wallet data after
-				return Err(Error::Node(e));
+	let mut api_blocks: HashMap<pedersen::Commitment, api::BlockHeaderInfo> = HashMap::new();
+	match api::client::get::<Vec<api::BlockOutputs>>(url.as_str()) {
+		Ok(blocks) => {
+			for block in blocks {
+				for out in block.outputs {
+					if let Ok(c) = util::from_hex(String::from(out.commit)) {
+						let commit = pedersen::Commitment::from_vec(c);
+						api_blocks.insert(commit, block.header.clone());
+					}
+				}
 			}
-		};
+		}
+		Err(e) => {
+			// if we got anything other than 200 back from server, bye
+			error!(LOGGER, "Refresh failed... unable to contact node: {}", e);
+			return Err(Error::Node(e));
+		}
 	}

 	// now for each commit, find the output in the wallet and
@@ -125,18 +134,87 @@ pub fn refresh_outputs(config: &WalletConfig, keychain: &Keychain) -> Result<(),
 	// and refresh it in-place in the wallet.
 	// Note: minimizing the time we spend holding the wallet lock.
 	WalletData::with_wallet(&config.data_file_dir, |wallet_data| {
-		for commit in commits {
+		for commit in wallet_outputs.keys() {
 			let id = wallet_outputs.get(&commit).unwrap();
 			if let Entry::Occupied(mut output) = wallet_data.outputs.entry(id.to_hex()) {
-				match api_outputs.get(&commit) {
-					Some(api_output) => refresh_output(&mut output.get_mut(), api_output),
-					None => mark_spent_output(&mut output.get_mut()),
-				};
+				if let Some(b) = api_blocks.get(&commit) {
+					let output = output.get_mut();
+					output.block_hash = Hash::from_hex(&b.hash).unwrap();
+					output.height = b.height;
+				}
 			}
 		}
 	})
 }

+/// Builds a single api query to retrieve the latest output data from the node.
+/// So we can refresh the local wallet outputs.
+fn refresh_output_state(config: &WalletConfig, keychain: &Keychain) -> Result<(), Error> {
+	debug!(LOGGER, "Refreshing wallet outputs");
+
+	// build a local map of wallet outputs keyed by commit
+	// and a list of outputs we want to query the node for
+	let mut wallet_outputs: HashMap<pedersen::Commitment, Identifier> = HashMap::new();
+	let _ = WalletData::read_wallet(&config.data_file_dir, |wallet_data| {
+		for out in wallet_data
+			.outputs
+			.values()
+			.filter(|x| {
+				x.root_key_id == keychain.root_key_id() &&
+				x.status != OutputStatus::Spent
+			})
+		{
+			let commit = keychain.commit_with_key_index(out.value, out.n_child).unwrap();
+			wallet_outputs.insert(commit, out.key_id.clone());
+		}
+	});
+
+	// build the necessary query params -
+	// ?id=xxx&id=yyy&id=zzz
+	let query_params: Vec<String> = wallet_outputs
+		.keys()
+		.map(|commit| {
+			let id = util::to_hex(commit.as_ref().to_vec());
+			format!("id={}", id)
+		})
+		.collect();
+
+	// build a map of api outputs by commit so we can look them up efficiently
+	let mut api_utxos: HashMap<pedersen::Commitment, api::Utxo> = HashMap::new();
+
+	let query_string = query_params.join("&");
+
+	let url = format!(
+		"{}/v1/chain/utxos/byids?{}",
+		config.check_node_api_http_addr, query_string,
+	);
+
+	match api::client::get::<Vec<api::Utxo>>(url.as_str()) {
+		Ok(outputs) => for out in outputs {
+			api_utxos.insert(out.commit, out);
+		},
+		Err(e) => {
+			// if we got anything other than 200 back from server, don't attempt to refresh
+			// the wallet data after
+			return Err(Error::Node(e));
+		}
+	};
+
+	// now for each commit, find the output in the wallet and
+	// the corresponding api output (if it exists)
+	// and refresh it in-place in the wallet.
+	// Note: minimizing the time we spend holding the wallet lock.
+	WalletData::with_wallet(&config.data_file_dir, |wallet_data| for commit in wallet_outputs.keys() {
+		let id = wallet_outputs.get(&commit).unwrap();
+		if let Entry::Occupied(mut output) = wallet_data.outputs.entry(id.to_hex()) {
+			match api_utxos.get(&commit) {
+				Some(_) => mark_unspent_output(&mut output.get_mut()),
+				None => mark_spent_output(&mut output.get_mut()),
+			};
+		}
+	})
+}
+
 pub fn get_tip_from_node(config: &WalletConfig) -> Result<api::Tip, Error> {
 	let url = format!("{}/v1/chain", config.check_node_api_http_addr);
 	api::client::get::<api::Tip>(url.as_str()).map_err(|e| Error::Node(e))
@@ -21,7 +21,6 @@ use prettytable;
 pub fn show_info(config: &WalletConfig, keychain: &Keychain) {
 	let result = checker::refresh_outputs(&config, &keychain);

-
 	let _ = WalletData::read_wallet(&config.data_file_dir, |wallet_data| {
 		let (current_height, from) = match checker::get_tip_from_node(config) {
 			Ok(tip) => (tip.height, "from server node"),
@@ -30,26 +29,26 @@ pub fn show_info(config: &WalletConfig, keychain: &Keychain) {
 				None => (0, "node/wallet unavailable"),
 			},
 		};
-		let mut unspent_total=0;
-		let mut unspent_but_locked_total=0;
-		let mut unconfirmed_total=0;
-		let mut locked_total=0;
+		let mut unspent_total = 0;
+		let mut unspent_but_locked_total = 0;
+		let mut unconfirmed_total = 0;
+		let mut locked_total = 0;
 		for out in wallet_data
 			.outputs
 			.values()
 			.filter(|out| out.root_key_id == keychain.root_key_id())
 		{
 			if out.status == OutputStatus::Unspent {
-				unspent_total+=out.value;
+				unspent_total += out.value;
 				if out.lock_height > current_height {
-						unspent_but_locked_total+=out.value;
+						unspent_but_locked_total += out.value;
 				}
 			}
 			if out.status == OutputStatus::Unconfirmed && !out.is_coinbase {
-				unconfirmed_total+=out.value;
+				unconfirmed_total += out.value;
 			}
 			if out.status == OutputStatus::Locked {
-				locked_total+=out.value;
+				locked_total += out.value;
 			}
 		};

@@ -25,7 +25,8 @@ use serde_json;
 use api;
 use core::consensus::reward;
 use core::core::{build, Block, Output, Transaction, TxKernel, amount_to_hr_string};
-use core::ser;
+use core::core::hash::Hash;
+use core::{global, ser};
 use keychain::{Identifier, Keychain};
 use types::*;
 use util::{LOGGER, to_hex, secp};
@@ -68,7 +69,7 @@ fn handle_sender_initiation(

    if fee > amount {
 		info!(
-			LOGGER, 
+			LOGGER,
 			"Rejected the transfer because transaction fee ({}) exceeds received amount ({}).",
 			amount_to_hr_string(fee),
 			amount_to_hr_string(amount)
@@ -96,6 +97,7 @@ fn handle_sender_initiation(
 			height: 0,
 			lock_height: 0,
 			is_coinbase: false,
+			block_hash: Hash::zero(),
 		});

 		key_id
@@ -259,6 +261,9 @@ pub fn receive_coinbase(
 ) -> Result<(Output, TxKernel, BlockFees), Error> {
 	let root_key_id = keychain.root_key_id();

+	let height = block_fees.height;
+	let lock_height = height + global::coinbase_maturity();
+
 	// Now acquire the wallet lock and write the new output.
 	let (key_id, derivation) = WalletData::with_wallet(&config.data_file_dir, |wallet_data| {
 		let key_id = block_fees.key_id();
@@ -274,9 +279,10 @@ pub fn receive_coinbase(
 			n_child: derivation,
 			value: reward(block_fees.fees),
 			status: OutputStatus::Unconfirmed,
-			height: 0,
-			lock_height: 0,
+			height: height,
+			lock_height: lock_height,
 			is_coinbase: true,
+			block_hash: Hash::zero(),
 		});

 		(key_id, derivation)
@@ -284,20 +290,22 @@ pub fn receive_coinbase(

 	debug!(
 		LOGGER,
-		"Received coinbase and built candidate output - {:?}, {:?}, {}",
-		root_key_id.clone(),
+		"receive_coinbase: built candidate output - {:?}, {}",
 		key_id.clone(),
 		derivation,
 	);

-	debug!(LOGGER, "block_fees - {:?}", block_fees);
-
 	let mut block_fees = block_fees.clone();
 	block_fees.key_id = Some(key_id.clone());

-	debug!(LOGGER, "block_fees updated - {:?}", block_fees);
+	debug!(LOGGER, "receive_coinbase: {:?}", block_fees);

-	let (out, kern) = Block::reward_output(&keychain, &key_id, block_fees.fees)?;
+	let (out, kern) = Block::reward_output(
+		&keychain,
+		&key_id,
+		block_fees.fees,
+		block_fees.height,
+	)?;
 	Ok((out, kern, block_fees))
 }

@@ -313,8 +321,8 @@ fn build_final_transaction(
 	let root_key_id = keychain.root_key_id();

 	// double check the fee amount included in the partial tx
- // we don't necessarily want to just trust the sender
- // we could just overwrite the fee here (but we won't) due to the ecdsa sig
+	// we don't necessarily want to just trust the sender
+	// we could just overwrite the fee here (but we won't) due to the ecdsa sig
 	let fee = tx_fee(tx.inputs.len(), tx.outputs.len() + 1, None);
 	if fee != tx.fee {
 		return Err(Error::FeeDispute {
@@ -325,7 +333,7 @@ fn build_final_transaction(

    if fee > amount {
 		info!(
-			LOGGER, 
+			LOGGER,
 			"Rejected the transfer because transaction fee ({}) exceeds received amount ({}).",
 			amount_to_hr_string(fee),
 			amount_to_hr_string(amount)
@@ -355,6 +363,7 @@ fn build_final_transaction(
 			height: 0,
 			lock_height: 0,
 			is_coinbase: false,
+			block_hash: Hash::zero(),
 		});

 		(key_id, derivation)
@@ -16,11 +16,14 @@ use keychain::{Keychain, Identifier};
 use util::{LOGGER, from_hex};
 use util::secp::pedersen;
 use api;
+use core::global;
 use core::core::{Output, SwitchCommitHash};
-use core::core::transaction::{COINBASE_OUTPUT, DEFAULT_OUTPUT, SWITCH_COMMIT_HASH_SIZE};
+use core::core::hash::Hash;
+use core::core::transaction::{COINBASE_OUTPUT, DEFAULT_OUTPUT};
 use types::{WalletConfig, WalletData, OutputData, OutputStatus, Error};
 use byteorder::{BigEndian, ByteOrder};

+
 pub fn get_chain_height(config: &WalletConfig) -> Result<u64, Error> {
 	let url = format!("{}/v1/chain", config.check_node_api_http_addr);

@@ -39,18 +42,28 @@ pub fn get_chain_height(config: &WalletConfig) -> Result<u64, Error> {
 	}
 }

-fn output_with_range_proof(config: &WalletConfig, commit_id: &str) -> Result<api::Output, Error> {
+fn output_with_range_proof(
+	config: &WalletConfig,
+	commit_id: &str,
+	height: u64,
+) -> Result<api::OutputPrintable, Error> {
 	let url =
 		format!(
-		"{}/v1/chain/utxos/byids?id={}&include_rp&include_switch",
+		"{}/v1/chain/utxos/byheight?start_height={}&end_height={}&id={}&include_rp",
 		config.check_node_api_http_addr,
+		height,
+		height,
 		commit_id,
 	);

-	match api::client::get::<Vec<api::Output>>(url.as_str()) {
-		Ok(outputs) => {
-			if let Some(output) = outputs.first() {
-				Ok(output.clone())
+	match api::client::get::<Vec<api::BlockOutputs>>(url.as_str()) {
+		Ok(block_outputs) => {
+			if let Some(block_output) = block_outputs.first() {
+				if let Some(output) = block_output.outputs.first() {
+					Ok(output.clone())
+				} else {
+					Err(Error::Node(api::Error::NotFound))
+				}
 			} else {
 				Err(Error::Node(api::Error::NotFound))
 			}
@@ -69,17 +82,20 @@ fn retrieve_amount_and_coinbase_status(
 	keychain: &Keychain,
 	key_id: Identifier,
 	commit_id: &str,
+	height: u64,
 ) -> Result<(u64, bool), Error> {
-	let output = output_with_range_proof(config, commit_id)?;
+	let output = output_with_range_proof(config, commit_id, height)?;
+
 	let core_output = Output {
 		features: match output.output_type {
 			api::OutputType::Coinbase => COINBASE_OUTPUT,
 			api::OutputType::Transaction => DEFAULT_OUTPUT,
 		},
-		proof: output.proof.expect("output with proof"),
-		switch_commit_hash: output.switch_commit_hash.expect("output with switch_commit_hash"),
-		commit: output.commit,
+		proof: output.range_proof()?,
+		switch_commit_hash: output.switch_commit_hash()?,
+		commit: output.commit()?,
 	};
+
 	if let Some(amount) = core_output.recover_value(keychain, &key_id) {
 		let is_coinbase = match output.output_type {
 			api::OutputType::Coinbase => true,
@@ -96,8 +112,6 @@ pub fn utxos_batch_block(
 	start_height: u64,
 	end_height: u64,
 ) -> Result<Vec<api::BlockOutputs>, Error> {
-	// build the necessary query param -
-	// ?height=x
 	let query_param = format!("start_height={}&end_height={}", start_height, end_height);

 	let url =
@@ -122,15 +136,16 @@ pub fn utxos_batch_block(
 	}
 }

+// TODO - wrap the many return values in a struct
 fn find_utxos_with_key(
 	config: &WalletConfig,
 	keychain: &Keychain,
-	switch_commit_cache: &Vec<[u8; SWITCH_COMMIT_HASH_SIZE]>,
+	switch_commit_cache: &Vec<pedersen::Commitment>,
 	block_outputs: api::BlockOutputs,
 	key_iterations: &mut usize,
 	padding: &mut usize,
-) -> Vec<(pedersen::Commitment, Identifier, u32, u64, u64, bool)> {
-	let mut wallet_outputs: Vec<(pedersen::Commitment, Identifier, u32, u64, u64, bool)> =
+) -> Vec<(pedersen::Commitment, Identifier, u32, u64, u64, u64, bool)> {
+	let mut wallet_outputs: Vec<(pedersen::Commitment, Identifier, u32, u64, u64, u64, bool)> =
 		Vec::new();

 	info!(
@@ -141,55 +156,69 @@ fn find_utxos_with_key(
 		*key_iterations,
 	);

-	for output in block_outputs.outputs {
+	for output in block_outputs.outputs.iter().filter(|x| !x.spent) {
 		for i in 0..*key_iterations {
-			if switch_commit_cache[i as usize] == output.switch_commit_hash {
-				info!(
-					LOGGER,
-					"Output found: {:?}, key_index: {:?}",
-					output.switch_commit_hash,
-					i,
-				);
+			let expected_hash = SwitchCommitHash::from_switch_commit(switch_commit_cache[i as usize]);

-				// add it to result set here
-				let commit_id = from_hex(output.commit.clone()).unwrap();
-				let key_id = keychain.derive_key_id(i as u32).unwrap();
-				let res = retrieve_amount_and_coinbase_status(
-					config,
-					keychain,
-					key_id.clone(),
-					&output.commit,
-				);
-
-				if let Ok((amount, is_coinbase)) = res {
-					info!(LOGGER, "Amount: {}", amount);
-
-					let commit = keychain
-						.commit_with_key_index(BigEndian::read_u64(&commit_id), i as u32)
-						.expect("commit with key index");
-
-					wallet_outputs.push((
-						commit,
-						key_id.clone(),
-						i as u32,
-						amount,
-						output.height,
-						is_coinbase,
-					));
-
-					// probably don't have to look for indexes greater than this now
-					*key_iterations = i + *padding;
-					if *key_iterations > switch_commit_cache.len() {
-						*key_iterations = switch_commit_cache.len();
-					}
-					info!(LOGGER, "Setting max key index to: {}", *key_iterations);
-					break;
-				} else {
+			if let Ok(x) = output.switch_commit_hash() {
+				if x == expected_hash {
 					info!(
 						LOGGER,
-						"Unable to retrieve the amount (needs investigating)",
+						"Output found: {:?}, key_index: {:?}",
+						output,
+						i,
 					);

+					// add it to result set here
+					let commit_id = from_hex(output.commit.clone()).unwrap();
+					let key_id = keychain.derive_key_id(i as u32).unwrap();
+
+					let res = retrieve_amount_and_coinbase_status(
+						config,
+						keychain,
+						key_id.clone(),
+						&output.commit,
+						block_outputs.header.height,
+					);
+
+					if let Ok((amount, is_coinbase)) = res {
+						info!(LOGGER, "Amount: {}", amount);
+
+						let commit = keychain
+							.commit_with_key_index(BigEndian::read_u64(&commit_id), i as u32)
+							.expect("commit with key index");
+
+						let height = block_outputs.header.height;
+						let lock_height = if is_coinbase {
+							height + global::coinbase_maturity()
+						} else {
+							0
+						};
+
+						wallet_outputs.push((
+							commit,
+							key_id.clone(),
+							i as u32,
+							amount,
+							height,
+							lock_height,
+							is_coinbase,
+						));
+
+						// probably don't have to look for indexes greater than this now
+						*key_iterations = i + *padding;
+						if *key_iterations > switch_commit_cache.len() {
+							*key_iterations = switch_commit_cache.len();
+						}
+						info!(LOGGER, "Setting max key index to: {}", *key_iterations);
+						break;
+					} else {
+						info!(
+							LOGGER,
+							"Unable to retrieve the amount (needs investigating) {:?}",
+							res,
+						);
+					}
 				}
 			}
 		}
@@ -229,7 +258,7 @@ pub fn restore(
 		chain_height
 	);

-	let mut switch_commit_cache: Vec<[u8; SWITCH_COMMIT_HASH_SIZE]> = vec![];
+	let mut switch_commit_cache: Vec<pedersen::Commitment> = vec![];
 	info!(
 		LOGGER,
 		"Building key derivation cache ({}) ...",
@@ -237,8 +266,7 @@ pub fn restore(
 	);
 	for i in 0..key_derivations {
 		let switch_commit = keychain.switch_commit_from_index(i as u32).unwrap();
-		let switch_commit_hash = SwitchCommitHash::from_switch_commit(switch_commit);
-		switch_commit_cache.push(switch_commit_hash.hash);
+		switch_commit_cache.push(switch_commit);
 	}
 	debug!(LOGGER, "... done");

@@ -281,8 +309,9 @@ pub fn restore(
 							value: output.3,
 							status: OutputStatus::Unconfirmed,
 							height: output.4,
-							lock_height: 0,
-							is_coinbase: output.5,
+							lock_height: output.5,
+							is_coinbase: output.6,
+							block_hash: Hash::zero(),
 						});
 					};
 				}
@@ -16,6 +16,7 @@ use api;
 use client;
 use checker;
 use core::core::{build, Transaction, amount_to_hr_string};
+use core::core::hash::Hash;
 use core::ser;
 use keychain::{BlindingFactor, Identifier, Keychain};
 use receiver::TxWrapper;
@@ -27,7 +28,6 @@ use util;
 /// wallet
 /// UTXOs. The destination can be "stdout" (for command line) (currently disabled) or a URL to the
 /// recipients wallet receiver (to be implemented).
-
 pub fn issue_send_tx(
 	config: &WalletConfig,
 	keychain: &Keychain,
@@ -98,9 +98,9 @@ pub fn issue_send_tx(
 	let res = client::send_partial_tx(&url, &partial_tx);
 	if let Err(e) = res {
 		match e {
-			Error::FeeExceedsAmount {sender_amount, recipient_fee} => 
+			Error::FeeExceedsAmount {sender_amount, recipient_fee} =>
 				error!(
-					LOGGER, 
+					LOGGER,
 					"Recipient rejected the transfer because transaction fee ({}) exceeded amount ({}).",
 					amount_to_hr_string(recipient_fee),
 					amount_to_hr_string(sender_amount)
@@ -126,7 +126,7 @@ pub fn issue_send_tx(

 	let sig_part=keychain.aggsig_calculate_partial_sig(&recp_pub_nonce, tx.fee, tx.lock_height).unwrap();

-	// Build the next stage, containing sS (and our pubkeys again, for the recipient's convenience) 
+	// Build the next stage, containing sS (and our pubkeys again, for the recipient's convenience)
 	let mut partial_tx = build_partial_tx(keychain, amount, Some(sig_part), tx);
 	partial_tx.phase = PartialTxPhase::SenderConfirmation;

@@ -134,9 +134,9 @@ pub fn issue_send_tx(
 	let res = client::send_partial_tx(&url, &partial_tx);
 	if let Err(e) = res {
 		match e {
-			Error::FeeExceedsAmount {sender_amount, recipient_fee} => 
+			Error::FeeExceedsAmount {sender_amount, recipient_fee} =>
 				error!(
-					LOGGER, 
+					LOGGER,
 					"Recipient rejected the transfer because transaction fee ({}) exceeded amount ({}).",
 					amount_to_hr_string(recipient_fee),
 					amount_to_hr_string(sender_amount)
@@ -266,7 +266,11 @@ fn inputs_and_change(
 	// build inputs using the appropriate derived key_ids
 	for coin in coins {
 		let key_id = keychain.derive_key_id(coin.n_child)?;
-		parts.push(build::input(coin.value, key_id));
+		if coin.is_coinbase {
+			parts.push(build::coinbase_input(coin.value, coin.block_hash, key_id));
+		} else {
+			parts.push(build::input(coin.value, coin.block_hash, key_id));
+		}
 	}

 	// track the output representing our change
@@ -284,6 +288,7 @@ fn inputs_and_change(
 			height: 0,
 			lock_height: 0,
 			is_coinbase: false,
+			block_hash: Hash::zero(),
 		});

 		change_key
@@ -297,8 +302,10 @@ fn inputs_and_change(
 #[cfg(test)]
 mod test {
 	use core::core::build::{input, output, transaction};
+	use core::core::hash::ZERO_HASH;
 	use keychain::Keychain;

+
 	#[test]
 	// demonstrate that input.commitment == referenced output.commitment
 	// based on the public key and amount begin spent
@@ -307,8 +314,9 @@ mod test {
 		let key_id1 = keychain.derive_key_id(1).unwrap();

 		let (tx1, _) = transaction(vec![output(105, key_id1.clone())], &keychain).unwrap();
-		let (tx2, _) = transaction(vec![input(105, key_id1.clone())], &keychain).unwrap();
+		let (tx2, _) = transaction(vec![input(105, ZERO_HASH, key_id1.clone())], &keychain).unwrap();

+		assert_eq!(tx1.outputs[0].features, tx2.inputs[0].features);
 		assert_eq!(tx1.outputs[0].commitment(), tx2.inputs[0].commitment());
 	}
 }
@@ -33,6 +33,7 @@ use tokio_retry::strategy::FibonacciBackoff;
 use api;
 use core::consensus;
 use core::core::{transaction, Transaction};
+use core::core::hash::Hash;
 use core::ser;
 use keychain;
 use util;
@@ -126,12 +127,20 @@ impl From<serde_json::Error> for Error {
 	}
 }

+// TODO - rethink this, would be nice not to have to worry about
+// low level hex conversion errors like this
 impl From<num::ParseIntError> for Error {
 	fn from(_: num::ParseIntError) -> Error {
 		Error::Format("Invalid hex".to_string())
 	}
 }

+impl From<ser::Error> for Error {
+	fn from(e: ser::Error) -> Error {
+		Error::Format(e.to_string())
+	}
+}
+
 impl From<api::Error> for Error {
 	fn from(e: api::Error) -> Error {
 		Error::Node(e)
@@ -236,6 +245,8 @@ pub struct OutputData {
 	pub lock_height: u64,
 	/// Is this a coinbase output? Is it subject to coinbase locktime?
 	pub is_coinbase: bool,
+	/// Hash of the block this output originated from.
+	pub block_hash: Hash,
 }

 impl OutputData {
@@ -252,7 +263,7 @@ impl OutputData {
 	pub fn num_confirmations(&self, current_height: u64) -> u64 {
 		if self.status == OutputStatus::Unconfirmed {
 			0
-		} else if self.status == OutputStatus::Spent && self.height == 0 {
+		} else if self.height == 0 {
 			0
 		} else {
 			// if an output has height n and we are at block n