hash (features|commitment) in output mmr (#615)

* experiment with lock_heights on outputs * playing around with lock_height as part of the switch commitment hash * cleanup * include features in the switch commit hash key * commit * rebase off master * commit * cleanup * missing docs * rework coinbase maturity test to build valid tx * pool and chain tests passing (inputs have switch commitments) * commit * cleanup * check inputs spending coinbase outputs have valid lock_heights * wip - got it building (tests still failing) * use zero key for non coinbase switch commit hash * fees and height wrong order... * send output lock_height over to wallet via api * no more header by height index workaround this for wallet refresh and wallet restore * refresh heights for unspent wallet outputs where missing * TODO - might be slow? * simplify - do not pass around lock_height for non coinbase outputs * commit * fix tests after merge * build input vs coinbase_input switch commit hash key encodes lock_height cleanup output by commit index (currently broken...) * is_unspent and get_unspent cleanup - we have no outputs, only switch_commit_hashes * separate concept of utxo vs output in the api utxos come from the sumtrees (and only the sumtrees, limited info) outputs come from blocks (and we need to look them up via block height) * cleanup * better api support for block outputs with range proofs * basic wallet operations appear to work restore is not working fully refresh refreshes heights correctly (at least appears to) * wallet refresh and wallet restore appear to be working now * fix core tests * fix some mine_simple_chain tests * fixup chain tests * rework so pool tests pass * wallet restore now safely habndles duplicate commitments (reused wallet keys) for coinbase outputs where lock_height is _very_ important * wip * validate_coinbase_maturity got things building tests are failing * lite vs full versions of is_unspent * builds and working locally zero-conf - what to do here? * handle zero-conf edge case (use latest block) * introduce OutputIdentifier, avoid leaking SumCommit everywhere * fix the bad merge * pool verifies coinbase maturity via is_matured this uses sumtree in a consistent way * cleanup * add docs, cleanup build warnings * fix core tests * fix chain tests * fix pool tests * cleanup debug logging that we no longer need * make out_block optional on an input (only care about it for spending coinbase outputs) * cleanup * bump the build
2018-01-16 22:03:40 -05:00
parent 7e7c8e157e
commit cbd3b2ff87
31 changed files with 1345 additions and 901 deletions
@@ -24,9 +24,8 @@ use serde::Serialize;
 use serde_json;

 use chain;
-use core::core::Transaction;
-use core::core::hash::Hash;
-use core::core::hash::Hashed;
+use core::core::{OutputIdentifier, Transaction, DEFAULT_OUTPUT, COINBASE_OUTPUT};
+use core::core::hash::{Hash, Hashed};
 use core::ser;
 use pool;
 use p2p;
@@ -54,38 +53,35 @@ impl Handler for IndexHandler {
 // Supports retrieval of multiple outputs in a single request -
 // GET /v1/chain/utxos/byids?id=xxx,yyy,zzz
 // GET /v1/chain/utxos/byids?id=xxx&id=yyy&id=zzz
-// GET /v1/chain/utxos/byheight?height=n
+// GET /v1/chain/utxos/byheight?start_height=101&end_height=200
 struct UtxoHandler {
 	chain: Arc<chain::Chain>,
 }

 impl UtxoHandler {
-	fn get_utxo(&self, id: &str, include_rp: bool, include_switch: bool) -> Result<Output, Error> {
-		debug!(LOGGER, "getting utxo: {}", id);
+	fn get_utxo(&self, id: &str) -> Result<Utxo, Error> {
 		let c = util::from_hex(String::from(id))
 			.map_err(|_| Error::Argument(format!("Not a valid commitment: {}", id)))?;
 		let commit = Commitment::from_vec(c);

-		let out = self.chain
-			.get_unspent(&commit)
-			.map_err(|_| Error::NotFound)?;
+		// We need the features here to be able to generate the necessary hash
+		// to compare against the hash in the output MMR.
+		// For now we can just try both (but this probably needs to be part of the api params)
+		let outputs = [
+			OutputIdentifier::new(DEFAULT_OUTPUT, &commit),
+			OutputIdentifier::new(COINBASE_OUTPUT, &commit)
+		];

-		let header = self.chain
-			.get_block_header_by_output_commit(&commit)
-			.map_err(|_| Error::NotFound)?;
-
-		Ok(Output::from_output(
-			&out,
-			&header,
-			include_rp,
-			include_switch,
-		))
+		for x in outputs.iter() {
+			if let Ok(_) = self.chain.is_unspent(&x) {
+				return Ok(Utxo::new(&commit))
+			}
+		}
+		Err(Error::NotFound)
 	}

-	fn utxos_by_ids(&self, req: &mut Request) -> Vec<Output> {
+	fn utxos_by_ids(&self, req: &mut Request) -> Vec<Utxo> {
 		let mut commitments: Vec<&str> = vec![];
-		let mut rp = false;
-		let mut switch = false;
 		if let Ok(params) = req.get_ref::<UrlEncodedQuery>() {
 			if let Some(ids) = params.get("id") {
 				for id in ids {
@@ -94,23 +90,25 @@ impl UtxoHandler {
 					}
 				}
 			}
-			if let Some(_) = params.get("include_rp") {
-				rp = true;
-			}
-			if let Some(_) = params.get("include_switch") {
-				switch = true;
-			}
 		}
-		let mut utxos: Vec<Output> = vec![];
-		for commit in commitments {
-			if let Ok(out) = self.get_utxo(commit, rp, switch) {
-				utxos.push(out);
+
+		debug!(LOGGER, "utxos_by_ids: {:?}", commitments);
+
+		let mut utxos: Vec<Utxo> = vec![];
+		for x in commitments {
+			if let Ok(utxo) = self.get_utxo(x) {
+				utxos.push(utxo);
 			}
 		}
 		utxos
 	}

-	fn utxos_at_height(&self, block_height: u64) -> BlockOutputs {
+	fn outputs_at_height(
+		&self,
+		block_height: u64,
+		commitments: Vec<Commitment>,
+		include_proof: bool,
+	) -> BlockOutputs {
 		let header = self.chain
 			.clone()
 			.get_header_by_height(block_height)
@@ -119,8 +117,12 @@ impl UtxoHandler {
 		let outputs = block
 			.outputs
 			.iter()
-			.filter(|c| self.chain.is_unspent(&c.commit).unwrap())
-			.map(|k| OutputSwitch::from_output(k, &header))
+			.filter(|output| {
+				commitments.is_empty() || commitments.contains(&output.commit)
+			})
+			.map(|output| {
+				OutputPrintable::from_output(output, self.chain.clone(), include_proof)
+			})
 			.collect();
 		BlockOutputs {
 			header: BlockHeaderInfo::from_header(&header),
@@ -128,11 +130,23 @@ impl UtxoHandler {
 		}
 	}

-	// returns utxos for a specified range of blocks
-	fn utxo_block_batch(&self, req: &mut Request) -> Vec<BlockOutputs> {
+	// returns outputs for a specified range of blocks
+	fn outputs_block_batch(&self, req: &mut Request) -> Vec<BlockOutputs> {
+		let mut commitments: Vec<Commitment> = vec![];
 		let mut start_height = 1;
 		let mut end_height = 1;
+		let mut include_rp = false;
+
 		if let Ok(params) = req.get_ref::<UrlEncodedQuery>() {
+			if let Some(ids) = params.get("id") {
+				for id in ids {
+					for id in id.split(",") {
+						if let Ok(x) = util::from_hex(String::from(id)) {
+							commitments.push(Commitment::from_vec(x));
+						}
+					}
+				}
+			}
 			if let Some(heights) = params.get("start_height") {
 				for height in heights {
 					start_height = height.parse().unwrap();
@@ -143,11 +157,28 @@ impl UtxoHandler {
 					end_height = height.parse().unwrap();
 				}
 			}
+			if let Some(_) = params.get("include_rp") {
+				include_rp = true;
+			}
 		}
+
+		debug!(
+			LOGGER,
+			"outputs_block_batch: {}-{}, {:?}, {:?}",
+			start_height,
+			end_height,
+			commitments,
+			include_rp,
+		);
+
 		let mut return_vec = vec![];
 		for i in start_height..end_height + 1 {
-			return_vec.push(self.utxos_at_height(i));
+			let res = self.outputs_at_height(i, commitments.clone(), include_rp);
+			if res.outputs.len() > 0 {
+				return_vec.push(res);
+			}
 		}
+
 		return_vec
 	}
 }
@@ -161,7 +192,7 @@ impl Handler for UtxoHandler {
 		}
 		match *path_elems.last().unwrap() {
 			"byids" => json_response(&self.utxos_by_ids(req)),
-			"byheight" => json_response(&self.utxo_block_batch(req)),
+			"byheight" => json_response(&self.outputs_block_batch(req)),
 			_ => Ok(Response::with((status::BadRequest, ""))),
 		}
 	}
@@ -363,11 +394,8 @@ pub struct BlockHandler {

 impl BlockHandler {
 	fn get_block(&self, h: &Hash) -> Result<BlockPrintable, Error> {
-		let block = self.chain
-			.clone()
-			.get_block(h)
-			.map_err(|_| Error::NotFound)?;
-		Ok(BlockPrintable::from_block(&block))
+		let block = self.chain.clone().get_block(h).map_err(|_| Error::NotFound)?;
+		Ok(BlockPrintable::from_block(&block, self.chain.clone(), false))
 	}

 	// Try to decode the string as a height or a hash.
@@ -460,11 +488,17 @@ where
 			tx.outputs.len()
 		);

-		let res = self.tx_pool.write().unwrap().add_to_memory_pool(source, tx);
+		let res = self.tx_pool
+			.write()
+			.unwrap()
+			.add_to_memory_pool(source, tx);

 		match res {
 			Ok(()) => Ok(Response::with(status::Ok)),
-			Err(e) => Err(IronError::from(Error::Argument(format!("{:?}", e)))),
+			Err(e) => {
+				debug!(LOGGER, "error - {:?}", e);
+				Err(IronError::from(Error::Argument(format!("{:?}", e))))
+			}
 		}
 	}
 }
@@ -13,13 +13,15 @@
 // limitations under the License.

 use std::sync::Arc;
-use core::{core, global};
+
+use core::{core, ser};
 use core::core::hash::Hashed;
+use core::core::SumCommit;
 use chain;
 use p2p;
-use util::secp::pedersen;
-use rest::*;
 use util;
+use util::secp::pedersen;
+use util::secp::constants::MAX_PROOF_SIZE;

 /// The state of the current fork tip
 #[derive(Serialize, Deserialize, Debug, Clone)]
@@ -86,10 +88,10 @@ impl SumTrees {
 	pub fn from_head(head: Arc<chain::Chain>) -> SumTrees {
 		let roots = head.get_sumtree_roots();
 		SumTrees {
-			utxo_root_hash: util::to_hex(roots.0.hash.to_vec()),
-			utxo_root_sum: util::to_hex(roots.0.sum.commit.0.to_vec()),
-			range_proof_root_hash: util::to_hex(roots.1.hash.to_vec()),
-			kernel_root_hash: util::to_hex(roots.2.hash.to_vec()),
+			utxo_root_hash: roots.0.hash.to_hex(),
+			utxo_root_sum: roots.0.sum.to_hex(),
+			range_proof_root_hash: roots.1.hash.to_hex(),
+			kernel_root_hash: roots.2.hash.to_hex(),
 		}
 	}
 }
@@ -100,26 +102,18 @@ impl SumTrees {
 pub struct SumTreeNode {
 	// The hash
 	pub hash: String,
-	// Output (if included)
-	pub output: Option<OutputPrintable>,
+	// SumCommit (features|commitment), optional (only for utxos)
+	pub sum: Option<SumCommit>,
 }

 impl SumTreeNode {
 	pub fn get_last_n_utxo(chain: Arc<chain::Chain>, distance: u64) -> Vec<SumTreeNode> {
 		let mut return_vec = Vec::new();
 		let last_n = chain.get_last_n_utxo(distance);
-		for elem_output in last_n {
-			let header = chain
-				.get_block_header_by_output_commit(&elem_output.1.commit)
-				.map_err(|_| Error::NotFound);
-			// Need to call further method to check if output is spent
-			let mut output = OutputPrintable::from_output(&elem_output.1, &header.unwrap(), true);
-			if let Ok(_) = chain.get_unspent(&elem_output.1.commit) {
-				output.spent = false;
-			}
+		for x in last_n {
 			return_vec.push(SumTreeNode {
-				hash: util::to_hex(elem_output.0.to_vec()),
-				output: Some(output),
+				hash: util::to_hex(x.hash.to_vec()),
+				sum: Some(x.sum),
 			});
 		}
 		return_vec
@@ -131,7 +125,7 @@ impl SumTreeNode {
 		for elem in last_n {
 			return_vec.push(SumTreeNode {
 				hash: util::to_hex(elem.hash.to_vec()),
-				output: None,
+				sum: None,
 			});
 		}
 		return_vec
@@ -143,7 +137,7 @@ impl SumTreeNode {
 		for elem in last_n {
 			return_vec.push(SumTreeNode {
 				hash: util::to_hex(elem.hash.to_vec()),
-				output: None,
+				sum: None,
 			});
 		}
 		return_vec
@@ -157,50 +151,14 @@ pub enum OutputType {
 }

 #[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct Output {
-	/// The type of output Coinbase|Transaction
-	pub output_type: OutputType,
-	/// The homomorphic commitment representing the output's amount
+pub struct Utxo {
+	/// The output commitment representing the amount
 	pub commit: pedersen::Commitment,
-	/// switch commit hash
-	pub switch_commit_hash: Option<core::SwitchCommitHash>,
-	/// A proof that the commitment is in the right range
-	pub proof: Option<pedersen::RangeProof>,
-	/// The height of the block creating this output
-	pub height: u64,
-	/// The lock height (earliest block this output can be spent)
-	pub lock_height: u64,
 }

-impl Output {
-	pub fn from_output(
-		output: &core::Output,
-		block_header: &core::BlockHeader,
-		include_proof: bool,
-		include_switch: bool,
-	) -> Output {
-		let (output_type, lock_height) = match output.features {
-			x if x.contains(core::transaction::COINBASE_OUTPUT) => (
-				OutputType::Coinbase,
-				block_header.height + global::coinbase_maturity(),
-			),
-			_ => (OutputType::Transaction, 0),
-		};
-
-		Output {
-			output_type: output_type,
-			commit: output.commit,
-			switch_commit_hash: match include_switch {
-				true => Some(output.switch_commit_hash),
-				false => None,
-			},
-			proof: match include_proof {
-				true => Some(output.proof),
-				false => None,
-			},
-			height: block_header.height,
-			lock_height: lock_height,
-		}
+impl Utxo {
+	pub fn new(commit: &pedersen::Commitment) -> Utxo {
+		Utxo { commit: commit.clone() }
 	}
 }

@@ -209,66 +167,73 @@ impl Output {
 pub struct OutputPrintable {
 	/// The type of output Coinbase|Transaction
 	pub output_type: OutputType,
-	/// The homomorphic commitment representing the output's amount (as hex
-	/// string)
+	/// The homomorphic commitment representing the output's amount
+	/// (as hex string)
 	pub commit: String,
 	/// switch commit hash
 	pub switch_commit_hash: String,
-	/// The height of the block creating this output
-	pub height: u64,
-	/// The lock height (earliest block this output can be spent)
-	pub lock_height: u64,
 	/// Whether the output has been spent
 	pub spent: bool,
-	/// Rangeproof hash  (as hex string)
-	pub proof_hash: Option<String>,
+	/// Rangeproof (as hex string)
+	pub proof: Option<String>,
+	/// Rangeproof hash (as hex string)
+	pub proof_hash: String,
 }

 impl OutputPrintable {
 	pub fn from_output(
 		output: &core::Output,
-		block_header: &core::BlockHeader,
-		include_proof_hash: bool,
+		chain: Arc<chain::Chain>,
+		include_proof: bool,
 	) -> OutputPrintable {
-		let (output_type, lock_height) = match output.features {
-			x if x.contains(core::transaction::COINBASE_OUTPUT) => (
-				OutputType::Coinbase,
-				block_header.height + global::coinbase_maturity(),
-			),
-			_ => (OutputType::Transaction, 0),
+		let output_type =
+			if output.features.contains(core::transaction::COINBASE_OUTPUT) {
+				OutputType::Coinbase
+			} else {
+				OutputType::Transaction
+			};
+
+		let out_id = core::OutputIdentifier::from_output(&output);
+		let spent = chain.is_unspent(&out_id).is_err();
+
+		let proof = if include_proof {
+			Some(util::to_hex(output.proof.bytes().to_vec()))
+		} else {
+			None
 		};
+
 		OutputPrintable {
 			output_type: output_type,
 			commit: util::to_hex(output.commit.0.to_vec()),
-			switch_commit_hash: util::to_hex(output.switch_commit_hash.hash.to_vec()),
-			height: block_header.height,
-			lock_height: lock_height,
-			spent: true,
-			proof_hash: match include_proof_hash {
-				true => Some(util::to_hex(output.proof.hash().to_vec())),
-				false => None,
-			},
+			switch_commit_hash: output.switch_commit_hash.to_hex(),
+			spent: spent,
+			proof: proof,
+			proof_hash: util::to_hex(output.proof.hash().to_vec()),
 		}
 	}
-}

-// As above, except just the info needed for wallet reconstruction
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct OutputSwitch {
-	/// the commit
-	pub commit: String,
-	/// switch commit hash
-	pub switch_commit_hash: [u8; core::SWITCH_COMMIT_HASH_SIZE],
-	/// The height of the block creating this output
-	pub height: u64,
-}
+	// Convert the hex string back into a switch_commit_hash instance
+	pub fn switch_commit_hash(&self) -> Result<core::SwitchCommitHash, ser::Error> {
+		core::SwitchCommitHash::from_hex(&self.switch_commit_hash)
+	}

-impl OutputSwitch {
-	pub fn from_output(output: &core::Output, block_header: &core::BlockHeader) -> OutputSwitch {
-		OutputSwitch {
-			commit: util::to_hex(output.commit.0.to_vec()),
-			switch_commit_hash: output.switch_commit_hash.hash,
-			height: block_header.height,
+	pub fn commit(&self) -> Result<pedersen::Commitment, ser::Error> {
+		let vec = util::from_hex(self.commit.clone())
+			.map_err(|_| ser::Error::HexError(format!("output commit hex_error")))?;
+		Ok(pedersen::Commitment::from_vec(vec))
+	}
+
+	pub fn range_proof(&self) -> Result<pedersen::RangeProof, ser::Error> {
+		if let Some(ref proof) = self.proof {
+			let vec = util::from_hex(proof.clone())
+				.map_err(|_| ser::Error::HexError(format!("output range_proof hex_error")))?;
+			let mut bytes = [0; MAX_PROOF_SIZE];
+			for i in 0..vec.len() {
+				bytes[i] = vec[i];
+			}
+			Ok(pedersen::RangeProof { proof: bytes, plen: vec.len() })
+		} else {
+			Err(ser::Error::HexError(format!("output range_proof missing")))
 		}
 	}
 }
@@ -374,16 +339,19 @@ pub struct BlockPrintable {
 }

 impl BlockPrintable {
-	pub fn from_block(block: &core::Block) -> BlockPrintable {
-		let inputs = block
-			.inputs
+	pub fn from_block(
+		block: &core::Block,
+		chain: Arc<chain::Chain>,
+		include_proof: bool,
+	) -> BlockPrintable {
+		let inputs = block.inputs
 			.iter()
-			.map(|input| util::to_hex((input.0).0.to_vec()))
+			.map(|x| util::to_hex(x.commitment().0.to_vec()))
 			.collect();
 		let outputs = block
 			.outputs
 			.iter()
-			.map(|output| OutputPrintable::from_output(output, &block.header, true))
+			.map(|output| OutputPrintable::from_output(output, chain.clone(), include_proof))
 			.collect();
 		let kernels = block
 			.kernels
@@ -406,7 +374,7 @@ pub struct BlockOutputs {
 	/// The block header
 	pub header: BlockHeaderInfo,
 	/// A printable version of the outputs
-	pub outputs: Vec<OutputSwitch>,
+	pub outputs: Vec<OutputPrintable>,
 }

 #[derive(Serialize, Deserialize)]