Introduce CommitOnly variant of Inputs (#3419)

* Introduce CommitOnly variant of Inputs.
Introduce CommitWrapper so we can sort commit only inputs correctly.

* rememebr to resort if converting

* write inputs based on variant and protocol version

* read and write protocol version specific inputs

* store full blocks in local db in v3
convert to v2 when relaying to v2 peers

* add debug version_str for inputs

* no assumptions about spent index sort order

* add additional version debug logs

* fix ser/deser tests for proto v3

* cleanup coinbase maturity

* rework pool to better handle v2 conversion robustly

* cleanup txpool add_to_pool

* fix nrd kernel test

* move init conversion earlier

* cleanup

* cleanup based on PR feedback

core/tests/block.rs

@@ -19,7 +19,7 @@ use crate::core::core::block::{Block, BlockHeader, Error, HeaderVersion, Untrust
 use crate::core::core::hash::Hashed;
 use crate::core::core::id::ShortIdentifiable;
 use crate::core::core::transaction::{
-	self, KernelFeatures, NRDRelativeHeight, Output, OutputFeatures, Transaction,
+	self, KernelFeatures, NRDRelativeHeight, Output, OutputFeatures, OutputIdentifier, Transaction,
 };
 use crate::core::core::verifier_cache::{LruVerifierCache, VerifierCache};
 use crate::core::core::{Committed, CompactBlock};
@@ -522,9 +522,53 @@ fn block_single_tx_serialized_size() {
 	let prev = BlockHeader::default();
 	let key_id = ExtKeychain::derive_key_id(1, 1, 0, 0, 0);
 	let b = new_block(&[tx1], &keychain, &builder, &prev, &key_id);
+
+	// Default protocol version (3)
 	let mut vec = Vec::new();
 	ser::serialize_default(&mut vec, &b).expect("serialization failed");
+	assert_eq!(vec.len(), 2_669);
+
+	// Protocol version 3
+	let mut vec = Vec::new();
+	ser::serialize(&mut vec, ser::ProtocolVersion(3), &b).expect("serialization failed");
+	assert_eq!(vec.len(), 2_669);
+
+	// Protocol version 2.
+	// Note: block must be in "v2" compatibility with "features and commit" inputs for this.
+	// Normally we would convert the block by looking inputs up in utxo but we fake it here for testing.
+	let inputs: Vec<_> = b.inputs().into();
+	let inputs: Vec<_> = inputs
+		.iter()
+		.map(|input| OutputIdentifier {
+			features: OutputFeatures::Plain,
+			commit: input.commitment(),
+		})
+		.collect();
+	let b = Block {
+		header: b.header,
+		body: b.body.replace_inputs(inputs.as_slice().into()),
+	};
+
+	// Protocol version 2
+	let mut vec = Vec::new();
+	ser::serialize(&mut vec, ser::ProtocolVersion(2), &b).expect("serialization failed");
 	assert_eq!(vec.len(), 2_670);
+
+	// Protocol version 1 (fixed size kernels)
+	let mut vec = Vec::new();
+	ser::serialize(&mut vec, ser::ProtocolVersion(1), &b).expect("serialization failed");
+	assert_eq!(vec.len(), 2_694);
+
+	// Check we can also serialize a v2 compatibility block in v3 protocol version
+	// without needing to explicitly convert the block.
+	let mut vec = Vec::new();
+	ser::serialize(&mut vec, ser::ProtocolVersion(3), &b).expect("serialization failed");
+	assert_eq!(vec.len(), 2_669);
+
+	// Default protocol version (3) for completeness
+	let mut vec = Vec::new();
+	ser::serialize_default(&mut vec, &b).expect("serialization failed");
+	assert_eq!(vec.len(), 2_669);
 }
 
 #[test]
@@ -571,25 +615,10 @@ fn block_10_tx_serialized_size() {
 	let key_id = ExtKeychain::derive_key_id(1, 1, 0, 0, 0);
 	let b = new_block(&txs, &keychain, &builder, &prev, &key_id);
 
-	// Default protocol version.
 	{
 		let mut vec = Vec::new();
 		ser::serialize_default(&mut vec, &b).expect("serialization failed");
-		assert_eq!(vec.len(), 16_836);
-	}
-
-	// Explicit protocol version 1
-	{
-		let mut vec = Vec::new();
-		ser::serialize(&mut vec, ser::ProtocolVersion(1), &b).expect("serialization failed");
-		assert_eq!(vec.len(), 16_932);
-	}
-
-	// Explicit protocol version 2
-	{
-		let mut vec = Vec::new();
-		ser::serialize(&mut vec, ser::ProtocolVersion(2), &b).expect("serialization failed");
-		assert_eq!(vec.len(), 16_836);
+		assert_eq!(vec.len(), 16_826);
 	}
 }
 
@@ -732,14 +761,13 @@ fn same_amount_outputs_copy_range_proof() {
 
 	// now we reconstruct the transaction, swapping the rangeproofs so they
 	// have the wrong privkey
-	let ins: Vec<_> = tx.inputs().into();
 	let mut outs = tx.outputs().to_vec();
 	outs[0].proof = outs[1].proof;
 
 	let key_id = keychain::ExtKeychain::derive_key_id(1, 4, 0, 0, 0);
 	let prev = BlockHeader::default();
 	let b = new_block(
-		&[Transaction::new(&ins, &outs, tx.kernels())],
+		&[Transaction::new(tx.inputs(), &outs, tx.kernels())],
 		&keychain,
 		&builder,
 		&prev,
@@ -784,7 +812,6 @@ fn wrong_amount_range_proof() {
 	.unwrap();
 
 	// we take the range proofs from tx2 into tx1 and rebuild the transaction
-	let ins: Vec<_> = tx1.inputs().into();
 	let mut outs = tx1.outputs().to_vec();
 	outs[0].proof = tx2.outputs()[0].proof;
 	outs[1].proof = tx2.outputs()[1].proof;
@@ -792,7 +819,7 @@ fn wrong_amount_range_proof() {
 	let key_id = keychain::ExtKeychain::derive_key_id(1, 4, 0, 0, 0);
 	let prev = BlockHeader::default();
 	let b = new_block(
-		&[Transaction::new(&ins, &outs, tx1.kernels())],
+		&[Transaction::new(tx1.inputs(), &outs, tx1.kernels())],
 		&keychain,
 		&builder,
 		&prev,

core/tests/common.rs

@@ -15,7 +15,9 @@
 //! Common test functions
 
 use grin_core::core::hash::DefaultHashable;
-use grin_core::core::{Block, BlockHeader, KernelFeatures, Transaction};
+use grin_core::core::{
+	Block, BlockHeader, KernelFeatures, OutputFeatures, OutputIdentifier, Transaction,
+};
 use grin_core::libtx::{
 	build::{self, input, output},
 	proof::{ProofBuild, ProofBuilder},
@@ -64,6 +66,24 @@ pub fn tx1i1o() -> Transaction {
 	tx
 }
 
+#[allow(dead_code)]
+pub fn tx1i10_v2_compatible() -> Transaction {
+	let tx = tx1i1o();
+
+	let inputs: Vec<_> = tx.inputs().into();
+	let inputs: Vec<_> = inputs
+		.iter()
+		.map(|input| OutputIdentifier {
+			features: OutputFeatures::Plain,
+			commit: input.commitment(),
+		})
+		.collect();
+	Transaction {
+		body: tx.body.replace_inputs(inputs.as_slice().into()),
+		..tx
+	}
+}
+
 // utility producing a transaction with a single input
 // and two outputs (one change output)
 // Note: this tx has an "offset" kernel

core/tests/core.rs

@@ -21,7 +21,8 @@ use self::core::core::block::Error::KernelLockHeight;
 use self::core::core::hash::{Hashed, ZERO_HASH};
 use self::core::core::verifier_cache::{LruVerifierCache, VerifierCache};
 use self::core::core::{
-	aggregate, deaggregate, KernelFeatures, Output, Transaction, TxKernel, Weighting,
+	aggregate, deaggregate, KernelFeatures, Output, OutputFeatures, OutputIdentifier, Transaction,
+	TxKernel, Weighting,
 };
 use self::core::libtx::build::{self, initial_tx, input, output, with_excess};
 use self::core::libtx::{aggsig, ProofBuilder};
@@ -42,26 +43,51 @@ fn test_setup() {
 fn simple_tx_ser() {
 	let tx = tx2i1o();
 
-	// Default protocol version.
-	{
-		let mut vec = Vec::new();
-		ser::serialize_default(&mut vec, &tx).expect("serialization failed");
-		assert_eq!(vec.len(), 947);
-	}
+	// Default protocol version (3).
+	let mut vec = Vec::new();
+	ser::serialize_default(&mut vec, &tx).expect("serialization failed");
+	assert_eq!(vec.len(), 945);
+
+	// Explicit protocol version 3.
+	let mut vec = Vec::new();
+	ser::serialize(&mut vec, ser::ProtocolVersion(3), &tx).expect("serialization failed");
+	assert_eq!(vec.len(), 945);
+
+	// We need to convert the tx to v2 compatibility with "features and commitment" inputs
+	// to serialize to any previous protocol version.
+	// Normally we would do this conversion against the utxo and txpool but we fake it here for testing.
+	let inputs: Vec<_> = tx.inputs().into();
+	let inputs: Vec<_> = inputs
+		.iter()
+		.map(|input| OutputIdentifier {
+			features: OutputFeatures::Plain,
+			commit: input.commitment(),
+		})
+		.collect();
+	let tx = Transaction {
+		body: tx.body.replace_inputs(inputs.as_slice().into()),
+		..tx
+	};
 
 	// Explicit protocol version 1.
-	{
-		let mut vec = Vec::new();
-		ser::serialize(&mut vec, ser::ProtocolVersion(1), &tx).expect("serialization failed");
-		assert_eq!(vec.len(), 955);
-	}
+	let mut vec = Vec::new();
+	ser::serialize(&mut vec, ser::ProtocolVersion(1), &tx).expect("serialization failed");
+	assert_eq!(vec.len(), 955);
 
 	// Explicit protocol version 2.
-	{
-		let mut vec = Vec::new();
-		ser::serialize(&mut vec, ser::ProtocolVersion(2), &tx).expect("serialization failed");
-		assert_eq!(vec.len(), 947);
-	}
+	let mut vec = Vec::new();
+	ser::serialize(&mut vec, ser::ProtocolVersion(2), &tx).expect("serialization failed");
+	assert_eq!(vec.len(), 947);
+
+	// Check we can still serialize to protocol version 3 without explicitly converting the tx.
+	let mut vec = Vec::new();
+	ser::serialize(&mut vec, ser::ProtocolVersion(3), &tx).expect("serialization failed");
+	assert_eq!(vec.len(), 945);
+
+	// And default protocol version for completeness.
+	let mut vec = Vec::new();
+	ser::serialize_default(&mut vec, &tx).expect("serialization failed");
+	assert_eq!(vec.len(), 945);
 }
 
 #[test]

core/tests/transaction.rs

@@ -15,7 +15,7 @@
 //! Transaction integration tests
 
 pub mod common;
-use crate::common::tx1i1o;
+use crate::common::tx1i10_v2_compatible;
 use crate::core::core::transaction::{self, Error};
 use crate::core::core::verifier_cache::LruVerifierCache;
 use crate::core::core::{KernelFeatures, Output, OutputFeatures, Transaction, Weighting};
@@ -32,8 +32,10 @@ use util::RwLock;
 // This test ensures we exercise this serialization/deserialization code.
 #[test]
 fn test_transaction_json_ser_deser() {
-	let tx1 = tx1i1o();
+	let tx1 = tx1i10_v2_compatible();
+
 	let value = serde_json::to_value(&tx1).unwrap();
+	println!("{:?}", value);
 
 	assert!(value["offset"].is_string());
 	assert_eq!(value["body"]["inputs"][0]["features"], "Plain");
@@ -50,7 +52,6 @@ fn test_transaction_json_ser_deser() {
 	let tx2: Transaction = serde_json::from_value(value).unwrap();
 	assert_eq!(tx1, tx2);
 
-	let tx1 = tx1i1o();
 	let str = serde_json::to_string(&tx1).unwrap();
 	println!("{}", str);
 	let tx2: Transaction = serde_json::from_str(&str).unwrap();