From ebaf99fadb07d863a5d4cdf2e26dcee07d715a87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bogdan-=C8=98tefan=20Neac=C5=9Fu?= Date: Fri, 9 Dec 2022 13:00:27 +0200 Subject: [PATCH] Feature/dkg state to disk (#1872) * Add PersistentState * Save and load state to/from disk * If in progress, don't continually write the same state * Fix tests and add serde one * Update changelog * Fix clippy --- CHANGELOG.md | 8 ++ common/crypto/dkg/src/bte/keys.rs | 2 +- .../crypto/dkg/src/bte/proof_discrete_log.rs | 2 +- common/crypto/dkg/src/dealing.rs | 73 +++++++++- validator-api/src/coconut/dkg/complaints.rs | 4 +- validator-api/src/coconut/dkg/controller.rs | 19 ++- validator-api/src/coconut/dkg/dealing.rs | 6 + validator-api/src/coconut/dkg/public_key.rs | 4 + validator-api/src/coconut/dkg/state.rs | 135 +++++++++++++++--- .../src/coconut/dkg/verification_key.rs | 4 + validator-api/src/coconut/error.rs | 3 + validator-api/src/config/mod.rs | 16 +++ 12 files changed, 247 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a2888c43c..c2926410d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ Post 1.0.0 release, the changelog format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Changed + +- validator-api: can recover from shutdown during DKG process ([#1872]) + +[#1872]: https://github.com/nymtech/nym/pull/1872 + ## [v1.1.2] ### Changed diff --git a/common/crypto/dkg/src/bte/keys.rs b/common/crypto/dkg/src/bte/keys.rs index 9412790db8..098e9638c7 100644 --- a/common/crypto/dkg/src/bte/keys.rs +++ b/common/crypto/dkg/src/bte/keys.rs @@ -53,7 +53,7 @@ impl PublicKey { } } -#[derive(Debug)] +#[derive(Clone, Debug)] #[cfg_attr(test, derive(PartialEq, Eq))] pub struct PublicKeyWithProof { pub(crate) key: PublicKey, diff --git a/common/crypto/dkg/src/bte/proof_discrete_log.rs b/common/crypto/dkg/src/bte/proof_discrete_log.rs index 5dbdbba702..489eb13244 100644 --- a/common/crypto/dkg/src/bte/proof_discrete_log.rs +++ b/common/crypto/dkg/src/bte/proof_discrete_log.rs @@ -13,7 +13,7 @@ use zeroize::Zeroize; const DISCRETE_LOG_DOMAIN: &[u8] = b"NYM_COCONUT_NIDKG_V01_CS01_WITH_BLS12381_XMD:SHA-256_SSWU_RO_PROOF_DISCRETE_LOG"; -#[derive(Debug)] +#[derive(Clone, Debug)] #[cfg_attr(test, derive(PartialEq, Eq))] pub struct ProofOfDiscreteLog { pub(crate) rand_commitment: G1Projective, diff --git a/common/crypto/dkg/src/dealing.rs b/common/crypto/dkg/src/dealing.rs index 817bffdf12..b3633ec82d 100644 --- a/common/crypto/dkg/src/dealing.rs +++ b/common/crypto/dkg/src/dealing.rs @@ -9,19 +9,79 @@ use crate::interpolation::polynomial::{Polynomial, PublicCoefficients}; use crate::interpolation::{ perform_lagrangian_interpolation_at_origin, perform_lagrangian_interpolation_at_x, }; +use crate::utils::deserialize_g2; use crate::{NodeIndex, Share, Threshold}; use bls12_381::{G2Projective, Scalar}; +use group::GroupEncoding; use rand_core::RngCore; use std::collections::BTreeMap; use zeroize::Zeroize; -#[derive(Debug)] +#[derive(Clone, Debug)] #[cfg_attr(test, derive(PartialEq, Eq))] pub struct RecoveredVerificationKeys { pub recovered_master: G2Projective, pub recovered_partials: Vec, } +impl RecoveredVerificationKeys { + pub fn to_bytes(&self) -> Vec { + let partials = self.recovered_partials.len(); + let mut bytes = Vec::with_capacity(96 + 4 + 96 * partials); + bytes.extend_from_slice(self.recovered_master.to_bytes().as_ref()); + bytes.extend_from_slice(&((partials as u32).to_be_bytes())); + for partial in &self.recovered_partials { + bytes.extend_from_slice(partial.to_bytes().as_ref()); + } + + bytes + } + + pub fn try_from_bytes(b: &[u8]) -> Result { + if b.len() < 96 + 4 { + return Err(DkgError::new_deserialization_failure( + "RecoveredVerificationKeys", + "insufficient number of bytes provided", + )); + } + + let recovered_master = deserialize_g2(&b[..96]).ok_or_else(|| { + DkgError::new_deserialization_failure( + "RecoveredVerificationKeys.recovered_master", + "invalid curve point", + ) + })?; + + let partials = u32::from_be_bytes([b[96], b[97], b[98], b[99]]) as usize; + let mut recovered_partials = Vec::with_capacity(partials); + + if b.len() != 96 + 4 + 96 * partials { + return Err(DkgError::new_deserialization_failure( + "RecoveredVerificationKeys", + "insufficient number of bytes provided", + )); + } + + let mut i = 96 + 4; + for _ in 0..partials { + let partial = deserialize_g2(&b[i..i + 96]).ok_or_else(|| { + DkgError::new_deserialization_failure( + "RecoveredVerificationKeys.recovered_partials", + "invalid curve point", + ) + })?; + + recovered_partials.push(partial); + i += 96; + } + + Ok(RecoveredVerificationKeys { + recovered_master, + recovered_partials, + }) + } +} + #[derive(Debug)] #[cfg_attr(test, derive(PartialEq, Eq))] pub struct Dealing { @@ -355,6 +415,17 @@ mod tests { use crate::combine_shares; use rand_core::SeedableRng; + #[test] + fn recovered_verification_keys_serde() { + let keys = RecoveredVerificationKeys { + recovered_master: Default::default(), + recovered_partials: vec![Default::default(), Default::default()], + }; + let bytes = keys.to_bytes(); + let recovered_keys = RecoveredVerificationKeys::try_from_bytes(&bytes).unwrap(); + assert_eq!(keys, recovered_keys); + } + #[test] #[ignore] // expensive test fn recovering_partial_verification_keys() { diff --git a/validator-api/src/coconut/dkg/complaints.rs b/validator-api/src/coconut/dkg/complaints.rs index 0001e4f76d..4ef0d32ce0 100644 --- a/validator-api/src/coconut/dkg/complaints.rs +++ b/validator-api/src/coconut/dkg/complaints.rs @@ -1,7 +1,9 @@ // Copyright 2022 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -#[derive(Debug, Eq, PartialEq)] +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub(crate) enum ComplaintReason { MalformedBTEPublicKey, InvalidBTEPublicKey, diff --git a/validator-api/src/coconut/dkg/controller.rs b/validator-api/src/coconut/dkg/controller.rs index 0cd538793b..6a7268af47 100644 --- a/validator-api/src/coconut/dkg/controller.rs +++ b/validator-api/src/coconut/dkg/controller.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use crate::coconut::dkg::client::DkgClient; -use crate::coconut::dkg::state::{ConsistentState, State}; +use crate::coconut::dkg::state::{ConsistentState, PersistentState, State}; use crate::coconut::dkg::verification_key::{ verification_key_finalization, verification_key_validation, }; @@ -63,12 +63,20 @@ impl DkgController { )) { coconut_keypair.set(coconut_keypair_value).await; } + let persistent_state = + PersistentState::load_from_file(config.persistent_state_path()).unwrap_or_default(); Ok(DkgController { dkg_client: DkgClient::new(nymd_client), secret_key_path: config.secret_key_path(), verification_key_path: config.verification_key_path(), - state: State::new(config.get_announce_address(), dkg_keypair, coconut_keypair), + state: State::new( + config.persistent_state_path(), + persistent_state, + config.get_announce_address(), + dkg_keypair, + coconut_keypair, + ), rng, polling_rate: config.get_dkg_contract_polling_rate(), }) @@ -114,6 +122,13 @@ impl DkgController { }; if let Err(e) = ret { warn!("Could not handle this iteration for the epoch state: {}", e); + } else if epoch_state != EpochState::InProgress { + let persistent_state = PersistentState::from(&self.state); + if let Err(e) = + persistent_state.save_to_file(self.state.persistent_state_path()) + { + warn!("Could not backup the state for this iteration: {}", e); + } } } } diff --git a/validator-api/src/coconut/dkg/dealing.rs b/validator-api/src/coconut/dkg/dealing.rs index b6ebed9853..7e53202dac 100644 --- a/validator-api/src/coconut/dkg/dealing.rs +++ b/validator-api/src/coconut/dkg/dealing.rs @@ -55,6 +55,7 @@ pub(crate) async fn dealing_exchange( pub(crate) mod tests { use super::*; use crate::coconut::dkg::complaints::ComplaintReason; + use crate::coconut::dkg::state::PersistentState; use crate::coconut::tests::DummyClient; use crate::coconut::KeyPair; use coconut_dkg_common::dealer::DealerDetails; @@ -63,6 +64,7 @@ pub(crate) mod tests { use dkg::bte::Params; use rand::rngs::OsRng; use std::collections::HashMap; + use std::path::PathBuf; use std::str::FromStr; use std::sync::{Arc, RwLock}; use url::Url; @@ -110,6 +112,8 @@ pub(crate) mod tests { ); let params = setup(); let mut state = State::new( + PathBuf::default(), + PersistentState::default(), Url::parse("localhost:8000").unwrap(), DkgKeyPair::new(¶ms, OsRng), KeyPair::new(), @@ -163,6 +167,8 @@ pub(crate) mod tests { ); let params = setup(); let mut state = State::new( + PathBuf::default(), + PersistentState::default(), Url::parse("localhost:8000").unwrap(), DkgKeyPair::new(¶ms, OsRng), KeyPair::new(), diff --git a/validator-api/src/coconut/dkg/public_key.rs b/validator-api/src/coconut/dkg/public_key.rs index 29a3e732f1..0ce3bd45b3 100644 --- a/validator-api/src/coconut/dkg/public_key.rs +++ b/validator-api/src/coconut/dkg/public_key.rs @@ -34,10 +34,12 @@ pub(crate) async fn public_key_submission( #[cfg(test)] pub(crate) mod tests { use super::*; + use crate::coconut::dkg::state::PersistentState; use crate::coconut::tests::DummyClient; use crate::coconut::KeyPair; use dkg::bte::keys::KeyPair as DkgKeyPair; use rand::rngs::OsRng; + use std::path::PathBuf; use std::str::FromStr; use url::Url; use validator_client::nymd::AccountId; @@ -51,6 +53,8 @@ pub(crate) mod tests { AccountId::from_str(TEST_VALIDATOR_ADDRESS).unwrap(), )); let mut state = State::new( + PathBuf::default(), + PersistentState::default(), Url::parse("localhost:8000").unwrap(), DkgKeyPair::new(&dkg::bte::setup(), OsRng), KeyPair::new(), diff --git a/validator-api/src/coconut/dkg/state.rs b/validator-api/src/coconut/dkg/state.rs index aebcdea495..b7d7796ca0 100644 --- a/validator-api/src/coconut/dkg/state.rs +++ b/validator-api/src/coconut/dkg/state.rs @@ -9,13 +9,33 @@ use coconut_dkg_common::types::EpochState; use cosmwasm_std::Addr; use dkg::bte::{keys::KeyPair as DkgKeyPair, PublicKey, PublicKeyWithProof}; use dkg::{NodeIndex, RecoveredVerificationKeys, Threshold}; +use serde::de::Error; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use std::collections::BTreeMap; +use std::path::PathBuf; use url::Url; +fn bte_pk_serialize( + val: &PublicKeyWithProof, + serializer: S, +) -> Result { + val.to_bytes().serialize(serializer) +} + +fn bte_pk_deserialize<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let vec: Vec = Deserialize::deserialize(deserializer)?; + PublicKeyWithProof::try_from_bytes(&vec).map_err(|err| Error::custom(format_args!("{:?}", err))) +} + // note: each dealer is also a receiver which simplifies some logic significantly -#[derive(Debug)] +#[derive(Clone, Deserialize, Debug, Serialize)] pub(crate) struct DkgParticipant { pub(crate) _address: Addr, + #[serde(serialize_with = "bte_pk_serialize")] + #[serde(deserialize_with = "bte_pk_deserialize")] pub(crate) bte_public_key_with_proof: PublicKeyWithProof, pub(crate) assigned_index: NodeIndex, } @@ -71,20 +91,6 @@ pub(crate) trait ConsistentState { } } -pub(crate) struct State { - announce_address: Url, - dkg_keypair: DkgKeyPair, - coconut_keypair: CoconutKeyPair, - node_index: Option, - dealers: BTreeMap>, - receiver_index: Option, - threshold: Option, - recovered_vks: Vec, - proposal_id: Option, - voted_vks: bool, - executed_proposal: bool, -} - #[async_trait] impl ConsistentState for State { fn node_index_value(&self) -> Result { @@ -131,27 +137,110 @@ impl ConsistentState for State { } } +fn vks_serialize( + val: &[RecoveredVerificationKeys], + serializer: S, +) -> Result { + let vec: Vec> = val.iter().map(|vk| vk.to_bytes()).collect(); + vec.serialize(serializer) +} + +fn vks_deserialize<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let vec: Vec> = Deserialize::deserialize(deserializer)?; + vec.into_iter() + .map(|b| { + RecoveredVerificationKeys::try_from_bytes(&b) + .map_err(|err| D::Error::custom(format_args!("{:?}", err))) + }) + .collect() +} + +#[derive(Default, Deserialize, Serialize)] +pub(crate) struct PersistentState { + node_index: Option, + dealers: BTreeMap>, + receiver_index: Option, + threshold: Option, + #[serde(serialize_with = "vks_serialize")] + #[serde(deserialize_with = "vks_deserialize")] + recovered_vks: Vec, + proposal_id: Option, + voted_vks: bool, + executed_proposal: bool, +} + +impl From<&State> for PersistentState { + fn from(s: &State) -> Self { + PersistentState { + node_index: s.node_index, + dealers: s.dealers.clone(), + receiver_index: s.receiver_index, + threshold: s.threshold, + recovered_vks: s.recovered_vks.clone(), + proposal_id: s.proposal_id, + voted_vks: s.voted_vks, + executed_proposal: s.executed_proposal, + } + } +} + +impl PersistentState { + pub fn save_to_file(&self, path: PathBuf) -> Result<(), CoconutError> { + std::fs::write(path, serde_json::to_string(self)?)?; + Ok(()) + } + + pub fn load_from_file(path: PathBuf) -> Result { + Ok(serde_json::from_str(&std::fs::read_to_string(path)?)?) + } +} + +pub(crate) struct State { + persistent_state_path: PathBuf, + announce_address: Url, + dkg_keypair: DkgKeyPair, + coconut_keypair: CoconutKeyPair, + node_index: Option, + dealers: BTreeMap>, + receiver_index: Option, + threshold: Option, + recovered_vks: Vec, + proposal_id: Option, + voted_vks: bool, + executed_proposal: bool, +} + impl State { pub fn new( + persistent_state_path: PathBuf, + persistent_state: PersistentState, announce_address: Url, dkg_keypair: DkgKeyPair, coconut_keypair: CoconutKeyPair, ) -> Self { State { + persistent_state_path, announce_address, dkg_keypair, coconut_keypair, - node_index: None, - dealers: BTreeMap::new(), - receiver_index: None, - threshold: None, - recovered_vks: vec![], - proposal_id: None, - voted_vks: false, - executed_proposal: false, + node_index: persistent_state.node_index, + dealers: persistent_state.dealers, + receiver_index: persistent_state.receiver_index, + threshold: persistent_state.threshold, + recovered_vks: persistent_state.recovered_vks, + proposal_id: persistent_state.proposal_id, + voted_vks: persistent_state.voted_vks, + executed_proposal: persistent_state.executed_proposal, } } + pub fn persistent_state_path(&self) -> PathBuf { + self.persistent_state_path.clone() + } + pub fn announce_address(&self) -> &Url { &self.announce_address } diff --git a/validator-api/src/coconut/dkg/verification_key.rs b/validator-api/src/coconut/dkg/verification_key.rs index 38ce4dd13c..cecd825154 100644 --- a/validator-api/src/coconut/dkg/verification_key.rs +++ b/validator-api/src/coconut/dkg/verification_key.rs @@ -249,6 +249,7 @@ pub(crate) mod tests { use super::*; use crate::coconut::dkg::dealing::dealing_exchange; use crate::coconut::dkg::public_key::public_key_submission; + use crate::coconut::dkg::state::PersistentState; use crate::coconut::tests::DummyClient; use crate::coconut::KeyPair; use coconut_dkg_common::dealer::DealerDetails; @@ -259,6 +260,7 @@ pub(crate) mod tests { use rand::Rng; use std::collections::HashMap; use std::env::temp_dir; + use std::path::PathBuf; use std::str::FromStr; use std::sync::{Arc, RwLock}; use url::Url; @@ -289,6 +291,8 @@ pub(crate) mod tests { ); let keypair = DkgKeyPair::new(¶ms, OsRng); let state = State::new( + PathBuf::default(), + PersistentState::default(), Url::parse("localhost:8000").unwrap(), keypair, KeyPair::new(), diff --git a/validator-api/src/coconut/error.rs b/validator-api/src/coconut/error.rs index 9b6fadf957..234d1cb96c 100644 --- a/validator-api/src/coconut/error.rs +++ b/validator-api/src/coconut/error.rs @@ -23,6 +23,9 @@ pub enum CoconutError { #[error("{0}")] IOError(#[from] std::io::Error), + #[error("{0}")] + SerdeJsonError(#[from] serde_json::Error), + #[error("Could not parse Ed25519 data")] Ed25519ParseError(#[from] Ed25519RecoveryError), diff --git a/validator-api/src/config/mod.rs b/validator-api/src/config/mod.rs index e08b3bb20a..715ed19311 100644 --- a/validator-api/src/config/mod.rs +++ b/validator-api/src/config/mod.rs @@ -282,6 +282,9 @@ pub struct CoconutSigner { /// Specifies whether rewarding service is enabled in this process. enabled: bool, + /// Path to a JSON file where state is persisted between different stages of DKG. + dkg_persistent_state_path: PathBuf, + /// Path to the coconut verification key. verification_key_path: PathBuf, @@ -299,6 +302,7 @@ pub struct CoconutSigner { } impl CoconutSigner { + pub const DKG_PERSISTENT_STATE_FILE: &'static str = "dkg_persistent_state.json"; pub const DKG_DECRYPTION_KEY_FILE: &'static str = "dkg_decryption_key.pem"; pub const DKG_PUBLIC_KEY_WITH_PROOF_FILE: &'static str = "dkg_public_key_with_proof.pem"; pub const COCONUT_VERIFICATION_KEY_FILE: &'static str = "coconut_verification_key.pem"; @@ -312,6 +316,10 @@ impl CoconutSigner { Config::default_data_directory(None).join(Self::COCONUT_SECRET_KEY_FILE) } + fn default_dkg_persistent_state_path() -> PathBuf { + Config::default_data_directory(None).join(Self::DKG_PERSISTENT_STATE_FILE) + } + fn default_dkg_decryption_key_path() -> PathBuf { Config::default_data_directory(None).join(Self::DKG_DECRYPTION_KEY_FILE) } @@ -325,6 +333,7 @@ impl Default for CoconutSigner { fn default() -> Self { Self { enabled: Default::default(), + dkg_persistent_state_path: CoconutSigner::default_dkg_persistent_state_path(), verification_key_path: CoconutSigner::default_coconut_verification_key_path(), secret_key_path: CoconutSigner::default_coconut_secret_key_path(), decryption_key_path: CoconutSigner::default_dkg_decryption_key_path(), @@ -345,6 +354,8 @@ impl Config { Config::default_data_directory(Some(id)).join(NodeStatusAPI::DB_FILE); self.network_monitor.credentials_database_path = Config::default_data_directory(Some(id)).join(NetworkMonitor::DB_FILE); + self.coconut_signer.dkg_persistent_state_path = + Config::default_data_directory(Some(id)).join(CoconutSigner::DKG_PERSISTENT_STATE_FILE); self.coconut_signer.verification_key_path = Config::default_data_directory(Some(id)) .join(CoconutSigner::COCONUT_VERIFICATION_KEY_FILE); self.coconut_signer.secret_key_path = @@ -509,6 +520,11 @@ impl Config { self.node_status_api.database_path.clone() } + #[cfg(feature = "coconut")] + pub fn persistent_state_path(&self) -> PathBuf { + self.coconut_signer.dkg_persistent_state_path.clone() + } + #[cfg(feature = "coconut")] pub fn verification_key_path(&self) -> PathBuf { self.coconut_signer.verification_key_path.clone()