Feature/dkg state to disk (#1872)

* Add PersistentState

* Save and load state to/from disk

* If in progress, don't continually write the same state

* Fix tests and add serde one

* Update changelog

* Fix clippy

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 Post 1.0.0 release, the changelog format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Changed
+
+- validator-api: can recover from shutdown during DKG process ([#1872])
+
+[#1872]: https://github.com/nymtech/nym/pull/1872
+
 ## [v1.1.2]
 
 ### Changed

common/crypto/dkg/src/bte/keys.rs

@@ -53,7 +53,7 @@ impl PublicKey {
     }
 }
 
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 #[cfg_attr(test, derive(PartialEq, Eq))]
 pub struct PublicKeyWithProof {
     pub(crate) key: PublicKey,

common/crypto/dkg/src/bte/proof_discrete_log.rs

@@ -13,7 +13,7 @@ use zeroize::Zeroize;
 const DISCRETE_LOG_DOMAIN: &[u8] =
     b"NYM_COCONUT_NIDKG_V01_CS01_WITH_BLS12381_XMD:SHA-256_SSWU_RO_PROOF_DISCRETE_LOG";
 
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 #[cfg_attr(test, derive(PartialEq, Eq))]
 pub struct ProofOfDiscreteLog {
     pub(crate) rand_commitment: G1Projective,

common/crypto/dkg/src/dealing.rs

@@ -9,19 +9,79 @@ use crate::interpolation::polynomial::{Polynomial, PublicCoefficients};
 use crate::interpolation::{
     perform_lagrangian_interpolation_at_origin, perform_lagrangian_interpolation_at_x,
 };
+use crate::utils::deserialize_g2;
 use crate::{NodeIndex, Share, Threshold};
 use bls12_381::{G2Projective, Scalar};
+use group::GroupEncoding;
 use rand_core::RngCore;
 use std::collections::BTreeMap;
 use zeroize::Zeroize;
 
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 #[cfg_attr(test, derive(PartialEq, Eq))]
 pub struct RecoveredVerificationKeys {
     pub recovered_master: G2Projective,
     pub recovered_partials: Vec<G2Projective>,
 }
 
+impl RecoveredVerificationKeys {
+    pub fn to_bytes(&self) -> Vec<u8> {
+        let partials = self.recovered_partials.len();
+        let mut bytes = Vec::with_capacity(96 + 4 + 96 * partials);
+        bytes.extend_from_slice(self.recovered_master.to_bytes().as_ref());
+        bytes.extend_from_slice(&((partials as u32).to_be_bytes()));
+        for partial in &self.recovered_partials {
+            bytes.extend_from_slice(partial.to_bytes().as_ref());
+        }
+
+        bytes
+    }
+
+    pub fn try_from_bytes(b: &[u8]) -> Result<Self, DkgError> {
+        if b.len() < 96 + 4 {
+            return Err(DkgError::new_deserialization_failure(
+                "RecoveredVerificationKeys",
+                "insufficient number of bytes provided",
+            ));
+        }
+
+        let recovered_master = deserialize_g2(&b[..96]).ok_or_else(|| {
+            DkgError::new_deserialization_failure(
+                "RecoveredVerificationKeys.recovered_master",
+                "invalid curve point",
+            )
+        })?;
+
+        let partials = u32::from_be_bytes([b[96], b[97], b[98], b[99]]) as usize;
+        let mut recovered_partials = Vec::with_capacity(partials);
+
+        if b.len() != 96 + 4 + 96 * partials {
+            return Err(DkgError::new_deserialization_failure(
+                "RecoveredVerificationKeys",
+                "insufficient number of bytes provided",
+            ));
+        }
+
+        let mut i = 96 + 4;
+        for _ in 0..partials {
+            let partial = deserialize_g2(&b[i..i + 96]).ok_or_else(|| {
+                DkgError::new_deserialization_failure(
+                    "RecoveredVerificationKeys.recovered_partials",
+                    "invalid curve point",
+                )
+            })?;
+
+            recovered_partials.push(partial);
+            i += 96;
+        }
+
+        Ok(RecoveredVerificationKeys {
+            recovered_master,
+            recovered_partials,
+        })
+    }
+}
+
 #[derive(Debug)]
 #[cfg_attr(test, derive(PartialEq, Eq))]
 pub struct Dealing {
@@ -355,6 +415,17 @@ mod tests {
     use crate::combine_shares;
     use rand_core::SeedableRng;
 
+    #[test]
+    fn recovered_verification_keys_serde() {
+        let keys = RecoveredVerificationKeys {
+            recovered_master: Default::default(),
+            recovered_partials: vec![Default::default(), Default::default()],
+        };
+        let bytes = keys.to_bytes();
+        let recovered_keys = RecoveredVerificationKeys::try_from_bytes(&bytes).unwrap();
+        assert_eq!(keys, recovered_keys);
+    }
+
     #[test]
     #[ignore] // expensive test
     fn recovering_partial_verification_keys() {

validator-api/src/coconut/dkg/complaints.rs

@@ -1,7 +1,9 @@
 // Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
-#[derive(Debug, Eq, PartialEq)]
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
 pub(crate) enum ComplaintReason {
     MalformedBTEPublicKey,
     InvalidBTEPublicKey,

validator-api/src/coconut/dkg/controller.rs

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use crate::coconut::dkg::client::DkgClient;
-use crate::coconut::dkg::state::{ConsistentState, State};
+use crate::coconut::dkg::state::{ConsistentState, PersistentState, State};
 use crate::coconut::dkg::verification_key::{
     verification_key_finalization, verification_key_validation,
 };
@@ -63,12 +63,20 @@ impl<R: RngCore + Clone> DkgController<R> {
         )) {
             coconut_keypair.set(coconut_keypair_value).await;
         }
+        let persistent_state =
+            PersistentState::load_from_file(config.persistent_state_path()).unwrap_or_default();
 
         Ok(DkgController {
             dkg_client: DkgClient::new(nymd_client),
             secret_key_path: config.secret_key_path(),
             verification_key_path: config.verification_key_path(),
-            state: State::new(config.get_announce_address(), dkg_keypair, coconut_keypair),
+            state: State::new(
+                config.persistent_state_path(),
+                persistent_state,
+                config.get_announce_address(),
+                dkg_keypair,
+                coconut_keypair,
+            ),
             rng,
             polling_rate: config.get_dkg_contract_polling_rate(),
         })
@@ -114,6 +122,13 @@ impl<R: RngCore + Clone> DkgController<R> {
                 };
                 if let Err(e) = ret {
                     warn!("Could not handle this iteration for the epoch state: {}", e);
+                } else if epoch_state != EpochState::InProgress {
+                    let persistent_state = PersistentState::from(&self.state);
+                    if let Err(e) =
+                        persistent_state.save_to_file(self.state.persistent_state_path())
+                    {
+                        warn!("Could not backup the state for this iteration: {}", e);
+                    }
                 }
             }
         }

validator-api/src/coconut/dkg/dealing.rs

@@ -55,6 +55,7 @@ pub(crate) async fn dealing_exchange(
 pub(crate) mod tests {
     use super::*;
     use crate::coconut::dkg::complaints::ComplaintReason;
+    use crate::coconut::dkg::state::PersistentState;
     use crate::coconut::tests::DummyClient;
     use crate::coconut::KeyPair;
     use coconut_dkg_common::dealer::DealerDetails;
@@ -63,6 +64,7 @@ pub(crate) mod tests {
     use dkg::bte::Params;
     use rand::rngs::OsRng;
     use std::collections::HashMap;
+    use std::path::PathBuf;
     use std::str::FromStr;
     use std::sync::{Arc, RwLock};
     use url::Url;
@@ -110,6 +112,8 @@ pub(crate) mod tests {
         );
         let params = setup();
         let mut state = State::new(
+            PathBuf::default(),
+            PersistentState::default(),
             Url::parse("localhost:8000").unwrap(),
             DkgKeyPair::new(&params, OsRng),
             KeyPair::new(),
@@ -163,6 +167,8 @@ pub(crate) mod tests {
         );
         let params = setup();
         let mut state = State::new(
+            PathBuf::default(),
+            PersistentState::default(),
             Url::parse("localhost:8000").unwrap(),
             DkgKeyPair::new(&params, OsRng),
             KeyPair::new(),

validator-api/src/coconut/dkg/public_key.rs

@@ -34,10 +34,12 @@ pub(crate) async fn public_key_submission(
 #[cfg(test)]
 pub(crate) mod tests {
     use super::*;
+    use crate::coconut::dkg::state::PersistentState;
     use crate::coconut::tests::DummyClient;
     use crate::coconut::KeyPair;
     use dkg::bte::keys::KeyPair as DkgKeyPair;
     use rand::rngs::OsRng;
+    use std::path::PathBuf;
     use std::str::FromStr;
     use url::Url;
     use validator_client::nymd::AccountId;
@@ -51,6 +53,8 @@ pub(crate) mod tests {
             AccountId::from_str(TEST_VALIDATOR_ADDRESS).unwrap(),
         ));
         let mut state = State::new(
+            PathBuf::default(),
+            PersistentState::default(),
             Url::parse("localhost:8000").unwrap(),
             DkgKeyPair::new(&dkg::bte::setup(), OsRng),
             KeyPair::new(),

validator-api/src/coconut/dkg/state.rs

@@ -9,13 +9,33 @@ use coconut_dkg_common::types::EpochState;
 use cosmwasm_std::Addr;
 use dkg::bte::{keys::KeyPair as DkgKeyPair, PublicKey, PublicKeyWithProof};
 use dkg::{NodeIndex, RecoveredVerificationKeys, Threshold};
+use serde::de::Error;
+use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use std::collections::BTreeMap;
+use std::path::PathBuf;
 use url::Url;
 
+fn bte_pk_serialize<S: Serializer>(
+    val: &PublicKeyWithProof,
+    serializer: S,
+) -> Result<S::Ok, S::Error> {
+    val.to_bytes().serialize(serializer)
+}
+
+fn bte_pk_deserialize<'de, D>(deserializer: D) -> Result<PublicKeyWithProof, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let vec: Vec<u8> = Deserialize::deserialize(deserializer)?;
+    PublicKeyWithProof::try_from_bytes(&vec).map_err(|err| Error::custom(format_args!("{:?}", err)))
+}
+
 // note: each dealer is also a receiver which simplifies some logic significantly
-#[derive(Debug)]
+#[derive(Clone, Deserialize, Debug, Serialize)]
 pub(crate) struct DkgParticipant {
     pub(crate) _address: Addr,
+    #[serde(serialize_with = "bte_pk_serialize")]
+    #[serde(deserialize_with = "bte_pk_deserialize")]
     pub(crate) bte_public_key_with_proof: PublicKeyWithProof,
     pub(crate) assigned_index: NodeIndex,
 }
@@ -71,20 +91,6 @@ pub(crate) trait ConsistentState {
     }
 }
 
-pub(crate) struct State {
-    announce_address: Url,
-    dkg_keypair: DkgKeyPair,
-    coconut_keypair: CoconutKeyPair,
-    node_index: Option<NodeIndex>,
-    dealers: BTreeMap<Addr, Result<DkgParticipant, ComplaintReason>>,
-    receiver_index: Option<usize>,
-    threshold: Option<Threshold>,
-    recovered_vks: Vec<RecoveredVerificationKeys>,
-    proposal_id: Option<u64>,
-    voted_vks: bool,
-    executed_proposal: bool,
-}
-
 #[async_trait]
 impl ConsistentState for State {
     fn node_index_value(&self) -> Result<NodeIndex, CoconutError> {
@@ -131,27 +137,110 @@ impl ConsistentState for State {
     }
 }
 
+fn vks_serialize<S: Serializer>(
+    val: &[RecoveredVerificationKeys],
+    serializer: S,
+) -> Result<S::Ok, S::Error> {
+    let vec: Vec<Vec<u8>> = val.iter().map(|vk| vk.to_bytes()).collect();
+    vec.serialize(serializer)
+}
+
+fn vks_deserialize<'de, D>(deserializer: D) -> Result<Vec<RecoveredVerificationKeys>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let vec: Vec<Vec<u8>> = Deserialize::deserialize(deserializer)?;
+    vec.into_iter()
+        .map(|b| {
+            RecoveredVerificationKeys::try_from_bytes(&b)
+                .map_err(|err| D::Error::custom(format_args!("{:?}", err)))
+        })
+        .collect()
+}
+
+#[derive(Default, Deserialize, Serialize)]
+pub(crate) struct PersistentState {
+    node_index: Option<NodeIndex>,
+    dealers: BTreeMap<Addr, Result<DkgParticipant, ComplaintReason>>,
+    receiver_index: Option<usize>,
+    threshold: Option<Threshold>,
+    #[serde(serialize_with = "vks_serialize")]
+    #[serde(deserialize_with = "vks_deserialize")]
+    recovered_vks: Vec<RecoveredVerificationKeys>,
+    proposal_id: Option<u64>,
+    voted_vks: bool,
+    executed_proposal: bool,
+}
+
+impl From<&State> for PersistentState {
+    fn from(s: &State) -> Self {
+        PersistentState {
+            node_index: s.node_index,
+            dealers: s.dealers.clone(),
+            receiver_index: s.receiver_index,
+            threshold: s.threshold,
+            recovered_vks: s.recovered_vks.clone(),
+            proposal_id: s.proposal_id,
+            voted_vks: s.voted_vks,
+            executed_proposal: s.executed_proposal,
+        }
+    }
+}
+
+impl PersistentState {
+    pub fn save_to_file(&self, path: PathBuf) -> Result<(), CoconutError> {
+        std::fs::write(path, serde_json::to_string(self)?)?;
+        Ok(())
+    }
+
+    pub fn load_from_file(path: PathBuf) -> Result<Self, CoconutError> {
+        Ok(serde_json::from_str(&std::fs::read_to_string(path)?)?)
+    }
+}
+
+pub(crate) struct State {
+    persistent_state_path: PathBuf,
+    announce_address: Url,
+    dkg_keypair: DkgKeyPair,
+    coconut_keypair: CoconutKeyPair,
+    node_index: Option<NodeIndex>,
+    dealers: BTreeMap<Addr, Result<DkgParticipant, ComplaintReason>>,
+    receiver_index: Option<usize>,
+    threshold: Option<Threshold>,
+    recovered_vks: Vec<RecoveredVerificationKeys>,
+    proposal_id: Option<u64>,
+    voted_vks: bool,
+    executed_proposal: bool,
+}
+
 impl State {
     pub fn new(
+        persistent_state_path: PathBuf,
+        persistent_state: PersistentState,
         announce_address: Url,
         dkg_keypair: DkgKeyPair,
         coconut_keypair: CoconutKeyPair,
     ) -> Self {
         State {
+            persistent_state_path,
             announce_address,
             dkg_keypair,
             coconut_keypair,
-            node_index: None,
-            dealers: BTreeMap::new(),
-            receiver_index: None,
-            threshold: None,
-            recovered_vks: vec![],
-            proposal_id: None,
-            voted_vks: false,
-            executed_proposal: false,
+            node_index: persistent_state.node_index,
+            dealers: persistent_state.dealers,
+            receiver_index: persistent_state.receiver_index,
+            threshold: persistent_state.threshold,
+            recovered_vks: persistent_state.recovered_vks,
+            proposal_id: persistent_state.proposal_id,
+            voted_vks: persistent_state.voted_vks,
+            executed_proposal: persistent_state.executed_proposal,
         }
     }
 
+    pub fn persistent_state_path(&self) -> PathBuf {
+        self.persistent_state_path.clone()
+    }
+
     pub fn announce_address(&self) -> &Url {
         &self.announce_address
     }

validator-api/src/coconut/dkg/verification_key.rs

@@ -249,6 +249,7 @@ pub(crate) mod tests {
     use super::*;
     use crate::coconut::dkg::dealing::dealing_exchange;
     use crate::coconut::dkg::public_key::public_key_submission;
+    use crate::coconut::dkg::state::PersistentState;
     use crate::coconut::tests::DummyClient;
     use crate::coconut::KeyPair;
     use coconut_dkg_common::dealer::DealerDetails;
@@ -259,6 +260,7 @@ pub(crate) mod tests {
     use rand::Rng;
     use std::collections::HashMap;
     use std::env::temp_dir;
+    use std::path::PathBuf;
     use std::str::FromStr;
     use std::sync::{Arc, RwLock};
     use url::Url;
@@ -289,6 +291,8 @@ pub(crate) mod tests {
             );
             let keypair = DkgKeyPair::new(&params, OsRng);
             let state = State::new(
+                PathBuf::default(),
+                PersistentState::default(),
                 Url::parse("localhost:8000").unwrap(),
                 keypair,
                 KeyPair::new(),

validator-api/src/coconut/error.rs

@@ -23,6 +23,9 @@ pub enum CoconutError {
     #[error("{0}")]
     IOError(#[from] std::io::Error),
 
+    #[error("{0}")]
+    SerdeJsonError(#[from] serde_json::Error),
+
     #[error("Could not parse Ed25519 data")]
     Ed25519ParseError(#[from] Ed25519RecoveryError),
 

validator-api/src/config/mod.rs

@@ -282,6 +282,9 @@ pub struct CoconutSigner {
     /// Specifies whether rewarding service is enabled in this process.
     enabled: bool,
 
+    /// Path to a JSON file where state is persisted between different stages of DKG.
+    dkg_persistent_state_path: PathBuf,
+
     /// Path to the coconut verification key.
     verification_key_path: PathBuf,
 
@@ -299,6 +302,7 @@ pub struct CoconutSigner {
 }
 
 impl CoconutSigner {
+    pub const DKG_PERSISTENT_STATE_FILE: &'static str = "dkg_persistent_state.json";
     pub const DKG_DECRYPTION_KEY_FILE: &'static str = "dkg_decryption_key.pem";
     pub const DKG_PUBLIC_KEY_WITH_PROOF_FILE: &'static str = "dkg_public_key_with_proof.pem";
     pub const COCONUT_VERIFICATION_KEY_FILE: &'static str = "coconut_verification_key.pem";
@@ -312,6 +316,10 @@ impl CoconutSigner {
         Config::default_data_directory(None).join(Self::COCONUT_SECRET_KEY_FILE)
     }
 
+    fn default_dkg_persistent_state_path() -> PathBuf {
+        Config::default_data_directory(None).join(Self::DKG_PERSISTENT_STATE_FILE)
+    }
+
     fn default_dkg_decryption_key_path() -> PathBuf {
         Config::default_data_directory(None).join(Self::DKG_DECRYPTION_KEY_FILE)
     }
@@ -325,6 +333,7 @@ impl Default for CoconutSigner {
     fn default() -> Self {
         Self {
             enabled: Default::default(),
+            dkg_persistent_state_path: CoconutSigner::default_dkg_persistent_state_path(),
             verification_key_path: CoconutSigner::default_coconut_verification_key_path(),
             secret_key_path: CoconutSigner::default_coconut_secret_key_path(),
             decryption_key_path: CoconutSigner::default_dkg_decryption_key_path(),
@@ -345,6 +354,8 @@ impl Config {
             Config::default_data_directory(Some(id)).join(NodeStatusAPI::DB_FILE);
         self.network_monitor.credentials_database_path =
             Config::default_data_directory(Some(id)).join(NetworkMonitor::DB_FILE);
+        self.coconut_signer.dkg_persistent_state_path =
+            Config::default_data_directory(Some(id)).join(CoconutSigner::DKG_PERSISTENT_STATE_FILE);
         self.coconut_signer.verification_key_path = Config::default_data_directory(Some(id))
             .join(CoconutSigner::COCONUT_VERIFICATION_KEY_FILE);
         self.coconut_signer.secret_key_path =
@@ -509,6 +520,11 @@ impl Config {
         self.node_status_api.database_path.clone()
     }
 
+    #[cfg(feature = "coconut")]
+    pub fn persistent_state_path(&self) -> PathBuf {
+        self.coconut_signer.dkg_persistent_state_path.clone()
+    }
+
     #[cfg(feature = "coconut")]
     pub fn verification_key_path(&self) -> PathBuf {
         self.coconut_signer.verification_key_path.clone()