Feature/spend coconut (#1210)

* Import cw3-flex-multisig and cw4-group contracts * Add release_funds to coconut-bandwidth-contract * Create contract.rs file * Add cw multi test and a test that uses it * Use mnemonic for coconut mode too * Stricter access to config file, which contains mnemonic * Update tests * Remove signed deposits dir after merging that into sql db * Clippy nits * More clippy * Remove backtraces features to pass clippy tests * Merge the same mnemonic for rewarding and coconut * Simplify things, letting network monitor use testnet-mode with gateways * Unify the nymd clients * Sqlx common storage for buying/consuming credentials * Link credential storage to credential client * Trigger rewarded_set update on bootstrap error * Fix bug on message signing * Simplify coconut feature in code and set it in validator-api * Update some local consts * Link clients to credential storage * Simplify sql query and change socks5 too * Update attr handling such that public ones are usable * Normalize test addresses * Fix clippy * Merge storages for (non)coconut creds * Fmt miss * Disable wasm client support for now Co-authored-by: durch <durch@users.noreply.github.com>
2022-04-14 14:25:21 +02:00
parent 94be4c71a4
commit 76a61cb3ae
83 changed files with 4293 additions and 553 deletions
@@ -1,35 +1,10 @@
 // Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0

-use crate::coconut::error::{CoconutError, Result};
-use crate::config::DEFAULT_LOCAL_VALIDATOR;
-
-use validator_client::nymd::{tx::Hash, NymdClient, QueryNymdClient, TxResponse};
-
-use async_trait::async_trait;
+use crate::coconut::error::Result;
+use validator_client::nymd::TxResponse;

 #[async_trait]
 pub trait Client {
    async fn get_tx(&self, tx_hash: &str) -> Result<TxResponse>;
 }
-
-pub struct QueryClient {
-    inner: NymdClient<QueryNymdClient>,
-}
-
-impl QueryClient {
-    pub fn new() -> Result<Self> {
-        let inner = NymdClient::connect(DEFAULT_LOCAL_VALIDATOR, None, None, None)?;
-        Ok(Self { inner })
-    }
-}
-
-#[async_trait]
-impl Client for QueryClient {
-    async fn get_tx(&self, tx_hash: &str) -> Result<TxResponse> {
-        let tx_hash = tx_hash
-            .parse::<Hash>()
-            .map_err(|_| CoconutError::TxHashParseError)?;
-        Ok(self.inner.get_tx(tx_hash).await?)
-    }
-}
@@ -3,7 +3,7 @@

 pub(crate) mod client;
 mod deposit;
-mod error;
+pub(crate) mod error;
 #[cfg(test)]
 mod tests;

@@ -27,6 +27,12 @@ const DEFAULT_GATEWAY_PING_INTERVAL: Duration = Duration::from_secs(60);
 const DEFAULT_GATEWAY_RESPONSE_TIMEOUT: Duration = Duration::from_secs(5 * 60);
 const DEFAULT_GATEWAY_CONNECTION_TIMEOUT: Duration = Duration::from_millis(2_500);

+#[cfg(not(feature = "coconut"))]
+const DEFAULT_ETH_ENDPOINT: &str = "https://rinkeby.infura.io/v3/00000000000000000000000000000000";
+#[cfg(not(feature = "coconut"))]
+const DEFAULT_ETH_PRIVATE_KEY: &str =
+    "0000000000000000000000000000000000000000000000000000000000000001";
+
 const DEFAULT_TEST_ROUTES: usize = 3;
 const DEFAULT_MINIMUM_TEST_ROUTES: usize = 1;
 const DEFAULT_ROUTE_TEST_PACKETS: usize = 1000;
@@ -53,6 +59,10 @@ pub struct Config {

    #[serde(default)]
    rewarding: Rewarding,
+
+    #[serde(default)]
+    #[cfg(feature = "coconut")]
+    coconut_signer: CoconutSigner,
 }

 impl NymConfig for Config {
@@ -88,9 +98,8 @@ pub struct Base {
    /// Address of the validator contract managing the network
    mixnet_contract_address: String,

-    // Avoid breaking derives for now
-    #[cfg(feature = "coconut")]
-    keypair_bs58: String,
+    /// Mnemonic used for rewarding and/or multisig operations
+    mnemonic: String,
 }

 impl Default for Base {
@@ -100,8 +109,7 @@ impl Default for Base {
                .parse()
                .expect("default local validator is malformed!"),
            mixnet_contract_address: DEFAULT_NETWORK.mixnet_contract_address().to_string(),
-            #[cfg(feature = "coconut")]
-            keypair_bs58: String::default(),
+            mnemonic: String::default(),
        }
    }
 }
@@ -154,11 +162,8 @@ pub struct NetworkMonitor {
    #[serde(with = "humantime_serde")]
    packet_delivery_timeout: Duration,

-    /// Path to directory containing public/private keys used for bandwidth token purchase.
-    /// Those are saved in case of emergency, to be able to reclaim bandwidth tokens.
-    /// The public key is the name of the file, while the private key is the content.
-    #[cfg(not(feature = "coconut"))]
-    backup_bandwidth_token_keys_dir: PathBuf,
+    /// Path to the database containing bandwidth credentials of this client.
+    credentials_database_path: PathBuf,

    /// Ethereum private key.
    #[cfg(not(feature = "coconut"))]
@@ -184,9 +189,8 @@ pub struct NetworkMonitor {
 }

 impl NetworkMonitor {
-    #[cfg(not(feature = "coconut"))]
-    fn default_backup_bandwidth_token_keys_dir() -> PathBuf {
-        Config::default_data_directory(None).join("backup_bandwidth_token_keys_dir")
+    fn default_credentials_database_path() -> PathBuf {
+        Config::default_data_directory(None).join("credentials_database.db")
    }
 }

@@ -205,12 +209,11 @@ impl Default for NetworkMonitor {
            gateway_response_timeout: DEFAULT_GATEWAY_RESPONSE_TIMEOUT,
            gateway_connection_timeout: DEFAULT_GATEWAY_CONNECTION_TIMEOUT,
            packet_delivery_timeout: DEFAULT_PACKET_DELIVERY_TIMEOUT,
+            credentials_database_path: Self::default_credentials_database_path(),
            #[cfg(not(feature = "coconut"))]
-            backup_bandwidth_token_keys_dir: Self::default_backup_bandwidth_token_keys_dir(),
+            eth_private_key: DEFAULT_ETH_PRIVATE_KEY.to_string(),
            #[cfg(not(feature = "coconut"))]
-            eth_private_key: "".to_string(),
-            #[cfg(not(feature = "coconut"))]
-            eth_endpoint: "".to_string(),
+            eth_endpoint: DEFAULT_ETH_ENDPOINT.to_string(),
            test_routes: DEFAULT_TEST_ROUTES,
            minimum_test_routes: DEFAULT_MINIMUM_TEST_ROUTES,
            route_test_packets: DEFAULT_ROUTE_TEST_PACKETS,
@@ -261,9 +264,6 @@ pub struct Rewarding {
    /// Specifies whether rewarding service is enabled in this process.
    enabled: bool,

-    /// Mnemonic (currently of the network monitor) used for rewarding
-    mnemonic: String,
-
    /// Specifies the minimum percentage of monitor test run data present in order to
    /// distribute rewards for given interval.
    /// Note, only values in range 0-100 are valid
@@ -274,12 +274,22 @@ impl Default for Rewarding {
    fn default() -> Self {
        Rewarding {
            enabled: false,
-            mnemonic: String::default(),
            minimum_interval_monitor_threshold: DEFAULT_MONITOR_THRESHOLD,
        }
    }
 }

+#[derive(Debug, Default, Deserialize, PartialEq, Serialize)]
+#[serde(default)]
+#[cfg(feature = "coconut")]
+pub struct CoconutSigner {
+    /// Specifies whether rewarding service is enabled in this process.
+    enabled: bool,
+
+    /// Base58 encoded signing keypair
+    keypair_bs58: String,
+}
+
 impl Config {
    pub fn new() -> Self {
        Config::default()
@@ -287,7 +297,7 @@ impl Config {

    #[cfg(feature = "coconut")]
    pub fn keypair(&self) -> KeyPair {
-        KeyPair::try_from_bs58(self.base.keypair_bs58.clone()).unwrap()
+        KeyPair::try_from_bs58(self.coconut_signer.keypair_bs58.clone()).unwrap()
    }

    pub fn with_network_monitor_enabled(mut self, enabled: bool) -> Self {
@@ -305,6 +315,12 @@ impl Config {
        self
    }

+    #[cfg(feature = "coconut")]
+    pub fn with_coconut_signer_enabled(mut self, enabled: bool) -> Self {
+        self.coconut_signer.enabled = enabled;
+        self
+    }
+
    pub fn with_custom_nymd_validator(mut self, validator: Url) -> Self {
        self.base.local_validator = validator;
        self
@@ -316,13 +332,13 @@ impl Config {
    }

    pub fn with_mnemonic<S: Into<String>>(mut self, mnemonic: S) -> Self {
-        self.rewarding.mnemonic = mnemonic.into();
+        self.base.mnemonic = mnemonic.into();
        self
    }

    #[cfg(feature = "coconut")]
    pub fn with_keypair<S: Into<String>>(mut self, keypair_bs58: S) -> Self {
-        self.base.keypair_bs58 = keypair_bs58.into();
+        self.coconut_signer.keypair_bs58 = keypair_bs58.into();
        self
    }

@@ -362,13 +378,17 @@ impl Config {
        self.network_monitor.enabled
    }

+    #[cfg(feature = "coconut")]
+    pub fn get_coconut_signer_enabled(&self) -> bool {
+        self.coconut_signer.enabled
+    }
+
    pub fn get_testnet_mode(&self) -> bool {
        self.network_monitor.testnet_mode
    }

-    #[cfg(not(feature = "coconut"))]
-    pub fn get_backup_bandwidth_token_keys_dir(&self) -> PathBuf {
-        self.network_monitor.backup_bandwidth_token_keys_dir.clone()
+    pub fn get_credentials_database_path(&self) -> PathBuf {
+        self.network_monitor.credentials_database_path.clone()
    }

    #[cfg(not(feature = "coconut"))]
@@ -396,7 +416,7 @@ impl Config {
    }

    pub fn get_mnemonic(&self) -> String {
-        self.rewarding.mnemonic.clone()
+        self.base.mnemonic.clone()
    }

    pub fn get_network_monitor_run_interval(&self) -> Duration {
@@ -58,10 +58,7 @@ gateway_connection_timeout = '{{ network_monitor.gateway_connection_timeout }}'
 # packets before declaring nodes unreachable.
 packet_delivery_timeout = '{{ network_monitor.packet_delivery_timeout }}'

-# Path to directory containing public/private keys used for bandwidth token purchase.
-# Those are saved in case of emergency, to be able to reclaim bandwidth tokens.
-# The public key is the name of the file, while the private key is the content.
-backup_bandwidth_token_keys_dir = '{{ network_monitor.backup_bandwidth_token_keys_dir }}'
+credentials_database_path = '{{ network_monitor.credentials_database_path }}'

 # Ethereum private key.
 eth_private_key = '{{ network_monitor.eth_private_key }}'
@@ -128,9 +128,13 @@ impl<C> ValidatorCacheRefresher<C> {
            self.nymd_client.get_gateways(),
        )?;

-        let rewarded_set_identities = self.nymd_client.get_rewarded_set_identities().await?;
-        let (rewarded_set, active_set) =
-            self.collect_rewarded_and_active_set_details(&mixnodes, rewarded_set_identities);
+        let (rewarded_set, active_set) = if let Ok(rewarded_set_identities) =
+            self.nymd_client.get_rewarded_set_identities().await
+        {
+            self.collect_rewarded_and_active_set_details(&mixnodes, rewarded_set_identities)
+        } else {
+            (Vec::new(), Vec::new())
+        };

        let epoch_rewarding_params = self.nymd_client.get_current_epoch_reward_params().await?;
        let current_epoch = self.nymd_client.get_current_epoch().await?;
@@ -29,7 +29,8 @@ use url::Url;

 use crate::rewarded_set_updater::RewardedSetUpdater;
 #[cfg(feature = "coconut")]
-use coconut::{client::QueryClient, InternalSignRequest};
+use coconut::InternalSignRequest;
+use validator_client::nymd::SigningNymdClient;

 pub(crate) mod config;
 pub(crate) mod contract_cache;
@@ -55,10 +56,7 @@ const TESTNET_MODE_ARG_NAME: &str = "testnet-mode";
 const KEYPAIR_ARG: &str = "keypair";

 #[cfg(feature = "coconut")]
-const SIGNED_DEPOSITS_ARG: &str = "signed-deposits";
-
-#[cfg(feature = "coconut")]
-const COCONUT_ONLY_FLAG: &str = "coconut-only";
+const COCONUT_ENABLED: &str = "enable-coconut";

 #[cfg(not(feature = "coconut"))]
 const ETH_ENDPOINT: &str = "eth_endpoint";
@@ -113,10 +111,6 @@ fn long_version() -> String {
 }

 fn parse_args<'a>() -> ArgMatches<'a> {
-    #[cfg(feature = "coconut")]
-    let monitor_reqs = &[];
-    #[cfg(not(feature = "coconut"))]
-    let monitor_reqs = &[ETH_ENDPOINT, ETH_PRIVATE_KEY];
    let build_details = long_version();
    let base_app = App::new("Nym Validator API")
        .version(crate_version!())
@@ -127,14 +121,13 @@ fn parse_args<'a>() -> ArgMatches<'a> {
                .help("specifies whether a network monitoring is enabled on this API")
                .long(MONITORING_ENABLED)
                .short("m")
-                .requires_all(monitor_reqs)
        )
        .arg(
            Arg::with_name(REWARDING_ENABLED)
                .help("specifies whether a network rewarding is enabled on this API")
                .long(REWARDING_ENABLED)
                .short("r")
-                .requires(MONITORING_ENABLED)
+                .requires_all(&[MONITORING_ENABLED, MNEMONIC_ARG])
        )
        .arg(
            Arg::with_name(NYMD_VALIDATOR_ARG)
@@ -151,7 +144,6 @@ fn parse_args<'a>() -> ArgMatches<'a> {
                 .long(MNEMONIC_ARG)
                 .help("Mnemonic of the network monitor used for rewarding operators")
                 .takes_value(true)
-                 .requires(REWARDING_ENABLED),
        )
        .arg(
            Arg::with_name(WRITE_CONFIG_ARG)
@@ -170,7 +162,6 @@ fn parse_args<'a>() -> ArgMatches<'a> {
                .help("Specifies the minimum percentage of monitor test run data present in order to distribute rewards for given interval.")
                .takes_value(true)
                .long(REWARDING_MONITOR_THRESHOLD_ARG)
-                .requires(REWARDING_ENABLED)
        )
        .arg(
            Arg::with_name(TESTNET_MODE_ARG_NAME)
@@ -179,21 +170,19 @@ fn parse_args<'a>() -> ArgMatches<'a> {
        );

    #[cfg(feature = "coconut")]
-        let base_app = base_app.arg(
-        Arg::with_name(KEYPAIR_ARG)
-            .help("Path to the secret key file")
-            .takes_value(true)
-            .long(KEYPAIR_ARG),
-    ).arg(
-        Arg::with_name(SIGNED_DEPOSITS_ARG)
-            .help("Path to the directory used to store the already signed deposit transactions. This prevents the validator for double signing for the same deposit")
-            .takes_value(true)
-            .long(SIGNED_DEPOSITS_ARG),
-    ).arg(
-        Arg::with_name(COCONUT_ONLY_FLAG)
-            .help("Flag to indicate whether validator api should only be used for credential issuance with no blockchain connection")
-            .long(COCONUT_ONLY_FLAG),
-    );
+    let base_app = base_app
+        .arg(
+            Arg::with_name(KEYPAIR_ARG)
+                .help("Path to the secret key file")
+                .takes_value(true)
+                .long(KEYPAIR_ARG),
+        )
+        .arg(
+            Arg::with_name(COCONUT_ENABLED)
+                .help("Flag to indicate whether coconut signer authority is enabled on this API")
+                .requires_all(&[KEYPAIR_ARG, MNEMONIC_ARG])
+                .long(COCONUT_ENABLED),
+        );

    #[cfg(not(feature = "coconut"))]
        let base_app = base_app.arg(
@@ -253,6 +242,11 @@ fn override_config(mut config: Config, matches: &ArgMatches<'_>) -> Config {
        config = config.with_rewarding_enabled(true)
    }

+    #[cfg(feature = "coconut")]
+    if matches.is_present(COCONUT_ENABLED) {
+        config = config.with_coconut_signer_enabled(true)
+    }
+
    if let Some(raw_validators) = matches.value_of(API_VALIDATORS_ARG) {
        config = config.with_custom_validator_apis(parse_validators(raw_validators));
    }
@@ -397,7 +391,11 @@ fn expected_monitor_test_runs(config: &Config, interval_length: Duration) -> usi
    (interval_length.as_secs() / test_delay.as_secs()) as usize
 }

-async fn setup_rocket(config: &Config, liftoff_notify: Arc<Notify>) -> Result<Rocket<Ignite>> {
+async fn setup_rocket(
+    config: &Config,
+    liftoff_notify: Arc<Notify>,
+    _nymd_client: Option<Client<SigningNymdClient>>,
+) -> Result<Rocket<Ignite>> {
    // let's build our rocket!
    let rocket = rocket::build()
        .attach(setup_cors()?)
@@ -413,11 +411,15 @@ async fn setup_rocket(config: &Config, liftoff_notify: Arc<Notify>) -> Result<Ro
    };

    #[cfg(feature = "coconut")]
-    let rocket = rocket.attach(InternalSignRequest::stage(
-        QueryClient::new()?,
-        config.keypair(),
-        storage.clone().unwrap(),
-    ));
+    let rocket = if config.get_coconut_signer_enabled() {
+        rocket.attach(InternalSignRequest::stage(
+            _nymd_client.expect("Should have a signing client here"),
+            config.keypair(),
+            storage.clone().unwrap(),
+        ))
+    } else {
+        rocket
+    };

    // see if we should start up network monitor and if so, attach the node status api
    if config.get_network_monitor_enabled() {
@@ -459,25 +461,21 @@ async fn run_validator_api(matches: ArgMatches<'static>) -> Result<()> {
        return Ok(());
    }

-    #[cfg(feature = "coconut")]
-    if matches.is_present(COCONUT_ONLY_FLAG) {
-        // this simplifies everything - we just want to run coconut things
-        return rocket::build()
-            .attach(setup_cors()?)
-            .attach(InternalSignRequest::stage(
-                QueryClient::new()?,
-                config.keypair(),
-                ValidatorApiStorage::init(config.get_node_status_api_database_path()).await?,
-            ))
-            .launch()
-            .await
-            .map_err(|err| err.into());
-    }
+    let signing_nymd_client = if matches.is_present(MNEMONIC_ARG) {
+        Some(Client::new_signing(&config))
+    } else {
+        None
+    };

    let liftoff_notify = Arc::new(Notify::new());

    // let's build our rocket!
-    let rocket = setup_rocket(&config, Arc::clone(&liftoff_notify)).await?;
+    let rocket = setup_rocket(
+        &config,
+        Arc::clone(&liftoff_notify),
+        signing_nymd_client.clone(),
+    )
+    .await?;
    let monitor_builder = setup_network_monitor(&config, system_version, &rocket);

    let validator_cache = rocket.state::<ValidatorCache>().unwrap().clone();
@@ -485,7 +483,7 @@ async fn run_validator_api(matches: ArgMatches<'static>) -> Result<()> {
    // if network monitor is disabled, we're not going to be sending any rewarding hence
    // we're not starting signing client
    if config.get_network_monitor_enabled() {
-        let nymd_client = Client::new_signing(&config);
+        let nymd_client = signing_nymd_client.expect("We should have a signing client here");
        let validator_cache_refresher = ValidatorCacheRefresher::new(
            nymd_client.clone(),
            config.get_caching_interval(),
@@ -1,6 +1,7 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0

+use credential_storage::PersistentStorage;
 use crypto::asymmetric::{encryption, identity};
 use futures::channel::mpsc;
 use gateway_client::bandwidth::BandwidthController;
@@ -73,14 +74,16 @@ impl<'a> NetworkMonitorBuilder<'a> {

        #[cfg(feature = "coconut")]
        let bandwidth_controller = BandwidthController::new(
+            credential_storage::initialise_storage(self.config.get_credentials_database_path())
+                .await,
            self.config.get_all_validator_api_endpoints(),
-            *identity_keypair.public_key(),
        );
        #[cfg(not(feature = "coconut"))]
        let bandwidth_controller = BandwidthController::new(
+            credential_storage::initialise_storage(self.config.get_credentials_database_path())
+                .await,
            self.config.get_network_monitor_eth_endpoint(),
            self.config.get_network_monitor_eth_private_key(),
-            self.config.get_backup_bandwidth_token_keys_dir(),
        )
        .expect("Could not create bandwidth controller");

@@ -157,7 +160,7 @@ fn new_packet_sender(
    gateways_status_updater: GatewayClientUpdateSender,
    local_identity: Arc<identity::KeyPair>,
    max_sending_rate: usize,
-    bandwidth_controller: BandwidthController,
+    bandwidth_controller: BandwidthController<PersistentStorage>,
    testnet_mode: bool,
 ) -> PacketSender {
    PacketSender::new(
@@ -7,6 +7,7 @@ use crate::network_monitor::monitor::gateway_clients_cache::{
 use crate::network_monitor::monitor::gateways_pinger::GatewayPinger;
 use crate::network_monitor::monitor::receiver::{GatewayClientUpdate, GatewayClientUpdateSender};
 use config::defaults::REMAINING_BANDWIDTH_THRESHOLD;
+use credential_storage::PersistentStorage;
 use crypto::asymmetric::identity::{self, PUBLIC_KEY_LENGTH};
 use futures::channel::mpsc;
 use futures::stream::{self, FuturesUnordered, StreamExt};
@@ -97,7 +98,7 @@ struct FreshGatewayClientData {
    // for coconut bandwidth credentials we currently have no double spending protection, just to
    // get things running we're re-using the same credential for all gateways all the time.
    // THIS IS VERY BAD!!
-    bandwidth_controller: BandwidthController,
+    bandwidth_controller: BandwidthController<PersistentStorage>,
    testnet_mode: bool,
 }

@@ -157,7 +158,7 @@ impl PacketSender {
        gateway_connection_timeout: Duration,
        max_concurrent_clients: usize,
        max_sending_rate: usize,
-        bandwidth_controller: BandwidthController,
+        bandwidth_controller: BandwidthController<PersistentStorage>,
        testnet_mode: bool,
    ) -> Self {
        PacketSender {
@@ -3,6 +3,8 @@

 use crate::config::Config;
 use crate::rewarded_set_updater::error::RewardingError;
+#[cfg(feature = "coconut")]
+use async_trait::async_trait;
 use config::defaults::{DEFAULT_NETWORK, DEFAULT_VALIDATOR_API_PORT};
 use mixnet_contract_common::Interval;
 use mixnet_contract_common::{
@@ -21,7 +23,7 @@ use validator_client::nymd::{
 };
 use validator_client::ValidatorClientError;

-pub(crate) struct Client<C>(Arc<RwLock<validator_client::Client<C>>>);
+pub(crate) struct Client<C>(pub(crate) Arc<RwLock<validator_client::Client<C>>>);

 impl<C> Clone for Client<C> {
    fn clone(&self) -> Self {
@@ -401,3 +403,20 @@ impl<C> Client<C> {
        }
    }
 }
+
+#[async_trait]
+#[cfg(feature = "coconut")]
+impl<C> crate::coconut::client::Client for Client<C>
+where
+    C: CosmWasmClient + Sync + Send,
+{
+    async fn get_tx(
+        &self,
+        tx_hash: &str,
+    ) -> crate::coconut::error::Result<validator_client::nymd::TxResponse> {
+        let tx_hash = tx_hash
+            .parse::<validator_client::nymd::tx::Hash>()
+            .map_err(|_| crate::coconut::error::CoconutError::TxHashParseError)?;
+        Ok(self.0.read().await.nymd.get_tx(tx_hash).await?)
+    }
+}