From d952f3233a68077a8fe9fd03aef7b31220114bad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Stuczy=C5=84ski?= Date: Wed, 10 Nov 2021 12:52:13 +0000 Subject: [PATCH] Active sets => Rewarded + Active/Idle sets (#864) * Introduced rewarding_interval_nonce to contract state * Queries for ibid. * Mixnode demanded set size * Routes for obtaining demanded/active mixnode sets * Testing only demanded nodes * Typo * Initial state * Feature-locking unused imports * Generating pseudorandom (with deterministic seed) demanded mixnodes set * cargo fmt * Fixed tauri state * [ci skip] Generate TS types * Renamed 'demanded' set to 'rewarded' set * Some renaming action * [ci skip] Generate TS types * Fixed post-merge dependency issue in tests Co-authored-by: jstuczyn --- Cargo.lock | 2 + .../validator-client/src/client.rs | 13 +- .../validator-client/src/nymd/mod.rs | 31 ++- common/mixnet-contract/src/lib.rs | 5 +- common/mixnet-contract/src/mixnode.rs | 8 + common/mixnet-contract/src/msg.rs | 1 + common/mixnet-contract/src/types.rs | 20 +- contracts/mixnet/src/contract.rs | 13 +- contracts/mixnet/src/error.rs | 3 + contracts/mixnet/src/queries.rs | 17 +- contracts/mixnet/src/state.rs | 6 + contracts/mixnet/src/transactions.rs | 16 +- .../src-tauri/src/operations/admin.rs | 3 + tauri-wallet/src/types/rust/stateparams.ts | 1 + validator-api/Cargo.toml | 4 +- validator-api/src/cache/mod.rs | 196 +++++++++++++----- validator-api/src/cache/routes.rs | 13 +- validator-api/src/config/mod.rs | 2 +- validator-api/src/network_monitor/chunker.rs | 2 +- validator-api/src/network_monitor/mod.rs | 2 +- .../src/network_monitor/monitor/preparer.rs | 71 ++++--- .../src/network_monitor/test_packet.rs | 3 +- validator-api/src/nymd_client.rs | 36 +++- validator-api/src/rewarding/mod.rs | 7 +- 24 files changed, 356 insertions(+), 119 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ee1f369724..a948b8b0c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3473,6 +3473,8 @@ dependencies = [ "pin-project", "pretty_env_logger", "rand 0.7.3", + "rand 0.8.4", + "rand_chacha 0.3.1", "reqwest", "rocket", "rocket_cors", diff --git a/common/client-libs/validator-client/src/client.rs b/common/client-libs/validator-client/src/client.rs index 5312e248c0..c50a9cffd0 100644 --- a/common/client-libs/validator-client/src/client.rs +++ b/common/client-libs/validator-client/src/client.rs @@ -10,9 +10,9 @@ use mixnet_contract::StateParams; use crate::{validator_api, ValidatorClientError}; use coconut_interface::{BlindSignRequestBody, BlindedSignatureResponse, VerificationKeyResponse}; -#[cfg(feature = "nymd-client")] -use mixnet_contract::RawDelegationData; use mixnet_contract::{GatewayBond, MixNodeBond}; +#[cfg(feature = "nymd-client")] +use mixnet_contract::{RawDelegationData, RewardingIntervalResponse}; use url::Url; #[cfg(feature = "nymd-client")] @@ -172,6 +172,15 @@ impl Client { Ok(self.nymd.get_state_params().await?) } + pub async fn get_current_rewarding_interval( + &self, + ) -> Result + where + C: CosmWasmClient + Sync, + { + Ok(self.nymd.get_current_rewarding_interval().await?) + } + pub async fn get_reward_pool(&self) -> Result where C: CosmWasmClient + Sync, diff --git a/common/client-libs/validator-client/src/nymd/mod.rs b/common/client-libs/validator-client/src/nymd/mod.rs index 7fc1685849..4102803365 100644 --- a/common/client-libs/validator-client/src/nymd/mod.rs +++ b/common/client-libs/validator-client/src/nymd/mod.rs @@ -16,7 +16,8 @@ use mixnet_contract::{ Addr, Delegation, ExecuteMsg, Gateway, GatewayOwnershipResponse, IdentityKey, LayerDistribution, MixNode, MixOwnershipResponse, PagedAllDelegationsResponse, PagedGatewayResponse, PagedMixDelegationsResponse, PagedMixnodeResponse, - PagedReverseMixDelegationsResponse, QueryMsg, RawDelegationData, StateParams, + PagedReverseMixDelegationsResponse, QueryMsg, RawDelegationData, RewardingIntervalResponse, + StateParams, }; use serde::Serialize; use std::collections::HashMap; @@ -27,6 +28,7 @@ pub use crate::nymd::cosmwasm_client::signing_client::SigningCosmWasmClient; pub use crate::nymd::gas_price::GasPrice; pub use cosmrs::rpc::HttpClient as QueryNymdClient; pub use cosmrs::tendermint::block::Height; +pub use cosmrs::tendermint::hash; pub use cosmrs::tendermint::Time as TendermintTime; pub use cosmrs::tx::{Fee, Gas}; pub use cosmrs::Coin as CosmosCoin; @@ -184,6 +186,21 @@ impl NymdClient { self.client.get_height().await } + /// Obtains the hash of a block specified by the provided height. + /// + /// # Arguments + /// + /// * `height`: height of the block for which we want to obtain the hash. + pub async fn get_block_hash(&self, height: u32) -> Result + where + C: CosmWasmClient + Sync, + { + self.client + .get_block(Some(height)) + .await + .map(|block| block.block_id.hash) + } + pub async fn get_balance(&self, address: &AccountId) -> Result, NymdError> where C: CosmWasmClient + Sync, @@ -201,6 +218,18 @@ impl NymdClient { .await } + pub async fn get_current_rewarding_interval( + &self, + ) -> Result + where + C: CosmWasmClient + Sync, + { + let request = QueryMsg::CurrentRewardingInterval {}; + self.client + .query_contract_smart(self.contract_address()?, &request) + .await + } + pub async fn get_layer_distribution(&self) -> Result where C: CosmWasmClient + Sync, diff --git a/common/mixnet-contract/src/lib.rs b/common/mixnet-contract/src/lib.rs index 04c758ec29..c3162adf4a 100644 --- a/common/mixnet-contract/src/lib.rs +++ b/common/mixnet-contract/src/lib.rs @@ -16,4 +16,7 @@ pub use delegation::{ pub use gateway::{Gateway, GatewayBond, GatewayOwnershipResponse, PagedGatewayResponse}; pub use mixnode::{Layer, MixNode, MixNodeBond, MixOwnershipResponse, PagedMixnodeResponse}; pub use msg::{ExecuteMsg, InstantiateMsg, MigrateMsg, QueryMsg}; -pub use types::{IdentityKey, IdentityKeyRef, LayerDistribution, SphinxKey, StateParams}; +pub use types::{ + IdentityKey, IdentityKeyRef, LayerDistribution, RewardingIntervalResponse, SphinxKey, + StateParams, +}; diff --git a/common/mixnet-contract/src/mixnode.rs b/common/mixnet-contract/src/mixnode.rs index 82d8b4c33c..287271e373 100644 --- a/common/mixnet-contract/src/mixnode.rs +++ b/common/mixnet-contract/src/mixnode.rs @@ -191,6 +191,14 @@ impl MixNodeBond { &self.mix_node } + pub fn total_stake(&self) -> Option { + if self.bond_amount.denom != self.total_delegation.denom { + None + } else { + Some(self.bond_amount.amount.u128() + self.total_delegation.amount.u128()) + } + } + pub fn total_delegation(&self) -> Coin { self.total_delegation.clone() } diff --git a/common/mixnet-contract/src/msg.rs b/common/mixnet-contract/src/msg.rs index 84f872fdc0..f60ee411f1 100644 --- a/common/mixnet-contract/src/msg.rs +++ b/common/mixnet-contract/src/msg.rs @@ -63,6 +63,7 @@ pub enum QueryMsg { address: Addr, }, StateParams {}, + CurrentRewardingInterval {}, GetMixDelegations { mix_identity: IdentityKey, start_after: Option, diff --git a/common/mixnet-contract/src/types.rs b/common/mixnet-contract/src/types.rs index b2a16515fe..447bfc710c 100644 --- a/common/mixnet-contract/src/types.rs +++ b/common/mixnet-contract/src/types.rs @@ -26,15 +26,28 @@ impl LayerDistribution { } } +#[derive(Debug, Default, Serialize, Deserialize, Copy, Clone, Eq, PartialEq)] +pub struct RewardingIntervalResponse { + pub current_rewarding_interval_starting_block: u64, + pub rewarding_in_progress: bool, +} + #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, JsonSchema)] pub struct StateParams { - pub epoch_length: u32, // length of an epoch, expressed in hours + pub epoch_length: u32, // length of a rewarding epoch/interval, expressed in hours pub minimum_mixnode_bond: Uint128, // minimum amount a mixnode must bond to get into the system pub minimum_gateway_bond: Uint128, // minimum amount a gateway must bond to get into the system pub mixnode_bond_reward_rate: Decimal, // annual reward rate, expressed as a decimal like 1.25 pub mixnode_delegation_reward_rate: Decimal, // annual reward rate, expressed as a decimal like 1.25 + + // number of mixnode that are going to get rewarded during current rewarding interval (k_m) + // based on overall demand for private bandwidth- + pub mixnode_rewarded_set_size: u32, + + // subset of rewarded mixnodes that are actively receiving mix traffic + // used to handle shorter-term (e.g. hourly) fluctuations of demand pub mixnode_active_set_size: u32, } @@ -54,6 +67,11 @@ impl Display for StateParams { "mixnode delegation reward rate: {}; ", self.mixnode_delegation_reward_rate )?; + write!( + f, + "mixnode rewarded set size: {}", + self.mixnode_rewarded_set_size + )?; write!( f, "mixnode active set size: {}", diff --git a/contracts/mixnet/src/contract.rs b/contracts/mixnet/src/contract.rs index eb360bd686..7b9862aff2 100644 --- a/contracts/mixnet/src/contract.rs +++ b/contracts/mixnet/src/contract.rs @@ -26,6 +26,7 @@ pub const INITIAL_MIXNODE_BOND: Uint128 = Uint128(100_000000); pub const INITIAL_MIXNODE_BOND_REWARD_RATE: u64 = 110; pub const INITIAL_MIXNODE_DELEGATION_REWARD_RATE: u64 = 110; +pub const INITIAL_MIXNODE_REWARDED_SET_SIZE: u32 = 200; pub const INITIAL_MIXNODE_ACTIVE_SET_SIZE: u32 = 100; pub const INITIAL_REWARD_POOL: u128 = 250_000_000_000_000; @@ -35,7 +36,7 @@ pub const DEFAULT_SYBIL_RESISTANCE_PERCENT: u8 = 30; // We'll be assuming a few more things, profit margin and cost function. Since we don't have relialable package measurement, we'll be using uptime. We'll also set the value of 1 Nym to 1 $, to be able to translate epoch costs to Nyms. We'll also assume a cost of 40$ per epoch(month), converting that to Nym at our 1$ rate translates to 40_000_000 uNyms pub const DEFAULT_COST_PER_EPOCH: u32 = 40_000_000; -fn default_initial_state(owner: Addr) -> State { +fn default_initial_state(owner: Addr, env: Env) -> State { let mixnode_bond_reward_rate = Decimal::percent(INITIAL_MIXNODE_BOND_REWARD_RATE); let mixnode_delegation_reward_rate = Decimal::percent(INITIAL_MIXNODE_DELEGATION_REWARD_RATE); @@ -48,8 +49,11 @@ fn default_initial_state(owner: Addr) -> State { minimum_gateway_bond: INITIAL_GATEWAY_BOND, mixnode_bond_reward_rate, mixnode_delegation_reward_rate, + mixnode_rewarded_set_size: INITIAL_MIXNODE_REWARDED_SET_SIZE, mixnode_active_set_size: INITIAL_MIXNODE_ACTIVE_SET_SIZE, }, + rewarding_interval_starting_block: env.block.height, + rewarding_in_progress: false, mixnode_epoch_bond_reward: calculate_epoch_reward_rate( INITIAL_DEFAULT_EPOCH_LENGTH, mixnode_bond_reward_rate, @@ -69,11 +73,11 @@ fn default_initial_state(owner: Addr) -> State { #[entry_point] pub fn instantiate( deps: DepsMut, - _env: Env, + env: Env, info: MessageInfo, _msg: InstantiateMsg, ) -> Result { - let state = default_initial_state(info.sender); + let state = default_initial_state(info.sender, env); config(deps.storage).save(&state)?; layer_distribution(deps.storage).save(&Default::default())?; @@ -131,6 +135,9 @@ pub fn query(deps: Deps, _env: Env, msg: QueryMsg) -> Result to_binary(&queries::query_state_params(deps)), + QueryMsg::CurrentRewardingInterval {} => { + to_binary(&queries::query_rewarding_interval(deps)) + } QueryMsg::LayerDistribution {} => to_binary(&queries::query_layer_distribution(deps)), QueryMsg::GetMixDelegations { mix_identity, diff --git a/contracts/mixnet/src/error.rs b/contracts/mixnet/src/error.rs index 6d3db45be5..18b7289664 100644 --- a/contracts/mixnet/src/error.rs +++ b/contracts/mixnet/src/error.rs @@ -51,6 +51,9 @@ pub enum ContractError { #[error("The delegation reward rate for mixnode was set to be lower than 1")] DecreasingMixnodeDelegationReward, + #[error("Provided active set size is bigger than the demanded set")] + InvalidActiveSetSize, + #[error("The node had uptime larger than 100%")] UnexpectedUptime, diff --git a/contracts/mixnet/src/queries.rs b/contracts/mixnet/src/queries.rs index 7453b2e1f9..1d63b87702 100644 --- a/contracts/mixnet/src/queries.rs +++ b/contracts/mixnet/src/queries.rs @@ -4,7 +4,7 @@ use crate::error::ContractError; use crate::helpers::get_all_delegations_paged; use crate::storage::{ - all_mix_delegations_read, circulating_supply, gateways_owners_read, gateways_read, + all_mix_delegations_read, circulating_supply, config_read, gateways_owners_read, gateways_read, mix_delegations_read, mixnodes_owners_read, mixnodes_read, read_layer_distribution, read_state_params, reverse_mix_delegations_read, reward_pool_value, }; @@ -14,7 +14,7 @@ use mixnet_contract::{ Delegation, GatewayBond, GatewayOwnershipResponse, IdentityKey, LayerDistribution, MixNodeBond, MixOwnershipResponse, PagedAllDelegationsResponse, PagedGatewayResponse, PagedMixDelegationsResponse, PagedMixnodeResponse, PagedReverseMixDelegationsResponse, - RawDelegationData, StateParams, + RawDelegationData, RewardingIntervalResponse, StateParams, }; const BOND_PAGE_MAX_LIMIT: u32 = 100; @@ -87,6 +87,14 @@ pub(crate) fn query_state_params(deps: Deps) -> StateParams { read_state_params(deps.storage) } +pub(crate) fn query_rewarding_interval(deps: Deps) -> RewardingIntervalResponse { + let state = config_read(deps.storage).load().unwrap(); + RewardingIntervalResponse { + current_rewarding_interval_starting_block: state.rewarding_interval_starting_block, + rewarding_in_progress: state.rewarding_in_progress, + } +} + pub(crate) fn query_layer_distribution(deps: Deps) -> LayerDistribution { read_layer_distribution(deps.storage) } @@ -577,8 +585,11 @@ pub(crate) mod tests { minimum_gateway_bond: 456u128.into(), mixnode_bond_reward_rate: "1.23".parse().unwrap(), mixnode_delegation_reward_rate: "7.89".parse().unwrap(), - mixnode_active_set_size: 1000, + mixnode_rewarded_set_size: 1000, + mixnode_active_set_size: 500, }, + rewarding_interval_starting_block: 123, + rewarding_in_progress: false, mixnode_epoch_bond_reward: "1.23".parse().unwrap(), mixnode_epoch_delegation_reward: "7.89".parse().unwrap(), }; diff --git a/contracts/mixnet/src/state.rs b/contracts/mixnet/src/state.rs index 6371e6529f..c7cbea5848 100644 --- a/contracts/mixnet/src/state.rs +++ b/contracts/mixnet/src/state.rs @@ -12,6 +12,12 @@ pub struct State { pub network_monitor_address: Addr, pub params: StateParams, + // keep track of the changes to the current rewarding interval, + // i.e. at which block has the latest rewarding occurred + // and whether another run is already in progress + pub rewarding_interval_starting_block: u64, + pub rewarding_in_progress: bool, + // helper values to avoid having to recalculate them on every single payment operation pub mixnode_epoch_bond_reward: Decimal, // reward per epoch expressed as a decimal like 0.05 pub mixnode_epoch_delegation_reward: Decimal, // reward per epoch expressed as a decimal like 0.05 diff --git a/contracts/mixnet/src/transactions.rs b/contracts/mixnet/src/transactions.rs index 48254473a6..b91b9af25b 100644 --- a/contracts/mixnet/src/transactions.rs +++ b/contracts/mixnet/src/transactions.rs @@ -315,6 +315,12 @@ pub(crate) fn try_update_state_params( return Err(ContractError::DecreasingMixnodeDelegationReward); } + // note: rewarded_set = active_set + idle_set + // hence rewarded set must always be bigger than (or equal to) the active set + if params.mixnode_rewarded_set_size < params.mixnode_active_set_size { + return Err(ContractError::InvalidActiveSetSize); + } + // if we're updating epoch length, recalculate rewards for mixnodes if state.params.epoch_length != params.epoch_length { state.mixnode_epoch_bond_reward = @@ -1460,7 +1466,8 @@ pub mod tests { mixnode_delegation_reward_rate: Decimal::percent( INITIAL_MIXNODE_DELEGATION_REWARD_RATE, ), - mixnode_active_set_size: 42, // change something + mixnode_rewarded_set_size: 100, + mixnode_active_set_size: 50, }; // cannot be updated from non-owner account @@ -1534,6 +1541,13 @@ pub mod tests { expected_mixnode_delegation, new_state.mixnode_epoch_delegation_reward ); + + // error is thrown if rewarded set is smaller than the active set + let info = mock_info("creator", &[]); + let mut new_params = current_state.params.clone(); + new_params.mixnode_rewarded_set_size = new_params.mixnode_active_set_size - 1; + let res = try_update_state_params(deps.as_mut(), info, new_params.clone()); + assert_eq!(Err(ContractError::InvalidActiveSetSize), res) } #[test] diff --git a/tauri-wallet/src-tauri/src/operations/admin.rs b/tauri-wallet/src-tauri/src/operations/admin.rs index 71ac69af8a..0c9db7a115 100644 --- a/tauri-wallet/src-tauri/src/operations/admin.rs +++ b/tauri-wallet/src-tauri/src/operations/admin.rs @@ -17,6 +17,7 @@ pub struct TauriStateParams { minimum_gateway_bond: String, mixnode_bond_reward_rate: String, mixnode_delegation_reward_rate: String, + mixnode_rewarded_set_size: u32, mixnode_active_set_size: u32, } @@ -28,6 +29,7 @@ impl From for TauriStateParams { minimum_gateway_bond: p.minimum_gateway_bond.to_string(), mixnode_bond_reward_rate: p.mixnode_bond_reward_rate.to_string(), mixnode_delegation_reward_rate: p.mixnode_delegation_reward_rate.to_string(), + mixnode_rewarded_set_size: p.mixnode_rewarded_set_size, mixnode_active_set_size: p.mixnode_active_set_size, } } @@ -43,6 +45,7 @@ impl TryFrom for StateParams { minimum_gateway_bond: Uint128::try_from(p.minimum_gateway_bond.as_str())?, mixnode_bond_reward_rate: Decimal::from_str(p.mixnode_bond_reward_rate.as_str())?, mixnode_delegation_reward_rate: Decimal::from_str(p.mixnode_delegation_reward_rate.as_str())?, + mixnode_rewarded_set_size: p.mixnode_rewarded_set_size, mixnode_active_set_size: p.mixnode_active_set_size, }) } diff --git a/tauri-wallet/src/types/rust/stateparams.ts b/tauri-wallet/src/types/rust/stateparams.ts index 4f6a8c6eea..ebe12a6bf1 100644 --- a/tauri-wallet/src/types/rust/stateparams.ts +++ b/tauri-wallet/src/types/rust/stateparams.ts @@ -4,5 +4,6 @@ export interface TauriStateParams { minimum_gateway_bond: string; mixnode_bond_reward_rate: string; mixnode_delegation_reward_rate: string; + mixnode_rewarded_set_size: number; mixnode_active_set_size: number; } \ No newline at end of file diff --git a/validator-api/Cargo.toml b/validator-api/Cargo.toml index 72150c1c6e..f55a71a6bc 100644 --- a/validator-api/Cargo.toml +++ b/validator-api/Cargo.toml @@ -23,7 +23,9 @@ humantime-serde = "1.0" log = "0.4" pin-project = "1.0" pretty_env_logger = "0.4" -rand = "0.7" +rand-07 = { package = "rand", version = "0.7" } # required for compatibility +rand = "0.8" +rand_chacha = "0.3.1" reqwest = { version = "0.11", features = ["json"] } rocket = { version = "0.5.0-rc.1", features = ["json"] } serde = "1.0" diff --git a/validator-api/src/cache/mod.rs b/validator-api/src/cache/mod.rs index 33de1177c2..c0e9d32c92 100644 --- a/validator-api/src/cache/mod.rs +++ b/validator-api/src/cache/mod.rs @@ -4,14 +4,18 @@ use crate::nymd_client::Client; use anyhow::Result; use config::defaults::VALIDATOR_API_VERSION; -use mixnet_contract::{GatewayBond, MixNodeBond, StateParams}; +use mixnet_contract::{GatewayBond, MixNodeBond, RewardingIntervalResponse, StateParams}; +use rand::prelude::SliceRandom; +use rand_chacha::rand_core::SeedableRng; +use rand_chacha::ChaCha20Rng; use rocket::fairing::AdHoc; use serde::Serialize; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tokio::sync::RwLock; use tokio::time; +use validator_client::nymd::hash::SHA256_HASH_SIZE; use validator_client::nymd::CosmWasmClient; pub(crate) mod routes; @@ -29,11 +33,15 @@ pub struct ValidatorCache { struct ValidatorCacheInner { initialised: AtomicBool, + latest_known_rewarding_block: AtomicU64, + mixnodes: RwLock>>, gateways: RwLock>>, - active_mixnodes_available: AtomicBool, - current_mixnode_active_set_size: AtomicUsize, + rewarded_mixnodes: RwLock>>, + + current_mixnode_rewarded_set_size: AtomicU32, + current_mixnode_active_set_size: AtomicU32, } #[derive(Default, Serialize, Clone)] @@ -51,6 +59,13 @@ impl Cache { .as_secs() } + fn renew(&mut self) { + self.as_at = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() + } + pub fn into_inner(self) -> T { self.value } @@ -79,6 +94,13 @@ impl ValidatorCacheRefresher { )?; let state_params = self.nymd_client.get_state_params().await?; + let current_rewarding_interval = self.nymd_client.get_current_rewarding_interval().await?; + let rewarding_block_hash = self + .nymd_client + .get_block_hash( + current_rewarding_interval.current_rewarding_interval_starting_block as u32, + ) + .await?; info!( "Updating validator cache. There are {} mixnodes and {} gateways", @@ -87,7 +109,13 @@ impl ValidatorCacheRefresher { ); self.cache - .update_cache(mixnodes, gateways, state_params) + .update_cache( + mixnodes, + gateways, + state_params, + current_rewarding_interval, + rewarding_block_hash, + ) .await; Ok(()) @@ -128,53 +156,106 @@ impl ValidatorCache { routes::get_mixnodes, routes::get_gateways, routes::get_active_mixnodes, + routes::get_rewarded_mixnodes, ], ) }) } - // TODO: check if all nodes can be compared together, - // i.e. they all have the same denom for bonds and delegations - fn verify_mixnodes(&self, mixnodes: &[MixNodeBond]) -> bool { + fn determine_rewarded_set( + &self, + mixnodes: &[MixNodeBond], + nodes_to_select: u32, + block_hash: Option<[u8; SHA256_HASH_SIZE]>, + ) -> Vec { if mixnodes.is_empty() { - return true; - } - let expected_denom = &mixnodes[0].bond_amount.denom; - for mixnode in mixnodes { - if &mixnode.bond_amount.denom != expected_denom - || &mixnode.total_delegation.denom != expected_denom - { - return false; - } + return Vec::new(); } - true + if block_hash.is_none() { + // I'm not entirely sure under what condition can hash of a block be empty + // (note that we know the block exists otherwise we would have gotten an error + // when attempting to retrieve the hash) + error!("The hash of the block of the rewarding interval is None - we're not going to update the set"); + return Vec::new(); + } + + // seed our rng with the hash of the block of the most recent rewarding interval + let mut rng = ChaCha20Rng::from_seed(block_hash.unwrap()); + + // generate list of mixnodes and their relatively weight (by total stake) + let choices = mixnodes + .iter() + .map(|mix| { + // note that the theoretical maximum possible stake is equal to the total + // supply of all tokens, i.e. 1B (which is 1 quadrillion of native tokens, i.e. 10^15 ~ 2^50) + // which is way below maximum value of f64, so the cast is fine + let total_stake = mix.total_stake().unwrap_or_default() as f64; + (mix, total_stake) + }) // if for some reason node is invalid, treat it as 0 stake/weight + .collect::>(); + + // the unwrap here is fine as an error can only be thrown under one of the following conditions: + // - our mixnode list is empty - we have already checked for that + // - we have invalid weights, i.e. less than zero or NaNs - it shouldn't happen in our case as we safely cast down from u128 + // - all weights are zero - it's impossible in our case as the list of nodes is not empty and weight is proportional to stake. You must have non-zero stake in order to bond + // - we have more than u32::MAX values (which is incredibly unrealistic to have 4B mixnodes bonded... literally every other person on the planet would need one) + choices + .choose_multiple_weighted(&mut rng, nodes_to_select as usize, |item| item.1) + .unwrap() + .map(|(bond, _weight)| *bond) + .cloned() + .collect() } async fn update_cache( &self, - mut mixnodes: Vec, + mixnodes: Vec, gateways: Vec, state: StateParams, + rewarding_interval: RewardingIntervalResponse, + rewarding_block_hash: Option<[u8; SHA256_HASH_SIZE]>, ) { + // if the rewarding is currently in progress, don't mess with the rewarded/active sets + // as most likely will be changed next time this function is called + // // if our data is valid, it means the active sets are available, // otherwise we must explicitly indicate nobody can read this data + if !rewarding_interval.rewarding_in_progress { + // if we're still in the same rewarding interval, i.e. the latest block is the same, + // there's nothing we have to do + if rewarding_interval.current_rewarding_interval_starting_block + > self + .inner + .latest_known_rewarding_block + .load(Ordering::SeqCst) + { + let rewarded_nodes = self.determine_rewarded_set( + &mixnodes, + state.mixnode_rewarded_set_size, + rewarding_block_hash, + ); - if self.verify_mixnodes(&mixnodes) { - // partial_cmp can only fail if the nodes have different denomination, - // but we just checked for that hence the unwraps are fine here - // Note the reverse order of comparison so that the "highest" node would be first - mixnodes.sort_by(|a, b| b.partial_cmp(a).unwrap()); - self.inner - .active_mixnodes_available - .store(true, Ordering::SeqCst); - self.inner - .current_mixnode_active_set_size - .store(state.mixnode_active_set_size as usize, Ordering::SeqCst); - } else { - self.inner - .active_mixnodes_available - .store(false, Ordering::SeqCst); + self.inner + .current_mixnode_rewarded_set_size + .store(state.mixnode_rewarded_set_size, Ordering::SeqCst); + self.inner + .current_mixnode_active_set_size + .store(state.mixnode_active_set_size, Ordering::SeqCst); + self.inner.latest_known_rewarding_block.store( + rewarding_interval.current_rewarding_interval_starting_block, + Ordering::SeqCst, + ); + + self.inner + .rewarded_mixnodes + .write() + .await + .set(rewarded_nodes); + } else { + // however, update the timestamp on the cache + self.inner.rewarded_mixnodes.write().await.renew() + } } self.inner.mixnodes.write().await.set(mixnodes); @@ -189,27 +270,28 @@ impl ValidatorCache { self.inner.gateways.read().await.clone() } - pub async fn active_mixnodes(&self) -> Option>> { - // if active set is available, it means it is already sorted - if self.inner.active_mixnodes_available.load(Ordering::SeqCst) { - let cache = self.inner.mixnodes.read().await; - let timestamp = cache.as_at; - let nodes = cache - .value - .iter() - .take( - self.inner - .current_mixnode_active_set_size - .load(Ordering::SeqCst), - ) - .cloned() - .collect(); - Some(Cache { - value: nodes, - as_at: timestamp, - }) - } else { - None + pub async fn rewarded_mixnodes(&self) -> Cache> { + self.inner.rewarded_mixnodes.read().await.clone() + } + + pub async fn active_mixnodes(&self) -> Cache> { + // rewarded set is already "sorted" by pseudo-randomly choosing mixnodes from + // all bonded nodes, weighted by stake. For the active set choose first k nodes. + let cache = self.inner.rewarded_mixnodes.read().await; + let timestamp = cache.as_at; + let nodes = cache + .value + .iter() + .take( + self.inner + .current_mixnode_active_set_size + .load(Ordering::SeqCst) as usize, + ) + .cloned() + .collect(); + Cache { + value: nodes, + as_at: timestamp, } } @@ -234,9 +316,11 @@ impl ValidatorCacheInner { fn new() -> Self { ValidatorCacheInner { initialised: AtomicBool::new(false), + latest_known_rewarding_block: Default::default(), mixnodes: RwLock::new(Cache::default()), gateways: RwLock::new(Cache::default()), - active_mixnodes_available: AtomicBool::new(false), + rewarded_mixnodes: RwLock::new(Cache::default()), + current_mixnode_rewarded_set_size: Default::default(), current_mixnode_active_set_size: Default::default(), } } diff --git a/validator-api/src/cache/routes.rs b/validator-api/src/cache/routes.rs index c3e3c858aa..cf59be469e 100644 --- a/validator-api/src/cache/routes.rs +++ b/validator-api/src/cache/routes.rs @@ -16,9 +16,12 @@ pub(crate) async fn get_gateways(cache: &State) -> Json, -) -> Option>> { - cache.active_mixnodes().await.map(|cache| Json(cache.value)) +#[get("/mixnodes/rewarded")] +pub(crate) async fn get_rewarded_mixnodes(cache: &State) -> Json> { + Json(cache.rewarded_mixnodes().await.value) +} + +#[get("/mixnodes/active")] +pub(crate) async fn get_active_mixnodes(cache: &State) -> Json> { + Json(cache.active_mixnodes().await.value) } diff --git a/validator-api/src/config/mod.rs b/validator-api/src/config/mod.rs index 5b8be3ac00..5a7c215da5 100644 --- a/validator-api/src/config/mod.rs +++ b/validator-api/src/config/mod.rs @@ -33,7 +33,7 @@ const DEFAULT_MINIMUM_TEST_ROUTES: usize = 1; const DEFAULT_ROUTE_TEST_PACKETS: usize = 1000; const DEFAULT_PER_NODE_TEST_PACKETS: usize = 3; -const DEFAULT_CACHE_INTERVAL: Duration = Duration::from_secs(60); +const DEFAULT_CACHE_INTERVAL: Duration = Duration::from_secs(10 * 60); const DEFAULT_MONITOR_THRESHOLD: u8 = 60; #[derive(Debug, Default, Deserialize, PartialEq, Serialize)] diff --git a/validator-api/src/network_monitor/chunker.rs b/validator-api/src/network_monitor/chunker.rs index a5732600f0..c95f8421f5 100644 --- a/validator-api/src/network_monitor/chunker.rs +++ b/validator-api/src/network_monitor/chunker.rs @@ -5,7 +5,7 @@ use nymsphinx::forwarding::packet::MixPacket; use nymsphinx::{ acknowledgements::AckKey, addressing::clients::Recipient, preparer::MessagePreparer, }; -use rand::rngs::OsRng; +use rand_07::rngs::OsRng; use std::time::Duration; use topology::NymTopology; diff --git a/validator-api/src/network_monitor/mod.rs b/validator-api/src/network_monitor/mod.rs index e8d0437058..8a35b386bf 100644 --- a/validator-api/src/network_monitor/mod.rs +++ b/validator-api/src/network_monitor/mod.rs @@ -57,7 +57,7 @@ impl<'a> NetworkMonitorBuilder<'a> { // TODO: those keys change constant throughout the whole execution of the monitor. // and on top of that, they are used with ALL the gateways -> presumably this should change // in the future - let mut rng = rand::rngs::OsRng; + let mut rng = rand_07::rngs::OsRng; let identity_keypair = Arc::new(identity::KeyPair::new(&mut rng)); let encryption_keypair = Arc::new(encryption::KeyPair::new(&mut rng)); diff --git a/validator-api/src/network_monitor/monitor/preparer.rs b/validator-api/src/network_monitor/monitor/preparer.rs index 7a2f838055..53c1eabe0e 100644 --- a/validator-api/src/network_monitor/monitor/preparer.rs +++ b/validator-api/src/network_monitor/monitor/preparer.rs @@ -185,32 +185,30 @@ impl PacketPreparer { let initialisation_backoff = Duration::from_secs(30); loop { let gateways = self.validator_cache.gateways().await; - let mixnodes = self.validator_cache.active_mixnodes().await; + let mixnodes = self.validator_cache.rewarded_mixnodes().await; - if let Some(mixnodes) = mixnodes { - if gateways.into_inner().len() < minimum_full_routes { - continue; - } + if gateways.into_inner().len() < minimum_full_routes { + continue; + } - let mut layered_mixes = HashMap::new(); - for mix in mixnodes.into_inner() { - let layer = mix.layer; - let mixes = layered_mixes.entry(layer).or_insert_with(Vec::new); - mixes.push(mix) - } + let mut layered_mixes = HashMap::new(); + for mix in mixnodes.into_inner() { + let layer = mix.layer; + let mixes = layered_mixes.entry(layer).or_insert_with(Vec::new); + mixes.push(mix) + } - // we remove the entries as this gives us the ownership and thus we can unwrap to default value - // which makes the code slightly nicer without having to deal with options - let layer1 = layered_mixes.remove(&Layer::One).unwrap_or_default(); - let layer2 = layered_mixes.remove(&Layer::Two).unwrap_or_default(); - let layer3 = layered_mixes.remove(&Layer::Three).unwrap_or_default(); + // we remove the entries as this gives us the ownership and thus we can unwrap to default value + // which makes the code slightly nicer without having to deal with options + let layer1 = layered_mixes.remove(&Layer::One).unwrap_or_default(); + let layer2 = layered_mixes.remove(&Layer::Two).unwrap_or_default(); + let layer3 = layered_mixes.remove(&Layer::Three).unwrap_or_default(); - if layer1.len() >= minimum_full_routes - && layer2.len() >= minimum_full_routes - && layer3.len() >= minimum_full_routes - { - break; - } + if layer1.len() >= minimum_full_routes + && layer2.len() >= minimum_full_routes + && layer3.len() >= minimum_full_routes + { + break; } info!( @@ -221,8 +219,10 @@ impl PacketPreparer { } } - async fn get_network_nodes(&self) -> (Vec, Vec) { - let mixnodes = self.validator_cache.mixnodes().await.into_inner(); + async fn get_rewarded_nodes(&self) -> (Vec, Vec) { + info!(target: "Monitor", "Obtaining network topology..."); + + let mixnodes = self.validator_cache.rewarded_mixnodes().await.into_inner(); let gateways = self.validator_cache.gateways().await.into_inner(); (mixnodes, gateways) @@ -247,7 +247,7 @@ impl PacketPreparer { gateway.try_into().map_err(|_| identity) } - // gets active nodes + // gets rewarded nodes // chooses n random nodes from each layer (and gateway) such that they are not on the blacklist // if failed to parsed => onto the blacklist they go // if generated fewer than n, blacklist will be updated by external function with correctly generated @@ -257,19 +257,19 @@ impl PacketPreparer { n: usize, blacklist: &mut HashSet, ) -> Option> { - let active_mixnodes = self.validator_cache.active_mixnodes().await?.into_inner(); + let rewarded_mixnodes = self.validator_cache.rewarded_mixnodes().await.into_inner(); let gateways = self.validator_cache.gateways().await.into_inner(); // separate mixes into layers for easier selection let mut layered_mixes = HashMap::new(); - for active_mix in active_mixnodes { + for rewarded_mix in rewarded_mixnodes { // filter out mixes on the blacklist - if blacklist.contains(&active_mix.mix_node.identity_key) { + if blacklist.contains(&rewarded_mix.mix_node.identity_key) { continue; } - let layer = active_mix.layer; + let layer = rewarded_mix.layer; let mixes = layered_mixes.entry(layer).or_insert_with(Vec::new); - mixes.push(active_mix) + mixes.push(rewarded_mix) } // filter out gateways on the blacklist let gateways = gateways @@ -445,11 +445,16 @@ impl PacketPreparer { test_nonce: u64, test_routes: &[TestRoute], ) -> PreparedPackets { - let (mixnode_bonds, gateway_bonds) = self.get_network_nodes().await; + // only test mixnodes that are rewarded, i.e. that will be rewarded in this epoch. + // (remember that "idle" nodes are still part of that set) + // we don't care about other nodes, i.e. nodes that are bonded but will not get + // any reward during the current rewarding interval + let (rewarded_mixnodes, all_gateways) = self.get_rewarded_nodes().await; - let (mixes, invalid_mixnodes) = self.filter_outdated_and_malformed_mixnodes(mixnode_bonds); + let (mixes, invalid_mixnodes) = + self.filter_outdated_and_malformed_mixnodes(rewarded_mixnodes); let (gateways, invalid_gateways) = - self.filter_outdated_and_malformed_gateways(gateway_bonds); + self.filter_outdated_and_malformed_gateways(all_gateways); let tested_mixnodes = mixes.iter().map(|node| node.into()).collect::>(); let tested_gateways = gateways.iter().map(|node| node.into()).collect::>(); diff --git a/validator-api/src/network_monitor/test_packet.rs b/validator-api/src/network_monitor/test_packet.rs index 4852c2f2e7..79c2971674 100644 --- a/validator-api/src/network_monitor/test_packet.rs +++ b/validator-api/src/network_monitor/test_packet.rs @@ -188,11 +188,10 @@ impl From for TestedNode { #[cfg(test)] mod tests { use super::*; - use rand::thread_rng; #[test] fn test_packet_roundtrip() { - let mut rng = thread_rng(); + let mut rng = rand_07::thread_rng(); let dummy_keypair = identity::KeyPair::new(&mut rng); let owner = "some owner".to_string(); let packet = TestPacket::new( diff --git a/validator-api/src/nymd_client.rs b/validator-api/src/nymd_client.rs index f4f73a41e4..4fa3302092 100644 --- a/validator-api/src/nymd_client.rs +++ b/validator-api/src/nymd_client.rs @@ -7,12 +7,16 @@ use crate::rewarding::{ PER_MIXNODE_DELEGATION_GAS_INCREASE, REWARDING_GAS_LIMIT_MULTIPLIER, }; use config::defaults::DEFAULT_VALIDATOR_API_PORT; -use mixnet_contract::{Delegation, ExecuteMsg, GatewayBond, IdentityKey, MixNodeBond, StateParams}; +use mixnet_contract::{ + Delegation, ExecuteMsg, GatewayBond, IdentityKey, MixNodeBond, RewardingIntervalResponse, + StateParams, +}; use std::sync::Arc; use std::time::Duration; use tokio::sync::RwLock; use tokio::time::sleep; use validator_client::nymd::{ + hash::{Hash, SHA256_HASH_SIZE}, CosmWasmClient, Fee, QueryNymdClient, SigningCosmWasmClient, SigningNymdClient, TendermintTime, }; use validator_client::ValidatorClientError; @@ -142,6 +146,36 @@ impl Client { self.0.read().await.get_state_params().await } + pub(crate) async fn get_current_rewarding_interval( + &self, + ) -> Result + where + C: CosmWasmClient + Sync, + { + self.0.read().await.get_current_rewarding_interval().await + } + + /// Obtains the hash of a block specified by the provided height. + /// If the resulting digest is empty, a `None` is returned instead. + /// + /// # Arguments + /// + /// * `height`: height of the block for which we want to obtain the hash. + pub async fn get_block_hash( + &self, + height: u32, + ) -> Result, ValidatorClientError> + where + C: CosmWasmClient + Sync, + { + let hash = match self.0.read().await.nymd.get_block_hash(height).await? { + Hash::Sha256(hash) => Some(hash), + Hash::None => None, + }; + + Ok(hash) + } + pub(crate) async fn get_mixnode_delegations( &self, identity: IdentityKey, diff --git a/validator-api/src/rewarding/mod.rs b/validator-api/src/rewarding/mod.rs index 94dfc7ef3f..b18f8a50e0 100644 --- a/validator-api/src/rewarding/mod.rs +++ b/validator-api/src/rewarding/mod.rs @@ -164,12 +164,7 @@ impl Rewarder { // instantaneous. let mut map = HashMap::new(); - let active_bonded_mixnodes = self - .validator_cache - .active_mixnodes() - .await - .ok_or(RewardingError::NoMixnodesToReward)? - .into_inner(); + let active_bonded_mixnodes = self.validator_cache.active_mixnodes().await.into_inner(); for mix in active_bonded_mixnodes.into_iter() { let delegator_count = self .get_mixnode_delegators_count(mix.mix_node.identity_key.clone())