diff --git a/common/client-libs/gateway-client/src/client.rs b/common/client-libs/gateway-client/src/client.rs index b91be4929e..da9fee7bd2 100644 --- a/common/client-libs/gateway-client/src/client.rs +++ b/common/client-libs/gateway-client/src/client.rs @@ -139,6 +139,11 @@ impl GatewayClient { self.gateway_identity } + #[cfg(feature = "coconut")] + pub fn remaining_bandwidth(&self) -> i64 { + self.bandwidth_remaining + } + #[cfg(not(target_arch = "wasm32"))] async fn _close_connection(&mut self) -> Result<(), GatewayClientError> { match std::mem::replace(&mut self.connection, SocketState::NotConnected) { diff --git a/common/mixnet-contract/src/mixnode.rs b/common/mixnet-contract/src/mixnode.rs index 25cce43c11..f6b3281580 100644 --- a/common/mixnet-contract/src/mixnode.rs +++ b/common/mixnet-contract/src/mixnode.rs @@ -23,7 +23,17 @@ pub struct MixNode { } #[derive( - Copy, Clone, Debug, Serialize_repr, PartialEq, PartialOrd, Deserialize_repr, JsonSchema, + Copy, + Clone, + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize_repr, + Deserialize_repr, + JsonSchema, )] #[repr(u8)] pub enum Layer { diff --git a/validator-api/migrations/20211018120000_core_nodes_testing.sql b/validator-api/migrations/20211018120000_core_nodes_testing.sql new file mode 100644 index 0000000000..dcad1ca533 --- /dev/null +++ b/validator-api/migrations/20211018120000_core_nodes_testing.sql @@ -0,0 +1,100 @@ +/* + * Copyright 2021 - Nym Technologies SA + * SPDX-License-Identifier: Apache-2.0 + */ + + +ALTER TABLE mixnode_historical_uptime + RENAME TO _mixnode_historical_uptime_old; + + +CREATE TABLE mixnode_historical_uptime +( + mixnode_details_id INTEGER NOT NULL, + date VARCHAR NOT NULL, + uptime INTEGER NOT NULL, + FOREIGN KEY (mixnode_details_id) + REFERENCES mixnode_details (id) +); + +INSERT INTO mixnode_historical_uptime (mixnode_details_id, date, uptime) +SELECT mixnode_details_id, date, (ipv4_uptime + ipv6_uptime) / 2 as uptime +from _mixnode_historical_uptime_old; + +DROP TABLE _mixnode_historical_uptime_old; + +ALTER TABLE gateway_historical_uptime + RENAME TO _gateway_historical_uptime_old; + + +CREATE TABLE gateway_historical_uptime +( + gateway_details_id INTEGER NOT NULL, + date VARCHAR NOT NULL, + uptime INTEGER NOT NULL, + FOREIGN KEY (gateway_details_id) + REFERENCES gateway_details (id) +); + +INSERT INTO gateway_historical_uptime (gateway_details_id, date, uptime) +SELECT gateway_details_id, date, (ipv4_uptime + ipv6_uptime) / 2 as uptime +from _gateway_historical_uptime_old; + +DROP TABLE _gateway_historical_uptime_old; + +CREATE TABLE mixnode_status +( + mixnode_details_id INTEGER NOT NULL, + reliability INTEGER NOT NULL, + timestamp INTEGER NOT NULL, + FOREIGN KEY (mixnode_details_id) REFERENCES mixnode_details (id) +); + +INSERT INTO mixnode_status (mixnode_details_id, timestamp, reliability) +SELECT mixnode_ipv4_status.mixnode_details_id, + mixnode_ipv4_status.timestamp, + (mixnode_ipv4_status.up * 100 + mixnode_ipv6_status.up * 100) / 2 as reliability +FROM mixnode_ipv4_status + JOIN mixnode_ipv6_status ON mixnode_ipv4_status.mixnode_details_id = mixnode_ipv6_status.mixnode_details_id AND + mixnode_ipv4_status.timestamp = mixnode_ipv6_status.timestamp; + +DROP TABLE mixnode_ipv4_status; +DROP TABLE mixnode_ipv6_status; + +CREATE TABLE gateway_status +( + gateway_details_id INTEGER NOT NULL, + reliability INTEGER NOT NULL, + timestamp INTEGER NOT NULL, + FOREIGN KEY (gateway_details_id) REFERENCES gateway_details (id) +); + +INSERT INTO gateway_status (gateway_details_id, timestamp, reliability) +SELECT gateway_ipv4_status.gateway_details_id, + gateway_ipv4_status.timestamp, + (gateway_ipv4_status.up * 100 + gateway_ipv6_status.up * 100) / 2 as reliability +FROM gateway_ipv4_status + JOIN gateway_ipv6_status ON gateway_ipv4_status.gateway_details_id = gateway_ipv6_status.gateway_details_id AND + gateway_ipv4_status.timestamp = gateway_ipv6_status.timestamp; + +DROP TABLE gateway_ipv4_status; +DROP TABLE gateway_ipv6_status; + +CREATE INDEX `mixnode_status_index` ON `mixnode_status` (`mixnode_details_id`, `timestamp` desc); +CREATE INDEX `gateway_status_index` ON `gateway_status` (`gateway_details_id`, `timestamp` desc); + +CREATE TABLE testing_route ( + gateway_id INTEGER NOT NULL, + layer1_mix_id INTEGER NOT NULL, + layer2_mix_id INTEGER NOT NULL, + layer3_mix_id INTEGER NOT NULL, + monitor_run_id INTEGER NOT NULL, + + FOREIGN KEY (layer1_mix_id) REFERENCES mixnode_details (id), + FOREIGN KEY (layer2_mix_id) REFERENCES mixnode_details (id), + FOREIGN KEY (layer3_mix_id) REFERENCES mixnode_details (id), + + FOREIGN KEY (gateway_id) REFERENCES gateway_details (id), + + FOREIGN KEY (monitor_run_id) references monitor_run (id) +); diff --git a/validator-api/src/config/mod.rs b/validator-api/src/config/mod.rs index afd672f83b..5b8be3ac00 100644 --- a/validator-api/src/config/mod.rs +++ b/validator-api/src/config/mod.rs @@ -8,7 +8,7 @@ use config::defaults::{ }; use config::NymConfig; use serde::{Deserialize, Serialize}; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::time::Duration; use time::OffsetDateTime; use url::Url; @@ -20,7 +20,7 @@ mod template; const DEFAULT_LOCAL_VALIDATOR: &str = "http://localhost:26657"; -const DEFAULT_GATEWAY_SENDING_RATE: usize = 500; +const DEFAULT_GATEWAY_SENDING_RATE: usize = 200; const DEFAULT_MAX_CONCURRENT_GATEWAY_CLIENTS: usize = 50; const DEFAULT_PACKET_DELIVERY_TIMEOUT: Duration = Duration::from_secs(20); const DEFAULT_MONITOR_RUN_INTERVAL: Duration = Duration::from_secs(15 * 60); @@ -28,6 +28,11 @@ const DEFAULT_GATEWAY_PING_INTERVAL: Duration = Duration::from_secs(60); const DEFAULT_GATEWAY_RESPONSE_TIMEOUT: Duration = Duration::from_millis(1_500); const DEFAULT_GATEWAY_CONNECTION_TIMEOUT: Duration = Duration::from_millis(2_500); +const DEFAULT_TEST_ROUTES: usize = 3; +const DEFAULT_MINIMUM_TEST_ROUTES: usize = 1; +const DEFAULT_ROUTE_TEST_PACKETS: usize = 1000; +const DEFAULT_PER_NODE_TEST_PACKETS: usize = 3; + const DEFAULT_CACHE_INTERVAL: Duration = Duration::from_secs(60); const DEFAULT_MONITOR_THRESHOLD: u8 = 60; @@ -111,17 +116,6 @@ pub struct NetworkMonitor { /// The list must also contain THIS validator that is running the test all_validator_apis: Vec, - /// Specifies whether a detailed report should be printed after each run - print_detailed_report: bool, - - // I guess in the future this will be deprecated/removed in favour - // of choosing 'good' network based on current nodes with best behaviour - /// Location of .json file containing IPv4 'good' network topology - good_v4_topology_file: PathBuf, - - /// Location of .json file containing IPv6 'good' network topology - good_v6_topology_file: PathBuf, - /// Specifies the interval at which the network monitor sends the test packets. #[serde(with = "humantime_serde")] run_interval: Duration, @@ -150,16 +144,20 @@ pub struct NetworkMonitor { /// packets before declaring nodes unreachable. #[serde(with = "humantime_serde")] packet_delivery_timeout: Duration, -} -impl NetworkMonitor { - fn default_good_v4_topology_file() -> PathBuf { - Config::default_data_directory(None).join("v4-topology.json") - } + /// Desired number of test routes to be constructed (and working) during a monitor test run. + test_routes: usize, - fn default_good_v6_topology_file() -> PathBuf { - Config::default_data_directory(None).join("v6-topology.json") - } + /// The minimum number of test routes that need to be constructed (and working) in order for + /// a monitor test run to be valid. + minimum_test_routes: usize, + + /// Number of test packets sent via each pseudorandom route to verify whether they work correctly, + /// before using them for testing the rest of the network. + route_test_packets: usize, + + /// Number of test packets sent to each node during regular monitor test run. + per_node_test_packets: usize, } impl Default for NetworkMonitor { @@ -167,9 +165,6 @@ impl Default for NetworkMonitor { NetworkMonitor { enabled: false, all_validator_apis: default_api_endpoints(), - print_detailed_report: false, - good_v4_topology_file: Self::default_good_v4_topology_file(), - good_v6_topology_file: Self::default_good_v6_topology_file(), run_interval: DEFAULT_MONITOR_RUN_INTERVAL, gateway_ping_interval: DEFAULT_GATEWAY_PING_INTERVAL, gateway_sending_rate: DEFAULT_GATEWAY_SENDING_RATE, @@ -177,6 +172,10 @@ impl Default for NetworkMonitor { gateway_response_timeout: DEFAULT_GATEWAY_RESPONSE_TIMEOUT, gateway_connection_timeout: DEFAULT_GATEWAY_CONNECTION_TIMEOUT, packet_delivery_timeout: DEFAULT_PACKET_DELIVERY_TIMEOUT, + test_routes: DEFAULT_TEST_ROUTES, + minimum_test_routes: DEFAULT_MINIMUM_TEST_ROUTES, + route_test_packets: DEFAULT_ROUTE_TEST_PACKETS, + per_node_test_packets: DEFAULT_PER_NODE_TEST_PACKETS, } } } @@ -272,21 +271,6 @@ impl Config { self } - pub fn with_detailed_network_monitor_report(mut self, detailed: bool) -> Self { - self.network_monitor.print_detailed_report = detailed; - self - } - - pub fn with_v4_good_topology>(mut self, path: P) -> Self { - self.network_monitor.good_v4_topology_file = path.as_ref().to_owned(); - self - } - - pub fn with_v6_good_topology>(mut self, path: P) -> Self { - self.network_monitor.good_v6_topology_file = path.as_ref().to_owned(); - self - } - pub fn with_custom_nymd_validator(mut self, validator: Url) -> Self { self.base.local_validator = validator; self @@ -336,18 +320,6 @@ impl Config { self.rewarding.enabled } - pub fn get_detailed_report(&self) -> bool { - self.network_monitor.print_detailed_report - } - - pub fn get_v4_good_topology_file(&self) -> PathBuf { - self.network_monitor.good_v4_topology_file.clone() - } - - pub fn get_v6_good_topology_file(&self) -> PathBuf { - self.network_monitor.good_v6_topology_file.clone() - } - pub fn get_nymd_validator_url(&self) -> Url { self.base.local_validator.clone() } @@ -388,6 +360,22 @@ impl Config { self.network_monitor.gateway_connection_timeout } + pub fn get_test_routes(&self) -> usize { + self.network_monitor.test_routes + } + + pub fn get_minimum_test_routes(&self) -> usize { + self.network_monitor.minimum_test_routes + } + + pub fn get_route_test_packets(&self) -> usize { + self.network_monitor.route_test_packets + } + + pub fn get_per_node_test_packets(&self) -> usize { + self.network_monitor.per_node_test_packets + } + pub fn get_caching_interval(&self) -> Duration { self.topology_cacher.caching_interval } diff --git a/validator-api/src/config/template.rs b/validator-api/src/config/template.rs index 87ad939301..59d61c1e90 100644 --- a/validator-api/src/config/template.rs +++ b/validator-api/src/config/template.rs @@ -35,15 +35,6 @@ all_validator_apis = [ {{/each}} ] -# Specifies whether a detailed report should be printed after each run -print_detailed_report = {{ network_monitor.print_detailed_report }} - -# Location of .json file containing IPv4 'good' network topology -good_v4_topology_file = '{{ network_monitor.good_v4_topology_file }}' - -# Location of .json file containing IPv6 'good' network topology -good_v6_topology_file = '{{ network_monitor.good_v6_topology_file }}' - # Specifies the interval at which the network monitor sends the test packets. run_interval = '{{ network_monitor.run_interval }}' @@ -66,6 +57,20 @@ gateway_connection_timeout = '{{ network_monitor.gateway_connection_timeout }}' # Specifies the duration the monitor is going to wait after sending all measurement # packets before declaring nodes unreachable. packet_delivery_timeout = '{{ network_monitor.packet_delivery_timeout }}' + +# Desired number of test routes to be constructed (and working) during a monitor test run. +test_routes = {{ network_monitor.test_routes }} + +# The minimum number of test routes that need to be constructed (and working) in order for +# a monitor test run to be valid. +minimum_test_routes = {{ network_monitor.minimum_test_routes }} + +# Number of test packets sent via each pseudorandom route to verify whether they work correctly, +# before using them for testing the rest of the network. +route_test_packets = {{ network_monitor.route_test_packets }} + +# Number of test packets sent to each node during regular monitor test run. +per_node_test_packets = {{ network_monitor.per_node_test_packets }} [node_status_api] diff --git a/validator-api/src/main.rs b/validator-api/src/main.rs index 19a1f3b8bd..e31d8ff115 100644 --- a/validator-api/src/main.rs +++ b/validator-api/src/main.rs @@ -6,7 +6,6 @@ extern crate rocket; use crate::cache::ValidatorCacheRefresher; use crate::config::Config; -use crate::network_monitor::tested_network::good_topology::parse_topology_file; use crate::network_monitor::NetworkMonitorBuilder; use crate::node_status_api::uptime_updater::HistoricalUptimeUpdater; use crate::nymd_client::Client; @@ -47,9 +46,6 @@ mod coconut; const MONITORING_ENABLED: &str = "enable-monitor"; const REWARDING_ENABLED: &str = "enable-rewarding"; -const V4_TOPOLOGY_ARG: &str = "v4-topology-filepath"; -const V6_TOPOLOGY_ARG: &str = "v6-topology-filepath"; -const DETAILED_REPORT_ARG: &str = "detailed-report"; const MIXNET_CONTRACT_ARG: &str = "mixnet-contract"; const MNEMONIC_ARG: &str = "mnemonic"; const WRITE_CONFIG_ARG: &str = "save-config"; @@ -66,8 +62,6 @@ const EPOCH_LENGTH_ARG: &str = "epoch-length"; const FIRST_REWARDING_EPOCH_ARG: &str = "first-epoch"; const REWARDING_MONITOR_THRESHOLD_ARG: &str = "monitor-threshold"; -pub(crate) const PENALISE_OUTDATED: bool = false; - fn parse_validators(raw: &str) -> Vec { raw.split(',') .map(|raw_validator| { @@ -95,19 +89,6 @@ fn parse_args<'a>() -> ArgMatches<'a> { .short("r") .requires(MONITORING_ENABLED) ) - .arg( - Arg::with_name(V4_TOPOLOGY_ARG) - .help("location of .json file containing IPv4 'good' network topology") - .long(V4_TOPOLOGY_ARG) - .requires(MONITORING_ENABLED) - ) - .arg( - Arg::with_name(V6_TOPOLOGY_ARG) - .help("location of .json file containing IPv6 'good' network topology") - .long(V6_TOPOLOGY_ARG) - .takes_value(true) - .requires(MONITORING_ENABLED) - ) .arg( Arg::with_name(NYMD_VALIDATOR_ARG) .help("Endpoint to nymd part of the validator from which the monitor will grab nodes to test") @@ -125,12 +106,6 @@ fn parse_args<'a>() -> ArgMatches<'a> { .takes_value(true) .requires(REWARDING_ENABLED), ) - .arg( - Arg::with_name(DETAILED_REPORT_ARG) - .help("specifies whether a detailed report should be printed after each run") - .long(DETAILED_REPORT_ARG) - .requires(MONITORING_ENABLED) - ) .arg( Arg::with_name(WRITE_CONFIG_ARG) .help("specifies whether a config file based on provided arguments should be saved to a file") @@ -220,14 +195,6 @@ fn override_config(mut config: Config, matches: &ArgMatches) -> Config { config = config.with_rewarding_enabled(true) } - if let Some(v4_topology_path) = matches.value_of(V4_TOPOLOGY_ARG) { - config = config.with_v4_good_topology(v4_topology_path) - } - - if let Some(v6_topology_path) = matches.value_of(V6_TOPOLOGY_ARG) { - config = config.with_v6_good_topology(v6_topology_path) - } - if let Some(raw_validators) = matches.value_of(API_VALIDATORS_ARG) { config = config.with_custom_validator_apis(parse_validators(raw_validators)); } @@ -278,10 +245,6 @@ fn override_config(mut config: Config, matches: &ArgMatches) -> Config { config = config.with_minimum_epoch_monitor_threshold(monitor_threshold) } - if matches.is_present(DETAILED_REPORT_ARG) { - config = config.with_detailed_network_monitor_report(true) - } - #[cfg(feature = "coconut")] if let Some(keypair_path) = matches.value_of(KEYPAIR_ARG) { let keypair_bs58 = std::fs::read_to_string(keypair_path) @@ -329,6 +292,7 @@ fn setup_liftoff_notify(notify: Arc) -> AdHoc { fn setup_network_monitor<'a>( config: &'a Config, + system_version: &str, rocket: &Rocket, ) -> Option> { if !config.get_network_monitor_enabled() { @@ -339,14 +303,9 @@ fn setup_network_monitor<'a>( let node_status_storage = rocket.state::().unwrap().clone(); let validator_cache = rocket.state::().unwrap().clone(); - let v4_topology = parse_topology_file(config.get_v4_good_topology_file()); - let v6_topology = parse_topology_file(config.get_v6_good_topology_file()); - network_monitor::check_if_up_to_date(&v4_topology, &v6_topology); - Some(NetworkMonitorBuilder::new( config, - v4_topology, - v6_topology, + system_version, node_status_storage, validator_cache, )) @@ -417,6 +376,7 @@ async fn setup_rocket(config: &Config, liftoff_notify: Arc) -> Result Result<()> { setup_logging(); + let system_version = env!("CARGO_PKG_VERSION"); println!("Starting validator api..."); @@ -459,7 +419,7 @@ async fn main() -> Result<()> { // let's build our rocket! let rocket = setup_rocket(&config, Arc::clone(&liftoff_notify)).await?; - let monitor_builder = setup_network_monitor(&config, &rocket); + let monitor_builder = setup_network_monitor(&config, system_version, &rocket); let validator_cache = rocket.state::().unwrap().clone(); diff --git a/validator-api/src/network_monitor/gateways_reader.rs b/validator-api/src/network_monitor/gateways_reader.rs index 1938431cae..2d3b140885 100644 --- a/validator-api/src/network_monitor/gateways_reader.rs +++ b/validator-api/src/network_monitor/gateways_reader.rs @@ -75,7 +75,7 @@ impl Stream for GatewayChannel { // it is fine enough to use the same constant here as in the main GatewayReader // as only a single channel, i.e. the main gateway will be capable // of returning more than 2 values - if polled == YIELD_EVERY { + if polled >= YIELD_EVERY { cx.waker().wake_by_ref(); break; } diff --git a/validator-api/src/network_monitor/mod.rs b/validator-api/src/network_monitor/mod.rs index 019f16e9d0..e8d0437058 100644 --- a/validator-api/src/network_monitor/mod.rs +++ b/validator-api/src/network_monitor/mod.rs @@ -13,14 +13,10 @@ use crate::network_monitor::monitor::receiver::{ use crate::network_monitor::monitor::sender::PacketSender; use crate::network_monitor::monitor::summary_producer::SummaryProducer; use crate::network_monitor::monitor::Monitor; -use crate::network_monitor::tested_network::TestedNetwork; use crate::storage::ValidatorApiStorage; use crypto::asymmetric::{encryption, identity}; use futures::channel::mpsc; -use log::info; -use nymsphinx::addressing::clients::Recipient; use std::sync::Arc; -use topology::NymTopology; #[cfg(feature = "coconut")] use coconut_interface::Credential; @@ -31,11 +27,13 @@ pub(crate) mod chunker; pub(crate) mod gateways_reader; pub(crate) mod monitor; pub(crate) mod test_packet; -pub(crate) mod tested_network; +pub(crate) mod test_route; + +pub(crate) const ROUTE_TESTING_TEST_NONCE: u64 = 0; pub(crate) struct NetworkMonitorBuilder<'a> { config: &'a Config, - tested_network: TestedNetwork, + system_version: String, node_status_storage: ValidatorApiStorage, validator_cache: ValidatorCache, } @@ -43,29 +41,19 @@ pub(crate) struct NetworkMonitorBuilder<'a> { impl<'a> NetworkMonitorBuilder<'a> { pub(crate) fn new( config: &'a Config, - v4_topology: NymTopology, - v6_topology: NymTopology, + system_version: &str, node_status_storage: ValidatorApiStorage, validator_cache: ValidatorCache, ) -> Self { - let tested_network = TestedNetwork::new_good(v4_topology, v6_topology); - NetworkMonitorBuilder { config, - tested_network, + system_version: system_version.to_string(), node_status_storage, validator_cache, } } pub(crate) async fn build(self) -> NetworkMonitorRunnables { - // TODO: in the future I guess this should somehow change to distribute the load - let tested_mix_gateway = self.tested_network.main_v4_gateway().clone(); - info!( - "* gateway for testing mixnodes: {}", - tested_mix_gateway.identity_key.to_base58_string() - ); - // TODO: those keys change constant throughout the whole execution of the monitor. // and on top of that, they are used with ALL the gateways -> presumably this should change // in the future @@ -74,20 +62,14 @@ impl<'a> NetworkMonitorBuilder<'a> { let identity_keypair = Arc::new(identity::KeyPair::new(&mut rng)); let encryption_keypair = Arc::new(encryption::KeyPair::new(&mut rng)); - let test_mixnode_sender = Recipient::new( - *identity_keypair.public_key(), - *encryption_keypair.public_key(), - tested_mix_gateway.identity_key, - ); - let (gateway_status_update_sender, gateway_status_update_receiver) = mpsc::unbounded(); let (received_processor_sender_channel, received_processor_receiver_channel) = mpsc::unbounded(); let packet_preparer = new_packet_preparer( + &self.system_version, self.validator_cache, - self.tested_network.clone(), - test_mixnode_sender, + self.config.get_per_node_test_packets(), *identity_keypair.public_key(), *encryption_keypair.public_key(), ); @@ -109,7 +91,7 @@ impl<'a> NetworkMonitorBuilder<'a> { received_processor_receiver_channel, Arc::clone(&encryption_keypair), ); - let summary_producer = new_summary_producer(self.config.get_detailed_report()); + let summary_producer = new_summary_producer(self.config.get_per_node_test_packets()); let packet_receiver = new_packet_receiver( gateway_status_update_receiver, received_processor_sender_channel, @@ -122,7 +104,6 @@ impl<'a> NetworkMonitorBuilder<'a> { received_processor, summary_producer, self.node_status_storage, - self.tested_network, ); NetworkMonitorRunnables { @@ -150,16 +131,16 @@ impl NetworkMonitorRunnables { } fn new_packet_preparer( + system_version: &str, validator_cache: ValidatorCache, - tested_network: TestedNetwork, - test_mixnode_sender: Recipient, + per_node_test_packets: usize, self_public_identity: identity::PublicKey, self_public_encryption: encryption::PublicKey, ) -> PacketPreparer { PacketPreparer::new( + system_version, validator_cache, - tested_network, - test_mixnode_sender, + per_node_test_packets, self_public_identity, self_public_encryption, ) @@ -219,15 +200,10 @@ fn new_received_processor( ReceivedProcessor::new(packets_receiver, client_encryption_keypair) } -fn new_summary_producer(detailed_report: bool) -> SummaryProducer { +fn new_summary_producer(per_node_test_packets: usize) -> SummaryProducer { // right now always print the basic report. If we feel like we need to change it, it can // be easily adjusted by adding some flag or something - let summary_producer = SummaryProducer::default().with_report(); - if detailed_report { - summary_producer.with_detailed_report() - } else { - summary_producer - } + SummaryProducer::new(per_node_test_packets).with_report() } fn new_packet_receiver( @@ -236,46 +212,3 @@ fn new_packet_receiver( ) -> PacketReceiver { PacketReceiver::new(gateways_status_updater, processor_packets_sender) } - -pub(crate) fn check_if_up_to_date(v4_topology: &NymTopology, v6_topology: &NymTopology) { - let monitor_version = env!("CARGO_PKG_VERSION"); - for (_, layer_mixes) in v4_topology.mixes().iter() { - for mix in layer_mixes.iter() { - if !version_checker::is_minor_version_compatible(monitor_version, &*mix.version) { - panic!( - "Our good topology is not compatible with monitor! Mix runs {}, we have {}", - mix.version, monitor_version - ) - } - } - } - - for gateway in v4_topology.gateways().iter() { - if !version_checker::is_minor_version_compatible(monitor_version, &*gateway.version) { - panic!( - "Our good topology is not compatible with monitor! Gateway runs {}, we have {}", - gateway.version, monitor_version - ) - } - } - - for (_, layer_mixes) in v6_topology.mixes().iter() { - for mix in layer_mixes.iter() { - if !version_checker::is_minor_version_compatible(monitor_version, &*mix.version) { - panic!( - "Our good topology is not compatible with monitor! Mix runs {}, we have {}", - mix.version, monitor_version - ) - } - } - } - - for gateway in v6_topology.gateways().iter() { - if !version_checker::is_minor_version_compatible(monitor_version, &*gateway.version) { - panic!( - "Our good topology is not compatible with monitor! Gateway runs {}, we have {}", - gateway.version, monitor_version - ) - } - } -} diff --git a/validator-api/src/network_monitor/monitor/gateway_clients_cache.rs b/validator-api/src/network_monitor/monitor/gateway_clients_cache.rs new file mode 100644 index 0000000000..6d7f415912 --- /dev/null +++ b/validator-api/src/network_monitor/monitor/gateway_clients_cache.rs @@ -0,0 +1,94 @@ +// Copyright 2021 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crypto::asymmetric::identity::PUBLIC_KEY_LENGTH; +use gateway_client::GatewayClient; +use std::collections::HashMap; +use std::sync::Arc; +use tokio::sync::{Mutex, MutexGuard, TryLockError}; + +pub(crate) struct GatewayClientHandle(Arc); + +struct GatewayClientHandleInner { + client: Mutex>, + raw_identity: [u8; PUBLIC_KEY_LENGTH], +} + +pub(crate) struct UnlockedGatewayClientHandle<'a>(MutexGuard<'a, Option>); + +impl GatewayClientHandle { + pub(crate) fn new(gateway_client: GatewayClient) -> Self { + GatewayClientHandle(Arc::new(GatewayClientHandleInner { + raw_identity: gateway_client.gateway_identity().to_bytes(), + client: Mutex::new(Some(gateway_client)), + })) + } + + pub(crate) fn ptr_eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.0, &other.0) + } + + // this could have also been achieved with a normal #[derive(Clone)] but I prefer to be explicit about it, + // because clippy would suggest some potentially confusing 'simplifications' regarding clone + pub(crate) fn clone_data_pointer(&self) -> Self { + GatewayClientHandle(Arc::clone(&self.0)) + } + + pub(crate) fn raw_identity(&self) -> [u8; PUBLIC_KEY_LENGTH] { + self.0.raw_identity + } + + pub(crate) async fn is_invalid(&self) -> bool { + self.0.client.lock().await.is_none() + } + + pub(crate) async fn lock_client(&self) -> UnlockedGatewayClientHandle<'_> { + UnlockedGatewayClientHandle(self.0.client.lock().await) + } + + pub(crate) fn lock_client_unchecked(&self) -> UnlockedGatewayClientHandle<'_> { + UnlockedGatewayClientHandle(self.0.client.try_lock().unwrap()) + } + + pub(crate) fn try_lock_client(&self) -> Result, TryLockError> { + self.0.client.try_lock().map(UnlockedGatewayClientHandle) + } +} + +impl<'a> UnlockedGatewayClientHandle<'a> { + pub(crate) fn get_mut_unchecked(&mut self) -> &mut GatewayClient { + self.0.as_mut().unwrap() + } + + pub(crate) fn inner_mut(&mut self) -> Option<&mut GatewayClient> { + self.0.as_mut() + } + + pub(crate) fn invalidate(&mut self) { + *self.0 = None + } +} + +pub(crate) type GatewayClientsMap = HashMap<[u8; PUBLIC_KEY_LENGTH], GatewayClientHandle>; + +#[derive(Clone)] +pub(crate) struct ActiveGatewayClients { + // there is no point in using an RwLock here as there will only ever be two readers here and both + // potentially need write access. + // A BiLock would have been slightly better than a normal Mutex since it's optimised for two + // owners, but it's behind `unstable` feature flag in futures and it would be a headache if the API + // changed. + inner: Arc>, +} + +impl ActiveGatewayClients { + pub(crate) fn new() -> Self { + ActiveGatewayClients { + inner: Arc::new(Mutex::new(HashMap::new())), + } + } + + pub(crate) async fn lock(&self) -> MutexGuard<'_, GatewayClientsMap> { + self.inner.lock().await + } +} diff --git a/validator-api/src/network_monitor/monitor/gateways_pinger.rs b/validator-api/src/network_monitor/monitor/gateways_pinger.rs new file mode 100644 index 0000000000..b62c780db0 --- /dev/null +++ b/validator-api/src/network_monitor/monitor/gateways_pinger.rs @@ -0,0 +1,142 @@ +// Copyright 2021 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::network_monitor::monitor::gateway_clients_cache::ActiveGatewayClients; +use crate::network_monitor::monitor::receiver::{GatewayClientUpdate, GatewayClientUpdateSender}; +use crypto::asymmetric::identity; +use crypto::asymmetric::identity::PUBLIC_KEY_LENGTH; +use log::{debug, info, trace, warn}; +use std::time::Duration; +use tokio::time::{sleep, Instant}; + +// TODO: should it perhaps be moved to config along other timeout values? +const PING_TIMEOUT: Duration = Duration::from_secs(3); + +pub(crate) struct GatewayPinger { + gateway_clients: ActiveGatewayClients, + gateways_status_updater: GatewayClientUpdateSender, + pinging_interval: Duration, +} + +impl GatewayPinger { + pub(crate) fn new( + gateway_clients: ActiveGatewayClients, + gateways_status_updater: GatewayClientUpdateSender, + pinging_interval: Duration, + ) -> Self { + GatewayPinger { + gateway_clients, + gateways_status_updater, + pinging_interval, + } + } + + fn notify_connection_failure(&self, raw_gateway_id: [u8; PUBLIC_KEY_LENGTH]) { + // if this unwrap failed it means something extremely weird is going on + // and we got some solar flare bitflip type of corruption + let gateway_key = identity::PublicKey::from_bytes(&raw_gateway_id) + .expect("failed to recover gateways public key from valid bytes"); + + // remove the gateway listener channels + self.gateways_status_updater + .unbounded_send(GatewayClientUpdate::Failure(gateway_key)) + .expect("packet receiver seems to have died!"); + } + + async fn ping_and_cleanup_all_gateways(&self) { + info!(target: "GatewayPinger", "Pinging all active gateways"); + + let lock_acquire_start = Instant::now(); + let active_gateway_clients_guard = self.gateway_clients.lock().await; + trace!(target: "GatewayPinger", "Acquiring lock took {:?}", Instant::now().duration_since(lock_acquire_start)); + + if active_gateway_clients_guard.is_empty() { + debug!(target: "GatewayPinger", "no gateways to ping"); + return; + } + + // don't keep the guard the entire time - clone all Arcs and drop it + // + // this clippy warning is a false positive as we cannot get rid of the collect by moving + // everything into a single iterator as it would require us to hold the lock the entire time + // and that is exactly what we want to avoid + #[allow(clippy::needless_collect)] + let active_gateway_clients = active_gateway_clients_guard + .iter() + .map(|(_, handle)| handle.clone_data_pointer()) + .collect::>(); + drop(active_gateway_clients_guard); + + let ping_start = Instant::now(); + + let mut clients_to_purge = Vec::new(); + + // since we don't need to wait for response, we can just ping all gateways sequentially + // if it becomes problem later on, we can adjust it. + for client_handle in active_gateway_clients.into_iter() { + trace!( + "Pinging: {}", + identity::PublicKey::from_bytes(&client_handle.raw_identity()) + .unwrap() + .to_base58_string() + ); + // if we fail to obtain the lock it means the client is being currently used to send messages + // and hence we don't need to ping it to keep connection alive + if let Ok(mut unlocked_handle) = client_handle.try_lock_client() { + if let Some(active_client) = unlocked_handle.inner_mut() { + match tokio::time::timeout(PING_TIMEOUT, active_client.send_ping_message()) + .await + { + Err(_timeout) => { + warn!( + target: "GatewayPinger", + "we timed out trying to ping {} - assuming the connection is dead.", + active_client.gateway_identity().to_base58_string(), + ); + clients_to_purge.push(client_handle.raw_identity()); + } + Ok(Err(err)) => { + warn!( + target: "GatewayPinger", + "failed to send ping message to gateway {} - {} - assuming the connection is dead.", + active_client.gateway_identity().to_base58_string(), + err, + ); + clients_to_purge.push(client_handle.raw_identity()); + } + _ => {} + } + } else { + clients_to_purge.push(client_handle.raw_identity()); + } + } + } + + // purge all dead connections + // reacquire the guard + let lock_acquire_start = Instant::now(); + let mut active_gateway_clients_guard = self.gateway_clients.lock().await; + trace!(target: "GatewayPinger", "Acquiring lock took {:?}", Instant::now().duration_since(lock_acquire_start)); + + for gateway_id in clients_to_purge.into_iter() { + if let Some(removed_handle) = active_gateway_clients_guard.remove(&gateway_id) { + if !removed_handle.is_invalid().await { + // it was not invalidated by the packet sender meaning it probably was some unbonded node + // that was never cleared + self.notify_connection_failure(gateway_id); + } + } + } + + let ping_end = Instant::now(); + let time_taken = ping_end.duration_since(ping_start); + debug!(target: "GatewayPinger", "Pinging all active gateways took {:?}", time_taken); + } + + pub(crate) async fn run(&self) { + loop { + sleep(self.pinging_interval).await; + self.ping_and_cleanup_all_gateways().await + } + } +} diff --git a/validator-api/src/network_monitor/monitor/mod.rs b/validator-api/src/network_monitor/monitor/mod.rs index c4246c4100..6b00caea72 100644 --- a/validator-api/src/network_monitor/monitor/mod.rs +++ b/validator-api/src/network_monitor/monitor/mod.rs @@ -2,17 +2,20 @@ // SPDX-License-Identifier: Apache-2.0 use crate::config::Config; -use crate::network_monitor::monitor::preparer::{PacketPreparer, TestedNode}; +use crate::network_monitor::monitor::preparer::PacketPreparer; use crate::network_monitor::monitor::processor::ReceivedProcessor; use crate::network_monitor::monitor::sender::PacketSender; -use crate::network_monitor::monitor::summary_producer::{NodeResult, SummaryProducer, TestReport}; -use crate::network_monitor::test_packet::NodeType; -use crate::network_monitor::tested_network::TestedNetwork; +use crate::network_monitor::monitor::summary_producer::{SummaryProducer, TestSummary}; +use crate::network_monitor::test_packet::TestPacket; +use crate::network_monitor::test_route::TestRoute; use crate::storage::ValidatorApiStorage; -use log::{debug, error, info, warn}; +use log::{debug, error, info}; +use std::collections::{HashMap, HashSet}; use std::process; use tokio::time::{sleep, Duration, Instant}; +pub(crate) mod gateway_clients_cache; +pub(crate) mod gateways_pinger; pub(crate) mod preparer; pub(crate) mod processor; pub(crate) mod receiver; @@ -20,16 +23,25 @@ pub(crate) mod sender; pub(crate) mod summary_producer; pub(super) struct Monitor { - nonce: u64, + test_nonce: u64, packet_preparer: PacketPreparer, packet_sender: PacketSender, received_processor: ReceivedProcessor, summary_producer: SummaryProducer, node_status_storage: ValidatorApiStorage, - tested_network: TestedNetwork, run_interval: Duration, gateway_ping_interval: Duration, packet_delivery_timeout: Duration, + + /// Number of test packets sent via each "random" route to verify whether they work correctly. + route_test_packets: usize, + + /// Desired number of test routes to be constructed (and working) during a monitor test run. + test_routes: usize, + + /// The minimum number of test routes that need to be constructed (and working) in order for + /// a monitor test run to be valid. + minimum_test_routes: usize, } impl Monitor { @@ -40,47 +52,41 @@ impl Monitor { received_processor: ReceivedProcessor, summary_producer: SummaryProducer, node_status_storage: ValidatorApiStorage, - tested_network: TestedNetwork, ) -> Self { Monitor { - nonce: 1, + test_nonce: 1, packet_preparer, packet_sender, received_processor, summary_producer, node_status_storage, - tested_network, run_interval: config.get_network_monitor_run_interval(), gateway_ping_interval: config.get_gateway_ping_interval(), packet_delivery_timeout: config.get_packet_delivery_timeout(), + route_test_packets: config.get_route_test_packets(), + test_routes: config.get_test_routes(), + minimum_test_routes: config.get_minimum_test_routes(), } } // while it might have been cleaner to put this into a separate `Notifier` structure, // I don't see much point considering it's only a single, small, method - async fn submit_new_node_statuses( - &self, - mixnode_results: Vec, - gateway_results: Vec, - ) { - if let Err(err) = self - .node_status_storage - .submit_new_statuses(mixnode_results, gateway_results) - .await - { - // this can only fail if there's an issue with the database - we can't really recover - error!( - "Failed to submit new monitoring results to the database - {}", - err - ); - - // TODO: slightly more graceful shutdown here - process::exit(1); - } - + async fn submit_new_node_statuses(&self, test_summary: TestSummary) { // indicate our run has completed successfully and should be used in any future // uptime calculations - if let Err(err) = self.node_status_storage.insert_monitor_run().await { + if let Err(err) = self + .node_status_storage + .insert_monitor_run_results( + test_summary.mixnode_results, + test_summary.gateway_results, + test_summary + .route_results + .into_iter() + .map(|result| result.route) + .collect(), + ) + .await + { error!( "Failed to submit monitor run information to the database - {}", err @@ -91,79 +97,159 @@ impl Monitor { } } - // checking it this way with a TestReport is rather suboptimal but given the fact we're only - // doing this fewer than 10 times, it's not that problematic - fn check_good_nodes_status(&self, report: &TestReport) -> bool { - let mut good_nodes_status = true; - for v4_mixes in self.tested_network.v4_topology().mixes().values() { - for v4_mix in v4_mixes { - let node = &TestedNode { - identity: v4_mix.identity_key.to_base58_string(), - owner: v4_mix.owner.clone(), - node_type: NodeType::Mixnode, - }; - if !report.fully_working_mixes.contains(node) { - warn!("Mixnode {} has not passed the ipv4 check", node.identity); - good_nodes_status = false; - } - } + fn analyse_received_test_route_packets(&self, packets: &[TestPacket]) -> HashMap { + let mut received = HashMap::new(); + for packet in packets { + *received.entry(packet.route_id).or_insert(0usize) += 1usize } - for v4_gateway in self.tested_network.v4_topology().gateways() { - let node = &TestedNode { - identity: v4_gateway.identity_key.to_base58_string(), - owner: v4_gateway.owner.clone(), - node_type: NodeType::Gateway, - }; - if !report.fully_working_gateways.contains(node) { - warn!("Gateway {} has not passed the ipv4 check", node.identity); - good_nodes_status = false; - } - } - - for v6_mixes in self.tested_network.v6_topology().mixes().values() { - for v6_mix in v6_mixes { - let node = &TestedNode { - identity: v6_mix.identity_key.to_base58_string(), - owner: v6_mix.owner.clone(), - node_type: NodeType::Mixnode, - }; - if !report.fully_working_mixes.contains(node) { - warn!("Mixnode {} has not passed the ipv6 check", node.identity); - good_nodes_status = false; - } - } - } - - for v6_gateway in self.tested_network.v6_topology().gateways() { - let node = &TestedNode { - identity: v6_gateway.identity_key.to_base58_string(), - owner: v6_gateway.owner.clone(), - node_type: NodeType::Gateway, - }; - if !report.fully_working_gateways.contains(node) { - warn!("Gateway {} has not passed the ipv6 check", node.identity); - good_nodes_status = false; - } - } - - good_nodes_status + received } - async fn test_run(&mut self) { - info!(target: "Monitor", "Starting test run no. {}", self.nonce); + async fn test_chosen_test_routes(&mut self, routes: &[TestRoute]) -> HashMap { + // notes for the future improvements: + /* + - gateway authentication failure should only 'blacklist' gateways, not mixnodes - debug!(target: "Monitor", "Preparing mix packets to all nodes..."); - let prepared_packets = self.packet_preparer.prepare_test_packets(self.nonce).await; + */ - self.received_processor.set_new_expected(self.nonce).await; + if routes.is_empty() { + return HashMap::new(); + } - info!(target: "Monitor", "Starting to send all the packets..."); + debug!("Testing the following test routes: {:#?}", routes); + + let mut packets = Vec::with_capacity(routes.len()); + for route in routes { + packets.push( + self.packet_preparer + .prepare_test_route_viability_packets(route, self.route_test_packets) + .await, + ); + } + + self.received_processor.set_route_test_nonce().await; + self.packet_sender.send_packets(packets).await; + + // give the packets some time to traverse the network + sleep(self.packet_delivery_timeout).await; + + let received = self.received_processor.return_received().await; + let mut results = self.analyse_received_test_route_packets(&received); + + // create entry for routes that might have not forwarded a single packet + for route in routes { + results.entry(route.id()).or_insert(0); + } + + for entry in results.iter() { + if *entry.1 == self.route_test_packets { + debug!("✔️ {} succeeded", entry.0) + } else { + debug!( + "❌️ {} failed ({}/{} received)", + entry.0, entry.1, self.route_test_packets + ) + } + } + + results + .into_iter() + .map(|(k, v)| (k, v == self.route_test_packets)) + .collect() + } + + fn blacklist_route_nodes(&self, route: &TestRoute, blacklist: &mut HashSet) { + for mix in route.topology().mixes_as_vec() { + blacklist.insert(mix.identity_key.to_base58_string()); + } + blacklist.insert(route.gateway_identity().to_base58_string()); + } + + async fn prepare_test_routes(&mut self) -> Option> { + info!(target: "Monitor", "Generating test routes..."); + + // keep track of nodes that should not be used for route construction + let mut blacklist = HashSet::new(); + let mut verified_routes = Vec::new(); + let mut remaining = self.test_routes; + let mut current_attempt = 0; + + // todo: tweak this to something more appropriate + let max_attempts = self.test_routes * 2; + + 'outer: loop { + if current_attempt >= max_attempts { + if verified_routes.len() >= self.minimum_test_routes { + return Some(verified_routes); + } + return None; + } + + // try to construct slightly more than what we actually need to more quickly reach + // the actual target + let candidates = match self + .packet_preparer + .prepare_test_routes(remaining * 2, &mut blacklist) + .await + { + Some(candidates) => candidates, + // if there are no more routes to generate, see if we have managed to construct + // at least the minimum number of routes + None => { + if verified_routes.len() >= self.minimum_test_routes { + return Some(verified_routes); + } + return None; + } + }; + let results = self.test_chosen_test_routes(&candidates).await; + for candidate in candidates { + // ideally we would blacklist all nodes regardless of the result so we would not use them anymore + // however, currently we have huge imbalance of gateways to mixnodes so we might accidentally + // discard working gateway because it was paired with broken mixnode + if *results.get(&candidate.id()).unwrap() { + // if the path is fully working, blacklist those nodes so we wouldn't construct + // any other path through any of those nodes + self.blacklist_route_nodes(&candidate, &mut blacklist); + + verified_routes.push(candidate); + if verified_routes.len() == self.test_routes { + break 'outer; + } + } + } + + remaining = self.test_routes - verified_routes.len(); + current_attempt += 1; + } + + Some(verified_routes) + } + + async fn test_network_against(&mut self, routes: &[TestRoute]) { + info!("Generating test mix packets for all the network nodes..."); + let prepared_packets = self + .packet_preparer + .prepare_test_packets(self.test_nonce, routes) + .await; + + let total_sent = prepared_packets + .packets + .iter() + .flat_map(|packets| packets.packets.iter()) + .count(); + + self.received_processor + .set_new_test_nonce(self.test_nonce) + .await; + + debug!("Sending packets to all gateways..."); self.packet_sender .send_packets(prepared_packets.packets) .await; - info!( + debug!( target: "Monitor", "Sending is over, waiting for {:?} before checking what we received", self.packet_delivery_timeout @@ -173,29 +259,37 @@ impl Monitor { sleep(self.packet_delivery_timeout).await; let received = self.received_processor.return_received().await; + let total_received = received.len(); - let test_summary = self.summary_producer.produce_summary( - prepared_packets.tested_nodes, + let summary = self.summary_producer.produce_summary( + prepared_packets.tested_mixnodes, + prepared_packets.tested_gateways, received, - prepared_packets.invalid_nodes, + prepared_packets.invalid_mixnodes, + prepared_packets.invalid_gateways, + routes, ); - // our "good" nodes MUST be working correctly otherwise we cannot trust the results - if self.check_good_nodes_status(&test_summary.test_report) { - self.submit_new_node_statuses( - test_summary.mixnode_results, - test_summary.gateway_results, - ) - .await; - } else { - error!("our own 'good' nodes did not pass the check - we are not going to submit results to the node status API"); - } + let report = summary.create_report(total_sent, total_received); + info!("{}", report); - self.nonce += 1; + self.submit_new_node_statuses(summary).await; } - async fn ping_all_gateways(&mut self) { - self.packet_sender.ping_all_active_gateways().await; + async fn test_run(&mut self) { + info!(target: "Monitor", "Starting test run no. {}", self.test_nonce); + let start = Instant::now(); + + if let Some(test_routes) = self.prepare_test_routes().await { + debug!(target: "Monitor", "Determined reliable routes to test all other nodes against. : {:?}", test_routes); + self.test_network_against(&test_routes).await; + } else { + error!("We failed to construct sufficient number of test routes to test the network against") + } + + debug!("Test run took {:?}", Instant::now().duration_since(start)); + + self.test_nonce += 1; } pub(crate) async fn run(&mut self) { @@ -203,36 +297,16 @@ impl Monitor { // wait for validator cache to be ready self.packet_preparer - .wait_for_validator_cache_initial_values() + .wait_for_validator_cache_initial_values(self.test_routes) .await; - // start from 0 to run test immediately on startup - let test_delay = sleep(Duration::from_secs(0)); - tokio::pin!(test_delay); - - let ping_delay = sleep(self.gateway_ping_interval); - tokio::pin!(ping_delay); + self.packet_sender + .spawn_gateways_pinger(self.gateway_ping_interval); + let mut run_interval = tokio::time::interval(self.run_interval); loop { - tokio::select! { - _ = &mut test_delay => { - self.test_run().await; - info!(target: "Monitor", "Next test run will happen in {:?}", self.run_interval); - - let now = Instant::now(); - test_delay.as_mut().reset(now + self.run_interval); - // since we just sent packets through gateways, there's no need to ping them - ping_delay.as_mut().reset(now + self.gateway_ping_interval); - - } - _ = &mut ping_delay => { - info!(target: "Monitor", "Pinging all active gateways"); - self.ping_all_gateways().await; - - let now = Instant::now(); - ping_delay.as_mut().reset(now + self.gateway_ping_interval); - } - } + run_interval.tick().await; + self.test_run().await; } } } diff --git a/validator-api/src/network_monitor/monitor/preparer.rs b/validator-api/src/network_monitor/monitor/preparer.rs index 1a4fb63f99..7a2f838055 100644 --- a/validator-api/src/network_monitor/monitor/preparer.rs +++ b/validator-api/src/network_monitor/monitor/preparer.rs @@ -5,61 +5,62 @@ use crate::cache::ValidatorCache; use crate::network_monitor::chunker::Chunker; use crate::network_monitor::monitor::sender::GatewayPackets; use crate::network_monitor::test_packet::{NodeType, TestPacket}; -use crate::network_monitor::tested_network::TestedNetwork; +use crate::network_monitor::test_route::TestRoute; use crypto::asymmetric::{encryption, identity}; -use log::{info, warn}; -use mixnet_contract::{GatewayBond, MixNodeBond}; +use log::info; +use mixnet_contract::{Addr, GatewayBond, Layer, MixNodeBond}; use nymsphinx::addressing::clients::Recipient; use nymsphinx::forwarding::packet::MixPacket; +use rand::seq::SliceRandom; +use rand::{thread_rng, Rng}; +use std::collections::{HashMap, HashSet}; use std::convert::TryInto; use std::fmt::{self, Display, Formatter}; use std::time::Duration; -use topology::{gateway, mix}; +use topology::{gateway, mix, NymTopology}; // declared type aliases for easier code reasoning type Version = String; type Id = String; -type Owner = String; +type Owner = Addr; #[derive(Clone)] pub(crate) enum InvalidNode { - OutdatedMix(Id, Owner, Version), - MalformedMix(Id, Owner), - OutdatedGateway(Id, Owner, Version), - MalformedGateway(Id, Owner), + Outdated(Id, Owner, Version), + Malformed(Id, Owner), } impl Display for InvalidNode { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { match self { - InvalidNode::OutdatedMix(id, owner, version) => { + InvalidNode::Outdated(id, owner, version) => { write!( f, - "Mixnode {} (v {}) owned by {} is outdated", + "Node {} (v{}) owned by {} is outdated", id, version, owner ) } - InvalidNode::MalformedMix(id, owner) => { - write!(f, "Mixnode {} owner by {} is malformed", id, owner) - } - InvalidNode::OutdatedGateway(id, owner, version) => { - write!( - f, - "Gateway {} (v {}) owned by {} is outdated", - id, version, owner - ) - } - InvalidNode::MalformedGateway(id, owner) => { - write!(f, "Gateway {} owned by {} is malformed", id, owner) + InvalidNode::Malformed(id, owner) => { + write!(f, "Node {} owned by {} is malformed", id, owner) } } } } -enum PreparedNode { - TestedGateway(gateway::Node, [TestPacket; 2]), - TestedMix(mix::Node, [TestPacket; 2]), - Invalid(InvalidNode), +impl InvalidNode { + pub(crate) fn identity(&self) -> String { + match self { + InvalidNode::Outdated(id, _, _) => id.clone(), + InvalidNode::Malformed(id, _) => id.clone(), + } + } + + pub(crate) fn owner(&self) -> String { + match self { + InvalidNode::Outdated(_, owner, _) => owner.into(), + InvalidNode::Malformed(_, owner) => owner.into(), + } + } } #[derive(Eq, PartialEq, Debug, Hash, Clone)] @@ -69,50 +70,24 @@ pub(crate) struct TestedNode { pub(crate) node_type: NodeType, } -impl TestedNode { - pub(crate) fn new_mix(identity: String, owner: String) -> Self { +impl<'a> From<&'a mix::Node> for TestedNode { + fn from(node: &'a mix::Node) -> Self { TestedNode { - identity, - owner, + identity: node.identity_key.to_base58_string(), + owner: node.owner.clone(), node_type: NodeType::Mixnode, } } +} - pub(crate) fn new_gateway(identity: String, owner: String) -> Self { +impl<'a> From<&'a gateway::Node> for TestedNode { + fn from(node: &'a gateway::Node) -> Self { TestedNode { - identity, - owner, + identity: node.identity_key.to_base58_string(), + owner: node.owner.clone(), node_type: NodeType::Gateway, } } - - pub(crate) fn from_raw_mix(identity: S1, owner: S2) -> Self - where - S1: Into, - S2: Into, - { - TestedNode { - identity: identity.into(), - owner: owner.into(), - node_type: NodeType::Mixnode, - } - } - - pub(crate) fn from_raw_gateway(identity: S1, owner: S2) -> Self - where - S1: Into, - S2: Into, - { - TestedNode { - identity: identity.into(), - owner: owner.into(), - node_type: NodeType::Gateway, - } - } - - pub(crate) fn is_gateway(&self) -> bool { - self.node_type == NodeType::Gateway - } } impl Display for TestedNode { @@ -126,216 +101,259 @@ pub(crate) struct PreparedPackets { /// which they ought to be sent. pub(super) packets: Vec, - /// Vector containing list of public keys and owners of all nodes (mixnodes and gateways) being tested. - /// We do not need to specify the exact test parameters as for each of them we expect to receive - /// two packets back: ipv4 and ipv6 regardless of which gateway they originate. - pub(super) tested_nodes: Vec, + /// Vector containing list of public keys and owners of all nodes mixnodes being tested. + pub(super) tested_mixnodes: Vec, - /// All nodes that failed to get parsed correctly. They will be marked to the validator as being - /// down on ipv4 and ipv6. - pub(super) invalid_nodes: Vec, + /// Vector containing list of public keys and owners of all gateways being tested. + pub(super) tested_gateways: Vec, + + /// All mixnodes that failed to get parsed correctly or were not version compatible. + /// They will be marked to the validator as being down for the test. + pub(super) invalid_mixnodes: Vec, + + /// All gateways that failed to get parsed correctly or were not version compatible. + /// They will be marked to the validator as being down for the test. + pub(super) invalid_gateways: Vec, } pub(crate) struct PacketPreparer { - chunker: Chunker, + system_version: String, + chunker: Option, validator_cache: ValidatorCache, - tested_network: TestedNetwork, - // currently all test MIXNODE packets are sent via the same gateway - test_mixnode_sender: Recipient, + /// Number of test packets sent to each node + per_node_test_packets: usize, - // keys required to create sender of any other gateway + // TODO: security: + // in the future we should really create unique set of keys every time otherwise + // gateways might recognise our "test" keys and take special care to always forward those packets + // even if otherwise they are malicious. self_public_identity: identity::PublicKey, self_public_encryption: encryption::PublicKey, } impl PacketPreparer { pub(crate) fn new( + system_version: &str, validator_cache: ValidatorCache, - tested_network: TestedNetwork, - test_mixnode_sender: Recipient, + per_node_test_packets: usize, self_public_identity: identity::PublicKey, self_public_encryption: encryption::PublicKey, ) -> Self { PacketPreparer { - chunker: Chunker::new(test_mixnode_sender), + system_version: system_version.to_owned(), + chunker: None, validator_cache, - tested_network, - test_mixnode_sender, + per_node_test_packets, self_public_identity, self_public_encryption, } } - pub(crate) async fn wait_for_validator_cache_initial_values(&self) { + async fn wrap_test_packet( + &mut self, + packet: &TestPacket, + topology: &NymTopology, + packet_recipient: Recipient, + ) -> MixPacket { + // this should be done only once. We can't really do it at construction time + // as there's no sane Default for Recipient + if self.chunker.is_none() { + self.chunker = Some(Chunker::new(packet_recipient)); + } + let mut mix_packets = self + .chunker + .as_mut() + .unwrap() + .prepare_packets_from(packet.to_bytes(), topology, packet_recipient) + .await; + assert_eq!( + mix_packets.len(), + 1, + "Our test packets data is longer than a single sphinx packet!" + ); + + mix_packets.pop().unwrap() + } + + pub(crate) async fn wait_for_validator_cache_initial_values(&self, minimum_full_routes: usize) { // wait for the cache to get initialised self.validator_cache.wait_for_initial_values().await; - // now wait for our "good" topology to be online - info!("Waiting for 'good' topology to be online"); + // now wait for at least `minimum_full_routes` mixnodes per layer and `minimum_full_routes` gateway to be online + info!("Waiting for minimal topology to be online"); let initialisation_backoff = Duration::from_secs(30); loop { let gateways = self.validator_cache.gateways().await; - let mixnodes = self.validator_cache.mixnodes().await; - if self - .tested_network - .is_online(&mixnodes.into_inner(), &gateways.into_inner()) - { - break; - } else { - info!( - "Our 'good' topology is still not offline. Going to check again in {:?}", - initialisation_backoff - ); - tokio::time::sleep(initialisation_backoff).await; + let mixnodes = self.validator_cache.active_mixnodes().await; + + if let Some(mixnodes) = mixnodes { + if gateways.into_inner().len() < minimum_full_routes { + continue; + } + + let mut layered_mixes = HashMap::new(); + for mix in mixnodes.into_inner() { + let layer = mix.layer; + let mixes = layered_mixes.entry(layer).or_insert_with(Vec::new); + mixes.push(mix) + } + + // we remove the entries as this gives us the ownership and thus we can unwrap to default value + // which makes the code slightly nicer without having to deal with options + let layer1 = layered_mixes.remove(&Layer::One).unwrap_or_default(); + let layer2 = layered_mixes.remove(&Layer::Two).unwrap_or_default(); + let layer3 = layered_mixes.remove(&Layer::Three).unwrap_or_default(); + + if layer1.len() >= minimum_full_routes + && layer2.len() >= minimum_full_routes + && layer3.len() >= minimum_full_routes + { + break; + } } + + info!( + "Minimal topology is still not offline. Going to check again in {:?}", + initialisation_backoff + ); + tokio::time::sleep(initialisation_backoff).await; } } async fn get_network_nodes(&self) -> (Vec, Vec) { - info!(target: "Monitor", "Obtaining network topology..."); - let mixnodes = self.validator_cache.mixnodes().await.into_inner(); let gateways = self.validator_cache.gateways().await.into_inner(); - info!(target: "Monitor", "Obtained network topology"); - (mixnodes, gateways) } - fn check_version_compatibility(&self, mix_version: &str) -> bool { - let semver_compatibility = version_checker::is_minor_version_compatible( - mix_version, - self.tested_network.system_version(), - ); + pub(crate) fn try_parse_mix_bond(&self, mix: &MixNodeBond) -> Result { + let identity = mix.mix_node.identity_key.clone(); + if !self.check_version_compatibility(&mix.mix_node.version) { + return Err(identity); + } + mix.try_into().map_err(|_| identity) + } - if semver_compatibility { - // this can't fail as we know it's semver compatible - let version = version_checker::parse_version(mix_version).unwrap(); + pub(crate) fn try_parse_gateway_bond( + &self, + gateway: &GatewayBond, + ) -> Result { + let identity = gateway.gateway.identity_key.clone(); + if !self.check_version_compatibility(&gateway.gateway.version) { + return Err(identity); + } + gateway.try_into().map_err(|_| identity) + } - // check if it's at least 0.9.2 - reject anything below it due to significant bugs present - // if it's 1.Y.Z it's definitely >= 0.9.2 - if version.major >= 1 { - return true; + // gets active nodes + // chooses n random nodes from each layer (and gateway) such that they are not on the blacklist + // if failed to parsed => onto the blacklist they go + // if generated fewer than n, blacklist will be updated by external function with correctly generated + // routes so that they wouldn't be reused + pub(crate) async fn prepare_test_routes( + &self, + n: usize, + blacklist: &mut HashSet, + ) -> Option> { + let active_mixnodes = self.validator_cache.active_mixnodes().await?.into_inner(); + let gateways = self.validator_cache.gateways().await.into_inner(); + + // separate mixes into layers for easier selection + let mut layered_mixes = HashMap::new(); + for active_mix in active_mixnodes { + // filter out mixes on the blacklist + if blacklist.contains(&active_mix.mix_node.identity_key) { + continue; } - // if it's 0.10.Z it's definitely >= 0.9.2 - if version.major == 0 && version.minor >= 10 { - return true; - } - // if it's 0.9.Z, ensure Z >= 2 - version.minor == 9 && version.patch >= 2 + let layer = active_mix.layer; + let mixes = layered_mixes.entry(layer).or_insert_with(Vec::new); + mixes.push(active_mix) + } + // filter out gateways on the blacklist + let gateways = gateways + .into_iter() + .filter(|gateway| !blacklist.contains(&gateway.gateway.identity_key)) + .collect::>(); + + // get all nodes from each layer... + let l1 = layered_mixes.get(&Layer::One)?; + let l2 = layered_mixes.get(&Layer::Two)?; + let l3 = layered_mixes.get(&Layer::Three)?; + + // try to choose n nodes from each of them (+ gateways)... + let mut rng = thread_rng(); + + let rand_l1 = l1.choose_multiple(&mut rng, n).collect::>(); + let rand_l2 = l2.choose_multiple(&mut rng, n).collect::>(); + let rand_l3 = l3.choose_multiple(&mut rng, n).collect::>(); + let rand_gateways = gateways.choose_multiple(&mut rng, n).collect::>(); + + // the unwrap on `min()` is fine as we know the iterator is not empty + let most_available = *[ + rand_l1.len(), + rand_l2.len(), + rand_l3.len(), + rand_gateways.len(), + ] + .iter() + .min() + .unwrap(); + + if most_available == 0 { + // it's impossible to generate a single route + None } else { - false - } - } + let mut routes = Vec::new(); + for i in 0..most_available { + let node_1 = match self.try_parse_mix_bond(rand_l1[i]) { + Ok(node) => node, + Err(id) => { + blacklist.insert(id); + continue; + } + }; - fn mix_into_prepared_node(&self, nonce: u64, mixnode_bond: &MixNodeBond) -> PreparedNode { - if !self.check_version_compatibility(&mixnode_bond.mix_node().version) { - return PreparedNode::Invalid(InvalidNode::OutdatedMix( - mixnode_bond.mix_node().identity_key.clone(), - mixnode_bond.owner.to_string(), - mixnode_bond.mix_node().version.clone(), - )); - } - match TryInto::::try_into(mixnode_bond) { - Ok(mix) => { - let v4_packet = TestPacket::new_v4( - mix.identity_key, - mix.owner.clone(), - nonce, - NodeType::Mixnode, - ); - let v6_packet = TestPacket::new_v6( - mix.identity_key, - mix.owner.clone(), - nonce, - NodeType::Mixnode, - ); - PreparedNode::TestedMix(mix, [v4_packet, v6_packet]) - } - Err(err) => { - warn!( - target: "Bad node", - "Mix {} is malformed - {}", - mixnode_bond.mix_node().identity_key, - err - ); - PreparedNode::Invalid(InvalidNode::MalformedMix( - mixnode_bond.mix_node().identity_key.clone(), - mixnode_bond.owner.to_string(), + let node_2 = match self.try_parse_mix_bond(rand_l2[i]) { + Ok(node) => node, + Err(id) => { + blacklist.insert(id); + continue; + } + }; + + let node_3 = match self.try_parse_mix_bond(rand_l3[i]) { + Ok(node) => node, + Err(id) => { + blacklist.insert(id); + continue; + } + }; + + let gateway = match self.try_parse_gateway_bond(rand_gateways[i]) { + Ok(node) => node, + Err(id) => { + blacklist.insert(id); + continue; + } + }; + + routes.push(TestRoute::new( + rng.gen(), + &self.system_version, + node_1, + node_2, + node_3, + gateway, )) } + Some(routes) } } - fn gateway_into_prepared_node(&self, nonce: u64, gateway_bond: &GatewayBond) -> PreparedNode { - if !self.check_version_compatibility(&gateway_bond.gateway().version) { - return PreparedNode::Invalid(InvalidNode::OutdatedGateway( - gateway_bond.gateway().identity_key.clone(), - gateway_bond.owner.to_string(), - gateway_bond.gateway().version.clone(), - )); - } - match TryInto::::try_into(gateway_bond) { - Ok(gateway) => { - let v4_packet = TestPacket::new_v4( - gateway.identity_key, - gateway.owner.clone(), - nonce, - NodeType::Gateway, - ); - let v6_packet = TestPacket::new_v6( - gateway.identity_key, - gateway.owner.clone(), - nonce, - NodeType::Gateway, - ); - PreparedNode::TestedGateway(gateway, [v4_packet, v6_packet]) - } - Err(err) => { - warn!( - target: "Bad node", - "gateway {} is malformed - {:?}", - gateway_bond.gateway().identity_key, - err - ); - PreparedNode::Invalid(InvalidNode::MalformedGateway( - gateway_bond.gateway().identity_key.clone(), - gateway_bond.owner.to_string(), - )) - } - } - } - - fn prepare_mixnodes(&self, nonce: u64, nodes: &[MixNodeBond]) -> Vec { - nodes - .iter() - .map(|mix| self.mix_into_prepared_node(nonce, mix)) - .collect() - } - - fn prepare_gateways(&self, nonce: u64, nodes: &[GatewayBond]) -> Vec { - nodes - .iter() - .map(|gateway| self.gateway_into_prepared_node(nonce, gateway)) - .collect() - } - - fn tested_nodes(&self, nodes: &[PreparedNode]) -> Vec { - nodes - .iter() - .filter_map(|node| match node { - PreparedNode::TestedGateway(gateway, _) => Some(TestedNode::new_gateway( - gateway.identity_key.to_base58_string(), - gateway.owner.clone(), - )), - PreparedNode::TestedMix(mix, _) => Some(TestedNode::new_mix( - mix.identity_key.to_base58_string(), - mix.owner.clone(), - )), - PreparedNode::Invalid(..) => None, - }) - .collect() + fn check_version_compatibility(&self, node_version: &str) -> bool { + version_checker::is_minor_version_compatible(node_version, &self.system_version) } fn create_packet_sender(&self, gateway: &gateway::Node) -> Recipient { @@ -346,137 +364,163 @@ impl PacketPreparer { ) } - async fn create_mixnode_mix_packets( + pub(crate) async fn prepare_test_route_viability_packets( &mut self, - mixes: Vec, - invalid: &mut Vec, - ) -> Vec { - // all of the mixnode mix packets are going to get sent via our one 'main' gateway - // TODO: in the future this should probably be changed... - - // this might be slightly overestimating the number of packets we are going to produce, - // however, it should be negligible as we don't expect to ever see high number of - // invalid nodes (and even if we do see them, they shouldn't persist for long) - let mut packets = Vec::with_capacity(mixes.len() * 2); - - for mix in mixes.into_iter() { - match mix { - PreparedNode::TestedMix(node, test_packets) => { - for test_packet in test_packets.iter() { - let topology_to_test = self - .tested_network - .substitute_mix(node.clone(), test_packet.ip_version()); - let mix_message = test_packet.to_bytes(); - let mut mix_packet = self - .chunker - .prepare_packets_from( - mix_message, - &topology_to_test, - self.test_mixnode_sender, - ) - .await; - debug_assert_eq!(mix_packet.len(), 1); - packets.push(mix_packet.pop().unwrap()); - } - } - PreparedNode::Invalid(node) => invalid.push(node), - // `prepare_mixnodes` should NEVER return prepared gateways - _ => unreachable!(), - } - } - packets - } - - async fn create_gateway_mix_packets( - &mut self, - gateways: Vec, - invalid: &mut Vec, - ) -> Vec { - // again, there might be a slight overestimation here, but in the grand scheme of things - // it will be negligible - let mut packets = Vec::with_capacity(gateways.len()); - - // unfortunately this can't be done more cleanly with iterators as we require an async call - for gateway in gateways.into_iter() { - match gateway { - PreparedNode::TestedGateway(node, test_packets) => { - let mut gateway_packets = Vec::with_capacity(2); - for test_packet in test_packets.iter() { - let packet_sender = self.create_packet_sender(&node); - let topology_to_test = self - .tested_network - .substitute_gateway(node.clone(), test_packet.ip_version()); - let mix_message = test_packet.to_bytes(); - let mut mix_packet = self - .chunker - .prepare_packets_from(mix_message, &topology_to_test, packet_sender) - .await; - debug_assert_eq!(mix_packet.len(), 1); - - gateway_packets.push(mix_packet.pop().unwrap()); - } - packets.push(GatewayPackets::new( - node.clients_address(), - node.identity_key, - gateway_packets, - )) - } - PreparedNode::Invalid(node) => invalid.push(node), - // `prepare_gateways` should NEVER return prepared mixnodes - _ => unreachable!(), - } + route: &TestRoute, + num: usize, + ) -> GatewayPackets { + let mut mix_packets = Vec::with_capacity(num); + let test_packet = route.self_test_packet(); + let recipient = self.create_packet_sender(route.gateway()); + for _ in 0..num { + let mix_packet = self + .wrap_test_packet(&test_packet, route.topology(), recipient) + .await; + mix_packets.push(mix_packet) } - packets + GatewayPackets::new( + route.gateway_clients_address(), + route.gateway_identity(), + mix_packets, + ) } - pub(super) async fn prepare_test_packets(&mut self, nonce: u64) -> PreparedPackets { + fn filter_outdated_and_malformed_mixnodes( + &self, + nodes: Vec, + ) -> (Vec, Vec) { + let mut parsed_nodes = Vec::new(); + let mut invalid_nodes = Vec::new(); + for mixnode in nodes { + if !self.check_version_compatibility(&mixnode.mix_node.version) { + invalid_nodes.push(InvalidNode::Outdated( + mixnode.mix_node.identity_key, + mixnode.owner, + mixnode.mix_node.version, + )); + continue; + } + if let Ok(parsed_node) = (&mixnode).try_into() { + parsed_nodes.push(parsed_node) + } else { + invalid_nodes.push(InvalidNode::Malformed( + mixnode.mix_node.identity_key, + mixnode.owner, + )); + } + } + (parsed_nodes, invalid_nodes) + } + + fn filter_outdated_and_malformed_gateways( + &self, + nodes: Vec, + ) -> (Vec, Vec) { + let mut parsed_nodes = Vec::new(); + let mut invalid_nodes = Vec::new(); + for gateway in nodes { + if !self.check_version_compatibility(&gateway.gateway.version) { + invalid_nodes.push(InvalidNode::Outdated( + gateway.gateway.identity_key, + gateway.owner, + gateway.gateway.version, + )); + continue; + } + if let Ok(parsed_node) = (&gateway).try_into() { + parsed_nodes.push(parsed_node) + } else { + invalid_nodes.push(InvalidNode::Malformed( + gateway.gateway.identity_key, + gateway.owner, + )); + } + } + (parsed_nodes, invalid_nodes) + } + + pub(super) async fn prepare_test_packets( + &mut self, + test_nonce: u64, + test_routes: &[TestRoute], + ) -> PreparedPackets { let (mixnode_bonds, gateway_bonds) = self.get_network_nodes().await; - let mut invalid_nodes = Vec::new(); - let mixes = self.prepare_mixnodes(nonce, &mixnode_bonds); - let gateways = self.prepare_gateways(nonce, &gateway_bonds); + let (mixes, invalid_mixnodes) = self.filter_outdated_and_malformed_mixnodes(mixnode_bonds); + let (gateways, invalid_gateways) = + self.filter_outdated_and_malformed_gateways(gateway_bonds); - // get the keys of all nodes that will get tested this round - let tested_nodes: Vec<_> = self - .tested_nodes(&mixes) - .into_iter() - .chain(self.tested_nodes(&gateways).into_iter()) - .collect(); + let tested_mixnodes = mixes.iter().map(|node| node.into()).collect::>(); + let tested_gateways = gateways.iter().map(|node| node.into()).collect::>(); - // those packets are going to go to our 'main' gateway - let mix_packets = self - .create_mixnode_mix_packets(mixes, &mut invalid_nodes) - .await; + let packets_to_create = (test_routes.len() * self.per_node_test_packets) + * (tested_mixnodes.len() + tested_gateways.len()); + info!(target: "TestPreparer", "Need to create {} mix packets", packets_to_create); - let main_gateway_id = self.tested_network.main_v4_gateway().identity_key; + let mut all_gateway_packets = HashMap::new(); - let mut gateway_packets = self - .create_gateway_mix_packets(gateways, &mut invalid_nodes) - .await; + // for each test route... + for test_route in test_routes { + let recipient = self.create_packet_sender(test_route.gateway()); + let gateway_identity = test_route.gateway_identity(); + let gateway_address = test_route.gateway_clients_address(); - // check whether our 'good' gateway is being tested - if let Some(tested_main_gateway_packets) = gateway_packets - .iter_mut() - .find(|gateway| gateway.gateway_address() == main_gateway_id) - { - // we are testing the gateway we specified in our 'good' topology - tested_main_gateway_packets.push_packets(mix_packets); - } else { - // we are not testing the gateway from our 'good' topology -> it's probably - // situation similar to using 'good' qa-topology but testing testnet nodes. - let main_gateway_packets = GatewayPackets::new( - self.tested_network.main_v4_gateway().clients_address(), - main_gateway_id, - mix_packets, - ); - gateway_packets.push(main_gateway_packets); + // it's actually going to be a tiny bit more due to gateway testing, but it's a good enough approximation + let mut mix_packets = Vec::with_capacity(mixes.len() * self.per_node_test_packets); + + // and for each mixnode... + for mixnode in &mixes { + let test_packet = TestPacket::from_mixnode(mixnode, test_route.id(), test_nonce); + let topology = test_route.substitute_mix(mixnode); + // produce n mix packets + for _ in 0..self.per_node_test_packets { + let mix_packet = self + .wrap_test_packet(&test_packet, &topology, recipient) + .await; + mix_packets.push(mix_packet); + } + } + + let gateway_packets = all_gateway_packets + .entry(gateway_identity.to_bytes()) + .or_insert_with(|| GatewayPackets::empty(gateway_address, gateway_identity)); + gateway_packets.push_packets(mix_packets); + + // and for each gateway... + for gateway in &gateways { + let mut gateway_mix_packets = Vec::new(); + let test_packet = TestPacket::from_gateway(gateway, test_route.id(), test_nonce); + let gateway_identity = gateway.identity_key; + let gateway_address = gateway.clients_address(); + let recipient = self.create_packet_sender(gateway); + let topology = test_route.substitute_gateway(gateway); + // produce n mix packets + for _ in 0..self.per_node_test_packets { + let mix_packet = self + .wrap_test_packet(&test_packet, &topology, recipient) + .await; + gateway_mix_packets.push(mix_packet); + } + + // and push it into existing struct (if it's a "core" gateway being tested against another route) + // or create a new one + let gateway_packets = all_gateway_packets + .entry(gateway_identity.to_bytes()) + .or_insert_with(|| GatewayPackets::empty(gateway_address, gateway_identity)); + gateway_packets.push_packets(gateway_mix_packets); + } } + // convert our hashmap back into a vec + let packets = all_gateway_packets.into_iter().map(|(_, v)| v).collect(); + PreparedPackets { - packets: gateway_packets, - tested_nodes, - invalid_nodes, + packets, + tested_mixnodes, + tested_gateways, + invalid_mixnodes, + invalid_gateways, } } } diff --git a/validator-api/src/network_monitor/monitor/processor.rs b/validator-api/src/network_monitor/monitor/processor.rs index 7ac13febf2..11d995f1c8 100644 --- a/validator-api/src/network_monitor/monitor/processor.rs +++ b/validator-api/src/network_monitor/monitor/processor.rs @@ -3,6 +3,7 @@ use crate::network_monitor::gateways_reader::GatewayMessages; use crate::network_monitor::test_packet::TestPacket; +use crate::network_monitor::ROUTE_TESTING_TEST_NONCE; use crypto::asymmetric::encryption; use futures::channel::mpsc; use futures::lock::{Mutex, MutexGuard}; @@ -50,7 +51,7 @@ enum LockPermit { struct ReceivedProcessorInner { /// Nonce of the current test run indicating which packets should get rejected. - nonce: Option, + test_nonce: Option, /// Channel for receiving packets/messages from the gateway clients packets_receiver: ReceivedProcessorReceiver, @@ -70,7 +71,7 @@ impl ReceivedProcessorInner { fn on_message(&mut self, message: Vec) -> Result<(), ProcessingError> { // if the nonce is none it means the packet was received during the 'waiting' for the // next test run - if self.nonce.is_none() { + if self.test_nonce.is_none() { return Err(ProcessingError::ReceivedOutsideTestRun); } @@ -91,8 +92,8 @@ impl ReceivedProcessorInner { .map_err(|_| ProcessingError::MalformedPacketReceived)?; // we know nonce is NOT none - if test_packet.nonce() != self.nonce.unwrap() { - return Err(ProcessingError::NonMatchingNonce(test_packet.nonce())); + if test_packet.test_nonce() != self.test_nonce.unwrap() { + return Err(ProcessingError::NonMatchingNonce(test_packet.test_nonce())); } self.received_packets.push(test_packet); @@ -101,7 +102,7 @@ impl ReceivedProcessorInner { } fn finish_run(&mut self) -> Vec { - self.nonce = None; + self.test_nonce = None; mem::take(&mut self.received_packets) } } @@ -118,7 +119,7 @@ impl ReceivedProcessor { ) -> Self { let inner: Arc> = Arc::new(Mutex::new(ReceivedProcessorInner { - nonce: None, + test_nonce: None, packets_receiver, client_encryption_keypair, message_receiver: MessageReceiver::new(), @@ -185,7 +186,11 @@ impl ReceivedProcessor { }); } - pub(super) async fn set_new_expected(&mut self, nonce: u64) { + pub(super) async fn set_route_test_nonce(&mut self) { + self.set_new_test_nonce(ROUTE_TESTING_TEST_NONCE).await + } + + pub(super) async fn set_new_test_nonce(&mut self, test_nonce: u64) { // ask for the lock back self.permit_changer .as_mut() @@ -195,7 +200,7 @@ impl ReceivedProcessor { .expect("processing task has died!"); let mut inner = self.inner.lock().await; - inner.nonce = Some(nonce); + inner.test_nonce = Some(test_nonce); // give the permit back drop(inner); diff --git a/validator-api/src/network_monitor/monitor/sender.rs b/validator-api/src/network_monitor/monitor/sender.rs index eb314d5301..0bdd116898 100644 --- a/validator-api/src/network_monitor/monitor/sender.rs +++ b/validator-api/src/network_monitor/monitor/sender.rs @@ -1,6 +1,10 @@ // Copyright 2021 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 +use crate::network_monitor::monitor::gateway_clients_cache::{ + ActiveGatewayClients, GatewayClientHandle, +}; +use crate::network_monitor::monitor::gateways_pinger::GatewayPinger; use crate::network_monitor::monitor::receiver::{GatewayClientUpdate, GatewayClientUpdateSender}; use crypto::asymmetric::identity::{self, PUBLIC_KEY_LENGTH}; use futures::channel::mpsc; @@ -9,33 +13,35 @@ use futures::task::Context; use futures::{Future, Stream}; use gateway_client::error::GatewayClientError; use gateway_client::{AcknowledgementReceiver, GatewayClient, MixnetMessageReceiver}; -use log::{debug, info, warn}; +use log::{debug, info, trace, warn}; use nymsphinx::forwarding::packet::MixPacket; use pin_project::pin_project; -use std::collections::HashMap; use std::mem; use std::num::NonZeroUsize; use std::pin::Pin; use std::sync::Arc; use std::task::Poll; use std::time::Duration; -use tokio::time::Instant; #[cfg(feature = "coconut")] use coconut_interface::Credential; const TIME_CHUNK_SIZE: Duration = Duration::from_millis(50); +// If we're below 10MB of bandwidth, claim some more +#[cfg(feature = "coconut")] +const REMAINING_BANDWIDTH_THRESHOLD: i64 = 10 * 1000 * 1000; + pub(crate) struct GatewayPackets { /// Network address of the target gateway if wanted to be accessed by the client. /// It is a websocket address. - clients_address: String, + pub(crate) clients_address: String, /// Public key of the target gateway. - pub_key: identity::PublicKey, + pub(crate) pub_key: identity::PublicKey, /// All the packets that are going to get sent to the gateway. - packets: Vec, + pub(crate) packets: Vec, } impl GatewayPackets { @@ -51,12 +57,23 @@ impl GatewayPackets { } } - pub(super) fn push_packets(&mut self, mut packets: Vec) { - self.packets.append(&mut packets) + pub(crate) fn empty(clients_address: String, pub_key: identity::PublicKey) -> Self { + GatewayPackets { + clients_address, + pub_key, + packets: Vec::new(), + } } - pub(super) fn gateway_address(&self) -> identity::PublicKey { - self.pub_key + pub(super) fn push_packets(&mut self, mut packets: Vec) { + if self.packets.is_empty() { + self.packets = packets + } else if self.packets.len() > packets.len() { + self.packets.append(&mut packets) + } else { + packets.append(&mut self.packets); + self.packets = packets; + } } } @@ -77,12 +94,42 @@ struct FreshGatewayClientData { coconut_bandwidth_credential: Credential, } +impl FreshGatewayClientData { + fn notify_connection_failure( + self: Arc, + raw_gateway_id: [u8; PUBLIC_KEY_LENGTH], + ) { + // if this unwrap failed it means something extremely weird is going on + // and we got some solar flare bitflip type of corruption + let gateway_key = identity::PublicKey::from_bytes(&raw_gateway_id) + .expect("failed to recover gateways public key from valid bytes"); + + // remove the gateway listener channels + self.gateways_status_updater + .unbounded_send(GatewayClientUpdate::Failure(gateway_key)) + .expect("packet receiver seems to have died!"); + } + + fn notify_new_connection( + self: Arc, + gateway_id: identity::PublicKey, + gateway_channels: Option<(MixnetMessageReceiver, AcknowledgementReceiver)>, + ) { + self.gateways_status_updater + .unbounded_send(GatewayClientUpdate::New( + gateway_id, + gateway_channels.expect("we created a new client, yet the channels are a None!"), + )) + .expect("packet receiver seems to have died!") + } +} + pub(crate) struct PacketSender { // TODO: this has a potential long-term issue. If we keep those clients cached between runs, // malicious gateways could figure out which traffic comes from the network monitor and always // forward that traffic while dropping the rest. However, at the current stage such sophisticated // behaviour is unlikely. - active_gateway_clients: HashMap<[u8; PUBLIC_KEY_LENGTH], GatewayClient>, + active_gateway_clients: ActiveGatewayClients, // I guess that will be required later on if credentials are got per gateway // aggregated_verification_key: Arc, @@ -103,7 +150,7 @@ impl PacketSender { #[cfg(feature = "coconut")] coconut_bandwidth_credential: Credential, ) -> Self { PacketSender { - active_gateway_clients: HashMap::new(), + active_gateway_clients: ActiveGatewayClients::new(), fresh_gateway_client_data: Arc::new(FreshGatewayClientData { gateways_status_updater, local_identity, @@ -117,12 +164,24 @@ impl PacketSender { } } - async fn new_gateway_client( + pub(crate) fn spawn_gateways_pinger(&self, pinging_interval: Duration) { + let gateway_pinger = GatewayPinger::new( + self.active_gateway_clients.clone(), + self.fresh_gateway_client_data + .gateways_status_updater + .clone(), + pinging_interval, + ); + + tokio::spawn(async move { gateway_pinger.run().await }); + } + + fn new_gateway_client_handle( address: String, identity: identity::PublicKey, fresh_gateway_client_data: &FreshGatewayClientData, ) -> ( - GatewayClient, + GatewayClientHandle, (MixnetMessageReceiver, AcknowledgementReceiver), ) { // TODO: future optimization: if we're remaking client for a gateway to which we used to be connected in the past, @@ -133,7 +192,7 @@ impl PacketSender { // so that the gateway client would not crash let (ack_sender, ack_receiver) = mpsc::unbounded(); ( - GatewayClient::new( + GatewayClientHandle::new(GatewayClient::new( address, Arc::clone(&fresh_gateway_client_data.local_identity), identity, @@ -141,7 +200,7 @@ impl PacketSender { message_sender, ack_sender, fresh_gateway_client_data.gateway_response_timeout, - ), + )), (message_receiver, ack_receiver), ) } @@ -189,7 +248,7 @@ impl PacketSender { // this way we won't have to do reallocations in here as they're unavoidable when // splitting a vector into multiple vectors while let Some(retained) = split_off_vec(&mut mix_packets, packets_per_time_chunk) { - debug!(target: "MessageSender","Sending {} packets...", mix_packets.len()); + trace!(target: "MessageSender","Sending {} packets...", mix_packets.len()); if mix_packets.len() == 1 { client.send_mix_packet(mix_packets.pop().unwrap()).await?; @@ -207,103 +266,212 @@ impl PacketSender { Ok(()) } + async fn create_new_gateway_client_handle_and_authenticate( + address: String, + identity: identity::PublicKey, + fresh_gateway_client_data: &FreshGatewayClientData, + gateway_connection_timeout: Duration, + ) -> Option<( + GatewayClientHandle, + (MixnetMessageReceiver, AcknowledgementReceiver), + )> { + let (new_client, (message_receiver, ack_receiver)) = + Self::new_gateway_client_handle(address, identity, fresh_gateway_client_data); + + // Put this in timeout in case the gateway has incorrectly set their ulimit and our connection + // gets stuck in their TCP queue and just hangs on our end but does not terminate + // (an actual bug we experienced) + // + // Note: locking the client in unchecked manner is fine here as we just created the lock + // and it wasn't shared with anyone, therefore we're the only one holding reference to it + // and hence it's impossible to fail to obtain the permit. + let mut unlocked_client = new_client.lock_client_unchecked(); + match tokio::time::timeout( + gateway_connection_timeout, + unlocked_client.get_mut_unchecked().authenticate_and_start( + #[cfg(feature = "coconut")] + Some( + fresh_gateway_client_data + .coconut_bandwidth_credential + .clone(), + ), + ), + ) + .await + { + Ok(Ok(_)) => { + drop(unlocked_client); + Some((new_client, (message_receiver, ack_receiver))) + } + Ok(Err(err)) => { + warn!( + "failed to authenticate with new gateway ({}) - {}", + identity.to_base58_string(), + err + ); + // we failed to create a client, can't do much here + None + } + Err(_) => { + warn!( + "timed out while trying to authenticate with new gateway ({})", + identity.to_base58_string() + ); + None + } + } + } + + #[cfg(feature = "coconut")] + async fn check_remaining_bandwidth( + client: &mut GatewayClient, + fresh_gateway_client_data: &FreshGatewayClientData, + ) -> Result<(), GatewayClientError> { + if client.remaining_bandwidth() < REMAINING_BANDWIDTH_THRESHOLD { + // TODO: SECURITY: + // We're using exactly the same credential again because we have no double + // spending protection... + info!( + "Client to gateway {} is running out of bandwidth... Claiming some more...", + client.gateway_identity().to_base58_string() + ); + client + .claim_coconut_bandwidth( + fresh_gateway_client_data + .coconut_bandwidth_credential + .clone(), + ) + .await + } else { + Ok(()) + } + } + // TODO: perhaps it should be spawned as a task to execute it in parallel rather // than just concurrently? async fn send_gateway_packets( gateway_connection_timeout: Duration, packets: GatewayPackets, fresh_gateway_client_data: Arc, - client: Option, + client: Option, max_sending_rate: usize, - ) -> Option { - let was_present = client.is_some(); + ) -> Option { + let existing_client = client.is_some(); - let (mut client, gateway_channels) = if let Some(client) = client { + // Note that in the worst case scenario we will only wait for a second or two to obtain the lock + // as other possibly entity holding the lock (the gateway pinger) is attempting to send + // the ping messages with a maximum timeout. + let (client, gateway_channels) = if let Some(client) = client { + if client.is_invalid().await { + warn!("Our existing client was invalid - two test runs happened back to back without cleanup"); + return None; + } (client, None) } else { - let (mut new_client, (message_receiver, ack_receiver)) = Self::new_gateway_client( - packets.clients_address, - packets.pub_key, - &fresh_gateway_client_data, - ) - .await; - - // Put this in timeout in case the gateway has incorrectly set their ulimit and our connection - // gets stuck in their TCP queue and just hangs on our end but does not terminate - // (an actual bug we experienced) - match tokio::time::timeout( - gateway_connection_timeout, - new_client.authenticate_and_start( - #[cfg(feature = "coconut")] - Some( - fresh_gateway_client_data - .coconut_bandwidth_credential - .clone(), - ), - ), - ) - .await - { - Ok(Ok(_)) => {} - Ok(Err(err)) => { - warn!( - "failed to authenticate with new gateway ({}) - {}", - packets.pub_key.to_base58_string(), - err - ); - // we failed to create a client, can't do much here - return None; - } - Err(_) => { - warn!( - "timed out while trying to authenticate with new gateway ({})", - packets.pub_key.to_base58_string() - ); - return None; - } - }; - - (new_client, Some((message_receiver, ack_receiver))) + let (client, gateway_channels) = + Self::create_new_gateway_client_handle_and_authenticate( + packets.clients_address, + packets.pub_key, + &fresh_gateway_client_data, + gateway_connection_timeout, + ) + .await?; + (client, Some(gateway_channels)) }; + let estimated_time = + Duration::from_secs_f64(packets.packets.len() as f64 / max_sending_rate as f64); + // give some leeway + let timeout = estimated_time * 3; + + let mut guard = client.lock_client().await; + let unwrapped_client = guard.get_mut_unchecked(); + + #[cfg(feature = "coconut")] if let Err(err) = - Self::attempt_to_send_packets(&mut client, packets.packets, max_sending_rate).await + Self::check_remaining_bandwidth(unwrapped_client, &fresh_gateway_client_data).await { warn!( - "failed to send packets to {} - {:?}", - packets.pub_key.to_base58_string(), + "Failed to claim additional bandwidth for {} - {}", + unwrapped_client.gateway_identity().to_base58_string(), err ); - // if this was a fresh client, there's no need to do anything as it was never - // registered to get read - if was_present { - fresh_gateway_client_data - .gateways_status_updater - .unbounded_send(GatewayClientUpdate::Failure(packets.pub_key)) - .expect("packet receiver seems to have died!"); + if existing_client { + guard.invalidate(); + fresh_gateway_client_data.notify_connection_failure(packets.pub_key.to_bytes()); } return None; - } else if !was_present { - // this is a fresh and working client - fresh_gateway_client_data - .gateways_status_updater - .unbounded_send(GatewayClientUpdate::New( - packets.pub_key, - gateway_channels - .expect("we created a new client, yet the channels are a None!"), - )) - .expect("packet receiver seems to have died!") } + + match tokio::time::timeout( + timeout, + Self::attempt_to_send_packets(unwrapped_client, packets.packets, max_sending_rate), + ) + .await + { + Err(_timeout) => { + warn!( + "failed to send packets to {} - we timed out", + packets.pub_key.to_base58_string(), + ); + // if this was a fresh client, there's no need to do anything as it was never + // registered to get read + if existing_client { + guard.invalidate(); + fresh_gateway_client_data.notify_connection_failure(packets.pub_key.to_bytes()); + } + return None; + } + Ok(Err(err)) => { + warn!( + "failed to send packets to {} - {:?}", + packets.pub_key.to_base58_string(), + err + ); + // if this was a fresh client, there's no need to do anything as it was never + // registered to get read + if existing_client { + guard.invalidate(); + fresh_gateway_client_data.notify_connection_failure(packets.pub_key.to_bytes()); + } + return None; + } + Ok(Ok(_)) => { + if !existing_client { + fresh_gateway_client_data + .notify_new_connection(packets.pub_key, gateway_channels); + } + } + } + + drop(guard); Some(client) } + // point of this is to basically insert handles of fresh clients that didn't exist here before + async fn merge_client_handles(&self, handles: Vec) { + let mut guard = self.active_gateway_clients.lock().await; + for handle in handles { + let raw_identity = handle.raw_identity(); + if let Some(existing) = guard.get(&raw_identity) { + if !handle.ptr_eq(existing) { + panic!("Duplicate client detected!") + } + + if handle.is_invalid().await { + guard.remove(&raw_identity); + } + } else { + // client never existed -> just insert it + guard.insert(raw_identity, handle); + } + } + } + pub(super) async fn send_packets(&mut self, packets: Vec) { // we know that each of the elements in the packets array will only ever access a single, // unique element from the existing clients - // while it may seem weird that each time we send packets we remove the entries from the map, - // and then put them back in, this way we remove the need for having locks instead, like - // Arc>>> let gateway_connection_timeout = self.gateway_connection_timeout; let max_concurrent_clients = if self.max_concurrent_clients > 0 { Some(self.max_concurrent_clients) @@ -312,21 +480,34 @@ impl PacketSender { }; let max_sending_rate = self.max_sending_rate; + let guard = self.active_gateway_clients.lock().await; + // this clippy warning is a false positive as we cannot get rid of the collect by moving + // everything into a single iterator as it would require us to hold the lock the entire time + // and that is exactly what we want to avoid + #[allow(clippy::needless_collect)] + let stream_data = packets + .into_iter() + .map(|packets| { + let existing_client = guard + .get(&packets.pub_key.to_bytes()) + .map(|client| client.clone_data_pointer()); + ( + packets, + Arc::clone(&self.fresh_gateway_client_data), + existing_client, + ) + }) + .collect::>(); + + // drop the guard immediately so that the other task (gateway pinger) would not need to wait until + // we're done sending packets (note: without this drop, we wouldn't be able to ping gateways that + // we're not interacting with right now) + drop(guard); + // can't chain it all nicely together as there's no adapter method defined on Stream directly // for ForEachConcurrentClientUse - let stream = stream::iter(packets.into_iter().map(|packets| { - let existing_client = self - .active_gateway_clients - .remove(&(packets.pub_key.to_bytes())); - ( - packets, - Arc::clone(&self.fresh_gateway_client_data), - existing_client, - ) - })); - - ForEachConcurrentClientUse::new( - stream, + let used_clients = ForEachConcurrentClientUse::new( + stream::iter(stream_data.into_iter()), max_concurrent_clients, |(packets, fresh_data, client)| async move { Self::send_gateway_packets( @@ -341,65 +522,10 @@ impl PacketSender { ) .await .into_iter() - .for_each(|client| { - if let Some(client) = client { - if let Some(existing) = self - .active_gateway_clients - .insert(client.gateway_identity().to_bytes(), client) - { - panic!( - "we got duplicate gateway client for {}!", - existing.gateway_identity().to_base58_string() - ); - } - } - }) - } + .flatten() + .collect(); - pub(super) async fn ping_all_active_gateways(&mut self) { - if self.active_gateway_clients.is_empty() { - info!(target: "Monitor", "no gateways to ping"); - return; - } - - let ping_start = Instant::now(); - - let mut clients_to_purge = Vec::new(); - - // since we don't need to wait for response, we can just ping all gateways sequentially - // if it becomes problem later on, we can adjust it. - for (gateway_id, active_client) in self.active_gateway_clients.iter_mut() { - if let Err(err) = active_client.send_ping_message().await { - warn!( - target: "Monitor", - "failed to send ping message to gateway {} - {} - assuming the connection is dead.", - active_client.gateway_identity().to_base58_string(), - err, - ); - clients_to_purge.push(*gateway_id); - } - } - - // purge all dead connections - for gateway_id in clients_to_purge.into_iter() { - // if this unwrap failed it means something extremely weird is going on - // and we got some solar flare bitflip type of corruption - let gateway_key = identity::PublicKey::from_bytes(&gateway_id) - .expect("failed to recover gateways public key from valid bytes"); - - // remove the gateway listener channels - self.fresh_gateway_client_data - .gateways_status_updater - .unbounded_send(GatewayClientUpdate::Failure(gateway_key)) - .expect("packet receiver seems to have died!"); - - // and remove it from our cache - self.active_gateway_clients.remove(&gateway_id); - } - - let ping_end = Instant::now(); - let time_taken = ping_end.duration_since(ping_start); - debug!(target: "Monitor", "pinging all active gateways took {:?}", time_taken); + self.merge_client_handles(used_clients).await; } } @@ -412,14 +538,14 @@ struct ForEachConcurrentClientUse { f: F, futures: FuturesUnordered, limit: Option, - result: Vec>, + result: Vec>, } impl ForEachConcurrentClientUse where St: Stream, F: FnMut(St::Item) -> Fut, - Fut: Future>, + Fut: Future>, { pub(super) fn new(stream: St, limit: Option, f: F) -> Self { let size_hint = stream.size_hint(); @@ -438,9 +564,9 @@ impl Future for ForEachConcurrentClientUse where St: Stream, F: FnMut(St::Item) -> Fut, - Fut: Future>, + Fut: Future>, { - type Output = Vec>; + type Output = Vec>; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let mut this = self.project(); diff --git a/validator-api/src/network_monitor/monitor/summary_producer.rs b/validator-api/src/network_monitor/monitor/summary_producer.rs index c8041c86fc..81e901e2df 100644 --- a/validator-api/src/network_monitor/monitor/summary_producer.rs +++ b/validator-api/src/network_monitor/monitor/summary_producer.rs @@ -2,258 +2,363 @@ // SPDX-License-Identifier: Apache-2.0 use crate::network_monitor::monitor::preparer::{InvalidNode, TestedNode}; -use crate::network_monitor::test_packet::{NodeType, TestPacket}; -use crate::PENALISE_OUTDATED; +use crate::network_monitor::test_packet::TestPacket; +use crate::network_monitor::test_route::TestRoute; use std::collections::HashMap; +use std::fmt::{Display, Formatter}; + +// just some approximate measures to print to stdout (well, technically stderr since it's being printed via log) +const EXCEPTIONAL_THRESHOLD: u8 = 95; // 95 - 100 +const FINE_THRESHOLD: u8 = 80; // 80 - 95 +const POOR_THRESHOLD: u8 = 60; // 60 - 80 +const UNRELIABLE_THRESHOLD: u8 = 1; // 1 - 60 + +// I didn't have time to implement it for this PR, however, an idea for the future is as follows: +// After testing network against N routes, if any one of them is worse than ALLOWED_RELIABILITY_DEVIATION +// from the average result, remove this data and recalculate scores. +// const ALLOWED_RELIABILITY_DEVIATION: f32 = 5.0; #[derive(Debug)] pub(crate) struct NodeResult { pub(crate) identity: String, pub(crate) owner: String, - pub(crate) working_ipv4: bool, - pub(crate) working_ipv6: bool, + pub(crate) reliability: u8, } -#[derive(Default)] -struct NodeStatus { - ip_v4_compatible: bool, - ip_v6_compatible: bool, -} - -impl NodeStatus { - fn into_node_status(self, identity: String, owner: String) -> NodeResult { +impl NodeResult { + pub(crate) fn new(identity: String, owner: String, reliability: u8) -> Self { NodeResult { identity, owner, - working_ipv4: self.ip_v4_compatible, - working_ipv6: self.ip_v6_compatible, + reliability, } } } -#[derive(Default)] +#[derive(Debug, Clone)] +pub(crate) struct RouteResult { + pub(crate) route: TestRoute, + reliability: u8, +} + +impl RouteResult { + pub(crate) fn new(route: TestRoute, reliability: u8) -> Self { + RouteResult { route, reliability } + } +} + +#[derive(Default, Debug)] pub(crate) struct TestReport { + pub(crate) network_reliability: f32, pub(crate) total_sent: usize, pub(crate) total_received: usize, - pub(crate) malformed: Vec, - // below are only populated if we're going to be printing the report - pub(crate) only_ipv4_compatible_mixes: Vec, - // can't speak v6, but can speak v4 - pub(crate) only_ipv6_compatible_mixes: Vec, - // can't speak v4, but can speak v6 - pub(crate) completely_unroutable_mixes: Vec, - // can't speak either v4 or v6 - pub(crate) fully_working_mixes: Vec, + pub(crate) route_results: Vec, - pub(crate) only_ipv4_compatible_gateways: Vec, - // can't speak v6, but can speak v4 - pub(crate) only_ipv6_compatible_gateways: Vec, - // can't speak v4, but can speak v6 - pub(crate) completely_unroutable_gateways: Vec, - // can't speak either v4 or v6 - pub(crate) fully_working_gateways: Vec, + pub(crate) exceptional_mixnodes: usize, + pub(crate) exceptional_gateways: usize, + + pub(crate) fine_mixnodes: usize, + pub(crate) fine_gateways: usize, + + pub(crate) poor_mixnodes: usize, + pub(crate) poor_gateways: usize, + + pub(crate) unreliable_mixnodes: usize, + pub(crate) unreliable_gateways: usize, + + pub(crate) unroutable_mixnodes: usize, + pub(crate) unroutable_gateways: usize, } impl TestReport { - fn print(&self, detailed: bool) { - info!("Sent total of {} packets", self.total_sent); - info!("Received total of {} packets", self.total_received); - info!("{} nodes are invalid", self.malformed.len()); + fn new( + total_sent: usize, + total_received: usize, + mixnode_results: &[NodeResult], + gateway_results: &[NodeResult], + route_results: &[RouteResult], + invalid_mixnodes: usize, + invalid_gateways: usize, + ) -> Self { + let mut exceptional_mixnodes = 0; + let mut exceptional_gateways = 0; - info!( - "{} mixnodes speak ONLY IPv4 (NO IPv6 connectivity)", - self.only_ipv4_compatible_mixes.len() - ); - info!( - "{} mixnodes speak ONLY IPv6 (NO IPv4 connectivity)", - self.only_ipv6_compatible_mixes.len() - ); - info!( - "{} mixnodes are totally unroutable!", - self.completely_unroutable_mixes.len() - ); - info!("{} mixnodes work fine!", self.fully_working_mixes.len()); + let mut fine_mixnodes = 0; + let mut fine_gateways = 0; - info!( - "{} gateways speak ONLY IPv4 (NO IPv6 connectivity)", - self.only_ipv4_compatible_gateways.len() - ); - info!( - "{} gateways speak ONLY IPv6 (NO IPv4 connectivity)", - self.only_ipv6_compatible_gateways.len() - ); - info!( - "{} gateways are totally unroutable!", - self.completely_unroutable_gateways.len() - ); - info!("{} gateways work fine!", self.fully_working_gateways.len()); + let mut poor_mixnodes = 0; + let mut poor_gateways = 0; - if detailed { - info!("full summary:"); - for malformed in self.malformed.iter() { - info!("{}", malformed) + let mut unreliable_mixnodes = 0; + let mut unreliable_gateways = 0; + + let mut unroutable_mixnodes = invalid_mixnodes; + let mut unroutable_gateways = invalid_gateways; + + for mixnode_result in mixnode_results { + if mixnode_result.reliability >= EXCEPTIONAL_THRESHOLD { + exceptional_mixnodes += 1; + } else if mixnode_result.reliability >= FINE_THRESHOLD { + fine_mixnodes += 1; + } else if mixnode_result.reliability >= POOR_THRESHOLD { + poor_mixnodes += 1; + } else if mixnode_result.reliability >= UNRELIABLE_THRESHOLD { + unreliable_mixnodes += 1; + } else { + unroutable_mixnodes += 1; } + } - for v4_node in self.only_ipv4_compatible_mixes.iter() { - info!("{}", v4_node) + for gateway_result in gateway_results { + if gateway_result.reliability >= EXCEPTIONAL_THRESHOLD { + exceptional_gateways += 1; + } else if gateway_result.reliability >= FINE_THRESHOLD { + fine_gateways += 1; + } else if gateway_result.reliability >= POOR_THRESHOLD { + poor_gateways += 1; + } else if gateway_result.reliability >= UNRELIABLE_THRESHOLD { + unreliable_gateways += 1; + } else { + unroutable_gateways += 1; } + } - for v6_node in self.only_ipv6_compatible_mixes.iter() { - info!("{}", v6_node) - } + let network_reliability = total_received as f32 / total_sent as f32 * 100.0; - for unroutable in self.completely_unroutable_mixes.iter() { - info!("{}", unroutable) - } - - for working in self.fully_working_mixes.iter() { - info!("{}", working) - } - - for v4_node in self.only_ipv4_compatible_gateways.iter() { - info!("{}", v4_node) - } - - for v6_node in self.only_ipv6_compatible_gateways.iter() { - info!("{}", v6_node) - } - - for unroutable in self.completely_unroutable_gateways.iter() { - info!("{}", unroutable) - } - - for working in self.fully_working_gateways.iter() { - info!("{}", working) - } + TestReport { + network_reliability, + total_sent, + total_received, + route_results: route_results.to_vec(), + exceptional_mixnodes, + exceptional_gateways, + fine_mixnodes, + fine_gateways, + poor_mixnodes, + poor_gateways, + unreliable_mixnodes, + unreliable_gateways, + unroutable_mixnodes, + unroutable_gateways, } } +} - fn parse_summary(&mut self, summary: &HashMap) { - for (node, result) in summary.iter() { - let owned_node = node.clone(); - if node.is_gateway() { - if result.ip_v4_compatible && result.ip_v6_compatible { - self.fully_working_gateways.push(owned_node) - } else if result.ip_v4_compatible { - self.only_ipv4_compatible_gateways.push(owned_node) - } else if result.ip_v6_compatible { - self.only_ipv6_compatible_gateways.push(owned_node) - } else { - self.completely_unroutable_gateways.push(owned_node) - } - } else if result.ip_v4_compatible && result.ip_v6_compatible { - self.fully_working_mixes.push(owned_node) - } else if result.ip_v4_compatible { - self.only_ipv4_compatible_mixes.push(owned_node) - } else if result.ip_v6_compatible { - self.only_ipv6_compatible_mixes.push(owned_node) - } else { - self.completely_unroutable_mixes.push(owned_node) - } +impl Display for TestReport { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + writeln!(f, "Mix Network Test Report")?; + writeln!( + f, + "Overall reliability: {:.2} ({} / {} received)", + self.network_reliability, self.total_received, self.total_sent, + )?; + + writeln!(f, "Routes used for testing:")?; + for route_result in &self.route_results { + writeln!( + f, + "{:?}, reliability: {:.2}", + route_result.route, route_result.reliability + )?; } + + writeln!( + f, + "Exceptional mixnodes (reliability >= {}): {}", + EXCEPTIONAL_THRESHOLD, self.exceptional_mixnodes + )?; + writeln!( + f, + "Exceptional gateways (reliability >= {}): {}", + EXCEPTIONAL_THRESHOLD, self.exceptional_gateways + )?; + + writeln!( + f, + "Fine mixnodes (reliability {} - {}): {}", + FINE_THRESHOLD, EXCEPTIONAL_THRESHOLD, self.fine_mixnodes + )?; + writeln!( + f, + "Fine gateways (reliability {} - {}): {}", + FINE_THRESHOLD, EXCEPTIONAL_THRESHOLD, self.fine_gateways + )?; + + writeln!( + f, + "Poor mixnodes (reliability {} - {}): {}", + POOR_THRESHOLD, FINE_THRESHOLD, self.poor_mixnodes + )?; + writeln!( + f, + "Poor gateways (reliability {} - {}): {}", + POOR_THRESHOLD, FINE_THRESHOLD, self.poor_gateways + )?; + + writeln!( + f, + "Unreliable mixnodes (reliability {} - {}): {}", + UNRELIABLE_THRESHOLD, POOR_THRESHOLD, self.unreliable_mixnodes + )?; + writeln!( + f, + "Unreliable gateways (reliability {} - {}): {}", + UNRELIABLE_THRESHOLD, POOR_THRESHOLD, self.unreliable_gateways + )?; + + writeln!( + f, + "Unroutable mixnodes (reliability < {}): {}", + UNRELIABLE_THRESHOLD, self.unroutable_mixnodes + )?; + writeln!( + f, + "Unroutable gateways (reliability < {}): {}", + UNRELIABLE_THRESHOLD, self.unroutable_gateways + )?; + + Ok(()) } } pub(crate) struct TestSummary { pub(crate) mixnode_results: Vec, pub(crate) gateway_results: Vec, - pub(crate) test_report: TestReport, + pub(crate) route_results: Vec, + + // technically we don't need to keep them here, but I couldn't figure out a better way + // of keeping them on hand to later produce the report + pub(crate) invalid_mixnodes: usize, + pub(crate) invalid_gateways: usize, +} + +impl TestSummary { + pub(crate) fn create_report(&self, total_sent: usize, total_received: usize) -> TestReport { + TestReport::new( + total_sent, + total_received, + &self.mixnode_results, + &self.gateway_results, + &self.route_results, + self.invalid_mixnodes, + self.invalid_gateways, + ) + } } #[derive(Default)] pub(crate) struct SummaryProducer { + per_node_test_packets: usize, print_report: bool, - print_detailed_report: bool, } impl SummaryProducer { + pub(crate) fn new(per_node_test_packets: usize) -> Self { + SummaryProducer { + per_node_test_packets, + print_report: false, + } + } + pub(crate) fn with_report(mut self) -> Self { self.print_report = true; self } - pub(crate) fn with_detailed_report(mut self) -> Self { - self.print_report = true; - self.print_detailed_report = true; - self - } - pub(super) fn produce_summary( &self, - expected_nodes: Vec, + tested_mixnodes: Vec, + tested_gateways: Vec, received_packets: Vec, - invalid_nodes: Vec, + invalid_mixnodes: Vec, + invalid_gateways: Vec, + test_routes: &[TestRoute], ) -> TestSummary { - let expected_nodes_count = expected_nodes.len(); - let received_packets_count = received_packets.len(); + let invalid_mixnodes_count = invalid_mixnodes.len(); + let invalid_gateways_count = invalid_gateways.len(); - // contains map of all (seemingly valid) nodes and whether they speak ipv4/ipv6 - let mut summary: HashMap = HashMap::new(); + let mut raw_mixnode_results = HashMap::new(); + let mut raw_gateway_results = HashMap::new(); - // update based on data we actually got - for received_status in received_packets.into_iter() { - let is_received_v4 = received_status.ip_version().is_v4(); - let entry = summary.entry(received_status.into()).or_default(); - if is_received_v4 { - entry.ip_v4_compatible = true + let mut raw_route_results = HashMap::new(); + + // we expect each route to receive this many packets in the ideal world + let per_route_expected = + (tested_mixnodes.len() + tested_gateways.len()) * self.per_node_test_packets; + let per_node_expected = test_routes.len() * self.per_node_test_packets; + + for tested_mixnode in tested_mixnodes { + raw_mixnode_results.insert((tested_mixnode.identity, tested_mixnode.owner), 0); + } + + for tested_gateway in tested_gateways { + raw_gateway_results.insert((tested_gateway.identity, tested_gateway.owner), 0); + } + + for invalid_mixnode in invalid_mixnodes { + raw_mixnode_results.insert((invalid_mixnode.identity(), invalid_mixnode.owner()), 0); + } + + for invalid_gateway in invalid_gateways { + raw_gateway_results.insert((invalid_gateway.identity(), invalid_gateway.owner()), 0); + } + + for test_route in test_routes { + raw_route_results.insert(test_route.id(), 0); + } + + for received in received_packets { + let id_owner = (received.pub_key.to_base58_string(), received.owner); + + if received.node_type.is_mixnode() { + *raw_mixnode_results.entry(id_owner).or_default() += 1usize; } else { - entry.ip_v6_compatible = true + *raw_gateway_results.entry(id_owner).or_default() += 1usize; } + + *raw_route_results.entry(received.route_id).or_default() += 1usize; } - // insert entries we didn't get but were expecting - for expected in expected_nodes.into_iter() { - summary.entry(expected).or_default(); - } - - // finally insert malformed nodes - for malformed in invalid_nodes.iter() { - match malformed { - InvalidNode::OutdatedMix(id, owner, _) - | InvalidNode::OutdatedGateway(id, owner, _) => { - if PENALISE_OUTDATED { - summary.insert(TestedNode::from_raw_gateway(id, owner), Default::default()); - } - } - InvalidNode::MalformedMix(id, owner) | InvalidNode::MalformedGateway(id, owner) => { - summary.insert(TestedNode::from_raw_mix(id, owner), Default::default()); - } - } - } - - let mut report = TestReport { - total_sent: expected_nodes_count * 2, // we sent two packets per node (one ipv4 and one ipv6) - total_received: received_packets_count, - malformed: invalid_nodes, - - ..Default::default() - }; - - report.parse_summary(&summary); - - if self.print_report { - report.print(self.print_detailed_report); - } - - let (mixes, gateways): (Vec<_>, Vec<_>) = summary + let mixnode_results = raw_mixnode_results .into_iter() - .partition(|(node, _)| node.node_type == NodeType::Mixnode); - - let mixnode_results = mixes - .into_iter() - .map(|(node, result)| result.into_node_status(node.identity, node.owner)) + .map(|((id, owner), received)| { + let reliability = + (received as f32 / per_node_expected as f32 * 100.0).round() as u8; + NodeResult::new(id, owner, reliability) + }) .collect(); - let gateway_results = gateways + let gateway_results = raw_gateway_results .into_iter() - .map(|(node, result)| result.into_node_status(node.identity, node.owner)) + .map(|((id, owner), received)| { + let reliability = + (received as f32 / per_node_expected as f32 * 100.0).round() as u8; + NodeResult::new(id, owner, reliability) + }) + .collect(); + + let route_results = raw_route_results + .into_iter() + .filter_map(|(id, received)| { + let reliability = + (received as f32 / per_route_expected as f32 * 100.0).round() as u8; + + // this might be suboptimal as we're going through the entire slice every time + // but realistically this slice will never have more than ~ 10 elements AT MOST + test_routes + .iter() + .find(|route| route.id() == id) + .map(|route| RouteResult::new(route.clone(), reliability)) + }) .collect(); TestSummary { mixnode_results, gateway_results, - test_report: report, + route_results, + invalid_mixnodes: invalid_mixnodes_count, + invalid_gateways: invalid_gateways_count, } } } diff --git a/validator-api/src/network_monitor/test_packet.rs b/validator-api/src/network_monitor/test_packet.rs index 33dd4f3f3f..4852c2f2e7 100644 --- a/validator-api/src/network_monitor/test_packet.rs +++ b/validator-api/src/network_monitor/test_packet.rs @@ -8,6 +8,7 @@ use std::fmt::{self, Display, Formatter}; use std::hash::{Hash, Hasher}; use std::mem; use std::str::Utf8Error; +use topology::{gateway, mix}; #[repr(u8)] #[derive(Eq, PartialEq, Debug, Hash, Clone, Copy)] @@ -16,6 +17,12 @@ pub(crate) enum NodeType { Gateway = 1, } +impl NodeType { + pub(crate) fn is_mixnode(&self) -> bool { + matches!(self, NodeType::Mixnode) + } +} + impl TryFrom for NodeType { type Error = TestPacketError; @@ -31,7 +38,6 @@ impl TryFrom for NodeType { #[derive(Debug)] pub(crate) enum TestPacketError { IncompletePacket, - InvalidIpVersion, InvalidNodeType, InvalidNodeKey, InvalidOwner(Utf8Error), @@ -49,154 +55,120 @@ impl From for TestPacketError { } } -#[repr(u8)] -#[derive(PartialEq, Eq, Hash, Copy, Clone, Debug)] -pub(crate) enum IpVersion { - V4 = 4, - V6 = 6, -} - -impl TryFrom for IpVersion { - type Error = TestPacketError; - - fn try_from(value: u8) -> Result { - match value { - _ if value == (Self::V4 as u8) => Ok(Self::V4), - _ if value == (Self::V6 as u8) => Ok(Self::V6), - _ => Err(TestPacketError::InvalidIpVersion), - } - } -} - -impl IpVersion { - pub(crate) fn is_v4(&self) -> bool { - *self == IpVersion::V4 - } -} - -impl From for String { - fn from(ipv: IpVersion) -> Self { - format!("{}", ipv) - } -} - -impl Display for IpVersion { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "{}", *self as u8) - } -} - #[derive(Eq, Clone, Debug)] pub(crate) struct TestPacket { - ip_version: IpVersion, - nonce: u64, - pub_key: identity::PublicKey, - owner: String, - node_type: NodeType, + pub(crate) route_id: u64, + pub(crate) test_nonce: u64, + pub(crate) pub_key: identity::PublicKey, + pub(crate) owner: String, + pub(crate) node_type: NodeType, } impl Display for TestPacket { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { write!( f, - "TestPacket {{ ip: {}, pub_key: {}, owner: {}, nonce: {} }}", - self.ip_version, + "TestPacket {{ pub_key: {}, owner: {}, route: {} test nonce: {} }}", self.pub_key.to_base58_string(), self.owner, - self.nonce + self.route_id, + self.test_nonce ) } } impl Hash for TestPacket { fn hash(&self, state: &mut H) { - self.ip_version.hash(state); - self.nonce.hash(state); + self.route_id.hash(state); + self.test_nonce.hash(state); self.pub_key.to_bytes().hash(state); self.owner.hash(state); + self.node_type.hash(state); } } impl PartialEq for TestPacket { fn eq(&self, other: &Self) -> bool { - self.ip_version == other.ip_version - && self.nonce == other.nonce + self.route_id == other.route_id + && self.test_nonce == other.test_nonce && self.pub_key.to_bytes() == other.pub_key.to_bytes() + && self.owner == other.owner + && self.node_type == other.node_type } } impl TestPacket { - pub(crate) fn new_v4( + pub(crate) fn from_mixnode(mix: &mix::Node, route_id: u64, test_nonce: u64) -> Self { + TestPacket { + pub_key: mix.identity_key, + owner: mix.owner.clone(), + route_id, + test_nonce, + node_type: NodeType::Mixnode, + } + } + + pub(crate) fn from_gateway(gateway: &gateway::Node, route_id: u64, test_nonce: u64) -> Self { + TestPacket { + pub_key: gateway.identity_key, + owner: gateway.owner.clone(), + route_id, + test_nonce, + node_type: NodeType::Gateway, + } + } + + pub(crate) fn new( pub_key: identity::PublicKey, owner: String, - nonce: u64, + route_id: u64, + test_nonce: u64, node_type: NodeType, ) -> Self { TestPacket { - ip_version: IpVersion::V4, - nonce, + route_id, + test_nonce, pub_key, owner, node_type, } } - pub(crate) fn new_v6( - pub_key: identity::PublicKey, - owner: String, - nonce: u64, - node_type: NodeType, - ) -> Self { - TestPacket { - ip_version: IpVersion::V6, - nonce, - pub_key, - owner, - node_type, - } - } - - pub(crate) fn nonce(&self) -> u64 { - self.nonce - } - - pub(crate) fn ip_version(&self) -> IpVersion { - self.ip_version + pub(crate) fn test_nonce(&self) -> u64 { + self.test_nonce } pub(crate) fn to_bytes(&self) -> Vec { - self.nonce - .to_be_bytes() - .iter() - .cloned() + IntoIterator::into_iter(self.route_id.to_be_bytes()) + .chain(IntoIterator::into_iter(self.test_nonce.to_be_bytes())) .chain(std::iter::once(self.node_type as u8)) - .chain(std::iter::once(self.ip_version as u8)) .chain(self.pub_key.to_bytes().iter().cloned()) .chain(self.owner.as_bytes().iter().cloned()) .collect() } pub(crate) fn try_from_bytes(b: &[u8]) -> Result { - // nonce size + // route id + test nonce size let n = mem::size_of::(); - if b.len() < n + 1 + identity::PUBLIC_KEY_LENGTH { + if b.len() < 2 * n + 1 + identity::PUBLIC_KEY_LENGTH { return Err(TestPacketError::IncompletePacket); } - // this unwrap can't fail as we've already checked for the size - let nonce = u64::from_be_bytes(b[0..n].try_into().unwrap()); - let node_type = NodeType::try_from(b[n])?; + // those unwraps can't fail as we've already checked for the size + let route_id = u64::from_be_bytes(b[0..n].try_into().unwrap()); + let test_nonce = u64::from_be_bytes(b[n..2 * n].try_into().unwrap()); + let node_type = NodeType::try_from(b[2 * n])?; - let ip_version = IpVersion::try_from(b[n + 1])?; - let pub_key = - identity::PublicKey::from_bytes(&b[n + 2..n + 2 + identity::PUBLIC_KEY_LENGTH])?; - let owner = std::str::from_utf8(&b[n + 2 + identity::PUBLIC_KEY_LENGTH..])?; + let pub_key = identity::PublicKey::from_bytes( + &b[2 * n + 1..2 * n + 1 + identity::PUBLIC_KEY_LENGTH], + )?; + let owner = std::str::from_utf8(&b[2 * n + 1 + identity::PUBLIC_KEY_LENGTH..])?; Ok(TestPacket { + route_id, node_type, - ip_version, - nonce, + test_nonce, pub_key, owner: owner.to_owned(), }) @@ -212,3 +184,27 @@ impl From for TestedNode { } } } + +#[cfg(test)] +mod tests { + use super::*; + use rand::thread_rng; + + #[test] + fn test_packet_roundtrip() { + let mut rng = thread_rng(); + let dummy_keypair = identity::KeyPair::new(&mut rng); + let owner = "some owner".to_string(); + let packet = TestPacket::new( + *dummy_keypair.public_key(), + owner, + 42, + 123, + NodeType::Mixnode, + ); + + let bytes = packet.to_bytes(); + let recovered = TestPacket::try_from_bytes(&bytes).unwrap(); + assert_eq!(packet, recovered); + } +} diff --git a/validator-api/src/network_monitor/test_route/mod.rs b/validator-api/src/network_monitor/test_route/mod.rs new file mode 100644 index 0000000000..c947f8e001 --- /dev/null +++ b/validator-api/src/network_monitor/test_route/mod.rs @@ -0,0 +1,120 @@ +// Copyright 2021 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::network_monitor::test_packet::{NodeType, TestPacket}; +use crate::network_monitor::ROUTE_TESTING_TEST_NONCE; +use crypto::asymmetric::identity; +use std::fmt::{Debug, Formatter}; +use topology::{gateway, mix, NymTopology}; + +#[derive(Clone)] +pub(crate) struct TestRoute { + id: u64, + system_version: String, + nodes: NymTopology, +} + +impl TestRoute { + pub(crate) fn new( + id: u64, + system_version: &str, + l1_mix: mix::Node, + l2_mix: mix::Node, + l3_mix: mix::Node, + gateway: gateway::Node, + ) -> Self { + // When somebody gets to refactor this in the future and Rust 2021 is being used, + // the call could be changed to a simple `.into_iter()` + let layered_mixes = IntoIterator::into_iter([ + (1u8, vec![l1_mix]), + (2u8, vec![l2_mix]), + (3u8, vec![l3_mix]), + ]) + .collect(); + + TestRoute { + id, + system_version: system_version.to_string(), + nodes: NymTopology::new(layered_mixes, vec![gateway]), + } + } + + pub(crate) fn id(&self) -> u64 { + self.id + } + + pub(crate) fn gateway(&self) -> &gateway::Node { + &self.nodes.gateways()[0] + } + + pub(crate) fn layer_one_mix(&self) -> &mix::Node { + &self.nodes.mixes().get(&1).unwrap()[0] + } + + pub(crate) fn layer_two_mix(&self) -> &mix::Node { + &self.nodes.mixes().get(&2).unwrap()[0] + } + + pub(crate) fn layer_three_mix(&self) -> &mix::Node { + &self.nodes.mixes().get(&3).unwrap()[0] + } + + pub(crate) fn gateway_clients_address(&self) -> String { + self.gateway().clients_address() + } + + pub(crate) fn gateway_identity(&self) -> identity::PublicKey { + self.gateway().identity_key + } + + pub(crate) fn topology(&self) -> &NymTopology { + &self.nodes + } + + pub(crate) fn self_test_packet(&self) -> TestPacket { + // it doesn't really matter which node is "chosen" as the packet has to always + // go through the same sequence of hops. + // let's just use layer 1 mixnode for this (this choice is completely arbitrary) + let mix = &self.nodes.mixes()[&1][0]; + TestPacket::new( + mix.identity_key, + mix.owner.clone(), + self.id, + ROUTE_TESTING_TEST_NONCE, + NodeType::Mixnode, + ) + } + + pub(crate) fn substitute_mix(&self, node: &mix::Node) -> NymTopology { + let mut topology = self.nodes.clone(); + topology.set_mixes_in_layer(node.layer as u8, vec![node.clone()]); + topology + } + + pub(crate) fn substitute_gateway(&self, gateway: &gateway::Node) -> NymTopology { + let mut topology = self.nodes.clone(); + topology.set_gateways(vec![gateway.clone()]); + topology + } +} + +impl Debug for TestRoute { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + write!( + f, + "[v{}] Route {}: [G] {} => [M1] {} => [M2] {} => [M3] {}", + self.system_version, + self.id, + self.nodes.gateways()[0].identity().to_base58_string(), + self.nodes.mixes_in_layer(1)[0] + .identity_key + .to_base58_string(), + self.nodes.mixes_in_layer(2)[0] + .identity_key + .to_base58_string(), + self.nodes.mixes_in_layer(3)[0] + .identity_key + .to_base58_string() + ) + } +} diff --git a/validator-api/src/network_monitor/tested_network/good_topology.rs b/validator-api/src/network_monitor/tested_network/good_topology.rs deleted file mode 100644 index 9f26ab7c41..0000000000 --- a/validator-api/src/network_monitor/tested_network/good_topology.rs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2021 - Nym Technologies SA -// SPDX-License-Identifier: Apache-2.0 - -use mixnet_contract::{GatewayBond, MixNodeBond}; -use serde::Deserialize; -use std::fs; -use std::path::Path; -use topology::{nym_topology_from_bonds, NymTopology}; - -#[derive(Deserialize)] -struct GoodTopology { - mixnodes: Vec, - gateways: Vec, -} - -pub(crate) fn parse_topology_file>(file_path: P) -> NymTopology { - let file_content = - fs::read_to_string(file_path).expect("specified topology file does not exist"); - let good_topology = serde_json::from_str::(&file_content) - .expect("topology in specified file is malformed"); - let nym_topology = nym_topology_from_bonds(good_topology.mixnodes, good_topology.gateways); - if nym_topology.mixes().len() != 3 { - panic!("topology has different than 3 number of layers") - } - if nym_topology.gateways().is_empty() { - panic!("topology does not include a gateway") - } - - nym_topology -} diff --git a/validator-api/src/network_monitor/tested_network/mod.rs b/validator-api/src/network_monitor/tested_network/mod.rs deleted file mode 100644 index 6f04b6b3bb..0000000000 --- a/validator-api/src/network_monitor/tested_network/mod.rs +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2021 - Nym Technologies SA -// SPDX-License-Identifier: Apache-2.0 - -use crate::network_monitor::test_packet::IpVersion; -use mixnet_contract::{GatewayBond, MixNodeBond}; -use topology::{gateway, mix, NymTopology}; - -pub(crate) mod good_topology; - -#[derive(Clone)] -pub(crate) struct TestedNetwork { - system_version: String, - good_v4_topology: NymTopology, - good_v6_topology: NymTopology, -} - -impl TestedNetwork { - pub(crate) fn new_good(good_v4_topology: NymTopology, good_v6_topology: NymTopology) -> Self { - TestedNetwork { - system_version: env!("CARGO_PKG_VERSION").to_owned(), - good_v4_topology, - good_v6_topology, - } - } - - pub(crate) fn main_v4_gateway(&self) -> &gateway::Node { - if self.good_v4_topology.gateways().len() > 1 { - warn!("we have more than a single 'good' gateway and in few places we made assumptions that only a single one existed!") - } - - self.good_v4_topology - .gateways() - .get(0) - .expect("our good v4 topology does not have any gateway specified!") - } - - pub(crate) fn system_version(&self) -> &str { - &self.system_version - } - - pub(crate) fn substitute_mix(&self, node: mix::Node, ip_version: IpVersion) -> NymTopology { - let mut good_topology = match ip_version { - IpVersion::V4 => self.good_v4_topology.clone(), - IpVersion::V6 => self.good_v6_topology.clone(), - }; - - good_topology.set_mixes_in_layer(node.layer as u8, vec![node]); - good_topology - } - - pub(crate) fn substitute_gateway( - &self, - gateway: gateway::Node, - ip_version: IpVersion, - ) -> NymTopology { - let mut good_topology = match ip_version { - IpVersion::V4 => self.good_v4_topology.clone(), - IpVersion::V6 => self.good_v6_topology.clone(), - }; - - good_topology.set_gateways(vec![gateway]); - good_topology - } - - pub(crate) fn v4_topology(&self) -> &NymTopology { - &self.good_v4_topology - } - - pub(crate) fn v6_topology(&self) -> &NymTopology { - &self.good_v6_topology - } - - /// Given slices of bonded mixnodes and gateways, checks whether all 'good' nodes are present - /// in the lists. - /// - /// # Arguments - /// - /// * `bonded_mixnodes`: slice of currently bonded mixnodes - /// * `bonded_gateways`: slice of currently bonded gateways - pub(crate) fn is_online( - &self, - bonded_mixnodes: &[MixNodeBond], - bonded_gateways: &[GatewayBond], - ) -> bool { - // while technically this is not the most optimal way of checking all nodes as we have to - // go through entire slice multiple times, we only do it every 30s before monitor startup - // so it's not really that bad - for layer_mixes in self.good_v4_topology.mixes().values() { - for mix in layer_mixes { - if !bonded_mixnodes.iter().any(|bonded| { - bonded.mix_node.identity_key == mix.identity_key.to_base58_string() - }) { - return false; - } - } - } - - for gateway in self.good_v4_topology.gateways() { - if !bonded_gateways.iter().any(|bonded| { - bonded.gateway.identity_key == gateway.identity_key.to_base58_string() - }) { - return false; - } - } - - true - } -} diff --git a/validator-api/src/node_status_api/mod.rs b/validator-api/src/node_status_api/mod.rs index cb16bcf6d5..3316aec6f4 100644 --- a/validator-api/src/node_status_api/mod.rs +++ b/validator-api/src/node_status_api/mod.rs @@ -27,6 +27,8 @@ pub(crate) fn stage(database_path: PathBuf) -> AdHoc { routes::gateway_report, routes::mixnode_uptime_history, routes::gateway_uptime_history, + routes::mixnode_core_status_count, + routes::gateway_core_status_count, ], ) }) diff --git a/validator-api/src/node_status_api/models.rs b/validator-api/src/node_status_api/models.rs index 308e3a91d1..a4ff5f54e2 100644 --- a/validator-api/src/node_status_api/models.rs +++ b/validator-api/src/node_status_api/models.rs @@ -30,7 +30,21 @@ impl Uptime { return Ok(Self::zero()); } - let uptime = ((numerator as f32 / denominator as f32) * 100.0) as u8; + let uptime = ((numerator as f32 / denominator as f32) * 100.0).round() as u8; + + if uptime > 100 { + Err(InvalidUptime) + } else { + Ok(Uptime(uptime)) + } + } + + pub fn from_uptime_sum(running_sum: f32, count: usize) -> Result { + if count == 0 { + return Ok(Self::zero()); + } + + let uptime = (running_sum / count as f32).round() as u8; if uptime > 100 { Err(InvalidUptime) @@ -79,14 +93,10 @@ pub struct MixnodeStatusReport { pub(crate) identity: String, pub(crate) owner: String, - pub(crate) most_recent_ipv4: bool, - pub(crate) most_recent_ipv6: bool, + pub(crate) most_recent: Uptime, - pub(crate) last_hour_ipv4: Uptime, - pub(crate) last_hour_ipv6: Uptime, - - pub(crate) last_day_ipv4: Uptime, - pub(crate) last_day_ipv6: Uptime, + pub(crate) last_hour: Uptime, + pub(crate) last_day: Uptime, } impl MixnodeStatusReport { @@ -94,15 +104,13 @@ impl MixnodeStatusReport { report_time: OffsetDateTime, identity: String, owner: String, - last_day_ipv4: Vec, - last_day_ipv6: Vec, + last_day: Vec, last_hour_test_runs: usize, last_day_test_runs: usize, ) -> Self { let node_uptimes = NodeUptimes::calculate_from_last_day_reports( report_time, - last_day_ipv4, - last_day_ipv6, + last_day, last_hour_test_runs, last_day_test_runs, ); @@ -110,12 +118,9 @@ impl MixnodeStatusReport { MixnodeStatusReport { identity, owner, - most_recent_ipv4: node_uptimes.most_recent_ipv4, - most_recent_ipv6: node_uptimes.most_recent_ipv6, - last_hour_ipv4: node_uptimes.last_hour_ipv4, - last_hour_ipv6: node_uptimes.last_hour_ipv6, - last_day_ipv4: node_uptimes.last_day_ipv4, - last_day_ipv6: node_uptimes.last_day_ipv6, + most_recent: node_uptimes.most_recent, + last_hour: node_uptimes.last_hour, + last_day: node_uptimes.last_day, } } } @@ -125,14 +130,10 @@ pub struct GatewayStatusReport { pub(crate) identity: String, pub(crate) owner: String, - pub(crate) most_recent_ipv4: bool, - pub(crate) most_recent_ipv6: bool, + pub(crate) most_recent: Uptime, - pub(crate) last_hour_ipv4: Uptime, - pub(crate) last_hour_ipv6: Uptime, - - pub(crate) last_day_ipv4: Uptime, - pub(crate) last_day_ipv6: Uptime, + pub(crate) last_hour: Uptime, + pub(crate) last_day: Uptime, } impl GatewayStatusReport { @@ -140,15 +141,13 @@ impl GatewayStatusReport { report_time: OffsetDateTime, identity: String, owner: String, - last_day_ipv4: Vec, - last_day_ipv6: Vec, + last_day: Vec, last_hour_test_runs: usize, last_day_test_runs: usize, ) -> Self { let node_uptimes = NodeUptimes::calculate_from_last_day_reports( report_time, - last_day_ipv4, - last_day_ipv6, + last_day, last_hour_test_runs, last_day_test_runs, ); @@ -156,12 +155,9 @@ impl GatewayStatusReport { GatewayStatusReport { identity, owner, - most_recent_ipv4: node_uptimes.most_recent_ipv4, - most_recent_ipv6: node_uptimes.most_recent_ipv6, - last_hour_ipv4: node_uptimes.last_hour_ipv4, - last_hour_ipv6: node_uptimes.last_hour_ipv6, - last_day_ipv4: node_uptimes.last_day_ipv4, - last_day_ipv6: node_uptimes.last_day_ipv6, + most_recent: node_uptimes.most_recent, + last_hour: node_uptimes.last_hour, + last_day: node_uptimes.last_day, } } } @@ -208,8 +204,7 @@ pub struct HistoricalUptime { // I think this is more than enough, we don't need the uber precision of timezone offsets, etc pub(crate) date: String, - pub(crate) ipv4_uptime: Uptime, - pub(crate) ipv6_uptime: Uptime, + pub(crate) uptime: Uptime, } pub(crate) struct ErrorResponse { @@ -274,3 +269,9 @@ impl Display for ValidatorApiStorageError { } } } + +#[derive(Serialize)] +pub struct CoreNodeStatus { + pub(crate) identity: String, + pub(crate) count: i32, +} diff --git a/validator-api/src/node_status_api/routes.rs b/validator-api/src/node_status_api/routes.rs index 756f10a9a2..7b732a5977 100644 --- a/validator-api/src/node_status_api/routes.rs +++ b/validator-api/src/node_status_api/routes.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use crate::node_status_api::models::{ - ErrorResponse, GatewayStatusReport, GatewayUptimeHistory, MixnodeStatusReport, + CoreNodeStatus, ErrorResponse, GatewayStatusReport, GatewayUptimeHistory, MixnodeStatusReport, MixnodeUptimeHistory, }; use crate::storage::ValidatorApiStorage; @@ -57,3 +57,37 @@ pub(crate) async fn gateway_uptime_history( .map(Json) .map_err(|err| ErrorResponse::new(err, Status::NotFound)) } + +#[get("/mixnode//core-status-count?")] +pub(crate) async fn mixnode_core_status_count( + storage: &State, + pubkey: &str, + since: Option, +) -> Json { + let count = storage + .get_core_mixnode_status_count(pubkey, since) + .await + .unwrap_or_default(); + + Json(CoreNodeStatus { + identity: pubkey.to_string(), + count, + }) +} + +#[get("/gateway//core-status-count?")] +pub(crate) async fn gateway_core_status_count( + storage: &State, + pubkey: &str, + since: Option, +) -> Json { + let count = storage + .get_core_gateway_status_count(pubkey, since) + .await + .unwrap_or_default(); + + Json(CoreNodeStatus { + identity: pubkey.to_string(), + count, + }) +} diff --git a/validator-api/src/node_status_api/utils.rs b/validator-api/src/node_status_api/utils.rs index d9a6bcc72b..004fcebab4 100644 --- a/validator-api/src/node_status_api/utils.rs +++ b/validator-api/src/node_status_api/utils.rs @@ -5,100 +5,105 @@ use crate::node_status_api::models::Uptime; use crate::node_status_api::{FIFTEEN_MINUTES, ONE_HOUR}; use crate::storage::models::NodeStatus; use log::warn; -use std::cmp::min; +use std::convert::TryInto; use time::OffsetDateTime; // A temporary helper struct used to produce reports for active nodes. -pub(crate) struct ActiveNodeDayStatuses { +pub(crate) struct ActiveNodeStatuses { pub(crate) identity: String, pub(crate) owner: String, - pub(crate) ipv4_statuses: Vec, - pub(crate) ipv6_statuses: Vec, + pub(crate) statuses: Vec, } // A helper intermediate struct to remove duplicate code for construction of mixnode and gateway reports pub(crate) struct NodeUptimes { - pub(crate) most_recent_ipv4: bool, - pub(crate) most_recent_ipv6: bool, + pub(crate) most_recent: Uptime, - pub(crate) last_hour_ipv4: Uptime, - pub(crate) last_hour_ipv6: Uptime, - - pub(crate) last_day_ipv4: Uptime, - pub(crate) last_day_ipv6: Uptime, + pub(crate) last_hour: Uptime, + pub(crate) last_day: Uptime, } impl NodeUptimes { pub(crate) fn calculate_from_last_day_reports( report_time: OffsetDateTime, - last_day_ipv4: Vec, - last_day_ipv6: Vec, + last_day: Vec, last_hour_test_runs: usize, last_day_test_runs: usize, ) -> Self { let hour_ago = (report_time - ONE_HOUR).unix_timestamp(); let fifteen_minutes_ago = (report_time - FIFTEEN_MINUTES).unix_timestamp(); - let mut ipv4_day_up = last_day_ipv4.iter().filter(|report| report.up).count(); - let mut ipv6_day_up = last_day_ipv6.iter().filter(|report| report.up).count(); - - let mut ipv4_hour_up = last_day_ipv4 - .iter() - .filter(|report| report.up && report.timestamp >= hour_ago) - .count(); - let mut ipv6_hour_up = last_day_ipv6 - .iter() - .filter(|report| report.up && report.timestamp >= hour_ago) - .count(); - - // most recent status MUST BE within last 15min - let most_recent_ipv4 = last_day_ipv4 - .iter() - .max_by_key(|report| report.timestamp) // find the most recent - .map(|status| status.timestamp >= fifteen_minutes_ago && status.up) // make sure its within last 15min - .unwrap_or_default(); - let most_recent_ipv6 = last_day_ipv6 - .iter() - .max_by_key(|report| report.timestamp) // find the most recent - .map(|status| status.timestamp >= fifteen_minutes_ago && status.up) // make sure its within last 15min - .unwrap_or_default(); - - // If somehow we have more "up" reports than the actual test runs it means something weird is going on + // If somehow we have more reports than the actual test runs it means something weird is going on // (or we just started running this code on old data, so if it appears for first 24h, it's fine and actually expected // as we would not have any run information from the past) // Either way, bound the the number of "up" reports by number of test runs and log warnings // if that happens - if ipv4_hour_up > last_hour_test_runs || ipv6_hour_up > last_hour_test_runs { - warn!( - "We have more 'up' reports than the actual number of test runs in last hour! ({} ipv4 'ups', {} ipv6 'ups' for {} test runs)", - ipv4_hour_up, - ipv6_hour_up, - last_hour_test_runs, - ); - ipv4_hour_up = min(ipv4_hour_up, last_hour_test_runs); - ipv6_hour_up = min(ipv6_hour_up, last_hour_test_runs); - } - if ipv4_day_up > last_day_test_runs || ipv6_day_up > last_day_test_runs { + let last_day_sum: f32 = if last_day.len() > last_day_test_runs { warn!( - "We have more 'up' reports than the actual number of test runs in last day! ({} ipv4 'ups', {} ipv6 'ups' for {} test runs)", - ipv4_day_up, - ipv6_day_up, + "We have more reports than the actual number of test runs in last day! ({} reports for {} test runs)", + last_day.len(), last_day_test_runs, ); - ipv4_day_up = min(ipv4_day_up, last_day_test_runs); - ipv6_day_up = min(ipv6_day_up, last_day_test_runs); - } + last_day + .iter() + .take(last_day_test_runs) + .map(|report| report.reliability as f32) + .sum() + } else { + // we average over expected number of test runs so if a node was not online for some of them + // it's treated as if it had a "zero" status. + last_day + .iter() + .map(|report| report.reliability as f32) + .sum() + }; + + let last_hour_reports = last_day + .iter() + .filter(|report| report.timestamp >= hour_ago) + .count(); + + let last_hour_sum: f32 = if last_hour_reports > last_hour_test_runs { + warn!( + "We have more reports than the actual number of test runs in last hour! ({} reports for {} test runs)", + last_hour_reports, + last_hour_test_runs, + ); + last_day + .iter() + .filter(|report| report.timestamp >= hour_ago) + .take(last_hour_test_runs) + .map(|report| report.reliability as f32) + .sum() + } else { + last_day + .iter() + .filter(|report| report.timestamp >= hour_ago) + .map(|report| report.reliability as f32) + .sum() + }; + + // find the most recent + let most_recent_report = last_day.iter().max_by_key(|report| report.timestamp); + + let most_recent = if let Some(most_recent_report) = most_recent_report { + // make sure its within last 15min + if most_recent_report.timestamp >= fifteen_minutes_ago { + most_recent_report.reliability + } else { + 0 + } + } else { + 0 + }; // the unwraps in Uptime::from_ratio are fine because it's impossible for us to have more "up" results // than total test runs as we just bounded them NodeUptimes { - most_recent_ipv4, - most_recent_ipv6, - last_hour_ipv4: Uptime::from_ratio(ipv4_hour_up, last_hour_test_runs).unwrap(), - last_hour_ipv6: Uptime::from_ratio(ipv6_hour_up, last_hour_test_runs).unwrap(), - last_day_ipv4: Uptime::from_ratio(ipv4_day_up, last_day_test_runs).unwrap(), - last_day_ipv6: Uptime::from_ratio(ipv6_day_up, last_day_test_runs).unwrap(), + most_recent: most_recent.try_into().unwrap(), + last_hour: Uptime::from_uptime_sum(last_hour_sum, last_hour_test_runs).unwrap(), + last_day: Uptime::from_uptime_sum(last_day_sum, last_hour_test_runs).unwrap(), } } } diff --git a/validator-api/src/rewarding/mod.rs b/validator-api/src/rewarding/mod.rs index 81d6aaacae..e1bac6fe73 100644 --- a/validator-api/src/rewarding/mod.rs +++ b/validator-api/src/rewarding/mod.rs @@ -14,7 +14,7 @@ use crate::storage::ValidatorApiStorage; use log::{error, info}; use mixnet_contract::{ExecuteMsg, IdentityKey}; use std::collections::HashMap; -use std::convert::{TryFrom, TryInto}; +use std::convert::TryInto; use std::time::Duration; use time::OffsetDateTime; use tokio::time::sleep; @@ -159,21 +159,6 @@ impl Rewarder { Ok(map) } - /// Calculates the absolute uptime of given node that is then passed as one of the arguments - /// in the smart contract to determine the actual reward value. - /// - /// Currently both ipv4 and ipv6 uptimes carry the same weight in the calculation. - /// - /// # Arguments - /// - /// * `ipv4_uptime`: ipv4 uptime of the node in the last epoch. - /// * `ipv6_uptime`: ipv6 uptime of the node in the last epoch. - fn calculate_absolute_uptime(&self, ipv4_uptime: Uptime, ipv6_uptime: Uptime) -> Uptime { - // just take average of ipv4 and ipv6 uptimes using equal weights - let abs = ((ipv4_uptime.u8() as f32 + ipv6_uptime.u8() as f32) / 2.0).round(); - Uptime::try_from(abs as i64).unwrap() - } - /// Given the list of mixnodes that were tested in the last epoch, tries to determine the /// subset that are eligible for any rewards. /// @@ -212,8 +197,7 @@ impl Rewarder { .get(&mix.identity) .map(|&total_delegations| MixnodeToReward { identity: mix.identity.clone(), - uptime: self - .calculate_absolute_uptime(mix.last_day_ipv4, mix.last_day_ipv6), + uptime: mix.last_day, total_delegations, }) }) diff --git a/validator-api/src/storage/manager.rs b/validator-api/src/storage/manager.rs index a31816f45e..d746c846bd 100644 --- a/validator-api/src/storage/manager.rs +++ b/validator-api/src/storage/manager.rs @@ -3,10 +3,10 @@ use crate::network_monitor::monitor::summary_producer::NodeResult; use crate::node_status_api::models::{HistoricalUptime, Uptime}; -use crate::node_status_api::utils::ActiveNodeDayStatuses; +use crate::node_status_api::utils::ActiveNodeStatuses; use crate::storage::models::{ ActiveNode, EpochRewarding, FailedMixnodeRewardChunk, NodeStatus, PossiblyUnrewardedMixnode, - RewardingReport, + RewardingReport, TestingRoute, }; use crate::storage::UnixTimestamp; use std::convert::TryFrom; @@ -18,7 +18,11 @@ pub(crate) struct StorageManager { // all SQL goes here impl StorageManager { - /// Tries to obtain row id of given mixnode given its identity + /// Tries to obtain row id of given mixnode given its identity. + /// + /// # Arguments + /// + /// * `identity`: identity (base58-encoded public key) of the mixnode. pub(super) async fn get_mixnode_id(&self, identity: &str) -> Result, sqlx::Error> { let id = sqlx::query!( "SELECT id FROM mixnode_details WHERE identity = ?", @@ -32,6 +36,10 @@ impl StorageManager { } /// Tries to obtain row id of given gateway given its identity + /// + /// # Arguments + /// + /// * `identity`: identity (base58-encoded public key) of the gateway. pub(super) async fn get_gateway_id(&self, identity: &str) -> Result, sqlx::Error> { let id = sqlx::query!( "SELECT id FROM gateway_details WHERE identity = ?", @@ -45,6 +53,10 @@ impl StorageManager { } /// Tries to obtain owner value of given mixnode given its identity + /// + /// # Arguments + /// + /// * `identity`: identity (base58-encoded public key) of the mixnode. pub(super) async fn get_mixnode_owner( &self, identity: &str, @@ -61,6 +73,10 @@ impl StorageManager { } /// Tries to obtain owner value of given gateway given its identity + /// + /// # Arguments + /// + /// * `identity`: identity (base58-encoded public key) of the gateway. pub(super) async fn get_gateway_owner( &self, identity: &str, @@ -76,9 +92,14 @@ impl StorageManager { Ok(owner) } - /// Gets all ipv4 statuses for mixnode with particular identity that were inserted + /// Gets all reliability statuses for mixnode with particular identity that were inserted /// into the database after the specified unix timestamp. - pub(super) async fn get_mixnode_ipv4_statuses_since( + /// + /// # Arguments + /// + /// * `identity`: identity (base58-encoded public key) of the mixnode. + /// * `timestamp`: unix timestamp of the lower bound of the selection. + pub(super) async fn get_mixnode_statuses_since( &self, identity: &str, timestamp: UnixTimestamp, @@ -86,11 +107,11 @@ impl StorageManager { sqlx::query_as!( NodeStatus, r#" - SELECT timestamp, up - FROM mixnode_ipv4_status + SELECT timestamp, reliability as "reliability: u8" + FROM mixnode_status JOIN mixnode_details - ON mixnode_ipv4_status.mixnode_details_id = mixnode_details.id - WHERE mixnode_details.identity=? AND mixnode_ipv4_status.timestamp > ?; + ON mixnode_status.mixnode_details_id = mixnode_details.id + WHERE mixnode_details.identity=? AND mixnode_status.timestamp > ?; "#, identity, timestamp, @@ -99,9 +120,14 @@ impl StorageManager { .await } - /// Gets all ipv6 statuses for mixnode with particular identity that were inserted + /// Gets all reliability statuses for gateway with particular identity that were inserted /// into the database after the specified unix timestamp. - pub(super) async fn get_mixnode_ipv6_statuses_since( + /// + /// # Arguments + /// + /// * `identity`: identity (base58-encoded public key) of the gateway. + /// * `timestamp`: unix timestamp of the lower bound of the selection. + pub(super) async fn get_gateway_statuses_since( &self, identity: &str, timestamp: UnixTimestamp, @@ -109,34 +135,11 @@ impl StorageManager { sqlx::query_as!( NodeStatus, r#" - SELECT timestamp, up - FROM mixnode_ipv6_status - JOIN mixnode_details - ON mixnode_ipv6_status.mixnode_details_id = mixnode_details.id - WHERE mixnode_details.identity=? AND mixnode_ipv6_status.timestamp > ?; - "#, - identity, - timestamp - ) - .fetch_all(&self.connection_pool) - .await - } - - /// Gets all ipv4 statuses for gateway with particular identity that were inserted - /// into the database after the specified unix timestamp. - pub(super) async fn get_gateway_ipv4_statuses_since( - &self, - identity: &str, - timestamp: UnixTimestamp, - ) -> Result, sqlx::Error> { - sqlx::query_as!( - NodeStatus, - r#" - SELECT timestamp, up - FROM gateway_ipv4_status + SELECT timestamp, reliability as "reliability: u8" + FROM gateway_status JOIN gateway_details - ON gateway_ipv4_status.gateway_details_id = gateway_details.id - WHERE gateway_details.identity=? AND gateway_ipv4_status.timestamp > ?; + ON gateway_status.gateway_details_id = gateway_details.id + WHERE gateway_details.identity=? AND gateway_status.timestamp > ?; "#, identity, timestamp, @@ -145,37 +148,18 @@ impl StorageManager { .await } - /// Gets all ipv6 statuses for gateway with particular identity that were inserted - /// into the database after the specified unix timestamp. - pub(super) async fn get_gateway_ipv6_statuses_since( - &self, - identity: &str, - timestamp: UnixTimestamp, - ) -> Result, sqlx::Error> { - sqlx::query_as!( - NodeStatus, - r#" - SELECT timestamp, up - FROM gateway_ipv6_status - JOIN gateway_details - ON gateway_ipv6_status.gateway_details_id = gateway_details.id - WHERE gateway_details.identity=? AND gateway_ipv6_status.timestamp > ?; - "#, - identity, - timestamp - ) - .fetch_all(&self.connection_pool) - .await - } - /// Gets the historical daily uptime associated with the particular mixnode + /// + /// # Arguments + /// + /// * `identity`: identity (base58-encoded public key) of the mixnode. pub(super) async fn get_mixnode_historical_uptimes( &self, identity: &str, ) -> Result, sqlx::Error> { let uptimes = sqlx::query!( r#" - SELECT date, ipv4_uptime, ipv6_uptime + SELECT date, uptime FROM mixnode_historical_uptime JOIN mixnode_details ON mixnode_historical_uptime.mixnode_details_id = mixnode_details.id @@ -190,18 +174,12 @@ impl StorageManager { // filter out nodes with valid uptime (in theory all should be 100% valid since we insert them ourselves, but // better safe than sorry and not use an unwrap) .filter_map(|row| { - Uptime::try_from(row.ipv4_uptime) - .ok() - .map(|ipv4_uptime| { - Uptime::try_from(row.ipv6_uptime) - .ok() - .map(|ipv6_uptime| HistoricalUptime { - date: row.date, - ipv4_uptime, - ipv6_uptime, - }) + Uptime::try_from(row.uptime) + .map(|uptime| HistoricalUptime { + date: row.date, + uptime, }) - .flatten() + .ok() }) .collect(); @@ -209,13 +187,17 @@ impl StorageManager { } /// Gets the historical daily uptime associated with the particular gateway + /// + /// # Arguments + /// + /// * `identity`: identity (base58-encoded public key) of the gateway. pub(super) async fn get_gateway_historical_uptimes( &self, identity: &str, ) -> Result, sqlx::Error> { let uptimes = sqlx::query!( r#" - SELECT date, ipv4_uptime, ipv6_uptime + SELECT date, uptime FROM gateway_historical_uptime JOIN gateway_details ON gateway_historical_uptime.gateway_details_id = gateway_details.id @@ -230,32 +212,26 @@ impl StorageManager { // filter out nodes with valid uptime (in theory all should be 100% valid since we insert them ourselves, but // better safe than sorry and not use an unwrap) .filter_map(|row| { - Uptime::try_from(row.ipv4_uptime) - .ok() - .map(|ipv4_uptime| { - Uptime::try_from(row.ipv6_uptime) - .ok() - .map(|ipv6_uptime| HistoricalUptime { - date: row.date, - ipv4_uptime, - ipv6_uptime, - }) + Uptime::try_from(row.uptime) + .map(|uptime| HistoricalUptime { + date: row.date, + uptime, }) - .flatten() + .ok() }) .collect(); Ok(uptimes) } - /// Gets all ipv4 statuses for mixnode with particular id that were inserted + /// Gets all reliability statuses for mixnode with particular id that were inserted /// into the database within the specified time interval. /// /// # Arguments /// /// * `since`: unix timestamp indicating the lower bound interval of the selection. /// * `until`: unix timestamp indicating the upper bound interval of the selection. - pub(super) async fn get_mixnode_ipv4_statuses_by_id( + pub(super) async fn get_mixnode_statuses_by_id( &self, id: i64, since: UnixTimestamp, @@ -264,8 +240,8 @@ impl StorageManager { sqlx::query_as!( NodeStatus, r#" - SELECT timestamp, up - FROM mixnode_ipv4_status + SELECT timestamp, reliability as "reliability: u8" + FROM mixnode_status WHERE mixnode_details_id=? AND timestamp > ? AND timestamp < ?; "#, id, @@ -276,14 +252,14 @@ impl StorageManager { .await } - /// Gets all ipv6 statuses for mixnode with particular id that were inserted + /// Gets all reliability statuses for gateway with particular id that were inserted /// into the database within the specified time interval. /// /// # Arguments /// /// * `since`: unix timestamp indicating the lower bound interval of the selection. /// * `until`: unix timestamp indicating the upper bound interval of the selection. - pub(super) async fn get_mixnode_ipv6_statuses_by_id( + pub(super) async fn get_gateway_statuses_by_id( &self, id: i64, since: UnixTimestamp, @@ -292,64 +268,8 @@ impl StorageManager { sqlx::query_as!( NodeStatus, r#" - SELECT timestamp, up - FROM mixnode_ipv6_status - WHERE mixnode_details_id=? AND timestamp > ? AND timestamp < ?; - "#, - id, - since, - until, - ) - .fetch_all(&self.connection_pool) - .await - } - - /// Gets all ipv4 statuses for gateway with particular id that were inserted - /// into the database within the specified time interval. - /// - /// # Arguments - /// - /// * `since`: unix timestamp indicating the lower bound interval of the selection. - /// * `until`: unix timestamp indicating the upper bound interval of the selection. - pub(super) async fn get_gateway_ipv4_statuses_by_id( - &self, - id: i64, - since: UnixTimestamp, - until: UnixTimestamp, - ) -> Result, sqlx::Error> { - sqlx::query_as!( - NodeStatus, - r#" - SELECT timestamp, up - FROM gateway_ipv4_status - WHERE gateway_details_id=? AND timestamp > ? AND timestamp < ?; - "#, - id, - since, - until, - ) - .fetch_all(&self.connection_pool) - .await - } - - /// Gets all ipv6 statuses for gateway with particular id that were inserted - /// into the database within the specified time interval. - /// - /// # Arguments - /// - /// * `since`: unix timestamp indicating the lower bound interval of the selection. - /// * `until`: unix timestamp indicating the upper bound interval of the selection. - pub(super) async fn get_gateway_ipv6_statuses_by_id( - &self, - id: i64, - since: UnixTimestamp, - until: UnixTimestamp, - ) -> Result, sqlx::Error> { - sqlx::query_as!( - NodeStatus, - r#" - SELECT timestamp, up - FROM gateway_ipv6_status + SELECT timestamp, reliability as "reliability: u8" + FROM gateway_status WHERE gateway_details_id=? AND timestamp > ? AND timestamp < ?; "#, id, @@ -361,6 +281,11 @@ impl StorageManager { } /// Tries to submit mixnode [`NodeResult`] from the network monitor to the database. + /// + /// # Arguments + /// + /// * `timestamp`: unix timestamp indicating when the measurements took place. + /// * `mixnode_results`: reliability results of each node that got tested. pub(super) async fn submit_mixnode_statuses( &self, timestamp: UnixTimestamp, @@ -383,27 +308,15 @@ impl StorageManager { .await? .id; - // insert ipv4 status + // insert the actual status sqlx::query!( - r#" - INSERT INTO mixnode_ipv4_status (mixnode_details_id, up, timestamp) VALUES (?, ?, ?); - "#, - mixnode_id, - mixnode_result.working_ipv4, - timestamp - ) - .execute(&mut tx) - .await?; - - // insert ipv6 status - sqlx::query!( - r#" - INSERT INTO mixnode_ipv6_status (mixnode_details_id, up, timestamp) VALUES (?, ?, ?); - "#, - mixnode_id, - mixnode_result.working_ipv6, - timestamp - ) + r#" + INSERT INTO mixnode_status (mixnode_details_id, reliability, timestamp) VALUES (?, ?, ?); + "#, + mixnode_id, + mixnode_result.reliability, + timestamp + ) .execute(&mut tx) .await?; } @@ -413,6 +326,11 @@ impl StorageManager { } /// Tries to submit gateway [`NodeResult`] from the network monitor to the database. + /// + /// # Arguments + /// + /// * `timestamp`: unix timestamp indicating when the measurements took place. + /// * `gateway_results`: reliability results of each node that got tested. pub(super) async fn submit_gateway_statuses( &self, timestamp: UnixTimestamp, @@ -439,27 +357,15 @@ impl StorageManager { .await? .id; - // insert ipv4 status + // insert the actual status sqlx::query!( - r#" - INSERT INTO gateway_ipv4_status (gateway_details_id, up, timestamp) VALUES (?, ?, ?); - "#, - gateway_id, - gateway_result.working_ipv4, - timestamp - ) - .execute(&mut tx) - .await?; - - // insert ipv6 status - sqlx::query!( - r#" - INSERT INTO gateway_ipv6_status (gateway_details_id, up, timestamp) VALUES (?, ?, ?); - "#, - gateway_id, - gateway_result.working_ipv6, - timestamp - ) + r#" + INSERT INTO gateway_status (gateway_details_id, reliability, timestamp) VALUES (?, ?, ?); + "#, + gateway_id, + gateway_result.reliability, + timestamp + ) .execute(&mut tx) .await?; } @@ -468,6 +374,110 @@ impl StorageManager { tx.commit().await } + /// Saves the information about which nodes were used as core nodes during this particular + /// network monitor test run. + /// + /// # Arguments + /// + /// * `testing_route`: test route used for this particular network monitor run. + pub(super) async fn submit_testing_route_used( + &self, + testing_route: TestingRoute, + ) -> Result<(), sqlx::Error> { + sqlx::query!( + r#" + INSERT INTO testing_route + (gateway_id, layer1_mix_id, layer2_mix_id, layer3_mix_id, monitor_run_id) + VALUES (?, ?, ?, ?, ?); + "#, + testing_route.gateway_id, + testing_route.layer1_mix_id, + testing_route.layer2_mix_id, + testing_route.layer3_mix_id, + testing_route.monitor_run_id, + ) + .execute(&self.connection_pool) + .await?; + Ok(()) + } + + /// Get the number of times mixnode with the particular id is present in any `testing_route` + /// since the provided unix timestamp. + /// + /// # Arguments + /// + /// * `mixnode_id`: id (as saved in the database) of the mixnode. + /// * `since`: unix timestamp indicating the lower bound interval of the selection. + pub(super) async fn get_mixnode_testing_route_presence_count_since( + &self, + mixnode_id: i64, + since: UnixTimestamp, + ) -> Result { + let count = sqlx::query!( + r#" + SELECT COUNT(*) as count FROM + ( + SELECT monitor_run_id + FROM testing_route + WHERE testing_route.layer1_mix_id = ? OR testing_route.layer2_mix_id = ? OR testing_route.layer3_mix_id = ? + ) testing_route + JOIN + ( + SELECT id + FROM monitor_run + WHERE monitor_run.timestamp > ? + ) monitor_run + ON monitor_run.id = testing_route.monitor_run_id; + "#, + mixnode_id, + mixnode_id, + mixnode_id, + since, + ).fetch_one(&self.connection_pool) + .await? + .count; + + Ok(count) + } + + /// Get the number of times gateway with the particular id is present in any `testing_route` + /// since the provided unix timestamp. + /// + /// # Arguments + /// + /// * `gateway_id`: id (as saved in the database) of the gateway. + /// * `since`: unix timestamp indicating the lower bound interval of the selection. + pub(super) async fn get_gateway_testing_route_presence_count_since( + &self, + gateway_id: i64, + since: UnixTimestamp, + ) -> Result { + let count = sqlx::query!( + r#" + SELECT COUNT(*) as count FROM + ( + SELECT monitor_run_id + FROM testing_route + WHERE testing_route.gateway_id = ? + ) testing_route + JOIN + ( + SELECT id + FROM monitor_run + WHERE monitor_run.timestamp > ? + ) monitor_run + ON monitor_run.id = testing_route.monitor_run_id; + "#, + gateway_id, + since, + ) + .fetch_one(&self.connection_pool) + .await? + .count; + + Ok(count) + } + /// Checks whether there are already any historical uptimes with this particular date. pub(super) async fn check_for_historical_uptime_existence( &self, @@ -483,42 +493,51 @@ impl StorageManager { } /// Creates new entry for mixnode historical uptime + /// + /// # Arguments + /// + /// * `node_id`: id of the mixnode (as inserted in `mixnode_details_id` table). + /// * `date`: date associated with the uptime represented in ISO 8601, i.e. YYYY-MM-DD. + /// * `uptime`: the actual uptime of the node during the specified day. pub(super) async fn insert_mixnode_historical_uptime( &self, node_id: i64, date: &str, - ipv4_uptime: u8, - ipv6_uptime: u8, + uptime: u8, ) -> Result<(), sqlx::Error> { sqlx::query!( - "INSERT INTO mixnode_historical_uptime(mixnode_details_id, date, ipv4_uptime, ipv6_uptime) VALUES (?, ?, ?, ?)", - node_id, + "INSERT INTO mixnode_historical_uptime(mixnode_details_id, date, uptime) VALUES (?, ?, ?)", + node_id, date, - ipv4_uptime, - ipv6_uptime, + uptime, ).execute(&self.connection_pool).await?; Ok(()) } /// Creates new entry for gateway historical uptime + /// + /// # Arguments + /// + /// * `node_id`: id of the gateway (as inserted in `gateway_details_id` table). + /// * `date`: date associated with the uptime represented in ISO 8601, i.e. YYYY-MM-DD. + /// * `uptime`: the actual uptime of the node during the specified day. pub(super) async fn insert_gateway_historical_uptime( &self, node_id: i64, date: &str, - ipv4_uptime: u8, - ipv6_uptime: u8, + uptime: u8, ) -> Result<(), sqlx::Error> { sqlx::query!( - "INSERT INTO gateway_historical_uptime(gateway_details_id, date, ipv4_uptime, ipv6_uptime) VALUES (?, ?, ?, ?)", - node_id, + "INSERT INTO gateway_historical_uptime(gateway_details_id, date, uptime) VALUES (?, ?, ?)", + node_id, date, - ipv4_uptime, - ipv6_uptime, + uptime, ).execute(&self.connection_pool).await?; Ok(()) } /// Creates a database entry for a finished network monitor test run. + /// Returns id of the newly created entry. /// /// # Arguments /// @@ -526,11 +545,11 @@ impl StorageManager { pub(super) async fn insert_monitor_run( &self, timestamp: UnixTimestamp, - ) -> Result<(), sqlx::Error> { - sqlx::query!("INSERT INTO monitor_run(timestamp) VALUES (?)", timestamp) + ) -> Result { + let res = sqlx::query!("INSERT INTO monitor_run(timestamp) VALUES (?)", timestamp) .execute(&self.connection_pool) .await?; - Ok(()) + Ok(res.last_insert_rowid()) } /// Obtains number of network monitor test runs that have occurred within the specified interval. @@ -555,83 +574,39 @@ impl StorageManager { Ok(count) } - /// Removes all ipv4 statuses for all mixnodes that are older than the + /// Removes all statuses for all mixnodes that are older than the /// provided timestamp. This method is indirectly called at every reward cycle. /// /// # Arguments /// /// * `until`: timestamp specifying the purge cutoff. - pub(super) async fn purge_old_mixnode_ipv4_statuses( + pub(super) async fn purge_old_mixnode_statuses( &self, timestamp: UnixTimestamp, ) -> Result<(), sqlx::Error> { - sqlx::query!( - "DELETE FROM mixnode_ipv4_status WHERE timestamp < ?", - timestamp - ) - .execute(&self.connection_pool) - .await?; + sqlx::query!("DELETE FROM mixnode_status WHERE timestamp < ?", timestamp) + .execute(&self.connection_pool) + .await?; Ok(()) } - /// Removes all ipv6 statuses for all mixnodes that are older than the + /// Removes all statuses for all gateways that are older than the /// provided timestamp. This method is indirectly called at every reward cycle. /// /// # Arguments /// /// * `until`: timestamp specifying the purge cutoff. - pub(super) async fn purge_old_mixnode_ipv6_statuses( + pub(super) async fn purge_old_gateway_statuses( &self, timestamp: UnixTimestamp, ) -> Result<(), sqlx::Error> { - sqlx::query!( - "DELETE FROM mixnode_ipv6_status WHERE timestamp < ?", - timestamp - ) - .execute(&self.connection_pool) - .await?; + sqlx::query!("DELETE FROM gateway_status WHERE timestamp < ?", timestamp) + .execute(&self.connection_pool) + .await?; Ok(()) } - /// Removes all ipv4 statuses for all gateways that are older than the - /// provided timestamp. This method is indirectly called at every reward cycle. - /// - /// # Arguments - /// - /// * `until`: timestamp specifying the purge cutoff. - pub(super) async fn purge_old_gateway_ipv4_statuses( - &self, - timestamp: UnixTimestamp, - ) -> Result<(), sqlx::Error> { - sqlx::query!( - "DELETE FROM gateway_ipv4_status WHERE timestamp < ?", - timestamp - ) - .execute(&self.connection_pool) - .await?; - Ok(()) - } - - /// Removes all ipv6 statuses for all gateways that are older than the - /// provided timestamp. This method is indirectly called at every reward cycle. - /// - /// # Arguments - /// - /// * `until`: timestamp specifying the purge cutoff. - pub(super) async fn purge_old_gateway_ipv6_statuses( - &self, - timestamp: UnixTimestamp, - ) -> Result<(), sqlx::Error> { - sqlx::query!( - "DELETE FROM gateway_ipv6_status WHERE timestamp < ?", - timestamp - ) - .execute(&self.connection_pool) - .await?; - Ok(()) - } - - /// Returns public key, owner and id of all mixnodes that have had any ipv4 statuses submitted + /// Returns public key, owner and id of all mixnodes that have had any statuses submitted /// within the provided time interval. /// /// # Arguments @@ -643,7 +618,7 @@ impl StorageManager { since: UnixTimestamp, until: UnixTimestamp, ) -> Result, sqlx::Error> { - // find mixnode details of all nodes that have had at least 1 ipv4 status since the provided + // find mixnode details of all nodes that have had at least 1 status information since the provided // timestamp // TODO: I dont know if theres a potential issue of if we have a lot of inactive nodes that // haven't mixed in ages, they might increase the query times? @@ -652,10 +627,10 @@ impl StorageManager { r#" SELECT DISTINCT identity, owner, id FROM mixnode_details - JOIN mixnode_ipv4_status - ON mixnode_details.id = mixnode_ipv4_status.mixnode_details_id + JOIN mixnode_status + ON mixnode_details.id = mixnode_status.mixnode_details_id WHERE EXISTS ( - SELECT 1 FROM mixnode_ipv4_status WHERE timestamp > ? AND timestamp < ? + SELECT 1 FROM mixnode_status WHERE timestamp > ? AND timestamp < ? ) "#, since, @@ -665,7 +640,7 @@ impl StorageManager { .await } - /// Returns public key, owner and id of all gateways that have had any ipv4 statuses submitted + /// Returns public key, owner and id of all gateways that have had any statuses submitted /// within the provided time interval. /// /// # Arguments @@ -682,10 +657,10 @@ impl StorageManager { r#" SELECT DISTINCT identity, owner, id FROM gateway_details - JOIN gateway_ipv4_status - ON gateway_details.id = gateway_ipv4_status.gateway_details_id + JOIN gateway_status + ON gateway_details.id = gateway_status.gateway_details_id WHERE EXISTS ( - SELECT 1 FROM gateway_ipv4_status WHERE timestamp > ? AND timestamp < ? + SELECT 1 FROM gateway_status WHERE timestamp > ? AND timestamp < ? ) "#, since, @@ -854,25 +829,21 @@ impl StorageManager { &self, since: UnixTimestamp, until: UnixTimestamp, - ) -> Result, sqlx::Error> { + ) -> Result, sqlx::Error> { let active_nodes = self .get_all_active_mixnodes_in_interval(since, until) .await?; let mut active_day_statuses = Vec::with_capacity(active_nodes.len()); for active_node in active_nodes.into_iter() { - let ipv4_statuses = self - .get_mixnode_ipv4_statuses_by_id(active_node.id, since, until) - .await?; - let ipv6_statuses = self - .get_mixnode_ipv6_statuses_by_id(active_node.id, since, until) + let statuses = self + .get_mixnode_statuses_by_id(active_node.id, since, until) .await?; - let statuses = ActiveNodeDayStatuses { + let statuses = ActiveNodeStatuses { identity: active_node.identity, owner: active_node.owner, - ipv4_statuses, - ipv6_statuses, + statuses, }; active_day_statuses.push(statuses); @@ -891,25 +862,21 @@ impl StorageManager { &self, since: UnixTimestamp, until: UnixTimestamp, - ) -> Result, sqlx::Error> { + ) -> Result, sqlx::Error> { let active_nodes = self .get_all_active_gateways_in_interval(since, until) .await?; let mut active_day_statuses = Vec::with_capacity(active_nodes.len()); for active_node in active_nodes.into_iter() { - let ipv4_statuses = self - .get_gateway_ipv4_statuses_by_id(active_node.id, since, until) - .await?; - let ipv6_statuses = self - .get_gateway_ipv6_statuses_by_id(active_node.id, since, until) + let statuses = self + .get_gateway_statuses_by_id(active_node.id, since, until) .await?; - let statuses = ActiveNodeDayStatuses { + let statuses = ActiveNodeStatuses { identity: active_node.identity, owner: active_node.owner, - ipv4_statuses, - ipv6_statuses, + statuses, }; active_day_statuses.push(statuses); diff --git a/validator-api/src/storage/mod.rs b/validator-api/src/storage/mod.rs index 2954af2f35..44c466ed02 100644 --- a/validator-api/src/storage/mod.rs +++ b/validator-api/src/storage/mod.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use crate::network_monitor::monitor::summary_producer::NodeResult; +use crate::network_monitor::test_route::TestRoute; use crate::node_status_api::models::{ GatewayStatusReport, GatewayUptimeHistory, MixnodeStatusReport, MixnodeUptimeHistory, ValidatorApiStorageError, @@ -10,7 +11,7 @@ use crate::node_status_api::{ONE_DAY, ONE_HOUR}; use crate::storage::manager::StorageManager; use crate::storage::models::{ EpochRewarding, FailedMixnodeRewardChunk, NodeStatus, PossiblyUnrewardedMixnode, - RewardingReport, + RewardingReport, TestingRoute, }; use rocket::fairing::{self, AdHoc}; use rocket::{Build, Rocket}; @@ -70,11 +71,9 @@ impl ValidatorApiStorage { }) } - /// Gets all statuses for particular mixnode (ipv4 and ipv6) that were inserted + /// Gets all statuses for particular mixnode that were inserted /// since the provided timestamp. /// - /// Returns tuple containing vectors of ipv4 statuses and ipv6 statuses. - /// /// # Arguments /// /// * `identity`: identity key of the mixnode to query. @@ -83,27 +82,19 @@ impl ValidatorApiStorage { &self, identity: &str, since: UnixTimestamp, - ) -> Result<(Vec, Vec), ValidatorApiStorageError> { - let ipv4_statuses = self + ) -> Result, ValidatorApiStorageError> { + let statuses = self .manager - .get_mixnode_ipv4_statuses_since(identity, since) + .get_mixnode_statuses_since(identity, since) .await .map_err(|_| ValidatorApiStorageError::InternalDatabaseError)?; - let ipv6_statuses = self - .manager - .get_mixnode_ipv6_statuses_since(identity, since) - .await - .map_err(|_| ValidatorApiStorageError::InternalDatabaseError)?; - - Ok((ipv4_statuses, ipv6_statuses)) + Ok(statuses) } - /// Gets all statuses for particular gateway (ipv4 and ipv6) that were inserted + /// Gets all statuses for particular gateway that were inserted /// since the provided timestamp. /// - /// Returns tuple containing vectors of ipv4 statuses and ipv6 statuses. - /// /// # Arguments /// /// * `identity`: identity key of the gateway to query. @@ -112,23 +103,21 @@ impl ValidatorApiStorage { &self, identity: &str, since: UnixTimestamp, - ) -> Result<(Vec, Vec), ValidatorApiStorageError> { - let ipv4_statuses = self + ) -> Result, ValidatorApiStorageError> { + let statuses = self .manager - .get_gateway_ipv4_statuses_since(identity, since) + .get_gateway_statuses_since(identity, since) .await .map_err(|_| ValidatorApiStorageError::InternalDatabaseError)?; - let ipv6_statuses = self - .manager - .get_gateway_ipv6_statuses_since(identity, since) - .await - .map_err(|_| ValidatorApiStorageError::InternalDatabaseError)?; - - Ok((ipv4_statuses, ipv6_statuses)) + Ok(statuses) } /// Tries to construct a status report for mixnode with the specified identity. + /// + /// # Arguments + /// + /// * `identity`: identity (base58-encoded public key) of the mixnode. pub(crate) async fn construct_mixnode_report( &self, identity: &str, @@ -137,10 +126,10 @@ impl ValidatorApiStorage { let day_ago = (now - ONE_DAY).unix_timestamp(); let hour_ago = (now - ONE_HOUR).unix_timestamp(); - let (ipv4_statuses, ipv6_statuses) = self.get_mixnode_statuses(identity, day_ago).await?; + let statuses = self.get_mixnode_statuses(identity, day_ago).await?; // if we have no statuses, the node doesn't exist (or monitor is down), but either way, we can't make a report - if ipv4_statuses.is_empty() { + if statuses.is_empty() { return Err(ValidatorApiStorageError::MixnodeReportNotFound( identity.to_owned(), )); @@ -154,16 +143,6 @@ impl ValidatorApiStorage { .get_monitor_runs_count(day_ago, now.unix_timestamp()) .await?; - // now, technically this is not a critical error, but this should have NEVER happened in the first place - // so something super weird is going on - if ipv4_statuses.len() != ipv6_statuses.len() { - error!("Somehow we have different number of ipv4 and ipv6 statuses for mixnode {}! (ipv4: {}, ipv6: {})", - identity, - ipv4_statuses.len(), - ipv6_statuses.len(), - ) - } - let mixnode_owner = self .manager .get_mixnode_owner(identity) @@ -175,8 +154,7 @@ impl ValidatorApiStorage { now, identity.to_owned(), mixnode_owner, - ipv4_statuses, - ipv6_statuses, + statuses, last_hour_runs_count, last_day_runs_count, )) @@ -190,10 +168,10 @@ impl ValidatorApiStorage { let day_ago = (now - ONE_DAY).unix_timestamp(); let hour_ago = (now - ONE_HOUR).unix_timestamp(); - let (ipv4_statuses, ipv6_statuses) = self.get_gateway_statuses(identity, day_ago).await?; + let statuses = self.get_gateway_statuses(identity, day_ago).await?; // if we have no statuses, the node doesn't exist (or monitor is down), but either way, we can't make a report - if ipv4_statuses.is_empty() { + if statuses.is_empty() { return Err(ValidatorApiStorageError::GatewayReportNotFound( identity.to_owned(), )); @@ -207,16 +185,6 @@ impl ValidatorApiStorage { .get_monitor_runs_count(day_ago, now.unix_timestamp()) .await?; - // now, technically this is not a critical error, but this should have NEVER happened in the first place - // so something super weird is going on - if ipv4_statuses.len() != ipv6_statuses.len() { - error!("Somehow we have different number of ipv4 and ipv6 statuses for gateway {}! (ipv4: {}, ipv6: {})", - identity, - ipv4_statuses.len(), - ipv6_statuses.len(), - ) - } - let gateway_owner = self .manager .get_gateway_owner(identity) @@ -230,8 +198,7 @@ impl ValidatorApiStorage { now, identity.to_owned(), gateway_owner, - ipv4_statuses, - ipv6_statuses, + statuses, last_hour_runs_count, last_day_runs_count, )) @@ -331,8 +298,7 @@ impl ValidatorApiStorage { OffsetDateTime::from_unix_timestamp(end).unwrap(), statuses.identity, statuses.owner, - statuses.ipv4_statuses, - statuses.ipv6_statuses, + statuses.statuses, last_hour_runs_count, last_day_runs_count, ) @@ -376,8 +342,7 @@ impl ValidatorApiStorage { OffsetDateTime::from_unix_timestamp(end).unwrap(), statuses.identity, statuses.owner, - statuses.ipv4_statuses, - statuses.ipv6_statuses, + statuses.statuses, last_hour_runs_count, last_day_runs_count, ) @@ -387,16 +352,148 @@ impl ValidatorApiStorage { Ok(reports) } - // Used by network monitor - pub(crate) async fn submit_new_statuses( + /// Saves information about test route used during the network monitor run to the database. + /// + /// # Arguments + /// + /// * `monitor_run_id` id (as saved in the database) of the associated network monitor test run. + /// * `test_route`: one of the test routes used during network testing. + async fn insert_test_route( + &self, + monitor_run_id: i64, + test_route: TestRoute, + ) -> Result<(), ValidatorApiStorageError> { + // we MUST have those entries in the database, otherwise the route wouldn't have been chosen + // in the first place + let layer1_mix_id = self + .manager + .get_mixnode_id(&test_route.layer_one_mix().identity_key.to_base58_string()) + .await + .map_err(|_| ValidatorApiStorageError::InternalDatabaseError)? + .ok_or(ValidatorApiStorageError::InternalDatabaseError)?; + + let layer2_mix_id = self + .manager + .get_mixnode_id(&test_route.layer_two_mix().identity_key.to_base58_string()) + .await + .map_err(|_| ValidatorApiStorageError::InternalDatabaseError)? + .ok_or(ValidatorApiStorageError::InternalDatabaseError)?; + + let layer3_mix_id = self + .manager + .get_mixnode_id(&test_route.layer_three_mix().identity_key.to_base58_string()) + .await + .map_err(|_| ValidatorApiStorageError::InternalDatabaseError)? + .ok_or(ValidatorApiStorageError::InternalDatabaseError)?; + + let gateway_id = self + .manager + .get_gateway_id(&test_route.gateway().identity_key.to_base58_string()) + .await + .map_err(|_| ValidatorApiStorageError::InternalDatabaseError)? + .ok_or(ValidatorApiStorageError::InternalDatabaseError)?; + + self.manager + .submit_testing_route_used(TestingRoute { + gateway_id, + layer1_mix_id, + layer2_mix_id, + layer3_mix_id, + monitor_run_id, + }) + .await + .map_err(|_| ValidatorApiStorageError::InternalDatabaseError)?; + Ok(()) + } + + /// Retrieves number of times particular mixnode was used as a core node during network monitor + /// test runs since the specified unix timestamp. If no value is provided, last 30 days of data + /// are used instead. + /// + /// # Arguments + /// + /// * `identity`: identity (base58-encoded public key) of the mixnode. + /// * `since`: optional unix timestamp indicating the lower bound interval of the selection. + pub(crate) async fn get_core_mixnode_status_count( + &self, + identity: &str, + since: Option, + ) -> Result { + let node_id = self + .manager + .get_mixnode_id(identity) + .await + .map_err(|_| ValidatorApiStorageError::InternalDatabaseError)?; + + if let Some(node_id) = node_id { + let since = since + .unwrap_or_else(|| (OffsetDateTime::now_utc() - (30 * ONE_DAY)).unix_timestamp()); + + self.manager + .get_mixnode_testing_route_presence_count_since(node_id, since) + .await + .map_err(|_| ValidatorApiStorageError::InternalDatabaseError) + } else { + Ok(0) + } + } + + /// Retrieves number of times particular gateway was used as a core node during network monitor + /// test runs since the specified unix timestamp. If no value is provided, last 30 days of data + /// are used instead. + /// + /// # Arguments + /// + /// * `identity`: identity (base58-encoded public key) of the gateway. + /// * `since`: optional unix timestamp indicating the lower bound interval of the selection. + pub(crate) async fn get_core_gateway_status_count( + &self, + identity: &str, + since: Option, + ) -> Result { + let node_id = self + .manager + .get_gateway_id(identity) + .await + .map_err(|_| ValidatorApiStorageError::InternalDatabaseError)?; + + if let Some(node_id) = node_id { + let since = since + .unwrap_or_else(|| (OffsetDateTime::now_utc() - (30 * ONE_DAY)).unix_timestamp()); + + self.manager + .get_gateway_testing_route_presence_count_since(node_id, since) + .await + .map_err(|_| ValidatorApiStorageError::InternalDatabaseError) + } else { + Ok(0) + } + } + + /// Inserts an entry to the database with the network monitor test run information + /// that has occurred at this instant alongside the results of all the measurements performed. + /// + /// # Arguments + /// + /// * `mixnode_results`: + /// * `gateway_results`: + /// * `route_results`: + pub(crate) async fn insert_monitor_run_results( &self, mixnode_results: Vec, gateway_results: Vec, + test_routes: Vec, ) -> Result<(), ValidatorApiStorageError> { info!("Submitting new node results to the database. There are {} mixnode results and {} gateway results", mixnode_results.len(), gateway_results.len()); let now = OffsetDateTime::now_utc().unix_timestamp(); + let monitor_run_id = self + .manager + .insert_monitor_run(now) + .await + .map_err(|_| ValidatorApiStorageError::InternalDatabaseError)?; + self.manager .submit_mixnode_statuses(now, mixnode_results) .await @@ -405,18 +502,13 @@ impl ValidatorApiStorage { self.manager .submit_gateway_statuses(now, gateway_results) .await - .map_err(|_| ValidatorApiStorageError::InternalDatabaseError) - } + .map_err(|_| ValidatorApiStorageError::InternalDatabaseError)?; - /// Inserts an entry to the database with the network monitor test run information - /// that has occurred at this instant. - pub(crate) async fn insert_monitor_run(&self) -> Result<(), ValidatorApiStorageError> { - let now = OffsetDateTime::now_utc().unix_timestamp(); + for test_route in test_routes { + self.insert_test_route(monitor_run_id, test_route).await?; + } - self.manager - .insert_monitor_run(now) - .await - .map_err(|_| ValidatorApiStorageError::InternalDatabaseError) + Ok(()) } /// Obtains number of network monitor test runs that have occurred within the specified interval. @@ -477,12 +569,7 @@ impl ValidatorApiStorage { }; self.manager - .insert_mixnode_historical_uptime( - node_id, - today_iso_8601, - report.last_day_ipv4.u8(), - report.last_day_ipv4.u8(), - ) + .insert_mixnode_historical_uptime(node_id, today_iso_8601, report.last_day.u8()) .await .map_err(|_| ValidatorApiStorageError::InternalDatabaseError)?; } @@ -507,12 +594,7 @@ impl ValidatorApiStorage { }; self.manager - .insert_gateway_historical_uptime( - node_id, - today_iso_8601, - report.last_day_ipv4.u8(), - report.last_day_ipv4.u8(), - ) + .insert_gateway_historical_uptime(node_id, today_iso_8601, report.last_day.u8()) .await .map_err(|_| ValidatorApiStorageError::InternalDatabaseError)?; } @@ -541,19 +623,11 @@ impl ValidatorApiStorage { until: UnixTimestamp, ) -> Result<(), ValidatorApiStorageError> { self.manager - .purge_old_mixnode_ipv4_statuses(until) + .purge_old_mixnode_statuses(until) .await .map_err(|_| ValidatorApiStorageError::InternalDatabaseError)?; self.manager - .purge_old_mixnode_ipv6_statuses(until) - .await - .map_err(|_| ValidatorApiStorageError::InternalDatabaseError)?; - self.manager - .purge_old_gateway_ipv4_statuses(until) - .await - .map_err(|_| ValidatorApiStorageError::InternalDatabaseError)?; - self.manager - .purge_old_gateway_ipv6_statuses(until) + .purge_old_gateway_statuses(until) .await .map_err(|_| ValidatorApiStorageError::InternalDatabaseError) } @@ -583,11 +657,11 @@ impl ValidatorApiStorage { // /// Returns None if no data exists. // pub(crate) async fn get_most_recent_epoch_rewarding_entry( // &self, - // ) -> Result, NodeStatusApiError> { + // ) -> Result, ValidatorApiStorageError> { // self.manager // .get_most_recent_epoch_rewarding_entry() // .await - // .map_err(|_| NodeStatusApiError::InternalDatabaseError) + // .map_err(|_| ValidatorApiStorageError::InternalDatabaseError) // } /// Tries to obtain the epoch rewarding entry that has the provided timestamp. diff --git a/validator-api/src/storage/models.rs b/validator-api/src/storage/models.rs index 1c5ffdf883..65891524eb 100644 --- a/validator-api/src/storage/models.rs +++ b/validator-api/src/storage/models.rs @@ -6,7 +6,7 @@ use crate::storage::UnixTimestamp; // Internally used struct to catch results from the database to calculate uptimes for given mixnode/gateway pub(crate) struct NodeStatus { pub(crate) timestamp: UnixTimestamp, - pub(crate) up: bool, + pub(crate) reliability: u8, } // Internally used struct to catch results from the database to find active mixnodes/gateways @@ -16,6 +16,14 @@ pub(crate) struct ActiveNode { pub(crate) owner: String, } +pub(crate) struct TestingRoute { + pub(crate) gateway_id: i64, + pub(crate) layer1_mix_id: i64, + pub(crate) layer2_mix_id: i64, + pub(crate) layer3_mix_id: i64, + pub(crate) monitor_run_id: i64, +} + pub(crate) struct EpochRewarding { #[allow(dead_code)] pub(crate) id: i64,