From caa17d933cf882cd735629d32385e3a98fbb0d80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bogdan-=C8=98tefan=20Neac=C5=9Fu?= Date: Tue, 17 Dec 2024 15:18:11 +0200 Subject: [PATCH 01/19] Add windows to CI builds (#5269) * Add windows to CI builds * Fix win build for node status api * Fix win build for sdk * Fix win build for cred proxy --- .github/workflows/ci-build.yml | 2 +- nym-api/src/network_monitor/monitor/sender.rs | 1 + .../nym-credential-proxy/src/helpers.rs | 101 ++++++++++++- .../nym-credential-proxy/src/main.rs | 136 ++++-------------- .../nym-node-status-api/build.rs | 7 +- sdk/rust/nym-sdk/src/mixnet/client.rs | 10 +- 6 files changed, 145 insertions(+), 112 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 56fd0d15eb..d07f7edcbd 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -30,7 +30,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ arc-ubuntu-20.04, custom-runner-mac-m1 ] + os: [ arc-ubuntu-20.04, custom-windows-11, custom-runner-mac-m1 ] runs-on: ${{ matrix.os }} env: CARGO_TERM_COLOR: always diff --git a/nym-api/src/network_monitor/monitor/sender.rs b/nym-api/src/network_monitor/monitor/sender.rs index 172d6b3680..c20feadac1 100644 --- a/nym-api/src/network_monitor/monitor/sender.rs +++ b/nym-api/src/network_monitor/monitor/sender.rs @@ -183,6 +183,7 @@ impl PacketSender { gateway_packet_router, Some(fresh_gateway_client_data.bandwidth_controller.clone()), nym_statistics_common::clients::ClientStatsSender::new(None), + #[cfg(unix)] None, task_client, ); diff --git a/nym-credential-proxy/nym-credential-proxy/src/helpers.rs b/nym-credential-proxy/nym-credential-proxy/src/helpers.rs index 12ba427a70..8a528ff575 100644 --- a/nym-credential-proxy/nym-credential-proxy/src/helpers.rs +++ b/nym-credential-proxy/nym-credential-proxy/src/helpers.rs @@ -1,8 +1,22 @@ // Copyright 2024 Nym Technologies SA // SPDX-License-Identifier: GPL-3.0-only +use nym_bin_common::bin_info; use time::OffsetDateTime; -use tracing::{debug, info, warn}; +use tokio_util::sync::CancellationToken; +use tracing::{debug, error, info, warn}; + +use crate::{ + cli::Cli, + deposit_maker::DepositMaker, + error::VpnApiError, + http::{ + state::{ApiState, ChainClient}, + HttpServer, + }, + storage::VpnApiStorage, + tasks::StoragePruner, +}; pub struct LockTimer { created: OffsetDateTime, @@ -40,3 +54,88 @@ impl Default for LockTimer { } } } + +pub async fn wait_for_signal() { + use tokio::signal::unix::{signal, SignalKind}; + + // if we fail to setup the signals, we should just blow up + #[allow(clippy::expect_used)] + let mut sigterm = signal(SignalKind::terminate()).expect("Failed to setup SIGTERM channel"); + #[allow(clippy::expect_used)] + let mut sigquit = signal(SignalKind::quit()).expect("Failed to setup SIGQUIT channel"); + + tokio::select! { + _ = tokio::signal::ctrl_c() => { + info!("Received SIGINT"); + }, + _ = sigterm.recv() => { + info!("Received SIGTERM"); + } + _ = sigquit.recv() => { + info!("Received SIGQUIT"); + } + } +} + +fn build_sha_short() -> &'static str { + let bin_info = bin_info!(); + if bin_info.commit_sha.len() < 7 { + panic!("unavailable build commit sha") + } + + if bin_info.commit_sha == "VERGEN_IDEMPOTENT_OUTPUT" { + error!("the binary hasn't been built correctly. it doesn't have a commit sha information"); + return "unknown"; + } + + &bin_info.commit_sha[..7] +} + +pub(crate) async fn run_api(cli: Cli) -> Result<(), VpnApiError> { + // create the tasks + let bind_address = cli.bind_address(); + + let storage = VpnApiStorage::init(cli.persistent_storage_path()).await?; + let mnemonic = cli.mnemonic; + let auth_token = cli.http_auth_token; + let webhook_cfg = cli.webhook; + let chain_client = ChainClient::new(mnemonic)?; + let cancellation_token = CancellationToken::new(); + + let deposit_maker = DepositMaker::new( + build_sha_short(), + chain_client.clone(), + cli.max_concurrent_deposits, + cancellation_token.clone(), + ); + + let deposit_request_sender = deposit_maker.deposit_request_sender(); + let api_state = ApiState::new( + storage.clone(), + webhook_cfg, + chain_client, + deposit_request_sender, + cancellation_token.clone(), + ) + .await?; + let http_server = HttpServer::new( + bind_address, + api_state.clone(), + auth_token, + cancellation_token.clone(), + ); + let storage_pruner = StoragePruner::new(cancellation_token, storage); + + // spawn all the tasks + api_state.try_spawn(http_server.run_forever()); + api_state.try_spawn(storage_pruner.run_forever()); + api_state.try_spawn(deposit_maker.run_forever()); + + // wait for cancel signal (SIGINT, SIGTERM or SIGQUIT) + wait_for_signal().await; + + // cancel all the tasks and wait for all task to terminate + api_state.cancel_and_wait().await; + + Ok(()) +} diff --git a/nym-credential-proxy/nym-credential-proxy/src/main.rs b/nym-credential-proxy/nym-credential-proxy/src/main.rs index 71dd082b10..1ca9ccf316 100644 --- a/nym-credential-proxy/nym-credential-proxy/src/main.rs +++ b/nym-credential-proxy/nym-credential-proxy/src/main.rs @@ -6,117 +6,30 @@ #![warn(clippy::todo)] #![warn(clippy::dbg_macro)] -use crate::cli::Cli; -use crate::deposit_maker::DepositMaker; -use crate::error::VpnApiError; -use crate::http::state::{ApiState, ChainClient}; -use crate::http::HttpServer; -use crate::storage::VpnApiStorage; -use crate::tasks::StoragePruner; -use clap::Parser; -use nym_bin_common::logging::setup_tracing_logger; -use nym_bin_common::{bin_info, bin_info_owned}; -use nym_network_defaults::setup_env; -use tokio_util::sync::CancellationToken; -use tracing::{error, info, trace}; +cfg_if::cfg_if! { + if #[cfg(unix)] { + use crate::cli::Cli; + use clap::Parser; + use nym_bin_common::bin_info_owned; + use nym_bin_common::logging::setup_tracing_logger; + use nym_network_defaults::setup_env; + use tracing::{info, trace}; -pub mod cli; -pub mod config; -pub mod credentials; -mod deposit_maker; -pub mod error; -pub mod helpers; -pub mod http; -pub mod nym_api_helpers; -pub mod storage; -pub mod tasks; -mod webhook; - -pub async fn wait_for_signal() { - use tokio::signal::unix::{signal, SignalKind}; - - // if we fail to setup the signals, we should just blow up - #[allow(clippy::expect_used)] - let mut sigterm = signal(SignalKind::terminate()).expect("Failed to setup SIGTERM channel"); - #[allow(clippy::expect_used)] - let mut sigquit = signal(SignalKind::quit()).expect("Failed to setup SIGQUIT channel"); - - tokio::select! { - _ = tokio::signal::ctrl_c() => { - info!("Received SIGINT"); - }, - _ = sigterm.recv() => { - info!("Received SIGTERM"); - } - _ = sigquit.recv() => { - info!("Received SIGQUIT"); - } + pub mod cli; + pub mod config; + pub mod credentials; + mod deposit_maker; + pub mod error; + pub mod helpers; + pub mod http; + pub mod nym_api_helpers; + pub mod storage; + pub mod tasks; + mod webhook; } } -fn build_sha_short() -> &'static str { - let bin_info = bin_info!(); - if bin_info.commit_sha.len() < 7 { - panic!("unavailable build commit sha") - } - - if bin_info.commit_sha == "VERGEN_IDEMPOTENT_OUTPUT" { - error!("the binary hasn't been built correctly. it doesn't have a commit sha information"); - return "unknown"; - } - - &bin_info.commit_sha[..7] -} - -async fn run_api(cli: Cli) -> Result<(), VpnApiError> { - // create the tasks - let bind_address = cli.bind_address(); - - let storage = VpnApiStorage::init(cli.persistent_storage_path()).await?; - let mnemonic = cli.mnemonic; - let auth_token = cli.http_auth_token; - let webhook_cfg = cli.webhook; - let chain_client = ChainClient::new(mnemonic)?; - let cancellation_token = CancellationToken::new(); - - let deposit_maker = DepositMaker::new( - build_sha_short(), - chain_client.clone(), - cli.max_concurrent_deposits, - cancellation_token.clone(), - ); - - let deposit_request_sender = deposit_maker.deposit_request_sender(); - let api_state = ApiState::new( - storage.clone(), - webhook_cfg, - chain_client, - deposit_request_sender, - cancellation_token.clone(), - ) - .await?; - let http_server = HttpServer::new( - bind_address, - api_state.clone(), - auth_token, - cancellation_token.clone(), - ); - let storage_pruner = StoragePruner::new(cancellation_token, storage); - - // spawn all the tasks - api_state.try_spawn(http_server.run_forever()); - api_state.try_spawn(storage_pruner.run_forever()); - api_state.try_spawn(deposit_maker.run_forever()); - - // wait for cancel signal (SIGINT, SIGTERM or SIGQUIT) - wait_for_signal().await; - - // cancel all the tasks and wait for all task to terminate - api_state.cancel_and_wait().await; - - Ok(()) -} - +#[cfg(unix)] #[tokio::main] async fn main() -> anyhow::Result<()> { // std::env::set_var( @@ -134,6 +47,13 @@ async fn main() -> anyhow::Result<()> { let bin_info = bin_info_owned!(); info!("using the following version: {bin_info}"); - run_api(cli).await?; + helpers::run_api(cli).await?; Ok(()) } + +#[cfg(not(unix))] +#[tokio::main] +async fn main() -> anyhow::Result<()> { + eprintln!("This tool is only supported on Unix systems"); + std::process::exit(1) +} diff --git a/nym-node-status-api/nym-node-status-api/build.rs b/nym-node-status-api/nym-node-status-api/build.rs index 025e755088..3a1b933bc4 100644 --- a/nym-node-status-api/nym-node-status-api/build.rs +++ b/nym-node-status-api/nym-node-status-api/build.rs @@ -1,6 +1,8 @@ use anyhow::{anyhow, Result}; use sqlx::{Connection, SqliteConnection}; +#[cfg(target_family = "unix")] use std::fs::Permissions; +#[cfg(target_family = "unix")] use std::os::unix::fs::PermissionsExt; use tokio::{fs::File, io::AsyncWriteExt}; @@ -39,7 +41,10 @@ async fn write_db_path_to_file(out_dir: &str, db_filename: &str) -> anyhow::Resu file.write_all(format!("sqlite3 {}/{}", out_dir, db_filename).as_bytes()) .await?; + #[cfg(target_family = "unix")] file.set_permissions(Permissions::from_mode(0o755)) .await - .map_err(From::from) + .map_err(anyhow::Error::from)?; + + Ok(()) } diff --git a/sdk/rust/nym-sdk/src/mixnet/client.rs b/sdk/rust/nym-sdk/src/mixnet/client.rs index 598bbd4808..249bbd182c 100644 --- a/sdk/rust/nym-sdk/src/mixnet/client.rs +++ b/sdk/rust/nym-sdk/src/mixnet/client.rs @@ -37,6 +37,7 @@ use nym_validator_client::{nyxd, QueryHttpRpcNyxdClient, UserAgent}; use rand::rngs::OsRng; use std::path::Path; use std::path::PathBuf; +#[cfg(unix)] use std::sync::Arc; use url::Url; use zeroize::Zeroizing; @@ -56,6 +57,7 @@ pub struct MixnetClientBuilder { custom_shutdown: Option, force_tls: bool, user_agent: Option, + #[cfg(unix)] connection_fd_callback: Option>, // TODO: incorporate it properly into `MixnetClientStorage` (I will need it in wasm anyway) @@ -256,6 +258,7 @@ where self } + #[cfg(unix)] #[must_use] pub fn with_connection_fd_callback( mut self, @@ -293,7 +296,10 @@ where client.wait_for_gateway = self.wait_for_gateway; client.force_tls = self.force_tls; client.user_agent = self.user_agent; - client.connection_fd_callback = self.connection_fd_callback; + #[cfg(unix)] + if self.connection_fd_callback.is_some() { + client.connection_fd_callback = self.connection_fd_callback; + } client.forget_me = self.forget_me; Ok(client) } @@ -345,6 +351,7 @@ where user_agent: Option, /// Callback on the websocket fd as soon as the connection has been established + #[cfg(unix)] connection_fd_callback: Option>, forget_me: ForgetMe, @@ -397,6 +404,7 @@ where force_tls: false, custom_shutdown: None, user_agent: None, + #[cfg(unix)] connection_fd_callback: None, forget_me: Default::default(), }) From acd068e5abd13bc74baf434bdb23ca394392bbff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Wed, 18 Dec 2024 12:37:16 +0100 Subject: [PATCH 02/19] Add close to credential storage (#5283) * Add close method to credential storage * wip --- common/credential-storage/src/backends/sqlite.rs | 5 +++++ common/credential-storage/src/ephemeral_storage.rs | 4 ++++ common/credential-storage/src/persistent_storage/mod.rs | 4 ++++ common/credential-storage/src/storage.rs | 2 ++ 4 files changed, 15 insertions(+) diff --git a/common/credential-storage/src/backends/sqlite.rs b/common/credential-storage/src/backends/sqlite.rs index 9267bbddb3..dec0899064 100644 --- a/common/credential-storage/src/backends/sqlite.rs +++ b/common/credential-storage/src/backends/sqlite.rs @@ -23,6 +23,11 @@ impl SqliteEcashTicketbookManager { SqliteEcashTicketbookManager { connection_pool } } + /// Closes the connection pool. + pub async fn close(&self) { + self.connection_pool.close().await + } + pub(crate) async fn cleanup_expired(&self, deadline: Date) -> Result<(), sqlx::Error> { sqlx::query!( "DELETE FROM ecash_ticketbook WHERE expiration_date <= ?", diff --git a/common/credential-storage/src/ephemeral_storage.rs b/common/credential-storage/src/ephemeral_storage.rs index 91436d4d8c..b6a113f414 100644 --- a/common/credential-storage/src/ephemeral_storage.rs +++ b/common/credential-storage/src/ephemeral_storage.rs @@ -43,6 +43,10 @@ impl Debug for EphemeralStorage { impl Storage for EphemeralStorage { type StorageError = StorageError; + async fn close(&self) { + // nothing to do here + } + async fn cleanup_expired(&self) -> Result<(), Self::StorageError> { self.storage_manager.cleanup_expired().await; Ok(()) diff --git a/common/credential-storage/src/persistent_storage/mod.rs b/common/credential-storage/src/persistent_storage/mod.rs index e8c9eca5aa..32b6f581de 100644 --- a/common/credential-storage/src/persistent_storage/mod.rs +++ b/common/credential-storage/src/persistent_storage/mod.rs @@ -89,6 +89,10 @@ impl PersistentStorage { impl Storage for PersistentStorage { type StorageError = StorageError; + async fn close(&self) { + self.storage_manager.close().await + } + /// remove all expired ticketbooks and expiration date signatures async fn cleanup_expired(&self) -> Result<(), Self::StorageError> { let ecash_yesterday = ecash_today().date().previous_day().unwrap(); diff --git a/common/credential-storage/src/storage.rs b/common/credential-storage/src/storage.rs index 19ddc44e86..4c0602ea85 100644 --- a/common/credential-storage/src/storage.rs +++ b/common/credential-storage/src/storage.rs @@ -22,6 +22,8 @@ use std::error::Error; pub trait Storage: Send + Sync { type StorageError: Error; + async fn close(&self); + /// remove all expired ticketbooks and expiration date signatures async fn cleanup_expired(&self) -> Result<(), Self::StorageError>; From f7a7a8072fb74e53837ae9cb29a7471021549b2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bogdan-=C8=98tefan=20Neac=C5=9Fu?= Date: Wed, 18 Dec 2024 16:23:18 +0200 Subject: [PATCH 03/19] Move tun constants to network defaults (#5286) (#5287) --- common/network-defaults/src/constants.rs | 11 +++++++++++ service-providers/ip-packet-router/src/constants.rs | 9 --------- .../ip-packet-router/src/ip_packet_router.rs | 13 +++++++------ .../ip-packet-router/src/util/generate_new_ip.rs | 7 ++++--- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/common/network-defaults/src/constants.rs b/common/network-defaults/src/constants.rs index 6723b45e41..e9b70f54e3 100644 --- a/common/network-defaults/src/constants.rs +++ b/common/network-defaults/src/constants.rs @@ -57,3 +57,14 @@ pub mod wireguard { pub const WG_TUN_DEVICE_IP_ADDRESS_V6: Ipv6Addr = Ipv6Addr::new(0xfc01, 0, 0, 0, 0, 0, 0, 0x1); // fc01::1 pub const WG_TUN_DEVICE_NETMASK_V6: u8 = 112; } + +pub mod mixnet_vpn { + use std::net::{Ipv4Addr, Ipv6Addr}; + + // The interface used to route traffic + pub const NYM_TUN_BASE_NAME: &str = "nymtun"; + pub const NYM_TUN_DEVICE_ADDRESS_V4: Ipv4Addr = Ipv4Addr::new(10, 0, 0, 1); + pub const NYM_TUN_DEVICE_NETMASK_V4: Ipv4Addr = Ipv4Addr::new(255, 255, 0, 0); + pub const NYM_TUN_DEVICE_ADDRESS_V6: Ipv6Addr = Ipv6Addr::new(0xfc00, 0, 0, 0, 0, 0, 0, 0x1); // fc00::1 + pub const NYM_TUN_DEVICE_NETMASK_V6: &str = "112"; +} diff --git a/service-providers/ip-packet-router/src/constants.rs b/service-providers/ip-packet-router/src/constants.rs index 73e744890d..bfee848df2 100644 --- a/service-providers/ip-packet-router/src/constants.rs +++ b/service-providers/ip-packet-router/src/constants.rs @@ -1,14 +1,5 @@ -use std::net::{Ipv4Addr, Ipv6Addr}; use std::time::Duration; -// The interface used to route traffic -pub const TUN_BASE_NAME: &str = "nymtun"; -pub const TUN_DEVICE_ADDRESS_V4: Ipv4Addr = Ipv4Addr::new(10, 0, 0, 1); -pub const TUN_DEVICE_NETMASK_V4: Ipv4Addr = Ipv4Addr::new(255, 255, 0, 0); -pub const TUN_DEVICE_ADDRESS_V6: Ipv6Addr = Ipv6Addr::new(0xfc00, 0, 0, 0, 0, 0, 0, 0x1); // fc00::1 - -pub const TUN_DEVICE_NETMASK_V6: &str = "112"; - // We routinely check if any clients needs to be disconnected at this interval pub(crate) const DISCONNECT_TIMER_INTERVAL: Duration = Duration::from_secs(10); diff --git a/service-providers/ip-packet-router/src/ip_packet_router.rs b/service-providers/ip-packet-router/src/ip_packet_router.rs index cff86d6fb0..a0f185cc76 100644 --- a/service-providers/ip-packet-router/src/ip_packet_router.rs +++ b/service-providers/ip-packet-router/src/ip_packet_router.rs @@ -119,7 +119,7 @@ impl IpPacketRouter { log::error!("ip packet router service provider is not yet supported on this platform"); Ok(()) } else { - unimplemented!("service provider is not yet supported on this platform") + todo!("service provider is not yet supported on this platform") } } @@ -145,11 +145,12 @@ impl IpPacketRouter { // Create the TUN device that we interact with the rest of the world with let config = nym_tun::tun_device::TunDeviceConfig { - base_name: crate::constants::TUN_BASE_NAME.to_string(), - ipv4: crate::constants::TUN_DEVICE_ADDRESS_V4, - netmaskv4: crate::constants::TUN_DEVICE_NETMASK_V4, - ipv6: crate::constants::TUN_DEVICE_ADDRESS_V6, - netmaskv6: crate::constants::TUN_DEVICE_NETMASK_V6.to_string(), + base_name: nym_network_defaults::constants::mixnet_vpn::NYM_TUN_BASE_NAME.to_string(), + ipv4: nym_network_defaults::constants::mixnet_vpn::NYM_TUN_DEVICE_ADDRESS_V4, + netmaskv4: nym_network_defaults::constants::mixnet_vpn::NYM_TUN_DEVICE_NETMASK_V4, + ipv6: nym_network_defaults::constants::mixnet_vpn::NYM_TUN_DEVICE_ADDRESS_V6, + netmaskv6: nym_network_defaults::constants::mixnet_vpn::NYM_TUN_DEVICE_NETMASK_V6 + .to_string(), }; let (tun_reader, tun_writer) = tokio::io::split(nym_tun::tun_device::TunDevice::new_device_only(config)?); diff --git a/service-providers/ip-packet-router/src/util/generate_new_ip.rs b/service-providers/ip-packet-router/src/util/generate_new_ip.rs index ea5325f237..2743be3ef6 100644 --- a/service-providers/ip-packet-router/src/util/generate_new_ip.rs +++ b/service-providers/ip-packet-router/src/util/generate_new_ip.rs @@ -1,9 +1,10 @@ use nym_ip_packet_requests::IpPair; +use nym_network_defaults::constants::mixnet_vpn::{ + NYM_TUN_DEVICE_ADDRESS_V4, NYM_TUN_DEVICE_ADDRESS_V6, +}; use std::net::Ipv6Addr; use std::{collections::HashMap, net::Ipv4Addr}; -use crate::constants::{TUN_DEVICE_ADDRESS_V4, TUN_DEVICE_ADDRESS_V6}; - // Find an available IP address in self.connected_clients // TODO: make this nicer fn generate_random_ips_within_subnet(rng: &mut R) -> IpPair { @@ -36,7 +37,7 @@ pub(crate) fn find_new_ips( let mut rng = rand::thread_rng(); let mut new_ips = generate_random_ips_within_subnet(&mut rng); let mut tries = 0; - let tun_ips = IpPair::new(TUN_DEVICE_ADDRESS_V4, TUN_DEVICE_ADDRESS_V6); + let tun_ips = IpPair::new(NYM_TUN_DEVICE_ADDRESS_V4, NYM_TUN_DEVICE_ADDRESS_V6); while is_ip_taken( connected_clients_ipv4, From c482350ec6c9c5477893e19446be68b18113b1c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Stuczy=C5=84ski?= Date: Thu, 19 Dec 2024 10:49:56 +0000 Subject: [PATCH 04/19] feature: wireguard metrics (#5278) * experimental log * introduce wireguard metrics updates * add wireguard traffic rates to console logger * missing import * changed order of displayed values * expose bytes information via rest endpoint * clippy --- Cargo.lock | 1 + common/wireguard/Cargo.toml | 1 + common/wireguard/src/lib.rs | 2 + common/wireguard/src/peer_controller.rs | 52 +++++++++++++++++++ gateway/src/node/mod.rs | 8 +++ nym-node/nym-node-metrics/src/lib.rs | 3 ++ nym-node/nym-node-metrics/src/wireguard.rs | 44 ++++++++++++++++ .../src/api/v1/metrics/models.rs | 13 +++++ nym-node/nym-node-requests/src/lib.rs | 6 +++ .../node/http/router/api/v1/metrics/mod.rs | 3 ++ .../router/api/v1/metrics/packets_stats.rs | 2 +- .../http/router/api/v1/metrics/wireguard.rs | 40 ++++++++++++++ nym-node/src/node/metrics/console_logger.rs | 24 +++++++++ nym-node/src/node/mod.rs | 1 + 14 files changed, 199 insertions(+), 1 deletion(-) create mode 100644 nym-node/nym-node-metrics/src/wireguard.rs create mode 100644 nym-node/src/node/http/router/api/v1/metrics/wireguard.rs diff --git a/Cargo.lock b/Cargo.lock index d84ab9916c..cc9ed04a83 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6807,6 +6807,7 @@ dependencies = [ "nym-crypto", "nym-gateway-storage", "nym-network-defaults", + "nym-node-metrics", "nym-task", "nym-wireguard-types", "thiserror", diff --git a/common/wireguard/Cargo.toml b/common/wireguard/Cargo.toml index df333cab82..c999861cd6 100644 --- a/common/wireguard/Cargo.toml +++ b/common/wireguard/Cargo.toml @@ -36,3 +36,4 @@ nym-gateway-storage = { path = "../gateway-storage" } nym-network-defaults = { path = "../network-defaults" } nym-task = { path = "../task" } nym-wireguard-types = { path = "../wireguard-types" } +nym-node-metrics = { path = "../../nym-node/nym-node-metrics" } diff --git a/common/wireguard/src/lib.rs b/common/wireguard/src/lib.rs index 7b5f190193..f397a3bbea 100644 --- a/common/wireguard/src/lib.rs +++ b/common/wireguard/src/lib.rs @@ -85,6 +85,7 @@ pub struct WireguardData { #[cfg(target_os = "linux")] pub async fn start_wireguard( storage: nym_gateway_storage::GatewayStorage, + metrics: nym_node_metrics::NymNodeMetrics, all_peers: Vec, task_client: nym_task::TaskClient, wireguard_data: WireguardData, @@ -175,6 +176,7 @@ pub async fn start_wireguard( let wg_api = std::sync::Arc::new(WgApiWrapper::new(wg_api)); let mut controller = PeerController::new( storage, + metrics, wg_api.clone(), host, peer_bandwidth_managers, diff --git a/common/wireguard/src/peer_controller.rs b/common/wireguard/src/peer_controller.rs index 5f2cf6399f..b002db761b 100644 --- a/common/wireguard/src/peer_controller.rs +++ b/common/wireguard/src/peer_controller.rs @@ -16,7 +16,9 @@ use nym_credential_verification::{ ClientBandwidth, }; use nym_gateway_storage::GatewayStorage; +use nym_node_metrics::NymNodeMetrics; use nym_wireguard_types::DEFAULT_PEER_TIMEOUT_CHECK; +use std::time::{Duration, SystemTime}; use std::{collections::HashMap, sync::Arc}; use tokio::sync::{mpsc, RwLock}; use tokio_stream::{wrappers::IntervalStream, StreamExt}; @@ -65,6 +67,11 @@ pub struct QueryBandwidthControlResponse { pub struct PeerController { storage: GatewayStorage, + + // we have "all" metrics of a node, but they're behind a single Arc pointer, + // so the overhead is minimal + metrics: NymNodeMetrics, + // used to receive commands from individual handles too request_tx: mpsc::Sender, request_rx: mpsc::Receiver, @@ -76,8 +83,10 @@ pub struct PeerController { } impl PeerController { + #[allow(clippy::too_many_arguments)] pub fn new( storage: GatewayStorage, + metrics: NymNodeMetrics, wg_api: Arc, initial_host_information: Host, bw_storage_managers: HashMap, Peer)>, @@ -123,6 +132,7 @@ impl PeerController { request_rx, timeout_check_interval, task_client, + metrics, } } @@ -257,6 +267,46 @@ impl PeerController { })) } + fn update_metrics(&self, new_host: &Host) { + let now = SystemTime::now(); + const ACTIVITY_THRESHOLD: Duration = Duration::from_secs(60); + + let total_peers = new_host.peers.len(); + let mut active_peers = 0; + let mut total_rx = 0; + let mut total_tx = 0; + + for peer in new_host.peers.values() { + total_rx += peer.rx_bytes; + total_tx += peer.tx_bytes; + + // if a peer hasn't performed a handshake in last minute, + // I think it's reasonable to assume it's no longer active + let Some(last_handshake) = peer.last_handshake else { + continue; + }; + let Ok(elapsed) = now.duration_since(last_handshake) else { + continue; + }; + if elapsed < ACTIVITY_THRESHOLD { + active_peers += 1; + } + } + + self.metrics.wireguard.update( + // if the conversion fails it means we're running not running on a 64bit system + // and that's a reason enough for this failure. + total_rx.try_into().expect( + "failed to convert bytes from u64 to usize - are you running on non 64bit system?", + ), + total_tx.try_into().expect( + "failed to convert bytes from u64 to usize - are you running on non 64bit system?", + ), + total_peers, + active_peers, + ); + } + pub async fn run(&mut self) { info!("started wireguard peer controller"); loop { @@ -266,6 +316,8 @@ impl PeerController { log::error!("Can't read wireguard kernel data"); continue; }; + self.update_metrics(&host); + *self.host_information.write().await = host; } _ = self.task_client.recv() => { diff --git a/gateway/src/node/mod.rs b/gateway/src/node/mod.rs index 84b1990351..b3b41961a1 100644 --- a/gateway/src/node/mod.rs +++ b/gateway/src/node/mod.rs @@ -37,6 +37,7 @@ mod internal_service_providers; pub use client_handling::active_clients::ActiveClientsStore; pub use nym_gateway_stats_storage::PersistentStatsStorage; pub use nym_gateway_storage::{error::GatewayStorageError, GatewayStorage}; +use nym_node_metrics::NymNodeMetrics; pub use nym_sdk::{NymApiTopologyProvider, NymApiTopologyProviderConfig, UserAgent}; #[derive(Debug, Clone)] @@ -81,6 +82,8 @@ pub struct GatewayTasksBuilder { metrics_sender: MetricEventsSender, + metrics: NymNodeMetrics, + mnemonic: Arc>, shutdown: TaskClient, @@ -102,12 +105,14 @@ impl Drop for GatewayTasksBuilder { } impl GatewayTasksBuilder { + #[allow(clippy::too_many_arguments)] pub fn new( config: Config, identity: Arc, storage: GatewayStorage, mix_packet_sender: MixForwardingSender, metrics_sender: MetricEventsSender, + metrics: NymNodeMetrics, mnemonic: Arc>, shutdown: TaskClient, ) -> GatewayTasksBuilder { @@ -121,6 +126,7 @@ impl GatewayTasksBuilder { storage, mix_packet_sender, metrics_sender, + metrics, mnemonic, shutdown, ecash_manager: None, @@ -443,6 +449,7 @@ impl GatewayTasksBuilder { pub async fn try_start_wireguard( &mut self, ) -> Result, Box> { + let _ = self.metrics.clone(); unimplemented!("wireguard is not supported on this platform") } @@ -460,6 +467,7 @@ impl GatewayTasksBuilder { let wg_handle = nym_wireguard::start_wireguard( self.storage.clone(), + self.metrics.clone(), all_peers, self.shutdown.fork("wireguard"), wireguard_data, diff --git a/nym-node/nym-node-metrics/src/lib.rs b/nym-node/nym-node-metrics/src/lib.rs index 58ab0f77f7..57a8c74bb3 100644 --- a/nym-node/nym-node-metrics/src/lib.rs +++ b/nym-node/nym-node-metrics/src/lib.rs @@ -4,6 +4,7 @@ use crate::entry::EntryStats; use crate::mixnet::MixingStats; use crate::network::NetworkStats; +use crate::wireguard::WireguardStats; use std::ops::Deref; use std::sync::Arc; @@ -11,6 +12,7 @@ pub mod entry; pub mod events; pub mod mixnet; pub mod network; +pub mod wireguard; #[derive(Clone, Default)] pub struct NymNodeMetrics { @@ -34,6 +36,7 @@ impl Deref for NymNodeMetrics { pub struct NymNodeMetricsInner { pub mixnet: MixingStats, pub entry: EntryStats, + pub wireguard: WireguardStats, pub network: NetworkStats, } diff --git a/nym-node/nym-node-metrics/src/wireguard.rs b/nym-node/nym-node-metrics/src/wireguard.rs new file mode 100644 index 0000000000..8aee90d34b --- /dev/null +++ b/nym-node/nym-node-metrics/src/wireguard.rs @@ -0,0 +1,44 @@ +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use std::sync::atomic::{AtomicUsize, Ordering}; + +#[derive(Default)] +pub struct WireguardStats { + bytes_rx: AtomicUsize, + bytes_tx: AtomicUsize, + + total_peers: AtomicUsize, + active_peers: AtomicUsize, +} + +impl WireguardStats { + pub fn bytes_rx(&self) -> usize { + self.bytes_rx.load(Ordering::Relaxed) + } + + pub fn bytes_tx(&self) -> usize { + self.bytes_tx.load(Ordering::Relaxed) + } + + pub fn total_peers(&self) -> usize { + self.total_peers.load(Ordering::Relaxed) + } + + pub fn active_peers(&self) -> usize { + self.active_peers.load(Ordering::Relaxed) + } + + pub fn update( + &self, + bytes_rx: usize, + bytes_tx: usize, + total_peers: usize, + active_peers: usize, + ) { + self.bytes_rx.store(bytes_rx, Ordering::Relaxed); + self.bytes_tx.store(bytes_tx, Ordering::Relaxed); + self.total_peers.store(total_peers, Ordering::Relaxed); + self.active_peers.store(active_peers, Ordering::Relaxed); + } +} diff --git a/nym-node/nym-node-requests/src/api/v1/metrics/models.rs b/nym-node/nym-node-requests/src/api/v1/metrics/models.rs index 8e430534c5..0ee4170dfe 100644 --- a/nym-node/nym-node-requests/src/api/v1/metrics/models.rs +++ b/nym-node/nym-node-requests/src/api/v1/metrics/models.rs @@ -4,6 +4,19 @@ pub use mixing::*; pub use session::*; pub use verloc::*; +pub use wireguard::*; + +pub mod wireguard { + use serde::{Deserialize, Serialize}; + + #[derive(Serialize, Deserialize, Debug, Clone, Copy)] + #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] + pub struct WireguardStats { + pub bytes_tx: usize, + + pub bytes_rx: usize, + } +} pub mod packets { use serde::{Deserialize, Serialize}; diff --git a/nym-node/nym-node-requests/src/lib.rs b/nym-node/nym-node-requests/src/lib.rs index 661936797a..1d7cd01846 100644 --- a/nym-node/nym-node-requests/src/lib.rs +++ b/nym-node/nym-node-requests/src/lib.rs @@ -66,12 +66,18 @@ pub mod routes { pub const LEGACY_MIXING: &str = "/mixing"; pub const PACKETS_STATS: &str = "/packets-stats"; + pub const WIREGUARD_STATS: &str = "/wireguard-stats"; pub const SESSIONS: &str = "/sessions"; pub const VERLOC: &str = "/verloc"; pub const PROMETHEUS: &str = "/prometheus"; absolute_route!(legacy_mixing_absolute, metrics_absolute(), LEGACY_MIXING); absolute_route!(packets_stats_absolute, metrics_absolute(), PACKETS_STATS); + absolute_route!( + wireguard_stats_absolute, + metrics_absolute(), + WIREGUARD_STATS + ); absolute_route!(sessions_absolute, metrics_absolute(), SESSIONS); absolute_route!(verloc_absolute, metrics_absolute(), VERLOC); absolute_route!(prometheus_absolute, metrics_absolute(), PROMETHEUS); diff --git a/nym-node/src/node/http/router/api/v1/metrics/mod.rs b/nym-node/src/node/http/router/api/v1/metrics/mod.rs index 201dbc7a26..71a9760273 100644 --- a/nym-node/src/node/http/router/api/v1/metrics/mod.rs +++ b/nym-node/src/node/http/router/api/v1/metrics/mod.rs @@ -5,6 +5,7 @@ use crate::node::http::api::v1::metrics::packets_stats::packets_stats; use crate::node::http::api::v1::metrics::prometheus::prometheus_metrics; use crate::node::http::api::v1::metrics::sessions::sessions_stats; use crate::node::http::api::v1::metrics::verloc::verloc_stats; +use crate::node::http::api::v1::metrics::wireguard::wireguard_stats; use crate::node::http::state::metrics::MetricsAppState; use axum::extract::FromRef; use axum::routing::get; @@ -16,6 +17,7 @@ pub mod packets_stats; pub mod prometheus; pub mod sessions; pub mod verloc; +pub mod wireguard; #[derive(Debug, Clone, Default)] pub struct Config { @@ -34,6 +36,7 @@ where get(legacy_mixing::legacy_mixing_stats), ) .route(metrics::PACKETS_STATS, get(packets_stats)) + .route(metrics::WIREGUARD_STATS, get(wireguard_stats)) .route(metrics::SESSIONS, get(sessions_stats)) .route(metrics::VERLOC, get(verloc_stats)) .route(metrics::PROMETHEUS, get(prometheus_metrics)) diff --git a/nym-node/src/node/http/router/api/v1/metrics/packets_stats.rs b/nym-node/src/node/http/router/api/v1/metrics/packets_stats.rs index d1a8e27f55..490bcffb57 100644 --- a/nym-node/src/node/http/router/api/v1/metrics/packets_stats.rs +++ b/nym-node/src/node/http/router/api/v1/metrics/packets_stats.rs @@ -1,5 +1,5 @@ // Copyright 2024 - Nym Technologies SA -// SPDX-License-Identifier: Apache-2.0 +// SPDX-License-Identifier: GPL-3.0-only use crate::node::http::state::metrics::MetricsAppState; use axum::extract::{Query, State}; diff --git a/nym-node/src/node/http/router/api/v1/metrics/wireguard.rs b/nym-node/src/node/http/router/api/v1/metrics/wireguard.rs new file mode 100644 index 0000000000..3519ff30dd --- /dev/null +++ b/nym-node/src/node/http/router/api/v1/metrics/wireguard.rs @@ -0,0 +1,40 @@ +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +use crate::node::http::state::metrics::MetricsAppState; +use axum::extract::{Query, State}; +use nym_http_api_common::{FormattedResponse, OutputParams}; +use nym_node_metrics::NymNodeMetrics; +use nym_node_requests::api::v1::metrics::models::WireguardStats; + +/// If applicable, returns wireguard statistics information of this node. +/// This information is **PURELY** self-reported and in no way validated. +#[utoipa::path( + get, + path = "/wireguard-stats", + context_path = "/api/v1/metrics", + tag = "Metrics", + responses( + (status = 200, content( + ("application/json" = WireguardStats), + ("application/yaml" = WireguardStats) + )) + ), + params(OutputParams), +)] +pub(crate) async fn wireguard_stats( + Query(output): Query, + State(metrics_state): State, +) -> WireguardStatsResponse { + let output = output.output.unwrap_or_default(); + output.to_response(build_response(&metrics_state.metrics)) +} + +fn build_response(metrics: &NymNodeMetrics) -> WireguardStats { + WireguardStats { + bytes_tx: metrics.wireguard.bytes_tx(), + bytes_rx: metrics.wireguard.bytes_rx(), + } +} + +pub type WireguardStatsResponse = FormattedResponse; diff --git a/nym-node/src/node/metrics/console_logger.rs b/nym-node/src/node/metrics/console_logger.rs index ea3d11e94c..37ad46b7f9 100644 --- a/nym-node/src/node/metrics/console_logger.rs +++ b/nym-node/src/node/metrics/console_logger.rs @@ -25,6 +25,9 @@ struct AtLastUpdate { // EGRESS ack_packets_sent: usize, + + wg_tx: usize, + wg_rx: usize, } impl AtLastUpdate { @@ -35,6 +38,8 @@ impl AtLastUpdate { final_hop_packets_received: 0, forward_hop_packets_sent: 0, ack_packets_sent: 0, + wg_tx: 0, + wg_rx: 0, } } } @@ -70,6 +75,9 @@ impl ConsoleLogger { let forward_sent = self.metrics.mixnet.egress.forward_hop_packets_sent(); let acks = self.metrics.mixnet.egress.ack_packets_sent(); + let wg_tx = self.metrics.wireguard.bytes_tx(); + let wg_rx = self.metrics.wireguard.bytes_rx(); + let forward_received_rate = (forward_received - self.at_last_update.forward_hop_packets_received) as f64 / delta_secs; @@ -79,6 +87,9 @@ impl ConsoleLogger { (forward_sent - self.at_last_update.forward_hop_packets_sent) as f64 / delta_secs; let acks_rate = (acks - self.at_last_update.ack_packets_sent) as f64 / delta_secs; + let wg_tx_rate = (wg_tx - self.at_last_update.wg_tx) as f64 / delta_secs; + let wg_rx_rate = (wg_rx - self.at_last_update.wg_rx) as f64 / delta_secs; + info!("↑↓ Packets sent [total] / sent [acks] / received [mix] / received [gw]: {} ({}) / {} ({}) / {} ({}) / {} ({})", forward_sent.human_count_bare(), forward_sent_rate.human_throughput_bare(), @@ -90,11 +101,24 @@ impl ConsoleLogger { final_rate.human_throughput_bare(), ); + // only log wireguard if we have transmitted ANY bytes + if self.at_last_update.wg_rx != 0 { + info!( + "↑↓ Wireguard tx/rx: {} ({}) / {} ({})", + wg_tx.human_count_bytes(), + wg_tx_rate.human_throughput_bytes(), + wg_rx.human_count_bytes(), + wg_rx_rate.human_throughput_bytes() + ) + } + self.at_last_update.time = now; self.at_last_update.forward_hop_packets_received = forward_received; self.at_last_update.final_hop_packets_received = final_received; self.at_last_update.forward_hop_packets_sent = forward_sent; self.at_last_update.ack_packets_sent = acks; + self.at_last_update.wg_tx = wg_tx; + self.at_last_update.wg_rx = wg_rx; // TODO: add websocket-client traffic } diff --git a/nym-node/src/node/mod.rs b/nym-node/src/node/mod.rs index 1311bb4f57..b4e8824eaa 100644 --- a/nym-node/src/node/mod.rs +++ b/nym-node/src/node/mod.rs @@ -581,6 +581,7 @@ impl NymNode { self.entry_gateway.client_storage.clone(), mix_packet_sender, metrics_sender, + self.metrics.clone(), self.entry_gateway.mnemonic.clone(), task_client, ); From 3efeededc519f040de89d0ad8a33e1dc94f0ee48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Stuczy=C5=84ski?= Date: Fri, 20 Dec 2024 10:32:56 +0000 Subject: [PATCH 05/19] feature: expand nym-node prometheus metrics (#5298) * fixed bearer auth for prometheus route * basic prometheus metrics * added rates on global values * improved structure on the prometheus metrics * added additional metrics for ingress websockets and egress mixnet connections * some channel business metrics * fixed metrics registration and added additional variants * added counter for number of disk persisted packets * counter for pending egress packets * counter for pending egress forward packets * clippy --- Cargo.lock | 4 +- common/client-libs/mixnet-client/Cargo.toml | 4 +- .../client-libs/mixnet-client/src/client.rs | 227 +++++---- common/nonexhaustive-delayqueue/src/lib.rs | 8 + common/nym-metrics/Cargo.toml | 2 +- common/nym-metrics/src/lib.rs | 471 +++++++++++++++--- .../node/client_handling/active_clients.rs | 7 + .../client_handling/websocket/common_state.rs | 2 + .../client_handling/websocket/listener.rs | 8 +- gateway/src/node/mod.rs | 1 + nym-node/nym-node-metrics/src/lib.rs | 9 + nym-node/nym-node-metrics/src/mixnet.rs | 17 +- nym-node/nym-node-metrics/src/network.rs | 32 ++ nym-node/nym-node-metrics/src/process.rs | 57 +++ .../src/prometheus_wrapper.rs | 390 +++++++++++++++ nym-node/src/config/metrics.rs | 17 +- nym-node/src/config/mod.rs | 1 + nym-node/src/node/http/error.rs | 2 +- .../node/http/router/api/v1/metrics/mod.rs | 33 +- .../http/router/api/v1/metrics/prometheus.rs | 26 +- nym-node/src/node/http/router/mod.rs | 7 + nym-node/src/node/http/state/metrics.rs | 2 - nym-node/src/node/http/state/mod.rs | 12 +- nym-node/src/node/metrics/aggregator.rs | 2 +- nym-node/src/node/metrics/events_listener.rs | 2 +- .../node/metrics/handler/client_sessions.rs | 11 +- .../at_last_update.rs | 219 ++++++++ .../handler/global_prometheus_updater/mod.rs | 223 +++++++++ .../metrics/handler/legacy_packet_data.rs | 2 +- nym-node/src/node/metrics/handler/mod.rs | 19 +- .../handler/pending_egress_packets_updater.rs | 60 +++ .../handler/prometheus_events_handler.rs | 6 + nym-node/src/node/mixnet/handler.rs | 9 +- nym-node/src/node/mixnet/packet_forwarding.rs | 25 +- nym-node/src/node/mod.rs | 62 ++- nym-node/src/node/shared_topology.rs | 5 + .../ip-packet-router/src/error.rs | 2 +- .../src/request_filter/mod.rs | 11 +- 38 files changed, 1741 insertions(+), 256 deletions(-) create mode 100644 nym-node/nym-node-metrics/src/process.rs create mode 100644 nym-node/nym-node-metrics/src/prometheus_wrapper.rs create mode 100644 nym-node/src/node/metrics/handler/global_prometheus_updater/at_last_update.rs create mode 100644 nym-node/src/node/metrics/handler/global_prometheus_updater/mod.rs create mode 100644 nym-node/src/node/metrics/handler/pending_egress_packets_updater.rs create mode 100644 nym-node/src/node/metrics/handler/prometheus_events_handler.rs diff --git a/Cargo.lock b/Cargo.lock index cc9ed04a83..b0c33bb62b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5679,18 +5679,20 @@ version = "0.1.0" dependencies = [ "dashmap", "lazy_static", - "log", "prometheus", + "tracing", ] [[package]] name = "nym-mixnet-client" version = "0.1.0" dependencies = [ + "dashmap", "futures", "nym-sphinx", "nym-task", "tokio", + "tokio-stream", "tokio-util", "tracing", ] diff --git a/common/client-libs/mixnet-client/Cargo.toml b/common/client-libs/mixnet-client/Cargo.toml index 25dd62f702..b240aab513 100644 --- a/common/client-libs/mixnet-client/Cargo.toml +++ b/common/client-libs/mixnet-client/Cargo.toml @@ -8,10 +8,12 @@ license.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +dashmap = { workspace = true } futures = { workspace = true } tracing = { workspace = true } -tokio = { workspace = true, features = ["time"] } +tokio = { workspace = true, features = ["time", "sync"] } tokio-util = { workspace = true, features = ["codec"], optional = true } +tokio-stream = { workspace = true } # internal nym-sphinx = { path = "../../nymsphinx" } diff --git a/common/client-libs/mixnet-client/src/client.rs b/common/client-libs/mixnet-client/src/client.rs index b8eebcdcc5..94e32f8404 100644 --- a/common/client-libs/mixnet-client/src/client.rs +++ b/common/client-libs/mixnet-client/src/client.rs @@ -1,21 +1,24 @@ -// Copyright 2021 - Nym Technologies SA +// Copyright 2021-2024 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use futures::channel::mpsc; +use dashmap::DashMap; use futures::StreamExt; use nym_sphinx::addressing::nodes::NymNodeRoutingAddress; use nym_sphinx::framing::codec::NymCodec; use nym_sphinx::framing::packet::FramedNymPacket; use nym_sphinx::params::PacketType; use nym_sphinx::NymPacket; -use std::collections::HashMap; use std::io; use std::net::SocketAddr; -use std::sync::atomic::{AtomicU32, Ordering}; +use std::ops::Deref; +use std::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; use std::sync::Arc; use std::time::Duration; use tokio::net::TcpStream; +use tokio::sync::mpsc; +use tokio::sync::mpsc::error::TrySendError; use tokio::time::sleep; +use tokio_stream::wrappers::ReceiverStream; use tokio_util::codec::Framed; use tracing::*; @@ -55,11 +58,37 @@ pub trait SendWithoutResponse { } pub struct Client { - conn_new: HashMap, + active_connections: ActiveConnections, + connections_count: Arc, config: Config, } -struct ConnectionSender { +#[derive(Default, Clone)] +pub struct ActiveConnections { + inner: Arc>, +} + +impl ActiveConnections { + pub fn pending_packets(&self) -> usize { + self.inner + .iter() + .map(|sender| { + let max_capacity = sender.channel.max_capacity(); + let capacity = sender.channel.capacity(); + max_capacity - capacity + }) + .sum() + } +} + +impl Deref for ActiveConnections { + type Target = DashMap; + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +pub struct ConnectionSender { channel: mpsc::Sender, current_reconnection_attempt: Arc, } @@ -73,46 +102,53 @@ impl ConnectionSender { } } -impl Client { - pub fn new(config: Config) -> Client { - Client { - conn_new: HashMap::new(), - config, +struct ManagedConnection { + address: SocketAddr, + message_receiver: ReceiverStream, + connection_timeout: Duration, + current_reconnection: Arc, +} + +impl ManagedConnection { + fn new( + address: SocketAddr, + message_receiver: mpsc::Receiver, + connection_timeout: Duration, + current_reconnection: Arc, + ) -> Self { + ManagedConnection { + address, + message_receiver: ReceiverStream::new(message_receiver), + connection_timeout, + current_reconnection, } } - async fn manage_connection( - address: SocketAddr, - receiver: mpsc::Receiver, - connection_timeout: Duration, - current_reconnection: &AtomicU32, - ) { + async fn run(self) { + let address = self.address; let connection_fut = TcpStream::connect(address); - let conn = match tokio::time::timeout(connection_timeout, connection_fut).await { + let conn = match tokio::time::timeout(self.connection_timeout, connection_fut).await { Ok(stream_res) => match stream_res { Ok(stream) => { - debug!("Managed to establish connection to {}", address); + debug!("Managed to establish connection to {}", self.address); // if we managed to connect, reset the reconnection count (whatever it might have been) - current_reconnection.store(0, Ordering::Release); + self.current_reconnection.store(0, Ordering::Release); Framed::new(stream, NymCodec) } Err(err) => { - debug!( - "failed to establish connection to {} (err: {})", - address, err - ); + debug!("failed to establish connection to {address} (err: {err})",); return; } }, Err(_) => { debug!( - "failed to connect to {} within {:?}", - address, connection_timeout + "failed to connect to {address} within {:?}", + self.connection_timeout ); // we failed to connect - increase reconnection attempt - current_reconnection.fetch_add(1, Ordering::SeqCst); + self.current_reconnection.fetch_add(1, Ordering::SeqCst); return; } }; @@ -120,15 +156,28 @@ impl Client { // Take whatever the receiver channel produces and put it on the connection. // We could have as well used conn.send_all(receiver.map(Ok)), but considering we don't care // about neither receiver nor the connection, it doesn't matter which one gets consumed - if let Err(err) = receiver.map(Ok).forward(conn).await { - warn!("Failed to forward packets to {} - {err}", address); + if let Err(err) = self.message_receiver.map(Ok).forward(conn).await { + warn!("Failed to forward packets to {address}: {err}"); } debug!( - "connection manager to {} is finished. Either the connection failed or mixnet client got dropped", - address + "connection manager to {address} is finished. Either the connection failed or mixnet client got dropped", ); } +} + +impl Client { + pub fn new(config: Config, connections_count: Arc) -> Client { + Client { + active_connections: Default::default(), + connections_count, + config, + } + } + + pub fn active_connections(&self) -> ActiveConnections { + self.active_connections.clone() + } /// If we're trying to reconnect, determine how long we should wait. fn determine_backoff(&self, current_attempt: u32) -> Option { @@ -148,7 +197,7 @@ impl Client { } fn make_connection(&mut self, address: NymNodeRoutingAddress, pending_packet: FramedNymPacket) { - let (mut sender, receiver) = mpsc::channel(self.config.maximum_connection_buffer_size); + let (sender, receiver) = mpsc::channel(self.config.maximum_connection_buffer_size); // this CAN'T fail because we just created the channel which has a non-zero capacity if self.config.maximum_connection_buffer_size > 0 { @@ -156,15 +205,16 @@ impl Client { } // if we already tried to connect to `address` before, grab the current attempt count - let current_reconnection_attempt = if let Some(existing) = self.conn_new.get_mut(&address) { - existing.channel = sender; - Arc::clone(&existing.current_reconnection_attempt) - } else { - let new_entry = ConnectionSender::new(sender); - let current_attempt = Arc::clone(&new_entry.current_reconnection_attempt); - self.conn_new.insert(address, new_entry); - current_attempt - }; + let current_reconnection_attempt = + if let Some(mut existing) = self.active_connections.get_mut(&address) { + existing.channel = sender; + Arc::clone(&existing.current_reconnection_attempt) + } else { + let new_entry = ConnectionSender::new(sender); + let current_attempt = Arc::clone(&new_entry.current_reconnection_attempt); + self.active_connections.insert(address, new_entry); + current_attempt + }; // load the actual value. let reconnection_attempt = current_reconnection_attempt.load(Ordering::Acquire); @@ -173,6 +223,7 @@ impl Client { // copy the value before moving into another task let initial_connection_timeout = self.config.initial_connection_timeout; + let connections_count = self.connections_count.clone(); tokio::spawn(async move { // before executing the manager, wait for what was specified, if anything if let Some(backoff) = backoff { @@ -180,13 +231,16 @@ impl Client { sleep(backoff).await; } - Self::manage_connection( + connections_count.fetch_add(1, Ordering::SeqCst); + ManagedConnection::new( address.into(), receiver, initial_connection_timeout, - ¤t_reconnection_attempt, + current_reconnection_attempt, ) - .await + .run() + .await; + connections_count.fetch_sub(1, Ordering::SeqCst); }); } } @@ -201,49 +255,47 @@ impl SendWithoutResponse for Client { trace!("Sending packet to {address:?}"); let framed_packet = FramedNymPacket::new(packet, packet_type); - if let Some(sender) = self.conn_new.get_mut(&address) { - if let Err(err) = sender.channel.try_send(framed_packet) { - if err.is_full() { - debug!("Connection to {} seems to not be able to handle all the traffic - dropping the current packet", address); - // it's not a 'big' error, but we did not manage to send the packet - // if the queue is full, we can't really do anything but to drop the packet - Err(io::Error::new( - io::ErrorKind::WouldBlock, - "connection queue is full", - )) - } else if err.is_disconnected() { - debug!( - "Connection to {} seems to be dead. attempting to re-establish it...", - address - ); - // it's not a 'big' error, but we did not manage to send the packet, but queue - // it up to send it as soon as the connection is re-established - self.make_connection(address, err.into_inner()); - Err(io::Error::new( - io::ErrorKind::ConnectionAborted, - "reconnection attempt is in progress", - )) - } else { - // this can't really happen, but let's safe-guard against it in case something changes in futures library - Err(io::Error::new( - io::ErrorKind::Other, - "unknown connection buffer error", - )) - } - } else { - Ok(()) - } - } else { + let Some(sender) = self.active_connections.get_mut(&address) else { // there was never a connection to begin with debug!("establishing initial connection to {}", address); // it's not a 'big' error, but we did not manage to send the packet, but queue the packet // for sending for as soon as the connection is created self.make_connection(address, framed_packet); - Err(io::Error::new( + return Err(io::Error::new( io::ErrorKind::NotConnected, "connection is in progress", - )) - } + )); + }; + + let sending_res = sender.channel.try_send(framed_packet); + drop(sender); + + sending_res.map_err(|err| { + match err { + TrySendError::Full(_) => { + debug!("Connection to {address} seems to not be able to handle all the traffic - dropping the current packet"); + // it's not a 'big' error, but we did not manage to send the packet + // if the queue is full, we can't really do anything but to drop the packet + io::Error::new( + io::ErrorKind::WouldBlock, + "connection queue is full", + ) + } + TrySendError::Closed(dropped) => { + debug!( + "Connection to {address} seems to be dead. attempting to re-establish it...", + ); + + // it's not a 'big' error, but we did not manage to send the packet, but queue + // it up to send it as soon as the connection is re-established + self.make_connection(address, dropped); + io::Error::new( + io::ErrorKind::ConnectionAborted, + "reconnection attempt is in progress", + ) + } + } + } ) } } @@ -252,12 +304,15 @@ mod tests { use super::*; fn dummy_client() -> Client { - Client::new(Config { - initial_reconnection_backoff: Duration::from_millis(10_000), - maximum_reconnection_backoff: Duration::from_millis(300_000), - initial_connection_timeout: Duration::from_millis(1_500), - maximum_connection_buffer_size: 128, - }) + Client::new( + Config { + initial_reconnection_backoff: Duration::from_millis(10_000), + maximum_reconnection_backoff: Duration::from_millis(300_000), + initial_connection_timeout: Duration::from_millis(1_500), + maximum_connection_buffer_size: 128, + }, + Default::default(), + ) } #[test] diff --git a/common/nonexhaustive-delayqueue/src/lib.rs b/common/nonexhaustive-delayqueue/src/lib.rs index ee8c9f214c..6c4853f675 100644 --- a/common/nonexhaustive-delayqueue/src/lib.rs +++ b/common/nonexhaustive-delayqueue/src/lib.rs @@ -65,6 +65,14 @@ impl NonExhaustiveDelayQueue { pub fn remove(&mut self, key: &QueueKey) -> Expired { self.inner.remove(key) } + + pub fn len(&self) -> usize { + self.inner.len() + } + + pub fn is_empty(&self) -> bool { + self.inner.is_empty() + } } impl Default for NonExhaustiveDelayQueue { diff --git a/common/nym-metrics/Cargo.toml b/common/nym-metrics/Cargo.toml index 4310ae28f6..1ac3bce16a 100644 --- a/common/nym-metrics/Cargo.toml +++ b/common/nym-metrics/Cargo.toml @@ -12,6 +12,6 @@ license.workspace = true [dependencies] prometheus = { workspace = true } -log = { workspace = true } +tracing = { workspace = true } dashmap = { workspace = true } lazy_static = { workspace = true } diff --git a/common/nym-metrics/src/lib.rs b/common/nym-metrics/src/lib.rs index 565d62cd5f..7c6373ccc7 100644 --- a/common/nym-metrics/src/lib.rs +++ b/common/nym-metrics/src/lib.rs @@ -1,14 +1,18 @@ use dashmap::DashMap; -pub use log::error; -use log::{debug, warn}; use std::fmt; -pub use std::time::Instant; +use tracing::{debug, error, warn}; -use prometheus::{core::Collector, Encoder as _, IntCounter, IntGauge, Registry, TextEncoder}; +use prometheus::{ + core::Collector, Encoder as _, Gauge, Histogram, HistogramOpts, IntCounter, IntGauge, Registry, + TextEncoder, +}; + +pub use prometheus::HistogramTimer; +pub use std::time::Instant; #[macro_export] macro_rules! prepend_package_name { - ($name: literal) => { + ($name: tt) => { &format!( "{}_{}", std::module_path!() @@ -23,15 +27,29 @@ macro_rules! prepend_package_name { #[macro_export] macro_rules! inc_by { + ($name:literal, $x:expr, $help: expr) => { + $crate::REGISTRY.maybe_register_and_inc_by( + $crate::prepend_package_name!($name), + $x as i64, + $help, + ); + }; ($name:literal, $x:expr) => { - $crate::REGISTRY.inc_by($crate::prepend_package_name!($name), $x as i64); + $crate::REGISTRY.maybe_register_and_inc_by( + $crate::prepend_package_name!($name), + $x as i64, + None, + ); }; } #[macro_export] macro_rules! inc { + ($name:literal, $help: expr) => { + $crate::REGISTRY.maybe_register_and_inc($crate::prepend_package_name!($name), $help); + }; ($name:literal) => { - $crate::REGISTRY.inc($crate::prepend_package_name!($name)); + $crate::REGISTRY.maybe_register_and_inc($crate::prepend_package_name!($name), None); }; } @@ -42,6 +60,71 @@ macro_rules! metrics { }; } +#[macro_export] +macro_rules! set_metric { + ($name:literal, $x:expr, $help: expr) => { + $crate::REGISTRY.maybe_register_and_set( + $crate::prepend_package_name!($name), + $x as i64, + $help, + ); + }; + ($name:literal, $x:expr) => { + $crate::REGISTRY.maybe_register_and_set( + $crate::prepend_package_name!($name), + $x as i64, + None, + ); + }; +} + +#[macro_export] +macro_rules! set_metric_float { + ($name:literal, $x:expr, $help: expr) => { + $crate::REGISTRY.maybe_register_and_set_float( + $crate::prepend_package_name!($name), + $x as f64, + $help, + ); + }; + ($name:literal, $x:expr) => { + $crate::REGISTRY.maybe_register_and_set_float( + $crate::prepend_package_name!($name), + $x as f64, + None, + ); + }; +} + +#[macro_export] +macro_rules! add_histogram_obs { + ($name:expr, $x:expr, $b:expr, $help:expr) => { + $crate::REGISTRY.maybe_register_and_add_to_histogram( + $crate::prepend_package_name!($name), + $x as f64, + Some($b), + $help, + ); + }; + + ($name:expr, $x:expr, $b:expr) => { + $crate::REGISTRY.maybe_register_and_add_to_histogram( + $crate::prepend_package_name!($name), + $x as f64, + Some($b), + None, + ); + }; + ($name:expr, $x:expr) => { + $crate::REGISTRY.maybe_register_and_add_to_histogram( + $crate::prepend_package_name!($name), + $x as f64, + None, + None, + ); + }; +} + #[macro_export] macro_rules! nanos { ( $name:literal, $x:expr ) => {{ @@ -50,7 +133,7 @@ macro_rules! nanos { let r = $x; let duration = start.elapsed().as_nanos() as i64; let name = $crate::prepend_package_name!($name); - $crate::REGISTRY.inc_by(&format!("{}_nanos", $name), duration); + $crate::REGISTRY.maybe_register_and_inc_by(&format!("{}_nanos", $name), duration, None); r }}; } @@ -59,15 +142,100 @@ lazy_static::lazy_static! { pub static ref REGISTRY: MetricsController = MetricsController::default(); } +pub fn metrics_registry() -> &'static MetricsController { + ®ISTRY +} + #[derive(Default)] pub struct MetricsController { registry: Registry, registry_index: DashMap, } -enum Metric { - C(Box), - G(Box), +pub enum Metric { + IntCounter(Box), + IntGauge(Box), + FloatGauge(Box), + Histogram(Box), +} + +impl Metric { + pub fn new_int_counter(name: &str, help: &str) -> Option { + match IntCounter::new(sanitize_metric_name(name), help) { + Ok(c) => Some(c.into()), + Err(err) => { + error!("Failed to create counter {name:?}: {err}"); + None + } + } + } + + pub fn new_int_gauge(name: &str, help: &str) -> Option { + match IntGauge::new(sanitize_metric_name(name), help) { + Ok(g) => Some(g.into()), + Err(err) => { + error!("Failed to create gauge {name:?}: {err}"); + None + } + } + } + + pub fn new_float_gauge(name: &str, help: &str) -> Option { + match Gauge::new(sanitize_metric_name(name), help) { + Ok(g) => Some(g.into()), + Err(err) => { + error!("Failed to create gauge {name:?}: {err}"); + None + } + } + } + + pub fn new_histogram(name: &str, help: &str, buckets: Option<&[f64]>) -> Option { + let mut opts = HistogramOpts::new(sanitize_metric_name(name), help); + if let Some(buckets) = buckets { + opts = opts.buckets(buckets.to_vec()) + } + match Histogram::with_opts(opts) { + Ok(h) => Some(Metric::Histogram(Box::new(h))), + Err(err) => { + error!("failed to create histogram {name:?}: {err}"); + None + } + } + } + + fn as_collector(&self) -> Box { + match self { + Metric::IntCounter(c) => c.clone(), + Metric::IntGauge(g) => g.clone(), + Metric::FloatGauge(g) => g.clone(), + Metric::Histogram(h) => h.clone(), + } + } +} + +impl From for Metric { + fn from(v: IntCounter) -> Self { + Metric::IntCounter(Box::new(v)) + } +} + +impl From for Metric { + fn from(v: IntGauge) -> Self { + Metric::IntGauge(Box::new(v)) + } +} + +impl From for Metric { + fn from(v: Gauge) -> Self { + Metric::FloatGauge(Box::new(v)) + } +} + +impl From for Metric { + fn from(v: Histogram) -> Self { + Metric::Histogram(Box::new(v)) + } } fn fq_name(c: &dyn Collector) -> String { @@ -81,34 +249,92 @@ impl Metric { #[inline(always)] fn fq_name(&self) -> String { match self { - Metric::C(c) => fq_name(c.as_ref()), - Metric::G(g) => fq_name(g.as_ref()), + Metric::IntCounter(c) => fq_name(c.as_ref()), + Metric::IntGauge(g) => fq_name(g.as_ref()), + Metric::FloatGauge(g) => fq_name(g.as_ref()), + Metric::Histogram(h) => fq_name(h.as_ref()), } } #[inline(always)] fn inc(&self) { match self { - Metric::C(c) => c.inc(), - Metric::G(g) => g.inc(), + Metric::IntCounter(c) => c.inc(), + Metric::IntGauge(g) => g.inc(), + Metric::FloatGauge(g) => g.inc(), + Metric::Histogram(_) => { + warn!("invalid operation: attempted to call increment on a histogram") + } } } #[inline(always)] fn inc_by(&self, value: i64) { match self { - Metric::C(c) => c.inc_by(value as u64), - Metric::G(g) => g.add(value), + Metric::IntCounter(c) => c.inc_by(value as u64), + Metric::IntGauge(g) => g.add(value), + Metric::FloatGauge(g) => { + warn!("attempted to increment a float gauge ('{}') by an integer - this is most likely a bug", self.fq_name()); + g.add(value as f64) + } + Metric::Histogram(_) => { + warn!("invalid operation: attempted to call increment on a histogram") + } } } #[inline(always)] fn set(&self, value: i64) { match self { - Metric::C(_c) => { + Metric::IntCounter(_c) => { warn!("Cannot set value for counter {:?}", self.fq_name()); } - Metric::G(g) => g.set(value), + Metric::IntGauge(g) => g.set(value), + Metric::FloatGauge(g) => { + warn!("attempted to set a float gauge ('{}') to an integer value - this is most likely a bug", self.fq_name()); + g.set(value as f64) + } + Metric::Histogram(_) => { + warn!("invalid operation: attempted to call set on a histogram") + } + } + } + + #[inline(always)] + fn set_float(&self, value: f64) { + match self { + Metric::IntCounter(_c) => { + warn!("Cannot set value for counter {:?}", self.fq_name()); + } + Metric::IntGauge(g) => { + warn!("attempted to set a integer gauge ('{}') to a float value - this is most likely a bug", self.fq_name()); + g.set(value as i64) + } + Metric::FloatGauge(g) => g.set(value), + Metric::Histogram(_) => { + warn!("invalid operation: attempted to call increment on a histogram") + } + } + } + + #[inline(always)] + fn add_histogram_observation(&self, value: f64) { + match self { + Metric::Histogram(h) => { + h.observe(value); + } + _ => warn!("attempted to add histogram observation on a non-histogram metric"), + } + } + + #[inline(always)] + fn start_timer(&self) -> Option { + match self { + Metric::Histogram(h) => Some(h.start_timer()), + _ => { + warn!("attempted to start histogram observation on a non-histogram metric"); + None + } } } } @@ -145,93 +371,165 @@ impl MetricsController { } } - pub fn set(&self, name: &str, value: i64) { + pub fn register_int_gauge<'a>(&self, name: &str, help: impl Into>) { + let Some(metric) = Metric::new_int_gauge(name, help.into().unwrap_or(name)) else { + return; + }; + self.register_metric(metric); + } + + pub fn register_float_gauge<'a>(&self, name: &str, help: impl Into>) { + let Some(metric) = Metric::new_float_gauge(name, help.into().unwrap_or(name)) else { + return; + }; + self.register_metric(metric); + } + + pub fn register_int_counter<'a>(&self, name: &str, help: impl Into>) { + let Some(metric) = Metric::new_int_counter(name, help.into().unwrap_or(name)) else { + return; + }; + self.register_metric(metric); + } + + pub fn register_histogram<'a>( + &self, + name: &str, + help: impl Into>, + buckets: Option<&[f64]>, + ) { + let Some(metric) = Metric::new_histogram(name, help.into().unwrap_or(name), buckets) else { + return; + }; + self.register_metric(metric); + } + + pub fn set(&self, name: &str, value: i64) -> bool { if let Some(metric) = self.registry_index.get(name) { metric.set(value); + true } else { - let gauge = match IntGauge::new(sanitize_metric_name(name), name) { - Ok(g) => g, - Err(e) => { - debug!("Failed to create gauge {:?}:\n{}", name, e); - return; - } - }; - self.register_gauge(Box::new(gauge)); - self.set(name, value) + false } } - pub fn inc(&self, name: &str) { + pub fn set_float(&self, name: &str, value: f64) -> bool { + if let Some(metric) = self.registry_index.get(name) { + metric.set_float(value); + true + } else { + false + } + } + + pub fn add_to_histogram(&self, name: &str, value: f64) -> bool { + if let Some(metric) = self.registry_index.get(name) { + metric.add_histogram_observation(value); + true + } else { + false + } + } + + pub fn start_timer(&self, name: &str) -> Option { + self.registry_index + .get(name) + .and_then(|metric| metric.start_timer()) + } + + pub fn inc(&self, name: &str) -> bool { if let Some(metric) = self.registry_index.get(name) { metric.inc(); + true } else { - let counter = match IntCounter::new(sanitize_metric_name(name), name) { - Ok(c) => c, - Err(e) => { - debug!("Failed to create counter {:?}:\n{}", name, e); - return; - } - }; - self.register_counter(Box::new(counter)); - self.inc(name) + false } } - pub fn inc_by(&self, name: &str, value: i64) { + pub fn inc_by(&self, name: &str, value: i64) -> bool { if let Some(metric) = self.registry_index.get(name) { metric.inc_by(value); + true } else { - let counter = match IntCounter::new(sanitize_metric_name(name), name) { - Ok(c) => c, - Err(e) => { - debug!("Failed to create counter {:?}:\n{}", name, e); - return; - } - }; - self.register_counter(Box::new(counter)); - self.inc_by(name, value) + false } } - fn register_gauge(&self, metric: Box) { - let fq_name = metric - .desc() - .first() - .map(|d| d.fq_name.clone()) - .unwrap_or_default(); + pub fn maybe_register_and_set<'a>( + &self, + name: &str, + value: i64, + help: impl Into>, + ) { + if !self.set(name, value) { + let help = help.into(); + self.register_int_gauge(name, help); + self.set(name, value); + } + } + + pub fn maybe_register_and_set_float<'a>( + &self, + name: &str, + value: f64, + help: impl Into>, + ) { + if !self.set_float(name, value) { + let help = help.into(); + self.register_float_gauge(name, help); + self.set_float(name, value); + } + } + + pub fn maybe_register_and_add_to_histogram<'a>( + &self, + name: &str, + value: f64, + buckets: Option<&[f64]>, + help: impl Into>, + ) { + if !self.add_to_histogram(name, value) { + let help = help.into(); + self.register_histogram(name, help, buckets); + self.add_to_histogram(name, value); + } + } + + pub fn maybe_register_and_inc<'a>(&self, name: &str, help: impl Into>) { + if !self.inc(name) { + let help = help.into(); + self.register_int_counter(name, help); + self.inc(name); + } + } + + pub fn maybe_register_and_inc_by<'a>( + &self, + name: &str, + value: i64, + help: impl Into>, + ) { + if !self.inc_by(name, value) { + let help = help.into(); + self.register_int_counter(name, help); + self.inc_by(name, value); + } + } + + pub fn register_metric(&self, metric: impl Into) { + let m = metric.into(); + let fq_name = m.fq_name(); if self.registry_index.contains_key(&fq_name) { return; } - match self.registry.register(metric.clone()) { + match self.registry.register(m.as_collector()) { Ok(_) => { - self.registry_index - .insert(fq_name, Metric::G(metric.clone())); + self.registry_index.insert(fq_name, m); } - Err(e) => { - debug!("Failed to register {:?}:\n{}", fq_name, e) - } - } - } - - fn register_counter(&self, metric: Box) { - let fq_name = metric - .desc() - .first() - .map(|d| d.fq_name.clone()) - .unwrap_or_default(); - - if self.registry_index.contains_key(&fq_name) { - return; - } - match self.registry.register(metric.clone()) { - Ok(_) => { - self.registry_index - .insert(fq_name, Metric::C(metric.clone())); - } - Err(e) => { - debug!("Failed to register {:?}:\n{}", fq_name, e) + Err(err) => { + debug!("Failed to register '{fq_name}': {err}") } } } @@ -275,4 +573,15 @@ mod tests { "packets_sent_34_242_65_133:1789" ) } + + #[test] + fn prepend_package_name() { + let literal = prepend_package_name!("foo"); + assert_eq!(literal, "nym_metrics_foo"); + + let bar = "bar"; + let format = format!("foomp_{}", bar); + let formatted = prepend_package_name!(format); + assert_eq!(formatted, "nym_metrics_foomp_bar"); + } } diff --git a/gateway/src/node/client_handling/active_clients.rs b/gateway/src/node/client_handling/active_clients.rs index 3dacfd3f63..4a3e55bef5 100644 --- a/gateway/src/node/client_handling/active_clients.rs +++ b/gateway/src/node/client_handling/active_clients.rs @@ -163,4 +163,11 @@ impl ActiveClientsStore { pub(crate) fn size(&self) -> usize { self.inner.len() } + + pub fn pending_packets(&self) -> usize { + self.inner + .iter() + .map(|client| client.get_sender_ref().len()) + .sum() + } } diff --git a/gateway/src/node/client_handling/websocket/common_state.rs b/gateway/src/node/client_handling/websocket/common_state.rs index 19a5943c53..223c534b26 100644 --- a/gateway/src/node/client_handling/websocket/common_state.rs +++ b/gateway/src/node/client_handling/websocket/common_state.rs @@ -7,6 +7,7 @@ use nym_crypto::asymmetric::identity; use nym_gateway_storage::GatewayStorage; use nym_mixnet_client::forwarder::MixForwardingSender; use nym_node_metrics::events::MetricEventsSender; +use nym_node_metrics::NymNodeMetrics; use std::sync::Arc; // I can see this being possible expanded with say storage or client store @@ -17,6 +18,7 @@ pub(crate) struct CommonHandlerState { pub(crate) local_identity: Arc, pub(crate) only_coconut_credentials: bool, pub(crate) bandwidth_cfg: BandwidthFlushingBehaviourConfig, + pub(crate) metrics: NymNodeMetrics, pub(crate) metrics_sender: MetricEventsSender, pub(crate) outbound_mix_sender: MixForwardingSender, pub(crate) active_clients_store: ActiveClientsStore, diff --git a/gateway/src/node/client_handling/websocket/listener.rs b/gateway/src/node/client_handling/websocket/listener.rs index ed484d17b6..faa659f0fa 100644 --- a/gateway/src/node/client_handling/websocket/listener.rs +++ b/gateway/src/node/client_handling/websocket/listener.rs @@ -61,7 +61,13 @@ impl Listener { remote_addr, shutdown, ); - tokio::spawn(handle.start_handling()); + tokio::spawn(async move { + // TODO: refactor it similarly to the mixnet listener on the nym-node + let metrics_ref = handle.shared_state.metrics.clone(); + metrics_ref.network.new_ingress_websocket_client(); + handle.start_handling().await; + metrics_ref.network.disconnected_ingress_websocket_client(); + }); } Err(err) => warn!("failed to get client: {err}"), } diff --git a/gateway/src/node/mod.rs b/gateway/src/node/mod.rs index b3b41961a1..3304f223b3 100644 --- a/gateway/src/node/mod.rs +++ b/gateway/src/node/mod.rs @@ -252,6 +252,7 @@ impl GatewayTasksBuilder { local_identity: Arc::clone(&self.identity_keypair), only_coconut_credentials: self.config.gateway.enforce_zk_nyms, bandwidth_cfg: (&self.config).into(), + metrics: self.metrics.clone(), metrics_sender: self.metrics_sender.clone(), outbound_mix_sender: self.mix_packet_sender.clone(), active_clients_store: active_clients_store.clone(), diff --git a/nym-node/nym-node-metrics/src/lib.rs b/nym-node/nym-node-metrics/src/lib.rs index 57a8c74bb3..904cdc1407 100644 --- a/nym-node/nym-node-metrics/src/lib.rs +++ b/nym-node/nym-node-metrics/src/lib.rs @@ -1,9 +1,15 @@ // Copyright 2024 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 +#![warn(clippy::expect_used)] +#![warn(clippy::unwrap_used)] +#![warn(clippy::todo)] +#![warn(clippy::dbg_macro)] + use crate::entry::EntryStats; use crate::mixnet::MixingStats; use crate::network::NetworkStats; +use crate::process::NodeStats; use crate::wireguard::WireguardStats; use std::ops::Deref; use std::sync::Arc; @@ -12,6 +18,8 @@ pub mod entry; pub mod events; pub mod mixnet; pub mod network; +pub mod process; +pub mod prometheus_wrapper; pub mod wireguard; #[derive(Clone, Default)] @@ -39,4 +47,5 @@ pub struct NymNodeMetricsInner { pub wireguard: WireguardStats, pub network: NetworkStats, + pub process: NodeStats, } diff --git a/nym-node/nym-node-metrics/src/mixnet.rs b/nym-node/nym-node-metrics/src/mixnet.rs index 043d2d1e7c..32d84e509b 100644 --- a/nym-node/nym-node-metrics/src/mixnet.rs +++ b/nym-node/nym-node-metrics/src/mixnet.rs @@ -131,13 +131,6 @@ impl MixingStats { .or_default() .dropped += 1; } - - pub fn egress_dropped_final_hop_packet(&self) { - todo!() - // self.egress - // .final_hop_packets_dropped - // .fetch_add(1, Ordering::Relaxed); - } } #[derive(Clone, Copy, Default, PartialEq)] @@ -148,6 +141,8 @@ pub struct EgressRecipientStats { #[derive(Default)] pub struct EgressMixingStats { + disk_persisted_packets: AtomicUsize, + // this includes ACKS! forward_hop_packets_sent: AtomicUsize, @@ -159,6 +154,14 @@ pub struct EgressMixingStats { } impl EgressMixingStats { + pub fn add_disk_persisted_packet(&self) { + self.disk_persisted_packets.fetch_add(1, Ordering::Relaxed); + } + + pub fn disk_persisted_packets(&self) -> usize { + self.disk_persisted_packets.load(Ordering::Relaxed) + } + pub fn forward_hop_packets_sent(&self) -> usize { self.forward_hop_packets_sent.load(Ordering::Relaxed) } diff --git a/nym-node/nym-node-metrics/src/network.rs b/nym-node/nym-node-metrics/src/network.rs index de00d78560..7de912fc3c 100644 --- a/nym-node/nym-node-metrics/src/network.rs +++ b/nym-node/nym-node-metrics/src/network.rs @@ -2,11 +2,19 @@ // SPDX-License-Identifier: Apache-2.0 use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; #[derive(Default)] pub struct NetworkStats { // for now just experiment with basic data, we could always extend it active_ingress_mixnet_connections: AtomicUsize, + + active_ingress_websocket_connections: AtomicUsize, + + // the reason for additional `Arc` on this one is that the handler wasn't + // designed with metrics in mind and this single counter has been woven through + // the call stack + active_egress_mixnet_connections: Arc, } impl NetworkStats { @@ -20,8 +28,32 @@ impl NetworkStats { .fetch_sub(1, Ordering::Relaxed); } + pub fn new_ingress_websocket_client(&self) { + self.active_ingress_websocket_connections + .fetch_add(1, Ordering::Relaxed); + } + + pub fn disconnected_ingress_websocket_client(&self) { + self.active_ingress_websocket_connections + .fetch_sub(1, Ordering::Relaxed); + } + pub fn active_ingress_mixnet_connections_count(&self) -> usize { self.active_ingress_mixnet_connections .load(Ordering::Relaxed) } + + pub fn active_ingress_websocket_connections_count(&self) -> usize { + self.active_ingress_websocket_connections + .load(Ordering::Relaxed) + } + + pub fn active_egress_mixnet_connections_counter(&self) -> Arc { + self.active_egress_mixnet_connections.clone() + } + + pub fn active_egress_mixnet_connections_count(&self) -> usize { + self.active_egress_mixnet_connections + .load(Ordering::Relaxed) + } } diff --git a/nym-node/nym-node-metrics/src/process.rs b/nym-node/nym-node-metrics/src/process.rs new file mode 100644 index 0000000000..8e19688249 --- /dev/null +++ b/nym-node/nym-node-metrics/src/process.rs @@ -0,0 +1,57 @@ +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use std::sync::atomic::{AtomicUsize, Ordering}; + +#[derive(Default)] +pub struct NodeStats { + pub final_hop_packets_pending_delivery: AtomicUsize, + + pub forward_hop_packets_pending_delivery: AtomicUsize, + + pub forward_hop_packets_being_delayed: AtomicUsize, + + // packets that haven't yet been delayed and are waiting for their chance + pub packet_forwarder_queue_size: AtomicUsize, +} + +impl NodeStats { + pub fn update_final_hop_packets_pending_delivery(&self, current: usize) { + self.final_hop_packets_pending_delivery + .store(current, Ordering::Relaxed); + } + + pub fn final_hop_packets_pending_delivery_count(&self) -> usize { + self.final_hop_packets_pending_delivery + .load(Ordering::Relaxed) + } + + pub fn update_forward_hop_packets_pending_delivery(&self, current: usize) { + self.forward_hop_packets_pending_delivery + .store(current, Ordering::Relaxed); + } + + pub fn forward_hop_packets_pending_delivery_count(&self) -> usize { + self.forward_hop_packets_pending_delivery + .load(Ordering::Relaxed) + } + + pub fn update_forward_hop_packets_being_delayed(&self, current: usize) { + self.forward_hop_packets_being_delayed + .store(current, Ordering::Relaxed); + } + + pub fn forward_hop_packets_being_delayed_count(&self) -> usize { + self.forward_hop_packets_being_delayed + .load(Ordering::Relaxed) + } + + pub fn update_packet_forwarder_queue_size(&self, current: usize) { + self.packet_forwarder_queue_size + .store(current, Ordering::Relaxed); + } + + pub fn packet_forwarder_queue_size(&self) -> usize { + self.packet_forwarder_queue_size.load(Ordering::Relaxed) + } +} diff --git a/nym-node/nym-node-metrics/src/prometheus_wrapper.rs b/nym-node/nym-node-metrics/src/prometheus_wrapper.rs new file mode 100644 index 0000000000..2d61ee55c4 --- /dev/null +++ b/nym-node/nym-node-metrics/src/prometheus_wrapper.rs @@ -0,0 +1,390 @@ +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +use nym_metrics::{metrics_registry, HistogramTimer, Metric}; +use std::sync::LazyLock; +use strum::{Display, EnumCount, EnumIter, EnumProperty, IntoEnumIterator}; + +pub static PROMETHEUS_METRICS: LazyLock = + LazyLock::new(NymNodePrometheusMetrics::initialise); + +const CLIENT_SESSION_DURATION_BUCKETS: &[f64] = &[ + // sub 3s (implicitly) + 3., // 3s - 15s + 15., // 15s - 70s + 70., // 70s - 2min + 120., // 2 min - 5 min + 300., // 5min - 15min + 900., // 15min - 1h + 3600., // 1h - 12h + 43200., // 12h - 23.5h + 88200., // 23.5h - 24.5h + 86400., // 24.5h - 72h + 259200., // 72h+ (implicitly) +]; + +#[derive(Clone, Debug, EnumIter, Display, EnumProperty, EnumCount, Eq, Hash, PartialEq)] +#[strum(serialize_all = "snake_case", prefix = "nym_node_")] +pub enum PrometheusMetric { + // # MIXNET + // ## INGRESS + #[strum(props(help = "The number of ingress forward hop sphinx packets received"))] + MixnetIngressForwardPacketsReceived, + + #[strum(props(help = "The number of ingress final hop sphinx packets received"))] + MixnetIngressFinalHopPacketsReceived, + + #[strum(props(help = "The number of ingress malformed sphinx packets received"))] + MixnetIngressMalformedPacketsReceived, + + #[strum(props( + help = "The number of ingress forward sphinx packets that specified excessive delay received" + ))] + MixnetIngressExcessiveDelayPacketsReceived, + + #[strum(props(help = "The number of ingress forward hop sphinx packets dropped"))] + MixnetIngressForwardPacketsDropped, + + #[strum(props(help = "The number of ingress final hop sphinx packets dropped"))] + MixnetIngressFinalHopPacketsDropped, + + #[strum(props(help = "The current rate of receiving ingress forward hop sphinx packets"))] + MixnetIngressForwardPacketsReceivedRate, + + #[strum(props(help = "The current rate of receiving ingress final hop sphinx packets"))] + MixnetIngressFinalHopPacketsReceivedRate, + + #[strum(props(help = "The current rate of receiving ingress malformed sphinx packets"))] + MixnetIngressMalformedPacketsReceivedRate, + + #[strum(props( + help = "The current rate of receiving ingress sphinx packets that specified excessive delay" + ))] + MixnetIngressExcessiveDelayPacketsReceivedRate, + + #[strum(props(help = "The current rate of dropping ingress forward hop sphinx packets"))] + MixnetIngressForwardPacketsDroppedRate, + + #[strum(props(help = "The current rate of dropping ingress final hop sphinx packets"))] + MixnetIngressFinalHopPacketsDroppedRate, + + // ## EGRESS + #[strum(props( + help = "The number of unwrapped final hop packets stored on disk for offline clients" + ))] + MixnetEgressStoredOnDiskFinalHopPackets, + + #[strum(props(help = "The number of egress forward hop sphinx packets sent/forwarded"))] + MixnetEgressForwardPacketsSent, + + #[strum(props( + help = "The number of egress forward hop sphinx packets sent/forwarded (acks only)" + ))] + MixnetEgressAckSent, + + #[strum(props(help = "The number of egress forward hop sphinx packets dropped"))] + MixnetEgressForwardPacketsDropped, + + #[strum(props( + help = "The current rate of sending/forwarding egress forward hop sphinx packets" + ))] + MixnetEgressForwardPacketsSendRate, + + #[strum(props( + help = "The current rate of sending/forwarding egress forward hop sphinx packets (acks only)" + ))] + MixnetEgressAckSendRate, + + #[strum(props(help = "The current rate of dropping egress forward hop sphinx packets"))] + MixnetEgressForwardPacketsDroppedRate, + + // # ENTRY + #[strum(props(help = "The number of unique users"))] + EntryClientUniqueUsers, + + #[strum(props(help = "The number of client sessions started"))] + EntryClientSessionsStarted, + + #[strum(props(help = "The number of client sessions finished"))] + EntryClientSessionsFinished, + + #[strum(to_string = "entry_client_sessions_durations_{typ}")] + #[strum(props(help = "The distribution of client sessions duration of the specified type"))] + EntryClientSessionsDurations { typ: String }, + + // # WIREGUARD + #[strum(props(help = "The amount of bytes transmitted via wireguard"))] + WireguardBytesTx, + + #[strum(props(help = "The amount of bytes received via wireguard"))] + WireguardBytesRx, + + #[strum(props(help = "The current number of all registered wireguard peers"))] + WireguardTotalPeers, + + #[strum(props(help = "The current number of active wireguard peers"))] + WireguardActivePeers, + + #[strum(props(help = "The current sending rate of wireguard"))] + WireguardBytesTxRate, + + #[strum(props(help = "The current receiving rate of wireguard"))] + WireguardBytesRxRate, + + // # NETWORK + #[strum(props(help = "The number of active ingress mixnet connections"))] + NetworkActiveIngressMixnetConnections, + + #[strum(props(help = "The number of active ingress websocket connections"))] + NetworkActiveIngressWebSocketConnections, + + #[strum(props(help = "The number of active egress mixnet connections"))] + NetworkActiveEgressMixnetConnections, + + // # PROCESS + #[strum(props(help = "The current number of packets being delayed"))] + ProcessForwardHopPacketsBeingDelayed, + + #[strum(props( + help = "The current number of packets waiting in the queue to get delayed and sent into the mixnet" + ))] + ProcessPacketForwarderQueueSize, + + #[strum(props( + help = "The latency distribution of attempting to retrieve network topology (from nym-api)" + ))] + ProcessTopologyQueryResolutionLatency, + + #[strum(props( + help = "The current number of final hop packets stuck in channels waiting to get delivered to appropriate websocket connections" + ))] + ProcessFinalHopPacketsPendingDelivery, + + #[strum(props( + help = "The current number of forward hop packets stuck in channels waiting to get delivered to appropriate TCP connections" + ))] + ProcessForwardHopPacketsPendingDelivery, +} + +impl PrometheusMetric { + fn name(&self) -> String { + self.to_string() + } + + fn help(&self) -> &'static str { + // SAFETY: every variant has a `help` prop defined (and there's a unit test is checking for that) + #[allow(clippy::unwrap_used)] + self.get_str("help").unwrap() + } + + fn is_complex(&self) -> bool { + matches!(self, PrometheusMetric::EntryClientSessionsDurations { .. }) + // match self { + // PrometheusMetric::EntryClientSessionsDurations { .. } => true, + // _ => false, + // } + } + + fn to_registrable_metric(&self) -> Option { + let name = self.name(); + let help = self.help(); + + match self { + PrometheusMetric::MixnetIngressForwardPacketsReceived => { + Metric::new_int_gauge(&name, help) + } + PrometheusMetric::MixnetIngressFinalHopPacketsReceived => { + Metric::new_int_gauge(&name, help) + } + PrometheusMetric::MixnetIngressMalformedPacketsReceived => { + Metric::new_int_gauge(&name, help) + } + PrometheusMetric::MixnetIngressExcessiveDelayPacketsReceived => { + Metric::new_int_gauge(&name, help) + } + PrometheusMetric::MixnetIngressForwardPacketsDropped => { + Metric::new_int_gauge(&name, help) + } + PrometheusMetric::MixnetIngressFinalHopPacketsDropped => { + Metric::new_int_gauge(&name, help) + } + PrometheusMetric::MixnetIngressForwardPacketsReceivedRate => { + Metric::new_float_gauge(&name, help) + } + PrometheusMetric::MixnetIngressFinalHopPacketsReceivedRate => { + Metric::new_float_gauge(&name, help) + } + PrometheusMetric::MixnetIngressMalformedPacketsReceivedRate => { + Metric::new_float_gauge(&name, help) + } + PrometheusMetric::MixnetIngressExcessiveDelayPacketsReceivedRate => { + Metric::new_float_gauge(&name, help) + } + PrometheusMetric::MixnetIngressForwardPacketsDroppedRate => { + Metric::new_float_gauge(&name, help) + } + PrometheusMetric::MixnetIngressFinalHopPacketsDroppedRate => { + Metric::new_float_gauge(&name, help) + } + PrometheusMetric::MixnetEgressStoredOnDiskFinalHopPackets => { + Metric::new_int_gauge(&name, help) + } + PrometheusMetric::MixnetEgressForwardPacketsSent => Metric::new_int_gauge(&name, help), + PrometheusMetric::MixnetEgressAckSent => Metric::new_int_gauge(&name, help), + PrometheusMetric::MixnetEgressForwardPacketsDropped => { + Metric::new_int_gauge(&name, help) + } + PrometheusMetric::MixnetEgressForwardPacketsSendRate => { + Metric::new_float_gauge(&name, help) + } + PrometheusMetric::MixnetEgressAckSendRate => Metric::new_float_gauge(&name, help), + PrometheusMetric::MixnetEgressForwardPacketsDroppedRate => { + Metric::new_float_gauge(&name, help) + } + PrometheusMetric::EntryClientUniqueUsers => Metric::new_int_gauge(&name, help), + PrometheusMetric::EntryClientSessionsStarted => Metric::new_int_gauge(&name, help), + PrometheusMetric::EntryClientSessionsFinished => Metric::new_int_gauge(&name, help), + PrometheusMetric::EntryClientSessionsDurations { .. } => { + Metric::new_histogram(&name, help, Some(CLIENT_SESSION_DURATION_BUCKETS)) + } + PrometheusMetric::WireguardBytesTx => Metric::new_int_gauge(&name, help), + PrometheusMetric::WireguardBytesRx => Metric::new_int_gauge(&name, help), + PrometheusMetric::WireguardTotalPeers => Metric::new_int_gauge(&name, help), + PrometheusMetric::WireguardActivePeers => Metric::new_int_gauge(&name, help), + PrometheusMetric::WireguardBytesTxRate => Metric::new_float_gauge(&name, help), + PrometheusMetric::WireguardBytesRxRate => Metric::new_float_gauge(&name, help), + PrometheusMetric::NetworkActiveIngressMixnetConnections => { + Metric::new_int_gauge(&name, help) + } + PrometheusMetric::NetworkActiveIngressWebSocketConnections => { + Metric::new_int_gauge(&name, help) + } + PrometheusMetric::NetworkActiveEgressMixnetConnections => { + Metric::new_int_gauge(&name, help) + } + PrometheusMetric::ProcessForwardHopPacketsBeingDelayed => { + Metric::new_int_gauge(&name, help) + } + PrometheusMetric::ProcessPacketForwarderQueueSize => Metric::new_int_gauge(&name, help), + PrometheusMetric::ProcessTopologyQueryResolutionLatency => { + Metric::new_histogram(&name, help, None) + } + PrometheusMetric::ProcessFinalHopPacketsPendingDelivery => { + Metric::new_int_gauge(&name, help) + } + PrometheusMetric::ProcessForwardHopPacketsPendingDelivery => { + Metric::new_int_gauge(&name, help) + } + } + } + + fn set(&self, value: i64) { + metrics_registry().set(&self.name(), value); + } + + fn set_float(&self, value: f64) { + metrics_registry().set_float(&self.name(), value); + } + + fn inc(&self) { + metrics_registry().inc(&self.name()); + } + + fn inc_by(&self, value: i64) { + metrics_registry().inc_by(&self.name(), value); + } + + fn observe_histogram(&self, value: f64) { + let reg = metrics_registry(); + if !reg.add_to_histogram(&self.name(), value) { + if let Some(registrable) = self.to_registrable_metric() { + reg.register_metric(registrable); + reg.add_to_histogram(&self.name(), value); + } + } + } + + fn start_timer(&self) -> Option { + metrics_registry().start_timer(&self.name()) + } +} + +#[non_exhaustive] +pub struct NymNodePrometheusMetrics {} + +impl NymNodePrometheusMetrics { + // initialise all fields on startup with default values so that they'd be immediately available for query + pub(crate) fn initialise() -> Self { + let registry = metrics_registry(); + + // we can't initialise complex metrics as their names will only be fully known at runtime + for kind in PrometheusMetric::iter() { + if !kind.is_complex() { + if let Some(metric) = kind.to_registrable_metric() { + registry.register_metric(metric); + } + } + } + + NymNodePrometheusMetrics {} + } + + pub fn set(&self, metric: PrometheusMetric, value: i64) { + metric.set(value) + } + + pub fn set_float(&self, metric: PrometheusMetric, value: f64) { + metric.set_float(value) + } + + pub fn inc(&self, metric: PrometheusMetric) { + metric.inc() + } + + pub fn inc_by(&self, metric: PrometheusMetric, value: i64) { + metric.inc_by(value) + } + + pub fn observe_histogram(&self, metric: PrometheusMetric, value: f64) { + metric.observe_histogram(value) + } + + pub fn start_timer(&self, metric: PrometheusMetric) -> Option { + metric.start_timer() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use strum::IntoEnumIterator; + + #[test] + fn prometheus_metrics() { + // a sanity check for anyone adding new metrics. if this test fails, + // make sure any methods on `PrometheusMetric` enum don't need updating + // or require custom Display impl + assert_eq!(37, PrometheusMetric::COUNT) + } + + #[test] + fn every_variant_has_help_property() { + for variant in PrometheusMetric::iter() { + assert!(variant.get_str("help").is_some()) + } + } + + #[test] + fn prometheus_metrics_names() { + // make sure nothing changed in our serialisation + let simple = PrometheusMetric::MixnetIngressForwardPacketsReceived.to_string(); + assert_eq!("nym_node_mixnet_ingress_forward_packets_received", simple); + + let parameterised = + PrometheusMetric::EntryClientSessionsDurations { typ: "vpn".into() }.to_string(); + assert_eq!( + "nym_node_entry_client_sessions_durations_vpn", + parameterised + ) + } +} diff --git a/nym-node/src/config/metrics.rs b/nym-node/src/config/metrics.rs index 80f9dcea6d..5bdec5bdbf 100644 --- a/nym-node/src/config/metrics.rs +++ b/nym-node/src/config/metrics.rs @@ -25,6 +25,14 @@ pub struct Debug { #[serde(with = "humantime_serde")] pub stale_mixnet_metrics_cleaner_rate: Duration, + /// Specify the target rate of updating global prometheus counters. + #[serde(with = "humantime_serde")] + pub global_prometheus_counters_update_rate: Duration, + + /// Specify the target rate of updating egress packets pending delivery counter. + #[serde(with = "humantime_serde")] + pub pending_egress_packets_update_rate: Duration, + /// Specify the rate of updating clients sessions #[serde(with = "humantime_serde")] pub clients_sessions_update_rate: Duration, @@ -42,8 +50,10 @@ impl Debug { const DEFAULT_CONSOLE_LOGGING_INTERVAL: Duration = Duration::from_millis(60_000); const DEFAULT_LEGACY_MIXING_UPDATE_RATE: Duration = Duration::from_millis(30_000); const DEFAULT_AGGREGATOR_UPDATE_RATE: Duration = Duration::from_secs(5); - const DEFAULT_STALE_MIXNET_ETRICS_UPDATE_RATE: Duration = Duration::from_secs(3600); + const DEFAULT_STALE_MIXNET_METRICS_UPDATE_RATE: Duration = Duration::from_secs(3600); const DEFAULT_CLIENT_SESSIONS_UPDATE_RATE: Duration = Duration::from_secs(3600); + const GLOBAL_PROMETHEUS_COUNTERS_UPDATE_INTERVAL: Duration = Duration::from_secs(30); + const DEFAULT_PENDING_EGRESS_PACKETS_UPDATE_RATE: Duration = Duration::from_secs(30); } impl Default for Debug { @@ -53,7 +63,10 @@ impl Default for Debug { console_logging_update_interval: Self::DEFAULT_CONSOLE_LOGGING_INTERVAL, legacy_mixing_metrics_update_rate: Self::DEFAULT_LEGACY_MIXING_UPDATE_RATE, aggregator_update_rate: Self::DEFAULT_AGGREGATOR_UPDATE_RATE, - stale_mixnet_metrics_cleaner_rate: Self::DEFAULT_STALE_MIXNET_ETRICS_UPDATE_RATE, + stale_mixnet_metrics_cleaner_rate: Self::DEFAULT_STALE_MIXNET_METRICS_UPDATE_RATE, + global_prometheus_counters_update_rate: + Self::GLOBAL_PROMETHEUS_COUNTERS_UPDATE_INTERVAL, + pending_egress_packets_update_rate: Self::DEFAULT_PENDING_EGRESS_PACKETS_UPDATE_RATE, clients_sessions_update_rate: Self::DEFAULT_CLIENT_SESSIONS_UPDATE_RATE, } } diff --git a/nym-node/src/config/mod.rs b/nym-node/src/config/mod.rs index 6b996f24be..8f0cb9f220 100644 --- a/nym-node/src/config/mod.rs +++ b/nym-node/src/config/mod.rs @@ -444,6 +444,7 @@ pub struct Http { /// An optional bearer token for accessing certain http endpoints. /// Currently only used for obtaining mixnode's stats. #[serde(default)] + #[serde(deserialize_with = "de_maybe_stringified")] pub access_token: Option, /// Specify whether basic system information should be exposed. diff --git a/nym-node/src/node/http/error.rs b/nym-node/src/node/http/error.rs index 8019c5fd58..251705f520 100644 --- a/nym-node/src/node/http/error.rs +++ b/nym-node/src/node/http/error.rs @@ -1,5 +1,5 @@ // Copyright 2024 - Nym Technologies SA -// SPDX-License-Identifier: Apache-2.0 +// SPDX-License-Identifier: GPL-3.0-only use std::io; use std::net::SocketAddr; diff --git a/nym-node/src/node/http/router/api/v1/metrics/mod.rs b/nym-node/src/node/http/router/api/v1/metrics/mod.rs index 71a9760273..51af714434 100644 --- a/nym-node/src/node/http/router/api/v1/metrics/mod.rs +++ b/nym-node/src/node/http/router/api/v1/metrics/mod.rs @@ -10,7 +10,12 @@ use crate::node::http::state::metrics::MetricsAppState; use axum::extract::FromRef; use axum::routing::get; use axum::Router; +use nym_http_api_common::middleware::bearer_auth::AuthLayer; use nym_node_requests::routes::api::v1::metrics; +use nym_node_requests::routes::api::v1::metrics::prometheus_absolute; +use std::sync::Arc; +use tracing::info; +use zeroize::Zeroizing; pub mod legacy_mixing; pub mod packets_stats; @@ -21,16 +26,23 @@ pub mod wireguard; #[derive(Debug, Clone, Default)] pub struct Config { - // + pub bearer_token: Option>>, } #[allow(deprecated)] -pub(super) fn routes(_config: Config) -> Router +pub(super) fn routes(config: Config) -> Router where S: Send + Sync + 'static + Clone, MetricsAppState: FromRef, { - Router::new() + if config.bearer_token.is_none() { + info!( + "bearer token hasn't been set. '{}' route will not be exposed", + prometheus_absolute() + ) + } + + let router = Router::new() .route( metrics::LEGACY_MIXING, get(legacy_mixing::legacy_mixing_stats), @@ -38,6 +50,17 @@ where .route(metrics::PACKETS_STATS, get(packets_stats)) .route(metrics::WIREGUARD_STATS, get(wireguard_stats)) .route(metrics::SESSIONS, get(sessions_stats)) - .route(metrics::VERLOC, get(verloc_stats)) - .route(metrics::PROMETHEUS, get(prometheus_metrics)) + .route(metrics::VERLOC, get(verloc_stats)); + + let auth_middleware = config.bearer_token.map(AuthLayer::new); + + // don't expose prometheus route without bearer token set + if let Some(auth_middleware) = auth_middleware { + router.route( + metrics::PROMETHEUS, + get(prometheus_metrics).route_layer(auth_middleware), + ) + } else { + router + } } diff --git a/nym-node/src/node/http/router/api/v1/metrics/prometheus.rs b/nym-node/src/node/http/router/api/v1/metrics/prometheus.rs index 197cc5a1c8..2bd8a3356b 100644 --- a/nym-node/src/node/http/router/api/v1/metrics/prometheus.rs +++ b/nym-node/src/node/http/router/api/v1/metrics/prometheus.rs @@ -1,12 +1,6 @@ // Copyright 2024 - Nym Technologies SA // SPDX-License-Identifier: GPL-3.0-only -use crate::node::http::state::metrics::MetricsAppState; -use axum::extract::State; -use axum::http::StatusCode; -use axum_extra::TypedHeader; -use headers::authorization::Bearer; -use headers::Authorization; use nym_metrics::metrics; /// Returns `prometheus` compatible metrics @@ -25,21 +19,7 @@ use nym_metrics::metrics; ("prometheus_token" = []) ) )] -pub(crate) async fn prometheus_metrics<'a>( - TypedHeader(authorization): TypedHeader>, - State(state): State, -) -> Result { - if authorization.token().is_empty() { - return Err(StatusCode::UNAUTHORIZED); - } - // TODO: is 500 the correct error code here? - let Some(metrics_key) = state.prometheus_access_token else { - return Err(StatusCode::INTERNAL_SERVER_ERROR); - }; - - if metrics_key != authorization.token() { - return Err(StatusCode::UNAUTHORIZED); - } - - Ok(metrics!()) +// the AuthLayer is protecting access to this endpoint +pub(crate) async fn prometheus_metrics() -> String { + metrics!() } diff --git a/nym-node/src/node/http/router/mod.rs b/nym-node/src/node/http/router/mod.rs index 449638c8d6..3d40e9a67e 100644 --- a/nym-node/src/node/http/router/mod.rs +++ b/nym-node/src/node/http/router/mod.rs @@ -20,6 +20,8 @@ use nym_node_requests::api::SignedHostInformation; use nym_node_requests::routes; use std::net::SocketAddr; use std::path::Path; +use std::sync::Arc; +use zeroize::Zeroizing; pub mod api; pub mod landing_page; @@ -115,6 +117,11 @@ impl HttpServerConfig { self.api.v1_config.authenticator.details = Some(authenticator); self } + + pub fn with_prometheus_bearer_token(mut self, bearer_token: Option) -> Self { + self.api.v1_config.metrics.bearer_token = bearer_token.map(|b| Arc::new(Zeroizing::new(b))); + self + } } pub struct NymNodeRouter { diff --git a/nym-node/src/node/http/state/metrics.rs b/nym-node/src/node/http/state/metrics.rs index c9156c021e..8b10fb0a30 100644 --- a/nym-node/src/node/http/state/metrics.rs +++ b/nym-node/src/node/http/state/metrics.rs @@ -9,8 +9,6 @@ pub use nym_verloc::measurements::metrics::SharedVerlocStats; #[derive(Clone)] pub struct MetricsAppState { - pub(crate) prometheus_access_token: Option, - pub(crate) metrics: NymNodeMetrics, pub(crate) verloc: SharedVerlocStats, diff --git a/nym-node/src/node/http/state/mod.rs b/nym-node/src/node/http/state/mod.rs index b9b58c0cce..0e2fa98786 100644 --- a/nym-node/src/node/http/state/mod.rs +++ b/nym-node/src/node/http/state/mod.rs @@ -24,17 +24,7 @@ impl AppState { // does it have to be? // also no. startup_time: Instant::now(), - metrics: MetricsAppState { - prometheus_access_token: None, - metrics, - verloc, - }, + metrics: MetricsAppState { metrics, verloc }, } } - - #[must_use] - pub fn with_metrics_key(mut self, bearer_token: impl Into>) -> Self { - self.metrics.prometheus_access_token = bearer_token.into(); - self - } } diff --git a/nym-node/src/node/metrics/aggregator.rs b/nym-node/src/node/metrics/aggregator.rs index c166eb0d3c..8f76b14edb 100644 --- a/nym-node/src/node/metrics/aggregator.rs +++ b/nym-node/src/node/metrics/aggregator.rs @@ -44,7 +44,7 @@ impl MetricsAggregator { self.event_sender.clone() } - pub fn register_handler(&mut self, handler: H, update_interval: Duration) + pub fn register_handler(&mut self, handler: H, update_interval: impl Into>) where H: MetricsHandler, { diff --git a/nym-node/src/node/metrics/events_listener.rs b/nym-node/src/node/metrics/events_listener.rs index 755fb6cc8b..939f19b3a9 100644 --- a/nym-node/src/node/metrics/events_listener.rs +++ b/nym-node/src/node/metrics/events_listener.rs @@ -1,2 +1,2 @@ // Copyright 2024 - Nym Technologies SA -// SPDX-License-Identifier: Apache-2.0 +// SPDX-License-Identifier: GPL-3.0-only diff --git a/nym-node/src/node/metrics/handler/client_sessions.rs b/nym-node/src/node/metrics/handler/client_sessions.rs index 8394eb1d89..2f2326c541 100644 --- a/nym-node/src/node/metrics/handler/client_sessions.rs +++ b/nym-node/src/node/metrics/handler/client_sessions.rs @@ -1,5 +1,5 @@ // Copyright 2024 - Nym Technologies SA -// SPDX-License-Identifier: Apache-2.0 +// SPDX-License-Identifier: GPL-3.0-only use crate::node::metrics::handler::{ MetricsHandler, OnStartMetricsHandler, OnUpdateMetricsHandler, @@ -10,6 +10,8 @@ use nym_gateway_stats_storage::error::StatsStorageError; use nym_gateway_stats_storage::models::{TicketType, ToSessionType}; use nym_node_metrics::entry::{ActiveSession, ClientSessions, FinishedSession}; use nym_node_metrics::events::GatewaySessionEvent; +use nym_node_metrics::prometheus_wrapper::PrometheusMetric::EntryClientSessionsDurations; +use nym_node_metrics::prometheus_wrapper::PROMETHEUS_METRICS; use nym_node_metrics::NymNodeMetrics; use nym_sphinx_types::DestinationAddressBytes; use time::{Date, Duration, OffsetDateTime}; @@ -53,6 +55,13 @@ impl GatewaySessionStatsHandler { ) -> Result<(), StatsStorageError> { if let Some(session) = self.storage.get_active_session(client).await? { if let Some(finished_session) = session.end_at(stop_time) { + PROMETHEUS_METRICS.observe_histogram( + EntryClientSessionsDurations { + typ: finished_session.typ.to_string(), + }, + finished_session.duration.as_secs_f64(), + ); + self.storage .insert_finished_session(self.current_day, finished_session) .await?; diff --git a/nym-node/src/node/metrics/handler/global_prometheus_updater/at_last_update.rs b/nym-node/src/node/metrics/handler/global_prometheus_updater/at_last_update.rs new file mode 100644 index 0000000000..af16be7ecb --- /dev/null +++ b/nym-node/src/node/metrics/handler/global_prometheus_updater/at_last_update.rs @@ -0,0 +1,219 @@ +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +use nym_node_metrics::mixnet::{EgressMixingStats, IngressMixingStats, MixingStats}; +use nym_node_metrics::wireguard::WireguardStats; +use nym_node_metrics::NymNodeMetrics; +use time::OffsetDateTime; + +// used to calculate traffic rates +#[derive(Debug)] +pub(crate) struct AtLastUpdate { + time: OffsetDateTime, + + mixnet: LastMixnet, + wireguard: LastWireguard, +} + +impl AtLastUpdate { + pub(crate) fn is_initial(&self) -> bool { + self.time == OffsetDateTime::UNIX_EPOCH + } + + pub(crate) fn rates(&self, previous: &Self) -> RateSinceUpdate { + let delta_secs = (self.time - previous.time).as_seconds_f64(); + + RateSinceUpdate { + mixnet: self.mixnet.rates(&previous.mixnet, delta_secs), + wireguard: self.wireguard.rates(&previous.wireguard, delta_secs), + } + } +} + +impl Default for AtLastUpdate { + fn default() -> Self { + AtLastUpdate { + time: OffsetDateTime::now_utc(), + mixnet: Default::default(), + wireguard: Default::default(), + } + } +} + +impl From<&NymNodeMetrics> for AtLastUpdate { + fn from(metrics: &NymNodeMetrics) -> Self { + AtLastUpdate { + time: OffsetDateTime::now_utc(), + mixnet: (&metrics.mixnet).into(), + wireguard: (&metrics.wireguard).into(), + } + } +} + +#[derive(Debug, Default)] +struct LastMixnet { + ingres: LastMixnetIngress, + egress: LastMixnetEgress, +} + +impl LastMixnet { + fn rates(&self, previous: &Self, time_delta_secs: f64) -> MixnetRateSinceUpdate { + MixnetRateSinceUpdate { + ingress: self.ingres.rates(&previous.ingres, time_delta_secs), + egress: self.egress.rates(&previous.egress, time_delta_secs), + } + } +} + +impl From<&MixingStats> for LastMixnet { + fn from(value: &MixingStats) -> Self { + LastMixnet { + ingres: (&value.ingress).into(), + egress: (&value.egress).into(), + } + } +} + +#[derive(Debug, Default)] +struct LastMixnetIngress { + forward_hop_packets_received: usize, + final_hop_packets_received: usize, + malformed_packets_received: usize, + excessive_delay_packets: usize, + forward_hop_packets_dropped: usize, + final_hop_packets_dropped: usize, +} + +impl LastMixnetIngress { + fn rates(&self, previous: &Self, time_delta_secs: f64) -> MixnetIngressRateSinceUpdate { + let forward_hop_packets_received_delta = + self.forward_hop_packets_received - previous.forward_hop_packets_received; + let final_hop_packets_received_delta = + self.final_hop_packets_received - previous.final_hop_packets_received; + let malformed_packets_received_delta = + self.malformed_packets_received - previous.malformed_packets_received; + let excessive_delay_packets_delta = + self.excessive_delay_packets - previous.excessive_delay_packets; + let forward_hop_packets_dropped_delta = + self.forward_hop_packets_dropped - previous.forward_hop_packets_dropped; + let final_hop_packets_dropped_delta = + self.final_hop_packets_dropped - previous.final_hop_packets_dropped; + + MixnetIngressRateSinceUpdate { + forward_hop_packets_received_sec: forward_hop_packets_received_delta as f64 + / time_delta_secs, + final_hop_packets_received_sec: final_hop_packets_received_delta as f64 + / time_delta_secs, + malformed_packets_received_sec: malformed_packets_received_delta as f64 + / time_delta_secs, + excessive_delay_packets_sec: excessive_delay_packets_delta as f64 / time_delta_secs, + forward_hop_packets_dropped_sec: forward_hop_packets_dropped_delta as f64 + / time_delta_secs, + final_hop_packets_dropped_sec: final_hop_packets_dropped_delta as f64 / time_delta_secs, + } + } +} + +impl From<&IngressMixingStats> for LastMixnetIngress { + fn from(value: &IngressMixingStats) -> Self { + LastMixnetIngress { + forward_hop_packets_received: value.forward_hop_packets_received(), + final_hop_packets_received: value.final_hop_packets_received(), + malformed_packets_received: value.malformed_packets_received(), + excessive_delay_packets: value.excessive_delay_packets(), + forward_hop_packets_dropped: value.forward_hop_packets_dropped(), + final_hop_packets_dropped: value.final_hop_packets_dropped(), + } + } +} + +#[derive(Debug, Default)] +struct LastMixnetEgress { + forward_hop_packets_sent: usize, + ack_packets_sent: usize, + forward_hop_packets_dropped: usize, +} + +impl LastMixnetEgress { + fn rates(&self, previous: &Self, time_delta_secs: f64) -> MixnetEgressRateSinceUpdate { + let forward_hop_packets_sent_delta = + self.forward_hop_packets_sent - previous.forward_hop_packets_sent; + let ack_packets_sent_delta = self.ack_packets_sent - previous.ack_packets_sent; + let forward_hop_packets_dropped_delta = + self.forward_hop_packets_dropped - previous.forward_hop_packets_dropped; + + MixnetEgressRateSinceUpdate { + forward_hop_packets_sent_sec: forward_hop_packets_sent_delta as f64 / time_delta_secs, + ack_packets_sent_sec: ack_packets_sent_delta as f64 / time_delta_secs, + forward_hop_packets_dropped_sec: forward_hop_packets_dropped_delta as f64 + / time_delta_secs, + } + } +} + +impl From<&EgressMixingStats> for LastMixnetEgress { + fn from(value: &EgressMixingStats) -> Self { + LastMixnetEgress { + forward_hop_packets_sent: value.forward_hop_packets_sent(), + ack_packets_sent: value.ack_packets_sent(), + forward_hop_packets_dropped: value.forward_hop_packets_dropped(), + } + } +} + +#[derive(Debug, Default)] +struct LastWireguard { + bytes_tx: usize, + bytes_rx: usize, +} + +impl LastWireguard { + fn rates(&self, previous: &Self, time_delta_secs: f64) -> WireguardRateSinceUpdate { + let bytes_tx_delta = self.bytes_tx - previous.bytes_tx; + let bytes_rx_delta = self.bytes_rx - previous.bytes_rx; + + WireguardRateSinceUpdate { + bytes_tx_sec: bytes_tx_delta as f64 / time_delta_secs, + bytes_rx_sec: bytes_rx_delta as f64 / time_delta_secs, + } + } +} + +impl From<&WireguardStats> for LastWireguard { + fn from(value: &WireguardStats) -> Self { + LastWireguard { + bytes_tx: value.bytes_tx(), + bytes_rx: value.bytes_rx(), + } + } +} + +pub(crate) struct RateSinceUpdate { + pub(crate) mixnet: MixnetRateSinceUpdate, + pub(crate) wireguard: WireguardRateSinceUpdate, +} + +pub(crate) struct MixnetRateSinceUpdate { + pub(crate) ingress: MixnetIngressRateSinceUpdate, + pub(crate) egress: MixnetEgressRateSinceUpdate, +} + +pub(crate) struct MixnetIngressRateSinceUpdate { + pub(crate) forward_hop_packets_received_sec: f64, + pub(crate) final_hop_packets_received_sec: f64, + pub(crate) malformed_packets_received_sec: f64, + pub(crate) excessive_delay_packets_sec: f64, + pub(crate) forward_hop_packets_dropped_sec: f64, + pub(crate) final_hop_packets_dropped_sec: f64, +} + +pub(crate) struct MixnetEgressRateSinceUpdate { + pub(crate) forward_hop_packets_sent_sec: f64, + pub(crate) ack_packets_sent_sec: f64, + pub(crate) forward_hop_packets_dropped_sec: f64, +} + +pub(crate) struct WireguardRateSinceUpdate { + pub(crate) bytes_tx_sec: f64, + pub(crate) bytes_rx_sec: f64, +} diff --git a/nym-node/src/node/metrics/handler/global_prometheus_updater/mod.rs b/nym-node/src/node/metrics/handler/global_prometheus_updater/mod.rs new file mode 100644 index 0000000000..81c61d9db8 --- /dev/null +++ b/nym-node/src/node/metrics/handler/global_prometheus_updater/mod.rs @@ -0,0 +1,223 @@ +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +use crate::node::metrics::handler::global_prometheus_updater::at_last_update::AtLastUpdate; +use crate::node::metrics::handler::{ + MetricsHandler, OnStartMetricsHandler, OnUpdateMetricsHandler, +}; +use async_trait::async_trait; +use nym_node_metrics::prometheus_wrapper::{ + NymNodePrometheusMetrics, PrometheusMetric, PROMETHEUS_METRICS, +}; +use nym_node_metrics::NymNodeMetrics; + +mod at_last_update; + +// it can be anything, we just need a unique type_id to register our handler +pub struct GlobalPrometheusData; + +pub struct PrometheusGlobalNodeMetricsRegistryUpdater { + metrics: NymNodeMetrics, + prometheus_wrapper: &'static NymNodePrometheusMetrics, + at_last_update: AtLastUpdate, +} + +impl PrometheusGlobalNodeMetricsRegistryUpdater { + pub(crate) fn new(metrics: NymNodeMetrics) -> Self { + Self { + metrics, + prometheus_wrapper: &PROMETHEUS_METRICS, + at_last_update: Default::default(), + } + } +} + +#[async_trait] +impl OnStartMetricsHandler for PrometheusGlobalNodeMetricsRegistryUpdater {} + +#[async_trait] +impl OnUpdateMetricsHandler for PrometheusGlobalNodeMetricsRegistryUpdater { + async fn on_update(&mut self) { + let entry_guard = self.metrics.entry.client_sessions().await; + use PrometheusMetric::*; + + // # MIXNET + // ## INGRESS + self.prometheus_wrapper.set( + MixnetIngressForwardPacketsReceived, + self.metrics.mixnet.ingress.forward_hop_packets_received() as i64, + ); + self.prometheus_wrapper.set( + MixnetIngressFinalHopPacketsReceived, + self.metrics.mixnet.ingress.final_hop_packets_received() as i64, + ); + self.prometheus_wrapper.set( + MixnetIngressMalformedPacketsReceived, + self.metrics.mixnet.ingress.malformed_packets_received() as i64, + ); + self.prometheus_wrapper.set( + MixnetIngressExcessiveDelayPacketsReceived, + self.metrics.mixnet.ingress.excessive_delay_packets() as i64, + ); + self.prometheus_wrapper.set( + MixnetEgressForwardPacketsDropped, + self.metrics.mixnet.ingress.forward_hop_packets_dropped() as i64, + ); + self.prometheus_wrapper.set( + MixnetIngressFinalHopPacketsDropped, + self.metrics.mixnet.ingress.final_hop_packets_dropped() as i64, + ); + + // ## EGRESS + self.prometheus_wrapper.set( + MixnetEgressStoredOnDiskFinalHopPackets, + self.metrics.mixnet.egress.disk_persisted_packets() as i64, + ); + self.prometheus_wrapper.set( + MixnetEgressForwardPacketsSent, + self.metrics.mixnet.egress.forward_hop_packets_sent() as i64, + ); + self.prometheus_wrapper.set( + MixnetEgressAckSent, + self.metrics.mixnet.egress.ack_packets_sent() as i64, + ); + self.prometheus_wrapper.set( + MixnetEgressForwardPacketsDropped, + self.metrics.mixnet.egress.forward_hop_packets_dropped() as i64, + ); + + // # ENTRY + self.prometheus_wrapper.set( + EntryClientUniqueUsers, + entry_guard.unique_users.len() as i64, + ); + self.prometheus_wrapper.set( + EntryClientSessionsStarted, + entry_guard.sessions_started as i64, + ); + self.prometheus_wrapper.set( + EntryClientSessionsFinished, + entry_guard.finished_sessions.len() as i64, + ); + + // # WIREGUARD + self.prometheus_wrapper + .set(WireguardBytesRx, self.metrics.wireguard.bytes_rx() as i64); + self.prometheus_wrapper + .set(WireguardBytesTx, self.metrics.wireguard.bytes_tx() as i64); + self.prometheus_wrapper.set( + WireguardTotalPeers, + self.metrics.wireguard.total_peers() as i64, + ); + self.prometheus_wrapper.set( + WireguardActivePeers, + self.metrics.wireguard.active_peers() as i64, + ); + + // # NETWORK + self.prometheus_wrapper.set( + NetworkActiveIngressMixnetConnections, + self.metrics + .network + .active_ingress_mixnet_connections_count() as i64, + ); + self.prometheus_wrapper.set( + NetworkActiveIngressWebSocketConnections, + self.metrics + .network + .active_ingress_websocket_connections_count() as i64, + ); + self.prometheus_wrapper.set( + NetworkActiveIngressWebSocketConnections, + self.metrics + .network + .active_egress_mixnet_connections_count() as i64, + ); + + // # PROCESS + self.prometheus_wrapper.set( + ProcessForwardHopPacketsBeingDelayed, + self.metrics + .process + .forward_hop_packets_being_delayed_count() as i64, + ); + self.prometheus_wrapper.set( + ProcessPacketForwarderQueueSize, + self.metrics.process.packet_forwarder_queue_size() as i64, + ); + self.prometheus_wrapper.set( + ProcessFinalHopPacketsPendingDelivery, + self.metrics + .process + .final_hop_packets_pending_delivery_count() as i64, + ); + self.prometheus_wrapper.set( + ProcessForwardHopPacketsPendingDelivery, + self.metrics + .process + .forward_hop_packets_pending_delivery_count() as i64, + ); + + let updated = AtLastUpdate::from(&self.metrics); + + // # RATES + if !self.at_last_update.is_initial() { + let diff = updated.rates(&self.at_last_update); + + self.prometheus_wrapper.set_float( + MixnetIngressForwardPacketsReceivedRate, + diff.mixnet.ingress.forward_hop_packets_received_sec, + ); + self.prometheus_wrapper.set_float( + MixnetIngressFinalHopPacketsReceivedRate, + diff.mixnet.ingress.final_hop_packets_received_sec, + ); + self.prometheus_wrapper.set_float( + MixnetIngressMalformedPacketsReceivedRate, + diff.mixnet.ingress.malformed_packets_received_sec, + ); + self.prometheus_wrapper.set_float( + MixnetIngressExcessiveDelayPacketsReceivedRate, + diff.mixnet.ingress.excessive_delay_packets_sec, + ); + self.prometheus_wrapper.set_float( + MixnetIngressForwardPacketsDroppedRate, + diff.mixnet.ingress.forward_hop_packets_dropped_sec, + ); + self.prometheus_wrapper.set_float( + MixnetIngressFinalHopPacketsDroppedRate, + diff.mixnet.ingress.final_hop_packets_dropped_sec, + ); + + // ## EGRESS + self.prometheus_wrapper.set_float( + MixnetEgressForwardPacketsSendRate, + diff.mixnet.egress.forward_hop_packets_sent_sec, + ); + self.prometheus_wrapper.set_float( + MixnetEgressAckSendRate, + diff.mixnet.egress.ack_packets_sent_sec, + ); + self.prometheus_wrapper.set_float( + MixnetEgressForwardPacketsDroppedRate, + diff.mixnet.egress.forward_hop_packets_dropped_sec, + ); + + // # WIREGUARD + self.prometheus_wrapper + .set_float(WireguardBytesRxRate, diff.wireguard.bytes_rx_sec); + self.prometheus_wrapper + .set_float(WireguardBytesTxRate, diff.wireguard.bytes_tx_sec); + } + self.at_last_update = updated; + } +} + +#[async_trait] +impl MetricsHandler for PrometheusGlobalNodeMetricsRegistryUpdater { + type Events = GlobalPrometheusData; + + async fn handle_event(&mut self, _event: Self::Events) { + panic!("this should have never been called! MetricsHandler has been incorrectly called on PrometheusNodeMetricsRegistryUpdater") + } +} diff --git a/nym-node/src/node/metrics/handler/legacy_packet_data.rs b/nym-node/src/node/metrics/handler/legacy_packet_data.rs index 4b649e831a..6240993797 100644 --- a/nym-node/src/node/metrics/handler/legacy_packet_data.rs +++ b/nym-node/src/node/metrics/handler/legacy_packet_data.rs @@ -1,5 +1,5 @@ // Copyright 2024 - Nym Technologies SA -// SPDX-License-Identifier: Apache-2.0 +// SPDX-License-Identifier: GPL-3.0-only use crate::node::metrics::handler::{ MetricsHandler, OnStartMetricsHandler, OnUpdateMetricsHandler, diff --git a/nym-node/src/node/metrics/handler/mod.rs b/nym-node/src/node/metrics/handler/mod.rs index 025b1419b7..764acc0ff9 100644 --- a/nym-node/src/node/metrics/handler/mod.rs +++ b/nym-node/src/node/metrics/handler/mod.rs @@ -1,5 +1,5 @@ // Copyright 2024 - Nym Technologies SA -// SPDX-License-Identifier: Apache-2.0 +// SPDX-License-Identifier: GPL-3.0-only use async_trait::async_trait; use std::any; @@ -9,8 +9,11 @@ use tokio::time::Instant; use tracing::trace; pub(crate) mod client_sessions; +pub(crate) mod global_prometheus_updater; pub(crate) mod legacy_packet_data; pub(crate) mod mixnet_data_cleaner; +pub(crate) mod pending_egress_packets_updater; +pub(crate) mod prometheus_events_handler; pub(crate) trait RegistrableHandler: Downcast + OnStartMetricsHandler + OnUpdateMetricsHandler + Send + Sync + 'static @@ -63,23 +66,23 @@ pub(crate) trait OnStartMetricsHandler { #[async_trait] pub(crate) trait OnUpdateMetricsHandler { - async fn on_update(&mut self); + async fn on_update(&mut self) {} } pub(crate) struct HandlerWrapper { handler: Box>, - update_interval: Duration, + update_interval: Option, last_updated: Instant, } impl HandlerWrapper { - pub fn new(update_interval: Duration, handler: U) -> Self + pub fn new(update_interval: impl Into>, handler: U) -> Self where U: MetricsHandler, { HandlerWrapper { handler: Box::new(handler), - update_interval, + update_interval: update_interval.into(), last_updated: Instant::now(), } } @@ -107,11 +110,15 @@ impl OnStartMetricsHandler for HandlerWrapper { #[async_trait] impl OnUpdateMetricsHandler for HandlerWrapper { async fn on_update(&mut self) { + let Some(update_interval) = self.update_interval else { + return; + }; + let name = any::type_name::(); trace!("on update for handler for events of type {name}"); let elapsed = self.last_updated.elapsed(); - if elapsed < self.update_interval { + if elapsed < update_interval { trace!("too soon for updates"); return; } diff --git a/nym-node/src/node/metrics/handler/pending_egress_packets_updater.rs b/nym-node/src/node/metrics/handler/pending_egress_packets_updater.rs new file mode 100644 index 0000000000..d1fde40f70 --- /dev/null +++ b/nym-node/src/node/metrics/handler/pending_egress_packets_updater.rs @@ -0,0 +1,60 @@ +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +use crate::node::metrics::handler::{ + MetricsHandler, OnStartMetricsHandler, OnUpdateMetricsHandler, +}; +use async_trait::async_trait; +use nym_gateway::node::ActiveClientsStore; +use nym_mixnet_client::client::ActiveConnections; +use nym_node_metrics::NymNodeMetrics; + +// it can be anything, we just need a unique type_id to register our handler +pub struct PendingEgressPackets; + +pub struct PendingEgressPacketsUpdater { + metrics: NymNodeMetrics, + active_websocket_clients: ActiveClientsStore, + active_mixnet_connections: ActiveConnections, +} + +impl PendingEgressPacketsUpdater { + pub(crate) fn new( + metrics: NymNodeMetrics, + active_clients: ActiveClientsStore, + active_mixnet_connections: ActiveConnections, + ) -> Self { + PendingEgressPacketsUpdater { + metrics, + active_websocket_clients: active_clients, + active_mixnet_connections, + } + } +} + +#[async_trait] +impl OnStartMetricsHandler for PendingEgressPacketsUpdater {} + +#[async_trait] +impl OnUpdateMetricsHandler for PendingEgressPacketsUpdater { + async fn on_update(&mut self) { + let pending_final = self.active_websocket_clients.pending_packets(); + self.metrics + .process + .update_final_hop_packets_pending_delivery(pending_final); + + let pending_forward = self.active_mixnet_connections.pending_packets(); + self.metrics + .process + .update_forward_hop_packets_pending_delivery(pending_forward) + } +} + +#[async_trait] +impl MetricsHandler for PendingEgressPacketsUpdater { + type Events = PendingEgressPackets; + + async fn handle_event(&mut self, _event: Self::Events) { + panic!("this should have never been called! MetricsHandler has been incorrectly called on PendingEgressPacketsUpdater") + } +} diff --git a/nym-node/src/node/metrics/handler/prometheus_events_handler.rs b/nym-node/src/node/metrics/handler/prometheus_events_handler.rs new file mode 100644 index 0000000000..28a1696278 --- /dev/null +++ b/nym-node/src/node/metrics/handler/prometheus_events_handler.rs @@ -0,0 +1,6 @@ +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +// pub struct PrometheusEventsHandler { +// // +// } diff --git a/nym-node/src/node/mixnet/handler.rs b/nym-node/src/node/mixnet/handler.rs index 6cfb7d50d1..e645636c96 100644 --- a/nym-node/src/node/mixnet/handler.rs +++ b/nym-node/src/node/mixnet/handler.rs @@ -112,7 +112,14 @@ impl ConnectionHandler { .await { Err(err) => error!("Failed to store client data - {err}"), - Ok(_) => trace!("Stored packet for {client}"), + Ok(_) => { + self.shared + .metrics + .mixnet + .egress + .add_disk_persisted_packet(); + trace!("Stored packet for {client}") + } } } Ok(_) => trace!("Pushed received packet to {client}"), diff --git a/nym-node/src/node/mixnet/packet_forwarding.rs b/nym-node/src/node/mixnet/packet_forwarding.rs index bcd51a52a9..5e70841c86 100644 --- a/nym-node/src/node/mixnet/packet_forwarding.rs +++ b/nym-node/src/node/mixnet/packet_forwarding.rs @@ -1,5 +1,5 @@ // Copyright 2024 - Nym Technologies SA -// SPDX-License-Identifier: Apache-2.0 +// SPDX-License-Identifier: GPL-3.0-only use futures::StreamExt; use nym_mixnet_client::forwarder::{ @@ -80,7 +80,7 @@ impl PacketForwarder { C: SendWithoutResponse, { let delayed_packet = packet.into_inner(); - self.forward_packet(delayed_packet) + self.forward_packet(delayed_packet); } fn handle_new_packet(&mut self, new_packet: PacketToForward) @@ -102,6 +102,18 @@ impl PacketForwarder { } } + fn update_queue_len_metric(&self) { + self.metrics + .process + .update_forward_hop_packets_being_delayed(self.delay_queue.len()); + } + + fn update_channel_size_metric(&self, channel_size: usize) { + self.metrics + .process + .update_packet_forwarder_queue_size(channel_size) + } + pub async fn run(&mut self) where C: SendWithoutResponse, @@ -125,18 +137,21 @@ impl PacketForwarder { // and hence it can't happen that ALL senders are dropped #[allow(clippy::unwrap_used)] self.handle_new_packet(new_packet.unwrap()); + let channel_len = self.packet_sender.len(); if processed % 1000 == 0 { - let queue_len = self.packet_sender.len(); - match queue_len { + match channel_len { n if n > 200 => error!("there are currently {n} mix packets waiting to get forwarded!"), n if n > 50 => warn!("there are currently {n} mix packets waiting to get forwarded"), n => trace!("there are currently {n} mix packets waiting to get forwarded"), } } - + self.update_channel_size_metric(channel_len); processed += 1; } } + + // update the metrics on either new packet being inserted or packet being removed + self.update_queue_len_metric(); } trace!("PacketForwarder: Exiting"); } diff --git a/nym-node/src/node/mod.rs b/nym-node/src/node/mod.rs index b4e8824eaa..09d349dd5a 100644 --- a/nym-node/src/node/mod.rs +++ b/nym-node/src/node/mod.rs @@ -21,8 +21,10 @@ use crate::node::http::{HttpServerConfig, NymNodeHttpServer, NymNodeRouter}; use crate::node::metrics::aggregator::MetricsAggregator; use crate::node::metrics::console_logger::ConsoleLogger; use crate::node::metrics::handler::client_sessions::GatewaySessionStatsHandler; +use crate::node::metrics::handler::global_prometheus_updater::PrometheusGlobalNodeMetricsRegistryUpdater; use crate::node::metrics::handler::legacy_packet_data::LegacyMixingStatsUpdater; use crate::node::metrics::handler::mixnet_data_cleaner::MixnetMetricsCleaner; +use crate::node::metrics::handler::pending_egress_packets_updater::PendingEgressPacketsUpdater; use crate::node::mixnet::packet_forwarding::PacketForwarder; use crate::node::mixnet::shared::ProcessingConfig; use crate::node::mixnet::SharedFinalHopData; @@ -30,6 +32,7 @@ use crate::node::shared_topology::NymNodeTopologyProvider; use nym_bin_common::bin_info; use nym_crypto::asymmetric::{ed25519, x25519}; use nym_gateway::node::{ActiveClientsStore, GatewayTasksBuilder}; +use nym_mixnet_client::client::ActiveConnections; use nym_mixnet_client::forwarder::MixForwardingSender; use nym_network_requester::{ set_active_gateway, setup_fs_gateways_storage, store_gateway_details, CustomGatewayDetails, @@ -743,7 +746,8 @@ impl NymNode { .with_authenticator_details(auth_details) .with_used_exit_policy(exit_policy_details) .with_description(self.description.clone()) - .with_auxiliary_details(auxiliary_details); + .with_auxiliary_details(auxiliary_details) + .with_prometheus_bearer_token(self.config.http.access_token.clone()); if self.config.http.expose_system_info { config = config.with_system_info(get_system_info( @@ -764,8 +768,7 @@ impl NymNode { config.api.v1_config.node.roles.ip_packet_router_enabled = true; } - let app_state = AppState::new(self.metrics.clone(), self.verloc_stats.clone()) - .with_metrics_key(self.config.http.access_token.clone()); + let app_state = AppState::new(self.metrics.clone(), self.verloc_stats.clone()); Ok(NymNodeRouter::new(config, app_state) .build_server(&self.config.http.bind_address) @@ -836,7 +839,12 @@ impl NymNode { tokio::spawn(async move { verloc_measurer.run().await }); } - pub(crate) fn setup_metrics_backend(&self, shutdown: TaskClient) -> MetricEventsSender { + pub(crate) fn setup_metrics_backend( + &self, + active_clients_store: ActiveClientsStore, + active_egress_mixnet_connections: ActiveConnections, + shutdown: TaskClient, + ) -> MetricEventsSender { info!("setting up node metrics..."); // aggregator (to listen for any metrics events) @@ -862,12 +870,35 @@ impl NymNode { self.config.metrics.debug.clients_sessions_update_rate, ); - // handler for periodically cleaning up stale recipient/sender darta + // handler for periodically cleaning up stale recipient/sender data metrics_aggregator.register_handler( MixnetMetricsCleaner::new(self.metrics.clone()), self.config.metrics.debug.stale_mixnet_metrics_cleaner_rate, ); + // handler for updating the value of forward/final hop packets pending delivery + metrics_aggregator.register_handler( + PendingEgressPacketsUpdater::new( + self.metrics.clone(), + active_clients_store, + active_egress_mixnet_connections, + ), + self.config.metrics.debug.pending_egress_packets_update_rate, + ); + + // handler for updating the prometheus registry from the global atomic metrics counters + // such as number of packets received + metrics_aggregator.register_handler( + PrometheusGlobalNodeMetricsRegistryUpdater::new(self.metrics.clone()), + self.config + .metrics + .debug + .global_prometheus_counters_update_rate, + ); + + // handler for handling prometheus metrics events + // metrics_aggregator.register_handler(PrometheusEventsHandler{}, None); + // note: we're still measuring things such as number of mixed packets, // but since they're stored as atomic integers, they are incremented directly at source // rather than going through event pipeline @@ -901,7 +932,7 @@ impl NymNode { &self, active_clients_store: &ActiveClientsStore, shutdown: TaskClient, - ) -> MixForwardingSender { + ) -> (MixForwardingSender, ActiveConnections) { let processing_config = ProcessingConfig::new(&self.config); // we're ALWAYS listening for mixnet packets, either for forward or final hops (or both) @@ -918,7 +949,13 @@ impl NymNode { self.config.mixnet.debug.initial_connection_timeout, self.config.mixnet.debug.maximum_connection_buffer_size, ); - let mixnet_client = nym_mixnet_client::Client::new(mixnet_client_config); + let mixnet_client = nym_mixnet_client::Client::new( + mixnet_client_config, + self.metrics + .network + .active_egress_mixnet_connections_counter(), + ); + let active_connections = mixnet_client.active_connections(); let mut packet_forwarder = PacketForwarder::new( mixnet_client, @@ -943,7 +980,7 @@ impl NymNode { ); mixnet::Listener::new(self.config.mixnet.bind_address, shared).start(); - mix_packet_sender + (mix_packet_sender, active_connections) } pub(crate) async fn run(mut self) -> Result<(), NymNodeError> { @@ -973,14 +1010,19 @@ impl NymNode { self.start_verloc_measurements(task_manager.subscribe_named("verloc-measurements")); - let metrics_sender = self.setup_metrics_backend(task_manager.subscribe_named("metrics")); let active_clients_store = ActiveClientsStore::new(); - let mix_packet_sender = self.start_mixnet_listener( + let (mix_packet_sender, active_egress_mixnet_connections) = self.start_mixnet_listener( &active_clients_store, task_manager.subscribe_named("mixnet-traffic"), ); + let metrics_sender = self.setup_metrics_backend( + active_clients_store.clone(), + active_egress_mixnet_connections, + task_manager.subscribe_named("metrics"), + ); + self.start_gateway_tasks( metrics_sender, active_clients_store, diff --git a/nym-node/src/node/shared_topology.rs b/nym-node/src/node/shared_topology.rs index 594fb5f4fb..756a4895c4 100644 --- a/nym-node/src/node/shared_topology.rs +++ b/nym-node/src/node/shared_topology.rs @@ -3,6 +3,7 @@ use async_trait::async_trait; use nym_gateway::node::{NymApiTopologyProvider, NymApiTopologyProviderConfig, UserAgent}; +use nym_node_metrics::prometheus_wrapper::{PrometheusMetric, PROMETHEUS_METRICS}; use nym_topology::{gateway, NymTopology, TopologyProvider}; use std::sync::Arc; use std::time::Duration; @@ -93,6 +94,10 @@ impl TopologyProvider for NymNodeTopologyProvider { if let Some(cached) = guard.cached_topology() { return Some(cached); } + + // the observation will be included on drop + let _timer = + PROMETHEUS_METRICS.start_timer(PrometheusMetric::ProcessTopologyQueryResolutionLatency); guard.update_cache().await } } diff --git a/service-providers/ip-packet-router/src/error.rs b/service-providers/ip-packet-router/src/error.rs index 28e21e6edd..7c8a4590d5 100644 --- a/service-providers/ip-packet-router/src/error.rs +++ b/service-providers/ip-packet-router/src/error.rs @@ -29,7 +29,7 @@ pub enum IpPacketRouterError { #[error("failed to connect to mixnet: {source}")] FailedToConnectToMixnet { source: nym_sdk::Error }, - #[error("the entity wrapping the network requester has disconnected")] + #[error("the entity wrapping the ip packet router has disconnected")] DisconnectedParent, #[error("received packet has an invalid version: {0}")] diff --git a/service-providers/ip-packet-router/src/request_filter/mod.rs b/service-providers/ip-packet-router/src/request_filter/mod.rs index c11fb6c9d6..599a00a1d2 100644 --- a/service-providers/ip-packet-router/src/request_filter/mod.rs +++ b/service-providers/ip-packet-router/src/request_filter/mod.rs @@ -56,13 +56,10 @@ impl RequestFilter { pub(crate) async fn check_address(&self, address: &SocketAddr) -> bool { match &*self.inner { RequestFilterInner::ExitPolicy { policy_filter } => { - match policy_filter.check(address).await { - Err(err) => { - warn!("failed to validate '{address}' against the exit policy: {err}"); - false - } - Ok(res) => res, - } + policy_filter.check(address).await.unwrap_or_else(|err| { + warn!("failed to validate '{address}' against the exit policy: {err}"); + false + }) } } } From f6a2f62ea9fefc5204852df259edde9193f39e6c Mon Sep 17 00:00:00 2001 From: benedettadavico Date: Wed, 8 Jan 2025 09:28:48 +0100 Subject: [PATCH 06/19] bump versions of binaries --- Cargo.lock | 16 ++++++++-------- clients/native/Cargo.toml | 2 +- clients/socks5/Cargo.toml | 2 +- explorer-api/Cargo.toml | 2 +- nym-api/Cargo.toml | 2 +- nym-node/Cargo.toml | 2 +- service-providers/network-requester/Cargo.toml | 2 +- tools/nym-cli/Cargo.toml | 2 +- tools/nymvisor/Cargo.toml | 2 +- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b0c33bb62b..521303658e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2416,7 +2416,7 @@ dependencies = [ [[package]] name = "explorer-api" -version = "1.1.43" +version = "1.1.44" dependencies = [ "chrono", "clap 4.5.20", @@ -4393,7 +4393,7 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "nym-api" -version = "1.1.47" +version = "1.1.48" dependencies = [ "anyhow", "async-trait", @@ -4642,7 +4642,7 @@ dependencies = [ [[package]] name = "nym-cli" -version = "1.1.45" +version = "1.1.46" dependencies = [ "anyhow", "base64 0.22.1", @@ -4725,7 +4725,7 @@ dependencies = [ [[package]] name = "nym-client" -version = "1.1.45" +version = "1.1.46" dependencies = [ "bs58", "clap 4.5.20", @@ -5812,7 +5812,7 @@ dependencies = [ [[package]] name = "nym-network-requester" -version = "1.1.46" +version = "1.1.47" dependencies = [ "addr", "anyhow", @@ -5863,7 +5863,7 @@ dependencies = [ [[package]] name = "nym-node" -version = "1.2.0" +version = "1.2.1" dependencies = [ "anyhow", "async-trait", @@ -6235,7 +6235,7 @@ dependencies = [ [[package]] name = "nym-socks5-client" -version = "1.1.45" +version = "1.1.46" dependencies = [ "bs58", "clap 4.5.20", @@ -6837,7 +6837,7 @@ dependencies = [ [[package]] name = "nymvisor" -version = "0.1.10" +version = "0.1.11" dependencies = [ "anyhow", "bytes", diff --git a/clients/native/Cargo.toml b/clients/native/Cargo.toml index 03eb55186e..5f95bbb088 100644 --- a/clients/native/Cargo.toml +++ b/clients/native/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nym-client" -version = "1.1.45" +version = "1.1.46" authors = ["Dave Hrycyszyn ", "Jędrzej Stuczyński "] description = "Implementation of the Nym Client" edition = "2021" diff --git a/clients/socks5/Cargo.toml b/clients/socks5/Cargo.toml index 4363c56387..d538cd5b75 100644 --- a/clients/socks5/Cargo.toml +++ b/clients/socks5/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nym-socks5-client" -version = "1.1.45" +version = "1.1.46" authors = ["Dave Hrycyszyn "] description = "A SOCKS5 localhost proxy that converts incoming messages to Sphinx and sends them to a Nym address" edition = "2021" diff --git a/explorer-api/Cargo.toml b/explorer-api/Cargo.toml index 6fe9830dae..fd2880c4fd 100644 --- a/explorer-api/Cargo.toml +++ b/explorer-api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "explorer-api" -version = "1.1.43" +version = "1.1.44" edition = "2021" license.workspace = true diff --git a/nym-api/Cargo.toml b/nym-api/Cargo.toml index 27814af1db..e28fea6309 100644 --- a/nym-api/Cargo.toml +++ b/nym-api/Cargo.toml @@ -4,7 +4,7 @@ [package] name = "nym-api" license = "GPL-3.0" -version = "1.1.47" +version = "1.1.48" authors.workspace = true edition = "2021" rust-version.workspace = true diff --git a/nym-node/Cargo.toml b/nym-node/Cargo.toml index 75cfde1b15..8d78e6a3dc 100644 --- a/nym-node/Cargo.toml +++ b/nym-node/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "nym-node" -version = "1.2.0" +version = "1.2.1" authors.workspace = true repository.workspace = true homepage.workspace = true diff --git a/service-providers/network-requester/Cargo.toml b/service-providers/network-requester/Cargo.toml index 1508c7d775..2f14ad8021 100644 --- a/service-providers/network-requester/Cargo.toml +++ b/service-providers/network-requester/Cargo.toml @@ -4,7 +4,7 @@ [package] name = "nym-network-requester" license = "GPL-3.0" -version = "1.1.46" +version = "1.1.47" authors.workspace = true edition.workspace = true rust-version = "1.70" diff --git a/tools/nym-cli/Cargo.toml b/tools/nym-cli/Cargo.toml index c156e28679..8b4583dcbc 100644 --- a/tools/nym-cli/Cargo.toml +++ b/tools/nym-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nym-cli" -version = "1.1.45" +version = "1.1.46" authors.workspace = true edition = "2021" license.workspace = true diff --git a/tools/nymvisor/Cargo.toml b/tools/nymvisor/Cargo.toml index d44e9fc12b..d1cde0c60d 100644 --- a/tools/nymvisor/Cargo.toml +++ b/tools/nymvisor/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nymvisor" -version = "0.1.10" +version = "0.1.11" authors.workspace = true repository.workspace = true homepage.workspace = true From 78f45012dbabec35b69eb686c94216e5011ef292 Mon Sep 17 00:00:00 2001 From: Tommy Verrall Date: Wed, 8 Jan 2025 09:44:14 +0100 Subject: [PATCH 07/19] amend 250gb limit --- common/authenticator-requests/src/v4/registration.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/authenticator-requests/src/v4/registration.rs b/common/authenticator-requests/src/v4/registration.rs index 2595922f00..d6e2ee9682 100644 --- a/common/authenticator-requests/src/v4/registration.rs +++ b/common/authenticator-requests/src/v4/registration.rs @@ -28,7 +28,7 @@ pub type HmacSha256 = Hmac; pub type Nonce = u64; pub type Taken = Option; -pub const BANDWIDTH_CAP_PER_DAY: u64 = 1024 * 1024 * 1024; // 1 GB +pub const BANDWIDTH_CAP_PER_DAY: u64 = 250 * 1024 * 1024 * 1024; // 250 GB #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct IpPair { From b47a742dd09f8e2b887dcfc736f5c1585afd561a Mon Sep 17 00:00:00 2001 From: benedettadavico Date: Wed, 8 Jan 2025 10:37:48 +0100 Subject: [PATCH 08/19] update nym-node binary version --- Cargo.lock | 2 +- nym-node/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 521303658e..c2e25e992f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5863,7 +5863,7 @@ dependencies = [ [[package]] name = "nym-node" -version = "1.2.1" +version = "1.2.2" dependencies = [ "anyhow", "async-trait", diff --git a/nym-node/Cargo.toml b/nym-node/Cargo.toml index 8d78e6a3dc..8b04ce3387 100644 --- a/nym-node/Cargo.toml +++ b/nym-node/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "nym-node" -version = "1.2.1" +version = "1.2.2" authors.workspace = true repository.workspace = true homepage.workspace = true From 836a93cd967ea198085acf07af17c9a34beadbcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Stuczy=C5=84ski?= Date: Wed, 8 Jan 2025 11:26:40 +0100 Subject: [PATCH 09/19] fixed client session histogram buckets (#5316) --- nym-node/nym-node-metrics/src/prometheus_wrapper.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nym-node/nym-node-metrics/src/prometheus_wrapper.rs b/nym-node/nym-node-metrics/src/prometheus_wrapper.rs index 2d61ee55c4..cfee39ee3e 100644 --- a/nym-node/nym-node-metrics/src/prometheus_wrapper.rs +++ b/nym-node/nym-node-metrics/src/prometheus_wrapper.rs @@ -18,8 +18,8 @@ const CLIENT_SESSION_DURATION_BUCKETS: &[f64] = &[ 900., // 15min - 1h 3600., // 1h - 12h 43200., // 12h - 23.5h - 88200., // 23.5h - 24.5h - 86400., // 24.5h - 72h + 84600., // 23.5h - 24.5h + 88200., // 24.5h - 72h 259200., // 72h+ (implicitly) ]; From 7c1c13e139dc055b593885fdfaa3f1a2d1209a97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Stuczy=C5=84ski?= Date: Thu, 9 Jan 2025 10:02:37 +0100 Subject: [PATCH 10/19] reduce log severity for number of packets being delayed (#5321) --- nym-node/src/node/mixnet/packet_forwarding.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nym-node/src/node/mixnet/packet_forwarding.rs b/nym-node/src/node/mixnet/packet_forwarding.rs index 5e70841c86..1c0c38d3a7 100644 --- a/nym-node/src/node/mixnet/packet_forwarding.rs +++ b/nym-node/src/node/mixnet/packet_forwarding.rs @@ -140,8 +140,8 @@ impl PacketForwarder { let channel_len = self.packet_sender.len(); if processed % 1000 == 0 { match channel_len { - n if n > 200 => error!("there are currently {n} mix packets waiting to get forwarded!"), - n if n > 50 => warn!("there are currently {n} mix packets waiting to get forwarded"), + n if n > 1000 => error!("there are currently {n} mix packets waiting to get forwarded - the node seems to be significantly overloaded!"), + n if n > 500 => warn!("there are currently {n} mix packets waiting to get forwarded - is the node overloaded?"), n => trace!("there are currently {n} mix packets waiting to get forwarded"), } } From a46245ffe3f788f752aa0bd8c5aefb63bfe57ca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Stuczy=C5=84ski?= Date: Thu, 9 Jan 2025 10:02:52 +0100 Subject: [PATCH 11/19] feat: warn users if node is run in exit mode only (#5320) * added 'full-gateway' nymnode mode to enable both entry and exit at the same time * warning for running node in exit mode only --- nym-node/src/cli/commands/run/mod.rs | 4 ++++ nym-node/src/config/mod.rs | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/nym-node/src/cli/commands/run/mod.rs b/nym-node/src/cli/commands/run/mod.rs index bcaa1e5106..9dfd1061f0 100644 --- a/nym-node/src/cli/commands/run/mod.rs +++ b/nym-node/src/cli/commands/run/mod.rs @@ -83,6 +83,10 @@ pub(crate) async fn execute(mut args: Args) -> Result<(), NymNodeError> { warn!("this node is going to run without mixnode or gateway support! consider providing `mode` value"); } + if config.modes.standalone_exit() { + warn!("this node is going to run in EXIT gateway mode only - it will not be able to accept client traffic and thus will NOT be eligible for any rewards. consider running it alongside `entry` (or `full-gateway`) mode") + } + if config.host.public_ips.is_empty() { return Err(NymNodeError::NoPublicIps); } diff --git a/nym-node/src/config/mod.rs b/nym-node/src/config/mod.rs index 8f0cb9f220..c9d614e6b1 100644 --- a/nym-node/src/config/mod.rs +++ b/nym-node/src/config/mod.rs @@ -77,6 +77,9 @@ pub enum NodeMode { #[clap(alias = "exit")] ExitGateway, + + // entry + exit + FullGateway, } impl Display for NodeMode { @@ -85,6 +88,7 @@ impl Display for NodeMode { NodeMode::Mixnode => "mixnode".fmt(f), NodeMode::EntryGateway => "entry-gateway".fmt(f), NodeMode::ExitGateway => "exit-gateway".fmt(f), + NodeMode::FullGateway => "full-gateway".fmt(f), } } } @@ -117,11 +121,16 @@ impl NodeModes { self.mixnode || self.entry || self.exit } + pub fn standalone_exit(&self) -> bool { + !self.mixnode && !self.entry && self.exit + } + pub fn with_mode(&mut self, mode: NodeMode) -> &mut Self { match mode { NodeMode::Mixnode => self.with_mixnode(), NodeMode::EntryGateway => self.with_entry(), NodeMode::ExitGateway => self.with_exit(), + NodeMode::FullGateway => self.with_entry().with_exit(), } } From 24480418f0184009de4f1c32684f6371f013faa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Stuczy=C5=84ski?= Date: Thu, 9 Jan 2025 11:00:37 +0100 Subject: [PATCH 12/19] Bugfix/contract version assignment (#5318) * fixed contract version being overwritten * introduced migration to fix existing [mainnet] state * updated contract schema * updated testnet manager migrate msg code --- .../mixnet-contract/src/msg.rs | 7 - .../src/support/setup.rs | 3 - .../mixnet/schema/nym-mixnet-contract.json | 88 +--- contracts/mixnet/schema/raw/migrate.json | 88 +--- contracts/mixnet/src/contract.rs | 4 +- .../src/mixnet_contract_settings/storage.rs | 233 ++++++++- .../mixnet_contract_settings/transactions.rs | 442 ++++++++++++++++++ contracts/mixnet/src/queued_migrations.rs | 203 +++++--- contracts/mixnet/src/support/tests/mod.rs | 18 +- .../src/manager/network_init.rs | 3 - 10 files changed, 824 insertions(+), 265 deletions(-) diff --git a/common/cosmwasm-smart-contracts/mixnet-contract/src/msg.rs b/common/cosmwasm-smart-contracts/mixnet-contract/src/msg.rs index 214e7e95df..1f32e71441 100644 --- a/common/cosmwasm-smart-contracts/mixnet-contract/src/msg.rs +++ b/common/cosmwasm-smart-contracts/mixnet-contract/src/msg.rs @@ -863,11 +863,4 @@ pub enum QueryMsg { pub struct MigrateMsg { pub unsafe_skip_state_updates: Option, pub vesting_contract_address: Option, - pub current_nym_node_semver: String, - - #[serde(default)] - pub version_score_weights: OutdatedVersionWeights, - - #[serde(default)] - pub version_score_params: VersionScoreFormulaParams, } diff --git a/contracts/mixnet-vesting-integration-tests/src/support/setup.rs b/contracts/mixnet-vesting-integration-tests/src/support/setup.rs index 7193097aa4..f1d0328f69 100644 --- a/contracts/mixnet-vesting-integration-tests/src/support/setup.rs +++ b/contracts/mixnet-vesting-integration-tests/src/support/setup.rs @@ -464,10 +464,7 @@ pub fn instantiate_contracts( mixnet_contract_address.clone(), &nym_mixnet_contract_common::MigrateMsg { vesting_contract_address: Some(vesting_contract_address.to_string()), - current_nym_node_semver: "1.1.10".to_string(), - version_score_weights: Default::default(), unsafe_skip_state_updates: Some(true), - version_score_params: Default::default(), }, mixnet_code_id, ) diff --git a/contracts/mixnet/schema/nym-mixnet-contract.json b/contracts/mixnet/schema/nym-mixnet-contract.json index 3549011058..2cf7b7675f 100644 --- a/contracts/mixnet/schema/nym-mixnet-contract.json +++ b/contracts/mixnet/schema/nym-mixnet-contract.json @@ -3470,43 +3470,13 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "MigrateMsg", "type": "object", - "required": [ - "current_nym_node_semver" - ], "properties": { - "current_nym_node_semver": { - "type": "string" - }, "unsafe_skip_state_updates": { "type": [ "boolean", "null" ] }, - "version_score_params": { - "default": { - "penalty": "0.995", - "penalty_scaling": "1.65" - }, - "allOf": [ - { - "$ref": "#/definitions/VersionScoreFormulaParams" - } - ] - }, - "version_score_weights": { - "default": { - "major": 100, - "minor": 10, - "patch": 1, - "prerelease": 1 - }, - "allOf": [ - { - "$ref": "#/definitions/OutdatedVersionWeights" - } - ] - }, "vesting_contract_address": { "type": [ "string", @@ -3514,63 +3484,7 @@ ] } }, - "additionalProperties": false, - "definitions": { - "Decimal": { - "description": "A fixed-point decimal value with 18 fractional digits, i.e. Decimal(1_000_000_000_000_000_000) == 1.0\n\nThe greatest possible value that can be represented is 340282366920938463463.374607431768211455 (which is (2^128 - 1) / 10^18)", - "type": "string" - }, - "OutdatedVersionWeights": { - "description": "Defines weights for calculating numbers of versions behind the current release.", - "type": "object", - "required": [ - "major", - "minor", - "patch", - "prerelease" - ], - "properties": { - "major": { - "type": "integer", - "format": "uint32", - "minimum": 0.0 - }, - "minor": { - "type": "integer", - "format": "uint32", - "minimum": 0.0 - }, - "patch": { - "type": "integer", - "format": "uint32", - "minimum": 0.0 - }, - "prerelease": { - "type": "integer", - "format": "uint32", - "minimum": 0.0 - } - }, - "additionalProperties": false - }, - "VersionScoreFormulaParams": { - "description": "Given the formula of version_score = penalty ^ (versions_behind_factor ^ penalty_scaling) define the relevant parameters", - "type": "object", - "required": [ - "penalty", - "penalty_scaling" - ], - "properties": { - "penalty": { - "$ref": "#/definitions/Decimal" - }, - "penalty_scaling": { - "$ref": "#/definitions/Decimal" - } - }, - "additionalProperties": false - } - } + "additionalProperties": false }, "sudo": null, "responses": { diff --git a/contracts/mixnet/schema/raw/migrate.json b/contracts/mixnet/schema/raw/migrate.json index 3844bce308..57f0d2acdb 100644 --- a/contracts/mixnet/schema/raw/migrate.json +++ b/contracts/mixnet/schema/raw/migrate.json @@ -2,43 +2,13 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "MigrateMsg", "type": "object", - "required": [ - "current_nym_node_semver" - ], "properties": { - "current_nym_node_semver": { - "type": "string" - }, "unsafe_skip_state_updates": { "type": [ "boolean", "null" ] }, - "version_score_params": { - "default": { - "penalty": "0.995", - "penalty_scaling": "1.65" - }, - "allOf": [ - { - "$ref": "#/definitions/VersionScoreFormulaParams" - } - ] - }, - "version_score_weights": { - "default": { - "major": 100, - "minor": 10, - "patch": 1, - "prerelease": 1 - }, - "allOf": [ - { - "$ref": "#/definitions/OutdatedVersionWeights" - } - ] - }, "vesting_contract_address": { "type": [ "string", @@ -46,61 +16,5 @@ ] } }, - "additionalProperties": false, - "definitions": { - "Decimal": { - "description": "A fixed-point decimal value with 18 fractional digits, i.e. Decimal(1_000_000_000_000_000_000) == 1.0\n\nThe greatest possible value that can be represented is 340282366920938463463.374607431768211455 (which is (2^128 - 1) / 10^18)", - "type": "string" - }, - "OutdatedVersionWeights": { - "description": "Defines weights for calculating numbers of versions behind the current release.", - "type": "object", - "required": [ - "major", - "minor", - "patch", - "prerelease" - ], - "properties": { - "major": { - "type": "integer", - "format": "uint32", - "minimum": 0.0 - }, - "minor": { - "type": "integer", - "format": "uint32", - "minimum": 0.0 - }, - "patch": { - "type": "integer", - "format": "uint32", - "minimum": 0.0 - }, - "prerelease": { - "type": "integer", - "format": "uint32", - "minimum": 0.0 - } - }, - "additionalProperties": false - }, - "VersionScoreFormulaParams": { - "description": "Given the formula of version_score = penalty ^ (versions_behind_factor ^ penalty_scaling) define the relevant parameters", - "type": "object", - "required": [ - "penalty", - "penalty_scaling" - ], - "properties": { - "penalty": { - "$ref": "#/definitions/Decimal" - }, - "penalty_scaling": { - "$ref": "#/definitions/Decimal" - } - }, - "additionalProperties": false - } - } + "additionalProperties": false } diff --git a/contracts/mixnet/src/contract.rs b/contracts/mixnet/src/contract.rs index 3526e58636..60dd9e4fe3 100644 --- a/contracts/mixnet/src/contract.rs +++ b/contracts/mixnet/src/contract.rs @@ -603,7 +603,7 @@ pub fn query( #[entry_point] pub fn migrate( mut deps: DepsMut<'_>, - env: Env, + _env: Env, msg: MigrateMsg, ) -> Result { set_build_information!(deps.storage)?; @@ -612,7 +612,7 @@ pub fn migrate( let skip_state_updates = msg.unsafe_skip_state_updates.unwrap_or(false); if !skip_state_updates { - crate::queued_migrations::add_config_score_params(deps.branch(), env, &msg)?; + crate::queued_migrations::restore_node_version_history(deps.branch())?; } // due to circular dependency on contract addresses (i.e. mixnet contract requiring vesting contract address diff --git a/contracts/mixnet/src/mixnet_contract_settings/storage.rs b/contracts/mixnet/src/mixnet_contract_settings/storage.rs index 6201972e51..992d15ab8a 100644 --- a/contracts/mixnet/src/mixnet_contract_settings/storage.rs +++ b/contracts/mixnet/src/mixnet_contract_settings/storage.rs @@ -34,9 +34,13 @@ impl NymNodeVersionHistory<'_> { } fn next_id(&self, storage: &mut dyn Storage) -> Result { - let next = self.id_counter.may_load(storage)?.unwrap_or_default(); - self.id_counter.save(storage, &next)?; - Ok(next) + let id = self + .id_counter + .may_load(storage)? + .map(|current| current + 1) + .unwrap_or_default(); + self.id_counter.save(storage, &id)?; + Ok(id) } pub fn current_version( @@ -56,10 +60,10 @@ impl NymNodeVersionHistory<'_> { pub fn insert_new( &self, storage: &mut dyn Storage, - entry: HistoricalNymNodeVersion, + entry: &HistoricalNymNodeVersion, ) -> Result { let next_id = self.next_id(storage)?; - self.version_history.save(storage, next_id, &entry)?; + self.version_history.save(storage, next_id, entry)?; Ok(next_id) } @@ -79,7 +83,7 @@ impl NymNodeVersionHistory<'_> { // treat this as genesis let genesis = HistoricalNymNodeVersion::genesis(raw_semver.to_string(), env.block.height); - return self.insert_new(storage, genesis); + return self.insert_new(storage, &genesis); }; let current_semver = current.version_information.semver_unchecked(); @@ -99,7 +103,7 @@ impl NymNodeVersionHistory<'_> { introduced_at_height: env.block.height, difference_since_genesis: diff, }; - self.insert_new(storage, entry) + self.insert_new(storage, &entry) } } @@ -170,3 +174,218 @@ pub(crate) fn initialise_storage( ADMIN.set(deps, Some(initial_admin))?; Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(test)] + mod nym_node_version_history { + use super::*; + use crate::support::tests::test_helpers::TestSetup; + use cosmwasm_std::testing::{mock_dependencies, mock_env}; + + #[test] + fn getting_current() -> anyhow::Result<()> { + // empty storage + let deps = mock_dependencies(); + let storage = NymNodeVersionHistory::new(); + assert!(storage.current_version(&deps.storage)?.is_none()); + + let mut test = TestSetup::new(); + + let zeroth = storage.current_version(test.storage())?.unwrap(); + let manual_zeroth = storage.version_history.load(test.storage(), 0)?; + assert_eq!(zeroth.version_information, manual_zeroth); + + // manually update the counter to make sure data is still read correctly + let dummy = HistoricalNymNodeVersion { + semver: "1.2.3".to_string(), + introduced_at_height: 1234, + difference_since_genesis: Default::default(), + }; + storage.id_counter.save(test.storage_mut(), &123)?; + storage + .version_history + .save(test.storage_mut(), 123, &dummy)?; + + let updated = storage.current_version(test.storage())?.unwrap(); + assert_eq!(updated.version_information, dummy); + + Ok(()) + } + + #[test] + fn inserting_new_entry() -> anyhow::Result<()> { + let mut test = TestSetup::new(); + let storage = NymNodeVersionHistory::new(); + + let first = HistoricalNymNodeVersion { + semver: "1.1.1".to_string(), + introduced_at_height: 12, + difference_since_genesis: Default::default(), + }; + let second = HistoricalNymNodeVersion { + semver: "1.1.2".to_string(), + introduced_at_height: 123, + difference_since_genesis: Default::default(), + }; + let third = HistoricalNymNodeVersion { + semver: "1.1.3".to_string(), + introduced_at_height: 1234, + difference_since_genesis: Default::default(), + }; + + assert_eq!(storage.id_counter.load(test.storage())?, 0); + + // id is correctly incremented for each case and no entry is overwritten + storage.insert_new(test.storage_mut(), &first)?; + assert_eq!(storage.id_counter.load(test.storage())?, 1); + + storage.insert_new(test.storage_mut(), &second)?; + assert_eq!(storage.id_counter.load(test.storage())?, 2); + + storage.insert_new(test.storage_mut(), &third)?; + assert_eq!(storage.id_counter.load(test.storage())?, 3); + + assert_eq!(storage.version_history.load(test.storage(), 1)?, first); + assert_eq!(storage.version_history.load(test.storage(), 2)?, second); + assert_eq!(storage.version_history.load(test.storage(), 3)?, third); + + Ok(()) + } + + #[test] + fn inserting_initial_semver() -> anyhow::Result<()> { + // empty storage + let mut deps = mock_dependencies(); + let env = mock_env(); + let storage = NymNodeVersionHistory::new(); + + assert!(storage + .id_counter + .may_load(deps.as_mut().storage)? + .is_none()); + + storage.try_insert_new(deps.as_mut().storage, &env, "1.1.1")?; + assert_eq!(storage.id_counter.load(deps.as_mut().storage)?, 0); + + assert_eq!( + storage + .version_history + .load(deps.as_ref().storage, 0)? + .semver, + "1.1.1" + ); + assert_eq!( + storage + .current_version(deps.as_ref().storage)? + .unwrap() + .version_information + .semver, + "1.1.1" + ); + + Ok(()) + } + + #[test] + fn inserting_second_semver() -> anyhow::Result<()> { + let mut test = TestSetup::new(); + let env = test.env(); + let storage = NymNodeVersionHistory::new(); + + // lower version + assert!(storage + .try_insert_new(test.storage_mut(), &env, "1.1.9") + .is_err()); + assert!(storage + .try_insert_new(test.storage_mut(), &env, "1.0.1") + .is_err()); + + // malformed + assert!(storage + .try_insert_new(test.storage_mut(), &env, "1.0") + .is_err()); + assert!(storage + .try_insert_new(test.storage_mut(), &env, "1.0bad") + .is_err()); + assert!(storage + .try_insert_new(test.storage_mut(), &env, "foomp") + .is_err()); + + // patch + let mut test = TestSetup::new(); + storage.try_insert_new(test.storage_mut(), &env, "1.1.11")?; + let current = storage + .current_version(test.storage_mut())? + .unwrap() + .version_information; + assert_eq!(current.semver, "1.1.11"); + assert_eq!(current.difference_since_genesis.major, 0); + assert_eq!(current.difference_since_genesis.minor, 0); + assert_eq!(current.difference_since_genesis.patch, 1); + assert_eq!(current.difference_since_genesis.prerelease, 0); + + // minor + let mut test = TestSetup::new(); + storage.try_insert_new(test.storage_mut(), &env, "1.2.0")?; + let current = storage + .current_version(test.storage_mut())? + .unwrap() + .version_information; + assert_eq!(current.semver, "1.2.0"); + assert_eq!(current.difference_since_genesis.major, 0); + assert_eq!(current.difference_since_genesis.minor, 1); + assert_eq!(current.difference_since_genesis.patch, 0); + assert_eq!(current.difference_since_genesis.prerelease, 0); + + // minor alt. + let mut test = TestSetup::new(); + storage.try_insert_new(test.storage_mut(), &env, "1.2.3")?; + let current = storage + .current_version(test.storage_mut())? + .unwrap() + .version_information; + assert_eq!(current.semver, "1.2.3"); + assert_eq!(current.difference_since_genesis.major, 0); + assert_eq!(current.difference_since_genesis.minor, 1); + assert_eq!(current.difference_since_genesis.patch, 0); + assert_eq!(current.difference_since_genesis.prerelease, 0); + + Ok(()) + } + + #[test] + fn inserting_subsequent_semver() -> anyhow::Result<()> { + let mut test = TestSetup::new(); + let env = test.env(); + let storage = NymNodeVersionHistory::new(); + + storage.try_insert_new(test.storage_mut(), &env, "1.2.0")?; + storage.try_insert_new(test.storage_mut(), &env, "1.2.1")?; + storage.try_insert_new(test.storage_mut(), &env, "1.2.3")?; + let current = storage + .current_version(test.storage_mut())? + .unwrap() + .version_information; + assert_eq!(current.semver, "1.2.3"); + assert_eq!(current.difference_since_genesis.major, 0); + assert_eq!(current.difference_since_genesis.minor, 1); + assert_eq!(current.difference_since_genesis.patch, 3); + assert_eq!(current.difference_since_genesis.prerelease, 0); + + storage.try_insert_new(test.storage_mut(), &env, "1.3.0")?; + let current = storage + .current_version(test.storage_mut())? + .unwrap() + .version_information; + assert_eq!(current.semver, "1.3.0"); + assert_eq!(current.difference_since_genesis.major, 0); + assert_eq!(current.difference_since_genesis.minor, 2); + assert_eq!(current.difference_since_genesis.patch, 3); + assert_eq!(current.difference_since_genesis.prerelease, 0); + Ok(()) + } + } +} diff --git a/contracts/mixnet/src/mixnet_contract_settings/transactions.rs b/contracts/mixnet/src/mixnet_contract_settings/transactions.rs index d3c3a211f1..f48f7b401a 100644 --- a/contracts/mixnet/src/mixnet_contract_settings/transactions.rs +++ b/contracts/mixnet/src/mixnet_contract_settings/transactions.rs @@ -238,4 +238,446 @@ pub mod tests { // let res = try_update_contract_settings(deps.as_mut(), info, new_params); // assert_eq!(Err(MixnetContractError::ZeroActiveSet), res); } + + #[cfg(test)] + mod updating_current_nym_node_semver { + use super::*; + use crate::mixnet_contract_settings::queries::query_current_nym_node_version; + use crate::support::tests::test_helpers::TestSetup; + + #[test] + fn is_restricted_to_the_admin() -> anyhow::Result<()> { + let mut test = TestSetup::new(); + + let not_admin = mock_info("not-admin", &[]); + let admin = mock_info(test.admin().as_ref(), &[]); + + let env = test.env(); + let res = try_update_current_nym_node_semver( + test.deps_mut(), + env, + not_admin, + "1.2.1".to_string(), + ); + assert!(res.is_err()); + + let env = test.env(); + let res = try_update_current_nym_node_semver( + test.deps_mut(), + env, + admin, + "1.2.1".to_string(), + ); + assert!(res.is_ok()); + Ok(()) + } + + #[test] + fn updates_current_semver_value() -> anyhow::Result<()> { + let mut test = TestSetup::new(); + + let res = query_current_nym_node_version(test.deps())?; + + let initial = res.version.unwrap().version_information.semver; + // sanity check to make sure our contract init hasn't changed + assert_eq!(initial, "1.1.10"); + + let update = "1.2.0".to_string(); + + let env = test.env(); + let sender = test.admin_sender(); + try_update_current_nym_node_semver(test.deps_mut(), env, sender, update.clone())?; + + let updated = query_current_nym_node_version(test.deps())?; + let version = updated.version.unwrap().version_information.semver; + assert_eq!(version, update); + + Ok(()) + } + + #[cfg(test)] + mod semver_chain_updates { + use super::*; + use crate::mixnet_contract_settings::queries::query_nym_node_version_history_paged; + use mixnet_contract_common::{ + HistoricalNymNodeVersion, HistoricalNymNodeVersionEntry, TotalVersionDifference, + }; + + fn test_setup_with_initial_checks() -> anyhow::Result { + let test = TestSetup::new(); + + let res = query_current_nym_node_version(test.deps())?; + let initial = res.version.unwrap().version_information.semver; + + // sanity check to make sure our contract init hasn't changed + assert_eq!(initial, "1.1.10"); + + let history = query_nym_node_version_history_paged(test.deps(), None, None)?; + assert_eq!(history.history.len(), 1); + + Ok(test) + } + + #[test] + fn single_patch() -> anyhow::Result<()> { + let mut test = test_setup_with_initial_checks()?; + let initial = query_current_nym_node_version(test.deps())? + .version + .unwrap(); + + let env = test.env(); + let sender = test.admin_sender(); + try_update_current_nym_node_semver( + test.deps_mut(), + env.clone(), + sender, + "1.1.11".to_string(), + )?; + + let history = + query_nym_node_version_history_paged(test.deps(), None, None)?.history; + assert_eq!(history.len(), 2); + assert_eq!(history[0], initial); + assert_eq!( + history[1], + HistoricalNymNodeVersionEntry { + id: 1, + version_information: HistoricalNymNodeVersion { + semver: "1.1.11".to_string(), + introduced_at_height: env.block.height, + difference_since_genesis: TotalVersionDifference { + major: 0, + minor: 0, + patch: 1, + prerelease: 0, + }, + }, + } + ); + + Ok(()) + } + + #[test] + fn single_minor() -> anyhow::Result<()> { + let mut test = test_setup_with_initial_checks()?; + let initial = query_current_nym_node_version(test.deps())? + .version + .unwrap(); + + let env = test.env(); + let sender = test.admin_sender(); + try_update_current_nym_node_semver( + test.deps_mut(), + env.clone(), + sender, + "1.2.0".to_string(), + )?; + + let history = + query_nym_node_version_history_paged(test.deps(), None, None)?.history; + assert_eq!(history.len(), 2); + assert_eq!(history[0], initial); + assert_eq!( + history[1], + HistoricalNymNodeVersionEntry { + id: 1, + version_information: HistoricalNymNodeVersion { + semver: "1.2.0".to_string(), + introduced_at_height: env.block.height, + difference_since_genesis: TotalVersionDifference { + major: 0, + minor: 1, + patch: 0, + prerelease: 0, + }, + }, + } + ); + + Ok(()) + } + + #[test] + fn multiple_patches() -> anyhow::Result<()> { + let mut test = test_setup_with_initial_checks()?; + let initial = query_current_nym_node_version(test.deps())? + .version + .unwrap(); + + let mut env = test.env(); + let sender = test.admin_sender(); + try_update_current_nym_node_semver( + test.deps_mut(), + env.clone(), + sender.clone(), + "1.1.11".to_string(), + )?; + env.block.height += 1; + try_update_current_nym_node_semver( + test.deps_mut(), + env.clone(), + sender.clone(), + "1.1.12".to_string(), + )?; + env.block.height += 1; + try_update_current_nym_node_semver( + test.deps_mut(), + env.clone(), + sender, + "1.1.13".to_string(), + )?; + + let history = + query_nym_node_version_history_paged(test.deps(), None, None)?.history; + assert_eq!(history.len(), 4); + assert_eq!(history[0], initial); + assert_eq!( + history[1], + HistoricalNymNodeVersionEntry { + id: 1, + version_information: HistoricalNymNodeVersion { + semver: "1.1.11".to_string(), + introduced_at_height: env.block.height - 2, + difference_since_genesis: TotalVersionDifference { + major: 0, + minor: 0, + patch: 1, + prerelease: 0, + }, + }, + } + ); + assert_eq!( + history[2], + HistoricalNymNodeVersionEntry { + id: 2, + version_information: HistoricalNymNodeVersion { + semver: "1.1.12".to_string(), + introduced_at_height: env.block.height - 1, + difference_since_genesis: TotalVersionDifference { + major: 0, + minor: 0, + patch: 2, + prerelease: 0, + }, + }, + } + ); + assert_eq!( + history[3], + HistoricalNymNodeVersionEntry { + id: 3, + version_information: HistoricalNymNodeVersion { + semver: "1.1.13".to_string(), + introduced_at_height: env.block.height, + difference_since_genesis: TotalVersionDifference { + major: 0, + minor: 0, + patch: 3, + prerelease: 0, + }, + }, + } + ); + + Ok(()) + } + + #[test] + fn multiple_minors() -> anyhow::Result<()> { + let mut test = test_setup_with_initial_checks()?; + let initial = query_current_nym_node_version(test.deps())? + .version + .unwrap(); + + let mut env = test.env(); + let sender = test.admin_sender(); + try_update_current_nym_node_semver( + test.deps_mut(), + env.clone(), + sender.clone(), + "1.2.0".to_string(), + )?; + env.block.height += 1; + try_update_current_nym_node_semver( + test.deps_mut(), + env.clone(), + sender.clone(), + "1.3.0".to_string(), + )?; + env.block.height += 1; + try_update_current_nym_node_semver( + test.deps_mut(), + env.clone(), + sender, + "1.4.0".to_string(), + )?; + + let history = + query_nym_node_version_history_paged(test.deps(), None, None)?.history; + assert_eq!(history.len(), 4); + assert_eq!(history[0], initial); + assert_eq!( + history[1], + HistoricalNymNodeVersionEntry { + id: 1, + version_information: HistoricalNymNodeVersion { + semver: "1.2.0".to_string(), + introduced_at_height: env.block.height - 2, + difference_since_genesis: TotalVersionDifference { + major: 0, + minor: 1, + patch: 0, + prerelease: 0, + }, + }, + } + ); + assert_eq!( + history[2], + HistoricalNymNodeVersionEntry { + id: 2, + version_information: HistoricalNymNodeVersion { + semver: "1.3.0".to_string(), + introduced_at_height: env.block.height - 1, + difference_since_genesis: TotalVersionDifference { + major: 0, + minor: 2, + patch: 0, + prerelease: 0, + }, + }, + } + ); + assert_eq!( + history[3], + HistoricalNymNodeVersionEntry { + id: 3, + version_information: HistoricalNymNodeVersion { + semver: "1.4.0".to_string(), + introduced_at_height: env.block.height, + difference_since_genesis: TotalVersionDifference { + major: 0, + minor: 3, + patch: 0, + prerelease: 0, + }, + }, + } + ); + + Ok(()) + } + + #[test] + fn mixed_multiple_updates() -> anyhow::Result<()> { + let mut test = test_setup_with_initial_checks()?; + let initial = query_current_nym_node_version(test.deps())? + .version + .unwrap(); + + let mut env = test.env(); + let sender = test.admin_sender(); + try_update_current_nym_node_semver( + test.deps_mut(), + env.clone(), + sender.clone(), + "1.2.0".to_string(), + )?; + env.block.height += 1; + try_update_current_nym_node_semver( + test.deps_mut(), + env.clone(), + sender.clone(), + "1.2.1".to_string(), + )?; + env.block.height += 1; + try_update_current_nym_node_semver( + test.deps_mut(), + env.clone(), + sender.clone(), + "1.2.3".to_string(), + )?; + env.block.height += 1; + try_update_current_nym_node_semver( + test.deps_mut(), + env.clone(), + sender, + "1.3.0".to_string(), + )?; + + let history = + query_nym_node_version_history_paged(test.deps(), None, None)?.history; + assert_eq!(history.len(), 5); + assert_eq!(history[0], initial); + assert_eq!( + history[1], + HistoricalNymNodeVersionEntry { + id: 1, + version_information: HistoricalNymNodeVersion { + semver: "1.2.0".to_string(), + introduced_at_height: env.block.height - 3, + difference_since_genesis: TotalVersionDifference { + major: 0, + minor: 1, + patch: 0, + prerelease: 0, + }, + }, + } + ); + assert_eq!( + history[2], + HistoricalNymNodeVersionEntry { + id: 2, + version_information: HistoricalNymNodeVersion { + semver: "1.2.1".to_string(), + introduced_at_height: env.block.height - 2, + difference_since_genesis: TotalVersionDifference { + major: 0, + minor: 1, + patch: 1, + prerelease: 0, + }, + }, + } + ); + assert_eq!( + history[3], + HistoricalNymNodeVersionEntry { + id: 3, + version_information: HistoricalNymNodeVersion { + semver: "1.2.3".to_string(), + introduced_at_height: env.block.height - 1, + difference_since_genesis: TotalVersionDifference { + major: 0, + minor: 1, + patch: 3, + prerelease: 0, + }, + }, + } + ); + assert_eq!( + history[4], + HistoricalNymNodeVersionEntry { + id: 4, + version_information: HistoricalNymNodeVersion { + semver: "1.3.0".to_string(), + introduced_at_height: env.block.height, + difference_since_genesis: TotalVersionDifference { + major: 0, + minor: 2, + patch: 3, + prerelease: 0, + }, + }, + } + ); + + Ok(()) + } + } + } } diff --git a/contracts/mixnet/src/queued_migrations.rs b/contracts/mixnet/src/queued_migrations.rs index 476d484d80..ebcce35112 100644 --- a/contracts/mixnet/src/queued_migrations.rs +++ b/contracts/mixnet/src/queued_migrations.rs @@ -1,86 +1,153 @@ // Copyright 2022-2024 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -mod config_score_params { - use crate::constants::CONTRACT_STATE_KEY; - use crate::mixnet_contract_settings::storage as mixnet_params_storage; +mod node_version_history { use crate::mixnet_contract_settings::storage::NymNodeVersionHistory; - use cosmwasm_std::{Addr, Coin, DepsMut, Env}; - use cw_storage_plus::Item; + use cosmwasm_std::DepsMut; use mixnet_contract_common::error::MixnetContractError; - use mixnet_contract_common::{ - ConfigScoreParams, ContractState, ContractStateParams, DelegationsParams, MigrateMsg, - OperatingCostRange, OperatorsParams, ProfitMarginRange, - }; - use serde::{Deserialize, Serialize}; - use std::str::FromStr; + use mixnet_contract_common::{HistoricalNymNodeVersion, TotalVersionDifference}; - pub(crate) fn add_config_score_params( + pub(crate) fn restore_node_version_history( deps: DepsMut<'_>, - env: Env, - msg: &MigrateMsg, ) -> Result<(), MixnetContractError> { - if semver::Version::from_str(&msg.current_nym_node_semver).is_err() { - return Err(MixnetContractError::InvalidNymNodeSemver { - provided: msg.current_nym_node_semver.to_string(), + // sanity check: + let storage = NymNodeVersionHistory::new(); + let Some(current) = storage.current_version(deps.storage)? else { + return Err(MixnetContractError::FailedMigration { + comment: "no node version history set".to_string(), + }); + }; + if current.version_information.semver != "1.2.1" + || current.version_information.introduced_at_height != 15902170 + { + return Err(MixnetContractError::FailedMigration { + comment: format!("unexpected current node version history. got: {current:?}"), }); } - - #[derive(Serialize, Deserialize)] - pub struct OldContractState { - pub owner: Option, - pub rewarding_validator_address: Addr, - pub vesting_contract_address: Addr, - pub rewarding_denom: String, - pub params: OldContractStateParams, - } - - #[derive(Serialize, Deserialize)] - pub struct OldContractStateParams { - pub minimum_delegation: Option, - pub minimum_pledge: Coin, - #[serde(default)] - pub profit_margin: ProfitMarginRange, - #[serde(default)] - pub interval_operating_cost: OperatingCostRange, - } - - const OLD_CONTRACT_STATE: Item<'_, OldContractState> = Item::new(CONTRACT_STATE_KEY); - let old_state = OLD_CONTRACT_STATE.load(deps.storage)?; - - #[allow(deprecated)] - let new_state = ContractState { - owner: old_state.owner, - rewarding_validator_address: old_state.rewarding_validator_address, - vesting_contract_address: old_state.vesting_contract_address, - rewarding_denom: old_state.rewarding_denom, - params: ContractStateParams { - delegations_params: DelegationsParams { - minimum_delegation: old_state.params.minimum_delegation, - }, - operators_params: OperatorsParams { - minimum_pledge: old_state.params.minimum_pledge, - profit_margin: old_state.params.profit_margin, - interval_operating_cost: old_state.params.interval_operating_cost, - }, - config_score_params: ConfigScoreParams { - version_weights: msg.version_score_weights, - version_score_formula_params: msg.version_score_params, - }, - }, + let lost = HistoricalNymNodeVersion { + semver: "1.1.12".to_string(), + introduced_at_height: 15779133, + difference_since_genesis: TotalVersionDifference::default(), }; - mixnet_params_storage::CONTRACT_STATE.save(deps.storage, &new_state)?; + #[allow(clippy::unwrap_used)] + // SAFETY: this information was already stored in the contract, so it must be a valid semver + let difference_since_genesis = lost.cumulative_difference_since_genesis( + ¤t.version_information.semver.parse().unwrap(), + ); + let updated_current = HistoricalNymNodeVersion { + semver: current.version_information.semver, + introduced_at_height: current.version_information.introduced_at_height, + difference_since_genesis, + }; - // initialise the version chain - NymNodeVersionHistory::new().try_insert_new( - deps.storage, - &env, - &msg.current_nym_node_semver, - )?; + // restore overwritten entry for 1.1.12 + storage.version_history.save(deps.storage, 0, &lost)?; + + // re-insert 1.2.1 as the current + storage + .version_history + .save(deps.storage, 1, &updated_current)?; + storage.id_counter.save(deps.storage, &1)?; Ok(()) } } -pub(crate) use config_score_params::add_config_score_params; +pub(crate) use node_version_history::restore_node_version_history; + +#[cfg(test)] +mod tests { + use super::*; + use crate::mixnet_contract_settings::queries::query_nym_node_version_history_paged; + use crate::mixnet_contract_settings::storage::NymNodeVersionHistory; + use cosmwasm_std::testing::{mock_dependencies, mock_env}; + use mixnet_contract_common::{ + HistoricalNymNodeVersion, HistoricalNymNodeVersionEntry, TotalVersionDifference, + }; + + #[test] + fn fixing_history_storage() -> anyhow::Result<()> { + // current state on mainnet: + let mut deps = mock_dependencies(); + let storage = NymNodeVersionHistory::new(); + + storage.id_counter.save(deps.as_mut().storage, &0)?; + storage.version_history.save( + deps.as_mut().storage, + 0, + &HistoricalNymNodeVersion { + semver: "1.2.1".to_string(), + introduced_at_height: 15902170, + difference_since_genesis: Default::default(), + }, + )?; + + // run migration + restore_node_version_history(deps.as_mut())?; + + let current = storage.current_version(deps.as_ref().storage)?.unwrap(); + assert_eq!(current.version_information.semver, "1.2.1"); + assert_eq!(current.version_information.introduced_at_height, 15902170); + assert_eq!( + current.version_information.difference_since_genesis, + TotalVersionDifference { + major: 0, + minor: 1, + patch: 0, + prerelease: 0, + } + ); + + let history = query_nym_node_version_history_paged(deps.as_ref(), None, None)?.history; + assert_eq!(history.len(), 2); + assert_eq!( + history, + vec![ + HistoricalNymNodeVersionEntry { + id: 0, + version_information: HistoricalNymNodeVersion { + semver: "1.1.12".to_string(), + introduced_at_height: 15779133, + difference_since_genesis: Default::default(), + }, + }, + HistoricalNymNodeVersionEntry { + id: 1, + version_information: HistoricalNymNodeVersion { + semver: "1.2.1".to_string(), + introduced_at_height: 15902170, + difference_since_genesis: TotalVersionDifference { + major: 0, + minor: 1, + patch: 0, + prerelease: 0, + }, + }, + } + ] + ); + + let counter = storage.id_counter.load(deps.as_ref().storage)?; + assert_eq!(counter, 1); + + // make sure adding another version doesn't mess anything up + storage.try_insert_new(deps.as_mut().storage, &mock_env(), "1.3.0")?; + + let current = storage.current_version(deps.as_ref().storage)?.unwrap(); + assert_eq!(current.version_information.semver, "1.3.0"); + assert_eq!( + current.version_information.difference_since_genesis, + TotalVersionDifference { + major: 0, + minor: 2, + patch: 0, + prerelease: 0, + } + ); + let counter = storage.id_counter.load(deps.as_ref().storage)?; + assert_eq!(counter, 2); + + Ok(()) + } +} diff --git a/contracts/mixnet/src/support/tests/mod.rs b/contracts/mixnet/src/support/tests/mod.rs index cd40ef7ca4..4c0f2d85a3 100644 --- a/contracts/mixnet/src/support/tests/mod.rs +++ b/contracts/mixnet/src/support/tests/mod.rs @@ -18,7 +18,7 @@ pub mod test_helpers { }; use crate::interval::{pending_events, storage as interval_storage}; use crate::mixnet_contract_settings::storage::{ - self as mixnet_params_storage, minimum_node_pledge, + self as mixnet_params_storage, minimum_node_pledge, ADMIN, }; use crate::mixnet_contract_settings::storage::{rewarding_denom, rewarding_validator_address}; use crate::mixnodes::helpers::get_mixnode_details_by_id; @@ -318,6 +318,10 @@ pub mod test_helpers { compare_decimals(mix_info.delegates, subtotal, Some(epsilon)) } + pub fn admin(&self) -> Addr { + ADMIN.get(self.deps()).unwrap().unwrap() + } + pub fn random_address(&mut self) -> String { format!("n1foomp{}", self.rng.next_u64()) } @@ -330,6 +334,14 @@ pub mod test_helpers { self.deps.as_mut() } + pub fn storage(&self) -> &dyn Storage { + self.deps().storage + } + + pub fn storage_mut(&mut self) -> &mut dyn Storage { + self.deps_mut().storage + } + pub fn env(&self) -> Env { self.env.clone() } @@ -470,6 +482,10 @@ pub mod test_helpers { .unwrap() } + pub fn admin_sender(&self) -> MessageInfo { + mock_info(self.admin().as_ref(), &[]) + } + pub fn owner(&self) -> MessageInfo { self.owner.clone() } diff --git a/tools/internal/testnet-manager/src/manager/network_init.rs b/tools/internal/testnet-manager/src/manager/network_init.rs index 5b7f36eea8..1a0fc0e41d 100644 --- a/tools/internal/testnet-manager/src/manager/network_init.rs +++ b/tools/internal/testnet-manager/src/manager/network_init.rs @@ -109,10 +109,7 @@ impl NetworkManager { ) -> Result { Ok(nym_mixnet_contract_common::MigrateMsg { vesting_contract_address: Some(ctx.network.contracts.vesting.address()?.to_string()), - current_nym_node_semver: "irrelevant".to_string(), - version_score_weights: Default::default(), unsafe_skip_state_updates: Some(true), - version_score_params: Default::default(), }) } From a94c035c0a6363be80b08cafb7fc228d22e613b3 Mon Sep 17 00:00:00 2001 From: benedettadavico Date: Thu, 9 Jan 2025 12:36:05 +0100 Subject: [PATCH 13/19] correct the nym-node bumped version --- Cargo.lock | 2 +- nym-node/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c2e25e992f..863266592a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5863,7 +5863,7 @@ dependencies = [ [[package]] name = "nym-node" -version = "1.2.2" +version = "1.3.0" dependencies = [ "anyhow", "async-trait", diff --git a/nym-node/Cargo.toml b/nym-node/Cargo.toml index 8b04ce3387..07c67d9961 100644 --- a/nym-node/Cargo.toml +++ b/nym-node/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "nym-node" -version = "1.2.2" +version = "1.3.0" authors.workspace = true repository.workspace = true homepage.workspace = true From 529e8d49eef8d67f65d5e5156bc9b6cf044a3255 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Stuczy=C5=84ski?= Date: Fri, 10 Jan 2025 14:00:18 +0100 Subject: [PATCH 14/19] chore: apply 1.84 linter suggestions (#5330) * chore: apply 1.84 linter suggestions * updated wasm dependencies to fix the macro issue * second batch of clippy fixes --- Cargo.lock | 52 ++++++++++--------- Cargo.toml | 18 +++---- common/client-core/src/init/helpers.rs | 2 +- .../src/registration/handshake/client.rs | 2 +- .../src/registration/handshake/gateway.rs | 2 +- .../src/scheme/aggregation.rs | 5 +- .../src/clients/packet_statistics.rs | 4 +- explorer-api/src/gateways/models.rs | 4 +- explorer-api/src/mix_nodes/models.rs | 4 +- explorer-api/src/unstable/models.rs | 4 +- .../src/http/state/mod.rs | 2 +- nym-wallet/Cargo.lock | 4 +- .../testnet-manager/src/manager/dkg_skip.rs | 26 +++++----- .../testnet-manager/src/manager/local_apis.rs | 6 +-- .../src/manager/local_client.rs | 17 +++--- .../src/manager/local_nodes.rs | 26 +++++----- 16 files changed, 83 insertions(+), 95 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 863266592a..b3197fd475 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2483,9 +2483,9 @@ checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" [[package]] name = "fancy_constructor" -version = "1.2.2" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f71f317e4af73b2f8f608fac190c52eac4b1879d2145df1db2fe48881ca69435" +checksum = "07b19d0e43eae2bfbafe4931b5e79c73fb1a849ca15cd41a761a7b8587f9a1a2" dependencies = [ "macroific", "proc-macro2", @@ -2846,15 +2846,15 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "gloo-net" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43aaa242d1239a8822c15c645f02166398da4f8b5c4bae795c1f5b44e9eee173" +checksum = "c06f627b1a58ca3d42b45d6104bf1e1a03799df472df00988b6ba21accc10580" dependencies = [ "futures-channel", "futures-core", "futures-sink", "gloo-utils 0.2.0", - "http 0.2.12", + "http 1.1.0", "js-sys", "pin-project", "serde", @@ -3432,7 +3432,8 @@ checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" [[package]] name = "indexed_db_futures" version = "0.4.2" -source = "git+https://github.com/TiemenSch/rust-indexed-db?branch=update-uuid#9745d015707008b0c410115d787014a6d1af2efb" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0704b71f13f81b5933d791abf2de26b33c40935143985220299a357721166706" dependencies = [ "accessory", "cfg-if", @@ -3670,10 +3671,11 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "js-sys" -version = "0.3.72" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" +checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -10762,9 +10764,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" dependencies = [ "cfg-if", "once_cell", @@ -10773,13 +10775,12 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", "syn 2.0.90", @@ -10788,21 +10789,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.45" +version = "0.4.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" +checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" dependencies = [ "cfg-if", "js-sys", + "once_cell", "wasm-bindgen", "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -10810,9 +10812,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", @@ -10823,9 +10825,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" [[package]] name = "wasm-bindgen-test" @@ -10933,9 +10935,9 @@ dependencies = [ [[package]] name = "wasmtimer" -version = "0.2.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f656cd8858a5164932d8a90f936700860976ec21eb00e0fe2aa8cab13f6b4cf" +checksum = "0048ad49a55b9deb3953841fa1fc5858f0efbcb7a18868c899a360269fac1b23" dependencies = [ "futures", "js-sys", @@ -10947,9 +10949,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.72" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" +checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 52ef268311..fa53897399 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -394,19 +394,17 @@ prost = { version = "0.12", default-features = false } # wasm-related dependencies gloo-utils = "0.2.0" -gloo-net = "0.5.0" +gloo-net = "0.6.0" -# use a separate branch due to feature unification failures -# this is blocked until the upstream removes outdates `wasm_bindgen` feature usage -# indexed_db_futures = "0.4.1" -indexed_db_futures = { git = "https://github.com/TiemenSch/rust-indexed-db", branch = "update-uuid" } -js-sys = "0.3.70" +# TODO: migrate to 0.6+ +indexed_db_futures = "0.4.2" +js-sys = "0.3.76" serde-wasm-bindgen = "0.6.5" tsify = "0.4.5" -wasm-bindgen = "0.2.95" -wasm-bindgen-futures = "0.4.45" -wasmtimer = "0.2.0" -web-sys = "0.3.72" +wasm-bindgen = "0.2.99" +wasm-bindgen-futures = "0.4.49" +wasmtimer = "0.4.1" +web-sys = "0.3.76" # Profile settings for individual crates diff --git a/common/client-core/src/init/helpers.rs b/common/client-core/src/init/helpers.rs index 68b3b8d457..16aa042045 100644 --- a/common/client-core/src/init/helpers.rs +++ b/common/client-core/src/init/helpers.rs @@ -190,7 +190,7 @@ where Ok(GatewayWithLatency::new(gateway, avg)) } -pub async fn choose_gateway_by_latency<'a, R: Rng, G: ConnectableGateway + Clone>( +pub async fn choose_gateway_by_latency( rng: &mut R, gateways: &[G], must_use_tls: bool, diff --git a/common/gateway-requests/src/registration/handshake/client.rs b/common/gateway-requests/src/registration/handshake/client.rs index 5bdb239a66..549cddca39 100644 --- a/common/gateway-requests/src/registration/handshake/client.rs +++ b/common/gateway-requests/src/registration/handshake/client.rs @@ -9,7 +9,7 @@ use futures::{Sink, Stream}; use rand::{CryptoRng, RngCore}; use tungstenite::Message as WsMessage; -impl<'a, S, R> State<'a, S, R> { +impl State<'_, S, R> { async fn client_handshake_inner(&mut self) -> Result<(), HandshakeError> where S: Stream + Sink + Unpin, diff --git a/common/gateway-requests/src/registration/handshake/gateway.rs b/common/gateway-requests/src/registration/handshake/gateway.rs index fc439b53c0..5fec717c46 100644 --- a/common/gateway-requests/src/registration/handshake/gateway.rs +++ b/common/gateway-requests/src/registration/handshake/gateway.rs @@ -10,7 +10,7 @@ use crate::registration::handshake::{error::HandshakeError, WsItem}; use futures::{Sink, Stream}; use tungstenite::Message as WsMessage; -impl<'a, S, R> State<'a, S, R> { +impl State<'_, S, R> { async fn gateway_handshake_inner( &mut self, raw_init_message: Vec, diff --git a/common/nym_offline_compact_ecash/src/scheme/aggregation.rs b/common/nym_offline_compact_ecash/src/scheme/aggregation.rs index 9018cde7ea..148a619a41 100644 --- a/common/nym_offline_compact_ecash/src/scheme/aggregation.rs +++ b/common/nym_offline_compact_ecash/src/scheme/aggregation.rs @@ -115,10 +115,7 @@ pub fn aggregate_signatures( let params = ecash_group_parameters(); // aggregate the signature - let signature = match Aggregatable::aggregate(signatures, indices) { - Ok(res) => res, - Err(err) => return Err(err), - }; + let signature = Aggregatable::aggregate(signatures, indices)?; // Ensure the aggregated signature is not an infinity point if bool::from(signature.is_at_infinity()) { diff --git a/common/statistics/src/clients/packet_statistics.rs b/common/statistics/src/clients/packet_statistics.rs index 5d6d1f9c1b..ba335d3627 100644 --- a/common/statistics/src/clients/packet_statistics.rs +++ b/common/statistics/src/clients/packet_statistics.rs @@ -428,7 +428,7 @@ impl PacketStatisticsControl { while self .history .front() - .map_or(false, |&(t, _)| t < recording_window) + .is_some_and(|&(t, _)| t < recording_window) { self.history.pop_front(); } @@ -462,7 +462,7 @@ impl PacketStatisticsControl { while self .rates .front() - .map_or(false, |&(t, _)| t < recording_window) + .is_some_and(|&(t, _)| t < recording_window) { self.rates.pop_front(); } diff --git a/explorer-api/src/gateways/models.rs b/explorer-api/src/gateways/models.rs index d2298b2a0c..9b8a6dfaef 100644 --- a/explorer-api/src/gateways/models.rs +++ b/explorer-api/src/gateways/models.rs @@ -120,9 +120,7 @@ impl ThreadsafeGatewayCache { .read() .await .get(&identity_key) - .map_or(false, |cache_item| { - cache_item.valid_until > SystemTime::now() - }) + .is_some_and(|cache_item| cache_item.valid_until > SystemTime::now()) } pub(crate) async fn get_locations(&self) -> GatewayLocationCache { diff --git a/explorer-api/src/mix_nodes/models.rs b/explorer-api/src/mix_nodes/models.rs index ce5876c949..a9e85ab97e 100644 --- a/explorer-api/src/mix_nodes/models.rs +++ b/explorer-api/src/mix_nodes/models.rs @@ -106,9 +106,7 @@ impl ThreadsafeMixNodesCache { .read() .await .get(&mix_id) - .map_or(false, |cache_item| { - cache_item.valid_until > SystemTime::now() - }) + .is_some_and(|cache_item| cache_item.valid_until > SystemTime::now()) } pub(crate) async fn get_locations(&self) -> MixnodeLocationCache { diff --git a/explorer-api/src/unstable/models.rs b/explorer-api/src/unstable/models.rs index 39254d9c07..9abb81942d 100644 --- a/explorer-api/src/unstable/models.rs +++ b/explorer-api/src/unstable/models.rs @@ -60,9 +60,7 @@ impl ThreadSafeNymNodesCache { .read() .await .get(&node_id) - .map_or(false, |cache_item| { - cache_item.valid_until > SystemTime::now() - }) + .is_some_and(|cache_item| cache_item.valid_until > SystemTime::now()) } pub(crate) async fn get_bonded_nymnodes( diff --git a/nym-credential-proxy/nym-credential-proxy/src/http/state/mod.rs b/nym-credential-proxy/nym-credential-proxy/src/http/state/mod.rs index dc6309e8a7..392fa71992 100644 --- a/nym-credential-proxy/nym-credential-proxy/src/http/state/mod.rs +++ b/nym-credential-proxy/nym-credential-proxy/src/http/state/mod.rs @@ -710,7 +710,7 @@ pub(crate) struct ChainWritePermit<'a> { inner: RwLockWriteGuard<'a, DirectSigningHttpRpcNyxdClient>, } -impl<'a> ChainWritePermit<'a> { +impl ChainWritePermit<'_> { pub(crate) async fn make_deposits( self, short_sha: &'static str, diff --git a/nym-wallet/Cargo.lock b/nym-wallet/Cargo.lock index 6d1ec90fd4..63fb5fff3f 100644 --- a/nym-wallet/Cargo.lock +++ b/nym-wallet/Cargo.lock @@ -6388,9 +6388,9 @@ checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "wasmtimer" -version = "0.2.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f656cd8858a5164932d8a90f936700860976ec21eb00e0fe2aa8cab13f6b4cf" +checksum = "0048ad49a55b9deb3953841fa1fc5858f0efbcb7a18868c899a360269fac1b23" dependencies = [ "futures", "js-sys", diff --git a/tools/internal/testnet-manager/src/manager/dkg_skip.rs b/tools/internal/testnet-manager/src/manager/dkg_skip.rs index 7720202665..ec74a01af3 100644 --- a/tools/internal/testnet-manager/src/manager/dkg_skip.rs +++ b/tools/internal/testnet-manager/src/manager/dkg_skip.rs @@ -168,9 +168,9 @@ impl NetworkManager { Ok(()) } - async fn validate_existing_contracts<'a>( + async fn validate_existing_contracts( &self, - ctx: &DkgSkipCtx<'a>, + ctx: &DkgSkipCtx<'_>, ) -> Result { ctx.println(format!( "🔬 {}Validating the current DKG and group contracts...", @@ -215,9 +215,9 @@ impl NetworkManager { Ok(current_code) } - async fn persist_dkg_keys<'a, P: AsRef>( + async fn persist_dkg_keys>( &self, - ctx: &mut DkgSkipCtx<'a>, + ctx: &mut DkgSkipCtx<'_>, output_dir: P, ) -> Result<(), NetworkManagerError> { ctx.println(format!( @@ -272,9 +272,9 @@ impl NetworkManager { Ok(()) } - async fn upload_bypass_contract<'a, P: AsRef>( + async fn upload_bypass_contract>( &self, - ctx: &DkgSkipCtx<'a>, + ctx: &DkgSkipCtx<'_>, dkg_bypass_contract: P, ) -> Result { ctx.println(format!( @@ -297,9 +297,9 @@ impl NetworkManager { Ok(res.code_id) } - async fn migrate_to_bypass_contract<'a>( + async fn migrate_to_bypass_contract( &self, - ctx: &DkgSkipCtx<'a>, + ctx: &DkgSkipCtx<'_>, code_id: ContractCodeId, ) -> Result<(), NetworkManagerError> { ctx.println(format!( @@ -336,9 +336,9 @@ impl NetworkManager { Ok(()) } - async fn restore_dkg_contract<'a>( + async fn restore_dkg_contract( &self, - ctx: &DkgSkipCtx<'a>, + ctx: &DkgSkipCtx<'_>, code_id: ContractCodeId, ) -> Result<(), NetworkManagerError> { ctx.println(format!( @@ -363,7 +363,7 @@ impl NetworkManager { Ok(()) } - async fn add_group_members<'a>(&self, ctx: &DkgSkipCtx<'a>) -> Result<(), NetworkManagerError> { + async fn add_group_members(&self, ctx: &DkgSkipCtx<'_>) -> Result<(), NetworkManagerError> { ctx.println(format!( "👪 {}Adding all the cw4 group members...", style("[7/8]").bold().dim() @@ -387,9 +387,9 @@ impl NetworkManager { Ok(()) } - async fn transfer_signer_tokens<'a>( + async fn transfer_signer_tokens( &self, - ctx: &DkgSkipCtx<'a>, + ctx: &DkgSkipCtx<'_>, ) -> Result<(), NetworkManagerError> { ctx.println(format!( "💸 {}Transferring tokens to the new signers...", diff --git a/tools/internal/testnet-manager/src/manager/local_apis.rs b/tools/internal/testnet-manager/src/manager/local_apis.rs index 8ad28e8743..7da09928c3 100644 --- a/tools/internal/testnet-manager/src/manager/local_apis.rs +++ b/tools/internal/testnet-manager/src/manager/local_apis.rs @@ -67,9 +67,9 @@ impl NetworkManager { .join(DEFAULT_CONFIG_FILENAME) } - async fn initialise_api<'a>( + async fn initialise_api( &self, - ctx: &LocalApisCtx<'a>, + ctx: &LocalApisCtx<'_>, info: &EcashSignerWithPaths, ) -> Result<(), NetworkManagerError> { let address = &info.data.cosmos_account.address; @@ -139,7 +139,7 @@ impl NetworkManager { Ok(()) } - async fn initialise_apis<'a>(&self, ctx: &LocalApisCtx<'a>) -> Result<(), NetworkManagerError> { + async fn initialise_apis(&self, ctx: &LocalApisCtx<'_>) -> Result<(), NetworkManagerError> { ctx.println(format!( "🔏 {}Initialising local nym-apis...", style("[1/1]").bold().dim() diff --git a/tools/internal/testnet-manager/src/manager/local_client.rs b/tools/internal/testnet-manager/src/manager/local_client.rs index f5c8ba173a..c95c3062df 100644 --- a/tools/internal/testnet-manager/src/manager/local_client.rs +++ b/tools/internal/testnet-manager/src/manager/local_client.rs @@ -80,9 +80,9 @@ impl NetworkManager { .join(DEFAULT_CONFIG_FILENAME) } - async fn wait_for_api_gateway<'a>( + async fn wait_for_api_gateway( &self, - ctx: &LocalClientCtx<'a>, + ctx: &LocalClientCtx<'_>, ) -> Result { // create api client // hehe, that's disgusting, but it's not meant to be used by users @@ -145,9 +145,9 @@ impl NetworkManager { } } - async fn wait_for_gateway_endpoint<'a>( + async fn wait_for_gateway_endpoint( &self, - ctx: &LocalClientCtx<'a>, + ctx: &LocalClientCtx<'_>, gateway: SocketAddr, ) -> Result<(), NetworkManagerError> { ctx.set_pb_message(format!( @@ -177,17 +177,14 @@ impl NetworkManager { Ok(()) } - async fn wait_for_gateway<'a>( - &self, - ctx: &LocalClientCtx<'a>, - ) -> Result<(), NetworkManagerError> { + async fn wait_for_gateway(&self, ctx: &LocalClientCtx<'_>) -> Result<(), NetworkManagerError> { let endpoint = self.wait_for_api_gateway(ctx).await?; self.wait_for_gateway_endpoint(ctx, endpoint).await } - async fn prepare_nym_client<'a>( + async fn prepare_nym_client( &self, - ctx: &LocalClientCtx<'a>, + ctx: &LocalClientCtx<'_>, ) -> Result<(), NetworkManagerError> { ctx.println(format!( "🔏 {}Initialising local nym-client...", diff --git a/tools/internal/testnet-manager/src/manager/local_nodes.rs b/tools/internal/testnet-manager/src/manager/local_nodes.rs index 792c67e8fe..e140ccb20a 100644 --- a/tools/internal/testnet-manager/src/manager/local_nodes.rs +++ b/tools/internal/testnet-manager/src/manager/local_nodes.rs @@ -102,9 +102,9 @@ struct ReducedSignatureOut { } impl NetworkManager { - async fn initialise_nym_node<'a>( + async fn initialise_nym_node( &self, - ctx: &mut LocalNodesCtx<'a>, + ctx: &mut LocalNodesCtx<'_>, offset: u16, is_gateway: bool, ) -> Result<(), NetworkManagerError> { @@ -222,9 +222,9 @@ impl NetworkManager { Ok(()) } - async fn initialise_nym_nodes<'a>( + async fn initialise_nym_nodes( &self, - ctx: &mut LocalNodesCtx<'a>, + ctx: &mut LocalNodesCtx<'_>, mixnodes: u16, gateways: u16, ) -> Result<(), NetworkManagerError> { @@ -250,9 +250,9 @@ impl NetworkManager { Ok(()) } - async fn transfer_bonding_tokens<'a>( + async fn transfer_bonding_tokens( &self, - ctx: &LocalNodesCtx<'a>, + ctx: &LocalNodesCtx<'_>, ) -> Result<(), NetworkManagerError> { ctx.println(format!( "💸 {}Transferring tokens to the bond owners...", @@ -281,9 +281,9 @@ impl NetworkManager { Ok(()) } - async fn bond_node<'a>( + async fn bond_node( &self, - ctx: &LocalNodesCtx<'a>, + ctx: &LocalNodesCtx<'_>, node: &NymNode, is_gateway: bool, ) -> Result<(), NetworkManagerError> { @@ -318,7 +318,7 @@ impl NetworkManager { Ok(()) } - async fn bond_nym_nodes<'a>(&self, ctx: &LocalNodesCtx<'a>) -> Result<(), NetworkManagerError> { + async fn bond_nym_nodes(&self, ctx: &LocalNodesCtx<'_>) -> Result<(), NetworkManagerError> { ctx.println(format!( "⛓️ {}Bonding the local nym-nodes...", style("[3/5]").bold().dim() @@ -336,9 +336,9 @@ impl NetworkManager { Ok(()) } - async fn assign_to_active_set<'a>( + async fn assign_to_active_set( &self, - ctx: &LocalNodesCtx<'a>, + ctx: &LocalNodesCtx<'_>, ) -> Result<(), NetworkManagerError> { ctx.println(format!( "🔌 {}Assigning nodes to the active set...", @@ -460,9 +460,9 @@ impl NetworkManager { ctx.progress.output_run_commands(cmds) } - async fn persist_nodes_in_database<'a>( + async fn persist_nodes_in_database( &self, - ctx: &LocalNodesCtx<'a>, + ctx: &LocalNodesCtx<'_>, ) -> Result<(), NetworkManagerError> { ctx.println(format!( "📦 {}Storing the node information in the database", From 5a6770e5e2f9d00e122f38b813d3dd213a403adf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Stuczy=C5=84ski?= Date: Fri, 10 Jan 2025 14:17:03 +0100 Subject: [PATCH 15/19] chore: readjusted --mode behaviour to fix the regression (#5331) --- nym-node/src/config/mod.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/nym-node/src/config/mod.rs b/nym-node/src/config/mod.rs index c9d614e6b1..243f0b403f 100644 --- a/nym-node/src/config/mod.rs +++ b/nym-node/src/config/mod.rs @@ -75,11 +75,12 @@ pub enum NodeMode { #[clap(alias = "entry", alias = "gateway")] EntryGateway, + // to not break existing behaviour, this means exit capabilities AND entry capabilities #[clap(alias = "exit")] ExitGateway, - // entry + exit - FullGateway, + // will start only SP needed for exit capabilities WITHOUT entry routing + ExitProvidersOnly, } impl Display for NodeMode { @@ -88,7 +89,7 @@ impl Display for NodeMode { NodeMode::Mixnode => "mixnode".fmt(f), NodeMode::EntryGateway => "entry-gateway".fmt(f), NodeMode::ExitGateway => "exit-gateway".fmt(f), - NodeMode::FullGateway => "full-gateway".fmt(f), + NodeMode::ExitProvidersOnly => "exit-providers-only".fmt(f), } } } @@ -129,8 +130,8 @@ impl NodeModes { match mode { NodeMode::Mixnode => self.with_mixnode(), NodeMode::EntryGateway => self.with_entry(), - NodeMode::ExitGateway => self.with_exit(), - NodeMode::FullGateway => self.with_entry().with_exit(), + NodeMode::ExitGateway => self.with_entry().with_exit(), + NodeMode::ExitProvidersOnly => self.with_exit(), } } From 676e93a3727a4abed7cc054b8c486b0a849a9460 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Stuczy=C5=84ski?= Date: Fri, 10 Jan 2025 15:52:52 +0100 Subject: [PATCH 16/19] bugfix: make sure refresh data key matches bond info (#5329) --- nym-api/src/node_describe_cache/mod.rs | 65 ++++++++++++++++----- nym-api/src/nym_contract_cache/cache/mod.rs | 6 +- 2 files changed, 53 insertions(+), 18 deletions(-) diff --git a/nym-api/src/node_describe_cache/mod.rs b/nym-api/src/node_describe_cache/mod.rs index adf25c29f1..0cd2d0fe3e 100644 --- a/nym-api/src/node_describe_cache/mod.rs +++ b/nym-api/src/node_describe_cache/mod.rs @@ -12,6 +12,7 @@ use futures::{stream, StreamExt}; use nym_api_requests::legacy::{LegacyGatewayBondWithId, LegacyMixNodeDetailsWithLayer}; use nym_api_requests::models::{DescribedNodeType, NymNodeData, NymNodeDescription}; use nym_config::defaults::DEFAULT_NYM_NODE_HTTP_PORT; +use nym_crypto::asymmetric::ed25519; use nym_mixnet_contract_common::{LegacyMixLayer, NodeId, NymNodeDetails}; use nym_node_requests::api::client::{NymNodeApiClientError, NymNodeApiClientExt}; use nym_topology::gateway::GatewayConversionError; @@ -58,6 +59,13 @@ pub enum NodeDescribeCacheError { #[error("could not verify signed host information for node {node_id}")] MissignedHostInformation { node_id: NodeId }, + #[error("identity of node {node_id} does not match. expected {expected} but got {got}")] + MismatchedIdentity { + node_id: NodeId, + expected: String, + got: String, + }, + #[error("node {node_id} is announcing an illegal ip address")] IllegalIpAddress { node_id: NodeId }, } @@ -289,6 +297,15 @@ async fn try_get_description( let host_info = client.get_host_information().await.map_err(map_query_err)?; + // check if the identity key matches the information provided during bonding + if data.expected_identity != host_info.keys.ed25519_identity { + return Err(NodeDescribeCacheError::MismatchedIdentity { + node_id: data.node_id, + expected: data.expected_identity.to_base58_string(), + got: host_info.keys.ed25519_identity.to_base58_string(), + }); + } + if !host_info.verify_host_information() { return Err(NodeDescribeCacheError::MissignedHostInformation { node_id: data.node_id, @@ -315,47 +332,58 @@ async fn try_get_description( pub(crate) struct RefreshData { host: String, node_id: NodeId, + expected_identity: ed25519::PublicKey, node_type: DescribedNodeType, port: Option, } -impl<'a> From<&'a LegacyMixNodeDetailsWithLayer> for RefreshData { - fn from(node: &'a LegacyMixNodeDetailsWithLayer) -> Self { - RefreshData::new( +impl<'a> TryFrom<&'a LegacyMixNodeDetailsWithLayer> for RefreshData { + type Error = ed25519::Ed25519RecoveryError; + + fn try_from(node: &'a LegacyMixNodeDetailsWithLayer) -> Result { + Ok(RefreshData::new( &node.bond_information.mix_node.host, + node.bond_information.identity().parse()?, DescribedNodeType::LegacyMixnode, node.mix_id(), Some(node.bond_information.mix_node.http_api_port), - ) + )) } } -impl<'a> From<&'a LegacyGatewayBondWithId> for RefreshData { - fn from(node: &'a LegacyGatewayBondWithId) -> Self { - RefreshData::new( +impl<'a> TryFrom<&'a LegacyGatewayBondWithId> for RefreshData { + type Error = ed25519::Ed25519RecoveryError; + + fn try_from(node: &'a LegacyGatewayBondWithId) -> Result { + Ok(RefreshData::new( &node.bond.gateway.host, + node.bond.identity().parse()?, DescribedNodeType::LegacyGateway, node.node_id, None, - ) + )) } } -impl<'a> From<&'a NymNodeDetails> for RefreshData { - fn from(node: &'a NymNodeDetails) -> Self { - RefreshData::new( +impl<'a> TryFrom<&'a NymNodeDetails> for RefreshData { + type Error = ed25519::Ed25519RecoveryError; + + fn try_from(node: &'a NymNodeDetails) -> Result { + Ok(RefreshData::new( &node.bond_information.node.host, + node.bond_information.identity().parse()?, DescribedNodeType::NymNode, node.node_id(), node.bond_information.node.custom_http_port, - ) + )) } } impl RefreshData { pub fn new( host: impl Into, + expected_identity: ed25519::PublicKey, node_type: DescribedNodeType, node_id: NodeId, port: Option, @@ -363,6 +391,7 @@ impl RefreshData { RefreshData { host: host.into(), node_id, + expected_identity, node_type, port, } @@ -404,7 +433,9 @@ impl CacheItemProvider for NodeDescriptionProvider { None => error!("failed to obtain mixnodes information from the cache"), Some(legacy_mixnodes) => { for node in &**legacy_mixnodes { - nodes_to_query.push(node.into()) + if let Ok(data) = node.try_into() { + nodes_to_query.push(data); + } } } } @@ -413,7 +444,9 @@ impl CacheItemProvider for NodeDescriptionProvider { None => error!("failed to obtain gateways information from the cache"), Some(legacy_gateways) => { for node in &**legacy_gateways { - nodes_to_query.push(node.into()) + if let Ok(data) = node.try_into() { + nodes_to_query.push(data); + } } } } @@ -422,7 +455,9 @@ impl CacheItemProvider for NodeDescriptionProvider { None => error!("failed to obtain nym-nodes information from the cache"), Some(nym_nodes) => { for node in &**nym_nodes { - nodes_to_query.push(node.into()) + if let Ok(data) = node.try_into() { + nodes_to_query.push(data); + } } } } diff --git a/nym-api/src/nym_contract_cache/cache/mod.rs b/nym-api/src/nym_contract_cache/cache/mod.rs index 1738901550..236a4f22cf 100644 --- a/nym-api/src/nym_contract_cache/cache/mod.rs +++ b/nym-api/src/nym_contract_cache/cache/mod.rs @@ -385,7 +385,7 @@ impl NymContractCache { .iter() .find(|n| n.bond_information.identity() == encoded_identity) { - return Some(nym_node.into()); + return nym_node.try_into().ok(); } // 2. check legacy mixnodes @@ -394,7 +394,7 @@ impl NymContractCache { .iter() .find(|n| n.bond_information.identity() == encoded_identity) { - return Some(mixnode.into()); + return mixnode.try_into().ok(); } // 3. check legacy gateways @@ -403,7 +403,7 @@ impl NymContractCache { .iter() .find(|n| n.identity() == &encoded_identity) { - return Some(gateway.into()); + return gateway.try_into().ok(); } None From 25766dc0ec2449cac8520516ad562931eb665899 Mon Sep 17 00:00:00 2001 From: RadekSabacky Date: Tue, 14 Jan 2025 13:22:31 +0100 Subject: [PATCH 17/19] + add alert message into nav components --- explorer-nextjs/app/components/Nav/DesktopNav.tsx | 2 ++ explorer-nextjs/app/components/Nav/MobileNav.tsx | 2 ++ 2 files changed, 4 insertions(+) diff --git a/explorer-nextjs/app/components/Nav/DesktopNav.tsx b/explorer-nextjs/app/components/Nav/DesktopNav.tsx index 618037ea04..e1faab46f1 100644 --- a/explorer-nextjs/app/components/Nav/DesktopNav.tsx +++ b/explorer-nextjs/app/components/Nav/DesktopNav.tsx @@ -20,6 +20,7 @@ import { NYM_WEBSITE } from '@/app/api/constants' import { useMainContext } from '@/app/context/main' import { MobileDrawerClose } from '@/app/icons/MobileDrawerClose' import { NavOptionType, originalNavOptions } from '@/app/context/nav' +import { ReleaseAlert } from '@/app/components/ReleaseAlert' import { DarkLightSwitchDesktop } from '@/app/components/Switch' import { Footer } from '@/app/components/Footer' import { ConnectKeplrWallet } from '@/app/components/Wallet/ConnectKeplrWallet' @@ -369,6 +370,7 @@ export const Nav: FCWithChildren = ({ children }) => { style={{ width: `calc(100% - ${drawerWidth}px` }} sx={{ py: 5, px: 6, mt: 7 }} > + {children}