diff --git a/Cargo.lock b/Cargo.lock index 40e54deb50..dbc4c27625 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6354,6 +6354,7 @@ dependencies = [ name = "nym-gateway" version = "1.1.36" dependencies = [ + "anyhow", "async-trait", "bincode", "bip39", @@ -6392,6 +6393,7 @@ dependencies = [ "nym-sphinx", "nym-statistics-common", "nym-task", + "nym-test-utils", "nym-topology", "nym-upgrade-mode-check", "nym-validator-client", @@ -8365,11 +8367,13 @@ dependencies = [ name = "nym-wireguard" version = "1.20.1" dependencies = [ + "anyhow", "base64 0.22.1", "defguard_wireguard_rs", "futures", "ip_network", "ipnetwork", + "mock_instant", "nym-authenticator-requests", "nym-credential-verification", "nym-credentials-interface", @@ -8380,6 +8384,7 @@ dependencies = [ "nym-network-defaults", "nym-node-metrics", "nym-task", + "nym-test-utils", "nym-wireguard-types", "rand 0.8.5", "thiserror 2.0.17", diff --git a/Cargo.toml b/Cargo.toml index a9a240b504..c6b1bc64a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -303,6 +303,7 @@ ledger-transport = "0.10.0" ledger-transport-hid = "0.10.0" log = "0.4" mime = "0.3.17" +mock_instant = "0.6.0" moka = { version = "0.12", features = ["future"] } nix = "0.30.1" notify = "5.1.0" diff --git a/common/authenticator-requests/src/lib.rs b/common/authenticator-requests/src/lib.rs index 226c78adee..eb5b14b5df 100644 --- a/common/authenticator-requests/src/lib.rs +++ b/common/authenticator-requests/src/lib.rs @@ -18,6 +18,7 @@ mod util; mod version; pub use error::Error; +pub use util::{authenticator_ipv4_to_ipv6, authenticator_ipv6_to_ipv4}; pub use v6 as latest; pub use version::AuthenticatorVersion; diff --git a/common/authenticator-requests/src/response.rs b/common/authenticator-requests/src/response.rs index 84b6ae8526..467fd25514 100644 --- a/common/authenticator-requests/src/response.rs +++ b/common/authenticator-requests/src/response.rs @@ -7,6 +7,7 @@ use crate::traits::{ TopUpBandwidthResponse, UpgradeModeStatus, }; use crate::{v2, v3, v4, v5, v6}; +use nym_sphinx::addressing::Recipient; #[derive(Debug)] pub enum AuthenticatorResponse { @@ -17,6 +18,17 @@ pub enum AuthenticatorResponse { UpgradeMode(Box), } +pub struct SerialisedResponse { + pub bytes: Vec, + pub reply_to: Option, +} + +impl SerialisedResponse { + pub fn new(bytes: Vec, reply_to: Option) -> Self { + Self { bytes, reply_to } + } +} + impl UpgradeModeStatus for AuthenticatorResponse { fn upgrade_mode_status(&self) -> CurrentUpgradeModeStatus { match self { diff --git a/common/authenticator-requests/src/util.rs b/common/authenticator-requests/src/util.rs index cb1269f10c..451b4a77b5 100644 --- a/common/authenticator-requests/src/util.rs +++ b/common/authenticator-requests/src/util.rs @@ -1,6 +1,38 @@ // Copyright 2024 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 +use nym_network_defaults::{WG_TUN_DEVICE_IP_ADDRESS_V4, WG_TUN_DEVICE_IP_ADDRESS_V6}; +use std::net::{Ipv4Addr, Ipv6Addr}; + +pub fn authenticator_ipv6_to_ipv4(addr: Ipv6Addr) -> Ipv4Addr { + let before_last_byte = addr.octets()[14]; + let last_byte = addr.octets()[15]; + + Ipv4Addr::new( + WG_TUN_DEVICE_IP_ADDRESS_V4.octets()[0], + WG_TUN_DEVICE_IP_ADDRESS_V4.octets()[1], + before_last_byte, + last_byte, + ) +} + +pub fn authenticator_ipv4_to_ipv6(addr: Ipv4Addr) -> Ipv6Addr { + let before_last_byte = addr.octets()[2]; + let last_byte = addr.octets()[3]; + + let last_bytes = ((before_last_byte as u16) << 8) | last_byte as u16; + Ipv6Addr::new( + WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[0], + WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[1], + WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[2], + WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[3], + WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[4], + WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[5], + WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[6], + last_bytes, + ) +} + #[cfg(test)] pub(crate) mod tests { pub(crate) const CREDENTIAL_BYTES: [u8; 1245] = [ diff --git a/common/authenticator-requests/src/v4/registration.rs b/common/authenticator-requests/src/v4/registration.rs index b1ee074dfd..e718bbab5c 100644 --- a/common/authenticator-requests/src/v4/registration.rs +++ b/common/authenticator-requests/src/v4/registration.rs @@ -2,9 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 use crate::error::Error; +use crate::util::{authenticator_ipv4_to_ipv6, authenticator_ipv6_to_ipv4}; use base64::{Engine, engine::general_purpose}; use nym_credentials_interface::CredentialSpendingData; -use nym_network_defaults::constants::{WG_TUN_DEVICE_IP_ADDRESS_V4, WG_TUN_DEVICE_IP_ADDRESS_V6}; use nym_wireguard_types::PeerPublicKey; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -56,27 +56,11 @@ impl fmt::Display for IpPair { impl From for IpPair { fn from(value: IpAddr) -> Self { - let (before_last_byte, last_byte) = match value { - std::net::IpAddr::V4(ipv4_addr) => (ipv4_addr.octets()[2], ipv4_addr.octets()[3]), - std::net::IpAddr::V6(ipv6_addr) => (ipv6_addr.octets()[14], ipv6_addr.octets()[15]), + let (ipv4, ipv6) = match value { + IpAddr::V4(ipv4) => (ipv4, authenticator_ipv4_to_ipv6(ipv4)), + IpAddr::V6(ipv6_addr) => (authenticator_ipv6_to_ipv4(ipv6_addr), ipv6_addr), }; - let last_bytes = ((before_last_byte as u16) << 8) | last_byte as u16; - let ipv4 = Ipv4Addr::new( - WG_TUN_DEVICE_IP_ADDRESS_V4.octets()[0], - WG_TUN_DEVICE_IP_ADDRESS_V4.octets()[1], - before_last_byte, - last_byte, - ); - let ipv6 = Ipv6Addr::new( - WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[0], - WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[1], - WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[2], - WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[3], - WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[4], - WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[5], - WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[6], - last_bytes, - ); + IpPair::new(ipv4, ipv6) } } diff --git a/common/authenticator-requests/src/v5/registration.rs b/common/authenticator-requests/src/v5/registration.rs index 5154400f93..6000dd6760 100644 --- a/common/authenticator-requests/src/v5/registration.rs +++ b/common/authenticator-requests/src/v5/registration.rs @@ -2,9 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 use crate::error::Error; +use crate::util::{authenticator_ipv4_to_ipv6, authenticator_ipv6_to_ipv4}; use base64::{Engine, engine::general_purpose}; use nym_credentials_interface::CredentialSpendingData; -use nym_network_defaults::constants::{WG_TUN_DEVICE_IP_ADDRESS_V4, WG_TUN_DEVICE_IP_ADDRESS_V6}; use nym_wireguard_types::PeerPublicKey; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -54,27 +54,11 @@ impl fmt::Display for IpPair { impl From for IpPair { fn from(value: IpAddr) -> Self { - let (before_last_byte, last_byte) = match value { - std::net::IpAddr::V4(ipv4_addr) => (ipv4_addr.octets()[2], ipv4_addr.octets()[3]), - std::net::IpAddr::V6(ipv6_addr) => (ipv6_addr.octets()[14], ipv6_addr.octets()[15]), + let (ipv4, ipv6) = match value { + IpAddr::V4(ipv4) => (ipv4, authenticator_ipv4_to_ipv6(ipv4)), + IpAddr::V6(ipv6_addr) => (authenticator_ipv6_to_ipv4(ipv6_addr), ipv6_addr), }; - let last_bytes = ((before_last_byte as u16) << 8) | last_byte as u16; - let ipv4 = Ipv4Addr::new( - WG_TUN_DEVICE_IP_ADDRESS_V4.octets()[0], - WG_TUN_DEVICE_IP_ADDRESS_V4.octets()[1], - before_last_byte, - last_byte, - ); - let ipv6 = Ipv6Addr::new( - WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[0], - WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[1], - WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[2], - WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[3], - WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[4], - WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[5], - WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[6], - last_bytes, - ); + IpPair::new(ipv4, ipv6) } } diff --git a/common/authenticator-requests/src/v6/registration.rs b/common/authenticator-requests/src/v6/registration.rs index 14e7462c05..a433a4e7e9 100644 --- a/common/authenticator-requests/src/v6/registration.rs +++ b/common/authenticator-requests/src/v6/registration.rs @@ -3,13 +3,12 @@ use crate::error::Error; use crate::models::BandwidthClaim; +use crate::util::{authenticator_ipv4_to_ipv6, authenticator_ipv6_to_ipv4}; use base64::{Engine, engine::general_purpose}; -use nym_network_defaults::constants::{WG_TUN_DEVICE_IP_ADDRESS_V4, WG_TUN_DEVICE_IP_ADDRESS_V6}; use nym_wireguard_types::PeerPublicKey; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; -use std::time::SystemTime; use std::{fmt, ops::Deref, str::FromStr}; #[cfg(feature = "verify")] @@ -20,7 +19,6 @@ use nym_crypto::asymmetric::x25519::{PrivateKey, PublicKey}; use sha2::Sha256; pub type PendingRegistrations = HashMap; -pub type PrivateIPs = HashMap; #[cfg(feature = "verify")] pub type HmacSha256 = Hmac; @@ -53,27 +51,11 @@ impl fmt::Display for IpPair { impl From for IpPair { fn from(value: IpAddr) -> Self { - let (before_last_byte, last_byte) = match value { - IpAddr::V4(ipv4_addr) => (ipv4_addr.octets()[2], ipv4_addr.octets()[3]), - IpAddr::V6(ipv6_addr) => (ipv6_addr.octets()[14], ipv6_addr.octets()[15]), + let (ipv4, ipv6) = match value { + IpAddr::V4(ipv4) => (ipv4, authenticator_ipv4_to_ipv6(ipv4)), + IpAddr::V6(ipv6_addr) => (authenticator_ipv6_to_ipv4(ipv6_addr), ipv6_addr), }; - let last_bytes = ((before_last_byte as u16) << 8) | last_byte as u16; - let ipv4 = Ipv4Addr::new( - WG_TUN_DEVICE_IP_ADDRESS_V4.octets()[0], - WG_TUN_DEVICE_IP_ADDRESS_V4.octets()[1], - before_last_byte, - last_byte, - ); - let ipv6 = Ipv6Addr::new( - WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[0], - WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[1], - WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[2], - WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[3], - WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[4], - WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[5], - WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[6], - last_bytes, - ); + IpPair::new(ipv4, ipv6) } } diff --git a/common/credential-verification/src/lib.rs b/common/credential-verification/src/lib.rs index cafc5bfcfd..fbf8152c8c 100644 --- a/common/credential-verification/src/lib.rs +++ b/common/credential-verification/src/lib.rs @@ -22,6 +22,8 @@ pub mod ecash; pub mod error; pub mod upgrade_mode; +const MOCK_BANDWIDTH: i64 = 2024 * 1024 * 1024; + // Histogram buckets for ecash verification duration (in seconds) const ECASH_VERIFICATION_DURATION_BUCKETS: &[f64] = &[0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0]; @@ -111,6 +113,13 @@ impl CredentialVerifier { } pub async fn verify(&mut self) -> Result { + if self.ecash_verifier.is_mock() { + // if we're in the mock mode (local testing), skip cryptographic verification + // and just return a dummy bandwidth value since we don't have blockchain access + // Return a reasonable test bandwidth value (e.g., 1GB in bytes) + return Ok(MOCK_BANDWIDTH); + } + let start = Instant::now(); nym_metrics::inc!("ecash_verification_attempts"); diff --git a/common/credential-verification/src/upgrade_mode.rs b/common/credential-verification/src/upgrade_mode.rs index 7dec1010f7..93177c688e 100644 --- a/common/credential-verification/src/upgrade_mode.rs +++ b/common/credential-verification/src/upgrade_mode.rs @@ -291,3 +291,40 @@ struct UpgradeModeStateInner { // (and dealing with the async consequences of that) status: UpgradeModeStatus, } + +pub mod testing { + use crate::UpgradeModeState; + use crate::upgrade_mode::{ + CheckRequest, UpgradeModeCheckConfig, UpgradeModeCheckRequestSender, UpgradeModeDetails, + }; + use futures::channel::mpsc::UnboundedReceiver; + use nym_crypto::asymmetric::ed25519; + use std::time::Duration; + + pub fn mock_dummy_upgrade_mode_details() -> (UpgradeModeDetails, UnboundedReceiver) + { + let (um_recheck_tx, um_recheck_rx) = futures::channel::mpsc::unbounded(); + + const DUMMY_ATTESTER_ED25519_PRIVATE_KEY: [u8; 32] = [ + 108, 49, 193, 21, 126, 161, 249, 85, 242, 207, 74, 195, 238, 6, 64, 149, 201, 140, 248, + 163, 122, 170, 79, 198, 87, 85, 36, 29, 243, 92, 64, 161, + ]; + + pub(crate) fn dummy_attester_public_key() -> ed25519::PublicKey { + let private_key = + ed25519::PrivateKey::from_bytes(&DUMMY_ATTESTER_ED25519_PRIVATE_KEY).unwrap(); + private_key.public_key() + } + + let upgrade_mode_state = UpgradeModeState::new(dummy_attester_public_key()); + let upgrade_mode_details = UpgradeModeDetails::new( + UpgradeModeCheckConfig { + // essentially we never want to trigger this in our tests + min_staleness_recheck: Duration::from_nanos(1), + }, + UpgradeModeCheckRequestSender::new(um_recheck_tx), + upgrade_mode_state.clone(), + ); + (upgrade_mode_details, um_recheck_rx) + } +} diff --git a/common/registration/src/lib.rs b/common/registration/src/lib.rs index 2beb0632ed..f297de143d 100644 --- a/common/registration/src/lib.rs +++ b/common/registration/src/lib.rs @@ -27,6 +27,22 @@ pub struct NymNodeInformation { pub version: AuthenticatorVersion, } +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +pub struct WireguardRegistrationData { + /// Public x25519 key of this gateway + #[serde(with = "bs58_x25519_pubkey")] + pub public_key: x25519::PublicKey, + + /// Port at which this gateway is accessible for wireguard + pub port: u16, + + /// Ipv4 address assigned to this peer + pub private_ipv4: Ipv4Addr, + + /// Ipv6 address assigned to this peer + pub private_ipv6: Ipv6Addr, +} + #[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] pub struct WireguardConfiguration { #[serde(with = "bs58_x25519_pubkey")] diff --git a/common/registration/src/lp_messages.rs b/common/registration/src/lp_messages.rs index ff64fc3746..c294b87abb 100644 --- a/common/registration/src/lp_messages.rs +++ b/common/registration/src/lp_messages.rs @@ -3,7 +3,7 @@ //! LP (Lewes Protocol) registration message types shared between client and gateway. -use crate::WireguardConfiguration; +use crate::WireguardRegistrationData; use crate::dvpn::{ LpDvpnRegistrationFinalisation, LpDvpnRegistrationInitialRequest, LpDvpnRegistrationRequestMessage, LpDvpnRegistrationRequestMessageContent, @@ -16,7 +16,6 @@ use crate::mixnet::{ }; use crate::serialisation::{BincodeError, BincodeOptions, lp_bincode_serializer}; use nym_authenticator_requests::models::BandwidthClaim; -use nym_credentials_interface::TicketType; use serde::{Deserialize, Serialize}; use tracing::error; @@ -135,16 +134,11 @@ impl LpRegistrationRequest { pub fn new_initial_dvpn( wg_public_key: nym_wireguard_types::PeerPublicKey, psk: [u8; 32], - ticket_type: TicketType, ) -> Self { Self::new(LpRegistrationRequestData::Dvpn { data: Box::new(LpDvpnRegistrationRequestMessage { content: LpDvpnRegistrationRequestMessageContent::InitialRequest( - LpDvpnRegistrationInitialRequest { - wg_public_key, - psk, - ticket_type, - }, + LpDvpnRegistrationInitialRequest { wg_public_key, psk }, ), }), }) @@ -180,7 +174,7 @@ impl LpRegistrationRequest { impl LpRegistrationResponse { /// Create a success response with GatewayData (for dVPN mode) - pub fn success_dvpn(config: WireguardConfiguration, available_bandwidth: i64) -> Self { + pub fn success_dvpn(config: WireguardRegistrationData, upgrade_mode: bool) -> Self { Self { status: RegistrationStatus::Completed, response_data: LpRegistrationResponseData::Dvpn { @@ -188,7 +182,7 @@ impl LpRegistrationResponse { content: LpDvpnRegistrationResponseMessageContent::CompletedRegistration( dvpn::CompletedRegistrationResponse { config, - available_bandwidth, + upgrade_mode, }, ), }, @@ -196,16 +190,13 @@ impl LpRegistrationResponse { } } - pub fn success_mixnet(config: LpMixnetGatewayData, available_bandwidth: i64) -> Self { + pub fn success_mixnet(config: LpMixnetGatewayData) -> Self { Self { status: RegistrationStatus::Completed, response_data: LpRegistrationResponseData::Mixnet { data: LpMixnetRegistrationResponseMessage { content: LpMixnetRegistrationResponseMessageContent::CompletedRegistration( - mixnet::CompletedRegistrationResponse { - config, - available_bandwidth, - }, + mixnet::CompletedRegistrationResponse { config }, ), }, }, @@ -284,9 +275,8 @@ impl LpRegistrationResponse { } pub mod dvpn { - use crate::WireguardConfiguration; + use crate::WireguardRegistrationData; use nym_authenticator_requests::models::BandwidthClaim; - use nym_credentials_interface::TicketType; use serde::{Deserialize, Serialize}; // client @@ -310,9 +300,6 @@ pub mod dvpn { /// Preshared key to be used for the connection pub psk: [u8; 32], - - /// Type of the ticket/gateway we're going to register with - pub ticket_type: TicketType, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -350,10 +337,11 @@ pub mod dvpn { pub struct CompletedRegistrationResponse { /// Gateway configuration data for dVPN mode (WireGuard) /// This matches what WireguardRegistrationResult expects - pub config: WireguardConfiguration, + pub config: WireguardRegistrationData, - /// The bandwidth available to this client, - pub available_bandwidth: i64, + /// Flag indicating whether the gateway has detected the system is undergoing the upgrade + /// (thus it will not meter bandwidth) + pub upgrade_mode: bool, } #[derive(Debug, Clone, Copy, Serialize, Deserialize)] @@ -427,9 +415,6 @@ pub mod mixnet { /// /// Contains gateway identity and sphinx key needed for nym address construction. pub config: LpMixnetGatewayData, - - /// The bandwidth available to this client, - pub available_bandwidth: i64, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -446,15 +431,14 @@ mod tests { use std::net::{Ipv4Addr, Ipv6Addr}; // ==================== Helper Functions ==================== - fn create_test_gateway_data() -> WireguardConfiguration { - WireguardConfiguration { + fn create_test_wg_config() -> WireguardRegistrationData { + WireguardRegistrationData { public_key: nym_crypto::asymmetric::x25519::PublicKey::from( nym_sphinx::PublicKey::from([1u8; 32]), ), - psk: Some([42u8; 32]), + port: 1234, private_ipv4: Ipv4Addr::new(10, 0, 0, 1), private_ipv6: Ipv6Addr::new(0xfc00, 0, 0, 0, 0, 0, 0, 1), - endpoint: "192.168.1.1:8080".parse().expect("Valid test endpoint"), } } @@ -499,10 +483,9 @@ mod tests { #[test] fn test_lp_registration_response_success_dvpn() { - let cfg = create_test_gateway_data(); - let allocated_bandwidth = 500_000_000; + let cfg = create_test_wg_config(); - let response = LpRegistrationResponse::success_dvpn(cfg, allocated_bandwidth); + let response = LpRegistrationResponse::success_dvpn(cfg, false); assert!(response.status.is_successful()); let LpRegistrationResponseData::Dvpn { data } = response.response_data else { @@ -515,7 +498,7 @@ mod tests { panic!("unexpected response") }; assert_eq!(complete.config, cfg); - assert_eq!(complete.available_bandwidth, allocated_bandwidth); + assert!(!complete.upgrade_mode); } #[test] @@ -526,10 +509,7 @@ mod tests { let lp_gateway_data = LpMixnetGatewayData { gateway_identity: *valid_key.public_key(), }; - let allocated_bandwidth = 500_000_000; - - let response = - LpRegistrationResponse::success_mixnet(lp_gateway_data.clone(), allocated_bandwidth); + let response = LpRegistrationResponse::success_mixnet(lp_gateway_data.clone()); assert!(response.status.is_successful()); let LpRegistrationResponseData::Mixnet { data } = response.response_data else { @@ -542,6 +522,5 @@ mod tests { panic!("unexpected response") }; assert_eq!(complete.config, lp_gateway_data); - assert_eq!(complete.available_bandwidth, allocated_bandwidth); } } diff --git a/common/service-provider-requests-common/src/lib.rs b/common/service-provider-requests-common/src/lib.rs index 44e54dc2af..c43ea89074 100644 --- a/common/service-provider-requests-common/src/lib.rs +++ b/common/service-provider-requests-common/src/lib.rs @@ -11,7 +11,7 @@ pub enum ProtocolError { InvalidServiceProviderType(u8), } -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] #[repr(u8)] pub enum ServiceProviderType { NetworkRequester = 0, @@ -76,7 +76,7 @@ impl ServiceProviderTypeExt for u8 { } } -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)] pub struct Protocol { pub version: u8, pub service_provider_type: ServiceProviderType, diff --git a/common/test-utils/src/helpers.rs b/common/test-utils/src/helpers.rs index 29d6c184d3..e96494dfc0 100644 --- a/common/test-utils/src/helpers.rs +++ b/common/test-utils/src/helpers.rs @@ -2,16 +2,16 @@ // SPDX-License-Identifier: Apache-2.0 use crate::traits::Timeboxed; -use rand_chacha::ChaCha20Rng; +use nym_bin_common::logging::tracing_subscriber::EnvFilter; +use nym_bin_common::logging::tracing_subscriber::layer::SubscriberExt; +use nym_bin_common::logging::tracing_subscriber::util::SubscriberInitExt; +use nym_bin_common::logging::{default_tracing_fmt_layer, tracing_subscriber}; use rand_chacha::rand_core::SeedableRng; use std::future::Future; use tokio::task::JoinHandle; use tokio::time::error::Elapsed; -use nym_bin_common::logging::tracing_subscriber::EnvFilter; -use nym_bin_common::logging::tracing_subscriber::layer::SubscriberExt; -use nym_bin_common::logging::tracing_subscriber::util::SubscriberInitExt; -use nym_bin_common::logging::{default_tracing_fmt_layer, tracing_subscriber}; +pub use rand_chacha::ChaCha20Rng as DeterministicRng; pub use rand_chacha::rand_core::{CryptoRng, RngCore}; pub fn leak(val: T) -> &'static mut T { @@ -26,16 +26,16 @@ where tokio::spawn(async move { fut.timeboxed().await }) } -pub fn deterministic_rng() -> ChaCha20Rng { +pub fn deterministic_rng() -> DeterministicRng { seeded_rng([42u8; 32]) } -pub fn seeded_rng(seed: [u8; 32]) -> ChaCha20Rng { - ChaCha20Rng::from_seed(seed) +pub fn seeded_rng(seed: [u8; 32]) -> DeterministicRng { + DeterministicRng::from_seed(seed) } -pub fn u64_seeded_rng(seed: u64) -> ChaCha20Rng { - ChaCha20Rng::seed_from_u64(seed) +pub fn u64_seeded_rng(seed: u64) -> DeterministicRng { + DeterministicRng::seed_from_u64(seed) } // test logger to use during debugging diff --git a/common/wireguard-private-metadata/tests/src/v2/peer_controller.rs b/common/wireguard-private-metadata/tests/src/v2/peer_controller.rs index 15312a1f52..2b29de63c6 100644 --- a/common/wireguard-private-metadata/tests/src/v2/peer_controller.rs +++ b/common/wireguard-private-metadata/tests/src/v2/peer_controller.rs @@ -26,7 +26,7 @@ impl From<&PeerControlRequest> for PeerControlRequestTypeV2 { fn from(req: &PeerControlRequest) -> Self { match req { PeerControlRequest::AddPeer { .. } => PeerControlRequestTypeV2::AddPeer, - PeerControlRequest::AllocatePeerIpPair { .. } => PeerControlRequestTypeV2::AddPeer, + PeerControlRequest::PreAllocateIpPair { .. } => PeerControlRequestTypeV2::AddPeer, PeerControlRequest::RemovePeer { .. } => PeerControlRequestTypeV2::RemovePeer, PeerControlRequest::QueryPeer { .. } => PeerControlRequestTypeV2::QueryPeer, PeerControlRequest::GetClientBandwidthByKey { .. } => { @@ -41,6 +41,7 @@ impl From<&PeerControlRequest> for PeerControlRequestTypeV2 { PeerControlRequest::GetVerifierByIp { ip, .. } => { PeerControlRequestTypeV2::GetVerifierByIp { ip: *ip } } + PeerControlRequest::CheckActivePeer { .. } => unreachable!(), PeerControlRequest::ReleaseIpPair { .. } => unreachable!(), } } @@ -114,7 +115,7 @@ impl MockPeerControllerV2 { ) .unwrap(); } - PeerControlRequest::AllocatePeerIpPair { response_tx, .. } => { + PeerControlRequest::PreAllocateIpPair { response_tx, .. } => { response_tx .send( *response @@ -186,6 +187,15 @@ impl MockPeerControllerV2 { ) .ok(); } + PeerControlRequest::CheckActivePeer { response_tx, .. } => { + response_tx + .send( + *response + .downcast() + .expect("registered response has mismatched type"), + ) + .ok(); + } } } diff --git a/common/wireguard/Cargo.toml b/common/wireguard/Cargo.toml index 3857d1165a..f6b3c0e6fb 100644 --- a/common/wireguard/Cargo.toml +++ b/common/wireguard/Cargo.toml @@ -36,7 +36,10 @@ nym-wireguard-types = { workspace = true } nym-node-metrics = { workspace = true } [dev-dependencies] +anyhow = { workspace = true } nym-gateway-storage = { workspace = true, features = ["mock"] } +mock_instant = { workspace = true } +nym-test-utils = { workspace = true } [features] default = [] diff --git a/common/wireguard/src/error.rs b/common/wireguard/src/error.rs index 7f5437d630..68d465a534 100644 --- a/common/wireguard/src/error.rs +++ b/common/wireguard/src/error.rs @@ -1,6 +1,8 @@ // Copyright 2024 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 +use crate::IpPoolError; + #[derive(Debug, thiserror::Error)] pub enum Error { #[error("{0}")] @@ -22,7 +24,7 @@ pub enum Error { SystemTime(#[from] std::time::SystemTimeError), #[error("IP pool error: {0}")] - IpPool(String), + IpPool(#[from] IpPoolError), } pub type Result = std::result::Result; diff --git a/common/wireguard/src/ip_pool/mod.rs b/common/wireguard/src/ip_pool/mod.rs index b34a7134c2..7044c8a806 100644 --- a/common/wireguard/src/ip_pool/mod.rs +++ b/common/wireguard/src/ip_pool/mod.rs @@ -1,20 +1,22 @@ // Copyright 2024 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -mod compat; - use defguard_wireguard_rs::host::Peer; use ipnetwork::IpNetwork; use rand::seq::IteratorRandom; use std::collections::HashMap; use std::fmt::{Display, Formatter}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; -use std::sync::Arc; -use std::time::{Duration, SystemTime}; -use tokio::sync::RwLock; -use tokio::time::Instant; +use std::time::Duration; use tracing::{trace, warn}; +mod compat; + +#[cfg(test)] +use mock_instant::thread_local::Instant; +#[cfg(not(test))] +use std::time::Instant; + // helper to convert peer's allocation into an `IpPair` pub fn allocated_ip_pair(peer: &Peer) -> Option { for allowed_ip in &peer.allowed_ips { @@ -57,17 +59,38 @@ impl Display for IpPair { pub enum AllocationState { /// IP is available for allocation Free, - /// IP is allocated and in use, with timestamp of allocation - Allocated(SystemTime), + + /// The IP has been pre-allocated for a peer, but the corresponding registration has not yet been finalised + PreAllocated { allocated_at: Instant }, + + /// IP is allocated and in use, with timestamp + Allocated { allocated_at: Instant }, +} + +impl AllocationState { + pub fn is_free(&self) -> bool { + matches!(self, AllocationState::Free) + } + + pub fn new_pre_allocated() -> Self { + AllocationState::PreAllocated { + allocated_at: Instant::now(), + } + } + + pub fn new_allocated() -> Self { + AllocationState::Allocated { + allocated_at: Instant::now(), + } + } } /// Thread-safe IP address pool manager /// /// Manages allocation of IPv4/IPv6 address pairs from configured CIDR ranges. /// Ensures collision-free allocation and supports stale cleanup. -#[derive(Clone)] pub struct IpPool { - allocations: Arc>>, + allocations: HashMap, } impl IpPool { @@ -98,7 +121,7 @@ impl IpPool { .iter() .filter_map(|ip| { if let IpAddr::V4(v4) = ip { - Some(v4) + if v4 != ipv4_network { Some(v4) } else { None } } else { None } @@ -109,7 +132,7 @@ impl IpPool { .iter() .filter_map(|ip| { if let IpAddr::V6(v6) = ip { - Some(v6) + if v6 != ipv6_network { Some(v6) } else { None } } else { None } @@ -129,21 +152,18 @@ impl IpPool { allocations.len(), ); - Ok(IpPool { - allocations: Arc::new(RwLock::new(allocations)), - }) + Ok(IpPool { allocations }) } - /// Allocate a free IP pair from the pool + /// Preallocate a free IP pair from the pool /// /// # Errors /// Returns `IpPoolError::NoFreeIp` if no IPs are available - pub async fn allocate(&self) -> Result { - let mut pool = self.allocations.write().await; - + pub fn pre_allocate(&mut self) -> Result { // Find a free IP and allocate it let assignment_start = Instant::now(); - let free_ip = pool + let free_ip = self + .allocations .iter_mut() .filter(|(_, state)| matches!(state, AllocationState::Free)) .choose(&mut rand::thread_rng()) @@ -155,18 +175,42 @@ impl IpPool { } let ip_pair = *free_ip.0; - *free_ip.1 = AllocationState::Allocated(SystemTime::now()); + *free_ip.1 = AllocationState::new_pre_allocated(); tracing::debug!("Allocated IP pair: {ip_pair}"); Ok(ip_pair) } + pub fn confirm_allocation(&mut self, ip_pair: IpPair) -> Result<(), IpPoolError> { + let Some(allocation) = self.allocations.get_mut(&ip_pair) else { + return Err(IpPoolError::UnknownIpPair { ip_pair }); + }; + match allocation { + AllocationState::Free => { + // seems the IpPair has been released before the confirmation, but it has not yet been re-allocated + warn!( + "{ip_pair} seems to have already been released, but has not been allocated to a new peer yet" + ); + *allocation = AllocationState::Allocated { + allocated_at: Instant::now(), + }; + Ok(()) + } + AllocationState::PreAllocated { allocated_at } => { + *allocation = AllocationState::Allocated { + allocated_at: *allocated_at, + }; + Ok(()) + } + AllocationState::Allocated { .. } => Err(IpPoolError::AlreadyUsed { ip_pair }), + } + } + /// Release an IP pair back to the pool /// /// Marks the IP as free for future allocations. - pub async fn release(&self, ip_pair: IpPair) { - let mut pool = self.allocations.write().await; - if let Some(state) = pool.get_mut(&ip_pair) { + pub fn release(&mut self, ip_pair: IpPair) { + if let Some(state) = self.allocations.get_mut(&ip_pair) { *state = AllocationState::Free; tracing::debug!("Released IP pair: {ip_pair}"); } @@ -175,58 +219,67 @@ impl IpPool { /// Mark an IP pair as allocated (used during initialization from database) /// /// This is used when restoring state from the database on gateway startup. - pub async fn mark_used(&self, ip_pair: IpPair) { - let mut pool = self.allocations.write().await; - if let Some(state) = pool.get_mut(&ip_pair) { - *state = AllocationState::Allocated(SystemTime::now()); - tracing::debug!("Marked IP pair as used: {ip_pair}"); - } else { - tracing::warn!("Attempted to mark unknown IP pair as used: {ip_pair}"); + pub fn mark_used(&mut self, ip_pair: IpPair) -> Result<(), IpPoolError> { + let Some(state) = self.allocations.get_mut(&ip_pair) else { + return Err(IpPoolError::UnknownIpPair { ip_pair }); + }; + + if !state.is_free() { + return Err(IpPoolError::AlreadyUsed { ip_pair }); } + tracing::debug!("Marked IP pair as used: {ip_pair}"); + *state = AllocationState::new_allocated(); + Ok(()) } /// Get the number of free IPs in the pool - pub async fn free_count(&self) -> usize { - let pool = self.allocations.read().await; - pool.iter() + pub fn free_count(&self) -> usize { + self.allocations + .iter() .filter(|(_, state)| matches!(state, AllocationState::Free)) .count() } /// Get the number of allocated IPs in the pool - pub async fn allocated_count(&self) -> usize { - let pool = self.allocations.read().await; - pool.iter() - .filter(|(_, state)| matches!(state, AllocationState::Allocated(_))) + pub fn pre_allocated_count(&self) -> usize { + self.allocations + .iter() + .filter(|(_, state)| matches!(state, AllocationState::PreAllocated { .. })) + .count() + } + + /// Get the number of allocated IPs in the pool + pub fn allocated_count(&self) -> usize { + self.allocations + .iter() + .filter(|(_, state)| matches!(state, AllocationState::Allocated { .. })) .count() } /// Get the total pool size - pub async fn total_count(&self) -> usize { - let pool = self.allocations.read().await; - pool.len() + pub fn total_count(&self) -> usize { + self.allocations.len() } /// Clean up stale allocations older than the specified duration /// /// Returns the number of IPs that were freed - pub async fn cleanup_stale(&self, max_age: std::time::Duration) -> usize { - let mut pool = self.allocations.write().await; - let now = SystemTime::now(); + pub fn cleanup_stale(&mut self, max_age: Duration) -> usize { + let now = Instant::now(); let mut freed = 0; - for (_ip, state) in pool.iter_mut() { - if let AllocationState::Allocated(allocated_at) = state - && let Ok(age) = now.duration_since(*allocated_at) - && age > max_age - { - *state = AllocationState::Free; - freed += 1; + for state in self.allocations.values_mut() { + if let AllocationState::PreAllocated { allocated_at, .. } = state { + let age = now.duration_since(*allocated_at); + if age > max_age { + *state = AllocationState::Free; + freed += 1; + } } } if freed > 0 { - tracing::info!("Cleaned up {} stale IP allocations", freed); + tracing::info!("Cleaned up {freed} stale IP allocations"); } freed @@ -234,11 +287,243 @@ impl IpPool { } /// Errors that can occur during IP pool operations -#[derive(Debug, thiserror::Error)] +#[derive(Debug, thiserror::Error, PartialEq, Eq)] pub enum IpPoolError { #[error("No free IP addresses available in pool")] NoFreeIp, + #[error("Attempted to mark an IpPair that is already in used: {ip_pair}")] + AlreadyUsed { ip_pair: IpPair }, + + #[error("Attempted to mark an unknown ip pair: {ip_pair}")] + UnknownIpPair { ip_pair: IpPair }, + #[error("Invalid IP network configuration: {0}")] InvalidNetwork(#[from] ipnetwork::IpNetworkError), } + +#[cfg(test)] +mod tests { + use super::*; + use anyhow::bail; + use mock_instant::thread_local::MockClock; + + // 3 addresses in each pool + fn small_ip_pool() -> IpPool { + IpPool::new( + Ipv4Addr::new(10, 0, 0, 0), + 30, + Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 0), + 126, + ) + .unwrap() + } + + #[test] + fn ip_pool_initial_allocation() -> anyhow::Result<()> { + let base_ipv4_network = Ipv4Addr::new(10, 0, 0, 0); + let base_ipv4_prefix = 24; // 255 addresses + let base_ipv6_network = Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 0); + let base_ipv6_prefix = 112; // 65535 addresses + + // ipv4 pool size < ipv6 pool size + let base_ip_pool = IpPool::new( + base_ipv4_network, + base_ipv4_prefix, + base_ipv6_network, + base_ipv6_prefix, + )?; + let inner = base_ip_pool.allocations; + // minimum of ipv4 and ipv6 allocations + assert_eq!(inner.len(), 255); + + // no ipv4 addresses + let base_ip_pool = IpPool::new(base_ipv4_network, 32, base_ipv6_network, base_ipv6_prefix)?; + let inner = base_ip_pool.allocations; + assert_eq!(inner.len(), 0); + + // no ipv6 addresses + let base_ip_pool = + IpPool::new(base_ipv4_network, base_ipv4_prefix, base_ipv6_network, 128)?; + let inner = base_ip_pool.allocations; + assert_eq!(inner.len(), 0); + + // ipv4 pool size == ipv6 pool size + let base_ip_pool = IpPool::new(base_ipv4_network, 16, base_ipv6_network, base_ipv6_prefix)?; + let inner = base_ip_pool.allocations; + assert_eq!(inner.len(), 65535); + + // ipv4 pool size > ipv6 pool size + let base_ip_pool = IpPool::new(base_ipv4_network, 12, base_ipv6_network, base_ipv6_prefix)?; + let inner = base_ip_pool.allocations; + assert_eq!(inner.len(), 65535); + + Ok(()) + } + + fn ensure_different_allocation(left: IpPair, right: IpPair) -> anyhow::Result<()> { + if left.ipv4 == right.ipv4 || left.ipv6 == right.ipv6 { + bail!("ip allocation overlap") + } + + Ok(()) + } + + #[test] + fn ip_pool_allocation() -> anyhow::Result<()> { + let mut pool = small_ip_pool(); + assert_eq!(pool.allocations.len(), 3); + + let gateway_pair = IpPair { + ipv4: Ipv4Addr::new(10, 0, 0, 0), + ipv6: Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 0), + }; + + assert_eq!(pool.free_count(), 3); + assert_eq!(pool.pre_allocated_count(), 0); + + let allocation1 = pool.pre_allocate()?; + assert_eq!(pool.free_count(), 2); + assert_eq!(pool.pre_allocated_count(), 1); + + let allocation2 = pool.pre_allocate()?; + assert_eq!(pool.free_count(), 1); + assert_eq!(pool.pre_allocated_count(), 2); + + pool.confirm_allocation(allocation1)?; + assert_eq!(pool.free_count(), 1); + assert_eq!(pool.pre_allocated_count(), 1); + assert_eq!(pool.allocated_count(), 1); + + let allocation3 = pool.pre_allocate()?; + assert_eq!(pool.free_count(), 0); + assert_eq!(pool.pre_allocated_count(), 2); + assert_eq!(pool.allocated_count(), 1); + + // make sure each was unique and different from the gateway + ensure_different_allocation(allocation1, allocation2)?; + ensure_different_allocation(allocation1, allocation3)?; + ensure_different_allocation(allocation2, allocation3)?; + + ensure_different_allocation(allocation1, gateway_pair)?; + ensure_different_allocation(allocation2, gateway_pair)?; + ensure_different_allocation(allocation3, gateway_pair)?; + + // allocation 4 will fail as we have run out of addresses + assert_eq!(pool.pre_allocate().unwrap_err(), IpPoolError::NoFreeIp); + + // if pair gets released, it's eligible for allocation again + pool.release(allocation2); + assert_eq!(pool.free_count(), 1); + assert_eq!(pool.pre_allocated_count(), 1); + assert_eq!(pool.allocated_count(), 1); + + let reallocation = pool.pre_allocate()?; + assert_eq!(reallocation, allocation2); + + assert_eq!(pool.free_count(), 0); + assert_eq!(pool.pre_allocated_count(), 2); + assert_eq!(pool.allocated_count(), 1); + + Ok(()) + } + + #[test] + fn ip_pool_mark_used() -> anyhow::Result<()> { + let mut pool = small_ip_pool(); + + let pair1 = IpPair::new( + Ipv4Addr::new(10, 0, 0, 1), + Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 1), + ); + let pair2 = IpPair::new( + Ipv4Addr::new(10, 0, 0, 2), + Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 2), + ); + let pair3 = IpPair::new( + Ipv4Addr::new(10, 0, 0, 3), + Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 3), + ); + + let bad_pair1 = IpPair::new( + Ipv4Addr::new(10, 0, 0, 1), + Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 4), + ); + let bad_pair2 = IpPair::new( + Ipv4Addr::new(10, 0, 0, 4), + Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 1), + ); + + assert!(pool.mark_used(pair1,).is_ok()); + assert_eq!( + pool.mark_used(pair1).unwrap_err(), + IpPoolError::AlreadyUsed { ip_pair: pair1 } + ); + + assert!(pool.mark_used(pair2).is_ok()); + assert!(pool.mark_used(pair3).is_ok()); + + assert_eq!( + pool.mark_used(bad_pair1).unwrap_err(), + IpPoolError::UnknownIpPair { ip_pair: bad_pair1 } + ); + + assert_eq!( + pool.mark_used(bad_pair2,).unwrap_err(), + IpPoolError::UnknownIpPair { ip_pair: bad_pair2 } + ); + + Ok(()) + } + + #[test] + fn ip_pool_cleanup() -> anyhow::Result<()> { + MockClock::set_time(Duration::ZERO); + + let mut pool = small_ip_pool(); + + let age_threshold = Duration::from_secs(1); + + // nothing to cleanup + assert_eq!(pool.cleanup_stale(age_threshold), 0); + + // just allocated + let pair1 = pool.pre_allocate()?; + let pair2 = pool.pre_allocate()?; + assert_eq!(pool.cleanup_stale(age_threshold), 0); + + // advance time to go beyond the allocation threshold + MockClock::advance(Duration::from_millis(1001)); + assert_eq!(pool.cleanup_stale(age_threshold), 2); + + // ensure those pairs are now marked as free + assert!(pool.allocations.get(&pair1).unwrap().is_free()); + assert!(pool.allocations.get(&pair2).unwrap().is_free()); + + pool.pre_allocate()?; + MockClock::advance(Duration::from_millis(500)); + pool.pre_allocate()?; + + assert_eq!(pool.cleanup_stale(age_threshold), 0); + MockClock::advance(Duration::from_millis(501)); + assert_eq!(pool.cleanup_stale(age_threshold), 1); + + MockClock::advance(Duration::from_millis(500)); + assert_eq!(pool.cleanup_stale(age_threshold), 1); + + let mut new_pool = small_ip_pool(); + let pair1 = new_pool.pre_allocate()?; + let pair2 = new_pool.pre_allocate()?; + + // complete allocation for pair2 + new_pool.confirm_allocation(pair2)?; + MockClock::advance(Duration::from_millis(2000)); + + // only pair1 should have got cleaned up + assert_eq!(new_pool.cleanup_stale(age_threshold), 1); + assert!(new_pool.allocations.get(&pair1).unwrap().is_free()); + assert!(!new_pool.allocations.get(&pair2).unwrap().is_free()); + + Ok(()) + } +} diff --git a/common/wireguard/src/lib.rs b/common/wireguard/src/lib.rs index e73da973a2..e64ab3b9e2 100644 --- a/common/wireguard/src/lib.rs +++ b/common/wireguard/src/lib.rs @@ -284,7 +284,7 @@ pub async fn start_wireguard( // Initialize IP pool from configuration info!("Initializing IP pool for WireGuard peer allocation"); - let ip_pool = IpPool::new( + let mut ip_pool = IpPool::new( wireguard_data.inner.config().private_ipv4, wireguard_data.inner.config().private_network_prefix_v4, wireguard_data.inner.config().private_ipv6, @@ -294,7 +294,7 @@ pub async fn start_wireguard( // Mark existing peer IPs as used in the pool for peer in &peers { if let Some(ip_pair) = crate::ip_pool::allocated_ip_pair(peer) { - ip_pool.mark_used(ip_pair).await; + ip_pool.mark_used(ip_pair)?; } } diff --git a/common/wireguard/src/peer_controller/mock.rs b/common/wireguard/src/peer_controller/mock.rs index 89f2821b31..37f9ba1f0a 100644 --- a/common/wireguard/src/peer_controller/mock.rs +++ b/common/wireguard/src/peer_controller/mock.rs @@ -75,6 +75,7 @@ pub enum PeerControlRequestType { ReleaseIpPair { ip_pair: IpPair }, RemovePeer { key: KeyWrapper }, QueryPeer { key: KeyWrapper }, + CheckActivePeer { key: KeyWrapper }, GetClientBandwidthByKey { key: KeyWrapper }, GetClientBandwidthByIp { ip: IpAddr }, GetVerifierByKey { key: KeyWrapper }, @@ -93,6 +94,7 @@ impl PeerControlRequestType { PeerControlRequestType::GetClientBandwidthByIp { .. } => None, PeerControlRequestType::GetVerifierByKey { key } => Some(key.clone()), PeerControlRequestType::GetVerifierByIp { .. } => None, + PeerControlRequestType::CheckActivePeer { key } => Some(key.clone()), } } @@ -107,7 +109,7 @@ impl From<&PeerControlRequest> for PeerControlRequestType { PeerControlRequest::AddPeer { peer, .. } => PeerControlRequestType::AddPeer { public_key: (&peer.public_key).into(), }, - PeerControlRequest::AllocatePeerIpPair { .. } => { + PeerControlRequest::PreAllocateIpPair { .. } => { PeerControlRequestType::AllocatePeerIpPair {} } PeerControlRequest::ReleaseIpPair { ip_pair, .. } => { @@ -131,6 +133,9 @@ impl From<&PeerControlRequest> for PeerControlRequestType { PeerControlRequest::GetVerifierByIp { ip, .. } => { PeerControlRequestType::GetVerifierByIp { ip: *ip } } + PeerControlRequest::CheckActivePeer { key, .. } => { + PeerControlRequestType::CheckActivePeer { key: key.into() } + } } } } @@ -266,7 +271,7 @@ impl MockPeerController { } response_tx.send_downcasted(response.content) } - PeerControlRequest::AllocatePeerIpPair { response_tx, .. } => { + PeerControlRequest::PreAllocateIpPair { response_tx, .. } => { response_tx.send_downcasted(response.content) } PeerControlRequest::ReleaseIpPair { @@ -295,6 +300,9 @@ impl MockPeerController { PeerControlRequest::GetVerifierByIp { response_tx, .. } => { response_tx.send_downcasted(response.content) } + PeerControlRequest::CheckActivePeer { response_tx, .. } => { + response_tx.send_downcasted(response.content) + } } } diff --git a/common/wireguard/src/peer_controller/mod.rs b/common/wireguard/src/peer_controller/mod.rs index e8ba3c4078..9aae772a8f 100644 --- a/common/wireguard/src/peer_controller/mod.rs +++ b/common/wireguard/src/peer_controller/mod.rs @@ -77,7 +77,7 @@ pub enum PeerControlRequest { response_tx: oneshot::Sender, }, /// Attempt to allocate an IP pair from the pool - AllocatePeerIpPair { + PreAllocateIpPair { response_tx: oneshot::Sender, }, /// Attempt to return an IP pair back to the pool @@ -93,6 +93,10 @@ pub enum PeerControlRequest { key: Key, response_tx: oneshot::Sender, }, + CheckActivePeer { + key: Key, + response_tx: oneshot::Sender, + }, GetClientBandwidthByKey { key: Key, response_tx: oneshot::Sender, @@ -118,6 +122,7 @@ pub type AllocatePeerControlResponse = Result; pub type ReleaseIpPairControlResponse = Result<()>; pub type RemovePeerControlResponse = Result<()>; pub type QueryPeerControlResponse = Result>; +pub type CheckActivePeerResponse = Result; pub type GetClientBandwidthControlResponse = Result; pub type QueryVerifierControlResponse = Result>; @@ -216,7 +221,7 @@ impl PeerController { if let Ok(Some(peer)) = self.handle_query_peer_by_key(key).await && let Some(ip_pair) = allocated_ip_pair(&peer) { - self.ip_pool.release(ip_pair).await + self.ip_pool.release(ip_pair) } let ret = self.wg_api.remove_peer(key); @@ -258,6 +263,13 @@ impl PeerController { async fn handle_add_request(&mut self, peer: &Peer) -> Result<()> { nym_metrics::inc!("wg_peer_addition_attempts"); + // confirm ip allocation so that it wouldn't be released for as long as the peer exists + let Some(ip_pair) = allocated_ip_pair(peer) else { + return Err(Error::Internal( + "could not determine ip pair allocated to the peer".to_string(), + )); + }; + // Try to configure WireGuard peer if let Err(e) = self.wg_api.configure_peer(peer) { nym_metrics::inc!("wg_peer_addition_failed"); @@ -289,6 +301,9 @@ impl PeerController { *self.host_information.write().await = host_information; } let public_key = peer.public_key.clone(); + + self.ip_pool.confirm_allocation(ip_pair)?; + tokio::spawn(async move { handle.run().await; debug!("Peer handle shut down for {public_key}"); @@ -302,15 +317,11 @@ impl PeerController { /// /// This only allocates IPs - the caller must handle database storage and /// then call AddPeer with a complete Peer struct. - async fn handle_ip_allocation_request(&mut self) -> Result { + fn handle_ip_allocation_request(&mut self) -> Result { nym_metrics::inc!("wg_ip_allocation_attempts"); // Allocate IP pair from pool - let ip_pair = self - .ip_pool - .allocate() - .await - .map_err(|e| Error::IpPool(e.to_string()))?; + let ip_pair = self.ip_pool.pre_allocate()?; nym_metrics::inc!("wg_ip_allocation_success"); tracing::debug!("Allocated IP pair: {ip_pair}"); @@ -319,8 +330,8 @@ impl PeerController { } /// Return IP pair back to the pool - async fn handle_ip_release(&mut self, ip_pair: IpPair) { - self.ip_pool.release(ip_pair).await + fn handle_ip_release(&mut self, ip_pair: IpPair) { + self.ip_pool.release(ip_pair) } async fn ip_to_key(&self, ip: IpAddr) -> Result> { @@ -359,6 +370,12 @@ impl PeerController { .client_bandwidth()) } + fn check_active_peer(&self, key: Key) -> Result { + // peer is active as long as we still have an entry inside the bandwidth storage manager, + // as it is removed upon peer removal + Ok(self.bw_storage_managers.contains_key(&key)) + } + async fn handle_get_client_bandwidth_by_ip(&self, ip: IpAddr) -> Result { let Some(key) = self.ip_to_key(ip).await? else { return Err(Error::MissingClientKernelEntry(ip.to_string())); @@ -492,16 +509,14 @@ impl PeerController { PeerControlRequest::AddPeer { peer, response_tx } => { response_tx.send(self.handle_add_request(&peer).await).ok(); } - PeerControlRequest::AllocatePeerIpPair { response_tx } => { - response_tx - .send(self.handle_ip_allocation_request().await) - .ok(); + PeerControlRequest::PreAllocateIpPair { response_tx } => { + response_tx.send(self.handle_ip_allocation_request()).ok(); } PeerControlRequest::ReleaseIpPair { response_tx, ip_pair, } => { - self.handle_ip_release(ip_pair).await; + self.handle_ip_release(ip_pair); response_tx.send(Ok(())).ok(); } PeerControlRequest::RemovePeer { key, response_tx } => { @@ -540,6 +555,9 @@ impl PeerController { .send(self.handle_query_verifier_by_ip(ip, *credential).await) .ok(); } + PeerControlRequest::CheckActivePeer { key, response_tx } => { + response_tx.send(self.check_active_peer(key)).ok(); + } } } @@ -558,7 +576,7 @@ impl PeerController { } _ = self.ip_cleanup_interval.next() => { // Periodically cleanup stale IP allocations - let freed = self.ip_pool.cleanup_stale(DEFAULT_IP_STALE_AGE).await; + let freed = self.ip_pool.cleanup_stale(DEFAULT_IP_STALE_AGE); if freed > 0 { nym_metrics::inc_by!("wg_stale_ips_cleaned", freed as u64); info!("Cleaned up {} stale IP allocations", freed); diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml index 8f729975e5..e912053e34 100644 --- a/gateway/Cargo.toml +++ b/gateway/Cargo.toml @@ -89,9 +89,11 @@ bytes = { workspace = true } defguard_wireguard_rs = { workspace = true } [dev-dependencies] +anyhow = { workspace = true } +nym-test-utils = { workspace = true } nym-gateway-storage = { workspace = true, features = ["mock"] } nym-wireguard = { workspace = true, features = ["mock"] } -mock_instant = "0.6.0" +mock_instant = { workspace = true } time = { workspace = true } [lints] diff --git a/gateway/src/node/internal_service_providers/authenticator/mixnet_listener.rs b/gateway/src/node/internal_service_providers/authenticator/mixnet_listener.rs index 3607ff7926..d9591e0090 100644 --- a/gateway/src/node/internal_service_providers/authenticator/mixnet_listener.rs +++ b/gateway/src/node/internal_service_providers/authenticator/mixnet_listener.rs @@ -4,29 +4,15 @@ use crate::node::internal_service_providers::authenticator::{ config::Config, error::AuthenticatorError, seen_credential_cache::SeenCredentialCache, }; -use crate::node::wireguard::PeerManager; -use defguard_wireguard_rs::net::IpAddrMask; -use defguard_wireguard_rs::{host::Peer, key::Key}; +use crate::node::wireguard::{PeerManager, PeerRegistrator}; use futures::StreamExt; -use nym_authenticator_requests::models::BandwidthClaim; use nym_authenticator_requests::traits::UpgradeModeMessage; -use nym_authenticator_requests::{latest, v4::registration::IpPair}; use nym_authenticator_requests::{ - latest::registration::{GatewayClient, PendingRegistrations, PrivateIPs}, request::AuthenticatorRequest, traits::{FinalMessage, InitMessage, QueryBandwidthMessage, TopUpMessage}, v1, v2, v3, v4, v5, v6, AuthenticatorVersion, CURRENT_VERSION, }; -use nym_credential_verification::ecash::traits::EcashManager; use nym_credential_verification::upgrade_mode::UpgradeModeDetails; -use nym_credential_verification::{ - bandwidth_storage_manager::BandwidthStorageManager, BandwidthFlushingBehaviourConfig, - ClientBandwidth, CredentialVerifier, -}; -use nym_credentials_interface::{BandwidthCredential, CredentialSpendingData}; -use nym_crypto::asymmetric::x25519::KeyPair; -use nym_gateway_requests::models::CredentialSpendingRequest; -use nym_gateway_storage::models::PersistedBandwidth; use nym_sdk::mixnet::{ AnonymousSenderTag, InputMessage, MixnetMessageSender, Recipient, TransmissionLane, }; @@ -36,50 +22,28 @@ use nym_task::ShutdownToken; use nym_wireguard::WireguardGatewayData; use nym_wireguard_types::PeerPublicKey; use std::cmp::max; -use std::{ - net::IpAddr, - sync::Arc, - time::{Duration, SystemTime}, -}; -use tokio::sync::RwLock; +use std::time::Duration; use tokio_stream::wrappers::IntervalStream; type AuthenticatorHandleResult = Result<(Vec, Option), AuthenticatorError>; -const DEFAULT_REGISTRATION_TIMEOUT_CHECK: Duration = Duration::from_secs(60); // 1 minute +const DEFAULT_CREDENTIAL_TIMEOUT_CHECK: Duration = Duration::from_secs(60); // 1 minute // we need to be above MINIMUM_REMAINING_BANDWIDTH (500MB) plus we also have to trick the client // its depletion is low enough to not require sending new tickets const DEFAULT_WG_CLIENT_BANDWIDTH_THRESHOLD: i64 = 1024 * 1024 * 1024; -pub(crate) struct RegisteredAndFree { - registration_in_progress: PendingRegistrations, - taken_private_network_ips: PrivateIPs, -} - -impl RegisteredAndFree { - pub(crate) fn new() -> Self { - RegisteredAndFree { - registration_in_progress: Default::default(), - taken_private_network_ips: Default::default(), - } - } -} - pub(crate) struct MixnetListener { // The configuration for the mixnet listener - pub(crate) config: Config, + pub(crate) _config: Config, // The mixnet client that we use to send and receive packets from the mixnet pub(crate) mixnet_client: nym_sdk::mixnet::MixnetClient, - // Registrations awaiting confirmation - pub(crate) registered_and_free: RwLock, - pub(crate) peer_manager: PeerManager, pub(crate) upgrade_mode: UpgradeModeDetails, - pub(crate) ecash_verifier: Arc, + pub(crate) peer_registrator: PeerRegistrator, pub(crate) timeout_check_interval: IntervalStream, @@ -91,18 +55,17 @@ impl MixnetListener { config: Config, wireguard_gateway_data: WireguardGatewayData, mixnet_client: nym_sdk::mixnet::MixnetClient, + peer_registrator: PeerRegistrator, upgrade_mode: UpgradeModeDetails, - ecash_verifier: Arc, ) -> Self { let timeout_check_interval = - IntervalStream::new(tokio::time::interval(DEFAULT_REGISTRATION_TIMEOUT_CHECK)); + IntervalStream::new(tokio::time::interval(DEFAULT_CREDENTIAL_TIMEOUT_CHECK)); MixnetListener { - config, + _config: config, mixnet_client, - registered_and_free: RwLock::new(RegisteredAndFree::new()), peer_manager: PeerManager::new(wireguard_gateway_data), upgrade_mode, - ecash_verifier, + peer_registrator, timeout_check_interval, seen_credential_cache: SeenCredentialCache::new(), } @@ -112,10 +75,6 @@ impl MixnetListener { self.upgrade_mode.enabled() } - fn keypair(&self) -> &Arc { - self.peer_manager.wireguard_gateway_data.keypair() - } - async fn upgrade_mode_bandwidth(&self, peer: PeerPublicKey) -> Result { // if we're undergoing upgrade mode, we don't meter bandwidth, // we simply return MAX of clients current bandwidth and minimum bandwidth before default @@ -129,47 +88,6 @@ impl MixnetListener { )) } - async fn remove_stale_registrations(&self) -> Result<(), AuthenticatorError> { - let mut registered_and_free = self.registered_and_free.write().await; - let registered_values: Vec<_> = registered_and_free - .registration_in_progress - .values() - .cloned() - .collect(); - for reg in registered_values { - let timestamp = registered_and_free - .taken_private_network_ips - .get_mut(®.gateway_data.private_ips) - .ok_or(AuthenticatorError::InternalDataCorruption(format!( - "IPs {} should be present", - reg.gateway_data.private_ips - )))?; - - let duration = SystemTime::now().duration_since(*timestamp).map_err(|_| { - AuthenticatorError::InternalDataCorruption( - "set timestamp shouldn't have been set in the future".to_string(), - ) - })?; - if duration > DEFAULT_REGISTRATION_TIMEOUT_CHECK { - registered_and_free - .registration_in_progress - .remove(®.gateway_data.pub_key()); - registered_and_free - .taken_private_network_ips - .remove(®.gateway_data.private_ips); - self.peer_manager - .release_ip_pair(reg.gateway_data.private_ips.into()) - .await?; - - tracing::debug!( - "Removed stale registration of {}", - reg.gateway_data.pub_key() - ); - } - } - Ok(()) - } - async fn on_initial_request( &mut self, init_message: Box, @@ -177,352 +95,12 @@ impl MixnetListener { request_id: u64, reply_to: Option, ) -> AuthenticatorHandleResult { - let remote_public = init_message.pub_key(); - let nonce: u64 = fastrand::u64(..); - let mut registered_and_free = self.registered_and_free.write().await; - if let Some(registration_data) = registered_and_free - .registration_in_progress - .get(&remote_public) - { - let gateway_data = registration_data.gateway_data.clone(); - let bytes = match AuthenticatorVersion::from(protocol) { - AuthenticatorVersion::V1 => { - v1::response::AuthenticatorResponse::new_pending_registration_success( - v1::registration::RegistrationData { - nonce: registration_data.nonce, - gateway_data: v1::GatewayClient { - pub_key: gateway_data.pub_key, - private_ip: gateway_data.private_ips.ipv4.into(), - mac: v1::ClientMac::new(gateway_data.mac.to_vec()), - }, - wg_port: registration_data.wg_port, - }, - request_id, - reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?, - ) - .to_bytes() - .map_err(AuthenticatorError::response_serialisation)? - } - AuthenticatorVersion::V2 => { - v2::response::AuthenticatorResponse::new_pending_registration_success( - v2::registration::RegistrationData { - nonce: registration_data.nonce, - gateway_data: v2::registration::GatewayClient::new( - self.keypair().private_key(), - remote_public.inner(), - registration_data.gateway_data.private_ips.ipv4.into(), - registration_data.nonce, - ), - wg_port: registration_data.wg_port, - }, - request_id, - reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?, - ) - .to_bytes() - .map_err(AuthenticatorError::response_serialisation)? - } - AuthenticatorVersion::V3 => { - v3::response::AuthenticatorResponse::new_pending_registration_success( - v3::registration::RegistrationData { - nonce: registration_data.nonce, - gateway_data: v3::registration::GatewayClient::new( - self.keypair().private_key(), - remote_public.inner(), - registration_data.gateway_data.private_ips.ipv4.into(), - registration_data.nonce, - ), - wg_port: registration_data.wg_port, - }, - request_id, - reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?, - ) - .to_bytes() - .map_err(AuthenticatorError::response_serialisation)? - } - AuthenticatorVersion::V4 => { - v4::response::AuthenticatorResponse::new_pending_registration_success( - v4::registration::RegistrationData { - nonce: registration_data.nonce, - // convert current to v5 and then v5 to v4 (current as of 28.08.25) - gateway_data: v5::registration::GatewayClient::from( - registration_data.gateway_data.clone(), - ) - .into(), - wg_port: registration_data.wg_port, - }, - request_id, - reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?, - ) - .to_bytes() - .map_err(AuthenticatorError::response_serialisation)? - } - AuthenticatorVersion::V5 => { - v5::response::AuthenticatorResponse::new_pending_registration_success( - v5::registration::RegistrationData { - nonce: registration_data.nonce, - gateway_data: registration_data.gateway_data.clone().into(), - wg_port: registration_data.wg_port, - }, - request_id, - ) - .to_bytes() - .map_err(AuthenticatorError::response_serialisation)? - } - AuthenticatorVersion::V6 => { - v6::response::AuthenticatorResponse::new_pending_registration_success( - v6::registration::RegistrationData { - nonce: registration_data.nonce, - gateway_data: registration_data.gateway_data.clone(), - wg_port: registration_data.wg_port, - }, - request_id, - self.upgrade_mode_enabled(), - ) - .to_bytes() - .map_err(AuthenticatorError::response_serialisation)? - } - AuthenticatorVersion::UNKNOWN => return Err(AuthenticatorError::UnknownVersion), - }; - return Ok((bytes, reply_to)); - } + let response = self + .peer_registrator + .on_initial_authenticator_request(init_message, protocol, request_id, reply_to) + .await?; - let peer = self.peer_manager.query_peer(remote_public).await?; - if let Some(peer) = peer { - let allowed_ipv4 = peer - .allowed_ips - .iter() - .find_map(|ip_mask| match ip_mask.address { - IpAddr::V4(ipv4_addr) => Some(ipv4_addr), - _ => None, - }) - .ok_or(AuthenticatorError::InternalError( - "there should be one private IPv4 in the list".to_string(), - ))?; - let allowed_ipv6 = peer - .allowed_ips - .iter() - .find_map(|ip_mask| match ip_mask.address { - IpAddr::V6(ipv6_addr) => Some(ipv6_addr), - _ => None, - }) - .unwrap_or(IpPair::from(IpAddr::from(allowed_ipv4)).ipv6); - let bytes = match AuthenticatorVersion::from(protocol) { - AuthenticatorVersion::V1 => v1::response::AuthenticatorResponse::new_registered( - v1::registration::RegisteredData { - pub_key: self.keypair().public_key().into(), - private_ip: allowed_ipv4.into(), - wg_port: self.config.authenticator.tunnel_announced_port, - }, - reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?, - request_id, - ) - .to_bytes() - .map_err(AuthenticatorError::response_serialisation)?, - AuthenticatorVersion::V2 => v2::response::AuthenticatorResponse::new_registered( - v2::registration::RegisteredData { - pub_key: self.keypair().public_key().into(), - private_ip: allowed_ipv4.into(), - wg_port: self.config.authenticator.tunnel_announced_port, - }, - reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?, - request_id, - ) - .to_bytes() - .map_err(AuthenticatorError::response_serialisation)?, - AuthenticatorVersion::V3 => v3::response::AuthenticatorResponse::new_registered( - v3::registration::RegisteredData { - pub_key: self.keypair().public_key().into(), - private_ip: allowed_ipv4.into(), - wg_port: self.config.authenticator.tunnel_announced_port, - }, - reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?, - request_id, - ) - .to_bytes() - .map_err(AuthenticatorError::response_serialisation)?, - AuthenticatorVersion::V4 => v4::response::AuthenticatorResponse::new_registered( - v4::registration::RegisteredData { - pub_key: self.keypair().public_key().into(), - private_ips: (allowed_ipv4, allowed_ipv6).into(), - wg_port: self.config.authenticator.tunnel_announced_port, - }, - reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?, - request_id, - ) - .to_bytes() - .map_err(AuthenticatorError::response_serialisation)?, - AuthenticatorVersion::V5 => v5::response::AuthenticatorResponse::new_registered( - v5::registration::RegisteredData { - pub_key: self.keypair().public_key().into(), - private_ips: (allowed_ipv4, allowed_ipv6).into(), - wg_port: self.config.authenticator.tunnel_announced_port, - }, - request_id, - ) - .to_bytes() - .map_err(AuthenticatorError::response_serialisation)?, - AuthenticatorVersion::V6 => v6::response::AuthenticatorResponse::new_registered( - v6::registration::RegisteredData { - pub_key: self.keypair().public_key().into(), - private_ips: (allowed_ipv4, allowed_ipv6).into(), - wg_port: self.config.authenticator.tunnel_announced_port, - }, - request_id, - self.upgrade_mode_enabled(), - ) - .to_bytes() - .map_err(AuthenticatorError::response_serialisation)?, - AuthenticatorVersion::UNKNOWN => return Err(AuthenticatorError::UnknownVersion), - }; - return Ok((bytes, reply_to)); - } - - // mark it as used, even though it's not final - let ip_allocation = self.peer_manager.allocate_peer_ip_pair().await?; - self.registered_and_free - .write() - .await - .taken_private_network_ips - .insert(ip_allocation.into(), SystemTime::now()); - - let gateway_data = GatewayClient::new( - self.keypair().private_key(), - remote_public.inner(), - ip_allocation.into(), - nonce, - ); - let registration_data = latest::registration::RegistrationData { - nonce, - gateway_data: gateway_data.clone(), - wg_port: self.config.authenticator.tunnel_announced_port, - }; - registered_and_free - .registration_in_progress - .insert(remote_public, registration_data.clone()); - let bytes = match AuthenticatorVersion::from(protocol) { - AuthenticatorVersion::V1 => { - v1::response::AuthenticatorResponse::new_pending_registration_success( - v1::registration::RegistrationData { - nonce: registration_data.nonce, - gateway_data: v1::registration::GatewayClient::new( - self.keypair().private_key(), - remote_public.inner(), - ip_allocation.ipv4.into(), - nonce, - ), - wg_port: registration_data.wg_port, - }, - request_id, - reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?, - ) - .to_bytes() - .map_err(AuthenticatorError::response_serialisation)? - } - AuthenticatorVersion::V2 => { - v2::response::AuthenticatorResponse::new_pending_registration_success( - v2::registration::RegistrationData { - nonce: registration_data.nonce, - gateway_data: v2::registration::GatewayClient::new( - self.keypair().private_key(), - remote_public.inner(), - ip_allocation.ipv4.into(), - nonce, - ), - wg_port: registration_data.wg_port, - }, - request_id, - reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?, - ) - .to_bytes() - .map_err(AuthenticatorError::response_serialisation)? - } - AuthenticatorVersion::V3 => { - v3::response::AuthenticatorResponse::new_pending_registration_success( - v3::registration::RegistrationData { - nonce: registration_data.nonce, - gateway_data: v3::registration::GatewayClient::new( - self.keypair().private_key(), - remote_public.inner(), - ip_allocation.ipv4.into(), - nonce, - ), - wg_port: registration_data.wg_port, - }, - request_id, - reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?, - ) - .to_bytes() - .map_err(AuthenticatorError::response_serialisation)? - } - AuthenticatorVersion::V4 => { - v4::response::AuthenticatorResponse::new_pending_registration_success( - v4::registration::RegistrationData { - nonce: registration_data.nonce, - // convert current to v5 and then v5 to v4 (current as of 28.08.25) - gateway_data: v5::registration::GatewayClient::from( - registration_data.gateway_data.clone(), - ) - .into(), - wg_port: registration_data.wg_port, - }, - request_id, - reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?, - ) - .to_bytes() - .map_err(AuthenticatorError::response_serialisation)? - } - AuthenticatorVersion::V5 => { - v5::response::AuthenticatorResponse::new_pending_registration_success( - v5::registration::RegistrationData { - nonce: registration_data.nonce, - gateway_data: registration_data.gateway_data.into(), - wg_port: registration_data.wg_port, - }, - request_id, - ) - .to_bytes() - .map_err(AuthenticatorError::response_serialisation)? - } - AuthenticatorVersion::V6 => { - v6::response::AuthenticatorResponse::new_pending_registration_success( - v6::registration::RegistrationData { - nonce: registration_data.nonce, - gateway_data: registration_data.gateway_data, - wg_port: registration_data.wg_port, - }, - request_id, - self.upgrade_mode_enabled(), - ) - .to_bytes() - .map_err(AuthenticatorError::response_serialisation)? - } - AuthenticatorVersion::UNKNOWN => return Err(AuthenticatorError::UnknownVersion), - }; - - Ok((bytes, reply_to)) - } - - async fn handle_final_credential_claim( - &self, - claim: BandwidthClaim, - client_id: i64, - ) -> Result<(), AuthenticatorError> { - match claim.credential { - BandwidthCredential::ZkNym(zk_nym) => { - // if we got zk-nym, we just try to verify it - credential_verification(self.ecash_verifier.clone(), *zk_nym, client_id).await?; - Ok(()) - } - BandwidthCredential::UpgradeModeJWT { token } => { - // if we're already in the upgrade mode, don't bother validating the token - if self.upgrade_mode_enabled() { - return Ok(()); - } - - self.upgrade_mode.try_enable_via_received_jwt(token).await?; - Ok(()) - } - } + Ok((response.bytes, response.reply_to)) } async fn on_final_request( @@ -532,139 +110,12 @@ impl MixnetListener { request_id: u64, reply_to: Option, ) -> AuthenticatorHandleResult { - let mut registered_and_free = self.registered_and_free.write().await; - let registration_data = registered_and_free - .registration_in_progress - .get(&final_message.gateway_client_pub_key()) - .ok_or(AuthenticatorError::RegistrationNotInProgress)? - .clone(); - - if final_message - .verify(self.keypair().private_key(), registration_data.nonce) - .is_err() - { - return Err(AuthenticatorError::MacVerificationFailure); - } - - let mut peer = Peer::new(Key::new(final_message.gateway_client_pub_key().to_bytes())); - peer.allowed_ips - .push(IpAddrMask::new(final_message.private_ips().ipv4.into(), 32)); - peer.allowed_ips.push(IpAddrMask::new( - final_message.private_ips().ipv6.into(), - 128, - )); - - // ideally credential wouldn't have been required in upgrade mode, - // however, we need some basic information to insert valid wg peer - let Some(credential) = final_message.credential() else { - return Err(AuthenticatorError::NoCredentialReceived); - }; - - let typ = credential.kind; - - let client_id = self - .ecash_verifier - .storage() - .insert_wireguard_peer(&peer, typ.into()) + let response = self + .peer_registrator + .on_final_authenticator_request(final_message, protocol, request_id, reply_to) .await?; - if let Err(err) = self - .handle_final_credential_claim(credential, client_id) - .await - { - self.ecash_verifier - .storage() - .remove_wireguard_peer(&peer.public_key.to_string()) - .await?; - return Err(err); - } - - let public_key = peer.public_key.to_string(); - if let Err(e) = self.peer_manager.add_peer(peer).await { - self.ecash_verifier - .storage() - .remove_wireguard_peer(&public_key) - .await?; - return Err(e.into()); - } - - registered_and_free - .registration_in_progress - .remove(&final_message.gateway_client_pub_key()); - - let bytes = match AuthenticatorVersion::from(protocol) { - AuthenticatorVersion::V1 => v1::response::AuthenticatorResponse::new_registered( - v1::registration::RegisteredData { - pub_key: registration_data.gateway_data.pub_key, - private_ip: registration_data.gateway_data.private_ips.ipv4.into(), - wg_port: registration_data.wg_port, - }, - reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?, - request_id, - ) - .to_bytes() - .map_err(AuthenticatorError::response_serialisation)?, - AuthenticatorVersion::V2 => v2::response::AuthenticatorResponse::new_registered( - v2::registration::RegisteredData { - pub_key: registration_data.gateway_data.pub_key, - private_ip: registration_data.gateway_data.private_ips.ipv4.into(), - wg_port: registration_data.wg_port, - }, - reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?, - request_id, - ) - .to_bytes() - .map_err(AuthenticatorError::response_serialisation)?, - AuthenticatorVersion::V3 => v3::response::AuthenticatorResponse::new_registered( - v3::registration::RegisteredData { - pub_key: registration_data.gateway_data.pub_key, - private_ip: registration_data.gateway_data.private_ips.ipv4.into(), - wg_port: registration_data.wg_port, - }, - reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?, - request_id, - ) - .to_bytes() - .map_err(AuthenticatorError::response_serialisation)?, - AuthenticatorVersion::V4 => v4::response::AuthenticatorResponse::new_registered( - v4::registration::RegisteredData { - pub_key: registration_data.gateway_data.pub_key, - // convert current to v5 and then v5 to v4 (current as of 28.08.25) - private_ips: v5::registration::IpPair::from( - registration_data.gateway_data.private_ips, - ) - .into(), - wg_port: registration_data.wg_port, - }, - reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?, - request_id, - ) - .to_bytes() - .map_err(AuthenticatorError::response_serialisation)?, - AuthenticatorVersion::V5 => v5::response::AuthenticatorResponse::new_registered( - v5::registration::RegisteredData { - pub_key: registration_data.gateway_data.pub_key, - private_ips: registration_data.gateway_data.private_ips.into(), - wg_port: registration_data.wg_port, - }, - request_id, - ) - .to_bytes() - .map_err(AuthenticatorError::response_serialisation)?, - AuthenticatorVersion::V6 => v6::response::AuthenticatorResponse::new_registered( - v6::registration::RegisteredData { - pub_key: registration_data.gateway_data.pub_key, - private_ips: registration_data.gateway_data.private_ips, - wg_port: registration_data.wg_port, - }, - request_id, - self.upgrade_mode_enabled(), - ) - .to_bytes() - .map_err(AuthenticatorError::response_serialisation)?, - AuthenticatorVersion::UNKNOWN => return Err(AuthenticatorError::UnknownVersion), - }; - Ok((bytes, reply_to)) + Ok((response.bytes, response.reply_to)) } async fn on_query_bandwidth_request( @@ -955,9 +406,6 @@ impl MixnetListener { break; }, _ = self.timeout_check_interval.next() => { - if let Err(e) = self.remove_stale_registrations().await { - tracing::error!("Could not clear stale registrations. The registration process might get jammed soon - {e:?}"); - } self.seen_credential_cache.remove_stale(); } msg = self.mixnet_client.next() => { @@ -987,45 +435,6 @@ impl MixnetListener { } } -pub async fn credential_storage_preparation( - ecash_verifier: Arc, - client_id: i64, -) -> Result { - ecash_verifier - .storage() - .create_bandwidth_entry(client_id) - .await?; - let bandwidth = ecash_verifier - .storage() - .get_available_bandwidth(client_id) - .await? - .ok_or(AuthenticatorError::InternalError( - "bandwidth entry should have just been created".to_string(), - ))?; - Ok(bandwidth) -} - -async fn credential_verification( - ecash_verifier: Arc, - credential: CredentialSpendingData, - client_id: i64, -) -> Result { - let bandwidth = credential_storage_preparation(ecash_verifier.clone(), client_id).await?; - let client_bandwidth = ClientBandwidth::new(bandwidth.into()); - let mut verifier = CredentialVerifier::new( - CredentialSpendingRequest::new(credential), - ecash_verifier.clone(), - BandwidthStorageManager::new( - ecash_verifier.storage(), - client_bandwidth, - client_id, - BandwidthFlushingBehaviourConfig::default(), - true, - ), - ); - Ok(verifier.verify().await?) -} - fn deserialize_request( reconstructed: &ReconstructedMessage, ) -> Result { diff --git a/gateway/src/node/internal_service_providers/authenticator/mod.rs b/gateway/src/node/internal_service_providers/authenticator/mod.rs index 95423a66e7..fb50f6ddb0 100644 --- a/gateway/src/node/internal_service_providers/authenticator/mod.rs +++ b/gateway/src/node/internal_service_providers/authenticator/mod.rs @@ -2,13 +2,14 @@ // SPDX-License-Identifier: Apache-2.0 use crate::node::internal_service_providers::authenticator::error::AuthenticatorError; +use crate::node::wireguard::PeerRegistrator; use futures::channel::oneshot; use nym_client_core::{HardcodedTopologyProvider, TopologyProvider}; use nym_credential_verification::upgrade_mode::UpgradeModeDetails; use nym_sdk::{mixnet::Recipient, GatewayTransceiver}; use nym_task::ShutdownTracker; use nym_wireguard::WireguardGatewayData; -use std::{path::Path, sync::Arc}; +use std::path::Path; pub use config::Config; @@ -32,12 +33,12 @@ impl OnStartData { pub struct Authenticator { #[allow(unused)] config: Config, + peer_registrator: PeerRegistrator, upgrade_mode_state: UpgradeModeDetails, wait_for_gateway: bool, custom_topology_provider: Option>, custom_gateway_transceiver: Option>, wireguard_gateway_data: WireguardGatewayData, - ecash_verifier: Arc, shutdown: ShutdownTracker, on_start: Option>, } @@ -45,20 +46,18 @@ pub struct Authenticator { impl Authenticator { pub fn new( config: Config, + peer_registrator: PeerRegistrator, upgrade_mode_state: UpgradeModeDetails, wireguard_gateway_data: WireguardGatewayData, - ecash_verifier: Arc< - dyn nym_credential_verification::ecash::traits::EcashManager + Send + Sync, - >, shutdown: ShutdownTracker, ) -> Self { Self { config, + peer_registrator, upgrade_mode_state, wait_for_gateway: false, custom_topology_provider: None, custom_gateway_transceiver: None, - ecash_verifier, wireguard_gateway_data, shutdown, on_start: None, @@ -134,8 +133,8 @@ impl Authenticator { self.config, self.wireguard_gateway_data, mixnet_client, + self.peer_registrator, self.upgrade_mode_state, - self.ecash_verifier, ); tracing::info!("The address of this client is: {self_address}"); diff --git a/gateway/src/node/internal_service_providers/authenticator/seen_credential_cache.rs b/gateway/src/node/internal_service_providers/authenticator/seen_credential_cache.rs index 0c2253f7c5..e06b66760e 100644 --- a/gateway/src/node/internal_service_providers/authenticator/seen_credential_cache.rs +++ b/gateway/src/node/internal_service_providers/authenticator/seen_credential_cache.rs @@ -2,9 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 #[cfg(test)] -use mock_instant::thread_local::SystemTime; +use mock_instant::thread_local::Instant; #[cfg(not(test))] -use std::time::SystemTime; +use std::time::Instant; use std::{collections::HashMap, time::Duration}; use nym_credentials_interface::CredentialSpendingData; @@ -15,7 +15,7 @@ const SEEN_CREDENTIAL_CACHE_TIME: Duration = Duration::from_secs(60 * 60); // 1 #[derive(Eq, Hash, PartialEq)] struct TimestampedPeerPubKey { peer_pub_key: PeerPublicKey, - timestamp: SystemTime, + timestamp: Instant, } pub(crate) struct SeenCredentialCache { @@ -36,7 +36,7 @@ impl SeenCredentialCache { ) { let value = TimestampedPeerPubKey { peer_pub_key, - timestamp: SystemTime::now(), + timestamp: Instant::now(), }; self.cached_credentials .insert(credential.serial_number_b58(), value); @@ -52,12 +52,9 @@ impl SeenCredentialCache { } pub(crate) fn remove_stale(&mut self) { - let now = SystemTime::now(); + let now = Instant::now(); self.cached_credentials.retain(|_, value| { - let Ok(cache_time) = now.duration_since(value.timestamp) else { - tracing::warn!("Got decreasing consecutive system timestamps"); - return false; - }; + let cache_time = now.duration_since(value.timestamp); cache_time < SEEN_CREDENTIAL_CACHE_TIME }); } @@ -159,30 +156,9 @@ mod test { cache.remove_stale(); assert!(cache.get_peer_pub_key(&credential).is_some()); - MockClock::advance_system_time(SEEN_CREDENTIAL_CACHE_TIME * 2); + MockClock::advance(SEEN_CREDENTIAL_CACHE_TIME * 2); cache.remove_stale(); assert!(cache.get_peer_pub_key(&credential).is_none()); } - - #[test] - fn invalid_time() { - assert!(MockClock::is_thread_local()); - assert!(SystemTime::now().is_thread_local()); - - let mut cache = SeenCredentialCache::new(); - let credential = CredentialSpendingData::try_from_bytes(&CREDENTIAL_BYTES).unwrap(); - let peer_pub_key = PeerPublicKey::from_str(PUB_KEY).unwrap(); - - // set some value for time - MockClock::set_system_time(Duration::from_secs(10)); - cache.insert_credential(credential.clone(), peer_pub_key); - - // then set the time in the past - MockClock::set_system_time(Duration::ZERO); - cache.remove_stale(); - - // invalid time should remove the credential, just in case - assert!(cache.get_peer_pub_key(&credential).is_none()); - } } diff --git a/gateway/src/node/lp_listener/handler.rs b/gateway/src/node/lp_listener/handler.rs index 599e3796ed..c29f9298f1 100644 --- a/gateway/src/node/lp_listener/handler.rs +++ b/gateway/src/node/lp_listener/handler.rs @@ -1241,23 +1241,15 @@ where mod tests { use super::*; use crate::node::lp_listener::{LpConfig, LpDebug}; - use crate::node::wireguard::PeerManager; use crate::node::ActiveClientsStore; use bytes::BytesMut; - use nym_credential_verification::upgrade_mode::{ - UpgradeModeCheckConfig, UpgradeModeCheckRequestSender, UpgradeModeDetails, - }; - use nym_credential_verification::UpgradeModeState; use nym_lp::codec::{parse_lp_packet, serialize_lp_packet}; use nym_lp::message::{ClientHelloData, EncryptedDataPayload, HandshakeData, LpMessage}; use nym_lp::packet::{LpHeader, LpPacket}; use nym_lp::peer::LpLocalPeer; - use nym_wireguard::{PeerControlRequest, WireguardConfig, WireguardGatewayData}; - use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; - use tokio::sync::mpsc::Receiver; // ==================== Test Helpers ==================== /// Create a minimal test state for handler tests @@ -1265,23 +1257,6 @@ mod tests { use nym_crypto::asymmetric::ed25519; use rand::rngs::OsRng; - fn wireguard_data( - keys: Arc, - ) -> (WireguardGatewayData, Receiver) { - // some sensible default values (ports don't matter anyway) - let cfg = WireguardConfig { - bind_address: SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 51822), - private_ipv4: Ipv4Addr::new(10, 1, 0, 1), - private_ipv6: Ipv6Addr::new(0xfc01, 0, 0, 0, 0, 0, 0, 0x1), // fc01::1, - announced_tunnel_port: 51822, - announced_metadata_port: 51830, - private_network_prefix_v4: 16, - private_network_prefix_v6: 112, - }; - - WireguardGatewayData::new(cfg, keys) - } - // Create in-memory storage for testing let storage = nym_gateway_storage::GatewayStorage::init(":memory:", 100) .await @@ -1308,20 +1283,6 @@ mod tests { let id_keys = Arc::new(ed25519::KeyPair::new(&mut OsRng)); let x_keys = Arc::new(id_keys.to_x25519()); - let (wireguard_data, _) = wireguard_data(x_keys.clone()); - - let (um_recheck_tx, _) = futures::channel::mpsc::unbounded(); - - let upgrade_mode_state = UpgradeModeState::new(*id_keys.public_key()); - let upgrade_mode_details = UpgradeModeDetails::new( - UpgradeModeCheckConfig { - // essentially we never want to trigger this in our tests - min_staleness_recheck: Duration::from_nanos(1), - }, - UpgradeModeCheckRequestSender::new(um_recheck_tx), - upgrade_mode_state.clone(), - ); - let lp_peer = LpLocalPeer::new(id_keys, x_keys.clone()).with_kem_psq_key(x_keys); LpHandlerState { @@ -1332,13 +1293,11 @@ mod tests { local_lp_peer: lp_peer, metrics: nym_node_metrics::NymNodeMetrics::default(), active_clients_store: ActiveClientsStore::new(), - upgrade_mode: upgrade_mode_details, outbound_mix_sender: mix_sender, handshake_states: Arc::new(dashmap::DashMap::new()), session_states: Arc::new(dashmap::DashMap::new()), - registrations_in_progress: Default::default(), forward_semaphore, - peer_manager: Arc::new(PeerManager::new(wireguard_data)), + peer_registrator: None, } } diff --git a/gateway/src/node/lp_listener/mod.rs b/gateway/src/node/lp_listener/mod.rs index 0114e28d95..367e16e9bf 100644 --- a/gateway/src/node/lp_listener/mod.rs +++ b/gateway/src/node/lp_listener/mod.rs @@ -68,12 +68,11 @@ // They can be exported via Prometheus format using the metrics endpoint. use crate::error::GatewayError; -use crate::node::lp_listener::registration::RegistrationsInProgress; +use crate::node::wireguard::PeerRegistrator; use crate::node::ActiveClientsStore; use dashmap::DashMap; use nym_config::serde_helpers::de_maybe_port; use nym_credential_verification::ecash::traits::EcashManager; -use nym_credential_verification::upgrade_mode::UpgradeModeDetails; use nym_gateway_storage::GatewayStorage; use nym_lp::state_machine::LpStateMachine; use nym_node_metrics::NymNodeMetrics; @@ -85,7 +84,6 @@ use tokio::net::TcpListener; use tokio::sync::Semaphore; use tracing::*; -use crate::node::wireguard::PeerManager; pub use nym_lp::peer::LpLocalPeer; pub use nym_mixnet_client::forwarder::{ mix_forwarding_channels, MixForwardingReceiver, MixForwardingSender, @@ -184,10 +182,6 @@ pub struct LpDebug { #[serde(with = "humantime_serde")] pub demoted_session_ttl: Duration, - /// Maximum age of in-progress dVPN registration before cleanup (default: 60s) - #[serde(with = "humantime_serde")] - pub pending_registration_ttl: Duration, - /// How often to run the state cleanup task (default: 5 minutes) /// /// The cleanup task scans for and removes stale handshakes and sessions. @@ -257,9 +251,6 @@ impl LpDebug { // 5 minutes - balances memory reclamation with task overhead pub const DEFAULT_STATE_CLEANUP_INTERVAL: Duration = Duration::from_secs(300); - // 1 minute - enough for client to send retrieve credential from its storage and send it across - pub const DEFAULT_PENDING_REGISTRATION_TTL: Duration = Duration::from_secs(60); - // Limits concurrent outbound connections to prevent fd exhaustion pub const DEFAULT_MAX_CONCURRENT_FORWARDS: usize = 1000; } @@ -273,7 +264,6 @@ impl Default for LpDebug { handshake_ttl: Self::DEFAULT_HANDSHAKE_TTL, session_ttl: Self::DEFAULT_SESSION_TTL, demoted_session_ttl: Self::DEFAULT_DEMOTED_SESSION_TTL, - pending_registration_ttl: Self::DEFAULT_PENDING_REGISTRATION_TTL, state_cleanup_interval: Self::DEFAULT_STATE_CLEANUP_INTERVAL, max_concurrent_forwards: Self::DEFAULT_MAX_CONCURRENT_FORWARDS, } @@ -360,12 +350,8 @@ pub struct LpHandlerState { /// Active clients tracking pub active_clients_store: ActiveClientsStore, - /// Current state of the Upgrade Mode as perceived by this gateway - pub upgrade_mode: UpgradeModeDetails, - - /// WireGuard gateway data (contains keypair and config) - /// alongside helpers for managing peers - pub peer_manager: Arc, + /// Handle registering new wireguard peers + pub peer_registrator: Option, /// LP configuration (for timestamp validation, etc.) pub lp_config: LpConfig, @@ -399,10 +385,6 @@ pub struct LpHandlerState { /// to rekey without re-authentication. pub session_states: Arc>>, - /// In-progress dVPN registrations that require additional data (e.g. credentials) - /// to finalise. - pub registrations_in_progress: RegistrationsInProgress, - /// Semaphore limiting concurrent forward connections /// /// Prevents file descriptor exhaustion when forwarding LP packets during @@ -577,31 +559,26 @@ impl LpListener { /// /// The task automatically stops when the shutdown signal is received. fn spawn_state_cleanup_task(&self) -> tokio::task::JoinHandle<()> { - let peer_manager = Arc::clone(&self.handler_state.peer_manager); let handshake_states = Arc::clone(&self.handler_state.handshake_states); let session_states = Arc::clone(&self.handler_state.session_states); - let pending_registrations = self.handler_state.registrations_in_progress.clone(); let dbg_cfg = self.handler_state.lp_config.debug; let handshake_ttl = dbg_cfg.handshake_ttl; let session_ttl = dbg_cfg.session_ttl; let demoted_session_ttl = dbg_cfg.demoted_session_ttl; - let pending_reg_ttl = dbg_cfg.pending_registration_ttl; let interval = dbg_cfg.state_cleanup_interval; let shutdown = self.shutdown.clone_shutdown_token(); let metrics = self.handler_state.metrics.clone(); info!( - "Starting LP state cleanup task (handshake_ttl={}s, session_ttl={}s, demoted_ttl={}s, reg_ttl={}s, interval={}s)", - handshake_ttl.as_secs(), session_ttl.as_secs(), demoted_session_ttl.as_secs(),pending_reg_ttl.as_secs(), interval.as_secs() + "Starting LP state cleanup task (handshake_ttl={}s, session_ttl={}s, demoted_ttl={}s, interval={}s)", + handshake_ttl.as_secs(), session_ttl.as_secs(), demoted_session_ttl.as_secs(), interval.as_secs() ); self.shutdown.try_spawn_named( cleanup_task::cleanup_loop( - peer_manager, handshake_states, session_states, - pending_registrations, dbg_cfg, shutdown, metrics, @@ -619,33 +596,27 @@ impl LpListener { } pub(crate) mod cleanup_task { - use crate::node::lp_listener::registration::RegistrationsInProgress; use crate::node::lp_listener::{LpDebug, TimestampedState}; - use crate::node::wireguard::PeerManager; use dashmap::DashMap; use nym_lp::state_machine::LpStateBare; use nym_lp::LpStateMachine; use nym_metrics::inc_by; use nym_node_metrics::NymNodeMetrics; use std::sync::Arc; - use tracing::{debug, error, info}; + use tracing::{debug, info}; async fn perform_cleanup( - peer_manager: &PeerManager, handshake_states: &Arc>>, session_states: &Arc>>, - registrations_in_progress: &RegistrationsInProgress, cfg: LpDebug, ) { let handshake_ttl = cfg.handshake_ttl; let session_ttl = cfg.session_ttl; let demoted_session_ttl = cfg.demoted_session_ttl; - let pending_registration_ttl = cfg.pending_registration_ttl; let start = std::time::Instant::now(); let mut hs_removed = 0u64; let mut ss_removed = 0u64; - let mut pending_reg_removed = 0u64; let mut demoted_removed = 0u64; // Remove stale handshakes (based on age since creation) @@ -680,33 +651,10 @@ pub(crate) mod cleanup_task { } }); - // Remove stale registrations (based on time since last activity) - let mut reg_guard = registrations_in_progress.lock().await; - let mut stale_registrations = Vec::new(); - for (k, timestamped) in reg_guard.iter() { - if timestamped.age() > pending_registration_ttl { - stale_registrations.push(*k) - } - } - - for to_remove in stale_registrations { - pending_reg_removed += 1; - - // SAFETY: we never dropped the guard and the entry existed - #[allow(clippy::unwrap_used)] - let entry = reg_guard.remove(&to_remove).unwrap(); - if let Err(err) = peer_manager - .release_ip_pair(entry.state.allocated_ip_pair()) - .await - { - error!("failed to release allocated ip pair: {err}") - } - } - - if hs_removed > 0 || ss_removed > 0 || demoted_removed > 0 || pending_reg_removed > 0 { + if hs_removed > 0 || ss_removed > 0 || demoted_removed > 0 { let duration = start.elapsed(); info!( - "LP state cleanup: removed {hs_removed} handshakes, {pending_reg_removed} pending registrations, {ss_removed} sessions, {demoted_removed} demoted (took {:.3}s)", + "LP state cleanup: removed {hs_removed} handshakes, {ss_removed} sessions, {demoted_removed} demoted (took {:.3}s)", duration.as_secs_f64() ); @@ -720,12 +668,6 @@ pub(crate) mod cleanup_task { if demoted_removed > 0 { inc_by!("lp_states_cleanup_demoted_removed", demoted_removed as i64); } - if pending_reg_removed > 0 { - inc_by!( - "lp_states_cleanup_pending_registrations_removed", - pending_reg_removed as i64 - ); - } } } @@ -737,10 +679,8 @@ pub(crate) mod cleanup_task { /// Demoted sessions (ReadOnlyTransport) use shorter TTL since they /// only need to drain in-flight packets after subsession promotion. pub(crate) async fn cleanup_loop( - peer_manager: Arc, handshake_states: Arc>>, session_states: Arc>>, - registrations_in_progress: RegistrationsInProgress, cfg: LpDebug, shutdown: nym_task::ShutdownToken, _metrics: NymNodeMetrics, @@ -757,7 +697,7 @@ pub(crate) mod cleanup_task { break; } _ = cleanup_interval.tick() => { - perform_cleanup(&peer_manager, &handshake_states, &session_states, ®istrations_in_progress, cfg).await; + perform_cleanup(&handshake_states, &session_states, cfg).await; } } } diff --git a/gateway/src/node/lp_listener/registration.rs b/gateway/src/node/lp_listener/registration.rs index 421dc14260..7f2e7b875c 100644 --- a/gateway/src/node/lp_listener/registration.rs +++ b/gateway/src/node/lp_listener/registration.rs @@ -1,22 +1,8 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: GPL-3.0-only -use super::{LpHandlerState, ReceiverIndex, TimestampedState}; -use crate::error::GatewayError; -use defguard_wireguard_rs::host::Peer; -use defguard_wireguard_rs::key::Key; -use nym_authenticator_requests::models::BandwidthClaim; -use nym_credential_verification::ecash::traits::EcashManager; -use nym_credential_verification::{ - bandwidth_storage_manager::BandwidthStorageManager, BandwidthFlushingBehaviourConfig, - ClientBandwidth, CredentialVerifier, -}; -use nym_credentials_interface::{BandwidthCredential, CredentialSpendingData, TicketType}; -use nym_crypto::asymmetric::encryption::KeyPair; -use nym_gateway_requests::models::CredentialSpendingRequest; -use nym_gateway_storage::models::PersistedBandwidth; -use nym_gateway_storage::traits::BandwidthGatewayStorage; -use nym_metrics::{add_histogram_obs, inc, inc_by}; +use crate::node::lp_listener::{LpHandlerState, ReceiverIndex}; +use nym_metrics::{add_histogram_obs, inc}; use nym_registration_common::dvpn::{ LpDvpnRegistrationFinalisation, LpDvpnRegistrationInitialRequest, LpDvpnRegistrationRequestMessage, LpDvpnRegistrationRequestMessageContent, @@ -24,15 +10,8 @@ use nym_registration_common::dvpn::{ use nym_registration_common::mixnet::LpMixnetRegistrationRequestMessage; use nym_registration_common::{ LpRegistrationRequest, LpRegistrationRequestData, LpRegistrationResponse, RegistrationMode, - RegistrationStatus, WireguardConfiguration, + RegistrationStatus, }; -use nym_wireguard::peer_controller::IpPair; -use nym_wireguard::WireguardConfig; -use nym_wireguard_types::PeerPublicKey; -use std::collections::HashMap; -use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; -use std::sync::Arc; -use tokio::sync::{Mutex, MutexGuard}; use tracing::*; // Histogram buckets for LP registration duration tracking @@ -49,225 +28,28 @@ const LP_REGISTRATION_DURATION_BUCKETS: &[f64] = &[ 30.0, // 30s ]; -// Histogram buckets for WireGuard peer controller channel latency -// Measures time to send request and receive response from peer controller -// Expected: 1ms-100ms for normal operations, up to 2s for slow conditions -const WG_CONTROLLER_LATENCY_BUCKETS: &[f64] = &[ - 0.001, // 1ms - 0.005, // 5ms - 0.01, // 10ms - 0.05, // 50ms - 0.1, // 100ms - 0.25, // 250ms - 0.5, // 500ms - 1.0, // 1s - 2.0, // 2s -]; - -#[derive(Clone, Copy)] -pub struct PendingRegistrationState { - client_id: i64, - peer_key: PeerPublicKey, - ticket_type: TicketType, - wireguard_config: WireguardConfiguration, -} - -impl PendingRegistrationState { - pub(crate) fn allocated_ip_pair(&self) -> IpPair { - IpPair::new( - self.wireguard_config.private_ipv4, - self.wireguard_config.private_ipv6, - ) - } -} - -#[derive(Clone, Default)] -pub struct RegistrationsInProgress { - /// Wrapped in TimestampedState for TTL-based cleanup of stale data. - inner: Arc>>>, -} - -impl RegistrationsInProgress { - pub async fn lock( - &self, - ) -> MutexGuard<'_, HashMap>> { - self.inner.lock().await - } -} - impl LpHandlerState { - fn upgrade_mode_enabled(&self) -> bool { - self.upgrade_mode.enabled() - } - - fn keypair(&self) -> &Arc { - self.peer_manager.wireguard_gateway_data.keypair() - } - - fn wireguard_config(&self) -> WireguardConfig { - self.peer_manager.wireguard_gateway_data.config() - } - - fn successful_dvpn_registration( - &self, - peer_private_ipv4: Ipv4Addr, - peer_private_ipv6: Ipv6Addr, - bandwidth: i64, - ) -> LpRegistrationResponse { - LpRegistrationResponse::success_dvpn( - WireguardConfiguration { - public_key: *self.keypair().public_key(), - psk: None, - // TODO: according to @SW this is most likely very wrong - endpoint: self.wireguard_config().bind_address, - private_ipv4: peer_private_ipv4, - private_ipv6: peer_private_ipv6, - }, - bandwidth, - ) - } - - /// Check if WG peer already registered, return cached response if so. - /// - /// This enables idempotent registration: if a client retries registration - /// with the same WG public key (e.g., after network failure), we return - /// the existing registration data instead of re-processing. This prevents - /// wasting credentials on network issues. - async fn check_existing_dvpn_registration( - &self, - public_key: PeerPublicKey, - ) -> Option { - // Look up existing peer - let Ok(maybe_peer) = self.peer_manager.query_peer(public_key).await else { - return Some(LpRegistrationResponse::error( - "iternal failure: failed to resolve peer information", - RegistrationMode::Dvpn, - )); - }; - - let peer = maybe_peer?; - - // Extract IPv4 and IPv6 from allowed_ips - let mut private_ipv4 = None; - let mut private_ipv6 = None; - for ip_mask in &peer.allowed_ips { - match ip_mask.address { - IpAddr::V4(v4) => private_ipv4 = Some(v4), - IpAddr::V6(v6) => private_ipv6 = Some(v6), - } - if private_ipv4.is_some() && private_ipv6.is_some() { - break; - } - } - - // Incomplete data, treat as new registration - let (Some(private_ipv4), Some(private_ipv6)) = (private_ipv4, private_ipv6) else { - return None; - }; - - // Get current bandwidth - let Ok(bandwidth) = self.peer_manager.query_client_bandwidth(public_key).await else { - return Some(LpRegistrationResponse::error( - "iternal failure: failed to resolve peer bandwidth", - RegistrationMode::Dvpn, - )); - }; - - Some(self.successful_dvpn_registration( - private_ipv4, - private_ipv6, - bandwidth.available().await, - )) - } - - /// In the case of an already registered WG peer, update its PSK. - async fn update_peer_psk(&self, peer: PeerPublicKey, psk: Key) -> Result<(), GatewayError> { - let encoded_psk = psk.to_lower_hex(); - self.storage - .update_peer_psk(&peer.to_string(), Some(&encoded_psk)) - .await?; - - // TODO: do we have to go through a peer manager to also update PSK if a peer is currently active? - // seems like an edge case. maybe we should force disconnect here? - Ok(()) - } - async fn process_dvpn_initial_registration( &self, sender: ReceiverIndex, request: LpDvpnRegistrationInitialRequest, ) -> LpRegistrationResponse { - let wg_key_str = request.wg_public_key.to_string(); - - // check for an existing registration (same WG key already registered) - // This allows clients to retry registration after network failures - // or to re-use gateway without spending additional bandwidth - if let Some(existing_response) = self - .check_existing_dvpn_registration(request.wg_public_key) - .await - { - // if there already exists registration for this client, update the psk and return the peer data - if let Err(err) = self - .update_peer_psk(request.wg_public_key, Key::new(request.psk)) - .await - { - return LpRegistrationResponse::error( - format!("WireGuard peer PSK update failed: {err}"), - RegistrationMode::Dvpn, - ); - } - info!("LP dVPN re-registration for existing peer {wg_key_str} (idempotent)",); - inc!("lp_registration_dvpn_idempotent"); - return existing_response; - } - - // TODO: this could be a source of some issue as we pre-allocate ip before validating credentials - // (but we do the same in the authenticator anyway...) - if let Err(err) = self - .register_wg_peer( - sender, - request.wg_public_key, - request.ticket_type, - Key::new(request.psk), - ) - .await - { + let Some(registrator) = self.peer_registrator.as_ref() else { return LpRegistrationResponse::error( - format!("WireGuard peer IP allocation failed: {err}"), + "dVPN via LP is not enabled on this node", RegistrationMode::Dvpn, ); - } + }; - LpRegistrationResponse::request_dvpn_credential() - } - - // TODO: dedup - async fn handle_final_credential_claim( - &self, - claim: BandwidthClaim, - client_id: i64, - ) -> Result { - match claim.credential { - BandwidthCredential::ZkNym(zk_nym) => { - // if we got zk-nym, we just try to verify it - let bandwidth = - credential_verification(self.ecash_verifier.clone(), *zk_nym, client_id) - .await?; - Ok(bandwidth) - } - BandwidthCredential::UpgradeModeJWT { token } => { - // TODO: move - const UM_BANDWIDTH: i64 = 1024 * 1024 * 1024; - - // if we're already in the upgrade mode, don't bother validating the token - if self.upgrade_mode_enabled() { - return Ok(UM_BANDWIDTH); - } - - self.upgrade_mode.try_enable_via_received_jwt(token).await?; - Ok(UM_BANDWIDTH) - } - } + registrator + .on_initial_lp_request(request, sender) + .await + .unwrap_or_else(|err| { + LpRegistrationResponse::error( + format!("LP registration has failed: {err}"), + RegistrationMode::Dvpn, + ) + }) } async fn process_dvpn_registration_finalisation( @@ -275,63 +57,22 @@ impl LpHandlerState { sender: ReceiverIndex, request: LpDvpnRegistrationFinalisation, ) -> LpRegistrationResponse { - // see if we still have the pending registration - // (e.g. it's illegal for client to request registration and only finalise it, - // for example the next day; we can't keep the data forever) - let Some(pending) = self - .registrations_in_progress - .lock() - .await - .get(&sender) - .map(|pending| pending.state) - else { + let Some(registrator) = self.peer_registrator.as_ref() else { return LpRegistrationResponse::error( - "no pending registration", + "dVPN via LP is not enabled on this node", RegistrationMode::Dvpn, ); }; - if pending.ticket_type != request.credential.kind { - return LpRegistrationResponse::error( - format!( - "inconsistent ticket type. used {} for initial request and {} for finalisation", - pending.ticket_type, request.credential.kind - ), - RegistrationMode::Dvpn, - ); - } - - let client_id = pending.client_id; - - let allocated_bandwidth = match self - .handle_final_credential_claim(request.credential, client_id) + registrator + .on_final_lp_request(request, sender) .await - { - Ok(bandwidth) => bandwidth, - Err(err) => { - // Credential verification failed, remove the peer - warn!("LP credential verification failed for client {client_id}: {err}"); - inc!("lp_registration_dvpn_failed"); - if let Err(remove_err) = self - .storage - .remove_wireguard_peer(&pending.peer_key.to_string()) - .await - { - error!( - "Failed to remove peer after credential verification failure: {remove_err}" - ); - } - self.registrations_in_progress.lock().await.remove(&sender); - return LpRegistrationResponse::error( - format!("Credential verification failed: {err}"), + .unwrap_or_else(|err| { + LpRegistrationResponse::error( + format!("LP registration has failed: {err}"), RegistrationMode::Dvpn, - ); - } - }; - - info!("LP dVPN registration successful (client_id: {client_id})"); - inc!("lp_registration_dvpn_success"); - LpRegistrationResponse::success_dvpn(pending.wireguard_config, allocated_bandwidth) + ) + }) } async fn process_dvpn_registration( @@ -415,163 +156,4 @@ impl LpHandlerState { result } - - /// Register a WireGuard peer and return gateway data along with the client_id - async fn register_wg_peer( - &self, - sender: ReceiverIndex, - peer_key: PeerPublicKey, - ticket_type: nym_credentials_interface::TicketType, - psk: Key, - ) -> Result<(), GatewayError> { - // Allocate IPs from centralized pool managed by PeerController - let defguard_key = Key::new(peer_key.to_bytes()); - - // Request IP allocation from PeerController - let ip_pair = self.peer_manager.allocate_peer_ip_pair().await?; - - let client_ipv4 = ip_pair.ipv4; - let client_ipv6 = ip_pair.ipv6; - - info!("Allocated IPs for peer {peer_key}: {client_ipv4} / {client_ipv6}"); - - // Create WireGuard peer with allocated IPs - let mut peer = Peer::new(defguard_key); - peer.endpoint = None; - peer.allowed_ips = vec![ - format!("{client_ipv4}/32").parse()?, - format!("{client_ipv6}/128").parse()?, - ]; - peer.persistent_keepalive_interval = Some(25); - peer.preshared_key = Some(psk); - - // Store peer in database FIRST (before adding to controller) - // This ensures bandwidth storage exists when controller's generate_bandwidth_manager() is called - let client_id = self - .storage - .insert_wireguard_peer(&peer, ticket_type.into()) - .await - .map_err(|e| { - error!("Failed to store WireGuard peer in database: {}", e); - GatewayError::InternalError(format!("Failed to store peer: {}", e)) - })?; - - // Create bandwidth entry for the client - // This must happen BEFORE AddPeer because generate_bandwidth_manager() expects it to exist - credential_storage_preparation(self.ecash_verifier.clone(), client_id).await?; - - // Now send peer to WireGuard controller and track latency - let controller_start = std::time::Instant::now(); - let result = self.peer_manager.add_peer(peer).await; - - // Record peer controller channel latency - let latency = controller_start.elapsed().as_secs_f64(); - add_histogram_obs!( - "wg_peer_controller_channel_latency_seconds", - latency, - WG_CONTROLLER_LATENCY_BUCKETS - ); - - result?; - - // Get gateway's actual WireGuard public key - let gateway_pubkey = *self.keypair().public_key(); - - // Get gateway's WireGuard endpoint from config - let gateway_endpoint = self.wireguard_config().bind_address; - self.registrations_in_progress.lock().await.insert( - sender, - TimestampedState::new(PendingRegistrationState { - client_id, - peer_key, - ticket_type, - wireguard_config: WireguardConfiguration { - public_key: gateway_pubkey, - psk: None, - endpoint: gateway_endpoint, - private_ipv4: client_ipv4, - private_ipv6: client_ipv6, - }, - }), - ); - Ok(()) - } -} - -// TODO: dedup -/// Prepare bandwidth storage for a client -async fn credential_storage_preparation( - ecash_verifier: Arc, - client_id: i64, -) -> Result { - // Check if bandwidth entry already exists (idempotent) - let existing_bandwidth = ecash_verifier - .storage() - .get_available_bandwidth(client_id) - .await?; - - // Only create if it doesn't exist - if existing_bandwidth.is_none() { - ecash_verifier - .storage() - .create_bandwidth_entry(client_id) - .await?; - } - - let bandwidth = ecash_verifier - .storage() - .get_available_bandwidth(client_id) - .await? - .ok_or_else(|| GatewayError::InternalError("bandwidth entry should exist".to_string()))?; - Ok(bandwidth) -} - -// TODO: dedup -/// Verify credential and allocate bandwidth using CredentialVerifier -async fn credential_verification( - ecash_verifier: Arc, - credential: CredentialSpendingData, - client_id: i64, -) -> Result { - let bandwidth = credential_storage_preparation(ecash_verifier.clone(), client_id).await?; - let client_bandwidth = ClientBandwidth::new(bandwidth.into()); - let mut verifier = CredentialVerifier::new( - CredentialSpendingRequest::new(credential), - ecash_verifier.clone(), - BandwidthStorageManager::new( - ecash_verifier.storage(), - client_bandwidth, - client_id, - BandwidthFlushingBehaviourConfig::default(), - true, - ), - ); - - // Track credential verification attempts - inc!("lp_credential_verification_attempts"); - - // For mock ecash mode (local testing), skip cryptographic verification - // and just return a dummy bandwidth value since we don't have blockchain access - let allocated = if ecash_verifier.is_mock() { - // Return a reasonable test bandwidth value (e.g., 1GB in bytes) - const MOCK_BANDWIDTH: i64 = 1024 * 1024 * 1024; - inc!("lp_credential_verification_success"); - inc_by!("lp_bandwidth_allocated_bytes_total", MOCK_BANDWIDTH); - Ok::(MOCK_BANDWIDTH) - } else { - match verifier.verify().await { - Ok(allocated) => { - inc!("lp_credential_verification_success"); - // Track allocated bandwidth - inc_by!("lp_bandwidth_allocated_bytes_total", allocated); - Ok(allocated) - } - Err(e) => { - inc!("lp_credential_verification_failed"); - Err(e.into()) - } - } - }?; - - Ok(allocated) } diff --git a/gateway/src/node/mod.rs b/gateway/src/node/mod.rs index a8a613f2ec..d8376289a3 100644 --- a/gateway/src/node/mod.rs +++ b/gateway/src/node/mod.rs @@ -38,7 +38,7 @@ use tracing::*; use zeroize::Zeroizing; pub use crate::node::upgrade_mode::watcher::UpgradeModeWatcher; -use crate::node::wireguard::PeerManager; +use crate::node::wireguard::{PeerManager, PeerRegistrator}; pub use client_handling::active_clients::ActiveClientsStore; pub use lp_listener::LpConfig; pub use nym_credential_verification::upgrade_mode::UpgradeModeCheckRequestSender; @@ -299,6 +299,22 @@ impl GatewayTasksBuilder { } } + pub async fn build_peer_registrator( + &mut self, + upgrade_mode_details: UpgradeModeDetails, + ) -> Result, GatewayError> { + let Some(wireguard_data) = &self.wireguard_data else { + return Ok(None); + }; + + let peer_manager = PeerManager::new(wireguard_data.inner.clone()); + Ok(Some(PeerRegistrator::new( + self.ecash_manager().await?, + peer_manager, + upgrade_mode_details, + ))) + } + pub async fn build_websocket_listener( &mut self, active_clients_store: ActiveClientsStore, @@ -330,19 +346,9 @@ impl GatewayTasksBuilder { pub async fn build_lp_listener( &mut self, - upgrade_mode_common_state: UpgradeModeDetails, + peer_registrator: Option, active_clients_store: ActiveClientsStore, ) -> Result { - // Get WireGuard peer controller if available - let Some(wireguard_data) = &self.wireguard_data else { - return Err(GatewayError::InternalWireguardError( - "wireguard not set".to_string(), - )); - }; - - // TODO: combine this `PeerManager` with the one used within the authenticator - let peer_manager = Arc::new(PeerManager::new(wireguard_data.inner.clone())); - let handler_state = lp_listener::LpHandlerState { ecash_verifier: self.ecash_manager().await?, storage: self.storage.clone(), @@ -353,13 +359,11 @@ impl GatewayTasksBuilder { .with_kem_psq_key(self.kem_psq_keys.clone()), metrics: self.metrics.clone(), active_clients_store, - upgrade_mode: upgrade_mode_common_state, - peer_manager, + peer_registrator, lp_config: self.config.lp, outbound_mix_sender: self.mix_packet_sender.clone(), handshake_states: Arc::new(dashmap::DashMap::new()), session_states: Arc::new(dashmap::DashMap::new()), - registrations_in_progress: Default::default(), forward_semaphore: Arc::new(Semaphore::new( self.config.lp.debug.max_concurrent_forwards, )), @@ -495,11 +499,10 @@ impl GatewayTasksBuilder { pub async fn build_wireguard_authenticator( &mut self, + peer_registrator: PeerRegistrator, upgrade_mode_common: UpgradeModeDetails, topology_provider: Box, ) -> Result, GatewayError> { - let ecash_manager = self.ecash_manager().await?; - let Some(opts) = &self.authenticator_opts else { return Err(GatewayError::UnspecifiedAuthenticatorConfig); }; @@ -519,9 +522,9 @@ impl GatewayTasksBuilder { let mut authenticator_server = Authenticator::new( opts.config.clone(), + peer_registrator, upgrade_mode_common, wireguard_data.inner.clone(), - ecash_manager, self.shutdown_tracker.clone(), ) .with_custom_gateway_transceiver(transceiver) diff --git a/gateway/src/node/wireguard/error.rs b/gateway/src/node/wireguard/error.rs index de0a6aaa9f..3b32f12257 100644 --- a/gateway/src/node/wireguard/error.rs +++ b/gateway/src/node/wireguard/error.rs @@ -1,6 +1,7 @@ // Copyright 2026 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 +use nym_credential_verification::upgrade_mode::UpgradeModeEnableError; use thiserror::Error; #[derive(Debug, Error)] @@ -10,10 +11,48 @@ pub enum GatewayWireguardError { #[error("peers can't be interacted with anymore")] PeerInteractionStopped, + + #[error("registration is not in progress for the provided peer key")] + RegistrationNotInProgress, + + #[error("missing reply_to for old client")] + MissingReplyToForOldClient, + + #[error("unknown version number")] + UnknownAuthenticatorVersion, + + #[error("unsupported authenticator version")] + UnsupportedAuthenticatorVersion, + + #[error("mac does not verify")] + AuthenticatorMacVerificationFailure, + + #[error("no credential received")] + MissingAuthenticatorCredential, + + #[error(transparent)] + UpgradeModeEnable(#[from] UpgradeModeEnableError), + + #[error("credential verification failed: {0}")] + CredentialVerificationError(#[from] nym_credential_verification::Error), + + #[error(transparent)] + GatewayStorageError(#[from] nym_gateway_storage::error::GatewayStorageError), + + #[error("failed to serialise authenticator response packet: {source}")] + AuthenticatorResponseSerialisationFailure { source: Box }, } impl GatewayWireguardError { pub fn internal(message: impl Into) -> Self { GatewayWireguardError::InternalError(message.into()) } + + pub fn authenticator_response_serialisation( + source: impl Into>, + ) -> Self { + GatewayWireguardError::AuthenticatorResponseSerialisationFailure { + source: source.into(), + } + } } diff --git a/gateway/src/node/wireguard/mod.rs b/gateway/src/node/wireguard/mod.rs index 7c9a02b8e1..b65654a9d1 100644 --- a/gateway/src/node/wireguard/mod.rs +++ b/gateway/src/node/wireguard/mod.rs @@ -2,7 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 pub mod error; +pub mod new_peer_registration; pub mod peer_manager; pub use error::GatewayWireguardError; +pub use new_peer_registration::PeerRegistrator; pub use peer_manager::PeerManager; diff --git a/gateway/src/node/wireguard/new_peer_registration/authenticator.rs b/gateway/src/node/wireguard/new_peer_registration/authenticator.rs new file mode 100644 index 0000000000..45da0e7ae4 --- /dev/null +++ b/gateway/src/node/wireguard/new_peer_registration/authenticator.rs @@ -0,0 +1,157 @@ +// Copyright 2026 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::node::wireguard::new_peer_registration::helpers::build_final_authenticator_response; +use crate::node::wireguard::new_peer_registration::pending::{ + PendingRegistration, PendingRegistrationData, +}; +use crate::node::wireguard::{GatewayWireguardError, PeerRegistrator}; +use defguard_wireguard_rs::host::Peer; +use nym_authenticator_requests::authenticator_ipv4_to_ipv6; +use nym_authenticator_requests::response::SerialisedResponse; +use nym_registration_common::WireguardRegistrationData; +use nym_sdk::mixnet::Recipient; +use nym_service_provider_requests_common::Protocol; +use nym_wireguard::peer_controller::IpPair; +use nym_wireguard_types::PeerPublicKey; +use std::net::IpAddr; +use std::time::Instant; + +impl PeerRegistrator { + fn authenticator_peer_to_final_response( + &self, + peer: Peer, + protocol: Protocol, + request_id: u64, + reply_to: Option, + ) -> Result { + let allowed_ipv4 = peer + .allowed_ips + .iter() + .find_map(|ip_mask| match ip_mask.address { + IpAddr::V4(ipv4_addr) => Some(ipv4_addr), + _ => None, + }) + .ok_or(GatewayWireguardError::internal( + "there should be one private IPv4 in the list", + ))?; + let allowed_ipv6 = peer + .allowed_ips + .iter() + .find_map(|ip_mask| match ip_mask.address { + IpAddr::V6(ipv6_addr) => Some(ipv6_addr), + _ => None, + }) + .unwrap_or(authenticator_ipv4_to_ipv6(allowed_ipv4)); + + let ip_allocation = IpPair::new(allowed_ipv4, allowed_ipv6); + let wg_port = self.wg_port(); + let local_pub_key = (*self.keypair().public_key()).into(); + let upgrade_mode_enabled = self.upgrade_mode_enabled(); + + build_final_authenticator_response( + ip_allocation, + wg_port, + local_pub_key, + upgrade_mode_enabled, + request_id, + protocol.into(), + reply_to, + ) + } + + pub(super) async fn check_pending_authenticator_registration( + &self, + protocol: Protocol, + request_id: u64, + remote_public: PeerPublicKey, + reply_to: Option, + ) -> Result, GatewayWireguardError> { + let Some(pending_registration) = self + .pending_registrations + .check_authenticator(&remote_public) + .await + else { + return Ok(None); + }; + + Ok(Some( + pending_registration.to_pending_authenticator_response( + self.keypair().private_key(), + self.upgrade_mode_enabled(), + request_id, + protocol.into(), + reply_to, + )?, + )) + } + + pub(super) async fn check_existing_authenticator_peer( + &self, + protocol: Protocol, + request_id: u64, + remote_public: PeerPublicKey, + reply_to: Option, + ) -> Result, GatewayWireguardError> { + let Some(peer) = self.peer_manager.query_peer(remote_public).await? else { + return Ok(None); + }; + Ok(Some(self.authenticator_peer_to_final_response( + peer, protocol, request_id, reply_to, + )?)) + } + + pub(super) fn new_pending_authenticator( + &self, + peer: PeerPublicKey, + ip_allocation: IpPair, + ) -> PendingRegistration { + let nonce: u64 = fastrand::u64(..); + + PendingRegistration { + requested_on: Instant::now(), + data: PendingRegistrationData { + nonce, + peer_key: peer, + psk: None, + wireguard_config: WireguardRegistrationData { + public_key: *self.keypair().public_key(), + port: self.wg_port(), + private_ipv4: ip_allocation.ipv4, + private_ipv6: ip_allocation.ipv6, + }, + }, + } + } + + pub(super) async fn process_fresh_initial_authenticator_registration( + &self, + protocol: Protocol, + request_id: u64, + remote_public: PeerPublicKey, + reply_to: Option, + ) -> Result { + // 1. allocate ip pair + let ip_allocation = self.peer_manager.preallocate_peer_ip_pair().await?; + + let pending = self.new_pending_authenticator(remote_public, ip_allocation); + + // 2. construct response + let response = pending.to_pending_authenticator_response( + self.keypair().private_key(), + self.upgrade_mode_enabled(), + request_id, + protocol.into(), + reply_to, + )?; + + // 3. insert pending data into cache + self.pending_registrations + .authenticator + .write() + .await + .insert(remote_public, pending); + + Ok(response) + } +} diff --git a/gateway/src/node/wireguard/new_peer_registration/helpers.rs b/gateway/src/node/wireguard/new_peer_registration/helpers.rs new file mode 100644 index 0000000000..c301d25c76 --- /dev/null +++ b/gateway/src/node/wireguard/new_peer_registration/helpers.rs @@ -0,0 +1,209 @@ +// Copyright 2026 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::node::wireguard::GatewayWireguardError; +use nym_authenticator_requests::response::SerialisedResponse; +use nym_authenticator_requests::{v1, v2, v3, v4, v5, v6, AuthenticatorVersion}; +use nym_crypto::asymmetric::x25519; +use nym_sdk::mixnet::Recipient; +use nym_wireguard::ip_pool::IpPair; +use nym_wireguard_types::PeerPublicKey; + +#[allow(clippy::too_many_arguments)] +pub(crate) fn build_pending_authenticator_response( + ip_allocation: IpPair, + wg_port: u16, + local_key: &x25519::PrivateKey, + peer_key: PeerPublicKey, + upgrade_mode_enabled: bool, + nonce: u64, + request_id: u64, + version: AuthenticatorVersion, + reply_to: Option, +) -> Result { + let private_ipv4 = ip_allocation.ipv4; + let private_ipv6 = ip_allocation.ipv6; + + let bytes = match version { + AuthenticatorVersion::V1 => Err(GatewayWireguardError::UnsupportedAuthenticatorVersion), + AuthenticatorVersion::V2 => { + v2::response::AuthenticatorResponse::new_pending_registration_success( + v2::registration::RegistrationData { + nonce, + gateway_data: v2::registration::GatewayClient::new( + local_key, + peer_key.inner(), + private_ipv4.into(), + nonce, + ), + wg_port, + }, + request_id, + reply_to.ok_or(GatewayWireguardError::MissingReplyToForOldClient)?, + ) + .to_bytes() + .map_err(GatewayWireguardError::authenticator_response_serialisation) + } + AuthenticatorVersion::V3 => { + v3::response::AuthenticatorResponse::new_pending_registration_success( + v3::registration::RegistrationData { + nonce, + gateway_data: v3::registration::GatewayClient::new( + local_key, + peer_key.inner(), + private_ipv4.into(), + nonce, + ), + wg_port, + }, + request_id, + reply_to.ok_or(GatewayWireguardError::MissingReplyToForOldClient)?, + ) + .to_bytes() + .map_err(GatewayWireguardError::authenticator_response_serialisation) + } + AuthenticatorVersion::V4 => { + v4::response::AuthenticatorResponse::new_pending_registration_success( + v4::registration::RegistrationData { + nonce, + gateway_data: v4::registration::GatewayClient::new( + local_key, + peer_key.inner(), + v4::registration::IpPair::new(private_ipv4, private_ipv6), + nonce, + ), + wg_port, + }, + request_id, + reply_to.ok_or(GatewayWireguardError::MissingReplyToForOldClient)?, + ) + .to_bytes() + .map_err(GatewayWireguardError::authenticator_response_serialisation) + } + AuthenticatorVersion::V5 => { + v5::response::AuthenticatorResponse::new_pending_registration_success( + v5::registration::RegistrationData { + nonce, + gateway_data: v5::registration::GatewayClient::new( + local_key, + peer_key.inner(), + v5::registration::IpPair::new(private_ipv4, private_ipv6), + nonce, + ), + wg_port, + }, + request_id, + ) + .to_bytes() + .map_err(GatewayWireguardError::authenticator_response_serialisation) + } + AuthenticatorVersion::V6 => { + v6::response::AuthenticatorResponse::new_pending_registration_success( + v6::registration::RegistrationData { + nonce, + gateway_data: v6::registration::GatewayClient::new( + local_key, + peer_key.inner(), + v6::registration::IpPair::new(private_ipv4, private_ipv6), + nonce, + ), + wg_port, + }, + request_id, + upgrade_mode_enabled, + ) + .to_bytes() + .map_err(GatewayWireguardError::authenticator_response_serialisation) + } + AuthenticatorVersion::UNKNOWN => { + return Err(GatewayWireguardError::UnknownAuthenticatorVersion) + } + }?; + + Ok(nym_authenticator_requests::response::SerialisedResponse::new(bytes, reply_to)) +} + +pub(crate) fn build_final_authenticator_response( + ip_allocation: IpPair, + wg_port: u16, + pub_key: PeerPublicKey, + upgrade_mode_enabled: bool, + request_id: u64, + version: AuthenticatorVersion, + reply_to: Option, +) -> Result { + let private_ipv4 = ip_allocation.ipv4; + let private_ipv6 = ip_allocation.ipv6; + + let bytes = match version { + AuthenticatorVersion::V1 => v1::response::AuthenticatorResponse::new_registered( + v1::registration::RegisteredData { + pub_key, + private_ip: private_ipv4.into(), + wg_port, + }, + reply_to.ok_or(GatewayWireguardError::MissingReplyToForOldClient)?, + request_id, + ) + .to_bytes() + .map_err(GatewayWireguardError::authenticator_response_serialisation)?, + AuthenticatorVersion::V2 => v2::response::AuthenticatorResponse::new_registered( + v2::registration::RegisteredData { + pub_key, + private_ip: private_ipv4.into(), + wg_port, + }, + reply_to.ok_or(GatewayWireguardError::MissingReplyToForOldClient)?, + request_id, + ) + .to_bytes() + .map_err(GatewayWireguardError::authenticator_response_serialisation)?, + AuthenticatorVersion::V3 => v3::response::AuthenticatorResponse::new_registered( + v3::registration::RegisteredData { + pub_key, + private_ip: private_ipv4.into(), + wg_port, + }, + reply_to.ok_or(GatewayWireguardError::MissingReplyToForOldClient)?, + request_id, + ) + .to_bytes() + .map_err(GatewayWireguardError::authenticator_response_serialisation)?, + AuthenticatorVersion::V4 => v4::response::AuthenticatorResponse::new_registered( + v4::registration::RegisteredData { + pub_key, + private_ips: v4::registration::IpPair::new(private_ipv4, private_ipv6), + wg_port, + }, + reply_to.ok_or(GatewayWireguardError::MissingReplyToForOldClient)?, + request_id, + ) + .to_bytes() + .map_err(GatewayWireguardError::authenticator_response_serialisation)?, + AuthenticatorVersion::V5 => v5::response::AuthenticatorResponse::new_registered( + v5::registration::RegisteredData { + pub_key, + private_ips: v5::registration::IpPair::new(private_ipv4, private_ipv6), + wg_port, + }, + request_id, + ) + .to_bytes() + .map_err(GatewayWireguardError::authenticator_response_serialisation)?, + AuthenticatorVersion::V6 => v6::response::AuthenticatorResponse::new_registered( + v6::registration::RegisteredData { + pub_key, + private_ips: v6::registration::IpPair::new(private_ipv4, private_ipv6), + wg_port, + }, + request_id, + upgrade_mode_enabled, + ) + .to_bytes() + .map_err(GatewayWireguardError::authenticator_response_serialisation)?, + AuthenticatorVersion::UNKNOWN => { + return Err(GatewayWireguardError::UnknownAuthenticatorVersion) + } + }; + Ok(SerialisedResponse::new(bytes, reply_to)) +} diff --git a/gateway/src/node/wireguard/new_peer_registration/lp.rs b/gateway/src/node/wireguard/new_peer_registration/lp.rs new file mode 100644 index 0000000000..dcc505ab94 --- /dev/null +++ b/gateway/src/node/wireguard/new_peer_registration/lp.rs @@ -0,0 +1,128 @@ +// Copyright 2026 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::node::lp_listener::ReceiverIndex; +use crate::node::wireguard::new_peer_registration::pending::{ + PendingRegistration, PendingRegistrationData, +}; +use crate::node::wireguard::{GatewayWireguardError, PeerRegistrator}; +use defguard_wireguard_rs::host::Peer; +use defguard_wireguard_rs::key::Key; +use nym_registration_common::{LpRegistrationResponse, WireguardRegistrationData}; +use nym_wireguard::ip_pool::{allocated_ip_pair, IpPair}; +use nym_wireguard_types::PeerPublicKey; +use std::time::Instant; + +impl PeerRegistrator { + /// In the case of an already registered WG peer, update its PSK. + pub(super) async fn update_peer_psk( + &self, + peer: PeerPublicKey, + psk: Key, + ) -> Result<(), GatewayWireguardError> { + // 1. check if the peer is currently being handled + if self.peer_manager.check_active_peer(peer).await? { + // 2. if so, force disconnect it (as we're handling new request from the same peer) + self.peer_manager.remove_peer(peer).await?; + } + + // 3. update the on-disk PSK + let encoded_psk = psk.to_lower_hex(); + self.ecash_verifier + .storage() + .update_peer_psk(&peer.to_string(), Some(&encoded_psk)) + .await?; + + Ok(()) + } + + fn lp_peer_to_final_response( + &self, + peer: Peer, + ) -> Result, GatewayWireguardError> { + // Incomplete data, treat as new registration + let Some(allocated_ips) = allocated_ip_pair(&peer) else { + return Ok(None); + }; + + Ok(Some(LpRegistrationResponse::success_dvpn( + WireguardRegistrationData { + public_key: *self.keypair().public_key(), + port: self.wg_port(), + private_ipv4: allocated_ips.ipv4, + private_ipv6: allocated_ips.ipv6, + }, + self.upgrade_mode_enabled(), + ))) + } + + pub(super) async fn check_pending_lp_registration( + &self, + sender: ReceiverIndex, + ) -> Result, GatewayWireguardError> { + let Some(pending_registration) = self.pending_registrations.check_lp(sender).await else { + return Ok(None); + }; + + Ok(Some(pending_registration.to_pending_lp_response())) + } + + pub(super) async fn check_existing_lp_peer( + &self, + remote_public: PeerPublicKey, + ) -> Result, GatewayWireguardError> { + let Some(peer) = self.peer_manager.query_peer(remote_public).await? else { + return Ok(None); + }; + + self.lp_peer_to_final_response(peer) + } + + pub(super) fn new_pending_lp( + &self, + peer: PeerPublicKey, + psk: Key, + ip_allocation: IpPair, + ) -> PendingRegistration { + let nonce: u64 = fastrand::u64(..); + + PendingRegistration { + requested_on: Instant::now(), + data: PendingRegistrationData { + nonce, + peer_key: peer, + psk: Some(psk), + wireguard_config: WireguardRegistrationData { + public_key: *self.keypair().public_key(), + port: self.wg_port(), + private_ipv4: ip_allocation.ipv4, + private_ipv6: ip_allocation.ipv6, + }, + }, + } + } + + pub(super) async fn process_fresh_initial_lp_registration( + &self, + sender: ReceiverIndex, + remote_public: PeerPublicKey, + psk: Key, + ) -> Result { + // 1. allocate ip pair + let ip_allocation = self.peer_manager.preallocate_peer_ip_pair().await?; + + let pending = self.new_pending_lp(remote_public, psk, ip_allocation); + + // 2. construct response + let response = pending.to_pending_lp_response(); + + // 3. insert pending data into cache + self.pending_registrations + .lp + .write() + .await + .insert(sender, pending); + + Ok(response) + } +} diff --git a/gateway/src/node/wireguard/new_peer_registration/mod.rs b/gateway/src/node/wireguard/new_peer_registration/mod.rs new file mode 100644 index 0000000000..43dcce7bb0 --- /dev/null +++ b/gateway/src/node/wireguard/new_peer_registration/mod.rs @@ -0,0 +1,391 @@ +// Copyright 2026 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +//! Unification of Nym registration flow +//! In general the registration has the following structure: +//! 1. Initial request message is received +//! 1.1. We check if the peer has already registered before -> if so, we returned the past information +//! 1.2. We check if the peer already has a pending registration -> if so, we return the past information +//! 1.3. We pre-allocated [`nym_wireguard::ip_pool::IpPair`] and save time-sensitive pending registration. +//! If it does not complete within specified time interval, the information is going to get removed. +//! 2. Finalisation request message is received, where credential has to be attached is verified. +//! Upon successful completion, pending registration is transformed into a properly inserted peer. + +use crate::node::lp_listener::ReceiverIndex; +use crate::node::wireguard::new_peer_registration::pending::{ + PendingRegistration, PendingRegistrations, +}; +use crate::node::wireguard::{GatewayWireguardError, PeerManager}; +use defguard_wireguard_rs::host::Peer; +use defguard_wireguard_rs::key::Key; +use defguard_wireguard_rs::net::IpAddrMask; +use nym_authenticator_requests::models::BandwidthClaim; +use nym_authenticator_requests::response::SerialisedResponse; +use nym_authenticator_requests::traits::{FinalMessage, InitMessage}; +use nym_credential_verification::bandwidth_storage_manager::BandwidthStorageManager; +use nym_credential_verification::ecash::traits::EcashManager; +use nym_credential_verification::upgrade_mode::UpgradeModeDetails; +use nym_credential_verification::{ + BandwidthFlushingBehaviourConfig, ClientBandwidth, CredentialVerifier, +}; +use nym_credentials_interface::{BandwidthCredential, CredentialSpendingData}; +use nym_crypto::asymmetric::x25519; +use nym_gateway_requests::models::CredentialSpendingRequest; +use nym_gateway_storage::models::PersistedBandwidth; +use nym_registration_common::dvpn::{ + LpDvpnRegistrationFinalisation, LpDvpnRegistrationInitialRequest, +}; +use nym_registration_common::LpRegistrationResponse; +use nym_sdk::mixnet::Recipient; +use nym_service_provider_requests_common::Protocol; +use nym_task::ShutdownToken; +use nym_wireguard::WireguardConfig; +use std::sync::Arc; +use std::time::Duration; +use tokio::time::{interval_at, Instant}; +use tracing::trace; + +mod authenticator; +mod helpers; +mod lp; +mod pending; + +#[derive(Clone)] +pub struct PeerRegistrator { + /// Handle for the structure managing verification of the ecash credentials for the bandwidth control + pub(crate) ecash_verifier: Arc, + + /// Handle for communication with the [`nym_wireguard::peer_controller::PeerController`] + pub(crate) peer_manager: PeerManager, + + /// Information about the current state of the upgrade mode as well as a handle + /// to remotely trigger the recheck + pub(crate) upgrade_mode: UpgradeModeDetails, + + /// Registrations in progress + pub(crate) pending_registrations: PendingRegistrations, +} + +impl PeerRegistrator { + pub fn new( + ecash_verifier: Arc, + peer_manager: PeerManager, + upgrade_mode: UpgradeModeDetails, + ) -> Self { + PeerRegistrator { + ecash_verifier, + peer_manager, + upgrade_mode, + pending_registrations: Default::default(), + } + } + + pub fn cleanup_task(&self, shutdown_token: ShutdownToken) -> StaleRegistrationRemover { + StaleRegistrationRemover { + pending_registrations: self.pending_registrations.clone(), + shutdown_token, + } + } + + fn upgrade_mode_enabled(&self) -> bool { + self.upgrade_mode.enabled() + } + + fn keypair(&self) -> &Arc { + self.peer_manager.wireguard_gateway_data.keypair() + } + + fn wireguard_config(&self) -> WireguardConfig { + self.peer_manager.wireguard_gateway_data.config() + } + + fn wg_port(&self) -> u16 { + self.wireguard_config().announced_tunnel_port + } + + pub async fn credential_storage_preparation( + &self, + client_id: i64, + ) -> Result { + self.ecash_verifier + .storage() + .create_bandwidth_entry(client_id) + .await?; + + self.ecash_verifier + .storage() + .get_available_bandwidth(client_id) + .await? + .ok_or(GatewayWireguardError::internal( + "missing bandwidth entry after it has just been created", + )) + } + + async fn credential_verification( + &self, + credential: CredentialSpendingData, + client_id: i64, + ) -> Result { + let bandwidth = self.credential_storage_preparation(client_id).await?; + let client_bandwidth = ClientBandwidth::new(bandwidth.into()); + let mut verifier = CredentialVerifier::new( + CredentialSpendingRequest::new(credential), + self.ecash_verifier.clone(), + BandwidthStorageManager::new( + self.ecash_verifier.storage(), + client_bandwidth, + client_id, + BandwidthFlushingBehaviourConfig::default(), + true, + ), + ); + + Ok(verifier.verify().await?) + } + + async fn handle_final_credential_claim( + &self, + claim: BandwidthClaim, + client_id: i64, + ) -> Result<(), GatewayWireguardError> { + match claim.credential { + BandwidthCredential::ZkNym(zk_nym) => { + // if we got zk-nym, we just try to verify it + self.credential_verification(*zk_nym, client_id).await?; + Ok(()) + } + BandwidthCredential::UpgradeModeJWT { token } => { + // if we're already in the upgrade mode, don't bother validating the token + if self.upgrade_mode_enabled() { + return Ok(()); + } + + self.upgrade_mode.try_enable_via_received_jwt(token).await?; + Ok(()) + } + } + } + + /// Attempt to process new peer by: + /// 1. retrieving previous IP allocation + /// 2. inserting it into the storage + /// 3. verifying bandwidth claim and increasing the allowance + /// 4. spawning the peer handler + async fn process_new_peer( + &self, + pending: PendingRegistration, + credential: BandwidthClaim, + ) -> Result<(), GatewayWireguardError> { + // 1. create peer based on the cached registration information + let defguard_key = Key::new(pending.data.peer_key.to_bytes()); + let mut peer = Peer::new(defguard_key); + if let Some(psk) = pending.data.psk { + peer.preshared_key = Some(psk); + } + let private_ipv4 = pending.data.wireguard_config.private_ipv4; + let private_ipv6 = pending.data.wireguard_config.private_ipv6; + peer.allowed_ips = vec![ + IpAddrMask::new(private_ipv4.into(), 32), + IpAddrMask::new(private_ipv6.into(), 128), + ]; + + let typ = credential.kind; + + // 2. attempt to pre-insert peer into the storage + let client_id = self + .ecash_verifier + .storage() + .insert_wireguard_peer(&peer, typ.into()) + .await?; + + // 3. verify the credential + if let Err(err) = self + .handle_final_credential_claim(credential, client_id) + .await + { + // 3.1. on failure -> remove the inserted peer + self.ecash_verifier + .storage() + .remove_wireguard_peer(&peer.public_key.to_string()) + .await?; + return Err(err); + } + + // 4. attempt to start the actual handle for the peer + let public_key = peer.public_key.to_string(); + if let Err(err) = self.peer_manager.add_peer(peer).await { + // 4.1. on failure -> remove the inserted peer (from the storage) + self.ecash_verifier + .storage() + .remove_wireguard_peer(&public_key) + .await?; + return Err(err); + } + + Ok(()) + } + + pub(crate) async fn on_initial_authenticator_request( + &mut self, + init_message: Box, + protocol: Protocol, + request_id: u64, + reply_to: Option, + ) -> Result { + let remote_public = init_message.pub_key(); + + // 1. check if there's any pending registration already in progress, + // if so, return the same data again without additional processing + if let Some(pending_registration) = self + .check_pending_authenticator_registration(protocol, request_id, remote_public, reply_to) + .await? + { + return Ok(pending_registration); + } + + // 2. check if there is already a peer associated with this sender, + // if so, retrieve the "final" data without additional processing + if let Some(existing_registration) = self + .check_existing_authenticator_peer(protocol, request_id, remote_public, reply_to) + .await? + { + return Ok(existing_registration); + } + + // 3. process fresh registration request + self.process_fresh_initial_authenticator_registration( + protocol, + request_id, + remote_public, + reply_to, + ) + .await + } + + pub(crate) async fn on_final_authenticator_request( + &mut self, + final_message: Box, + protocol: Protocol, + request_id: u64, + reply_to: Option, + ) -> Result { + let peer = final_message.gateway_client_pub_key(); + // 1. check if there's any pending registration associated with this peer + let pending_data = self + .pending_registrations + .check_authenticator(&peer) + .await + .ok_or(GatewayWireguardError::RegistrationNotInProgress)? + .clone(); + + // 2. verify the correctness of the received request based on the prior nonce + if final_message + .verify(self.keypair().private_key(), pending_data.data.nonce) + .is_err() + { + return Err(GatewayWireguardError::AuthenticatorMacVerificationFailure); + } + + // 3. ensure we have received a credential + let Some(credential) = final_message.credential() else { + return Err(GatewayWireguardError::MissingAuthenticatorCredential); + }; + + // 4. prepare new peer information and verify the credential + self.process_new_peer(pending_data.clone(), credential) + .await?; + + // 5. remove pending registration + self.pending_registrations.remove_authenticator(&peer).await; + + // 6. construct and return the response + pending_data.to_registered_authenticator_response( + self.upgrade_mode_enabled(), + request_id, + protocol.into(), + reply_to, + ) + } + + pub(crate) async fn on_initial_lp_request( + &self, + init_msg: LpDvpnRegistrationInitialRequest, + sender: ReceiverIndex, + ) -> Result { + let remote_public = init_msg.wg_public_key; + let psk = Key::new(init_msg.psk); + + // 1. check if there's any pending registration already in progress, + // if so, return the same data again without additional processing, + // but update stored PSK + if let Some(pending_registration) = self.check_pending_lp_registration(sender).await? { + self.update_peer_psk(remote_public, psk).await?; + return Ok(pending_registration); + } + + // 2. check if there is already a peer associated with this sender, + // if so, retrieve the "final" data without additional processing, + // but do update stored PSK + if let Some(existing_registration) = self.check_existing_lp_peer(remote_public).await? { + self.update_peer_psk(remote_public, psk).await?; + return Ok(existing_registration); + } + + // 3. process fresh registration request + self.process_fresh_initial_lp_registration(sender, remote_public, psk) + .await + } + + pub(crate) async fn on_final_lp_request( + &self, + final_msg: LpDvpnRegistrationFinalisation, + sender: ReceiverIndex, + ) -> Result { + // 1. check if there's any pending registration associated with this peer + let pending_data = self + .pending_registrations + .check_lp(sender) + .await + .ok_or(GatewayWireguardError::RegistrationNotInProgress)? + .clone(); + + let credential = final_msg.credential; + + // 2. prepare new peer information and verify the credential + self.process_new_peer(pending_data.clone(), credential) + .await?; + + // 3 remove pending registration + self.pending_registrations.remove_lp(sender).await; + + // 4. construct and return the response + Ok(pending_data.to_registered_lp_response(self.upgrade_mode_enabled())) + } +} + +pub struct StaleRegistrationRemover { + pending_registrations: PendingRegistrations, + shutdown_token: ShutdownToken, +} + +impl StaleRegistrationRemover { + // TODO: make it configurable + const STALE_REG_CHECK_INTERVAL: Duration = Duration::from_secs(60); + + pub async fn run(&self) { + let start = Instant::now() + Self::STALE_REG_CHECK_INTERVAL; + let mut interval = interval_at(start, Self::STALE_REG_CHECK_INTERVAL); + loop { + tokio::select! { + biased; + _ = self.shutdown_token.cancelled() => { + trace!("StaleRegistrationRemover: received shutdown"); + break + } + _ = interval.tick() => { + self.pending_registrations.remove_stale_registrations().await + } + } + } + } +} diff --git a/gateway/src/node/wireguard/new_peer_registration/pending.rs b/gateway/src/node/wireguard/new_peer_registration/pending.rs new file mode 100644 index 0000000000..cb933514db --- /dev/null +++ b/gateway/src/node/wireguard/new_peer_registration/pending.rs @@ -0,0 +1,158 @@ +// Copyright 2026 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::node::lp_listener::ReceiverIndex; +use crate::node::wireguard::new_peer_registration::helpers::{ + build_final_authenticator_response, build_pending_authenticator_response, +}; +use crate::node::wireguard::GatewayWireguardError; +use defguard_wireguard_rs::key::Key; +use nym_authenticator_requests::AuthenticatorVersion; +use nym_crypto::asymmetric::x25519; +use nym_registration_common::{LpRegistrationResponse, WireguardRegistrationData}; +use nym_sdk::mixnet::Recipient; +use nym_wireguard::ip_pool::IpPair; +use nym_wireguard_types::PeerPublicKey; +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::RwLock; + +const DEFAULT_PENDING_REGISTRATION_TTL: Duration = Duration::from_secs(120); // 2 minutes + +#[derive(Clone)] +pub struct PendingRegistration { + pub(super) requested_on: Instant, + pub(super) data: PendingRegistrationData, +} + +#[derive(Clone)] +pub struct PendingRegistrationData { + pub(super) nonce: u64, + + pub(super) peer_key: PeerPublicKey, + + // will not be set if registering via the Authenticator + pub(super) psk: Option, + + pub(super) wireguard_config: WireguardRegistrationData, +} + +impl PendingRegistration { + pub(crate) fn to_pending_authenticator_response( + &self, + local_key: &x25519::PrivateKey, + upgrade_mode_enabled: bool, + request_id: u64, + version: AuthenticatorVersion, + reply_to: Option, + ) -> Result + { + let nonce = self.data.nonce; + let remote_public = self.data.peer_key; + let wg_port = self.data.wireguard_config.port; + let ip_allocation = IpPair::new( + self.data.wireguard_config.private_ipv4, + self.data.wireguard_config.private_ipv6, + ); + + build_pending_authenticator_response( + ip_allocation, + wg_port, + local_key, + remote_public, + upgrade_mode_enabled, + nonce, + request_id, + version, + reply_to, + ) + } + + pub(crate) fn to_registered_authenticator_response( + &self, + upgrade_mode_enabled: bool, + request_id: u64, + version: AuthenticatorVersion, + reply_to: Option, + ) -> Result + { + let wg_port = self.data.wireguard_config.port; + let local_pub_key = self.data.wireguard_config.public_key.into(); + + let ip_allocation = IpPair::new( + self.data.wireguard_config.private_ipv4, + self.data.wireguard_config.private_ipv6, + ); + + build_final_authenticator_response( + ip_allocation, + wg_port, + local_pub_key, + upgrade_mode_enabled, + request_id, + version, + reply_to, + ) + } + + pub(crate) fn to_pending_lp_response(&self) -> LpRegistrationResponse { + LpRegistrationResponse::request_dvpn_credential() + } + + pub(crate) fn to_registered_lp_response( + &self, + upgrade_mode_enabled: bool, + ) -> LpRegistrationResponse { + LpRegistrationResponse::success_dvpn(self.data.wireguard_config, upgrade_mode_enabled) + } +} + +#[derive(Clone, Default)] +pub(crate) struct PendingRegistrations { + // TODO: unify those, somehow, later + /// Registrations in progress received from the Authenticator service provider via the + /// [`crate::node::internal_service_providers::authenticator::mixnet_listener::MixnetListener`] + pub(crate) authenticator: Arc>>, + + /// Registrations in progress received from the LP Listener via the + /// [`crate::node::lp_listener::handler::LpConnectionHandler`] and handle through + /// [`crate::node::lp_listener::registration::LpHandlerState`] + pub(crate) lp: Arc>>, +} + +impl PendingRegistrations { + pub(crate) async fn check_authenticator( + &self, + peer: &PeerPublicKey, + ) -> Option { + self.authenticator.read().await.get(peer).cloned() + } + + pub(crate) async fn remove_authenticator(&self, peer: &PeerPublicKey) { + self.authenticator.write().await.remove(peer); + } + + pub(crate) async fn remove_lp(&self, receiver_index: ReceiverIndex) { + self.lp.write().await.remove(&receiver_index); + } + + pub(crate) async fn check_lp( + &self, + receiver_index: ReceiverIndex, + ) -> Option { + self.lp.read().await.get(&receiver_index).cloned() + } + + pub(crate) async fn remove_stale_registrations(&self) { + // note: `IpPool` will release stale pre-allocated addresses by itself during the cleanup, + // so there's no need to send explicit messages over + let now = Instant::now(); + self.authenticator.write().await.retain(|_, pending| { + now.duration_since(pending.requested_on) < DEFAULT_PENDING_REGISTRATION_TTL + }); + self.lp.write().await.retain(|_, pending| { + now.duration_since(pending.requested_on) < DEFAULT_PENDING_REGISTRATION_TTL + }); + } +} diff --git a/gateway/src/node/wireguard/peer_manager.rs b/gateway/src/node/wireguard/peer_manager.rs index 0921b758ca..ffb25840e9 100644 --- a/gateway/src/node/wireguard/peer_manager.rs +++ b/gateway/src/node/wireguard/peer_manager.rs @@ -6,11 +6,28 @@ use defguard_wireguard_rs::{host::Peer, key::Key}; use futures::channel::oneshot; use nym_credential_verification::{ClientBandwidth, TicketVerifier}; use nym_credentials_interface::CredentialSpendingData; +use nym_metrics::add_histogram_obs; use nym_wireguard::peer_controller::IpPair; use nym_wireguard::{peer_controller::PeerControlRequest, WireguardGatewayData}; use nym_wireguard_types::PeerPublicKey; +use std::time::Instant; use tracing::error; +// Histogram buckets for WireGuard peer controller channel latency +// Measures time to send request and receive response from peer controller +// Expected: 1ms-100ms for normal operations, up to 2s for slow conditions +const WG_CONTROLLER_LATENCY_BUCKETS: &[f64] = &[ + 0.001, // 1ms + 0.005, // 5ms + 0.01, // 10ms + 0.05, // 50ms + 0.1, // 100ms + 0.25, // 250ms + 0.5, // 500ms + 1.0, // 1s + 2.0, // 2s +]; + #[derive(Clone)] pub struct PeerManager { pub(crate) wireguard_gateway_data: WireguardGatewayData, @@ -23,16 +40,17 @@ impl PeerManager { } } - pub async fn allocate_peer_ip_pair(&self) -> Result { + pub async fn preallocate_peer_ip_pair(&self) -> Result { + let controller_start = Instant::now(); let (response_tx, response_rx) = oneshot::channel(); - let msg = PeerControlRequest::AllocatePeerIpPair { response_tx }; + let msg = PeerControlRequest::PreAllocateIpPair { response_tx }; self.wireguard_gateway_data .peer_tx() .send(msg) .await .map_err(|_| GatewayWireguardError::PeerInteractionStopped)?; - response_rx + let res = response_rx .await .map_err(|e| { GatewayWireguardError::InternalError(format!( @@ -42,7 +60,16 @@ impl PeerManager { .map_err(|e| { error!("Failed to allocate IPs from pool: {e}"); GatewayWireguardError::InternalError(format!("Failed to allocate IPs: {e}")) - }) + }); + + let latency = controller_start.elapsed().as_secs_f64(); + add_histogram_obs!( + "wg_peer_controller_channel_latency_seconds", + latency, + WG_CONTROLLER_LATENCY_BUCKETS + ); + + res } pub async fn release_ip_pair(&self, ip_pair: IpPair) -> Result<(), GatewayWireguardError> { @@ -70,6 +97,7 @@ impl PeerManager { } pub async fn add_peer(&self, peer: Peer) -> Result<(), GatewayWireguardError> { + let controller_start = Instant::now(); let (response_tx, response_rx) = oneshot::channel(); let msg = PeerControlRequest::AddPeer { peer, response_tx }; self.wireguard_gateway_data @@ -78,17 +106,27 @@ impl PeerManager { .await .map_err(|_| GatewayWireguardError::PeerInteractionStopped)?; - response_rx + let res = response_rx .await .map_err(|_| GatewayWireguardError::internal("no response for add peer".to_string()))? .map_err(|err| { GatewayWireguardError::internal(format!( "adding peer could not be performed: {err:?}" )) - }) + }); + + let latency = controller_start.elapsed().as_secs_f64(); + add_histogram_obs!( + "wg_peer_controller_channel_latency_seconds", + latency, + WG_CONTROLLER_LATENCY_BUCKETS + ); + + res } - pub async fn _remove_peer(&self, pub_key: PeerPublicKey) -> Result<(), GatewayWireguardError> { + pub async fn remove_peer(&self, pub_key: PeerPublicKey) -> Result<(), GatewayWireguardError> { + let controller_start = Instant::now(); let key = Key::new(pub_key.to_bytes()); let (response_tx, response_rx) = oneshot::channel(); let msg = PeerControlRequest::RemovePeer { key, response_tx }; @@ -98,20 +136,63 @@ impl PeerManager { .await .map_err(|_| GatewayWireguardError::PeerInteractionStopped)?; - response_rx + let res = response_rx .await .map_err(|_| GatewayWireguardError::internal("no response for remove peer"))? .map_err(|err| { GatewayWireguardError::InternalError(format!( "removing peer could not be performed: {err:?}" )) - }) + }); + + let latency = controller_start.elapsed().as_secs_f64(); + add_histogram_obs!( + "wg_peer_controller_channel_latency_seconds", + latency, + WG_CONTROLLER_LATENCY_BUCKETS + ); + + res + } + + pub async fn check_active_peer( + &self, + pub_key: PeerPublicKey, + ) -> Result { + let controller_start = Instant::now(); + let key = Key::new(pub_key.to_bytes()); + let (response_tx, response_rx) = oneshot::channel(); + let msg = PeerControlRequest::CheckActivePeer { key, response_tx }; + self.wireguard_gateway_data + .peer_tx() + .send(msg) + .await + .map_err(|_| GatewayWireguardError::PeerInteractionStopped)?; + + let res = response_rx + .await + .map_err(|_| GatewayWireguardError::internal("no response for check active peer"))? + .map_err(|err| { + GatewayWireguardError::InternalError(format!( + "check active peer could not be performed: {err:?}" + )) + }); + + let latency = controller_start.elapsed().as_secs_f64(); + add_histogram_obs!( + "wg_peer_controller_channel_latency_seconds", + latency, + WG_CONTROLLER_LATENCY_BUCKETS + ); + + res } pub async fn query_peer( &self, public_key: PeerPublicKey, ) -> Result, GatewayWireguardError> { + let controller_start = Instant::now(); let key = Key::new(public_key.to_bytes()); let (response_tx, response_rx) = oneshot::channel(); let msg = PeerControlRequest::QueryPeer { key, response_tx }; @@ -121,14 +202,23 @@ impl PeerManager { .await .map_err(|_| GatewayWireguardError::PeerInteractionStopped)?; - response_rx + let res = response_rx .await .map_err(|_| GatewayWireguardError::internal("no response for query peer".to_string()))? .map_err(|err| { GatewayWireguardError::internal(format!( "querying peer could not be performed: {err:?}" )) - }) + }); + + let latency = controller_start.elapsed().as_secs_f64(); + add_histogram_obs!( + "wg_peer_controller_channel_latency_seconds", + latency, + WG_CONTROLLER_LATENCY_BUCKETS + ); + + res } pub async fn query_bandwidth( @@ -143,6 +233,7 @@ impl PeerManager { &self, key: PeerPublicKey, ) -> Result { + let controller_start = Instant::now(); let key = Key::new(key.to_bytes()); let (response_tx, response_rx) = oneshot::channel(); let msg = PeerControlRequest::GetClientBandwidthByKey { key, response_tx }; @@ -152,14 +243,23 @@ impl PeerManager { .await .map_err(|_| GatewayWireguardError::PeerInteractionStopped)?; - response_rx + let res = response_rx .await .map_err(|_| GatewayWireguardError::internal("no response for query client bandwidth"))? .map_err(|err| { GatewayWireguardError::internal(format!( "querying client bandwidth could not be performed: {err:?}" )) - }) + }); + + let latency = controller_start.elapsed().as_secs_f64(); + add_histogram_obs!( + "wg_peer_controller_channel_latency_seconds", + latency, + WG_CONTROLLER_LATENCY_BUCKETS + ); + + res } pub async fn query_verifier_by_key( @@ -167,6 +267,7 @@ impl PeerManager { key: PeerPublicKey, credential: CredentialSpendingData, ) -> Result, GatewayWireguardError> { + let controller_start = Instant::now(); let key = Key::new(key.to_bytes()); let (response_tx, response_rx) = oneshot::channel(); let msg = PeerControlRequest::GetVerifierByKey { @@ -180,7 +281,7 @@ impl PeerManager { .await .map_err(|_| GatewayWireguardError::PeerInteractionStopped)?; - response_rx + let res = response_rx .await .map_err(|_| { GatewayWireguardError::internal("no response for query verifier".to_string()) @@ -189,31 +290,39 @@ impl PeerManager { GatewayWireguardError::internal(format!( "querying verifier could not be performed: {err:?}" )) - }) + }); + + let latency = controller_start.elapsed().as_secs_f64(); + add_histogram_obs!( + "wg_peer_controller_channel_latency_seconds", + latency, + WG_CONTROLLER_LATENCY_BUCKETS + ); + + res } } #[cfg(test)] mod tests { - use std::{str::FromStr, sync::Arc}; - + use super::*; + use crate::node::wireguard::PeerRegistrator; + use crate::nym_authenticator::config::Authenticator; + use defguard_wireguard_rs::net::IpAddrMask; + use nym_credential_verification::upgrade_mode::testing::mock_dummy_upgrade_mode_details; use nym_credential_verification::{ bandwidth_storage_manager::BandwidthStorageManager, ecash::MockEcashManager, }; use nym_credentials_interface::Bandwidth; use nym_crypto::asymmetric::x25519::KeyPair; use nym_gateway_storage::traits::{mock::MockGatewayStorage, BandwidthGatewayStorage}; + use nym_task::ShutdownManager; + use nym_test_utils::helpers::{deterministic_rng, DeterministicRng, RngCore}; use nym_wireguard::peer_controller::{start_controller, stop_controller}; - use rand::rngs::OsRng; + use std::{str::FromStr, sync::Arc}; use time::{Duration, OffsetDateTime}; use tokio::sync::RwLock; - use crate::nym_authenticator::{ - config::Authenticator, mixnet_listener::credential_storage_preparation, - }; - - use super::*; - const CREDENTIAL_BYTES: [u8; 1245] = [ 0, 0, 4, 133, 96, 179, 223, 185, 136, 23, 213, 166, 59, 203, 66, 69, 209, 181, 227, 254, 16, 102, 98, 237, 59, 119, 170, 111, 31, 194, 51, 59, 120, 17, 115, 229, 79, 91, 11, 139, @@ -279,141 +388,202 @@ mod tests { 0, 0, 0, 0, 0, 1, ]; - #[tokio::test] - async fn add_peer() { - let (wireguard_data, request_rx) = WireguardGatewayData::new( - Authenticator::default().into(), - Arc::new(KeyPair::new(&mut OsRng)), - ); - let peer_manager = PeerManager::new(wireguard_data); - let (storage, task_manager) = start_controller( - peer_manager.wireguard_gateway_data.peer_tx().clone(), - request_rx, - ); - let peer = Peer::default(); - let ecash_manager = MockEcashManager::new(Box::new(storage.clone())); - - assert!(peer_manager.add_peer(peer.clone()).await.is_err()); - - let client_id = storage - .insert_wireguard_peer(&peer, FromStr::from_str("entry_wireguard").unwrap()) - .await - .unwrap(); - assert!(peer_manager.add_peer(peer.clone()).await.is_err()); - - credential_storage_preparation(Arc::new(ecash_manager), client_id) - .await - .unwrap(); - peer_manager.add_peer(peer.clone()).await.unwrap(); - - stop_controller(task_manager).await; + struct TestSetup { + rng: DeterministicRng, + _ecash_manager: Arc, + storage: Arc>, + peer_registrator: PeerRegistrator, + peer_manager: PeerManager, + task_manager: ShutdownManager, } - async fn helper_add_peer( - storage: &Arc>, - peer_manager: &mut PeerManager, - ) -> i64 { - let peer = Peer::default(); - let ecash_manager = MockEcashManager::new(Box::new(storage.clone())); - let client_id = storage + struct GeneratedPeer { + peer: Peer, + client_id: i64, + } + + impl GeneratedPeer { + fn key(&self) -> PeerPublicKey { + PeerPublicKey::from_str(self.peer.public_key.to_string().as_str()).unwrap() + } + } + + impl TestSetup { + fn new() -> TestSetup { + let mut rng = deterministic_rng(); + let (wireguard_data, request_rx) = WireguardGatewayData::new( + Authenticator::default().into(), + Arc::new(KeyPair::new(&mut rng)), + ); + + let (upgrade_mode_details, _) = mock_dummy_upgrade_mode_details(); + let peer_manager = PeerManager::new(wireguard_data); + + let (storage, task_manager) = start_controller( + peer_manager.wireguard_gateway_data.peer_tx().clone(), + request_rx, + ); + + let ecash_manager = Arc::new(MockEcashManager::new(Box::new(storage.clone()))); + let peer_registrator = PeerRegistrator::new( + ecash_manager.clone(), + peer_manager.clone(), + upgrade_mode_details, + ); + + TestSetup { + rng, + _ecash_manager: ecash_manager, + storage, + peer_registrator, + peer_manager, + task_manager, + } + } + + async fn peer_with_pre_allocated_ip(&mut self) -> Peer { + let mut peer = Peer::default(); + let mut key = [0u8; 32]; + self.rng.fill_bytes(&mut key); + peer.public_key = Key::new(key); + + let allocation = self.peer_manager.preallocate_peer_ip_pair().await.unwrap(); + peer.allowed_ips = vec![ + IpAddrMask::new(allocation.ipv4.into(), 32), + IpAddrMask::new(allocation.ipv6.into(), 128), + ]; + + peer + } + + async fn _add_peer(&self, peer: &Peer) -> i64 { + let client_id = self + .storage + .insert_wireguard_peer(peer, FromStr::from_str("entry_wireguard").unwrap()) + .await + .unwrap(); + self.peer_registrator + .credential_storage_preparation(client_id) + .await + .unwrap(); + self.peer_manager.add_peer(peer.clone()).await.unwrap(); + client_id + } + + async fn add_peer(&mut self) -> GeneratedPeer { + let peer = self.peer_with_pre_allocated_ip().await; + let client_id = self._add_peer(&peer).await; + + GeneratedPeer { peer, client_id } + } + + async fn finish(self) { + stop_controller(self.task_manager).await + } + } + + #[tokio::test] + async fn assign_peer_ip() -> anyhow::Result<()> { + let test = TestSetup::new(); + + let ip_pair1 = test.peer_manager.preallocate_peer_ip_pair().await?; + let ip_pair2 = test.peer_manager.preallocate_peer_ip_pair().await?; + assert_ne!(ip_pair1, ip_pair2); + + test.finish().await; + + Ok(()) + } + + #[tokio::test] + async fn add_peer() { + let mut test = TestSetup::new(); + let peer = test.peer_with_pre_allocated_ip().await; + + assert!(test.peer_manager.add_peer(peer.clone()).await.is_err()); + + let client_id = test + .storage .insert_wireguard_peer(&peer, FromStr::from_str("entry_wireguard").unwrap()) .await .unwrap(); - credential_storage_preparation(Arc::new(ecash_manager), client_id) + assert!(test.peer_manager.add_peer(peer.clone()).await.is_err()); + + test.peer_registrator + .credential_storage_preparation(client_id) .await .unwrap(); - peer_manager.add_peer(peer.clone()).await.unwrap(); + test.peer_manager.add_peer(peer.clone()).await.unwrap(); - client_id + test.finish().await } #[tokio::test] async fn remove_peer() { - let (wireguard_data, request_rx) = WireguardGatewayData::new( - Authenticator::default().into(), - Arc::new(KeyPair::new(&mut OsRng)), - ); - let mut peer_manager = PeerManager::new(wireguard_data); - let key = Key::default(); - let public_key = PeerPublicKey::from_str(&key.to_string()).unwrap(); - let (storage, task_manager) = start_controller( - peer_manager.wireguard_gateway_data.peer_tx().clone(), - request_rx, - ); + let mut test = TestSetup::new(); + let peer = test.add_peer().await; + let public_key = peer.key(); - helper_add_peer(&storage, &mut peer_manager).await; - peer_manager._remove_peer(public_key).await.unwrap(); + test.peer_manager.remove_peer(public_key).await.unwrap(); - stop_controller(task_manager).await; + test.finish().await } #[tokio::test] async fn query_peer() { - let (wireguard_data, request_rx) = WireguardGatewayData::new( - Authenticator::default().into(), - Arc::new(KeyPair::new(&mut OsRng)), - ); - let mut peer_manager = PeerManager::new(wireguard_data); - let key = Key::default(); - let public_key = PeerPublicKey::from_str(&key.to_string()).unwrap(); - let (storage, task_manager) = start_controller( - peer_manager.wireguard_gateway_data.peer_tx().clone(), - request_rx, - ); + let mut test = TestSetup::new(); + let peer = test.peer_with_pre_allocated_ip().await; + let public_key = PeerPublicKey::from_str(peer.public_key.to_string().as_str()).unwrap(); - assert!(peer_manager.query_peer(public_key).await.unwrap().is_none()); + assert!(test + .peer_manager + .query_peer(public_key) + .await + .unwrap() + .is_none()); - helper_add_peer(&storage, &mut peer_manager).await; - let peer = peer_manager.query_peer(public_key).await.unwrap().unwrap(); - assert_eq!(peer.public_key, key); + test._add_peer(&peer).await; + let peer_query = test + .peer_manager + .query_peer(public_key) + .await + .unwrap() + .unwrap(); + assert_eq!(peer.public_key, peer_query.public_key); - stop_controller(task_manager).await; + test.finish().await } #[tokio::test] async fn query_bandwidth() { - let (wireguard_data, request_rx) = WireguardGatewayData::new( - Authenticator::default().into(), - Arc::new(KeyPair::new(&mut OsRng)), - ); - let mut peer_manager = PeerManager::new(wireguard_data); - let key = Key::default(); - let public_key = PeerPublicKey::from_str(&key.to_string()).unwrap(); - let (storage, task_manager) = start_controller( - peer_manager.wireguard_gateway_data.peer_tx().clone(), - request_rx, - ); + let mut test = TestSetup::new(); + let peer = test.peer_with_pre_allocated_ip().await; + let public_key = PeerPublicKey::from_str(peer.public_key.to_string().as_str()).unwrap(); - assert!(peer_manager.query_bandwidth(public_key).await.is_err()); + assert!(test.peer_manager.query_bandwidth(public_key).await.is_err()); - helper_add_peer(&storage, &mut peer_manager).await; - let available_bandwidth = peer_manager.query_bandwidth(public_key).await.unwrap(); + test._add_peer(&peer).await; + let available_bandwidth = test.peer_manager.query_bandwidth(public_key).await.unwrap(); assert_eq!(available_bandwidth, 0); - stop_controller(task_manager).await; + test.finish().await } #[tokio::test] async fn query_client_bandwidth() { - let (wireguard_data, request_rx) = WireguardGatewayData::new( - Authenticator::default().into(), - Arc::new(KeyPair::new(&mut OsRng)), - ); - let mut peer_manager = PeerManager::new(wireguard_data); - let key = Key::default(); - let public_key = PeerPublicKey::from_str(&key.to_string()).unwrap(); - let (storage, task_manager) = start_controller( - peer_manager.wireguard_gateway_data.peer_tx().clone(), - request_rx, - ); + let mut test = TestSetup::new(); + let peer = test.peer_with_pre_allocated_ip().await; + let public_key = PeerPublicKey::from_str(peer.public_key.to_string().as_str()).unwrap(); - assert!(peer_manager + assert!(test + .peer_manager .query_client_bandwidth(public_key) .await .is_err()); - helper_add_peer(&storage, &mut peer_manager).await; - let available_bandwidth = peer_manager + test._add_peer(&peer).await; + let available_bandwidth = test + .peer_manager .query_client_bandwidth(public_key) .await .unwrap() @@ -421,64 +591,51 @@ mod tests { .await; assert_eq!(available_bandwidth, 0); - stop_controller(task_manager).await; + test.finish().await } #[tokio::test] async fn query_verifier() { - let (wireguard_data, request_rx) = WireguardGatewayData::new( - Authenticator::default().into(), - Arc::new(KeyPair::new(&mut OsRng)), - ); - let mut peer_manager = PeerManager::new(wireguard_data); - let key = Key::default(); - let public_key = PeerPublicKey::from_str(&key.to_string()).unwrap(); - let (storage, task_manager) = start_controller( - peer_manager.wireguard_gateway_data.peer_tx().clone(), - request_rx, - ); + let mut test = TestSetup::new(); + let peer = test.peer_with_pre_allocated_ip().await; + let public_key = PeerPublicKey::from_str(peer.public_key.to_string().as_str()).unwrap(); + let credential = CredentialSpendingData::try_from_bytes(&CREDENTIAL_BYTES).unwrap(); - assert!(peer_manager + assert!(test + .peer_manager .query_verifier_by_key(public_key, credential.clone()) .await .is_err()); - helper_add_peer(&storage, &mut peer_manager).await; - peer_manager + test._add_peer(&peer).await; + test.peer_manager .query_verifier_by_key(public_key, credential) .await .unwrap(); - stop_controller(task_manager).await; + test.finish().await } #[tokio::test] async fn increase_decrease_bandwidth() { - let (wireguard_data, request_rx) = WireguardGatewayData::new( - Authenticator::default().into(), - Arc::new(KeyPair::new(&mut OsRng)), - ); - let mut peer_manager = PeerManager::new(wireguard_data); - let key = Key::default(); - let public_key = PeerPublicKey::from_str(&key.to_string()).unwrap(); + let mut test = TestSetup::new(); + let peer = test.add_peer().await; + let public_key = peer.key(); + let top_up = 42; let consume = 4; - let (storage, task_manager) = start_controller( - peer_manager.wireguard_gateway_data.peer_tx().clone(), - request_rx, - ); - let client_id = helper_add_peer(&storage, &mut peer_manager).await; - let client_bandwidth = peer_manager - .query_client_bandwidth(public_key) + let client_bandwidth = test + .peer_manager + .query_client_bandwidth(peer.key()) .await .unwrap(); let mut bw_manager = BandwidthStorageManager::new( - Box::new(storage), + Box::new(test.storage.clone()), client_bandwidth.clone(), - client_id, + peer.client_id, Default::default(), true, ); @@ -494,7 +651,7 @@ mod tests { assert_eq!(client_bandwidth.available().await, top_up); assert_eq!( - peer_manager.query_bandwidth(public_key).await.unwrap(), + test.peer_manager.query_bandwidth(public_key).await.unwrap(), top_up ); @@ -502,10 +659,10 @@ mod tests { let remaining = top_up - consume; assert_eq!(client_bandwidth.available().await, remaining); assert_eq!( - peer_manager.query_bandwidth(public_key).await.unwrap(), + test.peer_manager.query_bandwidth(public_key).await.unwrap(), remaining ); - stop_controller(task_manager).await; + test.finish().await } } diff --git a/integration-tests/src/lp_registration.rs b/integration-tests/src/lp_registration.rs index 81112582ed..15074efc0e 100644 --- a/integration-tests/src/lp_registration.rs +++ b/integration-tests/src/lp_registration.rs @@ -5,11 +5,8 @@ mod tests { use anyhow::Context; use nym_bandwidth_controller::mock::MockBandwidthController; - use nym_credential_verification::UpgradeModeState; use nym_credential_verification::ecash::MockEcashManager; - use nym_credential_verification::upgrade_mode::{ - UpgradeModeCheckConfig, UpgradeModeCheckRequestSender, UpgradeModeDetails, - }; + use nym_credential_verification::upgrade_mode::testing::mock_dummy_upgrade_mode_details; use nym_credentials_interface::TicketType; use nym_crypto::asymmetric::{ed25519, x25519}; use nym_gateway::GatewayError; @@ -18,7 +15,7 @@ mod tests { LpDebug, LpHandlerState, LpLocalPeer, MixForwardingReceiver, PeerControlRequest, WireguardGatewayData, mix_forwarding_channels, }; - use nym_gateway::node::wireguard::PeerManager; + use nym_gateway::node::wireguard::{PeerManager, PeerRegistrator}; use nym_gateway::node::{ActiveClientsStore, GatewayStorage, LpConfig}; use nym_registration_client::{LpClientError, LpRegistrationClient}; use nym_test_utils::helpers::{CryptoRng, RngCore, u64_seeded_rng}; @@ -166,10 +163,9 @@ mod tests { .unwrap() } - async fn allocate_ip_pair(&mut self) -> IpPair { + fn pre_allocate_ip_pair(&mut self) -> IpPair { self.ip_pool - .allocate() - .await + .pre_allocate() .expect("unexpected ip allocation failure!") } @@ -181,17 +177,6 @@ mod tests { Ok(GatewayStorage::from_connection_pool(conn_pool, 100).await?) } - const DUMMY_ATTESTER_ED25519_PRIVATE_KEY: [u8; 32] = [ - 108, 49, 193, 21, 126, 161, 249, 85, 242, 207, 74, 195, 238, 6, 64, 149, 201, 140, 248, - 163, 122, 170, 79, 198, 87, 85, 36, 29, 243, 92, 64, 161, - ]; - - pub(crate) fn dummy_attester_public_key() -> ed25519::PublicKey { - let private_key = - ed25519::PrivateKey::from_bytes(&Self::DUMMY_ATTESTER_ED25519_PRIVATE_KEY).unwrap(); - private_key.public_key() - } - async fn mock(rng: &mut (impl RngCore + CryptoRng)) -> anyhow::Result { let base = Party::generate(rng); @@ -217,31 +202,24 @@ mod tests { // create wireguard data let (wireguard_data, peer_request_rx) = Self::wireguard_data(&base); - let (um_recheck_tx, um_recheck_rx) = futures::channel::mpsc::unbounded(); - - // TODO: use it if we ever want to test UM - let _ = um_recheck_rx; + let (upgrade_mode_details, _) = mock_dummy_upgrade_mode_details(); // mock the wg peer controller let (mock_peer_controller, peer_controller_state) = mock_peer_controller(peer_request_rx); - let upgrade_mode_state = UpgradeModeState::new(Self::dummy_attester_public_key()); - let upgrade_mode_details = UpgradeModeDetails::new( - UpgradeModeCheckConfig { - // essentially we never want to trigger this in our tests - min_staleness_recheck: Duration::from_nanos(1), - }, - UpgradeModeCheckRequestSender::new(um_recheck_tx), - upgrade_mode_state.clone(), - ); - // registering particular responses for peer controller is up to given test - let peer_manager = Arc::new(PeerManager::new(wireguard_data)); + let ecash_verifier = Arc::new(ecash_verifier); + + let peer_registrator = PeerRegistrator::new( + ecash_verifier.clone(), + PeerManager::new(wireguard_data), + upgrade_mode_details, + ); let lp_state = LpHandlerState { // use mock instance of ecash verifier - ecash_verifier: Arc::new(ecash_verifier), + ecash_verifier, // use in-memory database (no need for persistency) storage, @@ -253,10 +231,6 @@ mod tests { // no clients at the beginning active_clients_store: ActiveClientsStore::new(), - // handles required for wg registration - upgrade_mode: upgrade_mode_details, - peer_manager, - // use default lp config (with enabled flag) lp_config, @@ -269,9 +243,10 @@ mod tests { // we start with empty state session_states: Arc::new(Default::default()), - // sensible default value for tests - registrations_in_progress: Default::default(), forward_semaphore, + + // handles for dealing with new peers + peer_registrator: Some(peer_registrator), }; Ok(Gateway { @@ -444,7 +419,7 @@ mod tests { // 3. register all needed responses for the dvpn registration that will reach the peer controller // 1) peer registration - ip pair allocation - let ip_pair = entry.allocate_ip_pair().await; + let ip_pair = entry.pre_allocate_ip_pair(); let reg_res = Ok::<_, nym_wireguard::Error>(ip_pair); entry @@ -617,7 +592,7 @@ mod tests { // 4. register all needed responses for the dvpn registration that will reach the peer controller // 1) peer registration - ip pair allocation - let entry_ip_pair = entry.allocate_ip_pair().await; + let entry_ip_pair = entry.pre_allocate_ip_pair(); let reg_res = Ok::<_, nym_wireguard::Error>(entry_ip_pair); entry @@ -668,7 +643,7 @@ mod tests { // 10. register all needed responses for the dvpn registration that will reach the peer controller // 1) peer registration - ip pair allocation - let exit_ip_pair = exit.allocate_ip_pair().await; + let exit_ip_pair = exit.pre_allocate_ip_pair(); let reg_res = Ok::<_, nym_wireguard::Error>(exit_ip_pair); exit.register_peer_controller_response( diff --git a/nym-gateway-probe/src/common/netstack.rs b/nym-gateway-probe/src/common/netstack.rs index c7dff8a5cb..7b6f473bed 100644 --- a/nym-gateway-probe/src/common/netstack.rs +++ b/nym-gateway-probe/src/common/netstack.rs @@ -5,6 +5,7 @@ use crate::config::NetstackArgs; use anyhow::Context; use serde::Deserialize; use std::ffi::{CStr, CString}; +use std::net::SocketAddr; mod sys { use std::ffi::{c_char, c_void}; @@ -223,14 +224,14 @@ pub struct TwoHopNetstackRequestGo { pub entry_wg_ip: String, pub entry_private_key: String, pub entry_public_key: String, - pub entry_endpoint: String, + pub entry_endpoint: SocketAddr, pub entry_awg_args: String, // Exit tunnel configuration (connects via forwarder through entry) pub exit_wg_ip: String, pub exit_private_key: String, pub exit_public_key: String, - pub exit_endpoint: String, + pub exit_endpoint: SocketAddr, pub exit_awg_args: String, // Test parameters diff --git a/nym-gateway-probe/src/common/probe_tests.rs b/nym-gateway-probe/src/common/probe_tests.rs index fa0a9aa2c3..0ee8140f77 100644 --- a/nym-gateway-probe/src/common/probe_tests.rs +++ b/nym-gateway-probe/src/common/probe_tests.rs @@ -299,9 +299,6 @@ pub async fn wg_probe_lp( let entry_lp_version = entry_lp_data.lp_version; let exit_lp_version = exit_lp_data.lp_version; - let entry_ip = entry_address.ip(); - let exit_ip = exit_address.ip(); - info!("Starting LP-based WireGuard probe (entry→exit via forwarding)"); let mut wg_outcome = WgProbeResults::default(); @@ -405,9 +402,9 @@ pub async fn wg_probe_lp( // Build WireGuard endpoint addresses // Entry endpoint uses entry_ip (host-reachable) + port from registration - let entry_wg_endpoint = format!("{}:{}", entry_ip, entry_gateway_data.endpoint.port()); + let entry_wg_endpoint = entry_gateway_data.endpoint; // Exit endpoint uses exit_ip + port from registration (forwarded via entry) - let exit_wg_endpoint = format!("{}:{}", exit_ip, exit_gateway_data.endpoint.port()); + let exit_wg_endpoint = exit_gateway_data.endpoint; info!("Two-hop WireGuard configuration:"); info!(" Entry gateway:"); diff --git a/nym-gateway-probe/src/common/wireguard.rs b/nym-gateway-probe/src/common/wireguard.rs index 20a0e7ec8d..2ded0a0ae3 100644 --- a/nym-gateway-probe/src/common/wireguard.rs +++ b/nym-gateway-probe/src/common/wireguard.rs @@ -7,6 +7,7 @@ //! that is shared between different test modes (authenticator-based and LP-based). use nym_config::defaults::{WG_METADATA_PORT, WG_TUN_DEVICE_IP_ADDRESS_V4}; +use std::net::SocketAddr; use tracing::{error, info}; use crate::NetstackArgs; @@ -185,7 +186,7 @@ pub struct TwoHopWgTunnelConfig { /// Entry gateway's WireGuard public key (hex encoded) pub entry_public_key_hex: String, /// Entry WireGuard endpoint address (entry_gateway_ip:port) - pub entry_endpoint: String, + pub entry_endpoint: SocketAddr, /// Entry Amnezia WG args (empty for standard WG) pub entry_awg_args: String, @@ -197,7 +198,7 @@ pub struct TwoHopWgTunnelConfig { /// Exit gateway's WireGuard public key (hex encoded) pub exit_public_key_hex: String, /// Exit WireGuard endpoint address (exit_gateway_ip:port, forwarded via entry) - pub exit_endpoint: String, + pub exit_endpoint: SocketAddr, /// Exit Amnezia WG args (empty for standard WG) pub exit_awg_args: String, } @@ -209,24 +210,24 @@ impl TwoHopWgTunnelConfig { entry_private_ipv4: impl Into, entry_private_key_hex: impl Into, entry_public_key_hex: impl Into, - entry_endpoint: impl Into, + entry_endpoint: SocketAddr, entry_awg_args: impl Into, exit_private_ipv4: impl Into, exit_private_key_hex: impl Into, exit_public_key_hex: impl Into, - exit_endpoint: impl Into, + exit_endpoint: SocketAddr, exit_awg_args: impl Into, ) -> Self { Self { entry_private_ipv4: entry_private_ipv4.into(), entry_private_key_hex: entry_private_key_hex.into(), entry_public_key_hex: entry_public_key_hex.into(), - entry_endpoint: entry_endpoint.into(), + entry_endpoint, entry_awg_args: entry_awg_args.into(), exit_private_ipv4: exit_private_ipv4.into(), exit_private_key_hex: exit_private_key_hex.into(), exit_public_key_hex: exit_public_key_hex.into(), - exit_endpoint: exit_endpoint.into(), + exit_endpoint, exit_awg_args: exit_awg_args.into(), } } @@ -256,14 +257,14 @@ pub fn run_two_hop_tunnel_tests( entry_wg_ip: config.entry_private_ipv4.clone(), entry_private_key: config.entry_private_key_hex.clone(), entry_public_key: config.entry_public_key_hex.clone(), - entry_endpoint: config.entry_endpoint.clone(), + entry_endpoint: config.entry_endpoint, entry_awg_args: config.entry_awg_args.clone(), // Exit tunnel config exit_wg_ip: config.exit_private_ipv4.clone(), exit_private_key: config.exit_private_key_hex.clone(), exit_public_key: config.exit_public_key_hex.clone(), - exit_endpoint: config.exit_endpoint.clone(), + exit_endpoint: config.exit_endpoint, exit_awg_args: config.exit_awg_args.clone(), // Test parameters (use IPv4 config) diff --git a/nym-node/src/node/mod.rs b/nym-node/src/node/mod.rs index f3c6ab940f..a61c522846 100644 --- a/nym-node/src/node/mod.rs +++ b/nym-node/src/node/mod.rs @@ -673,6 +673,26 @@ impl NymNode { let upgrade_mode_common_state = gateway_tasks_builder.build_upgrade_mode_common_state(upgrade_check_request_sender); + // Set WireGuard data early so other builders can access it + if self.config.wireguard.enabled { + let Some(wg_data) = self.wireguard.take() else { + return Err(NymNodeError::WireguardDataUnavailable); + }; + gateway_tasks_builder.set_wireguard_data(wg_data.into()); + } + + let wg_peer_registrator = gateway_tasks_builder + .build_peer_registrator(upgrade_mode_common_state.clone()) + .await?; + + if let Some(wg_peer_registrator) = wg_peer_registrator.as_ref() { + let cleanup_task = wg_peer_registrator.cleanup_task(self.shutdown_token()); + self.shutdown_tracker().try_spawn_named( + async move { cleanup_task.run().await }, + "StaleRegistrationRemover", + ); + }; + // if we're running in entry mode, start the websocket if self.modes().entry { info!( @@ -688,15 +708,6 @@ impl NymNode { self.shutdown_tracker() .try_spawn_named(async move { websocket.run().await }, "EntryWebsocket"); - // Set WireGuard data early so LP listener can access it - // (LP listener needs wg_peer_controller for dVPN registrations) - if self.config.wireguard.enabled { - let Some(wg_data) = self.wireguard.take() else { - return Err(NymNodeError::WireguardDataUnavailable); - }; - gateway_tasks_builder.set_wireguard_data(wg_data.into()); - } - // Start LP listener if enabled info!( "starting the LP listener on {} (data handler on: {})", @@ -704,10 +715,7 @@ impl NymNode { self.config.gateway_tasks.lp.data_bind_address, ); let mut lp_listener = gateway_tasks_builder - .build_lp_listener( - upgrade_mode_common_state.clone(), - active_clients_store.clone(), - ) + .build_lp_listener(wg_peer_registrator.clone(), active_clients_store.clone()) .await?; self.shutdown_tracker() .try_spawn_named(async move { lp_listener.run().await }, "LpListener"); @@ -747,8 +755,16 @@ impl NymNode { gateway_tasks_builder.set_authenticator_opts(config.auth_opts); + let Some(peer_registrator) = wg_peer_registrator else { + return Err(NymNodeError::WireguardDataUnavailable); + }; + let authenticator = gateway_tasks_builder - .build_wireguard_authenticator(upgrade_mode_common_state.clone(), topology_provider) + .build_wireguard_authenticator( + peer_registrator, + upgrade_mode_common_state.clone(), + topology_provider, + ) .await?; let started_authenticator = authenticator.start_service_provider().await?; active_clients_store.insert_embedded(started_authenticator.handle); diff --git a/nym-registration-client/src/lp_client/client.rs b/nym-registration-client/src/lp_client/client.rs index 99a6d08a1f..1ae732dfb6 100644 --- a/nym-registration-client/src/lp_client/client.rs +++ b/nym-registration-client/src/lp_client/client.rs @@ -25,6 +25,7 @@ use nym_lp_transport::traits::LpTransport; use nym_registration_common::dvpn::LpDvpnRegistrationResponseMessageContent; use nym_registration_common::{ LpRegistrationRequest, LpRegistrationResponse, WireguardConfiguration, + WireguardRegistrationData, }; use nym_wireguard_types::PeerPublicKey; use rand::{CryptoRng, RngCore}; @@ -748,7 +749,7 @@ where gateway_identity: ed25519::PublicKey, bandwidth_controller: &dyn BandwidthTicketProvider, ticket_type: TicketType, - ) -> Result { + ) -> Result { tracing::debug!("Acquiring bandwidth credential for registration"); // 1. Get bandwidth credential from controller @@ -805,14 +806,7 @@ where tracing::warn!("Gateway rejected registration: {reason}"); Err(LpClientError::RegistrationRejected { reason }) } - LpDvpnRegistrationResponseMessageContent::CompletedRegistration(res) => { - // we have managed to complete the registration - tracing::info!( - "LP registration successful! Allocated bandwidth: {} bytes", - res.available_bandwidth - ); - Ok(res.config) - } + LpDvpnRegistrationResponseMessageContent::CompletedRegistration(res) => Ok(res.config), LpDvpnRegistrationResponseMessageContent::RequiresCredential(_) => { Err(LpClientError::unexpected_response( "received request for additional dvpn data after sending credential!", @@ -860,7 +854,7 @@ where let wg_public_key = PeerPublicKey::from(*wg_keypair.public_key()); let mut psk = [0u8; 32]; rng.fill_bytes(&mut psk); - let request = LpRegistrationRequest::new_initial_dvpn(wg_public_key, psk, ticket_type); + let request = LpRegistrationRequest::new_initial_dvpn(wg_public_key, psk); tracing::trace!("Built dVPN registration request: {request:?}"); @@ -896,14 +890,7 @@ where tracing::warn!("Gateway rejected registration: {reason}"); return Err(LpClientError::RegistrationRejected { reason }); } - LpDvpnRegistrationResponseMessageContent::CompletedRegistration(res) => { - // we have already registered with this gateway before, the gateway has updated the psk and sent us the config - tracing::info!( - "LP registration successful! Allocated bandwidth: {} bytes", - res.available_bandwidth - ); - res.config - } + LpDvpnRegistrationResponseMessageContent::CompletedRegistration(res) => res.config, LpDvpnRegistrationResponseMessageContent::RequiresCredential(_) => { // we're registering for the first time with this gateway - we need to attach a credential @@ -920,7 +907,7 @@ where Ok(WireguardConfiguration { public_key: final_response.public_key, psk: Some(psk), - endpoint: SocketAddr::new(self.gateway_lp_address.ip(), final_response.endpoint.port()), + endpoint: SocketAddr::new(self.gateway_lp_address.ip(), final_response.port), private_ipv4: final_response.private_ipv4, private_ipv6: final_response.private_ipv6, }) diff --git a/nym-registration-client/src/lp_client/nested_session.rs b/nym-registration-client/src/lp_client/nested_session.rs index 887338f2e0..631206785c 100644 --- a/nym-registration-client/src/lp_client/nested_session.rs +++ b/nym-registration-client/src/lp_client/nested_session.rs @@ -38,6 +38,7 @@ use nym_lp_transport::traits::LpTransport; use nym_registration_common::dvpn::LpDvpnRegistrationResponseMessageContent; use nym_registration_common::{ LpRegistrationRequest, LpRegistrationResponse, WireguardConfiguration, + WireguardRegistrationData, }; use nym_wireguard_types::PeerPublicKey; use rand::{CryptoRng, RngCore}; @@ -372,7 +373,7 @@ impl NestedLpSession { gateway_identity: ed25519::PublicKey, bandwidth_controller: &dyn BandwidthTicketProvider, ticket_type: TicketType, - ) -> Result + ) -> Result where S: LpTransport + Unpin, { @@ -433,14 +434,7 @@ impl NestedLpSession { tracing::warn!("Gateway rejected registration: {reason}"); Err(LpClientError::RegistrationRejected { reason }) } - LpDvpnRegistrationResponseMessageContent::CompletedRegistration(res) => { - // we have managed to complete the registration - tracing::info!( - "LP registration successful! Allocated bandwidth: {} bytes", - res.available_bandwidth - ); - Ok(res.config) - } + LpDvpnRegistrationResponseMessageContent::CompletedRegistration(res) => Ok(res.config), LpDvpnRegistrationResponseMessageContent::RequiresCredential(_) => { Err(LpClientError::unexpected_response( "received request for additional dvpn data after sending credential!", @@ -499,7 +493,7 @@ impl NestedLpSession { let mut psk = [0u8; 32]; rng.fill_bytes(&mut psk); - let request = LpRegistrationRequest::new_initial_dvpn(wg_public_key, psk, ticket_type); + let request = LpRegistrationRequest::new_initial_dvpn(wg_public_key, psk); // Step 3: Serialize the request let send_data = request.to_lp_data()?; @@ -536,14 +530,7 @@ impl NestedLpSession { tracing::warn!("Gateway rejected registration: {reason}"); return Err(LpClientError::RegistrationRejected { reason }); } - LpDvpnRegistrationResponseMessageContent::CompletedRegistration(res) => { - // we have already registered with this gateway before, the gateway has updated the psk and sent us the config - tracing::info!( - "LP registration successful! Allocated bandwidth: {} bytes", - res.available_bandwidth - ); - res.config - } + LpDvpnRegistrationResponseMessageContent::CompletedRegistration(res) => res.config, LpDvpnRegistrationResponseMessageContent::RequiresCredential(_) => { // we're registering for the first time with this gateway - we need to attach a credential @@ -562,7 +549,7 @@ impl NestedLpSession { Ok(WireguardConfiguration { public_key: final_response.public_key, psk: Some(psk), - endpoint: SocketAddr::new(self.exit_address.ip(), final_response.endpoint.port()), + endpoint: SocketAddr::new(self.exit_address.ip(), final_response.port), private_ipv4: final_response.private_ipv4, private_ipv6: final_response.private_ipv6, })