diff --git a/Cargo.lock b/Cargo.lock index 351e140330..25ac7b4776 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4108,6 +4108,7 @@ name = "integration-tests" version = "0.1.0" dependencies = [ "anyhow", + "futures", "nym-bandwidth-controller", "nym-credential-verification", "nym-credentials-interface", @@ -6906,6 +6907,7 @@ version = "0.1.0" dependencies = [ "nym-test-utils", "tokio", + "tracing", ] [[package]] @@ -7525,6 +7527,7 @@ dependencies = [ "nym-test-utils", "nym-wireguard-types", "serde", + "tracing", ] [[package]] diff --git a/common/bandwidth-controller/src/mock.rs b/common/bandwidth-controller/src/mock.rs index d889c83987..5bd7a04ebe 100644 --- a/common/bandwidth-controller/src/mock.rs +++ b/common/bandwidth-controller/src/mock.rs @@ -21,7 +21,7 @@ pub struct MockBandwidthController { impl BandwidthTicketProvider for MockBandwidthController { async fn get_ecash_ticket( &self, - _ticket_type: TicketType, + ticket_type: TicketType, _gateway_id: PublicKey, tickets_to_spend: u32, ) -> Result { @@ -100,6 +100,10 @@ impl BandwidthTicketProvider for MockBandwidthController { let mut credential = CredentialSpendingData::try_from_bytes(&CREDENTIAL_BYTES) .expect("Failed to deserialize test credential - this is a bug in the test harness"); + // change the ticket type to the requested ticket + // note that verification outside mocks is going to fail + credential.payment.t_type = ticket_type.to_repr() as u8; + // Update spend_date to today to pass validation credential.spend_date = OffsetDateTime::now_utc().date(); diff --git a/common/bandwidth-controller/src/traits.rs b/common/bandwidth-controller/src/traits.rs index d611649004..bb958f482b 100644 --- a/common/bandwidth-controller/src/traits.rs +++ b/common/bandwidth-controller/src/traits.rs @@ -57,3 +57,22 @@ where Ok(Some(token)) } } + +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +impl BandwidthTicketProvider for Box { + async fn get_ecash_ticket( + &self, + ticket_type: TicketType, + gateway_id: ed25519::PublicKey, + tickets_to_spend: u32, + ) -> Result { + (**self) + .get_ecash_ticket(ticket_type, gateway_id, tickets_to_spend) + .await + } + + async fn get_upgrade_mode_token(&self) -> Result, BandwidthControllerError> { + (**self).get_upgrade_mode_token().await + } +} diff --git a/common/client-core/src/client/base_client/storage/mod.rs b/common/client-core/src/client/base_client/storage/mod.rs index 5f24c8ef7f..bc92fb86bc 100644 --- a/common/client-core/src/client/base_client/storage/mod.rs +++ b/common/client-core/src/client/base_client/storage/mod.rs @@ -26,7 +26,7 @@ use crate::{ error::ClientCoreError, }; #[cfg(all(not(target_arch = "wasm32"), feature = "fs-credentials-storage"))] -use nym_credential_storage::persistent_storage::PersistentStorage as PersistentCredentialStorage; +pub use nym_credential_storage::persistent_storage::PersistentStorage as PersistentCredentialStorage; pub use nym_client_core_gateways_storage as gateways_storage; pub use nym_client_core_gateways_storage::{GatewaysDetailsStore, InMemGatewaysDetails}; diff --git a/common/nym-lp-transport/Cargo.toml b/common/nym-lp-transport/Cargo.toml index 929d68da37..d4448e49c8 100644 --- a/common/nym-lp-transport/Cargo.toml +++ b/common/nym-lp-transport/Cargo.toml @@ -12,8 +12,9 @@ readme.workspace = true publish = false [dependencies] -tokio = { workspace = true, features = ["net"] } +tokio = { workspace = true, features = ["net", "io-util"] } nym-test-utils = { path = "../test-utils", optional = true } +tracing = { workspace = true } [features] io-mocks = ["nym-test-utils"] diff --git a/common/nym-lp-transport/src/traits.rs b/common/nym-lp-transport/src/traits.rs index 5b3d300884..888c800f64 100644 --- a/common/nym-lp-transport/src/traits.rs +++ b/common/nym-lp-transport/src/traits.rs @@ -4,8 +4,9 @@ #[cfg(feature = "io-mocks")] use nym_test_utils::mocks::async_read_write::MockIOStream; use std::net::SocketAddr; -use tokio::io::{AsyncRead, AsyncWrite}; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use tokio::net::TcpStream; +use tracing::debug; // only used in internal code (and tests) #[allow(async_fn_in_trait)] @@ -13,6 +14,78 @@ pub trait LpTransport: AsyncRead + AsyncWrite + Sized { async fn connect(endpoint: SocketAddr) -> std::io::Result; fn set_no_delay(&mut self, nodelay: bool) -> std::io::Result<()>; + + /// Sends a serialised (and optionally encrypted) LP packet over the data stream with length-prefixed framing. + /// + /// Format: 4-byte big-endian u32 length + packet bytes + /// + /// # Arguments + /// * `packet_data` - The serialised LP packet to send + /// + /// # Errors + /// Returns an error on network transmission fails. + async fn send_serialised_packet(&mut self, packet_data: &[u8]) -> std::io::Result<()> + where + Self: Unpin, + { + // Send 4-byte length prefix (u32 big-endian) + let len = packet_data.len() as u32; + self.write_all(&len.to_be_bytes()) + .await + .inspect_err(|e| debug!("Failed to send packet length: {e}"))?; + + // Send the actual packet data + self.write_all(packet_data) + .await + .inspect_err(|e| debug!("Failed to send packet data: {e}"))?; + + // Flush to ensure data is sent immediately + self.flush() + .await + .inspect_err(|e| debug!("Failed to flush stream: {e}"))?; + + tracing::trace!( + "Sent LP packet ({} bytes + 4 byte header)", + packet_data.len() + ); + Ok(()) + } + + /// Receives an LP packet from a TCP stream with length-prefixed framing. + /// + /// Format: 4-byte big-endian u32 length + packet bytes + /// + /// # Errors + /// Returns an error on network transmission fails. + async fn receive_raw_packet(&mut self) -> std::io::Result> + where + Self: Unpin, + { + // Read 4-byte length prefix (u32 big-endian) + let mut len_buf = [0u8; 4]; + self.read_exact(&mut len_buf) + .await + .inspect_err(|e| debug!("Failed to read packet length: {e}"))?; + + let packet_len = u32::from_be_bytes(len_buf) as usize; + + // Sanity check to prevent huge allocations + const MAX_PACKET_SIZE: usize = 65536; // 64KB max + if packet_len > MAX_PACKET_SIZE { + return Err(std::io::Error::other(format!( + "Packet size {packet_len} exceeds maximum {MAX_PACKET_SIZE}", + ))); + } + + // Read the actual packet data + let mut packet_buf = vec![0u8; packet_len]; + self.read_exact(&mut packet_buf) + .await + .inspect_err(|e| debug!("Failed to read packet data: {e}"))?; + + tracing::trace!("Received LP packet ({packet_len} bytes + 4 byte header)"); + Ok(packet_buf) + } } impl LpTransport for TcpStream { diff --git a/common/nym-lp/src/message.rs b/common/nym-lp/src/message.rs index 8447b44684..fb836e63f5 100644 --- a/common/nym-lp/src/message.rs +++ b/common/nym-lp/src/message.rs @@ -239,6 +239,18 @@ pub struct ForwardPacketData { } impl ForwardPacketData { + pub fn new( + target_gateway_identity: ed25519::PublicKey, + target_lp_address: String, + inner_packet_bytes: Vec, + ) -> Self { + ForwardPacketData { + target_gateway_identity: target_gateway_identity.to_bytes(), + target_lp_address, + inner_packet_bytes, + } + } + fn len(&self) -> usize { // 32 bytes target gateway identity // + diff --git a/common/nym-lp/src/peer.rs b/common/nym-lp/src/peer.rs index 93f1fb2746..fd6e5ab7f3 100644 --- a/common/nym-lp/src/peer.rs +++ b/common/nym-lp/src/peer.rs @@ -1,6 +1,7 @@ // Copyright 2026 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 +use crate::ClientHelloData; use nym_crypto::asymmetric::{ed25519, x25519}; use nym_kkt::ciphersuite::{KEM, KEMKeyDigests, SignatureScheme, SigningKeyDigests}; use std::collections::HashMap; @@ -29,6 +30,14 @@ impl LpLocalPeer { } } + pub fn build_client_hello_data(&self, timestamp: u64) -> ClientHelloData { + ClientHelloData::new_with_fresh_salt( + *self.x25519().public_key(), + *self.ed25519().public_key(), + timestamp, + ) + } + #[must_use] pub fn with_kem_psq_key(mut self, key: Arc) -> Self { self.kem_psq = Some(key); diff --git a/common/registration/Cargo.toml b/common/registration/Cargo.toml index 230833559c..c361b21461 100644 --- a/common/registration/Cargo.toml +++ b/common/registration/Cargo.toml @@ -15,6 +15,7 @@ workspace = true [dependencies] bincode = { workspace = true } serde = { workspace = true, features = ["derive"] } +tracing = { workspace = true } nym-authenticator-requests = { workspace = true } nym-credentials-interface = { workspace = true } diff --git a/common/registration/src/lib.rs b/common/registration/src/lib.rs index 96d23bcf0f..ddc46a593f 100644 --- a/common/registration/src/lib.rs +++ b/common/registration/src/lib.rs @@ -11,10 +11,7 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; -pub use lp_messages::{ - LpDvpnRegistrationRequest, LpMixnetGatewayData, LpMixnetRegistrationRequest, - LpRegistrationData, LpRegistrationRequest, LpRegistrationResponse, RegistrationMode, -}; +pub use lp_messages::*; pub use serialisation::BincodeError; mod lp_messages; @@ -30,7 +27,7 @@ pub struct NymNodeInformation { pub version: AuthenticatorVersion, } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] pub struct WireguardConfiguration { #[serde(with = "bs58_x25519_pubkey")] pub public_key: x25519::PublicKey, diff --git a/common/registration/src/lp_messages.rs b/common/registration/src/lp_messages.rs index 5e36fb4885..9a7bcc59fb 100644 --- a/common/registration/src/lp_messages.rs +++ b/common/registration/src/lp_messages.rs @@ -4,65 +4,22 @@ //! LP (Lewes Protocol) registration message types shared between client and gateway. use crate::WireguardConfiguration; +use crate::dvpn::{ + LpDvpnRegistrationFinalisation, LpDvpnRegistrationInitialRequest, + LpDvpnRegistrationRequestMessage, LpDvpnRegistrationRequestMessageContent, + LpDvpnRegistrationResponseMessage, LpDvpnRegistrationResponseMessageContent, + RequiresCredentialResponse, +}; +use crate::mixnet::{ + LpMixnetGatewayData, LpMixnetRegistrationRequestMessage, LpMixnetRegistrationResponseMessage, + LpMixnetRegistrationResponseMessageContent, +}; use crate::serialisation::{BincodeError, BincodeOptions, lp_bincode_serializer}; -use nym_credentials_interface::{CredentialSpendingData, TicketType}; +use nym_authenticator_requests::models::BandwidthClaim; +use nym_credentials_interface::TicketType; use nym_crypto::aes::cipher::crypto_common::rand_core::{CryptoRng, RngCore}; -use nym_crypto::asymmetric::ed25519; use serde::{Deserialize, Serialize}; - -/// Registration request sent by client after LP handshake -/// Aligned with existing authenticator registration flow -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct LpRegistrationRequest { - /// Mode specific registration data - pub registration_data: LpRegistrationData, - - /// Unix timestamp for replay protection - pub timestamp: u64, -} - -impl LpRegistrationRequest { - pub fn mode(&self) -> RegistrationMode { - match self.registration_data { - LpRegistrationData::Dvpn { .. } => RegistrationMode::Dvpn, - LpRegistrationData::Mixnet { .. } => RegistrationMode::Mixnet, - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum LpRegistrationData { - /// dVPN mode - register as WireGuard peer (most common) - Dvpn { - data: Box, - }, - - /// Mixnet mode - register for mixnet routing via IPR - Mixnet { data: LpMixnetRegistrationRequest }, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct LpDvpnRegistrationRequest { - /// Client's WireGuard public key (for dVPN mode) - pub wg_public_key: nym_wireguard_types::PeerPublicKey, - - /// Bandwidth credential for payment - pub credential: CredentialSpendingData, - - /// Ticket type for bandwidth allocation - pub ticket_type: TicketType, - - /// Preshared key to be used for the connection - pub psk: [u8; 32], -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct LpMixnetRegistrationRequest { - /// Client's ed25519 public key (identity) - /// - /// Used to derive DestinationAddressBytes for ActiveClientsStore lookup. - pub client_ed25519_pubkey: ed25519::PublicKey, -} +use tracing::error; #[derive(Debug, Clone, Serialize, Deserialize)] pub enum RegistrationMode { @@ -73,49 +30,112 @@ pub enum RegistrationMode { Mixnet, } -/// Gateway data for mixnet mode registration -/// -/// Contains the gateway's identity and sphinx key needed for the client -/// to construct its full nym Recipient address. +/// Registration request sent by client after LP handshake +/// Aligned with existing authenticator registration flow #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct LpMixnetGatewayData { - /// Gateway's ed25519 identity public key - /// - /// Forms part of the client's nym Recipient address. - pub gateway_identity: ed25519::PublicKey, - // TODO: what we really need in here is the address of internal IPR +pub struct LpRegistrationRequest { + /// Mode specific registration data + pub registration_data: LpRegistrationRequestData, + + /// Unix timestamp for replay protection + pub timestamp: u64, +} + +impl LpRegistrationRequest { + pub fn mode(&self) -> RegistrationMode { + match self.registration_data { + LpRegistrationRequestData::Dvpn { .. } => RegistrationMode::Dvpn, + LpRegistrationRequestData::Mixnet { .. } => RegistrationMode::Mixnet, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum LpRegistrationRequestData { + /// dVPN mode - register as WireGuard peer (most common) + Dvpn { + data: Box, + }, + + /// Mixnet mode - register for mixnet routing via IPR + Mixnet { + data: LpMixnetRegistrationRequestMessage, + }, } /// Registration response from gateway /// Contains GatewayData for compatibility with existing client code #[derive(Debug, Clone, Serialize, Deserialize)] pub struct LpRegistrationResponse { - /// Whether registration succeeded - pub success: bool, + /// The status of this registration after the last received client message + pub status: RegistrationStatus, - /// Error message if registration failed - pub error: Option, + /// Mode specific registration response + pub response_data: LpRegistrationResponseData, +} - /// Gateway configuration data for dVPN mode (WireGuard) - /// This matches what WireguardRegistrationResult expects - pub gateway_data: Option, +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum LpRegistrationResponseData { + /// dVPN mode - register as WireGuard peer (most common) + Dvpn { + data: LpDvpnRegistrationResponseMessage, + }, - /// Gateway data for mixnet mode - /// - /// Contains gateway identity and sphinx key needed for nym address construction. - /// Only populated for Mixnet mode registrations. - pub lp_gateway_data: Option, + /// Mixnet mode - register for mixnet routing via IPR + Mixnet { + data: LpMixnetRegistrationResponseMessage, + }, +} - /// Allocated bandwidth in bytes - pub allocated_bandwidth: i64, +/// Represents the registration status after the last received client message. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum RegistrationStatus { + /// The registration has been completed successfully + Completed, + + /// The registration has failed + Failed, + + /// To complete registration the client needs to send additional data, + /// e.g. a credential. it is context dependent. + PendingMoreData, +} + +impl RegistrationStatus { + pub fn is_successful(&self) -> bool { + matches!(self, RegistrationStatus::Completed) + } + + pub fn is_failed(&self) -> bool { + matches!(self, RegistrationStatus::Failed) + } + + pub fn is_pending(&self) -> bool { + matches!(self, RegistrationStatus::PendingMoreData) + } +} + +fn current_timestamp() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .inspect_err(|_| error!("the current timestamp predates unix epoch!")) + .unwrap_or_default() + .as_secs() } impl LpRegistrationRequest { - /// Create a new dVPN registration request - pub fn new_dvpn( + /// Helper wrapping timestamp extraction + fn new(registration_data: LpRegistrationRequestData) -> LpRegistrationRequest { + Self { + registration_data, + timestamp: current_timestamp(), + } + } + + /// Create new dVPN registration initialisation request + pub fn new_initial_dvpn( rng: &mut R, wg_public_key: nym_wireguard_types::PeerPublicKey, - credential: CredentialSpendingData, ticket_type: TicketType, ) -> Self where @@ -124,29 +144,32 @@ impl LpRegistrationRequest { let mut psk = [0u8; 32]; rng.fill_bytes(&mut psk); - Self { - registration_data: LpRegistrationData::Dvpn { - data: Box::new(LpDvpnRegistrationRequest { - wg_public_key, - credential, - ticket_type, - psk, - }), - }, - #[allow(clippy::expect_used)] - timestamp: std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .expect("System time before UNIX epoch") - .as_secs(), - } + Self::new(LpRegistrationRequestData::Dvpn { + data: Box::new(LpDvpnRegistrationRequestMessage { + content: LpDvpnRegistrationRequestMessageContent::InitialRequest( + LpDvpnRegistrationInitialRequest { + wg_public_key, + psk, + ticket_type, + }, + ), + }), + }) + } + + pub fn new_finalise_dvpn(credential: BandwidthClaim) -> Self { + Self::new(LpRegistrationRequestData::Dvpn { + data: Box::new(LpDvpnRegistrationRequestMessage { + content: LpDvpnRegistrationRequestMessageContent::Finalisation( + LpDvpnRegistrationFinalisation { credential }, + ), + }), + }) } /// Validate the request timestamp is within acceptable bounds pub fn validate_timestamp(&self, max_skew_secs: u64) -> bool { - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); + let now = current_timestamp(); (now as i64 - self.timestamp as i64).abs() <= max_skew_secs as i64 } @@ -164,35 +187,95 @@ impl LpRegistrationRequest { impl LpRegistrationResponse { /// Create a success response with GatewayData (for dVPN mode) - pub fn success(allocated_bandwidth: i64, gateway_data: WireguardConfiguration) -> Self { + pub fn success_dvpn(config: WireguardConfiguration, available_bandwidth: i64) -> Self { Self { - success: true, - error: None, - gateway_data: Some(gateway_data), - lp_gateway_data: None, - allocated_bandwidth, + status: RegistrationStatus::Completed, + response_data: LpRegistrationResponseData::Dvpn { + data: LpDvpnRegistrationResponseMessage { + content: LpDvpnRegistrationResponseMessageContent::CompletedRegistration( + dvpn::CompletedRegistrationResponse { + config, + available_bandwidth, + }, + ), + }, + }, } } - /// Create a success response for mixnet mode with LpGatewayData - pub fn success_mixnet(allocated_bandwidth: i64, lp_gateway_data: LpMixnetGatewayData) -> Self { + pub fn success_mixnet(config: LpMixnetGatewayData, available_bandwidth: i64) -> Self { Self { - success: true, - error: None, - gateway_data: None, - lp_gateway_data: Some(lp_gateway_data), - allocated_bandwidth, + status: RegistrationStatus::Completed, + response_data: LpRegistrationResponseData::Mixnet { + data: LpMixnetRegistrationResponseMessage { + content: LpMixnetRegistrationResponseMessageContent::CompletedRegistration( + mixnet::CompletedRegistrationResponse { + config, + available_bandwidth, + }, + ), + }, + }, } } /// Create an error response - pub fn error(error: String) -> Self { - Self { - success: false, - error: Some(error), - gateway_data: None, - lp_gateway_data: None, - allocated_bandwidth: 0, + pub fn error(error: impl Into, mode: RegistrationMode) -> Self { + let response_data = match mode { + RegistrationMode::Dvpn => LpRegistrationResponseData::Dvpn { + data: LpDvpnRegistrationResponseMessage::error(error), + }, + RegistrationMode::Mixnet => LpRegistrationResponseData::Mixnet { + data: LpMixnetRegistrationResponseMessage::error(error), + }, + }; + LpRegistrationResponse { + status: RegistrationStatus::Failed, + response_data, + } + } + + pub fn request_dvpn_credential() -> Self { + LpRegistrationResponse { + status: RegistrationStatus::PendingMoreData, + response_data: LpRegistrationResponseData::Dvpn { + data: LpDvpnRegistrationResponseMessage { + content: LpDvpnRegistrationResponseMessageContent::RequiresCredential( + RequiresCredentialResponse, + ), + }, + }, + } + } + + pub fn into_dvpn_response(self) -> Option { + match self.response_data { + LpRegistrationResponseData::Dvpn { data } => Some(data), + LpRegistrationResponseData::Mixnet { .. } => None, + } + } + + pub fn into_mixnet_response(self) -> Option { + match self.response_data { + LpRegistrationResponseData::Mixnet { data } => Some(data), + LpRegistrationResponseData::Dvpn { .. } => None, + } + } + + pub fn error_message(&self) -> Option<&str> { + match &self.response_data { + LpRegistrationResponseData::Dvpn { data } => match &data.content { + LpDvpnRegistrationResponseMessageContent::RegistrationFailure(response) => { + Some(&response.error) + } + _ => None, + }, + LpRegistrationResponseData::Mixnet { data } => match &data.content { + LpMixnetRegistrationResponseMessageContent::RegistrationFailure(response) => { + Some(&response.error) + } + _ => None, + }, } } @@ -207,16 +290,170 @@ impl LpRegistrationResponse { } } +pub mod dvpn { + use crate::WireguardConfiguration; + use nym_authenticator_requests::models::BandwidthClaim; + use nym_credentials_interface::TicketType; + use serde::{Deserialize, Serialize}; + + // client + #[derive(Debug, Clone, Serialize, Deserialize)] + pub struct LpDvpnRegistrationRequestMessage { + pub content: LpDvpnRegistrationRequestMessageContent, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + pub enum LpDvpnRegistrationRequestMessageContent { + InitialRequest(LpDvpnRegistrationInitialRequest), + Finalisation(LpDvpnRegistrationFinalisation), + // in theory, we could also extend it with Bandwidth-related messages, + // but that shouldn't really be the responsibility of a Registration client. + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + pub struct LpDvpnRegistrationInitialRequest { + /// Client's WireGuard public key (for dVPN mode) + pub wg_public_key: nym_wireguard_types::PeerPublicKey, + + /// Preshared key to be used for the connection + pub psk: [u8; 32], + + /// Type of the ticket/gateway we're going to register with + pub ticket_type: TicketType, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + pub struct LpDvpnRegistrationFinalisation { + /// Ecash credential + pub credential: BandwidthClaim, + } + + // gateway + #[derive(Debug, Clone, Serialize, Deserialize)] + pub struct LpDvpnRegistrationResponseMessage { + pub content: LpDvpnRegistrationResponseMessageContent, + } + + impl LpDvpnRegistrationResponseMessage { + pub fn error(error: impl Into) -> Self { + LpDvpnRegistrationResponseMessage { + content: LpDvpnRegistrationResponseMessageContent::RegistrationFailure( + RegistrationFailureResponse { + error: error.into(), + }, + ), + } + } + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + pub enum LpDvpnRegistrationResponseMessageContent { + RequiresCredential(RequiresCredentialResponse), + CompletedRegistration(CompletedRegistrationResponse), + RegistrationFailure(RegistrationFailureResponse), + } + + #[derive(Debug, Clone, Copy, Serialize, Deserialize)] + pub struct CompletedRegistrationResponse { + /// Gateway configuration data for dVPN mode (WireGuard) + /// This matches what WireguardRegistrationResult expects + pub config: WireguardConfiguration, + + /// The bandwidth available to this client, + pub available_bandwidth: i64, + } + + #[derive(Debug, Clone, Copy, Serialize, Deserialize)] + pub struct RequiresCredentialResponse; + + #[derive(Debug, Clone, Serialize, Deserialize)] + pub struct RegistrationFailureResponse { + pub error: String, + } +} + +pub mod mixnet { + use nym_crypto::asymmetric::ed25519; + use serde::{Deserialize, Serialize}; + + // client + #[derive(Debug, Clone, Serialize, Deserialize)] + pub struct LpMixnetRegistrationRequestMessage { + pub content: LpMixnetRegistrationRequestContent, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + pub struct LpMixnetRegistrationRequestContent { + /// Client's ed25519 public key (identity) + /// + /// Used to derive DestinationAddressBytes for ActiveClientsStore lookup. + pub client_ed25519_pubkey: ed25519::PublicKey, + } + + // gateway + + #[derive(Debug, Clone, Serialize, Deserialize)] + pub struct LpMixnetRegistrationResponseMessage { + pub content: LpMixnetRegistrationResponseMessageContent, + } + + impl LpMixnetRegistrationResponseMessage { + pub fn error(error: impl Into) -> Self { + LpMixnetRegistrationResponseMessage { + content: LpMixnetRegistrationResponseMessageContent::RegistrationFailure( + RegistrationFailureResponse { + error: error.into(), + }, + ), + } + } + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + pub enum LpMixnetRegistrationResponseMessageContent { + CompletedRegistration(CompletedRegistrationResponse), + RegistrationFailure(RegistrationFailureResponse), + } + + /// Gateway data for mixnet mode registration + /// + /// Contains the gateway's identity and sphinx key needed for the client + /// to construct its full nym Recipient address. + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] + pub struct LpMixnetGatewayData { + /// Gateway's ed25519 identity public key + /// + /// Forms part of the client's nym Recipient address. + pub gateway_identity: ed25519::PublicKey, + // TODO: what we really need in here is the address of internal IPR + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + pub struct CompletedRegistrationResponse { + /// Gateway data for mixnet mode + /// + /// Contains gateway identity and sphinx key needed for nym address construction. + pub config: LpMixnetGatewayData, + + /// The bandwidth available to this client, + pub available_bandwidth: i64, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + pub struct RegistrationFailureResponse { + pub error: String, + } +} + #[cfg(test)] mod tests { use super::*; + use nym_crypto::asymmetric::ed25519; use nym_test_utils::helpers::deterministic_rng; - use std::net::Ipv4Addr; + use std::net::{Ipv4Addr, Ipv6Addr}; // ==================== Helper Functions ==================== fn create_test_gateway_data() -> WireguardConfiguration { - use std::net::Ipv6Addr; - WireguardConfiguration { public_key: nym_crypto::asymmetric::x25519::PublicKey::from( nym_sphinx::PublicKey::from([1u8; 32]), @@ -231,37 +468,60 @@ mod tests { // ==================== LpRegistrationResponse Tests ==================== - #[test] - fn test_lp_registration_response_success() { - let gateway_data = create_test_gateway_data(); - let allocated_bandwidth = 1_000_000_000; - - let response = LpRegistrationResponse::success(allocated_bandwidth, gateway_data.clone()); - - assert!(response.success); - assert!(response.error.is_none()); - assert!(response.gateway_data.is_some()); - assert_eq!(response.allocated_bandwidth, allocated_bandwidth); - - let returned_gw_data = response - .gateway_data - .expect("Gateway data should be present in success response"); - assert_eq!(returned_gw_data.public_key, gateway_data.public_key); - assert_eq!(returned_gw_data.private_ipv4, gateway_data.private_ipv4); - assert_eq!(returned_gw_data.private_ipv6, gateway_data.private_ipv6); - assert_eq!(returned_gw_data.endpoint, gateway_data.endpoint); - } - #[test] fn test_lp_registration_response_error() { let error_msg = String::from("Insufficient bandwidth"); - let response = LpRegistrationResponse::error(error_msg.clone()); + let response_mixnet = + LpRegistrationResponse::error(error_msg.clone(), RegistrationMode::Mixnet); + let response_dvpn = + LpRegistrationResponse::error(error_msg.clone(), RegistrationMode::Dvpn); - assert!(!response.success); - assert_eq!(response.error, Some(error_msg)); - assert!(response.gateway_data.is_none()); - assert_eq!(response.allocated_bandwidth, 0); + assert!(response_mixnet.status.is_failed()); + assert!(response_dvpn.status.is_failed()); + + // check mixnet + let LpRegistrationResponseData::Mixnet { data } = response_mixnet.response_data else { + panic!("unexpected response") + }; + + let LpMixnetRegistrationResponseMessageContent::RegistrationFailure(failure) = data.content + else { + panic!("unexpected response") + }; + assert_eq!(failure.error, error_msg); + + // check dvpn + let LpRegistrationResponseData::Dvpn { data } = response_dvpn.response_data else { + panic!("unexpected response") + }; + + let LpDvpnRegistrationResponseMessageContent::RegistrationFailure(failure) = data.content + else { + panic!("unexpected response") + }; + assert_eq!(failure.error, error_msg); + } + + #[test] + fn test_lp_registration_response_success_dvpn() { + let cfg = create_test_gateway_data(); + let allocated_bandwidth = 500_000_000; + + let response = LpRegistrationResponse::success_dvpn(cfg, allocated_bandwidth); + assert!(response.status.is_successful()); + + let LpRegistrationResponseData::Dvpn { data } = response.response_data else { + panic!("unexpected response") + }; + + let LpDvpnRegistrationResponseMessageContent::CompletedRegistration(complete) = + data.content + else { + panic!("unexpected response") + }; + assert_eq!(complete.config, cfg); + assert_eq!(complete.available_bandwidth, allocated_bandwidth); } #[test] @@ -274,17 +534,20 @@ mod tests { }; let allocated_bandwidth = 500_000_000; - let response = LpRegistrationResponse::success_mixnet(allocated_bandwidth, lp_gateway_data); + let response = + LpRegistrationResponse::success_mixnet(lp_gateway_data.clone(), allocated_bandwidth); + assert!(response.status.is_successful()); - assert!(response.success); - assert!(response.error.is_none()); - assert!(response.gateway_data.is_none()); - assert!(response.lp_gateway_data.is_some()); - assert_eq!(response.allocated_bandwidth, allocated_bandwidth); + let LpRegistrationResponseData::Mixnet { data } = response.response_data else { + panic!("unexpected response") + }; - let gw_data = response - .lp_gateway_data - .expect("LpGatewayData should be present"); - assert_eq!(gw_data.gateway_identity, *valid_key.public_key()); + let LpMixnetRegistrationResponseMessageContent::CompletedRegistration(complete) = + data.content + else { + panic!("unexpected response") + }; + assert_eq!(complete.config, lp_gateway_data); + assert_eq!(complete.available_bandwidth, allocated_bandwidth); } } diff --git a/common/wireguard/src/lib.rs b/common/wireguard/src/lib.rs index 1bf229db03..459677b492 100644 --- a/common/wireguard/src/lib.rs +++ b/common/wireguard/src/lib.rs @@ -28,6 +28,7 @@ pub mod peer_controller; pub mod peer_handle; pub mod peer_storage_manager; +pub use defguard_wireguard_rs::host::Peer as DefguardPeer; pub use error::Error; pub use ip_pool::{IpPool, IpPoolError}; pub use nym_wireguard_types::Config as WireguardConfig; diff --git a/common/wireguard/src/peer_controller/mod.rs b/common/wireguard/src/peer_controller/mod.rs index 7848409b75..60d7c98ee2 100644 --- a/common/wireguard/src/peer_controller/mod.rs +++ b/common/wireguard/src/peer_controller/mod.rs @@ -27,12 +27,12 @@ use nym_wireguard_types::{ }; use std::{collections::HashMap, sync::Arc}; use std::{ - net::{IpAddr, SocketAddr}, + net::IpAddr, time::{Duration, SystemTime}, }; use tokio::sync::{RwLock, mpsc}; use tokio_stream::{StreamExt, wrappers::IntervalStream}; -use tracing::{debug, error, info, trace}; +use tracing::{debug, error, info, trace, warn}; pub use nym_ip_packet_requests::IpPair; @@ -43,35 +43,30 @@ pub mod mock; #[derive(Debug, Clone)] pub struct PeerRegistrationData { pub public_key: Key, - pub preshared_key: Option, - pub endpoint: Option, - pub persistent_keepalive_interval: Option, + pub preshared_key: Key, + // pub endpoint: Option, + // pub persistent_keepalive_interval: Option, } impl PeerRegistrationData { - pub fn new(public_key: Key) -> Self { + pub fn new(public_key: Key, psk: Key) -> Self { Self { public_key, - preshared_key: None, - endpoint: None, - persistent_keepalive_interval: None, + preshared_key: psk, + // endpoint: None, + // persistent_keepalive_interval: None, } } - - pub fn with_preshared_key(mut self, key: Key) -> Self { - self.preshared_key = Some(key); - self - } - - pub fn with_endpoint(mut self, endpoint: SocketAddr) -> Self { - self.endpoint = Some(endpoint); - self - } - - pub fn with_keepalive(mut self, interval: u16) -> Self { - self.persistent_keepalive_interval = Some(interval); - self - } + // + // pub fn with_endpoint(mut self, endpoint: SocketAddr) -> Self { + // self.endpoint = Some(endpoint); + // self + // } + // + // pub fn with_keepalive(mut self, interval: u16) -> Self { + // self.persistent_keepalive_interval = Some(interval); + // self + // } } pub enum PeerControlRequest { @@ -211,6 +206,11 @@ impl PeerController { .remove_wireguard_peer(&key.to_string()) .await?; self.bw_storage_managers.remove(key); + + warn!("MISSING CALL TO IP POOL RELEASE"); + // need to figure out what addresses to release + // self.ip_pool.release() + let ret = self.wg_api.remove_peer(key); if ret.is_err() { nym_metrics::inc!("wg_peer_removal_failed"); @@ -308,7 +308,7 @@ impl PeerController { .map_err(|e| Error::IpPool(e.to_string()))?; nym_metrics::inc!("wg_ip_allocation_success"); - tracing::debug!("Allocated IP pair: {}", ip_pair); + tracing::debug!("Allocated IP pair: {ip_pair}"); Ok(ip_pair) } diff --git a/gateway/src/error.rs b/gateway/src/error.rs index 7801ef01d2..c620e019d8 100644 --- a/gateway/src/error.rs +++ b/gateway/src/error.rs @@ -1,7 +1,9 @@ // Copyright 2023 - Nym Technologies SA // SPDX-License-Identifier: GPL-3.0-only +pub use crate::node::client_handling::websocket::connection_handler::authenticated::RequestHandlingError; use crate::node::internal_service_providers::authenticator::error::AuthenticatorError; +use nym_credential_verification::upgrade_mode::UpgradeModeEnableError; use nym_gateway_stats_storage::error::StatsStorageError; use nym_gateway_storage::error::GatewayStorageError; use nym_ip_packet_router::error::IpPacketRouterError; @@ -12,8 +14,6 @@ use nym_validator_client::ValidatorClientError; use std::net::IpAddr; use thiserror::Error; -pub use crate::node::client_handling::websocket::connection_handler::authenticated::RequestHandlingError; - #[derive(Debug, Error)] pub enum GatewayError { #[error("the configured version of the gateway ({config_version}) is incompatible with the binary version ({binary_version})")] @@ -155,6 +155,9 @@ pub enum GatewayError { #[error("Invalid SystemTime: {0}")] InvalidSystemTime(#[from] std::time::SystemTimeError), + + #[error(transparent)] + UpgradeModeEnable(#[from] UpgradeModeEnableError), } impl From for GatewayError { diff --git a/gateway/src/node/internal_service_providers/authenticator/mixnet_listener.rs b/gateway/src/node/internal_service_providers/authenticator/mixnet_listener.rs index d5f14c70e9..7b13c5cc42 100644 --- a/gateway/src/node/internal_service_providers/authenticator/mixnet_listener.rs +++ b/gateway/src/node/internal_service_providers/authenticator/mixnet_listener.rs @@ -53,14 +53,14 @@ const DEFAULT_REGISTRATION_TIMEOUT_CHECK: Duration = Duration::from_secs(60); // const DEFAULT_WG_CLIENT_BANDWIDTH_THRESHOLD: i64 = 1024 * 1024 * 1024; pub(crate) struct RegisteredAndFree { - registration_in_progres: PendingRegistrations, + registration_in_progress: PendingRegistrations, free_private_network_ips: PrivateIPs, } impl RegisteredAndFree { pub(crate) fn new(free_private_network_ips: PrivateIPs) -> Self { RegisteredAndFree { - registration_in_progres: Default::default(), + registration_in_progress: Default::default(), free_private_network_ips, } } @@ -134,7 +134,7 @@ impl MixnetListener { async fn remove_stale_registrations(&self) -> Result<(), AuthenticatorError> { let mut registered_and_free = self.registered_and_free.write().await; let registered_values: Vec<_> = registered_and_free - .registration_in_progres + .registration_in_progress .values() .cloned() .collect(); @@ -149,7 +149,7 @@ impl MixnetListener { let Some(timestamp) = ip else { registered_and_free - .registration_in_progres + .registration_in_progress .remove(®.gateway_data.pub_key()); tracing::debug!( "Removed stale registration of {}", @@ -165,7 +165,7 @@ impl MixnetListener { if duration > DEFAULT_REGISTRATION_TIMEOUT_CHECK { *ip = None; registered_and_free - .registration_in_progres + .registration_in_progress .remove(®.gateway_data.pub_key()); tracing::debug!( "Removed stale registration of {}", @@ -187,7 +187,7 @@ impl MixnetListener { let nonce: u64 = fastrand::u64(..); let mut registered_and_free = self.registered_and_free.write().await; if let Some(registration_data) = registered_and_free - .registration_in_progres + .registration_in_progress .get(&remote_public) { let gateway_data = registration_data.gateway_data.clone(); @@ -404,7 +404,7 @@ impl MixnetListener { wg_port: self.config.authenticator.tunnel_announced_port, }; registered_and_free - .registration_in_progres + .registration_in_progress .insert(remote_public, registration_data.clone()); let bytes = match AuthenticatorVersion::from(protocol) { AuthenticatorVersion::V1 => { @@ -541,7 +541,7 @@ impl MixnetListener { ) -> AuthenticatorHandleResult { let mut registered_and_free = self.registered_and_free.write().await; let registration_data = registered_and_free - .registration_in_progres + .registration_in_progress .get(&final_message.gateway_client_pub_key()) .ok_or(AuthenticatorError::RegistrationNotInProgress)? .clone(); @@ -596,7 +596,7 @@ impl MixnetListener { } registered_and_free - .registration_in_progres + .registration_in_progress .remove(&final_message.gateway_client_pub_key()); let bytes = match AuthenticatorVersion::from(protocol) { diff --git a/gateway/src/node/internal_service_providers/authenticator/mod.rs b/gateway/src/node/internal_service_providers/authenticator/mod.rs index a98c31868e..cee2c9041b 100644 --- a/gateway/src/node/internal_service_providers/authenticator/mod.rs +++ b/gateway/src/node/internal_service_providers/authenticator/mod.rs @@ -152,7 +152,7 @@ impl Authenticator { } }) .collect(); - let mixnet_listener = crate::node::internal_service_providers::authenticator::mixnet_listener::MixnetListener::new( + let mixnet_listener = mixnet_listener::MixnetListener::new( self.config, free_private_network_ips, self.wireguard_gateway_data, diff --git a/gateway/src/node/lp_listener/handler.rs b/gateway/src/node/lp_listener/handler.rs index b2bfac5a91..35e94d4b66 100644 --- a/gateway/src/node/lp_listener/handler.rs +++ b/gateway/src/node/lp_listener/handler.rs @@ -1,8 +1,7 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: GPL-3.0-only -use super::registration::process_registration; -use super::LpHandlerState; +use super::{LpHandlerState, ReceiverIndex}; use crate::error::GatewayError; use nym_crypto::asymmetric::{ed25519, x25519}; use nym_lp::state_machine::{LpAction, LpData, LpDataKind, LpInput}; @@ -12,7 +11,7 @@ use nym_lp::{ }; use nym_lp_transport::traits::LpTransport; use nym_metrics::{add_histogram_obs, inc}; -use nym_registration_common::LpRegistrationRequest; +use nym_registration_common::{LpRegistrationRequest, RegistrationStatus}; use std::net::SocketAddr; use std::time::Duration; use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -742,11 +741,11 @@ where /// Handle registration request on an established session async fn handle_registration_request( &mut self, - receiver_idx: u32, + receiver_idx: ReceiverIndex, request: LpRegistrationRequest, ) -> Result<(), GatewayError> { // Process registration (might modify state) - let response = process_registration(request, &self.state).await; + let response = self.state.process_registration(receiver_idx, request).await; let response_bytes = response.serialise().map_err(|e| { GatewayError::LpProtocolError(format!("Failed to serialize response: {e}")) })?; @@ -754,13 +753,23 @@ where self.send_response_packet(receiver_idx, response_bytes, LpDataKind::Registration) .await?; - if response.success { - info!("LP registration successful for {})", self.remote_addr); - } else { - warn!( - "LP registration failed for {}: {:?}", - self.remote_addr, response.error - ); + match response.status { + RegistrationStatus::Completed => { + info!("LP registration successful for {}", self.remote_addr); + } + RegistrationStatus::Failed => { + warn!( + "LP registration failed for {}: {:?}", + self.remote_addr, + response.error_message() + ); + } + RegistrationStatus::PendingMoreData => { + info!( + "we required more deta from {} to complete registration", + self.remote_addr + ); + } } Ok(()) @@ -1219,16 +1228,24 @@ where #[cfg(test)] mod tests { use super::*; + use crate::node::lp_listener::peer_manager::PeerManager; use crate::node::lp_listener::{LpConfig, LpDebug}; use crate::node::ActiveClientsStore; use bytes::BytesMut; + use nym_credential_verification::upgrade_mode::{ + UpgradeModeCheckConfig, UpgradeModeCheckRequestSender, UpgradeModeDetails, + }; + use nym_credential_verification::UpgradeModeState; use nym_lp::codec::{parse_lp_packet, serialize_lp_packet}; use nym_lp::message::{ClientHelloData, EncryptedDataPayload, HandshakeData, LpMessage}; use nym_lp::packet::{LpHeader, LpPacket}; use nym_lp::peer::LpLocalPeer; + use nym_wireguard::{PeerControlRequest, WireguardConfig, WireguardGatewayData}; + use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; + use tokio::sync::mpsc::Receiver; // ==================== Test Helpers ==================== /// Create a minimal test state for handler tests @@ -1236,6 +1253,23 @@ mod tests { use nym_crypto::asymmetric::ed25519; use rand::rngs::OsRng; + fn wireguard_data( + keys: Arc, + ) -> (WireguardGatewayData, Receiver) { + // some sensible default values (ports don't matter anyway) + let cfg = WireguardConfig { + bind_address: SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 51822), + private_ipv4: Ipv4Addr::new(10, 1, 0, 1), + private_ipv6: Ipv6Addr::new(0xfc01, 0, 0, 0, 0, 0, 0, 0x1), // fc01::1, + announced_tunnel_port: 51822, + announced_metadata_port: 51830, + private_network_prefix_v4: 16, + private_network_prefix_v6: 112, + }; + + WireguardGatewayData::new(cfg, keys) + } + // Create in-memory storage for testing let storage = nym_gateway_storage::GatewayStorage::init(":memory:", 100) .await @@ -1262,6 +1296,20 @@ mod tests { let id_keys = Arc::new(ed25519::KeyPair::new(&mut OsRng)); let x_keys = Arc::new(id_keys.to_x25519()); + let (wireguard_data, _) = wireguard_data(x_keys.clone()); + + let (um_recheck_tx, _) = futures::channel::mpsc::unbounded(); + + let upgrade_mode_state = UpgradeModeState::new(*id_keys.public_key()); + let upgrade_mode_details = UpgradeModeDetails::new( + UpgradeModeCheckConfig { + // essentially we never want to trigger this in our tests + min_staleness_recheck: Duration::from_nanos(1), + }, + UpgradeModeCheckRequestSender::new(um_recheck_tx), + upgrade_mode_state.clone(), + ); + let lp_peer = LpLocalPeer::new(id_keys, x_keys.clone()).with_kem_psq_key(x_keys); LpHandlerState { @@ -1272,12 +1320,13 @@ mod tests { local_lp_peer: lp_peer, metrics: nym_node_metrics::NymNodeMetrics::default(), active_clients_store: ActiveClientsStore::new(), - wg_peer_controller: None, - wireguard_data: None, + upgrade_mode: upgrade_mode_details, outbound_mix_sender: mix_sender, handshake_states: Arc::new(dashmap::DashMap::new()), session_states: Arc::new(dashmap::DashMap::new()), + registrations_in_progress: Default::default(), forward_semaphore, + peer_manager: Arc::new(PeerManager::new(wireguard_data)), } } diff --git a/gateway/src/node/lp_listener/mod.rs b/gateway/src/node/lp_listener/mod.rs index 4fc45c4f2f..d20b83205d 100644 --- a/gateway/src/node/lp_listener/mod.rs +++ b/gateway/src/node/lp_listener/mod.rs @@ -68,30 +68,37 @@ // They can be exported via Prometheus format using the metrics endpoint. use crate::error::GatewayError; +use crate::node::lp_listener::peer_manager::PeerManager; +use crate::node::lp_listener::registration::RegistrationsInProgress; use crate::node::ActiveClientsStore; use dashmap::DashMap; use nym_config::serde_helpers::de_maybe_port; +use nym_credential_verification::ecash::traits::EcashManager; +use nym_credential_verification::upgrade_mode::UpgradeModeDetails; use nym_gateway_storage::GatewayStorage; use nym_lp::state_machine::LpStateMachine; -pub use nym_mixnet_client::forwarder::{ - mix_forwarding_channels, MixForwardingReceiver, MixForwardingSender, -}; use nym_node_metrics::NymNodeMetrics; use nym_task::ShutdownTracker; use std::net::{IpAddr, Ipv6Addr, SocketAddr}; use std::sync::Arc; use std::time::Duration; use tokio::net::TcpListener; -use tokio::sync::{mpsc, Semaphore}; +use tokio::sync::Semaphore; use tracing::*; pub use nym_lp::peer::LpLocalPeer; +pub use nym_mixnet_client::forwarder::{ + mix_forwarding_channels, MixForwardingReceiver, MixForwardingSender, +}; pub use nym_wireguard::{PeerControlRequest, WireguardGatewayData}; mod data_handler; pub mod handler; +pub mod peer_manager; mod registration; +pub type ReceiverIndex = u32; + /// Configuration for LP listener #[derive(Debug, Clone, Copy, serde::Deserialize, serde::Serialize)] #[serde(default)] @@ -111,7 +118,7 @@ pub struct LpConfig { pub announce_control_port: Option, /// Custom announced port for listening for the UDP LP data traffic. - /// If unspecified, the value from the `data_bind_address` will be used instead + /// If unspecified, the value from the `data_bind_address` will be used instead /// (default: None) #[serde(deserialize_with = "de_maybe_port")] pub announce_data_port: Option, @@ -178,6 +185,10 @@ pub struct LpDebug { #[serde(with = "humantime_serde")] pub demoted_session_ttl: Duration, + /// Maximum age of in-progress dVPN registration before cleanup (default: 60s) + #[serde(with = "humantime_serde")] + pub pending_registration_ttl: Duration, + /// How often to run the state cleanup task (default: 5 minutes) /// /// The cleanup task scans for and removes stale handshakes and sessions. @@ -247,6 +258,9 @@ impl LpDebug { // 5 minutes - balances memory reclamation with task overhead pub const DEFAULT_STATE_CLEANUP_INTERVAL: Duration = Duration::from_secs(300); + // 1 minute - enough for client to send retrieve credential from its storage and send it across + pub const DEFAULT_PENDING_REGISTRATION_TTL: Duration = Duration::from_secs(60); + // Limits concurrent outbound connections to prevent fd exhaustion pub const DEFAULT_MAX_CONCURRENT_FORWARDS: usize = 1000; } @@ -260,6 +274,7 @@ impl Default for LpDebug { handshake_ttl: Self::DEFAULT_HANDSHAKE_TTL, session_ttl: Self::DEFAULT_SESSION_TTL, demoted_session_ttl: Self::DEFAULT_DEMOTED_SESSION_TTL, + pending_registration_ttl: Self::DEFAULT_PENDING_REGISTRATION_TTL, state_cleanup_interval: Self::DEFAULT_STATE_CLEANUP_INTERVAL, max_concurrent_forwards: Self::DEFAULT_MAX_CONCURRENT_FORWARDS, } @@ -311,12 +326,12 @@ impl TimestampedState { } /// Get age since creation - pub fn age(&self) -> std::time::Duration { + pub fn age(&self) -> Duration { self.created_at.elapsed() } /// Get time since last activity - pub fn since_activity(&self) -> std::time::Duration { + pub fn since_activity(&self) -> Duration { let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default() @@ -332,8 +347,7 @@ impl TimestampedState { #[derive(Clone)] pub struct LpHandlerState { /// Ecash verifier for bandwidth credentials - pub ecash_verifier: - Arc, + pub ecash_verifier: Arc, /// Storage backend for persistence pub storage: GatewayStorage, @@ -347,11 +361,12 @@ pub struct LpHandlerState { /// Active clients tracking pub active_clients_store: ActiveClientsStore, - /// WireGuard peer controller channel (for dVPN registrations) - pub wg_peer_controller: Option>, + /// Current state of the Upgrade Mode as perceived by this gateway + pub upgrade_mode: UpgradeModeDetails, /// WireGuard gateway data (contains keypair and config) - pub wireguard_data: Option, + /// alongside helpers for managing peers + pub peer_manager: Arc, /// LP configuration (for timestamp validation, etc.) pub lp_config: LpConfig, @@ -369,7 +384,7 @@ pub struct LpHandlerState { /// state moves to session_states map. /// /// Wrapped in TimestampedState for TTL-based cleanup of stale handshakes. - pub handshake_states: Arc>>, + pub handshake_states: Arc>>, /// Established sessions keyed by session_id /// @@ -383,7 +398,11 @@ pub struct LpHandlerState { /// subsession/rekeying support. The state machine handles subsession initiation /// (SubsessionKK1/KK2/Ready) during transport phase, allowing long-lived connections /// to rekey without re-authentication. - pub session_states: Arc>>, + pub session_states: Arc>>, + + /// In-progress dVPN registrations that require additional data (e.g. credentials) + /// to finalise. + pub registrations_in_progress: RegistrationsInProgress, /// Semaphore limiting concurrent forward connections /// @@ -561,28 +580,28 @@ impl LpListener { fn spawn_state_cleanup_task(&self) -> tokio::task::JoinHandle<()> { let handshake_states = Arc::clone(&self.handler_state.handshake_states); let session_states = Arc::clone(&self.handler_state.session_states); + let pending_registrations = self.handler_state.registrations_in_progress.clone(); let dbg_cfg = self.handler_state.lp_config.debug; let handshake_ttl = dbg_cfg.handshake_ttl; let session_ttl = dbg_cfg.session_ttl; let demoted_session_ttl = dbg_cfg.demoted_session_ttl; + let pending_reg_ttl = dbg_cfg.pending_registration_ttl; let interval = dbg_cfg.state_cleanup_interval; let shutdown = self.shutdown.clone_shutdown_token(); let metrics = self.handler_state.metrics.clone(); info!( - "Starting LP state cleanup task (handshake_ttl={}s, session_ttl={}s, demoted_ttl={}s, interval={}s)", - handshake_ttl.as_secs(), session_ttl.as_secs(), demoted_session_ttl.as_secs(), interval.as_secs() + "Starting LP state cleanup task (handshake_ttl={}s, session_ttl={}s, demoted_ttl={}s, reg_ttl={}s, interval={}s)", + handshake_ttl.as_secs(), session_ttl.as_secs(), demoted_session_ttl.as_secs(),pending_reg_ttl.as_secs(), interval.as_secs() ); self.shutdown.try_spawn_named( - Self::cleanup_loop( + cleanup_task::cleanup_loop( handshake_states, session_states, - handshake_ttl, - session_ttl, - demoted_session_ttl, - interval, + pending_registrations, + dbg_cfg, shutdown, metrics, ), @@ -590,104 +609,6 @@ impl LpListener { ) } - /// Background loop for cleaning up stale state entries - /// - /// Runs periodically to scan handshake_states and session_states maps, - /// removing entries that have exceeded their TTL. - /// - /// Demoted sessions (ReadOnlyTransport) use shorter TTL since they - /// only need to drain in-flight packets after subsession promotion. - #[allow(clippy::too_many_arguments)] - async fn cleanup_loop( - handshake_states: Arc>>, - session_states: Arc>>, - handshake_ttl: Duration, - session_ttl: Duration, - demoted_session_ttl: Duration, - interval: Duration, - shutdown: nym_task::ShutdownToken, - _metrics: NymNodeMetrics, - ) { - use nym_lp::state_machine::LpStateBare; - use nym_metrics::inc_by; - - let mut cleanup_interval = tokio::time::interval(interval); - - loop { - tokio::select! { - biased; - - _ = shutdown.cancelled() => { - debug!("LP state cleanup task: received shutdown signal"); - break; - } - - _ = cleanup_interval.tick() => { - let start = std::time::Instant::now(); - let mut hs_removed = 0u64; - let mut ss_removed = 0u64; - let mut demoted_removed = 0u64; - - // Remove stale handshakes (based on age since creation) - handshake_states.retain(|_, timestamped| { - if timestamped.age() > handshake_ttl { - hs_removed += 1; - false - } else { - true - } - }); - - // Remove stale sessions (based on time since last activity) - // Use shorter TTL for demoted (ReadOnlyTransport) sessions - session_states.retain(|_, timestamped| { - let is_demoted = timestamped.state.bare_state() == LpStateBare::ReadOnlyTransport; - let ttl = if is_demoted { - demoted_session_ttl - } else { - session_ttl - }; - - if timestamped.since_activity() > ttl { - if is_demoted { - demoted_removed += 1; - } else { - ss_removed += 1; - } - false - } else { - true - } - }); - - if hs_removed > 0 || ss_removed > 0 || demoted_removed > 0 { - let duration = start.elapsed(); - info!( - "LP state cleanup: removed {} handshakes, {} sessions, {} demoted (took {:.3}s)", - hs_removed, - ss_removed, - demoted_removed, - duration.as_secs_f64() - ); - - // Track metrics - if hs_removed > 0 { - inc_by!("lp_states_cleanup_handshake_removed", hs_removed as i64); - } - if ss_removed > 0 { - inc_by!("lp_states_cleanup_session_removed", ss_removed as i64); - } - if demoted_removed > 0 { - inc_by!("lp_states_cleanup_demoted_removed", demoted_removed as i64); - } - } - } - } - } - - info!("LP state cleanup task shutdown complete"); - } - fn active_lp_connections(&self) -> usize { self.handler_state .metrics @@ -695,3 +616,138 @@ impl LpListener { .active_lp_connections_count() } } + +pub(crate) mod cleanup_task { + use crate::node::lp_listener::registration::RegistrationsInProgress; + use crate::node::lp_listener::{LpDebug, TimestampedState}; + use dashmap::DashMap; + use nym_lp::state_machine::LpStateBare; + use nym_lp::LpStateMachine; + use nym_metrics::inc_by; + use nym_node_metrics::NymNodeMetrics; + use std::sync::Arc; + use tracing::{debug, info}; + + async fn perform_cleanup( + handshake_states: &Arc>>, + session_states: &Arc>>, + registrations_in_progress: &RegistrationsInProgress, + cfg: LpDebug, + ) { + let handshake_ttl = cfg.handshake_ttl; + let session_ttl = cfg.session_ttl; + let demoted_session_ttl = cfg.demoted_session_ttl; + let pending_registration_ttl = cfg.pending_registration_ttl; + + let start = std::time::Instant::now(); + let mut hs_removed = 0u64; + let mut ss_removed = 0u64; + let mut pending_reg_removed = 0u64; + let mut demoted_removed = 0u64; + + // Remove stale handshakes (based on age since creation) + handshake_states.retain(|_, timestamped| { + if timestamped.age() > handshake_ttl { + hs_removed += 1; + false + } else { + true + } + }); + + // Remove stale sessions (based on time since last activity) + // Use shorter TTL for demoted (ReadOnlyTransport) sessions + session_states.retain(|_, timestamped| { + let is_demoted = timestamped.state.bare_state() == LpStateBare::ReadOnlyTransport; + let ttl = if is_demoted { + demoted_session_ttl + } else { + session_ttl + }; + + if timestamped.since_activity() > ttl { + if is_demoted { + demoted_removed += 1; + } else { + ss_removed += 1; + } + false + } else { + true + } + }); + + // Remove stale registrations (based on time since last activity) + registrations_in_progress + .lock() + .await + .retain(|_, timestamped| { + if timestamped.age() > pending_registration_ttl { + pending_reg_removed += 1; + false + } else { + true + } + }); + + if hs_removed > 0 || ss_removed > 0 || demoted_removed > 0 || pending_reg_removed > 0 { + let duration = start.elapsed(); + info!( + "LP state cleanup: removed {hs_removed} handshakes, {pending_reg_removed} pending registrations, {ss_removed} sessions, {demoted_removed} demoted (took {:.3}s)", + duration.as_secs_f64() + ); + + // Track metrics + if hs_removed > 0 { + inc_by!("lp_states_cleanup_handshake_removed", hs_removed as i64); + } + if ss_removed > 0 { + inc_by!("lp_states_cleanup_session_removed", ss_removed as i64); + } + if demoted_removed > 0 { + inc_by!("lp_states_cleanup_demoted_removed", demoted_removed as i64); + } + if pending_reg_removed > 0 { + inc_by!( + "lp_states_cleanup_pending_registrations_removed", + pending_reg_removed as i64 + ); + } + } + } + + /// Background loop for cleaning up stale state entries + /// + /// Runs periodically to scan handshake_states and session_states maps, + /// removing entries that have exceeded their TTL. + /// + /// Demoted sessions (ReadOnlyTransport) use shorter TTL since they + /// only need to drain in-flight packets after subsession promotion. + pub(crate) async fn cleanup_loop( + handshake_states: Arc>>, + session_states: Arc>>, + registrations_in_progress: RegistrationsInProgress, + cfg: LpDebug, + shutdown: nym_task::ShutdownToken, + _metrics: NymNodeMetrics, + ) { + let interval = cfg.state_cleanup_interval; + + let mut cleanup_interval = tokio::time::interval(interval); + + loop { + tokio::select! { + biased; + _ = shutdown.cancelled() => { + debug!("LP state cleanup task: received shutdown signal"); + break; + } + _ = cleanup_interval.tick() => { + perform_cleanup(&handshake_states, &session_states, ®istrations_in_progress, cfg).await; + } + } + } + + info!("LP state cleanup task shutdown complete"); + } +} diff --git a/gateway/src/node/lp_listener/peer_manager.rs b/gateway/src/node/lp_listener/peer_manager.rs new file mode 100644 index 0000000000..027fe9221f --- /dev/null +++ b/gateway/src/node/lp_listener/peer_manager.rs @@ -0,0 +1,127 @@ +// Copyright 2026 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::GatewayError; +use defguard_wireguard_rs::host::Peer; +use defguard_wireguard_rs::key::Key; +use futures::channel::oneshot; +use nym_credential_verification::ClientBandwidth; +use nym_wireguard::peer_controller::IpPair; +use nym_wireguard::{PeerControlRequest, PeerRegistrationData, WireguardGatewayData}; +use nym_wireguard_types::PeerPublicKey; +use tracing::error; + +/// attempts to replicate [`crate::node::internal_service_providers::authenticator::peer_manager::PeerManager`] +// TODO: put those in the shared crate +pub struct PeerManager { + pub(crate) wireguard_gateway_data: WireguardGatewayData, +} + +impl PeerManager { + pub fn new(wireguard_gateway_data: WireguardGatewayData) -> Self { + PeerManager { + wireguard_gateway_data, + } + } + + pub async fn register_peer( + &self, + registration_data: PeerRegistrationData, + ) -> Result { + let (response_tx, response_rx) = oneshot::channel(); + let msg = PeerControlRequest::RegisterPeer { + registration_data, + response_tx, + }; + self.wireguard_gateway_data + .peer_tx() + .send(msg) + .await + .map_err(|e| { + GatewayError::InternalError(format!("Failed to send IP allocation request: {e}")) + })?; + + response_rx + .await + .map_err(|e| { + GatewayError::InternalError(format!("Failed to receive IP allocation: {e}")) + })? + .map_err(|e| { + error!("Failed to allocate IPs from pool: {e}"); + GatewayError::InternalError(format!("Failed to allocate IPs: {e}")) + }) + } + + pub async fn add_peer(&self, peer: Peer) -> Result<(), GatewayError> { + let (response_tx, response_rx) = oneshot::channel(); + let msg = PeerControlRequest::AddPeer { peer, response_tx }; + self.wireguard_gateway_data + .peer_tx() + .send(msg) + .await + .map_err(|e| { + GatewayError::InternalError(format!("Failed to send peer request: {e}")) + })?; + + response_rx + .await + .map_err(|_| GatewayError::InternalError("no response for add peer".to_string()))? + .map_err(|err| { + GatewayError::InternalError(format!("adding peer could not be performed: {err:?}")) + }) + } + + pub async fn query_peer( + &self, + public_key: PeerPublicKey, + ) -> Result, GatewayError> { + let key = Key::new(public_key.to_bytes()); + let (response_tx, response_rx) = oneshot::channel(); + let msg = PeerControlRequest::QueryPeer { key, response_tx }; + self.wireguard_gateway_data + .peer_tx() + .send(msg) + .await + .map_err(|_| { + GatewayError::InternalError("Failed to send peer query request".to_string()) + })?; + + response_rx + .await + .map_err(|_| GatewayError::InternalError("no response for query peer".to_string()))? + .map_err(|err| { + GatewayError::InternalError(format!( + "querying peer could not be performed: {err:?}" + )) + }) + } + + pub async fn query_client_bandwidth( + &self, + key: PeerPublicKey, + ) -> Result { + let key = Key::new(key.to_bytes()); + let (response_tx, response_rx) = oneshot::channel(); + let msg = PeerControlRequest::GetClientBandwidthByKey { key, response_tx }; + self.wireguard_gateway_data + .peer_tx() + .send(msg) + .await + .map_err(|_| { + GatewayError::InternalError( + "Failed to send peer bandwidth query request".to_string(), + ) + })?; + + response_rx + .await + .map_err(|_| { + GatewayError::InternalError("no response for query peer bandwidth".to_string()) + })? + .map_err(|err| { + GatewayError::InternalError(format!( + "querying client bandwidth could not be performed: {err:?}" + )) + }) + } +} diff --git a/gateway/src/node/lp_listener/registration.rs b/gateway/src/node/lp_listener/registration.rs index 8d004d1d28..5df0c609d3 100644 --- a/gateway/src/node/lp_listener/registration.rs +++ b/gateway/src/node/lp_listener/registration.rs @@ -1,30 +1,37 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: GPL-3.0-only -use super::LpHandlerState; +use super::{LpHandlerState, ReceiverIndex, TimestampedState}; use crate::error::GatewayError; -use crate::node::client_handling::websocket::message_receiver::IsActive; use defguard_wireguard_rs::host::Peer; use defguard_wireguard_rs::key::Key; -use futures::channel::{mpsc, oneshot}; +use nym_authenticator_requests::models::BandwidthClaim; use nym_credential_verification::ecash::traits::EcashManager; use nym_credential_verification::{ bandwidth_storage_manager::BandwidthStorageManager, BandwidthFlushingBehaviourConfig, ClientBandwidth, CredentialVerifier, }; -use nym_credentials_interface::CredentialSpendingData; +use nym_credentials_interface::{BandwidthCredential, CredentialSpendingData, TicketType}; +use nym_crypto::asymmetric::encryption::KeyPair; use nym_gateway_requests::models::CredentialSpendingRequest; use nym_gateway_storage::models::PersistedBandwidth; use nym_gateway_storage::traits::BandwidthGatewayStorage; use nym_metrics::{add_histogram_obs, inc, inc_by}; -use nym_registration_common::{ - LpDvpnRegistrationRequest, LpMixnetGatewayData, LpMixnetRegistrationRequest, - LpRegistrationData, LpRegistrationRequest, LpRegistrationResponse, WireguardConfiguration, +use nym_registration_common::dvpn::{ + LpDvpnRegistrationFinalisation, LpDvpnRegistrationInitialRequest, + LpDvpnRegistrationRequestMessage, LpDvpnRegistrationRequestMessageContent, }; -use nym_wireguard::PeerControlRequest; +use nym_registration_common::mixnet::LpMixnetRegistrationRequestMessage; +use nym_registration_common::{ + LpRegistrationRequest, LpRegistrationRequestData, LpRegistrationResponse, RegistrationMode, + RegistrationStatus, WireguardConfiguration, +}; +use nym_wireguard::WireguardConfig; use nym_wireguard_types::PeerPublicKey; +use std::collections::HashMap; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use std::sync::Arc; -use time::OffsetDateTime; +use tokio::sync::{Mutex, MutexGuard}; use tracing::*; // Histogram buckets for LP registration duration tracking @@ -56,6 +63,434 @@ const WG_CONTROLLER_LATENCY_BUCKETS: &[f64] = &[ 2.0, // 2s ]; +#[derive(Clone, Copy)] +pub struct PendingRegistrationState { + client_id: i64, + peer_key: PeerPublicKey, + ticket_type: TicketType, + wireguard_config: WireguardConfiguration, +} + +#[derive(Clone, Default)] +pub struct RegistrationsInProgress { + /// Wrapped in TimestampedState for TTL-based cleanup of stale data. + inner: Arc>>>, +} + +impl RegistrationsInProgress { + pub async fn lock( + &self, + ) -> MutexGuard<'_, HashMap>> { + self.inner.lock().await + } +} + +impl LpHandlerState { + fn upgrade_mode_enabled(&self) -> bool { + self.upgrade_mode.enabled() + } + + fn keypair(&self) -> &Arc { + self.peer_manager.wireguard_gateway_data.keypair() + } + + fn wireguard_config(&self) -> WireguardConfig { + self.peer_manager.wireguard_gateway_data.config() + } + + fn successful_dvpn_registration( + &self, + peer_private_ipv4: Ipv4Addr, + peer_private_ipv6: Ipv6Addr, + bandwidth: i64, + ) -> LpRegistrationResponse { + LpRegistrationResponse::success_dvpn( + WireguardConfiguration { + public_key: *self.keypair().public_key(), + // TODO: according to @SW this is most likely very wrong + endpoint: self.wireguard_config().bind_address, + private_ipv4: peer_private_ipv4, + private_ipv6: peer_private_ipv6, + }, + bandwidth, + ) + } + + /// Check if WG peer already registered, return cached response if so. + /// + /// This enables idempotent registration: if a client retries registration + /// with the same WG public key (e.g., after network failure), we return + /// the existing registration data instead of re-processing. This prevents + /// wasting credentials on network issues. + async fn check_existing_dvpn_registration( + &self, + public_key: PeerPublicKey, + ) -> Option { + // Look up existing peer + let Ok(maybe_peer) = self.peer_manager.query_peer(public_key).await else { + return Some(LpRegistrationResponse::error( + "iternal failure: failed to resolve peer information", + RegistrationMode::Dvpn, + )); + }; + + let peer = maybe_peer?; + + // Extract IPv4 and IPv6 from allowed_ips + let mut private_ipv4 = None; + let mut private_ipv6 = None; + for ip_mask in &peer.allowed_ips { + match ip_mask.address { + IpAddr::V4(v4) => private_ipv4 = Some(v4), + IpAddr::V6(v6) => private_ipv6 = Some(v6), + } + if private_ipv4.is_some() && private_ipv6.is_some() { + break; + } + } + + // Incomplete data, treat as new registration + let (Some(private_ipv4), Some(private_ipv6)) = (private_ipv4, private_ipv6) else { + return None; + }; + + // Get current bandwidth + let Ok(bandwidth) = self.peer_manager.query_client_bandwidth(public_key).await else { + return Some(LpRegistrationResponse::error( + "iternal failure: failed to resolve peer bandwidth", + RegistrationMode::Dvpn, + )); + }; + + Some(self.successful_dvpn_registration( + private_ipv4, + private_ipv6, + bandwidth.available().await, + )) + } + + /// In the case of an already registered WG peer, update its PSK. + async fn update_peer_psk(&self, peer: PeerPublicKey, psk: Key) -> Result<(), GatewayError> { + let encoded_psk = psk.to_lower_hex(); + self.storage + .update_peer_psk(&peer.to_string(), Some(&encoded_psk)) + .await?; + + // TODO: do we have to go through a peer manager to also update PSK if a peer is currently active? + // seems like an edge case. maybe we should force disconnect here? + Ok(()) + } + + async fn process_dvpn_initial_registration( + &self, + sender: ReceiverIndex, + request: LpDvpnRegistrationInitialRequest, + ) -> LpRegistrationResponse { + let wg_key_str = request.wg_public_key.to_string(); + + // check for an existing registration (same WG key already registered) + // This allows clients to retry registration after network failures + // or to re-use gateway without spending additional bandwidth + if let Some(existing_response) = self + .check_existing_dvpn_registration(request.wg_public_key) + .await + { + // if there already exists registration for this client, update the psk and return the peer data + if let Err(err) = self + .update_peer_psk(request.wg_public_key, Key::new(request.psk)) + .await + { + return LpRegistrationResponse::error( + format!("WireGuard peer PSK update failed: {err}"), + RegistrationMode::Dvpn, + ); + } + info!("LP dVPN re-registration for existing peer {wg_key_str} (idempotent)",); + inc!("lp_registration_dvpn_idempotent"); + return existing_response; + } + + // TODO: this could be a source of some issue as we pre-allocate ip before validating credentials + // (but we do the same in the authenticator anyway...) + if let Err(err) = self + .register_wg_peer( + sender, + request.wg_public_key, + request.ticket_type, + Key::new(request.psk), + ) + .await + { + return LpRegistrationResponse::error( + format!("WireGuard peer IP allocation failed: {err}"), + RegistrationMode::Dvpn, + ); + } + + LpRegistrationResponse::request_dvpn_credential() + } + + // TODO: dedup + async fn handle_final_credential_claim( + &self, + claim: BandwidthClaim, + client_id: i64, + ) -> Result { + match claim.credential { + BandwidthCredential::ZkNym(zk_nym) => { + // if we got zk-nym, we just try to verify it + let bandwidth = + credential_verification(self.ecash_verifier.clone(), *zk_nym, client_id) + .await?; + Ok(bandwidth) + } + BandwidthCredential::UpgradeModeJWT { token } => { + // TODO: move + const UM_BANDWIDTH: i64 = 1024 * 1024 * 1024; + + // if we're already in the upgrade mode, don't bother validating the token + if self.upgrade_mode_enabled() { + return Ok(UM_BANDWIDTH); + } + + self.upgrade_mode.try_enable_via_received_jwt(token).await?; + Ok(UM_BANDWIDTH) + } + } + } + + async fn process_dvpn_registration_finalisation( + &self, + sender: ReceiverIndex, + request: LpDvpnRegistrationFinalisation, + ) -> LpRegistrationResponse { + // see if we still have the pending registration + // (e.g. it's illegal for client to request registration and only finalise it, + // for example the next day; we can't keep the data forever) + let Some(pending) = self + .registrations_in_progress + .lock() + .await + .get(&sender) + .map(|pending| pending.state) + else { + return LpRegistrationResponse::error( + "no pending registration", + RegistrationMode::Dvpn, + ); + }; + + if pending.ticket_type != request.credential.kind { + return LpRegistrationResponse::error( + format!( + "inconsistent ticket type. used {} for initial request and {} for finalisation", + pending.ticket_type, request.credential.kind + ), + RegistrationMode::Dvpn, + ); + } + + let client_id = pending.client_id; + + let allocated_bandwidth = match self + .handle_final_credential_claim(request.credential, client_id) + .await + { + Ok(bandwidth) => bandwidth, + Err(err) => { + // Credential verification failed, remove the peer + warn!("LP credential verification failed for client {client_id}: {err}"); + inc!("lp_registration_dvpn_failed"); + if let Err(remove_err) = self + .storage + .remove_wireguard_peer(&pending.peer_key.to_string()) + .await + { + error!( + "Failed to remove peer after credential verification failure: {remove_err}" + ); + } + self.registrations_in_progress.lock().await.remove(&sender); + return LpRegistrationResponse::error( + format!("Credential verification failed: {err}"), + RegistrationMode::Dvpn, + ); + } + }; + + info!("LP dVPN registration successful (client_id: {client_id})"); + inc!("lp_registration_dvpn_success"); + LpRegistrationResponse::success_dvpn(pending.wireguard_config, allocated_bandwidth) + } + + async fn process_dvpn_registration( + &self, + sender: ReceiverIndex, + request: Box, + ) -> LpRegistrationResponse { + // Track dVPN registration attempts + inc!("lp_registration_dvpn_attempts"); + + match request.content { + LpDvpnRegistrationRequestMessageContent::InitialRequest(req) => { + self.process_dvpn_initial_registration(sender, req).await + } + LpDvpnRegistrationRequestMessageContent::Finalisation(req) => { + self.process_dvpn_registration_finalisation(sender, req) + .await + } + } + } + + async fn process_mixnet_registration( + &self, + request: LpMixnetRegistrationRequestMessage, + ) -> LpRegistrationResponse { + let _ = request; + LpRegistrationResponse::error( + "mixnet registration is not yet supported", + RegistrationMode::Mixnet, + ) + } + + /// Process an LP registration request + pub async fn process_registration( + &self, + sender: ReceiverIndex, + request: LpRegistrationRequest, + ) -> LpRegistrationResponse { + let registration_start = std::time::Instant::now(); + + // Track total registration attempts + inc!("lp_registration_attempts_total"); + + // 1. Validate timestamp for replay protection + if !request.validate_timestamp(30) { + warn!("LP registration failed: timestamp too old or too far in future"); + inc!("lp_registration_failed_timestamp"); + return LpRegistrationResponse::error("invalid timestamp", request.mode()); + } + + // 2. Process based on mode + let result = match request.registration_data { + LpRegistrationRequestData::Dvpn { data } => { + self.process_dvpn_registration(sender, data).await + } + LpRegistrationRequestData::Mixnet { data } => { + self.process_mixnet_registration(data).await + } + }; + + // Track registration duration + let duration = registration_start.elapsed().as_secs_f64(); + add_histogram_obs!( + "lp_registration_duration_seconds", + duration, + LP_REGISTRATION_DURATION_BUCKETS + ); + + // Track overall success/failure + match result.status { + RegistrationStatus::Completed => { + inc!("lp_registration_success_total"); + } + RegistrationStatus::Failed => { + inc!("lp_registration_failed_total"); + } + RegistrationStatus::PendingMoreData => { + inc!("lp_registration_pending_more_data"); + } + } + + result + } + + /// Register a WireGuard peer and return gateway data along with the client_id + async fn register_wg_peer( + &self, + sender: ReceiverIndex, + peer_key: PeerPublicKey, + ticket_type: nym_credentials_interface::TicketType, + psk: Key, + ) -> Result<(), GatewayError> { + // Allocate IPs from centralized pool managed by PeerController + let defguard_key = Key::new(peer_key.to_bytes()); + + let registration_data = nym_wireguard::PeerRegistrationData::new(defguard_key.clone(), psk); + + let psk = registration_data.preshared_key.clone(); + + // Request IP allocation from PeerController + let ip_pair = self.peer_manager.register_peer(registration_data).await?; + + let client_ipv4 = ip_pair.ipv4; + let client_ipv6 = ip_pair.ipv6; + + info!("Allocated IPs for peer {peer_key}: {client_ipv4} / {client_ipv6}"); + + // Create WireGuard peer with allocated IPs + let mut peer = Peer::new(defguard_key); + peer.endpoint = None; + peer.allowed_ips = vec![ + format!("{client_ipv4}/32").parse()?, + format!("{client_ipv6}/128").parse()?, + ]; + peer.persistent_keepalive_interval = Some(25); + peer.preshared_key = Some(psk); + + // Store peer in database FIRST (before adding to controller) + // This ensures bandwidth storage exists when controller's generate_bandwidth_manager() is called + let client_id = self + .storage + .insert_wireguard_peer(&peer, ticket_type.into()) + .await + .map_err(|e| { + error!("Failed to store WireGuard peer in database: {}", e); + GatewayError::InternalError(format!("Failed to store peer: {}", e)) + })?; + + // Create bandwidth entry for the client + // This must happen BEFORE AddPeer because generate_bandwidth_manager() expects it to exist + credential_storage_preparation(self.ecash_verifier.clone(), client_id).await?; + + // Now send peer to WireGuard controller and track latency + let controller_start = std::time::Instant::now(); + let result = self.peer_manager.add_peer(peer).await; + + // Record peer controller channel latency + let latency = controller_start.elapsed().as_secs_f64(); + add_histogram_obs!( + "wg_peer_controller_channel_latency_seconds", + latency, + WG_CONTROLLER_LATENCY_BUCKETS + ); + + result?; + + // Get gateway's actual WireGuard public key + let gateway_pubkey = *self.keypair().public_key(); + + // Get gateway's WireGuard endpoint from config + let gateway_endpoint = self.wireguard_config().bind_address; + self.registrations_in_progress.lock().await.insert( + sender, + TimestampedState::new(PendingRegistrationState { + client_id, + peer_key, + ticket_type, + wireguard_config: WireguardConfiguration { + public_key: gateway_pubkey, + endpoint: gateway_endpoint, + private_ipv4: client_ipv4, + private_ipv6: client_ipv6, + }, + }), + ); + Ok(()) + } +} + +// TODO: dedup /// Prepare bandwidth storage for a client async fn credential_storage_preparation( ecash_verifier: Arc, @@ -83,6 +518,7 @@ async fn credential_storage_preparation( Ok(bandwidth) } +// TODO: dedup /// Verify credential and allocate bandwidth using CredentialVerifier async fn credential_verification( ecash_verifier: Arc, @@ -131,375 +567,3 @@ async fn credential_verification( Ok(allocated) } - -/// Check if WG peer already registered, return cached response if so. -/// -/// This enables idempotent registration: if a client retries registration -/// with the same WG public key (e.g., after network failure), we return -/// the existing registration data instead of re-processing. This prevents -/// wasting credentials on network issues. -async fn check_existing_registration( - wg_key_str: &str, - state: &LpHandlerState, -) -> Option { - // Need WG data to build GatewayData - let wg_data = state.wireguard_data.as_ref()?; - - // Look up existing peer - let peer = state.storage.get_wireguard_peer(wg_key_str).await.ok()??; - - // Convert to defguard Peer to access allowed_ips - let defguard_peer: Peer = peer.clone().try_into().ok()?; - - // Extract IPv4 and IPv6 from allowed_ips - let mut ipv4 = None; - let mut ipv6 = None; - for ip_mask in &defguard_peer.allowed_ips { - match ip_mask.address { - std::net::IpAddr::V4(v4) => ipv4 = Some(v4), - std::net::IpAddr::V6(v6) => ipv6 = Some(v6), - } - } - - let (private_ipv4, private_ipv6) = match (ipv4, ipv6) { - (Some(v4), Some(v6)) => (v4, v6), - _ => return None, // Incomplete data, treat as new registration - }; - - // Get current bandwidth - let bandwidth = state - .ecash_verifier - .storage() - .get_available_bandwidth(peer.client_id) - .await - .ok()? - .map(|b| b.available) - .unwrap_or(0); - - // Only return cached response if bandwidth was actually allocated. - // If bandwidth is 0, registration was incomplete (peer exists but - // credential verification failed or never completed). Let the caller - // proceed with normal registration flow which will handle cleanup. - if bandwidth == 0 { - return None; - } - - Some(LpRegistrationResponse::success( - bandwidth, - WireguardConfiguration { - public_key: *wg_data.keypair().public_key(), - endpoint: wg_data.config().bind_address, - private_ipv4, - private_ipv6, - }, - )) -} - -/// In the case of an already registered WG peer, update its PSK. -async fn update_peer_psk( - peer: PeerPublicKey, - psk: Key, - state: &LpHandlerState, -) -> Result<(), GatewayError> { - let encoded_psk = psk.to_lower_hex(); - state - .storage - .update_peer_psk(&peer.to_string(), Some(&encoded_psk)) - .await?; - Ok(()) -} - -async fn process_dvpn_registration( - request: Box, - state: &LpHandlerState, -) -> LpRegistrationResponse { - // Track dVPN registration attempts - inc!("lp_registration_dvpn_attempts"); - - // Check for idempotent re-registration (same WG key already registered) - // This allows clients to retry registration after network failures - // without wasting credentials - let wg_key_str = request.wg_public_key.to_string(); - if let Some(existing_response) = check_existing_registration(&wg_key_str, state).await { - // TODO: this flow will be changed in subsequent PRs as it's wasting credentials regardless - if let Err(err) = update_peer_psk(request.wg_public_key, Key::new(request.psk), state).await - { - return LpRegistrationResponse::error(format!( - "WireGuard peer PSK update failed: {err}" - )); - } - info!("LP dVPN re-registration for existing peer {wg_key_str} (idempotent)",); - inc!("lp_registration_dvpn_idempotent"); - return existing_response; - } - - // Register as WireGuard peer first to get client_id - let (gateway_data, client_id) = match register_wg_peer( - request.wg_public_key.inner().as_ref(), - request.ticket_type, - Key::new(request.psk), - state, - ) - .await - { - Ok(result) => result, - Err(e) => { - error!("LP WireGuard peer registration failed: {e}"); - inc!("lp_registration_dvpn_failed"); - inc!("lp_errors_wg_peer_registration"); - return LpRegistrationResponse::error(format!( - "WireGuard peer registration failed: {e}", - )); - } - }; - - // Verify credential with CredentialVerifier (handles double-spend, storage, etc.) - let allocated_bandwidth = - match credential_verification(state.ecash_verifier.clone(), request.credential, client_id) - .await - { - Ok(bandwidth) => bandwidth, - Err(e) => { - // Credential verification failed, remove the peer - warn!("LP credential verification failed for client {client_id}: {e}",); - inc!("lp_registration_dvpn_failed"); - if let Err(remove_err) = state - .storage - .remove_wireguard_peer(&request.wg_public_key.to_string()) - .await - { - error!( - "Failed to remove peer after credential verification failure: {remove_err}" - ); - } - return LpRegistrationResponse::error(format!( - "Credential verification failed: {e}", - )); - } - }; - - info!("LP dVPN registration successful (client_id: {client_id})"); - inc!("lp_registration_dvpn_success"); - LpRegistrationResponse::success(allocated_bandwidth, gateway_data) -} - -async fn process_mixnet_registration( - request: LpMixnetRegistrationRequest, - state: &LpHandlerState, -) -> LpRegistrationResponse { - let session_id = rand::random::(); - - // Track mixnet registration attempts - inc!("lp_registration_mixnet_attempts"); - - // Derive destination address for ActiveClientsStore lookup - let client_identity = request.client_ed25519_pubkey; - let client_address = client_identity.derive_destination_address(); - - info!("LP Mixnet registration for client {client_identity}, session {session_id}"); - - warn!("unimplemented: LP mixnet registration initial bandwidth allocation"); - // (the old implementation was wrong - it wasn't creating correct db entries) - - // Create channels for client message delivery - let (mix_sender, _mix_receiver) = mpsc::unbounded(); - let (is_active_request_sender, _is_active_request_receiver) = - mpsc::unbounded::>(); - - // Insert client into ActiveClientsStore for SURB reply delivery - if !state.active_clients_store.insert_remote( - client_address, - mix_sender, - is_active_request_sender, - OffsetDateTime::now_utc(), - ) { - warn!("LP Mixnet registration failed: client {client_identity} already registered",); - inc!("lp_registration_mixnet_failed"); - return LpRegistrationResponse::error("Client already registered".to_string()); - } - - // Get gateway identity and derive sphinx key - let gateway_identity = *state.local_lp_peer.ed25519().public_key(); - - info!("LP Mixnet registration successful (client: {client_identity})",); - inc!("lp_registration_mixnet_success"); - - LpRegistrationResponse::success_mixnet(0, LpMixnetGatewayData { gateway_identity }) -} - -/// Process an LP registration request -pub async fn process_registration( - request: LpRegistrationRequest, - state: &LpHandlerState, -) -> LpRegistrationResponse { - let registration_start = std::time::Instant::now(); - - // Track total registration attempts - inc!("lp_registration_attempts_total"); - - // 1. Validate timestamp for replay protection - if !request.validate_timestamp(30) { - warn!("LP registration failed: timestamp too old or too far in future"); - inc!("lp_registration_failed_timestamp"); - return LpRegistrationResponse::error("Invalid timestamp".to_string()); - } - - // 2. Process based on mode - let result = match request.registration_data { - LpRegistrationData::Dvpn { data } => process_dvpn_registration(data, state).await, - LpRegistrationData::Mixnet { data } => process_mixnet_registration(data, state).await, - }; - - // Track registration duration - let duration = registration_start.elapsed().as_secs_f64(); - add_histogram_obs!( - "lp_registration_duration_seconds", - duration, - LP_REGISTRATION_DURATION_BUCKETS - ); - - // Track overall success/failure - if result.success { - inc!("lp_registration_success_total"); - } else { - inc!("lp_registration_failed_total"); - } - - result -} - -/// Register a WireGuard peer and return gateway data along with the client_id -async fn register_wg_peer( - public_key_bytes: &[u8], - ticket_type: nym_credentials_interface::TicketType, - psk: Key, - state: &LpHandlerState, -) -> Result<(WireguardConfiguration, i64), GatewayError> { - let Some(wg_controller) = &state.wg_peer_controller else { - return Err(GatewayError::ServiceProviderNotRunning { - service: "WireGuard".to_string(), - }); - }; - - let Some(wg_data) = &state.wireguard_data else { - return Err(GatewayError::ServiceProviderNotRunning { - service: "WireGuard".to_string(), - }); - }; - - // Convert public key bytes to WireGuard Key - let mut key_bytes = [0u8; 32]; - if public_key_bytes.len() != 32 { - return Err(GatewayError::LpProtocolError( - "Invalid WireGuard public key length".to_string(), - )); - } - key_bytes.copy_from_slice(public_key_bytes); - let peer_key = Key::new(key_bytes); - - // Allocate IPs from centralized pool managed by PeerController - let registration_data = - nym_wireguard::PeerRegistrationData::new(peer_key.clone()).with_preshared_key(psk); - - let psk = registration_data.preshared_key.clone(); - // Request IP allocation from PeerController - let (tx, rx) = oneshot::channel(); - wg_controller - .send(PeerControlRequest::RegisterPeer { - registration_data, - response_tx: tx, - }) - .await - .map_err(|e| { - GatewayError::InternalError(format!("Failed to send IP allocation request: {}", e)) - })?; - - // Wait for IP allocation from pool - let ip_pair = rx - .await - .map_err(|e| { - GatewayError::InternalError(format!("Failed to receive IP allocation: {}", e)) - })? - .map_err(|e| { - error!("Failed to allocate IPs from pool: {}", e); - GatewayError::InternalError(format!("Failed to allocate IPs: {:?}", e)) - })?; - - let client_ipv4 = ip_pair.ipv4; - let client_ipv6 = ip_pair.ipv6; - - info!( - "Allocated IPs for peer {}: {} / {}", - peer_key, client_ipv4, client_ipv6 - ); - - // Create WireGuard peer with allocated IPs - let mut peer = Peer::new(peer_key.clone()); - peer.endpoint = None; - peer.allowed_ips = vec![ - format!("{client_ipv4}/32").parse()?, - format!("{client_ipv6}/128").parse()?, - ]; - peer.persistent_keepalive_interval = Some(25); - peer.preshared_key = psk; - - // Store peer in database FIRST (before adding to controller) - // This ensures bandwidth storage exists when controller's generate_bandwidth_manager() is called - let client_id = state - .storage - .insert_wireguard_peer(&peer, ticket_type.into()) - .await - .map_err(|e| { - error!("Failed to store WireGuard peer in database: {}", e); - GatewayError::InternalError(format!("Failed to store peer: {}", e)) - })?; - - // Create bandwidth entry for the client - // This must happen BEFORE AddPeer because generate_bandwidth_manager() expects it to exist - credential_storage_preparation(state.ecash_verifier.clone(), client_id).await?; - - // Now send peer to WireGuard controller and track latency - let controller_start = std::time::Instant::now(); - let (tx, rx) = oneshot::channel(); - wg_controller - .send(PeerControlRequest::AddPeer { - peer: peer.clone(), - response_tx: tx, - }) - .await - .map_err(|e| GatewayError::InternalError(format!("Failed to send peer request: {}", e)))?; - - let result = rx - .await - .map_err(|e| { - GatewayError::InternalError(format!("Failed to receive peer response: {}", e)) - })? - .map_err(|e| GatewayError::InternalError(format!("Failed to add peer: {:?}", e))); - - // Record peer controller channel latency - let latency = controller_start.elapsed().as_secs_f64(); - add_histogram_obs!( - "wg_peer_controller_channel_latency_seconds", - latency, - WG_CONTROLLER_LATENCY_BUCKETS - ); - - result?; - - // Get gateway's actual WireGuard public key - let gateway_pubkey = *wg_data.keypair().public_key(); - - // Get gateway's WireGuard endpoint from config - let gateway_endpoint = wg_data.config().bind_address; - - // Create GatewayData response (matching authenticator response format) - Ok(( - WireguardConfiguration { - public_key: gateway_pubkey, - endpoint: gateway_endpoint, - private_ipv4: client_ipv4, - private_ipv6: client_ipv6, - }, - client_id, - )) -} diff --git a/gateway/src/node/mod.rs b/gateway/src/node/mod.rs index b904143ac6..62263d521c 100644 --- a/gateway/src/node/mod.rs +++ b/gateway/src/node/mod.rs @@ -36,6 +36,7 @@ use tokio::sync::Semaphore; use tracing::*; use zeroize::Zeroizing; +use crate::node::lp_listener::peer_manager::PeerManager; pub use crate::node::upgrade_mode::watcher::UpgradeModeWatcher; pub use client_handling::active_clients::ActiveClientsStore; pub use lp_listener::LpConfig; @@ -328,13 +329,18 @@ impl GatewayTasksBuilder { pub async fn build_lp_listener( &mut self, + upgrade_mode_common_state: UpgradeModeDetails, active_clients_store: ActiveClientsStore, ) -> Result { // Get WireGuard peer controller if available - let wg_peer_controller = self - .wireguard_data - .as_ref() - .map(|wg_data| wg_data.inner.peer_tx().clone()); + let Some(wireguard_data) = &self.wireguard_data else { + return Err(GatewayError::InternalWireguardError( + "wireguard not set".to_string(), + )); + }; + + // TODO: combine this `PeerManager` with the one used within the authenticator + let peer_manager = Arc::new(PeerManager::new(wireguard_data.inner.clone())); let handler_state = lp_listener::LpHandlerState { ecash_verifier: self.ecash_manager().await?, @@ -346,12 +352,13 @@ impl GatewayTasksBuilder { .with_kem_psq_key(self.kem_psq_keys.clone()), metrics: self.metrics.clone(), active_clients_store, - wg_peer_controller, - wireguard_data: self.wireguard_data.as_ref().map(|wd| wd.inner.clone()), + upgrade_mode: upgrade_mode_common_state, + peer_manager, lp_config: self.config.lp, outbound_mix_sender: self.mix_packet_sender.clone(), handshake_states: Arc::new(dashmap::DashMap::new()), session_states: Arc::new(dashmap::DashMap::new()), + registrations_in_progress: Default::default(), forward_semaphore: Arc::new(Semaphore::new( self.config.lp.debug.max_concurrent_forwards, )), diff --git a/integration-tests/Cargo.toml b/integration-tests/Cargo.toml index f7d2f0897c..3d825c7e73 100644 --- a/integration-tests/Cargo.toml +++ b/integration-tests/Cargo.toml @@ -26,7 +26,7 @@ nym-lp-transport = { path = "../common/nym-lp-transport", features = ["io-mocks" nym-gateway = { path = "../gateway" } sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite"] } tracing = { workspace = true } - +futures = { workspace = true } [lints] workspace = true diff --git a/integration-tests/src/lp_registration.rs b/integration-tests/src/lp_registration.rs index 595c48128f..d292a2a441 100644 --- a/integration-tests/src/lp_registration.rs +++ b/integration-tests/src/lp_registration.rs @@ -5,11 +5,16 @@ mod tests { use anyhow::Context; use nym_bandwidth_controller::mock::MockBandwidthController; + use nym_credential_verification::UpgradeModeState; use nym_credential_verification::ecash::MockEcashManager; + use nym_credential_verification::upgrade_mode::{ + UpgradeModeCheckConfig, UpgradeModeCheckRequestSender, UpgradeModeDetails, + }; use nym_credentials_interface::TicketType; use nym_crypto::asymmetric::{ed25519, x25519}; use nym_gateway::GatewayError; use nym_gateway::node::lp_listener::handler::LpConnectionHandler; + use nym_gateway::node::lp_listener::peer_manager::PeerManager; use nym_gateway::node::lp_listener::{ LpDebug, LpHandlerState, LpLocalPeer, MixForwardingReceiver, PeerControlRequest, WireguardGatewayData, mix_forwarding_channels, @@ -174,6 +179,17 @@ mod tests { Ok(GatewayStorage::from_connection_pool(conn_pool, 100).await?) } + const DUMMY_ATTESTER_ED25519_PRIVATE_KEY: [u8; 32] = [ + 108, 49, 193, 21, 126, 161, 249, 85, 242, 207, 74, 195, 238, 6, 64, 149, 201, 140, 248, + 163, 122, 170, 79, 198, 87, 85, 36, 29, 243, 92, 64, 161, + ]; + + pub(crate) fn dummy_attester_public_key() -> ed25519::PublicKey { + let private_key = + ed25519::PrivateKey::from_bytes(&Self::DUMMY_ATTESTER_ED25519_PRIVATE_KEY).unwrap(); + private_key.public_key() + } + async fn mock(rng: &mut (impl RngCore + CryptoRng)) -> anyhow::Result { let base = Party::generate(rng); @@ -199,11 +215,27 @@ mod tests { // create wireguard data let (wireguard_data, peer_request_rx) = Self::wireguard_data(&base); + let (um_recheck_tx, um_recheck_rx) = futures::channel::mpsc::unbounded(); + + // TODO: use it if we ever want to test UM + let _ = um_recheck_rx; + // mock the wg peer controller let (mock_peer_controller, peer_controller_state) = mock_peer_controller(peer_request_rx); + let upgrade_mode_state = UpgradeModeState::new(Self::dummy_attester_public_key()); + let upgrade_mode_details = UpgradeModeDetails::new( + UpgradeModeCheckConfig { + // essentially we never want to trigger this in our tests + min_staleness_recheck: Duration::from_nanos(1), + }, + UpgradeModeCheckRequestSender::new(um_recheck_tx), + upgrade_mode_state.clone(), + ); + // registering particular responses for peer controller is up to given test + let peer_manager = Arc::new(PeerManager::new(wireguard_data)); let lp_state = LpHandlerState { // use mock instance of ecash verifier @@ -220,9 +252,8 @@ mod tests { active_clients_store: ActiveClientsStore::new(), // handles required for wg registration - wg_peer_controller: Some(wireguard_data.peer_tx().clone()), - - wireguard_data: Some(wireguard_data), + upgrade_mode: upgrade_mode_details, + peer_manager, // use default lp config (with enabled flag) lp_config, @@ -237,6 +268,7 @@ mod tests { session_states: Arc::new(Default::default()), // sensible default value for tests + registrations_in_progress: Default::default(), forward_semaphore, }; @@ -376,6 +408,7 @@ mod tests { mod using_lp_registration_client { use super::*; use nym_registration_client::NestedLpSession; + use nym_wireguard::DefguardPeer; #[tokio::test] async fn test_basic_lp_entry_registration() -> anyhow::Result<()> { @@ -430,6 +463,16 @@ mod tests { ) .await; + // 3) peer query - check for prior registrations + let query_res = Ok::<_, nym_wireguard::Error>(Option::::None); + let key = client_key.to_wg_key(); + entry + .register_peer_controller_response( + PeerControlRequestType::QueryPeer { key }, + query_res, + ) + .await; + // 4. spawn peer controller to be able to handle dvpn registration requests entry.spawn_peer_controller(); @@ -440,7 +483,7 @@ mod tests { let wg_keypair = client_data.base.x25519_wg_keys; let gateway_identity = entry.base.peer.ed25519().public_key(); let registration_result = client - .register( + .register_dvpn( &mut client_rng, &wg_keypair, gateway_identity, @@ -506,7 +549,7 @@ mod tests { let wg_keypair = client_data.base.x25519_wg_keys; let gateway_identity = entry.base.peer.ed25519().public_key(); let registration_result = client - .register( + .register_dvpn( &mut client_rng, &wg_keypair, gateway_identity, @@ -520,7 +563,10 @@ mod tests { let LpClientError::Transport(err) = registration_result else { panic!("unexpected error"); }; - assert_eq!(err, "Cannot register: handshake not completed"); + assert_eq!( + err, + "State machine not available - has the handshake been completed?" + ); // 5. stop the gateway task and finish the test entry.stop_tasks().await?; @@ -590,6 +636,16 @@ mod tests { ) .await; + // 3) peer query - check for prior registrations + let query_res = Ok::<_, nym_wireguard::Error>(Option::::None); + let key = client_key.to_wg_key(); + entry + .register_peer_controller_response( + PeerControlRequestType::QueryPeer { key }, + query_res, + ) + .await; + // 5. spawn peer controller to be able to handle dvpn registration requests entry.spawn_peer_controller(); @@ -630,6 +686,15 @@ mod tests { ) .await; + // 3) peer query - check for prior registrations + let query_res = Ok::<_, nym_wireguard::Error>(Option::::None); + let key = client_key.to_wg_key(); + exit.register_peer_controller_response( + PeerControlRequestType::QueryPeer { key }, + query_res, + ) + .await; + // 11. spawn peer controller to be able to handle dvpn registration requests exit.spawn_peer_controller(); @@ -646,7 +711,7 @@ mod tests { // 13. Perform handshake and registration with exit gateway (all via entry forwarding) let exit_registration_result = nested_session - .handshake_and_register( + .handshake_and_register_dvpn( &mut entry_client, &mut client_rng, &client_data.base.x25519_wg_keys, @@ -659,7 +724,7 @@ mod tests { // 14. complete registration with the entry let entry_registration_result = entry_client - .register( + .register_dvpn( &mut client_rng, &client_data.base.x25519_wg_keys, entry.base.peer.ed25519().public_key(), diff --git a/nym-gateway-probe/src/common/bandwidth_helpers.rs b/nym-gateway-probe/src/common/bandwidth_helpers.rs index 1f0048d722..dea3d6bfd5 100644 --- a/nym-gateway-probe/src/common/bandwidth_helpers.rs +++ b/nym-gateway-probe/src/common/bandwidth_helpers.rs @@ -2,18 +2,38 @@ // SPDX-License-Identifier: Apache-2.0 use anyhow::{Context, bail}; +use nym_bandwidth_controller::BandwidthTicketProvider; use nym_bandwidth_controller::error::BandwidthControllerError; +use nym_bandwidth_controller::mock::MockBandwidthController; use nym_client_core::client::base_client::storage::OnDiskPersistent; -use nym_credentials::CredentialSpendingData; use nym_credentials_interface::TicketType; use nym_node_status_client::models::AttachedTicketMaterials; use nym_sdk::bandwidth::BandwidthImporter; -use nym_sdk::mixnet::{DisconnectedMixnetClient, EphemeralCredentialStorage}; +use nym_sdk::mixnet::{CredentialStorage, DisconnectedMixnetClient, EphemeralCredentialStorage}; +use nym_validator_client::QueryHttpRpcNyxdClient; use nym_validator_client::nyxd::error::NyxdError; use std::time::Duration; -use time::OffsetDateTime; use tracing::{error, info}; +pub(crate) fn build_bandwidth_controller( + rpc_client: QueryHttpRpcNyxdClient, + on_disk_storage: S, + use_mock_ecash: bool, +) -> Box +where + S: CredentialStorage + 'static, + S::StorageError: Send + Sync + 'static, +{ + if !use_mock_ecash { + Box::new(nym_bandwidth_controller::BandwidthController::new( + on_disk_storage, + rpc_client, + )) + } else { + Box::new(MockBandwidthController::default()) + } +} + pub(crate) async fn import_bandwidth( bandwidth_importer: BandwidthImporter<'_, EphemeralCredentialStorage>, attached_ticket_materials: AttachedTicketMaterials, @@ -157,93 +177,3 @@ pub(crate) async fn acquire_bandwidth( bail!("failed to acquire bandwidth after {MAX_RETRIES} attempts") } - -/// Create a dummy credential for mock ecash testing -/// -/// Gateway with --lp-use-mock-ecash accepts any credential without verification, -/// so we only need to provide properly structured data with correct types. -/// -/// This is useful for local testing without requiring blockchain access or funded accounts. -/// -/// This uses a pre-serialized test credential from the wireguard tests - since MockEcashManager -/// doesn't verify anything, any valid CredentialSpendingData structure will work. -#[allow(clippy::expect_used)] // Test helper with hardcoded valid data -pub(crate) fn create_dummy_credential( - _gateway_identity: &[u8; 32], - _ticket_type: TicketType, -) -> CredentialSpendingData { - // This is a valid serialized CredentialSpendingData taken from integration tests - // See: common/wireguard-private-metadata/tests/src/lib.rs:CREDENTIAL_BYTES - const CREDENTIAL_BYTES: [u8; 1245] = [ - 0, 0, 4, 133, 96, 179, 223, 185, 136, 23, 213, 166, 59, 203, 66, 69, 209, 181, 227, 254, - 16, 102, 98, 237, 59, 119, 170, 111, 31, 194, 51, 59, 120, 17, 115, 229, 79, 91, 11, 139, - 154, 2, 212, 23, 68, 70, 167, 3, 240, 54, 224, 171, 221, 1, 69, 48, 60, 118, 119, 249, 123, - 35, 172, 227, 131, 96, 232, 209, 187, 123, 4, 197, 102, 90, 96, 45, 125, 135, 140, 99, 1, - 151, 17, 131, 143, 157, 97, 107, 139, 232, 212, 87, 14, 115, 253, 255, 166, 167, 186, 43, - 90, 96, 173, 105, 120, 40, 10, 163, 250, 224, 214, 200, 178, 4, 160, 16, 130, 59, 76, 193, - 39, 240, 3, 101, 141, 209, 183, 226, 186, 207, 56, 210, 187, 7, 164, 240, 164, 205, 37, 81, - 184, 214, 193, 195, 90, 205, 238, 225, 195, 104, 12, 123, 203, 57, 233, 243, 215, 145, 195, - 196, 57, 38, 125, 172, 18, 47, 63, 165, 110, 219, 180, 40, 58, 116, 92, 254, 160, 98, 48, - 92, 254, 232, 107, 184, 80, 234, 60, 160, 235, 249, 76, 41, 38, 165, 28, 40, 136, 74, 48, - 166, 50, 245, 23, 201, 140, 101, 79, 93, 235, 128, 186, 146, 126, 180, 134, 43, 13, 186, - 19, 195, 48, 168, 201, 29, 216, 95, 176, 198, 132, 188, 64, 39, 212, 150, 32, 52, 53, 38, - 228, 199, 122, 226, 217, 75, 40, 191, 151, 48, 164, 242, 177, 79, 14, 122, 105, 151, 85, - 88, 199, 162, 17, 96, 103, 83, 178, 128, 9, 24, 30, 74, 108, 241, 85, 240, 166, 97, 241, - 85, 199, 11, 198, 226, 234, 70, 107, 145, 28, 208, 114, 51, 12, 234, 108, 101, 202, 112, - 48, 185, 22, 159, 67, 109, 49, 27, 149, 90, 109, 32, 226, 112, 7, 201, 208, 209, 104, 31, - 97, 134, 204, 145, 27, 181, 206, 181, 106, 32, 110, 136, 115, 249, 201, 111, 5, 245, 203, - 71, 121, 169, 126, 151, 178, 236, 59, 221, 195, 48, 135, 115, 6, 50, 227, 74, 97, 107, 107, - 213, 90, 2, 203, 154, 138, 47, 128, 52, 134, 128, 224, 51, 65, 240, 90, 8, 55, 175, 180, - 178, 204, 206, 168, 110, 51, 57, 189, 169, 48, 169, 136, 121, 99, 51, 170, 178, 214, 74, 1, - 96, 151, 167, 25, 173, 180, 171, 155, 10, 55, 142, 234, 190, 113, 90, 79, 80, 244, 71, 166, - 30, 235, 113, 150, 133, 1, 218, 17, 109, 111, 223, 24, 216, 177, 41, 2, 204, 65, 221, 212, - 207, 236, 144, 6, 65, 224, 55, 42, 1, 1, 161, 134, 118, 127, 111, 220, 110, 127, 240, 71, - 223, 129, 12, 93, 20, 220, 60, 56, 71, 146, 184, 95, 132, 69, 28, 56, 53, 192, 213, 22, - 119, 230, 152, 225, 182, 188, 163, 219, 37, 175, 247, 73, 14, 247, 38, 72, 243, 1, 48, 131, - 59, 8, 13, 96, 143, 185, 127, 241, 161, 217, 24, 149, 193, 40, 16, 30, 202, 151, 28, 119, - 240, 153, 101, 156, 61, 193, 72, 245, 199, 181, 12, 231, 65, 166, 67, 142, 121, 207, 202, - 58, 197, 113, 188, 248, 42, 124, 105, 48, 161, 241, 55, 209, 36, 194, 27, 63, 233, 144, - 189, 85, 117, 234, 9, 139, 46, 31, 206, 114, 95, 131, 29, 240, 13, 81, 142, 140, 133, 33, - 30, 41, 141, 37, 80, 217, 95, 221, 76, 115, 86, 201, 165, 51, 252, 9, 28, 209, 1, 48, 150, - 74, 248, 212, 187, 222, 66, 210, 3, 200, 19, 217, 171, 184, 42, 148, 53, 150, 57, 50, 6, - 227, 227, 62, 49, 42, 148, 148, 157, 82, 191, 58, 24, 34, 56, 98, 120, 89, 105, 176, 85, - 15, 253, 241, 41, 153, 195, 136, 1, 48, 142, 126, 213, 101, 223, 79, 133, 230, 105, 38, - 161, 149, 2, 21, 136, 150, 42, 72, 218, 85, 146, 63, 223, 58, 108, 186, 183, 248, 62, 20, - 47, 34, 113, 160, 177, 204, 181, 16, 24, 212, 224, 35, 84, 51, 168, 56, 136, 11, 1, 48, - 135, 242, 62, 149, 230, 178, 32, 224, 119, 26, 234, 163, 237, 224, 114, 95, 112, 140, 170, - 150, 96, 125, 136, 221, 180, 78, 18, 11, 12, 184, 2, 198, 217, 119, 43, 69, 4, 172, 109, - 55, 183, 40, 131, 172, 161, 88, 183, 101, 1, 48, 173, 216, 22, 73, 42, 255, 211, 93, 249, - 87, 159, 115, 61, 91, 55, 130, 17, 216, 60, 34, 122, 55, 8, 244, 244, 153, 151, 57, 5, 144, - 178, 55, 249, 64, 211, 168, 34, 148, 56, 89, 92, 203, 70, 124, 219, 152, 253, 165, 0, 32, - 203, 116, 63, 7, 240, 222, 82, 86, 11, 149, 167, 72, 224, 55, 190, 66, 201, 65, 168, 184, - 96, 47, 194, 241, 168, 124, 7, 74, 214, 250, 37, 76, 32, 218, 69, 122, 103, 215, 145, 169, - 24, 212, 229, 168, 106, 10, 144, 31, 13, 25, 178, 242, 250, 106, 159, 40, 48, 163, 165, 61, - 130, 57, 146, 4, 73, 32, 254, 233, 125, 135, 212, 29, 111, 4, 177, 114, 15, 210, 170, 82, - 108, 110, 62, 166, 81, 209, 106, 176, 156, 14, 133, 242, 60, 127, 120, 242, 28, 97, 0, 1, - 32, 103, 93, 109, 89, 240, 91, 1, 84, 150, 50, 206, 157, 203, 49, 220, 120, 234, 175, 234, - 150, 126, 225, 94, 163, 164, 199, 138, 114, 62, 99, 106, 112, 1, 32, 171, 40, 220, 82, 241, - 203, 76, 146, 111, 139, 182, 179, 237, 182, 115, 75, 128, 201, 107, 43, 214, 0, 135, 217, - 160, 68, 150, 232, 144, 114, 237, 98, 32, 30, 134, 232, 59, 93, 163, 253, 244, 13, 202, 52, - 147, 168, 83, 121, 123, 95, 21, 210, 209, 225, 223, 143, 49, 10, 205, 238, 1, 22, 83, 81, - 70, 1, 32, 26, 76, 6, 234, 160, 50, 139, 102, 161, 232, 155, 106, 130, 171, 226, 210, 233, - 178, 85, 247, 71, 123, 55, 53, 46, 67, 148, 137, 156, 207, 208, 107, 1, 32, 102, 31, 4, 98, - 110, 156, 144, 61, 229, 140, 198, 84, 196, 238, 128, 35, 131, 182, 137, 125, 241, 95, 69, - 131, 170, 27, 2, 144, 75, 72, 242, 102, 3, 32, 121, 80, 45, 173, 56, 65, 218, 27, 40, 251, - 197, 32, 169, 104, 123, 110, 90, 78, 153, 166, 38, 9, 129, 228, 99, 8, 1, 116, 142, 233, - 162, 69, 32, 216, 169, 159, 116, 95, 12, 63, 176, 195, 6, 183, 123, 135, 75, 61, 112, 106, - 83, 235, 176, 41, 27, 248, 48, 71, 165, 170, 12, 92, 103, 103, 81, 32, 58, 74, 75, 145, - 192, 94, 153, 69, 80, 128, 241, 3, 16, 117, 192, 86, 161, 103, 44, 174, 211, 196, 182, 124, - 55, 11, 107, 142, 49, 88, 6, 41, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, - 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, - 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 0, 37, 139, 240, 0, 0, - 0, 0, 0, 0, 0, 1, - ]; - - let mut credential = CredentialSpendingData::try_from_bytes(&CREDENTIAL_BYTES) - .expect("Failed to deserialize test credential - this is a bug in the test harness"); - - // Update spend_date to today to pass validation - credential.spend_date = OffsetDateTime::now_utc().date(); - - credential -} diff --git a/nym-gateway-probe/src/common/probe_tests.rs b/nym-gateway-probe/src/common/probe_tests.rs index ef039e6ab4..a892901aa7 100644 --- a/nym-gateway-probe/src/common/probe_tests.rs +++ b/nym-gateway-probe/src/common/probe_tests.rs @@ -8,7 +8,7 @@ use crate::common::types::{ use crate::common::wireguard::{ TwoHopWgTunnelConfig, WgTunnelConfig, run_tunnel_tests, run_two_hop_tunnel_tests, }; -use crate::common::{bandwidth_helpers, helpers, icmp}; +use crate::common::{helpers, icmp}; use crate::config::NetstackArgs; use anyhow::bail; use base64::{Engine, engine::general_purpose}; @@ -19,6 +19,7 @@ use nym_authenticator_requests::{ AuthenticatorVersion, client_message::ClientMessage, response::AuthenticatorResponse, v2, v3, v4, v5, v6, }; +use nym_bandwidth_controller::BandwidthTicketProvider; use nym_config::defaults::mixnet_vpn::{NYM_TUN_DEVICE_ADDRESS_V4, NYM_TUN_DEVICE_ADDRESS_V6}; use nym_connection_monitor::self_ping_and_wait; use nym_credentials_interface::{CredentialSpendingData, TicketType}; @@ -147,22 +148,11 @@ pub async fn wg_probe( Ok(wg_outcome) } -pub async fn lp_registration_probe( +pub async fn lp_registration_probe( gateway_identity: NodeIdentity, gateway_lp_data: TestedNodeLpDetails, - bandwidth_controller: &nym_bandwidth_controller::BandwidthController< - nym_validator_client::nyxd::NyxdClient, - St, - >, - use_mock_ecash: bool, -) -> anyhow::Result -where - St: nym_sdk::mixnet::CredentialStorage + Clone + Send + Sync + 'static, - ::StorageError: Send + Sync, -{ - use nym_crypto::asymmetric::ed25519; - use nym_registration_client::LpRegistrationClient; - + bandwidth_controller: &dyn BandwidthTicketProvider, +) -> anyhow::Result { let lp_address = gateway_lp_data.address; let peer = helpers::to_lp_remote_peer(gateway_identity, gateway_lp_data); @@ -223,44 +213,22 @@ where // Register using the new packet-per-connection API (returns GatewayData directly) let ticket_type = TicketType::V1WireguardEntry; - let gateway_data = if use_mock_ecash { - info!("Using mock ecash credential for LP registration"); - let credential = bandwidth_helpers::create_dummy_credential( - &gateway_ed25519_pubkey.to_bytes(), + let gateway_data = match client + .register_dvpn( + &mut rng, + &wg_keypair, + &gateway_ed25519_pubkey, + bandwidth_controller, ticket_type, - ); - - match client - .register_with_credential(&mut rng, &wg_keypair, credential, ticket_type) - .await - { - Ok(data) => data, - Err(e) => { - let error_msg = format!("LP registration failed (mock ecash): {}", e); - error!("{}", error_msg); - lp_outcome.error = Some(error_msg); - return Ok(lp_outcome); - } - } - } else { - info!("Using real bandwidth controller for LP registration"); - match client - .register( - &mut rng, - &wg_keypair, - &gateway_ed25519_pubkey, - bandwidth_controller, - ticket_type, - ) - .await - { - Ok(data) => data, - Err(e) => { - let error_msg = format!("LP registration failed: {}", e); - error!("{}", error_msg); - lp_outcome.error = Some(error_msg); - return Ok(lp_outcome); - } + ) + .await + { + Ok(data) => data, + Err(e) => { + let error_msg = format!("LP registration failed: {}", e); + error!("{}", error_msg); + lp_outcome.error = Some(error_msg); + return Ok(lp_outcome); } }; @@ -291,21 +259,13 @@ where // but subsequent DNS/ping tests may timeout. This appears to be related to Apple Container // Runtime networking quirks combined with our NAT/iptables configuration. Tracked in // beads issue nym-vbdo. Workaround: restart the localnet containers between probe runs. -pub async fn wg_probe_lp( +pub async fn wg_probe_lp( entry_gateway: &TestedNodeDetails, exit_gateway: &TestedNodeDetails, - bandwidth_controller: &nym_bandwidth_controller::BandwidthController< - nym_validator_client::nyxd::NyxdClient, - St, - >, - use_mock_ecash: bool, + bandwidth_controller: &dyn BandwidthTicketProvider, awg_args: String, netstack_args: NetstackArgs, -) -> anyhow::Result -where - St: nym_sdk::mixnet::CredentialStorage + Clone + Send + Sync + 'static, - ::StorageError: Send + Sync, -{ +) -> anyhow::Result { // Validate that both gateways have required information let entry_lp_data = entry_gateway .lp_data @@ -366,47 +326,24 @@ where .map_err(|e| anyhow::anyhow!("Invalid exit gateway identity: {}", e))?; // Perform handshake and registration with exit gateway via forwarding - let exit_gateway_data = if use_mock_ecash { - info!("Using mock ecash credential for exit gateway registration"); - let credential = bandwidth_helpers::create_dummy_credential( - &exit_gateway_pubkey.to_bytes(), + let exit_gateway_data = match nested_session + .handshake_and_register_dvpn( + &mut entry_client, + &mut rng, + &exit_wg_keypair, + &exit_gateway_pubkey, + bandwidth_controller, TicketType::V1WireguardExit, - ); - match nested_session - .handshake_and_register_with_credential( - &mut entry_client, - &mut rng, - &exit_wg_keypair, - credential, - TicketType::V1WireguardExit, - ) - .await - { - Ok(data) => data, - Err(e) => { - error!("Failed to register with exit gateway (mock ecash): {}", e); - return Ok(wg_outcome); - } - } - } else { - match nested_session - .handshake_and_register( - &mut entry_client, - &mut rng, - &exit_wg_keypair, - &exit_gateway_pubkey, - bandwidth_controller, - TicketType::V1WireguardExit, - ) - .await - { - Ok(data) => data, - Err(e) => { - error!("Failed to register with exit gateway: {}", e); - return Ok(wg_outcome); - } + ) + .await + { + Ok(data) => data, + Err(e) => { + error!("Failed to register with exit gateway: {}", e); + return Ok(wg_outcome); } }; + info!("Exit gateway registration successful via forwarding"); // STEP 3: Register with entry gateway @@ -416,43 +353,20 @@ where .map_err(|e| anyhow::anyhow!("Invalid entry gateway identity: {}", e))?; // Use packet-per-connection register() which returns GatewayData directly - let entry_gateway_data = if use_mock_ecash { - info!("Using mock ecash credential for entry gateway registration"); - let credential = bandwidth_helpers::create_dummy_credential( - &entry_gateway_pubkey.to_bytes(), + let entry_gateway_data = match entry_client + .register_dvpn( + &mut rng, + &entry_wg_keypair, + &entry_gateway_pubkey, + bandwidth_controller, TicketType::V1WireguardEntry, - ); - match entry_client - .register_with_credential( - &mut rng, - &entry_wg_keypair, - credential, - TicketType::V1WireguardEntry, - ) - .await - { - Ok(data) => data, - Err(e) => { - error!("Failed to register with entry gateway (mock ecash): {}", e); - return Ok(wg_outcome); - } - } - } else { - match entry_client - .register( - &mut rng, - &entry_wg_keypair, - &entry_gateway_pubkey, - bandwidth_controller, - TicketType::V1WireguardEntry, - ) - .await - { - Ok(data) => data, - Err(e) => { - error!("Failed to register with entry gateway: {}", e); - return Ok(wg_outcome); - } + ) + .await + { + Ok(data) => data, + Err(e) => { + error!("Failed to register with entry gateway: {}", e); + return Ok(wg_outcome); } }; info!("Entry gateway registration successful"); diff --git a/nym-gateway-probe/src/lib.rs b/nym-gateway-probe/src/lib.rs index b74cf2df7d..ccd42a4121 100644 --- a/nym-gateway-probe/src/lib.rs +++ b/nym-gateway-probe/src/lib.rs @@ -24,7 +24,9 @@ use url::Url; mod common; pub mod config; -use crate::common::bandwidth_helpers::{acquire_bandwidth, import_bandwidth}; +use crate::common::bandwidth_helpers::{ + acquire_bandwidth, build_bandwidth_controller, import_bandwidth, +}; pub use crate::common::nodes::{ DirectoryNode, NymApiDirectory, TestedNode, TestedNodeDetails, TestedNodeLpDetails, query_gateway_by_ip, @@ -435,16 +437,14 @@ impl Probe { &NymNetworkDetails::new_from_env(), )?; let client = nym_validator_client::nyxd::NyxdClient::connect(config, nyxd_url.as_str())?; - let bw_controller = nym_bandwidth_controller::BandwidthController::new( - storage.credential_store().clone(), - client, - ); + + let bw_controller = + build_bandwidth_controller(client, storage.credential_store().clone(), use_mock_ecash); // Run LP registration probe - let lp_outcome = - lp_registration_probe(node_info.identity, lp_data, &bw_controller, use_mock_ecash) - .await - .unwrap_or_default(); + let lp_outcome = lp_registration_probe(node_info.identity, lp_data, &bw_controller) + .await + .unwrap_or_default(); // Return result with only LP outcome Ok(ProbeResult { @@ -644,9 +644,11 @@ impl Probe { )?; let client = nym_validator_client::nyxd::NyxdClient::connect(config, nyxd_url.as_str())?; - let bw_controller = nym_bandwidth_controller::BandwidthController::new( - storage.credential_store().clone(), + + let bw_controller = build_bandwidth_controller( client, + storage.credential_store().clone(), + use_mock_ecash, ); // Determine entry and exit gateways @@ -691,7 +693,6 @@ impl Probe { &entry_gateway, &exit_gateway, &bw_controller, - use_mock_ecash, self.amnezia_args.clone(), self.netstack_args.clone(), ) @@ -773,15 +774,15 @@ impl Probe { )?; let client = nym_validator_client::nyxd::NyxdClient::connect(config, nyxd_url.as_str())?; - let bw_controller = nym_bandwidth_controller::BandwidthController::new( - storage.credential_store().clone(), + let bw_controller = build_bandwidth_controller( client, + storage.credential_store().clone(), + use_mock_ecash, ); - let outcome = - lp_registration_probe(node_info.identity, lp_data, &bw_controller, use_mock_ecash) - .await - .unwrap_or_default(); + let outcome = lp_registration_probe(node_info.identity, lp_data, &bw_controller) + .await + .unwrap_or_default(); Some(outcome) } else { diff --git a/nym-node/src/node/mod.rs b/nym-node/src/node/mod.rs index 1c9f3bb3fb..01fb355825 100644 --- a/nym-node/src/node/mod.rs +++ b/nym-node/src/node/mod.rs @@ -704,7 +704,10 @@ impl NymNode { self.config.gateway_tasks.lp.data_bind_address, ); let mut lp_listener = gateway_tasks_builder - .build_lp_listener(active_clients_store.clone()) + .build_lp_listener( + upgrade_mode_common_state.clone(), + active_clients_store.clone(), + ) .await?; self.shutdown_tracker() .try_spawn_named(async move { lp_listener.run().await }, "LpListener"); diff --git a/nym-registration-client/src/lib.rs b/nym-registration-client/src/lib.rs index 42bd7b4be6..e89143fad6 100644 --- a/nym-registration-client/src/lib.rs +++ b/nym-registration-client/src/lib.rs @@ -218,7 +218,7 @@ impl RegistrationClient { // Perform handshake and registration with exit gateway (all via entry forwarding) let exit_gateway_data = nested_session - .handshake_and_register::( + .handshake_and_register_dvpn::( &mut entry_client, rng, &self.config.exit.keys, @@ -238,7 +238,7 @@ impl RegistrationClient { // STEP 3: Register with entry gateway (packet-per-connection) tracing::info!("Registering with entry gateway"); let entry_gateway_data = entry_client - .register( + .register_dvpn( rng, &self.config.entry.keys, &self.config.entry.node.identity, diff --git a/nym-registration-client/src/lp_client/client.rs b/nym-registration-client/src/lp_client/client.rs index 4d15e2a5bc..827c98e1b1 100644 --- a/nym-registration-client/src/lp_client/client.rs +++ b/nym-registration-client/src/lp_client/client.rs @@ -6,12 +6,14 @@ use super::config::LpConfig; use super::error::{LpClientError, Result}; use crate::lp_client::helpers::{ - convert_forward_data, convert_registration_request, try_convert_forward_response, - try_convert_registration_response, + LpDataDeliverExt, LpDataSendExt, convert_forward_data, try_convert_forward_response, +}; +use crate::lp_client::state_machine_helpers::{ + extract_forwarded_response, get_recv_key, get_send_key, prepare_send_packet, }; use bytes::BytesMut; use nym_bandwidth_controller::{BandwidthTicketProvider, DEFAULT_TICKETS_TO_SPEND}; -use nym_credentials_interface::{CredentialSpendingData, TicketType}; +use nym_credentials_interface::TicketType; use nym_crypto::asymmetric::{ed25519, x25519}; use nym_lp::LpPacket; use nym_lp::codec::{OuterAeadKey, parse_lp_packet, serialize_lp_packet}; @@ -19,13 +21,15 @@ use nym_lp::message::ForwardPacketData; use nym_lp::peer::{LpLocalPeer, LpRemotePeer}; use nym_lp::state_machine::{LpAction, LpData, LpInput, LpStateMachine}; use nym_lp_transport::traits::LpTransport; -use nym_registration_common::{LpRegistrationRequest, WireguardConfiguration}; +use nym_registration_common::dvpn::LpDvpnRegistrationResponseMessageContent; +use nym_registration_common::{ + LpRegistrationRequest, LpRegistrationResponse, WireguardConfiguration, +}; use nym_wireguard_types::PeerPublicKey; use rand::{CryptoRng, RngCore}; use std::net::SocketAddr; use std::sync::Arc; -use std::time::{SystemTime, UNIX_EPOCH}; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tokio::net::TcpStream; /// LP (Lewes Protocol) registration client for direct gateway connections. @@ -95,6 +99,28 @@ where } } + fn state_machine(&self) -> Result<&LpStateMachine> { + self.state_machine.as_ref().ok_or_else(|| { + LpClientError::transport( + "State machine not available - has the handshake been completed?", + ) + }) + } + + fn state_machine_mut(&mut self) -> Result<&mut LpStateMachine> { + self.state_machine.as_mut().ok_or_else(|| { + LpClientError::transport( + "State machine not available - has the handshake been completed?", + ) + }) + } + + fn stream_mut(&mut self) -> Result<&mut S> { + self.stream + .as_mut() + .ok_or_else(|| LpClientError::transport("Cannot send: not connected")) + } + /// Creates a new LP registration client with default configuration. /// /// # Arguments @@ -194,27 +220,86 @@ where Ok(()) } + /// Attempt to send an Lp packet on the persistent stream + /// and attempt to immediately read a response. + /// + /// Both packets are going to be optionally encrypted/decrypted based on the availability of keys + /// within the internal `LpStateMachine` + /// + /// # Arguments + /// * `packet` - The LP packet to send + /// + /// # Errors + /// Returns an error if not connected or if send or receive fails. + async fn send_and_receive_packet(&mut self, packet: &LpPacket) -> Result { + self.try_send_packet(packet).await?; + self.try_receive_packet().await + } + + /// Attempt to send an Lp packet on the persistent stream + /// and attempt to immediately read a response + /// within the provided timeout. + /// + /// Both packets are going to be optionally encrypted/decrypted based on the availability of keys + /// within the internal `LpStateMachine` + /// + /// # Arguments + /// * `packet` - The LP packet to send + /// + /// # Errors + /// Returns an error if not connected, the timeout has been reached, or if send or receive fails. + async fn send_and_receive_packet_with_timeout( + &mut self, + packet: &LpPacket, + timeout: Duration, + ) -> Result { + tokio::time::timeout(timeout, self.send_and_receive_packet(packet)) + .await + .map_err(|_| LpClientError::ResponseReceiveTimeout { timeout })? + } + /// Sends an LP packet on the persistent stream. /// /// # Arguments /// * `packet` - The LP packet to send + /// + /// # Errors + /// Returns an error if not connected or if send fails. + async fn try_send_packet(&mut self, packet: &LpPacket) -> Result<()> { + let state_machine = self.state_machine()?; + let send_key = get_send_key(state_machine); + self.try_send_packet_with_key(packet, send_key.as_ref()) + .await + } + + /// Sends an LP packet (and optionally encrypted) on the persistent stream. + /// + /// # Arguments + /// * `packet` - The LP packet to send /// * `outer_key` - Optional outer AEAD key for encryption /// /// # Errors /// Returns an error if not connected or if send fails. - async fn send_packet( + async fn try_send_packet_with_key( &mut self, packet: &LpPacket, outer_key: Option<&OuterAeadKey>, ) -> Result<()> { - let stream = self - .stream - .as_mut() - .ok_or_else(|| LpClientError::Transport("Cannot send: not connected".to_string()))?; - + let stream = self.stream_mut()?; Self::send_packet_with_key(stream, packet, outer_key).await } + /// Receives an LP packet from the persistent stream. + /// + /// # Errors + /// Returns an error if not connected or if receive fails. + async fn try_receive_packet(&mut self) -> Result { + let state_machine = self.state_machine()?; + let recv_key = get_recv_key(state_machine); + + self.try_receive_packet_with_key(recv_key.as_ref()).await + } + /// Receives an LP packet from the persistent stream. /// /// # Arguments @@ -222,11 +307,11 @@ where /// /// # Errors /// Returns an error if not connected or if receive fails. - async fn receive_packet(&mut self, outer_key: Option<&OuterAeadKey>) -> Result { - let stream = self - .stream - .as_mut() - .ok_or_else(|| LpClientError::Transport("Cannot receive: not connected".to_string()))?; + async fn try_receive_packet_with_key( + &mut self, + outer_key: Option<&OuterAeadKey>, + ) -> Result { + let stream = self.stream_mut()?; Self::receive_packet_with_key(stream, outer_key).await } @@ -319,11 +404,7 @@ where .as_secs(); // Step 1: Generate ClientHelloData with fresh salt and both public keys - let client_hello_data = nym_lp::ClientHelloData::new_with_fresh_salt( - *self.lp_local_peer.x25519().public_key(), - *self.lp_local_peer.ed25519().public_key(), - timestamp, - ); + let client_hello_data = self.lp_local_peer.build_client_hello_data(timestamp); let salt = client_hello_data.salt; let receiver_index = client_hello_data.receiver_index; @@ -344,9 +425,10 @@ where ); // Send ClientHello (no outer key - before PSK) - self.send_packet(&client_hello_packet, None).await?; + self.try_send_packet_with_key(&client_hello_packet, None) + .await?; // Receive Ack (no outer key - before PSK) - let ack_response = self.receive_packet(None).await?; + let ack_response = self.try_receive_packet_with_key(None).await?; // Verify we received Ack match ack_response.message() { @@ -408,8 +490,9 @@ where send_key.is_some(), recv_key.is_some() ); - self.send_packet(&packet, send_key.as_ref()).await?; - let response = self.receive_packet(recv_key.as_ref()).await?; + self.try_send_packet_with_key(&packet, send_key.as_ref()) + .await?; + let response = self.try_receive_packet_with_key(recv_key.as_ref()).await?; tracing::trace!("Received handshake response"); // Process the received packet @@ -433,9 +516,10 @@ where .ok() .and_then(|s| s.outer_aead_key()); tracing::trace!("Sending final handshake packet"); - self.send_packet(&final_packet, send_key.as_ref()).await?; + self.try_send_packet_with_key(&final_packet, send_key.as_ref()) + .await?; let ack_response = - self.receive_packet(recv_key.as_ref()).await?; + self.try_receive_packet_with_key(recv_key.as_ref()).await?; // Validate Ack response match ack_response.message() { @@ -580,30 +664,10 @@ where serialize_lp_packet(packet, &mut packet_buf, outer_key) .map_err(|e| LpClientError::Transport(format!("Failed to serialize packet: {e}")))?; - // Send 4-byte length prefix (u32 big-endian) - let len = packet_buf.len() as u32; stream - .write_all(&len.to_be_bytes()) + .send_serialised_packet(&packet_buf) .await - .map_err(|e| LpClientError::Transport(format!("Failed to send packet length: {e}")))?; - - // Send the actual packet data - stream - .write_all(&packet_buf) - .await - .map_err(|e| LpClientError::Transport(format!("Failed to send packet data: {e}")))?; - - // Flush to ensure data is sent immediately - stream - .flush() - .await - .map_err(|e| LpClientError::Transport(format!("Failed to flush stream: {e}")))?; - - tracing::trace!( - "Sent LP packet ({} bytes + 4 byte header)", - packet_buf.len() - ); - Ok(()) + .map_err(|err| LpClientError::Transport(err.to_string())) } /// Receives an LP packet from a TCP stream with length-prefixed framing. @@ -623,39 +687,115 @@ where stream: &mut S, outer_key: Option<&OuterAeadKey>, ) -> Result { - // Read 4-byte length prefix (u32 big-endian) - let mut len_buf = [0u8; 4]; - stream - .read_exact(&mut len_buf) + let packet_buf = stream + .receive_raw_packet() .await - .map_err(|e| LpClientError::Transport(format!("Failed to read packet length: {e}")))?; - - let packet_len = u32::from_be_bytes(len_buf) as usize; - - // Sanity check to prevent huge allocations - const MAX_PACKET_SIZE: usize = 65536; // 64KB max - if packet_len > MAX_PACKET_SIZE { - return Err(LpClientError::Transport(format!( - "Packet size {packet_len} exceeds maximum {MAX_PACKET_SIZE}", - ))); - } - - // Read the actual packet data - let mut packet_buf = vec![0u8; packet_len]; - stream - .read_exact(&mut packet_buf) - .await - .map_err(|e| LpClientError::Transport(format!("Failed to read packet data: {e}")))?; + .map_err(|err| LpClientError::transport(err.to_string()))?; let packet = parse_lp_packet(&packet_buf, outer_key) .map_err(|e| LpClientError::Transport(format!("Failed to parse packet: {e}")))?; - tracing::trace!("Received LP packet ({} bytes + 4 byte header)", packet_len); Ok(packet) } - /// Sends registration request and receives response in a single operation. + /// This is an internal method only meant to be called by `Self::register_dvpn` if the gateway + /// responds with a credential request. This is expected in every initial interaction with a particular gateway. /// + /// This method will actually attempt to retrieve a valid credential from the `bandwidth_controller` + /// + /// # Arguments + /// * `gateway_identity` - Gateway's ed25519 identity for credential verification + /// * `bandwidth_controller` - Provider for bandwidth credentials + /// * `ticket_type` - Type of bandwidth ticket to use + /// + /// # Returns + /// * `Ok(WireguardConfiguration)` - Gateway configuration data on successful registration + /// + /// # Errors + /// Returns an error if: + /// - Credential acquisition fails + /// - Request serialization/encryption fails + /// - Network communication fails + /// - Gateway rejected the registration + /// - Response times out (see LpConfig::registration_timeout) + async fn finalise_dvpn_registration( + &mut self, + gateway_identity: ed25519::PublicKey, + bandwidth_controller: &dyn BandwidthTicketProvider, + ticket_type: TicketType, + ) -> Result { + tracing::debug!("Acquiring bandwidth credential for registration"); + + // 1. Get bandwidth credential from controller + let credential_spending = bandwidth_controller + .get_ecash_ticket(ticket_type, gateway_identity, DEFAULT_TICKETS_TO_SPEND) + .await + .map_err(|e| { + LpClientError::SendRegistrationRequest(format!( + "Failed to acquire bandwidth credential: {e}", + )) + })? + .data; + + // 2. Build registration request + + // for now we do NOT support upgrade mode (yeah... no.) + let credential = credential_spending + .try_into() + .map_err(|err| LpClientError::Other(format!("malformed stored credential: {err}")))?; + + let request = LpRegistrationRequest::new_finalise_dvpn(credential); + + tracing::trace!("Built dVPN registration finalisation request"); + + // 3. Serialize the request + let lp_data = request.to_lp_data()?; + + // 4. Encrypt and prepare packet via state machine + let state_machine = self.state_machine_mut()?; + let request_packet = prepare_send_packet(lp_data, state_machine)?; + + // 5. Send initial request and receive response on persistent connection with timeout + let response_packet = self + .send_and_receive_packet_with_timeout(&request_packet, self.config.registration_timeout) + .await?; + + // 6. Decrypt via state machine (re-borrow) + let state_machine = self.state_machine_mut()?; + let received_data = extract_forwarded_response(response_packet, state_machine)?; + + // 7. Extract decrypted data and deserialise the response + let response = LpRegistrationResponse::from_lp_data(received_data)?; + let Some(dvpn_response) = response.into_dvpn_response() else { + return Err(LpClientError::unexpected_response( + "did not get a dvpn registration response after sending initial request", + )); + }; + + // 8. check response to the initial request + match dvpn_response.content { + LpDvpnRegistrationResponseMessageContent::RegistrationFailure(res) => { + let reason = res.error; + // the registration has failed + tracing::warn!("Gateway rejected registration: {reason}"); + Err(LpClientError::RegistrationRejected { reason }) + } + LpDvpnRegistrationResponseMessageContent::CompletedRegistration(res) => { + // we have managed to complete the registration + tracing::info!( + "LP registration successful! Allocated bandwidth: {} bytes", + res.available_bandwidth + ); + Ok(res.config) + } + LpDvpnRegistrationResponseMessageContent::RequiresCredential(_) => { + Err(LpClientError::unexpected_response( + "received request for additional dvpn data after sending credential!", + )) + } + } + } + /// This is the primary registration method. It acquires a bandwidth credential, /// sends the registration request, and receives the response /// on the same underlying connection. @@ -670,7 +810,7 @@ where /// * `ticket_type` - Type of bandwidth ticket to use /// /// # Returns - /// * `Ok(GatewayData)` - Gateway configuration data on successful registration + /// * `Ok(WireguardConfiguration)` - Gateway configuration data on successful registration /// /// # Errors /// Returns an error if: @@ -680,7 +820,7 @@ where /// - Network communication fails /// - Gateway rejected the registration /// - Response times out (see LpConfig::registration_timeout) - pub async fn register( + pub async fn register_dvpn( &mut self, rng: &mut R, wg_keypair: &x25519::KeyPair, @@ -691,159 +831,64 @@ where where R: RngCore + CryptoRng, { - tracing::debug!("Acquiring bandwidth credential for registration"); - - // Get bandwidth credential from controller - let credential = bandwidth_controller - .get_ecash_ticket(ticket_type, *gateway_identity, DEFAULT_TICKETS_TO_SPEND) - .await - .map_err(|e| { - LpClientError::SendRegistrationRequest(format!( - "Failed to acquire bandwidth credential: {e}", - )) - })? - .data; - - self.register_with_credential(rng, wg_keypair, credential, ticket_type) - .await - } - - /// Sends registration request with a pre-generated credential. - /// - /// This is useful for testing with mock ecash credentials. - /// Uses the persistent TCP connection established during handshake. - /// - /// # Arguments - /// * `rng` - RNG instance for generating PSK - /// * `wg_keypair` - Client's WireGuard x25519 keypair - /// * `credential` - Pre-generated bandwidth credential - /// * `ticket_type` - Type of bandwidth ticket - /// - /// # Returns - /// * `Ok(GatewayData)` - Gateway configuration data on successful registration - /// - /// # Connection Lifecycle - /// The connection stays open after registration to support `send_forward_packet()`. - /// Callers should call `close()` when done with all operations. - /// - /// # Panics / Errors - /// Returns error if handshake not completed or if connection was closed. - pub async fn register_with_credential( - &mut self, - rng: &mut R, - wg_keypair: &x25519::KeyPair, - credential: CredentialSpendingData, - ticket_type: TicketType, - ) -> Result - where - R: RngCore + CryptoRng, - { - tracing::debug!("Sending registration request (persistent connection)"); - // 1. Build registration request - let wg_public_key = PeerPublicKey::new(wg_keypair.public_key().to_bytes().into()); - let request = LpRegistrationRequest::new_dvpn(rng, wg_public_key, credential, ticket_type); + let wg_public_key = PeerPublicKey::from(*wg_keypair.public_key()); + let request = LpRegistrationRequest::new_initial_dvpn(rng, wg_public_key, ticket_type); - tracing::trace!("Built registration request: {:?}", request); + tracing::trace!("Built dVPN registration request: {request:?}"); // 2. Serialize the request - let input = convert_registration_request(request)?; + let lp_data = request.to_lp_data()?; - // 3. Encrypt and prepare packet via state machine (scoped borrow) - let (request_packet, send_key, recv_key) = { - let state_machine = self.state_machine.as_mut().ok_or_else(|| { - LpClientError::transport("Cannot register: handshake not completed") - })?; + // 3. Encrypt and prepare packet via state machine + let state_machine = self.state_machine_mut()?; + let request_packet = prepare_send_packet(lp_data, state_machine)?; - let action = state_machine - .process_input(input) - .ok_or_else(|| LpClientError::transport("State machine returned no action"))? - .map_err(|e| { - LpClientError::SendRegistrationRequest(format!( - "Failed to encrypt registration request: {e}", - )) - })?; - - let request_packet = match action { - LpAction::SendPacket(packet) => packet, - other => { - return Err(LpClientError::Transport(format!( - "Unexpected action when sending registration data: {other:?}", - ))); - } - }; - - // Get outer keys from session - let send_key = state_machine - .session() - .ok() - .and_then(|s| s.outer_aead_key_for_sending()); - let recv_key = state_machine - .session() - .ok() - .and_then(|s| s.outer_aead_key()); - - (request_packet, send_key, recv_key) - }; // state_machine borrow ends here - - // 4. Send request and receive response on persistent connection with timeout - let response_packet = tokio::time::timeout(self.config.registration_timeout, async { - self.send_packet(&request_packet, send_key.as_ref()).await?; - self.receive_packet(recv_key.as_ref()).await - }) - .await - .map_err(|_| { - LpClientError::ReceiveRegistrationResponse(format!( - "Registration timeout after {:?}", - self.config.registration_timeout - )) - })??; - - tracing::trace!("Received registration response packet"); + // 4. Send initial request and receive response on persistent connection with timeout + let response_packet = self + .send_and_receive_packet_with_timeout(&request_packet, self.config.registration_timeout) + .await?; // 5. Decrypt via state machine (re-borrow) - let state_machine = self - .state_machine - .as_mut() - .ok_or_else(|| LpClientError::transport("State machine disappeared unexpectedly"))?; - let action = state_machine - .process_input(LpInput::ReceivePacket(response_packet)) - .ok_or_else(|| LpClientError::transport("State machine returned no action"))? - .map_err(|e| { - LpClientError::ReceiveRegistrationResponse(format!( - "Failed to decrypt registration response: {e}", - )) - })?; + let state_machine = self.state_machine_mut()?; + let received_data = extract_forwarded_response(response_packet, state_machine)?; // 6. Extract decrypted data and deserialise the response - let response = try_convert_registration_response(action)?; + let response = LpRegistrationResponse::from_lp_data(received_data)?; + let Some(dvpn_response) = response.into_dvpn_response() else { + return Err(LpClientError::unexpected_response( + "did not get a dvpn registration response after sending initial request", + )); + }; - tracing::debug!( - "Received registration response: success={}", - response.success, - ); + // 7. check response to the initial request + match dvpn_response.content { + LpDvpnRegistrationResponseMessageContent::RegistrationFailure(res) => { + let reason = res.error; + // the registration has failed + tracing::warn!("Gateway rejected registration: {reason}"); + Err(LpClientError::RegistrationRejected { reason }) + } + LpDvpnRegistrationResponseMessageContent::CompletedRegistration(res) => { + // we have already registered with this gateway before, the gateway has updated the psk and sent us the config + tracing::info!( + "LP registration successful! Allocated bandwidth: {} bytes", + res.available_bandwidth + ); + Ok(res.config) + } + LpDvpnRegistrationResponseMessageContent::RequiresCredential(_) => { + // we're registering for the first time with this gateway - we need to attach a credential - // 7. Validate and extract GatewayData - if !response.success { - let error_msg = response - .error - .unwrap_or_else(|| "Unknown error".to_string()); - tracing::warn!("Gateway rejected registration: {error_msg}"); - return Err(LpClientError::RegistrationRejected { reason: error_msg }); + // 8. retrieve credential from the controller + self.finalise_dvpn_registration( + *gateway_identity, + bandwidth_controller, + ticket_type, + ) + .await + } } - - let gateway_data = response.gateway_data.ok_or_else(|| { - LpClientError::ReceiveRegistrationResponse( - "Gateway response missing gateway_data despite success=true".to_string(), - ) - })?; - - tracing::info!( - "LP registration successful! Allocated bandwidth: {} bytes", - response.allocated_bandwidth - ); - - Ok(gateway_data) } /// Register with automatic retry on network failure. @@ -891,29 +936,16 @@ where { tracing::debug!("Starting resilient registration (max_retries={max_retries})",); - // Acquire credential ONCE before any attempts - let credential = bandwidth_controller - .get_ecash_ticket(ticket_type, *gateway_identity, DEFAULT_TICKETS_TO_SPEND) - .await - .map_err(|e| { - LpClientError::SendRegistrationRequest(format!( - "Failed to acquire bandwidth credential: {e}", - )) - })? - .data; - let mut last_error = None; for attempt in 0..=max_retries { + let attempt_display = attempt + 1; + if attempt > 0 { // Exponential backoff with jitter: 100ms, 200ms, 400ms, 800ms, 1600ms (capped) let base_delay_ms = 100u64 * (1 << attempt.min(4)); let jitter_ms = rand::random::() % (base_delay_ms / 4 + 1); let delay = std::time::Duration::from_millis(base_delay_ms + jitter_ms); - tracing::info!( - "Retrying registration (attempt {}) after {:?}", - attempt + 1, - delay - ); + tracing::info!("Retrying registration (attempt {attempt_display}) after {delay:?}"); tokio::time::sleep(delay).await; } @@ -925,24 +957,30 @@ where self.state_machine = None; if let Err(e) = self.perform_handshake().await { - tracing::warn!("Handshake failed on attempt {}: {e}", attempt + 1); + tracing::warn!("Handshake failed on attempt {attempt_display}: {e}"); last_error = Some(e); continue; } } match self - .register_with_credential(rng, wg_keypair, credential.clone(), ticket_type) + .register_dvpn( + rng, + wg_keypair, + gateway_identity, + bandwidth_controller, + ticket_type, + ) .await { Ok(data) => { if attempt > 0 { - tracing::info!("Registration succeeded on retry attempt {}", attempt + 1); + tracing::info!("Registration succeeded on retry attempt {attempt_display}"); } return Ok(data); } Err(e) => { - tracing::warn!("Registration attempt {} failed: {e}", attempt + 1); + tracing::warn!("Registration attempt {attempt_display} failed: {e}"); last_error = Some(e); } } @@ -962,9 +1000,7 @@ where /// Multiple forward packets can be sent on the same connection. /// /// # Arguments - /// * `target_identity` - Target gateway's Ed25519 identity (32 bytes) - /// * `target_address` - Target gateway's LP address (e.g., "1.1.1.1:41264") - /// * `inner_packet_bytes` - Complete inner LP packet bytes to forward to exit gateway + /// * `forward_data` - encapsulated target gateway's ed25519 identity, socket address and serialised inner LP packet /// /// # Returns /// * `Ok(Vec)` - Decrypted response bytes from the exit gateway @@ -989,28 +1025,21 @@ where /// inner_bytes.to_vec(), /// ).await?; /// ``` - pub async fn send_forward_packet( + pub async fn send_forward_packet_with_response( &mut self, - target_identity: [u8; 32], - target_address: String, - inner_packet_bytes: Vec, + forward_data: ForwardPacketData, ) -> Result> { + let target_address = forward_data.target_lp_address.clone(); + tracing::debug!( - "Sending ForwardPacket to {} ({} inner bytes, persistent connection)", - target_address, - inner_packet_bytes.len() + "Sending ForwardPacket to {target_address} ({} inner bytes, persistent connection)", + forward_data.inner_packet_bytes.len() ); - // 1. Construct ForwardPacketData - let forward_data = ForwardPacketData { - target_gateway_identity: target_identity, - target_lp_address: target_address.clone(), - inner_packet_bytes, - }; - // 2. Serialize the ForwardPacketData - let input = convert_forward_data(forward_data); + // 1. Serialize the ForwardPacketData + let input = convert_forward_data(forward_data)?; - // 3. Encrypt and prepare packet via state machine (scoped borrow) + // 2. Encrypt and prepare packet via state machine (scoped borrow) let (forward_packet, send_key, recv_key) = { let state_machine = self.state_machine.as_mut().ok_or_else(|| { LpClientError::transport("Cannot send forward packet: handshake not completed") @@ -1046,10 +1075,11 @@ where (forward_packet, send_key, recv_key) }; // state_machine borrow ends here - // 4. Send and receive on persistent connection with timeout + // 3. Send and receive on persistent connection with timeout let response_packet = tokio::time::timeout(self.config.forward_timeout, async { - self.send_packet(&forward_packet, send_key.as_ref()).await?; - self.receive_packet(recv_key.as_ref()).await + self.try_send_packet_with_key(&forward_packet, send_key.as_ref()) + .await?; + self.try_receive_packet_with_key(recv_key.as_ref()).await }) .await .map_err(|_| { @@ -1060,7 +1090,7 @@ where })??; tracing::trace!("Received response packet from entry gateway"); - // 5. Decrypt via state machine (re-borrow) + // 4. Decrypt via state machine (re-borrow) let state_machine = self .state_machine .as_mut() @@ -1072,12 +1102,11 @@ where LpClientError::Transport(format!("Failed to decrypt forward response: {e}")) })?; - // 7. Extract decrypted response data + // 5. Extract decrypted response data let response_data = try_convert_forward_response(action)?; tracing::debug!( - "Successfully received forward response from {} ({} bytes)", - target_address, + "Successfully received forward response from {target_address} ({} bytes)", response_data.len() ); diff --git a/nym-registration-client/src/lp_client/error.rs b/nym-registration-client/src/lp_client/error.rs index 2026e15991..32443f4e01 100644 --- a/nym-registration-client/src/lp_client/error.rs +++ b/nym-registration-client/src/lp_client/error.rs @@ -6,6 +6,7 @@ use nym_lp::LpError; use nym_registration_common::BincodeError; use std::io; +use std::time::Duration; use thiserror::Error; /// Errors that can occur during LP client operations. @@ -35,6 +36,10 @@ pub enum LpClientError { #[error("Gateway rejected registration: {reason}")] RegistrationRejected { reason: String }, + /// Failed to receive response within specified deadline + #[error("Failed to receive response within the set timeout: {timeout:?}")] + ResponseReceiveTimeout { timeout: Duration }, + /// LP transport error #[error("LP transport error: {0}")] Transport(String), @@ -55,6 +60,9 @@ pub enum LpClientError { #[error("Timeout waiting for {operation}")] Timeout { operation: String }, + #[error("received an unexpected response: {message}")] + UnexpectedResponse { message: String }, + /// Another uncategorized error #[error("{0}")] Other(String), @@ -64,6 +72,12 @@ impl LpClientError { pub fn transport(message: impl Into) -> LpClientError { LpClientError::Transport(message.into()) } + + pub fn unexpected_response(message: impl Into) -> LpClientError { + LpClientError::UnexpectedResponse { + message: message.into(), + } + } } pub type Result = std::result::Result; diff --git a/nym-registration-client/src/lp_client/helpers.rs b/nym-registration-client/src/lp_client/helpers.rs index 79b1b027b1..e972b21b45 100644 --- a/nym-registration-client/src/lp_client/helpers.rs +++ b/nym-registration-client/src/lp_client/helpers.rs @@ -10,59 +10,61 @@ use nym_registration_common::{ LpRegistrationRequest, LpRegistrationResponse, NymNodeLPInformation, }; -pub(crate) fn convert_registration_request( - request: LpRegistrationRequest, -) -> Result { - let request_bytes = request.serialise().map_err(|e| { - LpClientError::SendRegistrationRequest(format!("Failed to serialize request: {e}")) - })?; - - tracing::debug!( - "Sending registration request ({} bytes)", - request_bytes.len() - ); - - let data = LpData::new_registration(request_bytes); - Ok(LpInput::SendData(data)) +pub(crate) trait LpDataSendExt { + fn to_lp_data(&self) -> Result; } -pub(crate) fn try_convert_registration_response( - action: LpAction, -) -> Result { - let response_data = match action { - LpAction::DeliverData(data) => data, - other => { +pub(crate) trait LpDataDeliverExt: Sized { + fn from_lp_data(data: LpData) -> Result; +} + +impl LpDataSendExt for LpRegistrationRequest { + fn to_lp_data(&self) -> Result { + let request_bytes = self.serialise().map_err(|e| { + LpClientError::SendRegistrationRequest(format!("Failed to serialize request: {e}")) + })?; + + tracing::debug!( + "Sending registration request ({} bytes)", + request_bytes.len() + ); + + Ok(LpData::new_registration(request_bytes)) + } +} + +impl LpDataDeliverExt for LpRegistrationResponse { + fn from_lp_data(data: LpData) -> Result { + if data.kind != LpDataKind::Registration { return Err(LpClientError::Transport(format!( - "Unexpected action when receiving registration response: {other:?}" + "did not receive a valid registration response. got {:?} instead", + data.kind ))); } - }; - if response_data.kind != LpDataKind::Registration { - return Err(LpClientError::Transport(format!( - "did not receive a valid registration response. got {:?} instead", - response_data.kind - ))); - } - - let response = - LpRegistrationResponse::try_deserialise(&response_data.content).map_err(|e| { + let response = LpRegistrationResponse::try_deserialise(&data.content).map_err(|e| { LpClientError::Transport(format!("Failed to deserialize registration response: {e}",)) })?; - Ok(response) + Ok(response) + } } -pub(crate) fn convert_forward_data(request: ForwardPacketData) -> LpInput { - let request_bytes = request.to_bytes(); +impl LpDataSendExt for ForwardPacketData { + fn to_lp_data(&self) -> Result { + let request_bytes = self.to_bytes(); - tracing::trace!( - "Sending forward packet data request ({} bytes)", - request_bytes.len() - ); + tracing::trace!( + "Sending forward packet data request ({} bytes)", + request_bytes.len() + ); - let data = LpData::new_forward(request_bytes); - LpInput::SendData(data) + Ok(LpData::new_forward(request_bytes)) + } +} + +pub(crate) fn convert_forward_data(request: ForwardPacketData) -> Result { + Ok(LpInput::SendData(request.to_lp_data()?)) } pub(crate) fn try_convert_forward_response(action: LpAction) -> Result, LpClientError> { @@ -78,7 +80,7 @@ pub(crate) fn try_convert_forward_response(action: LpAction) -> Result, if response_data.kind != LpDataKind::Forward { return Err(LpClientError::Transport(format!( - "did not receive a valid foreward response. got {:?} instead", + "did not receive a valid forward response. got {:?} instead", response_data.kind ))); } diff --git a/nym-registration-client/src/lp_client/mod.rs b/nym-registration-client/src/lp_client/mod.rs index b0dba1db25..d533303d29 100644 --- a/nym-registration-client/src/lp_client/mod.rs +++ b/nym-registration-client/src/lp_client/mod.rs @@ -36,6 +36,7 @@ mod config; pub(crate) mod error; pub(crate) mod helpers; mod nested_session; +mod state_machine_helpers; pub use client::LpRegistrationClient; pub use config::LpConfig; diff --git a/nym-registration-client/src/lp_client/nested_session.rs b/nym-registration-client/src/lp_client/nested_session.rs index fd4e486378..ed10f09880 100644 --- a/nym-registration-client/src/lp_client/nested_session.rs +++ b/nym-registration-client/src/lp_client/nested_session.rs @@ -20,17 +20,24 @@ use super::client::LpRegistrationClient; use super::error::{LpClientError, Result}; -use crate::lp_client::helpers::{convert_registration_request, try_convert_registration_response}; -use bytes::BytesMut; -use nym_bandwidth_controller::BandwidthTicketProvider; +use crate::lp_client::helpers::{LpDataDeliverExt, LpDataSendExt}; +use crate::lp_client::state_machine_helpers::{ + extract_forwarded_response, get_recv_key, get_send_key, prepare_serialised_send_packet, + serialize_packet, +}; +use nym_bandwidth_controller::{BandwidthTicketProvider, DEFAULT_TICKETS_TO_SPEND}; use nym_credentials_interface::TicketType; use nym_crypto::asymmetric::{ed25519, x25519}; -use nym_lp::codec::{OuterAeadKey, parse_lp_packet, serialize_lp_packet}; +use nym_lp::codec::{OuterAeadKey, parse_lp_packet}; +use nym_lp::message::ForwardPacketData; use nym_lp::peer::{LpLocalPeer, LpRemotePeer}; -use nym_lp::state_machine::{LpAction, LpInput, LpStateMachine}; +use nym_lp::state_machine::{LpAction, LpData, LpInput, LpStateMachine}; use nym_lp::{LpMessage, LpPacket}; use nym_lp_transport::traits::LpTransport; -use nym_registration_common::{LpRegistrationRequest, WireguardConfiguration}; +use nym_registration_common::dvpn::LpDvpnRegistrationResponseMessageContent; +use nym_registration_common::{ + LpRegistrationRequest, LpRegistrationResponse, WireguardConfiguration, +}; use nym_wireguard_types::PeerPublicKey; use rand::{CryptoRng, RngCore}; use std::sync::Arc; @@ -93,6 +100,51 @@ impl NestedLpSession { } } + fn state_machine(&self) -> Result<&LpStateMachine> { + self.state_machine.as_ref().ok_or_else(|| { + LpClientError::transport( + "State machine not available - has the handshake been completed?", + ) + }) + } + + fn state_machine_mut(&mut self) -> Result<&mut LpStateMachine> { + self.state_machine.as_mut().ok_or_else(|| { + LpClientError::transport( + "State machine not available - has the handshake been completed?", + ) + }) + } + + /// Attempt to parse received bytes into an LpPacket + fn parse_received_lp_packet(&self, response_bytes: Vec) -> Result { + let state_machine = self.state_machine()?; + let outer_key = get_recv_key(state_machine); + Self::parse_packet(&response_bytes, outer_key.as_ref()) + } + + /// Attempt to wrap the provided `LpData` into a `ForwardPacketData` + /// using the inner state machine. + fn prepare_forward_packet(&mut self, data: LpData) -> Result { + let target_gateway_identity = self.gateway_lp_peer.ed25519(); + let target_lp_address = self.exit_address.clone(); + + let state_machine = self.state_machine_mut()?; + let inner_packet_bytes = prepare_serialised_send_packet(data, state_machine)?; + Ok(ForwardPacketData { + target_gateway_identity: target_gateway_identity.to_bytes(), + target_lp_address, + inner_packet_bytes, + }) + } + + /// Attempt to recover received `LpData` from the received `LpPacket` + /// using the inner state machine. + fn extract_forwarded_response(&mut self, response_packet: LpPacket) -> Result { + let state_machine = self.state_machine_mut()?; + extract_forwarded_response(response_packet, state_machine) + } + /// Performs the LP handshake with the exit gateway by forwarding packets /// through the entry gateway. /// @@ -129,11 +181,7 @@ impl NestedLpSession { .as_secs(); // Step 1: Generate ClientHello for exit gateway - let client_hello_data = nym_lp::ClientHelloData::new_with_fresh_salt( - *self.lp_local_peer.x25519().public_key(), - *self.lp_local_peer.ed25519().public_key(), - timestamp, - ); + let client_hello_data = self.lp_local_peer.build_client_hello_data(timestamp); let salt = client_hello_data.salt; let receiver_index = client_hello_data.receiver_index; @@ -153,13 +201,15 @@ impl NestedLpSession { ); // Serialize and forward ClientHello (no state machine yet, no outer key) - let client_hello_bytes = Self::serialize_packet(&client_hello_packet, None)?; + let client_hello_bytes = serialize_packet(&client_hello_packet, None)?; + let forward_packet_data = ForwardPacketData::new( + self.gateway_lp_peer.ed25519(), + self.exit_address.clone(), + client_hello_bytes, + ); + let response_bytes = outer_client - .send_forward_packet( - self.gateway_lp_peer.ed25519().to_bytes(), - self.exit_address.clone(), - client_hello_bytes, - ) + .send_forward_packet_with_response(forward_packet_data) .await?; // Parse and validate Ack response (cleartext, no outer key before PSK derivation) @@ -273,132 +323,109 @@ impl NestedLpSession { Ok(()) } - /// Performs handshake and registration with the exit gateway via forwarding, - /// using a pre-made credential. + /// This is an internal method only meant to be called by `Self::handshake_and_register_dvpn` if the gateway + /// responds with a credential request. This is expected in every initial interaction with a particular gateway. /// - /// This variant is useful for mock ecash testing where the credential is provided - /// directly instead of being acquired from a bandwidth controller. + /// This method will actually attempt to retrieve a valid credential from the `bandwidth_controller` /// /// # Arguments /// * `outer_client` - Connected LP client with established outer session to entry gateway - /// * `wg_keypair` - Client's WireGuard x25519 keypair - /// * `credential` - Pre-made bandwidth credential (e.g., mock ecash) + /// * `gateway_identity` - Gateway's ed25519 identity for credential verification + /// * `bandwidth_controller` - Provider for bandwidth credentials /// * `ticket_type` - Type of bandwidth ticket to use - /// * `client_ip` - Client IP address for registration metadata /// /// # Returns - /// * `Ok(GatewayData)` - Exit gateway configuration data on successful registration - pub async fn handshake_and_register_with_credential( + /// * `Ok(WireguardConfiguration)` - Gateway configuration data on successful registration + /// + /// # Errors + /// Returns an error if: + /// - Credential acquisition fails + /// - Request serialization/encryption fails + /// - Forwarding through entry gateway fails + /// - Network communication fails + /// - Gateway rejected the registration + /// - Response times out (see LpConfig::registration_timeout) + async fn finalise_dvpn_registration( &mut self, outer_client: &mut LpRegistrationClient, - rng: &mut R, - wg_keypair: &x25519::KeyPair, - credential: nym_credentials_interface::CredentialSpendingData, + gateway_identity: ed25519::PublicKey, + bandwidth_controller: &dyn BandwidthTicketProvider, ticket_type: TicketType, ) -> Result where S: LpTransport + Unpin, - R: RngCore + CryptoRng, { - // Step 1: Perform handshake with exit gateway via forwarding - self.perform_handshake(outer_client).await?; + tracing::debug!("Acquiring bandwidth credential for registration"); - // Step 2: Get the state machine (must exist after successful handshake) - let state_machine = self.state_machine.as_mut().ok_or_else(|| { - LpClientError::Transport("State machine missing after handshake".to_string()) - })?; - - tracing::debug!( - "Building registration request for exit gateway (with pre-made credential)" - ); - - // Step 3: Build registration request (credential already provided) - let wg_public_key = PeerPublicKey::new(wg_keypair.public_key().to_bytes().into()); - let request = LpRegistrationRequest::new_dvpn(rng, wg_public_key, credential, ticket_type); - - tracing::trace!("Built registration request: {:?}", request); - - // Step 4: Serialize the request - let input = convert_registration_request(request)?; - tracing::debug!("Sending registration request to exit gateway via forwarding",); - - // Step 5: Encrypt and prepare packet via state machine - let action = state_machine - .process_input(input) - .ok_or_else(|| { - LpClientError::Transport("State machine returned no action".to_string()) - })? + // Step 1: Get bandwidth credential from controller + let credential_spending = bandwidth_controller + .get_ecash_ticket(ticket_type, gateway_identity, DEFAULT_TICKETS_TO_SPEND) + .await .map_err(|e| { - LpClientError::Transport(format!("Failed to encrypt registration request: {}", e)) - })?; + LpClientError::SendRegistrationRequest(format!( + "Failed to acquire bandwidth credential: {e}", + )) + })? + .data; - // Step 6: Send the encrypted packet via forwarding - let outer_key = Self::get_send_key(state_machine); - let response_bytes = match action { - LpAction::SendPacket(packet) => { - let packet_bytes = Self::serialize_packet(&packet, outer_key.as_ref())?; - outer_client - .send_forward_packet( - self.gateway_lp_peer.ed25519().to_bytes(), - self.exit_address.clone(), - packet_bytes, - ) - .await? - } - other => { - return Err(LpClientError::Transport(format!( - "Unexpected action when sending registration data: {:?}", - other - ))); - } + // Step 2: Build registration request + + // for now we do NOT support upgrade mode (yeah... no.) + let credential = credential_spending + .try_into() + .map_err(|err| LpClientError::Other(format!("malformed stored credential: {err}")))?; + + let request = LpRegistrationRequest::new_finalise_dvpn(credential); + + tracing::trace!("Built dVPN registration finalisation request"); + + // Step 3: Serialize the request + let send_data = request.to_lp_data()?; + + // Step 4: Encrypt and prepare packet via state machine + let forward_packet = self.prepare_forward_packet(send_data)?; + + // Step 5: Send the encrypted packet via forwarding + let response_bytes = outer_client + .send_forward_packet_with_response(forward_packet) + .await?; + + // Step 6: Parse response bytes to LP packet + let response_packet = self.parse_received_lp_packet(response_bytes)?; + + // Step 7: Decrypt via state machine + let response_data = self.extract_forwarded_response(response_packet)?; + + // Step 8: Extract decrypted data and deserialise the response + let response = LpRegistrationResponse::from_lp_data(response_data)?; + let Some(dvpn_response) = response.into_dvpn_response() else { + return Err(LpClientError::unexpected_response( + "did not get a dvpn registration response after sending initial request", + )); }; - tracing::trace!("Received registration response from exit gateway"); - - // Step 7: Parse response bytes to LP packet - let outer_key = Self::get_recv_key(state_machine); - let response_packet = Self::parse_packet(&response_bytes, outer_key.as_ref())?; - - // Step 8: Decrypt via state machine - let action = state_machine - .process_input(LpInput::ReceivePacket(response_packet)) - .ok_or_else(|| { - LpClientError::Transport("State machine returned no action".to_string()) - })? - .map_err(|e| { - LpClientError::Transport(format!("Failed to decrypt registration response: {}", e)) - })?; - - // Step 9: Extract decrypted data and deserialise the response - let response = try_convert_registration_response(action)?; - - tracing::debug!( - "Received registration response from exit: success={}", - response.success, - ); - - // Step 10: Validate and extract GatewayData - if !response.success { - let error_msg = response - .error - .unwrap_or_else(|| "Unknown error".to_string()); - tracing::warn!("Exit gateway rejected registration: {error_msg}"); - return Err(LpClientError::RegistrationRejected { reason: error_msg }); + // Step 9: check response to the initial request + match dvpn_response.content { + LpDvpnRegistrationResponseMessageContent::RegistrationFailure(res) => { + let reason = res.error; + // the registration has failed + tracing::warn!("Gateway rejected registration: {reason}"); + Err(LpClientError::RegistrationRejected { reason }) + } + LpDvpnRegistrationResponseMessageContent::CompletedRegistration(res) => { + // we have managed to complete the registration + tracing::info!( + "LP registration successful! Allocated bandwidth: {} bytes", + res.available_bandwidth + ); + Ok(res.config) + } + LpDvpnRegistrationResponseMessageContent::RequiresCredential(_) => { + Err(LpClientError::unexpected_response( + "received request for additional dvpn data after sending credential!", + )) + } } - - // Extract gateway_data - let gateway_data = response.gateway_data.ok_or_else(|| { - LpClientError::Transport( - "Gateway response missing gateway_data despite success=true".to_string(), - ) - })?; - - tracing::info!( - "Exit gateway registration successful! Allocated bandwidth: {} bytes", - response.allocated_bandwidth - ); - - Ok(gateway_data) } /// Performs handshake and registration with the exit gateway via forwarding. @@ -428,7 +455,7 @@ impl NestedLpSession { /// - Forwarding through entry gateway fails /// - Response decryption/deserialization fails /// - Gateway rejects the registration - pub async fn handshake_and_register( + pub async fn handshake_and_register_dvpn( &mut self, outer_client: &mut LpRegistrationClient, rng: &mut R, @@ -444,113 +471,68 @@ impl NestedLpSession { // Step 1: Perform handshake with exit gateway via forwarding self.perform_handshake(outer_client).await?; - // Step 2: Get the state machine (must exist after successful handshake) - let state_machine = self.state_machine.as_mut().ok_or_else(|| { - LpClientError::Transport("State machine missing after handshake".to_string()) - })?; - tracing::debug!("Building registration request for exit gateway"); - // Step 3: Acquire bandwidth credential - let credential = bandwidth_controller - .get_ecash_ticket( - ticket_type, - *gateway_identity, - nym_bandwidth_controller::DEFAULT_TICKETS_TO_SPEND, - ) - .await - .map_err(|e| { - LpClientError::Transport(format!("Failed to acquire bandwidth credential: {}", e)) - })? - .data; + // Step 2: Build registration request + let wg_public_key = PeerPublicKey::from(*wg_keypair.public_key()); + let request = LpRegistrationRequest::new_initial_dvpn(rng, wg_public_key, ticket_type); - // Step 4: Build registration request - let wg_public_key = PeerPublicKey::new(wg_keypair.public_key().to_bytes().into()); - let request = LpRegistrationRequest::new_dvpn(rng, wg_public_key, credential, ticket_type); + // Step 3: Serialize the request + let send_data = request.to_lp_data()?; - tracing::trace!("Built registration request: {:?}", request); + // Step 4: Encrypt and prepare packet via state machine + let forward_packet = self.prepare_forward_packet(send_data)?; - // Step 5: Serialize the request - let input = convert_registration_request(request)?; - tracing::debug!("Sending registration request to exit gateway via forwarding"); - - // Step 6: Encrypt and prepare packet via state machine - let action = state_machine - .process_input(input) - .ok_or_else(|| { - LpClientError::Transport("State machine returned no action".to_string()) - })? - .map_err(|e| { - LpClientError::Transport(format!("Failed to encrypt registration request: {}", e)) - })?; - - // Step 7: Send the encrypted packet via forwarding - let outer_key = Self::get_send_key(state_machine); - let response_bytes = match action { - LpAction::SendPacket(packet) => { - let packet_bytes = Self::serialize_packet(&packet, outer_key.as_ref())?; - outer_client - .send_forward_packet( - self.gateway_lp_peer.ed25519().to_bytes(), - self.exit_address.clone(), - packet_bytes, - ) - .await? - } - other => { - return Err(LpClientError::Transport(format!( - "Unexpected action when sending registration data: {:?}", - other - ))); - } - }; + // Step 5: Send the encrypted packet via forwarding + let response_bytes = outer_client + .send_forward_packet_with_response(forward_packet) + .await?; tracing::trace!("Received registration response from exit gateway"); - // Step 8: Parse response bytes to LP packet - let outer_key = Self::get_recv_key(state_machine); - let response_packet = Self::parse_packet(&response_bytes, outer_key.as_ref())?; + // Step 6: Parse response bytes to LP packet + let response_packet = self.parse_received_lp_packet(response_bytes)?; - // Step 9: Decrypt via state machine - let action = state_machine - .process_input(LpInput::ReceivePacket(response_packet)) - .ok_or_else(|| { - LpClientError::Transport("State machine returned no action".to_string()) - })? - .map_err(|e| { - LpClientError::Transport(format!("Failed to decrypt registration response: {}", e)) - })?; + // Step 7: Decrypt via state machine + let response_data = self.extract_forwarded_response(response_packet)?; - // Step 10: Extract decrypted data and deserialise the response - let response = try_convert_registration_response(action)?; + // Step 8: Extract decrypted data and deserialise the response + let response = LpRegistrationResponse::from_lp_data(response_data)?; + let Some(dvpn_response) = response.into_dvpn_response() else { + return Err(LpClientError::unexpected_response( + "did not get a dvpn registration response after sending initial request", + )); + }; - tracing::debug!( - "Received registration response from exit: success={}", - response.success, - ); + // Step 9: check response to the initial request + match dvpn_response.content { + LpDvpnRegistrationResponseMessageContent::RegistrationFailure(res) => { + let reason = res.error; + // the registration has failed + tracing::warn!("Gateway rejected registration: {reason}"); + Err(LpClientError::RegistrationRejected { reason }) + } + LpDvpnRegistrationResponseMessageContent::CompletedRegistration(res) => { + // we have already registered with this gateway before, the gateway has updated the psk and sent us the config + tracing::info!( + "LP registration successful! Allocated bandwidth: {} bytes", + res.available_bandwidth + ); + Ok(res.config) + } + LpDvpnRegistrationResponseMessageContent::RequiresCredential(_) => { + // we're registering for the first time with this gateway - we need to attach a credential - // Step 11: Validate and extract GatewayData - if !response.success { - let error_msg = response - .error - .unwrap_or_else(|| "Unknown error".to_string()); - tracing::warn!("Exit gateway rejected registration: {}", error_msg); - return Err(LpClientError::RegistrationRejected { reason: error_msg }); + // Step 10: retrieve credential from the controller + self.finalise_dvpn_registration( + outer_client, + *gateway_identity, + bandwidth_controller, + ticket_type, + ) + .await + } } - - // Extract gateway_data - let gateway_data = response.gateway_data.ok_or_else(|| { - LpClientError::Transport( - "Gateway response missing gateway_data despite success=true".to_string(), - ) - })?; - - tracing::info!( - "Exit gateway registration successful! Allocated bandwidth: {} bytes", - response.allocated_bandwidth - ); - - Ok(gateway_data) } /// Performs handshake and registration with the exit gateway via forwarding, @@ -582,10 +564,10 @@ impl NestedLpSession { /// # Errors /// Returns an error if all retry attempts fail. #[allow(clippy::too_many_arguments)] - pub async fn handshake_and_register_with_retry( + pub async fn handshake_and_register_dvpn_with_retry( &mut self, - rng: &mut R, outer_client: &mut LpRegistrationClient, + rng: &mut R, wg_keypair: &x25519::KeyPair, gateway_identity: &ed25519::PublicKey, bandwidth_controller: &dyn BandwidthTicketProvider, @@ -601,19 +583,6 @@ impl NestedLpSession { max_retries ); - // Acquire credential ONCE before any attempts - let credential = bandwidth_controller - .get_ecash_ticket( - ticket_type, - *gateway_identity, - nym_bandwidth_controller::DEFAULT_TICKETS_TO_SPEND, - ) - .await - .map_err(|e| { - LpClientError::Transport(format!("Failed to acquire bandwidth credential: {}", e)) - })? - .data; - let mut last_error = None; for attempt in 0..=max_retries { if attempt > 0 { @@ -640,11 +609,12 @@ impl NestedLpSession { } match self - .handshake_and_register_with_credential( + .handshake_and_register_dvpn( outer_client, rng, wg_keypair, - credential.clone(), + gateway_identity, + bandwidth_controller, ticket_type, ) .await @@ -686,58 +656,20 @@ impl NestedLpSession { where S: LpTransport + Unpin, { - let send_key = Self::get_send_key(state_machine); - let packet_bytes = Self::serialize_packet(packet, send_key.as_ref())?; + let send_key = get_send_key(state_machine); + let packet_bytes = serialize_packet(packet, send_key.as_ref())?; + let forward_data = ForwardPacketData::new( + self.gateway_lp_peer.ed25519(), + self.exit_address.clone(), + packet_bytes, + ); let response_bytes = outer_client - .send_forward_packet( - self.gateway_lp_peer.ed25519().to_bytes(), - self.exit_address.clone(), - packet_bytes, - ) + .send_forward_packet_with_response(forward_data) .await?; - let recv_key = Self::get_recv_key(state_machine); + let recv_key = get_recv_key(state_machine); Self::parse_packet(&response_bytes, recv_key.as_ref()) } - /// Gets the outer AEAD key for sending (encryption) from the state machine. - /// - /// Returns `None` during early handshake before PSK derivation. - fn get_send_key(state_machine: &LpStateMachine) -> Option { - state_machine - .session() - .ok() - .and_then(|s| s.outer_aead_key_for_sending()) - } - - /// Gets the outer AEAD key for receiving (decryption) from the state machine. - /// - /// Returns `None` during early handshake before PSK derivation. - fn get_recv_key(state_machine: &LpStateMachine) -> Option { - state_machine - .session() - .ok() - .and_then(|s| s.outer_aead_key()) - } - - /// Serializes an LP packet to bytes. - /// - /// # Arguments - /// * `packet` - The LP packet to serialize - /// - /// # Returns - /// * `Ok(Vec)` - Serialized packet bytes - /// - /// # Errors - /// Returns an error if serialization fails - fn serialize_packet(packet: &LpPacket, outer_key: Option<&OuterAeadKey>) -> Result> { - let mut buf = BytesMut::new(); - // Use outer AEAD key when available (after PSK derivation) - serialize_lp_packet(packet, &mut buf, outer_key).map_err(|e| { - LpClientError::Transport(format!("Failed to serialize LP packet: {}", e)) - })?; - Ok(buf.to_vec()) - } - /// Parses an LP packet from bytes. /// /// # Arguments @@ -751,6 +683,6 @@ impl NestedLpSession { fn parse_packet(bytes: &[u8], outer_key: Option<&OuterAeadKey>) -> Result { // Use outer AEAD key when available (after PSK derivation) parse_lp_packet(bytes, outer_key) - .map_err(|e| LpClientError::Transport(format!("Failed to parse LP packet: {}", e))) + .map_err(|e| LpClientError::Transport(format!("Failed to parse LP packet: {e}"))) } } diff --git a/nym-registration-client/src/lp_client/state_machine_helpers.rs b/nym-registration-client/src/lp_client/state_machine_helpers.rs new file mode 100644 index 0000000000..47da7c1255 --- /dev/null +++ b/nym-registration-client/src/lp_client/state_machine_helpers.rs @@ -0,0 +1,106 @@ +// Copyright 2026 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::LpClientError; +use bytes::BytesMut; +use nym_lp::codec::{OuterAeadKey, serialize_lp_packet}; +use nym_lp::state_machine::{LpAction, LpData, LpInput}; +use nym_lp::{LpPacket, LpStateMachine}; + +/// Gets the outer AEAD key for sending (encryption) from the state machine. +/// +/// Returns `None` during early handshake before PSK derivation. +pub(crate) fn get_send_key(state_machine: &LpStateMachine) -> Option { + state_machine + .session() + .ok() + .and_then(|s| s.outer_aead_key_for_sending()) +} + +/// Gets the outer AEAD key for receiving (decryption) from the state machine. +/// +/// Returns `None` during early handshake before PSK derivation. +pub(crate) fn get_recv_key(state_machine: &LpStateMachine) -> Option { + state_machine + .session() + .ok() + .and_then(|s| s.outer_aead_key()) +} + +/// Serializes an LP packet to bytes. +/// +/// # Arguments +/// * `packet` - The LP packet to serialize +/// +/// # Returns +/// * `Ok(Vec)` - Serialized packet bytes +/// +/// # Errors +/// Returns an error if serialization fails +pub(crate) fn serialize_packet( + packet: &LpPacket, + outer_key: Option<&OuterAeadKey>, +) -> Result, LpClientError> { + let mut buf = BytesMut::new(); + // Use outer AEAD key when available (after PSK derivation) + serialize_lp_packet(packet, &mut buf, outer_key) + .map_err(|e| LpClientError::Transport(format!("Failed to serialize LP packet: {}", e)))?; + Ok(buf.to_vec()) +} + +/// Attempt to prepare the provided data for sending by wrapping it in appropriate `LpAction`, +/// and attempting to extract `LpPacket` from the provided srtate machine. +pub(crate) fn prepare_send_packet( + data: LpData, + state_machine: &mut LpStateMachine, +) -> Result { + let action = state_machine + .process_input(LpInput::SendData(data)) + .ok_or_else(|| LpClientError::transport("State machine returned no action"))? + .map_err(|e| { + LpClientError::SendRegistrationRequest(format!( + "Failed to encrypt registration request: {e}", + )) + })?; + + match action { + LpAction::SendPacket(packet) => Ok(packet), + other => Err(LpClientError::Transport(format!( + "Unexpected action when trying to send packet data: {other:?}", + ))), + } +} + +/// Attempt to prepare the provided data for sending by wrapping it in appropriate `LpAction`, +/// serialising and finally encrypting (if appropriate key is available) the resultant `LpPacket` +/// It uses the provided state machine. +pub(crate) fn prepare_serialised_send_packet( + data: LpData, + state_machine: &mut LpStateMachine, +) -> Result, LpClientError> { + let packet = prepare_send_packet(data, state_machine)?; + + let send_key = get_send_key(state_machine); + serialize_packet(&packet, send_key.as_ref()) +} + +/// Attempt to recover received `LpData` from the received `LpPacket` +/// using the provided state machine. +pub(crate) fn extract_forwarded_response( + response_packet: LpPacket, + state_machine: &mut LpStateMachine, +) -> Result { + let action = state_machine + .process_input(LpInput::ReceivePacket(response_packet)) + .ok_or_else(|| LpClientError::Transport("State machine returned no action".to_string()))? + .map_err(|e| { + LpClientError::Transport(format!("Failed to decrypt received response: {e}")) + })?; + + match action { + LpAction::DeliverData(data) => Ok(data), + other => Err(LpClientError::Transport(format!( + "Unexpected action when receiving response: {other:?}" + ))), + } +}