GRIN/nym
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Lp/ip pool fixes (#6412)

Browse Source 
* squashing Lp/ip pool fixes#6412

removed unused imports

gateway probe fixes

PSK injection + test fixes

cleanup minus PSK injection

combine with lp reg

moved authenticator peer registration to centralised location

bugfix: ensure IpPool never allocates gateway ip

ip pool allocation tests

* review fixes

* test fixes
This commit is contained in:
Jędrzej Stuczyński
2026-02-05 14:47:37 +00:00
committed by GitHub
parent b19e82d4f7
commit a151a03181
44 changed files with 2121 additions and 1678 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+5
View File
@@ -6354,6 +6354,7 @@ dependencies = [
name = "nym-gateway"
version = "1.1.36"
dependencies = [
"anyhow",
"async-trait",
"bincode",
"bip39",
@@ -6392,6 +6393,7 @@ dependencies = [
"nym-sphinx",
"nym-statistics-common",
"nym-task",
"nym-test-utils",
"nym-topology",
"nym-upgrade-mode-check",
"nym-validator-client",
@@ -8365,11 +8367,13 @@ dependencies = [
name = "nym-wireguard"
version = "1.20.1"
dependencies = [
"anyhow",
"base64 0.22.1",
"defguard_wireguard_rs",
"futures",
"ip_network",
"ipnetwork",
"mock_instant",
"nym-authenticator-requests",
"nym-credential-verification",
"nym-credentials-interface",
@@ -8380,6 +8384,7 @@ dependencies = [
"nym-network-defaults",
"nym-node-metrics",
"nym-task",
"nym-test-utils",
"nym-wireguard-types",
"rand 0.8.5",
"thiserror 2.0.17",
Cargo.toml
+1
View File
@@ -303,6 +303,7 @@ ledger-transport = "0.10.0"
ledger-transport-hid = "0.10.0"
log = "0.4"
mime = "0.3.17"
mock_instant = "0.6.0"
moka = { version = "0.12", features = ["future"] }
nix = "0.30.1"
notify = "5.1.0"
common/authenticator-requests/src/lib.rs
+1
View File
@@ -18,6 +18,7 @@ mod util;
mod version;
pub use error::Error;
pub use util::{authenticator_ipv4_to_ipv6, authenticator_ipv6_to_ipv4};
pub use v6 as latest;
pub use version::AuthenticatorVersion;
common/authenticator-requests/src/response.rs
+12
View File
@@ -7,6 +7,7 @@ use crate::traits::{
TopUpBandwidthResponse, UpgradeModeStatus,
};
use crate::{v2, v3, v4, v5, v6};
use nym_sphinx::addressing::Recipient;
#[derive(Debug)]
pub enum AuthenticatorResponse {
@@ -17,6 +18,17 @@ pub enum AuthenticatorResponse {
UpgradeMode(Box<dyn UpgradeModeStatus + Send + Sync + 'static>),
}
pub struct SerialisedResponse {
pub bytes: Vec<u8>,
pub reply_to: Option<Recipient>,
}
impl SerialisedResponse {
pub fn new(bytes: Vec<u8>, reply_to: Option<Recipient>) -> Self {
Self { bytes, reply_to }
}
}
impl UpgradeModeStatus for AuthenticatorResponse {
fn upgrade_mode_status(&self) -> CurrentUpgradeModeStatus {
match self {
common/authenticator-requests/src/util.rs
+32
View File
@@ -1,6 +1,38 @@
// Copyright 2024 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use nym_network_defaults::{WG_TUN_DEVICE_IP_ADDRESS_V4, WG_TUN_DEVICE_IP_ADDRESS_V6};
use std::net::{Ipv4Addr, Ipv6Addr};
pub fn authenticator_ipv6_to_ipv4(addr: Ipv6Addr) -> Ipv4Addr {
let before_last_byte = addr.octets()[14];
let last_byte = addr.octets()[15];
Ipv4Addr::new(
WG_TUN_DEVICE_IP_ADDRESS_V4.octets()[0],
WG_TUN_DEVICE_IP_ADDRESS_V4.octets()[1],
before_last_byte,
last_byte,
)
}
pub fn authenticator_ipv4_to_ipv6(addr: Ipv4Addr) -> Ipv6Addr {
let before_last_byte = addr.octets()[2];
let last_byte = addr.octets()[3];
let last_bytes = ((before_last_byte as u16) << 8) | last_byte as u16;
Ipv6Addr::new(
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[0],
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[1],
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[2],
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[3],
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[4],
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[5],
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[6],
last_bytes,
)
}
#[cfg(test)]
pub(crate) mod tests {
pub(crate) const CREDENTIAL_BYTES: [u8; 1245] = [
common/authenticator-requests/src/v4/registration.rs
+5 -21
View File
@@ -2,9 +2,9 @@
// SPDX-License-Identifier: Apache-2.0
use crate::error::Error;
use crate::util::{authenticator_ipv4_to_ipv6, authenticator_ipv6_to_ipv4};
use base64::{Engine, engine::general_purpose};
use nym_credentials_interface::CredentialSpendingData;
use nym_network_defaults::constants::{WG_TUN_DEVICE_IP_ADDRESS_V4, WG_TUN_DEVICE_IP_ADDRESS_V6};
use nym_wireguard_types::PeerPublicKey;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
@@ -56,27 +56,11 @@ impl fmt::Display for IpPair {
impl From<IpAddr> for IpPair {
fn from(value: IpAddr) -> Self {
let (before_last_byte, last_byte) = match value {
std::net::IpAddr::V4(ipv4_addr) => (ipv4_addr.octets()[2], ipv4_addr.octets()[3]),
std::net::IpAddr::V6(ipv6_addr) => (ipv6_addr.octets()[14], ipv6_addr.octets()[15]),
let (ipv4, ipv6) = match value {
IpAddr::V4(ipv4) => (ipv4, authenticator_ipv4_to_ipv6(ipv4)),
IpAddr::V6(ipv6_addr) => (authenticator_ipv6_to_ipv4(ipv6_addr), ipv6_addr),
};
let last_bytes = ((before_last_byte as u16) << 8) | last_byte as u16;
let ipv4 = Ipv4Addr::new(
WG_TUN_DEVICE_IP_ADDRESS_V4.octets()[0],
WG_TUN_DEVICE_IP_ADDRESS_V4.octets()[1],
before_last_byte,
last_byte,
);
let ipv6 = Ipv6Addr::new(
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[0],
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[1],
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[2],
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[3],
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[4],
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[5],
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[6],
last_bytes,
);
IpPair::new(ipv4, ipv6)
}
}
common/authenticator-requests/src/v5/registration.rs
+5 -21
View File
@@ -2,9 +2,9 @@
// SPDX-License-Identifier: Apache-2.0
use crate::error::Error;
use crate::util::{authenticator_ipv4_to_ipv6, authenticator_ipv6_to_ipv4};
use base64::{Engine, engine::general_purpose};
use nym_credentials_interface::CredentialSpendingData;
use nym_network_defaults::constants::{WG_TUN_DEVICE_IP_ADDRESS_V4, WG_TUN_DEVICE_IP_ADDRESS_V6};
use nym_wireguard_types::PeerPublicKey;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
@@ -54,27 +54,11 @@ impl fmt::Display for IpPair {
impl From<IpAddr> for IpPair {
fn from(value: IpAddr) -> Self {
let (before_last_byte, last_byte) = match value {
std::net::IpAddr::V4(ipv4_addr) => (ipv4_addr.octets()[2], ipv4_addr.octets()[3]),
std::net::IpAddr::V6(ipv6_addr) => (ipv6_addr.octets()[14], ipv6_addr.octets()[15]),
let (ipv4, ipv6) = match value {
IpAddr::V4(ipv4) => (ipv4, authenticator_ipv4_to_ipv6(ipv4)),
IpAddr::V6(ipv6_addr) => (authenticator_ipv6_to_ipv4(ipv6_addr), ipv6_addr),
};
let last_bytes = ((before_last_byte as u16) << 8) | last_byte as u16;
let ipv4 = Ipv4Addr::new(
WG_TUN_DEVICE_IP_ADDRESS_V4.octets()[0],
WG_TUN_DEVICE_IP_ADDRESS_V4.octets()[1],
before_last_byte,
last_byte,
);
let ipv6 = Ipv6Addr::new(
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[0],
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[1],
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[2],
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[3],
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[4],
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[5],
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[6],
last_bytes,
);
IpPair::new(ipv4, ipv6)
}
}
common/authenticator-requests/src/v6/registration.rs
+5 -23
View File
@@ -3,13 +3,12 @@
use crate::error::Error;
use crate::models::BandwidthClaim;
use crate::util::{authenticator_ipv4_to_ipv6, authenticator_ipv6_to_ipv4};
use base64::{Engine, engine::general_purpose};
use nym_network_defaults::constants::{WG_TUN_DEVICE_IP_ADDRESS_V4, WG_TUN_DEVICE_IP_ADDRESS_V6};
use nym_wireguard_types::PeerPublicKey;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
use std::time::SystemTime;
use std::{fmt, ops::Deref, str::FromStr};
#[cfg(feature = "verify")]
@@ -20,7 +19,6 @@ use nym_crypto::asymmetric::x25519::{PrivateKey, PublicKey};
use sha2::Sha256;
pub type PendingRegistrations = HashMap<PeerPublicKey, RegistrationData>;
pub type PrivateIPs = HashMap<IpPair, SystemTime>;
#[cfg(feature = "verify")]
pub type HmacSha256 = Hmac<Sha256>;
@@ -53,27 +51,11 @@ impl fmt::Display for IpPair {
impl From<IpAddr> for IpPair {
fn from(value: IpAddr) -> Self {
let (before_last_byte, last_byte) = match value {
IpAddr::V4(ipv4_addr) => (ipv4_addr.octets()[2], ipv4_addr.octets()[3]),
IpAddr::V6(ipv6_addr) => (ipv6_addr.octets()[14], ipv6_addr.octets()[15]),
let (ipv4, ipv6) = match value {
IpAddr::V4(ipv4) => (ipv4, authenticator_ipv4_to_ipv6(ipv4)),
IpAddr::V6(ipv6_addr) => (authenticator_ipv6_to_ipv4(ipv6_addr), ipv6_addr),
};
let last_bytes = ((before_last_byte as u16) << 8) | last_byte as u16;
let ipv4 = Ipv4Addr::new(
WG_TUN_DEVICE_IP_ADDRESS_V4.octets()[0],
WG_TUN_DEVICE_IP_ADDRESS_V4.octets()[1],
before_last_byte,
last_byte,
);
let ipv6 = Ipv6Addr::new(
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[0],
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[1],
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[2],
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[3],
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[4],
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[5],
WG_TUN_DEVICE_IP_ADDRESS_V6.segments()[6],
last_bytes,
);
IpPair::new(ipv4, ipv6)
}
}
common/credential-verification/src/lib.rs
+9
View File
@@ -22,6 +22,8 @@ pub mod ecash;
pub mod error;
pub mod upgrade_mode;
const MOCK_BANDWIDTH: i64 = 2024 * 1024 * 1024;
// Histogram buckets for ecash verification duration (in seconds)
const ECASH_VERIFICATION_DURATION_BUCKETS: &[f64] =
&[0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0];
@@ -111,6 +113,13 @@ impl CredentialVerifier {
}
pub async fn verify(&mut self) -> Result<i64> {
if self.ecash_verifier.is_mock() {
// if we're in the mock mode (local testing), skip cryptographic verification
// and just return a dummy bandwidth value since we don't have blockchain access
// Return a reasonable test bandwidth value (e.g., 1GB in bytes)
return Ok(MOCK_BANDWIDTH);
}
let start = Instant::now();
nym_metrics::inc!("ecash_verification_attempts");
common/credential-verification/src/upgrade_mode.rs
+37
View File
@@ -291,3 +291,40 @@ struct UpgradeModeStateInner {
// (and dealing with the async consequences of that)
status: UpgradeModeStatus,
}
pub mod testing {
use crate::UpgradeModeState;
use crate::upgrade_mode::{
CheckRequest, UpgradeModeCheckConfig, UpgradeModeCheckRequestSender, UpgradeModeDetails,
};
use futures::channel::mpsc::UnboundedReceiver;
use nym_crypto::asymmetric::ed25519;
use std::time::Duration;
pub fn mock_dummy_upgrade_mode_details() -> (UpgradeModeDetails, UnboundedReceiver<CheckRequest>)
{
let (um_recheck_tx, um_recheck_rx) = futures::channel::mpsc::unbounded();
const DUMMY_ATTESTER_ED25519_PRIVATE_KEY: [u8; 32] = [
108, 49, 193, 21, 126, 161, 249, 85, 242, 207, 74, 195, 238, 6, 64, 149, 201, 140, 248,
163, 122, 170, 79, 198, 87, 85, 36, 29, 243, 92, 64, 161,
];
pub(crate) fn dummy_attester_public_key() -> ed25519::PublicKey {
let private_key =
ed25519::PrivateKey::from_bytes(&DUMMY_ATTESTER_ED25519_PRIVATE_KEY).unwrap();
private_key.public_key()
}
let upgrade_mode_state = UpgradeModeState::new(dummy_attester_public_key());
let upgrade_mode_details = UpgradeModeDetails::new(
UpgradeModeCheckConfig {
// essentially we never want to trigger this in our tests
min_staleness_recheck: Duration::from_nanos(1),
},
UpgradeModeCheckRequestSender::new(um_recheck_tx),
upgrade_mode_state.clone(),
);
(upgrade_mode_details, um_recheck_rx)
}
}
common/registration/src/lib.rs
+16
View File
@@ -27,6 +27,22 @@ pub struct NymNodeInformation {
pub version: AuthenticatorVersion,
}
#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
pub struct WireguardRegistrationData {
/// Public x25519 key of this gateway
#[serde(with = "bs58_x25519_pubkey")]
pub public_key: x25519::PublicKey,
/// Port at which this gateway is accessible for wireguard
pub port: u16,
/// Ipv4 address assigned to this peer
pub private_ipv4: Ipv4Addr,
/// Ipv6 address assigned to this peer
pub private_ipv6: Ipv6Addr,
}
#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
pub struct WireguardConfiguration {
#[serde(with = "bs58_x25519_pubkey")]
common/registration/src/lp_messages.rs
+18 -39
View File
@@ -3,7 +3,7 @@
//! LP (Lewes Protocol) registration message types shared between client and gateway.
use crate::WireguardConfiguration;
use crate::WireguardRegistrationData;
use crate::dvpn::{
LpDvpnRegistrationFinalisation, LpDvpnRegistrationInitialRequest,
LpDvpnRegistrationRequestMessage, LpDvpnRegistrationRequestMessageContent,
@@ -16,7 +16,6 @@ use crate::mixnet::{
};
use crate::serialisation::{BincodeError, BincodeOptions, lp_bincode_serializer};
use nym_authenticator_requests::models::BandwidthClaim;
use nym_credentials_interface::TicketType;
use serde::{Deserialize, Serialize};
use tracing::error;
@@ -135,16 +134,11 @@ impl LpRegistrationRequest {
pub fn new_initial_dvpn(
wg_public_key: nym_wireguard_types::PeerPublicKey,
psk: [u8; 32],
ticket_type: TicketType,
) -> Self {
Self::new(LpRegistrationRequestData::Dvpn {
data: Box::new(LpDvpnRegistrationRequestMessage {
content: LpDvpnRegistrationRequestMessageContent::InitialRequest(
LpDvpnRegistrationInitialRequest {
wg_public_key,
psk,
ticket_type,
},
LpDvpnRegistrationInitialRequest { wg_public_key, psk },
),
}),
})
@@ -180,7 +174,7 @@ impl LpRegistrationRequest {
impl LpRegistrationResponse {
/// Create a success response with GatewayData (for dVPN mode)
pub fn success_dvpn(config: WireguardConfiguration, available_bandwidth: i64) -> Self {
pub fn success_dvpn(config: WireguardRegistrationData, upgrade_mode: bool) -> Self {
Self {
status: RegistrationStatus::Completed,
response_data: LpRegistrationResponseData::Dvpn {
@@ -188,7 +182,7 @@ impl LpRegistrationResponse {
content: LpDvpnRegistrationResponseMessageContent::CompletedRegistration(
dvpn::CompletedRegistrationResponse {
config,
available_bandwidth,
upgrade_mode,
},
),
},
@@ -196,16 +190,13 @@ impl LpRegistrationResponse {
}
}
pub fn success_mixnet(config: LpMixnetGatewayData, available_bandwidth: i64) -> Self {
pub fn success_mixnet(config: LpMixnetGatewayData) -> Self {
Self {
status: RegistrationStatus::Completed,
response_data: LpRegistrationResponseData::Mixnet {
data: LpMixnetRegistrationResponseMessage {
content: LpMixnetRegistrationResponseMessageContent::CompletedRegistration(
mixnet::CompletedRegistrationResponse {
config,
available_bandwidth,
},
mixnet::CompletedRegistrationResponse { config },
),
},
},
@@ -284,9 +275,8 @@ impl LpRegistrationResponse {
}
pub mod dvpn {
use crate::WireguardConfiguration;
use crate::WireguardRegistrationData;
use nym_authenticator_requests::models::BandwidthClaim;
use nym_credentials_interface::TicketType;
use serde::{Deserialize, Serialize};
// client
@@ -310,9 +300,6 @@ pub mod dvpn {
/// Preshared key to be used for the connection
pub psk: [u8; 32],
/// Type of the ticket/gateway we're going to register with
pub ticket_type: TicketType,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -350,10 +337,11 @@ pub mod dvpn {
pub struct CompletedRegistrationResponse {
/// Gateway configuration data for dVPN mode (WireGuard)
/// This matches what WireguardRegistrationResult expects
pub config: WireguardConfiguration,
pub config: WireguardRegistrationData,
/// The bandwidth available to this client,
pub available_bandwidth: i64,
/// Flag indicating whether the gateway has detected the system is undergoing the upgrade
/// (thus it will not meter bandwidth)
pub upgrade_mode: bool,
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
@@ -427,9 +415,6 @@ pub mod mixnet {
///
/// Contains gateway identity and sphinx key needed for nym address construction.
pub config: LpMixnetGatewayData,
/// The bandwidth available to this client,
pub available_bandwidth: i64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -446,15 +431,14 @@ mod tests {
use std::net::{Ipv4Addr, Ipv6Addr};
// ==================== Helper Functions ====================
fn create_test_gateway_data() -> WireguardConfiguration {
WireguardConfiguration {
fn create_test_wg_config() -> WireguardRegistrationData {
WireguardRegistrationData {
public_key: nym_crypto::asymmetric::x25519::PublicKey::from(
nym_sphinx::PublicKey::from([1u8; 32]),
),
psk: Some([42u8; 32]),
port: 1234,
private_ipv4: Ipv4Addr::new(10, 0, 0, 1),
private_ipv6: Ipv6Addr::new(0xfc00, 0, 0, 0, 0, 0, 0, 1),
endpoint: "192.168.1.1:8080".parse().expect("Valid test endpoint"),
}
}
@@ -499,10 +483,9 @@ mod tests {
#[test]
fn test_lp_registration_response_success_dvpn() {
let cfg = create_test_gateway_data();
let allocated_bandwidth = 500_000_000;
let cfg = create_test_wg_config();
let response = LpRegistrationResponse::success_dvpn(cfg, allocated_bandwidth);
let response = LpRegistrationResponse::success_dvpn(cfg, false);
assert!(response.status.is_successful());
let LpRegistrationResponseData::Dvpn { data } = response.response_data else {
@@ -515,7 +498,7 @@ mod tests {
panic!("unexpected response")
};
assert_eq!(complete.config, cfg);
assert_eq!(complete.available_bandwidth, allocated_bandwidth);
assert!(!complete.upgrade_mode);
}
#[test]
@@ -526,10 +509,7 @@ mod tests {
let lp_gateway_data = LpMixnetGatewayData {
gateway_identity: *valid_key.public_key(),
};
let allocated_bandwidth = 500_000_000;
let response =
LpRegistrationResponse::success_mixnet(lp_gateway_data.clone(), allocated_bandwidth);
let response = LpRegistrationResponse::success_mixnet(lp_gateway_data.clone());
assert!(response.status.is_successful());
let LpRegistrationResponseData::Mixnet { data } = response.response_data else {
@@ -542,6 +522,5 @@ mod tests {
panic!("unexpected response")
};
assert_eq!(complete.config, lp_gateway_data);
assert_eq!(complete.available_bandwidth, allocated_bandwidth);
}
}
common/service-provider-requests-common/src/lib.rs
+2 -2
View File
@@ -11,7 +11,7 @@ pub enum ProtocolError {
InvalidServiceProviderType(u8),
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
#[repr(u8)]
pub enum ServiceProviderType {
NetworkRequester = 0,
@@ -76,7 +76,7 @@ impl ServiceProviderTypeExt for u8 {
}
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)]
pub struct Protocol {
pub version: u8,
pub service_provider_type: ServiceProviderType,
common/test-utils/src/helpers.rs
+10 -10
View File
@@ -2,16 +2,16 @@
// SPDX-License-Identifier: Apache-2.0
use crate::traits::Timeboxed;
use rand_chacha::ChaCha20Rng;
use nym_bin_common::logging::tracing_subscriber::EnvFilter;
use nym_bin_common::logging::tracing_subscriber::layer::SubscriberExt;
use nym_bin_common::logging::tracing_subscriber::util::SubscriberInitExt;
use nym_bin_common::logging::{default_tracing_fmt_layer, tracing_subscriber};
use rand_chacha::rand_core::SeedableRng;
use std::future::Future;
use tokio::task::JoinHandle;
use tokio::time::error::Elapsed;
use nym_bin_common::logging::tracing_subscriber::EnvFilter;
use nym_bin_common::logging::tracing_subscriber::layer::SubscriberExt;
use nym_bin_common::logging::tracing_subscriber::util::SubscriberInitExt;
use nym_bin_common::logging::{default_tracing_fmt_layer, tracing_subscriber};
pub use rand_chacha::ChaCha20Rng as DeterministicRng;
pub use rand_chacha::rand_core::{CryptoRng, RngCore};
pub fn leak<T>(val: T) -> &'static mut T {
@@ -26,16 +26,16 @@ where
tokio::spawn(async move { fut.timeboxed().await })
}
pub fn deterministic_rng() -> ChaCha20Rng {
pub fn deterministic_rng() -> DeterministicRng {
seeded_rng([42u8; 32])
}
pub fn seeded_rng(seed: [u8; 32]) -> ChaCha20Rng {
ChaCha20Rng::from_seed(seed)
pub fn seeded_rng(seed: [u8; 32]) -> DeterministicRng {
DeterministicRng::from_seed(seed)
}
pub fn u64_seeded_rng(seed: u64) -> ChaCha20Rng {
ChaCha20Rng::seed_from_u64(seed)
pub fn u64_seeded_rng(seed: u64) -> DeterministicRng {
DeterministicRng::seed_from_u64(seed)
}
// test logger to use during debugging
common/wireguard-private-metadata/tests/src/v2/peer_controller.rs
+12 -2
View File
@@ -26,7 +26,7 @@ impl From<&PeerControlRequest> for PeerControlRequestTypeV2 {
fn from(req: &PeerControlRequest) -> Self {
match req {
PeerControlRequest::AddPeer { .. } => PeerControlRequestTypeV2::AddPeer,
PeerControlRequest::AllocatePeerIpPair { .. } => PeerControlRequestTypeV2::AddPeer,
PeerControlRequest::PreAllocateIpPair { .. } => PeerControlRequestTypeV2::AddPeer,
PeerControlRequest::RemovePeer { .. } => PeerControlRequestTypeV2::RemovePeer,
PeerControlRequest::QueryPeer { .. } => PeerControlRequestTypeV2::QueryPeer,
PeerControlRequest::GetClientBandwidthByKey { .. } => {
@@ -41,6 +41,7 @@ impl From<&PeerControlRequest> for PeerControlRequestTypeV2 {
PeerControlRequest::GetVerifierByIp { ip, .. } => {
PeerControlRequestTypeV2::GetVerifierByIp { ip: *ip }
}
PeerControlRequest::CheckActivePeer { .. } => unreachable!(),
PeerControlRequest::ReleaseIpPair { .. } => unreachable!(),
}
}
@@ -114,7 +115,7 @@ impl MockPeerControllerV2 {
)
.unwrap();
}
PeerControlRequest::AllocatePeerIpPair { response_tx, .. } => {
PeerControlRequest::PreAllocateIpPair { response_tx, .. } => {
response_tx
.send(
*response
@@ -186,6 +187,15 @@ impl MockPeerControllerV2 {
)
.ok();
}
PeerControlRequest::CheckActivePeer { response_tx, .. } => {
response_tx
.send(
*response
.downcast()
.expect("registered response has mismatched type"),
)
.ok();
}
}
}
common/wireguard/Cargo.toml
+3
View File
@@ -36,7 +36,10 @@ nym-wireguard-types = { workspace = true }
nym-node-metrics = { workspace = true }
[dev-dependencies]
anyhow = { workspace = true }
nym-gateway-storage = { workspace = true, features = ["mock"] }
mock_instant = { workspace = true }
nym-test-utils = { workspace = true }
[features]
default = []
common/wireguard/src/error.rs
+3 -1
View File
@@ -1,6 +1,8 @@
// Copyright 2024 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use crate::IpPoolError;
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("{0}")]
@@ -22,7 +24,7 @@ pub enum Error {
SystemTime(#[from] std::time::SystemTimeError),
#[error("IP pool error: {0}")]
IpPool(String),
IpPool(#[from] IpPoolError),
}
pub type Result<T> = std::result::Result<T, Error>;
common/wireguard/src/ip_pool/mod.rs
+338 -53
View File
@@ -1,20 +1,22 @@
// Copyright 2024 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
mod compat;
use defguard_wireguard_rs::host::Peer;
use ipnetwork::IpNetwork;
use rand::seq::IteratorRandom;
use std::collections::HashMap;
use std::fmt::{Display, Formatter};
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
use std::sync::Arc;
use std::time::{Duration, SystemTime};
use tokio::sync::RwLock;
use tokio::time::Instant;
use std::time::Duration;
use tracing::{trace, warn};
mod compat;
#[cfg(test)]
use mock_instant::thread_local::Instant;
#[cfg(not(test))]
use std::time::Instant;
// helper to convert peer's allocation into an `IpPair`
pub fn allocated_ip_pair(peer: &Peer) -> Option<IpPair> {
for allowed_ip in &peer.allowed_ips {
@@ -57,17 +59,38 @@ impl Display for IpPair {
pub enum AllocationState {
/// IP is available for allocation
Free,
/// IP is allocated and in use, with timestamp of allocation
Allocated(SystemTime),
/// The IP has been pre-allocated for a peer, but the corresponding registration has not yet been finalised
PreAllocated { allocated_at: Instant },
/// IP is allocated and in use, with timestamp
Allocated { allocated_at: Instant },
}
impl AllocationState {
pub fn is_free(&self) -> bool {
matches!(self, AllocationState::Free)
}
pub fn new_pre_allocated() -> Self {
AllocationState::PreAllocated {
allocated_at: Instant::now(),
}
}
pub fn new_allocated() -> Self {
AllocationState::Allocated {
allocated_at: Instant::now(),
}
}
}
/// Thread-safe IP address pool manager
///
/// Manages allocation of IPv4/IPv6 address pairs from configured CIDR ranges.
/// Ensures collision-free allocation and supports stale cleanup.
#[derive(Clone)]
pub struct IpPool {
allocations: Arc<RwLock<HashMap<IpPair, AllocationState>>>,
allocations: HashMap<IpPair, AllocationState>,
}
impl IpPool {
@@ -98,7 +121,7 @@ impl IpPool {
.iter()
.filter_map(|ip| {
if let IpAddr::V4(v4) = ip {
Some(v4)
if v4 != ipv4_network { Some(v4) } else { None }
} else {
None
}
@@ -109,7 +132,7 @@ impl IpPool {
.iter()
.filter_map(|ip| {
if let IpAddr::V6(v6) = ip {
Some(v6)
if v6 != ipv6_network { Some(v6) } else { None }
} else {
None
}
@@ -129,21 +152,18 @@ impl IpPool {
allocations.len(),
);
Ok(IpPool {
allocations: Arc::new(RwLock::new(allocations)),
})
Ok(IpPool { allocations })
}
/// Allocate a free IP pair from the pool
/// Preallocate a free IP pair from the pool
///
/// # Errors
/// Returns `IpPoolError::NoFreeIp` if no IPs are available
pub async fn allocate(&self) -> Result<IpPair, IpPoolError> {
let mut pool = self.allocations.write().await;
pub fn pre_allocate(&mut self) -> Result<IpPair, IpPoolError> {
// Find a free IP and allocate it
let assignment_start = Instant::now();
let free_ip = pool
let free_ip = self
.allocations
.iter_mut()
.filter(|(_, state)| matches!(state, AllocationState::Free))
.choose(&mut rand::thread_rng())
@@ -155,18 +175,42 @@ impl IpPool {
}
let ip_pair = *free_ip.0;
*free_ip.1 = AllocationState::Allocated(SystemTime::now());
*free_ip.1 = AllocationState::new_pre_allocated();
tracing::debug!("Allocated IP pair: {ip_pair}");
Ok(ip_pair)
}
pub fn confirm_allocation(&mut self, ip_pair: IpPair) -> Result<(), IpPoolError> {
let Some(allocation) = self.allocations.get_mut(&ip_pair) else {
return Err(IpPoolError::UnknownIpPair { ip_pair });
};
match allocation {
AllocationState::Free => {
// seems the IpPair has been released before the confirmation, but it has not yet been re-allocated
warn!(
"{ip_pair} seems to have already been released, but has not been allocated to a new peer yet"
);
*allocation = AllocationState::Allocated {
allocated_at: Instant::now(),
};
Ok(())
}
AllocationState::PreAllocated { allocated_at } => {
*allocation = AllocationState::Allocated {
allocated_at: *allocated_at,
};
Ok(())
}
AllocationState::Allocated { .. } => Err(IpPoolError::AlreadyUsed { ip_pair }),
}
}
/// Release an IP pair back to the pool
///
/// Marks the IP as free for future allocations.
pub async fn release(&self, ip_pair: IpPair) {
let mut pool = self.allocations.write().await;
if let Some(state) = pool.get_mut(&ip_pair) {
pub fn release(&mut self, ip_pair: IpPair) {
if let Some(state) = self.allocations.get_mut(&ip_pair) {
*state = AllocationState::Free;
tracing::debug!("Released IP pair: {ip_pair}");
}
@@ -175,58 +219,67 @@ impl IpPool {
/// Mark an IP pair as allocated (used during initialization from database)
///
/// This is used when restoring state from the database on gateway startup.
pub async fn mark_used(&self, ip_pair: IpPair) {
let mut pool = self.allocations.write().await;
if let Some(state) = pool.get_mut(&ip_pair) {
*state = AllocationState::Allocated(SystemTime::now());
tracing::debug!("Marked IP pair as used: {ip_pair}");
} else {
tracing::warn!("Attempted to mark unknown IP pair as used: {ip_pair}");
pub fn mark_used(&mut self, ip_pair: IpPair) -> Result<(), IpPoolError> {
let Some(state) = self.allocations.get_mut(&ip_pair) else {
return Err(IpPoolError::UnknownIpPair { ip_pair });
};
if !state.is_free() {
return Err(IpPoolError::AlreadyUsed { ip_pair });
}
tracing::debug!("Marked IP pair as used: {ip_pair}");
*state = AllocationState::new_allocated();
Ok(())
}
/// Get the number of free IPs in the pool
pub async fn free_count(&self) -> usize {
let pool = self.allocations.read().await;
pool.iter()
pub fn free_count(&self) -> usize {
self.allocations
.iter()
.filter(|(_, state)| matches!(state, AllocationState::Free))
.count()
}
/// Get the number of allocated IPs in the pool
pub async fn allocated_count(&self) -> usize {
let pool = self.allocations.read().await;
pool.iter()
.filter(|(_, state)| matches!(state, AllocationState::Allocated(_)))
pub fn pre_allocated_count(&self) -> usize {
self.allocations
.iter()
.filter(|(_, state)| matches!(state, AllocationState::PreAllocated { .. }))
.count()
}
/// Get the number of allocated IPs in the pool
pub fn allocated_count(&self) -> usize {
self.allocations
.iter()
.filter(|(_, state)| matches!(state, AllocationState::Allocated { .. }))
.count()
}
/// Get the total pool size
pub async fn total_count(&self) -> usize {
let pool = self.allocations.read().await;
pool.len()
pub fn total_count(&self) -> usize {
self.allocations.len()
}
/// Clean up stale allocations older than the specified duration
///
/// Returns the number of IPs that were freed
pub async fn cleanup_stale(&self, max_age: std::time::Duration) -> usize {
let mut pool = self.allocations.write().await;
let now = SystemTime::now();
pub fn cleanup_stale(&mut self, max_age: Duration) -> usize {
let now = Instant::now();
let mut freed = 0;
for (_ip, state) in pool.iter_mut() {
if let AllocationState::Allocated(allocated_at) = state
&& let Ok(age) = now.duration_since(*allocated_at)
&& age > max_age
{
*state = AllocationState::Free;
freed += 1;
for state in self.allocations.values_mut() {
if let AllocationState::PreAllocated { allocated_at, .. } = state {
let age = now.duration_since(*allocated_at);
if age > max_age {
*state = AllocationState::Free;
freed += 1;
}
}
}
if freed > 0 {
tracing::info!("Cleaned up {} stale IP allocations", freed);
tracing::info!("Cleaned up {freed} stale IP allocations");
}
freed
@@ -234,11 +287,243 @@ impl IpPool {
}
/// Errors that can occur during IP pool operations
#[derive(Debug, thiserror::Error)]
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
pub enum IpPoolError {
#[error("No free IP addresses available in pool")]
NoFreeIp,
#[error("Attempted to mark an IpPair that is already in used: {ip_pair}")]
AlreadyUsed { ip_pair: IpPair },
#[error("Attempted to mark an unknown ip pair: {ip_pair}")]
UnknownIpPair { ip_pair: IpPair },
#[error("Invalid IP network configuration: {0}")]
InvalidNetwork(#[from] ipnetwork::IpNetworkError),
}
#[cfg(test)]
mod tests {
use super::*;
use anyhow::bail;
use mock_instant::thread_local::MockClock;
// 3 addresses in each pool
fn small_ip_pool() -> IpPool {
IpPool::new(
Ipv4Addr::new(10, 0, 0, 0),
30,
Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 0),
126,
)
.unwrap()
}
#[test]
fn ip_pool_initial_allocation() -> anyhow::Result<()> {
let base_ipv4_network = Ipv4Addr::new(10, 0, 0, 0);
let base_ipv4_prefix = 24; // 255 addresses
let base_ipv6_network = Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 0);
let base_ipv6_prefix = 112; // 65535 addresses
// ipv4 pool size < ipv6 pool size
let base_ip_pool = IpPool::new(
base_ipv4_network,
base_ipv4_prefix,
base_ipv6_network,
base_ipv6_prefix,
)?;
let inner = base_ip_pool.allocations;
// minimum of ipv4 and ipv6 allocations
assert_eq!(inner.len(), 255);
// no ipv4 addresses
let base_ip_pool = IpPool::new(base_ipv4_network, 32, base_ipv6_network, base_ipv6_prefix)?;
let inner = base_ip_pool.allocations;
assert_eq!(inner.len(), 0);
// no ipv6 addresses
let base_ip_pool =
IpPool::new(base_ipv4_network, base_ipv4_prefix, base_ipv6_network, 128)?;
let inner = base_ip_pool.allocations;
assert_eq!(inner.len(), 0);
// ipv4 pool size == ipv6 pool size
let base_ip_pool = IpPool::new(base_ipv4_network, 16, base_ipv6_network, base_ipv6_prefix)?;
let inner = base_ip_pool.allocations;
assert_eq!(inner.len(), 65535);
// ipv4 pool size > ipv6 pool size
let base_ip_pool = IpPool::new(base_ipv4_network, 12, base_ipv6_network, base_ipv6_prefix)?;
let inner = base_ip_pool.allocations;
assert_eq!(inner.len(), 65535);
Ok(())
}
fn ensure_different_allocation(left: IpPair, right: IpPair) -> anyhow::Result<()> {
if left.ipv4 == right.ipv4 || left.ipv6 == right.ipv6 {
bail!("ip allocation overlap")
}
Ok(())
}
#[test]
fn ip_pool_allocation() -> anyhow::Result<()> {
let mut pool = small_ip_pool();
assert_eq!(pool.allocations.len(), 3);
let gateway_pair = IpPair {
ipv4: Ipv4Addr::new(10, 0, 0, 0),
ipv6: Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 0),
};
assert_eq!(pool.free_count(), 3);
assert_eq!(pool.pre_allocated_count(), 0);
let allocation1 = pool.pre_allocate()?;
assert_eq!(pool.free_count(), 2);
assert_eq!(pool.pre_allocated_count(), 1);
let allocation2 = pool.pre_allocate()?;
assert_eq!(pool.free_count(), 1);
assert_eq!(pool.pre_allocated_count(), 2);
pool.confirm_allocation(allocation1)?;
assert_eq!(pool.free_count(), 1);
assert_eq!(pool.pre_allocated_count(), 1);
assert_eq!(pool.allocated_count(), 1);
let allocation3 = pool.pre_allocate()?;
assert_eq!(pool.free_count(), 0);
assert_eq!(pool.pre_allocated_count(), 2);
assert_eq!(pool.allocated_count(), 1);
// make sure each was unique and different from the gateway
ensure_different_allocation(allocation1, allocation2)?;
ensure_different_allocation(allocation1, allocation3)?;
ensure_different_allocation(allocation2, allocation3)?;
ensure_different_allocation(allocation1, gateway_pair)?;
ensure_different_allocation(allocation2, gateway_pair)?;
ensure_different_allocation(allocation3, gateway_pair)?;
// allocation 4 will fail as we have run out of addresses
assert_eq!(pool.pre_allocate().unwrap_err(), IpPoolError::NoFreeIp);
// if pair gets released, it's eligible for allocation again
pool.release(allocation2);
assert_eq!(pool.free_count(), 1);
assert_eq!(pool.pre_allocated_count(), 1);
assert_eq!(pool.allocated_count(), 1);
let reallocation = pool.pre_allocate()?;
assert_eq!(reallocation, allocation2);
assert_eq!(pool.free_count(), 0);
assert_eq!(pool.pre_allocated_count(), 2);
assert_eq!(pool.allocated_count(), 1);
Ok(())
}
#[test]
fn ip_pool_mark_used() -> anyhow::Result<()> {
let mut pool = small_ip_pool();
let pair1 = IpPair::new(
Ipv4Addr::new(10, 0, 0, 1),
Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 1),
);
let pair2 = IpPair::new(
Ipv4Addr::new(10, 0, 0, 2),
Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 2),
);
let pair3 = IpPair::new(
Ipv4Addr::new(10, 0, 0, 3),
Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 3),
);
let bad_pair1 = IpPair::new(
Ipv4Addr::new(10, 0, 0, 1),
Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 4),
);
let bad_pair2 = IpPair::new(
Ipv4Addr::new(10, 0, 0, 4),
Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 1),
);
assert!(pool.mark_used(pair1,).is_ok());
assert_eq!(
pool.mark_used(pair1).unwrap_err(),
IpPoolError::AlreadyUsed { ip_pair: pair1 }
);
assert!(pool.mark_used(pair2).is_ok());
assert!(pool.mark_used(pair3).is_ok());
assert_eq!(
pool.mark_used(bad_pair1).unwrap_err(),
IpPoolError::UnknownIpPair { ip_pair: bad_pair1 }
);
assert_eq!(
pool.mark_used(bad_pair2,).unwrap_err(),
IpPoolError::UnknownIpPair { ip_pair: bad_pair2 }
);
Ok(())
}
#[test]
fn ip_pool_cleanup() -> anyhow::Result<()> {
MockClock::set_time(Duration::ZERO);
let mut pool = small_ip_pool();
let age_threshold = Duration::from_secs(1);
// nothing to cleanup
assert_eq!(pool.cleanup_stale(age_threshold), 0);
// just allocated
let pair1 = pool.pre_allocate()?;
let pair2 = pool.pre_allocate()?;
assert_eq!(pool.cleanup_stale(age_threshold), 0);
// advance time to go beyond the allocation threshold
MockClock::advance(Duration::from_millis(1001));
assert_eq!(pool.cleanup_stale(age_threshold), 2);
// ensure those pairs are now marked as free
assert!(pool.allocations.get(&pair1).unwrap().is_free());
assert!(pool.allocations.get(&pair2).unwrap().is_free());
pool.pre_allocate()?;
MockClock::advance(Duration::from_millis(500));
pool.pre_allocate()?;
assert_eq!(pool.cleanup_stale(age_threshold), 0);
MockClock::advance(Duration::from_millis(501));
assert_eq!(pool.cleanup_stale(age_threshold), 1);
MockClock::advance(Duration::from_millis(500));
assert_eq!(pool.cleanup_stale(age_threshold), 1);
let mut new_pool = small_ip_pool();
let pair1 = new_pool.pre_allocate()?;
let pair2 = new_pool.pre_allocate()?;
// complete allocation for pair2
new_pool.confirm_allocation(pair2)?;
MockClock::advance(Duration::from_millis(2000));
// only pair1 should have got cleaned up
assert_eq!(new_pool.cleanup_stale(age_threshold), 1);
assert!(new_pool.allocations.get(&pair1).unwrap().is_free());
assert!(!new_pool.allocations.get(&pair2).unwrap().is_free());
Ok(())
}
}
common/wireguard/src/lib.rs
+2 -2
View File
@@ -284,7 +284,7 @@ pub async fn start_wireguard(
// Initialize IP pool from configuration
info!("Initializing IP pool for WireGuard peer allocation");
let ip_pool = IpPool::new(
let mut ip_pool = IpPool::new(
wireguard_data.inner.config().private_ipv4,
wireguard_data.inner.config().private_network_prefix_v4,
wireguard_data.inner.config().private_ipv6,
@@ -294,7 +294,7 @@ pub async fn start_wireguard(
// Mark existing peer IPs as used in the pool
for peer in &peers {
if let Some(ip_pair) = crate::ip_pool::allocated_ip_pair(peer) {
ip_pool.mark_used(ip_pair).await;
ip_pool.mark_used(ip_pair)?;
}
}
common/wireguard/src/peer_controller/mock.rs
+10 -2
View File
@@ -75,6 +75,7 @@ pub enum PeerControlRequestType {
ReleaseIpPair { ip_pair: IpPair },
RemovePeer { key: KeyWrapper },
QueryPeer { key: KeyWrapper },
CheckActivePeer { key: KeyWrapper },
GetClientBandwidthByKey { key: KeyWrapper },
GetClientBandwidthByIp { ip: IpAddr },
GetVerifierByKey { key: KeyWrapper },
@@ -93,6 +94,7 @@ impl PeerControlRequestType {
PeerControlRequestType::GetClientBandwidthByIp { .. } => None,
PeerControlRequestType::GetVerifierByKey { key } => Some(key.clone()),
PeerControlRequestType::GetVerifierByIp { .. } => None,
PeerControlRequestType::CheckActivePeer { key } => Some(key.clone()),
}
}
@@ -107,7 +109,7 @@ impl From<&PeerControlRequest> for PeerControlRequestType {
PeerControlRequest::AddPeer { peer, .. } => PeerControlRequestType::AddPeer {
public_key: (&peer.public_key).into(),
},
PeerControlRequest::AllocatePeerIpPair { .. } => {
PeerControlRequest::PreAllocateIpPair { .. } => {
PeerControlRequestType::AllocatePeerIpPair {}
}
PeerControlRequest::ReleaseIpPair { ip_pair, .. } => {
@@ -131,6 +133,9 @@ impl From<&PeerControlRequest> for PeerControlRequestType {
PeerControlRequest::GetVerifierByIp { ip, .. } => {
PeerControlRequestType::GetVerifierByIp { ip: *ip }
}
PeerControlRequest::CheckActivePeer { key, .. } => {
PeerControlRequestType::CheckActivePeer { key: key.into() }
}
}
}
}
@@ -266,7 +271,7 @@ impl MockPeerController {
}
response_tx.send_downcasted(response.content)
}
PeerControlRequest::AllocatePeerIpPair { response_tx, .. } => {
PeerControlRequest::PreAllocateIpPair { response_tx, .. } => {
response_tx.send_downcasted(response.content)
}
PeerControlRequest::ReleaseIpPair {
@@ -295,6 +300,9 @@ impl MockPeerController {
PeerControlRequest::GetVerifierByIp { response_tx, .. } => {
response_tx.send_downcasted(response.content)
}
PeerControlRequest::CheckActivePeer { response_tx, .. } => {
response_tx.send_downcasted(response.content)
}
}
}
common/wireguard/src/peer_controller/mod.rs
+34 -16
View File
@@ -77,7 +77,7 @@ pub enum PeerControlRequest {
response_tx: oneshot::Sender<AddPeerControlResponse>,
},
/// Attempt to allocate an IP pair from the pool
AllocatePeerIpPair {
PreAllocateIpPair {
response_tx: oneshot::Sender<AllocatePeerControlResponse>,
},
/// Attempt to return an IP pair back to the pool
@@ -93,6 +93,10 @@ pub enum PeerControlRequest {
key: Key,
response_tx: oneshot::Sender<QueryPeerControlResponse>,
},
CheckActivePeer {
key: Key,
response_tx: oneshot::Sender<CheckActivePeerResponse>,
},
GetClientBandwidthByKey {
key: Key,
response_tx: oneshot::Sender<GetClientBandwidthControlResponse>,
@@ -118,6 +122,7 @@ pub type AllocatePeerControlResponse = Result<IpPair>;
pub type ReleaseIpPairControlResponse = Result<()>;
pub type RemovePeerControlResponse = Result<()>;
pub type QueryPeerControlResponse = Result<Option<Peer>>;
pub type CheckActivePeerResponse = Result<bool>;
pub type GetClientBandwidthControlResponse = Result<ClientBandwidth>;
pub type QueryVerifierControlResponse = Result<Box<dyn TicketVerifier + Send + Sync>>;
@@ -216,7 +221,7 @@ impl PeerController {
if let Ok(Some(peer)) = self.handle_query_peer_by_key(key).await
&& let Some(ip_pair) = allocated_ip_pair(&peer)
{
self.ip_pool.release(ip_pair).await
self.ip_pool.release(ip_pair)
}
let ret = self.wg_api.remove_peer(key);
@@ -258,6 +263,13 @@ impl PeerController {
async fn handle_add_request(&mut self, peer: &Peer) -> Result<()> {
nym_metrics::inc!("wg_peer_addition_attempts");
// confirm ip allocation so that it wouldn't be released for as long as the peer exists
let Some(ip_pair) = allocated_ip_pair(peer) else {
return Err(Error::Internal(
"could not determine ip pair allocated to the peer".to_string(),
));
};
// Try to configure WireGuard peer
if let Err(e) = self.wg_api.configure_peer(peer) {
nym_metrics::inc!("wg_peer_addition_failed");
@@ -289,6 +301,9 @@ impl PeerController {
*self.host_information.write().await = host_information;
}
let public_key = peer.public_key.clone();
self.ip_pool.confirm_allocation(ip_pair)?;
tokio::spawn(async move {
handle.run().await;
debug!("Peer handle shut down for {public_key}");
@@ -302,15 +317,11 @@ impl PeerController {
///
/// This only allocates IPs - the caller must handle database storage and
/// then call AddPeer with a complete Peer struct.
async fn handle_ip_allocation_request(&mut self) -> Result<IpPair> {
fn handle_ip_allocation_request(&mut self) -> Result<IpPair> {
nym_metrics::inc!("wg_ip_allocation_attempts");
// Allocate IP pair from pool
let ip_pair = self
.ip_pool
.allocate()
.await
.map_err(|e| Error::IpPool(e.to_string()))?;
let ip_pair = self.ip_pool.pre_allocate()?;
nym_metrics::inc!("wg_ip_allocation_success");
tracing::debug!("Allocated IP pair: {ip_pair}");
@@ -319,8 +330,8 @@ impl PeerController {
}
/// Return IP pair back to the pool
async fn handle_ip_release(&mut self, ip_pair: IpPair) {
self.ip_pool.release(ip_pair).await
fn handle_ip_release(&mut self, ip_pair: IpPair) {
self.ip_pool.release(ip_pair)
}
async fn ip_to_key(&self, ip: IpAddr) -> Result<Option<Key>> {
@@ -359,6 +370,12 @@ impl PeerController {
.client_bandwidth())
}
fn check_active_peer(&self, key: Key) -> Result<bool> {
// peer is active as long as we still have an entry inside the bandwidth storage manager,
// as it is removed upon peer removal
Ok(self.bw_storage_managers.contains_key(&key))
}
async fn handle_get_client_bandwidth_by_ip(&self, ip: IpAddr) -> Result<ClientBandwidth> {
let Some(key) = self.ip_to_key(ip).await? else {
return Err(Error::MissingClientKernelEntry(ip.to_string()));
@@ -492,16 +509,14 @@ impl PeerController {
PeerControlRequest::AddPeer { peer, response_tx } => {
response_tx.send(self.handle_add_request(&peer).await).ok();
}
PeerControlRequest::AllocatePeerIpPair { response_tx } => {
response_tx
.send(self.handle_ip_allocation_request().await)
.ok();
PeerControlRequest::PreAllocateIpPair { response_tx } => {
response_tx.send(self.handle_ip_allocation_request()).ok();
}
PeerControlRequest::ReleaseIpPair {
response_tx,
ip_pair,
} => {
self.handle_ip_release(ip_pair).await;
self.handle_ip_release(ip_pair);
response_tx.send(Ok(())).ok();
}
PeerControlRequest::RemovePeer { key, response_tx } => {
@@ -540,6 +555,9 @@ impl PeerController {
.send(self.handle_query_verifier_by_ip(ip, *credential).await)
.ok();
}
PeerControlRequest::CheckActivePeer { key, response_tx } => {
response_tx.send(self.check_active_peer(key)).ok();
}
}
}
@@ -558,7 +576,7 @@ impl PeerController {
}
_ = self.ip_cleanup_interval.next() => {
// Periodically cleanup stale IP allocations
let freed = self.ip_pool.cleanup_stale(DEFAULT_IP_STALE_AGE).await;
let freed = self.ip_pool.cleanup_stale(DEFAULT_IP_STALE_AGE);
if freed > 0 {
nym_metrics::inc_by!("wg_stale_ips_cleaned", freed as u64);
info!("Cleaned up {} stale IP allocations", freed);
gateway/Cargo.toml
+3 -1
View File
@@ -89,9 +89,11 @@ bytes = { workspace = true }
defguard_wireguard_rs = { workspace = true }
[dev-dependencies]
anyhow = { workspace = true }
nym-test-utils = { workspace = true }
nym-gateway-storage = { workspace = true, features = ["mock"] }
nym-wireguard = { workspace = true, features = ["mock"] }
mock_instant = "0.6.0"
mock_instant = { workspace = true }
time = { workspace = true }
[lints]
gateway/src/node/internal_service_providers/authenticator/mixnet_listener.rs
+18 -609
View File
@@ -4,29 +4,15 @@
use crate::node::internal_service_providers::authenticator::{
config::Config, error::AuthenticatorError, seen_credential_cache::SeenCredentialCache,
};
use crate::node::wireguard::PeerManager;
use defguard_wireguard_rs::net::IpAddrMask;
use defguard_wireguard_rs::{host::Peer, key::Key};
use crate::node::wireguard::{PeerManager, PeerRegistrator};
use futures::StreamExt;
use nym_authenticator_requests::models::BandwidthClaim;
use nym_authenticator_requests::traits::UpgradeModeMessage;
use nym_authenticator_requests::{latest, v4::registration::IpPair};
use nym_authenticator_requests::{
latest::registration::{GatewayClient, PendingRegistrations, PrivateIPs},
request::AuthenticatorRequest,
traits::{FinalMessage, InitMessage, QueryBandwidthMessage, TopUpMessage},
v1, v2, v3, v4, v5, v6, AuthenticatorVersion, CURRENT_VERSION,
};
use nym_credential_verification::ecash::traits::EcashManager;
use nym_credential_verification::upgrade_mode::UpgradeModeDetails;
use nym_credential_verification::{
bandwidth_storage_manager::BandwidthStorageManager, BandwidthFlushingBehaviourConfig,
ClientBandwidth, CredentialVerifier,
};
use nym_credentials_interface::{BandwidthCredential, CredentialSpendingData};
use nym_crypto::asymmetric::x25519::KeyPair;
use nym_gateway_requests::models::CredentialSpendingRequest;
use nym_gateway_storage::models::PersistedBandwidth;
use nym_sdk::mixnet::{
AnonymousSenderTag, InputMessage, MixnetMessageSender, Recipient, TransmissionLane,
};
@@ -36,50 +22,28 @@ use nym_task::ShutdownToken;
use nym_wireguard::WireguardGatewayData;
use nym_wireguard_types::PeerPublicKey;
use std::cmp::max;
use std::{
net::IpAddr,
sync::Arc,
time::{Duration, SystemTime},
};
use tokio::sync::RwLock;
use std::time::Duration;
use tokio_stream::wrappers::IntervalStream;
type AuthenticatorHandleResult = Result<(Vec<u8>, Option<Recipient>), AuthenticatorError>;
const DEFAULT_REGISTRATION_TIMEOUT_CHECK: Duration = Duration::from_secs(60); // 1 minute
const DEFAULT_CREDENTIAL_TIMEOUT_CHECK: Duration = Duration::from_secs(60); // 1 minute
// we need to be above MINIMUM_REMAINING_BANDWIDTH (500MB) plus we also have to trick the client
// its depletion is low enough to not require sending new tickets
const DEFAULT_WG_CLIENT_BANDWIDTH_THRESHOLD: i64 = 1024 * 1024 * 1024;
pub(crate) struct RegisteredAndFree {
registration_in_progress: PendingRegistrations,
taken_private_network_ips: PrivateIPs,
}
impl RegisteredAndFree {
pub(crate) fn new() -> Self {
RegisteredAndFree {
registration_in_progress: Default::default(),
taken_private_network_ips: Default::default(),
}
}
}
pub(crate) struct MixnetListener {
// The configuration for the mixnet listener
pub(crate) config: Config,
pub(crate) _config: Config,
// The mixnet client that we use to send and receive packets from the mixnet
pub(crate) mixnet_client: nym_sdk::mixnet::MixnetClient,
// Registrations awaiting confirmation
pub(crate) registered_and_free: RwLock<RegisteredAndFree>,
pub(crate) peer_manager: PeerManager,
pub(crate) upgrade_mode: UpgradeModeDetails,
pub(crate) ecash_verifier: Arc<dyn EcashManager + Send + Sync>,
pub(crate) peer_registrator: PeerRegistrator,
pub(crate) timeout_check_interval: IntervalStream,
@@ -91,18 +55,17 @@ impl MixnetListener {
config: Config,
wireguard_gateway_data: WireguardGatewayData,
mixnet_client: nym_sdk::mixnet::MixnetClient,
peer_registrator: PeerRegistrator,
upgrade_mode: UpgradeModeDetails,
ecash_verifier: Arc<dyn EcashManager + Send + Sync>,
) -> Self {
let timeout_check_interval =
IntervalStream::new(tokio::time::interval(DEFAULT_REGISTRATION_TIMEOUT_CHECK));
IntervalStream::new(tokio::time::interval(DEFAULT_CREDENTIAL_TIMEOUT_CHECK));
MixnetListener {
config,
_config: config,
mixnet_client,
registered_and_free: RwLock::new(RegisteredAndFree::new()),
peer_manager: PeerManager::new(wireguard_gateway_data),
upgrade_mode,
ecash_verifier,
peer_registrator,
timeout_check_interval,
seen_credential_cache: SeenCredentialCache::new(),
}
@@ -112,10 +75,6 @@ impl MixnetListener {
self.upgrade_mode.enabled()
}
fn keypair(&self) -> &Arc<KeyPair> {
self.peer_manager.wireguard_gateway_data.keypair()
}
async fn upgrade_mode_bandwidth(&self, peer: PeerPublicKey) -> Result<i64, AuthenticatorError> {
// if we're undergoing upgrade mode, we don't meter bandwidth,
// we simply return MAX of clients current bandwidth and minimum bandwidth before default
@@ -129,47 +88,6 @@ impl MixnetListener {
))
}
async fn remove_stale_registrations(&self) -> Result<(), AuthenticatorError> {
let mut registered_and_free = self.registered_and_free.write().await;
let registered_values: Vec<_> = registered_and_free
.registration_in_progress
.values()
.cloned()
.collect();
for reg in registered_values {
let timestamp = registered_and_free
.taken_private_network_ips
.get_mut(&reg.gateway_data.private_ips)
.ok_or(AuthenticatorError::InternalDataCorruption(format!(
"IPs {} should be present",
reg.gateway_data.private_ips
)))?;
let duration = SystemTime::now().duration_since(*timestamp).map_err(|_| {
AuthenticatorError::InternalDataCorruption(
"set timestamp shouldn't have been set in the future".to_string(),
)
})?;
if duration > DEFAULT_REGISTRATION_TIMEOUT_CHECK {
registered_and_free
.registration_in_progress
.remove(&reg.gateway_data.pub_key());
registered_and_free
.taken_private_network_ips
.remove(&reg.gateway_data.private_ips);
self.peer_manager
.release_ip_pair(reg.gateway_data.private_ips.into())
.await?;
tracing::debug!(
"Removed stale registration of {}",
reg.gateway_data.pub_key()
);
}
}
Ok(())
}
async fn on_initial_request(
&mut self,
init_message: Box<dyn InitMessage + Send + Sync + 'static>,
@@ -177,352 +95,12 @@ impl MixnetListener {
request_id: u64,
reply_to: Option<Recipient>,
) -> AuthenticatorHandleResult {
let remote_public = init_message.pub_key();
let nonce: u64 = fastrand::u64(..);
let mut registered_and_free = self.registered_and_free.write().await;
if let Some(registration_data) = registered_and_free
.registration_in_progress
.get(&remote_public)
{
let gateway_data = registration_data.gateway_data.clone();
let bytes = match AuthenticatorVersion::from(protocol) {
AuthenticatorVersion::V1 => {
v1::response::AuthenticatorResponse::new_pending_registration_success(
v1::registration::RegistrationData {
nonce: registration_data.nonce,
gateway_data: v1::GatewayClient {
pub_key: gateway_data.pub_key,
private_ip: gateway_data.private_ips.ipv4.into(),
mac: v1::ClientMac::new(gateway_data.mac.to_vec()),
},
wg_port: registration_data.wg_port,
},
request_id,
reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?,
)
.to_bytes()
.map_err(AuthenticatorError::response_serialisation)?
}
AuthenticatorVersion::V2 => {
v2::response::AuthenticatorResponse::new_pending_registration_success(
v2::registration::RegistrationData {
nonce: registration_data.nonce,
gateway_data: v2::registration::GatewayClient::new(
self.keypair().private_key(),
remote_public.inner(),
registration_data.gateway_data.private_ips.ipv4.into(),
registration_data.nonce,
),
wg_port: registration_data.wg_port,
},
request_id,
reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?,
)
.to_bytes()
.map_err(AuthenticatorError::response_serialisation)?
}
AuthenticatorVersion::V3 => {
v3::response::AuthenticatorResponse::new_pending_registration_success(
v3::registration::RegistrationData {
nonce: registration_data.nonce,
gateway_data: v3::registration::GatewayClient::new(
self.keypair().private_key(),
remote_public.inner(),
registration_data.gateway_data.private_ips.ipv4.into(),
registration_data.nonce,
),
wg_port: registration_data.wg_port,
},
request_id,
reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?,
)
.to_bytes()
.map_err(AuthenticatorError::response_serialisation)?
}
AuthenticatorVersion::V4 => {
v4::response::AuthenticatorResponse::new_pending_registration_success(
v4::registration::RegistrationData {
nonce: registration_data.nonce,
// convert current to v5 and then v5 to v4 (current as of 28.08.25)
gateway_data: v5::registration::GatewayClient::from(
registration_data.gateway_data.clone(),
)
.into(),
wg_port: registration_data.wg_port,
},
request_id,
reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?,
)
.to_bytes()
.map_err(AuthenticatorError::response_serialisation)?
}
AuthenticatorVersion::V5 => {
v5::response::AuthenticatorResponse::new_pending_registration_success(
v5::registration::RegistrationData {
nonce: registration_data.nonce,
gateway_data: registration_data.gateway_data.clone().into(),
wg_port: registration_data.wg_port,
},
request_id,
)
.to_bytes()
.map_err(AuthenticatorError::response_serialisation)?
}
AuthenticatorVersion::V6 => {
v6::response::AuthenticatorResponse::new_pending_registration_success(
v6::registration::RegistrationData {
nonce: registration_data.nonce,
gateway_data: registration_data.gateway_data.clone(),
wg_port: registration_data.wg_port,
},
request_id,
self.upgrade_mode_enabled(),
)
.to_bytes()
.map_err(AuthenticatorError::response_serialisation)?
}
AuthenticatorVersion::UNKNOWN => return Err(AuthenticatorError::UnknownVersion),
};
return Ok((bytes, reply_to));
}
let response = self
.peer_registrator
.on_initial_authenticator_request(init_message, protocol, request_id, reply_to)
.await?;
let peer = self.peer_manager.query_peer(remote_public).await?;
if let Some(peer) = peer {
let allowed_ipv4 = peer
.allowed_ips
.iter()
.find_map(|ip_mask| match ip_mask.address {
IpAddr::V4(ipv4_addr) => Some(ipv4_addr),
_ => None,
})
.ok_or(AuthenticatorError::InternalError(
"there should be one private IPv4 in the list".to_string(),
))?;
let allowed_ipv6 = peer
.allowed_ips
.iter()
.find_map(|ip_mask| match ip_mask.address {
IpAddr::V6(ipv6_addr) => Some(ipv6_addr),
_ => None,
})
.unwrap_or(IpPair::from(IpAddr::from(allowed_ipv4)).ipv6);
let bytes = match AuthenticatorVersion::from(protocol) {
AuthenticatorVersion::V1 => v1::response::AuthenticatorResponse::new_registered(
v1::registration::RegisteredData {
pub_key: self.keypair().public_key().into(),
private_ip: allowed_ipv4.into(),
wg_port: self.config.authenticator.tunnel_announced_port,
},
reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?,
request_id,
)
.to_bytes()
.map_err(AuthenticatorError::response_serialisation)?,
AuthenticatorVersion::V2 => v2::response::AuthenticatorResponse::new_registered(
v2::registration::RegisteredData {
pub_key: self.keypair().public_key().into(),
private_ip: allowed_ipv4.into(),
wg_port: self.config.authenticator.tunnel_announced_port,
},
reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?,
request_id,
)
.to_bytes()
.map_err(AuthenticatorError::response_serialisation)?,
AuthenticatorVersion::V3 => v3::response::AuthenticatorResponse::new_registered(
v3::registration::RegisteredData {
pub_key: self.keypair().public_key().into(),
private_ip: allowed_ipv4.into(),
wg_port: self.config.authenticator.tunnel_announced_port,
},
reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?,
request_id,
)
.to_bytes()
.map_err(AuthenticatorError::response_serialisation)?,
AuthenticatorVersion::V4 => v4::response::AuthenticatorResponse::new_registered(
v4::registration::RegisteredData {
pub_key: self.keypair().public_key().into(),
private_ips: (allowed_ipv4, allowed_ipv6).into(),
wg_port: self.config.authenticator.tunnel_announced_port,
},
reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?,
request_id,
)
.to_bytes()
.map_err(AuthenticatorError::response_serialisation)?,
AuthenticatorVersion::V5 => v5::response::AuthenticatorResponse::new_registered(
v5::registration::RegisteredData {
pub_key: self.keypair().public_key().into(),
private_ips: (allowed_ipv4, allowed_ipv6).into(),
wg_port: self.config.authenticator.tunnel_announced_port,
},
request_id,
)
.to_bytes()
.map_err(AuthenticatorError::response_serialisation)?,
AuthenticatorVersion::V6 => v6::response::AuthenticatorResponse::new_registered(
v6::registration::RegisteredData {
pub_key: self.keypair().public_key().into(),
private_ips: (allowed_ipv4, allowed_ipv6).into(),
wg_port: self.config.authenticator.tunnel_announced_port,
},
request_id,
self.upgrade_mode_enabled(),
)
.to_bytes()
.map_err(AuthenticatorError::response_serialisation)?,
AuthenticatorVersion::UNKNOWN => return Err(AuthenticatorError::UnknownVersion),
};
return Ok((bytes, reply_to));
}
// mark it as used, even though it's not final
let ip_allocation = self.peer_manager.allocate_peer_ip_pair().await?;
self.registered_and_free
.write()
.await
.taken_private_network_ips
.insert(ip_allocation.into(), SystemTime::now());
let gateway_data = GatewayClient::new(
self.keypair().private_key(),
remote_public.inner(),
ip_allocation.into(),
nonce,
);
let registration_data = latest::registration::RegistrationData {
nonce,
gateway_data: gateway_data.clone(),
wg_port: self.config.authenticator.tunnel_announced_port,
};
registered_and_free
.registration_in_progress
.insert(remote_public, registration_data.clone());
let bytes = match AuthenticatorVersion::from(protocol) {
AuthenticatorVersion::V1 => {
v1::response::AuthenticatorResponse::new_pending_registration_success(
v1::registration::RegistrationData {
nonce: registration_data.nonce,
gateway_data: v1::registration::GatewayClient::new(
self.keypair().private_key(),
remote_public.inner(),
ip_allocation.ipv4.into(),
nonce,
),
wg_port: registration_data.wg_port,
},
request_id,
reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?,
)
.to_bytes()
.map_err(AuthenticatorError::response_serialisation)?
}
AuthenticatorVersion::V2 => {
v2::response::AuthenticatorResponse::new_pending_registration_success(
v2::registration::RegistrationData {
nonce: registration_data.nonce,
gateway_data: v2::registration::GatewayClient::new(
self.keypair().private_key(),
remote_public.inner(),
ip_allocation.ipv4.into(),
nonce,
),
wg_port: registration_data.wg_port,
},
request_id,
reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?,
)
.to_bytes()
.map_err(AuthenticatorError::response_serialisation)?
}
AuthenticatorVersion::V3 => {
v3::response::AuthenticatorResponse::new_pending_registration_success(
v3::registration::RegistrationData {
nonce: registration_data.nonce,
gateway_data: v3::registration::GatewayClient::new(
self.keypair().private_key(),
remote_public.inner(),
ip_allocation.ipv4.into(),
nonce,
),
wg_port: registration_data.wg_port,
},
request_id,
reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?,
)
.to_bytes()
.map_err(AuthenticatorError::response_serialisation)?
}
AuthenticatorVersion::V4 => {
v4::response::AuthenticatorResponse::new_pending_registration_success(
v4::registration::RegistrationData {
nonce: registration_data.nonce,
// convert current to v5 and then v5 to v4 (current as of 28.08.25)
gateway_data: v5::registration::GatewayClient::from(
registration_data.gateway_data.clone(),
)
.into(),
wg_port: registration_data.wg_port,
},
request_id,
reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?,
)
.to_bytes()
.map_err(AuthenticatorError::response_serialisation)?
}
AuthenticatorVersion::V5 => {
v5::response::AuthenticatorResponse::new_pending_registration_success(
v5::registration::RegistrationData {
nonce: registration_data.nonce,
gateway_data: registration_data.gateway_data.into(),
wg_port: registration_data.wg_port,
},
request_id,
)
.to_bytes()
.map_err(AuthenticatorError::response_serialisation)?
}
AuthenticatorVersion::V6 => {
v6::response::AuthenticatorResponse::new_pending_registration_success(
v6::registration::RegistrationData {
nonce: registration_data.nonce,
gateway_data: registration_data.gateway_data,
wg_port: registration_data.wg_port,
},
request_id,
self.upgrade_mode_enabled(),
)
.to_bytes()
.map_err(AuthenticatorError::response_serialisation)?
}
AuthenticatorVersion::UNKNOWN => return Err(AuthenticatorError::UnknownVersion),
};
Ok((bytes, reply_to))
}
async fn handle_final_credential_claim(
&self,
claim: BandwidthClaim,
client_id: i64,
) -> Result<(), AuthenticatorError> {
match claim.credential {
BandwidthCredential::ZkNym(zk_nym) => {
// if we got zk-nym, we just try to verify it
credential_verification(self.ecash_verifier.clone(), *zk_nym, client_id).await?;
Ok(())
}
BandwidthCredential::UpgradeModeJWT { token } => {
// if we're already in the upgrade mode, don't bother validating the token
if self.upgrade_mode_enabled() {
return Ok(());
}
self.upgrade_mode.try_enable_via_received_jwt(token).await?;
Ok(())
}
}
Ok((response.bytes, response.reply_to))
}
async fn on_final_request(
@@ -532,139 +110,12 @@ impl MixnetListener {
request_id: u64,
reply_to: Option<Recipient>,
) -> AuthenticatorHandleResult {
let mut registered_and_free = self.registered_and_free.write().await;
let registration_data = registered_and_free
.registration_in_progress
.get(&final_message.gateway_client_pub_key())
.ok_or(AuthenticatorError::RegistrationNotInProgress)?
.clone();
if final_message
.verify(self.keypair().private_key(), registration_data.nonce)
.is_err()
{
return Err(AuthenticatorError::MacVerificationFailure);
}
let mut peer = Peer::new(Key::new(final_message.gateway_client_pub_key().to_bytes()));
peer.allowed_ips
.push(IpAddrMask::new(final_message.private_ips().ipv4.into(), 32));
peer.allowed_ips.push(IpAddrMask::new(
final_message.private_ips().ipv6.into(),
128,
));
// ideally credential wouldn't have been required in upgrade mode,
// however, we need some basic information to insert valid wg peer
let Some(credential) = final_message.credential() else {
return Err(AuthenticatorError::NoCredentialReceived);
};
let typ = credential.kind;
let client_id = self
.ecash_verifier
.storage()
.insert_wireguard_peer(&peer, typ.into())
let response = self
.peer_registrator
.on_final_authenticator_request(final_message, protocol, request_id, reply_to)
.await?;
if let Err(err) = self
.handle_final_credential_claim(credential, client_id)
.await
{
self.ecash_verifier
.storage()
.remove_wireguard_peer(&peer.public_key.to_string())
.await?;
return Err(err);
}
let public_key = peer.public_key.to_string();
if let Err(e) = self.peer_manager.add_peer(peer).await {
self.ecash_verifier
.storage()
.remove_wireguard_peer(&public_key)
.await?;
return Err(e.into());
}
registered_and_free
.registration_in_progress
.remove(&final_message.gateway_client_pub_key());
let bytes = match AuthenticatorVersion::from(protocol) {
AuthenticatorVersion::V1 => v1::response::AuthenticatorResponse::new_registered(
v1::registration::RegisteredData {
pub_key: registration_data.gateway_data.pub_key,
private_ip: registration_data.gateway_data.private_ips.ipv4.into(),
wg_port: registration_data.wg_port,
},
reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?,
request_id,
)
.to_bytes()
.map_err(AuthenticatorError::response_serialisation)?,
AuthenticatorVersion::V2 => v2::response::AuthenticatorResponse::new_registered(
v2::registration::RegisteredData {
pub_key: registration_data.gateway_data.pub_key,
private_ip: registration_data.gateway_data.private_ips.ipv4.into(),
wg_port: registration_data.wg_port,
},
reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?,
request_id,
)
.to_bytes()
.map_err(AuthenticatorError::response_serialisation)?,
AuthenticatorVersion::V3 => v3::response::AuthenticatorResponse::new_registered(
v3::registration::RegisteredData {
pub_key: registration_data.gateway_data.pub_key,
private_ip: registration_data.gateway_data.private_ips.ipv4.into(),
wg_port: registration_data.wg_port,
},
reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?,
request_id,
)
.to_bytes()
.map_err(AuthenticatorError::response_serialisation)?,
AuthenticatorVersion::V4 => v4::response::AuthenticatorResponse::new_registered(
v4::registration::RegisteredData {
pub_key: registration_data.gateway_data.pub_key,
// convert current to v5 and then v5 to v4 (current as of 28.08.25)
private_ips: v5::registration::IpPair::from(
registration_data.gateway_data.private_ips,
)
.into(),
wg_port: registration_data.wg_port,
},
reply_to.ok_or(AuthenticatorError::MissingReplyToForOldClient)?,
request_id,
)
.to_bytes()
.map_err(AuthenticatorError::response_serialisation)?,
AuthenticatorVersion::V5 => v5::response::AuthenticatorResponse::new_registered(
v5::registration::RegisteredData {
pub_key: registration_data.gateway_data.pub_key,
private_ips: registration_data.gateway_data.private_ips.into(),
wg_port: registration_data.wg_port,
},
request_id,
)
.to_bytes()
.map_err(AuthenticatorError::response_serialisation)?,
AuthenticatorVersion::V6 => v6::response::AuthenticatorResponse::new_registered(
v6::registration::RegisteredData {
pub_key: registration_data.gateway_data.pub_key,
private_ips: registration_data.gateway_data.private_ips,
wg_port: registration_data.wg_port,
},
request_id,
self.upgrade_mode_enabled(),
)
.to_bytes()
.map_err(AuthenticatorError::response_serialisation)?,
AuthenticatorVersion::UNKNOWN => return Err(AuthenticatorError::UnknownVersion),
};
Ok((bytes, reply_to))
Ok((response.bytes, response.reply_to))
}
async fn on_query_bandwidth_request(
@@ -955,9 +406,6 @@ impl MixnetListener {
break;
},
_ = self.timeout_check_interval.next() => {
if let Err(e) = self.remove_stale_registrations().await {
tracing::error!("Could not clear stale registrations. The registration process might get jammed soon - {e:?}");
}
self.seen_credential_cache.remove_stale();
}
msg = self.mixnet_client.next() => {
@@ -987,45 +435,6 @@ impl MixnetListener {
}
}
pub async fn credential_storage_preparation(
ecash_verifier: Arc<dyn EcashManager + Send + Sync>,
client_id: i64,
) -> Result<PersistedBandwidth, AuthenticatorError> {
ecash_verifier
.storage()
.create_bandwidth_entry(client_id)
.await?;
let bandwidth = ecash_verifier
.storage()
.get_available_bandwidth(client_id)
.await?
.ok_or(AuthenticatorError::InternalError(
"bandwidth entry should have just been created".to_string(),
))?;
Ok(bandwidth)
}
async fn credential_verification(
ecash_verifier: Arc<dyn EcashManager + Send + Sync>,
credential: CredentialSpendingData,
client_id: i64,
) -> Result<i64, AuthenticatorError> {
let bandwidth = credential_storage_preparation(ecash_verifier.clone(), client_id).await?;
let client_bandwidth = ClientBandwidth::new(bandwidth.into());
let mut verifier = CredentialVerifier::new(
CredentialSpendingRequest::new(credential),
ecash_verifier.clone(),
BandwidthStorageManager::new(
ecash_verifier.storage(),
client_bandwidth,
client_id,
BandwidthFlushingBehaviourConfig::default(),
true,
),
);
Ok(verifier.verify().await?)
}
fn deserialize_request(
reconstructed: &ReconstructedMessage,
) -> Result<AuthenticatorRequest, AuthenticatorError> {
gateway/src/node/internal_service_providers/authenticator/mod.rs
+6 -7
View File
@@ -2,13 +2,14 @@
// SPDX-License-Identifier: Apache-2.0
use crate::node::internal_service_providers::authenticator::error::AuthenticatorError;
use crate::node::wireguard::PeerRegistrator;
use futures::channel::oneshot;
use nym_client_core::{HardcodedTopologyProvider, TopologyProvider};
use nym_credential_verification::upgrade_mode::UpgradeModeDetails;
use nym_sdk::{mixnet::Recipient, GatewayTransceiver};
use nym_task::ShutdownTracker;
use nym_wireguard::WireguardGatewayData;
use std::{path::Path, sync::Arc};
use std::path::Path;
pub use config::Config;
@@ -32,12 +33,12 @@ impl OnStartData {
pub struct Authenticator {
#[allow(unused)]
config: Config,
peer_registrator: PeerRegistrator,
upgrade_mode_state: UpgradeModeDetails,
wait_for_gateway: bool,
custom_topology_provider: Option<Box<dyn TopologyProvider + Send + Sync>>,
custom_gateway_transceiver: Option<Box<dyn GatewayTransceiver + Send + Sync>>,
wireguard_gateway_data: WireguardGatewayData,
ecash_verifier: Arc<dyn nym_credential_verification::ecash::traits::EcashManager + Send + Sync>,
shutdown: ShutdownTracker,
on_start: Option<oneshot::Sender<OnStartData>>,
}
@@ -45,20 +46,18 @@ pub struct Authenticator {
impl Authenticator {
pub fn new(
config: Config,
peer_registrator: PeerRegistrator,
upgrade_mode_state: UpgradeModeDetails,
wireguard_gateway_data: WireguardGatewayData,
ecash_verifier: Arc<
dyn nym_credential_verification::ecash::traits::EcashManager + Send + Sync,
>,
shutdown: ShutdownTracker,
) -> Self {
Self {
config,
peer_registrator,
upgrade_mode_state,
wait_for_gateway: false,
custom_topology_provider: None,
custom_gateway_transceiver: None,
ecash_verifier,
wireguard_gateway_data,
shutdown,
on_start: None,
@@ -134,8 +133,8 @@ impl Authenticator {
self.config,
self.wireguard_gateway_data,
mixnet_client,
self.peer_registrator,
self.upgrade_mode_state,
self.ecash_verifier,
);
tracing::info!("The address of this client is: {self_address}");
gateway/src/node/internal_service_providers/authenticator/seen_credential_cache.rs
+7 -31
View File
@@ -2,9 +2,9 @@
// SPDX-License-Identifier: Apache-2.0
#[cfg(test)]
use mock_instant::thread_local::SystemTime;
use mock_instant::thread_local::Instant;
#[cfg(not(test))]
use std::time::SystemTime;
use std::time::Instant;
use std::{collections::HashMap, time::Duration};
use nym_credentials_interface::CredentialSpendingData;
@@ -15,7 +15,7 @@ const SEEN_CREDENTIAL_CACHE_TIME: Duration = Duration::from_secs(60 * 60); // 1
#[derive(Eq, Hash, PartialEq)]
struct TimestampedPeerPubKey {
peer_pub_key: PeerPublicKey,
timestamp: SystemTime,
timestamp: Instant,
}
pub(crate) struct SeenCredentialCache {
@@ -36,7 +36,7 @@ impl SeenCredentialCache {
) {
let value = TimestampedPeerPubKey {
peer_pub_key,
timestamp: SystemTime::now(),
timestamp: Instant::now(),
};
self.cached_credentials
.insert(credential.serial_number_b58(), value);
@@ -52,12 +52,9 @@ impl SeenCredentialCache {
}
pub(crate) fn remove_stale(&mut self) {
let now = SystemTime::now();
let now = Instant::now();
self.cached_credentials.retain(|_, value| {
let Ok(cache_time) = now.duration_since(value.timestamp) else {
tracing::warn!("Got decreasing consecutive system timestamps");
return false;
};
let cache_time = now.duration_since(value.timestamp);
cache_time < SEEN_CREDENTIAL_CACHE_TIME
});
}
@@ -159,30 +156,9 @@ mod test {
cache.remove_stale();
assert!(cache.get_peer_pub_key(&credential).is_some());
MockClock::advance_system_time(SEEN_CREDENTIAL_CACHE_TIME * 2);
MockClock::advance(SEEN_CREDENTIAL_CACHE_TIME * 2);
cache.remove_stale();
assert!(cache.get_peer_pub_key(&credential).is_none());
}
#[test]
fn invalid_time() {
assert!(MockClock::is_thread_local());
assert!(SystemTime::now().is_thread_local());
let mut cache = SeenCredentialCache::new();
let credential = CredentialSpendingData::try_from_bytes(&CREDENTIAL_BYTES).unwrap();
let peer_pub_key = PeerPublicKey::from_str(PUB_KEY).unwrap();
// set some value for time
MockClock::set_system_time(Duration::from_secs(10));
cache.insert_credential(credential.clone(), peer_pub_key);
// then set the time in the past
MockClock::set_system_time(Duration::ZERO);
cache.remove_stale();
// invalid time should remove the credential, just in case
assert!(cache.get_peer_pub_key(&credential).is_none());
}
}
gateway/src/node/lp_listener/handler.rs
+1 -42
View File
@@ -1241,23 +1241,15 @@ where
mod tests {
use super::*;
use crate::node::lp_listener::{LpConfig, LpDebug};
use crate::node::wireguard::PeerManager;
use crate::node::ActiveClientsStore;
use bytes::BytesMut;
use nym_credential_verification::upgrade_mode::{
UpgradeModeCheckConfig, UpgradeModeCheckRequestSender, UpgradeModeDetails,
};
use nym_credential_verification::UpgradeModeState;
use nym_lp::codec::{parse_lp_packet, serialize_lp_packet};
use nym_lp::message::{ClientHelloData, EncryptedDataPayload, HandshakeData, LpMessage};
use nym_lp::packet::{LpHeader, LpPacket};
use nym_lp::peer::LpLocalPeer;
use nym_wireguard::{PeerControlRequest, WireguardConfig, WireguardGatewayData};
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt};
use tokio::sync::mpsc::Receiver;
// ==================== Test Helpers ====================
/// Create a minimal test state for handler tests
@@ -1265,23 +1257,6 @@ mod tests {
use nym_crypto::asymmetric::ed25519;
use rand::rngs::OsRng;
fn wireguard_data(
keys: Arc<x25519::KeyPair>,
) -> (WireguardGatewayData, Receiver<PeerControlRequest>) {
// some sensible default values (ports don't matter anyway)
let cfg = WireguardConfig {
bind_address: SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 51822),
private_ipv4: Ipv4Addr::new(10, 1, 0, 1),
private_ipv6: Ipv6Addr::new(0xfc01, 0, 0, 0, 0, 0, 0, 0x1), // fc01::1,
announced_tunnel_port: 51822,
announced_metadata_port: 51830,
private_network_prefix_v4: 16,
private_network_prefix_v6: 112,
};
WireguardGatewayData::new(cfg, keys)
}
// Create in-memory storage for testing
let storage = nym_gateway_storage::GatewayStorage::init(":memory:", 100)
.await
@@ -1308,20 +1283,6 @@ mod tests {
let id_keys = Arc::new(ed25519::KeyPair::new(&mut OsRng));
let x_keys = Arc::new(id_keys.to_x25519());
let (wireguard_data, _) = wireguard_data(x_keys.clone());
let (um_recheck_tx, _) = futures::channel::mpsc::unbounded();
let upgrade_mode_state = UpgradeModeState::new(*id_keys.public_key());
let upgrade_mode_details = UpgradeModeDetails::new(
UpgradeModeCheckConfig {
// essentially we never want to trigger this in our tests
min_staleness_recheck: Duration::from_nanos(1),
},
UpgradeModeCheckRequestSender::new(um_recheck_tx),
upgrade_mode_state.clone(),
);
let lp_peer = LpLocalPeer::new(id_keys, x_keys.clone()).with_kem_psq_key(x_keys);
LpHandlerState {
@@ -1332,13 +1293,11 @@ mod tests {
local_lp_peer: lp_peer,
metrics: nym_node_metrics::NymNodeMetrics::default(),
active_clients_store: ActiveClientsStore::new(),
upgrade_mode: upgrade_mode_details,
outbound_mix_sender: mix_sender,
handshake_states: Arc::new(dashmap::DashMap::new()),
session_states: Arc::new(dashmap::DashMap::new()),
registrations_in_progress: Default::default(),
forward_semaphore,
peer_manager: Arc::new(PeerManager::new(wireguard_data)),
peer_registrator: None,
}
}
gateway/src/node/lp_listener/mod.rs
+9 -69
View File
@@ -68,12 +68,11 @@
// They can be exported via Prometheus format using the metrics endpoint.
use crate::error::GatewayError;
use crate::node::lp_listener::registration::RegistrationsInProgress;
use crate::node::wireguard::PeerRegistrator;
use crate::node::ActiveClientsStore;
use dashmap::DashMap;
use nym_config::serde_helpers::de_maybe_port;
use nym_credential_verification::ecash::traits::EcashManager;
use nym_credential_verification::upgrade_mode::UpgradeModeDetails;
use nym_gateway_storage::GatewayStorage;
use nym_lp::state_machine::LpStateMachine;
use nym_node_metrics::NymNodeMetrics;
@@ -85,7 +84,6 @@ use tokio::net::TcpListener;
use tokio::sync::Semaphore;
use tracing::*;
use crate::node::wireguard::PeerManager;
pub use nym_lp::peer::LpLocalPeer;
pub use nym_mixnet_client::forwarder::{
mix_forwarding_channels, MixForwardingReceiver, MixForwardingSender,
@@ -184,10 +182,6 @@ pub struct LpDebug {
#[serde(with = "humantime_serde")]
pub demoted_session_ttl: Duration,
/// Maximum age of in-progress dVPN registration before cleanup (default: 60s)
#[serde(with = "humantime_serde")]
pub pending_registration_ttl: Duration,
/// How often to run the state cleanup task (default: 5 minutes)
///
/// The cleanup task scans for and removes stale handshakes and sessions.
@@ -257,9 +251,6 @@ impl LpDebug {
// 5 minutes - balances memory reclamation with task overhead
pub const DEFAULT_STATE_CLEANUP_INTERVAL: Duration = Duration::from_secs(300);
// 1 minute - enough for client to send retrieve credential from its storage and send it across
pub const DEFAULT_PENDING_REGISTRATION_TTL: Duration = Duration::from_secs(60);
// Limits concurrent outbound connections to prevent fd exhaustion
pub const DEFAULT_MAX_CONCURRENT_FORWARDS: usize = 1000;
}
@@ -273,7 +264,6 @@ impl Default for LpDebug {
handshake_ttl: Self::DEFAULT_HANDSHAKE_TTL,
session_ttl: Self::DEFAULT_SESSION_TTL,
demoted_session_ttl: Self::DEFAULT_DEMOTED_SESSION_TTL,
pending_registration_ttl: Self::DEFAULT_PENDING_REGISTRATION_TTL,
state_cleanup_interval: Self::DEFAULT_STATE_CLEANUP_INTERVAL,
max_concurrent_forwards: Self::DEFAULT_MAX_CONCURRENT_FORWARDS,
}
@@ -360,12 +350,8 @@ pub struct LpHandlerState {
/// Active clients tracking
pub active_clients_store: ActiveClientsStore,
/// Current state of the Upgrade Mode as perceived by this gateway
pub upgrade_mode: UpgradeModeDetails,
/// WireGuard gateway data (contains keypair and config)
/// alongside helpers for managing peers
pub peer_manager: Arc<PeerManager>,
/// Handle registering new wireguard peers
pub peer_registrator: Option<PeerRegistrator>,
/// LP configuration (for timestamp validation, etc.)
pub lp_config: LpConfig,
@@ -399,10 +385,6 @@ pub struct LpHandlerState {
/// to rekey without re-authentication.
pub session_states: Arc<DashMap<ReceiverIndex, TimestampedState<LpStateMachine>>>,
/// In-progress dVPN registrations that require additional data (e.g. credentials)
/// to finalise.
pub registrations_in_progress: RegistrationsInProgress,
/// Semaphore limiting concurrent forward connections
///
/// Prevents file descriptor exhaustion when forwarding LP packets during
@@ -577,31 +559,26 @@ impl LpListener {
///
/// The task automatically stops when the shutdown signal is received.
fn spawn_state_cleanup_task(&self) -> tokio::task::JoinHandle<()> {
let peer_manager = Arc::clone(&self.handler_state.peer_manager);
let handshake_states = Arc::clone(&self.handler_state.handshake_states);
let session_states = Arc::clone(&self.handler_state.session_states);
let pending_registrations = self.handler_state.registrations_in_progress.clone();
let dbg_cfg = self.handler_state.lp_config.debug;
let handshake_ttl = dbg_cfg.handshake_ttl;
let session_ttl = dbg_cfg.session_ttl;
let demoted_session_ttl = dbg_cfg.demoted_session_ttl;
let pending_reg_ttl = dbg_cfg.pending_registration_ttl;
let interval = dbg_cfg.state_cleanup_interval;
let shutdown = self.shutdown.clone_shutdown_token();
let metrics = self.handler_state.metrics.clone();
info!(
"Starting LP state cleanup task (handshake_ttl={}s, session_ttl={}s, demoted_ttl={}s, reg_ttl={}s, interval={}s)",
handshake_ttl.as_secs(), session_ttl.as_secs(), demoted_session_ttl.as_secs(),pending_reg_ttl.as_secs(), interval.as_secs()
"Starting LP state cleanup task (handshake_ttl={}s, session_ttl={}s, demoted_ttl={}s, interval={}s)",
handshake_ttl.as_secs(), session_ttl.as_secs(), demoted_session_ttl.as_secs(), interval.as_secs()
);
self.shutdown.try_spawn_named(
cleanup_task::cleanup_loop(
peer_manager,
handshake_states,
session_states,
pending_registrations,
dbg_cfg,
shutdown,
metrics,
@@ -619,33 +596,27 @@ impl LpListener {
}
pub(crate) mod cleanup_task {
use crate::node::lp_listener::registration::RegistrationsInProgress;
use crate::node::lp_listener::{LpDebug, TimestampedState};
use crate::node::wireguard::PeerManager;
use dashmap::DashMap;
use nym_lp::state_machine::LpStateBare;
use nym_lp::LpStateMachine;
use nym_metrics::inc_by;
use nym_node_metrics::NymNodeMetrics;
use std::sync::Arc;
use tracing::{debug, error, info};
use tracing::{debug, info};
async fn perform_cleanup(
peer_manager: &PeerManager,
handshake_states: &Arc<DashMap<u32, TimestampedState<LpStateMachine>>>,
session_states: &Arc<DashMap<u32, TimestampedState<LpStateMachine>>>,
registrations_in_progress: &RegistrationsInProgress,
cfg: LpDebug,
) {
let handshake_ttl = cfg.handshake_ttl;
let session_ttl = cfg.session_ttl;
let demoted_session_ttl = cfg.demoted_session_ttl;
let pending_registration_ttl = cfg.pending_registration_ttl;
let start = std::time::Instant::now();
let mut hs_removed = 0u64;
let mut ss_removed = 0u64;
let mut pending_reg_removed = 0u64;
let mut demoted_removed = 0u64;
// Remove stale handshakes (based on age since creation)
@@ -680,33 +651,10 @@ pub(crate) mod cleanup_task {
}
});
// Remove stale registrations (based on time since last activity)
let mut reg_guard = registrations_in_progress.lock().await;
let mut stale_registrations = Vec::new();
for (k, timestamped) in reg_guard.iter() {
if timestamped.age() > pending_registration_ttl {
stale_registrations.push(*k)
}
}
for to_remove in stale_registrations {
pending_reg_removed += 1;
// SAFETY: we never dropped the guard and the entry existed
#[allow(clippy::unwrap_used)]
let entry = reg_guard.remove(&to_remove).unwrap();
if let Err(err) = peer_manager
.release_ip_pair(entry.state.allocated_ip_pair())
.await
{
error!("failed to release allocated ip pair: {err}")
}
}
if hs_removed > 0 || ss_removed > 0 || demoted_removed > 0 || pending_reg_removed > 0 {
if hs_removed > 0 || ss_removed > 0 || demoted_removed > 0 {
let duration = start.elapsed();
info!(
"LP state cleanup: removed {hs_removed} handshakes, {pending_reg_removed} pending registrations, {ss_removed} sessions, {demoted_removed} demoted (took {:.3}s)",
"LP state cleanup: removed {hs_removed} handshakes, {ss_removed} sessions, {demoted_removed} demoted (took {:.3}s)",
duration.as_secs_f64()
);
@@ -720,12 +668,6 @@ pub(crate) mod cleanup_task {
if demoted_removed > 0 {
inc_by!("lp_states_cleanup_demoted_removed", demoted_removed as i64);
}
if pending_reg_removed > 0 {
inc_by!(
"lp_states_cleanup_pending_registrations_removed",
pending_reg_removed as i64
);
}
}
}
@@ -737,10 +679,8 @@ pub(crate) mod cleanup_task {
/// Demoted sessions (ReadOnlyTransport) use shorter TTL since they
/// only need to drain in-flight packets after subsession promotion.
pub(crate) async fn cleanup_loop(
peer_manager: Arc<PeerManager>,
handshake_states: Arc<DashMap<u32, TimestampedState<LpStateMachine>>>,
session_states: Arc<DashMap<u32, TimestampedState<LpStateMachine>>>,
registrations_in_progress: RegistrationsInProgress,
cfg: LpDebug,
shutdown: nym_task::ShutdownToken,
_metrics: NymNodeMetrics,
@@ -757,7 +697,7 @@ pub(crate) mod cleanup_task {
break;
}
_ = cleanup_interval.tick() => {
perform_cleanup(&peer_manager, &handshake_states, &session_states, &registrations_in_progress, cfg).await;
perform_cleanup(&handshake_states, &session_states, cfg).await;
}
}
}
gateway/src/node/lp_listener/registration.rs
+24 -442
View File
@@ -1,22 +1,8 @@
// Copyright 2025 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: GPL-3.0-only
use super::{LpHandlerState, ReceiverIndex, TimestampedState};
use crate::error::GatewayError;
use defguard_wireguard_rs::host::Peer;
use defguard_wireguard_rs::key::Key;
use nym_authenticator_requests::models::BandwidthClaim;
use nym_credential_verification::ecash::traits::EcashManager;
use nym_credential_verification::{
bandwidth_storage_manager::BandwidthStorageManager, BandwidthFlushingBehaviourConfig,
ClientBandwidth, CredentialVerifier,
};
use nym_credentials_interface::{BandwidthCredential, CredentialSpendingData, TicketType};
use nym_crypto::asymmetric::encryption::KeyPair;
use nym_gateway_requests::models::CredentialSpendingRequest;
use nym_gateway_storage::models::PersistedBandwidth;
use nym_gateway_storage::traits::BandwidthGatewayStorage;
use nym_metrics::{add_histogram_obs, inc, inc_by};
use crate::node::lp_listener::{LpHandlerState, ReceiverIndex};
use nym_metrics::{add_histogram_obs, inc};
use nym_registration_common::dvpn::{
LpDvpnRegistrationFinalisation, LpDvpnRegistrationInitialRequest,
LpDvpnRegistrationRequestMessage, LpDvpnRegistrationRequestMessageContent,
@@ -24,15 +10,8 @@ use nym_registration_common::dvpn::{
use nym_registration_common::mixnet::LpMixnetRegistrationRequestMessage;
use nym_registration_common::{
LpRegistrationRequest, LpRegistrationRequestData, LpRegistrationResponse, RegistrationMode,
RegistrationStatus, WireguardConfiguration,
RegistrationStatus,
};
use nym_wireguard::peer_controller::IpPair;
use nym_wireguard::WireguardConfig;
use nym_wireguard_types::PeerPublicKey;
use std::collections::HashMap;
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
use std::sync::Arc;
use tokio::sync::{Mutex, MutexGuard};
use tracing::*;
// Histogram buckets for LP registration duration tracking
@@ -49,225 +28,28 @@ const LP_REGISTRATION_DURATION_BUCKETS: &[f64] = &[
30.0, // 30s
];
// Histogram buckets for WireGuard peer controller channel latency
// Measures time to send request and receive response from peer controller
// Expected: 1ms-100ms for normal operations, up to 2s for slow conditions
const WG_CONTROLLER_LATENCY_BUCKETS: &[f64] = &[
0.001, // 1ms
0.005, // 5ms
0.01, // 10ms
0.05, // 50ms
0.1, // 100ms
0.25, // 250ms
0.5, // 500ms
1.0, // 1s
2.0, // 2s
];
#[derive(Clone, Copy)]
pub struct PendingRegistrationState {
client_id: i64,
peer_key: PeerPublicKey,
ticket_type: TicketType,
wireguard_config: WireguardConfiguration,
}
impl PendingRegistrationState {
pub(crate) fn allocated_ip_pair(&self) -> IpPair {
IpPair::new(
self.wireguard_config.private_ipv4,
self.wireguard_config.private_ipv6,
)
}
}
#[derive(Clone, Default)]
pub struct RegistrationsInProgress {
/// Wrapped in TimestampedState for TTL-based cleanup of stale data.
inner: Arc<Mutex<HashMap<ReceiverIndex, TimestampedState<PendingRegistrationState>>>>,
}
impl RegistrationsInProgress {
pub async fn lock(
&self,
) -> MutexGuard<'_, HashMap<ReceiverIndex, TimestampedState<PendingRegistrationState>>> {
self.inner.lock().await
}
}
impl LpHandlerState {
fn upgrade_mode_enabled(&self) -> bool {
self.upgrade_mode.enabled()
}
fn keypair(&self) -> &Arc<KeyPair> {
self.peer_manager.wireguard_gateway_data.keypair()
}
fn wireguard_config(&self) -> WireguardConfig {
self.peer_manager.wireguard_gateway_data.config()
}
fn successful_dvpn_registration(
&self,
peer_private_ipv4: Ipv4Addr,
peer_private_ipv6: Ipv6Addr,
bandwidth: i64,
) -> LpRegistrationResponse {
LpRegistrationResponse::success_dvpn(
WireguardConfiguration {
public_key: *self.keypair().public_key(),
psk: None,
// TODO: according to @SW this is most likely very wrong
endpoint: self.wireguard_config().bind_address,
private_ipv4: peer_private_ipv4,
private_ipv6: peer_private_ipv6,
},
bandwidth,
)
}
/// Check if WG peer already registered, return cached response if so.
///
/// This enables idempotent registration: if a client retries registration
/// with the same WG public key (e.g., after network failure), we return
/// the existing registration data instead of re-processing. This prevents
/// wasting credentials on network issues.
async fn check_existing_dvpn_registration(
&self,
public_key: PeerPublicKey,
) -> Option<LpRegistrationResponse> {
// Look up existing peer
let Ok(maybe_peer) = self.peer_manager.query_peer(public_key).await else {
return Some(LpRegistrationResponse::error(
"iternal failure: failed to resolve peer information",
RegistrationMode::Dvpn,
));
};
let peer = maybe_peer?;
// Extract IPv4 and IPv6 from allowed_ips
let mut private_ipv4 = None;
let mut private_ipv6 = None;
for ip_mask in &peer.allowed_ips {
match ip_mask.address {
IpAddr::V4(v4) => private_ipv4 = Some(v4),
IpAddr::V6(v6) => private_ipv6 = Some(v6),
}
if private_ipv4.is_some() && private_ipv6.is_some() {
break;
}
}
// Incomplete data, treat as new registration
let (Some(private_ipv4), Some(private_ipv6)) = (private_ipv4, private_ipv6) else {
return None;
};
// Get current bandwidth
let Ok(bandwidth) = self.peer_manager.query_client_bandwidth(public_key).await else {
return Some(LpRegistrationResponse::error(
"iternal failure: failed to resolve peer bandwidth",
RegistrationMode::Dvpn,
));
};
Some(self.successful_dvpn_registration(
private_ipv4,
private_ipv6,
bandwidth.available().await,
))
}
/// In the case of an already registered WG peer, update its PSK.
async fn update_peer_psk(&self, peer: PeerPublicKey, psk: Key) -> Result<(), GatewayError> {
let encoded_psk = psk.to_lower_hex();
self.storage
.update_peer_psk(&peer.to_string(), Some(&encoded_psk))
.await?;
// TODO: do we have to go through a peer manager to also update PSK if a peer is currently active?
// seems like an edge case. maybe we should force disconnect here?
Ok(())
}
async fn process_dvpn_initial_registration(
&self,
sender: ReceiverIndex,
request: LpDvpnRegistrationInitialRequest,
) -> LpRegistrationResponse {
let wg_key_str = request.wg_public_key.to_string();
// check for an existing registration (same WG key already registered)
// This allows clients to retry registration after network failures
// or to re-use gateway without spending additional bandwidth
if let Some(existing_response) = self
.check_existing_dvpn_registration(request.wg_public_key)
.await
{
// if there already exists registration for this client, update the psk and return the peer data
if let Err(err) = self
.update_peer_psk(request.wg_public_key, Key::new(request.psk))
.await
{
return LpRegistrationResponse::error(
format!("WireGuard peer PSK update failed: {err}"),
RegistrationMode::Dvpn,
);
}
info!("LP dVPN re-registration for existing peer {wg_key_str} (idempotent)",);
inc!("lp_registration_dvpn_idempotent");
return existing_response;
}
// TODO: this could be a source of some issue as we pre-allocate ip before validating credentials
// (but we do the same in the authenticator anyway...)
if let Err(err) = self
.register_wg_peer(
sender,
request.wg_public_key,
request.ticket_type,
Key::new(request.psk),
)
.await
{
let Some(registrator) = self.peer_registrator.as_ref() else {
return LpRegistrationResponse::error(
format!("WireGuard peer IP allocation failed: {err}"),
"dVPN via LP is not enabled on this node",
RegistrationMode::Dvpn,
);
}
};
LpRegistrationResponse::request_dvpn_credential()
}
// TODO: dedup
async fn handle_final_credential_claim(
&self,
claim: BandwidthClaim,
client_id: i64,
) -> Result<i64, GatewayError> {
match claim.credential {
BandwidthCredential::ZkNym(zk_nym) => {
// if we got zk-nym, we just try to verify it
let bandwidth =
credential_verification(self.ecash_verifier.clone(), *zk_nym, client_id)
.await?;
Ok(bandwidth)
}
BandwidthCredential::UpgradeModeJWT { token } => {
// TODO: move
const UM_BANDWIDTH: i64 = 1024 * 1024 * 1024;
// if we're already in the upgrade mode, don't bother validating the token
if self.upgrade_mode_enabled() {
return Ok(UM_BANDWIDTH);
}
self.upgrade_mode.try_enable_via_received_jwt(token).await?;
Ok(UM_BANDWIDTH)
}
}
registrator
.on_initial_lp_request(request, sender)
.await
.unwrap_or_else(|err| {
LpRegistrationResponse::error(
format!("LP registration has failed: {err}"),
RegistrationMode::Dvpn,
)
})
}
async fn process_dvpn_registration_finalisation(
@@ -275,63 +57,22 @@ impl LpHandlerState {
sender: ReceiverIndex,
request: LpDvpnRegistrationFinalisation,
) -> LpRegistrationResponse {
// see if we still have the pending registration
// (e.g. it's illegal for client to request registration and only finalise it,
// for example the next day; we can't keep the data forever)
let Some(pending) = self
.registrations_in_progress
.lock()
.await
.get(&sender)
.map(|pending| pending.state)
else {
let Some(registrator) = self.peer_registrator.as_ref() else {
return LpRegistrationResponse::error(
"no pending registration",
"dVPN via LP is not enabled on this node",
RegistrationMode::Dvpn,
);
};
if pending.ticket_type != request.credential.kind {
return LpRegistrationResponse::error(
format!(
"inconsistent ticket type. used {} for initial request and {} for finalisation",
pending.ticket_type, request.credential.kind
),
RegistrationMode::Dvpn,
);
}
let client_id = pending.client_id;
let allocated_bandwidth = match self
.handle_final_credential_claim(request.credential, client_id)
registrator
.on_final_lp_request(request, sender)
.await
{
Ok(bandwidth) => bandwidth,
Err(err) => {
// Credential verification failed, remove the peer
warn!("LP credential verification failed for client {client_id}: {err}");
inc!("lp_registration_dvpn_failed");
if let Err(remove_err) = self
.storage
.remove_wireguard_peer(&pending.peer_key.to_string())
.await
{
error!(
"Failed to remove peer after credential verification failure: {remove_err}"
);
}
self.registrations_in_progress.lock().await.remove(&sender);
return LpRegistrationResponse::error(
format!("Credential verification failed: {err}"),
.unwrap_or_else(|err| {
LpRegistrationResponse::error(
format!("LP registration has failed: {err}"),
RegistrationMode::Dvpn,
);
}
};
info!("LP dVPN registration successful (client_id: {client_id})");
inc!("lp_registration_dvpn_success");
LpRegistrationResponse::success_dvpn(pending.wireguard_config, allocated_bandwidth)
)
})
}
async fn process_dvpn_registration(
@@ -415,163 +156,4 @@ impl LpHandlerState {
result
}
/// Register a WireGuard peer and return gateway data along with the client_id
async fn register_wg_peer(
&self,
sender: ReceiverIndex,
peer_key: PeerPublicKey,
ticket_type: nym_credentials_interface::TicketType,
psk: Key,
) -> Result<(), GatewayError> {
// Allocate IPs from centralized pool managed by PeerController
let defguard_key = Key::new(peer_key.to_bytes());
// Request IP allocation from PeerController
let ip_pair = self.peer_manager.allocate_peer_ip_pair().await?;
let client_ipv4 = ip_pair.ipv4;
let client_ipv6 = ip_pair.ipv6;
info!("Allocated IPs for peer {peer_key}: {client_ipv4} / {client_ipv6}");
// Create WireGuard peer with allocated IPs
let mut peer = Peer::new(defguard_key);
peer.endpoint = None;
peer.allowed_ips = vec![
format!("{client_ipv4}/32").parse()?,
format!("{client_ipv6}/128").parse()?,
];
peer.persistent_keepalive_interval = Some(25);
peer.preshared_key = Some(psk);
// Store peer in database FIRST (before adding to controller)
// This ensures bandwidth storage exists when controller's generate_bandwidth_manager() is called
let client_id = self
.storage
.insert_wireguard_peer(&peer, ticket_type.into())
.await
.map_err(|e| {
error!("Failed to store WireGuard peer in database: {}", e);
GatewayError::InternalError(format!("Failed to store peer: {}", e))
})?;
// Create bandwidth entry for the client
// This must happen BEFORE AddPeer because generate_bandwidth_manager() expects it to exist
credential_storage_preparation(self.ecash_verifier.clone(), client_id).await?;
// Now send peer to WireGuard controller and track latency
let controller_start = std::time::Instant::now();
let result = self.peer_manager.add_peer(peer).await;
// Record peer controller channel latency
let latency = controller_start.elapsed().as_secs_f64();
add_histogram_obs!(
"wg_peer_controller_channel_latency_seconds",
latency,
WG_CONTROLLER_LATENCY_BUCKETS
);
result?;
// Get gateway's actual WireGuard public key
let gateway_pubkey = *self.keypair().public_key();
// Get gateway's WireGuard endpoint from config
let gateway_endpoint = self.wireguard_config().bind_address;
self.registrations_in_progress.lock().await.insert(
sender,
TimestampedState::new(PendingRegistrationState {
client_id,
peer_key,
ticket_type,
wireguard_config: WireguardConfiguration {
public_key: gateway_pubkey,
psk: None,
endpoint: gateway_endpoint,
private_ipv4: client_ipv4,
private_ipv6: client_ipv6,
},
}),
);
Ok(())
}
}
// TODO: dedup
/// Prepare bandwidth storage for a client
async fn credential_storage_preparation(
ecash_verifier: Arc<dyn EcashManager + Send + Sync>,
client_id: i64,
) -> Result<PersistedBandwidth, GatewayError> {
// Check if bandwidth entry already exists (idempotent)
let existing_bandwidth = ecash_verifier
.storage()
.get_available_bandwidth(client_id)
.await?;
// Only create if it doesn't exist
if existing_bandwidth.is_none() {
ecash_verifier
.storage()
.create_bandwidth_entry(client_id)
.await?;
}
let bandwidth = ecash_verifier
.storage()
.get_available_bandwidth(client_id)
.await?
.ok_or_else(|| GatewayError::InternalError("bandwidth entry should exist".to_string()))?;
Ok(bandwidth)
}
// TODO: dedup
/// Verify credential and allocate bandwidth using CredentialVerifier
async fn credential_verification(
ecash_verifier: Arc<dyn EcashManager + Send + Sync>,
credential: CredentialSpendingData,
client_id: i64,
) -> Result<i64, GatewayError> {
let bandwidth = credential_storage_preparation(ecash_verifier.clone(), client_id).await?;
let client_bandwidth = ClientBandwidth::new(bandwidth.into());
let mut verifier = CredentialVerifier::new(
CredentialSpendingRequest::new(credential),
ecash_verifier.clone(),
BandwidthStorageManager::new(
ecash_verifier.storage(),
client_bandwidth,
client_id,
BandwidthFlushingBehaviourConfig::default(),
true,
),
);
// Track credential verification attempts
inc!("lp_credential_verification_attempts");
// For mock ecash mode (local testing), skip cryptographic verification
// and just return a dummy bandwidth value since we don't have blockchain access
let allocated = if ecash_verifier.is_mock() {
// Return a reasonable test bandwidth value (e.g., 1GB in bytes)
const MOCK_BANDWIDTH: i64 = 1024 * 1024 * 1024;
inc!("lp_credential_verification_success");
inc_by!("lp_bandwidth_allocated_bytes_total", MOCK_BANDWIDTH);
Ok::<i64, GatewayError>(MOCK_BANDWIDTH)
} else {
match verifier.verify().await {
Ok(allocated) => {
inc!("lp_credential_verification_success");
// Track allocated bandwidth
inc_by!("lp_bandwidth_allocated_bytes_total", allocated);
Ok(allocated)
}
Err(e) => {
inc!("lp_credential_verification_failed");
Err(e.into())
}
}
}?;
Ok(allocated)
}
gateway/src/node/mod.rs
+21 -18
View File
@@ -38,7 +38,7 @@ use tracing::*;
use zeroize::Zeroizing;
pub use crate::node::upgrade_mode::watcher::UpgradeModeWatcher;
use crate::node::wireguard::PeerManager;
use crate::node::wireguard::{PeerManager, PeerRegistrator};
pub use client_handling::active_clients::ActiveClientsStore;
pub use lp_listener::LpConfig;
pub use nym_credential_verification::upgrade_mode::UpgradeModeCheckRequestSender;
@@ -299,6 +299,22 @@ impl GatewayTasksBuilder {
}
}
pub async fn build_peer_registrator(
&mut self,
upgrade_mode_details: UpgradeModeDetails,
) -> Result<Option<PeerRegistrator>, GatewayError> {
let Some(wireguard_data) = &self.wireguard_data else {
return Ok(None);
};
let peer_manager = PeerManager::new(wireguard_data.inner.clone());
Ok(Some(PeerRegistrator::new(
self.ecash_manager().await?,
peer_manager,
upgrade_mode_details,
)))
}
pub async fn build_websocket_listener(
&mut self,
active_clients_store: ActiveClientsStore,
@@ -330,19 +346,9 @@ impl GatewayTasksBuilder {
pub async fn build_lp_listener(
&mut self,
upgrade_mode_common_state: UpgradeModeDetails,
peer_registrator: Option<PeerRegistrator>,
active_clients_store: ActiveClientsStore,
) -> Result<lp_listener::LpListener, GatewayError> {
// Get WireGuard peer controller if available
let Some(wireguard_data) = &self.wireguard_data else {
return Err(GatewayError::InternalWireguardError(
"wireguard not set".to_string(),
));
};
// TODO: combine this `PeerManager` with the one used within the authenticator
let peer_manager = Arc::new(PeerManager::new(wireguard_data.inner.clone()));
let handler_state = lp_listener::LpHandlerState {
ecash_verifier: self.ecash_manager().await?,
storage: self.storage.clone(),
@@ -353,13 +359,11 @@ impl GatewayTasksBuilder {
.with_kem_psq_key(self.kem_psq_keys.clone()),
metrics: self.metrics.clone(),
active_clients_store,
upgrade_mode: upgrade_mode_common_state,
peer_manager,
peer_registrator,
lp_config: self.config.lp,
outbound_mix_sender: self.mix_packet_sender.clone(),
handshake_states: Arc::new(dashmap::DashMap::new()),
session_states: Arc::new(dashmap::DashMap::new()),
registrations_in_progress: Default::default(),
forward_semaphore: Arc::new(Semaphore::new(
self.config.lp.debug.max_concurrent_forwards,
)),
@@ -495,11 +499,10 @@ impl GatewayTasksBuilder {
pub async fn build_wireguard_authenticator(
&mut self,
peer_registrator: PeerRegistrator,
upgrade_mode_common: UpgradeModeDetails,
topology_provider: Box<dyn TopologyProvider + Send + Sync>,
) -> Result<ServiceProviderBeingBuilt<Authenticator>, GatewayError> {
let ecash_manager = self.ecash_manager().await?;
let Some(opts) = &self.authenticator_opts else {
return Err(GatewayError::UnspecifiedAuthenticatorConfig);
};
@@ -519,9 +522,9 @@ impl GatewayTasksBuilder {
let mut authenticator_server = Authenticator::new(
opts.config.clone(),
peer_registrator,
upgrade_mode_common,
wireguard_data.inner.clone(),
ecash_manager,
self.shutdown_tracker.clone(),
)
.with_custom_gateway_transceiver(transceiver)
gateway/src/node/wireguard/error.rs
+39
View File
@@ -1,6 +1,7 @@
// Copyright 2026 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use nym_credential_verification::upgrade_mode::UpgradeModeEnableError;
use thiserror::Error;
#[derive(Debug, Error)]
@@ -10,10 +11,48 @@ pub enum GatewayWireguardError {
#[error("peers can't be interacted with anymore")]
PeerInteractionStopped,
#[error("registration is not in progress for the provided peer key")]
RegistrationNotInProgress,
#[error("missing reply_to for old client")]
MissingReplyToForOldClient,
#[error("unknown version number")]
UnknownAuthenticatorVersion,
#[error("unsupported authenticator version")]
UnsupportedAuthenticatorVersion,
#[error("mac does not verify")]
AuthenticatorMacVerificationFailure,
#[error("no credential received")]
MissingAuthenticatorCredential,
#[error(transparent)]
UpgradeModeEnable(#[from] UpgradeModeEnableError),
#[error("credential verification failed: {0}")]
CredentialVerificationError(#[from] nym_credential_verification::Error),
#[error(transparent)]
GatewayStorageError(#[from] nym_gateway_storage::error::GatewayStorageError),
#[error("failed to serialise authenticator response packet: {source}")]
AuthenticatorResponseSerialisationFailure { source: Box<bincode::ErrorKind> },
}
impl GatewayWireguardError {
pub fn internal(message: impl Into<String>) -> Self {
GatewayWireguardError::InternalError(message.into())
}
pub fn authenticator_response_serialisation(
source: impl Into<Box<bincode::ErrorKind>>,
) -> Self {
GatewayWireguardError::AuthenticatorResponseSerialisationFailure {
source: source.into(),
}
}
}
gateway/src/node/wireguard/mod.rs
+2
View File
@@ -2,7 +2,9 @@
// SPDX-License-Identifier: Apache-2.0
pub mod error;
pub mod new_peer_registration;
pub mod peer_manager;
pub use error::GatewayWireguardError;
pub use new_peer_registration::PeerRegistrator;
pub use peer_manager::PeerManager;
gateway/src/node/wireguard/new_peer_registration/authenticator.rs
+157
View File
@@ -0,0 +1,157 @@
// Copyright 2026 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use crate::node::wireguard::new_peer_registration::helpers::build_final_authenticator_response;
use crate::node::wireguard::new_peer_registration::pending::{
PendingRegistration, PendingRegistrationData,
};
use crate::node::wireguard::{GatewayWireguardError, PeerRegistrator};
use defguard_wireguard_rs::host::Peer;
use nym_authenticator_requests::authenticator_ipv4_to_ipv6;
use nym_authenticator_requests::response::SerialisedResponse;
use nym_registration_common::WireguardRegistrationData;
use nym_sdk::mixnet::Recipient;
use nym_service_provider_requests_common::Protocol;
use nym_wireguard::peer_controller::IpPair;
use nym_wireguard_types::PeerPublicKey;
use std::net::IpAddr;
use std::time::Instant;
impl PeerRegistrator {
fn authenticator_peer_to_final_response(
&self,
peer: Peer,
protocol: Protocol,
request_id: u64,
reply_to: Option<Recipient>,
) -> Result<SerialisedResponse, GatewayWireguardError> {
let allowed_ipv4 = peer
.allowed_ips
.iter()
.find_map(|ip_mask| match ip_mask.address {
IpAddr::V4(ipv4_addr) => Some(ipv4_addr),
_ => None,
})
.ok_or(GatewayWireguardError::internal(
"there should be one private IPv4 in the list",
))?;
let allowed_ipv6 = peer
.allowed_ips
.iter()
.find_map(|ip_mask| match ip_mask.address {
IpAddr::V6(ipv6_addr) => Some(ipv6_addr),
_ => None,
})
.unwrap_or(authenticator_ipv4_to_ipv6(allowed_ipv4));
let ip_allocation = IpPair::new(allowed_ipv4, allowed_ipv6);
let wg_port = self.wg_port();
let local_pub_key = (*self.keypair().public_key()).into();
let upgrade_mode_enabled = self.upgrade_mode_enabled();
build_final_authenticator_response(
ip_allocation,
wg_port,
local_pub_key,
upgrade_mode_enabled,
request_id,
protocol.into(),
reply_to,
)
}
pub(super) async fn check_pending_authenticator_registration(
&self,
protocol: Protocol,
request_id: u64,
remote_public: PeerPublicKey,
reply_to: Option<Recipient>,
) -> Result<Option<SerialisedResponse>, GatewayWireguardError> {
let Some(pending_registration) = self
.pending_registrations
.check_authenticator(&remote_public)
.await
else {
return Ok(None);
};
Ok(Some(
pending_registration.to_pending_authenticator_response(
self.keypair().private_key(),
self.upgrade_mode_enabled(),
request_id,
protocol.into(),
reply_to,
)?,
))
}
pub(super) async fn check_existing_authenticator_peer(
&self,
protocol: Protocol,
request_id: u64,
remote_public: PeerPublicKey,
reply_to: Option<Recipient>,
) -> Result<Option<SerialisedResponse>, GatewayWireguardError> {
let Some(peer) = self.peer_manager.query_peer(remote_public).await? else {
return Ok(None);
};
Ok(Some(self.authenticator_peer_to_final_response(
peer, protocol, request_id, reply_to,
)?))
}
pub(super) fn new_pending_authenticator(
&self,
peer: PeerPublicKey,
ip_allocation: IpPair,
) -> PendingRegistration {
let nonce: u64 = fastrand::u64(..);
PendingRegistration {
requested_on: Instant::now(),
data: PendingRegistrationData {
nonce,
peer_key: peer,
psk: None,
wireguard_config: WireguardRegistrationData {
public_key: *self.keypair().public_key(),
port: self.wg_port(),
private_ipv4: ip_allocation.ipv4,
private_ipv6: ip_allocation.ipv6,
},
},
}
}
pub(super) async fn process_fresh_initial_authenticator_registration(
&self,
protocol: Protocol,
request_id: u64,
remote_public: PeerPublicKey,
reply_to: Option<Recipient>,
) -> Result<SerialisedResponse, GatewayWireguardError> {
// 1. allocate ip pair
let ip_allocation = self.peer_manager.preallocate_peer_ip_pair().await?;
let pending = self.new_pending_authenticator(remote_public, ip_allocation);
// 2. construct response
let response = pending.to_pending_authenticator_response(
self.keypair().private_key(),
self.upgrade_mode_enabled(),
request_id,
protocol.into(),
reply_to,
)?;
// 3. insert pending data into cache
self.pending_registrations
.authenticator
.write()
.await
.insert(remote_public, pending);
Ok(response)
}
}
gateway/src/node/wireguard/new_peer_registration/helpers.rs
+209
View File
@@ -0,0 +1,209 @@
// Copyright 2026 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use crate::node::wireguard::GatewayWireguardError;
use nym_authenticator_requests::response::SerialisedResponse;
use nym_authenticator_requests::{v1, v2, v3, v4, v5, v6, AuthenticatorVersion};
use nym_crypto::asymmetric::x25519;
use nym_sdk::mixnet::Recipient;
use nym_wireguard::ip_pool::IpPair;
use nym_wireguard_types::PeerPublicKey;
#[allow(clippy::too_many_arguments)]
pub(crate) fn build_pending_authenticator_response(
ip_allocation: IpPair,
wg_port: u16,
local_key: &x25519::PrivateKey,
peer_key: PeerPublicKey,
upgrade_mode_enabled: bool,
nonce: u64,
request_id: u64,
version: AuthenticatorVersion,
reply_to: Option<Recipient>,
) -> Result<SerialisedResponse, GatewayWireguardError> {
let private_ipv4 = ip_allocation.ipv4;
let private_ipv6 = ip_allocation.ipv6;
let bytes = match version {
AuthenticatorVersion::V1 => Err(GatewayWireguardError::UnsupportedAuthenticatorVersion),
AuthenticatorVersion::V2 => {
v2::response::AuthenticatorResponse::new_pending_registration_success(
v2::registration::RegistrationData {
nonce,
gateway_data: v2::registration::GatewayClient::new(
local_key,
peer_key.inner(),
private_ipv4.into(),
nonce,
),
wg_port,
},
request_id,
reply_to.ok_or(GatewayWireguardError::MissingReplyToForOldClient)?,
)
.to_bytes()
.map_err(GatewayWireguardError::authenticator_response_serialisation)
}
AuthenticatorVersion::V3 => {
v3::response::AuthenticatorResponse::new_pending_registration_success(
v3::registration::RegistrationData {
nonce,
gateway_data: v3::registration::GatewayClient::new(
local_key,
peer_key.inner(),
private_ipv4.into(),
nonce,
),
wg_port,
},
request_id,
reply_to.ok_or(GatewayWireguardError::MissingReplyToForOldClient)?,
)
.to_bytes()
.map_err(GatewayWireguardError::authenticator_response_serialisation)
}
AuthenticatorVersion::V4 => {
v4::response::AuthenticatorResponse::new_pending_registration_success(
v4::registration::RegistrationData {
nonce,
gateway_data: v4::registration::GatewayClient::new(
local_key,
peer_key.inner(),
v4::registration::IpPair::new(private_ipv4, private_ipv6),
nonce,
),
wg_port,
},
request_id,
reply_to.ok_or(GatewayWireguardError::MissingReplyToForOldClient)?,
)
.to_bytes()
.map_err(GatewayWireguardError::authenticator_response_serialisation)
}
AuthenticatorVersion::V5 => {
v5::response::AuthenticatorResponse::new_pending_registration_success(
v5::registration::RegistrationData {
nonce,
gateway_data: v5::registration::GatewayClient::new(
local_key,
peer_key.inner(),
v5::registration::IpPair::new(private_ipv4, private_ipv6),
nonce,
),
wg_port,
},
request_id,
)
.to_bytes()
.map_err(GatewayWireguardError::authenticator_response_serialisation)
}
AuthenticatorVersion::V6 => {
v6::response::AuthenticatorResponse::new_pending_registration_success(
v6::registration::RegistrationData {
nonce,
gateway_data: v6::registration::GatewayClient::new(
local_key,
peer_key.inner(),
v6::registration::IpPair::new(private_ipv4, private_ipv6),
nonce,
),
wg_port,
},
request_id,
upgrade_mode_enabled,
)
.to_bytes()
.map_err(GatewayWireguardError::authenticator_response_serialisation)
}
AuthenticatorVersion::UNKNOWN => {
return Err(GatewayWireguardError::UnknownAuthenticatorVersion)
}
}?;
Ok(nym_authenticator_requests::response::SerialisedResponse::new(bytes, reply_to))
}
pub(crate) fn build_final_authenticator_response(
ip_allocation: IpPair,
wg_port: u16,
pub_key: PeerPublicKey,
upgrade_mode_enabled: bool,
request_id: u64,
version: AuthenticatorVersion,
reply_to: Option<Recipient>,
) -> Result<SerialisedResponse, GatewayWireguardError> {
let private_ipv4 = ip_allocation.ipv4;
let private_ipv6 = ip_allocation.ipv6;
let bytes = match version {
AuthenticatorVersion::V1 => v1::response::AuthenticatorResponse::new_registered(
v1::registration::RegisteredData {
pub_key,
private_ip: private_ipv4.into(),
wg_port,
},
reply_to.ok_or(GatewayWireguardError::MissingReplyToForOldClient)?,
request_id,
)
.to_bytes()
.map_err(GatewayWireguardError::authenticator_response_serialisation)?,
AuthenticatorVersion::V2 => v2::response::AuthenticatorResponse::new_registered(
v2::registration::RegisteredData {
pub_key,
private_ip: private_ipv4.into(),
wg_port,
},
reply_to.ok_or(GatewayWireguardError::MissingReplyToForOldClient)?,
request_id,
)
.to_bytes()
.map_err(GatewayWireguardError::authenticator_response_serialisation)?,
AuthenticatorVersion::V3 => v3::response::AuthenticatorResponse::new_registered(
v3::registration::RegisteredData {
pub_key,
private_ip: private_ipv4.into(),
wg_port,
},
reply_to.ok_or(GatewayWireguardError::MissingReplyToForOldClient)?,
request_id,
)
.to_bytes()
.map_err(GatewayWireguardError::authenticator_response_serialisation)?,
AuthenticatorVersion::V4 => v4::response::AuthenticatorResponse::new_registered(
v4::registration::RegisteredData {
pub_key,
private_ips: v4::registration::IpPair::new(private_ipv4, private_ipv6),
wg_port,
},
reply_to.ok_or(GatewayWireguardError::MissingReplyToForOldClient)?,
request_id,
)
.to_bytes()
.map_err(GatewayWireguardError::authenticator_response_serialisation)?,
AuthenticatorVersion::V5 => v5::response::AuthenticatorResponse::new_registered(
v5::registration::RegisteredData {
pub_key,
private_ips: v5::registration::IpPair::new(private_ipv4, private_ipv6),
wg_port,
},
request_id,
)
.to_bytes()
.map_err(GatewayWireguardError::authenticator_response_serialisation)?,
AuthenticatorVersion::V6 => v6::response::AuthenticatorResponse::new_registered(
v6::registration::RegisteredData {
pub_key,
private_ips: v6::registration::IpPair::new(private_ipv4, private_ipv6),
wg_port,
},
request_id,
upgrade_mode_enabled,
)
.to_bytes()
.map_err(GatewayWireguardError::authenticator_response_serialisation)?,
AuthenticatorVersion::UNKNOWN => {
return Err(GatewayWireguardError::UnknownAuthenticatorVersion)
}
};
Ok(SerialisedResponse::new(bytes, reply_to))
}
gateway/src/node/wireguard/new_peer_registration/lp.rs
+128
View File
@@ -0,0 +1,128 @@
// Copyright 2026 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use crate::node::lp_listener::ReceiverIndex;
use crate::node::wireguard::new_peer_registration::pending::{
PendingRegistration, PendingRegistrationData,
};
use crate::node::wireguard::{GatewayWireguardError, PeerRegistrator};
use defguard_wireguard_rs::host::Peer;
use defguard_wireguard_rs::key::Key;
use nym_registration_common::{LpRegistrationResponse, WireguardRegistrationData};
use nym_wireguard::ip_pool::{allocated_ip_pair, IpPair};
use nym_wireguard_types::PeerPublicKey;
use std::time::Instant;
impl PeerRegistrator {
/// In the case of an already registered WG peer, update its PSK.
pub(super) async fn update_peer_psk(
&self,
peer: PeerPublicKey,
psk: Key,
) -> Result<(), GatewayWireguardError> {
// 1. check if the peer is currently being handled
if self.peer_manager.check_active_peer(peer).await? {
// 2. if so, force disconnect it (as we're handling new request from the same peer)
self.peer_manager.remove_peer(peer).await?;
}
// 3. update the on-disk PSK
let encoded_psk = psk.to_lower_hex();
self.ecash_verifier
.storage()
.update_peer_psk(&peer.to_string(), Some(&encoded_psk))
.await?;
Ok(())
}
fn lp_peer_to_final_response(
&self,
peer: Peer,
) -> Result<Option<LpRegistrationResponse>, GatewayWireguardError> {
// Incomplete data, treat as new registration
let Some(allocated_ips) = allocated_ip_pair(&peer) else {
return Ok(None);
};
Ok(Some(LpRegistrationResponse::success_dvpn(
WireguardRegistrationData {
public_key: *self.keypair().public_key(),
port: self.wg_port(),
private_ipv4: allocated_ips.ipv4,
private_ipv6: allocated_ips.ipv6,
},
self.upgrade_mode_enabled(),
)))
}
pub(super) async fn check_pending_lp_registration(
&self,
sender: ReceiverIndex,
) -> Result<Option<LpRegistrationResponse>, GatewayWireguardError> {
let Some(pending_registration) = self.pending_registrations.check_lp(sender).await else {
return Ok(None);
};
Ok(Some(pending_registration.to_pending_lp_response()))
}
pub(super) async fn check_existing_lp_peer(
&self,
remote_public: PeerPublicKey,
) -> Result<Option<LpRegistrationResponse>, GatewayWireguardError> {
let Some(peer) = self.peer_manager.query_peer(remote_public).await? else {
return Ok(None);
};
self.lp_peer_to_final_response(peer)
}
pub(super) fn new_pending_lp(
&self,
peer: PeerPublicKey,
psk: Key,
ip_allocation: IpPair,
) -> PendingRegistration {
let nonce: u64 = fastrand::u64(..);
PendingRegistration {
requested_on: Instant::now(),
data: PendingRegistrationData {
nonce,
peer_key: peer,
psk: Some(psk),
wireguard_config: WireguardRegistrationData {
public_key: *self.keypair().public_key(),
port: self.wg_port(),
private_ipv4: ip_allocation.ipv4,
private_ipv6: ip_allocation.ipv6,
},
},
}
}
pub(super) async fn process_fresh_initial_lp_registration(
&self,
sender: ReceiverIndex,
remote_public: PeerPublicKey,
psk: Key,
) -> Result<LpRegistrationResponse, GatewayWireguardError> {
// 1. allocate ip pair
let ip_allocation = self.peer_manager.preallocate_peer_ip_pair().await?;
let pending = self.new_pending_lp(remote_public, psk, ip_allocation);
// 2. construct response
let response = pending.to_pending_lp_response();
// 3. insert pending data into cache
self.pending_registrations
.lp
.write()
.await
.insert(sender, pending);
Ok(response)
}
}
gateway/src/node/wireguard/new_peer_registration/mod.rs
+391
View File
@@ -0,0 +1,391 @@
// Copyright 2026 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
//! Unification of Nym registration flow
//! In general the registration has the following structure:
//! 1. Initial request message is received
//! 1.1. We check if the peer has already registered before -> if so, we returned the past information
//! 1.2. We check if the peer already has a pending registration -> if so, we return the past information
//! 1.3. We pre-allocated [`nym_wireguard::ip_pool::IpPair`] and save time-sensitive pending registration.
//! If it does not complete within specified time interval, the information is going to get removed.
//! 2. Finalisation request message is received, where credential has to be attached is verified.
//! Upon successful completion, pending registration is transformed into a properly inserted peer.
use crate::node::lp_listener::ReceiverIndex;
use crate::node::wireguard::new_peer_registration::pending::{
PendingRegistration, PendingRegistrations,
};
use crate::node::wireguard::{GatewayWireguardError, PeerManager};
use defguard_wireguard_rs::host::Peer;
use defguard_wireguard_rs::key::Key;
use defguard_wireguard_rs::net::IpAddrMask;
use nym_authenticator_requests::models::BandwidthClaim;
use nym_authenticator_requests::response::SerialisedResponse;
use nym_authenticator_requests::traits::{FinalMessage, InitMessage};
use nym_credential_verification::bandwidth_storage_manager::BandwidthStorageManager;
use nym_credential_verification::ecash::traits::EcashManager;
use nym_credential_verification::upgrade_mode::UpgradeModeDetails;
use nym_credential_verification::{
BandwidthFlushingBehaviourConfig, ClientBandwidth, CredentialVerifier,
};
use nym_credentials_interface::{BandwidthCredential, CredentialSpendingData};
use nym_crypto::asymmetric::x25519;
use nym_gateway_requests::models::CredentialSpendingRequest;
use nym_gateway_storage::models::PersistedBandwidth;
use nym_registration_common::dvpn::{
LpDvpnRegistrationFinalisation, LpDvpnRegistrationInitialRequest,
};
use nym_registration_common::LpRegistrationResponse;
use nym_sdk::mixnet::Recipient;
use nym_service_provider_requests_common::Protocol;
use nym_task::ShutdownToken;
use nym_wireguard::WireguardConfig;
use std::sync::Arc;
use std::time::Duration;
use tokio::time::{interval_at, Instant};
use tracing::trace;
mod authenticator;
mod helpers;
mod lp;
mod pending;
#[derive(Clone)]
pub struct PeerRegistrator {
/// Handle for the structure managing verification of the ecash credentials for the bandwidth control
pub(crate) ecash_verifier: Arc<dyn EcashManager + Send + Sync>,
/// Handle for communication with the [`nym_wireguard::peer_controller::PeerController`]
pub(crate) peer_manager: PeerManager,
/// Information about the current state of the upgrade mode as well as a handle
/// to remotely trigger the recheck
pub(crate) upgrade_mode: UpgradeModeDetails,
/// Registrations in progress
pub(crate) pending_registrations: PendingRegistrations,
}
impl PeerRegistrator {
pub fn new(
ecash_verifier: Arc<dyn EcashManager + Send + Sync>,
peer_manager: PeerManager,
upgrade_mode: UpgradeModeDetails,
) -> Self {
PeerRegistrator {
ecash_verifier,
peer_manager,
upgrade_mode,
pending_registrations: Default::default(),
}
}
pub fn cleanup_task(&self, shutdown_token: ShutdownToken) -> StaleRegistrationRemover {
StaleRegistrationRemover {
pending_registrations: self.pending_registrations.clone(),
shutdown_token,
}
}
fn upgrade_mode_enabled(&self) -> bool {
self.upgrade_mode.enabled()
}
fn keypair(&self) -> &Arc<x25519::KeyPair> {
self.peer_manager.wireguard_gateway_data.keypair()
}
fn wireguard_config(&self) -> WireguardConfig {
self.peer_manager.wireguard_gateway_data.config()
}
fn wg_port(&self) -> u16 {
self.wireguard_config().announced_tunnel_port
}
pub async fn credential_storage_preparation(
&self,
client_id: i64,
) -> Result<PersistedBandwidth, GatewayWireguardError> {
self.ecash_verifier
.storage()
.create_bandwidth_entry(client_id)
.await?;
self.ecash_verifier
.storage()
.get_available_bandwidth(client_id)
.await?
.ok_or(GatewayWireguardError::internal(
"missing bandwidth entry after it has just been created",
))
}
async fn credential_verification(
&self,
credential: CredentialSpendingData,
client_id: i64,
) -> Result<i64, GatewayWireguardError> {
let bandwidth = self.credential_storage_preparation(client_id).await?;
let client_bandwidth = ClientBandwidth::new(bandwidth.into());
let mut verifier = CredentialVerifier::new(
CredentialSpendingRequest::new(credential),
self.ecash_verifier.clone(),
BandwidthStorageManager::new(
self.ecash_verifier.storage(),
client_bandwidth,
client_id,
BandwidthFlushingBehaviourConfig::default(),
true,
),
);
Ok(verifier.verify().await?)
}
async fn handle_final_credential_claim(
&self,
claim: BandwidthClaim,
client_id: i64,
) -> Result<(), GatewayWireguardError> {
match claim.credential {
BandwidthCredential::ZkNym(zk_nym) => {
// if we got zk-nym, we just try to verify it
self.credential_verification(*zk_nym, client_id).await?;
Ok(())
}
BandwidthCredential::UpgradeModeJWT { token } => {
// if we're already in the upgrade mode, don't bother validating the token
if self.upgrade_mode_enabled() {
return Ok(());
}
self.upgrade_mode.try_enable_via_received_jwt(token).await?;
Ok(())
}
}
}
/// Attempt to process new peer by:
/// 1. retrieving previous IP allocation
/// 2. inserting it into the storage
/// 3. verifying bandwidth claim and increasing the allowance
/// 4. spawning the peer handler
async fn process_new_peer(
&self,
pending: PendingRegistration,
credential: BandwidthClaim,
) -> Result<(), GatewayWireguardError> {
// 1. create peer based on the cached registration information
let defguard_key = Key::new(pending.data.peer_key.to_bytes());
let mut peer = Peer::new(defguard_key);
if let Some(psk) = pending.data.psk {
peer.preshared_key = Some(psk);
}
let private_ipv4 = pending.data.wireguard_config.private_ipv4;
let private_ipv6 = pending.data.wireguard_config.private_ipv6;
peer.allowed_ips = vec![
IpAddrMask::new(private_ipv4.into(), 32),
IpAddrMask::new(private_ipv6.into(), 128),
];
let typ = credential.kind;
// 2. attempt to pre-insert peer into the storage
let client_id = self
.ecash_verifier
.storage()
.insert_wireguard_peer(&peer, typ.into())
.await?;
// 3. verify the credential
if let Err(err) = self
.handle_final_credential_claim(credential, client_id)
.await
{
// 3.1. on failure -> remove the inserted peer
self.ecash_verifier
.storage()
.remove_wireguard_peer(&peer.public_key.to_string())
.await?;
return Err(err);
}
// 4. attempt to start the actual handle for the peer
let public_key = peer.public_key.to_string();
if let Err(err) = self.peer_manager.add_peer(peer).await {
// 4.1. on failure -> remove the inserted peer (from the storage)
self.ecash_verifier
.storage()
.remove_wireguard_peer(&public_key)
.await?;
return Err(err);
}
Ok(())
}
pub(crate) async fn on_initial_authenticator_request(
&mut self,
init_message: Box<dyn InitMessage + Send + Sync + 'static>,
protocol: Protocol,
request_id: u64,
reply_to: Option<Recipient>,
) -> Result<SerialisedResponse, GatewayWireguardError> {
let remote_public = init_message.pub_key();
// 1. check if there's any pending registration already in progress,
// if so, return the same data again without additional processing
if let Some(pending_registration) = self
.check_pending_authenticator_registration(protocol, request_id, remote_public, reply_to)
.await?
{
return Ok(pending_registration);
}
// 2. check if there is already a peer associated with this sender,
// if so, retrieve the "final" data without additional processing
if let Some(existing_registration) = self
.check_existing_authenticator_peer(protocol, request_id, remote_public, reply_to)
.await?
{
return Ok(existing_registration);
}
// 3. process fresh registration request
self.process_fresh_initial_authenticator_registration(
protocol,
request_id,
remote_public,
reply_to,
)
.await
}
pub(crate) async fn on_final_authenticator_request(
&mut self,
final_message: Box<dyn FinalMessage + Send + Sync + 'static>,
protocol: Protocol,
request_id: u64,
reply_to: Option<Recipient>,
) -> Result<SerialisedResponse, GatewayWireguardError> {
let peer = final_message.gateway_client_pub_key();
// 1. check if there's any pending registration associated with this peer
let pending_data = self
.pending_registrations
.check_authenticator(&peer)
.await
.ok_or(GatewayWireguardError::RegistrationNotInProgress)?
.clone();
// 2. verify the correctness of the received request based on the prior nonce
if final_message
.verify(self.keypair().private_key(), pending_data.data.nonce)
.is_err()
{
return Err(GatewayWireguardError::AuthenticatorMacVerificationFailure);
}
// 3. ensure we have received a credential
let Some(credential) = final_message.credential() else {
return Err(GatewayWireguardError::MissingAuthenticatorCredential);
};
// 4. prepare new peer information and verify the credential
self.process_new_peer(pending_data.clone(), credential)
.await?;
// 5. remove pending registration
self.pending_registrations.remove_authenticator(&peer).await;
// 6. construct and return the response
pending_data.to_registered_authenticator_response(
self.upgrade_mode_enabled(),
request_id,
protocol.into(),
reply_to,
)
}
pub(crate) async fn on_initial_lp_request(
&self,
init_msg: LpDvpnRegistrationInitialRequest,
sender: ReceiverIndex,
) -> Result<LpRegistrationResponse, GatewayWireguardError> {
let remote_public = init_msg.wg_public_key;
let psk = Key::new(init_msg.psk);
// 1. check if there's any pending registration already in progress,
// if so, return the same data again without additional processing,
// but update stored PSK
if let Some(pending_registration) = self.check_pending_lp_registration(sender).await? {
self.update_peer_psk(remote_public, psk).await?;
return Ok(pending_registration);
}
// 2. check if there is already a peer associated with this sender,
// if so, retrieve the "final" data without additional processing,
// but do update stored PSK
if let Some(existing_registration) = self.check_existing_lp_peer(remote_public).await? {
self.update_peer_psk(remote_public, psk).await?;
return Ok(existing_registration);
}
// 3. process fresh registration request
self.process_fresh_initial_lp_registration(sender, remote_public, psk)
.await
}
pub(crate) async fn on_final_lp_request(
&self,
final_msg: LpDvpnRegistrationFinalisation,
sender: ReceiverIndex,
) -> Result<LpRegistrationResponse, GatewayWireguardError> {
// 1. check if there's any pending registration associated with this peer
let pending_data = self
.pending_registrations
.check_lp(sender)
.await
.ok_or(GatewayWireguardError::RegistrationNotInProgress)?
.clone();
let credential = final_msg.credential;
// 2. prepare new peer information and verify the credential
self.process_new_peer(pending_data.clone(), credential)
.await?;
// 3 remove pending registration
self.pending_registrations.remove_lp(sender).await;
// 4. construct and return the response
Ok(pending_data.to_registered_lp_response(self.upgrade_mode_enabled()))
}
}
pub struct StaleRegistrationRemover {
pending_registrations: PendingRegistrations,
shutdown_token: ShutdownToken,
}
impl StaleRegistrationRemover {
// TODO: make it configurable
const STALE_REG_CHECK_INTERVAL: Duration = Duration::from_secs(60);
pub async fn run(&self) {
let start = Instant::now() + Self::STALE_REG_CHECK_INTERVAL;
let mut interval = interval_at(start, Self::STALE_REG_CHECK_INTERVAL);
loop {
tokio::select! {
biased;
_ = self.shutdown_token.cancelled() => {
trace!("StaleRegistrationRemover: received shutdown");
break
}
_ = interval.tick() => {
self.pending_registrations.remove_stale_registrations().await
}
}
}
}
}
gateway/src/node/wireguard/new_peer_registration/pending.rs
+158
View File
@@ -0,0 +1,158 @@
// Copyright 2026 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use crate::node::lp_listener::ReceiverIndex;
use crate::node::wireguard::new_peer_registration::helpers::{
build_final_authenticator_response, build_pending_authenticator_response,
};
use crate::node::wireguard::GatewayWireguardError;
use defguard_wireguard_rs::key::Key;
use nym_authenticator_requests::AuthenticatorVersion;
use nym_crypto::asymmetric::x25519;
use nym_registration_common::{LpRegistrationResponse, WireguardRegistrationData};
use nym_sdk::mixnet::Recipient;
use nym_wireguard::ip_pool::IpPair;
use nym_wireguard_types::PeerPublicKey;
use std::collections::HashMap;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::RwLock;
const DEFAULT_PENDING_REGISTRATION_TTL: Duration = Duration::from_secs(120); // 2 minutes
#[derive(Clone)]
pub struct PendingRegistration {
pub(super) requested_on: Instant,
pub(super) data: PendingRegistrationData,
}
#[derive(Clone)]
pub struct PendingRegistrationData {
pub(super) nonce: u64,
pub(super) peer_key: PeerPublicKey,
// will not be set if registering via the Authenticator
pub(super) psk: Option<Key>,
pub(super) wireguard_config: WireguardRegistrationData,
}
impl PendingRegistration {
pub(crate) fn to_pending_authenticator_response(
&self,
local_key: &x25519::PrivateKey,
upgrade_mode_enabled: bool,
request_id: u64,
version: AuthenticatorVersion,
reply_to: Option<Recipient>,
) -> Result<nym_authenticator_requests::response::SerialisedResponse, GatewayWireguardError>
{
let nonce = self.data.nonce;
let remote_public = self.data.peer_key;
let wg_port = self.data.wireguard_config.port;
let ip_allocation = IpPair::new(
self.data.wireguard_config.private_ipv4,
self.data.wireguard_config.private_ipv6,
);
build_pending_authenticator_response(
ip_allocation,
wg_port,
local_key,
remote_public,
upgrade_mode_enabled,
nonce,
request_id,
version,
reply_to,
)
}
pub(crate) fn to_registered_authenticator_response(
&self,
upgrade_mode_enabled: bool,
request_id: u64,
version: AuthenticatorVersion,
reply_to: Option<Recipient>,
) -> Result<nym_authenticator_requests::response::SerialisedResponse, GatewayWireguardError>
{
let wg_port = self.data.wireguard_config.port;
let local_pub_key = self.data.wireguard_config.public_key.into();
let ip_allocation = IpPair::new(
self.data.wireguard_config.private_ipv4,
self.data.wireguard_config.private_ipv6,
);
build_final_authenticator_response(
ip_allocation,
wg_port,
local_pub_key,
upgrade_mode_enabled,
request_id,
version,
reply_to,
)
}
pub(crate) fn to_pending_lp_response(&self) -> LpRegistrationResponse {
LpRegistrationResponse::request_dvpn_credential()
}
pub(crate) fn to_registered_lp_response(
&self,
upgrade_mode_enabled: bool,
) -> LpRegistrationResponse {
LpRegistrationResponse::success_dvpn(self.data.wireguard_config, upgrade_mode_enabled)
}
}
#[derive(Clone, Default)]
pub(crate) struct PendingRegistrations {
// TODO: unify those, somehow, later
/// Registrations in progress received from the Authenticator service provider via the
/// [`crate::node::internal_service_providers::authenticator::mixnet_listener::MixnetListener`]
pub(crate) authenticator: Arc<RwLock<HashMap<PeerPublicKey, PendingRegistration>>>,
/// Registrations in progress received from the LP Listener via the
/// [`crate::node::lp_listener::handler::LpConnectionHandler`] and handle through
/// [`crate::node::lp_listener::registration::LpHandlerState`]
pub(crate) lp: Arc<RwLock<HashMap<ReceiverIndex, PendingRegistration>>>,
}
impl PendingRegistrations {
pub(crate) async fn check_authenticator(
&self,
peer: &PeerPublicKey,
) -> Option<PendingRegistration> {
self.authenticator.read().await.get(peer).cloned()
}
pub(crate) async fn remove_authenticator(&self, peer: &PeerPublicKey) {
self.authenticator.write().await.remove(peer);
}
pub(crate) async fn remove_lp(&self, receiver_index: ReceiverIndex) {
self.lp.write().await.remove(&receiver_index);
}
pub(crate) async fn check_lp(
&self,
receiver_index: ReceiverIndex,
) -> Option<PendingRegistration> {
self.lp.read().await.get(&receiver_index).cloned()
}
pub(crate) async fn remove_stale_registrations(&self) {
// note: `IpPool` will release stale pre-allocated addresses by itself during the cleanup,
// so there's no need to send explicit messages over
let now = Instant::now();
self.authenticator.write().await.retain(|_, pending| {
now.duration_since(pending.requested_on) < DEFAULT_PENDING_REGISTRATION_TTL
});
self.lp.write().await.retain(|_, pending| {
now.duration_since(pending.requested_on) < DEFAULT_PENDING_REGISTRATION_TTL
});
}
}
gateway/src/node/wireguard/peer_manager.rs
+313 -156
View File
@@ -6,11 +6,28 @@ use defguard_wireguard_rs::{host::Peer, key::Key};
use futures::channel::oneshot;
use nym_credential_verification::{ClientBandwidth, TicketVerifier};
use nym_credentials_interface::CredentialSpendingData;
use nym_metrics::add_histogram_obs;
use nym_wireguard::peer_controller::IpPair;
use nym_wireguard::{peer_controller::PeerControlRequest, WireguardGatewayData};
use nym_wireguard_types::PeerPublicKey;
use std::time::Instant;
use tracing::error;
// Histogram buckets for WireGuard peer controller channel latency
// Measures time to send request and receive response from peer controller
// Expected: 1ms-100ms for normal operations, up to 2s for slow conditions
const WG_CONTROLLER_LATENCY_BUCKETS: &[f64] = &[
0.001, // 1ms
0.005, // 5ms
0.01, // 10ms
0.05, // 50ms
0.1, // 100ms
0.25, // 250ms
0.5, // 500ms
1.0, // 1s
2.0, // 2s
];
#[derive(Clone)]
pub struct PeerManager {
pub(crate) wireguard_gateway_data: WireguardGatewayData,
@@ -23,16 +40,17 @@ impl PeerManager {
}
}
pub async fn allocate_peer_ip_pair(&self) -> Result<IpPair, GatewayWireguardError> {
pub async fn preallocate_peer_ip_pair(&self) -> Result<IpPair, GatewayWireguardError> {
let controller_start = Instant::now();
let (response_tx, response_rx) = oneshot::channel();
let msg = PeerControlRequest::AllocatePeerIpPair { response_tx };
let msg = PeerControlRequest::PreAllocateIpPair { response_tx };
self.wireguard_gateway_data
.peer_tx()
.send(msg)
.await
.map_err(|_| GatewayWireguardError::PeerInteractionStopped)?;
response_rx
let res = response_rx
.await
.map_err(|e| {
GatewayWireguardError::InternalError(format!(
@@ -42,7 +60,16 @@ impl PeerManager {
.map_err(|e| {
error!("Failed to allocate IPs from pool: {e}");
GatewayWireguardError::InternalError(format!("Failed to allocate IPs: {e}"))
})
});
let latency = controller_start.elapsed().as_secs_f64();
add_histogram_obs!(
"wg_peer_controller_channel_latency_seconds",
latency,
WG_CONTROLLER_LATENCY_BUCKETS
);
res
}
pub async fn release_ip_pair(&self, ip_pair: IpPair) -> Result<(), GatewayWireguardError> {
@@ -70,6 +97,7 @@ impl PeerManager {
}
pub async fn add_peer(&self, peer: Peer) -> Result<(), GatewayWireguardError> {
let controller_start = Instant::now();
let (response_tx, response_rx) = oneshot::channel();
let msg = PeerControlRequest::AddPeer { peer, response_tx };
self.wireguard_gateway_data
@@ -78,17 +106,27 @@ impl PeerManager {
.await
.map_err(|_| GatewayWireguardError::PeerInteractionStopped)?;
response_rx
let res = response_rx
.await
.map_err(|_| GatewayWireguardError::internal("no response for add peer".to_string()))?
.map_err(|err| {
GatewayWireguardError::internal(format!(
"adding peer could not be performed: {err:?}"
))
})
});
let latency = controller_start.elapsed().as_secs_f64();
add_histogram_obs!(
"wg_peer_controller_channel_latency_seconds",
latency,
WG_CONTROLLER_LATENCY_BUCKETS
);
res
}
pub async fn _remove_peer(&self, pub_key: PeerPublicKey) -> Result<(), GatewayWireguardError> {
pub async fn remove_peer(&self, pub_key: PeerPublicKey) -> Result<(), GatewayWireguardError> {
let controller_start = Instant::now();
let key = Key::new(pub_key.to_bytes());
let (response_tx, response_rx) = oneshot::channel();
let msg = PeerControlRequest::RemovePeer { key, response_tx };
@@ -98,20 +136,63 @@ impl PeerManager {
.await
.map_err(|_| GatewayWireguardError::PeerInteractionStopped)?;
response_rx
let res = response_rx
.await
.map_err(|_| GatewayWireguardError::internal("no response for remove peer"))?
.map_err(|err| {
GatewayWireguardError::InternalError(format!(
"removing peer could not be performed: {err:?}"
))
})
});
let latency = controller_start.elapsed().as_secs_f64();
add_histogram_obs!(
"wg_peer_controller_channel_latency_seconds",
latency,
WG_CONTROLLER_LATENCY_BUCKETS
);
res
}
pub async fn check_active_peer(
&self,
pub_key: PeerPublicKey,
) -> Result<bool, GatewayWireguardError> {
let controller_start = Instant::now();
let key = Key::new(pub_key.to_bytes());
let (response_tx, response_rx) = oneshot::channel();
let msg = PeerControlRequest::CheckActivePeer { key, response_tx };
self.wireguard_gateway_data
.peer_tx()
.send(msg)
.await
.map_err(|_| GatewayWireguardError::PeerInteractionStopped)?;
let res = response_rx
.await
.map_err(|_| GatewayWireguardError::internal("no response for check active peer"))?
.map_err(|err| {
GatewayWireguardError::InternalError(format!(
"check active peer could not be performed: {err:?}"
))
});
let latency = controller_start.elapsed().as_secs_f64();
add_histogram_obs!(
"wg_peer_controller_channel_latency_seconds",
latency,
WG_CONTROLLER_LATENCY_BUCKETS
);
res
}
pub async fn query_peer(
&self,
public_key: PeerPublicKey,
) -> Result<Option<Peer>, GatewayWireguardError> {
let controller_start = Instant::now();
let key = Key::new(public_key.to_bytes());
let (response_tx, response_rx) = oneshot::channel();
let msg = PeerControlRequest::QueryPeer { key, response_tx };
@@ -121,14 +202,23 @@ impl PeerManager {
.await
.map_err(|_| GatewayWireguardError::PeerInteractionStopped)?;
response_rx
let res = response_rx
.await
.map_err(|_| GatewayWireguardError::internal("no response for query peer".to_string()))?
.map_err(|err| {
GatewayWireguardError::internal(format!(
"querying peer could not be performed: {err:?}"
))
})
});
let latency = controller_start.elapsed().as_secs_f64();
add_histogram_obs!(
"wg_peer_controller_channel_latency_seconds",
latency,
WG_CONTROLLER_LATENCY_BUCKETS
);
res
}
pub async fn query_bandwidth(
@@ -143,6 +233,7 @@ impl PeerManager {
&self,
key: PeerPublicKey,
) -> Result<ClientBandwidth, GatewayWireguardError> {
let controller_start = Instant::now();
let key = Key::new(key.to_bytes());
let (response_tx, response_rx) = oneshot::channel();
let msg = PeerControlRequest::GetClientBandwidthByKey { key, response_tx };
@@ -152,14 +243,23 @@ impl PeerManager {
.await
.map_err(|_| GatewayWireguardError::PeerInteractionStopped)?;
response_rx
let res = response_rx
.await
.map_err(|_| GatewayWireguardError::internal("no response for query client bandwidth"))?
.map_err(|err| {
GatewayWireguardError::internal(format!(
"querying client bandwidth could not be performed: {err:?}"
))
})
});
let latency = controller_start.elapsed().as_secs_f64();
add_histogram_obs!(
"wg_peer_controller_channel_latency_seconds",
latency,
WG_CONTROLLER_LATENCY_BUCKETS
);
res
}
pub async fn query_verifier_by_key(
@@ -167,6 +267,7 @@ impl PeerManager {
key: PeerPublicKey,
credential: CredentialSpendingData,
) -> Result<Box<dyn TicketVerifier + Send + Sync>, GatewayWireguardError> {
let controller_start = Instant::now();
let key = Key::new(key.to_bytes());
let (response_tx, response_rx) = oneshot::channel();
let msg = PeerControlRequest::GetVerifierByKey {
@@ -180,7 +281,7 @@ impl PeerManager {
.await
.map_err(|_| GatewayWireguardError::PeerInteractionStopped)?;
response_rx
let res = response_rx
.await
.map_err(|_| {
GatewayWireguardError::internal("no response for query verifier".to_string())
@@ -189,31 +290,39 @@ impl PeerManager {
GatewayWireguardError::internal(format!(
"querying verifier could not be performed: {err:?}"
))
})
});
let latency = controller_start.elapsed().as_secs_f64();
add_histogram_obs!(
"wg_peer_controller_channel_latency_seconds",
latency,
WG_CONTROLLER_LATENCY_BUCKETS
);
res
}
}
#[cfg(test)]
mod tests {
use std::{str::FromStr, sync::Arc};
use super::*;
use crate::node::wireguard::PeerRegistrator;
use crate::nym_authenticator::config::Authenticator;
use defguard_wireguard_rs::net::IpAddrMask;
use nym_credential_verification::upgrade_mode::testing::mock_dummy_upgrade_mode_details;
use nym_credential_verification::{
bandwidth_storage_manager::BandwidthStorageManager, ecash::MockEcashManager,
};
use nym_credentials_interface::Bandwidth;
use nym_crypto::asymmetric::x25519::KeyPair;
use nym_gateway_storage::traits::{mock::MockGatewayStorage, BandwidthGatewayStorage};
use nym_task::ShutdownManager;
use nym_test_utils::helpers::{deterministic_rng, DeterministicRng, RngCore};
use nym_wireguard::peer_controller::{start_controller, stop_controller};
use rand::rngs::OsRng;
use std::{str::FromStr, sync::Arc};
use time::{Duration, OffsetDateTime};
use tokio::sync::RwLock;
use crate::nym_authenticator::{
config::Authenticator, mixnet_listener::credential_storage_preparation,
};
use super::*;
const CREDENTIAL_BYTES: [u8; 1245] = [
0, 0, 4, 133, 96, 179, 223, 185, 136, 23, 213, 166, 59, 203, 66, 69, 209, 181, 227, 254,
16, 102, 98, 237, 59, 119, 170, 111, 31, 194, 51, 59, 120, 17, 115, 229, 79, 91, 11, 139,
@@ -279,141 +388,202 @@ mod tests {
0, 0, 0, 0, 0, 1,
];
#[tokio::test]
async fn add_peer() {
let (wireguard_data, request_rx) = WireguardGatewayData::new(
Authenticator::default().into(),
Arc::new(KeyPair::new(&mut OsRng)),
);
let peer_manager = PeerManager::new(wireguard_data);
let (storage, task_manager) = start_controller(
peer_manager.wireguard_gateway_data.peer_tx().clone(),
request_rx,
);
let peer = Peer::default();
let ecash_manager = MockEcashManager::new(Box::new(storage.clone()));
assert!(peer_manager.add_peer(peer.clone()).await.is_err());
let client_id = storage
.insert_wireguard_peer(&peer, FromStr::from_str("entry_wireguard").unwrap())
.await
.unwrap();
assert!(peer_manager.add_peer(peer.clone()).await.is_err());
credential_storage_preparation(Arc::new(ecash_manager), client_id)
.await
.unwrap();
peer_manager.add_peer(peer.clone()).await.unwrap();
stop_controller(task_manager).await;
struct TestSetup {
rng: DeterministicRng,
_ecash_manager: Arc<MockEcashManager>,
storage: Arc<RwLock<MockGatewayStorage>>,
peer_registrator: PeerRegistrator,
peer_manager: PeerManager,
task_manager: ShutdownManager,
}
async fn helper_add_peer(
storage: &Arc<RwLock<MockGatewayStorage>>,
peer_manager: &mut PeerManager,
) -> i64 {
let peer = Peer::default();
let ecash_manager = MockEcashManager::new(Box::new(storage.clone()));
let client_id = storage
struct GeneratedPeer {
peer: Peer,
client_id: i64,
}
impl GeneratedPeer {
fn key(&self) -> PeerPublicKey {
PeerPublicKey::from_str(self.peer.public_key.to_string().as_str()).unwrap()
}
}
impl TestSetup {
fn new() -> TestSetup {
let mut rng = deterministic_rng();
let (wireguard_data, request_rx) = WireguardGatewayData::new(
Authenticator::default().into(),
Arc::new(KeyPair::new(&mut rng)),
);
let (upgrade_mode_details, _) = mock_dummy_upgrade_mode_details();
let peer_manager = PeerManager::new(wireguard_data);
let (storage, task_manager) = start_controller(
peer_manager.wireguard_gateway_data.peer_tx().clone(),
request_rx,
);
let ecash_manager = Arc::new(MockEcashManager::new(Box::new(storage.clone())));
let peer_registrator = PeerRegistrator::new(
ecash_manager.clone(),
peer_manager.clone(),
upgrade_mode_details,
);
TestSetup {
rng,
_ecash_manager: ecash_manager,
storage,
peer_registrator,
peer_manager,
task_manager,
}
}
async fn peer_with_pre_allocated_ip(&mut self) -> Peer {
let mut peer = Peer::default();
let mut key = [0u8; 32];
self.rng.fill_bytes(&mut key);
peer.public_key = Key::new(key);
let allocation = self.peer_manager.preallocate_peer_ip_pair().await.unwrap();
peer.allowed_ips = vec![
IpAddrMask::new(allocation.ipv4.into(), 32),
IpAddrMask::new(allocation.ipv6.into(), 128),
];
peer
}
async fn _add_peer(&self, peer: &Peer) -> i64 {
let client_id = self
.storage
.insert_wireguard_peer(peer, FromStr::from_str("entry_wireguard").unwrap())
.await
.unwrap();
self.peer_registrator
.credential_storage_preparation(client_id)
.await
.unwrap();
self.peer_manager.add_peer(peer.clone()).await.unwrap();
client_id
}
async fn add_peer(&mut self) -> GeneratedPeer {
let peer = self.peer_with_pre_allocated_ip().await;
let client_id = self._add_peer(&peer).await;
GeneratedPeer { peer, client_id }
}
async fn finish(self) {
stop_controller(self.task_manager).await
}
}
#[tokio::test]
async fn assign_peer_ip() -> anyhow::Result<()> {
let test = TestSetup::new();
let ip_pair1 = test.peer_manager.preallocate_peer_ip_pair().await?;
let ip_pair2 = test.peer_manager.preallocate_peer_ip_pair().await?;
assert_ne!(ip_pair1, ip_pair2);
test.finish().await;
Ok(())
}
#[tokio::test]
async fn add_peer() {
let mut test = TestSetup::new();
let peer = test.peer_with_pre_allocated_ip().await;
assert!(test.peer_manager.add_peer(peer.clone()).await.is_err());
let client_id = test
.storage
.insert_wireguard_peer(&peer, FromStr::from_str("entry_wireguard").unwrap())
.await
.unwrap();
credential_storage_preparation(Arc::new(ecash_manager), client_id)
assert!(test.peer_manager.add_peer(peer.clone()).await.is_err());
test.peer_registrator
.credential_storage_preparation(client_id)
.await
.unwrap();
peer_manager.add_peer(peer.clone()).await.unwrap();
test.peer_manager.add_peer(peer.clone()).await.unwrap();
client_id
test.finish().await
}
#[tokio::test]
async fn remove_peer() {
let (wireguard_data, request_rx) = WireguardGatewayData::new(
Authenticator::default().into(),
Arc::new(KeyPair::new(&mut OsRng)),
);
let mut peer_manager = PeerManager::new(wireguard_data);
let key = Key::default();
let public_key = PeerPublicKey::from_str(&key.to_string()).unwrap();
let (storage, task_manager) = start_controller(
peer_manager.wireguard_gateway_data.peer_tx().clone(),
request_rx,
);
let mut test = TestSetup::new();
let peer = test.add_peer().await;
let public_key = peer.key();
helper_add_peer(&storage, &mut peer_manager).await;
peer_manager._remove_peer(public_key).await.unwrap();
test.peer_manager.remove_peer(public_key).await.unwrap();
stop_controller(task_manager).await;
test.finish().await
}
#[tokio::test]
async fn query_peer() {
let (wireguard_data, request_rx) = WireguardGatewayData::new(
Authenticator::default().into(),
Arc::new(KeyPair::new(&mut OsRng)),
);
let mut peer_manager = PeerManager::new(wireguard_data);
let key = Key::default();
let public_key = PeerPublicKey::from_str(&key.to_string()).unwrap();
let (storage, task_manager) = start_controller(
peer_manager.wireguard_gateway_data.peer_tx().clone(),
request_rx,
);
let mut test = TestSetup::new();
let peer = test.peer_with_pre_allocated_ip().await;
let public_key = PeerPublicKey::from_str(peer.public_key.to_string().as_str()).unwrap();
assert!(peer_manager.query_peer(public_key).await.unwrap().is_none());
assert!(test
.peer_manager
.query_peer(public_key)
.await
.unwrap()
.is_none());
helper_add_peer(&storage, &mut peer_manager).await;
let peer = peer_manager.query_peer(public_key).await.unwrap().unwrap();
assert_eq!(peer.public_key, key);
test._add_peer(&peer).await;
let peer_query = test
.peer_manager
.query_peer(public_key)
.await
.unwrap()
.unwrap();
assert_eq!(peer.public_key, peer_query.public_key);
stop_controller(task_manager).await;
test.finish().await
}
#[tokio::test]
async fn query_bandwidth() {
let (wireguard_data, request_rx) = WireguardGatewayData::new(
Authenticator::default().into(),
Arc::new(KeyPair::new(&mut OsRng)),
);
let mut peer_manager = PeerManager::new(wireguard_data);
let key = Key::default();
let public_key = PeerPublicKey::from_str(&key.to_string()).unwrap();
let (storage, task_manager) = start_controller(
peer_manager.wireguard_gateway_data.peer_tx().clone(),
request_rx,
);
let mut test = TestSetup::new();
let peer = test.peer_with_pre_allocated_ip().await;
let public_key = PeerPublicKey::from_str(peer.public_key.to_string().as_str()).unwrap();
assert!(peer_manager.query_bandwidth(public_key).await.is_err());
assert!(test.peer_manager.query_bandwidth(public_key).await.is_err());
helper_add_peer(&storage, &mut peer_manager).await;
let available_bandwidth = peer_manager.query_bandwidth(public_key).await.unwrap();
test._add_peer(&peer).await;
let available_bandwidth = test.peer_manager.query_bandwidth(public_key).await.unwrap();
assert_eq!(available_bandwidth, 0);
stop_controller(task_manager).await;
test.finish().await
}
#[tokio::test]
async fn query_client_bandwidth() {
let (wireguard_data, request_rx) = WireguardGatewayData::new(
Authenticator::default().into(),
Arc::new(KeyPair::new(&mut OsRng)),
);
let mut peer_manager = PeerManager::new(wireguard_data);
let key = Key::default();
let public_key = PeerPublicKey::from_str(&key.to_string()).unwrap();
let (storage, task_manager) = start_controller(
peer_manager.wireguard_gateway_data.peer_tx().clone(),
request_rx,
);
let mut test = TestSetup::new();
let peer = test.peer_with_pre_allocated_ip().await;
let public_key = PeerPublicKey::from_str(peer.public_key.to_string().as_str()).unwrap();
assert!(peer_manager
assert!(test
.peer_manager
.query_client_bandwidth(public_key)
.await
.is_err());
helper_add_peer(&storage, &mut peer_manager).await;
let available_bandwidth = peer_manager
test._add_peer(&peer).await;
let available_bandwidth = test
.peer_manager
.query_client_bandwidth(public_key)
.await
.unwrap()
@@ -421,64 +591,51 @@ mod tests {
.await;
assert_eq!(available_bandwidth, 0);
stop_controller(task_manager).await;
test.finish().await
}
#[tokio::test]
async fn query_verifier() {
let (wireguard_data, request_rx) = WireguardGatewayData::new(
Authenticator::default().into(),
Arc::new(KeyPair::new(&mut OsRng)),
);
let mut peer_manager = PeerManager::new(wireguard_data);
let key = Key::default();
let public_key = PeerPublicKey::from_str(&key.to_string()).unwrap();
let (storage, task_manager) = start_controller(
peer_manager.wireguard_gateway_data.peer_tx().clone(),
request_rx,
);
let mut test = TestSetup::new();
let peer = test.peer_with_pre_allocated_ip().await;
let public_key = PeerPublicKey::from_str(peer.public_key.to_string().as_str()).unwrap();
let credential = CredentialSpendingData::try_from_bytes(&CREDENTIAL_BYTES).unwrap();
assert!(peer_manager
assert!(test
.peer_manager
.query_verifier_by_key(public_key, credential.clone())
.await
.is_err());
helper_add_peer(&storage, &mut peer_manager).await;
peer_manager
test._add_peer(&peer).await;
test.peer_manager
.query_verifier_by_key(public_key, credential)
.await
.unwrap();
stop_controller(task_manager).await;
test.finish().await
}
#[tokio::test]
async fn increase_decrease_bandwidth() {
let (wireguard_data, request_rx) = WireguardGatewayData::new(
Authenticator::default().into(),
Arc::new(KeyPair::new(&mut OsRng)),
);
let mut peer_manager = PeerManager::new(wireguard_data);
let key = Key::default();
let public_key = PeerPublicKey::from_str(&key.to_string()).unwrap();
let mut test = TestSetup::new();
let peer = test.add_peer().await;
let public_key = peer.key();
let top_up = 42;
let consume = 4;
let (storage, task_manager) = start_controller(
peer_manager.wireguard_gateway_data.peer_tx().clone(),
request_rx,
);
let client_id = helper_add_peer(&storage, &mut peer_manager).await;
let client_bandwidth = peer_manager
.query_client_bandwidth(public_key)
let client_bandwidth = test
.peer_manager
.query_client_bandwidth(peer.key())
.await
.unwrap();
let mut bw_manager = BandwidthStorageManager::new(
Box::new(storage),
Box::new(test.storage.clone()),
client_bandwidth.clone(),
client_id,
peer.client_id,
Default::default(),
true,
);
@@ -494,7 +651,7 @@ mod tests {
assert_eq!(client_bandwidth.available().await, top_up);
assert_eq!(
peer_manager.query_bandwidth(public_key).await.unwrap(),
test.peer_manager.query_bandwidth(public_key).await.unwrap(),
top_up
);
@@ -502,10 +659,10 @@ mod tests {
let remaining = top_up - consume;
assert_eq!(client_bandwidth.available().await, remaining);
assert_eq!(
peer_manager.query_bandwidth(public_key).await.unwrap(),
test.peer_manager.query_bandwidth(public_key).await.unwrap(),
remaining
);
stop_controller(task_manager).await;
test.finish().await
}
}
integration-tests/src/lp_registration.rs
+19 -44
View File
@@ -5,11 +5,8 @@
mod tests {
use anyhow::Context;
use nym_bandwidth_controller::mock::MockBandwidthController;
use nym_credential_verification::UpgradeModeState;
use nym_credential_verification::ecash::MockEcashManager;
use nym_credential_verification::upgrade_mode::{
UpgradeModeCheckConfig, UpgradeModeCheckRequestSender, UpgradeModeDetails,
};
use nym_credential_verification::upgrade_mode::testing::mock_dummy_upgrade_mode_details;
use nym_credentials_interface::TicketType;
use nym_crypto::asymmetric::{ed25519, x25519};
use nym_gateway::GatewayError;
@@ -18,7 +15,7 @@ mod tests {
LpDebug, LpHandlerState, LpLocalPeer, MixForwardingReceiver, PeerControlRequest,
WireguardGatewayData, mix_forwarding_channels,
};
use nym_gateway::node::wireguard::PeerManager;
use nym_gateway::node::wireguard::{PeerManager, PeerRegistrator};
use nym_gateway::node::{ActiveClientsStore, GatewayStorage, LpConfig};
use nym_registration_client::{LpClientError, LpRegistrationClient};
use nym_test_utils::helpers::{CryptoRng, RngCore, u64_seeded_rng};
@@ -166,10 +163,9 @@ mod tests {
.unwrap()
}
async fn allocate_ip_pair(&mut self) -> IpPair {
fn pre_allocate_ip_pair(&mut self) -> IpPair {
self.ip_pool
.allocate()
.await
.pre_allocate()
.expect("unexpected ip allocation failure!")
}
@@ -181,17 +177,6 @@ mod tests {
Ok(GatewayStorage::from_connection_pool(conn_pool, 100).await?)
}
const DUMMY_ATTESTER_ED25519_PRIVATE_KEY: [u8; 32] = [
108, 49, 193, 21, 126, 161, 249, 85, 242, 207, 74, 195, 238, 6, 64, 149, 201, 140, 248,
163, 122, 170, 79, 198, 87, 85, 36, 29, 243, 92, 64, 161,
];
pub(crate) fn dummy_attester_public_key() -> ed25519::PublicKey {
let private_key =
ed25519::PrivateKey::from_bytes(&Self::DUMMY_ATTESTER_ED25519_PRIVATE_KEY).unwrap();
private_key.public_key()
}
async fn mock(rng: &mut (impl RngCore + CryptoRng)) -> anyhow::Result<Self> {
let base = Party::generate(rng);
@@ -217,31 +202,24 @@ mod tests {
// create wireguard data
let (wireguard_data, peer_request_rx) = Self::wireguard_data(&base);
let (um_recheck_tx, um_recheck_rx) = futures::channel::mpsc::unbounded();
// TODO: use it if we ever want to test UM
let _ = um_recheck_rx;
let (upgrade_mode_details, _) = mock_dummy_upgrade_mode_details();
// mock the wg peer controller
let (mock_peer_controller, peer_controller_state) =
mock_peer_controller(peer_request_rx);
let upgrade_mode_state = UpgradeModeState::new(Self::dummy_attester_public_key());
let upgrade_mode_details = UpgradeModeDetails::new(
UpgradeModeCheckConfig {
// essentially we never want to trigger this in our tests
min_staleness_recheck: Duration::from_nanos(1),
},
UpgradeModeCheckRequestSender::new(um_recheck_tx),
upgrade_mode_state.clone(),
);
// registering particular responses for peer controller is up to given test
let peer_manager = Arc::new(PeerManager::new(wireguard_data));
let ecash_verifier = Arc::new(ecash_verifier);
let peer_registrator = PeerRegistrator::new(
ecash_verifier.clone(),
PeerManager::new(wireguard_data),
upgrade_mode_details,
);
let lp_state = LpHandlerState {
// use mock instance of ecash verifier
ecash_verifier: Arc::new(ecash_verifier),
ecash_verifier,
// use in-memory database (no need for persistency)
storage,
@@ -253,10 +231,6 @@ mod tests {
// no clients at the beginning
active_clients_store: ActiveClientsStore::new(),
// handles required for wg registration
upgrade_mode: upgrade_mode_details,
peer_manager,
// use default lp config (with enabled flag)
lp_config,
@@ -269,9 +243,10 @@ mod tests {
// we start with empty state
session_states: Arc::new(Default::default()),
// sensible default value for tests
registrations_in_progress: Default::default(),
forward_semaphore,
// handles for dealing with new peers
peer_registrator: Some(peer_registrator),
};
Ok(Gateway {
@@ -444,7 +419,7 @@ mod tests {
// 3. register all needed responses for the dvpn registration that will reach the peer controller
// 1) peer registration - ip pair allocation
let ip_pair = entry.allocate_ip_pair().await;
let ip_pair = entry.pre_allocate_ip_pair();
let reg_res = Ok::<_, nym_wireguard::Error>(ip_pair);
entry
@@ -617,7 +592,7 @@ mod tests {
// 4. register all needed responses for the dvpn registration that will reach the peer controller
// 1) peer registration - ip pair allocation
let entry_ip_pair = entry.allocate_ip_pair().await;
let entry_ip_pair = entry.pre_allocate_ip_pair();
let reg_res = Ok::<_, nym_wireguard::Error>(entry_ip_pair);
entry
@@ -668,7 +643,7 @@ mod tests {
// 10. register all needed responses for the dvpn registration that will reach the peer controller
// 1) peer registration - ip pair allocation
let exit_ip_pair = exit.allocate_ip_pair().await;
let exit_ip_pair = exit.pre_allocate_ip_pair();
let reg_res = Ok::<_, nym_wireguard::Error>(exit_ip_pair);
exit.register_peer_controller_response(
nym-gateway-probe/src/common/netstack.rs
+3 -2
View File
@@ -5,6 +5,7 @@ use crate::config::NetstackArgs;
use anyhow::Context;
use serde::Deserialize;
use std::ffi::{CStr, CString};
use std::net::SocketAddr;
mod sys {
use std::ffi::{c_char, c_void};
@@ -223,14 +224,14 @@ pub struct TwoHopNetstackRequestGo {
pub entry_wg_ip: String,
pub entry_private_key: String,
pub entry_public_key: String,
pub entry_endpoint: String,
pub entry_endpoint: SocketAddr,
pub entry_awg_args: String,
// Exit tunnel configuration (connects via forwarder through entry)
pub exit_wg_ip: String,
pub exit_private_key: String,
pub exit_public_key: String,
pub exit_endpoint: String,
pub exit_endpoint: SocketAddr,
pub exit_awg_args: String,
// Test parameters
nym-gateway-probe/src/common/probe_tests.rs
+2 -5
View File
@@ -299,9 +299,6 @@ pub async fn wg_probe_lp(
let entry_lp_version = entry_lp_data.lp_version;
let exit_lp_version = exit_lp_data.lp_version;
let entry_ip = entry_address.ip();
let exit_ip = exit_address.ip();
info!("Starting LP-based WireGuard probe (entry→exit via forwarding)");
let mut wg_outcome = WgProbeResults::default();
@@ -405,9 +402,9 @@ pub async fn wg_probe_lp(
// Build WireGuard endpoint addresses
// Entry endpoint uses entry_ip (host-reachable) + port from registration
let entry_wg_endpoint = format!("{}:{}", entry_ip, entry_gateway_data.endpoint.port());
let entry_wg_endpoint = entry_gateway_data.endpoint;
// Exit endpoint uses exit_ip + port from registration (forwarded via entry)
let exit_wg_endpoint = format!("{}:{}", exit_ip, exit_gateway_data.endpoint.port());
let exit_wg_endpoint = exit_gateway_data.endpoint;
info!("Two-hop WireGuard configuration:");
info!(" Entry gateway:");
nym-gateway-probe/src/common/wireguard.rs
+9 -8
View File
@@ -7,6 +7,7 @@
//! that is shared between different test modes (authenticator-based and LP-based).
use nym_config::defaults::{WG_METADATA_PORT, WG_TUN_DEVICE_IP_ADDRESS_V4};
use std::net::SocketAddr;
use tracing::{error, info};
use crate::NetstackArgs;
@@ -185,7 +186,7 @@ pub struct TwoHopWgTunnelConfig {
/// Entry gateway's WireGuard public key (hex encoded)
pub entry_public_key_hex: String,
/// Entry WireGuard endpoint address (entry_gateway_ip:port)
pub entry_endpoint: String,
pub entry_endpoint: SocketAddr,
/// Entry Amnezia WG args (empty for standard WG)
pub entry_awg_args: String,
@@ -197,7 +198,7 @@ pub struct TwoHopWgTunnelConfig {
/// Exit gateway's WireGuard public key (hex encoded)
pub exit_public_key_hex: String,
/// Exit WireGuard endpoint address (exit_gateway_ip:port, forwarded via entry)
pub exit_endpoint: String,
pub exit_endpoint: SocketAddr,
/// Exit Amnezia WG args (empty for standard WG)
pub exit_awg_args: String,
}
@@ -209,24 +210,24 @@ impl TwoHopWgTunnelConfig {
entry_private_ipv4: impl Into<String>,
entry_private_key_hex: impl Into<String>,
entry_public_key_hex: impl Into<String>,
entry_endpoint: impl Into<String>,
entry_endpoint: SocketAddr,
entry_awg_args: impl Into<String>,
exit_private_ipv4: impl Into<String>,
exit_private_key_hex: impl Into<String>,
exit_public_key_hex: impl Into<String>,
exit_endpoint: impl Into<String>,
exit_endpoint: SocketAddr,
exit_awg_args: impl Into<String>,
) -> Self {
Self {
entry_private_ipv4: entry_private_ipv4.into(),
entry_private_key_hex: entry_private_key_hex.into(),
entry_public_key_hex: entry_public_key_hex.into(),
entry_endpoint: entry_endpoint.into(),
entry_endpoint,
entry_awg_args: entry_awg_args.into(),
exit_private_ipv4: exit_private_ipv4.into(),
exit_private_key_hex: exit_private_key_hex.into(),
exit_public_key_hex: exit_public_key_hex.into(),
exit_endpoint: exit_endpoint.into(),
exit_endpoint,
exit_awg_args: exit_awg_args.into(),
}
}
@@ -256,14 +257,14 @@ pub fn run_two_hop_tunnel_tests(
entry_wg_ip: config.entry_private_ipv4.clone(),
entry_private_key: config.entry_private_key_hex.clone(),
entry_public_key: config.entry_public_key_hex.clone(),
entry_endpoint: config.entry_endpoint.clone(),
entry_endpoint: config.entry_endpoint,
entry_awg_args: config.entry_awg_args.clone(),
// Exit tunnel config
exit_wg_ip: config.exit_private_ipv4.clone(),
exit_private_key: config.exit_private_key_hex.clone(),
exit_public_key: config.exit_public_key_hex.clone(),
exit_endpoint: config.exit_endpoint.clone(),
exit_endpoint: config.exit_endpoint,
exit_awg_args: config.exit_awg_args.clone(),
// Test parameters (use IPv4 config)
nym-node/src/node/mod.rs
+30 -14
View File
@@ -673,6 +673,26 @@ impl NymNode {
let upgrade_mode_common_state =
gateway_tasks_builder.build_upgrade_mode_common_state(upgrade_check_request_sender);
// Set WireGuard data early so other builders can access it
if self.config.wireguard.enabled {
let Some(wg_data) = self.wireguard.take() else {
return Err(NymNodeError::WireguardDataUnavailable);
};
gateway_tasks_builder.set_wireguard_data(wg_data.into());
}
let wg_peer_registrator = gateway_tasks_builder
.build_peer_registrator(upgrade_mode_common_state.clone())
.await?;
if let Some(wg_peer_registrator) = wg_peer_registrator.as_ref() {
let cleanup_task = wg_peer_registrator.cleanup_task(self.shutdown_token());
self.shutdown_tracker().try_spawn_named(
async move { cleanup_task.run().await },
"StaleRegistrationRemover",
);
};
// if we're running in entry mode, start the websocket
if self.modes().entry {
info!(
@@ -688,15 +708,6 @@ impl NymNode {
self.shutdown_tracker()
.try_spawn_named(async move { websocket.run().await }, "EntryWebsocket");
// Set WireGuard data early so LP listener can access it
// (LP listener needs wg_peer_controller for dVPN registrations)
if self.config.wireguard.enabled {
let Some(wg_data) = self.wireguard.take() else {
return Err(NymNodeError::WireguardDataUnavailable);
};
gateway_tasks_builder.set_wireguard_data(wg_data.into());
}
// Start LP listener if enabled
info!(
"starting the LP listener on {} (data handler on: {})",
@@ -704,10 +715,7 @@ impl NymNode {
self.config.gateway_tasks.lp.data_bind_address,
);
let mut lp_listener = gateway_tasks_builder
.build_lp_listener(
upgrade_mode_common_state.clone(),
active_clients_store.clone(),
)
.build_lp_listener(wg_peer_registrator.clone(), active_clients_store.clone())
.await?;
self.shutdown_tracker()
.try_spawn_named(async move { lp_listener.run().await }, "LpListener");
@@ -747,8 +755,16 @@ impl NymNode {
gateway_tasks_builder.set_authenticator_opts(config.auth_opts);
let Some(peer_registrator) = wg_peer_registrator else {
return Err(NymNodeError::WireguardDataUnavailable);
};
let authenticator = gateway_tasks_builder
.build_wireguard_authenticator(upgrade_mode_common_state.clone(), topology_provider)
.build_wireguard_authenticator(
peer_registrator,
upgrade_mode_common_state.clone(),
topology_provider,
)
.await?;
let started_authenticator = authenticator.start_service_provider().await?;
active_clients_store.insert_embedded(started_authenticator.handle);
nym-registration-client/src/lp_client/client.rs
+6 -19
View File
@@ -25,6 +25,7 @@ use nym_lp_transport::traits::LpTransport;
use nym_registration_common::dvpn::LpDvpnRegistrationResponseMessageContent;
use nym_registration_common::{
LpRegistrationRequest, LpRegistrationResponse, WireguardConfiguration,
WireguardRegistrationData,
};
use nym_wireguard_types::PeerPublicKey;
use rand::{CryptoRng, RngCore};
@@ -748,7 +749,7 @@ where
gateway_identity: ed25519::PublicKey,
bandwidth_controller: &dyn BandwidthTicketProvider,
ticket_type: TicketType,
) -> Result<WireguardConfiguration> {
) -> Result<WireguardRegistrationData> {
tracing::debug!("Acquiring bandwidth credential for registration");
// 1. Get bandwidth credential from controller
@@ -805,14 +806,7 @@ where
tracing::warn!("Gateway rejected registration: {reason}");
Err(LpClientError::RegistrationRejected { reason })
}
LpDvpnRegistrationResponseMessageContent::CompletedRegistration(res) => {
// we have managed to complete the registration
tracing::info!(
"LP registration successful! Allocated bandwidth: {} bytes",
res.available_bandwidth
);
Ok(res.config)
}
LpDvpnRegistrationResponseMessageContent::CompletedRegistration(res) => Ok(res.config),
LpDvpnRegistrationResponseMessageContent::RequiresCredential(_) => {
Err(LpClientError::unexpected_response(
"received request for additional dvpn data after sending credential!",
@@ -860,7 +854,7 @@ where
let wg_public_key = PeerPublicKey::from(*wg_keypair.public_key());
let mut psk = [0u8; 32];
rng.fill_bytes(&mut psk);
let request = LpRegistrationRequest::new_initial_dvpn(wg_public_key, psk, ticket_type);
let request = LpRegistrationRequest::new_initial_dvpn(wg_public_key, psk);
tracing::trace!("Built dVPN registration request: {request:?}");
@@ -896,14 +890,7 @@ where
tracing::warn!("Gateway rejected registration: {reason}");
return Err(LpClientError::RegistrationRejected { reason });
}
LpDvpnRegistrationResponseMessageContent::CompletedRegistration(res) => {
// we have already registered with this gateway before, the gateway has updated the psk and sent us the config
tracing::info!(
"LP registration successful! Allocated bandwidth: {} bytes",
res.available_bandwidth
);
res.config
}
LpDvpnRegistrationResponseMessageContent::CompletedRegistration(res) => res.config,
LpDvpnRegistrationResponseMessageContent::RequiresCredential(_) => {
// we're registering for the first time with this gateway - we need to attach a credential
@@ -920,7 +907,7 @@ where
Ok(WireguardConfiguration {
public_key: final_response.public_key,
psk: Some(psk),
endpoint: SocketAddr::new(self.gateway_lp_address.ip(), final_response.endpoint.port()),
endpoint: SocketAddr::new(self.gateway_lp_address.ip(), final_response.port),
private_ipv4: final_response.private_ipv4,
private_ipv6: final_response.private_ipv6,
})
nym-registration-client/src/lp_client/nested_session.rs
+6 -19
View File
@@ -38,6 +38,7 @@ use nym_lp_transport::traits::LpTransport;
use nym_registration_common::dvpn::LpDvpnRegistrationResponseMessageContent;
use nym_registration_common::{
LpRegistrationRequest, LpRegistrationResponse, WireguardConfiguration,
WireguardRegistrationData,
};
use nym_wireguard_types::PeerPublicKey;
use rand::{CryptoRng, RngCore};
@@ -372,7 +373,7 @@ impl NestedLpSession {
gateway_identity: ed25519::PublicKey,
bandwidth_controller: &dyn BandwidthTicketProvider,
ticket_type: TicketType,
) -> Result<WireguardConfiguration>
) -> Result<WireguardRegistrationData>
where
S: LpTransport + Unpin,
{
@@ -433,14 +434,7 @@ impl NestedLpSession {
tracing::warn!("Gateway rejected registration: {reason}");
Err(LpClientError::RegistrationRejected { reason })
}
LpDvpnRegistrationResponseMessageContent::CompletedRegistration(res) => {
// we have managed to complete the registration
tracing::info!(
"LP registration successful! Allocated bandwidth: {} bytes",
res.available_bandwidth
);
Ok(res.config)
}
LpDvpnRegistrationResponseMessageContent::CompletedRegistration(res) => Ok(res.config),
LpDvpnRegistrationResponseMessageContent::RequiresCredential(_) => {
Err(LpClientError::unexpected_response(
"received request for additional dvpn data after sending credential!",
@@ -499,7 +493,7 @@ impl NestedLpSession {
let mut psk = [0u8; 32];
rng.fill_bytes(&mut psk);
let request = LpRegistrationRequest::new_initial_dvpn(wg_public_key, psk, ticket_type);
let request = LpRegistrationRequest::new_initial_dvpn(wg_public_key, psk);
// Step 3: Serialize the request
let send_data = request.to_lp_data()?;
@@ -536,14 +530,7 @@ impl NestedLpSession {
tracing::warn!("Gateway rejected registration: {reason}");
return Err(LpClientError::RegistrationRejected { reason });
}
LpDvpnRegistrationResponseMessageContent::CompletedRegistration(res) => {
// we have already registered with this gateway before, the gateway has updated the psk and sent us the config
tracing::info!(
"LP registration successful! Allocated bandwidth: {} bytes",
res.available_bandwidth
);
res.config
}
LpDvpnRegistrationResponseMessageContent::CompletedRegistration(res) => res.config,
LpDvpnRegistrationResponseMessageContent::RequiresCredential(_) => {
// we're registering for the first time with this gateway - we need to attach a credential
@@ -562,7 +549,7 @@ impl NestedLpSession {
Ok(WireguardConfiguration {
public_key: final_response.public_key,
psk: Some(psk),
endpoint: SocketAddr::new(self.exit_address.ip(), final_response.endpoint.port()),
endpoint: SocketAddr::new(self.exit_address.ip(), final_response.port),
private_ipv4: final_response.private_ipv4,
private_ipv6: final_response.private_ipv6,
})