Persist used wireguard private IPs (#4771)

* Persist used wireguard private IPs

* Fix imports

* Remove unnecessary type specification

common/gateway-storage/src/lib.rs

@@ -22,7 +22,7 @@ use tracing::{debug, error};
 pub mod bandwidth;
 pub mod error;
 mod inboxes;
-pub(crate) mod models;
+pub mod models;
 mod shared_keys;
 mod tickets;
 #[cfg(feature = "wireguard")]

common/wireguard/src/lib.rs

@@ -84,6 +84,7 @@ pub struct WireguardData {
 #[cfg(target_os = "linux")]
 pub async fn start_wireguard<St: nym_gateway_storage::Storage + 'static>(
     storage: St,
+    all_peers: Vec<nym_gateway_storage::models::WireguardPeer>,
     task_client: nym_task::TaskClient,
     wireguard_data: WireguardData,
     control_tx: UnboundedSender<peer_controller::PeerControlResponse>,
@@ -95,7 +96,7 @@ pub async fn start_wireguard<St: nym_gateway_storage::Storage + 'static>(
 
     let mut peers = vec![];
     let mut suspended_peers = vec![];
-    for storage_peer in storage.get_all_wireguard_peers().await? {
+    for storage_peer in all_peers {
         let suspended = storage_peer.suspended;
         let peer = Peer::try_from(storage_peer)?;
         if suspended {

gateway/src/error.rs

@@ -204,8 +204,8 @@ pub enum GatewayError {
     WireguardInterfaceError(#[from] defguard_wireguard_rs::error::WireguardInterfaceError),
 
     #[cfg(all(feature = "wireguard", target_os = "linux"))]
-    #[error("wireguard not set")]
-    WireguardNotSet,
+    #[error("internal wireguard error {0}")]
+    InternalWireguardError(String),
 
     #[error("failed to start authenticator: {source}")]
     AuthenticatorStartError {

gateway/src/node/mod.rs

@@ -267,12 +267,29 @@ impl<St> Gateway<St> {
             forwarding_channel,
             router_tx,
         );
+        let all_peers = self.storage.get_all_wireguard_peers().await?;
+        let used_private_network_ips = all_peers
+            .iter()
+            .cloned()
+            .map(|wireguard_peer| {
+                defguard_wireguard_rs::host::Peer::try_from(wireguard_peer).map(|mut peer| {
+                    peer.allowed_ips
+                        .pop()
+                        .ok_or(Box::new(GatewayError::InternalWireguardError(format!(
+                            "no private IP set for peer {}",
+                            peer.public_key
+                        ))))
+                        .map(|p| p.ip)
+                })
+            })
+            .collect::<Result<Result<Vec<_>, _>, _>>()??;
 
         if let Some(wireguard_data) = self.wireguard_data.take() {
             let (on_start_tx, on_start_rx) = oneshot::channel();
             let mut authenticator_server = nym_authenticator::Authenticator::new(
                 opts.config.clone(),
                 wireguard_data.inner.clone(),
+                used_private_network_ips,
                 peer_response_rx,
             )
             .with_custom_gateway_transceiver(Box::new(transceiver))
@@ -306,6 +323,7 @@ impl<St> Gateway<St> {
 
             let wg_api = nym_wireguard::start_wireguard(
                 self.storage.clone(),
+                all_peers,
                 shutdown,
                 wireguard_data,
                 peer_response_tx,
@@ -317,7 +335,9 @@ impl<St> Gateway<St> {
                 handle: LocalEmbeddedClientHandle::new(start_data.address, auth_mix_sender),
             })
         } else {
-            Err(Box::new(GatewayError::WireguardNotSet))
+            Err(Box::new(GatewayError::InternalWireguardError(
+                "wireguard not set".to_string(),
+            )))
         }
     }
 

service-providers/authenticator/src/authenticator.rs

@@ -1,7 +1,7 @@
 // Copyright 2024 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
-use std::path::Path;
+use std::{net::IpAddr, path::Path, time::SystemTime};
 
 use futures::channel::oneshot;
 use ipnetwork::IpNetwork;
@@ -31,6 +31,7 @@ pub struct Authenticator {
     custom_topology_provider: Option<Box<dyn TopologyProvider + Send + Sync>>,
     custom_gateway_transceiver: Option<Box<dyn GatewayTransceiver + Send + Sync>>,
     wireguard_gateway_data: WireguardGatewayData,
+    used_private_network_ips: Vec<IpAddr>,
     response_rx: UnboundedReceiver<PeerControlResponse>,
     shutdown: Option<TaskClient>,
     on_start: Option<oneshot::Sender<OnStartData>>,
@@ -40,6 +41,7 @@ impl Authenticator {
     pub fn new(
         config: Config,
         wireguard_gateway_data: WireguardGatewayData,
+        used_private_network_ips: Vec<IpAddr>,
         response_rx: UnboundedReceiver<PeerControlResponse>,
     ) -> Self {
         Self {
@@ -48,6 +50,7 @@ impl Authenticator {
             custom_topology_provider: None,
             custom_gateway_transceiver: None,
             wireguard_gateway_data,
+            used_private_network_ips,
             response_rx,
             shutdown: None,
             on_start: None,
@@ -128,13 +131,26 @@ impl Authenticator {
 
         let self_address = *mixnet_client.nym_address();
 
+        let used_private_network_ips =
+            std::collections::BTreeSet::from_iter(self.used_private_network_ips.iter());
         let private_ip_network = IpNetwork::new(
             self.config.authenticator.private_ip,
             self.config.authenticator.private_network_prefix,
         )?;
+        let now = SystemTime::now();
+        let free_private_network_ips = private_ip_network
+            .iter()
+            .map(|ip| {
+                if used_private_network_ips.contains(&ip) {
+                    (ip, Some(now))
+                } else {
+                    (ip, None)
+                }
+            })
+            .collect();
         let mixnet_listener = crate::mixnet_listener::MixnetListener::new(
             self.config,
-            private_ip_network,
+            free_private_network_ips,
             self.wireguard_gateway_data,
             self.response_rx,
             mixnet_client,

service-providers/authenticator/src/cli/run.rs

@@ -55,7 +55,7 @@ pub(crate) async fn execute(args: &Run) -> Result<(), AuthenticatorError> {
         handler.run().await;
     });
     let mut server =
-        nym_authenticator::Authenticator::new(config, wireguard_gateway_data, response_rx);
+        nym_authenticator::Authenticator::new(config, wireguard_gateway_data, vec![], response_rx);
     if let Some(custom_mixnet) = &args.common_args.custom_mixnet {
         server = server.with_stored_topology(custom_mixnet)?
     }

service-providers/authenticator/src/mixnet_listener.rs

@@ -8,7 +8,6 @@ use std::{
 
 use crate::{error::AuthenticatorError, peer_manager::PeerManager};
 use futures::StreamExt;
-use ipnetwork::IpNetwork;
 use nym_authenticator_requests::v1::{
     self,
     request::{AuthenticatorRequest, AuthenticatorRequestData},
@@ -67,7 +66,7 @@ pub(crate) struct MixnetListener {
 impl MixnetListener {
     pub fn new(
         config: Config,
-        private_ip_network: IpNetwork,
+        free_private_network_ips: PrivateIPs,
         wireguard_gateway_data: WireguardGatewayData,
         response_rx: UnboundedReceiver<PeerControlResponse>,
         mixnet_client: nym_sdk::mixnet::MixnetClient,
@@ -75,7 +74,6 @@ impl MixnetListener {
     ) -> Self {
         let timeout_check_interval =
             IntervalStream::new(tokio::time::interval(DEFAULT_REGISTRATION_TIMEOUT_CHECK));
-        let free_private_network_ips = private_ip_network.iter().map(|ip| (ip, None)).collect();
         MixnetListener {
             config,
             mixnet_client,