Merge pull request #6158 from nymtech/simon/registration_fail

[bugfix] Disconnect mixnet client if registration fails

Cargo.toml

@@ -215,7 +215,6 @@ base64 = "0.22.1"
 base85rs = "0.1.3"
 bincode = "1.3.3"
 bip39 = { version = "2.0.0", features = ["zeroize"] }
-bit-vec = "0.7.0"                                     # can we unify those?
 bitvec = "1.0.0"
 blake3 = "1.7.0"
 bloomfilter = "3.0.1"
@@ -243,13 +242,11 @@ criterion = "0.5"
 csv = "1.3.1"
 ctr = "0.9.1"
 cupid = "0.6.1"
-curve25519-dalek = "4.1"
 dashmap = "5.5.3"
 # We want https://github.com/DefGuard/wireguard-rs/pull/64 , but there's no crates.io release being pushed out anymore
 defguard_wireguard_rs = { git = "https://github.com/DefGuard/wireguard-rs.git", rev = "v0.4.7" }
 digest = "0.10.7"
 dirs = "6.0"
-doc-comment = "0.3"
 dotenvy = "0.15.6"
 dyn-clone = "1.0.19"
 ecdsa = "0.16"
@@ -265,11 +262,8 @@ futures = "0.3.31"
 futures-util = "0.3"
 generic-array = "0.14.7"
 getrandom = "0.2.10"
-getset = "0.1.5"
 handlebars = "3.5.5"
-headers = "0.4.0"
 hex = "0.4.3"
-hex-literal = "0.3.3"
 hickory-resolver = "0.25"
 hkdf = "0.12.3"
 hmac = "0.12.1"
@@ -293,12 +287,10 @@ lazy_static = "1.5.0"
 ledger-transport = "0.10.0"
 ledger-transport-hid = "0.10.0"
 log = "0.4"
-maxminddb = "0.23.0"
 mime = "0.3.17"
 moka = { version = "0.12", features = ["future"] }
 nix = "0.27.1"
 notify = "5.1.0"
-okapi = "0.7.0"
 once_cell = "1.21.3"
 opentelemetry = "0.19.0"
 opentelemetry-jaeger = "0.18.0"
@@ -307,7 +299,6 @@ pem = "0.8"
 petgraph = "0.6.5"
 pin-project = "1.1"
 pnet_packet = "0.35.0"
-pin-project-lite = "0.2.16"
 publicsuffix = "2.3.0"
 proc_pidinfo = "0.1.3"
 quote = "1"
@@ -315,13 +306,10 @@ rand = "0.8.5"
 rand_chacha = "0.3"
 rand_core = "0.6.3"
 rand_distr = "0.4"
-rand_pcg = "0.3.1"
-rand_seeder = "0.2.3"
 rayon = "1.5.1"
 regex = "1.10.6"
 reqwest = { version = "0.12.15", default-features = false }
 rs_merkle = "1.5.0"
-safer-ffi = "0.1.13"
 schemars = "0.8.22"
 semver = "1.0.26"
 serde = "1.0.219"

documentation/docs/components/operators/snippets/quic-bridge-deployment-script-setup.mdx

@@ -15,6 +15,7 @@ Operators can use [Nym Bridge Configuration Tool](https://github.com/nymtech/nym
 <Steps>
 ###### 1. Download [`quic_bridge_deployment.sh`](https://github.com/nymtech/nym/blob/develop/scripts/nym-node-setup/quic_bridge_deployment.sh) script
 - SSH to your server
+- **Run as root**
 - Download the script and make executable
 ```sh
 wget https://raw.githubusercontent.com/nymtech/nym/refs/heads/develop/scripts/nym-node-setup/quic_bridge_deployment.sh && \
@@ -26,7 +27,7 @@ chmod +x quic_bridge_deployment.sh
 - Optional: open `tmux` in case you will need to run another commands on the VPS
 - Run the script with a command `full_bridge_setup`
 ```sh
-./nym-node-setup/quic_bridge_deployment.sh full_bridge_setup
+./quic_bridge_deployment.sh full_bridge_setup
 ```
 
 ###### 3. Follow the interactive prompts

nym-authenticator-client/src/mixnet_listener.rs

@@ -50,7 +50,7 @@ impl AuthClientMixnetListener {
         }
     }
 
-    async fn run(mut self) -> Self {
+    async fn run(mut self) {
         let mixnet_cancel_token = self.mixnet_client.cancellation_token();
         self.shutdown_token.run_until_cancelled(async {
             loop {
@@ -95,12 +95,8 @@ impl AuthClientMixnetListener {
             tracing::debug!("AuthClientMixnetListener is shutting down");
         }).await;
 
-        self
-    }
-
-    // Disconnects the mixnet client and effectively drop itself, since it doesn't work without one, and reconnecting isn't supported
-    pub async fn disconnect_mixnet_client(self) {
-        if !self.mixnet_client.cancellation_token().is_cancelled() {
+        tracing::debug!("AuthClientMixnetListener: Disconnect mixnet client");
+        if !mixnet_cancel_token.is_cancelled() {
             self.mixnet_client.disconnect().await;
         }
     }
@@ -128,7 +124,7 @@ pub struct AuthClientMixnetListenerHandle {
     message_sender: MixnetMessageInputSender,
     cancellation_token: CancellationToken,
     mixnet_cancellation_token: CancellationToken,
-    handle: JoinHandle<AuthClientMixnetListener>,
+    handle: JoinHandle<()>,
 }
 
 impl AuthClientMixnetListenerHandle {
@@ -148,13 +144,8 @@ impl AuthClientMixnetListenerHandle {
         // If shutdown was externally called, that call is a no-op
         // If we're only stopping this, it is very much needed
         self.cancellation_token.cancel();
-        match self.handle.await {
-            Ok(auth_client_mixnet_listener) => {
-                auth_client_mixnet_listener.disconnect_mixnet_client().await;
-            }
-            Err(e) => {
-                tracing::error!("Error waiting for auth clients mixnet listener to stop: {e}");
-            }
+        if let Err(e) = self.handle.await {
+            tracing::error!("Error waiting for auth clients mixnet listener to stop: {e}")
         }
     }
 }

nym-gateway-probe/src/lib.rs

@@ -765,7 +765,7 @@ async fn connect_exit(
     );
     // The IPR supports cancellation, but it's unused in the gateway probe
     let cancel_token = CancellationToken::new();
-    let mut ipr_client = IprClientConnect::new(mixnet_client, cancel_token).await;
+    let mut ipr_client = IprClientConnect::new(mixnet_client, cancel_token);
 
     let maybe_ip_pair = ipr_client.connect(exit_router_address).await;
     let mixnet_client = ipr_client.into_mixnet_client();

nym-ip-packet-client/src/connect.rs

@@ -43,7 +43,7 @@ pub struct IprClientConnect {
 }
 
 impl IprClientConnect {
-    pub async fn new(mixnet_client: MixnetClient, cancel_token: CancellationToken) -> Self {
+    pub fn new(mixnet_client: MixnetClient, cancel_token: CancellationToken) -> Self {
         Self {
             mixnet_client,
             connected: ConnectionState::Disconnected,

nym-registration-client/src/lib.rs

@@ -9,6 +9,7 @@ use nym_credentials_interface::TicketType;
 use nym_ip_packet_client::IprClientConnect;
 use nym_registration_common::AssignedAddresses;
 use nym_sdk::mixnet::{EventReceiver, MixnetClient, Recipient};
+use tracing::debug;
 
 use crate::config::RegistrationClientConfig;
 
@@ -34,23 +35,49 @@ pub struct RegistrationClient {
     event_rx: EventReceiver,
 }
 
+// Bundle of an actual error and the underlying mixnet client so it can be shutdown correctly if needed
+struct RegistrationError {
+    mixnet_client: Option<MixnetClient>,
+    source: crate::RegistrationClientError,
+}
+
 impl RegistrationClient {
-    async fn register_mix_exit(self) -> Result<RegistrationResult, RegistrationClientError> {
+    async fn register_mix_exit(self) -> Result<RegistrationResult, RegistrationError> {
         let entry_mixnet_gateway_ip = self.config.entry.node.ip_address;
 
         let exit_mixnet_gateway_ip = self.config.exit.node.ip_address;
 
-        let ipr_address = self.config.exit.node.ipr_address.ok_or(
-            RegistrationClientError::NoIpPacketRouterAddress {
-                node_id: self.config.exit.node.identity.to_base58_string(),
-            },
-        )?;
+        let Some(ipr_address) = self.config.exit.node.ipr_address else {
+            return Err(RegistrationError {
+                mixnet_client: Some(self.mixnet_client),
+                source: RegistrationClientError::NoIpPacketRouterAddress {
+                    node_id: self.config.exit.node.identity.to_base58_string(),
+                },
+            });
+        };
+
         let mut ipr_client =
-            IprClientConnect::new(self.mixnet_client, self.cancel_token.clone()).await;
-        let interface_addresses = ipr_client
-            .connect(ipr_address)
+            IprClientConnect::new(self.mixnet_client, self.cancel_token.child_token());
+
+        let interface_addresses = match self
+            .cancel_token
+            .run_until_cancelled(ipr_client.connect(ipr_address))
             .await
-            .map_err(RegistrationClientError::ConnectToIpPacketRouter)?;
+        {
+            Some(Ok(addr)) => addr,
+            Some(Err(e)) => {
+                return Err(RegistrationError {
+                    mixnet_client: Some(ipr_client.into_mixnet_client()),
+                    source: RegistrationClientError::ConnectToIpPacketRouter(e),
+                });
+            }
+            None => {
+                return Err(RegistrationError {
+                    mixnet_client: Some(ipr_client.into_mixnet_client()),
+                    source: RegistrationClientError::Cancelled,
+                });
+            }
+        };
 
         Ok(RegistrationResult::Mixnet(Box::new(
             MixnetRegistrationResult {
@@ -67,18 +94,24 @@ impl RegistrationClient {
         )))
     }
 
-    async fn register_wg(self) -> Result<RegistrationResult, RegistrationClientError> {
-        let entry_auth_address = self.config.entry.node.authenticator_address.ok_or(
-            RegistrationClientError::AuthenticationNotPossible {
-                node_id: self.config.entry.node.identity.to_base58_string(),
-            },
-        )?;
+    async fn register_wg(self) -> Result<RegistrationResult, RegistrationError> {
+        let Some(entry_auth_address) = self.config.entry.node.authenticator_address else {
+            return Err(RegistrationError {
+                mixnet_client: Some(self.mixnet_client),
+                source: RegistrationClientError::AuthenticationNotPossible {
+                    node_id: self.config.entry.node.identity.to_base58_string(),
+                },
+            });
+        };
 
-        let exit_auth_address = self.config.exit.node.authenticator_address.ok_or(
-            RegistrationClientError::AuthenticationNotPossible {
-                node_id: self.config.exit.node.identity.to_base58_string(),
-            },
-        )?;
+        let Some(exit_auth_address) = self.config.exit.node.authenticator_address else {
+            return Err(RegistrationError {
+                mixnet_client: Some(self.mixnet_client),
+                source: RegistrationClientError::AuthenticationNotPossible {
+                    node_id: self.config.exit.node.identity.to_base58_string(),
+                },
+            });
+        };
 
         let entry_version = self.config.entry.node.version;
         tracing::debug!("Entry gateway version: {entry_version}");
@@ -87,8 +120,10 @@ impl RegistrationClient {
 
         // Start the auth client mixnet listener, which will listen for incoming messages from the
         // mixnet and rebroadcast them to the auth clients.
+        // From this point on, we don't need to care about the mixnet client anymore
         let mixnet_listener =
-            AuthClientMixnetListener::new(self.mixnet_client, self.cancel_token.clone()).start();
+            AuthClientMixnetListener::new(self.mixnet_client, self.cancel_token.child_token())
+                .start();
 
         let mut entry_auth_client = AuthenticatorClient::new(
             mixnet_listener.subscribe(),
@@ -115,24 +150,33 @@ impl RegistrationClient {
         let exit_fut = exit_auth_client
             .register_wireguard(&*self.bandwidth_controller, TicketType::V1WireguardExit);
 
-        let (entry, exit) = Box::pin(async { tokio::join!(entry_fut, exit_fut) }).await;
+        let (entry, exit) = Box::pin(
+            self.cancel_token
+                .run_until_cancelled(async { tokio::join!(entry_fut, exit_fut) }),
+        )
+        .await
+        .ok_or(RegistrationError {
+            mixnet_client: None,
+            source: RegistrationClientError::Cancelled,
+        })?;
 
-        let entry =
-            entry.map_err(
-                |source| RegistrationClientError::EntryGatewayRegisterWireguard {
-                    gateway_id: self.config.entry.node.identity.to_base58_string(),
-                    authenticator_address: Box::new(entry_auth_address),
-                    source: Box::new(source),
-                },
-            )?;
-        let exit =
-            exit.map_err(
-                |source| RegistrationClientError::ExitGatewayRegisterWireguard {
-                    gateway_id: self.config.exit.node.identity.to_base58_string(),
-                    authenticator_address: Box::new(exit_auth_address),
-                    source: Box::new(source),
-                },
-            )?;
+        let entry = entry.map_err(|source| RegistrationError {
+            mixnet_client: None,
+            source: RegistrationClientError::EntryGatewayRegisterWireguard {
+                gateway_id: self.config.entry.node.identity.to_base58_string(),
+                authenticator_address: Box::new(entry_auth_address),
+                source: Box::new(source),
+            },
+        })?;
+
+        let exit = exit.map_err(|source| RegistrationError {
+            mixnet_client: None,
+            source: RegistrationClientError::EntryGatewayRegisterWireguard {
+                gateway_id: self.config.exit.node.identity.to_base58_string(),
+                authenticator_address: Box::new(exit_auth_address),
+                source: Box::new(source),
+            },
+        })?;
 
         Ok(RegistrationResult::Wireguard(Box::new(
             WireguardRegistrationResult {
@@ -147,16 +191,23 @@ impl RegistrationClient {
     }
 
     pub async fn register(self) -> Result<RegistrationResult, RegistrationClientError> {
-        self.cancel_token
-            .clone()
-            .run_until_cancelled(async {
-                if self.config.two_hops {
-                    self.register_wg().await
-                } else {
-                    self.register_mix_exit().await
+        let registration_result = if self.config.two_hops {
+            self.register_wg().await
+        } else {
+            self.register_mix_exit().await
+        };
+
+        // If we failed to register, and we were the owner of the mixnet client, shut it down
+        match registration_result {
+            Ok(result) => Ok(result),
+            Err(error) => {
+                debug!("Registration failed");
+                if let Some(mixnet_client) = error.mixnet_client {
+                    debug!("Shutting down mixnet client");
+                    mixnet_client.disconnect().await;
                 }
-            })
-            .await
-            .ok_or(RegistrationClientError::Cancelled)?
+                Err(error.source)
+            }
+        }
     }
 }