feature: CancellationToken-based shutdowns (#5325)

* initial stub for ShutdownToken

* attempting to start using new ShutdownManager in NymNode

* migrated verloc tasks

* added custom shutdown signal registration

* integrated legacy task support

* migrated additional tasks inside nym-node

* removed import thats unused in wasm

* apply review comments

* windows fixes

nym-node/src/node/http/mod.rs

@@ -6,9 +6,7 @@ use axum::extract::ConnectInfo;
 use axum::middleware::AddExtension;
 use axum::serve::Serve;
 use axum::Router;
-use nym_task::TaskClient;
 use std::net::SocketAddr;
-use tracing::{debug, error};
 
 pub use router::{api, HttpServerConfig, NymNodeRouter};
 
@@ -19,47 +17,4 @@ pub mod state;
 
 type InnerService = IntoMakeServiceWithConnectInfo<Router, SocketAddr>;
 type ConnectInfoExt = AddExtension<Router, ConnectInfo<SocketAddr>>;
-pub type ServeService = Serve<InnerService, ConnectInfoExt>;
-
-pub struct NymNodeHttpServer {
-    task_client: Option<TaskClient>,
-    inner: ServeService,
-}
-
-impl NymNodeHttpServer {
-    pub(crate) fn new(inner: ServeService) -> Self {
-        NymNodeHttpServer {
-            task_client: None,
-            inner,
-        }
-    }
-
-    #[must_use]
-    pub fn with_task_client(mut self, task_client: TaskClient) -> Self {
-        self.task_client = Some(task_client);
-        self
-    }
-
-    async fn run_server_forever(server: ServeService) {
-        if let Err(err) = server.await {
-            error!("the HTTP server has terminated with the error: {err}");
-        } else {
-            error!("the HTTP server has terminated with producing any errors");
-        }
-    }
-
-    pub async fn run(self) {
-        if let Some(mut task_client) = self.task_client {
-            tokio::select! {
-                _ = task_client.recv_with_delay() => {
-                    debug!("NymNodeHTTPServer: Received shutdown");
-                }
-                _ = Self::run_server_forever(self.inner) => { }
-            }
-        } else {
-            Self::run_server_forever(self.inner).await
-        }
-
-        debug!("NymNodeHTTPServer: Exiting");
-    }
-}
+pub type NymNodeHttpServer = Serve<InnerService, ConnectInfoExt>;

nym-node/src/node/http/router/mod.rs

@@ -175,12 +175,10 @@ impl NymNodeRouter {
                 source,
             })?;
 
-        let axum_server = axum::serve(
+        Ok(axum::serve(
             listener,
             self.inner
                 .into_make_service_with_connect_info::<SocketAddr>(),
-        );
-
-        Ok(NymNodeHttpServer::new(axum_server))
+        ))
     }
 }

nym-node/src/node/metrics/aggregator.rs

@@ -6,6 +6,7 @@ use futures::StreamExt;
 use nym_node_metrics::events::{
     events_channels, MetricEventsReceiver, MetricEventsSender, MetricsEvent,
 };
+use nym_task::ShutdownToken;
 use std::any;
 use std::any::TypeId;
 use std::collections::HashMap;
@@ -24,11 +25,11 @@ pub(crate) struct MetricsAggregator {
     // registered_handlers: HashMap<TypeId, Box<dyn Any + Send + Sync + 'static>>,
     event_sender: MetricEventsSender,
     event_receiver: MetricEventsReceiver,
-    shutdown: nym_task::TaskClient,
+    shutdown: ShutdownToken,
 }
 
 impl MetricsAggregator {
-    pub fn new(handlers_update_interval: Duration, shutdown: nym_task::TaskClient) -> Self {
+    pub fn new(handlers_update_interval: Duration, shutdown: ShutdownToken) -> Self {
         let (event_sender, event_receiver) = events_channels();
 
         MetricsAggregator {
@@ -113,7 +114,7 @@ impl MetricsAggregator {
         loop {
             tokio::select! {
                 biased;
-                _ = self.shutdown.recv() => {
+                _ = self.shutdown.cancelled() => {
                     debug!("MetricsAggregator: Received shutdown");
                     break;
                 }

nym-node/src/node/metrics/console_logger.rs

@@ -4,7 +4,7 @@
 use human_repr::HumanCount;
 use human_repr::HumanThroughput;
 use nym_node_metrics::NymNodeMetrics;
-use nym_task::TaskClient;
+use nym_task::ShutdownToken;
 use std::time::Duration;
 use time::OffsetDateTime;
 use tokio::task::JoinHandle;
@@ -49,14 +49,14 @@ pub(crate) struct ConsoleLogger {
     logging_delay: Duration,
     at_last_update: AtLastUpdate,
     metrics: NymNodeMetrics,
-    shutdown: TaskClient,
+    shutdown: ShutdownToken,
 }
 
 impl ConsoleLogger {
     pub(crate) fn new(
         logging_delay: Duration,
         metrics: NymNodeMetrics,
-        shutdown: TaskClient,
+        shutdown: ShutdownToken,
     ) -> Self {
         ConsoleLogger {
             logging_delay,
@@ -126,11 +126,12 @@ impl ConsoleLogger {
     async fn run(&mut self) {
         trace!("Starting ConsoleLogger");
         let mut interval = interval_at(Instant::now() + self.logging_delay, self.logging_delay);
-        while !self.shutdown.is_shutdown() {
+        loop {
             tokio::select! {
                 biased;
-               _ = self.shutdown.recv() => {
+               _ = self.shutdown.cancelled() => {
                     trace!("ConsoleLogger: Received shutdown");
+                    break
                 }
                 _ = interval.tick() => self.log_running_stats().await,
             };

nym-node/src/node/mixnet/handler.rs

@@ -38,10 +38,7 @@ impl ConnectionHandler {
         tcp_stream: TcpStream,
         remote_address: SocketAddr,
     ) -> Self {
-        let mut task_client = shared.task_client.fork(remote_address.to_string());
-        // we don't want dropped connections to cause global shutdown
-        task_client.disarm();
-
+        let shutdown = shared.shutdown.child_token(remote_address.to_string());
         shared.metrics.network.new_active_ingress_mixnet_client();
 
         ConnectionHandler {
@@ -51,7 +48,7 @@ impl ConnectionHandler {
                 mixnet_forwarder: shared.mixnet_forwarder.clone(),
                 final_hop: shared.final_hop.clone(),
                 metrics: shared.metrics.clone(),
-                task_client,
+                shutdown,
             },
             remote_address,
             mixnet_connection: Framed::new(tcp_stream, NymCodec),
@@ -165,11 +162,12 @@ impl ConnectionHandler {
         )
     )]
     pub(crate) async fn handle_stream(&mut self) {
-        while !self.shared.task_client.is_shutdown() {
+        loop {
             tokio::select! {
                 biased;
-                _ = self.shared.task_client.recv() => {
+                _ = self.shared.shutdown.cancelled() => {
                     trace!("connection handler: received shutdown");
+                    break
                 }
                 maybe_framed_nym_packet = self.mixnet_connection.next() => {
                     match maybe_framed_nym_packet {
@@ -186,6 +184,7 @@ impl ConnectionHandler {
                 }
             }
         }
+
         debug!("exiting and closing connection");
     }
 }

nym-node/src/node/mixnet/listener.rs

@@ -2,14 +2,14 @@
 // SPDX-License-Identifier: GPL-3.0-only
 
 use crate::node::mixnet::SharedData;
-use nym_task::TaskClient;
+use nym_task::ShutdownToken;
 use std::net::SocketAddr;
 use tokio::task::JoinHandle;
 use tracing::{error, info, trace};
 
 pub(crate) struct Listener {
     bind_address: SocketAddr,
-    shutdown: TaskClient,
+    shutdown: ShutdownToken,
     shared_data: SharedData,
 }
 
@@ -17,7 +17,7 @@ impl Listener {
     pub(crate) fn new(bind_address: SocketAddr, shared_data: SharedData) -> Self {
         Listener {
             bind_address,
-            shutdown: shared_data.task_client.fork("socket-listener"),
+            shutdown: shared_data.shutdown.clone_with_suffix("socket-listener"),
             shared_data,
         }
     }
@@ -29,19 +29,15 @@ impl Listener {
             Ok(listener) => listener,
             Err(err) => {
                 error!("Failed to bind to {}: {err}. Are you sure nothing else is running on the specified port and your user has sufficient permission to bind to the requested address?", self.bind_address);
-
-                // that's a bit gnarly, but we need to make sure we trigger shutdown
-                let mut shutdown_bomb = self.shutdown.fork("shutdown-bomb");
-                shutdown_bomb.rearm();
-                drop(shutdown_bomb);
+                self.shutdown.cancel();
                 return;
             }
         };
 
-        while !self.shutdown.is_shutdown() {
+        loop {
             tokio::select! {
                 biased;
-                _ = self.shutdown.recv() => {
+                _ = self.shutdown.cancelled() => {
                     trace!("mixnet listener: received shutdown");
                 }
                 connection = tcp_listener.accept() => {

nym-node/src/node/mixnet/packet_forwarding.rs

@@ -9,6 +9,7 @@ use nym_mixnet_client::SendWithoutResponse;
 use nym_node_metrics::NymNodeMetrics;
 use nym_nonexhaustive_delayqueue::{Expired, NonExhaustiveDelayQueue};
 use nym_sphinx_forwarding::packet::MixPacket;
+use nym_task::ShutdownToken;
 use std::io;
 use tokio::time::Instant;
 use tracing::{debug, error, trace, warn};
@@ -21,11 +22,11 @@ pub struct PacketForwarder<C> {
 
     packet_sender: MixForwardingSender,
     packet_receiver: MixForwardingReceiver,
-    shutdown: nym_task::TaskClient,
+    shutdown: ShutdownToken,
 }
 
 impl<C> PacketForwarder<C> {
-    pub fn new(client: C, metrics: NymNodeMetrics, shutdown: nym_task::TaskClient) -> Self {
+    pub fn new(client: C, metrics: NymNodeMetrics, shutdown: ShutdownToken) -> Self {
         let (packet_sender, packet_receiver) = mix_forwarding_channels();
 
         PacketForwarder {
@@ -123,7 +124,7 @@ impl<C> PacketForwarder<C> {
         loop {
             tokio::select! {
                 biased;
-                _ = self.shutdown.recv() => {
+                _ = self.shutdown.cancelled() => {
                     debug!("PacketForwarder: Received shutdown");
                     break;
                 }

nym-node/src/node/mixnet/shared/mod.rs

@@ -11,7 +11,7 @@ use nym_node_metrics::NymNodeMetrics;
 use nym_sphinx_forwarding::packet::MixPacket;
 use nym_sphinx_framing::processing::{MixProcessingResult, PacketProcessingError};
 use nym_sphinx_types::DestinationAddressBytes;
-use nym_task::TaskClient;
+use nym_task::ShutdownToken;
 use std::io;
 use std::net::{IpAddr, SocketAddr};
 use std::sync::Arc;
@@ -43,7 +43,6 @@ impl ProcessingConfig {
 }
 
 // explicitly do NOT derive clone as we want to manually apply relevant suffixes to the task clients
-// as well as immediately disarm them
 pub(crate) struct SharedData {
     pub(super) processing_config: ProcessingConfig,
     // TODO: this type is not `Zeroize` : (
@@ -56,7 +55,7 @@ pub(crate) struct SharedData {
     pub(super) final_hop: SharedFinalHopData,
 
     pub(super) metrics: NymNodeMetrics,
-    pub(super) task_client: TaskClient,
+    pub(super) shutdown: ShutdownToken,
 }
 
 impl SharedData {
@@ -66,7 +65,7 @@ impl SharedData {
         mixnet_forwarder: MixForwardingSender,
         final_hop: SharedFinalHopData,
         metrics: NymNodeMetrics,
-        task_client: TaskClient,
+        shutdown: ShutdownToken,
     ) -> Self {
         SharedData {
             processing_config,
@@ -74,7 +73,7 @@ impl SharedData {
             mixnet_forwarder,
             final_hop,
             metrics,
-            task_client,
+            shutdown,
         }
     }
 
@@ -144,13 +143,10 @@ impl SharedData {
             .mixnet_forwarder
             .forward_packet(PacketToForward::new(packet, delay_until))
             .is_err()
-            && !self.task_client.is_shutdown()
+            && !self.shutdown.is_cancelled()
         {
             error!("failed to forward sphinx packet on the channel while the process is not going through the shutdown!");
-            // this is a critical error, we're in uncharted lands, we have to shut down
-            let mut shutdown_bomb = self.task_client.fork("shutdown bomb");
-            shutdown_bomb.rearm();
-            drop(shutdown_bomb)
+            self.shutdown.cancel();
         }
     }
 

nym-node/src/node/mod.rs

@@ -43,7 +43,7 @@ use nym_node_metrics::NymNodeMetrics;
 use nym_node_requests::api::v1::node::models::{AnnouncePorts, NodeDescription};
 use nym_sphinx_acknowledgements::AckKey;
 use nym_sphinx_addressing::Recipient;
-use nym_task::{TaskClient, TaskManager};
+use nym_task::{ShutdownManager, ShutdownToken, TaskClient};
 use nym_validator_client::client::NymApiClientExt;
 use nym_validator_client::models::NodeRefreshBody;
 use nym_validator_client::{NymApiClient, UserAgent};
@@ -351,6 +351,7 @@ impl From<WireguardData> for nym_wireguard::WireguardData {
 pub(crate) struct NymNode {
     config: Config,
     accepted_operator_terms_and_conditions: bool,
+    shutdown_manager: ShutdownManager,
 
     description: NodeDescription,
 
@@ -447,6 +448,10 @@ impl NymNode {
             wireguard: Some(wireguard_data),
             config,
             accepted_operator_terms_and_conditions: false,
+            shutdown_manager: ShutdownManager::new("NymNode")
+                .with_legacy_task_manager()
+                .with_default_shutdown_signals()
+                .map_err(|source| NymNodeError::ShutdownSignalFailure { source })?,
         })
     }
 
@@ -823,7 +828,7 @@ impl NymNode {
         }
     }
 
-    pub(crate) fn start_verloc_measurements(&self, shutdown: TaskClient) {
+    pub(crate) fn start_verloc_measurements(&self) {
         info!(
             "Starting the [verloc] round-trip-time measurer on {} ...",
             self.config.verloc.bind_address
@@ -845,8 +850,11 @@ impl NymNode {
         .retry_timeout(self.config.verloc.debug.retry_timeout)
         .build();
 
-        let mut verloc_measurer =
-            VerlocMeasurer::new(config, self.ed25519_identity_keys.clone(), shutdown);
+        let mut verloc_measurer = VerlocMeasurer::new(
+            config,
+            self.ed25519_identity_keys.clone(),
+            self.shutdown_manager.clone_token("verloc"),
+        );
         verloc_measurer.set_shared_state(self.verloc_stats.clone());
         tokio::spawn(async move { verloc_measurer.run().await });
     }
@@ -855,14 +863,14 @@ impl NymNode {
         &self,
         active_clients_store: ActiveClientsStore,
         active_egress_mixnet_connections: ActiveConnections,
-        shutdown: TaskClient,
+        shutdown: ShutdownToken,
     ) -> MetricEventsSender {
         info!("setting up node metrics...");
 
         // aggregator (to listen for any metrics events)
         let mut metrics_aggregator = MetricsAggregator::new(
             self.config.metrics.debug.aggregator_update_rate,
-            shutdown.fork("aggregator"),
+            shutdown.clone_with_suffix("aggregator"),
         );
 
         // >>>> START: register all relevant handlers for custom events
@@ -924,12 +932,9 @@ impl NymNode {
             ConsoleLogger::new(
                 self.config.metrics.debug.console_logging_update_interval,
                 self.metrics.clone(),
-                shutdown.named("metrics-console-logger"),
+                shutdown.clone_with_suffix("metrics-console-logger"),
             )
             .start();
-        } else {
-            let mut shutdown = shutdown;
-            shutdown.disarm()
         }
 
         let events_sender = metrics_aggregator.sender();
@@ -943,7 +948,7 @@ impl NymNode {
     pub(crate) fn start_mixnet_listener(
         &self,
         active_clients_store: &ActiveClientsStore,
-        shutdown: TaskClient,
+        shutdown: ShutdownToken,
     ) -> (MixForwardingSender, ActiveConnections) {
         let processing_config = ProcessingConfig::new(&self.config);
 
@@ -972,7 +977,7 @@ impl NymNode {
         let mut packet_forwarder = PacketForwarder::new(
             mixnet_client,
             self.metrics.clone(),
-            shutdown.fork("mix-packet-forwarder"),
+            shutdown.clone_with_suffix("mix-packet-forwarder"),
         );
         let mix_packet_sender = packet_forwarder.sender();
         tokio::spawn(async move { packet_forwarder.run().await });
@@ -1005,45 +1010,47 @@ impl NymNode {
         );
         debug!("config: {:#?}", self.config);
 
-        let mut task_manager = TaskManager::default().named("NymNode");
-        let http_server = self
-            .build_http_server()
-            .await?
-            .with_task_client(task_manager.subscribe_named("http-server"));
+        let http_server = self.build_http_server().await?;
         let bind_address = self.config.http.bind_address;
-        tokio::spawn(async move {
+        let server_shutdown = self.shutdown_manager.clone_token("http-server");
+
+        self.shutdown_manager.spawn(async move {
             {
-                info!("started NymNodeHTTPServer on {bind_address}");
-                http_server.run().await
+                info!("starting NymNodeHTTPServer on {bind_address}");
+                http_server
+                    .with_graceful_shutdown(async move { server_shutdown.cancelled().await })
+                    .await
             }
         });
 
         self.try_refresh_remote_nym_api_cache().await;
 
-        self.start_verloc_measurements(task_manager.subscribe_named("verloc-measurements"));
+        self.start_verloc_measurements();
 
         let active_clients_store = ActiveClientsStore::new();
 
         let (mix_packet_sender, active_egress_mixnet_connections) = self.start_mixnet_listener(
             &active_clients_store,
-            task_manager.subscribe_named("mixnet-traffic"),
+            self.shutdown_manager.clone_token("mixnet-traffic"),
         );
 
         let metrics_sender = self.setup_metrics_backend(
             active_clients_store.clone(),
             active_egress_mixnet_connections,
-            task_manager.subscribe_named("metrics"),
+            self.shutdown_manager.clone_token("metrics"),
         );
 
         self.start_gateway_tasks(
             metrics_sender,
             active_clients_store,
             mix_packet_sender,
-            task_manager.subscribe_named("gateway-tasks"),
+            self.shutdown_manager.subscribe_legacy("gateway-tasks"),
         )
         .await?;
 
-        let _ = task_manager.catch_interrupt().await;
+        self.shutdown_manager.close();
+        self.shutdown_manager.wait_for_shutdown_signal().await;
+
         Ok(())
     }
 }