Feature/validator api shutdown (#1573)

* Rewarded set updater shutdown (partial) handling

* Shutdown handling in monitor

* Remove shutdown from packet receiver

* Configurable shutdown timeout

* Select on test_run too

* Remove unnecessary await/async

* Add bias to shutdown select and concurrency for big tasks

* Put cpu-bound packet prep on separate thread, to avoid blocking

* Use a better fit timeout value

* Fix clippy warnings

* Update changelog

* Fix wasm client

validator-api/src/node_status_api/cache.rs

@@ -123,17 +123,28 @@ impl NodeStatusCacheRefresher {
         let mut fallback_interval = time::interval(self.fallback_caching_interval);
         while !shutdown.is_shutdown() {
             tokio::select! {
+                biased;
+                _ = shutdown.recv() => {
+                    log::trace!("NodeStatusCacheRefresher: Received shutdown");
+                }
                 // Update node status cache when the contract cache / validator cache is updated
                 Ok(_) = self.contract_cache_listener.changed() => {
-                    self.update_on_notify(&mut fallback_interval).await;
+                    tokio::select! {
+                        _ = self.update_on_notify(&mut fallback_interval) => (),
+                        _ = shutdown.recv() => {
+                            log::trace!("NodeStatusCacheRefresher: Received shutdown");
+                        }
+                    }
                 }
                 // ... however, if we don't receive any notifications we fall back to periodic
                 // refreshes
                 _ = fallback_interval.tick() => {
-                    self.update_on_timer().await;
-                }
-                _ = shutdown.recv() => {
-                    log::trace!("NodeStatusCacheRefresher: Received shutdown");
+                    tokio::select! {
+                        _ = self.update_on_timer() => (),
+                        _ = shutdown.recv() => {
+                            log::trace!("NodeStatusCacheRefresher: Received shutdown");
+                        }
+                    }
                 }
             }
         }

validator-api/src/node_status_api/uptime_updater.rs

@@ -72,14 +72,20 @@ impl HistoricalUptimeUpdater {
         while !shutdown.is_shutdown() {
             tokio::select! {
                 _ = sleep(ONE_DAY) => {
-                    if let Err(err) = self.update_uptimes().await {
-                        // normally that would have been a warning rather than an error,
-                        // however, in this case it implies some underlying issues with our database
-                        // that might affect the entire program
-                        error!(
-                            "We failed to update daily uptimes of active nodes - {}",
-                            err
-                        )
+                    tokio::select! {
+                        biased;
+                        _ = shutdown.recv() => {
+                            trace!("UpdateHandler: Received shutdown");
+                        }
+                        Err(err) = self.update_uptimes() => {
+                            // normally that would have been a warning rather than an error,
+                            // however, in this case it implies some underlying issues with our database
+                            // that might affect the entire program
+                            error!(
+                                "We failed to update daily uptimes of active nodes - {}",
+                                err
+                            );
+                        }
                     }
                 }
                 _ = shutdown.recv() => {