diff --git a/CHANGELOG.md b/CHANGELOG.md index b6adc1c809..931457972f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,7 +53,7 @@ Post 1.0.0 release, the changelog format is based on [Keep a Changelog](https:// - validator: fixed local docker-compose setup to work on Apple M1 ([#1329]) - explorer-api: listen out for SIGTERM and SIGQUIT too, making it play nicely as a system service ([#1482]). - network-requester: fix filter for suffix-only domains ([#1487]) -- validator-api: listen out for SIGTERM and SIGQUIT too, making it play nicely as a system service ([#1496]). +- validator-api: listen out for SIGTERM and SIGQUIT too, making it play nicely as a system service; cleaner shutdown, without panics ([#1496], [#1573]). ### Changed @@ -99,6 +99,7 @@ Post 1.0.0 release, the changelog format is based on [Keep a Changelog](https:// [#1496]: https://github.com/nymtech/nym/pull/1496 [#1503]: https://github.com/nymtech/nym/pull/1503 [#1520]: https://github.com/nymtech/nym/pull/1520 +[#1573]: https://github.com/nymtech/nym/pull/1573 ## [v1.0.1](https://github.com/nymtech/nym/tree/v1.0.1) (2022-05-04) diff --git a/clients/client-core/src/client/real_messages_control/acknowledgement_control/input_message_listener.rs b/clients/client-core/src/client/real_messages_control/acknowledgement_control/input_message_listener.rs index 2e34451de3..e44eef38c8 100644 --- a/clients/client-core/src/client/real_messages_control/acknowledgement_control/input_message_listener.rs +++ b/clients/client-core/src/client/real_messages_control/acknowledgement_control/input_message_listener.rs @@ -133,7 +133,6 @@ where let prepared_fragment = self .message_preparer .prepare_chunk_for_sending(chunk_clone, topology, &self.ack_key, &recipient) - .await .unwrap(); real_messages.push(RealMessage::new( diff --git a/clients/client-core/src/client/real_messages_control/acknowledgement_control/retransmission_request_listener.rs b/clients/client-core/src/client/real_messages_control/acknowledgement_control/retransmission_request_listener.rs index d0081f4d04..3e9e2ee65a 100644 --- a/clients/client-core/src/client/real_messages_control/acknowledgement_control/retransmission_request_listener.rs +++ b/clients/client-core/src/client/real_messages_control/acknowledgement_control/retransmission_request_listener.rs @@ -83,7 +83,6 @@ where let prepared_fragment = self .message_preparer .prepare_chunk_for_sending(chunk_clone, topology_ref, &self.ack_key, packet_recipient) - .await .unwrap(); // if we have the ONLY strong reference to the ack data, it means it was removed from the diff --git a/clients/webassembly/src/client/mod.rs b/clients/webassembly/src/client/mod.rs index 573ef60bc7..57e20931ca 100644 --- a/clients/webassembly/src/client/mod.rs +++ b/clients/webassembly/src/client/mod.rs @@ -197,7 +197,6 @@ impl NymClient { // don't bother with acks etc. for time being let prepared_fragment = message_preparer .prepare_chunk_for_sending(message_chunk, topology, &self.ack_key, &recipient) - .await .unwrap(); console_warn!("packet is going to have round trip time of {:?}, but we're not going to do anything for acks anyway ", prepared_fragment.total_delay); diff --git a/common/nymsphinx/src/preparer/mod.rs b/common/nymsphinx/src/preparer/mod.rs index 3f56f9bada..a51075290e 100644 --- a/common/nymsphinx/src/preparer/mod.rs +++ b/common/nymsphinx/src/preparer/mod.rs @@ -213,7 +213,7 @@ where /// - compute vk_b = g^x || v_b /// - compute sphinx_plaintext = SURB_ACK || g^x || v_b /// - compute sphinx_packet = Sphinx(recipient, sphinx_plaintext) - pub async fn prepare_chunk_for_sending( + pub fn prepare_chunk_for_sending( &mut self, fragment: Fragment, topology: &NymTopology, @@ -222,8 +222,7 @@ where ) -> Result { // create an ack let (ack_delay, surb_ack_bytes) = self - .generate_surb_ack(fragment.fragment_identifier(), topology, ack_key) - .await? + .generate_surb_ack(fragment.fragment_identifier(), topology, ack_key)? .prepare_for_sending(); // TODO: @@ -294,7 +293,7 @@ where } /// Construct an acknowledgement SURB for the given [`FragmentIdentifier`] - async fn generate_surb_ack( + fn generate_surb_ack( &mut self, fragment_id: FragmentIdentifier, topology: &NymTopology, @@ -357,8 +356,7 @@ where // gateways could not distinguish reply packets from normal messages due to lack of said acks // note: the ack delay is irrelevant since we do not know the delay of actual surb let (_, surb_ack_bytes) = self - .generate_surb_ack(reply_id, topology, ack_key) - .await? + .generate_surb_ack(reply_id, topology, ack_key)? .prepare_for_sending(); let zero_pad_len = self.packet_size.plaintext_size() diff --git a/common/task/src/shutdown.rs b/common/task/src/shutdown.rs index 8c8b472459..1fe873e9aa 100644 --- a/common/task/src/shutdown.rs +++ b/common/task/src/shutdown.rs @@ -5,13 +5,14 @@ use std::time::Duration; use tokio::sync::watch::{self, error::SendError}; -const SHUTDOWN_TIMER_SECS: u64 = 5; +const DEFAULT_SHUTDOWN_TIMER_SECS: u64 = 5; /// Used to notify other tasks to gracefully shutdown #[derive(Debug)] pub struct ShutdownNotifier { notify_tx: watch::Sender<()>, notify_rx: Option>, + shutdown_timer_secs: u64, } impl Default for ShutdownNotifier { @@ -20,11 +21,19 @@ impl Default for ShutdownNotifier { Self { notify_tx, notify_rx: Some(notify_rx), + shutdown_timer_secs: DEFAULT_SHUTDOWN_TIMER_SECS, } } } impl ShutdownNotifier { + pub fn new(shutdown_timer_secs: u64) -> Self { + Self { + shutdown_timer_secs, + ..Default::default() + } + } + pub fn subscribe(&self) -> ShutdownListener { ShutdownListener::new( self.notify_rx @@ -50,7 +59,7 @@ impl ShutdownNotifier { _ = tokio::signal::ctrl_c() => { log::info!("Forcing shutdown"); } - _ = tokio::time::sleep(Duration::from_secs(SHUTDOWN_TIMER_SECS)) => { + _ = tokio::time::sleep(Duration::from_secs(self.shutdown_timer_secs)) => { log::info!("Timout reached, forcing shutdown"); }, } diff --git a/validator-api/src/contract_cache/mod.rs b/validator-api/src/contract_cache/mod.rs index a7f2b91b0b..5a3d3d8978 100644 --- a/validator-api/src/contract_cache/mod.rs +++ b/validator-api/src/contract_cache/mod.rs @@ -281,13 +281,21 @@ impl ValidatorCacheRefresher { while !shutdown.is_shutdown() { tokio::select! { _ = interval.tick() => { - if let Err(err) = self.refresh_cache().await { - error!("Failed to refresh validator cache - {}", err); - } else { - // relaxed memory ordering is fine here. worst case scenario network monitor - // will just have to wait for an additional backoff to see the change. - // And so this will not really incur any performance penalties by setting it every loop iteration - self.cache.initialised.store(true, Ordering::Relaxed) + tokio::select! { + biased; + _ = shutdown.recv() => { + trace!("ValidatorCacheRefresher: Received shutdown"); + } + ret = self.refresh_cache() => { + if let Err(err) = ret { + error!("Failed to refresh validator cache - {}", err); + } else { + // relaxed memory ordering is fine here. worst case scenario network monitor + // will just have to wait for an additional backoff to see the change. + // And so this will not really incur any performance penalties by setting it every loop iteration + self.cache.initialised.store(true, Ordering::Relaxed) + } + } } } _ = shutdown.recv() => { diff --git a/validator-api/src/main.rs b/validator-api/src/main.rs index dd4ec49172..96abe8408a 100644 --- a/validator-api/src/main.rs +++ b/validator-api/src/main.rs @@ -568,7 +568,8 @@ async fn run_validator_api(matches: ArgMatches<'static>) -> Result<()> { let signing_nymd_client = Client::new_signing(&config); let liftoff_notify = Arc::new(Notify::new()); - let shutdown = ShutdownNotifier::default(); + // We need a bigger timeout + let shutdown = ShutdownNotifier::new(10); // let's build our rocket! let rocket = setup_rocket( @@ -609,7 +610,8 @@ async fn run_validator_api(matches: ArgMatches<'static>) -> Result<()> { // spawn rewarded set updater let mut rewarded_set_updater = RewardedSetUpdater::new(signing_nymd_client, validator_cache.clone(), storage).await?; - tokio::spawn(async move { rewarded_set_updater.run().await.unwrap() }); + let shutdown_listener = shutdown.subscribe(); + tokio::spawn(async move { rewarded_set_updater.run(shutdown_listener).await.unwrap() }); validator_cache_listener } else { diff --git a/validator-api/src/network_monitor/chunker.rs b/validator-api/src/network_monitor/chunker.rs index c95f8421f5..ec0da08871 100644 --- a/validator-api/src/network_monitor/chunker.rs +++ b/validator-api/src/network_monitor/chunker.rs @@ -12,6 +12,7 @@ use topology::NymTopology; const DEFAULT_AVERAGE_PACKET_DELAY: Duration = Duration::from_millis(200); const DEFAULT_AVERAGE_ACK_DELAY: Duration = Duration::from_millis(200); +#[derive(Clone)] pub(crate) struct Chunker { rng: OsRng, message_preparer: MessagePreparer, @@ -30,7 +31,7 @@ impl Chunker { } } - pub(crate) async fn prepare_packets_from( + pub(crate) fn prepare_packets_from( &mut self, message: Vec, topology: &NymTopology, @@ -40,10 +41,10 @@ impl Chunker { // but without some significant API changes in the `MessagePreparer` this was the easiest // way to being able to have variable sender address. self.message_preparer.set_sender_address(packet_sender); - self.prepare_packets(message, topology, packet_sender).await + self.prepare_packets(message, topology, packet_sender) } - async fn prepare_packets( + fn prepare_packets( &mut self, message: Vec, topology: &NymTopology, @@ -62,7 +63,6 @@ impl Chunker { let prepared_fragment = self .message_preparer .prepare_chunk_for_sending(message_chunk, topology, &ack_key, &packet_sender) - .await .unwrap(); mix_packets.push(prepared_fragment.mix_packet); diff --git a/validator-api/src/network_monitor/monitor/gateways_pinger.rs b/validator-api/src/network_monitor/monitor/gateways_pinger.rs index f7d47ca302..24cd581bf3 100644 --- a/validator-api/src/network_monitor/monitor/gateways_pinger.rs +++ b/validator-api/src/network_monitor/monitor/gateways_pinger.rs @@ -7,6 +7,7 @@ use crypto::asymmetric::identity; use crypto::asymmetric::identity::PUBLIC_KEY_LENGTH; use log::{debug, info, trace, warn}; use std::time::Duration; +use task::ShutdownListener; use tokio::time::{sleep, Instant}; // TODO: should it perhaps be moved to config along other timeout values? @@ -143,10 +144,22 @@ impl GatewayPinger { info!("Pinging all active gateways took {:?}", time_taken); } - pub(crate) async fn run(&self) { - loop { - sleep(self.pinging_interval).await; - self.ping_and_cleanup_all_gateways().await + pub(crate) async fn run(&self, mut shutdown: ShutdownListener) { + while !shutdown.is_shutdown() { + tokio::select! { + _ = sleep(self.pinging_interval) => { + tokio::select! { + biased; + _ = shutdown.recv() => { + trace!("GatewaysPinger: Received shutdown"); + } + _ = self.ping_and_cleanup_all_gateways() => (), + } + } + _ = shutdown.recv() => { + trace!("GatewaysPinger: Received shutdown"); + } + } } } } diff --git a/validator-api/src/network_monitor/monitor/mod.rs b/validator-api/src/network_monitor/monitor/mod.rs index 2cd1950f96..e2d1a07128 100644 --- a/validator-api/src/network_monitor/monitor/mod.rs +++ b/validator-api/src/network_monitor/monitor/mod.rs @@ -122,11 +122,15 @@ impl Monitor { let mut packets = Vec::with_capacity(routes.len()); for route in routes { - packets.push( - self.packet_preparer - .prepare_test_route_viability_packets(route, self.route_test_packets) - .await, - ); + let mut packet_preparer = self.packet_preparer.clone(); + let route = route.clone(); + let route_test_packets = self.route_test_packets; + let gateway_packets = tokio::spawn(async move { + packet_preparer.prepare_test_route_viability_packets(&route, route_test_packets) + }) + .await + .unwrap(); + packets.push(gateway_packets); } self.received_processor.set_route_test_nonce().await; @@ -306,12 +310,20 @@ impl Monitor { .await; self.packet_sender - .spawn_gateways_pinger(self.gateway_ping_interval); + .spawn_gateways_pinger(self.gateway_ping_interval, shutdown.clone()); let mut run_interval = tokio::time::interval(self.run_interval); while !shutdown.is_shutdown() { tokio::select! { - _ = run_interval.tick() => self.test_run().await, + _ = run_interval.tick() => { + tokio::select! { + biased; + _ = shutdown.recv() => { + trace!("UpdateHandler: Received shutdown"); + } + _ = self.test_run() => (), + } + } _ = shutdown.recv() => { trace!("UpdateHandler: Received shutdown"); } diff --git a/validator-api/src/network_monitor/monitor/preparer.rs b/validator-api/src/network_monitor/monitor/preparer.rs index d67eb59629..2173cc04da 100644 --- a/validator-api/src/network_monitor/monitor/preparer.rs +++ b/validator-api/src/network_monitor/monitor/preparer.rs @@ -117,6 +117,7 @@ pub(crate) struct PreparedPackets { pub(super) invalid_gateways: Vec, } +#[derive(Clone)] pub(crate) struct PacketPreparer { system_version: String, chunker: Option, @@ -151,7 +152,7 @@ impl PacketPreparer { } } - async fn wrap_test_packet( + fn wrap_test_packet( &mut self, packet: &TestPacket, topology: &NymTopology, @@ -162,12 +163,11 @@ impl PacketPreparer { if self.chunker.is_none() { self.chunker = Some(Chunker::new(packet_recipient)); } - let mut mix_packets = self - .chunker - .as_mut() - .unwrap() - .prepare_packets_from(packet.to_bytes(), topology, packet_recipient) - .await; + let mut mix_packets = self.chunker.as_mut().unwrap().prepare_packets_from( + packet.to_bytes(), + topology, + packet_recipient, + ); assert_eq!( mix_packets.len(), 1, @@ -351,7 +351,7 @@ impl PacketPreparer { ) } - pub(crate) async fn prepare_test_route_viability_packets( + pub(crate) fn prepare_test_route_viability_packets( &mut self, route: &TestRoute, num: usize, @@ -360,9 +360,7 @@ impl PacketPreparer { let test_packet = route.self_test_packet(); let recipient = self.create_packet_sender(route.gateway()); for _ in 0..num { - let mix_packet = self - .wrap_test_packet(&test_packet, route.topology(), recipient) - .await; + let mix_packet = self.wrap_test_packet(&test_packet, route.topology(), recipient); mix_packets.push(mix_packet) } @@ -451,9 +449,7 @@ impl PacketPreparer { let topology = test_route.substitute_mix(mixnode); // produce n mix packets for _ in 0..self.per_node_test_packets { - let mix_packet = self - .wrap_test_packet(&test_packet, &topology, recipient) - .await; + let mix_packet = self.wrap_test_packet(&test_packet, &topology, recipient); mix_packets.push(mix_packet); } } @@ -476,9 +472,7 @@ impl PacketPreparer { let topology = test_route.substitute_gateway(gateway); // produce n mix packets for _ in 0..self.per_node_test_packets { - let mix_packet = self - .wrap_test_packet(&test_packet, &topology, recipient) - .await; + let mix_packet = self.wrap_test_packet(&test_packet, &topology, recipient); gateway_mix_packets.push(mix_packet); } diff --git a/validator-api/src/network_monitor/monitor/processor.rs b/validator-api/src/network_monitor/monitor/processor.rs index 11d995f1c8..2d504c01c2 100644 --- a/validator-api/src/network_monitor/monitor/processor.rs +++ b/validator-api/src/network_monitor/monitor/processor.rs @@ -140,8 +140,7 @@ impl ReceivedProcessor { self.permit_changer = Some(permit_sender); tokio::spawn(async move { - loop { - let permit = wait_for_permit(&mut permit_receiver, &*inner).await; + while let Some(permit) = wait_for_permit(&mut permit_receiver, &*inner).await { receive_or_release_permit(&mut permit_receiver, permit).await; } @@ -151,16 +150,20 @@ impl ReceivedProcessor { ) { loop { tokio::select! { - permit_receiver = permit_receiver.next() => match permit_receiver.unwrap() { - LockPermit::Release => return, - LockPermit::Free => error!("somehow we got notification that the lock is free to take while we already hold it!"), + permit_receiver = permit_receiver.next() => match permit_receiver { + Some(LockPermit::Release) => return, + Some(LockPermit::Free) => error!("somehow we got notification that the lock is free to take while we already hold it!"), + None => return, }, - messages = inner.packets_receiver.next() => { - for message in messages.expect("packet receiver has died!") { - if let Err(err) = inner.on_message(message) { - warn!(target: "Monitor", "failed to process received gateway message - {}", err) + messages = inner.packets_receiver.next() => match messages { + Some(messages) => { + for message in messages { + if let Err(err) = inner.on_message(message) { + warn!(target: "Monitor", "failed to process received gateway message - {}", err) + } } } + None => return, }, } } @@ -172,14 +175,15 @@ impl ReceivedProcessor { async fn wait_for_permit<'a>( permit_receiver: &mut mpsc::Receiver, inner: &'a Mutex, - ) -> MutexGuard<'a, ReceivedProcessorInner> { + ) -> Option> { loop { - match permit_receiver.next().await.unwrap() { + match permit_receiver.next().await { // we should only ever get this on the very first run - LockPermit::Release => debug!( + Some(LockPermit::Release) => debug!( "somehow got request to drop our lock permit while we do not hold it!" ), - LockPermit::Free => return inner.lock().await, + Some(LockPermit::Free) => return Some(inner.lock().await), + None => return None, } } } diff --git a/validator-api/src/network_monitor/monitor/receiver.rs b/validator-api/src/network_monitor/monitor/receiver.rs index 36913241db..991da5fa05 100644 --- a/validator-api/src/network_monitor/monitor/receiver.rs +++ b/validator-api/src/network_monitor/monitor/receiver.rs @@ -59,6 +59,10 @@ impl PacketReceiver { pub(crate) async fn run(&mut self, mut shutdown: ShutdownListener) { while !shutdown.is_shutdown() { tokio::select! { + biased; + _ = shutdown.recv() => { + trace!("UpdateHandler: Received shutdown"); + } // unwrap here is fine as it can only return a `None` if the PacketSender has died // and if that was the case, then the entire monitor is already in an undefined state update = self.clients_updater.next() => self.process_gateway_update(update.unwrap()), @@ -68,9 +72,6 @@ impl PacketReceiver { Some((_gateway_id, message)) = self.gateways_reader.stream_map().next() => { self.process_gateway_messages(message) } - _ = shutdown.recv() => { - trace!("UpdateHandler: Received shutdown"); - } } } } diff --git a/validator-api/src/network_monitor/monitor/sender.rs b/validator-api/src/network_monitor/monitor/sender.rs index 48fea699dd..c11f38b826 100644 --- a/validator-api/src/network_monitor/monitor/sender.rs +++ b/validator-api/src/network_monitor/monitor/sender.rs @@ -24,6 +24,7 @@ use std::pin::Pin; use std::sync::Arc; use std::task::Poll; use std::time::Duration; +use task::ShutdownListener; use gateway_client::bandwidth::BandwidthController; @@ -176,7 +177,11 @@ impl PacketSender { } } - pub(crate) fn spawn_gateways_pinger(&self, pinging_interval: Duration) { + pub(crate) fn spawn_gateways_pinger( + &self, + pinging_interval: Duration, + shutdown: ShutdownListener, + ) { let gateway_pinger = GatewayPinger::new( self.active_gateway_clients.clone(), self.fresh_gateway_client_data @@ -185,7 +190,7 @@ impl PacketSender { pinging_interval, ); - tokio::spawn(async move { gateway_pinger.run().await }); + tokio::spawn(async move { gateway_pinger.run(shutdown).await }); } fn new_gateway_client_handle( diff --git a/validator-api/src/node_status_api/cache.rs b/validator-api/src/node_status_api/cache.rs index 8a7259679b..9d4b20067c 100644 --- a/validator-api/src/node_status_api/cache.rs +++ b/validator-api/src/node_status_api/cache.rs @@ -123,17 +123,28 @@ impl NodeStatusCacheRefresher { let mut fallback_interval = time::interval(self.fallback_caching_interval); while !shutdown.is_shutdown() { tokio::select! { + biased; + _ = shutdown.recv() => { + log::trace!("NodeStatusCacheRefresher: Received shutdown"); + } // Update node status cache when the contract cache / validator cache is updated Ok(_) = self.contract_cache_listener.changed() => { - self.update_on_notify(&mut fallback_interval).await; + tokio::select! { + _ = self.update_on_notify(&mut fallback_interval) => (), + _ = shutdown.recv() => { + log::trace!("NodeStatusCacheRefresher: Received shutdown"); + } + } } // ... however, if we don't receive any notifications we fall back to periodic // refreshes _ = fallback_interval.tick() => { - self.update_on_timer().await; - } - _ = shutdown.recv() => { - log::trace!("NodeStatusCacheRefresher: Received shutdown"); + tokio::select! { + _ = self.update_on_timer() => (), + _ = shutdown.recv() => { + log::trace!("NodeStatusCacheRefresher: Received shutdown"); + } + } } } } diff --git a/validator-api/src/node_status_api/uptime_updater.rs b/validator-api/src/node_status_api/uptime_updater.rs index 56e7655aae..0e60650f65 100644 --- a/validator-api/src/node_status_api/uptime_updater.rs +++ b/validator-api/src/node_status_api/uptime_updater.rs @@ -72,14 +72,20 @@ impl HistoricalUptimeUpdater { while !shutdown.is_shutdown() { tokio::select! { _ = sleep(ONE_DAY) => { - if let Err(err) = self.update_uptimes().await { - // normally that would have been a warning rather than an error, - // however, in this case it implies some underlying issues with our database - // that might affect the entire program - error!( - "We failed to update daily uptimes of active nodes - {}", - err - ) + tokio::select! { + biased; + _ = shutdown.recv() => { + trace!("UpdateHandler: Received shutdown"); + } + Err(err) = self.update_uptimes() => { + // normally that would have been a warning rather than an error, + // however, in this case it implies some underlying issues with our database + // that might affect the entire program + error!( + "We failed to update daily uptimes of active nodes - {}", + err + ); + } } } _ = shutdown.recv() => { diff --git a/validator-api/src/rewarded_set_updater/mod.rs b/validator-api/src/rewarded_set_updater/mod.rs index 627452d673..d7cef3e373 100644 --- a/validator-api/src/rewarded_set_updater/mod.rs +++ b/validator-api/src/rewarded_set_updater/mod.rs @@ -24,6 +24,7 @@ use rand::rngs::OsRng; use std::collections::HashSet; use std::ops::Deref; use std::time::Duration; +use task::ShutdownListener; use time::OffsetDateTime; use tokio::time::sleep; use validator_client::nymd::{Coin, SigningNymdClient}; @@ -328,11 +329,14 @@ impl RewardedSetUpdater { Ok(()) } - pub(crate) async fn run(&mut self) -> Result<(), RewardingError> { + pub(crate) async fn run( + &mut self, + mut shutdown: ShutdownListener, + ) -> Result<(), RewardingError> { self.validator_cache.wait_for_initial_values().await; let mut epoch = self.epoch().await?; - loop { + while !shutdown.is_shutdown() { // wait until the cache refresher determined its time to update the rewarded/active sets let time = OffsetDateTime::now_utc().unix_timestamp(); epoch.update_to_latest(self).await?; @@ -351,7 +355,16 @@ impl RewardedSetUpdater { ); // Sleep at most 300 before checking again, to keep logs busy let s = time_to_epoch_change.min(300).max(0) as u64; - sleep(Duration::from_secs(s)).await; + tokio::select! { + _ = sleep(Duration::from_secs(s)) => { + trace!("Checking again for updating rewarded/active sets"); + } + _ = shutdown.recv() => { + trace!("RewardedSetUpdater: Received shutdown"); + // This break should not be necessary, but there's a following sleep after this + break; + } + } } // allow some blocks to pass sleep(Duration::from_secs(10)).await;