additional bugfixes and guards against stuck epoch

nym-node/src/node/key_rotation/controller.rs

@@ -11,7 +11,7 @@ use nym_validator_client::models::{KeyRotationInfoResponse, KeyRotationState};
 use std::time::Duration;
 use time::OffsetDateTime;
 use tokio::time::{interval, sleep, Instant};
-use tracing::{error, info, trace, warn};
+use tracing::{debug, error, info, trace, warn};
 
 pub(crate) struct RotationConfig {
     epoch_duration: Duration,
@@ -60,7 +60,7 @@ impl NextAction {
     }
 }
 
-#[derive(Clone, Copy)]
+#[derive(Debug, Clone, Copy)]
 enum KeyRotationActionState {
     // generate and pre-announce new key to the nym-api(s)
     PreAnnounce { rotation_id: u32 },
@@ -100,6 +100,10 @@ impl KeyRotationController {
     async fn determine_next_action(&self) -> NextAction {
         loop {
             if let Some(next) = self.try_determine_next_action().await {
+                debug!(
+                    "next key rotation action to take: {:?} at {}",
+                    next.typ, next.deadline
+                );
                 return next;
             }
 
@@ -110,8 +114,12 @@ impl KeyRotationController {
 
     async fn try_determine_next_action(&self) -> Option<NextAction> {
         let key_rotation_info = self.try_get_key_rotation_info().await?;
-        let current_rotation = key_rotation_info.current_key_rotation_id();
+        if key_rotation_info.is_epoch_stuck() {
+            warn!("the epoch is stuck - can't progress with key rotation");
+            return None;
+        }
 
+        let current_rotation = key_rotation_info.current_key_rotation_id();
         let current_epoch = key_rotation_info.current_absolute_epoch_id;
         let next_rotation_epoch = key_rotation_info.next_rotation_starting_epoch_id();
         let current_rotation_epoch = key_rotation_info.current_rotation_starting_epoch_id();
@@ -172,6 +180,7 @@ impl KeyRotationController {
     }
 
     async fn pre_announce_new_key(&self, rotation_id: u32) {
+        info!("pre-announcing new key for rotation {rotation_id}");
         if let Err(err) = self.managed_keys.generate_key_for_new_rotation(rotation_id) {
             error!("failed to generate and store new sphinx key: {err}");
             return;
@@ -192,6 +201,7 @@ impl KeyRotationController {
     }
 
     fn swap_default_key(&self) {
+        info!("attempting to swap the primary key to the previously generated one");
         if let Err(err) = self.managed_keys.rotate_keys() {
             error!("failed to perform sphinx key swap: {err}")
         };
@@ -207,6 +217,7 @@ impl KeyRotationController {
     }
 
     fn purge_old_rotation_data(&self) {
+        info!("purging data associated with the old sphinx key");
         if let Err(err) = self.managed_keys.remove_overlap_key() {
             error!("failed to remove old sphinx key: {err}");
         };

nym-node/src/node/key_rotation/manager.rs

@@ -52,13 +52,11 @@ impl SphinxKeyManager {
         let tmp_path = primary_path.as_ref().with_extension("tmp");
 
         // 1. COPY: primary -> temp
-        fs::copy(primary_path.as_ref(), secondary_path.as_ref()).map_err(|err| {
-            KeyIOFailure::KeyCopyFailure {
-                key: "old x25519 sphinx primary".to_string(),
-                source: primary_path.as_ref().to_path_buf(),
-                destination: secondary_path.as_ref().to_path_buf(),
-                err,
-            }
+        fs::copy(primary_path.as_ref(), &tmp_path).map_err(|err| KeyIOFailure::KeyCopyFailure {
+            key: "old x25519 sphinx primary".to_string(),
+            source: primary_path.as_ref().to_path_buf(),
+            destination: tmp_path.clone(),
+            err,
         })?;
 
         // 2. MOVE: secondary -> primary
@@ -81,12 +79,6 @@ impl SphinxKeyManager {
             }
         })?;
 
-        // 4. REMOVE: temp
-        fs::remove_file(&tmp_path).map_err(|err| KeyIOFailure::KeyRemovalFailure {
-            key: "old x25519 sphinx primary (temp location)".to_string(),
-            path: tmp_path,
-            err,
-        })?;
         Ok(())
     }
 

nym-node/src/node/mod.rs

@@ -547,6 +547,7 @@ impl NymNode {
         self.x25519_noise_keys.public_key()
     }
 
+    #[track_caller]
     pub(crate) fn active_sphinx_keys(&self) -> Result<ActiveSphinxKeys, NymNodeError> {
         Ok(self.sphinx_keys()?.keys.clone())
     }
@@ -985,6 +986,7 @@ impl NymNode {
         NymApisClient::new(&self.config.mixnet.nym_api_urls)
     }
 
+    #[track_caller]
     fn sphinx_keys(&self) -> Result<&SphinxKeyManager, NymNodeError> {
         self.sphinx_key_manager
             .as_ref()
@@ -1136,9 +1138,6 @@ impl NymNode {
             )
             .await?;
 
-        self.setup_key_rotation(nym_apis_client, bloomfilters_manager)
-            .await?;
-
         let metrics_sender = self.setup_metrics_backend(
             active_clients_store.clone(),
             active_egress_mixnet_connections,
@@ -1154,6 +1153,9 @@ impl NymNode {
         )
         .await?;
 
+        self.setup_key_rotation(nym_apis_client, bloomfilters_manager)
+            .await?;
+
         network_refresher.start();
 
         self.shutdown_manager.close();

nym-node/src/node/replay_protection/background_task.rs

@@ -51,6 +51,10 @@ impl ReplayProtectionDiskFlush {
             path: bloomfilters_directory.clone(),
         };
 
+        if !bloomfilters_directory.exists() {
+            fs::create_dir_all(&bloomfilters_directory).map_err(dir_read_err)?;
+        }
+
         let available_filters_dir = fs::read_dir(&bloomfilters_directory).map_err(dir_read_err)?;
 
         // figure out what bloomfilters we have available on disk