additional bugfixes and guards against stuck epoch

nym-api/nym-api-requests/src/models.rs

@@ -1475,6 +1475,21 @@ impl KeyRotationInfoResponse {
         self.key_rotation_state
             .current_rotation_starting_epoch_id(self.current_absolute_epoch_id)
     }
+
+    pub fn is_epoch_stuck(&self) -> bool {
+        let now = OffsetDateTime::now_utc();
+        let expected_epoch_end = self.current_epoch_start + self.epoch_duration;
+        if now > expected_epoch_end {
+            let diff = now - expected_epoch_end;
+            // if epoch hasn't progressed for more than 20% of its duration, mark is as stuck
+            let threshold = Duration::from_secs_f32(self.epoch_duration.as_secs_f32() * 0.2);
+            if diff > threshold {
+                return true;
+            }
+        }
+
+        false
+    }
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize, schemars::JsonSchema, ToSchema)]

nym-api/src/key_rotation/mod.rs

@@ -62,8 +62,6 @@ pub(crate) struct KeyRotationController {
     pub(crate) contract_cache: NymContractCache,
 }
 
-const todo: &str = "remove those println";
-
 impl KeyRotationController {
     pub(crate) fn new(
         describe_cache_refresher: RefreshRequester,
@@ -90,16 +88,12 @@ impl KeyRotationController {
     }
 
     async fn handle_contract_cache_update(&mut self) {
-        println!("contract cache refreshed - checking rotation status");
-
         let updated = self.get_contract_data().await;
-        println!("status: {updated:#?}");
 
         // if we're only 1/4 epoch away from the next rotation, and we haven't yet performed the refresh,
         // update the self-described cache, as all nodes should have already pre-announced their new sphinx keys
         if let Some(remaining) = updated.epochs_until_next_rotation() {
             debug!("{remaining} epoch(s) remaining until next key rotation");
-            println!("{remaining} epoch(s) remaining until next key rotation");
             let expected = Some(updated.upcoming_rotation_id());
             if remaining < 0.25 && self.last_described_refreshed_for != expected {
                 info!("{remaining} epoch(s) remaining until next key rotation - requesting full refresh of self-described cache");

nym-api/src/node_describe_cache/refresh.rs

@@ -5,10 +5,12 @@ use crate::node_describe_cache::query_helpers::query_for_described_data;
 use crate::node_describe_cache::NodeDescribeCacheError;
 use nym_api_requests::legacy::{LegacyGatewayBondWithId, LegacyMixNodeDetailsWithLayer};
 use nym_api_requests::models::{DescribedNodeType, NymNodeDescription};
+use nym_bin_common::bin_info;
 use nym_config::defaults::DEFAULT_NYM_NODE_HTTP_PORT;
 use nym_crypto::asymmetric::ed25519;
 use nym_mixnet_contract_common::{NodeId, NymNodeDetails};
 use nym_node_requests::api::client::NymNodeApiClientExt;
+use nym_validator_client::UserAgent;
 use std::time::Duration;
 use tracing::debug;
 
@@ -122,7 +124,7 @@ async fn try_get_client(
         let client = match nym_node_requests::api::Client::builder(address).and_then(|b| {
             b.with_timeout(Duration::from_secs(5))
                 .no_hickory_dns()
-                .with_user_agent("nym-api-describe-cache")
+                .with_user_agent(UserAgent::from(bin_info!()))
                 .build()
         }) {
             Ok(client) => client,

nym-node/src/config/template.rs

@@ -84,11 +84,11 @@ private_ed25519_identity_key_file = '{{ storage_paths.keys.private_ed25519_ident
 # Path to file containing ed25519 identity public key.
 public_ed25519_identity_key_file = '{{ storage_paths.keys.public_ed25519_identity_key_file }}'
 
-# Path to file containing x25519 sphinx private key.
-private_x25519_sphinx_key_file = '{{ storage_paths.keys.private_x25519_sphinx_key_file }}'
+# Path to file containing the primary x25519 sphinx private key.
+primary_x25519_sphinx_key_file = '{{ storage_paths.keys.primary_x25519_sphinx_key_file }}'
 
-# Path to file containing x25519 sphinx public key.
-public_x25519_sphinx_key_file = '{{ storage_paths.keys.public_x25519_sphinx_key_file }}'
+# Path to file containing the secondary x25519 sphinx private key.
+secondary_x25519_sphinx_key_file = '{{ storage_paths.keys.secondary_x25519_sphinx_key_file }}'
 
 # Path to file containing x25519 noise private key.
 private_x25519_noise_key_file = '{{ storage_paths.keys.private_x25519_noise_key_file }}'

nym-node/src/config/upgrade_helpers.rs

@@ -19,7 +19,7 @@ async fn try_upgrade_config(path: &Path) -> Result<(), NymNodeError> {
     match try_upgrade_config_v9(path, cfg).await {
         Ok(cfg) => cfg.save(),
         Err(e) => {
-            tracing::error!("Failed to finish upgrade - {e}");
+            tracing::error!("Failed to finish upgrade: {e}");
             Err(NymNodeError::FailedUpgrade)
         }
     }

nym-node/src/node/key_rotation/controller.rs

@@ -11,7 +11,7 @@ use nym_validator_client::models::{KeyRotationInfoResponse, KeyRotationState};
 use std::time::Duration;
 use time::OffsetDateTime;
 use tokio::time::{interval, sleep, Instant};
-use tracing::{error, info, trace, warn};
+use tracing::{debug, error, info, trace, warn};
 
 pub(crate) struct RotationConfig {
     epoch_duration: Duration,
@@ -60,7 +60,7 @@ impl NextAction {
     }
 }
 
-#[derive(Clone, Copy)]
+#[derive(Debug, Clone, Copy)]
 enum KeyRotationActionState {
     // generate and pre-announce new key to the nym-api(s)
     PreAnnounce { rotation_id: u32 },
@@ -100,6 +100,10 @@ impl KeyRotationController {
     async fn determine_next_action(&self) -> NextAction {
         loop {
             if let Some(next) = self.try_determine_next_action().await {
+                debug!(
+                    "next key rotation action to take: {:?} at {}",
+                    next.typ, next.deadline
+                );
                 return next;
             }
 
@@ -110,8 +114,12 @@ impl KeyRotationController {
 
     async fn try_determine_next_action(&self) -> Option<NextAction> {
         let key_rotation_info = self.try_get_key_rotation_info().await?;
-        let current_rotation = key_rotation_info.current_key_rotation_id();
+        if key_rotation_info.is_epoch_stuck() {
+            warn!("the epoch is stuck - can't progress with key rotation");
+            return None;
+        }
 
+        let current_rotation = key_rotation_info.current_key_rotation_id();
         let current_epoch = key_rotation_info.current_absolute_epoch_id;
         let next_rotation_epoch = key_rotation_info.next_rotation_starting_epoch_id();
         let current_rotation_epoch = key_rotation_info.current_rotation_starting_epoch_id();
@@ -172,6 +180,7 @@ impl KeyRotationController {
     }
 
     async fn pre_announce_new_key(&self, rotation_id: u32) {
+        info!("pre-announcing new key for rotation {rotation_id}");
         if let Err(err) = self.managed_keys.generate_key_for_new_rotation(rotation_id) {
             error!("failed to generate and store new sphinx key: {err}");
             return;
@@ -192,6 +201,7 @@ impl KeyRotationController {
     }
 
     fn swap_default_key(&self) {
+        info!("attempting to swap the primary key to the previously generated one");
         if let Err(err) = self.managed_keys.rotate_keys() {
             error!("failed to perform sphinx key swap: {err}")
         };
@@ -207,6 +217,7 @@ impl KeyRotationController {
     }
 
     fn purge_old_rotation_data(&self) {
+        info!("purging data associated with the old sphinx key");
         if let Err(err) = self.managed_keys.remove_overlap_key() {
             error!("failed to remove old sphinx key: {err}");
         };

nym-node/src/node/key_rotation/manager.rs

@@ -52,13 +52,11 @@ impl SphinxKeyManager {
         let tmp_path = primary_path.as_ref().with_extension("tmp");
 
         // 1. COPY: primary -> temp
-        fs::copy(primary_path.as_ref(), secondary_path.as_ref()).map_err(|err| {
-            KeyIOFailure::KeyCopyFailure {
-                key: "old x25519 sphinx primary".to_string(),
-                source: primary_path.as_ref().to_path_buf(),
-                destination: secondary_path.as_ref().to_path_buf(),
-                err,
-            }
+        fs::copy(primary_path.as_ref(), &tmp_path).map_err(|err| KeyIOFailure::KeyCopyFailure {
+            key: "old x25519 sphinx primary".to_string(),
+            source: primary_path.as_ref().to_path_buf(),
+            destination: tmp_path.clone(),
+            err,
         })?;
 
         // 2. MOVE: secondary -> primary
@@ -81,12 +79,6 @@ impl SphinxKeyManager {
             }
         })?;
 
-        // 4. REMOVE: temp
-        fs::remove_file(&tmp_path).map_err(|err| KeyIOFailure::KeyRemovalFailure {
-            key: "old x25519 sphinx primary (temp location)".to_string(),
-            path: tmp_path,
-            err,
-        })?;
         Ok(())
     }
 

nym-node/src/node/mod.rs

@@ -547,6 +547,7 @@ impl NymNode {
         self.x25519_noise_keys.public_key()
     }
 
+    #[track_caller]
     pub(crate) fn active_sphinx_keys(&self) -> Result<ActiveSphinxKeys, NymNodeError> {
         Ok(self.sphinx_keys()?.keys.clone())
     }
@@ -985,6 +986,7 @@ impl NymNode {
         NymApisClient::new(&self.config.mixnet.nym_api_urls)
     }
 
+    #[track_caller]
     fn sphinx_keys(&self) -> Result<&SphinxKeyManager, NymNodeError> {
         self.sphinx_key_manager
             .as_ref()
@@ -1136,9 +1138,6 @@ impl NymNode {
             )
             .await?;
 
-        self.setup_key_rotation(nym_apis_client, bloomfilters_manager)
-            .await?;
-
         let metrics_sender = self.setup_metrics_backend(
             active_clients_store.clone(),
             active_egress_mixnet_connections,
@@ -1154,6 +1153,9 @@ impl NymNode {
         )
         .await?;
 
+        self.setup_key_rotation(nym_apis_client, bloomfilters_manager)
+            .await?;
+
         network_refresher.start();
 
         self.shutdown_manager.close();

nym-node/src/node/replay_protection/background_task.rs

@@ -51,6 +51,10 @@ impl ReplayProtectionDiskFlush {
             path: bloomfilters_directory.clone(),
         };
 
+        if !bloomfilters_directory.exists() {
+            fs::create_dir_all(&bloomfilters_directory).map_err(dir_read_err)?;
+        }
+
         let available_filters_dir = fs::read_dir(&bloomfilters_directory).map_err(dir_read_err)?;
 
         // figure out what bloomfilters we have available on disk

tools/internal/testnet-manager/src/manager/local_nodes.rs

@@ -18,6 +18,7 @@ use std::ops::Deref;
 use std::path::{Path, PathBuf};
 use std::process::Stdio;
 use tokio::process::Command;
+use tracing::error;
 use zeroize::Zeroizing;
 
 struct LocalNodesCtx<'a> {
@@ -158,8 +159,8 @@ impl NetworkManager {
             &output_file_path.display().to_string(),
         ])
         .stdout(Stdio::null())
+        .stderr(Stdio::piped())
         .stdin(Stdio::null())
-        .stderr(Stdio::null())
         .kill_on_drop(true);
 
         if is_gateway {
@@ -169,10 +170,12 @@ impl NetworkManager {
             cmd.args(["--mode", "mixnode"]);
         }
 
-        let mut child = cmd.spawn()?;
-        let child_fut = child.wait();
+        let child = cmd.spawn()?;
+        let child_fut = child.wait_with_output();
         let out = ctx.async_with_progress(child_fut).await?;
-        if !out.success() {
+        if !out.status.success() {
+            error!("nym node failure");
+            println!("{}", String::from_utf8_lossy(&out.stderr));
             return Err(NetworkManagerError::NymNodeExecutionFailure);
         }
 
@@ -196,14 +199,16 @@ impl NetworkManager {
                 "--output",
                 "json",
             ])
-            .stdin(Stdio::null())
             .stdout(Stdio::null())
-            .stderr(Stdio::null())
+            .stderr(Stdio::piped())
+            .stdin(Stdio::null())
             .kill_on_drop(true)
             .output();
 
         let out = ctx.async_with_progress(child).await?;
         if !out.status.success() {
+            error!("nym node failure");
+            println!("{}", String::from_utf8_lossy(&out.stderr));
             return Err(NetworkManagerError::NymNodeExecutionFailure);
         }
         let signature: ReducedSignatureOut = serde_json::from_slice(&out.stdout)?;