From 2d5b614777ea644cdbfb9bd2f48064f7c5326fb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Stuczy=C5=84ski?= Date: Mon, 19 May 2025 09:32:36 +0100 Subject: [PATCH] additional bugfixes and guards against stuck epoch --- nym-api/nym-api-requests/src/models.rs | 15 +++++++++++++++ nym-api/src/key_rotation/mod.rs | 6 ------ nym-api/src/node_describe_cache/refresh.rs | 4 +++- nym-node/src/config/template.rs | 8 ++++---- nym-node/src/config/upgrade_helpers.rs | 2 +- nym-node/src/node/key_rotation/controller.rs | 17 ++++++++++++++--- nym-node/src/node/key_rotation/manager.rs | 18 +++++------------- nym-node/src/node/mod.rs | 8 +++++--- .../node/replay_protection/background_task.rs | 4 ++++ .../testnet-manager/src/manager/local_nodes.rs | 17 +++++++++++------ 10 files changed, 62 insertions(+), 37 deletions(-) diff --git a/nym-api/nym-api-requests/src/models.rs b/nym-api/nym-api-requests/src/models.rs index 86e6fc7664..d0b43ea5e4 100644 --- a/nym-api/nym-api-requests/src/models.rs +++ b/nym-api/nym-api-requests/src/models.rs @@ -1475,6 +1475,21 @@ impl KeyRotationInfoResponse { self.key_rotation_state .current_rotation_starting_epoch_id(self.current_absolute_epoch_id) } + + pub fn is_epoch_stuck(&self) -> bool { + let now = OffsetDateTime::now_utc(); + let expected_epoch_end = self.current_epoch_start + self.epoch_duration; + if now > expected_epoch_end { + let diff = now - expected_epoch_end; + // if epoch hasn't progressed for more than 20% of its duration, mark is as stuck + let threshold = Duration::from_secs_f32(self.epoch_duration.as_secs_f32() * 0.2); + if diff > threshold { + return true; + } + } + + false + } } #[derive(Clone, Debug, Serialize, Deserialize, schemars::JsonSchema, ToSchema)] diff --git a/nym-api/src/key_rotation/mod.rs b/nym-api/src/key_rotation/mod.rs index db8d807767..ec88099395 100644 --- a/nym-api/src/key_rotation/mod.rs +++ b/nym-api/src/key_rotation/mod.rs @@ -62,8 +62,6 @@ pub(crate) struct KeyRotationController { pub(crate) contract_cache: NymContractCache, } -const todo: &str = "remove those println"; - impl KeyRotationController { pub(crate) fn new( describe_cache_refresher: RefreshRequester, @@ -90,16 +88,12 @@ impl KeyRotationController { } async fn handle_contract_cache_update(&mut self) { - println!("contract cache refreshed - checking rotation status"); - let updated = self.get_contract_data().await; - println!("status: {updated:#?}"); // if we're only 1/4 epoch away from the next rotation, and we haven't yet performed the refresh, // update the self-described cache, as all nodes should have already pre-announced their new sphinx keys if let Some(remaining) = updated.epochs_until_next_rotation() { debug!("{remaining} epoch(s) remaining until next key rotation"); - println!("{remaining} epoch(s) remaining until next key rotation"); let expected = Some(updated.upcoming_rotation_id()); if remaining < 0.25 && self.last_described_refreshed_for != expected { info!("{remaining} epoch(s) remaining until next key rotation - requesting full refresh of self-described cache"); diff --git a/nym-api/src/node_describe_cache/refresh.rs b/nym-api/src/node_describe_cache/refresh.rs index ebc0dff38d..6f3949c6c6 100644 --- a/nym-api/src/node_describe_cache/refresh.rs +++ b/nym-api/src/node_describe_cache/refresh.rs @@ -5,10 +5,12 @@ use crate::node_describe_cache::query_helpers::query_for_described_data; use crate::node_describe_cache::NodeDescribeCacheError; use nym_api_requests::legacy::{LegacyGatewayBondWithId, LegacyMixNodeDetailsWithLayer}; use nym_api_requests::models::{DescribedNodeType, NymNodeDescription}; +use nym_bin_common::bin_info; use nym_config::defaults::DEFAULT_NYM_NODE_HTTP_PORT; use nym_crypto::asymmetric::ed25519; use nym_mixnet_contract_common::{NodeId, NymNodeDetails}; use nym_node_requests::api::client::NymNodeApiClientExt; +use nym_validator_client::UserAgent; use std::time::Duration; use tracing::debug; @@ -122,7 +124,7 @@ async fn try_get_client( let client = match nym_node_requests::api::Client::builder(address).and_then(|b| { b.with_timeout(Duration::from_secs(5)) .no_hickory_dns() - .with_user_agent("nym-api-describe-cache") + .with_user_agent(UserAgent::from(bin_info!())) .build() }) { Ok(client) => client, diff --git a/nym-node/src/config/template.rs b/nym-node/src/config/template.rs index 96f66b246b..bfe413a62b 100644 --- a/nym-node/src/config/template.rs +++ b/nym-node/src/config/template.rs @@ -84,11 +84,11 @@ private_ed25519_identity_key_file = '{{ storage_paths.keys.private_ed25519_ident # Path to file containing ed25519 identity public key. public_ed25519_identity_key_file = '{{ storage_paths.keys.public_ed25519_identity_key_file }}' -# Path to file containing x25519 sphinx private key. -private_x25519_sphinx_key_file = '{{ storage_paths.keys.private_x25519_sphinx_key_file }}' +# Path to file containing the primary x25519 sphinx private key. +primary_x25519_sphinx_key_file = '{{ storage_paths.keys.primary_x25519_sphinx_key_file }}' -# Path to file containing x25519 sphinx public key. -public_x25519_sphinx_key_file = '{{ storage_paths.keys.public_x25519_sphinx_key_file }}' +# Path to file containing the secondary x25519 sphinx private key. +secondary_x25519_sphinx_key_file = '{{ storage_paths.keys.secondary_x25519_sphinx_key_file }}' # Path to file containing x25519 noise private key. private_x25519_noise_key_file = '{{ storage_paths.keys.private_x25519_noise_key_file }}' diff --git a/nym-node/src/config/upgrade_helpers.rs b/nym-node/src/config/upgrade_helpers.rs index 9c8276bacc..85e2cced9c 100644 --- a/nym-node/src/config/upgrade_helpers.rs +++ b/nym-node/src/config/upgrade_helpers.rs @@ -19,7 +19,7 @@ async fn try_upgrade_config(path: &Path) -> Result<(), NymNodeError> { match try_upgrade_config_v9(path, cfg).await { Ok(cfg) => cfg.save(), Err(e) => { - tracing::error!("Failed to finish upgrade - {e}"); + tracing::error!("Failed to finish upgrade: {e}"); Err(NymNodeError::FailedUpgrade) } } diff --git a/nym-node/src/node/key_rotation/controller.rs b/nym-node/src/node/key_rotation/controller.rs index 88c0eadfa3..98d72692b3 100644 --- a/nym-node/src/node/key_rotation/controller.rs +++ b/nym-node/src/node/key_rotation/controller.rs @@ -11,7 +11,7 @@ use nym_validator_client::models::{KeyRotationInfoResponse, KeyRotationState}; use std::time::Duration; use time::OffsetDateTime; use tokio::time::{interval, sleep, Instant}; -use tracing::{error, info, trace, warn}; +use tracing::{debug, error, info, trace, warn}; pub(crate) struct RotationConfig { epoch_duration: Duration, @@ -60,7 +60,7 @@ impl NextAction { } } -#[derive(Clone, Copy)] +#[derive(Debug, Clone, Copy)] enum KeyRotationActionState { // generate and pre-announce new key to the nym-api(s) PreAnnounce { rotation_id: u32 }, @@ -100,6 +100,10 @@ impl KeyRotationController { async fn determine_next_action(&self) -> NextAction { loop { if let Some(next) = self.try_determine_next_action().await { + debug!( + "next key rotation action to take: {:?} at {}", + next.typ, next.deadline + ); return next; } @@ -110,8 +114,12 @@ impl KeyRotationController { async fn try_determine_next_action(&self) -> Option { let key_rotation_info = self.try_get_key_rotation_info().await?; - let current_rotation = key_rotation_info.current_key_rotation_id(); + if key_rotation_info.is_epoch_stuck() { + warn!("the epoch is stuck - can't progress with key rotation"); + return None; + } + let current_rotation = key_rotation_info.current_key_rotation_id(); let current_epoch = key_rotation_info.current_absolute_epoch_id; let next_rotation_epoch = key_rotation_info.next_rotation_starting_epoch_id(); let current_rotation_epoch = key_rotation_info.current_rotation_starting_epoch_id(); @@ -172,6 +180,7 @@ impl KeyRotationController { } async fn pre_announce_new_key(&self, rotation_id: u32) { + info!("pre-announcing new key for rotation {rotation_id}"); if let Err(err) = self.managed_keys.generate_key_for_new_rotation(rotation_id) { error!("failed to generate and store new sphinx key: {err}"); return; @@ -192,6 +201,7 @@ impl KeyRotationController { } fn swap_default_key(&self) { + info!("attempting to swap the primary key to the previously generated one"); if let Err(err) = self.managed_keys.rotate_keys() { error!("failed to perform sphinx key swap: {err}") }; @@ -207,6 +217,7 @@ impl KeyRotationController { } fn purge_old_rotation_data(&self) { + info!("purging data associated with the old sphinx key"); if let Err(err) = self.managed_keys.remove_overlap_key() { error!("failed to remove old sphinx key: {err}"); }; diff --git a/nym-node/src/node/key_rotation/manager.rs b/nym-node/src/node/key_rotation/manager.rs index 090f0cbb77..ee087bc366 100644 --- a/nym-node/src/node/key_rotation/manager.rs +++ b/nym-node/src/node/key_rotation/manager.rs @@ -52,13 +52,11 @@ impl SphinxKeyManager { let tmp_path = primary_path.as_ref().with_extension("tmp"); // 1. COPY: primary -> temp - fs::copy(primary_path.as_ref(), secondary_path.as_ref()).map_err(|err| { - KeyIOFailure::KeyCopyFailure { - key: "old x25519 sphinx primary".to_string(), - source: primary_path.as_ref().to_path_buf(), - destination: secondary_path.as_ref().to_path_buf(), - err, - } + fs::copy(primary_path.as_ref(), &tmp_path).map_err(|err| KeyIOFailure::KeyCopyFailure { + key: "old x25519 sphinx primary".to_string(), + source: primary_path.as_ref().to_path_buf(), + destination: tmp_path.clone(), + err, })?; // 2. MOVE: secondary -> primary @@ -81,12 +79,6 @@ impl SphinxKeyManager { } })?; - // 4. REMOVE: temp - fs::remove_file(&tmp_path).map_err(|err| KeyIOFailure::KeyRemovalFailure { - key: "old x25519 sphinx primary (temp location)".to_string(), - path: tmp_path, - err, - })?; Ok(()) } diff --git a/nym-node/src/node/mod.rs b/nym-node/src/node/mod.rs index 3620a9348a..918199f60f 100644 --- a/nym-node/src/node/mod.rs +++ b/nym-node/src/node/mod.rs @@ -547,6 +547,7 @@ impl NymNode { self.x25519_noise_keys.public_key() } + #[track_caller] pub(crate) fn active_sphinx_keys(&self) -> Result { Ok(self.sphinx_keys()?.keys.clone()) } @@ -985,6 +986,7 @@ impl NymNode { NymApisClient::new(&self.config.mixnet.nym_api_urls) } + #[track_caller] fn sphinx_keys(&self) -> Result<&SphinxKeyManager, NymNodeError> { self.sphinx_key_manager .as_ref() @@ -1136,9 +1138,6 @@ impl NymNode { ) .await?; - self.setup_key_rotation(nym_apis_client, bloomfilters_manager) - .await?; - let metrics_sender = self.setup_metrics_backend( active_clients_store.clone(), active_egress_mixnet_connections, @@ -1154,6 +1153,9 @@ impl NymNode { ) .await?; + self.setup_key_rotation(nym_apis_client, bloomfilters_manager) + .await?; + network_refresher.start(); self.shutdown_manager.close(); diff --git a/nym-node/src/node/replay_protection/background_task.rs b/nym-node/src/node/replay_protection/background_task.rs index 1c8d6ad033..68092e0208 100644 --- a/nym-node/src/node/replay_protection/background_task.rs +++ b/nym-node/src/node/replay_protection/background_task.rs @@ -51,6 +51,10 @@ impl ReplayProtectionDiskFlush { path: bloomfilters_directory.clone(), }; + if !bloomfilters_directory.exists() { + fs::create_dir_all(&bloomfilters_directory).map_err(dir_read_err)?; + } + let available_filters_dir = fs::read_dir(&bloomfilters_directory).map_err(dir_read_err)?; // figure out what bloomfilters we have available on disk diff --git a/tools/internal/testnet-manager/src/manager/local_nodes.rs b/tools/internal/testnet-manager/src/manager/local_nodes.rs index e140ccb20a..9e5450e4ba 100644 --- a/tools/internal/testnet-manager/src/manager/local_nodes.rs +++ b/tools/internal/testnet-manager/src/manager/local_nodes.rs @@ -18,6 +18,7 @@ use std::ops::Deref; use std::path::{Path, PathBuf}; use std::process::Stdio; use tokio::process::Command; +use tracing::error; use zeroize::Zeroizing; struct LocalNodesCtx<'a> { @@ -158,8 +159,8 @@ impl NetworkManager { &output_file_path.display().to_string(), ]) .stdout(Stdio::null()) + .stderr(Stdio::piped()) .stdin(Stdio::null()) - .stderr(Stdio::null()) .kill_on_drop(true); if is_gateway { @@ -169,10 +170,12 @@ impl NetworkManager { cmd.args(["--mode", "mixnode"]); } - let mut child = cmd.spawn()?; - let child_fut = child.wait(); + let child = cmd.spawn()?; + let child_fut = child.wait_with_output(); let out = ctx.async_with_progress(child_fut).await?; - if !out.success() { + if !out.status.success() { + error!("nym node failure"); + println!("{}", String::from_utf8_lossy(&out.stderr)); return Err(NetworkManagerError::NymNodeExecutionFailure); } @@ -196,14 +199,16 @@ impl NetworkManager { "--output", "json", ]) - .stdin(Stdio::null()) .stdout(Stdio::null()) - .stderr(Stdio::null()) + .stderr(Stdio::piped()) + .stdin(Stdio::null()) .kill_on_drop(true) .output(); let out = ctx.async_with_progress(child).await?; if !out.status.success() { + error!("nym node failure"); + println!("{}", String::from_utf8_lossy(&out.stderr)); return Err(NetworkManagerError::NymNodeExecutionFailure); } let signature: ReducedSignatureOut = serde_json::from_slice(&out.stdout)?;