diff --git a/Cargo.lock b/Cargo.lock index 2d0759e936..78b5c86b53 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6563,6 +6563,8 @@ dependencies = [ "serde", "thiserror 2.0.12", "time", + "tokio", + "wasmtimer", "zeroize", ] @@ -7768,7 +7770,7 @@ dependencies = [ [[package]] name = "nym-node-status-api" -version = "4.6.2-rc9" +version = "4.6.2-rc10" dependencies = [ "ammonia", "anyhow", diff --git a/common/credential-proxy/src/quorum_checker.rs b/common/credential-proxy/src/quorum_checker.rs index 29a9ca0d4f..59e6cc0eaf 100644 --- a/common/credential-proxy/src/quorum_checker.rs +++ b/common/credential-proxy/src/quorum_checker.rs @@ -27,6 +27,9 @@ pub struct QuorumStateChecker { cancellation_token: CancellationToken, check_interval: Duration, quorum_state: QuorumState, + + /// indicates whether the last check has been a failure + last_failed: bool, } impl QuorumStateChecker { @@ -42,6 +45,7 @@ impl QuorumStateChecker { quorum_state: QuorumState { available: Arc::new(Default::default()), }, + last_failed: false, }; // first check MUST succeed, otherwise we shouldn't start @@ -107,7 +111,7 @@ impl QuorumStateChecker { Ok(available) } - pub async fn run_forever(self) { + pub async fn run_forever(mut self) { info!("starting quorum state checker"); loop { tokio::select! { @@ -117,7 +121,23 @@ impl QuorumStateChecker { } _ = tokio::time::sleep(self.check_interval) => { match self.check_quorum_state().await { - Ok(available) => self.quorum_state.available.store(available, Ordering::SeqCst), + Ok(available) => { + let previous = self.quorum_state.available.load(Ordering::SeqCst); + // only update the quorum state to a failed state if we've had two consecutive failures + if available { + if !previous { + info!("quorum recovered"); + } + self.quorum_state.available.store(true, Ordering::SeqCst); + } else if self.last_failed { + if previous { + warn!("quorum became unavailable after 2 consecutive failed checks"); + } + self.quorum_state.available.store(false, Ordering::SeqCst); + } + + self.last_failed = !available; + }, Err(err) => error!("failed to check current quorum state: {err}"), } } diff --git a/common/credentials/Cargo.toml b/common/credentials/Cargo.toml index ab45bf851f..b73222c612 100644 --- a/common/credentials/Cargo.toml +++ b/common/credentials/Cargo.toml @@ -36,5 +36,13 @@ nym-ecash-contract-common = { workspace = true } nym-network-defaults = { workspace = true } nym-serde-helpers = { workspace = true, features = ["date"] } +[target."cfg(not(target_arch = \"wasm32\"))".dependencies.tokio] +workspace = true +features = ["time"] + +[target."cfg(target_arch = \"wasm32\")".dependencies.wasmtimer] +workspace = true +features = ["tokio"] + [dev-dependencies] rand = { workspace = true } diff --git a/common/credentials/src/ecash/bandwidth/issuance.rs b/common/credentials/src/ecash/bandwidth/issuance.rs index 6bde4eb9e9..ee84cfb19c 100644 --- a/common/credentials/src/ecash/bandwidth/issuance.rs +++ b/common/credentials/src/ecash/bandwidth/issuance.rs @@ -6,6 +6,7 @@ use crate::ecash::bandwidth::serialiser::VersionedSerialise; use crate::ecash::bandwidth::CredentialSigningData; use crate::ecash::utils::cred_exp_date; use crate::error::Error; +use log::{debug, warn}; use nym_api_requests::ecash::BlindSignRequestBody; use nym_credentials_interface::{ aggregate_wallets, generate_keypair_user_from_seed, issue_verify, withdrawal_request, @@ -17,8 +18,15 @@ use nym_ecash_contract_common::deposit::DepositId; use nym_ecash_time::{ecash_default_expiration_date, ecash_today, EcashTime}; use nym_validator_client::nym_api::{EpochId, NymApiClientExt}; use serde::{Deserialize, Serialize}; +use std::time::Duration; use time::Date; +#[cfg(not(target_arch = "wasm32"))] +use tokio::time::sleep; + +#[cfg(target_arch = "wasm32")] +use wasmtimer::tokio::sleep; + pub use nym_validator_client::nyxd::{Coin, Hash}; #[derive(Serialize, Deserialize)] @@ -192,6 +200,49 @@ impl IssuanceTicketBook { Ok(unblinded_signature) } + // ideally this would have been generic over credential type, but we really don't need secp256k1 keys for bandwidth vouchers + pub async fn obtain_partial_ticketbook_credential_with_retries( + &self, + client: &nym_http_api_client::Client, + signer_index: u64, + validator_vk: &VerificationKeyAuth, + signing_data: CredentialSigningData, + max_attempts: usize, + ) -> Result { + let Some(client_url) = client.base_urls().first() else { + return Err(Error::CredentialShareObtainFailed); + }; + let mut last_err = None; + for attempt in 0..max_attempts { + if attempt > 0 { + sleep(Duration::from_millis(500 * attempt as u64)).await; + } + debug!( + "attempt {} / {max_attempts} to obtain partial ticketbook credential from {client_url}", + attempt + 1, + ); + match self + .obtain_partial_ticketbook_credential( + client, + signer_index, + validator_vk, + signing_data.clone(), + ) + .await + { + Ok(partial_wallet) => return Ok(partial_wallet), + Err(err) => { + warn!( + "attempt {} / {max_attempts} to obtain partial ticketbook credential from {client_url} failed: {err}", + attempt + 1, + ); + last_err = Some(err); + } + } + } + Err(last_err.unwrap_or(Error::CredentialShareObtainFailed)) + } + // ideally this would have been generic over credential type, but we really don't need secp256k1 keys for bandwidth vouchers pub async fn obtain_partial_ticketbook_credential( &self, diff --git a/common/credentials/src/ecash/utils.rs b/common/credentials/src/ecash/utils.rs index b2b1321051..dba3448a45 100644 --- a/common/credentials/src/ecash/utils.rs +++ b/common/credentials/src/ecash/utils.rs @@ -137,6 +137,8 @@ pub async fn obtain_aggregate_wallet( ecash_api_clients: &[EcashApiClient], threshold: u64, ) -> Result { + const MAX_ATTEMPTS: usize = 2; + if ecash_api_clients.len() < threshold as usize { return Err(Error::NoValidatorsAvailable); } @@ -154,11 +156,12 @@ pub async fn obtain_aggregate_wallet( ); match voucher - .obtain_partial_ticketbook_credential( + .obtain_partial_ticketbook_credential_with_retries( &ecash_api_client.api_client, ecash_api_client.node_id, &ecash_api_client.verification_key, request.clone(), + MAX_ATTEMPTS, ) .await { @@ -167,6 +170,11 @@ pub async fn obtain_aggregate_wallet( warn!("failed to obtain partial credential from API {ecash_api_client}: {err}",); } }; + + // we got sufficient number of shares + if wallets.len() >= threshold as usize { + break; + } } if wallets.len() < threshold as usize { return Err(Error::NotEnoughShares); diff --git a/common/credentials/src/error.rs b/common/credentials/src/error.rs index daabbcdb1b..b17aaa4734 100644 --- a/common/credentials/src/error.rs +++ b/common/credentials/src/error.rs @@ -63,6 +63,9 @@ pub enum Error { #[error("failed to create a secp256k1 signature")] Secp256k1SignFailure, + + #[error("failed to obtain a valid credential share")] + CredentialShareObtainFailed, } impl From for Error { diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-2ee6b058d423a66114d8411e7c287ade31137b30407dc0254d30f60e2d0101cf.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-7c2f58e63efd85010408f812692ecad1c89d9df3ffaf4b5d00db5adfdef854c4.json similarity index 56% rename from nym-node-status-api/nym-node-status-api/.sqlx/query-2ee6b058d423a66114d8411e7c287ade31137b30407dc0254d30f60e2d0101cf.json rename to nym-node-status-api/nym-node-status-api/.sqlx/query-7c2f58e63efd85010408f812692ecad1c89d9df3ffaf4b5d00db5adfdef854c4.json index 4a11df7ad6..9eba21b14b 100644 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-2ee6b058d423a66114d8411e7c287ade31137b30407dc0254d30f60e2d0101cf.json +++ b/nym-node-status-api/nym-node-status-api/.sqlx/query-7c2f58e63efd85010408f812692ecad1c89d9df3ffaf4b5d00db5adfdef854c4.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO pending_issuance\n (deposit_id, serialization_revision, pending_ticketbook_data, expiration_date)\n VALUES ($1, $2, $3, $4)\n ", + "query": "\n INSERT INTO pending_issuance\n (deposit_id, serialization_revision, pending_ticketbook_data, expiration_date, epoch_id, failure_message)\n VALUES ($1, $2, $3, $4, $5, $6)\n ", "describe": { "columns": [], "parameters": { @@ -8,10 +8,12 @@ "Int4", "Int2", "Bytea", - "Date" + "Date", + "Int4", + "Text" ] }, "nullable": [] }, - "hash": "2ee6b058d423a66114d8411e7c287ade31137b30407dc0254d30f60e2d0101cf" + "hash": "7c2f58e63efd85010408f812692ecad1c89d9df3ffaf4b5d00db5adfdef854c4" } diff --git a/nym-node-status-api/nym-node-status-api/Cargo.toml b/nym-node-status-api/nym-node-status-api/Cargo.toml index 261905a0c2..03016d7b13 100644 --- a/nym-node-status-api/nym-node-status-api/Cargo.toml +++ b/nym-node-status-api/nym-node-status-api/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "nym-node-status-api" -version = "4.6.2-rc9" +version = "4.6.2-rc10" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/nym-node-status-api/nym-node-status-api/src/db/queries/ecash_data.rs b/nym-node-status-api/nym-node-status-api/src/db/queries/ecash_data.rs index 0f90e5884a..02d648edb3 100644 --- a/nym-node-status-api/nym-node-status-api/src/db/queries/ecash_data.rs +++ b/nym-node-status-api/nym-node-status-api/src/db/queries/ecash_data.rs @@ -30,17 +30,21 @@ impl Storage { deposit_id: i32, data: &[u8], expiration_date: Date, + epoch_id: i32, + failure_message: &str, ) -> Result<(), sqlx::Error> { sqlx::query!( r#" INSERT INTO pending_issuance - (deposit_id, serialization_revision, pending_ticketbook_data, expiration_date) - VALUES ($1, $2, $3, $4) + (deposit_id, serialization_revision, pending_ticketbook_data, expiration_date, epoch_id, failure_message) + VALUES ($1, $2, $3, $4, $5, $6) "#, deposit_id, serialisation_revision, data, expiration_date, + epoch_id, + failure_message, ) .execute(&self.pool) .await?; diff --git a/nym-node-status-api/nym-node-status-api/src/ticketbook_manager/mod.rs b/nym-node-status-api/nym-node-status-api/src/ticketbook_manager/mod.rs index 0a7f6c8e11..db141ec7c5 100644 --- a/nym-node-status-api/nym-node-status-api/src/ticketbook_manager/mod.rs +++ b/nym-node-status-api/nym-node-status-api/src/ticketbook_manager/mod.rs @@ -121,11 +121,12 @@ impl TicketbookManager { { Err(err) => { error!("failed to obtain aggregated wallet: {err}"); + let failure_message = err.to_string(); self.state .storage() - .insert_pending_ticketbook(&issuance_data).await.inspect_err(|err| { + .insert_pending_ticketbook(&issuance_data, epoch_id, &failure_message).await.inspect_err(|store_err| { let deposit = issuance_data.deposit_id(); - error!("could not save the recovery data for deposit {deposit}: {err}. the data will unfortunately get lost") + error!("could not save the recovery data for deposit {deposit}: {store_err}. the data will unfortunately get lost") })?; return Err(err.into()); } diff --git a/nym-node-status-api/nym-node-status-api/src/ticketbook_manager/storage/mod.rs b/nym-node-status-api/nym-node-status-api/src/ticketbook_manager/storage/mod.rs index f2aebcfafd..73d790964d 100644 --- a/nym-node-status-api/nym-node-status-api/src/ticketbook_manager/storage/mod.rs +++ b/nym-node-status-api/nym-node-status-api/src/ticketbook_manager/storage/mod.rs @@ -43,6 +43,8 @@ impl TicketbookManagerStorage { pub(crate) async fn insert_pending_ticketbook( &self, ticketbook: &IssuanceTicketBook, + epoch_id: EpochId, + failure_message: &str, ) -> anyhow::Result<()> { let ser = ticketbook.pack(); let data = Zeroizing::new(ser.data); @@ -54,6 +56,8 @@ impl TicketbookManagerStorage { ticketbook.deposit_id() as i32, &data, ticketbook.expiration_date(), + epoch_id as i32, + failure_message, ) .await?;