add additional leniency in ticketbook requests

Cargo.lock

@@ -6563,6 +6563,8 @@ dependencies = [
  "serde",
  "thiserror 2.0.12",
  "time",
+ "tokio",
+ "wasmtimer",
  "zeroize",
 ]
 
@@ -7768,7 +7770,7 @@ dependencies = [
 
 [[package]]
 name = "nym-node-status-api"
-version = "4.6.2-rc9"
+version = "4.6.2-rc10"
 dependencies = [
  "ammonia",
  "anyhow",

common/credential-proxy/src/quorum_checker.rs

@@ -27,6 +27,9 @@ pub struct QuorumStateChecker {
     cancellation_token: CancellationToken,
     check_interval: Duration,
     quorum_state: QuorumState,
+
+    /// indicates whether the last check has been a failure
+    last_failed: bool,
 }
 
 impl QuorumStateChecker {
@@ -42,6 +45,7 @@ impl QuorumStateChecker {
             quorum_state: QuorumState {
                 available: Arc::new(Default::default()),
             },
+            last_failed: false,
         };
 
         // first check MUST succeed, otherwise we shouldn't start
@@ -107,7 +111,7 @@ impl QuorumStateChecker {
         Ok(available)
     }
 
-    pub async fn run_forever(self) {
+    pub async fn run_forever(mut self) {
         info!("starting quorum state checker");
         loop {
             tokio::select! {
@@ -117,7 +121,23 @@ impl QuorumStateChecker {
                 }
                 _ = tokio::time::sleep(self.check_interval) => {
                     match self.check_quorum_state().await {
-                        Ok(available) => self.quorum_state.available.store(available, Ordering::SeqCst),
+                        Ok(available) => {
+                            let previous = self.quorum_state.available.load(Ordering::SeqCst);
+                            // only update the quorum state to a failed state if we've had two consecutive failures
+                            if available {
+                                if !previous {
+                                    info!("quorum recovered");
+                                }
+                                self.quorum_state.available.store(true, Ordering::SeqCst);
+                            } else if self.last_failed {
+                                if previous {
+                                    warn!("quorum became unavailable after 2 consecutive failed checks");
+                                }
+                                self.quorum_state.available.store(false, Ordering::SeqCst);
+                            }
+
+                            self.last_failed = !available;
+                        },
                         Err(err) => error!("failed to check current quorum state: {err}"),
                     }
                 }

common/credentials/Cargo.toml

@@ -36,5 +36,13 @@ nym-ecash-contract-common = { workspace = true }
 nym-network-defaults = { workspace = true }
 nym-serde-helpers = { workspace = true, features = ["date"] }
 
+[target."cfg(not(target_arch = \"wasm32\"))".dependencies.tokio]
+workspace = true
+features = ["time"]
+
+[target."cfg(target_arch = \"wasm32\")".dependencies.wasmtimer]
+workspace = true
+features = ["tokio"]
+
 [dev-dependencies]
 rand = { workspace = true }

common/credentials/src/ecash/bandwidth/issuance.rs

@@ -6,6 +6,7 @@ use crate::ecash::bandwidth::serialiser::VersionedSerialise;
 use crate::ecash::bandwidth::CredentialSigningData;
 use crate::ecash::utils::cred_exp_date;
 use crate::error::Error;
+use log::{debug, warn};
 use nym_api_requests::ecash::BlindSignRequestBody;
 use nym_credentials_interface::{
     aggregate_wallets, generate_keypair_user_from_seed, issue_verify, withdrawal_request,
@@ -17,8 +18,15 @@ use nym_ecash_contract_common::deposit::DepositId;
 use nym_ecash_time::{ecash_default_expiration_date, ecash_today, EcashTime};
 use nym_validator_client::nym_api::{EpochId, NymApiClientExt};
 use serde::{Deserialize, Serialize};
+use std::time::Duration;
 use time::Date;
 
+#[cfg(not(target_arch = "wasm32"))]
+use tokio::time::sleep;
+
+#[cfg(target_arch = "wasm32")]
+use wasmtimer::tokio::sleep;
+
 pub use nym_validator_client::nyxd::{Coin, Hash};
 
 #[derive(Serialize, Deserialize)]
@@ -192,6 +200,49 @@ impl IssuanceTicketBook {
         Ok(unblinded_signature)
     }
 
+    // ideally this would have been generic over credential type, but we really don't need secp256k1 keys for bandwidth vouchers
+    pub async fn obtain_partial_ticketbook_credential_with_retries(
+        &self,
+        client: &nym_http_api_client::Client,
+        signer_index: u64,
+        validator_vk: &VerificationKeyAuth,
+        signing_data: CredentialSigningData,
+        max_attempts: usize,
+    ) -> Result<PartialWallet, Error> {
+        let Some(client_url) = client.base_urls().first() else {
+            return Err(Error::CredentialShareObtainFailed);
+        };
+        let mut last_err = None;
+        for attempt in 0..max_attempts {
+            if attempt > 0 {
+                sleep(Duration::from_millis(500 * attempt as u64)).await;
+            }
+            debug!(
+                "attempt {} / {max_attempts} to obtain partial ticketbook credential from {client_url}",
+                attempt + 1,
+            );
+            match self
+                .obtain_partial_ticketbook_credential(
+                    client,
+                    signer_index,
+                    validator_vk,
+                    signing_data.clone(),
+                )
+                .await
+            {
+                Ok(partial_wallet) => return Ok(partial_wallet),
+                Err(err) => {
+                    warn!(
+                        "attempt {} / {max_attempts} to obtain partial ticketbook credential from {client_url} failed: {err}",
+                        attempt + 1,
+                    );
+                    last_err = Some(err);
+                }
+            }
+        }
+        Err(last_err.unwrap_or(Error::CredentialShareObtainFailed))
+    }
+
     // ideally this would have been generic over credential type, but we really don't need secp256k1 keys for bandwidth vouchers
     pub async fn obtain_partial_ticketbook_credential(
         &self,

common/credentials/src/ecash/utils.rs

@@ -137,6 +137,8 @@ pub async fn obtain_aggregate_wallet(
     ecash_api_clients: &[EcashApiClient],
     threshold: u64,
 ) -> Result<WalletSignatures, Error> {
+    const MAX_ATTEMPTS: usize = 2;
+
     if ecash_api_clients.len() < threshold as usize {
         return Err(Error::NoValidatorsAvailable);
     }
@@ -154,11 +156,12 @@ pub async fn obtain_aggregate_wallet(
         );
 
         match voucher
-            .obtain_partial_ticketbook_credential(
+            .obtain_partial_ticketbook_credential_with_retries(
                 &ecash_api_client.api_client,
                 ecash_api_client.node_id,
                 &ecash_api_client.verification_key,
                 request.clone(),
+                MAX_ATTEMPTS,
             )
             .await
         {
@@ -167,6 +170,11 @@ pub async fn obtain_aggregate_wallet(
                 warn!("failed to obtain partial credential from API {ecash_api_client}: {err}",);
             }
         };
+
+        // we got sufficient number of shares
+        if wallets.len() >= threshold as usize {
+            break;
+        }
     }
     if wallets.len() < threshold as usize {
         return Err(Error::NotEnoughShares);

common/credentials/src/error.rs

@@ -63,6 +63,9 @@ pub enum Error {
 
     #[error("failed to create a secp256k1 signature")]
     Secp256k1SignFailure,
+
+    #[error("failed to obtain a valid credential share")]
+    CredentialShareObtainFailed,
 }
 
 impl From<NymAPIError> for Error {

nym-node-status-api/nym-node-status-api/.sqlx/query-7c2f58e63efd85010408f812692ecad1c89d9df3ffaf4b5d00db5adfdef854c4.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n                INSERT INTO pending_issuance\n                (deposit_id, serialization_revision, pending_ticketbook_data, expiration_date)\n                VALUES ($1, $2, $3, $4)\n            ",
+  "query": "\n                INSERT INTO pending_issuance\n                (deposit_id, serialization_revision, pending_ticketbook_data, expiration_date, epoch_id, failure_message)\n                VALUES ($1, $2, $3, $4, $5, $6)\n            ",
   "describe": {
     "columns": [],
     "parameters": {
@@ -8,10 +8,12 @@
         "Int4",
         "Int2",
         "Bytea",
-        "Date"
+        "Date",
+        "Int4",
+        "Text"
       ]
     },
     "nullable": []
   },
-  "hash": "2ee6b058d423a66114d8411e7c287ade31137b30407dc0254d30f60e2d0101cf"
+  "hash": "7c2f58e63efd85010408f812692ecad1c89d9df3ffaf4b5d00db5adfdef854c4"
 }

nym-node-status-api/nym-node-status-api/Cargo.toml

@@ -3,7 +3,7 @@
 
 [package]
 name = "nym-node-status-api"
-version = "4.6.2-rc9"
+version = "4.6.2-rc10"
 authors.workspace = true
 edition.workspace = true
 license.workspace = true

nym-node-status-api/nym-node-status-api/src/db/queries/ecash_data.rs

@@ -30,17 +30,21 @@ impl Storage {
         deposit_id: i32,
         data: &[u8],
         expiration_date: Date,
+        epoch_id: i32,
+        failure_message: &str,
     ) -> Result<(), sqlx::Error> {
         sqlx::query!(
             r#"
                 INSERT INTO pending_issuance
-                (deposit_id, serialization_revision, pending_ticketbook_data, expiration_date)
-                VALUES ($1, $2, $3, $4)
+                (deposit_id, serialization_revision, pending_ticketbook_data, expiration_date, epoch_id, failure_message)
+                VALUES ($1, $2, $3, $4, $5, $6)
             "#,
             deposit_id,
             serialisation_revision,
             data,
             expiration_date,
+            epoch_id,
+            failure_message,
         )
         .execute(&self.pool)
         .await?;

nym-node-status-api/nym-node-status-api/src/ticketbook_manager/mod.rs

@@ -121,11 +121,12 @@ impl TicketbookManager {
             {
                 Err(err) => {
                     error!("failed to obtain aggregated wallet: {err}");
+                    let failure_message = err.to_string();
                     self.state
                         .storage()
-                        .insert_pending_ticketbook(&issuance_data).await.inspect_err(|err| {
+                        .insert_pending_ticketbook(&issuance_data, epoch_id, &failure_message).await.inspect_err(|store_err| {
                             let deposit = issuance_data.deposit_id();
-                            error!("could not save the recovery data for deposit {deposit}: {err}. the data will unfortunately get lost")
+                            error!("could not save the recovery data for deposit {deposit}: {store_err}. the data will unfortunately get lost")
                         })?;
                     return Err(err.into());
                 }

nym-node-status-api/nym-node-status-api/src/ticketbook_manager/storage/mod.rs

@@ -43,6 +43,8 @@ impl TicketbookManagerStorage {
     pub(crate) async fn insert_pending_ticketbook(
         &self,
         ticketbook: &IssuanceTicketBook,
+        epoch_id: EpochId,
+        failure_message: &str,
     ) -> anyhow::Result<()> {
         let ser = ticketbook.pack();
         let data = Zeroizing::new(ser.data);
@@ -54,6 +56,8 @@ impl TicketbookManagerStorage {
                 ticketbook.deposit_id() as i32,
                 &data,
                 ticketbook.expiration_date(),
+                epoch_id as i32,
+                failure_message,
             )
             .await?;