feat: introduce on-disk cache persistance for major nym-api caches (#6302)

This includes:
- mixnet contract cache
- described nodes cache
- nodes annotations cache (performance)

those changes include taking some code developed for the purposes of #6277

nym-api/src/node_status_api/cache/data.rs

@@ -1,20 +1,20 @@
 // Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: GPL-3.0-only
 
-use crate::support::caching::Cache;
 use nym_api_requests::models::NodeAnnotation;
 use nym_mixnet_contract_common::NodeId;
+use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 
-#[derive(Default)]
+#[derive(Default, Serialize, Deserialize)]
 #[allow(deprecated)]
 pub(crate) struct NodeStatusCacheData {
     /// Basic annotation for nym-nodes
-    pub(crate) node_annotations: Cache<HashMap<NodeId, NodeAnnotation>>,
+    pub(crate) node_annotations: HashMap<NodeId, NodeAnnotation>,
 }
 
-impl NodeStatusCacheData {
-    pub fn new() -> Self {
-        Self::default()
+impl From<HashMap<NodeId, NodeAnnotation>> for NodeStatusCacheData {
+    fn from(node_annotations: HashMap<NodeId, NodeAnnotation>) -> Self {
+        NodeStatusCacheData { node_annotations }
     }
 }

nym-api/src/node_status_api/cache/mod.rs

@@ -3,19 +3,18 @@
 
 use self::data::NodeStatusCacheData;
 use crate::node_performance::provider::PerformanceRetrievalFailure;
-use crate::support::caching::cache::UninitialisedCache;
+use crate::support::caching::cache::{SharedCache, UninitialisedCache};
 use crate::support::caching::Cache;
 use nym_api_requests::models::NodeAnnotation;
 use nym_mixnet_contract_common::NodeId;
 use std::collections::HashMap;
-use std::{sync::Arc, time::Duration};
+use std::path::Path;
+use std::time::Duration;
 use thiserror::Error;
+use time::OffsetDateTime;
 use tokio::sync::RwLockReadGuard;
-use tokio::{sync::RwLock, time};
 use tracing::error;
 
-const CACHE_TIMEOUT_MS: u64 = 100;
-
 mod config_score;
 pub mod data;
 pub mod refresher;
@@ -44,43 +43,64 @@ impl From<UninitialisedCache> for NodeStatusCacheError {
 /// The cache can be triggered to update on contract cache changes, and/or periodically on a timer.
 #[derive(Clone)]
 pub struct NodeStatusCache {
-    inner: Arc<RwLock<NodeStatusCacheData>>,
+    inner: SharedCache<NodeStatusCacheData>,
+}
+
+impl From<SharedCache<NodeStatusCacheData>> for NodeStatusCache {
+    fn from(inner: SharedCache<NodeStatusCacheData>) -> Self {
+        NodeStatusCache { inner }
+    }
 }
 
 impl NodeStatusCache {
     /// Creates a new cache with no data.
-    pub(crate) fn new() -> NodeStatusCache {
+    pub(crate) fn new<P: AsRef<Path>>(store_path: P, max_cache_age: Duration) -> NodeStatusCache {
         NodeStatusCache {
-            inner: Arc::new(RwLock::new(NodeStatusCacheData::new())),
+            inner: SharedCache::new_with_persistent(
+                store_path,
+                max_cache_age,
+                Some(HashMap::new().into()),
+            ),
         }
     }
 
+    pub async fn cache_timestamp(&self) -> OffsetDateTime {
+        let Ok(cache) = self.inner.get().await else {
+            return OffsetDateTime::UNIX_EPOCH;
+        };
+
+        cache.timestamp()
+    }
+
     /// Updates the cache with the latest data.
     async fn update(&self, node_annotations: HashMap<NodeId, NodeAnnotation>) {
-        match time::timeout(Duration::from_millis(CACHE_TIMEOUT_MS), self.inner.write()).await {
-            Ok(mut cache) => {
-                cache.node_annotations.unchecked_update(node_annotations);
-            }
-            Err(e) => error!("{e}"),
+        if self
+            .inner
+            .try_overwrite_old_value(node_annotations, "node-status")
+            .await
+            .is_err()
+        {
+            error!("failed to update node status cache!")
         }
     }
 
+    pub(crate) async fn cache(
+        &self,
+    ) -> Result<RwLockReadGuard<'_, Cache<NodeStatusCacheData>>, UninitialisedCache> {
+        self.inner.get().await
+    }
+
     async fn get<'a, T: 'a>(
         &'a self,
-        fn_arg: impl FnOnce(&NodeStatusCacheData) -> &Cache<T>,
-    ) -> Option<RwLockReadGuard<'a, Cache<T>>> {
-        match time::timeout(Duration::from_millis(CACHE_TIMEOUT_MS), self.inner.read()).await {
-            Ok(cache) => Some(RwLockReadGuard::map(cache, |item| fn_arg(item))),
-            Err(e) => {
-                error!("{e}");
-                None
-            }
-        }
+        fn_arg: impl FnOnce(&Cache<NodeStatusCacheData>) -> &T,
+    ) -> Result<RwLockReadGuard<'a, T>, UninitialisedCache> {
+        let guard = self.inner.get().await?;
+        Ok(RwLockReadGuard::map(guard, fn_arg))
     }
 
     pub(crate) async fn node_annotations(
         &self,
-    ) -> Option<RwLockReadGuard<'_, Cache<HashMap<NodeId, NodeAnnotation>>>> {
+    ) -> Result<RwLockReadGuard<'_, HashMap<NodeId, NodeAnnotation>>, UninitialisedCache> {
         self.get(|c| &c.node_annotations).await
     }
 }

nym-api/src/node_status_api/cache/refresher.rs

@@ -8,6 +8,7 @@ use crate::node_performance::provider::{NodePerformanceProvider, NodesRoutingSco
 use crate::node_status_api::cache::config_score::calculate_config_score;
 use crate::node_status_api::models::Uptime;
 use crate::support::caching::cache::SharedCache;
+use crate::support::caching::refresher::RefreshRequester;
 use crate::{
     mixnet_contract_cache::cache::MixnetContractCache,
     node_status_api::cache::NodeStatusCacheError, support::caching::CacheNotification,
@@ -18,10 +19,11 @@ use nym_mixnet_contract_common::{NodeId, NymNodeDetails};
 use nym_task::ShutdownToken;
 use nym_topology::CachedEpochRewardedSet;
 use std::collections::HashMap;
+use std::path::PathBuf;
 use std::time::Duration;
 use tokio::sync::watch;
 use tokio::time;
-use tracing::{info, trace, warn};
+use tracing::{error, info, trace, warn};
 
 // Long running task responsible for keeping the node status cache up-to-date.
 pub struct NodeStatusCacheRefresher {
@@ -32,13 +34,27 @@ pub struct NodeStatusCacheRefresher {
     // Sources for when refreshing data
     mixnet_contract_cache: MixnetContractCache,
     described_cache: SharedCache<DescribedNodes>,
+
+    /// channel notifying us when mixnet cache has been refreshed,
+    /// so that this cache could also be recreated
     mixnet_contract_cache_listener: watch::Receiver<CacheNotification>,
+
+    /// channel notifying us when the describe cache has been refreshed,
+    /// so that this cache could also be recreated
     describe_cache_listener: watch::Receiver<CacheNotification>,
 
+    /// channel explicitly requesting cache refresh. it does not follow the usual rate limiting
+    refresh_requester: RefreshRequester,
+
+    /// Path to an on-disk location where the contents of the retrieved items should be written
+    /// upon refresh
+    on_disk_file: PathBuf,
+
     performance_provider: Box<dyn NodePerformanceProvider + Send + Sync>,
 }
 
 impl NodeStatusCacheRefresher {
+    #[allow(clippy::too_many_arguments)]
     pub(crate) fn new(
         cache: NodeStatusCache,
         fallback_caching_interval: Duration,
@@ -47,6 +63,7 @@ impl NodeStatusCacheRefresher {
         contract_cache_listener: watch::Receiver<CacheNotification>,
         describe_cache_listener: watch::Receiver<CacheNotification>,
         performance_provider: Box<dyn NodePerformanceProvider + Send + Sync>,
+        on_disk_file: PathBuf,
     ) -> Self {
         Self {
             cache,
@@ -55,6 +72,8 @@ impl NodeStatusCacheRefresher {
             described_cache,
             mixnet_contract_cache_listener: contract_cache_listener,
             describe_cache_listener,
+            refresh_requester: Default::default(),
+            on_disk_file,
             performance_provider,
         }
     }
@@ -89,6 +108,23 @@ impl NodeStatusCacheRefresher {
                         }
                     }
                 }
+                // note: `Notify` is not cancellation safe, HOWEVER, there's only one listener,
+                // so it doesn't matter if we lose our queue position
+                _ = self.refresh_requester.notified() => {
+                     tokio::select! {
+                        // perform full refresh regardless of the rates
+                        _ = self.refresh() => {
+                            last_update = OffsetDateTime::now_utc();
+                            fallback_interval.reset();
+                        },
+                        _ = shutdown_token.cancelled() => {
+                            trace!("NodeStatusCacheRefresher: Received shutdown");
+                            break;
+                        }
+                    }
+                }
+
+
                 // ... however, if we don't receive any notifications we fall back to periodic
                 // refreshes
                 _ = fallback_interval.tick() => {
@@ -219,6 +255,15 @@ impl NodeStatusCacheRefresher {
 
         // Update the cache
         self.cache.update(node_annotations).await;
+
+        // attempt to update on-disk cache
+        let Ok(new_cached) = self.cache.cache().await else {
+            error!("the node status cache is still not initialised!");
+            return Ok(());
+        };
+        // error reporting is handled by the serialise function itself
+        let _ = new_cached.try_serialise_to_file(&self.on_disk_file);
+
         Ok(())
     }
 }

nym-api/src/node_status_api/mod.rs

@@ -12,6 +12,7 @@ use crate::{
 };
 pub(crate) use cache::NodeStatusCache;
 use nym_task::ShutdownManager;
+use std::path::PathBuf;
 use std::time::Duration;
 use tokio::sync::watch;
 
@@ -39,6 +40,7 @@ pub(crate) fn start_cache_refresh(
     performance_provider: Box<dyn NodePerformanceProvider + Send + Sync>,
     nym_contract_cache_listener: watch::Receiver<support::caching::CacheNotification>,
     described_cache_cache_listener: watch::Receiver<support::caching::CacheNotification>,
+    on_disk_file: PathBuf,
     shutdown_manager: &ShutdownManager,
 ) {
     let mut nym_api_cache_refresher = NodeStatusCacheRefresher::new(
@@ -49,6 +51,7 @@ pub(crate) fn start_cache_refresh(
         nym_contract_cache_listener,
         described_cache_cache_listener,
         performance_provider,
+        on_disk_file,
     );
     let shutdown_listener = shutdown_manager.clone_shutdown_token();
     tokio::spawn(async move { nym_api_cache_refresher.run(shutdown_listener).await });

nym-api/src/node_status_api/models.rs

@@ -346,13 +346,6 @@ impl AxumErrorResponse {
         }
     }
 
-    pub(crate) fn internal() -> Self {
-        Self {
-            message: RequestError::new("Internal server error"),
-            status: StatusCode::INTERNAL_SERVER_ERROR,
-        }
-    }
-
     pub(crate) fn not_implemented() -> Self {
         Self {
             message: RequestError::empty(),