Feature/aggregated econ dynamics explorer endpoint (#1203)

* Economic dynamics stats endpoint on the explorer API with dummy fixture data

* Populating the endpoint with real data aggregated from validator api

* Introduced new cache functionalities

common/client-libs/validator-client/src/validator_api/mod.rs

@@ -39,6 +39,10 @@ impl Client {
         self.url = new_url
     }
 
+    pub fn current_url(&self) -> &Url {
+        &self.url
+    }
+
     async fn query_validator_api<T, K, V>(
         &self,
         path: PathSegments<'_>,

explorer-api/src/cache/mod.rs

@@ -1,18 +1,30 @@
 use std::collections::HashMap;
 use std::time::{Duration, SystemTime};
 
+const DEFAULT_CACHE_VALIDITY: Duration = Duration::from_secs(60 * 30);
+
 #[derive(Clone)]
 pub(crate) struct Cache<T: Clone> {
     inner: HashMap<String, CacheItem<T>>,
+    cache_validity_duration: Duration,
 }
 
 impl<T: Clone> Cache<T> {
     pub(crate) fn new() -> Self {
         Cache {
             inner: HashMap::new(),
+            cache_validity_duration: DEFAULT_CACHE_VALIDITY,
         }
     }
 
+    // it felt like this might be an useful addition if we want to keep our caches with different
+    // validity durations
+    #[allow(unused)]
+    pub(crate) fn with_validity_duration(mut self, new_cache_validity: Duration) -> Self {
+        self.cache_validity_duration = new_cache_validity;
+        self
+    }
+
     pub(crate) fn len(&self) -> usize {
         self.inner.len()
     }
@@ -27,7 +39,7 @@ impl<T: Clone> Cache<T> {
     pub(crate) fn get(&self, key: &str) -> Option<T> {
         self.inner
             .get(key)
-            .filter(|cache_item| cache_item.valid_until > SystemTime::now())
+            .filter(|cache_item| cache_item.valid_until >= SystemTime::now())
             .map(|cache_item| cache_item.value.clone())
     }
 
@@ -35,11 +47,31 @@ impl<T: Clone> Cache<T> {
         self.inner.insert(
             key.to_string(),
             CacheItem {
-                valid_until: SystemTime::now() + Duration::from_secs(60 * 30),
+                valid_until: SystemTime::now() + self.cache_validity_duration,
                 value,
             },
         );
     }
+
+    #[allow(unused)]
+    pub(crate) fn remove(&mut self, key: &str) -> Option<T> {
+        self.inner.remove(key).map(|item| item.value)
+    }
+
+    #[allow(unused)]
+    pub(crate) fn remove_if_expired(&mut self, key: &str) -> Option<T> {
+        if self.inner.get(key)?.has_expired() {
+            self.remove(key)
+        } else {
+            None
+        }
+    }
+
+    // it seems like something should be running on timer calling this method on all of our caches
+    #[allow(unused)]
+    pub(crate) fn remove_all_expired(&mut self) {
+        self.inner.retain(|_, v| !v.has_expired())
+    }
 }
 
 #[derive(Clone)]
@@ -47,3 +79,10 @@ pub(crate) struct CacheItem<T> {
     pub(crate) value: T,
     pub(crate) valid_until: std::time::SystemTime,
 }
+
+impl<T> CacheItem<T> {
+    fn has_expired(&self) -> bool {
+        let now = SystemTime::now();
+        self.valid_until < now
+    }
+}

explorer-api/src/client.rs

@@ -1,7 +1,28 @@
 use network_defaults::{default_api_endpoints, default_nymd_endpoints, DEFAULT_NETWORK};
+use reqwest::Url;
+use std::sync::Arc;
 use validator_client::nymd::QueryNymdClient;
 
-pub(crate) fn new_nymd_client() -> validator_client::Client<QueryNymdClient> {
+// since this is just a query client, we don't need any locking mechanism to keep sequence numbers in check
+// nor we need to access any of its methods taking mutable reference (like updating api URL)
+// when that becomes a requirement, we would simply put an extra RwLock (or Mutex) in here
+
+#[derive(Clone)]
+pub(crate) struct ThreadsafeValidatorClient(
+    pub(crate) Arc<validator_client::Client<QueryNymdClient>>,
+);
+
+impl ThreadsafeValidatorClient {
+    pub(crate) fn new() -> Self {
+        new_validator_client()
+    }
+
+    pub(crate) fn api_endpoint(&self) -> &Url {
+        self.0.validator_api.current_url()
+    }
+}
+
+pub(crate) fn new_validator_client() -> ThreadsafeValidatorClient {
     let network = DEFAULT_NETWORK;
     let mixnet_contract = network.mixnet_contract_address().to_string();
     let nymd_url = default_nymd_endpoints()[0].clone();
@@ -16,5 +37,7 @@ pub(crate) fn new_nymd_client() -> validator_client::Client<QueryNymdClient> {
         None,
     );
 
-    validator_client::Client::new_query(client_config).expect("Failed to connect to nymd!")
+    ThreadsafeValidatorClient(Arc::new(
+        validator_client::Client::new_query(client_config).expect("Failed to connect to nymd!"),
+    ))
 }

explorer-api/src/main.rs

@@ -42,7 +42,7 @@ impl ExplorerApi {
     async fn run(&mut self) {
         info!("Explorer API starting up...");
 
-        let validator_api_url = network_defaults::default_api_endpoints()[0].clone();
+        let validator_api_url = self.state.inner.validator_client.api_endpoint();
         info!("Using validator API - {}", validator_api_url);
 
         // spawn concurrent tasks

explorer-api/src/mix_node/delegations.rs

@@ -1,11 +1,15 @@
 // Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
+use crate::client::ThreadsafeValidatorClient;
 use mixnet_contract_common::Delegation;
 
-pub(crate) async fn get_single_mixnode_delegations(pubkey: &str) -> Vec<Delegation> {
-    let client = crate::client::new_nymd_client();
+pub(crate) async fn get_single_mixnode_delegations(
+    client: &ThreadsafeValidatorClient,
+    pubkey: &str,
+) -> Vec<Delegation> {
     let delegates = match client
+        .0
         .get_all_nymd_single_mixnode_delegations(pubkey.to_string())
         .await
     {

explorer-api/src/mix_node/econ_stats.rs

@@ -0,0 +1,41 @@
+// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::client::ThreadsafeValidatorClient;
+use crate::mix_node::models::EconomicDynamicsStats;
+
+pub(crate) async fn retrieve_mixnode_econ_stats(
+    client: &ThreadsafeValidatorClient,
+    identity: &str,
+) -> Option<EconomicDynamicsStats> {
+    let stake_saturation = client
+        .0
+        .validator_api
+        .get_mixnode_stake_saturation(identity)
+        .await
+        .ok()?;
+
+    let inclusion_probability = client
+        .0
+        .validator_api
+        .get_mixnode_inclusion_probability(identity)
+        .await
+        .ok()?;
+
+    let reward_estimation = client
+        .0
+        .validator_api
+        .get_mixnode_reward_estimation(identity)
+        .await
+        .ok()?;
+
+    Some(EconomicDynamicsStats {
+        stake_saturation: stake_saturation.saturation,
+        active_set_inclusion_probability: inclusion_probability.in_active,
+        reserve_set_inclusion_probability: inclusion_probability.in_reserve,
+        estimated_total_node_reward: reward_estimation.estimated_total_node_reward,
+        estimated_operator_reward: reward_estimation.estimated_operator_reward,
+        estimated_delegators_reward: reward_estimation.estimated_delegators_reward,
+        current_interval_uptime: reward_estimation.current_interval_uptime,
+    })
+}

explorer-api/src/mix_node/http.rs

@@ -11,8 +11,11 @@ use rocket_okapi::settings::OpenApiSettings;
 
 use mixnet_contract_common::Delegation;
 
-use crate::mix_node::models::{NodeDescription, NodeStats, PrettyDetailedMixNodeBond};
-use crate::mix_nodes::delegations::get_single_mixnode_delegations;
+use crate::mix_node::delegations::get_single_mixnode_delegations;
+use crate::mix_node::econ_stats::retrieve_mixnode_econ_stats;
+use crate::mix_node::models::{
+    EconomicDynamicsStats, NodeDescription, NodeStats, PrettyDetailedMixNodeBond,
+};
 use crate::state::ExplorerApiStateContext;
 
 pub fn mix_node_make_default_routes(settings: &OpenApiSettings) -> (Vec<Route>, OpenApi) {
@@ -21,6 +24,7 @@ pub fn mix_node_make_default_routes(settings: &OpenApiSettings) -> (Vec<Route>,
         get_by_id,
         get_description,
         get_stats,
+        get_economic_dynamics_stats,
     ]
 }
 
@@ -43,8 +47,11 @@ pub(crate) async fn get_by_id(
 
 #[openapi(tag = "mix_node")]
 #[get("/<pubkey>/delegations")]
-pub(crate) async fn get_delegations(pubkey: &str) -> Json<Vec<Delegation>> {
-    Json(get_single_mixnode_delegations(pubkey).await)
+pub(crate) async fn get_delegations(
+    pubkey: &str,
+    state: &State<ExplorerApiStateContext>,
+) -> Json<Vec<Delegation>> {
+    Json(get_single_mixnode_delegations(&state.inner.validator_client, pubkey).await)
 }
 
 #[openapi(tag = "mix_node")]
@@ -134,6 +141,31 @@ pub(crate) async fn get_stats(
     }
 }
 
+#[openapi(tag = "mix_node")]
+#[get("/<pubkey>/economic-dynamics-stats")]
+pub(crate) async fn get_economic_dynamics_stats(
+    pubkey: &str,
+    state: &State<ExplorerApiStateContext>,
+) -> Option<Json<EconomicDynamicsStats>> {
+    match state.inner.mixnode.get_econ_stats(pubkey).await {
+        Some(cache_value) => {
+            trace!("Returning cached value for {}", pubkey);
+            Some(Json(cache_value))
+        }
+        None => {
+            trace!("No valid cache value for {}", pubkey);
+
+            // get fresh value from the validator API
+            let econ_stats =
+                retrieve_mixnode_econ_stats(&state.inner.validator_client, pubkey).await?;
+
+            // update cache
+            state.inner.mixnode.set_econ_stats(pubkey, econ_stats).await;
+            Some(Json(econ_stats))
+        }
+    }
+}
+
 async fn get_mix_node_description(host: &str, port: &u16) -> Result<NodeDescription, ReqwestError> {
     reqwest::get(format!("http://{}:{}/description", host, port))
         .await?

explorer-api/src/mix_node/mod.rs

@@ -1,2 +1,4 @@
+pub(crate) mod delegations;
+pub(crate) mod econ_stats;
 pub(crate) mod http;
 pub(crate) mod models;

explorer-api/src/mix_node/models.rs

@@ -32,6 +32,7 @@ pub(crate) struct PrettyDetailedMixNodeBond {
 pub(crate) struct MixNodeCache {
     pub(crate) descriptions: Cache<NodeDescription>,
     pub(crate) node_stats: Cache<NodeStats>,
+    pub(crate) econ_stats: Cache<EconomicDynamicsStats>,
 }
 
 #[derive(Clone)]
@@ -45,6 +46,7 @@ impl ThreadsafeMixNodeCache {
             inner: Arc::new(RwLock::new(MixNodeCache {
                 descriptions: Cache::new(),
                 node_stats: Cache::new(),
+                econ_stats: Cache::new(),
             })),
         }
     }
@@ -57,6 +59,10 @@ impl ThreadsafeMixNodeCache {
         self.inner.read().await.node_stats.get(identity_key)
     }
 
+    pub(crate) async fn get_econ_stats(&self, identity_key: &str) -> Option<EconomicDynamicsStats> {
+        self.inner.read().await.econ_stats.get(identity_key)
+    }
+
     pub(crate) async fn set_description(&self, identity_key: &str, description: NodeDescription) {
         self.inner
             .write()
@@ -72,6 +78,18 @@ impl ThreadsafeMixNodeCache {
             .node_stats
             .set(identity_key, node_stats);
     }
+
+    pub(crate) async fn set_econ_stats(
+        &self,
+        identity_key: &str,
+        econ_stats: EconomicDynamicsStats,
+    ) {
+        self.inner
+            .write()
+            .await
+            .econ_stats
+            .set(identity_key, econ_stats);
+    }
 }
 
 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
@@ -103,3 +121,17 @@ pub(crate) struct NodeStats {
     packets_sent_since_last_update: u64,
     packets_explicitly_dropped_since_last_update: u64,
 }
+
+#[derive(Serialize, Clone, Copy, Deserialize, JsonSchema)]
+pub(crate) struct EconomicDynamicsStats {
+    pub(crate) stake_saturation: f32,
+
+    pub(crate) active_set_inclusion_probability: f32,
+    pub(crate) reserve_set_inclusion_probability: f32,
+
+    pub(crate) estimated_total_node_reward: u64,
+    pub(crate) estimated_operator_reward: u64,
+    pub(crate) estimated_delegators_reward: u64,
+
+    pub(crate) current_interval_uptime: u8,
+}

explorer-api/src/mix_nodes/mod.rs

@@ -3,7 +3,6 @@
 
 use std::time::Duration;
 
-pub(crate) mod delegations;
 pub(crate) mod http;
 pub(crate) mod location;
 pub(crate) mod models;

explorer-api/src/state.rs

@@ -5,6 +5,7 @@ use chrono::{DateTime, Utc};
 use log::info;
 use serde::{Deserialize, Serialize};
 
+use crate::client::ThreadsafeValidatorClient;
 use mixnet_contract_common::MixNodeBond;
 
 use crate::country_statistics::country_nodes_distribution::{
@@ -28,6 +29,9 @@ pub struct ExplorerApiState {
     pub(crate) mixnodes: ThreadsafeMixNodesCache,
     pub(crate) ping: ThreadsafePingCache,
     pub(crate) validators: ThreadsafeValidatorCache,
+
+    // TODO: discuss with @MS whether this is an appropriate spot for it
+    pub(crate) validator_client: ThreadsafeValidatorClient,
 }
 
 impl ExplorerApiState {
@@ -75,6 +79,7 @@ impl ExplorerApiStateContext {
                     ),
                     ping: ThreadsafePingCache::new(),
                     validators: ThreadsafeValidatorCache::new(),
+                    validator_client: ThreadsafeValidatorClient::new(),
                 }
             }
             _ => {
@@ -90,6 +95,7 @@ impl ExplorerApiStateContext {
                     mixnodes: ThreadsafeMixNodesCache::new(),
                     ping: ThreadsafePingCache::new(),
                     validators: ThreadsafeValidatorCache::new(),
+                    validator_client: ThreadsafeValidatorClient::new(),
                 }
             }
         }

explorer-api/src/tasks.rs

@@ -8,21 +8,16 @@ use validator_client::nymd::error::NymdError;
 use validator_client::nymd::{Paging, QueryNymdClient, ValidatorResponse};
 use validator_client::ValidatorClientError;
 
-use crate::client::new_nymd_client;
 use crate::mix_nodes::CACHE_REFRESH_RATE;
 use crate::state::ExplorerApiStateContext;
 
 pub(crate) struct ExplorerApiTasks {
     state: ExplorerApiStateContext,
-    validator_client: validator_client::Client<QueryNymdClient>,
 }
 
 impl ExplorerApiTasks {
     pub(crate) fn new(state: ExplorerApiStateContext) -> Self {
-        ExplorerApiTasks {
-            state,
-            validator_client: new_nymd_client(),
-        }
+        ExplorerApiTasks { state }
     }
 
     // a helper to remove duplicate code when grabbing active/rewarded/all mixnodes
@@ -31,7 +26,7 @@ impl ExplorerApiTasks {
         F: FnOnce(&'a validator_client::Client<QueryNymdClient>) -> Fut,
         Fut: Future<Output = Result<Vec<MixNodeBond>, ValidatorClientError>>,
     {
-        let bonds = match f(&self.validator_client).await {
+        let bonds = match f(&self.state.inner.validator_client.0).await {
             Ok(result) => result,
             Err(e) => {
                 error!("Unable to retrieve mixnode bonds: {:?}", e);
@@ -51,18 +46,29 @@ impl ExplorerApiTasks {
 
     async fn retrieve_all_gateways(&self) -> Result<Vec<GatewayBond>, ValidatorClientError> {
         info!("About to retrieve all gateways...");
-        self.validator_client.get_cached_gateways().await
+        self.state
+            .inner
+            .validator_client
+            .0
+            .get_cached_gateways()
+            .await
     }
 
     async fn retrieve_all_validators(&self) -> Result<ValidatorResponse, NymdError> {
         info!("About to retrieve all validators...");
         let height = self
+            .state
+            .inner
             .validator_client
+            .0
             .nymd
             .get_current_block_height()
             .await?;
         let response: ValidatorResponse = self
+            .state
+            .inner
             .validator_client
+            .0
             .nymd
             .get_validators(height.value(), Paging::All)
             .await?;