better date serilization

merge crunch into develop

.github/workflows/push-node-status-agent.yaml

@@ -8,7 +8,7 @@ on:
         description: Which gateway probe git ref to build the image with
 
 env:
-  WORKING_DIRECTORY: "nym-node-status-agent"
+  WORKING_DIRECTORY: "nym-node-status-api/nym-node-status-agent"
   CONTAINER_NAME: "node-status-agent"
 
 jobs:
@@ -58,4 +58,4 @@ jobs:
       - name: BuildAndPushImageOnHarbor
         run: |
           docker build --build-arg GIT_REF=${{ github.event.inputs.gateway_probe_git_ref }} -f ${{ env.WORKING_DIRECTORY }}/Dockerfile . -t harbor.nymte.ch/nym/${{ env.CONTAINER_NAME }}:${{ steps.get_version.outputs.result }}-${{ steps.cleanup_gateway_probe_ref.outputs.git_ref }}
-          docker push harbor.nymte.ch/nym/${{ env.CONTAINER_NAME }} --all-tags
+          docker push harbor.nymte.ch/nym/${{ env.CONTAINER_NAME }} --all-tags

.github/workflows/push-node-status-api.yaml

@@ -3,7 +3,7 @@ on:
   workflow_dispatch:
 
 env:
-  WORKING_DIRECTORY: "nym-node-status-api"
+  WORKING_DIRECTORY: "nym-node-status-api/nym-node-status-api"
   CONTAINER_NAME: "node-status-api"
 
 jobs:

.github/workflows/push-validator-rewarder.yaml

@@ -29,7 +29,7 @@ jobs:
         uses: mikefarah/yq@v4.44.5
         id: get_version
         with:
-          cmd: yq -oy '.package.version' ${{ env.WORKING_DIRECTORY }}/nym-credential-proxy/Cargo.toml
+          cmd: yq -oy '.package.version' ${{ env.WORKING_DIRECTORY }}/Cargo.toml
 
       - name: Remove existing tag if exists
         run: |

CHANGELOG.md

@@ -4,6 +4,10 @@ Post 1.0.0 release, the changelog format is based on [Keep a Changelog](https://
 
 ## [Unreleased]
 
+## [2024.13-magura-drift] (2024-11-29)
+
+- Optimised syncing bandwidth information to storage
+
 ## [2024.13-magura-patched] (2024-11-22)
 
 - [experimental] allow clients to change between deterministic route selection based on packet headers and a pseudorandom distribution

Cargo.lock

@@ -2428,7 +2428,7 @@ dependencies = [
 
 [[package]]
 name = "explorer-api"
-version = "1.1.42"
+version = "1.1.43"
 dependencies = [
  "chrono",
  "clap 4.5.20",
@@ -4451,7 +4451,7 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
 
 [[package]]
 name = "nym-api"
-version = "1.1.46"
+version = "1.1.47"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -4701,7 +4701,7 @@ dependencies = [
 
 [[package]]
 name = "nym-cli"
-version = "1.1.44"
+version = "1.1.45"
 dependencies = [
  "anyhow",
  "base64 0.22.1",
@@ -4784,7 +4784,7 @@ dependencies = [
 
 [[package]]
 name = "nym-client"
-version = "1.1.44"
+version = "1.1.45"
 dependencies = [
  "bs58",
  "clap 4.5.20",
@@ -5536,6 +5536,7 @@ version = "0.1.0"
 dependencies = [
  "nym-credentials-interface",
  "nym-sphinx",
+ "nym-statistics-common",
  "sqlx",
  "thiserror",
  "time",
@@ -5891,7 +5892,7 @@ dependencies = [
 
 [[package]]
 name = "nym-network-requester"
-version = "1.1.45"
+version = "1.1.46"
 dependencies = [
  "addr",
  "anyhow",
@@ -5942,7 +5943,7 @@ dependencies = [
 
 [[package]]
 name = "nym-node"
-version = "1.1.11"
+version = "1.1.12"
 dependencies = [
  "anyhow",
  "bip39",
@@ -6061,7 +6062,7 @@ dependencies = [
 
 [[package]]
 name = "nym-node-status-api"
-version = "1.0.0-rc.2"
+version = "1.0.0-rc.4"
 dependencies = [
  "anyhow",
  "axum 0.7.7",
@@ -6077,6 +6078,8 @@ dependencies = [
  "nym-network-defaults",
  "nym-node-requests",
  "nym-node-status-client",
+ "nym-serde-helpers",
+ "nym-statistics-common",
  "nym-task",
  "nym-validator-client",
  "regex",
@@ -6088,6 +6091,7 @@ dependencies = [
  "strum 0.26.3",
  "strum_macros 0.26.4",
  "thiserror",
+ "time",
  "tokio",
  "tokio-util",
  "tower-http",
@@ -6306,7 +6310,7 @@ dependencies = [
 
 [[package]]
 name = "nym-socks5-client"
-version = "1.1.44"
+version = "1.1.45"
 dependencies = [
  "bs58",
  "clap 4.5.20",
@@ -6886,6 +6890,7 @@ dependencies = [
  "nym-task",
  "nym-wireguard-types",
  "thiserror",
+ "time",
  "tokio",
  "tokio-stream",
  "x25519-dalek",
@@ -6908,7 +6913,7 @@ dependencies = [
 
 [[package]]
 name = "nymvisor"
-version = "0.1.9"
+version = "0.1.10"
 dependencies = [
  "anyhow",
  "bytes",
@@ -7019,9 +7024,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.102"
+version = "0.9.104"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2"
+checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741"
 dependencies = [
  "cc",
  "libc",

clients/native/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "nym-client"
-version = "1.1.44"
+version = "1.1.45"
 authors = ["Dave Hrycyszyn <futurechimp@users.noreply.github.com>", "Jędrzej Stuczyński <andrew@nymtech.net>"]
 description = "Implementation of the Nym Client"
 edition = "2021"

clients/socks5/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "nym-socks5-client"
-version = "1.1.44"
+version = "1.1.45"
 authors = ["Dave Hrycyszyn <futurechimp@users.noreply.github.com>"]
 description = "A SOCKS5 localhost proxy that converts incoming messages to Sphinx and sends them to a Nym address"
 edition = "2021"

common/client-core/src/client/topology_control/accessor.rs

@@ -38,7 +38,7 @@ pub struct TopologyReadPermit<'a> {
     permit: RwLockReadGuard<'a, Option<NymTopology>>,
 }
 
-impl<'a> Deref for TopologyReadPermit<'a> {
+impl Deref for TopologyReadPermit<'_> {
     type Target = Option<NymTopology>;
 
     fn deref(&self) -> &Self::Target {

common/client-libs/validator-client/src/nyxd/coin.rs

@@ -32,7 +32,7 @@ impl Div<GasPrice> for Coin {
     }
 }
 
-impl<'a> Div<GasPrice> for &'a Coin {
+impl Div<GasPrice> for &Coin {
     type Output = Gas;
 
     fn div(self, rhs: GasPrice) -> Self::Output {

common/client-libs/validator-client/src/nyxd/fee/gas_price.rs

@@ -22,7 +22,7 @@ pub struct GasPrice {
     pub denom: String,
 }
 
-impl<'a> Mul<Gas> for &'a GasPrice {
+impl Mul<Gas> for &GasPrice {
     type Output = Coin;
 
     fn mul(self, gas_limit: Gas) -> Self::Output {

common/cosmwasm-smart-contracts/mixnet-contract/src/interval.rs

@@ -32,7 +32,7 @@ pub(crate) mod string_rfc3339_offset_date_time {
 
     struct Rfc3339OffsetDateTimeVisitor;
 
-    impl<'de> Visitor<'de> for Rfc3339OffsetDateTimeVisitor {
+    impl Visitor<'_> for Rfc3339OffsetDateTimeVisitor {
         type Value = OffsetDateTime;
 
         fn expecting(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result {

common/credential-verification/src/bandwidth_storage_manager.rs

@@ -111,7 +111,7 @@ impl<S: Storage + Clone + 'static> BandwidthStorageManager<S> {
     }
 
     #[instrument(level = "trace", skip_all)]
-    async fn sync_storage_bandwidth(&mut self) -> Result<()> {
+    pub async fn sync_storage_bandwidth(&mut self) -> Result<()> {
         trace!("syncing client bandwidth with the underlying storage");
         let updated = self
             .storage

common/credential-verification/src/client_bandwidth.rs

@@ -8,8 +8,8 @@ use std::time::Duration;
 use time::OffsetDateTime;
 use tokio::sync::RwLock;
 
-const DEFAULT_CLIENT_BANDWIDTH_MAX_FLUSHING_RATE: Duration = Duration::from_millis(5);
-const DEFAULT_CLIENT_BANDWIDTH_MAX_DELTA_FLUSHING_AMOUNT: i64 = 512 * 1024; // 512kB
+const DEFAULT_CLIENT_BANDWIDTH_MAX_FLUSHING_RATE: Duration = Duration::from_secs(5 * 60); // 5 minutes
+const DEFAULT_CLIENT_BANDWIDTH_MAX_DELTA_FLUSHING_AMOUNT: i64 = 5 * 1024 * 1024; // 5MB
 
 #[derive(Debug, Clone, Copy)]
 pub struct BandwidthFlushingBehaviourConfig {

common/dkg/src/bte/proof_chunking.rs

@@ -26,9 +26,8 @@ const PARALLEL_RUNS: usize = 32;
 /// `lambda` ($\lambda$) in the DKG paper
 const SECURITY_PARAMETER: usize = 256;
 
-// note: ceiling in integer division can be achieved via q = (x + y - 1) / y;
 /// ceil(SECURITY_PARAMETER / PARALLEL_RUNS) in the paper
-const NUM_CHALLENGE_BITS: usize = (SECURITY_PARAMETER + PARALLEL_RUNS - 1) / PARALLEL_RUNS;
+const NUM_CHALLENGE_BITS: usize = SECURITY_PARAMETER.div_ceil(PARALLEL_RUNS);
 
 // type alias for ease of use
 type FirstChallenge = Vec<Vec<Vec<u64>>>;

common/dkg/src/interpolation/polynomial.rs

@@ -196,7 +196,7 @@ impl<'b> Add<&'b Polynomial> for Polynomial {
     }
 }
 
-impl<'a> Add<Polynomial> for &'a Polynomial {
+impl Add<Polynomial> for &Polynomial {
     type Output = Polynomial;
 
     fn add(self, rhs: Polynomial) -> Polynomial {
@@ -212,10 +212,10 @@ impl Add<Polynomial> for Polynomial {
     }
 }
 
-impl<'a, 'b> Add<&'b Polynomial> for &'a Polynomial {
+impl<'a> Add<&'a Polynomial> for &Polynomial {
     type Output = Polynomial;
 
-    fn add(self, rhs: &'b Polynomial) -> Self::Output {
+    fn add(self, rhs: &'a Polynomial) -> Self::Output {
         let len = self.coefficients.len();
         let rhs_len = rhs.coefficients.len();
 

common/gateway-requests/src/registration/handshake/mod.rs

@@ -37,7 +37,7 @@ pub struct GatewayHandshake<'a> {
     handshake_future: BoxFuture<'a, Result<SharedGatewayKey, HandshakeError>>,
 }
 
-impl<'a> Future for GatewayHandshake<'a> {
+impl Future for GatewayHandshake<'_> {
     type Output = Result<SharedGatewayKey, HandshakeError>;
 
     fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {

common/gateway-stats-storage/Cargo.toml

@@ -22,6 +22,7 @@ tracing = { workspace = true }
 
 nym-sphinx = { path = "../nymsphinx" }
 nym-credentials-interface = { path = "../credentials-interface" }
+nym-statistics-common = { path = "../statistics" }
 
 
 [build-dependencies]

common/gateway-stats-storage/src/lib.rs

@@ -2,8 +2,9 @@
 // SPDX-License-Identifier: GPL-3.0-only
 
 use error::StatsStorageError;
-use models::{ActiveSession, FinishedSession, SessionType, StoredFinishedSession};
+use models::{ActiveSession, FinishedSession, StoredFinishedSession};
 use nym_sphinx::DestinationAddressBytes;
+use nym_statistics_common::gateways::SessionType;
 use sessions::SessionManager;
 use sqlx::ConnectOptions;
 use std::path::Path;

common/gateway-stats-storage/src/models.rs

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: GPL-3.0-only
 
 use nym_credentials_interface::TicketType;
+use nym_statistics_common::gateways::SessionType;
 use sqlx::prelude::FromRow;
 use time::{Duration, OffsetDateTime};
 
@@ -25,42 +26,6 @@ pub struct FinishedSession {
     pub typ: SessionType,
 }
 
-#[derive(PartialEq)]
-pub enum SessionType {
-    Vpn,
-    Mixnet,
-    Unknown,
-}
-
-impl SessionType {
-    pub fn to_string(&self) -> &str {
-        match self {
-            Self::Vpn => "vpn",
-            Self::Mixnet => "mixnet",
-            Self::Unknown => "unknown",
-        }
-    }
-
-    pub fn from_string(s: &str) -> Self {
-        match s {
-            "vpn" => Self::Vpn,
-            "mixnet" => Self::Mixnet,
-            _ => Self::Unknown,
-        }
-    }
-}
-
-impl From<TicketType> for SessionType {
-    fn from(value: TicketType) -> Self {
-        match value {
-            TicketType::V1MixnetEntry => Self::Mixnet,
-            TicketType::V1MixnetExit => Self::Mixnet,
-            TicketType::V1WireguardEntry => Self::Vpn,
-            TicketType::V1WireguardExit => Self::Vpn,
-        }
-    }
-}
-
 #[derive(FromRow)]
 pub(crate) struct StoredActiveSession {
     start_time: OffsetDateTime,

common/nym_offline_compact_ecash/src/scheme/coin_indices_signatures.rs

@@ -324,18 +324,6 @@ pub fn unchecked_aggregate_indices_signatures(
     _aggregate_indices_signatures(params, vk, signatures_shares, false)
 }
 
-/// Generates parameters for the scheme setup.
-///
-/// # Arguments
-///
-/// * `total_coins` - it is the number of coins in a freshly generated wallet. It is the public parameter of the scheme.
-///
-/// # Returns
-///
-/// A `Parameters` struct containing group parameters, public key, the number of signatures (`total_coins`),
-/// and a map of signatures for each index `l`.
-///
-
 #[cfg(test)]
 mod tests {
     use super::*;

common/nym_offline_compact_ecash/src/scheme/keygen.rs

@@ -264,7 +264,7 @@ impl<'b> Add<&'b VerificationKeyAuth> for VerificationKeyAuth {
     }
 }
 
-impl<'a> Mul<Scalar> for &'a VerificationKeyAuth {
+impl Mul<Scalar> for &VerificationKeyAuth {
     type Output = VerificationKeyAuth;
 
     #[inline]

common/nym_offline_compact_ecash/src/scheme/mod.rs

@@ -984,7 +984,7 @@ pub struct SerialNumberRef<'a> {
     pub(crate) inner: &'a [G1Projective],
 }
 
-impl<'a> SerialNumberRef<'a> {
+impl SerialNumberRef<'_> {
     pub fn to_bytes(&self) -> Vec<u8> {
         let ss_len = self.inner.len();
         let mut bytes: Vec<u8> = Vec::with_capacity(ss_len * 48);

common/nymcoconut/src/elgamal.rs

@@ -206,10 +206,10 @@ impl Deref for PublicKey {
     }
 }
 
-impl<'a, 'b> Mul<&'b Scalar> for &'a PublicKey {
+impl<'a> Mul<&'a Scalar> for &PublicKey {
     type Output = G1Projective;
 
-    fn mul(self, rhs: &'b Scalar) -> Self::Output {
+    fn mul(self, rhs: &'a Scalar) -> Self::Output {
         self.0 * rhs
     }
 }

common/nymcoconut/src/scheme/keygen.rs

@@ -305,7 +305,7 @@ impl<'b> Add<&'b VerificationKey> for VerificationKey {
     }
 }
 
-impl<'a> Mul<Scalar> for &'a VerificationKey {
+impl Mul<Scalar> for &VerificationKey {
     type Output = VerificationKey;
 
     #[inline]

common/nymsphinx/addressing/src/clients.rs

@@ -64,7 +64,7 @@ impl<'de> Deserialize<'de> for Recipient {
     {
         struct RecipientVisitor;
 
-        impl<'de> Visitor<'de> for RecipientVisitor {
+        impl Visitor<'_> for RecipientVisitor {
             type Value = Recipient;
 
             fn expecting(&self, formatter: &mut Formatter<'_>) -> fmt::Result {

common/nymsphinx/addressing/src/nodes.rs

@@ -1,6 +1,13 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
+//! Encoding and decoding node routing information.
+//!
+//! This module is responsible for encoding and decoding node routing information, so that
+//! they could be later put into an appropriate field in a sphinx header.
+//! Currently, that routing information is an IP address, but in principle it can be anything
+//!  for as long as it's going to fit in the field.
+
 use nym_crypto::asymmetric::identity;
 use nym_sphinx_types::{NodeAddressBytes, NODE_ADDRESS_LENGTH};
 
@@ -12,13 +19,6 @@ use thiserror::Error;
 pub type NodeIdentity = identity::PublicKey;
 pub const NODE_IDENTITY_SIZE: usize = identity::PUBLIC_KEY_LENGTH;
 
-/// Encodoing and decoding node routing information.
-///
-/// This module is responsible for encoding and decoding node routing information, so that
-/// they could be later put into an appropriate field in a sphinx header.
-/// Currently, that routing information is an IP address, but in principle it can be anything
-/// for as long as it's going to fit in the field.
-
 /// MAX_UNPADDED_LEN represents maximum length an unpadded address could have.
 /// In this case it's an ipv6 socket address (with version prefix)
 pub const MAX_NODE_ADDRESS_UNPADDED_LEN: usize = 19;

common/nymsphinx/anonymous-replies/src/reply_surb.rs

@@ -56,7 +56,7 @@ impl<'de> Deserialize<'de> for ReplySurb {
     {
         struct ReplySurbVisitor;
 
-        impl<'de> Visitor<'de> for ReplySurbVisitor {
+        impl Visitor<'_> for ReplySurbVisitor {
             type Value = ReplySurb;
 
             fn expecting(&self, formatter: &mut Formatter<'_>) -> fmt::Result {

common/socks5/requests/src/request.rs

@@ -253,25 +253,15 @@ impl Socks5RequestContent {
     /// Deserialize the request type, connection id, destination address and port,
     /// and the request body from bytes.
     ///
-    // TODO: this was already inaccurate
-    // /// Serialized bytes looks like this:
-    // ///
-    // /// --------------------------------------------------------------------------------------
-    // ///  request_flag | connection_id | address_length | remote_address_bytes | request_data |
-    // ///        1      |       8       |      2         |    address_length    |    ...       |
-    // /// --------------------------------------------------------------------------------------
-    ///
     /// The request_flag tells us whether this is a new connection request (`new_connect`),
     /// an already-established connection we should send up (`new_send`), or
     /// a request to close an established connection (`new_close`).
-
     // connect:
     // RequestFlag::Connect || CONN_ID || ADDR_LEN || ADDR || <RETURN_ADDR>
     //
     // send:
     // RequestFlag::Send || CONN_ID || LOCAL_CLOSED || DATA
     // where DATA: SEQ || TRUE_DATA
-
     pub fn try_from_bytes(b: &[u8]) -> Result<Socks5RequestContent, RequestDeserializationError> {
         // each request needs to at least contain flag and ConnectionId
         if b.is_empty() {

common/statistics/src/gateways.rs

@@ -87,3 +87,39 @@ pub enum SessionEvent {
         client: DestinationAddressBytes,
     },
 }
+
+#[derive(PartialEq)]
+pub enum SessionType {
+    Vpn,
+    Mixnet,
+    Unknown,
+}
+
+impl SessionType {
+    pub fn to_string(&self) -> &str {
+        match self {
+            Self::Vpn => "vpn",
+            Self::Mixnet => "mixnet",
+            Self::Unknown => "unknown",
+        }
+    }
+
+    pub fn from_string(s: &str) -> Self {
+        match s {
+            "vpn" => Self::Vpn,
+            "mixnet" => Self::Mixnet,
+            _ => Self::Unknown,
+        }
+    }
+}
+
+impl From<TicketType> for SessionType {
+    fn from(value: TicketType) -> Self {
+        match value {
+            TicketType::V1MixnetEntry => Self::Mixnet,
+            TicketType::V1MixnetExit => Self::Mixnet,
+            TicketType::V1WireguardEntry => Self::Vpn,
+            TicketType::V1WireguardExit => Self::Vpn,
+        }
+    }
+}

common/wireguard/Cargo.toml

@@ -26,6 +26,7 @@ log.workspace = true
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["rt-multi-thread", "net", "io-util"] }
 tokio-stream = { workspace = true }
+time = { workspace = true }
 
 nym-authenticator-requests = { path = "../authenticator-requests" }
 nym-credential-verification = { path = "../credential-verification" }

common/wireguard/src/lib.rs

@@ -20,6 +20,7 @@ use tokio::sync::mpsc::{self, Receiver, Sender};
 pub(crate) mod error;
 pub mod peer_controller;
 pub mod peer_handle;
+pub mod peer_storage_manager;
 
 pub struct WgApiWrapper {
     inner: WGApi,
@@ -118,7 +119,7 @@ pub async fn start_wireguard<St: nym_gateway_storage::Storage + Clone + 'static>
         storage
             .insert_wireguard_peer(peer, bandwidth_manager.is_some())
             .await?;
-        peer_bandwidth_managers.insert(peer.public_key.clone(), bandwidth_manager);
+        peer_bandwidth_managers.insert(peer.public_key.clone(), (bandwidth_manager, peer.clone()));
     }
     wg_api.create_interface()?;
     let interface_config = InterfaceConfiguration {

common/wireguard/src/peer_controller.rs

@@ -20,9 +20,9 @@ use std::{collections::HashMap, sync::Arc};
 use tokio::sync::{mpsc, RwLock};
 use tokio_stream::{wrappers::IntervalStream, StreamExt};
 
-use crate::peer_handle::PeerHandle;
 use crate::WgApiWrapper;
 use crate::{error::Error, peer_handle::SharedBandwidthStorageManager};
+use crate::{peer_handle::PeerHandle, peer_storage_manager::PeerStorageManager};
 
 pub enum PeerControlRequest {
     AddPeer {
@@ -79,7 +79,7 @@ impl<St: Storage + Clone + 'static> PeerController<St> {
         storage: St,
         wg_api: Arc<WgApiWrapper>,
         initial_host_information: Host,
-        bw_storage_managers: HashMap<Key, Option<SharedBandwidthStorageManager<St>>>,
+        bw_storage_managers: HashMap<Key, (Option<SharedBandwidthStorageManager<St>>, Peer)>,
         request_tx: mpsc::Sender<PeerControlRequest>,
         request_rx: mpsc::Receiver<PeerControlRequest>,
         task_client: nym_task::TaskClient,
@@ -88,11 +88,16 @@ impl<St: Storage + Clone + 'static> PeerController<St> {
             tokio::time::interval(DEFAULT_PEER_TIMEOUT_CHECK),
         );
         let host_information = Arc::new(RwLock::new(initial_host_information));
-        for (public_key, bandwidth_storage_manager) in bw_storage_managers.iter() {
-            let mut handle = PeerHandle::new(
+        for (public_key, (bandwidth_storage_manager, peer)) in bw_storage_managers.iter() {
+            let peer_storage_manager = PeerStorageManager::new(
                 storage.clone(),
+                peer.clone(),
+                bandwidth_storage_manager.is_some(),
+            );
+            let mut handle = PeerHandle::new(
                 public_key.clone(),
                 host_information.clone(),
+                peer_storage_manager,
                 bandwidth_storage_manager.clone(),
                 request_tx.clone(),
                 &task_client,
@@ -103,6 +108,10 @@ impl<St: Storage + Clone + 'static> PeerController<St> {
                 }
             });
         }
+        let bw_storage_managers = bw_storage_managers
+            .into_iter()
+            .map(|(k, (m, _))| (k, m))
+            .collect();
 
         PeerController {
             storage,
@@ -184,10 +193,15 @@ impl<St: Storage + Clone + 'static> PeerController<St> {
             Self::generate_bandwidth_manager(self.storage.clone(), &peer.public_key)
                 .await?
                 .map(|bw_m| Arc::new(RwLock::new(bw_m)));
-        let mut handle = PeerHandle::new(
+        let peer_storage_manager = PeerStorageManager::new(
             self.storage.clone(),
+            peer.clone(),
+            bandwidth_storage_manager.is_some(),
+        );
+        let mut handle = PeerHandle::new(
             peer.public_key.clone(),
             self.host_information.clone(),
+            peer_storage_manager,
             bandwidth_storage_manager.clone(),
             self.request_tx.clone(),
             &self.task_client,

common/wireguard/src/peer_handle.rs

@@ -3,6 +3,7 @@
 
 use crate::error::Error;
 use crate::peer_controller::PeerControlRequest;
+use crate::peer_storage_manager::PeerStorageManager;
 use defguard_wireguard_rs::host::Peer;
 use defguard_wireguard_rs::{host::Host, key::Key};
 use futures::channel::oneshot;
@@ -21,9 +22,9 @@ pub(crate) type SharedBandwidthStorageManager<St> = Arc<RwLock<BandwidthStorageM
 const AUTO_REMOVE_AFTER: Duration = Duration::from_secs(60 * 60 * 24 * 30); // 30 days
 
 pub struct PeerHandle<St> {
-    storage: St,
     public_key: Key,
     host_information: Arc<RwLock<Host>>,
+    peer_storage_manager: PeerStorageManager<St>,
     bandwidth_storage_manager: Option<SharedBandwidthStorageManager<St>>,
     request_tx: mpsc::Sender<PeerControlRequest>,
     timeout_check_interval: IntervalStream,
@@ -33,9 +34,9 @@ pub struct PeerHandle<St> {
 
 impl<St: Storage + Clone + 'static> PeerHandle<St> {
     pub fn new(
-        storage: St,
         public_key: Key,
         host_information: Arc<RwLock<Host>>,
+        peer_storage_manager: PeerStorageManager<St>,
         bandwidth_storage_manager: Option<SharedBandwidthStorageManager<St>>,
         request_tx: mpsc::Sender<PeerControlRequest>,
         task_client: &TaskClient,
@@ -46,9 +47,9 @@ impl<St: Storage + Clone + 'static> PeerHandle<St> {
         let mut task_client = task_client.fork(format!("peer-{public_key}"));
         task_client.disarm();
         PeerHandle {
-            storage,
             public_key,
             host_information,
+            peer_storage_manager,
             bandwidth_storage_manager,
             request_tx,
             timeout_check_interval,
@@ -84,16 +85,19 @@ impl<St: Storage + Clone + 'static> PeerHandle<St> {
                 .ok_or(Error::InconsistentConsumedBytes)?
                 .try_into()
                 .map_err(|_| Error::InconsistentConsumedBytes)?;
-            if spent_bandwidth > 0
-                && bandwidth_manager
+            if spent_bandwidth > 0 {
+                self.peer_storage_manager.update_trx(kernel_peer);
+                if bandwidth_manager
                     .write()
                     .await
                     .try_use_bandwidth(spent_bandwidth)
                     .await
                     .is_err()
-            {
-                let success = self.remove_peer().await?;
-                return Ok(!success);
+                {
+                    let success = self.remove_peer().await?;
+                    self.peer_storage_manager.remove_peer();
+                    return Ok(!success);
+                }
             }
         } else {
             if SystemTime::now().duration_since(self.startup_timestamp)? >= AUTO_REMOVE_AFTER {
@@ -132,7 +136,7 @@ impl<St: Storage + Clone + 'static> PeerHandle<St> {
                             // the host information hasn't beed updated yet
                             continue;
                         };
-                    let Some(storage_peer) = self.storage.get_wireguard_peer(&self.public_key.to_string()).await? else {
+                    let Some(storage_peer) = self.peer_storage_manager.get_peer() else {
                         log::debug!("Peer {:?} not in storage anymore, shutting down handle", self.public_key);
                         return Ok(());
                     };
@@ -141,12 +145,18 @@ impl<St: Storage + Clone + 'static> PeerHandle<St> {
                         return Ok(());
                     } else {
                         // Update storage values
-                        self.storage.insert_wireguard_peer(&kernel_peer, self.bandwidth_storage_manager.is_some()).await?;
+                        self.peer_storage_manager.sync_storage_peer().await?;
                     }
                 }
 
                 _ = self.task_client.recv() => {
                     log::trace!("PeerHandle: Received shutdown");
+                    if let Some(bandwidth_manager) = &self.bandwidth_storage_manager {
+                        if let Err(e) = bandwidth_manager.write().await.sync_storage_bandwidth().await {
+                            log::error!("Storage sync failed - {e}, unaccounted bandwidth might have been consumed");
+                        }
+                    }
+                    log::trace!("PeerHandle: Finished shutdown");
                 }
             }
         }

common/wireguard/src/peer_storage_manager.rs

@@ -0,0 +1,138 @@
+// Copyright 2024 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::error::Error;
+use defguard_wireguard_rs::host::Peer;
+use nym_gateway_storage::models::WireguardPeer;
+use nym_gateway_storage::Storage;
+use std::time::Duration;
+use time::OffsetDateTime;
+
+const DEFAULT_PEER_MAX_FLUSHING_RATE: Duration = Duration::from_secs(60 * 60 * 24); // 24h
+const DEFAULT_PEER_MAX_DELTA_FLUSHING_AMOUNT: u64 = 512 * 1024 * 1024; // 512MB
+
+#[derive(Debug, Clone, Copy)]
+pub struct PeerFlushingBehaviourConfig {
+    /// Defines maximum delay between peer information being flushed to the persistent storage.
+    pub peer_max_flushing_rate: Duration,
+
+    /// Defines a maximum change in peer before it gets flushed to the persistent storage.
+    pub peer_max_delta_flushing_amount: u64,
+}
+
+impl Default for PeerFlushingBehaviourConfig {
+    fn default() -> Self {
+        Self {
+            peer_max_flushing_rate: DEFAULT_PEER_MAX_FLUSHING_RATE,
+            peer_max_delta_flushing_amount: DEFAULT_PEER_MAX_DELTA_FLUSHING_AMOUNT,
+        }
+    }
+}
+
+pub struct PeerStorageManager<S> {
+    pub(crate) storage: S,
+    pub(crate) peer_information: Option<PeerInformation>,
+    pub(crate) cfg: PeerFlushingBehaviourConfig,
+    pub(crate) with_client_id: bool,
+}
+
+impl<S: Storage + Clone + 'static> PeerStorageManager<S> {
+    pub(crate) fn new(storage: S, peer: Peer, with_client_id: bool) -> Self {
+        let peer_information = Some(PeerInformation::new(peer));
+        Self {
+            storage,
+            peer_information,
+            cfg: PeerFlushingBehaviourConfig::default(),
+            with_client_id,
+        }
+    }
+
+    pub(crate) fn get_peer(&self) -> Option<WireguardPeer> {
+        self.peer_information
+            .as_ref()
+            .map(|p| p.peer.clone().into())
+    }
+
+    pub(crate) fn remove_peer(&mut self) {
+        self.peer_information = None;
+    }
+
+    pub(crate) fn update_trx(&mut self, kernel_peer: &Peer) {
+        if let Some(peer_information) = self.peer_information.as_mut() {
+            peer_information.update_trx_bytes(kernel_peer.tx_bytes, kernel_peer.rx_bytes);
+        }
+    }
+
+    pub(crate) async fn sync_storage_peer(&mut self) -> Result<(), Error> {
+        let Some(peer_information) = self.peer_information.as_mut() else {
+            return Ok(());
+        };
+        if !peer_information.should_sync(self.cfg) {
+            return Ok(());
+        }
+        if self
+            .storage
+            .get_wireguard_peer(&peer_information.peer().public_key.to_string())
+            .await?
+            .is_none()
+        {
+            self.peer_information = None;
+            return Ok(());
+        }
+        self.storage
+            .insert_wireguard_peer(peer_information.peer(), self.with_client_id)
+            .await?;
+
+        peer_information.resync_peer_with_storage();
+
+        Ok(())
+    }
+}
+
+#[derive(Clone, Debug)]
+pub(crate) struct PeerInformation {
+    pub(crate) peer: Peer,
+    pub(crate) last_synced: OffsetDateTime,
+
+    pub(crate) bytes_delta_since_sync: u64,
+}
+
+impl PeerInformation {
+    pub fn new(peer: Peer) -> PeerInformation {
+        PeerInformation {
+            peer,
+            last_synced: OffsetDateTime::now_utc(),
+            bytes_delta_since_sync: 0,
+        }
+    }
+
+    pub(crate) fn should_sync(&self, cfg: PeerFlushingBehaviourConfig) -> bool {
+        if self.bytes_delta_since_sync >= cfg.peer_max_delta_flushing_amount {
+            return true;
+        }
+
+        if self.last_synced + cfg.peer_max_flushing_rate < OffsetDateTime::now_utc()
+            && self.bytes_delta_since_sync != 0
+        {
+            return true;
+        }
+
+        false
+    }
+
+    pub(crate) fn peer(&self) -> &Peer {
+        &self.peer
+    }
+
+    pub(crate) fn update_trx_bytes(&mut self, tx_bytes: u64, rx_bytes: u64) {
+        self.bytes_delta_since_sync += tx_bytes.saturating_sub(self.peer.tx_bytes)
+            + rx_bytes.saturating_sub(self.peer.rx_bytes);
+        self.peer.tx_bytes = tx_bytes;
+        self.peer.rx_bytes = rx_bytes;
+    }
+
+    pub(crate) fn resync_peer_with_storage(&mut self) {
+        self.bytes_delta_since_sync = 0;
+        self.last_synced = OffsetDateTime::now_utc();
+    }
+}

documentation/docs/pages/operators/changelog.mdx

@@ -34,6 +34,28 @@ This page displays a full list of all the changes during our release cycle from
 <VarInfo />
 
 
+## `magura-drift`
+
+Second patch to `v2024.13-magura` release version. 
+
+- [Release binaries](https://github.com/nymtech/nym/releases/tag/nym-binaries-v2024.13-magura-drift)
+- [`nym-node`](nodes/nym-node.mdx) version `1.1.12`
+
+```sh
+nym-node
+Binary Name:        nym-node
+Build Timestamp:    2024-11-29T13:10:51.813092288Z
+Build Version:      1.1.12
+Commit SHA:         4a9a5579c40ad956163ea02e01d7b53aef2ac8ef
+Commit Date:        2024-11-29T14:06:32.000000000+01:00
+Commit Branch:      HEAD
+rustc Version:      1.83.0
+rustc Channel:      stable
+cargo Profile:      release
+```
+
+-  This patch adds a peer storage manager to fix issues causing external clients to be blocked, ensuring they can successfully connect to different nodes. 
+
 ## `v2024.13-magura-patched`
 
 - [Release binaries](https://github.com/nymtech/nym/releases/tag/nym-binaries-v2024.13-magura-patched)
@@ -54,12 +76,11 @@ cargo Profile:      release
 
 
 <Callout type="warning" emoji="⚠️">
-After changes coming along with `v2024.13-magura` (`nym-node v1.1.10`), Nym Explorer is no longer picking all values correctly. Insstead of fixing this outdated explorer, we are working on a new one, coming out soon. 
+After changes coming along with `v2024.13-magura` (`nym-node v1.1.10`), Nym Explorer is no longer picking all values correctly. Instead of fixing this outdated explorer, we are working on a new one, coming out soon. 
 
 [Nym Harbourmaster](https://harbourmaster.nymtech.net) has cache of 90min, expect your values to be updated with delay. We are aware of some issues with Nym Harbourmaster and working hard to resolve them in the upcoming explorer v2. To check your routing values in real time, you can use [`nym-gateway-probe`](nodes/performance-and-testing/gateway-probe).
 </Callout>
 
-
 ### Operators Updates & Tools
 
 - Updated [`network_tunnel_manager.sh`](https://github.com/nymtech/nym/blob/develop/scripts/network_tunnel_manager.sh) (moved to our monorepo) helps operators to configure their IP tables rules for `nymtun` and `wireguard` routing.

documentation/docs/pages/operators/nodes/maintenance/manual-upgrade.mdx

@@ -53,7 +53,7 @@ journalctl -f -u nym-node.service
 </Steps>
 
 <Callout type="warning" emoji="⚠️">
-After changes coming along with `v2024.13-magura` (`nym-node v1.1.10`), Nym Explorer is no longer picking all values correctly. Insstead of fixing this outdated explorer, we are working on a new one, coming out soon.
+After changes coming along with `v2024.13-magura` (`nym-node v1.1.10`), Nym Explorer is no longer picking all values correctly. Instead of fixing this outdated explorer, we are working on a new one, coming out soon.
 
 [Nym Harbourmaster](https://harbourmaster.nymtech.net) has cache of 90min, expect your values to be updated with delay. We are aware of some issues with Nym Harbourmaster and working hard to resolve them in the upcoming explorer v2. To check your routing values in real time, you can use [`nym-gateway-probe`](../performance-and-testing/gateway-probe).
 </Callout>

documentation/docs/pages/operators/nodes/nym-node/bonding.mdx

@@ -120,7 +120,7 @@ From `nym-wallet` version `1.2.15` onward the application allows and prompts ope
 
 ###### 2. Verify the binary and extract it if needed
 
-- Download [`hashes.json`]https://github.com/nymtech/nym/releases/download/nym-wallet-v1.2.15/hashes.json
+- Download [`hashes.json`](https://github.com/nymtech/nym/releases/download/nym-wallet-v1.2.15/hashes.json)
 - Open it with your text editor or print it's content with `cat hashes.json`
 - Run `sha256sum <WALLET_BINARY>` for example `sha256sum ./nym-wallet_1.2.15_amd64.AppImage`
 - If your have to extract it (like `.tar.gz`) do it

documentation/docs/pages/operators/nodes/nym-node/setup.mdx

@@ -17,12 +17,12 @@ This documentation page provides a guide on how to set up and run a [NYM NODE](.
 ```sh
 nym-node
 Binary Name:        nym-node
-Build Timestamp:    2024-11-22T14:30:48.067329245Z
-Build Version:      1.1.11
-Commit SHA:         01c7b2819ee3d328deccd303b4113ff415d7e276
-Commit Date:        2024-11-22T10:50:59.000000000+01:00
+Build Timestamp:    2024-11-29T13:10:51.813092288Z
+Build Version:      1.1.12
+Commit SHA:         4a9a5579c40ad956163ea02e01d7b53aef2ac8ef
+Commit Date:        2024-11-29T14:06:32.000000000+01:00
 Commit Branch:      HEAD
-rustc Version:      1.82.0
+rustc Version:      1.83.0
 rustc Channel:      stable
 cargo Profile:      release
 ```

explorer-api/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "explorer-api"
-version = "1.1.42"
+version = "1.1.43"
 edition = "2021"
 license.workspace = true
 

explorer-api/src/geo_ip/location.rs

@@ -166,7 +166,7 @@ impl GeoIp {
     }
 }
 
-impl<'a> TryFrom<&City<'a>> for Location {
+impl TryFrom<&City<'_>> for Location {
     type Error = String;
 
     fn try_from(city: &City) -> Result<Self, Self::Error> {

explorer-api/src/guards/location.rs

@@ -65,7 +65,7 @@ impl<'r> FromRequest<'r> for Location {
     }
 }
 
-impl<'a> OpenApiFromRequest<'a> for Location {
+impl OpenApiFromRequest<'_> for Location {
     fn from_request_input(
         _gen: &mut OpenApiGenerator,
         _name: String,

gateway/src/node/helpers.rs

@@ -5,6 +5,8 @@ use async_trait::async_trait;
 use nym_sdk::{NymApiTopologyProvider, NymApiTopologyProviderConfig, UserAgent};
 use nym_topology::{gateway, NymTopology, TopologyProvider};
 use std::sync::Arc;
+use std::time::Duration;
+use time::OffsetDateTime;
 use tokio::sync::Mutex;
 use tracing::debug;
 use url::Url;
@@ -17,6 +19,7 @@ pub struct GatewayTopologyProvider {
 impl GatewayTopologyProvider {
     pub fn new(
         gateway_node: gateway::LegacyNode,
+        cache_ttl: Duration,
         user_agent: UserAgent,
         nym_api_url: Vec<Url>,
     ) -> GatewayTopologyProvider {
@@ -31,6 +34,9 @@ impl GatewayTopologyProvider {
                     env!("CARGO_PKG_VERSION").to_string(),
                     Some(user_agent),
                 ),
+                cache_ttl,
+                cached_at: OffsetDateTime::UNIX_EPOCH,
+                cached: None,
                 gateway_node,
             })),
         }
@@ -39,25 +45,53 @@ impl GatewayTopologyProvider {
 
 struct GatewayTopologyProviderInner {
     inner: NymApiTopologyProvider,
+    cache_ttl: Duration,
+    cached_at: OffsetDateTime,
+    cached: Option<NymTopology>,
     gateway_node: gateway::LegacyNode,
 }
 
+impl GatewayTopologyProviderInner {
+    fn cached_topology(&self) -> Option<NymTopology> {
+        if let Some(cached_topology) = &self.cached {
+            if self.cached_at + self.cache_ttl > OffsetDateTime::now_utc() {
+                return Some(cached_topology.clone());
+            }
+        }
+
+        None
+    }
+
+    async fn update_cache(&mut self) -> Option<NymTopology> {
+        let updated_cache = match self.inner.get_new_topology().await {
+            None => None,
+            Some(mut base) => {
+                if !base.gateway_exists(&self.gateway_node.identity_key) {
+                    debug!(
+                        "{} didn't exist in topology. inserting it.",
+                        self.gateway_node.identity_key
+                    );
+                    base.insert_gateway(self.gateway_node.clone());
+                }
+                Some(base)
+            }
+        };
+
+        self.cached_at = OffsetDateTime::now_utc();
+        self.cached = updated_cache.clone();
+
+        updated_cache
+    }
+}
+
 #[async_trait]
 impl TopologyProvider for GatewayTopologyProvider {
     async fn get_new_topology(&mut self) -> Option<NymTopology> {
         let mut guard = self.inner.lock().await;
-        match guard.inner.get_new_topology().await {
-            None => None,
-            Some(mut base) => {
-                if !base.gateway_exists(&guard.gateway_node.identity_key) {
-                    debug!(
-                        "{} didn't exist in topology. inserting it.",
-                        guard.gateway_node.identity_key
-                    );
-                    base.insert_gateway(guard.gateway_node.clone());
-                }
-                Some(base)
-            }
+        // check the cache
+        if let Some(cached) = guard.cached_topology() {
+            return Some(cached);
         }
+        guard.update_cache().await
     }
 }

gateway/src/node/mod.rs

@@ -30,6 +30,7 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr};
 use std::path::PathBuf;
 use std::process;
 use std::sync::Arc;
+use std::time::Duration;
 use tracing::*;
 
 pub(crate) mod client_handling;
@@ -148,8 +149,11 @@ impl<St> Gateway<St> {
     }
 
     fn gateway_topology_provider(&self) -> GatewayTopologyProvider {
+        // TODO: make topology ttl configurable
+        // (to be done in reeses with the final smooshing)
         GatewayTopologyProvider::new(
             self.as_topology_node(),
+            Duration::from_secs(5 * 60),
             self.user_agent.clone(),
             self.config.gateway.nym_api_urls.clone(),
         )
@@ -231,22 +235,28 @@ impl<St> Gateway<St> {
             forwarding_channel,
             router_tx,
         );
-        let all_peers = self.client_storage.get_all_wireguard_peers().await?;
-        let used_private_network_ips = all_peers
-            .iter()
-            .cloned()
-            .map(|wireguard_peer| {
-                defguard_wireguard_rs::host::Peer::try_from(wireguard_peer).map(|mut peer| {
-                    peer.allowed_ips
-                        .pop()
-                        .ok_or(Box::new(GatewayError::InternalWireguardError(format!(
-                            "no private IP set for peer {}",
-                            peer.public_key
-                        ))))
-                        .map(|p| p.ip)
-                })
-            })
-            .collect::<Result<Result<Vec<_>, _>, _>>()??;
+        let mut used_private_network_ips = vec![];
+        let mut all_peers = vec![];
+        for wireguard_peer in self
+            .client_storage
+            .get_all_wireguard_peers()
+            .await?
+            .into_iter()
+        {
+            let mut peer = defguard_wireguard_rs::host::Peer::try_from(wireguard_peer.clone())?;
+            let Some(peer) = peer.allowed_ips.pop() else {
+                tracing::warn!(
+                    "Peer {} has empty allowed ips. It will be removed",
+                    peer.public_key
+                );
+                self.client_storage
+                    .remove_wireguard_peer(&peer.public_key.to_string())
+                    .await?;
+                continue;
+            };
+            used_private_network_ips.push(peer.ip);
+            all_peers.push(wireguard_peer);
+        }
 
         if let Some(wireguard_data) = self.wireguard_data.take() {
             let (on_start_tx, on_start_rx) = oneshot::channel();

nym-api/Cargo.toml

@@ -4,7 +4,7 @@
 [package]
 name = "nym-api"
 license = "GPL-3.0"
-version = "1.1.46"
+version = "1.1.47"
 authors.workspace = true
 edition = "2021"
 rust-version.workspace = true
@@ -121,7 +121,7 @@ nym-types = { path = "../common/types" }
 nym-http-api-common = { path = "../common/http-api-common", features = ["utoipa"] }
 nym-serde-helpers = { path = "../common/serde-helpers", features = ["date"] }
 nym-ticketbooks-merkle = { path = "../common/ticketbooks-merkle" }
-nym-statistics-common = {path ="../common/statistics" }
+nym-statistics-common = { path = "../common/statistics" }
 
 [features]
 no-reward = []

nym-api/migrations/20241127110000_add_monitor_run_indexes.sql

@@ -0,0 +1,8 @@
+/*
+ * Copyright 2024 - Nym Technologies SA <contact@nymtech.net>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+CREATE INDEX IF NOT EXISTS monitor_run_id on monitor_run(id);
+CREATE INDEX IF NOT EXISTS monitor_run_timestamp on monitor_run(timestamp);
+CREATE INDEX IF NOT EXISTS testing_route_monitor_run_id on testing_route(monitor_run_id);

nym-api/nym-api-requests/src/helpers.rs

@@ -64,7 +64,7 @@ pub(crate) mod overengineered_offset_date_time_serde {
         ])),
     ];
 
-    impl<'de> Visitor<'de> for OffsetDateTimeVisitor {
+    impl Visitor<'_> for OffsetDateTimeVisitor {
         type Value = OffsetDateTime;
 
         fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result {

nym-api/src/ecash/api_routes/aggregation.rs

@@ -6,7 +6,7 @@ use crate::ecash::error::EcashError;
 use crate::ecash::state::EcashState;
 use crate::node_status_api::models::AxumResult;
 use crate::support::http::state::AppState;
-use axum::extract::Path;
+use axum::extract::{Query, State};
 use axum::{Json, Router};
 use nym_api_requests::ecash::models::{
     AggregatedCoinIndicesSignatureResponse, AggregatedExpirationDateSignatureResponse,
@@ -21,28 +21,19 @@ use tracing::trace;
 use utoipa::IntoParams;
 
 /// routes with globally aggregated keys, signatures, etc.
-pub(crate) fn aggregation_routes(ecash_state: Arc<EcashState>) -> Router<AppState> {
+pub(crate) fn aggregation_routes() -> Router<AppState> {
     Router::new()
         .route(
             "/master-verification-key",
-            axum::routing::get({
-                let ecash_state = Arc::clone(&ecash_state);
-                |epoch_id| master_verification_key(epoch_id, ecash_state)
-            }),
+            axum::routing::get(master_verification_key),
         )
         .route(
             "/aggregated-expiration-date-signatures",
-            axum::routing::get({
-                let ecash_state = Arc::clone(&ecash_state);
-                |expiration_date| expiration_date_signatures(expiration_date, ecash_state)
-            }),
+            axum::routing::get(expiration_date_signatures),
         )
         .route(
             "/aggregated-coin-indices-signatures",
-            axum::routing::get({
-                let ecash_state = Arc::clone(&ecash_state);
-                |epoch_id| coin_indices_signatures(epoch_id, ecash_state)
-            }),
+            axum::routing::get(coin_indices_signatures),
         )
 }
 
@@ -58,8 +49,8 @@ pub(crate) fn aggregation_routes(ecash_state: Arc<EcashState>) -> Router<AppStat
     )
 )]
 async fn master_verification_key(
-    Path(EpochIdParam { epoch_id }): Path<EpochIdParam>,
-    state: Arc<EcashState>,
+    State(state): State<Arc<EcashState>>,
+    Query(EpochIdParam { epoch_id }): Query<EpochIdParam>,
 ) -> AxumResult<Json<VerificationKeyResponse>> {
     trace!("aggregated_verification_key request");
 
@@ -72,7 +63,6 @@ async fn master_verification_key(
 }
 
 #[derive(Deserialize, IntoParams)]
-#[into_params(parameter_in = Path)]
 struct ExpirationDateParam {
     expiration_date: Option<String>,
 }
@@ -89,8 +79,8 @@ struct ExpirationDateParam {
     )
 )]
 async fn expiration_date_signatures(
-    Path(ExpirationDateParam { expiration_date }): Path<ExpirationDateParam>,
-    state: Arc<EcashState>,
+    State(state): State<Arc<EcashState>>,
+    Query(ExpirationDateParam { expiration_date }): Query<ExpirationDateParam>,
 ) -> AxumResult<Json<AggregatedExpirationDateSignatureResponse>> {
     trace!("aggregated_expiration_date_signatures request");
 
@@ -126,8 +116,8 @@ async fn expiration_date_signatures(
     )
 )]
 async fn coin_indices_signatures(
-    Path(EpochIdParam { epoch_id }): Path<EpochIdParam>,
-    state: Arc<EcashState>,
+    Query(EpochIdParam { epoch_id }): Query<EpochIdParam>,
+    State(state): State<Arc<EcashState>>,
 ) -> AxumResult<Json<AggregatedCoinIndicesSignatureResponse>> {
     trace!("aggregated_coin_indices_signatures request");
 

nym-api/src/ecash/api_routes/handlers.rs

@@ -5,15 +5,13 @@ use crate::ecash::api_routes::aggregation::aggregation_routes;
 use crate::ecash::api_routes::issued::issued_routes;
 use crate::ecash::api_routes::partial_signing::partial_signing_routes;
 use crate::ecash::api_routes::spending::spending_routes;
-use crate::ecash::state::EcashState;
 use crate::support::http::state::AppState;
 use axum::Router;
-use std::sync::Arc;
 
-pub(crate) fn ecash_routes(ecash_state: Arc<EcashState>) -> Router<AppState> {
+pub(crate) fn ecash_routes() -> Router<AppState> {
     Router::new()
-        .merge(aggregation_routes(Arc::clone(&ecash_state)))
-        .merge(issued_routes(Arc::clone(&ecash_state)))
-        .merge(partial_signing_routes(Arc::clone(&ecash_state)))
-        .merge(spending_routes(Arc::clone(&ecash_state)))
+        .merge(aggregation_routes())
+        .merge(issued_routes())
+        .merge(partial_signing_routes())
+        .merge(spending_routes())
 }

nym-api/src/ecash/api_routes/helpers.rs

@@ -2,7 +2,6 @@
 // SPDX-License-Identifier: GPL-3.0-only
 
 #[derive(serde::Deserialize, utoipa::IntoParams)]
-#[into_params(parameter_in = Path)]
 pub(super) struct EpochIdParam {
     pub(super) epoch_id: Option<u64>,
 }

nym-api/src/ecash/api_routes/issued.rs

@@ -4,7 +4,7 @@
 use crate::ecash::state::EcashState;
 use crate::node_status_api::models::AxumResult;
 use crate::support::http::state::AppState;
-use axum::extract::Path;
+use axum::extract::{Path, State};
 use axum::{Json, Router};
 use nym_api_requests::ecash::models::{
     IssuedTicketbooksChallengeRequest, IssuedTicketbooksChallengeResponse,
@@ -17,21 +17,15 @@ use time::Date;
 use tracing::trace;
 use utoipa::{IntoParams, ToSchema};
 
-pub(crate) fn issued_routes(ecash_state: Arc<EcashState>) -> Router<AppState> {
+pub(crate) fn issued_routes() -> Router<AppState> {
     Router::new()
         .route(
             "/issued-ticketbooks-for/:expiration_date",
-            axum::routing::get({
-                let ecash_state = Arc::clone(&ecash_state);
-                |expiration_date| issued_ticketbooks_for(expiration_date, ecash_state)
-            }),
+            axum::routing::get(issued_ticketbooks_for),
         )
         .route(
             "/issued-ticketbooks-challenge",
-            axum::routing::post({
-                let ecash_state = Arc::clone(&ecash_state);
-                |body| issued_ticketbooks_challenge(body, ecash_state)
-            }),
+            axum::routing::post(issued_ticketbooks_challenge),
         )
 }
 
@@ -58,8 +52,8 @@ pub(crate) struct ExpirationDatePathParam {
     )
 )]
 async fn issued_ticketbooks_for(
+    State(state): State<Arc<EcashState>>,
     Path(ExpirationDatePathParam { expiration_date }): Path<ExpirationDatePathParam>,
-    state: Arc<EcashState>,
 ) -> AxumResult<Json<IssuedTicketbooksForResponse>> {
     state.ensure_signer().await?;
 
@@ -83,8 +77,8 @@ async fn issued_ticketbooks_for(
     )
 )]
 async fn issued_ticketbooks_challenge(
+    State(state): State<Arc<EcashState>>,
     Json(challenge): Json<IssuedTicketbooksChallengeRequest>,
-    state: Arc<EcashState>,
 ) -> AxumResult<Json<IssuedTicketbooksChallengeResponse>> {
     trace!("replying to ticketbooks challenge: {:?}", challenge);
     state.ensure_signer().await?;

nym-api/src/ecash/api_routes/partial_signing.rs

@@ -7,7 +7,7 @@ use crate::ecash::helpers::blind_sign;
 use crate::ecash::state::EcashState;
 use crate::node_status_api::models::AxumResult;
 use crate::support::http::state::AppState;
-use axum::extract::Query;
+use axum::extract::{Query, State};
 use axum::{Json, Router};
 use nym_api_requests::ecash::{
     BlindSignRequestBody, BlindedSignatureResponse, PartialCoinIndicesSignatureResponse,
@@ -22,28 +22,16 @@ use time::Date;
 use tracing::{debug, trace};
 use utoipa::IntoParams;
 
-pub(crate) fn partial_signing_routes(ecash_state: Arc<EcashState>) -> Router<AppState> {
+pub(crate) fn partial_signing_routes() -> Router<AppState> {
     Router::new()
-        .route(
-            "/blind-sign",
-            axum::routing::post({
-                let ecash_state = Arc::clone(&ecash_state);
-                |body| post_blind_sign(body, ecash_state)
-            }),
-        )
+        .route("/blind-sign", axum::routing::post(post_blind_sign))
         .route(
             "/partial-expiration-date-signatures",
-            axum::routing::get({
-                let ecash_state = Arc::clone(&ecash_state);
-                |expiration_date| partial_expiration_date_signatures(expiration_date, ecash_state)
-            }),
+            axum::routing::get(partial_expiration_date_signatures),
         )
         .route(
             "/partial-coin-indices-signatures",
-            axum::routing::get({
-                let ecash_state = Arc::clone(&ecash_state);
-                |epoch_id| partial_coin_indices_signatures(epoch_id, ecash_state)
-            }),
+            axum::routing::get(partial_coin_indices_signatures),
         )
 }
 
@@ -59,8 +47,8 @@ pub(crate) fn partial_signing_routes(ecash_state: Arc<EcashState>) -> Router<App
     )
 )]
 async fn post_blind_sign(
+    State(state): State<Arc<EcashState>>,
     Json(blind_sign_request_body): Json<BlindSignRequestBody>,
-    state: Arc<EcashState>,
 ) -> AxumResult<Json<BlindedSignatureResponse>> {
     state.ensure_signer().await?;
 
@@ -134,8 +122,8 @@ struct ExpirationDateParam {
     )
 )]
 async fn partial_expiration_date_signatures(
+    State(state): State<Arc<EcashState>>,
     Query(ExpirationDateParam { expiration_date }): Query<ExpirationDateParam>,
-    state: Arc<EcashState>,
 ) -> AxumResult<Json<PartialExpirationDateSignatureResponse>> {
     state.ensure_signer().await?;
 
@@ -172,8 +160,8 @@ async fn partial_expiration_date_signatures(
     )
 )]
 async fn partial_coin_indices_signatures(
+    State(state): State<Arc<EcashState>>,
     Query(EpochIdParam { epoch_id }): Query<EpochIdParam>,
-    state: Arc<EcashState>,
 ) -> AxumResult<Json<PartialCoinIndicesSignatureResponse>> {
     state.ensure_signer().await?;
 

nym-api/src/ecash/api_routes/spending.rs

@@ -5,6 +5,7 @@ use crate::ecash::error::EcashError;
 use crate::ecash::state::EcashState;
 use crate::node_status_api::models::{AxumErrorResponse, AxumResult};
 use crate::support::http::state::AppState;
+use axum::extract::State;
 use axum::{Json, Router};
 use nym_api_requests::constants::MIN_BATCH_REDEMPTION_DELAY;
 use nym_api_requests::ecash::models::{
@@ -21,28 +22,16 @@ use time::{OffsetDateTime, Time};
 use tracing::{error, warn};
 
 #[allow(deprecated)]
-pub(crate) fn spending_routes(ecash_state: Arc<EcashState>) -> Router<AppState> {
+pub(crate) fn spending_routes() -> Router<AppState> {
     Router::new()
-        .route(
-            "/verify-ecash-ticket",
-            axum::routing::post({
-                let ecash_state = Arc::clone(&ecash_state);
-                |body| verify_ticket(body, ecash_state)
-            }),
-        )
+        .route("/verify-ecash-ticket", axum::routing::post(verify_ticket))
         .route(
             "/batch-redeem-ecash-tickets",
-            axum::routing::post({
-                let ecash_state = Arc::clone(&ecash_state);
-                |body| batch_redeem_tickets(body, ecash_state)
-            }),
+            axum::routing::post(batch_redeem_tickets),
         )
         .route(
             "/double-spending-filter-v1",
-            axum::routing::get({
-                let ecash_state = Arc::clone(&ecash_state);
-                || double_spending_filter_v1(ecash_state)
-            }),
+            axum::routing::get(double_spending_filter_v1),
         )
 }
 
@@ -67,9 +56,9 @@ fn reject_ticket(
     )
 )]
 async fn verify_ticket(
+    State(state): State<Arc<EcashState>>,
     // TODO in the future: make it send binary data rather than json
     Json(verify_ticket_body): Json<VerifyEcashTicketBody>,
-    state: Arc<EcashState>,
 ) -> AxumResult<Json<EcashTicketVerificationResponse>> {
     state.ensure_signer().await?;
 
@@ -170,9 +159,9 @@ async fn verify_ticket(
     )
 )]
 async fn batch_redeem_tickets(
+    State(state): State<Arc<EcashState>>,
     // TODO in the future: make it send binary data rather than json
     Json(batch_redeem_credentials_body): Json<BatchRedeemTicketsBody>,
-    state: Arc<EcashState>,
 ) -> AxumResult<Json<EcashBatchTicketRedemptionResponse>> {
     state.ensure_signer().await?;
 
@@ -244,8 +233,6 @@ async fn batch_redeem_tickets(
     )
 )]
 #[deprecated]
-async fn double_spending_filter_v1(
-    _state: Arc<EcashState>,
-) -> AxumResult<Json<SpentCredentialsResponse>> {
+async fn double_spending_filter_v1() -> AxumResult<Json<SpentCredentialsResponse>> {
     AxumResult::Err(AxumErrorResponse::internal_msg("permanently restricted"))
 }

nym-api/src/ecash/client.rs

@@ -27,7 +27,7 @@ use nym_validator_client::EcashApiClient;
 
 #[async_trait]
 pub trait Client {
-    async fn address(&self) -> AccountId;
+    async fn address(&self) -> Result<AccountId>;
 
     async fn dkg_contract_address(&self) -> Result<AccountId>;
 

nym-api/src/ecash/dkg/client.rs

@@ -35,7 +35,7 @@ impl DkgClient {
         }
     }
 
-    pub(crate) async fn get_address(&self) -> AccountId {
+    pub(crate) async fn get_address(&self) -> Result<AccountId, EcashError> {
         self.inner.address().await
     }
 
@@ -53,7 +53,7 @@ impl DkgClient {
 
     pub(crate) async fn group_member(&self) -> Result<MemberResponse, EcashError> {
         self.inner
-            .group_member(self.get_address().await.to_string())
+            .group_member(self.get_address().await?.to_string())
             .await
     }
 
@@ -126,7 +126,7 @@ impl DkgClient {
         &self,
         epoch_id: EpochId,
     ) -> Result<Option<ContractVKShare>, EcashError> {
-        let address = self.inner.address().await;
+        let address = self.inner.address().await?;
         self.get_verification_key_share(epoch_id, address).await
     }
 
@@ -138,7 +138,7 @@ impl DkgClient {
     }
 
     pub(crate) async fn get_vote(&self, proposal_id: u64) -> Result<VoteResponse, EcashError> {
-        let address = self.get_address().await.to_string();
+        let address = self.get_address().await?.to_string();
         self.inner.get_vote(proposal_id, address).await
     }
 

nym-api/src/ecash/dkg/dealing.rs

@@ -155,7 +155,7 @@ impl<R: RngCore + CryptoRng> DkgController<R> {
         resharing: bool,
     ) -> Result<(), DealingGenerationError> {
         let dealing_state = self.state.dealing_exchange_state(epoch_id)?;
-        let address = self.dkg_client.get_address().await.to_string();
+        let address = self.dkg_client.get_address().await?.to_string();
 
         let status = self
             .dkg_client
@@ -259,7 +259,7 @@ impl<R: RngCore + CryptoRng> DkgController<R> {
             .checked_sub(1)
             .expect("resharing epoch invariant has been broken");
 
-        let address = self.dkg_client.get_address().await;
+        let address = self.dkg_client.get_address().await?;
         Ok(self
             .dkg_client
             .dealer_in_epoch(previous_epoch_id, address.to_string())

nym-api/src/ecash/dkg/key_derivation.rs

@@ -494,7 +494,7 @@ impl<R: RngCore + CryptoRng> DkgController<R> {
         // submitted proposals and find the one with our address
         self.get_validation_proposals()
             .await?
-            .get(self.dkg_client.get_address().await.as_ref())
+            .get(self.dkg_client.get_address().await?.as_ref())
             .copied()
             .ok_or(KeyDerivationError::UnrecoverableProposalId)
     }

nym-api/src/ecash/dkg/key_validation.rs

@@ -155,7 +155,7 @@ impl<R: RngCore + CryptoRng> DkgController<R> {
             };
 
             // if this is our share, obviously vote for yes without spending time on verification
-            if owner.as_ref() == self.dkg_client.get_address().await.as_ref() {
+            if owner.as_ref() == self.dkg_client.get_address().await?.as_ref() {
                 votes.insert(*proposal_id, true);
                 continue;
             }
@@ -313,7 +313,7 @@ mod tests {
         exchange_dealings(&mut controllers, false).await;
         derive_keypairs(&mut controllers, false).await;
 
-        let first_dealer = controllers[0].dkg_client.get_address().await;
+        let first_dealer = controllers[0].dkg_client.get_address().await?;
 
         {
             let mut guard = chain.lock().unwrap();
@@ -365,8 +365,8 @@ mod tests {
         exchange_dealings(&mut controllers, false).await;
         derive_keypairs(&mut controllers, false).await;
 
-        let first_dealer = controllers[0].dkg_client.get_address().await;
-        let second_dealer = controllers[1].dkg_client.get_address().await;
+        let first_dealer = controllers[0].dkg_client.get_address().await?;
+        let second_dealer = controllers[1].dkg_client.get_address().await?;
 
         {
             let mut guard = chain.lock().unwrap();

nym-api/src/ecash/error.rs

@@ -25,6 +25,9 @@ pub enum EcashError {
     #[error(transparent)]
     IOError(#[from] std::io::Error),
 
+    #[error("this instance is running without on-chain signing capabilities so no transactions can be sent")]
+    ChainSignerNotEnabled,
+
     #[error("this operation couldn't be completed as this nym-api is not an active ecash signer")]
     NotASigner,
 

nym-api/src/ecash/state/mod.rs

@@ -139,7 +139,9 @@ impl EcashState {
             .local
             .active_signer
             .get_or_init(epoch_id, || async {
-                let address = self.aux.client.address().await;
+                let Ok(address) = self.aux.client.address().await else {
+                    return Ok(false);
+                };
                 let ecash_signers = self.aux.comm_channel.ecash_clients(epoch_id).await?;
 
                 // check if any ecash signers for this epoch has the same cosmos address as this api
@@ -246,7 +248,7 @@ impl EcashState {
                 let threshold = self.aux.comm_channel.ecash_threshold(epoch_id).await?;
 
                 // let mut shares = Mutex::new(Vec::with_capacity(all_apis.len()));
-                let cosmos_address = self.aux.client.address().await;
+                let cosmos_address = self.aux.client.address().await.ok();
 
                 let get_partial_signatures = |api: EcashApiClient| async {
                     // move the api into the closure
@@ -256,7 +258,7 @@ impl EcashState {
 
                     // check if we're attempting to query ourselves, in that case just get local signature
                     // rather than making the http query
-                    let partial = if api.cosmos_address == cosmos_address {
+                    let partial = if Some(api.cosmos_address) == cosmos_address {
                         self.partial_coin_index_signatures(Some(epoch_id))
                             .await?
                             .signatures
@@ -380,7 +382,7 @@ impl EcashState {
                 let all_apis = self.aux.comm_channel.ecash_clients(epoch_id).await?;
                 let threshold = self.aux.comm_channel.ecash_threshold(epoch_id).await?;
 
-                let cosmos_address = self.aux.client.address().await;
+                let cosmos_address = self.aux.client.address().await.ok();
 
                 let get_partial_signatures = |api: EcashApiClient| async {
                     // move the api into the closure
@@ -390,7 +392,7 @@ impl EcashState {
 
                     // check if we're attempting to query ourselves, in that case just get local signature
                     // rather than making the http query
-                    let partial = if api.cosmos_address == cosmos_address {
+                    let partial = if Some(api.cosmos_address) == cosmos_address {
                         self.partial_expiration_date_signatures(expiration_date)
                             .await?
                             .signatures

nym-api/src/ecash/tests/fixtures.rs

@@ -263,7 +263,7 @@ pub(crate) struct TestingDkgController {
 
 impl TestingDkgController {
     pub async fn address(&self) -> AccountId {
-        self.dkg_client.get_address().await
+        self.dkg_client.get_address().await.unwrap()
     }
 
     pub async fn cw_address(&self) -> Addr {

nym-api/src/ecash/tests/helpers.rs

@@ -51,7 +51,7 @@ pub(crate) async fn initialise_dkg(controllers: &mut [TestingDkgController], res
 
     // add every dealer to group contract
     for controller in controllers.iter() {
-        let address = controller.dkg_client.get_address().await;
+        let address = controller.dkg_client.get_address().await.unwrap();
         let mut chain = controllers[0].chain_state.lock().unwrap();
         chain.add_member(address.as_ref(), 10);
     }
@@ -76,7 +76,7 @@ pub(crate) async fn submit_public_keys(controllers: &mut [TestingDkgController],
             .unwrap();
     }
 
-    let threshold = (2 * controllers.len() as u64 + 3 - 1) / 3;
+    let threshold = (2 * controllers.len() as u64).div_ceil(3);
 
     let mut guard = controllers[0].chain_state.lock().unwrap();
     guard.dkg_contract.epoch.state = EpochState::DealingExchange { resharing };

nym-api/src/ecash/tests/mod.rs

@@ -11,6 +11,7 @@ use crate::node_describe_cache::DescribedNodes;
 use crate::node_status_api::handlers::unstable;
 use crate::node_status_api::NodeStatusCache;
 use crate::nym_contract_cache::cache::NymContractCache;
+use crate::status::ApiStatusState;
 use crate::support::caching::cache::SharedCache;
 use crate::support::config;
 use crate::support::http::state::{AppState, ForcedRefresh};
@@ -524,8 +525,8 @@ impl DummyClient {
 
 #[async_trait]
 impl super::client::Client for DummyClient {
-    async fn address(&self) -> AccountId {
-        self.validator_address.clone()
+    async fn address(&self) -> Result<AccountId> {
+        Ok(self.validator_address.clone())
     }
 
     async fn dkg_contract_address(&self) -> Result<AccountId> {
@@ -1262,7 +1263,7 @@ struct TestFixture {
 }
 
 impl TestFixture {
-    fn build_app_state(storage: NymApiStorage) -> AppState {
+    fn build_app_state(storage: NymApiStorage, ecash_state: EcashState) -> AppState {
         AppState {
             forced_refresh: ForcedRefresh::new(true),
             nym_contract_cache: NymContractCache::new(),
@@ -1275,6 +1276,8 @@ impl TestFixture {
                 NymNetworkDetails::new_empty(),
             ),
             node_info_cache: unstable::NodeInfoCache::default(),
+            api_status: ApiStatusState::new(None),
+            ecash_state: Arc::new(ecash_state),
         }
     }
 
@@ -1337,8 +1340,8 @@ impl TestFixture {
         TestFixture {
             axum: TestServer::new(
                 Router::new()
-                    .nest("/v1/ecash", ecash_routes(Arc::new(ecash_state)))
-                    .with_state(Self::build_app_state(storage.clone())),
+                    .nest("/v1/ecash", ecash_routes())
+                    .with_state(Self::build_app_state(storage.clone(), ecash_state)),
             )
             .unwrap(),
             storage,

nym-api/src/epoch_operations/error.rs

@@ -11,6 +11,9 @@ use thiserror::Error;
 
 #[derive(Debug, Error)]
 pub enum RewardingError {
+    #[error("this instance is running without on-chain signing capabilities so no transactions can be sent")]
+    ChainSignerNotEnabled,
+
     #[error("Our account ({our_address}) is not permitted to update rewarded set and perform rewarding. The allowed address is {allowed_address}")]
     Unauthorised {
         our_address: AccountId,

nym-api/src/epoch_operations/mod.rs

@@ -134,9 +134,12 @@ impl EpochAdvancer {
 
         let epoch_status = self.nyxd_client.get_current_epoch_status().await?;
         if !epoch_status.is_in_progress() {
-            if epoch_status.being_advanced_by.as_str()
-                != self.nyxd_client.client_address().await.as_ref()
-            {
+            // SAFETY: before `EpochAdvancer` is started, `ensure_rewarding_permission` is called
+            // which is not allowed to progress if this instance is not using a signing client
+            #[allow(clippy::unwrap_used)]
+            let address = self.nyxd_client.client_address().await.unwrap();
+
+            if epoch_status.being_advanced_by.as_str() != address.as_ref() {
                 // another nym-api is already handling
                 error!("another nym-api ({}) is already advancing the epoch... but we shouldn't have other nym-apis yet!", epoch_status.being_advanced_by);
                 return Ok(());
@@ -318,8 +321,10 @@ impl EpochAdvancer {
 pub(crate) async fn ensure_rewarding_permission(
     nyxd_client: &Client,
 ) -> Result<(), RewardingError> {
+    let Some(our_address) = nyxd_client.client_address().await else {
+        return Err(RewardingError::ChainSignerNotEnabled);
+    };
     let allowed_address = nyxd_client.get_rewarding_validator_address().await?;
-    let our_address = nyxd_client.client_address().await;
     if allowed_address != our_address {
         Err(RewardingError::Unauthorised {
             our_address,

nym-api/src/network_monitor/monitor/gateway_clients_cache.rs

@@ -59,7 +59,7 @@ impl GatewayClientHandle {
     }
 }
 
-impl<'a> UnlockedGatewayClientHandle<'a> {
+impl UnlockedGatewayClientHandle<'_> {
     pub(crate) fn get_mut_unchecked(
         &mut self,
     ) -> &mut GatewayClient<nyxd::Client, PersistentStorage> {

nym-api/src/status/handlers.rs

@@ -4,37 +4,20 @@
 use crate::node_status_api::models::{AxumErrorResponse, AxumResult};
 use crate::status::ApiStatusState;
 use crate::support::http::state::AppState;
+use axum::extract::State;
 use axum::Json;
 use axum::Router;
 use nym_api_requests::models::{ApiHealthResponse, SignerInformationResponse};
 use nym_bin_common::build_information::BinaryBuildInformationOwned;
 use nym_compact_ecash::Base58;
-use std::sync::Arc;
 
 pub(crate) fn api_status_routes() -> Router<AppState> {
-    let api_status_state = Arc::new(ApiStatusState::new());
-
     Router::new()
-        .route(
-            "/health",
-            axum::routing::get({
-                let state = Arc::clone(&api_status_state);
-                || health(state)
-            }),
-        )
-        .route(
-            "/build-information",
-            axum::routing::get({
-                let state = Arc::clone(&api_status_state);
-                || build_information(state)
-            }),
-        )
+        .route("/health", axum::routing::get(health))
+        .route("/build-information", axum::routing::get(build_information))
         .route(
             "/signer-information",
-            axum::routing::get({
-                let state = Arc::clone(&api_status_state);
-                || signer_information(state)
-            }),
+            axum::routing::get(signer_information),
         )
 }
 
@@ -46,7 +29,7 @@ pub(crate) fn api_status_routes() -> Router<AppState> {
         (status = 200, body = ApiHealthResponse)
     )
 )]
-async fn health(state: Arc<ApiStatusState>) -> Json<ApiHealthResponse> {
+async fn health(State(state): State<ApiStatusState>) -> Json<ApiHealthResponse> {
     let uptime = state.startup_time.elapsed();
     let health = ApiHealthResponse::new_healthy(uptime);
     Json(health)
@@ -60,7 +43,9 @@ async fn health(state: Arc<ApiStatusState>) -> Json<ApiHealthResponse> {
         (status = 200, body = BinaryBuildInformationOwned)
     )
 )]
-async fn build_information(state: Arc<ApiStatusState>) -> Json<BinaryBuildInformationOwned> {
+async fn build_information(
+    State(state): State<ApiStatusState>,
+) -> Json<BinaryBuildInformationOwned> {
     Json(state.build_information.to_owned())
 }
 
@@ -73,7 +58,7 @@ async fn build_information(state: Arc<ApiStatusState>) -> Json<BinaryBuildInform
     )
 )]
 async fn signer_information(
-    state: Arc<ApiStatusState>,
+    State(state): State<ApiStatusState>,
 ) -> AxumResult<Json<SignerInformationResponse>> {
     let signer_state = state.signer_information.as_ref().ok_or_else(|| {
         AxumErrorResponse::internal_msg("this api does not expose zk-nym signing functionalities")

nym-api/src/status/mod.rs

@@ -4,12 +4,25 @@
 use crate::ecash;
 use nym_bin_common::bin_info;
 use nym_bin_common::build_information::BinaryBuildInformation;
-
+use std::ops::Deref;
+use std::sync::Arc;
 use tokio::time::Instant;
 
 pub(crate) mod handlers;
 
+#[derive(Clone)]
 pub(crate) struct ApiStatusState {
+    inner: Arc<ApiStatusStateInner>,
+}
+
+impl Deref for ApiStatusState {
+    type Target = ApiStatusStateInner;
+    fn deref(&self) -> &Self::Target {
+        &self.inner
+    }
+}
+
+pub(crate) struct ApiStatusStateInner {
     startup_time: Instant,
     build_information: BinaryBuildInformation,
     signer_information: Option<SignerState>,
@@ -27,15 +40,13 @@ pub(crate) struct SignerState {
 }
 
 impl ApiStatusState {
-    pub fn new() -> Self {
+    pub fn new(signer_information: Option<SignerState>) -> Self {
         ApiStatusState {
-            startup_time: Instant::now(),
-            build_information: bin_info!(),
-            signer_information: None,
+            inner: Arc::new(ApiStatusStateInner {
+                startup_time: Instant::now(),
+                build_information: bin_info!(),
+                signer_information,
+            }),
         }
     }
-
-    pub fn add_zk_nym_signer(&mut self, signer_information: SignerState) {
-        self.signer_information = Some(signer_information)
-    }
 }

nym-api/src/support/cli/run.rs

@@ -2,7 +2,6 @@
 // SPDX-License-Identifier: GPL-3.0-only
 
 use crate::circulating_supply_api::cache::CirculatingSupplyCache;
-use crate::ecash::api_routes::handlers::ecash_routes;
 use crate::ecash::client::Client;
 use crate::ecash::comm::QueryCommunicationChannel;
 use crate::ecash::dkg::controller::keys::{
@@ -138,8 +137,6 @@ async fn start_nym_api_tasks_axum(config: &Config) -> anyhow::Result<ShutdownHan
     let described_nodes_cache = SharedCache::<DescribedNodes>::new();
     let node_info_cache = unstable::NodeInfoCache::default();
 
-    let mut status_state = ApiStatusState::new();
-
     let ecash_contract = nyxd_client
         .get_ecash_contract_address()
         .await
@@ -161,8 +158,8 @@ async fn start_nym_api_tasks_axum(config: &Config) -> anyhow::Result<ShutdownHan
 
     // if ecash signer is enabled, there are additional constraints on the nym-api,
     // such as having sufficient token balance
-    let router = if config.ecash_signer.enabled {
-        let cosmos_address = nyxd_client.address().await;
+    let signer_information = if config.ecash_signer.enabled {
+        let cosmos_address = nyxd_client.address().await?;
 
         // make sure we have some tokens to cover multisig fees
         let balance = nyxd_client.balance(&mix_denom).await?;
@@ -177,16 +174,14 @@ async fn start_nym_api_tasks_axum(config: &Config) -> anyhow::Result<ShutdownHan
             .clone()
             .map(|u| u.to_string())
             .unwrap_or_default();
-        status_state.add_zk_nym_signer(SignerState {
+        Some(SignerState {
             cosmos_address: cosmos_address.to_string(),
             identity: encoded_identity,
             announce_address,
             ecash_keypair: ecash_keypair_wrapper.clone(),
-        });
-
-        router.nest("/v1/ecash", ecash_routes(Arc::new(ecash_state)))
+        })
     } else {
-        router
+        None
     };
 
     let router = router.with_state(AppState {
@@ -200,6 +195,8 @@ async fn start_nym_api_tasks_axum(config: &Config) -> anyhow::Result<ShutdownHan
         described_nodes_cache: described_nodes_cache.clone(),
         network_details,
         node_info_cache,
+        api_status: ApiStatusState::new(signer_information),
+        ecash_state: Arc::new(ecash_state),
     });
 
     let task_manager = TaskManager::new(TASK_MANAGER_TIMEOUT_S);

nym-api/src/support/http/router.rs

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: GPL-3.0-only
 
 use crate::circulating_supply_api::handlers::circulating_supply_routes;
+use crate::ecash::api_routes::handlers::ecash_routes;
 use crate::network::handlers::nym_network_routes;
 use crate::node_status_api::handlers::node_status_routes;
 use crate::nym_contract_cache::handlers::nym_contract_cache_routes;
@@ -62,6 +63,7 @@ impl RouterBuilder {
                     .nest("/network", nym_network_routes())
                     .nest("/api-status", status::handlers::api_status_routes())
                     .nest("/nym-nodes", nym_node_routes())
+                    .nest("/ecash", ecash_routes())
                     .nest("/unstable", unstable_routes()), // CORS layer needs to be "outside" of routes
             );
 
@@ -70,6 +72,7 @@ impl RouterBuilder {
         }
     }
 
+    #[allow(dead_code)]
     pub(crate) fn nest(self, path: &str, router: Router<AppState>) -> Self {
         Self {
             unfinished_router: self.unfinished_router.nest(path, router),

nym-api/src/support/http/state.rs

@@ -2,15 +2,18 @@
 // SPDX-License-Identifier: GPL-3.0-only
 
 use crate::circulating_supply_api::cache::CirculatingSupplyCache;
+use crate::ecash::state::EcashState;
 use crate::network::models::NetworkDetails;
 use crate::node_describe_cache::DescribedNodes;
 use crate::node_status_api::handlers::unstable;
 use crate::node_status_api::models::AxumErrorResponse;
 use crate::node_status_api::NodeStatusCache;
 use crate::nym_contract_cache::cache::{CachedRewardedSet, NymContractCache};
+use crate::status::ApiStatusState;
 use crate::support::caching::cache::SharedCache;
 use crate::support::caching::Cache;
 use crate::support::storage;
+use axum::extract::FromRef;
 use nym_api_requests::models::{GatewayBondAnnotated, MixNodeBondAnnotated, NodeAnnotation};
 use nym_mixnet_contract_common::NodeId;
 use nym_task::TaskManager;
@@ -80,6 +83,21 @@ pub(crate) struct AppState {
     pub(crate) described_nodes_cache: SharedCache<DescribedNodes>,
     pub(crate) network_details: NetworkDetails,
     pub(crate) node_info_cache: unstable::NodeInfoCache,
+    pub(crate) api_status: ApiStatusState,
+    // todo: refactor it into inner: Arc<EcashStateInner>
+    pub(crate) ecash_state: Arc<EcashState>,
+}
+
+impl FromRef<AppState> for ApiStatusState {
+    fn from_ref(app_state: &AppState) -> Self {
+        app_state.api_status.clone()
+    }
+}
+
+impl FromRef<AppState> for Arc<EcashState> {
+    fn from_ref(app_state: &AppState) -> Self {
+        app_state.ecash_state.clone()
+    }
 }
 
 #[derive(Clone)]

nym-api/src/support/nyxd/mod.rs

@@ -77,16 +77,6 @@ macro_rules! nyxd_query {
     }};
 }
 
-macro_rules! nyxd_signing_shared {
-    ($self:expr, $($op:tt)*) => {{
-        let guard = $self.inner.read().await;
-        match &*guard {
-            $crate::support::nyxd::ClientInner::Signing(client) => client.$($op)*,
-            $crate::support::nyxd::ClientInner::Query(_) => panic!("attempted to use a signing method on a query client"),
-        }
-    }};
-}
-
 macro_rules! nyxd_signing {
     ($self:expr, $($op:tt)*) => {{
         let guard = $self.inner.write().await;
@@ -140,13 +130,19 @@ impl Client {
         self.inner.read().await
     }
 
-    pub(crate) async fn client_address(&self) -> AccountId {
-        nyxd_signing_shared!(self, address())
+    pub(crate) async fn client_address(&self) -> Option<AccountId> {
+        let guard = self.inner.read().await;
+        match &*guard {
+            ClientInner::Signing(client) => Some(client.address()),
+            ClientInner::Query(_) => None,
+        }
     }
 
     pub(crate) async fn balance<S: Into<String>>(&self, denom: S) -> Result<Coin, NyxdError> {
-        let address = self.client_address().await;
         let denom = denom.into();
+        let Some(address) = self.client_address().await else {
+            return Ok(Coin::new(0, denom));
+        };
         let balance = nyxd_query!(self, get_balance(&address, denom.clone()).await?);
 
         match balance {
@@ -394,8 +390,10 @@ impl Client {
 
 #[async_trait]
 impl crate::ecash::client::Client for Client {
-    async fn address(&self) -> AccountId {
-        self.client_address().await
+    async fn address(&self) -> Result<AccountId, EcashError> {
+        self.client_address()
+            .await
+            .ok_or(EcashError::ChainSignerNotEnabled)
     }
 
     async fn dkg_contract_address(&self) -> Result<AccountId, EcashError> {
@@ -481,7 +479,7 @@ impl crate::ecash::client::Client for Client {
     async fn get_self_registered_dealer_details(
         &self,
     ) -> crate::ecash::error::Result<DealerDetailsResponse> {
-        let self_address = &self.address().await;
+        let self_address = &self.address().await?;
         Ok(nyxd_query!(self, get_dealer_details(self_address).await?))
     }
 

nym-credential-proxy/nym-credential-proxy/src/http/state/mod.rs

@@ -674,7 +674,7 @@ pub(crate) struct ChainWritePermit<'a> {
     inner: RwLockWriteGuard<'a, DirectSigningHttpRpcNyxdClient>,
 }
 
-impl<'a> Deref for ChainWritePermit<'a> {
+impl Deref for ChainWritePermit<'_> {
     type Target = DirectSigningHttpRpcNyxdClient;
 
     fn deref(&self) -> &Self::Target {

nym-node-status-api/nym-node-status-api/Cargo.toml

@@ -3,7 +3,7 @@
 
 [package]
 name = "nym-node-status-api"
-version = "1.0.0-rc.2"
+version = "1.0.0-rc.4"
 authors.workspace = true
 repository.workspace = true
 homepage.workspace = true
@@ -26,6 +26,8 @@ nym-node-status-client = { path = "../nym-node-status-client" }
 nym-crypto = { path = "../../common/crypto", features = ["asymmetric", "serde"] }
 nym-explorer-client = { path = "../../explorer-api/explorer-client" }
 nym-network-defaults = { path = "../../common/network-defaults" }
+nym-serde-helpers = { path = "../../common/serde-helpers"}
+nym-statistics-common = { path = "../../common/statistics"}
 nym-validator-client = { path = "../../common/client-libs/validator-client" }
 nym-task = { path = "../../common/task" }
 nym-node-requests = { path = "../../nym-node/nym-node-requests", features = ["openapi"] }
@@ -36,8 +38,9 @@ serde_json = { workspace = true }
 serde_json_path = { workspace = true }
 strum = { workspace = true }
 strum_macros = { workspace = true }
-sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite"] }
+sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite", "time"] }
 thiserror = { workspace = true }
+time = { workspace = true }
 tokio = { workspace = true, features = ["rt-multi-thread"] }
 tokio-util = { workspace = true }
 tracing = { workspace = true }

nym-node-status-api/nym-node-status-api/launch_node_status_api.sh

@@ -10,7 +10,7 @@ export NODE_STATUS_API_TESTRUN_REFRESH_INTERVAL=120
 # For TESTING only. NOT used in any other environment
 export NODE_STATUS_API_AGENT_KEY_LIST="H4z8kx5Kkf5JMQHhxaW1MwYndjKCDHC7HsVhHTFfBZ4J"
 
-export ENVIRONMENT=${ENVIRONMENT:-"sandbox"}
+export ENVIRONMENT=${ENVIRONMENT:-"mainnet"}
 
 script_dir=$(dirname $(realpath "$0"))
 monorepo_root=$(realpath "${script_dir}/../..")

nym-node-status-api/nym-node-status-api/migrations/002_session_stats.sql

@@ -0,0 +1,17 @@
+
+CREATE TABLE gateway_session_stats (
+    id                    INTEGER PRIMARY KEY AUTOINCREMENT,
+    gateway_identity_key  VARCHAR NOT NULL,
+    node_id               INTEGER NOT NULL,
+    day                   DATE    NOT NULL,
+    unique_active_clients INTEGER NOT NULL,
+    session_started       INTEGER NOT NULL,
+    users_hashes          VARCHAR,
+    vpn_sessions          VARCHAR,
+    mixnet_sessions       VARCHAR,
+    unknown_sessions      VARCHAR,
+    UNIQUE (node_id, day) -- This constraint automatically creates an index
+  );
+CREATE INDEX idx_gateway_session_stats_identity_key ON gateway_session_stats (gateway_identity_key);
+CREATE INDEX idx_gateway_session_stats_day ON gateway_session_stats (day);
+

nym-node-status-api/nym-node-status-api/src/cli/mod.rs

@@ -45,11 +45,6 @@ pub(crate) struct Cli {
     #[arg(value_parser = parse_duration)]
     pub(crate) nym_api_client_timeout: Duration,
 
-    /// Explorer api client timeout.
-    #[clap(long, default_value = "15", env = "EXPLORER_CLIENT_TIMEOUT")]
-    #[arg(value_parser = parse_duration)]
-    pub(crate) explorer_client_timeout: Duration,
-
     /// Connection url for the database.
     #[clap(long, env = "DATABASE_URL")]
     pub(crate) database_url: String,
@@ -70,6 +65,10 @@ pub(crate) struct Cli {
     #[arg(value_parser = parse_duration)]
     pub(crate) testruns_refresh_interval: Duration,
 
+    #[clap(long, default_value = "86400", env = "NODE_STATUS_API_GEODATA_TTL")]
+    #[arg(value_parser = parse_duration)]
+    pub(crate) geodata_ttl: Duration,
+
     #[clap(env = "NODE_STATUS_API_AGENT_KEY_LIST")]
     #[arg(value_delimiter = ',')]
     pub(crate) agent_key_list: Vec<String>,

nym-node-status-api/nym-node-status-api/src/db/models.rs

@@ -4,7 +4,9 @@ use crate::{
 };
 use nym_node_requests::api::v1::node::models::NodeDescription;
 use serde::{Deserialize, Serialize};
+use sqlx::FromRow;
 use strum_macros::{EnumString, FromRepr};
+use time::Date;
 use utoipa::ToSchema;
 
 pub(crate) struct GatewayRecord {
@@ -12,6 +14,7 @@ pub(crate) struct GatewayRecord {
     pub(crate) bonded: bool,
     pub(crate) blacklisted: bool,
     pub(crate) self_described: String,
+    // TODO dz shouldn't be an option
     pub(crate) explorer_pretty_bond: Option<String>,
     pub(crate) last_updated_utc: i64,
     pub(crate) performance: u8,
@@ -215,7 +218,6 @@ pub(crate) const MIXNODES_BONDED_RESERVE: &str = "mixnodes.bonded.reserve";
 pub(crate) const MIXNODES_BLACKLISTED_COUNT: &str = "mixnodes.blacklisted.count";
 
 pub(crate) const GATEWAYS_BONDED_COUNT: &str = "gateways.bonded.count";
-pub(crate) const GATEWAYS_EXPLORER_COUNT: &str = "gateways.explorer.count";
 pub(crate) const GATEWAYS_BLACKLISTED_COUNT: &str = "gateways.blacklisted.count";
 
 pub(crate) const MIXNODES_HISTORICAL_COUNT: &str = "mixnodes.historical.count";
@@ -272,7 +274,6 @@ pub(crate) mod gateway {
         pub(crate) bonded: GatewaySummaryBonded,
         pub(crate) blacklisted: GatewaySummaryBlacklisted,
         pub(crate) historical: GatewaySummaryHistorical,
-        pub(crate) explorer: GatewaySummaryExplorer,
     }
 
     #[derive(Debug, Clone, Deserialize, Serialize, ToSchema)]
@@ -334,3 +335,44 @@ pub struct GatewayInfoDto {
     pub self_described: Option<String>,
     pub explorer_pretty_bond: Option<String>,
 }
+
+#[derive(Debug, Clone, FromRow)]
+pub struct GatewaySessionsRecord {
+    pub gateway_identity_key: String,
+    pub node_id: i64,
+    pub day: Date,
+    pub unique_active_clients: i64,
+    pub session_started: i64,
+    pub users_hashes: Option<String>,
+    pub vpn_sessions: Option<String>,
+    pub mixnet_sessions: Option<String>,
+    pub unknown_sessions: Option<String>,
+}
+
+impl TryFrom<GatewaySessionsRecord> for http::models::SessionStats {
+    type Error = anyhow::Error;
+
+    fn try_from(value: GatewaySessionsRecord) -> Result<Self, Self::Error> {
+        let users_hashes = value.users_hashes.clone().unwrap_or("null".to_string());
+        let vpn_sessions = value.vpn_sessions.clone().unwrap_or("null".to_string());
+        let mixnet_sessions = value.mixnet_sessions.clone().unwrap_or("null".to_string());
+        let unknown_sessions = value.unknown_sessions.clone().unwrap_or("null".to_string());
+
+        let users_hashes = serde_json::from_str(&users_hashes).unwrap_or(None);
+        let vpn_sessions = serde_json::from_str(&vpn_sessions).unwrap_or(None);
+        let mixnet_sessions = serde_json::from_str(&mixnet_sessions).unwrap_or(None);
+        let unknown_sessions = serde_json::from_str(&unknown_sessions).unwrap_or(None);
+
+        Ok(http::models::SessionStats {
+            gateway_identity_key: value.gateway_identity_key.clone(),
+            node_id: value.node_id as u32,
+            day: value.day,
+            unique_active_clients: value.unique_active_clients,
+            session_started: value.session_started,
+            users_hashes,
+            vpn_sessions,
+            mixnet_sessions,
+            unknown_sessions,
+        })
+    }
+}

nym-node-status-api/nym-node-status-api/src/db/queries/gateways_stats.rs

@@ -0,0 +1,75 @@
+use crate::{
+    db::{models::GatewaySessionsRecord, DbPool},
+    http::models::SessionStats,
+};
+use futures_util::TryStreamExt;
+use time::Date;
+use tracing::error;
+
+pub(crate) async fn insert_session_records(
+    pool: &DbPool,
+    records: Vec<GatewaySessionsRecord>,
+) -> anyhow::Result<()> {
+    let mut tx = pool.begin().await?;
+    for record in records {
+        sqlx::query!(
+            "INSERT OR IGNORE INTO gateway_session_stats
+                (gateway_identity_key, node_id, day,
+                    unique_active_clients, session_started, users_hashes,
+                    vpn_sessions, mixnet_sessions, unknown_sessions)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
+            record.gateway_identity_key,
+            record.node_id,
+            record.day,
+            record.unique_active_clients,
+            record.session_started,
+            record.users_hashes,
+            record.vpn_sessions,
+            record.mixnet_sessions,
+            record.unknown_sessions,
+        )
+        .execute(&mut *tx)
+        .await?;
+    }
+    tx.commit().await?;
+
+    Ok(())
+}
+
+pub(crate) async fn get_sessions_stats(pool: &DbPool) -> anyhow::Result<Vec<SessionStats>> {
+    let mut conn = pool.acquire().await?;
+    let items = sqlx::query_as(
+        "SELECT gateway_identity_key, 
+                node_id,              
+                day,                  
+                unique_active_clients,
+                session_started,      
+                users_hashes,         
+                vpn_sessions,         
+                mixnet_sessions,      
+                unknown_sessions
+        FROM gateway_session_stats",
+    )
+    .fetch(&mut *conn)
+    .try_collect::<Vec<GatewaySessionsRecord>>()
+    .await?;
+
+    let items: Vec<SessionStats> = items
+        .into_iter()
+        .map(|item| item.try_into())
+        .collect::<anyhow::Result<Vec<_>>>()
+        .map_err(|e| {
+            error!("Conversion from database failed: {e}. Invalidly stored data?");
+            e
+        })?;
+
+    Ok(items)
+}
+
+pub(crate) async fn delete_old_records(pool: &DbPool, cut_off: Date) -> anyhow::Result<()> {
+    let mut conn = pool.acquire().await?;
+    sqlx::query!("DELETE FROM gateway_session_stats WHERE day <= ?", cut_off)
+        .execute(&mut *conn)
+        .await?;
+    Ok(())
+}

nym-node-status-api/nym-node-status-api/src/db/queries/mod.rs

@@ -1,4 +1,5 @@
 mod gateways;
+mod gateways_stats;
 mod misc;
 mod mixnodes;
 mod summary;
@@ -13,3 +14,5 @@ pub(crate) use mixnodes::{
     ensure_mixnodes_still_bonded, get_all_mixnodes, get_daily_stats, insert_mixnodes,
 };
 pub(crate) use summary::{get_summary, get_summary_history};
+
+pub(crate) use gateways_stats::{delete_old_records, get_sessions_stats, insert_session_records};

nym-node-status-api/nym-node-status-api/src/db/queries/summary.rs

@@ -8,7 +8,7 @@ use crate::{
         models::{
             gateway::{
                 GatewaySummary, GatewaySummaryBlacklisted, GatewaySummaryBonded,
-                GatewaySummaryExplorer, GatewaySummaryHistorical,
+                GatewaySummaryHistorical,
             },
             mixnode::{
                 MixnodeSummary, MixnodeSummaryBlacklisted, MixnodeSummaryBonded,
@@ -82,7 +82,6 @@ async fn from_summary_dto(items: Vec<SummaryDto>) -> HttpResult<NetworkSummary>
     const MIXNODES_BONDED_RESERVE: &str = "mixnodes.bonded.reserve";
     const MIXNODES_BLACKLISTED_COUNT: &str = "mixnodes.blacklisted.count";
     const GATEWAYS_BONDED_COUNT: &str = "gateways.bonded.count";
-    const GATEWAYS_EXPLORER_COUNT: &str = "gateways.explorer.count";
     const GATEWAYS_BLACKLISTED_COUNT: &str = "gateways.blacklisted.count";
     const MIXNODES_HISTORICAL_COUNT: &str = "mixnodes.historical.count";
     const GATEWAYS_HISTORICAL_COUNT: &str = "gateways.historical.count";
@@ -96,7 +95,6 @@ async fn from_summary_dto(items: Vec<SummaryDto>) -> HttpResult<NetworkSummary>
     // check we have all the keys we are expecting, and build up a map of errors for missing one
     let keys = [
         GATEWAYS_BONDED_COUNT,
-        GATEWAYS_EXPLORER_COUNT,
         GATEWAYS_HISTORICAL_COUNT,
         GATEWAYS_BLACKLISTED_COUNT,
         MIXNODES_BLACKLISTED_COUNT,
@@ -139,10 +137,6 @@ async fn from_summary_dto(items: Vec<SummaryDto>) -> HttpResult<NetworkSummary>
         .unwrap_or_default();
     let gateways_bonded_count: SummaryDto =
         map.get(GATEWAYS_BONDED_COUNT).cloned().unwrap_or_default();
-    let gateways_explorer_count: SummaryDto = map
-        .get(GATEWAYS_EXPLORER_COUNT)
-        .cloned()
-        .unwrap_or_default();
     let mixnodes_historical_count: SummaryDto = map
         .get(MIXNODES_HISTORICAL_COUNT)
         .cloned()
@@ -187,10 +181,6 @@ async fn from_summary_dto(items: Vec<SummaryDto>) -> HttpResult<NetworkSummary>
                 count: to_count_i32(&gateways_historical_count),
                 last_updated_utc: to_timestamp(&gateways_historical_count),
             },
-            explorer: GatewaySummaryExplorer {
-                count: to_count_i32(&gateways_explorer_count),
-                last_updated_utc: to_timestamp(&gateways_explorer_count),
-            },
         },
     })
 }

nym-node-status-api/nym-node-status-api/src/http/api/metrics/mod.rs

@@ -0,0 +1,10 @@
+use axum::Router;
+
+use crate::http::state::AppState;
+
+pub(crate) mod sessions;
+
+pub(crate) fn routes() -> Router<AppState> {
+    Router::new().nest("/sessions", sessions::routes())
+    //eventually add other metrics type
+}

nym-node-status-api/nym-node-status-api/src/http/api/metrics/sessions.rs

@@ -0,0 +1,83 @@
+use axum::{
+    extract::{Query, State},
+    Json, Router,
+};
+use time::Date;
+use tracing::instrument;
+
+use crate::http::{
+    error::{HttpError, HttpResult},
+    models::SessionStats,
+    state::AppState,
+    PagedResult, Pagination,
+};
+
+pub(crate) fn routes() -> Router<AppState> {
+    Router::new().route("/", axum::routing::get(get_all_sessions))
+    // .route("/:node_id", axum::routing::get(get_node_sessions))
+    // .route("/:day", axum::routing::get(get_daily_sessions))
+}
+
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, utoipa::IntoParams)]
+#[into_params(parameter_in = Query)]
+pub(crate) struct SessionQueryParams {
+    size: Option<usize>,
+    page: Option<usize>,
+    node_id: Option<String>,
+    day: Option<String>,
+}
+
+#[utoipa::path(
+    tag = "Sessions",
+    get,
+    params(
+        SessionQueryParams
+    ),
+    path = "/v2/metrics/sessions",
+    responses(
+        (status = 200, body = PagedSessionStats)
+    )
+)]
+#[instrument(level = tracing::Level::DEBUG, skip(state))]
+async fn get_all_sessions(
+    Query(SessionQueryParams {
+        size,
+        page,
+        node_id,
+        day,
+    }): Query<SessionQueryParams>,
+    State(state): State<AppState>,
+) -> HttpResult<Json<PagedResult<SessionStats>>> {
+    let db = state.db_pool();
+    let res = state.cache().get_sessions_stats(db).await;
+
+    let day_filtered = if let Some(day) = day {
+        if let Ok(parsed_day) =
+            Date::parse(&day, &time::format_description::well_known::Iso8601::DATE)
+        {
+            res.into_iter().filter(|s| s.day == parsed_day).collect()
+        } else {
+            return Err(HttpError::invalid_input(day));
+        }
+    } else {
+        res
+    };
+
+    let day_and_node_filtered = if let Some(node_id) = node_id {
+        if let Ok(parsed_node_id) = node_id.parse::<u32>() {
+            day_filtered
+                .into_iter()
+                .filter(|s| s.node_id == parsed_node_id)
+                .collect()
+        } else {
+            return Err(HttpError::invalid_input(node_id));
+        }
+    } else {
+        day_filtered
+    };
+
+    Ok(Json(PagedResult::paginate(
+        Pagination { size, page },
+        day_and_node_filtered,
+    )))
+}

nym-node-status-api/nym-node-status-api/src/http/api/mod.rs

@@ -8,6 +8,7 @@ use utoipa_swagger_ui::SwaggerUi;
 use crate::http::{server::HttpServer, state::AppState};
 
 pub(crate) mod gateways;
+pub(crate) mod metrics;
 pub(crate) mod mixnodes;
 pub(crate) mod services;
 pub(crate) mod summary;
@@ -34,7 +35,8 @@ impl RouterBuilder {
                     .nest("/gateways", gateways::routes())
                     .nest("/mixnodes", mixnodes::routes())
                     .nest("/services", services::routes())
-                    .nest("/summary", summary::routes()),
+                    .nest("/summary", summary::routes())
+                    .nest("/metrics", metrics::routes()),
             )
             .nest(
                 "/internal",

nym-node-status-api/nym-node-status-api/src/http/api_docs.rs

@@ -1,4 +1,4 @@
-use crate::http::{Gateway, GatewaySkinny, Mixnode, Service};
+use crate::http::{Gateway, GatewaySkinny, Mixnode, Service, SessionStats};
 use utoipa::OpenApi;
 use utoipauto::utoipauto;
 

nym-node-status-api/nym-node-status-api/src/http/mod.rs

@@ -1,4 +1,4 @@
-use models::{Gateway, GatewaySkinny, Mixnode, Service};
+use models::{Gateway, GatewaySkinny, Mixnode, Service, SessionStats};
 
 pub(crate) mod api;
 pub(crate) mod api_docs;
@@ -20,6 +20,7 @@ pub(crate) mod state;
     PagedGatewaySkinny = PagedResult<GatewaySkinny>,
     PagedMixnode = PagedResult<Mixnode>,
     PagedService = PagedResult<Service>,
+    PagedSessionStats = PagedResult<SessionStats>
 )]
 pub struct PagedResult<T> {
     pub page: usize,

nym-node-status-api/nym-node-status-api/src/http/models.rs

@@ -74,3 +74,17 @@ pub(crate) struct SummaryHistory {
     pub value_json: serde_json::Value,
     pub timestamp_utc: String,
 }
+
+#[derive(Debug, Clone, Deserialize, Serialize, ToSchema)]
+pub struct SessionStats {
+    pub gateway_identity_key: String,
+    pub node_id: u32,
+    #[serde(with = "nym_serde_helpers::date")]
+    pub day: time::Date,
+    pub unique_active_clients: i64,
+    pub session_started: i64,
+    pub users_hashes: Option<serde_json::Value>,
+    pub vpn_sessions: Option<serde_json::Value>,
+    pub mixnet_sessions: Option<serde_json::Value>,
+    pub unknown_sessions: Option<serde_json::Value>,
+}

nym-node-status-api/nym-node-status-api/src/http/state.rs

@@ -9,6 +9,8 @@ use crate::{
     http::models::{DailyStats, Gateway, Mixnode, SummaryHistory},
 };
 
+use super::models::SessionStats;
+
 #[derive(Debug, Clone)]
 pub(crate) struct AppState {
     db_pool: DbPool,
@@ -53,6 +55,7 @@ static GATEWAYS_LIST_KEY: &str = "gateways";
 static MIXNODES_LIST_KEY: &str = "mixnodes";
 static MIXSTATS_LIST_KEY: &str = "mixstats";
 static SUMMARY_HISTORY_LIST_KEY: &str = "summary-history";
+static SESSION_STATS_LIST_KEY: &str = "session-stats";
 
 #[derive(Debug, Clone)]
 pub(crate) struct HttpCache {
@@ -60,6 +63,7 @@ pub(crate) struct HttpCache {
     mixnodes: Cache<String, Arc<RwLock<Vec<Mixnode>>>>,
     mixstats: Cache<String, Arc<RwLock<Vec<DailyStats>>>>,
     history: Cache<String, Arc<RwLock<Vec<SummaryHistory>>>>,
+    session_stats: Cache<String, Arc<RwLock<Vec<SessionStats>>>>,
 }
 
 impl HttpCache {
@@ -81,6 +85,10 @@ impl HttpCache {
                 .max_capacity(2)
                 .time_to_live(Duration::from_secs(ttl_seconds))
                 .build(),
+            session_stats: Cache::builder()
+                .max_capacity(2)
+                .time_to_live(Duration::from_secs(ttl_seconds))
+                .build(),
         }
     }
 
@@ -238,4 +246,39 @@ impl HttpCache {
             })
             .await
     }
+
+    pub async fn get_sessions_stats(&self, db: &DbPool) -> Vec<SessionStats> {
+        match self.session_stats.get(SESSION_STATS_LIST_KEY).await {
+            Some(guard) => {
+                let read_lock = guard.read().await;
+                read_lock.to_vec()
+            }
+            None => {
+                let session_stats = crate::db::queries::get_sessions_stats(db)
+                    .await
+                    .unwrap_or_default();
+                self.upsert_sessions_stats(session_stats.clone()).await;
+                session_stats
+            }
+        }
+    }
+
+    pub async fn upsert_sessions_stats(
+        &self,
+        session_stats: Vec<SessionStats>,
+    ) -> Entry<String, Arc<RwLock<Vec<SessionStats>>>> {
+        self.session_stats
+            .entry_by_ref(SESSION_STATS_LIST_KEY)
+            .and_upsert_with(|maybe_entry| async {
+                if let Some(entry) = maybe_entry {
+                    let v = entry.into_value();
+                    let mut guard = v.write().await;
+                    *guard = session_stats;
+                    v.clone()
+                } else {
+                    Arc::new(RwLock::new(session_stats))
+                }
+            })
+            .await
+    }
 }

nym-node-status-api/nym-node-status-api/src/main.rs

@@ -7,6 +7,7 @@ mod db;
 mod http;
 mod logging;
 mod monitor;
+mod node_scraper;
 mod testruns;
 
 #[tokio::main]
@@ -26,21 +27,13 @@ async fn main() -> anyhow::Result<()> {
     tracing::debug!("Using config:\n{:#?}", args);
 
     let storage = db::Storage::init(connection_url).await?;
-    let db_pool = storage.pool_owned();
     let args_clone = args.clone();
-    tokio::spawn(async move {
-        monitor::spawn_in_background(
-            db_pool,
-            args_clone.explorer_client_timeout,
-            args_clone.nym_api_client_timeout,
-            &args_clone.nyxd_addr,
-            args_clone.monitor_refresh_interval,
-        )
-        .await;
-        tracing::info!("Started monitor task");
-    });
 
-    testruns::spawn(storage.pool_owned(), args.testruns_refresh_interval).await;
+    let db_pool_scraper = storage.pool_owned();
+    tokio::spawn(async move {
+        node_scraper::spawn_in_background(db_pool_scraper, args_clone.nym_api_client_timeout).await;
+        tracing::info!("Started metrics scraper task");
+    });
 
     let shutdown_handles = http::server::start_http_api(
         storage.pool_owned(),

nym-node-status-api/nym-node-status-api/src/monitor/geodata.rs

@@ -0,0 +1,112 @@
+use cosmwasm_std::{Addr, Coin};
+use serde::{Deserialize, Serialize};
+
+pub(crate) struct IpInfoClient {
+    client: reqwest::Client,
+    token: String,
+}
+
+impl IpInfoClient {
+    pub(crate) fn new(token: impl Into<String>) -> Self {
+        let client = reqwest::Client::new();
+        let token = token.into();
+
+        Self { client, token }
+    }
+
+    pub(crate) async fn locate_ip(&self, ip: impl AsRef<str>) -> anyhow::Result<Location> {
+        let url = format!(
+            "https://ipinfo.io/{}/country?token={}",
+            ip.as_ref(),
+            &self.token
+        );
+        let response = self
+            .client
+            .get(url)
+            .send()
+            .await
+            // map non 2xx responses to error
+            .and_then(|res| res.error_for_status())
+            .map_err(|err| {
+                if matches!(err.status(), Some(reqwest::StatusCode::TOO_MANY_REQUESTS)) {
+                    tracing::error!("ipinfo rate limit exceeded");
+                }
+                anyhow::Error::from(err)
+            })?;
+        let response_text = response.text().await?.trim().to_string();
+
+        Ok(Location {
+            two_letter_iso_country_code: response_text,
+        })
+    }
+
+    /// check DOESN'T consume bandwidth allowance
+    pub(crate) async fn check_remaining_bandwidth(
+        &self,
+    ) -> anyhow::Result<ipinfo::MeResponseRequests> {
+        let url = format!("https://ipinfo.io/me?token={}", &self.token);
+        let response = self
+            .client
+            .get(url)
+            .send()
+            .await
+            // map non 2xx responses to error
+            .and_then(|res| res.error_for_status())
+            .map_err(|err| {
+                if matches!(err.status(), Some(reqwest::StatusCode::TOO_MANY_REQUESTS)) {
+                    tracing::error!("ipinfo rate limit exceeded");
+                }
+                anyhow::Error::from(err)
+            })?;
+        let response: ipinfo::MeResponse = response.json().await?;
+
+        Ok(response.requests)
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub(crate) struct NodeGeoData {
+    pub(crate) identity_key: String,
+    pub(crate) owner: Addr,
+    pub(crate) pledge_amount: Coin,
+    pub(crate) location: Location,
+}
+
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub(crate) struct Location {
+    pub(crate) two_letter_iso_country_code: String,
+}
+
+impl Location {
+    pub(crate) fn empty() -> Self {
+        Self {
+            two_letter_iso_country_code: String::new(),
+        }
+    }
+}
+
+pub(crate) mod ipinfo {
+    use super::*;
+
+    // clippy doesn't understand it's used for typed deserialization
+    #[allow(dead_code)]
+    #[derive(Debug, Clone, Deserialize)]
+    /// `/me` is undocumented in their developers page
+    /// https://ipinfo.io/developers/responses
+    /// but explained here
+    /// https://community.ipinfo.io/t/easy-way-to-check-allowance-usage/5755/2
+    pub(crate) struct MeResponse {
+        token: String,
+        pub(crate) requests: MeResponseRequests,
+    }
+
+    // clippy doesn't understand it's used for typed deserialization
+    #[allow(dead_code)]
+    #[derive(Debug, Clone, Deserialize)]
+    pub(crate) struct MeResponseRequests {
+        pub(crate) day: u64,
+        pub(crate) month: u64,
+        pub(crate) limit: u64,
+        pub(crate) remaining: u64,
+    }
+}

nym-node-status-api/nym-node-status-api/src/monitor/mod.rs

@@ -2,16 +2,17 @@
 
 use crate::db::models::{
     gateway, mixnode, GatewayRecord, MixnodeRecord, NetworkSummary, GATEWAYS_BLACKLISTED_COUNT,
-    GATEWAYS_BONDED_COUNT, GATEWAYS_EXPLORER_COUNT, GATEWAYS_HISTORICAL_COUNT,
-    MIXNODES_BLACKLISTED_COUNT, MIXNODES_BONDED_ACTIVE, MIXNODES_BONDED_COUNT,
-    MIXNODES_BONDED_INACTIVE, MIXNODES_BONDED_RESERVE, MIXNODES_HISTORICAL_COUNT,
+    GATEWAYS_BONDED_COUNT, GATEWAYS_HISTORICAL_COUNT, MIXNODES_BLACKLISTED_COUNT,
+    MIXNODES_BONDED_ACTIVE, MIXNODES_BONDED_COUNT, MIXNODES_BONDED_INACTIVE,
+    MIXNODES_BONDED_RESERVE, MIXNODES_HISTORICAL_COUNT,
 };
 use crate::db::{queries, DbPool};
+use crate::monitor::geodata::{Location, NodeGeoData};
 use anyhow::anyhow;
 use cosmwasm_std::Decimal;
-use nym_explorer_client::{ExplorerClient, PrettyDetailedGatewayBond};
+use moka::future::Cache;
 use nym_network_defaults::NymNetworkDetails;
-use nym_validator_client::client::NymApiClientExt;
+use nym_validator_client::client::{NodeId, NymApiClientExt};
 use nym_validator_client::models::{
     LegacyDescribedMixNode, MixNodeBondAnnotated, NymNodeDescription,
 };
@@ -20,40 +21,55 @@ use nym_validator_client::nyxd::contract_traits::PagedMixnetQueryClient;
 use nym_validator_client::nyxd::{AccountId, NyxdClient};
 use nym_validator_client::NymApiClient;
 use reqwest::Url;
-use std::collections::HashSet;
+use std::collections::{HashMap, HashSet};
 use std::str::FromStr;
 use tokio::time::Duration;
 use tracing::instrument;
 
+pub(crate) use geodata::IpInfoClient;
+
+mod geodata;
+
 // TODO dz should be configurable
 const FAILURE_RETRY_DELAY: Duration = Duration::from_secs(60);
 
 static DELEGATION_PROGRAM_WALLET: &str = "n1rnxpdpx3kldygsklfft0gech7fhfcux4zst5lw";
 
+struct Monitor {
+    db_pool: DbPool,
+    network_details: NymNetworkDetails,
+    nym_api_client_timeout: Duration,
+    nyxd_addr: Url,
+    ipinfo: IpInfoClient,
+    geocache: Cache<NodeId, Location>,
+}
+
 // TODO dz: query many NYM APIs:
 // multiple instances running directory cache, ask sachin
 #[instrument(level = "debug", name = "data_monitor", skip_all)]
 pub(crate) async fn spawn_in_background(
     db_pool: DbPool,
-    explorer_client_timeout: Duration,
     nym_api_client_timeout: Duration,
-    nyxd_addr: &Url,
+    nyxd_addr: Url,
     refresh_interval: Duration,
+    ipinfo_api_token: String,
+    geodata_ttl: Duration,
 ) {
-    let network_defaults = nym_network_defaults::NymNetworkDetails::new_from_env();
+    let geocache = Cache::builder().time_to_live(geodata_ttl).build();
+    let ipinfo = IpInfoClient::new(ipinfo_api_token.clone());
+    let mut monitor = Monitor {
+        db_pool,
+        network_details: nym_network_defaults::NymNetworkDetails::new_from_env(),
+        nym_api_client_timeout,
+        nyxd_addr,
+        ipinfo,
+        geocache,
+    };
 
     loop {
         tracing::info!("Refreshing node info...");
 
-        if let Err(e) = run(
-            &db_pool,
-            &network_defaults,
-            explorer_client_timeout,
-            nym_api_client_timeout,
-            nyxd_addr,
-        )
-        .await
-        {
+        if let Err(e) = monitor.run().await {
             tracing::error!(
                 "Monitor run failed: {e}, retrying in {}s...",
                 FAILURE_RETRY_DELAY.as_secs()
@@ -70,344 +86,370 @@ pub(crate) async fn spawn_in_background(
     }
 }
 
-async fn run(
-    pool: &DbPool,
-    network_details: &NymNetworkDetails,
-    explorer_client_timeout: Duration,
-    nym_api_client_timeout: Duration,
-    nyxd_addr: &Url,
-) -> anyhow::Result<()> {
-    let default_api_url = network_details
-        .endpoints
-        .first()
-        .expect("rust sdk mainnet default incorrectly configured")
-        .api_url()
-        .clone()
-        .expect("rust sdk mainnet default missing api_url");
-    let default_explorer_url = network_details.explorer_api.clone().map(|url| {
-        url.parse()
-            .expect("rust sdk mainnet default explorer url not parseable")
-    });
+impl Monitor {
+    async fn run(&mut self) -> anyhow::Result<()> {
+        self.check_ipinfo_bandwidth().await;
 
-    // TODO dz replace explorer api with ipinfo.io
-    let default_explorer_url =
-        default_explorer_url.expect("explorer url missing in network config");
-    let explorer_client =
-        ExplorerClient::new_with_timeout(default_explorer_url, explorer_client_timeout)?;
-    let explorer_gateways = explorer_client
-        .unstable_get_gateways()
-        .await
-        .log_error("unstable_get_gateways")?;
+        let default_api_url = self
+            .network_details
+            .endpoints
+            .first()
+            .expect("rust sdk mainnet default incorrectly configured")
+            .api_url()
+            .clone()
+            .expect("rust sdk mainnet default missing api_url");
 
-    let api_client = NymApiClient::new_with_timeout(default_api_url, nym_api_client_timeout);
+        let api_client =
+            NymApiClient::new_with_timeout(default_api_url, self.nym_api_client_timeout);
 
-    let all_nodes = api_client
-        .get_all_described_nodes()
-        .await
-        .log_error("get_all_described_nodes")?;
-    tracing::debug!("Fetched {} total nodes", all_nodes.len());
+        let all_nodes = api_client
+            .get_all_described_nodes()
+            .await
+            .log_error("get_all_described_nodes")?;
+        tracing::debug!("Fetched {} total nodes", all_nodes.len());
 
-    let gateways = all_nodes
-        .iter()
-        .filter(|node| node.description.declared_role.entry)
-        .collect::<Vec<_>>();
-    tracing::debug!("Of those, {} gateways", gateways.len());
-    for gw in gateways.iter() {
-        tracing::debug!("{}", gw.ed25519_identity_key().to_base58_string());
-    }
+        let gateways = all_nodes
+            .iter()
+            .filter(|node| node.description.declared_role.entry)
+            .collect::<Vec<_>>();
+        tracing::debug!(
+            "{}/{} with declared entry gateway capability",
+            gateways.len(),
+            all_nodes.len()
+        );
 
-    let mixnodes = all_nodes
-        .iter()
-        .filter(|node| node.description.declared_role.mixnode)
-        .collect::<Vec<_>>();
-    tracing::debug!("Of those, {} mixnodes", mixnodes.len());
+        let mixnodes = all_nodes
+            .iter()
+            .filter(|node| node.description.declared_role.mixnode)
+            .collect::<Vec<_>>();
+        tracing::debug!(
+            "{}/{} with declared mixnode capability",
+            mixnodes.len(),
+            all_nodes.len()
+        );
 
-    log_gw_in_explorer_not_api(explorer_gateways.as_slice(), gateways.as_slice());
+        let bonded_node_info = api_client
+            .get_all_bonded_nym_nodes()
+            .await?
+            .into_iter()
+            .map(|node| (node.bond_information.node_id, node.bond_information))
+            // for faster reads
+            .collect::<HashMap<_, _>>();
 
-    let all_skimmed_nodes = api_client
-        .get_all_basic_nodes(None)
-        .await
-        .log_error("get_all_basic_nodes")?;
-
-    let mixnodes = api_client
-        .get_cached_mixnodes()
-        .await
-        .log_error("get_cached_mixnodes")?;
-    tracing::debug!("Fetched {} mixnodes", mixnodes.len());
-
-    // let gateways_blacklisted = gateways.iter().filter(|gw|gw.)
-    let gateways_blacklisted = all_skimmed_nodes
-        .iter()
-        .filter_map(|node| {
-            if node.performance.round_to_integer() <= 50 && node.supported_roles.entry {
-                Some(node.ed25519_identity_pubkey.to_base58_string())
-            } else {
-                None
+        let mut gateway_geodata = Vec::new();
+        for gateway in gateways.iter() {
+            if let Some(node_info) = bonded_node_info.get(&gateway.node_id) {
+                let gw_geodata = NodeGeoData {
+                    identity_key: node_info.node.identity_key.to_owned(),
+                    owner: node_info.owner.to_owned(),
+                    pledge_amount: node_info.original_pledge.to_owned(),
+                    location: self.location_cached(gateway).await,
+                };
+                gateway_geodata.push(gw_geodata);
             }
-        })
-        .collect::<HashSet<_>>();
+        }
 
-    // Cached mixnodes don't include blacklisted nodes
-    // We need that to calculate the total locked tokens later
-    let mixnodes = api_client
-        .nym_api
-        .get_mixnodes_detailed_unfiltered()
-        .await
-        .log_error("get_mixnodes_detailed_unfiltered")?;
-    let mixnodes_described = api_client
-        .nym_api
-        .get_mixnodes_described()
-        .await
-        .log_error("get_mixnodes_described")?;
-    let mixnodes_active = api_client
-        .nym_api
-        .get_active_mixnodes()
-        .await
-        .log_error("get_active_mixnodes")?;
-    let delegation_program_members =
-        get_delegation_program_details(network_details, nyxd_addr).await?;
+        // contains performance data
+        let all_skimmed_nodes = api_client
+            .get_all_basic_nodes(None)
+            .await
+            .log_error("get_all_basic_nodes")?;
 
-    // keep stats for later
-    let count_bonded_mixnodes = mixnodes.len();
-    let count_bonded_gateways = gateways.len();
-    let count_explorer_gateways = explorer_gateways.len();
-    let count_bonded_mixnodes_active = mixnodes_active.len();
+        let gateways_blacklisted = all_skimmed_nodes
+            .iter()
+            .filter_map(|node| {
+                if node.performance.round_to_integer() <= 50 && node.supported_roles.entry {
+                    Some(node.ed25519_identity_pubkey.to_base58_string())
+                } else {
+                    None
+                }
+            })
+            .collect::<HashSet<_>>();
 
-    let gateway_records = prepare_gateway_data(
-        &gateways,
-        &gateways_blacklisted,
-        explorer_gateways,
-        all_skimmed_nodes,
-    )?;
-    queries::insert_gateways(pool, gateway_records)
-        .await
-        .map(|_| {
-            tracing::debug!("Gateway info written to DB!");
-        })?;
+        // Cached mixnodes don't include blacklisted nodes
+        // We need that to calculate the total locked tokens later
+        // TODO dz deprecated API, remove
+        let legacy_mixnodes = api_client
+            .nym_api
+            .get_mixnodes_detailed_unfiltered()
+            .await
+            .log_error("get_mixnodes_detailed_unfiltered")?;
+        let mixnodes_described = api_client
+            .nym_api
+            .get_mixnodes_described()
+            .await
+            .log_error("get_mixnodes_described")?;
+        let mixnodes_active = api_client
+            .nym_api
+            .get_basic_active_mixing_assigned_nodes(None, false, None, None)
+            .await
+            .log_error("get_active_mixnodes")?
+            .nodes
+            .data;
+        let delegation_program_members =
+            get_delegation_program_details(&self.network_details, &self.nyxd_addr).await?;
 
-    // instead of counting blacklisted GWs returned from API cache, count from the active set
-    let count_gateways_blacklisted = gateways
-        .iter()
-        .filter(|gw| {
-            let gw_identity = gw.ed25519_identity_key().to_base58_string();
-            gateways_blacklisted.contains(&gw_identity)
-        })
-        .count();
+        // keep stats for later
+        let count_bonded_mixnodes = mixnodes.len();
+        let count_bonded_gateways = gateways.len();
+        let count_bonded_mixnodes_active = mixnodes_active.len();
 
-    if count_gateways_blacklisted > 0 {
-        queries::write_blacklisted_gateways_to_db(pool, gateways_blacklisted.iter())
+        let gateway_records = self.prepare_gateway_data(
+            &gateways,
+            &gateways_blacklisted,
+            gateway_geodata,
+            all_skimmed_nodes,
+        )?;
+
+        let pool = self.db_pool.clone();
+        queries::insert_gateways(&pool, gateway_records)
             .await
             .map(|_| {
-                tracing::debug!(
-                    "Gateway blacklist info written to DB! {} blacklisted by Nym API",
-                    count_gateways_blacklisted
-                )
+                tracing::debug!("Gateway info written to DB!");
             })?;
-    }
 
-    let mixnode_records =
-        prepare_mixnode_data(&mixnodes, mixnodes_described, delegation_program_members)?;
-    queries::insert_mixnodes(pool, mixnode_records)
-        .await
-        .map(|_| {
-            tracing::debug!("Mixnode info written to DB!");
-        })?;
-
-    let count_mixnodes_blacklisted = mixnodes.iter().filter(|elem| elem.blacklisted).count();
-
-    let recently_unbonded_gateways = queries::ensure_gateways_still_bonded(pool, &gateways).await?;
-    let recently_unbonded_mixnodes = queries::ensure_mixnodes_still_bonded(pool, &mixnodes).await?;
-
-    let count_bonded_mixnodes_reserve = 0; // TODO: NymAPI doesn't report the reserve set size
-    let count_bonded_mixnodes_inactive = count_bonded_mixnodes - count_bonded_mixnodes_active;
-
-    let (all_historical_gateways, all_historical_mixnodes) = calculate_stats(pool).await?;
-
-    //
-    // write summary keys and values to table
-    //
-
-    let nodes_summary = vec![
-        (MIXNODES_BONDED_COUNT, &count_bonded_mixnodes),
-        (MIXNODES_BONDED_ACTIVE, &count_bonded_mixnodes_active),
-        (MIXNODES_BONDED_INACTIVE, &count_bonded_mixnodes_inactive),
-        (MIXNODES_BONDED_RESERVE, &count_bonded_mixnodes_reserve),
-        (MIXNODES_BLACKLISTED_COUNT, &count_mixnodes_blacklisted),
-        (GATEWAYS_BONDED_COUNT, &count_bonded_gateways),
-        (GATEWAYS_EXPLORER_COUNT, &count_explorer_gateways),
-        (MIXNODES_HISTORICAL_COUNT, &all_historical_mixnodes),
-        (GATEWAYS_HISTORICAL_COUNT, &all_historical_gateways),
-        (GATEWAYS_BLACKLISTED_COUNT, &count_gateways_blacklisted),
-    ];
-
-    let last_updated = chrono::offset::Utc::now();
-    let last_updated_utc = last_updated.timestamp().to_string();
-    let network_summary = NetworkSummary {
-        mixnodes: mixnode::MixnodeSummary {
-            bonded: mixnode::MixnodeSummaryBonded {
-                count: count_bonded_mixnodes.cast_checked()?,
-                active: count_bonded_mixnodes_active.cast_checked()?,
-                inactive: count_bonded_mixnodes_inactive.cast_checked()?,
-                reserve: count_bonded_mixnodes_reserve.cast_checked()?,
-                last_updated_utc: last_updated_utc.to_owned(),
-            },
-            blacklisted: mixnode::MixnodeSummaryBlacklisted {
-                count: count_mixnodes_blacklisted.cast_checked()?,
-                last_updated_utc: last_updated_utc.to_owned(),
-            },
-            historical: mixnode::MixnodeSummaryHistorical {
-                count: all_historical_mixnodes.cast_checked()?,
-                last_updated_utc: last_updated_utc.to_owned(),
-            },
-        },
-        gateways: gateway::GatewaySummary {
-            bonded: gateway::GatewaySummaryBonded {
-                count: count_bonded_gateways.cast_checked()?,
-                last_updated_utc: last_updated_utc.to_owned(),
-            },
-            blacklisted: gateway::GatewaySummaryBlacklisted {
-                count: count_gateways_blacklisted.cast_checked()?,
-                last_updated_utc: last_updated_utc.to_owned(),
-            },
-            historical: gateway::GatewaySummaryHistorical {
-                count: all_historical_gateways.cast_checked()?,
-                last_updated_utc: last_updated_utc.to_owned(),
-            },
-            explorer: gateway::GatewaySummaryExplorer {
-                count: count_explorer_gateways.cast_checked()?,
-                last_updated_utc: last_updated_utc.to_owned(),
-            },
-        },
-    };
-
-    queries::insert_summaries(pool, &nodes_summary, &network_summary, last_updated).await?;
-
-    let mut log_lines: Vec<String> = vec![];
-    for (key, value) in nodes_summary.iter() {
-        log_lines.push(format!("{} = {}", key, value));
-    }
-    log_lines.push(format!(
-        "recently_unbonded_mixnodes = {}",
-        recently_unbonded_mixnodes
-    ));
-    log_lines.push(format!(
-        "recently_unbonded_gateways = {}",
-        recently_unbonded_gateways
-    ));
-
-    tracing::info!("Directory summary: \n{}", log_lines.join("\n"));
-
-    Ok(())
-}
-
-fn prepare_gateway_data(
-    gateways: &[&NymNodeDescription],
-    gateways_blacklisted: &HashSet<String>,
-    explorer_gateways: Vec<PrettyDetailedGatewayBond>,
-    skimmed_gateways: Vec<SkimmedNode>,
-) -> anyhow::Result<Vec<GatewayRecord>> {
-    let mut gateway_records = Vec::new();
-
-    for gateway in gateways {
-        let identity_key = gateway.ed25519_identity_key().to_base58_string();
-        let bonded = true;
-        let last_updated_utc = chrono::offset::Utc::now().timestamp();
-        let blacklisted = gateways_blacklisted.contains(&identity_key);
-
-        let self_described = serde_json::to_string(&gateway.description)?;
-
-        let explorer_pretty_bond = explorer_gateways
+        let count_gateways_blacklisted = gateways
             .iter()
-            .find(|g| g.gateway.identity_key.eq(&identity_key));
-        let explorer_pretty_bond = explorer_pretty_bond.and_then(|g| serde_json::to_string(g).ok());
-
-        let performance = skimmed_gateways
-            .iter()
-            .find(|g| {
-                g.ed25519_identity_pubkey
-                    .to_base58_string()
-                    .eq(&identity_key)
+            .filter(|gw| {
+                let gw_identity = gw.ed25519_identity_key().to_base58_string();
+                gateways_blacklisted.contains(&gw_identity)
             })
-            .map(|g| g.performance)
-            .unwrap_or_default()
-            .round_to_integer();
+            .count();
 
-        gateway_records.push(GatewayRecord {
-            identity_key: identity_key.to_owned(),
-            bonded,
-            blacklisted,
-            self_described,
-            explorer_pretty_bond,
-            last_updated_utc,
-            performance,
-        });
+        if count_gateways_blacklisted > 0 {
+            queries::write_blacklisted_gateways_to_db(&pool, gateways_blacklisted.iter())
+                .await
+                .map(|_| {
+                    tracing::debug!(
+                        "Gateway blacklist info written to DB! {} blacklisted by Nym API",
+                        count_gateways_blacklisted
+                    )
+                })?;
+        }
+
+        let mixnode_records = self.prepare_mixnode_data(
+            &legacy_mixnodes,
+            mixnodes_described,
+            delegation_program_members,
+        )?;
+        queries::insert_mixnodes(&pool, mixnode_records)
+            .await
+            .map(|_| {
+                tracing::debug!("Mixnode info written to DB!");
+            })?;
+
+        let count_mixnodes_blacklisted = legacy_mixnodes
+            .iter()
+            .filter(|elem| elem.blacklisted)
+            .count();
+
+        let recently_unbonded_gateways =
+            queries::ensure_gateways_still_bonded(&pool, &gateways).await?;
+        let recently_unbonded_mixnodes =
+            queries::ensure_mixnodes_still_bonded(&pool, &legacy_mixnodes).await?;
+
+        let count_bonded_mixnodes_reserve = 0; // TODO: NymAPI doesn't report the reserve set size
+        let count_bonded_mixnodes_inactive = count_bonded_mixnodes - count_bonded_mixnodes_active;
+
+        let (all_historical_gateways, all_historical_mixnodes) = calculate_stats(&pool).await?;
+
+        //
+        // write summary keys and values to table
+        //
+
+        let nodes_summary = vec![
+            (MIXNODES_BONDED_COUNT, &count_bonded_mixnodes),
+            (MIXNODES_BONDED_ACTIVE, &count_bonded_mixnodes_active),
+            (MIXNODES_BONDED_INACTIVE, &count_bonded_mixnodes_inactive),
+            (MIXNODES_BONDED_RESERVE, &count_bonded_mixnodes_reserve),
+            (MIXNODES_BLACKLISTED_COUNT, &count_mixnodes_blacklisted),
+            (GATEWAYS_BONDED_COUNT, &count_bonded_gateways),
+            (MIXNODES_HISTORICAL_COUNT, &all_historical_mixnodes),
+            (GATEWAYS_HISTORICAL_COUNT, &all_historical_gateways),
+            (GATEWAYS_BLACKLISTED_COUNT, &count_gateways_blacklisted),
+        ];
+
+        let last_updated = chrono::offset::Utc::now();
+        let last_updated_utc = last_updated.timestamp().to_string();
+        let network_summary = NetworkSummary {
+            mixnodes: mixnode::MixnodeSummary {
+                bonded: mixnode::MixnodeSummaryBonded {
+                    count: count_bonded_mixnodes.cast_checked()?,
+                    active: count_bonded_mixnodes_active.cast_checked()?,
+                    inactive: count_bonded_mixnodes_inactive.cast_checked()?,
+                    reserve: count_bonded_mixnodes_reserve.cast_checked()?,
+                    last_updated_utc: last_updated_utc.to_owned(),
+                },
+                blacklisted: mixnode::MixnodeSummaryBlacklisted {
+                    count: count_mixnodes_blacklisted.cast_checked()?,
+                    last_updated_utc: last_updated_utc.to_owned(),
+                },
+                historical: mixnode::MixnodeSummaryHistorical {
+                    count: all_historical_mixnodes.cast_checked()?,
+                    last_updated_utc: last_updated_utc.to_owned(),
+                },
+            },
+            gateways: gateway::GatewaySummary {
+                bonded: gateway::GatewaySummaryBonded {
+                    count: count_bonded_gateways.cast_checked()?,
+                    last_updated_utc: last_updated_utc.to_owned(),
+                },
+                blacklisted: gateway::GatewaySummaryBlacklisted {
+                    count: count_gateways_blacklisted.cast_checked()?,
+                    last_updated_utc: last_updated_utc.to_owned(),
+                },
+                historical: gateway::GatewaySummaryHistorical {
+                    count: all_historical_gateways.cast_checked()?,
+                    last_updated_utc: last_updated_utc.to_owned(),
+                },
+            },
+        };
+
+        queries::insert_summaries(&pool, &nodes_summary, &network_summary, last_updated).await?;
+
+        let mut log_lines: Vec<String> = vec![];
+        for (key, value) in nodes_summary.iter() {
+            log_lines.push(format!("{} = {}", key, value));
+        }
+        log_lines.push(format!(
+            "recently_unbonded_mixnodes = {}",
+            recently_unbonded_mixnodes
+        ));
+        log_lines.push(format!(
+            "recently_unbonded_gateways = {}",
+            recently_unbonded_gateways
+        ));
+
+        tracing::info!("Directory summary: \n{}", log_lines.join("\n"));
+
+        Ok(())
     }
 
-    Ok(gateway_records)
-}
+    #[instrument(level = "debug", skip_all)]
+    async fn location_cached(&mut self, node: &NymNodeDescription) -> Location {
+        let node_id = node.node_id;
 
-fn prepare_mixnode_data(
-    mixnodes: &[MixNodeBondAnnotated],
-    mixnodes_described: Vec<LegacyDescribedMixNode>,
-    delegation_program_members: Vec<u32>,
-) -> anyhow::Result<Vec<MixnodeRecord>> {
-    let mut mixnode_records = Vec::new();
-
-    for mixnode in mixnodes {
-        let mix_id = mixnode.mix_id();
-        let identity_key = mixnode.identity_key();
-        let bonded = true;
-        let total_stake = decimal_to_i64(mixnode.mixnode_details.total_stake());
-        let blacklisted = mixnode.blacklisted;
-        let node_info = mixnode.mix_node();
-        let host = node_info.host.clone();
-        let http_port = node_info.http_api_port;
-        // Contains all the information including what's above
-        let full_details = serde_json::to_string(&mixnode)?;
-
-        let mixnode_described = mixnodes_described.iter().find(|m| m.bond.mix_id == mix_id);
-        let self_described = mixnode_described.and_then(|v| serde_json::to_string(v).ok());
-        let is_dp_delegatee = delegation_program_members.contains(&mix_id);
-
-        let last_updated_utc = chrono::offset::Utc::now().timestamp();
-
-        mixnode_records.push(MixnodeRecord {
-            mix_id,
-            identity_key: identity_key.to_owned(),
-            bonded,
-            total_stake,
-            host,
-            http_port,
-            blacklisted,
-            full_details,
-            self_described,
-            last_updated_utc,
-            is_dp_delegatee,
-        });
+        match self.geocache.get(&node_id).await {
+            Some(location) => return location,
+            None => {
+                for ip in node.description.host_information.ip_address.iter() {
+                    if let Ok(location) = self.ipinfo.locate_ip(ip.to_string()).await {
+                        self.geocache.insert(node_id, location.clone()).await;
+                        return location;
+                    }
+                }
+                // if no data could be retrieved
+                tracing::debug!("No geodata could be retrieved for {}", node_id);
+                Location::empty()
+            }
+        }
     }
 
-    Ok(mixnode_records)
-}
+    fn prepare_gateway_data(
+        &self,
+        gateways: &[&NymNodeDescription],
+        gateways_blacklisted: &HashSet<String>,
+        gateway_geodata: Vec<NodeGeoData>,
+        skimmed_gateways: Vec<SkimmedNode>,
+    ) -> anyhow::Result<Vec<GatewayRecord>> {
+        let mut gateway_records = Vec::new();
 
-fn log_gw_in_explorer_not_api(
-    explorer: &[PrettyDetailedGatewayBond],
-    api_gateways: &[&NymNodeDescription],
-) {
-    let api_gateways = api_gateways
-        .iter()
-        .map(|gw| gw.ed25519_identity_key().to_base58_string())
-        .collect::<HashSet<_>>();
-    let explorer_only = explorer
-        .iter()
-        .filter(|gw| !api_gateways.contains(&gw.gateway.identity_key.to_string()))
-        .collect::<Vec<_>>();
+        for gateway in gateways {
+            let identity_key = gateway.ed25519_identity_key().to_base58_string();
+            let bonded = true;
+            let last_updated_utc = chrono::offset::Utc::now().timestamp();
+            let blacklisted = gateways_blacklisted.contains(&identity_key);
 
-    tracing::debug!(
-        "Gateways listed by explorer but not by Nym API: {}",
-        explorer_only.len()
-    );
-    for gw in explorer_only.iter() {
-        tracing::debug!("{}", gw.gateway.identity_key.to_string());
+            let self_described = serde_json::to_string(&gateway.description)?;
+
+            let explorer_pretty_bond = gateway_geodata
+                .iter()
+                .find(|g| g.identity_key.eq(&identity_key));
+            let explorer_pretty_bond =
+                explorer_pretty_bond.and_then(|g| serde_json::to_string(g).ok());
+
+            let performance = skimmed_gateways
+                .iter()
+                .find(|g| {
+                    g.ed25519_identity_pubkey
+                        .to_base58_string()
+                        .eq(&identity_key)
+                })
+                .map(|g| g.performance)
+                .unwrap_or_default()
+                .round_to_integer();
+
+            gateway_records.push(GatewayRecord {
+                identity_key: identity_key.to_owned(),
+                bonded,
+                blacklisted,
+                self_described,
+                explorer_pretty_bond,
+                last_updated_utc,
+                performance,
+            });
+        }
+
+        Ok(gateway_records)
+    }
+
+    fn prepare_mixnode_data(
+        &self,
+        mixnodes: &[MixNodeBondAnnotated],
+        mixnodes_described: Vec<LegacyDescribedMixNode>,
+        delegation_program_members: Vec<u32>,
+    ) -> anyhow::Result<Vec<MixnodeRecord>> {
+        let mut mixnode_records = Vec::new();
+
+        for mixnode in mixnodes {
+            let mix_id = mixnode.mix_id();
+            let identity_key = mixnode.identity_key();
+            let bonded = true;
+            let total_stake = decimal_to_i64(mixnode.mixnode_details.total_stake());
+            let blacklisted = mixnode.blacklisted;
+            let node_info = mixnode.mix_node();
+            let host = node_info.host.clone();
+            let http_port = node_info.http_api_port;
+            // Contains all the information including what's above
+            let full_details = serde_json::to_string(&mixnode)?;
+
+            let mixnode_described = mixnodes_described.iter().find(|m| m.bond.mix_id == mix_id);
+            let self_described = mixnode_described.and_then(|v| serde_json::to_string(v).ok());
+            let is_dp_delegatee = delegation_program_members.contains(&mix_id);
+
+            let last_updated_utc = chrono::offset::Utc::now().timestamp();
+
+            mixnode_records.push(MixnodeRecord {
+                mix_id,
+                identity_key: identity_key.to_owned(),
+                bonded,
+                total_stake,
+                host,
+                http_port,
+                blacklisted,
+                full_details,
+                self_described,
+                last_updated_utc,
+                is_dp_delegatee,
+            });
+        }
+
+        Ok(mixnode_records)
+    }
+
+    async fn check_ipinfo_bandwidth(&self) {
+        match self.ipinfo.check_remaining_bandwidth().await {
+            Ok(bandwidth) => {
+                tracing::info!(
+                    "ipinfo monthly bandwidth: {}/{} spent",
+                    bandwidth.month,
+                    bandwidth.limit
+                );
+            }
+            Err(err) => {
+                tracing::debug!("Couldn't check ipinfo bandwidth: {}", err);
+            }
+        }
     }
 }
 

nym-node-status-api/nym-node-status-api/src/node_scraper/error.rs

@@ -0,0 +1,18 @@
+use nym_network_defaults::DEFAULT_NYM_NODE_HTTP_PORT;
+use nym_node_requests::api::client::NymNodeApiClientError;
+use nym_validator_client::client::NodeId;
+use thiserror::Error;
+
+#[derive(Debug, Error)]
+pub enum NodeScraperError {
+    #[error("node {node_id} has provided malformed host information ({host}: {source}")]
+    MalformedHost {
+        host: String,
+        node_id: NodeId,
+        #[source]
+        source: NymNodeApiClientError,
+    },
+
+    #[error("node {node_id} with host '{host}' doesn't seem to expose its declared http port nor any of the standard API ports, i.e.: 80, 443 or {}", DEFAULT_NYM_NODE_HTTP_PORT)]
+    NoHttpPortsAvailable { host: String, node_id: NodeId },
+}

nym-node-status-api/nym-node-status-api/src/node_scraper/mod.rs

@@ -0,0 +1,279 @@
+use crate::db::{models::GatewaySessionsRecord, queries, DbPool};
+use error::NodeScraperError;
+
+use nym_network_defaults::{NymNetworkDetails, DEFAULT_NYM_NODE_HTTP_PORT};
+use nym_node_requests::api::{client::NymNodeApiClientExt, v1::metrics::models::SessionStats};
+use nym_statistics_common::gateways::SessionType;
+use nym_validator_client::{
+    client::{NodeId, NymNodeDetails},
+    models::{DescribedNodeType, NymNodeDescription},
+    NymApiClient,
+};
+use time::OffsetDateTime;
+
+use std::collections::HashMap;
+use tokio::time::Duration;
+use tracing::instrument;
+
+mod error;
+
+const FAILURE_RETRY_DELAY: Duration = Duration::from_secs(60);
+const REFRESH_INTERVAL: Duration = Duration::from_secs(60 * 60 * 6); //6h, data only update once a day
+const STALE_DURATION: Duration = Duration::from_secs(86400 * 365); //one year
+
+#[instrument(level = "debug", name = "node_scraper", skip_all)]
+pub(crate) async fn spawn_in_background(db_pool: DbPool, nym_api_client_timeout: Duration) {
+    let network_defaults = nym_network_defaults::NymNetworkDetails::new_from_env();
+
+    loop {
+        //No graceful shutdown?
+        tracing::info!("Refreshing node self-described metrics...");
+
+        if let Err(e) = run(&db_pool, &network_defaults, nym_api_client_timeout).await {
+            tracing::error!(
+                "Metrics collection failed: {e}, retrying in {}s...",
+                FAILURE_RETRY_DELAY.as_secs()
+            );
+
+            tokio::time::sleep(FAILURE_RETRY_DELAY).await;
+        } else {
+            tracing::info!(
+                "Metrics successfully collected, sleeping for {}s...",
+                REFRESH_INTERVAL.as_secs()
+            );
+            tokio::time::sleep(REFRESH_INTERVAL).await;
+        }
+    }
+}
+
+async fn run(
+    pool: &DbPool,
+    network_details: &NymNetworkDetails,
+    nym_api_client_timeout: Duration,
+) -> anyhow::Result<()> {
+    let default_api_url = network_details
+        .endpoints
+        .first()
+        .expect("rust sdk mainnet default incorrectly configured")
+        .api_url()
+        .clone()
+        .expect("rust sdk mainnet default missing api_url");
+
+    let api_client = NymApiClient::new_with_timeout(default_api_url, nym_api_client_timeout);
+
+    //SW TBC what nodes exactly need to be scraped, the skimmed node endpoint seems to return more nodes
+    let bonded_nodes = api_client.get_all_bonded_nym_nodes().await?;
+    let all_nodes = api_client.get_all_described_nodes().await?; //legacy node that did not upgrade the contract bond yet
+    tracing::debug!("Fetched {} total nodes", all_nodes.len());
+
+    let mut nodes_to_scrape: HashMap<NodeId, MetricsScrapingData> = bonded_nodes
+        .into_iter()
+        .map(|n| (n.node_id(), n.into()))
+        .collect();
+
+    all_nodes
+        .into_iter()
+        .filter(|n| n.contract_node_type != DescribedNodeType::LegacyMixnode)
+        .for_each(|n| {
+            nodes_to_scrape.entry(n.node_id).or_insert_with(|| n.into());
+        });
+    tracing::debug!("Will try to scrape {} nodes", nodes_to_scrape.len());
+
+    let mut session_records = Vec::new();
+    for n in nodes_to_scrape.into_values() {
+        if let Some(stat) = n.try_scrape_metrics().await {
+            session_records.push(prepare_session_data(stat, &n));
+        }
+    }
+
+    queries::insert_session_records(pool, session_records)
+        .await
+        .map(|_| {
+            tracing::debug!("Session info written to DB!");
+        })?;
+    let cut_off_date = (OffsetDateTime::now_utc() - STALE_DURATION).date();
+    queries::delete_old_records(pool, cut_off_date)
+        .await
+        .map(|_| {
+            tracing::debug!("Cleared old data before {}", cut_off_date);
+        })?;
+
+    Ok(())
+}
+
+#[derive(Debug)]
+struct MetricsScrapingData {
+    host: String,
+    node_id: NodeId,
+    id_key: String,
+    port: Option<u16>,
+}
+
+impl MetricsScrapingData {
+    pub fn new(
+        host: impl Into<String>,
+        node_id: NodeId,
+        id_key: String,
+        port: Option<u16>,
+    ) -> Self {
+        MetricsScrapingData {
+            host: host.into(),
+            node_id,
+            id_key,
+            port,
+        }
+    }
+
+    async fn try_scrape_metrics(&self) -> Option<SessionStats> {
+        match self.try_get_client().await {
+            Ok(client) => {
+                match client.get_sessions_metrics().await {
+                    Ok(session_stats) => {
+                        if session_stats.update_time != OffsetDateTime::UNIX_EPOCH {
+                            Some(session_stats)
+                        } else {
+                            //means no data
+                            None
+                        }
+                    }
+                    Err(e) => {
+                        tracing::error!("[metrics scraper]: {e}");
+                        None
+                    }
+                }
+            }
+            Err(e) => {
+                tracing::error!("[metrics scraper]: {e}");
+                None
+            }
+        }
+    }
+
+    async fn try_get_client(&self) -> Result<nym_node_requests::api::Client, NodeScraperError> {
+        // first try the standard port in case the operator didn't put the node behind the proxy,
+        // then default https (443)
+        // finally default http (80)
+        let mut addresses_to_try = vec![
+            format!("http://{0}:{DEFAULT_NYM_NODE_HTTP_PORT}", self.host), // 'standard' nym-node
+            format!("https://{0}", self.host), // node behind https proxy (443)
+            format!("http://{0}", self.host),  // node behind http proxy (80)
+        ];
+
+        // note: I removed 'standard' legacy mixnode port because it should now be automatically pulled via
+        // the 'custom_port' since it should have been present in the contract.
+
+        if let Some(port) = self.port {
+            addresses_to_try.insert(0, format!("http://{0}:{port}", self.host));
+        }
+
+        for address in addresses_to_try {
+            // if provided host was malformed, no point in continuing
+            let client = match nym_node_requests::api::Client::builder(address).and_then(|b| {
+                b.with_timeout(Duration::from_secs(5))
+                    .with_user_agent("node-status-api-metrics-scraper")
+                    .build()
+            }) {
+                Ok(client) => client,
+                Err(err) => {
+                    return Err(NodeScraperError::MalformedHost {
+                        host: self.host.to_string(),
+                        node_id: self.node_id,
+                        source: err,
+                    });
+                }
+            };
+
+            if let Ok(health) = client.get_health().await {
+                if health.status.is_up() {
+                    return Ok(client);
+                }
+            }
+        }
+
+        Err(NodeScraperError::NoHttpPortsAvailable {
+            host: self.host.to_string(),
+            node_id: self.node_id,
+        })
+    }
+}
+
+impl From<NymNodeDetails> for MetricsScrapingData {
+    fn from(value: NymNodeDetails) -> Self {
+        MetricsScrapingData::new(
+            value.bond_information.node.host.clone(),
+            value.node_id(),
+            value.bond_information.node.identity_key,
+            value.bond_information.node.custom_http_port,
+        )
+    }
+}
+
+impl From<NymNodeDescription> for MetricsScrapingData {
+    fn from(value: NymNodeDescription) -> Self {
+        MetricsScrapingData::new(
+            value.description.host_information.ip_address[0].to_string(),
+            value.node_id,
+            value.ed25519_identity_key().to_base58_string(),
+            None,
+        )
+    }
+}
+
+fn prepare_session_data(
+    stat: SessionStats,
+    node_data: &MetricsScrapingData,
+) -> GatewaySessionsRecord {
+    let users_hashes = if !stat.unique_active_users_hashes.is_empty() {
+        Some(serde_json::to_string(&stat.unique_active_users_hashes).unwrap())
+    } else {
+        None
+    };
+    let vpn_durations = stat
+        .sessions
+        .iter()
+        .filter(|s| SessionType::from_string(&s.typ) == SessionType::Vpn)
+        .map(|s| s.duration_ms)
+        .collect::<Vec<_>>();
+
+    let mixnet_durations = stat
+        .sessions
+        .iter()
+        .filter(|s| SessionType::from_string(&s.typ) == SessionType::Mixnet)
+        .map(|s| s.duration_ms)
+        .collect::<Vec<_>>();
+
+    let unkown_durations = stat
+        .sessions
+        .iter()
+        .filter(|s| SessionType::from_string(&s.typ) == SessionType::Unknown)
+        .map(|s| s.duration_ms)
+        .collect::<Vec<_>>();
+
+    let vpn_sessions = if !vpn_durations.is_empty() {
+        Some(serde_json::to_string(&vpn_durations).unwrap())
+    } else {
+        None
+    };
+    let mixnet_sessions = if !mixnet_durations.is_empty() {
+        Some(serde_json::to_string(&mixnet_durations).unwrap())
+    } else {
+        None
+    };
+    let unknown_sessions = if !unkown_durations.is_empty() {
+        Some(serde_json::to_string(&unkown_durations).unwrap())
+    } else {
+        None
+    };
+
+    GatewaySessionsRecord {
+        gateway_identity_key: node_data.id_key.clone(),
+        node_id: node_data.node_id as i64,
+        day: stat.update_time.date(),
+        unique_active_clients: stat.unique_active_users as i64,
+        session_started: stat.sessions_started as i64,
+        users_hashes,
+        vpn_sessions,
+        mixnet_sessions,
+        unknown_sessions,
+    }
+}

nym-node/Cargo.toml

@@ -3,7 +3,7 @@
 
 [package]
 name = "nym-node"
-version = "1.1.11"
+version = "1.1.12"
 authors.workspace = true
 repository.workspace = true
 homepage.workspace = true

nym-node/nym-node-requests/src/api/client.rs

@@ -19,6 +19,7 @@ use crate::api::v1::network_requester::models::NetworkRequester;
 pub use nym_http_api_client::Client;
 
 use super::v1::gateway::models::Wireguard;
+use super::v1::metrics::models::SessionStats;
 
 pub type NymNodeApiClientError = HttpClientError<ErrorResponse>;
 
@@ -87,6 +88,11 @@ pub trait NymNodeApiClientExt: ApiClient {
         self.get_json_from(routes::api::v1::gateway::client_interfaces::wireguard_absolute())
             .await
     }
+
+    async fn get_sessions_metrics(&self) -> Result<SessionStats, NymNodeApiClientError> {
+        self.get_json_from(routes::api::v1::metrics::sessions_absolute())
+            .await
+    }
 }
 
 #[cfg_attr(target_arch = "wasm32", async_trait(?Send))]

nym-node/nym-node-requests/src/api/v1/metrics/models.rs

@@ -50,6 +50,7 @@ pub struct SessionStats {
 
     pub unique_active_users: u32,
 
+    #[serde(default = "Vec::new")] // field was added later
     pub unique_active_users_hashes: Vec<String>,
 
     pub sessions: Vec<Session>,

nym-node/src/config/mixnode.rs

@@ -36,7 +36,7 @@ impl MixnodeConfig {
 }
 
 #[derive(Debug, Clone, Deserialize, PartialEq, Serialize)]
-#[serde(deny_unknown_fields)]
+#[serde(deny_unknown_fields, default)]
 pub struct Verloc {
     /// Socket address this node will use for binding its verloc API.
     /// default: `0.0.0.0:1790`

nym-node/src/config/old_configs/old_config_v1.rs

@@ -11,6 +11,7 @@ use nym_pemstore::store_keypair;
 use old_configs::old_config_v2::*;
 use rand::rngs::OsRng;
 use serde::{Deserialize, Serialize};
+use tracing::instrument;
 
 #[derive(Debug, Clone, Deserialize, PartialEq, Eq, Serialize)]
 #[serde(deny_unknown_fields)]
@@ -756,15 +757,16 @@ fn initialise(config: &WireguardV2) -> std::io::Result<()> {
     Ok(())
 }
 
+#[instrument(skip_all)]
 pub async fn try_upgrade_config_v1<P: AsRef<Path>>(
     path: P,
     prev_config: Option<ConfigV1>,
 ) -> Result<ConfigV2, NymNodeError> {
-    tracing::debug!("Updating from 1.1.2");
+    debug!("attempting to load v1 config...");
     let old_cfg = if let Some(prev_config) = prev_config {
         prev_config
     } else {
-        ConfigV1::read_from_path(&path)?
+        ConfigV1::read_from_path(&path).inspect_err(|err| debug!("failed: {err}"))?
     };
     let wireguard = WireguardV2 {
         enabled: old_cfg.wireguard.enabled,

nym-node/src/config/old_configs/old_config_v2.rs

@@ -16,6 +16,7 @@ use nym_sphinx_acknowledgements::AckKey;
 use old_configs::old_config_v3::*;
 use rand::rngs::OsRng;
 use serde::{Deserialize, Serialize};
+use tracing::instrument;
 
 #[derive(Debug, Clone, Deserialize, PartialEq, Eq, Serialize)]
 #[serde(deny_unknown_fields)]
@@ -786,15 +787,16 @@ pub async fn initialise(
     Ok(())
 }
 
+#[instrument(skip_all)]
 pub async fn try_upgrade_config_v2<P: AsRef<Path>>(
     path: P,
     prev_config: Option<ConfigV2>,
 ) -> Result<ConfigV3, NymNodeError> {
-    tracing::debug!("Updating from 1.1.3");
+    debug!("attempting to load v2 config...");
     let old_cfg = if let Some(prev_config) = prev_config {
         prev_config
     } else {
-        ConfigV2::read_from_path(&path)?
+        ConfigV2::read_from_path(&path).inspect_err(|err| debug!("failed: {err}"))?
     };
 
     let authenticator_paths = AuthenticatorPathsV3::new(

nym-node/src/config/old_configs/old_config_v3.rs

@@ -17,6 +17,7 @@ use old_configs::old_config_v4::*;
 use persistence::*;
 use rand::rngs::OsRng;
 use serde::{Deserialize, Serialize};
+use tracing::instrument;
 
 #[derive(Debug, Clone, Deserialize, PartialEq, Eq, Serialize)]
 #[serde(deny_unknown_fields)]
@@ -938,15 +939,16 @@ pub async fn initialise(
     Ok(())
 }
 
+#[instrument(skip_all)]
 pub async fn try_upgrade_config_v3<P: AsRef<Path>>(
     path: P,
     prev_config: Option<ConfigV3>,
 ) -> Result<ConfigV4, NymNodeError> {
-    tracing::debug!("Updating from 1.1.4");
+    debug!("attempting to load v3 config...");
     let old_cfg = if let Some(prev_config) = prev_config {
         prev_config
     } else {
-        ConfigV3::read_from_path(&path)?
+        ConfigV3::read_from_path(&path).inspect_err(|err| debug!("failed: {err}"))?
     };
 
     let exit_gateway_paths = ExitGatewayPaths::new(

nym-node/src/config/old_configs/old_config_v4.rs

@@ -20,6 +20,7 @@ use old_configs::old_config_v5::*;
 use persistence::*;
 use rand::rngs::OsRng;
 use serde::{Deserialize, Serialize};
+use tracing::instrument;
 
 #[derive(Debug, Clone, Deserialize, PartialEq, Eq, Serialize)]
 #[serde(deny_unknown_fields)]
@@ -203,7 +204,7 @@ pub struct MixnetV4 {
     /// If applicable, custom port announced in the self-described API that other clients and nodes
     /// will use.
     /// Useful when the node is behind a proxy.
-    #[serde(deserialize_with = "de_maybe_port")]
+    #[serde(deserialize_with = "de_maybe_port", default)]
     pub announce_port: Option<u16>,
 
     /// Addresses to nym APIs from which the node gets the view of the network.
@@ -385,7 +386,7 @@ pub struct VerlocV4 {
     /// default: `0.0.0.0:1790`
     pub bind_address: SocketAddr,
 
-    #[serde(deserialize_with = "de_maybe_port")]
+    #[serde(deserialize_with = "de_maybe_port", default)]
     pub announce_port: Option<u16>,
 
     #[serde(default)]
@@ -1060,15 +1061,16 @@ pub async fn initialise(
     Ok(())
 }
 
+#[instrument(skip_all)]
 pub async fn try_upgrade_config_v4<P: AsRef<Path>>(
     path: P,
     prev_config: Option<ConfigV4>,
 ) -> Result<ConfigV5, NymNodeError> {
-    tracing::debug!("Updating from 1.1.5");
+    debug!("attempting to load v4 config...");
     let old_cfg = if let Some(prev_config) = prev_config {
         prev_config
     } else {
-        ConfigV4::read_from_path(&path)?
+        ConfigV4::read_from_path(&path).inspect_err(|err| debug!("failed: {err}"))?
     };
 
     let exit_gateway_paths = ExitGatewayPaths::new(