chore: use updated sphinx packets (client-side)

Display error messages if IPv4 or IPv6 address not found on nymtun0

Cargo.lock

@@ -702,7 +702,7 @@ checksum = "33415e24172c1b7d6066f6d999545375ab8e1d95421d6784bdfff9496f292387"
 dependencies = [
  "bitcoin_hashes",
  "rand 0.8.5",
- "rand_core 0.5.1",
+ "rand_core 0.6.4",
  "serde",
  "unicode-normalization",
  "zeroize",
@@ -4190,9 +4190,9 @@ checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e"
 
 [[package]]
 name = "log"
-version = "0.4.25"
+version = "0.4.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f"
+checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e"
 
 [[package]]
 name = "loom"
@@ -5940,6 +5940,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_yaml",
+ "subtle 2.6.1",
  "tower 0.5.2",
  "tracing",
  "utoipa",
@@ -6365,16 +6366,18 @@ dependencies = [
 
 [[package]]
 name = "nym-node-status-api"
-version = "1.0.0-rc.8"
+version = "1.0.2"
 dependencies = [
  "ammonia",
  "anyhow",
  "axum 0.7.9",
+ "bip39",
  "chrono",
  "clap",
  "cosmwasm-std",
  "envy",
  "futures-util",
+ "itertools 0.13.0",
  "moka",
  "nym-bin-common",
  "nym-contracts-common",
@@ -6388,6 +6391,8 @@ dependencies = [
  "nym-statistics-common",
  "nym-task",
  "nym-validator-client",
+ "rand 0.8.5",
+ "rand_chacha 0.3.1",
  "regex",
  "reqwest 0.12.4",
  "serde",
@@ -8209,7 +8214,7 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf2890aaef0aa82719a50e808de264f9484b74b442e1a3a0e5ee38243ac40bdb"
 dependencies = [
- "rand_core 0.5.1",
+ "rand_core 0.6.4",
 ]
 
 [[package]]
@@ -9967,9 +9972,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
 
 [[package]]
 name = "tar"
-version = "0.4.43"
+version = "0.4.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c65998313f8e17d0d553d28f91a0df93e4dbbbf770279c7bc21ca0f09ea1a1f6"
+checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a"
 dependencies = [
  "filetime",
  "libc",

Cargo.toml

@@ -322,7 +322,7 @@ serde_with = "3.9.0"
 serde_yaml = "0.9.25"
 sha2 = "0.10.8"
 si-scale = "0.2.3"
-sphinx-packet = "0.3.1"
+sphinx-packet = "=0.3.1" # make sure to use version below 0.4.0 that supports processing legacy and updated packets
 sqlx = "0.7.4"
 strum = "0.26"
 strum_macros = "0.26"
@@ -330,7 +330,7 @@ subtle-encoding = "0.5"
 syn = "1"
 sysinfo = "0.33.0"
 tap = "1.0.1"
-tar = "0.4.43"
+tar = "0.4.44"
 tempfile = "3.15"
 thiserror = "2.0"
 time = "0.3.37"

common/client-libs/gateway-client/src/client/mod.rs

@@ -204,15 +204,15 @@ impl<C, St> GatewayClient<C, St> {
             "Attemting to establish connection to gateway at: {}",
             self.gateway_address
         );
-        let (ws_stream, _) = connect_async(&self.gateway_address).await?;
+        let (ws_stream, _) = connect_async(
+            &self.gateway_address,
+            #[cfg(unix)]
+            self.connection_fd_callback.clone(),
+        )
+        .await?;
 
         self.connection = SocketState::Available(Box::new(ws_stream));
 
-        #[cfg(unix)]
-        if let (Some(callback), Some(fd)) = (self.connection_fd_callback.as_ref(), self.ws_fd()) {
-            callback.as_ref()(fd);
-        }
-
         Ok(())
     }
 

common/client-libs/gateway-client/src/client/websockets.rs

@@ -1,6 +1,11 @@
 use crate::error::GatewayClientError;
 
 use nym_http_api_client::HickoryDnsResolver;
+#[cfg(unix)]
+use std::{
+    os::fd::{AsRawFd, RawFd},
+    sync::Arc,
+};
 use tokio::net::TcpStream;
 use tokio_tungstenite::{MaybeTlsStream, WebSocketStream};
 use tungstenite::handshake::client::Response;
@@ -11,7 +16,10 @@ use std::net::SocketAddr;
 #[cfg(not(target_arch = "wasm32"))]
 pub(crate) async fn connect_async(
     endpoint: &str,
+    #[cfg(unix)] connection_fd_callback: Option<Arc<dyn Fn(RawFd) + Send + Sync>>,
 ) -> Result<(WebSocketStream<MaybeTlsStream<TcpStream>>, Response), GatewayClientError> {
+    use tokio::net::TcpSocket;
+
     let resolver = HickoryDnsResolver::default();
     let uri =
         Url::parse(endpoint).map_err(|_| GatewayClientError::InvalidUrl(endpoint.to_owned()))?;
@@ -37,14 +45,41 @@ pub(crate) async fn connect_async(
         }
     };
 
-    let stream = TcpStream::connect(&sock_addrs[..]).await.map_err(|error| {
-        GatewayClientError::NetworkConnectionFailed {
-            address: endpoint.to_owned(),
-            source: error.into(),
+    let mut stream = Err(GatewayClientError::NoEndpointForConnection {
+        address: endpoint.to_owned(),
+    });
+    for sock_addr in sock_addrs {
+        let socket = if sock_addr.is_ipv4() {
+            TcpSocket::new_v4()
+        } else {
+            TcpSocket::new_v6()
         }
-    })?;
+        .map_err(|err| GatewayClientError::NetworkConnectionFailed {
+            address: endpoint.to_owned(),
+            source: err.into(),
+        })?;
 
-    tokio_tungstenite::client_async_tls(endpoint, stream)
+        #[cfg(unix)]
+        if let Some(callback) = connection_fd_callback.as_ref() {
+            callback.as_ref()(socket.as_raw_fd());
+        }
+
+        match socket.connect(sock_addr).await {
+            Ok(s) => {
+                stream = Ok(s);
+                break;
+            }
+            Err(err) => {
+                stream = Err(GatewayClientError::NetworkConnectionFailed {
+                    address: endpoint.to_owned(),
+                    source: err.into(),
+                });
+                continue;
+            }
+        }
+    }
+
+    tokio_tungstenite::client_async_tls(endpoint, stream?)
         .await
         .map_err(|error| GatewayClientError::NetworkConnectionFailed {
             address: endpoint.to_owned(),

common/client-libs/gateway-client/src/error.rs

@@ -43,6 +43,9 @@ pub enum GatewayClientError {
     #[error("connection failed: {address}: {source}")]
     NetworkConnectionFailed { address: String, source: WsError },
 
+    #[error("no socket address for endpoint: {address}")]
+    NoEndpointForConnection { address: String },
+
     #[error("Invalid URL: {0}")]
     InvalidUrl(String),
 

common/http-api-common/Cargo.toml

@@ -20,6 +20,7 @@ mime = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 serde_json.workspace = true
 serde_yaml = { workspace = true }
+subtle.workspace = true
 tower = { workspace = true }
 tracing.workspace = true
 utoipa = { workspace = true, optional = true }

common/http-api-common/src/middleware/bearer_auth.rs

@@ -7,6 +7,7 @@ use axum::{extract::Request, response::Response};
 use futures::future::BoxFuture;
 use std::sync::Arc;
 use std::task::{Context, Poll};
+use subtle::ConstantTimeEq;
 use tower::{Layer, Service};
 use tracing::{debug, instrument, trace};
 use zeroize::Zeroizing;
@@ -76,7 +77,7 @@ impl<S> RequireAuth<S> {
             return Err("`Authorization` header must contain non-empty `Bearer` token");
         }
 
-        if self.bearer_token.as_str() != bearer_token {
+        if bool::from(self.bearer_token.as_bytes().ct_ne(bearer_token.as_bytes())) {
             return Err("`Authorization` header does not contain the correct `Bearer` token");
         }
 

common/nymsphinx/types/src/lib.rs

@@ -30,7 +30,6 @@ pub use sphinx_packet::{
     route::{Destination, DestinationAddressBytes, Node, NodeAddressBytes, SURBIdentifier},
     surb::{SURBMaterial, SURB},
     version::Version,
-    version::UPDATED_LEGACY_VERSION,
     Error as SphinxError, ProcessedPacket, ProcessedPacketData,
 };
 
@@ -91,12 +90,8 @@ impl NymPacket {
         destination: &Destination,
         delays: &[Delay],
     ) -> Result<NymPacket, NymPacketError> {
-        // FIXME:
-        // for now explicitly use the legacy version until sufficient number of nodes
-        // understand both variants
         Ok(NymPacket::Sphinx(
             SphinxPacketBuilder::new()
-                .with_version(UPDATED_LEGACY_VERSION)
                 .with_payload_size(size)
                 .build_packet(message, route, destination, delays)?,
         ))

nym-node-status-api/nym-node-status-api/Cargo.toml

@@ -3,7 +3,7 @@
 
 [package]
 name = "nym-node-status-api"
-version = "1.0.0-rc.8"
+version = "1.0.2"
 authors.workspace = true
 repository.workspace = true
 homepage.workspace = true
@@ -16,11 +16,13 @@ rust-version.workspace = true
 ammonia = { workspace = true }
 anyhow = { workspace = true }
 axum = { workspace = true, features = ["tokio", "macros"] }
+bip39 = { workspace = true }
 chrono = { workspace = true }
 clap = { workspace = true, features = ["cargo", "derive", "env", "string"] }
 cosmwasm-std = { workspace = true }
 envy = { workspace = true }
 futures-util = { workspace = true }
+itertools = { workspace = true }
 moka = { workspace = true, features = ["future"] }
 nym-contracts-common = { path = "../../common/cosmwasm-smart-contracts/contracts-common" }
 nym-bin-common = { path = "../../common/bin-common", features = ["models"] }
@@ -33,6 +35,8 @@ nym-statistics-common = { path = "../../common/statistics" }
 nym-validator-client = { path = "../../common/client-libs/validator-client" }
 nym-task = { path = "../../common/task" }
 nym-node-requests = { path = "../../nym-node/nym-node-requests", features = ["openapi"] }
+rand = { workspace = true }
+rand_chacha = { workspace = true }
 regex = { workspace = true }
 reqwest = { workspace = true }
 serde = { workspace = true, features = ["derive"] }

nym-node-status-api/nym-node-status-api/src/cli/mod.rs

@@ -83,9 +83,6 @@ pub(crate) struct Cli {
         env = "NYM_NODE_STATUS_API_MAX_AGENT_COUNT"
     )]
     pub(crate) max_agent_count: i64,
-
-    #[clap(long, default_value = "", env = "NYM_NODE_STATUS_API_HM_URL")]
-    pub(crate) hm_url: String,
 }
 
 fn parse_duration(arg: &str) -> Result<std::time::Duration, std::num::ParseIntError> {

nym-node-status-api/nym-node-status-api/src/db/models.rs

@@ -2,7 +2,7 @@ use std::str::FromStr;
 
 use crate::{
     http::{self, models::SummaryHistory},
-    monitor::NumericalCheckedCast,
+    utils::NumericalCheckedCast,
 };
 use anyhow::Context;
 use nym_contracts_common::Percent;
@@ -16,7 +16,7 @@ use strum_macros::{EnumString, FromRepr};
 use time::{Date, OffsetDateTime};
 use utoipa::ToSchema;
 
-pub(crate) struct GatewayRecord {
+pub(crate) struct GatewayInsertRecord {
     pub(crate) identity_key: String,
     pub(crate) bonded: bool,
     pub(crate) self_described: String,
@@ -360,14 +360,24 @@ impl TryFrom<GatewaySessionsRecord> for http::models::SessionStats {
     }
 }
 
-pub(crate) enum MixingNodeKind {
-    LegacyMixnode,
-    NymNode,
+pub(crate) enum ScrapeNodeKind {
+    LegacyMixnode { mix_id: i64 },
+    MixingNymNode { node_id: i64 },
+    EntryExitNymNode { node_id: i64, identity_key: String },
+}
+
+impl ScrapeNodeKind {
+    pub(crate) fn node_id(&self) -> &i64 {
+        match self {
+            ScrapeNodeKind::LegacyMixnode { mix_id } => mix_id,
+            ScrapeNodeKind::MixingNymNode { node_id } => node_id,
+            ScrapeNodeKind::EntryExitNymNode { node_id, .. } => node_id,
+        }
+    }
 }
 
 pub(crate) struct ScraperNodeInfo {
-    pub node_id: i64,
-    pub node_kind: MixingNodeKind,
+    pub node_kind: ScrapeNodeKind,
     pub hosts: Vec<String>,
     pub http_api_port: i64,
 }
@@ -390,6 +400,10 @@ impl ScraperNodeInfo {
 
         urls
     }
+
+    pub(crate) fn node_id(&self) -> &i64 {
+        self.node_kind.node_id()
+    }
 }
 
 #[derive(sqlx::Decode, Debug)]

nym-node-status-api/nym-node-status-api/src/db/queries/gateways.rs

@@ -1,6 +1,8 @@
+use std::collections::HashSet;
+
 use crate::{
     db::{
-        models::{GatewayDto, GatewayRecord},
+        models::{GatewayDto, GatewayInsertRecord},
         DbPool,
     },
     http::models::Gateway,
@@ -30,7 +32,7 @@ pub(crate) async fn select_gateway_identity(
 
 pub(crate) async fn insert_gateways(
     pool: &DbPool,
-    gateways: Vec<GatewayRecord>,
+    gateways: Vec<GatewayInsertRecord>,
 ) -> anyhow::Result<()> {
     let mut db = pool.acquire().await?;
     for record in gateways {
@@ -98,3 +100,21 @@ pub(crate) async fn get_all_gateways(pool: &DbPool) -> anyhow::Result<Vec<Gatewa
     tracing::trace!("Fetched {} gateways from DB", items.len());
     Ok(items)
 }
+
+pub(crate) async fn get_all_gateway_id_keys(pool: &DbPool) -> anyhow::Result<HashSet<String>> {
+    let mut conn = pool.acquire().await?;
+    let items = sqlx::query!(
+        r#"
+            SELECT gateway_identity_key
+            FROM gateways
+            WHERE bonded = true
+        "#
+    )
+    .fetch_all(&mut *conn)
+    .await?
+    .into_iter()
+    .map(|record| record.gateway_identity_key)
+    .collect::<HashSet<_>>();
+
+    Ok(items)
+}

nym-node-status-api/nym-node-status-api/src/db/queries/mixnodes.rs

@@ -1,3 +1,5 @@
+use std::collections::HashSet;
+
 use futures_util::TryStreamExt;
 use tracing::error;
 
@@ -83,8 +85,7 @@ pub(crate) async fn get_all_mixnodes(pool: &DbPool) -> anyhow::Result<Vec<Mixnod
     Ok(items)
 }
 
-/// `offset` = slides our fixed-day period further into the past by N days
-pub(crate) async fn get_daily_stats(pool: &DbPool, offset: i64) -> anyhow::Result<Vec<DailyStats>> {
+pub(crate) async fn get_daily_stats(pool: &DbPool) -> anyhow::Result<Vec<DailyStats>> {
     let mut conn = pool.acquire().await?;
     let items = sqlx::query_as!(
         DailyStats,
@@ -115,11 +116,8 @@ pub(crate) async fn get_daily_stats(pool: &DbPool, offset: i64) -> anyhow::Resul
             WHERE nym_node_daily_mixing_stats.node_id IS NULL
         )
         GROUP BY date_utc
-        ORDER BY date_utc DESC
-        LIMIT 30
-        OFFSET ?
+        ORDER BY date_utc ASC
         "#,
-        offset
     )
     .fetch(&mut *conn)
     .try_collect::<Vec<DailyStats>>()
@@ -127,3 +125,21 @@ pub(crate) async fn get_daily_stats(pool: &DbPool, offset: i64) -> anyhow::Resul
 
     Ok(items)
 }
+
+pub(crate) async fn get_all_mix_ids(pool: &DbPool) -> anyhow::Result<HashSet<i64>> {
+    let mut conn = pool.acquire().await?;
+    let items = sqlx::query!(
+        r#"
+            SELECT mix_id
+            FROM mixnodes
+            WHERE bonded = true
+        "#
+    )
+    .fetch_all(&mut *conn)
+    .await?
+    .into_iter()
+    .map(|record| record.mix_id)
+    .collect::<HashSet<_>>();
+
+    Ok(items)
+}

nym-node-status-api/nym-node-status-api/src/db/queries/mod.rs

@@ -8,13 +8,15 @@ pub(crate) mod scraper;
 mod summary;
 pub(crate) mod testruns;
 
-pub(crate) use gateways::{get_all_gateways, insert_gateways, select_gateway_identity};
+pub(crate) use gateways::{
+    get_all_gateway_id_keys, get_all_gateways, insert_gateways, select_gateway_identity,
+};
 pub(crate) use gateways_stats::{delete_old_records, get_sessions_stats, insert_session_records};
 pub(crate) use misc::insert_summaries;
-pub(crate) use mixnodes::{get_all_mixnodes, get_daily_stats, insert_mixnodes};
+pub(crate) use mixnodes::{get_all_mix_ids, get_all_mixnodes, get_daily_stats, insert_mixnodes};
 pub(crate) use nym_nodes::{get_nym_nodes, insert_nym_nodes};
 pub(crate) use packet_stats::{
     get_raw_node_stats, insert_daily_node_stats, insert_node_packet_stats,
 };
-pub(crate) use scraper::{get_mixing_nodes_for_scraping, insert_scraped_node_description};
+pub(crate) use scraper::{get_nodes_for_scraping, insert_scraped_node_description};
 pub(crate) use summary::{get_summary, get_summary_history};

nym-node-status-api/nym-node-status-api/src/db/queries/nym_nodes.rs

@@ -1,5 +1,6 @@
 use std::collections::HashMap;
 
+use anyhow::Context;
 use futures_util::TryStreamExt;
 use nym_validator_client::{client::NymNodeDetails, nym_api::SkimmedNode};
 use tracing::instrument;
@@ -9,7 +10,7 @@ use crate::{
         models::{NymNodeDto, NymNodeInsertRecord},
         DbPool,
     },
-    monitor::decimal_to_i64,
+    utils::decimal_to_i64,
 };
 
 pub(crate) async fn get_nym_nodes(pool: &DbPool) -> anyhow::Result<Vec<SkimmedNode>> {
@@ -100,7 +101,8 @@ pub(crate) async fn insert_nym_nodes(
             record.last_updated_utc,
         )
         .execute(&mut *conn)
-        .await?;
+        .await
+        .with_context(|| format!("node_id={}", record.node_id))?;
     }
 
     Ok(())

nym-node-status-api/nym-node-status-api/src/db/queries/packet_stats.rs

@@ -1,27 +1,26 @@
 use crate::db::{
-    models::{MixingNodeKind, NodeStats, ScraperNodeInfo},
+    models::{NodeStats, ScrapeNodeKind, ScraperNodeInfo},
     DbPool,
 };
 use anyhow::Result;
 
 pub(crate) async fn insert_node_packet_stats(
     pool: &DbPool,
-    node_id: i64,
-    node_kind: &MixingNodeKind,
+    node_kind: &ScrapeNodeKind,
     stats: &NodeStats,
     timestamp_utc: i64,
 ) -> Result<()> {
     let mut conn = pool.acquire().await?;
 
     match node_kind {
-        MixingNodeKind::LegacyMixnode => {
+        ScrapeNodeKind::LegacyMixnode { mix_id } => {
             sqlx::query!(
                 r#"
                 INSERT INTO mixnode_packet_stats_raw (
                     mix_id, timestamp_utc, packets_received, packets_sent, packets_dropped
                 ) VALUES (?, ?, ?, ?, ?)
                 "#,
-                node_id,
+                mix_id,
                 timestamp_utc,
                 stats.packets_received,
                 stats.packets_sent,
@@ -30,7 +29,8 @@ pub(crate) async fn insert_node_packet_stats(
             .execute(&mut *conn)
             .await?;
         }
-        MixingNodeKind::NymNode => {
+        ScrapeNodeKind::MixingNymNode { node_id }
+        | ScrapeNodeKind::EntryExitNymNode { node_id, .. } => {
             sqlx::query!(
                 r#"
                 INSERT INTO nym_nodes_packet_stats_raw (
@@ -60,7 +60,7 @@ pub(crate) async fn get_raw_node_stats(
     let packets = match node.node_kind {
         // if no packets are found, it's fine to assume 0 because that's also
         // SQL default value if none provided
-        MixingNodeKind::LegacyMixnode => {
+        ScrapeNodeKind::LegacyMixnode { mix_id } => {
             sqlx::query_as!(
                 NodeStats,
                 r#"
@@ -73,12 +73,13 @@ pub(crate) async fn get_raw_node_stats(
                 ORDER BY timestamp_utc DESC
                 LIMIT 1 OFFSET 1
                 "#,
-                node.node_id
+                mix_id
             )
             .fetch_optional(&mut *conn)
             .await?
         }
-        MixingNodeKind::NymNode => {
+        ScrapeNodeKind::MixingNymNode { node_id }
+        | ScrapeNodeKind::EntryExitNymNode { node_id, .. } => {
             sqlx::query_as!(
                 NodeStats,
                 r#"
@@ -91,7 +92,7 @@ pub(crate) async fn get_raw_node_stats(
                 ORDER BY timestamp_utc DESC
                 LIMIT 1 OFFSET 1
                 "#,
-                node.node_id
+                node_id
             )
             .fetch_optional(&mut *conn)
             .await?
@@ -110,7 +111,7 @@ pub(crate) async fn insert_daily_node_stats(
     let mut conn = pool.acquire().await?;
 
     match node.node_kind {
-        MixingNodeKind::LegacyMixnode => {
+        ScrapeNodeKind::LegacyMixnode { mix_id } => {
             let total_stake = sqlx::query_scalar!(
                 r#"
                     SELECT
@@ -118,7 +119,7 @@ pub(crate) async fn insert_daily_node_stats(
                     FROM mixnodes
                     WHERE mix_id = ?
                    "#,
-                node.node_id
+                mix_id
             )
             .fetch_one(&mut *conn)
             .await?;
@@ -136,7 +137,7 @@ pub(crate) async fn insert_daily_node_stats(
                     packets_sent = mixnode_daily_stats.packets_sent + excluded.packets_sent,
                     packets_dropped = mixnode_daily_stats.packets_dropped + excluded.packets_dropped
                 "#,
-                node.node_id,
+                mix_id,
                 date_utc,
                 total_stake,
                 packets.packets_received,
@@ -146,7 +147,8 @@ pub(crate) async fn insert_daily_node_stats(
             .execute(&mut *conn)
             .await?;
         }
-        MixingNodeKind::NymNode => {
+        ScrapeNodeKind::MixingNymNode { node_id }
+        | ScrapeNodeKind::EntryExitNymNode { node_id, .. } => {
             let total_stake = sqlx::query_scalar!(
                 r#"
                 SELECT
@@ -154,7 +156,7 @@ pub(crate) async fn insert_daily_node_stats(
                 FROM nym_nodes
                 WHERE node_id = ?
                 "#,
-                node.node_id
+                node_id
             )
             .fetch_one(&mut *conn)
             .await?;
@@ -167,12 +169,12 @@ pub(crate) async fn insert_daily_node_stats(
                     packets_sent, packets_dropped
                 ) VALUES (?, ?, ?, ?, ?, ?)
                 ON CONFLICT(node_id, date_utc) DO UPDATE SET
-                    total_stake = nym_node_daily_mixing_stats.total_stake + excluded.total_stake,
+                    total_stake = excluded.total_stake,
                     packets_received = nym_node_daily_mixing_stats.packets_received + excluded.packets_received,
                     packets_sent = nym_node_daily_mixing_stats.packets_sent + excluded.packets_sent,
                     packets_dropped = nym_node_daily_mixing_stats.packets_dropped + excluded.packets_dropped
                 "#,
-                node.node_id,
+                node_id,
                 date_utc,
                 total_stake,
                 packets.packets_received,

nym-node-status-api/nym-node-status-api/src/db/queries/scraper.rs

@@ -1,6 +1,6 @@
 use crate::{
     db::{
-        models::{MixingNodeKind, ScraperNodeInfo},
+        models::{ScrapeNodeKind, ScraperNodeInfo},
         queries, DbPool,
     },
     mixnet_scraper::helpers::NodeDescriptionResponse,
@@ -8,16 +8,36 @@ use crate::{
 use anyhow::Result;
 use chrono::Utc;
 
-pub(crate) async fn get_mixing_nodes_for_scraping(pool: &DbPool) -> Result<Vec<ScraperNodeInfo>> {
+pub(crate) async fn get_nodes_for_scraping(pool: &DbPool) -> Result<Vec<ScraperNodeInfo>> {
     let mut nodes_to_scrape = Vec::new();
 
+    let mixnode_ids = queries::get_all_mix_ids(pool).await?;
+    let gateway_keys = queries::get_all_gateway_id_keys(pool).await?;
+
+    let mut entry_exit_nodes = 0;
     queries::get_nym_nodes(pool)
         .await?
         .into_iter()
         .for_each(|node| {
+            // due to polyfilling, Nym nodes table might contain legacy mixnodes
+            // as well. Mark them as such here.
+            let node_kind = if mixnode_ids.contains(&node.node_id.into()) {
+                ScrapeNodeKind::LegacyMixnode {
+                    mix_id: node.node_id.into(),
+                }
+            } else if gateway_keys.contains(&node.ed25519_identity_pubkey.to_base58_string()) {
+                entry_exit_nodes += 1;
+                ScrapeNodeKind::EntryExitNymNode {
+                    node_id: node.node_id.into(),
+                    identity_key: node.ed25519_identity_pubkey.to_base58_string(),
+                }
+            } else {
+                ScrapeNodeKind::MixingNymNode {
+                    node_id: node.node_id.into(),
+                }
+            };
             nodes_to_scrape.push(ScraperNodeInfo {
-                node_id: node.node_id.into(),
-                node_kind: MixingNodeKind::NymNode,
+                node_kind,
                 hosts: node
                     .ip_addresses
                     .into_iter()
@@ -27,7 +47,8 @@ pub(crate) async fn get_mixing_nodes_for_scraping(pool: &DbPool) -> Result<Vec<S
             })
         });
 
-    tracing::debug!("Fetched {} 🌟 nym nodes", nodes_to_scrape.len());
+    tracing::debug!("Fetched {} 🌟 total nym nodes", nodes_to_scrape.len());
+    tracing::debug!("Fetched {} 🚪 entry/exit nodes", entry_exit_nodes);
 
     let mut conn = pool.acquire().await?;
     let mixnodes = sqlx::query!(
@@ -41,7 +62,7 @@ pub(crate) async fn get_mixing_nodes_for_scraping(pool: &DbPool) -> Result<Vec<S
     .await?;
     drop(conn);
 
-    tracing::debug!("Fetched {} 🦖 mixnodes", nodes_to_scrape.len());
+    tracing::debug!("Fetched {} 🦖 mixnodes", mixnodes.len());
 
     let mut duplicates = 0;
     let mut legacy_not_in_nym_node_list = 0;
@@ -49,26 +70,22 @@ pub(crate) async fn get_mixing_nodes_for_scraping(pool: &DbPool) -> Result<Vec<S
     for mixnode in mixnodes {
         if nodes_to_scrape
             .iter()
-            .all(|node| node.node_id != mixnode.node_id)
+            .all(|node| node.node_id() != &mixnode.node_id)
         {
+            // in case polyfilling on Nym API gets removed, this part ensures
+            // mixnodes are added to the final list of nodes to scrape
+            nodes_to_scrape.push(ScraperNodeInfo {
+                node_kind: ScrapeNodeKind::LegacyMixnode {
+                    mix_id: mixnode.node_id,
+                },
+                hosts: vec![mixnode.host],
+                http_api_port: mixnode.http_api_port,
+            });
+
             legacy_not_in_nym_node_list += 1;
         } else {
             duplicates += 1;
         }
-
-        // technically, mixnodes shouldn't be in nym_nodes table, but it's
-        // possible due to polyfilling on Nym API
-        if nodes_to_scrape
-            .iter()
-            .all(|node| node.node_id != mixnode.node_id)
-        {
-            nodes_to_scrape.push(ScraperNodeInfo {
-                node_id: mixnode.node_id,
-                node_kind: MixingNodeKind::LegacyMixnode,
-                hosts: vec![mixnode.host],
-                http_api_port: mixnode.http_api_port,
-            })
-        }
     }
     tracing::debug!(
         "{}/{} legacy mixnodes already included in nym_node list",
@@ -85,19 +102,16 @@ pub(crate) async fn get_mixing_nodes_for_scraping(pool: &DbPool) -> Result<Vec<S
     Ok(nodes_to_scrape)
 }
 
-// TODO: add stuff for gateways
-
 pub(crate) async fn insert_scraped_node_description(
     pool: &DbPool,
-    node_kind: &MixingNodeKind,
-    node_id: i64,
+    node_kind: &ScrapeNodeKind,
     description: &NodeDescriptionResponse,
 ) -> Result<()> {
     let timestamp = Utc::now().timestamp();
     let mut conn = pool.acquire().await?;
 
     match node_kind {
-        MixingNodeKind::LegacyMixnode => {
+        ScrapeNodeKind::LegacyMixnode { mix_id } => {
             sqlx::query!(
                 r#"
                 INSERT INTO mixnode_description (
@@ -110,7 +124,7 @@ pub(crate) async fn insert_scraped_node_description(
                     details = excluded.details,
                     last_updated_utc = excluded.last_updated_utc
                 "#,
-                node_id,
+                mix_id,
                 description.moniker,
                 description.website,
                 description.security_contact,
@@ -120,7 +134,7 @@ pub(crate) async fn insert_scraped_node_description(
             .execute(&mut *conn)
             .await?;
         }
-        MixingNodeKind::NymNode => {
+        ScrapeNodeKind::MixingNymNode { node_id } => {
             sqlx::query!(
                 r#"
                 INSERT INTO nym_node_descriptions (
@@ -143,6 +157,34 @@ pub(crate) async fn insert_scraped_node_description(
             .execute(&mut *conn)
             .await?;
         }
+        ScrapeNodeKind::EntryExitNymNode { identity_key, .. } => {
+            sqlx::query!(
+                r#"
+            INSERT INTO gateway_description (
+                gateway_identity_key,
+                moniker,
+                website,
+                security_contact,
+                details,
+                last_updated_utc
+            ) VALUES (?, ?, ?, ?, ?, ?)
+            ON CONFLICT (gateway_identity_key) DO UPDATE SET
+                moniker = excluded.moniker,
+                website = excluded.website,
+                security_contact = excluded.security_contact,
+                details = excluded.details,
+                last_updated_utc = excluded.last_updated_utc
+            "#,
+                identity_key,
+                description.moniker,
+                description.website,
+                description.security_contact,
+                description.details,
+                timestamp,
+            )
+            .execute(&mut *conn)
+            .await?;
+        }
     }
 
     Ok(())

nym-node-status-api/nym-node-status-api/src/http/api/mixnodes.rs

@@ -99,7 +99,10 @@ async fn get_stats(
     Query(MixStatsQueryParams { offset }): Query<MixStatsQueryParams>,
     State(state): State<AppState>,
 ) -> HttpResult<Json<Vec<DailyStats>>> {
-    let offset = offset.unwrap_or(0);
+    let offset: usize = offset
+        .unwrap_or(0)
+        .try_into()
+        .map_err(|_| HttpError::invalid_input("Offset must be non-negative"))?;
     let last_30_days = state
         .cache()
         .get_mixnode_stats(state.db_pool(), offset)

nym-node-status-api/nym-node-status-api/src/http/server.rs

@@ -17,18 +17,10 @@ pub(crate) async fn start_http_api(
     nym_http_cache_ttl: u64,
     agent_key_list: Vec<PublicKey>,
     agent_max_count: i64,
-    hm_url: String,
 ) -> anyhow::Result<ShutdownHandles> {
     let router_builder = RouterBuilder::with_default_routes();
 
-    let state = AppState::new(
-        db_pool,
-        nym_http_cache_ttl,
-        agent_key_list,
-        agent_max_count,
-        hm_url,
-    )
-    .await;
+    let state = AppState::new(db_pool, nym_http_cache_ttl, agent_key_list, agent_max_count).await;
     let router = router_builder.with_state(state);
 
     let bind_addr = format!("0.0.0.0:{}", http_port);

nym-node-status-api/nym-node-status-api/src/http/state.rs

@@ -25,11 +25,10 @@ impl AppState {
         cache_ttl: u64,
         agent_key_list: Vec<PublicKey>,
         agent_max_count: i64,
-        hm_url: String,
     ) -> Self {
         Self {
             db_pool,
-            cache: HttpCache::new(cache_ttl, hm_url).await,
+            cache: HttpCache::new(cache_ttl).await,
             agent_key_list,
             agent_max_count,
         }
@@ -52,96 +51,14 @@ impl AppState {
     }
 }
 
-#[derive(Debug, Clone)]
-struct HistoricMixingStats {
-    historic_stats: Vec<DailyStats>,
-}
-
-impl HistoricMixingStats {
-    /// Collect historic stats only on initialization. From this point onwards,
-    /// service will collect its own stats
-    async fn init(hm_url: String) -> Self {
-        tracing::info!("Fetching historic mixnode stats from {}", hm_url);
-
-        let target_url = format!("{}/v2/mixnodes/stats", hm_url);
-        if let Ok(response) = reqwest::get(&target_url)
-            .await
-            .and_then(|res| res.error_for_status())
-            .inspect_err(|err| tracing::error!("Failed to fetch cache from HM: {}", err))
-        {
-            if let Ok(mut daily_stats) = response.json::<Vec<DailyStats>>().await {
-                // sorting required for seamless comparison later (descending, newest first)
-                daily_stats.sort_by(|left, right| right.date_utc.cmp(&left.date_utc));
-
-                tracing::info!(
-                    "Successfully fetched {} historic entries from {}",
-                    daily_stats.len(),
-                    hm_url
-                );
-                return Self {
-                    historic_stats: daily_stats,
-                };
-            }
-        };
-
-        tracing::warn!("Failed to get historic daily stats from {}", hm_url);
-        Self {
-            historic_stats: Vec::new(),
-        }
-    }
-
-    /// polyfill with historical data obtained from Harbour Master
-    fn merge_with_historic_stats(&self, mut new_stats: Vec<DailyStats>) -> Vec<DailyStats> {
-        // newest first
-        new_stats.sort_by(|left, right| right.date_utc.cmp(&left.date_utc));
-
-        // historic stats are only used for dates when we don't have new data
-        let oldest_date_in_new_stats = new_stats
-            .last()
-            .map(|day| day.date_utc.to_owned())
-            .unwrap_or(String::from("1900-01-01"));
-
-        // given 2 arrays
-        // index    historic_stats      new_stats
-        //   0        30-01               31-01
-        //   1        29-01               30-01
-        //   2        28-01
-        //            ...
-        //   N        01-01
-        // cutoff point would be at historic_stats[1]
-        // (first date smaller than oldest we've already got)
-        if let Some(cutoff) = self
-            .historic_stats
-            .iter()
-            .position(|elem| elem.date_utc < oldest_date_in_new_stats)
-        {
-            // missing data = (all historic data) - (however many days we already have)
-            let missing_data = self.historic_stats.iter().skip(cutoff).cloned();
-
-            // extend new data with missing days
-            tracing::debug!(
-                "Polyfilled with {} historic records from {:?} to {:?}",
-                missing_data.len(),
-                self.historic_stats.last(),
-                self.historic_stats.get(cutoff)
-            );
-            new_stats.extend(missing_data);
-
-            // oldest first
-            new_stats.into_iter().rev().collect::<Vec<_>>()
-        } else {
-            // if all historic data is older than what we've got, don't use it
-            new_stats
-        }
-    }
-}
-
 static GATEWAYS_LIST_KEY: &str = "gateways";
 static MIXNODES_LIST_KEY: &str = "mixnodes";
 static MIXSTATS_LIST_KEY: &str = "mixstats";
 static SUMMARY_HISTORY_LIST_KEY: &str = "summary-history";
 static SESSION_STATS_LIST_KEY: &str = "session-stats";
 
+const MIXNODE_STATS_HISTORY_DAYS: usize = 30;
+
 #[derive(Debug, Clone)]
 pub(crate) struct HttpCache {
     gateways: Cache<String, Arc<RwLock<Vec<Gateway>>>>,
@@ -149,11 +66,10 @@ pub(crate) struct HttpCache {
     mixstats: Cache<String, Arc<RwLock<Vec<DailyStats>>>>,
     history: Cache<String, Arc<RwLock<Vec<SummaryHistory>>>>,
     session_stats: Cache<String, Arc<RwLock<Vec<SessionStats>>>>,
-    mixnode_historic_daily_stats: HistoricMixingStats,
 }
 
 impl HttpCache {
-    pub async fn new(ttl_seconds: u64, hm_url: String) -> Self {
+    pub async fn new(ttl_seconds: u64) -> Self {
         HttpCache {
             gateways: Cache::builder()
                 .max_capacity(2)
@@ -175,7 +91,6 @@ impl HttpCache {
                 .max_capacity(2)
                 .time_to_live(Duration::from_secs(ttl_seconds))
                 .build(),
-            mixnode_historic_daily_stats: HistoricMixingStats::init(hm_url).await,
         }
     }
 
@@ -285,26 +200,27 @@ impl HttpCache {
             .await
     }
 
-    pub async fn get_mixnode_stats(&self, db: &DbPool, offset: i64) -> Vec<DailyStats> {
-        match self.mixstats.get(MIXSTATS_LIST_KEY).await {
+    pub async fn get_mixnode_stats(&self, db: &DbPool, offset: usize) -> Vec<DailyStats> {
+        let mut stats = match self.mixstats.get(MIXSTATS_LIST_KEY).await {
             Some(guard) => {
                 let read_lock = guard.read().await;
                 read_lock.to_vec()
             }
             None => {
-                let new_node_stats = crate::db::queries::get_daily_stats(db, offset)
+                let new_node_stats = crate::db::queries::get_daily_stats(db)
                     .await
-                    .unwrap_or_default();
-                // for every day that's missing, fill it with cached historic data
-                let mut mixnode_stats = self
-                    .mixnode_historic_daily_stats
-                    .merge_with_historic_stats(new_node_stats);
-                mixnode_stats.truncate(30);
-
-                self.upsert_mixnode_stats(mixnode_stats.clone()).await;
-                mixnode_stats
+                    .unwrap_or_default()
+                    .into_iter()
+                    .rev()
+                    .collect::<Vec<_>>();
+                // cache result without offset
+                self.upsert_mixnode_stats(new_node_stats.clone()).await;
+                new_node_stats
             }
-        }
+        };
+
+        stats.truncate(MIXNODE_STATS_HISTORY_DAYS + offset);
+        stats.into_iter().skip(offset).rev().collect()
     }
 
     pub async fn get_summary_history(&self, db: &DbPool) -> Vec<SummaryHistory> {

nym-node-status-api/nym-node-status-api/src/logging.rs

@@ -34,6 +34,8 @@ pub(crate) fn setup_tracing_logger() -> anyhow::Result<()> {
         "tower_http",
         "axum",
         "html5ever",
+        "hickory_proto",
+        "hickory_resolver",
     ];
     for crate_name in warn_crates {
         filter = filter.add_directive(directive_checked(format!("{}=warn", crate_name))?);

nym-node-status-api/nym-node-status-api/src/main.rs

@@ -10,6 +10,7 @@ mod mixnet_scraper;
 mod monitor;
 mod node_scraper;
 mod testruns;
+mod utils;
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
@@ -66,7 +67,6 @@ async fn main() -> anyhow::Result<()> {
         args.nym_http_cache_ttl,
         agent_key_list.to_owned(),
         args.max_agent_count,
-        args.hm_url,
     )
     .await
     .expect("Failed to start server");

nym-node-status-api/nym-node-status-api/src/mixnet_scraper/helpers.rs

@@ -1,12 +1,15 @@
-use crate::db::{
-    models::{NodeStats, ScraperNodeInfo},
-    queries::{
-        get_raw_node_stats, insert_daily_node_stats, insert_node_packet_stats,
-        insert_scraped_node_description,
+use crate::{
+    db::{
+        models::{NodeStats, ScraperNodeInfo},
+        queries::{
+            get_raw_node_stats, insert_daily_node_stats, insert_node_packet_stats,
+            insert_scraped_node_description,
+        },
     },
+    utils::generate_node_name,
 };
 use ammonia::Builder;
-use anyhow::Result;
+use anyhow::{anyhow, Result};
 use chrono::{DateTime, Datelike, Utc};
 use reqwest;
 use serde::{Deserialize, Serialize};
@@ -80,22 +83,33 @@ pub fn build_client() -> Result<reqwest::Client> {
         .map_err(|e| anyhow::anyhow!("Failed to build HTTP client: {}", e))
 }
 
-pub fn sanitize_description(description: NodeDescriptionResponse) -> NodeDescriptionResponse {
+pub fn sanitize_description(
+    description: NodeDescriptionResponse,
+    node_id: i64,
+) -> NodeDescriptionResponse {
     let mut sanitizer = Builder::new();
     sanitizer
         .tags(std::collections::HashSet::new())
         .generic_attributes(std::collections::HashSet::new())
         .url_schemes(std::collections::HashSet::new());
 
+    const UNKNOWN: &str = "N/A";
     let sanitize_field = |opt: Option<String>| -> Option<String> {
         Some(
             opt.filter(|s| !s.trim().is_empty())
-                .map_or_else(|| "N/A".to_string(), |s| sanitizer.clean(&s).to_string()),
+                .map_or_else(|| UNKNOWN.to_string(), |s| sanitizer.clean(&s).to_string()),
         )
     };
 
+    let mut moniker = sanitize_field(description.moniker);
+    if let Some(sanitized) = &moniker {
+        if sanitized == UNKNOWN {
+            moniker = Some(generate_node_name(node_id));
+        }
+    };
+
     NodeDescriptionResponse {
-        moniker: sanitize_field(description.moniker),
+        moniker,
         website: sanitize_field(description.website),
         security_contact: sanitize_field(description.security_contact),
         details: sanitize_field(description.details),
@@ -108,18 +122,26 @@ pub async fn scrape_and_store_description(pool: &SqlitePool, node: &ScraperNodeI
 
     let mut description = None;
     let mut error = None;
+    let mut tried_url_list = Vec::new();
 
     for mut url in urls {
         url = format!("{}{}", url.trim_end_matches('/'), DESCRIPTION_URL);
+        tried_url_list.push(url.clone());
 
-        match client.get(&url).send().await {
+        match client
+            .get(&url)
+            .send()
+            .await
+            // convert 404 and similar to error
+            .and_then(|res| res.error_for_status())
+        {
             Ok(response) => {
                 if let Ok(desc) = response.json::<NodeDescriptionResponse>().await {
                     description = Some(desc);
                     break;
                 }
             }
-            Err(e) => error = Some(e),
+            Err(e) => error = Some(anyhow!("{:?} ({})", tried_url_list, e)),
         }
     }
 
@@ -128,9 +150,8 @@ pub async fn scrape_and_store_description(pool: &SqlitePool, node: &ScraperNodeI
         anyhow::anyhow!("Failed to fetch description from any URL: {}", err_msg)
     })?;
 
-    let sanitized_description = sanitize_description(description);
-    insert_scraped_node_description(pool, &node.node_kind, node.node_id, &sanitized_description)
-        .await?;
+    let sanitized_description = sanitize_description(description, *node.node_id());
+    insert_scraped_node_description(pool, &node.node_kind, &sanitized_description).await?;
 
     Ok(())
 }
@@ -144,9 +165,11 @@ pub async fn scrape_and_store_packet_stats(
 
     let mut stats = None;
     let mut error = None;
+    let mut tried_url_list = Vec::new();
 
     for mut url in urls {
         url = format!("{}{}", url.trim_end_matches('/'), PACKET_STATS_URL);
+        tried_url_list.push(url.clone());
 
         match client.get(&url).send().await {
             Ok(response) => {
@@ -155,18 +178,18 @@ pub async fn scrape_and_store_packet_stats(
                     break;
                 }
             }
-            Err(e) => error = Some(e),
+            Err(e) => error = Some(anyhow!("{:?} ({})", tried_url_list, e)),
         }
     }
 
     let stats = stats.ok_or_else(|| {
         let err_msg = error.map_or_else(|| "Unknown error".to_string(), |e| e.to_string());
-        anyhow::anyhow!("Failed to fetch stats from any URL: {}", err_msg)
+        anyhow::anyhow!("Failed to fetch description from any URL: {}", err_msg)
     })?;
 
     let timestamp = Utc::now();
     let timestamp_utc = timestamp.timestamp();
-    insert_node_packet_stats(pool, node.node_id, &node.node_kind, &stats, timestamp_utc).await?;
+    insert_node_packet_stats(pool, &node.node_kind, &stats, timestamp_utc).await?;
 
     // Update daily stats
     update_daily_stats(pool, node, timestamp, &stats).await?;

nym-node-status-api/nym-node-status-api/src/mixnet_scraper/mod.rs

@@ -8,7 +8,7 @@ use sqlx::SqlitePool;
 use tracing::{debug, error, instrument, warn};
 
 use crate::db::models::ScraperNodeInfo;
-use crate::db::queries::get_mixing_nodes_for_scraping;
+use crate::db::queries::get_nodes_for_scraping;
 
 const DESCRIPTION_SCRAPE_INTERVAL: Duration = Duration::from_secs(60 * 60 * 4);
 const PACKET_SCRAPE_INTERVAL: Duration = Duration::from_secs(60 * 60);
@@ -74,7 +74,7 @@ impl Scraper {
         pool: &SqlitePool,
         queue: Arc<Mutex<Vec<ScraperNodeInfo>>>,
     ) -> Result<()> {
-        let nodes = get_mixing_nodes_for_scraping(pool).await?;
+        let nodes = get_nodes_for_scraping(pool).await?;
         if let Ok(mut queue_lock) = queue.lock() {
             queue_lock.extend(nodes);
         } else {
@@ -82,7 +82,7 @@ impl Scraper {
             return Ok(());
         }
 
-        Self::process_description_queue(pool, queue).await?;
+        Self::process_description_queue(pool, queue).await;
         Ok(())
     }
 
@@ -91,7 +91,7 @@ impl Scraper {
         pool: &SqlitePool,
         queue: Arc<Mutex<Vec<ScraperNodeInfo>>>,
     ) -> Result<()> {
-        let nodes = get_mixing_nodes_for_scraping(pool).await?;
+        let nodes = get_nodes_for_scraping(pool).await?;
         tracing::info!("Querying {} mixing nodes", nodes.len());
         if let Ok(mut queue_lock) = queue.lock() {
             queue_lock.extend(nodes);
@@ -100,14 +100,11 @@ impl Scraper {
             return Ok(());
         }
 
-        Self::process_packet_queue(pool, queue).await?;
+        Self::process_packet_queue(pool, queue).await;
         Ok(())
     }
 
-    async fn process_description_queue(
-        pool: &SqlitePool,
-        queue: Arc<Mutex<Vec<ScraperNodeInfo>>>,
-    ) -> Result<()> {
+    async fn process_description_queue(pool: &SqlitePool, queue: Arc<Mutex<Vec<ScraperNodeInfo>>>) {
         loop {
             let running_tasks = TASK_COUNTER.load(Ordering::Relaxed);
 
@@ -132,12 +129,15 @@ impl Scraper {
                 tokio::spawn(async move {
                     match scrape_and_store_description(&pool, &node).await {
                         Ok(_) => debug!(
-                            "✅ Description task #{} for node {} complete",
-                            task_id, node.node_id
+                            "📝 ✅ Description task #{} for node {} complete",
+                            task_id,
+                            node.node_id()
                         ),
                         Err(e) => debug!(
-                            "❌ Description task #{} for node {} failed: {}",
-                            task_id, node.node_id, e
+                            "📝 ❌ Description task #{} for node {} failed: {}",
+                            task_id,
+                            node.node_id(),
+                            e
                         ),
                     }
                     TASK_COUNTER.fetch_sub(1, Ordering::Relaxed);
@@ -146,13 +146,9 @@ impl Scraper {
                 tokio::time::sleep(QUEUE_CHECK_INTERVAL).await;
             }
         }
-        Ok(())
     }
 
-    async fn process_packet_queue(
-        pool: &SqlitePool,
-        queue: Arc<Mutex<Vec<ScraperNodeInfo>>>,
-    ) -> Result<()> {
+    async fn process_packet_queue(pool: &SqlitePool, queue: Arc<Mutex<Vec<ScraperNodeInfo>>>) {
         loop {
             let running_tasks = TASK_COUNTER.load(Ordering::Relaxed);
 
@@ -177,12 +173,15 @@ impl Scraper {
                 tokio::spawn(async move {
                     match scrape_and_store_packet_stats(&pool, &node).await {
                         Ok(_) => debug!(
-                            "✅ Packet stats task #{} for node {} complete",
-                            task_id, node.node_id
+                            "📊 ✅ Packet stats task #{} for node {} complete",
+                            task_id,
+                            node.node_id()
                         ),
                         Err(e) => debug!(
-                            "❌ Packet stats task #{} for node {} failed: {}",
-                            task_id, node.node_id, e
+                            "📊 ❌ Packet stats task #{} for node {} failed: {}",
+                            task_id,
+                            node.node_id(),
+                            e
                         ),
                     }
                     TASK_COUNTER.fetch_sub(1, Ordering::Relaxed);
@@ -191,6 +190,5 @@ impl Scraper {
                 tokio::time::sleep(QUEUE_CHECK_INTERVAL).await;
             }
         }
-        Ok(())
     }
 }

nym-node-status-api/nym-node-status-api/src/monitor/mod.rs

@@ -1,14 +1,14 @@
 #![allow(deprecated)]
 
 use crate::db::models::{
-    gateway, mixnode, GatewayRecord, MixnodeRecord, NetworkSummary, ASSIGNED_ENTRY_COUNT,
+    gateway, mixnode, GatewayInsertRecord, MixnodeRecord, NetworkSummary, ASSIGNED_ENTRY_COUNT,
     ASSIGNED_EXIT_COUNT, ASSIGNED_MIXING_COUNT, GATEWAYS_BONDED_COUNT, GATEWAYS_HISTORICAL_COUNT,
     MIXNODES_HISTORICAL_COUNT, MIXNODES_LEGACY_COUNT, NYMNODES_DESCRIBED_COUNT, NYMNODE_COUNT,
 };
 use crate::db::{queries, DbPool};
 use crate::monitor::geodata::{Location, NodeGeoData};
+use crate::utils::{decimal_to_i64, LogError, NumericalCheckedCast};
 use anyhow::anyhow;
-use cosmwasm_std::Decimal;
 use moka::future::Cache;
 use nym_network_defaults::NymNetworkDetails;
 use nym_validator_client::client::{NodeId, NymApiClientExt};
@@ -29,7 +29,6 @@ pub(crate) use geodata::IpInfoClient;
 
 mod geodata;
 
-// TODO dz should be configurable
 const FAILURE_RETRY_DELAY: Duration = Duration::from_secs(60);
 
 static DELEGATION_PROGRAM_WALLET: &str = "n1rnxpdpx3kldygsklfft0gech7fhfcux4zst5lw";
@@ -109,7 +108,11 @@ impl Monitor {
 
         let gateways = described_nodes
             .iter()
-            .filter(|node| node.description.declared_role.entry)
+            .filter(|node| {
+                node.description.declared_role.entry
+                    || node.description.declared_role.exit_ipr
+                    || node.description.declared_role.exit_nr
+            })
             .collect::<Vec<_>>();
 
         let bonded_node_info = api_client
@@ -120,12 +123,18 @@ impl Monitor {
             // for faster reads
             .collect::<HashMap<_, _>>();
 
+        tracing::info!("🟣 bonded_nodes: {}", bonded_node_info.len());
+
         let nym_nodes = api_client
             .get_all_basic_nodes()
             .await
             .log_error("get_all_basic_nodes")?;
 
-        queries::insert_nym_nodes(&self.db_pool, nym_nodes.clone(), &bonded_node_info).await?;
+        queries::insert_nym_nodes(&self.db_pool, nym_nodes.clone(), &bonded_node_info)
+            .await
+            .map(|_| {
+                tracing::debug!("{} nym nodes written to DB!", nym_nodes.len());
+            })?;
 
         let mut gateway_geodata = Vec::new();
         for gateway in gateways.iter() {
@@ -198,10 +207,11 @@ impl Monitor {
         let gateway_records = self.prepare_gateway_data(&gateways, gateway_geodata, &nym_nodes)?;
 
         let pool = self.db_pool.clone();
+        let gateways_count = gateway_records.len();
         queries::insert_gateways(&pool, gateway_records)
             .await
             .map(|_| {
-                tracing::debug!("Gateway info written to DB!");
+                tracing::debug!("{} gateway records written to DB!", gateways_count);
             })?;
 
         let mixnode_records = self.prepare_mixnode_data(
@@ -209,10 +219,11 @@ impl Monitor {
             mixnodes_described,
             delegation_program_members,
         )?;
+        let mixnodes_count = mixnode_records.len();
         queries::insert_mixnodes(&pool, mixnode_records)
             .await
             .map(|_| {
-                tracing::debug!("Mixnode info written to DB!");
+                tracing::debug!("{} mixnode info written to DB!", mixnodes_count);
             })?;
 
         let (all_historical_gateways, all_historical_mixnodes) = calculate_stats(&pool).await?;
@@ -299,13 +310,13 @@ impl Monitor {
 
     fn prepare_gateway_data(
         &self,
-        gateways: &[&NymNodeDescription],
+        described_gateways: &[&NymNodeDescription],
         gateway_geodata: Vec<NodeGeoData>,
         skimmed_gateways: &[SkimmedNode],
-    ) -> anyhow::Result<Vec<GatewayRecord>> {
+    ) -> anyhow::Result<Vec<GatewayInsertRecord>> {
         let mut gateway_records = Vec::new();
 
-        for gateway in gateways {
+        for gateway in described_gateways {
             let identity_key = gateway.ed25519_identity_key().to_base58_string();
             let bonded = true;
             let last_updated_utc = chrono::offset::Utc::now().timestamp();
@@ -329,7 +340,7 @@ impl Monitor {
                 .unwrap_or_default()
                 .round_to_integer();
 
-            gateway_records.push(GatewayRecord {
+            gateway_records.push(GatewayInsertRecord {
                 identity_key: identity_key.to_owned(),
                 bonded,
                 self_described,
@@ -400,33 +411,6 @@ impl Monitor {
     }
 }
 
-// TODO dz is there a common monorepo place this can be put?
-pub trait NumericalCheckedCast<T>
-where
-    T: TryFrom<Self>,
-    <T as TryFrom<Self>>::Error: std::error::Error,
-    Self: std::fmt::Display + Copy,
-{
-    fn cast_checked(self) -> anyhow::Result<T> {
-        T::try_from(self).map_err(|e| {
-            anyhow::anyhow!(
-                "Couldn't cast {} to {}: {}",
-                self,
-                std::any::type_name::<T>(),
-                e
-            )
-        })
-    }
-}
-
-impl<T, U> NumericalCheckedCast<U> for T
-where
-    U: TryFrom<T>,
-    <U as TryFrom<T>>::Error: std::error::Error,
-    T: std::fmt::Display + Copy,
-{
-}
-
 async fn calculate_stats(pool: &DbPool) -> anyhow::Result<(usize, usize)> {
     let mut conn = pool.acquire().await?;
 
@@ -464,39 +448,3 @@ async fn get_delegation_program_details(
 
     Ok(mix_ids)
 }
-
-pub(crate) fn decimal_to_i64(decimal: Decimal) -> i64 {
-    // Convert the underlying Uint128 to a u128
-    let atomics = decimal.atomics().u128();
-    let precision = 1_000_000_000_000_000_000u128;
-
-    // Get the fractional part
-    let fractional = atomics % precision;
-
-    // Get the integer part
-    let integer = atomics / precision;
-
-    // Combine them into a float
-    let float_value = integer as f64 + (fractional as f64 / 1_000_000_000_000_000_000_f64);
-
-    // Limit to 6 decimal places
-    let rounded_value = (float_value * 1_000_000.0).round() / 1_000_000.0;
-
-    rounded_value as i64
-}
-
-trait LogError<T, E> {
-    fn log_error(self, msg: &str) -> Result<T, E>;
-}
-
-impl<T, E> LogError<T, E> for anyhow::Result<T, E>
-where
-    E: std::error::Error,
-{
-    fn log_error(self, msg: &str) -> Result<T, E> {
-        if let Err(e) = &self {
-            tracing::error!("[{msg}]:\t{e}");
-        }
-        self
-    }
-}

nym-node-status-api/nym-node-status-api/src/node_scraper/mod.rs

@@ -17,15 +17,14 @@ use tracing::instrument;
 mod error;
 
 const FAILURE_RETRY_DELAY: Duration = Duration::from_secs(60);
-const REFRESH_INTERVAL: Duration = Duration::from_secs(60 * 60 * 6); //6h, data only update once a day
+const REFRESH_INTERVAL: Duration = Duration::from_secs(60 * 60 * 6);
 const STALE_DURATION: Duration = Duration::from_secs(86400 * 365); //one year
 
-#[instrument(level = "debug", name = "node_scraper", skip_all)]
+#[instrument(level = "info", name = "metrics_scraper", skip_all)]
 pub(crate) async fn spawn_in_background(db_pool: DbPool, nym_api_client_timeout: Duration) {
     let network_defaults = nym_network_defaults::NymNetworkDetails::new_from_env();
 
     loop {
-        //No graceful shutdown?
         tracing::info!("Refreshing node self-described metrics...");
 
         if let Err(e) = run(&db_pool, &network_defaults, nym_api_client_timeout).await {
@@ -123,7 +122,7 @@ impl MetricsScrapingData {
         }
     }
 
-    #[instrument(level = "debug", name = "metrics_scraper", skip_all)]
+    #[instrument(level = "info", name = "metrics_scraper", skip_all)]
     async fn try_scrape_metrics(&self) -> Option<SessionStats> {
         match self.try_get_client().await {
             Ok(client) => {

nym-node-status-api/nym-node-status-api/src/utils.rs

@@ -0,0 +1,104 @@
+use cosmwasm_std::Decimal;
+use itertools::Itertools;
+use rand::prelude::SliceRandom;
+use rand::SeedableRng;
+
+// pub(crate) fn generate_node_name(identity: ed25519::PublicKey) -> String {
+pub(crate) fn generate_node_name(node_id: i64) -> String {
+    let seed = {
+        let node_id_bytes = node_id.to_le_bytes();
+        let mut seed = [0u8; 32];
+        for i in 0..4 {
+            seed[i * 8..(i + 1) * 8].copy_from_slice(&node_id_bytes);
+        }
+        seed
+    };
+    let mut rng = rand_chacha::ChaCha20Rng::from_seed(seed);
+    let words = bip39::Language::English.word_list();
+    words.choose_multiple(&mut rng, 3).join(" ")
+}
+
+#[allow(clippy::items_after_test_module)]
+#[cfg(test)]
+mod test {
+    use rand::Rng;
+
+    use super::*;
+
+    #[test]
+    fn generate_node_name_should_be_deterministic() {
+        let mut rng = rand::thread_rng();
+
+        let node_id: i64 = rng.gen();
+        let different_node_id: i64 = rng.gen();
+
+        let node_name = generate_node_name(node_id);
+        let node_name_different = generate_node_name(different_node_id);
+        assert_ne!(node_name, node_name_different);
+
+        let node_name_same = generate_node_name(node_id);
+        assert_eq!(node_name, node_name_same);
+    }
+}
+
+pub trait NumericalCheckedCast<T>
+where
+    T: TryFrom<Self>,
+    <T as TryFrom<Self>>::Error: std::error::Error,
+    Self: std::fmt::Display + Copy,
+{
+    fn cast_checked(self) -> anyhow::Result<T> {
+        T::try_from(self).map_err(|e| {
+            anyhow::anyhow!(
+                "Couldn't cast {} to {}: {}",
+                self,
+                std::any::type_name::<T>(),
+                e
+            )
+        })
+    }
+}
+
+impl<T, U> NumericalCheckedCast<U> for T
+where
+    U: TryFrom<T>,
+    <U as TryFrom<T>>::Error: std::error::Error,
+    T: std::fmt::Display + Copy,
+{
+}
+
+pub(crate) fn decimal_to_i64(decimal: Decimal) -> i64 {
+    // Convert the underlying Uint128 to a u128
+    let atomics = decimal.atomics().u128();
+    let precision = 1_000_000_000_000_000_000u128;
+
+    // Get the fractional part
+    let fractional = atomics % precision;
+
+    // Get the integer part
+    let integer = atomics / precision;
+
+    // Combine them into a float
+    let float_value = integer as f64 + (fractional as f64 / 1_000_000_000_000_000_000_f64);
+
+    // Limit to 6 decimal places
+    let rounded_value = (float_value * 1_000_000.0).round() / 1_000_000.0;
+
+    rounded_value as i64
+}
+
+pub(crate) trait LogError<T, E> {
+    fn log_error(self, msg: &str) -> Result<T, E>;
+}
+
+impl<T, E> LogError<T, E> for anyhow::Result<T, E>
+where
+    E: std::error::Error,
+{
+    fn log_error(self, msg: &str) -> Result<T, E> {
+        if let Err(e) = &self {
+            tracing::error!("[{msg}]:\t{e}");
+        }
+        self
+    }
+}

nym-wallet/src/components/Send/SendInputModal.tsx

@@ -121,6 +121,19 @@ export const SendInputModal = ({
           initialValue={amount?.amount}
           denom={denom}
         />
+        <TextField
+          name="memo"
+          label="Memo"
+          onChange={(e) => onMemoChange(e.target.value)}
+          value={memo}
+          error={!memoIsValid}
+          placeholder="Optional"
+          helperText={
+            !memoIsValid ? ' The text is invalid, only alphanumeric characters and white spaces are allowed' : undefined
+          }
+          InputLabelProps={{ shrink: true }}
+          fullWidth
+        />
         <Typography fontSize="smaller" sx={{ color: 'error.main' }}>
           {error}
         </Typography>

nyx-chain-watcher/build.rs

@@ -1,13 +1,12 @@
 use anyhow::Result;
 use sqlx::{sqlite::SqliteConnectOptions, Connection, SqliteConnection};
+use std::env::var;
 use std::io::Write;
 use std::{collections::HashMap, fs::File, path::PathBuf, str::FromStr};
 
 #[tokio::main]
 async fn main() -> Result<()> {
-    let db_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
-        .join(".build")
-        .join("nyx_chain_watcher.sqlite");
+    let db_path = PathBuf::from(var("OUT_DIR").unwrap()).join("nyx_chain_watcher.sqlite");
 
     // Create the database directory if it doesn't exist
     if let Some(parent) = db_path.parent() {

scripts/network_tunnel_manager.sh

@@ -167,6 +167,8 @@ joke_through_tunnel() {
         else
             echo -e "${red}IPv4 connectivity is not working for $interface. verify your routing and NAT settings.${reset}"
         fi
+    else
+        echo -e "${red}no IPv4 address found on $interface. unable to fetch a joke via IPv4.${reset}"
     fi
 
     if [[ -n "$ipv6_address" ]]; then
@@ -183,6 +185,8 @@ joke_through_tunnel() {
         else
             echo -e "${red}IPv6 connectivity is not working for $interface. verify your routing and NAT settings.${reset}"
         fi
+    else
+        echo -e "${red}no IPv6 address found on $interface. unable to fetch a joke via IPv6.${reset}"
     fi
 
     echo -e "${green}joke fetching processes completed for $interface.${reset}"