WIP

Cargo.lock

@@ -6411,6 +6411,7 @@ dependencies = [
  "utoipa",
  "utoipa-swagger-ui",
  "utoipauto",
+ "validator-status-check",
 ]
 
 [[package]]

Cargo.toml

@@ -137,7 +137,8 @@ members = [
     "tools/internal/testnet-manager",
     "tools/internal/testnet-manager",
     "tools/internal/testnet-manager/dkg-bypass-contract",
-    "tools/internal/testnet-manager/dkg-bypass-contract", "tools/internal/validator-status-check",
+    "tools/internal/testnet-manager/dkg-bypass-contract",
+    "tools/internal/validator-status-check",
     "tools/nym-cli",
     "tools/nym-id-cli",
     "tools/nym-nr-query",

nym-node-status-api/nym-node-status-api/Cargo.toml

@@ -36,6 +36,7 @@ nym-statistics-common = { path = "../../common/statistics" }
 nym-validator-client = { path = "../../common/client-libs/validator-client" }
 nym-task = { path = "../../common/task" }
 nym-node-requests = { path = "../../nym-node/nym-node-requests", features = ["openapi"] }
+validator-status-check = { path = "../../tools/internal/validator-status-check" }
 rand = { workspace = true }
 rand_chacha = { workspace = true }
 regex = { workspace = true }

nym-node-status-api/nym-node-status-api/src/db/queries/mod.rs

@@ -2,9 +2,9 @@ mod gateways;
 mod gateways_stats;
 mod misc;
 mod mixnodes;
+pub(crate) mod node_description;
 mod nym_nodes;
 mod packet_stats;
-pub(crate) mod scraper;
 mod summary;
 pub(crate) mod testruns;
 
@@ -14,6 +14,7 @@ pub(crate) use gateways::{
 pub(crate) use gateways_stats::{delete_old_records, get_sessions_stats, insert_session_records};
 pub(crate) use misc::insert_summaries;
 pub(crate) use mixnodes::{get_all_mixnodes, get_bonded_mix_ids, get_daily_stats, update_mixnodes};
+pub(crate) use node_description::{get_nodes_for_scraping, insert_scraped_node_description};
 pub(crate) use nym_nodes::{
     get_active_nym_nodes, get_all_nym_nodes, get_described_node_bond_info, get_node_descriptions,
     update_nym_nodes,
@@ -21,5 +22,4 @@ pub(crate) use nym_nodes::{
 pub(crate) use packet_stats::{
     get_raw_node_stats, insert_daily_node_stats, insert_node_packet_stats,
 };
-pub(crate) use scraper::{get_nodes_for_scraping, insert_scraped_node_description};
 pub(crate) use summary::{get_summary, get_summary_history};

nym-node-status-api/nym-node-status-api/src/db/queries/node_description.rs

@@ -3,7 +3,7 @@ use crate::{
         models::{ScrapeNodeKind, ScraperNodeInfo},
         queries, DbPool,
     },
-    mixnet_scraper::helpers::NodeDescriptionResponse,
+    scrapers::node_info::helpers::NodeDescriptionResponse,
 };
 use anyhow::Result;
 use chrono::Utc;

nym-node-status-api/nym-node-status-api/src/main.rs

@@ -6,9 +6,8 @@ mod cli;
 mod db;
 mod http;
 mod logging;
-mod mixnet_scraper;
 mod monitor;
-mod node_scraper;
+mod scrapers;
 mod testruns;
 mod utils;
 
@@ -32,14 +31,19 @@ async fn main() -> anyhow::Result<()> {
     let db_pool = storage.pool_owned();
 
     // Start the node scraper
-    let scraper = mixnet_scraper::Scraper::new(storage.pool_owned());
+    let scraper = scrapers::node_info::Scraper::new(storage.pool_owned());
     tokio::spawn(async move {
         scraper.start().await;
     });
 
-    // Start the monitor
-    let args_clone = args.clone();
+    let db_pool = storage.pool_owned();
+    tokio::spawn(async move {
+        scrapers::nym_api_versions::spawn(db_pool).await;
+        tracing::info!("Started nym_api scraper task");
+    });
 
+    let args_clone = args.clone();
+    let db_pool = storage.pool_owned();
     tokio::spawn(async move {
         monitor::spawn_in_background(
             db_pool,
@@ -57,7 +61,8 @@ async fn main() -> anyhow::Result<()> {
 
     let db_pool_scraper = storage.pool_owned();
     tokio::spawn(async move {
-        node_scraper::spawn_in_background(db_pool_scraper, args_clone.nym_api_client_timeout).await;
+        scrapers::sessions::spawn_in_background(db_pool_scraper, args_clone.nym_api_client_timeout)
+            .await;
         tracing::info!("Started metrics scraper task");
     });
 

nym-node-status-api/nym-node-status-api/src/scrapers/mod.rs

@@ -0,0 +1,3 @@
+pub(crate) mod node_info;
+pub(crate) mod sessions;
+pub(crate) mod nym_api_versions;

nym-node-status-api/nym-node-status-api/src/scrapers/nym_api_versions/mod.rs

@@ -0,0 +1,151 @@
+use crate::db::DbPool;
+use nym_network_defaults::NymNetworkDetails;
+use nym_validator_client::nyxd::NyxdClient;
+use reqwest::Url;
+use serde::Deserialize;
+use sqlx::SqlitePool;
+use std::time::Duration;
+use tracing::{debug, error, instrument};
+use validator_status_check::{helpers::get_known_dealers, models::SignerStatus};
+
+// TODO dz configurable?
+const SCRAPE_INTERVAL: Duration = Duration::from_secs(60 * 60);
+const COSMOS_REST_API: &str = "https://api.nymtech.net/cosmwasm/wasm/v1/contract/n19604yflqggs9mk2z26mqygq43q2kr3n932egxx630svywd5mpxjsztfpvx/smart/eyJnZXRfY3VycmVudF9kZWFsZXJzIjogeyJsaW1pdCI6IDMwfX0=";
+const BUILD_INFORMATION_API: &str = "/v1/api-status/build-information";
+
+pub struct Scraper {
+    pool: SqlitePool,
+    http_client: reqwest::Client,
+}
+
+#[instrument(level = "debug", name = "nym_api_versions", skip_all)]
+pub(crate) async fn spawn(pool: DbPool) {
+    tracing::info!("Starting Nym API scraper");
+
+    let pool_cloned = pool.clone();
+    let scraper = Scraper::new(pool_cloned);
+
+    tokio::spawn(async move {
+        loop {
+            if let Err(e) = scraper.run().await {
+                error!(name: "nym_api_scraper", "Failed: {}", e);
+            }
+            debug!(name: "nym_api_scraper", "Sleeping for {}s", SCRAPE_INTERVAL.as_secs());
+            tokio::time::sleep(SCRAPE_INTERVAL).await;
+        }
+    });
+}
+
+impl Scraper {
+    pub fn new(pool: SqlitePool) -> Self {
+        Self {
+            pool,
+            http_client: reqwest::Client::new(),
+        }
+    }
+
+    async fn run(&self) -> anyhow::Result<()> {
+        let dealers = get_known_dealers().await?;
+        let mut signer_statuses = Vec::new();
+        for dealer in dealers {
+            let mut status = SignerStatus::new(dealer.announce_address);
+            status.try_update_api_version().await;
+            status.try_update_rpc_status().await;
+
+            signer_statuses.push(status);
+        }
+
+        Ok(())
+    }
+
+    async fn get_build_info_from_all_signers(
+        &self,
+    ) -> anyhow::Result<Vec<cosmos_response::Status>> {
+        let signer_address_list = {
+            let response: cosmos_response::Response = self
+                .http_client
+                .get(COSMOS_REST_API)
+                .send()
+                .await
+                .and_then(|res| res.error_for_status())?
+                .json()
+                .await?;
+            response
+                .data
+                .dealers
+                .into_iter()
+                .map(|dealer| dealer.announce_address)
+                .collect::<Vec<_>>()
+        };
+
+        let mut build_info = Vec::new();
+        for signer_address in signer_address_list {
+            let target_url = format!("{}/{}", signer_address, BUILD_INFORMATION_API);
+            let signer_status = match self
+                .http_client
+                .get(target_url)
+                .send()
+                .await
+                .and_then(|res| res.error_for_status())
+            {
+                Ok(response) => match response.json::<BuildInformation>().await {
+                    Ok(build_info) => cosmos_response::Status::Ok(build_info),
+                    Err(err) => cosmos_response::Status::Unreachable(err.to_string()),
+                },
+                Err(err) => cosmos_response::Status::Unreachable(err.to_string()),
+            };
+            build_info.push(signer_status);
+        }
+
+        Ok(build_info)
+    }
+}
+
+// async fn get_build_info(client: &reqwest::Client, dealer_address: &str) -> anyhow::Result<String>
+
+#[allow(dead_code)]
+mod cosmos_response {
+    use serde::Deserialize;
+
+    #[derive(Debug, Deserialize)]
+    pub struct Response {
+        pub data: Data,
+    }
+
+    #[derive(Debug, Deserialize)]
+    pub struct Data {
+        pub dealers: Vec<Dealer>,
+        pub per_page: u64,
+        pub start_next_after: String,
+    }
+
+    #[derive(Debug, Deserialize)]
+    pub struct Dealer {
+        pub address: String,
+        pub bte_public_key_with_proof: String,
+        pub ed25519_identity: String,
+        pub announce_address: String,
+        pub assigned_index: u64,
+    }
+
+    #[derive(Debug, Deserialize)]
+    pub enum Status {
+        Ok(super::BuildInformation),
+        Unreachable(String),
+    }
+}
+
+#[allow(dead_code)]
+#[derive(Debug, Deserialize)]
+struct BuildInformation {
+    binary_name: String,
+    build_timestamp: String,
+    build_version: String,
+    commit_sha: String,
+    commit_timestamp: String,
+    commit_branch: String,
+    rustc_version: String,
+    rustc_channel: String,
+    cargo_profile: String,
+    cargo_triple: String,
+}

nym-node-status-api/nym-node-status-api/src/scrapers/sessions/mod.rs

@@ -20,7 +20,7 @@ const FAILURE_RETRY_DELAY: Duration = Duration::from_secs(60);
 const REFRESH_INTERVAL: Duration = Duration::from_secs(60 * 60 * 6);
 const STALE_DURATION: Duration = Duration::from_secs(86400 * 365); //one year
 
-#[instrument(level = "info", name = "metrics_scraper", skip_all)]
+#[instrument(level = "debug", name = "metrics_scraper", skip_all)]
 pub(crate) async fn spawn_in_background(db_pool: DbPool, nym_api_client_timeout: Duration) {
     let network_defaults = nym_network_defaults::NymNetworkDetails::new_from_env();
 

tools/internal/validator-status-check/Cargo.toml

@@ -28,3 +28,11 @@ nym-network-defaults = { path = "../../../common/network-defaults" }
 
 [lints]
 workspace = true
+
+[[bin]]
+name = "validator_status_check"
+path = "src/main.rs"
+
+[lib]
+name = "validator_status_check"
+path = "src/lib.rs"

tools/internal/validator-status-check/src/helpers.rs

@@ -10,7 +10,7 @@ use nym_validator_client::nyxd::Config;
 use nym_validator_client::QueryHttpRpcNyxdClient;
 use tracing::info;
 
-async fn get_query_client() -> anyhow::Result<QueryHttpRpcNyxdClient> {
+pub async fn get_query_client() -> anyhow::Result<QueryHttpRpcNyxdClient> {
     let network = NymNetworkDetails::new_from_env();
 
     let Some(endpoint_info) = network.endpoints.first() else {
@@ -24,12 +24,12 @@ async fn get_query_client() -> anyhow::Result<QueryHttpRpcNyxdClient> {
     )?)
 }
 
-pub(crate) async fn get_known_dealers() -> anyhow::Result<Vec<DealerDetails>> {
+pub async fn get_known_dealers() -> anyhow::Result<Vec<DealerDetails>> {
     let client = get_query_client().await?;
     Ok(client.get_all_current_dealers().await?)
 }
 
-pub(crate) async fn get_signer_status(raw_api_endpoint: &str) -> SignerStatus {
+pub async fn get_signer_status(raw_api_endpoint: &str) -> SignerStatus {
     info!("attempting to get signer status of {raw_api_endpoint}...");
     let mut status = SignerStatus::new(raw_api_endpoint.to_string());
 

tools/internal/validator-status-check/src/lib.rs

@@ -0,0 +1,2 @@
+pub mod helpers;
+pub mod models;

tools/internal/validator-status-check/src/main.rs

@@ -8,8 +8,8 @@ use nym_network_defaults::setup_env;
 use tracing::trace;
 
 mod commands;
-mod helpers;
-mod models;
+
+use validator_status_check::{helpers, models};
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {

tools/internal/validator-status-check/src/models.rs

@@ -10,7 +10,7 @@ use time::{Duration, OffsetDateTime};
 use tracing::error;
 
 #[derive(Serialize, Deserialize)]
-pub(crate) struct SignerStatus {
+pub struct SignerStatus {
     api_endpoint: String,
     api_version: ApiVersion,
     rpc_status: RpcStatus,
@@ -30,7 +30,7 @@ impl Display for SignerStatus {
 }
 
 impl SignerStatus {
-    pub(crate) fn new(api_endpoint: String) -> Self {
+    pub fn new(api_endpoint: String) -> Self {
         SignerStatus {
             api_endpoint,
             api_version: Default::default(),
@@ -40,11 +40,11 @@ impl SignerStatus {
         }
     }
 
-    pub(crate) fn api_up(&self) -> bool {
+    pub fn api_up(&self) -> bool {
         matches!(self.api_version, ApiVersion::Available { .. })
     }
 
-    pub(crate) fn rpc_up(&self) -> bool {
+    pub fn rpc_up(&self) -> bool {
         matches!(self.rpc_status, RpcStatus::Up)
     }
 
@@ -60,7 +60,7 @@ impl SignerStatus {
         Some(NymApiClient::new(api_endpoint))
     }
 
-    pub(crate) async fn try_update_api_version(&mut self) {
+    pub async fn try_update_api_version(&mut self) {
         let Some(client) = self.build_api_client() else {
             return;
         };
@@ -79,7 +79,7 @@ impl SignerStatus {
         }
     }
 
-    pub(crate) async fn try_update_rpc_status(&mut self) {
+    pub async fn try_update_rpc_status(&mut self) {
         let Some(client) = self.build_api_client() else {
             return;
         };
@@ -109,7 +109,7 @@ impl SignerStatus {
         }
     }
 
-    pub(crate) fn to_table_row(&self) -> Vec<String> {
+    pub fn to_table_row(&self) -> Vec<String> {
         vec![
             self.api_endpoint.to_string(),
             self.api_version.as_cell(),