cleanup psql fork for push

Rebase + point to earlier network client code

Adjust to new Nym API types

Refer to earlier client code

Revert "Rebase + point to earlier network client code"

This reverts commit dd75e7dc0695c25b0883e2f5dd15b7d70165e9e8.

Point to earlier commit

Cargo.toml

@@ -79,6 +79,7 @@ members = [
     "common/nymsphinx/routing",
     "common/nymsphinx/types",
     "common/nyxd-scraper",
+    "common/nyxd-scraper-psql",
     "common/pemstore",
     "common/serde-helpers",
     "common/service-provider-requests-common",
@@ -118,6 +119,7 @@ members = [
     "nym-node",
     "nym-node/nym-node-http-api",
     "nym-node/nym-node-requests",
+    "nym-node-status-api",
     "nym-outfox",
     "nym-validator-rewarder",
     "tools/echo-server",
@@ -152,6 +154,7 @@ default-members = [
     "nym-data-observatory",
     "nym-node",
     "nym-validator-rewarder",
+    "nym-node-status-api",
     "service-providers/authenticator",
     "service-providers/ip-packet-router",
     "service-providers/network-requester",
@@ -232,10 +235,12 @@ dotenvy = "0.15.6"
 ecdsa = "0.16"
 ed25519-dalek = "2.1"
 etherparse = "0.13.0"
+envy = "0.4"
 eyre = "0.6.9"
 fastrand = "2.1.1"
 flate2 = "1.0.34"
 futures = "0.3.28"
+futures-util = "0.3"
 generic-array = "0.14.7"
 getrandom = "0.2.10"
 getset = "0.1.3"
@@ -265,6 +270,7 @@ ledger-transport-hid = "0.10.0"
 log = "0.4"
 maxminddb = "0.23.0"
 mime = "0.3.17"
+moka = { version = "0.12", features = ["future"] }
 nix = "0.27.1"
 notify = "5.1.0"
 okapi = "0.7.0"
@@ -298,6 +304,7 @@ serde = "1.0.210"
 serde_bytes = "0.11.15"
 serde_derive = "1.0"
 serde_json = "1.0.128"
+serde_json_path = "0.6.7"
 serde_repr = "0.1"
 serde_with = "3.9.0"
 serde_yaml = "0.9.25"
@@ -327,6 +334,7 @@ tracing = "0.1.37"
 tracing-opentelemetry = "0.19.0"
 tracing-subscriber = "0.3.16"
 tracing-tree = "0.2.2"
+tracing-log = "0.2"
 ts-rs = "10.0.0"
 tungstenite = { version = "0.20.1", default-features = false }
 url = "2.5"

common/client-libs/validator-client/Cargo.toml

@@ -25,7 +25,7 @@ serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
 nym-http-api-client = { path = "../../../common/http-api-client" }
 thiserror = { workspace = true }
-log = { workspace = true }
+tracing = { workspace = true }
 url = { workspace = true, features = ["serde"] }
 tokio = { workspace = true, features = ["sync", "time"] }
 time = { workspace = true, features = ["formatting"] }

common/client-libs/validator-client/src/client.rs

@@ -265,6 +265,13 @@ impl NymApiClient {
         NymApiClient { nym_api }
     }
 
+    #[cfg(not(target_arch = "wasm32"))]
+    pub fn new_with_timeout(api_url: Url, timeout: std::time::Duration) -> Self {
+        let nym_api = nym_api::Client::new(api_url, Some(timeout));
+
+        NymApiClient { nym_api }
+    }
+
     pub fn new_with_user_agent(api_url: Url, user_agent: UserAgent) -> Self {
         let nym_api = nym_api::Client::builder::<_, ValidatorClientError>(api_url)
             .expect("invalid api url")

common/client-libs/validator-client/src/connection_tester.rs

@@ -121,36 +121,36 @@ async fn test_nyxd_connection(
     {
         Ok(Err(NyxdError::TendermintErrorRpc(e))) => {
             // If we get a tendermint-rpc error, we classify the node as not contactable
-            log::warn!("Checking: nyxd url: {url}: {}: {}", "failed".red(), e);
+            tracing::warn!("Checking: nyxd url: {url}: {}: {}", "failed".red(), e);
             false
         }
         Ok(Err(NyxdError::AbciError { code, log, .. })) => {
             // We accept the mixnet contract not found as ok from a connection standpoint. This happens
             // for example on a pre-launch network.
-            log::debug!(
+            tracing::debug!(
                 "Checking: nyxd url: {url}: {}, but with abci error: {code}: {log}",
                 "success".green()
             );
             code == 18
         }
         Ok(Err(error @ NyxdError::NoContractAddressAvailable(_))) => {
-            log::warn!("Checking: nyxd url: {url}: {}: {error}", "failed".red());
+            tracing::warn!("Checking: nyxd url: {url}: {}: {error}", "failed".red());
             false
         }
         Ok(Err(e)) => {
             // For any other error, we're optimistic and just try anyway.
-            log::warn!(
+            tracing::warn!(
                 "Checking: nyxd_url: {url}: {}, but with error: {e}",
                 "success".green()
             );
             true
         }
         Ok(Ok(_)) => {
-            log::debug!("Checking: nyxd_url: {url}: {}", "success".green());
+            tracing::debug!("Checking: nyxd_url: {url}: {}", "success".green());
             true
         }
         Err(e) => {
-            log::warn!("Checking: nyxd_url: {url}: {}: {e}", "failed".red());
+            tracing::warn!("Checking: nyxd_url: {url}: {}: {e}", "failed".red());
             false
         }
     };
@@ -169,15 +169,15 @@ async fn test_nym_api_connection(
     .await
     {
         Ok(Ok(_)) => {
-            log::debug!("Checking: api_url: {url}: {}", "success".green());
+            tracing::debug!("Checking: api_url: {url}: {}", "success".green());
             true
         }
         Ok(Err(e)) => {
-            log::debug!("Checking: api_url: {url}: {}: {e}", "failed".red());
+            tracing::debug!("Checking: api_url: {url}: {}: {e}", "failed".red());
             false
         }
         Err(e) => {
-            log::debug!("Checking: api_url: {url}: {}: {e}", "failed".red());
+            tracing::debug!("Checking: api_url: {url}: {}: {e}", "failed".red());
             false
         }
     };

common/client-libs/validator-client/src/nym_api/mod.rs

@@ -40,6 +40,7 @@ use nym_mixnet_contract_common::mixnode::MixNodeDetails;
 use nym_mixnet_contract_common::{GatewayBond, IdentityKeyRef, NodeId};
 use time::format_description::BorrowedFormatItem;
 use time::Date;
+use tracing::instrument;
 
 pub mod error;
 pub mod routes;
@@ -51,11 +52,13 @@ pub fn rfc_3339_date() -> Vec<BorrowedFormatItem<'static>> {
 #[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
 #[cfg_attr(not(target_arch = "wasm32"), async_trait)]
 pub trait NymApiClientExt: ApiClient {
+    #[instrument(level = "debug", skip(self))]
     async fn get_mixnodes(&self) -> Result<Vec<MixNodeDetails>, NymAPIError> {
         self.get_json(&[routes::API_VERSION, routes::MIXNODES], NO_PARAMS)
             .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn get_mixnodes_detailed(&self) -> Result<Vec<MixNodeBondAnnotated>, NymAPIError> {
         self.get_json(
             &[
@@ -69,6 +72,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn get_gateways_detailed(&self) -> Result<Vec<GatewayBondAnnotated>, NymAPIError> {
         self.get_json(
             &[
@@ -82,6 +86,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn get_mixnodes_detailed_unfiltered(
         &self,
     ) -> Result<Vec<MixNodeBondAnnotated>, NymAPIError> {
@@ -97,11 +102,13 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn get_gateways(&self) -> Result<Vec<GatewayBond>, NymAPIError> {
         self.get_json(&[routes::API_VERSION, routes::GATEWAYS], NO_PARAMS)
             .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn get_gateways_described(&self) -> Result<Vec<LegacyDescribedGateway>, NymAPIError> {
         self.get_json(
             &[routes::API_VERSION, routes::GATEWAYS, routes::DESCRIBED],
@@ -110,6 +117,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn get_mixnodes_described(&self) -> Result<Vec<LegacyDescribedMixNode>, NymAPIError> {
         self.get_json(
             &[routes::API_VERSION, routes::MIXNODES, routes::DESCRIBED],
@@ -118,6 +126,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[tracing::instrument(level = "debug", skip_all)]
     async fn get_basic_mixnodes(
         &self,
         semver_compatibility: Option<String>,
@@ -141,6 +150,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn get_basic_gateways(
         &self,
         semver_compatibility: Option<String>,
@@ -164,6 +174,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn get_active_mixnodes(&self) -> Result<Vec<MixNodeDetails>, NymAPIError> {
         self.get_json(
             &[routes::API_VERSION, routes::MIXNODES, routes::ACTIVE],
@@ -172,6 +183,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn get_active_mixnodes_detailed(&self) -> Result<Vec<MixNodeBondAnnotated>, NymAPIError> {
         self.get_json(
             &[
@@ -186,6 +198,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn get_rewarded_mixnodes(&self) -> Result<Vec<MixNodeDetails>, NymAPIError> {
         self.get_json(
             &[routes::API_VERSION, routes::MIXNODES, routes::REWARDED],
@@ -194,6 +207,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn get_mixnode_report(
         &self,
         mix_id: NodeId,
@@ -211,6 +225,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn get_gateway_report(
         &self,
         identity: IdentityKeyRef<'_>,
@@ -228,6 +243,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn get_mixnode_history(
         &self,
         mix_id: NodeId,
@@ -245,6 +261,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn get_gateway_history(
         &self,
         identity: IdentityKeyRef<'_>,
@@ -262,6 +279,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn get_rewarded_mixnodes_detailed(
         &self,
     ) -> Result<Vec<MixNodeBondAnnotated>, NymAPIError> {
@@ -278,6 +296,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn get_gateway_core_status_count(
         &self,
         identity: IdentityKeyRef<'_>,
@@ -309,6 +328,7 @@ pub trait NymApiClientExt: ApiClient {
         }
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn get_mixnode_core_status_count(
         &self,
         mix_id: NodeId,
@@ -341,6 +361,7 @@ pub trait NymApiClientExt: ApiClient {
         }
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn get_mixnode_status(
         &self,
         mix_id: NodeId,
@@ -358,6 +379,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn get_mixnode_reward_estimation(
         &self,
         mix_id: NodeId,
@@ -375,6 +397,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn compute_mixnode_reward_estimation(
         &self,
         mix_id: NodeId,
@@ -394,6 +417,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn get_mixnode_stake_saturation(
         &self,
         mix_id: NodeId,
@@ -411,6 +435,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn get_mixnode_inclusion_probability(
         &self,
         mix_id: NodeId,
@@ -428,6 +453,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn get_current_node_performance(
         &self,
         node_id: NodeId,
@@ -458,6 +484,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn get_mixnodes_blacklisted(&self) -> Result<Vec<NodeId>, NymAPIError> {
         self.get_json(
             &[routes::API_VERSION, routes::MIXNODES, routes::BLACKLISTED],
@@ -466,6 +493,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn get_gateways_blacklisted(&self) -> Result<Vec<IdentityKey>, NymAPIError> {
         self.get_json(
             &[routes::API_VERSION, routes::GATEWAYS, routes::BLACKLISTED],
@@ -474,6 +502,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self, request_body))]
     async fn blind_sign(
         &self,
         request_body: &BlindSignRequestBody,
@@ -490,6 +519,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self, request_body))]
     async fn verify_ecash_ticket(
         &self,
         request_body: &VerifyEcashTicketBody,
@@ -506,6 +536,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self, request_body))]
     async fn batch_redeem_ecash_tickets(
         &self,
         request_body: &BatchRedeemTicketsBody,
@@ -522,6 +553,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn double_spending_filter_v1(&self) -> Result<SpentCredentialsResponse, NymAPIError> {
         self.get_json(
             &[
@@ -534,6 +566,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn partial_expiration_date_signatures(
         &self,
         expiration_date: Option<Date>,
@@ -557,6 +590,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn partial_coin_indices_signatures(
         &self,
         epoch_id: Option<EpochId>,
@@ -577,6 +611,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn global_expiration_date_signatures(
         &self,
         expiration_date: Option<Date>,
@@ -600,6 +635,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn global_coin_indices_signatures(
         &self,
         epoch_id: Option<EpochId>,
@@ -620,6 +656,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn master_verification_key(
         &self,
         epoch_id: Option<EpochId>,
@@ -639,6 +676,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn epoch_credentials(
         &self,
         dkg_epoch: EpochId,
@@ -655,6 +693,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn issued_credential(
         &self,
         credential_id: i64,
@@ -671,6 +710,7 @@ pub trait NymApiClientExt: ApiClient {
         .await
     }
 
+    #[instrument(level = "debug", skip(self))]
     async fn issued_credentials(
         &self,
         credential_ids: Vec<i64>,

common/client-libs/validator-client/src/nyxd/contract_traits/dkg_query_client.rs

@@ -8,9 +8,9 @@ use crate::nyxd::CosmWasmClient;
 use async_trait::async_trait;
 use cosmrs::AccountId;
 use cosmwasm_std::Addr;
-use log::trace;
 use nym_coconut_dkg_common::types::{ChunkIndex, NodeIndex, StateAdvanceResponse};
 use serde::Deserialize;
+use tracing::trace;
 
 use nym_coconut_dkg_common::dealer::RegisteredDealerDetails;
 pub use nym_coconut_dkg_common::{

common/client-libs/validator-client/src/nyxd/cosmwasm_client/client_traits/query_client.rs

@@ -29,7 +29,6 @@ use cosmrs::proto::cosmwasm::wasm::v1::{
 };
 use cosmrs::tendermint::{block, chain, Hash};
 use cosmrs::{AccountId, Coin as CosmosCoin, Tx};
-use log::trace;
 use prost::Message;
 use serde::{Deserialize, Serialize};
 
@@ -68,7 +67,7 @@ pub trait CosmWasmClient: TendermintRpcClient {
         Res: Message + Default,
     {
         if let Some(ref abci_path) = path {
-            trace!("performing query on abci path {abci_path}")
+            tracing::trace!("performing query on abci path {abci_path}")
         }
         let mut buf = Vec::with_capacity(req.encoded_len());
         req.encode(&mut buf)?;
@@ -297,7 +296,7 @@ pub trait CosmWasmClient: TendermintRpcClient {
 
         let start = Instant::now();
         loop {
-            log::debug!(
+            tracing::debug!(
                 "Polling for result of including {} in a block...",
                 broadcasted.hash
             );
@@ -522,7 +521,7 @@ pub trait CosmWasmClient: TendermintRpcClient {
             .make_abci_query::<_, QuerySmartContractStateResponse>(path, req)
             .await?;
 
-        trace!("raw query response: {}", String::from_utf8_lossy(&res.data));
+        tracing::trace!("raw query response: {}", String::from_utf8_lossy(&res.data));
         Ok(serde_json::from_slice(&res.data)?)
     }
 

common/client-libs/validator-client/src/nyxd/cosmwasm_client/client_traits/signing_client.rs

@@ -25,12 +25,12 @@ use cosmrs::proto::cosmos::tx::signing::v1beta1::SignMode;
 use cosmrs::staking::{MsgDelegate, MsgUndelegate};
 use cosmrs::tx::{self, Msg};
 use cosmrs::{cosmwasm, AccountId, Any, Tx};
-use log::debug;
 use serde::Serialize;
 use sha2::Digest;
 use sha2::Sha256;
 use std::time::SystemTime;
 use tendermint_rpc::endpoint::broadcast;
+use tracing::debug;
 
 fn empty_fee() -> tx::Fee {
     tx::Fee {

common/client-libs/validator-client/src/nyxd/cosmwasm_client/helpers.rs

@@ -7,9 +7,9 @@ use base64::Engine;
 use cosmrs::abci::TxMsgData;
 use cosmrs::cosmwasm::MsgExecuteContractResponse;
 use cosmrs::proto::cosmos::base::query::v1beta1::{PageRequest, PageResponse};
-use log::error;
 use prost::bytes::Bytes;
 use tendermint_rpc::endpoint::broadcast;
+use tracing::error;
 
 pub use cosmrs::abci::MsgResponse;
 

common/http-api-client/src/lib.rs

@@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize};
 use std::fmt::Display;
 use std::time::Duration;
 use thiserror::Error;
-use tracing::warn;
+use tracing::{instrument, warn};
 use url::Url;
 
 pub use reqwest::IntoUrl;
@@ -202,6 +202,7 @@ impl Client {
         self.reqwest_client.get(url)
     }
 
+    #[instrument(level = "debug", skip_all, fields(path=?path))]
     async fn send_get_request<K, V, E>(
         &self,
         path: PathSegments<'_>,
@@ -212,6 +213,7 @@ impl Client {
         V: AsRef<str>,
         E: Display,
     {
+        tracing::trace!("Sending GET request");
         let url = sanitize_url(&self.base_url, path, params);
 
         #[cfg(target_arch = "wasm32")]
@@ -277,6 +279,7 @@ impl Client {
         }
     }
 
+    #[instrument(level = "debug", skip_all)]
     pub async fn get_json<T, K, V, E>(
         &self,
         path: PathSegments<'_>,
@@ -512,12 +515,14 @@ pub fn sanitize_url<K: AsRef<str>, V: AsRef<str>>(
     url
 }
 
+#[tracing::instrument(level = "debug", skip_all)]
 pub async fn parse_response<T, E>(res: Response, allow_empty: bool) -> Result<T, HttpClientError<E>>
 where
     T: DeserializeOwned,
     E: DeserializeOwned + Display,
 {
     let status = res.status();
+    tracing::debug!("Status: {} (success: {})", &status, status.is_success());
 
     if !allow_empty {
         if let Some(0) = res.content_length() {

common/nyxd-scraper-psql/Cargo.toml

@@ -0,0 +1,43 @@
+[package]
+name = "nyxd-scraper-psql"
+version = "0.1.0"
+authors.workspace = true
+repository.workspace = true
+homepage.workspace = true
+documentation.workspace = true
+edition.workspace = true
+license.workspace = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+async-trait.workspace = true
+base64.workspace = true
+const_format = { workspace = true }
+cosmrs.workspace = true
+chrono = {workspace = true}
+eyre = { workspace = true }
+futures.workspace = true
+humantime = { workspace = true }
+sha2 = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+serde_json = {workspace = true}
+sqlx = { workspace = true, features = ["runtime-tokio-rustls", "postgres", "macros", "migrate", "time", "json"] }
+tendermint.workspace = true
+tendermint-rpc = { workspace = true, features = ["websocket-client", "http-client"] }
+thiserror.workspace = true
+time = { workspace = true }
+tokio = { workspace = true, features = ["full"] }
+tokio-stream = { workspace = true }
+tokio-util = { workspace = true, features = ["rt"] }
+tracing.workspace = true
+url.workspace = true
+
+
+# TEMP
+#nym-bin-common = { path = "../bin-common", features = ["basic_tracing"]}
+
+
+[build-dependencies]
+sqlx = { workspace = true, features = ["runtime-tokio-rustls", "postgres", "macros", "migrate"] }
+tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }

common/nyxd-scraper-psql/Dockerfile

@@ -0,0 +1,7 @@
+FROM postgres:latest
+
+ENV POSTGRES_USER=nym
+ENV POSTGRES_PASSWORD=password123
+ENV POSTGRES_DB=nyxd_scraper
+
+EXPOSE 5432 

common/nyxd-scraper-psql/README.md

@@ -0,0 +1,23 @@
+# Nyxd Scraper
+
+## Pruning
+
+Similarly to cosmos-sdk, we incorporate pruning into our (scraped) chain data. We attempt to follow their strategies as
+closely as possible for convenience's sake. Therefore, the following are available:
+
+### Strategies
+
+The strategies are configured in `config.toml`, with the format `pruning = "<strategy>"` where the options are:
+
+* `default`: only the last 362,880 states(approximately 3.5 weeks worth of state) are kept; pruning at 10 block
+  intervals
+* `nothing`: all historic states will be saved, nothing will be deleted (i.e. archiving node)
+* `everything`: 2 latest states will be kept; pruning at 10 block intervals.
+* `custom`: allow pruning options to be manually specified through `pruning.keep_recent`, and `pruning.interval`
+
+### Custom Pruning
+
+These are applied if and only if the pruning strategy is `custom`:
+
+* `pruning.keep_recent`: N means to keep all of the last N blocks
+* `pruning.interval`: N means to delete old block data from disk every Nth block.

common/nyxd-scraper-psql/build.rs

@@ -0,0 +1,41 @@
+#[tokio::main]
+async fn main() {
+    use sqlx::{Connection, Executor, PgConnection};
+
+    const POSTGRES_USER: &str = "nym";
+    const POSTGRES_PASSWORD: &str = "password123";
+    const POSTGRES_DB: &str = "nyxd_scraper";
+
+    let admin_url = format!(
+        "postgres://{}:{}@localhost:5432/postgres",
+        POSTGRES_USER, POSTGRES_PASSWORD
+    );
+    // Connect to postgres to create test database
+    let database_url =
+        format!("postgres://{POSTGRES_USER}:{POSTGRES_PASSWORD}@localhost:5432/{POSTGRES_DB}");
+
+    let mut conn = PgConnection::connect(&admin_url)
+        .await
+        .expect("Failed to connect to Postgres");
+
+    conn.execute(format!(r#"DROP DATABASE IF EXISTS {}"#, POSTGRES_DB).as_str())
+        .await
+        .expect("Failed to drop test database");
+
+    conn.execute(format!(r#"CREATE DATABASE {}"#, POSTGRES_DB).as_str())
+        .await
+        .expect("Failed to create test database");
+
+    let mut test_conn = PgConnection::connect(&database_url)
+        .await
+        .expect("Failed to connect to test database");
+
+    // Run migrations
+    sqlx::migrate!("./sql_migrations")
+        .run(&mut test_conn)
+        .await
+        .expect("Failed to perform SQLx migrations");
+
+    // Set the database URL as an environment variable
+    println!("cargo:rustc-env=DATABASE_URL={}", database_url);
+}

common/nyxd-scraper-psql/docker-compose.yml

@@ -0,0 +1,15 @@
+version: '3.8'
+services:
+  postgres:
+    image: postgres:latest
+    environment:
+      POSTGRES_USER: nym
+      POSTGRES_PASSWORD: password123
+      POSTGRES_DB: nyxd_scraper
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+
+volumes:
+  postgres_data:

common/nyxd-scraper-psql/sql_migrations/01_metadata.sql

@@ -0,0 +1,8 @@
+/*
+ * Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+CREATE TABLE METADATA (
+    id INTEGER PRIMARY KEY CHECK (id = 0),
+    last_processed_height BIGINT NOT NULL
+);

common/nyxd-scraper-psql/sql_migrations/02_cosmos.sql

@@ -0,0 +1,73 @@
+CREATE TABLE validator (
+    consensus_address TEXT NOT NULL PRIMARY KEY,
+    /* Validator consensus address */
+    consensus_pubkey TEXT NOT NULL UNIQUE
+    /* Validator consensus public key */
+);
+
+CREATE TABLE pre_commit (
+    validator_address TEXT NOT NULL REFERENCES validator (consensus_address),
+    height BIGINT NOT NULL,
+    timestamp TIMESTAMPTZ NOT NULL,
+    voting_power BIGINT NOT NULL,
+    proposer_priority BIGINT NOT NULL,
+    UNIQUE (validator_address, timestamp)
+);
+
+CREATE INDEX pre_commit_validator_address_index ON pre_commit (validator_address);
+
+CREATE INDEX pre_commit_height_index ON pre_commit (height);
+
+CREATE TABLE block (
+    height BIGINT UNIQUE PRIMARY KEY,
+    hash TEXT NOT NULL UNIQUE,
+    num_txs INTEGER DEFAULT 0,
+    total_gas BIGINT DEFAULT 0,
+    proposer_address TEXT REFERENCES validator (consensus_address),
+    timestamp TIMESTAMPTZ NOT NULL
+);
+
+CREATE INDEX block_height_index ON block (height);
+
+CREATE INDEX block_hash_index ON block (hash);
+
+CREATE INDEX block_proposer_address_index ON block (proposer_address);
+
+CREATE TABLE "transaction" (
+    hash TEXT UNIQUE NOT NULL,
+    height BIGINT NOT NULL REFERENCES block (height),
+    "index" INTEGER NOT NULL,
+    success BOOLEAN NOT NULL,
+    /* Body */
+    num_messages INTEGER NOT NULL,
+    messages JSONB NOT NULL DEFAULT '[]',
+    memo TEXT,
+    signatures TEXT [] NOT NULL,
+    /* AuthInfo */
+    signer_infos JSONB NOT NULL DEFAULT '[]'::JSONB,
+    fee JSONB NOT NULL DEFAULT '{}'::JSONB,
+    /* Tx response */
+    gas_wanted BIGINT DEFAULT 0,
+    gas_used BIGINT DEFAULT 0,
+    raw_log TEXT
+);
+
+CREATE INDEX transaction_hash_index ON "transaction" (hash);
+
+CREATE INDEX transaction_height_index ON "transaction" (height);
+
+CREATE TABLE message (
+    transaction_hash TEXT NOT NULL REFERENCES "transaction" (hash),
+    "index" BIGINT NOT NULL,
+    TYPE TEXT NOT NULL,
+    value JSONB NOT NULL,
+    involved_accounts_addresses TEXT [] NOT NULL,
+    height BIGINT NOT NULL,
+    CONSTRAINT unique_message_per_tx UNIQUE (transaction_hash, "index")
+);
+
+CREATE INDEX message_transaction_hash_index ON message (transaction_hash);
+
+CREATE INDEX message_type_index ON message (TYPE);
+
+CREATE TABLE pruning (last_pruned_height BIGINT NOT NULL);

common/nyxd-scraper-psql/src/block_processor/helpers.rs

@@ -0,0 +1,65 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::block_processor::MAX_RANGE_SIZE;
+use std::cmp::min;
+use std::collections::VecDeque;
+use std::ops::Range;
+
+pub(crate) fn split_request_range(request_range: Range<u32>) -> VecDeque<Range<u32>> {
+    let mut requests = VecDeque::new();
+
+    let mut start = request_range.start;
+    let mut end = min(request_range.end, start + MAX_RANGE_SIZE as u32);
+
+    loop {
+        requests.push_back(start..end);
+        start = min(start + MAX_RANGE_SIZE as u32, request_range.end);
+        end = min(end + MAX_RANGE_SIZE as u32, request_range.end);
+
+        if start == end {
+            break;
+        }
+    }
+
+    requests
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn splitting_request_range() {
+        let range = 0..100;
+        let mut expected = VecDeque::new();
+        expected.push_back(0..30);
+        expected.push_back(30..60);
+        expected.push_back(60..90);
+        expected.push_back(90..100);
+        assert_eq!(expected, split_request_range(range));
+
+        let range = 0..30;
+        let mut expected = VecDeque::new();
+        expected.push_back(0..30);
+        assert_eq!(expected, split_request_range(range));
+
+        let range = 0..60;
+        let mut expected = VecDeque::new();
+        expected.push_back(0..30);
+        expected.push_back(30..60);
+        assert_eq!(expected, split_request_range(range));
+
+        let range = 0..5;
+        let mut expected = VecDeque::new();
+        expected.push_back(0..5);
+        assert_eq!(expected, split_request_range(range));
+
+        let range = 123..200;
+        let mut expected = VecDeque::new();
+        expected.push_back(123..153);
+        expected.push_back(153..183);
+        expected.push_back(183..200);
+        assert_eq!(expected, split_request_range(range));
+    }
+}

common/nyxd-scraper-psql/src/block_processor/mod.rs

@@ -0,0 +1,426 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::block_processor::helpers::split_request_range;
+use crate::block_processor::types::BlockToProcess;
+use crate::block_requester::BlockRequest;
+use crate::error::ScraperError;
+use crate::modules::{BlockModule, MsgModule, TxModule};
+use crate::rpc_client::RpcClient;
+use crate::storage::{persist_block, ScraperStorage};
+use crate::PruningOptions;
+use futures::StreamExt;
+use std::cmp::max;
+use std::collections::{BTreeMap, HashSet, VecDeque};
+use std::ops::{Add, Range};
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::mpsc::{Sender, UnboundedReceiver};
+use tokio::sync::Notify;
+use tokio::time::{interval_at, Instant};
+use tokio_stream::wrappers::UnboundedReceiverStream;
+use tokio_util::sync::CancellationToken;
+use tracing::{debug, error, info, instrument, trace, warn};
+
+mod helpers;
+pub(crate) mod pruning;
+pub(crate) mod types;
+
+const MISSING_BLOCKS_CHECK_INTERVAL: Duration = Duration::from_secs(30);
+const MAX_MISSING_BLOCKS_DELAY: Duration = Duration::from_secs(15);
+const MAX_RANGE_SIZE: usize = 30;
+
+#[derive(Debug, Default)]
+struct PendingSync {
+    request_in_flight: HashSet<u32>,
+    queued_requests: VecDeque<Range<u32>>,
+}
+
+impl PendingSync {
+    fn is_empty(&self) -> bool {
+        self.request_in_flight.is_empty() && self.queued_requests.is_empty()
+    }
+}
+
+pub struct BlockProcessor {
+    pruning_options: PruningOptions,
+    cancel: CancellationToken,
+    synced: Arc<Notify>,
+    last_processed_height: u32,
+    last_pruned_height: u32,
+    last_processed_at: Instant,
+    pending_sync: PendingSync,
+    queued_blocks: BTreeMap<u32, BlockToProcess>,
+
+    rpc_client: RpcClient,
+    incoming: UnboundedReceiverStream<BlockToProcess>,
+    block_requester: Sender<BlockRequest>,
+    storage: ScraperStorage,
+
+    // future work: rather than sending each msg to every msg module,
+    // let them subscribe based on `type_url` inside the message itself
+    // (like "/cosmwasm.wasm.v1.MsgExecuteContract")
+    block_modules: Vec<Box<dyn BlockModule + Send>>,
+    tx_modules: Vec<Box<dyn TxModule + Send>>,
+    msg_modules: Vec<Box<dyn MsgModule + Send>>,
+}
+
+impl BlockProcessor {
+    pub async fn new(
+        pruning_options: PruningOptions,
+        cancel: CancellationToken,
+        synced: Arc<Notify>,
+        incoming: UnboundedReceiver<BlockToProcess>,
+        block_requester: Sender<BlockRequest>,
+        storage: ScraperStorage,
+        rpc_client: RpcClient,
+    ) -> Result<Self, ScraperError> {
+        let last_processed = storage.get_last_processed_height().await?;
+        let last_processed_height = last_processed.try_into().unwrap_or_default();
+
+        let last_pruned = storage.get_pruned_height().await?;
+        let last_pruned_height = last_pruned.try_into().unwrap_or_default();
+
+        Ok(BlockProcessor {
+            pruning_options,
+            cancel,
+            synced,
+            last_processed_height,
+            last_pruned_height,
+            last_processed_at: Instant::now(),
+            pending_sync: Default::default(),
+            queued_blocks: Default::default(),
+            rpc_client,
+            incoming: incoming.into(),
+            block_requester,
+            storage,
+            block_modules: vec![],
+            tx_modules: vec![],
+            msg_modules: vec![],
+        })
+    }
+
+    pub fn with_pruning(mut self, pruning_options: PruningOptions) -> Self {
+        self.pruning_options = pruning_options;
+        self
+    }
+
+    pub(super) async fn process_block(
+        &mut self,
+        block: BlockToProcess,
+    ) -> Result<(), ScraperError> {
+        info!("processing block at height {}", block.height);
+
+        let full_info = self.rpc_client.try_get_full_details(block).await?;
+
+        debug!(
+            "this block has {} transaction(s)",
+            full_info.transactions.len()
+        );
+        for tx in &full_info.transactions {
+            debug!("{} has {} message(s)", tx.hash, tx.tx.body.messages.len());
+            for (index, msg) in tx.tx.body.messages.iter().enumerate() {
+                debug!("{index}: {:?}", msg.type_url)
+            }
+        }
+
+        // process the entire block as a transaction so that if anything fails,
+        // we won't end up with a corrupted storage.
+        let mut tx = self.storage.begin_processing_tx().await?;
+
+        persist_block(&full_info, &mut tx).await?;
+
+        // let the modules do whatever they want
+        // the ones wanting the full block:
+        for block_module in &mut self.block_modules {
+            block_module.handle_block(&full_info, &mut tx).await?;
+        }
+
+        // the ones wanting transactions:
+        for block_tx in full_info.transactions {
+            for tx_module in &mut self.tx_modules {
+                tx_module.handle_tx(&block_tx, &mut tx).await?;
+            }
+            // the ones concerned with individual messages
+            for (index, msg) in block_tx.tx.body.messages.iter().enumerate() {
+                for msg_module in &mut self.msg_modules {
+                    msg_module
+                        .handle_msg(index, msg, &block_tx, &mut tx)
+                        .await?
+                }
+            }
+        }
+
+        let commit_start = Instant::now();
+        tx.commit()
+            .await
+            .map_err(|source| ScraperError::StorageTxCommitFailure { source })?;
+        crate::storage::log_db_operation_time("committing processing tx", commit_start);
+
+        self.last_processed_height = full_info.block.header.height.value() as u32;
+        self.last_processed_at = Instant::now();
+        if let Err(err) = self.maybe_prune_storage().await {
+            error!("failed to prune the storage: {err}");
+        }
+
+        Ok(())
+    }
+
+    pub fn set_block_modules(&mut self, modules: Vec<Box<dyn BlockModule + Send>>) {
+        self.block_modules = modules;
+    }
+
+    pub fn set_tx_modules(&mut self, modules: Vec<Box<dyn TxModule + Send>>) {
+        self.tx_modules = modules;
+    }
+
+    pub fn set_msg_modules(&mut self, modules: Vec<Box<dyn MsgModule + Send>>) {
+        self.msg_modules = modules;
+    }
+
+    pub(super) fn last_process_height(&self) -> u32 {
+        self.last_processed_height
+    }
+
+    async fn maybe_request_missing_blocks(&mut self) -> Result<(), ScraperError> {
+        // we're still processing, so we're good
+        if self.last_processed_at.elapsed() < MAX_MISSING_BLOCKS_DELAY {
+            debug!("no need to request missing blocks");
+            return Ok(());
+        }
+
+        if self.try_request_pending().await {
+            return Ok(());
+        }
+
+        // TODO: properly fill in the gaps later with BlockRequest::Specific,
+        let request_range = if let Some((next_available, _)) = self.queued_blocks.first_key_value()
+        {
+            self.last_processed_height + 1..*next_available
+        } else {
+            let current_height = self.rpc_client.current_block_height().await? as u32;
+            self.last_processed_height + 1..current_height + 1
+        };
+
+        self.request_missing_blocks(request_range).await?;
+
+        Ok(())
+    }
+
+    async fn request_missing_blocks(
+        &mut self,
+        request_range: Range<u32>,
+    ) -> Result<(), ScraperError> {
+        let request_range = if request_range.len() > MAX_RANGE_SIZE {
+            let mut split = split_request_range(request_range);
+
+            // SAFETY: we know that after the split of a non-empty range we have AT LEAST one value
+            #[allow(clippy::unwrap_used)]
+            let first = split.pop_front().unwrap();
+            self.pending_sync.queued_requests = split;
+            self.pending_sync.request_in_flight = first.clone().collect();
+
+            first
+        } else {
+            request_range
+        };
+
+        self.send_blocks_request(request_range).await
+    }
+
+    // technically we're not mutating self here,
+    // but we need it to help the compiler figure out the future is `Send`
+    async fn send_blocks_request(&mut self, request_range: Range<u32>) -> Result<(), ScraperError> {
+        debug!("requesting missing blocks: {request_range:?}");
+
+        self.block_requester
+            .send(BlockRequest::Range(request_range))
+            .await?;
+        Ok(())
+    }
+
+    #[instrument(skip(self))]
+    async fn prune_storage(&mut self) -> Result<(), ScraperError> {
+        let keep_recent = self.pruning_options.strategy_keep_recent();
+        let last_to_keep = self.last_processed_height - keep_recent;
+
+        info!(
+            keep_recent,
+            oldest_to_keep = last_to_keep,
+            "pruning the storage"
+        );
+
+        let lowest: u32 = self
+            .storage
+            .lowest_block_height()
+            .await?
+            .unwrap_or_default()
+            .try_into()
+            .unwrap_or_default();
+
+        let to_prune = last_to_keep.saturating_sub(lowest);
+        match to_prune {
+            v if v > 1000 => warn!("approximately {v} blocks worth of data will be pruned"),
+            v if v > 100 => info!("approximately {v} blocks worth of data will be pruned"),
+            0 => trace!("no blocks to prune"),
+            v => debug!("approximately {v} blocks worth of data will be pruned"),
+        }
+
+        if to_prune == 0 {
+            self.last_pruned_height = self.last_processed_height;
+            return Ok(());
+        }
+
+        self.storage
+            .prune_storage(last_to_keep, self.last_processed_height)
+            .await?;
+
+        self.last_pruned_height = self.last_processed_height;
+        Ok(())
+    }
+
+    async fn maybe_prune_storage(&mut self) -> Result<(), ScraperError> {
+        debug!("checking for storage pruning");
+
+        if self.pruning_options.strategy.is_nothing() {
+            trace!("the current pruning strategy is 'nothing'");
+            return Ok(());
+        }
+
+        let interval = self.pruning_options.strategy_interval();
+        if self.last_pruned_height + interval <= self.last_processed_height {
+            self.prune_storage().await?;
+        }
+
+        Ok(())
+    }
+
+    async fn next_incoming(&mut self, block: BlockToProcess) {
+        let height = block.height;
+
+        self.pending_sync.request_in_flight.remove(&height);
+
+        if self.last_processed_height == 0 {
+            // this is the first time we've started up the process
+            debug!("setting up initial processing height");
+            self.last_processed_height = height - 1
+        }
+
+        if height <= self.last_processed_height {
+            warn!("we have already processed block for height {height}");
+            return;
+        }
+
+        if self.last_processed_height + 1 != height {
+            if self.queued_blocks.insert(height, block).is_some() {
+                warn!("we have already queued up block for height {height}");
+            }
+            return;
+        }
+
+        if let Err(err) = self.process_block(block).await {
+            error!("failed to process block at height {height}: {err}");
+            return;
+        }
+
+        // process as much as we can from the queue
+        let mut next = height + 1;
+        while let Some(next_block) = self.queued_blocks.remove(&next) {
+            if let Err(err) = self.process_block(next_block).await {
+                error!("failed to process queued-up block at height {next}: {err}")
+            }
+            next += 1;
+        }
+
+        self.try_request_pending().await;
+
+        if self.pending_sync.is_empty() {
+            self.synced.notify_one();
+        }
+    }
+
+    async fn try_request_pending(&mut self) -> bool {
+        if self.pending_sync.request_in_flight.is_empty() {
+            if let Some(next_sync) = self.pending_sync.queued_requests.pop_front() {
+                debug!(
+                    "current request range has been resolved. requesting another bunch of blocks"
+                );
+                if let Err(err) = self.send_blocks_request(next_sync.clone()).await {
+                    error!("failed to request resync blocks: {err}");
+                    self.pending_sync.queued_requests.push_front(next_sync);
+                } else {
+                    self.pending_sync.request_in_flight = next_sync.collect()
+                }
+
+                return true;
+            }
+        }
+
+        false
+    }
+
+    // technically we're not mutating self here,
+    // but we need it to help the compiler figure out the future is `Send`
+    async fn startup_resync(&mut self) -> Result<(), ScraperError> {
+        assert!(self.pending_sync.is_empty());
+
+        self.maybe_prune_storage().await?;
+
+        let latest_block = self.rpc_client.current_block_height().await? as u32;
+
+        if latest_block > self.last_processed_height && self.last_processed_height != 0 {
+            // in case we were offline for a while,
+            // make sure we don't request blocks we'd have to prune anyway
+            let keep_recent = self.pruning_options.strategy_keep_recent();
+            let last_to_keep = latest_block - keep_recent;
+            self.last_processed_height = max(self.last_processed_height, last_to_keep);
+
+            let request_range = self.last_processed_height + 1..latest_block + 1;
+            info!("we need to request {request_range:?} to resync");
+            self.request_missing_blocks(request_range).await?;
+        }
+
+        Ok(())
+    }
+
+    pub(crate) async fn run(&mut self) {
+        info!("starting processing loop");
+
+        // sure, we could be more efficient and reset it on every processed block,
+        // but the overhead is so minimal that it doesn't matter
+        let mut missing_check_interval = interval_at(
+            Instant::now().add(MISSING_BLOCKS_CHECK_INTERVAL),
+            MISSING_BLOCKS_CHECK_INTERVAL,
+        );
+
+        if let Err(err) = self.startup_resync().await {
+            error!("failed to perform startup sync: {err}");
+            self.cancel.cancel();
+            return;
+        };
+
+        loop {
+            tokio::select! {
+                _ = self.cancel.cancelled() => {
+                    info!("received cancellation token");
+                    break
+                }
+                _ = missing_check_interval.tick() => {
+                    if let Err(err) = self.maybe_request_missing_blocks().await {
+                        error!("failed to request missing blocks: {err}")
+                    }
+                }
+                block = self.incoming.next() => {
+                    match block {
+                        Some(block) => self.next_incoming(block).await,
+                        None => {
+                            warn!("stopped receiving new blocks");
+                            self.cancel.cancel();
+                            break
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

common/nyxd-scraper-psql/src/block_processor/pruning.rs

@@ -0,0 +1,122 @@
+// Copyright 2024 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::error::ScraperError;
+use serde::{Deserialize, Serialize};
+
+pub const DEFAULT_PRUNING_KEEP_RECENT: u32 = 362880;
+pub const DEFAULT_PRUNING_INTERVAL: u32 = 10;
+pub const EVERYTHING_PRUNING_KEEP_RECENT: u32 = 2;
+pub const EVERYTHING_PRUNING_INTERVAL: u32 = 10;
+
+/// We follow cosmos-sdk pruning strategies for convenience’s sake.
+#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum PruningStrategy {
+    /// 'Default' strategy defines a pruning strategy where the last 362880 heights are
+    /// kept where to-be pruned heights are pruned at every 10th height.
+    /// The last 362880 heights are kept(approximately 3.5 weeks worth of state) assuming the typical
+    /// block time is 6s. If these values do not match the applications' requirements, use the "custom" option.
+    #[default]
+    Default,
+
+    /// 'Everything' strategy defines a pruning strategy where all committed heights are
+    /// deleted, storing only the current height and last 2 states. To-be pruned heights are
+    /// pruned at every 10th height.
+    Everything,
+
+    /// 'Nothing' strategy defines a pruning strategy where all heights are kept on disk.
+    Nothing,
+
+    /// 'Custom' strategy defines a pruning strategy where the user specifies the pruning.
+    Custom,
+}
+
+impl PruningStrategy {
+    pub fn is_custom(&self) -> bool {
+        matches!(self, PruningStrategy::Custom)
+    }
+
+    pub fn is_nothing(&self) -> bool {
+        matches!(self, PruningStrategy::Nothing)
+    }
+
+    pub fn is_everything(&self) -> bool {
+        matches!(self, PruningStrategy::Everything)
+    }
+}
+
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub struct PruningOptions {
+    /// keep_recent defines how many recent heights to keep on disk.
+    pub keep_recent: u32,
+
+    /// interval defines the frequency of removing the pruned heights from the disk.
+    pub interval: u32,
+
+    /// strategy defines the currently used kind of [PruningStrategy].
+    pub strategy: PruningStrategy,
+}
+
+impl PruningOptions {
+    pub fn validate(&self) -> Result<(), ScraperError> {
+        // if strategy is not set to custom, other options are meaningless since they won't be applied
+        if !self.strategy.is_custom() {
+            return Ok(());
+        }
+
+        if self.interval == 0 {
+            return Err(ScraperError::ZeroPruningInterval);
+        }
+
+        if self.interval < EVERYTHING_PRUNING_INTERVAL {
+            return Err(ScraperError::TooSmallPruningInterval {
+                interval: self.interval,
+            });
+        }
+
+        if self.keep_recent < EVERYTHING_PRUNING_KEEP_RECENT {
+            return Err(ScraperError::TooSmallKeepRecent {
+                keep_recent: self.keep_recent,
+            });
+        }
+
+        Ok(())
+    }
+
+    pub fn nothing() -> Self {
+        PruningOptions {
+            keep_recent: 0,
+            interval: 0,
+            strategy: PruningStrategy::Nothing,
+        }
+    }
+
+    pub fn strategy_interval(&self) -> u32 {
+        match self.strategy {
+            PruningStrategy::Default => DEFAULT_PRUNING_INTERVAL,
+            PruningStrategy::Everything => EVERYTHING_PRUNING_INTERVAL,
+            PruningStrategy::Nothing => 0,
+            PruningStrategy::Custom => self.interval,
+        }
+    }
+
+    pub fn strategy_keep_recent(&self) -> u32 {
+        match self.strategy {
+            PruningStrategy::Default => DEFAULT_PRUNING_KEEP_RECENT,
+            PruningStrategy::Everything => EVERYTHING_PRUNING_KEEP_RECENT,
+            PruningStrategy::Nothing => 0,
+            PruningStrategy::Custom => self.keep_recent,
+        }
+    }
+}
+
+impl Default for PruningOptions {
+    fn default() -> Self {
+        PruningOptions {
+            keep_recent: DEFAULT_PRUNING_KEEP_RECENT,
+            interval: DEFAULT_PRUNING_INTERVAL,
+            strategy: Default::default(),
+        }
+    }
+}

common/nyxd-scraper-psql/src/block_processor/types.rs

@@ -0,0 +1,121 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::error::ScraperError;
+use crate::helpers;
+use tendermint::{abci, block, tx, Block, Hash};
+use tendermint_rpc::endpoint::{block as block_endpoint, block_results, validators};
+use tendermint_rpc::event::{Event, EventData};
+
+// just get all everything out of tx::Response, but parse raw `tx` bytes
+#[derive(Clone, Debug)]
+pub struct ParsedTransactionResponse {
+    /// The hash of the transaction.
+    ///
+    /// Deserialized from a hex-encoded string (there is a discrepancy between
+    /// the format used for the request and the format used for the response in
+    /// the Tendermint RPC).
+    pub hash: Hash,
+
+    pub height: block::Height,
+
+    pub index: u32,
+
+    pub tx_result: abci::types::ExecTxResult,
+
+    pub tx: cosmrs::tx::Tx,
+
+    pub proof: Option<tx::Proof>,
+}
+
+#[derive(Debug)]
+pub struct FullBlockInformation {
+    /// Basic block information, including its signers.
+    pub block: Block,
+
+    /// All of the emitted events alongside any tx results.
+    pub results: block_results::Response,
+
+    /// Validator set for this particular block
+    pub validators: validators::Response,
+
+    /// Transaction results from this particular block
+    pub transactions: Vec<ParsedTransactionResponse>,
+}
+
+impl FullBlockInformation {
+    pub fn ensure_proposer(&self) -> Result<(), ScraperError> {
+        let block_proposer = self.block.header.proposer_address;
+        if !self
+            .validators
+            .validators
+            .iter()
+            .any(|v| v.address == block_proposer)
+        {
+            let proposer = helpers::validator_consensus_address(block_proposer)?;
+            return Err(ScraperError::BlockProposerNotInValidatorSet {
+                height: self.block.header.height.value() as u32,
+                proposer: proposer.to_string(),
+            });
+        }
+        Ok(())
+    }
+}
+
+pub(crate) struct BlockToProcess {
+    pub(crate) height: u32,
+    pub(crate) block: Block,
+}
+
+impl From<Block> for BlockToProcess {
+    fn from(block: Block) -> Self {
+        BlockToProcess {
+            height: block.header.height.value() as u32,
+            block,
+        }
+    }
+}
+
+impl TryFrom<Event> for BlockToProcess {
+    type Error = ScraperError;
+
+    fn try_from(event: Event) -> Result<Self, Self::Error> {
+        let query = event.query.clone();
+
+        // TODO: we're losing `result_begin_block` and `result_end_block` here but maybe that's fine?
+        let maybe_block = match event.data {
+            // we don't care about `NewBlock` until CometBFT 0.38, i.e. until we upgrade to wasmd 0.50
+            EventData::NewBlock { .. } => {
+                return Err(ScraperError::InvalidSubscriptionEvent {
+                    query,
+                    kind: "NewBlock".to_string(),
+                })
+            }
+            EventData::LegacyNewBlock { block, .. } => block,
+            EventData::Tx { .. } => {
+                return Err(ScraperError::InvalidSubscriptionEvent {
+                    query,
+                    kind: "Tx".to_string(),
+                })
+            }
+            EventData::GenericJsonEvent(_) => {
+                return Err(ScraperError::InvalidSubscriptionEvent {
+                    query,
+                    kind: "GenericJsonEvent".to_string(),
+                })
+            }
+        };
+
+        let Some(block) = maybe_block else {
+            return Err(ScraperError::EmptyBlockData { query });
+        };
+
+        Ok((*block).into())
+    }
+}
+
+impl From<block_endpoint::Response> for BlockToProcess {
+    fn from(value: block_endpoint::Response) -> Self {
+        value.block.into()
+    }
+}

common/nyxd-scraper-psql/src/block_requester/mod.rs

@@ -0,0 +1,91 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::block_processor::types::BlockToProcess;
+use crate::error::ScraperError;
+use crate::rpc_client::RpcClient;
+use futures::StreamExt;
+use std::ops::Range;
+use tokio::sync::mpsc::{Receiver, UnboundedSender};
+use tokio_stream::wrappers::ReceiverStream;
+use tokio_util::sync::CancellationToken;
+use tracing::{error, info, instrument, warn};
+
+#[derive(Debug)]
+pub enum BlockRequest {
+    Range(Range<u32>),
+
+    // UNIMPLEMENTED:
+    #[allow(dead_code)]
+    Specific(Vec<u32>),
+}
+
+pub(crate) struct BlockRequester {
+    cancel: CancellationToken,
+    rpc_client: RpcClient,
+    requests: ReceiverStream<BlockRequest>,
+    blocks: UnboundedSender<BlockToProcess>,
+}
+
+impl BlockRequester {
+    pub(crate) fn new(
+        cancel: CancellationToken,
+        rpc_client: RpcClient,
+        requests: Receiver<BlockRequest>,
+        blocks: UnboundedSender<BlockToProcess>,
+    ) -> Self {
+        BlockRequester {
+            cancel,
+            rpc_client,
+            requests: requests.into(),
+            blocks,
+        }
+    }
+
+    async fn request_and_send(&self, height: u32) -> Result<(), ScraperError> {
+        let block = self.rpc_client.get_basic_block_details(height).await?;
+        self.blocks.send(block.into())?;
+        Ok(())
+    }
+
+    async fn request_blocks<I: IntoIterator<Item = u32>>(&self, heights: I) {
+        futures::stream::iter(heights)
+            .for_each_concurrent(4, |height| async move {
+                if let Err(err) = self.request_and_send(height).await {
+                    error!("failed to request block data: {err}")
+                }
+            })
+            .await
+    }
+
+    #[instrument(skip(self))]
+    async fn handle_blocks_request(&self, request: BlockRequest) {
+        info!("received request for missed blocks");
+
+        match request {
+            BlockRequest::Range(range) => self.request_blocks(range).await,
+            BlockRequest::Specific(heights) => self.request_blocks(heights).await,
+        }
+    }
+
+    pub(crate) async fn run(&mut self) {
+        loop {
+            tokio::select! {
+               _ = self.cancel.cancelled() => {
+                    info!("received cancellation token");
+                    break
+                }
+                maybe_request = self.requests.next() => {
+                    match maybe_request {
+                        Some(request) => self.handle_blocks_request(request).await,
+                        None => {
+                            warn!("stopped receiving new requests");
+                            self.cancel.cancel();
+                            break
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

common/nyxd-scraper-psql/src/constants.rs

@@ -0,0 +1,47 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use const_format::concatcp;
+
+// TODO: make those configurable via 'NymNetworkDetails'
+
+// BECH32_PREFIX defines the main SDK Bech32 prefix of an account's address
+pub const BECH32_PREFIX: &str = "n";
+
+// ACCOUNT_PREFIX is the prefix for account keys
+pub const ACCOUNT_PREFIX: &str = "acc";
+// VALIDATOR_PREFIX is the prefix for validator keys
+pub const VALIDATOR_PREFIX: &str = "val";
+// CONSENSUS_PREFIX is the prefix for consensus keys
+pub const CONSENSUS_PREFIX: &str = "cons";
+// PUBKEY_PREFIX is the prefix for public keys
+pub const PUBKEY_PREFIX: &str = "pub";
+// OPERATOR_PREFIX is the prefix for operator keys
+pub const OPERATOR_PREFIX: &str = "oper";
+// ADDRESS_PREFIX is the prefix for addresses
+pub const ADDRESS_PREFIX: &str = "addr";
+
+// BECH32_ACCOUNT_ADDRESS_PREFIX defines the Bech32 prefix of an account's address
+pub const BECH32_ACCOUNT_ADDRESS_PREFIX: &str = BECH32_PREFIX;
+// BECH32_ACCOUNT_PUBKEY_PREFIX defines the Bech32 prefix of an account's public key
+pub const BECH32_ACCOUNT_PUBKEY_PREFIX: &str = concatcp!(BECH32_PREFIX, PUBKEY_PREFIX);
+// BECH32_VALIDATOR_ADDRESS_PREFIX defines the Bech32 prefix of a validator's operator address
+pub const BECH32_VALIDATOR_ADDRESS_PREFIX: &str =
+    concatcp!(BECH32_PREFIX, VALIDATOR_PREFIX, OPERATOR_PREFIX);
+// BECH32_VALIDATOR_PUBKEY_PREFIX defines the Bech32 prefix of a validator's operator public key
+pub const BECH32_VALIDATOR_PUBKEY_PREFIX: &str = concatcp!(
+    BECH32_PREFIX,
+    VALIDATOR_PREFIX,
+    OPERATOR_PREFIX,
+    PUBKEY_PREFIX
+);
+// BECH32_CONSENSUS_ADDRESS_PREFIX defines the Bech32 prefix of a consensus node address
+pub const BECH32_CONSENSUS_ADDRESS_PREFIX: &str =
+    concatcp!(BECH32_PREFIX, VALIDATOR_PREFIX, CONSENSUS_PREFIX);
+// BECH32_CONESNSUS_PUBKEY_PREFIX defines the Bech32 prefix of a consensus node public key
+pub const BECH32_CONESNSUS_PUBKEY_PREFIX: &str = concatcp!(
+    BECH32_PREFIX,
+    VALIDATOR_PREFIX,
+    CONSENSUS_PREFIX,
+    PUBKEY_PREFIX
+);

common/nyxd-scraper-psql/src/error.rs

@@ -0,0 +1,143 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::block_processor::pruning::{
+    EVERYTHING_PRUNING_INTERVAL, EVERYTHING_PRUNING_KEEP_RECENT,
+};
+use tendermint::Hash;
+use thiserror::Error;
+use tokio::sync::mpsc::error::SendError;
+
+#[derive(Debug, Error)]
+pub enum ScraperError {
+    #[error("experienced internal database error: {0}")]
+    InternalDatabaseError(#[from] sqlx::Error),
+
+    #[error("failed to perform startup SQL migration: {0}")]
+    StartupMigrationFailure(#[from] sqlx::migrate::MigrateError),
+
+    #[error("the block scraper is already running")]
+    ScraperAlreadyRunning,
+
+    #[error("failed to establish websocket connection to {url}: {source}")]
+    WebSocketConnectionFailure {
+        url: String,
+        #[source]
+        source: tendermint_rpc::Error,
+    },
+
+    #[error("failed to establish rpc connection to {url}: {source}")]
+    HttpConnectionFailure {
+        url: String,
+        #[source]
+        source: tendermint_rpc::Error,
+    },
+
+    #[error("failed to create chain subscription: {source}")]
+    ChainSubscriptionFailure {
+        #[source]
+        source: tendermint_rpc::Error,
+    },
+
+    #[error("could not obtain basic block information at height: {height}: {source}")]
+    BlockQueryFailure {
+        height: u32,
+        #[source]
+        source: tendermint_rpc::Error,
+    },
+
+    #[error("could not obtain block results information at height: {height}: {source}")]
+    BlockResultsQueryFailure {
+        height: u32,
+        #[source]
+        source: tendermint_rpc::Error,
+    },
+
+    #[error("could not obtain validators information at height: {height}: {source}")]
+    ValidatorsQueryFailure {
+        height: u32,
+        #[source]
+        source: tendermint_rpc::Error,
+    },
+
+    #[error("could not obtain tx results for tx: {hash}: {source}")]
+    TxResultsQueryFailure {
+        hash: Hash,
+        #[source]
+        source: tendermint_rpc::Error,
+    },
+
+    #[error("could not obtain current abci info: {source}")]
+    AbciInfoQueryFailure {
+        #[source]
+        source: tendermint_rpc::Error,
+    },
+
+    #[error("could not parse tx {hash}: {source}")]
+    TxParseFailure {
+        hash: Hash,
+        #[source]
+        source: cosmrs::ErrorReport,
+    },
+
+    #[error("received an invalid chain subscription event of kind {kind} while we were waiting for new block data (query: '{query}')")]
+    InvalidSubscriptionEvent { query: String, kind: String },
+
+    #[error("received block data was empty (query: '{query}')")]
+    EmptyBlockData { query: String },
+
+    #[error("reached maximum number of allowed errors for subscription events")]
+    MaximumWebSocketFailures,
+
+    #[error("failed to begin storage tx: {source}")]
+    StorageTxBeginFailure {
+        #[source]
+        source: sqlx::Error,
+    },
+
+    #[error("failed to commit storage tx: {source}")]
+    StorageTxCommitFailure {
+        #[source]
+        source: sqlx::Error,
+    },
+
+    #[error("failed to send on a closed channel")]
+    ClosedChannelError,
+
+    #[error("failed to parse validator's address: {source}")]
+    MalformedValidatorAddress {
+        #[source]
+        source: eyre::Report,
+    },
+
+    #[error("failed to parse validator's address: {source}")]
+    MalformedValidatorPubkey {
+        #[source]
+        source: eyre::Report,
+    },
+
+    #[error(
+        "could not find the block proposer ('{proposer}') for height {height} in the validator set"
+    )]
+    BlockProposerNotInValidatorSet { height: u32, proposer: String },
+
+    #[error(
+        "could not find validator information for {address}; the validator has signed a commit"
+    )]
+    MissingValidatorInfoCommitted { address: String },
+
+    #[error("pruning.interval must not be set to 0. If you want to disable pruning, select pruning.strategy = \"nothing\"")]
+    ZeroPruningInterval,
+
+    #[error("pruning.interval must not be smaller than {}. got: {interval}. for most aggressive pruning, select pruning.strategy = \"everything\"", EVERYTHING_PRUNING_INTERVAL)]
+    TooSmallPruningInterval { interval: u32 },
+
+    #[error("pruning.keep_recent must not be smaller than {}. got: {keep_recent}. for most aggressive pruning, select pruning.strategy = \"everything\"", EVERYTHING_PRUNING_KEEP_RECENT)]
+    TooSmallKeepRecent { keep_recent: u32 },
+}
+
+impl<T> From<SendError<T>> for ScraperError {
+    fn from(_: SendError<T>) -> Self {
+        ScraperError::ClosedChannelError
+    }
+}

common/nyxd-scraper-psql/src/helpers.rs

@@ -0,0 +1,46 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::block_processor::types::ParsedTransactionResponse;
+use crate::constants::{BECH32_CONESNSUS_PUBKEY_PREFIX, BECH32_CONSENSUS_ADDRESS_PREFIX};
+use crate::error::ScraperError;
+use cosmrs::AccountId;
+use sha2::{Digest, Sha256};
+use tendermint::{account, PublicKey};
+use tendermint::{validator, Hash};
+use tendermint_rpc::endpoint::validators;
+
+pub(crate) fn tx_hash<M: AsRef<[u8]>>(raw_tx: M) -> Hash {
+    Hash::Sha256(Sha256::digest(raw_tx).into())
+}
+
+pub(crate) fn validator_pubkey_to_bech32(pubkey: PublicKey) -> Result<AccountId, ScraperError> {
+    // TODO: this one seem to attach additional prefix to they pubkeys, is that what we want instead maybe?
+    // Ok(pubkey.to_bech32(BECH32_CONESNSUS_PUBKEY_PREFIX))
+    AccountId::new(BECH32_CONESNSUS_PUBKEY_PREFIX, &pubkey.to_bytes())
+        .map_err(|source| ScraperError::MalformedValidatorPubkey { source })
+}
+
+pub(crate) fn validator_consensus_address(id: account::Id) -> Result<AccountId, ScraperError> {
+    AccountId::new(BECH32_CONSENSUS_ADDRESS_PREFIX, id.as_ref())
+        .map_err(|source| ScraperError::MalformedValidatorAddress { source })
+}
+
+pub(crate) fn tx_gas_sum(txs: &[ParsedTransactionResponse]) -> i64 {
+    txs.iter().map(|tx| tx.tx_result.gas_used).sum()
+}
+
+pub(crate) fn validator_info(
+    id: account::Id,
+    validators: &validators::Response,
+) -> Result<&validator::Info, ScraperError> {
+    match validators.validators.iter().find(|v| v.address == id) {
+        Some(info) => Ok(info),
+        None => {
+            let addr = validator_consensus_address(id)?;
+            Err(ScraperError::MissingValidatorInfoCommitted {
+                address: addr.to_string(),
+            })
+        }
+    }
+}

common/nyxd-scraper-psql/src/lib.rs

@@ -0,0 +1,20 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+#![warn(clippy::expect_used)]
+#![warn(clippy::unwrap_used)]
+
+pub(crate) mod block_processor;
+pub(crate) mod block_requester;
+pub mod constants;
+pub mod error;
+pub(crate) mod helpers;
+pub mod modules;
+pub(crate) mod rpc_client;
+pub(crate) mod scraper;
+pub mod storage;
+
+pub use block_processor::pruning::{PruningOptions, PruningStrategy};
+pub use modules::{BlockModule, MsgModule, TxModule};
+pub use scraper::{Config, NyxdScraper};
+pub use storage::models;

common/nyxd-scraper-psql/src/modules/block_module.rs

@@ -0,0 +1,16 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::block_processor::types::FullBlockInformation;
+use crate::error::ScraperError;
+use crate::storage::StorageTransaction;
+use async_trait::async_trait;
+
+#[async_trait]
+pub trait BlockModule {
+    async fn handle_block(
+        &mut self,
+        block: &FullBlockInformation,
+        storage_tx: &mut StorageTransaction,
+    ) -> Result<(), ScraperError>;
+}

common/nyxd-scraper-psql/src/modules/mod.rs

@@ -0,0 +1,10 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+mod block_module;
+mod msg_module;
+mod tx_module;
+
+pub use block_module::BlockModule;
+pub use msg_module::MsgModule;
+pub use tx_module::TxModule;

common/nyxd-scraper-psql/src/modules/msg_module.rs

@@ -0,0 +1,19 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::block_processor::types::ParsedTransactionResponse;
+use crate::error::ScraperError;
+use crate::storage::StorageTransaction;
+use async_trait::async_trait;
+use cosmrs::Any;
+
+#[async_trait]
+pub trait MsgModule {
+    async fn handle_msg(
+        &mut self,
+        index: usize,
+        msg: &Any,
+        tx: &ParsedTransactionResponse,
+        storage_tx: &mut StorageTransaction,
+    ) -> Result<(), ScraperError>;
+}

common/nyxd-scraper-psql/src/modules/tx_module.rs

@@ -0,0 +1,16 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::block_processor::types::ParsedTransactionResponse;
+use crate::error::ScraperError;
+use crate::storage::StorageTransaction;
+use async_trait::async_trait;
+
+#[async_trait]
+pub trait TxModule {
+    async fn handle_tx(
+        &mut self,
+        tx: &ParsedTransactionResponse,
+        storage_tx: &mut StorageTransaction,
+    ) -> Result<(), ScraperError>;
+}

common/nyxd-scraper-psql/src/rpc_client.rs

@@ -0,0 +1,175 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::block_processor::types::{
+    BlockToProcess, FullBlockInformation, ParsedTransactionResponse,
+};
+use crate::error::ScraperError;
+use crate::helpers::tx_hash;
+use futures::future::join3;
+use futures::StreamExt;
+use std::collections::BTreeMap;
+use std::sync::Arc;
+use tendermint::Hash;
+use tendermint_rpc::endpoint::{block, block_results, tx, validators};
+use tendermint_rpc::{Client, HttpClient, Paging};
+use tokio::sync::Mutex;
+use tracing::{debug, instrument};
+use url::Url;
+
+#[derive(Clone)]
+pub struct RpcClient {
+    // right now I don't care about anything nym specific, so a simple http client is sufficient,
+    // once this is inadequate, we can switch to a NyxdClient
+    inner: Arc<HttpClient>,
+}
+
+impl RpcClient {
+    pub fn new(url: &Url) -> Result<Self, ScraperError> {
+        let http_client = HttpClient::new(url.as_str()).map_err(|source| {
+            ScraperError::HttpConnectionFailure {
+                url: url.to_string(),
+                source,
+            }
+        })?;
+
+        Ok(RpcClient {
+            inner: Arc::new(http_client),
+        })
+    }
+
+    #[instrument(skip(self, block), fields(height = block.height))]
+    pub async fn try_get_full_details(
+        &self,
+        block: BlockToProcess,
+    ) -> Result<FullBlockInformation, ScraperError> {
+        debug!("getting complete block details");
+        let height = block.height;
+
+        // make all the http requests concurrently
+        let (results, validators, raw_transactions) = join3(
+            self.get_block_results(height),
+            self.get_validators_details(height),
+            self.get_transaction_results(&block.block.data),
+        )
+        .await;
+
+        let raw_transactions = raw_transactions?;
+        let mut transactions = Vec::with_capacity(raw_transactions.len());
+        for tx in raw_transactions {
+            transactions.push(ParsedTransactionResponse {
+                hash: tx.hash,
+                height: tx.height,
+                index: tx.index,
+                tx_result: tx.tx_result,
+                tx: cosmrs::Tx::from_bytes(&tx.tx).map_err(|source| {
+                    ScraperError::TxParseFailure {
+                        hash: tx.hash,
+                        source,
+                    }
+                })?,
+                proof: tx.proof,
+            })
+        }
+
+        Ok(FullBlockInformation {
+            block: block.block,
+            results: results?,
+            validators: validators?,
+            transactions,
+        })
+    }
+
+    #[instrument(skip(self), err(Display))]
+    pub async fn get_basic_block_details(
+        &self,
+        height: u32,
+    ) -> Result<block::Response, ScraperError> {
+        debug!("getting basic block details");
+
+        self.inner
+            .block(height)
+            .await
+            .map_err(|source| ScraperError::BlockQueryFailure { height, source })
+    }
+
+    #[instrument(skip(self), err(Display))]
+    pub async fn get_block_results(
+        &self,
+        height: u32,
+    ) -> Result<block_results::Response, ScraperError> {
+        debug!("getting block results");
+
+        self.inner
+            .block_results(height)
+            .await
+            .map_err(|source| ScraperError::BlockResultsQueryFailure { height, source })
+    }
+
+    pub(crate) async fn current_block_height(&self) -> Result<u64, ScraperError> {
+        debug!("getting current block height");
+
+        let info = self
+            .inner
+            .abci_info()
+            .await
+            .map_err(|source| ScraperError::AbciInfoQueryFailure { source })?;
+        Ok(info.last_block_height.value())
+    }
+
+    async fn get_transaction_results(
+        &self,
+        raw: &[Vec<u8>],
+    ) -> Result<Vec<tx::Response>, ScraperError> {
+        let ordered_results = Arc::new(Mutex::new(BTreeMap::new()));
+
+        // "Data is just a wrapper for a list of transactions, where transactions are arbitrary byte arrays"
+        // source: https://github.com/tendermint/spec/blob/d46cd7f573a2c6a2399fcab2cde981330aa63f37/spec/core/data_structures.md#data
+        //
+        // I hate that zip as much as you, dear reader, but for some reason the compiler didn't let me remove the `move`
+        futures::stream::iter(
+            raw.iter()
+                .map(tx_hash)
+                .enumerate()
+                .zip(std::iter::repeat(ordered_results.clone())),
+        )
+        .for_each_concurrent(4, |((id, tx_hash), ordered_results)| async move {
+            let res = self.get_transaction_result(tx_hash).await;
+            ordered_results.lock().await.insert(id, res);
+        })
+        .await;
+
+        // safety the futures have completed so we MUST have the only arc reference
+        #[allow(clippy::unwrap_used)]
+        let inner = Arc::into_inner(ordered_results).unwrap().into_inner();
+
+        // BTreeMap is ordered by its keys so we're guaranteed to get txs in correct order
+        inner.into_values().collect()
+    }
+
+    #[instrument(skip(self, tx_hash), fields(tx_hash = %tx_hash), err(Display))]
+    async fn get_transaction_result(&self, tx_hash: Hash) -> Result<tx::Response, ScraperError> {
+        debug!("getting tx results");
+
+        self.inner
+            .tx(tx_hash, false)
+            .await
+            .map_err(|source| ScraperError::TxResultsQueryFailure {
+                hash: tx_hash,
+                source,
+            })
+    }
+
+    #[instrument(skip(self))]
+    pub async fn get_validators_details(
+        &self,
+        height: u32,
+    ) -> Result<validators::Response, ScraperError> {
+        debug!("getting validators set");
+
+        self.inner
+            .validators(height, Paging::All)
+            .await
+            .map_err(|source| ScraperError::ValidatorsQueryFailure { height, source })
+    }
+}

common/nyxd-scraper-psql/src/scraper/mod.rs

@@ -0,0 +1,339 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::block_processor::types::BlockToProcess;
+use crate::block_processor::BlockProcessor;
+use crate::block_requester::{BlockRequest, BlockRequester};
+use crate::error::ScraperError;
+use crate::modules::{BlockModule, MsgModule, TxModule};
+use crate::rpc_client::RpcClient;
+use crate::scraper::subscriber::ChainSubscriber;
+use crate::storage::ScraperStorage;
+use crate::PruningOptions;
+use futures::future::join_all;
+use std::path::PathBuf;
+use std::sync::Arc;
+use tokio::sync::mpsc::{
+    channel, unbounded_channel, Receiver, Sender, UnboundedReceiver, UnboundedSender,
+};
+use tokio::sync::Notify;
+use tokio_util::sync::CancellationToken;
+use tokio_util::task::TaskTracker;
+use tracing::{error, info};
+use url::Url;
+
+mod subscriber;
+
+pub struct Config {
+    /// Url to the websocket endpoint of a validator, for example `wss://rpc.nymtech.net/websocket`
+    pub websocket_url: Url,
+
+    /// Url to the rpc endpoint of a validator, for example `https://rpc.nymtech.net/`
+    pub rpc_url: Url,
+
+    pub database_path: PathBuf,
+
+    pub pruning_options: PruningOptions,
+}
+
+pub struct NyxdScraperBuilder {
+    config: Config,
+
+    block_modules: Vec<Box<dyn BlockModule + Send>>,
+    tx_modules: Vec<Box<dyn TxModule + Send>>,
+    msg_modules: Vec<Box<dyn MsgModule + Send>>,
+}
+
+impl NyxdScraperBuilder {
+    pub async fn build_and_start(self) -> Result<NyxdScraper, ScraperError> {
+        let scraper = NyxdScraper::new(self.config).await?;
+
+        let (processing_tx, processing_rx) = unbounded_channel();
+        let (req_tx, req_rx) = channel(5);
+
+        let rpc_client = RpcClient::new(&scraper.config.rpc_url)?;
+
+        // create the tasks
+        let block_requester = BlockRequester::new(
+            scraper.cancel_token.clone(),
+            rpc_client.clone(),
+            req_rx,
+            processing_tx.clone(),
+        );
+        let mut block_processor = BlockProcessor::new(
+            scraper.config.pruning_options,
+            scraper.cancel_token.clone(),
+            scraper.startup_sync.clone(),
+            processing_rx,
+            req_tx,
+            scraper.storage.clone(),
+            rpc_client,
+        )
+        .await?;
+        block_processor.set_block_modules(self.block_modules);
+        block_processor.set_tx_modules(self.tx_modules);
+        block_processor.set_msg_modules(self.msg_modules);
+
+        let chain_subscriber = ChainSubscriber::new(
+            &scraper.config.websocket_url,
+            scraper.cancel_token.clone(),
+            scraper.task_tracker.clone(),
+            processing_tx,
+        )
+        .await?;
+
+        scraper.start_tasks(block_requester, block_processor, chain_subscriber);
+
+        Ok(scraper)
+    }
+
+    pub fn new(config: Config) -> Self {
+        NyxdScraperBuilder {
+            config,
+            block_modules: vec![],
+            tx_modules: vec![],
+            msg_modules: vec![],
+        }
+    }
+
+    pub fn with_block_module<M: BlockModule + Send + 'static>(mut self, module: M) -> Self {
+        self.block_modules.push(Box::new(module));
+        self
+    }
+
+    pub fn with_tx_module<M: TxModule + Send + 'static>(mut self, module: M) -> Self {
+        self.tx_modules.push(Box::new(module));
+        self
+    }
+
+    pub fn with_msg_module<M: MsgModule + Send + 'static>(mut self, module: M) -> Self {
+        self.msg_modules.push(Box::new(module));
+        self
+    }
+}
+
+pub struct NyxdScraper {
+    config: Config,
+
+    task_tracker: TaskTracker,
+    cancel_token: CancellationToken,
+    startup_sync: Arc<Notify>,
+    pub storage: ScraperStorage,
+    rpc_client: RpcClient,
+}
+
+impl NyxdScraper {
+    pub fn builder(config: Config) -> NyxdScraperBuilder {
+        NyxdScraperBuilder::new(config)
+    }
+
+    pub async fn new(config: Config) -> Result<Self, ScraperError> {
+        config.pruning_options.validate()?;
+        let storage =
+            ScraperStorage::init(&config.database_path.to_str().unwrap_or_default()).await?;
+        let rpc_client = RpcClient::new(&config.rpc_url)?;
+
+        Ok(NyxdScraper {
+            config,
+            task_tracker: TaskTracker::new(),
+            cancel_token: CancellationToken::new(),
+            startup_sync: Arc::new(Default::default()),
+            storage,
+            rpc_client,
+        })
+    }
+
+    fn start_tasks(
+        &self,
+        mut block_requester: BlockRequester,
+        mut block_processor: BlockProcessor,
+        mut chain_subscriber: ChainSubscriber,
+    ) {
+        self.task_tracker
+            .spawn(async move { block_requester.run().await });
+        self.task_tracker
+            .spawn(async move { block_processor.run().await });
+        self.task_tracker
+            .spawn(async move { chain_subscriber.run().await });
+
+        self.task_tracker.close();
+    }
+
+    pub async fn process_single_block(&self, height: u32) -> Result<(), ScraperError> {
+        info!(height = height, "attempting to process a single block");
+        if !self.task_tracker.is_empty() {
+            return Err(ScraperError::ScraperAlreadyRunning);
+        }
+
+        let (_, processing_rx) = unbounded_channel();
+        let (req_tx, _) = channel(5);
+
+        let mut block_processor = self
+            .new_block_processor(req_tx.clone(), processing_rx)
+            .await?
+            .with_pruning(PruningOptions::nothing());
+
+        let block = self.rpc_client.get_basic_block_details(height).await?;
+
+        block_processor.process_block(block.into()).await
+    }
+
+    pub async fn process_block_range(
+        &self,
+        starting_height: Option<u32>,
+        end_height: Option<u32>,
+    ) -> Result<(), ScraperError> {
+        if !self.task_tracker.is_empty() {
+            return Err(ScraperError::ScraperAlreadyRunning);
+        }
+
+        let (_, processing_rx) = unbounded_channel();
+        let (req_tx, _) = channel(5);
+
+        let mut block_processor = self
+            .new_block_processor(req_tx.clone(), processing_rx)
+            .await?
+            .with_pruning(PruningOptions::nothing());
+
+        let current_height = self.rpc_client.current_block_height().await? as u32;
+        let last_processed = block_processor.last_process_height();
+
+        let starting_height = match starting_height {
+            // always attempt to use whatever the user has provided
+            Some(explicit) => explicit,
+            None => {
+                // otherwise, attempt to resume where we last stopped
+                // and if we haven't processed anything, start from the current height
+                if last_processed != 0 {
+                    last_processed
+                } else {
+                    current_height
+                }
+            }
+        };
+
+        let end_height = match end_height {
+            // always attempt to use whatever the user has provided
+            Some(explicit) => explicit,
+            None => {
+                // otherwise, attempt to either go from the start height to the height right
+                // before the final processed block held in the storage (in case there are gaps)
+                // or finally, just go to the current block height
+                if last_processed > starting_height {
+                    last_processed - 1
+                } else {
+                    current_height
+                }
+            }
+        };
+
+        info!(
+            starting_height = starting_height,
+            end_height = end_height,
+            "attempting to process block range"
+        );
+
+        let range = (starting_height..=end_height).collect::<Vec<_>>();
+
+        // the most likely bottleneck here are going to be the chain queries,
+        // so batch multiple requests
+        for batch in range.chunks(4) {
+            let batch_result = join_all(
+                batch
+                    .iter()
+                    .map(|height| self.rpc_client.get_basic_block_details(*height)),
+            )
+            .await;
+            for result in batch_result {
+                match result {
+                    Ok(block) => block_processor.process_block(block.into()).await?,
+                    Err(err) => {
+                        error!("failed to retrieve the block: {err}. stopping...");
+                        return Err(err);
+                    }
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    fn new_block_requester(
+        &self,
+        req_rx: Receiver<BlockRequest>,
+        processing_tx: UnboundedSender<BlockToProcess>,
+    ) -> BlockRequester {
+        BlockRequester::new(
+            self.cancel_token.clone(),
+            self.rpc_client.clone(),
+            req_rx,
+            processing_tx.clone(),
+        )
+    }
+
+    async fn new_block_processor(
+        &self,
+        req_tx: Sender<BlockRequest>,
+        processing_rx: UnboundedReceiver<BlockToProcess>,
+    ) -> Result<BlockProcessor, ScraperError> {
+        BlockProcessor::new(
+            self.config.pruning_options,
+            self.cancel_token.clone(),
+            self.startup_sync.clone(),
+            processing_rx,
+            req_tx,
+            self.storage.clone(),
+            self.rpc_client.clone(),
+        )
+        .await
+    }
+
+    async fn new_chain_subscriber(
+        &self,
+        processing_tx: UnboundedSender<BlockToProcess>,
+    ) -> Result<ChainSubscriber, ScraperError> {
+        ChainSubscriber::new(
+            &self.config.websocket_url,
+            self.cancel_token.clone(),
+            self.task_tracker.clone(),
+            processing_tx,
+        )
+        .await
+    }
+
+    pub async fn start(&self) -> Result<(), ScraperError> {
+        let (processing_tx, processing_rx) = unbounded_channel();
+        let (req_tx, req_rx) = channel(5);
+
+        // create the tasks
+        let block_requester = self.new_block_requester(req_rx, processing_tx.clone());
+        let block_processor = self.new_block_processor(req_tx, processing_rx).await?;
+        let chain_subscriber = self.new_chain_subscriber(processing_tx).await?;
+
+        // spawn them
+        self.start_tasks(block_requester, block_processor, chain_subscriber);
+
+        Ok(())
+    }
+
+    pub async fn wait_for_startup_sync(&self) {
+        info!("awaiting startup chain sync");
+        self.startup_sync.notified().await
+    }
+
+    pub async fn stop(self) {
+        info!("stopping the chain scraper");
+        assert!(self.task_tracker.is_closed());
+
+        self.cancel_token.cancel();
+        self.task_tracker.wait().await
+    }
+
+    pub fn cancel_token(&self) -> CancellationToken {
+        self.cancel_token.clone()
+    }
+
+    pub fn is_cancelled(&self) -> bool {
+        self.cancel_token.is_cancelled()
+    }
+}

common/nyxd-scraper-psql/src/scraper/subscriber.rs

@@ -0,0 +1,241 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::block_processor::types::BlockToProcess;
+use crate::error::ScraperError;
+use tendermint_rpc::event::Event;
+use tendermint_rpc::query::EventType;
+use tendermint_rpc::{SubscriptionClient, WebSocketClient, WebSocketClientDriver};
+use time::{Duration, OffsetDateTime};
+use tokio::sync::mpsc::UnboundedSender;
+use tokio_stream::StreamExt;
+use tokio_util::sync::CancellationToken;
+use tokio_util::task::TaskTracker;
+use tracing::{error, info, warn};
+use url::Url;
+
+const MAX_FAILURES: usize = 10;
+const MAX_RECONNECTION_ATTEMPTS: usize = 8;
+const SOCKET_FAILURE_RESET: Duration = Duration::minutes(15);
+
+pub struct ChainSubscriber {
+    cancel: CancellationToken,
+    task_tracker: TaskTracker,
+
+    block_sender: UnboundedSender<BlockToProcess>,
+
+    websocket_endpoint: Url,
+    websocket_client: WebSocketClient,
+    websocket_driver: Option<WebSocketClientDriver>,
+}
+
+impl ChainSubscriber {
+    pub async fn new(
+        websocket_endpoint: &Url,
+        cancel: CancellationToken,
+        task_tracker: TaskTracker,
+        block_sender: UnboundedSender<BlockToProcess>,
+    ) -> Result<Self, ScraperError> {
+        // sure, we could have just used websocket client entirely, but let's keep the logic for
+        // getting current blocks and historical blocks completely separate with the dual connection
+        let (client, driver) = WebSocketClient::new(websocket_endpoint.as_str())
+            .await
+            .map_err(|source| ScraperError::WebSocketConnectionFailure {
+                url: websocket_endpoint.to_string(),
+                source,
+            })?;
+
+        Ok(ChainSubscriber {
+            cancel,
+            task_tracker,
+            block_sender,
+            websocket_endpoint: websocket_endpoint.clone(),
+            websocket_client: client,
+            websocket_driver: Some(driver),
+        })
+    }
+
+    fn handle_new_event(&mut self, event: Event) -> Result<(), ScraperError> {
+        if let Err(err) = self.block_sender.send(event.try_into()?) {
+            // this error has nothing to do with the websocket or chain
+            error!("failed to send block for processing: {err} - are we shutting down?")
+        }
+        Ok(())
+    }
+
+    async fn remake_connection(&mut self) -> Result<(), ScraperError> {
+        info!(
+            "attempting to reestablish connection to {}",
+            self.websocket_endpoint
+        );
+
+        let (client, driver) = WebSocketClient::new(self.websocket_endpoint.as_str())
+            .await
+            .map_err(|source| ScraperError::WebSocketConnectionFailure {
+                url: self.websocket_endpoint.to_string(),
+                source,
+            })?;
+        self.websocket_client = client;
+        self.websocket_driver = Some(driver);
+
+        info!(
+            "managed to reestablish the websocket connection to {}",
+            self.websocket_endpoint
+        );
+        Ok(())
+    }
+
+    /// Returns whether the method exited due to the cancellation
+    async fn run_chain_subscription(&mut self) -> Result<bool, ScraperError> {
+        let Some(ws_driver) = self.websocket_driver.take() else {
+            error!("the websocket driver hasn't been created - we probably failed to establish the connection");
+            return Ok(false);
+        };
+
+        let driver_cancel = CancellationToken::new();
+        let _driver_guard = driver_cancel.clone().drop_guard();
+
+        // spawn the websocket driver task
+        let driver_handle = {
+            self.task_tracker.reopen();
+            let handle = self
+                .task_tracker
+                .spawn(run_websocket_driver(ws_driver, driver_cancel));
+            self.task_tracker.close();
+            handle
+        };
+        tokio::pin!(driver_handle);
+
+        info!("creating chain subscription");
+        let mut subs = self
+            .websocket_client
+            .subscribe(EventType::NewBlock.into())
+            .await
+            .map_err(|source| ScraperError::ChainSubscriptionFailure { source })?;
+
+        let mut failures = 0;
+
+        info!("starting processing loop");
+        loop {
+            tokio::select! {
+                _ = self.cancel.cancelled() => {
+                    info!("received cancellation token");
+                    // note: `_driver_guard` will get dropped here thus causing cancellation of the driver task
+                    return Ok(true)
+                }
+                _ = &mut driver_handle => {
+                    error!("our websocket driver has finished execution");
+                    return Ok(self.cancel.is_cancelled())
+                }
+                maybe_event = subs.next() => {
+                    let Some(maybe_event) = maybe_event else {
+                        warn!("stopped receiving new events");
+                        return Ok(false)
+                    };
+                    match maybe_event {
+                        Ok(event) => {
+                            if let Err(err) = self.handle_new_event(event) {
+                                error!("failed to process received block: {err}");
+                                failures += 1
+                            } else {
+                                failures = 0;
+                            }
+                        }
+                        Err(err) => {
+                            error!("failed to receive a valid subscription event: {err}");
+                            failures += 1
+                        }
+                    }
+                    if failures >= MAX_FAILURES {
+                        return Ok(false)
+                    }
+                }
+            }
+        }
+    }
+
+    async fn websocket_backoff(&mut self, failure_count: usize) -> bool {
+        const MINIMUM_WAIT_MS: u64 = 10_000;
+        const INCREMENTAL_WAIT_MS: u64 = 30_000;
+
+        let backoff_duration_ms = MINIMUM_WAIT_MS + INCREMENTAL_WAIT_MS * failure_count as u64;
+        info!("going to wait {backoff_duration_ms} ms before re-attempting the reconnection");
+
+        tokio::select! {
+            _ = self.cancel.cancelled() => {
+                info!("received cancellation token");
+                true
+            }
+            _ = tokio::time::sleep(std::time::Duration::from_millis(backoff_duration_ms)) => false,
+        }
+    }
+
+    pub(crate) async fn run(&mut self) -> Result<(), ScraperError> {
+        let _drop_guard = self.cancel.clone().drop_guard();
+        let mut socket_failures = 0;
+        let mut last_failure = OffsetDateTime::now_utc();
+
+        loop {
+            if self.cancel.is_cancelled() {
+                return Ok(());
+            }
+
+            match self.run_chain_subscription().await {
+                Ok(cancelled) => {
+                    if cancelled {
+                        // we're in the middle of a shutdown
+                        return Ok(());
+                    }
+                    socket_failures += 1;
+                }
+                Err(err) => {
+                    error!("failed to create chain subscription: {err}");
+                    socket_failures += 1;
+                }
+            }
+
+            warn!("current socket failure count: {socket_failures}. the last failure was at {last_failure}");
+
+            let now = OffsetDateTime::now_utc();
+
+            // if it's been a while since the last failure, reset the count
+            if now - last_failure > SOCKET_FAILURE_RESET {
+                warn!("resetting the failure count to 1");
+                socket_failures = 1;
+            }
+            last_failure = now;
+
+            if socket_failures >= MAX_RECONNECTION_ATTEMPTS {
+                error!("reached the maximum allowed failure count");
+                return Err(ScraperError::MaximumWebSocketFailures);
+            }
+
+            // BACKOFF
+            let cancelled = self.websocket_backoff(socket_failures).await;
+            if cancelled {
+                return Ok(());
+            }
+
+            if let Err(err) = self.remake_connection().await {
+                error!("failed to re-establish the websocket connection: {err}");
+            }
+        }
+    }
+}
+
+pub async fn run_websocket_driver(driver: WebSocketClientDriver, driver_cancel: CancellationToken) {
+    info!("starting websocket driver");
+    tokio::select! {
+        _ = driver_cancel.cancelled() => {
+            info!("received cancellation token")
+        }
+        res = driver.run() => {
+            match res {
+                Ok(_) => info!("our websocket driver has finished execution"),
+                Err(err) => {
+                    error!("our websocket driver has errored out: {err}");
+                }
+            }
+        }
+    }
+}

common/nyxd-scraper-psql/src/storage/helpers.rs

@@ -0,0 +1,2 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0

common/nyxd-scraper-psql/src/storage/manager.rs

@@ -0,0 +1,597 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use std::vec;
+
+use crate::storage::log_db_operation_time;
+use crate::storage::models::{CommitSignature, Validator};
+use base64;
+use serde_json::Value as JsonValue;
+use sqlx::types::time::OffsetDateTime;
+use sqlx::{Executor, Postgres};
+use tokio::time::Instant;
+use tracing::{instrument, trace};
+
+#[derive(Clone)]
+pub(crate) struct StorageManager {
+    pub(crate) connection_pool: sqlx::PgPool,
+}
+
+impl StorageManager {
+    pub(crate) async fn set_initial_metadata(&self) -> Result<(), sqlx::Error> {
+        if sqlx::query::<Postgres>("SELECT * from metadata")
+            .fetch_optional(&self.connection_pool)
+            .await?
+            .is_none()
+        {
+            sqlx::query("INSERT INTO metadata (id, last_processed_height) VALUES (0, 0)")
+                .execute(&self.connection_pool)
+                .await?;
+        }
+        Ok(())
+    }
+
+    pub(crate) async fn get_lowest_block(&self) -> Result<Option<i64>, sqlx::Error> {
+        trace!("get_lowest_block");
+        let start = Instant::now();
+
+        let maybe_record = sqlx::query!(
+            r#"
+                SELECT height
+                FROM block
+                ORDER BY height ASC
+                LIMIT 1
+            "#,
+        )
+        .fetch_optional(&self.connection_pool)
+        .await?;
+        log_db_operation_time("get_lowest_block", start);
+
+        if let Some(row) = maybe_record {
+            Ok(Some(row.height))
+        } else {
+            Ok(None)
+        }
+    }
+
+    pub(crate) async fn get_first_block_height_after(
+        &self,
+        time: OffsetDateTime,
+    ) -> Result<Option<i64>, sqlx::Error> {
+        trace!("get_first_block_height_after");
+        let start = Instant::now();
+
+        let maybe_record = sqlx::query!(
+            r#"
+                SELECT height
+                FROM block
+                WHERE timestamp > $1
+                ORDER BY timestamp
+                LIMIT 1
+            "#,
+            time
+        )
+        .fetch_optional(&self.connection_pool)
+        .await?;
+        log_db_operation_time("get_first_block_height_after", start);
+
+        if let Some(row) = maybe_record {
+            Ok(Some(row.height))
+        } else {
+            Ok(None)
+        }
+    }
+
+    pub(crate) async fn get_last_block_height_before(
+        &self,
+        time: OffsetDateTime,
+    ) -> Result<Option<i64>, sqlx::Error> {
+        trace!("get_last_block_height_before");
+        let start = Instant::now();
+
+        let maybe_record = sqlx::query!(
+            r#"
+                SELECT height
+                FROM block
+                WHERE timestamp < $1
+                ORDER BY timestamp DESC
+                LIMIT 1
+            "#,
+            time
+        )
+        .fetch_optional(&self.connection_pool)
+        .await?;
+        log_db_operation_time("get_last_block_height_before", start);
+
+        if let Some(row) = maybe_record {
+            Ok(Some(row.height))
+        } else {
+            Ok(None)
+        }
+    }
+
+    pub(crate) async fn get_signed_between(
+        &self,
+        consensus_address: &str,
+        start_height: i64,
+        end_height: i64,
+    ) -> Result<i64, sqlx::Error> {
+        trace!("get_signed_between");
+        let start = Instant::now();
+
+        let count = sqlx::query!(
+            r#"
+                SELECT COUNT(*) as count FROM pre_commit
+                WHERE 
+                    validator_address = $1
+                    AND height >= $2
+                    AND height <= $3
+            "#,
+            consensus_address,
+            start_height,
+            end_height
+        )
+        .fetch_one(&self.connection_pool)
+        .await?
+        .count;
+        log_db_operation_time("get_signed_between", start);
+
+        Ok(count.expect("Could not find the count"))
+    }
+
+    pub(crate) async fn get_precommit(
+        &self,
+        consensus_address: &str,
+        height: i64,
+    ) -> Result<Option<CommitSignature>, sqlx::Error> {
+        trace!("get_precommit");
+        let start = Instant::now();
+
+        let res = sqlx::query_as(
+            r#"
+                SELECT * FROM pre_commit
+                WHERE validator_address = $1
+                AND height = $2
+            "#,
+        )
+        .bind(consensus_address)
+        .bind(height)
+        .fetch_optional(&self.connection_pool)
+        .await?;
+        log_db_operation_time("get_precommit", start);
+
+        Ok(res)
+    }
+
+    pub(crate) async fn get_block_validators(
+        &self,
+        height: i64,
+    ) -> Result<Vec<Validator>, sqlx::Error> {
+        trace!("get_block_validators");
+        let start = Instant::now();
+
+        let res = sqlx::query_as!(
+            Validator,
+            r#"
+                SELECT * FROM validator 
+                WHERE EXISTS (
+                    SELECT 1 FROM pre_commit
+                    WHERE height = $1
+                    AND pre_commit.validator_address = validator.consensus_address
+                )
+            "#,
+            height
+        )
+        .fetch_all(&self.connection_pool)
+        .await?;
+        log_db_operation_time("get_block_validators", start);
+
+        Ok(res)
+    }
+
+    pub(crate) async fn get_validators(&self) -> Result<Vec<Validator>, sqlx::Error> {
+        trace!("get_validators");
+        let start = Instant::now();
+
+        let res = sqlx::query_as("SELECT * FROM validator")
+            .fetch_all(&self.connection_pool)
+            .await?;
+        log_db_operation_time("get_validators", start);
+
+        Ok(res)
+    }
+
+    pub(crate) async fn get_last_processed_height(&self) -> Result<i64, sqlx::Error> {
+        trace!("get_last_processed_height");
+        let start = Instant::now();
+
+        let maybe_record = sqlx::query!(
+            r#"
+                SELECT last_processed_height FROM metadata
+            "#
+        )
+        .fetch_optional(&self.connection_pool)
+        .await?;
+        log_db_operation_time("get_last_processed_height", start);
+
+        if let Some(row) = maybe_record {
+            Ok(row.last_processed_height)
+        } else {
+            Ok(-1)
+        }
+    }
+
+    pub(crate) async fn get_pruned_height(&self) -> Result<i64, sqlx::Error> {
+        trace!("get_pruned_height");
+        let start = Instant::now();
+
+        let maybe_record = sqlx::query!(
+            r#"
+                SELECT last_pruned_height FROM pruning
+            "#
+        )
+        .fetch_optional(&self.connection_pool)
+        .await?;
+
+        log_db_operation_time("get_pruned_height", start);
+
+        if let Some(row) = maybe_record {
+            Ok(row.last_pruned_height)
+        } else {
+            Ok(-1)
+        }
+    }
+}
+
+// make those generic over executor so that they could be performed over connection pool and a tx
+
+#[instrument(skip(executor))]
+pub(crate) async fn insert_validator<'a, E>(
+    consensus_address: String,
+    consensus_pubkey: String,
+    executor: E,
+) -> Result<(), sqlx::Error>
+where
+    E: Executor<'a, Database = Postgres>,
+{
+    trace!("insert_validator");
+    let start = Instant::now();
+
+    sqlx::query!(
+        r#"
+            INSERT INTO validator (consensus_address, consensus_pubkey)
+            VALUES ($1, $2)
+            ON CONFLICT (consensus_address) DO NOTHING
+        "#,
+        consensus_address,
+        consensus_pubkey
+    )
+    .execute(executor)
+    .await?;
+    log_db_operation_time("insert_validator", start);
+
+    Ok(())
+}
+
+#[instrument(skip(executor))]
+pub(crate) async fn insert_block<'a, E>(
+    height: i64,
+    hash: String,
+    num_txs: u32,
+    total_gas: i64,
+    proposer_address: String,
+    timestamp: OffsetDateTime,
+    executor: E,
+) -> Result<(), sqlx::Error>
+where
+    E: Executor<'a, Database = Postgres>,
+{
+    trace!("insert_block");
+    let start = Instant::now();
+
+    sqlx::query!(
+        r#"
+            INSERT INTO block (height, hash, num_txs, total_gas, proposer_address, timestamp)
+            VALUES ($1, $2, $3, $4, $5, $6)
+            ON CONFLICT DO NOTHING
+        "#,
+        height,
+        hash,
+        num_txs as i32,
+        total_gas,
+        proposer_address,
+        timestamp
+    )
+    .execute(executor)
+    .await?;
+    log_db_operation_time("insert_block", start);
+
+    Ok(())
+}
+
+#[instrument(skip(executor))]
+pub(crate) async fn insert_precommit<'a, E>(
+    validator_address: String,
+    height: i64,
+    timestamp: OffsetDateTime,
+    voting_power: i64,
+    proposer_priority: i64,
+    executor: E,
+) -> Result<(), sqlx::Error>
+where
+    E: Executor<'a, Database = Postgres>,
+{
+    trace!("insert_precommit");
+    let start = Instant::now();
+
+    sqlx::query!(
+        r#"
+            INSERT INTO pre_commit (validator_address, height, timestamp, voting_power, proposer_priority)
+            VALUES ($1, $2, $3, $4, $5)
+            ON CONFLICT (validator_address, timestamp) DO NOTHING
+        "#,
+        validator_address,
+        height,
+        timestamp,
+        voting_power,
+        proposer_priority
+    )
+    .execute(executor)
+    .await?;
+    log_db_operation_time("insert_precommit", start);
+
+    Ok(())
+}
+
+#[instrument(skip(executor))]
+#[allow(clippy::too_many_arguments)]
+pub(crate) async fn insert_transaction<'a, E>(
+    hash: String,
+    height: i64,
+    index: i64,
+    success: bool,
+    num_messages: i64,
+    messages: Vec<cosmrs::Any>,
+    memo: String,
+    signatures: Vec<String>,
+    signer_infos: Vec<SignerInfo>,
+    fee: Fee,
+    gas_wanted: i64,
+    gas_used: i64,
+    raw_log: String,
+    executor: E,
+) -> Result<(), sqlx::Error>
+where
+    E: Executor<'a, Database = Postgres>,
+{
+    trace!("insert_transaction");
+    let start = Instant::now();
+
+    sqlx::query!(
+        r#"
+            INSERT INTO transaction (hash, height, "index", success, num_messages, messages, memo, signatures, signer_infos, fee, gas_wanted, gas_used, raw_log)
+            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
+            ON CONFLICT (hash) DO UPDATE
+                SET height = excluded.height,
+                    "index" = excluded."index",
+                    success = excluded.success,
+                    num_messages = excluded.num_messages,
+                    messages = excluded.messages,
+                    memo = excluded.memo,
+                    signatures = excluded.signatures,
+                    signer_infos = excluded.signer_infos,
+                    fee = excluded.fee,
+                    gas_wanted = excluded.gas_wanted,
+                    gas_used = excluded.gas_used,
+                    raw_log = excluded.raw_log
+        "#,
+        hash,
+        height,
+        index as i32,
+        success,
+        num_messages as i32,
+        serde_json::json!(messages.iter().map(|msg| {
+            serde_json::json!({
+                "type_url": msg.type_url,
+                "value": base64::encode(&msg.value)
+            })
+        }).collect::<Vec<_>>()),
+        memo,
+        &signatures,
+        serde_json::json!(signer_infos.iter().map(|info| {
+            serde_json::json!({
+                "public_key": {
+                    "type_url": info.public_key.type_url,
+                    "value": base64::encode(&info.public_key.value)
+                },
+                "mode_info": info.mode_info,
+                "sequence": info.sequence
+            })
+        }).collect::<Vec<_>>()),
+        serde_json::json!({
+            "amount": fee.amount,
+            "gas_limit": fee.gas_limit,
+            "payer": fee.payer,
+            "granter": fee.granter
+        }),
+        gas_wanted,
+        gas_used,
+        raw_log
+    )
+    .execute(executor)
+    .await?;
+    log_db_operation_time("insert_transaction", start);
+
+    Ok(())
+}
+
+#[instrument(skip(executor))]
+pub(crate) async fn insert_message<'a, E>(
+    transaction_hash: String,
+    index: i64,
+    typ: String,
+    value: cosmrs::Any,
+    involved_accounts: Vec<String>,
+    height: i64,
+    executor: E,
+) -> Result<(), sqlx::Error>
+where
+    E: Executor<'a, Database = Postgres>,
+{
+    trace!("insert_message");
+    let start = Instant::now();
+
+    sqlx::query!(
+        r#"
+            INSERT INTO message (transaction_hash, "index", type, value, involved_accounts_addresses, height)
+            VALUES ($1, $2, $3, $4, $5, $6)
+            ON CONFLICT (transaction_hash, "index") DO UPDATE
+                SET height = excluded.height,
+                type = excluded.type,
+                value = excluded.value,
+                involved_accounts_addresses = excluded.involved_accounts_addresses
+        "#,
+            transaction_hash,
+            index,
+            typ,
+            serde_json::json!({
+                "type_url": value.type_url,
+                "value": base64::encode(&value.value)
+            }),
+            &involved_accounts,
+            height
+    )
+    .execute(executor)
+    .await?;
+    log_db_operation_time("insert_message", start);
+
+    Ok(())
+}
+
+#[instrument(skip(executor))]
+pub(crate) async fn update_last_processed<'a, E>(
+    height: i64,
+    executor: E,
+) -> Result<(), sqlx::Error>
+where
+    E: Executor<'a, Database = Postgres>,
+{
+    trace!("update_last_processed");
+    let start = Instant::now();
+
+    sqlx::query!(
+        "UPDATE metadata SET last_processed_height = GREATEST(last_processed_height, $1)",
+        height
+    )
+    .execute(executor)
+    .await?;
+    log_db_operation_time("update_last_processed", start);
+
+    Ok(())
+}
+
+#[instrument(skip(executor))]
+pub(crate) async fn update_last_pruned<'a, E>(height: i64, executor: E) -> Result<(), sqlx::Error>
+where
+    E: Executor<'a, Database = Postgres>,
+{
+    trace!("update_last_pruned");
+    let start = Instant::now();
+
+    sqlx::query!("UPDATE pruning SET last_pruned_height = $1", height)
+        .execute(executor)
+        .await?;
+    log_db_operation_time("update_last_pruned", start);
+
+    Ok(())
+}
+
+pub(crate) async fn prune_blocks<'a, E>(oldest_to_keep: i64, executor: E) -> Result<(), sqlx::Error>
+where
+    E: Executor<'a, Database = Postgres>,
+{
+    trace!("prune_blocks");
+    let start = Instant::now();
+
+    sqlx::query!("DELETE FROM block WHERE height < $1", oldest_to_keep)
+        .execute(executor)
+        .await?;
+    log_db_operation_time("prune_blocks", start);
+
+    Ok(())
+}
+
+pub(crate) async fn prune_pre_commits<'a, E>(
+    oldest_to_keep: i64,
+    executor: E,
+) -> Result<(), sqlx::Error>
+where
+    E: Executor<'a, Database = Postgres>,
+{
+    trace!("prune_pre_commits");
+    let start = Instant::now();
+
+    sqlx::query!("DELETE FROM pre_commit WHERE height < $1", oldest_to_keep)
+        .execute(executor)
+        .await?;
+    log_db_operation_time("prune_pre_commits", start);
+
+    Ok(())
+}
+
+pub(crate) async fn prune_transactions<'a, E>(
+    oldest_to_keep: i64,
+    executor: E,
+) -> Result<(), sqlx::Error>
+where
+    E: Executor<'a, Database = Postgres>,
+{
+    trace!("prune_transactions");
+    let start = Instant::now();
+
+    sqlx::query!("DELETE FROM transaction WHERE height < $1", oldest_to_keep)
+        .execute(executor)
+        .await?;
+    log_db_operation_time("prune_transactions", start);
+
+    Ok(())
+}
+
+pub(crate) async fn prune_messages<'a, E>(
+    oldest_to_keep: i64,
+    executor: E,
+) -> Result<(), sqlx::Error>
+where
+    E: Executor<'a, Database = Postgres>,
+{
+    trace!("prune_messages");
+    let start = Instant::now();
+
+    sqlx::query!("DELETE FROM message WHERE height < $1", oldest_to_keep)
+        .execute(executor)
+        .await?;
+    log_db_operation_time("prune_messages", start);
+
+    Ok(())
+}
+
+#[derive(Debug, Clone, serde::Serialize)]
+pub struct Coin {
+    pub denom: String,
+    pub amount: String,
+}
+
+#[derive(Debug, Clone)]
+pub struct SignerInfo {
+    pub public_key: cosmrs::Any,
+    pub mode_info: String,
+    pub sequence: i64,
+}
+
+#[derive(Debug, Clone)]
+pub struct Fee {
+    pub amount: Vec<Coin>,
+    pub gas_limit: i64,
+    pub payer: String,
+    pub granter: String,
+}

common/nyxd-scraper-psql/src/storage/mod.rs

@@ -0,0 +1,420 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::block_processor::types::{FullBlockInformation, ParsedTransactionResponse};
+use crate::error::ScraperError;
+use crate::storage::manager::{
+    insert_block, insert_message, insert_precommit, insert_transaction,
+    insert_validator, prune_blocks, prune_messages, prune_pre_commits, prune_transactions,
+    update_last_processed, update_last_pruned, StorageManager, Coin, Fee, SignerInfo,
+};
+use crate::storage::models::{CommitSignature, Validator};
+use sqlx::types::time::OffsetDateTime;
+use sqlx::{ConnectOptions, Postgres, Transaction};
+use std::str::FromStr;
+use tendermint::block::{Commit, CommitSig};
+use tendermint::Block;
+use tendermint_rpc::endpoint::validators;
+use tokio::time::Instant;
+use tracing::{debug, error, info, instrument, trace, warn};
+
+mod helpers;
+mod manager;
+pub mod models;
+
+pub type StorageTransaction = Transaction<'static, Postgres>;
+
+#[derive(Clone)]
+pub struct ScraperStorage {
+    pub(crate) manager: StorageManager,
+}
+
+pub(crate) fn log_db_operation_time(op_name: &str, start_time: Instant) {
+    let elapsed = start_time.elapsed();
+    let formatted = humantime::format_duration(elapsed);
+
+    match elapsed.as_millis() {
+        v if v > 10000 => error!("{op_name} took {formatted} to execute"),
+        v if v > 1000 => warn!("{op_name} took {formatted} to execute"),
+        v if v > 100 => info!("{op_name} took {formatted} to execute"),
+        v if v > 10 => debug!("{op_name} took {formatted} to execute"),
+        _ => trace!("{op_name} took {formatted} to execute"),
+    }
+}
+
+impl ScraperStorage {
+    #[instrument]
+    pub async fn init(database_url: &str) -> Result<Self, ScraperError> {
+        let mut opts = sqlx::postgres::PgConnectOptions::from_str(database_url)
+            .map_err(|err| ScraperError::InternalDatabaseError(err))?;
+
+        // TODO: do we want auto_vacuum ?
+
+        opts.disable_statement_logging();
+
+        let connection_pool = match sqlx::PgPool::connect_with(opts).await {
+            Ok(db) => db,
+            Err(err) => {
+                error!("Failed to connect to PostgreSQL database: {err}");
+                return Err(err.into());
+            }
+        };
+
+        if let Err(err) = sqlx::migrate!("./sql_migrations")
+            .run(&connection_pool)
+            .await
+        {
+            error!("Failed to initialize SQLx database: {err}");
+            return Err(err.into());
+        }
+
+        info!("Database migration finished!");
+
+        let manager = StorageManager { connection_pool };
+        manager.set_initial_metadata().await?;
+
+        let storage = ScraperStorage { manager };
+
+        Ok(storage)
+    }
+
+    #[instrument(skip(self))]
+    pub async fn prune_storage(
+        &self,
+        oldest_to_keep: u32,
+        current_height: u32,
+    ) -> Result<(), ScraperError> {
+        let start = Instant::now();
+
+        let mut tx = self.begin_processing_tx().await?;
+
+        prune_messages(oldest_to_keep.into(), &mut tx).await?;
+        prune_transactions(oldest_to_keep.into(), &mut tx).await?;
+        prune_pre_commits(oldest_to_keep.into(), &mut tx).await?;
+        prune_blocks(oldest_to_keep.into(), &mut tx).await?;
+        update_last_pruned(current_height.into(), &mut tx).await?;
+
+        let commit_start = Instant::now();
+        tx.commit()
+            .await
+            .map_err(|source| ScraperError::StorageTxCommitFailure { source })?;
+        log_db_operation_time("committing pruning tx", commit_start);
+
+        log_db_operation_time("pruning storage", start);
+        Ok(())
+    }
+
+    #[instrument(skip_all)]
+    pub async fn begin_processing_tx(&self) -> Result<StorageTransaction, ScraperError> {
+        debug!("starting storage tx");
+        self.manager
+            .connection_pool
+            .begin()
+            .await
+            .map_err(|source| ScraperError::StorageTxBeginFailure { source })
+    }
+
+    pub async fn lowest_block_height(&self) -> Result<Option<i64>, ScraperError> {
+        Ok(self.manager.get_lowest_block().await?)
+    }
+
+    pub async fn get_first_block_height_after(
+        &self,
+        time: OffsetDateTime,
+    ) -> Result<Option<i64>, ScraperError> {
+        Ok(self.manager.get_first_block_height_after(time).await?)
+    }
+
+    pub async fn get_last_block_height_before(
+        &self,
+        time: OffsetDateTime,
+    ) -> Result<Option<i64>, ScraperError> {
+        Ok(self.manager.get_last_block_height_before(time).await?)
+    }
+
+    pub async fn get_blocks_between(
+        &self,
+        start_time: OffsetDateTime,
+        end_time: OffsetDateTime,
+    ) -> Result<i64, ScraperError> {
+        let Some(block_start) = self.get_first_block_height_after(start_time).await? else {
+            return Ok(0);
+        };
+        let Some(block_end) = self.get_last_block_height_before(end_time).await? else {
+            return Ok(0);
+        };
+
+        Ok(block_end - block_start)
+    }
+
+    pub async fn get_signed_between(
+        &self,
+        consensus_address: &str,
+        start_height: i64,
+        end_height: i64,
+    ) -> Result<i64, ScraperError> {
+        Ok(self
+            .manager
+            .get_signed_between(consensus_address, start_height, end_height)
+            .await?)
+    }
+
+    pub async fn get_signed_between_times(
+        &self,
+        consensus_address: &str,
+        start_time: OffsetDateTime,
+        end_time: OffsetDateTime,
+    ) -> Result<i64, ScraperError> {
+        let Some(block_start) = self.get_first_block_height_after(start_time).await? else {
+            return Ok(0);
+        };
+        let Some(block_end) = self.get_last_block_height_before(end_time).await? else {
+            return Ok(0);
+        };
+
+        self.get_signed_between(consensus_address, block_start, block_end)
+            .await
+    }
+
+    pub async fn get_precommit(
+        &self,
+        consensus_address: &str,
+        height: i64,
+    ) -> Result<Option<CommitSignature>, ScraperError> {
+        Ok(self
+            .manager
+            .get_precommit(consensus_address, height)
+            .await?)
+    }
+
+    pub async fn get_block_signers(&self, height: i64) -> Result<Vec<Validator>, ScraperError> {
+        Ok(self.manager.get_block_validators(height).await?)
+    }
+
+    pub async fn get_all_known_validators(&self) -> Result<Vec<Validator>, ScraperError> {
+        Ok(self.manager.get_validators().await?)
+    }
+
+    pub async fn get_last_processed_height(&self) -> Result<i64, ScraperError> {
+        Ok(self.manager.get_last_processed_height().await?)
+    }
+
+    pub async fn get_pruned_height(&self) -> Result<i64, ScraperError> {
+        Ok(self.manager.get_pruned_height().await?)
+    }
+}
+
+pub async fn persist_block(
+    block: &FullBlockInformation,
+    tx: &mut StorageTransaction,
+) -> Result<(), ScraperError> {
+    let total_gas = crate::helpers::tx_gas_sum(&block.transactions);
+
+    // SANITY CHECK: make sure the block proposer is present in the validator set
+    block.ensure_proposer()?;
+
+    // persist validators
+    persist_validators(&block.validators, tx).await?;
+
+    // persist block data
+    persist_block_data(&block.block, total_gas, tx).await?;
+
+    // persist commits
+    if let Some(commit) = &block.block.last_commit {
+        persist_commits(commit, &block.validators, tx).await?;
+    } else {
+        warn!("no commits for block {}", block.block.header.height)
+    }
+
+    // persist txs
+    for chain_tx in &block.transactions {
+        persist_transaction(chain_tx, tx).await?;
+    }
+
+    update_last_processed(block.block.header.height.into(), tx).await?;
+
+    Ok(())
+}
+
+async fn persist_validators(
+    validators: &validators::Response,
+    tx: &mut StorageTransaction,
+) -> Result<(), ScraperError> {
+    debug!("persisting {} validators", validators.total);
+    for validator in &validators.validators {
+        let consensus_address = crate::helpers::validator_consensus_address(validator.address)?;
+        let consensus_pubkey = crate::helpers::validator_pubkey_to_bech32(validator.pub_key)?;
+
+        insert_validator(
+            consensus_address.to_string(),
+            consensus_pubkey.to_string(),
+            &mut *tx,
+        )
+        .await?;
+    }
+
+    Ok(())
+}
+
+async fn persist_block_data(
+    block: &Block,
+    total_gas: i64,
+    tx: &mut StorageTransaction,
+) -> Result<(), ScraperError> {
+    let proposer_address =
+        crate::helpers::validator_consensus_address(block.header.proposer_address)?.to_string();
+
+    insert_block(
+        block.header.height.into(),
+        block.header.hash().to_string(),
+        block.data.len() as u32,
+        total_gas,
+        proposer_address,
+        block.header.time.into(),
+        tx,
+    )
+    .await?;
+    Ok(())
+}
+
+async fn persist_commits(
+    commits: &Commit,
+    validators: &validators::Response,
+    tx: &mut StorageTransaction,
+) -> Result<(), ScraperError> {
+    debug!("persisting up to {} commits", commits.signatures.len());
+    let height: i64 = commits.height.into();
+
+    for commit_sig in &commits.signatures {
+        let (validator_id, timestamp, signature) = match commit_sig {
+            CommitSig::BlockIdFlagAbsent => {
+                trace!("absent signature");
+                continue;
+            }
+            CommitSig::BlockIdFlagCommit {
+                validator_address,
+                timestamp,
+                signature,
+            } => (validator_address, timestamp, signature),
+            CommitSig::BlockIdFlagNil {
+                validator_address,
+                timestamp,
+                signature,
+            } => (validator_address, timestamp, signature),
+        };
+
+        let validator = crate::helpers::validator_info(*validator_id, validators)?;
+        let validator_address = crate::helpers::validator_consensus_address(*validator_id)?;
+
+        if signature.is_none() {
+            warn!("empty signature for {validator_address} at height {height}");
+            continue;
+        }
+
+        insert_precommit(
+            validator_address.to_string(),
+            height,
+            (*timestamp).into(),
+            validator.power.into(),
+            validator.proposer_priority.value(),
+            &mut *tx,
+        )
+        .await?;
+    }
+
+    Ok(())
+}
+
+async fn persist_transaction(
+    chain_tx: &ParsedTransactionResponse,
+    tx: &mut Transaction<'_, Postgres>,
+) -> Result<(), ScraperError> {
+    let signer_infos = chain_tx
+        .tx
+        .auth_info
+        .signer_infos
+        .iter()
+        .map(|info| SignerInfo {
+            public_key: info.public_key.clone()
+                .map(|pk| pk.into())
+                .unwrap_or_default(),
+            mode_info: serde_json::to_string(&serde_json::json!({
+                "single": {
+                    "mode": match &info.mode_info {
+                        cosmrs::tx::ModeInfo::Single(s) => s.mode as u32,
+                        _ => 0,
+                    }
+                }
+            })).unwrap_or_default(),
+            sequence: info.sequence as i64,
+        })
+        .collect();
+
+    let fee = Fee {
+        amount: chain_tx
+            .tx
+            .auth_info
+            .fee
+            .amount
+            .iter()
+            .map(|coin| Coin {
+                denom: coin.denom.to_string(),
+                amount: coin.amount.to_string(),
+            })
+            .collect(),
+        gas_limit: chain_tx.tx.auth_info.fee.gas_limit as i64,
+        payer: chain_tx.tx.auth_info.fee.payer
+            .clone()
+            .map(|id| id.to_string())
+            .unwrap_or_default(),
+        granter: chain_tx.tx.auth_info.fee.granter
+            .clone()
+            .map(|id| id.to_string())
+            .unwrap_or_default(),
+    };
+
+    let signatures = chain_tx.tx.signatures
+        .iter()
+        .map(|sig| base64::encode(sig))
+        .collect();
+
+    insert_transaction(
+        chain_tx.hash.to_string(),
+        chain_tx.height.into(),
+        chain_tx.index as i64,
+        chain_tx.tx_result.code.is_ok(),
+        chain_tx.tx.body.messages.len() as i64,
+        chain_tx.tx.body.messages.clone(),
+        chain_tx.tx.body.memo.clone(),
+        signatures,
+        signer_infos,
+        fee,
+        chain_tx.tx_result.gas_wanted,
+        chain_tx.tx_result.gas_used,
+        chain_tx.tx_result.log.clone(),
+        &mut *tx,
+    )
+    .await?;
+
+    for (index, msg) in chain_tx.tx.body.messages.iter().enumerate() {
+        let involved_accounts = extract_involved_accounts(msg);
+        insert_message(
+            chain_tx.hash.to_string(),
+            index as i64,
+            msg.type_url.clone(),
+            msg.clone(),
+            involved_accounts,
+            chain_tx.height.into(),
+            &mut *tx,
+        )
+        .await?;
+    }
+
+    Ok(())
+}
+
+fn extract_involved_accounts(_msg: &cosmrs::Any) -> Vec<String> {
+    // This is a placeholder implementation
+    // TODO: Implement proper account extraction based on message type
+    vec![]
+}

common/nyxd-scraper-psql/src/storage/models.rs

@@ -0,0 +1,30 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use sqlx::types::time::OffsetDateTime;
+use sqlx::FromRow;
+
+#[derive(Debug, Clone, Eq, PartialEq, Hash, FromRow)]
+pub struct Validator {
+    pub consensus_address: String,
+    pub consensus_pubkey: String,
+}
+
+#[derive(Debug, Clone, FromRow)]
+pub struct Block {
+    pub height: i64,
+    pub hash: String,
+    pub num_txs: u32,
+    pub total_gas: i64,
+    pub proposer_address: String,
+    pub timestamp: OffsetDateTime,
+}
+
+#[derive(Debug, Clone, FromRow)]
+pub struct CommitSignature {
+    pub height: i64,
+    pub validator_address: String,
+    pub voting_power: i64,
+    pub proposer_priority: i64,
+    pub timestamp: OffsetDateTime,
+}

common/nyxd-scraper/Cargo.toml

@@ -12,14 +12,17 @@ license.workspace = true
 
 [dependencies]
 async-trait.workspace = true
+base64.workspace = true
 const_format = { workspace = true }
 cosmrs.workspace = true
+chrono = {workspace = true}
 eyre = { workspace = true }
 futures.workspace = true
 humantime = { workspace = true }
 sha2 = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
-sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate", "time"] }
+serde_json = {workspace = true}
+sqlx = { workspace = true, features = ["runtime-tokio-rustls", "postgres", "macros", "migrate", "time", "json"] }
 tendermint.workspace = true
 tendermint-rpc = { workspace = true, features = ["websocket-client", "http-client"] }
 thiserror.workspace = true
@@ -36,5 +39,5 @@ url.workspace = true
 
 
 [build-dependencies]
-sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate"] }
+sqlx = { workspace = true, features = ["runtime-tokio-rustls", "postgres", "macros", "migrate"] }
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }

common/nyxd-scraper/build.rs

@@ -1,28 +1,41 @@
-// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
-// SPDX-License-Identifier: Apache-2.0
-
 #[tokio::main]
 async fn main() {
-    use sqlx::{Connection, SqliteConnection};
-    use std::env;
+    use sqlx::{Connection, Executor, PgConnection};
 
-    let out_dir = env::var("OUT_DIR").unwrap();
-    let database_path = format!("{out_dir}/scraper-example.sqlite");
+    const POSTGRES_USER: &str = "nym";
+    const POSTGRES_PASSWORD: &str = "password123";
+    const POSTGRES_DB: &str = "nyxd_scraper";
 
-    let mut conn = SqliteConnection::connect(&format!("sqlite://{database_path}?mode=rwc"))
+    let admin_url = format!(
+        "postgres://{}:{}@localhost:5432/postgres",
+        POSTGRES_USER, POSTGRES_PASSWORD
+    );
+    // Connect to postgres to create test database
+    let database_url =
+        format!("postgres://{POSTGRES_USER}:{POSTGRES_PASSWORD}@localhost:5432/{POSTGRES_DB}");
+
+    let mut conn = PgConnection::connect(&admin_url)
         .await
-        .expect("Failed to create SQLx database connection");
+        .expect("Failed to connect to Postgres");
 
+    conn.execute(format!(r#"DROP DATABASE IF EXISTS {}"#, POSTGRES_DB).as_str())
+        .await
+        .expect("Failed to drop test database");
+
+    conn.execute(format!(r#"CREATE DATABASE {}"#, POSTGRES_DB).as_str())
+        .await
+        .expect("Failed to create test database");
+
+    let mut test_conn = PgConnection::connect(&database_url)
+        .await
+        .expect("Failed to connect to test database");
+
+    // Run migrations
     sqlx::migrate!("./sql_migrations")
-        .run(&mut conn)
+        .run(&mut test_conn)
         .await
         .expect("Failed to perform SQLx migrations");
 
-    #[cfg(target_family = "unix")]
-    println!("cargo:rustc-env=DATABASE_URL=sqlite://{}", &database_path);
-
-    #[cfg(target_family = "windows")]
-    // for some strange reason we need to add a leading `/` to the windows path even though it's
-    // not a valid windows path... but hey, it works...
-    println!("cargo:rustc-env=DATABASE_URL=sqlite:///{}", &database_path);
+    // Set the database URL as an environment variable
+    println!("cargo:rustc-env=DATABASE_URL={}", database_url);
 }

common/nyxd-scraper/sql_migrations/01_metadata.sql

@@ -2,9 +2,7 @@
  * Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
  * SPDX-License-Identifier: Apache-2.0
  */
-
-CREATE TABLE METADATA
-(
-    id                    INTEGER PRIMARY KEY CHECK (id = 0),
-    last_processed_height INTEGER NOT NULL
+CREATE TABLE METADATA (
+    id INTEGER PRIMARY KEY CHECK (id = 0),
+    last_processed_height BIGINT NOT NULL
 );

common/nyxd-scraper/sql_migrations/02_cosmos.sql

@@ -1,76 +1,73 @@
-CREATE TABLE validator
-(
-    consensus_address TEXT NOT NULL PRIMARY KEY, /* Validator consensus address */
-    consensus_pubkey  TEXT NOT NULL UNIQUE /* Validator consensus public key */
+CREATE TABLE validator (
+    consensus_address TEXT NOT NULL PRIMARY KEY,
+    /* Validator consensus address */
+    consensus_pubkey TEXT NOT NULL UNIQUE
+    /* Validator consensus public key */
 );
 
-CREATE TABLE pre_commit
-(
-    validator_address TEXT                        NOT NULL REFERENCES validator (consensus_address),
-    height            BIGINT                      NOT NULL,
-    timestamp         TIMESTAMP WITHOUT TIME ZONE NOT NULL,
-    voting_power      BIGINT                      NOT NULL,
-    proposer_priority BIGINT                      NOT NULL,
+CREATE TABLE pre_commit (
+    validator_address TEXT NOT NULL REFERENCES validator (consensus_address),
+    height BIGINT NOT NULL,
+    timestamp TIMESTAMPTZ NOT NULL,
+    voting_power BIGINT NOT NULL,
+    proposer_priority BIGINT NOT NULL,
     UNIQUE (validator_address, timestamp)
 );
+
 CREATE INDEX pre_commit_validator_address_index ON pre_commit (validator_address);
+
 CREATE INDEX pre_commit_height_index ON pre_commit (height);
 
-CREATE TABLE block
-(
-    height           BIGINT  UNIQUE PRIMARY KEY,
-    hash             TEXT    NOT NULL UNIQUE,
-    num_txs          INTEGER DEFAULT 0,
-    total_gas        BIGINT  DEFAULT 0,
+CREATE TABLE block (
+    height BIGINT UNIQUE PRIMARY KEY,
+    hash TEXT NOT NULL UNIQUE,
+    num_txs INTEGER DEFAULT 0,
+    total_gas BIGINT DEFAULT 0,
     proposer_address TEXT REFERENCES validator (consensus_address),
-    timestamp        TIMESTAMP WITHOUT TIME ZONE NOT NULL
+    timestamp TIMESTAMPTZ NOT NULL
 );
+
 CREATE INDEX block_height_index ON block (height);
+
 CREATE INDEX block_hash_index ON block (hash);
+
 CREATE INDEX block_proposer_address_index ON block (proposer_address);
 
--- no JSONB in sqlite (yet) : )
-CREATE TABLE "transaction"
-(
-    hash         TEXT    UNIQUE NOT NULL,
-    height       BIGINT  NOT NULL REFERENCES block (height),
-    "index"      INTEGER NOT NULL,
-    success      BOOLEAN NOT NULL,
-
+CREATE TABLE "transaction" (
+    hash TEXT UNIQUE NOT NULL,
+    height BIGINT NOT NULL REFERENCES block (height),
+    "index" INTEGER NOT NULL,
+    success BOOLEAN NOT NULL,
     /* Body */
     num_messages INTEGER NOT NULL,
---     messages     JSONB   NOT NULL,-- DEFAULT '[]'::JSONB,
-    memo         TEXT,
---     signatures   TEXT[]  NOT NULL,
-
+    messages JSONB NOT NULL DEFAULT '[]',
+    memo TEXT,
+    signatures TEXT [] NOT NULL,
     /* AuthInfo */
---     signer_infos JSONB   NOT NULL,-- DEFAULT '[]'::JSONB,
---     fee          JSONB   NOT NULL,-- DEFAULT '{}'::JSONB,
-
+    signer_infos JSONB NOT NULL DEFAULT '[]'::JSONB,
+    fee JSONB NOT NULL DEFAULT '{}'::JSONB,
     /* Tx response */
-    gas_wanted   BIGINT           DEFAULT 0,
-    gas_used     BIGINT           DEFAULT 0,
-    raw_log      TEXT
---     logs         JSONB
+    gas_wanted BIGINT DEFAULT 0,
+    gas_used BIGINT DEFAULT 0,
+    raw_log TEXT
 );
+
 CREATE INDEX transaction_hash_index ON "transaction" (hash);
+
 CREATE INDEX transaction_height_index ON "transaction" (height);
 
-CREATE TABLE message
-(
-    transaction_hash            TEXT   NOT NULL REFERENCES "transaction" (hash),
-    "index"                       BIGINT NOT NULL,
-    type                        TEXT   NOT NULL,
---     value                       JSONB  NOT NULL,
---     involved_accounts_addresses TEXT[] NOT NULL,
-
-    height                      BIGINT NOT NULL,
+CREATE TABLE message (
+    transaction_hash TEXT NOT NULL REFERENCES "transaction" (hash),
+    "index" BIGINT NOT NULL,
+    TYPE TEXT NOT NULL,
+    value JSONB NOT NULL,
+    involved_accounts_addresses TEXT [] NOT NULL,
+    height BIGINT NOT NULL,
     CONSTRAINT unique_message_per_tx UNIQUE (transaction_hash, "index")
 );
-CREATE INDEX message_transaction_hash_index ON message (transaction_hash);
-CREATE INDEX message_type_index ON message (type);
 
-CREATE TABLE pruning
-(
-    last_pruned_height BIGINT NOT NULL
-);
+CREATE INDEX message_transaction_hash_index ON message (transaction_hash);
+
+CREATE INDEX message_type_index ON message (TYPE);
+
+CREATE TABLE pruning (last_pruned_height BIGINT NOT NULL);

common/nyxd-scraper/src/scraper/mod.rs

@@ -129,7 +129,8 @@ impl NyxdScraper {
 
     pub async fn new(config: Config) -> Result<Self, ScraperError> {
         config.pruning_options.validate()?;
-        let storage = ScraperStorage::init(&config.database_path).await?;
+        let storage =
+            ScraperStorage::init(&config.database_path.to_str().unwrap_or_default()).await?;
         let rpc_client = RpcClient::new(&config.rpc_url)?;
 
         Ok(NyxdScraper {

common/nyxd-scraper/src/storage/manager.rs

@@ -1,21 +1,25 @@
 // Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
+use std::vec;
+
 use crate::storage::log_db_operation_time;
 use crate::storage::models::{CommitSignature, Validator};
+use base64;
+use serde_json::Value as JsonValue;
 use sqlx::types::time::OffsetDateTime;
-use sqlx::{Executor, Sqlite};
+use sqlx::{Executor, Postgres};
 use tokio::time::Instant;
 use tracing::{instrument, trace};
 
 #[derive(Clone)]
 pub(crate) struct StorageManager {
-    pub(crate) connection_pool: sqlx::SqlitePool,
+    pub(crate) connection_pool: sqlx::PgPool,
 }
 
 impl StorageManager {
     pub(crate) async fn set_initial_metadata(&self) -> Result<(), sqlx::Error> {
-        if sqlx::query("SELECT * from metadata")
+        if sqlx::query::<Postgres>("SELECT * from metadata")
             .fetch_optional(&self.connection_pool)
             .await?
             .is_none()
@@ -44,7 +48,7 @@ impl StorageManager {
         log_db_operation_time("get_lowest_block", start);
 
         if let Some(row) = maybe_record {
-            Ok(row.height)
+            Ok(Some(row.height))
         } else {
             Ok(None)
         }
@@ -61,7 +65,7 @@ impl StorageManager {
             r#"
                 SELECT height
                 FROM block
-                WHERE timestamp > ?
+                WHERE timestamp > $1
                 ORDER BY timestamp
                 LIMIT 1
             "#,
@@ -72,7 +76,7 @@ impl StorageManager {
         log_db_operation_time("get_first_block_height_after", start);
 
         if let Some(row) = maybe_record {
-            Ok(row.height)
+            Ok(Some(row.height))
         } else {
             Ok(None)
         }
@@ -89,7 +93,7 @@ impl StorageManager {
             r#"
                 SELECT height
                 FROM block
-                WHERE timestamp < ?
+                WHERE timestamp < $1
                 ORDER BY timestamp DESC
                 LIMIT 1
             "#,
@@ -100,7 +104,7 @@ impl StorageManager {
         log_db_operation_time("get_last_block_height_before", start);
 
         if let Some(row) = maybe_record {
-            Ok(row.height)
+            Ok(Some(row.height))
         } else {
             Ok(None)
         }
@@ -111,7 +115,7 @@ impl StorageManager {
         consensus_address: &str,
         start_height: i64,
         end_height: i64,
-    ) -> Result<i32, sqlx::Error> {
+    ) -> Result<i64, sqlx::Error> {
         trace!("get_signed_between");
         let start = Instant::now();
 
@@ -119,9 +123,9 @@ impl StorageManager {
             r#"
                 SELECT COUNT(*) as count FROM pre_commit
                 WHERE 
-                    validator_address == ?
-                    AND height >= ? 
-                    AND height <= ?
+                    validator_address = $1
+                    AND height >= $2
+                    AND height <= $3
             "#,
             consensus_address,
             start_height,
@@ -132,7 +136,7 @@ impl StorageManager {
         .count;
         log_db_operation_time("get_signed_between", start);
 
-        Ok(count)
+        Ok(count.expect("Could not find the count"))
     }
 
     pub(crate) async fn get_precommit(
@@ -146,8 +150,8 @@ impl StorageManager {
         let res = sqlx::query_as(
             r#"
                 SELECT * FROM pre_commit
-                WHERE validator_address = ? 
-                AND height = ?
+                WHERE validator_address = $1
+                AND height = $2
             "#,
         )
         .bind(consensus_address)
@@ -172,7 +176,7 @@ impl StorageManager {
                 SELECT * FROM validator 
                 WHERE EXISTS (
                     SELECT 1 FROM pre_commit
-                    WHERE height == ?
+                    WHERE height = $1
                     AND pre_commit.validator_address = validator.consensus_address
                 )
             "#,
@@ -248,7 +252,7 @@ pub(crate) async fn insert_validator<'a, E>(
     executor: E,
 ) -> Result<(), sqlx::Error>
 where
-    E: Executor<'a, Database = Sqlite>,
+    E: Executor<'a, Database = Postgres>,
 {
     trace!("insert_validator");
     let start = Instant::now();
@@ -256,8 +260,8 @@ where
     sqlx::query!(
         r#"
             INSERT INTO validator (consensus_address, consensus_pubkey)
-            VALUES (?, ?)
-            ON CONFLICT DO NOTHING
+            VALUES ($1, $2)
+            ON CONFLICT (consensus_address) DO NOTHING
         "#,
         consensus_address,
         consensus_pubkey
@@ -280,7 +284,7 @@ pub(crate) async fn insert_block<'a, E>(
     executor: E,
 ) -> Result<(), sqlx::Error>
 where
-    E: Executor<'a, Database = Sqlite>,
+    E: Executor<'a, Database = Postgres>,
 {
     trace!("insert_block");
     let start = Instant::now();
@@ -288,12 +292,12 @@ where
     sqlx::query!(
         r#"
             INSERT INTO block (height, hash, num_txs, total_gas, proposer_address, timestamp)
-            VALUES (?, ?, ?, ?, ?, ?)
+            VALUES ($1, $2, $3, $4, $5, $6)
             ON CONFLICT DO NOTHING
         "#,
         height,
         hash,
-        num_txs,
+        num_txs as i32,
         total_gas,
         proposer_address,
         timestamp
@@ -315,7 +319,7 @@ pub(crate) async fn insert_precommit<'a, E>(
     executor: E,
 ) -> Result<(), sqlx::Error>
 where
-    E: Executor<'a, Database = Sqlite>,
+    E: Executor<'a, Database = Postgres>,
 {
     trace!("insert_precommit");
     let start = Instant::now();
@@ -323,7 +327,7 @@ where
     sqlx::query!(
         r#"
             INSERT INTO pre_commit (validator_address, height, timestamp, voting_power, proposer_priority)
-            VALUES (?, ?, ?, ?, ?)
+            VALUES ($1, $2, $3, $4, $5)
             ON CONFLICT (validator_address, timestamp) DO NOTHING
         "#,
         validator_address,
@@ -346,45 +350,76 @@ pub(crate) async fn insert_transaction<'a, E>(
     height: i64,
     index: i64,
     success: bool,
-    messages: i64,
+    num_messages: i64,
+    messages: Vec<cosmrs::Any>,
     memo: String,
+    signatures: Vec<String>,
+    signer_infos: Vec<SignerInfo>,
+    fee: Fee,
     gas_wanted: i64,
     gas_used: i64,
     raw_log: String,
     executor: E,
 ) -> Result<(), sqlx::Error>
 where
-    E: Executor<'a, Database = Sqlite>,
+    E: Executor<'a, Database = Postgres>,
 {
     trace!("insert_transaction");
     let start = Instant::now();
 
     sqlx::query!(
         r#"
-            INSERT INTO "transaction" (hash, height, "index", success, num_messages, memo, gas_wanted, gas_used, raw_log)
-            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
-               ON CONFLICT (hash) DO UPDATE
-               SET height = excluded.height,
-               "index" = excluded."index",
-               success = excluded.success,
-               num_messages = excluded.num_messages,
-               memo = excluded.memo,
-               gas_wanted = excluded.gas_wanted,
-               gas_used = excluded.gas_used,
-               raw_log = excluded.raw_log
+            INSERT INTO transaction (hash, height, "index", success, num_messages, messages, memo, signatures, signer_infos, fee, gas_wanted, gas_used, raw_log)
+            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
+            ON CONFLICT (hash) DO UPDATE
+                SET height = excluded.height,
+                    "index" = excluded."index",
+                    success = excluded.success,
+                    num_messages = excluded.num_messages,
+                    messages = excluded.messages,
+                    memo = excluded.memo,
+                    signatures = excluded.signatures,
+                    signer_infos = excluded.signer_infos,
+                    fee = excluded.fee,
+                    gas_wanted = excluded.gas_wanted,
+                    gas_used = excluded.gas_used,
+                    raw_log = excluded.raw_log
         "#,
-            hash,
-            height,
-            index,
-            success,
-            messages,
-            memo,
-            gas_wanted,
-            gas_used,
-            raw_log,
+        hash,
+        height,
+        index as i32,
+        success,
+        num_messages as i32,
+        serde_json::json!(messages.iter().map(|msg| {
+            serde_json::json!({
+                "type_url": msg.type_url,
+                "value": base64::encode(&msg.value)
+            })
+        }).collect::<Vec<_>>()),
+        memo,
+        &signatures,
+        serde_json::json!(signer_infos.iter().map(|info| {
+            serde_json::json!({
+                "public_key": {
+                    "type_url": info.public_key.type_url,
+                    "value": base64::encode(&info.public_key.value)
+                },
+                "mode_info": info.mode_info,
+                "sequence": info.sequence
+            })
+        }).collect::<Vec<_>>()),
+        serde_json::json!({
+            "amount": fee.amount,
+            "gas_limit": fee.gas_limit,
+            "payer": fee.payer,
+            "granter": fee.granter
+        }),
+        gas_wanted,
+        gas_used,
+        raw_log
     )
-        .execute(executor)
-        .await?;
+    .execute(executor)
+    .await?;
     log_db_operation_time("insert_transaction", start);
 
     Ok(())
@@ -395,27 +430,36 @@ pub(crate) async fn insert_message<'a, E>(
     transaction_hash: String,
     index: i64,
     typ: String,
+    value: cosmrs::Any,
+    involved_accounts: Vec<String>,
     height: i64,
     executor: E,
 ) -> Result<(), sqlx::Error>
 where
-    E: Executor<'a, Database = Sqlite>,
+    E: Executor<'a, Database = Postgres>,
 {
     trace!("insert_message");
     let start = Instant::now();
 
     sqlx::query!(
         r#"
-            INSERT INTO message (transaction_hash, "index", type, height)
-            VALUES (?, ?, ?, ?)
+            INSERT INTO message (transaction_hash, "index", type, value, involved_accounts_addresses, height)
+            VALUES ($1, $2, $3, $4, $5, $6)
             ON CONFLICT (transaction_hash, "index") DO UPDATE
                 SET height = excluded.height,
-                type = excluded.type
+                type = excluded.type,
+                value = excluded.value,
+                involved_accounts_addresses = excluded.involved_accounts_addresses
         "#,
-        transaction_hash,
-        index,
-        typ,
-        height
+            transaction_hash,
+            index,
+            typ,
+            serde_json::json!({
+                "type_url": value.type_url,
+                "value": base64::encode(&value.value)
+            }),
+            &involved_accounts,
+            height
     )
     .execute(executor)
     .await?;
@@ -430,13 +474,13 @@ pub(crate) async fn update_last_processed<'a, E>(
     executor: E,
 ) -> Result<(), sqlx::Error>
 where
-    E: Executor<'a, Database = Sqlite>,
+    E: Executor<'a, Database = Postgres>,
 {
     trace!("update_last_processed");
     let start = Instant::now();
 
     sqlx::query!(
-        "UPDATE metadata SET last_processed_height = MAX(last_processed_height, ?)",
+        "UPDATE metadata SET last_processed_height = GREATEST(last_processed_height, $1)",
         height
     )
     .execute(executor)
@@ -449,12 +493,12 @@ where
 #[instrument(skip(executor))]
 pub(crate) async fn update_last_pruned<'a, E>(height: i64, executor: E) -> Result<(), sqlx::Error>
 where
-    E: Executor<'a, Database = Sqlite>,
+    E: Executor<'a, Database = Postgres>,
 {
     trace!("update_last_pruned");
     let start = Instant::now();
 
-    sqlx::query!("UPDATE pruning SET last_pruned_height = ?", height)
+    sqlx::query!("UPDATE pruning SET last_pruned_height = $1", height)
         .execute(executor)
         .await?;
     log_db_operation_time("update_last_pruned", start);
@@ -464,12 +508,12 @@ where
 
 pub(crate) async fn prune_blocks<'a, E>(oldest_to_keep: i64, executor: E) -> Result<(), sqlx::Error>
 where
-    E: Executor<'a, Database = Sqlite>,
+    E: Executor<'a, Database = Postgres>,
 {
     trace!("prune_blocks");
     let start = Instant::now();
 
-    sqlx::query!("DELETE FROM block WHERE height < ?", oldest_to_keep)
+    sqlx::query!("DELETE FROM block WHERE height < $1", oldest_to_keep)
         .execute(executor)
         .await?;
     log_db_operation_time("prune_blocks", start);
@@ -482,12 +526,12 @@ pub(crate) async fn prune_pre_commits<'a, E>(
     executor: E,
 ) -> Result<(), sqlx::Error>
 where
-    E: Executor<'a, Database = Sqlite>,
+    E: Executor<'a, Database = Postgres>,
 {
     trace!("prune_pre_commits");
     let start = Instant::now();
 
-    sqlx::query!("DELETE FROM pre_commit WHERE height < ?", oldest_to_keep)
+    sqlx::query!("DELETE FROM pre_commit WHERE height < $1", oldest_to_keep)
         .execute(executor)
         .await?;
     log_db_operation_time("prune_pre_commits", start);
@@ -500,13 +544,13 @@ pub(crate) async fn prune_transactions<'a, E>(
     executor: E,
 ) -> Result<(), sqlx::Error>
 where
-    E: Executor<'a, Database = Sqlite>,
+    E: Executor<'a, Database = Postgres>,
 {
     trace!("prune_transactions");
     let start = Instant::now();
 
     sqlx::query!(
-        "DELETE FROM \"transaction\" WHERE height < ?",
+        "DELETE FROM transaction WHERE height < $1",
         oldest_to_keep
     )
     .execute(executor)
@@ -521,15 +565,36 @@ pub(crate) async fn prune_messages<'a, E>(
     executor: E,
 ) -> Result<(), sqlx::Error>
 where
-    E: Executor<'a, Database = Sqlite>,
+    E: Executor<'a, Database = Postgres>,
 {
     trace!("prune_messages");
     let start = Instant::now();
 
-    sqlx::query!("DELETE FROM message WHERE height < ?", oldest_to_keep)
+    sqlx::query!("DELETE FROM message WHERE height < $1", oldest_to_keep)
         .execute(executor)
         .await?;
     log_db_operation_time("prune_messages", start);
 
     Ok(())
 }
+
+#[derive(Debug, Clone, serde::Serialize)]
+pub struct Coin {
+    pub denom: String,
+    pub amount: String,
+}
+
+#[derive(Debug, Clone)]
+pub struct SignerInfo {
+    pub public_key: cosmrs::Any,
+    pub mode_info: String,
+    pub sequence: i64,
+}
+
+#[derive(Debug, Clone)]
+pub struct Fee {
+    pub amount: Vec<Coin>,
+    pub gas_limit: i64,
+    pub payer: String,
+    pub granter: String,
+}

common/nyxd-scraper/src/storage/mod.rs

@@ -4,15 +4,14 @@
 use crate::block_processor::types::{FullBlockInformation, ParsedTransactionResponse};
 use crate::error::ScraperError;
 use crate::storage::manager::{
-    insert_block, insert_message, insert_precommit, insert_transaction, insert_validator,
-    prune_blocks, prune_messages, prune_pre_commits, prune_transactions, update_last_processed,
-    update_last_pruned, StorageManager,
+    insert_block, insert_message, insert_precommit, insert_transaction,
+    insert_validator, prune_blocks, prune_messages, prune_pre_commits, prune_transactions,
+    update_last_processed, update_last_pruned, StorageManager, Coin, Fee, SignerInfo,
 };
 use crate::storage::models::{CommitSignature, Validator};
 use sqlx::types::time::OffsetDateTime;
-use sqlx::{ConnectOptions, Sqlite, Transaction};
-use std::fmt::Debug;
-use std::path::Path;
+use sqlx::{ConnectOptions, Postgres, Transaction};
+use std::str::FromStr;
 use tendermint::block::{Commit, CommitSig};
 use tendermint::Block;
 use tendermint_rpc::endpoint::validators;
@@ -23,7 +22,7 @@ mod helpers;
 mod manager;
 pub mod models;
 
-pub type StorageTransaction = Transaction<'static, Sqlite>;
+pub type StorageTransaction = Transaction<'static, Postgres>;
 
 #[derive(Clone)]
 pub struct ScraperStorage {
@@ -45,19 +44,18 @@ pub(crate) fn log_db_operation_time(op_name: &str, start_time: Instant) {
 
 impl ScraperStorage {
     #[instrument]
-    pub async fn init<P: AsRef<Path> + Debug>(database_path: P) -> Result<Self, ScraperError> {
-        let mut opts = sqlx::sqlite::SqliteConnectOptions::new()
-            .filename(database_path)
-            .create_if_missing(true);
+    pub async fn init(database_url: &str) -> Result<Self, ScraperError> {
+        let mut opts = sqlx::postgres::PgConnectOptions::from_str(database_url)
+            .map_err(|err| ScraperError::InternalDatabaseError(err))?;
 
         // TODO: do we want auto_vacuum ?
 
         opts.disable_statement_logging();
 
-        let connection_pool = match sqlx::SqlitePool::connect_with(opts).await {
+        let connection_pool = match sqlx::PgPool::connect_with(opts).await {
             Ok(db) => db,
             Err(err) => {
-                error!("Failed to connect to SQLx database: {err}");
+                error!("Failed to connect to PostgreSQL database: {err}");
                 return Err(err.into());
             }
         };
@@ -154,7 +152,7 @@ impl ScraperStorage {
         consensus_address: &str,
         start_height: i64,
         end_height: i64,
-    ) -> Result<i32, ScraperError> {
+    ) -> Result<i64, ScraperError> {
         Ok(self
             .manager
             .get_signed_between(consensus_address, start_height, end_height)
@@ -166,7 +164,7 @@ impl ScraperStorage {
         consensus_address: &str,
         start_time: OffsetDateTime,
         end_time: OffsetDateTime,
-    ) -> Result<i32, ScraperError> {
+    ) -> Result<i64, ScraperError> {
         let Some(block_start) = self.get_first_block_height_after(start_time).await? else {
             return Ok(0);
         };
@@ -229,10 +227,9 @@ pub async fn persist_block(
     }
 
     // persist txs
-    persist_txs(&block.transactions, tx).await?;
-
-    // persist messages (inside the transactions)
-    persist_messages(&block.transactions, tx).await?;
+    for chain_tx in &block.transactions {
+        persist_transaction(chain_tx, tx).await?;
+    }
 
     update_last_processed(block.block.header.height.into(), tx).await?;
 
@@ -328,23 +325,86 @@ async fn persist_commits(
     Ok(())
 }
 
-async fn persist_txs(
-    txs: &[ParsedTransactionResponse],
-    tx: &mut StorageTransaction,
+async fn persist_transaction(
+    chain_tx: &ParsedTransactionResponse,
+    tx: &mut Transaction<'_, Postgres>,
 ) -> Result<(), ScraperError> {
-    debug!("persisting {} txs", txs.len());
+    let signer_infos = chain_tx
+        .tx
+        .auth_info
+        .signer_infos
+        .iter()
+        .map(|info| SignerInfo {
+            public_key: info.public_key.clone()
+                .map(|pk| pk.into())
+                .unwrap_or_default(),
+            mode_info: serde_json::to_string(&serde_json::json!({
+                "single": {
+                    "mode": match &info.mode_info {
+                        cosmrs::tx::ModeInfo::Single(s) => s.mode as u32,
+                        _ => 0,
+                    }
+                }
+            })).unwrap_or_default(),
+            sequence: info.sequence as i64,
+        })
+        .collect();
 
-    for chain_tx in txs {
-        insert_transaction(
+    let fee = Fee {
+        amount: chain_tx
+            .tx
+            .auth_info
+            .fee
+            .amount
+            .iter()
+            .map(|coin| Coin {
+                denom: coin.denom.to_string(),
+                amount: coin.amount.to_string(),
+            })
+            .collect(),
+        gas_limit: chain_tx.tx.auth_info.fee.gas_limit as i64,
+        payer: chain_tx.tx.auth_info.fee.payer
+            .clone()
+            .map(|id| id.to_string())
+            .unwrap_or_default(),
+        granter: chain_tx.tx.auth_info.fee.granter
+            .clone()
+            .map(|id| id.to_string())
+            .unwrap_or_default(),
+    };
+
+    let signatures = chain_tx.tx.signatures
+        .iter()
+        .map(|sig| base64::encode(sig))
+        .collect();
+
+    insert_transaction(
+        chain_tx.hash.to_string(),
+        chain_tx.height.into(),
+        chain_tx.index as i64,
+        chain_tx.tx_result.code.is_ok(),
+        chain_tx.tx.body.messages.len() as i64,
+        chain_tx.tx.body.messages.clone(),
+        chain_tx.tx.body.memo.clone(),
+        signatures,
+        signer_infos,
+        fee,
+        chain_tx.tx_result.gas_wanted,
+        chain_tx.tx_result.gas_used,
+        chain_tx.tx_result.log.clone(),
+        &mut *tx,
+    )
+    .await?;
+
+    for (index, msg) in chain_tx.tx.body.messages.iter().enumerate() {
+        let involved_accounts = extract_involved_accounts(msg);
+        insert_message(
             chain_tx.hash.to_string(),
+            index as i64,
+            msg.type_url.clone(),
+            msg.clone(),
+            involved_accounts,
             chain_tx.height.into(),
-            chain_tx.index as i64,
-            chain_tx.tx_result.code.is_ok(),
-            chain_tx.tx.body.messages.len() as i64,
-            chain_tx.tx.body.memo.clone(),
-            chain_tx.tx_result.gas_wanted,
-            chain_tx.tx_result.gas_used,
-            chain_tx.tx_result.log.clone(),
             &mut *tx,
         )
         .await?;
@@ -353,24 +413,8 @@ async fn persist_txs(
     Ok(())
 }
 
-async fn persist_messages(
-    txs: &[ParsedTransactionResponse],
-    tx: &mut StorageTransaction,
-) -> Result<(), ScraperError> {
-    debug!("persisting messages");
-
-    for chain_tx in txs {
-        for (index, msg) in chain_tx.tx.body.messages.iter().enumerate() {
-            insert_message(
-                chain_tx.hash.to_string(),
-                index as i64,
-                msg.type_url.clone(),
-                chain_tx.height.into(),
-                &mut *tx,
-            )
-            .await?
-        }
-    }
-
-    Ok(())
-}
+fn extract_involved_accounts(_msg: &cosmrs::Any) -> Vec<String> {
+    // This is a placeholder implementation
+    // TODO: Implement proper account extraction based on message type
+    vec![]
+}

explorer-api/explorer-client/Cargo.toml

@@ -7,12 +7,12 @@ license.workspace = true
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-log.workspace = true
 nym-explorer-api-requests = { path = "../explorer-api-requests" }
 reqwest = { workspace = true, features = ["json"] }
 serde.workspace = true
 thiserror.workspace = true
 url.workspace = true
+tracing = {workspace = true, features = ["attributes"]}
 
 [dev-dependencies]
 tokio = { workspace = true, features = ["full"] }

explorer-api/explorer-client/src/lib.rs

@@ -3,6 +3,7 @@ use std::time::Duration;
 
 use reqwest::StatusCode;
 use thiserror::Error;
+use tracing::instrument;
 use url::Url;
 
 // Re-export request types
@@ -47,6 +48,12 @@ impl ExplorerClient {
         Ok(Self { client, url })
     }
 
+    #[cfg(not(target_arch = "wasm32"))]
+    pub fn new_with_timeout(url: url::Url, timeout: Duration) -> Result<Self, ExplorerApiError> {
+        let client = reqwest::Client::builder().timeout(timeout).build()?;
+        Ok(Self { client, url })
+    }
+
     #[cfg(target_arch = "wasm32")]
     pub fn new(url: url::Url) -> Result<Self, ExplorerApiError> {
         let client = reqwest::Client::builder().build()?;
@@ -58,10 +65,11 @@ impl ExplorerClient {
         paths: &[&str],
     ) -> Result<reqwest::Response, ExplorerApiError> {
         let url = combine_url(self.url.clone(), paths)?;
-        log::trace!("Sending GET request {url:?}");
+        tracing::debug!("Sending GET request");
         Ok(self.client.get(url).send().await?)
     }
 
+    #[instrument(level = "trace", skip_all, fields(paths=?paths))]
     async fn query_explorer_api<T>(&self, paths: &[&str]) -> Result<T, ExplorerApiError>
     where
         T: std::fmt::Debug,
@@ -70,7 +78,7 @@ impl ExplorerClient {
         let response = self.send_get_request(paths).await?;
         if response.status().is_success() {
             let res = response.json::<T>().await?;
-            log::trace!("Got response: {res:?}");
+            tracing::trace!("Got response: {res:?}");
             Ok(res)
         } else if response.status() == StatusCode::NOT_FOUND {
             Err(ExplorerApiError::NotFound)

nym-node-status-api/.gitignore

@@ -0,0 +1,2 @@
+data/
+enter_db.sh

nym-node-status-api/Cargo.toml

@@ -0,0 +1,57 @@
+# Copyright 2024 - Nym Technologies SA <contact@nymtech.net>
+# SPDX-License-Identifier: Apache-2.0
+
+[package]
+name = "nym-node-status-api"
+version = "0.1.0"
+authors.workspace = true
+repository.workspace = true
+homepage.workspace = true
+documentation.workspace = true
+edition.workspace = true
+license.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+anyhow = { workspace = true }
+axum = { workspace = true, features = ["tokio"] }
+chrono = { workspace = true }
+clap = { workspace = true, features = ["cargo", "derive"] }
+cosmwasm-std = { workspace = true }
+envy = { workspace = true }
+futures-util = { workspace = true }
+moka = { workspace = true, features = ["future"] }
+nym-bin-common = { path = "../common/bin-common" }
+nym-explorer-client = { path = "../explorer-api/explorer-client" }
+# TODO dz: ref before Nym API client changes. Update to latest develop once new Nym API is live
+nym-network-defaults = { git = "https://github.com/nymtech/nym", rev = "f86e08866" }
+nym-validator-client = { git = "https://github.com/nymtech/nym", rev = "f86e08866" }
+# nym-network-defaults = { path = "../common/network-defaults" }
+# nym-validator-client = { path = "../common/client-libs/validator-client" }
+nym-task = { path = "../common/task" }
+nym-node-requests = { git = "https://github.com/nymtech/nym", rev = "f86e08866" }
+# nym-node-requests = { path = "../nym-node/nym-node-requests", features = ["openapi"] }
+reqwest = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true }
+serde_json_path = { workspace = true }
+sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite"] }
+thiserror = { workspace = true }
+tokio = { workspace = true, features = ["rt-multi-thread"] }
+tokio-util = { workspace = true }
+tracing = { workspace = true }
+tracing-subscriber = { workspace = true, features = ["env-filter"] }
+tracing-log = { workspace = true }
+tower-http = { workspace = true, features = ["cors", "trace"] }
+utoipa = { workspace = true, features = ["axum_extras", "time"] }
+utoipa-swagger-ui = { workspace = true, features = ["axum"] }
+# TODO dz `cargo update async-trait`
+# for automatic schema detection, which was merged, but not released yet
+# https://github.com/ProbablyClem/utoipauto/pull/38
+# utoipauto = { git = "https://github.com/ProbablyClem/utoipauto", rev = "eb04cba" }
+utoipauto = { workspace = true }
+
+[build-dependencies]
+anyhow = { workspace = true }
+tokio = { workspace = true, features = ["macros" ] }
+sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate"] }

nym-node-status-api/build.rs

@@ -0,0 +1,46 @@
+use anyhow::{anyhow, Result};
+use sqlx::{Connection, SqliteConnection};
+use tokio::{fs::File, io::AsyncWriteExt};
+
+const SQLITE_DB_FILENAME: &str = "nym-node-status-api.sqlite";
+
+/// If you need to re-run migrations or reset the db, just run
+/// cargo clean -p nym-node-status-api
+#[tokio::main]
+async fn main() -> Result<()> {
+    let out_dir = read_env_var("OUT_DIR")?;
+    let database_path = format!("sqlite://{}/{}?mode=rwc", out_dir, SQLITE_DB_FILENAME);
+
+    write_db_path_to_file(&out_dir, SQLITE_DB_FILENAME).await?;
+    let mut conn = SqliteConnection::connect(&database_path).await?;
+    sqlx::migrate!("./migrations").run(&mut conn).await?;
+
+    #[cfg(target_family = "unix")]
+    println!("cargo::rustc-env=DATABASE_URL=sqlite://{}", &database_path);
+
+    #[cfg(target_family = "windows")]
+    // for some strange reason we need to add a leading `/` to the windows path even though it's
+    // not a valid windows path... but hey, it works...
+    println!("cargo::rustc-env=DATABASE_URL=sqlite:///{}", &database_path);
+
+    rerun_if_changed();
+    Ok(())
+}
+
+fn read_env_var(var: &str) -> Result<String> {
+    std::env::var(var).map_err(|_| anyhow!("You need to set {} env var", var))
+}
+
+fn rerun_if_changed() {
+    println!("cargo::rerun-if-changed=migrations");
+    println!("cargo::rerun-if-changed=src/db/queries");
+}
+
+/// use `./enter_db.sh` to inspect DB
+async fn write_db_path_to_file(out_dir: &str, db_filename: &str) -> anyhow::Result<()> {
+    let mut file = File::create("enter_db.sh").await?;
+    let _ = file.write(b"#!/bin/bash\n").await?;
+    file.write_all(format!("sqlite3 {}/{}", out_dir, db_filename).as_bytes())
+        .await
+        .map_err(From::from)
+}

nym-node-status-api/launch_node_status_api.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -e
+
+export RUST_LOG=${RUST_LOG:-debug}
+
+export NYM_API_CLIENT_TIMEOUT=60;
+export EXPLORER_CLIENT_TIMEOUT=60;
+
+cargo run --package nym-node-status-api --release -- --config-env-file ../envs/mainnet.env

nym-node-status-api/migrations/000_init.sql

@@ -0,0 +1,100 @@
+CREATE TABLE gateways
+(
+  id                      INTEGER PRIMARY KEY AUTOINCREMENT,
+  gateway_identity_key    VARCHAR NOT NULL UNIQUE,
+  self_described          VARCHAR,
+  explorer_pretty_bond    VARCHAR,
+  last_probe_result       VARCHAR,
+  last_probe_log          VARCHAR,
+  config_score            INTEGER NOT NULL DEFAULT (0),
+  config_score_successes  REAL    NOT NULL DEFAULT (0),
+  config_score_samples    REAL    NOT NULL DEFAULT (0),
+  routing_score           INTEGER NOT NULL DEFAULT (0),
+  routing_score_successes REAL    NOT NULL DEFAULT (0),
+  routing_score_samples   REAL    NOT NULL DEFAULT (0),
+  test_run_samples        REAL    NOT NULL DEFAULT (0),
+  last_testrun_utc        INTEGER,
+  last_updated_utc        INTEGER NOT NULL,
+  bonded INTEGER CHECK (bonded in (0, 1)) NOT NULL DEFAULT 0,
+  blacklisted INTEGER CHECK (bonded in (0, 1)) NOT NULL DEFAULT 0,
+  performance INTEGER NOT NULL DEFAULT 0
+);
+
+CREATE INDEX idx_gateway_description_gateway_identity_key ON gateways (gateway_identity_key);
+
+
+CREATE TABLE mixnodes (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    identity_key VARCHAR NOT NULL UNIQUE,
+    mix_id INTEGER NOT NULL UNIQUE,
+    bonded INTEGER CHECK (bonded in (0, 1)) NOT NULL DEFAULT 0,
+    total_stake INTEGER NOT NULL,
+    host VARCHAR NOT NULL,
+    http_api_port INTEGER NOT NULL,
+    blacklisted INTEGER CHECK (blacklisted in (0, 1)) NOT NULL DEFAULT 0,
+    full_details VARCHAR,
+    self_described VARCHAR,
+    last_updated_utc INTEGER NOT NULL
+  , is_dp_delegatee INTEGER CHECK (is_dp_delegatee IN (0, 1)) NOT NULL DEFAULT 0);
+CREATE INDEX idx_mixnodes_mix_id ON mixnodes (mix_id);
+CREATE INDEX idx_mixnodes_identity_key ON mixnodes (identity_key);
+
+CREATE TABLE
+  mixnode_description (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    mix_id INTEGER UNIQUE NOT NULL,
+    moniker VARCHAR,
+    website VARCHAR,
+    security_contact VARCHAR,
+    details VARCHAR,
+    last_updated_utc INTEGER NOT NULL,
+    FOREIGN KEY (mix_id) REFERENCES mixnodes (mix_id)
+  );
+
+-- Indexes for description table
+CREATE INDEX idx_mixnode_description_mix_id ON mixnode_description (mix_id);
+
+
+CREATE TABLE summary
+(
+  key              VARCHAR PRIMARY KEY,
+  value_json       VARCHAR,
+  last_updated_utc INTEGER NOT NULL
+);
+
+
+CREATE TABLE summary_history
+(
+  id            INTEGER PRIMARY KEY AUTOINCREMENT,
+  date          VARCHAR UNIQUE NOT NULL,
+  timestamp_utc INTEGER NOT NULL,
+  value_json    VARCHAR
+);
+CREATE INDEX idx_summary_history_timestamp_utc ON summary_history (timestamp_utc);
+CREATE INDEX idx_summary_history_date ON summary_history (date);
+
+
+CREATE TABLE gateway_description (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    gateway_identity_key VARCHAR UNIQUE NOT NULL,
+    moniker VARCHAR,
+    website VARCHAR,
+    security_contact VARCHAR,
+    details VARCHAR,
+    last_updated_utc INTEGER NOT NULL,
+    FOREIGN KEY (gateway_identity_key) REFERENCES gateways (gateway_identity_key)
+  );
+
+
+CREATE TABLE
+  mixnode_daily_stats (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    mix_id INTEGER NOT NULL,
+    total_stake BIGINT NOT NULL,
+    date_utc VARCHAR NOT NULL,
+    packets_received INTEGER DEFAULT 0,
+    packets_sent INTEGER DEFAULT 0,
+    packets_dropped INTEGER DEFAULT 0,
+    FOREIGN KEY (mix_id) REFERENCES mixnodes (mix_id),
+    UNIQUE (mix_id, date_utc) -- This constraint automatically creates an index
+  );

nym-node-status-api/src/cli/mod.rs

@@ -0,0 +1,17 @@
+use clap::Parser;
+use nym_bin_common::bin_info;
+use std::sync::OnceLock;
+
+// Helper for passing LONG_VERSION to clap
+fn pretty_build_info_static() -> &'static str {
+    static PRETTY_BUILD_INFORMATION: OnceLock<String> = OnceLock::new();
+    PRETTY_BUILD_INFORMATION.get_or_init(|| bin_info!().pretty_print())
+}
+
+#[derive(Parser, Debug)]
+#[clap(author = "Nymtech", version, long_version = pretty_build_info_static(), about)]
+pub(crate) struct Cli {
+    /// Path pointing to an env file that configures the Nym API.
+    #[clap(short, long)]
+    pub(crate) config_env_file: Option<std::path::PathBuf>,
+}

nym-node-status-api/src/config.rs

@@ -0,0 +1,72 @@
+use anyhow::anyhow;
+use reqwest::Url;
+use serde::Deserialize;
+use std::time::Duration;
+
+#[derive(Debug, Clone, Deserialize)]
+pub(crate) struct Config {
+    #[serde(default = "Config::default_http_cache_seconds")]
+    nym_http_cache_ttl: u64,
+    #[serde(default = "Config::default_http_port")]
+    http_port: u16,
+    #[serde(rename = "nyxd")]
+    nyxd_addr: Url,
+    #[serde(default = "Config::default_client_timeout")]
+    #[serde(deserialize_with = "parse_duration")]
+    explorer_client_timeout: Duration,
+}
+
+impl Config {
+    pub(crate) fn from_env() -> anyhow::Result<Self> {
+        envy::from_env::<Self>().map_err(|e| {
+            tracing::error!("Failed to load config from env: {e}");
+            anyhow::Error::from(e)
+        })
+    }
+
+    fn default_client_timeout() -> Duration {
+        Duration::from_secs(15)
+    }
+
+    fn default_http_port() -> u16 {
+        8000
+    }
+
+    fn default_http_cache_seconds() -> u64 {
+        30
+    }
+
+    pub(crate) fn nym_http_cache_ttl(&self) -> u64 {
+        self.nym_http_cache_ttl
+    }
+
+    pub(crate) fn http_port(&self) -> u16 {
+        self.http_port
+    }
+
+    pub(crate) fn nyxd_addr(&self) -> &Url {
+        &self.nyxd_addr
+    }
+
+    pub(crate) fn nym_explorer_client_timeout(&self) -> Duration {
+        self.explorer_client_timeout.to_owned()
+    }
+}
+
+fn parse_duration<'de, D>(deserializer: D) -> Result<Duration, D::Error>
+where
+    D: serde::Deserializer<'de>,
+{
+    let s: String = Deserialize::deserialize(deserializer)?;
+    let secs: u64 = s.parse().map_err(serde::de::Error::custom)?;
+    Ok(Duration::from_secs(secs))
+}
+
+pub(super) fn read_env_var(env_var: &str) -> anyhow::Result<String> {
+    std::env::var(env_var)
+        .map_err(|_| anyhow!("You need to set {}", env_var))
+        .map(|value| {
+            tracing::trace!("{}={}", env_var, value);
+            value
+        })
+}

nym-node-status-api/src/db/mod.rs

@@ -0,0 +1,42 @@
+use std::str::FromStr;
+
+use crate::read_env_var;
+use anyhow::{anyhow, Result};
+use sqlx::{migrate::Migrator, sqlite::SqliteConnectOptions, ConnectOptions, SqlitePool};
+
+pub(crate) mod models;
+pub(crate) mod queries;
+
+pub(crate) const DATABASE_URL_ENV_VAR: &str = "DATABASE_URL";
+static MIGRATOR: Migrator = sqlx::migrate!("./migrations");
+
+pub(crate) type DbPool = SqlitePool;
+
+pub(crate) struct Storage {
+    pool: DbPool,
+}
+
+impl Storage {
+    pub async fn init() -> Result<Self> {
+        let connection_url = read_env_var(DATABASE_URL_ENV_VAR)?;
+        let connect_options = {
+            let connect_options = SqliteConnectOptions::from_str(&connection_url)?;
+            let mut connect_options = connect_options.create_if_missing(true);
+            let connect_options = connect_options.disable_statement_logging();
+            (*connect_options).clone()
+        };
+
+        let pool = sqlx::SqlitePool::connect_with(connect_options)
+            .await
+            .map_err(|err| anyhow!("Failed to connect to {}: {}", &connection_url, err))?;
+
+        MIGRATOR.run(&pool).await?;
+
+        Ok(Storage { pool })
+    }
+
+    /// Cloning pool is cheap, it's the same underlying set of connections
+    pub async fn pool_owned(&self) -> DbPool {
+        self.pool.clone()
+    }
+}

nym-node-status-api/src/db/models.rs

@@ -0,0 +1,300 @@
+use crate::{
+    http::{self, models::SummaryHistory},
+    monitor::NumericalCheckedCast,
+};
+use nym_node_requests::api::v1::node::models::NodeDescription;
+use serde::{Deserialize, Serialize};
+use utoipa::ToSchema;
+
+pub(crate) struct GatewayRecord {
+    pub(crate) identity_key: String,
+    pub(crate) bonded: bool,
+    pub(crate) blacklisted: bool,
+    pub(crate) self_described: Option<String>,
+    pub(crate) explorer_pretty_bond: Option<String>,
+    pub(crate) last_updated_utc: i64,
+    pub(crate) performance: u8,
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct GatewayDto {
+    pub(crate) gateway_identity_key: String,
+    pub(crate) bonded: bool,
+    pub(crate) blacklisted: bool,
+    pub(crate) performance: i64,
+    pub(crate) self_described: Option<String>,
+    pub(crate) explorer_pretty_bond: Option<String>,
+    pub(crate) last_probe_result: Option<String>,
+    pub(crate) last_probe_log: Option<String>,
+    pub(crate) last_testrun_utc: Option<i64>,
+    pub(crate) last_updated_utc: i64,
+    pub(crate) moniker: String,
+    pub(crate) security_contact: String,
+    pub(crate) details: String,
+    pub(crate) website: String,
+}
+
+impl TryFrom<GatewayDto> for http::models::Gateway {
+    type Error = anyhow::Error;
+
+    fn try_from(value: GatewayDto) -> Result<Self, Self::Error> {
+        // Instead of using routing_score_successes / routing_score_samples, we use the
+        // number of successful testruns in the last 24h.
+        let routing_score = 0f32;
+        let config_score = 0u32;
+        let last_updated_utc =
+            timestamp_as_utc(value.last_updated_utc.cast_checked()?).to_rfc3339();
+        let last_testrun_utc = value
+            .last_testrun_utc
+            .and_then(|i| i.cast_checked().ok())
+            .map(|t| timestamp_as_utc(t).to_rfc3339());
+
+        let self_described = value.self_described.clone().unwrap_or("null".to_string());
+        let explorer_pretty_bond = value
+            .explorer_pretty_bond
+            .clone()
+            .unwrap_or("null".to_string());
+        let last_probe_result = value
+            .last_probe_result
+            .clone()
+            .unwrap_or("null".to_string());
+        let last_probe_log = value.last_probe_log.clone();
+
+        let self_described = serde_json::from_str(&self_described).unwrap_or(None);
+        let explorer_pretty_bond = serde_json::from_str(&explorer_pretty_bond).unwrap_or(None);
+        let last_probe_result = serde_json::from_str(&last_probe_result).unwrap_or(None);
+
+        let bonded = value.bonded;
+        let blacklisted = value.blacklisted;
+        let performance = value.performance as u8;
+
+        let description = NodeDescription {
+            moniker: value.moniker.clone(),
+            website: value.website.clone(),
+            security_contact: value.security_contact.clone(),
+            details: value.details.clone(),
+        };
+
+        Ok(http::models::Gateway {
+            gateway_identity_key: value.gateway_identity_key.clone(),
+            bonded,
+            blacklisted,
+            performance,
+            self_described,
+            explorer_pretty_bond,
+            description,
+            last_probe_result,
+            last_probe_log,
+            routing_score,
+            config_score,
+            last_testrun_utc,
+            last_updated_utc,
+        })
+    }
+}
+
+fn timestamp_as_utc(unix_timestamp: u64) -> chrono::DateTime<chrono::Utc> {
+    let d = std::time::UNIX_EPOCH + std::time::Duration::from_secs(unix_timestamp);
+    d.into()
+}
+
+pub(crate) struct MixnodeRecord {
+    pub(crate) mix_id: u32,
+    pub(crate) identity_key: String,
+    pub(crate) bonded: bool,
+    pub(crate) total_stake: i64,
+    pub(crate) host: String,
+    pub(crate) http_port: u16,
+    pub(crate) blacklisted: bool,
+    pub(crate) full_details: String,
+    pub(crate) self_described: Option<String>,
+    pub(crate) last_updated_utc: i64,
+    pub(crate) is_dp_delegatee: bool,
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct MixnodeDto {
+    pub(crate) mix_id: i64,
+    pub(crate) bonded: bool,
+    pub(crate) blacklisted: bool,
+    pub(crate) is_dp_delegatee: bool,
+    pub(crate) total_stake: i64,
+    pub(crate) full_details: String,
+    pub(crate) self_described: Option<String>,
+    pub(crate) last_updated_utc: i64,
+    pub(crate) moniker: String,
+    pub(crate) website: String,
+    pub(crate) security_contact: String,
+    pub(crate) details: String,
+}
+
+impl TryFrom<MixnodeDto> for http::models::Mixnode {
+    type Error = anyhow::Error;
+
+    fn try_from(value: MixnodeDto) -> Result<Self, Self::Error> {
+        let mix_id = value.mix_id.cast_checked()?;
+        let full_details = value.full_details.clone();
+        let full_details = serde_json::from_str(&full_details).unwrap_or(None);
+
+        let self_described = value
+            .self_described
+            .clone()
+            .map(|v| serde_json::from_str(&v).unwrap_or(serde_json::Value::Null));
+
+        let last_updated_utc =
+            timestamp_as_utc(value.last_updated_utc.cast_checked()?).to_rfc3339();
+        let blacklisted = value.blacklisted;
+        let is_dp_delegatee = value.is_dp_delegatee;
+        let moniker = value.moniker.clone();
+        let website = value.website.clone();
+        let security_contact = value.security_contact.clone();
+        let details = value.details.clone();
+
+        Ok(http::models::Mixnode {
+            mix_id,
+            bonded: value.bonded,
+            blacklisted,
+            is_dp_delegatee,
+            total_stake: value.total_stake,
+            full_details,
+            description: NodeDescription {
+                moniker,
+                website,
+                security_contact,
+                details,
+            },
+            self_described,
+            last_updated_utc,
+        })
+    }
+}
+
+#[allow(unused)]
+#[derive(Debug, Clone)]
+pub(crate) struct BondedStatusDto {
+    pub(crate) id: i64,
+    pub(crate) identity_key: String,
+    pub(crate) bonded: bool,
+}
+
+#[allow(unused)]
+#[derive(Debug, Clone, Default)]
+pub(crate) struct SummaryDto {
+    pub(crate) key: String,
+    pub(crate) value_json: String,
+    pub(crate) last_updated_utc: i64,
+}
+
+#[derive(Debug, Clone, Default)]
+pub(crate) struct SummaryHistoryDto {
+    #[allow(dead_code)]
+    pub id: i64,
+    pub date: String,
+    pub value_json: String,
+    pub timestamp_utc: i64,
+}
+
+impl TryFrom<SummaryHistoryDto> for SummaryHistory {
+    type Error = anyhow::Error;
+
+    fn try_from(value: SummaryHistoryDto) -> Result<Self, Self::Error> {
+        let value_json = serde_json::from_str(&value.value_json).unwrap_or_default();
+        Ok(SummaryHistory {
+            value_json,
+            date: value.date.clone(),
+            timestamp_utc: timestamp_as_utc(value.timestamp_utc.cast_checked()?).to_rfc3339(),
+        })
+    }
+}
+
+pub(crate) const MIXNODES_BONDED_COUNT: &str = "mixnodes.bonded.count";
+pub(crate) const MIXNODES_BONDED_ACTIVE: &str = "mixnodes.bonded.active";
+pub(crate) const MIXNODES_BONDED_INACTIVE: &str = "mixnodes.bonded.inactive";
+pub(crate) const MIXNODES_BONDED_RESERVE: &str = "mixnodes.bonded.reserve";
+pub(crate) const MIXNODES_BLACKLISTED_COUNT: &str = "mixnodes.blacklisted.count";
+
+pub(crate) const GATEWAYS_BONDED_COUNT: &str = "gateways.bonded.count";
+pub(crate) const GATEWAYS_EXPLORER_COUNT: &str = "gateways.explorer.count";
+pub(crate) const GATEWAYS_BLACKLISTED_COUNT: &str = "gateways.blacklisted.count";
+
+pub(crate) const MIXNODES_HISTORICAL_COUNT: &str = "mixnodes.historical.count";
+pub(crate) const GATEWAYS_HISTORICAL_COUNT: &str = "gateways.historical.count";
+
+// `utoipa`` goes crazy if you use module-qualified prefix as field type so we
+//  have to import it
+use gateway::GatewaySummary;
+use mixnode::MixnodeSummary;
+
+#[derive(Debug, Clone, Deserialize, Serialize, ToSchema)]
+pub(crate) struct NetworkSummary {
+    pub(crate) mixnodes: MixnodeSummary,
+    pub(crate) gateways: GatewaySummary,
+}
+
+pub(crate) mod mixnode {
+    use super::*;
+
+    #[derive(Debug, Clone, Deserialize, Serialize, ToSchema)]
+    pub(crate) struct MixnodeSummary {
+        pub(crate) bonded: MixnodeSummaryBonded,
+        pub(crate) blacklisted: MixnodeSummaryBlacklisted,
+        pub(crate) historical: MixnodeSummaryHistorical,
+    }
+
+    #[derive(Debug, Clone, Deserialize, Serialize, ToSchema)]
+    pub(crate) struct MixnodeSummaryBonded {
+        pub(crate) count: i32,
+        pub(crate) active: i32,
+        pub(crate) inactive: i32,
+        pub(crate) reserve: i32,
+        pub(crate) last_updated_utc: String,
+    }
+
+    #[derive(Debug, Clone, Deserialize, Serialize, ToSchema)]
+    pub(crate) struct MixnodeSummaryBlacklisted {
+        pub(crate) count: i32,
+        pub(crate) last_updated_utc: String,
+    }
+
+    #[derive(Debug, Clone, Deserialize, Serialize, ToSchema)]
+    pub(crate) struct MixnodeSummaryHistorical {
+        pub(crate) count: i32,
+        pub(crate) last_updated_utc: String,
+    }
+}
+
+pub(crate) mod gateway {
+    use super::*;
+
+    #[derive(Debug, Clone, Deserialize, Serialize, ToSchema)]
+    pub(crate) struct GatewaySummary {
+        pub(crate) bonded: GatewaySummaryBonded,
+        pub(crate) blacklisted: GatewaySummaryBlacklisted,
+        pub(crate) historical: GatewaySummaryHistorical,
+        pub(crate) explorer: GatewaySummaryExplorer,
+    }
+
+    #[derive(Debug, Clone, Deserialize, Serialize, ToSchema)]
+    pub(crate) struct GatewaySummaryExplorer {
+        pub(crate) count: i32,
+        pub(crate) last_updated_utc: String,
+    }
+
+    #[derive(Debug, Clone, Deserialize, Serialize, ToSchema)]
+    pub(crate) struct GatewaySummaryBonded {
+        pub(crate) count: i32,
+        pub(crate) last_updated_utc: String,
+    }
+
+    #[derive(Debug, Clone, Deserialize, Serialize, ToSchema)]
+    pub(crate) struct GatewaySummaryHistorical {
+        pub(crate) count: i32,
+        pub(crate) last_updated_utc: String,
+    }
+
+    #[derive(Debug, Clone, Deserialize, Serialize, ToSchema)]
+    pub(crate) struct GatewaySummaryBlacklisted {
+        pub(crate) count: i32,
+        pub(crate) last_updated_utc: String,
+    }
+}

nym-node-status-api/src/db/queries/gateways.rs

@@ -0,0 +1,160 @@
+use crate::{
+    db::{
+        models::{BondedStatusDto, GatewayDto, GatewayRecord},
+        DbPool,
+    },
+    http::models::Gateway,
+};
+use futures_util::TryStreamExt;
+use nym_validator_client::models::DescribedGateway;
+use tracing::error;
+
+pub(crate) async fn insert_gateways(
+    pool: &DbPool,
+    gateways: Vec<GatewayRecord>,
+) -> anyhow::Result<()> {
+    let mut db = pool.acquire().await?;
+    for record in gateways {
+        sqlx::query!(
+            "INSERT INTO gateways
+                (gateway_identity_key, bonded, blacklisted,
+                    self_described, explorer_pretty_bond,
+                    last_updated_utc, performance)
+                VALUES (?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(gateway_identity_key) DO UPDATE SET
+                bonded=excluded.bonded,
+                blacklisted=excluded.blacklisted,
+                self_described=excluded.self_described,
+                explorer_pretty_bond=excluded.explorer_pretty_bond,
+                last_updated_utc=excluded.last_updated_utc,
+                performance = excluded.performance;",
+            record.identity_key,
+            record.bonded,
+            record.blacklisted,
+            record.self_described,
+            record.explorer_pretty_bond,
+            record.last_updated_utc,
+            record.performance
+        )
+        .execute(&mut *db)
+        .await?;
+    }
+
+    Ok(())
+}
+
+pub(crate) async fn write_blacklisted_gateways_to_db<'a, I>(
+    pool: &DbPool,
+    gateways: I,
+) -> anyhow::Result<()>
+where
+    I: Iterator<Item = &'a String>,
+{
+    let mut conn = pool.acquire().await?;
+    for gateway_identity_key in gateways {
+        sqlx::query!(
+            "UPDATE gateways
+             SET blacklisted = true
+             WHERE gateway_identity_key = ?;",
+            gateway_identity_key,
+        )
+        .execute(&mut *conn)
+        .await?;
+    }
+
+    Ok(())
+}
+
+/// Ensure all gateways that are set as bonded, are still bonded
+pub(crate) async fn ensure_gateways_still_bonded(
+    pool: &DbPool,
+    gateways: &[DescribedGateway],
+) -> anyhow::Result<usize> {
+    let bonded_gateways_rows = get_all_bonded_gateways_row_ids_by_status(pool, true).await?;
+    let unbonded_gateways_rows = bonded_gateways_rows.iter().filter(|v| {
+        !gateways
+            .iter()
+            .any(|bonded| *bonded.bond.identity() == v.identity_key)
+    });
+
+    let recently_unbonded_gateways = unbonded_gateways_rows.to_owned().count();
+    let last_updated_utc = chrono::offset::Utc::now().timestamp();
+    let mut transaction = pool.begin().await?;
+    for row in unbonded_gateways_rows {
+        sqlx::query!(
+            "UPDATE gateways
+                SET bonded = ?, last_updated_utc = ?
+                WHERE id = ?;",
+            false,
+            last_updated_utc,
+            row.id,
+        )
+        .execute(&mut *transaction)
+        .await?;
+    }
+    transaction.commit().await?;
+
+    Ok(recently_unbonded_gateways)
+}
+
+async fn get_all_bonded_gateways_row_ids_by_status(
+    pool: &DbPool,
+    status: bool,
+) -> anyhow::Result<Vec<BondedStatusDto>> {
+    let mut conn = pool.acquire().await?;
+    let items = sqlx::query_as!(
+        BondedStatusDto,
+        r#"SELECT
+            id as "id!",
+            gateway_identity_key as "identity_key!",
+            bonded as "bonded: bool"
+         FROM gateways
+         WHERE bonded = ?"#,
+        status,
+    )
+    .fetch(&mut *conn)
+    .try_collect::<Vec<_>>()
+    .await?;
+
+    Ok(items)
+}
+
+pub(crate) async fn get_all_gateways(pool: &DbPool) -> anyhow::Result<Vec<Gateway>> {
+    let mut conn = pool.acquire().await?;
+    let items = sqlx::query_as!(
+        GatewayDto,
+        r#"SELECT
+            gw.gateway_identity_key as "gateway_identity_key!",
+            gw.bonded as "bonded: bool",
+            gw.blacklisted as "blacklisted: bool",
+            gw.performance as "performance!",
+        gw.self_described as "self_described?",
+            gw.explorer_pretty_bond as "explorer_pretty_bond?",
+            gw.last_probe_result as "last_probe_result?",
+            gw.last_probe_log as "last_probe_log?",
+            gw.last_testrun_utc as "last_testrun_utc?",
+            gw.last_updated_utc as "last_updated_utc!",
+            COALESCE(gd.moniker, "NA") as "moniker!",
+            COALESCE(gd.website, "NA") as "website!",
+            COALESCE(gd.security_contact, "NA") as "security_contact!",
+            COALESCE(gd.details, "NA") as "details!"
+         FROM gateways gw
+         LEFT JOIN gateway_description gd
+         ON gw.gateway_identity_key = gd.gateway_identity_key
+         ORDER BY gw.gateway_identity_key"#,
+    )
+    .fetch(&mut conn)
+    .try_collect::<Vec<_>>()
+    .await?;
+
+    let items: Vec<Gateway> = items
+        .into_iter()
+        .map(|item| item.try_into())
+        .collect::<anyhow::Result<Vec<_>>>()
+        .map_err(|e| {
+            error!("Conversion from DTO failed: {e}. Invalidly stored data?");
+            e
+        })?;
+    tracing::trace!("Fetched {} gateways from DB", items.len());
+    Ok(items)
+}

nym-node-status-api/src/db/queries/misc.rs

@@ -0,0 +1,88 @@
+use crate::db::{models::NetworkSummary, DbPool};
+use chrono::{DateTime, Utc};
+
+/// take `last_updated` instead of calculating it so that `summary` matches
+/// `daily_summary`
+pub(crate) async fn insert_summaries(
+    pool: &DbPool,
+    summaries: &[(&str, &usize)],
+    summary: &NetworkSummary,
+    last_updated: DateTime<Utc>,
+) -> anyhow::Result<()> {
+    insert_summary(pool, summaries, last_updated).await?;
+
+    insert_summary_history(pool, summary, last_updated).await?;
+
+    Ok(())
+}
+
+async fn insert_summary(
+    pool: &DbPool,
+    summaries: &[(&str, &usize)],
+    last_updated: DateTime<Utc>,
+) -> anyhow::Result<()> {
+    let timestamp = last_updated.timestamp();
+    let mut tx = pool.begin().await?;
+
+    for (kind, value) in summaries {
+        let value = value.to_string();
+        sqlx::query!(
+            "INSERT INTO summary
+                    (key, value_json, last_updated_utc)
+                    VALUES (?, ?, ?)
+                    ON CONFLICT(key) DO UPDATE SET
+                    value_json=excluded.value_json,
+                    last_updated_utc=excluded.last_updated_utc;",
+            kind,
+            value,
+            timestamp
+        )
+        .execute(&mut tx)
+        .await
+        .map_err(|err| {
+            tracing::error!("Failed to insert data for {kind}: {err}, aborting transaction",);
+            err
+        })?;
+    }
+
+    tx.commit().await?;
+
+    Ok(())
+}
+
+/// For `<date_N>`, `summary_history` is updated with fresh data on every
+/// iteration.
+///
+/// After UTC midnight, summary is inserted for `<date_N+1>` and last entry for
+/// `<date_N>` stays there forever.
+///
+/// This is not aggregate data, it's a set of latest data points
+async fn insert_summary_history(
+    pool: &DbPool,
+    summary: &NetworkSummary,
+    last_updated: DateTime<Utc>,
+) -> anyhow::Result<()> {
+    let mut conn = pool.acquire().await?;
+
+    let value_json = serde_json::to_string(&summary)?;
+    let timestamp = last_updated.timestamp();
+    let now_rfc3339 = last_updated.to_rfc3339();
+    // YYYY-MM-DD, without time
+    let date = &now_rfc3339[..10];
+
+    sqlx::query!(
+        "INSERT INTO summary_history
+                (date, timestamp_utc, value_json)
+                VALUES (?, ?, ?)
+                ON CONFLICT(date) DO UPDATE SET
+                timestamp_utc=excluded.timestamp_utc,
+                value_json=excluded.value_json;",
+        date,
+        timestamp,
+        value_json
+    )
+    .execute(&mut *conn)
+    .await?;
+
+    Ok(())
+}

nym-node-status-api/src/db/queries/mixnodes.rs

@@ -0,0 +1,177 @@
+use futures_util::TryStreamExt;
+use nym_validator_client::models::MixNodeBondAnnotated;
+use tracing::error;
+
+use crate::{
+    db::{
+        models::{BondedStatusDto, MixnodeDto, MixnodeRecord},
+        DbPool,
+    },
+    http::models::{DailyStats, Mixnode},
+};
+
+pub(crate) async fn insert_mixnodes(
+    pool: &DbPool,
+    mixnodes: Vec<MixnodeRecord>,
+) -> anyhow::Result<()> {
+    let mut conn = pool.acquire().await?;
+
+    for record in mixnodes.iter() {
+        // https://www.sqlite.org/lang_upsert.html
+        sqlx::query!(
+            "INSERT INTO mixnodes
+                (mix_id, identity_key, bonded, total_stake,
+                    host, http_api_port, blacklisted, full_details,
+                    self_described, last_updated_utc, is_dp_delegatee)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(mix_id) DO UPDATE SET
+                bonded=excluded.bonded,
+                total_stake=excluded.total_stake, host=excluded.host,
+                http_api_port=excluded.http_api_port,blacklisted=excluded.blacklisted,
+                full_details=excluded.full_details,self_described=excluded.self_described,
+                last_updated_utc=excluded.last_updated_utc,
+                is_dp_delegatee = excluded.is_dp_delegatee;",
+            record.mix_id,
+            record.identity_key,
+            record.bonded,
+            record.total_stake,
+            record.host,
+            record.http_port,
+            record.blacklisted,
+            record.full_details,
+            record.self_described,
+            record.last_updated_utc,
+            record.is_dp_delegatee
+        )
+        .execute(&mut *conn)
+        .await?;
+    }
+
+    Ok(())
+}
+
+pub(crate) async fn get_all_mixnodes(pool: &DbPool) -> anyhow::Result<Vec<Mixnode>> {
+    let mut conn = pool.acquire().await?;
+    let items = sqlx::query_as!(
+        MixnodeDto,
+        r#"SELECT
+            mn.mix_id as "mix_id!",
+            mn.bonded as "bonded: bool",
+            mn.blacklisted as "blacklisted: bool",
+            mn.is_dp_delegatee as "is_dp_delegatee: bool",
+            mn.total_stake as "total_stake!",
+            mn.full_details as "full_details!",
+            mn.self_described as "self_described",
+            mn.last_updated_utc as "last_updated_utc!",
+            COALESCE(md.moniker, "NA") as "moniker!",
+            COALESCE(md.website, "NA") as "website!",
+            COALESCE(md.security_contact, "NA") as "security_contact!",
+            COALESCE(md.details, "NA") as "details!"
+         FROM mixnodes mn
+         LEFT JOIN mixnode_description md ON mn.mix_id = md.mix_id
+         ORDER BY mn.mix_id"#
+    )
+    .fetch(&mut conn)
+    .try_collect::<Vec<_>>()
+    .await?;
+
+    let items = items
+        .into_iter()
+        .map(|item| item.try_into())
+        .collect::<anyhow::Result<Vec<_>>>()
+        .map_err(|e| {
+            error!("Conversion from DTO failed: {e}. Invalidly stored data?");
+            e
+        })?;
+    Ok(items)
+}
+
+/// We fetch the latest 30 days of data as a subquery and then
+/// return it in ascending order, so we don't break existing UI
+pub(crate) async fn get_daily_stats(pool: &DbPool) -> anyhow::Result<Vec<DailyStats>> {
+    let mut conn = pool.acquire().await?;
+    let items = sqlx::query_as!(
+        DailyStats,
+        r#"
+        SELECT
+            date_utc as "date_utc!",
+            packets_received as "total_packets_received!: i64",
+            packets_sent as "total_packets_sent!: i64",
+            packets_dropped as "total_packets_dropped!: i64",
+            total_stake as "total_stake!: i64"
+        FROM (
+            SELECT
+                date_utc,
+                SUM(packets_received) as packets_received,
+                SUM(packets_sent) as packets_sent,
+                SUM(packets_dropped) as packets_dropped,
+                SUM(total_stake) as total_stake
+            FROM mixnode_daily_stats
+            GROUP BY date_utc
+            ORDER BY date_utc DESC
+            LIMIT 30
+        )
+        GROUP BY date_utc
+        ORDER BY date_utc
+        "#
+    )
+    .fetch(&mut conn)
+    .try_collect::<Vec<DailyStats>>()
+    .await?;
+
+    Ok(items)
+}
+
+/// Ensure all mixnodes that are set as bonded, are still bonded
+pub(crate) async fn ensure_mixnodes_still_bonded(
+    pool: &DbPool,
+    mixnodes: &[MixNodeBondAnnotated],
+) -> anyhow::Result<usize> {
+    let bonded_mixnodes_rows = get_all_bonded_mixnodes_row_ids_by_status(pool, true).await?;
+    let unbonded_mixnodes_rows = bonded_mixnodes_rows.iter().filter(|v| {
+        !mixnodes
+            .iter()
+            .any(|bonded| *bonded.mixnode_details.bond_information.identity() == v.identity_key)
+    });
+
+    let recently_unbonded_mixnodes = unbonded_mixnodes_rows.to_owned().count();
+    let last_updated_utc = chrono::offset::Utc::now().timestamp();
+    let mut transaction = pool.begin().await?;
+    for row in unbonded_mixnodes_rows {
+        sqlx::query!(
+            "UPDATE mixnodes
+                SET bonded = ?, last_updated_utc = ?
+                WHERE id = ?;",
+            false,
+            last_updated_utc,
+            row.id,
+        )
+        .execute(&mut *transaction)
+        .await?;
+    }
+    transaction.commit().await?;
+
+    Ok(recently_unbonded_mixnodes)
+}
+
+async fn get_all_bonded_mixnodes_row_ids_by_status(
+    pool: &DbPool,
+    status: bool,
+) -> anyhow::Result<Vec<BondedStatusDto>> {
+    let mut conn = pool.acquire().await?;
+    let items = sqlx::query_as!(
+        BondedStatusDto,
+        r#"SELECT
+            id as "id!",
+            identity_key as "identity_key!",
+            bonded as "bonded: bool"
+         FROM mixnodes
+         WHERE bonded = ?"#,
+        status,
+    )
+    .fetch(&mut *conn)
+    .try_collect::<Vec<_>>()
+    .await?;
+
+    Ok(items)
+}

nym-node-status-api/src/db/queries/mod.rs

@@ -0,0 +1,14 @@
+mod gateways;
+mod misc;
+mod mixnodes;
+mod summary;
+
+pub(crate) use gateways::{
+    ensure_gateways_still_bonded, get_all_gateways, insert_gateways,
+    write_blacklisted_gateways_to_db,
+};
+pub(crate) use misc::insert_summaries;
+pub(crate) use mixnodes::{
+    ensure_mixnodes_still_bonded, get_all_mixnodes, get_daily_stats, insert_mixnodes,
+};
+pub(crate) use summary::{get_summary, get_summary_history};

nym-node-status-api/src/db/queries/summary.rs

@@ -0,0 +1,209 @@
+use chrono::{DateTime, Utc};
+use futures_util::TryStreamExt;
+use std::collections::HashMap;
+use tracing::error;
+
+use crate::{
+    db::{
+        models::{
+            gateway::{
+                GatewaySummary, GatewaySummaryBlacklisted, GatewaySummaryBonded,
+                GatewaySummaryExplorer, GatewaySummaryHistorical,
+            },
+            mixnode::{
+                MixnodeSummary, MixnodeSummaryBlacklisted, MixnodeSummaryBonded,
+                MixnodeSummaryHistorical,
+            },
+            NetworkSummary, SummaryDto, SummaryHistoryDto,
+        },
+        DbPool,
+    },
+    http::{
+        error::{HttpError, HttpResult},
+        models::SummaryHistory,
+    },
+};
+
+pub(crate) async fn get_summary_history(pool: &DbPool) -> anyhow::Result<Vec<SummaryHistory>> {
+    let mut conn = pool.acquire().await?;
+    let items = sqlx::query_as!(
+        SummaryHistoryDto,
+        r#"SELECT
+            id as "id!",
+            date as "date!",
+            timestamp_utc as "timestamp_utc!",
+            value_json as "value_json!"
+         FROM summary_history
+         ORDER BY date DESC
+         LIMIT 30"#,
+    )
+    .fetch(&mut conn)
+    .try_collect::<Vec<_>>()
+    .await?;
+
+    let items = items
+        .into_iter()
+        .map(|item| item.try_into())
+        .collect::<anyhow::Result<Vec<_>>>()
+        .map_err(|e| {
+            error!("Conversion from DTO failed: {e}. Invalidly stored data?");
+            e
+        })?;
+    Ok(items)
+}
+
+async fn get_summary_dto(pool: &DbPool) -> anyhow::Result<Vec<SummaryDto>> {
+    let mut conn = pool.acquire().await?;
+    Ok(sqlx::query_as!(
+        SummaryDto,
+        r#"SELECT
+            key as "key!",
+            value_json as "value_json!",
+            last_updated_utc as "last_updated_utc!"
+         FROM summary"#
+    )
+    .fetch(&mut conn)
+    .try_collect::<Vec<_>>()
+    .await?)
+}
+
+pub(crate) async fn get_summary(pool: &DbPool) -> HttpResult<NetworkSummary> {
+    let items = get_summary_dto(pool).await.map_err(|err| {
+        tracing::error!("Couldn't get Summary from DB: {err}");
+        HttpError::internal()
+    })?;
+    from_summary_dto(items).await
+}
+
+async fn from_summary_dto(items: Vec<SummaryDto>) -> HttpResult<NetworkSummary> {
+    const MIXNODES_BONDED_COUNT: &str = "mixnodes.bonded.count";
+    const MIXNODES_BONDED_ACTIVE: &str = "mixnodes.bonded.active";
+    const MIXNODES_BONDED_INACTIVE: &str = "mixnodes.bonded.inactive";
+    const MIXNODES_BONDED_RESERVE: &str = "mixnodes.bonded.reserve";
+    const MIXNODES_BLACKLISTED_COUNT: &str = "mixnodes.blacklisted.count";
+    const GATEWAYS_BONDED_COUNT: &str = "gateways.bonded.count";
+    const GATEWAYS_EXPLORER_COUNT: &str = "gateways.explorer.count";
+    const GATEWAYS_BLACKLISTED_COUNT: &str = "gateways.blacklisted.count";
+    const MIXNODES_HISTORICAL_COUNT: &str = "mixnodes.historical.count";
+    const GATEWAYS_HISTORICAL_COUNT: &str = "gateways.historical.count";
+
+    // convert database rows into a map by key
+    let mut map = HashMap::new();
+    for item in items {
+        map.insert(item.key.clone(), item);
+    }
+
+    // check we have all the keys we are expecting, and build up a map of errors for missing one
+    let keys = [
+        GATEWAYS_BONDED_COUNT,
+        GATEWAYS_EXPLORER_COUNT,
+        GATEWAYS_HISTORICAL_COUNT,
+        GATEWAYS_BLACKLISTED_COUNT,
+        MIXNODES_BLACKLISTED_COUNT,
+        MIXNODES_BONDED_ACTIVE,
+        MIXNODES_BONDED_COUNT,
+        MIXNODES_BONDED_INACTIVE,
+        MIXNODES_BONDED_RESERVE,
+        MIXNODES_HISTORICAL_COUNT,
+    ];
+
+    let mut errors: Vec<&str> = vec![];
+    for key in keys {
+        if !map.contains_key(key) {
+            errors.push(key);
+        }
+    }
+
+    // return an error if anything is missing, with a nice list
+    if !errors.is_empty() {
+        tracing::error!("Summary value missing: {}", errors.join(", "));
+        return Err(HttpError::internal());
+    }
+
+    // strip the options and use default values (anything missing is trapped above)
+    let mixnodes_bonded_count: SummaryDto =
+        map.get(MIXNODES_BONDED_COUNT).cloned().unwrap_or_default();
+    let mixnodes_bonded_active: SummaryDto =
+        map.get(MIXNODES_BONDED_ACTIVE).cloned().unwrap_or_default();
+    let mixnodes_bonded_inactive: SummaryDto = map
+        .get(MIXNODES_BONDED_INACTIVE)
+        .cloned()
+        .unwrap_or_default();
+    let mixnodes_bonded_reserve: SummaryDto = map
+        .get(MIXNODES_BONDED_RESERVE)
+        .cloned()
+        .unwrap_or_default();
+    let mixnodes_blacklisted_count: SummaryDto = map
+        .get(MIXNODES_BLACKLISTED_COUNT)
+        .cloned()
+        .unwrap_or_default();
+    let gateways_bonded_count: SummaryDto =
+        map.get(GATEWAYS_BONDED_COUNT).cloned().unwrap_or_default();
+    let gateways_explorer_count: SummaryDto = map
+        .get(GATEWAYS_EXPLORER_COUNT)
+        .cloned()
+        .unwrap_or_default();
+    let mixnodes_historical_count: SummaryDto = map
+        .get(MIXNODES_HISTORICAL_COUNT)
+        .cloned()
+        .unwrap_or_default();
+    let gateways_historical_count: SummaryDto = map
+        .get(GATEWAYS_HISTORICAL_COUNT)
+        .cloned()
+        .unwrap_or_default();
+    let gateways_blacklisted_count: SummaryDto = map
+        .get(GATEWAYS_BLACKLISTED_COUNT)
+        .cloned()
+        .unwrap_or_default();
+
+    Ok(NetworkSummary {
+        mixnodes: MixnodeSummary {
+            bonded: MixnodeSummaryBonded {
+                count: to_count_i32(&mixnodes_bonded_count),
+                active: to_count_i32(&mixnodes_bonded_active),
+                reserve: to_count_i32(&mixnodes_bonded_reserve),
+                inactive: to_count_i32(&mixnodes_bonded_inactive),
+                last_updated_utc: to_timestamp(&mixnodes_bonded_count),
+            },
+            blacklisted: MixnodeSummaryBlacklisted {
+                count: to_count_i32(&mixnodes_blacklisted_count),
+                last_updated_utc: to_timestamp(&mixnodes_blacklisted_count),
+            },
+            historical: MixnodeSummaryHistorical {
+                count: to_count_i32(&mixnodes_historical_count),
+                last_updated_utc: to_timestamp(&mixnodes_historical_count),
+            },
+        },
+        gateways: GatewaySummary {
+            bonded: GatewaySummaryBonded {
+                count: to_count_i32(&gateways_bonded_count),
+                last_updated_utc: to_timestamp(&gateways_bonded_count),
+            },
+            blacklisted: GatewaySummaryBlacklisted {
+                count: to_count_i32(&gateways_blacklisted_count),
+                last_updated_utc: to_timestamp(&gateways_blacklisted_count),
+            },
+            historical: GatewaySummaryHistorical {
+                count: to_count_i32(&gateways_historical_count),
+                last_updated_utc: to_timestamp(&gateways_historical_count),
+            },
+            explorer: GatewaySummaryExplorer {
+                count: to_count_i32(&gateways_explorer_count),
+                last_updated_utc: to_timestamp(&gateways_explorer_count),
+            },
+        },
+    })
+}
+
+fn to_count_i32(value: &SummaryDto) -> i32 {
+    value.value_json.parse::<i32>().unwrap_or_default()
+}
+
+fn to_timestamp(value: &SummaryDto) -> String {
+    timestamp_as_utc(value.last_updated_utc as u64).to_rfc3339()
+}
+
+fn timestamp_as_utc(unix_timestamp: u64) -> DateTime<Utc> {
+    let d = std::time::UNIX_EPOCH + std::time::Duration::from_secs(unix_timestamp);
+    d.into()
+}

nym-node-status-api/src/http/api/gateways.rs

@@ -0,0 +1,110 @@
+use axum::{
+    extract::{Path, Query, State},
+    Json, Router,
+};
+use serde::Deserialize;
+use utoipa::IntoParams;
+
+use crate::http::{
+    error::{HttpError, HttpResult},
+    models::{Gateway, GatewaySkinny},
+    state::AppState,
+    PagedResult, Pagination,
+};
+
+pub(crate) fn routes() -> Router<AppState> {
+    Router::new()
+        .route("/", axum::routing::get(gateways))
+        .route("/skinny", axum::routing::get(gateways_skinny))
+        .route("/skinny/:identity_key", axum::routing::get(get_gateway))
+}
+
+#[utoipa::path(
+    tag = "Gateways",
+    get,
+    params(
+        Pagination
+    ),
+    path = "/v2/gateways",
+    responses(
+        (status = 200, body = PagedGateway)
+    )
+)]
+async fn gateways(
+    Query(pagination): Query<Pagination>,
+    State(state): State<AppState>,
+) -> HttpResult<Json<PagedResult<Gateway>>> {
+    let db = state.db_pool();
+    let res = state.cache().get_gateway_list(db).await;
+
+    Ok(Json(PagedResult::paginate(pagination, res)))
+}
+
+#[utoipa::path(
+    tag = "Gateways",
+    get,
+    params(
+        Pagination
+    ),
+    path = "/v2/gateways/skinny",
+    responses(
+        (status = 200, body = PagedGatewaySkinny)
+    )
+)]
+async fn gateways_skinny(
+    Query(pagination): Query<Pagination>,
+    State(state): State<AppState>,
+) -> HttpResult<Json<PagedResult<GatewaySkinny>>> {
+    let db = state.db_pool();
+    let res = state.cache().get_gateway_list(db).await;
+    let res: Vec<GatewaySkinny> = res
+        .iter()
+        .filter(|g| g.bonded)
+        .map(|g| GatewaySkinny {
+            gateway_identity_key: g.gateway_identity_key.clone(),
+            self_described: g.self_described.clone(),
+            performance: g.performance,
+            explorer_pretty_bond: g.explorer_pretty_bond.clone(),
+            last_probe_result: g.last_probe_result.clone(),
+            last_testrun_utc: g.last_testrun_utc.clone(),
+            last_updated_utc: g.last_updated_utc.clone(),
+            routing_score: g.routing_score,
+            config_score: g.config_score,
+        })
+        .collect();
+
+    Ok(Json(PagedResult::paginate(pagination, res)))
+}
+
+#[derive(Deserialize, IntoParams)]
+#[into_params(parameter_in = Path)]
+struct IdentityKeyParam {
+    identity_key: String,
+}
+
+#[utoipa::path(
+    tag = "Gateways",
+    get,
+    params(
+        IdentityKeyParam
+    ),
+    path = "/v2/gateways/{identity_key}",
+    responses(
+        (status = 200, body = Gateway)
+    )
+)]
+async fn get_gateway(
+    Path(IdentityKeyParam { identity_key }): Path<IdentityKeyParam>,
+    State(state): State<AppState>,
+) -> HttpResult<Json<Gateway>> {
+    let db = state.db_pool();
+    let res = state.cache().get_gateway_list(db).await;
+
+    match res
+        .iter()
+        .find(|item| item.gateway_identity_key == identity_key)
+    {
+        Some(res) => Ok(Json(res.clone())),
+        None => Err(HttpError::invalid_input(identity_key)),
+    }
+}

nym-node-status-api/src/http/api/mixnodes.rs

@@ -0,0 +1,91 @@
+use axum::{
+    extract::{Path, Query, State},
+    Json, Router,
+};
+use serde::Deserialize;
+use tracing::instrument;
+use utoipa::IntoParams;
+
+use crate::http::{
+    error::{HttpError, HttpResult},
+    models::{DailyStats, Mixnode},
+    state::AppState,
+    PagedResult, Pagination,
+};
+
+pub(crate) fn routes() -> Router<AppState> {
+    Router::new()
+        .route("/", axum::routing::get(mixnodes))
+        .route("/:mix_id", axum::routing::get(get_mixnodes))
+        .route("/stats", axum::routing::get(get_stats))
+}
+
+#[utoipa::path(
+    tag = "Mixnodes",
+    get,
+    params(
+        Pagination
+    ),
+    path = "/v2/mixnodes",
+    responses(
+        (status = 200, body = PagedMixnode)
+    )
+)]
+#[instrument(level = tracing::Level::DEBUG, skip_all, fields(page=pagination.page, size=pagination.size))]
+async fn mixnodes(
+    Query(pagination): Query<Pagination>,
+    State(state): State<AppState>,
+) -> HttpResult<Json<PagedResult<Mixnode>>> {
+    let db = state.db_pool();
+    let res = state.cache().get_mixnodes_list(db).await;
+
+    Ok(Json(PagedResult::paginate(pagination, res)))
+}
+
+#[derive(Deserialize, IntoParams)]
+#[into_params(parameter_in = Path)]
+struct MixIdParam {
+    mix_id: String,
+}
+
+#[utoipa::path(
+    tag = "Mixnodes",
+    get,
+    params(
+        MixIdParam
+    ),
+    path = "/v2/mixnodes/{mix_id}",
+    responses(
+        (status = 200, body = Mixnode)
+    )
+)]
+#[instrument(level = tracing::Level::DEBUG, skip_all, fields(mix_id = mix_id))]
+async fn get_mixnodes(
+    Path(MixIdParam { mix_id }): Path<MixIdParam>,
+    State(state): State<AppState>,
+) -> HttpResult<Json<Mixnode>> {
+    match mix_id.parse::<u32>() {
+        Ok(parsed_mix_id) => {
+            let res = state.cache().get_mixnodes_list(state.db_pool()).await;
+
+            match res.iter().find(|item| item.mix_id == parsed_mix_id) {
+                Some(res) => Ok(Json(res.clone())),
+                None => Err(HttpError::invalid_input(mix_id)),
+            }
+        }
+        Err(_e) => Err(HttpError::invalid_input(mix_id)),
+    }
+}
+
+#[utoipa::path(
+    tag = "Mixnodes",
+    get,
+    path = "/v2/mixnodes/stats",
+    responses(
+        (status = 200, body = Vec<DailyStats>)
+    )
+)]
+async fn get_stats(State(state): State<AppState>) -> HttpResult<Json<Vec<DailyStats>>> {
+    let stats = state.cache().get_mixnode_stats(state.db_pool()).await;
+    Ok(Json(stats))
+}

nym-node-status-api/src/http/api/mod.rs

@@ -0,0 +1,85 @@
+use anyhow::anyhow;
+use axum::{response::Redirect, Router};
+use tokio::net::ToSocketAddrs;
+use tower_http::{cors::CorsLayer, trace::TraceLayer};
+use utoipa::OpenApi;
+use utoipa_swagger_ui::SwaggerUi;
+
+use crate::http::{server::HttpServer, state::AppState};
+
+pub(crate) mod gateways;
+pub(crate) mod mixnodes;
+pub(crate) mod services;
+pub(crate) mod summary;
+pub(crate) mod testruns;
+
+pub(crate) struct RouterBuilder {
+    unfinished_router: Router<AppState>,
+}
+
+impl RouterBuilder {
+    pub(crate) fn with_default_routes() -> Self {
+        let router = Router::new()
+            .merge(
+                SwaggerUi::new("/swagger")
+                    .url("/api-docs/openapi.json", super::api_docs::ApiDoc::openapi()),
+            )
+            .route(
+                "/",
+                axum::routing::get(|| async { Redirect::permanent("/swagger") }),
+            )
+            .nest(
+                "/v2",
+                Router::new()
+                    .nest("/gateways", gateways::routes())
+                    .nest("/mixnodes", mixnodes::routes())
+                    .nest("/services", services::routes())
+                    .nest("/summary", summary::routes()),
+                // .nest("/testruns", testruns::_routes()),
+            );
+
+        Self {
+            unfinished_router: router,
+        }
+    }
+
+    pub(crate) fn with_state(self, state: AppState) -> RouterWithState {
+        RouterWithState {
+            router: self.finalize_routes().with_state(state),
+        }
+    }
+
+    fn finalize_routes(self) -> Router<AppState> {
+        // layers added later wrap earlier layers
+        self.unfinished_router
+            // CORS layer needs to wrap other API layers
+            .layer(setup_cors())
+            // logger should be outermost layer
+            .layer(TraceLayer::new_for_http())
+    }
+}
+
+pub(crate) struct RouterWithState {
+    router: Router,
+}
+
+impl RouterWithState {
+    pub(crate) async fn build_server<A: ToSocketAddrs>(
+        self,
+        bind_address: A,
+    ) -> anyhow::Result<HttpServer> {
+        tokio::net::TcpListener::bind(bind_address)
+            .await
+            .map(|listener| HttpServer::new(self.router, listener))
+            .map_err(|err| anyhow!("Couldn't bind to address due to {}", err))
+    }
+}
+
+fn setup_cors() -> CorsLayer {
+    use axum::http::Method;
+    CorsLayer::new()
+        .allow_origin(tower_http::cors::Any)
+        .allow_methods([Method::POST, Method::GET, Method::PATCH, Method::OPTIONS])
+        .allow_headers(tower_http::cors::Any)
+        .allow_credentials(false)
+}

nym-node-status-api/src/http/api/services/json_path.rs

@@ -0,0 +1,58 @@
+use serde_json_path::JsonPath;
+
+use crate::http::models::Gateway;
+
+pub(super) struct ParseJsonPaths {
+    pub(super) path_ip_address: JsonPath,
+    pub(super) path_hostname: JsonPath,
+    pub(super) path_service_provider_client_id: JsonPath,
+}
+
+impl ParseJsonPaths {
+    pub fn new() -> Result<Self, serde_json_path::ParseError> {
+        Ok(ParseJsonPaths {
+            path_ip_address: JsonPath::parse("$.host_information.ip_address[0]")?,
+            path_hostname: JsonPath::parse("$.host_information.hostname")?,
+            path_service_provider_client_id: JsonPath::parse("$.network_requester.address")?,
+        })
+    }
+}
+
+pub(super) struct ParsedDetails {
+    pub(super) ip_address: Option<String>,
+    pub(super) hostname: Option<String>,
+    pub(super) service_provider_client_id: Option<String>,
+}
+
+impl ParsedDetails {
+    fn get_string_from_json_path(
+        value: &Option<serde_json::Value>,
+        path: &JsonPath,
+    ) -> Option<String> {
+        match value {
+            Some(value) => path
+                .query(value)
+                .exactly_one()
+                .map(|v2| v2.as_str().map(|v3| v3.to_string()))
+                .ok()
+                .flatten(),
+            None => None,
+        }
+    }
+    pub fn new(paths: &ParseJsonPaths, g: &Gateway) -> ParsedDetails {
+        ParsedDetails {
+            hostname: ParsedDetails::get_string_from_json_path(
+                &g.self_described,
+                &paths.path_hostname,
+            ),
+            ip_address: ParsedDetails::get_string_from_json_path(
+                &g.self_described,
+                &paths.path_ip_address,
+            ),
+            service_provider_client_id: ParsedDetails::get_string_from_json_path(
+                &g.self_described,
+                &paths.path_service_provider_client_id,
+            ),
+        }
+    }
+}

nym-node-status-api/src/http/api/services/mod.rs

@@ -0,0 +1,134 @@
+use crate::http::{
+    error::{HttpError, HttpResult},
+    models::Service,
+    state::AppState,
+    PagedResult, Pagination,
+};
+use axum::{
+    extract::{Query, State},
+    Json, Router,
+};
+use json_path::{ParseJsonPaths, ParsedDetails};
+use tracing::instrument;
+
+mod json_path;
+
+pub(crate) fn routes() -> Router<AppState> {
+    Router::new().route("/", axum::routing::get(mixnodes))
+}
+
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, utoipa::IntoParams)]
+#[into_params(parameter_in = Query)]
+pub(crate) struct ServicesQueryParams {
+    size: Option<usize>,
+    page: Option<usize>,
+    wss: Option<bool>,
+    hostname: Option<bool>,
+    entry: Option<bool>,
+}
+
+#[utoipa::path(
+    tag = "Services",
+    get,
+    params(
+        ServicesQueryParams,
+    ),
+    path = "/v2/services",
+    responses(
+        (status = 200, body = PagedService)
+    )
+)]
+#[instrument(level = tracing::Level::DEBUG, skip(state))]
+async fn mixnodes(
+    Query(ServicesQueryParams {
+        size,
+        page,
+        wss,
+        hostname,
+        entry,
+    }): Query<ServicesQueryParams>,
+    State(state): State<AppState>,
+) -> HttpResult<Json<PagedResult<Service>>> {
+    let db = state.db_pool();
+    let cache = state.cache();
+
+    let show_only_wss = wss.unwrap_or(false);
+    let show_only_with_hostname = hostname.unwrap_or(false);
+    let show_entry_gateways_only = entry.unwrap_or(false);
+
+    let paths = ParseJsonPaths::new().map_err(|e| {
+        tracing::error!("Invalidly configured ParseJsonPaths: {e}");
+        HttpError::internal()
+    })?;
+    let res = cache.get_gateway_list(db).await;
+    let res: Vec<Service> = res
+        .iter()
+        .map(|g| {
+            let details = ParsedDetails::new(&paths, g);
+
+            let s = Service {
+                gateway_identity_key: g.gateway_identity_key.clone(),
+                ip_address: details.ip_address,
+                service_provider_client_id: details.service_provider_client_id,
+                hostname: details.hostname,
+                last_successful_ping_utc: g.last_testrun_utc.clone(),
+                last_updated_utc: g.last_updated_utc.clone(),
+                // routing_score: g.routing_score,
+                routing_score: 1f32,
+                mixnet_websockets: g
+                    .self_described
+                    .clone()
+                    .and_then(|s| s.get("mixnet_websockets").cloned()),
+            };
+
+            let f = ServiceFilter::new(&s);
+
+            (s, f)
+        })
+        .filter(|(_, f)| {
+            let mut keep = f.has_network_requester_sp;
+
+            if show_entry_gateways_only {
+                keep = true;
+            }
+
+            if show_only_wss {
+                keep &= f.has_wss;
+            }
+            if show_only_with_hostname {
+                keep &= f.has_hostname;
+            }
+
+            keep
+        })
+        .map(|(s, _)| s)
+        .collect();
+
+    Ok(Json(PagedResult::paginate(Pagination { size, page }, res)))
+}
+
+struct ServiceFilter {
+    has_wss: bool,
+    has_network_requester_sp: bool,
+    has_hostname: bool,
+}
+
+impl ServiceFilter {
+    fn new(s: &Service) -> Self {
+        let has_wss = match &s.mixnet_websockets {
+            Some(v) => v.get("wss_port").map(|v2| !v2.is_null()).unwrap_or(false),
+            None => false,
+        };
+        let has_hostname = s.hostname.is_some();
+        let has_network_requester_sp = match &s.service_provider_client_id {
+            Some(v) => !v.is_empty(),
+            None => false,
+        };
+
+        ServiceFilter {
+            has_wss,
+            has_hostname,
+            has_network_requester_sp,
+        }
+    }
+}

nym-node-status-api/src/http/api/summary.rs

@@ -0,0 +1,43 @@
+use axum::{extract::State, Json, Router};
+use tracing::instrument;
+
+use crate::{
+    db::models::NetworkSummary,
+    http::{error::HttpResult, models::SummaryHistory, state::AppState},
+};
+
+pub(crate) fn routes() -> Router<AppState> {
+    Router::new()
+        .route("/", axum::routing::get(summary))
+        .route("/history", axum::routing::get(summary_history))
+}
+
+#[utoipa::path(
+    tag = "Summary",
+    get,
+    path = "/v2/summary",
+    responses(
+        (status = 200, body = NetworkSummary)
+    )
+)]
+#[instrument(level = tracing::Level::DEBUG, skip_all)]
+async fn summary(State(state): State<AppState>) -> HttpResult<Json<NetworkSummary>> {
+    crate::db::queries::get_summary(state.db_pool())
+        .await
+        .map(Json)
+}
+
+#[utoipa::path(
+    tag = "Summary",
+    get,
+    path = "/v2/summary/history",
+    responses(
+        (status = 200, body = Vec<SummaryHistory>)
+    )
+)]
+#[instrument(level = tracing::Level::DEBUG, skip_all)]
+async fn summary_history(State(state): State<AppState>) -> HttpResult<Json<Vec<SummaryHistory>>> {
+    Ok(Json(
+        state.cache().get_summary_history(state.db_pool()).await,
+    ))
+}

nym-node-status-api/src/http/api/testruns.rs

@@ -0,0 +1,7 @@
+use axum::Router;
+
+use crate::http::state::AppState;
+
+pub(crate) fn _routes() -> Router<AppState> {
+    unimplemented!()
+}

nym-node-status-api/src/http/api_docs.rs

@@ -0,0 +1,15 @@
+use crate::http::{Gateway, GatewaySkinny, Mixnode, Service};
+use utoipa::OpenApi;
+use utoipauto::utoipauto;
+
+// manually import external structs which are behind feature flags because they
+// can't be automatically discovered
+// https://github.com/ProbablyClem/utoipauto/issues/13#issuecomment-1974911829
+#[utoipauto(paths = "./nym-node-status-api/src")]
+#[derive(OpenApi)]
+#[openapi(
+    info(title = "Nym API"),
+    tags(),
+    components(schemas(nym_node_requests::api::v1::node::models::NodeDescription,))
+)]
+pub(super) struct ApiDoc;

nym-node-status-api/src/http/error.rs

@@ -0,0 +1,28 @@
+pub(crate) type HttpResult<T> = Result<T, HttpError>;
+
+pub(crate) struct HttpError {
+    message: String,
+    status: axum::http::StatusCode,
+}
+
+impl HttpError {
+    pub(crate) fn invalid_input(message: String) -> Self {
+        Self {
+            message,
+            status: axum::http::StatusCode::BAD_REQUEST,
+        }
+    }
+
+    pub(crate) fn internal() -> Self {
+        Self {
+            message: serde_json::json!({"message": "Internal server error"}).to_string(),
+            status: axum::http::StatusCode::INTERNAL_SERVER_ERROR,
+        }
+    }
+}
+
+impl axum::response::IntoResponse for HttpError {
+    fn into_response(self) -> axum::response::Response {
+        (self.status, self.message).into_response()
+    }
+}

nym-node-status-api/src/http/mod.rs

@@ -0,0 +1,71 @@
+use models::{Gateway, GatewaySkinny, Mixnode, Service};
+
+pub(crate) mod api;
+pub(crate) mod api_docs;
+pub(crate) mod error;
+pub(crate) mod models;
+pub(crate) mod server;
+pub(crate) mod state;
+
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, utoipa::ToSchema)]
+// exclude generic from auto-discovery
+#[utoipauto::utoipa_ignore]
+// https://docs.rs/utoipa/latest/utoipa/derive.ToSchema.html#generic-schemas-with-aliases
+// Generic structs can only be included via aliases, not directly, because they
+// it would cause an error in generated Swagger docs.
+// Instead, you have to manually monomorphize each generic struct that
+// you wish to document
+#[aliases(
+    PagedGateway = PagedResult<Gateway>,
+    PagedGatewaySkinny = PagedResult<GatewaySkinny>,
+    PagedMixnode = PagedResult<Mixnode>,
+    PagedService = PagedResult<Service>,
+)]
+pub struct PagedResult<T> {
+    pub page: usize,
+    pub size: usize,
+    pub total: usize,
+    pub items: Vec<T>,
+}
+
+impl<T: Clone> PagedResult<T> {
+    pub fn paginate(pagination: Pagination, res: Vec<T>) -> Self {
+        let total = res.len();
+        let (size, mut page) = pagination.intoto_inner_values();
+
+        if page * size > total {
+            page = total / size;
+        }
+
+        let chunks: Vec<&[T]> = res.chunks(size).collect();
+
+        PagedResult {
+            page,
+            size,
+            total,
+            items: chunks.get(page).cloned().unwrap_or_default().into(),
+        }
+    }
+}
+
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, utoipa::IntoParams)]
+#[into_params(parameter_in = Query)]
+pub(crate) struct Pagination {
+    size: Option<usize>,
+    page: Option<usize>,
+}
+
+impl Pagination {
+    // unwrap stored values or use predefined defaults
+    pub(crate) fn intoto_inner_values(self) -> (usize, usize) {
+        const SIZE_DEFAULT: usize = 10;
+        const SIZE_MAX: usize = 200;
+
+        const PAGE_DEFAULT: usize = 0;
+
+        (
+            self.size.unwrap_or(SIZE_DEFAULT).min(SIZE_MAX),
+            self.page.unwrap_or(PAGE_DEFAULT),
+        )
+    }
+}

nym-node-status-api/src/http/models.rs

@@ -0,0 +1,74 @@
+use nym_node_requests::api::v1::node::models::NodeDescription;
+use serde::{Deserialize, Serialize};
+use utoipa::ToSchema;
+
+#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
+pub struct Gateway {
+    pub gateway_identity_key: String,
+    pub bonded: bool,
+    pub blacklisted: bool,
+    pub performance: u8,
+    pub self_described: Option<serde_json::Value>,
+    pub explorer_pretty_bond: Option<serde_json::Value>,
+    pub description: NodeDescription,
+    pub last_probe_result: Option<serde_json::Value>,
+    pub last_probe_log: Option<String>,
+    pub last_testrun_utc: Option<String>,
+    pub last_updated_utc: String,
+    pub routing_score: f32,
+    pub config_score: u32,
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize, ToSchema)]
+pub struct GatewaySkinny {
+    pub gateway_identity_key: String,
+    pub self_described: Option<serde_json::Value>,
+    pub explorer_pretty_bond: Option<serde_json::Value>,
+    pub last_probe_result: Option<serde_json::Value>,
+    pub last_testrun_utc: Option<String>,
+    pub last_updated_utc: String,
+    pub routing_score: f32,
+    pub config_score: u32,
+    pub performance: u8,
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize, ToSchema)]
+pub struct Mixnode {
+    pub mix_id: u32,
+    pub bonded: bool,
+    pub blacklisted: bool,
+    pub is_dp_delegatee: bool,
+    pub total_stake: i64,
+    pub full_details: Option<serde_json::Value>,
+    pub self_described: Option<serde_json::Value>,
+    pub description: NodeDescription,
+    pub last_updated_utc: String,
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize, ToSchema)]
+pub struct DailyStats {
+    pub date_utc: String,
+    pub total_packets_received: i64,
+    pub total_packets_sent: i64,
+    pub total_packets_dropped: i64,
+    pub total_stake: i64,
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize, ToSchema)]
+pub struct Service {
+    pub gateway_identity_key: String,
+    pub last_updated_utc: String,
+    pub routing_score: f32,
+    pub service_provider_client_id: Option<String>,
+    pub ip_address: Option<String>,
+    pub hostname: Option<String>,
+    pub mixnet_websockets: Option<serde_json::Value>,
+    pub last_successful_ping_utc: Option<String>,
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize, ToSchema)]
+pub(crate) struct SummaryHistory {
+    pub date: String,
+    pub value_json: serde_json::Value,
+    pub timestamp_utc: String,
+}

nym-node-status-api/src/http/server.rs

@@ -0,0 +1,92 @@
+use axum::Router;
+use core::net::SocketAddr;
+use tokio::{net::TcpListener, task::JoinHandle};
+use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned};
+
+use crate::{
+    db::DbPool,
+    http::{api::RouterBuilder, state::AppState},
+};
+
+/// Return handles that allow for graceful shutdown of server + awaiting its
+/// background tokio task
+pub(crate) async fn start_http_api(
+    db_pool: DbPool,
+    http_port: u16,
+    nym_http_cache_ttl: u64,
+) -> anyhow::Result<ShutdownHandles> {
+    let router_builder = RouterBuilder::with_default_routes();
+
+    let state = AppState::new(db_pool, nym_http_cache_ttl);
+    let router = router_builder.with_state(state);
+
+    // TODO dz do we need this to be configurable?
+    let bind_addr = format!("0.0.0.0:{}", http_port);
+    let server = router.build_server(bind_addr).await?;
+
+    Ok(start_server(server))
+}
+
+fn start_server(server: HttpServer) -> ShutdownHandles {
+    // one copy is stored to trigger a graceful shutdown later
+    let shutdown_button = CancellationToken::new();
+    // other copy is given to server to listen for a shutdown
+    let shutdown_receiver = shutdown_button.clone();
+    let shutdown_receiver = shutdown_receiver.cancelled_owned();
+
+    let server_handle = tokio::spawn(async move { server.run(shutdown_receiver).await });
+
+    ShutdownHandles {
+        server_handle,
+        shutdown_button,
+    }
+}
+
+pub(crate) struct ShutdownHandles {
+    server_handle: JoinHandle<std::io::Result<()>>,
+    shutdown_button: CancellationToken,
+}
+
+impl ShutdownHandles {
+    /// Send graceful shutdown signal to server and wait for server task to complete
+    pub(crate) async fn shutdown(self) -> anyhow::Result<()> {
+        self.shutdown_button.cancel();
+
+        match self.server_handle.await {
+            Ok(Ok(_)) => {
+                tracing::info!("HTTP server shut down without errors");
+            }
+            Ok(Err(err)) => {
+                tracing::error!("HTTP server terminated with: {err}");
+                anyhow::bail!(err)
+            }
+            Err(err) => {
+                tracing::error!("Server task panicked: {err}");
+            }
+        };
+
+        Ok(())
+    }
+}
+
+pub(crate) struct HttpServer {
+    router: Router,
+    listener: TcpListener,
+}
+
+impl HttpServer {
+    pub(crate) fn new(router: Router, listener: TcpListener) -> Self {
+        Self { router, listener }
+    }
+
+    pub(crate) async fn run(self, receiver: WaitForCancellationFutureOwned) -> std::io::Result<()> {
+        // into_make_service_with_connect_info allows us to see client ip address
+        axum::serve(
+            self.listener,
+            self.router
+                .into_make_service_with_connect_info::<SocketAddr>(),
+        )
+        .with_graceful_shutdown(receiver)
+        .await
+    }
+}

nym-node-status-api/src/http/state.rs

@@ -0,0 +1,223 @@
+use std::{sync::Arc, time::Duration};
+
+use moka::{future::Cache, Entry};
+use tokio::sync::RwLock;
+
+use crate::{
+    db::DbPool,
+    http::models::{DailyStats, Gateway, Mixnode, SummaryHistory},
+};
+
+#[derive(Debug, Clone)]
+pub(crate) struct AppState {
+    db_pool: DbPool,
+    cache: HttpCache,
+}
+
+impl AppState {
+    pub(crate) fn new(db_pool: DbPool, cache_ttl: u64) -> Self {
+        Self {
+            db_pool,
+            cache: HttpCache::new(cache_ttl),
+        }
+    }
+
+    pub(crate) fn db_pool(&self) -> &DbPool {
+        &self.db_pool
+    }
+
+    pub(crate) fn cache(&self) -> &HttpCache {
+        &self.cache
+    }
+}
+
+static GATEWAYS_LIST_KEY: &str = "gateways";
+static MIXNODES_LIST_KEY: &str = "mixnodes";
+static MIXSTATS_LIST_KEY: &str = "mixstats";
+static SUMMARY_HISTORY_LIST_KEY: &str = "summary-history";
+
+#[derive(Debug, Clone)]
+pub(crate) struct HttpCache {
+    gateways: Cache<String, Arc<RwLock<Vec<Gateway>>>>,
+    mixnodes: Cache<String, Arc<RwLock<Vec<Mixnode>>>>,
+    mixstats: Cache<String, Arc<RwLock<Vec<DailyStats>>>>,
+    history: Cache<String, Arc<RwLock<Vec<SummaryHistory>>>>,
+}
+
+impl HttpCache {
+    pub fn new(ttl_seconds: u64) -> Self {
+        HttpCache {
+            gateways: Cache::builder()
+                .max_capacity(2)
+                .time_to_live(Duration::from_secs(ttl_seconds))
+                .build(),
+            mixnodes: Cache::builder()
+                .max_capacity(2)
+                .time_to_live(Duration::from_secs(ttl_seconds))
+                .build(),
+            mixstats: Cache::builder()
+                .max_capacity(2)
+                .time_to_live(Duration::from_secs(ttl_seconds))
+                .build(),
+            history: Cache::builder()
+                .max_capacity(2)
+                .time_to_live(Duration::from_secs(ttl_seconds))
+                .build(),
+        }
+    }
+
+    pub async fn upsert_gateway_list(
+        &self,
+        new_gateway_list: Vec<Gateway>,
+    ) -> Entry<String, Arc<RwLock<Vec<Gateway>>>> {
+        self.gateways
+            .entry_by_ref(GATEWAYS_LIST_KEY)
+            .and_upsert_with(|maybe_entry| async {
+                if let Some(entry) = maybe_entry {
+                    let v = entry.into_value();
+                    let mut guard = v.write().await;
+                    *guard = new_gateway_list;
+                    v.clone()
+                } else {
+                    Arc::new(RwLock::new(new_gateway_list))
+                }
+            })
+            .await
+    }
+
+    pub async fn get_gateway_list(&self, db: &DbPool) -> Vec<Gateway> {
+        match self.gateways.get(GATEWAYS_LIST_KEY).await {
+            Some(guard) => {
+                let read_lock = guard.read().await;
+                read_lock.clone()
+            }
+            None => {
+                // the key is missing so populate it
+                tracing::warn!("No gateways in cache, refreshing cache from DB...");
+
+                let gateways = crate::db::queries::get_all_gateways(db)
+                    .await
+                    .unwrap_or_default();
+                self.upsert_gateway_list(gateways.clone()).await;
+
+                if gateways.is_empty() {
+                    tracing::warn!("Database contains 0 gateways");
+                }
+
+                gateways
+            }
+        }
+    }
+
+    pub async fn upsert_mixnode_list(
+        &self,
+        new_mixnode_list: Vec<Mixnode>,
+    ) -> Entry<String, Arc<RwLock<Vec<Mixnode>>>> {
+        self.mixnodes
+            .entry_by_ref(MIXNODES_LIST_KEY)
+            .and_upsert_with(|maybe_entry| async {
+                if let Some(entry) = maybe_entry {
+                    let v = entry.into_value();
+                    let mut guard = v.write().await;
+                    *guard = new_mixnode_list;
+                    v.clone()
+                } else {
+                    Arc::new(RwLock::new(new_mixnode_list))
+                }
+            })
+            .await
+    }
+
+    pub async fn get_mixnodes_list(&self, db: &DbPool) -> Vec<Mixnode> {
+        match self.mixnodes.get(MIXNODES_LIST_KEY).await {
+            Some(guard) => {
+                let read_lock = guard.read().await;
+                read_lock.clone()
+            }
+            None => {
+                tracing::warn!("No mixnodes in cache, refreshing cache from DB...");
+
+                let mixnodes = crate::db::queries::get_all_mixnodes(db)
+                    .await
+                    .unwrap_or_default();
+                self.upsert_mixnode_list(mixnodes.clone()).await;
+
+                if mixnodes.is_empty() {
+                    tracing::warn!("Database contains 0 mixnodes");
+                }
+
+                mixnodes
+            }
+        }
+    }
+
+    pub async fn upsert_mixnode_stats(
+        &self,
+        mixnode_stats: Vec<DailyStats>,
+    ) -> Entry<String, Arc<RwLock<Vec<DailyStats>>>> {
+        self.mixstats
+            .entry_by_ref(MIXSTATS_LIST_KEY)
+            .and_upsert_with(|maybe_entry| async {
+                if let Some(entry) = maybe_entry {
+                    let v = entry.into_value();
+                    let mut guard = v.write().await;
+                    *guard = mixnode_stats;
+                    v.clone()
+                } else {
+                    Arc::new(RwLock::new(mixnode_stats))
+                }
+            })
+            .await
+    }
+
+    pub async fn get_mixnode_stats(&self, db: &DbPool) -> Vec<DailyStats> {
+        match self.mixstats.get(MIXSTATS_LIST_KEY).await {
+            Some(guard) => {
+                let read_lock = guard.read().await;
+                read_lock.to_vec()
+            }
+            None => {
+                let mixnode_stats = crate::db::queries::get_daily_stats(db)
+                    .await
+                    .unwrap_or_default();
+                self.upsert_mixnode_stats(mixnode_stats.clone()).await;
+                mixnode_stats
+            }
+        }
+    }
+
+    pub async fn get_summary_history(&self, db: &DbPool) -> Vec<SummaryHistory> {
+        match self.history.get(SUMMARY_HISTORY_LIST_KEY).await {
+            Some(guard) => {
+                let read_lock = guard.read().await;
+                read_lock.to_vec()
+            }
+            None => {
+                let summary_history = crate::db::queries::get_summary_history(db)
+                    .await
+                    .unwrap_or(vec![]);
+                self.upsert_summary_history(summary_history.clone()).await;
+                summary_history
+            }
+        }
+    }
+
+    pub async fn upsert_summary_history(
+        &self,
+        summary_history: Vec<SummaryHistory>,
+    ) -> Entry<String, Arc<RwLock<Vec<SummaryHistory>>>> {
+        self.history
+            .entry_by_ref(SUMMARY_HISTORY_LIST_KEY)
+            .and_upsert_with(|maybe_entry| async {
+                if let Some(entry) = maybe_entry {
+                    let v = entry.into_value();
+                    let mut guard = v.write().await;
+                    *guard = summary_history;
+                    v.clone()
+                } else {
+                    Arc::new(RwLock::new(summary_history))
+                }
+            })
+            .await
+    }
+}

nym-node-status-api/src/logging.rs

@@ -0,0 +1,48 @@
+use tracing::level_filters::LevelFilter;
+use tracing_subscriber::{filter::Directive, EnvFilter};
+
+pub(crate) fn setup_tracing_logger() {
+    fn directive_checked(directive: impl Into<String>) -> Directive {
+        directive
+            .into()
+            .parse()
+            .expect("Failed to parse log directive")
+    }
+
+    let log_builder = tracing_subscriber::fmt()
+        // Use a more compact, abbreviated log format
+        .compact()
+        // Display source code file paths
+        .with_file(true)
+        // Display source code line numbers
+        .with_line_number(true)
+        .with_thread_ids(true)
+        // Don't display the event's target (module path)
+        .with_target(false);
+
+    let mut filter = EnvFilter::builder()
+        // if RUST_LOG isn't set, set default level
+        .with_default_directive(LevelFilter::INFO.into())
+        .from_env_lossy();
+    // these crates are more granularly filtered
+    let filter_crates = [
+        "reqwest",
+        "rustls",
+        "hyper",
+        "sqlx",
+        "h2",
+        "tendermint_rpc",
+        "tower_http",
+        "axum",
+    ];
+    for crate_name in filter_crates {
+        filter = filter.add_directive(directive_checked(format!("{}=warn", crate_name)));
+    }
+
+    filter = filter.add_directive(directive_checked("nym_bin_common=debug"));
+    filter = filter.add_directive(directive_checked("nym_explorer_client=debug"));
+    filter = filter.add_directive(directive_checked("nym_network_defaults=debug"));
+    filter = filter.add_directive(directive_checked("nym_validator_client=debug"));
+
+    log_builder.with_env_filter(filter).init();
+}

nym-node-status-api/src/main.rs

@@ -0,0 +1,54 @@
+use clap::Parser;
+use nym_network_defaults::setup_env;
+use nym_task::signal::wait_for_signal;
+
+use crate::config::read_env_var;
+
+mod cli;
+mod config;
+mod db;
+mod http;
+mod logging;
+mod monitor;
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    logging::setup_tracing_logger();
+
+    let args = cli::Cli::parse();
+    // if dotenv file is present, load its values
+    // otherwise, default to mainnet
+    setup_env(args.config_env_file.as_ref());
+    tracing::debug!("{:?}", read_env_var("NETWORK_NAME"));
+    tracing::debug!("{:?}", read_env_var("EXPLORER_API"));
+    tracing::debug!("{:?}", read_env_var("NYM_API"));
+
+    let conf = config::Config::from_env()?;
+    tracing::debug!("Using config:\n{:#?}", conf);
+
+    let storage = db::Storage::init().await?;
+    let db_pool = storage.pool_owned().await;
+    let conf_clone = conf.clone();
+    tokio::spawn(async move {
+        monitor::spawn_in_background(db_pool, conf_clone).await;
+    });
+    tracing::info!("Started monitor task");
+
+    let shutdown_handles = http::server::start_http_api(
+        storage.pool_owned().await,
+        conf.http_port(),
+        conf.nym_http_cache_ttl(),
+    )
+    .await
+    .expect("Failed to start server");
+
+    tracing::info!("Started HTTP server on port {}", conf.http_port());
+
+    wait_for_signal().await;
+
+    if let Err(err) = shutdown_handles.shutdown().await {
+        tracing::error!("{err}");
+    };
+
+    Ok(())
+}

nym-node-status-api/src/monitor/mod.rs

@@ -0,0 +1,451 @@
+use crate::config::Config;
+use crate::db::models::{
+    gateway, mixnode, GatewayRecord, MixnodeRecord, NetworkSummary, GATEWAYS_BLACKLISTED_COUNT,
+    GATEWAYS_BONDED_COUNT, GATEWAYS_EXPLORER_COUNT, GATEWAYS_HISTORICAL_COUNT,
+    MIXNODES_BLACKLISTED_COUNT, MIXNODES_BONDED_ACTIVE, MIXNODES_BONDED_COUNT,
+    MIXNODES_BONDED_INACTIVE, MIXNODES_BONDED_RESERVE, MIXNODES_HISTORICAL_COUNT,
+};
+use crate::db::{queries, DbPool};
+use anyhow::anyhow;
+use cosmwasm_std::Decimal;
+use nym_explorer_client::{ExplorerClient, PrettyDetailedGatewayBond};
+use nym_network_defaults::NymNetworkDetails;
+use nym_validator_client::client::NymApiClientExt;
+use nym_validator_client::models::{DescribedGateway, DescribedMixNode, MixNodeBondAnnotated};
+use nym_validator_client::nym_nodes::SkimmedNode;
+use nym_validator_client::nyxd::contract_traits::PagedMixnetQueryClient;
+use nym_validator_client::nyxd::{AccountId, NyxdClient};
+use nym_validator_client::NymApiClient;
+use reqwest::Url;
+use std::collections::HashSet;
+use std::str::FromStr;
+use tokio::task::JoinHandle;
+use tokio::time::Duration;
+
+const REFRESH_DELAY: Duration = Duration::from_secs(60 * 5);
+const FAILURE_RETRY_DELAY: Duration = Duration::from_secs(15);
+
+static DELEGATION_PROGRAM_WALLET: &str = "n1rnxpdpx3kldygsklfft0gech7fhfcux4zst5lw";
+
+// TODO dz: query many NYM APIs:
+// multiple instances running directory cache, ask sachin
+pub(crate) async fn spawn_in_background(db_pool: DbPool, config: Config) -> JoinHandle<()> {
+    let network_defaults = nym_network_defaults::NymNetworkDetails::new_from_env();
+
+    loop {
+        tracing::info!("Refreshing node info...");
+
+        if let Err(e) = run(&db_pool, &network_defaults, &config).await {
+            tracing::error!(
+                "Monitor run failed: {e}, retrying in {}s...",
+                FAILURE_RETRY_DELAY.as_secs()
+            );
+            tokio::time::sleep(FAILURE_RETRY_DELAY).await;
+        } else {
+            tracing::info!(
+                "Info successfully collected, sleeping for {}s...",
+                REFRESH_DELAY.as_secs()
+            );
+            tokio::time::sleep(REFRESH_DELAY).await;
+        }
+    }
+}
+
+async fn run(
+    pool: &DbPool,
+    network_details: &NymNetworkDetails,
+    config: &Config,
+) -> anyhow::Result<()> {
+    let default_api_url = network_details
+        .endpoints
+        .first()
+        .expect("rust sdk mainnet default incorrectly configured")
+        .api_url()
+        .clone()
+        .expect("rust sdk mainnet default missing api_url");
+    let default_explorer_url = network_details.explorer_api.clone().map(|url| {
+        url.parse()
+            .expect("rust sdk mainnet default explorer url not parseable")
+    });
+
+    let default_explorer_url =
+        default_explorer_url.expect("explorer url missing in network config");
+    let explorer_client = ExplorerClient::new_with_timeout(
+        default_explorer_url,
+        config.nym_explorer_client_timeout(),
+    )?;
+    let explorer_gateways = explorer_client
+        .get_gateways()
+        .await
+        .log_error("get_gateways")?;
+    tracing::debug!("6");
+
+    let api_client =
+    // TODO dz introduce timeout ?
+        NymApiClient::new(default_api_url);
+    let gateways = api_client
+        .get_cached_described_gateways()
+        .await
+        .log_error("get_described_gateways")?;
+    tracing::debug!("Fetched {} gateways", gateways.len());
+    let skimmed_gateways = api_client
+        .get_basic_gateways(None)
+        .await
+        .log_error("get_basic_gateways")?;
+
+    let mixnodes = api_client
+        .get_cached_mixnodes()
+        .await
+        .log_error("get_cached_mixnodes")?;
+    tracing::debug!("Fetched {} mixnodes", mixnodes.len());
+
+    // TODO dz can we calculate blacklisted GWs from their performance?
+    // where do we get their performance?
+    let gateways_blacklisted = api_client
+        .nym_api
+        .get_gateways_blacklisted()
+        .await
+        .map(|vec| vec.into_iter().collect::<HashSet<_>>())
+        .log_error("get_gateways_blacklisted")?;
+
+    // Cached mixnodes don't include blacklisted nodes
+    // We need that to calculate the total locked tokens later
+    let mixnodes = api_client
+        .nym_api
+        .get_mixnodes_detailed_unfiltered()
+        .await
+        .log_error("get_mixnodes_detailed_unfiltered")?;
+    let mixnodes_described = api_client
+        .nym_api
+        .get_mixnodes_described()
+        .await
+        .log_error("get_mixnodes_described")?;
+    let mixnodes_active = api_client
+        .nym_api
+        .get_active_mixnodes()
+        .await
+        .log_error("get_active_mixnodes")?;
+    let delegation_program_members =
+        get_delegation_program_details(network_details, config.nyxd_addr()).await?;
+
+    // keep stats for later
+    let count_bonded_mixnodes = mixnodes.len();
+    let count_bonded_gateways = gateways.len();
+    let count_explorer_gateways = explorer_gateways.len();
+    let count_bonded_mixnodes_active = mixnodes_active.len();
+
+    let gateway_records = prepare_gateway_data(
+        &gateways,
+        &gateways_blacklisted,
+        explorer_gateways,
+        skimmed_gateways,
+    )?;
+    queries::insert_gateways(pool, gateway_records)
+        .await
+        .map(|_| {
+            tracing::debug!("Gateway info written to DB!");
+        })?;
+
+    // instead of counting blacklisted GWs returned from API cache, count from the active set
+    let count_gateways_blacklisted = gateways
+        .iter()
+        .filter(|gw| {
+            let gw_identity = gw.bond.identity();
+            gateways_blacklisted.contains(gw_identity)
+        })
+        .count();
+
+    if count_gateways_blacklisted > 0 {
+        queries::write_blacklisted_gateways_to_db(pool, gateways_blacklisted.iter())
+            .await
+            .map(|_| {
+                tracing::debug!(
+                    "Gateway blacklist info written to DB! {} blacklisted by Nym API",
+                    count_gateways_blacklisted
+                )
+            })?;
+    }
+
+    let mixnode_records =
+        prepare_mixnode_data(&mixnodes, mixnodes_described, delegation_program_members)?;
+    queries::insert_mixnodes(pool, mixnode_records)
+        .await
+        .map(|_| {
+            tracing::debug!("Mixnode info written to DB!");
+        })?;
+
+    let count_mixnodes_blacklisted = mixnodes.iter().filter(|elem| elem.blacklisted).count();
+
+    let recently_unbonded_gateways = queries::ensure_gateways_still_bonded(pool, &gateways).await?;
+    let recently_unbonded_mixnodes = queries::ensure_mixnodes_still_bonded(pool, &mixnodes).await?;
+
+    let count_bonded_mixnodes_reserve = 0; // TODO: NymAPI doesn't report the reserve set size
+    let count_bonded_mixnodes_inactive = count_bonded_mixnodes - count_bonded_mixnodes_active;
+
+    let (all_historical_gateways, all_historical_mixnodes) = calculate_stats(pool).await?;
+
+    //
+    // write summary keys and values to table
+    //
+
+    let nodes_summary = vec![
+        (MIXNODES_BONDED_COUNT, &count_bonded_mixnodes),
+        (MIXNODES_BONDED_ACTIVE, &count_bonded_mixnodes_active),
+        (MIXNODES_BONDED_INACTIVE, &count_bonded_mixnodes_inactive),
+        (MIXNODES_BONDED_RESERVE, &count_bonded_mixnodes_reserve),
+        (MIXNODES_BLACKLISTED_COUNT, &count_mixnodes_blacklisted),
+        (GATEWAYS_BONDED_COUNT, &count_bonded_gateways),
+        (GATEWAYS_EXPLORER_COUNT, &count_explorer_gateways),
+        (MIXNODES_HISTORICAL_COUNT, &all_historical_mixnodes),
+        (GATEWAYS_HISTORICAL_COUNT, &all_historical_gateways),
+        (GATEWAYS_BLACKLISTED_COUNT, &count_gateways_blacklisted),
+    ];
+
+    let last_updated = chrono::offset::Utc::now();
+    let last_updated_utc = last_updated.timestamp().to_string();
+    let network_summary = NetworkSummary {
+        mixnodes: mixnode::MixnodeSummary {
+            bonded: mixnode::MixnodeSummaryBonded {
+                count: count_bonded_mixnodes.cast_checked()?,
+                active: count_bonded_mixnodes_active.cast_checked()?,
+                inactive: count_bonded_mixnodes_inactive.cast_checked()?,
+                reserve: count_bonded_mixnodes_reserve.cast_checked()?,
+                last_updated_utc: last_updated_utc.to_owned(),
+            },
+            blacklisted: mixnode::MixnodeSummaryBlacklisted {
+                count: count_mixnodes_blacklisted.cast_checked()?,
+                last_updated_utc: last_updated_utc.to_owned(),
+            },
+            historical: mixnode::MixnodeSummaryHistorical {
+                count: all_historical_mixnodes.cast_checked()?,
+                last_updated_utc: last_updated_utc.to_owned(),
+            },
+        },
+        gateways: gateway::GatewaySummary {
+            bonded: gateway::GatewaySummaryBonded {
+                count: count_bonded_gateways.cast_checked()?,
+                last_updated_utc: last_updated_utc.to_owned(),
+            },
+            blacklisted: gateway::GatewaySummaryBlacklisted {
+                count: count_gateways_blacklisted.cast_checked()?,
+                last_updated_utc: last_updated_utc.to_owned(),
+            },
+            historical: gateway::GatewaySummaryHistorical {
+                count: all_historical_gateways.cast_checked()?,
+                last_updated_utc: last_updated_utc.to_owned(),
+            },
+            explorer: gateway::GatewaySummaryExplorer {
+                count: count_explorer_gateways.cast_checked()?,
+                last_updated_utc: last_updated_utc.to_owned(),
+            },
+        },
+    };
+
+    queries::insert_summaries(pool, &nodes_summary, &network_summary, last_updated).await?;
+
+    let mut log_lines: Vec<String> = vec![];
+    for (key, value) in nodes_summary.iter() {
+        log_lines.push(format!("{} = {}", key, value));
+    }
+    log_lines.push(format!(
+        "recently_unbonded_mixnodes = {}",
+        recently_unbonded_mixnodes
+    ));
+    log_lines.push(format!(
+        "recently_unbonded_gateways = {}",
+        recently_unbonded_gateways
+    ));
+
+    tracing::info!("Directory summary: \n{}", log_lines.join("\n"));
+
+    Ok(())
+}
+
+fn prepare_gateway_data(
+    gateways: &[DescribedGateway],
+    gateways_blacklisted: &HashSet<String>,
+    explorer_gateways: Vec<PrettyDetailedGatewayBond>,
+    skimmed_gateways: Vec<SkimmedNode>,
+) -> anyhow::Result<Vec<GatewayRecord>> {
+    let mut gateway_records = Vec::new();
+
+    for gateway in gateways {
+        let identity_key = gateway.bond.identity();
+        let bonded = true;
+        let last_updated_utc = chrono::offset::Utc::now().timestamp();
+        let blacklisted = gateways_blacklisted.contains(identity_key);
+
+        let self_described = gateway
+            .self_described
+            .as_ref()
+            .and_then(|v| serde_json::to_string(&v).ok());
+
+        let explorer_pretty_bond = explorer_gateways
+            .iter()
+            .find(|g| g.gateway.identity_key.eq(identity_key));
+        let explorer_pretty_bond = explorer_pretty_bond.and_then(|g| serde_json::to_string(g).ok());
+
+        let performance = skimmed_gateways
+            .iter()
+            .find(|g| g.ed25519_identity_pubkey.eq(identity_key))
+            .map(|g| g.performance)
+            .unwrap_or_default()
+            .round_to_integer();
+
+        gateway_records.push(GatewayRecord {
+            identity_key: identity_key.to_owned(),
+            bonded,
+            blacklisted,
+            self_described,
+            explorer_pretty_bond,
+            last_updated_utc,
+            performance,
+        });
+    }
+
+    Ok(gateway_records)
+}
+
+fn prepare_mixnode_data(
+    mixnodes: &[MixNodeBondAnnotated],
+    mixnodes_described: Vec<DescribedMixNode>,
+    delegation_program_members: Vec<u32>,
+) -> anyhow::Result<Vec<MixnodeRecord>> {
+    let mut mixnode_records = Vec::new();
+
+    for mixnode in mixnodes {
+        let mix_id = mixnode.mix_id();
+        let identity_key = mixnode.identity_key();
+        let bonded = true;
+        let total_stake = decimal_to_i64(mixnode.mixnode_details.total_stake());
+        let blacklisted = mixnode.blacklisted;
+        let node_info = mixnode.mix_node();
+        let host = node_info.host.clone();
+        let http_port = node_info.http_api_port;
+        // Contains all the information including what's above
+        let full_details = serde_json::to_string(&mixnode)?;
+
+        let mixnode_described = mixnodes_described.iter().find(|m| m.bond.mix_id == mix_id);
+        let self_described = mixnode_described.and_then(|v| serde_json::to_string(v).ok());
+        let is_dp_delegatee = delegation_program_members.contains(&mix_id);
+
+        let last_updated_utc = chrono::offset::Utc::now().timestamp();
+
+        mixnode_records.push(MixnodeRecord {
+            mix_id,
+            identity_key: identity_key.to_owned(),
+            bonded,
+            total_stake,
+            host,
+            http_port,
+            blacklisted,
+            full_details,
+            self_described,
+            last_updated_utc,
+            is_dp_delegatee,
+        });
+    }
+
+    Ok(mixnode_records)
+}
+
+// TODO dz is there a common monorepo place this can be put?
+pub trait NumericalCheckedCast<T>
+where
+    T: TryFrom<Self>,
+    <T as TryFrom<Self>>::Error: std::error::Error,
+    Self: std::fmt::Display + Copy,
+{
+    fn cast_checked(self) -> anyhow::Result<T> {
+        T::try_from(self).map_err(|e| {
+            anyhow::anyhow!(
+                "Couldn't cast {} to {}: {}",
+                self,
+                std::any::type_name::<T>(),
+                e
+            )
+        })
+    }
+}
+
+impl<T, U> NumericalCheckedCast<U> for T
+where
+    U: TryFrom<T>,
+    <U as TryFrom<T>>::Error: std::error::Error,
+    T: std::fmt::Display + Copy,
+{
+}
+
+async fn calculate_stats(pool: &DbPool) -> anyhow::Result<(usize, usize)> {
+    let mut conn = pool.acquire().await?;
+
+    let all_historical_gateways = sqlx::query_scalar!(r#"SELECT count(id) FROM gateways"#)
+        .fetch_one(&mut *conn)
+        .await?
+        .cast_checked()?;
+
+    let all_historical_mixnodes = sqlx::query_scalar!(r#"SELECT count(id) FROM mixnodes"#)
+        .fetch_one(&mut *conn)
+        .await?
+        .cast_checked()?;
+
+    Ok((all_historical_gateways, all_historical_mixnodes))
+}
+
+async fn get_delegation_program_details(
+    network_details: &NymNetworkDetails,
+    nyxd_addr: &Url,
+) -> anyhow::Result<Vec<u32>> {
+    let config = nym_validator_client::nyxd::Config::try_from_nym_network_details(network_details)?;
+
+    let client = NyxdClient::connect(config, nyxd_addr.as_str())
+        .map_err(|err| anyhow::anyhow!("Couldn't connect: {}", err))?;
+
+    let account_id = AccountId::from_str(DELEGATION_PROGRAM_WALLET)
+        .map_err(|e| anyhow!("Invalid bech32 address: {}", e))?;
+
+    let delegations = client.get_all_delegator_delegations(&account_id).await?;
+
+    let mix_ids: Vec<u32> = delegations
+        .iter()
+        .map(|delegation| delegation.mix_id)
+        .collect();
+
+    Ok(mix_ids)
+}
+
+fn decimal_to_i64(decimal: Decimal) -> i64 {
+    // Convert the underlying Uint128 to a u128
+    let atomics = decimal.atomics().u128();
+    let precision = 1_000_000_000_000_000_000u128;
+
+    // Get the fractional part
+    let fractional = atomics % precision;
+
+    // Get the integer part
+    let integer = atomics / precision;
+
+    // Combine them into a float
+    let float_value = integer as f64 + (fractional as f64 / 1_000_000_000_000_000_000_f64);
+
+    // Limit to 6 decimal places
+    let rounded_value = (float_value * 1_000_000.0).round() / 1_000_000.0;
+
+    rounded_value as i64
+}
+
+trait LogError<T, E> {
+    fn log_error(self, msg: &str) -> Result<T, E>;
+}
+
+impl<T, E> LogError<T, E> for anyhow::Result<T, E>
+where
+    E: std::error::Error,
+{
+    fn log_error(self, msg: &str) -> Result<T, E> {
+        if let Err(e) = &self {
+            tracing::error!("[{msg}]:\t{e}");
+        }
+        self
+    }
+}