Compare commits

..

1 Commits

Author SHA1 Message Date
dynco-nym c14b010f9e Eliminate duplicate node_ids from endpoint (#5728)
* Improve swagger definitions

* Sort data in DB

* Improve logging

* Store gw description to nym nodes table

* Move explorer related path to /explorer

* Bump package version
2025-04-23 15:19:15 +02:00
20 changed files with 221 additions and 127 deletions
+8 -12
View File
@@ -44,24 +44,21 @@ jobs:
- name: Download EV CodeSignTool from ssl.com
working-directory: nym-wallet/src-tauri
if: ${{ inputs.sign || startsWith(github.ref, 'refs/tags/nym-wallet-') }}
if: ${{ inputs.sign }}
shell: bash
run: |
curl -L0 https://www.ssl.com/download/codesigntool-for-linux-and-macos/ -o codesigntool.zip
unzip codesigntool.zip
chmod +x ./CodeSignTool.sh
- name: Get EV certificate credential id
working-directory: nym-wallet/src-tauri
if: ${{ inputs.sign || startsWith(github.ref, 'refs/tags/nym-wallet-') }}
if: ${{ inputs.sign }}
id: get_credential_ids
shell: bash
run: |
echo "SSL_COM_CREDENTIAL_ID=$(./CodeSignTool.sh get_credential_ids -username=${{ secrets.SSL_COM_USERNAME }} -password=${{ secrets.SSL_COM_PASSWORD }} | sed -n '1!p' | sed 's/- //')" >> "$GITHUB_OUTPUT"
- name: Add custom sign command to tauri.conf.json
working-directory: nym-wallet/src-tauri
if: ${{ inputs.sign || startsWith(github.ref, 'refs/tags/nym-wallet-') }}
if: ${{ inputs.sign }}
shell: bash
run: |
yq eval --inplace '.bundle.windows +=
@@ -82,7 +79,6 @@ jobs:
]
}
}' tauri.conf.json
- name: Install project dependencies
shell: bash
run: cd .. && yarn --network-timeout 100000
@@ -97,10 +93,10 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_PRIVATE_KEY }}
TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ${{ secrets.TAURI_KEY_PASSWORD }}
SSL_COM_USERNAME: ${{ (inputs.sign || startsWith(github.ref, 'refs/tags/nym-wallet-')) && secrets.SSL_COM_USERNAME }}
SSL_COM_PASSWORD: ${{ (inputs.sign || startsWith(github.ref, 'refs/tags/nym-wallet-')) && secrets.SSL_COM_PASSWORD }}
SSL_COM_CREDENTIAL_ID: ${{ (inputs.sign || startsWith(github.ref, 'refs/tags/nym-wallet-')) && steps.get_credential_ids.outputs.SSL_COM_CREDENTIAL_ID }}
SSL_COM_TOTP_SECRET: ${{ (inputs.sign || startsWith(github.ref, 'refs/tags/nym-wallet-')) && secrets.SSL_COM_TOTP_SECRET }}
SSL_COM_USERNAME: ${{ inputs.sign && secrets.SSL_COM_USERNAME }}
SSL_COM_PASSWORD: ${{ inputs.sign && secrets.SSL_COM_PASSWORD }}
SSL_COM_CREDENTIAL_ID: ${{ inputs.sign && steps.get_credential_ids.outputs.SSL_COM_CREDENTIAL_ID }}
SSL_COM_TOTP_SECRET: ${{ inputs.sign && secrets.SSL_COM_TOTP_SECRET }}
run: |
echo "Starting build process..."
yarn build
@@ -144,7 +140,7 @@ jobs:
- id: create-release
name: Upload to release based on tag name
uses: softprops/action-gh-release@v2
if: ${{ startsWith(github.ref, 'refs/tags/nym-wallet-') && github.event_name == 'release' }}
if: github.event_name == 'release'
with:
files: |
nym-wallet/${{ env.BUNDLE_PATH }}/msi/*.msi
Generated
+3 -1
View File
@@ -6252,7 +6252,7 @@ dependencies = [
[[package]]
name = "nym-node-status-api"
version = "2.1.0"
version = "2.2.0"
dependencies = [
"ammonia",
"anyhow",
@@ -6269,6 +6269,8 @@ dependencies = [
"nym-contracts-common",
"nym-crypto",
"nym-http-api-client",
"nym-http-api-common",
"nym-mixnet-contract-common",
"nym-network-defaults",
"nym-node-metrics",
"nym-node-requests",
@@ -9,13 +9,35 @@ use axum::response::IntoResponse;
use axum_client_ip::InsecureClientIp;
use colored::Colorize;
use std::time::Instant;
use tracing::info;
use tracing::{debug, info};
enum LogLevel {
Debug,
Info,
}
pub async fn log_request_info(
insecure_client_ip: InsecureClientIp,
request: Request,
next: Next,
) -> impl IntoResponse {
log_request(insecure_client_ip, request, next, LogLevel::Info).await
}
pub async fn log_request_debug(
insecure_client_ip: InsecureClientIp,
request: Request,
next: Next,
) -> impl IntoResponse {
log_request(insecure_client_ip, request, next, LogLevel::Debug).await
}
/// Simple logger for requests
pub async fn logger(
async fn log_request(
InsecureClientIp(addr): InsecureClientIp,
request: Request,
next: Next,
level: LogLevel,
) -> impl IntoResponse {
// TODO dz use `OriginalUri` extractor to get full URI even for nested
// routers if routes aren't logged correctly in handlers
@@ -58,7 +80,14 @@ pub async fn logger(
let agent_str = "agent".bold();
info!("[{addr} -> {host}] {method} '{uri}': {print_status} {time_taken} {agent_str}: {agent}");
match level {
LogLevel::Debug => debug!(
"[{addr} -> {host}] {method} '{uri}': {print_status} {time_taken} {agent_str}: {agent}"
),
LogLevel::Info => info!(
"[{addr} -> {host}] {method} '{uri}': {print_status} {time_taken} {agent_str}: {agent}"
),
}
res
}
+2 -2
View File
@@ -17,7 +17,7 @@ use axum::response::Redirect;
use axum::routing::get;
use axum::Router;
use core::net::SocketAddr;
use nym_http_api_common::middleware::logging::logger;
use nym_http_api_common::middleware::logging::log_request_info;
use tokio::net::TcpListener;
use tokio_util::sync::WaitForCancellationFutureOwned;
use tower_http::cors::CorsLayer;
@@ -91,7 +91,7 @@ impl RouterBuilder {
fn finalize_routes(self) -> Router<AppState> {
self.unfinished_router
.layer(setup_cors())
.layer(axum::middleware::from_fn(logger))
.layer(axum::middleware::from_fn(log_request_info))
}
}
@@ -31,7 +31,7 @@ pub fn build_router(state: ApiState, auth_token: String) -> Router {
.nest(routes::API, api::routes(auth_middleware))
// we don't have to be using middleware, but we already had that code
// we might want something like: https://github.com/tokio-rs/axum/blob/main/examples/tracing-aka-logging/src/main.rs#L44 instead
.layer(axum::middleware::from_fn(logging::logger))
.layer(axum::middleware::from_fn(logging::log_request_info))
.with_state(state);
cfg_if::cfg_if! {
@@ -20,7 +20,7 @@ nym-bin-common = { path = "../../common/bin-common", features = ["models"]}
nym-node-status-client = { path = "../nym-node-status-client" }
nym-crypto = { path = "../../common/crypto", features = ["asymmetric", "rand"] }
rand = { workspace = true }
tokio = { workspace = true, features = ["macros", "rt-multi-thread", "process"] }
tokio = { workspace = true, features = ["macros", "rt-multi-thread", "process", "fs"] }
tracing = { workspace = true }
tracing-subscriber = { workspace = true, features = ["env-filter"] }
@@ -3,7 +3,7 @@
[package]
name = "nym-node-status-api"
version = "2.1.0"
version = "2.2.0"
authors.workspace = true
repository.workspace = true
homepage.workspace = true
@@ -25,10 +25,12 @@ futures-util = { workspace = true }
itertools = { workspace = true }
moka = { workspace = true, features = ["future"] }
nym-contracts-common = { path = "../../common/cosmwasm-smart-contracts/contracts-common" }
nym-mixnet-contract-common = { path = "../../common/cosmwasm-smart-contracts/mixnet-contract", features = ["utoipa"] }
nym-bin-common = { path = "../../common/bin-common", features = ["models"] }
nym-node-status-client = { path = "../nym-node-status-client" }
nym-crypto = { path = "../../common/crypto", features = ["asymmetric", "serde"] }
nym-http-api-client = { path = "../../common/http-api-client" }
nym-http-api-common = { path = "../../common/http-api-common" }
nym-network-defaults = { path = "../../common/network-defaults" }
nym-serde-helpers = { path = "../../common/serde-helpers" }
nym-statistics-common = { path = "../../common/statistics" }
@@ -6,6 +6,7 @@ use crate::{
DbPool,
},
http::models::Gateway,
mixnet_scraper::helpers::NodeDescriptionResponse,
};
use futures_util::TryStreamExt;
use sqlx::{pool::PoolConnection, Sqlite};
@@ -131,3 +132,39 @@ pub(crate) async fn get_bonded_gateway_id_keys(pool: &DbPool) -> anyhow::Result<
Ok(items)
}
pub(crate) async fn insert_gateway_description(
conn: &mut PoolConnection<Sqlite>,
identity_key: &str,
description: &NodeDescriptionResponse,
timestamp: i64,
) -> anyhow::Result<()> {
sqlx::query!(
r#"
INSERT INTO gateway_description (
gateway_identity_key,
moniker,
website,
security_contact,
details,
last_updated_utc
) VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT (gateway_identity_key) DO UPDATE SET
moniker = excluded.moniker,
website = excluded.website,
security_contact = excluded.security_contact,
details = excluded.details,
last_updated_utc = excluded.last_updated_utc
"#,
identity_key,
description.moniker,
description.website,
description.security_contact,
description.details,
timestamp,
)
.execute(conn.as_mut())
.await
.map(drop)
.map_err(From::from)
}
@@ -1,6 +1,7 @@
use std::collections::HashSet;
use futures_util::TryStreamExt;
use sqlx::{pool::PoolConnection, Sqlite};
use tracing::error;
use crate::{
@@ -9,6 +10,7 @@ use crate::{
DbPool,
},
http::models::{DailyStats, Mixnode},
mixnet_scraper::helpers::NodeDescriptionResponse,
};
pub(crate) async fn update_mixnodes(
@@ -157,3 +159,34 @@ pub(crate) async fn get_bonded_mix_ids(pool: &DbPool) -> anyhow::Result<HashSet<
Ok(items)
}
pub(crate) async fn insert_mixnode_description(
conn: &mut PoolConnection<Sqlite>,
mix_id: &i64,
description: &NodeDescriptionResponse,
timestamp: i64,
) -> anyhow::Result<()> {
sqlx::query!(
r#"
INSERT INTO mixnode_description (
mix_id, moniker, website, security_contact, details, last_updated_utc
) VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT (mix_id) DO UPDATE SET
moniker = excluded.moniker,
website = excluded.website,
security_contact = excluded.security_contact,
details = excluded.details,
last_updated_utc = excluded.last_updated_utc
"#,
mix_id,
description.moniker,
description.website,
description.security_contact,
description.details,
timestamp,
)
.execute(conn.as_mut())
.await
.map(drop)
.map_err(From::from)
}
@@ -4,12 +4,16 @@ use nym_validator_client::{
client::{NodeId, NymNodeDetails},
models::NymNodeDescription,
};
use sqlx::{pool::PoolConnection, Sqlite};
use std::collections::HashMap;
use tracing::instrument;
use crate::db::{
models::{NymNodeDto, NymNodeInsertRecord},
DbPool,
use crate::{
db::{
models::{NymNodeDto, NymNodeInsertRecord},
DbPool,
},
mixnet_scraper::helpers::NodeDescriptionResponse,
};
pub(crate) async fn get_all_nym_nodes(pool: &DbPool) -> anyhow::Result<Vec<NymNodeDto>> {
@@ -194,6 +198,8 @@ pub(crate) async fn get_node_self_description(
nym_nodes
WHERE
self_described IS NOT NULL
ORDER BY
node_id
"#,
)
.fetch_all(&mut *conn)
@@ -256,3 +262,34 @@ pub(crate) async fn get_bonded_node_description(
})
.map_err(From::from)
}
pub(crate) async fn insert_nym_node_description(
conn: &mut PoolConnection<Sqlite>,
node_id: &i64,
description: &NodeDescriptionResponse,
timestamp: i64,
) -> anyhow::Result<()> {
sqlx::query!(
r#"
INSERT INTO nym_node_descriptions (
node_id, moniker, website, security_contact, details, last_updated_utc
) VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT (node_id) DO UPDATE SET
moniker = excluded.moniker,
website = excluded.website,
security_contact = excluded.security_contact,
details = excluded.details,
last_updated_utc = excluded.last_updated_utc
"#,
node_id,
description.moniker,
description.website,
description.security_contact,
description.details,
timestamp,
)
.execute(conn.as_mut())
.await
.map(drop)
.map_err(From::from)
}
@@ -1,7 +1,11 @@
use crate::{
db::{
models::{ScrapeNodeKind, ScraperNodeInfo},
queries, DbPool,
queries::{
self, gateways::insert_gateway_description, mixnodes::insert_mixnode_description,
nym_nodes::insert_nym_node_description,
},
DbPool,
},
mixnet_scraper::helpers::NodeDescriptionResponse,
};
@@ -125,78 +129,18 @@ pub(crate) async fn insert_scraped_node_description(
match node_kind {
ScrapeNodeKind::LegacyMixnode { mix_id } => {
sqlx::query!(
r#"
INSERT INTO mixnode_description (
mix_id, moniker, website, security_contact, details, last_updated_utc
) VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT (mix_id) DO UPDATE SET
moniker = excluded.moniker,
website = excluded.website,
security_contact = excluded.security_contact,
details = excluded.details,
last_updated_utc = excluded.last_updated_utc
"#,
mix_id,
description.moniker,
description.website,
description.security_contact,
description.details,
timestamp,
)
.execute(&mut *conn)
.await?;
insert_mixnode_description(&mut conn, mix_id, description, timestamp).await?;
}
ScrapeNodeKind::MixingNymNode { node_id } => {
sqlx::query!(
r#"
INSERT INTO nym_node_descriptions (
node_id, moniker, website, security_contact, details, last_updated_utc
) VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT (node_id) DO UPDATE SET
moniker = excluded.moniker,
website = excluded.website,
security_contact = excluded.security_contact,
details = excluded.details,
last_updated_utc = excluded.last_updated_utc
"#,
node_id,
description.moniker,
description.website,
description.security_contact,
description.details,
timestamp,
)
.execute(&mut *conn)
.await?;
insert_nym_node_description(&mut conn, node_id, description, timestamp).await?;
}
ScrapeNodeKind::EntryExitNymNode { identity_key, .. } => {
sqlx::query!(
r#"
INSERT INTO gateway_description (
gateway_identity_key,
moniker,
website,
security_contact,
details,
last_updated_utc
) VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT (gateway_identity_key) DO UPDATE SET
moniker = excluded.moniker,
website = excluded.website,
security_contact = excluded.security_contact,
details = excluded.details,
last_updated_utc = excluded.last_updated_utc
"#,
identity_key,
description.moniker,
description.website,
description.security_contact,
description.details,
timestamp,
)
.execute(&mut *conn)
.await?;
ScrapeNodeKind::EntryExitNymNode {
node_id,
identity_key,
} => {
insert_nym_node_description(&mut conn, node_id, description, timestamp).await?;
// for historic reasons (/gateways API), store this info into gateways table as well
insert_gateway_description(&mut conn, identity_key, description, timestamp).await?;
}
}
@@ -77,7 +77,7 @@ async fn get_all_sessions(
};
Ok(Json(PagedResult::paginate(
Pagination { size, page },
Pagination::new(size, page),
day_and_node_filtered,
)))
}
@@ -1,7 +1,8 @@
use anyhow::anyhow;
use axum::{response::Redirect, Router};
use nym_http_api_common::middleware::logging::log_request_debug;
use tokio::net::ToSocketAddrs;
use tower_http::{cors::CorsLayer, trace::TraceLayer};
use tower_http::cors::CorsLayer;
use utoipa::OpenApi;
use utoipa_swagger_ui::SwaggerUi;
@@ -39,7 +40,10 @@ impl RouterBuilder {
.nest("/summary", summary::routes())
.nest("/metrics", metrics::routes()),
)
.nest("/v3", Router::new().nest("/nym-nodes", nym_nodes::routes()))
.nest(
"/explorer/v3",
Router::new().nest("/nym-nodes", nym_nodes::routes()),
)
.nest(
"/internal",
Router::new().nest("/testruns", testruns::routes()),
@@ -62,7 +66,7 @@ impl RouterBuilder {
// CORS layer needs to wrap other API layers
.layer(setup_cors())
// logger should be outermost layer
.layer(TraceLayer::new_for_http())
.layer(axum::middleware::from_fn(log_request_debug))
}
}
@@ -16,12 +16,13 @@ pub(crate) fn routes() -> Router<AppState> {
}
#[utoipa::path(
tag = "Nym Nodes",
tag = "Nym Explorer",
get,
params(
Pagination
),
path = "/v3/nym-nodes",
path = "/nym-nodes",
context_path = "/explorer/v3",
responses(
(status = 200, body = PagedResult<ExtendedNymNode>)
)
@@ -104,7 +104,10 @@ async fn mixnodes(
.map(|(s, _)| s)
.collect();
Ok(Json(PagedResult::paginate(Pagination { size, page }, res)))
Ok(Json(PagedResult::paginate(
Pagination::new(size, page),
res,
)))
}
struct ServiceFilter {
@@ -18,7 +18,7 @@ pub struct PagedResult<T: ToSchema> {
impl<T: Clone + ToSchema> PagedResult<T> {
pub fn paginate(pagination: Pagination, res: Vec<T>) -> Self {
let total = res.len();
let (size, mut page) = pagination.intoto_inner_values();
let (size, mut page) = pagination.into_inner_values();
if page * size > total {
page = total / size;
@@ -42,14 +42,25 @@ pub(crate) struct Pagination {
page: Option<usize>,
}
const SIZE_DEFAULT: usize = 10;
const SIZE_MAX: usize = 200;
const PAGE_DEFAULT: usize = 0;
impl Default for Pagination {
fn default() -> Self {
Self {
size: Some(SIZE_DEFAULT),
page: Some(PAGE_DEFAULT),
}
}
}
impl Pagination {
// unwrap stored values or use predefined defaults
pub(crate) fn intoto_inner_values(self) -> (usize, usize) {
const SIZE_DEFAULT: usize = 10;
const SIZE_MAX: usize = 200;
const PAGE_DEFAULT: usize = 0;
pub(crate) fn new(size: Option<usize>, page: Option<usize>) -> Self {
Self { size, page }
}
pub(crate) fn into_inner_values(self) -> (usize, usize) {
(
self.size.unwrap_or(SIZE_DEFAULT).min(SIZE_MAX),
self.page.unwrap_or(PAGE_DEFAULT),
@@ -55,13 +55,13 @@ pub(crate) struct ExtendedNymNode {
#[schema(value_type = String)]
pub(crate) total_stake: Decimal,
pub(crate) original_pledge: u128,
pub(crate) bonding_address: String,
pub(crate) bonding_address: Option<String>,
pub(crate) bonded: bool,
pub(crate) node_type: String,
pub(crate) node_type: nym_validator_client::models::DescribedNodeType,
pub(crate) ip_address: String,
pub(crate) accepted_tnc: bool,
pub(crate) self_description: serde_json::Value,
pub(crate) rewarding_details: serde_json::Value,
pub(crate) self_description: nym_validator_client::models::NymNodeData,
pub(crate) rewarding_details: Option<nym_mixnet_contract_common::NodeRewarding>,
pub(crate) description: NodeDescription,
pub(crate) geoip: Option<NodeGeoData>,
}
@@ -4,7 +4,7 @@ use cosmwasm_std::Decimal;
use moka::{future::Cache, Entry};
use nym_contracts_common::NaiveFloat;
use nym_crypto::asymmetric::ed25519::PublicKey;
use nym_validator_client::{models::DescribedNodeType, nym_api::SkimmedNode};
use nym_validator_client::nym_api::SkimmedNode;
use tokio::sync::RwLock;
use tracing::instrument;
@@ -401,11 +401,7 @@ async fn aggregate_node_info_from_db(
.get(&node_id)
.map(|node| node.performance.naive_to_f64())
.unwrap_or(0.0);
let node_type = match described_node.contract_node_type {
DescribedNodeType::NymNode => "nym_node".to_string(),
DescribedNodeType::LegacyMixnode => "legacy_mixnode".to_string(),
DescribedNodeType::LegacyGateway => "legacy_gateway".to_string(),
};
let node_type = described_node.contract_node_type;
let ip_address = described_node
.description
.host_information
@@ -417,11 +413,10 @@ async fn aggregate_node_info_from_db(
.description
.auxiliary_details
.accepted_operator_terms_and_conditions;
let description = described_node.description;
let self_described = described_node.description;
let bonding_address = bond_details
.map(|details| details.bond_information.owner.to_string())
.unwrap_or_default();
let bonding_address =
bond_details.map(|details| details.bond_information.owner.to_string());
let node_description = node_descriptions.get(&node_id).cloned().unwrap_or_default();
let geoip = {
@@ -449,8 +444,8 @@ async fn aggregate_node_info_from_db(
bonded,
node_type,
accepted_tnc,
self_description: serde_json::to_value(description).unwrap_or_default(),
rewarding_details: serde_json::to_value(rewarding_details).unwrap_or_default(),
self_description: self_described,
rewarding_details: rewarding_details.to_owned(),
description: node_description,
geoip,
});
@@ -151,7 +151,7 @@ impl Monitor {
})?;
// refresh geodata for all nodes
for (_, node_description) in described_nodes.iter() {
for node_description in described_nodes.values() {
self.location_cached(node_description).await;
}
@@ -295,7 +295,7 @@ impl Monitor {
Ok(())
}
#[instrument(level = "debug", skip_all)]
#[instrument(level = "info", skip_all)]
async fn location_cached(&mut self, node: &NymNodeDescription) -> Location {
let node_id = node.node_id;
+1 -1
View File
@@ -159,7 +159,7 @@ impl NymNodeRouter {
)
.nest(routes::LANDING_PAGE, landing_page::routes(config.landing))
.nest(routes::API, api::routes(config.api))
.layer(axum::middleware::from_fn(logging::logger))
.layer(axum::middleware::from_fn(logging::log_request_info))
.with_state(state),
}
}