From 9517beecfac71c0753b5ab4ce9615bbf0b435f65 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Tue, 11 Nov 2025 19:36:17 +0000 Subject: [PATCH] change to clap args and use url::Url to parse args --- nym-data-observatory/Cargo.toml | 1 + nym-data-observatory/src/chain_scraper/mod.rs | 48 ++++++------------- nym-data-observatory/src/cli/commands/init.rs | 2 +- .../src/cli/commands/run/args.rs | 17 ++++++- nym-data-observatory/src/cli/mod.rs | 2 +- .../src/config/data_observatory.rs | 2 +- nym-data-observatory/src/config/mod.rs | 8 ++-- nym-data-observatory/src/env.rs | 3 ++ nym-data-observatory/src/error.rs | 3 ++ nym-data-observatory/src/http/models.rs | 4 +- .../src/cli/process_block.rs | 2 +- .../src/cli/process_until.rs | 2 +- nym-validator-rewarder/src/config/mod.rs | 13 +++-- .../src/config/persistence/paths.rs | 6 +-- nym-validator-rewarder/src/error.rs | 5 +- 15 files changed, 64 insertions(+), 54 deletions(-) diff --git a/nym-data-observatory/Cargo.toml b/nym-data-observatory/Cargo.toml index 33178009f4..0de46b9e4a 100644 --- a/nym-data-observatory/Cargo.toml +++ b/nym-data-observatory/Cargo.toml @@ -38,6 +38,7 @@ tokio-util = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter"] } tower-http = { workspace = true, features = ["cors", "trace"] } +url = { workspace = true } utoipa = { workspace = true, features = ["axum_extras", "time"] } utoipa-swagger-ui = { workspace = true, features = ["axum"] } utoipauto = { workspace = true } diff --git a/nym-data-observatory/src/chain_scraper/mod.rs b/nym-data-observatory/src/chain_scraper/mod.rs index a1eaeeb2fc..cbc1036208 100644 --- a/nym-data-observatory/src/chain_scraper/mod.rs +++ b/nym-data-observatory/src/chain_scraper/mod.rs @@ -1,8 +1,5 @@ +use crate::cli::commands::run::Args; use crate::db::DbPool; -use crate::env::vars::{ - NYXD_SCRAPER_START_HEIGHT, NYXD_SCRAPER_UNSAFE_NUKE_DB, - NYXD_SCRAPER_USE_BEST_EFFORT_START_HEIGHT, -}; use nyxd_scraper_psql::{PostgresNyxdScraper, PruningOptions}; use std::fs; use tracing::{info, warn}; @@ -10,48 +7,31 @@ use tracing::{info, warn}; pub(crate) mod webhook; pub(crate) async fn run_chain_scraper( + args: Args, config: &crate::config::Config, connection_pool: DbPool, ) -> anyhow::Result { - let websocket_url = std::env::var("NYXD_WS").expect("NYXD_WS not defined"); + let use_best_effort_start_height = args.start_block_height.is_some(); - let rpc_url = std::env::var("NYXD").expect("NYXD not defined"); - let websocket_url = reqwest::Url::parse(&websocket_url)?; - let rpc_url = reqwest::Url::parse(&rpc_url)?; - - // why are those not part of CLI? : ( - let start_block_height = match std::env::var(NYXD_SCRAPER_START_HEIGHT).ok() { - None => None, - // blow up if passed malformed env value - Some(raw) => Some(raw.parse()?), - }; - - let use_best_effort_start_height = - match std::env::var(NYXD_SCRAPER_USE_BEST_EFFORT_START_HEIGHT).ok() { - None => false, - // blow up if passed malformed env value - Some(raw) => raw.parse()?, - }; - - let nuke_db: bool = match std::env::var(NYXD_SCRAPER_UNSAFE_NUKE_DB).ok() { - None => false, - // blow up if passed malformed env value - Some(raw) => raw.parse()?, - }; - - if nuke_db { + if args.nuke_db { warn!("☢️☢️☢️ NUKING THE SCRAPER DATABASE"); fs::remove_file(config.chain_scraper_connection_string())?; } + let database_storage = config + .chain_scraper_connection_string + .clone() + .and(args.db_connection_string) + .expect("no database connection string set in config"); + let scraper = PostgresNyxdScraper::builder(nyxd_scraper_psql::Config { - websocket_url, - rpc_url, - database_storage: config.chain_scraper_connection_string.clone(), + websocket_url: args.websocket_url, + rpc_url: args.rpc_url, + database_storage, pruning_options: PruningOptions::nothing(), store_precommits: false, start_block: nyxd_scraper_psql::StartingBlockOpts { - start_block_height, + start_block_height: args.start_block_height, use_best_effort_start_height, }, }) diff --git a/nym-data-observatory/src/cli/commands/init.rs b/nym-data-observatory/src/cli/commands/init.rs index f57f3fc43f..4a0a2b9e92 100644 --- a/nym-data-observatory/src/cli/commands/init.rs +++ b/nym-data-observatory/src/cli/commands/init.rs @@ -28,7 +28,7 @@ pub(crate) async fn execute(args: Args) -> Result<(), NymDataObservatoryError> { .with_data_observatory_config(DataObservatoryConfig { webhooks: vec![WebhookConfig { id: DEFAULT_NYM_DATA_OBSERVATORY_ID.to_string(), - webhook_url: "https://webhook.site".to_string(), + webhook_url: url::Url::parse("https://webhook.site")?, authentication: Some(AuthorizationBearerToken { token: "1234".to_string(), }), diff --git a/nym-data-observatory/src/cli/commands/run/args.rs b/nym-data-observatory/src/cli/commands/run/args.rs index dc34af5715..6d4f71b272 100644 --- a/nym-data-observatory/src/cli/commands/run/args.rs +++ b/nym-data-observatory/src/cli/commands/run/args.rs @@ -2,9 +2,22 @@ // SPDX-License-Identifier: GPL-3.0-only use crate::env::vars::*; +use url::Url; -#[derive(clap::Args, Debug)] +#[derive(clap::Args, Debug, Clone)] pub(crate) struct Args { + #[arg(long, env = NYXD_WS, alias = "nyxd_ws")] + pub(crate) websocket_url: Url, + + #[arg(long, env = NYXD, alias = "nyxd")] + pub(crate) rpc_url: Url, + + #[arg(long, env = NYXD_SCRAPER_START_HEIGHT)] + pub(crate) start_block_height: Option, + + #[arg(long, env = NYXD_SCRAPER_UNSAFE_NUKE_DB, default_value = "false")] + pub(crate) nuke_db: bool, + /// (Override) Postgres connection string for chain scraper history #[arg(long, env = NYM_DATA_OBSERVATORY_DB_URL, alias = "db_url")] pub(crate) db_connection_string: Option, @@ -22,7 +35,7 @@ pub(crate) struct Args { long, env = NYM_DATA_OBSERVATORY_WEBHOOK_URL )] - pub webhook_url: Option, + pub webhook_url: Option, /// (Override) Optionally, authenticate with the webhook #[clap( diff --git a/nym-data-observatory/src/cli/mod.rs b/nym-data-observatory/src/cli/mod.rs index 698bb9d293..7aa4d99dbb 100644 --- a/nym-data-observatory/src/cli/mod.rs +++ b/nym-data-observatory/src/cli/mod.rs @@ -8,7 +8,7 @@ use clap::{Parser, Subcommand}; use nym_bin_common::bin_info; use std::sync::OnceLock; -mod commands; +pub(crate) mod commands; pub const DEFAULT_NYM_DATA_OBSERVATORY_ID: &str = "default-nym-data-observatory"; diff --git a/nym-data-observatory/src/config/data_observatory.rs b/nym-data-observatory/src/config/data_observatory.rs index 36ef39e881..fa9341e35b 100644 --- a/nym-data-observatory/src/config/data_observatory.rs +++ b/nym-data-observatory/src/config/data_observatory.rs @@ -9,7 +9,7 @@ pub struct DataObservatoryConfig { pub struct WebhookConfig { pub id: String, pub description: Option, - pub webhook_url: String, + pub webhook_url: url::Url, pub watch_for_chain_message_types: Vec, pub authentication: Option, } diff --git a/nym-data-observatory/src/config/mod.rs b/nym-data-observatory/src/config/mod.rs index da75f453ea..86cd357faa 100644 --- a/nym-data-observatory/src/config/mod.rs +++ b/nym-data-observatory/src/config/mod.rs @@ -81,7 +81,7 @@ impl ConfigBuilder { save_path: Some(self.config_path), data_observatory_config: self.data_observatory_config.unwrap_or_default(), data_dir: self.data_dir, - chain_scraper_connection_string: self.chain_scraper_connection_string, + chain_scraper_connection_string: Some(self.chain_scraper_connection_string), } } } @@ -96,7 +96,7 @@ pub struct Config { #[serde(skip)] pub(crate) data_dir: PathBuf, - pub chain_scraper_connection_string: String, + pub chain_scraper_connection_string: Option, #[serde(default)] pub data_observatory_config: DataObservatoryConfig, @@ -182,7 +182,9 @@ impl Config { } pub fn chain_scraper_connection_string(&self) -> String { - self.chain_scraper_connection_string.clone() + self.chain_scraper_connection_string + .clone() + .expect("database connection string not set") } // simple wrapper that reads config file and assigns path location diff --git a/nym-data-observatory/src/env.rs b/nym-data-observatory/src/env.rs index 45f054a839..cff2212b57 100644 --- a/nym-data-observatory/src/env.rs +++ b/nym-data-observatory/src/env.rs @@ -3,6 +3,9 @@ #[allow(unused)] pub mod vars { + pub const NYXD_WS: &str = "NYXD_WS"; + pub const NYXD: &str = "NYXD"; + pub const NYM_DATA_OBSERVATORY_NO_BANNER_ARG: &str = "NYM_DATA_OBSERVATORY_NO_BANNER"; pub const NYM_DATA_OBSERVATORY_CONFIG_ENV_FILE_ARG: &str = "NYM_DATA_OBSERVATORY_CONFIG_ENV_FILE_ARG"; diff --git a/nym-data-observatory/src/error.rs b/nym-data-observatory/src/error.rs index 1dd5b302c9..03d49219ea 100644 --- a/nym-data-observatory/src/error.rs +++ b/nym-data-observatory/src/error.rs @@ -42,4 +42,7 @@ pub enum NymDataObservatoryError { #[error(transparent)] NymConfigTomlE(#[from] nym_config::error::NymConfigTomlError), + + #[error(transparent)] + UrlParseFailure(#[from] url::ParseError), } diff --git a/nym-data-observatory/src/http/models.rs b/nym-data-observatory/src/http/models.rs index c1798eb59d..34d6d305f1 100644 --- a/nym-data-observatory/src/http/models.rs +++ b/nym-data-observatory/src/http/models.rs @@ -8,6 +8,7 @@ pub mod status { use crate::db::models::CoingeckoPriceResponse; use serde::{Deserialize, Serialize}; use time::OffsetDateTime; + use url::Url; use utoipa::ToSchema; #[derive(Clone, Copy, Debug, Serialize, Deserialize, schemars::JsonSchema, ToSchema)] @@ -31,7 +32,8 @@ pub mod status { pub struct Webhook { pub id: String, pub description: String, - pub webhook_url: String, + #[schema(value_type = String)] + pub webhook_url: Url, pub watched_message_types: Vec, } diff --git a/nym-validator-rewarder/src/cli/process_block.rs b/nym-validator-rewarder/src/cli/process_block.rs index aeb249076d..ba802de275 100644 --- a/nym-validator-rewarder/src/cli/process_block.rs +++ b/nym-validator-rewarder/src/cli/process_block.rs @@ -24,7 +24,7 @@ pub(crate) async fn execute(args: Args) -> Result<(), NymRewarderError> { let config = try_load_current_config(&args.custom_config_path)?.with_override(args.config_override); - SqliteNyxdScraper::new(config.scraper_config()) + SqliteNyxdScraper::new(config.scraper_config()?) .await? .unsafe_process_single_block(args.height) .await?; diff --git a/nym-validator-rewarder/src/cli/process_until.rs b/nym-validator-rewarder/src/cli/process_until.rs index 06c3c68f87..0a34c0852f 100644 --- a/nym-validator-rewarder/src/cli/process_until.rs +++ b/nym-validator-rewarder/src/cli/process_until.rs @@ -37,7 +37,7 @@ pub(crate) async fn execute(args: Args) -> Result<(), NymRewarderError> { let config = try_load_current_config(&args.custom_config_path)?.with_override(args.config_override); - SqliteNyxdScraper::new(config.scraper_config()) + SqliteNyxdScraper::new(config.scraper_config()?) .await? .unsafe_process_block_range(args.start_height, args.stop_height) .await?; diff --git a/nym-validator-rewarder/src/config/mod.rs b/nym-validator-rewarder/src/config/mod.rs index b3824f21ff..146f971bad 100644 --- a/nym-validator-rewarder/src/config/mod.rs +++ b/nym-validator-rewarder/src/config/mod.rs @@ -119,18 +119,23 @@ impl Config { } } - pub fn scraper_config(&self) -> nyxd_scraper_sqlite::Config { - nyxd_scraper_sqlite::Config { + pub fn scraper_config(&self) -> Result { + let database_storage = self.storage_paths.nyxd_scraper.as_path(); + let database_storage = database_storage + .to_str() + .ok_or(NymRewarderError::ConfigError)? + .to_string(); + Ok(nyxd_scraper_sqlite::Config { websocket_url: self.nyxd_scraper.websocket_url.clone(), rpc_url: self.base.upstream_nyxd.clone(), - database_storage: self.storage_paths.nyxd_scraper.clone(), + database_storage, pruning_options: self.nyxd_scraper.pruning, store_precommits: self.nyxd_scraper.store_precommits, start_block: StartingBlockOpts { start_block_height: None, use_best_effort_start_height: true, }, - } + }) } pub fn verification_config(&self) -> ticketbook_issuance::VerificationConfig { diff --git a/nym-validator-rewarder/src/config/persistence/paths.rs b/nym-validator-rewarder/src/config/persistence/paths.rs index 47dd3bf811..c003f900d1 100644 --- a/nym-validator-rewarder/src/config/persistence/paths.rs +++ b/nym-validator-rewarder/src/config/persistence/paths.rs @@ -15,7 +15,7 @@ pub const DEFAULT_ED25519_PUBLIC_IDENTITY_KEY_FILENAME: &str = "ed25519_identity #[derive(Debug, Clone, Deserialize, Serialize)] pub struct ValidatorRewarderPaths { - pub nyxd_scraper: String, + pub nyxd_scraper: PathBuf, pub reward_history: PathBuf, @@ -47,9 +47,7 @@ impl Default for ValidatorRewarderPaths { fn default() -> Self { ValidatorRewarderPaths { // validator rewarder uses sqlite - nyxd_scraper: (default_data_directory().join(DEFAULT_SCRAPER_DB_FILENAME)) - .to_string_lossy() - .to_string(), + nyxd_scraper: default_data_directory().join(DEFAULT_SCRAPER_DB_FILENAME), reward_history: default_data_directory().join(DEFAULT_REWARD_HISTORY_DB_FILENAME), private_ed25519_identity_key_file: default_data_directory() .join(DEFAULT_ED25519_PRIVATE_IDENTITY_KEY_FILENAME), diff --git a/nym-validator-rewarder/src/error.rs b/nym-validator-rewarder/src/error.rs index d20ee0e81c..f082e908ed 100644 --- a/nym-validator-rewarder/src/error.rs +++ b/nym-validator-rewarder/src/error.rs @@ -11,7 +11,7 @@ use nym_validator_client::nyxd::tx::ErrorReport; use nym_validator_client::nyxd::{AccountId, Coin}; use nyxd_scraper_sqlite::error::SqliteScraperError; use std::io; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use thiserror::Error; #[derive(Debug, Error)] @@ -25,6 +25,9 @@ pub enum NymRewarderError { #[error("failed to perform startup SQL migration: {0}")] StartupMigrationFailure(#[from] sqlx::migrate::MigrateError), + #[error("config error: database storage path invalid")] + ConfigError, + #[error( "failed to load config file using path '{}'. detailed message: {source}", path.display() )]