diff --git a/.github/workflows/ci-build-upload-binaries.yml b/.github/workflows/ci-build-upload-binaries.yml index 5a0f325b45..8ceb473118 100644 --- a/.github/workflows/ci-build-upload-binaries.yml +++ b/.github/workflows/ci-build-upload-binaries.yml @@ -109,7 +109,6 @@ jobs: target/release/nym-socks5-client target/release/nym-api target/release/nym-network-requester - target/release/nym-network-statistics target/release/nym-cli target/release/nymvisor target/release/nym-node @@ -129,7 +128,6 @@ jobs: cp target/release/nym-socks5-client $OUTPUT_DIR cp target/release/nym-api $OUTPUT_DIR cp target/release/nym-network-requester $OUTPUT_DIR - cp target/release/nym-network-statistics $OUTPUT_DIR cp target/release/nymvisor $OUTPUT_DIR cp target/release/nym-node $OUTPUT_DIR cp target/release/nym-cli $OUTPUT_DIR diff --git a/Cargo.lock b/Cargo.lock index 03807367fc..c993671a89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4506,7 +4506,7 @@ dependencies = [ [[package]] name = "nym-gateway" -version = "1.1.37" +version = "1.1.36" dependencies = [ "anyhow", "async-trait", @@ -4805,7 +4805,7 @@ dependencies = [ [[package]] name = "nym-mixnode" -version = "1.1.38" +version = "1.1.37" dependencies = [ "anyhow", "axum 0.7.5", @@ -4960,23 +4960,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "nym-network-statistics" -version = "1.1.35" -dependencies = [ - "dirs 4.0.0", - "log", - "nym-bin-common", - "nym-statistics-common", - "nym-task", - "pretty_env_logger", - "rocket", - "serde", - "sqlx", - "thiserror", - "tokio", -] - [[package]] name = "nym-node" version = "1.1.3" diff --git a/Cargo.toml b/Cargo.toml index 1691e9d5ca..c4afac4d69 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -99,7 +99,6 @@ members = [ "service-providers/common", "service-providers/ip-packet-router", "service-providers/network-requester", - "service-providers/network-statistics", "nym-api", "nym-browser-extension/storage", "nym-api/nym-api-requests", @@ -127,7 +126,6 @@ default-members = [ "clients/socks5", "gateway", "service-providers/network-requester", - "service-providers/network-statistics", "mixnode", "nym-api", "tools/nymvisor", diff --git a/service-providers/network-statistics/Cargo.toml b/service-providers/network-statistics/Cargo.toml deleted file mode 100644 index 10c19872c4..0000000000 --- a/service-providers/network-statistics/Cargo.toml +++ /dev/null @@ -1,24 +0,0 @@ -[package] -name = "nym-network-statistics" -version = "1.1.34" -edition = "2021" -license.workspace = true - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -dirs = "4.0" -log = { workspace = true } -pretty_env_logger = { workspace = true } -rocket = { workspace = true, features = ["json"] } -serde = { workspace = true, features = ["derive"] } -sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate", "chrono"]} -thiserror = { workspace = true } -tokio = { workspace = true, features = ["net", "rt-multi-thread", "macros", "time"] } -nym-bin-common = { path = "../../common/bin-common"} -nym-statistics-common = { path = "../../common/statistics" } -nym-task = { path = "../../common/task" } - -[build-dependencies] -sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate"] } -tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } diff --git a/service-providers/network-statistics/Rocket.toml b/service-providers/network-statistics/Rocket.toml deleted file mode 100644 index 5828739b6f..0000000000 --- a/service-providers/network-statistics/Rocket.toml +++ /dev/null @@ -1,7 +0,0 @@ -[default] -limits = { forms = "64 kB", json = "1 MiB" } -port = 8090 -address = "127.0.0.1" - -[release] -address = "0.0.0.0" \ No newline at end of file diff --git a/service-providers/network-statistics/build.rs b/service-providers/network-statistics/build.rs deleted file mode 100644 index 88d3505015..0000000000 --- a/service-providers/network-statistics/build.rs +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2022 - Nym Technologies SA -// SPDX-License-Identifier: Apache-2.0 - -use sqlx::{Connection, SqliteConnection}; -use std::env; - -#[tokio::main] -async fn main() { - let out_dir = env::var("OUT_DIR").unwrap(); - let database_path = format!("{out_dir}/network-statistics-example.sqlite"); - - let mut conn = SqliteConnection::connect(&format!("sqlite://{database_path}?mode=rwc")) - .await - .expect("Failed to create SQLx database connection"); - - sqlx::migrate!("./migrations") - .run(&mut conn) - .await - .expect("Failed to perform SQLx migrations"); - - #[cfg(target_family = "unix")] - println!("cargo:rustc-env=DATABASE_URL=sqlite://{}", &database_path); - - #[cfg(target_family = "windows")] - // for some strange reason we need to add a leading `/` to the windows path even though it's - // not a valid windows path... but hey, it works... - println!("cargo:rustc-env=DATABASE_URL=sqlite:///{}", &database_path); -} diff --git a/service-providers/network-statistics/migrations/20220512120000_mixnet_statistics.sql b/service-providers/network-statistics/migrations/20220512120000_mixnet_statistics.sql deleted file mode 100644 index da13b953ca..0000000000 --- a/service-providers/network-statistics/migrations/20220512120000_mixnet_statistics.sql +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2022 - Nym Technologies SA - * SPDX-License-Identifier: Apache-2.0 - */ - -CREATE TABLE service_statistics -( - id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - requested_service VARCHAR NOT NULL, - request_processed_bytes INTEGER NOT NULL, - response_processed_bytes INTEGER NOT NULL, - interval_seconds INTEGER NOT NULL, - timestamp DATETIME NOT NULL -); - -CREATE TABLE gateway_statistics -( - id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - gateway_id VARCHAR NOT NULL, - inbox_count INTEGER NOT NULL, - timestamp DATETIME NOT NULL -); \ No newline at end of file diff --git a/service-providers/network-statistics/src/api/error.rs b/service-providers/network-statistics/src/api/error.rs deleted file mode 100644 index bebafc2fde..0000000000 --- a/service-providers/network-statistics/src/api/error.rs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2022 - Nym Technologies SA -// SPDX-License-Identifier: Apache-2.0 - -use rocket::http::{ContentType, Status}; -use rocket::response::Responder; -use rocket::{response, Request, Response}; -use std::io::Cursor; - -use crate::storage::error::NetworkStatisticsStorageError; - -pub type Result = std::result::Result; - -#[derive(Debug, thiserror::Error)] -pub enum NetworkStatisticsAPIError { - #[error(transparent)] - RocketError(#[from] Box), - - #[error(transparent)] - StorageError(#[from] NetworkStatisticsStorageError), -} - -impl<'r, 'o: 'r> Responder<'r, 'o> for NetworkStatisticsAPIError { - fn respond_to(self, _: &'r Request<'_>) -> response::Result<'o> { - let err_msg = self.to_string(); - Response::build() - .header(ContentType::Plain) - .sized_body(err_msg.len(), Cursor::new(err_msg)) - .status(Status::BadRequest) - .ok() - } -} diff --git a/service-providers/network-statistics/src/api/mod.rs b/service-providers/network-statistics/src/api/mod.rs deleted file mode 100644 index fc32cc8d7b..0000000000 --- a/service-providers/network-statistics/src/api/mod.rs +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2022 - Nym Technologies SA -// SPDX-License-Identifier: Apache-2.0 - -use log::*; -use rocket::{Ignite, Rocket}; - -use crate::storage::NetworkStatisticsStorage; -use error::Result; -use routes::{post_all_statistics, post_statistic}; - -use nym_statistics_common::api::STATISTICS_SERVICE_VERSION; -use nym_task::TaskManager; - -mod error; -mod routes; - -pub(crate) struct NetworkStatisticsAPI { - rocket: Rocket, -} - -impl NetworkStatisticsAPI { - pub async fn init(storage: NetworkStatisticsStorage) -> Result { - let rocket = rocket::build() - .mount( - STATISTICS_SERVICE_VERSION, - rocket::routes![post_all_statistics, post_statistic], - ) - .manage(storage.clone()) - .ignite() - .await - .map_err(Box::new)?; - - Ok(NetworkStatisticsAPI { rocket }) - } - - pub async fn run(self) { - let rocket_shutdown_handle = self.rocket.shutdown(); - let mut shutdown = TaskManager::new(10); - tokio::spawn(self.rocket.launch()); - - shutdown.catch_interrupt().await.ok(); - info!("Stopping network statistics"); - rocket_shutdown_handle.notify(); - } -} diff --git a/service-providers/network-statistics/src/api/routes.rs b/service-providers/network-statistics/src/api/routes.rs deleted file mode 100644 index 33bad5d108..0000000000 --- a/service-providers/network-statistics/src/api/routes.rs +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2022 - Nym Technologies SA -// SPDX-License-Identifier: Apache-2.0 - -use crate::api::error::Result; -use crate::storage::NetworkStatisticsStorage; -use nym_statistics_common::StatsMessage; -use rocket::serde::json::Json; -use rocket::State; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Serialize, Deserialize, Debug)] -pub struct StatisticsRequest { - // date, RFC 3339 format - since: String, - // date, RFC 3339 format - until: String, -} - -#[derive(Clone, Serialize, Deserialize, Debug)] -pub enum GenericStatistic { - Service(ServiceStatistic), - Gateway(GatewayStatistic), -} - -#[derive(Clone, Serialize, Deserialize, Debug)] -pub struct ServiceStatistic { - pub requested_service: String, - pub request_processed_bytes: u32, - pub response_processed_bytes: u32, - pub interval_seconds: u32, - pub timestamp: String, -} - -#[derive(Clone, Serialize, Deserialize, Debug)] -pub struct GatewayStatistic { - pub gateway_id: String, - pub inbox_count: u32, - pub timestamp: String, -} - -#[rocket::post("/all-statistics", data = "")] -pub(crate) async fn post_all_statistics( - all_statistics_request: Json, - storage: &State, -) -> Result>> { - let all_statistics = storage - .get_service_statistics_in_interval( - &all_statistics_request.since, - &all_statistics_request.until, - ) - .await? - .into_iter() - .map(|data| { - GenericStatistic::Service(ServiceStatistic { - requested_service: data.requested_service, - request_processed_bytes: data.request_processed_bytes as u32, - response_processed_bytes: data.response_processed_bytes as u32, - interval_seconds: data.interval_seconds as u32, - timestamp: data.timestamp.to_string(), - }) - }) - .chain( - storage - .get_gateway_statistics_in_interval( - &all_statistics_request.since, - &all_statistics_request.until, - ) - .await? - .into_iter() - .map(|data| { - GenericStatistic::Gateway(GatewayStatistic { - gateway_id: data.gateway_id, - inbox_count: data.inbox_count as u32, - timestamp: data.timestamp.to_string(), - }) - }), - ) - .collect(); - - Ok(Json(all_statistics)) -} - -#[rocket::post("/statistic", data = "")] -pub(crate) async fn post_statistic( - statistic: Json, - storage: &State, -) -> Result> { - storage.insert_statistics(statistic.0).await?; - Ok(Json(())) -} diff --git a/service-providers/network-statistics/src/main.rs b/service-providers/network-statistics/src/main.rs deleted file mode 100644 index e9ced2c1ef..0000000000 --- a/service-providers/network-statistics/src/main.rs +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2022 - Nym Technologies SA -// SPDX-License-Identifier: Apache-2.0 - -use api::NetworkStatisticsAPI; -use nym_bin_common::logging::setup_logging; -use std::path::PathBuf; - -mod api; -mod storage; - -#[tokio::main] -async fn main() { - setup_logging(); - - let base_dir = default_base_dir(); - let storage = storage::NetworkStatisticsStorage::init(&base_dir) - .await - .expect("Could not create network statistics storage"); - - let api = NetworkStatisticsAPI::init(storage) - .await - .expect("Could not ignite stats api service"); - api.run().await; -} - -/// Returns the default base directory for the storefile. -/// -/// This is split out so we can easily inject our own base_dir for unit tests. -fn default_base_dir() -> PathBuf { - dirs::home_dir() - .expect("no home directory known for this OS") - .join(".nym") - .join("service-providers") - .join("network-statistics") -} diff --git a/service-providers/network-statistics/src/storage/error.rs b/service-providers/network-statistics/src/storage/error.rs deleted file mode 100644 index 48253a261b..0000000000 --- a/service-providers/network-statistics/src/storage/error.rs +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2022 - Nym Technologies SA -// SPDX-License-Identifier: Apache-2.0 - -#[derive(Debug, thiserror::Error)] -pub enum NetworkStatisticsStorageError { - #[error("File system error - {0}")] - FSError(#[from] std::io::Error), - - #[error("SQL error - {0}")] - InternalDatabaseError(#[from] sqlx::Error), - - #[error("SQL migrate error - {0}")] - DatabaseMigrateError(#[from] sqlx::migrate::MigrateError), - - #[error("Timestamp could not be parsed")] - TimestampParse, -} diff --git a/service-providers/network-statistics/src/storage/manager.rs b/service-providers/network-statistics/src/storage/manager.rs deleted file mode 100644 index f77a5aace4..0000000000 --- a/service-providers/network-statistics/src/storage/manager.rs +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2022 - Nym Technologies SA -// SPDX-License-Identifier: Apache-2.0 - -use sqlx::types::chrono::{DateTime, Utc}; - -use crate::storage::models::{GatewayStatistics, ServiceStatistics}; - -#[derive(Clone)] -pub(crate) struct StorageManager { - pub(crate) connection_pool: sqlx::SqlitePool, -} - -// all SQL goes here -impl StorageManager { - /// Adds an entry for some service statistical data. - /// - /// # Arguments - /// - /// * `requested_service`: Address of the service requested. - /// * `request_processed_bytes`: Number of bytes for socks5 requests. - /// * `response_processed_bytes`: Number of bytes for socks5 responses. - /// * `interval_seconds`: Duration in seconds in which the data was gathered. - /// * `timestamp`: The moment in time when the data started being collected. - pub(super) async fn insert_service_statistics( - &self, - requested_service: String, - request_processed_bytes: u32, - response_processed_bytes: u32, - interval_seconds: u32, - timestamp: DateTime, - ) -> Result<(), sqlx::Error> { - sqlx::query!( - "INSERT INTO service_statistics(requested_service, request_processed_bytes, response_processed_bytes, interval_seconds, timestamp) VALUES (?, ?, ?, ?, ?)", - requested_service, - request_processed_bytes, - response_processed_bytes, - interval_seconds, - timestamp, - ) - .execute(&self.connection_pool) - .await?; - - Ok(()) - } - - /// Adds an entry for some gateway statistical data. - /// - /// # Arguments - /// - /// * `inbox_count`: Number of clients of a gateway. - /// * `interval_seconds`: Duration in seconds in which the data was gathered. - /// * `timestamp`: The moment in time when the data started being collected. - pub(super) async fn insert_gateway_statistics( - &self, - gateway_id: String, - inbox_count: u32, - timestamp: DateTime, - ) -> Result<(), sqlx::Error> { - sqlx::query!( - "INSERT INTO gateway_statistics(gateway_id, inbox_count, timestamp) VALUES (?, ?, ?)", - gateway_id, - inbox_count, - timestamp, - ) - .execute(&self.connection_pool) - .await?; - - Ok(()) - } - - /// Returns service statistical data submitted within the provided time interval. - /// - /// # Arguments - /// - /// * `since`: indicates the lower bound timestamp for the data - /// * `until`: indicates the upper bound timestamp for the data - pub(super) async fn get_service_statistics_in_interval( - &self, - since: DateTime, - until: DateTime, - ) -> Result, sqlx::Error> { - sqlx::query_as!( - ServiceStatistics, - "SELECT * FROM service_statistics WHERE timestamp BETWEEN ? AND ?", - since, - until - ) - .fetch_all(&self.connection_pool) - .await - } - - /// Returns gateway statistical data submitted within the provided time interval. - /// - /// # Arguments - /// - /// * `since`: indicates the lower bound timestamp for the data - /// * `until`: indicates the upper bound timestamp for the data - pub(super) async fn get_gateway_statistics_in_interval( - &self, - since: DateTime, - until: DateTime, - ) -> Result, sqlx::Error> { - sqlx::query_as!( - GatewayStatistics, - "SELECT * FROM gateway_statistics WHERE timestamp BETWEEN ? AND ?", - since, - until - ) - .fetch_all(&self.connection_pool) - .await - } -} diff --git a/service-providers/network-statistics/src/storage/mod.rs b/service-providers/network-statistics/src/storage/mod.rs deleted file mode 100644 index f485b08145..0000000000 --- a/service-providers/network-statistics/src/storage/mod.rs +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright 2022 - Nym Technologies SA -// SPDX-License-Identifier: Apache-2.0 - -use log::*; -use sqlx::types::chrono::{DateTime, Utc}; -use sqlx::ConnectOptions; -use std::path::PathBuf; - -use nym_statistics_common::StatsMessage; - -use crate::storage::error::NetworkStatisticsStorageError; -use crate::storage::manager::StorageManager; -use crate::storage::models::{GatewayStatistics, ServiceStatistics}; - -pub(crate) mod error; -mod manager; -mod models; - -// note that clone here is fine as upon cloning the same underlying pool will be used -#[derive(Clone)] -pub(crate) struct NetworkStatisticsStorage { - manager: StorageManager, -} - -impl NetworkStatisticsStorage { - pub async fn init(base_dir: &PathBuf) -> Result { - std::fs::create_dir_all(base_dir)?; - let database_path = base_dir.join("db.sqlite"); - let mut opts = sqlx::sqlite::SqliteConnectOptions::new() - .filename(database_path) - .create_if_missing(true); - - opts.disable_statement_logging(); - - let connection_pool = sqlx::SqlitePool::connect_with(opts).await?; - - sqlx::migrate!("./migrations").run(&connection_pool).await?; - info!("Database migration finished!"); - - let storage = NetworkStatisticsStorage { - manager: StorageManager { connection_pool }, - }; - - Ok(storage) - } - - /// Adds an entry for some statistical data. - /// - /// # Arguments - /// - /// * `msg`: Message containing the statistical data. - pub(super) async fn insert_statistics( - &self, - msg: StatsMessage, - ) -> Result<(), NetworkStatisticsStorageError> { - let timestamp: DateTime = DateTime::parse_from_rfc3339(&msg.timestamp) - .map_err(|_| NetworkStatisticsStorageError::TimestampParse)? - .into(); - for stats_data in msg.stats_data { - match stats_data { - nym_statistics_common::StatsData::Service(service_data) => { - self.manager - .insert_service_statistics( - service_data.requested_service.clone(), - service_data.request_bytes, - service_data.response_bytes, - msg.interval_seconds, - timestamp, - ) - .await?; - } - nym_statistics_common::StatsData::Gateway(gateway_data) => { - self.manager - .insert_gateway_statistics( - gateway_data.gateway_id, - gateway_data.inbox_count, - timestamp, - ) - .await? - } - } - } - - Ok(()) - } - - /// Returns service data submitted within the provided time interval. - /// - /// # Arguments - /// - /// * `since`: indicates the lower bound timestamp for the data, RFC 3339 format - /// * `until`: indicates the upper bound timestamp for the data, RFC 3339 format - pub(super) async fn get_service_statistics_in_interval( - &self, - since: &str, - until: &str, - ) -> Result, NetworkStatisticsStorageError> { - let since = DateTime::parse_from_rfc3339(since) - .map_err(|_| NetworkStatisticsStorageError::TimestampParse)? - .into(); - let until = DateTime::parse_from_rfc3339(until) - .map_err(|_| NetworkStatisticsStorageError::TimestampParse)? - .into(); - Ok(self - .manager - .get_service_statistics_in_interval(since, until) - .await?) - } - - /// Returns gateway data submitted within the provided time interval. - /// - /// # Arguments - /// - /// * `since`: indicates the lower bound timestamp for the data, RFC 3339 format - /// * `until`: indicates the upper bound timestamp for the data, RFC 3339 format - pub(super) async fn get_gateway_statistics_in_interval( - &self, - since: &str, - until: &str, - ) -> Result, NetworkStatisticsStorageError> { - let since = DateTime::parse_from_rfc3339(since) - .map_err(|_| NetworkStatisticsStorageError::TimestampParse)? - .into(); - let until = DateTime::parse_from_rfc3339(until) - .map_err(|_| NetworkStatisticsStorageError::TimestampParse)? - .into(); - Ok(self - .manager - .get_gateway_statistics_in_interval(since, until) - .await?) - } -} diff --git a/service-providers/network-statistics/src/storage/models.rs b/service-providers/network-statistics/src/storage/models.rs deleted file mode 100644 index d179914540..0000000000 --- a/service-providers/network-statistics/src/storage/models.rs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2022 - Nym Technologies SA -// SPDX-License-Identifier: Apache-2.0 - -use sqlx::types::chrono::NaiveDateTime; - -// Internally used struct to catch results from the database to get mixnet statistics -pub(crate) struct ServiceStatistics { - #[allow(dead_code)] - pub(crate) id: i64, - pub(crate) requested_service: String, - pub(crate) request_processed_bytes: i64, - pub(crate) response_processed_bytes: i64, - pub(crate) interval_seconds: i64, - pub(crate) timestamp: NaiveDateTime, -} - -pub(crate) struct GatewayStatistics { - #[allow(dead_code)] - pub(crate) id: i64, - pub(crate) gateway_id: String, - pub(crate) inbox_count: i64, - pub(crate) timestamp: NaiveDateTime, -}