GRIN/nym
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove nym-network-statistics (#4678)

Browse Source 
* Remove nym-network-statistics from workspace

* Remove nym-network-statistics

* Cargo.lock

* Update ci workflow
This commit is contained in:
Jon Häggblad
2024-06-27 10:32:09 +02:00
committed by GitHub
parent 24ffb8fe8c
commit 14a904eff0
15 changed files with 2 additions and 589 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci-build-upload-binaries.yml
-2
View File
@@ -109,7 +109,6 @@ jobs:
target/release/nym-socks5-client
target/release/nym-api
target/release/nym-network-requester
target/release/nym-network-statistics
target/release/nym-cli
target/release/nymvisor
target/release/nym-node
@@ -129,7 +128,6 @@ jobs:
cp target/release/nym-socks5-client $OUTPUT_DIR
cp target/release/nym-api $OUTPUT_DIR
cp target/release/nym-network-requester $OUTPUT_DIR
cp target/release/nym-network-statistics $OUTPUT_DIR
cp target/release/nymvisor $OUTPUT_DIR
cp target/release/nym-node $OUTPUT_DIR
cp target/release/nym-cli $OUTPUT_DIR
Cargo.lock
Generated
+2 -19
View File
@@ -4506,7 +4506,7 @@ dependencies = [
[[package]]
name = "nym-gateway"
version = "1.1.37"
version = "1.1.36"
dependencies = [
"anyhow",
"async-trait",
@@ -4805,7 +4805,7 @@ dependencies = [
[[package]]
name = "nym-mixnode"
version = "1.1.38"
version = "1.1.37"
dependencies = [
"anyhow",
"axum 0.7.5",
@@ -4960,23 +4960,6 @@ dependencies = [
"zeroize",
]
[[package]]
name = "nym-network-statistics"
version = "1.1.35"
dependencies = [
"dirs 4.0.0",
"log",
"nym-bin-common",
"nym-statistics-common",
"nym-task",
"pretty_env_logger",
"rocket",
"serde",
"sqlx",
"thiserror",
"tokio",
]
[[package]]
name = "nym-node"
version = "1.1.3"
Cargo.toml
-2
View File
@@ -99,7 +99,6 @@ members = [
"service-providers/common",
"service-providers/ip-packet-router",
"service-providers/network-requester",
"service-providers/network-statistics",
"nym-api",
"nym-browser-extension/storage",
"nym-api/nym-api-requests",
@@ -127,7 +126,6 @@ default-members = [
"clients/socks5",
"gateway",
"service-providers/network-requester",
"service-providers/network-statistics",
"mixnode",
"nym-api",
"tools/nymvisor",
service-providers/network-statistics/Cargo.toml
-24
View File
@@ -1,24 +0,0 @@
[package]
name = "nym-network-statistics"
version = "1.1.34"
edition = "2021"
license.workspace = true
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
dirs = "4.0"
log = { workspace = true }
pretty_env_logger = { workspace = true }
rocket = { workspace = true, features = ["json"] }
serde = { workspace = true, features = ["derive"] }
sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate", "chrono"]}
thiserror = { workspace = true }
tokio = { workspace = true, features = ["net", "rt-multi-thread", "macros", "time"] }
nym-bin-common = { path = "../../common/bin-common"}
nym-statistics-common = { path = "../../common/statistics" }
nym-task = { path = "../../common/task" }
[build-dependencies]
sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate"] }
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
service-providers/network-statistics/Rocket.toml
-7
View File
@@ -1,7 +0,0 @@
[default]
limits = { forms = "64 kB", json = "1 MiB" }
port = 8090
address = "127.0.0.1"
[release]
address = "0.0.0.0"
service-providers/network-statistics/build.rs
-28
View File
@@ -1,28 +0,0 @@
// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use sqlx::{Connection, SqliteConnection};
use std::env;
#[tokio::main]
async fn main() {
let out_dir = env::var("OUT_DIR").unwrap();
let database_path = format!("{out_dir}/network-statistics-example.sqlite");
let mut conn = SqliteConnection::connect(&format!("sqlite://{database_path}?mode=rwc"))
.await
.expect("Failed to create SQLx database connection");
sqlx::migrate!("./migrations")
.run(&mut conn)
.await
.expect("Failed to perform SQLx migrations");
#[cfg(target_family = "unix")]
println!("cargo:rustc-env=DATABASE_URL=sqlite://{}", &database_path);
#[cfg(target_family = "windows")]
// for some strange reason we need to add a leading `/` to the windows path even though it's
// not a valid windows path... but hey, it works...
println!("cargo:rustc-env=DATABASE_URL=sqlite:///{}", &database_path);
}
service-providers/network-statistics/migrations/20220512120000_mixnet_statistics.sql
-22
View File
@@ -1,22 +0,0 @@
/*
* Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
* SPDX-License-Identifier: Apache-2.0
*/
CREATE TABLE service_statistics
(
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
requested_service VARCHAR NOT NULL,
request_processed_bytes INTEGER NOT NULL,
response_processed_bytes INTEGER NOT NULL,
interval_seconds INTEGER NOT NULL,
timestamp DATETIME NOT NULL
);
CREATE TABLE gateway_statistics
(
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
gateway_id VARCHAR NOT NULL,
inbox_count INTEGER NOT NULL,
timestamp DATETIME NOT NULL
);
service-providers/network-statistics/src/api/error.rs
-31
View File
@@ -1,31 +0,0 @@
// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use rocket::http::{ContentType, Status};
use rocket::response::Responder;
use rocket::{response, Request, Response};
use std::io::Cursor;
use crate::storage::error::NetworkStatisticsStorageError;
pub type Result<T> = std::result::Result<T, NetworkStatisticsAPIError>;
#[derive(Debug, thiserror::Error)]
pub enum NetworkStatisticsAPIError {
#[error(transparent)]
RocketError(#[from] Box<rocket::Error>),
#[error(transparent)]
StorageError(#[from] NetworkStatisticsStorageError),
}
impl<'r, 'o: 'r> Responder<'r, 'o> for NetworkStatisticsAPIError {
fn respond_to(self, _: &'r Request<'_>) -> response::Result<'o> {
let err_msg = self.to_string();
Response::build()
.header(ContentType::Plain)
.sized_body(err_msg.len(), Cursor::new(err_msg))
.status(Status::BadRequest)
.ok()
}
}
service-providers/network-statistics/src/api/mod.rs
-45
View File
@@ -1,45 +0,0 @@
// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use log::*;
use rocket::{Ignite, Rocket};
use crate::storage::NetworkStatisticsStorage;
use error::Result;
use routes::{post_all_statistics, post_statistic};
use nym_statistics_common::api::STATISTICS_SERVICE_VERSION;
use nym_task::TaskManager;
mod error;
mod routes;
pub(crate) struct NetworkStatisticsAPI {
rocket: Rocket<Ignite>,
}
impl NetworkStatisticsAPI {
pub async fn init(storage: NetworkStatisticsStorage) -> Result<Self> {
let rocket = rocket::build()
.mount(
STATISTICS_SERVICE_VERSION,
rocket::routes![post_all_statistics, post_statistic],
)
.manage(storage.clone())
.ignite()
.await
.map_err(Box::new)?;
Ok(NetworkStatisticsAPI { rocket })
}
pub async fn run(self) {
let rocket_shutdown_handle = self.rocket.shutdown();
let mut shutdown = TaskManager::new(10);
tokio::spawn(self.rocket.launch());
shutdown.catch_interrupt().await.ok();
info!("Stopping network statistics");
rocket_shutdown_handle.notify();
}
}
service-providers/network-statistics/src/api/routes.rs
-90
View File
@@ -1,90 +0,0 @@
// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use crate::api::error::Result;
use crate::storage::NetworkStatisticsStorage;
use nym_statistics_common::StatsMessage;
use rocket::serde::json::Json;
use rocket::State;
use serde::{Deserialize, Serialize};
#[derive(Clone, Serialize, Deserialize, Debug)]
pub struct StatisticsRequest {
// date, RFC 3339 format
since: String,
// date, RFC 3339 format
until: String,
}
#[derive(Clone, Serialize, Deserialize, Debug)]
pub enum GenericStatistic {
Service(ServiceStatistic),
Gateway(GatewayStatistic),
}
#[derive(Clone, Serialize, Deserialize, Debug)]
pub struct ServiceStatistic {
pub requested_service: String,
pub request_processed_bytes: u32,
pub response_processed_bytes: u32,
pub interval_seconds: u32,
pub timestamp: String,
}
#[derive(Clone, Serialize, Deserialize, Debug)]
pub struct GatewayStatistic {
pub gateway_id: String,
pub inbox_count: u32,
pub timestamp: String,
}
#[rocket::post("/all-statistics", data = "<all_statistics_request>")]
pub(crate) async fn post_all_statistics(
all_statistics_request: Json<StatisticsRequest>,
storage: &State<NetworkStatisticsStorage>,
) -> Result<Json<Vec<GenericStatistic>>> {
let all_statistics = storage
.get_service_statistics_in_interval(
&all_statistics_request.since,
&all_statistics_request.until,
)
.await?
.into_iter()
.map(|data| {
GenericStatistic::Service(ServiceStatistic {
requested_service: data.requested_service,
request_processed_bytes: data.request_processed_bytes as u32,
response_processed_bytes: data.response_processed_bytes as u32,
interval_seconds: data.interval_seconds as u32,
timestamp: data.timestamp.to_string(),
})
})
.chain(
storage
.get_gateway_statistics_in_interval(
&all_statistics_request.since,
&all_statistics_request.until,
)
.await?
.into_iter()
.map(|data| {
GenericStatistic::Gateway(GatewayStatistic {
gateway_id: data.gateway_id,
inbox_count: data.inbox_count as u32,
timestamp: data.timestamp.to_string(),
})
}),
)
.collect();
Ok(Json(all_statistics))
}
#[rocket::post("/statistic", data = "<statistic>")]
pub(crate) async fn post_statistic(
statistic: Json<StatsMessage>,
storage: &State<NetworkStatisticsStorage>,
) -> Result<Json<()>> {
storage.insert_statistics(statistic.0).await?;
Ok(Json(()))
}
service-providers/network-statistics/src/main.rs
-35
View File
@@ -1,35 +0,0 @@
// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use api::NetworkStatisticsAPI;
use nym_bin_common::logging::setup_logging;
use std::path::PathBuf;
mod api;
mod storage;
#[tokio::main]
async fn main() {
setup_logging();
let base_dir = default_base_dir();
let storage = storage::NetworkStatisticsStorage::init(&base_dir)
.await
.expect("Could not create network statistics storage");
let api = NetworkStatisticsAPI::init(storage)
.await
.expect("Could not ignite stats api service");
api.run().await;
}
/// Returns the default base directory for the storefile.
///
/// This is split out so we can easily inject our own base_dir for unit tests.
fn default_base_dir() -> PathBuf {
dirs::home_dir()
.expect("no home directory known for this OS")
.join(".nym")
.join("service-providers")
.join("network-statistics")
}
service-providers/network-statistics/src/storage/error.rs
-17
View File
@@ -1,17 +0,0 @@
// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
#[derive(Debug, thiserror::Error)]
pub enum NetworkStatisticsStorageError {
#[error("File system error - {0}")]
FSError(#[from] std::io::Error),
#[error("SQL error - {0}")]
InternalDatabaseError(#[from] sqlx::Error),
#[error("SQL migrate error - {0}")]
DatabaseMigrateError(#[from] sqlx::migrate::MigrateError),
#[error("Timestamp could not be parsed")]
TimestampParse,
}
service-providers/network-statistics/src/storage/manager.rs
-112
View File
@@ -1,112 +0,0 @@
// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use sqlx::types::chrono::{DateTime, Utc};
use crate::storage::models::{GatewayStatistics, ServiceStatistics};
#[derive(Clone)]
pub(crate) struct StorageManager {
pub(crate) connection_pool: sqlx::SqlitePool,
}
// all SQL goes here
impl StorageManager {
/// Adds an entry for some service statistical data.
///
/// # Arguments
///
/// * `requested_service`: Address of the service requested.
/// * `request_processed_bytes`: Number of bytes for socks5 requests.
/// * `response_processed_bytes`: Number of bytes for socks5 responses.
/// * `interval_seconds`: Duration in seconds in which the data was gathered.
/// * `timestamp`: The moment in time when the data started being collected.
pub(super) async fn insert_service_statistics(
&self,
requested_service: String,
request_processed_bytes: u32,
response_processed_bytes: u32,
interval_seconds: u32,
timestamp: DateTime<Utc>,
) -> Result<(), sqlx::Error> {
sqlx::query!(
"INSERT INTO service_statistics(requested_service, request_processed_bytes, response_processed_bytes, interval_seconds, timestamp) VALUES (?, ?, ?, ?, ?)",
requested_service,
request_processed_bytes,
response_processed_bytes,
interval_seconds,
timestamp,
)
.execute(&self.connection_pool)
.await?;
Ok(())
}
/// Adds an entry for some gateway statistical data.
///
/// # Arguments
///
/// * `inbox_count`: Number of clients of a gateway.
/// * `interval_seconds`: Duration in seconds in which the data was gathered.
/// * `timestamp`: The moment in time when the data started being collected.
pub(super) async fn insert_gateway_statistics(
&self,
gateway_id: String,
inbox_count: u32,
timestamp: DateTime<Utc>,
) -> Result<(), sqlx::Error> {
sqlx::query!(
"INSERT INTO gateway_statistics(gateway_id, inbox_count, timestamp) VALUES (?, ?, ?)",
gateway_id,
inbox_count,
timestamp,
)
.execute(&self.connection_pool)
.await?;
Ok(())
}
/// Returns service statistical data submitted within the provided time interval.
///
/// # Arguments
///
/// * `since`: indicates the lower bound timestamp for the data
/// * `until`: indicates the upper bound timestamp for the data
pub(super) async fn get_service_statistics_in_interval(
&self,
since: DateTime<Utc>,
until: DateTime<Utc>,
) -> Result<Vec<ServiceStatistics>, sqlx::Error> {
sqlx::query_as!(
ServiceStatistics,
"SELECT * FROM service_statistics WHERE timestamp BETWEEN ? AND ?",
since,
until
)
.fetch_all(&self.connection_pool)
.await
}
/// Returns gateway statistical data submitted within the provided time interval.
///
/// # Arguments
///
/// * `since`: indicates the lower bound timestamp for the data
/// * `until`: indicates the upper bound timestamp for the data
pub(super) async fn get_gateway_statistics_in_interval(
&self,
since: DateTime<Utc>,
until: DateTime<Utc>,
) -> Result<Vec<GatewayStatistics>, sqlx::Error> {
sqlx::query_as!(
GatewayStatistics,
"SELECT * FROM gateway_statistics WHERE timestamp BETWEEN ? AND ?",
since,
until
)
.fetch_all(&self.connection_pool)
.await
}
}
service-providers/network-statistics/src/storage/mod.rs
-132
View File
@@ -1,132 +0,0 @@
// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use log::*;
use sqlx::types::chrono::{DateTime, Utc};
use sqlx::ConnectOptions;
use std::path::PathBuf;
use nym_statistics_common::StatsMessage;
use crate::storage::error::NetworkStatisticsStorageError;
use crate::storage::manager::StorageManager;
use crate::storage::models::{GatewayStatistics, ServiceStatistics};
pub(crate) mod error;
mod manager;
mod models;
// note that clone here is fine as upon cloning the same underlying pool will be used
#[derive(Clone)]
pub(crate) struct NetworkStatisticsStorage {
manager: StorageManager,
}
impl NetworkStatisticsStorage {
pub async fn init(base_dir: &PathBuf) -> Result<Self, NetworkStatisticsStorageError> {
std::fs::create_dir_all(base_dir)?;
let database_path = base_dir.join("db.sqlite");
let mut opts = sqlx::sqlite::SqliteConnectOptions::new()
.filename(database_path)
.create_if_missing(true);
opts.disable_statement_logging();
let connection_pool = sqlx::SqlitePool::connect_with(opts).await?;
sqlx::migrate!("./migrations").run(&connection_pool).await?;
info!("Database migration finished!");
let storage = NetworkStatisticsStorage {
manager: StorageManager { connection_pool },
};
Ok(storage)
}
/// Adds an entry for some statistical data.
///
/// # Arguments
///
/// * `msg`: Message containing the statistical data.
pub(super) async fn insert_statistics(
&self,
msg: StatsMessage,
) -> Result<(), NetworkStatisticsStorageError> {
let timestamp: DateTime<Utc> = DateTime::parse_from_rfc3339(&msg.timestamp)
.map_err(|_| NetworkStatisticsStorageError::TimestampParse)?
.into();
for stats_data in msg.stats_data {
match stats_data {
nym_statistics_common::StatsData::Service(service_data) => {
self.manager
.insert_service_statistics(
service_data.requested_service.clone(),
service_data.request_bytes,
service_data.response_bytes,
msg.interval_seconds,
timestamp,
)
.await?;
}
nym_statistics_common::StatsData::Gateway(gateway_data) => {
self.manager
.insert_gateway_statistics(
gateway_data.gateway_id,
gateway_data.inbox_count,
timestamp,
)
.await?
}
}
}
Ok(())
}
/// Returns service data submitted within the provided time interval.
///
/// # Arguments
///
/// * `since`: indicates the lower bound timestamp for the data, RFC 3339 format
/// * `until`: indicates the upper bound timestamp for the data, RFC 3339 format
pub(super) async fn get_service_statistics_in_interval(
&self,
since: &str,
until: &str,
) -> Result<Vec<ServiceStatistics>, NetworkStatisticsStorageError> {
let since = DateTime::parse_from_rfc3339(since)
.map_err(|_| NetworkStatisticsStorageError::TimestampParse)?
.into();
let until = DateTime::parse_from_rfc3339(until)
.map_err(|_| NetworkStatisticsStorageError::TimestampParse)?
.into();
Ok(self
.manager
.get_service_statistics_in_interval(since, until)
.await?)
}
/// Returns gateway data submitted within the provided time interval.
///
/// # Arguments
///
/// * `since`: indicates the lower bound timestamp for the data, RFC 3339 format
/// * `until`: indicates the upper bound timestamp for the data, RFC 3339 format
pub(super) async fn get_gateway_statistics_in_interval(
&self,
since: &str,
until: &str,
) -> Result<Vec<GatewayStatistics>, NetworkStatisticsStorageError> {
let since = DateTime::parse_from_rfc3339(since)
.map_err(|_| NetworkStatisticsStorageError::TimestampParse)?
.into();
let until = DateTime::parse_from_rfc3339(until)
.map_err(|_| NetworkStatisticsStorageError::TimestampParse)?
.into();
Ok(self
.manager
.get_gateway_statistics_in_interval(since, until)
.await?)
}
}
service-providers/network-statistics/src/storage/models.rs
-23
View File
@@ -1,23 +0,0 @@
// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use sqlx::types::chrono::NaiveDateTime;
// Internally used struct to catch results from the database to get mixnet statistics
pub(crate) struct ServiceStatistics {
#[allow(dead_code)]
pub(crate) id: i64,
pub(crate) requested_service: String,
pub(crate) request_processed_bytes: i64,
pub(crate) response_processed_bytes: i64,
pub(crate) interval_seconds: i64,
pub(crate) timestamp: NaiveDateTime,
}
pub(crate) struct GatewayStatistics {
#[allow(dead_code)]
pub(crate) id: i64,
pub(crate) gateway_id: String,
pub(crate) inbox_count: i64,
pub(crate) timestamp: NaiveDateTime,
}