[Product Data] First step in gateway usage data collection (#4963)

* add stats model

* add stats collection

* add stats route

* propagate stuff and run stuff

* cargo stuff

* sqlx unused what?

* add sessions started stat

* session durations in miliseconds

* apply Jon's comments

* [Product Data] Second step in gateway usage data collection  (#4964)

* turn stats collection into event based

* move events into a common crate for future use elsewhere

* apply Jon's comments
This commit is contained in:
Simon Wicky
2024-10-15 09:18:02 +02:00
committed by GitHub
parent 1fc7e07028
commit 435f236812
17 changed files with 394 additions and 13 deletions
Generated
+11
View File
@@ -5219,6 +5219,7 @@ dependencies = [
"nym-node-http-api",
"nym-pemstore",
"nym-sphinx",
"nym-statistics-common",
"nym-task",
"nym-types",
"nym-validator-client",
@@ -5232,6 +5233,7 @@ dependencies = [
"sqlx",
"subtle-encoding",
"thiserror",
"time",
"tokio",
"tokio-stream",
"tokio-tungstenite",
@@ -6283,6 +6285,15 @@ dependencies = [
"thiserror",
]
[[package]]
name = "nym-statistics-common"
version = "0.1.0"
dependencies = [
"futures",
"nym-sphinx",
"time",
]
[[package]]
name = "nym-store-cipher"
version = "0.1.0"
+1
View File
@@ -85,6 +85,7 @@ members = [
"common/socks5-client-core",
"common/socks5/proxy-helpers",
"common/socks5/requests",
"common/statistics",
"common/store-cipher",
"common/task",
"common/topology",
+16
View File
@@ -0,0 +1,16 @@
# Copyright 2024 - Nym Technologies SA <contact@nymtech.net>
# SPDX-License-Identifier: Apache-2.0
[package]
name = "nym-statistics-common"
version = "0.1.0"
edition.workspace = true
license.workspace = true
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
futures = { workspace = true }
time = { workspace = true }
nym-sphinx = { path = "../nymsphinx" }
+39
View File
@@ -0,0 +1,39 @@
// Copyright 2024 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: GPL-3.0-only
use futures::channel::mpsc;
use nym_sphinx::DestinationAddressBytes;
use time::OffsetDateTime;
pub type StatsEventSender = mpsc::UnboundedSender<StatsEvent>;
pub type StatsEventReceiver = mpsc::UnboundedReceiver<StatsEvent>;
pub enum StatsEvent {
SessionStatsEvent(SessionEvent),
}
impl StatsEvent {
pub fn new_session_start(client: DestinationAddressBytes) -> StatsEvent {
StatsEvent::SessionStatsEvent(SessionEvent::SessionStart {
start_time: OffsetDateTime::now_utc(),
client,
})
}
pub fn new_session_stop(client: DestinationAddressBytes) -> StatsEvent {
StatsEvent::SessionStatsEvent(SessionEvent::SessionStop {
stop_time: OffsetDateTime::now_utc(),
client,
})
}
}
pub enum SessionEvent {
SessionStart {
start_time: OffsetDateTime,
client: DestinationAddressBytes,
},
SessionStop {
stop_time: OffsetDateTime,
client: DestinationAddressBytes,
},
}
+4
View File
@@ -0,0 +1,4 @@
// Copyright 2024 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: GPL-3.0-only
pub mod events;
+2 -7
View File
@@ -41,15 +41,9 @@ rand = { workspace = true }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
si-scale = { workspace = true }
sqlx = { workspace = true, features = [
"runtime-tokio-rustls",
"sqlite",
"macros",
"migrate",
"time",
] }
subtle-encoding = { workspace = true, features = ["bech32-preview"] }
thiserror = { workspace = true }
time = { workspace = true }
tokio = { workspace = true, features = [
"rt-multi-thread",
"net",
@@ -83,6 +77,7 @@ nym-network-requester = { path = "../service-providers/network-requester" }
nym-node-http-api = { path = "../nym-node/nym-node-http-api" }
nym-pemstore = { path = "../common/pemstore" }
nym-sphinx = { path = "../common/nymsphinx" }
nym-statistics-common = { path = "../common/statistics" }
nym-task = { path = "../common/task" }
nym-types = { path = "../common/types" }
nym-validator-client = { path = "../common/client-libs/validator-client" }
@@ -5,6 +5,8 @@ use super::websocket::message_receiver::{IsActiveRequestSender, MixMessageSender
use crate::node::client_handling::embedded_clients::LocalEmbeddedClientHandle;
use dashmap::DashMap;
use nym_sphinx::DestinationAddressBytes;
use nym_statistics_common::events;
use nym_statistics_common::events::StatsEventSender;
use std::sync::Arc;
use tracing::warn;
@@ -35,6 +37,7 @@ impl ActiveClient {
#[derive(Clone)]
pub(crate) struct ActiveClientsStore {
inner: Arc<DashMap<DestinationAddressBytes, ActiveClient>>,
stats_event_sender: StatsEventSender,
}
#[derive(Clone)]
@@ -48,9 +51,10 @@ pub(crate) struct ClientIncomingChannels {
impl ActiveClientsStore {
/// Creates new instance of `ActiveClientsStore` to store in-memory handles to all currently connected clients.
pub(crate) fn new() -> Self {
pub(crate) fn new(stats_event_sender: StatsEventSender) -> Self {
ActiveClientsStore {
inner: Arc::new(DashMap::new()),
stats_event_sender,
}
}
@@ -126,6 +130,12 @@ impl ActiveClientsStore {
/// * `client`: address of the client for which to remove the handle.
pub(crate) fn disconnect(&self, client: DestinationAddressBytes) {
self.inner.remove(&client);
if let Err(e) = self
.stats_event_sender
.unbounded_send(events::StatsEvent::new_session_stop(client))
{
warn!("Failed to send session stop event to collector : {e}")
};
}
/// Insert new client handle into the store.
@@ -147,6 +157,12 @@ impl ActiveClientsStore {
if self.inner.insert(client, entry).is_some() {
panic!("inserted a duplicate remote client")
}
if let Err(e) = self
.stats_event_sender
.unbounded_send(events::StatsEvent::new_session_start(client))
{
warn!("Failed to send session start event to collector : {e}")
};
}
/// Inserts a handle to the embedded client
+31 -1
View File
@@ -22,12 +22,15 @@ use nym_crypto::asymmetric::{encryption, identity};
use nym_mixnet_client::forwarder::{MixForwardingSender, PacketForwarder};
use nym_network_defaults::NymNetworkDetails;
use nym_network_requester::{LocalGateway, NRServiceProviderBuilder, RequestFilter};
use nym_node_http_api::state::metrics::SharedSessionStats;
use nym_statistics_common::events;
use nym_task::{TaskClient, TaskHandle, TaskManager};
use nym_types::gateway::GatewayNodeDetailsResponse;
use nym_validator_client::nyxd::{Coin, CosmWasmClient};
use nym_validator_client::{nyxd, DirectSigningHttpRpcNyxdClient};
use rand::seq::SliceRandom;
use rand::thread_rng;
use statistics::GatewayStatisticsCollector;
use std::net::SocketAddr;
use std::path::PathBuf;
use std::sync::Arc;
@@ -36,6 +39,7 @@ use tracing::*;
pub(crate) mod client_handling;
pub(crate) mod helpers;
pub(crate) mod mixnet_handling;
pub(crate) mod statistics;
pub use nym_gateway_storage::{PersistentStorage, Storage};
@@ -147,6 +151,8 @@ pub struct Gateway<St = PersistentStorage> {
wireguard_data: Option<nym_wireguard::WireguardData>,
session_stats: Option<SharedSessionStats>,
run_http_server: bool,
task_client: Option<TaskClient>,
}
@@ -168,6 +174,7 @@ impl<St> Gateway<St> {
ip_packet_router_opts,
authenticator_opts: None,
wireguard_data: None,
session_stats: None,
run_http_server: true,
task_client: None,
})
@@ -191,6 +198,7 @@ impl<St> Gateway<St> {
sphinx_keypair,
storage,
wireguard_data: None,
session_stats: None,
run_http_server: true,
task_client: None,
}
@@ -204,6 +212,10 @@ impl<St> Gateway<St> {
self.task_client = Some(task_client)
}
pub fn set_session_stats(&mut self, session_stats: SharedSessionStats) {
self.session_stats = Some(session_stats);
}
pub fn set_wireguard_data(&mut self, wireguard_data: nym_wireguard::WireguardData) {
self.wireguard_data = Some(wireguard_data)
}
@@ -393,6 +405,19 @@ impl<St> Gateway<St> {
packet_sender
}
fn start_stats_collector(
&self,
shared_session_stats: SharedSessionStats,
shutdown: TaskClient,
) -> events::StatsEventSender {
info!("Starting gateway stats collector...");
let (mut stats_collector, stats_event_sender) =
GatewayStatisticsCollector::new(shared_session_stats);
tokio::spawn(async move { stats_collector.run(shutdown).await });
stats_event_sender
}
// TODO: rethink the logic in this function...
async fn start_network_requester(
&self,
@@ -599,6 +624,11 @@ impl<St> Gateway<St> {
return Err(GatewayError::InsufficientNodeBalance { account, balance });
}
}
let shared_session_stats = self.session_stats.take().unwrap_or_default();
let stats_event_sender = self.start_stats_collector(
shared_session_stats,
shutdown.fork("statistics::GatewayStatisticsCollector"),
);
let handler_config = CredentialHandlerConfig {
revocation_bandwidth_penalty: self
@@ -629,7 +659,7 @@ impl<St> Gateway<St> {
let mix_forwarding_channel = self.start_packet_forwarder(shutdown.fork("PacketForwarder"));
let active_clients_store = ActiveClientsStore::new();
let active_clients_store = ActiveClientsStore::new(stats_event_sender.clone());
self.start_mix_socket_listener(
mix_forwarding_channel.clone(),
active_clients_store.clone(),
+63
View File
@@ -0,0 +1,63 @@
// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: GPL-3.0-only
use futures::{channel::mpsc, StreamExt};
use nym_node_http_api::state::metrics::SharedSessionStats;
use nym_statistics_common::events::{StatsEvent, StatsEventReceiver, StatsEventSender};
use nym_task::TaskClient;
use sessions::SessionStatsHandler;
use std::time::Duration;
use time::OffsetDateTime;
use tracing::trace;
pub mod sessions;
const STATISTICS_UPDATE_TIMER_INTERVAL: Duration = Duration::from_secs(3600); //update timer, no need to check everytime
pub(crate) struct GatewayStatisticsCollector {
stats_event_rx: StatsEventReceiver,
session_stats: SessionStatsHandler,
//here goes additionnal stats handler
}
impl GatewayStatisticsCollector {
pub fn new(
shared_session_stats: SharedSessionStats,
) -> (GatewayStatisticsCollector, StatsEventSender) {
let (stats_event_tx, stats_event_rx) = mpsc::unbounded();
let collector = GatewayStatisticsCollector {
stats_event_rx,
session_stats: SessionStatsHandler::new(shared_session_stats),
};
(collector, stats_event_tx)
}
async fn update_shared_state(&mut self, update_time: OffsetDateTime) {
self.session_stats.update_shared_state(update_time).await;
//here goes additionnal stats handler update
}
pub async fn run(&mut self, mut shutdown: TaskClient) {
let mut update_interval = tokio::time::interval(STATISTICS_UPDATE_TIMER_INTERVAL);
while !shutdown.is_shutdown() {
tokio::select! {
biased;
_ = shutdown.recv() => {
trace!("StatisticsCollector: Received shutdown");
},
_ = update_interval.tick() => {
let now = OffsetDateTime::now_utc();
self.update_shared_state(now).await;
},
Some(stat_event) = self.stats_event_rx.next() => {
//dispatching event to proper handler
match stat_event {
StatsEvent::SessionStatsEvent(event) => self.session_stats.handle_event(event),
}
},
}
}
}
}
+86
View File
@@ -0,0 +1,86 @@
// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: GPL-3.0-only
use nym_node_http_api::state::metrics::SharedSessionStats;
use nym_sphinx::DestinationAddressBytes;
use std::collections::{HashMap, HashSet};
use time::{Date, OffsetDateTime};
use nym_statistics_common::events::SessionEvent;
type SessionDuration = u64; //in miliseconds
pub(crate) struct SessionStatsHandler {
last_update_day: Date,
shared_session_stats: SharedSessionStats,
active_sessions: HashMap<DestinationAddressBytes, OffsetDateTime>,
unique_users: HashSet<DestinationAddressBytes>,
sessions_started: u32,
finished_sessions: Vec<SessionDuration>,
}
impl SessionStatsHandler {
pub fn new(shared_session_stats: SharedSessionStats) -> Self {
SessionStatsHandler {
last_update_day: OffsetDateTime::now_utc().date(),
shared_session_stats,
active_sessions: Default::default(),
unique_users: Default::default(),
sessions_started: 0,
finished_sessions: Default::default(),
}
}
pub(crate) fn handle_event(&mut self, event: SessionEvent) {
match event {
SessionEvent::SessionStart { start_time, client } => {
self.handle_session_start(start_time, client);
}
SessionEvent::SessionStop { stop_time, client } => {
self.handle_session_stop(stop_time, client);
}
}
}
fn handle_session_start(
&mut self,
start_time: OffsetDateTime,
client: DestinationAddressBytes,
) {
self.sessions_started += 1;
self.unique_users.insert(client);
self.active_sessions.insert(client, start_time);
}
fn handle_session_stop(&mut self, stop_time: OffsetDateTime, client: DestinationAddressBytes) {
if let Some(session_start) = self.active_sessions.remove(&client) {
let session_duration = (stop_time - session_start).whole_milliseconds();
//this should always happen because it should always be positive and u64::max milliseconds is 500k millenia, but anyway
if let Ok(duration_u64) = session_duration.try_into() {
self.finished_sessions.push(duration_u64);
}
}
}
//update shared state once a day has passed, with data from the previous day
pub(crate) async fn update_shared_state(&mut self, update_time: OffsetDateTime) {
let update_date = update_time.date();
if update_date != self.last_update_day {
{
let mut shared_state = self.shared_session_stats.write().await;
shared_state.update_time = self.last_update_day;
shared_state.unique_active_users = self.unique_users.len() as u32;
shared_state.session_started = self.sessions_started;
shared_state.session_durations = self.finished_sessions.clone();
}
self.reset_stats(update_date);
}
}
fn reset_stats(&mut self, reset_day: Date) {
self.last_update_day = reset_day;
self.unique_users = self.active_sessions.keys().copied().collect();
self.finished_sessions = Default::default();
self.sessions_started = 0;
}
}
@@ -3,6 +3,7 @@
use crate::api::v1::metrics::mixing::mixing_stats;
use crate::api::v1::metrics::prometheus::prometheus_metrics;
use crate::api::v1::metrics::sessions::sessions_stats;
use crate::api::v1::metrics::verloc::verloc_stats;
use crate::state::metrics::MetricsAppState;
use axum::extract::FromRef;
@@ -12,6 +13,7 @@ use nym_node_requests::routes::api::v1::metrics;
pub mod mixing;
pub mod prometheus;
pub mod sessions;
pub mod verloc;
#[derive(Debug, Clone, Default)]
@@ -26,6 +28,7 @@ where
{
Router::new()
.route(metrics::MIXING, get(mixing_stats))
.route(metrics::SESSIONS, get(sessions_stats))
.route(metrics::VERLOC, get(verloc_stats))
.route(metrics::PROMETHEUS, get(prometheus_metrics))
}
@@ -0,0 +1,33 @@
// Copyright 2024 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: GPL-3.0-only
use crate::state::metrics::MetricsAppState;
use axum::extract::{Query, State};
use nym_http_api_common::{FormattedResponse, OutputParams};
use nym_node_requests::api::v1::metrics::models::SessionStats;
/// If applicable, returns sessions statistics information of this node.
/// This information is **PURELY** self-reported and in no way validated.
#[utoipa::path(
get,
path = "/sessions",
context_path = "/api/v1/metrics",
tag = "Metrics",
responses(
(status = 200, content(
("application/json" = SessionStats),
("application/yaml" = SessionStats)
))
),
params(OutputParams),
)]
pub(crate) async fn sessions_stats(
Query(output): Query<OutputParams>,
State(metrics_state): State<MetricsAppState>,
) -> SessionStatsResponse {
let output = output.output.unwrap_or_default();
let response = metrics_state.session_stats.read().await.as_response();
output.to_response(response)
}
pub type SessionStatsResponse = FormattedResponse<SessionStats>;
@@ -4,11 +4,12 @@
use crate::state::AppState;
use axum::extract::FromRef;
use nym_node_requests::api::v1::metrics::models::{
MixingStats, VerlocResult, VerlocResultData, VerlocStats,
MixingStats, SessionStats, VerlocResult, VerlocResultData, VerlocStats,
};
use std::collections::HashMap;
use std::sync::Arc;
use time::OffsetDateTime;
use time::macros::time;
use time::{Date, OffsetDateTime};
use tokio::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard};
pub use nym_node_requests::api::v1::metrics::models::{VerlocMeasurement, VerlocNodeResult};
@@ -132,12 +133,66 @@ impl VerlocStatsState {
}
}
#[derive(Clone, Debug, Default)]
pub struct SharedSessionStats {
inner: Arc<RwLock<SessionStatsState>>,
}
impl SharedSessionStats {
pub fn new() -> SharedSessionStats {
SharedSessionStats {
inner: Arc::new(RwLock::new(Default::default())),
}
}
pub async fn read(&self) -> RwLockReadGuard<'_, SessionStatsState> {
self.inner.read().await
}
pub async fn write(&self) -> RwLockWriteGuard<'_, SessionStatsState> {
self.inner.write().await
}
}
#[derive(Debug, Clone)]
pub struct SessionStatsState {
pub update_time: Date,
pub unique_active_users: u32,
pub session_started: u32,
pub session_durations: Vec<u64>,
}
impl SessionStatsState {
pub fn as_response(&self) -> SessionStats {
SessionStats {
update_time: self.update_time.with_time(time!(0:00)).assume_utc(),
unique_active_users: self.unique_active_users,
session_durations: self.session_durations.clone(),
sessions_started: self.session_started,
sessions_finished: self.session_durations.len() as u32,
}
}
}
impl Default for SessionStatsState {
fn default() -> Self {
SessionStatsState {
update_time: OffsetDateTime::UNIX_EPOCH.date(),
unique_active_users: 0,
session_started: 0,
session_durations: Default::default(),
}
}
}
#[derive(Debug, Clone, Default)]
pub struct MetricsAppState {
pub(crate) prometheus_access_token: Option<String>,
pub(crate) mixing_stats: SharedMixingStats,
pub(crate) session_stats: SharedSessionStats,
pub(crate) verloc: SharedVerlocStats,
}
+9 -1
View File
@@ -1,7 +1,9 @@
// Copyright 2023-2024 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: GPL-3.0-only
use crate::state::metrics::{MetricsAppState, SharedMixingStats, SharedVerlocStats};
use crate::state::metrics::{
MetricsAppState, SharedMixingStats, SharedSessionStats, SharedVerlocStats,
};
use tokio::time::Instant;
pub mod metrics;
@@ -32,6 +34,12 @@ impl AppState {
self
}
#[must_use]
pub fn with_sessions_stats(mut self, session_stats: SharedSessionStats) -> Self {
self.metrics.session_stats = session_stats;
self
}
#[must_use]
pub fn with_verloc_stats(mut self, verloc_stats: SharedVerlocStats) -> Self {
self.metrics.verloc = verloc_stats;
@@ -35,6 +35,21 @@ pub struct MixingStats {
pub dropped_since_last_update: u64,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
pub struct SessionStats {
#[serde(with = "time::serde::rfc3339")]
pub update_time: OffsetDateTime,
pub unique_active_users: u32,
pub session_durations: Vec<u64>,
pub sessions_started: u32,
pub sessions_finished: u32,
}
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
pub struct VerlocStats {
+2
View File
@@ -65,10 +65,12 @@ pub mod routes {
use super::*;
pub const MIXING: &str = "/mixing";
pub const SESSIONS: &str = "/sessions";
pub const VERLOC: &str = "/verloc";
pub const PROMETHEUS: &str = "/prometheus";
absolute_route!(mixing_absolute, metrics_absolute(), MIXING);
absolute_route!(sessions_absolute, metrics_absolute(), SESSIONS);
absolute_route!(verloc_absolute, metrics_absolute(), VERLOC);
absolute_route!(prometheus_absolute, metrics_absolute(), PROMETHEUS);
}
+5 -1
View File
@@ -26,7 +26,7 @@ use nym_node::config::{
use nym_node::error::{EntryGatewayError, ExitGatewayError, MixnodeError, NymNodeError};
use nym_node_http_api::api::api_requests;
use nym_node_http_api::api::api_requests::v1::node::models::{AnnouncePorts, NodeDescription};
use nym_node_http_api::state::metrics::{SharedMixingStats, SharedVerlocStats};
use nym_node_http_api::state::metrics::{SharedMixingStats, SharedSessionStats, SharedVerlocStats};
use nym_node_http_api::state::AppState;
use nym_node_http_api::{NymNodeHTTPServer, NymNodeRouter};
use nym_sphinx_acknowledgements::AckKey;
@@ -67,6 +67,7 @@ impl MixnodeData {
pub struct EntryGatewayData {
mnemonic: Zeroizing<bip39::Mnemonic>,
client_storage: nym_gateway::node::PersistentStorage,
sessions_stats: SharedSessionStats,
}
impl EntryGatewayData {
@@ -93,6 +94,7 @@ impl EntryGatewayData {
)
.await
.map_err(nym_gateway::GatewayError::from)?,
sessions_stats: SharedSessionStats::new(),
})
}
}
@@ -581,6 +583,7 @@ impl NymNode {
);
entry_gateway.disable_http_server();
entry_gateway.set_task_client(task_client);
entry_gateway.set_session_stats(self.entry_gateway.sessions_stats.clone());
if self.config.wireguard.enabled {
entry_gateway.set_wireguard_data(self.wireguard.into());
}
@@ -728,6 +731,7 @@ impl NymNode {
let app_state = AppState::new()
.with_mixing_stats(self.mixnode.mixing_stats.clone())
.with_sessions_stats(self.entry_gateway.sessions_stats.clone())
.with_verloc_stats(self.verloc_stats.clone())
.with_metrics_key(self.config.http.access_token.clone());