feature: introduce /load endpoint for self-reported quantised NymNode load (#5326)
* feature: introduce /load endpoint for self-reported quantised NymNode load * return Load::Unknown for value of 0 because it means we misread some data * add additional filtering on 'en...' endpoints
This commit is contained in:
committed by
GitHub
parent
dcfb092758
commit
03c4895f2b
@@ -72,7 +72,7 @@ pub(crate) fn get_system_info(
|
||||
system_name: System::name(),
|
||||
kernel_version: System::kernel_version(),
|
||||
os_version: System::os_version(),
|
||||
cpu_arch: System::cpu_arch(),
|
||||
cpu_arch: Some(System::cpu_arch()),
|
||||
hardware,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,8 +14,8 @@ use nym_node_requests::api::v1::health::models::NodeHealth;
|
||||
tag = "Health",
|
||||
responses(
|
||||
(status = 200, content(
|
||||
(Vec<NodeHealth> = "application/json"),
|
||||
(Vec<NodeHealth> = "application/yaml")
|
||||
(NodeHealth = "application/json"),
|
||||
(NodeHealth = "application/yaml")
|
||||
), description = "the api is available and healthy")
|
||||
),
|
||||
params(OutputParams)
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
// Copyright 2025 - Nym Technologies SA <contact@nymtech.net>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
use crate::node::http::state::AppState;
|
||||
use axum::extract::{Query, State};
|
||||
use nym_http_api_common::{FormattedResponse, OutputParams};
|
||||
use nym_node_requests::api::v1::node_load::models::NodeLoad;
|
||||
|
||||
/// Returns current relative load this node.
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/load",
|
||||
context_path = "/api/v1",
|
||||
tag = "Node",
|
||||
responses(
|
||||
(status = 200, content(
|
||||
(NodeLoad = "application/json"),
|
||||
(NodeLoad = "application/yaml")
|
||||
), description = "current node load")
|
||||
),
|
||||
params(OutputParams)
|
||||
)]
|
||||
pub(crate) async fn root_load(
|
||||
Query(output): Query<OutputParams>,
|
||||
State(state): State<AppState>,
|
||||
) -> NodeLoadResponse {
|
||||
let output = output.output.unwrap_or_default();
|
||||
let load = state.cached_load.current_load();
|
||||
|
||||
output.to_response(load)
|
||||
}
|
||||
|
||||
pub type NodeLoadResponse = FormattedResponse<NodeLoad>;
|
||||
@@ -10,6 +10,7 @@ pub mod authenticator;
|
||||
pub mod gateway;
|
||||
pub mod health;
|
||||
pub mod ip_packet_router;
|
||||
pub mod load;
|
||||
pub mod metrics;
|
||||
pub mod mixnode;
|
||||
pub mod network_requester;
|
||||
@@ -30,6 +31,7 @@ pub struct Config {
|
||||
pub(super) fn routes(config: Config) -> Router<AppState> {
|
||||
Router::new()
|
||||
.route(v1::HEALTH, get(health::root_health))
|
||||
.route(v1::LOAD, get(load::root_load))
|
||||
.nest(v1::METRICS, metrics::routes(config.metrics))
|
||||
.nest(v1::GATEWAY, gateway::routes(config.gateway))
|
||||
.nest(v1::MIXNODE, mixnode::routes(config.mixnode))
|
||||
|
||||
@@ -24,6 +24,7 @@ use utoipa_swagger_ui::SwaggerUi;
|
||||
api::v1::metrics::verloc::verloc_stats,
|
||||
api::v1::metrics::prometheus::prometheus_metrics,
|
||||
api::v1::health::root_health,
|
||||
api::v1::load::root_load,
|
||||
api::v1::gateway::root::root_gateway,
|
||||
api::v1::gateway::client_interfaces::client_interfaces,
|
||||
api::v1::gateway::client_interfaces::mixnet_websockets,
|
||||
@@ -38,6 +39,8 @@ use utoipa_swagger_ui::SwaggerUi;
|
||||
nym_http_api_common::OutputParams,
|
||||
api_requests::v1::health::models::NodeHealth,
|
||||
api_requests::v1::health::models::NodeStatus,
|
||||
api_requests::v1::node_load::models::NodeLoad,
|
||||
api_requests::v1::node_load::models::Load,
|
||||
api_requests::v1::node::models::BinaryBuildInformationOwned,
|
||||
api_requests::v1::node::models::HostInformation,
|
||||
api_requests::v1::node::models::HostKeys,
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
// Copyright 2025 - Nym Technologies SA <contact@nymtech.net>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use nym_node_requests::api::v1::node_load::models::{Load, NodeLoad};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use sysinfo::{CpuRefreshKind, MemoryRefreshKind, Networks, System};
|
||||
use time::OffsetDateTime;
|
||||
use tracing::debug;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct CachedNodeLoad {
|
||||
ttl: Duration,
|
||||
being_updated: Arc<AtomicBool>,
|
||||
inner: Arc<ArcSwap<CachedNodeLoadInner>>,
|
||||
}
|
||||
|
||||
impl CachedNodeLoad {
|
||||
pub(crate) fn new(ttl: Duration) -> Self {
|
||||
CachedNodeLoad {
|
||||
ttl,
|
||||
being_updated: Arc::new(Default::default()),
|
||||
inner: Arc::new(ArcSwap::new(Arc::new(CachedNodeLoadInner::initial()))),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn current_load(&self) -> NodeLoad {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
|
||||
let inner = self.inner.load();
|
||||
if inner.timestamp + self.ttl < now {
|
||||
// new
|
||||
let already_being_updated = self.being_updated.swap(true, Ordering::SeqCst);
|
||||
if already_being_updated {
|
||||
// use the 'stale' entry because it is already being updated by another thread
|
||||
inner.load
|
||||
} else {
|
||||
self.update_cache()
|
||||
}
|
||||
} else {
|
||||
inner.load
|
||||
}
|
||||
}
|
||||
|
||||
fn update_cache(&self) -> NodeLoad {
|
||||
let old = self.inner.load();
|
||||
let current_load = CachedNodeLoadInner::update(&old);
|
||||
let load = current_load.load;
|
||||
self.inner.store(Arc::new(current_load));
|
||||
self.being_updated.store(false, Ordering::SeqCst);
|
||||
load
|
||||
}
|
||||
}
|
||||
|
||||
pub struct CachedNodeLoadInner {
|
||||
timestamp: OffsetDateTime,
|
||||
last_transmitted: u64,
|
||||
last_received: u64,
|
||||
load: NodeLoad,
|
||||
}
|
||||
|
||||
struct RawUsage {
|
||||
load_per_cpu: f64,
|
||||
memory_usage: f64,
|
||||
swap_usage: f64,
|
||||
eth_transmitted: u64,
|
||||
eth_received: u64,
|
||||
total_swap: u64,
|
||||
}
|
||||
|
||||
impl RawUsage {
|
||||
fn current() -> RawUsage {
|
||||
let mut system = sysinfo::System::new();
|
||||
let networks = Networks::new_with_refreshed_list();
|
||||
|
||||
system.refresh_memory_specifics(MemoryRefreshKind::everything());
|
||||
system.refresh_cpu_specifics(CpuRefreshKind::nothing());
|
||||
|
||||
let average_load = System::load_average();
|
||||
let cpu_count = system.cpus().len();
|
||||
|
||||
let load_per_cpu = average_load.five / cpu_count as f64;
|
||||
|
||||
let total_memory = system.total_memory();
|
||||
let used_memory = system.used_memory();
|
||||
|
||||
let memory_usage = used_memory as f64 / total_memory as f64;
|
||||
|
||||
let total_swap = system.total_swap();
|
||||
let used_swap = system.free_swap();
|
||||
|
||||
let swap_usage = used_swap as f64 / total_swap as f64;
|
||||
|
||||
let mut eth_transmitted = 0;
|
||||
let mut eth_received = 0;
|
||||
|
||||
// we're only interested in interfaces with 'eth' or 'en' prefix
|
||||
// (that's a very weak assumption, but that's just first iteration of this endpoint)
|
||||
for (interface, data) in networks.list() {
|
||||
if interface.starts_with("eth") || interface.starts_with("en") {
|
||||
eth_transmitted += data.total_transmitted();
|
||||
eth_received += data.total_received();
|
||||
}
|
||||
}
|
||||
|
||||
debug!(average_load = ?average_load, memory_usage=memory_usage, swap_usage = swap_usage, "current load");
|
||||
|
||||
RawUsage {
|
||||
load_per_cpu,
|
||||
memory_usage,
|
||||
swap_usage,
|
||||
eth_transmitted,
|
||||
eth_received,
|
||||
total_swap,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CachedNodeLoadInner {
|
||||
pub fn initial() -> CachedNodeLoadInner {
|
||||
let timestamp = OffsetDateTime::now_utc();
|
||||
let raw_usage = RawUsage::current();
|
||||
|
||||
let mut base_load = Load::from(raw_usage.load_per_cpu);
|
||||
let memory_load = Load::from(raw_usage.memory_usage);
|
||||
|
||||
// if memory load is of higher tier, increment the base load by one level
|
||||
// (i.e. for example from 'Low' to 'Medium')
|
||||
if memory_load > base_load {
|
||||
base_load = base_load.increment();
|
||||
}
|
||||
|
||||
if raw_usage.total_swap > 1024 * 1024 * 1024 {
|
||||
// same with swap
|
||||
let swap_load = Load::from(raw_usage.swap_usage);
|
||||
if swap_load > base_load {
|
||||
base_load = base_load.increment();
|
||||
}
|
||||
}
|
||||
|
||||
CachedNodeLoadInner {
|
||||
timestamp,
|
||||
last_transmitted: raw_usage.eth_transmitted,
|
||||
last_received: raw_usage.eth_received,
|
||||
load: NodeLoad {
|
||||
total: base_load,
|
||||
machine: base_load,
|
||||
network: Load::Unknown,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn update(previous: &Self) -> CachedNodeLoadInner {
|
||||
let timestamp = OffsetDateTime::now_utc();
|
||||
let raw_usage = RawUsage::current();
|
||||
|
||||
let time_delta = (timestamp - previous.timestamp).as_seconds_f64();
|
||||
let tx_delta = raw_usage.eth_transmitted - previous.last_transmitted;
|
||||
let rx_delta = raw_usage.eth_received - previous.last_received;
|
||||
|
||||
let tx_bs = tx_delta as f64 / time_delta;
|
||||
let rx_bs = rx_delta as f64 / time_delta;
|
||||
|
||||
// currently we consider value of 1Gbps to be maximum load
|
||||
// in the future we should allow specifying custom sizes of network cards
|
||||
// (Gbps = Bytes/s * 0.000000008)
|
||||
let tx_gbps = tx_bs * 0.000000008;
|
||||
let rx_gbps = rx_bs * 0.000000008;
|
||||
let network_load = Load::from(tx_gbps.max(rx_gbps));
|
||||
|
||||
debug!(tx_gbps = tx_gbps, rx_gbps = rx_gbps, "network load");
|
||||
|
||||
let mut base_load = Load::from(raw_usage.load_per_cpu);
|
||||
let memory_load = Load::from(raw_usage.memory_usage);
|
||||
|
||||
// if memory load is of higher tier, increment the base load by one level
|
||||
// (i.e. for example from 'Low' to 'Medium')
|
||||
if memory_load > base_load {
|
||||
base_load = base_load.increment();
|
||||
}
|
||||
|
||||
if raw_usage.total_swap > 1024 * 1024 * 1024 {
|
||||
// same with swap
|
||||
let swap_load = Load::from(raw_usage.swap_usage);
|
||||
if swap_load > base_load {
|
||||
base_load = base_load.increment();
|
||||
}
|
||||
}
|
||||
|
||||
let total_load = if base_load > network_load {
|
||||
base_load
|
||||
} else {
|
||||
base_load.increment()
|
||||
};
|
||||
|
||||
CachedNodeLoadInner {
|
||||
timestamp,
|
||||
last_transmitted: raw_usage.eth_transmitted,
|
||||
last_received: raw_usage.eth_received,
|
||||
load: NodeLoad {
|
||||
total: total_load,
|
||||
machine: base_load,
|
||||
network: network_load,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,29 +1,39 @@
|
||||
// Copyright 2023-2024 - Nym Technologies SA <contact@nymtech.net>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
use crate::node::http::state::load::CachedNodeLoad;
|
||||
use crate::node::http::state::metrics::MetricsAppState;
|
||||
use nym_node_metrics::NymNodeMetrics;
|
||||
use nym_verloc::measurements::SharedVerlocStats;
|
||||
use std::time::Duration;
|
||||
use tokio::time::Instant;
|
||||
|
||||
pub mod load;
|
||||
pub mod metrics;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct AppState {
|
||||
pub(crate) startup_time: Instant,
|
||||
|
||||
pub(crate) cached_load: CachedNodeLoad,
|
||||
|
||||
pub(crate) metrics: MetricsAppState,
|
||||
}
|
||||
|
||||
impl AppState {
|
||||
#[allow(clippy::new_without_default)]
|
||||
pub fn new(metrics: NymNodeMetrics, verloc: SharedVerlocStats) -> Self {
|
||||
pub fn new(
|
||||
metrics: NymNodeMetrics,
|
||||
verloc: SharedVerlocStats,
|
||||
load_cache_ttl: Duration,
|
||||
) -> Self {
|
||||
AppState {
|
||||
// is it 100% accurate?
|
||||
// no.
|
||||
// does it have to be?
|
||||
// also no.
|
||||
startup_time: Instant::now(),
|
||||
cached_load: CachedNodeLoad::new(load_cache_ttl),
|
||||
metrics: MetricsAppState { metrics, verloc },
|
||||
}
|
||||
}
|
||||
|
||||
@@ -785,7 +785,11 @@ impl NymNode {
|
||||
config.api.v1_config.node.roles.ip_packet_router_enabled = true;
|
||||
}
|
||||
|
||||
let app_state = AppState::new(self.metrics.clone(), self.verloc_stats.clone());
|
||||
let app_state = AppState::new(
|
||||
self.metrics.clone(),
|
||||
self.verloc_stats.clone(),
|
||||
self.config.http.node_load_cache_ttl,
|
||||
);
|
||||
|
||||
Ok(NymNodeRouter::new(config, app_state)
|
||||
.build_server(&self.config.http.bind_address)
|
||||
|
||||
Reference in New Issue
Block a user