GRIN/nym
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Raw route handling and reliability corrections

Browse Source
This commit is contained in:
durch
2025-05-15 13:10:54 +02:00
parent 9c7d79683b
commit fcffebfe45
22 changed files with 1005 additions and 816 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -62,3 +62,4 @@ nym-api/redocly/formatted-openapi.json
**/settings.sql
**/enter_db.sh
CLAUDE.md
Cargo.lock
Generated
+545 -577
View File
File diff suppressed because it is too large Load Diff
common/types/src/monitoring.rs
+28 -7
View File
@@ -11,6 +11,27 @@ static NETWORK_MONITORS: LazyLock<HashSet<String>> = LazyLock::new(|| {
nm
});
#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone, ToSchema)]
pub struct RouteResult {
pub layer1: u32,
pub layer2: u32,
pub layer3: u32,
pub gw: u32,
pub success: bool,
}
impl RouteResult {
pub fn new(layer1: u32, layer2: u32, layer3: u32, gw: u32, success: bool) -> Self {
RouteResult {
layer1,
layer2,
layer3,
gw,
success,
}
}
}
#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone, ToSchema)]
pub struct NodeResult {
#[schema(value_type = u32)]
@@ -29,23 +50,23 @@ impl NodeResult {
}
}
#[derive(Serialize, Deserialize, JsonSchema)]
#[derive(Serialize, Deserialize, JsonSchema, ToSchema)]
#[serde(untagged)]
pub enum MonitorResults {
Mixnode(Vec<NodeResult>),
Gateway(Vec<NodeResult>),
Node(Vec<NodeResult>),
Route(Vec<RouteResult>),
}
#[derive(Serialize, Deserialize, JsonSchema, ToSchema)]
pub struct MonitorMessage {
results: Vec<NodeResult>,
results: MonitorResults,
signature: String,
signer: String,
timestamp: i64,
}
impl MonitorMessage {
fn message_to_sign(results: &[NodeResult], timestamp: i64) -> Vec<u8> {
fn message_to_sign(results: &MonitorResults, timestamp: i64) -> Vec<u8> {
let mut msg = serde_json::to_vec(results).unwrap_or_default();
msg.extend_from_slice(&timestamp.to_le_bytes());
msg
@@ -60,7 +81,7 @@ impl MonitorMessage {
now - self.timestamp < 5
}
pub fn new(results: Vec<NodeResult>, private_key: &PrivateKey) -> Self {
pub fn new(results: MonitorResults, private_key: &PrivateKey) -> Self {
let timestamp = SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.expect("Time went backwards")
@@ -82,7 +103,7 @@ impl MonitorMessage {
NETWORK_MONITORS.contains(&self.signer)
}
pub fn results(&self) -> &[NodeResult] {
pub fn results(&self) -> &MonitorResults {
&self.results
}
nym-api/.sqlx/query-00d857b624e7edab1198114b17cbad1e16988a3f9989d135840500e1143ce5e5.json
Generated
-32
View File
@@ -1,32 +0,0 @@
{
"db_name": "SQLite",
"query": "\n SELECT epoch_id as \"epoch_id: u32\", serialised_signatures, serialization_revision as \"serialization_revision: u8\"\n FROM expiration_date_signatures\n WHERE expiration_date = ?\n ",
"describe": {
"columns": [
{
"name": "epoch_id: u32",
"ordinal": 0,
"type_info": "Int64"
},
{
"name": "serialised_signatures",
"ordinal": 1,
"type_info": "Blob"
},
{
"name": "serialization_revision: u8",
"ordinal": 2,
"type_info": "Int64"
}
],
"parameters": {
"Right": 1
},
"nullable": [
false,
false,
false
]
},
"hash": "00d857b624e7edab1198114b17cbad1e16988a3f9989d135840500e1143ce5e5"
}
nym-api/.sqlx/query-0112296b190328a3856d1adf51aafa2525da6c0b871633aad80ad555db9cf47c.json
Generated
-32
View File
@@ -1,32 +0,0 @@
{
"db_name": "SQLite",
"query": "\n SELECT epoch_id as \"epoch_id: u32\", serialised_key, serialization_revision as \"serialization_revision: u8\"\n FROM master_verification_key WHERE epoch_id = ?\n ",
"describe": {
"columns": [
{
"name": "epoch_id: u32",
"ordinal": 0,
"type_info": "Int64"
},
{
"name": "serialised_key",
"ordinal": 1,
"type_info": "Blob"
},
{
"name": "serialization_revision: u8",
"ordinal": 2,
"type_info": "Int64"
}
],
"parameters": {
"Right": 1
},
"nullable": [
false,
false,
false
]
},
"hash": "0112296b190328a3856d1adf51aafa2525da6c0b871633aad80ad555db9cf47c"
}
nym-api/.sqlx/query-16d10f0ac0ed9ce4239937f46df3797a6a9ee7db2aab9f1b5e55f7c13c53bcc1.json
Generated
-12
View File
@@ -1,12 +0,0 @@
{
"db_name": "SQLite",
"query": "\n INSERT OR IGNORE INTO expiration_date_signatures(expiration_date, epoch_id, serialised_signatures, serialization_revision)\n VALUES (?, ?, ?, ?);\n UPDATE expiration_date_signatures\n SET\n serialised_signatures = ?,\n serialization_revision = ?\n WHERE expiration_date = ?\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 7
},
"nullable": []
},
"hash": "16d10f0ac0ed9ce4239937f46df3797a6a9ee7db2aab9f1b5e55f7c13c53bcc1"
}
nym-api/.sqlx/query-284b3ceae42f9320c30323dde47765854899103fd3c0fa670eb6809492270e02.json
Generated
-12
View File
@@ -1,12 +0,0 @@
{
"db_name": "SQLite",
"query": "\n INSERT INTO ecash_ticketbook\n (serialization_revision, ticketbook_data, expiration_date, ticketbook_type, epoch_id, total_tickets, used_tickets)\n VALUES (?, ?, ?, ?, ?, ?, ?)\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 7
},
"nullable": []
},
"hash": "284b3ceae42f9320c30323dde47765854899103fd3c0fa670eb6809492270e02"
}
nym-api/.sqlx/query-37f82c9ec26b53d01601a2d6df82038a77ec37cca9f9aef18008dcd03030c2c4.json
Generated
-12
View File
@@ -1,12 +0,0 @@
{
"db_name": "SQLite",
"query": "DELETE FROM ecash_ticketbook WHERE expiration_date <= ?",
"describe": {
"columns": [],
"parameters": {
"Right": 1
},
"nullable": []
},
"hash": "37f82c9ec26b53d01601a2d6df82038a77ec37cca9f9aef18008dcd03030c2c4"
}
nym-api/.sqlx/query-5c5d4bfabf18bc6fa56e76a9b98e38b7f6ceb8e9191a7b9201922efcf6b07966.json
Generated
-12
View File
@@ -1,12 +0,0 @@
{
"db_name": "SQLite",
"query": "DELETE FROM pending_issuance WHERE deposit_id = ?",
"describe": {
"columns": [],
"parameters": {
"Right": 1
},
"nullable": []
},
"hash": "5c5d4bfabf18bc6fa56e76a9b98e38b7f6ceb8e9191a7b9201922efcf6b07966"
}
nym-api/.sqlx/query-66109c1d856e1ca2b5126e4bf4c58c7a27b8c303bfa079cf74909354202dcc49.json
Generated
+12
View File
@@ -0,0 +1,12 @@
{
"db_name": "SQLite",
"query": "\n INSERT OR IGNORE INTO routes (layer1, layer2, layer3, gw, success) VALUES (?, ?, ?, ?, ?);\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 5
},
"nullable": []
},
"hash": "66109c1d856e1ca2b5126e4bf4c58c7a27b8c303bfa079cf74909354202dcc49"
}
nym-api/.sqlx/query-81a12a8a419c88b1c28a5533fde4d63462e9ea0049e2edafea1dc3f8476b33e4.json
Generated
-12
View File
@@ -1,12 +0,0 @@
{
"db_name": "SQLite",
"query": "\n INSERT INTO pending_issuance\n (deposit_id, serialization_revision, pending_ticketbook_data, expiration_date)\n VALUES (?, ?, ?, ?)\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 4
},
"nullable": []
},
"hash": "81a12a8a419c88b1c28a5533fde4d63462e9ea0049e2edafea1dc3f8476b33e4"
}
nym-api/.sqlx/query-84cad8b1078a4000830835e6349de3eb76fed954b7336530401db72cd008aff3.json
Generated
-12
View File
@@ -1,12 +0,0 @@
{
"db_name": "SQLite",
"query": "UPDATE ecash_ticketbook SET used_tickets = used_tickets + ? WHERE id = ?",
"describe": {
"columns": [],
"parameters": {
"Right": 2
},
"nullable": []
},
"hash": "84cad8b1078a4000830835e6349de3eb76fed954b7336530401db72cd008aff3"
}
nym-api/.sqlx/query-a5b18e66d77ff802e274623605e15dcfcffb502ba8398caefd56c481f44eb84e.json
Generated
-12
View File
@@ -1,12 +0,0 @@
{
"db_name": "SQLite",
"query": "\n INSERT OR IGNORE INTO master_verification_key(epoch_id, serialised_key, serialization_revision) VALUES (?, ?, ?);\n UPDATE master_verification_key\n SET\n serialised_key = ?,\n serialization_revision = ?\n WHERE epoch_id = ?\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 6
},
"nullable": []
},
"hash": "a5b18e66d77ff802e274623605e15dcfcffb502ba8398caefd56c481f44eb84e"
}
nym-api/.sqlx/query-ba96344db31b0f2155e2af53eaaeafc9b5f64061b6c9a829e2912945b6cffc82.json
Generated
-32
View File
@@ -1,32 +0,0 @@
{
"db_name": "SQLite",
"query": "\n SELECT epoch_id as \"epoch_id: u32\", serialised_signatures, serialization_revision as \"serialization_revision: u8\"\n FROM coin_indices_signatures WHERE epoch_id = ?\n ",
"describe": {
"columns": [
{
"name": "epoch_id: u32",
"ordinal": 0,
"type_info": "Int64"
},
{
"name": "serialised_signatures",
"ordinal": 1,
"type_info": "Blob"
},
{
"name": "serialization_revision: u8",
"ordinal": 2,
"type_info": "Int64"
}
],
"parameters": {
"Right": 1
},
"nullable": [
false,
false,
false
]
},
"hash": "ba96344db31b0f2155e2af53eaaeafc9b5f64061b6c9a829e2912945b6cffc82"
}
nym-api/.sqlx/query-bc823c54143e2dc590b91347cd089dde284b38a3a4960afed758206d03ca1cf4.json
Generated
-12
View File
@@ -1,12 +0,0 @@
{
"db_name": "SQLite",
"query": "\n UPDATE ecash_ticketbook\n SET used_tickets = used_tickets - ?\n WHERE id = ?\n AND used_tickets = ?\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 3
},
"nullable": []
},
"hash": "bc823c54143e2dc590b91347cd089dde284b38a3a4960afed758206d03ca1cf4"
}
nym-api/.sqlx/query-bd1973696121b6128bd75ae80fab253c071e04eb853d4b0f3b21782ea57c2f68.json
Generated
-12
View File
@@ -1,12 +0,0 @@
{
"db_name": "SQLite",
"query": "\n INSERT OR IGNORE INTO coin_indices_signatures(epoch_id, serialised_signatures, serialization_revision) VALUES (?, ?, ?);\n UPDATE coin_indices_signatures\n SET\n serialised_signatures = ?,\n serialization_revision = ?\n WHERE epoch_id = ?\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 6
},
"nullable": []
},
"hash": "bd1973696121b6128bd75ae80fab253c071e04eb853d4b0f3b21782ea57c2f68"
}
nym-api/migrations/20250513104800_routes_table.sql
+16
View File
@@ -0,0 +1,16 @@
-- Drop the table if it exists
DROP TABLE IF EXISTS routes;
-- Create the routes table
CREATE TABLE routes (
id INTEGER PRIMARY KEY AUTOINCREMENT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
layer1 INTEGER,
layer2 INTEGER,
layer3 INTEGER,
gw INTEGER,
success BOOLEAN
);
-- Create an index on created_at
CREATE INDEX routes_created_at ON routes(created_at);
nym-api/src/node_status_api/handlers/without_monitor.rs
+80 -23
View File
@@ -20,7 +20,7 @@ use nym_api_requests::models::{
};
use nym_http_api_common::{FormattedResponse, OutputParams};
use nym_mixnet_contract_common::NodeId;
use nym_types::monitoring::MonitorMessage;
use nym_types::monitoring::{MonitorMessage, MonitorResults};
use tracing::error;
pub(super) fn mandatory_routes() -> Router<AppState> {
@@ -33,6 +33,10 @@ pub(super) fn mandatory_routes() -> Router<AppState> {
"/submit-node-monitoring-results",
post(submit_node_monitoring_results),
)
.route(
"/submit-route-monitoring-results",
post(submit_route_monitoring_results),
)
.nest(
"/mixnode/:mix_id",
Router::new()
@@ -58,6 +62,53 @@ pub(super) fn mandatory_routes() -> Router<AppState> {
)
}
#[utoipa::path(
tag = "status",
post,
path = "/v1/status/submit-route-monitoring-results",
responses(
(status = 200),
(status = 400, body = String, description = "TBD"),
(status = 403, body = String, description = "TBD"),
(status = 500, body = String, description = "TBD"),
),
)]
pub(crate) async fn submit_route_monitoring_results(
State(state): State<AppState>,
Json(message): Json<MonitorMessage>,
) -> AxumResult<()> {
if !message.is_in_allowed() {
return Err(AxumErrorResponse::forbidden(
"Monitor not registered to submit results",
));
}
if !message.timely() {
return Err(AxumErrorResponse::bad_request("Message is too old"));
}
if !message.verify() {
return Err(AxumErrorResponse::bad_request("invalid signature"));
}
match message.results() {
MonitorResults::Route(results) => {
match state.storage.submit_route_monitoring_results(results).await {
Ok(_) => Ok(()),
Err(err) => {
error!("failed to submit node monitoring results: {err}");
Err(AxumErrorResponse::internal_msg(
"failed to submit node monitoring results",
))
}
}
}
MonitorResults::Node(_results) => Err(AxumErrorResponse::bad_request(
"Node monitoring results not supported for this endpoint",
)),
}
}
#[utoipa::path(
tag = "status",
post,
@@ -87,18 +138,21 @@ pub(crate) async fn submit_gateway_monitoring_results(
return Err(AxumErrorResponse::bad_request("invalid signature"));
}
match state
.storage
.submit_gateway_statuses_v2(message.results())
.await
{
Ok(_) => Ok(()),
Err(err) => {
error!("failed to submit gateway monitoring results: {err}");
Err(AxumErrorResponse::internal_msg(
"failed to submit gateway monitoring results",
))
match message.results() {
MonitorResults::Node(results) => {
match state.storage.submit_gateway_statuses_v2(results).await {
Ok(_) => Ok(()),
Err(err) => {
error!("failed to submit node monitoring results: {err}");
Err(AxumErrorResponse::internal_msg(
"failed to submit node monitoring results",
))
}
}
}
MonitorResults::Route(_results) => Err(AxumErrorResponse::bad_request(
"Gateway monitoring results not supported for this endpoint",
)),
}
}
@@ -131,18 +185,21 @@ pub(crate) async fn submit_node_monitoring_results(
return Err(AxumErrorResponse::bad_request("invalid signature"));
}
match state
.storage
.submit_mixnode_statuses_v2(message.results())
.await
{
Ok(_) => Ok(()),
Err(err) => {
error!("failed to submit node monitoring results: {err}");
Err(AxumErrorResponse::internal_msg(
"failed to submit node monitoring results",
))
match message.results() {
MonitorResults::Node(results) => {
match state.storage.submit_mixnode_statuses_v2(results).await {
Ok(_) => Ok(()),
Err(err) => {
error!("failed to submit node monitoring results: {err}");
Err(AxumErrorResponse::internal_msg(
"failed to submit node monitoring results",
))
}
}
}
MonitorResults::Route(_results) => Err(AxumErrorResponse::bad_request(
"Node monitoring results not supported for this endpoint",
)),
}
}
nym-api/src/support/storage/manager.rs
+179 -1
View File
@@ -10,10 +10,11 @@ use crate::support::storage::models::{
};
use crate::support::storage::DbIdCache;
use nym_mixnet_contract_common::{EpochId, IdentityKey, NodeId};
use nym_types::monitoring::NodeResult;
use nym_types::monitoring::{NodeResult, RouteResult};
use sqlx::FromRow;
use time::{Date, OffsetDateTime};
use tracing::info;
use std::collections::HashMap;
#[derive(Clone)]
pub(crate) struct StorageManager {
@@ -51,6 +52,25 @@ impl AvgGatewayReliability {
}
}
// Helper struct for in-memory state during calculation for an interval
#[derive(Debug, Default, Clone)]
struct NodeCorrectionIntervalState {
pos_samples: u32,
neg_samples: u32,
fail_seq: u32,
}
// Output struct for the calculated corrected reliability for an interval
#[derive(Debug, serde::Serialize)] // Add utoipa::ToSchema if this struct is directly exposed via API
pub struct CorrectedNodeIntervalReliability {
pub node_id: NodeId, // nym_mixnet_contract_common::NodeId (typically u32)
pub identity: Option<String>, // Base58 public key
pub reliability: f64,
pub pos_samples_in_interval: u32,
pub neg_samples_in_interval: u32,
pub final_fail_seq_in_interval: u32,
}
// all SQL goes here
impl StorageManager {
pub(super) async fn get_all_avg_mix_reliability_in_last_24hr(
@@ -664,6 +684,33 @@ impl StorageManager {
tx.commit().await
}
pub(super) async fn submit_route_monitoring_results(
&self,
route_results: &[RouteResult],
) -> Result<(), sqlx::Error> {
info!("Inserting {} route monitoring results", route_results.len());
// insert it all in a transaction to make sure all nodes are updated at the same time
// (plus it's a nice guard against new nodes)
let mut tx = self.connection_pool.begin().await?;
for route_result in route_results {
sqlx::query!(
r#"
INSERT OR IGNORE INTO routes (layer1, layer2, layer3, gw, success) VALUES (?, ?, ?, ?, ?);
"#,
route_result.layer1,
route_result.layer2,
route_result.layer3,
route_result.gw,
route_result.success,
)
.execute(&mut *tx)
.await?;
}
tx.commit().await
}
pub(super) async fn submit_gateway_statuses_v2(
&self,
gateway_results: &[NodeResult],
@@ -1329,6 +1376,137 @@ impl StorageManager {
.fetch_all(&self.connection_pool)
.await
}
/// Fetches raw route results from the database for a given time interval.
/// Assumes the 'routes' table has layer1, layer2, layer3, gw, success, and a timestamp.
async fn get_raw_routes_in_interval(
&self,
start_ts_secs: i64,
end_ts_secs: i64,
) -> Result<Vec<(NodeId, NodeId, NodeId, NodeId, bool)>, sqlx::Error> {
// Temporary struct to match the expected columns from the 'routes' table
// NodeId here is assumed to be compatible with how layer1, etc. are stored (e.g. u32/i32/i64)
struct RawRouteData {
layer1: NodeId,
layer2: NodeId,
layer3: NodeId,
gw: NodeId,
success: bool,
// timestamp: i64, // Not explicitly selected into struct, but used in WHERE and ORDER BY
}
// Ensure your 'routes' table has:
// layer1 (NodeId type), layer2 (NodeId type), layer3 (NodeId type),
// gw (NodeId type), success (bool/INTEGER), timestamp (BIGINT/INTEGER)
// The "NodeId:" type hint for sqlx::query_as! helps map to nym_mixnet_contract_common::NodeId
let db_routes = sqlx::query_as!(
RawRouteData,
r#"
SELECT
layer1 as "layer1",
layer2 as "layer2",
layer3 as "layer3",
gw as "gw",
success
FROM routes
WHERE timestamp >= ? AND timestamp <= ?
ORDER BY timestamp ASC
"#,
start_ts_secs,
end_ts_secs
)
.fetch_all(&self.connection_pool)
.await?;
Ok(db_routes
.into_iter()
.map(|r| (r.layer1, r.layer2, r.layer3, r.gw, r.success))
.collect())
}
pub async fn calculate_corrected_node_reliabilities_for_interval(
&self,
start_ts_secs: i64,
end_ts_secs: i64,
) -> Result<Vec<CorrectedNodeIntervalReliability>, anyhow::Error> {
let raw_routes = self
.get_raw_routes_in_interval(start_ts_secs, end_ts_secs)
.await?;
let mut node_states: HashMap<NodeId, NodeCorrectionIntervalState> = HashMap::new();
for (l1, l2, l3, gw, success) in raw_routes {
let path_node_ids = [l1, l2, l3, gw];
if success {
for &node_id in &path_node_ids {
let state = node_states.entry(node_id).or_default();
state.pos_samples += 1;
state.fail_seq = 0;
}
} else {
// Path test failed
let mut current_path_node_ids_for_blame: Vec<NodeId> = Vec::new();
for &node_id in &path_node_ids {
let state = node_states.entry(node_id).or_default();
state.fail_seq += 1;
current_path_node_ids_for_blame.push(node_id);
}
let mut guilty_nodes_in_path: Vec<NodeId> = Vec::new();
for &node_id in &current_path_node_ids_for_blame {
if let Some(state_after_update) = node_states.get(&node_id) {
if state_after_update.fail_seq > 2 {
guilty_nodes_in_path.push(node_id);
}
}
}
if !guilty_nodes_in_path.is_empty() {
for &guilty_node_id in &guilty_nodes_in_path {
if let Some(state) = node_states.get_mut(&guilty_node_id) {
state.neg_samples += 1;
}
}
} else {
// No single guilty party, distribute blame
for &node_id_in_path in &current_path_node_ids_for_blame {
if let Some(state) = node_states.get_mut(&node_id_in_path) {
state.neg_samples += 1;
}
}
}
}
}
let mut final_reliabilities = Vec::new();
for (node_id, state) in node_states {
let total_samples = state.pos_samples + state.neg_samples;
let reliability = if total_samples == 0 {
0.0 // Default for no samples in this interval. Consider Option<f64> or filtering.
} else {
state.pos_samples as f64 / total_samples as f64
};
// Attempt to fetch identity, first as mixnode, then as gateway if not found.
// This assumes get_mixnode_identity_key and get_gateway_identity_key return Result<Option<String>, sqlx::Error>
let mut identity: Option<String> = self.get_mixnode_identity_key(node_id).await.unwrap_or(None);
if identity.is_none() {
identity = self.get_gateway_identity_key(node_id).await.unwrap_or(None);
}
final_reliabilities.push(CorrectedNodeIntervalReliability {
node_id,
identity,
reliability,
pos_samples_in_interval: state.pos_samples,
neg_samples_in_interval: state.neg_samples,
final_fail_seq_in_interval: state.fail_seq,
});
}
Ok(final_reliabilities)
}
}
pub(crate) mod v3_migration {
nym-api/src/support/storage/mod.rs
+11 -1
View File
@@ -17,7 +17,7 @@ use crate::support::storage::models::{
};
use dashmap::DashMap;
use nym_mixnet_contract_common::NodeId;
use nym_types::monitoring::NodeResult;
use nym_types::monitoring::{NodeResult, RouteResult};
use sqlx::sqlite::{SqliteAutoVacuum, SqliteSynchronous};
use sqlx::ConnectOptions;
use std::path::Path;
@@ -835,6 +835,16 @@ impl NymApiStorage {
Ok(())
}
pub(crate) async fn submit_route_monitoring_results(
&self,
route_results: &[RouteResult],
) -> Result<(), NymApiStorageError> {
self.manager
.submit_route_monitoring_results(route_results)
.await?;
Ok(())
}
/// Obtains number of network monitor test runs that have occurred within the specified interval.
///
/// # Arguments
nym-network-monitor/main-1747305481101.py
+104
View File
@@ -0,0 +1,104 @@
import csv
import arrow
class Measurement:
def __init__(self, serial, ts, route, received): #, tstamp):
self.serial = serial # serial number identifying the packet (ordered in time)
self.ts = ts # timestamp
self.route = route # route of the packet in the form [node_L1, node_L2, node_L3, gateway]
self.received = received # True if received, False if not
self.duplicates = 0 # increases with the number of duplicates
class Node:
def __init__(self, node_id):
self.node_id = node_id # serial number identifying the node
self.mix = False # it's set to true when the node is seen in a mix layer position
self.gateway = False # it's set to true when the node is seen in a gateway position
self.pos_samples = 0 # measurement samples routed by this node and successfully received
self.neg_samples = 0 # measurement samples considered as possibly dropped by this node
self.fail_seq = 0 # nr of packets dropped in a sequence
self.pos_samples_v2 = 0 # measurement samples routed by this node and successfully received (current system)
self.neg_samples_v2 = 0 # measurement samples considered as possibly dropped by this node (current system)
self.score = 0 # performance score computed after new postprocessing
self.score_v2 = 0 # current performance score
def read_input_file(file_name):
list_measurements = []
dict_nodes = {}
with open(file_name, newline='') as csvfile:
reader = csv.reader(csvfile, delimiter=',', quotechar='|')
for row in reader:
if row[0] == 'id':
continue
serial = int(row[0])
ts = arrow.get(row[1])
ts_seconds = ts.timestamp()
n1 = int(row[2])
n2 = int(row[3])
n3 = int(row[4])
gw = int(row[5])
if row[6] == 'false':
received = False
elif row[6] == 'true':
received = True
else:
exit('ERROR booloan from list, value = ' + str('row[6]'))
if len(list_measurements) > 0:
last_msm = list_measurements[-1]
if n1 == last_msm.route[0] and n2 == last_msm.route[1] and n3 == last_msm.route[2] and gw == last_msm.route[3]:
last_msm.duplicates += 1
else:
msm = Measurement(serial, ts_seconds, [n1, n2, n3, gw], received)
list_measurements.append(msm)
else:
msm = Measurement(serial, ts_seconds, [n1, n2, n3, gw], received)
list_measurements.append(msm)
for n in [n1, n2, n3, gw]:
if n not in dict_nodes.keys():
dict_nodes[n] = Node(n)
return list_measurements, dict_nodes
def compute_node_scores(list_measurements, dict_nodes):
for msg in list_measurements:
if msg.received:
for node in msg.route:
dict_nodes[node].pos_samples += 1 # count measurement as positive
dict_nodes[node].fail_seq = 0 # reset sequence of failed messages
dict_nodes[node].pos_samples_v2 += 1 # count measurement as positive
else: # message was dropped
guilty = [] # candidates for being responsible for the drop
for node in msg.route:
dict_nodes[node].fail_seq += 1
dict_nodes[node].neg_samples_v2 += 1 # count measurement as negative
if dict_nodes[node].fail_seq > 2:
guilty.append(node)
dict_nodes[node].neg_samples += 1
if len(guilty) == 0: # none of them is obviously dropping packets
for node in msg.route:
dict_nodes[node].neg_samples += 1 # punish all three with a negative sample
for node in dict_nodes.values():
if node.pos_samples + node.neg_samples == 0:
exit("not enough samples causes division by zero")
estimated_performance = node.pos_samples / (node.pos_samples + node.neg_samples)
estimated_performance_v2 = node.pos_samples_v2 / (node.pos_samples_v2 + node.neg_samples_v2)
print("\nnode id: ", node.node_id)
print("estimated performance (new v2.5): ", round(estimated_performance, 2))
print("estimated performance (current v2): ", round(estimated_performance_v2, 2))
if __name__ == '__main__':
file_name = 'routes_v0.csv'
list_measurements, dict_nodes = read_input_file(file_name)
compute_node_scores(list_measurements, dict_nodes)
nym-network-monitor/src/accounting.rs
+29 -3
View File
@@ -8,7 +8,7 @@ use futures::{pin_mut, stream::FuturesUnordered, StreamExt};
use log::{debug, error, info};
use nym_sphinx::chunking::{monitoring, SentFragment};
use nym_topology::{NymRouteProvider, RoutingNode};
use nym_types::monitoring::{MonitorMessage, NodeResult};
use nym_types::monitoring::{MonitorMessage, MonitorResults, NodeResult, RouteResult};
use nym_validator_client::nym_api::routes::{API_VERSION, STATUS, SUBMIT_GATEWAY, SUBMIT_NODE};
use rand::SeedableRng;
use rand_chacha::ChaCha8Rng;
@@ -439,6 +439,7 @@ async fn db_connection(database_url: Option<&String>) -> Result<Option<(Client,
Ok(None)
}
}
pub async fn submit_metrics_to_db(database_url: Option<&String>) -> anyhow::Result<()> {
if let Some((client, handle)) = db_connection(database_url).await? {
let client = Arc::new(client);
@@ -509,7 +510,8 @@ pub async fn submit_metrics(database_url: Option<&String>) -> anyhow::Result<()>
node_stats
.chunks(10)
.map(|chunk| {
let monitor_message = MonitorMessage::new(chunk.to_vec(), private_key);
let monitor_results = MonitorResults::Node(chunk.to_vec());
let monitor_message = MonitorMessage::new(monitor_results, private_key);
client.post(&node_submit_url).json(&monitor_message).send()
})
.collect::<FuturesUnordered<_>>()
@@ -523,8 +525,9 @@ pub async fn submit_metrics(database_url: Option<&String>) -> anyhow::Result<()>
gateway_stats
.chunks(10)
.map(|chunk| {
let monitor_results = MonitorResults::Node(chunk.to_vec());
let monitor_message = MonitorMessage::new(
chunk.to_vec(),
monitor_results,
PRIVATE_KEY.get().expect("We've set this!"),
);
client
@@ -537,6 +540,29 @@ pub async fn submit_metrics(database_url: Option<&String>) -> anyhow::Result<()>
.await
.into_iter()
.collect::<Result<Vec<_>, _>>()?;
let network_account = NetworkAccount::finalize()?;
let accounting_routes = network_account.accounting_routes;
let route_results = accounting_routes
.iter()
.map(|route| {
RouteResult::new(
route.mix_nodes.0,
route.mix_nodes.1,
route.mix_nodes.2,
route.gateway_node,
route.success,
)
})
.collect::<Vec<RouteResult>>();
let monitor_results = MonitorResults::Route(route_results);
let monitor_message = MonitorMessage::new(monitor_results, private_key);
client
.post(&node_submit_url)
.json(&monitor_message)
.send()
.await?;
}
}
}