From 3bc7ced2cf91464b0da142651b47e0f9a93070c8 Mon Sep 17 00:00:00 2001 From: dynco-nym <173912580+dynco-nym@users.noreply.github.com> Date: Mon, 24 Mar 2025 11:21:09 +0100 Subject: [PATCH] Remove UNIQUE constraint on node pubkey (#5649) * Migration to remove UNIQUE constraint * Don't remove old nodes * Bump package version * Update function name --- Cargo.lock | 2 +- ...a5b483d9e1a395c65c536fdc4596b69b4584b.json | 12 ++++ ...335d9c26fe1447309f46623bcfee4537991b1.json | 12 ---- ...69682c4ae96f774261f5c298264d3c12e5b67.json | 12 ++++ ...c266223814c19ab2a5763a0575274d684d02e.json | 20 ------ .../nym-node-status-api/Cargo.toml | 2 +- .../006_remove_unique_constraint.sql | 68 +++++++++++++++++++ .../nym-node-status-api/src/db/queries/mod.rs | 4 +- .../src/db/queries/nym_nodes.rs | 45 ++++-------- .../src/db/queries/scraper.rs | 24 ++++--- 10 files changed, 121 insertions(+), 80 deletions(-) create mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-12524c93213c29a4f231785ba25a5b483d9e1a395c65c536fdc4596b69b4584b.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-b41fc93f5dc7a397e8898776bf1335d9c26fe1447309f46623bcfee4537991b1.json create mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-b68796d1d8d2384b30f1aace06269682c4ae96f774261f5c298264d3c12e5b67.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-fb2fcd26974ce4c3f5055fa2ca8c266223814c19ab2a5763a0575274d684d02e.json create mode 100644 nym-node-status-api/nym-node-status-api/migrations/006_remove_unique_constraint.sql diff --git a/Cargo.lock b/Cargo.lock index 0020669369..158018c47d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6226,7 +6226,7 @@ dependencies = [ [[package]] name = "nym-node-status-api" -version = "1.0.3" +version = "2.0.0" dependencies = [ "ammonia", "anyhow", diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-12524c93213c29a4f231785ba25a5b483d9e1a395c65c536fdc4596b69b4584b.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-12524c93213c29a4f231785ba25a5b483d9e1a395c65c536fdc4596b69b4584b.json new file mode 100644 index 0000000000..d58822be6c --- /dev/null +++ b/nym-node-status-api/nym-node-status-api/.sqlx/query-12524c93213c29a4f231785ba25a5b483d9e1a395c65c536fdc4596b69b4584b.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "DELETE FROM nym_nodes\n WHERE last_updated_utc < ?", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + }, + "hash": "12524c93213c29a4f231785ba25a5b483d9e1a395c65c536fdc4596b69b4584b" +} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-b41fc93f5dc7a397e8898776bf1335d9c26fe1447309f46623bcfee4537991b1.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-b41fc93f5dc7a397e8898776bf1335d9c26fe1447309f46623bcfee4537991b1.json deleted file mode 100644 index cf560141fd..0000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-b41fc93f5dc7a397e8898776bf1335d9c26fe1447309f46623bcfee4537991b1.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "DELETE FROM nym_nodes\n WHERE node_id = ?", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - }, - "hash": "b41fc93f5dc7a397e8898776bf1335d9c26fe1447309f46623bcfee4537991b1" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-b68796d1d8d2384b30f1aace06269682c4ae96f774261f5c298264d3c12e5b67.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-b68796d1d8d2384b30f1aace06269682c4ae96f774261f5c298264d3c12e5b67.json new file mode 100644 index 0000000000..383c3681f0 --- /dev/null +++ b/nym-node-status-api/nym-node-status-api/.sqlx/query-b68796d1d8d2384b30f1aace06269682c4ae96f774261f5c298264d3c12e5b67.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "UPDATE nym_nodes\n SET\n self_described = NULL,\n bond_info = NULL", + "describe": { + "columns": [], + "parameters": { + "Right": 0 + }, + "nullable": [] + }, + "hash": "b68796d1d8d2384b30f1aace06269682c4ae96f774261f5c298264d3c12e5b67" +} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-fb2fcd26974ce4c3f5055fa2ca8c266223814c19ab2a5763a0575274d684d02e.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-fb2fcd26974ce4c3f5055fa2ca8c266223814c19ab2a5763a0575274d684d02e.json deleted file mode 100644 index 50a09435c0..0000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-fb2fcd26974ce4c3f5055fa2ca8c266223814c19ab2a5763a0575274d684d02e.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "SQLite", - "query": "SELECT\n node_id as \"node_id!: i64\"\n FROM\n nym_nodes\n ", - "describe": { - "columns": [ - { - "name": "node_id!: i64", - "ordinal": 0, - "type_info": "Int64" - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - true - ] - }, - "hash": "fb2fcd26974ce4c3f5055fa2ca8c266223814c19ab2a5763a0575274d684d02e" -} diff --git a/nym-node-status-api/nym-node-status-api/Cargo.toml b/nym-node-status-api/nym-node-status-api/Cargo.toml index 2cd1ed4d72..5acd155f94 100644 --- a/nym-node-status-api/nym-node-status-api/Cargo.toml +++ b/nym-node-status-api/nym-node-status-api/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "nym-node-status-api" -version = "1.0.3" +version = "2.0.0" authors.workspace = true repository.workspace = true homepage.workspace = true diff --git a/nym-node-status-api/nym-node-status-api/migrations/006_remove_unique_constraint.sql b/nym-node-status-api/nym-node-status-api/migrations/006_remove_unique_constraint.sql new file mode 100644 index 0000000000..1d747b33b6 --- /dev/null +++ b/nym-node-status-api/nym-node-status-api/migrations/006_remove_unique_constraint.sql @@ -0,0 +1,68 @@ +-- Removing UNIQUE constraint on nym_nodes +-- https://www.sqlite.org/lang_altertable.html + +-- To avoid invalidating existing FK references, temporarily disable FK enforcement. +PRAGMA foreign_keys=off; + +CREATE TABLE nym_nodes_new ( + node_id INTEGER PRIMARY KEY, + ed25519_identity_pubkey VARCHAR NOT NULL, + total_stake INTEGER NOT NULL, + ip_addresses TEXT NOT NULL, -- JSON serialized + mix_port INTEGER NOT NULL, + x25519_sphinx_pubkey VARCHAR NOT NULL, + node_role TEXT NOT NULL, -- JSON serialized + supported_roles TEXT NOT NULL, -- JSON serialized + performance VARCHAR NOT NULL, + entry TEXT, -- JSON serialized + self_described TEXT, -- JSON serialized + bond_info TEXT, -- JSON serialized + last_updated_utc INTEGER NOT NULL +); + +-- columns are misaligned because old nym_nodes has 2 subsequently added columns +-- which come at the end of schema definition. +-- To correctly insert values into corresponding columns, named columns are required +INSERT INTO nym_nodes_new ( + node_id, + ed25519_identity_pubkey, + total_stake, + ip_addresses, + mix_port, + x25519_sphinx_pubkey, + node_role, + supported_roles, + performance, + entry, + self_described, + bond_info, + last_updated_utc +) +SELECT + existing.node_id, + existing.ed25519_identity_pubkey, + existing.total_stake, + existing.ip_addresses, + existing.mix_port, + existing.x25519_sphinx_pubkey, + existing.node_role, + existing.supported_roles, + existing.performance, + existing.entry, + existing.self_described, + existing.bond_info, + existing.last_updated_utc +FROM nym_nodes as existing; + +DROP INDEX IF EXISTS idx_nym_nodes_node_id; +DROP INDEX IF EXISTS idx_nym_nodes_ed25519_identity_pubkey; + +DROP TABLE nym_nodes; + +ALTER TABLE nym_nodes_new RENAME TO nym_nodes; + +CREATE INDEX idx_nym_nodes_node_id ON nym_nodes (node_id); +CREATE INDEX idx_nym_nodes_ed25519_identity_pubkey ON nym_nodes (ed25519_identity_pubkey); + + +PRAGMA foreign_keys=on; diff --git a/nym-node-status-api/nym-node-status-api/src/db/queries/mod.rs b/nym-node-status-api/nym-node-status-api/src/db/queries/mod.rs index 2e5a864ba6..f9ca28923c 100644 --- a/nym-node-status-api/nym-node-status-api/src/db/queries/mod.rs +++ b/nym-node-status-api/nym-node-status-api/src/db/queries/mod.rs @@ -15,8 +15,8 @@ pub(crate) use gateways_stats::{delete_old_records, get_sessions_stats, insert_s pub(crate) use misc::insert_summaries; pub(crate) use mixnodes::{get_all_mixnodes, get_bonded_mix_ids, get_daily_stats, update_mixnodes}; pub(crate) use nym_nodes::{ - get_active_nym_nodes, get_all_nym_nodes, get_described_node_bond_info, get_node_descriptions, - update_nym_nodes, + get_all_nym_nodes, get_described_bonded_nym_nodes, get_described_node_bond_info, + get_node_descriptions, update_nym_nodes, }; pub(crate) use packet_stats::{ get_raw_node_stats, insert_daily_node_stats, insert_node_packet_stats, diff --git a/nym-node-status-api/nym-node-status-api/src/db/queries/nym_nodes.rs b/nym-node-status-api/nym-node-status-api/src/db/queries/nym_nodes.rs index 6280af8148..6d2aa2aa36 100644 --- a/nym-node-status-api/nym-node-status-api/src/db/queries/nym_nodes.rs +++ b/nym-node-status-api/nym-node-status-api/src/db/queries/nym_nodes.rs @@ -1,10 +1,9 @@ -use anyhow::Context; use futures_util::TryStreamExt; use nym_validator_client::{ client::{NodeId, NymNodeDetails}, models::NymNodeDescription, }; -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use tracing::instrument; use crate::db::{ @@ -45,7 +44,9 @@ pub(crate) async fn get_all_nym_nodes(pool: &DbPool) -> anyhow::Result anyhow::Result> { +pub(crate) async fn get_described_bonded_nym_nodes( + pool: &DbPool, +) -> anyhow::Result> { let mut conn = pool.acquire().await?; sqlx::query_as!( @@ -84,17 +85,14 @@ pub(crate) async fn update_nym_nodes( ) -> anyhow::Result { let mut tx = pool.begin().await?; - let mut nodes_to_delete = sqlx::query!( - r#"SELECT - node_id as "node_id!: i64" - FROM - nym_nodes - "#, + sqlx::query!( + "UPDATE nym_nodes + SET + self_described = NULL, + bond_info = NULL", ) - .fetch_all(&mut *tx) - .await - .map(|records| records.into_iter().map(|record| record.node_id as NodeId))? - .collect::>(); + .execute(&mut *tx) + .await?; let inserted = node_records.len(); for record in node_records { @@ -140,26 +138,7 @@ pub(crate) async fn update_nym_nodes( ) .execute(&mut *tx) .await - .with_context(|| format!("Failed to INSERT node_id={}", record.node_id))?; - - // if node was updated, remove it from the list - nodes_to_delete.remove(&(record.node_id as NodeId)); - } - - if !nodes_to_delete.is_empty() { - tracing::debug!("DELETING {} obsolete nodes", nodes_to_delete.len()); - } - - // clean up leftover nodes, which weren't inserted/updated - for node_id in nodes_to_delete { - sqlx::query!( - "DELETE FROM nym_nodes - WHERE node_id = ?", - node_id, - ) - .execute(&mut *tx) - .await - .map_err(|e| anyhow::anyhow!("Failed to DELETE node_id={}: {}", node_id, e))?; + .map_err(|e| anyhow::anyhow!("Failed to INSERT node_id={}: {}", record.node_id, e))?; } tx.commit().await?; diff --git a/nym-node-status-api/nym-node-status-api/src/db/queries/scraper.rs b/nym-node-status-api/nym-node-status-api/src/db/queries/scraper.rs index 87d9fea229..aff700595d 100644 --- a/nym-node-status-api/nym-node-status-api/src/db/queries/scraper.rs +++ b/nym-node-status-api/nym-node-status-api/src/db/queries/scraper.rs @@ -16,18 +16,20 @@ pub(crate) async fn get_nodes_for_scraping(pool: &DbPool) -> Result Some(node), - Err(e) => { - tracing::error!("Failed to decode node_id={}: {}", node_id, e); - None + let skimmed_nodes = queries::get_described_bonded_nym_nodes(pool) + .await + .map(|nodes_dto| { + nodes_dto.into_iter().filter_map(|node| { + let node_id = node.node_id; + match SkimmedNode::try_from(node) { + Ok(node) => Some(node), + Err(e) => { + tracing::error!("Failed to decode node_id={}: {}", node_id, e); + None + } } - } - }) - })?; + }) + })?; skimmed_nodes.for_each(|node| { // TODO: relies on polyfilling: Nym nodes table might contain legacy mixnodes