Remove UNIQUE constraint on node pubkey (#5649)

* Migration to remove UNIQUE constraint * Don't remove old nodes * Bump package version * Update function name
2025-03-24 11:21:09 +01:00
parent 8abcc58055
commit 3bc7ced2cf
10 changed files with 121 additions and 80 deletions
@@ -6226,7 +6226,7 @@ dependencies = [

 [[package]]
 name = "nym-node-status-api"
-version = "1.0.3"
+version = "2.0.0"
 dependencies = [
 "ammonia",
 "anyhow",
@@ -0,0 +1,12 @@
+{
+  "db_name": "SQLite",
+  "query": "DELETE FROM nym_nodes\n        WHERE last_updated_utc < ?",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 1
+    },
+    "nullable": []
+  },
+  "hash": "12524c93213c29a4f231785ba25a5b483d9e1a395c65c536fdc4596b69b4584b"
+}
@@ -1,12 +0,0 @@
-{
-  "db_name": "SQLite",
-  "query": "DELETE FROM nym_nodes\n            WHERE node_id = ?",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Right": 1
-    },
-    "nullable": []
-  },
-  "hash": "b41fc93f5dc7a397e8898776bf1335d9c26fe1447309f46623bcfee4537991b1"
-}
@@ -0,0 +1,12 @@
+{
+  "db_name": "SQLite",
+  "query": "UPDATE nym_nodes\n        SET\n            self_described = NULL,\n            bond_info = NULL",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 0
+    },
+    "nullable": []
+  },
+  "hash": "b68796d1d8d2384b30f1aace06269682c4ae96f774261f5c298264d3c12e5b67"
+}
@@ -1,20 +0,0 @@
-{
-  "db_name": "SQLite",
-  "query": "SELECT\n            node_id as \"node_id!: i64\"\n        FROM\n            nym_nodes\n        ",
-  "describe": {
-    "columns": [
-      {
-        "name": "node_id!: i64",
-        "ordinal": 0,
-        "type_info": "Int64"
-      }
-    ],
-    "parameters": {
-      "Right": 0
-    },
-    "nullable": [
-      true
-    ]
-  },
-  "hash": "fb2fcd26974ce4c3f5055fa2ca8c266223814c19ab2a5763a0575274d684d02e"
-}
@@ -3,7 +3,7 @@

 [package]
 name = "nym-node-status-api"
-version = "1.0.3"
+version = "2.0.0"
 authors.workspace = true
 repository.workspace = true
 homepage.workspace = true
@@ -0,0 +1,68 @@
+-- Removing UNIQUE constraint on nym_nodes
+-- https://www.sqlite.org/lang_altertable.html
+
+-- To avoid invalidating existing FK references, temporarily disable FK enforcement.
+PRAGMA foreign_keys=off;
+
+CREATE TABLE nym_nodes_new (
+    node_id INTEGER PRIMARY KEY,
+    ed25519_identity_pubkey VARCHAR NOT NULL,
+    total_stake INTEGER NOT NULL,
+    ip_addresses TEXT NOT NULL, -- JSON serialized
+    mix_port INTEGER NOT NULL,
+    x25519_sphinx_pubkey VARCHAR NOT NULL,
+    node_role TEXT NOT NULL, -- JSON serialized
+    supported_roles TEXT NOT NULL, -- JSON serialized
+    performance VARCHAR NOT NULL,
+    entry TEXT, -- JSON serialized
+    self_described TEXT, -- JSON serialized
+    bond_info TEXT, -- JSON serialized
+    last_updated_utc INTEGER NOT NULL
+);
+
+-- columns are misaligned because old nym_nodes has 2 subsequently added columns
+-- which come at the end of schema definition.
+-- To correctly insert values into corresponding columns, named columns are required
+INSERT INTO nym_nodes_new (
+    node_id,
+    ed25519_identity_pubkey,
+    total_stake,
+    ip_addresses,
+    mix_port,
+    x25519_sphinx_pubkey,
+    node_role,
+    supported_roles,
+    performance,
+    entry,
+    self_described,
+    bond_info,
+    last_updated_utc
+)
+SELECT
+    existing.node_id,
+    existing.ed25519_identity_pubkey,
+    existing.total_stake,
+    existing.ip_addresses,
+    existing.mix_port,
+    existing.x25519_sphinx_pubkey,
+    existing.node_role,
+    existing.supported_roles,
+    existing.performance,
+    existing.entry,
+    existing.self_described,
+    existing.bond_info,
+    existing.last_updated_utc
+FROM nym_nodes as existing;
+
+DROP INDEX IF EXISTS idx_nym_nodes_node_id;
+DROP INDEX IF EXISTS idx_nym_nodes_ed25519_identity_pubkey;
+
+DROP TABLE nym_nodes;
+
+ALTER TABLE nym_nodes_new RENAME TO nym_nodes;
+
+CREATE INDEX idx_nym_nodes_node_id ON nym_nodes (node_id);
+CREATE INDEX idx_nym_nodes_ed25519_identity_pubkey ON nym_nodes (ed25519_identity_pubkey);
+
+
+PRAGMA foreign_keys=on;
@@ -15,8 +15,8 @@ pub(crate) use gateways_stats::{delete_old_records, get_sessions_stats, insert_s
 pub(crate) use misc::insert_summaries;
 pub(crate) use mixnodes::{get_all_mixnodes, get_bonded_mix_ids, get_daily_stats, update_mixnodes};
 pub(crate) use nym_nodes::{
-    get_active_nym_nodes, get_all_nym_nodes, get_described_node_bond_info, get_node_descriptions,
-    update_nym_nodes,
+    get_all_nym_nodes, get_described_bonded_nym_nodes, get_described_node_bond_info,
+    get_node_descriptions, update_nym_nodes,
 };
 pub(crate) use packet_stats::{
    get_raw_node_stats, insert_daily_node_stats, insert_node_packet_stats,
@@ -1,10 +1,9 @@
-use anyhow::Context;
 use futures_util::TryStreamExt;
 use nym_validator_client::{
    client::{NodeId, NymNodeDetails},
    models::NymNodeDescription,
 };
-use std::collections::{HashMap, HashSet};
+use std::collections::HashMap;
 use tracing::instrument;

 use crate::db::{
@@ -45,7 +44,9 @@ pub(crate) async fn get_all_nym_nodes(pool: &DbPool) -> anyhow::Result<Vec<NymNo
 ///
 /// same if it's not bonded in the mixnet smart contract
 /// - https://nym.com/docs/operators/tokenomics/mixnet-rewards#rewarded-set-selection
-pub(crate) async fn get_active_nym_nodes(pool: &DbPool) -> anyhow::Result<Vec<NymNodeDto>> {
+pub(crate) async fn get_described_bonded_nym_nodes(
+    pool: &DbPool,
+) -> anyhow::Result<Vec<NymNodeDto>> {
    let mut conn = pool.acquire().await?;

    sqlx::query_as!(
@@ -84,17 +85,14 @@ pub(crate) async fn update_nym_nodes(
 ) -> anyhow::Result<usize> {
    let mut tx = pool.begin().await?;

-    let mut nodes_to_delete = sqlx::query!(
-        r#"SELECT
-            node_id as "node_id!: i64"
-        FROM
-            nym_nodes
-        "#,
+    sqlx::query!(
+        "UPDATE nym_nodes
+        SET
+            self_described = NULL,
+            bond_info = NULL",
    )
-    .fetch_all(&mut *tx)
-    .await
-    .map(|records| records.into_iter().map(|record| record.node_id as NodeId))?
-    .collect::<HashSet<NodeId>>();
+    .execute(&mut *tx)
+    .await?;

    let inserted = node_records.len();
    for record in node_records {
@@ -140,26 +138,7 @@ pub(crate) async fn update_nym_nodes(
        )
        .execute(&mut *tx)
        .await
-        .with_context(|| format!("Failed to INSERT node_id={}", record.node_id))?;
-
-        // if node was updated, remove it from the list
-        nodes_to_delete.remove(&(record.node_id as NodeId));
-    }
-
-    if !nodes_to_delete.is_empty() {
-        tracing::debug!("DELETING {} obsolete nodes", nodes_to_delete.len());
-    }
-
-    // clean up leftover nodes, which weren't inserted/updated
-    for node_id in nodes_to_delete {
-        sqlx::query!(
-            "DELETE FROM nym_nodes
-            WHERE node_id = ?",
-            node_id,
-        )
-        .execute(&mut *tx)
-        .await
-        .map_err(|e| anyhow::anyhow!("Failed to DELETE node_id={}: {}", node_id, e))?;
+        .map_err(|e| anyhow::anyhow!("Failed to INSERT node_id={}: {}", record.node_id, e))?;
    }

    tx.commit().await?;
@@ -16,18 +16,20 @@ pub(crate) async fn get_nodes_for_scraping(pool: &DbPool) -> Result<Vec<ScraperN
    let gateway_keys = queries::get_bonded_gateway_id_keys(pool).await?;

    let mut entry_exit_nodes = 0;
-    let skimmed_nodes = queries::get_active_nym_nodes(pool).await.map(|nodes_dto| {
-        nodes_dto.into_iter().filter_map(|node| {
-            let node_id = node.node_id;
-            match SkimmedNode::try_from(node) {
-                Ok(node) => Some(node),
-                Err(e) => {
-                    tracing::error!("Failed to decode node_id={}: {}", node_id, e);
-                    None
+    let skimmed_nodes = queries::get_described_bonded_nym_nodes(pool)
+        .await
+        .map(|nodes_dto| {
+            nodes_dto.into_iter().filter_map(|node| {
+                let node_id = node.node_id;
+                match SkimmedNode::try_from(node) {
+                    Ok(node) => Some(node),
+                    Err(e) => {
+                        tracing::error!("Failed to decode node_id={}: {}", node_id, e);
+                        None
+                    }
                }
-            }
-        })
-    })?;
+            })
+        })?;

    skimmed_nodes.for_each(|node| {
        // TODO: relies on polyfilling: Nym nodes table might contain legacy mixnodes