Agent 0.1.4

- log gw identity key
- better agent testrun logging
- log responses
- change response code for agents

Cargo.lock

@@ -2440,7 +2440,7 @@ dependencies = [
  "maxminddb",
  "nym-bin-common 0.6.0",
  "nym-contracts-common 0.5.0",
- "nym-explorer-api-requests",
+ "nym-explorer-api-requests 0.1.0",
  "nym-mixnet-contract-common 0.6.0",
  "nym-network-defaults 0.1.0",
  "nym-task",
@@ -4586,6 +4586,34 @@ dependencies = [
  "utoipa",
 ]
 
+[[package]]
+name = "nym-api-requests"
+version = "0.1.0"
+source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
+dependencies = [
+ "bs58",
+ "cosmrs 0.17.0-pre",
+ "cosmwasm-std",
+ "ecdsa",
+ "getset",
+ "nym-compact-ecash 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "nym-credentials-interface 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "nym-crypto 0.4.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "nym-ecash-time 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "nym-mixnet-contract-common 0.6.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "nym-network-defaults 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "nym-node-requests 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "nym-serde-helpers 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "schemars",
+ "serde",
+ "serde_json",
+ "sha2 0.10.8",
+ "tendermint 0.37.0",
+ "thiserror",
+ "time",
+ "utoipa",
+]
+
 [[package]]
 name = "nym-async-file-watcher"
 version = "0.1.0"
@@ -4716,6 +4744,21 @@ dependencies = [
  "vergen",
 ]
 
+[[package]]
+name = "nym-bin-common"
+version = "0.6.0"
+source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
+dependencies = [
+ "const-str",
+ "log",
+ "pretty_env_logger",
+ "schemars",
+ "semver 1.0.23",
+ "serde",
+ "utoipa",
+ "vergen",
+]
+
 [[package]]
 name = "nym-bity-integration"
 version = "0.1.0"
@@ -4876,7 +4919,7 @@ dependencies = [
  "nym-credentials-interface 0.1.0",
  "nym-crypto 0.4.0",
  "nym-ecash-time 0.1.0",
- "nym-explorer-client",
+ "nym-explorer-client 0.1.0",
  "nym-gateway-client",
  "nym-gateway-requests",
  "nym-id",
@@ -5115,6 +5158,29 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "nym-compact-ecash"
+version = "0.1.0"
+source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
+dependencies = [
+ "bincode",
+ "bls12_381",
+ "bs58",
+ "cfg-if",
+ "digest 0.9.0",
+ "ff",
+ "group",
+ "itertools 0.13.0",
+ "nym-network-defaults 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "nym-pemstore 0.3.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "rand",
+ "serde",
+ "sha2 0.9.9",
+ "subtle 2.5.0",
+ "thiserror",
+ "zeroize",
+]
+
 [[package]]
 name = "nym-config"
 version = "0.1.0"
@@ -5142,6 +5208,20 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "nym-config"
+version = "0.1.0"
+source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
+dependencies = [
+ "dirs",
+ "handlebars",
+ "log",
+ "nym-network-defaults 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "serde",
+ "toml 0.8.14",
+ "url",
+]
+
 [[package]]
 name = "nym-contracts-common"
 version = "0.5.0"
@@ -5172,6 +5252,21 @@ dependencies = [
  "vergen",
 ]
 
+[[package]]
+name = "nym-contracts-common"
+version = "0.5.0"
+source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
+dependencies = [
+ "bs58",
+ "cosmwasm-schema",
+ "cosmwasm-std",
+ "cw-storage-plus",
+ "schemars",
+ "serde",
+ "thiserror",
+ "vergen",
+]
+
 [[package]]
 name = "nym-country-group"
 version = "0.1.0"
@@ -5377,6 +5472,22 @@ dependencies = [
  "time",
 ]
 
+[[package]]
+name = "nym-credentials-interface"
+version = "0.1.0"
+source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
+dependencies = [
+ "bls12_381",
+ "nym-compact-ecash 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "nym-ecash-time 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "nym-network-defaults 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "rand",
+ "serde",
+ "strum 0.26.3",
+ "thiserror",
+ "time",
+]
+
 [[package]]
 name = "nym-crypto"
 version = "0.4.0"
@@ -5422,6 +5533,23 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "nym-crypto"
+version = "0.4.0"
+source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
+dependencies = [
+ "bs58",
+ "ed25519-dalek",
+ "nym-pemstore 0.3.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "nym-sphinx-types 0.2.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "serde",
+ "serde_bytes",
+ "subtle-encoding",
+ "thiserror",
+ "x25519-dalek",
+ "zeroize",
+]
+
 [[package]]
 name = "nym-data-observatory"
 version = "0.1.0"
@@ -5523,6 +5651,14 @@ dependencies = [
  "time",
 ]
 
+[[package]]
+name = "nym-ecash-time"
+version = "0.1.0"
+source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
+dependencies = [
+ "time",
+]
+
 [[package]]
 name = "nym-execute"
 version = "0.1.0"
@@ -5555,6 +5691,18 @@ dependencies = [
  "utoipa",
 ]
 
+[[package]]
+name = "nym-exit-policy"
+version = "0.1.0"
+source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
+dependencies = [
+ "serde",
+ "serde_json",
+ "thiserror",
+ "tracing",
+ "utoipa",
+]
+
 [[package]]
 name = "nym-explorer-api-requests"
 version = "0.1.0"
@@ -5567,11 +5715,23 @@ dependencies = [
  "ts-rs",
 ]
 
+[[package]]
+name = "nym-explorer-api-requests"
+version = "0.1.0"
+source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
+dependencies = [
+ "nym-api-requests 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "nym-contracts-common 0.5.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "nym-mixnet-contract-common 0.6.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "schemars",
+ "serde",
+]
+
 [[package]]
 name = "nym-explorer-client"
 version = "0.1.0"
 dependencies = [
- "nym-explorer-api-requests",
+ "nym-explorer-api-requests 0.1.0",
  "reqwest 0.12.4",
  "serde",
  "thiserror",
@@ -5580,6 +5740,19 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "nym-explorer-client"
+version = "0.1.0"
+source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
+dependencies = [
+ "log",
+ "nym-explorer-api-requests 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "reqwest 0.12.4",
+ "serde",
+ "thiserror",
+ "url",
+]
+
 [[package]]
 name = "nym-ffi-shared"
 version = "0.2.0"
@@ -6009,6 +6182,27 @@ dependencies = [
  "time",
 ]
 
+[[package]]
+name = "nym-mixnet-contract-common"
+version = "0.6.0"
+source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
+dependencies = [
+ "bs58",
+ "cosmwasm-schema",
+ "cosmwasm-std",
+ "cw-controllers",
+ "cw-storage-plus",
+ "humantime-serde",
+ "log",
+ "nym-contracts-common 0.5.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "schemars",
+ "serde",
+ "serde-json-wasm",
+ "serde_repr",
+ "thiserror",
+ "time",
+]
+
 [[package]]
 name = "nym-mixnode"
 version = "1.1.37"
@@ -6140,6 +6334,19 @@ dependencies = [
  "utoipa",
 ]
 
+[[package]]
+name = "nym-network-defaults"
+version = "0.1.0"
+source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
+dependencies = [
+ "dotenvy",
+ "log",
+ "schemars",
+ "serde",
+ "url",
+ "utoipa",
+]
+
 [[package]]
 name = "nym-network-monitor"
 version = "0.1.0"
@@ -6346,8 +6553,29 @@ dependencies = [
 ]
 
 [[package]]
-name = "nym-node-status-agent"
+name = "nym-node-requests"
 version = "0.1.0"
+source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
+dependencies = [
+ "base64 0.22.1",
+ "celes",
+ "humantime 2.1.0",
+ "humantime-serde",
+ "nym-bin-common 0.6.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "nym-crypto 0.4.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "nym-exit-policy 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "nym-wireguard-types 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "schemars",
+ "serde",
+ "serde_json",
+ "thiserror",
+ "time",
+ "utoipa",
+]
+
+[[package]]
+name = "nym-node-status-agent"
+version = "0.1.4"
 dependencies = [
  "anyhow",
  "clap 4.5.20",
@@ -6363,7 +6591,7 @@ dependencies = [
 
 [[package]]
 name = "nym-node-status-api"
-version = "0.1.0"
+version = "0.1.4"
 dependencies = [
  "anyhow",
  "axum 0.7.7",
@@ -6375,7 +6603,7 @@ dependencies = [
  "moka",
  "nym-bin-common 0.6.0",
  "nym-common-models",
- "nym-explorer-client",
+ "nym-explorer-client 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
  "nym-network-defaults 0.1.0 (git+https://github.com/nymtech/nym?branch=pre-dir-v2-fork)",
  "nym-node-requests 0.1.0 (git+https://github.com/nymtech/nym?branch=pre-dir-v2-fork)",
  "nym-task",
@@ -6508,6 +6736,14 @@ dependencies = [
  "pem",
 ]
 
+[[package]]
+name = "nym-pemstore"
+version = "0.3.0"
+source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
+dependencies = [
+ "pem",
+]
+
 [[package]]
 name = "nym-sdk"
 version = "0.1.0"
@@ -6584,6 +6820,17 @@ dependencies = [
  "time",
 ]
 
+[[package]]
+name = "nym-serde-helpers"
+version = "0.1.0"
+source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
+dependencies = [
+ "base64 0.22.1",
+ "bs58",
+ "serde",
+ "time",
+]
+
 [[package]]
 name = "nym-service-provider-requests-common"
 version = "0.1.0"
@@ -6897,6 +7144,15 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "nym-sphinx-types"
+version = "0.2.0"
+source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
+dependencies = [
+ "sphinx-packet",
+ "thiserror",
+]
+
 [[package]]
 name = "nym-statistics-common"
 version = "0.1.0"
@@ -7263,6 +7519,20 @@ dependencies = [
  "x25519-dalek",
 ]
 
+[[package]]
+name = "nym-wireguard-types"
+version = "0.1.0"
+source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
+dependencies = [
+ "base64 0.22.1",
+ "log",
+ "nym-config 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "nym-network-defaults 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "serde",
+ "thiserror",
+ "x25519-dalek",
+]
+
 [[package]]
 name = "nymvisor"
 version = "0.1.8"

common/models/src/ns_api.rs

@@ -2,7 +2,6 @@ use serde::{Deserialize, Serialize};
 
 #[derive(Debug, Clone, Deserialize, Serialize)]
 pub struct TestrunAssignment {
-    /// has nothing to do with GW identity key. This is PK from `gateways` table
     pub testrun_id: i64,
-    pub gateway_pk_id: i64,
+    pub gateway_identity_key: String,
 }

nym-node-status-agent/Cargo.toml

@@ -4,7 +4,7 @@
 
 [package]
 name = "nym-node-status-agent"
-version = "0.1.0"
+version = "0.1.4"
 authors.workspace = true
 repository.workspace = true
 homepage.workspace = true

nym-node-status-agent/src/cli.rs

@@ -31,24 +31,19 @@ pub(crate) enum Command {
         /// path of binary to run
         #[arg(long, env = "NODE_STATUS_AGENT_PROBE_PATH")]
         probe_path: String,
-        #[arg(short, long, env = "NODE_STATUS_AGENT_GATEWAY_ID")]
-        gateway_id: Option<String>,
     },
 }
 
 impl Args {
     pub(crate) async fn execute(&self) -> anyhow::Result<()> {
         match &self.command {
-            Command::RunProbe {
-                probe_path,
-                gateway_id,
-            } => self.run_probe(probe_path, gateway_id).await?,
+            Command::RunProbe { probe_path } => self.run_probe(probe_path).await?,
         }
 
         Ok(())
     }
 
-    async fn run_probe(&self, probe_path: &str, gateway_id: &Option<String>) -> anyhow::Result<()> {
+    async fn run_probe(&self, probe_path: &str) -> anyhow::Result<()> {
         let server_address = format!("{}:{}", &self.server_address, self.server_port);
 
         let probe = GwProbe::new(probe_path.to_string());
@@ -58,7 +53,7 @@ impl Args {
 
         let testrun = request_testrun(&server_address).await?;
 
-        let log = probe.run_and_get_log(gateway_id);
+        let log = probe.run_and_get_log(&Some(testrun.gateway_identity_key));
 
         submit_results(&server_address, testrun.testrun_id, log).await?;
 
@@ -97,6 +92,7 @@ async fn submit_results(
 ) -> anyhow::Result<()> {
     let target_url = format!("{}/{}/{}", server_addr, URL_BASE, testrun_id);
     let client = reqwest::Client::new();
+
     let res = client
         .post(target_url)
         .body(probe_outcome)

nym-node-status-agent/src/probe.rs

@@ -40,6 +40,12 @@ impl GwProbe {
         match command.spawn() {
             Ok(child) => {
                 if let Ok(output) = child.wait_with_output() {
+                    if !output.status.success() {
+                        let out = String::from_utf8_lossy(&output.stdout);
+                        let err = String::from_utf8_lossy(&output.stderr);
+                        tracing::error!("Probe exited with {:?}:\n{}\n{}", output.status, out, err);
+                    }
+
                     return String::from_utf8(output.stdout)
                         .unwrap_or("Unable to get log from test run".to_string());
                 }

nym-node-status-api/Cargo.toml

@@ -3,7 +3,7 @@
 
 [package]
 name = "nym-node-status-api"
-version = "0.1.0"
+version = "0.1.4"
 authors.workspace = true
 repository.workspace = true
 homepage.workspace = true
@@ -23,7 +23,8 @@ futures-util = { workspace = true }
 moka = { workspace = true, features = ["future"] }
 nym-bin-common = { path = "../common/bin-common", features = ["models"]}
 nym-common-models = { path = "../common/models" }
-nym-explorer-client = { path = "../explorer-api/explorer-client" }
+# nym-explorer-client = { path = "../explorer-api/explorer-client" }
+nym-explorer-client = { git = "https://github.com/nymtech/nym", branch = "release/2024.13-magura" }
 # TODO dz: before Nym API client breaking changes. Update to latest develop once new Nym API is live
 nym-network-defaults = { git = "https://github.com/nymtech/nym", branch = "pre-dir-v2-fork" }
 nym-validator-client = { git = "https://github.com/nymtech/nym", branch = "pre-dir-v2-fork" }

nym-node-status-api/build.rs

@@ -25,7 +25,6 @@ async fn main() -> Result<()> {
     // not a valid windows path... but hey, it works...
     println!("cargo::rustc-env=DATABASE_URL=sqlite:///{}", &database_path);
 
-    rerun_if_changed();
     Ok(())
 }
 
@@ -33,11 +32,6 @@ fn read_env_var(var: &str) -> Result<String> {
     std::env::var(var).map_err(|_| anyhow!("You need to set {} env var", var))
 }
 
-fn rerun_if_changed() {
-    println!("cargo::rerun-if-changed=migrations");
-    println!("cargo::rerun-if-changed=src/db/queries");
-}
-
 /// use `./enter_db.sh` to inspect DB
 async fn write_db_path_to_file(out_dir: &str, db_filename: &str) -> anyhow::Result<()> {
     let mut file = File::create("enter_db.sh").await?;

nym-node-status-api/launch_node_status_api.sh

@@ -7,7 +7,7 @@ export RUST_LOG=${RUST_LOG:-debug}
 export NYM_API_CLIENT_TIMEOUT=60
 export EXPLORER_CLIENT_TIMEOUT=60
 
-export ENVIRONMENT="mainnet.env"
+export ENVIRONMENT="qa.env"
 
 function run_bare() {
     # export necessary env vars

nym-node-status-api/migrations/000_init.sql

@@ -103,7 +103,7 @@ CREATE TABLE
 CREATE TABLE testruns
 (
   id            INTEGER PRIMARY KEY AUTOINCREMENT,
-  gateway_id    INTEGER,
+  gateway_id    INTEGER NOT NULL,
   status        INTEGER NOT NULL, -- 0=pending, 1=in-progress, 2=complete
   timestamp_utc INTEGER NOT NULL,
   ip_address    VARCHAR NOT NULL,

nym-node-status-api/src/db/models.rs

@@ -300,6 +300,7 @@ pub(crate) mod gateway {
     }
 }
 
+#[allow(dead_code)] // not dead code, this is SQL data model
 #[derive(Debug, Clone)]
 pub struct TestRunDto {
     pub id: i64,
@@ -315,7 +316,7 @@ pub struct TestRunDto {
 pub(crate) enum TestRunStatus {
     Complete = 2,
     InProgress = 1,
-    Pending = 0,
+    Queued = 0,
 }
 
 #[derive(Debug, Clone)]

nym-node-status-api/src/db/queries/gateways.rs

@@ -7,8 +7,28 @@ use crate::{
 };
 use futures_util::TryStreamExt;
 use nym_validator_client::models::DescribedGateway;
+use sqlx::{pool::PoolConnection, Sqlite};
 use tracing::error;
 
+pub(crate) async fn select_gateway_identity(
+    conn: &mut PoolConnection<Sqlite>,
+    gateway_pk: i64,
+) -> anyhow::Result<String> {
+    let record = sqlx::query!(
+        r#"SELECT
+            gateway_identity_key
+        FROM
+            gateways
+        WHERE
+            id = ?"#,
+        gateway_pk
+    )
+    .fetch_one(conn.as_mut())
+    .await?;
+
+    Ok(record.gateway_identity_key)
+}
+
 pub(crate) async fn insert_gateways(
     pool: &DbPool,
     gateways: Vec<GatewayRecord>,

nym-node-status-api/src/db/queries/mod.rs

@@ -5,7 +5,7 @@ mod summary;
 pub(crate) mod testruns;
 
 pub(crate) use gateways::{
-    ensure_gateways_still_bonded, get_all_gateways, insert_gateways,
+    ensure_gateways_still_bonded, get_all_gateways, insert_gateways, select_gateway_identity,
     write_blacklisted_gateways_to_db,
 };
 pub(crate) use misc::insert_summaries;

nym-node-status-api/src/db/queries/testruns.rs

@@ -1,12 +1,14 @@
+use crate::db::DbPool;
 use crate::http::models::TestrunAssignment;
 use crate::{
     db::models::{TestRunDto, TestRunStatus},
     testruns::now_utc,
 };
 use anyhow::Context;
+use chrono::Duration;
 use sqlx::{pool::PoolConnection, Sqlite};
 
-pub(crate) async fn get_testrun_by_id(
+pub(crate) async fn get_in_progress_testrun_by_id(
     conn: &mut PoolConnection<Sqlite>,
     testrun_id: i64,
 ) -> anyhow::Result<TestRunDto> {
@@ -20,22 +22,58 @@ pub(crate) async fn get_testrun_by_id(
             ip_address as "ip_address!",
             log as "log!"
          FROM testruns
-         WHERE id = ?
+         WHERE
+            id = ?
+         AND
+            status = ?
          ORDER BY timestamp_utc"#,
-        testrun_id
+        testrun_id,
+        TestRunStatus::InProgress as i64,
     )
     .fetch_one(conn.as_mut())
     .await
     .context(format!("Couldn't retrieve testrun {testrun_id}"))
 }
 
+pub(crate) async fn update_testruns_older_than(db: &DbPool, age: Duration) -> anyhow::Result<u64> {
+    let mut conn = db.acquire().await?;
+    let previous_run = now_utc() - age;
+    let cutoff_timestamp = previous_run.timestamp();
+
+    let res = sqlx::query!(
+        r#"UPDATE
+            testruns
+        SET
+            status = ?
+        WHERE
+            status = ?
+        AND
+            timestamp_utc < ?
+            "#,
+        TestRunStatus::Queued as i64,
+        TestRunStatus::InProgress as i64,
+        cutoff_timestamp
+    )
+    .execute(conn.as_mut())
+    .await?;
+
+    let stale_testruns = res.rows_affected();
+    if stale_testruns > 0 {
+        tracing::debug!(
+            "Refreshed {} stale testruns, scheduled before {} but not yet finished",
+            stale_testruns,
+            previous_run
+        );
+    }
+
+    Ok(stale_testruns)
+}
+
 pub(crate) async fn get_oldest_testrun_and_make_it_pending(
-    // TODO dz accept mut reference, repeat in all similar functions
-    conn: PoolConnection<Sqlite>,
+    conn: &mut PoolConnection<Sqlite>,
 ) -> anyhow::Result<Option<TestrunAssignment>> {
-    let mut conn = conn;
-    let assignment = sqlx::query_as!(
-        TestrunAssignment,
+    // find & mark as "In progress" in the same transaction to avoid race conditions
+    let returning = sqlx::query!(
         r#"UPDATE testruns
             SET status = ?
             WHERE rowid =
@@ -47,16 +85,36 @@ pub(crate) async fn get_oldest_testrun_and_make_it_pending(
             LIMIT 1
         )
         RETURNING
-            id as "testrun_id!",
-            gateway_id as "gateway_pk_id!"
+            id as "id!",
+            gateway_id
             "#,
         TestRunStatus::InProgress as i64,
-        TestRunStatus::Pending as i64,
+        TestRunStatus::Queued as i64,
     )
-    .fetch_optional(&mut *conn)
+    .fetch_optional(conn.as_mut())
     .await?;
 
-    Ok(assignment)
+    if let Some(testrun) = returning {
+        let gw_identity = sqlx::query!(
+            r#"
+                SELECT
+                    id,
+                    gateway_identity_key
+                FROM gateways
+                WHERE id = ?
+                LIMIT 1"#,
+            testrun.gateway_id
+        )
+        .fetch_one(conn.as_mut())
+        .await?;
+
+        return Ok(Some(TestrunAssignment {
+            testrun_id: testrun.id,
+            gateway_identity_key: gw_identity.gateway_identity_key,
+        }));
+    } else {
+        return Ok(None);
+    }
 }
 
 pub(crate) async fn update_testrun_status(

nym-node-status-api/src/http/api/mod.rs

@@ -1,7 +1,10 @@
 use anyhow::anyhow;
 use axum::{response::Redirect, Router};
 use tokio::net::ToSocketAddrs;
-use tower_http::{cors::CorsLayer, trace::TraceLayer};
+use tower_http::{
+    cors::CorsLayer,
+    trace::{DefaultOnResponse, TraceLayer},
+};
 use utoipa::OpenApi;
 use utoipa_swagger_ui::SwaggerUi;
 
@@ -58,7 +61,10 @@ impl RouterBuilder {
             // CORS layer needs to wrap other API layers
             .layer(setup_cors())
             // logger should be outermost layer
-            .layer(TraceLayer::new_for_http())
+            .layer(
+                TraceLayer::new_for_http()
+                    .on_response(DefaultOnResponse::new().level(tracing::Level::DEBUG)),
+            )
     }
 }
 

nym-node-status-api/src/http/api/testruns.rs

@@ -1,3 +1,4 @@
+use axum::extract::DefaultBodyLimit;
 use axum::Json;
 use axum::{
     extract::{Path, State},
@@ -23,31 +24,32 @@ pub(crate) fn routes() -> Router<AppState> {
     Router::new()
         .route("/", axum::routing::get(request_testrun))
         .route("/:testrun_id", axum::routing::post(submit_testrun))
+        .layer(DefaultBodyLimit::max(1024 * 1024 * 5))
 }
 
 #[tracing::instrument(level = "debug", skip_all)]
 async fn request_testrun(State(state): State<AppState>) -> HttpResult<Json<TestrunAssignment>> {
     // TODO dz log agent's key
     // TODO dz log agent's network probe version
-    tracing::debug!("Agent X requested testrun");
+    tracing::debug!("Agent requested testrun");
 
     let db = state.db_pool();
-    let conn = db
+    let mut conn = db
         .acquire()
         .await
         .map_err(HttpError::internal_with_logging)?;
 
-    return match db::queries::testruns::get_oldest_testrun_and_make_it_pending(conn).await {
+    return match db::queries::testruns::get_oldest_testrun_and_make_it_pending(&mut conn).await {
         Ok(res) => {
             if let Some(testrun) = res {
-                // TODO dz consider adding a column to testruns table with agent's public key
                 tracing::debug!(
-                    "🏃‍ Assigned testrun row_id {} to agent X",
-                    &testrun.testrun_id
+                    "🏃‍ Assigned testrun row_id {} gateway {} to agent",
+                    &testrun.testrun_id,
+                    testrun.gateway_identity_key
                 );
                 Ok(Json(testrun))
             } else {
-                Err(HttpError::not_found("No testruns available"))
+                Err(HttpError::no_available_testruns())
             }
         }
         Err(err) => Err(HttpError::internal_with_logging(err)),
@@ -61,25 +63,32 @@ async fn submit_testrun(
     State(state): State<AppState>,
     body: String,
 ) -> HttpResult<StatusCode> {
-    tracing::debug!(
-        "Agent submitted testrun {}. Total length: {}",
-        testrun_id,
-        body.len(),
-    );
-    // TODO dz store testrun results
-
     let db = state.db_pool();
     let mut conn = db
         .acquire()
         .await
         .map_err(HttpError::internal_with_logging)?;
 
-    let testrun = queries::testruns::get_testrun_by_id(&mut conn, testrun_id)
+    let testrun = queries::testruns::get_in_progress_testrun_by_id(&mut conn, testrun_id)
         .await
         .map_err(|e| {
             tracing::error!("{e}");
             HttpError::not_found(testrun_id)
         })?;
+
+    let gw_identity = db::queries::select_gateway_identity(&mut conn, testrun.gateway_id)
+        .await
+        .map_err(|_| {
+            // should never happen:
+            HttpError::internal_with_logging("No gateway found for testrun")
+        })?;
+    tracing::debug!(
+        "Agent submitted testrun {} for gateway {} ({} bytes)",
+        testrun_id,
+        gw_identity,
+        body.len(),
+    );
+
     // TODO dz this should be part of a single transaction: commit after everything is done
     queries::testruns::update_testrun_status(&mut conn, testrun_id, TestRunStatus::Complete)
         .await
@@ -99,7 +108,7 @@ async fn submit_testrun(
     tracing::info!(
         "✅ Testrun row_id {} for gateway {} complete",
         testrun.id,
-        testrun.gateway_id
+        gw_identity
     );
 
     Ok(StatusCode::CREATED)

nym-node-status-api/src/http/error.rs

@@ -8,9 +8,9 @@ pub(crate) struct HttpError {
 }
 
 impl HttpError {
-    pub(crate) fn invalid_input(message: String) -> Self {
+    pub(crate) fn invalid_input(msg: impl Display) -> Self {
         Self {
-            message,
+            message: serde_json::json!({"message": msg.to_string()}).to_string(),
             status: axum::http::StatusCode::BAD_REQUEST,
         }
     }
@@ -27,6 +27,12 @@ impl HttpError {
         }
     }
 
+    pub(crate) fn no_available_testruns() -> Self {
+        Self {
+            message: serde_json::json!({"message": "No available testruns"}).to_string(),
+            status: axum::http::StatusCode::SERVICE_UNAVAILABLE,
+        }
+    }
     pub(crate) fn not_found(msg: impl Display) -> Self {
         Self {
             message: serde_json::json!({"message": msg.to_string()}).to_string(),

nym-node-status-api/src/http/models.rs

@@ -1,4 +1,3 @@
-use crate::db::models::TestRunDto;
 use nym_node_requests::api::v1::node::models::NodeDescription;
 use serde::{Deserialize, Serialize};
 use utoipa::ToSchema;
@@ -75,12 +74,3 @@ pub(crate) struct SummaryHistory {
     pub value_json: serde_json::Value,
     pub timestamp_utc: String,
 }
-
-impl From<TestRunDto> for TestrunAssignment {
-    fn from(value: TestRunDto) -> Self {
-        Self {
-            gateway_pk_id: value.gateway_id,
-            testrun_id: value.id,
-        }
-    }
-}

nym-node-status-api/src/monitor/mod.rs

@@ -91,7 +91,7 @@ async fn run(
     let explorer_client =
         ExplorerClient::new_with_timeout(default_explorer_url, explorer_client_timeout)?;
     let explorer_gateways = explorer_client
-        .get_gateways()
+        .unstable_get_gateways()
         .await
         .log_error("get_gateways")?;
 

nym-node-status-api/src/testruns/mod.rs

@@ -11,10 +11,12 @@ pub(crate) use queue::now_utc;
 pub(crate) async fn spawn(pool: DbPool, refresh_interval: Duration) {
     tokio::spawn(async move {
         loop {
-            tracing::info!("Spawning testruns...");
+            if let Err(e) = refresh_stale_testruns(&pool, refresh_interval).await {
+                tracing::error!("{e}");
+            }
 
             if let Err(e) = run(&pool).await {
-                tracing::error!("Cron job failed: {}", e);
+                tracing::error!("Assigning testruns failed: {}", e);
             }
             tracing::debug!("Sleeping for {}s...", refresh_interval.as_secs());
             tokio::time::sleep(refresh_interval).await;
@@ -24,9 +26,9 @@ pub(crate) async fn spawn(pool: DbPool, refresh_interval: Duration) {
 
 // TODO dz make number of max agents configurable
 
-// TODO dz periodically clean up stale pending testruns
 #[instrument(level = "debug", name = "testrun_queue", skip_all)]
 async fn run(pool: &DbPool) -> anyhow::Result<()> {
+    tracing::info!("Spawning testruns...");
     if pool.is_closed() {
         tracing::debug!("DB pool closed, returning early");
         return Ok(());
@@ -74,3 +76,11 @@ async fn run(pool: &DbPool) -> anyhow::Result<()> {
 
     Ok(())
 }
+
+#[instrument(level = "debug", skip_all)]
+async fn refresh_stale_testruns(pool: &DbPool, refresh_interval: Duration) -> anyhow::Result<()> {
+    let chrono_duration = chrono::Duration::from_std(refresh_interval)?;
+    crate::db::queries::testruns::update_testruns_older_than(pool, chrono_duration).await?;
+
+    Ok(())
+}

nym-node-status-api/src/testruns/queue.rs

@@ -82,7 +82,7 @@ pub(crate) async fn try_queue_testrun(
     //
     // save test run
     //
-    let status = TestRunStatus::Pending as u32;
+    let status = TestRunStatus::Queued as u32;
     let log = format!(
         "Test for {identity_key} requested at {} UTC\n\n",
         timestamp_pretty
@@ -103,7 +103,7 @@ pub(crate) async fn try_queue_testrun(
     Ok(TestRun {
         id: id as u32,
         identity_key,
-        status: format!("{}", TestRunStatus::Pending),
+        status: format!("{}", TestRunStatus::Queued),
         log,
     })
 }