fix review comments

* feat: add dockerfile

* add github workflow for node status agent

---------

Co-authored-by: Fran Arbanas <arbanasfran@gmail.com>

.github/workflows/push-node-status-agent.yaml

@@ -3,9 +3,54 @@ name: Build and upload Node Status agent container to harbor.nymte.ch
 on:
   workflow_dispatch:
 
+env:
+  WORKING_DIRECTORY: "nym-node-status-agent"
+  CONTAINER_NAME: "node-status-agent"
+
 jobs:
-  my-job:
-    runs-on: arc-ubuntu-22.04
+  build-container:
+    runs-on: arc-ubuntu-22.04-dind
     steps:
-      - name: my-step
-        run: echo "Hello World!"
+      - name: Login to Harbor
+        uses: docker/login-action@v3
+        with:
+          registry: harbor.nymte.ch
+          username: ${{ secrets.HARBOR_ROBOT_USERNAME }}
+          password: ${{ secrets.HARBOR_ROBOT_SECRET }}
+
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Configure git identity
+        run: |
+          git config --global user.email "lawrence@nymtech.net"
+          git config --global user.name "Lawrence Stalder"
+
+      - name: Get version from cargo.toml
+        uses: mikefarah/yq@v4.44.3
+        id: get_version
+        with:
+          cmd: yq -oy '.package.version' ${{ env.WORKING_DIRECTORY }}/Cargo.toml
+
+      - name: Check if tag exists
+        run: |
+          if git rev-parse ${{ steps.get_version.outputs.value }} >/dev/null 2>&1; then
+            echo "Tag ${{ steps.get_version.outputs.value }} already exists"
+          fi
+
+      - name: Remove existing tag if exists
+        run: |
+          if git rev-parse ${{ env.WORKING_DIRECTORY }}-${{ steps.get_version.outputs.result }} >/dev/null 2>&1; then
+            git push --delete origin ${{ env.WORKING_DIRECTORY }}-${{ steps.get_version.outputs.result }}
+            git tag -d ${{ env.WORKING_DIRECTORY }}-${{ steps.get_version.outputs.result }}
+          fi
+
+      - name: Create tag
+        run: |
+          git tag -a ${{ env.WORKING_DIRECTORY }}-${{ steps.get_version.outputs.result }} -m "Version ${{ steps.get_version.outputs.result }}"
+          git push origin ${{ env.WORKING_DIRECTORY }}-${{ steps.get_version.outputs.result }}
+
+      - name: BuildAndPushImageOnHarbor
+        run: |
+          docker build -f ${{ env.WORKING_DIRECTORY }}/Dockerfile . -t harbor.nymte.ch/nym/${{ env.CONTAINER_NAME }}:${{ steps.get_version.outputs.result }} -t harbor.nymte.ch/nym/${{ env.CONTAINER_NAME }}:latest
+          docker push harbor.nymte.ch/nym/${{ env.CONTAINER_NAME }} --all-tags

Cargo.lock

@@ -2429,7 +2429,7 @@ dependencies = [
  "maxminddb",
  "nym-bin-common 0.6.0",
  "nym-contracts-common 0.5.0",
- "nym-explorer-api-requests 0.1.0",
+ "nym-explorer-api-requests",
  "nym-mixnet-contract-common 0.6.0",
  "nym-network-defaults 0.1.0",
  "nym-task",
@@ -4559,34 +4559,6 @@ dependencies = [
  "utoipa",
 ]
 
-[[package]]
-name = "nym-api-requests"
-version = "0.1.0"
-source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
-dependencies = [
- "bs58",
- "cosmrs 0.17.0-pre",
- "cosmwasm-std",
- "ecdsa",
- "getset",
- "nym-compact-ecash 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "nym-credentials-interface 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "nym-crypto 0.4.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "nym-ecash-time 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "nym-mixnet-contract-common 0.6.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "nym-network-defaults 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "nym-node-requests 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "nym-serde-helpers 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "schemars",
- "serde",
- "serde_json",
- "sha2 0.10.8",
- "tendermint 0.37.0",
- "thiserror",
- "time",
- "utoipa",
-]
-
 [[package]]
 name = "nym-async-file-watcher"
 version = "0.1.0"
@@ -4717,21 +4689,6 @@ dependencies = [
  "vergen",
 ]
 
-[[package]]
-name = "nym-bin-common"
-version = "0.6.0"
-source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
-dependencies = [
- "const-str",
- "log",
- "pretty_env_logger",
- "schemars",
- "semver 1.0.23",
- "serde",
- "utoipa",
- "vergen",
-]
-
 [[package]]
 name = "nym-bity-integration"
 version = "0.1.0"
@@ -4892,7 +4849,7 @@ dependencies = [
  "nym-credentials-interface 0.1.0",
  "nym-crypto 0.4.0",
  "nym-ecash-time 0.1.0",
- "nym-explorer-client 0.1.0",
+ "nym-explorer-client",
  "nym-gateway-client",
  "nym-gateway-requests",
  "nym-id",
@@ -5131,29 +5088,6 @@ dependencies = [
  "zeroize",
 ]
 
-[[package]]
-name = "nym-compact-ecash"
-version = "0.1.0"
-source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
-dependencies = [
- "bincode",
- "bls12_381",
- "bs58",
- "cfg-if",
- "digest 0.9.0",
- "ff",
- "group",
- "itertools 0.13.0",
- "nym-network-defaults 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "nym-pemstore 0.3.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "rand",
- "serde",
- "sha2 0.9.9",
- "subtle 2.5.0",
- "thiserror",
- "zeroize",
-]
-
 [[package]]
 name = "nym-config"
 version = "0.1.0"
@@ -5181,20 +5115,6 @@ dependencies = [
  "url",
 ]
 
-[[package]]
-name = "nym-config"
-version = "0.1.0"
-source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
-dependencies = [
- "dirs",
- "handlebars",
- "log",
- "nym-network-defaults 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "serde",
- "toml 0.8.14",
- "url",
-]
-
 [[package]]
 name = "nym-contracts-common"
 version = "0.5.0"
@@ -5225,21 +5145,6 @@ dependencies = [
  "vergen",
 ]
 
-[[package]]
-name = "nym-contracts-common"
-version = "0.5.0"
-source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
-dependencies = [
- "bs58",
- "cosmwasm-schema",
- "cosmwasm-std",
- "cw-storage-plus",
- "schemars",
- "serde",
- "thiserror",
- "vergen",
-]
-
 [[package]]
 name = "nym-country-group"
 version = "0.1.0"
@@ -5376,22 +5281,6 @@ dependencies = [
  "time",
 ]
 
-[[package]]
-name = "nym-credentials-interface"
-version = "0.1.0"
-source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
-dependencies = [
- "bls12_381",
- "nym-compact-ecash 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "nym-ecash-time 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "nym-network-defaults 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "rand",
- "serde",
- "strum 0.26.3",
- "thiserror",
- "time",
-]
-
 [[package]]
 name = "nym-crypto"
 version = "0.4.0"
@@ -5437,23 +5326,6 @@ dependencies = [
  "zeroize",
 ]
 
-[[package]]
-name = "nym-crypto"
-version = "0.4.0"
-source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
-dependencies = [
- "bs58",
- "ed25519-dalek",
- "nym-pemstore 0.3.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "nym-sphinx-types 0.2.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "serde",
- "serde_bytes",
- "subtle-encoding",
- "thiserror",
- "x25519-dalek",
- "zeroize",
-]
-
 [[package]]
 name = "nym-data-observatory"
 version = "0.1.0"
@@ -5555,14 +5427,6 @@ dependencies = [
  "time",
 ]
 
-[[package]]
-name = "nym-ecash-time"
-version = "0.1.0"
-source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
-dependencies = [
- "time",
-]
-
 [[package]]
 name = "nym-execute"
 version = "0.1.0"
@@ -5595,18 +5459,6 @@ dependencies = [
  "utoipa",
 ]
 
-[[package]]
-name = "nym-exit-policy"
-version = "0.1.0"
-source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
-dependencies = [
- "serde",
- "serde_json",
- "thiserror",
- "tracing",
- "utoipa",
-]
-
 [[package]]
 name = "nym-explorer-api-requests"
 version = "0.1.0"
@@ -5619,23 +5471,11 @@ dependencies = [
  "ts-rs",
 ]
 
-[[package]]
-name = "nym-explorer-api-requests"
-version = "0.1.0"
-source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
-dependencies = [
- "nym-api-requests 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "nym-contracts-common 0.5.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "nym-mixnet-contract-common 0.6.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "schemars",
- "serde",
-]
-
 [[package]]
 name = "nym-explorer-client"
 version = "0.1.0"
 dependencies = [
- "nym-explorer-api-requests 0.1.0",
+ "nym-explorer-api-requests",
  "reqwest 0.12.4",
  "serde",
  "thiserror",
@@ -5644,19 +5484,6 @@ dependencies = [
  "url",
 ]
 
-[[package]]
-name = "nym-explorer-client"
-version = "0.1.0"
-source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
-dependencies = [
- "log",
- "nym-explorer-api-requests 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "reqwest 0.12.4",
- "serde",
- "thiserror",
- "url",
-]
-
 [[package]]
 name = "nym-ffi-shared"
 version = "0.2.0"
@@ -6083,27 +5910,6 @@ dependencies = [
  "time",
 ]
 
-[[package]]
-name = "nym-mixnet-contract-common"
-version = "0.6.0"
-source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
-dependencies = [
- "bs58",
- "cosmwasm-schema",
- "cosmwasm-std",
- "cw-controllers",
- "cw-storage-plus",
- "humantime-serde",
- "log",
- "nym-contracts-common 0.5.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "schemars",
- "serde",
- "serde-json-wasm",
- "serde_repr",
- "thiserror",
- "time",
-]
-
 [[package]]
 name = "nym-mixnode"
 version = "1.1.37"
@@ -6235,19 +6041,6 @@ dependencies = [
  "utoipa",
 ]
 
-[[package]]
-name = "nym-network-defaults"
-version = "0.1.0"
-source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
-dependencies = [
- "dotenvy",
- "log",
- "schemars",
- "serde",
- "url",
- "utoipa",
-]
-
 [[package]]
 name = "nym-network-monitor"
 version = "0.1.0"
@@ -6453,27 +6246,6 @@ dependencies = [
  "utoipa",
 ]
 
-[[package]]
-name = "nym-node-requests"
-version = "0.1.0"
-source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
-dependencies = [
- "base64 0.22.1",
- "celes",
- "humantime 2.1.0",
- "humantime-serde",
- "nym-bin-common 0.6.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "nym-crypto 0.4.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "nym-exit-policy 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "nym-wireguard-types 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "schemars",
- "serde",
- "serde_json",
- "thiserror",
- "time",
- "utoipa",
-]
-
 [[package]]
 name = "nym-node-status-agent"
 version = "0.1.0"
@@ -6492,7 +6264,7 @@ dependencies = [
 
 [[package]]
 name = "nym-node-status-api"
-version = "0.1.3"
+version = "0.1.0"
 dependencies = [
  "anyhow",
  "axum 0.7.7",
@@ -6504,7 +6276,7 @@ dependencies = [
  "moka",
  "nym-bin-common 0.6.0",
  "nym-common-models",
- "nym-explorer-client 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
+ "nym-explorer-client",
  "nym-network-defaults 0.1.0 (git+https://github.com/nymtech/nym?branch=pre-dir-v2-fork)",
  "nym-node-requests 0.1.0 (git+https://github.com/nymtech/nym?branch=pre-dir-v2-fork)",
  "nym-task",
@@ -6637,14 +6409,6 @@ dependencies = [
  "pem",
 ]
 
-[[package]]
-name = "nym-pemstore"
-version = "0.3.0"
-source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
-dependencies = [
- "pem",
-]
-
 [[package]]
 name = "nym-sdk"
 version = "0.1.0"
@@ -6721,17 +6485,6 @@ dependencies = [
  "time",
 ]
 
-[[package]]
-name = "nym-serde-helpers"
-version = "0.1.0"
-source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
-dependencies = [
- "base64 0.22.1",
- "bs58",
- "serde",
- "time",
-]
-
 [[package]]
 name = "nym-service-provider-requests-common"
 version = "0.1.0"
@@ -7045,15 +6798,6 @@ dependencies = [
  "thiserror",
 ]
 
-[[package]]
-name = "nym-sphinx-types"
-version = "0.2.0"
-source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
-dependencies = [
- "sphinx-packet",
- "thiserror",
-]
-
 [[package]]
 name = "nym-statistics-common"
 version = "0.1.0"
@@ -7395,20 +7139,6 @@ dependencies = [
  "x25519-dalek",
 ]
 
-[[package]]
-name = "nym-wireguard-types"
-version = "0.1.0"
-source = "git+https://github.com/nymtech/nym?branch=release/2024.13-magura#317f7fffa986dff856e5b980b15872620bc052f0"
-dependencies = [
- "base64 0.22.1",
- "log",
- "nym-config 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "nym-network-defaults 0.1.0 (git+https://github.com/nymtech/nym?branch=release/2024.13-magura)",
- "serde",
- "thiserror",
- "x25519-dalek",
-]
-
 [[package]]
 name = "nymvisor"
 version = "0.1.8"

common/nyxd-scraper/src/block_processor/mod.rs

@@ -42,8 +42,32 @@ impl PendingSync {
     }
 }
 
+#[derive(Debug, Clone)]
+pub struct BlockProcessorConfig {
+    pub pruning_options: PruningOptions,
+    pub store_precommits: bool,
+}
+
+impl Default for BlockProcessorConfig {
+    fn default() -> Self {
+        Self {
+            pruning_options: PruningOptions::nothing(),
+            store_precommits: true,
+        }
+    }
+}
+
+impl BlockProcessorConfig {
+    pub fn new(pruning_options: PruningOptions, store_precommits: bool) -> Self {
+        Self {
+            pruning_options,
+            store_precommits,
+        }
+    }
+}
+
 pub struct BlockProcessor {
-    pruning_options: PruningOptions,
+    config: BlockProcessorConfig,
     cancel: CancellationToken,
     synced: Arc<Notify>,
     last_processed_height: u32,
@@ -65,9 +89,10 @@ pub struct BlockProcessor {
     msg_modules: Vec<Box<dyn MsgModule + Send>>,
 }
 
+#[allow(clippy::too_many_arguments)]
 impl BlockProcessor {
     pub async fn new(
-        pruning_options: PruningOptions,
+        config: BlockProcessorConfig,
         cancel: CancellationToken,
         synced: Arc<Notify>,
         incoming: UnboundedReceiver<BlockToProcess>,
@@ -82,7 +107,7 @@ impl BlockProcessor {
         let last_pruned_height = last_pruned.try_into().unwrap_or_default();
 
         Ok(BlockProcessor {
-            pruning_options,
+            config,
             cancel,
             synced,
             last_processed_height,
@@ -101,7 +126,7 @@ impl BlockProcessor {
     }
 
     pub fn with_pruning(mut self, pruning_options: PruningOptions) -> Self {
-        self.pruning_options = pruning_options;
+        self.config.pruning_options = pruning_options;
         self
     }
 
@@ -128,7 +153,7 @@ impl BlockProcessor {
         // we won't end up with a corrupted storage.
         let mut tx = self.storage.begin_processing_tx().await?;
 
-        persist_block(&full_info, &mut tx).await?;
+        persist_block(&full_info, &mut tx, self.config.store_precommits).await?;
 
         // let the modules do whatever they want
         // the ones wanting the full block:
@@ -241,7 +266,7 @@ impl BlockProcessor {
 
     #[instrument(skip(self))]
     async fn prune_storage(&mut self) -> Result<(), ScraperError> {
-        let keep_recent = self.pruning_options.strategy_keep_recent();
+        let keep_recent = self.config.pruning_options.strategy_keep_recent();
         let last_to_keep = self.last_processed_height - keep_recent;
 
         info!(
@@ -282,12 +307,12 @@ impl BlockProcessor {
     async fn maybe_prune_storage(&mut self) -> Result<(), ScraperError> {
         debug!("checking for storage pruning");
 
-        if self.pruning_options.strategy.is_nothing() {
+        if self.config.pruning_options.strategy.is_nothing() {
             trace!("the current pruning strategy is 'nothing'");
             return Ok(());
         }
 
-        let interval = self.pruning_options.strategy_interval();
+        let interval = self.config.pruning_options.strategy_interval();
         if self.last_pruned_height + interval <= self.last_processed_height {
             self.prune_storage().await?;
         }
@@ -371,7 +396,7 @@ impl BlockProcessor {
         if latest_block > self.last_processed_height && self.last_processed_height != 0 {
             // in case we were offline for a while,
             // make sure we don't request blocks we'd have to prune anyway
-            let keep_recent = self.pruning_options.strategy_keep_recent();
+            let keep_recent = self.config.pruning_options.strategy_keep_recent();
             let last_to_keep = latest_block - keep_recent;
             self.last_processed_height = max(self.last_processed_height, last_to_keep);
 

common/nyxd-scraper/src/scraper/mod.rs

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use crate::block_processor::types::BlockToProcess;
-use crate::block_processor::BlockProcessor;
+use crate::block_processor::{BlockProcessor, BlockProcessorConfig};
 use crate::block_requester::{BlockRequest, BlockRequester};
 use crate::error::ScraperError;
 use crate::modules::{BlockModule, MsgModule, TxModule};
@@ -34,6 +34,8 @@ pub struct Config {
     pub database_path: PathBuf,
 
     pub pruning_options: PruningOptions,
+
+    pub store_precommits: bool,
 }
 
 pub struct NyxdScraperBuilder {
@@ -60,8 +62,14 @@ impl NyxdScraperBuilder {
             req_rx,
             processing_tx.clone(),
         );
-        let mut block_processor = BlockProcessor::new(
+
+        let block_processor_config = BlockProcessorConfig::new(
             scraper.config.pruning_options,
+            scraper.config.store_precommits,
+        );
+
+        let mut block_processor = BlockProcessor::new(
+            block_processor_config,
             scraper.cancel_token.clone(),
             scraper.startup_sync.clone(),
             processing_rx,
@@ -275,8 +283,11 @@ impl NyxdScraper {
         req_tx: Sender<BlockRequest>,
         processing_rx: UnboundedReceiver<BlockToProcess>,
     ) -> Result<BlockProcessor, ScraperError> {
+        let block_processor_config =
+            BlockProcessorConfig::new(self.config.pruning_options, self.config.store_precommits);
+
         BlockProcessor::new(
-            self.config.pruning_options,
+            block_processor_config,
             self.cancel_token.clone(),
             self.startup_sync.clone(),
             processing_rx,

common/nyxd-scraper/src/storage/mod.rs

@@ -212,6 +212,7 @@ impl ScraperStorage {
 pub async fn persist_block(
     block: &FullBlockInformation,
     tx: &mut StorageTransaction,
+    store_precommits: bool,
 ) -> Result<(), ScraperError> {
     let total_gas = crate::helpers::tx_gas_sum(&block.transactions);
 
@@ -224,11 +225,12 @@ pub async fn persist_block(
     // persist block data
     persist_block_data(&block.block, total_gas, tx).await?;
 
-    // persist commits
-    if let Some(commit) = &block.block.last_commit {
-        persist_commits(commit, &block.validators, tx).await?;
-    } else {
-        warn!("no commits for block {}", block.block.header.height)
+    if store_precommits {
+        if let Some(commit) = &block.block.last_commit {
+            persist_commits(commit, &block.validators, tx).await?;
+        } else {
+            warn!("no commits for block {}", block.block.header.height)
+        }
     }
 
     // persist txs

nym-node-status-agent/Dockerfile

@@ -0,0 +1,28 @@
+FROM rust:latest AS builder
+
+RUN apt update && apt install -yy libdbus-1-dev pkg-config libclang-dev
+
+# Install go
+RUN wget https://go.dev/dl/go1.22.5.linux-amd64.tar.gz -O go.tar.gz
+RUN tar -xzvf go.tar.gz -C /usr/local
+
+RUN git clone https://github.com/nymtech/nym-vpn-client /usr/src/nym-vpn-client
+ENV PATH=/go/bin:/usr/local/go/bin:$PATH
+WORKDIR /usr/src/nym-vpn-client/nym-vpn-core
+RUN cargo build --release --package nym-gateway-probe
+
+COPY ./ /usr/src/nym
+WORKDIR /usr/src/nym/nym-node-status-agent
+RUN cargo build --release
+
+FROM ubuntu:24.04
+
+RUN apt-get update && apt-get install -y ca-certificates
+
+WORKDIR /nym
+
+COPY --from=builder /usr/src/nym/target/release/nym-node-status-agent ./
+COPY --from=builder /usr/src/nym-vpn-client/nym-vpn-core/target/release/nym-gateway-probe ./
+
+ENV NODE_STATUS_AGENT_PROBE_PATH=/nym/nym-gateway-probe
+ENTRYPOINT [ "/nym/nym-node-status-agent", "run-probe" ]

nym-node-status-agent/src/cli.rs

@@ -97,7 +97,6 @@ async fn submit_results(
 ) -> anyhow::Result<()> {
     let target_url = format!("{}/{}/{}", server_addr, URL_BASE, testrun_id);
     let client = reqwest::Client::new();
-
     let res = client
         .post(target_url)
         .body(probe_outcome)

nym-node-status-agent/src/probe.rs

@@ -40,12 +40,6 @@ impl GwProbe {
         match command.spawn() {
             Ok(child) => {
                 if let Ok(output) = child.wait_with_output() {
-                    if !output.status.success() {
-                        let out = String::from_utf8_lossy(&output.stdout);
-                        let err = String::from_utf8_lossy(&output.stderr);
-                        tracing::error!("Probe exited with {:?}:\n{}\n{}", output.status, out, err);
-                    }
-
                     return String::from_utf8(output.stdout)
                         .unwrap_or("Unable to get log from test run".to_string());
                 }

nym-node-status-api/Cargo.toml

@@ -3,7 +3,7 @@
 
 [package]
 name = "nym-node-status-api"
-version = "0.1.3"
+version = "0.1.0"
 authors.workspace = true
 repository.workspace = true
 homepage.workspace = true
@@ -23,8 +23,7 @@ futures-util = { workspace = true }
 moka = { workspace = true, features = ["future"] }
 nym-bin-common = { path = "../common/bin-common", features = ["models"]}
 nym-common-models = { path = "../common/models" }
-# nym-explorer-client = { path = "../explorer-api/explorer-client" }
-nym-explorer-client = { git = "https://github.com/nymtech/nym", branch = "release/2024.13-magura" }
+nym-explorer-client = { path = "../explorer-api/explorer-client" }
 # TODO dz: before Nym API client breaking changes. Update to latest develop once new Nym API is live
 nym-network-defaults = { git = "https://github.com/nymtech/nym", branch = "pre-dir-v2-fork" }
 nym-validator-client = { git = "https://github.com/nymtech/nym", branch = "pre-dir-v2-fork" }

nym-node-status-api/build.rs

@@ -25,6 +25,7 @@ async fn main() -> Result<()> {
     // not a valid windows path... but hey, it works...
     println!("cargo::rustc-env=DATABASE_URL=sqlite:///{}", &database_path);
 
+    rerun_if_changed();
     Ok(())
 }
 
@@ -32,6 +33,11 @@ fn read_env_var(var: &str) -> Result<String> {
     std::env::var(var).map_err(|_| anyhow!("You need to set {} env var", var))
 }
 
+fn rerun_if_changed() {
+    println!("cargo::rerun-if-changed=migrations");
+    println!("cargo::rerun-if-changed=src/db/queries");
+}
+
 /// use `./enter_db.sh` to inspect DB
 async fn write_db_path_to_file(out_dir: &str, db_filename: &str) -> anyhow::Result<()> {
     let mut file = File::create("enter_db.sh").await?;

nym-node-status-api/launch_node_status_api.sh

@@ -7,7 +7,7 @@ export RUST_LOG=${RUST_LOG:-debug}
 export NYM_API_CLIENT_TIMEOUT=60
 export EXPLORER_CLIENT_TIMEOUT=60
 
-export ENVIRONMENT="qa.env"
+export ENVIRONMENT="mainnet.env"
 
 function run_bare() {
     # export necessary env vars

nym-node-status-api/src/db/models.rs

@@ -315,7 +315,7 @@ pub struct TestRunDto {
 pub(crate) enum TestRunStatus {
     Complete = 2,
     InProgress = 1,
-    Queued = 0,
+    Pending = 0,
 }
 
 #[derive(Debug, Clone)]

nym-node-status-api/src/db/queries/gateways.rs

@@ -7,28 +7,8 @@ use crate::{
 };
 use futures_util::TryStreamExt;
 use nym_validator_client::models::DescribedGateway;
-use sqlx::{pool::PoolConnection, Sqlite};
 use tracing::error;
 
-pub(crate) async fn select_gateway_identity(
-    conn: &mut PoolConnection<Sqlite>,
-    gateway_pk: i64,
-) -> anyhow::Result<String> {
-    let record = sqlx::query!(
-        r#"SELECT
-            gateway_identity_key
-        FROM
-            gateways
-        WHERE
-            id = ?"#,
-        gateway_pk
-    )
-    .fetch_one(conn.as_mut())
-    .await?;
-
-    Ok(record.gateway_identity_key)
-}
-
 pub(crate) async fn insert_gateways(
     pool: &DbPool,
     gateways: Vec<GatewayRecord>,

nym-node-status-api/src/db/queries/mod.rs

@@ -5,7 +5,7 @@ mod summary;
 pub(crate) mod testruns;
 
 pub(crate) use gateways::{
-    ensure_gateways_still_bonded, get_all_gateways, insert_gateways, select_gateway_identity,
+    ensure_gateways_still_bonded, get_all_gateways, insert_gateways,
     write_blacklisted_gateways_to_db,
 };
 pub(crate) use misc::insert_summaries;

nym-node-status-api/src/db/queries/testruns.rs

@@ -1,14 +1,12 @@
-use crate::db::DbPool;
 use crate::http::models::TestrunAssignment;
 use crate::{
     db::models::{TestRunDto, TestRunStatus},
     testruns::now_utc,
 };
 use anyhow::Context;
-use chrono::Duration;
 use sqlx::{pool::PoolConnection, Sqlite};
 
-pub(crate) async fn get_in_progress_testrun_by_id(
+pub(crate) async fn get_testrun_by_id(
     conn: &mut PoolConnection<Sqlite>,
     testrun_id: i64,
 ) -> anyhow::Result<TestRunDto> {
@@ -22,56 +20,20 @@ pub(crate) async fn get_in_progress_testrun_by_id(
             ip_address as "ip_address!",
             log as "log!"
          FROM testruns
-         WHERE
-            id = ?
-         AND
-            status = ?
+         WHERE id = ?
          ORDER BY timestamp_utc"#,
-        testrun_id,
-        TestRunStatus::InProgress as i64,
+        testrun_id
     )
     .fetch_one(conn.as_mut())
     .await
     .context(format!("Couldn't retrieve testrun {testrun_id}"))
 }
 
-pub(crate) async fn update_testruns_older_than(db: &DbPool, age: Duration) -> anyhow::Result<u64> {
-    let mut conn = db.acquire().await?;
-    let previous_run = now_utc() - age;
-    let cutoff_timestamp = previous_run.timestamp();
-
-    let res = sqlx::query!(
-        r#"UPDATE
-            testruns
-        SET
-            status = ?
-        WHERE
-            status = ?
-        AND
-            timestamp_utc < ?
-            "#,
-        TestRunStatus::Queued as i64,
-        TestRunStatus::InProgress as i64,
-        cutoff_timestamp
-    )
-    .execute(conn.as_mut())
-    .await?;
-
-    let stale_testruns = res.rows_affected();
-    if stale_testruns > 0 {
-        tracing::debug!(
-            "Refreshed {} stale testruns, scheduled before {} but not yet finished",
-            stale_testruns,
-            previous_run
-        );
-    }
-
-    Ok(stale_testruns)
-}
-
 pub(crate) async fn get_oldest_testrun_and_make_it_pending(
-    conn: &mut PoolConnection<Sqlite>,
+    // TODO dz accept mut reference, repeat in all similar functions
+    conn: PoolConnection<Sqlite>,
 ) -> anyhow::Result<Option<TestrunAssignment>> {
+    let mut conn = conn;
     let assignment = sqlx::query_as!(
         TestrunAssignment,
         r#"UPDATE testruns
@@ -89,9 +51,9 @@ pub(crate) async fn get_oldest_testrun_and_make_it_pending(
             gateway_id as "gateway_pk_id!"
             "#,
         TestRunStatus::InProgress as i64,
-        TestRunStatus::Queued as i64,
+        TestRunStatus::Pending as i64,
     )
-    .fetch_optional(conn.as_mut())
+    .fetch_optional(&mut *conn)
     .await?;
 
     Ok(assignment)

nym-node-status-api/src/http/api/mod.rs

@@ -1,10 +1,7 @@
 use anyhow::anyhow;
 use axum::{response::Redirect, Router};
 use tokio::net::ToSocketAddrs;
-use tower_http::{
-    cors::CorsLayer,
-    trace::{DefaultOnResponse, TraceLayer},
-};
+use tower_http::{cors::CorsLayer, trace::TraceLayer};
 use utoipa::OpenApi;
 use utoipa_swagger_ui::SwaggerUi;
 
@@ -61,10 +58,7 @@ impl RouterBuilder {
             // CORS layer needs to wrap other API layers
             .layer(setup_cors())
             // logger should be outermost layer
-            .layer(
-                TraceLayer::new_for_http()
-                    .on_response(DefaultOnResponse::new().level(tracing::Level::DEBUG)),
-            )
+            .layer(TraceLayer::new_for_http())
     }
 }
 

nym-node-status-api/src/http/api/testruns.rs

@@ -1,4 +1,3 @@
-use axum::extract::DefaultBodyLimit;
 use axum::Json;
 use axum::{
     extract::{Path, State},
@@ -24,40 +23,31 @@ pub(crate) fn routes() -> Router<AppState> {
     Router::new()
         .route("/", axum::routing::get(request_testrun))
         .route("/:testrun_id", axum::routing::post(submit_testrun))
-        .layer(DefaultBodyLimit::max(1024 * 1024 * 5))
 }
 
 #[tracing::instrument(level = "debug", skip_all)]
 async fn request_testrun(State(state): State<AppState>) -> HttpResult<Json<TestrunAssignment>> {
     // TODO dz log agent's key
     // TODO dz log agent's network probe version
-    tracing::debug!("Agent requested testrun");
+    tracing::debug!("Agent X requested testrun");
 
     let db = state.db_pool();
-    let mut conn = db
+    let conn = db
         .acquire()
         .await
         .map_err(HttpError::internal_with_logging)?;
 
-    return match db::queries::testruns::get_oldest_testrun_and_make_it_pending(&mut conn).await {
+    return match db::queries::testruns::get_oldest_testrun_and_make_it_pending(conn).await {
         Ok(res) => {
             if let Some(testrun) = res {
-                let gw_identity =
-                    db::queries::select_gateway_identity(&mut conn, testrun.gateway_pk_id)
-                        .await
-                        .map_err(|_| {
-                            // should never happen:
-                            HttpError::internal_with_logging("No gateway found for testrun")
-                        })?;
                 // TODO dz consider adding a column to testruns table with agent's public key
                 tracing::debug!(
-                    "🏃‍ Assigned testrun row_id {} gateway {} to agent",
-                    &testrun.testrun_id,
-                    gw_identity
+                    "🏃‍ Assigned testrun row_id {} to agent X",
+                    &testrun.testrun_id
                 );
                 Ok(Json(testrun))
             } else {
-                Err(HttpError::no_available_testruns())
+                Err(HttpError::not_found("No testruns available"))
             }
         }
         Err(err) => Err(HttpError::internal_with_logging(err)),
@@ -71,32 +61,25 @@ async fn submit_testrun(
     State(state): State<AppState>,
     body: String,
 ) -> HttpResult<StatusCode> {
+    tracing::debug!(
+        "Agent submitted testrun {}. Total length: {}",
+        testrun_id,
+        body.len(),
+    );
+    // TODO dz store testrun results
+
     let db = state.db_pool();
     let mut conn = db
         .acquire()
         .await
         .map_err(HttpError::internal_with_logging)?;
 
-    let testrun = queries::testruns::get_in_progress_testrun_by_id(&mut conn, testrun_id)
+    let testrun = queries::testruns::get_testrun_by_id(&mut conn, testrun_id)
         .await
         .map_err(|e| {
             tracing::error!("{e}");
             HttpError::not_found(testrun_id)
         })?;
-
-    let gw_identity = db::queries::select_gateway_identity(&mut conn, testrun.gateway_id)
-        .await
-        .map_err(|_| {
-            // should never happen:
-            HttpError::internal_with_logging("No gateway found for testrun")
-        })?;
-    tracing::debug!(
-        "Agent submitted testrun {} for gateway {} ({} bytes)",
-        testrun_id,
-        gw_identity,
-        body.len(),
-    );
-
     // TODO dz this should be part of a single transaction: commit after everything is done
     queries::testruns::update_testrun_status(&mut conn, testrun_id, TestRunStatus::Complete)
         .await
@@ -116,7 +99,7 @@ async fn submit_testrun(
     tracing::info!(
         "✅ Testrun row_id {} for gateway {} complete",
         testrun.id,
-        gw_identity
+        testrun.gateway_id
     );
 
     Ok(StatusCode::CREATED)

nym-node-status-api/src/http/error.rs

@@ -8,9 +8,9 @@ pub(crate) struct HttpError {
 }
 
 impl HttpError {
-    pub(crate) fn invalid_input(msg: impl Display) -> Self {
+    pub(crate) fn invalid_input(message: String) -> Self {
         Self {
-            message: serde_json::json!({"message": msg.to_string()}).to_string(),
+            message,
             status: axum::http::StatusCode::BAD_REQUEST,
         }
     }
@@ -27,12 +27,6 @@ impl HttpError {
         }
     }
 
-    pub(crate) fn no_available_testruns() -> Self {
-        Self {
-            message: serde_json::json!({"message": "No available testruns"}).to_string(),
-            status: axum::http::StatusCode::SERVICE_UNAVAILABLE,
-        }
-    }
     pub(crate) fn not_found(msg: impl Display) -> Self {
         Self {
             message: serde_json::json!({"message": msg.to_string()}).to_string(),

nym-node-status-api/src/monitor/mod.rs

@@ -91,7 +91,7 @@ async fn run(
     let explorer_client =
         ExplorerClient::new_with_timeout(default_explorer_url, explorer_client_timeout)?;
     let explorer_gateways = explorer_client
-        .unstable_get_gateways()
+        .get_gateways()
         .await
         .log_error("get_gateways")?;
 

nym-node-status-api/src/testruns/mod.rs

@@ -11,12 +11,10 @@ pub(crate) use queue::now_utc;
 pub(crate) async fn spawn(pool: DbPool, refresh_interval: Duration) {
     tokio::spawn(async move {
         loop {
-            if let Err(e) = refresh_stale_testruns(&pool, refresh_interval).await {
-                tracing::error!("{e}");
-            }
+            tracing::info!("Spawning testruns...");
 
             if let Err(e) = run(&pool).await {
-                tracing::error!("Assigning testruns failed: {}", e);
+                tracing::error!("Cron job failed: {}", e);
             }
             tracing::debug!("Sleeping for {}s...", refresh_interval.as_secs());
             tokio::time::sleep(refresh_interval).await;
@@ -26,9 +24,9 @@ pub(crate) async fn spawn(pool: DbPool, refresh_interval: Duration) {
 
 // TODO dz make number of max agents configurable
 
+// TODO dz periodically clean up stale pending testruns
 #[instrument(level = "debug", name = "testrun_queue", skip_all)]
 async fn run(pool: &DbPool) -> anyhow::Result<()> {
-    tracing::info!("Spawning testruns...");
     if pool.is_closed() {
         tracing::debug!("DB pool closed, returning early");
         return Ok(());
@@ -76,11 +74,3 @@ async fn run(pool: &DbPool) -> anyhow::Result<()> {
 
     Ok(())
 }
-
-#[instrument(level = "debug", skip_all)]
-async fn refresh_stale_testruns(pool: &DbPool, refresh_interval: Duration) -> anyhow::Result<()> {
-    let chrono_duration = chrono::Duration::from_std(refresh_interval)?;
-    crate::db::queries::testruns::update_testruns_older_than(pool, chrono_duration).await?;
-
-    Ok(())
-}

nym-node-status-api/src/testruns/queue.rs

@@ -82,7 +82,7 @@ pub(crate) async fn try_queue_testrun(
     //
     // save test run
     //
-    let status = TestRunStatus::Queued as u32;
+    let status = TestRunStatus::Pending as u32;
     let log = format!(
         "Test for {identity_key} requested at {} UTC\n\n",
         timestamp_pretty
@@ -103,7 +103,7 @@ pub(crate) async fn try_queue_testrun(
     Ok(TestRun {
         id: id as u32,
         identity_key,
-        status: format!("{}", TestRunStatus::Queued),
+        status: format!("{}", TestRunStatus::Pending),
         log,
     })
 }

nym-validator-rewarder/src/config/mod.rs

@@ -107,6 +107,7 @@ impl Config {
             nyxd_scraper: NyxdScraper {
                 websocket_url,
                 pruning: Default::default(),
+                store_precommits: true,
             },
             base: Base {
                 upstream_nyxd: nyxd_url,
@@ -122,6 +123,7 @@ impl Config {
             rpc_url: self.base.upstream_nyxd.clone(),
             database_path: self.storage_paths.nyxd_scraper.clone(),
             pruning_options: self.nyxd_scraper.pruning,
+            store_precommits: self.nyxd_scraper.store_precommits,
         }
     }
 
@@ -249,7 +251,14 @@ pub struct NyxdScraper {
     // if the value is missing, use `nothing` pruning as this was the past behaviour
     #[serde(default = "PruningOptions::nothing")]
     pub pruning: PruningOptions,
-    // TODO: debug with everything that's currently hardcoded in the scraper
+
+    /// Specifies whether to store pre-commits within the database.
+    #[serde(default = "default_store_precommits")]
+    pub store_precommits: bool,
+}
+
+fn default_store_precommits() -> bool {
+    true
 }
 
 impl NyxdScraper {